d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

scripts/nvmf_perf: use common method name for calling subprocesses

Browse Source 
Use common method name and parameters for calling
subprocesses for local (Target) and remote systems
(Initiators) instead of using "check_output" (from
subprocess module) and "exec_command" (from paramiko)
separately.
Having these functions wrapped by a single common
method will allow to create common methods in Server
class more eaisly, instead of creating two copies in
Target and Initiator classes.

Signed-off-by: Karol Latecki <karol.latecki@intel.com>
Change-Id: I1c10f6a88f3d7300c227e969ad6fd901763ac52c
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/6261
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Michal Berger <michalx.berger@intel.com>
Reviewed-by: Maciej Wawryk <maciejx.wawryk@intel.com>
This commit is contained in:
Karol Latecki 2021-02-03 13:08:15 +01:00 committed by Tomasz Zawadzki
parent f9dd94b515
commit 26d7e3d4bb
2 changed files with 79 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scripts/perf/nvmf/common.py
View File

@ -2,7 +2,7 @@ import os
import re import re
import json import json
from itertools import product, chain from itertools import product, chain
from subprocess import check_output, Popen from subprocess import check_output, CalledProcessError, Popen
def get_used_numa_nodes(): def get_used_numa_nodes():

137
scripts/perf/nvmf/run_nvmf.py
View File

@ -74,6 +74,9 @@ class Server:
self.local_nic_info = extract_network_elements(pci_info) self.local_nic_info = extract_network_elements(pci_info)
def exec_cmd(self, cmd, stderr_redirect=False):
return ""
class Target(Server): class Target(Server):
def __init__(self, name, general_config, target_config): def __init__(self, name, general_config, target_config):
@ -96,7 +99,7 @@ class Target(Server):
self.enable_zcopy = False self.enable_zcopy = False
self.scheduler_name = "static" self.scheduler_name = "static"
self.null_block = 0 self.null_block = 0
self._nics_json_obj = json.loads(check_output(["ip", "-j", "address", "show"])) self._nics_json_obj = json.loads(self.exec_cmd(["ip", "-j", "address", "show"]))
if "null_block_devices" in target_config: if "null_block_devices" in target_config:
self.null_block = target_config["null_block_devices"] self.null_block = target_config["null_block_devices"]
@ -120,7 +123,14 @@ class Target(Server):
self.sys_config() self.sys_config()
def set_local_nic_info_helper(self): def set_local_nic_info_helper(self):
return json.loads(check_output(["lshw", "-json"])) return json.loads(self.exec_cmd(["lshw", "-json"]))
def exec_cmd(self, cmd, stderr_redirect=False):
stderr_opt = None
if stderr_redirect:
stderr_opt = subprocess.STDOUT
out = check_output(cmd, stderr=stderr_opt).decode(encoding="utf-8")
return out
def zip_spdk_sources(self, spdk_dir, dest_file): def zip_spdk_sources(self, spdk_dir, dest_file):
self.log_print("Zipping SPDK source directory") self.log_print("Zipping SPDK source directory")
@ -300,8 +310,7 @@ class Target(Server):
def measure_sar(self, results_dir, sar_file_name): def measure_sar(self, results_dir, sar_file_name):
self.log_print("Waiting %d delay before measuring SAR stats" % self.sar_delay) self.log_print("Waiting %d delay before measuring SAR stats" % self.sar_delay)
time.sleep(self.sar_delay) time.sleep(self.sar_delay)
cmd = ["sar", "-P", "ALL", "%s" % self.sar_interval, "%s" % self.sar_count] out = self.exec_cmd(["sar", "-P", "ALL", "%s" % self.sar_interval, "%s" % self.sar_count])
out = subprocess.check_output(cmd).decode(encoding="utf-8")
with open(os.path.join(results_dir, sar_file_name), "w") as fh: with open(os.path.join(results_dir, sar_file_name), "w") as fh:
for line in out.split("\n"): for line in out.split("\n"):
if "Average" in line and "CPU" in line: if "Average" in line and "CPU" in line:
@ -330,14 +339,13 @@ class Target(Server):
def measure_pcm_power(self, results_dir, pcm_power_file_name): def measure_pcm_power(self, results_dir, pcm_power_file_name):
time.sleep(self.pcm_delay) time.sleep(self.pcm_delay)
cmd = ["%s/pcm-power.x" % self.pcm_dir, "%s" % self.pcm_interval, "-i=%s" % self.pcm_count] out = self.exec_cmd(["%s/pcm-power.x" % self.pcm_dir, "%s" % self.pcm_interval, "-i=%s" % self.pcm_count])
out = subprocess.check_output(cmd).decode(encoding="utf-8")
with open(os.path.join(results_dir, pcm_power_file_name), "w") as fh: with open(os.path.join(results_dir, pcm_power_file_name), "w") as fh:
fh.write(out) fh.write(out)
def measure_bandwidth(self, results_dir, bandwidth_file_name): def measure_bandwidth(self, results_dir, bandwidth_file_name):
cmd = ["bwm-ng", "-o csv", "-F %s/%s" % (results_dir, bandwidth_file_name), "-a 1", "-t 1000", "-c %s" % self.bandwidth_count] self.exec_cmd(["bwm-ng", "-o csv", "-F %s/%s" % (results_dir, bandwidth_file_name),
bwm = subprocess.run(cmd) "-a 1", "-t 1000", "-c %s" % self.bandwidth_count])
def measure_dpdk_memory(self, results_dir): def measure_dpdk_memory(self, results_dir):
self.log_print("INFO: waiting to generate DPDK memory usage") self.log_print("INFO: waiting to generate DPDK memory usage")
@ -358,7 +366,7 @@ class Target(Server):
sysctl = f.readlines() sysctl = f.readlines()
self.log_print('\n'.join(self.get_uncommented_lines(sysctl))) self.log_print('\n'.join(self.get_uncommented_lines(sysctl)))
self.log_print("====Cpu power info:====") self.log_print("====Cpu power info:====")
subprocess.run(["cpupower", "frequency-info"]) self.log_print(self.exec_cmd(["cpupower", "frequency-info"]))
self.log_print("====zcopy settings:====") self.log_print("====zcopy settings:====")
self.log_print("zcopy enabled: %s" % (self.enable_zcopy)) self.log_print("zcopy enabled: %s" % (self.enable_zcopy))
self.log_print("====Scheduler settings:====") self.log_print("====Scheduler settings:====")
@ -399,19 +407,32 @@ class Initiator(Server):
self.ssh_connection = paramiko.SSHClient() self.ssh_connection = paramiko.SSHClient()
self.ssh_connection.set_missing_host_key_policy(paramiko.AutoAddPolicy()) self.ssh_connection.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh_connection.connect(self.ip, username=self.username, password=self.password) self.ssh_connection.connect(self.ip, username=self.username, password=self.password)
self.remote_call("sudo rm -rf %s/nvmf_perf" % self.spdk_dir) self.exec_cmd(["sudo", "rm", "-rf", "%s/nvmf_perf" % self.spdk_dir])
self.remote_call("mkdir -p %s" % self.spdk_dir) self.exec_cmd(["mkdir", "-p", "%s" % self.spdk_dir])
self._nics_json_obj = json.loads(self.remote_call("ip -j address show")[0]) self._nics_json_obj = json.loads(self.exec_cmd(["ip", "-j", "address", "show"]))
self.set_local_nic_info(self.set_local_nic_info_helper()) self.set_local_nic_info(self.set_local_nic_info_helper())
self.set_cpu_frequency() self.set_cpu_frequency()
self.sys_config() self.sys_config()
def set_local_nic_info_helper(self): def set_local_nic_info_helper(self):
return json.loads(self.remote_call("lshw -json")[0]) return json.loads(self.exec_cmd(["lshw", "-json"]))
def __del__(self): def __del__(self):
self.ssh_connection.close() self.ssh_connection.close()
def exec_cmd(self, cmd, stderr_redirect=False):
# Redirect stderr to stdout thanks using get_pty option if needed
cmd = " ".join(cmd)
_, stdout, _ = self.ssh_connection.exec_command(cmd, get_pty=stderr_redirect)
out = stdout.read().decode(encoding="utf-8")
# Check the return code
rc = stdout.channel.recv_exit_status()
if rc:
raise CalledProcessError(int(rc), cmd, out)
return out
def put_file(self, local, remote_dest): def put_file(self, local, remote_dest):
ftp = self.ssh_connection.open_sftp() ftp = self.ssh_connection.open_sftp()
ftp.put(local, remote_dest) ftp.put(local, remote_dest)
@ -422,12 +443,6 @@ class Initiator(Server):
ftp.get(remote, local_dest) ftp.get(remote, local_dest)
ftp.close() ftp.close()
def remote_call(self, cmd):
stdin, stdout, stderr = self.ssh_connection.exec_command(cmd)
out = stdout.read().decode(encoding="utf-8")
err = stderr.read().decode(encoding="utf-8")
return out, err
def copy_result_files(self, dest_dir): def copy_result_files(self, dest_dir):
self.log_print("Copying results") self.log_print("Copying results")
@ -435,8 +450,7 @@ class Initiator(Server):
os.mkdir(dest_dir) os.mkdir(dest_dir)
# Get list of result files from initiator and copy them back to target # Get list of result files from initiator and copy them back to target
stdout, stderr = self.remote_call("ls %s/nvmf_perf" % self.spdk_dir) file_list = self.exec_cmd(["ls", "%s/nvmf_perf" % self.spdk_dir]).strip().split("\n")
file_list = stdout.strip().split("\n")
for file in file_list: for file in file_list:
self.get_file(os.path.join(self.spdk_dir, "nvmf_perf", file), self.get_file(os.path.join(self.spdk_dir, "nvmf_perf", file),
@ -450,14 +464,19 @@ class Initiator(Server):
self.log_print("Trying to discover: %s:%s" % (ip, 4420 + subsys_no)) self.log_print("Trying to discover: %s:%s" % (ip, 4420 + subsys_no))
nvme_discover_cmd = ["sudo", nvme_discover_cmd = ["sudo",
"%s" % self.nvmecli_bin, "%s" % self.nvmecli_bin,
"discover", "-t %s" % self.transport, "discover", "-t", "%s" % self.transport,
"-s %s" % (4420 + subsys_no), "-s", "%s" % (4420 + subsys_no),
"-a %s" % ip] "-a", "%s" % ip]
nvme_discover_cmd = " ".join(nvme_discover_cmd)
stdout, stderr = self.remote_call(nvme_discover_cmd) try:
if stdout: stdout = self.exec_cmd(nvme_discover_cmd)
nvme_discover_output = nvme_discover_output + stdout if stdout:
nvme_discover_output = nvme_discover_output + stdout
except CalledProcessError:
# Do nothing. In case of discovering remote subsystems of kernel target
# we expect "nvme discover" to fail a bunch of times because we basically
# scan ports.
pass
subsystems = re.findall(r'trsvcid:\s(\d+)\s+' # get svcid number subsystems = re.findall(r'trsvcid:\s(\d+)\s+' # get svcid number
r'subnqn:\s+([a-zA-Z0-9\.\-\:]+)\s+' # get NQN id r'subnqn:\s+([a-zA-Z0-9\.\-\:]+)\s+' # get NQN id
@ -497,13 +516,14 @@ runtime={run_time}
if "spdk" in self.mode: if "spdk" in self.mode:
subsystems = self.discover_subsystems(self.remote_nic_ips, subsys_no) subsystems = self.discover_subsystems(self.remote_nic_ips, subsys_no)
bdev_conf = self.gen_spdk_bdev_conf(subsystems) bdev_conf = self.gen_spdk_bdev_conf(subsystems)
self.remote_call("echo '%s' > %s/bdev.conf" % (bdev_conf, self.spdk_dir)) self.exec_cmd(["echo", "'%s'" % bdev_conf, ">", "%s/bdev.conf" % self.spdk_dir])
ioengine = "%s/build/fio/spdk_bdev" % self.spdk_dir ioengine = "%s/build/fio/spdk_bdev" % self.spdk_dir
spdk_conf = "spdk_json_conf=%s/bdev.conf" % self.spdk_dir spdk_conf = "spdk_json_conf=%s/bdev.conf" % self.spdk_dir
else: else:
ioengine = "libaio" ioengine = "libaio"
spdk_conf = "" spdk_conf = ""
out, err = self.remote_call("sudo nvme list | grep -E 'SPDK|Linux' | awk '{print $1}'") out = self.exec_cmd(["sudo", "nvme", "list", "|", "grep", "-E", "'SPDK|Linux'",
"|", "awk", "'{print $1}'"])
subsystems = [x for x in out.split("\n") if "nvme" in x] subsystems = [x for x in out.split("\n") if "nvme" in x]
if self.cpus_allowed is not None: if self.cpus_allowed is not None:
@ -545,8 +565,8 @@ runtime={run_time}
fio_config_filename += "_%sCPU" % self.num_cores fio_config_filename += "_%sCPU" % self.num_cores
fio_config_filename += ".fio" fio_config_filename += ".fio"
self.remote_call("mkdir -p %s/nvmf_perf" % self.spdk_dir) self.exec_cmd(["mkdir", "-p", "%s/nvmf_perf" % self.spdk_dir])
self.remote_call("echo '%s' > %s/nvmf_perf/%s" % (fio_config, self.spdk_dir, fio_config_filename)) self.exec_cmd(["echo", "'%s'" % fio_config, ">", "%s/nvmf_perf/%s" % (self.spdk_dir, fio_config_filename)])
self.log_print("Created FIO Config:") self.log_print("Created FIO Config:")
self.log_print(fio_config) self.log_print(fio_config)
@ -555,12 +575,9 @@ runtime={run_time}
def set_cpu_frequency(self): def set_cpu_frequency(self):
if self.cpu_frequency is not None: if self.cpu_frequency is not None:
try: try:
self.remote_call('sudo cpupower frequency-set -g userspace') self.exec_cmd(["sudo", "cpupower", "frequency-set", "-g", "userspace"], True)
self.remote_call('sudo cpupower frequency-set -f %s' % self.cpu_frequency) self.exec_cmd(["sudo", "cpupower", "frequency-set", "-f", "%s" % self.cpu_frequency], True)
cmd = "sudo cpupower frequency-info" self.log_print(self.exec_cmd(["sudo", "cpupower", "frequency-info"]))
output, error = self.remote_call(cmd)
self.log_print(output)
self.log_print(error)
except Exception: except Exception:
self.log_print("ERROR: cpu_frequency will not work when intel_pstate is enabled!") self.log_print("ERROR: cpu_frequency will not work when intel_pstate is enabled!")
sys.exit() sys.exit()
@ -575,29 +592,29 @@ runtime={run_time}
if run_num: if run_num:
for i in range(1, run_num + 1): for i in range(1, run_num + 1):
output_filename = job_name + "_run_" + str(i) + "_" + self.name + ".json" output_filename = job_name + "_run_" + str(i) + "_" + self.name + ".json"
cmd = "sudo %s %s --output-format=json --output=%s" % (self.fio_bin, fio_config_file, output_filename) output = self.exec_cmd(["sudo", self.fio_bin,
output, error = self.remote_call(cmd) fio_config_file, "--output-format=json",
"--output=%s" % output_filename], True)
self.log_print(output) self.log_print(output)
self.log_print(error)
else: else:
output_filename = job_name + "_" + self.name + ".json" output_filename = job_name + "_" + self.name + ".json"
cmd = "sudo %s %s --output-format=json --output=%s" % (self.fio_bin, fio_config_file, output_filename) output = self.exec_cmd(["sudo", self.fio_bin,
output, error = self.remote_call(cmd) fio_config_file, "--output-format=json",
"--output" % output_filename], True)
self.log_print(output) self.log_print(output)
self.log_print(error)
self.log_print("FIO run finished. Results in: %s" % output_filename) self.log_print("FIO run finished. Results in: %s" % output_filename)
def sys_config(self): def sys_config(self):
self.log_print("====Kernel release:====") self.log_print("====Kernel release:====")
self.log_print(self.remote_call('uname -r')[0]) self.log_print(self.exec_cmd(["uname", "-r"]))
self.log_print("====Kernel command line:====") self.log_print("====Kernel command line:====")
cmdline, error = self.remote_call('cat /proc/cmdline') cmdline = self.exec_cmd(["cat", "/proc/cmdline"])
self.log_print('\n'.join(self.get_uncommented_lines(cmdline.splitlines()))) self.log_print('\n'.join(self.get_uncommented_lines(cmdline.splitlines())))
self.log_print("====sysctl conf:====") self.log_print("====sysctl conf:====")
sysctl, error = self.remote_call('cat /etc/sysctl.conf') sysctl = self.exec_cmd(["cat", "/etc/sysctl.conf"])
self.log_print('\n'.join(self.get_uncommented_lines(sysctl.splitlines()))) self.log_print('\n'.join(self.get_uncommented_lines(sysctl.splitlines())))
self.log_print("====Cpu power info:====") self.log_print("====Cpu power info:====")
self.remote_call("cpupower frequency-info") self.log_print(self.exec_cmd(["cpupower", "frequency-info"]))
class KernelTarget(Target): class KernelTarget(Target):
@ -856,20 +873,19 @@ class KernelInitiator(Initiator):
self.log_print("Below connection attempts may result in error messages, this is expected!") self.log_print("Below connection attempts may result in error messages, this is expected!")
for subsystem in subsystems: for subsystem in subsystems:
self.log_print("Trying to connect %s %s %s" % subsystem) self.log_print("Trying to connect %s %s %s" % subsystem)
self.remote_call("sudo %s connect -t %s -s %s -n %s -a %s %s" % (self.nvmecli_bin, self.exec_cmd(["sudo", self.nvmecli_bin, "connect", "-t", self.transport,
self.transport, "-s", subsystem[0], "-n", subsystem[1], "-a", subsystem[2], self.extra_params])
*subsystem,
self.extra_params))
time.sleep(2) time.sleep(2)
def kernel_init_disconnect(self, address_list, subsys_no): def kernel_init_disconnect(self, address_list, subsys_no):
subsystems = self.discover_subsystems(address_list, subsys_no) subsystems = self.discover_subsystems(address_list, subsys_no)
for subsystem in subsystems: for subsystem in subsystems:
self.remote_call("sudo %s disconnect -n %s" % (self.nvmecli_bin, subsystem[1])) self.exec_cmd(["sudo", self.nvmecli_bin, "disconnect", "-n", subsystem[1]])
time.sleep(1) time.sleep(1)
def gen_fio_filename_conf(self, threads, io_depth, num_jobs=1): def gen_fio_filename_conf(self, threads, io_depth, num_jobs=1):
out, err = self.remote_call("sudo nvme list | grep -E 'SPDK|Linux' | awk '{print $1}'") out = self.exec_cmd(["sudo", "nvme", "list", "|", "grep", "-E", "'SPDK|Linux'",
"|", "awk", "'{print $1}'"])
nvme_list = [x for x in out.split("\n") if "nvme" in x] nvme_list = [x for x in out.split("\n") if "nvme" in x]
filename_section = "" filename_section = ""
@ -906,15 +922,18 @@ class SPDKInitiator(Initiator):
def install_spdk(self, local_spdk_zip): def install_spdk(self, local_spdk_zip):
self.put_file(local_spdk_zip, "/tmp/spdk_drop.zip") self.put_file(local_spdk_zip, "/tmp/spdk_drop.zip")
self.log_print("Copied sources zip from target") self.log_print("Copied sources zip from target")
self.remote_call("unzip -qo /tmp/spdk_drop.zip -d %s" % self.spdk_dir) self.exec_cmd(["unzip", "-qo", "/tmp/spdk_drop.zip", "-d", self.spdk_dir])
self.log_print("Sources unpacked") self.log_print("Sources unpacked")
self.log_print("Using fio binary %s" % self.fio_bin) self.log_print("Using fio binary %s" % self.fio_bin)
self.remote_call("cd %s; git submodule update --init; make clean; ./configure --with-rdma --with-fio=%s;" self.exec_cmd(["git", "-C", self.spdk_dir, "submodule", "update", "--init"])
"make -j$(($(nproc)*2))" % (self.spdk_dir, os.path.dirname(self.fio_bin))) self.exec_cmd(["git", "-C", self.spdk_dir, "clean", "-ffdx"])
self.exec_cmd(["cd", self.spdk_dir, "&&", "./configure", "--with-rdma", "--with-fio=%s" % os.path.dirname(self.fio_bin)])
self.exec_cmd(["make", "-C", self.spdk_dir, "clean"])
self.exec_cmd(["make", "-C", self.spdk_dir, "-j$(($(nproc)*2))"])
self.log_print("SPDK built") self.log_print("SPDK built")
self.remote_call("sudo %s/scripts/setup.sh" % self.spdk_dir) self.exec_cmd(["sudo", "%s/scripts/setup.sh" % self.spdk_dir])
def gen_spdk_bdev_conf(self, remote_subsystem_list): def gen_spdk_bdev_conf(self, remote_subsystem_list):
bdev_cfg_section = { bdev_cfg_section = {
@ -1091,7 +1110,7 @@ if __name__ == "__main__":
for i in initiators: for i in initiators:
if i.mode == "kernel": if i.mode == "kernel":
i.kernel_init_disconnect(i.nic_ips, target_obj.subsys_no) i.kernel_init_disconnect(i.remote_nic_ips, target_obj.subsys_no)
i.copy_result_files(target_results_dir) i.copy_result_files(target_results_dir)
target_obj.parse_results(target_results_dir) target_obj.parse_results(target_results_dir)