numam-spdk/scripts/config_converter.py
Ben Walker af8affd1a9 iscsi: Deprecate MinConnectionsPerCore
iSCSI is currently being adapted to SPDK's new threading model,
and this parameter doesn't make sense anymore. Remove it for
now until something functionally equivalent is added later.

Change-Id: Ia0e2f5aa81b72d99467c5a900619fbeeb42b2069
Signed-off-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/452779
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
2019-05-06 17:10:48 +00:00

730 lines
25 KiB
Python
Executable File

#!/usr/bin/env python3
import configparser
import re
import sys
import json
from collections import OrderedDict
bdev_dict = OrderedDict()
bdev_dict["set_bdev_options"] = []
bdev_dict["construct_split_vbdev"] = []
bdev_dict["set_bdev_nvme_options"] = []
bdev_dict["construct_nvme_bdev"] = []
bdev_dict["set_bdev_nvme_hotplug"] = []
bdev_dict["construct_malloc_bdev"] = []
bdev_dict["construct_aio_bdev"] = []
bdev_dict["construct_pmem_bdev"] = []
bdev_dict["construct_virtio_dev"] = []
vhost_dict = OrderedDict()
vhost_dict["construct_vhost_scsi_controller"] = []
vhost_dict["construct_vhost_blk_controller"] = []
vhost_dict["construct_vhost_nvme_controller"] = []
iscsi_dict = OrderedDict()
iscsi_dict["set_iscsi_options"] = []
iscsi_dict["add_portal_group"] = []
iscsi_dict["add_initiator_group"] = []
iscsi_dict["construct_target_node"] = []
nvmf_dict = OrderedDict()
nvmf_dict["set_nvmf_target_config"] = []
nvmf_dict["set_nvmf_target_max_subsystems"] = []
nvmf_dict["subsystems"] = []
# dictionary with new config that will be written to new json config file
subsystem = {
"copy": None,
"interface": None,
"net_framework": None,
"bdev": bdev_dict,
"scsi": [],
"nvmf": nvmf_dict,
"nbd": [],
"vhost": vhost_dict,
"iscsi": iscsi_dict
}
class OptionOrderedDict(OrderedDict):
def __setitem__(self, option, value):
if option in self and isinstance(value, list):
self[option].extend(value)
return
super(OptionOrderedDict, self).__setitem__(option, value)
no_yes_map = {"no": False, "No": False, "Yes": True, "yes": True}
def generate_new_json_config():
json_subsystem = [
{'subsystem': "copy", 'config': None},
{"subsystem": "interface", "config": None},
{"subsystem": "net_framework", "config": None},
{"subsystem": "bdev", "config": []},
{"subsystem": "scsi", "config": None},
{"subsystem": "nvmf", "config": []},
{"subsystem": "nbd", "config": []},
{"subsystem": "vhost", "config": []},
{"subsystem": "iscsi", "config": []}
]
for method in subsystem['bdev']:
for item in subsystem['bdev'][method]:
json_subsystem[3]['config'].append(item)
for item in subsystem['scsi']:
if json_subsystem[4]['config'] is None:
json_subsystem[4]['config'] = []
json_subsystem[4]['config'].append(item)
for method in subsystem['nvmf']:
for item in subsystem['nvmf'][method]:
json_subsystem[5]['config'].append(item)
for method in subsystem['vhost']:
for item in subsystem['vhost'][method]:
json_subsystem[7]['config'].append(item)
for method in subsystem['iscsi']:
for item in subsystem['iscsi'][method]:
json_subsystem[8]['config'].append(item)
return {"subsystems": json_subsystem}
section_to_subsystem = {
"Bdev": subsystem['bdev'],
"AIO": subsystem['bdev'],
"Malloc": subsystem['bdev'],
"Nvme": subsystem['bdev'],
"Pmem": subsystem['bdev'],
"Split": subsystem['bdev'],
"Nvmf": subsystem['nvmf'],
"Subsystem": subsystem['nvmf'],
"VhostScsi": subsystem['vhost'],
"VhostBlk": subsystem['vhost'],
"VhostNvme": subsystem['vhost'],
"VirtioUser": subsystem['bdev'],
"iSCSI": subsystem['iscsi'],
"PortalGroup": subsystem['iscsi'],
"InitiatorGroup": subsystem['iscsi'],
"TargetNode": subsystem['iscsi']
}
def set_param(params, cfg_name, value):
for param in params:
if param[0] != cfg_name:
continue
if param[1] == "disable_chap":
param[3] = True if value == "None" else False
elif param[1] == "require_chap":
param[3] = True if value in ["CHAP", "Mutual"] else False
elif param[1] == "mutual_chap":
param[3] = True if value == "Mutual" else False
elif param[1] == "chap_group":
param[3] = int(value.replace("AuthGroup", ""))
elif param[2] == bool:
param[3] = True if value in ("yes", "true", "Yes") else False
elif param[2] == "hex":
param[3] = str(int(value, 16))
elif param[2] == int:
param[3] = int(value)
elif param[2] == list:
param[3].append(value)
elif param[2] == "dev_type":
if value.lower() == "blk":
param[3] = "blk"
else:
param[3] = param[2](value.replace("\"", ""))
def to_json_params(params):
out = {}
for param in params:
if param[3] is not None:
out[param[1]] = param[3]
return out
def get_bdev_options_json(config, section):
params = [
["BdevIoPoolSize", "bdev_io_pool_size", int, 65536],
["BdevIoCacheSize", "bdev_io_cache_size", int, 256]
]
for option in config.options("Bdev"):
set_param(params, option, config.get("Bdev", option))
return [{"params": to_json_params(params), "method": "set_bdev_options"}]
def get_aio_bdev_json(config, section):
aio_json = []
value = None
for option in config.options("AIO"):
if option == "AIO":
value = config.get("AIO", option).split("\n")
if value is None:
return aio_json
for item in value:
items = re.findall(r"\S+", item)
params = {}
params['filename'] = items[0]
params['name'] = items[1]
if len(items) == 3:
params['block_size'] = int(items[2])
aio_json.append({
"params": params,
"method": "construct_aio_bdev"
})
return aio_json
def get_malloc_bdev_json(config, section):
malloc_json = []
params = [
['NumberOfLuns', '', int, -1],
['LunSizeInMB', '', int, 20],
['BlockSize', '', int, 512]
]
for option in config.options("Malloc"):
set_param(params, option, config.get("Malloc", option))
for lun in range(0, params[0][3]):
malloc_json.append({
"params": {
"block_size": params[2][3],
"num_blocks": params[1][3] * 1024 * 1024 / params[2][3],
"name": "Malloc%s" % lun
},
"method": "construct_malloc_bdev"
})
return malloc_json
def get_nvme_bdev_json(config, section):
params = [
["RetryCount", "retry_count", int, 4],
["TimeoutuSec", "timeout_us", int, 0],
["AdminPollRate", "nvme_adminq_poll_period_us", int, 1000000],
["ActionOnTimeout", "action_on_timeout", str, "none"],
["IOPollRate", "nvme_ioq_poll_period_us", int, 0],
["HotplugEnable", "enable", bool, False],
["AdminPollRate", "period_us", int, 1000]
]
nvme_json = []
for option in config.options("Nvme"):
value = config.get("Nvme", option)
if "TransportID" == option:
entry = re.findall(r"\S+", value)
nvme_name = entry[-1]
trtype = re.findall(r"trtype:\S+", value)
if trtype:
trtype = trtype[0].replace("trtype:", "").replace("\"", "")
traddr = re.findall(r"traddr:\S+", value)
if traddr:
traddr = traddr[0].replace("traddr:", "").replace("\"", "")
nvme_json.append({
"params": {
"trtype": trtype,
"name": nvme_name,
"traddr": traddr
},
"method": "construct_nvme_bdev"
})
else:
set_param(params, option, value)
params[3][3] = params[3][3].lower()
params[6][3] = params[6][3] * 100
nvme_json.append({
"params": to_json_params(params[5:7]),
"method": "set_bdev_nvme_hotplug"
})
nvme_json.append({
"params": to_json_params(params[0:5]),
"method": "set_bdev_nvme_options"
})
return nvme_json
def get_pmem_bdev_json(config, section):
pmem_json = []
for option in config.options(section):
if "Blk" == option:
for value in config.get(section, option).split("\n"):
items = re.findall(r"\S+", value)
pmem_json.append({
"params": {
"name": items[1],
"pmem_file": items[0]
},
"method": "construct_pmem_bdev"
})
return pmem_json
def get_split_bdev_json(config, section):
split_json = []
value = []
for option in config.options("Split"):
if "Split" == option:
value = config.get("Split", option)
if value and not isinstance(value, list):
value = [value]
for split in value:
items = re.findall(r"\S+", split)
split_size_mb = 0
base_bdev = items[0]
split_count = int(items[1])
if len(items) == 3:
split_size_mb = items[2]
split_json.append({
"params": {
"base_bdev": base_bdev,
"split_size_mb": split_size_mb,
"split_count": split_count
},
"method": "construct_split_vbdev"
})
return split_json
def get_nvmf_options_json(config, section):
params = [
["AcceptorPollRate", "acceptor_poll_rate", int, 10000],
["MaxSubsystems", "max_subsystems", int, 1024]
]
for option in config.options("Nvmf"):
set_param(params, option, config.get("Nvmf", option))
nvmf_json = []
nvmf_json.append({
"params": to_json_params([params[0]]),
"method": "set_nvmf_target_config"
})
nvmf_json.append({
"params": to_json_params(params[1:7]),
"method": "set_nvmf_target_max_subsystems"
})
return nvmf_json
def get_nvmf_subsystem_json(config, section):
nvmf_subsystem_methods = []
params = [
# Last items are default values if given entry is not set
["Host", "hosts", list, []],
["NQN", "nqn", str, ""],
["AllowAnyHost", "allow_any_host", bool, False],
["SN", "serial_number", str, "00000000000000000000"],
["MN", "model_number", str, "SPDK bdev Controller"],
["MaxNamespaces", "max_namespaces", str, ""],
]
listen_address = []
namespaces = []
nsid = 0
searched_items = [param[0] for param in params]
for option in config.options(section):
value = config.get(section, option)
if option in searched_items:
set_param(params, option, value)
continue
if "Listen" == option:
items = re.findall(r"\S+", value)
adrfam = "IPv4"
if len(items[1].split(":")) > 2:
adrfam = "IPv6"
listen_address.append({
"trtype": items[0],
"adrfam": adrfam,
"trsvcid": items[1].rsplit(":", 1)[-1],
"traddr": items[1].rsplit(":", 1)[0].replace(
"]", "").replace("[", "")
})
if "Namespace" == option:
for item in value.split("\n"):
items = re.findall(r"\S+", item)
if len(items) == 2:
nsid = items[1]
else:
nsid += 1
namespaces.append({
"nsid": int(nsid),
"bdev_name": items[0],
})
# Get parameters: nqn, allow_any_host, serial_number
# for nvmf_subsystem_create rpc method
parameters = to_json_params(params[1:5])
nvmf_subsystem_methods.append({
"params": parameters,
"method": "nvmf_subsystem_create"
})
for listen in listen_address:
nvmf_subsystem_methods.append({
"params": {
"listen_address": listen,
"nqn": parameters['nqn']
},
"method": "nvmf_subsystem_add_listener"
})
for host in to_json_params([params[0]])['hosts']:
nvmf_subsystem_methods.append({
"params": {
"host": host,
"nqn": parameters['nqn']
},
"method": "nvmf_subsystem_add_host"
})
for namespace in namespaces:
nvmf_subsystem_methods.append({
"params": {
"namespace": namespace,
"nqn": parameters['nqn']
},
"method": "nvmf_subsystem_add_ns"
})
# Define max_namespaces if it is set in old config
if params[5][3]:
nvmf_subsystem_methods[0]['params']['max_namespaces'] = int(params[5][3])
return nvmf_subsystem_methods
def get_vhost_scsi_json(config, section):
params = [
["Name", "ctrlr", str, None],
["Cpumask", "cpumask", "hex", "1"],
]
targets = []
vhost_scsi_json = []
for option in config.options(section):
value = config.get(section, option)
if option in ["Name", "Cpumask"]:
set_param(params, option, value)
if "Target" == option:
for item in value.split("\n"):
items = re.findall(r"\S+", item)
targets.append({
"scsi_target_num": int(items[0]),
"ctrlr": params[0][3],
"bdev_name": items[1]
})
vhost_scsi_json.append({
"params": to_json_params(params),
"method": "construct_vhost_scsi_controller"
})
for target in targets:
vhost_scsi_json.append({
"params": target,
"method": "add_vhost_scsi_lun"
})
return vhost_scsi_json
def get_vhost_blk_json(config, section):
params = [
["ReadOnly", "readonly", bool, False],
["Dev", "dev_name", str, ""],
["Name", "ctrlr", str, ""],
["Cpumask", "cpumask", "hex", ""]
]
for option in config.options(section):
set_param(params, option, config.get(section, option))
return [{"method": "construct_vhost_blk_controller",
"params": to_json_params(params)}]
def get_vhost_nvme_json(config, section):
params = [
["Name", "ctrlr", str, ""],
["NumberOfQueues", "io_queues", int, -1],
["Cpumask", "cpumask", "hex", 0x1],
["Namespace", "bdev_name", list, []]
]
for option in config.options(section):
values = config.get(section, option).split("\n")
for value in values:
set_param(params, option, value)
vhost_nvme_json = []
vhost_nvme_json.append({
"params": to_json_params(params[:3]),
"method": "construct_vhost_nvme_controller"
})
for namespace in params[3][3]:
vhost_nvme_json.append({
"params": {
"ctrlr": params[0][3],
"bdev_name": namespace,
},
"method": "add_vhost_nvme_ns"
})
return vhost_nvme_json
def get_virtio_user_json(config, section):
params = [
["Path", "traddr", str, ""],
["Queues", "vq_count", int, 1],
["Type", "dev_type", "dev_type", "scsi"],
["Name", "name", str, section],
# Define parameters with default values.
# These params are set by rpc commands and
# do not occur in ini config file.
# But they are visible in json config file
# with default values even if not set by rpc.
[None, "trtype", str, "user"],
[None, "vq_size", int, 512]
]
for option in config.options(section):
set_param(params, option, config.get(section, option))
dev_name = "Scsi"
if params[2][3] == "blk":
dev_name = "Blk"
params[3][3] = params[3][3].replace("User", dev_name)
return [{
"params": to_json_params(params),
"method": "construct_virtio_dev"
}]
def get_iscsi_options_json(config, section):
params = [
['AllowDuplicateIsid', 'allow_duplicated_isid', bool, False],
['DefaultTime2Retain', 'default_time2retain', int, 20],
['DiscoveryAuthMethod', 'mutual_chap', bool, False],
['MaxConnectionsPerSession', 'max_connections_per_session', int, 2],
['Timeout', 'nop_timeout', int, 60],
['DiscoveryAuthMethod', 'disable_chap', bool, False],
['DiscoveryAuthMethod', 'require_chap', bool, False],
['NodeBase', 'node_base', str, "iqn.2016-06.io.spdk"],
['AuthFile', 'auth_file', str, None],
['DiscoveryAuthGroup', 'chap_group', int, 0],
['MaxSessions', 'max_sessions', int, 128],
['ImmediateData', 'immediate_data', bool, True],
['ErrorRecoveryLevel', 'error_recovery_level', int, 0],
['NopInInterval', 'nop_in_interval', int, 30],
['DefaultTime2Wait', 'default_time2wait', int, 2],
['QueueDepth', 'max_queue_depth', int, 64],
['', 'first_burst_length', int, 8192]
]
for option in config.options(section):
set_param(params, option, config.get(section, option))
return [{"method": "set_iscsi_options", "params": to_json_params(params)}]
def get_iscsi_portal_group_json(config, name):
portal_group_json = []
portals = []
for option in config.options(name):
if "Portal" == option:
for value in config.get(name, option).split("\n"):
items = re.findall(r"\S+", value)
portal = {'host': items[1].rsplit(":", 1)[0]}
if "@" in items[1]:
portal['port'] =\
items[1].rsplit(":", 1)[1].split("@")[0]
portal['cpumask'] =\
items[1].rsplit(":", 1)[1].split("@")[1]
else:
portal['port'] = items[1].rsplit(":", 1)[1]
portals.append(portal)
portal_group_json.append({
"params": {
"portals": portals,
"tag": int(re.findall(r'\d+', name)[0])
},
"method": "add_portal_group"
})
return portal_group_json
def get_iscsi_initiator_group_json(config, name):
initiators = []
netmasks = []
for option in config.options(name):
if "InitiatorName" == option:
initiators.append(config.get(name, option))
if "Netmask" == option:
netmasks.append(config.get(name, option))
initiator_group_json = {
"params": {
"initiators": initiators,
"tag": int(re.findall(r'\d+', name)[0]),
"netmasks": netmasks
},
"method": "add_initiator_group"
}
return [initiator_group_json]
def get_iscsi_target_node_json(config, section):
luns = []
mutual_chap = False
name = ""
alias_name = ""
require_chap = False
chap_group = 1
pg_ig_maps = []
data_digest = False
disable_chap = False
header_digest = False
queue_depth = 64
for option in config.options(section):
value = config.get(section, option)
if "TargetName" == option:
name = value
if "TargetAlias" == option:
alias_name = value.replace("\"", "")
if "Mapping" == option:
items = re.findall(r"\S+", value)
pg_ig_maps.append({
"ig_tag": int(re.findall(r'\d+', items[1])[0]),
"pg_tag": int(re.findall(r'\d+', items[0])[0])
})
if "AuthMethod" == option:
items = re.findall(r"\S+", value)
for item in items:
if "CHAP" == item:
require_chap = True
elif "Mutual" == item:
mutual_chap = True
elif "Auto" == item:
disable_chap = False
require_chap = False
mutual_chap = False
elif "None" == item:
disable_chap = True
require_chap = False
mutual_chap = False
if "AuthGroup" == option: # AuthGroup1
items = re.findall(r"\S+", value)
chap_group = int(re.findall(r'\d+', items[0])[0])
if "UseDigest" == option:
items = re.findall(r"\S+", value)
for item in items:
if "Header" == item:
header_digest = True
elif "Data" == item:
data_digest = True
elif "Auto" == item:
header_digest = False
data_digest = False
if re.match(r"LUN\d+", option):
luns.append({"lun_id": len(luns),
"bdev_name": value})
if "QueueDepth" == option:
queue_depth = int(value)
params = {"alias_name": alias_name}
params["name"] = "iqn.2016-06.io.spdk:%s" % name
params["luns"] = luns
params["pg_ig_maps"] = pg_ig_maps
params["queue_depth"] = queue_depth
params["chap_group"] = chap_group
params["header_digest"] = header_digest
params["mutual_chap"] = mutual_chap
params["require_chap"] = require_chap
params["data_digest"] = data_digest
params["disable_chap"] = disable_chap
target_json = {
"params": params,
"method": "construct_target_node"
}
return [target_json]
if __name__ == "__main__":
try:
config = configparser.ConfigParser(strict=False, delimiters=(' '),
dict_type=OptionOrderedDict,
allow_no_value=True)
# Do not parse options and values. Capital letters are relevant.
config.optionxform = str
config.read_file(sys.stdin)
except Exception as e:
print("Exception while parsing config: %s" % e)
exit(1)
# Add missing sections to generate default configuration
for section in ['Nvme', 'Nvmf', 'Bdev', 'iSCSI']:
if section not in config.sections():
config.add_section(section)
for section in config.sections():
match = re.match(r'(Bdev|Nvme|Malloc|VirtioUser\d+|Split|Pmem|AIO|'
r'iSCSI|PortalGroup\d+|InitiatorGroup\d+|'
r'TargetNode\d+|Nvmf|Subsystem\d+|VhostScsi\d+|'
r'VhostBlk\d+|VhostNvme\d+)', section)
if match:
match_section = ''.join(letter for letter in match.group(0)
if not letter.isdigit())
if match_section == "Bdev":
items = get_bdev_options_json(config, section)
elif match_section == "AIO":
items = get_aio_bdev_json(config, section)
elif match_section == "Malloc":
items = get_malloc_bdev_json(config, section)
elif match_section == "Nvme":
items = get_nvme_bdev_json(config, section)
elif match_section == "Pmem":
items = get_pmem_bdev_json(config, section)
elif match_section == "Split":
items = get_split_bdev_json(config, section)
elif match_section == "Nvmf":
items = get_nvmf_options_json(config, section)
elif match_section == "Subsystem":
items = get_nvmf_subsystem_json(config, section)
elif match_section == "VhostScsi":
items = get_vhost_scsi_json(config, section)
elif match_section == "VhostBlk":
items = get_vhost_blk_json(config, section)
elif match_section == "VhostNvme":
items = get_vhost_nvme_json(config, section)
elif match_section == "VirtioUser":
items = get_virtio_user_json(config, section)
elif match_section == "iSCSI":
items = get_iscsi_options_json(config, section)
elif match_section == "PortalGroup":
items = get_iscsi_portal_group_json(config, section)
elif match_section == "InitiatorGroup":
items = get_iscsi_initiator_group_json(config, section)
elif match_section == "TargetNode":
items = get_iscsi_target_node_json(config, section)
for item in items:
if match_section == "VhostScsi":
section_to_subsystem[match_section]["construct_vhost_scsi_controller"].append(item)
elif match_section == "VhostNvme":
section_to_subsystem[match_section]["construct_vhost_nvme_controller"].append(item)
elif match_section == "Subsystem":
section_to_subsystem[match_section]["subsystems"].append(item)
else:
section_to_subsystem[match_section][
item['method']].append(item)
elif section == "Global":
pass
elif section == "Ioat":
# Ioat doesn't support JSON config yet.
pass
elif section == "VirtioPci":
print("Please use spdk target flags.")
exit(1)
else:
print("An invalid section detected: %s.\n"
"Please revise your config file." % section)
exit(1)
json.dump(generate_new_json_config(), sys.stdout, indent=2)
print("")