nvmf: allow user to specify NSID via conf and RPC

The nvmf library now supports the ability to assign arbitrary NSIDs,
rather than automatically assigning the next one in line.  Expose this
functionality to the user via the configuration file and RPC interfaces.

Change-Id: Ia85a9a6dfe31a2cd0605c7a6c098eec0c1b7de68
Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-on: https://review.gerrithub.io/376463
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

CHANGELOG.md

@@ -45,6 +45,9 @@ connection will be denied; this is a behavior change from previous releases,
 which allowed any host NQN to connect if the Host list was empty.
 AllowAnyHost is disabled by default.
 
+NVMe-oF namespaces may now be assigned arbitrary namespace IDs, and the number
+of namespaces per subsystem is no longer limited.
+
 ### Environment Abstraction Layer
 
 A new default value, SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, was added to provide

app/nvmf_tgt/conf.c

@@ -233,8 +233,8 @@ spdk_nvmf_parse_subsystem(struct spdk_conf_section *sp)
 	char *hosts[MAX_HOSTS];
 	bool allow_any_host;
 	const char *sn;
-	int num_devs;
+	size_t num_ns;
-	char *devs[MAX_NAMESPACES];
+	struct spdk_nvmf_ns_params ns_list[MAX_NAMESPACES] = {};
 
 	nqn = spdk_conf_section_get_val(sp, "NQN");
 	mode = spdk_conf_section_get_val(sp, "Mode");
@@ -296,21 +296,39 @@ spdk_nvmf_parse_subsystem(struct spdk_conf_section *sp)
 
 	sn = spdk_conf_section_get_val(sp, "SN");
 
-	num_devs = 0;
+	num_ns = 0;
-	for (i = 0; i < SPDK_COUNTOF(devs); i++) {
+	for (i = 0; i < SPDK_COUNTOF(ns_list); i++) {
-		devs[i] = spdk_conf_section_get_nmval(sp, "Namespace", i, 0);
+		char *nsid_str;
-		if (!devs[i]) {
+
+		ns_list[i].bdev_name = spdk_conf_section_get_nmval(sp, "Namespace", i, 0);
+		if (!ns_list[i].bdev_name) {
 			break;
 		}
 
-		num_devs++;
+		nsid_str = spdk_conf_section_get_nmval(sp, "Namespace", i, 1);
+		if (nsid_str) {
+			char *end;
+			unsigned long nsid_ul = strtoul(nsid_str, &end, 0);
+
+			if (*end != '\0' || nsid_ul == 0 || nsid_ul >= UINT32_MAX) {
+				SPDK_ERRLOG("Invalid NSID %s\n", nsid_str);
+				return -1;
+			}
+
+			ns_list[i].nsid = (uint32_t)nsid_ul;
+		} else {
+			/* Automatically assign the next available NSID. */
+			ns_list[i].nsid = 0;
+		}
+
+		num_ns++;
 	}
 
 	ret = spdk_nvmf_construct_subsystem(nqn, lcore,
 					    num_listen_addrs, listen_addrs,
 					    num_hosts, hosts, allow_any_host,
 					    sn,
-					    num_devs, devs);
+					    num_ns, ns_list);
 
 	for (i = 0; i < MAX_LISTEN_ADDRESSES; i++) {
 		free(listen_addrs_str[i]);
@@ -362,14 +380,14 @@ int
 spdk_nvmf_construct_subsystem(const char *name, int32_t lcore,
 			      int num_listen_addresses, struct rpc_listen_address *addresses,
 			      int num_hosts, char *hosts[], bool allow_any_host,
-			      const char *sn, int num_devs, char *dev_list[])
+			      const char *sn, size_t num_ns, struct spdk_nvmf_ns_params *ns_list)
 {
 	struct spdk_nvmf_subsystem *subsystem;
 	struct nvmf_tgt_subsystem *app_subsys;
 	int i, rc;
+	size_t j;
 	uint64_t mask;
 	struct spdk_bdev *bdev;
-	const char *namespace;
 
 	if (name == NULL) {
 		SPDK_ERRLOG("No NQN specified for subsystem\n");
@@ -395,7 +413,7 @@ spdk_nvmf_construct_subsystem(const char *name, int32_t lcore,
 	lcore = spdk_nvmf_allocate_lcore(mask, lcore);
 	g_last_core = lcore;
 
-	app_subsys = nvmf_tgt_create_subsystem(name, SPDK_NVMF_SUBTYPE_NVME, num_devs, lcore);
+	app_subsys = nvmf_tgt_create_subsystem(name, SPDK_NVMF_SUBTYPE_NVME, num_ns, lcore);
 	if (app_subsys == NULL) {
 		SPDK_ERRLOG("Subsystem creation failed\n");
 		return -1;
@@ -460,18 +478,21 @@ spdk_nvmf_construct_subsystem(const char *name, int32_t lcore,
 		goto error;
 	}
 
-	for (i = 0; i < num_devs; i++) {
+	for (j = 0; j < num_ns; j++) {
-		namespace = dev_list[i];
+		struct spdk_nvmf_ns_params *ns_params = &ns_list[j];
-		if (!namespace) {
+
-			SPDK_ERRLOG("Namespace %d: missing block device\n", i);
+		if (!ns_params->bdev_name) {
+			SPDK_ERRLOG("Namespace missing bdev name\n");
 			goto error;
 		}
-		bdev = spdk_bdev_get_by_name(namespace);
+
+		bdev = spdk_bdev_get_by_name(ns_params->bdev_name);
 		if (bdev == NULL) {
-			SPDK_ERRLOG("Could not find namespace bdev '%s'\n", namespace);
+			SPDK_ERRLOG("Could not find namespace bdev '%s'\n", ns_params->bdev_name);
 			goto error;
 		}
-		if (spdk_nvmf_subsystem_add_ns(subsystem, bdev, 0) == 0) {
+
+		if (spdk_nvmf_subsystem_add_ns(subsystem, bdev, ns_params->nsid) == 0) {
 			goto error;
 		}
 

app/nvmf_tgt/nvmf_rpc.c

@@ -116,6 +116,9 @@ dump_nvmf_subsystem(struct spdk_json_write_ctx *w, struct nvmf_tgt_subsystem *tg
 			spdk_json_write_object_begin(w);
 			spdk_json_write_name(w, "nsid");
 			spdk_json_write_int32(w, spdk_nvmf_ns_get_id(ns));
+			spdk_json_write_name(w, "bdev_name");
+			spdk_json_write_string(w, spdk_bdev_get_name(spdk_nvmf_ns_get_bdev(ns)));
+			/* NOTE: "name" is kept for compatibility only - new code should use bdev_name. */
 			spdk_json_write_name(w, "name");
 			spdk_json_write_string(w, spdk_bdev_get_name(spdk_nvmf_ns_get_bdev(ns)));
 			spdk_json_write_object_end(w);
@@ -210,31 +213,74 @@ decode_rpc_hosts(const struct spdk_json_val *val, void *out)
 
 
 
-struct rpc_dev_names {
+struct rpc_namespaces {
-	size_t num_names;
+	size_t num_ns;
-	char *names[RPC_MAX_NAMESPACES];
+	struct spdk_nvmf_ns_params ns_params[RPC_MAX_NAMESPACES];
 };
 
-static int
-decode_rpc_dev_names(const struct spdk_json_val *val, void *out)
-{
-	struct rpc_dev_names *dev_names = out;
 
-	return spdk_json_decode_array(val, spdk_json_decode_string, dev_names->names,
+static const struct spdk_json_object_decoder rpc_ns_params_decoders[] = {
-				      SPDK_COUNTOF(dev_names->names),
+	{"nsid", offsetof(struct spdk_nvmf_ns_params, nsid), spdk_json_decode_uint32, true},
-				      &dev_names->num_names, sizeof(char *));
+	{"bdev_name", offsetof(struct spdk_nvmf_ns_params, bdev_name), spdk_json_decode_string},
-}
+};
 
 static void
-free_rpc_dev_names(struct rpc_dev_names *r)
+free_rpc_namespaces(struct rpc_namespaces *r)
 {
 	size_t i;
 
-	for (i = 0; i < r->num_names; i++) {
+	for (i = 0; i < r->num_ns; i++) {
-		free(r->names[i]);
+		free(r->ns_params[i].bdev_name);
 	}
 }
 
+static int
+decode_rpc_ns_params(const struct spdk_json_val *val, void *out)
+{
+	struct spdk_nvmf_ns_params *ns_params = out;
+
+	return spdk_json_decode_object(val, rpc_ns_params_decoders,
+				       SPDK_COUNTOF(rpc_ns_params_decoders),
+				       ns_params);
+}
+
+static int
+decode_rpc_namespaces(const struct spdk_json_val *val, void *out)
+{
+	struct rpc_namespaces *namespaces = out;
+	char *names[RPC_MAX_NAMESPACES]; /* old format - array of strings (bdev names) */
+	size_t i;
+	int rc;
+
+	/* First try to decode namespaces as an array of objects (new format) */
+	if (spdk_json_decode_array(val, decode_rpc_ns_params, namespaces->ns_params,
+				   SPDK_COUNTOF(namespaces->ns_params),
+				   &namespaces->num_ns, sizeof(*namespaces->ns_params)) == 0) {
+		return 0;
+	}
+
+	/* If that fails, try to decode namespaces as an array of strings (old format) */
+	free_rpc_namespaces(namespaces);
+	memset(namespaces, 0, sizeof(*namespaces));
+	rc = spdk_json_decode_array(val, spdk_json_decode_string, names,
+				    SPDK_COUNTOF(names),
+				    &namespaces->num_ns, sizeof(char *));
+	if (rc == 0) {
+		/* Decoded old format - copy to ns_params (new format) */
+		for (i = 0; i < namespaces->num_ns; i++) {
+			namespaces->ns_params[i].bdev_name = names[i];
+		}
+		return 0;
+	}
+
+	/* Failed to decode - don't leave dangling string pointers around */
+	for (i = 0; i < namespaces->num_ns; i++) {
+		free(names[i]);
+	}
+
+	return rc;
+}
+
 static void
 free_rpc_listen_addresses(struct rpc_listen_addresses *r)
 {
@@ -267,7 +313,7 @@ struct rpc_subsystem {
 	bool allow_any_host;
 	char *pci_address;
 	char *serial_number;
-	struct rpc_dev_names namespaces;
+	struct rpc_namespaces namespaces;
 };
 
 static void
@@ -276,7 +322,7 @@ free_rpc_subsystem(struct rpc_subsystem *req)
 	free(req->mode);
 	free(req->nqn);
 	free(req->serial_number);
-	free_rpc_dev_names(&req->namespaces);
+	free_rpc_namespaces(&req->namespaces);
 	free_rpc_listen_addresses(&req->listen_addresses);
 	free_rpc_hosts(&req->hosts);
 }
@@ -289,7 +335,7 @@ static const struct spdk_json_object_decoder rpc_subsystem_decoders[] = {
 	{"hosts", offsetof(struct rpc_subsystem, hosts), decode_rpc_hosts, true},
 	{"allow_any_host", offsetof(struct rpc_subsystem, allow_any_host), spdk_json_decode_bool, true},
 	{"serial_number", offsetof(struct rpc_subsystem, serial_number), spdk_json_decode_string, true},
-	{"namespaces", offsetof(struct rpc_subsystem, namespaces), decode_rpc_dev_names, true},
+	{"namespaces", offsetof(struct rpc_subsystem, namespaces), decode_rpc_namespaces, true},
 };
 
 static void
@@ -328,7 +374,7 @@ spdk_rpc_construct_nvmf_subsystem(struct spdk_jsonrpc_request *request,
 					    req.listen_addresses.addresses,
 					    req.hosts.num_hosts, req.hosts.hosts, req.allow_any_host,
 					    req.serial_number,
-					    req.namespaces.num_names, req.namespaces.names);
+					    req.namespaces.num_ns, req.namespaces.ns_params);
 	if (ret) {
 		goto invalid;
 	}

app/nvmf_tgt/nvmf_tgt.h

@@ -79,12 +79,17 @@ struct nvmf_tgt_subsystem *nvmf_tgt_create_subsystem(const char *name,
 		enum spdk_nvmf_subtype subtype, uint32_t num_ns,
 		uint32_t lcore);
 
+struct spdk_nvmf_ns_params {
+	char *bdev_name;
+	uint32_t nsid;
+};
+
 int
 spdk_nvmf_construct_subsystem(const char *name,
 			      int32_t lcore,
 			      int num_listen_addresses, struct rpc_listen_address *addresses,
 			      int num_hosts, char *hosts[], bool allow_any_host,
-			      const char *sn, int num_devs, char *dev_list[]);
+			      const char *sn, size_t num_ns, struct spdk_nvmf_ns_params *ns_list);
 
 int
 nvmf_tgt_shutdown_subsystem_by_nqn(const char *nqn);

doc/nvmf.md

@@ -163,7 +163,7 @@ Listen RDMA 192.168.100.8:4420
 AllowAnyHost No
 Host nqn.2016-06.io.spdk:init
 SN SPDK00000000000001
-Namespace Nvme0n1
+Namespace Nvme0n1 1
 
 [Subsystem2]
 NQN nqn.2016-06.io.spdk:cnode2
@@ -171,7 +171,7 @@ Core 26
 Listen RDMA 192.168.100.9:4420
 AllowAnyHost Yes
 SN SPDK00000000000002
-Namespace Nvme1n1
+Namespace Nvme1n1 1
 ~~~
 SPDK executes all code for an NVMe-oF subsystem on a single thread. Different subsystems may execute
 on different threads. SPDK gives the user maximum control to determine how many CPU cores are used
@@ -188,7 +188,8 @@ file as follows:
 **Create malloc LUNs:** See @ref bdev_getting_started for details on creating Malloc block devices.
 
 **Create a virtual controller:** Any bdev may be presented as a namespace. For example, to create a
-virtual controller with two namespaces backed by the malloc LUNs named Malloc0 and Malloc1:
+virtual controller with two namespaces backed by the malloc LUNs named Malloc0 and Malloc1 and made
+available as NSID 1 and 2:
 ~~~{.sh}
 # Virtual controller
 [Subsystem2]
@@ -198,6 +199,6 @@ virtual controller with two namespaces backed by the malloc LUNs named Malloc0 a
   AllowAnyHost No
   Host nqn.2016-06.io.spdk:init
   SN SPDK00000000000001
-  Namespace Malloc0
+  Namespace Malloc0 1
-  Namespace Malloc1
+  Namespace Malloc1 2
 ~~~

etc/spdk/nvmf.conf.in

@@ -130,9 +130,14 @@
 # - Between 0 and 255 Host directives are allowed. This defines the
 #   NQNs of allowed hosts. If no Host directive is specified, all hosts
 #   are allowed to connect.
-# - Exactly 1 NVMe directive specifying an NVMe device by PCI BDF. The
+# - Between 0 and 255 Namespace directives are allowed.  These define the
-#   PCI domain:bus:device.function can be replaced by "*" to indicate
+#   namespaces accessible from this subsystem.
-#   any PCI device.
+#   The user must specify a bdev name for each namespace, and may optionally
+#   specify a namespace ID.  If nsid is omitted, the namespace will be
+#   assigned the next available NSID.  The NSID must be unique within the
+#   subsystem.
+#   Syntax:
+#     Namespace <bdev_name> [<nsid>]
 
 # Namespaces backed by physical NVMe devices
 [Subsystem1]
@@ -142,8 +147,8 @@
   AllowAnyHost No
   Host nqn.2016-06.io.spdk:init
   SN SPDK00000000000001
-  Namespace Nvme0n1
+  Namespace Nvme0n1 1
-  Namespace Nvme1n1
+  Namespace Nvme1n1 2
 
 # Multiple subsystems are allowed.
 # Namespaces backed by non-NVMe devices

scripts/rpc.py

@@ -426,7 +426,18 @@ def construct_nvmf_subsystem(args):
     if args.namespaces:
         namespaces = []
         for u in args.namespaces.strip().split(" "):
-            namespaces.append(u)
+            bdev_name = u
+            nsid = 0
+            if ':' in u:
+                (bdev_name, nsid) = u.split(":")
+
+            ns_params = {'bdev_name': bdev_name}
+
+            nsid = int(nsid)
+            if nsid != 0:
+                ns_params['nsid'] = nsid
+
+            namespaces.append(ns_params)
         params['namespaces'] = namespaces
 
     jsonrpc_call('construct_nvmf_subsystem', params)
@@ -444,9 +455,9 @@ p.add_argument("-a", "--allow-any-host", action='store_true', help="Allow any ho
 p.add_argument("-s", "--serial_number", help="""
 Format:  'sn' etc
 Example: 'SPDK00000000000001'""", default='0000:00:01.0')
-p.add_argument("-n", "--namespaces", help="""Whitespace-separated list of namespaces.
+p.add_argument("-n", "--namespaces", help="""Whitespace-separated list of namespaces
-Format:  'dev1 dev2 dev3' etc
+Format:  'bdev_name1[:nsid1] bdev_name2[:nsid2] bdev_name3[:nsid3]' etc
-Example: 'Malloc0 Malloc1 Malloc2'
+Example: '1:Malloc0 2:Malloc1 3:Malloc2'
 *** The devices must pre-exist ***""")
 p.set_defaults(func=construct_nvmf_subsystem)