lib/nvme: add RPC for interacting with CUSE devices

Added RPC commands to register/unregister CUSE devices
to NVMe controllers:

 - bdev_nvme_cuse_register
 - bdev_nvme_cuse_unregister

Additionally two RPC now return CUSE device names:

 - bdev_get_bdevs for namespaces
 - bdev_nvme_get_controllers for controllers


Signed-off-by: Maciej Szwed <maciej.szwed@intel.com>
Change-Id: I69c4bf41ec8f78a7522894268a67dd733881712f
Signed-off-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Signed-off-by: Tomasz Kulasek <tomaszx.kulasek@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/472211
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

include/spdk/nvme.h

@@ -2603,6 +2603,9 @@ char *spdk_nvme_cuse_get_ctrlr_name(struct spdk_nvme_ctrlr *ctrlr);
  */
 char *spdk_nvme_cuse_get_ns_name(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid);
 
+int spdk_nvme_cuse_register(struct spdk_nvme_ctrlr *ctrlr, const char *dev_path);
+void spdk_nvme_cuse_unregister(struct spdk_nvme_ctrlr *ctrlr);
+
 #ifdef __cplusplus
 }
 #endif

lib/nvme/nvme_cuse.c

@@ -48,7 +48,6 @@ struct cuse_device {
 	struct spdk_nvme_ctrlr		*ctrlr;		/**< NVMe controller */
 	uint32_t			nsid;		/**< NVMe name space id, or 0 */
 
-	uint32_t			idx;
 	pthread_t			tid;
 	struct fuse_session		*session;
 
@@ -59,7 +58,6 @@ struct cuse_device {
 };
 
 static TAILQ_HEAD(, cuse_device) g_ctrlr_ctx_head = TAILQ_HEAD_INITIALIZER(g_ctrlr_ctx_head);
-static int g_controllers_found = 0;
 
 struct cuse_io_ctx {
 	struct spdk_nvme_cmd	nvme_cmd;
@@ -660,7 +658,7 @@ end:
  */
 
 static int
-cuse_nvme_ns_start(struct cuse_device *ctrlr_device, uint32_t nsid)
+cuse_nvme_ns_start(struct cuse_device *ctrlr_device, uint32_t nsid, const char *dev_path)
 {
 	struct cuse_device *ns_device;
 
@@ -672,10 +670,9 @@ cuse_nvme_ns_start(struct cuse_device *ctrlr_device, uint32_t nsid)
 
 	ns_device->ctrlr = ctrlr_device->ctrlr;
 	ns_device->ctrlr_device = ctrlr_device;
-	ns_device->idx = nsid;
 	ns_device->nsid = nsid;
-	snprintf(ns_device->dev_name, sizeof(ns_device->dev_name), "spdk/nvme%dn%d",
-		 ctrlr_device->idx, ns_device->idx);
+	snprintf(ns_device->dev_name, sizeof(ns_device->dev_name), "%sn%d",
+		 dev_path, ns_device->nsid);
 
 	if (pthread_create(&ns_device->tid, NULL, cuse_thread, ns_device)) {
 		SPDK_ERRLOG("pthread_create failed\n");
@@ -708,7 +705,7 @@ cuse_nvme_ctrlr_stop(struct cuse_device *ctrlr_device)
 }
 
 static int
-nvme_cuse_start(struct spdk_nvme_ctrlr *ctrlr)
+nvme_cuse_start(struct spdk_nvme_ctrlr *ctrlr, const char *dev_path)
 {
 	uint32_t i, nsid;
 	struct cuse_device *ctrlr_device;
@@ -723,9 +720,7 @@ nvme_cuse_start(struct spdk_nvme_ctrlr *ctrlr)
 
 	TAILQ_INIT(&ctrlr_device->ns_devices);
 	ctrlr_device->ctrlr = ctrlr;
-	ctrlr_device->idx = g_controllers_found++;
-	snprintf(ctrlr_device->dev_name, sizeof(ctrlr_device->dev_name),
-		 "spdk/nvme%d", ctrlr_device->idx);
+	snprintf(ctrlr_device->dev_name, sizeof(ctrlr_device->dev_name), "%s", dev_path);
 
 	if (pthread_create(&ctrlr_device->tid, NULL, cuse_thread, ctrlr_device)) {
 		SPDK_ERRLOG("pthread_create failed\n");
@@ -741,7 +736,7 @@ nvme_cuse_start(struct spdk_nvme_ctrlr *ctrlr)
 			continue;
 		}
 
-		if (cuse_nvme_ns_start(ctrlr_device, nsid) < 0) {
+		if (cuse_nvme_ns_start(ctrlr_device, nsid, dev_path) < 0) {
 			SPDK_ERRLOG("Cannot start CUSE namespace device.");
 			cuse_nvme_ctrlr_stop(ctrlr_device);
 			return -1;
@@ -775,16 +770,20 @@ static struct nvme_io_msg_producer cuse_nvme_io_msg_producer = {
 };
 
 int
-nvme_cuse_register(struct spdk_nvme_ctrlr *ctrlr)
+spdk_nvme_cuse_register(struct spdk_nvme_ctrlr *ctrlr, const char *dev_path)
 {
 	int rc;
 
+	if (dev_path == NULL) {
+		return -EINVAL;
+	}
+
 	rc = nvme_io_msg_ctrlr_start(ctrlr, &cuse_nvme_io_msg_producer);
 	if (rc) {
 		return rc;
 	}
 
-	rc = nvme_cuse_start(ctrlr);
+	rc = nvme_cuse_start(ctrlr, dev_path);
 	if (rc) {
 		nvme_io_msg_ctrlr_stop(ctrlr, &cuse_nvme_io_msg_producer, false);
 	}
@@ -793,7 +792,7 @@ nvme_cuse_register(struct spdk_nvme_ctrlr *ctrlr)
 }
 
 void
-nvme_cuse_unregister(struct spdk_nvme_ctrlr *ctrlr)
+spdk_nvme_cuse_unregister(struct spdk_nvme_ctrlr *ctrlr)
 {
 	nvme_cuse_stop(ctrlr);
 

lib/nvme/nvme_cuse.h

@@ -36,7 +36,7 @@
 
 #include "spdk/nvme.h"
 
-int nvme_cuse_register(struct spdk_nvme_ctrlr *ctrlr);
+int nvme_cuse_register(struct spdk_nvme_ctrlr *ctrlr, const char *dev_path);
 void nvme_cuse_unregister(struct spdk_nvme_ctrlr *ctrlr);
 
 #endif /* __NVME_CUSE_H__ */

module/bdev/nvme/Makefile

@@ -35,6 +35,8 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
 C_SRCS = bdev_nvme.c bdev_nvme_rpc.c nvme_rpc.c common.c
+C_SRCS-$(CONFIG_NVME_CUSE) += bdev_nvme_cuse_rpc.c
+
 ifeq ($(OS),Linux)
 C_SRCS += bdev_ftl.c bdev_ftl_rpc.c vbdev_opal.c vbdev_opal_rpc.c
 endif

module/bdev/nvme/bdev_nvme.c

@@ -615,6 +615,16 @@ bdev_nvme_dump_info_json(void *ctx, struct spdk_json_write_ctx *w)
 
 	spdk_json_write_object_end(w);
 
+#ifdef SPDK_CONFIG_NVME_CUSE
+	char *cuse_device;
+
+	cuse_device = spdk_nvme_cuse_get_ns_name(nvme_bdev->nvme_bdev_ctrlr->ctrlr,
+			spdk_nvme_ns_get_id(ns));
+	if (cuse_device) {
+		spdk_json_write_named_string(w, "cuse_device", cuse_device);
+	}
+#endif
+
 	spdk_json_write_named_object_begin(w, "ctrlr_data");
 
 	spdk_json_write_named_string_fmt(w, "vendor_id", "0x%04x", cdata->vid);

module/bdev/nvme/bdev_nvme_cuse_rpc.c

@@ -0,0 +1,149 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "bdev_nvme.h"
+
+#include "spdk/string.h"
+#include "spdk/rpc.h"
+#include "spdk/util.h"
+#include "spdk/nvme.h"
+
+#include "spdk_internal/log.h"
+
+struct rpc_nvme_cuse_register {
+	char *name;
+	char *dev_path;
+};
+
+static void
+free_rpc_nvme_cuse_register(struct rpc_nvme_cuse_register *req)
+{
+	free(req->name);
+	free(req->dev_path);
+}
+
+static const struct spdk_json_object_decoder rpc_nvme_cuse_register_decoders[] = {
+	{"name", offsetof(struct rpc_nvme_cuse_register, name), spdk_json_decode_string},
+	{"dev_path", offsetof(struct rpc_nvme_cuse_register, dev_path), spdk_json_decode_string},
+};
+
+static void
+spdk_rpc_nvme_cuse_register(struct spdk_jsonrpc_request *request,
+			    const struct spdk_json_val *params)
+{
+	struct rpc_nvme_cuse_register req = {};
+	struct spdk_json_write_ctx *w;
+	struct nvme_bdev_ctrlr *bdev_ctrlr = NULL;
+	int rc;
+
+	if (spdk_json_decode_object(params, rpc_nvme_cuse_register_decoders,
+				    SPDK_COUNTOF(rpc_nvme_cuse_register_decoders),
+				    &req)) {
+		SPDK_ERRLOG("spdk_json_decode_object failed\n");
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
+						 "spdk_json_decode_object failed");
+		goto cleanup;
+	}
+
+	bdev_ctrlr = nvme_bdev_ctrlr_get_by_name(req.name);
+	if (!bdev_ctrlr) {
+		SPDK_ERRLOG("No such controller\n");
+		spdk_jsonrpc_send_error_response(request, -ENODEV, spdk_strerror(ENODEV));
+		goto cleanup;
+	}
+
+	rc = spdk_nvme_cuse_register(bdev_ctrlr->ctrlr, req.dev_path);
+	if (rc) {
+		SPDK_ERRLOG("Failed to register CUSE devices\n");
+		spdk_jsonrpc_send_error_response(request, -rc, spdk_strerror(rc));
+	}
+
+	w = spdk_jsonrpc_begin_result(request);
+	spdk_json_write_bool(w, true);
+	spdk_jsonrpc_end_result(request, w);
+
+cleanup:
+	free_rpc_nvme_cuse_register(&req);
+}
+SPDK_RPC_REGISTER("bdev_nvme_cuse_register", spdk_rpc_nvme_cuse_register, SPDK_RPC_RUNTIME)
+
+struct rpc_nvme_cuse_unregister {
+	char *name;
+};
+
+static void
+free_rpc_nvme_cuse_unregister(struct rpc_nvme_cuse_unregister *req)
+{
+	free(req->name);
+}
+
+static const struct spdk_json_object_decoder rpc_nvme_cuse_unregister_decoders[] = {
+	{"name", offsetof(struct rpc_nvme_cuse_unregister, name), spdk_json_decode_string, true},
+};
+
+static void
+spdk_rpc_nvme_cuse_unregister(struct spdk_jsonrpc_request *request,
+			      const struct spdk_json_val *params)
+{
+	struct rpc_nvme_cuse_unregister req = {};
+	struct spdk_json_write_ctx *w;
+	struct nvme_bdev_ctrlr *bdev_ctrlr = NULL;
+
+	if (spdk_json_decode_object(params, rpc_nvme_cuse_unregister_decoders,
+				    SPDK_COUNTOF(rpc_nvme_cuse_unregister_decoders),
+				    &req)) {
+		SPDK_ERRLOG("spdk_json_decode_object failed\n");
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
+						 "spdk_json_decode_object failed");
+		goto cleanup;
+	}
+
+	bdev_ctrlr = nvme_bdev_ctrlr_get_by_name(req.name);
+	if (!bdev_ctrlr) {
+		SPDK_ERRLOG("No such controller\n");
+		spdk_jsonrpc_send_error_response(request, -ENODEV, spdk_strerror(ENODEV));
+		goto cleanup;
+	}
+
+	spdk_nvme_cuse_unregister(bdev_ctrlr->ctrlr);
+
+	w = spdk_jsonrpc_begin_result(request);
+	spdk_json_write_bool(w, true);
+	spdk_jsonrpc_end_result(request, w);
+
+cleanup:
+	free_rpc_nvme_cuse_unregister(&req);
+}
+SPDK_RPC_REGISTER("bdev_nvme_cuse_unregister", spdk_rpc_nvme_cuse_unregister, SPDK_RPC_RUNTIME)

module/bdev/nvme/bdev_nvme_rpc.c

@@ -36,6 +36,8 @@
 #include "bdev_nvme.h"
 #include "common.h"
 
+#include "spdk/config.h"
+
 #include "spdk/string.h"
 #include "spdk/rpc.h"
 #include "spdk/util.h"
@@ -345,6 +347,15 @@ spdk_rpc_dump_nvme_controller_info(struct spdk_json_write_ctx *w,
 	spdk_json_write_object_begin(w);
 	spdk_json_write_named_string(w, "name", nvme_bdev_ctrlr->name);
 
+#ifdef SPDK_CONFIG_NVME_CUSE
+	char *cuse_device;
+
+	cuse_device = spdk_nvme_cuse_get_ctrlr_name(nvme_bdev_ctrlr->ctrlr);
+	if (cuse_device) {
+		spdk_json_write_named_string(w, "cuse_device", cuse_device);
+	}
+#endif
+
 	spdk_json_write_named_object_begin(w, "trid");
 	nvme_bdev_dump_trid_json(trid, w);
 	spdk_json_write_object_end(w);

scripts/rpc.py

@@ -448,6 +448,29 @@ if __name__ == "__main__":
     p.add_argument('name', help="Name of the controller")
     p.set_defaults(func=bdev_nvme_detach_controller)
 
+    def bdev_nvme_cuse_register(args):
+        rpc.bdev.bdev_nvme_cuse_register(args.client,
+                                         name=args.name,
+                                         dev_path=args.dev_path)
+
+    p = subparsers.add_parser('bdev_nvme_cuse_register',
+                              help='Register CUSE devices on NVMe controller')
+    p.add_argument('-n', '--name',
+                   help='Name of the NVMe controller. Example: Nvme0', required=True)
+    p.add_argument('-p', '--dev_path',
+                   help='CUSE dev path including prefix: e.g. spdk/nvme0 will result: /dev/spdk/nvme0n1', required=True)
+    p.set_defaults(func=bdev_nvme_cuse_register)
+
+    def bdev_nvme_cuse_unregister(args):
+        rpc.bdev.bdev_nvme_cuse_unregister(args.client,
+                                           name=args.name)
+
+    p = subparsers.add_parser('bdev_nvme_cuse_unregister',
+                              help='Unregister CUSE devices on NVMe controller')
+    p.add_argument('-n', '--name',
+                   help='Name of the NVMe controller. Example: Nvme0', required=True)
+    p.set_defaults(func=bdev_nvme_cuse_unregister)
+
     def bdev_rbd_create(args):
         config = None
         if args.config:

scripts/rpc/bdev.py

@@ -480,6 +480,30 @@ def bdev_nvme_detach_controller(client, name):
     return client.call('bdev_nvme_detach_controller', params)
 
 
+def bdev_nvme_cuse_register(client, name, dev_path):
+    """Register CUSE devices on NVMe controller.
+
+    Args:
+        name: Name of the operating NVMe controller
+        dev_path: CUSE dev path with dev prefix
+    """
+    params = {'name': name,
+              'dev_path': dev_path}
+
+    return client.call('bdev_nvme_cuse_register', params)
+
+
+def bdev_nvme_cuse_unregister(client, name):
+    """Unregister CUSE devices on NVMe controller.
+
+    Args:
+        name: Name of the operating NVMe controller
+    """
+    params = {'name': name}
+
+    return client.call('bdev_nvme_cuse_unregister', params)
+
+
 @deprecated_alias('construct_rbd_bdev')
 def bdev_rbd_create(client, pool_name, rbd_name, block_size, name=None, user=None, config=None):
     """Create a Ceph RBD block device.