diff --git a/include/spdk/nvme.h b/include/spdk/nvme.h index 4110bb6a34..13982ae144 100644 --- a/include/spdk/nvme.h +++ b/include/spdk/nvme.h @@ -2603,6 +2603,9 @@ char *spdk_nvme_cuse_get_ctrlr_name(struct spdk_nvme_ctrlr *ctrlr); */ char *spdk_nvme_cuse_get_ns_name(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid); +int spdk_nvme_cuse_register(struct spdk_nvme_ctrlr *ctrlr, const char *dev_path); +void spdk_nvme_cuse_unregister(struct spdk_nvme_ctrlr *ctrlr); + #ifdef __cplusplus } #endif diff --git a/lib/nvme/nvme_cuse.c b/lib/nvme/nvme_cuse.c index a7e85a2002..18d36bb251 100644 --- a/lib/nvme/nvme_cuse.c +++ b/lib/nvme/nvme_cuse.c @@ -48,7 +48,6 @@ struct cuse_device { struct spdk_nvme_ctrlr *ctrlr; /**< NVMe controller */ uint32_t nsid; /**< NVMe name space id, or 0 */ - uint32_t idx; pthread_t tid; struct fuse_session *session; @@ -59,7 +58,6 @@ struct cuse_device { }; static TAILQ_HEAD(, cuse_device) g_ctrlr_ctx_head = TAILQ_HEAD_INITIALIZER(g_ctrlr_ctx_head); -static int g_controllers_found = 0; struct cuse_io_ctx { struct spdk_nvme_cmd nvme_cmd; @@ -660,7 +658,7 @@ end: */ static int -cuse_nvme_ns_start(struct cuse_device *ctrlr_device, uint32_t nsid) +cuse_nvme_ns_start(struct cuse_device *ctrlr_device, uint32_t nsid, const char *dev_path) { struct cuse_device *ns_device; @@ -672,10 +670,9 @@ cuse_nvme_ns_start(struct cuse_device *ctrlr_device, uint32_t nsid) ns_device->ctrlr = ctrlr_device->ctrlr; ns_device->ctrlr_device = ctrlr_device; - ns_device->idx = nsid; ns_device->nsid = nsid; - snprintf(ns_device->dev_name, sizeof(ns_device->dev_name), "spdk/nvme%dn%d", - ctrlr_device->idx, ns_device->idx); + snprintf(ns_device->dev_name, sizeof(ns_device->dev_name), "%sn%d", + dev_path, ns_device->nsid); if (pthread_create(&ns_device->tid, NULL, cuse_thread, ns_device)) { SPDK_ERRLOG("pthread_create failed\n"); @@ -708,7 +705,7 @@ cuse_nvme_ctrlr_stop(struct cuse_device *ctrlr_device) } static int -nvme_cuse_start(struct spdk_nvme_ctrlr *ctrlr) +nvme_cuse_start(struct spdk_nvme_ctrlr *ctrlr, const char *dev_path) { uint32_t i, nsid; struct cuse_device *ctrlr_device; @@ -723,9 +720,7 @@ nvme_cuse_start(struct spdk_nvme_ctrlr *ctrlr) TAILQ_INIT(&ctrlr_device->ns_devices); ctrlr_device->ctrlr = ctrlr; - ctrlr_device->idx = g_controllers_found++; - snprintf(ctrlr_device->dev_name, sizeof(ctrlr_device->dev_name), - "spdk/nvme%d", ctrlr_device->idx); + snprintf(ctrlr_device->dev_name, sizeof(ctrlr_device->dev_name), "%s", dev_path); if (pthread_create(&ctrlr_device->tid, NULL, cuse_thread, ctrlr_device)) { SPDK_ERRLOG("pthread_create failed\n"); @@ -741,7 +736,7 @@ nvme_cuse_start(struct spdk_nvme_ctrlr *ctrlr) continue; } - if (cuse_nvme_ns_start(ctrlr_device, nsid) < 0) { + if (cuse_nvme_ns_start(ctrlr_device, nsid, dev_path) < 0) { SPDK_ERRLOG("Cannot start CUSE namespace device."); cuse_nvme_ctrlr_stop(ctrlr_device); return -1; @@ -775,16 +770,20 @@ static struct nvme_io_msg_producer cuse_nvme_io_msg_producer = { }; int -nvme_cuse_register(struct spdk_nvme_ctrlr *ctrlr) +spdk_nvme_cuse_register(struct spdk_nvme_ctrlr *ctrlr, const char *dev_path) { int rc; + if (dev_path == NULL) { + return -EINVAL; + } + rc = nvme_io_msg_ctrlr_start(ctrlr, &cuse_nvme_io_msg_producer); if (rc) { return rc; } - rc = nvme_cuse_start(ctrlr); + rc = nvme_cuse_start(ctrlr, dev_path); if (rc) { nvme_io_msg_ctrlr_stop(ctrlr, &cuse_nvme_io_msg_producer, false); } @@ -793,7 +792,7 @@ nvme_cuse_register(struct spdk_nvme_ctrlr *ctrlr) } void -nvme_cuse_unregister(struct spdk_nvme_ctrlr *ctrlr) +spdk_nvme_cuse_unregister(struct spdk_nvme_ctrlr *ctrlr) { nvme_cuse_stop(ctrlr); diff --git a/lib/nvme/nvme_cuse.h b/lib/nvme/nvme_cuse.h index 174431a887..92b4751908 100644 --- a/lib/nvme/nvme_cuse.h +++ b/lib/nvme/nvme_cuse.h @@ -36,7 +36,7 @@ #include "spdk/nvme.h" -int nvme_cuse_register(struct spdk_nvme_ctrlr *ctrlr); +int nvme_cuse_register(struct spdk_nvme_ctrlr *ctrlr, const char *dev_path); void nvme_cuse_unregister(struct spdk_nvme_ctrlr *ctrlr); #endif /* __NVME_CUSE_H__ */ diff --git a/module/bdev/nvme/Makefile b/module/bdev/nvme/Makefile index 536af9c1ba..7062775531 100644 --- a/module/bdev/nvme/Makefile +++ b/module/bdev/nvme/Makefile @@ -35,6 +35,8 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..) include $(SPDK_ROOT_DIR)/mk/spdk.common.mk C_SRCS = bdev_nvme.c bdev_nvme_rpc.c nvme_rpc.c common.c +C_SRCS-$(CONFIG_NVME_CUSE) += bdev_nvme_cuse_rpc.c + ifeq ($(OS),Linux) C_SRCS += bdev_ftl.c bdev_ftl_rpc.c vbdev_opal.c vbdev_opal_rpc.c endif diff --git a/module/bdev/nvme/bdev_nvme.c b/module/bdev/nvme/bdev_nvme.c index bc2c61b803..0accdeba1d 100644 --- a/module/bdev/nvme/bdev_nvme.c +++ b/module/bdev/nvme/bdev_nvme.c @@ -615,6 +615,16 @@ bdev_nvme_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) spdk_json_write_object_end(w); +#ifdef SPDK_CONFIG_NVME_CUSE + char *cuse_device; + + cuse_device = spdk_nvme_cuse_get_ns_name(nvme_bdev->nvme_bdev_ctrlr->ctrlr, + spdk_nvme_ns_get_id(ns)); + if (cuse_device) { + spdk_json_write_named_string(w, "cuse_device", cuse_device); + } +#endif + spdk_json_write_named_object_begin(w, "ctrlr_data"); spdk_json_write_named_string_fmt(w, "vendor_id", "0x%04x", cdata->vid); diff --git a/module/bdev/nvme/bdev_nvme_cuse_rpc.c b/module/bdev/nvme/bdev_nvme_cuse_rpc.c new file mode 100644 index 0000000000..ad61165095 --- /dev/null +++ b/module/bdev/nvme/bdev_nvme_cuse_rpc.c @@ -0,0 +1,149 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/stdinc.h" + +#include "bdev_nvme.h" + +#include "spdk/string.h" +#include "spdk/rpc.h" +#include "spdk/util.h" +#include "spdk/nvme.h" + +#include "spdk_internal/log.h" + +struct rpc_nvme_cuse_register { + char *name; + char *dev_path; +}; + +static void +free_rpc_nvme_cuse_register(struct rpc_nvme_cuse_register *req) +{ + free(req->name); + free(req->dev_path); +} + +static const struct spdk_json_object_decoder rpc_nvme_cuse_register_decoders[] = { + {"name", offsetof(struct rpc_nvme_cuse_register, name), spdk_json_decode_string}, + {"dev_path", offsetof(struct rpc_nvme_cuse_register, dev_path), spdk_json_decode_string}, +}; + +static void +spdk_rpc_nvme_cuse_register(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_nvme_cuse_register req = {}; + struct spdk_json_write_ctx *w; + struct nvme_bdev_ctrlr *bdev_ctrlr = NULL; + int rc; + + if (spdk_json_decode_object(params, rpc_nvme_cuse_register_decoders, + SPDK_COUNTOF(rpc_nvme_cuse_register_decoders), + &req)) { + SPDK_ERRLOG("spdk_json_decode_object failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "spdk_json_decode_object failed"); + goto cleanup; + } + + bdev_ctrlr = nvme_bdev_ctrlr_get_by_name(req.name); + if (!bdev_ctrlr) { + SPDK_ERRLOG("No such controller\n"); + spdk_jsonrpc_send_error_response(request, -ENODEV, spdk_strerror(ENODEV)); + goto cleanup; + } + + rc = spdk_nvme_cuse_register(bdev_ctrlr->ctrlr, req.dev_path); + if (rc) { + SPDK_ERRLOG("Failed to register CUSE devices\n"); + spdk_jsonrpc_send_error_response(request, -rc, spdk_strerror(rc)); + } + + w = spdk_jsonrpc_begin_result(request); + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(request, w); + +cleanup: + free_rpc_nvme_cuse_register(&req); +} +SPDK_RPC_REGISTER("bdev_nvme_cuse_register", spdk_rpc_nvme_cuse_register, SPDK_RPC_RUNTIME) + +struct rpc_nvme_cuse_unregister { + char *name; +}; + +static void +free_rpc_nvme_cuse_unregister(struct rpc_nvme_cuse_unregister *req) +{ + free(req->name); +} + +static const struct spdk_json_object_decoder rpc_nvme_cuse_unregister_decoders[] = { + {"name", offsetof(struct rpc_nvme_cuse_unregister, name), spdk_json_decode_string, true}, +}; + +static void +spdk_rpc_nvme_cuse_unregister(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_nvme_cuse_unregister req = {}; + struct spdk_json_write_ctx *w; + struct nvme_bdev_ctrlr *bdev_ctrlr = NULL; + + if (spdk_json_decode_object(params, rpc_nvme_cuse_unregister_decoders, + SPDK_COUNTOF(rpc_nvme_cuse_unregister_decoders), + &req)) { + SPDK_ERRLOG("spdk_json_decode_object failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "spdk_json_decode_object failed"); + goto cleanup; + } + + bdev_ctrlr = nvme_bdev_ctrlr_get_by_name(req.name); + if (!bdev_ctrlr) { + SPDK_ERRLOG("No such controller\n"); + spdk_jsonrpc_send_error_response(request, -ENODEV, spdk_strerror(ENODEV)); + goto cleanup; + } + + spdk_nvme_cuse_unregister(bdev_ctrlr->ctrlr); + + w = spdk_jsonrpc_begin_result(request); + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(request, w); + +cleanup: + free_rpc_nvme_cuse_unregister(&req); +} +SPDK_RPC_REGISTER("bdev_nvme_cuse_unregister", spdk_rpc_nvme_cuse_unregister, SPDK_RPC_RUNTIME) diff --git a/module/bdev/nvme/bdev_nvme_rpc.c b/module/bdev/nvme/bdev_nvme_rpc.c index 6a83b2b168..795dfbcfe4 100644 --- a/module/bdev/nvme/bdev_nvme_rpc.c +++ b/module/bdev/nvme/bdev_nvme_rpc.c @@ -36,6 +36,8 @@ #include "bdev_nvme.h" #include "common.h" +#include "spdk/config.h" + #include "spdk/string.h" #include "spdk/rpc.h" #include "spdk/util.h" @@ -345,6 +347,15 @@ spdk_rpc_dump_nvme_controller_info(struct spdk_json_write_ctx *w, spdk_json_write_object_begin(w); spdk_json_write_named_string(w, "name", nvme_bdev_ctrlr->name); +#ifdef SPDK_CONFIG_NVME_CUSE + char *cuse_device; + + cuse_device = spdk_nvme_cuse_get_ctrlr_name(nvme_bdev_ctrlr->ctrlr); + if (cuse_device) { + spdk_json_write_named_string(w, "cuse_device", cuse_device); + } +#endif + spdk_json_write_named_object_begin(w, "trid"); nvme_bdev_dump_trid_json(trid, w); spdk_json_write_object_end(w); diff --git a/scripts/rpc.py b/scripts/rpc.py index 160abb9e2b..be0a3b2dfb 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -448,6 +448,29 @@ if __name__ == "__main__": p.add_argument('name', help="Name of the controller") p.set_defaults(func=bdev_nvme_detach_controller) + def bdev_nvme_cuse_register(args): + rpc.bdev.bdev_nvme_cuse_register(args.client, + name=args.name, + dev_path=args.dev_path) + + p = subparsers.add_parser('bdev_nvme_cuse_register', + help='Register CUSE devices on NVMe controller') + p.add_argument('-n', '--name', + help='Name of the NVMe controller. Example: Nvme0', required=True) + p.add_argument('-p', '--dev_path', + help='CUSE dev path including prefix: e.g. spdk/nvme0 will result: /dev/spdk/nvme0n1', required=True) + p.set_defaults(func=bdev_nvme_cuse_register) + + def bdev_nvme_cuse_unregister(args): + rpc.bdev.bdev_nvme_cuse_unregister(args.client, + name=args.name) + + p = subparsers.add_parser('bdev_nvme_cuse_unregister', + help='Unregister CUSE devices on NVMe controller') + p.add_argument('-n', '--name', + help='Name of the NVMe controller. Example: Nvme0', required=True) + p.set_defaults(func=bdev_nvme_cuse_unregister) + def bdev_rbd_create(args): config = None if args.config: diff --git a/scripts/rpc/bdev.py b/scripts/rpc/bdev.py index d187ca11e1..e8e9a42c41 100644 --- a/scripts/rpc/bdev.py +++ b/scripts/rpc/bdev.py @@ -480,6 +480,30 @@ def bdev_nvme_detach_controller(client, name): return client.call('bdev_nvme_detach_controller', params) +def bdev_nvme_cuse_register(client, name, dev_path): + """Register CUSE devices on NVMe controller. + + Args: + name: Name of the operating NVMe controller + dev_path: CUSE dev path with dev prefix + """ + params = {'name': name, + 'dev_path': dev_path} + + return client.call('bdev_nvme_cuse_register', params) + + +def bdev_nvme_cuse_unregister(client, name): + """Unregister CUSE devices on NVMe controller. + + Args: + name: Name of the operating NVMe controller + """ + params = {'name': name} + + return client.call('bdev_nvme_cuse_unregister', params) + + @deprecated_alias('construct_rbd_bdev') def bdev_rbd_create(client, pool_name, rbd_name, block_size, name=None, user=None, config=None): """Create a Ceph RBD block device.