From b940247b3e9b8f0b5a00f64c3a9bc4111a2f06ec Mon Sep 17 00:00:00 2001 From: Ziye Yang Date: Fri, 23 Apr 2021 20:30:43 +0800 Subject: [PATCH] bdev/rbd: Use the same Rados cluster object in the same RBD bdev This patch is design to use the single Rados Cluster object in the same RBD if there are multiple I/O channels created. And this patch will be prepared for the next patch to share the same cluster among different RBD bdevs. Change-Id: I1509f29a9c1088da308a3f88980f0c7fed26476f Signed-off-by: Ziye Yang Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/7601 Community-CI: Broadcom CI Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Changpeng Liu Reviewed-by: Shuhei Matsumoto --- module/bdev/rbd/bdev_rbd.c | 78 +++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 30 deletions(-) diff --git a/module/bdev/rbd/bdev_rbd.c b/module/bdev/rbd/bdev_rbd.c index 26adc96ef6..5a334b2558 100644 --- a/module/bdev/rbd/bdev_rbd.c +++ b/module/bdev/rbd/bdev_rbd.c @@ -62,6 +62,7 @@ struct bdev_rbd { char *user_id; char *pool_name; char **config; + rados_t cluster; rbd_image_info_t info; TAILQ_ENTRY(bdev_rbd) tailq; struct spdk_poller *reset_timer; @@ -75,7 +76,6 @@ struct bdev_rbd_group_channel { struct bdev_rbd_io_channel { rados_ioctx_t io_ctx; - rados_t cluster; int pfd; rbd_image_t image; struct bdev_rbd *disk; @@ -93,6 +93,10 @@ bdev_rbd_free(struct bdev_rbd *rbd) return; } + if (rbd->cluster) { + rados_shutdown(rbd->cluster); + } + free(rbd->disk.name); free(rbd->rbd_name); free(rbd->user_id); @@ -138,8 +142,8 @@ bdev_rbd_dup_config(const char *const *config) } static int -bdev_rados_context_init(const char *user_id, const char *rbd_pool_name, const char *const *config, - rados_t *cluster, rados_ioctx_t *io_ctx) +bdev_rados_cluster_init(const char *user_id, const char *const *config, + rados_t *cluster) { int ret; @@ -176,30 +180,44 @@ bdev_rados_context_init(const char *user_id, const char *rbd_pool_name, const ch return -1; } - ret = rados_ioctx_create(*cluster, rbd_pool_name, io_ctx); + return 0; +} - if (ret < 0) { - SPDK_ERRLOG("Failed to create ioctx\n"); - rados_shutdown(*cluster); - return -1; +static void * +bdev_rbd_cluster_handle(void *arg) +{ + struct bdev_rbd *rbd = arg; + void *ret = arg; + int rc; + + rc = bdev_rados_cluster_init(rbd->user_id, (const char *const *)rbd->config, + &rbd->cluster); + if (rc < 0) { + SPDK_ERRLOG("Failed to create rados cluster for user_id=%s and rbd_pool=%s\n", + rbd->user_id ? rbd->user_id : "admin (the default)", rbd->pool_name); + ret = NULL; } - return 0; + return ret; } static int bdev_rbd_init(struct bdev_rbd *rbd) { int ret = 0; - rados_t cluster = NULL; rados_ioctx_t io_ctx = NULL; rbd_image_t image = NULL; - ret = bdev_rados_context_init(rbd->user_id, rbd->pool_name, (const char *const *)rbd->config, - &cluster, &io_ctx); + /* Cluster should be created in non-SPDK thread to avoid conflict between + * Rados and SPDK thread */ + if (spdk_call_unaffinitized(bdev_rbd_cluster_handle, rbd) == NULL) { + SPDK_ERRLOG("Cannot create the rados object on rbd=%p\n", rbd); + return -1; + } + + ret = rados_ioctx_create(rbd->cluster, rbd->pool_name, &io_ctx); if (ret < 0) { - SPDK_ERRLOG("Failed to create rados context for user_id=%s and rbd_pool=%s\n", - rbd->user_id ? rbd->user_id : "admin (the default)", rbd->pool_name); + SPDK_ERRLOG("Failed to create ioctx\n"); return -1; } @@ -216,7 +234,6 @@ bdev_rbd_init(struct bdev_rbd *rbd) end: rados_ioctx_destroy(io_ctx); - rados_shutdown(cluster); return ret; } @@ -325,14 +342,23 @@ bdev_rbd_reset(struct bdev_rbd *disk, struct spdk_bdev_io *bdev_io) disk->reset_timer = SPDK_POLLER_REGISTER(bdev_rbd_reset_timer, disk, 1 * 1000 * 1000); } +static void +bdev_rbd_free_cb(void *io_device) +{ + struct bdev_rbd *rbd = io_device; + + assert(rbd != NULL); + + bdev_rbd_free((struct bdev_rbd *)rbd); +} + static int bdev_rbd_destruct(void *ctx) { struct bdev_rbd *rbd = ctx; - spdk_io_device_unregister(rbd, NULL); + spdk_io_device_unregister(rbd, bdev_rbd_free_cb); - bdev_rbd_free(rbd); return 0; } @@ -447,10 +473,6 @@ bdev_rbd_free_channel(struct bdev_rbd_io_channel *ch) rados_ioctx_destroy(ch->io_ctx); } - if (ch->cluster) { - rados_shutdown(ch->cluster); - } - if (ch->pfd >= 0) { close(ch->pfd); } @@ -465,16 +487,13 @@ bdev_rbd_handle(void *arg) { struct bdev_rbd_io_channel *ch = arg; void *ret = arg; - int rc; - rc = bdev_rados_context_init(ch->disk->user_id, ch->disk->pool_name, - (const char *const *)ch->disk->config, - &ch->cluster, &ch->io_ctx); - if (rc < 0) { - SPDK_ERRLOG("Failed to create rados context for user_id %s and rbd_pool=%s\n", - ch->disk->user_id ? ch->disk->user_id : "admin (the default)", ch->disk->pool_name); + assert(ch->disk->cluster != NULL); + + if (rados_ioctx_create(ch->disk->cluster, ch->disk->pool_name, &ch->io_ctx) < 0) { + SPDK_ERRLOG("Failed to create ioctx\n"); ret = NULL; - goto end; + return ret; } if (rbd_open(ch->io_ctx, ch->disk->rbd_name, &ch->image, NULL) < 0) { @@ -482,7 +501,6 @@ bdev_rbd_handle(void *arg) ret = NULL; } -end: return ret; }