bdev/rbd: Use the same Rados cluster object in the same RBD bdev

This patch is design to use the single Rados Cluster
object in the same RBD if there are multiple I/O channels created.

And this patch will be prepared for the next patch to share
the same cluster among different RBD bdevs.

Change-Id: I1509f29a9c1088da308a3f88980f0c7fed26476f
Signed-off-by: Ziye Yang <ziye.yang@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/7601
Community-CI: Broadcom CI
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
This commit is contained in:
Ziye Yang 2021-04-23 20:30:43 +08:00 committed by Tomasz Zawadzki
parent 67eb36c7fa
commit b940247b3e

View File

@ -62,6 +62,7 @@ struct bdev_rbd {
char *user_id;
char *pool_name;
char **config;
rados_t cluster;
rbd_image_info_t info;
TAILQ_ENTRY(bdev_rbd) tailq;
struct spdk_poller *reset_timer;
@ -75,7 +76,6 @@ struct bdev_rbd_group_channel {
struct bdev_rbd_io_channel {
rados_ioctx_t io_ctx;
rados_t cluster;
int pfd;
rbd_image_t image;
struct bdev_rbd *disk;
@ -93,6 +93,10 @@ bdev_rbd_free(struct bdev_rbd *rbd)
return;
}
if (rbd->cluster) {
rados_shutdown(rbd->cluster);
}
free(rbd->disk.name);
free(rbd->rbd_name);
free(rbd->user_id);
@ -138,8 +142,8 @@ bdev_rbd_dup_config(const char *const *config)
}
static int
bdev_rados_context_init(const char *user_id, const char *rbd_pool_name, const char *const *config,
rados_t *cluster, rados_ioctx_t *io_ctx)
bdev_rados_cluster_init(const char *user_id, const char *const *config,
rados_t *cluster)
{
int ret;
@ -176,30 +180,44 @@ bdev_rados_context_init(const char *user_id, const char *rbd_pool_name, const ch
return -1;
}
ret = rados_ioctx_create(*cluster, rbd_pool_name, io_ctx);
return 0;
}
if (ret < 0) {
SPDK_ERRLOG("Failed to create ioctx\n");
rados_shutdown(*cluster);
return -1;
static void *
bdev_rbd_cluster_handle(void *arg)
{
struct bdev_rbd *rbd = arg;
void *ret = arg;
int rc;
rc = bdev_rados_cluster_init(rbd->user_id, (const char *const *)rbd->config,
&rbd->cluster);
if (rc < 0) {
SPDK_ERRLOG("Failed to create rados cluster for user_id=%s and rbd_pool=%s\n",
rbd->user_id ? rbd->user_id : "admin (the default)", rbd->pool_name);
ret = NULL;
}
return 0;
return ret;
}
static int
bdev_rbd_init(struct bdev_rbd *rbd)
{
int ret = 0;
rados_t cluster = NULL;
rados_ioctx_t io_ctx = NULL;
rbd_image_t image = NULL;
ret = bdev_rados_context_init(rbd->user_id, rbd->pool_name, (const char *const *)rbd->config,
&cluster, &io_ctx);
/* Cluster should be created in non-SPDK thread to avoid conflict between
* Rados and SPDK thread */
if (spdk_call_unaffinitized(bdev_rbd_cluster_handle, rbd) == NULL) {
SPDK_ERRLOG("Cannot create the rados object on rbd=%p\n", rbd);
return -1;
}
ret = rados_ioctx_create(rbd->cluster, rbd->pool_name, &io_ctx);
if (ret < 0) {
SPDK_ERRLOG("Failed to create rados context for user_id=%s and rbd_pool=%s\n",
rbd->user_id ? rbd->user_id : "admin (the default)", rbd->pool_name);
SPDK_ERRLOG("Failed to create ioctx\n");
return -1;
}
@ -216,7 +234,6 @@ bdev_rbd_init(struct bdev_rbd *rbd)
end:
rados_ioctx_destroy(io_ctx);
rados_shutdown(cluster);
return ret;
}
@ -325,14 +342,23 @@ bdev_rbd_reset(struct bdev_rbd *disk, struct spdk_bdev_io *bdev_io)
disk->reset_timer = SPDK_POLLER_REGISTER(bdev_rbd_reset_timer, disk, 1 * 1000 * 1000);
}
static void
bdev_rbd_free_cb(void *io_device)
{
struct bdev_rbd *rbd = io_device;
assert(rbd != NULL);
bdev_rbd_free((struct bdev_rbd *)rbd);
}
static int
bdev_rbd_destruct(void *ctx)
{
struct bdev_rbd *rbd = ctx;
spdk_io_device_unregister(rbd, NULL);
spdk_io_device_unregister(rbd, bdev_rbd_free_cb);
bdev_rbd_free(rbd);
return 0;
}
@ -447,10 +473,6 @@ bdev_rbd_free_channel(struct bdev_rbd_io_channel *ch)
rados_ioctx_destroy(ch->io_ctx);
}
if (ch->cluster) {
rados_shutdown(ch->cluster);
}
if (ch->pfd >= 0) {
close(ch->pfd);
}
@ -465,16 +487,13 @@ bdev_rbd_handle(void *arg)
{
struct bdev_rbd_io_channel *ch = arg;
void *ret = arg;
int rc;
rc = bdev_rados_context_init(ch->disk->user_id, ch->disk->pool_name,
(const char *const *)ch->disk->config,
&ch->cluster, &ch->io_ctx);
if (rc < 0) {
SPDK_ERRLOG("Failed to create rados context for user_id %s and rbd_pool=%s\n",
ch->disk->user_id ? ch->disk->user_id : "admin (the default)", ch->disk->pool_name);
assert(ch->disk->cluster != NULL);
if (rados_ioctx_create(ch->disk->cluster, ch->disk->pool_name, &ch->io_ctx) < 0) {
SPDK_ERRLOG("Failed to create ioctx\n");
ret = NULL;
goto end;
return ret;
}
if (rbd_open(ch->io_ctx, ch->disk->rbd_name, &ch->image, NULL) < 0) {
@ -482,7 +501,6 @@ bdev_rbd_handle(void *arg)
ret = NULL;
}
end:
return ret;
}