From d5aeeaf2f761d563ed10a5b8275670edd588d642 Mon Sep 17 00:00:00 2001 From: Vitaliy Mysak Date: Wed, 3 Apr 2019 20:43:15 +0000 Subject: [PATCH] ocf: adopt examine to asynchronous construct Make examine work with asynchronous bdev startup. We need to count references of bdevs that are being examined, for the case when single cache bdev is referenced by multiple OCF vbdevs. The construct function is not in fact asynchronous yet, but will be after adopting "trylock" API. Adopting trylock requires a lot of changes, so this patch is really to split implementation. Signed-off-by: Vitaliy Mysak Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/450033 (master) (cherry picked from commit ac2b06c8096f686e502aa1db0a42ee29190d9bfd) Change-Id: I6ac091f2dd48462e74aa89cf54acb0306c30673b Signed-off-by: Tomasz Zawadzki Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/457207 Tested-by: SPDK CI Jenkins Reviewed-by: Darek Stojaczyk --- lib/bdev/ocf/vbdev_ocf.c | 132 ++++++++++++++++++++++++++++++++------- 1 file changed, 108 insertions(+), 24 deletions(-) diff --git a/lib/bdev/ocf/vbdev_ocf.c b/lib/bdev/ocf/vbdev_ocf.c index 1f817b6ce5..f678a5df87 100644 --- a/lib/bdev/ocf/vbdev_ocf.c +++ b/lib/bdev/ocf/vbdev_ocf.c @@ -53,6 +53,52 @@ static struct spdk_bdev_module ocf_if; static TAILQ_HEAD(, vbdev_ocf) g_ocf_vbdev_head = TAILQ_HEAD_INITIALIZER(g_ocf_vbdev_head); +static TAILQ_HEAD(, examining_bdev) g_ocf_examining_bdevs_head + = TAILQ_HEAD_INITIALIZER(g_ocf_examining_bdevs_head); + +/* Structure for keeping list of bdevs that are claimed but not used yet */ +struct examining_bdev { + struct spdk_bdev *bdev; + TAILQ_ENTRY(examining_bdev) tailq; +}; + +/* Add bdev to list of claimed */ +static void +examine_start(struct spdk_bdev *bdev) +{ + struct examining_bdev *entry = malloc(sizeof(*entry)); + + assert(entry); + entry->bdev = bdev; + TAILQ_INSERT_TAIL(&g_ocf_examining_bdevs_head, entry, tailq); +} + +/* Find bdev on list of claimed bdevs, then remove it, + * if it was the last one on list then report examine done */ +static void +examine_done(int status, struct vbdev_ocf *vbdev, void *cb_arg) +{ + struct spdk_bdev *bdev = cb_arg; + struct examining_bdev *entry, *safe, *found = NULL; + + TAILQ_FOREACH_SAFE(entry, &g_ocf_examining_bdevs_head, tailq, safe) { + if (entry->bdev == bdev) { + if (found) { + goto remove; + } else { + found = entry; + } + } + } + + assert(found); + spdk_bdev_module_examine_done(&ocf_if); + +remove: + TAILQ_REMOVE(&g_ocf_examining_bdevs_head, found, tailq); + free(found); +} + /* Free allocated strings and structure itself * Used at shutdown only */ static void @@ -731,25 +777,26 @@ io_device_destroy_cb(void *io_device, void *ctx_buf) } /* Start OCF cache and register vbdev_ocf at bdev layer */ -static int -register_vbdev(struct vbdev_ocf *vbdev) +static void +register_vbdev(struct vbdev_ocf *vbdev, void (*cb)(int, struct vbdev_ocf *, void *), void *cb_arg) { int result; - if (!vbdev->cache.attached || !vbdev->core.attached) { - return -EPERM; + if (!vbdev->cache.attached || !vbdev->core.attached || vbdev->state.started) { + result = -EPERM; + goto callback; } result = start_cache(vbdev); if (result) { SPDK_ERRLOG("Failed to start cache instance\n"); - return result; + goto callback; } result = add_core(vbdev); if (result) { SPDK_ERRLOG("Failed to add core to cache instance\n"); - return result; + goto callback; } /* Create exported spdk object */ @@ -773,12 +820,13 @@ register_vbdev(struct vbdev_ocf *vbdev) result = spdk_bdev_register(&vbdev->exp_bdev); if (result) { SPDK_ERRLOG("Could not register exposed bdev\n"); - return result; + goto callback; } vbdev->state.started = true; - return result; +callback: + cb(result, vbdev, cb_arg); } /* Init OCF configuration options @@ -1042,12 +1090,11 @@ attach_base(struct vbdev_ocf_base *base) return status; } -/* Attach base bdevs - * If they attached, start vbdev - * otherwise wait for them to appear at examine */ +/* Attach base bdevs */ static int -create_from_bdevs(struct vbdev_ocf *vbdev, - struct spdk_bdev *cache_bdev, struct spdk_bdev *core_bdev) +attach_base_bdevs(struct vbdev_ocf *vbdev, + struct spdk_bdev *cache_bdev, + struct spdk_bdev *core_bdev) { int rc = 0; @@ -1061,10 +1108,6 @@ create_from_bdevs(struct vbdev_ocf *vbdev, rc |= attach_base(&vbdev->core); } - if (rc == 0 && vbdev->core.attached && vbdev->cache.attached) { - rc = register_vbdev(vbdev); - } - return rc; } @@ -1103,14 +1146,22 @@ vbdev_ocf_construct(const char *vbdev_name, vbdev->name, core_name); } - rc = create_from_bdevs(vbdev, cache_bdev, core_bdev); - cb(rc, vbdev, cb_arg); + rc = attach_base_bdevs(vbdev, cache_bdev, core_bdev); + if (rc) { + cb(rc, vbdev, cb_arg); + return; + } + + if (core_bdev && cache_bdev) { + register_vbdev(vbdev, cb, cb_arg); + } else { + cb(0, vbdev, cb_arg); + } } /* This called if new device is created in SPDK application - * If that device named as one of base bdevs of cache_vbdev, - * attach them - * If last device attached here, vbdev starts here */ + * If that device named as one of base bdevs of OCF vbdev, + * claim and open them */ static void vbdev_ocf_examine(struct spdk_bdev *bdev) { @@ -1123,17 +1174,49 @@ vbdev_ocf_examine(struct spdk_bdev *bdev) } if (!strcmp(bdev_name, vbdev->cache.name)) { - create_from_bdevs(vbdev, bdev, NULL); + attach_base_bdevs(vbdev, bdev, NULL); continue; } if (!strcmp(bdev_name, vbdev->core.name)) { - create_from_bdevs(vbdev, NULL, bdev); + attach_base_bdevs(vbdev, NULL, bdev); break; } } spdk_bdev_module_examine_done(&ocf_if); } +/* This is called after vbdev_ocf_examine + * It allows to delay application initialization + * until all OCF bdevs get registered + * If vbdev has all of its base devices it starts asynchronously here */ +static void +vbdev_ocf_examine_disk(struct spdk_bdev *bdev) +{ + const char *bdev_name = spdk_bdev_get_name(bdev); + struct vbdev_ocf *vbdev; + + examine_start(bdev); + + TAILQ_FOREACH(vbdev, &g_ocf_vbdev_head, tailq) { + if (vbdev->state.doing_finish || vbdev->state.started) { + continue; + } + + if (!strcmp(bdev_name, vbdev->cache.name)) { + examine_start(bdev); + register_vbdev(vbdev, examine_done, bdev); + continue; + } + if (!strcmp(bdev_name, vbdev->core.name)) { + examine_start(bdev); + register_vbdev(vbdev, examine_done, bdev); + break; + } + } + + examine_done(0, NULL, bdev); +} + static int vbdev_ocf_get_ctx_size(void) { @@ -1150,6 +1233,7 @@ static struct spdk_bdev_module ocf_if = { .config_text = NULL, .get_ctx_size = vbdev_ocf_get_ctx_size, .examine_config = vbdev_ocf_examine, + .examine_disk = vbdev_ocf_examine_disk, }; SPDK_BDEV_MODULE_REGISTER(ocf, &ocf_if);