From 2fa51eeb46d1c5148a5ae77e4134c125929db3a5 Mon Sep 17 00:00:00 2001 From: Shuhei Matsumoto Date: Thu, 13 Feb 2020 05:03:16 -0500 Subject: [PATCH] lib/nvmf: Make spdk_nvmf_poll_group_destroy() asynchronous The next patch will create poll group threads dynamically for NVMe-oF target, and will need to wait for completion of poll group and I/O channel destroy. This is a preparation for the next patch. Add callback function and its argument to spdk_nvmf_poll_group_destroy(), and to struct spdk_nvmf_poll_group, respectively. The callback has not only cb_arg but also status as its parameters even if the next patch always sets the status to zero. The reason is to follow spdk_nvmf_tgt_destroy's callback and to process any case that the status is nonzero in future. spdk_nvmf_poll_group_destroy() sets the passed callback to the passed poll group. Then spdk_nvmf_tgt_destroy_poll_group() calls the held callback in the end. This change will ensure all pollers are being unregistered and all I/O channels are being released. Signed-off-by: Shuhei Matsumoto Change-Id: Ifb854066a5259a6029d55b88de358e3346c63f18 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/495 Tested-by: SPDK CI Jenkins Reviewed-by: Tomasz Zawadzki Reviewed-by: Ben Walker --- CHANGELOG.md | 3 +++ examples/nvmf/nvmf/nvmf.c | 2 +- include/spdk/nvmf.h | 8 +++++++- include/spdk/nvmf_transport.h | 3 +++ lib/nvmf/nvmf.c | 12 +++++++++++- module/event/subsystems/nvmf/nvmf_tgt.c | 2 +- test/unit/lib/nvmf/fc.c/fc_ut.c | 2 +- 7 files changed, 27 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 606d450b91..2a8c8dca68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ engine have been renamed to identify the engine as a software accelerator. A new function, `spdk_vmd_fini`, has been added. It releases all resources acquired by the VMD library through the `spdk_vmd_init` call. +### nvmf +`spdk_nvmf_poll_group_destroy()` is now asynchronous and accepts a completion callback. + ### Miscellaneous `--json-ignore-init-errors` command line param has been added to ignore initialization errors diff --git a/examples/nvmf/nvmf/nvmf.c b/examples/nvmf/nvmf/nvmf.c index 2ca5771780..1bde3e3ca3 100644 --- a/examples/nvmf/nvmf/nvmf.c +++ b/examples/nvmf/nvmf/nvmf.c @@ -663,7 +663,7 @@ nvmf_tgt_destroy_poll_group(struct spdk_io_channel_iter *i) TAILQ_FOREACH_SAFE(pg, &g_poll_groups, link, tmp) { if (pg->group == group) { TAILQ_REMOVE(&g_poll_groups, pg, link); - spdk_nvmf_poll_group_destroy(group); + spdk_nvmf_poll_group_destroy(group, NULL, NULL); free(pg); break; } diff --git a/include/spdk/nvmf.h b/include/spdk/nvmf.h index bb68fd2541..ca76ce1159 100644 --- a/include/spdk/nvmf.h +++ b/include/spdk/nvmf.h @@ -261,12 +261,18 @@ struct spdk_nvmf_poll_group *spdk_nvmf_poll_group_create(struct spdk_nvmf_tgt *t */ struct spdk_nvmf_poll_group *spdk_nvmf_get_optimal_poll_group(struct spdk_nvmf_qpair *qpair); +typedef void(*spdk_nvmf_poll_group_destroy_done_fn)(void *cb_arg, int status); + /** * Destroy a poll group. * * \param group The poll group to destroy. + * \param cb_fn A callback that will be called once the poll group is destroyed. + * \param cb_arg A context argument passed to cb_fn. */ -void spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group); +void spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group, + spdk_nvmf_poll_group_destroy_done_fn cb_fn, + void *cb_arg); /** * Add the given qpair to the poll group. diff --git a/include/spdk/nvmf_transport.h b/include/spdk/nvmf_transport.h index 1d267b3980..2b423a178d 100644 --- a/include/spdk/nvmf_transport.h +++ b/include/spdk/nvmf_transport.h @@ -157,6 +157,9 @@ struct spdk_nvmf_poll_group { /* Statistics */ struct spdk_nvmf_poll_group_stat stat; + + spdk_nvmf_poll_group_destroy_done_fn destroy_cb_fn; + void *destroy_cb_arg; }; struct spdk_nvmf_listener { diff --git a/lib/nvmf/nvmf.c b/lib/nvmf/nvmf.c index 03378b7fbf..35dc9732e4 100644 --- a/lib/nvmf/nvmf.c +++ b/lib/nvmf/nvmf.c @@ -175,6 +175,10 @@ spdk_nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf) } free(group->sgroups); + + if (group->destroy_cb_fn) { + group->destroy_cb_fn(group->destroy_cb_arg, 0); + } } static void @@ -728,8 +732,14 @@ spdk_nvmf_poll_group_create(struct spdk_nvmf_tgt *tgt) } void -spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group) +spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group, + spdk_nvmf_poll_group_destroy_done_fn cb_fn, + void *cb_arg) { + assert(group->destroy_cb_fn == NULL); + group->destroy_cb_fn = cb_fn; + group->destroy_cb_arg = cb_arg; + /* This function will put the io_channel associated with this poll group */ spdk_nvmf_tgt_destroy_poll_group_qpairs(group); } diff --git a/module/event/subsystems/nvmf/nvmf_tgt.c b/module/event/subsystems/nvmf/nvmf_tgt.c index e9b15ab446..e98b3a2a8e 100644 --- a/module/event/subsystems/nvmf/nvmf_tgt.c +++ b/module/event/subsystems/nvmf/nvmf_tgt.c @@ -324,7 +324,7 @@ nvmf_tgt_destroy_poll_group(void *ctx) TAILQ_FOREACH_SAFE(pg, &g_poll_groups, link, tpg) { if (pg->thread == thread) { TAILQ_REMOVE(&g_poll_groups, pg, link); - spdk_nvmf_poll_group_destroy(pg->group); + spdk_nvmf_poll_group_destroy(pg->group, NULL, NULL); free(pg); assert(g_num_poll_groups > 0); g_num_poll_groups--; diff --git a/test/unit/lib/nvmf/fc.c/fc_ut.c b/test/unit/lib/nvmf/fc.c/fc_ut.c index e531117e21..2406afdc4b 100644 --- a/test/unit/lib/nvmf/fc.c/fc_ut.c +++ b/test/unit/lib/nvmf/fc.c/fc_ut.c @@ -451,7 +451,7 @@ destroy_transport_test(void) for (i = 0; i < MAX_FC_UT_POLL_THREADS; i++) { set_thread(i); - spdk_nvmf_poll_group_destroy(g_poll_groups[i]); + spdk_nvmf_poll_group_destroy(g_poll_groups[i], NULL, NULL); poll_thread(0); }