lib/nvmf: Make spdk_nvmf_poll_group_destroy() asynchronous

The next patch will create poll group threads dynamically for
NVMe-oF target, and will need to wait for completion of poll group and
I/O channel destroy. This is a preparation for the next patch.

Add callback function and its argument to spdk_nvmf_poll_group_destroy(),
and to struct spdk_nvmf_poll_group, respectively.

The callback has not only cb_arg but also status as its parameters even
if the next patch always sets the status to zero. The reason is to follow
spdk_nvmf_tgt_destroy's callback and to process any case that the status
is nonzero in future.

spdk_nvmf_poll_group_destroy() sets the passed callback to the passed
poll group.

Then spdk_nvmf_tgt_destroy_poll_group() calls the held callback in the
end.

This change will ensure all pollers are being unregistered and
all I/O channels are being released.

Signed-off-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Change-Id: Ifb854066a5259a6029d55b88de358e3346c63f18
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/495
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

CHANGELOG.md

@@ -19,6 +19,9 @@ engine have been renamed to identify the engine as a software accelerator.
 A new function, `spdk_vmd_fini`, has been added. It releases all resources acquired by the VMD
 library through the `spdk_vmd_init` call.
 
+### nvmf
+`spdk_nvmf_poll_group_destroy()` is now asynchronous and accepts a completion callback.
+
 ### Miscellaneous
 
 `--json-ignore-init-errors` command line param has been added to ignore initialization errors

examples/nvmf/nvmf/nvmf.c

@@ -663,7 +663,7 @@ nvmf_tgt_destroy_poll_group(struct spdk_io_channel_iter *i)
 	TAILQ_FOREACH_SAFE(pg, &g_poll_groups, link, tmp) {
 		if (pg->group == group) {
 			TAILQ_REMOVE(&g_poll_groups, pg, link);
-			spdk_nvmf_poll_group_destroy(group);
+			spdk_nvmf_poll_group_destroy(group, NULL, NULL);
 			free(pg);
 			break;
 		}

include/spdk/nvmf.h

@@ -261,12 +261,18 @@ struct spdk_nvmf_poll_group *spdk_nvmf_poll_group_create(struct spdk_nvmf_tgt *t
  */
 struct spdk_nvmf_poll_group *spdk_nvmf_get_optimal_poll_group(struct spdk_nvmf_qpair *qpair);
 
+typedef void(*spdk_nvmf_poll_group_destroy_done_fn)(void *cb_arg, int status);
+
 /**
  * Destroy a poll group.
  *
  * \param group The poll group to destroy.
+ * \param cb_fn A callback that will be called once the poll group is destroyed.
+ * \param cb_arg A context argument passed to cb_fn.
  */
-void spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group);
+void spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group,
+				  spdk_nvmf_poll_group_destroy_done_fn cb_fn,
+				  void *cb_arg);
 
 /**
  * Add the given qpair to the poll group.

include/spdk/nvmf_transport.h

@@ -157,6 +157,9 @@ struct spdk_nvmf_poll_group {
 
 	/* Statistics */
 	struct spdk_nvmf_poll_group_stat		stat;
+
+	spdk_nvmf_poll_group_destroy_done_fn		destroy_cb_fn;
+	void						*destroy_cb_arg;
 };
 
 struct spdk_nvmf_listener {

lib/nvmf/nvmf.c

@@ -175,6 +175,10 @@ spdk_nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf)
 	}
 
 	free(group->sgroups);
+
+	if (group->destroy_cb_fn) {
+		group->destroy_cb_fn(group->destroy_cb_arg, 0);
+	}
 }
 
 static void
@@ -728,8 +732,14 @@ spdk_nvmf_poll_group_create(struct spdk_nvmf_tgt *tgt)
 }
 
 void
-spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group)
+spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group,
+			     spdk_nvmf_poll_group_destroy_done_fn cb_fn,
+			     void *cb_arg)
 {
+	assert(group->destroy_cb_fn == NULL);
+	group->destroy_cb_fn = cb_fn;
+	group->destroy_cb_arg = cb_arg;
+
 	/* This function will put the io_channel associated with this poll group */
 	spdk_nvmf_tgt_destroy_poll_group_qpairs(group);
 }

module/event/subsystems/nvmf/nvmf_tgt.c

@@ -324,7 +324,7 @@ nvmf_tgt_destroy_poll_group(void *ctx)
 	TAILQ_FOREACH_SAFE(pg, &g_poll_groups, link, tpg) {
 		if (pg->thread == thread) {
 			TAILQ_REMOVE(&g_poll_groups, pg, link);
-			spdk_nvmf_poll_group_destroy(pg->group);
+			spdk_nvmf_poll_group_destroy(pg->group, NULL, NULL);
 			free(pg);
 			assert(g_num_poll_groups > 0);
 			g_num_poll_groups--;

test/unit/lib/nvmf/fc.c/fc_ut.c

@@ -451,7 +451,7 @@ destroy_transport_test(void)
 
 	for (i = 0; i < MAX_FC_UT_POLL_THREADS; i++) {
 		set_thread(i);
-		spdk_nvmf_poll_group_destroy(g_poll_groups[i]);
+		spdk_nvmf_poll_group_destroy(g_poll_groups[i], NULL, NULL);
 		poll_thread(0);
 	}