From 590b6e7507f4b818fa497caceabcddfbc6d98530 Mon Sep 17 00:00:00 2001 From: Ziye Yang Date: Fri, 18 Dec 2020 01:12:44 +0800 Subject: [PATCH] nbd: Use async manner to stop nbd device. Purpose: In order to free all the allocated resources. Our current code uses sync behaviour, and it will not wait for all the resources are freed if there is active I/Os, and thus we will not free some resources, e.g., some fds will not be closed. Signed-off-by: Ziye Yang Change-Id: Iaf9a606da2049ffd0096860c46d89d094038a5ff Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5601 Community-CI: Broadcom CI Tested-by: SPDK CI Jenkins Reviewed-by: Changpeng Liu Reviewed-by: Jim Harris Reviewed-by: Sun Zhenyuan Reviewed-by: Shuhei Matsumoto Reviewed-by: --- CHANGELOG.md | 5 ++++ include/spdk/nbd.h | 10 ++++++-- lib/nbd/Makefile | 2 +- lib/nbd/nbd.c | 39 ++++++++++++++++++++++++++----- module/event/subsystems/nbd/nbd.c | 9 +++++-- 5 files changed, 54 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9abcc11c0..c82f05e9de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,11 @@ An `opts_size`element was added in the `spdk_app_opts` structure to solve the ABI compatiblity issue between different SPDK version. An `opts_size` parameter is added into `spdk_app_opts_init` function. +### nbd + +Change the return type of function `spdk_nbd_stop` from void to int. And update the +`spdk_nbd_fini` with two parameters to make its behavior from sync to async. + ### nvmf Broadcom FC LLD driver and SPDK NVMe-oF FC transport consolidated one LLD API, diff --git a/include/spdk/nbd.h b/include/spdk/nbd.h index be57c09cd3..9a8a8f7594 100644 --- a/include/spdk/nbd.h +++ b/include/spdk/nbd.h @@ -45,6 +45,7 @@ extern "C" { struct spdk_bdev; struct spdk_nbd_disk; struct spdk_json_write_ctx; +typedef void (*spdk_nbd_fini_cb)(void *arg); /** * Initialize the network block device layer. @@ -55,8 +56,11 @@ int spdk_nbd_init(void); /** * Stop and close all the running network block devices. + * + * \param cb_fn Callback to be always called. + * \param cb_arg Passed to cb_fn. */ -void spdk_nbd_fini(void); +void spdk_nbd_fini(spdk_nbd_fini_cb cb_fn, void *cb_arg); /** * Called when an NBD device has been started. @@ -80,8 +84,10 @@ void spdk_nbd_start(const char *bdev_name, const char *nbd_path, * Stop the running network block device safely. * * \param nbd A pointer to the network block device to stop. + * + * \return 0 on success. */ -void spdk_nbd_stop(struct spdk_nbd_disk *nbd); +int spdk_nbd_stop(struct spdk_nbd_disk *nbd); /** * Get the local filesystem path used for the network block device. diff --git a/lib/nbd/Makefile b/lib/nbd/Makefile index 69b13d133f..3e9f0384a5 100644 --- a/lib/nbd/Makefile +++ b/lib/nbd/Makefile @@ -34,7 +34,7 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..) include $(SPDK_ROOT_DIR)/mk/spdk.common.mk -SO_VER := 2 +SO_VER := 3 SO_MINOR := 0 LIBNAME = nbd diff --git a/lib/nbd/nbd.c b/lib/nbd/nbd.c index 7493d38222..b28f997ced 100644 --- a/lib/nbd/nbd.c +++ b/lib/nbd/nbd.c @@ -120,6 +120,8 @@ struct spdk_nbd_disk_globals { }; static struct spdk_nbd_disk_globals g_spdk_nbd; +static spdk_nbd_fini_cb g_fini_cb_fn; +static void *g_fini_cb_arg; static int nbd_submit_bdev_io(struct spdk_nbd_disk *nbd, struct nbd_io *io); @@ -132,10 +134,11 @@ spdk_nbd_init(void) return 0; } -void -spdk_nbd_fini(void) +static void +_nbd_fini(void *arg1) { struct spdk_nbd_disk *nbd_idx, *nbd_tmp; + int rc = 0; /* * Stop running spdk_nbd_disk. @@ -144,8 +147,26 @@ spdk_nbd_fini(void) * remove nbd from TAILQ. */ TAILQ_FOREACH_SAFE(nbd_idx, &g_spdk_nbd.disk_head, tailq, nbd_tmp) { - spdk_nbd_stop(nbd_idx); + rc = spdk_nbd_stop(nbd_idx); + if (rc) { + break; + } } + + if (!rc) { + g_fini_cb_fn(g_fini_cb_arg); + } else { + spdk_thread_send_msg(spdk_get_thread(), _nbd_fini, NULL); + } +} + +void +spdk_nbd_fini(spdk_nbd_fini_cb cb_fn, void *cb_arg) +{ + g_fini_cb_fn = cb_fn; + g_fini_cb_arg = cb_arg; + + _nbd_fini(NULL); } static int @@ -388,11 +409,13 @@ _nbd_stop(struct spdk_nbd_disk *nbd) free(nbd); } -void +int spdk_nbd_stop(struct spdk_nbd_disk *nbd) { + int rc = 0; + if (nbd == NULL) { - return; + return rc; } nbd->state = NBD_DISK_STATE_HARDDISC; @@ -400,9 +423,13 @@ spdk_nbd_stop(struct spdk_nbd_disk *nbd) /* * Stop action should be called only after all nbd_io are executed. */ - if (!nbd_cleanup_io(nbd)) { + + rc = nbd_cleanup_io(nbd); + if (!rc) { _nbd_stop(nbd); } + + return rc; } static int64_t diff --git a/module/event/subsystems/nbd/nbd.c b/module/event/subsystems/nbd/nbd.c index 1730fd448b..50340c5c7f 100644 --- a/module/event/subsystems/nbd/nbd.c +++ b/module/event/subsystems/nbd/nbd.c @@ -47,11 +47,16 @@ nbd_subsystem_init(void) spdk_subsystem_init_next(rc); } +static void +nbd_subsystem_fini_done(void *arg) +{ + spdk_subsystem_fini_next(); +} + static void nbd_subsystem_fini(void) { - spdk_nbd_fini(); - spdk_subsystem_fini_next(); + spdk_nbd_fini(nbd_subsystem_fini_done, NULL); } static void