d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

util/nvme: added io_device unregister callback

Browse Source 
Patch afe860ae deferred freeing the io_device. However, for nvme, the
io_device context (spdk_nvme_ctrlr) is still being destructed before
io_channels are destroyed, causing segfaults on hotremove.

This patch defers io_device context destruction and fixes nvme
hotremove.

Fixes: afe860aeb1 ("channel: Correctly defer unregisters if channels exist")
Fixes: 5533c3d208 ("util: defer put_io_channel")

Change-Id: I7af699174cac0c6c6a6faa2cc65418c47347eb9a
Signed-off-by: Dariusz Stojaczyk <dariuszx.stojaczyk@intel.com>
Reviewed-on: https://review.gerrithub.io/370459
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
This commit is contained in:
Dariusz Stojaczyk 2017-07-20 14:40:51 +02:00 committed by Daniel Verkamp
parent 542b541588
commit 73358c99e1
8 changed files with 40 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
include/spdk/io_channel.h
View File

@ -52,6 +52,8 @@ typedef void (*spdk_thread_pass_msg)(spdk_thread_fn fn, void *ctx,
typedef int (*spdk_io_channel_create_cb)(void *io_device, void *ctx_buf);
typedef void (*spdk_io_channel_destroy_cb)(void *io_device, void *ctx_buf);
typedef void (*spdk_io_device_unregister_cb)(void *io_device);
typedef void (*spdk_channel_msg)(void *io_device, struct spdk_io_channel *ch,
void *ctx);
typedef void (*spdk_channel_for_each_cpl)(void *io_device, void *ctx);
@ -110,10 +112,11 @@ void spdk_io_device_register(void *io_device, spdk_io_channel_create_cb create_c
/**
* \brief Unregister the opaque io_device context as an I/O device.
*
* Callers must ensure they release references to any I/O channel related to this
* device before calling this function.
* The actual unregistration might be deferred until all active I/O channels are destroyed.
* unregister_cb is an optional callback function invoked to release any references to
* this I/O device.
*/
void spdk_io_device_unregister(void *io_device);
void spdk_io_device_unregister(void *io_device, spdk_io_device_unregister_cb unregister_cb);
/**
* \brief Gets an I/O channel for the specified io_device to be used by the calling thread.

4
lib/bdev/bdev.c
View File

@ -522,7 +522,7 @@ spdk_bdev_finish(void)
spdk_mempool_free(g_bdev_mgr.buf_small_pool);
spdk_mempool_free(g_bdev_mgr.buf_large_pool);
spdk_io_device_unregister(&g_bdev_mgr);
spdk_io_device_unregister(&g_bdev_mgr, NULL);
return 0;
}
@ -1462,7 +1462,7 @@ spdk_bdev_unregister(struct spdk_bdev *bdev)
pthread_mutex_destroy(&bdev->mutex);
spdk_io_device_unregister(bdev);
spdk_io_device_unregister(bdev, NULL);
rc = bdev->fn_table->destruct(bdev->ctxt);
if (rc < 0) {

11
lib/bdev/nvme/bdev_nvme.c
View File

@ -224,6 +224,14 @@ bdev_nvme_poll_adminq(void *arg)
spdk_nvme_ctrlr_process_admin_completions(ctrlr);
}
static void
bdev_nvme_unregister_cb(void *io_device)
{
struct spdk_nvme_ctrlr *ctrlr = io_device;
spdk_nvme_detach(ctrlr);
}
static int
bdev_nvme_destruct(void *ctx)
{
@ -240,9 +248,8 @@ bdev_nvme_destruct(void *ctx)
if (nvme_ctrlr->ref == 0) {
TAILQ_REMOVE(&g_nvme_ctrlrs, nvme_ctrlr, tailq);
pthread_mutex_unlock(&g_bdev_nvme_mutex);
spdk_io_device_unregister(nvme_ctrlr->ctrlr);
spdk_io_device_unregister(nvme_ctrlr->ctrlr, bdev_nvme_unregister_cb);
spdk_bdev_poller_stop(&nvme_ctrlr->adminq_timer_poller);
spdk_nvme_detach(nvme_ctrlr->ctrlr);
free(nvme_ctrlr->name);
free(nvme_ctrlr);
return 0;

4
lib/blob/blobstore.c
View File

@ -1141,8 +1141,8 @@ _spdk_bs_free(struct spdk_blob_store *bs)
struct spdk_blob *blob, *blob_tmp;
spdk_bs_unregister_md_thread(bs);
spdk_io_device_unregister(&bs->io_target);
spdk_io_device_unregister(&bs->md_target);
spdk_io_device_unregister(&bs->io_target, NULL);
spdk_io_device_unregister(&bs->md_target, NULL);
TAILQ_FOREACH_SAFE(blob, &bs->blobs, link, blob_tmp) {
TAILQ_REMOVE(&bs->blobs, blob, link);

18
lib/blobfs/blobfs.c
View File

@ -440,10 +440,10 @@ spdk_fs_init(struct spdk_bs_dev *dev, fs_send_request_fn send_request_fn,
req = alloc_fs_request(fs->md_target.md_fs_channel);
if (req == NULL) {
spdk_put_io_channel(fs->md_target.md_io_channel);
spdk_io_device_unregister(&fs->md_target);
spdk_io_device_unregister(&fs->md_target, NULL);
spdk_put_io_channel(fs->sync_target.sync_io_channel);
spdk_io_device_unregister(&fs->sync_target);
spdk_io_device_unregister(&fs->io_target);
spdk_io_device_unregister(&fs->sync_target, NULL);
spdk_io_device_unregister(&fs->io_target, NULL);
free(fs);
cb_fn(cb_arg, NULL, -ENOMEM);
return;
@ -566,10 +566,10 @@ spdk_fs_load(struct spdk_bs_dev *dev, fs_send_request_fn send_request_fn,
req = alloc_fs_request(fs->md_target.md_fs_channel);
if (req == NULL) {
spdk_put_io_channel(fs->md_target.md_io_channel);
spdk_io_device_unregister(&fs->md_target);
spdk_io_device_unregister(&fs->md_target, NULL);
spdk_put_io_channel(fs->sync_target.sync_io_channel);
spdk_io_device_unregister(&fs->sync_target);
spdk_io_device_unregister(&fs->io_target);
spdk_io_device_unregister(&fs->sync_target, NULL);
spdk_io_device_unregister(&fs->io_target, NULL);
free(fs);
cb_fn(cb_arg, NULL, -ENOMEM);
return;
@ -600,9 +600,9 @@ unload_cb(void *ctx, int bserrno)
args->fn.fs_op(args->arg, bserrno);
free(req);
spdk_io_device_unregister(&fs->io_target);
spdk_io_device_unregister(&fs->sync_target);
spdk_io_device_unregister(&fs->md_target);
spdk_io_device_unregister(&fs->io_target, NULL);
spdk_io_device_unregister(&fs->sync_target, NULL);
spdk_io_device_unregister(&fs->md_target, NULL);
free(fs);
}

9
lib/util/io_channel.c
View File

@ -42,6 +42,7 @@ struct io_device {
void *io_device;
spdk_io_channel_create_cb create_cb;
spdk_io_channel_destroy_cb destroy_cb;
spdk_io_device_unregister_cb unregister_cb;
uint32_t ctx_size;
TAILQ_ENTRY(io_device) tailq;
@ -183,6 +184,7 @@ spdk_io_device_register(void *io_device, spdk_io_channel_create_cb create_cb,
dev->io_device = io_device;
dev->create_cb = create_cb;
dev->destroy_cb = destroy_cb;
dev->unregister_cb = NULL;
dev->ctx_size = ctx_size;
dev->unregistered = false;
@ -217,11 +219,15 @@ _spdk_io_device_attempt_free(struct io_device *dev)
}
}
if (dev->unregister_cb) {
dev->unregister_cb(dev->io_device);
}
free(dev);
}
void
spdk_io_device_unregister(void *io_device)
spdk_io_device_unregister(void *io_device, spdk_io_device_unregister_cb unregister_cb)
{
struct io_device *dev;
@ -238,6 +244,7 @@ spdk_io_device_unregister(void *io_device)
return;
}
dev->unregister_cb = unregister_cb;
dev->unregistered = true;
TAILQ_REMOVE(&g_io_devices, dev, tailq);
_spdk_io_device_attempt_free(dev);

2
test/unit/lib/bdev/bdev.c/bdev_ut.c
View File

@ -55,7 +55,7 @@ spdk_io_device_register(void *io_device, spdk_io_channel_create_cb create_cb,
}
void
spdk_io_device_unregister(void *io_device)
spdk_io_device_unregister(void *io_device, spdk_io_device_unregister_cb unregister_cb)
{
}

6
test/unit/lib/util/io_channel.c/io_channel_ut.c
View File

@ -146,9 +146,9 @@ channel(void)
ch1 = spdk_get_io_channel(&device3);
CU_ASSERT(ch1 == NULL);
spdk_io_device_unregister(&device1);
spdk_io_device_unregister(&device2);
spdk_io_device_unregister(&device3);
spdk_io_device_unregister(&device1, NULL);
spdk_io_device_unregister(&device2, NULL);
spdk_io_device_unregister(&device3, NULL);
CU_ASSERT(TAILQ_EMPTY(&g_io_devices));
spdk_free_thread();
CU_ASSERT(TAILQ_EMPTY(&g_threads));