diff --git a/lib/ftl/ftl_core.c b/lib/ftl/ftl_core.c index 9e801f4426..250ce8784b 100644 --- a/lib/ftl/ftl_core.c +++ b/lib/ftl/ftl_core.c @@ -393,6 +393,21 @@ ftl_check_read_thread(const struct spdk_ftl_dev *dev) return dev->read_thread.thread == spdk_get_thread(); } +struct spdk_io_channel * +ftl_get_io_channel(const struct spdk_ftl_dev *dev) +{ + if (ftl_check_core_thread(dev)) { + return dev->core_thread.ioch; + } + if (ftl_check_read_thread(dev)) { + return dev->read_thread.ioch; + } + + assert(0); + return NULL; +} + + int ftl_io_erase(struct ftl_io *io) { @@ -1265,7 +1280,7 @@ ftl_nv_cache_write_header(struct ftl_nv_cache *nv_cache, bool shutdown, struct ftl_io_channel *ioch; bdev = spdk_bdev_desc_get_bdev(nv_cache->bdev_desc); - ioch = spdk_io_channel_get_ctx(dev->ioch); + ioch = spdk_io_channel_get_ctx(ftl_get_io_channel(dev)); memset(hdr, 0, spdk_bdev_get_block_size(bdev)); @@ -1287,7 +1302,7 @@ ftl_nv_cache_scrub(struct ftl_nv_cache *nv_cache, spdk_bdev_io_completion_cb cb_ struct ftl_io_channel *ioch; struct spdk_bdev *bdev; - ioch = spdk_io_channel_get_ctx(dev->ioch); + ioch = spdk_io_channel_get_ctx(ftl_get_io_channel(dev)); bdev = spdk_bdev_desc_get_bdev(nv_cache->bdev_desc); return spdk_bdev_write_zeroes_blocks(nv_cache->bdev_desc, ioch->cache_ioch, 1, diff --git a/lib/ftl/ftl_core.h b/lib/ftl/ftl_core.h index 1435d0a901..e07f7867ee 100644 --- a/lib/ftl/ftl_core.h +++ b/lib/ftl/ftl_core.h @@ -90,6 +90,8 @@ struct ftl_thread { /* Thread on which the poller is running */ struct spdk_thread *thread; + /* IO channel */ + struct spdk_io_channel *ioch; /* Poller */ struct spdk_poller *poller; /* Poller's function */ @@ -160,9 +162,6 @@ struct spdk_ftl_dev { /* Destruction context */ struct ftl_init_context fini_ctx; - /* IO channel */ - struct spdk_io_channel *ioch; - /* NVMe controller */ struct spdk_nvme_ctrlr *ctrlr; /* NVMe namespace */ @@ -301,6 +300,9 @@ int ftl_nv_cache_write_header(struct ftl_nv_cache *nv_cache, bool shutdown, int ftl_nv_cache_scrub(struct ftl_nv_cache *nv_cache, spdk_bdev_io_completion_cb cb_fn, void *cb_arg); +struct spdk_io_channel * +ftl_get_io_channel(const struct spdk_ftl_dev *dev); + #define ftl_to_ppa(addr) \ (struct ftl_ppa) { .ppa = (uint64_t)(addr) } diff --git a/lib/ftl/ftl_init.c b/lib/ftl/ftl_init.c index fd7595482f..a28a3d1cb3 100644 --- a/lib/ftl/ftl_init.c +++ b/lib/ftl/ftl_init.c @@ -732,6 +732,8 @@ _ftl_dev_init_thread(void *ctx) if (spdk_get_thread() == ftl_get_core_thread(dev)) { ftl_anm_register_device(dev, ftl_process_anm_event); } + + thread->ioch = spdk_get_io_channel(dev); } static int @@ -778,8 +780,10 @@ ftl_dev_free_thread(struct spdk_ftl_dev *dev, struct ftl_thread *thread) { assert(thread->poller == NULL); + spdk_put_io_channel(thread->ioch); spdk_nvme_ctrlr_free_io_qpair(thread->qpair); thread->thread = NULL; + thread->ioch = NULL; thread->qpair = NULL; } @@ -1069,12 +1073,6 @@ ftl_dev_init_io_channel(struct spdk_ftl_dev *dev) sizeof(struct ftl_io_channel), NULL); - dev->ioch = spdk_get_io_channel(dev); - if (!dev->ioch) { - spdk_io_device_unregister(dev, NULL); - return -1; - } - return 0; } @@ -1227,10 +1225,7 @@ ftl_dev_free_sync(struct spdk_ftl_dev *dev) ftl_dev_dump_bands(dev); ftl_dev_dump_stats(dev); - if (dev->ioch) { - spdk_put_io_channel(dev->ioch); - spdk_io_device_unregister(dev, NULL); - } + spdk_io_device_unregister(dev, NULL); if (dev->bands) { for (i = 0; i < ftl_dev_num_bands(dev); ++i) { diff --git a/lib/ftl/ftl_io.c b/lib/ftl/ftl_io.c index 398e195c0e..e237b5b80e 100644 --- a/lib/ftl/ftl_io.c +++ b/lib/ftl/ftl_io.c @@ -284,7 +284,7 @@ ftl_io_init_internal(const struct ftl_io_init_opts *opts) if (parent) { io = ftl_io_alloc_child(parent); } else { - io = ftl_io_alloc(dev->ioch); + io = ftl_io_alloc(ftl_get_io_channel(dev)); } if (!io) { diff --git a/lib/ftl/ftl_restore.c b/lib/ftl/ftl_restore.c index ef8598b05e..e7260dffcb 100644 --- a/lib/ftl/ftl_restore.c +++ b/lib/ftl/ftl_restore.c @@ -322,9 +322,10 @@ ftl_restore_head_cb(struct ftl_io *io, void *ctx, int status) } } -static int -ftl_restore_head_md(struct ftl_restore *restore) +static void +ftl_restore_head_md(void *ctx) { + struct ftl_restore *restore = ctx; struct spdk_ftl_dev *dev = restore->dev; struct ftl_restore_band *rband; struct ftl_lba_map *lba_map; @@ -347,8 +348,7 @@ ftl_restore_head_md(struct ftl_restore *restore) /* If the first IO fails, don't bother sending anything else */ if (i == 0) { - ftl_restore_free(restore); - return -EIO; + ftl_restore_complete(restore, -EIO); } } @@ -359,12 +359,9 @@ ftl_restore_head_md(struct ftl_restore *restore) if (spdk_unlikely(num_failed > 0)) { num_ios = __atomic_fetch_sub(&restore->num_ios, num_failed, __ATOMIC_SEQ_CST); if (num_ios == num_failed) { - ftl_restore_free(restore); - return -EIO; + ftl_restore_complete(restore, -EIO); } } - - return 0; } int @@ -377,7 +374,9 @@ ftl_restore_md(struct spdk_ftl_dev *dev, ftl_restore_fn cb) return -ENOMEM; } - return ftl_restore_head_md(restore); + spdk_thread_send_msg(ftl_get_core_thread(dev), ftl_restore_head_md, restore); + + return 0; } static int @@ -1003,7 +1002,7 @@ ftl_restore_nv_cache(struct ftl_restore *restore, ftl_restore_fn cb) size_t alignment; int rc, i; - ioch = spdk_io_channel_get_ctx(dev->ioch); + ioch = spdk_io_channel_get_ctx(ftl_get_io_channel(dev)); bdev = spdk_bdev_desc_get_bdev(nv_cache->bdev_desc); alignment = spdk_max(spdk_bdev_get_buf_align(bdev), sizeof(uint64_t)); diff --git a/test/unit/lib/ftl/ftl_io.c/ftl_io_ut.c b/test/unit/lib/ftl/ftl_io.c/ftl_io_ut.c index 0fbe1e4b9c..14a136555d 100644 --- a/test/unit/lib/ftl/ftl_io.c/ftl_io_ut.c +++ b/test/unit/lib/ftl/ftl_io.c/ftl_io_ut.c @@ -50,10 +50,10 @@ setup_device(void) dev = calloc(1, sizeof(*dev)); SPDK_CU_ASSERT_FATAL(dev != NULL); - dev->ioch = calloc(1, sizeof(*ioch) + sizeof(struct spdk_io_channel)); - SPDK_CU_ASSERT_FATAL(dev->ioch != NULL); + dev->core_thread.ioch = calloc(1, sizeof(*ioch) + sizeof(struct spdk_io_channel)); + SPDK_CU_ASSERT_FATAL(dev->core_thread.ioch != NULL); - ioch = spdk_io_channel_get_ctx(dev->ioch); + ioch = spdk_io_channel_get_ctx(dev->core_thread.ioch); ioch->elem_size = sizeof(struct ftl_md_io); ioch->io_pool = spdk_mempool_create("io-pool", 4096, ioch->elem_size, 0, 0); @@ -68,10 +68,10 @@ free_device(struct spdk_ftl_dev *dev) { struct ftl_io_channel *ioch; - ioch = spdk_io_channel_get_ctx(dev->ioch); + ioch = spdk_io_channel_get_ctx(dev->core_thread.ioch); spdk_mempool_free(ioch->io_pool); - free(dev->ioch); + free(dev->core_thread.ioch); free(dev); } @@ -88,7 +88,7 @@ alloc_io(struct spdk_ftl_dev *dev, ftl_io_fn cb, void *ctx) { struct ftl_io *io; - io = ftl_io_alloc(dev->ioch); + io = ftl_io_alloc(dev->core_thread.ioch); SPDK_CU_ASSERT_FATAL(io != NULL); setup_io(io, dev, cb, ctx); @@ -111,7 +111,7 @@ test_completion(void) size_t pool_size; dev = setup_device(); - ioch = spdk_io_channel_get_ctx(dev->ioch); + ioch = spdk_io_channel_get_ctx(dev->core_thread.ioch); pool_size = spdk_mempool_count(ioch->io_pool); io = alloc_io(dev, io_complete_cb, &status); @@ -153,7 +153,7 @@ test_alloc_free(void) size_t pool_size; dev = setup_device(); - ioch = spdk_io_channel_get_ctx(dev->ioch); + ioch = spdk_io_channel_get_ctx(dev->core_thread.ioch); pool_size = spdk_mempool_count(ioch->io_pool); parent = alloc_io(dev, io_complete_cb, &parent_status); @@ -199,7 +199,7 @@ test_child_requests(void) size_t pool_size; dev = setup_device(); - ioch = spdk_io_channel_get_ctx(dev->ioch); + ioch = spdk_io_channel_get_ctx(dev->core_thread.ioch); pool_size = spdk_mempool_count(ioch->io_pool); /* Verify correct behaviour when children finish first */ @@ -299,7 +299,7 @@ test_child_status(void) size_t pool_size, i; dev = setup_device(); - ioch = spdk_io_channel_get_ctx(dev->ioch); + ioch = spdk_io_channel_get_ctx(dev->core_thread.ioch); pool_size = spdk_mempool_count(ioch->io_pool); /* Verify the first error is returned by the parent */ @@ -386,7 +386,7 @@ test_multi_generation(void) int i, j; dev = setup_device(); - ioch = spdk_io_channel_get_ctx(dev->ioch); + ioch = spdk_io_channel_get_ctx(dev->core_thread.ioch); pool_size = spdk_mempool_count(ioch->io_pool); /* Verify correct behaviour when children finish first */