From cd55b3886ec295c9eebed9d8e1cdec3db36e48c4 Mon Sep 17 00:00:00 2001 From: Jim Harris Date: Fri, 5 Jan 2018 14:55:38 -0700 Subject: [PATCH] bdev: change spdk_bdev_io buf_link to STAILQ This is more efficient and buf_link users don't need the extra flexibility that TAILQ provides. link could also be changed to an STAILQ, but its usage is not in the performance path so not touching that as part of this commit. Signed-off-by: Jim Harris Change-Id: I7df2cc24a3784add8370db859003783e92cbfc21 Reviewed-on: https://review.gerrithub.io/393834 Reviewed-by: Ben Walker Reviewed-by: Changpeng Liu Tested-by: SPDK Automated Test System --- include/spdk_internal/bdev.h | 2 +- lib/bdev/bdev.c | 63 ++++++++++++++++++++---------------- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/include/spdk_internal/bdev.h b/include/spdk_internal/bdev.h index fac00ec4f4..44de5948b1 100644 --- a/include/spdk_internal/bdev.h +++ b/include/spdk_internal/bdev.h @@ -284,7 +284,7 @@ struct spdk_bdev_io { spdk_bdev_io_get_buf_cb get_buf_cb; /** Entry to the list need_buf of struct spdk_bdev. */ - TAILQ_ENTRY(spdk_bdev_io) buf_link; + STAILQ_ENTRY(spdk_bdev_io) buf_link; /** Enumerated value representing the I/O type. */ int16_t type; diff --git a/lib/bdev/bdev.c b/lib/bdev/bdev.c index 5127e423eb..50e259ae3b 100644 --- a/lib/bdev/bdev.c +++ b/lib/bdev/bdev.c @@ -63,6 +63,7 @@ int __itt_init_ittlib(const char *, __itt_group_id); #define ZERO_BUFFER_SIZE 0x100000 typedef TAILQ_HEAD(, spdk_bdev_io) bdev_io_tailq_t; +typedef STAILQ_HEAD(, spdk_bdev_io) bdev_io_stailq_t; struct spdk_bdev_mgr { struct spdk_mempool *bdev_io_pool; @@ -100,8 +101,8 @@ static struct spdk_thread *g_fini_thread = NULL; struct spdk_bdev_mgmt_channel { - bdev_io_tailq_t need_buf_small; - bdev_io_tailq_t need_buf_large; + bdev_io_stailq_t need_buf_small; + bdev_io_stailq_t need_buf_large; /* * Each thread keeps a cache of bdev_io - this allows @@ -110,7 +111,7 @@ struct spdk_bdev_mgmt_channel { * this, non-DPDK threads fetching from the mempool * incur a cmpxchg on get and put. */ - bdev_io_tailq_t per_thread_cache; + bdev_io_stailq_t per_thread_cache; uint32_t per_thread_cache_count; }; @@ -276,7 +277,7 @@ spdk_bdev_io_put_buf(struct spdk_bdev_io *bdev_io) struct spdk_mempool *pool; struct spdk_bdev_io *tmp; void *buf; - bdev_io_tailq_t *tailq; + bdev_io_stailq_t *stailq; struct spdk_bdev_mgmt_channel *ch; assert(bdev_io->u.bdev.iovcnt == 1); @@ -286,17 +287,17 @@ spdk_bdev_io_put_buf(struct spdk_bdev_io *bdev_io) if (bdev_io->buf_len <= SPDK_BDEV_SMALL_BUF_MAX_SIZE) { pool = g_bdev_mgr.buf_small_pool; - tailq = &ch->need_buf_small; + stailq = &ch->need_buf_small; } else { pool = g_bdev_mgr.buf_large_pool; - tailq = &ch->need_buf_large; + stailq = &ch->need_buf_large; } - if (TAILQ_EMPTY(tailq)) { + if (STAILQ_EMPTY(stailq)) { spdk_mempool_put(pool, buf); } else { - tmp = TAILQ_FIRST(tailq); - TAILQ_REMOVE(tailq, tmp, buf_link); + tmp = STAILQ_FIRST(stailq); + STAILQ_REMOVE_HEAD(stailq, buf_link); spdk_bdev_io_set_buf(tmp, buf); } } @@ -305,7 +306,7 @@ void spdk_bdev_io_get_buf(struct spdk_bdev_io *bdev_io, spdk_bdev_io_get_buf_cb cb, uint64_t len) { struct spdk_mempool *pool; - bdev_io_tailq_t *tailq; + bdev_io_stailq_t *stailq; void *buf = NULL; struct spdk_bdev_mgmt_channel *ch; @@ -325,16 +326,16 @@ spdk_bdev_io_get_buf(struct spdk_bdev_io *bdev_io, spdk_bdev_io_get_buf_cb cb, u bdev_io->get_buf_cb = cb; if (len <= SPDK_BDEV_SMALL_BUF_MAX_SIZE) { pool = g_bdev_mgr.buf_small_pool; - tailq = &ch->need_buf_small; + stailq = &ch->need_buf_small; } else { pool = g_bdev_mgr.buf_large_pool; - tailq = &ch->need_buf_large; + stailq = &ch->need_buf_large; } buf = spdk_mempool_get(pool); if (!buf) { - TAILQ_INSERT_TAIL(tailq, bdev_io, buf_link); + STAILQ_INSERT_TAIL(stailq, bdev_io, buf_link); } else { spdk_bdev_io_set_buf(bdev_io, buf); } @@ -372,10 +373,10 @@ spdk_bdev_mgmt_channel_create(void *io_device, void *ctx_buf) { struct spdk_bdev_mgmt_channel *ch = ctx_buf; - TAILQ_INIT(&ch->need_buf_small); - TAILQ_INIT(&ch->need_buf_large); + STAILQ_INIT(&ch->need_buf_small); + STAILQ_INIT(&ch->need_buf_large); - TAILQ_INIT(&ch->per_thread_cache); + STAILQ_INIT(&ch->per_thread_cache); ch->per_thread_cache_count = 0; return 0; @@ -386,13 +387,13 @@ spdk_bdev_mgmt_channel_free_resources(struct spdk_bdev_mgmt_channel *ch) { struct spdk_bdev_io *bdev_io; - if (!TAILQ_EMPTY(&ch->need_buf_small) || !TAILQ_EMPTY(&ch->need_buf_large)) { + if (!STAILQ_EMPTY(&ch->need_buf_small) || !STAILQ_EMPTY(&ch->need_buf_large)) { SPDK_ERRLOG("Pending I/O list wasn't empty on channel free\n"); } - while (!TAILQ_EMPTY(&ch->per_thread_cache)) { - bdev_io = TAILQ_FIRST(&ch->per_thread_cache); - TAILQ_REMOVE(&ch->per_thread_cache, bdev_io, buf_link); + while (!STAILQ_EMPTY(&ch->per_thread_cache)) { + bdev_io = STAILQ_FIRST(&ch->per_thread_cache); + STAILQ_REMOVE_HEAD(&ch->per_thread_cache, buf_link); ch->per_thread_cache_count--; spdk_mempool_put(g_bdev_mgr.bdev_io_pool, (void *)bdev_io); } @@ -742,8 +743,8 @@ spdk_bdev_get_io(struct spdk_io_channel *_ch) struct spdk_bdev_io *bdev_io; if (ch->per_thread_cache_count > 0) { - bdev_io = TAILQ_FIRST(&ch->per_thread_cache); - TAILQ_REMOVE(&ch->per_thread_cache, bdev_io, buf_link); + bdev_io = STAILQ_FIRST(&ch->per_thread_cache); + STAILQ_REMOVE_HEAD(&ch->per_thread_cache, buf_link); ch->per_thread_cache_count--; } else { bdev_io = spdk_mempool_get(g_bdev_mgr.bdev_io_pool); @@ -767,7 +768,7 @@ spdk_bdev_put_io(struct spdk_bdev_io *bdev_io) if (ch->per_thread_cache_count < SPDK_BDEV_IO_CACHE_SIZE) { ch->per_thread_cache_count++; - TAILQ_INSERT_TAIL(&ch->per_thread_cache, bdev_io, buf_link); + STAILQ_INSERT_TAIL(&ch->per_thread_cache, bdev_io, buf_link); } else { spdk_mempool_put(g_bdev_mgr.bdev_io_pool, (void *)bdev_io); } @@ -893,16 +894,24 @@ spdk_bdev_channel_create(void *io_device, void *ctx_buf) * linked using the spdk_bdev_io buf_link TAILQ_ENTRY. */ static void -_spdk_bdev_abort_buf_io(bdev_io_tailq_t *queue, struct spdk_bdev_channel *ch) +_spdk_bdev_abort_buf_io(bdev_io_stailq_t *queue, struct spdk_bdev_channel *ch) { - struct spdk_bdev_io *bdev_io, *tmp; + bdev_io_stailq_t tmp; + struct spdk_bdev_io *bdev_io; - TAILQ_FOREACH_SAFE(bdev_io, queue, buf_link, tmp) { + STAILQ_INIT(&tmp); + + while (!STAILQ_EMPTY(queue)) { + bdev_io = STAILQ_FIRST(queue); + STAILQ_REMOVE_HEAD(queue, buf_link); if (bdev_io->ch == ch) { - TAILQ_REMOVE(queue, bdev_io, buf_link); spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + } else { + STAILQ_INSERT_TAIL(&tmp, bdev_io, buf_link); } } + + STAILQ_SWAP(&tmp, queue, spdk_bdev_io); } /*