From de5f1ea0ba5b6f522ee4c1aaedecaeeab8f80f80 Mon Sep 17 00:00:00 2001 From: Shuhei Matsumoto Date: Thu, 30 Jan 2020 19:10:20 -0500 Subject: [PATCH] lib/iscsi: Create and terminate poll group threads explicitly Login acceptor still runs on one of the default reactor threads, but we move iSCSI poll group from the default reactor threads to dedicated threads created at startup. At startup, use reference count to detect completion and move to the next step. By returning completion message to the init thread, we can avoid using any atomic operation. At shutdown, we can use spdk_for_each_channel() conveniently. Put voluntary spdk_thread_exit() calls into the callback to spdk_put_io_channel(). Moving login acceptor to a dedicated thread is another task. To maintain the original behavior, number of threads created is the number of cores that SPDK app uses. Change-Id: Ifd1de9343ac0183254ca608d1fd8faa94acc254e Signed-off-by: Shuhei Matsumoto Signed-off-by: Vitaliy Mysak Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/492 Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Ben Walker --- lib/iscsi/iscsi.h | 1 + lib/iscsi/iscsi_subsystem.c | 52 ++++++++++++++++++++++++++++++++----- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/lib/iscsi/iscsi.h b/lib/iscsi/iscsi.h index 044f6043db..674382f875 100644 --- a/lib/iscsi/iscsi.h +++ b/lib/iscsi/iscsi.h @@ -342,6 +342,7 @@ struct spdk_iscsi_globals { char *authfile; char *nodebase; pthread_mutex_t mutex; + uint32_t refcnt; TAILQ_HEAD(, spdk_iscsi_portal) portal_head; TAILQ_HEAD(, spdk_iscsi_portal_grp) pg_head; TAILQ_HEAD(, spdk_iscsi_init_grp) ig_head; diff --git a/lib/iscsi/iscsi_subsystem.c b/lib/iscsi/iscsi_subsystem.c index dcfaf0c39a..7f25f0ce34 100644 --- a/lib/iscsi/iscsi_subsystem.c +++ b/lib/iscsi/iscsi_subsystem.c @@ -50,6 +50,7 @@ struct spdk_iscsi_opts *g_spdk_iscsi_opts = NULL; +static struct spdk_thread *g_init_thread = NULL; static spdk_iscsi_init_cb g_init_cb_fn = NULL; static void *g_init_cb_arg = NULL; @@ -1143,7 +1144,7 @@ iscsi_init_complete(int rc) } static void -iscsi_parse_configuration(void *ctx) +iscsi_parse_configuration(void) { int rc; @@ -1238,6 +1239,8 @@ static void iscsi_poll_group_destroy(void *io_device, void *ctx_buf) { struct spdk_iscsi_poll_group *pg = ctx_buf; + struct spdk_io_channel *ch; + struct spdk_thread *thread; assert(pg->poller != NULL); assert(pg->sock_group != NULL); @@ -1245,6 +1248,24 @@ iscsi_poll_group_destroy(void *io_device, void *ctx_buf) spdk_sock_group_close(&pg->sock_group); spdk_poller_unregister(&pg->poller); spdk_poller_unregister(&pg->nop_poller); + + ch = spdk_io_channel_from_ctx(pg); + thread = spdk_io_channel_get_thread(ch); + + assert(thread == spdk_get_thread()); + + spdk_thread_exit(thread); +} + +static void +_iscsi_init_thread_done(void *ctx) +{ + struct spdk_iscsi_poll_group *pg = ctx; + + TAILQ_INSERT_TAIL(&g_spdk_iscsi.poll_group_head, pg, link); + if (--g_spdk_iscsi.refcnt == 0) { + iscsi_parse_configuration(); + } } static void @@ -1256,19 +1277,38 @@ _iscsi_init_thread(void *ctx) ch = spdk_get_io_channel(&g_spdk_iscsi); pg = spdk_io_channel_get_ctx(ch); - pthread_mutex_lock(&g_spdk_iscsi.mutex); - TAILQ_INSERT_TAIL(&g_spdk_iscsi.poll_group_head, pg, link); - pthread_mutex_unlock(&g_spdk_iscsi.mutex); + spdk_thread_send_msg(g_init_thread, _iscsi_init_thread_done, pg); } static void initialize_iscsi_poll_group(void) { + struct spdk_cpuset tmp_cpumask = {}; + uint32_t i; + char thread_name[32]; + struct spdk_thread *thread; + spdk_io_device_register(&g_spdk_iscsi, iscsi_poll_group_create, iscsi_poll_group_destroy, sizeof(struct spdk_iscsi_poll_group), "iscsi_tgt"); - /* Send a message to each thread and create a poll group */ - spdk_for_each_thread(_iscsi_init_thread, NULL, iscsi_parse_configuration); + /* Create threads for CPU cores active for this application, and send a + * message to each thread to create a poll group on it. + */ + g_init_thread = spdk_get_thread(); + assert(g_init_thread != NULL); + assert(g_spdk_iscsi.refcnt == 0); + + SPDK_ENV_FOREACH_CORE(i) { + spdk_cpuset_zero(&tmp_cpumask); + spdk_cpuset_set_cpu(&tmp_cpumask, i, true); + snprintf(thread_name, sizeof(thread_name), "iscsi_poll_group_%u", i); + + thread = spdk_thread_create(thread_name, &tmp_cpumask); + assert(thread != NULL); + + g_spdk_iscsi.refcnt++; + spdk_thread_send_msg(thread, _iscsi_init_thread, NULL); + } } static int