d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

lib/iscsi: Create and terminate poll group threads explicitly

Browse Source 
Login acceptor still runs on one of the default reactor threads,
but we move iSCSI poll group from the default reactor threads to
dedicated threads created at startup.

At startup, use reference count to detect completion and
move to the next step. By returning completion message to the init
thread, we can avoid using any atomic operation.

At shutdown, we can use spdk_for_each_channel() conveniently. Put
voluntary spdk_thread_exit() calls into the callback to
spdk_put_io_channel().

Moving login acceptor to a dedicated thread is another task.

To maintain the original behavior, number of threads created is
the number of cores that SPDK app uses.

Change-Id: Ifd1de9343ac0183254ca608d1fd8faa94acc254e
Signed-off-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Signed-off-by: Vitaliy Mysak <vitaliy.mysak@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/492
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
This commit is contained in:
Shuhei Matsumoto 2020-01-30 19:10:20 -05:00 committed by Tomasz Zawadzki
parent 7d26adb6c0
commit de5f1ea0ba
2 changed files with 47 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lib/iscsi/iscsi.h
View File

@ -342,6 +342,7 @@ struct spdk_iscsi_globals {
char *authfile;
char *nodebase;
pthread_mutex_t mutex;
uint32_t refcnt;
TAILQ_HEAD(, spdk_iscsi_portal) portal_head;
TAILQ_HEAD(, spdk_iscsi_portal_grp) pg_head;
TAILQ_HEAD(, spdk_iscsi_init_grp) ig_head;

52
lib/iscsi/iscsi_subsystem.c
View File

@ -50,6 +50,7 @@
struct spdk_iscsi_opts *g_spdk_iscsi_opts = NULL;
static struct spdk_thread *g_init_thread = NULL;
static spdk_iscsi_init_cb g_init_cb_fn = NULL;
static void *g_init_cb_arg = NULL;
@ -1143,7 +1144,7 @@ iscsi_init_complete(int rc)
}
static void
iscsi_parse_configuration(void *ctx)
iscsi_parse_configuration(void)
{
int rc;
@ -1238,6 +1239,8 @@ static void
iscsi_poll_group_destroy(void *io_device, void *ctx_buf)
{
struct spdk_iscsi_poll_group *pg = ctx_buf;
struct spdk_io_channel *ch;
struct spdk_thread *thread;
assert(pg->poller != NULL);
assert(pg->sock_group != NULL);
@ -1245,6 +1248,24 @@ iscsi_poll_group_destroy(void *io_device, void *ctx_buf)
spdk_sock_group_close(&pg->sock_group);
spdk_poller_unregister(&pg->poller);
spdk_poller_unregister(&pg->nop_poller);
ch = spdk_io_channel_from_ctx(pg);
thread = spdk_io_channel_get_thread(ch);
assert(thread == spdk_get_thread());
spdk_thread_exit(thread);
}
static void
_iscsi_init_thread_done(void *ctx)
{
struct spdk_iscsi_poll_group *pg = ctx;
TAILQ_INSERT_TAIL(&g_spdk_iscsi.poll_group_head, pg, link);
if (--g_spdk_iscsi.refcnt == 0) {
iscsi_parse_configuration();
}
}
static void
@ -1256,19 +1277,38 @@ _iscsi_init_thread(void *ctx)
ch = spdk_get_io_channel(&g_spdk_iscsi);
pg = spdk_io_channel_get_ctx(ch);
pthread_mutex_lock(&g_spdk_iscsi.mutex);
TAILQ_INSERT_TAIL(&g_spdk_iscsi.poll_group_head, pg, link);
pthread_mutex_unlock(&g_spdk_iscsi.mutex);
spdk_thread_send_msg(g_init_thread, _iscsi_init_thread_done, pg);
}
static void
initialize_iscsi_poll_group(void)
{
struct spdk_cpuset tmp_cpumask = {};
uint32_t i;
char thread_name[32];
struct spdk_thread *thread;
spdk_io_device_register(&g_spdk_iscsi, iscsi_poll_group_create, iscsi_poll_group_destroy,
sizeof(struct spdk_iscsi_poll_group), "iscsi_tgt");
/* Send a message to each thread and create a poll group */
spdk_for_each_thread(_iscsi_init_thread, NULL, iscsi_parse_configuration);
/* Create threads for CPU cores active for this application, and send a
* message to each thread to create a poll group on it.
*/
g_init_thread = spdk_get_thread();
assert(g_init_thread != NULL);
assert(g_spdk_iscsi.refcnt == 0);
SPDK_ENV_FOREACH_CORE(i) {
spdk_cpuset_zero(&tmp_cpumask);
spdk_cpuset_set_cpu(&tmp_cpumask, i, true);
snprintf(thread_name, sizeof(thread_name), "iscsi_poll_group_%u", i);
thread = spdk_thread_create(thread_name, &tmp_cpumask);
assert(thread != NULL);
g_spdk_iscsi.refcnt++;
spdk_thread_send_msg(thread, _iscsi_init_thread, NULL);
}
}
static int