nvmf: Remove use of events from nvmf application
This is all based on SPDK threads instead. Change-Id: Idfd23d6541a548179be24b16540d62e5d7ea2a5c Signed-off-by: Ben Walker <benjamin.walker@intel.com> Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/451760 Tested-by: SPDK CI Jenkins <sys_sgci@intel.com> Reviewed-by: Seth Howell <seth.howell5141@gmail.com> Reviewed-by: Jim Harris <james.r.harris@intel.com> Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
This commit is contained in:
Ben Walker
Jim Harris
parent
c2c0ece7a2
commit
266dd568e6
@ -56,8 +56,9 @@ enum nvmf_tgt_state {
|
||||
};
|
||||
|
||||
struct nvmf_tgt_poll_group {
|
||||
struct spdk_nvmf_poll_group *group;
|
||||
uint32_t core;
|
||||
struct spdk_nvmf_poll_group *group;
|
||||
struct spdk_thread *thread;
|
||||
TAILQ_ENTRY(nvmf_tgt_poll_group) link;
|
||||
};
|
||||
|
||||
struct nvmf_tgt_host_trid {
|
||||
@ -75,10 +76,10 @@ struct spdk_nvmf_tgt *g_spdk_nvmf_tgt = NULL;
|
||||
|
||||
static enum nvmf_tgt_state g_tgt_state;
|
||||
|
||||
/* Round-Robin/IP-based tracking of cores for qpair assignment */
|
||||
static uint32_t g_tgt_core;
|
||||
/* Round-Robin/IP-based tracking of threads to poll group assignment */
|
||||
static struct nvmf_tgt_poll_group *g_next_poll_group = NULL;
|
||||
|
||||
static struct nvmf_tgt_poll_group *g_poll_groups = NULL;
|
||||
static TAILQ_HEAD(, nvmf_tgt_poll_group) g_poll_groups = TAILQ_HEAD_INITIALIZER(g_poll_groups);
|
||||
static size_t g_num_poll_groups = 0;
|
||||
|
||||
static struct spdk_poller *g_acceptor_poller = NULL;
|
||||
@ -111,15 +112,15 @@ spdk_nvmf_subsystem_fini(void)
|
||||
static struct nvmf_tgt_poll_group *
|
||||
spdk_nvmf_get_next_pg(void)
|
||||
{
|
||||
uint32_t core;
|
||||
struct nvmf_tgt_poll_group *pg;
|
||||
|
||||
core = g_tgt_core;
|
||||
g_tgt_core = spdk_env_get_next_core(core);
|
||||
if (g_tgt_core == UINT32_MAX) {
|
||||
g_tgt_core = spdk_env_get_first_core();
|
||||
pg = g_next_poll_group;
|
||||
g_next_poll_group = TAILQ_NEXT(pg, link);
|
||||
if (g_next_poll_group == NULL) {
|
||||
g_next_poll_group = TAILQ_FIRST(&g_poll_groups);
|
||||
}
|
||||
|
||||
return &g_poll_groups[core];
|
||||
return pg;
|
||||
}
|
||||
|
||||
static void
|
||||
@ -157,16 +158,15 @@ nvmf_tgt_get_pg(struct spdk_nvmf_qpair *qpair)
|
||||
{
|
||||
struct spdk_nvme_transport_id trid;
|
||||
struct nvmf_tgt_host_trid *tmp_trid = NULL, *new_trid = NULL;
|
||||
struct nvmf_tgt_poll_group *pg = NULL;
|
||||
struct nvmf_tgt_poll_group *pg;
|
||||
int ret;
|
||||
uint32_t core = spdk_env_get_first_core();
|
||||
|
||||
switch (g_spdk_nvmf_tgt_conf->conn_sched) {
|
||||
case CONNECT_SCHED_HOST_IP:
|
||||
ret = spdk_nvmf_qpair_get_peer_trid(qpair, &trid);
|
||||
if (ret) {
|
||||
SPDK_ERRLOG("Invalid host transport Id. Assigning to core %d\n", core);
|
||||
pg = &g_poll_groups[core];
|
||||
pg = g_next_poll_group;
|
||||
SPDK_ERRLOG("Invalid host transport Id. Assigning to poll group %p\n", pg);
|
||||
break;
|
||||
}
|
||||
|
||||
@ -181,11 +181,11 @@ nvmf_tgt_get_pg(struct spdk_nvmf_qpair *qpair)
|
||||
if (!tmp_trid) {
|
||||
new_trid = calloc(1, sizeof(*new_trid));
|
||||
if (!new_trid) {
|
||||
SPDK_ERRLOG("Insufficient memory. Assigning to core %d\n", core);
|
||||
pg = &g_poll_groups[core];
|
||||
pg = g_next_poll_group;
|
||||
SPDK_ERRLOG("Insufficient memory. Assigning to poll group %p\n", pg);
|
||||
break;
|
||||
}
|
||||
/* Get the next available core for the new host */
|
||||
/* Get the next available poll group for the new host */
|
||||
pg = spdk_nvmf_get_next_pg();
|
||||
new_trid->pg = pg;
|
||||
memcpy(new_trid->host_trid.traddr, trid.traddr,
|
||||
@ -202,11 +202,19 @@ nvmf_tgt_get_pg(struct spdk_nvmf_qpair *qpair)
|
||||
return pg;
|
||||
}
|
||||
|
||||
struct nvmf_tgt_pg_ctx {
|
||||
struct spdk_nvmf_qpair *qpair;
|
||||
struct nvmf_tgt_poll_group *pg;
|
||||
};
|
||||
|
||||
static void
|
||||
nvmf_tgt_poll_group_add(void *arg1, void *arg2)
|
||||
nvmf_tgt_poll_group_add(void *_ctx)
|
||||
{
|
||||
struct spdk_nvmf_qpair *qpair = arg1;
|
||||
struct nvmf_tgt_poll_group *pg = arg2;
|
||||
struct nvmf_tgt_pg_ctx *ctx = _ctx;
|
||||
struct spdk_nvmf_qpair *qpair = ctx->qpair;
|
||||
struct nvmf_tgt_poll_group *pg = ctx->pg;
|
||||
|
||||
free(_ctx);
|
||||
|
||||
if (spdk_nvmf_poll_group_add(pg->group, qpair) != 0) {
|
||||
SPDK_ERRLOG("Unable to add the qpair to a poll group.\n");
|
||||
@ -217,7 +225,7 @@ nvmf_tgt_poll_group_add(void *arg1, void *arg2)
|
||||
static void
|
||||
new_qpair(struct spdk_nvmf_qpair *qpair)
|
||||
{
|
||||
struct spdk_event *event;
|
||||
struct nvmf_tgt_pg_ctx *ctx;
|
||||
struct nvmf_tgt_poll_group *pg;
|
||||
uint32_t attempts;
|
||||
|
||||
@ -241,8 +249,17 @@ new_qpair(struct spdk_nvmf_qpair *qpair)
|
||||
return;
|
||||
}
|
||||
|
||||
event = spdk_event_allocate(pg->core, nvmf_tgt_poll_group_add, qpair, pg);
|
||||
spdk_event_call(event);
|
||||
ctx = calloc(1, sizeof(*ctx));
|
||||
if (!ctx) {
|
||||
SPDK_ERRLOG("Unable to send message to poll group.\n");
|
||||
spdk_nvmf_qpair_disconnect(qpair, NULL, NULL);
|
||||
return;
|
||||
}
|
||||
|
||||
ctx->qpair = qpair;
|
||||
ctx->pg = pg;
|
||||
|
||||
spdk_thread_send_msg(pg->thread, nvmf_tgt_poll_group_add, ctx);
|
||||
}
|
||||
|
||||
static int
|
||||
@ -259,19 +276,27 @@ static void
|
||||
nvmf_tgt_destroy_poll_group_done(void *ctx)
|
||||
{
|
||||
g_tgt_state = NVMF_TGT_FINI_STOP_ACCEPTOR;
|
||||
assert(g_num_poll_groups == 0);
|
||||
nvmf_tgt_advance_state();
|
||||
}
|
||||
|
||||
static void
|
||||
nvmf_tgt_destroy_poll_group(void *ctx)
|
||||
{
|
||||
struct nvmf_tgt_poll_group *pg;
|
||||
struct nvmf_tgt_poll_group *pg, *tpg;
|
||||
struct spdk_thread *thread;
|
||||
|
||||
pg = &g_poll_groups[spdk_env_get_current_core()];
|
||||
thread = spdk_get_thread();
|
||||
|
||||
if (pg->group) {
|
||||
spdk_nvmf_poll_group_destroy(pg->group);
|
||||
pg->group = NULL;
|
||||
TAILQ_FOREACH_SAFE(pg, &g_poll_groups, link, tpg) {
|
||||
if (pg->thread == thread) {
|
||||
TAILQ_REMOVE(&g_poll_groups, pg, link);
|
||||
spdk_nvmf_poll_group_destroy(pg->group);
|
||||
free(pg);
|
||||
assert(g_num_poll_groups > 0);
|
||||
g_num_poll_groups--;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -286,14 +311,22 @@ static void
|
||||
nvmf_tgt_create_poll_group(void *ctx)
|
||||
{
|
||||
struct nvmf_tgt_poll_group *pg;
|
||||
uint32_t core;
|
||||
|
||||
core = spdk_env_get_current_core();
|
||||
pg = calloc(1, sizeof(*pg));
|
||||
if (!pg) {
|
||||
SPDK_ERRLOG("Not enough memory to allocate poll groups\n");
|
||||
spdk_app_stop(-ENOMEM);
|
||||
return;
|
||||
}
|
||||
|
||||
pg = &g_poll_groups[core];
|
||||
assert(pg->core == SPDK_ENV_LCORE_ID_ANY);
|
||||
pg->core = core;
|
||||
pg->thread = spdk_get_thread();
|
||||
pg->group = spdk_nvmf_poll_group_create(g_spdk_nvmf_tgt);
|
||||
TAILQ_INSERT_TAIL(&g_poll_groups, pg, link);
|
||||
g_num_poll_groups++;
|
||||
|
||||
if (g_next_poll_group == NULL) {
|
||||
g_next_poll_group = pg;
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
@ -365,7 +398,6 @@ nvmf_tgt_advance_state(void)
|
||||
{
|
||||
enum nvmf_tgt_state prev_state;
|
||||
int rc = -1;
|
||||
uint32_t i;
|
||||
|
||||
do {
|
||||
prev_state = g_tgt_state;
|
||||
@ -373,21 +405,6 @@ nvmf_tgt_advance_state(void)
|
||||
switch (g_tgt_state) {
|
||||
case NVMF_TGT_INIT_NONE: {
|
||||
g_tgt_state = NVMF_TGT_INIT_PARSE_CONFIG;
|
||||
|
||||
/* Find the maximum core number */
|
||||
g_num_poll_groups = spdk_env_get_last_core() + 1;
|
||||
|
||||
g_poll_groups = calloc(g_num_poll_groups, sizeof(*g_poll_groups));
|
||||
if (g_poll_groups == NULL) {
|
||||
g_tgt_state = NVMF_TGT_ERROR;
|
||||
rc = -ENOMEM;
|
||||
break;
|
||||
}
|
||||
SPDK_ENV_FOREACH_CORE(i) {
|
||||
g_poll_groups[i].core = SPDK_ENV_LCORE_ID_ANY;
|
||||
}
|
||||
|
||||
g_tgt_core = spdk_env_get_first_core();
|
||||
break;
|
||||
}
|
||||
case NVMF_TGT_INIT_PARSE_CONFIG:
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user