d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

nvmf: Make transport-chosen the only valid qpair scheduler

Browse Source 
The transport always knows best.

Signed-off-by: Ben Walker <benjamin.walker@intel.com>
Change-Id: If66f539033fe11c50dc42e12ec990468588d5739
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2812
Community-CI: Broadcom CI
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Ziye Yang <ziye.yang@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>
This commit is contained in:
Ben Walker 2020-05-27 14:42:07 -07:00 committed by Tomasz Zawadzki
parent e0cd084bcb
commit c37cf9fbd8
7 changed files with 33 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@ -2,6 +2,11 @@
## v20.07: (Upcoming Release)
### nvmf
The NVMe-oF target no longer supports connecting scheduling configuration and instead
always uses what was previously called "transport" scheduling.
### nvme
Add `opts_size` in `spdk_nvme_ctrlr_opts` structure in order to solve the compatiblity issue

19
module/event/subsystems/nvmf/conf.c
View File

@ -117,23 +117,7 @@ nvmf_read_config_file_tgt_conf(struct spdk_conf_section *sp,
conn_scheduler = spdk_conf_section_get_val(sp, "ConnectionScheduler");
if (conn_scheduler) {
if (strcasecmp(conn_scheduler, "RoundRobin") == 0) {
conf->conn_sched = CONNECT_SCHED_ROUND_ROBIN;
} else if (strcasecmp(conn_scheduler, "Host") == 0) {
conf->conn_sched = CONNECT_SCHED_HOST_IP;
} else if (strcasecmp(conn_scheduler, "Transport") == 0) {
conf->conn_sched = CONNECT_SCHED_TRANSPORT_OPTIMAL_GROUP;
} else {
SPDK_ERRLOG("The valid value of ConnectionScheduler should be:\n"
"\t RoundRobin\n"
"\t Host\n"
"\t Transport\n");
rc = -1;
}
} else {
SPDK_NOTICELOG("The value of ConnectionScheduler is not configured,\n"
"we will use RoundRobin as the default scheduler\n");
SPDK_NOTICELOG("The ConnectionScheduler option is no longer valid. Ignoring it.\n");
}
conf->admin_passthru.identify_ctrlr = spdk_conf_section_get_boolval(sp,
@ -170,7 +154,6 @@ nvmf_parse_tgt_conf(void)
}
conf->acceptor_poll_rate = ACCEPT_TIMEOUT_US;
conf->conn_sched = DEFAULT_CONN_SCHED;
conf->admin_passthru.identify_ctrlr = false;
sp = spdk_conf_find_section(NULL, "Nvmf");

9
module/event/subsystems/nvmf/event_nvmf.h
View File

@ -43,13 +43,6 @@
#include "spdk_internal/log.h"
#define ACCEPT_TIMEOUT_US 10000 /* 10ms */
#define DEFAULT_CONN_SCHED CONNECT_SCHED_TRANSPORT_OPTIMAL_GROUP
enum spdk_nvmf_connect_sched {
CONNECT_SCHED_ROUND_ROBIN = 0,
CONNECT_SCHED_HOST_IP,
CONNECT_SCHED_TRANSPORT_OPTIMAL_GROUP,
};
struct spdk_nvmf_admin_passthru_conf {
bool identify_ctrlr;
@ -57,7 +50,7 @@ struct spdk_nvmf_admin_passthru_conf {
struct spdk_nvmf_tgt_conf {
uint32_t acceptor_poll_rate;
enum spdk_nvmf_connect_sched conn_sched;
uint32_t conn_sched; /* Deprecated. */
struct spdk_nvmf_admin_passthru_conf admin_passthru;
};

14
module/event/subsystems/nvmf/nvmf_rpc.c
View File

@ -76,18 +76,9 @@ SPDK_RPC_REGISTER_ALIAS_DEPRECATED(nvmf_set_max_subsystems, set_nvmf_target_max_
static int decode_conn_sched(const struct spdk_json_val *val, void *out)
{
enum spdk_nvmf_connect_sched *sched = out;
*(uint32_t *)out = 0;
if (spdk_json_strequal(val, "roundrobin") == true) {
*sched = CONNECT_SCHED_ROUND_ROBIN;
} else if (spdk_json_strequal(val, "hostip") == true) {
*sched = CONNECT_SCHED_HOST_IP;
} else if (spdk_json_strequal(val, "transport") == true) {
*sched = CONNECT_SCHED_TRANSPORT_OPTIMAL_GROUP;
} else {
SPDK_ERRLOG("Invalid connection scheduling parameter\n");
return -EINVAL;
}
SPDK_NOTICELOG("conn_sched is no longer a supported parameter. Ignoring.");
return 0;
}
@ -139,7 +130,6 @@ rpc_nvmf_set_config(struct spdk_jsonrpc_request *request,
}
conf->acceptor_poll_rate = ACCEPT_TIMEOUT_US;
conf->conn_sched = DEFAULT_CONN_SCHED;
conf->admin_passthru.identify_ctrlr = false;
if (params != NULL) {

173
module/event/subsystems/nvmf/nvmf_tgt.c
View File

@ -62,17 +62,6 @@ struct nvmf_tgt_poll_group {
TAILQ_ENTRY(nvmf_tgt_poll_group) link;
};
struct nvmf_tgt_host_trid {
struct spdk_nvme_transport_id host_trid;
struct nvmf_tgt_poll_group *pg;
uint32_t ref;
TAILQ_ENTRY(nvmf_tgt_host_trid) link;
};
/* List of host trids that are connected to the target */
static TAILQ_HEAD(, nvmf_tgt_host_trid) g_nvmf_tgt_host_trids =
TAILQ_HEAD_INITIALIZER(g_nvmf_tgt_host_trids);
struct spdk_nvmf_tgt *g_spdk_nvmf_tgt = NULL;
static enum nvmf_tgt_state g_tgt_state;
@ -80,7 +69,7 @@ static enum nvmf_tgt_state g_tgt_state;
static struct spdk_thread *g_tgt_init_thread = NULL;
static struct spdk_thread *g_tgt_fini_thread = NULL;
/* Round-Robin/IP-based tracking of threads to poll group assignment */
/* Round-Robin assignment of connections to poll groups */
static struct nvmf_tgt_poll_group *g_next_poll_group = NULL;
static TAILQ_HEAD(, nvmf_tgt_poll_group) g_poll_groups = TAILQ_HEAD_INITIALIZER(g_poll_groups);
@ -117,124 +106,34 @@ nvmf_subsystem_fini(void)
nvmf_shutdown_cb(NULL);
}
/* Round robin selection of poll groups */
static struct nvmf_tgt_poll_group *
nvmf_get_next_pg(void)
{
struct nvmf_tgt_poll_group *pg;
pg = g_next_poll_group;
g_next_poll_group = TAILQ_NEXT(pg, link);
if (g_next_poll_group == NULL) {
g_next_poll_group = TAILQ_FIRST(&g_poll_groups);
}
return pg;
}
static struct nvmf_tgt_poll_group *
nvmf_get_optimal_pg(struct spdk_nvmf_qpair *qpair)
{
struct nvmf_tgt_poll_group *pg, *_pg = NULL;
struct spdk_nvmf_poll_group *group = spdk_nvmf_get_optimal_poll_group(qpair);
if (group == NULL) {
_pg = nvmf_get_next_pg();
goto end;
}
TAILQ_FOREACH(pg, &g_poll_groups, link) {
if (pg->group == group) {
_pg = pg;
break;
}
}
end:
assert(_pg != NULL);
return _pg;
}
static void
nvmf_tgt_remove_host_trid(struct spdk_nvmf_qpair *qpair)
{
struct spdk_nvme_transport_id trid_to_remove;
struct nvmf_tgt_host_trid *trid = NULL, *tmp_trid = NULL;
if (g_spdk_nvmf_tgt_conf->conn_sched != CONNECT_SCHED_HOST_IP) {
return;
}
if (spdk_nvmf_qpair_get_peer_trid(qpair, &trid_to_remove) != 0) {
return;
}
TAILQ_FOREACH_SAFE(trid, &g_nvmf_tgt_host_trids, link, tmp_trid) {
if (trid && !strncmp(trid->host_trid.traddr,
trid_to_remove.traddr, SPDK_NVMF_TRADDR_MAX_LEN + 1)) {
trid->ref--;
if (trid->ref == 0) {
TAILQ_REMOVE(&g_nvmf_tgt_host_trids, trid, link);
free(trid);
}
break;
}
}
return;
}
static struct nvmf_tgt_poll_group *
nvmf_tgt_get_pg(struct spdk_nvmf_qpair *qpair)
{
struct spdk_nvme_transport_id trid;
struct nvmf_tgt_host_trid *tmp_trid = NULL, *new_trid = NULL;
struct nvmf_tgt_poll_group *pg;
int ret;
struct spdk_nvmf_poll_group *group;
switch (g_spdk_nvmf_tgt_conf->conn_sched) {
case CONNECT_SCHED_HOST_IP:
ret = spdk_nvmf_qpair_get_peer_trid(qpair, &trid);
if (ret) {
pg = g_next_poll_group;
SPDK_ERRLOG("Invalid host transport Id. Assigning to poll group %p\n", pg);
break;
}
group = spdk_nvmf_get_optimal_poll_group(qpair);
TAILQ_FOREACH(tmp_trid, &g_nvmf_tgt_host_trids, link) {
if (tmp_trid && !strncmp(tmp_trid->host_trid.traddr,
trid.traddr, SPDK_NVMF_TRADDR_MAX_LEN + 1)) {
tmp_trid->ref++;
pg = tmp_trid->pg;
break;
if (group != NULL) {
/* Look up the nvmf_tgt_poll_group that matches this spdk_nvmf_poll_group */
TAILQ_FOREACH(pg, &g_poll_groups, link) {
if (pg->group == group) {
return pg;
}
}
if (!tmp_trid) {
new_trid = calloc(1, sizeof(*new_trid));
if (!new_trid) {
pg = g_next_poll_group;
SPDK_ERRLOG("Insufficient memory. Assigning to poll group %p\n", pg);
break;
}
/* Get the next available poll group for the new host */
pg = nvmf_get_next_pg();
new_trid->pg = pg;
memcpy(new_trid->host_trid.traddr, trid.traddr,
SPDK_NVMF_TRADDR_MAX_LEN + 1);
TAILQ_INSERT_TAIL(&g_nvmf_tgt_host_trids, new_trid, link);
}
break;
case CONNECT_SCHED_TRANSPORT_OPTIMAL_GROUP:
pg = nvmf_get_optimal_pg(qpair);
break;
case CONNECT_SCHED_ROUND_ROBIN:
default:
pg = nvmf_get_next_pg();
break;
return NULL;
}
if (g_next_poll_group == NULL) {
g_next_poll_group = TAILQ_FIRST(&g_poll_groups);
if (g_next_poll_group == NULL) {
return NULL;
}
}
pg = g_next_poll_group;
g_next_poll_group = TAILQ_NEXT(pg, link);
return pg;
}
@ -263,23 +162,14 @@ new_qpair(struct spdk_nvmf_qpair *qpair, void *cb_arg)
{
struct nvmf_tgt_pg_ctx *ctx;
struct nvmf_tgt_poll_group *pg;
uint32_t attempts;
if (g_tgt_state != NVMF_TGT_RUNNING) {
spdk_nvmf_qpair_disconnect(qpair, NULL, NULL);
return;
}
for (attempts = 0; attempts < g_num_poll_groups; attempts++) {
pg = nvmf_tgt_get_pg(qpair);
if (pg->group != NULL) {
break;
} else {
nvmf_tgt_remove_host_trid(qpair);
}
}
if (attempts == g_num_poll_groups) {
pg = nvmf_tgt_get_pg(qpair);
if (pg == NULL) {
SPDK_ERRLOG("No poll groups exist.\n");
spdk_nvmf_qpair_disconnect(qpair, NULL, NULL);
return;
@ -447,15 +337,8 @@ nvmf_tgt_subsystem_stopped(struct spdk_nvmf_subsystem *subsystem,
static void
nvmf_tgt_destroy_done(void *ctx, int status)
{
struct nvmf_tgt_host_trid *trid, *tmp_trid;
g_tgt_state = NVMF_TGT_STOPPED;
TAILQ_FOREACH_SAFE(trid, &g_nvmf_tgt_host_trids, link, tmp_trid) {
TAILQ_REMOVE(&g_nvmf_tgt_host_trids, trid, link);
free(trid);
}
free(g_spdk_nvmf_tgt_conf);
g_spdk_nvmf_tgt_conf = NULL;
nvmf_tgt_advance_state();
@ -645,18 +528,6 @@ nvmf_subsystem_init(void)
nvmf_tgt_advance_state();
}
static char *
get_conn_sched_string(enum spdk_nvmf_connect_sched sched)
{
if (sched == CONNECT_SCHED_HOST_IP) {
return "hostip";
} else if (sched == CONNECT_SCHED_TRANSPORT_OPTIMAL_GROUP) {
return "transport";
} else {
return "roundrobin";
}
}
static void
nvmf_subsystem_write_config_json(struct spdk_json_write_ctx *w)
{
@ -667,8 +538,6 @@ nvmf_subsystem_write_config_json(struct spdk_json_write_ctx *w)
spdk_json_write_named_object_begin(w, "params");
spdk_json_write_named_uint32(w, "acceptor_poll_rate", g_spdk_nvmf_tgt_conf->acceptor_poll_rate);
spdk_json_write_named_string(w, "conn_sched",
get_conn_sched_string(g_spdk_nvmf_tgt_conf->conn_sched));
spdk_json_write_named_object_begin(w, "admin_cmd_passthru");
spdk_json_write_named_bool(w, "identify_ctrlr",
g_spdk_nvmf_tgt_conf->admin_passthru.identify_ctrlr);

5
scripts/rpc.py
View File

@ -1704,10 +1704,7 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse
p = subparsers.add_parser('nvmf_set_config', aliases=['set_nvmf_target_config'],
help='Set NVMf target config')
p.add_argument('-r', '--acceptor-poll-rate', help='Polling interval of the acceptor for incoming connections (usec)', type=int)
p.add_argument('-s', '--conn-sched', help="""'roundrobin' - Schedule the incoming connections from any host
on the cores in a round robin manner (Default). 'hostip' - Schedule all the incoming connections from a
specific host IP on to the same core. Connections from different IP will be assigned to cores in a round
robin manner. 'transport' - Schedule the connection according to the transport characteristics.""")
p.add_argument('-s', '--conn-sched', help='(Deprecated). Ignored.')
p.add_argument('-i', '--passthru-identify-ctrlr', help="""Passthrough fields like serial number and model number
when the controller has a single namespace that is an NVMe bdev""", action='store_true')
p.set_defaults(func=nvmf_set_config)

4
scripts/rpc/nvmf.py
View File

@ -27,7 +27,7 @@ def nvmf_set_config(client,
Args:
acceptor_poll_rate: Acceptor poll period in microseconds (optional)
conn_sched: Scheduling of incoming connections (optional)
conn_sched: (Deprecated) Ignored
Returns:
True or False
@ -37,7 +37,7 @@ def nvmf_set_config(client,
if acceptor_poll_rate:
params['acceptor_poll_rate'] = acceptor_poll_rate
if conn_sched:
params['conn_sched'] = conn_sched
print("WARNING: conn_sched is deprecated and ignored.")
if passthru_identify_ctrlr:
admin_cmd_passthru = {}
admin_cmd_passthru['identify_ctrlr'] = passthru_identify_ctrlr