From 1ad2c3ea7313794aaaec5500c3d1f35e363641cc Mon Sep 17 00:00:00 2001 From: Ben Walker Date: Mon, 19 Sep 2016 13:29:17 -0700 Subject: [PATCH] nvmf: Add support for multiple sessions per subsystem. This feature should only be used if clients are coordinating with one another. Change-Id: I89a437441a7e3fbcc1e5f6efa1c8e970ade7c2ec Signed-off-by: Ben Walker --- lib/nvmf/session.c | 32 +++++++++++++++++--------------- lib/nvmf/session.h | 5 ++++- lib/nvmf/subsystem.c | 28 ++++++++++++++-------------- lib/nvmf/subsystem.h | 4 +++- lib/nvmf/virtual.c | 12 ++++++------ 5 files changed, 44 insertions(+), 37 deletions(-) diff --git a/lib/nvmf/session.c b/lib/nvmf/session.c index d34058e02d..14c3084c8f 100644 --- a/lib/nvmf/session.c +++ b/lib/nvmf/session.c @@ -91,7 +91,7 @@ nvmf_init_nvme_session_properties(struct spdk_nvmf_session *session) session->subsys->ops->ctrlr_get_data(session); session->vcdata.aerl = 0; - session->vcdata.cntlid = 0; + session->vcdata.cntlid = session->id; session->vcdata.kas = 10; session->vcdata.maxcmd = g_nvmf_tgt.max_queue_depth; session->vcdata.mdts = nvmf_u32log2(g_nvmf_tgt.max_io_size / 4096); @@ -158,7 +158,7 @@ nvmf_init_nvme_session_properties(struct spdk_nvmf_session *session) static void session_destruct(struct spdk_nvmf_session *session) { - session->subsys->session = NULL; + TAILQ_REMOVE(&session->subsys->sessions, session, link); session->transport->session_fini(session); free(session); } @@ -245,15 +245,8 @@ spdk_nvmf_session_connect(struct spdk_nvmf_conn *conn, return; } - if (subsystem->session) { - SPDK_ERRLOG("Cannot connect to already-connected controller\n"); - rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; - rsp->status.sc = SPDK_NVMF_FABRIC_SC_CONTROLLER_BUSY; - return; - } - /* Establish a new session */ - subsystem->session = session = calloc(1, sizeof(struct spdk_nvmf_session)); + session = calloc(1, sizeof(struct spdk_nvmf_session)); if (session == NULL) { SPDK_ERRLOG("Memory allocation failure\n"); rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; @@ -261,6 +254,7 @@ spdk_nvmf_session_connect(struct spdk_nvmf_conn *conn, } TAILQ_INIT(&session->connections); + session->id = subsystem->session_id++; session->kato = cmd->kato; session->num_connections = 0; session->subsys = subsystem; @@ -269,7 +263,6 @@ spdk_nvmf_session_connect(struct spdk_nvmf_conn *conn, rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; conn->transport->session_fini(session); free(session); - subsystem->session = NULL; return; } @@ -278,19 +271,28 @@ spdk_nvmf_session_connect(struct spdk_nvmf_conn *conn, } else { nvmf_init_discovery_session_properties(session); } + + TAILQ_INSERT_TAIL(&subsystem->sessions, session, link); } else { + struct spdk_nvmf_session *tmp; + conn->type = CONN_TYPE_IOQ; SPDK_TRACELOG(SPDK_TRACE_NVMF, "Connect I/O Queue for controller id 0x%x\n", data->cntlid); - /* We always return CNTLID 0, so verify that the I/O connect CNTLID matches */ - if (data->cntlid != 0) { + session = NULL; + TAILQ_FOREACH(tmp, &subsystem->sessions, link) { + if (tmp->id == data->cntlid) { + session = tmp; + break; + } + } + if (session == NULL) { SPDK_ERRLOG("Unknown controller ID 0x%x\n", data->cntlid); INVALID_CONNECT_DATA(cntlid); return; } - session = subsystem->session; - if (session == NULL || !session->vcprop.cc.bits.en) { + if (!session->vcprop.cc.bits.en) { SPDK_ERRLOG("Got I/O connect before ctrlr was enabled\n"); INVALID_CONNECT_CMD(qid); return; diff --git a/lib/nvmf/session.h b/lib/nvmf/session.h index cd11d1ab51..4d0e25d407 100644 --- a/lib/nvmf/session.h +++ b/lib/nvmf/session.h @@ -67,7 +67,8 @@ struct spdk_nvmf_conn { * At least one admin connection and additional IOQ connections. */ struct spdk_nvmf_session { - struct spdk_nvmf_subsystem *subsys; + uint32_t id; + struct spdk_nvmf_subsystem *subsys; struct { union spdk_nvme_cap_register cap; @@ -86,6 +87,8 @@ struct spdk_nvmf_session { /* This is filled in by calling the transport's * session_init function. */ void *trctx; + + TAILQ_ENTRY(spdk_nvmf_session) link; }; void spdk_nvmf_session_connect(struct spdk_nvmf_conn *conn, diff --git a/lib/nvmf/subsystem.c b/lib/nvmf/subsystem.c index 1f6ded6509..57bdc863c5 100644 --- a/lib/nvmf/subsystem.c +++ b/lib/nvmf/subsystem.c @@ -76,20 +76,17 @@ nvmf_find_subsystem(const char *subnqn, const char *hostnqn) void spdk_nvmf_subsystem_poll(struct spdk_nvmf_subsystem *subsystem) { - struct spdk_nvmf_session *session = subsystem->session; + struct spdk_nvmf_session *session; - if (!session) { - /* No active connections, so just return */ - return; + TAILQ_FOREACH(session, &subsystem->sessions, link) { + /* For NVMe subsystems, check the backing physical device for completions. */ + if (subsystem->subtype == SPDK_NVMF_SUBTYPE_NVME) { + session->subsys->ops->poll_for_completions(session); + } + + /* For each connection in the session, check for completions */ + spdk_nvmf_session_poll(session); } - - /* For NVMe subsystems, check the backing physical device for completions. */ - if (subsystem->subtype == SPDK_NVMF_SUBTYPE_NVME) { - session->subsys->ops->poll_for_completions(session); - } - - /* For each connection in the session, check for completions */ - spdk_nvmf_session_poll(session); } static bool @@ -143,6 +140,7 @@ spdk_nvmf_create_subsystem(int num, const char *name, snprintf(subsystem->subnqn, sizeof(subsystem->subnqn), "%s", name); TAILQ_INIT(&subsystem->listen_addrs); TAILQ_INIT(&subsystem->hosts); + TAILQ_INIT(&subsystem->sessions); TAILQ_INSERT_HEAD(&g_subsystems, subsystem, entries); @@ -154,6 +152,7 @@ spdk_nvmf_delete_subsystem(struct spdk_nvmf_subsystem *subsystem) { struct spdk_nvmf_listen_addr *listen_addr, *listen_addr_tmp; struct spdk_nvmf_host *host, *host_tmp; + struct spdk_nvmf_session *session, *session_tmp; if (!subsystem) { return; @@ -176,9 +175,10 @@ spdk_nvmf_delete_subsystem(struct spdk_nvmf_subsystem *subsystem) subsystem->num_hosts--; } - if (subsystem->session) { - spdk_nvmf_session_destruct(subsystem->session); + TAILQ_FOREACH_SAFE(session, &subsystem->sessions, link, session_tmp) { + spdk_nvmf_session_destruct(session); } + if (subsystem->ops) { subsystem->ops->detach(subsystem); } diff --git a/lib/nvmf/subsystem.h b/lib/nvmf/subsystem.h index b2ef979c29..29ef397678 100644 --- a/lib/nvmf/subsystem.h +++ b/lib/nvmf/subsystem.h @@ -106,7 +106,6 @@ struct spdk_nvmf_subsystem { char subnqn[SPDK_NVMF_NQN_MAX_LEN]; enum spdk_nvmf_subsystem_mode mode; enum spdk_nvmf_subtype subtype; - struct spdk_nvmf_session *session; union { struct { @@ -129,6 +128,9 @@ struct spdk_nvmf_subsystem { spdk_nvmf_subsystem_connect_fn connect_cb; spdk_nvmf_subsystem_disconnect_fn disconnect_cb; + TAILQ_HEAD(, spdk_nvmf_session) sessions; + uint32_t session_id; + TAILQ_HEAD(, spdk_nvmf_listen_addr) listen_addrs; uint32_t num_listen_addrs; diff --git a/lib/nvmf/virtual.c b/lib/nvmf/virtual.c index 19b6407fac..48e6d7f898 100644 --- a/lib/nvmf/virtual.c +++ b/lib/nvmf/virtual.c @@ -58,12 +58,12 @@ struct __attribute__((packed)) nvme_read_cdw12 { uint8_t lr : 1; /* limited retry */ }; -static void nvmf_virtual_set_dsm(struct spdk_nvmf_subsystem *subsys) +static void nvmf_virtual_set_dsm(struct spdk_nvmf_session *session) { int i; - for (i = 0; i < subsys->dev.virtual.ns_count; i++) { - struct spdk_bdev *bdev = subsys->dev.virtual.ns_list[i]; + for (i = 0; i < session->subsys->dev.virtual.ns_count; i++) { + struct spdk_bdev *bdev = session->subsys->dev.virtual.ns_list[i]; if (!spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_UNMAP)) { SPDK_TRACELOG(SPDK_TRACE_NVMF, @@ -74,8 +74,8 @@ static void nvmf_virtual_set_dsm(struct spdk_nvmf_subsystem *subsys) } SPDK_TRACELOG(SPDK_TRACE_NVMF, "All devices in Subsystem%d support unmap - enabling DSM\n", - subsys->num); - subsys->session->vcdata.oncs.dsm = 1; + session->subsys->num); + session->vcdata.oncs.dsm = 1; } static void @@ -108,7 +108,7 @@ nvmf_virtual_ctrlr_get_data(struct spdk_nvmf_session *session) session->vcdata.vwc.present = 1; session->vcdata.sgls.supported = 1; strncpy(session->vcdata.subnqn, session->subsys->subnqn, sizeof(session->vcdata.subnqn)); - nvmf_virtual_set_dsm(subsys); + nvmf_virtual_set_dsm(session); } static void