d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

nvmf: Fail when sessions span RNICs

Browse Source 
The shared memory pool for a session is associated with
a particular RNIC via the protection domain. New connections
attempting to join a session that came in on a different RNIC
can't use that memory, so must be rejected.

Change-Id: Ibd79fe90566a231f76b7472e5e9b484c3e528454
Signed-off-by: Ben Walker <benjamin.walker@intel.com>
This commit is contained in:
Ben Walker 2016-10-19 11:16:17 -07:00
parent 349295caf2
commit 8b487155fa
3 changed files with 74 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
lib/nvmf/rdma.c
View File

@ -134,6 +134,8 @@ static TAILQ_HEAD(, spdk_nvmf_rdma_conn) g_pending_conns = TAILQ_HEAD_INITIALIZE
struct spdk_nvmf_rdma_session {
SLIST_HEAD(, spdk_nvmf_rdma_buf) data_buf_pool;
struct ibv_context *verbs;
uint8_t *buf;
struct ibv_mr *buf_mr;
};
@ -1169,10 +1171,9 @@ spdk_nvmf_rdma_discover(struct spdk_nvmf_listen_addr *listen_addr,
}
static int
spdk_nvmf_rdma_session_init(struct spdk_nvmf_session *session, struct spdk_nvmf_conn *conn)
spdk_nvmf_rdma_session_init(struct spdk_nvmf_session *session)
{
struct spdk_nvmf_rdma_session *rdma_sess;
struct spdk_nvmf_rdma_conn *rdma_conn = get_rdma_conn(conn);
int i;
struct spdk_nvmf_rdma_buf *buf;
@ -1193,27 +1194,13 @@ spdk_nvmf_rdma_session_init(struct spdk_nvmf_session *session, struct spdk_nvmf_
return -1;
}
rdma_sess->buf_mr = ibv_reg_mr(rdma_conn->cm_id->pd, rdma_sess->buf,
g_rdma.max_queue_depth * g_rdma.max_io_size,
IBV_ACCESS_LOCAL_WRITE);
if (!rdma_sess->buf_mr) {
SPDK_ERRLOG("Large buffer pool registration failed (%d x %d)\n",
g_rdma.max_queue_depth, g_rdma.max_io_size);
spdk_free(rdma_sess->buf);
free(rdma_sess);
return -1;
}
SPDK_TRACELOG(SPDK_TRACE_RDMA, "Session Shared Data Pool: %p Length: %x LKey: %x\n",
rdma_sess->buf, g_rdma.max_queue_depth * g_rdma.max_io_size, rdma_sess->buf_mr->lkey);
SLIST_INIT(&rdma_sess->data_buf_pool);
for (i = 0; i < g_rdma.max_queue_depth; i++) {
buf = (struct spdk_nvmf_rdma_buf *)(rdma_sess->buf + (i * g_rdma.max_io_size));
SLIST_INSERT_HEAD(&rdma_sess->data_buf_pool, buf, link);
}
session->transport = conn->transport;
session->transport = &spdk_nvmf_transport_rdma;
session->trctx = rdma_sess;
return 0;
@ -1234,6 +1221,48 @@ spdk_nvmf_rdma_session_fini(struct spdk_nvmf_session *session)
session->trctx = NULL;
}
static int
spdk_nvmf_rdma_session_add_conn(struct spdk_nvmf_session *session,
struct spdk_nvmf_conn *conn)
{
struct spdk_nvmf_rdma_session *rdma_sess = session->trctx;
struct spdk_nvmf_rdma_conn *rdma_conn = get_rdma_conn(conn);
if (rdma_sess->verbs != NULL) {
if (rdma_sess->verbs != rdma_conn->cm_id->verbs) {
SPDK_ERRLOG("Two connections belonging to the same session cannot connect using different RDMA devices.\n");
return -1;
}
/* Nothing else to do. */
return 0;
}
rdma_sess->verbs = rdma_conn->cm_id->verbs;
rdma_sess->buf_mr = ibv_reg_mr(rdma_conn->cm_id->pd, rdma_sess->buf,
g_rdma.max_queue_depth * g_rdma.max_io_size,
IBV_ACCESS_LOCAL_WRITE);
if (!rdma_sess->buf_mr) {
SPDK_ERRLOG("Large buffer pool registration failed (%d x %d)\n",
g_rdma.max_queue_depth, g_rdma.max_io_size);
spdk_free(rdma_sess->buf);
free(rdma_sess);
return -1;
}
SPDK_TRACELOG(SPDK_TRACE_RDMA, "Session Shared Data Pool: %p Length: %x LKey: %x\n",
rdma_sess->buf, g_rdma.max_queue_depth * g_rdma.max_io_size, rdma_sess->buf_mr->lkey);
return 0;
}
static int
spdk_nvmf_rdma_session_remove_conn(struct spdk_nvmf_session *session,
struct spdk_nvmf_conn *conn)
{
return 0;
}
static int
spdk_nvmf_rdma_request_complete(struct spdk_nvmf_request *req)
{
@ -1412,6 +1441,8 @@ const struct spdk_nvmf_transport spdk_nvmf_transport_rdma = {
.session_init = spdk_nvmf_rdma_session_init,
.session_fini = spdk_nvmf_rdma_session_fini,
.session_add_conn = spdk_nvmf_rdma_session_add_conn,
.session_remove_conn = spdk_nvmf_rdma_session_remove_conn,
.req_complete = spdk_nvmf_rdma_request_complete,
.req_release = spdk_nvmf_rdma_request_release,

16
lib/nvmf/session.c
View File

@ -259,7 +259,14 @@ spdk_nvmf_session_connect(struct spdk_nvmf_conn *conn,
session->num_connections = 0;
session->subsys = subsystem;
session->max_connections_allowed = g_nvmf_tgt.max_queues_per_session;
if (conn->transport->session_init(session, conn)) {
if (conn->transport->session_init(session)) {
rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
conn->transport->session_fini(session);
free(session);
return;
}
if (conn->transport->session_add_conn(session, conn)) {
rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
conn->transport->session_fini(session);
free(session);
@ -319,6 +326,11 @@ spdk_nvmf_session_connect(struct spdk_nvmf_conn *conn,
rsp->status.sc = SPDK_NVMF_FABRIC_SC_CONTROLLER_BUSY;
return;
}
if (conn->transport->session_add_conn(session, conn)) {
INVALID_CONNECT_CMD(qid);
return;
}
}
session->num_connections++;
@ -339,6 +351,8 @@ spdk_nvmf_session_disconnect(struct spdk_nvmf_conn *conn)
assert(session != NULL);
session->num_connections--;
TAILQ_REMOVE(&session->connections, conn, link);
conn->transport->session_remove_conn(session, conn);
conn->transport->conn_fini(conn);
if (session->num_connections == 0) {

12
lib/nvmf/transport.h
View File

@ -75,13 +75,23 @@ struct spdk_nvmf_transport {
/**
* Initialize the transport for the given session
*/
int (*session_init)(struct spdk_nvmf_session *session, struct spdk_nvmf_conn *conn);
int (*session_init)(struct spdk_nvmf_session *session);
/**
* Deinitiallize the transport for the given session
*/
void (*session_fini)(struct spdk_nvmf_session *session);
/**
* Add a connection to a session
*/
int (*session_add_conn)(struct spdk_nvmf_session *session, struct spdk_nvmf_conn *conn);
/**
* Remove a connection from a session
*/
int (*session_remove_conn)(struct spdk_nvmf_session *session, struct spdk_nvmf_conn *conn);
/*
* Signal request completion, which sends a response
* to the originator. A request can either