d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

nvme/rdma: separate req/rsp allocation from registration

Browse Source 
This will simplify some upcoming changes to reconnect
a qpair.  In these cases we only need to re-register
the memory - we shouldn't have to allocate it again.

Signed-off-by: Jim Harris <james.r.harris@intel.com>
Change-Id: Id8adff313f191fbf11d7502127a2b961f2ca2f6e

Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/453478
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
This commit is contained in:
Jim Harris 2019-05-06 14:35:21 -07:00 committed by Changpeng Liu
parent ac2b06c809
commit 6949c71dca
1 changed files with 57 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
lib/nvme/nvme_rdma.c
View File

@ -326,13 +326,17 @@ nvme_rdma_post_recv(struct nvme_rdma_qpair *rqpair, uint16_t rsp_idx)
}
static void
nvme_rdma_free_rsps(struct nvme_rdma_qpair *rqpair)
nvme_rdma_unregister_rsps(struct nvme_rdma_qpair *rqpair)
{
if (rqpair->rsp_mr && rdma_dereg_mr(rqpair->rsp_mr)) {
SPDK_ERRLOG("Unable to de-register rsp_mr\n");
}
rqpair->rsp_mr = NULL;
}
static void
nvme_rdma_free_rsps(struct nvme_rdma_qpair *rqpair)
{
free(rqpair->rsps);
rqpair->rsps = NULL;
free(rqpair->rsp_sgls);
@ -344,9 +348,6 @@ nvme_rdma_free_rsps(struct nvme_rdma_qpair *rqpair)
static int
nvme_rdma_alloc_rsps(struct nvme_rdma_qpair *rqpair)
{
uint16_t i;
rqpair->rsp_mr = NULL;
rqpair->rsps = NULL;
rqpair->rsp_recv_wrs = NULL;
@ -369,6 +370,17 @@ nvme_rdma_alloc_rsps(struct nvme_rdma_qpair *rqpair)
goto fail;
}
return 0;
fail:
nvme_rdma_free_rsps(rqpair);
return -ENOMEM;
}
static int
nvme_rdma_register_rsps(struct nvme_rdma_qpair *rqpair)
{
int i;
rqpair->rsp_mr = rdma_reg_msgs(rqpair->cm_id, rqpair->rsps,
rqpair->num_entries * sizeof(*rqpair->rsps));
if (rqpair->rsp_mr == NULL) {
@ -397,10 +409,19 @@ nvme_rdma_alloc_rsps(struct nvme_rdma_qpair *rqpair)
return 0;
fail:
nvme_rdma_free_rsps(rqpair);
nvme_rdma_unregister_rsps(rqpair);
return -ENOMEM;
}
static void
nvme_rdma_unregister_reqs(struct nvme_rdma_qpair *rqpair)
{
if (rqpair->cmd_mr && rdma_dereg_mr(rqpair->cmd_mr)) {
SPDK_ERRLOG("Unable to de-register cmd_mr\n");
}
rqpair->cmd_mr = NULL;
}
static void
nvme_rdma_free_reqs(struct nvme_rdma_qpair *rqpair)
{
@ -408,11 +429,6 @@ nvme_rdma_free_reqs(struct nvme_rdma_qpair *rqpair)
return;
}
if (rqpair->cmd_mr && rdma_dereg_mr(rqpair->cmd_mr)) {
SPDK_ERRLOG("Unable to de-register cmd_mr\n");
}
rqpair->cmd_mr = NULL;
free(rqpair->cmds);
rqpair->cmds = NULL;
@ -423,8 +439,6 @@ nvme_rdma_free_reqs(struct nvme_rdma_qpair *rqpair)
static int
nvme_rdma_alloc_reqs(struct nvme_rdma_qpair *rqpair)
{
int i;
rqpair->rdma_reqs = calloc(rqpair->num_entries, sizeof(struct spdk_nvme_rdma_req));
if (rqpair->rdma_reqs == NULL) {
SPDK_ERRLOG("Failed to allocate rdma_reqs\n");
@ -437,6 +451,17 @@ nvme_rdma_alloc_reqs(struct nvme_rdma_qpair *rqpair)
goto fail;
}
return 0;
fail:
nvme_rdma_free_reqs(rqpair);
return -ENOMEM;
}
static int
nvme_rdma_register_reqs(struct nvme_rdma_qpair *rqpair)
{
int i;
rqpair->cmd_mr = rdma_reg_msgs(rqpair->cm_id, rqpair->cmds,
rqpair->num_entries * sizeof(*rqpair->cmds));
if (!rqpair->cmd_mr) {
@ -475,7 +500,7 @@ nvme_rdma_alloc_reqs(struct nvme_rdma_qpair *rqpair)
return 0;
fail:
nvme_rdma_free_reqs(rqpair);
nvme_rdma_unregister_reqs(rqpair);
return -ENOMEM;
}
@ -849,11 +874,19 @@ nvme_rdma_qpair_connect(struct nvme_rdma_qpair *rqpair)
rc = nvme_rdma_alloc_reqs(rqpair);
SPDK_DEBUGLOG(SPDK_LOG_NVME, "rc =%d\n", rc);
if (rc) {
SPDK_ERRLOG("Unable to allocate rqpair RDMA requests\n");
SPDK_ERRLOG("Unable to allocate rqpair RDMA requests\n");
return -1;
}
SPDK_DEBUGLOG(SPDK_LOG_NVME, "RDMA requests allocated\n");
rc = nvme_rdma_register_reqs(rqpair);
SPDK_DEBUGLOG(SPDK_LOG_NVME, "rc =%d\n", rc);
if (rc) {
SPDK_ERRLOG("Unable to register rqpair RDMA requests\n");
return -1;
}
SPDK_DEBUGLOG(SPDK_LOG_NVME, "RDMA requests registered\n");
rc = nvme_rdma_alloc_rsps(rqpair);
SPDK_DEBUGLOG(SPDK_LOG_NVME, "rc =%d\n", rc);
if (rc < 0) {
@ -862,6 +895,14 @@ nvme_rdma_qpair_connect(struct nvme_rdma_qpair *rqpair)
}
SPDK_DEBUGLOG(SPDK_LOG_NVME, "RDMA responses allocated\n");
rc = nvme_rdma_register_rsps(rqpair);
SPDK_DEBUGLOG(SPDK_LOG_NVME, "rc =%d\n", rc);
if (rc < 0) {
SPDK_ERRLOG("Unable to register rqpair RDMA responses\n");
return -1;
}
SPDK_DEBUGLOG(SPDK_LOG_NVME, "RDMA responses registered\n");
rc = nvme_rdma_register_mem(rqpair);
if (rc < 0) {
SPDK_ERRLOG("Unable to register memory for RDMA\n");
@ -1299,7 +1340,9 @@ nvme_rdma_qpair_destroy(struct spdk_nvme_qpair *qpair)
rqpair = nvme_rdma_qpair(qpair);
nvme_rdma_unregister_mem(rqpair);
nvme_rdma_unregister_reqs(rqpair);
nvme_rdma_free_reqs(rqpair);
nvme_rdma_unregister_rsps(rqpair);
nvme_rdma_free_rsps(rqpair);
if (rqpair->cm_id) {