diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index f7f496fffe..cbeaa607de 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -68,6 +68,12 @@ struct spdk_nvmf_rdma_request { /* In Capsule data buffer */ uint8_t *buf; + union { + struct ibv_recv_wr recv; + struct ibv_send_wr send; + } wr; + struct ibv_sge sg_list[2]; + TAILQ_ENTRY(spdk_nvmf_rdma_request) link; }; @@ -327,6 +333,8 @@ nvmf_ibv_send_wr_init(struct ibv_send_wr *wr, assert(wr != NULL); assert(sg_list != NULL); + memset(wr, 0, sizeof(*wr)); + wr->wr_id = (uint64_t)rdma_req; wr->opcode = opcode; wr->send_flags = send_flags; @@ -351,31 +359,31 @@ nvmf_ibv_send_wr_set_rkey(struct ibv_send_wr *wr, struct spdk_nvmf_request *req) static int nvmf_post_rdma_read(struct spdk_nvmf_request *req) { - struct ibv_send_wr wr = {}; struct ibv_send_wr *bad_wr = NULL; struct spdk_nvmf_conn *conn = req->conn; + struct spdk_nvmf_rdma_request *rdma_req = get_rdma_req(req); struct spdk_nvmf_rdma_conn *rdma_conn = get_rdma_conn(conn); struct spdk_nvmf_rdma_session *rdma_sess; - struct ibv_sge sge; int rc; SPDK_TRACELOG(SPDK_TRACE_RDMA, "RDMA READ POSTED. Request: %p Connection: %p\n", req, conn); - sge.addr = (uintptr_t)req->data; + rdma_req->sg_list[0].addr = (uintptr_t)req->data; if (req->length > g_rdma.in_capsule_data_size) { rdma_sess = conn->sess->trctx; - sge.lkey = rdma_sess->buf_mr->lkey; + rdma_req->sg_list[0].lkey = rdma_sess->buf_mr->lkey; } else { - sge.lkey = rdma_conn->bufs_mr->lkey; + rdma_req->sg_list[0].lkey = rdma_conn->bufs_mr->lkey; } - sge.length = req->length; - nvmf_trace_ibv_sge(&sge); + rdma_req->sg_list[0].length = req->length; + nvmf_trace_ibv_sge(&rdma_req->sg_list[0]); - nvmf_ibv_send_wr_init(&wr, req, &sge, IBV_WR_RDMA_READ, IBV_SEND_SIGNALED); - nvmf_ibv_send_wr_set_rkey(&wr, req); + nvmf_ibv_send_wr_init(&rdma_req->wr.send, req, rdma_req->sg_list, IBV_WR_RDMA_READ, + IBV_SEND_SIGNALED); + nvmf_ibv_send_wr_set_rkey(&rdma_req->wr.send, req); spdk_trace_record(TRACE_RDMA_READ_START, 0, 0, (uintptr_t)req, 0); - rc = ibv_post_send(rdma_conn->cm_id->qp, &wr, &bad_wr); + rc = ibv_post_send(rdma_conn->cm_id->qp, &rdma_req->wr.send, &bad_wr); if (rc) { SPDK_ERRLOG("Failure posting rdma read send, rc = 0x%x\n", rc); } @@ -386,31 +394,31 @@ nvmf_post_rdma_read(struct spdk_nvmf_request *req) static int nvmf_post_rdma_write(struct spdk_nvmf_request *req) { - struct ibv_send_wr wr = {}; struct ibv_send_wr *bad_wr = NULL; struct spdk_nvmf_conn *conn = req->conn; + struct spdk_nvmf_rdma_request *rdma_req = get_rdma_req(req); struct spdk_nvmf_rdma_conn *rdma_conn = get_rdma_conn(conn); struct spdk_nvmf_rdma_session *rdma_sess; - struct ibv_sge sge; int rc; SPDK_TRACELOG(SPDK_TRACE_RDMA, "RDMA WRITE POSTED. Request: %p Connection: %p\n", req, conn); - sge.addr = (uintptr_t)req->data; + rdma_req->sg_list[0].addr = (uintptr_t)req->data; if (req->length > g_rdma.in_capsule_data_size) { rdma_sess = conn->sess->trctx; - sge.lkey = rdma_sess->buf_mr->lkey; + rdma_req->sg_list[0].lkey = rdma_sess->buf_mr->lkey; } else { - sge.lkey = rdma_conn->bufs_mr->lkey; + rdma_req->sg_list[0].lkey = rdma_conn->bufs_mr->lkey; } - sge.length = req->length; - nvmf_trace_ibv_sge(&sge); + rdma_req->sg_list[0].length = req->length; + nvmf_trace_ibv_sge(&rdma_req->sg_list[0]); - nvmf_ibv_send_wr_init(&wr, req, &sge, IBV_WR_RDMA_WRITE, IBV_SEND_SIGNALED); - nvmf_ibv_send_wr_set_rkey(&wr, req); + nvmf_ibv_send_wr_init(&rdma_req->wr.send, req, rdma_req->sg_list, IBV_WR_RDMA_WRITE, + IBV_SEND_SIGNALED); + nvmf_ibv_send_wr_set_rkey(&rdma_req->wr.send, req); spdk_trace_record(TRACE_RDMA_WRITE_START, 0, 0, (uintptr_t)req, 0); - rc = ibv_post_send(rdma_conn->cm_id->qp, &wr, &bad_wr); + rc = ibv_post_send(rdma_conn->cm_id->qp, &rdma_req->wr.send, &bad_wr); if (rc) { SPDK_ERRLOG("Failure posting rdma write send, rc = 0x%x\n", rc); } @@ -421,32 +429,31 @@ nvmf_post_rdma_write(struct spdk_nvmf_request *req) static int nvmf_post_rdma_recv(struct spdk_nvmf_request *req) { - struct ibv_recv_wr wr, *bad_wr = NULL; + struct ibv_recv_wr *bad_wr = NULL; struct spdk_nvmf_conn *conn = req->conn; struct spdk_nvmf_rdma_conn *rdma_conn = get_rdma_conn(conn); struct spdk_nvmf_rdma_request *rdma_req = get_rdma_req(req); - struct ibv_sge sg_list[2]; int rc; SPDK_TRACELOG(SPDK_TRACE_RDMA, "RDMA RECV POSTED. Request: %p Connection: %p\n", req, conn); - sg_list[0].addr = (uintptr_t)req->cmd; - sg_list[0].length = sizeof(*req->cmd); - sg_list[0].lkey = rdma_conn->cmds_mr->lkey; - nvmf_trace_ibv_sge(&sg_list[0]); + rdma_req->sg_list[0].addr = (uintptr_t)req->cmd; + rdma_req->sg_list[0].length = sizeof(*req->cmd); + rdma_req->sg_list[0].lkey = rdma_conn->cmds_mr->lkey; + nvmf_trace_ibv_sge(&rdma_req->sg_list[0]); - sg_list[1].addr = (uintptr_t)rdma_req->buf; - sg_list[1].length = g_rdma.in_capsule_data_size; - sg_list[1].lkey = rdma_conn->bufs_mr->lkey; - nvmf_trace_ibv_sge(&sg_list[1]); + rdma_req->sg_list[1].addr = (uintptr_t)rdma_req->buf; + rdma_req->sg_list[1].length = g_rdma.in_capsule_data_size; + rdma_req->sg_list[1].lkey = rdma_conn->bufs_mr->lkey; + nvmf_trace_ibv_sge(&rdma_req->sg_list[1]); - memset(&wr, 0, sizeof(wr)); - wr.wr_id = (uintptr_t)rdma_req; - wr.next = NULL; - wr.sg_list = sg_list; - wr.num_sge = 2; + memset(&rdma_req->wr.recv, 0, sizeof(struct ibv_recv_wr)); + rdma_req->wr.recv.wr_id = (uintptr_t)rdma_req; + rdma_req->wr.recv.next = NULL; + rdma_req->wr.recv.sg_list = rdma_req->sg_list; + rdma_req->wr.recv.num_sge = 2; - rc = ibv_post_recv(rdma_conn->cm_id->qp, &wr, &bad_wr); + rc = ibv_post_recv(rdma_conn->cm_id->qp, &rdma_req->wr.recv, &bad_wr); if (rc) { SPDK_ERRLOG("Failure posting rdma recv, rc = 0x%x\n", rc); } @@ -457,24 +464,23 @@ nvmf_post_rdma_recv(struct spdk_nvmf_request *req) static int nvmf_post_rdma_send(struct spdk_nvmf_request *req) { - struct ibv_send_wr wr = {}; struct ibv_send_wr *bad_wr = NULL; struct spdk_nvmf_conn *conn = req->conn; + struct spdk_nvmf_rdma_request *rdma_req = get_rdma_req(req); struct spdk_nvmf_rdma_conn *rdma_conn = get_rdma_conn(conn); - struct ibv_sge sge; int rc; SPDK_TRACELOG(SPDK_TRACE_RDMA, "RDMA SEND POSTED. Request: %p Connection: %p\n", req, conn); - sge.addr = (uintptr_t)req->rsp; - sge.length = sizeof(*req->rsp); - sge.lkey = rdma_conn->cpls_mr->lkey; - nvmf_trace_ibv_sge(&sge); + rdma_req->sg_list[0].addr = (uintptr_t)req->rsp; + rdma_req->sg_list[0].length = sizeof(*req->rsp); + rdma_req->sg_list[0].lkey = rdma_conn->cpls_mr->lkey; + nvmf_trace_ibv_sge(&rdma_req->sg_list[0]); - nvmf_ibv_send_wr_init(&wr, req, &sge, IBV_WR_SEND, IBV_SEND_SIGNALED); + nvmf_ibv_send_wr_init(&rdma_req->wr.send, req, rdma_req->sg_list, IBV_WR_SEND, IBV_SEND_SIGNALED); spdk_trace_record(TRACE_NVMF_IO_COMPLETE, 0, 0, (uintptr_t)req, 0); - rc = ibv_post_send(rdma_conn->cm_id->qp, &wr, &bad_wr); + rc = ibv_post_send(rdma_conn->cm_id->qp, &rdma_req->wr.send, &bad_wr); if (rc) { SPDK_ERRLOG("Failure posting rdma send for NVMf completion, rc = 0x%x\n", rc); }