diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index 4091f2163f..abe12edc6e 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -826,7 +826,7 @@ spdk_nvmf_rdma_qpair_destroy(struct spdk_nvmf_rdma_qpair *rqpair) rdma_destroy_qp(rqpair->cm_id); rdma_destroy_id(rqpair->cm_id); - if (rqpair->poller) { + if (rqpair->poller != NULL && rqpair->srq == NULL) { rqpair->poller->required_num_wr -= MAX_WR_PER_QP(rqpair->max_queue_depth); } } @@ -906,7 +906,7 @@ spdk_nvmf_rdma_qpair_initialize(struct spdk_nvmf_qpair *qpair) ibv_init_attr.cap.max_send_sge = spdk_min(device->attr.max_sge, NVMF_DEFAULT_TX_SGE); ibv_init_attr.cap.max_recv_sge = spdk_min(device->attr.max_sge, NVMF_DEFAULT_RX_SGE); - if (nvmf_rdma_resize_cq(rqpair, device) < 0) { + if (rqpair->srq == NULL && nvmf_rdma_resize_cq(rqpair, device) < 0) { SPDK_ERRLOG("Failed to resize the completion queue. Cannot initialize qpair.\n"); goto error; } @@ -2598,6 +2598,7 @@ spdk_nvmf_rdma_poll_group_create(struct spdk_nvmf_transport *transport) struct spdk_nvmf_rdma_device *device; struct ibv_srq_init_attr srq_init_attr; struct spdk_nvmf_rdma_resource_opts opts; + int num_cqe; rtransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_rdma_transport, transport); @@ -2624,15 +2625,6 @@ spdk_nvmf_rdma_poll_group_create(struct spdk_nvmf_transport *transport) TAILQ_INIT(&poller->qpairs); - poller->cq = ibv_create_cq(device->context, DEFAULT_NVMF_RDMA_CQ_SIZE, poller, NULL, 0); - if (!poller->cq) { - SPDK_ERRLOG("Unable to create completion queue\n"); - spdk_nvmf_rdma_poll_group_destroy(&rgroup->group); - pthread_mutex_unlock(&rtransport->lock); - return NULL; - } - poller->num_cqe = DEFAULT_NVMF_RDMA_CQ_SIZE; - TAILQ_INSERT_TAIL(&rgroup->pollers, poller, link); if (device->attr.max_srq != 0) { poller->max_srq_depth = transport->opts.max_srq_depth; @@ -2662,6 +2654,27 @@ spdk_nvmf_rdma_poll_group_create(struct spdk_nvmf_transport *transport) pthread_mutex_unlock(&rtransport->lock); } } + + /* + * When using an srq, we can limit the completion queue at startup. + * The following formula represents the calculation: + * num_cqe = num_recv + num_data_wr + num_send_wr. + * where num_recv=num_data_wr=and num_send_wr=poller->max_srq_depth + */ + if (poller->srq) { + num_cqe = poller->max_srq_depth * 3; + } else { + num_cqe = DEFAULT_NVMF_RDMA_CQ_SIZE; + } + + poller->cq = ibv_create_cq(device->context, num_cqe, poller, NULL, 0); + if (!poller->cq) { + SPDK_ERRLOG("Unable to create completion queue\n"); + spdk_nvmf_rdma_poll_group_destroy(&rgroup->group); + pthread_mutex_unlock(&rtransport->lock); + return NULL; + } + poller->num_cqe = num_cqe; } pthread_mutex_unlock(&rtransport->lock);