d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

nvme/rdma: Limit max_sges by command capsule size

Browse Source 
According to NVMe over Fabrics spec number of SGLs supported by the
controller is reported in MSDBD. But it is also implicitly limited by
command capsule size (IOCCSZ) since SGL are passed in capsule.

This patch adjusts max_sges to capsule size if required. Adjustment to
MSDBD is also moved to transport layer because it is fabrics specific
parameter and is not valid for PCIe transport.

Signed-off-by: Evgeniy Kochetov <evgeniik@nvidia.com>
Change-Id: I44918eb949345c61242ca50a524d21d04b6ac058
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/11669
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: Shuhei Matsumoto <smatsumoto@nvidia.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
This commit is contained in:
Evgeniy Kochetov 2022-02-18 12:03:30 +02:00 committed by Tomasz Zawadzki
parent 742d818e29
commit 5c80b1e5ab
3 changed files with 44 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
lib/nvme/nvme_ctrlr.c
View File

@ -1986,16 +1986,8 @@ nvme_ctrlr_identify_done(void *arg, const struct spdk_nvme_cpl *cpl)
if (ctrlr->cdata.sgls.supported == 0x2) {
ctrlr->flags |= SPDK_NVME_CTRLR_SGL_REQUIRES_DWORD_ALIGNMENT;
}
/*
* Use MSDBD to ensure our max_sges doesn't exceed what the
* controller supports.
*/
ctrlr->max_sges = nvme_transport_ctrlr_get_max_sges(ctrlr);
if (ctrlr->cdata.nvmf_specific.msdbd != 0) {
ctrlr->max_sges = spdk_min(ctrlr->cdata.nvmf_specific.msdbd, ctrlr->max_sges);
} else {
/* A value 0 indicates no limit. */
}
NVME_CTRLR_DEBUGLOG(ctrlr, "transport max_sges %u\n", ctrlr->max_sges);
}

15
lib/nvme/nvme_rdma.c
View File

@ -2434,8 +2434,21 @@ static uint16_t
nvme_rdma_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr)
{
struct nvme_rdma_ctrlr *rctrlr = nvme_rdma_ctrlr(ctrlr);
uint32_t max_sge = rctrlr->max_sge;
uint32_t max_in_capsule_sge = (ctrlr->cdata.nvmf_specific.ioccsz * 16 -
sizeof(struct spdk_nvme_cmd)) /
sizeof(struct spdk_nvme_sgl_descriptor);
return rctrlr->max_sge;
/* Max SGE is limited by capsule size */
max_sge = spdk_min(max_sge, max_in_capsule_sge);
/* Max SGE may be limited by MSDBD */
if (ctrlr->cdata.nvmf_specific.msdbd != 0) {
max_sge = spdk_min(max_sge, ctrlr->cdata.nvmf_specific.msdbd);
}
/* Max SGE can't be less than 1 */
max_sge = spdk_max(1, max_sge);
return max_sge;
}
static int

29
test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c
View File

@ -1370,6 +1370,34 @@ test_nvme_rdma_poll_group_get_qpair_by_id(void)
STAILQ_REMOVE_HEAD(&group.destroyed_qpairs, link);
}
static void
test_nvme_rdma_ctrlr_get_max_sges(void)
{
struct nvme_rdma_ctrlr rctrlr = {};
rctrlr.ctrlr.trid.trtype = SPDK_NVME_TRANSPORT_RDMA;
rctrlr.max_sge = NVME_RDMA_MAX_SGL_DESCRIPTORS;
rctrlr.ctrlr.cdata.nvmf_specific.msdbd = 16;
rctrlr.ctrlr.cdata.nvmf_specific.ioccsz = 4096;
CU_ASSERT(nvme_rdma_ctrlr_get_max_sges(&rctrlr.ctrlr) == 16);
rctrlr.ctrlr.cdata.nvmf_specific.msdbd = 32;
rctrlr.ctrlr.cdata.nvmf_specific.ioccsz = 4096;
CU_ASSERT(nvme_rdma_ctrlr_get_max_sges(&rctrlr.ctrlr) == 16);
rctrlr.ctrlr.cdata.nvmf_specific.msdbd = 8;
rctrlr.ctrlr.cdata.nvmf_specific.ioccsz = 4096;
CU_ASSERT(nvme_rdma_ctrlr_get_max_sges(&rctrlr.ctrlr) == 8);
rctrlr.ctrlr.cdata.nvmf_specific.msdbd = 16;
rctrlr.ctrlr.cdata.nvmf_specific.ioccsz = 4;
CU_ASSERT(nvme_rdma_ctrlr_get_max_sges(&rctrlr.ctrlr) == 1);
rctrlr.ctrlr.cdata.nvmf_specific.msdbd = 16;
rctrlr.ctrlr.cdata.nvmf_specific.ioccsz = 6;
CU_ASSERT(nvme_rdma_ctrlr_get_max_sges(&rctrlr.ctrlr) == 2);
}
int main(int argc, char **argv)
{
CU_pSuite suite = NULL;
@ -1402,6 +1430,7 @@ int main(int argc, char **argv)
CU_ADD_TEST(suite, test_rdma_ctrlr_get_memory_domains);
CU_ADD_TEST(suite, test_rdma_get_memory_translation);
CU_ADD_TEST(suite, test_nvme_rdma_poll_group_get_qpair_by_id);
CU_ADD_TEST(suite, test_nvme_rdma_ctrlr_get_max_sges);
CU_basic_set_mode(CU_BRM_VERBOSE);
CU_basic_run_tests();