d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

nvme: Update spdk_nvme_ctrlr_get_memory_domain

Browse Source 
Allow to return more than one memory domain.
This change aligns bdev and nvme API and provides
more flexibility for custom transports.

Signed-off-by: Alexey Marchuk <alexeymar@mellanox.com>
Change-Id: Ica9b12ad8463c361be6cb62ee2c0513eec0b486d
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9546
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
This commit is contained in:
Alexey Marchuk 2021-09-20 17:26:57 +03:00 committed by Tomasz Zawadzki
parent ecfc50872f
commit 9381d8d399
12 changed files with 89 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -58,7 +58,7 @@ log level.
Added new functions `spdk_nvme_zns_set_zone_desc_ext` and `spdk_nvme_zns_ext_report_zones`
to set zone descriptor extension and to get extended zone report respectively.
New API `spdk_nvme_ctrlr_get_memory_domain` has been added, it allows to get SPDK memory domain used by nvme controller.
New API `spdk_nvme_ctrlr_get_memory_domains` has been added, it allows to get SPDK memory domains used by nvme controller.
New API functions `spdk_nvme_ns_cmd_readv_ext` and `spdk_nvme_ns_cmd_writev_ext`
have been added. These functions accept `spdk_nvme_ns_cmd_ext_io_opts` structure with extended IO request

21
include/spdk/nvme.h
View File

@ -3779,12 +3779,23 @@ int spdk_nvme_cuse_register(struct spdk_nvme_ctrlr *ctrlr);
int spdk_nvme_cuse_unregister(struct spdk_nvme_ctrlr *ctrlr);
/**
* Get SPDK memory domain used by the given nvme controller.
* Get SPDK memory domains used by the given nvme controller.
*
* The user can call this function with \b domains set to NULL and \b array_size set to 0 to get the
* number of memory domains used by nvme controller
*
* \param ctrlr Opaque handle to the NVMe controller.
* \return Pointer to memory domain used by this controller or NULL
* \param domains Pointer to an array of memory domains to be filled by this function. The user should allocate big enough
* array to keep all memory domains used by nvme controller
* \param array_size size of \b domains array
* \return the number of entries in \b domains array or negated errno. If returned value is bigger than \b array_size passed by the user
* then the user should increase the size of \b domains array and call this function again. There is no guarantees that
* the content of \b domains array is valid in that case.
* -EINVAL if input parameters were invalid
*/
struct spdk_memory_domain *spdk_nvme_ctrlr_get_memory_domain(const struct spdk_nvme_ctrlr *ctrlr);
int spdk_nvme_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr,
struct spdk_memory_domain **domains, int array_size);
/**
* Opaque handle for a transport poll group. Used by the transport function table.
@ -3912,7 +3923,9 @@ struct spdk_nvme_transport_ops {
void (*poll_group_free_stats)(struct spdk_nvme_transport_poll_group *tgroup,
struct spdk_nvme_transport_poll_group_stat *stats);
struct spdk_memory_domain *(*ctrlr_get_memory_domain)(const struct spdk_nvme_ctrlr *ctrlr);
int (*ctrlr_get_memory_domains)(const struct spdk_nvme_ctrlr *ctrlr,
struct spdk_memory_domain **domains,
int array_size);
};
/**

7
lib/nvme/nvme_ctrlr.c
View File

@ -5100,8 +5100,9 @@ spdk_nvme_ctrlr_free_qid(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid)
nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
}
struct spdk_memory_domain *
spdk_nvme_ctrlr_get_memory_domain(const struct spdk_nvme_ctrlr *ctrlr)
int
spdk_nvme_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr,
struct spdk_memory_domain **domains, int array_size)
{
return nvme_transport_ctrlr_get_memory_domain(ctrlr);
return nvme_transport_ctrlr_get_memory_domains(ctrlr, domains, array_size);
}

4
lib/nvme/nvme_internal.h
View File

@ -1483,8 +1483,8 @@ int nvme_transport_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr,
struct spdk_nvme_qpair *qpair);
void nvme_transport_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr,
struct spdk_nvme_qpair *qpair);
struct spdk_memory_domain *nvme_transport_ctrlr_get_memory_domain(const struct spdk_nvme_ctrlr
*ctrlr);
int nvme_transport_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr,
struct spdk_memory_domain **domains, int array_size);
void nvme_transport_qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr);
int nvme_transport_qpair_reset(struct spdk_nvme_qpair *qpair);
int nvme_transport_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req);

13
lib/nvme/nvme_rdma.c
View File

@ -2865,12 +2865,17 @@ nvme_rdma_poll_group_free_stats(struct spdk_nvme_transport_poll_group *tgroup,
free(stats);
}
static struct spdk_memory_domain *
nvme_rdma_ctrlr_get_memory_domain(const struct spdk_nvme_ctrlr *ctrlr)
static int
nvme_rdma_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr,
struct spdk_memory_domain **domains, int array_size)
{
struct nvme_rdma_qpair *rqpair = nvme_rdma_qpair(ctrlr->adminq);
return rqpair->memory_domain->domain;
if (domains && array_size > 0) {
domains[0] = rqpair->memory_domain->domain;
}
return 1;
}
void
@ -2900,7 +2905,7 @@ const struct spdk_nvme_transport_ops rdma_ops = {
.ctrlr_connect_qpair = nvme_rdma_ctrlr_connect_qpair,
.ctrlr_disconnect_qpair = nvme_rdma_ctrlr_disconnect_qpair,
.ctrlr_get_memory_domain = nvme_rdma_ctrlr_get_memory_domain,
.ctrlr_get_memory_domains = nvme_rdma_ctrlr_get_memory_domains,
.qpair_abort_reqs = nvme_rdma_qpair_abort_reqs,
.qpair_reset = nvme_rdma_qpair_reset,

11
lib/nvme/nvme_transport.c
View File

@ -540,17 +540,18 @@ nvme_transport_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk
nvme_qpair_set_state(qpair, NVME_QPAIR_DISCONNECTED);
}
struct spdk_memory_domain *
nvme_transport_ctrlr_get_memory_domain(const struct spdk_nvme_ctrlr *ctrlr)
int
nvme_transport_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr,
struct spdk_memory_domain **domains, int array_size)
{
const struct spdk_nvme_transport *transport = nvme_get_transport(ctrlr->trid.trstring);
assert(transport != NULL);
if (transport->ops.ctrlr_get_memory_domain) {
return transport->ops.ctrlr_get_memory_domain(ctrlr);
if (transport->ops.ctrlr_get_memory_domains) {
return transport->ops.ctrlr_get_memory_domains(ctrlr, domains, array_size);
}
return NULL;
return 0;
}
void

2
lib/nvme/spdk_nvme.map
View File

@ -109,7 +109,7 @@
spdk_nvme_ctrlr_alloc_qid;
spdk_nvme_ctrlr_free_qid;
spdk_nvme_ctrlr_set_remove_cb;
spdk_nvme_ctrlr_get_memory_domain;
spdk_nvme_ctrlr_get_memory_domains;
spdk_nvme_poll_group_create;
spdk_nvme_poll_group_add;

12
module/bdev/nvme/bdev_nvme.c
View File

@ -1513,18 +1513,8 @@ static int
bdev_nvme_get_memory_domains(void *ctx, struct spdk_memory_domain **domains, int array_size)
{
struct nvme_bdev *nbdev = ctx;
struct spdk_memory_domain *domain;
domain = spdk_nvme_ctrlr_get_memory_domain(nbdev->nvme_ns->ctrlr->ctrlr);
if (domain) {
if (array_size > 0 && domains) {
domains[0] = domain;
}
return 1;
}
return 0;
return spdk_nvme_ctrlr_get_memory_domains(nbdev->nvme_ns->ctrlr->ctrlr, domains, array_size);
}
static int

28
test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c
View File

@ -70,13 +70,14 @@ DEFINE_STUB(spdk_nvme_ctrlr_get_flags, uint64_t, (struct spdk_nvme_ctrlr *ctrlr)
DEFINE_STUB(accel_engine_create_cb, int, (void *io_device, void *ctx_buf), 0);
DEFINE_STUB_V(accel_engine_destroy_cb, (void *io_device, void *ctx_buf));
DEFINE_RETURN_MOCK(spdk_nvme_ctrlr_get_memory_domain, struct spdk_memory_domain *);
DEFINE_RETURN_MOCK(spdk_nvme_ctrlr_get_memory_domain, int);
struct spdk_memory_domain *spdk_nvme_ctrlr_get_memory_domain(const struct spdk_nvme_ctrlr *ctrlr)
int spdk_nvme_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr,
struct spdk_memory_domain **domains, int array_size)
{
HANDLE_RETURN_MOCK(spdk_nvme_ctrlr_get_memory_domain);
return NULL;
return 0;
}
struct spdk_io_channel *
@ -2628,31 +2629,18 @@ test_get_memory_domains(void)
struct nvme_ctrlr ctrlr = { .ctrlr = (struct spdk_nvme_ctrlr *) 0xbaadbeef };
struct nvme_ns ns = { .ctrlr = &ctrlr };
struct nvme_bdev nbdev = { .nvme_ns = &ns };
struct spdk_memory_domain *domain = (struct spdk_memory_domain *) 0xf00df00d;
struct spdk_memory_domain *domains[2] = {};
int rc = 0;
/* nvme controller doesn't have a memory domain */
MOCK_SET(spdk_nvme_ctrlr_get_memory_domain, NULL);
/* nvme controller doesn't have memory domainы */
MOCK_SET(spdk_nvme_ctrlr_get_memory_domain, 0);
rc = bdev_nvme_get_memory_domains(&nbdev, domains, 2);
CU_ASSERT(rc == 0)
/* nvme controller has a memory domain but array size is insufficient */
MOCK_SET(spdk_nvme_ctrlr_get_memory_domain, domain);
rc = bdev_nvme_get_memory_domains(&nbdev, domains, 0);
CU_ASSERT(rc == 1);
/* nvme controller has a memory domain but domains array is NULL */
MOCK_SET(spdk_nvme_ctrlr_get_memory_domain, domain);
rc = bdev_nvme_get_memory_domains(&nbdev, domains, 0);
CU_ASSERT(rc == 1);
/* nvme controller has a memory domain */
MOCK_SET(spdk_nvme_ctrlr_get_memory_domain, domain);
rc = bdev_nvme_get_memory_domains(&nbdev, domains, 1);
MOCK_SET(spdk_nvme_ctrlr_get_memory_domain, 1);
rc = bdev_nvme_get_memory_domains(&nbdev, domains, 2);
CU_ASSERT(rc == 1);
CU_ASSERT(domains[0] == domain);
MOCK_CLEAR(spdk_nvme_ctrlr_get_memory_domain);
}

27
test/unit/lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c
View File

@ -82,12 +82,14 @@ DEFINE_STUB(spdk_nvme_ctrlr_cmd_security_send, int, (struct spdk_nvme_ctrlr *ctr
uint8_t secp, uint16_t spsp, uint8_t nssf, void *payload,
uint32_t payload_size, spdk_nvme_cmd_cb cb_fn, void *cb_arg), 0);
DEFINE_RETURN_MOCK(nvme_transport_ctrlr_get_memory_domain, struct spdk_memory_domain *);
struct spdk_memory_domain *
nvme_transport_ctrlr_get_memory_domain(const struct spdk_nvme_ctrlr *ctrlr)
DEFINE_RETURN_MOCK(nvme_transport_ctrlr_get_memory_domains, int);
int
nvme_transport_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr,
struct spdk_memory_domain **domains, int array_size)
{
HANDLE_RETURN_MOCK(nvme_transport_ctrlr_get_memory_domain);
return NULL;
HANDLE_RETURN_MOCK(nvme_transport_ctrlr_get_memory_domains);
return 0;
}
struct spdk_nvme_ctrlr *nvme_transport_ctrlr_construct(const struct spdk_nvme_transport_id *trid,
@ -3142,18 +3144,17 @@ test_nvme_ctrlr_ana_resize(void)
}
static void
test_nvme_ctrlr_get_memory_domain(void)
test_nvme_ctrlr_get_memory_domains(void)
{
struct spdk_nvme_ctrlr ctrlr = {};
struct spdk_memory_domain *domain = (struct spdk_memory_domain *)0xbaadbeef;
MOCK_SET(nvme_transport_ctrlr_get_memory_domain, domain);
CU_ASSERT(spdk_nvme_ctrlr_get_memory_domain(&ctrlr) == domain);
MOCK_SET(nvme_transport_ctrlr_get_memory_domains, 1);
CU_ASSERT(spdk_nvme_ctrlr_get_memory_domains(&ctrlr, NULL, 0) == 1);
MOCK_SET(nvme_transport_ctrlr_get_memory_domain, NULL);
CU_ASSERT(spdk_nvme_ctrlr_get_memory_domain(&ctrlr) == NULL);
MOCK_SET(nvme_transport_ctrlr_get_memory_domains, 0);
CU_ASSERT(spdk_nvme_ctrlr_get_memory_domains(&ctrlr, NULL, 0) == 0);
MOCK_CLEAR(nvme_transport_ctrlr_get_memory_domain);
MOCK_CLEAR(nvme_transport_ctrlr_get_memory_domains);
}
int main(int argc, char **argv)
@ -3208,7 +3209,7 @@ int main(int argc, char **argv)
CU_ADD_TEST(suite, test_nvme_ctrlr_set_supported_log_pages);
CU_ADD_TEST(suite, test_nvme_ctrlr_parse_ana_log_page);
CU_ADD_TEST(suite, test_nvme_ctrlr_ana_resize);
CU_ADD_TEST(suite, test_nvme_ctrlr_get_memory_domain);
CU_ADD_TEST(suite, test_nvme_ctrlr_get_memory_domains);
CU_basic_set_mode(CU_BRM_VERBOSE);
CU_basic_run_tests();

19
test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c
View File

@ -1220,18 +1220,31 @@ test_nvme_rdma_memory_domain(void)
}
static void
test_rdma_ctrlr_get_memory_domain(void)
test_rdma_ctrlr_get_memory_domains(void)
{
struct nvme_rdma_ctrlr rctrlr = {};
struct nvme_rdma_qpair rqpair = {};
struct spdk_memory_domain *domain = (struct spdk_memory_domain *)0xbaadbeef;
struct nvme_rdma_memory_domain rdma_domain = { .domain = domain };
struct spdk_memory_domain *domains[1] = {NULL};
rqpair.memory_domain = &rdma_domain;
rqpair.qpair.trtype = SPDK_NVME_TRANSPORT_RDMA;
rctrlr.ctrlr.adminq = &rqpair.qpair;
CU_ASSERT(nvme_rdma_ctrlr_get_memory_domain(&rctrlr.ctrlr) == domain);
/* Test 1, input domains pointer is NULL */
CU_ASSERT(nvme_rdma_ctrlr_get_memory_domains(&rctrlr.ctrlr, NULL, 1) == 1);
/* Test 2, input array_size is 0 */
CU_ASSERT(nvme_rdma_ctrlr_get_memory_domains(&rctrlr.ctrlr, domains, 0) == 1);
CU_ASSERT(domains[0] == NULL);
/* Test 3, both input domains pointer and array_size are NULL/0 */
CU_ASSERT(nvme_rdma_ctrlr_get_memory_domains(&rctrlr.ctrlr, NULL, 0) == 1);
/* Test 2, input parameters are valid */
CU_ASSERT(nvme_rdma_ctrlr_get_memory_domains(&rctrlr.ctrlr, domains, 1) == 1);
CU_ASSERT(domains[0] == domain);
}
static void
@ -1362,7 +1375,7 @@ int main(int argc, char **argv)
CU_ADD_TEST(suite, test_nvme_rdma_qpair_init);
CU_ADD_TEST(suite, test_nvme_rdma_qpair_submit_request);
CU_ADD_TEST(suite, test_nvme_rdma_memory_domain);
CU_ADD_TEST(suite, test_rdma_ctrlr_get_memory_domain);
CU_ADD_TEST(suite, test_rdma_ctrlr_get_memory_domains);
CU_ADD_TEST(suite, test_rdma_get_memory_translation);
CU_ADD_TEST(suite, test_nvme_rdma_poll_group_get_qpair_by_id);

22
test/unit/lib/nvme/nvme_transport.c/nvme_transport_ut.c
View File

@ -220,17 +220,15 @@ test_nvme_transport_poll_group_add_remove(void)
CU_ASSERT(rc == -ENOENT);
}
static struct spdk_memory_domain *g_ut_ctrlr_memory_domain = (struct spdk_memory_domain *)
0xfeedbeef;
static struct spdk_memory_domain *
g_ut_ctrlr_get_memory_domain(const struct spdk_nvme_ctrlr *ctrlr)
static int
g_ut_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr,
struct spdk_memory_domain **domains, int array_size)
{
return g_ut_ctrlr_memory_domain;
return 1;
}
static void
test_ctrlr_get_memory_domain(void)
test_ctrlr_get_memory_domains(void)
{
struct spdk_nvme_ctrlr ctrlr = {
.trid = {
@ -238,17 +236,17 @@ test_ctrlr_get_memory_domain(void)
}
};
struct spdk_nvme_transport new_transport = {
.ops = { .ctrlr_get_memory_domain = g_ut_ctrlr_get_memory_domain }
.ops = { .ctrlr_get_memory_domains = g_ut_ctrlr_get_memory_domains }
};
ut_construct_transport(&new_transport, "new_transport");
/* transport contains necessary op */
CU_ASSERT(nvme_transport_ctrlr_get_memory_domain(&ctrlr) == g_ut_ctrlr_memory_domain);
CU_ASSERT(nvme_transport_ctrlr_get_memory_domains(&ctrlr, NULL, 0) == 1);
/* transport doesn't contain necessary op */
new_transport.ops.ctrlr_get_memory_domain = NULL;
CU_ASSERT(nvme_transport_ctrlr_get_memory_domain(&ctrlr) == NULL);
new_transport.ops.ctrlr_get_memory_domains = NULL;
CU_ASSERT(nvme_transport_ctrlr_get_memory_domains(&ctrlr, NULL, 0) == 0);
TAILQ_REMOVE(&g_spdk_nvme_transports, &new_transport, link);
}
@ -266,7 +264,7 @@ int main(int argc, char **argv)
CU_ADD_TEST(suite, test_nvme_transport_poll_group_connect_qpair);
CU_ADD_TEST(suite, test_nvme_transport_poll_group_disconnect_qpair);
CU_ADD_TEST(suite, test_nvme_transport_poll_group_add_remove);
CU_ADD_TEST(suite, test_ctrlr_get_memory_domain);
CU_ADD_TEST(suite, test_ctrlr_get_memory_domains);
CU_basic_set_mode(CU_BRM_VERBOSE);
CU_basic_run_tests();