d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

nvme: allocate requests on a per-queue basis

Browse Source 
Change-Id: I7bec816e518a0a6f2e9fb719128c83d4b908d46c
Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>
This commit is contained in:
Daniel Verkamp 2017-04-03 15:52:53 -07:00
parent cd13f280f4
commit 5742e9b9e7
11 changed files with 114 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
include/spdk/nvme.h
View File

@ -107,6 +107,17 @@ struct spdk_nvme_ctrlr_opts {
* Unused for local PCIe-attached NVMe devices.
*/
char hostnqn[SPDK_NVMF_NQN_MAX_LEN + 1];
/**
* The number of requests to allocate for each NVMe I/O queue.
*
* This should be at least as large as io_queue_size.
*
* A single I/O may allocate more than one request, since splitting may be necessary to
* conform to the device's maximum transfer size, PRP list compatibility requirements,
* or driver-assisted striping.
*/
uint32_t io_queue_requests;
};
/**

22
lib/nvme/nvme.c
View File

@ -75,13 +75,15 @@ nvme_allocate_request(struct spdk_nvme_qpair *qpair,
const struct nvme_payload *payload, uint32_t payload_size,
spdk_nvme_cmd_cb cb_fn, void *cb_arg)
{
struct nvme_request *req = NULL;
struct nvme_request *req;
req = spdk_mempool_get(g_spdk_nvme_driver->request_mempool);
req = STAILQ_FIRST(&qpair->free_req);
if (req == NULL) {
return req;
}
STAILQ_REMOVE_HEAD(&qpair->free_req, stailq);
/*
* Only memset up to (but not including) the children
* TAILQ_ENTRY. children, and following members, are
@ -190,8 +192,9 @@ nvme_free_request(struct nvme_request *req)
{
assert(req != NULL);
assert(req->num_children == 0);
assert(req->qpair != NULL);
spdk_mempool_put(g_spdk_nvme_driver->request_mempool, req);
STAILQ_INSERT_HEAD(&req->qpair->free_req, req, stailq);
}
int
@ -285,19 +288,6 @@ nvme_driver_init(void)
TAILQ_INIT(&g_spdk_nvme_driver->init_ctrlrs);
TAILQ_INIT(&g_spdk_nvme_driver->attached_ctrlrs);
g_spdk_nvme_driver->request_mempool = spdk_mempool_create("nvme_request", 8192,
sizeof(struct nvme_request), 128, SPDK_ENV_SOCKET_ID_ANY);
if (g_spdk_nvme_driver->request_mempool == NULL) {
SPDK_ERRLOG("unable to allocate pool of requests\n");
nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
pthread_mutex_destroy(&g_spdk_nvme_driver->lock);
spdk_memzone_free(SPDK_NVME_DRIVER_NAME);
return -1;
}
nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock);
return ret;

5
lib/nvme/nvme_ctrlr.c
View File

@ -82,6 +82,7 @@ spdk_nvme_ctrlr_opts_set_defaults(struct spdk_nvme_ctrlr_opts *opts)
opts->keep_alive_timeout_ms = 10 * 1000;
opts->io_queue_size = DEFAULT_IO_QUEUE_SIZE;
strncpy(opts->hostnqn, DEFAULT_HOSTNQN, sizeof(opts->hostnqn));
opts->io_queue_requests = DEFAULT_IO_QUEUE_REQUESTS;
}
/**
@ -223,6 +224,8 @@ spdk_nvme_ctrlr_free_io_qpair(struct spdk_nvme_qpair *qpair)
TAILQ_REMOVE(&ctrlr->active_io_qpairs, qpair, tailq);
spdk_bit_array_set(ctrlr->free_io_qids, qpair->id);
spdk_free(qpair->req_buf);
if (nvme_transport_ctrlr_delete_io_qpair(ctrlr, qpair)) {
nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
return -1;
@ -1350,6 +1353,8 @@ nvme_ctrlr_init_cap(struct spdk_nvme_ctrlr *ctrlr, const union spdk_nvme_cap_reg
ctrlr->opts.io_queue_size = spdk_max(ctrlr->opts.io_queue_size, SPDK_NVME_IO_QUEUE_MIN_ENTRIES);
ctrlr->opts.io_queue_size = spdk_min(ctrlr->opts.io_queue_size, ctrlr->cap.bits.mqes + 1u);
ctrlr->opts.io_queue_size = spdk_min(ctrlr->opts.io_queue_size, max_io_queue_size);
ctrlr->opts.io_queue_requests = spdk_max(ctrlr->opts.io_queue_requests, ctrlr->opts.io_queue_size);
}
void

10
lib/nvme/nvme_internal.h
View File

@ -112,6 +112,9 @@
#define DEFAULT_MAX_IO_QUEUES (1024)
#define DEFAULT_IO_QUEUE_SIZE (256)
#define DEFAULT_ADMIN_QUEUE_REQUESTS (32)
#define DEFAULT_IO_QUEUE_REQUESTS (512)
#define DEFAULT_HOSTNQN "nqn.2016-06.io.spdk:host"
enum nvme_payload_type {
@ -254,6 +257,7 @@ struct nvme_async_event_request {
};
struct spdk_nvme_qpair {
STAILQ_HEAD(, nvme_request) free_req;
STAILQ_HEAD(, nvme_request) queued_req;
enum spdk_nvme_transport_type trtype;
@ -277,6 +281,8 @@ struct spdk_nvme_qpair {
/* List entry for spdk_nvme_ctrlr_process::allocated_io_qpairs */
TAILQ_ENTRY(spdk_nvme_qpair) per_process_tailq;
void *req_buf;
};
struct spdk_nvme_ns {
@ -458,7 +464,6 @@ struct nvme_driver {
pthread_mutex_t lock;
TAILQ_HEAD(, spdk_nvme_ctrlr) init_ctrlrs;
TAILQ_HEAD(, spdk_nvme_ctrlr) attached_ctrlrs;
struct spdk_mempool *request_mempool;
bool initialized;
};
@ -547,7 +552,8 @@ int nvme_ctrlr_get_cap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cap_regist
void nvme_ctrlr_init_cap(struct spdk_nvme_ctrlr *ctrlr, const union spdk_nvme_cap_register *cap);
int nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id,
struct spdk_nvme_ctrlr *ctrlr,
enum spdk_nvme_qprio qprio);
enum spdk_nvme_qprio qprio,
uint32_t num_requests);
void nvme_qpair_enable(struct spdk_nvme_qpair *qpair);
void nvme_qpair_disable(struct spdk_nvme_qpair *qpair);
int nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair,

5
lib/nvme/nvme_pcie.c
View File

@ -573,7 +573,8 @@ nvme_pcie_ctrlr_construct_admin_qpair(struct spdk_nvme_ctrlr *ctrlr)
rc = nvme_qpair_init(ctrlr->adminq,
0, /* qpair ID */
ctrlr,
SPDK_NVME_QPRIO_URGENT);
SPDK_NVME_QPRIO_URGENT,
NVME_ADMIN_ENTRIES);
if (rc != 0) {
return rc;
}
@ -1418,7 +1419,7 @@ nvme_pcie_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid,
qpair = &pqpair->qpair;
rc = nvme_qpair_init(qpair, qid, ctrlr, qprio);
rc = nvme_qpair_init(qpair, qid, ctrlr, qprio, ctrlr->opts.io_queue_requests);
if (rc != 0) {
nvme_pcie_qpair_destroy(qpair);
return NULL;

20
lib/nvme/nvme_qpair.c
View File

@ -361,8 +361,12 @@ spdk_nvme_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_
int
nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id,
struct spdk_nvme_ctrlr *ctrlr,
enum spdk_nvme_qprio qprio)
enum spdk_nvme_qprio qprio,
uint32_t num_requests)
{
size_t req_size_padded;
uint32_t i;
qpair->id = id;
qpair->qprio = qprio;
@ -372,8 +376,22 @@ nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id,
qpair->ctrlr = ctrlr;
qpair->trtype = ctrlr->trid.trtype;
STAILQ_INIT(&qpair->free_req);
STAILQ_INIT(&qpair->queued_req);
req_size_padded = (sizeof(struct nvme_request) + 63) & ~(size_t)63;
qpair->req_buf = spdk_zmalloc(req_size_padded * num_requests, 64, NULL);
if (qpair->req_buf == NULL) {
return -ENOMEM;
}
for (i = 0; i < num_requests; i++) {
struct nvme_request *req = qpair->req_buf + i * req_size_padded;
STAILQ_INSERT_HEAD(&qpair->free_req, req, stailq);
}
return 0;
}

10
lib/nvme/nvme_rdma.c
View File

@ -981,7 +981,8 @@ nvme_rdma_fabric_prop_get_cmd(struct spdk_nvme_ctrlr *ctrlr,
static struct spdk_nvme_qpair *
nvme_rdma_ctrlr_create_qpair(struct spdk_nvme_ctrlr *ctrlr,
uint16_t qid, uint32_t qsize,
enum spdk_nvme_qprio qprio)
enum spdk_nvme_qprio qprio,
uint32_t num_requests)
{
struct nvme_rdma_qpair *rqpair;
struct spdk_nvme_qpair *qpair;
@ -997,7 +998,7 @@ nvme_rdma_ctrlr_create_qpair(struct spdk_nvme_ctrlr *ctrlr,
qpair = &rqpair->qpair;
rc = nvme_qpair_init(qpair, qid, ctrlr, qprio);
rc = nvme_qpair_init(qpair, qid, ctrlr, qprio, num_requests);
if (rc != 0) {
return NULL;
}
@ -1050,7 +1051,8 @@ struct spdk_nvme_qpair *
nvme_rdma_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid,
enum spdk_nvme_qprio qprio)
{
return nvme_rdma_ctrlr_create_qpair(ctrlr, qid, ctrlr->opts.io_queue_size, qprio);
return nvme_rdma_ctrlr_create_qpair(ctrlr, qid, ctrlr->opts.io_queue_size, qprio,
ctrlr->opts.io_queue_requests);
}
int
@ -1232,7 +1234,7 @@ struct spdk_nvme_ctrlr *nvme_rdma_ctrlr_construct(const struct spdk_nvme_transpo
}
rctrlr->ctrlr.adminq = nvme_rdma_ctrlr_create_qpair(&rctrlr->ctrlr, 0,
SPDK_NVMF_MIN_ADMIN_QUEUE_ENTRIES, 0);
SPDK_NVMF_MIN_ADMIN_QUEUE_ENTRIES, 0, SPDK_NVMF_MIN_ADMIN_QUEUE_ENTRIES);
if (!rctrlr->ctrlr.adminq) {
SPDK_ERRLOG("failed to create admin qpair\n");
return NULL;

10
test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c
View File

@ -48,7 +48,6 @@ struct spdk_trace_flag SPDK_TRACE_NVME = {
struct nvme_driver _g_nvme_driver = {
.lock = PTHREAD_MUTEX_INITIALIZER,
.request_mempool = NULL,
};
uint64_t g_ut_tsc = 0;
@ -156,7 +155,8 @@ nvme_transport_qpair_reset(struct spdk_nvme_qpair *qpair)
int nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id,
struct spdk_nvme_ctrlr *ctrlr,
enum spdk_nvme_qprio qprio)
enum spdk_nvme_qprio qprio,
uint32_t num_requests)
{
qpair->id = id;
qpair->qprio = qprio;
@ -210,7 +210,7 @@ nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *re
* Free the request here so it does not leak.
* For the purposes of this unit test, we don't need to bother emulating request submission.
*/
spdk_mempool_put(_g_nvme_driver.request_mempool, req);
free(req);
return 0;
}
@ -334,7 +334,7 @@ nvme_allocate_request(struct spdk_nvme_qpair *qpair,
void *cb_arg)
{
struct nvme_request *req = NULL;
req = spdk_mempool_get(_g_nvme_driver.request_mempool);
req = calloc(1, sizeof(*req));
if (req != NULL) {
memset(req, 0, offsetof(struct nvme_request, children));
@ -372,7 +372,7 @@ nvme_allocate_request_null(struct spdk_nvme_qpair *qpair, spdk_nvme_cmd_cb cb_fn
void
nvme_free_request(struct nvme_request *req)
{
spdk_mempool_put(_g_nvme_driver.request_mempool, req);
free(req);
}
static void

40
test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c
View File

@ -39,7 +39,6 @@
static struct nvme_driver _g_nvme_driver = {
.lock = PTHREAD_MUTEX_INITIALIZER,
.request_mempool = NULL,
};
static struct nvme_request *g_request = NULL;
@ -194,6 +193,9 @@ prepare_for_test(struct spdk_nvme_ns *ns, struct spdk_nvme_ctrlr *ctrlr,
uint32_t sector_size, uint32_t md_size, uint32_t max_xfer_size,
uint32_t stripe_size, bool extended_lba)
{
uint32_t num_requests = 32;
uint32_t i;
ctrlr->max_xfer_size = max_xfer_size;
/*
* Clear the flags field - we especially want to make sure the SGL_SUPPORTED flag is not set
@ -214,10 +216,24 @@ prepare_for_test(struct spdk_nvme_ns *ns, struct spdk_nvme_ctrlr *ctrlr,
ns->sectors_per_stripe = ns->stripe_size / ns->extended_lba_size;
memset(qpair, 0, sizeof(*qpair));
qpair->req_buf = calloc(num_requests, sizeof(struct nvme_request));
SPDK_CU_ASSERT_FATAL(qpair->req_buf != NULL);
for (i = 0; i < num_requests; i++) {
struct nvme_request *req = qpair->req_buf + i * sizeof(struct nvme_request);
STAILQ_INSERT_HEAD(&qpair->free_req, req, stailq);
}
g_request = NULL;
}
static void
cleanup_after_test(struct spdk_nvme_qpair *qpair)
{
free(qpair->req_buf);
}
static void
nvme_cmd_interpret_rw(const struct spdk_nvme_cmd *cmd,
uint64_t *lba, uint32_t *num_blocks)
@ -254,6 +270,7 @@ split_test(void)
free(payload);
nvme_free_request(g_request);
cleanup_after_test(&qpair);
}
static void
@ -308,6 +325,7 @@ split_test2(void)
free(payload);
nvme_free_request(g_request);
cleanup_after_test(&qpair);
}
static void
@ -364,6 +382,7 @@ split_test3(void)
free(payload);
nvme_free_request(g_request);
cleanup_after_test(&qpair);
}
static void
@ -441,6 +460,7 @@ split_test4(void)
free(payload);
nvme_free_request(g_request);
cleanup_after_test(&qpair);
}
static void
@ -496,6 +516,7 @@ test_cmd_child_request(void)
free(payload);
nvme_free_request(g_request);
cleanup_after_test(&qpair);
}
static void
@ -517,6 +538,7 @@ test_nvme_ns_cmd_flush(void)
CU_ASSERT(g_request->cmd.nsid == ns.id);
nvme_free_request(g_request);
cleanup_after_test(&qpair);
}
static void
@ -543,6 +565,7 @@ test_nvme_ns_cmd_write_zeroes(void)
CU_ASSERT_EQUAL(cmd_lba_count, 2);
nvme_free_request(g_request);
cleanup_after_test(&qpair);
}
static void
@ -592,6 +615,7 @@ test_nvme_ns_cmd_dataset_management(void)
rc = spdk_nvme_ns_cmd_dataset_management(&ns, &qpair, SPDK_NVME_DSM_ATTR_DEALLOCATE,
NULL, 0, cb_fn, cb_arg);
CU_ASSERT(rc != 0);
cleanup_after_test(&qpair);
}
static void
@ -626,6 +650,7 @@ test_nvme_ns_cmd_readv(void)
free(cb_arg);
nvme_free_request(g_request);
cleanup_after_test(&qpair);
}
static void
@ -660,6 +685,7 @@ test_nvme_ns_cmd_writev(void)
free(cb_arg);
nvme_free_request(g_request);
cleanup_after_test(&qpair);
}
static void
@ -695,7 +721,7 @@ test_io_flags(void)
nvme_free_request(g_request);
free(payload);
cleanup_after_test(&qpair);
}
static void
@ -733,6 +759,7 @@ test_nvme_ns_cmd_reservation_register(void)
spdk_free(g_request->payload.u.contig);
nvme_free_request(g_request);
free(payload);
cleanup_after_test(&qpair);
}
static void
@ -770,6 +797,7 @@ test_nvme_ns_cmd_reservation_release(void)
spdk_free(g_request->payload.u.contig);
nvme_free_request(g_request);
free(payload);
cleanup_after_test(&qpair);
}
static void
@ -807,6 +835,7 @@ test_nvme_ns_cmd_reservation_acquire(void)
spdk_free(g_request->payload.u.contig);
nvme_free_request(g_request);
free(payload);
cleanup_after_test(&qpair);
}
static void
@ -838,6 +867,7 @@ test_nvme_ns_cmd_reservation_report(void)
spdk_free(g_request->payload.u.contig);
nvme_free_request(g_request);
free(payload);
cleanup_after_test(&qpair);
}
static void
@ -881,6 +911,7 @@ test_nvme_ns_cmd_write_with_md(void)
CU_ASSERT(g_request->payload_size == 256 * 512);
nvme_free_request(g_request);
cleanup_after_test(&qpair);
/*
* 512 byte data + 128 byte metadata
@ -915,6 +946,7 @@ test_nvme_ns_cmd_write_with_md(void)
nvme_request_free_children(g_request);
nvme_free_request(g_request);
cleanup_after_test(&qpair);
/*
* 512 byte data + 8 byte metadata
@ -950,6 +982,7 @@ test_nvme_ns_cmd_write_with_md(void)
nvme_request_free_children(g_request);
nvme_free_request(g_request);
cleanup_after_test(&qpair);
/*
* 512 byte data + 8 byte metadata
@ -988,6 +1021,7 @@ test_nvme_ns_cmd_write_with_md(void)
nvme_request_free_children(g_request);
nvme_free_request(g_request);
cleanup_after_test(&qpair);
/*
* 512 byte data + 8 byte metadata
@ -1010,6 +1044,7 @@ test_nvme_ns_cmd_write_with_md(void)
CU_ASSERT(g_request->payload_size == 256 * 512);
nvme_free_request(g_request);
cleanup_after_test(&qpair);
/*
* 512 byte data + 8 byte metadata
@ -1046,6 +1081,7 @@ test_nvme_ns_cmd_write_with_md(void)
nvme_request_free_children(g_request);
nvme_free_request(g_request);
cleanup_after_test(&qpair);
free(buffer);
free(metadata);

6
test/lib/nvme/unit/nvme_pcie_c/nvme_pcie_ut.c
View File

@ -47,7 +47,6 @@ struct spdk_trace_flag SPDK_TRACE_NVME = {
static struct nvme_driver _g_nvme_driver = {
.lock = PTHREAD_MUTEX_INITIALIZER,
.request_mempool = NULL,
};
struct nvme_driver *g_spdk_nvme_driver = &_g_nvme_driver;
@ -92,7 +91,8 @@ spdk_pci_device_get_id(struct spdk_pci_device *dev)
int
nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id,
struct spdk_nvme_ctrlr *ctrlr,
enum spdk_nvme_qprio qprio)
enum spdk_nvme_qprio qprio,
uint32_t num_requests)
{
abort();
}
@ -347,7 +347,7 @@ ut_insert_cq_entry(struct spdk_nvme_qpair *qpair, uint32_t slot)
struct nvme_tracker *tr;
struct spdk_nvme_cpl *cpl;
req = spdk_mempool_get(_g_nvme_driver.request_mempool);
req = calloc(1, sizeof(*req));
SPDK_CU_ASSERT_FATAL(req != NULL);
memset(req, 0, sizeof(*req));

17
test/lib/nvme/unit/nvme_qpair_c/nvme_qpair_ut.c
View File

@ -44,7 +44,6 @@ bool trace_flag = false;
struct nvme_driver _g_nvme_driver = {
.lock = PTHREAD_MUTEX_INITIALIZER,
.request_mempool = NULL,
};
struct nvme_request *
@ -53,14 +52,15 @@ nvme_allocate_request(struct spdk_nvme_qpair *qpair,
spdk_nvme_cmd_cb cb_fn,
void *cb_arg)
{
struct nvme_request *req = NULL;
req = spdk_mempool_get(_g_nvme_driver.request_mempool);
struct nvme_request *req;
req = STAILQ_FIRST(&qpair->free_req);
if (req == NULL) {
return req;
return NULL;
}
STAILQ_REMOVE_HEAD(&qpair->free_req, stailq);
/*
* Only memset up to (but not including) the children
* TAILQ_ENTRY. children, and following members, are
@ -101,7 +101,9 @@ nvme_allocate_request_null(struct spdk_nvme_qpair *qpair, spdk_nvme_cmd_cb cb_fn
void
nvme_free_request(struct nvme_request *req)
{
spdk_mempool_put(_g_nvme_driver.request_mempool, req);
SPDK_CU_ASSERT_FATAL(req != NULL);
SPDK_CU_ASSERT_FATAL(req->qpair != NULL);
STAILQ_INSERT_HEAD(&req->qpair->free_req, req, stailq);
}
void
@ -158,12 +160,13 @@ prepare_submit_request_test(struct spdk_nvme_qpair *qpair,
ctrlr->free_io_qids = NULL;
TAILQ_INIT(&ctrlr->active_io_qpairs);
TAILQ_INIT(&ctrlr->active_procs);
nvme_qpair_init(qpair, 1, ctrlr, 0);
nvme_qpair_init(qpair, 1, ctrlr, 0, 32);
}
static void
cleanup_submit_request_test(struct spdk_nvme_qpair *qpair)
{
free(qpair->req_buf);
}
static void