From 5742e9b9e782b9666e10c9224389e4d015c3cdee Mon Sep 17 00:00:00 2001 From: Daniel Verkamp Date: Mon, 3 Apr 2017 15:52:53 -0700 Subject: [PATCH] nvme: allocate requests on a per-queue basis Change-Id: I7bec816e518a0a6f2e9fb719128c83d4b908d46c Signed-off-by: Daniel Verkamp --- include/spdk/nvme.h | 11 +++++ lib/nvme/nvme.c | 22 +++------- lib/nvme/nvme_ctrlr.c | 5 +++ lib/nvme/nvme_internal.h | 10 ++++- lib/nvme/nvme_pcie.c | 5 ++- lib/nvme/nvme_qpair.c | 20 +++++++++- lib/nvme/nvme_rdma.c | 10 +++-- .../nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c | 10 ++--- .../nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c | 40 ++++++++++++++++++- test/lib/nvme/unit/nvme_pcie_c/nvme_pcie_ut.c | 6 +-- .../nvme/unit/nvme_qpair_c/nvme_qpair_ut.c | 17 ++++---- 11 files changed, 114 insertions(+), 42 deletions(-) diff --git a/include/spdk/nvme.h b/include/spdk/nvme.h index 86b0d0e85f..79958ec621 100644 --- a/include/spdk/nvme.h +++ b/include/spdk/nvme.h @@ -107,6 +107,17 @@ struct spdk_nvme_ctrlr_opts { * Unused for local PCIe-attached NVMe devices. */ char hostnqn[SPDK_NVMF_NQN_MAX_LEN + 1]; + + /** + * The number of requests to allocate for each NVMe I/O queue. + * + * This should be at least as large as io_queue_size. + * + * A single I/O may allocate more than one request, since splitting may be necessary to + * conform to the device's maximum transfer size, PRP list compatibility requirements, + * or driver-assisted striping. + */ + uint32_t io_queue_requests; }; /** diff --git a/lib/nvme/nvme.c b/lib/nvme/nvme.c index 1fb8fe8dbf..c6a80f02f2 100644 --- a/lib/nvme/nvme.c +++ b/lib/nvme/nvme.c @@ -75,13 +75,15 @@ nvme_allocate_request(struct spdk_nvme_qpair *qpair, const struct nvme_payload *payload, uint32_t payload_size, spdk_nvme_cmd_cb cb_fn, void *cb_arg) { - struct nvme_request *req = NULL; + struct nvme_request *req; - req = spdk_mempool_get(g_spdk_nvme_driver->request_mempool); + req = STAILQ_FIRST(&qpair->free_req); if (req == NULL) { return req; } + STAILQ_REMOVE_HEAD(&qpair->free_req, stailq); + /* * Only memset up to (but not including) the children * TAILQ_ENTRY. children, and following members, are @@ -190,8 +192,9 @@ nvme_free_request(struct nvme_request *req) { assert(req != NULL); assert(req->num_children == 0); + assert(req->qpair != NULL); - spdk_mempool_put(g_spdk_nvme_driver->request_mempool, req); + STAILQ_INSERT_HEAD(&req->qpair->free_req, req, stailq); } int @@ -285,19 +288,6 @@ nvme_driver_init(void) TAILQ_INIT(&g_spdk_nvme_driver->init_ctrlrs); TAILQ_INIT(&g_spdk_nvme_driver->attached_ctrlrs); - g_spdk_nvme_driver->request_mempool = spdk_mempool_create("nvme_request", 8192, - sizeof(struct nvme_request), 128, SPDK_ENV_SOCKET_ID_ANY); - if (g_spdk_nvme_driver->request_mempool == NULL) { - SPDK_ERRLOG("unable to allocate pool of requests\n"); - - nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock); - pthread_mutex_destroy(&g_spdk_nvme_driver->lock); - - spdk_memzone_free(SPDK_NVME_DRIVER_NAME); - - return -1; - } - nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock); return ret; diff --git a/lib/nvme/nvme_ctrlr.c b/lib/nvme/nvme_ctrlr.c index 9dab0e7692..66f947d2c3 100644 --- a/lib/nvme/nvme_ctrlr.c +++ b/lib/nvme/nvme_ctrlr.c @@ -82,6 +82,7 @@ spdk_nvme_ctrlr_opts_set_defaults(struct spdk_nvme_ctrlr_opts *opts) opts->keep_alive_timeout_ms = 10 * 1000; opts->io_queue_size = DEFAULT_IO_QUEUE_SIZE; strncpy(opts->hostnqn, DEFAULT_HOSTNQN, sizeof(opts->hostnqn)); + opts->io_queue_requests = DEFAULT_IO_QUEUE_REQUESTS; } /** @@ -223,6 +224,8 @@ spdk_nvme_ctrlr_free_io_qpair(struct spdk_nvme_qpair *qpair) TAILQ_REMOVE(&ctrlr->active_io_qpairs, qpair, tailq); spdk_bit_array_set(ctrlr->free_io_qids, qpair->id); + spdk_free(qpair->req_buf); + if (nvme_transport_ctrlr_delete_io_qpair(ctrlr, qpair)) { nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); return -1; @@ -1350,6 +1353,8 @@ nvme_ctrlr_init_cap(struct spdk_nvme_ctrlr *ctrlr, const union spdk_nvme_cap_reg ctrlr->opts.io_queue_size = spdk_max(ctrlr->opts.io_queue_size, SPDK_NVME_IO_QUEUE_MIN_ENTRIES); ctrlr->opts.io_queue_size = spdk_min(ctrlr->opts.io_queue_size, ctrlr->cap.bits.mqes + 1u); ctrlr->opts.io_queue_size = spdk_min(ctrlr->opts.io_queue_size, max_io_queue_size); + + ctrlr->opts.io_queue_requests = spdk_max(ctrlr->opts.io_queue_requests, ctrlr->opts.io_queue_size); } void diff --git a/lib/nvme/nvme_internal.h b/lib/nvme/nvme_internal.h index 38d0fb0722..9341246663 100644 --- a/lib/nvme/nvme_internal.h +++ b/lib/nvme/nvme_internal.h @@ -112,6 +112,9 @@ #define DEFAULT_MAX_IO_QUEUES (1024) #define DEFAULT_IO_QUEUE_SIZE (256) +#define DEFAULT_ADMIN_QUEUE_REQUESTS (32) +#define DEFAULT_IO_QUEUE_REQUESTS (512) + #define DEFAULT_HOSTNQN "nqn.2016-06.io.spdk:host" enum nvme_payload_type { @@ -254,6 +257,7 @@ struct nvme_async_event_request { }; struct spdk_nvme_qpair { + STAILQ_HEAD(, nvme_request) free_req; STAILQ_HEAD(, nvme_request) queued_req; enum spdk_nvme_transport_type trtype; @@ -277,6 +281,8 @@ struct spdk_nvme_qpair { /* List entry for spdk_nvme_ctrlr_process::allocated_io_qpairs */ TAILQ_ENTRY(spdk_nvme_qpair) per_process_tailq; + + void *req_buf; }; struct spdk_nvme_ns { @@ -458,7 +464,6 @@ struct nvme_driver { pthread_mutex_t lock; TAILQ_HEAD(, spdk_nvme_ctrlr) init_ctrlrs; TAILQ_HEAD(, spdk_nvme_ctrlr) attached_ctrlrs; - struct spdk_mempool *request_mempool; bool initialized; }; @@ -547,7 +552,8 @@ int nvme_ctrlr_get_cap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cap_regist void nvme_ctrlr_init_cap(struct spdk_nvme_ctrlr *ctrlr, const union spdk_nvme_cap_register *cap); int nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id, struct spdk_nvme_ctrlr *ctrlr, - enum spdk_nvme_qprio qprio); + enum spdk_nvme_qprio qprio, + uint32_t num_requests); void nvme_qpair_enable(struct spdk_nvme_qpair *qpair); void nvme_qpair_disable(struct spdk_nvme_qpair *qpair); int nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, diff --git a/lib/nvme/nvme_pcie.c b/lib/nvme/nvme_pcie.c index d4b9294363..55d0303f54 100644 --- a/lib/nvme/nvme_pcie.c +++ b/lib/nvme/nvme_pcie.c @@ -573,7 +573,8 @@ nvme_pcie_ctrlr_construct_admin_qpair(struct spdk_nvme_ctrlr *ctrlr) rc = nvme_qpair_init(ctrlr->adminq, 0, /* qpair ID */ ctrlr, - SPDK_NVME_QPRIO_URGENT); + SPDK_NVME_QPRIO_URGENT, + NVME_ADMIN_ENTRIES); if (rc != 0) { return rc; } @@ -1418,7 +1419,7 @@ nvme_pcie_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid, qpair = &pqpair->qpair; - rc = nvme_qpair_init(qpair, qid, ctrlr, qprio); + rc = nvme_qpair_init(qpair, qid, ctrlr, qprio, ctrlr->opts.io_queue_requests); if (rc != 0) { nvme_pcie_qpair_destroy(qpair); return NULL; diff --git a/lib/nvme/nvme_qpair.c b/lib/nvme/nvme_qpair.c index 5e06b784ba..853fb786b7 100644 --- a/lib/nvme/nvme_qpair.c +++ b/lib/nvme/nvme_qpair.c @@ -361,8 +361,12 @@ spdk_nvme_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_ int nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id, struct spdk_nvme_ctrlr *ctrlr, - enum spdk_nvme_qprio qprio) + enum spdk_nvme_qprio qprio, + uint32_t num_requests) { + size_t req_size_padded; + uint32_t i; + qpair->id = id; qpair->qprio = qprio; @@ -372,8 +376,22 @@ nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id, qpair->ctrlr = ctrlr; qpair->trtype = ctrlr->trid.trtype; + STAILQ_INIT(&qpair->free_req); STAILQ_INIT(&qpair->queued_req); + req_size_padded = (sizeof(struct nvme_request) + 63) & ~(size_t)63; + + qpair->req_buf = spdk_zmalloc(req_size_padded * num_requests, 64, NULL); + if (qpair->req_buf == NULL) { + return -ENOMEM; + } + + for (i = 0; i < num_requests; i++) { + struct nvme_request *req = qpair->req_buf + i * req_size_padded; + + STAILQ_INSERT_HEAD(&qpair->free_req, req, stailq); + } + return 0; } diff --git a/lib/nvme/nvme_rdma.c b/lib/nvme/nvme_rdma.c index d186c46014..1fefd5fd95 100644 --- a/lib/nvme/nvme_rdma.c +++ b/lib/nvme/nvme_rdma.c @@ -981,7 +981,8 @@ nvme_rdma_fabric_prop_get_cmd(struct spdk_nvme_ctrlr *ctrlr, static struct spdk_nvme_qpair * nvme_rdma_ctrlr_create_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid, uint32_t qsize, - enum spdk_nvme_qprio qprio) + enum spdk_nvme_qprio qprio, + uint32_t num_requests) { struct nvme_rdma_qpair *rqpair; struct spdk_nvme_qpair *qpair; @@ -997,7 +998,7 @@ nvme_rdma_ctrlr_create_qpair(struct spdk_nvme_ctrlr *ctrlr, qpair = &rqpair->qpair; - rc = nvme_qpair_init(qpair, qid, ctrlr, qprio); + rc = nvme_qpair_init(qpair, qid, ctrlr, qprio, num_requests); if (rc != 0) { return NULL; } @@ -1050,7 +1051,8 @@ struct spdk_nvme_qpair * nvme_rdma_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid, enum spdk_nvme_qprio qprio) { - return nvme_rdma_ctrlr_create_qpair(ctrlr, qid, ctrlr->opts.io_queue_size, qprio); + return nvme_rdma_ctrlr_create_qpair(ctrlr, qid, ctrlr->opts.io_queue_size, qprio, + ctrlr->opts.io_queue_requests); } int @@ -1232,7 +1234,7 @@ struct spdk_nvme_ctrlr *nvme_rdma_ctrlr_construct(const struct spdk_nvme_transpo } rctrlr->ctrlr.adminq = nvme_rdma_ctrlr_create_qpair(&rctrlr->ctrlr, 0, - SPDK_NVMF_MIN_ADMIN_QUEUE_ENTRIES, 0); + SPDK_NVMF_MIN_ADMIN_QUEUE_ENTRIES, 0, SPDK_NVMF_MIN_ADMIN_QUEUE_ENTRIES); if (!rctrlr->ctrlr.adminq) { SPDK_ERRLOG("failed to create admin qpair\n"); return NULL; diff --git a/test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c b/test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c index d19d33c1e3..856f1a176f 100644 --- a/test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c +++ b/test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c @@ -48,7 +48,6 @@ struct spdk_trace_flag SPDK_TRACE_NVME = { struct nvme_driver _g_nvme_driver = { .lock = PTHREAD_MUTEX_INITIALIZER, - .request_mempool = NULL, }; uint64_t g_ut_tsc = 0; @@ -156,7 +155,8 @@ nvme_transport_qpair_reset(struct spdk_nvme_qpair *qpair) int nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id, struct spdk_nvme_ctrlr *ctrlr, - enum spdk_nvme_qprio qprio) + enum spdk_nvme_qprio qprio, + uint32_t num_requests) { qpair->id = id; qpair->qprio = qprio; @@ -210,7 +210,7 @@ nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *re * Free the request here so it does not leak. * For the purposes of this unit test, we don't need to bother emulating request submission. */ - spdk_mempool_put(_g_nvme_driver.request_mempool, req); + free(req); return 0; } @@ -334,7 +334,7 @@ nvme_allocate_request(struct spdk_nvme_qpair *qpair, void *cb_arg) { struct nvme_request *req = NULL; - req = spdk_mempool_get(_g_nvme_driver.request_mempool); + req = calloc(1, sizeof(*req)); if (req != NULL) { memset(req, 0, offsetof(struct nvme_request, children)); @@ -372,7 +372,7 @@ nvme_allocate_request_null(struct spdk_nvme_qpair *qpair, spdk_nvme_cmd_cb cb_fn void nvme_free_request(struct nvme_request *req) { - spdk_mempool_put(_g_nvme_driver.request_mempool, req); + free(req); } static void diff --git a/test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c b/test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c index 66fe064c1a..0d296bd4a2 100644 --- a/test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c +++ b/test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c @@ -39,7 +39,6 @@ static struct nvme_driver _g_nvme_driver = { .lock = PTHREAD_MUTEX_INITIALIZER, - .request_mempool = NULL, }; static struct nvme_request *g_request = NULL; @@ -194,6 +193,9 @@ prepare_for_test(struct spdk_nvme_ns *ns, struct spdk_nvme_ctrlr *ctrlr, uint32_t sector_size, uint32_t md_size, uint32_t max_xfer_size, uint32_t stripe_size, bool extended_lba) { + uint32_t num_requests = 32; + uint32_t i; + ctrlr->max_xfer_size = max_xfer_size; /* * Clear the flags field - we especially want to make sure the SGL_SUPPORTED flag is not set @@ -214,10 +216,24 @@ prepare_for_test(struct spdk_nvme_ns *ns, struct spdk_nvme_ctrlr *ctrlr, ns->sectors_per_stripe = ns->stripe_size / ns->extended_lba_size; memset(qpair, 0, sizeof(*qpair)); + qpair->req_buf = calloc(num_requests, sizeof(struct nvme_request)); + SPDK_CU_ASSERT_FATAL(qpair->req_buf != NULL); + + for (i = 0; i < num_requests; i++) { + struct nvme_request *req = qpair->req_buf + i * sizeof(struct nvme_request); + + STAILQ_INSERT_HEAD(&qpair->free_req, req, stailq); + } g_request = NULL; } +static void +cleanup_after_test(struct spdk_nvme_qpair *qpair) +{ + free(qpair->req_buf); +} + static void nvme_cmd_interpret_rw(const struct spdk_nvme_cmd *cmd, uint64_t *lba, uint32_t *num_blocks) @@ -254,6 +270,7 @@ split_test(void) free(payload); nvme_free_request(g_request); + cleanup_after_test(&qpair); } static void @@ -308,6 +325,7 @@ split_test2(void) free(payload); nvme_free_request(g_request); + cleanup_after_test(&qpair); } static void @@ -364,6 +382,7 @@ split_test3(void) free(payload); nvme_free_request(g_request); + cleanup_after_test(&qpair); } static void @@ -441,6 +460,7 @@ split_test4(void) free(payload); nvme_free_request(g_request); + cleanup_after_test(&qpair); } static void @@ -496,6 +516,7 @@ test_cmd_child_request(void) free(payload); nvme_free_request(g_request); + cleanup_after_test(&qpair); } static void @@ -517,6 +538,7 @@ test_nvme_ns_cmd_flush(void) CU_ASSERT(g_request->cmd.nsid == ns.id); nvme_free_request(g_request); + cleanup_after_test(&qpair); } static void @@ -543,6 +565,7 @@ test_nvme_ns_cmd_write_zeroes(void) CU_ASSERT_EQUAL(cmd_lba_count, 2); nvme_free_request(g_request); + cleanup_after_test(&qpair); } static void @@ -592,6 +615,7 @@ test_nvme_ns_cmd_dataset_management(void) rc = spdk_nvme_ns_cmd_dataset_management(&ns, &qpair, SPDK_NVME_DSM_ATTR_DEALLOCATE, NULL, 0, cb_fn, cb_arg); CU_ASSERT(rc != 0); + cleanup_after_test(&qpair); } static void @@ -626,6 +650,7 @@ test_nvme_ns_cmd_readv(void) free(cb_arg); nvme_free_request(g_request); + cleanup_after_test(&qpair); } static void @@ -660,6 +685,7 @@ test_nvme_ns_cmd_writev(void) free(cb_arg); nvme_free_request(g_request); + cleanup_after_test(&qpair); } static void @@ -695,7 +721,7 @@ test_io_flags(void) nvme_free_request(g_request); free(payload); - + cleanup_after_test(&qpair); } static void @@ -733,6 +759,7 @@ test_nvme_ns_cmd_reservation_register(void) spdk_free(g_request->payload.u.contig); nvme_free_request(g_request); free(payload); + cleanup_after_test(&qpair); } static void @@ -770,6 +797,7 @@ test_nvme_ns_cmd_reservation_release(void) spdk_free(g_request->payload.u.contig); nvme_free_request(g_request); free(payload); + cleanup_after_test(&qpair); } static void @@ -807,6 +835,7 @@ test_nvme_ns_cmd_reservation_acquire(void) spdk_free(g_request->payload.u.contig); nvme_free_request(g_request); free(payload); + cleanup_after_test(&qpair); } static void @@ -838,6 +867,7 @@ test_nvme_ns_cmd_reservation_report(void) spdk_free(g_request->payload.u.contig); nvme_free_request(g_request); free(payload); + cleanup_after_test(&qpair); } static void @@ -881,6 +911,7 @@ test_nvme_ns_cmd_write_with_md(void) CU_ASSERT(g_request->payload_size == 256 * 512); nvme_free_request(g_request); + cleanup_after_test(&qpair); /* * 512 byte data + 128 byte metadata @@ -915,6 +946,7 @@ test_nvme_ns_cmd_write_with_md(void) nvme_request_free_children(g_request); nvme_free_request(g_request); + cleanup_after_test(&qpair); /* * 512 byte data + 8 byte metadata @@ -950,6 +982,7 @@ test_nvme_ns_cmd_write_with_md(void) nvme_request_free_children(g_request); nvme_free_request(g_request); + cleanup_after_test(&qpair); /* * 512 byte data + 8 byte metadata @@ -988,6 +1021,7 @@ test_nvme_ns_cmd_write_with_md(void) nvme_request_free_children(g_request); nvme_free_request(g_request); + cleanup_after_test(&qpair); /* * 512 byte data + 8 byte metadata @@ -1010,6 +1044,7 @@ test_nvme_ns_cmd_write_with_md(void) CU_ASSERT(g_request->payload_size == 256 * 512); nvme_free_request(g_request); + cleanup_after_test(&qpair); /* * 512 byte data + 8 byte metadata @@ -1046,6 +1081,7 @@ test_nvme_ns_cmd_write_with_md(void) nvme_request_free_children(g_request); nvme_free_request(g_request); + cleanup_after_test(&qpair); free(buffer); free(metadata); diff --git a/test/lib/nvme/unit/nvme_pcie_c/nvme_pcie_ut.c b/test/lib/nvme/unit/nvme_pcie_c/nvme_pcie_ut.c index 0d8fba6ba2..fc242d542e 100644 --- a/test/lib/nvme/unit/nvme_pcie_c/nvme_pcie_ut.c +++ b/test/lib/nvme/unit/nvme_pcie_c/nvme_pcie_ut.c @@ -47,7 +47,6 @@ struct spdk_trace_flag SPDK_TRACE_NVME = { static struct nvme_driver _g_nvme_driver = { .lock = PTHREAD_MUTEX_INITIALIZER, - .request_mempool = NULL, }; struct nvme_driver *g_spdk_nvme_driver = &_g_nvme_driver; @@ -92,7 +91,8 @@ spdk_pci_device_get_id(struct spdk_pci_device *dev) int nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id, struct spdk_nvme_ctrlr *ctrlr, - enum spdk_nvme_qprio qprio) + enum spdk_nvme_qprio qprio, + uint32_t num_requests) { abort(); } @@ -347,7 +347,7 @@ ut_insert_cq_entry(struct spdk_nvme_qpair *qpair, uint32_t slot) struct nvme_tracker *tr; struct spdk_nvme_cpl *cpl; - req = spdk_mempool_get(_g_nvme_driver.request_mempool); + req = calloc(1, sizeof(*req)); SPDK_CU_ASSERT_FATAL(req != NULL); memset(req, 0, sizeof(*req)); diff --git a/test/lib/nvme/unit/nvme_qpair_c/nvme_qpair_ut.c b/test/lib/nvme/unit/nvme_qpair_c/nvme_qpair_ut.c index 8c59f46d34..5ffa1586d5 100644 --- a/test/lib/nvme/unit/nvme_qpair_c/nvme_qpair_ut.c +++ b/test/lib/nvme/unit/nvme_qpair_c/nvme_qpair_ut.c @@ -44,7 +44,6 @@ bool trace_flag = false; struct nvme_driver _g_nvme_driver = { .lock = PTHREAD_MUTEX_INITIALIZER, - .request_mempool = NULL, }; struct nvme_request * @@ -53,14 +52,15 @@ nvme_allocate_request(struct spdk_nvme_qpair *qpair, spdk_nvme_cmd_cb cb_fn, void *cb_arg) { - struct nvme_request *req = NULL; - - req = spdk_mempool_get(_g_nvme_driver.request_mempool); + struct nvme_request *req; + req = STAILQ_FIRST(&qpair->free_req); if (req == NULL) { - return req; + return NULL; } + STAILQ_REMOVE_HEAD(&qpair->free_req, stailq); + /* * Only memset up to (but not including) the children * TAILQ_ENTRY. children, and following members, are @@ -101,7 +101,9 @@ nvme_allocate_request_null(struct spdk_nvme_qpair *qpair, spdk_nvme_cmd_cb cb_fn void nvme_free_request(struct nvme_request *req) { - spdk_mempool_put(_g_nvme_driver.request_mempool, req); + SPDK_CU_ASSERT_FATAL(req != NULL); + SPDK_CU_ASSERT_FATAL(req->qpair != NULL); + STAILQ_INSERT_HEAD(&req->qpair->free_req, req, stailq); } void @@ -158,12 +160,13 @@ prepare_submit_request_test(struct spdk_nvme_qpair *qpair, ctrlr->free_io_qids = NULL; TAILQ_INIT(&ctrlr->active_io_qpairs); TAILQ_INIT(&ctrlr->active_procs); - nvme_qpair_init(qpair, 1, ctrlr, 0); + nvme_qpair_init(qpair, 1, ctrlr, 0, 32); } static void cleanup_submit_request_test(struct spdk_nvme_qpair *qpair) { + free(qpair->req_buf); } static void