d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

nvme: Add SGL support in NVMe driver

Browse Source 
For those NVMe controllers which can support SGL feature in
firmware, we will use SGL for scattered payloads.

Change-Id: If688e6494ed62e8cba1d55fc6372c6e162cc09c3
Signed-off-by: Changpeng Liu <changpeng.liu@intel.com>
This commit is contained in:
Changpeng Liu 2016-03-01 10:50:31 +08:00 committed by Daniel Verkamp
parent 9841610855
commit eb9ef5cc2b
5 changed files with 136 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lib/nvme/nvme_ctrlr.c
View File

@ -842,6 +842,11 @@ nvme_ctrlr_start(struct spdk_nvme_ctrlr *ctrlr)
nvme_ctrlr_set_supported_log_pages(ctrlr);
nvme_ctrlr_set_supported_features(ctrlr);
if (ctrlr->cdata.sgls.supported) {
ctrlr->flags |= SPDK_NVME_CTRLR_SGL_SUPPORTED;
}
return 0;
}
@ -908,6 +913,7 @@ nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr, void *devhandle)
ctrlr->is_resetting = false;
ctrlr->is_failed = false;
ctrlr->flags = 0;
nvme_mutex_init_recursive(&ctrlr->ctrlr_lock);

23
lib/nvme/nvme_internal.h
View File

@ -98,6 +98,12 @@
#define NVME_MIN_IO_TRACKERS (4)
#define NVME_MAX_IO_TRACKERS (1024)
/*
* NVME_MAX_SGL_DESCRIPTORS defines the maximum number of descriptors in one SGL
* segment.
*/
#define NVME_MAX_SGL_DESCRIPTORS (256)
/*
* NVME_MAX_IO_ENTRIES is not defined, since it is specified in CC.MQES
* for each controller.
@ -128,6 +134,13 @@ enum nvme_payload_type {
NVME_PAYLOAD_TYPE_SGL,
};
/*
* Controller support flags.
*/
enum spdk_nvme_ctrlr_flags {
SPDK_NVME_CTRLR_SGL_SUPPORTED = 0x1, /**< The SGL is supported */
};
/**
* Descriptor for a request data payload.
*
@ -233,8 +246,11 @@ struct nvme_tracker {
struct nvme_request *req;
uint16_t cid;
uint64_t prp_bus_addr;
uint64_t prp[NVME_MAX_PRP_LIST_ENTRIES];
uint64_t prp_sgl_bus_addr;
union {
uint64_t prp[NVME_MAX_PRP_LIST_ENTRIES];
struct spdk_nvme_sgl_descriptor sgl[NVME_MAX_SGL_DESCRIPTORS];
} u;
};
struct spdk_nvme_qpair {
@ -342,6 +358,9 @@ struct spdk_nvme_ctrlr {
bool is_failed;
/** Controller support flags */
uint64_t flags;
/* Cold data (not accessed in normal I/O path) is after this point. */
enum nvme_ctrlr_state state;

94
lib/nvme/nvme_qpair.c
View File

@ -289,7 +289,7 @@ nvme_completion_is_retry(const struct spdk_nvme_cpl *cpl)
static void
nvme_qpair_construct_tracker(struct nvme_tracker *tr, uint16_t cid, uint64_t phys_addr)
{
tr->prp_bus_addr = phys_addr + offsetof(struct nvme_tracker, prp);
tr->prp_sgl_bus_addr = phys_addr + offsetof(struct nvme_tracker, u.prp);
tr->cid = cid;
}
@ -702,7 +702,7 @@ _nvme_qpair_build_contig_request(struct spdk_nvme_qpair *qpair, struct nvme_requ
tr->req->cmd.dptr.prp.prp2 = nvme_vtophys(seg_addr);
} else if (nseg > 2) {
cur_nseg = 1;
tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_bus_addr;
tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_sgl_bus_addr;
while (cur_nseg < nseg) {
seg_addr = payload + cur_nseg * PAGE_SIZE - unaligned;
phys_addr = nvme_vtophys(seg_addr);
@ -710,7 +710,7 @@ _nvme_qpair_build_contig_request(struct spdk_nvme_qpair *qpair, struct nvme_requ
_nvme_fail_request_bad_vtophys(qpair, tr);
return -1;
}
tr->prp[cur_nseg - 1] = phys_addr;
tr->u.prp[cur_nseg - 1] = phys_addr;
cur_nseg++;
}
}
@ -718,9 +718,79 @@ _nvme_qpair_build_contig_request(struct spdk_nvme_qpair *qpair, struct nvme_requ
return 0;
}
/**
* Build SGL list describing scattered payload buffer.
*/
static int
_nvme_qpair_build_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,
struct nvme_tracker *tr)
_nvme_qpair_build_hw_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,
struct nvme_tracker *tr)
{
int rc;
uint64_t phys_addr;
uint32_t remaining_transfer_len, length;
struct spdk_nvme_sgl_descriptor *sgl;
uint32_t nseg = 0;
/*
* Build scattered payloads.
*/
nvme_assert(req->payload.type == NVME_PAYLOAD_TYPE_SGL, ("sgl payload type required\n"));
nvme_assert(req->payload.u.sgl.reset_sgl_fn != NULL, ("sgl reset callback required\n"));
req->payload.u.sgl.reset_sgl_fn(req->payload.u.sgl.cb_arg, req->payload_offset);
sgl = (struct spdk_nvme_sgl_descriptor *)tr->u.sgl;
req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_SGL;
req->cmd.dptr.sgl1.type_specific = 0;
remaining_transfer_len = req->payload_size;
while (remaining_transfer_len > 0) {
nvme_assert(req->payload.u.sgl.next_sge_fn != NULL, ("sgl callback required\n"));
rc = req->payload.u.sgl.next_sge_fn(req->payload.u.sgl.cb_arg, &phys_addr, &length);
if (rc) {
_nvme_fail_request_bad_vtophys(qpair, tr);
return -1;
}
remaining_transfer_len -= length;
sgl->type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;
sgl->length = length;
sgl->address = phys_addr;
sgl->type_specific = 0;
sgl++;
nseg++;
if (nseg >= NVME_MAX_SGL_DESCRIPTORS) {
_nvme_fail_request_bad_vtophys(qpair, tr);
return -1;
}
}
if (nseg == 1) {
req->cmd.dptr.sgl1.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;
req->cmd.dptr.sgl1.address = phys_addr;
req->cmd.dptr.sgl1.length = length;
} else if (nseg > 1) {
/* For now we can only support 1 SGL segment in NVMe controller */
req->cmd.dptr.sgl1.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT;
req->cmd.dptr.sgl1.address = tr->prp_sgl_bus_addr;
req->cmd.dptr.sgl1.length = nseg * sizeof(struct spdk_nvme_sgl_descriptor);
} else {
_nvme_fail_request_bad_vtophys(qpair, tr);
return -1;
}
return 0;
}
/**
* Build PRP list describing scattered payload buffer.
*/
static int
_nvme_qpair_build_prps_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,
struct nvme_tracker *tr)
{
int rc;
uint64_t phys_addr;
@ -779,13 +849,13 @@ _nvme_qpair_build_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request
else
cur_nseg = 0;
tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_bus_addr;
tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_sgl_bus_addr;
while (cur_nseg < nseg) {
if (prp2) {
tr->prp[0] = prp2;
tr->prp[last_nseg + 1] = phys_addr + cur_nseg * PAGE_SIZE - unaligned;
tr->u.prp[0] = prp2;
tr->u.prp[last_nseg + 1] = phys_addr + cur_nseg * PAGE_SIZE - unaligned;
} else
tr->prp[last_nseg] = phys_addr + cur_nseg * PAGE_SIZE - unaligned;
tr->u.prp[last_nseg] = phys_addr + cur_nseg * PAGE_SIZE - unaligned;
last_nseg++;
cur_nseg++;
@ -810,6 +880,7 @@ nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *re
int rc;
struct nvme_tracker *tr;
struct nvme_request *child_req;
struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
nvme_qpair_check_enabled(qpair);
@ -860,7 +931,10 @@ nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *re
return;
}
} else if (req->payload.type == NVME_PAYLOAD_TYPE_SGL) {
rc = _nvme_qpair_build_sgl_request(qpair, req, tr);
if (ctrlr->flags & SPDK_NVME_CTRLR_SGL_SUPPORTED)
rc = _nvme_qpair_build_hw_sgl_request(qpair, req, tr);
else
rc = _nvme_qpair_build_prps_sgl_request(qpair, req, tr);
if (rc < 0) {
return;
}

19
test/lib/nvme/sgl/nvme_sgl.c
View File

@ -129,6 +129,22 @@ io_complete(void *ctx, const struct spdk_nvme_cpl *cpl)
io_complete_flag = 1;
}
static uint32_t build_io_request_0(struct io_request *req)
{
int i;
uint32_t len = 0;
req->nseg = 1;
req->iovs[0].iov_base = rte_zmalloc(NULL, 0x800, 4);
req->iovs[0].iov_len = 0x800;
for (i = 0; i < req->nseg; i++)
len += req->iovs[i].iov_len;
return len;
}
static uint32_t build_io_request_1(struct io_request *req)
{
int i, found = 0;
@ -456,7 +472,8 @@ int main(int argc, char **argv)
}
foreach_dev(iter) {
if (writev_readv_tests(iter, build_io_request_1)
if (writev_readv_tests(iter, build_io_request_0)
|| writev_readv_tests(iter, build_io_request_1)
|| writev_readv_tests(iter, build_io_request_2)
|| writev_readv_tests(iter, build_io_request_3)
|| writev_readv_tests(iter, build_io_request_4)

13
test/lib/nvme/unit/nvme_qpair_c/nvme_qpair_ut.c
View File

@ -389,13 +389,14 @@ test_sgl_req(void)
CU_ASSERT(req->cmd.dptr.prp.prp1 == 0);
CU_ASSERT(qpair.sq_tail == 1);
sgl_tr = LIST_FIRST(&qpair.outstanding_tr);
CU_ASSERT(sgl_tr != NULL);
for (i = 0; i < 32; i++) {
CU_ASSERT(sgl_tr->prp[i] == (PAGE_SIZE * (i + 1)));
}
if (sgl_tr != NULL) {
for (i = 0; i < 32; i++) {
CU_ASSERT(sgl_tr->u.prp[i] == (PAGE_SIZE * (i + 1)));
}
LIST_REMOVE(sgl_tr, list);
free(sgl_tr);
LIST_REMOVE(sgl_tr, list);
free(sgl_tr);
}
cleanup_submit_request_test(&qpair);
nvme_free_request(req);
}