d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Support doorbell strides != 0.

Browse Source 
The NVMe standard (1.4) states

>>> 8.6 Doorbell Stride for Software Emulation
>>> The doorbell stride,...is useful in software emulation of an NVM
>>> Express controller. ...  For hardware implementations of the NVM
>>> Express interface, the expected doorbell stride value is 0h.

However, hardware in the wild exists with a doorbell stride of 1
(meaning 8 byte separation). This change supports that hardware, as
well as software emulators as envisioned in Section 8.6. Since this is
the fast path, care has been taken to make this computation
efficient. The bit of math to compute an offset for each is replaced
by a memory load from cache of a pre-computed value.

MFC After: 3 days
Reviewed by: scottl@
Differential Revision: https://reviews.freebsd.org/D21514
This commit is contained in:
imp 2019-09-04 20:08:36 +00:00
parent 7555f0f24e
commit 362659d41a
3 changed files with 33 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
sys/dev/nvme/nvme_ctrlr.c
View File

@ -90,19 +90,25 @@ nvme_ctrlr_construct_io_qpairs(struct nvme_controller *ctrlr)
struct nvme_qpair *qpair;
uint32_t cap_lo;
uint16_t mqes;
int i, error, num_entries, num_trackers;
num_entries = NVME_IO_ENTRIES;
TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries);
int i, error, num_entries, num_trackers, max_entries;
/*
* NVMe spec sets a hard limit of 64K max entries, but
* devices may specify a smaller limit, so we need to check
* the MQES field in the capabilities register.
* NVMe spec sets a hard limit of 64K max entries, but devices may
* specify a smaller limit, so we need to check the MQES field in the
* capabilities register. We have to cap the number of entries to the
* current stride allows for in BAR 0/1, otherwise the remainder entries
* are inaccessable. MQES should reflect this, and this is just a
* fail-safe.
*/
max_entries =
(rman_get_size(ctrlr->resource) - nvme_mmio_offsetof(doorbell[0])) /
(1 << (ctrlr->dstrd + 1));
num_entries = NVME_IO_ENTRIES;
TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries);
cap_lo = nvme_mmio_read_4(ctrlr, cap_lo);
mqes = NVME_CAP_LO_MQES(cap_lo);
num_entries = min(num_entries, mqes + 1);
num_entries = min(num_entries, max_entries);
num_trackers = NVME_IO_TRACKERS;
TUNABLE_INT_FETCH("hw.nvme.io_trackers", &num_trackers);
@ -110,9 +116,9 @@ nvme_ctrlr_construct_io_qpairs(struct nvme_controller *ctrlr)
num_trackers = max(num_trackers, NVME_MIN_IO_TRACKERS);
num_trackers = min(num_trackers, NVME_MAX_IO_TRACKERS);
/*
* No need to have more trackers than entries in the submit queue.
* Note also that for a queue size of N, we can only have (N-1)
* commands outstanding, hence the "-1" here.
* No need to have more trackers than entries in the submit queue. Note
* also that for a queue size of N, we can only have (N-1) commands
* outstanding, hence the "-1" here.
*/
num_trackers = min(num_trackers, (num_entries-1));
@ -1120,7 +1126,6 @@ nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev)
uint32_t cap_lo;
uint32_t cap_hi;
uint32_t to;
uint8_t dstrd;
uint8_t mpsmin;
int status, timeout_period;
@ -1128,14 +1133,8 @@ nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev)
mtx_init(&ctrlr->lock, "nvme ctrlr lock", NULL, MTX_DEF);
/*
* Software emulators may set the doorbell stride to something
* other than zero, but this driver is not set up to handle that.
*/
cap_hi = nvme_mmio_read_4(ctrlr, cap_hi);
dstrd = NVME_CAP_HI_DSTRD(cap_hi);
if (dstrd != 0)
return (ENXIO);
ctrlr->dstrd = NVME_CAP_HI_DSTRD(cap_hi) + 2;
mpsmin = NVME_CAP_HI_MPSMIN(cap_hi);
ctrlr->min_page_size = 1 << (12 + mpsmin);

3
sys/dev/nvme/nvme_private.h
View File

@ -297,6 +297,9 @@ struct nvme_controller {
/** timeout period in seconds */
uint32_t timeout_period;
/** doorbell stride */
uint32_t dstrd;
struct nvme_qpair adminq;
struct nvme_qpair *ioq;

20
sys/dev/nvme/nvme_qpair.c
View File

@ -622,8 +622,8 @@ nvme_qpair_process_completions(struct nvme_qpair *qpair)
qpair->phase = !qpair->phase; /* 3 */
}
nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].cq_hdbl,
qpair->cq_head);
bus_space_write_4(qpair->ctrlr->bus_tag, qpair->ctrlr->bus_handle,
qpair->cq_hdbl_off, qpair->cq_head);
}
return (done != 0);
}
@ -731,8 +731,15 @@ nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id,
qpair->cpl_bus_addr = queuemem_phys + cmdsz;
prpmem_phys = queuemem_phys + cmdsz + cplsz;
qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[id].sq_tdbl);
qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[id].cq_hdbl);
/*
* Calcuate the stride of the doorbell register. Many emulators set this
* value to correspond to a cache line. However, some hardware has set
* it to various small values.
*/
qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[0]) +
(id << (ctrlr->dstrd + 1));
qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[0]) +
(id << (ctrlr->dstrd + 1)) + (1 << ctrlr->dstrd);
TAILQ_INIT(&qpair->free_tr);
TAILQ_INIT(&qpair->outstanding_tr);
@ -950,9 +957,8 @@ nvme_qpair_submit_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr)
wmb();
#endif
nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].sq_tdbl,
qpair->sq_tail);
bus_space_write_4(qpair->ctrlr->bus_tag, qpair->ctrlr->bus_handle,
qpair->sq_tdbl_off, qpair->sq_tail);
qpair->num_cmds++;
}