From 0a4b14e8cc38f1067c56b72bc9deebcb6eb85a84 Mon Sep 17 00:00:00 2001 From: Michal Meloun Date: Sun, 15 Dec 2019 14:28:38 +0000 Subject: [PATCH] Properly synchronize completion DMA buffers. Within command completion processing the callback function may access DMAed data buffer. Synchronize it before use, not after. This allows to use NVMe disk on non-DMA coherent arm64 system. MFC after: 3 weeks --- sys/dev/nvme/nvme_qpair.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/sys/dev/nvme/nvme_qpair.c b/sys/dev/nvme/nvme_qpair.c index fbd3d2c1578f..4929340fdb60 100644 --- a/sys/dev/nvme/nvme_qpair.c +++ b/sys/dev/nvme/nvme_qpair.c @@ -444,8 +444,15 @@ nvme_qpair_complete_tracker(struct nvme_tracker *tr, KASSERT(cpl->cid == req->cmd.cid, ("cpl cid does not match cmd cid\n")); - if (req->cb_fn && !retry) - req->cb_fn(req->cb_arg, cpl); + if (!retry) { + if (req->type != NVME_REQUEST_NULL) { + bus_dmamap_sync(qpair->dma_tag_payload, + tr->payload_dma_map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + } + if (req->cb_fn) + req->cb_fn(req->cb_arg, cpl); + } mtx_lock(&qpair->lock); callout_stop(&tr->timer); @@ -455,9 +462,6 @@ nvme_qpair_complete_tracker(struct nvme_tracker *tr, nvme_qpair_submit_tracker(qpair, tr); } else { if (req->type != NVME_REQUEST_NULL) { - bus_dmamap_sync(qpair->dma_tag_payload, - tr->payload_dma_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(qpair->dma_tag_payload, tr->payload_dma_map); }