From 35688b25ae2f5a29ff6d24fe20bb24fc44b3d9e1 Mon Sep 17 00:00:00 2001 From: imp Date: Wed, 14 Mar 2018 23:01:18 +0000 Subject: [PATCH] When tearing down a queue pair, also delete the queue entries. The NVME standard has required in section 7.2.6, since at least 1.1, that a clean shutdown is signalled by deleting the subission and the completion queues before setting the shutdown bit in CC. The 1.0 standard, apparently, did not and many of the early Intel cards didn't care. Some newer cards care, at least one whose beta firmware can scramble the card on an unclean shutdown. Linux has done this for some time. To make it possible to move forward with an evaluation of this pre-release card with wonky firmware, delete the queues on the card when we delete the qpair structures. Sponsored by: Netflix --- sys/dev/nvme/nvme_ctrlr.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/sys/dev/nvme/nvme_ctrlr.c b/sys/dev/nvme/nvme_ctrlr.c index b204c91ead11..c916c7a933a8 100644 --- a/sys/dev/nvme/nvme_ctrlr.c +++ b/sys/dev/nvme/nvme_ctrlr.c @@ -513,6 +513,34 @@ nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr) return (0); } +static int +nvme_ctrlr_destroy_qpair(struct nvme_controller *ctrlr, struct nvme_qpair *qpair) +{ + struct nvme_completion_poll_status status; + + status.done = 0; + nvme_ctrlr_cmd_delete_io_sq(qpair->ctrlr, qpair, + nvme_completion_poll_cb, &status); + while (!atomic_load_acq_int(&status.done)) + pause("nvme", 1); + if (nvme_completion_is_error(&status.cpl)) { + nvme_printf(ctrlr, "nvme_create_io_sq failed!\n"); + return (ENXIO); + } + + status.done = 0; + nvme_ctrlr_cmd_delete_io_cq(ctrlr, qpair, + nvme_completion_poll_cb, &status); + while (!atomic_load_acq_int(&status.done)) + pause("nvme", 1); + if (nvme_completion_is_error(&status.cpl)) { + nvme_printf(ctrlr, "nvme_create_io_cq failed!\n"); + return (ENXIO); + } + + return (0); +} + static int nvme_ctrlr_construct_namespaces(struct nvme_controller *ctrlr) { @@ -1274,6 +1302,7 @@ nvme_ctrlr_destruct(struct nvme_controller *ctrlr, device_t dev) destroy_dev(ctrlr->cdev); for (i = 0; i < ctrlr->num_io_queues; i++) { + nvme_ctrlr_destroy_qpair(ctrlr, &ctrlr->ioq[i]); nvme_io_qpair_destroy(&ctrlr->ioq[i]); }