2012-09-17 19:23:01 +00:00
|
|
|
/*-
|
2017-11-27 14:52:40 +00:00
|
|
|
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
|
|
|
|
*
|
2016-01-07 20:32:04 +00:00
|
|
|
* Copyright (C) 2012-2016 Intel Corporation
|
2012-09-17 19:23:01 +00:00
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
|
|
* SUCH DAMAGE.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <sys/cdefs.h>
|
|
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
|
2016-06-10 06:04:53 +00:00
|
|
|
#include "opt_cam.h"
|
|
|
|
|
2012-09-17 19:23:01 +00:00
|
|
|
#include <sys/param.h>
|
2013-04-12 17:52:17 +00:00
|
|
|
#include <sys/systm.h>
|
|
|
|
#include <sys/buf.h>
|
2012-09-17 19:23:01 +00:00
|
|
|
#include <sys/bus.h>
|
|
|
|
#include <sys/conf.h>
|
|
|
|
#include <sys/ioccom.h>
|
2013-04-12 17:52:17 +00:00
|
|
|
#include <sys/proc.h>
|
2012-09-17 19:23:01 +00:00
|
|
|
#include <sys/smp.h>
|
2013-04-12 17:52:17 +00:00
|
|
|
#include <sys/uio.h>
|
2020-04-30 00:27:19 +00:00
|
|
|
#include <sys/sbuf.h>
|
2018-02-22 13:32:31 +00:00
|
|
|
#include <sys/endian.h>
|
2020-04-30 00:27:19 +00:00
|
|
|
#include <machine/stdarg.h>
|
2019-09-23 17:53:47 +00:00
|
|
|
#include <vm/vm.h>
|
2012-09-17 19:23:01 +00:00
|
|
|
|
|
|
|
#include "nvme_private.h"
|
|
|
|
|
2018-02-22 13:32:31 +00:00
|
|
|
#define B4_CHK_RDY_DELAY_MS 2300 /* work around controller bug */
|
2017-12-18 18:38:00 +00:00
|
|
|
|
2013-03-26 18:37:36 +00:00
|
|
|
static void nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,
|
|
|
|
struct nvme_async_event_request *aer);
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2020-04-30 00:27:19 +00:00
|
|
|
static void
|
|
|
|
nvme_ctrlr_devctl_log(struct nvme_controller *ctrlr, const char *type, const char *msg, ...)
|
|
|
|
{
|
|
|
|
struct sbuf sb;
|
|
|
|
va_list ap;
|
|
|
|
int error;
|
|
|
|
|
2020-04-30 00:43:11 +00:00
|
|
|
if (sbuf_new(&sb, NULL, 0, SBUF_AUTOEXTEND | SBUF_NOWAIT) == NULL)
|
|
|
|
return;
|
2020-04-30 00:27:19 +00:00
|
|
|
sbuf_printf(&sb, "%s: ", device_get_nameunit(ctrlr->dev));
|
|
|
|
va_start(ap, msg);
|
|
|
|
sbuf_vprintf(&sb, msg, ap);
|
|
|
|
va_end(ap);
|
|
|
|
error = sbuf_finish(&sb);
|
|
|
|
if (error == 0)
|
|
|
|
printf("%s\n", sbuf_data(&sb));
|
|
|
|
|
|
|
|
sbuf_clear(&sb);
|
|
|
|
sbuf_printf(&sb, "name=\"%s\" reason=\"", device_get_nameunit(ctrlr->dev));
|
|
|
|
va_start(ap, msg);
|
|
|
|
sbuf_vprintf(&sb, msg, ap);
|
|
|
|
va_end(ap);
|
|
|
|
sbuf_printf(&sb, "\"");
|
|
|
|
error = sbuf_finish(&sb);
|
|
|
|
if (error == 0)
|
|
|
|
devctl_notify("nvme", "controller", type, sbuf_data(&sb));
|
|
|
|
sbuf_delete(&sb);
|
|
|
|
}
|
|
|
|
|
2016-11-08 00:24:49 +00:00
|
|
|
static int
|
2012-09-17 19:23:01 +00:00
|
|
|
nvme_ctrlr_construct_admin_qpair(struct nvme_controller *ctrlr)
|
|
|
|
{
|
|
|
|
struct nvme_qpair *qpair;
|
|
|
|
uint32_t num_entries;
|
2016-11-08 00:24:49 +00:00
|
|
|
int error;
|
2012-09-17 19:23:01 +00:00
|
|
|
|
|
|
|
qpair = &ctrlr->adminq;
|
2019-09-23 17:53:47 +00:00
|
|
|
qpair->id = 0;
|
|
|
|
qpair->cpu = CPU_FFS(&cpuset_domain[ctrlr->domain]) - 1;
|
|
|
|
qpair->domain = ctrlr->domain;
|
2012-09-17 19:23:01 +00:00
|
|
|
|
|
|
|
num_entries = NVME_ADMIN_ENTRIES;
|
|
|
|
TUNABLE_INT_FETCH("hw.nvme.admin_entries", &num_entries);
|
|
|
|
/*
|
|
|
|
* If admin_entries was overridden to an invalid value, revert it
|
|
|
|
* back to our default value.
|
|
|
|
*/
|
|
|
|
if (num_entries < NVME_MIN_ADMIN_ENTRIES ||
|
|
|
|
num_entries > NVME_MAX_ADMIN_ENTRIES) {
|
2013-03-26 22:17:10 +00:00
|
|
|
nvme_printf(ctrlr, "invalid hw.nvme.admin_entries=%d "
|
|
|
|
"specified\n", num_entries);
|
2012-09-17 19:23:01 +00:00
|
|
|
num_entries = NVME_ADMIN_ENTRIES;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The admin queue's max xfer size is treated differently than the
|
|
|
|
* max I/O xfer size. 16KB is sufficient here - maybe even less?
|
|
|
|
*/
|
2019-09-23 17:53:47 +00:00
|
|
|
error = nvme_qpair_construct(qpair, num_entries, NVME_ADMIN_TRACKERS,
|
|
|
|
ctrlr);
|
2016-11-08 00:24:49 +00:00
|
|
|
return (error);
|
2012-09-17 19:23:01 +00:00
|
|
|
}
|
|
|
|
|
2019-09-23 17:53:47 +00:00
|
|
|
#define QP(ctrlr, c) ((c) * (ctrlr)->num_io_queues / mp_ncpus)
|
|
|
|
|
2012-09-17 19:23:01 +00:00
|
|
|
static int
|
|
|
|
nvme_ctrlr_construct_io_qpairs(struct nvme_controller *ctrlr)
|
|
|
|
{
|
|
|
|
struct nvme_qpair *qpair;
|
2018-02-22 13:32:31 +00:00
|
|
|
uint32_t cap_lo;
|
|
|
|
uint16_t mqes;
|
2019-09-23 17:53:47 +00:00
|
|
|
int c, error, i, n;
|
|
|
|
int num_entries, num_trackers, max_entries;
|
2012-09-17 19:23:01 +00:00
|
|
|
|
|
|
|
/*
|
2019-09-04 20:08:36 +00:00
|
|
|
* NVMe spec sets a hard limit of 64K max entries, but devices may
|
|
|
|
* specify a smaller limit, so we need to check the MQES field in the
|
|
|
|
* capabilities register. We have to cap the number of entries to the
|
|
|
|
* current stride allows for in BAR 0/1, otherwise the remainder entries
|
|
|
|
* are inaccessable. MQES should reflect this, and this is just a
|
|
|
|
* fail-safe.
|
2012-09-17 19:23:01 +00:00
|
|
|
*/
|
2019-09-04 20:08:36 +00:00
|
|
|
max_entries =
|
|
|
|
(rman_get_size(ctrlr->resource) - nvme_mmio_offsetof(doorbell[0])) /
|
|
|
|
(1 << (ctrlr->dstrd + 1));
|
|
|
|
num_entries = NVME_IO_ENTRIES;
|
|
|
|
TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries);
|
2018-02-22 13:32:31 +00:00
|
|
|
cap_lo = nvme_mmio_read_4(ctrlr, cap_lo);
|
2019-07-18 15:41:10 +00:00
|
|
|
mqes = NVME_CAP_LO_MQES(cap_lo);
|
2018-02-22 13:32:31 +00:00
|
|
|
num_entries = min(num_entries, mqes + 1);
|
2019-09-04 20:08:36 +00:00
|
|
|
num_entries = min(num_entries, max_entries);
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2012-10-18 00:44:39 +00:00
|
|
|
num_trackers = NVME_IO_TRACKERS;
|
|
|
|
TUNABLE_INT_FETCH("hw.nvme.io_trackers", &num_trackers);
|
|
|
|
|
|
|
|
num_trackers = max(num_trackers, NVME_MIN_IO_TRACKERS);
|
|
|
|
num_trackers = min(num_trackers, NVME_MAX_IO_TRACKERS);
|
|
|
|
/*
|
2019-09-04 20:08:36 +00:00
|
|
|
* No need to have more trackers than entries in the submit queue. Note
|
|
|
|
* also that for a queue size of N, we can only have (N-1) commands
|
|
|
|
* outstanding, hence the "-1" here.
|
2012-10-18 00:44:39 +00:00
|
|
|
*/
|
|
|
|
num_trackers = min(num_trackers, (num_entries-1));
|
|
|
|
|
2017-08-28 23:54:20 +00:00
|
|
|
/*
|
|
|
|
* Our best estimate for the maximum number of I/Os that we should
|
Implement nvme suspend / resume for pci attachment
When we suspend, we need to properly shutdown the NVME controller. The
controller may go into D3 state (or may have the power removed), and
to properly flush the metadata to non-volatile RAM, we must complete a
normal shutdown. This consists of deleting the I/O queues and setting
the shutodown bit. We have to do some extra stuff to make sure we
reset the software state of the queues as well.
On resume, we have to reset the card twice, for reasons described in
the attach funcion. Once we've done that, we can restart the card. If
any of this fails, we'll fail the NVMe card, just like we do when a
reset fails.
Set is_resetting for the duration of the suspend / resume. This keeps
the reset taskqueue from running a concurrent reset, and also is
needed to prevent any hw completions from queueing more I/O to the
card. Pass resetting flag to nvme_ctrlr_start. It doesn't need to get
that from the global state of the ctrlr. Wait for any pending reset to
finish. All queued I/O will get sent to the hardware as part of
nvme_ctrlr_start(), though the upper layers shouldn't send any
down. Disabling the qpairs is the other failsafe to ensure all I/O is
queued.
Rename nvme_ctrlr_destory_qpairs to nvme_ctrlr_delete_qpairs to avoid
confusion with all the other destroy functions. It just removes the
queues in hardware, while the other _destroy_ functions tear down
driver data structures.
Split parts of the hardware reset function up so that I can
do part of the reset in suspsend. Split out the software disabling
of the qpairs into nvme_ctrlr_disable_qpairs.
Finally, fix a couple of spelling errors in comments related to
this.
Relnotes: Yes
MFC After: 1 week
Reviewed by: scottl@ (prior version)
Differential Revision: https://reviews.freebsd.org/D21493
2019-09-03 15:26:11 +00:00
|
|
|
* normally have in flight at one time. This should be viewed as a hint,
|
|
|
|
* not a hard limit and will need to be revisited when the upper layers
|
2017-08-28 23:54:20 +00:00
|
|
|
* of the storage system grows multi-queue support.
|
|
|
|
*/
|
2017-09-20 21:42:25 +00:00
|
|
|
ctrlr->max_hw_pend_io = num_trackers * ctrlr->num_io_queues * 3 / 4;
|
2017-08-28 23:54:20 +00:00
|
|
|
|
2012-09-17 19:23:01 +00:00
|
|
|
ctrlr->ioq = malloc(ctrlr->num_io_queues * sizeof(struct nvme_qpair),
|
2013-03-26 22:11:34 +00:00
|
|
|
M_NVME, M_ZERO | M_WAITOK);
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2019-09-23 17:53:47 +00:00
|
|
|
for (i = c = n = 0; i < ctrlr->num_io_queues; i++, c += n) {
|
2012-09-17 19:23:01 +00:00
|
|
|
qpair = &ctrlr->ioq[i];
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Admin queue has ID=0. IO queues start at ID=1 -
|
|
|
|
* hence the 'i+1' here.
|
2019-09-23 17:53:47 +00:00
|
|
|
*/
|
|
|
|
qpair->id = i + 1;
|
|
|
|
if (ctrlr->num_io_queues > 1) {
|
|
|
|
/* Find number of CPUs served by this queue. */
|
|
|
|
for (n = 1; QP(ctrlr, c + n) == i; n++)
|
|
|
|
;
|
|
|
|
/* Shuffle multiple NVMe devices between CPUs. */
|
|
|
|
qpair->cpu = c + (device_get_unit(ctrlr->dev)+n/2) % n;
|
|
|
|
qpair->domain = pcpu_find(qpair->cpu)->pc_domain;
|
|
|
|
} else {
|
|
|
|
qpair->cpu = CPU_FFS(&cpuset_domain[ctrlr->domain]) - 1;
|
|
|
|
qpair->domain = ctrlr->domain;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2012-09-17 19:23:01 +00:00
|
|
|
* For I/O queues, use the controller-wide max_xfer_size
|
|
|
|
* calculated in nvme_attach().
|
|
|
|
*/
|
2019-09-23 17:53:47 +00:00
|
|
|
error = nvme_qpair_construct(qpair, num_entries, num_trackers,
|
|
|
|
ctrlr);
|
2016-11-08 00:24:49 +00:00
|
|
|
if (error)
|
|
|
|
return (error);
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2016-01-07 16:18:32 +00:00
|
|
|
/*
|
|
|
|
* Do not bother binding interrupts if we only have one I/O
|
|
|
|
* interrupt thread for this controller.
|
|
|
|
*/
|
2016-01-07 16:09:56 +00:00
|
|
|
if (ctrlr->num_io_queues > 1)
|
2019-09-23 17:53:47 +00:00
|
|
|
bus_bind_intr(ctrlr->dev, qpair->res, qpair->cpu);
|
2012-09-17 19:23:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
2013-03-26 21:58:38 +00:00
|
|
|
static void
|
|
|
|
nvme_ctrlr_fail(struct nvme_controller *ctrlr)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
2019-12-13 18:35:48 +00:00
|
|
|
ctrlr->is_failed = true;
|
2019-08-21 20:17:30 +00:00
|
|
|
nvme_admin_qpair_disable(&ctrlr->adminq);
|
2013-03-26 21:58:38 +00:00
|
|
|
nvme_qpair_fail(&ctrlr->adminq);
|
2017-03-07 23:06:41 +00:00
|
|
|
if (ctrlr->ioq != NULL) {
|
2019-08-21 20:17:30 +00:00
|
|
|
for (i = 0; i < ctrlr->num_io_queues; i++) {
|
|
|
|
nvme_io_qpair_disable(&ctrlr->ioq[i]);
|
2017-03-07 23:06:41 +00:00
|
|
|
nvme_qpair_fail(&ctrlr->ioq[i]);
|
2019-08-21 20:17:30 +00:00
|
|
|
}
|
2017-03-07 23:06:41 +00:00
|
|
|
}
|
2013-03-26 21:58:38 +00:00
|
|
|
nvme_notify_fail_consumers(ctrlr);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
nvme_ctrlr_post_failed_request(struct nvme_controller *ctrlr,
|
|
|
|
struct nvme_request *req)
|
|
|
|
{
|
|
|
|
|
2013-04-12 17:36:48 +00:00
|
|
|
mtx_lock(&ctrlr->lock);
|
2013-03-26 21:58:38 +00:00
|
|
|
STAILQ_INSERT_TAIL(&ctrlr->fail_req, req, stailq);
|
2013-04-12 17:36:48 +00:00
|
|
|
mtx_unlock(&ctrlr->lock);
|
2013-03-26 21:58:38 +00:00
|
|
|
taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->fail_req_task);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
nvme_ctrlr_fail_req_task(void *arg, int pending)
|
|
|
|
{
|
|
|
|
struct nvme_controller *ctrlr = arg;
|
|
|
|
struct nvme_request *req;
|
|
|
|
|
2013-04-12 17:36:48 +00:00
|
|
|
mtx_lock(&ctrlr->lock);
|
2018-05-02 20:13:03 +00:00
|
|
|
while ((req = STAILQ_FIRST(&ctrlr->fail_req)) != NULL) {
|
2013-03-26 21:58:38 +00:00
|
|
|
STAILQ_REMOVE_HEAD(&ctrlr->fail_req, stailq);
|
2018-05-02 20:13:03 +00:00
|
|
|
mtx_unlock(&ctrlr->lock);
|
2013-03-26 21:58:38 +00:00
|
|
|
nvme_qpair_manual_complete_request(req->qpair, req,
|
2019-03-09 01:18:16 +00:00
|
|
|
NVME_SCT_GENERIC, NVME_SC_ABORTED_BY_REQUEST);
|
2018-05-02 20:13:03 +00:00
|
|
|
mtx_lock(&ctrlr->lock);
|
2013-03-26 21:58:38 +00:00
|
|
|
}
|
2013-04-12 17:36:48 +00:00
|
|
|
mtx_unlock(&ctrlr->lock);
|
2013-03-26 21:58:38 +00:00
|
|
|
}
|
|
|
|
|
2012-09-17 19:23:01 +00:00
|
|
|
static int
|
2015-07-23 15:50:39 +00:00
|
|
|
nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr, int desired_val)
|
2012-09-17 19:23:01 +00:00
|
|
|
{
|
2021-03-17 14:30:40 +00:00
|
|
|
int timeout = ticks + (uint64_t)ctrlr->ready_timeout_in_ms * hz / 1000;
|
2018-02-22 13:32:31 +00:00
|
|
|
uint32_t csts;
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2019-08-21 20:17:30 +00:00
|
|
|
while (1) {
|
|
|
|
csts = nvme_mmio_read_4(ctrlr, csts);
|
2021-02-08 20:08:48 +00:00
|
|
|
if (csts == NVME_GONE) /* Hot unplug. */
|
2019-08-21 20:17:30 +00:00
|
|
|
return (ENXIO);
|
|
|
|
if (((csts >> NVME_CSTS_REG_RDY_SHIFT) & NVME_CSTS_REG_RDY_MASK)
|
|
|
|
== desired_val)
|
|
|
|
break;
|
2021-03-17 14:30:40 +00:00
|
|
|
if (timeout - ticks < 0) {
|
2015-07-23 15:50:39 +00:00
|
|
|
nvme_printf(ctrlr, "controller ready did not become %d "
|
|
|
|
"within %d ms\n", desired_val, ctrlr->ready_timeout_in_ms);
|
2012-09-17 19:23:01 +00:00
|
|
|
return (ENXIO);
|
|
|
|
}
|
2021-03-17 14:30:40 +00:00
|
|
|
pause("nvmerdy", 1);
|
2012-09-17 19:23:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
2017-12-18 18:38:00 +00:00
|
|
|
static int
|
2012-09-17 19:23:01 +00:00
|
|
|
nvme_ctrlr_disable(struct nvme_controller *ctrlr)
|
|
|
|
{
|
2018-02-22 13:32:31 +00:00
|
|
|
uint32_t cc;
|
|
|
|
uint32_t csts;
|
|
|
|
uint8_t en, rdy;
|
2017-12-18 18:38:00 +00:00
|
|
|
int err;
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2018-02-22 13:32:31 +00:00
|
|
|
cc = nvme_mmio_read_4(ctrlr, cc);
|
|
|
|
csts = nvme_mmio_read_4(ctrlr, csts);
|
|
|
|
|
|
|
|
en = (cc >> NVME_CC_REG_EN_SHIFT) & NVME_CC_REG_EN_MASK;
|
|
|
|
rdy = (csts >> NVME_CSTS_REG_RDY_SHIFT) & NVME_CSTS_REG_RDY_MASK;
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2017-12-18 18:38:00 +00:00
|
|
|
/*
|
|
|
|
* Per 3.1.5 in NVME 1.3 spec, transitioning CC.EN from 0 to 1
|
|
|
|
* when CSTS.RDY is 1 or transitioning CC.EN from 1 to 0 when
|
|
|
|
* CSTS.RDY is 0 "has undefined results" So make sure that CSTS.RDY
|
|
|
|
* isn't the desired value. Short circuit if we're already disabled.
|
|
|
|
*/
|
2018-02-22 13:32:31 +00:00
|
|
|
if (en == 1) {
|
|
|
|
if (rdy == 0) {
|
2017-12-18 18:38:00 +00:00
|
|
|
/* EN == 1, wait for RDY == 1 or fail */
|
|
|
|
err = nvme_ctrlr_wait_for_ready(ctrlr, 1);
|
|
|
|
if (err != 0)
|
|
|
|
return (err);
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
/* EN == 0 already wait for RDY == 0 */
|
2018-02-22 13:32:31 +00:00
|
|
|
if (rdy == 0)
|
2017-12-18 18:38:00 +00:00
|
|
|
return (0);
|
|
|
|
else
|
|
|
|
return (nvme_ctrlr_wait_for_ready(ctrlr, 0));
|
|
|
|
}
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2018-02-22 13:32:31 +00:00
|
|
|
cc &= ~NVME_CC_REG_EN_MASK;
|
|
|
|
nvme_mmio_write_4(ctrlr, cc, cc);
|
2017-12-18 18:38:00 +00:00
|
|
|
/*
|
|
|
|
* Some drives have issues with accessing the mmio after we
|
|
|
|
* disable, so delay for a bit after we write the bit to
|
|
|
|
* cope with these issues.
|
|
|
|
*/
|
2017-12-18 20:11:21 +00:00
|
|
|
if (ctrlr->quirks & QUIRK_DELAY_B4_CHK_RDY)
|
2017-12-18 18:38:00 +00:00
|
|
|
pause("nvmeR", B4_CHK_RDY_DELAY_MS * hz / 1000);
|
|
|
|
return (nvme_ctrlr_wait_for_ready(ctrlr, 0));
|
2012-09-17 19:23:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
nvme_ctrlr_enable(struct nvme_controller *ctrlr)
|
|
|
|
{
|
2018-02-22 13:32:31 +00:00
|
|
|
uint32_t cc;
|
|
|
|
uint32_t csts;
|
|
|
|
uint32_t aqa;
|
|
|
|
uint32_t qsize;
|
|
|
|
uint8_t en, rdy;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
cc = nvme_mmio_read_4(ctrlr, cc);
|
|
|
|
csts = nvme_mmio_read_4(ctrlr, csts);
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2018-02-22 13:32:31 +00:00
|
|
|
en = (cc >> NVME_CC_REG_EN_SHIFT) & NVME_CC_REG_EN_MASK;
|
|
|
|
rdy = (csts >> NVME_CSTS_REG_RDY_SHIFT) & NVME_CSTS_REG_RDY_MASK;
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2017-12-18 18:38:00 +00:00
|
|
|
/*
|
|
|
|
* See note in nvme_ctrlr_disable. Short circuit if we're already enabled.
|
|
|
|
*/
|
2018-02-22 13:32:31 +00:00
|
|
|
if (en == 1) {
|
|
|
|
if (rdy == 1)
|
2012-09-17 19:23:01 +00:00
|
|
|
return (0);
|
|
|
|
else
|
2015-07-23 15:50:39 +00:00
|
|
|
return (nvme_ctrlr_wait_for_ready(ctrlr, 1));
|
2017-12-18 18:38:00 +00:00
|
|
|
} else {
|
|
|
|
/* EN == 0 already wait for RDY == 0 or fail */
|
|
|
|
err = nvme_ctrlr_wait_for_ready(ctrlr, 0);
|
|
|
|
if (err != 0)
|
|
|
|
return (err);
|
2012-09-17 19:23:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
nvme_mmio_write_8(ctrlr, asq, ctrlr->adminq.cmd_bus_addr);
|
|
|
|
DELAY(5000);
|
|
|
|
nvme_mmio_write_8(ctrlr, acq, ctrlr->adminq.cpl_bus_addr);
|
|
|
|
DELAY(5000);
|
|
|
|
|
|
|
|
/* acqs and asqs are 0-based. */
|
2018-02-22 13:32:31 +00:00
|
|
|
qsize = ctrlr->adminq.num_entries - 1;
|
|
|
|
|
|
|
|
aqa = 0;
|
|
|
|
aqa = (qsize & NVME_AQA_REG_ACQS_MASK) << NVME_AQA_REG_ACQS_SHIFT;
|
|
|
|
aqa |= (qsize & NVME_AQA_REG_ASQS_MASK) << NVME_AQA_REG_ASQS_SHIFT;
|
|
|
|
nvme_mmio_write_4(ctrlr, aqa, aqa);
|
2012-09-17 19:23:01 +00:00
|
|
|
DELAY(5000);
|
|
|
|
|
2018-02-22 13:32:31 +00:00
|
|
|
/* Initialization values for CC */
|
|
|
|
cc = 0;
|
|
|
|
cc |= 1 << NVME_CC_REG_EN_SHIFT;
|
|
|
|
cc |= 0 << NVME_CC_REG_CSS_SHIFT;
|
|
|
|
cc |= 0 << NVME_CC_REG_AMS_SHIFT;
|
|
|
|
cc |= 0 << NVME_CC_REG_SHN_SHIFT;
|
|
|
|
cc |= 6 << NVME_CC_REG_IOSQES_SHIFT; /* SQ entry size == 64 == 2^6 */
|
|
|
|
cc |= 4 << NVME_CC_REG_IOCQES_SHIFT; /* CQ entry size == 16 == 2^4 */
|
2012-09-17 19:23:01 +00:00
|
|
|
|
|
|
|
/* This evaluates to 0, which is according to spec. */
|
2018-02-22 13:32:31 +00:00
|
|
|
cc |= (PAGE_SIZE >> 13) << NVME_CC_REG_MPS_SHIFT;
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2018-02-22 13:32:31 +00:00
|
|
|
nvme_mmio_write_4(ctrlr, cc, cc);
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2015-07-23 15:50:39 +00:00
|
|
|
return (nvme_ctrlr_wait_for_ready(ctrlr, 1));
|
2012-09-17 19:23:01 +00:00
|
|
|
}
|
|
|
|
|
Implement nvme suspend / resume for pci attachment
When we suspend, we need to properly shutdown the NVME controller. The
controller may go into D3 state (or may have the power removed), and
to properly flush the metadata to non-volatile RAM, we must complete a
normal shutdown. This consists of deleting the I/O queues and setting
the shutodown bit. We have to do some extra stuff to make sure we
reset the software state of the queues as well.
On resume, we have to reset the card twice, for reasons described in
the attach funcion. Once we've done that, we can restart the card. If
any of this fails, we'll fail the NVMe card, just like we do when a
reset fails.
Set is_resetting for the duration of the suspend / resume. This keeps
the reset taskqueue from running a concurrent reset, and also is
needed to prevent any hw completions from queueing more I/O to the
card. Pass resetting flag to nvme_ctrlr_start. It doesn't need to get
that from the global state of the ctrlr. Wait for any pending reset to
finish. All queued I/O will get sent to the hardware as part of
nvme_ctrlr_start(), though the upper layers shouldn't send any
down. Disabling the qpairs is the other failsafe to ensure all I/O is
queued.
Rename nvme_ctrlr_destory_qpairs to nvme_ctrlr_delete_qpairs to avoid
confusion with all the other destroy functions. It just removes the
queues in hardware, while the other _destroy_ functions tear down
driver data structures.
Split parts of the hardware reset function up so that I can
do part of the reset in suspsend. Split out the software disabling
of the qpairs into nvme_ctrlr_disable_qpairs.
Finally, fix a couple of spelling errors in comments related to
this.
Relnotes: Yes
MFC After: 1 week
Reviewed by: scottl@ (prior version)
Differential Revision: https://reviews.freebsd.org/D21493
2019-09-03 15:26:11 +00:00
|
|
|
static void
|
|
|
|
nvme_ctrlr_disable_qpairs(struct nvme_controller *ctrlr)
|
2012-09-17 19:23:01 +00:00
|
|
|
{
|
Implement nvme suspend / resume for pci attachment
When we suspend, we need to properly shutdown the NVME controller. The
controller may go into D3 state (or may have the power removed), and
to properly flush the metadata to non-volatile RAM, we must complete a
normal shutdown. This consists of deleting the I/O queues and setting
the shutodown bit. We have to do some extra stuff to make sure we
reset the software state of the queues as well.
On resume, we have to reset the card twice, for reasons described in
the attach funcion. Once we've done that, we can restart the card. If
any of this fails, we'll fail the NVMe card, just like we do when a
reset fails.
Set is_resetting for the duration of the suspend / resume. This keeps
the reset taskqueue from running a concurrent reset, and also is
needed to prevent any hw completions from queueing more I/O to the
card. Pass resetting flag to nvme_ctrlr_start. It doesn't need to get
that from the global state of the ctrlr. Wait for any pending reset to
finish. All queued I/O will get sent to the hardware as part of
nvme_ctrlr_start(), though the upper layers shouldn't send any
down. Disabling the qpairs is the other failsafe to ensure all I/O is
queued.
Rename nvme_ctrlr_destory_qpairs to nvme_ctrlr_delete_qpairs to avoid
confusion with all the other destroy functions. It just removes the
queues in hardware, while the other _destroy_ functions tear down
driver data structures.
Split parts of the hardware reset function up so that I can
do part of the reset in suspsend. Split out the software disabling
of the qpairs into nvme_ctrlr_disable_qpairs.
Finally, fix a couple of spelling errors in comments related to
this.
Relnotes: Yes
MFC After: 1 week
Reviewed by: scottl@ (prior version)
Differential Revision: https://reviews.freebsd.org/D21493
2019-09-03 15:26:11 +00:00
|
|
|
int i;
|
2013-03-26 19:50:46 +00:00
|
|
|
|
|
|
|
nvme_admin_qpair_disable(&ctrlr->adminq);
|
2016-01-07 16:18:32 +00:00
|
|
|
/*
|
|
|
|
* I/O queues are not allocated before the initial HW
|
|
|
|
* reset, so do not try to disable them. Use is_initialized
|
|
|
|
* to determine if this is the initial HW reset.
|
|
|
|
*/
|
|
|
|
if (ctrlr->is_initialized) {
|
|
|
|
for (i = 0; i < ctrlr->num_io_queues; i++)
|
|
|
|
nvme_io_qpair_disable(&ctrlr->ioq[i]);
|
|
|
|
}
|
Implement nvme suspend / resume for pci attachment
When we suspend, we need to properly shutdown the NVME controller. The
controller may go into D3 state (or may have the power removed), and
to properly flush the metadata to non-volatile RAM, we must complete a
normal shutdown. This consists of deleting the I/O queues and setting
the shutodown bit. We have to do some extra stuff to make sure we
reset the software state of the queues as well.
On resume, we have to reset the card twice, for reasons described in
the attach funcion. Once we've done that, we can restart the card. If
any of this fails, we'll fail the NVMe card, just like we do when a
reset fails.
Set is_resetting for the duration of the suspend / resume. This keeps
the reset taskqueue from running a concurrent reset, and also is
needed to prevent any hw completions from queueing more I/O to the
card. Pass resetting flag to nvme_ctrlr_start. It doesn't need to get
that from the global state of the ctrlr. Wait for any pending reset to
finish. All queued I/O will get sent to the hardware as part of
nvme_ctrlr_start(), though the upper layers shouldn't send any
down. Disabling the qpairs is the other failsafe to ensure all I/O is
queued.
Rename nvme_ctrlr_destory_qpairs to nvme_ctrlr_delete_qpairs to avoid
confusion with all the other destroy functions. It just removes the
queues in hardware, while the other _destroy_ functions tear down
driver data structures.
Split parts of the hardware reset function up so that I can
do part of the reset in suspsend. Split out the software disabling
of the qpairs into nvme_ctrlr_disable_qpairs.
Finally, fix a couple of spelling errors in comments related to
this.
Relnotes: Yes
MFC After: 1 week
Reviewed by: scottl@ (prior version)
Differential Revision: https://reviews.freebsd.org/D21493
2019-09-03 15:26:11 +00:00
|
|
|
}
|
|
|
|
|
2021-02-08 20:28:18 +00:00
|
|
|
static int
|
Implement nvme suspend / resume for pci attachment
When we suspend, we need to properly shutdown the NVME controller. The
controller may go into D3 state (or may have the power removed), and
to properly flush the metadata to non-volatile RAM, we must complete a
normal shutdown. This consists of deleting the I/O queues and setting
the shutodown bit. We have to do some extra stuff to make sure we
reset the software state of the queues as well.
On resume, we have to reset the card twice, for reasons described in
the attach funcion. Once we've done that, we can restart the card. If
any of this fails, we'll fail the NVMe card, just like we do when a
reset fails.
Set is_resetting for the duration of the suspend / resume. This keeps
the reset taskqueue from running a concurrent reset, and also is
needed to prevent any hw completions from queueing more I/O to the
card. Pass resetting flag to nvme_ctrlr_start. It doesn't need to get
that from the global state of the ctrlr. Wait for any pending reset to
finish. All queued I/O will get sent to the hardware as part of
nvme_ctrlr_start(), though the upper layers shouldn't send any
down. Disabling the qpairs is the other failsafe to ensure all I/O is
queued.
Rename nvme_ctrlr_destory_qpairs to nvme_ctrlr_delete_qpairs to avoid
confusion with all the other destroy functions. It just removes the
queues in hardware, while the other _destroy_ functions tear down
driver data structures.
Split parts of the hardware reset function up so that I can
do part of the reset in suspsend. Split out the software disabling
of the qpairs into nvme_ctrlr_disable_qpairs.
Finally, fix a couple of spelling errors in comments related to
this.
Relnotes: Yes
MFC After: 1 week
Reviewed by: scottl@ (prior version)
Differential Revision: https://reviews.freebsd.org/D21493
2019-09-03 15:26:11 +00:00
|
|
|
nvme_ctrlr_hw_reset(struct nvme_controller *ctrlr)
|
|
|
|
{
|
|
|
|
int err;
|
|
|
|
|
|
|
|
nvme_ctrlr_disable_qpairs(ctrlr);
|
2013-03-26 19:50:46 +00:00
|
|
|
|
2021-03-17 14:30:40 +00:00
|
|
|
pause("nvmehwreset", hz / 10);
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2017-12-18 18:38:00 +00:00
|
|
|
err = nvme_ctrlr_disable(ctrlr);
|
|
|
|
if (err != 0)
|
|
|
|
return err;
|
2012-09-17 19:23:01 +00:00
|
|
|
return (nvme_ctrlr_enable(ctrlr));
|
|
|
|
}
|
|
|
|
|
2013-03-26 19:50:46 +00:00
|
|
|
void
|
|
|
|
nvme_ctrlr_reset(struct nvme_controller *ctrlr)
|
|
|
|
{
|
2013-03-26 20:56:58 +00:00
|
|
|
int cmpset;
|
|
|
|
|
|
|
|
cmpset = atomic_cmpset_32(&ctrlr->is_resetting, 0, 1);
|
|
|
|
|
2013-03-26 21:58:38 +00:00
|
|
|
if (cmpset == 0 || ctrlr->is_failed)
|
|
|
|
/*
|
|
|
|
* Controller is already resetting or has failed. Return
|
|
|
|
* immediately since there is no need to kick off another
|
|
|
|
* reset in these cases.
|
|
|
|
*/
|
2013-03-26 20:56:58 +00:00
|
|
|
return;
|
2013-03-26 19:50:46 +00:00
|
|
|
|
2013-03-26 20:32:57 +00:00
|
|
|
taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->reset_task);
|
2013-03-26 19:50:46 +00:00
|
|
|
}
|
|
|
|
|
2012-09-17 19:23:01 +00:00
|
|
|
static int
|
|
|
|
nvme_ctrlr_identify(struct nvme_controller *ctrlr)
|
|
|
|
{
|
2013-03-26 22:09:51 +00:00
|
|
|
struct nvme_completion_poll_status status;
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2018-01-29 00:00:52 +00:00
|
|
|
status.done = 0;
|
2012-09-17 19:23:01 +00:00
|
|
|
nvme_ctrlr_cmd_identify_controller(ctrlr, &ctrlr->cdata,
|
2013-03-26 22:09:51 +00:00
|
|
|
nvme_completion_poll_cb, &status);
|
2019-09-02 17:11:27 +00:00
|
|
|
nvme_completion_poll(&status);
|
2013-03-26 22:09:51 +00:00
|
|
|
if (nvme_completion_is_error(&status.cpl)) {
|
2013-03-26 22:17:10 +00:00
|
|
|
nvme_printf(ctrlr, "nvme_identify_controller failed!\n");
|
2012-09-17 19:23:01 +00:00
|
|
|
return (ENXIO);
|
|
|
|
}
|
|
|
|
|
2018-02-22 13:32:31 +00:00
|
|
|
/* Convert data to host endian */
|
|
|
|
nvme_controller_data_swapbytes(&ctrlr->cdata);
|
|
|
|
|
2013-03-26 21:16:53 +00:00
|
|
|
/*
|
|
|
|
* Use MDTS to ensure our default max_xfer_size doesn't exceed what the
|
|
|
|
* controller supports.
|
|
|
|
*/
|
|
|
|
if (ctrlr->cdata.mdts > 0)
|
|
|
|
ctrlr->max_xfer_size = min(ctrlr->max_xfer_size,
|
|
|
|
ctrlr->min_page_size * (1 << (ctrlr->cdata.mdts)));
|
|
|
|
|
2012-09-17 19:23:01 +00:00
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
nvme_ctrlr_set_num_qpairs(struct nvme_controller *ctrlr)
|
|
|
|
{
|
2013-03-26 22:09:51 +00:00
|
|
|
struct nvme_completion_poll_status status;
|
2016-01-07 16:18:32 +00:00
|
|
|
int cq_allocated, sq_allocated;
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2018-01-29 00:00:52 +00:00
|
|
|
status.done = 0;
|
2012-09-17 19:23:01 +00:00
|
|
|
nvme_ctrlr_cmd_set_num_queues(ctrlr, ctrlr->num_io_queues,
|
2013-03-26 22:09:51 +00:00
|
|
|
nvme_completion_poll_cb, &status);
|
2019-09-02 17:11:27 +00:00
|
|
|
nvme_completion_poll(&status);
|
2013-03-26 22:09:51 +00:00
|
|
|
if (nvme_completion_is_error(&status.cpl)) {
|
2017-03-07 23:06:41 +00:00
|
|
|
nvme_printf(ctrlr, "nvme_ctrlr_set_num_qpairs failed!\n");
|
2012-09-17 19:23:01 +00:00
|
|
|
return (ENXIO);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Data in cdw0 is 0-based.
|
|
|
|
* Lower 16-bits indicate number of submission queues allocated.
|
|
|
|
* Upper 16-bits indicate number of completion queues allocated.
|
|
|
|
*/
|
2013-03-26 22:09:51 +00:00
|
|
|
sq_allocated = (status.cpl.cdw0 & 0xFFFF) + 1;
|
|
|
|
cq_allocated = (status.cpl.cdw0 >> 16) + 1;
|
2012-09-17 19:23:01 +00:00
|
|
|
|
|
|
|
/*
|
2016-01-07 16:18:32 +00:00
|
|
|
* Controller may allocate more queues than we requested,
|
|
|
|
* so use the minimum of the number requested and what was
|
|
|
|
* actually allocated.
|
2012-09-17 19:23:01 +00:00
|
|
|
*/
|
2016-01-07 16:18:32 +00:00
|
|
|
ctrlr->num_io_queues = min(ctrlr->num_io_queues, sq_allocated);
|
|
|
|
ctrlr->num_io_queues = min(ctrlr->num_io_queues, cq_allocated);
|
2019-09-23 17:53:47 +00:00
|
|
|
if (ctrlr->num_io_queues > vm_ndomains)
|
|
|
|
ctrlr->num_io_queues -= ctrlr->num_io_queues % vm_ndomains;
|
2012-09-17 19:23:01 +00:00
|
|
|
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr)
|
|
|
|
{
|
2013-03-26 22:09:51 +00:00
|
|
|
struct nvme_completion_poll_status status;
|
|
|
|
struct nvme_qpair *qpair;
|
|
|
|
int i;
|
2012-09-17 19:23:01 +00:00
|
|
|
|
|
|
|
for (i = 0; i < ctrlr->num_io_queues; i++) {
|
|
|
|
qpair = &ctrlr->ioq[i];
|
|
|
|
|
2018-01-29 00:00:52 +00:00
|
|
|
status.done = 0;
|
2019-09-23 17:53:47 +00:00
|
|
|
nvme_ctrlr_cmd_create_io_cq(ctrlr, qpair,
|
2013-03-26 22:09:51 +00:00
|
|
|
nvme_completion_poll_cb, &status);
|
2019-09-02 17:11:27 +00:00
|
|
|
nvme_completion_poll(&status);
|
2013-03-26 22:09:51 +00:00
|
|
|
if (nvme_completion_is_error(&status.cpl)) {
|
2013-03-26 22:17:10 +00:00
|
|
|
nvme_printf(ctrlr, "nvme_create_io_cq failed!\n");
|
2012-09-17 19:23:01 +00:00
|
|
|
return (ENXIO);
|
|
|
|
}
|
|
|
|
|
2018-01-29 00:00:52 +00:00
|
|
|
status.done = 0;
|
2020-06-18 19:16:03 +00:00
|
|
|
nvme_ctrlr_cmd_create_io_sq(ctrlr, qpair,
|
2013-03-26 22:09:51 +00:00
|
|
|
nvme_completion_poll_cb, &status);
|
2019-09-02 17:11:27 +00:00
|
|
|
nvme_completion_poll(&status);
|
2013-03-26 22:09:51 +00:00
|
|
|
if (nvme_completion_is_error(&status.cpl)) {
|
2013-03-26 22:17:10 +00:00
|
|
|
nvme_printf(ctrlr, "nvme_create_io_sq failed!\n");
|
2012-09-17 19:23:01 +00:00
|
|
|
return (ENXIO);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
2018-03-14 23:01:18 +00:00
|
|
|
static int
|
Implement nvme suspend / resume for pci attachment
When we suspend, we need to properly shutdown the NVME controller. The
controller may go into D3 state (or may have the power removed), and
to properly flush the metadata to non-volatile RAM, we must complete a
normal shutdown. This consists of deleting the I/O queues and setting
the shutodown bit. We have to do some extra stuff to make sure we
reset the software state of the queues as well.
On resume, we have to reset the card twice, for reasons described in
the attach funcion. Once we've done that, we can restart the card. If
any of this fails, we'll fail the NVMe card, just like we do when a
reset fails.
Set is_resetting for the duration of the suspend / resume. This keeps
the reset taskqueue from running a concurrent reset, and also is
needed to prevent any hw completions from queueing more I/O to the
card. Pass resetting flag to nvme_ctrlr_start. It doesn't need to get
that from the global state of the ctrlr. Wait for any pending reset to
finish. All queued I/O will get sent to the hardware as part of
nvme_ctrlr_start(), though the upper layers shouldn't send any
down. Disabling the qpairs is the other failsafe to ensure all I/O is
queued.
Rename nvme_ctrlr_destory_qpairs to nvme_ctrlr_delete_qpairs to avoid
confusion with all the other destroy functions. It just removes the
queues in hardware, while the other _destroy_ functions tear down
driver data structures.
Split parts of the hardware reset function up so that I can
do part of the reset in suspsend. Split out the software disabling
of the qpairs into nvme_ctrlr_disable_qpairs.
Finally, fix a couple of spelling errors in comments related to
this.
Relnotes: Yes
MFC After: 1 week
Reviewed by: scottl@ (prior version)
Differential Revision: https://reviews.freebsd.org/D21493
2019-09-03 15:26:11 +00:00
|
|
|
nvme_ctrlr_delete_qpairs(struct nvme_controller *ctrlr)
|
2018-03-14 23:01:18 +00:00
|
|
|
{
|
|
|
|
struct nvme_completion_poll_status status;
|
2019-05-08 20:18:11 +00:00
|
|
|
struct nvme_qpair *qpair;
|
2018-03-14 23:01:18 +00:00
|
|
|
|
2019-05-08 20:18:11 +00:00
|
|
|
for (int i = 0; i < ctrlr->num_io_queues; i++) {
|
|
|
|
qpair = &ctrlr->ioq[i];
|
2018-03-14 23:01:18 +00:00
|
|
|
|
2019-05-08 20:18:11 +00:00
|
|
|
status.done = 0;
|
|
|
|
nvme_ctrlr_cmd_delete_io_sq(ctrlr, qpair,
|
|
|
|
nvme_completion_poll_cb, &status);
|
2019-09-02 17:11:27 +00:00
|
|
|
nvme_completion_poll(&status);
|
2019-05-08 20:18:11 +00:00
|
|
|
if (nvme_completion_is_error(&status.cpl)) {
|
|
|
|
nvme_printf(ctrlr, "nvme_destroy_io_sq failed!\n");
|
|
|
|
return (ENXIO);
|
|
|
|
}
|
|
|
|
|
|
|
|
status.done = 0;
|
|
|
|
nvme_ctrlr_cmd_delete_io_cq(ctrlr, qpair,
|
|
|
|
nvme_completion_poll_cb, &status);
|
2019-09-02 17:11:27 +00:00
|
|
|
nvme_completion_poll(&status);
|
2019-05-08 20:18:11 +00:00
|
|
|
if (nvme_completion_is_error(&status.cpl)) {
|
|
|
|
nvme_printf(ctrlr, "nvme_destroy_io_cq failed!\n");
|
|
|
|
return (ENXIO);
|
|
|
|
}
|
2018-03-14 23:01:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
2012-09-17 19:23:01 +00:00
|
|
|
static int
|
|
|
|
nvme_ctrlr_construct_namespaces(struct nvme_controller *ctrlr)
|
|
|
|
{
|
|
|
|
struct nvme_namespace *ns;
|
2017-08-25 21:38:38 +00:00
|
|
|
uint32_t i;
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2017-03-07 21:47:54 +00:00
|
|
|
for (i = 0; i < min(ctrlr->cdata.nn, NVME_MAX_NAMESPACES); i++) {
|
2012-09-17 19:23:01 +00:00
|
|
|
ns = &ctrlr->ns[i];
|
2017-03-07 21:47:54 +00:00
|
|
|
nvme_ns_construct(ns, i+1, ctrlr);
|
2012-09-17 19:23:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
2019-12-13 18:35:48 +00:00
|
|
|
static bool
|
2013-03-26 21:05:15 +00:00
|
|
|
is_log_page_id_valid(uint8_t page_id)
|
|
|
|
{
|
|
|
|
|
|
|
|
switch (page_id) {
|
|
|
|
case NVME_LOG_ERROR:
|
|
|
|
case NVME_LOG_HEALTH_INFORMATION:
|
|
|
|
case NVME_LOG_FIRMWARE_SLOT:
|
2018-05-25 03:34:33 +00:00
|
|
|
case NVME_LOG_CHANGED_NAMESPACE:
|
2019-08-02 20:16:21 +00:00
|
|
|
case NVME_LOG_COMMAND_EFFECT:
|
|
|
|
case NVME_LOG_RES_NOTIFICATION:
|
|
|
|
case NVME_LOG_SANITIZE_STATUS:
|
2019-12-13 18:35:48 +00:00
|
|
|
return (true);
|
2013-03-26 21:05:15 +00:00
|
|
|
}
|
|
|
|
|
2019-12-13 18:35:48 +00:00
|
|
|
return (false);
|
2013-03-26 21:05:15 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static uint32_t
|
|
|
|
nvme_ctrlr_get_log_page_size(struct nvme_controller *ctrlr, uint8_t page_id)
|
|
|
|
{
|
|
|
|
uint32_t log_page_size;
|
|
|
|
|
|
|
|
switch (page_id) {
|
|
|
|
case NVME_LOG_ERROR:
|
|
|
|
log_page_size = min(
|
|
|
|
sizeof(struct nvme_error_information_entry) *
|
2018-02-22 13:32:31 +00:00
|
|
|
(ctrlr->cdata.elpe + 1), NVME_MAX_AER_LOG_SIZE);
|
2013-03-26 21:05:15 +00:00
|
|
|
break;
|
|
|
|
case NVME_LOG_HEALTH_INFORMATION:
|
|
|
|
log_page_size = sizeof(struct nvme_health_information_page);
|
|
|
|
break;
|
|
|
|
case NVME_LOG_FIRMWARE_SLOT:
|
|
|
|
log_page_size = sizeof(struct nvme_firmware_page);
|
|
|
|
break;
|
2018-05-25 03:34:33 +00:00
|
|
|
case NVME_LOG_CHANGED_NAMESPACE:
|
|
|
|
log_page_size = sizeof(struct nvme_ns_list);
|
|
|
|
break;
|
2019-08-02 20:16:21 +00:00
|
|
|
case NVME_LOG_COMMAND_EFFECT:
|
|
|
|
log_page_size = sizeof(struct nvme_command_effects_page);
|
|
|
|
break;
|
|
|
|
case NVME_LOG_RES_NOTIFICATION:
|
|
|
|
log_page_size = sizeof(struct nvme_res_notification_page);
|
|
|
|
break;
|
|
|
|
case NVME_LOG_SANITIZE_STATUS:
|
|
|
|
log_page_size = sizeof(struct nvme_sanitize_status_page);
|
|
|
|
break;
|
2013-03-26 21:05:15 +00:00
|
|
|
default:
|
|
|
|
log_page_size = 0;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return (log_page_size);
|
|
|
|
}
|
|
|
|
|
2013-10-08 16:00:12 +00:00
|
|
|
static void
|
|
|
|
nvme_ctrlr_log_critical_warnings(struct nvme_controller *ctrlr,
|
2018-02-22 13:32:31 +00:00
|
|
|
uint8_t state)
|
2013-10-08 16:00:12 +00:00
|
|
|
{
|
|
|
|
|
2018-02-22 13:32:31 +00:00
|
|
|
if (state & NVME_CRIT_WARN_ST_AVAILABLE_SPARE)
|
2020-04-30 00:27:19 +00:00
|
|
|
nvme_ctrlr_devctl_log(ctrlr, "critical",
|
|
|
|
"available spare space below threshold");
|
2013-10-08 16:00:12 +00:00
|
|
|
|
2018-02-22 13:32:31 +00:00
|
|
|
if (state & NVME_CRIT_WARN_ST_TEMPERATURE)
|
2020-04-30 00:27:19 +00:00
|
|
|
nvme_ctrlr_devctl_log(ctrlr, "critical",
|
|
|
|
"temperature above threshold");
|
2013-10-08 16:00:12 +00:00
|
|
|
|
2018-02-22 13:32:31 +00:00
|
|
|
if (state & NVME_CRIT_WARN_ST_DEVICE_RELIABILITY)
|
2020-04-30 00:27:19 +00:00
|
|
|
nvme_ctrlr_devctl_log(ctrlr, "critical",
|
|
|
|
"device reliability degraded");
|
2013-10-08 16:00:12 +00:00
|
|
|
|
2018-02-22 13:32:31 +00:00
|
|
|
if (state & NVME_CRIT_WARN_ST_READ_ONLY)
|
2020-04-30 00:27:19 +00:00
|
|
|
nvme_ctrlr_devctl_log(ctrlr, "critical",
|
|
|
|
"media placed in read only mode");
|
2013-10-08 16:00:12 +00:00
|
|
|
|
2018-02-22 13:32:31 +00:00
|
|
|
if (state & NVME_CRIT_WARN_ST_VOLATILE_MEMORY_BACKUP)
|
2020-04-30 00:27:19 +00:00
|
|
|
nvme_ctrlr_devctl_log(ctrlr, "critical",
|
|
|
|
"volatile memory backup device failed");
|
2013-10-08 16:00:12 +00:00
|
|
|
|
2018-02-22 13:32:31 +00:00
|
|
|
if (state & NVME_CRIT_WARN_ST_RESERVED_MASK)
|
2020-04-30 00:27:19 +00:00
|
|
|
nvme_ctrlr_devctl_log(ctrlr, "critical",
|
|
|
|
"unknown critical warning(s): state = 0x%02x", state);
|
2013-10-08 16:00:12 +00:00
|
|
|
}
|
|
|
|
|
2013-03-26 21:05:15 +00:00
|
|
|
static void
|
|
|
|
nvme_ctrlr_async_event_log_page_cb(void *arg, const struct nvme_completion *cpl)
|
|
|
|
{
|
2013-10-08 16:00:12 +00:00
|
|
|
struct nvme_async_event_request *aer = arg;
|
|
|
|
struct nvme_health_information_page *health_info;
|
2018-05-25 03:34:33 +00:00
|
|
|
struct nvme_ns_list *nsl;
|
2018-02-22 13:32:31 +00:00
|
|
|
struct nvme_error_information_entry *err;
|
|
|
|
int i;
|
2013-03-26 21:05:15 +00:00
|
|
|
|
2013-03-26 21:08:32 +00:00
|
|
|
/*
|
|
|
|
* If the log page fetch for some reason completed with an error,
|
|
|
|
* don't pass log page data to the consumers. In practice, this case
|
|
|
|
* should never happen.
|
|
|
|
*/
|
|
|
|
if (nvme_completion_is_error(cpl))
|
|
|
|
nvme_notify_async_consumers(aer->ctrlr, &aer->cpl,
|
|
|
|
aer->log_page_id, NULL, 0);
|
2013-10-08 16:00:12 +00:00
|
|
|
else {
|
2018-02-22 13:32:31 +00:00
|
|
|
/* Convert data to host endian */
|
|
|
|
switch (aer->log_page_id) {
|
|
|
|
case NVME_LOG_ERROR:
|
|
|
|
err = (struct nvme_error_information_entry *)aer->log_page_buffer;
|
|
|
|
for (i = 0; i < (aer->ctrlr->cdata.elpe + 1); i++)
|
|
|
|
nvme_error_information_entry_swapbytes(err++);
|
|
|
|
break;
|
|
|
|
case NVME_LOG_HEALTH_INFORMATION:
|
|
|
|
nvme_health_information_page_swapbytes(
|
|
|
|
(struct nvme_health_information_page *)aer->log_page_buffer);
|
|
|
|
break;
|
|
|
|
case NVME_LOG_FIRMWARE_SLOT:
|
|
|
|
nvme_firmware_page_swapbytes(
|
|
|
|
(struct nvme_firmware_page *)aer->log_page_buffer);
|
|
|
|
break;
|
2018-05-25 03:34:33 +00:00
|
|
|
case NVME_LOG_CHANGED_NAMESPACE:
|
|
|
|
nvme_ns_list_swapbytes(
|
|
|
|
(struct nvme_ns_list *)aer->log_page_buffer);
|
|
|
|
break;
|
2019-08-02 20:16:21 +00:00
|
|
|
case NVME_LOG_COMMAND_EFFECT:
|
|
|
|
nvme_command_effects_page_swapbytes(
|
|
|
|
(struct nvme_command_effects_page *)aer->log_page_buffer);
|
|
|
|
break;
|
|
|
|
case NVME_LOG_RES_NOTIFICATION:
|
|
|
|
nvme_res_notification_page_swapbytes(
|
|
|
|
(struct nvme_res_notification_page *)aer->log_page_buffer);
|
|
|
|
break;
|
|
|
|
case NVME_LOG_SANITIZE_STATUS:
|
|
|
|
nvme_sanitize_status_page_swapbytes(
|
|
|
|
(struct nvme_sanitize_status_page *)aer->log_page_buffer);
|
|
|
|
break;
|
2018-02-22 13:32:31 +00:00
|
|
|
case INTEL_LOG_TEMP_STATS:
|
|
|
|
intel_log_temp_stats_swapbytes(
|
|
|
|
(struct intel_log_temp_stats *)aer->log_page_buffer);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2013-10-08 16:00:12 +00:00
|
|
|
if (aer->log_page_id == NVME_LOG_HEALTH_INFORMATION) {
|
|
|
|
health_info = (struct nvme_health_information_page *)
|
|
|
|
aer->log_page_buffer;
|
|
|
|
nvme_ctrlr_log_critical_warnings(aer->ctrlr,
|
|
|
|
health_info->critical_warning);
|
|
|
|
/*
|
|
|
|
* Critical warnings reported through the
|
|
|
|
* SMART/health log page are persistent, so
|
|
|
|
* clear the associated bits in the async event
|
|
|
|
* config so that we do not receive repeated
|
|
|
|
* notifications for the same event.
|
|
|
|
*/
|
2018-02-22 13:32:31 +00:00
|
|
|
aer->ctrlr->async_event_config &=
|
|
|
|
~health_info->critical_warning;
|
2013-10-08 16:00:12 +00:00
|
|
|
nvme_ctrlr_cmd_set_async_event_config(aer->ctrlr,
|
|
|
|
aer->ctrlr->async_event_config, NULL, NULL);
|
2018-05-25 03:34:33 +00:00
|
|
|
} else if (aer->log_page_id == NVME_LOG_CHANGED_NAMESPACE &&
|
|
|
|
!nvme_use_nvd) {
|
|
|
|
nsl = (struct nvme_ns_list *)aer->log_page_buffer;
|
|
|
|
for (i = 0; i < nitems(nsl->ns) && nsl->ns[i] != 0; i++) {
|
|
|
|
if (nsl->ns[i] > NVME_MAX_NAMESPACES)
|
|
|
|
break;
|
|
|
|
nvme_notify_ns(aer->ctrlr, nsl->ns[i]);
|
|
|
|
}
|
2013-10-08 16:00:12 +00:00
|
|
|
}
|
|
|
|
|
2013-03-26 21:08:32 +00:00
|
|
|
/*
|
|
|
|
* Pass the cpl data from the original async event completion,
|
|
|
|
* not the log page fetch.
|
|
|
|
*/
|
|
|
|
nvme_notify_async_consumers(aer->ctrlr, &aer->cpl,
|
|
|
|
aer->log_page_id, aer->log_page_buffer, aer->log_page_size);
|
2013-10-08 16:00:12 +00:00
|
|
|
}
|
2013-03-26 21:05:15 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Repost another asynchronous event request to replace the one
|
|
|
|
* that just completed.
|
|
|
|
*/
|
|
|
|
nvme_ctrlr_construct_and_submit_aer(aer->ctrlr, aer);
|
|
|
|
}
|
|
|
|
|
2013-03-26 18:37:36 +00:00
|
|
|
static void
|
|
|
|
nvme_ctrlr_async_event_cb(void *arg, const struct nvme_completion *cpl)
|
|
|
|
{
|
2013-03-26 21:05:15 +00:00
|
|
|
struct nvme_async_event_request *aer = arg;
|
2013-03-26 18:37:36 +00:00
|
|
|
|
2013-07-09 21:03:39 +00:00
|
|
|
if (nvme_completion_is_error(cpl)) {
|
2013-03-26 18:37:36 +00:00
|
|
|
/*
|
2013-07-09 21:03:39 +00:00
|
|
|
* Do not retry failed async event requests. This avoids
|
|
|
|
* infinite loops where a new async event request is submitted
|
|
|
|
* to replace the one just failed, only to fail again and
|
|
|
|
* perpetuate the loop.
|
2013-03-26 18:37:36 +00:00
|
|
|
*/
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2013-03-26 21:05:15 +00:00
|
|
|
/* Associated log page is in bits 23:16 of completion entry dw0. */
|
2013-03-26 21:08:32 +00:00
|
|
|
aer->log_page_id = (cpl->cdw0 & 0xFF0000) >> 16;
|
2013-03-26 21:05:15 +00:00
|
|
|
|
2018-05-25 03:34:33 +00:00
|
|
|
nvme_printf(aer->ctrlr, "async event occurred (type 0x%x, info 0x%02x,"
|
2019-08-03 02:36:35 +00:00
|
|
|
" page 0x%02x)\n", (cpl->cdw0 & 0x07), (cpl->cdw0 & 0xFF00) >> 8,
|
2013-03-26 22:17:10 +00:00
|
|
|
aer->log_page_id);
|
|
|
|
|
2013-03-26 21:08:32 +00:00
|
|
|
if (is_log_page_id_valid(aer->log_page_id)) {
|
2013-03-26 21:05:15 +00:00
|
|
|
aer->log_page_size = nvme_ctrlr_get_log_page_size(aer->ctrlr,
|
2013-03-26 21:08:32 +00:00
|
|
|
aer->log_page_id);
|
2013-03-26 21:05:15 +00:00
|
|
|
memcpy(&aer->cpl, cpl, sizeof(*cpl));
|
2013-03-26 21:08:32 +00:00
|
|
|
nvme_ctrlr_cmd_get_log_page(aer->ctrlr, aer->log_page_id,
|
2013-03-26 21:05:15 +00:00
|
|
|
NVME_GLOBAL_NAMESPACE_TAG, aer->log_page_buffer,
|
|
|
|
aer->log_page_size, nvme_ctrlr_async_event_log_page_cb,
|
|
|
|
aer);
|
|
|
|
/* Wait to notify consumers until after log page is fetched. */
|
|
|
|
} else {
|
2013-03-26 21:08:32 +00:00
|
|
|
nvme_notify_async_consumers(aer->ctrlr, cpl, aer->log_page_id,
|
|
|
|
NULL, 0);
|
2013-03-26 18:37:36 +00:00
|
|
|
|
2013-03-26 21:05:15 +00:00
|
|
|
/*
|
|
|
|
* Repost another asynchronous event request to replace the one
|
|
|
|
* that just completed.
|
|
|
|
*/
|
|
|
|
nvme_ctrlr_construct_and_submit_aer(aer->ctrlr, aer);
|
|
|
|
}
|
2013-03-26 18:37:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,
|
|
|
|
struct nvme_async_event_request *aer)
|
|
|
|
{
|
|
|
|
struct nvme_request *req;
|
|
|
|
|
|
|
|
aer->ctrlr = ctrlr;
|
2013-03-29 20:34:28 +00:00
|
|
|
req = nvme_allocate_request_null(nvme_ctrlr_async_event_cb, aer);
|
2013-03-26 18:37:36 +00:00
|
|
|
aer->req = req;
|
|
|
|
|
|
|
|
/*
|
2013-03-26 20:02:35 +00:00
|
|
|
* Disable timeout here, since asynchronous event requests should by
|
|
|
|
* nature never be timed out.
|
2013-03-26 18:37:36 +00:00
|
|
|
*/
|
2019-12-13 18:35:48 +00:00
|
|
|
req->timeout = false;
|
2018-08-22 04:29:24 +00:00
|
|
|
req->cmd.opc = NVME_OPC_ASYNC_EVENT_REQUEST;
|
2013-03-26 18:37:36 +00:00
|
|
|
nvme_ctrlr_submit_admin_request(ctrlr, req);
|
|
|
|
}
|
|
|
|
|
2012-09-17 19:23:01 +00:00
|
|
|
static void
|
|
|
|
nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr)
|
|
|
|
{
|
2013-10-08 15:49:14 +00:00
|
|
|
struct nvme_completion_poll_status status;
|
2013-03-26 18:37:36 +00:00
|
|
|
struct nvme_async_event_request *aer;
|
|
|
|
uint32_t i;
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2018-05-25 03:34:33 +00:00
|
|
|
ctrlr->async_event_config = NVME_CRIT_WARN_ST_AVAILABLE_SPARE |
|
|
|
|
NVME_CRIT_WARN_ST_DEVICE_RELIABILITY |
|
|
|
|
NVME_CRIT_WARN_ST_READ_ONLY |
|
|
|
|
NVME_CRIT_WARN_ST_VOLATILE_MEMORY_BACKUP;
|
|
|
|
if (ctrlr->cdata.ver >= NVME_REV(1, 2))
|
2020-08-31 19:38:03 +00:00
|
|
|
ctrlr->async_event_config |= NVME_ASYNC_EVENT_NS_ATTRIBUTE |
|
|
|
|
NVME_ASYNC_EVENT_FW_ACTIVATE;
|
2013-10-08 15:49:14 +00:00
|
|
|
|
2018-01-29 00:00:52 +00:00
|
|
|
status.done = 0;
|
2013-10-08 15:49:14 +00:00
|
|
|
nvme_ctrlr_cmd_get_feature(ctrlr, NVME_FEAT_TEMPERATURE_THRESHOLD,
|
|
|
|
0, NULL, 0, nvme_completion_poll_cb, &status);
|
2019-09-02 17:11:27 +00:00
|
|
|
nvme_completion_poll(&status);
|
2013-10-08 15:49:14 +00:00
|
|
|
if (nvme_completion_is_error(&status.cpl) ||
|
|
|
|
(status.cpl.cdw0 & 0xFFFF) == 0xFFFF ||
|
|
|
|
(status.cpl.cdw0 & 0xFFFF) == 0x0000) {
|
|
|
|
nvme_printf(ctrlr, "temperature threshold not supported\n");
|
2018-05-25 03:34:33 +00:00
|
|
|
} else
|
|
|
|
ctrlr->async_event_config |= NVME_CRIT_WARN_ST_TEMPERATURE;
|
2013-10-08 15:49:14 +00:00
|
|
|
|
2013-10-08 16:00:12 +00:00
|
|
|
nvme_ctrlr_cmd_set_async_event_config(ctrlr,
|
|
|
|
ctrlr->async_event_config, NULL, NULL);
|
2012-09-17 19:23:01 +00:00
|
|
|
|
|
|
|
/* aerl is a zero-based value, so we need to add 1 here. */
|
2013-03-26 18:37:36 +00:00
|
|
|
ctrlr->num_aers = min(NVME_MAX_ASYNC_EVENTS, (ctrlr->cdata.aerl+1));
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2013-03-26 18:37:36 +00:00
|
|
|
for (i = 0; i < ctrlr->num_aers; i++) {
|
|
|
|
aer = &ctrlr->aer[i];
|
|
|
|
nvme_ctrlr_construct_and_submit_aer(ctrlr, aer);
|
|
|
|
}
|
2012-09-17 19:23:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
nvme_ctrlr_configure_int_coalescing(struct nvme_controller *ctrlr)
|
|
|
|
{
|
|
|
|
|
|
|
|
ctrlr->int_coal_time = 0;
|
|
|
|
TUNABLE_INT_FETCH("hw.nvme.int_coal_time",
|
|
|
|
&ctrlr->int_coal_time);
|
|
|
|
|
|
|
|
ctrlr->int_coal_threshold = 0;
|
|
|
|
TUNABLE_INT_FETCH("hw.nvme.int_coal_threshold",
|
|
|
|
&ctrlr->int_coal_threshold);
|
|
|
|
|
|
|
|
nvme_ctrlr_cmd_set_interrupt_coalescing(ctrlr, ctrlr->int_coal_time,
|
|
|
|
ctrlr->int_coal_threshold, NULL, NULL);
|
|
|
|
}
|
|
|
|
|
2020-01-07 21:17:11 +00:00
|
|
|
static void
|
|
|
|
nvme_ctrlr_hmb_free(struct nvme_controller *ctrlr)
|
|
|
|
{
|
|
|
|
struct nvme_hmb_chunk *hmbc;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (ctrlr->hmb_desc_paddr) {
|
|
|
|
bus_dmamap_unload(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_map);
|
|
|
|
bus_dmamem_free(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_vaddr,
|
|
|
|
ctrlr->hmb_desc_map);
|
|
|
|
ctrlr->hmb_desc_paddr = 0;
|
|
|
|
}
|
|
|
|
if (ctrlr->hmb_desc_tag) {
|
|
|
|
bus_dma_tag_destroy(ctrlr->hmb_desc_tag);
|
2020-01-08 18:26:23 +00:00
|
|
|
ctrlr->hmb_desc_tag = NULL;
|
2020-01-07 21:17:11 +00:00
|
|
|
}
|
|
|
|
for (i = 0; i < ctrlr->hmb_nchunks; i++) {
|
|
|
|
hmbc = &ctrlr->hmb_chunks[i];
|
|
|
|
bus_dmamap_unload(ctrlr->hmb_tag, hmbc->hmbc_map);
|
|
|
|
bus_dmamem_free(ctrlr->hmb_tag, hmbc->hmbc_vaddr,
|
|
|
|
hmbc->hmbc_map);
|
|
|
|
}
|
|
|
|
ctrlr->hmb_nchunks = 0;
|
|
|
|
if (ctrlr->hmb_tag) {
|
|
|
|
bus_dma_tag_destroy(ctrlr->hmb_tag);
|
|
|
|
ctrlr->hmb_tag = NULL;
|
|
|
|
}
|
|
|
|
if (ctrlr->hmb_chunks) {
|
|
|
|
free(ctrlr->hmb_chunks, M_NVME);
|
|
|
|
ctrlr->hmb_chunks = NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
nvme_ctrlr_hmb_alloc(struct nvme_controller *ctrlr)
|
|
|
|
{
|
|
|
|
struct nvme_hmb_chunk *hmbc;
|
|
|
|
size_t pref, min, minc, size;
|
|
|
|
int err, i;
|
|
|
|
uint64_t max;
|
|
|
|
|
2020-01-07 23:10:38 +00:00
|
|
|
/* Limit HMB to 5% of RAM size per device by default. */
|
|
|
|
max = (uint64_t)physmem * PAGE_SIZE / 20;
|
2020-01-07 21:17:11 +00:00
|
|
|
TUNABLE_UINT64_FETCH("hw.nvme.hmb_max", &max);
|
|
|
|
|
|
|
|
min = (long long unsigned)ctrlr->cdata.hmmin * 4096;
|
2020-01-07 23:29:54 +00:00
|
|
|
if (max == 0 || max < min)
|
2020-01-07 21:17:11 +00:00
|
|
|
return;
|
|
|
|
pref = MIN((long long unsigned)ctrlr->cdata.hmpre * 4096, max);
|
|
|
|
minc = MAX(ctrlr->cdata.hmminds * 4096, PAGE_SIZE);
|
|
|
|
if (min > 0 && ctrlr->cdata.hmmaxd > 0)
|
|
|
|
minc = MAX(minc, min / ctrlr->cdata.hmmaxd);
|
|
|
|
ctrlr->hmb_chunk = pref;
|
|
|
|
|
|
|
|
again:
|
|
|
|
ctrlr->hmb_chunk = roundup2(ctrlr->hmb_chunk, PAGE_SIZE);
|
|
|
|
ctrlr->hmb_nchunks = howmany(pref, ctrlr->hmb_chunk);
|
|
|
|
if (ctrlr->cdata.hmmaxd > 0 && ctrlr->hmb_nchunks > ctrlr->cdata.hmmaxd)
|
|
|
|
ctrlr->hmb_nchunks = ctrlr->cdata.hmmaxd;
|
|
|
|
ctrlr->hmb_chunks = malloc(sizeof(struct nvme_hmb_chunk) *
|
|
|
|
ctrlr->hmb_nchunks, M_NVME, M_WAITOK);
|
|
|
|
err = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev),
|
|
|
|
PAGE_SIZE, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
|
|
|
|
ctrlr->hmb_chunk, 1, ctrlr->hmb_chunk, 0, NULL, NULL, &ctrlr->hmb_tag);
|
|
|
|
if (err != 0) {
|
|
|
|
nvme_printf(ctrlr, "HMB tag create failed %d\n", err);
|
|
|
|
nvme_ctrlr_hmb_free(ctrlr);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < ctrlr->hmb_nchunks; i++) {
|
|
|
|
hmbc = &ctrlr->hmb_chunks[i];
|
|
|
|
if (bus_dmamem_alloc(ctrlr->hmb_tag,
|
|
|
|
(void **)&hmbc->hmbc_vaddr, BUS_DMA_NOWAIT,
|
|
|
|
&hmbc->hmbc_map)) {
|
|
|
|
nvme_printf(ctrlr, "failed to alloc HMB\n");
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (bus_dmamap_load(ctrlr->hmb_tag, hmbc->hmbc_map,
|
|
|
|
hmbc->hmbc_vaddr, ctrlr->hmb_chunk, nvme_single_map,
|
|
|
|
&hmbc->hmbc_paddr, BUS_DMA_NOWAIT) != 0) {
|
|
|
|
bus_dmamem_free(ctrlr->hmb_tag, hmbc->hmbc_vaddr,
|
|
|
|
hmbc->hmbc_map);
|
|
|
|
nvme_printf(ctrlr, "failed to load HMB\n");
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
bus_dmamap_sync(ctrlr->hmb_tag, hmbc->hmbc_map,
|
|
|
|
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (i < ctrlr->hmb_nchunks && i * ctrlr->hmb_chunk < min &&
|
|
|
|
ctrlr->hmb_chunk / 2 >= minc) {
|
|
|
|
ctrlr->hmb_nchunks = i;
|
|
|
|
nvme_ctrlr_hmb_free(ctrlr);
|
|
|
|
ctrlr->hmb_chunk /= 2;
|
|
|
|
goto again;
|
|
|
|
}
|
|
|
|
ctrlr->hmb_nchunks = i;
|
|
|
|
if (ctrlr->hmb_nchunks * ctrlr->hmb_chunk < min) {
|
|
|
|
nvme_ctrlr_hmb_free(ctrlr);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
size = sizeof(struct nvme_hmb_desc) * ctrlr->hmb_nchunks;
|
|
|
|
err = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev),
|
|
|
|
16, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
|
|
|
|
size, 1, size, 0, NULL, NULL, &ctrlr->hmb_desc_tag);
|
|
|
|
if (err != 0) {
|
|
|
|
nvme_printf(ctrlr, "HMB desc tag create failed %d\n", err);
|
|
|
|
nvme_ctrlr_hmb_free(ctrlr);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
if (bus_dmamem_alloc(ctrlr->hmb_desc_tag,
|
|
|
|
(void **)&ctrlr->hmb_desc_vaddr, BUS_DMA_WAITOK,
|
|
|
|
&ctrlr->hmb_desc_map)) {
|
|
|
|
nvme_printf(ctrlr, "failed to alloc HMB desc\n");
|
|
|
|
nvme_ctrlr_hmb_free(ctrlr);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
if (bus_dmamap_load(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_map,
|
|
|
|
ctrlr->hmb_desc_vaddr, size, nvme_single_map,
|
|
|
|
&ctrlr->hmb_desc_paddr, BUS_DMA_NOWAIT) != 0) {
|
|
|
|
bus_dmamem_free(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_vaddr,
|
|
|
|
ctrlr->hmb_desc_map);
|
|
|
|
nvme_printf(ctrlr, "failed to load HMB desc\n");
|
|
|
|
nvme_ctrlr_hmb_free(ctrlr);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < ctrlr->hmb_nchunks; i++) {
|
|
|
|
ctrlr->hmb_desc_vaddr[i].addr =
|
|
|
|
htole64(ctrlr->hmb_chunks[i].hmbc_paddr);
|
|
|
|
ctrlr->hmb_desc_vaddr[i].size = htole32(ctrlr->hmb_chunk / 4096);
|
|
|
|
}
|
|
|
|
bus_dmamap_sync(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_map,
|
|
|
|
BUS_DMASYNC_PREWRITE);
|
|
|
|
|
|
|
|
nvme_printf(ctrlr, "Allocated %lluMB host memory buffer\n",
|
|
|
|
(long long unsigned)ctrlr->hmb_nchunks * ctrlr->hmb_chunk
|
|
|
|
/ 1024 / 1024);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
nvme_ctrlr_hmb_enable(struct nvme_controller *ctrlr, bool enable, bool memret)
|
|
|
|
{
|
|
|
|
struct nvme_completion_poll_status status;
|
|
|
|
uint32_t cdw11;
|
|
|
|
|
|
|
|
cdw11 = 0;
|
|
|
|
if (enable)
|
|
|
|
cdw11 |= 1;
|
|
|
|
if (memret)
|
|
|
|
cdw11 |= 2;
|
|
|
|
status.done = 0;
|
|
|
|
nvme_ctrlr_cmd_set_feature(ctrlr, NVME_FEAT_HOST_MEMORY_BUFFER, cdw11,
|
|
|
|
ctrlr->hmb_nchunks * ctrlr->hmb_chunk / 4096, ctrlr->hmb_desc_paddr,
|
|
|
|
ctrlr->hmb_desc_paddr >> 32, ctrlr->hmb_nchunks, NULL, 0,
|
|
|
|
nvme_completion_poll_cb, &status);
|
|
|
|
nvme_completion_poll(&status);
|
|
|
|
if (nvme_completion_is_error(&status.cpl))
|
|
|
|
nvme_printf(ctrlr, "nvme_ctrlr_hmb_enable failed!\n");
|
|
|
|
}
|
|
|
|
|
2013-03-26 21:19:26 +00:00
|
|
|
static void
|
Implement nvme suspend / resume for pci attachment
When we suspend, we need to properly shutdown the NVME controller. The
controller may go into D3 state (or may have the power removed), and
to properly flush the metadata to non-volatile RAM, we must complete a
normal shutdown. This consists of deleting the I/O queues and setting
the shutodown bit. We have to do some extra stuff to make sure we
reset the software state of the queues as well.
On resume, we have to reset the card twice, for reasons described in
the attach funcion. Once we've done that, we can restart the card. If
any of this fails, we'll fail the NVMe card, just like we do when a
reset fails.
Set is_resetting for the duration of the suspend / resume. This keeps
the reset taskqueue from running a concurrent reset, and also is
needed to prevent any hw completions from queueing more I/O to the
card. Pass resetting flag to nvme_ctrlr_start. It doesn't need to get
that from the global state of the ctrlr. Wait for any pending reset to
finish. All queued I/O will get sent to the hardware as part of
nvme_ctrlr_start(), though the upper layers shouldn't send any
down. Disabling the qpairs is the other failsafe to ensure all I/O is
queued.
Rename nvme_ctrlr_destory_qpairs to nvme_ctrlr_delete_qpairs to avoid
confusion with all the other destroy functions. It just removes the
queues in hardware, while the other _destroy_ functions tear down
driver data structures.
Split parts of the hardware reset function up so that I can
do part of the reset in suspsend. Split out the software disabling
of the qpairs into nvme_ctrlr_disable_qpairs.
Finally, fix a couple of spelling errors in comments related to
this.
Relnotes: Yes
MFC After: 1 week
Reviewed by: scottl@ (prior version)
Differential Revision: https://reviews.freebsd.org/D21493
2019-09-03 15:26:11 +00:00
|
|
|
nvme_ctrlr_start(void *ctrlr_arg, bool resetting)
|
2012-09-17 19:23:01 +00:00
|
|
|
{
|
|
|
|
struct nvme_controller *ctrlr = ctrlr_arg;
|
2016-01-07 16:18:32 +00:00
|
|
|
uint32_t old_num_io_queues;
|
2013-03-26 19:50:46 +00:00
|
|
|
int i;
|
|
|
|
|
2016-01-07 16:18:32 +00:00
|
|
|
/*
|
|
|
|
* Only reset adminq here when we are restarting the
|
|
|
|
* controller after a reset. During initialization,
|
|
|
|
* we have already submitted admin commands to get
|
|
|
|
* the number of I/O queues supported, so cannot reset
|
|
|
|
* the adminq again here.
|
|
|
|
*/
|
2020-11-29 00:20:31 +00:00
|
|
|
if (resetting) {
|
2016-01-07 16:18:32 +00:00
|
|
|
nvme_qpair_reset(&ctrlr->adminq);
|
2020-11-29 00:20:31 +00:00
|
|
|
nvme_admin_qpair_enable(&ctrlr->adminq);
|
|
|
|
}
|
2016-01-07 16:18:32 +00:00
|
|
|
|
2020-06-25 20:29:29 +00:00
|
|
|
if (ctrlr->ioq != NULL) {
|
|
|
|
for (i = 0; i < ctrlr->num_io_queues; i++)
|
|
|
|
nvme_qpair_reset(&ctrlr->ioq[i]);
|
|
|
|
}
|
2013-03-26 21:14:51 +00:00
|
|
|
|
2020-06-25 20:29:29 +00:00
|
|
|
/*
|
|
|
|
* If it was a reset on initialization command timeout, just
|
|
|
|
* return here, letting initialization code fail gracefully.
|
|
|
|
*/
|
|
|
|
if (resetting && !ctrlr->is_initialized)
|
|
|
|
return;
|
|
|
|
|
2020-11-29 00:20:31 +00:00
|
|
|
if (resetting && nvme_ctrlr_identify(ctrlr) != 0) {
|
2013-03-26 21:58:38 +00:00
|
|
|
nvme_ctrlr_fail(ctrlr);
|
2013-03-26 21:19:26 +00:00
|
|
|
return;
|
2013-03-26 21:58:38 +00:00
|
|
|
}
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2016-01-07 16:18:32 +00:00
|
|
|
/*
|
|
|
|
* The number of qpairs are determined during controller initialization,
|
|
|
|
* including using NVMe SET_FEATURES/NUMBER_OF_QUEUES to determine the
|
|
|
|
* HW limit. We call SET_FEATURES again here so that it gets called
|
|
|
|
* after any reset for controllers that depend on the driver to
|
|
|
|
* explicit specify how many queues it will use. This value should
|
|
|
|
* never change between resets, so panic if somehow that does happen.
|
|
|
|
*/
|
Implement nvme suspend / resume for pci attachment
When we suspend, we need to properly shutdown the NVME controller. The
controller may go into D3 state (or may have the power removed), and
to properly flush the metadata to non-volatile RAM, we must complete a
normal shutdown. This consists of deleting the I/O queues and setting
the shutodown bit. We have to do some extra stuff to make sure we
reset the software state of the queues as well.
On resume, we have to reset the card twice, for reasons described in
the attach funcion. Once we've done that, we can restart the card. If
any of this fails, we'll fail the NVMe card, just like we do when a
reset fails.
Set is_resetting for the duration of the suspend / resume. This keeps
the reset taskqueue from running a concurrent reset, and also is
needed to prevent any hw completions from queueing more I/O to the
card. Pass resetting flag to nvme_ctrlr_start. It doesn't need to get
that from the global state of the ctrlr. Wait for any pending reset to
finish. All queued I/O will get sent to the hardware as part of
nvme_ctrlr_start(), though the upper layers shouldn't send any
down. Disabling the qpairs is the other failsafe to ensure all I/O is
queued.
Rename nvme_ctrlr_destory_qpairs to nvme_ctrlr_delete_qpairs to avoid
confusion with all the other destroy functions. It just removes the
queues in hardware, while the other _destroy_ functions tear down
driver data structures.
Split parts of the hardware reset function up so that I can
do part of the reset in suspsend. Split out the software disabling
of the qpairs into nvme_ctrlr_disable_qpairs.
Finally, fix a couple of spelling errors in comments related to
this.
Relnotes: Yes
MFC After: 1 week
Reviewed by: scottl@ (prior version)
Differential Revision: https://reviews.freebsd.org/D21493
2019-09-03 15:26:11 +00:00
|
|
|
if (resetting) {
|
2016-02-11 17:32:41 +00:00
|
|
|
old_num_io_queues = ctrlr->num_io_queues;
|
|
|
|
if (nvme_ctrlr_set_num_qpairs(ctrlr) != 0) {
|
|
|
|
nvme_ctrlr_fail(ctrlr);
|
|
|
|
return;
|
|
|
|
}
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2016-02-11 17:32:41 +00:00
|
|
|
if (old_num_io_queues != ctrlr->num_io_queues) {
|
|
|
|
panic("num_io_queues changed from %u to %u",
|
|
|
|
old_num_io_queues, ctrlr->num_io_queues);
|
|
|
|
}
|
2016-01-07 16:18:32 +00:00
|
|
|
}
|
|
|
|
|
2020-01-07 21:17:11 +00:00
|
|
|
if (ctrlr->cdata.hmpre > 0 && ctrlr->hmb_nchunks == 0) {
|
|
|
|
nvme_ctrlr_hmb_alloc(ctrlr);
|
|
|
|
if (ctrlr->hmb_nchunks > 0)
|
|
|
|
nvme_ctrlr_hmb_enable(ctrlr, true, false);
|
|
|
|
} else if (ctrlr->hmb_nchunks > 0)
|
|
|
|
nvme_ctrlr_hmb_enable(ctrlr, true, true);
|
|
|
|
|
2013-03-26 21:58:38 +00:00
|
|
|
if (nvme_ctrlr_create_qpairs(ctrlr) != 0) {
|
|
|
|
nvme_ctrlr_fail(ctrlr);
|
2013-03-26 21:19:26 +00:00
|
|
|
return;
|
2013-03-26 21:58:38 +00:00
|
|
|
}
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2013-03-26 21:58:38 +00:00
|
|
|
if (nvme_ctrlr_construct_namespaces(ctrlr) != 0) {
|
|
|
|
nvme_ctrlr_fail(ctrlr);
|
2013-03-26 21:19:26 +00:00
|
|
|
return;
|
2013-03-26 21:58:38 +00:00
|
|
|
}
|
2012-09-17 19:23:01 +00:00
|
|
|
|
|
|
|
nvme_ctrlr_configure_aer(ctrlr);
|
|
|
|
nvme_ctrlr_configure_int_coalescing(ctrlr);
|
|
|
|
|
2013-03-26 19:50:46 +00:00
|
|
|
for (i = 0; i < ctrlr->num_io_queues; i++)
|
|
|
|
nvme_io_qpair_enable(&ctrlr->ioq[i]);
|
2013-03-26 21:19:26 +00:00
|
|
|
}
|
2013-03-26 19:50:46 +00:00
|
|
|
|
2013-03-26 21:19:26 +00:00
|
|
|
void
|
|
|
|
nvme_ctrlr_start_config_hook(void *arg)
|
|
|
|
{
|
|
|
|
struct nvme_controller *ctrlr = arg;
|
2019-12-11 22:51:02 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Reset controller twice to ensure we do a transition from cc.en==1 to
|
|
|
|
* cc.en==0. This is because we don't really know what status the
|
|
|
|
* controller was left in when boot handed off to OS. Linux doesn't do
|
|
|
|
* this, however. If we adopt that policy, see also nvme_ctrlr_resume().
|
|
|
|
*/
|
2020-06-25 20:29:29 +00:00
|
|
|
if (nvme_ctrlr_hw_reset(ctrlr) != 0) {
|
|
|
|
fail:
|
2019-12-11 22:51:02 +00:00
|
|
|
nvme_ctrlr_fail(ctrlr);
|
2020-06-12 14:14:01 +00:00
|
|
|
config_intrhook_disestablish(&ctrlr->config_hook);
|
2019-12-11 22:51:02 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2020-06-25 20:29:29 +00:00
|
|
|
if (nvme_ctrlr_hw_reset(ctrlr) != 0)
|
|
|
|
goto fail;
|
2013-03-26 19:50:46 +00:00
|
|
|
|
2016-01-07 16:18:32 +00:00
|
|
|
nvme_qpair_reset(&ctrlr->adminq);
|
|
|
|
nvme_admin_qpair_enable(&ctrlr->adminq);
|
|
|
|
|
2020-11-29 00:20:31 +00:00
|
|
|
if (nvme_ctrlr_identify(ctrlr) == 0 &&
|
|
|
|
nvme_ctrlr_set_num_qpairs(ctrlr) == 0 &&
|
2016-01-07 16:18:32 +00:00
|
|
|
nvme_ctrlr_construct_io_qpairs(ctrlr) == 0)
|
Implement nvme suspend / resume for pci attachment
When we suspend, we need to properly shutdown the NVME controller. The
controller may go into D3 state (or may have the power removed), and
to properly flush the metadata to non-volatile RAM, we must complete a
normal shutdown. This consists of deleting the I/O queues and setting
the shutodown bit. We have to do some extra stuff to make sure we
reset the software state of the queues as well.
On resume, we have to reset the card twice, for reasons described in
the attach funcion. Once we've done that, we can restart the card. If
any of this fails, we'll fail the NVMe card, just like we do when a
reset fails.
Set is_resetting for the duration of the suspend / resume. This keeps
the reset taskqueue from running a concurrent reset, and also is
needed to prevent any hw completions from queueing more I/O to the
card. Pass resetting flag to nvme_ctrlr_start. It doesn't need to get
that from the global state of the ctrlr. Wait for any pending reset to
finish. All queued I/O will get sent to the hardware as part of
nvme_ctrlr_start(), though the upper layers shouldn't send any
down. Disabling the qpairs is the other failsafe to ensure all I/O is
queued.
Rename nvme_ctrlr_destory_qpairs to nvme_ctrlr_delete_qpairs to avoid
confusion with all the other destroy functions. It just removes the
queues in hardware, while the other _destroy_ functions tear down
driver data structures.
Split parts of the hardware reset function up so that I can
do part of the reset in suspsend. Split out the software disabling
of the qpairs into nvme_ctrlr_disable_qpairs.
Finally, fix a couple of spelling errors in comments related to
this.
Relnotes: Yes
MFC After: 1 week
Reviewed by: scottl@ (prior version)
Differential Revision: https://reviews.freebsd.org/D21493
2019-09-03 15:26:11 +00:00
|
|
|
nvme_ctrlr_start(ctrlr, false);
|
2016-01-07 16:18:32 +00:00
|
|
|
else
|
2020-06-25 20:29:29 +00:00
|
|
|
goto fail;
|
2016-01-07 16:18:32 +00:00
|
|
|
|
|
|
|
nvme_sysctl_initialize_ctrlr(ctrlr);
|
2013-03-26 21:19:26 +00:00
|
|
|
config_intrhook_disestablish(&ctrlr->config_hook);
|
2014-03-18 18:09:08 +00:00
|
|
|
|
|
|
|
ctrlr->is_initialized = 1;
|
|
|
|
nvme_notify_new_controller(ctrlr);
|
2012-09-17 19:23:01 +00:00
|
|
|
}
|
|
|
|
|
2013-03-26 19:58:17 +00:00
|
|
|
static void
|
2013-03-26 20:32:57 +00:00
|
|
|
nvme_ctrlr_reset_task(void *arg, int pending)
|
2013-03-26 19:58:17 +00:00
|
|
|
{
|
2013-03-26 20:32:57 +00:00
|
|
|
struct nvme_controller *ctrlr = arg;
|
|
|
|
int status;
|
2013-03-26 19:58:17 +00:00
|
|
|
|
2020-04-30 00:27:19 +00:00
|
|
|
nvme_ctrlr_devctl_log(ctrlr, "RESET", "resetting controller");
|
2013-03-26 20:32:57 +00:00
|
|
|
status = nvme_ctrlr_hw_reset(ctrlr);
|
|
|
|
/*
|
|
|
|
* Use pause instead of DELAY, so that we yield to any nvme interrupt
|
|
|
|
* handlers on this CPU that were blocked on a qpair lock. We want
|
|
|
|
* all nvme interrupts completed before proceeding with restarting the
|
|
|
|
* controller.
|
|
|
|
*
|
|
|
|
* XXX - any way to guarantee the interrupt handlers have quiesced?
|
|
|
|
*/
|
|
|
|
pause("nvmereset", hz / 10);
|
|
|
|
if (status == 0)
|
Implement nvme suspend / resume for pci attachment
When we suspend, we need to properly shutdown the NVME controller. The
controller may go into D3 state (or may have the power removed), and
to properly flush the metadata to non-volatile RAM, we must complete a
normal shutdown. This consists of deleting the I/O queues and setting
the shutodown bit. We have to do some extra stuff to make sure we
reset the software state of the queues as well.
On resume, we have to reset the card twice, for reasons described in
the attach funcion. Once we've done that, we can restart the card. If
any of this fails, we'll fail the NVMe card, just like we do when a
reset fails.
Set is_resetting for the duration of the suspend / resume. This keeps
the reset taskqueue from running a concurrent reset, and also is
needed to prevent any hw completions from queueing more I/O to the
card. Pass resetting flag to nvme_ctrlr_start. It doesn't need to get
that from the global state of the ctrlr. Wait for any pending reset to
finish. All queued I/O will get sent to the hardware as part of
nvme_ctrlr_start(), though the upper layers shouldn't send any
down. Disabling the qpairs is the other failsafe to ensure all I/O is
queued.
Rename nvme_ctrlr_destory_qpairs to nvme_ctrlr_delete_qpairs to avoid
confusion with all the other destroy functions. It just removes the
queues in hardware, while the other _destroy_ functions tear down
driver data structures.
Split parts of the hardware reset function up so that I can
do part of the reset in suspsend. Split out the software disabling
of the qpairs into nvme_ctrlr_disable_qpairs.
Finally, fix a couple of spelling errors in comments related to
this.
Relnotes: Yes
MFC After: 1 week
Reviewed by: scottl@ (prior version)
Differential Revision: https://reviews.freebsd.org/D21493
2019-09-03 15:26:11 +00:00
|
|
|
nvme_ctrlr_start(ctrlr, true);
|
2013-03-26 21:58:38 +00:00
|
|
|
else
|
|
|
|
nvme_ctrlr_fail(ctrlr);
|
2013-03-26 20:56:58 +00:00
|
|
|
|
|
|
|
atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);
|
2013-03-26 19:58:17 +00:00
|
|
|
}
|
|
|
|
|
2017-10-15 16:18:08 +00:00
|
|
|
/*
|
|
|
|
* Poll all the queues enabled on the device for completion.
|
|
|
|
*/
|
2016-06-10 06:04:53 +00:00
|
|
|
void
|
2017-10-15 16:18:08 +00:00
|
|
|
nvme_ctrlr_poll(struct nvme_controller *ctrlr)
|
2012-09-17 19:23:01 +00:00
|
|
|
{
|
2017-10-15 16:18:08 +00:00
|
|
|
int i;
|
2012-12-18 21:50:48 +00:00
|
|
|
|
2012-09-17 19:23:01 +00:00
|
|
|
nvme_qpair_process_completions(&ctrlr->adminq);
|
|
|
|
|
2017-10-15 16:18:08 +00:00
|
|
|
for (i = 0; i < ctrlr->num_io_queues; i++)
|
|
|
|
if (ctrlr->ioq && ctrlr->ioq[i].cpl)
|
|
|
|
nvme_qpair_process_completions(&ctrlr->ioq[i]);
|
|
|
|
}
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2017-10-15 16:18:08 +00:00
|
|
|
/*
|
Implement nvme suspend / resume for pci attachment
When we suspend, we need to properly shutdown the NVME controller. The
controller may go into D3 state (or may have the power removed), and
to properly flush the metadata to non-volatile RAM, we must complete a
normal shutdown. This consists of deleting the I/O queues and setting
the shutodown bit. We have to do some extra stuff to make sure we
reset the software state of the queues as well.
On resume, we have to reset the card twice, for reasons described in
the attach funcion. Once we've done that, we can restart the card. If
any of this fails, we'll fail the NVMe card, just like we do when a
reset fails.
Set is_resetting for the duration of the suspend / resume. This keeps
the reset taskqueue from running a concurrent reset, and also is
needed to prevent any hw completions from queueing more I/O to the
card. Pass resetting flag to nvme_ctrlr_start. It doesn't need to get
that from the global state of the ctrlr. Wait for any pending reset to
finish. All queued I/O will get sent to the hardware as part of
nvme_ctrlr_start(), though the upper layers shouldn't send any
down. Disabling the qpairs is the other failsafe to ensure all I/O is
queued.
Rename nvme_ctrlr_destory_qpairs to nvme_ctrlr_delete_qpairs to avoid
confusion with all the other destroy functions. It just removes the
queues in hardware, while the other _destroy_ functions tear down
driver data structures.
Split parts of the hardware reset function up so that I can
do part of the reset in suspsend. Split out the software disabling
of the qpairs into nvme_ctrlr_disable_qpairs.
Finally, fix a couple of spelling errors in comments related to
this.
Relnotes: Yes
MFC After: 1 week
Reviewed by: scottl@ (prior version)
Differential Revision: https://reviews.freebsd.org/D21493
2019-09-03 15:26:11 +00:00
|
|
|
* Poll the single-vector interrupt case: num_io_queues will be 1 and
|
2017-10-15 16:18:08 +00:00
|
|
|
* there's only a single vector. While we're polling, we mask further
|
|
|
|
* interrupts in the controller.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
nvme_ctrlr_intx_handler(void *arg)
|
|
|
|
{
|
|
|
|
struct nvme_controller *ctrlr = arg;
|
|
|
|
|
|
|
|
nvme_mmio_write_4(ctrlr, intms, 1);
|
|
|
|
nvme_ctrlr_poll(ctrlr);
|
2012-09-17 19:23:01 +00:00
|
|
|
nvme_mmio_write_4(ctrlr, intmc, 1);
|
|
|
|
}
|
|
|
|
|
2013-04-12 17:52:17 +00:00
|
|
|
static void
|
|
|
|
nvme_pt_done(void *arg, const struct nvme_completion *cpl)
|
|
|
|
{
|
|
|
|
struct nvme_pt_command *pt = arg;
|
2018-05-02 20:13:03 +00:00
|
|
|
struct mtx *mtx = pt->driver_lock;
|
2018-02-22 13:32:31 +00:00
|
|
|
uint16_t status;
|
2013-04-12 17:52:17 +00:00
|
|
|
|
|
|
|
bzero(&pt->cpl, sizeof(pt->cpl));
|
|
|
|
pt->cpl.cdw0 = cpl->cdw0;
|
2018-02-22 13:32:31 +00:00
|
|
|
|
|
|
|
status = cpl->status;
|
|
|
|
status &= ~NVME_STATUS_P_MASK;
|
|
|
|
pt->cpl.status = status;
|
2013-04-12 17:52:17 +00:00
|
|
|
|
2018-05-02 20:13:03 +00:00
|
|
|
mtx_lock(mtx);
|
|
|
|
pt->driver_lock = NULL;
|
2013-04-12 17:52:17 +00:00
|
|
|
wakeup(pt);
|
2018-05-02 20:13:03 +00:00
|
|
|
mtx_unlock(mtx);
|
2013-04-12 17:52:17 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
nvme_ctrlr_passthrough_cmd(struct nvme_controller *ctrlr,
|
|
|
|
struct nvme_pt_command *pt, uint32_t nsid, int is_user_buffer,
|
|
|
|
int is_admin_cmd)
|
|
|
|
{
|
|
|
|
struct nvme_request *req;
|
|
|
|
struct mtx *mtx;
|
|
|
|
struct buf *buf = NULL;
|
|
|
|
int ret = 0;
|
|
|
|
|
2013-06-26 23:32:45 +00:00
|
|
|
if (pt->len > 0) {
|
|
|
|
if (pt->len > ctrlr->max_xfer_size) {
|
|
|
|
nvme_printf(ctrlr, "pt->len (%d) "
|
|
|
|
"exceeds max_xfer_size (%d)\n", pt->len,
|
|
|
|
ctrlr->max_xfer_size);
|
|
|
|
return EIO;
|
|
|
|
}
|
2013-04-12 17:52:17 +00:00
|
|
|
if (is_user_buffer) {
|
|
|
|
/*
|
|
|
|
* Ensure the user buffer is wired for the duration of
|
Implement nvme suspend / resume for pci attachment
When we suspend, we need to properly shutdown the NVME controller. The
controller may go into D3 state (or may have the power removed), and
to properly flush the metadata to non-volatile RAM, we must complete a
normal shutdown. This consists of deleting the I/O queues and setting
the shutodown bit. We have to do some extra stuff to make sure we
reset the software state of the queues as well.
On resume, we have to reset the card twice, for reasons described in
the attach funcion. Once we've done that, we can restart the card. If
any of this fails, we'll fail the NVMe card, just like we do when a
reset fails.
Set is_resetting for the duration of the suspend / resume. This keeps
the reset taskqueue from running a concurrent reset, and also is
needed to prevent any hw completions from queueing more I/O to the
card. Pass resetting flag to nvme_ctrlr_start. It doesn't need to get
that from the global state of the ctrlr. Wait for any pending reset to
finish. All queued I/O will get sent to the hardware as part of
nvme_ctrlr_start(), though the upper layers shouldn't send any
down. Disabling the qpairs is the other failsafe to ensure all I/O is
queued.
Rename nvme_ctrlr_destory_qpairs to nvme_ctrlr_delete_qpairs to avoid
confusion with all the other destroy functions. It just removes the
queues in hardware, while the other _destroy_ functions tear down
driver data structures.
Split parts of the hardware reset function up so that I can
do part of the reset in suspsend. Split out the software disabling
of the qpairs into nvme_ctrlr_disable_qpairs.
Finally, fix a couple of spelling errors in comments related to
this.
Relnotes: Yes
MFC After: 1 week
Reviewed by: scottl@ (prior version)
Differential Revision: https://reviews.freebsd.org/D21493
2019-09-03 15:26:11 +00:00
|
|
|
* this pass-through command.
|
2013-04-12 17:52:17 +00:00
|
|
|
*/
|
|
|
|
PHOLD(curproc);
|
Allocate pager bufs from UMA instead of 80-ish mutex protected linked list.
o In vm_pager_bufferinit() create pbuf_zone and start accounting on how many
pbufs are we going to have set.
In various subsystems that are going to utilize pbufs create private zones
via call to pbuf_zsecond_create(). The latter calls uma_zsecond_create(),
and sets a limit on created zone. After startup preallocate pbufs according
to requirements of all pbuf zones.
Subsystems that used to have a private limit with old allocator now have
private pbuf zones: md(4), fusefs, NFS client, smbfs, VFS cluster, FFS,
swap, vnode pager.
The following subsystems use shared pbuf zone: cam(4), nvme(4), physio(9),
aio(4). They should have their private limits, but changing that is out of
scope of this commit.
o Fetch tunable value of kern.nswbuf from init_param2() and while here move
NSWBUF_MIN to opt_param.h and eliminate opt_swap.h, that was holding only
this option.
Default values aren't touched by this commit, but they probably should be
reviewed wrt to modern hardware.
This change removes a tight bottleneck from sendfile(2) operation, that
uses pbufs in vnode pager. Other pagers also would benefit from faster
allocation.
Together with: gallatin
Tested by: pho
2019-01-15 01:02:16 +00:00
|
|
|
buf = uma_zalloc(pbuf_zone, M_WAITOK);
|
2013-04-12 17:52:17 +00:00
|
|
|
buf->b_iocmd = pt->is_read ? BIO_READ : BIO_WRITE;
|
2020-10-21 16:00:15 +00:00
|
|
|
if (vmapbuf(buf, pt->buf, pt->len, 1) < 0) {
|
2013-04-12 17:52:17 +00:00
|
|
|
ret = EFAULT;
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
req = nvme_allocate_request_vaddr(buf->b_data, pt->len,
|
|
|
|
nvme_pt_done, pt);
|
|
|
|
} else
|
|
|
|
req = nvme_allocate_request_vaddr(pt->buf, pt->len,
|
|
|
|
nvme_pt_done, pt);
|
2013-06-26 23:32:45 +00:00
|
|
|
} else
|
2013-04-12 17:52:17 +00:00
|
|
|
req = nvme_allocate_request_null(nvme_pt_done, pt);
|
|
|
|
|
Implement nvme suspend / resume for pci attachment
When we suspend, we need to properly shutdown the NVME controller. The
controller may go into D3 state (or may have the power removed), and
to properly flush the metadata to non-volatile RAM, we must complete a
normal shutdown. This consists of deleting the I/O queues and setting
the shutodown bit. We have to do some extra stuff to make sure we
reset the software state of the queues as well.
On resume, we have to reset the card twice, for reasons described in
the attach funcion. Once we've done that, we can restart the card. If
any of this fails, we'll fail the NVMe card, just like we do when a
reset fails.
Set is_resetting for the duration of the suspend / resume. This keeps
the reset taskqueue from running a concurrent reset, and also is
needed to prevent any hw completions from queueing more I/O to the
card. Pass resetting flag to nvme_ctrlr_start. It doesn't need to get
that from the global state of the ctrlr. Wait for any pending reset to
finish. All queued I/O will get sent to the hardware as part of
nvme_ctrlr_start(), though the upper layers shouldn't send any
down. Disabling the qpairs is the other failsafe to ensure all I/O is
queued.
Rename nvme_ctrlr_destory_qpairs to nvme_ctrlr_delete_qpairs to avoid
confusion with all the other destroy functions. It just removes the
queues in hardware, while the other _destroy_ functions tear down
driver data structures.
Split parts of the hardware reset function up so that I can
do part of the reset in suspsend. Split out the software disabling
of the qpairs into nvme_ctrlr_disable_qpairs.
Finally, fix a couple of spelling errors in comments related to
this.
Relnotes: Yes
MFC After: 1 week
Reviewed by: scottl@ (prior version)
Differential Revision: https://reviews.freebsd.org/D21493
2019-09-03 15:26:11 +00:00
|
|
|
/* Assume user space already converted to little-endian */
|
2018-08-22 04:29:24 +00:00
|
|
|
req->cmd.opc = pt->cmd.opc;
|
|
|
|
req->cmd.fuse = pt->cmd.fuse;
|
2018-12-07 21:58:08 +00:00
|
|
|
req->cmd.rsvd2 = pt->cmd.rsvd2;
|
|
|
|
req->cmd.rsvd3 = pt->cmd.rsvd3;
|
2018-02-22 13:32:31 +00:00
|
|
|
req->cmd.cdw10 = pt->cmd.cdw10;
|
|
|
|
req->cmd.cdw11 = pt->cmd.cdw11;
|
|
|
|
req->cmd.cdw12 = pt->cmd.cdw12;
|
|
|
|
req->cmd.cdw13 = pt->cmd.cdw13;
|
|
|
|
req->cmd.cdw14 = pt->cmd.cdw14;
|
|
|
|
req->cmd.cdw15 = pt->cmd.cdw15;
|
2013-04-12 17:52:17 +00:00
|
|
|
|
2018-02-22 13:32:31 +00:00
|
|
|
req->cmd.nsid = htole32(nsid);
|
2013-04-12 17:52:17 +00:00
|
|
|
|
2018-05-02 20:13:03 +00:00
|
|
|
mtx = mtx_pool_find(mtxpool_sleep, pt);
|
2013-04-12 17:52:17 +00:00
|
|
|
pt->driver_lock = mtx;
|
|
|
|
|
|
|
|
if (is_admin_cmd)
|
|
|
|
nvme_ctrlr_submit_admin_request(ctrlr, req);
|
|
|
|
else
|
|
|
|
nvme_ctrlr_submit_io_request(ctrlr, req);
|
|
|
|
|
2018-05-02 20:13:03 +00:00
|
|
|
mtx_lock(mtx);
|
|
|
|
while (pt->driver_lock != NULL)
|
|
|
|
mtx_sleep(pt, mtx, PRIBIO, "nvme_pt", 0);
|
2013-04-12 17:52:17 +00:00
|
|
|
mtx_unlock(mtx);
|
|
|
|
|
|
|
|
err:
|
|
|
|
if (buf != NULL) {
|
Allocate pager bufs from UMA instead of 80-ish mutex protected linked list.
o In vm_pager_bufferinit() create pbuf_zone and start accounting on how many
pbufs are we going to have set.
In various subsystems that are going to utilize pbufs create private zones
via call to pbuf_zsecond_create(). The latter calls uma_zsecond_create(),
and sets a limit on created zone. After startup preallocate pbufs according
to requirements of all pbuf zones.
Subsystems that used to have a private limit with old allocator now have
private pbuf zones: md(4), fusefs, NFS client, smbfs, VFS cluster, FFS,
swap, vnode pager.
The following subsystems use shared pbuf zone: cam(4), nvme(4), physio(9),
aio(4). They should have their private limits, but changing that is out of
scope of this commit.
o Fetch tunable value of kern.nswbuf from init_param2() and while here move
NSWBUF_MIN to opt_param.h and eliminate opt_swap.h, that was holding only
this option.
Default values aren't touched by this commit, but they probably should be
reviewed wrt to modern hardware.
This change removes a tight bottleneck from sendfile(2) operation, that
uses pbufs in vnode pager. Other pagers also would benefit from faster
allocation.
Together with: gallatin
Tested by: pho
2019-01-15 01:02:16 +00:00
|
|
|
uma_zfree(pbuf_zone, buf);
|
2013-04-12 17:52:17 +00:00
|
|
|
PRELE(curproc);
|
|
|
|
}
|
|
|
|
|
|
|
|
return (ret);
|
|
|
|
}
|
|
|
|
|
2012-09-17 19:23:01 +00:00
|
|
|
static int
|
|
|
|
nvme_ctrlr_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int flag,
|
|
|
|
struct thread *td)
|
|
|
|
{
|
2013-03-26 22:09:51 +00:00
|
|
|
struct nvme_controller *ctrlr;
|
2013-04-12 17:52:17 +00:00
|
|
|
struct nvme_pt_command *pt;
|
2012-09-17 19:23:01 +00:00
|
|
|
|
|
|
|
ctrlr = cdev->si_drv1;
|
|
|
|
|
|
|
|
switch (cmd) {
|
2013-03-26 19:50:46 +00:00
|
|
|
case NVME_RESET_CONTROLLER:
|
|
|
|
nvme_ctrlr_reset(ctrlr);
|
|
|
|
break;
|
2013-04-12 17:52:17 +00:00
|
|
|
case NVME_PASSTHROUGH_CMD:
|
|
|
|
pt = (struct nvme_pt_command *)arg;
|
2018-02-22 13:32:31 +00:00
|
|
|
return (nvme_ctrlr_passthrough_cmd(ctrlr, pt, le32toh(pt->cmd.nsid),
|
2013-04-12 17:52:17 +00:00
|
|
|
1 /* is_user_buffer */, 1 /* is_admin_cmd */));
|
2019-08-01 21:44:07 +00:00
|
|
|
case NVME_GET_NSID:
|
|
|
|
{
|
|
|
|
struct nvme_get_nsid *gnsid = (struct nvme_get_nsid *)arg;
|
|
|
|
strncpy(gnsid->cdev, device_get_nameunit(ctrlr->dev),
|
|
|
|
sizeof(gnsid->cdev));
|
2020-05-02 20:47:58 +00:00
|
|
|
gnsid->cdev[sizeof(gnsid->cdev) - 1] = '\0';
|
2019-08-01 21:44:07 +00:00
|
|
|
gnsid->nsid = 0;
|
|
|
|
break;
|
|
|
|
}
|
2020-09-21 15:41:47 +00:00
|
|
|
case NVME_GET_MAX_XFER_SIZE:
|
|
|
|
*(uint64_t *)arg = ctrlr->max_xfer_size;
|
|
|
|
break;
|
2012-09-17 19:23:01 +00:00
|
|
|
default:
|
|
|
|
return (ENOTTY);
|
|
|
|
}
|
|
|
|
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct cdevsw nvme_ctrlr_cdevsw = {
|
|
|
|
.d_version = D_VERSION,
|
|
|
|
.d_flags = 0,
|
|
|
|
.d_ioctl = nvme_ctrlr_ioctl
|
|
|
|
};
|
|
|
|
|
2016-01-07 16:12:42 +00:00
|
|
|
int
|
|
|
|
nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev)
|
|
|
|
{
|
2018-04-30 23:05:57 +00:00
|
|
|
struct make_dev_args md_args;
|
2018-02-22 13:32:31 +00:00
|
|
|
uint32_t cap_lo;
|
|
|
|
uint32_t cap_hi;
|
2020-11-14 01:45:34 +00:00
|
|
|
uint32_t to, vs, pmrcap;
|
2018-02-22 13:32:31 +00:00
|
|
|
uint8_t mpsmin;
|
|
|
|
int status, timeout_period;
|
2014-03-18 18:10:35 +00:00
|
|
|
|
2016-01-07 16:12:42 +00:00
|
|
|
ctrlr->dev = dev;
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2016-01-07 16:12:42 +00:00
|
|
|
mtx_init(&ctrlr->lock, "nvme ctrlr lock", NULL, MTX_DEF);
|
2019-09-23 17:53:47 +00:00
|
|
|
if (bus_get_domain(dev, &ctrlr->domain) != 0)
|
|
|
|
ctrlr->domain = 0;
|
2016-01-07 16:12:42 +00:00
|
|
|
|
2020-10-28 15:43:29 +00:00
|
|
|
cap_lo = nvme_mmio_read_4(ctrlr, cap_lo);
|
|
|
|
if (bootverbose) {
|
|
|
|
device_printf(dev, "CapLo: 0x%08x: MQES %u%s%s%s%s, TO %u\n",
|
|
|
|
cap_lo, NVME_CAP_LO_MQES(cap_lo),
|
|
|
|
NVME_CAP_LO_CQR(cap_lo) ? ", CQR" : "",
|
|
|
|
NVME_CAP_LO_AMS(cap_lo) ? ", AMS" : "",
|
|
|
|
(NVME_CAP_LO_AMS(cap_lo) & 0x1) ? " WRRwUPC" : "",
|
|
|
|
(NVME_CAP_LO_AMS(cap_lo) & 0x2) ? " VS" : "",
|
|
|
|
NVME_CAP_LO_TO(cap_lo));
|
|
|
|
}
|
2018-02-22 13:32:31 +00:00
|
|
|
cap_hi = nvme_mmio_read_4(ctrlr, cap_hi);
|
2020-10-28 15:43:29 +00:00
|
|
|
if (bootverbose) {
|
|
|
|
device_printf(dev, "CapHi: 0x%08x: DSTRD %u%s, CSS %x%s, "
|
2020-11-14 01:45:34 +00:00
|
|
|
"MPSMIN %u, MPSMAX %u%s%s\n", cap_hi,
|
2020-10-28 15:43:29 +00:00
|
|
|
NVME_CAP_HI_DSTRD(cap_hi),
|
2020-11-14 01:45:34 +00:00
|
|
|
NVME_CAP_HI_NSSRS(cap_hi) ? ", NSSRS" : "",
|
2020-10-28 15:43:29 +00:00
|
|
|
NVME_CAP_HI_CSS(cap_hi),
|
2020-11-14 01:45:34 +00:00
|
|
|
NVME_CAP_HI_BPS(cap_hi) ? ", BPS" : "",
|
2020-10-28 15:43:29 +00:00
|
|
|
NVME_CAP_HI_MPSMIN(cap_hi),
|
|
|
|
NVME_CAP_HI_MPSMAX(cap_hi),
|
2020-11-14 01:45:34 +00:00
|
|
|
NVME_CAP_HI_PMRS(cap_hi) ? ", PMRS" : "",
|
|
|
|
NVME_CAP_HI_CMBS(cap_hi) ? ", CMBS" : "");
|
2020-10-28 15:43:29 +00:00
|
|
|
}
|
|
|
|
if (bootverbose) {
|
|
|
|
vs = nvme_mmio_read_4(ctrlr, vs);
|
|
|
|
device_printf(dev, "Version: 0x%08x: %d.%d\n", vs,
|
|
|
|
NVME_MAJOR(vs), NVME_MINOR(vs));
|
|
|
|
}
|
2020-11-14 01:45:34 +00:00
|
|
|
if (bootverbose && NVME_CAP_HI_PMRS(cap_hi)) {
|
|
|
|
pmrcap = nvme_mmio_read_4(ctrlr, pmrcap);
|
|
|
|
device_printf(dev, "PMRCap: 0x%08x: BIR %u%s%s, PMRTU %u, "
|
|
|
|
"PMRWBM %x, PMRTO %u%s\n", pmrcap,
|
|
|
|
NVME_PMRCAP_BIR(pmrcap),
|
|
|
|
NVME_PMRCAP_RDS(pmrcap) ? ", RDS" : "",
|
|
|
|
NVME_PMRCAP_WDS(pmrcap) ? ", WDS" : "",
|
|
|
|
NVME_PMRCAP_PMRTU(pmrcap),
|
|
|
|
NVME_PMRCAP_PMRWBM(pmrcap),
|
|
|
|
NVME_PMRCAP_PMRTO(pmrcap),
|
|
|
|
NVME_PMRCAP_CMSS(pmrcap) ? ", CMSS" : "");
|
|
|
|
}
|
2020-10-28 15:43:29 +00:00
|
|
|
|
2019-09-04 20:08:36 +00:00
|
|
|
ctrlr->dstrd = NVME_CAP_HI_DSTRD(cap_hi) + 2;
|
2016-01-07 16:12:42 +00:00
|
|
|
|
2019-07-18 15:41:10 +00:00
|
|
|
mpsmin = NVME_CAP_HI_MPSMIN(cap_hi);
|
2018-02-22 13:32:31 +00:00
|
|
|
ctrlr->min_page_size = 1 << (12 + mpsmin);
|
2016-01-07 16:12:42 +00:00
|
|
|
|
|
|
|
/* Get ready timeout value from controller, in units of 500ms. */
|
2019-07-18 15:41:10 +00:00
|
|
|
to = NVME_CAP_LO_TO(cap_lo) + 1;
|
2018-02-22 13:32:31 +00:00
|
|
|
ctrlr->ready_timeout_in_ms = to * 500;
|
2016-01-07 16:12:42 +00:00
|
|
|
|
|
|
|
timeout_period = NVME_DEFAULT_TIMEOUT_PERIOD;
|
|
|
|
TUNABLE_INT_FETCH("hw.nvme.timeout_period", &timeout_period);
|
|
|
|
timeout_period = min(timeout_period, NVME_MAX_TIMEOUT_PERIOD);
|
|
|
|
timeout_period = max(timeout_period, NVME_MIN_TIMEOUT_PERIOD);
|
|
|
|
ctrlr->timeout_period = timeout_period;
|
|
|
|
|
|
|
|
nvme_retry_count = NVME_DEFAULT_RETRY_COUNT;
|
|
|
|
TUNABLE_INT_FETCH("hw.nvme.retry_count", &nvme_retry_count);
|
|
|
|
|
|
|
|
ctrlr->enable_aborts = 0;
|
|
|
|
TUNABLE_INT_FETCH("hw.nvme.enable_aborts", &ctrlr->enable_aborts);
|
|
|
|
|
2013-06-26 23:27:17 +00:00
|
|
|
ctrlr->max_xfer_size = NVME_MAX_XFER_SIZE;
|
2016-11-08 00:24:49 +00:00
|
|
|
if (nvme_ctrlr_construct_admin_qpair(ctrlr) != 0)
|
|
|
|
return (ENXIO);
|
2012-09-17 19:23:01 +00:00
|
|
|
|
2013-03-26 19:58:17 +00:00
|
|
|
ctrlr->taskqueue = taskqueue_create("nvme_taskq", M_WAITOK,
|
|
|
|
taskqueue_thread_enqueue, &ctrlr->taskqueue);
|
|
|
|
taskqueue_start_threads(&ctrlr->taskqueue, 1, PI_DISK, "nvme taskq");
|
|
|
|
|
2013-03-26 20:56:58 +00:00
|
|
|
ctrlr->is_resetting = 0;
|
2014-03-18 18:09:08 +00:00
|
|
|
ctrlr->is_initialized = 0;
|
|
|
|
ctrlr->notification_sent = 0;
|
2013-03-26 21:58:38 +00:00
|
|
|
TASK_INIT(&ctrlr->reset_task, 0, nvme_ctrlr_reset_task, ctrlr);
|
|
|
|
TASK_INIT(&ctrlr->fail_req_task, 0, nvme_ctrlr_fail_req_task, ctrlr);
|
|
|
|
STAILQ_INIT(&ctrlr->fail_req);
|
2019-12-13 18:35:48 +00:00
|
|
|
ctrlr->is_failed = false;
|
2013-03-26 20:56:58 +00:00
|
|
|
|
2018-04-30 23:05:57 +00:00
|
|
|
make_dev_args_init(&md_args);
|
|
|
|
md_args.mda_devsw = &nvme_ctrlr_cdevsw;
|
|
|
|
md_args.mda_uid = UID_ROOT;
|
|
|
|
md_args.mda_gid = GID_WHEEL;
|
|
|
|
md_args.mda_mode = 0600;
|
|
|
|
md_args.mda_unit = device_get_unit(dev);
|
|
|
|
md_args.mda_si_drv1 = (void *)ctrlr;
|
|
|
|
status = make_dev_s(&md_args, &ctrlr->cdev, "nvme%d",
|
|
|
|
device_get_unit(dev));
|
|
|
|
if (status != 0)
|
|
|
|
return (ENXIO);
|
|
|
|
|
2012-09-17 19:23:01 +00:00
|
|
|
return (0);
|
|
|
|
}
|
2012-10-18 00:39:29 +00:00
|
|
|
|
2013-03-26 18:34:19 +00:00
|
|
|
void
|
|
|
|
nvme_ctrlr_destruct(struct nvme_controller *ctrlr, device_t dev)
|
|
|
|
{
|
2019-08-21 20:17:30 +00:00
|
|
|
int gone, i;
|
2013-03-26 18:34:19 +00:00
|
|
|
|
2018-04-30 23:05:57 +00:00
|
|
|
if (ctrlr->resource == NULL)
|
|
|
|
goto nores;
|
2013-03-26 19:58:17 +00:00
|
|
|
|
2019-08-21 20:17:30 +00:00
|
|
|
/*
|
|
|
|
* Check whether it is a hot unplug or a clean driver detach.
|
|
|
|
* If device is not there any more, skip any shutdown commands.
|
|
|
|
*/
|
2021-02-08 20:08:48 +00:00
|
|
|
gone = (nvme_mmio_read_4(ctrlr, csts) == NVME_GONE);
|
2019-08-21 20:17:30 +00:00
|
|
|
if (gone)
|
|
|
|
nvme_ctrlr_fail(ctrlr);
|
|
|
|
else
|
|
|
|
nvme_notify_fail_consumers(ctrlr);
|
2018-05-25 03:34:33 +00:00
|
|
|
|
2013-03-26 19:50:46 +00:00
|
|
|
for (i = 0; i < NVME_MAX_NAMESPACES; i++)
|
|
|
|
nvme_ns_destruct(&ctrlr->ns[i]);
|
2013-03-26 18:34:19 +00:00
|
|
|
|
|
|
|
if (ctrlr->cdev)
|
|
|
|
destroy_dev(ctrlr->cdev);
|
|
|
|
|
2019-08-22 21:56:11 +00:00
|
|
|
if (ctrlr->is_initialized) {
|
2020-01-07 21:17:11 +00:00
|
|
|
if (!gone) {
|
|
|
|
if (ctrlr->hmb_nchunks > 0)
|
|
|
|
nvme_ctrlr_hmb_enable(ctrlr, false, false);
|
Implement nvme suspend / resume for pci attachment
When we suspend, we need to properly shutdown the NVME controller. The
controller may go into D3 state (or may have the power removed), and
to properly flush the metadata to non-volatile RAM, we must complete a
normal shutdown. This consists of deleting the I/O queues and setting
the shutodown bit. We have to do some extra stuff to make sure we
reset the software state of the queues as well.
On resume, we have to reset the card twice, for reasons described in
the attach funcion. Once we've done that, we can restart the card. If
any of this fails, we'll fail the NVMe card, just like we do when a
reset fails.
Set is_resetting for the duration of the suspend / resume. This keeps
the reset taskqueue from running a concurrent reset, and also is
needed to prevent any hw completions from queueing more I/O to the
card. Pass resetting flag to nvme_ctrlr_start. It doesn't need to get
that from the global state of the ctrlr. Wait for any pending reset to
finish. All queued I/O will get sent to the hardware as part of
nvme_ctrlr_start(), though the upper layers shouldn't send any
down. Disabling the qpairs is the other failsafe to ensure all I/O is
queued.
Rename nvme_ctrlr_destory_qpairs to nvme_ctrlr_delete_qpairs to avoid
confusion with all the other destroy functions. It just removes the
queues in hardware, while the other _destroy_ functions tear down
driver data structures.
Split parts of the hardware reset function up so that I can
do part of the reset in suspsend. Split out the software disabling
of the qpairs into nvme_ctrlr_disable_qpairs.
Finally, fix a couple of spelling errors in comments related to
this.
Relnotes: Yes
MFC After: 1 week
Reviewed by: scottl@ (prior version)
Differential Revision: https://reviews.freebsd.org/D21493
2019-09-03 15:26:11 +00:00
|
|
|
nvme_ctrlr_delete_qpairs(ctrlr);
|
2020-01-07 21:17:11 +00:00
|
|
|
}
|
2020-06-25 20:29:29 +00:00
|
|
|
nvme_ctrlr_hmb_free(ctrlr);
|
|
|
|
}
|
|
|
|
if (ctrlr->ioq != NULL) {
|
2019-08-22 21:56:11 +00:00
|
|
|
for (i = 0; i < ctrlr->num_io_queues; i++)
|
|
|
|
nvme_io_qpair_destroy(&ctrlr->ioq[i]);
|
|
|
|
free(ctrlr->ioq, M_NVME);
|
|
|
|
}
|
2020-06-17 17:51:40 +00:00
|
|
|
nvme_admin_qpair_destroy(&ctrlr->adminq);
|
2013-03-26 18:34:19 +00:00
|
|
|
|
2018-04-30 23:05:57 +00:00
|
|
|
/*
|
|
|
|
* Notify the controller of a shutdown, even though this is due to
|
|
|
|
* a driver unload, not a system shutdown (this path is not invoked
|
|
|
|
* during shutdown). This ensures the controller receives a
|
|
|
|
* shutdown notification in case the system is shutdown before
|
|
|
|
* reloading the driver.
|
|
|
|
*/
|
2019-08-21 20:17:30 +00:00
|
|
|
if (!gone)
|
|
|
|
nvme_ctrlr_shutdown(ctrlr);
|
2013-03-26 18:34:19 +00:00
|
|
|
|
2019-08-21 20:17:30 +00:00
|
|
|
if (!gone)
|
|
|
|
nvme_ctrlr_disable(ctrlr);
|
2018-04-30 23:05:57 +00:00
|
|
|
|
|
|
|
if (ctrlr->taskqueue)
|
|
|
|
taskqueue_free(ctrlr->taskqueue);
|
2013-03-26 18:34:19 +00:00
|
|
|
|
|
|
|
if (ctrlr->tag)
|
|
|
|
bus_teardown_intr(ctrlr->dev, ctrlr->res, ctrlr->tag);
|
|
|
|
|
|
|
|
if (ctrlr->res)
|
|
|
|
bus_release_resource(ctrlr->dev, SYS_RES_IRQ,
|
|
|
|
rman_get_rid(ctrlr->res), ctrlr->res);
|
|
|
|
|
2018-04-30 23:05:57 +00:00
|
|
|
if (ctrlr->bar4_resource != NULL) {
|
|
|
|
bus_release_resource(dev, SYS_RES_MEMORY,
|
|
|
|
ctrlr->bar4_resource_id, ctrlr->bar4_resource);
|
|
|
|
}
|
|
|
|
|
|
|
|
bus_release_resource(dev, SYS_RES_MEMORY,
|
|
|
|
ctrlr->resource_id, ctrlr->resource);
|
|
|
|
|
|
|
|
nores:
|
|
|
|
mtx_destroy(&ctrlr->lock);
|
2013-03-26 18:34:19 +00:00
|
|
|
}
|
|
|
|
|
2013-08-13 21:47:08 +00:00
|
|
|
void
|
|
|
|
nvme_ctrlr_shutdown(struct nvme_controller *ctrlr)
|
|
|
|
{
|
2018-02-22 13:32:31 +00:00
|
|
|
uint32_t cc;
|
|
|
|
uint32_t csts;
|
2021-03-17 14:30:40 +00:00
|
|
|
int timeout;
|
2018-02-22 13:32:31 +00:00
|
|
|
|
|
|
|
cc = nvme_mmio_read_4(ctrlr, cc);
|
|
|
|
cc &= ~(NVME_CC_REG_SHN_MASK << NVME_CC_REG_SHN_SHIFT);
|
|
|
|
cc |= NVME_SHN_NORMAL << NVME_CC_REG_SHN_SHIFT;
|
|
|
|
nvme_mmio_write_4(ctrlr, cc, cc);
|
|
|
|
|
2021-03-17 14:30:40 +00:00
|
|
|
timeout = ticks + (ctrlr->cdata.rtd3e == 0 ? 5 * hz :
|
|
|
|
((uint64_t)ctrlr->cdata.rtd3e * hz + 999999) / 1000000);
|
2019-08-21 20:17:30 +00:00
|
|
|
while (1) {
|
2018-02-22 13:32:31 +00:00
|
|
|
csts = nvme_mmio_read_4(ctrlr, csts);
|
2021-02-08 20:08:48 +00:00
|
|
|
if (csts == NVME_GONE) /* Hot unplug. */
|
2019-08-21 20:17:30 +00:00
|
|
|
break;
|
|
|
|
if (NVME_CSTS_GET_SHST(csts) == NVME_SHST_COMPLETE)
|
|
|
|
break;
|
2021-03-17 14:30:40 +00:00
|
|
|
if (timeout - ticks < 0) {
|
|
|
|
nvme_printf(ctrlr, "shutdown timeout\n");
|
2019-08-21 20:17:30 +00:00
|
|
|
break;
|
|
|
|
}
|
2021-03-17 14:30:40 +00:00
|
|
|
pause("nvmeshut", 1);
|
2013-08-13 21:47:08 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-10-18 00:39:29 +00:00
|
|
|
void
|
|
|
|
nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr,
|
|
|
|
struct nvme_request *req)
|
|
|
|
{
|
2012-10-18 00:41:31 +00:00
|
|
|
|
2012-10-18 00:43:25 +00:00
|
|
|
nvme_qpair_submit_request(&ctrlr->adminq, req);
|
2012-10-18 00:39:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr,
|
|
|
|
struct nvme_request *req)
|
|
|
|
{
|
|
|
|
struct nvme_qpair *qpair;
|
|
|
|
|
2019-09-23 17:53:47 +00:00
|
|
|
qpair = &ctrlr->ioq[QP(ctrlr, curcpu)];
|
2012-10-18 00:43:25 +00:00
|
|
|
nvme_qpair_submit_request(qpair, req);
|
2012-10-18 00:39:29 +00:00
|
|
|
}
|
2013-03-26 18:39:54 +00:00
|
|
|
|
|
|
|
device_t
|
|
|
|
nvme_ctrlr_get_device(struct nvme_controller *ctrlr)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (ctrlr->dev);
|
|
|
|
}
|
2013-03-26 19:52:57 +00:00
|
|
|
|
|
|
|
const struct nvme_controller_data *
|
|
|
|
nvme_ctrlr_get_data(struct nvme_controller *ctrlr)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (&ctrlr->cdata);
|
|
|
|
}
|
Implement nvme suspend / resume for pci attachment
When we suspend, we need to properly shutdown the NVME controller. The
controller may go into D3 state (or may have the power removed), and
to properly flush the metadata to non-volatile RAM, we must complete a
normal shutdown. This consists of deleting the I/O queues and setting
the shutodown bit. We have to do some extra stuff to make sure we
reset the software state of the queues as well.
On resume, we have to reset the card twice, for reasons described in
the attach funcion. Once we've done that, we can restart the card. If
any of this fails, we'll fail the NVMe card, just like we do when a
reset fails.
Set is_resetting for the duration of the suspend / resume. This keeps
the reset taskqueue from running a concurrent reset, and also is
needed to prevent any hw completions from queueing more I/O to the
card. Pass resetting flag to nvme_ctrlr_start. It doesn't need to get
that from the global state of the ctrlr. Wait for any pending reset to
finish. All queued I/O will get sent to the hardware as part of
nvme_ctrlr_start(), though the upper layers shouldn't send any
down. Disabling the qpairs is the other failsafe to ensure all I/O is
queued.
Rename nvme_ctrlr_destory_qpairs to nvme_ctrlr_delete_qpairs to avoid
confusion with all the other destroy functions. It just removes the
queues in hardware, while the other _destroy_ functions tear down
driver data structures.
Split parts of the hardware reset function up so that I can
do part of the reset in suspsend. Split out the software disabling
of the qpairs into nvme_ctrlr_disable_qpairs.
Finally, fix a couple of spelling errors in comments related to
this.
Relnotes: Yes
MFC After: 1 week
Reviewed by: scottl@ (prior version)
Differential Revision: https://reviews.freebsd.org/D21493
2019-09-03 15:26:11 +00:00
|
|
|
|
|
|
|
int
|
|
|
|
nvme_ctrlr_suspend(struct nvme_controller *ctrlr)
|
|
|
|
{
|
|
|
|
int to = hz;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Can't touch failed controllers, so it's already suspended.
|
|
|
|
*/
|
|
|
|
if (ctrlr->is_failed)
|
|
|
|
return (0);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We don't want the reset taskqueue running, since it does similar
|
|
|
|
* things, so prevent it from running after we start. Wait for any reset
|
|
|
|
* that may have been started to complete. The reset process we follow
|
|
|
|
* will ensure that any new I/O will queue and be given to the hardware
|
|
|
|
* after we resume (though there should be none).
|
|
|
|
*/
|
|
|
|
while (atomic_cmpset_32(&ctrlr->is_resetting, 0, 1) == 0 && to-- > 0)
|
|
|
|
pause("nvmesusp", 1);
|
|
|
|
if (to <= 0) {
|
|
|
|
nvme_printf(ctrlr,
|
|
|
|
"Competing reset task didn't finish. Try again later.\n");
|
|
|
|
return (EWOULDBLOCK);
|
|
|
|
}
|
|
|
|
|
2020-01-07 21:17:11 +00:00
|
|
|
if (ctrlr->hmb_nchunks > 0)
|
|
|
|
nvme_ctrlr_hmb_enable(ctrlr, false, false);
|
|
|
|
|
Implement nvme suspend / resume for pci attachment
When we suspend, we need to properly shutdown the NVME controller. The
controller may go into D3 state (or may have the power removed), and
to properly flush the metadata to non-volatile RAM, we must complete a
normal shutdown. This consists of deleting the I/O queues and setting
the shutodown bit. We have to do some extra stuff to make sure we
reset the software state of the queues as well.
On resume, we have to reset the card twice, for reasons described in
the attach funcion. Once we've done that, we can restart the card. If
any of this fails, we'll fail the NVMe card, just like we do when a
reset fails.
Set is_resetting for the duration of the suspend / resume. This keeps
the reset taskqueue from running a concurrent reset, and also is
needed to prevent any hw completions from queueing more I/O to the
card. Pass resetting flag to nvme_ctrlr_start. It doesn't need to get
that from the global state of the ctrlr. Wait for any pending reset to
finish. All queued I/O will get sent to the hardware as part of
nvme_ctrlr_start(), though the upper layers shouldn't send any
down. Disabling the qpairs is the other failsafe to ensure all I/O is
queued.
Rename nvme_ctrlr_destory_qpairs to nvme_ctrlr_delete_qpairs to avoid
confusion with all the other destroy functions. It just removes the
queues in hardware, while the other _destroy_ functions tear down
driver data structures.
Split parts of the hardware reset function up so that I can
do part of the reset in suspsend. Split out the software disabling
of the qpairs into nvme_ctrlr_disable_qpairs.
Finally, fix a couple of spelling errors in comments related to
this.
Relnotes: Yes
MFC After: 1 week
Reviewed by: scottl@ (prior version)
Differential Revision: https://reviews.freebsd.org/D21493
2019-09-03 15:26:11 +00:00
|
|
|
/*
|
|
|
|
* Per Section 7.6.2 of NVMe spec 1.4, to properly suspend, we need to
|
|
|
|
* delete the hardware I/O queues, and then shutdown. This properly
|
|
|
|
* flushes any metadata the drive may have stored so it can survive
|
|
|
|
* having its power removed and prevents the unsafe shutdown count from
|
|
|
|
* incriminating. Once we delete the qpairs, we have to disable them
|
|
|
|
* before shutting down. The delay is out of paranoia in
|
|
|
|
* nvme_ctrlr_hw_reset, and is repeated here (though we should have no
|
|
|
|
* pending I/O that the delay copes with).
|
|
|
|
*/
|
|
|
|
nvme_ctrlr_delete_qpairs(ctrlr);
|
|
|
|
nvme_ctrlr_disable_qpairs(ctrlr);
|
2021-03-17 14:30:40 +00:00
|
|
|
pause("nvmesusp", hz / 10);
|
Implement nvme suspend / resume for pci attachment
When we suspend, we need to properly shutdown the NVME controller. The
controller may go into D3 state (or may have the power removed), and
to properly flush the metadata to non-volatile RAM, we must complete a
normal shutdown. This consists of deleting the I/O queues and setting
the shutodown bit. We have to do some extra stuff to make sure we
reset the software state of the queues as well.
On resume, we have to reset the card twice, for reasons described in
the attach funcion. Once we've done that, we can restart the card. If
any of this fails, we'll fail the NVMe card, just like we do when a
reset fails.
Set is_resetting for the duration of the suspend / resume. This keeps
the reset taskqueue from running a concurrent reset, and also is
needed to prevent any hw completions from queueing more I/O to the
card. Pass resetting flag to nvme_ctrlr_start. It doesn't need to get
that from the global state of the ctrlr. Wait for any pending reset to
finish. All queued I/O will get sent to the hardware as part of
nvme_ctrlr_start(), though the upper layers shouldn't send any
down. Disabling the qpairs is the other failsafe to ensure all I/O is
queued.
Rename nvme_ctrlr_destory_qpairs to nvme_ctrlr_delete_qpairs to avoid
confusion with all the other destroy functions. It just removes the
queues in hardware, while the other _destroy_ functions tear down
driver data structures.
Split parts of the hardware reset function up so that I can
do part of the reset in suspsend. Split out the software disabling
of the qpairs into nvme_ctrlr_disable_qpairs.
Finally, fix a couple of spelling errors in comments related to
this.
Relnotes: Yes
MFC After: 1 week
Reviewed by: scottl@ (prior version)
Differential Revision: https://reviews.freebsd.org/D21493
2019-09-03 15:26:11 +00:00
|
|
|
nvme_ctrlr_shutdown(ctrlr);
|
|
|
|
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
nvme_ctrlr_resume(struct nvme_controller *ctrlr)
|
|
|
|
{
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Can't touch failed controllers, so nothing to do to resume.
|
|
|
|
*/
|
|
|
|
if (ctrlr->is_failed)
|
|
|
|
return (0);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Have to reset the hardware twice, just like we do on attach. See
|
|
|
|
* nmve_attach() for why.
|
|
|
|
*/
|
|
|
|
if (nvme_ctrlr_hw_reset(ctrlr) != 0)
|
|
|
|
goto fail;
|
|
|
|
if (nvme_ctrlr_hw_reset(ctrlr) != 0)
|
|
|
|
goto fail;
|
|
|
|
|
|
|
|
/*
|
2020-05-02 20:47:58 +00:00
|
|
|
* Now that we've reset the hardware, we can restart the controller. Any
|
Implement nvme suspend / resume for pci attachment
When we suspend, we need to properly shutdown the NVME controller. The
controller may go into D3 state (or may have the power removed), and
to properly flush the metadata to non-volatile RAM, we must complete a
normal shutdown. This consists of deleting the I/O queues and setting
the shutodown bit. We have to do some extra stuff to make sure we
reset the software state of the queues as well.
On resume, we have to reset the card twice, for reasons described in
the attach funcion. Once we've done that, we can restart the card. If
any of this fails, we'll fail the NVMe card, just like we do when a
reset fails.
Set is_resetting for the duration of the suspend / resume. This keeps
the reset taskqueue from running a concurrent reset, and also is
needed to prevent any hw completions from queueing more I/O to the
card. Pass resetting flag to nvme_ctrlr_start. It doesn't need to get
that from the global state of the ctrlr. Wait for any pending reset to
finish. All queued I/O will get sent to the hardware as part of
nvme_ctrlr_start(), though the upper layers shouldn't send any
down. Disabling the qpairs is the other failsafe to ensure all I/O is
queued.
Rename nvme_ctrlr_destory_qpairs to nvme_ctrlr_delete_qpairs to avoid
confusion with all the other destroy functions. It just removes the
queues in hardware, while the other _destroy_ functions tear down
driver data structures.
Split parts of the hardware reset function up so that I can
do part of the reset in suspsend. Split out the software disabling
of the qpairs into nvme_ctrlr_disable_qpairs.
Finally, fix a couple of spelling errors in comments related to
this.
Relnotes: Yes
MFC After: 1 week
Reviewed by: scottl@ (prior version)
Differential Revision: https://reviews.freebsd.org/D21493
2019-09-03 15:26:11 +00:00
|
|
|
* I/O that was pending is requeued. Any admin commands are aborted with
|
|
|
|
* an error. Once we've restarted, take the controller out of reset.
|
|
|
|
*/
|
|
|
|
nvme_ctrlr_start(ctrlr, true);
|
2020-05-02 20:47:58 +00:00
|
|
|
(void)atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);
|
Implement nvme suspend / resume for pci attachment
When we suspend, we need to properly shutdown the NVME controller. The
controller may go into D3 state (or may have the power removed), and
to properly flush the metadata to non-volatile RAM, we must complete a
normal shutdown. This consists of deleting the I/O queues and setting
the shutodown bit. We have to do some extra stuff to make sure we
reset the software state of the queues as well.
On resume, we have to reset the card twice, for reasons described in
the attach funcion. Once we've done that, we can restart the card. If
any of this fails, we'll fail the NVMe card, just like we do when a
reset fails.
Set is_resetting for the duration of the suspend / resume. This keeps
the reset taskqueue from running a concurrent reset, and also is
needed to prevent any hw completions from queueing more I/O to the
card. Pass resetting flag to nvme_ctrlr_start. It doesn't need to get
that from the global state of the ctrlr. Wait for any pending reset to
finish. All queued I/O will get sent to the hardware as part of
nvme_ctrlr_start(), though the upper layers shouldn't send any
down. Disabling the qpairs is the other failsafe to ensure all I/O is
queued.
Rename nvme_ctrlr_destory_qpairs to nvme_ctrlr_delete_qpairs to avoid
confusion with all the other destroy functions. It just removes the
queues in hardware, while the other _destroy_ functions tear down
driver data structures.
Split parts of the hardware reset function up so that I can
do part of the reset in suspsend. Split out the software disabling
of the qpairs into nvme_ctrlr_disable_qpairs.
Finally, fix a couple of spelling errors in comments related to
this.
Relnotes: Yes
MFC After: 1 week
Reviewed by: scottl@ (prior version)
Differential Revision: https://reviews.freebsd.org/D21493
2019-09-03 15:26:11 +00:00
|
|
|
|
|
|
|
return (0);
|
|
|
|
fail:
|
|
|
|
/*
|
|
|
|
* Since we can't bring the controller out of reset, announce and fail
|
|
|
|
* the controller. However, we have to return success for the resume
|
|
|
|
* itself, due to questionable APIs.
|
|
|
|
*/
|
|
|
|
nvme_printf(ctrlr, "Failed to reset on resume, failing.\n");
|
|
|
|
nvme_ctrlr_fail(ctrlr);
|
2020-05-02 20:47:58 +00:00
|
|
|
(void)atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);
|
Implement nvme suspend / resume for pci attachment
When we suspend, we need to properly shutdown the NVME controller. The
controller may go into D3 state (or may have the power removed), and
to properly flush the metadata to non-volatile RAM, we must complete a
normal shutdown. This consists of deleting the I/O queues and setting
the shutodown bit. We have to do some extra stuff to make sure we
reset the software state of the queues as well.
On resume, we have to reset the card twice, for reasons described in
the attach funcion. Once we've done that, we can restart the card. If
any of this fails, we'll fail the NVMe card, just like we do when a
reset fails.
Set is_resetting for the duration of the suspend / resume. This keeps
the reset taskqueue from running a concurrent reset, and also is
needed to prevent any hw completions from queueing more I/O to the
card. Pass resetting flag to nvme_ctrlr_start. It doesn't need to get
that from the global state of the ctrlr. Wait for any pending reset to
finish. All queued I/O will get sent to the hardware as part of
nvme_ctrlr_start(), though the upper layers shouldn't send any
down. Disabling the qpairs is the other failsafe to ensure all I/O is
queued.
Rename nvme_ctrlr_destory_qpairs to nvme_ctrlr_delete_qpairs to avoid
confusion with all the other destroy functions. It just removes the
queues in hardware, while the other _destroy_ functions tear down
driver data structures.
Split parts of the hardware reset function up so that I can
do part of the reset in suspsend. Split out the software disabling
of the qpairs into nvme_ctrlr_disable_qpairs.
Finally, fix a couple of spelling errors in comments related to
this.
Relnotes: Yes
MFC After: 1 week
Reviewed by: scottl@ (prior version)
Differential Revision: https://reviews.freebsd.org/D21493
2019-09-03 15:26:11 +00:00
|
|
|
return (0);
|
|
|
|
}
|