From fe1b713e2cba0d02c0562c5cffe029d7461b3888 Mon Sep 17 00:00:00 2001 From: Chuck Tuffli Date: Fri, 5 Apr 2019 16:54:16 +0000 Subject: [PATCH] bhyve: Fix NVMe BAR size calculation The NVMe specification defines bits 13:4 of BAR0 as Reserved (i.e. 0x0). Most drivers do not enforce this, but the Windows NVMe driver does and will refuse to start the device (i.e. error 10) if any of these bits are set. The current BAR size calculation tries to minimize the amount of memory the device reserves by scaling the BAR size by the maximum number of queues supported by the device. But unless the device supports a large number of queue pairs (over 1536), it will reserve too little memory. The fix is to allocate a minimum of 16K bytes for BAR0. Tested on Windows Server 2016 and 2019 Reviewed by: imp (mentor), araujo, jhb, bhyve Approved by: imp (mentor), bhyve (jhb) MFC after: 2 weeks Differential Revision: https://reviews.freebsd.org/D19676 --- usr.sbin/bhyve/pci_nvme.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/usr.sbin/bhyve/pci_nvme.c b/usr.sbin/bhyve/pci_nvme.c index 8fe8d1b25df3..61d37d1f0836 100644 --- a/usr.sbin/bhyve/pci_nvme.c +++ b/usr.sbin/bhyve/pci_nvme.c @@ -85,6 +85,9 @@ static int nvme_debug = 0; #define NVME_IOSLOTS 8 +/* The NVMe spec defines bits 13:4 in BAR0 as reserved */ +#define NVME_MMIO_SPACE_MIN (1 << 14) + #define NVME_QUEUES 16 #define NVME_MAX_QENTRIES 2048 @@ -1847,9 +1850,16 @@ pci_nvme_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts) pci_set_cfgdata8(pi, PCIR_PROGIF, PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0); - /* allocate size of nvme registers + doorbell space for all queues */ + /* + * Allocate size of NVMe registers + doorbell space for all queues. + * + * The specification requires a minimum memory I/O window size of 16K. + * The Windows driver will refuse to start a device with a smaller + * window. + */ pci_membar_sz = sizeof(struct nvme_registers) + - 2*sizeof(uint32_t)*(sc->max_queues + 1); + 2 * sizeof(uint32_t) * (sc->max_queues + 1); + pci_membar_sz = MAX(pci_membar_sz, NVME_MMIO_SPACE_MIN); DPRINTF(("nvme membar size: %u\r\n", pci_membar_sz));