SPDK: Initial check-in

Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>

.gitignore

@@ -0,0 +1,6 @@
+*.a
+*.d
+*.o
+*~
+*.swp
+tags

CONFIG

@@ -0,0 +1,45 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+# Build with debug logging. Turn off for performance testing and normal usage
+CONFIG_DEBUG=y
+
+# This directory should contain 'include' and 'lib' directories for your DPDK
+# installation. Alternatively you can specify this on the command line
+# with 'make DPDK_DIR=/path/to/dpdk'.
+CONFIG_DPDK_DIR=/path/to/dpdk
+
+# Header file to use for NVMe implementation specific functions.
+# Defaults to depending on DPDK.
+CONFIG_NVME_IMPL=nvme_impl.h
+

LICENSE

@@ -0,0 +1,30 @@
+BSD LICENSE
+
+Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+  * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+  * Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+  * Neither the name of Intel Corporation nor the names of its
+    contributors may be used to endorse or promote products derived
+    from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Makefile

@@ -0,0 +1,49 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(CURDIR)
+export SPDK_ROOT_DIR
+
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+DIRS-y += lib test examples
+
+.PHONY: all clean $(DIRS-y)
+
+all: $(DIRS-y)
+clean: $(DIRS-y)
+
+test: lib
+examples: lib
+
+include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk

doc/Makefile

@@ -0,0 +1,15 @@
+DOXYFILES = Doxyfile.nvme
+OUTPUT_DIRS = $(patsubst Doxyfile.%,output.%,$(DOXYFILES))
+
+all: doc
+
+.PHONY: all doc clean
+
+doc: $(OUTPUT_DIRS)
+
+output.%: Doxyfile.%
+	rm -rf $@
+	doxygen $^
+
+clean:
+	rm -rf $(OUTPUT_DIRS)

doc/nvme.index.txt

@@ -0,0 +1,58 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*!
+
+\mainpage SPDK Userspace NVMe Driver
+
+\section interface Public Interface
+
+- nvme.h
+
+\section key_functions Key Functions
+
+- nvme_attach() \copybrief nvme_attach()
+- nvme_ns_cmd_read() \copybrief nvme_ns_cmd_read()
+- nvme_ns_cmd_write() \copybrief nvme_ns_cmd_write()
+- nvme_ns_cmd_deallocate() \copybrief nvme_ns_cmd_deallocate()
+- nvme_ns_cmd_flush() \copybrief nvme_ns_cmd_flush()
+- nvme_ctrlr_process_io_completions() \copybrief nvme_ctrlr_process_io_completions()
+
+\section key_concepts Key Concepts
+
+- \ref nvme_driver_integration
+- \ref nvme_initialization
+- \ref nvme_io_submission
+- \ref nvme_async_completion
+
+*/

examples/Makefile

@@ -0,0 +1,43 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+DIRS-y += nvme
+
+.PHONY: all clean $(DIRS-y)
+
+all: $(DIRS-y)
+clean: $(DIRS-y)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk

examples/nvme/Makefile

@@ -0,0 +1,43 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+DIRS-y += identify perf
+
+.PHONY: all clean $(DIRS-y)
+
+all: $(DIRS-y)
+clean: $(DIRS-y)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk

examples/nvme/identify/.gitignore

@@ -0,0 +1 @@
+identify

examples/nvme/identify/Makefile

@@ -0,0 +1,62 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+APP = identify
+
+C_SRCS := identify.c
+
+CFLAGS += -I. $(DPDK_INC)
+
+SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/nvme/libspdk_nvme.a \
+	     $(SPDK_ROOT_DIR)/lib/util/libspdk_util.a \
+	     $(SPDK_ROOT_DIR)/lib/memory/libspdk_memory.a
+
+LIBS += $(SPDK_LIBS) -lpciaccess -lpthread $(DPDK_LIB) -lrt
+
+OBJS  = $(C_SRCS:.c=.o)
+
+all : $(APP)
+
+objs : $(OBJS)
+
+$(APP) : $(OBJS) $(SPDK_LIBS)
+	@echo "  LINK $@"
+	$(Q)$(CC) $(LDFLAGS) -o $@ $(OBJS) $(LIBS)
+
+clean :
+	$(Q)rm -f $(OBJS) *.d $(APP)
+
+
+

examples/nvme/identify/identify.c

@@ -0,0 +1,497 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdbool.h>
+
+#include <pciaccess.h>
+
+#include <rte_config.h>
+#include <rte_malloc.h>
+#include <rte_mempool.h>
+#include <rte_lcore.h>
+
+#include "spdk/nvme.h"
+#include "spdk/pci.h"
+
+struct rte_mempool *request_mempool;
+
+static int outstanding_commands;
+
+struct feature {
+	uint32_t result;
+	bool valid;
+};
+
+static struct feature features[256];
+
+static struct nvme_health_information_page *health_page;
+
+static void
+get_feature_completion(void *cb_arg, const struct nvme_completion *cpl)
+{
+	struct feature *feature = cb_arg;
+	int fid = feature - features;
+	if (nvme_completion_is_error(cpl)) {
+		printf("get_feature(0x%02X) failed\n", fid);
+	} else {
+		feature->result = cpl->cdw0;
+		feature->valid = true;
+	}
+	outstanding_commands--;
+}
+
+static void
+get_log_page_completion(void *cb_arg, const struct nvme_completion *cpl)
+{
+	if (nvme_completion_is_error(cpl)) {
+		printf("get log page failed\n");
+	}
+	outstanding_commands--;
+}
+
+static int
+get_feature(struct nvme_controller *ctrlr, uint8_t fid)
+{
+	struct nvme_command cmd = {0};
+
+	cmd.opc = NVME_OPC_GET_FEATURES;
+	cmd.cdw10 = fid;
+
+	return nvme_ctrlr_cmd_admin_raw(ctrlr, &cmd, NULL, 0, get_feature_completion, &features[fid]);
+}
+
+static void
+get_features(struct nvme_controller *ctrlr)
+{
+	int i;
+
+	uint8_t features_to_get[] = {
+		NVME_FEAT_ARBITRATION,
+		NVME_FEAT_POWER_MANAGEMENT,
+		NVME_FEAT_TEMPERATURE_THRESHOLD,
+		NVME_FEAT_ERROR_RECOVERY,
+	};
+
+	/* Submit several GET FEATURES commands and wait for them to complete */
+	outstanding_commands = 0;
+	for (i = 0; i < sizeof(features_to_get) / sizeof(*features_to_get); i++) {
+		if (get_feature(ctrlr, features_to_get[i]) == 0) {
+			outstanding_commands++;
+		} else {
+			printf("get_feature(0x%02X) failed to submit command\n", features_to_get[i]);
+		}
+	}
+
+	while (outstanding_commands) {
+		nvme_ctrlr_process_admin_completions(ctrlr);
+	}
+}
+
+static int
+get_health_log_page(struct nvme_controller *ctrlr)
+{
+	struct nvme_command cmd = {0};
+
+	if (health_page == NULL) {
+		health_page = rte_zmalloc("nvme health", sizeof(*health_page), 4096);
+	}
+	if (health_page == NULL) {
+		printf("Allocation error (health page)\n");
+		exit(1);
+	}
+
+	cmd.opc = NVME_OPC_GET_LOG_PAGE;
+	cmd.cdw10 = NVME_LOG_HEALTH_INFORMATION;
+	cmd.cdw10 |= (sizeof(*health_page) / 4) << 16; // number of dwords
+	cmd.nsid = NVME_GLOBAL_NAMESPACE_TAG;
+
+	return nvme_ctrlr_cmd_admin_raw(ctrlr, &cmd, health_page, sizeof(*health_page),
+					get_log_page_completion, NULL);
+}
+
+static void
+get_log_pages(struct nvme_controller *ctrlr)
+{
+	outstanding_commands = 0;
+
+	if (get_health_log_page(ctrlr) == 0) {
+		outstanding_commands++;
+	} else {
+		printf("Get Log Page (SMART/health) failed\n");
+	}
+
+	while (outstanding_commands) {
+		nvme_ctrlr_process_admin_completions(ctrlr);
+	}
+}
+
+static void
+cleanup(void)
+{
+	if (health_page) {
+		rte_free(health_page);
+		health_page = NULL;
+	}
+}
+
+static void
+print_uint128_hex(uint64_t *v)
+{
+	unsigned long long lo = v[0], hi = v[1];
+	if (hi) {
+		printf("0x%llX%016llX", hi, lo);
+	} else {
+		printf("0x%llX", lo);
+	}
+}
+
+static void
+print_uint128_dec(uint64_t *v)
+{
+	unsigned long long lo = v[0], hi = v[1];
+	if (hi) {
+		/* can't handle large (>64-bit) decimal values for now, so fall back to hex */
+		print_uint128_hex(v);
+	} else {
+		printf("%llu", (unsigned long long)lo);
+	}
+}
+
+static void
+print_namespace(struct nvme_namespace *ns)
+{
+	const struct nvme_namespace_data	*nsdata;
+	uint32_t				i;
+	uint32_t				flags;
+
+	nsdata = nvme_ns_get_data(ns);
+	flags  = nvme_ns_get_flags(ns);
+
+	printf("Namespace ID:%d\n", nvme_ns_get_id(ns));
+	printf("Deallocate:                  %s\n",
+	       (flags & NVME_NS_DEALLOCATE_SUPPORTED) ? "Supported" : "Not Supported");
+	printf("Flush:                       %s\n",
+	       (flags & NVME_NS_FLUSH_SUPPORTED) ? "Supported" : "Not Supported");
+	printf("Size (in LBAs):              %lld (%lldM)\n",
+	       (long long)nsdata->nsze,
+	       (long long)nsdata->nsze / 1024 / 1024);
+	printf("Capacity (in LBAs):          %lld (%lldM)\n",
+	       (long long)nsdata->ncap,
+	       (long long)nsdata->ncap / 1024 / 1024);
+	printf("Utilization (in LBAs):       %lld (%lldM)\n",
+	       (long long)nsdata->nuse,
+	       (long long)nsdata->nuse / 1024 / 1024);
+	printf("Thin Provisioning:           %s\n",
+	       nsdata->nsfeat.thin_prov ? "Supported" : "Not Supported");
+	printf("Number of LBA Formats:       %d\n", nsdata->nlbaf + 1);
+	printf("Current LBA Format:          LBA Format #%02d\n",
+	       nsdata->flbas.format);
+	for (i = 0; i <= nsdata->nlbaf; i++)
+		printf("LBA Format #%02d: Data Size: %5d  Metadata Size: %5d\n",
+		       i, 1 << nsdata->lbaf[i].lbads, nsdata->lbaf[i].ms);
+	printf("\n");
+}
+
+static void
+print_controller(struct nvme_controller *ctrlr, struct pci_device *pci_dev)
+{
+	const struct nvme_controller_data	*cdata;
+	uint8_t					str[128];
+	uint32_t				i;
+
+	get_features(ctrlr);
+	get_log_pages(ctrlr);
+
+	cdata = nvme_ctrlr_get_data(ctrlr);
+
+	printf("=====================================================\n");
+	printf("NVMe Controller at PCI bus %d, device %d, function %d\n",
+	       pci_dev->bus, pci_dev->dev, pci_dev->func);
+	printf("=====================================================\n");
+	printf("Controller Capabilities/Features\n");
+	printf("================================\n");
+	printf("Vendor ID:                  %04x\n", cdata->vid);
+	printf("Subsystem Vendor ID:        %04x\n", cdata->ssvid);
+	snprintf(str, sizeof(cdata->sn) + 1, "%s", cdata->sn);
+	printf("Serial Number:              %s\n", str);
+	snprintf(str, sizeof(cdata->mn) + 1, "%s", cdata->mn);
+	printf("Model Number:               %s\n", str);
+	snprintf(str, sizeof(cdata->fr) + 1, "%s", cdata->fr);
+	printf("Firmware Version:           %s\n", str);
+	printf("Recommended Arb Burst:      %d\n", cdata->rab);
+	printf("IEEE OUI Identifier:        %02x %02x %02x\n",
+	       cdata->ieee[0], cdata->ieee[1], cdata->ieee[2]);
+	printf("Multi-Interface Cap:        %02x\n", cdata->mic);
+	/* TODO: Use CAP.MPSMIN to determine true memory page size. */
+	printf("Max Data Transfer Size:     ");
+	if (cdata->mdts == 0)
+		printf("Unlimited\n");
+	else
+		printf("%d\n", 4096 * (1 << cdata->mdts));
+	if (features[NVME_FEAT_ERROR_RECOVERY].valid) {
+		unsigned tler = features[NVME_FEAT_ERROR_RECOVERY].result & 0xFFFF;
+		printf("Error Recovery Timeout:     ");
+		if (tler == 0) {
+			printf("Unlimited\n");
+		} else {
+			printf("%u milliseconds\n", tler * 100);
+		}
+	}
+	printf("\n");
+
+	printf("Admin Command Set Attributes\n");
+	printf("============================\n");
+	printf("Security Send/Receive:       %s\n",
+	       cdata->oacs.security ? "Supported" : "Not Supported");
+	printf("Format NVM:                  %s\n",
+	       cdata->oacs.format ? "Supported" : "Not Supported");
+	printf("Firmware Activate/Download:  %s\n",
+	       cdata->oacs.firmware ? "Supported" : "Not Supported");
+	printf("Abort Command Limit:         %d\n", cdata->acl + 1);
+	printf("Async Event Request Limit:   %d\n", cdata->aerl + 1);
+	printf("Number of Firmware Slots:    ");
+	if (cdata->oacs.firmware != 0)
+		printf("%d\n", cdata->frmw.num_slots);
+	else
+		printf("N/A\n");
+	printf("Firmware Slot 1 Read-Only:   ");
+	if (cdata->oacs.firmware != 0)
+		printf("%s\n", cdata->frmw.slot1_ro ? "Yes" : "No");
+	else
+		printf("N/A\n");
+	printf("Per-Namespace SMART Log:     %s\n",
+	       cdata->lpa.ns_smart ? "Yes" : "No");
+	printf("Error Log Page Entries:      %d\n", cdata->elpe + 1);
+	printf("\n");
+
+	printf("NVM Command Set Attributes\n");
+	printf("==========================\n");
+	printf("Submission Queue Entry Size\n");
+	printf("  Max:                       %d\n", 1 << cdata->sqes.max);
+	printf("  Min:                       %d\n", 1 << cdata->sqes.min);
+	printf("Completion Queue Entry Size\n");
+	printf("  Max:                       %d\n", 1 << cdata->cqes.max);
+	printf("  Min:                       %d\n", 1 << cdata->cqes.min);
+	printf("Number of Namespaces:        %d\n", cdata->nn);
+	printf("Compare Command:             %s\n",
+	       cdata->oncs.compare ? "Supported" : "Not Supported");
+	printf("Write Uncorrectable Command: %s\n",
+	       cdata->oncs.write_unc ? "Supported" : "Not Supported");
+	printf("Dataset Management Command:  %s\n",
+	       cdata->oncs.dsm ? "Supported" : "Not Supported");
+	printf("Volatile Write Cache:        %s\n",
+	       cdata->vwc.present ? "Present" : "Not Present");
+	printf("\n");
+
+	if (features[NVME_FEAT_ARBITRATION].valid) {
+		uint32_t arb = features[NVME_FEAT_ARBITRATION].result;
+		unsigned ab, lpw, mpw, hpw;
+
+		ab = arb & 0x3;
+		lpw = ((arb >> 8) & 0xFF) + 1;
+		mpw = ((arb >> 16) & 0xFF) + 1;
+		hpw = ((arb >> 24) & 0xFF) + 1;
+
+		printf("Arbitration\n");
+		printf("===========\n");
+		printf("Arbitration Burst:           ");
+		if (ab == 7) {
+			printf("no limit\n");
+		} else {
+			printf("%u\n", 1u << ab);
+		}
+		printf("Low Priority Weight:         %u\n", lpw);
+		printf("Medium Priority Weight:      %u\n", mpw);
+		printf("High Priority Weight:        %u\n", hpw);
+		printf("\n");
+	}
+
+	if (features[NVME_FEAT_POWER_MANAGEMENT].valid) {
+		unsigned ps = features[NVME_FEAT_POWER_MANAGEMENT].result & 0x1F;
+		printf("Power Management\n");
+		printf("================\n");
+		printf("Number of Power States:      %u\n", cdata->npss + 1);
+		printf("Current Power State:         Power State #%u\n", ps);
+		for (i = 0; i <= cdata->npss; i++) {
+			const struct nvme_power_state *psd = &cdata->psd[i];
+			printf("Power State #%u:  ", i);
+			if (psd->mps) {
+				/* MP scale is 0.0001 W */
+				printf("Max Power: %u.%04u W\n",
+				       psd->mp / 10000,
+				       psd->mp % 10000);
+			} else {
+				/* MP scale is 0.01 W */
+				printf("Max Power: %3u.%02u W\n",
+				       psd->mp / 100,
+				       psd->mp % 100);
+			}
+			/* TODO: print other power state descriptor fields */
+		}
+		printf("\n");
+	}
+
+	if (features[NVME_FEAT_TEMPERATURE_THRESHOLD].valid && health_page) {
+		printf("Health Information\n");
+		printf("==================\n");
+		printf("Critical Warnings:\n");
+		printf("  Available Spare Space:     %s\n",
+		       health_page->critical_warning.bits.available_spare ? "WARNING" : "OK");
+		printf("  Temperature:               %s\n",
+		       health_page->critical_warning.bits.temperature ? "WARNING" : "OK");
+		printf("  Device Reliability:        %s\n",
+		       health_page->critical_warning.bits.device_reliability ? "WARNING" : "OK");
+		printf("  Read Only:                 %s\n",
+		       health_page->critical_warning.bits.read_only ? "Yes" : "No");
+		printf("  Volatile Memory Backup:    %s\n",
+		       health_page->critical_warning.bits.volatile_memory_backup ? "WARNING" : "OK");
+		printf("Current Temperature:         %u Kelvin (%u Celsius)\n",
+		       health_page->temperature,
+		       health_page->temperature - 273);
+		printf("Temperature Threshold:       %u Kelvin (%u Celsius)\n",
+		       features[NVME_FEAT_TEMPERATURE_THRESHOLD].result,
+		       features[NVME_FEAT_TEMPERATURE_THRESHOLD].result - 273);
+		printf("Available Spare:             %u%%\n", health_page->available_spare);
+		printf("Life Percentage Used:        %u%%\n", health_page->percentage_used);
+		printf("Data Units Read:             ");
+		print_uint128_dec(health_page->data_units_read);
+		printf("\n");
+		printf("Data Units Written:          ");
+		print_uint128_dec(health_page->data_units_written);
+		printf("\n");
+		printf("Host Read Commands:          ");
+		print_uint128_dec(health_page->host_read_commands);
+		printf("\n");
+		printf("Host Write Commands:         ");
+		print_uint128_dec(health_page->host_write_commands);
+		printf("\n");
+		printf("Controller Busy Time:        ");
+		print_uint128_dec(health_page->controller_busy_time);
+		printf(" minutes\n");
+		printf("Power Cycles:                ");
+		print_uint128_dec(health_page->power_cycles);
+		printf("\n");
+		printf("Power On Hours:              ");
+		print_uint128_dec(health_page->power_on_hours);
+		printf(" hours\n");
+		printf("Unsafe Shutdowns:            ");
+		print_uint128_dec(health_page->unsafe_shutdowns);
+		printf("\n");
+		printf("Unrecoverable Media Errors:  ");
+		print_uint128_dec(health_page->media_errors);
+		printf("\n");
+		printf("Lifetime Error Log Entries:  ");
+		print_uint128_dec(health_page->num_error_info_log_entries);
+		printf("\n");
+		printf("\n");
+	}
+
+	for (i = 1; i <= nvme_ctrlr_get_num_ns(ctrlr); i++) {
+		print_namespace(nvme_ctrlr_get_ns(ctrlr, i));
+	}
+}
+
+static const char *ealargs[] = {
+	"identify",
+	"-c 0x1",
+	"-n 4",
+};
+
+int main(int argc, char **argv)
+{
+	struct pci_device_iterator	*pci_dev_iter;
+	struct pci_device		*pci_dev;
+	struct pci_id_match		match;
+	int				rc;
+
+	rc = rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]),
+			  (char **)(void *)(uintptr_t)ealargs);
+
+	if (rc < 0) {
+		fprintf(stderr, "could not initialize dpdk\n");
+		exit(1);
+	}
+
+	request_mempool = rte_mempool_create("nvme_request", 8192,
+					     nvme_request_size(), 128, 0,
+					     NULL, NULL, NULL, NULL,
+					     SOCKET_ID_ANY, 0);
+
+	if (request_mempool == NULL) {
+		fprintf(stderr, "could not initialize request mempool\n");
+		exit(1);
+	}
+
+	pci_system_init();
+
+	match.vendor_id =	PCI_MATCH_ANY;
+	match.subvendor_id =	PCI_MATCH_ANY;
+	match.subdevice_id =	PCI_MATCH_ANY;
+	match.device_id =	PCI_MATCH_ANY;
+	match.device_class =	NVME_CLASS_CODE;
+	match.device_class_mask = 0xFFFFFF;
+
+	pci_dev_iter = pci_id_match_iterator_create(&match);
+
+	rc = 0;
+	while ((pci_dev = pci_device_next(pci_dev_iter))) {
+		struct nvme_controller *ctrlr;
+
+		if (pci_device_has_kernel_driver(pci_dev) &&
+		    !pci_device_has_uio_driver(pci_dev)) {
+			fprintf(stderr, "non-uio kernel driver attached to nvme\n");
+			fprintf(stderr, " controller at pci bdf %d:%d:%d\n",
+				pci_dev->bus, pci_dev->dev, pci_dev->func);
+			fprintf(stderr, " skipping...\n");
+			continue;
+		}
+
+		pci_device_probe(pci_dev);
+
+		ctrlr = nvme_attach(pci_dev);
+		if (ctrlr == NULL) {
+			fprintf(stderr, "failed to attach to NVMe controller at PCI BDF %d:%d:%d\n",
+				pci_dev->bus, pci_dev->dev, pci_dev->func);
+			rc = 1;
+			continue;
+		}
+
+		print_controller(ctrlr, pci_dev);
+		nvme_detach(ctrlr);
+	}
+
+	cleanup();
+
+	pci_iterator_destroy(pci_dev_iter);
+	return rc;
+}

examples/nvme/perf/.gitignore

@@ -0,0 +1 @@
+perf

examples/nvme/perf/Makefile

@@ -0,0 +1,61 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+APP = perf
+
+C_SRCS := perf.c
+
+CFLAGS += -I. $(DPDK_INC)
+
+SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/nvme/libspdk_nvme.a \
+	     $(SPDK_ROOT_DIR)/lib/util/libspdk_util.a \
+	     $(SPDK_ROOT_DIR)/lib/memory/libspdk_memory.a
+
+LIBS += $(SPDK_LIBS) -lpciaccess -lpthread $(DPDK_LIB) -lrt
+
+OBJS  = $(C_SRCS:.c=.o)
+
+all : $(APP)
+
+objs : $(OBJS)
+
+$(APP) : $(OBJS) $(SPDK_LIBS)
+	@echo "  LINK $@"
+	$(Q)$(CC) $(CPPFLAGS) $(LDFLAGS) -o $@ $(OBJS) $(LIBS)
+
+clean :
+	$(Q)rm -f $(OBJS) *.d $(APP)
+
+

examples/nvme/perf/perf.c

@@ -0,0 +1,522 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdbool.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <pciaccess.h>
+
+#include <rte_config.h>
+#include <rte_cycles.h>
+#include <rte_mempool.h>
+#include <rte_malloc.h>
+#include <rte_lcore.h>
+
+#include "spdk/nvme.h"
+#include "spdk/pci.h"
+
+struct ctrlr_entry {
+	struct nvme_controller	*ctrlr;
+	struct ctrlr_entry	*next;
+};
+
+struct ns_entry {
+	struct nvme_controller	*ctrlr;
+	struct nvme_namespace	*ns;
+	struct ns_entry		*next;
+	uint32_t		io_size_blocks;
+	int			io_completed;
+	int			current_queue_depth;
+	uint64_t		size_in_ios;
+	uint64_t		offset_in_ios;
+	bool			is_draining;
+	char			name[1024];
+};
+
+struct perf_task {
+	struct ns_entry		*entry;
+	void			*buf;
+};
+
+struct rte_mempool *request_mempool;
+static struct rte_mempool *task_pool;
+
+static struct ctrlr_entry *g_controllers = NULL;
+static struct ns_entry *g_namespaces = NULL;
+
+static uint64_t g_tsc_rate;
+
+static int g_io_size_bytes;
+static int g_rw_percentage;
+static int g_is_random;
+static int g_queue_depth;
+static int g_time_in_sec;
+
+
+static void
+register_ns(struct nvme_controller *ctrlr, struct pci_device *pci_dev, struct nvme_namespace *ns)
+{
+	struct ns_entry *entry = malloc(sizeof(struct ns_entry));
+	const struct nvme_controller_data *cdata = nvme_ctrlr_get_data(ctrlr);
+
+	entry->ctrlr = ctrlr;
+	entry->ns = ns;
+	entry->next = g_namespaces;
+	entry->io_completed = 0;
+	entry->current_queue_depth = 0;
+	entry->offset_in_ios = 0;
+	entry->size_in_ios = nvme_ns_get_size(ns) /
+			     g_io_size_bytes;
+	entry->io_size_blocks = g_io_size_bytes / nvme_ns_get_sector_size(ns);
+	entry->is_draining = false;
+
+	snprintf(entry->name, sizeof(cdata->mn), "%s", cdata->mn);
+	g_namespaces = entry;
+}
+
+static void
+register_ctrlr(struct nvme_controller *ctrlr, struct pci_device *pci_dev)
+{
+	int nsid, num_ns;
+	struct ctrlr_entry *entry = malloc(sizeof(struct ctrlr_entry));
+
+	entry->ctrlr = ctrlr;
+	entry->next = g_controllers;
+	g_controllers = entry;
+
+	num_ns = nvme_ctrlr_get_num_ns(ctrlr);
+	for (nsid = 1; nsid <= num_ns; nsid++) {
+		register_ns(ctrlr, pci_dev, nvme_ctrlr_get_ns(ctrlr, nsid));
+	}
+
+}
+
+void task_ctor(struct rte_mempool *mp, void *arg, void *__task, unsigned id)
+{
+	struct perf_task *task = __task;
+	task->buf = rte_malloc(NULL, g_io_size_bytes, 0x200);
+}
+
+static void io_complete(void *ctx, const struct nvme_completion *completion);
+
+static unsigned int __thread seed = 0;
+
+static void
+submit_single_io(struct ns_entry *entry)
+{
+	struct perf_task	*task = NULL;
+	uint64_t		offset_in_ios;
+	int			rc;
+
+	rte_mempool_get(task_pool, (void **)&task);
+
+	task->entry = entry;
+
+	if (g_is_random) {
+		offset_in_ios = rand_r(&seed) % entry->size_in_ios;
+	} else {
+		offset_in_ios = entry->offset_in_ios++;
+		if (entry->offset_in_ios == entry->size_in_ios) {
+			entry->offset_in_ios = 0;
+		}
+	}
+
+	if ((g_rw_percentage == 100) ||
+	    (g_rw_percentage != 0 && ((rand_r(&seed) % 100) < g_rw_percentage))) {
+		rc = nvme_ns_cmd_read(entry->ns, task->buf, offset_in_ios * entry->io_size_blocks,
+				      entry->io_size_blocks, io_complete, task);
+	} else {
+		rc = nvme_ns_cmd_write(entry->ns, task->buf, offset_in_ios * entry->io_size_blocks,
+				       entry->io_size_blocks, io_complete, task);
+	}
+
+	if (rc != 0) {
+		fprintf(stderr, "starting I/O failed\n");
+	}
+
+	entry->current_queue_depth++;
+}
+
+static void
+io_complete(void *ctx, const struct nvme_completion *completion)
+{
+	struct perf_task	*task;
+	struct ns_entry		*entry;
+
+	task = (struct perf_task *)ctx;
+
+	entry = task->entry;
+	entry->current_queue_depth--;
+	entry->io_completed++;
+
+	rte_mempool_put(task_pool, task);
+
+	/*
+	 * is_draining indicates when time has expired for the test run
+	 * and we are just waiting for the previously submitted I/O
+	 * to complete.  In this case, do not submit a new I/O to replace
+	 * the one just completed.
+	 */
+	if (!entry->is_draining) {
+		submit_single_io(entry);
+	}
+}
+
+static void
+check_io(struct ns_entry *entry)
+{
+	nvme_ctrlr_process_io_completions(entry->ctrlr);
+}
+
+static void
+submit_io(struct ns_entry *entry, int queue_depth)
+{
+	while (queue_depth-- > 0) {
+		submit_single_io(entry);
+	}
+}
+
+static void
+drain_io(struct ns_entry *entry)
+{
+	entry->is_draining = true;
+	while (entry->current_queue_depth > 0) {
+		check_io(entry);
+	}
+}
+
+static int
+work_fn(void *arg)
+{
+	uint64_t tsc_end = rte_get_timer_cycles() + g_time_in_sec * g_tsc_rate;
+	struct ns_entry *entry = (struct ns_entry *)arg;
+
+	nvme_register_io_thread();
+
+	/* Submit initial I/O for each namespace. */
+	while (entry != NULL) {
+		submit_io(entry, g_queue_depth);
+		entry = entry->next;
+	}
+
+	while (1) {
+		/*
+		 * Check for completed I/O for each controller. A new
+		 * I/O will be submitted in the io_complete callback
+		 * to replace each I/O that is completed.
+		 */
+		entry = (struct ns_entry *)arg;
+		while (entry != NULL) {
+			check_io(entry);
+			entry = entry->next;
+		}
+
+		rte_delay_us(1);
+
+		if (rte_get_timer_cycles() > tsc_end) {
+			break;
+		}
+	}
+
+	entry = (struct ns_entry *)arg;
+	while (entry != NULL) {
+		drain_io(entry);
+		entry = entry->next;
+	}
+
+	nvme_unregister_io_thread();
+
+	return 0;
+}
+
+static void usage(char *program_name)
+{
+	printf("%s options\n", program_name);
+	printf("\t[-q io depth]\n");
+	printf("\t[-s io size in bytes]\n");
+	printf("\t[-w io pattern type, must be one of\n");
+	printf("\t\t(read, write, randread, randwrite, rw, randrw)]\n");
+	printf("\t[-M rwmixread (100 for reads, 0 for writes)]\n");
+	printf("\t[-t time in seconds]\n");
+}
+
+static void
+print_stats(void)
+{
+	float io_per_second, mb_per_second;
+	float total_io_per_second, total_mb_per_second;
+
+	total_io_per_second = 0;
+	total_mb_per_second = 0;
+
+	struct ns_entry *entry = g_namespaces;
+	while (entry != NULL) {
+		io_per_second = (float)entry->io_completed /
+				g_time_in_sec;
+		mb_per_second = io_per_second * g_io_size_bytes /
+				(1024 * 1024);
+		printf("%-.20s: %10.2f IO/s %10.2f MB/s\n",
+		       entry->name, io_per_second,
+		       mb_per_second);
+		total_io_per_second += io_per_second;
+		total_mb_per_second += mb_per_second;
+		entry = entry->next;
+	}
+
+	printf("=====================================================\n");
+	printf("%-20s: %10.2f IO/s %10.2f MB/s\n",
+	       "Total", total_io_per_second, total_mb_per_second);
+}
+
+static int
+parse_args(int argc, char **argv)
+{
+	const char *workload_type;
+	int op;
+	bool mix_specified = false;
+
+	/* default value*/
+	g_queue_depth = 0;
+	g_io_size_bytes = 0;
+	workload_type = NULL;
+	g_time_in_sec = 0;
+	g_rw_percentage = -1;
+
+	while ((op = getopt(argc, argv, "q:s:t:w:M:")) != -1) {
+		switch (op) {
+		case 'q':
+			g_queue_depth = atoi(optarg);
+			break;
+		case 's':
+			g_io_size_bytes = atoi(optarg);
+			break;
+		case 't':
+			g_time_in_sec = atoi(optarg);
+			break;
+		case 'w':
+			workload_type = optarg;
+			break;
+		case 'M':
+			g_rw_percentage = atoi(optarg);
+			mix_specified = true;
+			break;
+		default:
+			usage(argv[0]);
+			return 1;
+		}
+	}
+
+	if (!g_queue_depth) {
+		usage(argv[0]);
+		return 1;
+	}
+	if (!g_io_size_bytes) {
+		usage(argv[0]);
+		return 1;
+	}
+	if (!workload_type) {
+		usage(argv[0]);
+		return 1;
+	}
+	if (!g_time_in_sec) {
+		usage(argv[0]);
+		return 1;
+	}
+
+	if (strcmp(workload_type, "read") &&
+	    strcmp(workload_type, "write") &&
+	    strcmp(workload_type, "randread") &&
+	    strcmp(workload_type, "randwrite") &&
+	    strcmp(workload_type, "rw") &&
+	    strcmp(workload_type, "randrw")) {
+		fprintf(stderr,
+			"io pattern type must be one of\n"
+			"(read, write, randread, randwrite, rw, randrw)\n");
+		return 1;
+	}
+
+	if (!strcmp(workload_type, "read") ||
+	    !strcmp(workload_type, "randread")) {
+		g_rw_percentage = 100;
+	}
+
+	if (!strcmp(workload_type, "write") ||
+	    !strcmp(workload_type, "randwrite")) {
+		g_rw_percentage = 0;
+	}
+
+	if (!strcmp(workload_type, "read") ||
+	    !strcmp(workload_type, "randread") ||
+	    !strcmp(workload_type, "write") ||
+	    !strcmp(workload_type, "randwrite")) {
+		if (mix_specified) {
+			fprintf(stderr, "Ignoring -M option... Please use -M option"
+				" only when using rw or randrw.\n");
+		}
+	}
+
+	if (!strcmp(workload_type, "rw") ||
+	    !strcmp(workload_type, "randrw")) {
+		if (g_rw_percentage < 0 || g_rw_percentage > 100) {
+			fprintf(stderr,
+				"-M must be specified to value from 0 to 100 "
+				"for rw or randrw.\n");
+			return 1;
+		}
+	}
+
+	if (!strcmp(workload_type, "read") ||
+	    !strcmp(workload_type, "write") ||
+	    !strcmp(workload_type, "rw")) {
+		g_is_random = 0;
+	} else {
+		g_is_random = 1;
+	}
+
+	optind = 1;
+	return 0;
+}
+
+static int
+register_controllers(void)
+{
+	struct pci_device_iterator	*pci_dev_iter;
+	struct pci_device		*pci_dev;
+	struct pci_id_match		match;
+	int				rc;
+
+	pci_system_init();
+
+	match.vendor_id =	PCI_MATCH_ANY;
+	match.subvendor_id =	PCI_MATCH_ANY;
+	match.subdevice_id =	PCI_MATCH_ANY;
+	match.device_id =	PCI_MATCH_ANY;
+	match.device_class =	NVME_CLASS_CODE;
+	match.device_class_mask = 0xFFFFFF;
+
+	pci_dev_iter = pci_id_match_iterator_create(&match);
+
+	rc = 0;
+	while ((pci_dev = pci_device_next(pci_dev_iter))) {
+		struct nvme_controller *ctrlr;
+
+		if (pci_device_has_kernel_driver(pci_dev) &&
+		    !pci_device_has_uio_driver(pci_dev)) {
+			fprintf(stderr, "non-uio kernel driver attached to nvme\n");
+			fprintf(stderr, " controller at pci bdf %d:%d:%d\n",
+				pci_dev->bus, pci_dev->dev, pci_dev->func);
+			fprintf(stderr, " skipping...\n");
+			continue;
+		}
+
+		pci_device_probe(pci_dev);
+
+		ctrlr = nvme_attach(pci_dev);
+		if (ctrlr == NULL) {
+			fprintf(stderr, "nvme_attach failed for controller at pci bdf %d:%d:%d\n",
+				pci_dev->bus, pci_dev->dev, pci_dev->func);
+			rc = 1;
+			continue;
+		}
+
+		register_ctrlr(ctrlr, pci_dev);
+	}
+
+	pci_iterator_destroy(pci_dev_iter);
+
+	return rc;
+}
+
+static void
+unregister_controllers(void)
+{
+	struct ctrlr_entry *entry = g_controllers;
+	while (entry) {
+		struct ctrlr_entry *next = entry->next;
+		nvme_detach(entry->ctrlr);
+		free(entry);
+		entry = next;
+	}
+}
+
+static const char *ealargs[] = {
+	"perf",
+	"-c 0x1",
+	"-n 4",
+};
+
+int main(int argc, char **argv)
+{
+	int rc;
+
+	rc = parse_args(argc, argv);
+	if (rc != 0) {
+		return rc;
+	}
+
+	rc = rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]),
+			  (char **)(void *)(uintptr_t)ealargs);
+
+	if (rc < 0) {
+		fprintf(stderr, "could not initialize dpdk\n");
+		return 1;
+	}
+
+	request_mempool = rte_mempool_create("nvme_request", 8192,
+					     nvme_request_size(), 128, 0,
+					     NULL, NULL, NULL, NULL,
+					     SOCKET_ID_ANY, 0);
+
+	if (request_mempool == NULL) {
+		fprintf(stderr, "could not initialize request mempool\n");
+		return 1;
+	}
+
+	task_pool = rte_mempool_create("task_pool", 2048,
+				       sizeof(struct perf_task),
+				       64, 0, NULL, NULL, task_ctor, NULL,
+				       SOCKET_ID_ANY, 0);
+
+	g_tsc_rate = rte_get_timer_hz();
+
+	rc = register_controllers();
+
+	work_fn(g_namespaces);
+	print_stats();
+
+	unregister_controllers();
+
+	return rc;
+}

include/spdk/barrier.h

@@ -0,0 +1,40 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef SPDK_BARRIER_H
+#define SPDK_BARRIER_H
+
+#define wmb()	__asm volatile("sfence" ::: "memory")
+#define mb()	__asm volatile("mfence" ::: "memory")
+
+#endif

include/spdk/nvme.h

@@ -0,0 +1,372 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef SPDK_NVME_H
+#define SPDK_NVME_H
+
+#include <stddef.h>
+#include "nvme_spec.h"
+
+/** \file
+ *
+ */
+
+#define NVME_DEFAULT_RETRY_COUNT	(4)
+extern int32_t		nvme_retry_count;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** \brief Opaque handle to a controller. Obtained by calling nvme_attach(). */
+struct nvme_controller;
+
+/**
+ * \brief Attaches specified device to the NVMe driver.
+ *
+ * On success, the nvme_controller handle is valid for other nvme_ctrlr_* functions.
+ * On failure, the return value will be NULL.
+ *
+ * This function should be called from a single thread while no other threads or drivers
+ * are actively using the NVMe device.
+ *
+ * To stop using the the controller and release its associated resources,
+ * call \ref nvme_detach with the nvme_controller instance returned by this function.
+ */
+struct nvme_controller *nvme_attach(void *devhandle);
+
+/**
+ * \brief Detaches specified device returned by \ref nvme_attach() from the NVMe driver.
+ *
+ * On success, the nvme_controller handle is no longer valid.
+ *
+ * This function should be called from a single thread while no other threads
+ * are actively using the NVMe device.
+ *
+ */
+int nvme_detach(struct nvme_controller *ctrlr);
+
+/**
+ * \brief Perform a full hardware reset of the NVMe controller.
+ *
+ * This function should be called from a single thread while no other threads
+ * are actively using the NVMe device.
+ *
+ * Any pointers returned from nvme_ctrlr_get_ns() and nvme_ns_get_data() may be invalidated
+ * by calling this function.  The number of namespaces as returned by nvme_ctrlr_get_num_ns() may
+ * also change.
+ */
+int nvme_ctrlr_reset(struct nvme_controller *ctrlr);
+
+/**
+ * \brief Get the identify controller data as defined by the NVMe specification.
+ *
+ * This function is thread safe and can be called at any point after nvme_attach().
+ *
+ */
+const struct nvme_controller_data *nvme_ctrlr_get_data(struct nvme_controller *ctrlr);
+
+/**
+ * \brief Get the number of namespaces for the given NVMe controller.
+ *
+ * This function is thread safe and can be called at any point after nvme_attach().
+ *
+ * This is equivalent to calling nvme_ctrlr_get_data() to get the
+ * nvme_controller_data and then reading the nn field.
+ *
+ */
+uint32_t nvme_ctrlr_get_num_ns(struct nvme_controller *ctrlr);
+
+/**
+ * Signature for callback function invoked when a command is completed.
+ *
+ * The nvme_completion parameter contains the completion status.
+ */
+typedef void (*nvme_cb_fn_t)(void *, const struct nvme_completion *);
+
+/**
+ * Signature for callback function invoked when an asynchronous error
+ *  request command is completed.
+ *
+ * The aer_cb_arg parameter is set to the context specified by
+ *  nvme_register_aer_callback().
+ * The nvme_completion parameter contains the completion status of the
+ *  asynchronous event request that was completed.
+ */
+typedef void (*nvme_aer_cb_fn_t)(void *aer_cb_arg,
+				 const struct nvme_completion *);
+
+void nvme_ctrlr_register_aer_callback(struct nvme_controller *ctrlr,
+				      nvme_aer_cb_fn_t aer_cb_fn,
+				      void *aer_cb_arg);
+
+/**
+ * \brief Send the given NVM I/O command to the NVMe controller.
+ *
+ * This is a low level interface for submitting I/O commands directly. Prefer
+ * the nvme_ns_cmd_* functions instead. The validity of the command will
+ * not be checked!
+ *
+ * When constructing the nvme_command it is not necessary to fill out the PRP
+ * list/SGL or the CID. The driver will handle both of those for you.
+ *
+ * This function is thread safe and can be called at any point after
+ * nvme_register_io_thread().
+ *
+ */
+int nvme_ctrlr_cmd_io_raw(struct nvme_controller *ctrlr,
+			  struct nvme_command *cmd,
+			  void *buf, uint32_t len,
+			  nvme_cb_fn_t cb_fn, void *cb_arg);
+
+/**
+ * \brief Process any outstanding completions for I/O submitted on the current thread.
+ *
+ * This will only process completions for I/O that were submitted on the same thread
+ * that this function is called from. This call is also non-blocking, i.e. it only
+ * processes completions that are ready at the time of this function call. It does not
+ * wait for outstanding commands to finish
+ *
+ * This function is thread safe and can be called at any point after nvme_attach().
+ *
+ */
+void nvme_ctrlr_process_io_completions(struct nvme_controller *ctrlr);
+
+/**
+ * \brief Send the given admin command to the NVMe controller.
+ *
+ * This is a low level interface for submitting admin commands directly. Prefer
+ * the nvme_ctrlr_cmd_* functions instead. The validity of the command will
+ * not be checked!
+ *
+ * When constructing the nvme_command it is not necessary to fill out the PRP
+ * list/SGL or the CID. The driver will handle both of those for you.
+ *
+ * This function is thread safe and can be called at any point after
+ * \ref nvme_attach().
+ *
+ * Call \ref nvme_ctrlr_process_admin_completions() to poll for completion
+ * of commands submitted through this function.
+ */
+int nvme_ctrlr_cmd_admin_raw(struct nvme_controller *ctrlr,
+			     struct nvme_command *cmd,
+			     void *buf, uint32_t len,
+			     nvme_cb_fn_t cb_fn, void *cb_arg);
+
+/**
+ * \brief Process any outstanding completions for admin commands.
+ *
+ * This will process completions for admin commands submitted on any thread.
+ *
+ * This call is non-blocking, i.e. it only processes completions that are ready
+ * at the time of this function call. It does not wait for outstanding commands to
+ * finish.
+ *
+ * This function is thread safe and can be called at any point after nvme_attach().
+ */
+void nvme_ctrlr_process_admin_completions(struct nvme_controller *ctrlr);
+
+
+/** \brief Opaque handle to a namespace. Obtained by calling nvme_ctrlr_get_ns(). */
+struct nvme_namespace;
+
+/**
+ * \brief Get a handle to a namespace for the given controller.
+ *
+ * Namespaces are numbered from 1 to the total number of namespaces. There will never
+ * be any gaps in the numbering. The number of namespaces is obtained by calling
+ * nvme_ctrlr_get_num_ns().
+ *
+ * This function is thread safe and can be called at any point after nvme_attach().
+ *
+ */
+struct nvme_namespace *nvme_ctrlr_get_ns(struct nvme_controller *ctrlr, uint32_t ns_id);
+
+/**
+ * \brief Get the identify namespace data as defined by the NVMe specification.
+ *
+ * This function is thread safe and can be called at any point after nvme_attach().
+ *
+ */
+const struct nvme_namespace_data *nvme_ns_get_data(struct nvme_namespace *ns);
+
+/**
+ * \brief Get the namespace id (index number) from the given namespace handle.
+ *
+ * This function is thread safe and can be called at any point after nvme_attach().
+ *
+ */
+uint32_t nvme_ns_get_id(struct nvme_namespace *ns);
+
+/**
+ * \brief Get the maximum transfer size, in bytes, for an I/O sent to the given namespace.
+ *
+ * This function is thread safe and can be called at any point after nvme_attach().
+ *
+ */
+uint32_t nvme_ns_get_max_io_xfer_size(struct nvme_namespace *ns);
+
+/**
+ * \brief Get the sector size, in bytes, of the given namespace.
+ *
+ * This function is thread safe and can be called at any point after nvme_attach().
+ *
+ */
+uint32_t nvme_ns_get_sector_size(struct nvme_namespace *ns);
+
+/**
+ * \brief Get the number of sectors for the given namespace.
+ *
+ * This function is thread safe and can be called at any point after nvme_attach().
+ *
+ */
+uint64_t nvme_ns_get_num_sectors(struct nvme_namespace *ns);
+
+/**
+ * \brief Get the size, in bytes, of the given namespace.
+ *
+ * This function is thread safe and can be called at any point after nvme_attach().
+ *
+ */
+uint64_t nvme_ns_get_size(struct nvme_namespace *ns);
+
+enum nvme_namespace_flags {
+	NVME_NS_DEALLOCATE_SUPPORTED	= 0x1,
+	NVME_NS_FLUSH_SUPPORTED		= 0x2,
+};
+
+/**
+ * \brief Get the flags for the given namespace.
+ *
+ * See nvme_namespace_flags for the possible flags returned.
+ *
+ * This function is thread safe and can be called at any point after nvme_attach().
+ *
+ */
+uint32_t nvme_ns_get_flags(struct nvme_namespace *ns);
+
+/**
+ * \brief Submits a write I/O to the specified NVMe namespace.
+ *
+ * \param ns NVMe namespace to submit the write I/O
+ * \param payload virtual address pointer to the data payload
+ * \param lba starting LBA to write the data
+ * \param lba_count length (in sectors) for the write operation
+ * \param cb_fn callback function to invoke when the I/O is completed
+ * \param cb_arg argument to pass to the callback function
+ *
+ * \return 0 if successfully submitted, ENOMEM if an nvme_request
+ *	     structure cannot be allocated for the I/O request
+ *
+ * This function is thread safe and can be called at any point after
+ * nvme_register_io_thread().
+ */
+int nvme_ns_cmd_write(struct nvme_namespace *ns, void *payload,
+		      uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn,
+		      void *cb_arg);
+
+/**
+ * \brief Submits a read I/O to the specified NVMe namespace.
+ *
+ * \param ns NVMe namespace to submit the read I/O
+ * \param payload virtual address pointer to the data payload
+ * \param lba starting LBA to read the data
+ * \param lba_count length (in sectors) for the read operation
+ * \param cb_fn callback function to invoke when the I/O is completed
+ * \param cb_arg argument to pass to the callback function
+ *
+ * \return 0 if successfully submitted, ENOMEM if an nvme_request
+ *	     structure cannot be allocated for the I/O request
+ *
+ * This function is thread safe and can be called at any point after
+ * nvme_register_io_thread().
+ */
+int nvme_ns_cmd_read(struct nvme_namespace *ns, void *payload,
+		     uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn,
+		     void *cb_arg);
+
+/**
+ * \brief Submits a deallocation request to the specified NVMe namespace.
+ *
+ * \param ns NVMe namespace to submit the deallocation request
+ * \param payload virtual address pointer to the list of LBA ranges to
+ *                deallocate
+ * \param num_ranges number of ranges in the list pointed to by payload
+ * \param cb_fn callback function to invoke when the I/O is completed
+ * \param cb_arg argument to pass to the callback function
+ *
+ * \return 0 if successfully submitted, ENOMEM if an nvme_request
+ *	     structure cannot be allocated for the I/O request
+ *
+ * This function is thread safe and can be called at any point after
+ * nvme_register_io_thread().
+ */
+int nvme_ns_cmd_deallocate(struct nvme_namespace *ns, void *payload,
+			   uint8_t num_ranges, nvme_cb_fn_t cb_fn,
+			   void *cb_arg);
+
+/**
+ * \brief Submits a flush request to the specified NVMe namespace.
+ *
+ * \param ns NVMe namespace to submit the flush request
+ * \param cb_fn callback function to invoke when the I/O is completed
+ * \param cb_arg argument to pass to the callback function
+ *
+ * \return 0 if successfully submitted, ENOMEM if an nvme_request
+ *	     structure cannot be allocated for the I/O request
+ *
+ * This function is thread safe and can be called at any point after
+ * nvme_register_io_thread().
+ */
+int nvme_ns_cmd_flush(struct nvme_namespace *ns, nvme_cb_fn_t cb_fn,
+		      void *cb_arg);
+
+/**
+ * \brief Get the size, in bytes, of an nvme_request.
+ *
+ * This is the size of the request objects that need to be allocated by the
+ * nvme_alloc_request macro in nvme_impl.h
+ *
+ * This function is thread safe and can be called at any time.
+ *
+ */
+size_t nvme_request_size(void);
+
+int nvme_register_io_thread(void);
+void nvme_unregister_io_thread(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/spdk/nvme_spec.h

@@ -0,0 +1,948 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef SPDK_NVME_SPEC_H
+#define SPDK_NVME_SPEC_H
+
+#include <stdint.h>
+#include <stddef.h>
+
+/**
+ * \file
+ *
+ */
+
+/**
+ * PCI class code for NVMe devices.
+ *
+ * Base class code 01h: mass storage
+ * Subclass code 08h: non-volatile memory
+ * Programming interface 02h: NVM Express
+ */
+#define NVME_CLASS_CODE 0x10802
+
+/**
+ * Use to mark a command to apply to all namespaces, or to retrieve global
+ *  log pages.
+ */
+#define NVME_GLOBAL_NAMESPACE_TAG	((uint32_t)0xFFFFFFFF)
+
+#define NVME_MAX_IO_QUEUES		(1 << 16)
+
+union nvme_cap_lo_register {
+	uint32_t	raw;
+	struct {
+		/** maximum queue entries supported */
+		uint32_t mqes		: 16;
+
+		/** contiguous queues required */
+		uint32_t cqr		: 1;
+
+		/** arbitration mechanism supported */
+		uint32_t ams		: 2;
+
+		uint32_t reserved1	: 5;
+
+		/** timeout */
+		uint32_t to		: 8;
+	} bits;
+};
+_Static_assert(sizeof(union nvme_cap_lo_register) == 4, "Incorrect size");
+
+union nvme_cap_hi_register {
+	uint32_t	raw;
+	struct {
+		/** doorbell stride */
+		uint32_t dstrd		: 4;
+
+		uint32_t reserved3	: 1;
+
+		/** command sets supported */
+		uint32_t css_nvm	: 1;
+
+		uint32_t css_reserved	: 3;
+		uint32_t reserved2	: 7;
+
+		/** memory page size minimum */
+		uint32_t mpsmin		: 4;
+
+		/** memory page size maximum */
+		uint32_t mpsmax		: 4;
+
+		uint32_t reserved1	: 8;
+	} bits;
+};
+_Static_assert(sizeof(union nvme_cap_hi_register) == 4, "Incorrect size");
+
+union nvme_cc_register {
+	uint32_t	raw;
+	struct {
+		/** enable */
+		uint32_t en		: 1;
+
+		uint32_t reserved1	: 3;
+
+		/** i/o command set selected */
+		uint32_t css		: 3;
+
+		/** memory page size */
+		uint32_t mps		: 4;
+
+		/** arbitration mechanism selected */
+		uint32_t ams		: 3;
+
+		/** shutdown notification */
+		uint32_t shn		: 2;
+
+		/** i/o submission queue entry size */
+		uint32_t iosqes		: 4;
+
+		/** i/o completion queue entry size */
+		uint32_t iocqes		: 4;
+
+		uint32_t reserved2	: 8;
+	} bits;
+};
+_Static_assert(sizeof(union nvme_cc_register) == 4, "Incorrect size");
+
+enum nvme_shn_value {
+	NVME_SHN_NORMAL		= 0x1,
+	NVME_SHN_ABRUPT		= 0x2,
+};
+
+union nvme_csts_register {
+	uint32_t	raw;
+	struct {
+		/** ready */
+		uint32_t rdy		: 1;
+
+		/** controller fatal status */
+		uint32_t cfs		: 1;
+
+		/** shutdown status */
+		uint32_t shst		: 2;
+
+		uint32_t reserved1	: 28;
+	} bits;
+};
+_Static_assert(sizeof(union nvme_csts_register) == 4, "Incorrect size");
+
+enum nvme_shst_value {
+	NVME_SHST_NORMAL	= 0x0,
+	NVME_SHST_OCCURRING	= 0x1,
+	NVME_SHST_COMPLETE	= 0x2,
+};
+
+union nvme_aqa_register {
+	uint32_t	raw;
+	struct {
+		/** admin submission queue size */
+		uint32_t asqs		: 12;
+
+		uint32_t reserved1	: 4;
+
+		/** admin completion queue size */
+		uint32_t acqs		: 12;
+
+		uint32_t reserved2	: 4;
+	} bits;
+};
+_Static_assert(sizeof(union nvme_aqa_register) == 4, "Incorrect size");
+
+struct nvme_registers {
+	/** controller capabilities */
+	union nvme_cap_lo_register	cap_lo;
+	union nvme_cap_hi_register	cap_hi;
+
+	uint32_t	vs;		/* version */
+	uint32_t	intms;		/* interrupt mask set */
+	uint32_t	intmc;		/* interrupt mask clear */
+
+	/** controller configuration */
+	union nvme_cc_register	cc;
+
+	uint32_t	reserved1;
+	uint32_t	csts;		/* controller status */
+	uint32_t	nssr;		/* NVM subsystem reset */
+
+	/** admin queue attributes */
+	union nvme_aqa_register	aqa;
+
+	uint64_t	asq;		/* admin submission queue base addr */
+	uint64_t	acq;		/* admin completion queue base addr */
+	uint32_t	reserved3[0x3f2];
+
+	struct {
+		uint32_t	sq_tdbl;	/* submission queue tail doorbell */
+		uint32_t	cq_hdbl;	/* completion queue head doorbell */
+	} doorbell[1];
+};
+
+/* NVMe controller register space offsets */
+_Static_assert(0x00 == offsetof(struct nvme_registers, cap_lo), "Incorrect register offset");
+_Static_assert(0x08 == offsetof(struct nvme_registers, vs), "Incorrect register offset");
+_Static_assert(0x0C == offsetof(struct nvme_registers, intms), "Incorrect register offset");
+_Static_assert(0x10 == offsetof(struct nvme_registers, intmc), "Incorrect register offset");
+_Static_assert(0x14 == offsetof(struct nvme_registers, cc), "Incorrect register offset");
+_Static_assert(0x1C == offsetof(struct nvme_registers, csts), "Incorrect register offset");
+_Static_assert(0x20 == offsetof(struct nvme_registers, nssr), "Incorrect register offset");
+_Static_assert(0x24 == offsetof(struct nvme_registers, aqa), "Incorrect register offset");
+_Static_assert(0x28 == offsetof(struct nvme_registers, asq), "Incorrect register offset");
+_Static_assert(0x30 == offsetof(struct nvme_registers, acq), "Incorrect register offset");
+
+enum nvme_sgl_descriptor_type {
+	NVME_SGL_TYPE_DATA_BLOCK	= 0x0,
+	NVME_SGL_TYPE_BIT_BUCKET	= 0x1,
+	NVME_SGL_TYPE_SEGMENT		= 0x2,
+	NVME_SGL_TYPE_LAST_SEGMENT	= 0x3,
+	/* 0x4 - 0xe reserved */
+	NVME_SGL_TYPE_VENDOR_SPECIFIC	= 0xf
+};
+
+struct __attribute__((packed)) nvme_sgl_descriptor {
+	uint64_t address;
+	uint32_t length;
+	uint8_t reserved[3];
+
+	/** SGL descriptor type */
+	uint8_t type : 4;
+
+	/** SGL descriptor type specific */
+	uint8_t type_specific : 4;
+};
+_Static_assert(sizeof(struct nvme_sgl_descriptor) == 16, "Incorrect size");
+
+enum nvme_psdt_value {
+	NVME_PSDT_PRP			= 0x0,
+	NVME_PSDT_SGL_MPTR_CONTIG	= 0x1,
+	NVME_PSDT_SGL_MPTR_SGL		= 0x2,
+	NVME_PSDT_RESERVED		= 0x3
+};
+
+struct nvme_command {
+	/* dword 0 */
+	uint16_t opc	:  8;	/* opcode */
+	uint16_t fuse	:  2;	/* fused operation */
+	uint16_t rsvd1	:  4;
+	uint16_t psdt	:  2;
+	uint16_t cid;		/* command identifier */
+
+	/* dword 1 */
+	uint32_t nsid;		/* namespace identifier */
+
+	/* dword 2-3 */
+	uint32_t rsvd2;
+	uint32_t rsvd3;
+
+	/* dword 4-5 */
+	uint64_t mptr;		/* metadata pointer */
+
+	/* dword 6-9: data pointer */
+	union {
+		struct {
+			uint64_t prp1;		/* prp entry 1 */
+			uint64_t prp2;		/* prp entry 2 */
+		} prp;
+
+		struct nvme_sgl_descriptor sgl1;
+	} dptr;
+
+	/* dword 10-15 */
+	uint32_t cdw10;		/* command-specific */
+	uint32_t cdw11;		/* command-specific */
+	uint32_t cdw12;		/* command-specific */
+	uint32_t cdw13;		/* command-specific */
+	uint32_t cdw14;		/* command-specific */
+	uint32_t cdw15;		/* command-specific */
+};
+_Static_assert(sizeof(struct nvme_command) == 64, "Incorrect size");
+
+struct nvme_status {
+	uint16_t p	:  1;	/* phase tag */
+	uint16_t sc	:  8;	/* status code */
+	uint16_t sct	:  3;	/* status code type */
+	uint16_t rsvd2	:  2;
+	uint16_t m	:  1;	/* more */
+	uint16_t dnr	:  1;	/* do not retry */
+};
+_Static_assert(sizeof(struct nvme_status) == 2, "Incorrect size");
+
+struct nvme_completion {
+	/* dword 0 */
+	uint32_t		cdw0;	/* command-specific */
+
+	/* dword 1 */
+	uint32_t		rsvd1;
+
+	/* dword 2 */
+	uint16_t		sqhd;	/* submission queue head pointer */
+	uint16_t		sqid;	/* submission queue identifier */
+
+	/* dword 3 */
+	uint16_t		cid;	/* command identifier */
+	struct nvme_status	status;
+};
+_Static_assert(sizeof(struct nvme_completion) == 16, "Incorrect size");
+
+struct nvme_dsm_range {
+	uint32_t attributes;
+	uint32_t length;
+	uint64_t starting_lba;
+};
+_Static_assert(sizeof(struct nvme_dsm_range) == 16, "Incorrect size");
+
+/* status code types */
+enum nvme_status_code_type {
+	NVME_SCT_GENERIC		= 0x0,
+	NVME_SCT_COMMAND_SPECIFIC	= 0x1,
+	NVME_SCT_MEDIA_ERROR		= 0x2,
+	/* 0x3-0x6 - reserved */
+	NVME_SCT_VENDOR_SPECIFIC	= 0x7,
+};
+
+/* generic command status codes */
+enum nvme_generic_command_status_code {
+	NVME_SC_SUCCESS				= 0x00,
+	NVME_SC_INVALID_OPCODE			= 0x01,
+	NVME_SC_INVALID_FIELD			= 0x02,
+	NVME_SC_COMMAND_ID_CONFLICT		= 0x03,
+	NVME_SC_DATA_TRANSFER_ERROR		= 0x04,
+	NVME_SC_ABORTED_POWER_LOSS		= 0x05,
+	NVME_SC_INTERNAL_DEVICE_ERROR		= 0x06,
+	NVME_SC_ABORTED_BY_REQUEST		= 0x07,
+	NVME_SC_ABORTED_SQ_DELETION		= 0x08,
+	NVME_SC_ABORTED_FAILED_FUSED		= 0x09,
+	NVME_SC_ABORTED_MISSING_FUSED		= 0x0a,
+	NVME_SC_INVALID_NAMESPACE_OR_FORMAT	= 0x0b,
+	NVME_SC_COMMAND_SEQUENCE_ERROR		= 0x0c,
+
+	NVME_SC_LBA_OUT_OF_RANGE		= 0x80,
+	NVME_SC_CAPACITY_EXCEEDED		= 0x81,
+	NVME_SC_NAMESPACE_NOT_READY		= 0x82,
+};
+
+/* command specific status codes */
+enum nvme_command_specific_status_code {
+	NVME_SC_COMPLETION_QUEUE_INVALID	= 0x00,
+	NVME_SC_INVALID_QUEUE_IDENTIFIER	= 0x01,
+	NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED	= 0x02,
+	NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED	= 0x03,
+	/* 0x04 - reserved */
+	NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED = 0x05,
+	NVME_SC_INVALID_FIRMWARE_SLOT		= 0x06,
+	NVME_SC_INVALID_FIRMWARE_IMAGE		= 0x07,
+	NVME_SC_INVALID_INTERRUPT_VECTOR	= 0x08,
+	NVME_SC_INVALID_LOG_PAGE		= 0x09,
+	NVME_SC_INVALID_FORMAT			= 0x0a,
+	NVME_SC_FIRMWARE_REQUIRES_RESET		= 0x0b,
+
+	NVME_SC_CONFLICTING_ATTRIBUTES		= 0x80,
+	NVME_SC_INVALID_PROTECTION_INFO		= 0x81,
+	NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE	= 0x82,
+};
+
+/* media error status codes */
+enum nvme_media_error_status_code {
+	NVME_SC_WRITE_FAULTS			= 0x80,
+	NVME_SC_UNRECOVERED_READ_ERROR		= 0x81,
+	NVME_SC_GUARD_CHECK_ERROR		= 0x82,
+	NVME_SC_APPLICATION_TAG_CHECK_ERROR	= 0x83,
+	NVME_SC_REFERENCE_TAG_CHECK_ERROR	= 0x84,
+	NVME_SC_COMPARE_FAILURE			= 0x85,
+	NVME_SC_ACCESS_DENIED			= 0x86,
+};
+
+/* admin opcodes */
+enum nvme_admin_opcode {
+	NVME_OPC_DELETE_IO_SQ			= 0x00,
+	NVME_OPC_CREATE_IO_SQ			= 0x01,
+	NVME_OPC_GET_LOG_PAGE			= 0x02,
+	/* 0x03 - reserved */
+	NVME_OPC_DELETE_IO_CQ			= 0x04,
+	NVME_OPC_CREATE_IO_CQ			= 0x05,
+	NVME_OPC_IDENTIFY			= 0x06,
+	/* 0x07 - reserved */
+	NVME_OPC_ABORT				= 0x08,
+	NVME_OPC_SET_FEATURES			= 0x09,
+	NVME_OPC_GET_FEATURES			= 0x0a,
+	/* 0x0b - reserved */
+	NVME_OPC_ASYNC_EVENT_REQUEST		= 0x0c,
+	NVME_OPC_NAMESPACE_MANAGEMENT		= 0x0d,
+	/* 0x0e-0x0f - reserved */
+	NVME_OPC_FIRMWARE_COMMIT		= 0x10,
+	NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD	= 0x11,
+
+	NVME_OPC_NAMESPACE_ATTACHMENT		= 0x15,
+
+	NVME_OPC_FORMAT_NVM			= 0x80,
+	NVME_OPC_SECURITY_SEND			= 0x81,
+	NVME_OPC_SECURITY_RECEIVE		= 0x82,
+};
+
+/* nvme nvm opcodes */
+enum nvme_nvm_opcode {
+	NVME_OPC_FLUSH				= 0x00,
+	NVME_OPC_WRITE				= 0x01,
+	NVME_OPC_READ				= 0x02,
+	/* 0x03 - reserved */
+	NVME_OPC_WRITE_UNCORRECTABLE		= 0x04,
+	NVME_OPC_COMPARE			= 0x05,
+	/* 0x06-0x07 - reserved */
+	NVME_OPC_WRITE_ZEROES			= 0x08,
+	NVME_OPC_DATASET_MANAGEMENT		= 0x09,
+
+	NVME_OPC_RESERVATION_REGISTER		= 0x0d,
+	NVME_OPC_RESERVATION_REPORT		= 0x0e,
+
+	NVME_OPC_RESERVATION_ACQUIRE		= 0x11,
+	NVME_OPC_RESERVATION_RELEASE		= 0x15,
+};
+
+enum nvme_feature {
+	/* 0x00 - reserved */
+	NVME_FEAT_ARBITRATION			= 0x01,
+	NVME_FEAT_POWER_MANAGEMENT		= 0x02,
+	NVME_FEAT_LBA_RANGE_TYPE		= 0x03,
+	NVME_FEAT_TEMPERATURE_THRESHOLD		= 0x04,
+	NVME_FEAT_ERROR_RECOVERY		= 0x05,
+	NVME_FEAT_VOLATILE_WRITE_CACHE		= 0x06,
+	NVME_FEAT_NUMBER_OF_QUEUES		= 0x07,
+	NVME_FEAT_INTERRUPT_COALESCING		= 0x08,
+	NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION = 0x09,
+	NVME_FEAT_WRITE_ATOMICITY		= 0x0A,
+	NVME_FEAT_ASYNC_EVENT_CONFIGURATION	= 0x0B,
+	/* 0x0C-0x7F - reserved */
+	NVME_FEAT_SOFTWARE_PROGRESS_MARKER	= 0x80,
+	/* 0x81-0xBF - command set specific (reserved) */
+	/* 0xC0-0xFF - vendor specific */
+};
+
+enum nvme_dsm_attribute {
+	NVME_DSM_ATTR_INTEGRAL_READ		= 0x1,
+	NVME_DSM_ATTR_INTEGRAL_WRITE		= 0x2,
+	NVME_DSM_ATTR_DEALLOCATE		= 0x4,
+};
+
+struct nvme_power_state {
+	uint16_t mp;				/* bits 15:00: maximum power */
+
+	uint8_t reserved1;
+
+	uint8_t mps		: 1;		/* bit 24: max power scale */
+	uint8_t nops		: 1;		/* bit 25: non-operational state */
+	uint8_t reserved2	: 6;
+
+	uint32_t enlat;				/* bits 63:32: entry latency in microseconds */
+	uint32_t exlat;				/* bits 95:64: exit latency in microseconds */
+
+	uint8_t rrt		: 5;		/* bits 100:96: relative read throughput */
+	uint8_t reserved3	: 3;
+
+	uint8_t rrl		: 5;		/* bits 108:104: relative read latency */
+	uint8_t reserved4	: 3;
+
+	uint8_t rwt		: 5;		/* bits 116:112: relative write throughput */
+	uint8_t reserved5	: 3;
+
+	uint8_t rwl		: 5;		/* bits 124:120: relative write latency */
+	uint8_t reserved6	: 3;
+
+	uint8_t reserved7[16];
+};
+_Static_assert(sizeof(struct nvme_power_state) == 32, "Incorrect size");
+
+struct __attribute__((packed)) nvme_controller_data {
+	/* bytes 0-255: controller capabilities and features */
+
+	/** pci vendor id */
+	uint16_t		vid;
+
+	/** pci subsystem vendor id */
+	uint16_t		ssvid;
+
+	/** serial number */
+	int8_t			sn[20];
+
+	/** model number */
+	int8_t			mn[40];
+
+	/** firmware revision */
+	uint8_t			fr[8];
+
+	/** recommended arbitration burst */
+	uint8_t			rab;
+
+	/** ieee oui identifier */
+	uint8_t			ieee[3];
+
+	/** multi-interface capabilities */
+	uint8_t			mic;
+
+	/** maximum data transfer size */
+	uint8_t			mdts;
+
+	/** controller id */
+	uint16_t		cntlid;
+
+	/** version */
+	uint32_t		ver;
+
+	/** RTD3 resume latency */
+	uint32_t		rtd3r;
+
+	/** RTD3 entry latency */
+	uint32_t		rtd3e;
+
+	/** optional asynchronous events supported */
+	uint32_t		oaes;
+
+	uint8_t			reserved1[160];
+
+	/* bytes 256-511: admin command set attributes */
+
+	/** optional admin command support */
+	struct {
+		/* supports security send/receive commands */
+		uint16_t	security  : 1;
+
+		/* supports format nvm command */
+		uint16_t	format    : 1;
+
+		/* supports firmware activate/download commands */
+		uint16_t	firmware  : 1;
+
+		uint16_t	oacs_rsvd : 13;
+	} oacs;
+
+	/** abort command limit */
+	uint8_t			acl;
+
+	/** asynchronous event request limit */
+	uint8_t			aerl;
+
+	/** firmware updates */
+	struct {
+		/* first slot is read-only */
+		uint8_t		slot1_ro  : 1;
+
+		/* number of firmware slots */
+		uint8_t		num_slots : 3;
+
+		uint8_t		frmw_rsvd : 4;
+	} frmw;
+
+	/** log page attributes */
+	struct {
+		/* per namespace smart/health log page */
+		uint8_t		ns_smart : 1;
+
+		uint8_t		lpa_rsvd : 7;
+	} lpa;
+
+	/** error log page entries */
+	uint8_t			elpe;
+
+	/** number of power states supported */
+	uint8_t			npss;
+
+	/** admin vendor specific command configuration */
+	struct {
+		/* admin vendor specific commands use disk format */
+		uint8_t		spec_format : 1;
+
+		uint8_t		avscc_rsvd  : 7;
+	} avscc;
+
+	/** autonomous power state transition attributes */
+	struct {
+		/** controller supports autonomous power state transitions */
+		uint8_t		supported  : 1;
+
+		uint8_t		apsta_rsvd : 7;
+	} apsta;
+
+	/** warning composite temperature threshold */
+	uint16_t		wctemp;
+
+	/** critical composite temperature threshold */
+	uint16_t		cctemp;
+
+	/** maximum time for firmware activation */
+	uint16_t		mtfa;
+
+	/** host memory buffer preferred size */
+	uint32_t		hmpre;
+
+	/** host memory buffer minimum size */
+	uint32_t		hmmin;
+
+	/** total NVM capacity */
+	uint64_t		tnvmcap[2];
+
+	/** unallocated NVM capacity */
+	uint64_t		unvmcap[2];
+
+	/** replay protected memory block support */
+	struct {
+		uint8_t		num_rpmb_units	: 3;
+		uint8_t		auth_method	: 3;
+		uint8_t		reserved1	: 2;
+
+		uint8_t		reserved2;
+
+		uint8_t		total_size;
+		uint8_t		access_size;
+	} rpmbs;
+
+	uint8_t			reserved2[196];
+
+	/* bytes 512-703: nvm command set attributes */
+
+	/** submission queue entry size */
+	struct {
+		uint8_t		min : 4;
+		uint8_t		max : 4;
+	} sqes;
+
+	/** completion queue entry size */
+	struct {
+		uint8_t		min : 4;
+		uint8_t		max : 4;
+	} cqes;
+
+	uint8_t			reserved3[2];
+
+	/** number of namespaces */
+	uint32_t		nn;
+
+	/** optional nvm command support */
+	struct {
+		uint16_t	compare : 1;
+		uint16_t	write_unc : 1;
+		uint16_t	dsm: 1;
+		uint16_t	reserved: 13;
+	} oncs;
+
+	/** fused operation support */
+	uint16_t		fuses;
+
+	/** format nvm attributes */
+	uint8_t			fna;
+
+	/** volatile write cache */
+	struct {
+		uint8_t		present : 1;
+		uint8_t		reserved : 7;
+	} vwc;
+
+	/** atomic write unit normal */
+	uint16_t		awun;
+
+	/** atomic write unit power fail */
+	uint16_t		awupf;
+
+	/** NVM vendor specific command configuration */
+	uint8_t			nvscc;
+
+	uint8_t			reserved531;
+
+	/** atomic compare & write unit */
+	uint16_t		acwu;
+
+	uint16_t		reserved534;
+
+	/** SGL support */
+	struct {
+		uint32_t	supported : 1;
+		uint32_t	reserved : 15;
+		uint32_t	bit_bucket_descriptor_supported : 1;
+		uint32_t	metadata_pointer_supported : 1;
+		uint32_t	oversized_sgl_supported : 1;
+	} sgls;
+
+	uint8_t			reserved4[164];
+
+	/* bytes 704-2047: i/o command set attributes */
+	uint8_t			reserved5[1344];
+
+	/* bytes 2048-3071: power state descriptors */
+	struct nvme_power_state	psd[32];
+
+	/* bytes 3072-4095: vendor specific */
+	uint8_t			vs[1024];
+};
+_Static_assert(sizeof(struct nvme_controller_data) == 4096, "Incorrect size");
+
+struct nvme_namespace_data {
+	/** namespace size */
+	uint64_t		nsze;
+
+	/** namespace capacity */
+	uint64_t		ncap;
+
+	/** namespace utilization */
+	uint64_t		nuse;
+
+	/** namespace features */
+	struct {
+		/** thin provisioning */
+		uint8_t		thin_prov : 1;
+		uint8_t		reserved1 : 7;
+	} nsfeat;
+
+	/** number of lba formats */
+	uint8_t			nlbaf;
+
+	/** formatted lba size */
+	struct {
+		uint8_t		format    : 4;
+		uint8_t		extended  : 1;
+		uint8_t		reserved2 : 3;
+	} flbas;
+
+	/** metadata capabilities */
+	struct {
+		/** metadata can be transferred as part of data prp list */
+		uint8_t		extended  : 1;
+
+		/** metadata can be transferred with separate metadata pointer */
+		uint8_t		pointer   : 1;
+
+		/** reserved */
+		uint8_t		reserved3 : 6;
+	} mc;
+
+	/** end-to-end data protection capabilities */
+	struct {
+		/** protection information type 1 */
+		uint8_t		pit1     : 1;
+
+		/** protection information type 2 */
+		uint8_t		pit2     : 1;
+
+		/** protection information type 3 */
+		uint8_t		pit3     : 1;
+
+		/** first eight bytes of metadata */
+		uint8_t		md_start : 1;
+
+		/** last eight bytes of metadata */
+		uint8_t		md_end   : 1;
+	} dpc;
+
+	/** end-to-end data protection type settings */
+	struct {
+		/** protection information type */
+		uint8_t		pit       : 3;
+
+		/** 1 == protection info transferred at start of metadata */
+		/** 0 == protection info transferred at end of metadata */
+		uint8_t		md_start  : 1;
+
+		uint8_t		reserved4 : 4;
+	} dps;
+
+	/** namespace multi-path I/O and namespace sharing capabilities */
+	struct {
+		uint8_t		can_share : 1;
+		uint8_t		reserved : 7;
+	} nmic;
+
+	/** reservation capabilities */
+	struct {
+		/** supports persist through power loss */
+		uint8_t		persist : 1;
+
+		/** supports write exclusive */
+		uint8_t		write_exclusive : 1;
+
+		/** supports exclusive access */
+		uint8_t		exclusive_access : 1;
+
+		/** supports write exclusive - registrants only */
+		uint8_t		write_exclusive_reg_only : 1;
+
+		/** supports exclusive access - registrants only */
+		uint8_t		exclusive_access_reg_only : 1;
+
+		/** supports write exclusive - all registrants */
+		uint8_t		write_exclusive_all_reg : 1;
+
+		/** supports exclusive access - all registrants */
+		uint8_t		exclusive_access_all_reg : 1;
+
+		uint8_t		reserved : 1;
+	} rescap;
+
+	/** format progress indicator */
+	uint8_t			fpi;
+
+	uint8_t			reserved33;
+
+	/** namespace atomic write unit normal */
+	uint16_t		nawun;
+
+	/** namespace atomic write unit power fail */
+	uint16_t		nawupf;
+
+	/** namespace atomic compare & write unit */
+	uint16_t		nacwu;
+
+	/** namespace atomic boundary size normal */
+	uint16_t		nabsn;
+
+	/** namespace atomic boundary offset */
+	uint16_t		nabo;
+
+	/** namespace atomic boundary size power fail */
+	uint16_t		nabspf;
+
+	uint16_t		reserved46;
+
+	/** NVM capacity */
+	uint64_t		nvmcap[2];
+
+	uint8_t			reserved64[40];
+
+	/** namespace globally unique identifier */
+	uint8_t			nguid[16];
+
+	/** IEEE extended unique identifier */
+	uint64_t		eui64;
+
+	/** lba format support */
+	struct {
+		/** metadata size */
+		uint32_t	ms	  : 16;
+
+		/** lba data size */
+		uint32_t	lbads	  : 8;
+
+		/** relative performance */
+		uint32_t	rp	  : 2;
+
+		uint32_t	reserved6 : 6;
+	} lbaf[16];
+
+	uint8_t			reserved6[192];
+
+	uint8_t			vendor_specific[3712];
+};
+_Static_assert(sizeof(struct nvme_namespace_data) == 4096, "Incorrect size");
+
+enum nvme_log_page {
+	/* 0x00 - reserved */
+	NVME_LOG_ERROR			= 0x01,
+	NVME_LOG_HEALTH_INFORMATION	= 0x02,
+	NVME_LOG_FIRMWARE_SLOT		= 0x03,
+	/* 0x04-0x7F - reserved */
+	/* 0x80-0xBF - I/O command set specific */
+	/* 0xC0-0xFF - vendor specific */
+};
+
+struct nvme_error_information_entry {
+	uint64_t		error_count;
+	uint16_t		sqid;
+	uint16_t		cid;
+	struct nvme_status	status;
+	uint16_t		error_location;
+	uint64_t		lba;
+	uint32_t		nsid;
+	uint8_t			vendor_specific;
+	uint8_t			reserved[35];
+};
+_Static_assert(sizeof(struct nvme_error_information_entry) == 64, "Incorrect size");
+
+union nvme_critical_warning_state {
+	uint8_t		raw;
+
+	struct {
+		uint8_t	available_spare		: 1;
+		uint8_t	temperature		: 1;
+		uint8_t	device_reliability	: 1;
+		uint8_t	read_only		: 1;
+		uint8_t	volatile_memory_backup	: 1;
+		uint8_t	reserved		: 3;
+	} bits;
+};
+_Static_assert(sizeof(union nvme_critical_warning_state) == 1, "Incorrect size");
+
+struct __attribute__((packed)) nvme_health_information_page {
+	union nvme_critical_warning_state	critical_warning;
+
+	uint16_t		temperature;
+	uint8_t			available_spare;
+	uint8_t			available_spare_threshold;
+	uint8_t			percentage_used;
+
+	uint8_t			reserved[26];
+
+	/*
+	 * Note that the following are 128-bit values, but are
+	 *  defined as an array of 2 64-bit values.
+	 */
+	/* Data Units Read is always in 512-byte units. */
+	uint64_t		data_units_read[2];
+	/* Data Units Written is always in 512-byte units. */
+	uint64_t		data_units_written[2];
+	/* For NVM command set, this includes Compare commands. */
+	uint64_t		host_read_commands[2];
+	uint64_t		host_write_commands[2];
+	/* Controller Busy Time is reported in minutes. */
+	uint64_t		controller_busy_time[2];
+	uint64_t		power_cycles[2];
+	uint64_t		power_on_hours[2];
+	uint64_t		unsafe_shutdowns[2];
+	uint64_t		media_errors[2];
+	uint64_t		num_error_info_log_entries[2];
+
+	uint8_t			reserved2[320];
+};
+_Static_assert(sizeof(struct nvme_health_information_page) == 512, "Incorrect size");
+
+struct nvme_firmware_page {
+	struct {
+		uint8_t	slot		: 3; /* slot for current FW */
+		uint8_t	reserved	: 5;
+	} afi;
+
+	uint8_t			reserved[7];
+	uint64_t		revision[7]; /* revisions for 7 slots */
+	uint8_t			reserved2[448];
+};
+_Static_assert(sizeof(struct nvme_firmware_page) == 512, "Incorrect size");
+
+#define nvme_completion_is_error(cpl)					\
+	((cpl)->status.sc != 0 || (cpl)->status.sct != 0)
+
+#endif

include/spdk/pci.h

@@ -0,0 +1,48 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef SPDK_PCI_H
+#define SPDK_PCI_H
+
+#define PCI_CFG_SIZE		256
+#define PCI_EXT_CAP_ID_SN	0x03
+#define PCI_UIO_DRIVER		"uio_pci_generic"
+
+int pci_device_get_serial_number(struct pci_device *dev, char *sn, int len);
+int pci_device_has_uio_driver(struct pci_device *dev);
+int pci_device_unbind_kernel_driver(struct pci_device *dev);
+int pci_device_bind_uio_driver(struct pci_device *dev, char *driver_name);
+int pci_device_switch_to_uio_driver(struct pci_device *pci_dev);
+int pci_device_claim(struct pci_device *dev);
+
+#endif

include/spdk/queue.h

@@ -0,0 +1,344 @@
+/*-
+ * Copyright (c) 1991, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)queue.h	8.5 (Berkeley) 8/20/94
+ * $FreeBSD$
+ */
+
+#ifndef SPDK_QUEUE_H
+#define SPDK_QUEUE_H
+
+#include <sys/cdefs.h>
+#include <sys/queue.h>
+
+/*
+ * This file defines four types of data structures: singly-linked lists,
+ * singly-linked tail queues, lists and tail queues.
+ *
+ * A singly-linked list is headed by a single forward pointer. The elements
+ * are singly linked for minimum space and pointer manipulation overhead at
+ * the expense of O(n) removal for arbitrary elements. New elements can be
+ * added to the list after an existing element or at the head of the list.
+ * Elements being removed from the head of the list should use the explicit
+ * macro for this purpose for optimum efficiency. A singly-linked list may
+ * only be traversed in the forward direction.  Singly-linked lists are ideal
+ * for applications with large datasets and few or no removals or for
+ * implementing a LIFO queue.
+ *
+ * A singly-linked tail queue is headed by a pair of pointers, one to the
+ * head of the list and the other to the tail of the list. The elements are
+ * singly linked for minimum space and pointer manipulation overhead at the
+ * expense of O(n) removal for arbitrary elements. New elements can be added
+ * to the list after an existing element, at the head of the list, or at the
+ * end of the list. Elements being removed from the head of the tail queue
+ * should use the explicit macro for this purpose for optimum efficiency.
+ * A singly-linked tail queue may only be traversed in the forward direction.
+ * Singly-linked tail queues are ideal for applications with large datasets
+ * and few or no removals or for implementing a FIFO queue.
+ *
+ * A list is headed by a single forward pointer (or an array of forward
+ * pointers for a hash table header). The elements are doubly linked
+ * so that an arbitrary element can be removed without a need to
+ * traverse the list. New elements can be added to the list before
+ * or after an existing element or at the head of the list. A list
+ * may be traversed in either direction.
+ *
+ * A tail queue is headed by a pair of pointers, one to the head of the
+ * list and the other to the tail of the list. The elements are doubly
+ * linked so that an arbitrary element can be removed without a need to
+ * traverse the list. New elements can be added to the list before or
+ * after an existing element, at the head of the list, or at the end of
+ * the list. A tail queue may be traversed in either direction.
+ *
+ * For details on the use of these macros, see the queue(3) manual page.
+ *
+ *
+ *				SLIST	LIST	STAILQ	TAILQ
+ * _HEAD			+	+	+	+
+ * _HEAD_INITIALIZER		+	+	+	+
+ * _ENTRY			+	+	+	+
+ * _INIT			+	+	+	+
+ * _EMPTY			+	+	+	+
+ * _FIRST			+	+	+	+
+ * _NEXT			+	+	+	+
+ * _PREV			-	+	-	+
+ * _LAST			-	-	+	+
+ * _FOREACH			+	+	+	+
+ * _FOREACH_FROM		+	+	+	+
+ * _FOREACH_SAFE		+	+	+	+
+ * _FOREACH_FROM_SAFE		+	+	+	+
+ * _FOREACH_REVERSE		-	-	-	+
+ * _FOREACH_REVERSE_FROM	-	-	-	+
+ * _FOREACH_REVERSE_SAFE	-	-	-	+
+ * _FOREACH_REVERSE_FROM_SAFE	-	-	-	+
+ * _INSERT_HEAD			+	+	+	+
+ * _INSERT_BEFORE		-	+	-	+
+ * _INSERT_AFTER		+	+	+	+
+ * _INSERT_TAIL			-	-	+	+
+ * _CONCAT			-	-	+	+
+ * _REMOVE_AFTER		+	-	+	-
+ * _REMOVE_HEAD			+	-	+	-
+ * _REMOVE			+	+	+	+
+ * _SWAP			+	+	+	+
+ *
+ */
+
+/*
+ * Singly-linked Tail queue declarations.
+ */
+#define	STAILQ_HEAD(name, type)						\
+struct name {								\
+	struct type *stqh_first;/* first element */			\
+	struct type **stqh_last;/* addr of last next element */		\
+}
+
+#define	STAILQ_HEAD_INITIALIZER(head)					\
+	{ NULL, &(head).stqh_first }
+
+/*
+ * Singly-linked Tail queue functions.
+ */
+#define	STAILQ_EMPTY(head)	((head)->stqh_first == NULL)
+
+#define	STAILQ_FIRST(head)	((head)->stqh_first)
+
+#define	STAILQ_FOREACH_FROM(var, head, field)				\
+	for ((var) = ((var) ? (var) : STAILQ_FIRST((head)));		\
+	   (var);							\
+	   (var) = STAILQ_NEXT((var), field))
+
+#define	STAILQ_FOREACH_SAFE(var, head, field, tvar)			\
+	for ((var) = STAILQ_FIRST((head));				\
+	    (var) && ((tvar) = STAILQ_NEXT((var), field), 1);		\
+	    (var) = (tvar))
+
+#define	STAILQ_FOREACH_FROM_SAFE(var, head, field, tvar)		\
+	for ((var) = ((var) ? (var) : STAILQ_FIRST((head)));		\
+	    (var) && ((tvar) = STAILQ_NEXT((var), field), 1);		\
+	    (var) = (tvar))
+
+#define	STAILQ_LAST(head, type, field)					\
+	(STAILQ_EMPTY((head)) ? NULL :					\
+	    __containerof((head)->stqh_last, struct type, field.stqe_next))
+
+#define	STAILQ_NEXT(elm, field)	((elm)->field.stqe_next)
+
+#define STAILQ_REMOVE_AFTER(head, elm, field) do {			\
+	if ((STAILQ_NEXT(elm, field) =					\
+	     STAILQ_NEXT(STAILQ_NEXT(elm, field), field)) == NULL)	\
+		(head)->stqh_last = &STAILQ_NEXT((elm), field);		\
+} while (0)
+
+#define STAILQ_SWAP(head1, head2, type) do {				\
+	struct type *swap_first = STAILQ_FIRST(head1);			\
+	struct type **swap_last = (head1)->stqh_last;			\
+	STAILQ_FIRST(head1) = STAILQ_FIRST(head2);			\
+	(head1)->stqh_last = (head2)->stqh_last;			\
+	STAILQ_FIRST(head2) = swap_first;				\
+	(head2)->stqh_last = swap_last;					\
+	if (STAILQ_EMPTY(head1))					\
+		(head1)->stqh_last = &STAILQ_FIRST(head1);		\
+	if (STAILQ_EMPTY(head2))					\
+		(head2)->stqh_last = &STAILQ_FIRST(head2);		\
+} while (0)
+
+/*
+ * List declarations.
+ */
+#define	LIST_HEAD(name, type)						\
+struct name {								\
+	struct type *lh_first;	/* first element */			\
+}
+
+#define	LIST_HEAD_INITIALIZER(head)					\
+	{ NULL }
+
+#define	LIST_ENTRY(type)						\
+struct {								\
+	struct type *le_next;	/* next element */			\
+	struct type **le_prev;	/* address of previous next element */	\
+}
+
+/*
+ * List functions.
+ */
+
+#if (defined(_KERNEL) && defined(INVARIANTS))
+#define	QMD_LIST_CHECK_HEAD(head, field) do {				\
+	if (LIST_FIRST((head)) != NULL &&				\
+	    LIST_FIRST((head))->field.le_prev !=			\
+	     &LIST_FIRST((head)))					\
+		panic("Bad list head %p first->prev != head", (head));	\
+} while (0)
+
+#define	QMD_LIST_CHECK_NEXT(elm, field) do {				\
+	if (LIST_NEXT((elm), field) != NULL &&				\
+	    LIST_NEXT((elm), field)->field.le_prev !=			\
+	     &((elm)->field.le_next))					\
+	     	panic("Bad link elm %p next->prev != elm", (elm));	\
+} while (0)
+
+#define	QMD_LIST_CHECK_PREV(elm, field) do {				\
+	if (*(elm)->field.le_prev != (elm))				\
+		panic("Bad link elm %p prev->next != elm", (elm));	\
+} while (0)
+#else
+#define	QMD_LIST_CHECK_HEAD(head, field)
+#define	QMD_LIST_CHECK_NEXT(elm, field)
+#define	QMD_LIST_CHECK_PREV(elm, field)
+#endif /* (_KERNEL && INVARIANTS) */
+
+#define	LIST_EMPTY(head)	((head)->lh_first == NULL)
+
+#define	LIST_FIRST(head)	((head)->lh_first)
+
+#define	LIST_FOREACH_FROM(var, head, field)				\
+	for ((var) = ((var) ? (var) : LIST_FIRST((head)));		\
+	    (var);							\
+	    (var) = LIST_NEXT((var), field))
+
+#define	LIST_FOREACH_SAFE(var, head, field, tvar)			\
+	for ((var) = LIST_FIRST((head));				\
+	    (var) && ((tvar) = LIST_NEXT((var), field), 1);		\
+	    (var) = (tvar))
+
+#define	LIST_FOREACH_FROM_SAFE(var, head, field, tvar)			\
+	for ((var) = ((var) ? (var) : LIST_FIRST((head)));		\
+	    (var) && ((tvar) = LIST_NEXT((var), field), 1);		\
+	    (var) = (tvar))
+
+#define	LIST_NEXT(elm, field)	((elm)->field.le_next)
+
+#define	LIST_PREV(elm, head, type, field)				\
+	((elm)->field.le_prev == &LIST_FIRST((head)) ? NULL :		\
+	    __containerof((elm)->field.le_prev, struct type, field.le_next))
+
+#define LIST_SWAP(head1, head2, type, field) do {			\
+	struct type *swap_tmp = LIST_FIRST((head1));			\
+	LIST_FIRST((head1)) = LIST_FIRST((head2));			\
+	LIST_FIRST((head2)) = swap_tmp;					\
+	if ((swap_tmp = LIST_FIRST((head1))) != NULL)			\
+		swap_tmp->field.le_prev = &LIST_FIRST((head1));		\
+	if ((swap_tmp = LIST_FIRST((head2))) != NULL)			\
+		swap_tmp->field.le_prev = &LIST_FIRST((head2));		\
+} while (0)
+
+/*
+ * Tail queue functions.
+ */
+#if (defined(_KERNEL) && defined(INVARIANTS))
+#define	QMD_TAILQ_CHECK_HEAD(head, field) do {				\
+	if (!TAILQ_EMPTY(head) &&					\
+	    TAILQ_FIRST((head))->field.tqe_prev !=			\
+	     &TAILQ_FIRST((head)))					\
+		panic("Bad tailq head %p first->prev != head", (head));	\
+} while (0)
+
+#define	QMD_TAILQ_CHECK_TAIL(head, field) do {				\
+	if (*(head)->tqh_last != NULL)					\
+	    	panic("Bad tailq NEXT(%p->tqh_last) != NULL", (head)); 	\
+} while (0)
+
+#define	QMD_TAILQ_CHECK_NEXT(elm, field) do {				\
+	if (TAILQ_NEXT((elm), field) != NULL &&				\
+	    TAILQ_NEXT((elm), field)->field.tqe_prev !=			\
+	     &((elm)->field.tqe_next))					\
+		panic("Bad link elm %p next->prev != elm", (elm));	\
+} while (0)
+
+#define	QMD_TAILQ_CHECK_PREV(elm, field) do {				\
+	if (*(elm)->field.tqe_prev != (elm))				\
+		panic("Bad link elm %p prev->next != elm", (elm));	\
+} while (0)
+#else
+#define	QMD_TAILQ_CHECK_HEAD(head, field)
+#define	QMD_TAILQ_CHECK_TAIL(head, headname)
+#define	QMD_TAILQ_CHECK_NEXT(elm, field)
+#define	QMD_TAILQ_CHECK_PREV(elm, field)
+#endif /* (_KERNEL && INVARIANTS) */
+
+#define	TAILQ_EMPTY(head)	((head)->tqh_first == NULL)
+
+#define	TAILQ_FIRST(head)	((head)->tqh_first)
+
+#define	TAILQ_FOREACH_FROM(var, head, field)				\
+	for ((var) = ((var) ? (var) : TAILQ_FIRST((head)));		\
+	    (var);							\
+	    (var) = TAILQ_NEXT((var), field))
+
+#define	TAILQ_FOREACH_SAFE(var, head, field, tvar)			\
+	for ((var) = TAILQ_FIRST((head));				\
+	    (var) && ((tvar) = TAILQ_NEXT((var), field), 1);		\
+	    (var) = (tvar))
+
+#define	TAILQ_FOREACH_FROM_SAFE(var, head, field, tvar)			\
+	for ((var) = ((var) ? (var) : TAILQ_FIRST((head)));		\
+	    (var) && ((tvar) = TAILQ_NEXT((var), field), 1);		\
+	    (var) = (tvar))
+
+#define	TAILQ_FOREACH_REVERSE_FROM(var, head, headname, field)		\
+	for ((var) = ((var) ? (var) : TAILQ_LAST((head), headname));	\
+	    (var);							\
+	    (var) = TAILQ_PREV((var), headname, field))
+
+#define	TAILQ_FOREACH_REVERSE_SAFE(var, head, headname, field, tvar)	\
+	for ((var) = TAILQ_LAST((head), headname);			\
+	    (var) && ((tvar) = TAILQ_PREV((var), headname, field), 1);	\
+	    (var) = (tvar))
+
+#define	TAILQ_FOREACH_REVERSE_FROM_SAFE(var, head, headname, field, tvar) \
+	for ((var) = ((var) ? (var) : TAILQ_LAST((head), headname));	\
+	    (var) && ((tvar) = TAILQ_PREV((var), headname, field), 1);	\
+	    (var) = (tvar))
+
+#define	TAILQ_LAST(head, headname)					\
+	(*(((struct headname *)((head)->tqh_last))->tqh_last))
+
+#define	TAILQ_NEXT(elm, field) ((elm)->field.tqe_next)
+
+#define	TAILQ_PREV(elm, headname, field)				\
+	(*(((struct headname *)((elm)->field.tqe_prev))->tqh_last))
+
+#define TAILQ_SWAP(head1, head2, type, field) do {			\
+	struct type *swap_first = (head1)->tqh_first;			\
+	struct type **swap_last = (head1)->tqh_last;			\
+	(head1)->tqh_first = (head2)->tqh_first;			\
+	(head1)->tqh_last = (head2)->tqh_last;				\
+	(head2)->tqh_first = swap_first;				\
+	(head2)->tqh_last = swap_last;					\
+	if ((swap_first = (head1)->tqh_first) != NULL)			\
+		swap_first->field.tqe_prev = &(head1)->tqh_first;	\
+	else								\
+		(head1)->tqh_last = &(head1)->tqh_first;		\
+	if ((swap_first = (head2)->tqh_first) != NULL)			\
+		swap_first->field.tqe_prev = &(head2)->tqh_first;	\
+	else								\
+		(head2)->tqh_last = &(head2)->tqh_first;		\
+} while (0)
+
+#endif

include/spdk/vtophys.h

@@ -0,0 +1,51 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef SPDK_VTOPHYS_H
+#define SPDK_VTOPHYS_H
+
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define VTOPHYS_ERROR	(0xFFFFFFFFFFFFFFFFULL)
+
+uint64_t vtophys(void *buf);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

lib/Makefile

@@ -0,0 +1,43 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+DIRS-y += memory util nvme
+
+.PHONY: all clean $(DIRS-y)
+
+all: $(DIRS-y)
+clean: $(DIRS-y)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk

lib/memory/Makefile

@@ -0,0 +1,55 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+CFLAGS += $(DPDK_INC)
+
+C_SRCS = vtophys.c
+
+C_OBJS := $(patsubst %.c,%.o,$(C_SRCS))
+
+OBJS = $(C_OBJS)
+
+LIB = libspdk_memory.a
+
+all : $(LIB)
+
+objs : $(OBJS)
+
+clean :
+	$(Q)rm -f $(LIB) $(OBJS) *.d
+
+$(LIB) : $(OBJS)
+	$(Q)ar crDs $(LIB) $(OBJS)
+

lib/memory/vtophys.c

@@ -0,0 +1,229 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#define _LARGEFILE64_SOURCE
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/user.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <string.h>
+#include <pthread.h>
+
+#include "spdk/vtophys.h"
+
+/* x86-64 userspace virtual addresses use only the low 47 bits [0..46],
+ * which is enough to cover 128 TB.
+ */
+#define SHIFT_128TB	47 /* (1 << 47) == 128 TB */
+#define MASK_128TB	((1ULL << SHIFT_128TB) - 1)
+
+#define SHIFT_1GB	30 /* (1 << 30) == 1 GB */
+#define MASK_1GB	((1ULL << SHIFT_1GB) - 1)
+
+#define SHIFT_2MB	21 /* (1 << 21) == 2MB */
+#define MASK_2MB	((1ULL << SHIFT_2MB) - 1)
+
+#define SHIFT_4KB	12 /* (1 << 12) == 4KB */
+#define MASK_4KB	((1ULL << SHIFT_4KB) - 1)
+
+#define FN_2MB_TO_4KB(fn)	(fn << (SHIFT_2MB - SHIFT_4KB))
+#define FN_4KB_TO_2MB(fn)	(fn >> (SHIFT_2MB - SHIFT_4KB))
+
+#define MAP_128TB_IDX(vfn_2mb)	((vfn_2mb) >> (SHIFT_1GB - SHIFT_2MB))
+#define MAP_1GB_IDX(vfn_2mb)	((vfn_2mb) & ((1ULL << (SHIFT_1GB - SHIFT_2MB + 1)) - 1))
+
+/* Defines related to Linux pagemap. */
+#define PAGEMAP_PFN_MASK	0x007FFFFFFFFFFFFFULL /* bits 54:0 */
+
+/* Defines related to Linux kpageflags. */
+#define KPAGEFLAGS_HUGE		17
+
+/* Physical page frame number of a single 2MB page. */
+struct map_2mb {
+	uint64_t pfn_2mb;
+};
+
+/* Second-level map table indexed by bits [21..29] of the virtual address.
+ * Each entry contains the 2MB physical page frame number or VTOPHYS_ERROR for entries that haven't
+ * been retrieved yet.
+ */
+struct map_1gb {
+	struct map_2mb map[1ULL << (SHIFT_1GB - SHIFT_2MB + 1)];
+};
+
+/* Top-level map table indexed by bits [30..46] of the virtual address.
+ * Each entry points to a second-level map table or NULL.
+ */
+struct map_128tb {
+	struct map_1gb *map[1ULL << (SHIFT_128TB - SHIFT_1GB + 1)];
+};
+
+static struct map_128tb vtophys_map_128tb = {};
+static pthread_mutex_t vtophys_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+static struct map_2mb *
+vtophys_get_map(uint64_t vfn_2mb)
+{
+	struct map_1gb *map_1gb;
+	struct map_2mb *map_2mb;
+	uint64_t idx_128tb = MAP_128TB_IDX(vfn_2mb);
+	uint64_t idx_1gb = MAP_1GB_IDX(vfn_2mb);
+
+	if (vfn_2mb & ~MASK_128TB) {
+		printf("invalid usermode virtual address\n");
+		return NULL;
+	}
+
+	map_1gb = vtophys_map_128tb.map[idx_128tb];
+
+	if (!map_1gb) {
+		pthread_mutex_lock(&vtophys_mutex);
+
+		/* Recheck to make sure nobody else got the mutex first. */
+		map_1gb = vtophys_map_128tb.map[idx_128tb];
+		if (!map_1gb) {
+			map_1gb = malloc(sizeof(struct map_1gb));
+			/* initialize all entries to all 0xFF (VTOPHYS_ERROR) */
+			memset(map_1gb, 0xFF, sizeof(struct map_1gb));
+			vtophys_map_128tb.map[idx_128tb] = map_1gb;
+		}
+
+		pthread_mutex_unlock(&vtophys_mutex);
+
+		if (!map_1gb) {
+			printf("allocation failed\n");
+			return NULL;
+		}
+	}
+
+	map_2mb = &map_1gb->map[idx_1gb];
+	return map_2mb;
+}
+
+uint64_t
+vtophys_get_pfn_2mb(uint64_t vfn_2mb)
+{
+	off64_t offset;
+	uint64_t pfn_4kb, pfn_2mb, pfn_flag_info;
+	static __thread int fd = -1;
+	static __thread int kpageflag_fd = -1;
+
+	if (fd == -1) {
+		fd = open("/proc/self/pagemap", O_RDONLY);
+		if (fd == -1) {
+			perror("pagemap open failed");
+			return VTOPHYS_ERROR;
+		}
+	}
+
+	/*
+	 * pagemap has a separate 8 byte entry for each 4KB
+	 * page.  So even though we are only storing a single
+	 * PFN for every 2MB VFN, we have to get the 4KB
+	 * VFN equivalent to lseek into the pagemap.
+	 *
+	 * vfn contains the 2MB virtual frame number, but we
+	 * we need the 4KB equivalent to lseek into the pagemap.
+	 * So do a left shift to convert the 2MB vfn to the
+	 * 4KB vfn equivalent.
+	 */
+	offset = FN_2MB_TO_4KB(vfn_2mb) * sizeof(uint64_t);
+
+	if (lseek64(fd, offset, SEEK_SET) < 0) {
+		perror("pagemap llseek failed");
+		return VTOPHYS_ERROR;
+	}
+
+	if (read(fd, &pfn_4kb, sizeof(pfn_4kb)) != sizeof(pfn_4kb)) {
+		perror("pagemap read failed");
+		return VTOPHYS_ERROR;
+	}
+
+	pfn_4kb &= PAGEMAP_PFN_MASK;
+
+	if (kpageflag_fd == -1) {
+		kpageflag_fd = open("/proc/kpageflags", O_RDONLY);
+		if (kpageflag_fd == -1) {
+			perror("kpageflags open failed");
+			return VTOPHYS_ERROR;
+		}
+	}
+
+	offset = pfn_4kb * sizeof(uint64_t);
+	if (lseek64(kpageflag_fd, offset, SEEK_SET) < 0) {
+		perror("kpageflags llseek failed");
+		return VTOPHYS_ERROR;
+	}
+
+	if (read(kpageflag_fd, &pfn_flag_info, sizeof(pfn_flag_info)) != sizeof(pfn_flag_info)) {
+		perror("kpageflags read failed");
+		return VTOPHYS_ERROR;
+	}
+
+	/* check whether it is a huge page */
+	if (!(pfn_flag_info & (1ULL << KPAGEFLAGS_HUGE))) {
+		printf("This is not a huge page\n");
+		return VTOPHYS_ERROR;
+	}
+
+	pfn_2mb = FN_4KB_TO_2MB(pfn_4kb);
+	return pfn_2mb;
+}
+
+uint64_t
+vtophys(void *buf)
+{
+	struct map_2mb *map_2mb;
+	uint64_t vfn_2mb, pfn_2mb;
+
+	vfn_2mb = (uint64_t)buf;
+	vfn_2mb >>= SHIFT_2MB;
+
+	map_2mb = vtophys_get_map(vfn_2mb);
+	if (!map_2mb) {
+		return VTOPHYS_ERROR;
+	}
+
+	pfn_2mb = map_2mb->pfn_2mb;
+	if (pfn_2mb == VTOPHYS_ERROR) {
+		pfn_2mb = vtophys_get_pfn_2mb(vfn_2mb);
+		if (pfn_2mb == VTOPHYS_ERROR) {
+			return VTOPHYS_ERROR;
+		}
+		map_2mb->pfn_2mb = pfn_2mb;
+	}
+
+	return (pfn_2mb << SHIFT_2MB) | ((uint64_t)buf & MASK_2MB);
+}

lib/nvme/Makefile

@@ -0,0 +1,53 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+CFLAGS += $(DPDK_INC) -include $(CONFIG_NVME_IMPL)
+
+C_SRCS = nvme_ctrlr_cmd.c nvme_ctrlr.c nvme_ns_cmd.c nvme_ns.c nvme_qpair.c nvme.c
+
+C_OBJS := $(patsubst %.c,%.o,$(C_SRCS))
+
+OBJS = $(C_OBJS)
+
+all : libspdk_nvme.a
+
+objs : $(OBJS)
+
+clean :
+	$(Q)rm -f libspdk_nvme.a $(OBJS) *.d
+
+libspdk_nvme.a : $(OBJS)
+	$(Q)ar crDs libspdk_nvme.a $(OBJS)
+

lib/nvme/nvme.c

@@ -0,0 +1,294 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "nvme_internal.h"
+
+/** \file
+ *
+ */
+
+struct nvme_driver g_nvme_driver = {
+	.lock = NVME_MUTEX_INITIALIZER,
+	.max_io_queues = NVME_MAX_IO_QUEUES
+};
+
+int32_t		nvme_retry_count;
+int __thread	nvme_thread_ioq_index = -1;
+
+
+void
+nvme_dump_command(struct nvme_command *cmd)
+{
+	printf(
+		"opc:%x f:%x r1:%x cid:%x nsid:%x r2:%x r3:%x mptr:%jx prp1:%jx prp2:%jx cdw:%x %x %x %x %x %x\n",
+		cmd->opc, cmd->fuse, cmd->rsvd1, cmd->cid, cmd->nsid,
+		cmd->rsvd2, cmd->rsvd3,
+		(uintmax_t)cmd->mptr, (uintmax_t)cmd->dptr.prp.prp1, (uintmax_t)cmd->dptr.prp.prp2,
+		cmd->cdw10, cmd->cdw11, cmd->cdw12, cmd->cdw13, cmd->cdw14,
+		cmd->cdw15);
+}
+
+void
+nvme_dump_completion(struct nvme_completion *cpl)
+{
+	printf("cdw0:%08x sqhd:%04x sqid:%04x "
+	       "cid:%04x p:%x sc:%02x sct:%x m:%x dnr:%x\n",
+	       cpl->cdw0, cpl->sqhd, cpl->sqid,
+	       cpl->cid, cpl->status.p, cpl->status.sc, cpl->status.sct,
+	       cpl->status.m, cpl->status.dnr);
+}
+
+/**
+ * \page nvme_initialization NVMe Initialization
+
+\msc
+
+	app [label="Application"], nvme [label="NVMe Driver"];
+	app=>nvme [label="nvme_attach(devhandle)"];
+	app<<nvme [label="nvme_controller ptr"];
+	app=>nvme [label="nvme_ctrlr_start(nvme_controller ptr)"];
+	nvme=>nvme [label="identify controller"];
+	nvme=>nvme [label="create queue pairs"];
+	nvme=>nvme [label="identify namespace(s)"];
+	app=>app [label="create block devices based on controller's namespaces"];
+
+\endmsc
+
+ */
+
+struct nvme_controller *
+nvme_attach(void *devhandle)
+{
+	struct nvme_controller	*ctrlr;
+	int			status;
+	uint64_t		phys_addr = 0;
+
+	ctrlr = nvme_malloc("nvme_ctrlr", sizeof(struct nvme_controller),
+			    64, &phys_addr);
+	if (ctrlr == NULL) {
+		nvme_printf(NULL, "could not allocate ctrlr\n");
+		return NULL;
+	}
+
+	status = nvme_ctrlr_construct(ctrlr, devhandle);
+	if (status != 0) {
+		nvme_free(ctrlr);
+		return NULL;
+	}
+
+	if (nvme_ctrlr_start(ctrlr) != 0) {
+		nvme_ctrlr_destruct(ctrlr);
+		nvme_free(ctrlr);
+		return NULL;
+	}
+
+	return ctrlr;
+}
+
+int
+nvme_detach(struct nvme_controller *ctrlr)
+{
+	nvme_ctrlr_destruct(ctrlr);
+	nvme_free(ctrlr);
+	return 0;
+}
+
+void
+nvme_completion_poll_cb(void *arg, const struct nvme_completion *cpl)
+{
+	struct nvme_completion_poll_status	*status = arg;
+
+	/*
+	 * Copy status into the argument passed by the caller, so that
+	 *  the caller can check the status to determine if the
+	 *  the request passed or failed.
+	 */
+	memcpy(&status->cpl, cpl, sizeof(*cpl));
+	status->done = true;
+}
+
+size_t
+nvme_request_size(void)
+{
+	return sizeof(struct nvme_request);
+}
+
+struct nvme_request *
+nvme_allocate_request(void *payload, uint32_t payload_size,
+		      nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+	struct nvme_request *req = NULL;
+
+	nvme_alloc_request(&req);
+
+	if (req == NULL) {
+		return req;
+	}
+
+	/*
+	 * Only memset up to (but not including) the children
+	 *  TAILQ_ENTRY.  children, and following members, are
+	 *  only used as part of I/O splitting so we avoid
+	 *  memsetting them until it is actually needed.
+	 */
+	memset(req, 0, offsetof(struct nvme_request, children));
+	req->cb_fn = cb_fn;
+	req->cb_arg = cb_arg;
+	req->timeout = true;
+	nvme_assert((payload == NULL && payload_size == 0) ||
+		    (payload != NULL && payload_size != 0),
+		    ("Invalid argument combination of payload and payload_size\n"));
+	if (payload == NULL || payload_size == 0) {
+		req->u.payload = NULL;
+		req->payload_size = 0;
+	} else {
+		req->u.payload = payload;
+		req->payload_size = payload_size;
+	}
+
+	return req;
+}
+
+void
+nvme_cb_complete_child(void *child_arg, const struct nvme_completion *cpl)
+{
+	struct nvme_request *child = child_arg;
+	struct nvme_request *parent = child->parent;
+
+	parent->num_children--;
+	TAILQ_REMOVE(&parent->children, child, child_tailq);
+
+	if (nvme_completion_is_error(cpl)) {
+		memcpy(&parent->parent_status, cpl, sizeof(*cpl));
+	}
+
+	if (parent->num_children == 0) {
+		if (parent->cb_fn) {
+			parent->cb_fn(parent->cb_arg, &parent->parent_status);
+		}
+		nvme_free_request(parent);
+	}
+}
+
+void
+nvme_request_add_child(struct nvme_request *parent, struct nvme_request *child)
+{
+	if (parent->num_children == 0) {
+		/*
+		 * Defer initialization of the children TAILQ since it falls
+		 *  on a separate cacheline.  This ensures we do not touch this
+		 *  cacheline except on request splitting cases, which are
+		 *  relatively rare.
+		 */
+		TAILQ_INIT(&parent->children);
+		memset(&parent->parent_status, 0, sizeof(struct nvme_completion));
+	}
+
+	parent->num_children++;
+	TAILQ_INSERT_TAIL(&parent->children, child, child_tailq);
+	child->parent = parent;
+	child->cb_fn = nvme_cb_complete_child;
+	child->cb_arg = child;
+}
+
+static int
+nvme_allocate_ioq_index(void)
+{
+	struct nvme_driver	*driver = &g_nvme_driver;
+	uint32_t		i;
+
+	nvme_mutex_lock(&driver->lock);
+	if (driver->ioq_index_pool == NULL) {
+		driver->ioq_index_pool =
+			calloc(driver->max_io_queues, sizeof(*driver->ioq_index_pool));
+		if (driver->ioq_index_pool) {
+			for (i = 0; i < driver->max_io_queues; i++) {
+				driver->ioq_index_pool[i] = i;
+			}
+		} else {
+			nvme_mutex_unlock(&driver->lock);
+			return -1;
+		}
+		driver->ioq_index_pool_next = 0;
+	}
+
+	if (driver->ioq_index_pool_next < driver->max_io_queues) {
+		nvme_thread_ioq_index = driver->ioq_index_pool[driver->ioq_index_pool_next];
+		driver->ioq_index_pool[driver->ioq_index_pool_next] = -1;
+		driver->ioq_index_pool_next++;
+	} else {
+		nvme_thread_ioq_index = -1;
+	}
+
+	nvme_mutex_unlock(&driver->lock);
+	return 0;
+}
+
+static void
+nvme_free_ioq_index(void)
+{
+	struct nvme_driver	*driver = &g_nvme_driver;
+
+	nvme_mutex_lock(&driver->lock);
+	if (nvme_thread_ioq_index >= 0) {
+		driver->ioq_index_pool_next--;
+		driver->ioq_index_pool[driver->ioq_index_pool_next] = nvme_thread_ioq_index;
+		nvme_thread_ioq_index = -1;
+	}
+	nvme_mutex_unlock(&driver->lock);
+}
+
+int
+nvme_register_io_thread(void)
+{
+	int rc = 0;
+
+	if (nvme_thread_ioq_index >= 0) {
+		nvme_printf(NULL, "thread already registered\n");
+		return -1;
+	}
+
+	rc = nvme_allocate_ioq_index();
+	if (rc) {
+		nvme_printf(NULL, "ioq_index_pool alloc failed\n");
+		return rc;
+	}
+	return (nvme_thread_ioq_index >= 0) ? 0 : -1;
+}
+
+void
+nvme_unregister_io_thread(void)
+{
+	nvme_free_ioq_index();
+}
+

lib/nvme/nvme_ctrlr.c

@@ -0,0 +1,787 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "nvme_internal.h"
+
+/**
+ * \file
+ *
+ */
+
+static void nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,
+		struct nvme_async_event_request *aer);
+
+static int
+nvme_ctrlr_construct_admin_qpair(struct nvme_controller *ctrlr)
+{
+	struct nvme_qpair	*qpair;
+	int rc;
+
+	qpair = &ctrlr->adminq;
+
+	/*
+	 * The admin queue's max xfer size is treated differently than the
+	 *  max I/O xfer size.  16KB is sufficient here - maybe even less?
+	 */
+	rc = nvme_qpair_construct(qpair,
+				  0, /* qpair ID */
+				  NVME_ADMIN_ENTRIES,
+				  NVME_ADMIN_TRACKERS,
+				  ctrlr);
+	return rc;
+}
+
+static int
+nvme_ctrlr_construct_io_qpairs(struct nvme_controller *ctrlr)
+{
+	struct nvme_qpair		*qpair;
+	union nvme_cap_lo_register	cap_lo;
+	int				i, num_entries, num_trackers, rc;
+
+	rc = 0;
+	if (ctrlr->ioq != NULL) {
+		/*
+		 * io_qpairs were already constructed, so just return.
+		 *  This typically happens when the controller is
+		 *  initialized a second (or subsequent) time after a
+		 *  controller reset.
+		 */
+		return 0;
+	}
+
+	/*
+	 * NVMe spec sets a hard limit of 64K max entries, but
+	 *  devices may specify a smaller limit, so we need to check
+	 *  the MQES field in the capabilities register.
+	 */
+	cap_lo.raw = nvme_mmio_read_4(ctrlr, cap_lo.raw);
+	num_entries = nvme_min(NVME_IO_ENTRIES, cap_lo.bits.mqes + 1);
+
+	/*
+	 * No need to have more trackers than entries in the submit queue.
+	 *  Note also that for a queue size of N, we can only have (N-1)
+	 *  commands outstanding, hence the "-1" here.
+	 */
+	num_trackers = nvme_min(NVME_IO_TRACKERS, (num_entries - 1));
+
+	ctrlr->max_xfer_size = NVME_MAX_XFER_SIZE;
+
+	ctrlr->ioq = calloc(ctrlr->num_io_queues, sizeof(struct nvme_qpair));
+
+	if (ctrlr->ioq == NULL)
+		return -1;
+
+	for (i = 0; i < ctrlr->num_io_queues; i++) {
+		qpair = &ctrlr->ioq[i];
+
+		/*
+		 * Admin queue has ID=0. IO queues start at ID=1 -
+		 *  hence the 'i+1' here.
+		 *
+		 * For I/O queues, use the controller-wide max_xfer_size
+		 *  calculated in nvme_attach().
+		 */
+		rc = nvme_qpair_construct(qpair,
+					  i + 1, /* qpair ID */
+					  num_entries,
+					  num_trackers,
+					  ctrlr);
+		if (rc)
+			return -1;
+	}
+
+	return 0;
+}
+
+static void
+nvme_ctrlr_fail(struct nvme_controller *ctrlr)
+{
+	int i;
+
+	ctrlr->is_failed = true;
+	nvme_qpair_fail(&ctrlr->adminq);
+	for (i = 0; i < ctrlr->num_io_queues; i++) {
+		nvme_qpair_fail(&ctrlr->ioq[i]);
+	}
+}
+
+static int
+_nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr, int desired_ready_value)
+{
+	int ms_waited, ready_timeout_in_ms;
+	union nvme_csts_register csts;
+	union nvme_cap_lo_register cap_lo;
+
+	/* Get ready timeout value from controller, in units of 500ms. */
+	cap_lo.raw = nvme_mmio_read_4(ctrlr, cap_lo.raw);
+	ready_timeout_in_ms = cap_lo.bits.to * 500;
+
+	csts.raw = nvme_mmio_read_4(ctrlr, csts);
+
+	ms_waited = 0;
+
+	while (csts.bits.rdy != desired_ready_value) {
+		nvme_delay(1000);
+		if (ms_waited++ > ready_timeout_in_ms) {
+			nvme_printf(ctrlr, "controller ready did not become %d "
+				    "within %d ms\n", desired_ready_value, ready_timeout_in_ms);
+			return ENXIO;
+		}
+		csts.raw = nvme_mmio_read_4(ctrlr, csts);
+	}
+
+	return 0;
+}
+
+static int
+nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr)
+{
+	union nvme_cc_register cc;
+
+	cc.raw = nvme_mmio_read_4(ctrlr, cc.raw);
+
+	if (!cc.bits.en) {
+		nvme_printf(ctrlr, "%s called with cc.en = 0\n", __func__);
+		return ENXIO;
+	}
+
+	return _nvme_ctrlr_wait_for_ready(ctrlr, 1);
+}
+
+static void
+nvme_ctrlr_disable(struct nvme_controller *ctrlr)
+{
+	union nvme_cc_register cc;
+	union nvme_csts_register csts;
+
+	cc.raw = nvme_mmio_read_4(ctrlr, cc.raw);
+	csts.raw = nvme_mmio_read_4(ctrlr, csts);
+
+	if (cc.bits.en == 1 && csts.bits.rdy == 0) {
+		_nvme_ctrlr_wait_for_ready(ctrlr, 1);
+	}
+
+	cc.bits.en = 0;
+	nvme_mmio_write_4(ctrlr, cc.raw, cc.raw);
+	nvme_delay(5000);
+
+	_nvme_ctrlr_wait_for_ready(ctrlr, 0);
+}
+
+static void
+nvme_ctrlr_shutdown(struct nvme_controller *ctrlr)
+{
+	union nvme_cc_register		cc;
+	union nvme_csts_register	csts;
+	int				ms_waited = 0;
+
+	cc.raw = nvme_mmio_read_4(ctrlr, cc.raw);
+	cc.bits.shn = NVME_SHN_NORMAL;
+	nvme_mmio_write_4(ctrlr, cc.raw, cc.raw);
+
+	csts.raw = nvme_mmio_read_4(ctrlr, csts);
+	/*
+	 * The NVMe spec does not define a timeout period
+	 *  for shutdown notification, so we just pick
+	 *  5 seconds as a reasonable amount of time to
+	 *  wait before proceeding.
+	 */
+	while (csts.bits.shst != NVME_SHST_COMPLETE) {
+		nvme_delay(1000);
+		csts.raw = nvme_mmio_read_4(ctrlr, csts);
+		if (ms_waited++ >= 5000)
+			break;
+	}
+	if (csts.bits.shst != NVME_SHST_COMPLETE)
+		nvme_printf(ctrlr, "did not shutdown within 5 seconds\n");
+}
+
+static int
+nvme_ctrlr_enable(struct nvme_controller *ctrlr)
+{
+	union nvme_cc_register		cc;
+	union nvme_csts_register	csts;
+	union nvme_aqa_register		aqa;
+
+	cc.raw = nvme_mmio_read_4(ctrlr, cc.raw);
+	csts.raw = nvme_mmio_read_4(ctrlr, csts);
+
+	if (cc.bits.en == 1) {
+		if (csts.bits.rdy == 1) {
+			return 0;
+		} else {
+			return nvme_ctrlr_wait_for_ready(ctrlr);
+		}
+	}
+
+	nvme_mmio_write_8(ctrlr, asq, ctrlr->adminq.cmd_bus_addr);
+	nvme_delay(5000);
+	nvme_mmio_write_8(ctrlr, acq, ctrlr->adminq.cpl_bus_addr);
+	nvme_delay(5000);
+
+	aqa.raw = 0;
+	/* acqs and asqs are 0-based. */
+	aqa.bits.acqs = ctrlr->adminq.num_entries - 1;
+	aqa.bits.asqs = ctrlr->adminq.num_entries - 1;
+	nvme_mmio_write_4(ctrlr, aqa.raw, aqa.raw);
+	nvme_delay(5000);
+
+	cc.bits.en = 1;
+	cc.bits.css = 0;
+	cc.bits.ams = 0;
+	cc.bits.shn = 0;
+	cc.bits.iosqes = 6; /* SQ entry size == 64 == 2^6 */
+	cc.bits.iocqes = 4; /* CQ entry size == 16 == 2^4 */
+
+	/* Page size is 2 ^ (12 + mps). */
+	cc.bits.mps = nvme_u32log2(PAGE_SIZE) - 12;
+
+	nvme_mmio_write_4(ctrlr, cc.raw, cc.raw);
+	nvme_delay(5000);
+
+	return nvme_ctrlr_wait_for_ready(ctrlr);
+}
+
+int
+nvme_ctrlr_hw_reset(struct nvme_controller *ctrlr)
+{
+	int i, rc;
+	union nvme_cc_register cc;
+
+	cc.raw = nvme_mmio_read_4(ctrlr, cc.raw);
+	if (cc.bits.en) {
+		nvme_qpair_disable(&ctrlr->adminq);
+		for (i = 0; i < ctrlr->num_io_queues; i++) {
+			nvme_qpair_disable(&ctrlr->ioq[i]);
+		}
+
+		nvme_delay(100 * 1000);
+	} else {
+		/*
+		 * Ensure we do a transition from cc.en==1 to cc.en==0.
+		 *  If we started disabled (cc.en==0), then we have to enable
+		 *  first to get a reset.
+		 */
+		nvme_ctrlr_enable(ctrlr);
+	}
+
+	nvme_ctrlr_disable(ctrlr);
+	rc = nvme_ctrlr_enable(ctrlr);
+
+	nvme_delay(100 * 1000);
+
+	return rc;
+}
+
+int
+nvme_ctrlr_reset(struct nvme_controller *ctrlr)
+{
+	int rc;
+
+	nvme_mutex_lock(&ctrlr->ctrlr_lock);
+
+	if (ctrlr->is_resetting || ctrlr->is_failed) {
+		/*
+		 * Controller is already resetting or has failed.  Return
+		 *  immediately since there is no need to kick off another
+		 *  reset in these cases.
+		 */
+		nvme_mutex_unlock(&ctrlr->ctrlr_lock);
+		return 0;
+	}
+
+	ctrlr->is_resetting = 1;
+
+	nvme_printf(ctrlr, "resetting controller\n");
+	/* nvme_ctrlr_start() issues a reset as its first step */
+	rc = nvme_ctrlr_start(ctrlr);
+	if (rc) {
+		nvme_ctrlr_fail(ctrlr);
+	}
+
+	ctrlr->is_resetting = 0;
+
+	nvme_mutex_unlock(&ctrlr->ctrlr_lock);
+
+	return rc;
+}
+
+static int
+nvme_ctrlr_identify(struct nvme_controller *ctrlr)
+{
+	struct nvme_completion_poll_status	status;
+
+	status.done = false;
+	nvme_ctrlr_cmd_identify_controller(ctrlr, &ctrlr->cdata,
+					   nvme_completion_poll_cb, &status);
+	while (status.done == false) {
+		nvme_qpair_process_completions(&ctrlr->adminq);
+	}
+	if (nvme_completion_is_error(&status.cpl)) {
+		nvme_printf(ctrlr, "nvme_identify_controller failed!\n");
+		return ENXIO;
+	}
+
+	/*
+	 * Use MDTS to ensure our default max_xfer_size doesn't exceed what the
+	 *  controller supports.
+	 */
+	if (ctrlr->cdata.mdts > 0) {
+		ctrlr->max_xfer_size = nvme_min(ctrlr->max_xfer_size,
+						ctrlr->min_page_size * (1 << (ctrlr->cdata.mdts)));
+	}
+
+	return 0;
+}
+
+static int
+nvme_ctrlr_set_num_qpairs(struct nvme_controller *ctrlr)
+{
+	struct nvme_driver			*driver = &g_nvme_driver;
+	struct nvme_completion_poll_status	status;
+	int					cq_allocated, sq_allocated;
+	uint32_t				max_io_queues;
+
+	status.done = false;
+
+	nvme_mutex_lock(&driver->lock);
+	max_io_queues = driver->max_io_queues;
+	nvme_mutex_unlock(&driver->lock);
+
+	nvme_ctrlr_cmd_set_num_queues(ctrlr, max_io_queues,
+				      nvme_completion_poll_cb, &status);
+	while (status.done == false) {
+		nvme_qpair_process_completions(&ctrlr->adminq);
+	}
+	if (nvme_completion_is_error(&status.cpl)) {
+		nvme_printf(ctrlr, "nvme_set_num_queues failed!\n");
+		return ENXIO;
+	}
+
+	/*
+	 * Data in cdw0 is 0-based.
+	 * Lower 16-bits indicate number of submission queues allocated.
+	 * Upper 16-bits indicate number of completion queues allocated.
+	 */
+	sq_allocated = (status.cpl.cdw0 & 0xFFFF) + 1;
+	cq_allocated = (status.cpl.cdw0 >> 16) + 1;
+
+	ctrlr->num_io_queues = nvme_min(sq_allocated, cq_allocated);
+
+	nvme_mutex_lock(&driver->lock);
+	driver->max_io_queues = nvme_min(driver->max_io_queues, ctrlr->num_io_queues);
+	nvme_mutex_unlock(&driver->lock);
+
+	return 0;
+}
+
+static int
+nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr)
+{
+	struct nvme_completion_poll_status	status;
+	struct nvme_qpair			*qpair;
+	int					i;
+
+	if (nvme_ctrlr_construct_io_qpairs(ctrlr)) {
+		nvme_printf(ctrlr, "nvme_ctrlr_construct_io_qpairs failed!\n");
+		return ENXIO;
+	}
+
+	for (i = 0; i < ctrlr->num_io_queues; i++) {
+		qpair = &ctrlr->ioq[i];
+
+		status.done = false;
+		nvme_ctrlr_cmd_create_io_cq(ctrlr, qpair,
+					    nvme_completion_poll_cb, &status);
+		while (status.done == false) {
+			nvme_qpair_process_completions(&ctrlr->adminq);
+		}
+		if (nvme_completion_is_error(&status.cpl)) {
+			nvme_printf(ctrlr, "nvme_create_io_cq failed!\n");
+			return ENXIO;
+		}
+
+		status.done = false;
+		nvme_ctrlr_cmd_create_io_sq(qpair->ctrlr, qpair,
+					    nvme_completion_poll_cb, &status);
+		while (status.done == false) {
+			nvme_qpair_process_completions(&ctrlr->adminq);
+		}
+		if (nvme_completion_is_error(&status.cpl)) {
+			nvme_printf(ctrlr, "nvme_create_io_sq failed!\n");
+			return ENXIO;
+		}
+
+		nvme_qpair_reset(qpair);
+	}
+
+	return 0;
+}
+
+static void
+nvme_ctrlr_destruct_namespaces(struct nvme_controller *ctrlr)
+{
+	if (ctrlr->ns) {
+		uint32_t i, num_ns = ctrlr->num_ns;
+
+		for (i = 0; i < num_ns; i++) {
+			nvme_ns_destruct(&ctrlr->ns[i]);
+		}
+
+		free(ctrlr->ns);
+		ctrlr->ns = NULL;
+		ctrlr->num_ns = 0;
+	}
+
+	if (ctrlr->nsdata) {
+		nvme_free(ctrlr->nsdata);
+		ctrlr->nsdata = NULL;
+	}
+}
+
+static int
+nvme_ctrlr_construct_namespaces(struct nvme_controller *ctrlr)
+{
+	uint32_t i, nn = ctrlr->cdata.nn;
+	uint64_t phys_addr = 0;
+
+	if (nn == 0) {
+		nvme_printf(ctrlr, "controller has 0 namespaces\n");
+		return -1;
+	}
+
+	/* ctrlr->num_ns may be 0 (startup) or a different number of namespaces (reset),
+	 * so check if we need to reallocate.
+	 */
+	if (nn != ctrlr->num_ns) {
+		nvme_ctrlr_destruct_namespaces(ctrlr);
+
+		ctrlr->ns = calloc(nn, sizeof(struct nvme_namespace));
+		if (ctrlr->ns == NULL) {
+			goto fail;
+		}
+
+		ctrlr->nsdata = nvme_malloc("nvme_namespaces",
+					    nn * sizeof(struct nvme_namespace_data), 64,
+					    &phys_addr);
+		if (ctrlr->nsdata == NULL) {
+			goto fail;
+		}
+
+		ctrlr->num_ns = nn;
+	}
+
+	for (i = 0; i < nn; i++) {
+		struct nvme_namespace	*ns = &ctrlr->ns[i];
+		uint32_t 		nsid = i + 1;
+
+		if (nvme_ns_construct(ns, nsid, ctrlr) != 0) {
+			goto fail;
+		}
+	}
+
+	return 0;
+
+fail:
+	nvme_ctrlr_destruct_namespaces(ctrlr);
+	return -1;
+}
+
+static void
+nvme_ctrlr_async_event_cb(void *arg, const struct nvme_completion *cpl)
+{
+	struct nvme_async_event_request	*aer = arg;
+	struct nvme_controller		*ctrlr = aer->ctrlr;
+
+	if (cpl->status.sc == NVME_SC_ABORTED_SQ_DELETION) {
+		/*
+		 *  This is simulated when controller is being shut down, to
+		 *  effectively abort outstanding asynchronous event requests
+		 *  and make sure all memory is freed.  Do not repost the
+		 *  request in this case.
+		 */
+		return;
+	}
+
+	if (ctrlr->aer_cb_fn != NULL) {
+		ctrlr->aer_cb_fn(ctrlr->aer_cb_arg, cpl);
+	}
+
+	/*
+	 * Repost another asynchronous event request to replace the one
+	 *  that just completed.
+	 */
+	nvme_ctrlr_construct_and_submit_aer(aer->ctrlr, aer);
+}
+
+static void
+nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,
+				    struct nvme_async_event_request *aer)
+{
+	struct nvme_request *req;
+
+	aer->ctrlr = ctrlr;
+	req = nvme_allocate_request(NULL, 0, nvme_ctrlr_async_event_cb, aer);
+	aer->req = req;
+
+	/*
+	 * Disable timeout here, since asynchronous event requests should by
+	 *  nature never be timed out.
+	 */
+	req->timeout = false;
+	req->cmd.opc = NVME_OPC_ASYNC_EVENT_REQUEST;
+	nvme_ctrlr_submit_admin_request(ctrlr, req);
+}
+
+static int
+nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr)
+{
+	union nvme_critical_warning_state	state;
+	struct nvme_async_event_request		*aer;
+	uint32_t				i;
+	struct nvme_completion_poll_status	status;
+
+	status.done = false;
+
+	state.raw = 0xFF;
+	state.bits.reserved = 0;
+	nvme_ctrlr_cmd_set_async_event_config(ctrlr, state, nvme_completion_poll_cb, &status);
+
+	while (status.done == false) {
+		nvme_qpair_process_completions(&ctrlr->adminq);
+	}
+	if (nvme_completion_is_error(&status.cpl)) {
+		nvme_printf(ctrlr, "nvme_ctrlr_cmd_set_async_event_config failed!\n");
+		return ENXIO;
+	}
+
+	/* aerl is a zero-based value, so we need to add 1 here. */
+	ctrlr->num_aers = nvme_min(NVME_MAX_ASYNC_EVENTS, (ctrlr->cdata.aerl + 1));
+
+	for (i = 0; i < ctrlr->num_aers; i++) {
+		aer = &ctrlr->aer[i];
+		nvme_ctrlr_construct_and_submit_aer(ctrlr, aer);
+	}
+
+	return 0;
+}
+
+int
+nvme_ctrlr_start(struct nvme_controller *ctrlr)
+{
+	if (nvme_ctrlr_hw_reset(ctrlr) != 0) {
+		return -1;
+	}
+
+	nvme_qpair_reset(&ctrlr->adminq);
+
+	nvme_qpair_enable(&ctrlr->adminq);
+
+	if (nvme_ctrlr_identify(ctrlr) != 0) {
+		return -1;
+	}
+
+	if (nvme_ctrlr_set_num_qpairs(ctrlr) != 0) {
+		return -1;
+	}
+
+	if (nvme_ctrlr_create_qpairs(ctrlr) != 0) {
+		return -1;
+	}
+
+	if (nvme_ctrlr_construct_namespaces(ctrlr) != 0) {
+		return -1;
+	}
+
+	if (nvme_ctrlr_configure_aer(ctrlr) != 0) {
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+nvme_ctrlr_allocate_bars(struct nvme_controller *ctrlr)
+{
+	int rc;
+	void *addr;
+
+	rc = nvme_pcicfg_map_bar(ctrlr->devhandle, 0, 0 /* writable */, &addr);
+	ctrlr->regs = (volatile struct nvme_registers *)addr;
+	if ((ctrlr->regs == NULL) || (rc != 0)) {
+		printf("pci_device_map_range failed with error code %d\n", rc);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+nvme_ctrlr_free_bars(struct nvme_controller *ctrlr)
+{
+	int rc = 0;
+	void *addr = (void *)ctrlr->regs;
+
+	if (addr) {
+		rc = nvme_pcicfg_unmap_bar(ctrlr->devhandle, 0, addr);
+	}
+	return rc;
+}
+
+int
+nvme_ctrlr_construct(struct nvme_controller *ctrlr, void *devhandle)
+{
+	union nvme_cap_hi_register	cap_hi;
+	uint32_t			cmd_reg;
+	int				status;
+	int				rc;
+
+	ctrlr->devhandle = devhandle;
+
+	status = nvme_ctrlr_allocate_bars(ctrlr);
+	if (status != 0) {
+		return status;
+	}
+
+	/* Enable PCI busmaster. */
+	nvme_pcicfg_read32(devhandle, &cmd_reg, 4);
+	cmd_reg |= 0x4;
+	nvme_pcicfg_write32(devhandle, cmd_reg, 4);
+
+	cap_hi.raw = nvme_mmio_read_4(ctrlr, cap_hi.raw);
+
+	/* Doorbell stride is 2 ^ (dstrd + 2),
+	 * but we want multiples of 4, so drop the + 2 */
+	ctrlr->doorbell_stride_u32 = 1 << cap_hi.bits.dstrd;
+
+	ctrlr->min_page_size = 1 << (12 + cap_hi.bits.mpsmin);
+
+	rc = nvme_ctrlr_construct_admin_qpair(ctrlr);
+	if (rc)
+		return rc;
+
+	ctrlr->is_resetting = 0;
+	ctrlr->is_failed = false;
+
+	nvme_mutex_init_recursive(&ctrlr->ctrlr_lock);
+
+	return 0;
+}
+
+void
+nvme_ctrlr_destruct(struct nvme_controller *ctrlr)
+{
+	int				i;
+
+	nvme_ctrlr_disable(ctrlr);
+	nvme_ctrlr_shutdown(ctrlr);
+
+	nvme_ctrlr_destruct_namespaces(ctrlr);
+
+	for (i = 0; i < ctrlr->num_io_queues; i++) {
+		nvme_qpair_destroy(&ctrlr->ioq[i]);
+	}
+
+	free(ctrlr->ioq);
+
+	nvme_qpair_destroy(&ctrlr->adminq);
+
+	nvme_ctrlr_free_bars(ctrlr);
+	nvme_mutex_destroy(&ctrlr->ctrlr_lock);
+}
+
+void
+nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr,
+				struct nvme_request *req)
+{
+	nvme_qpair_submit_request(&ctrlr->adminq, req);
+}
+
+void
+nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr,
+			     struct nvme_request *req)
+{
+	struct nvme_qpair       *qpair;
+
+	nvme_assert(nvme_thread_ioq_index >= 0, ("no ioq_index assigned for thread\n"));
+	qpair = &ctrlr->ioq[nvme_thread_ioq_index];
+
+	nvme_qpair_submit_request(qpair, req);
+}
+
+void
+nvme_ctrlr_process_io_completions(struct nvme_controller *ctrlr)
+{
+	nvme_assert(nvme_thread_ioq_index >= 0, ("no ioq_index assigned for thread\n"));
+	nvme_qpair_process_completions(&ctrlr->ioq[nvme_thread_ioq_index]);
+}
+
+void
+nvme_ctrlr_process_admin_completions(struct nvme_controller *ctrlr)
+{
+	nvme_mutex_lock(&ctrlr->ctrlr_lock);
+	nvme_qpair_process_completions(&ctrlr->adminq);
+	nvme_mutex_unlock(&ctrlr->ctrlr_lock);
+}
+
+const struct nvme_controller_data *
+nvme_ctrlr_get_data(struct nvme_controller *ctrlr)
+{
+
+	return &ctrlr->cdata;
+}
+
+uint32_t
+nvme_ctrlr_get_num_ns(struct nvme_controller *ctrlr)
+{
+	return ctrlr->num_ns;
+}
+
+struct nvme_namespace *
+nvme_ctrlr_get_ns(struct nvme_controller *ctrlr, uint32_t ns_id)
+{
+	if (ns_id < 1 || ns_id > ctrlr->num_ns) {
+		return NULL;
+	}
+
+	return &ctrlr->ns[ns_id - 1];
+}
+
+void
+nvme_ctrlr_register_aer_callback(struct nvme_controller *ctrlr,
+				 nvme_aer_cb_fn_t aer_cb_fn,
+				 void *aer_cb_arg)
+{
+	ctrlr->aer_cb_fn = aer_cb_fn;
+	ctrlr->aer_cb_arg = aer_cb_arg;
+}

lib/nvme/nvme_ctrlr_cmd.c

@@ -0,0 +1,310 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "nvme_internal.h"
+
+int
+nvme_ctrlr_cmd_io_raw(struct nvme_controller *ctrlr,
+		      struct nvme_command *cmd,
+		      void *buf, uint32_t len,
+		      nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+	struct nvme_request	*req;
+
+	req = nvme_allocate_request(buf, len, cb_fn, cb_arg);
+
+	if (req == NULL) {
+		return ENOMEM;
+	}
+
+	memcpy(&req->cmd, cmd, sizeof(req->cmd));
+
+	nvme_ctrlr_submit_io_request(ctrlr, req);
+	return 0;
+}
+
+int
+nvme_ctrlr_cmd_admin_raw(struct nvme_controller *ctrlr,
+			 struct nvme_command *cmd,
+			 void *buf, uint32_t len,
+			 nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+	struct nvme_request	*req;
+
+	nvme_mutex_lock(&ctrlr->ctrlr_lock);
+	req = nvme_allocate_request(buf, len, cb_fn, cb_arg);
+	if (req == NULL) {
+		nvme_mutex_unlock(&ctrlr->ctrlr_lock);
+		return ENOMEM;
+	}
+
+	memcpy(&req->cmd, cmd, sizeof(req->cmd));
+
+	nvme_ctrlr_submit_admin_request(ctrlr, req);
+
+	nvme_mutex_unlock(&ctrlr->ctrlr_lock);
+	return 0;
+}
+
+void
+nvme_ctrlr_cmd_identify_controller(struct nvme_controller *ctrlr, void *payload,
+				   nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+	struct nvme_command *cmd;
+
+	req = nvme_allocate_request(payload,
+				    sizeof(struct nvme_controller_data),
+				    cb_fn, cb_arg);
+
+	cmd = &req->cmd;
+	cmd->opc = NVME_OPC_IDENTIFY;
+
+	/*
+	 * TODO: create an identify command data structure, which
+	 *  includes this CNS bit in cdw10.
+	 */
+	cmd->cdw10 = 1;
+
+	nvme_ctrlr_submit_admin_request(ctrlr, req);
+}
+
+void
+nvme_ctrlr_cmd_identify_namespace(struct nvme_controller *ctrlr, uint16_t nsid,
+				  void *payload, nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+	struct nvme_command *cmd;
+
+	req = nvme_allocate_request(payload,
+				    sizeof(struct nvme_namespace_data),
+				    cb_fn, cb_arg);
+
+	cmd = &req->cmd;
+	cmd->opc = NVME_OPC_IDENTIFY;
+
+	/*
+	 * TODO: create an identify command data structure
+	 */
+	cmd->nsid = nsid;
+
+	nvme_ctrlr_submit_admin_request(ctrlr, req);
+}
+
+void
+nvme_ctrlr_cmd_create_io_cq(struct nvme_controller *ctrlr,
+			    struct nvme_qpair *io_que, nvme_cb_fn_t cb_fn,
+			    void *cb_arg)
+{
+	struct nvme_request *req;
+	struct nvme_command *cmd;
+
+	req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg);
+
+	cmd = &req->cmd;
+	cmd->opc = NVME_OPC_CREATE_IO_CQ;
+
+	/*
+	 * TODO: create a create io completion queue command data
+	 *  structure.
+	 */
+	cmd->cdw10 = ((io_que->num_entries - 1) << 16) | io_que->id;
+	/*
+	 * 0x2 = interrupts enabled
+	 * 0x1 = physically contiguous
+	 */
+	cmd->cdw11 = (io_que->id << 16) | 0x1;
+	cmd->dptr.prp.prp1 = io_que->cpl_bus_addr;
+
+	nvme_ctrlr_submit_admin_request(ctrlr, req);
+}
+
+void
+nvme_ctrlr_cmd_create_io_sq(struct nvme_controller *ctrlr,
+			    struct nvme_qpair *io_que, nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+	struct nvme_command *cmd;
+
+	req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg);
+
+	cmd = &req->cmd;
+	cmd->opc = NVME_OPC_CREATE_IO_SQ;
+
+	/*
+	 * TODO: create a create io submission queue command data
+	 *  structure.
+	 */
+	cmd->cdw10 = ((io_que->num_entries - 1) << 16) | io_que->id;
+	/* 0x1 = physically contiguous */
+	cmd->cdw11 = (io_que->id << 16) | 0x1;
+	cmd->dptr.prp.prp1 = io_que->cmd_bus_addr;
+
+	nvme_ctrlr_submit_admin_request(ctrlr, req);
+}
+
+void
+nvme_ctrlr_cmd_set_feature(struct nvme_controller *ctrlr, uint8_t feature,
+			   uint32_t cdw11, void *payload, uint32_t payload_size,
+			   nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+	struct nvme_command *cmd;
+
+	req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg);
+
+	cmd = &req->cmd;
+	cmd->opc = NVME_OPC_SET_FEATURES;
+	cmd->cdw10 = feature;
+	cmd->cdw11 = cdw11;
+
+	nvme_ctrlr_submit_admin_request(ctrlr, req);
+}
+
+void
+nvme_ctrlr_cmd_get_feature(struct nvme_controller *ctrlr, uint8_t feature,
+			   uint32_t cdw11, void *payload, uint32_t payload_size,
+			   nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+	struct nvme_command *cmd;
+
+	req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg);
+
+	cmd = &req->cmd;
+	cmd->opc = NVME_OPC_GET_FEATURES;
+	cmd->cdw10 = feature;
+	cmd->cdw11 = cdw11;
+
+	nvme_ctrlr_submit_admin_request(ctrlr, req);
+}
+
+void
+nvme_ctrlr_cmd_set_num_queues(struct nvme_controller *ctrlr,
+			      uint32_t num_queues, nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+	uint32_t cdw11;
+
+	cdw11 = ((num_queues - 1) << 16) | (num_queues - 1);
+	nvme_ctrlr_cmd_set_feature(ctrlr, NVME_FEAT_NUMBER_OF_QUEUES, cdw11,
+				   NULL, 0, cb_fn, cb_arg);
+}
+
+void
+nvme_ctrlr_cmd_set_async_event_config(struct nvme_controller *ctrlr,
+				      union nvme_critical_warning_state state, nvme_cb_fn_t cb_fn,
+				      void *cb_arg)
+{
+	uint32_t cdw11;
+
+	cdw11 = state.raw;
+	nvme_ctrlr_cmd_set_feature(ctrlr,
+				   NVME_FEAT_ASYNC_EVENT_CONFIGURATION, cdw11, NULL, 0, cb_fn,
+				   cb_arg);
+}
+
+void
+nvme_ctrlr_cmd_get_log_page(struct nvme_controller *ctrlr, uint8_t log_page,
+			    uint32_t nsid, void *payload, uint32_t payload_size, nvme_cb_fn_t cb_fn,
+			    void *cb_arg)
+{
+	struct nvme_request *req;
+	struct nvme_command *cmd;
+
+	req = nvme_allocate_request(payload, payload_size, cb_fn, cb_arg);
+
+	cmd = &req->cmd;
+	cmd->opc = NVME_OPC_GET_LOG_PAGE;
+	cmd->nsid = nsid;
+	cmd->cdw10 = ((payload_size / sizeof(uint32_t)) - 1) << 16;
+	cmd->cdw10 |= log_page;
+
+	nvme_ctrlr_submit_admin_request(ctrlr, req);
+}
+
+void
+nvme_ctrlr_cmd_get_error_page(struct nvme_controller *ctrlr,
+			      struct nvme_error_information_entry *payload, uint32_t num_entries,
+			      nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+
+	nvme_assert(num_entries > 0, ("%s called with num_entries==0\n", __func__));
+
+	/* Controller's error log page entries is 0-based. */
+	nvme_assert(num_entries <= (ctrlr->cdata.elpe + 1),
+		    ("%s called with num_entries=%d but (elpe+1)=%d\n", __func__,
+		     num_entries, ctrlr->cdata.elpe + 1));
+
+	if (num_entries > (ctrlr->cdata.elpe + 1))
+		num_entries = ctrlr->cdata.elpe + 1;
+
+	nvme_ctrlr_cmd_get_log_page(ctrlr, NVME_LOG_ERROR,
+				    NVME_GLOBAL_NAMESPACE_TAG, payload, sizeof(*payload) * num_entries,
+				    cb_fn, cb_arg);
+}
+
+void
+nvme_ctrlr_cmd_get_health_information_page(struct nvme_controller *ctrlr,
+		uint32_t nsid, struct nvme_health_information_page *payload,
+		nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+
+	nvme_ctrlr_cmd_get_log_page(ctrlr, NVME_LOG_HEALTH_INFORMATION,
+				    nsid, payload, sizeof(*payload), cb_fn, cb_arg);
+}
+
+void
+nvme_ctrlr_cmd_get_firmware_page(struct nvme_controller *ctrlr,
+				 struct nvme_firmware_page *payload, nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+
+	nvme_ctrlr_cmd_get_log_page(ctrlr, NVME_LOG_FIRMWARE_SLOT,
+				    NVME_GLOBAL_NAMESPACE_TAG, payload, sizeof(*payload), cb_fn,
+				    cb_arg);
+}
+
+void
+nvme_ctrlr_cmd_abort(struct nvme_controller *ctrlr, uint16_t cid,
+		     uint16_t sqid, nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+	struct nvme_command *cmd;
+
+	req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg);
+
+	cmd = &req->cmd;
+	cmd->opc = NVME_OPC_ABORT;
+	cmd->cdw10 = (cid << 16) | sqid;
+
+	nvme_ctrlr_submit_admin_request(ctrlr, req);
+}

lib/nvme/nvme_impl.h

@@ -0,0 +1,164 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __NVME_IMPL_H__
+#define __NVME_IMPL_H__
+
+#include "spdk/vtophys.h"
+#include <assert.h>
+#include <pciaccess.h>
+#include <rte_malloc.h>
+#include <rte_config.h>
+#include <rte_mempool.h>
+
+/**
+ * \file
+ *
+ * This file describes the callback functions required to integrate
+ * the userspace NVMe driver for a specific implementation.  This
+ * implementation is specific for DPDK for Storage.  Users would
+ * revise it as necessary for their own particular environment if not
+ * using it within the DPDK for Storage framework.
+ */
+
+/**
+ * \page nvme_driver_integration NVMe Driver Integration
+ *
+ * Users can integrate the userspace NVMe driver into their environment
+ * by implementing the callbacks in nvme_impl.h.  These callbacks
+ * enable users to specify how to allocate pinned and physically
+ * contiguous memory, performance virtual to physical address
+ * translations, log messages, PCI configuration and register mapping,
+ * and a number of other facilities that may differ depending on the
+ * environment.
+ */
+
+/**
+ * Allocate a pinned, physically contiguous memory buffer with the
+ *   given size and alignment.
+ * Note: these calls are only made during driver initialization.  Per
+ *   I/O allocations during driver operation use the nvme_alloc_request
+ *   callback.
+ */
+static inline void *
+nvme_malloc(const char *tag, size_t size, unsigned align, uint64_t *phys_addr)
+{
+	void *buf = rte_zmalloc(tag, size, align);
+	*phys_addr = rte_malloc_virt2phy(buf);
+	return buf;
+}
+
+/**
+ * Free a memory buffer previously allocated with nvme_malloc.
+ */
+#define nvme_free(buf)			rte_free(buf)
+
+/**
+ * Log or print a message from the NVMe driver.
+ */
+#define nvme_printf(ctrlr, fmt, args...) printf(fmt, ##args)
+
+/**
+ * Assert a condition and panic/abort as desired.  Failures of these
+ *  assertions indicate catastrophic failures within the driver.
+ */
+#define nvme_assert(check, str) assert(check)
+
+/**
+ * Return the physical address for the specified virtual address.
+ */
+#define nvme_vtophys(buf)		vtophys(buf)
+
+extern struct rte_mempool *request_mempool;
+
+/**
+ * Return a buffer for an nvme_request object.  These objects are allocated
+ *  for each I/O.  They do not need to be pinned nor physically contiguous.
+ */
+#define nvme_alloc_request(bufp)	rte_mempool_get(request_mempool, (void **)(bufp));
+
+/**
+ * Free a buffer previously allocated with nvme_alloc_request().
+ */
+#define nvme_free_request(buf)		rte_mempool_put(request_mempool, buf)
+
+/**
+ *
+ */
+#define nvme_pcicfg_read32(handle, var, offset)  pci_device_cfg_read_u32(handle, var, offset)
+#define nvme_pcicfg_write32(handle, var, offset) pci_device_cfg_write_u32(handle, var, offset)
+
+static inline int
+nvme_pcicfg_map_bar(void *devhandle, uint32_t bar, uint32_t read_only, void **mapped_addr)
+{
+	struct pci_device *dev = devhandle;
+	uint32_t flags = (read_only ? 0 : PCI_DEV_MAP_FLAG_WRITABLE);
+
+	return pci_device_map_range(dev, dev->regions[bar].base_addr, dev->regions[bar].size,
+				    flags, mapped_addr);
+}
+
+static inline int
+nvme_pcicfg_unmap_bar(void *devhandle, uint32_t bar, void *addr)
+{
+	struct pci_device *dev = devhandle;
+
+	return pci_device_unmap_range(dev, addr, dev->regions[bar].size);
+}
+
+typedef pthread_mutex_t nvme_mutex_t;
+
+#define nvme_mutex_init(x) pthread_mutex_init((x), NULL)
+#define nvme_mutex_destroy(x) pthread_mutex_destroy((x))
+#define nvme_mutex_lock pthread_mutex_lock
+#define nvme_mutex_unlock pthread_mutex_unlock
+#define NVME_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
+
+static inline int
+nvme_mutex_init_recursive(nvme_mutex_t *mtx)
+{
+	pthread_mutexattr_t attr;
+	int rc = 0;
+
+	if (pthread_mutexattr_init(&attr)) {
+		return -1;
+	}
+	if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) ||
+	    pthread_mutex_init(mtx, &attr)) {
+		rc = -1;
+	}
+	pthread_mutexattr_destroy(&attr);
+	return rc;
+}
+
+#endif /* __NVME_IMPL_H__ */

lib/nvme/nvme_internal.h

@@ -0,0 +1,442 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __NVME_INTERNAL_H__
+#define __NVME_INTERNAL_H__
+
+#include <errno.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <x86intrin.h>
+
+#include <sys/user.h>
+
+#include "spdk/nvme.h"
+
+#include "spdk/queue.h"
+#include "spdk/barrier.h"
+
+#define NVME_MAX_PRP_LIST_ENTRIES	(32)
+
+/*
+ * For commands requiring more than 2 PRP entries, one PRP will be
+ *  embedded in the command (prp1), and the rest of the PRP entries
+ *  will be in a list pointed to by the command (prp2).  This means
+ *  that real max number of PRP entries we support is 32+1, which
+ *  results in a max xfer size of 32*PAGE_SIZE.
+ */
+#define NVME_MAX_XFER_SIZE	NVME_MAX_PRP_LIST_ENTRIES * PAGE_SIZE
+
+#define NVME_ADMIN_TRACKERS	(16)
+#define NVME_ADMIN_ENTRIES	(128)
+/* min and max are defined in admin queue attributes section of spec */
+#define NVME_MIN_ADMIN_ENTRIES	(2)
+#define NVME_MAX_ADMIN_ENTRIES	(4096)
+
+/*
+ * NVME_IO_ENTRIES defines the size of an I/O qpair's submission and completion
+ *  queues, while NVME_IO_TRACKERS defines the maximum number of I/O that we
+ *  will allow outstanding on an I/O qpair at any time.  The only advantage in
+ *  having IO_ENTRIES > IO_TRACKERS is for debugging purposes - when dumping
+ *  the contents of the submission and completion queues, it will show a longer
+ *  history of data.
+ */
+#define NVME_IO_ENTRIES		(256)
+#define NVME_IO_TRACKERS	(128)
+#define NVME_MIN_IO_TRACKERS	(4)
+#define NVME_MAX_IO_TRACKERS	(1024)
+
+/*
+ * NVME_MAX_IO_ENTRIES is not defined, since it is specified in CC.MQES
+ *  for each controller.
+ */
+
+#define NVME_MAX_ASYNC_EVENTS	(8)
+
+#define NVME_MIN_TIMEOUT_PERIOD		(5)
+#define NVME_MAX_TIMEOUT_PERIOD		(120)
+
+/* Maximum log page size to fetch for AERs. */
+#define NVME_MAX_AER_LOG_SIZE		(4096)
+
+struct nvme_request {
+	struct nvme_command		cmd;
+
+	/**
+	 * Points to a parent request if part of a split request,
+	 *   NULL otherwise.
+	 */
+	struct nvme_request		*parent;
+	union {
+		void			*payload;
+	} u;
+
+	uint8_t				timeout;
+	uint8_t				retries;
+
+	/**
+	 * Number of children requests still outstanding for this
+	 *  request which was split into multiple child requests.
+	 */
+	uint8_t				num_children;
+	uint32_t			payload_size;
+	nvme_cb_fn_t			cb_fn;
+	void				*cb_arg;
+	STAILQ_ENTRY(nvme_request)	stailq;
+
+	/**
+	 * The following members should not be reordered with members
+	 *  above.  These members are only needed when splitting
+	 *  requests which is done rarely, and the driver is careful
+	 *  to not touch the following fields until a split operation is
+	 *  needed, to avoid touching an extra cacheline.
+	 */
+
+	/**
+	 * Points to the outstanding child requests for a parent request.
+	 *  Only valid if a request was split into multiple children
+	 *  requests, and is not initialized for non-split requests.
+	 */
+	TAILQ_HEAD(, nvme_request)	children;
+
+	/**
+	 * Linked-list pointers for a child request in its parent's list.
+	 */
+	TAILQ_ENTRY(nvme_request)	child_tailq;
+
+	/**
+	 * Completion status for a parent request.  Initialized to all 0's
+	 *  (SUCCESS) before child requests are submitted.  If a child
+	 *  request completes with error, the error status is copied here,
+	 *  to ensure that the parent request is also completed with error
+	 *  status once all child requests are completed.
+	 */
+	struct nvme_completion		parent_status;
+};
+
+struct nvme_completion_poll_status {
+	struct nvme_completion	cpl;
+	bool			done;
+};
+
+struct nvme_async_event_request {
+	struct nvme_controller		*ctrlr;
+	struct nvme_request		*req;
+	struct nvme_completion		cpl;
+};
+
+struct nvme_tracker {
+	LIST_ENTRY(nvme_tracker)	list;
+
+	struct nvme_request		*req;
+	uint16_t			cid;
+
+	uint64_t			prp_bus_addr;
+	uint64_t			prp[NVME_MAX_PRP_LIST_ENTRIES];
+};
+
+struct nvme_qpair {
+	volatile uint32_t		*sq_tdbl;
+	volatile uint32_t		*cq_hdbl;
+
+	/**
+	 * Submission queue
+	 */
+	struct nvme_command		*cmd;
+
+	/**
+	 * Completion queue
+	 */
+	struct nvme_completion		*cpl;
+
+	LIST_HEAD(, nvme_tracker)	free_tr;
+	LIST_HEAD(, nvme_tracker)	outstanding_tr;
+
+	STAILQ_HEAD(, nvme_request)	queued_req;
+
+	struct nvme_tracker		**act_tr;
+
+	uint16_t			id;
+
+	uint16_t			num_entries;
+	uint16_t			sq_tail;
+	uint16_t			cq_head;
+
+	uint8_t				phase;
+
+	bool				is_enabled;
+
+	/*
+	 * Fields below this point should not be touched on the normal I/O happy path.
+	 */
+	struct nvme_controller		*ctrlr;
+
+	uint64_t			cmd_bus_addr;
+	uint64_t			cpl_bus_addr;
+};
+
+struct nvme_namespace {
+	struct nvme_controller		*ctrlr;
+	uint32_t			stripe_size;
+	uint32_t			sector_size;
+	uint32_t			sectors_per_max_io;
+	uint32_t			sectors_per_stripe;
+	uint16_t			id;
+	uint16_t			flags;
+};
+
+/*
+ * One of these per allocated PCI device.
+ */
+struct nvme_controller {
+	/* Hot data (accessed in I/O path) starts here. */
+
+	/** NVMe MMIO register space */
+	volatile struct nvme_registers	*regs;
+
+	/** I/O queue pairs */
+	struct nvme_qpair		*ioq;
+
+	uint32_t			is_resetting;
+
+	uint32_t			num_ns;
+
+	/** Array of namespaces indexed by nsid - 1 */
+	struct nvme_namespace		*ns;
+
+
+	/* Cold data (not accessed in normal I/O path) is after this point. */
+
+	/* Opaque handle to associated PCI device. */
+	void				*devhandle;
+
+	uint32_t			num_io_queues;
+
+	/** maximum i/o size in bytes */
+	uint32_t			max_xfer_size;
+
+	/** minimum page size supported by this controller in bytes */
+	uint32_t			min_page_size;
+
+	/** stride in uint32_t units between doorbell registers (1 = 4 bytes, 2 = 8 bytes, ...) */
+	uint32_t			doorbell_stride_u32;
+
+	uint32_t			num_aers;
+	struct nvme_async_event_request	aer[NVME_MAX_ASYNC_EVENTS];
+	nvme_aer_cb_fn_t		aer_cb_fn;
+	void				*aer_cb_arg;
+
+	bool				is_failed;
+
+	/** guards access to the controller itself, including admin queues */
+	nvme_mutex_t			ctrlr_lock;
+
+
+	struct nvme_qpair		adminq;
+
+	/**
+	 * Identify Controller data.
+	 */
+	struct nvme_controller_data	cdata;
+
+	/**
+	 * Array of Identify Namespace data.
+	 *
+	 * Stored separately from ns since nsdata should not normally be accessed during I/O.
+	 */
+	struct nvme_namespace_data	*nsdata;
+};
+
+extern int __thread nvme_thread_ioq_index;
+
+struct nvme_driver {
+	nvme_mutex_t	lock;
+	uint16_t	*ioq_index_pool;
+	uint32_t	max_io_queues;
+	uint16_t	ioq_index_pool_next;
+};
+
+extern struct nvme_driver g_nvme_driver;
+
+#define nvme_min(a,b) (((a)<(b))?(a):(b))
+
+#define INTEL_DC_P3X00_DEVID	0x09538086
+
+static inline uint32_t
+_nvme_mmio_read_4(const volatile uint32_t *addr)
+{
+	return *addr;
+}
+
+static inline void
+_nvme_mmio_write_4(volatile uint32_t *addr, uint32_t val)
+{
+	*addr = val;
+}
+
+static inline void
+_nvme_mmio_write_8(volatile uint64_t *addr, uint64_t val)
+{
+	*addr = val;
+}
+
+#define nvme_mmio_read_4(sc, reg) \
+	_nvme_mmio_read_4(&(sc)->regs->reg)
+
+#define nvme_mmio_write_4(sc, reg, val) \
+	_nvme_mmio_write_4(&(sc)->regs->reg, val)
+
+#define nvme_mmio_write_8(sc, reg, val) \
+	_nvme_mmio_write_8(&(sc)->regs->reg, val)
+
+#define nvme_delay		usleep
+
+static inline uint32_t
+nvme_u32log2(uint32_t x)
+{
+	if (x == 0) {
+		/* __builtin_clz(0) is undefined, so just bail */
+		return 0;
+	}
+	return 31u - __builtin_clz(x);
+}
+
+static inline uint32_t
+nvme_align32pow2(uint32_t x)
+{
+	return 1u << (1 + nvme_u32log2(x - 1));
+}
+
+/* Admin functions */
+void	nvme_ctrlr_cmd_set_feature(struct nvme_controller *ctrlr,
+				   uint8_t feature, uint32_t cdw11,
+				   void *payload, uint32_t payload_size,
+				   nvme_cb_fn_t cb_fn, void *cb_arg);
+void	nvme_ctrlr_cmd_get_feature(struct nvme_controller *ctrlr,
+				   uint8_t feature, uint32_t cdw11,
+				   void *payload, uint32_t payload_size,
+				   nvme_cb_fn_t cb_fn, void *cb_arg);
+void	nvme_ctrlr_cmd_get_log_page(struct nvme_controller *ctrlr,
+				    uint8_t log_page, uint32_t nsid,
+				    void *payload, uint32_t payload_size,
+				    nvme_cb_fn_t cb_fn, void *cb_arg);
+
+void	nvme_ctrlr_cmd_identify_controller(struct nvme_controller *ctrlr,
+		void *payload,
+		nvme_cb_fn_t cb_fn, void *cb_arg);
+void	nvme_ctrlr_cmd_identify_namespace(struct nvme_controller *ctrlr,
+		uint16_t nsid, void *payload,
+		nvme_cb_fn_t cb_fn, void *cb_arg);
+void	nvme_ctrlr_cmd_get_error_page(struct nvme_controller *ctrlr,
+				      struct nvme_error_information_entry *payload,
+				      uint32_t num_entries, /* 0 = max */
+				      nvme_cb_fn_t cb_fn,
+				      void *cb_arg);
+void	nvme_ctrlr_cmd_get_health_information_page(struct nvme_controller *ctrlr,
+		uint32_t nsid,
+		struct nvme_health_information_page *payload,
+		nvme_cb_fn_t cb_fn,
+		void *cb_arg);
+void	nvme_ctrlr_cmd_get_firmware_page(struct nvme_controller *ctrlr,
+		struct nvme_firmware_page *payload,
+		nvme_cb_fn_t cb_fn,
+		void *cb_arg);
+void	nvme_ctrlr_cmd_create_io_cq(struct nvme_controller *ctrlr,
+				    struct nvme_qpair *io_que,
+				    nvme_cb_fn_t cb_fn, void *cb_arg);
+void	nvme_ctrlr_cmd_create_io_sq(struct nvme_controller *ctrlr,
+				    struct nvme_qpair *io_que,
+				    nvme_cb_fn_t cb_fn, void *cb_arg);
+void	nvme_ctrlr_cmd_set_num_queues(struct nvme_controller *ctrlr,
+				      uint32_t num_queues, nvme_cb_fn_t cb_fn,
+				      void *cb_arg);
+void	nvme_ctrlr_cmd_set_async_event_config(struct nvme_controller *ctrlr,
+		union nvme_critical_warning_state state,
+		nvme_cb_fn_t cb_fn, void *cb_arg);
+void	nvme_ctrlr_cmd_abort(struct nvme_controller *ctrlr, uint16_t cid,
+			     uint16_t sqid, nvme_cb_fn_t cb_fn, void *cb_arg);
+
+void	nvme_completion_poll_cb(void *arg, const struct nvme_completion *cpl);
+
+int	nvme_ctrlr_construct(struct nvme_controller *ctrlr, void *devhandle);
+void	nvme_ctrlr_destruct(struct nvme_controller *ctrlr);
+int	nvme_ctrlr_start(struct nvme_controller *ctrlr);
+int	nvme_ctrlr_hw_reset(struct nvme_controller *ctrlr);
+
+void	nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr,
+					struct nvme_request *req);
+void	nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr,
+				     struct nvme_request *req);
+void	nvme_ctrlr_post_failed_request(struct nvme_controller *ctrlr,
+				       struct nvme_request *req);
+
+int	nvme_qpair_construct(struct nvme_qpair *qpair, uint16_t id,
+			     uint16_t num_entries,
+			     uint16_t num_trackers,
+			     struct nvme_controller *ctrlr);
+void	nvme_qpair_destroy(struct nvme_qpair *qpair);
+void	nvme_qpair_enable(struct nvme_qpair *qpair);
+void	nvme_qpair_disable(struct nvme_qpair *qpair);
+void	nvme_qpair_submit_tracker(struct nvme_qpair *qpair,
+				  struct nvme_tracker *tr);
+void	nvme_qpair_process_completions(struct nvme_qpair *qpair);
+void	nvme_qpair_submit_request(struct nvme_qpair *qpair,
+				  struct nvme_request *req);
+void	nvme_qpair_reset(struct nvme_qpair *qpair);
+void	nvme_qpair_fail(struct nvme_qpair *qpair);
+void	nvme_qpair_manual_complete_request(struct nvme_qpair *qpair,
+		struct nvme_request *req,
+		uint32_t sct, uint32_t sc,
+		bool print_on_error);
+
+int	nvme_ns_construct(struct nvme_namespace *ns, uint16_t id,
+			  struct nvme_controller *ctrlr);
+void	nvme_ns_destruct(struct nvme_namespace *ns);
+
+void	nvme_dump_command(struct nvme_command *cmd);
+void	nvme_dump_completion(struct nvme_completion *cpl);
+
+struct nvme_request *
+nvme_allocate_request(void *payload, uint32_t payload_size,
+		      nvme_cb_fn_t cb_fn, void *cb_arg);
+
+void nvme_cb_complete_child(void *child, const struct nvme_completion *cpl);
+void nvme_request_add_child(struct nvme_request *parent, struct nvme_request *child);
+
+#endif /* __NVME_INTERNAL_H__ */

lib/nvme/nvme_ns.c

@@ -0,0 +1,135 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "nvme_internal.h"
+
+static inline struct nvme_namespace_data *
+_nvme_ns_get_data(struct nvme_namespace *ns)
+{
+	return &ns->ctrlr->nsdata[ns->id - 1];
+}
+
+uint32_t
+nvme_ns_get_id(struct nvme_namespace *ns)
+{
+	return ns->id;
+}
+
+uint32_t
+nvme_ns_get_max_io_xfer_size(struct nvme_namespace *ns)
+{
+	return ns->ctrlr->max_xfer_size;
+}
+
+uint32_t
+nvme_ns_get_sector_size(struct nvme_namespace *ns)
+{
+	return ns->sector_size;
+}
+
+uint64_t
+nvme_ns_get_num_sectors(struct nvme_namespace *ns)
+{
+	return _nvme_ns_get_data(ns)->nsze;
+}
+
+uint64_t
+nvme_ns_get_size(struct nvme_namespace *ns)
+{
+	return nvme_ns_get_num_sectors(ns) * nvme_ns_get_sector_size(ns);
+}
+
+uint32_t
+nvme_ns_get_flags(struct nvme_namespace *ns)
+{
+	return ns->flags;
+}
+
+const struct nvme_namespace_data *
+nvme_ns_get_data(struct nvme_namespace *ns)
+{
+	return _nvme_ns_get_data(ns);
+}
+
+int
+nvme_ns_construct(struct nvme_namespace *ns, uint16_t id,
+		  struct nvme_controller *ctrlr)
+{
+	struct nvme_completion_poll_status	status;
+	struct nvme_namespace_data		*nsdata;
+	uint32_t				pci_devid;
+
+	nvme_assert(id > 0, ("invalid namespace id %d", id));
+
+	ns->ctrlr = ctrlr;
+	ns->id = id;
+	ns->stripe_size = 0;
+
+	nvme_pcicfg_read32(ctrlr->devhandle, &pci_devid, 0);
+	if (pci_devid == INTEL_DC_P3X00_DEVID && ctrlr->cdata.vs[3] != 0) {
+		ns->stripe_size = (1 << ctrlr->cdata.vs[3]) * ctrlr->min_page_size;
+	}
+
+	nsdata = _nvme_ns_get_data(ns);
+
+	status.done = false;
+	nvme_ctrlr_cmd_identify_namespace(ctrlr, id, nsdata,
+					  nvme_completion_poll_cb, &status);
+	while (status.done == false) {
+		nvme_qpair_process_completions(&ctrlr->adminq);
+	}
+	if (nvme_completion_is_error(&status.cpl)) {
+		nvme_printf(ctrlr, "nvme_identify_namespace failed\n");
+		return ENXIO;
+	}
+
+	ns->sector_size = 1 << nsdata->lbaf[nsdata->flbas.format].lbads;
+
+	ns->sectors_per_max_io = nvme_ns_get_max_io_xfer_size(ns) / ns->sector_size;
+	ns->sectors_per_stripe = ns->stripe_size / ns->sector_size;
+
+	if (ctrlr->cdata.oncs.dsm) {
+		ns->flags |= NVME_NS_DEALLOCATE_SUPPORTED;
+	}
+
+	if (ctrlr->cdata.vwc.present) {
+		ns->flags |= NVME_NS_FLUSH_SUPPORTED;
+	}
+
+	return 0;
+}
+
+void nvme_ns_destruct(struct nvme_namespace *ns)
+{
+
+}

lib/nvme/nvme_ns_cmd.c

@@ -0,0 +1,197 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "nvme_internal.h"
+
+/**
+ * \file
+ *
+ */
+
+static struct nvme_request *
+_nvme_ns_cmd_rw(struct nvme_namespace *ns, void *payload, uint64_t lba,
+		uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg,
+		uint32_t opc)
+{
+	struct nvme_request	*req;
+	struct nvme_command	*cmd;
+	uint64_t		*tmp_lba;
+	uint32_t		sector_size;
+	uint32_t		sectors_per_max_io;
+	uint32_t		sectors_per_stripe;
+
+	sector_size = ns->sector_size;
+	sectors_per_max_io = ns->sectors_per_max_io;
+	sectors_per_stripe = ns->sectors_per_stripe;
+
+	req = nvme_allocate_request(payload, lba_count * sector_size, cb_fn, cb_arg);
+	if (req == NULL) {
+		return NULL;
+	}
+
+	/*
+	 * Intel DC P3*00 NVMe controllers benefit from driver-assisted striping.
+	 * If this controller defines a stripe boundary and this I/O spans a stripe
+	 *  boundary, split the request into multiple requests and submit each
+	 *  separately to hardware.
+	 */
+	if (sectors_per_stripe > 0 &&
+	    (((lba & (sectors_per_stripe - 1)) + lba_count) > sectors_per_stripe)) {
+		uint64_t		remaining_lba_count = lba_count;
+		struct nvme_request	*child;
+
+		while (remaining_lba_count > 0) {
+			lba_count = sectors_per_stripe - (lba & (sectors_per_stripe - 1));
+			lba_count = nvme_min(remaining_lba_count, lba_count);
+
+			child = _nvme_ns_cmd_rw(ns, payload, lba, lba_count, cb_fn,
+						cb_arg, opc);
+			if (child == NULL) {
+				nvme_free_request(req);
+				return NULL;
+			}
+			nvme_request_add_child(req, child);
+			remaining_lba_count -= lba_count;
+			lba += lba_count;
+			payload = (void *)((uintptr_t)payload + (lba_count * sector_size));
+		}
+	} else if (lba_count > sectors_per_max_io) {
+		uint64_t		remaining_lba_count = lba_count;
+		struct nvme_request	*child;
+
+		while (remaining_lba_count > 0) {
+			lba_count = nvme_min(remaining_lba_count, sectors_per_max_io);
+			child = _nvme_ns_cmd_rw(ns, payload, lba, lba_count, cb_fn,
+						cb_arg, opc);
+			if (child == NULL) {
+				nvme_free_request(req);
+				return NULL;
+			}
+			nvme_request_add_child(req, child);
+			remaining_lba_count -= lba_count;
+			lba += lba_count;
+			payload = (void *)((uintptr_t)payload + (lba_count * sector_size));
+		}
+	} else {
+		cmd = &req->cmd;
+		cmd->opc = opc;
+		cmd->nsid = ns->id;
+
+		tmp_lba = (uint64_t *)&cmd->cdw10;
+		*tmp_lba = lba;
+		cmd->cdw12 = lba_count - 1;
+	}
+
+	return req;
+}
+
+int
+nvme_ns_cmd_read(struct nvme_namespace *ns, void *payload, uint64_t lba,
+		 uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+
+	req = _nvme_ns_cmd_rw(ns, payload, lba, lba_count, cb_fn, cb_arg, NVME_OPC_READ);
+	if (req != NULL) {
+		nvme_ctrlr_submit_io_request(ns->ctrlr, req);
+		return 0;
+	} else {
+		return ENOMEM;
+	}
+}
+
+int
+nvme_ns_cmd_write(struct nvme_namespace *ns, void *payload, uint64_t lba,
+		  uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+
+	req = _nvme_ns_cmd_rw(ns, payload, lba, lba_count, cb_fn, cb_arg, NVME_OPC_WRITE);
+	if (req != NULL) {
+		nvme_ctrlr_submit_io_request(ns->ctrlr, req);
+		return 0;
+	} else {
+		return ENOMEM;
+	}
+}
+
+int
+nvme_ns_cmd_deallocate(struct nvme_namespace *ns, void *payload,
+		       uint8_t num_ranges, nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+	struct nvme_request	*req;
+	struct nvme_command	*cmd;
+
+	if (num_ranges == 0) {
+		return EINVAL;
+	}
+
+	req = nvme_allocate_request(payload,
+				    num_ranges * sizeof(struct nvme_dsm_range),
+				    cb_fn, cb_arg);
+	if (req == NULL) {
+		return ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = NVME_OPC_DATASET_MANAGEMENT;
+	cmd->nsid = ns->id;
+
+	/* TODO: create a delete command data structure */
+	cmd->cdw10 = num_ranges - 1;
+	cmd->cdw11 = NVME_DSM_ATTR_DEALLOCATE;
+
+	nvme_ctrlr_submit_io_request(ns->ctrlr, req);
+
+	return 0;
+}
+
+int
+nvme_ns_cmd_flush(struct nvme_namespace *ns, nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+	struct nvme_request	*req;
+	struct nvme_command	*cmd;
+
+	req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg);
+	if (req == NULL) {
+		return ENOMEM;
+	}
+
+	cmd = &req->cmd;
+	cmd->opc = NVME_OPC_FLUSH;
+	cmd->nsid = ns->id;
+
+	nvme_ctrlr_submit_io_request(ns->ctrlr, req);
+
+	return 0;
+}

lib/nvme/nvme_qpair.c

@@ -0,0 +1,878 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "nvme_internal.h"
+
+/**
+ * \file
+ *
+ */
+
+static inline bool nvme_qpair_is_admin_queue(struct nvme_qpair *qpair)
+{
+	return qpair->id == 0;
+}
+
+static inline bool nvme_qpair_is_io_queue(struct nvme_qpair *qpair)
+{
+	return qpair->id != 0;
+}
+
+struct nvme_string {
+	uint16_t	value;
+	const char 	*str;
+};
+
+static const struct nvme_string admin_opcode[] = {
+	{ NVME_OPC_DELETE_IO_SQ, "DELETE IO SQ" },
+	{ NVME_OPC_CREATE_IO_SQ, "CREATE IO SQ" },
+	{ NVME_OPC_GET_LOG_PAGE, "GET LOG PAGE" },
+	{ NVME_OPC_DELETE_IO_CQ, "DELETE IO CQ" },
+	{ NVME_OPC_CREATE_IO_CQ, "CREATE IO CQ" },
+	{ NVME_OPC_IDENTIFY, "IDENTIFY" },
+	{ NVME_OPC_ABORT, "ABORT" },
+	{ NVME_OPC_SET_FEATURES, "SET FEATURES" },
+	{ NVME_OPC_GET_FEATURES, "GET FEATURES" },
+	{ NVME_OPC_ASYNC_EVENT_REQUEST, "ASYNC EVENT REQUEST" },
+	{ NVME_OPC_NAMESPACE_MANAGEMENT, "NAMESPACE MANAGEMENT" },
+	{ NVME_OPC_FIRMWARE_COMMIT, "FIRMWARE COMMIT" },
+	{ NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD, "FIRMWARE IMAGE DOWNLOAD" },
+	{ NVME_OPC_NAMESPACE_ATTACHMENT, "NAMESPACE ATTACHMENT" },
+	{ NVME_OPC_FORMAT_NVM, "FORMAT NVM" },
+	{ NVME_OPC_SECURITY_SEND, "SECURITY SEND" },
+	{ NVME_OPC_SECURITY_RECEIVE, "SECURITY RECEIVE" },
+	{ 0xFFFF, "ADMIN COMMAND" }
+};
+
+static const struct nvme_string io_opcode[] = {
+	{ NVME_OPC_FLUSH, "FLUSH" },
+	{ NVME_OPC_WRITE, "WRITE" },
+	{ NVME_OPC_READ, "READ" },
+	{ NVME_OPC_WRITE_UNCORRECTABLE, "WRITE UNCORRECTABLE" },
+	{ NVME_OPC_COMPARE, "COMPARE" },
+	{ NVME_OPC_WRITE_ZEROES, "WRITE ZEROES" },
+	{ NVME_OPC_DATASET_MANAGEMENT, "DATASET MANAGEMENT" },
+	{ NVME_OPC_RESERVATION_REGISTER, "RESERVATION REGISTER" },
+	{ NVME_OPC_RESERVATION_REPORT, "RESERVATION REPORT" },
+	{ NVME_OPC_RESERVATION_ACQUIRE, "RESERVATION ACQUIRE" },
+	{ NVME_OPC_RESERVATION_RELEASE, "RESERVATION RELEASE" },
+	{ 0xFFFF, "IO COMMAND" }
+};
+
+static const char *
+nvme_get_string(const struct nvme_string *strings, uint16_t value)
+{
+	const struct nvme_string *entry;
+
+	entry = strings;
+
+	while (entry->value != 0xFFFF) {
+		if (entry->value == value) {
+			return entry->str;
+		}
+		entry++;
+	}
+	return entry->str;
+}
+
+static void
+nvme_admin_qpair_print_command(struct nvme_qpair *qpair,
+			       struct nvme_command *cmd)
+{
+
+	nvme_printf(qpair->ctrlr, "%s (%02x) sqid:%d cid:%d nsid:%x "
+		    "cdw10:%08x cdw11:%08x\n",
+		    nvme_get_string(admin_opcode, cmd->opc), cmd->opc, qpair->id, cmd->cid,
+		    cmd->nsid, cmd->cdw10, cmd->cdw11);
+}
+
+static void
+nvme_io_qpair_print_command(struct nvme_qpair *qpair,
+			    struct nvme_command *cmd)
+{
+
+	switch ((int)cmd->opc) {
+	case NVME_OPC_WRITE:
+	case NVME_OPC_READ:
+	case NVME_OPC_WRITE_UNCORRECTABLE:
+	case NVME_OPC_COMPARE:
+		nvme_printf(qpair->ctrlr, "%s sqid:%d cid:%d nsid:%d "
+			    "lba:%llu len:%d\n",
+			    nvme_get_string(io_opcode, cmd->opc), qpair->id, cmd->cid,
+			    cmd->nsid,
+			    ((unsigned long long)cmd->cdw11 << 32) + cmd->cdw10,
+			    (cmd->cdw12 & 0xFFFF) + 1);
+		break;
+	case NVME_OPC_FLUSH:
+	case NVME_OPC_DATASET_MANAGEMENT:
+		nvme_printf(qpair->ctrlr, "%s sqid:%d cid:%d nsid:%d\n",
+			    nvme_get_string(io_opcode, cmd->opc), qpair->id, cmd->cid,
+			    cmd->nsid);
+		break;
+	default:
+		nvme_printf(qpair->ctrlr, "%s (%02x) sqid:%d cid:%d nsid:%d\n",
+			    nvme_get_string(io_opcode, cmd->opc), cmd->opc, qpair->id,
+			    cmd->cid, cmd->nsid);
+		break;
+	}
+}
+
+static void
+nvme_qpair_print_command(struct nvme_qpair *qpair, struct nvme_command *cmd)
+{
+	if (nvme_qpair_is_admin_queue(qpair)) {
+		nvme_admin_qpair_print_command(qpair, cmd);
+	} else {
+		nvme_io_qpair_print_command(qpair, cmd);
+	}
+}
+
+static const struct nvme_string generic_status[] = {
+	{ NVME_SC_SUCCESS, "SUCCESS" },
+	{ NVME_SC_INVALID_OPCODE, "INVALID OPCODE" },
+	{ NVME_SC_INVALID_FIELD, "INVALID_FIELD" },
+	{ NVME_SC_COMMAND_ID_CONFLICT, "COMMAND ID CONFLICT" },
+	{ NVME_SC_DATA_TRANSFER_ERROR, "DATA TRANSFER ERROR" },
+	{ NVME_SC_ABORTED_POWER_LOSS, "ABORTED - POWER LOSS" },
+	{ NVME_SC_INTERNAL_DEVICE_ERROR, "INTERNAL DEVICE ERROR" },
+	{ NVME_SC_ABORTED_BY_REQUEST, "ABORTED - BY REQUEST" },
+	{ NVME_SC_ABORTED_SQ_DELETION, "ABORTED - SQ DELETION" },
+	{ NVME_SC_ABORTED_FAILED_FUSED, "ABORTED - FAILED FUSED" },
+	{ NVME_SC_ABORTED_MISSING_FUSED, "ABORTED - MISSING FUSED" },
+	{ NVME_SC_INVALID_NAMESPACE_OR_FORMAT, "INVALID NAMESPACE OR FORMAT" },
+	{ NVME_SC_COMMAND_SEQUENCE_ERROR, "COMMAND SEQUENCE ERROR" },
+	{ NVME_SC_LBA_OUT_OF_RANGE, "LBA OUT OF RANGE" },
+	{ NVME_SC_CAPACITY_EXCEEDED, "CAPACITY EXCEEDED" },
+	{ NVME_SC_NAMESPACE_NOT_READY, "NAMESPACE NOT READY" },
+	{ 0xFFFF, "GENERIC" }
+};
+
+static const struct nvme_string command_specific_status[] = {
+	{ NVME_SC_COMPLETION_QUEUE_INVALID, "INVALID COMPLETION QUEUE" },
+	{ NVME_SC_INVALID_QUEUE_IDENTIFIER, "INVALID QUEUE IDENTIFIER" },
+	{ NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED, "MAX QUEUE SIZE EXCEEDED" },
+	{ NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED, "ABORT CMD LIMIT EXCEEDED" },
+	{ NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED, "ASYNC LIMIT EXCEEDED" },
+	{ NVME_SC_INVALID_FIRMWARE_SLOT, "INVALID FIRMWARE SLOT" },
+	{ NVME_SC_INVALID_FIRMWARE_IMAGE, "INVALID FIRMWARE IMAGE" },
+	{ NVME_SC_INVALID_INTERRUPT_VECTOR, "INVALID INTERRUPT VECTOR" },
+	{ NVME_SC_INVALID_LOG_PAGE, "INVALID LOG PAGE" },
+	{ NVME_SC_INVALID_FORMAT, "INVALID FORMAT" },
+	{ NVME_SC_FIRMWARE_REQUIRES_RESET, "FIRMWARE REQUIRES RESET" },
+	{ NVME_SC_CONFLICTING_ATTRIBUTES, "CONFLICTING ATTRIBUTES" },
+	{ NVME_SC_INVALID_PROTECTION_INFO, "INVALID PROTECTION INFO" },
+	{ NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE, "WRITE TO RO PAGE" },
+	{ 0xFFFF, "COMMAND SPECIFIC" }
+};
+
+static const struct nvme_string media_error_status[] = {
+	{ NVME_SC_WRITE_FAULTS, "WRITE FAULTS" },
+	{ NVME_SC_UNRECOVERED_READ_ERROR, "UNRECOVERED READ ERROR" },
+	{ NVME_SC_GUARD_CHECK_ERROR, "GUARD CHECK ERROR" },
+	{ NVME_SC_APPLICATION_TAG_CHECK_ERROR, "APPLICATION TAG CHECK ERROR" },
+	{ NVME_SC_REFERENCE_TAG_CHECK_ERROR, "REFERENCE TAG CHECK ERROR" },
+	{ NVME_SC_COMPARE_FAILURE, "COMPARE FAILURE" },
+	{ NVME_SC_ACCESS_DENIED, "ACCESS DENIED" },
+	{ 0xFFFF, "MEDIA ERROR" }
+};
+
+static const char *
+get_status_string(uint16_t sct, uint16_t sc)
+{
+	const struct nvme_string *entry;
+
+	switch (sct) {
+	case NVME_SCT_GENERIC:
+		entry = generic_status;
+		break;
+	case NVME_SCT_COMMAND_SPECIFIC:
+		entry = command_specific_status;
+		break;
+	case NVME_SCT_MEDIA_ERROR:
+		entry = media_error_status;
+		break;
+	case NVME_SCT_VENDOR_SPECIFIC:
+		return "VENDOR SPECIFIC";
+	default:
+		return "RESERVED";
+	}
+
+	return nvme_get_string(entry, sc);
+}
+
+static void
+nvme_qpair_print_completion(struct nvme_qpair *qpair,
+			    struct nvme_completion *cpl)
+{
+	nvme_printf(qpair->ctrlr, "%s (%02x/%02x) sqid:%d cid:%d cdw0:%x\n",
+		    get_status_string(cpl->status.sct, cpl->status.sc),
+		    cpl->status.sct, cpl->status.sc, cpl->sqid, cpl->cid, cpl->cdw0);
+}
+
+static bool
+nvme_completion_is_retry(const struct nvme_completion *cpl)
+{
+	/*
+	 * TODO: spec is not clear how commands that are aborted due
+	 *  to TLER will be marked.  So for now, it seems
+	 *  NAMESPACE_NOT_READY is the only case where we should
+	 *  look at the DNR bit.
+	 */
+	switch ((int)cpl->status.sct) {
+	case NVME_SCT_GENERIC:
+		switch ((int)cpl->status.sc) {
+		case NVME_SC_ABORTED_BY_REQUEST:
+		case NVME_SC_NAMESPACE_NOT_READY:
+			if (cpl->status.dnr) {
+				return false;
+			} else {
+				return true;
+			}
+		case NVME_SC_INVALID_OPCODE:
+		case NVME_SC_INVALID_FIELD:
+		case NVME_SC_COMMAND_ID_CONFLICT:
+		case NVME_SC_DATA_TRANSFER_ERROR:
+		case NVME_SC_ABORTED_POWER_LOSS:
+		case NVME_SC_INTERNAL_DEVICE_ERROR:
+		case NVME_SC_ABORTED_SQ_DELETION:
+		case NVME_SC_ABORTED_FAILED_FUSED:
+		case NVME_SC_ABORTED_MISSING_FUSED:
+		case NVME_SC_INVALID_NAMESPACE_OR_FORMAT:
+		case NVME_SC_COMMAND_SEQUENCE_ERROR:
+		case NVME_SC_LBA_OUT_OF_RANGE:
+		case NVME_SC_CAPACITY_EXCEEDED:
+		default:
+			return false;
+		}
+	case NVME_SCT_COMMAND_SPECIFIC:
+	case NVME_SCT_MEDIA_ERROR:
+	case NVME_SCT_VENDOR_SPECIFIC:
+	default:
+		return false;
+	}
+}
+
+static void
+nvme_qpair_construct_tracker(struct nvme_tracker *tr, uint16_t cid, uint64_t phys_addr)
+{
+	tr->prp_bus_addr = phys_addr + offsetof(struct nvme_tracker, prp);
+	tr->cid = cid;
+}
+
+static void
+nvme_qpair_complete_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr,
+			    struct nvme_completion *cpl, bool print_on_error)
+{
+	struct nvme_request	*req;
+	bool			retry, error;
+
+	req = tr->req;
+	error = nvme_completion_is_error(cpl);
+	retry = error && nvme_completion_is_retry(cpl) &&
+		req->retries < nvme_retry_count;
+
+	if (error && print_on_error) {
+		nvme_qpair_print_command(qpair, &req->cmd);
+		nvme_qpair_print_completion(qpair, cpl);
+	}
+
+	qpair->act_tr[cpl->cid] = NULL;
+
+	nvme_assert(cpl->cid == req->cmd.cid, ("cpl cid does not match cmd cid\n"));
+
+	if (req->cb_fn && !retry) {
+		req->cb_fn(req->cb_arg, cpl);
+	}
+
+	if (retry) {
+		req->retries++;
+		nvme_qpair_submit_tracker(qpair, tr);
+	} else {
+		nvme_free_request(req);
+		tr->req = NULL;
+
+		LIST_REMOVE(tr, list);
+		LIST_INSERT_HEAD(&qpair->free_tr, tr, list);
+
+		/*
+		 * If the controller is in the middle of resetting, don't
+		 *  try to submit queued requests here - let the reset logic
+		 *  handle that instead.
+		 */
+		if (!STAILQ_EMPTY(&qpair->queued_req) &&
+		    !qpair->ctrlr->is_resetting) {
+			req = STAILQ_FIRST(&qpair->queued_req);
+			STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
+			nvme_qpair_submit_request(qpair, req);
+		}
+	}
+}
+
+static void
+nvme_qpair_manual_complete_tracker(struct nvme_qpair *qpair,
+				   struct nvme_tracker *tr, uint32_t sct, uint32_t sc, uint32_t dnr,
+				   bool print_on_error)
+{
+	struct nvme_completion	cpl;
+
+	memset(&cpl, 0, sizeof(cpl));
+	cpl.sqid = qpair->id;
+	cpl.cid = tr->cid;
+	cpl.status.sct = sct;
+	cpl.status.sc = sc;
+	cpl.status.dnr = dnr;
+	nvme_qpair_complete_tracker(qpair, tr, &cpl, print_on_error);
+}
+
+void
+nvme_qpair_manual_complete_request(struct nvme_qpair *qpair,
+				   struct nvme_request *req, uint32_t sct, uint32_t sc,
+				   bool print_on_error)
+{
+	struct nvme_completion	cpl;
+	bool			error;
+
+	memset(&cpl, 0, sizeof(cpl));
+	cpl.sqid = qpair->id;
+	cpl.status.sct = sct;
+	cpl.status.sc = sc;
+
+	error = nvme_completion_is_error(&cpl);
+
+	if (error && print_on_error) {
+		nvme_qpair_print_command(qpair, &req->cmd);
+		nvme_qpair_print_completion(qpair, &cpl);
+	}
+
+	if (req->cb_fn) {
+		req->cb_fn(req->cb_arg, &cpl);
+	}
+
+	nvme_free_request(req);
+}
+
+static inline bool
+nvme_qpair_check_enabled(struct nvme_qpair *qpair)
+{
+	if (!qpair->is_enabled &&
+	    !qpair->ctrlr->is_resetting) {
+		nvme_qpair_enable(qpair);
+	}
+	return qpair->is_enabled;
+}
+
+/**
+ * \page nvme_async_completion NVMe Asynchronous Completion
+ *
+ * The userspace NVMe driver follows an asynchronous polled model for
+ * I/O completion.
+ *
+ * \section async_io I/O commands
+ *
+ * The application may submit I/O from one or more threads
+ * and must call nvme_ctrlr_process_io_completions()
+ * from each thread that submitted I/O.
+ *
+ * When the application calls nvme_ctrlr_process_io_completions(),
+ * if the NVMe driver detects completed I/Os that were submitted on that thread,
+ * it will invoke the registered callback function
+ * for each I/O within the context of nvme_ctrlr_process_io_completions().
+ *
+ * \section async_admin Admin commands
+ *
+ * The application may submit admin commands from one or more threads
+ * and must call nvme_ctrlr_process_admin_completions()
+ * from at least one thread to receive admin command completions.
+ * The thread that processes admin completions need not be the same thread that submitted the
+ * admin commands.
+ *
+ * When the application calls nvme_ctrlr_process_admin_completions(),
+ * if the NVMe driver detects completed admin commands submitted from any thread,
+ * it will invote the registered callback function
+ * for each command within the context of nvme_ctrlr_process_admin_completions().
+ *
+ * It is the application's responsibility to manage the order of submitted admin commands.
+ * If certain admin commands must be submitted while no other commands are outstanding,
+ * it is the application's responsibility to enforce this rule
+ * using its own synchronization method.
+ */
+
+/**
+ * \brief Checks for and processes completions on the specified qpair.
+ *
+ * For each completed command, the request's callback function will
+ *  be called if specified as non-NULL when the request was submitted.
+ *
+ * \sa nvme_cb_fn_t
+ */
+void
+nvme_qpair_process_completions(struct nvme_qpair *qpair)
+{
+	struct nvme_tracker	*tr;
+	struct nvme_completion	*cpl;
+
+	if (!nvme_qpair_check_enabled(qpair)) {
+		/*
+		 * qpair is not enabled, likely because a controller reset is
+		 *  is in progress.  Ignore the interrupt - any I/O that was
+		 *  associated with this interrupt will get retried when the
+		 *  reset is complete.
+		 */
+		return;
+	}
+
+	while (1) {
+		cpl = &qpair->cpl[qpair->cq_head];
+
+		if (cpl->status.p != qpair->phase)
+			break;
+
+		tr = qpair->act_tr[cpl->cid];
+
+		if (tr != NULL) {
+			nvme_qpair_complete_tracker(qpair, tr, cpl, true);
+		} else {
+			nvme_printf(qpair->ctrlr,
+				    "cpl does not map to outstanding cmd\n");
+			nvme_dump_completion(cpl);
+			nvme_assert(0, ("received completion for unknown cmd\n"));
+		}
+
+		if (++qpair->cq_head == qpair->num_entries) {
+			qpair->cq_head = 0;
+			qpair->phase = !qpair->phase;
+		}
+
+		_nvme_mmio_write_4(qpair->cq_hdbl, qpair->cq_head);
+	}
+}
+
+int
+nvme_qpair_construct(struct nvme_qpair *qpair, uint16_t id,
+		     uint16_t num_entries, uint16_t num_trackers,
+		     struct nvme_controller *ctrlr)
+{
+	struct nvme_tracker	*tr;
+	uint16_t		i;
+	volatile uint32_t	*doorbell_base;
+	uint64_t		phys_addr = 0;
+
+	qpair->id = id;
+	qpair->num_entries = num_entries;
+
+	qpair->ctrlr = ctrlr;
+
+	/* cmd and cpl rings must be aligned on 4KB boundaries. */
+	qpair->cmd = nvme_malloc("qpair_cmd",
+				 qpair->num_entries * sizeof(struct nvme_command),
+				 0x1000,
+				 &qpair->cmd_bus_addr);
+	if (qpair->cmd == NULL) {
+		nvme_printf(ctrlr, "alloc qpair_cmd failed\n");
+		goto fail;
+	}
+	qpair->cpl = nvme_malloc("qpair_cpl",
+				 qpair->num_entries * sizeof(struct nvme_completion),
+				 0x1000,
+				 &qpair->cpl_bus_addr);
+	if (qpair->cpl == NULL) {
+		nvme_printf(ctrlr, "alloc qpair_cpl failed\n");
+		goto fail;
+	}
+
+	doorbell_base = &ctrlr->regs->doorbell[0].sq_tdbl;
+	qpair->sq_tdbl = doorbell_base + (2 * id + 0) * ctrlr->doorbell_stride_u32;
+	qpair->cq_hdbl = doorbell_base + (2 * id + 1) * ctrlr->doorbell_stride_u32;
+
+	LIST_INIT(&qpair->free_tr);
+	LIST_INIT(&qpair->outstanding_tr);
+	STAILQ_INIT(&qpair->queued_req);
+
+	for (i = 0; i < num_trackers; i++) {
+		/*
+		 * Round alignment up to next power of 2.  This ensures the PRP
+		 *  list embedded in the nvme_tracker object will not span a
+		 *  4KB boundary.
+		 */
+		tr = nvme_malloc("nvme_tr", sizeof(*tr), nvme_align32pow2(sizeof(*tr)), &phys_addr);
+		if (tr == NULL) {
+			nvme_printf(ctrlr, "nvme_tr failed\n");
+			goto fail;
+		}
+		nvme_qpair_construct_tracker(tr, i, phys_addr);
+		LIST_INSERT_HEAD(&qpair->free_tr, tr, list);
+	}
+
+	qpair->act_tr = calloc(num_trackers, sizeof(struct nvme_tracker *));
+	if (qpair->act_tr == NULL) {
+		nvme_printf(ctrlr, "alloc nvme_act_tr failed\n");
+		goto fail;
+	}
+	nvme_qpair_reset(qpair);
+	return 0;
+fail:
+	nvme_qpair_destroy(qpair);
+	return -1;
+}
+
+static void
+nvme_admin_qpair_abort_aers(struct nvme_qpair *qpair)
+{
+	struct nvme_tracker	*tr;
+
+	tr = LIST_FIRST(&qpair->outstanding_tr);
+	while (tr != NULL) {
+		if (tr->req->cmd.opc == NVME_OPC_ASYNC_EVENT_REQUEST) {
+			nvme_qpair_manual_complete_tracker(qpair, tr,
+							   NVME_SCT_GENERIC, NVME_SC_ABORTED_SQ_DELETION, 0,
+							   false);
+			tr = LIST_FIRST(&qpair->outstanding_tr);
+		} else {
+			tr = LIST_NEXT(tr, list);
+		}
+	}
+}
+
+static void
+_nvme_admin_qpair_destroy(struct nvme_qpair *qpair)
+{
+	nvme_admin_qpair_abort_aers(qpair);
+}
+
+static void
+_nvme_io_qpair_destroy(struct nvme_qpair *qpair)
+{
+}
+
+void
+nvme_qpair_destroy(struct nvme_qpair *qpair)
+{
+	struct nvme_tracker	*tr;
+
+	if (nvme_qpair_is_io_queue(qpair)) {
+		_nvme_io_qpair_destroy(qpair);
+	} else {
+		_nvme_admin_qpair_destroy(qpair);
+	}
+	if (qpair->cmd)
+		nvme_free(qpair->cmd);
+	if (qpair->cpl)
+		nvme_free(qpair->cpl);
+	if (qpair->act_tr)
+		free(qpair->act_tr);
+
+	while (!LIST_EMPTY(&qpair->free_tr)) {
+		tr = LIST_FIRST(&qpair->free_tr);
+		LIST_REMOVE(tr, list);
+		nvme_free(tr);
+	}
+}
+
+/**
+ * \page nvme_io_submission NVMe I/O Submission
+ *
+ * I/O is submitted to an NVMe namespace using nvme_ns_cmd_xxx functions
+ * defined in nvme_ns_cmd.c.  The NVMe driver submits the I/O request
+ * as an NVMe submission queue entry on the nvme_qpair associated with
+ * the logical core that submits the I/O.
+ *
+ * \sa nvme_ns_cmd_read, nvme_ns_cmd_write, nvme_ns_cmd_deallocate,
+ *     nvme_ns_cmd_flush, nvme_get_ioq_idx
+ */
+
+
+void
+nvme_qpair_submit_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr)
+{
+	struct nvme_request	*req;
+
+	req = tr->req;
+	qpair->act_tr[tr->cid] = tr;
+
+	/* Copy the command from the tracker to the submission queue. */
+	memcpy(&qpair->cmd[qpair->sq_tail], &req->cmd, sizeof(req->cmd));
+
+	if (++qpair->sq_tail == qpair->num_entries) {
+		qpair->sq_tail = 0;
+	}
+
+	wmb();
+	_nvme_mmio_write_4(qpair->sq_tdbl, qpair->sq_tail);
+}
+
+static void
+_nvme_fail_request_bad_vtophys(struct nvme_qpair *qpair, struct nvme_tracker *tr)
+{
+	/*
+	 * Bad vtophys translation, so abort this request and return
+	 *  immediately.
+	 */
+	nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
+					   NVME_SC_INVALID_FIELD,
+					   1 /* do not retry */, true);
+}
+
+static void
+_nvme_fail_request_ctrlr_failed(struct nvme_qpair *qpair, struct nvme_request *req)
+{
+	nvme_qpair_manual_complete_request(qpair, req, NVME_SCT_GENERIC,
+					   NVME_SC_ABORTED_BY_REQUEST, true);
+}
+
+void
+nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req)
+{
+	struct nvme_tracker	*tr;
+	struct nvme_request	*child_req;
+	uint64_t phys_addr;
+	void *seg_addr;
+	uint32_t nseg, cur_nseg, modulo, unaligned;
+
+	nvme_qpair_check_enabled(qpair);
+
+	if (req->num_children) {
+		/*
+		 * This is a split (parent) request. Submit all of the children but not the parent
+		 * request itself, since the parent is the original unsplit request.
+		 */
+		TAILQ_FOREACH(child_req, &req->children, child_tailq) {
+			nvme_qpair_submit_request(qpair, child_req);
+		}
+		return;
+	}
+
+	tr = LIST_FIRST(&qpair->free_tr);
+
+	if (tr == NULL || !qpair->is_enabled) {
+		/*
+		 * No tracker is available, or the qpair is disabled due to
+		 *  an in-progress controller-level reset or controller
+		 *  failure.
+		 */
+
+		if (qpair->ctrlr->is_failed) {
+			_nvme_fail_request_ctrlr_failed(qpair, req);
+		} else {
+			/*
+			 * Put the request on the qpair's request queue to be
+			 *  processed when a tracker frees up via a command
+			 *  completion or when the controller reset is
+			 *  completed.
+			 */
+			STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq);
+		}
+		return;
+	}
+
+	LIST_REMOVE(tr, list); /* remove tr from free_tr */
+	LIST_INSERT_HEAD(&qpair->outstanding_tr, tr, list);
+	tr->req = req;
+	req->cmd.cid = tr->cid;
+
+	if (req->payload_size) {
+		/*
+		 * Build PRP list describing payload buffer.
+		 */
+
+		phys_addr = nvme_vtophys(req->u.payload);
+		if (phys_addr == -1) {
+			_nvme_fail_request_bad_vtophys(qpair, tr);
+			return;
+		}
+		nseg = req->payload_size >> nvme_u32log2(PAGE_SIZE);
+		modulo = req->payload_size & (PAGE_SIZE - 1);
+		unaligned = phys_addr & (PAGE_SIZE - 1);
+		if (modulo || unaligned) {
+			nseg += 1 + ((modulo + unaligned - 1) >> nvme_u32log2(PAGE_SIZE));
+		}
+
+		tr->req->cmd.psdt = NVME_PSDT_PRP;
+		tr->req->cmd.dptr.prp.prp1 = phys_addr;
+		if (nseg == 2) {
+			seg_addr = req->u.payload + PAGE_SIZE - unaligned;
+			tr->req->cmd.dptr.prp.prp2 = nvme_vtophys(seg_addr);
+		} else if (nseg > 2) {
+			cur_nseg = 1;
+			tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_bus_addr;
+			while (cur_nseg < nseg) {
+				seg_addr = req->u.payload + cur_nseg * PAGE_SIZE - unaligned;
+				phys_addr = nvme_vtophys(seg_addr);
+				if (phys_addr == -1) {
+					_nvme_fail_request_bad_vtophys(qpair, tr);
+					return;
+				}
+				tr->prp[cur_nseg - 1] = phys_addr;
+				cur_nseg++;
+			}
+		}
+	}
+
+	nvme_qpair_submit_tracker(qpair, tr);
+}
+
+void
+nvme_qpair_reset(struct nvme_qpair *qpair)
+{
+	qpair->sq_tail = qpair->cq_head = 0;
+
+	/*
+	 * First time through the completion queue, HW will set phase
+	 *  bit on completions to 1.  So set this to 1 here, indicating
+	 *  we're looking for a 1 to know which entries have completed.
+	 *  we'll toggle the bit each time when the completion queue
+	 *  rolls over.
+	 */
+	qpair->phase = 1;
+
+	memset(qpair->cmd, 0,
+	       qpair->num_entries * sizeof(struct nvme_command));
+	memset(qpair->cpl, 0,
+	       qpair->num_entries * sizeof(struct nvme_completion));
+}
+
+static void
+_nvme_admin_qpair_enable(struct nvme_qpair *qpair)
+{
+	struct nvme_tracker		*tr;
+	struct nvme_tracker		*tr_temp;
+
+	/*
+	 * Manually abort each outstanding admin command.  Do not retry
+	 *  admin commands found here, since they will be left over from
+	 *  a controller reset and its likely the context in which the
+	 *  command was issued no longer applies.
+	 */
+	LIST_FOREACH_SAFE(tr, &qpair->outstanding_tr, list, tr_temp) {
+		nvme_printf(qpair->ctrlr,
+			    "aborting outstanding admin command\n");
+		nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
+						   NVME_SC_ABORTED_BY_REQUEST, 1 /* do not retry */, true);
+	}
+
+	qpair->is_enabled = true;
+}
+
+static void
+_nvme_io_qpair_enable(struct nvme_qpair *qpair)
+{
+	STAILQ_HEAD(, nvme_request)	temp;
+	struct nvme_tracker		*tr;
+	struct nvme_tracker		*tr_temp;
+	struct nvme_request		*req;
+
+	/*
+	 * Manually abort each outstanding I/O.  This normally results in a
+	 *  retry, unless the retry count on the associated request has
+	 *  reached its limit.
+	 */
+	LIST_FOREACH_SAFE(tr, &qpair->outstanding_tr, list, tr_temp) {
+		nvme_printf(qpair->ctrlr, "aborting outstanding i/o\n");
+		nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
+						   NVME_SC_ABORTED_BY_REQUEST, 0, true);
+	}
+
+	qpair->is_enabled = true;
+
+	STAILQ_INIT(&temp);
+	STAILQ_SWAP(&qpair->queued_req, &temp, nvme_request);
+
+	while (!STAILQ_EMPTY(&temp)) {
+		req = STAILQ_FIRST(&temp);
+		STAILQ_REMOVE_HEAD(&temp, stailq);
+
+		nvme_printf(qpair->ctrlr, "resubmitting queued i/o\n");
+		nvme_qpair_print_command(qpair, &req->cmd);
+		nvme_qpair_submit_request(qpair, req);
+	}
+}
+
+void
+nvme_qpair_enable(struct nvme_qpair *qpair)
+{
+	if (nvme_qpair_is_io_queue(qpair)) {
+		_nvme_io_qpair_enable(qpair);
+	} else {
+		_nvme_admin_qpair_enable(qpair);
+	}
+}
+
+static void
+_nvme_admin_qpair_disable(struct nvme_qpair *qpair)
+{
+	qpair->is_enabled = false;
+	nvme_admin_qpair_abort_aers(qpair);
+}
+
+static void
+_nvme_io_qpair_disable(struct nvme_qpair *qpair)
+{
+	qpair->is_enabled = false;
+}
+
+void
+nvme_qpair_disable(struct nvme_qpair *qpair)
+{
+	if (nvme_qpair_is_io_queue(qpair)) {
+		_nvme_io_qpair_disable(qpair);
+	} else {
+		_nvme_admin_qpair_disable(qpair);
+	}
+}
+
+void
+nvme_qpair_fail(struct nvme_qpair *qpair)
+{
+	struct nvme_tracker		*tr;
+	struct nvme_request		*req;
+
+	while (!STAILQ_EMPTY(&qpair->queued_req)) {
+		req = STAILQ_FIRST(&qpair->queued_req);
+		STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
+		nvme_printf(qpair->ctrlr, "failing queued i/o\n");
+		nvme_qpair_manual_complete_request(qpair, req, NVME_SCT_GENERIC,
+						   NVME_SC_ABORTED_BY_REQUEST, true);
+	}
+
+	/* Manually abort each outstanding I/O. */
+	while (!LIST_EMPTY(&qpair->outstanding_tr)) {
+		tr = LIST_FIRST(&qpair->outstanding_tr);
+		/*
+		 * Do not remove the tracker.  The abort_tracker path will
+		 *  do that for us.
+		 */
+		nvme_printf(qpair->ctrlr, "failing outstanding i/o\n");
+		nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
+						   NVME_SC_ABORTED_BY_REQUEST, 1 /* do not retry */, true);
+	}
+}
+

lib/util/Makefile

@@ -0,0 +1,53 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+CFLAGS += $(DPDK_INC)
+
+C_SRCS = pci.c
+
+C_OBJS := $(patsubst %.c,%.o,$(C_SRCS))
+
+OBJS = $(C_OBJS)
+
+all : libspdk_util.a
+
+objs : $(OBJS)
+
+clean :
+	$(Q)rm -f libspdk_util.a $(OBJS) *.d
+
+libspdk_util.a : $(OBJS)
+	$(Q)ar crDs libspdk_util.a $(OBJS)
+

lib/util/pci.c

@@ -0,0 +1,293 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <error.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <linux/limits.h>
+
+#include <pciaccess.h>
+
+#include "spdk/pci.h"
+
+#define SYSFS_PCI_DEVICES	"/sys/bus/pci/devices"
+#define SYSFS_PCI_DRIVERS	"/sys/bus/pci/drivers"
+#define PCI_PRI_FMT		"%04x:%02x:%02x.%1u"
+
+int
+pci_device_get_serial_number(struct pci_device *dev, char *sn, int len)
+{
+	int err;
+	uint32_t pos, header = 0;
+	uint32_t i, buf[2];
+
+	if (len < 17)
+		return -1;
+
+	err = pci_device_cfg_read_u32(dev, &header, PCI_CFG_SIZE);
+	if (err || !header)
+		return -1;
+
+	pos = PCI_CFG_SIZE;
+	while (1) {
+		if ((header & 0x0000ffff) == PCI_EXT_CAP_ID_SN) {
+			if (pos) {
+				/*skip the header*/
+				pos += 4;
+				for (i = 0; i < 2; i++) {
+					err = pci_device_cfg_read_u32(dev,
+								      &buf[i], pos + 4 * i);
+					if (err)
+						return -1;
+				}
+				sprintf(sn, "%08x%08x", buf[1], buf[0]);
+				return 0;
+			}
+		}
+		pos = (header >> 20) & 0xffc;
+		/*0 if no other items exist*/
+		if (pos < PCI_CFG_SIZE)
+			return -1;
+		err = pci_device_cfg_read_u32(dev, &header, pos);
+		if (err)
+			return -1;
+	}
+	return -1;
+}
+
+int
+pci_device_has_uio_driver(struct pci_device *dev)
+{
+	struct dirent *e;
+	DIR *dir;
+	char dirname[PATH_MAX];
+
+	snprintf(dirname, sizeof(dirname),
+		 SYSFS_PCI_DEVICES "/" PCI_PRI_FMT "/uio",
+		 dev->domain, dev->bus, dev->dev, dev->func);
+
+	dir = opendir(dirname);
+	if (!dir) {
+		snprintf(dirname, sizeof(dirname),
+			 SYSFS_PCI_DEVICES "/" PCI_PRI_FMT,
+			 dev->domain, dev->bus, dev->dev, dev->func);
+		dir = opendir(dirname);
+		if (!dir)
+			return 0;
+	}
+
+	while ((e = readdir(dir)) != NULL) {
+		if (strncmp(e->d_name, "uio", 3) == 0) {
+			break;
+		}
+	}
+
+	closedir(dir);
+
+	if (!e)
+		return 0;
+
+	return 1;
+}
+
+int
+pci_device_unbind_kernel_driver(struct pci_device *dev)
+{
+	int n;
+	FILE *fd;
+	char filename[PATH_MAX];
+	char buf[256];
+
+	snprintf(filename, sizeof(filename),
+		 SYSFS_PCI_DEVICES "/" PCI_PRI_FMT "/driver/unbind",
+		 dev->domain, dev->bus, dev->dev, dev->func);
+
+	fd = fopen(filename, "w");
+	if (!fd)
+		return 0;
+
+	n = snprintf(buf, sizeof(buf), PCI_PRI_FMT,
+		     dev->domain, dev->bus, dev->dev, dev->func);
+
+	if (fwrite(buf, n, 1, fd) == 0)
+		goto error;
+
+	fclose(fd);
+	return 0;
+
+error:
+	fclose(fd);
+	return -1;
+}
+
+static int
+check_modules(char *driver_name)
+{
+	FILE *fd;
+	const char *proc_modules = "/proc/modules";
+	char buffer[256];
+
+	fd = fopen(proc_modules, "r");
+	if (!fd)
+		return -1;
+
+	while (fgets(buffer, sizeof(buffer), fd)) {
+		if (strstr(buffer, driver_name) == NULL)
+			continue;
+		else {
+			fclose(fd);
+			return 0;
+		}
+	}
+	fclose(fd);
+
+	return -1;
+}
+
+int
+pci_device_bind_uio_driver(struct pci_device *dev, char *driver_name)
+{
+	int err, n;
+	FILE *fd;
+	char filename[PATH_MAX];
+	char buf[256];
+
+	err = check_modules(driver_name);
+	if (err < 0) {
+		fprintf(stderr, "No %s module loaded\n", driver_name);
+		return err;
+	}
+
+	snprintf(filename, sizeof(filename),
+		 SYSFS_PCI_DRIVERS "/" "%s" "/new_id", driver_name);
+
+	fd = fopen(filename, "w");
+	if (!fd) {
+		return -1;
+	}
+
+	n = snprintf(buf, sizeof(buf), "%04x %04x",
+		     dev->vendor_id, dev->device_id);
+
+	if (fwrite(buf, n, 1, fd) == 0)
+		goto error;
+
+	fclose(fd);
+	return 0;
+
+error:
+	fclose(fd);
+	return -1;
+}
+
+int
+pci_device_switch_to_uio_driver(struct pci_device *pci_dev)
+{
+	if (pci_device_unbind_kernel_driver(pci_dev)) {
+		fprintf(stderr, "Device %s %d:%d:%d unbind from "
+			"kernel driver failed\n",
+			pci_device_get_device_name(pci_dev), pci_dev->bus,
+			pci_dev->dev, pci_dev->func);
+		return -1;
+	}
+	if (pci_device_bind_uio_driver(pci_dev, PCI_UIO_DRIVER)) {
+		fprintf(stderr, "Device %s %d:%d:%d bind to "
+			"uio driver failed\n",
+			pci_device_get_device_name(pci_dev), pci_dev->bus,
+			pci_dev->dev, pci_dev->func);
+		return -1;
+	}
+	printf("Device %s %d:%d:%d bind to uio driver success\n",
+	       pci_device_get_device_name(pci_dev), pci_dev->bus,
+	       pci_dev->dev, pci_dev->func);
+	return 0;
+}
+
+int
+pci_device_claim(struct pci_device *dev)
+{
+	int dev_fd;
+	char shm_name[64];
+	int pid;
+	void *dev_map;
+	struct flock pcidev_lock = {
+		.l_type = F_WRLCK,
+		.l_whence = SEEK_SET,
+		.l_start = 0,
+		.l_len = 0,
+	};
+
+	sprintf(shm_name, PCI_PRI_FMT, dev->domain, dev->bus,
+		dev->dev, dev->func);
+
+	dev_fd = shm_open(shm_name, O_RDWR | O_CREAT, 0600);
+	if (dev_fd == -1) {
+		fprintf(stderr, "could not shm_open %s\n", shm_name);
+		return -1;
+	}
+
+	if (ftruncate(dev_fd, sizeof(int)) != 0) {
+		fprintf(stderr, "could not truncate shm %s\n", shm_name);
+		close(dev_fd);
+		return -1;
+	}
+
+	dev_map = mmap(NULL, sizeof(int), PROT_READ | PROT_WRITE,
+		       MAP_SHARED, dev_fd, 0);
+	if (dev_map == NULL) {
+		fprintf(stderr, "could not mmap shm %s\n", shm_name);
+		close(dev_fd);
+		return -1;
+	}
+
+	if (fcntl(dev_fd, F_SETLK, &pcidev_lock) != 0) {
+		pid = *(int *)dev_map;
+		fprintf(stderr, "Cannot create lock on device %s, probably"
+			" process %d has claimed it\n", shm_name, pid);
+		munmap(dev_map, sizeof(int));
+		close(dev_fd);
+		return -1;
+	}
+
+	*(int *)dev_map = (int)getpid();
+	munmap(dev_map, sizeof(int));
+	/* Keep dev_fd open to maintain the lock. */
+	return 0;
+}

mk/nvme.unittest.mk

@@ -0,0 +1,71 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/..
+NVME_DIR := $(SPDK_ROOT_DIR)/lib/nvme
+
+include $(SPDK_ROOT_DIR)/CONFIG
+
+C_OPT ?= -O2 -fno-omit-frame-pointer
+Q ?= @
+S ?= $(notdir $(CURDIR))
+
+C_SRCS = $(TEST_FILE) $(OTHER_FILES)
+
+OBJS = $(C_SRCS:.c=.o)
+
+CFLAGS += $(C_OPT) -I$(SPDK_ROOT_DIR)/lib -I$(SPDK_ROOT_DIR)/include -include $(SPDK_ROOT_DIR)/test/lib/nvme/unit/nvme_impl.h
+
+LIBS += -lcunit -lpthread
+
+UT_APP = $(TEST_FILE:.c=)
+
+all: $(UT_APP)
+
+$(UT_APP) : $(OBJS)
+	@echo "  LINK $@"
+	$(Q)$(CC) $(CFLAGS) -o $@ $(OBJS) $(LIBS)
+
+clean:
+	$(Q)rm -f $(UT_APP) $(OBJS) *.d
+
+%.o: $(NVME_DIR)/%.c
+	@echo "  CC $@"
+	$(Q)$(CC) $(CFLAGS) -c $< -o $@
+	$(Q)$(CC) -MM $(CFLAGS) $(NVME_DIR)/$*.c > $*.d
+	@mv -f $*.d $*.d.tmp
+	@sed -e 's|.*:|$*.o:|' < $*.d.tmp > $*.d
+	@sed -e 's/.*://' -e 's/\\$$//' < $*.d.tmp | fmt -1 | \
+		sed -e 's/^ *//' -e 's/$$/:/' >> $*.d
+	@rm -f $*.d.tmp
+

mk/spdk.common.mk

@@ -0,0 +1,77 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+include $(SPDK_ROOT_DIR)/CONFIG
+
+C_OPT ?= -fno-omit-frame-pointer
+Q ?= @
+S ?= $(notdir $(CURDIR))
+
+ifeq ($(MAKECMDGOALS),)
+MAKECMDGOALS=$(.DEFAULT_GOAL)
+endif
+
+COMMON_CFLAGS = -g $(C_OPT) -Wall -Werror -fno-strict-aliasing -march=native -m64 -I$(SPDK_ROOT_DIR)/include
+
+ifeq ($(CONFIG_DEBUG), y)
+COMMON_CFLAGS += -DDEBUG -O0
+else
+COMMON_CFLAGS += -O2
+endif
+
+CFLAGS   += $(COMMON_CFLAGS) -Wno-pointer-sign -std=gnu11
+
+MAKEFLAGS += --no-print-directory
+
+.PRECIOUS: $(OBJS)
+
+-include $(OBJS:.o=.d)
+
+%.o : %.c
+	@echo "  CC $@"
+	$(Q)$(CC) $(CFLAGS) -c $<
+	$(Q)$(CC) -MM $(CFLAGS) $*.c > $*.d
+	@mv -f $*.d $*.d.tmp
+	@sed -e 's|.*:|$*.o:|' < $*.d.tmp > $*.d
+	@sed -e 's/.*://' -e 's/\\$$//' < $*.d.tmp | fmt -1 | \
+		sed -e 's/^ *//' -e 's/$$/:/' >> $*.d
+	@rm -f $*.d.tmp
+
+DPDK_DIR ?= $(CONFIG_DPDK_DIR)
+DPDK_INC_DIR ?= $(DPDK_DIR)/include
+DPDK_LIB_DIR ?= $(DPDK_DIR)/lib
+
+DPDK_INC = -I$(DPDK_INC_DIR)
+# DPDK requires dl library for dlopen/dlclose.
+DPDK_LIB = -L$(DPDK_LIB_DIR) -lrte_eal -lrte_malloc -lrte_mempool -lrte_ring -ldl -Wl,-rpath=$(DPDK_LIB_DIR)
+

mk/spdk.subdirs.mk

@@ -0,0 +1,3 @@
+$(DIRS-y) :
+	@echo "== $S/$@ ($(MAKECMDGOALS))"
+	$(Q)$(MAKE) -C $@ S=$S/$@ $(MAKECMDGOALS) $(MAKESUBDIRFLAGS)

scripts/autotest_common.sh

@@ -0,0 +1,55 @@
+set -xe
+ulimit -c unlimited
+
+if [ -z "$rootdir" ] || [ ! -d "$rootdir/../output" ]; then
+	output_dir=.
+else
+	output_dir=$rootdir/../output
+fi
+
+function timing() {
+	direction="$1"
+	testname="$2"
+
+	now=$(date +%s)
+
+	if [ "$direction" = "enter" ]; then
+		export timing_stack="${timing_stack}/${now}"
+		export test_stack="${test_stack}/${testname}"
+	else
+		start_time=$(echo "$timing_stack" | sed -e 's@^.*/@@')
+		timing_stack=$(echo "$timing_stack" | sed -e 's@/[^/]*$@@')
+
+		elapsed=$((now - start_time))
+		echo "$elapsed $test_stack" >> $output_dir/timing.txt
+
+		test_stack=$(echo "$test_stack" | sed -e 's@/[^/]*$@@')
+	fi
+}
+
+function timing_enter() {
+	timing "enter" "$1"
+}
+
+function timing_exit() {
+	timing "exit" "$1"
+}
+
+function process_core() {
+	ret=0
+	for core in $(find . -type f -name 'core*'); do
+		exe=$(eu-readelf -n "$core" | grep psargs | awk '{ print $2 }')
+		echo "exe for $core is $exe"
+		if [[ ! -z "$exe" ]]; then
+			if hash gdb; then
+				gdb -batch -ex "bt" $exe $core
+			fi
+			cp $exe $output_dir
+		fi
+		mv $core $output_dir
+		chmod a+r $output_dir/$core
+		ret=1
+	done
+	return $ret
+}
+

test/Makefile

@@ -0,0 +1,46 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+# These directories contain tests.
+TESTDIRS = lib
+
+DIRS-y = $(TESTDIRS)
+
+.PHONY: all clean $(DIRS-y)
+
+all: $(DIRS-y)
+clean: $(DIRS-y)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk

test/lib/Makefile

@@ -0,0 +1,43 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+DIRS-y = nvme memory
+
+.PHONY: all clean $(DIRS-y)
+
+all: $(DIRS-y)
+clean: $(DIRS-y)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk

test/lib/memory/.gitignore

@@ -0,0 +1 @@
+vtophys

test/lib/memory/Makefile

@@ -0,0 +1,48 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+C_SRCS = vtophys.c
+
+CFLAGS += $(DPDK_INC)
+OBJS =	$(SPDK_ROOT_DIR)/lib/memory/vtophys.o
+
+all: vtophys
+
+vtophys: vtophys.o $(OBJS)
+	@echo "  LINK $@"
+	$(Q)$(CC) $(LDFLAGS) -o vtophys vtophys.o $(OBJS) -lpthread $(DPDK_LIB) -lrt
+
+clean:
+	$(Q)rm -f vtophys vtophys.o *.d

test/lib/memory/memory.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+testdir=$(readlink -f $(dirname $0))
+rootdir="$testdir/../../.."
+source $rootdir/scripts/autotest_common.sh
+
+$testdir/vtophys
+process_core

test/lib/memory/vtophys.c

@@ -0,0 +1,136 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <rte_config.h>
+#include <rte_eal.h>
+#include <rte_debug.h>
+#include <rte_mempool.h>
+#include <rte_malloc.h>
+
+#include "spdk/vtophys.h"
+
+static const char *ealargs[] = {
+	"vtophys",
+	"-c 0x1",
+	"-n 4",
+};
+
+static int
+vtophys_negative_test()
+{
+	void *p = NULL;
+	int i;
+	unsigned int size = 1;
+	int rc = 0;
+
+	for (i = 0; i < 31; i++) {
+		p = malloc(size);
+		if (p == NULL)
+			continue;
+
+		if (vtophys(p) != -1) {
+			rc = -1;
+			printf("Err: VA=%p is mapped to a huge_page,\n", p);
+			free(p);
+			break;
+		}
+
+		free(p);
+		size = size << 1;
+	}
+
+	if (!rc)
+		printf("vtophys_negative_test passed\n");
+	else
+		printf("vtophys_negative_test failed\n");
+
+	return rc;
+}
+
+static int
+vtophys_positive_test()
+{
+	void *p = NULL;
+	int i;
+	unsigned int size = 1;
+	int rc = 0;
+
+	for (i = 0; i < 31; i++) {
+		p = rte_malloc("vtophys_test", size, 512);
+		if (p == NULL)
+			continue;
+
+		if (vtophys(p) == -1) {
+			rc = -1;
+			printf("Err: VA=%p is not mapped to a huge_page,\n", p);
+			rte_free(p);
+			break;
+		}
+
+		rte_free(p);
+		size = size << 1;
+	}
+
+	if (!rc)
+		printf("vtophys_positive_test passed\n");
+	else
+		printf("vtophys_positive_test failed\n");
+
+	return rc;
+}
+
+
+int
+main(int argc, char **argv)
+{
+	int rc;
+
+	rc = rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]),
+			  (char **)(void *)(uintptr_t)ealargs);
+
+	if (rc < 0) {
+		fprintf(stderr, "Could not init eal\n");
+		exit(1);
+	}
+
+	rc = vtophys_negative_test();
+	if (rc < 0)
+		return rc;
+
+	rc = vtophys_positive_test();
+	return rc;
+}

test/lib/nvme/Makefile

@@ -0,0 +1,43 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+DIRS-y = unit aer
+
+.PHONY: all clean $(DIRS-y)
+
+all: $(DIRS-y)
+clean: $(DIRS-y)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk

test/lib/nvme/aer/.gitignore

@@ -0,0 +1 @@
+aer

test/lib/nvme/aer/Makefile

@@ -0,0 +1,59 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+APP = aer
+
+C_SRCS := aer.c
+
+CFLAGS += -I. $(DPDK_INC)
+
+SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/nvme/libspdk_nvme.a \
+	     $(SPDK_ROOT_DIR)/lib/util/libspdk_util.a \
+	     $(SPDK_ROOT_DIR)/lib/memory/libspdk_memory.a
+
+LIBS += $(SPDK_LIBS) -lpciaccess -lpthread $(DPDK_LIB) -lrt
+
+OBJS  = $(C_SRCS:.c=.o)
+
+all : $(APP)
+
+objs : $(OBJS)
+
+$(APP) : $(OBJS) $(SPDK_LIBS)
+	@echo "  LINK $@"
+	$(Q)$(CC) $(LDFLAGS) -o $@ $(OBJS) $(LIBS)
+
+clean :
+	$(Q)rm -f $(OBJS) *.d $(APP)

test/lib/nvme/aer/aer.c

@@ -0,0 +1,301 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdbool.h>
+
+#include <pciaccess.h>
+
+#include <rte_config.h>
+#include <rte_malloc.h>
+#include <rte_mempool.h>
+#include <rte_lcore.h>
+
+#include "spdk/nvme.h"
+#include "spdk/pci.h"
+
+struct rte_mempool *request_mempool;
+
+#define MAX_DEVS 64
+
+struct dev {
+	struct pci_device			*pci_dev;
+	struct nvme_controller 			*ctrlr;
+	struct nvme_health_information_page	*health_page;
+	uint32_t				orig_temp_threshold;
+	char 					name[100];
+};
+
+static struct dev devs[MAX_DEVS];
+static int num_devs = 0;
+
+static int aer_done = 0;
+
+
+#define foreach_dev(iter) \
+	for (iter = devs; iter - devs < num_devs; iter++)
+
+
+static int temperature_done = 0;
+
+static void set_feature_completion(void *arg, const struct nvme_completion *cpl)
+{
+	/* Admin command completions are synchronized by the NVMe driver,
+	 * so we don't need to do any special locking here. */
+	temperature_done++;
+}
+
+static int
+set_temp_threshold(struct dev *dev, uint32_t temp)
+{
+	struct nvme_command cmd = {0};
+
+	cmd.opc = NVME_OPC_SET_FEATURES;
+	cmd.cdw10 = NVME_FEAT_TEMPERATURE_THRESHOLD;
+	cmd.cdw11 = temp;
+
+	return nvme_ctrlr_cmd_admin_raw(dev->ctrlr, &cmd, NULL, 0, set_feature_completion, dev);
+}
+
+static void
+get_feature_completion(void *cb_arg, const struct nvme_completion *cpl)
+{
+	struct dev *dev = cb_arg;
+
+	if (nvme_completion_is_error(cpl)) {
+		printf("%s: get feature (temp threshold) failed\n", dev->name);
+	} else {
+		dev->orig_temp_threshold = cpl->cdw0;
+		printf("%s: original temperature threshold: %u Kelvin (%d Celsius)\n",
+		       dev->name, dev->orig_temp_threshold, dev->orig_temp_threshold - 273);
+	}
+
+	/* Set temperature threshold to a low value so the AER will trigger. */
+	set_temp_threshold(dev, 200);
+}
+
+static int
+get_temp_threshold(struct dev *dev)
+{
+	struct nvme_command cmd = {0};
+
+	cmd.opc = NVME_OPC_GET_FEATURES;
+	cmd.cdw10 = NVME_FEAT_TEMPERATURE_THRESHOLD;
+
+	return nvme_ctrlr_cmd_admin_raw(dev->ctrlr, &cmd, NULL, 0, get_feature_completion, dev);
+}
+
+static void
+print_health_page(struct dev *dev, struct nvme_health_information_page *hip)
+{
+	printf("%s: Current Temperature:         %u Kelvin (%d Celsius)\n",
+	       dev->name, hip->temperature, hip->temperature - 273);
+}
+
+static void
+get_log_page_completion(void *cb_arg, const struct nvme_completion *cpl)
+{
+	struct dev *dev = cb_arg;
+
+	if (nvme_completion_is_error(cpl)) {
+		printf("%s: get log page failed\n", dev->name);
+	} else {
+		print_health_page(dev, dev->health_page);
+	}
+
+	aer_done++;
+}
+
+static int
+get_health_log_page(struct dev *dev)
+{
+	struct nvme_command cmd = {0};
+
+	cmd.opc = NVME_OPC_GET_LOG_PAGE;
+	cmd.cdw10 = NVME_LOG_HEALTH_INFORMATION;
+	cmd.cdw10 |= (sizeof(*(dev->health_page)) / 4) << 16; // number of dwords
+	cmd.nsid = NVME_GLOBAL_NAMESPACE_TAG;
+
+	return nvme_ctrlr_cmd_admin_raw(dev->ctrlr, &cmd, dev->health_page, sizeof(*dev->health_page),
+					get_log_page_completion, dev);
+}
+
+static void
+cleanup(void)
+{
+	struct dev *dev;
+
+	foreach_dev(dev) {
+		if (dev->health_page) {
+			rte_free(dev->health_page);
+		}
+	}
+}
+
+static void aer_cb(void *arg, const struct nvme_completion *cpl)
+{
+	uint32_t log_page_id = (cpl->cdw0 & 0xFF0000) >> 16;
+	struct dev *dev = arg;
+
+	printf("%s: aer_cb for log page %d\n", dev->name, log_page_id);
+
+	/* Set the temperature threshold back to the original value
+	 * so the AER doesn't trigger again.
+	 */
+	set_temp_threshold(dev, dev->orig_temp_threshold);
+
+	get_health_log_page(dev);
+}
+
+static const char *ealargs[] = {
+	"aer",
+	"-c 0x1",
+	"-n 4",
+};
+
+int main(int argc, char **argv)
+{
+	struct pci_device_iterator	*pci_dev_iter;
+	struct pci_device		*pci_dev;
+	struct dev			*dev;
+	struct pci_id_match		match;
+	int				rc, i;
+
+	printf("Asynchronous Event Request test\n");
+
+	rc = rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]),
+			  (char **)(void *)(uintptr_t)ealargs);
+
+	if (rc < 0) {
+		fprintf(stderr, "could not initialize dpdk\n");
+		exit(1);
+	}
+
+	request_mempool = rte_mempool_create("nvme_request", 8192,
+					     nvme_request_size(), 128, 0,
+					     NULL, NULL, NULL, NULL,
+					     SOCKET_ID_ANY, 0);
+
+	if (request_mempool == NULL) {
+		fprintf(stderr, "could not initialize request mempool\n");
+		exit(1);
+	}
+
+	pci_system_init();
+
+	match.vendor_id =	PCI_MATCH_ANY;
+	match.subvendor_id =	PCI_MATCH_ANY;
+	match.subdevice_id =	PCI_MATCH_ANY;
+	match.device_id =	PCI_MATCH_ANY;
+	match.device_class =	NVME_CLASS_CODE;
+	match.device_class_mask = 0xFFFFFF;
+
+	pci_dev_iter = pci_id_match_iterator_create(&match);
+
+	rc = 0;
+	while ((pci_dev = pci_device_next(pci_dev_iter))) {
+		struct dev *dev;
+
+		if (pci_device_has_kernel_driver(pci_dev) &&
+		    !pci_device_has_uio_driver(pci_dev)) {
+			fprintf(stderr, "non-uio kernel driver attached to nvme\n");
+			fprintf(stderr, " controller at pci bdf %d:%d:%d\n",
+				pci_dev->bus, pci_dev->dev, pci_dev->func);
+			fprintf(stderr, " skipping...\n");
+			continue;
+		}
+
+		pci_device_probe(pci_dev);
+
+		/* add to dev list */
+		dev = &devs[num_devs++];
+
+		dev->pci_dev = pci_dev;
+
+		snprintf(dev->name, sizeof(dev->name), "%04X:%02X:%02X.%02X",
+			 pci_dev->domain, pci_dev->bus, pci_dev->dev, pci_dev->func);
+
+		printf("%s: attaching NVMe driver...\n", dev->name);
+
+		dev->health_page = rte_zmalloc("nvme health", sizeof(*dev->health_page), 4096);
+		if (dev->health_page == NULL) {
+			printf("Allocation error (health page)\n");
+			rc = 1;
+			continue; /* TODO: just abort */
+		}
+
+		dev->ctrlr = nvme_attach(pci_dev);
+		if (dev->ctrlr == NULL) {
+			fprintf(stderr, "failed to attach to NVMe controller %s\n", dev->name);
+			rc = 1;
+			continue; /* TODO: just abort */
+		}
+	}
+
+	printf("Registering asynchronous event callbacks...\n");
+	foreach_dev(dev) {
+		nvme_ctrlr_register_aer_callback(dev->ctrlr, aer_cb, dev);
+	}
+
+	printf("Setting temperature thresholds...\n");
+	foreach_dev(dev) {
+		/* Get the original temperature threshold and set it to a low value */
+		get_temp_threshold(dev);
+	}
+
+	while (temperature_done < num_devs) {
+		foreach_dev(dev) {
+			nvme_ctrlr_process_admin_completions(dev->ctrlr);
+		}
+	}
+
+	printf("Waiting for all controllers to trigger AER...\n");
+
+	while (aer_done < num_devs) {
+		foreach_dev(dev) {
+			nvme_ctrlr_process_admin_completions(dev->ctrlr);
+		}
+	}
+
+	printf("Cleaning up...\n");
+
+	for (i = 0; i < num_devs; i++) {
+		struct dev *dev = &devs[i];
+
+		nvme_detach(dev->ctrlr);
+	}
+
+	cleanup();
+
+	pci_iterator_destroy(pci_dev_iter);
+	return rc;
+}

test/lib/nvme/nvme.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+testdir=$(readlink -f $(dirname $0))
+rootdir="$testdir/../../.."
+source $rootdir/scripts/autotest_common.sh
+
+$testdir/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut
+$testdir/unit/nvme_c/nvme_ut
+$testdir/unit/nvme_qpair_c/nvme_qpair_ut
+$testdir/unit/nvme_ctrlr_c/nvme_ctrlr_ut
+$testdir/unit/nvme_ctrlr_cmd_c/nvme_ctrlr_cmd_ut
+
+$testdir/aer/aer
+process_core
+
+$rootdir/examples/nvme/identify/identify
+process_core
+
+$rootdir/examples/nvme/perf/perf -q 128 -w read -s 4096 -t 5
+process_core

test/lib/nvme/unit/Makefile

@@ -0,0 +1,43 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+DIRS-y = nvme_c nvme_ns_cmd_c nvme_qpair_c nvme_ctrlr_c nvme_ctrlr_cmd_c
+
+.PHONY: all clean $(DIRS-y)
+
+all: $(DIRS-y)
+clean: $(DIRS-y)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk

test/lib/nvme/unit/nvme_c/.gitignore

@@ -0,0 +1 @@
+nvme_ut

test/lib/nvme/unit/nvme_c/Makefile

@@ -0,0 +1,37 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+TEST_FILE = nvme_ut.c
+
+include $(SPDK_ROOT_DIR)/mk/nvme.unittest.mk
+

test/lib/nvme/unit/nvme_c/nvme_ut.c

@@ -0,0 +1,251 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "CUnit/Basic.h"
+
+#include "nvme/nvme.c"
+
+char outbuf[OUTBUF_SIZE];
+
+volatile int sync_start = 0;
+volatile int threads_pass = 0;
+volatile int threads_fail = 0;
+
+uint64_t nvme_vtophys(void *buf)
+{
+	return (uintptr_t)buf;
+}
+
+int
+nvme_ctrlr_hw_reset(struct nvme_controller *ctrlr)
+{
+	return 0;
+}
+
+int
+nvme_ctrlr_construct(struct nvme_controller *ctrlr, void *devhandle)
+{
+	return 0;
+}
+
+void
+nvme_ctrlr_destruct(struct nvme_controller *ctrlr)
+{
+}
+
+int
+nvme_ctrlr_start(struct nvme_controller *ctrlr)
+{
+	return 0;
+}
+
+void prepare_for_test(uint32_t max_io_queues)
+{
+	struct nvme_driver *driver = &g_nvme_driver;
+
+	driver->max_io_queues = max_io_queues;
+	if (driver->ioq_index_pool != NULL) {
+		free(driver->ioq_index_pool);
+		driver->ioq_index_pool = NULL;
+	}
+	driver->ioq_index_pool_next = 0;
+	nvme_thread_ioq_index = -1;
+
+	sync_start = 0;
+	threads_pass = 0;
+	threads_fail = 0;
+}
+
+void *
+nvme_thread(void *arg)
+{
+	int rc;
+
+	/* Try to synchronize the nvme_register_io_thread() calls
+	 *  as much as possible to ensure the mutex locking is tested
+	 *  correctly.
+	 */
+	while (sync_start == 0)
+		;
+
+	rc = nvme_register_io_thread();
+	if (rc == 0) {
+		__sync_fetch_and_add(&threads_pass, 1);
+	} else {
+		__sync_fetch_and_add(&threads_fail, 1);
+	}
+
+	pthread_exit(NULL);
+}
+
+void
+test1(void)
+{
+	struct nvme_driver *driver = &g_nvme_driver;
+	int rc;
+	int last_index;
+
+	prepare_for_test(1);
+
+	CU_ASSERT(nvme_thread_ioq_index == -1);
+
+	rc = nvme_register_io_thread();
+	CU_ASSERT(rc == 0);
+	CU_ASSERT(nvme_thread_ioq_index >= 0);
+	CU_ASSERT(driver->ioq_index_pool_next == 1);
+
+	/* try to register thread again - this should fail */
+	last_index = nvme_thread_ioq_index;
+	rc = nvme_register_io_thread();
+	CU_ASSERT(rc != 0);
+	/* assert that the ioq_index was unchanged */
+	CU_ASSERT(nvme_thread_ioq_index == last_index);
+
+	nvme_unregister_io_thread();
+	CU_ASSERT(nvme_thread_ioq_index == -1);
+	CU_ASSERT(driver->ioq_index_pool_next == 0);
+}
+
+void
+test2(void)
+{
+	int num_threads = 16;
+	int i;
+	pthread_t td;
+
+	/*
+	 * Start 16 threads, but only simulate a maximum of 12 I/O
+	 *  queues.  12 threads should be able to successfully
+	 *  register, while the other 4 should fail.
+	 */
+	prepare_for_test(12);
+
+	for (i = 0; i < num_threads; i++) {
+		pthread_create(&td, NULL, nvme_thread, NULL);
+	}
+
+	sync_start = 1;
+
+	while ((threads_pass + threads_fail) < num_threads)
+		;
+
+	CU_ASSERT(threads_pass == 12);
+	CU_ASSERT(threads_fail == 4);
+}
+
+void
+test_nvme_dump_command(void)
+{
+	struct nvme_command	*cmd = NULL;
+	uint64_t		physaddr = 0;
+
+	cmd = nvme_malloc("nvme_command", sizeof(struct nvme_command),
+			  64, &physaddr);
+	CU_ASSERT(cmd != NULL);
+
+	cmd->opc = 1;
+	cmd->fuse = 1;
+	cmd->rsvd1 = 1;
+	cmd->cid = 1;
+	cmd->nsid = 1;
+	cmd->rsvd2 = 1;
+	cmd->rsvd3 = 1;
+	cmd->mptr = 1;
+	cmd->dptr.prp.prp1 = 1;
+	cmd->dptr.prp.prp2 = 1;
+	cmd->cdw10 = 1;
+	cmd->cdw11 = 1;
+	cmd->cdw12 = 1;
+	cmd->cdw13 = 1;
+	cmd->cdw14 = 1;
+	cmd->cdw15 = 1;
+
+	nvme_dump_command(cmd);
+	nvme_free(cmd);
+}
+
+void
+test_nvme_dump_completion(void)
+{
+	struct nvme_completion	*cpl = NULL;
+	uint64_t		physaddr = 0;
+
+	cpl = nvme_malloc("nvme_completion", sizeof(struct nvme_completion),
+			  64, &physaddr);
+	CU_ASSERT(cpl != NULL);
+
+	cpl->cdw0 = 1;
+	cpl->sqhd = 1;
+	cpl->sqid = 1;
+	cpl->cid = 1;
+	cpl->status.p = 1;
+	cpl->status.sc = 1;
+	cpl->status.sct = 1;
+	cpl->status.m = 1;
+	cpl->status.dnr = 1;
+
+	nvme_dump_completion(cpl);
+	nvme_free(cpl);
+}
+
+int main(int argc, char **argv)
+{
+	CU_pSuite	suite = NULL;
+	unsigned int	num_failures;
+
+	if (CU_initialize_registry() != CUE_SUCCESS) {
+		return CU_get_error();
+	}
+
+	suite = CU_add_suite("nvme", NULL, NULL);
+	if (suite == NULL) {
+		CU_cleanup_registry();
+		return CU_get_error();
+	}
+
+	if (
+		CU_add_test(suite, "test1", test1) == NULL
+		|| CU_add_test(suite, "test2", test2) == NULL
+		|| CU_add_test(suite, "nvme_dump_command", test_nvme_dump_command) == NULL
+		|| CU_add_test(suite, "nvme_dump_completion", test_nvme_dump_completion) == NULL
+	) {
+		CU_cleanup_registry();
+		return CU_get_error();
+	}
+
+	CU_basic_set_mode(CU_BRM_VERBOSE);
+	CU_basic_run_tests();
+	num_failures = CU_get_number_of_failures();
+	CU_cleanup_registry();
+	return num_failures;
+}

test/lib/nvme/unit/nvme_ctrlr_c/.gitignore

@@ -0,0 +1 @@
+nvme_ctrlr_ut

test/lib/nvme/unit/nvme_ctrlr_c/Makefile

@@ -0,0 +1,37 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+TEST_FILE = nvme_ctrlr_ut.c
+
+include $(SPDK_ROOT_DIR)/mk/nvme.unittest.mk
+

test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c

@@ -0,0 +1,207 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "nvme/nvme_internal.h"
+
+#include "CUnit/Basic.h"
+
+#include "nvme/nvme_ctrlr.c"
+
+struct nvme_driver g_nvme_driver = {
+	.lock = NVME_MUTEX_INITIALIZER,
+	.max_io_queues = NVME_MAX_IO_QUEUES
+};
+
+char outbuf[OUTBUF_SIZE];
+
+int __thread    nvme_thread_ioq_index = -1;
+
+int nvme_qpair_construct(struct nvme_qpair *qpair, uint16_t id,
+			 uint16_t num_entries, uint16_t num_trackers,
+			 struct nvme_controller *ctrlr)
+{
+	return 0;
+}
+
+void
+nvme_qpair_fail(struct nvme_qpair *qpair)
+{
+}
+
+void
+nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req)
+{
+	CU_ASSERT(req->cmd.opc == NVME_OPC_ASYNC_EVENT_REQUEST);
+}
+
+void
+nvme_qpair_process_completions(struct nvme_qpair *qpair)
+{
+}
+
+void
+nvme_qpair_disable(struct nvme_qpair *qpair)
+{
+}
+
+void
+nvme_qpair_destroy(struct nvme_qpair *qpair)
+{
+}
+
+void
+nvme_qpair_enable(struct nvme_qpair *qpair)
+{
+}
+
+void
+nvme_qpair_reset(struct nvme_qpair *qpair)
+{
+}
+
+void
+nvme_completion_poll_cb(void *arg, const struct nvme_completion *cpl)
+{
+}
+
+void
+nvme_ctrlr_cmd_identify_controller(struct nvme_controller *ctrlr, void *payload,
+				   nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+}
+
+void
+nvme_ctrlr_cmd_set_num_queues(struct nvme_controller *ctrlr,
+			      uint32_t num_queues, nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+}
+
+void
+nvme_ctrlr_cmd_create_io_cq(struct nvme_controller *ctrlr,
+			    struct nvme_qpair *io_que, nvme_cb_fn_t cb_fn,
+			    void *cb_arg)
+{
+}
+
+void
+nvme_ctrlr_cmd_create_io_sq(struct nvme_controller *ctrlr,
+			    struct nvme_qpair *io_que, nvme_cb_fn_t cb_fn,
+			    void *cb_arg)
+{
+}
+
+void
+nvme_ns_destruct(struct nvme_namespace *ns)
+{
+}
+
+int
+nvme_ns_construct(struct nvme_namespace *ns, uint16_t id,
+		  struct nvme_controller *ctrlr)
+{
+	return 0;
+}
+
+
+struct nvme_request *
+nvme_allocate_request(void *payload, uint32_t payload_size,
+		      nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+	struct nvme_request *req = NULL;
+	nvme_alloc_request(&req);
+
+	if (req != NULL) {
+		memset(req, 0, offsetof(struct nvme_request, children));
+
+		if (payload == NULL || payload_size == 0) {
+			req->u.payload = NULL;
+			req->payload_size = 0;
+		} else {
+			req->u.payload = payload;
+			req->payload_size = payload_size;
+		}
+
+		req->cb_fn = cb_fn;
+		req->cb_arg = cb_arg;
+		req->timeout = true;
+	}
+	return req;
+}
+
+void
+test_nvme_ctrlr_fail()
+{
+	struct nvme_controller *ctrlr = NULL;
+	struct nvme_qpair	qpair = {};
+	uint64_t		phys_addr = 0;
+
+	ctrlr = nvme_malloc("nvme_controller", sizeof(struct nvme_controller),
+			    64, &phys_addr);
+	CU_ASSERT(ctrlr != NULL);
+
+	ctrlr->num_io_queues = 0;
+	ctrlr->adminq = qpair;
+	nvme_ctrlr_fail(ctrlr);
+
+	CU_ASSERT(ctrlr->is_failed == true);
+	nvme_free(ctrlr);
+}
+
+int main(int argc, char **argv)
+{
+	CU_pSuite	suite = NULL;
+	unsigned int	num_failures;
+
+	if (CU_initialize_registry() != CUE_SUCCESS) {
+		return CU_get_error();
+	}
+
+	suite = CU_add_suite("nvme_ctrlr", NULL, NULL);
+	if (suite == NULL) {
+		CU_cleanup_registry();
+		return CU_get_error();
+	}
+
+	if (
+		CU_add_test(suite, "test nvme_ctrlr function nvme_ctrlr_fail", test_nvme_ctrlr_fail) == NULL
+	) {
+		CU_cleanup_registry();
+		return CU_get_error();
+	}
+
+	CU_basic_set_mode(CU_BRM_VERBOSE);
+	CU_basic_run_tests();
+	num_failures = CU_get_number_of_failures();
+	CU_cleanup_registry();
+	return num_failures;
+}

test/lib/nvme/unit/nvme_ctrlr_cmd_c/.gitignore

@@ -0,0 +1 @@
+nvme_ctrlr_cmd_ut

test/lib/nvme/unit/nvme_ctrlr_cmd_c/Makefile

@@ -0,0 +1,37 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+TEST_FILE = nvme_ctrlr_cmd_ut.c
+
+include $(SPDK_ROOT_DIR)/mk/nvme.unittest.mk
+

test/lib/nvme/unit/nvme_ctrlr_cmd_c/nvme_ctrlr_cmd_ut.c

@@ -0,0 +1,310 @@
+
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdbool.h>
+#include "nvme/nvme_internal.h"
+
+#include "CUnit/Basic.h"
+
+#include "nvme/nvme_ctrlr_cmd.c"
+
+char outbuf[OUTBUF_SIZE];
+
+struct nvme_command *cmd = NULL;
+
+uint64_t nvme_vtophys(void *buf)
+{
+	return (uintptr_t)buf;
+}
+
+typedef void (*verify_request_fn_t)(struct nvme_request *req);
+verify_request_fn_t verify_fn;
+
+void verify_firmware_log_page(struct nvme_request *req)
+{
+	cmd = &req->cmd;
+	CU_ASSERT(cmd->opc == NVME_OPC_GET_LOG_PAGE);
+	nvme_free_request(req);
+}
+
+void verify_health_log_page(struct nvme_request *req)
+{
+	cmd = &req->cmd;
+	CU_ASSERT(cmd->opc == NVME_OPC_GET_LOG_PAGE);
+	nvme_free_request(req);
+}
+
+void verify_error_log_page(struct nvme_request *req)
+{
+	cmd = &req->cmd;
+	CU_ASSERT(cmd->opc == NVME_OPC_GET_LOG_PAGE);
+	nvme_free_request(req);
+}
+
+void verify_get_feature_cmd(struct nvme_request *req)
+{
+	cmd = &req->cmd;
+	CU_ASSERT(cmd->opc == NVME_OPC_GET_FEATURES);
+	nvme_free_request(req);
+}
+
+void verify_abort_cmd(struct nvme_request *req)
+{
+	cmd = &req->cmd;
+	CU_ASSERT(cmd->opc == NVME_OPC_ABORT);
+	nvme_free_request(req);
+}
+
+void verify_io_raw_cmd(struct nvme_request *req)
+{
+	struct nvme_command	command = {0};
+	uint64_t		phys_addr = 0;
+	int			rc = 100;
+
+
+	cmd = &req->cmd;
+	CU_ASSERT(cmd != NULL);
+	rc = memcmp(cmd, &command, sizeof(cmd));
+	CU_ASSERT(rc == 0);
+	nvme_free_request(req);
+}
+
+struct nvme_request *
+nvme_allocate_request(void *payload, uint32_t payload_size,
+		      nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+	struct nvme_request *req = NULL;
+	nvme_alloc_request(&req);
+
+	if (req != NULL) {
+		memset(req, 0, offsetof(struct nvme_request, children));
+
+		if (payload == NULL || payload_size == 0) {
+			req->u.payload = NULL;
+			req->payload_size = 0;
+		} else {
+			req->u.payload = payload;
+			req->payload_size = payload_size;
+		}
+
+		req->cb_fn = cb_fn;
+		req->cb_arg = cb_arg;
+		req->timeout = true;
+	}
+	return req;
+}
+
+void
+nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr,
+			     struct nvme_request *req)
+{
+	verify_fn(req);
+}
+
+void
+nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr, struct nvme_request *req)
+{
+	verify_fn(req);
+}
+
+
+void
+test_firmware_get_log_page()
+{
+	struct nvme_controller			ctrlr = {};
+	struct nvme_firmware_page		*payload = NULL;
+	nvme_cb_fn_t				cb_fn = NULL;
+	void					*cb_arg = NULL;
+	uint64_t				phys_addr = 0;
+
+	payload = nvme_malloc("nvme_firmware_page", sizeof(struct nvme_firmware_page),
+			      64, &phys_addr);
+	CU_ASSERT(payload != NULL);
+
+	verify_fn = verify_firmware_log_page;
+
+	nvme_ctrlr_cmd_get_firmware_page(&ctrlr,
+					 payload, cb_fn, cb_arg);
+
+	nvme_free(payload);
+}
+
+void
+test_health_get_log_page()
+{
+	struct nvme_controller			ctrlr = {};
+	struct nvme_health_information_page	*payload = NULL;
+	uint32_t				nsid = 0;
+	nvme_cb_fn_t				cb_fn = NULL;
+	void					*cb_arg = NULL;
+	uint64_t				phys_addr = 0;
+
+	payload = nvme_malloc("nvme_health_information_page", sizeof(struct nvme_health_information_page),
+			      64, &phys_addr);
+	CU_ASSERT(payload != NULL);
+
+	verify_fn = verify_health_log_page;
+
+	nvme_ctrlr_cmd_get_health_information_page(&ctrlr, nsid,
+			payload, cb_fn, cb_arg);
+
+	nvme_free(payload);
+}
+
+void
+test_error_get_log_page()
+{
+	struct nvme_controller			*ctrlr = NULL;
+	struct nvme_controller_data		*ctrldata = NULL;
+	struct nvme_error_information_entry	*payload = NULL;
+	uint32_t				num_entries = 1;
+	nvme_cb_fn_t				cb_fn = NULL;
+	void					*cb_arg = NULL;
+	uint64_t				phys_addr = 0;
+
+	payload = nvme_malloc("nvme_error_information_entry", sizeof(struct nvme_error_information_entry),
+			      64, &phys_addr);
+	CU_ASSERT(payload != NULL);
+
+	ctrlr = nvme_malloc("nvme_controller", sizeof(struct nvme_controller),
+			    64, &phys_addr);
+	CU_ASSERT(ctrlr != NULL);
+
+	ctrldata = nvme_malloc("nvme_controller_data", sizeof(struct nvme_controller_data),
+			       64, &phys_addr);
+	CU_ASSERT(ctrldata != NULL);
+
+	ctrlr->cdata = *ctrldata;
+	ctrlr->cdata.elpe = 5;
+
+	verify_fn = verify_error_log_page;
+
+	nvme_ctrlr_cmd_get_error_page(ctrlr, payload,
+				      num_entries, cb_fn, cb_arg);
+	num_entries = 50;
+	nvme_ctrlr_cmd_get_error_page(ctrlr, payload,
+				      num_entries, cb_fn, cb_arg);
+
+
+	nvme_free(payload);
+	nvme_free(ctrlr);
+	nvme_free(ctrldata);
+}
+
+void
+test_get_feature_cmd()
+{
+	struct nvme_controller	ctrlr = {};
+	uint8_t			feature = 1;
+	uint32_t		cdw11 = 1;
+	void			*payload = NULL;
+	uint32_t		payload_size = 0;
+	nvme_cb_fn_t		cb_fn = NULL;
+	void			*cb_arg = NULL;
+
+	verify_fn = verify_get_feature_cmd;
+
+	nvme_ctrlr_cmd_get_feature(&ctrlr, feature, cdw11, payload,
+				   payload_size, cb_fn, cb_arg);
+}
+
+void
+test_abort_cmd()
+{
+	struct nvme_controller ctrlr = {};
+	uint16_t		cid = 0;
+	uint16_t		sqid = 0;
+	nvme_cb_fn_t		cb_fn = NULL;
+	void			*cb_arg = NULL;
+
+	verify_fn = verify_abort_cmd;
+
+	nvme_ctrlr_cmd_abort(&ctrlr, cid, sqid, cb_fn, cb_arg);
+}
+
+void
+test_io_raw_cmd()
+{
+	struct nvme_controller ctrlr = {};
+	struct nvme_command	*cmd = NULL;
+	void			*buf = NULL;
+	uint32_t		len = 1;
+	nvme_cb_fn_t		cb_fn = NULL;
+	void			*cb_arg = NULL;
+	uint64_t		phys_addr = 0;
+
+	cmd = nvme_malloc("nvme_command", sizeof(struct nvme_command),
+			  64, &phys_addr);
+	CU_ASSERT(cmd != NULL);
+	memset(cmd, 0, sizeof(cmd));
+
+	verify_fn = verify_io_raw_cmd;
+
+	nvme_ctrlr_cmd_io_raw(&ctrlr, cmd, buf, len, cb_fn, cb_arg);
+	nvme_free(cmd);
+}
+
+int main(int argc, char **argv)
+{
+	CU_pSuite	suite = NULL;
+	unsigned int	num_failures;
+
+	if (CU_initialize_registry() != CUE_SUCCESS) {
+		return CU_get_error();
+	}
+
+	suite = CU_add_suite("nvme_ctrlr_cmd", NULL, NULL);
+	if (suite == NULL) {
+		CU_cleanup_registry();
+		return CU_get_error();
+	}
+
+	if (
+		CU_add_test(suite, "test ctrlr cmd get_firmware_page", test_firmware_get_log_page) == NULL
+		|| CU_add_test(suite, "test ctrlr cmd get_health_page", test_health_get_log_page) == NULL
+		|| CU_add_test(suite, "test ctrlr cmd get_error_page", test_error_get_log_page) == NULL
+		|| CU_add_test(suite, "test ctrlr cmd get_feature", test_get_feature_cmd) == NULL
+		|| CU_add_test(suite, "test ctrlr cmd abort_cmd", test_abort_cmd) == NULL
+		|| CU_add_test(suite, "test ctrlr cmd io_raw_cmd", test_io_raw_cmd) == NULL
+	) {
+		CU_cleanup_registry();
+		return CU_get_error();
+	}
+
+	CU_basic_set_mode(CU_BRM_VERBOSE);
+	CU_basic_run_tests();
+	num_failures = CU_get_number_of_failures();
+	CU_cleanup_registry();
+	return num_failures;
+}

test/lib/nvme/unit/nvme_impl.h

@@ -0,0 +1,102 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __NVME_IMPL_H__
+#define __NVME_IMPL_H__
+
+#include <stdio.h>
+#include <stdint.h>
+#include <pthread.h>
+
+#define nvme_malloc(tag, size, align, phys_addr)	malloc(size)
+#define nvme_free(buf)			free(buf)
+#define OUTBUF_SIZE 1024
+extern char outbuf[OUTBUF_SIZE];
+#define nvme_printf(ctrlr, fmt, args...) snprintf(outbuf, OUTBUF_SIZE, fmt, ##args)
+#define nvme_get_num_ioq()		8
+#define nvme_get_ioq_idx()		0
+#define nvme_assert(check, str)			\
+do							\
+	{						\
+		if (!(check))				\
+		printf str;				\
+	}						\
+	while (0);
+uint64_t nvme_vtophys(void *buf);
+#define nvme_alloc_request(bufp)	\
+do					\
+	{				\
+		*bufp = malloc(sizeof(struct nvme_request));	\
+	}				\
+	while (0);
+
+#define nvme_free_request(buf)		free(buf)
+#define nvme_pcicfg_read32(handle, var, offset)
+#define nvme_pcicfg_write32(handle, var, offset)
+
+static inline
+int nvme_pcicfg_map_bar(void *pci_handle, int bar, int read_only, void **addr)
+{
+	*addr = NULL;
+	return 0;
+}
+
+static inline int
+nvme_pcicfg_unmap_bar(void *devhandle, uint32_t bar, void *addr)
+{
+	return 0;
+}
+
+typedef pthread_mutex_t nvme_mutex_t;
+
+#define nvme_mutex_init(x) pthread_mutex_init((x), NULL)
+#define nvme_mutex_destroy(x) pthread_mutex_destroy((x))
+#define nvme_mutex_lock pthread_mutex_lock
+#define nvme_mutex_unlock pthread_mutex_unlock
+#define NVME_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
+
+static inline int
+nvme_mutex_init_recursive(nvme_mutex_t *mtx)
+{
+	pthread_mutexattr_t attr;
+
+	if (pthread_mutexattr_init(&attr)) {
+		return -1;
+	}
+	if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE)) {
+		return -1;
+	}
+	return pthread_mutex_init(mtx, &attr);
+}
+
+#endif /* __NVME_IMPL_H__ */

test/lib/nvme/unit/nvme_ns_cmd_c/.gitignore

@@ -0,0 +1 @@
+nvme_ns_cmd_ut

test/lib/nvme/unit/nvme_ns_cmd_c/Makefile

@@ -0,0 +1,38 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+TEST_FILE = nvme_ns_cmd_ut.c
+OTHER_FILES = nvme.c
+
+include $(SPDK_ROOT_DIR)/mk/nvme.unittest.mk
+

test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c

@@ -0,0 +1,243 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "nvme/nvme_internal.h"
+
+#include "CUnit/Basic.h"
+
+#include "nvme/nvme_ns_cmd.c"
+
+char outbuf[OUTBUF_SIZE];
+
+struct nvme_request *g_request = NULL;
+
+uint64_t nvme_vtophys(void *buf)
+{
+	return (uintptr_t)buf;
+}
+
+int
+nvme_ctrlr_hw_reset(struct nvme_controller *ctrlr)
+{
+	return 0;
+}
+
+int
+nvme_ctrlr_construct(struct nvme_controller *ctrlr, void *devhandle)
+{
+	return 0;
+}
+
+void
+nvme_ctrlr_destruct(struct nvme_controller *ctrlr)
+{
+}
+
+int
+nvme_ctrlr_start(struct nvme_controller *ctrlr)
+{
+	return 0;
+}
+
+uint32_t
+nvme_ns_get_sector_size(struct nvme_namespace *ns)
+{
+	return ns->sector_size;
+}
+
+uint32_t
+nvme_ns_get_max_io_xfer_size(struct nvme_namespace *ns)
+{
+	return ns->ctrlr->max_xfer_size;
+}
+
+void
+nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr,
+			     struct nvme_request *req)
+{
+	g_request = req;
+}
+
+void
+prepare_for_test(struct nvme_namespace *ns, struct nvme_controller *ctrlr,
+		 uint32_t sector_size, uint32_t max_xfer_size,
+		 uint32_t stripe_size)
+{
+	ctrlr->max_xfer_size = max_xfer_size;
+	ns->ctrlr = ctrlr;
+	ns->sector_size = sector_size;
+	ns->stripe_size = stripe_size;
+	ns->sectors_per_max_io = nvme_ns_get_max_io_xfer_size(ns) / ns->sector_size;
+	ns->sectors_per_stripe = ns->stripe_size / ns->sector_size;
+
+	g_request = NULL;
+}
+
+void
+split_test(void)
+{
+	struct nvme_namespace	ns;
+	struct nvme_controller	ctrlr;
+	void			*payload;
+	uint64_t		lba;
+	uint32_t		lba_count;
+	int			rc;
+
+	prepare_for_test(&ns, &ctrlr, 512, 128 * 1024, 0);
+	payload = 0x0;
+	lba = 0;
+	lba_count = 1;
+
+	rc = nvme_ns_cmd_read(&ns, payload, lba, lba_count, NULL, NULL);
+
+	CU_ASSERT(rc == 0);
+	CU_ASSERT(g_request != NULL);
+	CU_ASSERT(g_request->num_children == 0);
+
+	nvme_free_request(g_request);
+}
+
+void
+split_test2(void)
+{
+	struct nvme_namespace	ns;
+	struct nvme_controller	ctrlr;
+	struct nvme_request	*child;
+	void			*payload;
+	uint64_t		lba;
+	uint32_t		lba_count;
+	int			rc;
+
+	prepare_for_test(&ns, &ctrlr, 512, 128 * 1024, 0);
+	payload = malloc(256 * 1024);
+	lba = 0;
+	lba_count = (256 * 1024) / 512;
+
+	rc = nvme_ns_cmd_read(&ns, payload, lba, lba_count, NULL, NULL);
+
+	CU_ASSERT(rc == 0);
+	CU_ASSERT(g_request != NULL);
+	CU_ASSERT(g_request->num_children == 2);
+
+	child = TAILQ_FIRST(&g_request->children);
+	TAILQ_REMOVE(&g_request->children, child, child_tailq);
+	CU_ASSERT(child->num_children == 0);
+	CU_ASSERT(child->payload_size == 128 * 1024);
+
+	child = TAILQ_FIRST(&g_request->children);
+	TAILQ_REMOVE(&g_request->children, child, child_tailq);
+	CU_ASSERT(child->num_children == 0);
+	CU_ASSERT(child->payload_size == 128 * 1024);
+
+	CU_ASSERT(TAILQ_EMPTY(&g_request->children));
+
+	free(payload);
+	nvme_free_request(g_request);
+}
+
+void
+test_nvme_ns_cmd_flush(void)
+{
+	struct nvme_namespace	ns;
+	struct nvme_controller	ctrlr;
+	nvme_cb_fn_t		cb_fn = NULL;
+	void			*cb_arg = NULL;
+
+	prepare_for_test(&ns, &ctrlr, 512, 128 * 1024, 0);
+
+	nvme_ns_cmd_flush(&ns, cb_fn, cb_arg);
+	CU_ASSERT(g_request->cmd.opc == NVME_OPC_FLUSH);
+	CU_ASSERT(g_request->cmd.nsid == ns.id);
+
+	nvme_free_request(g_request);
+}
+
+void
+test_nvme_ns_cmd_deallocate(void)
+{
+	struct nvme_namespace	ns;
+	struct nvme_controller	ctrlr;
+	nvme_cb_fn_t		cb_fn = NULL;
+	void			*cb_arg = NULL;
+	uint8_t			num_ranges = 1;
+	void			*payload = NULL;
+	int			rc = 0;
+
+	prepare_for_test(&ns, &ctrlr, 512, 128 * 1024, 0);
+	payload = malloc(num_ranges * sizeof(struct nvme_dsm_range));
+
+	nvme_ns_cmd_deallocate(&ns, payload, num_ranges, cb_fn, cb_arg);
+	CU_ASSERT(g_request->cmd.opc == NVME_OPC_DATASET_MANAGEMENT);
+	CU_ASSERT(g_request->cmd.nsid == ns.id);
+	CU_ASSERT(g_request->cmd.cdw10 == num_ranges - 1);
+	CU_ASSERT(g_request->cmd.cdw11 == NVME_DSM_ATTR_DEALLOCATE);
+	free(payload);
+	nvme_free_request(g_request);
+
+	payload = NULL;
+	num_ranges = 0;
+	rc = nvme_ns_cmd_deallocate(&ns, payload, num_ranges, cb_fn, cb_arg);
+	CU_ASSERT(rc != 0);
+}
+
+int main(int argc, char **argv)
+{
+	CU_pSuite	suite = NULL;
+	unsigned int	num_failures;
+
+	if (CU_initialize_registry() != CUE_SUCCESS) {
+		return CU_get_error();
+	}
+
+	suite = CU_add_suite("nvme_ns_cmd", NULL, NULL);
+	if (suite == NULL) {
+		CU_cleanup_registry();
+		return CU_get_error();
+	}
+
+	if (
+		CU_add_test(suite, "split_test", split_test) == NULL
+		|| CU_add_test(suite, "split_test2", split_test2) == NULL
+		|| CU_add_test(suite, "nvme_ns_cmd_flush testing", test_nvme_ns_cmd_flush) == NULL
+		|| CU_add_test(suite, "nvme_ns_cmd_deallocate testing", test_nvme_ns_cmd_deallocate) == NULL
+	) {
+		CU_cleanup_registry();
+		return CU_get_error();
+	}
+
+	CU_basic_set_mode(CU_BRM_VERBOSE);
+	CU_basic_run_tests();
+	num_failures = CU_get_number_of_failures();
+	CU_cleanup_registry();
+	return num_failures;
+}

test/lib/nvme/unit/nvme_qpair_c/.gitignore

@@ -0,0 +1 @@
+nvme_qpair_ut

test/lib/nvme/unit/nvme_qpair_c/Makefile

@@ -0,0 +1,37 @@
+#
+#  BSD LICENSE
+#
+#  Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+TEST_FILE = nvme_qpair_ut.c
+
+include $(SPDK_ROOT_DIR)/mk/nvme.unittest.mk
+

test/lib/nvme/unit/nvme_qpair_c/nvme_qpair_ut.c

@@ -0,0 +1,433 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdbool.h>
+#include "nvme/nvme_internal.h"
+
+#include "CUnit/Basic.h"
+
+#include "nvme/nvme_qpair.c"
+
+struct nvme_driver g_nvme_driver = {
+	.lock = NVME_MUTEX_INITIALIZER,
+	.max_io_queues = NVME_MAX_IO_QUEUES,
+};
+
+int32_t nvme_retry_count = 1;
+
+char outbuf[OUTBUF_SIZE];
+
+bool fail_vtophys = false;
+
+uint64_t nvme_vtophys(void *buf)
+{
+	if (fail_vtophys) {
+		return (uint64_t) - 1;
+	} else {
+		return (uintptr_t)buf;
+	}
+}
+
+void nvme_dump_completion(struct nvme_completion *cpl)
+{
+}
+
+void prepare_for_test(void)
+{
+}
+
+void
+test1(void)
+{
+	struct nvme_qpair qpair = {};
+	struct nvme_command cmd = {};
+
+	outbuf[0] = '\0';
+
+	/*
+	 * qpair.id == 0 means it is an admin queue.  Ensure
+	 *  that the opc is decoded as an admin opc and not an
+	 *  I/o opc.
+	 */
+	qpair.id = 0;
+	cmd.opc = NVME_OPC_IDENTIFY;
+
+	nvme_qpair_print_command(&qpair, &cmd);
+
+	CU_ASSERT(strstr(outbuf, "IDENTIFY") != NULL);
+}
+
+void
+test2(void)
+{
+	struct nvme_qpair qpair = {};
+	struct nvme_command cmd = {};
+
+	outbuf[0] = '\0';
+
+	/*
+	 * qpair.id != 0 means it is an I/O queue.  Ensure
+	 *  that the opc is decoded as an I/O opc and not an
+	 *  admin opc.
+	 */
+	qpair.id = 1;
+	cmd.opc = NVME_OPC_DATASET_MANAGEMENT;
+
+	nvme_qpair_print_command(&qpair, &cmd);
+
+	CU_ASSERT(strstr(outbuf, "DATASET MANAGEMENT") != NULL);
+}
+
+void
+prepare_submit_request_test(struct nvme_qpair *qpair,
+			    struct nvme_controller *ctrlr,
+			    struct nvme_registers *regs)
+{
+	memset(ctrlr, 0, sizeof(*ctrlr));
+	ctrlr->regs = regs;
+	nvme_qpair_construct(qpair, 1, 128, 32, ctrlr);
+
+	CU_ASSERT(qpair->sq_tail == 0);
+	CU_ASSERT(qpair->cq_head == 0);
+
+	fail_vtophys = false;
+}
+
+void
+cleanup_submit_request_test(struct nvme_qpair *qpair)
+{
+	nvme_qpair_destroy(qpair);
+}
+
+void
+expected_success_callback(void *arg, const struct nvme_completion *cpl)
+{
+	CU_ASSERT(!nvme_completion_is_error(cpl));
+}
+
+void
+expected_failure_callback(void *arg, const struct nvme_completion *cpl)
+{
+	CU_ASSERT(nvme_completion_is_error(cpl));
+}
+
+void
+test3(void)
+{
+	struct nvme_qpair	qpair = {};
+	struct nvme_request	*req;
+	struct nvme_controller	ctrlr = {};
+	struct nvme_registers	regs = {};
+
+	prepare_submit_request_test(&qpair, &ctrlr, &regs);
+
+	nvme_alloc_request(&req);
+	req->payload_size = 4096;
+	req->cb_fn = expected_success_callback;
+
+	CU_ASSERT(qpair.sq_tail == 0);
+
+	nvme_qpair_submit_request(&qpair, req);
+
+	CU_ASSERT(qpair.sq_tail == 1);
+
+	cleanup_submit_request_test(&qpair);
+	nvme_free_request(req);
+}
+
+void
+test4(void)
+{
+	struct nvme_qpair	qpair = {};
+	struct nvme_request	*req;
+	struct nvme_controller	ctrlr = {};
+	struct nvme_registers	regs = {};
+
+	prepare_submit_request_test(&qpair, &ctrlr, &regs);
+
+	nvme_alloc_request(&req);
+	req->payload_size = 4096;
+	req->cb_fn = expected_failure_callback;
+
+	/* Force vtophys to return a failure.  This should
+	 *  result in the nvme_qpair manually failing
+	 *  the request with error status to signify
+	 *  a bad payload buffer.
+	 */
+	fail_vtophys = true;
+	outbuf[0] = '\0';
+
+	CU_ASSERT(qpair.sq_tail == 0);
+
+	nvme_qpair_submit_request(&qpair, req);
+
+	CU_ASSERT(qpair.sq_tail == 0);
+	/* Assert that command/completion data was printed to log. */
+	CU_ASSERT(strlen(outbuf) > 0);
+
+	cleanup_submit_request_test(&qpair);
+}
+
+void
+test_ctrlr_failed(void)
+{
+	struct nvme_qpair	qpair = {};
+	struct nvme_request	*req;
+	struct nvme_controller	ctrlr = {};
+	struct nvme_registers	regs = {};
+
+	prepare_submit_request_test(&qpair, &ctrlr, &regs);
+
+	nvme_alloc_request(&req);
+	req->payload_size = 4096;
+	req->cb_fn = expected_failure_callback;
+
+	/* Disable the queue and set the controller to failed.
+	 * Set the controller to resetting so that the qpair won't get re-enabled.
+	 */
+	qpair.is_enabled = false;
+	ctrlr.is_failed = true;
+	ctrlr.is_resetting = true;
+
+	outbuf[0] = '\0';
+
+	CU_ASSERT(qpair.sq_tail == 0);
+
+	nvme_qpair_submit_request(&qpair, req);
+
+	CU_ASSERT(qpair.sq_tail == 0);
+	/* Assert that command/completion data was printed to log. */
+	CU_ASSERT(strlen(outbuf) > 0);
+
+	cleanup_submit_request_test(&qpair);
+}
+
+void struct_packing(void)
+{
+	/* ctrlr is the first field in nvme_qpair after the fields
+	 * that are used in the I/O path. Make sure the I/O path fields
+	 * all fit into two cache lines.
+	 */
+	CU_ASSERT(offsetof(struct nvme_qpair, ctrlr) <= 128);
+}
+
+void test_nvme_qpair_fail(void)
+{
+	struct nvme_qpair	qpair = {0};
+	struct nvme_request	*req = NULL;
+	struct nvme_controller	ctrlr = {};
+	struct nvme_registers	regs = {0};
+	struct nvme_tracker	*tr_temp;
+	uint64_t		phys_addr = 0;
+
+	prepare_submit_request_test(&qpair, &ctrlr, &regs);
+
+	tr_temp = nvme_malloc("nvme_tracker", sizeof(struct nvme_tracker),
+			      64, &phys_addr);
+	tr_temp->req = NULL;
+	nvme_alloc_request(&tr_temp->req);
+
+	LIST_INSERT_HEAD(&qpair.outstanding_tr, tr_temp, list);
+	nvme_qpair_fail(&qpair);
+	CU_ASSERT_TRUE(LIST_EMPTY(&qpair.outstanding_tr));
+
+	nvme_alloc_request(&req);
+	STAILQ_INSERT_HEAD(&qpair.queued_req, req, stailq);
+	nvme_qpair_fail(&qpair);
+	CU_ASSERT_TRUE(STAILQ_EMPTY(&qpair.queued_req));
+
+	cleanup_submit_request_test(&qpair);
+}
+
+void test_nvme_qpair_process_completions(void)
+{
+	struct nvme_qpair	qpair = {0};
+	struct nvme_request	*req = NULL;
+	struct nvme_controller	ctrlr = {};
+	struct nvme_registers	regs = {0};
+
+	prepare_submit_request_test(&qpair, &ctrlr, &regs);
+	qpair.is_enabled = false;
+	qpair.ctrlr->is_resetting = true;
+
+	nvme_qpair_process_completions(&qpair);
+	cleanup_submit_request_test(&qpair);
+}
+
+void test_nvme_qpair_destroy(void)
+{
+	struct nvme_qpair	qpair = {0};
+	struct nvme_request	*req = NULL;
+	struct nvme_controller	ctrlr = {};
+	struct nvme_registers	regs = {0};
+	struct nvme_tracker	*tr_temp;
+	uint64_t		phys_addr = 0;
+
+	memset(&ctrlr, 0, sizeof(ctrlr));
+	ctrlr.regs = &regs;
+	nvme_qpair_construct(&qpair, 1, 128, 32, &ctrlr);
+	qpair.cmd = nvme_malloc("nvme_command", sizeof(struct nvme_command),
+				64, &phys_addr);
+
+	qpair.cpl = nvme_malloc("nvme_completion", sizeof(struct nvme_completion),
+				64, &phys_addr);
+
+	qpair.act_tr = nvme_malloc("nvme_tracker", sizeof(struct nvme_tracker),
+				   64, &phys_addr);
+
+	nvme_qpair_destroy(&qpair);
+}
+
+void test_nvme_completion_is_retry(void)
+{
+	struct nvme_completion	*cpl = NULL;
+	uint64_t		phys_addr = 0;
+	bool			ret_val = false;
+
+	cpl = nvme_malloc("nvme_completion", sizeof(struct nvme_completion),
+			  64, &phys_addr);
+
+	cpl->status.sct = NVME_SCT_GENERIC;
+	cpl->status.sc = NVME_SC_ABORTED_BY_REQUEST;
+	cpl->status.dnr = 0;
+	ret_val = nvme_completion_is_retry(cpl);
+	CU_ASSERT_TRUE(ret_val);
+
+	cpl->status.sc = NVME_SC_INVALID_OPCODE;
+	ret_val = nvme_completion_is_retry(cpl);
+	CU_ASSERT_FALSE(ret_val);
+
+	cpl->status.sc = NVME_SC_INVALID_FIELD;
+	ret_val = nvme_completion_is_retry(cpl);
+	CU_ASSERT_FALSE(ret_val);
+
+	cpl->status.sc = NVME_SC_COMMAND_ID_CONFLICT;
+	ret_val = nvme_completion_is_retry(cpl);
+	CU_ASSERT_FALSE(ret_val);
+
+	cpl->status.sc = NVME_SC_DATA_TRANSFER_ERROR;
+	ret_val = nvme_completion_is_retry(cpl);
+	CU_ASSERT_FALSE(ret_val);
+
+	cpl->status.sc = NVME_SC_ABORTED_POWER_LOSS;
+	ret_val = nvme_completion_is_retry(cpl);
+	CU_ASSERT_FALSE(ret_val);
+
+	cpl->status.sc = NVME_SC_INTERNAL_DEVICE_ERROR;
+	ret_val = nvme_completion_is_retry(cpl);
+	CU_ASSERT_FALSE(ret_val);
+
+	cpl->status.sc = NVME_SC_ABORTED_FAILED_FUSED;
+	ret_val = nvme_completion_is_retry(cpl);
+	CU_ASSERT_FALSE(ret_val);
+
+	cpl->status.sc = NVME_SC_ABORTED_MISSING_FUSED;
+	ret_val = nvme_completion_is_retry(cpl);
+	CU_ASSERT_FALSE(ret_val);
+
+	cpl->status.sc = NVME_SC_INVALID_NAMESPACE_OR_FORMAT;
+	ret_val = nvme_completion_is_retry(cpl);
+	CU_ASSERT_FALSE(ret_val);
+
+	cpl->status.sc = NVME_SC_COMMAND_SEQUENCE_ERROR;
+	ret_val = nvme_completion_is_retry(cpl);
+	CU_ASSERT_FALSE(ret_val);
+
+	cpl->status.sc = NVME_SC_LBA_OUT_OF_RANGE;
+	ret_val = nvme_completion_is_retry(cpl);
+	CU_ASSERT_FALSE(ret_val);
+
+	cpl->status.sc = NVME_SC_CAPACITY_EXCEEDED;
+	ret_val = nvme_completion_is_retry(cpl);
+	CU_ASSERT_FALSE(ret_val);
+
+	cpl->status.sc = 0x70;
+	ret_val = nvme_completion_is_retry(cpl);
+	CU_ASSERT_FALSE(ret_val);
+
+	cpl->status.sct = NVME_SCT_COMMAND_SPECIFIC;
+	ret_val = nvme_completion_is_retry(cpl);
+	CU_ASSERT_FALSE(ret_val);
+
+	cpl->status.sct = NVME_SCT_MEDIA_ERROR;
+	ret_val = nvme_completion_is_retry(cpl);
+	CU_ASSERT_FALSE(ret_val);
+
+	cpl->status.sct = NVME_SCT_VENDOR_SPECIFIC;
+	ret_val = nvme_completion_is_retry(cpl);
+	CU_ASSERT_FALSE(ret_val);
+
+	cpl->status.sct = 0x4;
+	ret_val = nvme_completion_is_retry(cpl);
+	CU_ASSERT_FALSE(ret_val);
+
+
+}
+
+int main(int argc, char **argv)
+{
+	CU_pSuite	suite = NULL;
+	unsigned int	num_failures;
+
+	if (CU_initialize_registry() != CUE_SUCCESS) {
+		return CU_get_error();
+	}
+
+	suite = CU_add_suite("nvme_qpair", NULL, NULL);
+	if (suite == NULL) {
+		CU_cleanup_registry();
+		return CU_get_error();
+	}
+
+	if (
+		CU_add_test(suite, "test1", test1) == NULL
+		|| CU_add_test(suite, "test2", test2) == NULL
+		|| CU_add_test(suite, "test3", test3) == NULL
+		|| CU_add_test(suite, "test4", test4) == NULL
+		|| CU_add_test(suite, "ctrlr_failed", test_ctrlr_failed) == NULL
+		|| CU_add_test(suite, "struct_packing", struct_packing) == NULL
+		|| CU_add_test(suite, "nvme_qpair_fail", test_nvme_qpair_fail) == NULL
+		|| CU_add_test(suite, "nvme_qpair_process_completions", test_nvme_qpair_process_completions) == NULL
+		|| CU_add_test(suite, "nvme_qpair_destroy", test_nvme_qpair_destroy) == NULL
+		|| CU_add_test(suite, "nvme_completion_is_retry", test_nvme_completion_is_retry) == NULL
+	) {
+		CU_cleanup_registry();
+		return CU_get_error();
+	}
+
+	CU_basic_set_mode(CU_BRM_VERBOSE);
+	CU_basic_run_tests();
+	num_failures = CU_get_number_of_failures();
+	CU_cleanup_registry();
+	return num_failures;
+}