Start 18.10.3

Change-Id: I7408ecd0cc0adad150e50b72bf074dbc57492f4a
Signed-off-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/449572
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

CHANGELOG.md

@@ -1,5 +1,62 @@
 # Changelog
 
+## v18.10.3 (upcoming release):
+
+## v18.10.2:
+
+DPDK 18.11 is now supported.
+
+## v18.10.1:
+
+### Packaging
+
+Spec file for creating RPM SPDK packages is now available:
+
+x86_64/spdk
+  Base SPDK package - all apps installed by `make install`
+  - spdk_* applications
+  - shared libs in libs64 dir
+
+x86_64/spdk-devel
+  SPDK development package:
+  - header files
+  - static/shared libs
+
+x86_64/spdk-debuginfo
+x86_64/spdk-debugsource
+  These two are autogenerated by rpmbuild
+
+no_arch/spdk-tools:
+  SPDK tools package:
+    scripts/rpc.py -> /usr/sbin/spdk-rpc
+    scripts/spdkcli.py -> /usr/sbin/spdk-cli
+
+no_arch/spdk-doc: optional, generated when adding '--with doc'
+  SPDK html doc package:
+    doc/output/html/ -> /usr/share/doc/spdk/html
+
+### bdev
+
+On shutdown, bdev unregister now proceeds in top-down fashion, with
+claimed bdevs skipped (these will be unregistered later, when virtual
+bdev built on top of the respective base bdev unclaims it). This
+allows virtual bdevs to be shut down cleanly as opposed to the
+previous behavior that didn't differentiate between hotremove and
+planned shutdown.
+
+### Bug fixes
+
+- nvmf: improve error handling during disconnect and QP shutdown
+- blobstore: remove snpashot from the list only on blob deletion
+- memory: return first translation from mem_map_translate
+- iSCSI: prevent recalculation of CRC multiple times
+- bdev/crypto: improve error handling when creating and unregistering
+- env_dpdk/memory: fix contiguous memory calculation for unaligned buffers
+- env_dpdk/memory: prevent freeing dynamically allocated memory back to OS
+- fio_plugin: Perform initialization and teardown on consistent thread
+- fio_plugin: exit immediately if spdk_fio_init_env fails
+- app/trace: fix app_name dereference on init
+
 ## v18.10:
 
 ### nvme

app/trace/trace.cpp

@@ -341,15 +341,14 @@ int main(int argc, char **argv)
 		exit(1);
 	}
 
+	if (file_name) {
+		fd = open(file_name, O_RDONLY);
+	} else {
 		if (shm_id >= 0) {
 			snprintf(shm_name, sizeof(shm_name), "/%s_trace.%d", app_name, shm_id);
 		} else {
 			snprintf(shm_name, sizeof(shm_name), "/%s_trace.pid%d", app_name, shm_pid);
 		}
-
-	if (file_name) {
-		fd = open(file_name, O_RDONLY);
-	} else {
 		fd = shm_open(shm_name, O_RDONLY, 0600);
 	}
 	if (fd < 0) {

autotest.sh

@@ -59,10 +59,41 @@ if [ $(uname -s) = Linux ]; then
 	# Let the kernel discover any filesystems or partitions
 	sleep 10
 
-	# Delete all partitions on NVMe devices
-	devs=`lsblk -l -o NAME | grep nvme | grep -v p` || true
-	for dev in $devs; do
-		parted -s /dev/$dev mklabel msdos
+	# OCSSD devices drivers don't support IO issues by kernel so
+	# detect OCSSD devices and blacklist them (unbind from any driver).
+	# If test scripts want to use this device it needs to do this explicitly.
+	#
+	# If some OCSSD device is bound to other driver than nvme we won't be able to
+	# discover if it is OCSSD or not so load the kernel driver first.
+
+
+	for dev in $(find /dev -maxdepth 1 -regex '/dev/nvme[0-9]+'); do
+		# Send Open Channel 2.0 Geometry opcode "0xe2" - not supported by NVMe device.
+		if nvme admin-passthru $dev --namespace-id=1 --data-len=4096  --opcode=0xe2 --read >/dev/null; then
+			bdf="$(basename $(readlink -e /sys/class/nvme/${dev#/dev/}/device))"
+			echo "INFO: blacklisting OCSSD device: $dev ($bdf)"
+			PCI_BLACKLIST+=" $bdf"
+		fi
+	done
+
+	# Now, bind blacklisted devices to pci-stub module. This will prevent
+	# automatic grabbing these devices when we add device/vendor ID to
+	# proper driver.
+	if [[ -n "$PCI_BLACKLIST" ]]; then
+		PCI_WHITELIST="$PCI_BLACKLIST" \
+		PCI_BLACKLIST="" \
+		DRIVER_OVERRIDE="pci-stub" \
+			./scripts/setup.sh
+
+		# Export our blacklist so it will take effect during next setup.sh
+		export PCI_BLACKLIST
+	fi
+
+	# Delete all leftover lvols and gpt partitions
+	# Matches both /dev/nvmeXnY on Linux and /dev/nvmeXnsY on BSD
+	# Filter out nvme with partitions - the "p*" suffix
+	for dev in $(ls /dev/nvme*n* | grep -v p || true); do
+		dd if=/dev/zero of="$dev" bs=1M count=1
 	done
 
 	# Load RAM disk driver if available
@@ -100,7 +131,13 @@ fi
 
 timing_enter lib
 
-if [ $SPDK_TEST_BLOCKDEV -eq 1 ]; then
+if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
+	timing_enter lib
+
+	run_test suite test/env/env.sh
+	run_test suite test/rpc_client/rpc_client.sh
+
+	if [ $SPDK_TEST_BLOCKDEV -eq 1 ]; then
 		run_test suite test/bdev/blockdev.sh
 		if [ $(uname -s) = Linux ]; then
 			run_test suite test/bdev/bdevjson/json_config.sh
@@ -108,17 +145,17 @@ if [ $SPDK_TEST_BLOCKDEV -eq 1 ]; then
 				run_test suite test/bdev/nbdjson/json_config.sh
 			fi
 		fi
-fi
+	fi
 
-if [ $SPDK_TEST_JSON -eq 1 ]; then
+	if [ $SPDK_TEST_JSON -eq 1 ]; then
 		run_test suite test/config_converter/test_converter.sh
-fi
+	fi
 
-if [ $SPDK_TEST_EVENT -eq 1 ]; then
+	if [ $SPDK_TEST_EVENT -eq 1 ]; then
 		run_test suite test/event/event.sh
-fi
+	fi
 
-if [ $SPDK_TEST_NVME -eq 1 ]; then
+	if [ $SPDK_TEST_NVME -eq 1 ]; then
 		run_test suite test/nvme/nvme.sh
 		if [ $SPDK_TEST_NVME_CLI -eq 1 ]; then
 			run_test suite test/nvme/spdk_nvme_cli.sh
@@ -129,35 +166,32 @@ if [ $SPDK_TEST_NVME -eq 1 ]; then
 		if [ $SPDK_RUN_ASAN -eq 0 ]; then
 			run_test suite test/nvme/hotplug.sh intel
 		fi
-fi
+	fi
 
-run_test suite test/env/env.sh
-run_test suite test/rpc_client/rpc_client.sh
-
-if [ $SPDK_TEST_IOAT -eq 1 ]; then
+	if [ $SPDK_TEST_IOAT -eq 1 ]; then
 		run_test suite test/ioat/ioat.sh
-fi
+	fi
 
-timing_exit lib
+	timing_exit lib
 
-if [ $SPDK_TEST_ISCSI -eq 1 ]; then
+	if [ $SPDK_TEST_ISCSI -eq 1 ]; then
 		run_test suite ./test/iscsi_tgt/iscsi_tgt.sh posix
 		run_test suite ./test/iscsi_tgt/iscsijson/json_config.sh
 		run_test suite ./test/spdkcli/iscsi.sh
-fi
+	fi
 
-if [ $SPDK_TEST_BLOBFS -eq 1 ]; then
+	if [ $SPDK_TEST_BLOBFS -eq 1 ]; then
 		run_test suite ./test/blobfs/rocksdb/rocksdb.sh
 		run_test suite ./test/blobstore/blobstore.sh
-fi
+	fi
 
-if [ $SPDK_TEST_NVMF -eq 1 ]; then
+	if [ $SPDK_TEST_NVMF -eq 1 ]; then
 		run_test suite ./test/nvmf/nvmf.sh
 		run_test suite ./test/nvmf/nvmfjson/json_config.sh
 		run_test suite ./test/spdkcli/nvmf.sh
-fi
+	fi
 
-if [ $SPDK_TEST_VHOST -eq 1 ]; then
+	if [ $SPDK_TEST_VHOST -eq 1 ]; then
 		timing_enter vhost
 		timing_enter negative
 		run_test suite ./test/vhost/spdk_vhost.sh --negative
@@ -218,9 +252,9 @@ if [ $SPDK_TEST_VHOST -eq 1 ]; then
 		timing_exit spdk_cli
 
 		timing_exit vhost
-fi
+	fi
 
-if [ $SPDK_TEST_LVOL -eq 1 ]; then
+	if [ $SPDK_TEST_LVOL -eq 1 ]; then
 		timing_enter lvol
 		test_cases="1,50,51,52,53,100,101,102,150,200,201,250,251,252,253,254,255,"
 		test_cases+="300,301,450,451,452,550,551,552,553,"
@@ -231,24 +265,25 @@ if [ $SPDK_TEST_LVOL -eq 1 ]; then
 		run_test suite ./test/blobstore/blob_io_wait/blob_io_wait.sh
 		report_test_completion "lvol"
 		timing_exit lvol
-fi
+	fi
 
-if [ $SPDK_TEST_VHOST_INIT -eq 1 ]; then
+	if [ $SPDK_TEST_VHOST_INIT -eq 1 ]; then
 		run_test suite ./test/vhost/initiator/blockdev.sh
 		run_test suite ./test/vhost/initiator/json_config.sh
 		run_test suite ./test/spdkcli/virtio.sh
 		report_test_completion "vhost_initiator"
-fi
+	fi
 
-if [ $SPDK_TEST_PMDK -eq 1 ]; then
+	if [ $SPDK_TEST_PMDK -eq 1 ]; then
 		run_test suite ./test/pmem/pmem.sh -x
 		run_test suite ./test/pmem/json_config/json_config.sh
 		run_test suite ./test/spdkcli/pmem.sh
-fi
+	fi
 
-if [ $SPDK_TEST_RBD -eq 1 ]; then
+	if [ $SPDK_TEST_RBD -eq 1 ]; then
 		run_test suite ./test/bdev/bdevjson/rbd_json_config.sh
 		run_test suite ./test/spdkcli/rbd.sh
+	fi
 fi
 
 timing_enter cleanup

examples/bdev/fio_plugin/fio_plugin.c

@@ -94,8 +94,6 @@ struct spdk_fio_thread {
 	unsigned int		iocq_size;	// number of iocq entries allocated
 };
 
-static struct spdk_fio_thread *g_init_thread = NULL;
-static pthread_t g_init_thread_id = 0;
 static bool g_spdk_env_initialized = false;
 
 static int spdk_fio_init(struct thread_data *td);
@@ -208,70 +206,79 @@ spdk_fio_init_thread(struct thread_data *td)
 	return 0;
 }
 
+static void
+spdk_fio_cleanup_thread(struct spdk_fio_thread *fio_thread)
+{
+	struct spdk_fio_target *target, *tmp;
+
+	TAILQ_FOREACH_SAFE(target, &fio_thread->targets, link, tmp) {
+		TAILQ_REMOVE(&fio_thread->targets, target, link);
+		spdk_put_io_channel(target->ch);
+		spdk_bdev_close(target->desc);
+		free(target);
+	}
+
+	while (spdk_fio_poll_thread(fio_thread) > 0) {}
+
+	spdk_free_thread();
+	spdk_ring_free(fio_thread->ring);
+	free(fio_thread->iocq);
+	free(fio_thread);
+}
+
+static void
+spdk_fio_module_finish_done(void *cb_arg)
+{
+	*(bool *)cb_arg = true;
+}
+
+static pthread_t g_init_thread_id = 0;
+static pthread_mutex_t g_init_mtx = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t g_init_cond = PTHREAD_COND_INITIALIZER;
+
 static void *
 spdk_init_thread_poll(void *arg)
 {
-	struct spdk_fio_thread *thread = arg;
-	int oldstate;
-	int rc;
-
-	/* Loop until the thread is cancelled */
-	while (true) {
-		rc = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate);
-		if (rc != 0) {
-			SPDK_ERRLOG("Unable to set cancel state disabled on g_init_thread (%d): %s\n",
-				    rc, spdk_strerror(rc));
-		}
-
-		spdk_fio_poll_thread(thread);
-
-		rc = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &oldstate);
-		if (rc != 0) {
-			SPDK_ERRLOG("Unable to set cancel state enabled on g_init_thread (%d): %s\n",
-				    rc, spdk_strerror(rc));
-		}
-
-		/* This is a pthread cancellation point and cannot be removed. */
-		sleep(1);
-	}
-
-	return NULL;
-}
-
-static int
-spdk_fio_init_env(struct thread_data *td)
-{
+	struct spdk_fio_options		*eo = arg;
 	struct spdk_fio_thread		*fio_thread;
-	struct spdk_fio_options		*eo;
-	bool				done = false;
-	int				rc;
 	struct spdk_conf		*config;
 	struct spdk_env_opts		opts;
+	bool				done;
+	int				rc;
 	size_t				count;
+	struct timespec			ts;
+	struct thread_data		td = {};
+
+	/* Create a dummy thread data for use on the initialization thread. */
+	td.o.iodepth = 32;
+	td.eo = eo;
 
 	/* Parse the SPDK configuration file */
-	eo = td->eo;
+	eo = arg;
 	if (!eo->conf || !strlen(eo->conf)) {
 		SPDK_ERRLOG("No configuration file provided\n");
-		return -1;
+		rc = EINVAL;
+		goto err_exit;
 	}
 
 	config = spdk_conf_allocate();
 	if (!config) {
 		SPDK_ERRLOG("Unable to allocate configuration file\n");
-		return -1;
+		rc = ENOMEM;
+		goto err_exit;
 	}
 
 	rc = spdk_conf_read(config, eo->conf);
 	if (rc != 0) {
 		SPDK_ERRLOG("Invalid configuration file format\n");
 		spdk_conf_free(config);
-		return -1;
+		goto err_exit;
 	}
 	if (spdk_conf_first_section(config) == NULL) {
 		SPDK_ERRLOG("Invalid configuration file format\n");
 		spdk_conf_free(config);
-		return -1;
+		rc = EINVAL;
+		goto err_exit;
 	}
 	spdk_conf_set_as_default(config);
 
@@ -287,23 +294,25 @@ spdk_fio_init_env(struct thread_data *td)
 	if (spdk_env_init(&opts) < 0) {
 		SPDK_ERRLOG("Unable to initialize SPDK env\n");
 		spdk_conf_free(config);
-		return -1;
+		rc = EINVAL;
+		goto err_exit;
 	}
 	spdk_unaffinitize_thread();
 
 	/* Create an SPDK thread temporarily */
-	rc = spdk_fio_init_thread(td);
+	rc = spdk_fio_init_thread(&td);
 	if (rc < 0) {
 		SPDK_ERRLOG("Failed to create initialization thread\n");
-		return -1;
+		goto err_exit;
 	}
 
-	g_init_thread = fio_thread = td->io_ops_data;
+	fio_thread = td.io_ops_data;
 
 	/* Initialize the copy engine */
 	spdk_copy_engine_initialize();
 
 	/* Initialize the bdev layer */
+	done = false;
 	spdk_bdev_initialize(spdk_fio_bdev_init_done, &done);
 
 	/* First, poll until initialization is done. */
@@ -319,16 +328,75 @@ spdk_fio_init_env(struct thread_data *td)
 		count = spdk_fio_poll_thread(fio_thread);
 	} while (count > 0);
 
+	/* Set condition variable */
+	pthread_mutex_lock(&g_init_mtx);
+	pthread_cond_signal(&g_init_cond);
+
+	while (true) {
+		spdk_fio_poll_thread(fio_thread);
+
+		clock_gettime(CLOCK_REALTIME, &ts);
+		ts.tv_sec += 1;
+		rc = pthread_cond_timedwait(&g_init_cond, &g_init_mtx, &ts);
+
+		if (rc != ETIMEDOUT) {
+			break;
+		}
+	}
+
+	pthread_mutex_unlock(&g_init_mtx);
+
+	done = false;
+	spdk_bdev_finish(spdk_fio_module_finish_done, &done);
+
+	do {
+		spdk_fio_poll_thread(fio_thread);
+	} while (!done);
+
+	do {
+		count = spdk_fio_poll_thread(fio_thread);
+	} while (count > 0);
+
+	done = false;
+	spdk_copy_engine_finish(spdk_fio_module_finish_done, &done);
+
+	do {
+		spdk_fio_poll_thread(fio_thread);
+	} while (!done);
+
+	do {
+		count = spdk_fio_poll_thread(fio_thread);
+	} while (count > 0);
+
+	spdk_fio_cleanup_thread(fio_thread);
+
+	pthread_exit(NULL);
+
+err_exit:
+	exit(rc);
+	return NULL;
+}
+
+static int
+spdk_fio_init_env(struct thread_data *td)
+{
+	int rc;
+
 	/*
-	 * Spawn a thread to continue polling this thread
-	 * occasionally.
+	 * Spawn a thread to handle initialization operations and to poll things
+	 * like the admin queues periodically.
 	 */
 
-	rc = pthread_create(&g_init_thread_id, NULL, &spdk_init_thread_poll, fio_thread);
+	rc = pthread_create(&g_init_thread_id, NULL, &spdk_init_thread_poll, td->eo);
 	if (rc != 0) {
 		SPDK_ERRLOG("Unable to spawn thread to poll admin queue. It won't be polled.\n");
 	}
 
+	/* Wait for background thread to advance past the initialization */
+	pthread_mutex_lock(&g_init_mtx);
+	pthread_cond_wait(&g_init_cond, &g_init_mtx);
+	pthread_mutex_unlock(&g_init_mtx);
+
 	return 0;
 }
 
@@ -430,26 +498,6 @@ spdk_fio_init(struct thread_data *td)
 	return 0;
 }
 
-static void
-spdk_fio_cleanup_thread(struct spdk_fio_thread *fio_thread)
-{
-	struct spdk_fio_target *target, *tmp;
-
-	TAILQ_FOREACH_SAFE(target, &fio_thread->targets, link, tmp) {
-		TAILQ_REMOVE(&fio_thread->targets, target, link);
-		spdk_put_io_channel(target->ch);
-		spdk_bdev_close(target->desc);
-		free(target);
-	}
-
-	while (spdk_fio_poll_thread(fio_thread) > 0) {}
-
-	spdk_free_thread();
-	spdk_ring_free(fio_thread->ring);
-	free(fio_thread->iocq);
-	free(fio_thread);
-}
-
 static void
 spdk_fio_cleanup(struct thread_data *td)
 {
@@ -725,48 +773,13 @@ static void fio_init spdk_fio_register(void)
 	register_ioengine(&ioengine);
 }
 
-static void
-spdk_fio_module_finish_done(void *cb_arg)
-{
-	*(bool *)cb_arg = true;
-}
-
 static void
 spdk_fio_finish_env(void)
 {
-	struct spdk_fio_thread		*fio_thread;
-	bool				done = false;
-	size_t				count;
-
-	/* the same thread that called spdk_fio_init_env */
-	fio_thread = g_init_thread;
-
-	if (pthread_cancel(g_init_thread_id) == 0) {
+	pthread_mutex_lock(&g_init_mtx);
+	pthread_cond_signal(&g_init_cond);
+	pthread_mutex_unlock(&g_init_mtx);
 	pthread_join(g_init_thread_id, NULL);
-	}
-
-	spdk_bdev_finish(spdk_fio_module_finish_done, &done);
-
-	do {
-		spdk_fio_poll_thread(fio_thread);
-	} while (!done);
-
-	do {
-		count = spdk_fio_poll_thread(fio_thread);
-	} while (count > 0);
-
-	done = false;
-	spdk_copy_engine_finish(spdk_fio_module_finish_done, &done);
-
-	do {
-		spdk_fio_poll_thread(fio_thread);
-	} while (!done);
-
-	do {
-		count = spdk_fio_poll_thread(fio_thread);
-	} while (count > 0);
-
-	spdk_fio_cleanup_thread(fio_thread);
 }
 
 static void fio_exit spdk_fio_unregister(void)

examples/nvme/identify/Makefile

@@ -36,4 +36,7 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
 APP = identify
 
+install: $(APP)
+	$(INSTALL_EXAMPLE)
+
 include $(SPDK_ROOT_DIR)/mk/nvme.libtest.mk

examples/nvme/perf/Makefile

@@ -41,4 +41,7 @@ SYS_LIBS += -laio
 CFLAGS += -DHAVE_LIBAIO
 endif
 
+install: $(APP)
+	$(INSTALL_EXAMPLE)
+
 include $(SPDK_ROOT_DIR)/mk/nvme.libtest.mk

include/spdk/barrier.h

@@ -64,7 +64,7 @@ extern "C" {
 #ifdef __PPC64__
 #define spdk_rmb()	__asm volatile("sync" ::: "memory")
 #elif defined(__aarch64__)
-#define spdk_rmb()	__asm volatile("dsb lt" ::: "memory")
+#define spdk_rmb()	__asm volatile("dsb ld" ::: "memory")
 #elif defined(__i386__) || defined(__x86_64__)
 #define spdk_rmb()	__asm volatile("lfence" ::: "memory")
 #else

include/spdk/version.h

@@ -54,12 +54,12 @@
  * Patch level is incremented on maintenance branch releases and reset to 0 for each
  * new major.minor release.
  */
-#define SPDK_VERSION_PATCH	0
+#define SPDK_VERSION_PATCH	3
 
 /**
  * Version string suffix.
  */
-#define SPDK_VERSION_SUFFIX	""
+#define SPDK_VERSION_SUFFIX	"pre"
 
 /**
  * Single numeric value representing a version number for compile-time comparisons.

lib/bdev/bdev.c

@@ -966,7 +966,7 @@ _spdk_bdev_finish_unregister_bdevs_iter(void *cb_arg, int bdeverrno)
 	if (TAILQ_EMPTY(&g_bdev_mgr.bdevs)) {
 		SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Done unregistering bdevs\n");
 		/*
-		 * Bdev module finish need to be deffered as we might be in the middle of some context
+		 * Bdev module finish need to be deferred as we might be in the middle of some context
 		 * (like bdev part free) that will use this bdev (or private bdev driver ctx data)
 		 * after returning.
 		 */
@@ -975,12 +975,38 @@ _spdk_bdev_finish_unregister_bdevs_iter(void *cb_arg, int bdeverrno)
 	}
 
 	/*
-	 * Unregister the last bdev in the list.  The last bdev in the list should be a bdev
-	 * that has no bdevs that depend on it.
+	 * Unregister last unclaimed bdev in the list, to ensure that bdev subsystem
+	 * shutdown proceeds top-down. The goal is to give virtual bdevs an opportunity
+	 * to detect clean shutdown as opposed to run-time hot removal of the underlying
+	 * base bdevs.
+	 *
+	 * Also, walk the list in the reverse order.
 	 */
-	bdev = TAILQ_LAST(&g_bdev_mgr.bdevs, spdk_bdev_list);
+	for (bdev = TAILQ_LAST(&g_bdev_mgr.bdevs, spdk_bdev_list);
+	     bdev; bdev = TAILQ_PREV(bdev, spdk_bdev_list, internal.link)) {
+		if (bdev->internal.claim_module != NULL) {
+			SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Skipping claimed bdev '%s'(<-'%s').\n",
+				      bdev->name, bdev->internal.claim_module->name);
+			continue;
+		}
+
 		SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Unregistering bdev '%s'\n", bdev->name);
 		spdk_bdev_unregister(bdev, _spdk_bdev_finish_unregister_bdevs_iter, bdev);
+		return;
+	}
+
+	/*
+	 * If any bdev fails to unclaim underlying bdev properly, we may face the
+	 * case of bdev list consisting of claimed bdevs only (if claims are managed
+	 * correctly, this would mean there's a loop in the claims graph which is
+	 * clearly impossible). Warn and unregister last bdev on the list then.
+	 */
+	for (bdev = TAILQ_LAST(&g_bdev_mgr.bdevs, spdk_bdev_list);
+	     bdev; bdev = TAILQ_PREV(bdev, spdk_bdev_list, internal.link)) {
+		SPDK_ERRLOG("Unregistering claimed bdev '%s'!\n", bdev->name);
+		spdk_bdev_unregister(bdev, _spdk_bdev_finish_unregister_bdevs_iter, bdev);
+		return;
+	}
 }
 
 void

lib/bdev/crypto/vbdev_crypto.c

@@ -974,6 +974,19 @@ vbdev_crypto_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type)
 	}
 }
 
+/* Callback for unregistering the IO device. */
+static void
+_device_unregister_cb(void *io_device)
+{
+	struct vbdev_crypto *crypto_bdev = io_device;
+
+	/* Done with this crypto_bdev. */
+	free(crypto_bdev->drv_name);
+	free(crypto_bdev->key);
+	free(crypto_bdev->crypto_bdev.name);
+	free(crypto_bdev);
+}
+
 /* Called after we've unregistered following a hot remove callback.
  * Our finish entry point will be called next.
  */
@@ -982,18 +995,18 @@ vbdev_crypto_destruct(void *ctx)
 {
 	struct vbdev_crypto *crypto_bdev = (struct vbdev_crypto *)ctx;
 
+	/* Remove this device from the internal list */
+	TAILQ_REMOVE(&g_vbdev_crypto, crypto_bdev, link);
+
 	/* Unclaim the underlying bdev. */
 	spdk_bdev_module_release_bdev(crypto_bdev->base_bdev);
 
 	/* Close the underlying bdev. */
 	spdk_bdev_close(crypto_bdev->base_desc);
 
-	/* Done with this crypto_bdev. */
-	TAILQ_REMOVE(&g_vbdev_crypto, crypto_bdev, link);
-	free(crypto_bdev->drv_name);
-	free(crypto_bdev->key);
-	free(crypto_bdev->crypto_bdev.name);
-	free(crypto_bdev);
+	/* Unregister the io_device. */
+	spdk_io_device_unregister(crypto_bdev, _device_unregister_cb);
+
 	return 0;
 }
 
@@ -1116,6 +1129,13 @@ vbdev_crypto_insert_name(const char *bdev_name, const char *vbdev_name,
 	int rc, j;
 	bool found = false;
 
+	TAILQ_FOREACH(name, &g_bdev_names, link) {
+		if (strcmp(vbdev_name, name->vbdev_name) == 0) {
+			SPDK_ERRLOG("crypto bdev %s already exists\n", vbdev_name);
+			return -EEXIST;
+		}
+	}
+
 	name = calloc(1, sizeof(struct bdev_names));
 	if (!name) {
 		SPDK_ERRLOG("could not allocate bdev_names\n");
@@ -1222,6 +1242,7 @@ create_crypto_disk(const char *bdev_name, const char *vbdev_name,
 				SPDK_ERRLOG("could not register crypto_bdev\n");
 				spdk_bdev_close(crypto_bdev->base_desc);
 				TAILQ_REMOVE(&g_vbdev_crypto, crypto_bdev, link);
+				spdk_io_device_unregister(crypto_bdev, NULL);
 				free(crypto_bdev->crypto_bdev.name);
 				free(crypto_bdev->key);
 				free(crypto_bdev);
@@ -1487,7 +1508,7 @@ vbdev_crypto_claim(struct spdk_bdev *bdev)
 			goto error_claim;
 		}
 
-		SPDK_NOTICELOG("registered crypto_bdev for: %s\n", name->vbdev_name);
+		SPDK_NOTICELOG("registered io_device for: %s\n", name->vbdev_name);
 	}
 
 	return rc;
@@ -1537,6 +1558,7 @@ delete_crypto_disk(struct spdk_bdev *bdev, spdk_delete_crypto_complete cb_fn,
 		}
 	}
 
+	/* Additional cleanup happens in the destruct callback. */
 	spdk_bdev_unregister(bdev, cb_fn, cb_arg);
 }
 
@@ -1553,7 +1575,11 @@ vbdev_crypto_examine(struct spdk_bdev *bdev)
 	struct vbdev_crypto *crypto_bdev, *tmp;
 	int rc;
 
-	vbdev_crypto_claim(bdev);
+	rc = vbdev_crypto_claim(bdev);
+	if (rc) {
+		spdk_bdev_module_examine_done(&crypto_if);
+		return;
+	}
 
 	TAILQ_FOREACH_SAFE(crypto_bdev, &g_vbdev_crypto, link, tmp) {
 		if (strcmp(crypto_bdev->base_bdev->name, bdev->name) == 0) {
@@ -1563,6 +1589,7 @@ vbdev_crypto_examine(struct spdk_bdev *bdev)
 				SPDK_ERRLOG("could not register crypto_bdev\n");
 				spdk_bdev_close(crypto_bdev->base_desc);
 				TAILQ_REMOVE(&g_vbdev_crypto, crypto_bdev, link);
+				spdk_io_device_unregister(crypto_bdev, NULL);
 				free(crypto_bdev->crypto_bdev.name);
 				free(crypto_bdev->key);
 				free(crypto_bdev);

lib/blob/blobstore.c

@@ -2361,11 +2361,6 @@ _spdk_bs_blob_list_remove(struct spdk_blob *blob)
 	free(clone_entry);
 
 	snapshot_entry->clone_count--;
-	if (snapshot_entry->clone_count == 0) {
-		/* Snapshot have no more clones */
-		TAILQ_REMOVE(&blob->bs->snapshots, snapshot_entry, link);
-		free(snapshot_entry);
-	}
 
 	return 0;
 }
@@ -4956,6 +4951,7 @@ static void
 _spdk_bs_delete_open_cpl(void *cb_arg, struct spdk_blob *blob, int bserrno)
 {
 	spdk_bs_sequence_t *seq = cb_arg;
+	struct spdk_blob_list *snapshot = NULL;
 	uint32_t page_num;
 
 	if (bserrno != 0) {
@@ -4987,6 +4983,16 @@ _spdk_bs_delete_open_cpl(void *cb_arg, struct spdk_blob *blob, int bserrno)
 	 *  get returned after this point by _spdk_blob_lookup().
 	 */
 	TAILQ_REMOVE(&blob->bs->blobs, blob, link);
+
+	/* If blob is a snapshot then remove it from the list */
+	TAILQ_FOREACH(snapshot, &blob->bs->snapshots, link) {
+		if (snapshot->id == blob->id) {
+			TAILQ_REMOVE(&blob->bs->snapshots, snapshot, link);
+			free(snapshot);
+			break;
+		}
+	}
+
 	page_num = _spdk_bs_blobid_to_page(blob->id);
 	spdk_bit_array_clear(blob->bs->used_blobids, page_num);
 	blob->state = SPDK_BLOB_STATE_DIRTY;

lib/env_dpdk/env.c

@@ -33,7 +33,7 @@
 
 #include "spdk/stdinc.h"
 
-#include "spdk/env.h"
+#include "env_internal.h"
 
 #include <rte_config.h>
 #include <rte_cycles.h>
@@ -246,9 +246,7 @@ spdk_mempool_get_name(struct spdk_mempool *mp)
 void
 spdk_mempool_free(struct spdk_mempool *mp)
 {
-#if RTE_VERSION >= RTE_VERSION_NUM(16, 7, 0, 1)
 	rte_mempool_free((struct rte_mempool *)mp);
-#endif
 }
 
 void *
@@ -285,11 +283,7 @@ spdk_mempool_put_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count)
 size_t
 spdk_mempool_count(const struct spdk_mempool *pool)
 {
-#if RTE_VERSION < RTE_VERSION_NUM(16, 7, 0, 1)
-	return rte_mempool_count((struct rte_mempool *)pool);
-#else
 	return rte_mempool_avail_count((struct rte_mempool *)pool);
-#endif
 }
 
 bool

lib/env_dpdk/env_internal.h

@@ -36,8 +36,6 @@
 
 #include "spdk/stdinc.h"
 
-#define spdk_pci_device rte_pci_device
-
 #include "spdk/env.h"
 
 #include <rte_config.h>
@@ -53,6 +51,10 @@ extern struct rte_pci_bus rte_pci_bus;
 #endif
 #include <rte_dev.h>
 
+#if RTE_VERSION < RTE_VERSION_NUM(16, 11, 0, 0)
+#error RTE_VERSION is too old! Minimum 16.11 is required.
+#endif
+
 /* x86-64 and ARM userspace virtual addresses use only the low 48 bits [0..47],
  * which is enough to cover 256 TB.
  */
@@ -68,22 +70,41 @@ extern struct rte_pci_bus rte_pci_bus;
 
 #define SHIFT_4KB	12 /* (1 << 12) == 4KB */
 #define MASK_4KB	((1ULL << SHIFT_4KB) - 1)
+#define VALUE_4KB	(1 << SHIFT_4KB)
 
-struct spdk_pci_enum_ctx {
+#define SPDK_PMD_REGISTER_PCI(pci_drv)									\
+__attribute__((constructor)) static void pci_drv ## _register(void)					\
+{													\
+	spdk_pci_driver_register(&pci_drv);								\
+}
+
+struct spdk_pci_device {
+	struct rte_pci_device		*dev_handle;
+	struct spdk_pci_driver		*driver;
+	struct spdk_pci_addr		addr;
+	struct spdk_pci_id		id;
+	int				socket_id;
+	bool				attached;
+	TAILQ_ENTRY(spdk_pci_device)	tailq;
+};
+
+struct spdk_pci_driver {
 	struct rte_pci_driver		driver;
 	spdk_pci_enum_cb		cb_fn;
 	void				*cb_arg;
-	pthread_mutex_t		mtx;
 	bool				is_registered;
+	TAILQ_ENTRY(spdk_pci_driver)	tailq;
 };
 
+void spdk_pci_driver_register(struct spdk_pci_driver *driver);
 int spdk_pci_device_init(struct rte_pci_driver *driver, struct rte_pci_device *device);
 int spdk_pci_device_fini(struct rte_pci_device *device);
 
-int spdk_pci_enumerate(struct spdk_pci_enum_ctx *ctx, spdk_pci_enum_cb enum_cb, void *enum_ctx);
-int spdk_pci_device_attach(struct spdk_pci_enum_ctx *ctx, spdk_pci_enum_cb enum_cb, void *enum_ctx,
+int spdk_pci_enumerate(struct spdk_pci_driver *driver, spdk_pci_enum_cb enum_cb, void *enum_ctx);
+int spdk_pci_device_attach(struct spdk_pci_driver *driver, spdk_pci_enum_cb enum_cb, void *enum_ctx,
 			   struct spdk_pci_addr *pci_address);
 
+void spdk_pci_init(void);
 int spdk_mem_map_init(void);
 int spdk_vtophys_init(void);
 

lib/env_dpdk/init.c

@@ -293,6 +293,17 @@ spdk_build_eal_cmdline(const struct spdk_env_opts *opts)
 	}
 
 #ifdef __linux__
+	/* Set the base virtual address - it must be an address that is not in the
+	 * ASAN shadow region, otherwise ASAN-enabled builds will ignore the
+	 * mmap hint.
+	 *
+	 * Ref: https://github.com/google/sanitizers/wiki/AddressSanitizerAlgorithm
+	 */
+	args = spdk_push_arg(args, &argcount, _sprintf_alloc("--base-virtaddr=0x200000000000"));
+	if (args == NULL) {
+		return -1;
+	}
+
 	if (opts->shm_id < 0) {
 		args = spdk_push_arg(args, &argcount, _sprintf_alloc("--file-prefix=spdk_pid%d",
 				     getpid()));
@@ -306,17 +317,6 @@ spdk_build_eal_cmdline(const struct spdk_env_opts *opts)
 			return -1;
 		}
 
-		/* Set the base virtual address - it must be an address that is not in the
-		 * ASAN shadow region, otherwise ASAN-enabled builds will ignore the
-		 * mmap hint.
-		 *
-		 * Ref: https://github.com/google/sanitizers/wiki/AddressSanitizerAlgorithm
-		 */
-		args = spdk_push_arg(args, &argcount, _sprintf_alloc("--base-virtaddr=0x200000000000"));
-		if (args == NULL) {
-			return -1;
-		}
-
 		/* set the process type */
 		args = spdk_push_arg(args, &argcount, _sprintf_alloc("--proc-type=auto"));
 		if (args == NULL) {
@@ -388,6 +388,8 @@ int spdk_env_init(const struct spdk_env_opts *opts)
 		spdk_env_unlink_shared_files();
 	}
 
+	spdk_pci_init();
+
 	if (spdk_mem_map_init() < 0) {
 		fprintf(stderr, "Failed to allocate mem_map\n");
 		return -1;

lib/env_dpdk/memory.c

@@ -57,6 +57,8 @@
 #define MAP_256TB_IDX(vfn_2mb)	((vfn_2mb) >> (SHIFT_1GB - SHIFT_2MB))
 #define MAP_1GB_IDX(vfn_2mb)	((vfn_2mb) & ((1ULL << (SHIFT_1GB - SHIFT_2MB)) - 1))
 
+#define _2MB_OFFSET(ptr)	(((uintptr_t)(ptr)) &  (VALUE_2MB - 1))
+
 /* Page is registered */
 #define REG_MAP_REGISTERED	(1ULL << 62)
 
@@ -592,6 +594,7 @@ spdk_mem_map_translate(const struct spdk_mem_map *map, uint64_t vaddr, uint64_t
 	uint64_t total_size = 0;
 	uint64_t cur_size;
 	uint64_t prev_translation;
+	uint64_t orig_translation;
 
 	if (size != NULL) {
 		total_size = *size;
@@ -612,9 +615,9 @@ spdk_mem_map_translate(const struct spdk_mem_map *map, uint64_t vaddr, uint64_t
 		return map->default_translation;
 	}
 
-	cur_size = VALUE_2MB;
+	cur_size = VALUE_2MB - _2MB_OFFSET(vaddr);
 	if (size != NULL) {
-		*size = VALUE_2MB;
+		*size = cur_size;
 	}
 
 	map_2mb = &map_1gb->map[idx_1gb];
@@ -623,7 +626,8 @@ spdk_mem_map_translate(const struct spdk_mem_map *map, uint64_t vaddr, uint64_t
 		return map_2mb->translation_2mb;
 	}
 
-	prev_translation = map_2mb->translation_2mb;;
+	orig_translation = map_2mb->translation_2mb;
+	prev_translation = orig_translation;
 	while (cur_size < total_size) {
 		vfn_2mb++;
 		idx_256tb = MAP_256TB_IDX(vfn_2mb);
@@ -644,7 +648,7 @@ spdk_mem_map_translate(const struct spdk_mem_map *map, uint64_t vaddr, uint64_t
 	}
 
 	*size = cur_size;
-	return prev_translation;
+	return orig_translation;
 }
 
 #if RTE_VERSION >= RTE_VERSION_NUM(18, 05, 0, 0)
@@ -653,14 +657,18 @@ memory_hotplug_cb(enum rte_mem_event event_type,
 		  const void *addr, size_t len, void *arg)
 {
 	if (event_type == RTE_MEM_EVENT_ALLOC) {
+		spdk_mem_register((void *)addr, len);
+
+		/* Now mark each segment so that DPDK won't later free it.
+		 * This ensures we don't have to deal with the memory
+		 * getting freed in different units than it was allocated.
+		 */
 		while (len > 0) {
 			struct rte_memseg *seg;
 
 			seg = rte_mem_virt2memseg(addr, NULL);
 			assert(seg != NULL);
-			assert(len >= seg->hugepage_sz);
-
-			spdk_mem_register((void *)seg->addr, seg->hugepage_sz);
+			seg->flags |= RTE_MEMSEG_FLAG_DO_NOT_FREE;
 			addr = (void *)((uintptr_t)addr + seg->hugepage_sz);
 			len -= seg->hugepage_sz;
 		}

lib/env_dpdk/pci.c

@@ -40,75 +40,155 @@
 #define PCI_CFG_SIZE		256
 #define PCI_EXT_CAP_ID_SN	0x03
 
-int
-spdk_pci_device_init(struct rte_pci_driver *driver,
-		     struct rte_pci_device *device)
+/* DPDK 18.11+ hotplug isn't robust. Multiple apps starting at the same time
+ * might cause the internal IPC to misbehave. Just retry in such case.
+ */
+#define DPDK_HOTPLUG_RETRY_COUNT 4
+
+static pthread_mutex_t g_pci_mutex = PTHREAD_MUTEX_INITIALIZER;
+static TAILQ_HEAD(, spdk_pci_device) g_pci_devices = TAILQ_HEAD_INITIALIZER(g_pci_devices);
+static TAILQ_HEAD(, spdk_pci_driver) g_pci_drivers = TAILQ_HEAD_INITIALIZER(g_pci_drivers);
+
+void
+spdk_pci_driver_register(struct spdk_pci_driver *driver)
 {
-	struct spdk_pci_enum_ctx *ctx = (struct spdk_pci_enum_ctx *)driver;
-	int rc;
-
-	if (!ctx->cb_fn) {
-#if RTE_VERSION >= RTE_VERSION_NUM(17, 05, 0, 4)
-		rte_pci_unmap_device(device);
-#elif RTE_VERSION >= RTE_VERSION_NUM(16, 11, 0, 0)
-		rte_eal_pci_unmap_device(device);
-#endif
-
-		/* Return a positive value to indicate that this device does not belong to this driver, but
-		 * this isn't an error. */
-		return 1;
-	}
-
-	rc = ctx->cb_fn(ctx->cb_arg, (struct spdk_pci_device *)device);
-	if (rc != 0) {
-		return rc;
-	}
-
-	spdk_vtophys_pci_device_added(device);
-	return 0;
-}
-
-int
-spdk_pci_device_fini(struct rte_pci_device *device)
-{
-	spdk_vtophys_pci_device_removed(device);
-	return 0;
+	TAILQ_INSERT_TAIL(&g_pci_drivers, driver, tailq);
 }
 
 void
-spdk_pci_device_detach(struct spdk_pci_device *device)
+spdk_pci_init(void)
 {
-#if RTE_VERSION >= RTE_VERSION_NUM(16, 11, 0, 0)
-#if RTE_VERSION < RTE_VERSION_NUM(17, 05, 0, 0)
-	rte_eal_device_remove(&device->device);
-#endif
-#endif
+#if RTE_VERSION >= RTE_VERSION_NUM(18, 11, 0, 0)
+	struct spdk_pci_driver *driver;
 
-#if RTE_VERSION >= RTE_VERSION_NUM(17, 11, 0, 3)
-	struct spdk_pci_addr	addr;
-	char			bdf[32];
-
-	addr.domain = device->addr.domain;
-	addr.bus = device->addr.bus;
-	addr.dev = device->addr.devid;
-	addr.func = device->addr.function;
-
-	spdk_pci_addr_fmt(bdf, sizeof(bdf), &addr);
-	if (rte_eal_dev_detach(&device->device) < 0) {
-		fprintf(stderr, "Failed to detach PCI device %s (device already removed?).\n", bdf);
+	/* We need to pre-register pci drivers for the pci devices to be
+	 * attachable in multi-process with DPDK 18.11+.
+	 *
+	 * DPDK 18.11+ does its best to ensure all devices are equally
+	 * attached or detached in all processes within a shared memory group.
+	 * For SPDK it means that if a device is hotplugged in the primary,
+	 * then DPDK will automatically send an IPC hotplug request to all other
+	 * processes. Those other processes may not have the same SPDK PCI
+	 * driver registered and may fail to attach the device. DPDK will send
+	 * back the failure status, and the the primary process will also fail
+	 * to hotplug the device. To prevent that, we need to pre-register the
+	 * pci drivers here.
+	 */
+	TAILQ_FOREACH(driver, &g_pci_drivers, tailq) {
+		assert(!driver->is_registered);
+		driver->is_registered = true;
+		rte_pci_register(&driver->driver);
 	}
+#endif
+}
+
+int
+spdk_pci_device_init(struct rte_pci_driver *_drv,
+		     struct rte_pci_device *_dev)
+{
+	struct spdk_pci_driver *driver = (struct spdk_pci_driver *)_drv;
+	struct spdk_pci_device *dev;
+	int rc;
+
+#if RTE_VERSION < RTE_VERSION_NUM(18, 11, 0, 0)
+	if (!driver->cb_fn) {
+#if RTE_VERSION < RTE_VERSION_NUM(17, 02, 0, 1)
+		rte_eal_pci_unmap_device(_dev);
+#endif
+		/* Return a positive value to indicate that this device does
+		 * not belong to this driver, but this isn't an error.
+		 */
+		return 1;
+	}
+#endif
+
+	dev = calloc(1, sizeof(*dev));
+	if (dev == NULL) {
+		return -1;
+	}
+
+	dev->dev_handle = _dev;
+	dev->driver = driver;
+
+	dev->addr.domain = _dev->addr.domain;
+	dev->addr.bus = _dev->addr.bus;
+	dev->addr.dev = _dev->addr.devid;
+	dev->addr.func = _dev->addr.function;
+	dev->id.vendor_id = _dev->id.vendor_id;
+	dev->id.device_id = _dev->id.device_id;
+	dev->id.subvendor_id = _dev->id.subsystem_vendor_id;
+	dev->id.subdevice_id = _dev->id.subsystem_device_id;
+	dev->socket_id = _dev->device.numa_node;
+
+	if (driver->cb_fn != NULL) {
+		rc = driver->cb_fn(driver->cb_arg, dev);
+		if (rc != 0) {
+			free(dev);
+			return rc;
+		}
+		dev->attached = true;
+	}
+
+	TAILQ_INSERT_TAIL(&g_pci_devices, dev, tailq);
+	spdk_vtophys_pci_device_added(dev->dev_handle);
+	return 0;
+}
+
+int
+spdk_pci_device_fini(struct rte_pci_device *_dev)
+{
+	struct spdk_pci_device *dev;
+
+	TAILQ_FOREACH(dev, &g_pci_devices, tailq) {
+		if (dev->dev_handle == _dev) {
+			break;
+		}
+	}
+
+	if (dev == NULL || dev->attached) {
+		/* The device might be still referenced somewhere in SPDK. */
+		return -1;
+	}
+
+	spdk_vtophys_pci_device_removed(dev->dev_handle);
+	TAILQ_REMOVE(&g_pci_devices, dev, tailq);
+	free(dev);
+	return 0;
+
+}
+
+void
+spdk_pci_device_detach(struct spdk_pci_device *dev)
+{
+	struct rte_pci_device *device = dev->dev_handle;
+
+	assert(dev->attached);
+	dev->attached = false;
+#if RTE_VERSION >= RTE_VERSION_NUM(18, 11, 0, 0)
+	char bdf[32];
+	int i = 0, rc;
+
+	snprintf(bdf, sizeof(bdf), "%s", device->device.name);
+	do {
+		rc = rte_eal_hotplug_remove("pci", bdf);
+	} while (rc == -ENOMSG && ++i <= DPDK_HOTPLUG_RETRY_COUNT);
+#elif RTE_VERSION >= RTE_VERSION_NUM(17, 11, 0, 3)
+	rte_eal_dev_detach(&device->device);
 #elif RTE_VERSION >= RTE_VERSION_NUM(17, 05, 0, 4)
 	rte_pci_detach(&device->addr);
 #else
+	rte_eal_device_remove(&device->device);
 	rte_eal_pci_detach(&device->addr);
 #endif
 }
 
 int
-spdk_pci_device_attach(struct spdk_pci_enum_ctx *ctx,
+spdk_pci_device_attach(struct spdk_pci_driver *driver,
 		       spdk_pci_enum_cb enum_cb,
 		       void *enum_ctx, struct spdk_pci_addr *pci_address)
 {
+	struct spdk_pci_device *dev;
+	int rc;
 #if RTE_VERSION >= RTE_VERSION_NUM(17, 11, 0, 3)
 	char bdf[32];
 
@@ -122,38 +202,66 @@ spdk_pci_device_attach(struct spdk_pci_enum_ctx *ctx,
 	addr.function = pci_address->func;
 #endif
 
-	pthread_mutex_lock(&ctx->mtx);
+	pthread_mutex_lock(&g_pci_mutex);
 
-	if (!ctx->is_registered) {
-		ctx->is_registered = true;
-#if RTE_VERSION >= RTE_VERSION_NUM(17, 05, 0, 4)
-		rte_pci_register(&ctx->driver);
-#else
-		rte_eal_pci_register(&ctx->driver);
-#endif
+	TAILQ_FOREACH(dev, &g_pci_devices, tailq) {
+		if (spdk_pci_addr_compare(&dev->addr, pci_address) == 0) {
+			break;
+		}
 	}
 
-	ctx->cb_fn = enum_cb;
-	ctx->cb_arg = enum_ctx;
-
-#if RTE_VERSION >= RTE_VERSION_NUM(17, 11, 0, 3)
-	if (rte_eal_dev_attach(bdf, "") != 0) {
-#elif RTE_VERSION >= RTE_VERSION_NUM(17, 05, 0, 4)
-	if (rte_pci_probe_one(&addr) != 0) {
-#else
-	if (rte_eal_pci_probe_one(&addr) != 0) {
-#endif
-		ctx->cb_arg = NULL;
-		ctx->cb_fn = NULL;
-		pthread_mutex_unlock(&ctx->mtx);
+	if (dev != NULL && dev->driver == driver) {
+		if (dev->attached) {
+			pthread_mutex_unlock(&g_pci_mutex);
 			return -1;
 		}
 
-	ctx->cb_arg = NULL;
-	ctx->cb_fn = NULL;
-	pthread_mutex_unlock(&ctx->mtx);
+		rc = enum_cb(enum_ctx, dev);
+		if (rc == 0) {
+			dev->attached = true;
+		}
+		pthread_mutex_unlock(&g_pci_mutex);
+		return rc;
+	}
 
-	return 0;
+	if (!driver->is_registered) {
+		driver->is_registered = true;
+#if RTE_VERSION >= RTE_VERSION_NUM(17, 05, 0, 4)
+		rte_pci_register(&driver->driver);
+#else
+		rte_eal_pci_register(&driver->driver);
+#endif
+	}
+
+	driver->cb_fn = enum_cb;
+	driver->cb_arg = enum_ctx;
+
+#if RTE_VERSION >= RTE_VERSION_NUM(18, 11, 0, 0)
+	int i = 0;
+
+	do {
+		rc = rte_eal_hotplug_add("pci", bdf, "");
+	} while (rc == -ENOMSG && ++i <= DPDK_HOTPLUG_RETRY_COUNT);
+
+	if (i > 1 && rc == -EEXIST) {
+		/* Even though the previous request timed out, the device
+		 * was attached successfully.
+		 */
+		rc = 0;
+	}
+#elif RTE_VERSION >= RTE_VERSION_NUM(17, 11, 0, 3)
+	rc = rte_eal_dev_attach(bdf, "");
+#elif RTE_VERSION >= RTE_VERSION_NUM(17, 05, 0, 4)
+	rc = rte_pci_probe_one(&addr);
+#else
+	rc = rte_eal_pci_probe_one(&addr);
+#endif
+
+	driver->cb_arg = NULL;
+	driver->cb_fn = NULL;
+	pthread_mutex_unlock(&g_pci_mutex);
+
+	return rc == 0 ? 0 : -1;
 }
 
 /* Note: You can call spdk_pci_enumerate from more than one thread
@@ -161,23 +269,40 @@ spdk_pci_device_attach(struct spdk_pci_enum_ctx *ctx,
  *       and rte_eal_pci_probe simultaneously.
  */
 int
-spdk_pci_enumerate(struct spdk_pci_enum_ctx *ctx,
+spdk_pci_enumerate(struct spdk_pci_driver *driver,
 		   spdk_pci_enum_cb enum_cb,
 		   void *enum_ctx)
 {
-	pthread_mutex_lock(&ctx->mtx);
+	struct spdk_pci_device *dev;
+	int rc;
 
-	if (!ctx->is_registered) {
-		ctx->is_registered = true;
+	pthread_mutex_lock(&g_pci_mutex);
+
+	TAILQ_FOREACH(dev, &g_pci_devices, tailq) {
+		if (dev->attached || dev->driver != driver) {
+			continue;
+		}
+
+		rc = enum_cb(enum_ctx, dev);
+		if (rc == 0) {
+			dev->attached = true;
+		} else if (rc < 0) {
+			pthread_mutex_unlock(&g_pci_mutex);
+			return -1;
+		}
+	}
+
+	if (!driver->is_registered) {
+		driver->is_registered = true;
 #if RTE_VERSION >= RTE_VERSION_NUM(17, 05, 0, 4)
-		rte_pci_register(&ctx->driver);
+		rte_pci_register(&driver->driver);
 #else
-		rte_eal_pci_register(&ctx->driver);
+		rte_eal_pci_register(&driver->driver);
 #endif
 	}
 
-	ctx->cb_fn = enum_cb;
-	ctx->cb_arg = enum_ctx;
+	driver->cb_fn = enum_cb;
+	driver->cb_arg = enum_ctx;
 
 #if RTE_VERSION >= RTE_VERSION_NUM(17, 11, 0, 3)
 	if (rte_bus_probe() != 0) {
@@ -186,15 +311,15 @@ spdk_pci_enumerate(struct spdk_pci_enum_ctx *ctx,
 #else
 	if (rte_eal_pci_probe() != 0) {
 #endif
-		ctx->cb_arg = NULL;
-		ctx->cb_fn = NULL;
-		pthread_mutex_unlock(&ctx->mtx);
+		driver->cb_arg = NULL;
+		driver->cb_fn = NULL;
+		pthread_mutex_unlock(&g_pci_mutex);
 		return -1;
 	}
 
-	ctx->cb_arg = NULL;
-	ctx->cb_fn = NULL;
-	pthread_mutex_unlock(&ctx->mtx);
+	driver->cb_arg = NULL;
+	driver->cb_fn = NULL;
+	pthread_mutex_unlock(&g_pci_mutex);
 
 	return 0;
 }
@@ -203,7 +328,7 @@ int
 spdk_pci_device_map_bar(struct spdk_pci_device *device, uint32_t bar,
 			void **mapped_addr, uint64_t *phys_addr, uint64_t *size)
 {
-	struct rte_pci_device *dev = device;
+	struct rte_pci_device *dev = device->dev_handle;
 
 	*mapped_addr = dev->mem_resource[bar].addr;
 	*phys_addr = (uint64_t)dev->mem_resource[bar].phys_addr;
@@ -233,13 +358,13 @@ spdk_pci_device_get_bus(struct spdk_pci_device *dev)
 uint8_t
 spdk_pci_device_get_dev(struct spdk_pci_device *dev)
 {
-	return dev->addr.devid;
+	return dev->addr.dev;
 }
 
 uint8_t
 spdk_pci_device_get_func(struct spdk_pci_device *dev)
 {
-	return dev->addr.function;
+	return dev->addr.func;
 }
 
 uint16_t
@@ -257,36 +382,25 @@ spdk_pci_device_get_device_id(struct spdk_pci_device *dev)
 uint16_t
 spdk_pci_device_get_subvendor_id(struct spdk_pci_device *dev)
 {
-	return dev->id.subsystem_vendor_id;
+	return dev->id.subvendor_id;
 }
 
 uint16_t
 spdk_pci_device_get_subdevice_id(struct spdk_pci_device *dev)
 {
-	return dev->id.subsystem_device_id;
+	return dev->id.subdevice_id;
 }
 
 struct spdk_pci_id
-spdk_pci_device_get_id(struct spdk_pci_device *pci_dev)
+spdk_pci_device_get_id(struct spdk_pci_device *dev)
 {
-	struct spdk_pci_id pci_id;
-
-	pci_id.vendor_id = spdk_pci_device_get_vendor_id(pci_dev);
-	pci_id.device_id = spdk_pci_device_get_device_id(pci_dev);
-	pci_id.subvendor_id = spdk_pci_device_get_subvendor_id(pci_dev);
-	pci_id.subdevice_id = spdk_pci_device_get_subdevice_id(pci_dev);
-
-	return pci_id;
+	return dev->id;
 }
 
 int
-spdk_pci_device_get_socket_id(struct spdk_pci_device *pci_dev)
+spdk_pci_device_get_socket_id(struct spdk_pci_device *dev)
 {
-#if RTE_VERSION >= RTE_VERSION_NUM(16, 11, 0, 0)
-	return pci_dev->device.numa_node;
-#else
-	return pci_dev->numa_node;
-#endif
+	return dev->socket_id;
 }
 
 int
@@ -295,9 +409,14 @@ spdk_pci_device_cfg_read(struct spdk_pci_device *dev, void *value, uint32_t len,
 	int rc;
 
 #if RTE_VERSION >= RTE_VERSION_NUM(17, 05, 0, 4)
-	rc = rte_pci_read_config(dev, value, len, offset);
+	rc = rte_pci_read_config(dev->dev_handle, value, len, offset);
 #else
-	rc = rte_eal_pci_read_config(dev, value, len, offset);
+	rc = rte_eal_pci_read_config(dev->dev_handle, value, len, offset);
+#endif
+
+#if defined(__FreeBSD__) && RTE_VERSION < RTE_VERSION_NUM(18, 11, 0, 0)
+	/* Older DPDKs return 0 on success and -1 on failure */
+	return rc;
 #endif
 	return (rc > 0 && (uint32_t) rc == len) ? 0 : -1;
 }
@@ -308,9 +427,14 @@ spdk_pci_device_cfg_write(struct spdk_pci_device *dev, void *value, uint32_t len
 	int rc;
 
 #if RTE_VERSION >= RTE_VERSION_NUM(17, 05, 0, 4)
-	rc = rte_pci_write_config(dev, value, len, offset);
+	rc = rte_pci_write_config(dev->dev_handle, value, len, offset);
 #else
-	rc = rte_eal_pci_write_config(dev, value, len, offset);
+	rc = rte_eal_pci_write_config(dev->dev_handle, value, len, offset);
+#endif
+
+#ifdef __FreeBSD__
+	/* DPDK returns 0 on success and -1 on failure */
+	return rc;
 #endif
 	return (rc > 0 && (uint32_t) rc == len) ? 0 : -1;
 }
@@ -397,16 +521,9 @@ spdk_pci_device_get_serial_number(struct spdk_pci_device *dev, char *sn, size_t
 }
 
 struct spdk_pci_addr
-spdk_pci_device_get_addr(struct spdk_pci_device *pci_dev)
+spdk_pci_device_get_addr(struct spdk_pci_device *dev)
 {
-	struct spdk_pci_addr pci_addr;
-
-	pci_addr.domain = spdk_pci_device_get_domain(pci_dev);
-	pci_addr.bus = spdk_pci_device_get_bus(pci_dev);
-	pci_addr.dev = spdk_pci_device_get_dev(pci_dev);
-	pci_addr.func = spdk_pci_device_get_func(pci_dev);
-
-	return pci_addr;
+	return dev->addr;
 }
 
 int

lib/env_dpdk/pci_ioat.c

@@ -88,24 +88,17 @@ static struct rte_pci_id ioat_driver_id[] = {
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
-static struct spdk_pci_enum_ctx g_ioat_pci_drv = {
+static struct spdk_pci_driver g_ioat_pci_drv = {
 	.driver = {
 		.drv_flags	= RTE_PCI_DRV_NEED_MAPPING,
 		.id_table	= ioat_driver_id,
-#if RTE_VERSION >= RTE_VERSION_NUM(16, 11, 0, 0)
 		.probe		= spdk_pci_device_init,
 		.remove		= spdk_pci_device_fini,
 		.driver.name	= "spdk_ioat",
-#else
-		.devinit	= spdk_pci_device_init,
-		.devuninit	= spdk_pci_device_fini,
-		.name		= "spdk_ioat",
-#endif
 	},
 
 	.cb_fn = NULL,
 	.cb_arg = NULL,
-	.mtx = PTHREAD_MUTEX_INITIALIZER,
 	.is_registered = false,
 };
 
@@ -121,3 +114,5 @@ spdk_pci_ioat_enumerate(spdk_pci_enum_cb enum_cb, void *enum_ctx)
 {
 	return spdk_pci_enumerate(&g_ioat_pci_drv, enum_cb, enum_ctx);
 }
+
+SPDK_PMD_REGISTER_PCI(g_ioat_pci_drv);

lib/env_dpdk/pci_nvme.c

@@ -36,7 +36,6 @@
 #include "spdk/pci_ids.h"
 
 static struct rte_pci_id nvme_pci_driver_id[] = {
-#if RTE_VERSION >= RTE_VERSION_NUM(16, 7, 0, 1)
 	{
 		.class_id = SPDK_PCI_CLASS_NVME,
 		.vendor_id = PCI_ANY_ID,
@@ -44,13 +43,10 @@ static struct rte_pci_id nvme_pci_driver_id[] = {
 		.subsystem_vendor_id = PCI_ANY_ID,
 		.subsystem_device_id = PCI_ANY_ID,
 	},
-#else
-	{RTE_PCI_DEVICE(0x8086, 0x0953)},
-#endif
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
-static struct spdk_pci_enum_ctx g_nvme_pci_drv = {
+static struct spdk_pci_driver g_nvme_pci_drv = {
 	.driver = {
 		.drv_flags	= RTE_PCI_DRV_NEED_MAPPING
 #if RTE_VERSION >= RTE_VERSION_NUM(18, 8, 0, 0)
@@ -58,20 +54,13 @@ static struct spdk_pci_enum_ctx g_nvme_pci_drv = {
 #endif
 		,
 		.id_table	= nvme_pci_driver_id,
-#if RTE_VERSION >= RTE_VERSION_NUM(16, 11, 0, 0)
 		.probe		= spdk_pci_device_init,
 		.remove		= spdk_pci_device_fini,
 		.driver.name	= "spdk_nvme",
-#else
-		.devinit	= spdk_pci_device_init,
-		.devuninit	= spdk_pci_device_fini,
-		.name		= "spdk_nvme",
-#endif
 	},
 
 	.cb_fn = NULL,
 	.cb_arg = NULL,
-	.mtx = PTHREAD_MUTEX_INITIALIZER,
 	.is_registered = false,
 };
 
@@ -87,3 +76,5 @@ spdk_pci_nvme_enumerate(spdk_pci_enum_cb enum_cb, void *enum_ctx)
 {
 	return spdk_pci_enumerate(&g_nvme_pci_drv, enum_cb, enum_ctx);
 }
+
+SPDK_PMD_REGISTER_PCI(g_nvme_pci_drv);

lib/env_dpdk/pci_virtio.c

@@ -41,7 +41,7 @@ static struct rte_pci_id virtio_pci_driver_id[] = {
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
-static struct spdk_pci_enum_ctx g_virtio_pci_drv = {
+static struct spdk_pci_driver g_virtio_pci_drv = {
 	.driver = {
 		.drv_flags	= RTE_PCI_DRV_NEED_MAPPING
 #if RTE_VERSION >= RTE_VERSION_NUM(18, 8, 0, 0)
@@ -49,20 +49,13 @@ static struct spdk_pci_enum_ctx g_virtio_pci_drv = {
 #endif
 		,
 		.id_table	= virtio_pci_driver_id,
-#if RTE_VERSION >= RTE_VERSION_NUM(16, 11, 0, 0)
 		.probe		= spdk_pci_device_init,
 		.remove		= spdk_pci_device_fini,
 		.driver.name	= "spdk_virtio",
-#else
-		.devinit	= spdk_pci_device_init,
-		.devuninit	= spdk_pci_device_fini,
-		.name		= "spdk_virtio",
-#endif
 	},
 
 	.cb_fn = NULL,
 	.cb_arg = NULL,
-	.mtx = PTHREAD_MUTEX_INITIALIZER,
 	.is_registered = false,
 };
 
@@ -78,3 +71,5 @@ spdk_pci_virtio_enumerate(spdk_pci_enum_cb enum_cb, void *enum_ctx)
 {
 	return spdk_pci_enumerate(&g_virtio_pci_drv, enum_cb, enum_ctx);
 }
+
+SPDK_PMD_REGISTER_PCI(g_virtio_pci_drv);

lib/env_dpdk/threads.c

@@ -31,7 +31,7 @@
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include "spdk/env.h"
+#include "env_internal.h"
 
 #include <rte_config.h>
 #include <rte_lcore.h>

lib/env_dpdk/vtophys.c

@@ -305,11 +305,7 @@ vtophys_get_paddr_pci(uint64_t vaddr)
 	struct spdk_vtophys_pci_device *vtophys_dev;
 	uintptr_t paddr;
 	struct rte_pci_device	*dev;
-#if RTE_VERSION >= RTE_VERSION_NUM(16, 11, 0, 1)
 	struct rte_mem_resource *res;
-#else
-	struct rte_pci_resource *res;
-#endif
 	unsigned r;
 
 	pthread_mutex_lock(&g_vtophys_pci_devices_mutex);

lib/iscsi/conn.c

@@ -51,6 +51,12 @@
 #include "iscsi/tgt_node.h"
 #include "iscsi/portal_grp.h"
 
+#define MAKE_DIGEST_WORD(BUF, CRC32C) \
+        (   ((*((uint8_t *)(BUF)+0)) = (uint8_t)((uint32_t)(CRC32C) >> 0)), \
+            ((*((uint8_t *)(BUF)+1)) = (uint8_t)((uint32_t)(CRC32C) >> 8)), \
+            ((*((uint8_t *)(BUF)+2)) = (uint8_t)((uint32_t)(CRC32C) >> 16)), \
+            ((*((uint8_t *)(BUF)+3)) = (uint8_t)((uint32_t)(CRC32C) >> 24)))
+
 #define SPDK_ISCSI_CONNECTION_MEMSET(conn)		\
 	memset(&(conn)->portal, 0, sizeof(*(conn)) -	\
 		offsetof(struct spdk_iscsi_conn, portal));
@@ -1261,6 +1267,22 @@ spdk_iscsi_conn_flush_pdus(void *_conn)
 void
 spdk_iscsi_conn_write_pdu(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu *pdu)
 {
+	uint32_t crc32c;
+
+	if (pdu->bhs.opcode != ISCSI_OP_LOGIN_RSP) {
+		/* Header Digest */
+		if (conn->header_digest) {
+			crc32c = spdk_iscsi_pdu_calc_header_digest(pdu);
+			MAKE_DIGEST_WORD(pdu->header_digest, crc32c);
+		}
+
+		/* Data Digest */
+		if (conn->data_digest && DGET24(pdu->bhs.data_segment_len) != 0) {
+			crc32c = spdk_iscsi_pdu_calc_data_digest(pdu);
+			MAKE_DIGEST_WORD(pdu->data_digest, crc32c);
+		}
+	}
+
 	TAILQ_INSERT_TAIL(&conn->write_pdu_list, pdu, tailq);
 	spdk_iscsi_conn_flush_pdus(conn);
 }

lib/iscsi/iscsi.c

@@ -117,12 +117,6 @@ static int spdk_iscsi_reject(struct spdk_iscsi_conn *conn, struct spdk_iscsi_pdu
 	    | (((uint32_t) *((uint8_t *)(BUF)+3)) << 24))	\
 	    == (CRC32C))
 
-#define MAKE_DIGEST_WORD(BUF, CRC32C) \
-	(   ((*((uint8_t *)(BUF)+0)) = (uint8_t)((uint32_t)(CRC32C) >> 0)), \
-	    ((*((uint8_t *)(BUF)+1)) = (uint8_t)((uint32_t)(CRC32C) >> 8)), \
-	    ((*((uint8_t *)(BUF)+2)) = (uint8_t)((uint32_t)(CRC32C) >> 16)), \
-	    ((*((uint8_t *)(BUF)+3)) = (uint8_t)((uint32_t)(CRC32C) >> 24)))
-
 #if 0
 static int
 spdk_match_digest_word(const uint8_t *buf, uint32_t crc32c)
@@ -307,7 +301,7 @@ spdk_islun2lun(uint64_t islun)
 	return lun_i;
 }
 
-static uint32_t
+uint32_t
 spdk_iscsi_pdu_calc_header_digest(struct spdk_iscsi_pdu *pdu)
 {
 	uint32_t crc32c;
@@ -325,7 +319,7 @@ spdk_iscsi_pdu_calc_header_digest(struct spdk_iscsi_pdu *pdu)
 	return crc32c;
 }
 
-static uint32_t
+uint32_t
 spdk_iscsi_pdu_calc_data_digest(struct spdk_iscsi_pdu *pdu)
 {
 	uint32_t data_len = DGET24(pdu->bhs.data_segment_len);
@@ -573,7 +567,6 @@ spdk_iscsi_build_iovecs(struct spdk_iscsi_conn *conn, struct iovec *iovec,
 			struct spdk_iscsi_pdu *pdu)
 {
 	int iovec_cnt = 0;
-	uint32_t crc32c;
 	int enable_digest;
 	int total_ahs_len;
 	int data_len;
@@ -601,9 +594,6 @@ spdk_iscsi_build_iovecs(struct spdk_iscsi_conn *conn, struct iovec *iovec,
 
 	/* Header Digest */
 	if (enable_digest && conn->header_digest) {
-		crc32c = spdk_iscsi_pdu_calc_header_digest(pdu);
-		MAKE_DIGEST_WORD(pdu->header_digest, crc32c);
-
 		iovec[iovec_cnt].iov_base = pdu->header_digest;
 		iovec[iovec_cnt].iov_len = ISCSI_DIGEST_LEN;
 		iovec_cnt++;
@@ -618,9 +608,6 @@ spdk_iscsi_build_iovecs(struct spdk_iscsi_conn *conn, struct iovec *iovec,
 
 	/* Data Digest */
 	if (enable_digest && conn->data_digest && data_len != 0) {
-		crc32c = spdk_iscsi_pdu_calc_data_digest(pdu);
-		MAKE_DIGEST_WORD(pdu->data_digest, crc32c);
-
 		iovec[iovec_cnt].iov_base = pdu->data_digest;
 		iovec[iovec_cnt].iov_len = ISCSI_DIGEST_LEN;
 		iovec_cnt++;

lib/iscsi/iscsi.h

@@ -435,6 +435,8 @@ int spdk_iscsi_copy_param2var(struct spdk_iscsi_conn *conn);
 
 void spdk_iscsi_task_cpl(struct spdk_scsi_task *scsi_task);
 void spdk_iscsi_task_mgmt_cpl(struct spdk_scsi_task *scsi_task);
+uint32_t spdk_iscsi_pdu_calc_header_digest(struct spdk_iscsi_pdu *pdu);
+uint32_t spdk_iscsi_pdu_calc_data_digest(struct spdk_iscsi_pdu *pdu);
 
 /* Memory management */
 void spdk_put_pdu(struct spdk_iscsi_pdu *pdu);

lib/nvme/nvme_ctrlr.c

@@ -1650,6 +1650,10 @@ nvme_ctrlr_remove_process(struct spdk_nvme_ctrlr *ctrlr,
 
 	TAILQ_REMOVE(&ctrlr->active_procs, proc, tailq);
 
+	if (ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
+		spdk_pci_device_detach(proc->devhandle);
+	}
+
 	spdk_dma_free(proc);
 }
 

lib/nvme/nvme_pcie.c

@@ -703,15 +703,14 @@ pcie_nvme_enum_cb(void *ctx, struct spdk_pci_device *pci_dev)
 	trid.trtype = SPDK_NVME_TRANSPORT_PCIE;
 	spdk_pci_addr_fmt(trid.traddr, sizeof(trid.traddr), &pci_addr);
 
-	/* Verify that this controller is not already attached */
 	ctrlr = spdk_nvme_get_ctrlr_by_trid_unsafe(&trid);
-	if (ctrlr) {
-		if (spdk_process_is_primary()) {
-			/* Already attached */
-			return 0;
-		} else {
-			return nvme_ctrlr_add_process(ctrlr, pci_dev);
+	if (!spdk_process_is_primary()) {
+		if (!ctrlr) {
+			SPDK_ERRLOG("Controller must be constructed in the primary process first.\n");
+			return -1;
 		}
+
+		return nvme_ctrlr_add_process(ctrlr, pci_dev);
 	}
 
 	/* check whether user passes the pci_addr */

lib/nvme/nvme_rdma.c

@@ -642,6 +642,13 @@ nvme_rdma_mr_map_notify(void *cb_ctx, struct spdk_mem_map *map,
 	return rc;
 }
 
+static int
+nvme_rdma_check_contiguous_entries(uint64_t addr_1, uint64_t addr_2)
+{
+	/* Two contiguous mappings will point to the same address which is the start of the RDMA MR. */
+	return addr_1 == addr_2;
+}
+
 static int
 nvme_rdma_register_mem(struct nvme_rdma_qpair *rqpair)
 {
@@ -649,7 +656,7 @@ nvme_rdma_register_mem(struct nvme_rdma_qpair *rqpair)
 	struct spdk_nvme_rdma_mr_map *mr_map;
 	const struct spdk_mem_map_ops nvme_rdma_map_ops = {
 		.notify_cb = nvme_rdma_mr_map_notify,
-		.are_contiguous = NULL
+		.are_contiguous = nvme_rdma_check_contiguous_entries
 	};
 
 	pthread_mutex_lock(&g_rdma_mr_maps_mutex);
@@ -873,6 +880,9 @@ nvme_rdma_build_contig_inline_request(struct nvme_rdma_qpair *rqpair,
 			(uint64_t)payload, &requested_size);
 
 	if (mr == NULL || requested_size < req->payload_size) {
+		if (mr) {
+			SPDK_ERRLOG("Data buffer split over multiple RDMA Memory Regions\n");
+		}
 		return -EINVAL;
 	}
 
@@ -920,7 +930,11 @@ nvme_rdma_build_contig_request(struct nvme_rdma_qpair *rqpair,
 	requested_size = req->payload_size;
 	mr = (struct ibv_mr *)spdk_mem_map_translate(rqpair->mr_map->map, (uint64_t)payload,
 			&requested_size);
+
 	if (mr == NULL || requested_size < req->payload_size) {
+		if (mr) {
+			SPDK_ERRLOG("Data buffer split over multiple RDMA Memory Regions\n");
+		}
 		return -1;
 	}
 
@@ -981,6 +995,9 @@ nvme_rdma_build_sgl_request(struct nvme_rdma_qpair *rqpair,
 				&mr_length);
 
 		if (mr == NULL || mr_length < sge_length) {
+			if (mr) {
+				SPDK_ERRLOG("Data buffer split over multiple RDMA Memory Regions\n");
+			}
 			return -1;
 		}
 
@@ -1074,6 +1091,9 @@ nvme_rdma_build_sgl_inline_request(struct nvme_rdma_qpair *rqpair,
 	mr = (struct ibv_mr *)spdk_mem_map_translate(rqpair->mr_map->map, (uint64_t)virt_addr,
 			&requested_size);
 	if (mr == NULL || requested_size < req->payload_size) {
+		if (mr) {
+			SPDK_ERRLOG("Data buffer split over multiple RDMA Memory Regions\n");
+		}
 		return -1;
 	}
 

lib/nvmf/nvmf.c

@@ -139,7 +139,7 @@ spdk_nvmf_tgt_create_poll_group(void *io_device, void *ctx_buf)
 	group->num_sgroups = tgt->opts.max_subsystems;
 	group->sgroups = calloc(tgt->opts.max_subsystems, sizeof(struct spdk_nvmf_subsystem_poll_group));
 	if (!group->sgroups) {
-		return -1;
+		return -ENOMEM;
 	}
 
 	for (sid = 0; sid < tgt->opts.max_subsystems; sid++) {
@@ -700,7 +700,7 @@ spdk_nvmf_poll_group_add(struct spdk_nvmf_poll_group *group,
 	if (rc == 0) {
 		spdk_nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ACTIVE);
 	} else {
-		spdk_nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_INACTIVE);
+		spdk_nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ERROR);
 	}
 
 	return rc;
@@ -743,7 +743,7 @@ _spdk_nvmf_qpair_destroy(void *ctx, int status)
 	struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr;
 
 	assert(qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING);
-	spdk_nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_INACTIVE);
+	spdk_nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ERROR);
 	qpair_ctx->qid = qpair->qid;
 
 	TAILQ_REMOVE(&qpair->group->qpairs, qpair, link);
@@ -781,8 +781,7 @@ spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_
 	/* The queue pair must be disconnected from the thread that owns it */
 	assert(qpair->group->thread == spdk_get_thread());
 
-	if (qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING ||
-	    qpair->state == SPDK_NVMF_QPAIR_INACTIVE) {
+	if (qpair->state != SPDK_NVMF_QPAIR_ACTIVE) {
 		/* This can occur if the connection is killed by the target,
 		 * which results in a notification that the connection
 		 * died. Send a message to defer the processing of this
@@ -1043,7 +1042,7 @@ _nvmf_subsystem_disconnect_next_qpair(void *ctx)
 	subsystem = qpair_ctx->subsystem;
 
 	TAILQ_FOREACH(qpair, &group->qpairs, link) {
-		if (qpair->ctrlr->subsys == subsystem) {
+		if ((qpair->ctrlr != NULL) && (qpair->ctrlr->subsys == subsystem)) {
 			break;
 		}
 	}
@@ -1084,7 +1083,7 @@ spdk_nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group,
 	sgroup->state = SPDK_NVMF_SUBSYSTEM_INACTIVE;
 
 	TAILQ_FOREACH(qpair, &group->qpairs, link) {
-		if (qpair->ctrlr->subsys == subsystem) {
+		if ((qpair->ctrlr != NULL) && (qpair->ctrlr->subsys == subsystem)) {
 			break;
 		}
 	}

lib/nvmf/nvmf_internal.h

@@ -59,7 +59,6 @@ enum spdk_nvmf_subsystem_state {
 
 enum spdk_nvmf_qpair_state {
 	SPDK_NVMF_QPAIR_UNINITIALIZED = 0,
-	SPDK_NVMF_QPAIR_INACTIVE,
 	SPDK_NVMF_QPAIR_ACTIVATING,
 	SPDK_NVMF_QPAIR_ACTIVE,
 	SPDK_NVMF_QPAIR_DEACTIVATING,

lib/nvmf/transport.c

@@ -1,8 +1,8 @@
 /*-
  *   BSD LICENSE
  *
- *   Copyright (c) Intel Corporation.
- *   All rights reserved.
+ *   Copyright (c) Intel Corporation. All rights reserved.
+ *   Copyright (c) 2018 Mellanox Technologies LTD. All rights reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
@@ -140,6 +140,9 @@ spdk_nvmf_transport_poll_group_create(struct spdk_nvmf_transport *transport)
 	struct spdk_nvmf_transport_poll_group *group;
 
 	group = transport->ops->poll_group_create(transport);
+	if (!group) {
+		return NULL;
+	}
 	group->transport = transport;
 
 	return group;

lib/thread/thread.c

@@ -530,7 +530,7 @@ spdk_get_io_channel(void *io_device)
 	pthread_mutex_unlock(&g_devlist_mutex);
 
 	rc = dev->create_cb(io_device, (uint8_t *)ch + sizeof(*ch));
-	if (rc == -1) {
+	if (rc != 0) {
 		pthread_mutex_lock(&g_devlist_mutex);
 		TAILQ_REMOVE(&ch->thread->io_channels, ch, tailq);
 		dev->refcnt--;

mk/spdk.common.mk

@@ -273,7 +273,12 @@ endef
 INSTALL_SHARED_LIB=\
 	$(Q)echo "  INSTALL $(DESTDIR)$(libdir)/$(notdir $(SHARED_LINKED_LIB))"; \
 	install -d -m 755 "$(DESTDIR)$(libdir)"; \
-	install -m 755 "$(SHARED_REALNAME_LIB)" "$(DESTDIR)$(libdir)/"; \
+	if file --mime-type $(SHARED_REALNAME_LIB) | grep -q 'application/x-sharedlib'; then \
+		perm_mode=755; \
+	else \
+		perm_mode=644; \
+	fi; \
+	install -m $$perm_mode "$(SHARED_REALNAME_LIB)" "$(DESTDIR)$(libdir)/"; \
 	$(call spdk_install_lib_symlink,$(notdir $(SHARED_REALNAME_LIB)),$(notdir $(SHARED_LINKED_LIB)));
 
 # Install an app binary
@@ -282,6 +287,11 @@ INSTALL_APP=\
 	install -d -m 755 "$(DESTDIR)$(bindir)"; \
 	install -m 755 "$(APP)" "$(DESTDIR)$(bindir)/"
 
+INSTALL_EXAMPLE=\
+	$(Q)echo "  INSTALL $(DESTDIR)$(bindir)/spdk_$(strip $(subst /,_,$(subst $(SPDK_ROOT_DIR)/examples/, ,$(CURDIR))))"; \
+	install -d -m 755 "$(DESTDIR)$(bindir)"; \
+	install -m 755 "$(APP)" "$(DESTDIR)$(bindir)/spdk_$(strip $(subst /,_,$(subst $(SPDK_ROOT_DIR)/examples/, ,$(CURDIR))))"
+
 # Install a header
 INSTALL_HEADER=\
 	$(Q)echo "  INSTALL $@"; \

pkg/spdk.spec

@@ -0,0 +1,171 @@
+# Build documentation package
+%bcond_with doc
+
+Name: spdk
+Version: 18.10.x
+Release: 0%{?dist}
+Epoch: 0
+URL: http://spdk.io
+
+Source: https://github.com/spdk/spdk/archive/v18.10.x.tar.gz#/%{name}-%{version}.tar.gz
+Summary: Set of libraries and utilities for high performance user-mode storage
+
+%define spdk_build_dir %{name}-%{version}
+
+%define package_version %{epoch}:%{version}-%{release}
+
+%define install_datadir %{buildroot}/%{_datadir}/%{name}
+%define install_sbindir %{buildroot}/%{_sbindir}
+%define install_docdir %{buildroot}/%{_docdir}/%{name}
+
+# Distros that don't support python3 will use python2
+%if "%{dist}" == ".el7"
+%define use_python2 1
+%else
+%define use_python2 0
+%endif
+
+License: BSD
+
+# Only x86_64 is supported
+ExclusiveArch: x86_64
+
+BuildRequires: gcc gcc-c++ make
+BuildRequires: dpdk-devel, numactl-devel
+BuildRequires: libiscsi-devel, libaio-devel, openssl-devel, libuuid-devel
+BuildRequires: libibverbs-devel, librdmacm-devel
+%if %{with doc}
+BuildRequires: doxygen mscgen graphviz
+%endif
+
+# Install dependencies
+Requires: dpdk >= 17.11, numactl-libs, openssl-libs
+Requires: libiscsi, libaio, libuuid
+# NVMe over Fabrics
+Requires: librdmacm, librdmacm
+Requires(post): /sbin/ldconfig
+Requires(postun): /sbin/ldconfig
+
+%description
+The Storage Performance Development Kit provides a set of tools
+and libraries for writing high performance, scalable, user-mode storage
+applications.
+
+
+%package devel
+Summary: Storage Performance Development Kit development files
+Requires: %{name}%{?_isa} = %{package_version}
+Provides: %{name}-static%{?_isa} = %{package_version}
+
+%description devel
+This package contains the headers and other files needed for
+developing applications with the Storage Performance Development Kit.
+
+
+%package tools
+Summary: Storage Performance Development Kit tools files
+%if "%{use_python2}" == "0"
+Requires: %{name}%{?_isa} = %{package_version} python3 python3-configshell python3-pexpect
+%else
+Requires: %{name}%{?_isa} = %{package_version} python python-configshell pexpect
+%endif
+BuildArch: noarch
+
+%description tools
+%{summary}
+
+
+%if %{with doc}
+%package doc
+Summary: Storage Performance Development Kit documentation
+BuildArch: noarch
+
+%description doc
+%{summary}
+%endif
+
+
+%prep
+# add -q
+%autosetup -n %{spdk_build_dir}
+
+
+%build
+./configure --prefix=%{_usr} \
+	--disable-tests \
+	--without-crypto \
+	--with-dpdk=/usr/share/dpdk/x86_64-default-linuxapp-gcc \
+	--without-fio \
+	--with-vhost \
+	--without-pmdk \
+	--without-vpp \
+	--without-rbd \
+	--with-rdma \
+	--with-shared \
+	--with-iscsi-initiator \
+	--without-vtune
+
+make -j`nproc` all
+
+%if %{with doc}
+make -C doc
+%endif
+
+%install
+%make_install -j`nproc` prefix=%{_usr} libdir=%{_libdir} datadir=%{_datadir}
+
+# Install tools
+mkdir -p %{install_datadir}
+find scripts -type f -regextype egrep -regex '.*(spdkcli|rpc).*[.]py' \
+	-exec cp --parents -t %{install_datadir} {} ";"
+
+# env is banned - replace '/usr/bin/env anything' with '/usr/bin/anything'
+find %{install_datadir}/scripts -type f -regextype egrep -regex '.*([.]py|[.]sh)' \
+	-exec sed -i -E '1s@#!/usr/bin/env (.*)@#!/usr/bin/\1@' {} +
+
+%if "%{use_python2}" == "1"
+find %{install_datadir}/scripts -type f -regextype egrep -regex '.*([.]py)' \
+	-exec sed -i -E '1s@#!/usr/bin/python3@#!/usr/bin/python2@' {} +
+%endif
+
+# synlinks to tools
+mkdir -p %{install_sbindir}
+ln -sf -r %{install_datadir}/scripts/rpc.py %{install_sbindir}/%{name}-rpc
+ln -sf -r %{install_datadir}/scripts/spdkcli.py %{install_sbindir}/%{name}-cli
+
+%if %{with doc}
+# Install doc
+mkdir -p %{install_docdir}
+mv doc/output/html/ %{install_docdir}
+%endif
+
+
+%post -p /sbin/ldconfig
+%postun -p /sbin/ldconfig
+
+
+%files
+%{_bindir}/spdk_*
+%{_libdir}/*.so.*
+
+
+%files devel
+%{_includedir}/%{name}
+%{_libdir}/*.a
+%{_libdir}/*.so
+
+
+%files tools
+%{_datadir}/%{name}/scripts
+%{_sbindir}/%{name}-rpc
+%{_sbindir}/%{name}-cli
+
+%if %{with doc}
+%files doc
+%{_docdir}/%{name}
+%endif
+
+
+%changelog
+* Tue Sep 18 2018 Pawel Wodkowski <pawelx.wodkowski@intel.com> - 0:18.07-3
+- Initial RPM release

scripts/common.sh

@@ -1,6 +1,35 @@
 # Common shell utility functions
 
-function iter_pci_class_code() {
+# Check if PCI device is on PCI_WHITELIST and not on PCI_BLACKLIST
+# Env:
+# if PCI_WHITELIST is empty assume device is whitelistened
+# if PCI_BLACKLIST is empty assume device is NOT blacklistened
+# Params:
+# $1 - PCI BDF
+function pci_can_use() {
+	local i
+
+	# The '\ ' part is important
+	if [[ " $PCI_BLACKLIST " =~ \ $1\  ]] ; then
+		return 1
+	fi
+
+	if [[ -z "$PCI_WHITELIST" ]]; then
+		#no whitelist specified, bind all devices
+		return 0
+	fi
+
+	for i in $PCI_WHITELIST; do
+		if [ "$i" == "$1" ] ; then
+			return 0
+		fi
+	done
+
+	return 1
+}
+
+# This function will ignore PCI PCI_WHITELIST and PCI_BLACKLIST
+function iter_all_pci_class_code() {
 	local class="$(printf %02x $((0x$1)))"
 	local subclass="$(printf %02x $((0x$2)))"
 	local progif="$(printf %02x $((0x$3)))"
@@ -17,7 +46,25 @@ function iter_pci_class_code() {
 				'{if (cc ~ $2) print $1}' | tr -d '"'
 		fi
 	elif hash pciconf &>/dev/null; then
-		addr=($(pciconf -l | grep -i "class=0x${class}${subclass}${progif}" | \
+		local addr=($(pciconf -l | grep -i "class=0x${class}${subclass}${progif}" | \
+			cut -d$'\t' -f1 | sed -e 's/^[a-zA-Z0-9_]*@pci//g' | tr ':' ' '))
+		printf "%04x:%02x:%02x:%x\n" ${addr[0]} ${addr[1]} ${addr[2]} ${addr[3]}
+	else
+		echo "Missing PCI enumeration utility"
+		exit 1
+	fi
+}
+
+# This function will ignore PCI PCI_WHITELIST and PCI_BLACKLIST
+function iter_all_pci_dev_id() {
+	local ven_id="$(printf %04x $((0x$1)))"
+	local dev_id="$(printf %04x $((0x$2)))"
+
+	if hash lspci &>/dev/null; then
+		lspci -mm -n -D | awk -v ven="\"$ven_id\"" -v dev="\"${dev_id}\"" -F " " \
+			'{if (ven ~ $3 && dev ~ $4) print $1}' | tr -d '"'
+	elif hash pciconf &>/dev/null; then
+		local addr=($(pciconf -l | grep -i "chip=0x${dev_id}${ven_id}" | \
 			cut -d$'\t' -f1 | sed -e 's/^[a-zA-Z0-9_]*@pci//g' | tr ':' ' '))
 		printf "%04x:%02x:%02x:%x\n" ${addr[0]} ${addr[1]} ${addr[2]} ${addr[3]}
 	else
@@ -27,18 +74,23 @@ function iter_pci_class_code() {
 }
 
 function iter_pci_dev_id() {
-	local ven_id="$(printf %04x $((0x$1)))"
-	local dev_id="$(printf %04x $((0x$2)))"
+	local bdf=""
 
-	if hash lspci &>/dev/null; then
-		lspci -mm -n -D | awk -v ven="\"$ven_id\"" -v dev="\"${dev_id}\"" -F " " \
-			'{if (ven ~ $3 && dev ~ $4) print $1}' | tr -d '"'
-	elif hash pciconf &>/dev/null; then
-		addr=($(pciconf -l | grep -i "chip=0x${dev_id}${ven_id}" | \
-			cut -d$'\t' -f1 | sed -e 's/^[a-zA-Z0-9_]*@pci//g' | tr ':' ' '))
-		printf "%04x:%02x:%02x:%x\n" ${addr[0]} ${addr[1]} ${addr[2]} ${addr[3]}
-	else
-		echo "Missing PCI enumeration utility"
-		exit 1
+	for bdf in $(iter_all_pci_dev_id "$@"); do
+		if pci_can_use "$bdf"; then
+			echo "$bdf"
 		fi
+	done
+}
+
+# This function will filter out PCI devices using PCI_WHITELIST and PCI_BLACKLIST
+# See function pci_can_use()
+function iter_pci_class_code() {
+	local bdf=""
+
+	for bdf in $(iter_all_pci_class_code "$@"); do
+		if pci_can_use "$bdf"; then
+			echo "$bdf"
+		fi
+	done
 }

scripts/setup.sh

@@ -41,47 +41,48 @@ function usage()
 	echo "HUGENODE          Specific NUMA node to allocate hugepages on. To allocate"
 	echo "                  hugepages on multiple nodes run this script multiple times -"
 	echo "                  once for each node."
-	echo "PCI_WHITELIST     Whitespace separated list of PCI devices (NVMe, I/OAT, Virtio) to bind."
+	echo "PCI_WHITELIST"
+	echo "PCI_BLACKLIST     Whitespace separated list of PCI devices (NVMe, I/OAT, Virtio)."
 	echo "                  Each device must be specified as a full PCI address."
 	echo "                  E.g. PCI_WHITELIST=\"0000:01:00.0 0000:02:00.0\""
 	echo "                  To blacklist all PCI devices use a non-valid address."
 	echo "                  E.g. PCI_WHITELIST=\"none\""
-	echo "                  If empty or unset, all PCI devices will be bound."
+	echo "                  If PCI_WHITELIST and PCI_BLACKLIST are empty or unset, all PCI devices"
+	echo "                  will be bound."
+	echo "                  Each device in PCI_BLACKLIST will be ignored (driver won't be changed)."
+	echo "                  PCI_BLACKLIST has precedence over PCI_WHITELIST."
 	echo "TARGET_USER       User that will own hugepage mountpoint directory and vfio groups."
 	echo "                  By default the current user will be used."
+	echo "DRIVER_OVERRIDE   Disable automatic vfio-pci/uio_pci_generic selection and forcefully"
+	echo "                  bind devices to the given driver."
+	echo "                  E.g. DRIVER_OVERRIDE=uio_pci_generic or DRIVER_OVERRIDE=vfio-pci"
 	exit 0
 }
 
 # In monolithic kernels the lsmod won't work. So
-# back that with a /sys/modules check. Return a different code for
-# built-in vs module just in case we want that down the road.
+# back that with a /sys/modules. We also check
+# /sys/bus/pci/drivers/ as neither lsmod nor /sys/modules might
+# contain needed info (like in Fedora-like OS).
 function check_for_driver {
-	$(lsmod | grep $1 > /dev/null)
-	if [ $? -eq 0 ]; then
+	if lsmod | grep -q ${1//-/_}; then
 		return 1
-	else
-		if [[ -d /sys/module/$1 ]]; then
-			return 2
-		else
-			return 0
 	fi
+
+	if [[ -d /sys/module/${1} || \
+			-d /sys/module/${1//-/_} || \
+			-d /sys/bus/pci/drivers/${1} || \
+			-d /sys/bus/pci/drivers/${1//-/_} ]]; then
+		return 2
 	fi
 	return 0
 }
 
-function pci_can_bind() {
-	if [[ ${#PCI_WHITELIST[@]} == 0 ]]; then
-		#no whitelist specified, bind all devices
-		return 1
-	fi
-
-	for i in ${PCI_WHITELIST[@]}
-	do
-		if [ "$i" == "$1" ] ; then
-			 return 1
-		fi
-	done
-	return 0
+function pci_dev_echo() {
+	local bdf="$1"
+	local vendor="$(cat /sys/bus/pci/devices/$bdf/vendor)"
+	local device="$(cat /sys/bus/pci/devices/$bdf/device)"
+	shift
+	echo "$bdf (${vendor#0x} ${device#0x}): $@"
 }
 
 function linux_bind_driver() {
@@ -94,6 +95,7 @@ function linux_bind_driver() {
 		old_driver_name=$(basename $(readlink /sys/bus/pci/devices/$bdf/driver))
 
 		if [ "$driver_name" = "$old_driver_name" ]; then
+			pci_dev_echo "$bdf" "Already using the $old_driver_name driver"
 			return 0
 		fi
 
@@ -101,7 +103,7 @@ function linux_bind_driver() {
 		echo "$bdf" > "/sys/bus/pci/devices/$bdf/driver/unbind"
 	fi
 
-	echo "$bdf ($ven_dev_id): $old_driver_name -> $driver_name"
+	pci_dev_echo "$bdf" "$old_driver_name -> $driver_name"
 
 	echo "$ven_dev_id" > "/sys/bus/pci/drivers/$driver_name/new_id" 2> /dev/null || true
 	echo "$bdf" > "/sys/bus/pci/drivers/$driver_name/bind" 2> /dev/null || true
@@ -164,19 +166,23 @@ function get_virtio_names_from_bdf {
 }
 
 function configure_linux_pci {
+	if [ -z "${DRIVER_OVERRIDE}" ]; then
 		driver_name=vfio-pci
 		if [ -z "$(ls /sys/kernel/iommu_groups)" ]; then
 			# No IOMMU. Use uio.
 			driver_name=uio_pci_generic
 		fi
+	else
+		driver_name="${DRIVER_OVERRIDE}"
+	fi
 
 	# NVMe
-	modprobe $driver_name || true
-	for bdf in $(iter_pci_class_code 01 08 02); do
+	modprobe $driver_name
+	for bdf in $(iter_all_pci_class_code 01 08 02); do
 		blkname=''
 		get_nvme_name_from_bdf "$bdf" blkname
-		if pci_can_bind $bdf == "0" ; then
-			echo "Skipping un-whitelisted NVMe controller $blkname ($bdf)"
+		if ! pci_can_use $bdf; then
+			pci_dev_echo "$bdf" "Skipping un-whitelisted NVMe controller $blkname"
 			continue
 		fi
 		if [ "$blkname" != "" ]; then
@@ -187,7 +193,7 @@ function configure_linux_pci {
 		if [ "$mountpoints" = "0" ]; then
 			linux_bind_driver "$bdf" "$driver_name"
 		else
-			echo Active mountpoints on /dev/$blkname, so not binding PCI dev $bdf
+			pci_dev_echo "$bdf" "Active mountpoints on /dev/$blkname, so not binding PCI dev"
 		fi
 	done
 
@@ -198,11 +204,12 @@ function configure_linux_pci {
 	| awk -F"x" '{print $2}' > $TMP
 
 	for dev_id in `cat $TMP`; do
-		for bdf in $(iter_pci_dev_id 8086 $dev_id); do
-			if pci_can_bind $bdf == "0" ; then
-				echo "Skipping un-whitelisted I/OAT device at $bdf"
+		for bdf in $(iter_all_pci_dev_id 8086 $dev_id); do
+			if ! pci_can_use $bdf; then
+				pci_dev_echo "$bdf" "Skipping un-whitelisted I/OAT device"
 				continue
 			fi
+
 			linux_bind_driver "$bdf" "$driver_name"
 		done
 	done
@@ -215,16 +222,16 @@ function configure_linux_pci {
 	| awk -F"x" '{print $2}' > $TMP
 
 	for dev_id in `cat $TMP`; do
-		for bdf in $(iter_pci_dev_id 1af4 $dev_id); do
-			if pci_can_bind $bdf == "0" ; then
-				echo "Skipping un-whitelisted Virtio device at $bdf"
+		for bdf in $(iter_all_pci_dev_id 1af4 $dev_id); do
+			if ! pci_can_use $bdf; then
+				pci_dev_echo "$bdf" "Skipping un-whitelisted Virtio device at $bdf"
 				continue
 			fi
 			blknames=''
 			get_virtio_names_from_bdf "$bdf" blknames
 			for blkname in $blknames; do
 				if mount | grep -q "/dev/$blkname"; then
-					echo Active mountpoints on /dev/$blkname, so not binding PCI dev $bdf
+					pci_dev_echo "$bdf" "Active mountpoints on /dev/$blkname, so not binding"
 					continue 2
 				fi
 			done
@@ -251,7 +258,7 @@ function cleanup_linux {
 	done
 	shopt -u extglob nullglob
 
-	files_to_clean+="$(echo /dev/shm/* | egrep '(spdk_tgt|iscsi|vhost|nvmf|rocksdb|bdevtest|bdevperf)_trace|spdk_iscsi_conns' || true) "
+	files_to_clean+="$(ls -1 /dev/shm/* | egrep '(spdk_tgt|iscsi|vhost|nvmf|rocksdb|bdevtest|bdevperf)_trace|spdk_iscsi_conns' || true) "
 	files_to_clean="$(readlink -e assert_not_empty $files_to_clean || true)"
 	if [[ -z "$files_to_clean" ]]; then
 		echo "Clean"
@@ -353,9 +360,9 @@ function reset_linux_pci {
 	check_for_driver nvme
 	driver_loaded=$?
 	set -e
-	for bdf in $(iter_pci_class_code 01 08 02); do
-		if pci_can_bind $bdf == "0" ; then
-			echo "Skipping un-whitelisted NVMe controller $blkname ($bdf)"
+	for bdf in $(iter_all_pci_class_code 01 08 02); do
+		if ! pci_can_use $bdf; then
+			pci_dev_echo "$bdf" "Skipping un-whitelisted NVMe controller $blkname"
 			continue
 		fi
 		if [ $driver_loaded -ne 0 ]; then
@@ -376,9 +383,9 @@ function reset_linux_pci {
 	driver_loaded=$?
 	set -e
 	for dev_id in `cat $TMP`; do
-		for bdf in $(iter_pci_dev_id 8086 $dev_id); do
-			if pci_can_bind $bdf == "0" ; then
-				echo "Skipping un-whitelisted I/OAT device at $bdf"
+		for bdf in $(iter_all_pci_dev_id 8086 $dev_id); do
+			if ! pci_can_use $bdf; then
+				pci_dev_echo "$bdf" "Skipping un-whitelisted I/OAT device"
 				continue
 			fi
 			if [ $driver_loaded -ne 0 ]; then
@@ -402,9 +409,9 @@ function reset_linux_pci {
 	#  underscore vs. dash right in the virtio_scsi name.
 	modprobe virtio-pci || true
 	for dev_id in `cat $TMP`; do
-		for bdf in $(iter_pci_dev_id 1af4 $dev_id); do
-			if pci_can_bind $bdf == "0" ; then
-				echo "Skipping un-whitelisted Virtio device at $bdf"
+		for bdf in $(iter_all_pci_dev_id 1af4 $dev_id); do
+			if ! pci_can_use $bdf; then
+				pci_dev_echo "$bdf" "Skipping un-whitelisted Virtio device at"
 				continue
 			fi
 			linux_bind_driver "$bdf" virtio-pci
@@ -453,47 +460,56 @@ function status_linux {
 		printf "%-6s %10s %8s / %6s\n" $node $huge_size $free_pages $all_pages
 	fi
 
+	echo ""
 	echo "NVMe devices"
 
-	echo -e "BDF\t\tNuma Node\tDriver name\t\tDevice name"
-	for bdf in $(iter_pci_class_code 01 08 02); do
-		driver=`grep DRIVER /sys/bus/pci/devices/$bdf/uevent |awk -F"=" '{print $2}'`
-		node=`cat /sys/bus/pci/devices/$bdf/numa_node`;
+	echo -e "BDF\t\tVendor\tDevice\tNUMA\tDriver\t\tDevice name"
+	for bdf in $(iter_all_pci_class_code 01 08 02); do
+		driver=$(grep DRIVER /sys/bus/pci/devices/$bdf/uevent |awk -F"=" '{print $2}')
+		node=$(cat /sys/bus/pci/devices/$bdf/numa_node)
+		device=$(cat /sys/bus/pci/devices/$bdf/device)
+		vendor=$(cat /sys/bus/pci/devices/$bdf/vendor)
 		if [ "$driver" = "nvme" -a -d /sys/bus/pci/devices/$bdf/nvme ]; then
 			name="\t"`ls /sys/bus/pci/devices/$bdf/nvme`;
 		else
 			name="-";
 		fi
-		echo -e "$bdf\t$node\t\t$driver\t\t$name";
+		echo -e "$bdf\t${vendor#0x}\t${device#0x}\t$node\t$driver\t\t$name";
 	done
 
+	echo ""
 	echo "I/OAT DMA"
 
 	#collect all the device_id info of ioat devices.
 	TMP=`grep "PCI_DEVICE_ID_INTEL_IOAT" $rootdir/include/spdk/pci_ids.h \
 	| awk -F"x" '{print $2}'`
-	echo -e "BDF\t\tNuma Node\tDriver Name"
+	echo -e "BDF\t\tVendor\tDevice\tNUMA\tDriver"
 	for dev_id in $TMP; do
-		for bdf in $(iter_pci_dev_id 8086 $dev_id); do
-			driver=`grep DRIVER /sys/bus/pci/devices/$bdf/uevent |awk -F"=" '{print $2}'`
-			node=`cat /sys/bus/pci/devices/$bdf/numa_node`;
-			echo -e "$bdf\t$node\t\t$driver"
+		for bdf in $(iter_all_pci_dev_id 8086 $dev_id); do
+			driver=$(grep DRIVER /sys/bus/pci/devices/$bdf/uevent |awk -F"=" '{print $2}')
+			node=$(cat /sys/bus/pci/devices/$bdf/numa_node)
+			device=$(cat /sys/bus/pci/devices/$bdf/device)
+			vendor=$(cat /sys/bus/pci/devices/$bdf/vendor)
+			echo -e "$bdf\t${vendor#0x}\t${device#0x}\t$node\t$driver"
 		done
 	done
 
+	echo ""
 	echo "virtio"
 
 	#collect all the device_id info of virtio devices.
 	TMP=`grep "PCI_DEVICE_ID_VIRTIO" $rootdir/include/spdk/pci_ids.h \
 	| awk -F"x" '{print $2}'`
-	echo -e "BDF\t\tNuma Node\tDriver Name\t\tDevice Name"
+	echo -e "BDF\t\tVendor\tDevice\tNUMA\tDriver\t\tDevice name"
 	for dev_id in $TMP; do
-		for bdf in $(iter_pci_dev_id 1af4 $dev_id); do
-			driver=`grep DRIVER /sys/bus/pci/devices/$bdf/uevent |awk -F"=" '{print $2}'`
-			node=`cat /sys/bus/pci/devices/$bdf/numa_node`;
+		for bdf in $(iter_all_pci_dev_id 1af4 $dev_id); do
+			driver=$(grep DRIVER /sys/bus/pci/devices/$bdf/uevent |awk -F"=" '{print $2}')
+			node=$(cat /sys/bus/pci/devices/$bdf/numa_node)
+			device=$(cat /sys/bus/pci/devices/$bdf/device)
+			vendor=$(cat /sys/bus/pci/devices/$bdf/vendor)
 			blknames=''
 			get_virtio_names_from_bdf "$bdf" blknames
-			echo -e "$bdf\t$node\t\t$driver\t\t$blknames"
+			echo -e "$bdf\t${vendor#0x}\t${device#0x}\t$node\t\t$driver\t\t$blknames"
 		done
 	done
 }
@@ -551,6 +567,7 @@ fi
 
 : ${HUGEMEM:=2048}
 : ${PCI_WHITELIST:=""}
+: ${PCI_BLACKLIST:=""}
 
 if [ -n "$NVME_WHITELIST" ]; then
 	PCI_WHITELIST="$PCI_WHITELIST $NVME_WHITELIST"
@@ -560,8 +577,6 @@ if [ -n "$SKIP_PCI" ]; then
 	PCI_WHITELIST="none"
 fi
 
-declare -a PCI_WHITELIST=(${PCI_WHITELIST})
-
 if [ -z "$TARGET_USER" ]; then
 	TARGET_USER="$SUDO_USER"
 	if [ -z "$TARGET_USER" ]; then

scripts/vagrant/autorun-spdk.conf

@@ -3,6 +3,7 @@ SPDK_BUILD_DOC=1
 SPDK_RUN_CHECK_FORMAT=1
 SPDK_RUN_SCANBUILD=1
 SPDK_RUN_VALGRIND=1
+SPDK_RUN_FUNCTIONAL_TEST=1
 SPDK_TEST_UNITTEST=1
 SPDK_TEST_ISCSI=0
 SPDK_TEST_ISCSI_INITIATOR=0

test/common/autotest_common.sh

@@ -40,6 +40,7 @@ fi
 : ${SPDK_RUN_CHECK_FORMAT=1}; export SPDK_RUN_CHECK_FORMAT
 : ${SPDK_RUN_SCANBUILD=1}; export SPDK_RUN_SCANBUILD
 : ${SPDK_RUN_VALGRIND=1}; export SPDK_RUN_VALGRIND
+: ${SPDK_RUN_FUNCTIONAL_TEST=1}; export SPDK_RUN_FUNCTIONAL_TEST
 : ${SPDK_TEST_UNITTEST=1}; export SPDK_TEST_UNITTEST
 : ${SPDK_TEST_ISCSI=1}; export SPDK_TEST_ISCSI
 : ${SPDK_TEST_ISCSI_INITIATOR=1}; export SPDK_TEST_ISCSI_INITIATOR
@@ -59,7 +60,9 @@ fi
 : ${SPDK_RUN_ASAN=1}; export SPDK_RUN_ASAN
 : ${SPDK_RUN_UBSAN=1}; export SPDK_RUN_UBSAN
 : ${SPDK_RUN_INSTALLED_DPDK=1}; export SPDK_RUN_INSTALLED_DPDK
-: ${SPDK_TEST_CRYPTO=1}; export SPDK_TEST_CRYPTO
+
+SPDK_TEST_CRYPTO=0
+export SPDK_TEST_CRYPTO
 
 if [ -z "$DEPENDENCY_DIR" ]; then
 	export DEPENDENCY_DIR=/home/sys_sgsw

test/common/config/vm_setup.sh

@@ -400,6 +400,7 @@ SPDK_BUILD_DOC=1
 SPDK_RUN_CHECK_FORMAT=1
 SPDK_RUN_SCANBUILD=1
 SPDK_RUN_VALGRIND=1
+SPDK_RUN_FUNCTIONAL_TEST=1
 SPDK_TEST_UNITTEST=1
 SPDK_TEST_ISCSI=1
 SPDK_TEST_ISCSI_INITIATOR=1

test/env/memory/memory_ut.c

@@ -257,6 +257,12 @@ test_mem_map_translation(void)
 	CU_ASSERT(addr == 0);
 	CU_ASSERT(mapping_length == VALUE_2MB * 3)
 
+	/* Translate an unaligned address */
+	mapping_length = VALUE_2MB * 3;
+	addr = spdk_mem_map_translate(map, VALUE_4KB, &mapping_length);
+	CU_ASSERT(addr == 0);
+	CU_ASSERT(mapping_length == VALUE_2MB * 3 - VALUE_4KB);
+
 	/* Clear translation for the middle page of the larger region. */
 	rc = spdk_mem_map_clear_translation(map, VALUE_2MB, VALUE_2MB);
 	CU_ASSERT(rc == 0);

test/unit/lib/blob/blob.c/blob_ut.c

@@ -109,7 +109,18 @@ _get_xattr_value_null(void *arg, const char *name,
 	*value = NULL;
 }
 
+static int
+_get_snapshots_count(struct spdk_blob_store *bs)
+{
+	struct spdk_blob_list *snapshot = NULL;
+	int count = 0;
 
+	TAILQ_FOREACH(snapshot, &bs->snapshots, link) {
+		count += 1;
+	}
+
+	return count;
+}
 
 static void
 bs_op_complete(void *cb_arg, int bserrno)
@@ -524,6 +535,7 @@ blob_snapshot(void)
 	struct spdk_blob_xattr_opts xattrs;
 	spdk_blob_id blobid;
 	spdk_blob_id snapshotid;
+	spdk_blob_id snapshotid2;
 	const void *value;
 	size_t value_len;
 	int rc;
@@ -551,9 +563,11 @@ blob_snapshot(void)
 	CU_ASSERT(spdk_blob_get_num_clusters(blob) == 10)
 
 	/* Create snapshot from blob */
+	CU_ASSERT_EQUAL(_get_snapshots_count(bs), 0);
 	spdk_bs_create_snapshot(bs, blobid, NULL, blob_op_with_id_complete, NULL);
 	CU_ASSERT(g_bserrno == 0);
 	CU_ASSERT(g_blobid != SPDK_BLOBID_INVALID);
+	CU_ASSERT_EQUAL(_get_snapshots_count(bs), 1);
 	snapshotid = g_blobid;
 
 	spdk_bs_open_blob(bs, snapshotid, blob_op_with_handle_complete, NULL);
@@ -577,9 +591,10 @@ blob_snapshot(void)
 	spdk_bs_create_snapshot(bs, blobid, &xattrs, blob_op_with_id_complete, NULL);
 	CU_ASSERT(g_bserrno == 0);
 	CU_ASSERT(g_blobid != SPDK_BLOBID_INVALID);
-	blobid = g_blobid;
+	CU_ASSERT_EQUAL(_get_snapshots_count(bs), 2);
+	snapshotid2 = g_blobid;
 
-	spdk_bs_open_blob(bs, blobid, blob_op_with_handle_complete, NULL);
+	spdk_bs_open_blob(bs, snapshotid2, blob_op_with_handle_complete, NULL);
 	CU_ASSERT(g_bserrno == 0);
 	SPDK_CU_ASSERT_FATAL(g_blob != NULL);
 	snapshot2 = g_blob;
@@ -620,16 +635,29 @@ blob_snapshot(void)
 	spdk_bs_create_snapshot(bs, snapshotid, NULL, blob_op_with_id_complete, NULL);
 	CU_ASSERT(g_bserrno == -EINVAL);
 	CU_ASSERT(g_blobid == SPDK_BLOBID_INVALID);
+	CU_ASSERT_EQUAL(_get_snapshots_count(bs), 2);
 
 	spdk_blob_close(blob, blob_op_complete, NULL);
 	CU_ASSERT(g_bserrno == 0);
 
-	spdk_blob_close(snapshot, blob_op_complete, NULL);
+	spdk_bs_delete_blob(bs, blobid, blob_op_complete, NULL);
 	CU_ASSERT(g_bserrno == 0);
+	CU_ASSERT_EQUAL(_get_snapshots_count(bs), 2);
 
 	spdk_blob_close(snapshot2, blob_op_complete, NULL);
 	CU_ASSERT(g_bserrno == 0);
 
+	spdk_bs_delete_blob(bs, snapshotid2, blob_op_complete, NULL);
+	CU_ASSERT(g_bserrno == 0);
+	CU_ASSERT_EQUAL(_get_snapshots_count(bs), 1);
+
+	spdk_blob_close(snapshot, blob_op_complete, NULL);
+	CU_ASSERT(g_bserrno == 0);
+
+	spdk_bs_delete_blob(bs, snapshotid, blob_op_complete, NULL);
+	CU_ASSERT(g_bserrno == 0);
+	CU_ASSERT_EQUAL(_get_snapshots_count(bs), 0);
+
 	spdk_bs_unload(g_bs, bs_op_complete, NULL);
 	CU_ASSERT(g_bserrno == 0);
 	g_bs = NULL;

test/unit/lib/iscsi/conn.c/conn_ut.c

@@ -279,6 +279,18 @@ spdk_iscsi_tgt_node_cleanup_luns(struct spdk_iscsi_conn *conn,
 	return 0;
 }
 
+uint32_t
+spdk_iscsi_pdu_calc_header_digest(struct spdk_iscsi_pdu *pdu)
+{
+	return 0;
+}
+
+uint32_t
+spdk_iscsi_pdu_calc_data_digest(struct spdk_iscsi_pdu *pdu)
+{
+	return 0;
+}
+
 void
 spdk_shutdown_iscsi_conns_done(void)
 {

test/unit/lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c

@@ -421,6 +421,11 @@ nvme_ns_construct(struct spdk_nvme_ns *ns, uint32_t id,
 	return 0;
 }
 
+void
+spdk_pci_device_detach(struct spdk_pci_device *device)
+{
+}
+
 #define DECLARE_AND_CONSTRUCT_CTRLR()	\
 	struct spdk_nvme_ctrlr	ctrlr = {};	\
 	struct spdk_nvme_qpair	adminq = {};	\

test/unit/lib/nvme/nvme_ns_ocssd_cmd.c/nvme_ns_ocssd_cmd_ut.c

@@ -39,6 +39,8 @@
 
 #include "common/lib/test_env.c"
 
+#define OCSSD_SECTOR_SIZE 0x1000
+
 DEFINE_STUB(spdk_nvme_qpair_process_completions, int32_t,
 	    (struct spdk_nvme_qpair *qpair,
 	     uint32_t max_completions), 0);
@@ -176,7 +178,7 @@ static void
 test_nvme_ocssd_ns_cmd_vector_reset_single_entry(void)
 {
 	const uint32_t	max_xfer_size = 0x10000;
-	const uint32_t	sector_size = 0x1000;
+	const uint32_t	sector_size = OCSSD_SECTOR_SIZE;
 
 	struct spdk_nvme_ns	ns;
 	struct spdk_nvme_ctrlr	ctrlr;
@@ -206,7 +208,7 @@ static void
 test_nvme_ocssd_ns_cmd_vector_reset(void)
 {
 	const uint32_t	max_xfer_size = 0x10000;
-	const uint32_t	sector_size = 0x1000;
+	const uint32_t	sector_size = OCSSD_SECTOR_SIZE;
 	const uint32_t	vector_size = 0x10;
 
 	struct spdk_nvme_ns	ns;
@@ -236,7 +238,7 @@ static void
 test_nvme_ocssd_ns_cmd_vector_read_with_md_single_entry(void)
 {
 	const uint32_t	max_xfer_size = 0x10000;
-	const uint32_t	sector_size = 0x1000;
+	const uint32_t	sector_size = OCSSD_SECTOR_SIZE;
 	const uint32_t	md_size = 0x80;
 
 	struct spdk_nvme_ns	ns;
@@ -261,7 +263,7 @@ test_nvme_ocssd_ns_cmd_vector_read_with_md_single_entry(void)
 	SPDK_CU_ASSERT_FATAL(g_request->num_children == 0);
 
 	CU_ASSERT(g_request->payload.md == metadata);
-	CU_ASSERT(g_request->payload_size == PAGE_SIZE);
+	CU_ASSERT(g_request->payload_size == OCSSD_SECTOR_SIZE);
 	CU_ASSERT(g_request->payload.contig_or_cb_arg == buffer);
 	CU_ASSERT(g_request->cmd.opc == SPDK_OCSSD_OPC_VECTOR_READ);
 	CU_ASSERT(g_request->cmd.nsid == ns.id);
@@ -279,7 +281,7 @@ static void
 test_nvme_ocssd_ns_cmd_vector_read_with_md(void)
 {
 	const uint32_t	max_xfer_size = 0x10000;
-	const uint32_t	sector_size = 0x1000;
+	const uint32_t	sector_size = OCSSD_SECTOR_SIZE;
 	const uint32_t	md_size = 0x80;
 	const uint32_t	vector_size = 0x10;
 
@@ -323,7 +325,7 @@ static void
 test_nvme_ocssd_ns_cmd_vector_read_single_entry(void)
 {
 	const uint32_t	max_xfer_size = 0x10000;
-	const uint32_t	sector_size = 0x1000;
+	const uint32_t	sector_size = OCSSD_SECTOR_SIZE;
 
 	struct spdk_nvme_ns	ns;
 	struct spdk_nvme_ctrlr	ctrlr;
@@ -344,7 +346,7 @@ test_nvme_ocssd_ns_cmd_vector_read_single_entry(void)
 	SPDK_CU_ASSERT_FATAL(g_request != NULL);
 	SPDK_CU_ASSERT_FATAL(g_request->num_children == 0);
 
-	CU_ASSERT(g_request->payload_size == PAGE_SIZE);
+	CU_ASSERT(g_request->payload_size == OCSSD_SECTOR_SIZE);
 	CU_ASSERT(g_request->payload.contig_or_cb_arg == buffer);
 	CU_ASSERT(g_request->cmd.opc == SPDK_OCSSD_OPC_VECTOR_READ);
 	CU_ASSERT(g_request->cmd.nsid == ns.id);
@@ -360,7 +362,7 @@ static void
 test_nvme_ocssd_ns_cmd_vector_read(void)
 {
 	const uint32_t	max_xfer_size = 0x10000;
-	const uint32_t	sector_size = 0x1000;
+	const uint32_t	sector_size = OCSSD_SECTOR_SIZE;
 	const uint32_t	vector_size = 0x10;
 
 	struct spdk_nvme_ns	ns;
@@ -397,7 +399,7 @@ static void
 test_nvme_ocssd_ns_cmd_vector_write_with_md_single_entry(void)
 {
 	const uint32_t	max_xfer_size = 0x10000;
-	const uint32_t	sector_size = 0x1000;
+	const uint32_t	sector_size = OCSSD_SECTOR_SIZE;
 	const uint32_t	md_size = 0x80;
 
 	struct spdk_nvme_ns	ns;
@@ -422,7 +424,7 @@ test_nvme_ocssd_ns_cmd_vector_write_with_md_single_entry(void)
 	SPDK_CU_ASSERT_FATAL(g_request->num_children == 0);
 
 	CU_ASSERT(g_request->payload.md == metadata);
-	CU_ASSERT(g_request->payload_size == PAGE_SIZE);
+	CU_ASSERT(g_request->payload_size == OCSSD_SECTOR_SIZE);
 	CU_ASSERT(g_request->payload.contig_or_cb_arg == buffer);
 	CU_ASSERT(g_request->cmd.opc == SPDK_OCSSD_OPC_VECTOR_WRITE);
 	CU_ASSERT(g_request->cmd.nsid == ns.id);
@@ -441,7 +443,7 @@ static void
 test_nvme_ocssd_ns_cmd_vector_write_with_md(void)
 {
 	const uint32_t	max_xfer_size = 0x10000;
-	const uint32_t	sector_size = 0x1000;
+	const uint32_t	sector_size = OCSSD_SECTOR_SIZE;
 	const uint32_t	md_size = 0x80;
 	const uint32_t	vector_size = 0x10;
 
@@ -485,7 +487,7 @@ static void
 test_nvme_ocssd_ns_cmd_vector_write_single_entry(void)
 {
 	const uint32_t	max_xfer_size = 0x10000;
-	const uint32_t	sector_size = 0x1000;
+	const uint32_t	sector_size = OCSSD_SECTOR_SIZE;
 
 	struct spdk_nvme_ns	ns;
 	struct spdk_nvme_ctrlr	ctrlr;
@@ -506,7 +508,7 @@ test_nvme_ocssd_ns_cmd_vector_write_single_entry(void)
 	SPDK_CU_ASSERT_FATAL(g_request != NULL);
 	SPDK_CU_ASSERT_FATAL(g_request->num_children == 0);
 
-	CU_ASSERT(g_request->payload_size == PAGE_SIZE);
+	CU_ASSERT(g_request->payload_size == OCSSD_SECTOR_SIZE);
 	CU_ASSERT(g_request->payload.contig_or_cb_arg == buffer);
 	CU_ASSERT(g_request->cmd.opc == SPDK_OCSSD_OPC_VECTOR_WRITE);
 	CU_ASSERT(g_request->cmd.nsid == ns.id);
@@ -523,7 +525,7 @@ static void
 test_nvme_ocssd_ns_cmd_vector_write(void)
 {
 	const uint32_t	max_xfer_size = 0x10000;
-	const uint32_t	sector_size = 0x1000;
+	const uint32_t	sector_size = OCSSD_SECTOR_SIZE;
 	const uint32_t	vector_size = 0x10;
 
 	struct spdk_nvme_ns	ns;
@@ -562,7 +564,7 @@ static void
 test_nvme_ocssd_ns_cmd_vector_copy_single_entry(void)
 {
 	const uint32_t	max_xfer_size = 0x10000;
-	const uint32_t	sector_size = 0x1000;
+	const uint32_t	sector_size = OCSSD_SECTOR_SIZE;
 
 	struct spdk_nvme_ns	ns;
 	struct spdk_nvme_ctrlr	ctrlr;
@@ -594,7 +596,7 @@ static void
 test_nvme_ocssd_ns_cmd_vector_copy(void)
 {
 	const uint32_t	max_xfer_size = 0x10000;
-	const uint32_t	sector_size = 0x1000;
+	const uint32_t	sector_size = OCSSD_SECTOR_SIZE;
 	const uint32_t	vector_size = 0x10;
 
 	struct spdk_nvme_ns	ns;