SPDK 19.07.1

Change-Id: I7149e5d81c92712357b3c07b732a59059e64fce9
Signed-off-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/469681
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## v19.07.1:
+
+### build
+
+Improved inter-lib dependencies to allow linking against only a subset of SPDK
+libraries when building SPDK applications.
+
 ## v19.07:
 
 ### ftl

Makefile

@@ -37,6 +37,7 @@ SPDK_ROOT_DIR := $(CURDIR)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
 DIRS-y += lib
+DIRS-y += module
 DIRS-$(CONFIG_SHARED) += shared_lib
 DIRS-y += examples app include
 DIRS-$(CONFIG_TESTS) += test
@@ -59,7 +60,7 @@ endif
 ifeq ($(CONFIG_SHARED),y)
 LIB = shared_lib
 else
-LIB = lib
+LIB = module
 endif
 
 ifeq ($(CONFIG_IPSEC_MB),y)
@@ -86,8 +87,9 @@ ifneq ($(SKIP_DPDK_BUILD),1)
 dpdkbuild: $(DPDK_DEPS)
 endif
 
-shared_lib: lib
 lib: $(DPDKBUILD)
+module: lib
+shared_lib: module
 app: $(LIB)
 test: $(LIB)
 examples: $(LIB)

app/iscsi_tgt/Makefile

@@ -46,7 +46,7 @@ C_SRCS := iscsi_tgt.c
 SPDK_LIB_LIST = $(ALL_MODULES_LIST)
 SPDK_LIB_LIST += event_bdev event_copy event_iscsi event_net event_scsi event_vmd event
 SPDK_LIB_LIST += jsonrpc json rpc bdev_rpc bdev iscsi scsi copy trace conf
-SPDK_LIB_LIST += thread util log log_rpc trace_rpc app_rpc net sock notify
+SPDK_LIB_LIST += thread util log log_rpc app_rpc net sock notify
 
 ifeq ($(OS),Linux)
 SPDK_LIB_LIST += event_nbd nbd

app/nvmf_tgt/Makefile

@@ -42,7 +42,7 @@ C_SRCS := nvmf_main.c
 SPDK_LIB_LIST = $(ALL_MODULES_LIST)
 SPDK_LIB_LIST += event_bdev event_copy event_nvmf event_net event_vmd
 SPDK_LIB_LIST += nvmf event log trace conf thread util bdev copy rpc jsonrpc json net sock
-SPDK_LIB_LIST += app_rpc log_rpc trace_rpc bdev_rpc notify
+SPDK_LIB_LIST += app_rpc log_rpc bdev_rpc notify
 
 ifeq ($(OS),Linux)
 SPDK_LIB_LIST += event_nbd nbd

app/spdk_tgt/Makefile

@@ -52,7 +52,7 @@ endif
 
 SPDK_LIB_LIST += event_bdev event_copy event_iscsi event_net event_scsi event_nvmf event_vmd event
 SPDK_LIB_LIST += nvmf trace log conf thread util bdev iscsi scsi copy rpc jsonrpc json
-SPDK_LIB_LIST += app_rpc log_rpc trace_rpc bdev_rpc net sock notify
+SPDK_LIB_LIST += app_rpc log_rpc bdev_rpc net sock notify
 
 ifeq ($(OS),Linux)
 SPDK_LIB_LIST += event_nbd nbd

app/vhost/Makefile

@@ -48,7 +48,7 @@ endif
 
 SPDK_LIB_LIST += event_bdev event_copy event_net event_scsi event_vmd event
 SPDK_LIB_LIST += jsonrpc json rpc bdev_rpc bdev scsi copy trace conf
-SPDK_LIB_LIST += thread util log log_rpc trace_rpc app_rpc
+SPDK_LIB_LIST += thread util log log_rpc app_rpc
 SPDK_LIB_LIST += event_nbd nbd net sock notify
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

autobuild.sh

@@ -28,7 +28,7 @@ if [ "$SPDK_TEST_OCF" -eq 1 ]; then
 	# So we precompile OCF now for further use as standalone static library
 	./configure $(echo $config_params | sed 's/--enable-coverage//g')
 	$MAKE $MAKEFLAGS include/spdk/config.h
-	CC=gcc CCAR=ar $MAKE $MAKEFLAGS -C lib/bdev/ocf/env exportlib O=$rootdir/build/ocf.a
+	CC=gcc CCAR=ar $MAKE $MAKEFLAGS -C lib/env_ocf exportlib O=$rootdir/build/ocf.a
 	# Set config to use precompiled library
 	config_params="$config_params --with-ocf=/$rootdir/build/ocf.a"
 fi
@@ -77,9 +77,7 @@ timing_enter "$make_timing_label"
 
 $MAKE $MAKEFLAGS clean
 if [ $SPDK_BUILD_SHARED_OBJECT -eq 1 ]; then
-	./configure $config_params --with-shared
-	$MAKE $MAKEFLAGS
-	$MAKE $MAKEFLAGS clean
+	$rootdir/test/make/check_so_deps.sh
 	report_test_completion "shared_object_build"
 fi
 

configure

@@ -369,7 +369,7 @@ scripts/detect_cc.sh --cc="$CC" --cxx="$CXX" --lto="${CONFIG[LTO]}" --ld="$LD" -
 CC=$(cat mk/cc.mk | grep "CC=" | cut -d "=" -f 2)
 CC_TYPE=$(cat mk/cc.mk | grep "CC_TYPE=" | cut -d "=" -f 2)
 
-BUILD_CMD="$CC -o /dev/null -x c $CPPFLAGS $CFLAGS $LDFLAGS"
+BUILD_CMD=($CC -o /dev/null -x c $CPPFLAGS $CFLAGS $LDFLAGS)
 
 # Detect architecture and force no ISA-L if non-x86 archtecture
 if [[ "${CONFIG[ISAL]}" = "y" ]]; then
@@ -417,7 +417,7 @@ if [ -z "${CONFIG[ENV]}" ]; then
 			# program, just compile it
 			if ! echo -e '#include <rte_vhost.h>\n' \
 				'int main(void) { return rte_vhost_extern_callback_register(0, NULL, NULL); }\n' \
-				| $BUILD_CMD -c -Wno-deprecated-declarations -Werror \
+				| ${BUILD_CMD[@]} -c -Wno-deprecated-declarations -Werror \
 				-I"${CONFIG[DPDK_DIR]}/include" - &>/dev/null; then
 				echo "Notice: DPDK's rte_vhost not found or version < 19.05, using internal," \
 				     "legacy rte_vhost library."
@@ -477,6 +477,10 @@ if [[ "$OSTYPE" == "freebsd"* ]]; then
 		echo "Vhost is only supported on Linux. Disabling it."
 		CONFIG[VHOST]="n"
 	fi
+	if [[ "${CONFIG[VHOST_INTERNAL_LIB]}" == "y" ]]; then
+		echo "Internal rte_vhost library is only supported on Linux. Disabling it."
+		CONFIG[VHOST_INTERNAL_LIB]="n"
+	fi
 	if [[ "${CONFIG[VIRTIO]}" == "y" ]]; then
 		echo "Virtio is only supported on Linux. Disabling it."
 		CONFIG[VIRTIO]="n"
@@ -486,7 +490,7 @@ fi
 if [ "${CONFIG[RDMA]}" = "y" ]; then
 	if ! echo -e '#include <infiniband/verbs.h>\n#include <rdma/rdma_verbs.h>\n' \
 		'int main(void) { return 0; }\n' \
-		| $BUILD_CMD -libverbs -lrdmacm - 2>/dev/null; then
+		| ${BUILD_CMD[@]} -libverbs -lrdmacm - 2>/dev/null; then
 		echo --with-rdma requires libverbs and librdmacm.
 		echo Please install then re-run this script.
 		exit 1
@@ -494,7 +498,7 @@ if [ "${CONFIG[RDMA]}" = "y" ]; then
 
 	if echo -e '#include <infiniband/verbs.h>\n' \
 		'int main(void) { return !!IBV_WR_SEND_WITH_INV; }\n' \
-		| $BUILD_CMD -c - 2>/dev/null; then
+		| ${BUILD_CMD[@]} -c - 2>/dev/null; then
 		CONFIG[RDMA_SEND_WITH_INVAL]="y"
 	else
 		CONFIG[RDMA_SEND_WITH_INVAL]="n"
@@ -565,7 +569,7 @@ fi
 
 if [[ "${CONFIG[PMDK]}" = "y" ]]; then
 	if ! echo -e '#include <libpmemblk.h>\nint main(void) { return 0; }\n' \
-		| $BUILD_CMD -lpmemblk - 2>/dev/null; then
+		| ${BUILD_CMD[@]} -lpmemblk - 2>/dev/null; then
 		echo --with-pmdk requires libpmemblk.
 		echo Please install then re-run this script.
 		exit 1
@@ -574,7 +578,7 @@ fi
 
 if [[ "${CONFIG[REDUCE]}" = "y" ]]; then
 	if ! echo -e '#include <libpmem.h>\nint main(void) { return 0; }\n' \
-		| $BUILD_CMD -lpmem - 2>/dev/null; then
+		| ${BUILD_CMD[@]} -lpmem - 2>/dev/null; then
 		echo --with-reduce requires libpmem.
 		echo Please install then re-run this script.
 		exit 1
@@ -586,7 +590,7 @@ if [[ "${CONFIG[VPP]}" = "y" ]]; then
 		VPP_CFLAGS="-L${CONFIG[VPP_DIR]}/lib -I${CONFIG[VPP_DIR]}/include"
 	fi
 	if ! echo -e '#include <vnet/session/application_interface.h>\nint main(void) { return 0; }\n' \
-		| $BUILD_CMD ${VPP_CFLAGS} -lvppinfra -lsvm -lvlibmemoryclient - 2>/dev/null; then
+		| ${BUILD_CMD[@]} ${VPP_CFLAGS} -lvppinfra -lsvm -lvlibmemoryclient - 2>/dev/null; then
 		echo --with-vpp requires installed vpp.
 		echo Please install then re-run this script.
 		exit 1
@@ -596,7 +600,7 @@ fi
 if [[ "${CONFIG[RBD]}" = "y" ]]; then
 	if ! echo -e '#include <rbd/librbd.h>\n#include <rados/librados.h>\n' \
 		'int main(void) { return 0; }\n' \
-		| $BUILD_CMD -lrados -lrbd - 2>/dev/null; then
+		| ${BUILD_CMD[@]} -lrados -lrbd - 2>/dev/null; then
 		echo --with-rbd requires librados and librbd.
 		echo Please install then re-run this script.
 		exit 1
@@ -610,7 +614,7 @@ if [[ "${CONFIG[ISCSI_INITIATOR]}" = "y" ]]; then
 		'#error\n' \
 		'#endif\n' \
 		'int main(void) { return 0; }\n' \
-		| $BUILD_CMD -L/usr/lib64/iscsi -liscsi - 2>/dev/null; then
+		| ${BUILD_CMD[@]} -L/usr/lib64/iscsi -liscsi - 2>/dev/null; then
 		echo --with-iscsi-initiator requires libiscsi with
 		echo 'LIBISCSI_API_VERSION >= 20150621.'
 		echo Please install then re-run this script.
@@ -620,7 +624,7 @@ fi
 
 if [[ "${CONFIG[LOG_BACKTRACE]}" = "y" ]]; then
 	if ! echo -e '#include <libunwind.h>\nint main(void) { return 0; }\n' \
-		| $BUILD_CMD -lunwind - 2>/dev/null; then
+		| ${BUILD_CMD[@]} -lunwind - 2>/dev/null; then
 		echo --enable-log-bt requires libunwind.
 		echo Please install then re-run this script.
 		exit 1
@@ -629,7 +633,7 @@ fi
 
 if [[ "${CONFIG[ASAN]}" = "y" ]]; then
 	if ! echo -e 'int main(void) { return 0; }\n' \
-		| $BUILD_CMD -fsanitize=address - 2>/dev/null; then
+		| ${BUILD_CMD[@]} -fsanitize=address - 2>/dev/null; then
 		echo --enable-asan requires libasan.
 		echo Please install then re-run this script.
 		exit 1
@@ -638,7 +642,7 @@ fi
 
 if [[ "${CONFIG[UBSAN]}" = "y" ]]; then
 	if ! echo -e 'int main(void) { return 0; }\n' \
-		| $BUILD_CMD -fsanitize=undefined - 2>/dev/null; then
+		| ${BUILD_CMD[@]} -fsanitize=undefined - 2>/dev/null; then
 		echo --enable-ubsan requires libubsan.
 		echo Please install then re-run this script.
 		exit 1
@@ -647,7 +651,7 @@ fi
 
 if [[ "${CONFIG[TSAN]}" = "y" ]]; then
 	if ! echo -e 'int main(void) { return 0; }\n' \
-		| $BUILD_CMD -fsanitize=thread - 2>/dev/null; then
+		| ${BUILD_CMD[@]} -fsanitize=thread - 2>/dev/null; then
 		echo --enable-tsan requires libtsan.
 		echo Please install then re-run this script.
 		exit 1

include/spdk/nvme_spec.h

@@ -654,7 +654,7 @@ enum spdk_nvme_command_specific_status_code {
 
 	SPDK_NVME_SC_CONFLICTING_ATTRIBUTES		= 0x80,
 	SPDK_NVME_SC_INVALID_PROTECTION_INFO		= 0x81,
-	SPDK_NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE		= 0x82,
+	SPDK_NVME_SC_ATTEMPTED_WRITE_TO_RO_RANGE	= 0x82,
 };
 
 /**

include/spdk/rpc.h

@@ -103,14 +103,20 @@ int spdk_rpc_is_method_allowed(const char *method, uint32_t state_mask);
 #define SPDK_RPC_STARTUP	0x1
 #define SPDK_RPC_RUNTIME	0x2
 
+/* Give SPDK_RPC_REGISTER a higher execution priority than
+ * SPDK_RPC_REGISTER_ALIAS_DEPRECATED to ensure all of the RPCs are registered
+ * before we try registering any aliases.  Some older versions of clang may
+ * otherwise execute the constructors in a different order than
+ * defined in the source file (see issue #892).
+ */
 #define SPDK_RPC_REGISTER(method, func, state_mask) \
-static void __attribute__((constructor)) rpc_register_##func(void) \
+static void __attribute__((constructor(1000))) rpc_register_##func(void) \
 { \
 	spdk_rpc_register_method(method, func, state_mask); \
 }
 
 #define SPDK_RPC_REGISTER_ALIAS_DEPRECATED(method, alias) \
-static void __attribute__((constructor)) rpc_register_##alias(void) \
+static void __attribute__((constructor(1001))) rpc_register_##alias(void) \
 { \
 	spdk_rpc_register_alias_deprecated(#method, #alias); \
 }

include/spdk/version.h

@@ -54,7 +54,7 @@
  * Patch level is incremented on maintenance branch releases and reset to 0 for each
  * new major.minor release.
  */
-#define SPDK_VERSION_PATCH	0
+#define SPDK_VERSION_PATCH	1
 
 /**
  * Version string suffix.

lib/Makefile

@@ -33,17 +33,23 @@
 
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+include $(SPDK_ROOT_DIR)/mk/spdk.lib_deps.mk
 
 DIRS-y += bdev blob blobfs conf copy event json jsonrpc \
-          log lvol net rpc sock thread trace util nvme vmd nvmf scsi ioat \
-          ut_mock iscsi notify
+          log log_rpc lvol net rpc sock thread trace util nvme vmd nvmf scsi \
+          ioat ut_mock iscsi notify
 ifeq ($(OS),Linux)
 DIRS-y += nbd ftl
 endif
 
+ifeq ($(CONFIG_OCF), y)
+DIRS-y += env_ocf
+endif
+
 DIRS-$(CONFIG_VHOST) += vhost
 DIRS-$(CONFIG_VIRTIO) += virtio
 DIRS-$(CONFIG_REDUCE) += reduce
+DIRS-$(CONFIG_VHOST_INTERNAL_LIB) += rte_vhost
 
 # If CONFIG_ENV is pointing at a directory in lib, build it.
 # Out-of-tree env implementations must be built separately by the user.
@@ -52,9 +58,6 @@ ifeq ($(abspath $(CONFIG_ENV)),$(SPDK_ROOT_DIR)/lib/$(ENV_NAME))
 DIRS-y += $(ENV_NAME)
 endif
 
-DEPDIRS-iscsi := scsi
-DEPDIRS-bdev := notify
-
 .PHONY: all clean $(DIRS-y)
 
 all: $(DIRS-y)

lib/bdev/Makefile

@@ -38,37 +38,8 @@ ifeq ($(CONFIG_VTUNE),y)
 CFLAGS += -I$(CONFIG_VTUNE_DIR)/include -I$(CONFIG_VTUNE_DIR)/sdk/src/ittnotify
 endif
 
-C_SRCS = bdev.c part.c scsi_nvme.c
+C_SRCS = bdev.c bdev_rpc.c part.c scsi_nvme.c
 C_SRCS-$(CONFIG_VTUNE) += vtune.c
 LIBNAME = bdev
 
-DIRS-y += delay error gpt lvol malloc null nvme passthru raid rpc split
-
-ifeq ($(CONFIG_CRYPTO),y)
-DIRS-y += crypto
-endif
-
-ifeq ($(CONFIG_OCF), y)
-DIRS-y += ocf
-DIRS-y += ocf/env
-DEPDIRS-ocf := ocf/env
-endif
-
-ifeq ($(CONFIG_REDUCE),y)
-DIRS-y += compress
-endif
-
-ifeq ($(CONFIG_URING),y)
-DIRS-y += uring
-endif
-
-ifeq ($(OS),Linux)
-DIRS-y += aio
-DIRS-$(CONFIG_ISCSI_INITIATOR) += iscsi
-DIRS-$(CONFIG_VIRTIO) += virtio
-DIRS-$(CONFIG_PMDK) += pmem
-endif
-
-DIRS-$(CONFIG_RBD) += rbd
-
 include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

lib/bdev/bdev.c

@@ -4308,6 +4308,12 @@ spdk_bdev_open(struct spdk_bdev *bdev, bool write, spdk_bdev_remove_cb_t remove_
 	*_desc = desc;
 
 	pthread_mutex_lock(&bdev->internal.mutex);
+	if (bdev->internal.status == SPDK_BDEV_STATUS_REMOVING) {
+		pthread_mutex_unlock(&bdev->internal.mutex);
+		free(desc);
+		*_desc = NULL;
+		return -ENODEV;
+	}
 
 	if (write && bdev->internal.claim_module) {
 		SPDK_ERRLOG("Could not open %s - %s module already claimed it\n",
@@ -4773,8 +4779,7 @@ spdk_bdev_set_qos_rate_limits(struct spdk_bdev *bdev, uint64_t *limits,
 			if (!bdev->internal.qos) {
 				pthread_mutex_unlock(&bdev->internal.mutex);
 				SPDK_ERRLOG("Unable to allocate memory for QoS tracking\n");
-				free(ctx);
-				cb_fn(cb_arg, -ENOMEM);
+				_spdk_bdev_set_qos_limit_done(ctx, -ENOMEM);
 				return;
 			}
 		}

lib/bdev/scsi_nvme.c

@@ -158,7 +158,7 @@ spdk_scsi_nvme_translate(const struct spdk_bdev_io *bdev_io, int *sc, int *sk,
 			*asc  = SPDK_SCSI_ASC_INVALID_FIELD_IN_CDB;
 			*ascq = SPDK_SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
 			break;
-		case SPDK_NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE:
+		case SPDK_NVME_SC_ATTEMPTED_WRITE_TO_RO_RANGE:
 			*sc   = SPDK_SCSI_STATUS_CHECK_CONDITION;
 			*sk   = SPDK_SCSI_SENSE_DATA_PROTECT;
 			*asc  = SPDK_SCSI_ASC_WRITE_PROTECTED;

lib/blob/Makefile

@@ -37,6 +37,4 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 C_SRCS = blobstore.c request.c zeroes.c blob_bs_dev.c
 LIBNAME = blob
 
-DIRS-y += bdev
-
 include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

lib/copy/Makefile

@@ -37,6 +37,4 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 LIBNAME = copy
 C_SRCS = copy_engine.c
 
-DIRS-y = ioat
-
 include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

lib/env_dpdk/env.mk

@@ -116,7 +116,7 @@ ENV_CFLAGS = $(DPDK_INC) -Wno-deprecated-declarations
 ENV_CXXFLAGS = $(ENV_CFLAGS)
 ENV_DPDK_FILE = $(call spdk_lib_list_to_static_libs,env_dpdk)
 ENV_LIBS = $(ENV_DPDK_FILE) $(DPDK_LIB)
-ENV_LINKER_ARGS = $(ENV_DPDK_FILE) -Wl,--whole-archive $(DPDK_LIB) -Wl,--no-whole-archive
+ENV_LINKER_ARGS = $(ENV_DPDK_FILE) -Wl,--whole-archive,--no-as-needed $(DPDK_LIB) -Wl,--no-whole-archive
 
 ifeq ($(CONFIG_IPSEC_MB),y)
 ENV_LINKER_ARGS += -lIPSec_MB -L$(IPSEC_MB_DIR)

lib/env_dpdk/memory.c

@@ -1005,87 +1005,151 @@ spdk_vtophys_notify(void *cb_ctx, struct spdk_mem_map *map,
 		return -EINVAL;
 	}
 
-	while (len > 0) {
-		/* Get the physical address from the DPDK memsegs */
-		paddr = vtophys_get_paddr_memseg((uint64_t)vaddr);
+	/* Get the physical address from the DPDK memsegs */
+	paddr = vtophys_get_paddr_memseg((uint64_t)vaddr);
 
-		switch (action) {
-		case SPDK_MEM_MAP_NOTIFY_REGISTER:
-			if (paddr == SPDK_VTOPHYS_ERROR) {
-				/* This is not an address that DPDK is managing. */
+	switch (action) {
+	case SPDK_MEM_MAP_NOTIFY_REGISTER:
+		if (paddr == SPDK_VTOPHYS_ERROR) {
+			/* This is not an address that DPDK is managing. */
 #if SPDK_VFIO_ENABLED
-				if (spdk_iommu_is_enabled()) {
-					/* We'll use the virtual address as the iova. DPDK
-					 * currently uses physical addresses as the iovas (or counts
-					 * up from 0 if it can't get physical addresses), so
-					 * the range of user space virtual addresses and physical
-					 * addresses will never overlap.
-					 */
-					paddr = (uint64_t)vaddr;
-					rc = vtophys_iommu_map_dma((uint64_t)vaddr, paddr, VALUE_2MB);
-					if (rc) {
+			if (spdk_iommu_is_enabled()) {
+				/* We'll use the virtual address as the iova. DPDK
+				 * currently uses physical addresses as the iovas (or counts
+				 * up from 0 if it can't get physical addresses), so
+				 * the range of user space virtual addresses and physical
+				 * addresses will never overlap.
+				 */
+				paddr = (uint64_t)vaddr;
+				rc = vtophys_iommu_map_dma((uint64_t)vaddr, paddr, len);
+				if (rc) {
+					return -EFAULT;
+				}
+				while (len > 0) {
+					rc = spdk_mem_map_set_translation(map, (uint64_t)vaddr, VALUE_2MB, paddr);
+					if (rc != 0) {
+						return rc;
+					}
+					vaddr += VALUE_2MB;
+					paddr += VALUE_2MB;
+					len -= VALUE_2MB;
+				}
+			} else
+#endif
+			{
+				/* Get the physical address from /proc/self/pagemap. */
+				paddr = vtophys_get_paddr_pagemap((uint64_t)vaddr);
+				if (paddr == SPDK_VTOPHYS_ERROR) {
+					/* Get the physical address from PCI devices */
+					paddr = vtophys_get_paddr_pci((uint64_t)vaddr);
+					if (paddr == SPDK_VTOPHYS_ERROR) {
+						DEBUG_PRINT("could not get phys addr for %p\n", vaddr);
 						return -EFAULT;
 					}
-				} else
-#endif
-				{
-					/* Get the physical address from /proc/self/pagemap. */
-					paddr = vtophys_get_paddr_pagemap((uint64_t)vaddr);
-					if (paddr == SPDK_VTOPHYS_ERROR) {
-						/* Get the physical address from PCI devices */
-						paddr = vtophys_get_paddr_pci((uint64_t)vaddr);
-						if (paddr == SPDK_VTOPHYS_ERROR) {
-							DEBUG_PRINT("could not get phys addr for %p\n", vaddr);
-							return -EFAULT;
-						}
-						pci_phys = 1;
-					}
+					/* The beginning of this address range points to a PCI resource,
+					 * so the rest must point to a PCI resource as well.
+					 */
+					pci_phys = 1;
 				}
-			}
-			/* Since PCI paddr can break the 2MiB physical alignment skip this check for that. */
-			if (!pci_phys && (paddr & MASK_2MB)) {
-				DEBUG_PRINT("invalid paddr 0x%" PRIx64 " - must be 2MB aligned\n", paddr);
-				return -EINVAL;
-			}
 
-			rc = spdk_mem_map_set_translation(map, (uint64_t)vaddr, VALUE_2MB, paddr);
-			break;
-		case SPDK_MEM_MAP_NOTIFY_UNREGISTER:
-#if SPDK_VFIO_ENABLED
-			if (paddr == SPDK_VTOPHYS_ERROR) {
-				/*
-				 * This is not an address that DPDK is managing. If vfio is enabled,
-				 * we need to unmap the range from the IOMMU
-				 */
-				if (spdk_iommu_is_enabled()) {
-					uint64_t buffer_len = VALUE_2MB;
-					paddr = spdk_mem_map_translate(map, (uint64_t)vaddr, &buffer_len);
-					if (buffer_len != VALUE_2MB) {
+				/* Get paddr for each 2MB chunk in this address range */
+				while (len > 0) {
+					/* Get the physical address from /proc/self/pagemap. */
+					if (pci_phys) {
+						paddr = vtophys_get_paddr_pci((uint64_t)vaddr);
+					} else {
+						paddr = vtophys_get_paddr_pagemap((uint64_t)vaddr);
+					}
+
+					if (paddr == SPDK_VTOPHYS_ERROR) {
+						DEBUG_PRINT("could not get phys addr for %p\n", vaddr);
+						return -EFAULT;
+					}
+
+					/* Since PCI paddr can break the 2MiB physical alignment skip this check for that. */
+					if (!pci_phys && (paddr & MASK_2MB)) {
+						DEBUG_PRINT("invalid paddr 0x%" PRIx64 " - must be 2MB aligned\n", paddr);
 						return -EINVAL;
 					}
-					rc = vtophys_iommu_unmap_dma(paddr, VALUE_2MB);
-					if (rc) {
-						return -EFAULT;
+
+					rc = spdk_mem_map_set_translation(map, (uint64_t)vaddr, VALUE_2MB, paddr);
+					if (rc != 0) {
+						return rc;
 					}
+
+					vaddr += VALUE_2MB;
+					len -= VALUE_2MB;
 				}
 			}
-#endif
-			rc = spdk_mem_map_clear_translation(map, (uint64_t)vaddr, VALUE_2MB);
-			break;
-		default:
-			SPDK_UNREACHABLE();
+		} else {
+			/* This is an address managed by DPDK. Just setup the translations. */
+			while (len > 0) {
+				paddr = vtophys_get_paddr_memseg((uint64_t)vaddr);
+				if (paddr == SPDK_VTOPHYS_ERROR) {
+					DEBUG_PRINT("could not get phys addr for %p\n", vaddr);
+					return -EFAULT;
+				}
+
+				rc = spdk_mem_map_set_translation(map, (uint64_t)vaddr, VALUE_2MB, paddr);
+				if (rc != 0) {
+					return rc;
+				}
+
+				vaddr += VALUE_2MB;
+				len -= VALUE_2MB;
+			}
 		}
 
-		if (rc != 0) {
-			return rc;
+		break;
+	case SPDK_MEM_MAP_NOTIFY_UNREGISTER:
+#if SPDK_VFIO_ENABLED
+		if (paddr == SPDK_VTOPHYS_ERROR) {
+			/*
+			 * This is not an address that DPDK is managing. If vfio is enabled,
+			 * we need to unmap the range from the IOMMU
+			 */
+			if (spdk_iommu_is_enabled()) {
+				uint64_t buffer_len = len;
+				paddr = spdk_mem_map_translate(map, (uint64_t)vaddr, &buffer_len);
+				if (buffer_len != len) {
+					return -EINVAL;
+				}
+				rc = vtophys_iommu_unmap_dma(paddr, len);
+				if (rc) {
+					return -EFAULT;
+				}
+			}
 		}
-		vaddr += VALUE_2MB;
-		len -= VALUE_2MB;
+#endif
+		while (len > 0) {
+			rc = spdk_mem_map_clear_translation(map, (uint64_t)vaddr, VALUE_2MB);
+			if (rc != 0) {
+				return rc;
+			}
+
+			vaddr += VALUE_2MB;
+			len -= VALUE_2MB;
+		}
+
+		break;
+	default:
+		SPDK_UNREACHABLE();
 	}
 
 	return rc;
 }
 
+static int
+vtophys_check_contiguous_entries(uint64_t paddr1, uint64_t paddr2)
+{
+	/* This function is always called with paddrs for two subsequent
+	 * 2MB chunks in virtual address space, so those chunks will be only
+	 * physically contiguous if the physical addresses are 2MB apart
+	 * from each other as well.
+	 */
+	return (paddr2 - paddr1 == VALUE_2MB);
+}
+
 #if SPDK_VFIO_ENABLED
 
 static bool
@@ -1274,7 +1338,7 @@ spdk_vtophys_init(void)
 {
 	const struct spdk_mem_map_ops vtophys_map_ops = {
 		.notify_cb = spdk_vtophys_notify,
-		.are_contiguous = NULL
+		.are_contiguous = vtophys_check_contiguous_entries,
 	};
 
 #if SPDK_VFIO_ENABLED

lib/env_ocf/Makefile

@@ -37,7 +37,7 @@
 # Else if SPDK is configured with OCF precompiled library
 # we just use it as SPDK lib by copying it to /build/lib/
 
-SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../..)
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 OCFDIR=$(CONFIG_OCF_DIR)
 
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
@@ -76,17 +76,17 @@ all: ocf_inc ocf_src
 	$(Q)$(MAKE) $(LIB)
 
 ocf_inc:
-	$(Q)$(MAKE) -C "$(CONFIG_OCF_PATH)" inc O="$(SPDK_ROOT_DIR)/lib/bdev/ocf/env/" --quiet
+	$(Q)$(MAKE) -C "$(CONFIG_OCF_PATH)" inc O="$(SPDK_ROOT_DIR)/lib/env_ocf/" ENV= --quiet
 
 ocf_src: ocf_inc
-	$(Q)$(MAKE) -C "$(CONFIG_OCF_PATH)" src O="$(SPDK_ROOT_DIR)/lib/bdev/ocf/env/" CMD=cp --quiet
+	$(Q)$(MAKE) -C "$(CONFIG_OCF_PATH)" src O="$(SPDK_ROOT_DIR)/lib/env_ocf/" CMD=cp ENV= --quiet
 
 ocf_distclean:
-	$(Q)$(MAKE) -C "$(CONFIG_OCF_PATH)" distclean O="$(SPDK_ROOT_DIR)/lib/bdev/ocf/env/" --quiet
+	$(Q)$(MAKE) -C "$(CONFIG_OCF_PATH)" distclean O="$(SPDK_ROOT_DIR)/lib/env_ocf/" ENV= --quiet
 
 clean: ocf_distclean
-	$(Q)rm -rf	"$(SPDK_ROOT_DIR)/lib/bdev/ocf/env/include" \
-				"$(SPDK_ROOT_DIR)/lib/bdev/ocf/env/src" \
+	$(Q)rm -rf	"$(SPDK_ROOT_DIR)/lib/env_ocf/include" \
+				"$(SPDK_ROOT_DIR)/lib/env_ocf/src" \
 				$(LIB) $(OBJS);
 
 $(LIB): $(OBJS)

lib/event/Makefile

@@ -37,6 +37,4 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 LIBNAME = event
 C_SRCS = app.c reactor.c rpc.c subsystem.c json_config.c
 
-DIRS-y = rpc subsystems
-
 include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

lib/event/subsystems/iscsi/iscsi_rpc.c

@@ -1,115 +0,0 @@
-/*-
- *   BSD LICENSE
- *
- *   Copyright (c) Intel Corporation.
- *   All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of Intel Corporation nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "iscsi/iscsi.h"
-#include "iscsi/conn.h"
-
-#include "spdk/rpc.h"
-#include "spdk/util.h"
-#include "spdk/event.h"
-
-#include "spdk_internal/log.h"
-
-static const struct spdk_json_object_decoder rpc_set_iscsi_opts_decoders[] = {
-	{"auth_file", offsetof(struct spdk_iscsi_opts, authfile), spdk_json_decode_string, true},
-	{"node_base", offsetof(struct spdk_iscsi_opts, nodebase), spdk_json_decode_string, true},
-	{"nop_timeout", offsetof(struct spdk_iscsi_opts, timeout), spdk_json_decode_int32, true},
-	{"nop_in_interval", offsetof(struct spdk_iscsi_opts, nopininterval), spdk_json_decode_int32, true},
-	{"no_discovery_auth", offsetof(struct spdk_iscsi_opts, disable_chap), spdk_json_decode_bool, true},
-	{"req_discovery_auth", offsetof(struct spdk_iscsi_opts, require_chap), spdk_json_decode_bool, true},
-	{"req_discovery_auth_mutual", offsetof(struct spdk_iscsi_opts, mutual_chap), spdk_json_decode_bool, true},
-	{"discovery_auth_group", offsetof(struct spdk_iscsi_opts, chap_group), spdk_json_decode_int32, true},
-	{"disable_chap", offsetof(struct spdk_iscsi_opts, disable_chap), spdk_json_decode_bool, true},
-	{"require_chap", offsetof(struct spdk_iscsi_opts, require_chap), spdk_json_decode_bool, true},
-	{"mutual_chap", offsetof(struct spdk_iscsi_opts, mutual_chap), spdk_json_decode_bool, true},
-	{"chap_group", offsetof(struct spdk_iscsi_opts, chap_group), spdk_json_decode_int32, true},
-	{"max_sessions", offsetof(struct spdk_iscsi_opts, MaxSessions), spdk_json_decode_uint32, true},
-	{"max_queue_depth", offsetof(struct spdk_iscsi_opts, MaxQueueDepth), spdk_json_decode_uint32, true},
-	{"max_connections_per_session", offsetof(struct spdk_iscsi_opts, MaxConnectionsPerSession), spdk_json_decode_uint32, true},
-	{"default_time2wait", offsetof(struct spdk_iscsi_opts, DefaultTime2Wait), spdk_json_decode_uint32, true},
-	{"default_time2retain", offsetof(struct spdk_iscsi_opts, DefaultTime2Retain), spdk_json_decode_uint32, true},
-	{"first_burst_length", offsetof(struct spdk_iscsi_opts, FirstBurstLength), spdk_json_decode_uint32, true},
-	{"immediate_data", offsetof(struct spdk_iscsi_opts, ImmediateData), spdk_json_decode_bool, true},
-	{"error_recovery_level", offsetof(struct spdk_iscsi_opts, ErrorRecoveryLevel), spdk_json_decode_uint32, true},
-	{"allow_duplicated_isid", offsetof(struct spdk_iscsi_opts, AllowDuplicateIsid), spdk_json_decode_bool, true},
-	{"min_connections_per_core", offsetof(struct spdk_iscsi_opts, min_connections_per_core), spdk_json_decode_uint32, true},
-};
-
-static void
-spdk_rpc_set_iscsi_options(struct spdk_jsonrpc_request *request,
-			   const struct spdk_json_val *params)
-{
-	struct spdk_iscsi_opts *opts;
-	struct spdk_json_write_ctx *w;
-
-	if (g_spdk_iscsi_opts != NULL) {
-		SPDK_ERRLOG("this RPC must not be called more than once.\n");
-		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
-						 "Must not call more than once");
-		return;
-	}
-
-	opts = spdk_iscsi_opts_alloc();
-	if (opts == NULL) {
-		SPDK_ERRLOG("spdk_iscsi_opts_alloc() failed.\n");
-		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
-						 "Out of memory");
-		return;
-	}
-
-	if (params != NULL) {
-		if (spdk_json_decode_object(params, rpc_set_iscsi_opts_decoders,
-					    SPDK_COUNTOF(rpc_set_iscsi_opts_decoders), opts)) {
-			SPDK_ERRLOG("spdk_json_decode_object() failed\n");
-			spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
-							 "Invalid parameters");
-			spdk_iscsi_opts_free(opts);
-			return;
-		}
-	}
-
-	g_spdk_iscsi_opts = spdk_iscsi_opts_copy(opts);
-	spdk_iscsi_opts_free(opts);
-
-	if (g_spdk_iscsi_opts == NULL) {
-		SPDK_ERRLOG("spdk_iscsi_opts_copy() failed\n");
-		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
-						 "Out of memory");
-		return;
-	}
-
-	w = spdk_jsonrpc_begin_result(request);
-	spdk_json_write_bool(w, true);
-	spdk_jsonrpc_end_result(request, w);
-}
-SPDK_RPC_REGISTER("set_iscsi_options", spdk_rpc_set_iscsi_options, SPDK_RPC_STARTUP)

lib/ftl/ftl_core.c

@@ -753,13 +753,14 @@ static void
 ftl_wptr_pad_band(struct ftl_wptr *wptr)
 {
 	struct spdk_ftl_dev *dev = wptr->dev;
-	size_t size = ftl_rwb_num_acquired(dev->rwb, FTL_RWB_TYPE_INTERNAL) +
-		      ftl_rwb_num_acquired(dev->rwb, FTL_RWB_TYPE_USER);
+	size_t size = ftl_rwb_num_pending(dev->rwb);
 	size_t blocks_left, rwb_size, pad_size;
 
 	blocks_left = ftl_wptr_user_lbks_left(wptr);
+	assert(size <= blocks_left);
+	assert(blocks_left % dev->xfer_size == 0);
 	rwb_size = ftl_rwb_size(dev->rwb) - size;
-	pad_size = spdk_min(blocks_left, rwb_size);
+	pad_size = spdk_min(blocks_left - size, rwb_size);
 
 	/* Pad write buffer until band is full */
 	ftl_rwb_pad(dev, pad_size);
@@ -769,8 +770,7 @@ static void
 ftl_wptr_process_shutdown(struct ftl_wptr *wptr)
 {
 	struct spdk_ftl_dev *dev = wptr->dev;
-	size_t size = ftl_rwb_num_acquired(dev->rwb, FTL_RWB_TYPE_INTERNAL) +
-		      ftl_rwb_num_acquired(dev->rwb, FTL_RWB_TYPE_USER);
+	size_t size = ftl_rwb_num_pending(dev->rwb);
 	size_t num_active = dev->xfer_size * ftl_rwb_get_active_batches(dev->rwb);
 
 	num_active = num_active ? num_active : dev->xfer_size;
@@ -2078,14 +2078,10 @@ _ftl_flush(void *ctx)
 }
 
 int
-spdk_ftl_flush(struct spdk_ftl_dev *dev, spdk_ftl_fn cb_fn, void *cb_arg)
+ftl_flush_rwb(struct spdk_ftl_dev *dev, spdk_ftl_fn cb_fn, void *cb_arg)
 {
 	struct ftl_flush *flush;
 
-	if (!dev->initialized) {
-		return -EBUSY;
-	}
-
 	flush = ftl_flush_init(dev, cb_fn, cb_arg);
 	if (!flush) {
 		return -ENOMEM;
@@ -2095,6 +2091,16 @@ spdk_ftl_flush(struct spdk_ftl_dev *dev, spdk_ftl_fn cb_fn, void *cb_arg)
 	return 0;
 }
 
+int
+spdk_ftl_flush(struct spdk_ftl_dev *dev, spdk_ftl_fn cb_fn, void *cb_arg)
+{
+	if (!dev->initialized) {
+		return -EBUSY;
+	}
+
+	return ftl_flush_rwb(dev, cb_fn, cb_arg);
+}
+
 static void
 _ftl_process_anm_event(void *ctx)
 {
@@ -2108,6 +2114,12 @@ ftl_process_anm_event(struct ftl_anm_event *event)
 	struct ftl_band *band;
 	size_t lbkoff;
 
+	/* Drop any ANM requests until the device is initialized */
+	if (!dev->initialized) {
+		ftl_anm_event_complete(event);
+		return;
+	}
+
 	if (!ftl_check_core_thread(dev)) {
 		spdk_thread_send_msg(ftl_get_core_thread(dev), _ftl_process_anm_event, event);
 		return;

lib/ftl/ftl_core.h

@@ -276,7 +276,7 @@ void	ftl_apply_limits(struct spdk_ftl_dev *dev);
 void	ftl_io_read(struct ftl_io *io);
 void	ftl_io_write(struct ftl_io *io);
 int	ftl_io_erase(struct ftl_io *io);
-int	ftl_io_flush(struct ftl_io *io);
+int	ftl_flush_rwb(struct spdk_ftl_dev *dev, spdk_ftl_fn cb_fn, void *cb_arg);
 int	ftl_current_limit(const struct spdk_ftl_dev *dev);
 int	ftl_invalidate_addr(struct spdk_ftl_dev *dev, struct ftl_ppa ppa);
 int	ftl_task_core(void *ctx);

lib/ftl/ftl_restore.c

@@ -544,6 +544,27 @@ ftl_nv_cache_band_flush_cb(void *ctx, int status)
 	}
 }
 
+static void
+ftl_nv_cache_rwb_flush_cb(void *ctx, int status)
+{
+	struct ftl_nv_cache_restore *restore = ctx;
+	struct ftl_nv_cache *nv_cache = restore->nv_cache;
+	struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(nv_cache, struct spdk_ftl_dev, nv_cache);
+	int rc;
+
+	if (spdk_unlikely(status != 0)) {
+		SPDK_ERRLOG("Flushing the write buffer failed: %s\n", spdk_strerror(-status));
+		ftl_nv_cache_restore_complete(restore, status);
+		return;
+	}
+
+	rc = ftl_flush_active_bands(dev, ftl_nv_cache_band_flush_cb, restore);
+	if (spdk_unlikely(rc != 0)) {
+		SPDK_ERRLOG("Unable to flush active bands: %s\n", spdk_strerror(-rc));
+		ftl_nv_cache_restore_complete(restore, rc);
+	}
+}
+
 static void
 ftl_nv_cache_recovery_done(struct ftl_nv_cache_restore *restore)
 {
@@ -567,9 +588,9 @@ ftl_nv_cache_recovery_done(struct ftl_nv_cache_restore *restore)
 	     range_current->start_addr < range_prev->last_addr)) {
 		SPDK_DEBUGLOG(SPDK_LOG_FTL_INIT, "Non-volatile cache inconsistency detected\n");
 
-		rc = ftl_flush_active_bands(dev, ftl_nv_cache_band_flush_cb, restore);
+		rc = ftl_flush_rwb(dev, ftl_nv_cache_rwb_flush_cb, restore);
 		if (spdk_unlikely(rc != 0)) {
-			SPDK_ERRLOG("Unable to flush active bands: %s\n", spdk_strerror(-rc));
+			SPDK_ERRLOG("Unable to flush the write buffer: %s\n", spdk_strerror(-rc));
 			ftl_nv_cache_restore_complete(restore, rc);
 		}
 

lib/ftl/ftl_rwb.c

@@ -83,6 +83,8 @@ struct ftl_rwb {
 
 	/* Number of acquired entries */
 	unsigned int				num_acquired[FTL_RWB_TYPE_MAX];
+	/* Number of acquired but not yet submitted entries */
+	unsigned int				num_pending;
 	/* User/internal limits */
 	size_t					limits[FTL_RWB_TYPE_MAX];
 
@@ -302,7 +304,7 @@ ftl_rwb_batch_release(struct ftl_rwb_batch *batch)
 		num_acquired = __atomic_fetch_sub(&rwb->num_acquired[ftl_rwb_entry_type(entry)], 1,
 						  __ATOMIC_SEQ_CST);
 		entry->band = NULL;
-		assert(num_acquired  > 0);
+		assert(num_acquired > 0);
 	}
 
 	pthread_spin_lock(&rwb->lock);
@@ -371,6 +373,14 @@ ftl_rwb_batch_revert(struct ftl_rwb_batch *batch)
 	if (spdk_ring_enqueue(rwb->prio_queue, (void **)&batch, 1, NULL) != 1) {
 		assert(0 && "Should never happen");
 	}
+
+	__atomic_fetch_add(&rwb->num_pending, rwb->xfer_size, __ATOMIC_SEQ_CST);
+}
+
+unsigned int
+ftl_rwb_num_pending(struct ftl_rwb *rwb)
+{
+	return __atomic_load_n(&rwb->num_pending, __ATOMIC_SEQ_CST);
 }
 
 void
@@ -457,6 +467,7 @@ ftl_rwb_acquire(struct ftl_rwb *rwb, enum ftl_rwb_entry_type type)
 
 	pthread_spin_unlock(&rwb->lock);
 	__atomic_fetch_add(&rwb->num_acquired[type], 1, __ATOMIC_SEQ_CST);
+	__atomic_fetch_add(&rwb->num_pending, 1, __ATOMIC_SEQ_CST);
 	return entry;
 error:
 	pthread_spin_unlock(&rwb->lock);
@@ -491,12 +502,19 @@ struct ftl_rwb_batch *
 ftl_rwb_pop(struct ftl_rwb *rwb)
 {
 	struct ftl_rwb_batch *batch = NULL;
+	unsigned int num_pending __attribute__((unused));
 
 	if (spdk_ring_dequeue(rwb->prio_queue, (void **)&batch, 1) == 1) {
+		num_pending = __atomic_fetch_sub(&rwb->num_pending, rwb->xfer_size,
+						 __ATOMIC_SEQ_CST);
+		assert(num_pending > 0);
 		return batch;
 	}
 
 	if (spdk_ring_dequeue(rwb->submit_queue, (void **)&batch, 1) == 1) {
+		num_pending = __atomic_fetch_sub(&rwb->num_pending, rwb->xfer_size,
+						 __ATOMIC_SEQ_CST);
+		assert(num_pending > 0);
 		return batch;
 	}
 

lib/ftl/ftl_rwb.h

@@ -118,6 +118,7 @@ struct ftl_rwb_entry *ftl_rwb_batch_first_entry(struct ftl_rwb_batch *batch);
 void	*ftl_rwb_batch_get_data(struct ftl_rwb_batch *batch);
 void	*ftl_rwb_batch_get_md(struct ftl_rwb_batch *batch);
 void	ftl_rwb_disable_interleaving(struct ftl_rwb *rwb);
+unsigned int ftl_rwb_num_pending(struct ftl_rwb *rwb);
 
 static inline void
 _ftl_rwb_entry_set_valid(struct ftl_rwb_entry *entry, bool valid)

lib/iscsi/Makefile

@@ -41,6 +41,5 @@ C_SRCS = conn.c \
 	 iscsi_rpc.c task.c
 LIBNAME = iscsi
 LOCAL_SYS_LIBS = -lcrypto
-SPDK_DEP_LIBNAMES = scsi
 
 include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

lib/iscsi/iscsi_rpc.c

@@ -1474,3 +1474,77 @@ spdk_rpc_get_iscsi_auth_groups(struct spdk_jsonrpc_request *request,
 	spdk_jsonrpc_end_result(request, w);
 }
 SPDK_RPC_REGISTER("get_iscsi_auth_groups", spdk_rpc_get_iscsi_auth_groups, SPDK_RPC_RUNTIME)
+
+static const struct spdk_json_object_decoder rpc_set_iscsi_opts_decoders[] = {
+	{"auth_file", offsetof(struct spdk_iscsi_opts, authfile), spdk_json_decode_string, true},
+	{"node_base", offsetof(struct spdk_iscsi_opts, nodebase), spdk_json_decode_string, true},
+	{"nop_timeout", offsetof(struct spdk_iscsi_opts, timeout), spdk_json_decode_int32, true},
+	{"nop_in_interval", offsetof(struct spdk_iscsi_opts, nopininterval), spdk_json_decode_int32, true},
+	{"no_discovery_auth", offsetof(struct spdk_iscsi_opts, disable_chap), spdk_json_decode_bool, true},
+	{"req_discovery_auth", offsetof(struct spdk_iscsi_opts, require_chap), spdk_json_decode_bool, true},
+	{"req_discovery_auth_mutual", offsetof(struct spdk_iscsi_opts, mutual_chap), spdk_json_decode_bool, true},
+	{"discovery_auth_group", offsetof(struct spdk_iscsi_opts, chap_group), spdk_json_decode_int32, true},
+	{"disable_chap", offsetof(struct spdk_iscsi_opts, disable_chap), spdk_json_decode_bool, true},
+	{"require_chap", offsetof(struct spdk_iscsi_opts, require_chap), spdk_json_decode_bool, true},
+	{"mutual_chap", offsetof(struct spdk_iscsi_opts, mutual_chap), spdk_json_decode_bool, true},
+	{"chap_group", offsetof(struct spdk_iscsi_opts, chap_group), spdk_json_decode_int32, true},
+	{"max_sessions", offsetof(struct spdk_iscsi_opts, MaxSessions), spdk_json_decode_uint32, true},
+	{"max_queue_depth", offsetof(struct spdk_iscsi_opts, MaxQueueDepth), spdk_json_decode_uint32, true},
+	{"max_connections_per_session", offsetof(struct spdk_iscsi_opts, MaxConnectionsPerSession), spdk_json_decode_uint32, true},
+	{"default_time2wait", offsetof(struct spdk_iscsi_opts, DefaultTime2Wait), spdk_json_decode_uint32, true},
+	{"default_time2retain", offsetof(struct spdk_iscsi_opts, DefaultTime2Retain), spdk_json_decode_uint32, true},
+	{"first_burst_length", offsetof(struct spdk_iscsi_opts, FirstBurstLength), spdk_json_decode_uint32, true},
+	{"immediate_data", offsetof(struct spdk_iscsi_opts, ImmediateData), spdk_json_decode_bool, true},
+	{"error_recovery_level", offsetof(struct spdk_iscsi_opts, ErrorRecoveryLevel), spdk_json_decode_uint32, true},
+	{"allow_duplicated_isid", offsetof(struct spdk_iscsi_opts, AllowDuplicateIsid), spdk_json_decode_bool, true},
+	{"min_connections_per_core", offsetof(struct spdk_iscsi_opts, min_connections_per_core), spdk_json_decode_uint32, true},
+};
+
+static void
+spdk_rpc_set_iscsi_options(struct spdk_jsonrpc_request *request,
+			   const struct spdk_json_val *params)
+{
+	struct spdk_iscsi_opts *opts;
+	struct spdk_json_write_ctx *w;
+
+	if (g_spdk_iscsi_opts != NULL) {
+		SPDK_ERRLOG("this RPC must not be called more than once.\n");
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
+						 "Must not call more than once");
+		return;
+	}
+
+	opts = spdk_iscsi_opts_alloc();
+	if (opts == NULL) {
+		SPDK_ERRLOG("spdk_iscsi_opts_alloc() failed.\n");
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
+						 "Out of memory");
+		return;
+	}
+
+	if (params != NULL) {
+		if (spdk_json_decode_object(params, rpc_set_iscsi_opts_decoders,
+					    SPDK_COUNTOF(rpc_set_iscsi_opts_decoders), opts)) {
+			SPDK_ERRLOG("spdk_json_decode_object() failed\n");
+			spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+							 "Invalid parameters");
+			spdk_iscsi_opts_free(opts);
+			return;
+		}
+	}
+
+	g_spdk_iscsi_opts = spdk_iscsi_opts_copy(opts);
+	spdk_iscsi_opts_free(opts);
+
+	if (g_spdk_iscsi_opts == NULL) {
+		SPDK_ERRLOG("spdk_iscsi_opts_copy() failed\n");
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
+						 "Out of memory");
+		return;
+	}
+
+	w = spdk_jsonrpc_begin_result(request);
+	spdk_json_write_bool(w, true);
+	spdk_jsonrpc_end_result(request, w);
+}
+SPDK_RPC_REGISTER("set_iscsi_options", spdk_rpc_set_iscsi_options, SPDK_RPC_STARTUP)

lib/log/Makefile

@@ -40,6 +40,4 @@ ifeq ($(CONFIG_LOG_BACKTRACE),y)
 LOCAL_SYS_LIBS += -lunwind
 endif
 
-DIRS-y = rpc
-
 include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

lib/log_rpc/Makefile

@@ -31,7 +31,7 @@
 #  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
 
-SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
 C_SRCS = log_rpc.c

lib/notify/Makefile

@@ -34,7 +34,7 @@
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-C_SRCS = notify.c
+C_SRCS = notify.c notify_rpc.c
 LIBNAME = notify
 
 include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

lib/nvme/nvme_opal.c

@@ -549,6 +549,9 @@ static size_t
 opal_response_get_string(const struct spdk_opal_resp_parsed *resp, int n,
 			 const char **store)
 {
+	uint8_t header_len;
+	struct spdk_opal_resp_token token = resp->resp_tokens[n];
+
 	*store = NULL;
 	if (!resp) {
 		SPDK_ERRLOG("Response is NULL\n");
@@ -561,13 +564,28 @@ opal_response_get_string(const struct spdk_opal_resp_parsed *resp, int n,
 		return 0;
 	}
 
-	if (resp->resp_tokens[n].type != OPAL_DTA_TOKENID_BYTESTRING) {
+	if (token.type != OPAL_DTA_TOKENID_BYTESTRING) {
 		SPDK_ERRLOG("Token is not a byte string!\n");
 		return 0;
 	}
 
-	*store = resp->resp_tokens[n].pos + 1;
-	return resp->resp_tokens[n].len - 1;
+	switch (token.width) {
+	case OPAL_WIDTH_SHORT:
+		header_len = 1;
+		break;
+	case OPAL_WIDTH_MEDIUM:
+		header_len = 2;
+		break;
+	case OPAL_WIDTH_LONG:
+		header_len = 4;
+		break;
+	default:
+		SPDK_ERRLOG("Can't get string from this Token\n");
+		return 0;
+	}
+
+	*store = token.pos + header_len;
+	return token.len - header_len;
 }
 
 static int

lib/nvme/nvme_qpair.c

@@ -239,7 +239,7 @@ static const struct nvme_string command_specific_status[] = {
 	{ SPDK_NVME_SC_INVALID_RESOURCE_ID, "INVALID RESOURCE IDENTIFIER" },
 	{ SPDK_NVME_SC_CONFLICTING_ATTRIBUTES, "CONFLICTING ATTRIBUTES" },
 	{ SPDK_NVME_SC_INVALID_PROTECTION_INFO, "INVALID PROTECTION INFO" },
-	{ SPDK_NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE, "WRITE TO RO PAGE" },
+	{ SPDK_NVME_SC_ATTEMPTED_WRITE_TO_RO_RANGE, "WRITE TO RO RANGE" },
 	{ 0xFFFF, "COMMAND SPECIFIC" }
 };
 

lib/nvme/nvme_rdma.c

@@ -1428,22 +1428,22 @@ nvme_rdma_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx,
 		return -1;
 	}
 
+	/* get the cdata info */
+	rc = nvme_ctrlr_cmd_identify(discovery_ctrlr, SPDK_NVME_IDENTIFY_CTRLR, 0, 0,
+				     &discovery_ctrlr->cdata, sizeof(discovery_ctrlr->cdata),
+				     nvme_completion_poll_cb, &status);
+	if (rc != 0) {
+		SPDK_ERRLOG("Failed to identify cdata\n");
+		return rc;
+	}
+
+	if (spdk_nvme_wait_for_completion(discovery_ctrlr->adminq, &status)) {
+		SPDK_ERRLOG("nvme_identify_controller failed!\n");
+		return -ENXIO;
+	}
+
 	/* Direct attach through spdk_nvme_connect() API */
 	if (direct_connect == true) {
-		/* get the cdata info */
-		rc = nvme_ctrlr_cmd_identify(discovery_ctrlr, SPDK_NVME_IDENTIFY_CTRLR, 0, 0,
-					     &discovery_ctrlr->cdata, sizeof(discovery_ctrlr->cdata),
-					     nvme_completion_poll_cb, &status);
-		if (rc != 0) {
-			SPDK_ERRLOG("Failed to identify cdata\n");
-			return rc;
-		}
-
-		if (spdk_nvme_wait_for_completion(discovery_ctrlr->adminq, &status)) {
-			SPDK_ERRLOG("nvme_identify_controller failed!\n");
-			return -ENXIO;
-		}
-
 		/* Set the ready state to skip the normal init process */
 		discovery_ctrlr->state = NVME_CTRLR_STATE_READY;
 		nvme_ctrlr_connected(probe_ctx, discovery_ctrlr);

lib/nvme/nvme_tcp.c

@@ -310,22 +310,26 @@ nvme_tcp_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx,
 		return -1;
 	}
 
+	/* get the cdata info */
+	status.done = false;
+	rc = nvme_ctrlr_cmd_identify(discovery_ctrlr, SPDK_NVME_IDENTIFY_CTRLR, 0, 0,
+				     &discovery_ctrlr->cdata, sizeof(discovery_ctrlr->cdata),
+				     nvme_completion_poll_cb, &status);
+	if (rc != 0) {
+		SPDK_ERRLOG("Failed to identify cdata\n");
+		return rc;
+	}
+
+	while (status.done == false) {
+		spdk_nvme_qpair_process_completions(discovery_ctrlr->adminq, 0);
+	}
+	if (spdk_nvme_cpl_is_error(&status.cpl)) {
+		SPDK_ERRLOG("nvme_identify_controller failed!\n");
+		return -ENXIO;
+	}
+
 	/* Direct attach through spdk_nvme_connect() API */
 	if (direct_connect == true) {
-		/* get the cdata info */
-		status.done = false;
-		rc = nvme_ctrlr_cmd_identify(discovery_ctrlr, SPDK_NVME_IDENTIFY_CTRLR, 0, 0,
-					     &discovery_ctrlr->cdata, sizeof(discovery_ctrlr->cdata),
-					     nvme_completion_poll_cb, &status);
-		if (rc != 0) {
-			SPDK_ERRLOG("Failed to identify cdata\n");
-			return rc;
-		}
-
-		if (spdk_nvme_wait_for_completion(discovery_ctrlr->adminq, &status)) {
-			SPDK_ERRLOG("nvme_identify_controller failed!\n");
-			return -ENXIO;
-		}
 		/* Set the ready state to skip the normal init process */
 		discovery_ctrlr->state = NVME_CTRLR_STATE_READY;
 		nvme_ctrlr_connected(probe_ctx, discovery_ctrlr);

lib/nvmf/ctrlr.c

@@ -1331,8 +1331,6 @@ spdk_nvmf_ctrlr_ns_changed(struct spdk_nvmf_ctrlr *ctrlr, uint32_t nsid)
 			ctrlr->changed_ns_list.ns_list[ctrlr->changed_ns_list_count++] = nsid;
 		}
 	}
-
-	spdk_nvmf_ctrlr_async_event_ns_notice(ctrlr);
 }
 
 static void

lib/nvmf/ctrlr_discovery.c

@@ -48,8 +48,8 @@
 #include "spdk/bdev_module.h"
 #include "spdk_internal/log.h"
 
-static void
-nvmf_update_discovery_log(struct spdk_nvmf_tgt *tgt, const char *hostnqn)
+static struct spdk_nvmf_discovery_log_page *
+nvmf_generate_discovery_log(struct spdk_nvmf_tgt *tgt, const char *hostnqn, size_t *log_page_size)
 {
 	uint64_t numrec = 0;
 	struct spdk_nvmf_subsystem *subsystem;
@@ -66,7 +66,7 @@ nvmf_update_discovery_log(struct spdk_nvmf_tgt *tgt, const char *hostnqn)
 	disc_log = calloc(1, cur_size);
 	if (disc_log == NULL) {
 		SPDK_ERRLOG("Discovery log page memory allocation error\n");
-		return;
+		return NULL;
 	}
 
 	for (sid = 0; sid < tgt->max_subsystems; sid++) {
@@ -114,11 +114,9 @@ nvmf_update_discovery_log(struct spdk_nvmf_tgt *tgt, const char *hostnqn)
 
 	disc_log->numrec = numrec;
 	disc_log->genctr = tgt->discovery_genctr;
+	*log_page_size = cur_size;
 
-	free(tgt->discovery_log_page);
-
-	tgt->discovery_log_page = disc_log;
-	tgt->discovery_log_page_size = cur_size;
+	return disc_log;
 }
 
 void
@@ -128,24 +126,23 @@ spdk_nvmf_get_discovery_log_page(struct spdk_nvmf_tgt *tgt, const char *hostnqn,
 	size_t copy_len = 0;
 	size_t zero_len = 0;
 	struct iovec *tmp;
+	size_t log_page_size = 0;
+	struct spdk_nvmf_discovery_log_page *discovery_log_page;
 
-	if (offset == 0 || tgt->discovery_log_page == NULL ||
-	    tgt->discovery_log_page->genctr != tgt->discovery_genctr) {
-		nvmf_update_discovery_log(tgt, hostnqn);
-	}
+	discovery_log_page = nvmf_generate_discovery_log(tgt, hostnqn, &log_page_size);
 
 	/* Copy the valid part of the discovery log page, if any */
-	if (tgt->discovery_log_page) {
+	if (discovery_log_page) {
 		for (tmp = iov; tmp < iov + iovcnt; tmp++) {
 			copy_len = spdk_min(tmp->iov_len, length);
-			copy_len = spdk_min(tgt->discovery_log_page_size - offset, copy_len);
+			copy_len = spdk_min(log_page_size - offset, copy_len);
 
-			memcpy(tmp->iov_base, (char *)tgt->discovery_log_page + offset, copy_len);
+			memcpy(tmp->iov_base, (char *)discovery_log_page + offset, copy_len);
 
 			offset += copy_len;
 			length -= copy_len;
 			zero_len = tmp->iov_len - copy_len;
-			if (tgt->discovery_log_page_size <= offset || length == 0) {
+			if (log_page_size <= offset || length == 0) {
 				break;
 			}
 		}
@@ -157,5 +154,7 @@ spdk_nvmf_get_discovery_log_page(struct spdk_nvmf_tgt *tgt, const char *hostnqn,
 		for (++tmp; tmp < iov + iovcnt; tmp++) {
 			memset((char *)tmp->iov_base, 0, tmp->iov_len);
 		}
+
+		free(discovery_log_page);
 	}
 }

lib/nvmf/nvmf.c

@@ -233,8 +233,6 @@ spdk_nvmf_tgt_create(uint32_t max_subsystems)
 	}
 
 	tgt->discovery_genctr = 0;
-	tgt->discovery_log_page = NULL;
-	tgt->discovery_log_page_size = 0;
 	TAILQ_INIT(&tgt->transports);
 
 	tgt->subsystems = calloc(tgt->max_subsystems, sizeof(struct spdk_nvmf_subsystem *));
@@ -261,10 +259,6 @@ spdk_nvmf_tgt_destroy_cb(void *io_device)
 	void					*destroy_cb_arg;
 	uint32_t i;
 
-	if (tgt->discovery_log_page) {
-		free(tgt->discovery_log_page);
-	}
-
 	if (tgt->subsystems) {
 		for (i = 0; i < tgt->max_subsystems; i++) {
 			if (tgt->subsystems[i]) {
@@ -872,6 +866,8 @@ poll_group_update_subsystem(struct spdk_nvmf_poll_group *group,
 	struct spdk_nvmf_registrant *reg, *tmp;
 	struct spdk_io_channel *ch;
 	struct spdk_nvmf_subsystem_pg_ns_info *ns_info;
+	struct spdk_nvmf_ctrlr *ctrlr;
+	bool ns_changed;
 
 	/* Make sure our poll group has memory for this subsystem allocated */
 	if (subsystem->id >= group->num_sgroups) {
@@ -884,6 +880,8 @@ poll_group_update_subsystem(struct spdk_nvmf_poll_group *group,
 	new_num_ns = subsystem->max_nsid;
 	old_num_ns = sgroup->num_ns;
 
+	ns_changed = false;
+
 	if (old_num_ns == 0) {
 		if (new_num_ns > 0) {
 			/* First allocation */
@@ -945,10 +943,12 @@ poll_group_update_subsystem(struct spdk_nvmf_poll_group *group,
 			/* Both NULL. Leave empty */
 		} else if (ns == NULL && ch != NULL) {
 			/* There was a channel here, but the namespace is gone. */
+			ns_changed = true;
 			spdk_put_io_channel(ch);
 			ns_info->channel = NULL;
 		} else if (ns != NULL && ch == NULL) {
 			/* A namespace appeared but there is no channel yet */
+			ns_changed = true;
 			ch = spdk_bdev_get_io_channel(ns->desc);
 			if (ch == NULL) {
 				SPDK_ERRLOG("Could not allocate I/O channel.\n");
@@ -957,6 +957,7 @@ poll_group_update_subsystem(struct spdk_nvmf_poll_group *group,
 			ns_info->channel = ch;
 		} else if (spdk_uuid_compare(&ns_info->uuid, spdk_bdev_get_uuid(ns->bdev)) != 0) {
 			/* A namespace was here before, but was replaced by a new one. */
+			ns_changed = true;
 			spdk_put_io_channel(ns_info->channel);
 			memset(ns_info, 0, sizeof(*ns_info));
 
@@ -990,6 +991,14 @@ poll_group_update_subsystem(struct spdk_nvmf_poll_group *group,
 		}
 	}
 
+	if (ns_changed) {
+		TAILQ_FOREACH(ctrlr, &subsystem->ctrlrs, link) {
+			if (ctrlr->admin_qpair->group == group) {
+				spdk_nvmf_ctrlr_async_event_ns_notice(ctrlr);
+			}
+		}
+	}
+
 	return 0;
 }
 

lib/nvmf/nvmf_internal.h

@@ -81,8 +81,6 @@ struct spdk_nvmf_tgt {
 	/* Array of subsystem pointers of size max_subsystems indexed by sid */
 	struct spdk_nvmf_subsystem		**subsystems;
 
-	struct spdk_nvmf_discovery_log_page	*discovery_log_page;
-	size_t					discovery_log_page_size;
 	TAILQ_HEAD(, spdk_nvmf_transport)	transports;
 
 	spdk_nvmf_tgt_destroy_done_fn		*destroy_cb_fn;

lib/nvmf/rdma.c

@@ -644,40 +644,22 @@ static void
 nvmf_rdma_request_free_data(struct spdk_nvmf_rdma_request *rdma_req,
 			    struct spdk_nvmf_rdma_transport *rtransport)
 {
-	struct spdk_nvmf_rdma_request_data	*current_data_wr = NULL, *next_data_wr = NULL;
-	struct ibv_send_wr			*send_wr;
-	int					i;
+	struct spdk_nvmf_rdma_request_data	*data_wr;
+	struct ibv_send_wr			*next_send_wr;
+	uint64_t				req_wrid;
 
 	rdma_req->num_outstanding_data_wr = 0;
-	current_data_wr = &rdma_req->data;
-	for (i = 0; i < current_data_wr->wr.num_sge; i++) {
-		current_data_wr->wr.sg_list[i].addr = 0;
-		current_data_wr->wr.sg_list[i].length = 0;
-		current_data_wr->wr.sg_list[i].lkey = 0;
-	}
-	current_data_wr->wr.num_sge = 0;
-
-	send_wr = current_data_wr->wr.next;
-	if (send_wr != NULL && send_wr != &rdma_req->rsp.wr) {
-		next_data_wr = SPDK_CONTAINEROF(send_wr, struct spdk_nvmf_rdma_request_data, wr);
-	}
-	while (next_data_wr) {
-		current_data_wr = next_data_wr;
-		send_wr = current_data_wr->wr.next;
-		if (send_wr != NULL && send_wr != &rdma_req->rsp.wr &&
-		    send_wr->wr_id == current_data_wr->wr.wr_id) {
-			next_data_wr = SPDK_CONTAINEROF(send_wr, struct spdk_nvmf_rdma_request_data, wr);
-		} else {
-			next_data_wr = NULL;
+	data_wr = &rdma_req->data;
+	req_wrid = data_wr->wr.wr_id;
+	while (data_wr && data_wr->wr.wr_id == req_wrid) {
+		memset(data_wr->sgl, 0, sizeof(data_wr->wr.sg_list[0]) * data_wr->wr.num_sge);
+		data_wr->wr.num_sge = 0;
+		next_send_wr = data_wr->wr.next;
+		if (data_wr != &rdma_req->data) {
+			spdk_mempool_put(rtransport->data_wr_pool, data_wr);
 		}
-
-		for (i = 0; i < current_data_wr->wr.num_sge; i++) {
-			current_data_wr->wr.sg_list[i].addr = 0;
-			current_data_wr->wr.sg_list[i].length = 0;
-			current_data_wr->wr.sg_list[i].lkey = 0;
-		}
-		current_data_wr->wr.num_sge = 0;
-		spdk_mempool_put(rtransport->data_wr_pool, current_data_wr);
+		data_wr = (!next_send_wr || next_send_wr == &rdma_req->rsp.wr) ? NULL :
+			  SPDK_CONTAINEROF(next_send_wr, struct spdk_nvmf_rdma_request_data, wr);
 	}
 }
 
@@ -3052,6 +3034,7 @@ spdk_nvmf_rdma_poll_group_create(struct spdk_nvmf_transport *transport)
 				SPDK_ERRLOG("Unable to allocate resources for shared receive queue.\n");
 				spdk_nvmf_rdma_poll_group_destroy(&rgroup->group);
 				pthread_mutex_unlock(&rtransport->lock);
+				return NULL;
 			}
 		}
 
@@ -3480,6 +3463,23 @@ spdk_nvmf_rdma_poller_poll(struct spdk_nvmf_rdma_transport *rtransport,
 			rdma_recv = SPDK_CONTAINEROF(rdma_wr, struct spdk_nvmf_rdma_recv, rdma_wr);
 			if (rpoller->srq != NULL) {
 				rdma_recv->qpair = get_rdma_qpair_from_wc(rpoller, &wc[i]);
+				/* It is possible that there are still some completions for destroyed QP
+				 * associated with SRQ. We just ignore these late completions and re-post
+				 * receive WRs back to SRQ.
+				 */
+				if (spdk_unlikely(NULL == rdma_recv->qpair)) {
+					struct ibv_recv_wr *bad_wr;
+					int rc;
+
+					rdma_recv->wr.next = NULL;
+					rc = ibv_post_srq_recv(rpoller->srq,
+							       &rdma_recv->wr,
+							       &bad_wr);
+					if (rc) {
+						SPDK_ERRLOG("Failed to re-post recv WR to SRQ, err %d\n", rc);
+					}
+					continue;
+				}
 			}
 			rqpair = rdma_recv->qpair;
 

lib/rte_vhost/Makefile

@@ -31,7 +31,7 @@
 #  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
 
-SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
 CFLAGS += -I.

lib/sock/Makefile

@@ -38,7 +38,4 @@ C_SRCS = sock.c net_framework.c
 
 LIBNAME = sock
 
-DIRS-y += posix
-DIRS-$(CONFIG_VPP) += vpp
-
 include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

lib/sock/sock.c

@@ -103,6 +103,7 @@ spdk_sock_map_release(int placement_id)
 			entry->ref--;
 			if (!entry->ref) {
 				STAILQ_REMOVE(&g_placement_id_map, entry, spdk_sock_placement_id_entry, link);
+				free(entry);
 			}
 			break;
 		}
@@ -139,6 +140,7 @@ spdk_sock_remove_sock_group_from_map_table(struct spdk_sock_group *group)
 	STAILQ_FOREACH_SAFE(entry, &g_placement_id_map, link, tmp) {
 		if (entry->group == group) {
 			STAILQ_REMOVE(&g_placement_id_map, entry, spdk_sock_placement_id_entry, link);
+			free(entry);
 		}
 	}
 	pthread_mutex_unlock(&g_map_table_mutex);

lib/trace/Makefile

@@ -34,9 +34,7 @@
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-C_SRCS = trace.c trace_flags.c
+C_SRCS = trace.c trace_flags.c trace_rpc.c
 LIBNAME = trace
 
-DIRS-y = rpc
-
 include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

lib/trace/rpc/Makefile

@@ -1,40 +0,0 @@
-#
-#  BSD LICENSE
-#
-#  Copyright (c) Intel Corporation.
-#  All rights reserved.
-#
-#  Redistribution and use in source and binary forms, with or without
-#  modification, are permitted provided that the following conditions
-#  are met:
-#
-#    * Redistributions of source code must retain the above copyright
-#      notice, this list of conditions and the following disclaimer.
-#    * Redistributions in binary form must reproduce the above copyright
-#      notice, this list of conditions and the following disclaimer in
-#      the documentation and/or other materials provided with the
-#      distribution.
-#    * Neither the name of Intel Corporation nor the names of its
-#      contributors may be used to endorse or promote products derived
-#      from this software without specific prior written permission.
-#
-#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
-
-SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
-include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
-
-C_SRCS = trace_rpc.c
-LIBNAME = trace_rpc
-
-include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

lib/vhost/Makefile

@@ -41,8 +41,7 @@ C_SRCS = vhost.c vhost_rpc.c vhost_scsi.c vhost_blk.c rte_vhost_compat.c
 
 ifeq ($(CONFIG_VHOST_INTERNAL_LIB),y)
 C_SRCS += vhost_nvme.c
-DIRS-y += rte_vhost
-CFLAGS := -Irte_vhost $(CFLAGS)
+CFLAGS := -I../rte_vhost $(CFLAGS)
 endif
 
 LIBNAME = vhost

mk/spdk.app_vars.mk

@@ -35,6 +35,7 @@ SPDK_LIB_FILES = $(call spdk_lib_list_to_static_libs,$(SPDK_LIB_LIST))
 SPDK_LIB_LINKER_ARGS = \
 	-L$(SPDK_ROOT_DIR)/build/lib \
 	-Wl,--whole-archive \
+	-Wl,--no-as-needed \
 	$(SPDK_LIB_LIST:%=-lspdk_%) \
 	-Wl,--no-whole-archive
 

mk/spdk.lib.mk

@@ -32,6 +32,7 @@
 #
 
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+include $(SPDK_ROOT_DIR)/mk/spdk.lib_deps.mk
 
 SPDK_MAP_FILE = $(SPDK_ROOT_DIR)/shared_lib/spdk.map
 LIB := $(call spdk_lib_list_to_static_libs,$(LIBNAME))
@@ -50,7 +51,7 @@ else
 LOCAL_SYS_LIBS += -lrt
 endif
 
-SPDK_DEP_LIBS = $(call spdk_lib_list_to_shared_libs,$(SPDK_DEP_LIBNAMES))
+SPDK_DEP_LIBS = $(call spdk_lib_list_to_shared_libs,$(DEPDIRS-$(LIBNAME)))
 
 .PHONY: all clean $(DIRS-y)
 

mk/spdk.lib_deps.mk

@@ -0,0 +1,154 @@
+#
+#  BSD LICENSE
+#
+#  Copyright (c) Intel Corporation.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+# A quick note on organization:
+#
+# Each grouping is independent from itself. it depends only on libraries
+# in the grouping above it. All dependencies are listed alphabetically within
+# groups. The only exception to this is the JSON_LIBS grouping which is a special
+# case since they almost always occur together.
+
+JSON_LIBS := json jsonrpc rpc
+
+DEPDIRS-env_ocf :=
+DEPDIRS-log :=
+DEPDIRS-rte_vhost :=
+
+DEPDIRS-ioat := log
+DEPDIRS-sock := log
+DEPDIRS-util := log
+DEPDIRS-vmd := log
+
+DEPDIRS-conf := log util
+DEPDIRS-json := log util
+DEPDIRS-nvme := log sock util
+DEPDIRS-reduce := log util
+DEPDIRS-thread := log util
+
+DEPDIRS-blob := log util thread
+DEPDIRS-copy := thread
+DEPDIRS-jsonrpc := log util json
+DEPDIRS-virtio := log util json thread
+
+DEPDIRS-lvol := log util blob
+DEPDIRS-rpc := log util json jsonrpc
+
+DEPDIRS-log_rpc := log $(JSON_LIBS)
+DEPDIRS-net := log util $(JSON_LIBS)
+DEPDIRS-notify := log util $(JSON_LIBS)
+DEPDIRS-trace := log util $(JSON_LIBS)
+
+DEPDIRS-bdev := log util conf thread $(JSON_LIBS) notify trace
+DEPDIRS-blobfs := log conf thread blob trace
+DEPDIRS-event := log util conf thread $(JSON_LIBS) trace
+
+DEPDIRS-ftl := log util nvme thread trace bdev
+DEPDIRS-nbd := log util thread $(JSON_LIBS) bdev
+DEPDIRS-nvmf := log sock util json nvme thread  trace bdev
+DEPDIRS-scsi := log util thread $(JSON_LIBS) trace bdev
+
+DEPDIRS-iscsi := log sock util conf thread $(JSON_LIBS) trace event scsi
+DEPDIRS-vhost = log util conf thread $(JSON_LIBS) bdev event scsi
+ifeq ($(CONFIG_VHOST_INTERNAL_LIB),y)
+DEPDIRS-vhost += rte_vhost
+endif
+
+# ------------------------------------------------------------------------
+# Start module/ directory - This section extends the organizational pattern from
+# above. However, it introduces several more groupings which may not strictly follow
+# the ordering pattern above. These are used for convenience and to help quickly
+# determine the unique dependencies of a given module. It is also grouped by directory.
+
+BDEV_DEPS = log util $(JSON_LIBS) bdev
+BDEV_DEPS_CONF = $(BDEV_DEPS) conf
+BDEV_DEPS_THREAD = $(BDEV_DEPS) thread
+BDEV_DEPS_CONF_THREAD = $(BDEV_DEPS) conf thread
+
+# module/blob
+DEPDIRS-blob_bdev := log thread bdev
+
+# module/copy
+DEPDIRS-copy_ioat := log ioat conf thread $(JSON_LIBS) copy
+
+# module/sock
+DEPDIRS-sock_posix := log sock
+DEPDIRS-sock_vpp := log sock util thread
+
+# module/bdev
+DEPDIRS-bdev_gpt := bdev conf json log thread util
+
+DEPDIRS-bdev_lvol := $(BDEV_DEPS) lvol blob blob_bdev
+DEPDIRS-bdev_rpc := $(BDEV_DEPS)
+
+DEPDIRS-bdev_error := $(BDEV_DEPS_CONF)
+DEPDIRS-bdev_malloc := $(BDEV_DEPS_CONF) copy
+DEPDIRS-bdev_split := $(BDEV_DEPS_CONF)
+
+DEPDIRS-bdev_compress := $(BDEV_DEPS_THREAD) reduce
+DEPDIRS-bdev_delay := $(BDEV_DEPS_THREAD)
+
+DEPDIRS-bdev_aio := $(BDEV_DEPS_CONF_THREAD)
+DEPDIRS-bdev_crypto := $(BDEV_DEPS_CONF_THREAD)
+DEPDIRS-bdev_iscsi := $(BDEV_DEPS_CONF_THREAD)
+DEPDIRS-bdev_null := $(BDEV_DEPS_CONF_THREAD)
+DEPDIRS-bdev_nvme = $(BDEV_DEPS_CONF_THREAD) nvme
+ifeq ($(OS),Linux)
+DEPDIRS-bdev_nvme += ftl
+endif
+DEPDIRS-bdev_ocf := $(BDEV_DEPS_CONF_THREAD)
+DEPDIRS-bdev_passthru := $(BDEV_DEPS_CONF_THREAD)
+DEPDIRS-bdev_pmem := $(BDEV_DEPS_CONF_THREAD)
+DEPDIRS-bdev_raid := $(BDEV_DEPS_CONF_THREAD)
+DEPDIRS-bdev_rbd := $(BDEV_DEPS_CONF_THREAD)
+DEPDIRS-bdev_virtio := $(BDEV_DEPS_CONF_THREAD) virtio
+
+# module/event
+# module/event/app
+DEPDIRS-app_rpc := log util  thread event $(JSON_LIBS)
+
+#module/event/subsystems
+# These depdirs include subsystem interdependencies which
+# are not related to symbols, but are defined directly in
+# the SPDK event subsystem code.
+DEPDIRS-event_copy := copy event
+DEPDIRS-event_net := sock net event
+DEPDIRS-event_vmd := vmd conf $(JSON_LIBS) event
+
+DEPDIRS-event_bdev := bdev event event_copy event_vmd
+
+DEPDIRS-event_nbd := event nbd event_bdev
+DEPDIRS-event_nvmf := $(BDEV_DEPS_CONF_THREAD) event nvme nvmf event_bdev
+DEPDIRS-event_scsi := event scsi event_bdev
+
+DEPDIRS-event_iscsi := event iscsi event_scsi
+DEPDIRS-event_vhost := event vhost event_scsi

mk/spdk.unittest.mk

@@ -38,6 +38,7 @@ include $(SPDK_ROOT_DIR)/mk/spdk.mock.unittest.mk
 C_SRCS = $(TEST_FILE)
 
 CFLAGS += -I$(SPDK_ROOT_DIR)/lib
+CFLAGS += -I$(SPDK_ROOT_DIR)/module
 CFLAGS += -I$(SPDK_ROOT_DIR)/test
 CFLAGS += -ffunction-sections
 LDFLAGS += -Wl,--gc-sections

module/Makefile

@@ -0,0 +1,50 @@
+#
+#  BSD LICENSE
+#
+#  Copyright (c) Intel Corporation.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+DIRS-y = bdev blob copy event sock
+
+DEPDIRS-blob :=
+DEPDIRS-copy :=
+DEPDIRS-sock :=
+DEPDIRS-bdev := blob
+DEPDIRS-event := bdev blob
+
+.PHONY: all clean $(DIRS-y)
+
+all: $(DIRS-y)
+clean: $(DIRS-y)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk

module/bdev/Makefile

@@ -0,0 +1,69 @@
+#
+#  BSD LICENSE
+#
+#  Copyright (c) Intel Corporation.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+DIRS-y += delay error gpt lvol malloc null nvme passthru raid rpc split
+
+ifeq ($(CONFIG_CRYPTO),y)
+DIRS-y += crypto
+endif
+
+ifeq ($(CONFIG_OCF), y)
+DIRS-y += ocf
+endif
+
+ifeq ($(CONFIG_REDUCE),y)
+DIRS-y += compress
+endif
+
+ifeq ($(CONFIG_URING),y)
+DIRS-y += uring
+endif
+
+ifeq ($(OS),Linux)
+DIRS-y += aio
+DIRS-$(CONFIG_ISCSI_INITIATOR) += iscsi
+DIRS-$(CONFIG_VIRTIO) += virtio
+DIRS-$(CONFIG_PMDK) += pmem
+endif
+
+DIRS-$(CONFIG_RBD) += rbd
+
+.PHONY: all clean $(DIRS-y)
+
+all: $(DIRS-y)
+clean: $(DIRS-y)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk

module/bdev/compress/vbdev_compress.c

@@ -54,7 +54,10 @@
 #define NUM_MAX_XFORMS 2
 #define NUM_MAX_INFLIGHT_OPS 128
 #define DEFAULT_WINDOW_SIZE 15
-#define MAX_MBUFS_PER_OP REDUCE_MAX_IOVECS
+/* We need extra mbufs per operation to accomodate host buffers that
+ *  span a 2MB boundary.
+ */
+#define MAX_MBUFS_PER_OP (REDUCE_MAX_IOVECS * 2)
 #define CHUNK_SIZE (1024 * 16)
 #define COMP_BDEV_NAME "compress"
 #define BACKING_IO_SZ (4 * 1024)
@@ -437,13 +440,15 @@ _compress_operation(struct spdk_reduce_backing_dev *backing_dev, struct iovec *s
 	struct rte_mbuf *src_mbufs[MAX_MBUFS_PER_OP];
 	struct rte_mbuf *dst_mbufs[MAX_MBUFS_PER_OP];
 	uint8_t cdev_id = comp_bdev->device_qp->device->cdev_id;
-	uint64_t total_length = 0;
-	struct iovec *current_src_iov = NULL;
-	struct iovec *current_dst_iov = NULL;
-	int iov_index;
+	uint64_t updated_length, remainder, phys_addr, total_length = 0;
+	uint8_t *current_src_base = NULL;
+	uint8_t *current_dst_base = NULL;
+	int iov_index, mbuf_index;
 	int rc = 0;
 	struct vbdev_comp_op *op_to_queue;
 	int i;
+	int src_mbuf_total = src_iovcnt;
+	int dst_mbuf_total = dst_iovcnt;
 
 	assert(src_iovcnt < MAX_MBUFS_PER_OP);
 
@@ -477,23 +482,53 @@ _compress_operation(struct spdk_reduce_backing_dev *backing_dev, struct iovec *s
 	 */
 
 	/* Setup src mbufs */
-	for (iov_index = 0; iov_index < src_iovcnt; iov_index++) {
+	iov_index = mbuf_index = 0;
+	while (iov_index < src_iovcnt) {
 
-		current_src_iov = src_iovs[iov_index].iov_base;
+		current_src_base = src_iovs[iov_index].iov_base;
 		total_length += src_iovs[iov_index].iov_len;
-		assert(src_mbufs[iov_index] != NULL);
+		assert(src_mbufs[mbuf_index] != NULL);
 		src_mbufs[iov_index]->userdata = reduce_cb_arg;
+		updated_length = src_iovs[iov_index].iov_len;
+		phys_addr = spdk_vtophys((void *)current_src_base, &updated_length);
 
-		rte_pktmbuf_attach_extbuf(src_mbufs[iov_index],
-					  current_src_iov,
-					  spdk_vtophys((void *)current_src_iov, NULL),
-					  src_iovs[iov_index].iov_len,
+		rte_pktmbuf_attach_extbuf(src_mbufs[mbuf_index],
+					  current_src_base,
+					  phys_addr,
+					  updated_length,
 					  &g_shinfo);
-		rte_pktmbuf_append(src_mbufs[iov_index], src_iovs[iov_index].iov_len);
+		rte_pktmbuf_append(src_mbufs[mbuf_index], updated_length);
+		remainder = src_iovs[iov_index].iov_len - updated_length;
 
-		if (iov_index > 0) {
-			rte_pktmbuf_chain(src_mbufs[0], src_mbufs[iov_index]);
+		if (mbuf_index > 0) {
+			rte_pktmbuf_chain(src_mbufs[0], src_mbufs[mbuf_index]);
 		}
+
+		/* If we crossed 2 2MB boundary we need another mbuf for the remainder */
+		if (remainder > 0) {
+			/* allocate an mbuf at the end of the array */
+			rc = rte_pktmbuf_alloc_bulk(g_mbuf_mp, (struct rte_mbuf **)&src_mbufs[src_mbuf_total], 1);
+			if (rc) {
+				SPDK_ERRLOG("ERROR trying to get an extra src_mbuf!\n");
+				goto error_src_dst;
+			}
+			src_mbuf_total++;
+			mbuf_index++;
+			current_src_base += updated_length;
+			phys_addr = spdk_vtophys((void *)current_src_base, &remainder);
+			/* assert we don't cross another */
+			assert(remainder == src_iovs[iov_index].iov_len - updated_length);
+
+			rte_pktmbuf_attach_extbuf(src_mbufs[mbuf_index],
+						  current_src_base,
+						  phys_addr,
+						  remainder,
+						  &g_shinfo);
+			rte_pktmbuf_append(src_mbufs[mbuf_index], remainder);
+			rte_pktmbuf_chain(src_mbufs[0], src_mbufs[mbuf_index]);
+		}
+		iov_index++;
+		mbuf_index++;
 	}
 
 	comp_op->m_src = src_mbufs[0];
@@ -501,21 +536,51 @@ _compress_operation(struct spdk_reduce_backing_dev *backing_dev, struct iovec *s
 	comp_op->src.length = total_length;
 
 	/* setup dst mbufs, for the current test being used with this code there's only one vector */
-	for (iov_index = 0; iov_index < dst_iovcnt; iov_index++) {
+	iov_index = mbuf_index = 0;
+	while (iov_index < dst_iovcnt) {
 
-		current_dst_iov = dst_iovs[iov_index].iov_base;
+		current_dst_base = dst_iovs[iov_index].iov_base;
+		updated_length = dst_iovs[iov_index].iov_len;
+		phys_addr = spdk_vtophys((void *)current_dst_base, &updated_length);
 
-		rte_pktmbuf_attach_extbuf(dst_mbufs[iov_index],
-					  current_dst_iov,
-					  spdk_vtophys((void *)current_dst_iov, NULL),
-					  dst_iovs[iov_index].iov_len,
+		rte_pktmbuf_attach_extbuf(dst_mbufs[mbuf_index],
+					  current_dst_base,
+					  phys_addr,
+					  updated_length,
 					  &g_shinfo);
-		rte_pktmbuf_append(dst_mbufs[iov_index], dst_iovs[iov_index].iov_len);
+		rte_pktmbuf_append(dst_mbufs[mbuf_index], updated_length);
+		remainder = dst_iovs[iov_index].iov_len - updated_length;
 
-		if (iov_index > 0) {
-			rte_pktmbuf_chain(dst_mbufs[0], dst_mbufs[iov_index]);
+		if (mbuf_index > 0) {
+			rte_pktmbuf_chain(dst_mbufs[0], dst_mbufs[mbuf_index]);
 		}
+
+		/* If we crossed 2 2MB boundary we need another mbuf for the remainder */
+		if (remainder > 0) {
+			rc = rte_pktmbuf_alloc_bulk(g_mbuf_mp, (struct rte_mbuf **)&dst_mbufs[dst_mbuf_total], 1);
+			if (rc) {
+				SPDK_ERRLOG("ERROR trying to get an extra dst_mbuf!\n");
+				goto error_src_dst;
+			}
+			dst_mbuf_total++;
+			mbuf_index++;
+			current_dst_base += updated_length;
+			phys_addr = spdk_vtophys((void *)current_dst_base, &remainder);
+			/* assert we don't cross another */
+			assert(remainder == dst_iovs[iov_index].iov_len - updated_length);
+
+			rte_pktmbuf_attach_extbuf(dst_mbufs[mbuf_index],
+						  current_dst_base,
+						  phys_addr,
+						  remainder,
+						  &g_shinfo);
+			rte_pktmbuf_append(dst_mbufs[mbuf_index], remainder);
+			rte_pktmbuf_chain(dst_mbufs[0], dst_mbufs[mbuf_index]);
+		}
+		iov_index++;
+		mbuf_index++;
 	}
+
 	comp_op->m_dst = dst_mbufs[0];
 	comp_op->dst.offset = 0;
 
@@ -542,8 +607,12 @@ _compress_operation(struct spdk_reduce_backing_dev *backing_dev, struct iovec *s
 	}
 
 	/* Error cleanup paths. */
+error_src_dst:
+	for (i = 0; i < dst_mbuf_total; i++) {
+		rte_pktmbuf_free((struct rte_mbuf *)&dst_mbufs[i]);
+	}
 error_get_dst:
-	for (i = 0; i < src_iovcnt; i++) {
+	for (i = 0; i < src_mbuf_total; i++) {
 		rte_pktmbuf_free((struct rte_mbuf *)&src_mbufs[i]);
 	}
 error_get_src:
@@ -1074,6 +1143,7 @@ bdev_hotremove_vol_unload_cb(void *cb_arg, int reduce_errno)
 		SPDK_ERRLOG("number %d\n", reduce_errno);
 	}
 
+	comp_bdev->vol = NULL;
 	spdk_bdev_unregister(&comp_bdev->comp_bdev, NULL, NULL);
 }