build: fix missing dependency for env_dpdk

Signed-off-by: Jacek Kalwas <jacek.kalwas@intel.com>
Change-Id: I414a7bdb63990df6ae411b8b03f055b9ffecd41f
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9014
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

CHANGELOG.md

@@ -1,6 +1,193 @@
 # Changelog
 
-## v21.04: (Upcoming Release)
+## v21.07.1: (Upcoming Release)
+
+## v21.07: Kernel DSA, Init lib, Userspace DTrace
+
+### accel_fw
+
+Added API `spdk_accel_submit_copy_crc32c` to perform a CRC32C while copying data.
+
+Added API `spdk_accel_batch_prep_copy_crc32c` to batch CRC32C + copy commands.
+
+Added API `spdk_accel_submit_copy_crc32cv` to submit chained CRC32C + copy commands.
+
+### bdev
+
+Change `spdk_bdev_read_blocks_with_md` arg offset definition from int64_t to uint64_t.
+
+Red-black tree has been used to organize the bdev names and aliases uniformly
+to provide faster lookup.
+
+Removed ZCOPY emulation: The bdev module can be checked to see if it supports ZCOPY
+and if not supported then use existing READ/WRITE commands.
+
+Added iov to spdk_bdev_zcopy_start.
+
+### dpdk
+
+Updated DPDK submodule to DPDK 21.05.
+
+### idxd
+
+Remove the `probe_cb` parameter in `spdk_idxd_probe` function. And remove the definition
+of `spdk_idxd_probe_cb` function pointer. It should be implemented in idxd_user.c.
+
+Added API `spdk_idxd_submit_copy_crc32c` to perform a CRC32C while copying data.
+
+Added API `spdk_idxd_batch_prep_copy_crc32c` to prepare a batch operation to perform
+a CRC32C while copying data.
+
+We have userspace idxd driver under accel_engine library (module/accel/idxd/accel_engine).
+And kernel idxd driver usage is also added under the accel_engine library. This approach
+implemented a wrapper library to use IDXD device by leveraging the kernel DSA driver in
+SPDK idxd library (lib/idxd). Then users can leverage the RPC later to configure how to
+use the DSA device by user space driver or kernel driver.
+
+### init
+
+Added new `init` library that initializes the SPDK subsystems, which previously was
+internal to application framework. That functionality has been made public and for
+use by applications which don't leverage SPDK's application framework.
+
+### iscsi
+
+New parameters, `pdu_pool_size`, `immediate_data_pool_size`, and `data_out_pool_size`,
+were added to the RPC `iscsi_set_options` to run iSCSI target with varying amount of
+available memory.
+
+### json
+
+Added API `spdk_json_write_named_uint128` and `spdk_json_write_uint128` to perform
+the uint128 related data.
+
+### net
+
+Removed deprecated `net` library.
+
+Added support for ZCOPY.  ZCOPY is used in preference to READ and WRITE if the
+bdev module supports ZCOPY and the bdev module has ZCOPY enabled.
+
+### nvme
+
+`spdk_nvme_map_prps` and `spdk_nvme_map_cmd` were moved to nvmf/vfio-user as internal APIs
+as vfio-user is the only user for the above two APIs.
+
+Added a new function `spdk_nvme_ns_cmd_copy` to submit a Simple Copy Command to a Namespace.
+
+Update the `spdk_nvme_generic_command_status_code` structure with new status code
+according to the definition in NVMe 1.4 spec.
+
+`spdk_nvme_ctrlr_get_default_ctrlr_opts` now sets `use_cmb_sqs` to false. This means
+that if a controller has a CMB and supports SQs in the CMB, SPDK will not use
+the CMB for SQs by default - the user must set `use_cmb_sqs` to true in
+the `spdk_nvme_ctrlr_opts` structure prior to controller attach.
+
+Add a new function `spdk_nvme_detach_poll` to simplify a common use case to continue
+polling until all detachments complete.
+
+Added new argument `timeout_admin_us` to `spdk_nvme_ctrlr_register_timeout_callback` so callers
+can specify a different timeout for admin commands vs. io commands.
+
+An existing function `spdk_nvme_detach_async` was updated to add one or more detachments
+to an active context while it is being polled.
+
+Rename a variable in the member `cmic` of the struct `spdk_nvme_ctrlr_data` from
+`multi_host` to `multi_ctrlr`. The variable means that the NVM subsystem may have two or
+more controllers if set to 1. However `multi_host` had indicated a particular use case
+such that the NVM subsystem is used by multiple hosts.
+
+A new option `disable_read_ana_log_page` was added to struct `spdk_nvme_ctrlr_opts` to disable
+reading ANA log page. The upper layer is expected to read ANA log page instead if `true`.
+The default value is `false`.
+
+New APIs, `spdk_nvme_ctrlr_reset_async` and `spdk_nvme_ctrlr_reset_poll_async`, have been added to
+reset a controller asynchronously.
+
+New RPC `bdev_nvme_reset_controller` was added, to reset an NVMe controller.
+
+Added `spdk_nvme_ns_get_nguid` function to get NGUID for the given namespace.
+
+Added `spdk_nvme_ctrlr_is_fabrics` function to indicate whether a ctrlr handle
+is associated with a fabrics controller.
+
+### nvmf
+
+Added `min_cntlid` and `max_cntlid` to `nvmf_create_subsystem` to limit the controller ID range.
+Added `spdk_nvmf_subsystem_get_min_cntlid` and `spdk_nvmf_subsystem_get_max_cntlid` to request those values.
+
+`spdk_nvmf_request_get_buffers_multi` API is removed.
+
+Added the `nvmf_set_crdt` RPC for setting command retry delay times.
+
+Expanded `spdk_nvmf_poll_group_stat` with current qpair count statistics.
+
+Removed following deprecated APIs:
+- `spdk_nvmf_poll_group_get_stat` (function in `nvmf.h`),
+- `spdk_nvmf_transport_poll_group_get_stat` (function in `nvmf.h`),
+- `spdk_nvmf_transport_poll_group_free_stat`(function in `nvmf.h`),
+- `spdk_nvmf_rdma_device_stat` (struct in `nvmf.h`),
+- `spdk_nvmf_transport_poll_group_stat` (struct in `nvmf.h`),
+- `poll_group_get_stat` (transport op in `nvmf_transport.h`),
+- `poll_group_free_stat` (transport op in `nvmf_transport.h`).
+
+### rpc
+
+New RPC `bdev_rbd_register_cluster` and `bdev_rbd_unregister_cluster` was added, it allows to create
+and delete the rados object cluster, then users can choose the cluster to create related rbd
+device.
+
+Revised `bdev_rbd_create` parameter, it allows to use an optional parameter `--cluster-name`
+to create a rbd bdev with  an already registered Rados Cluster Object.
+
+New RPC `bdev_rbd_get_clusters_info` was added, it allows to get the info of the registered
+Rados Cluster names.
+
+New optional parameter, `timeout_admin_us`, added to the bdev_nvme_set_options RPC.
+
+Revised a parameter `--stripe-size_kb` to `--stripe-size-kb` of `bdev_raid_create` method
+provided in `scripts/rpc.py` for consistency.
+
+An new optional parameter `config_kernel_mode` was added to the RPC `idxd_scan_accel_engine`,
+and this is used to enable using the kernel mode IDXD driver.
+
+Removed deprecated parameter `enable-zerocopy-send` of RPC `sock_impl_set_options`,
+use `enable-zerocopy-send-server` or `enable-zerocopy-send-client` instead.
+Removed deprecated parameter `disable-zerocopy-send` of RPC `sock_impl_set_options`,
+use `disable-zerocopy-send-server` or `disable-zerocopy-send-client` instead.
+
+### rpm
+
+Removed deprecated `pkg/spdk.spec`, please use `rpmbuild/spdk.spec` instead.
+See [RPM documentation](https://spdk.io/doc/rpm.html) for more details.
+
+### thread
+
+Red-black tree has been used for timed pollers to provide faster insertion and deletion
+and for io_devices to provide faster lookup.
+
+Removed `spdk_io_channel` structure from public header and moved it to thread_internal.h.
+
+### trace
+
+Added `spdk_trace_register_description_ext` function to register variable number of tracepoint arguments.
+
+Added ability to chain multiple trace entries together to extend the size of the argument buffer.
+
+### util
+
+Red-black tree macros has been added by using the macros provided by the FreeBSD operating system
+under the same BSD license.
+
+Add an new macro `SPDK_SIZEOF_MEMBER` to get the size of a member of a struct.
+
+`spdk_crc32c_iov_update` function was added to support calculating the crc32c of the iovs.
+
+Added zipf random number generator with power law probability distribution.
+When applied to performance testing of block devices, it will select blocks over
+the full range of LBAs, but will more frequently select lower-numbered LBAs.
+
+## v21.04:
 
 ### accel
 
@@ -161,6 +348,16 @@ For `bdev_raid_create` RPC, the deprecated parameter `strip_size` was removed.
 New RPC `bdev_nvme_get_transport_statistics` was added, it allows to get transport statistics
 of nvme poll groups.
 
+Parameter `enable-zerocopy-send` of RPC `sock_impl_set_options` is deprecated and will be removed in SPDK 21.07,
+use `enable-zerocopy-send-server` or `enable-zerocopy-send-client` instead.
+Parameter `disable-zerocopy-send` of RPC `sock_impl_set_options` is deprecated and will be removed in SPDK 21.07,
+use `disable-zerocopy-send-server` or `disable-zerocopy-send-client` instead.
+
+Added cmd_parser.py used to parse the args from argparse. There are
+two benefit to use command parser:
+- Simplify the definition of rpc method. It will reduce the rpc method code.
+- Make the rpc call more versatile. User can add private args into rpc method.
+
 ### rpm
 
 Added support for new RPM spec, rpmbuild/spdk.spec, which can be used for packaging the
@@ -179,6 +376,12 @@ sockets to be marked using the SO_MARK socket option as a hint for which hardwar
 queue they should be associated with. This mode leverages that by setting the same
 value for all sockets within a poll group.
 
+New parameters `enable_zerocopy_send_server` and `enable_zerocopy_send_client` were added
+to struct spdk_sock_impl_opts, these parameters enable or disable zero copy send for server
+and client sockets which are created using `spdk_sock_listen` and `spdk_sock_listen_ext` (server);
+`spdk_sock_connect` and `spdk_sock_connect_ext` (client) functions. Existing parameter
+`enable_zerocopy_send`  enables or disables zero copy send for both server and client sockets.
+
 ### thread
 
 A new API `spdk_io_channel_get_io_device` was added to get the io_device for the specified

CONFIG

@@ -76,6 +76,9 @@ CONFIG_UNIT_TESTS=y
 # Build examples
 CONFIG_EXAMPLES=y
 
+# Build apps
+CONFIG_APPS=y
+
 # Build with Control-flow Enforcement Technology (CET)
 CONFIG_CET=n
 
@@ -86,6 +89,10 @@ CONFIG_ENV=
 # This directory should contain 'include' and 'lib' directories for your DPDK
 # installation.
 CONFIG_DPDK_DIR=
+# Automatically set via pkg-config when bare --with-dpdk is set
+CONFIG_DPDK_LIB_DIR=
+CONFIG_DPDK_INC_DIR=
+CONFIG_DPDK_PKG_CONFIG=n
 
 # This directory should contain 'include' and 'lib' directories for WPDK.
 CONFIG_WPDK_DIR=
@@ -171,4 +178,12 @@ CONFIG_FUSE=n
 CONFIG_RAID5=n
 
 # Build with IDXD support
+# In this mode, SPDK fully controls the DSA device.
 CONFIG_IDXD=n
+
+# Build with USDT support
+CONFIG_USDT=n
+
+# Build with IDXD kernel support.
+# In this mode, SPDK shares the DSA device with the kernel.
+CONFIG_IDXD_KERNEL=n

Makefile

@@ -40,8 +40,9 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 DIRS-y += lib
 DIRS-y += module
 DIRS-$(CONFIG_SHARED) += shared_lib
-DIRS-y += app include
+DIRS-y += include
 DIRS-$(CONFIG_EXAMPLES) += examples
+DIRS-$(CONFIG_APPS) += app
 DIRS-y += test
 DIRS-$(CONFIG_IPSEC_MB) += ipsecbuild
 DIRS-$(CONFIG_ISAL) += isalbuild
@@ -57,11 +58,13 @@ export MAKE_PID := $(shell echo $$PPID)
 ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV))
 ifeq ($(CURDIR)/dpdk/build,$(CONFIG_DPDK_DIR))
 ifneq ($(SKIP_DPDK_BUILD),1)
+ifneq ($(CONFIG_DPDK_PKG_CONFIG),y)
 DPDKBUILD = dpdkbuild
 DIRS-y += dpdkbuild
 endif
 endif
 endif
+endif
 
 ifeq ($(OS),Windows)
 ifeq ($(CURDIR)/wpdk/build,$(CONFIG_WPDK_DIR))

app/iscsi_tgt/Makefile

@@ -43,7 +43,7 @@ CFLAGS += -I$(SPDK_ROOT_DIR)/lib
 
 C_SRCS := iscsi_tgt.c
 
-SPDK_LIB_LIST = $(ALL_MODULES_LIST) event_iscsi event_net
+SPDK_LIB_LIST = $(ALL_MODULES_LIST) event event_iscsi
 
 ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV))
 SPDK_LIB_LIST += env_dpdk_rpc

app/iscsi_tgt/iscsi_tgt.c

@@ -37,7 +37,6 @@
 #include "spdk/event.h"
 #include "iscsi/iscsi.h"
 #include "spdk/log.h"
-#include "spdk/net.h"
 
 static int g_daemon_mode = 0;
 

app/nvmf_tgt/Makefile

@@ -39,7 +39,7 @@ APP = nvmf_tgt
 
 C_SRCS := nvmf_main.c
 
-SPDK_LIB_LIST = $(ALL_MODULES_LIST) event_nvmf
+SPDK_LIB_LIST = $(ALL_MODULES_LIST) event event_nvmf
 
 ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV))
 SPDK_LIB_LIST += env_dpdk_rpc

app/spdk_dd/Makefile

@@ -39,6 +39,6 @@ APP = spdk_dd
 
 C_SRCS := spdk_dd.c
 
-SPDK_LIB_LIST = $(ALL_MODULES_LIST) event_bdev
+SPDK_LIB_LIST = $(ALL_MODULES_LIST) event event_bdev
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

app/spdk_tgt/Makefile

@@ -41,7 +41,7 @@ C_SRCS := spdk_tgt.c
 
 SPDK_LIB_LIST = $(ALL_MODULES_LIST)
 
-SPDK_LIB_LIST += event_iscsi event_nvmf
+SPDK_LIB_LIST += event event_iscsi event_nvmf
 
 ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV))
 SPDK_LIB_LIST += env_dpdk_rpc

app/spdk_top/Makefile

@@ -39,6 +39,6 @@ APP = spdk_top
 C_SRCS := spdk_top.c
 
 SPDK_LIB_LIST = rpc
-LIBS=-lncurses -lpanel -lmenu
+LIBS=-lpanel -lmenu -lncurses
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

app/trace/Makefile

@@ -38,6 +38,8 @@ include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
 APP = spdk_trace
 SPDK_NO_LINK_ENV = 1
 
+SPDK_LIB_LIST += json
+
 CXX_SRCS := trace.cpp
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app_cxx.mk

app/trace/trace.cpp

@@ -32,6 +32,11 @@
  */
 
 #include "spdk/stdinc.h"
+#include "spdk/env.h"
+#include "spdk/json.h"
+#include "spdk/likely.h"
+#include "spdk/string.h"
+#include "spdk/util.h"
 
 #include <map>
 
@@ -41,8 +46,29 @@ extern "C" {
 }
 
 static struct spdk_trace_histories *g_histories;
+static struct spdk_json_write_ctx *g_json;
 static bool g_print_tsc = false;
 
+/* This is a bit ugly, but we don't want to include env_dpdk in the app, while spdk_util, which we
+ * do need, uses some of the functions implemented there.  We're not actually using the functions
+ * that depend on those, so just define them as no-ops to allow the app to link.
+ */
+extern "C" {
+	void *
+	spdk_realloc(void *buf, size_t size, size_t align)
+	{
+		assert(false);
+
+		return NULL;
+	}
+
+	void
+	spdk_free(void *buf)
+	{
+		assert(false);
+	}
+} /* extern "C" */
+
 static void usage(void);
 
 struct entry_key {
@@ -79,10 +105,28 @@ struct object_stats {
 	object_stats() : start(), index(), size(), tpoint_id(), counter(0) {}
 };
 
+struct argument_context {
+	struct spdk_trace_entry		*entry;
+	struct spdk_trace_entry_buffer	*buffer;
+	uint16_t			lcore;
+	size_t				offset;
+
+	argument_context(struct spdk_trace_entry *entry,
+			 uint16_t lcore) : entry(entry), lcore(lcore)
+	{
+		buffer = (struct spdk_trace_entry_buffer *)entry;
+
+		/* The first argument resides within the spdk_trace_entry structure, so the initial
+		 * offset needs to be adjusted to the start of the spdk_trace_entry.args array
+		 */
+		offset = offsetof(struct spdk_trace_entry, args) -
+			 offsetof(struct spdk_trace_entry_buffer, data);
+	}
+};
+
 struct object_stats g_stats[SPDK_TRACE_MAX_OBJECT];
 
 static char *g_exe_name;
-static int g_verbose = 1;
 
 static uint64_t g_tsc_rate;
 static uint64_t g_first_tsc = 0x0;
@@ -93,10 +137,19 @@ get_us_from_tsc(uint64_t tsc, uint64_t tsc_rate)
 	return ((float)tsc) * 1000 * 1000 / tsc_rate;
 }
 
+static const char *
+format_argname(const char *name)
+{
+	static char namebuf[16];
+
+	snprintf(namebuf, sizeof(namebuf), "%s: ", name);
+	return namebuf;
+}
+
 static void
 print_ptr(const char *arg_string, uint64_t arg)
 {
-	printf("%-7.7s0x%-14jx ", arg_string, arg);
+	printf("%-7.7s0x%-14jx ", format_argname(arg_string), arg);
 }
 
 static void
@@ -107,14 +160,13 @@ print_uint64(const char *arg_string, uint64_t arg)
 	 *  for FLUSH WRITEBUF when writev() returns -1 due to full
 	 *  socket buffer.
 	 */
-	printf("%-7.7s%-16jd ", arg_string, arg);
+	printf("%-7.7s%-16jd ", format_argname(arg_string), arg);
 }
 
 static void
-print_string(const char *arg_string, uint64_t arg)
+print_string(const char *arg_string, const char *arg)
 {
-	char *str = (char *)&arg;
-	printf("%-7.7s%.8s ", arg_string, str);
+	printf("%-7.7s%-16.16s ", format_argname(arg_string), arg);
 }
 
 static void
@@ -136,26 +188,106 @@ print_object_id(uint8_t type, uint64_t id)
 static void
 print_float(const char *arg_string, float arg)
 {
-	printf("%-7s%-16.3f ", arg_string, arg);
+	printf("%-7s%-16.3f ", format_argname(arg_string), arg);
+}
+
+static struct spdk_trace_entry_buffer *
+get_next_buffer(struct spdk_trace_entry_buffer *buf, uint16_t lcore)
+{
+	struct spdk_trace_history *history;
+
+	history = spdk_get_per_lcore_history(g_histories, lcore);
+	assert(history);
+
+	if (spdk_unlikely((void *)buf == &history->entries[history->num_entries - 1])) {
+		return (struct spdk_trace_entry_buffer *)&history->entries[0];
+	} else {
+		return buf + 1;
+	}
+}
+
+static bool
+build_arg(struct argument_context *argctx, const struct spdk_trace_argument *arg,
+	  void *buf, size_t bufsize)
+{
+	struct spdk_trace_entry *entry = argctx->entry;
+	struct spdk_trace_entry_buffer *buffer = argctx->buffer;
+	size_t curlen, argoff;
+
+	argoff = 0;
+	while (argoff < arg->size) {
+		if (argctx->offset == sizeof(buffer->data)) {
+			buffer = get_next_buffer(buffer, argctx->lcore);
+			if (spdk_unlikely(buffer->tpoint_id != SPDK_TRACE_MAX_TPOINT_ID ||
+					  buffer->tsc != entry->tsc)) {
+				return false;
+			}
+
+			argctx->offset = 0;
+			argctx->buffer = buffer;
+		}
+
+		curlen = spdk_min(sizeof(buffer->data) - argctx->offset, arg->size - argoff);
+		if (argoff < bufsize) {
+			memcpy((uint8_t *)buf + argoff, &buffer->data[argctx->offset],
+			       spdk_min(curlen, bufsize - argoff));
+		}
+
+		argctx->offset += curlen;
+		argoff += curlen;
+	}
+
+	return true;
 }
 
 static void
-print_arg(uint8_t arg_type, const char *arg_string, uint64_t arg)
+print_arg(struct argument_context *argctx, const struct spdk_trace_argument *arg)
 {
-	if (arg_string[0] == 0) {
-		printf("%24s", "");
-		return;
-	}
+	uint64_t value = 0;
+	char strbuf[256];
 
-	switch (arg_type) {
+	switch (arg->type) {
 	case SPDK_TRACE_ARG_TYPE_PTR:
-		print_ptr(arg_string, arg);
+		if (build_arg(argctx, arg, &value, sizeof(value))) {
+			print_ptr(arg->name, value);
+		}
 		break;
 	case SPDK_TRACE_ARG_TYPE_INT:
-		print_uint64(arg_string, arg);
+		if (build_arg(argctx, arg, &value, sizeof(value))) {
+			print_uint64(arg->name, value);
+		}
 		break;
 	case SPDK_TRACE_ARG_TYPE_STR:
-		print_string(arg_string, arg);
+		assert((size_t)arg->size <= sizeof(strbuf));
+		if (build_arg(argctx, arg, strbuf, sizeof(strbuf))) {
+			print_string(arg->name, strbuf);
+		}
+		break;
+	}
+}
+
+static void
+print_arg_json(struct argument_context *argctx, const struct spdk_trace_argument *arg)
+{
+	uint64_t value = 0;
+	char strbuf[256];
+
+	switch (arg->type) {
+	case SPDK_TRACE_ARG_TYPE_PTR:
+	case SPDK_TRACE_ARG_TYPE_INT:
+		if (build_arg(argctx, arg, &value, sizeof(value))) {
+			spdk_json_write_uint64(g_json, value);
+		} else {
+			spdk_json_write_null(g_json);
+		}
+		break;
+	case SPDK_TRACE_ARG_TYPE_STR:
+		assert((size_t)arg->size <= sizeof(strbuf));
+		if (build_arg(argctx, arg, strbuf, sizeof(strbuf))) {
+			spdk_json_write_string(g_json, strbuf);
+		} else {
+			spdk_json_write_null(g_json);
+		}
 		break;
 	}
 }
@@ -167,17 +299,10 @@ print_event(struct spdk_trace_entry *e, uint64_t tsc_rate,
 	struct spdk_trace_tpoint	*d;
 	struct object_stats		*stats;
 	float				us;
+	size_t				i;
 
 	d = &g_histories->flags.tpoint[e->tpoint_id];
 	stats = &g_stats[d->object_type];
-
-	if (d->new_object) {
-		stats->index[e->object_id] = stats->counter++;
-		stats->tpoint_id[e->object_id] = e->tpoint_id;
-		stats->start[e->object_id] = e->tsc;
-		stats->size[e->object_id] = e->size;
-	}
-
 	us = get_us_from_tsc(e->tsc - tsc_offset, tsc_rate);
 
 	printf("%2d: %10.3f ", lcore, us);
@@ -193,7 +318,6 @@ print_event(struct spdk_trace_entry *e, uint64_t tsc_rate,
 	printf("%-*s ", (int)sizeof(d->name), d->name);
 	print_size(e->size);
 
-	print_arg(d->arg1_type, d->arg1_name, e->arg1);
 	if (d->new_object) {
 		print_object_id(d->object_type, stats->index[e->object_id]);
 	} else if (d->object_type != OBJECT_NONE) {
@@ -201,22 +325,100 @@ print_event(struct spdk_trace_entry *e, uint64_t tsc_rate,
 			us = get_us_from_tsc(e->tsc - stats->start[e->object_id],
 					     tsc_rate);
 			print_object_id(d->object_type, stats->index[e->object_id]);
-			print_float("time:", us);
+			print_float("time", us);
 		} else {
 			printf("id:    N/A");
 		}
 	} else if (e->object_id != 0) {
-		print_arg(SPDK_TRACE_ARG_TYPE_PTR, "object: ", e->object_id);
+		print_ptr("object", e->object_id);
+	}
+
+	struct argument_context argctx(e, lcore);
+	for (i = 0; i < d->num_args; ++i) {
+		print_arg(&argctx, &d->args[i]);
 	}
 	printf("\n");
 }
 
+static void
+print_event_json(struct spdk_trace_entry *e, uint64_t tsc_rate,
+		 uint64_t tsc_offset, uint16_t lcore)
+{
+	struct spdk_trace_tpoint *d;
+	struct object_stats *stats;
+	size_t i;
+
+	d = &g_histories->flags.tpoint[e->tpoint_id];
+	stats = &g_stats[d->object_type];
+
+	spdk_json_write_object_begin(g_json);
+	spdk_json_write_named_uint64(g_json, "lcore", lcore);
+	spdk_json_write_named_uint64(g_json, "tpoint", e->tpoint_id);
+	spdk_json_write_named_uint64(g_json, "tsc", e->tsc);
+
+	if (g_histories->flags.owner[d->owner_type].id_prefix) {
+		spdk_json_write_named_string_fmt(g_json, "poller", "%c%02d",
+						 g_histories->flags.owner[d->owner_type].id_prefix,
+						 e->poller_id);
+	}
+	if (e->size != 0) {
+		spdk_json_write_named_uint32(g_json, "size", e->size);
+	}
+	if (d->new_object || d->object_type != OBJECT_NONE || e->object_id != 0) {
+		char object_type;
+
+		spdk_json_write_named_object_begin(g_json, "object");
+		if (d->new_object) {
+			object_type =  g_histories->flags.object[d->object_type].id_prefix;
+			spdk_json_write_named_string_fmt(g_json, "id", "%c%" PRIu64, object_type,
+							 stats->index[e->object_id]);
+		} else if (d->object_type != OBJECT_NONE) {
+			object_type =  g_histories->flags.object[d->object_type].id_prefix;
+			if (stats->start.find(e->object_id) != stats->start.end()) {
+				spdk_json_write_named_string_fmt(g_json, "id", "%c%" PRIu64,
+								 object_type,
+								 stats->index[e->object_id]);
+				spdk_json_write_named_uint64(g_json, "time",
+							     e->tsc - stats->start[e->object_id]);
+			}
+		}
+		spdk_json_write_named_uint64(g_json, "value", e->object_id);
+		spdk_json_write_object_end(g_json);
+	}
+	if (d->num_args > 0) {
+		struct argument_context argctx(e, lcore);
+
+		spdk_json_write_named_array_begin(g_json, "args");
+		for (i = 0; i < d->num_args; ++i) {
+			print_arg_json(&argctx, &d->args[i]);
+		}
+		spdk_json_write_array_end(g_json);
+	}
+
+	spdk_json_write_object_end(g_json);
+}
+
 static void
 process_event(struct spdk_trace_entry *e, uint64_t tsc_rate,
 	      uint64_t tsc_offset, uint16_t lcore)
 {
-	if (g_verbose) {
+	struct spdk_trace_tpoint	*d;
+	struct object_stats		*stats;
+
+	d = &g_histories->flags.tpoint[e->tpoint_id];
+	stats = &g_stats[d->object_type];
+
+	if (d->new_object) {
+		stats->index[e->object_id] = stats->counter++;
+		stats->tpoint_id[e->object_id] = e->tpoint_id;
+		stats->start[e->object_id] = e->tsc;
+		stats->size[e->object_id] = e->size;
+	}
+
+	if (g_json == NULL) {
 		print_event(e, tsc_rate, tsc_offset, lcore);
+	} else {
+		print_event_json(e, tsc_rate, tsc_offset, lcore);
 	}
 }
 
@@ -263,7 +465,9 @@ populate_events(struct spdk_trace_history *history, int num_entries)
 
 	i = first;
 	while (1) {
-		g_entry_map[entry_key(lcore, e[i].tsc)] = &e[i];
+		if (e[i].tpoint_id != SPDK_TRACE_MAX_TPOINT_ID) {
+			g_entry_map[entry_key(lcore, e[i].tsc)] = &e[i];
+		}
 		if (i == last) {
 			break;
 		}
@@ -276,11 +480,68 @@ populate_events(struct spdk_trace_history *history, int num_entries)
 	return (0);
 }
 
+static void
+print_tpoint_definitions(void)
+{
+	struct spdk_trace_tpoint *tpoint;
+	size_t i, j;
+
+	/* We only care about these when printing JSON */
+	if (!g_json) {
+		return;
+	}
+
+	spdk_json_write_named_uint64(g_json, "tsc_rate", g_tsc_rate);
+	spdk_json_write_named_array_begin(g_json, "tpoints");
+
+	for (i = 0; i < SPDK_COUNTOF(g_histories->flags.tpoint); ++i) {
+		tpoint = &g_histories->flags.tpoint[i];
+		if (tpoint->tpoint_id == 0) {
+			continue;
+		}
+
+		spdk_json_write_object_begin(g_json);
+		spdk_json_write_named_string(g_json, "name", tpoint->name);
+		spdk_json_write_named_uint32(g_json, "id", tpoint->tpoint_id);
+		spdk_json_write_named_bool(g_json, "new_object", tpoint->new_object);
+
+		spdk_json_write_named_array_begin(g_json, "args");
+		for (j = 0; j < tpoint->num_args; ++j) {
+			spdk_json_write_object_begin(g_json);
+			spdk_json_write_named_string(g_json, "name", tpoint->args[j].name);
+			spdk_json_write_named_uint32(g_json, "type", tpoint->args[j].type);
+			spdk_json_write_named_uint32(g_json, "size", tpoint->args[j].size);
+			spdk_json_write_object_end(g_json);
+		}
+		spdk_json_write_array_end(g_json);
+		spdk_json_write_object_end(g_json);
+	}
+
+	spdk_json_write_array_end(g_json);
+}
+
+static int
+print_json(void *cb_ctx, const void *data, size_t size)
+{
+	ssize_t rc;
+
+	while (size > 0) {
+		rc = write(STDOUT_FILENO, data, size);
+		if (rc < 0) {
+			fprintf(stderr, "%s: %s\n", g_exe_name, spdk_strerror(errno));
+			abort();
+		}
+
+		size -= rc;
+	}
+
+	return 0;
+}
+
 static void usage(void)
 {
 	fprintf(stderr, "usage:\n");
 	fprintf(stderr, "   %s <option> <lcore#>\n", g_exe_name);
-	fprintf(stderr, "        option = '-q' to disable verbose mode\n");
 	fprintf(stderr, "                 '-c' to display single lcore history\n");
 	fprintf(stderr, "                 '-t' to display TSC offset for each event\n");
 	fprintf(stderr, "                 '-s' to specify spdk_trace shm name for a\n");
@@ -291,6 +552,7 @@ static void usage(void)
 	fprintf(stderr, "                       -i or -p must be specified)\n");
 	fprintf(stderr, "                 '-f' to specify a tracepoint file name\n");
 	fprintf(stderr, "                      (-s and -f are mutually exclusive)\n");
+	fprintf(stderr, "                 '-j' to use JSON to format the output\n");
 }
 
 int main(int argc, char **argv)
@@ -307,9 +569,10 @@ int main(int argc, char **argv)
 	int			shm_id = -1, shm_pid = -1;
 	uint64_t		trace_histories_size;
 	struct stat		_stat;
+	bool			json = false;
 
 	g_exe_name = argv[0];
-	while ((op = getopt(argc, argv, "c:f:i:p:qs:t")) != -1) {
+	while ((op = getopt(argc, argv, "c:f:i:jp:s:t")) != -1) {
 		switch (op) {
 		case 'c':
 			lcore = atoi(optarg);
@@ -326,9 +589,6 @@ int main(int argc, char **argv)
 		case 'p':
 			shm_pid = atoi(optarg);
 			break;
-		case 'q':
-			g_verbose = 0;
-			break;
 		case 's':
 			app_name = optarg;
 			break;
@@ -338,6 +598,9 @@ int main(int argc, char **argv)
 		case 't':
 			g_print_tsc = true;
 			break;
+		case 'j':
+			json = true;
+			break;
 		default:
 			usage();
 			exit(1);
@@ -356,6 +619,14 @@ int main(int argc, char **argv)
 		exit(1);
 	}
 
+	if (json) {
+		g_json = spdk_json_write_begin(print_json, NULL, 0);
+		if (g_json == NULL) {
+			fprintf(stderr, "Failed to allocate JSON write context\n");
+			exit(1);
+		}
+	}
+
 	if (file_name) {
 		fd = open(file_name, O_RDONLY);
 	} else {
@@ -402,7 +673,7 @@ int main(int argc, char **argv)
 		exit(-1);
 	}
 
-	if (g_verbose) {
+	if (!g_json) {
 		printf("TSC Rate: %ju\n", g_tsc_rate);
 	}
 
@@ -430,7 +701,7 @@ int main(int argc, char **argv)
 				continue;
 			}
 
-			if (g_verbose && history->num_entries) {
+			if (!g_json && history->num_entries) {
 				printf("Trace Size of lcore (%d): %ju\n", i, history->num_entries);
 			}
 
@@ -439,7 +710,7 @@ int main(int argc, char **argv)
 	} else {
 		history = spdk_get_per_lcore_history(g_histories, lcore);
 		if (history->num_entries > 0 && history->entries[0].tsc != 0) {
-			if (g_verbose && history->num_entries) {
+			if (!g_json && history->num_entries) {
 				printf("Trace Size of lcore (%d): %ju\n", lcore, history->num_entries);
 			}
 
@@ -447,6 +718,12 @@ int main(int argc, char **argv)
 		}
 	}
 
+	if (g_json != NULL) {
+		spdk_json_write_object_begin(g_json);
+		print_tpoint_definitions();
+		spdk_json_write_named_array_begin(g_json, "entries");
+	}
+
 	tsc_offset = g_first_tsc;
 	for (entry_map::iterator it = g_entry_map.begin(); it != g_entry_map.end(); it++) {
 		if (it->first.tsc < g_first_tsc) {
@@ -455,6 +732,12 @@ int main(int argc, char **argv)
 		process_event(it->second, g_tsc_rate, tsc_offset, it->first.lcore);
 	}
 
+	if (g_json != NULL) {
+		spdk_json_write_array_end(g_json);
+		spdk_json_write_object_end(g_json);
+		spdk_json_write_end(g_json);
+	}
+
 	munmap(history_ptr, trace_histories_size);
 	close(fd);
 

app/vhost/Makefile

@@ -39,7 +39,7 @@ APP = vhost
 
 C_SRCS := vhost.c
 
-SPDK_LIB_LIST = $(ALL_MODULES_LIST) event_vhost event_nbd
+SPDK_LIB_LIST = $(ALL_MODULES_LIST) event event_vhost event_nbd
 
 ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV))
 SPDK_LIB_LIST += env_dpdk_rpc

autobuild.sh

@@ -14,6 +14,10 @@ source "$1"
 source "$rootdir/test/common/autotest_common.sh"
 source "$rootdir/scripts/common.sh"
 
+if [[ -n $EXTERNAL_MAKE_HUGEMEM ]]; then
+	export EXTERNAL_MAKE_HUGEMEM
+fi
+
 out=$output_dir
 if [ -n "$SPDK_TEST_NATIVE_DPDK" ]; then
 	scanbuild_exclude=" --exclude $(dirname $SPDK_RUN_EXTERNAL_DPDK)"
@@ -51,7 +55,10 @@ function ocf_precompile() {
 function build_native_dpdk() {
 	local external_dpdk_dir
 	local external_dpdk_base_dir
+	local gcc_version
 
+	gcc_version=$(gcc -dumpversion)
+	gcc_version=${gcc_version//./}
 	external_dpdk_dir="$SPDK_RUN_EXTERNAL_DPDK"
 	external_dpdk_base_dir="$(dirname $external_dpdk_dir)"
 
@@ -65,9 +72,17 @@ function build_native_dpdk() {
 	git clone --branch $SPDK_TEST_NATIVE_DPDK --depth 1 http://dpdk.org/git/dpdk "$external_dpdk_base_dir"
 	git -C "$external_dpdk_base_dir" log --oneline -n 5
 
-	dpdk_cflags="-fPIC -g -Werror -fcommon"
+	dpdk_cflags="-fPIC -g -fcommon"
 	dpdk_ldflags=""
 
+	if [[ $gcc_version -ge 5 ]]; then
+		dpdk_cflags+=" -Werror"
+	fi
+
+	if [[ $gcc_version -ge 10 ]]; then
+		dpdk_cflags+=" -Wno-stringop-overflow"
+	fi
+
 	# the drivers we use
 	# net/i40e driver is not really needed by us, but it's built as a workaround
 	# for DPDK issue: https://bugs.dpdk.org/show_bug.cgi?id=576
@@ -113,7 +128,6 @@ function build_native_dpdk() {
 
 	cd $external_dpdk_base_dir
 	if [ "$(uname -s)" = "Linux" ]; then
-		dpdk_cflags+=" -Wno-stringop-overflow"
 		# Fix for freeing device if not kernel driver configured.
 		# TODO: Remove once this is merged in upstream DPDK
 		if grep "20.08.0" $external_dpdk_base_dir/VERSION; then
@@ -122,13 +136,17 @@ function build_native_dpdk() {
 		elif grep "20.11\|21.02" $external_dpdk_base_dir/VERSION; then
 			wget https://github.com/karlatec/dpdk/commit/3219c0cfc38803aec10c809dde16e013b370bda9.patch -O dpdk-pci.patch
 			wget https://github.com/karlatec/dpdk/commit/adf8f7638de29bc4bf9ba3faf12bbdae73acda0c.patch -O dpdk-qat.patch
+		elif grep "21.05" $external_dpdk_base_dir/VERSION; then
+			wget https://github.com/karlatec/dpdk/commit/f95e331be3a1f856b816948990dd2afc67ea4020.patch -O dpdk-pci.patch
+			wget https://github.com/karlatec/dpdk/commit/6fd2fa906ffdcee04e6ce5da40e61cb841be9827.patch -O dpdk-qat.patch
 		else
-			wget https://github.com/karlatec/dpdk/commit/eac05db0580091ef8e4d338aa5d2210695521894.patch -O dpdk-pci.patch
-			wget https://github.com/karlatec/dpdk/commit/d649d5efb7bb404ce59dea81768adeb994b284f7.patch -O dpdk-qat.patch
+			wget https://github.com/karlatec/dpdk/commit/6fd2fa906ffdcee04e6ce5da40e61cb841be9827.patch -O dpdk-qat.patch
 		fi
 		git config --local user.name "spdk"
 		git config --local user.email "nomail@all.com"
-		git am dpdk-pci.patch
+		if [[ -f dpdk-pci.patch ]]; then
+			git am dpdk-pci.patch
+		fi
 		git am dpdk-qat.patch
 	fi
 
@@ -158,6 +176,7 @@ function make_fail_cleanup() {
 
 function scanbuild_make() {
 	pass=true
+	"$rootdir/configure" $config_params --without-shared
 	$scanbuild $MAKE $MAKEFLAGS > $out/build_output.txt && rm -rf $out/scan-build-tmp || make_fail_cleanup
 	xtrace_disable
 
@@ -255,12 +274,9 @@ function build_doc() {
 function autobuild_test_suite() {
 	run_test "autobuild_check_format" ./scripts/check_format.sh
 	run_test "autobuild_external_code" sudo -E --preserve-env=PATH LD_LIBRARY_PATH=$LD_LIBRARY_PATH $rootdir/test/external_code/test_make.sh $rootdir
-	if [ "$SPDK_TEST_OCF" -eq 1 ]; then
-		run_test "autobuild_ocf_precompile" ocf_precompile
-	fi
 	run_test "autobuild_check_so_deps" $rootdir/test/make/check_so_deps.sh $1
 	./configure $config_params --without-shared
-	run_test "scanbuild_make" scanbuild_make
+	$MAKE $MAKEFLAGS
 	run_test "autobuild_generated_files_check" porcelain_check
 	run_test "autobuild_header_dependency_check" header_dependency_check
 	run_test "autobuild_make_install" $MAKE $MAKEFLAGS install DESTDIR="$SPDK_WORKSPACE" prefix=/usr
@@ -268,6 +284,11 @@ function autobuild_test_suite() {
 	run_test "autobuild_build_doc" build_doc
 }
 
+function unittest_build() {
+	"$rootdir/configure" $config_params --without-shared
+	$MAKE $MAKEFLAGS
+}
+
 if [ $SPDK_RUN_VALGRIND -eq 1 ]; then
 	run_test "valgrind" echo "using valgrind"
 fi
@@ -291,12 +312,17 @@ $MAKE cc_version
 $MAKE cxx_version
 echo "** END ** Info for Hostname: $HOSTNAME"
 
-if [ "$SPDK_TEST_AUTOBUILD" -eq 1 ]; then
+if [[ $SPDK_TEST_OCF -eq 1 ]]; then
+	run_test "autobuild_ocf_precompile" ocf_precompile
+fi
+
+if [[ $SPDK_TEST_AUTOBUILD -eq 1 ]]; then
 	run_test "autobuild" autobuild_test_suite $1
+elif [[ $SPDK_TEST_UNITTEST -eq 1 ]]; then
+	run_test "unittest_build" unittest_build
+elif [[ $SPDK_TEST_SCANBUILD -eq 1 ]]; then
+	run_test "scanbuild_make" scanbuild_make
 else
-	if [ "$SPDK_TEST_OCF" -eq 1 ]; then
-		run_test "autobuild_ocf_precompile" ocf_precompile
-	fi
 	# if we aren't testing the unittests, build with shared objects.
 	./configure $config_params --with-shared
 	run_test "make" $MAKE $MAKEFLAGS

autopackage.sh

@@ -11,6 +11,7 @@ fi
 source "$1"
 
 rootdir=$(readlink -f $(dirname $0))
+testdir=$rootdir # to get the storage space for tests
 source "$rootdir/test/common/autotest_common.sh"
 
 function build_rpms() (
@@ -20,7 +21,7 @@ function build_rpms() (
 	unset -v LD_LIBRARY_PATH
 
 	install_uninstall_rpms() {
-		rpms=("$HOME/rpmbuild/RPMS/x86_64/"spdk{,-devel,{,-dpdk}-libs}-$version-1.x86_64.rpm)
+		rpms=("$builddir/rpm/x86_64/"spdk{,-devel,{,-dpdk}-libs}-$version-1.x86_64.rpm)
 
 		sudo rpm -i "${rpms[@]}"
 		rpms=("${rpms[@]##*/}") rpms=("${rpms[@]%.rpm}")
@@ -31,11 +32,12 @@ function build_rpms() (
 	}
 
 	build_rpm() {
-		MAKEFLAGS="$MAKEFLAGS" SPDK_VERSION="$version" DEPS=no "$rootdir/rpmbuild/rpm.sh" "$@"
+		BUILDDIR=$builddir MAKEFLAGS="$MAKEFLAGS" SPDK_VERSION="$version" DEPS=no "$rootdir/rpmbuild/rpm.sh" "$@"
 		install_uninstall_rpms
 	}
 
 	version="test_shared"
+	builddir=$SPDK_TEST_STORAGE/test-rpm
 	run_test "build_shared_rpm" build_rpm --with-shared
 
 	if [[ -n $SPDK_TEST_NATIVE_DPDK ]]; then

autorun.sh

@@ -12,11 +12,14 @@ if [[ ! -f $conf ]]; then
 	echo "ERROR: $conf doesn't exist"
 	exit 1
 fi
+source "$conf"
 
 echo "Test configuration:"
 cat "$conf"
 
 # Runs agent scripts
 $rootdir/autobuild.sh "$conf"
-sudo -E $rootdir/autotest.sh "$conf"
+if ((SPDK_TEST_UNITTEST == 1 || SPDK_RUN_FUNCTIONAL_TEST == 1)); then
+	sudo -E $rootdir/autotest.sh "$conf"
+fi
 $rootdir/autopackage.sh "$conf"

autotest.sh

@@ -35,11 +35,6 @@ if [ $(uname -s) = Linux ]; then
 	echo "|$rootdir/scripts/core-collector.sh %P %s %t $output_dir/coredumps" > /proc/sys/kernel/core_pattern
 	echo 2 > /proc/sys/kernel/core_pipe_limit
 
-	# Make sure that the hugepage state for our VM is fresh so we don't fail
-	# hugepage allocation. Allow time for this action to complete.
-	echo 1 > /proc/sys/vm/drop_caches
-	sleep 3
-
 	# make sure nbd (network block device) driver is loaded if it is available
 	# this ensures that when tests need to use nbd, it will be fully initialized
 	modprobe nbd || true
@@ -50,7 +45,7 @@ if [ $(uname -s) = Linux ]; then
 	fi
 fi
 
-trap "process_core; autotest_cleanup; exit 1" SIGINT SIGTERM EXIT
+trap "process_core || :; autotest_cleanup; exit 1" SIGINT SIGTERM EXIT
 
 timing_enter autotest
 
@@ -152,11 +147,6 @@ timing_enter afterboot
 timing_exit afterboot
 
 if [[ $SPDK_TEST_CRYPTO -eq 1 || $SPDK_TEST_REDUCE -eq 1 ]]; then
-	# Make sure that memory is distributed across all NUMA nodes - by default, all goes to
-	# node0, but if QAT devices are attached to a different node, all of their VFs will end
-	# up under that node too and memory needs to be available there for the tests.
-	CLEAR_HUGE=yes HUGE_EVEN_ALLOC=yes ./scripts/setup.sh
-	./scripts/setup.sh status
 	if [[ $SPDK_TEST_USE_IGB_UIO -eq 1 ]]; then
 		./scripts/qat_setup.sh igb_uio
 	else
@@ -194,7 +184,20 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
 		run_test "bdev_raid" test/bdev/bdev_raid.sh
 		run_test "bdevperf_config" test/bdev/bdevperf/test_config.sh
 		if [[ $(uname -s) == Linux ]]; then
-			run_test "spdk_dd" test/dd/dd.sh
+			run_test "reactor_set_interrupt" test/interrupt/reactor_set_interrupt.sh
+		fi
+	fi
+
+	if [[ $(uname -s) == Linux ]]; then
+		if [[ $SPDK_TEST_BLOCKDEV -eq 1 || $SPDK_TEST_URING -eq 1 ]]; then
+			# The crypto job also includes the SPDK_TEST_BLOCKDEV in its configuration hence the
+			# dd tests are executed there as well. However, these tests can take a significant
+			# amount of time to complete (up to 4min) on a physical system leading to a potential
+			# job timeout. Avoid that by skipping these tests - this should not affect the coverage
+			# since dd tests are still run as part of the vg jobs.
+			if [[ $SPDK_TEST_CRYPTO -eq 0 ]]; then
+				run_test "spdk_dd" test/dd/dd.sh
+			fi
 		fi
 	fi
 
@@ -209,9 +212,15 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
 		if [[ $SPDK_TEST_NVME_PMR -eq 1 ]]; then
 			run_test "nvme_pmr" test/nvme/nvme_pmr.sh
 		fi
+		if [[ $SPDK_TEST_NVME_SCC -eq 1 ]]; then
+			run_test "nvme_scc" test/nvme/nvme_scc.sh
+		fi
 		if [[ $SPDK_TEST_NVME_CUSE -eq 1 ]]; then
 			run_test "nvme_cuse" test/nvme/cuse/nvme_cuse.sh
 		fi
+		if [[ $SPDK_TEST_NVME_CMB -eq 1 ]]; then
+			run_test "nvme_cmb" test/nvme/cmb/cmb.sh
+		fi
 		run_test "nvme_rpc" test/nvme/nvme_rpc.sh
 		# Only test hotplug without ASAN enabled. Since if it is
 		# enabled, it catches SEGV earlier than our handler which

configure

@@ -35,6 +35,7 @@ function usage() {
 	echo " --disable-tests           Disable building of functional tests"
 	echo " --disable-unit-tests      Disable building of unit tests"
 	echo " --disable-examples        Disable building of examples"
+	echo " --disable-apps            Disable building of apps"
 	echo ""
 	echo "Specifying Dependencies:"
 	echo "--with-DEPENDENCY[=path]   Use the given dependency. Optionally, provide the"
@@ -98,6 +99,8 @@ function usage() {
 	echo "                           No path required."
 	echo " wpdk                      Build using WPDK to provide support for Windows (experimental)."
 	echo "                           The argument must be a directory containing lib and include."
+	echo " usdt                      Build with userspace DTrace probes enabled."
+	echo "                           No path required."
 	echo ""
 	echo "Environment variables:"
 	echo ""
@@ -163,7 +166,7 @@ fi
 
 #check nasm only on x86
 if [[ $arch == x86_64* ]]; then
-	ver=$(nasm -v 2> /dev/null | awk '{print $3}')
+	ver=$(nasm -v 2> /dev/null | awk '{print $3}' | awk -Fr '{print $1}')
 	if lt "$ver" 2.14; then
 		# ISA-L, compression & crypto require NASM version 2.14 or newer.
 		CONFIG[ISAL]=n
@@ -263,6 +266,12 @@ for i in "$@"; do
 		--disable-examples)
 			CONFIG[EXAMPLES]=n
 			;;
+		--enable-apps)
+			CONFIG[APPS]=y
+			;;
+		--disable-apps)
+			CONFIG[APPS]=N
+			;;
 		--enable-werror)
 			CONFIG[WERROR]=y
 			;;
@@ -275,6 +284,17 @@ for i in "$@"; do
 		--disable-cet)
 			CONFIG[CET]=n
 			;;
+		--with-dpdk)
+			if pkg-config --exists libdpdk; then
+				CONFIG[DPDK_LIB_DIR]=$(pkg-config --variable=libdir libdpdk)
+				CONFIG[DPDK_INC_DIR]=$(pkg-config --variable=includedir libdpdk)
+				CONFIG[DPDK_PKG_CONFIG]=y
+				CFLAGS="${CFLAGS:+$CFLAGS }$(pkg-config --cflags libdpdk)"
+			else
+				echo "libdpdk.pc not found, aborting"
+				exit 1
+			fi
+			;;
 		--with-dpdk=*)
 			check_dir "$i"
 			CONFIG[DPDK_DIR]=$(readlink -f ${i#*=})
@@ -448,10 +468,17 @@ for i in "$@"; do
 			;;
 		--with-idxd)
 			CONFIG[IDXD]=y
+			CONFIG[IDXD_KERNEL]=n
 			;;
 		--without-idxd)
 			CONFIG[IDXD]=n
 			;;
+		--with-usdt)
+			CONFIG[USDT]=y
+			;;
+		--without-usdt)
+			CONFIG[USDT]=n
+			;;
 		--)
 			break
 			;;
@@ -472,7 +499,7 @@ BUILD_CMD+=(-I/usr/local/include -L/usr/local/lib)
 
 if [[ "${CONFIG[VFIO_USER]}" = "y" ]]; then
 
-	if ! hash cmake; then
+	if ! bash -c "command -v cmake3 cmake" > /dev/null; then
 		echo "ERROR: --with-vfio-user requires cmake"
 		echo "Please install then re-run this script"
 		exit 1
@@ -502,6 +529,10 @@ if [[ "${CONFIG[IDXD]}" = "y" ]]; then
 		echo "ERROR: IDXD cannot be used due to CPU incompatiblity."
 		exit 1
 	fi
+	if [ -e /usr/include/accel-config/libaccel_config.h ]; then
+		CONFIG[IDXD_KERNEL]=y
+	fi
+
 fi
 
 # Detect architecture and force no ISA-L if non-x86 or non-aarch64 architecture
@@ -520,7 +551,7 @@ fi
 if [ -z "${CONFIG[ENV]}" ]; then
 	CONFIG[ENV]=$rootdir/lib/env_dpdk
 	echo "Using default SPDK env in ${CONFIG[ENV]}"
-	if [ -z "${CONFIG[DPDK_DIR]}" ]; then
+	if [[ -z "${CONFIG[DPDK_DIR]}" && "${CONFIG[DPDK_PKG_CONFIG]}" == n ]]; then
 		if [ ! -f "$rootdir"/dpdk/config/meson.build ]; then
 			echo "DPDK not found; please specify --with-dpdk=<path> or run:"
 			echo
@@ -532,7 +563,7 @@ if [ -z "${CONFIG[ENV]}" ]; then
 		fi
 	fi
 else
-	if [ -n "${CONFIG[DPDK_DIR]}" ]; then
+	if [[ -n "${CONFIG[DPDK_DIR]}" || "${CONFIG[DPDK_PKG_CONFIG]}" == y ]]; then
 		echo "--with-env and --with-dpdk are mutually exclusive."
 		exit 1
 	fi
@@ -549,6 +580,14 @@ else
 	CONFIG[VIRTIO]="n"
 fi
 
+if [[ "${CONFIG[DPDK_PKG_CONFIG]}" == y ]]; then
+	if [[ "${CONFIG[SHARED]}" == n ]]; then
+		# dpdk-devel doesn't provide static libs
+		echo "Build against packaged DPDK requested, enabling shared libraries"
+		CONFIG[SHARED]=y
+	fi
+fi
+
 if [[ $sys_name == "Windows" ]]; then
 	if [ -z "${CONFIG[WPDK_DIR]}" ]; then
 		if [ ! -f "$rootdir"/wpdk/Makefile ]; then
@@ -830,6 +869,20 @@ if [[ "${CONFIG[ISAL]}" = "y" ]]; then
 	cd $rootdir
 fi
 
+# For ARM Neoverse-N1 platform, debug build needs gcc version newer than 8.4
+if [[ "${CONFIG[DEBUG]}" = "y" && $arch = aarch64* && "$CC_TYPE" = "gcc" ]]; then
+	GCC_VERSION=$($CC -dumpfullversion)
+	PART_NUM=$(grep -i -m 1 "CPU part" /proc/cpuinfo | awk '{print $4}')
+
+	if [[ "$(printf '%s\n' "8.4.0" "$GCC_VERSION" | sort -V | head -n1)" != "8.4.0" ]]; then
+		if [[ $PART_NUM = 0xd0c ]]; then
+			echo "WARNING: For ARM Neoverse-N1 platform, debug build needs GCC version newer than 8.4."
+			echo "         Will work around this by using armv8.2-a+crypto as target architecture for now."
+			CONFIG[ARCH]=armv8.2-a+crypto
+		fi
+	fi
+fi
+
 # We are now ready to generate final configuration. But first do sanity
 # check to see if all keys in CONFIG array have its reflection in CONFIG file.
 if (($(grep -cE "^\s*CONFIG_[[:alnum:]_]+=" "$rootdir/CONFIG") != ${#CONFIG[@]})); then
@@ -837,7 +890,7 @@ if (($(grep -cE "^\s*CONFIG_[[:alnum:]_]+=" "$rootdir/CONFIG") != ${#CONFIG[@]})
 	echo "BUG: Some configuration options are not present in CONFIG file. Please update this file."
 	echo "Missing options in CONFIG (+) file and in current config (-): "
 	diff -u --label "CONFIG file" --label "CONFIG[@]" \
-		<(sed -r -e '/^\s*$/d; /^\s*#.*/d; s/(CONFIG_[[:alnum:]_]+)=.*/\1/g' CONFIG | sort) \
+		<(sed -r -e '/^[[:space:]]*$/d; /^[[:space:]]*#.*/d; s/(CONFIG_[[:alnum:]_]+)=.*/\1/g' CONFIG | sort) \
 		<(printf "CONFIG_%s\n" "${!CONFIG[@]}" | sort)
 	exit 1
 fi
@@ -845,7 +898,7 @@ fi
 echo -n "Creating mk/config.mk..."
 cp -f $rootdir/CONFIG $rootdir/mk/config.mk
 for key in "${!CONFIG[@]}"; do
-	sed -i.bak -r "s#^\s*CONFIG_${key}=.*#CONFIG_${key}\?=${CONFIG[$key]}#g" $rootdir/mk/config.mk
+	sed -i.bak -r "s#[[:space:]]*CONFIG_${key}=.*#CONFIG_${key}\?=${CONFIG[$key]}#g" $rootdir/mk/config.mk
 done
 # On FreeBSD sed -i 'SUFFIX' - SUFFIX is mandatory. So no way but to delete the backed file.
 rm -f $rootdir/mk/config.mk.bak

deprecation.md

@@ -11,25 +11,3 @@ Specific future SPDK release for the removal must be provided.
 ABI cannot be removed without providing deprecation notice for at least single SPDK release.
 
 # Deprecation Notices {#deprecation-notices}
-
-## net
-
-The net library is deprecated and will be removed in the 21.07 release.
-
-## nvmf
-
-The following APIs have been deprecated and will be removed in SPDK 21.07:
-- `spdk_nvmf_poll_group_get_stat` (function in `nvmf.h`),
-- `spdk_nvmf_transport_poll_group_get_stat` (function in `nvmf.h`),
-- `spdk_nvmf_transport_poll_group_free_stat`(function in `nvmf.h`),
-- `spdk_nvmf_rdma_device_stat` (struct in `nvmf.h`),
-- `spdk_nvmf_transport_poll_group_stat` (struct in `nvmf.h`),
-- `poll_group_get_stat` (transport op in `nvmf_transport.h`),
-- `poll_group_free_stat` (transport op in `nvmf_transport.h`).
-Please use `spdk_nvmf_poll_group_dump_stat` and `poll_group_dump_stat` instead.
-
-## rpm
-
-`pkg/spdk.spec` is considered to be deprecated and scheduled for removal in SPDK 21.07.
-Please use `rpmbuild/spdk.spec` instead and see
-[RPM documentation](https://spdk.io/doc/rpm.html) for more details.

doc/Doxyfile

@@ -843,6 +843,7 @@ INPUT                  += \
                          spdk_top.md \
                          ssd_internals.md \
                          system_configuration.md \
+                         usdt.md \
                          userspace.md \
                          vagrant.md \
                          vhost.md \

doc/event.md

@@ -73,3 +73,11 @@ spdk_app_start() is called, it will block the current thread until the applicati
 terminates by calling spdk_app_stop() or an error condition occurs during the
 initialization code within spdk_app_start(), itself, before invoking the caller's
 supplied function.
+
+## Custom shutdown callback {#event_component_shutdown}
+
+When creating SPDK based application user may add custom shutdown callback which
+will be called before the application framework starts the shutdown process.
+To do that set shutdown_cb function callback in spdk_app_opts structure passed
+to spdk_app_start(). Custom shutdown callback should call spdk_app_stop() before
+returning to continue application shutdown process.

doc/jsonrpc.md

@@ -405,9 +405,6 @@ Example response:
     "framework_start_init",
     "rpc_get_methods",
     "scsi_get_devices",
-    "net_get_interfaces",
-    "delete_ip_address",
-    "net_interface_add_ip_address",
     "nbd_get_disks",
     "nbd_stop_disk",
     "nbd_start_disk",
@@ -460,6 +457,7 @@ Example response:
     "bdev_passthru_delete"
     "bdev_nvme_apply_firmware",
     "bdev_nvme_get_transport_statistics",
+    "bdev_nvme_get_controller_health_info",
     "bdev_nvme_detach_controller",
     "bdev_nvme_attach_controller",
     "bdev_null_create",
@@ -1410,6 +1408,7 @@ This feature is considered as experimental.
 Name                    | Optional | Type        | Description
 ----------------------- | -------- | ----------- | -----------
 config_number           | Required | number      | Pre-defined config # to use (ie 0, 1). See [docs.](https://spdk.io/doc/idxd.html)
+config_kernel_mode      | Optional | Boolean     | If set, will use kernel idxd driver.
 
 ### Example
 
@@ -1418,7 +1417,8 @@ Example request:
 ~~~
 {
   "params": {
-    "config_number": 0
+    "config_number": 0,
+    "config_kernel_mode": false
   },
   "jsonrpc": "2.0",
   "method": "idxd_scan_accel_engine",
@@ -2107,6 +2107,7 @@ Name                    | Optional | Type        | Description
 ----------------------- | -------- | ----------- | -----------
 name                    | Required | string      | Bdev name to use
 mode                    | Required | string      | OCF cache mode: wb, wt, pt, wa, wi, wo
+cache_line_size         | Optional | int         | OCF cache line size in KiB: 4, 8, 16, 32, 64
 cache_bdev_name         | Required | string      | Name of underlying cache bdev
 core_bdev_name          | Required | string      | Name of underlying core bdev
 
@@ -2123,7 +2124,8 @@ Example request:
   "params": {
     "name": "ocf0",
     "mode": "wt",
-    "cache_bdev_name": "Nvme0n1"
+    "cache_line_size": 64,
+    "cache_bdev_name": "Nvme0n1",
     "core_bdev_name": "aio0"
   },
   "jsonrpc": "2.0",
@@ -2762,6 +2764,7 @@ Name                       | Optional | Type        | Description
 -------------------------- | -------- | ----------- | -----------
 action_on_timeout          | Optional | string      | Action to take on command time out: none, reset or abort
 timeout_us                 | Optional | number      | Timeout for each command, in microseconds. If 0, don't track timeouts
+timeout_admin_us           | Optional | number      | Timeout for each admin command, in microseconds. If 0, treat same as io timeouts ('timeout_us')
 keep_alive_timeout_ms      | Optional | number      | Keep alive timeout period in milliseconds, default is 10s
 retry_count                | Optional | number      | The number of attempts per I/O before an I/O fails
 arbitration_burst          | Optional | number      | The value is expressed as a power of two, a value of 111b indicates no limit
@@ -2788,6 +2791,7 @@ request:
     "high_priority_weight": 8,
     "nvme_adminq_poll_period_us": 2000,
     "timeout_us": 10000000,
+    "timeout_admin_us": 20000000,
     "keep_alive_timeout_ms": 600000,
     "action_on_timeout": "reset",
     "io_queue_requests" : 2048,
@@ -3003,6 +3007,43 @@ Example response:
 }
 ~~~
 
+## bdev_nvme_reset_controller {#rpc_bdev_nvme_reset_controller}
+
+Reset NVMe controller.
+
+Returns true if the controller reset was successful, false otherwise.
+
+### Parameters
+
+Name                    | Optional | Type        | Description
+----------------------- | -------- | ----------- | -----------
+name                    | Required | string      | NVMe controller name
+
+### Example
+
+Example request:
+
+~~~
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "bdev_nvme_reset_controller",
+  "params": {
+    "name": "Nvme0"
+  }
+}
+~~~
+
+Example response:
+
+~~~
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": true
+}
+~~~
+
 ## bdev_nvme_cuse_register {#rpc_bdev_nvme_cuse_register}
 
 Register CUSE device on NVMe controller.
@@ -3301,6 +3342,203 @@ Example response:
 }
 ~~~
 
+## bdev_nvme_get_controller_health_info {#rpc_bdev_nvme_get_controller_health_info}
+
+Display health log of the required NVMe bdev device.
+
+### Parameters
+
+Name                    | Optional | Type        | Description
+----------------------- | -------- | ----------- | -----------
+name                    | Required | string      | Name of the NVMe bdev controller
+
+### Response
+
+The response is the object containing information about health log of the NVMe controller.
+
+### Example
+
+Example request:
+
+~~~
+{
+  "jsonrpc": "2.0",
+  "method": "bdev_nvme_get_controller_health_info",
+  "id": 1,
+  "params": {
+    "name": "Nvme0"
+  }
+}
+~~~
+
+Example response:
+
+~~~
+{
+  "model_number": "INTEL SSDPE2KX020T8",
+  "serial_number": "BTLJ72430ARH2P0BGN",
+  "firmware_revision": "VDV10110",
+  "traddr": "0000:08:00.0",
+  "temperature_celsius": 32,
+  "available_spare_percentage": 99,
+  "available_spare_threshold_percentage": 10,
+  "percentage_used": 2,
+  "data_units_read": 1013408619,
+  "data_units_written": 346792685,
+  "host_read_commands": 30457773282,
+  "host_write_commands": 18949677715,
+  "controller_busy_time": 4979,
+  "power_cycles": 49,
+  "power_on_hours": 31118,
+  "unsafe_shutdowns": 18,
+  "media_errors": 17,
+  "num_err_log_entries": 19,
+  "warning_temperature_time_minutes": 0,
+  "critical_composite_temperature_time_minutes": 0
+}
+~~~
+
+## bdev_rbd_register_cluster {#rpc_bdev_rbd_register_cluster}
+
+This method is available only if SPDK was build with Ceph RBD support.
+
+### Parameters
+
+Name                    | Optional | Type        | Description
+----------------------- | -------- | ----------- | -----------
+name                    | Required | string      | Registerd Rados cluster object name
+user_id                 | Optional | string      | Ceph ID (i.e. admin, not client.admin)
+config_param            | Optional | string map  | Explicit librados configuration
+config_file             | Optional | string      | File path of libraodos configuration file
+
+This RPC registers a Rados Cluster object handle which is only known
+to rbd module, it uses user_id + config_param or user_id + config_file to
+identify a Rados cluster object.
+
+If no config_param is specified, Ceph configuration files must exist with
+all relevant settings for accessing the Ceph cluster. If a config map is
+passed, the configuration files are ignored and instead all key/value
+pairs are passed to rados_conf_set to configure cluster access. In
+practice, "mon_host" (= list of monitor address+port) and "key" (= the
+secret key stored in Ceph keyrings) are enough.
+
+When accessing the Ceph cluster as some user other than "admin" (the
+default), the "user_id" has to be set.
+
+### Result
+
+Name of newly created Rados cluster object.
+
+### Example
+
+Example request with `key` from `/etc/ceph/ceph.client.admin.keyring`:
+
+~~
+{
+  "params": {
+    "name": "rbd_cluster",
+    "config_param": {
+      "mon_host": "192.168.7.1:6789,192.168.7.2:6789",
+      "key": "AQDwf8db7zR1GRAA5k7NKXjS5S5V4mntwUDnGQ==",
+    }
+  },
+  "jsonrpc": "2.0",
+  "method": "bdev_rbd_register_cluster",
+  "id": 1
+}
+~~
+
+Example response:
+
+~~
+response:
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": "rbd_cluster"
+}
+~~
+
+## bdev_rbd_unregister_cluster {#rpc_bdev_rbd_unregister_cluster}
+
+This method is available only if SPDK was build with Ceph RBD support.
+If there is still rbd bdev using this cluster, the unregisteration operation
+will fail.
+
+### Result
+
+`true` if Rados cluster object with provided name was deleted or `false` otherwise.
+
+### Parameters
+
+Name                    | Optional | Type        | Description
+----------------------- | -------- | ----------- | -------------------------
+name                    | Required | string      | Rados cluster object name
+
+### Example
+
+Example request:
+
+~~
+{
+  "params": {
+    "name": "rbd_cluster"
+  },
+  "jsonrpc": "2.0",
+  "method": "bdev_rbd_unregister_cluster",
+  "id": 1
+}
+~~
+
+Example response:
+
+~~
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": true
+}
+~~
+
+## bdev_rbd_get_clusters_info {#rpc_bdev_rbd_get_clusters_info}
+
+This method is available only if SPDK was build with Ceph RBD support.
+
+### Result
+
+Returns the cluster info of the Rados Cluster name if provided. Otherwise, it
+returns the cluster info of every registered Raods Cluster name.
+
+### Parameters
+
+Name                    | Optional | Type        | Description
+----------------------- | -------- | ----------- | -------------------------
+name                    | Optional | string      | Rados cluster object name
+
+### Example
+
+Example request:
+
+~~
+{
+  "params": {
+    "name": "rbd_cluster"
+  },
+  "jsonrpc": "2.0",
+  "method": "bdev_rbd_get_clusters_info",
+  "id": 1
+}
+~~
+
+Example response:
+
+~~
+{
+  "jsonrpc": "2.0",
+  "cluster_name": "rbd_cluster"
+}
+~~
+
 ## bdev_rbd_create {#rpc_bdev_rbd_create}
 
 Create @ref bdev_config_rbd bdev
@@ -3317,6 +3555,7 @@ pool_name               | Required | string      | Pool name
 rbd_name                | Required | string      | Image name
 block_size              | Required | number      | Block size
 config                  | Optional | string map  | Explicit librados configuration
+cluster_name            | Optional | string      | Rados cluster object name created in this module.
 
 If no config is specified, Ceph configuration files must exist with
 all relevant settings for accessing the pool. If a config map is
@@ -3328,6 +3567,10 @@ secret key stored in Ceph keyrings) are enough.
 When accessing the image as some user other than "admin" (the
 default), the "user_id" has to be set.
 
+If provided with cluster_name option, it will use the Rados cluster object
+referenced by the name (created by bdev_rbd_register_cluster RPC) and ignores
+"user_id + config" combination to create its own Rados cluster.
+
 ### Result
 
 Name of newly created bdev.
@@ -3364,6 +3607,33 @@ response:
 }
 ~~~
 
+Example request with `cluster_name`:
+
+~~
+{
+  "params": {
+    "pool_name": "rbd",
+    "rbd_name": "foo",
+    "block_size": 4096,
+    "cluster_name": "rbd_cluster"
+  },
+  "jsonrpc": "2.0",
+  "method": "bdev_rbd_create",
+  "id": 1
+}
+~~
+
+Example response:
+
+~~
+response:
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": "Ceph0"
+}
+~~
+
 ## bdev_rbd_delete {#rpc_bdev_rbd_delete}
 
 Delete @ref bdev_config_rbd bdev
@@ -4443,6 +4713,9 @@ error_recovery_level            | Optional | number  | Session specific paramete
 allow_duplicated_isid           | Optional | boolean | Allow duplicated initiator session ID (default: `false`)
 max_large_datain_per_connection | Optional | number  | Max number of outstanding split read I/Os per connection (default: 64)
 max_r2t_per_connection          | Optional | number  | Max number of outstanding R2Ts per connection (default: 4)
+pdu_pool_size                   | Optional | number  | Number of PDUs in the pool (default: approximately 2 * max_sessions * (max_queue_depth + max_connections_per_session))
+immediate_data_pool_size        | Optional | number  | Number of immediate data buffers in the pool (default: 128 * max_sessions)
+data_out_pool_size              | Optional | number  | Number of data out buffers in the pool (default: 16 * max_sessions)
 
 To load CHAP shared secret file, its path is required to specify explicitly in the parameter `auth_file`.
 
@@ -5909,6 +6182,8 @@ model_number            | Optional | string      | Model number of virtual contr
 max_namespaces          | Optional | number      | Maximum number of namespaces that can be attached to the subsystem. Default: 0 (Unlimited)
 allow_any_host          | Optional | boolean     | Allow any host (`true`) or enforce allowed host list (`false`). Default: `false`.
 ana_reporting           | Optional | boolean     | Enable ANA reporting feature (default: `false`).
+min_cntlid              | Optional | number      | Minimum controller ID. Default: 1
+max_cntlid              | Optional | number      | Maximum controller ID. Default: 0xffef
 
 ### Example
 
@@ -6593,6 +6868,7 @@ tgt_name                    | Optional | string      | Parent NVMe-oF target nam
 ### Response
 
 The response is an object containing NVMf subsystem statistics.
+In the response, `admin_qpairs` and `io_qpairs` are reflecting cumulative queue pair counts while `current_admin_qpairs` and `current_io_qpairs` are showing the current number.
 
 ### Example
 
@@ -6617,6 +6893,8 @@ Example response:
         "name": "app_thread",
         "admin_qpairs": 1,
         "io_qpairs": 4,
+        "current_admin_qpairs": 1,
+        "current_io_qpairs": 2,
         "pending_bdev_io": 1721,
         "transports": [
           {
@@ -6660,6 +6938,22 @@ Example response:
 }
 ~~~
 
+
+## nvmf_set_crdt {#rpc_nvmf_set_crdt}
+
+Set the 3 CRDT (Command Retry Delay Time) values. For details about
+CRDT, please refer to the NVMe specification. Currently all the
+SPDK nvmf subsystems share the same CRDT values. The default values
+are 0. This rpc can only be invoked in STARTUP stage. All values are
+in units of 100 milliseconds (same as the NVMe specification).
+
+### Parameters
+Name                    | Optional | Type        | Description
+----------------------- | -------- | ----------- | -----------
+crdt1                   | Optional | number      | Command Retry Delay Time 1
+crdt2                   | Optional | number      | Command Retry Delay Time 2
+crdt3                   | Optional | number      | Command Retry Delay Time 3
+
 # Vhost Target {#jsonrpc_components_vhost_tgt}
 
 The following common preconditions need to be met in all target types.
@@ -7667,7 +7961,7 @@ Example request:
       "Malloc2",
       "Malloc3"
     ],
-    "strip_size": 4096
+    "strip_size_kb": 4
   }
 }
 ~~~
@@ -8591,7 +8885,10 @@ Example response:
     "recv_buf_size": 2097152,
     "send_buf_size": 2097152,
     "enable_recv_pipe": true,
-    "enable_zerocopy_send": true
+    "enable_quickack": true,
+    "enable_placement_id": 0,
+    "enable_zerocopy_send_server": true,
+    "enable_zerocopy_send_client": false
   }
 }
 ~~~
@@ -8602,15 +8899,16 @@ Set parameters for the socket layer implementation.
 
 ### Parameters
 
-Name                    | Optional | Type        | Description
------------------------ | -------- | ----------- | -----------
-impl_name               | Required | string      | Name of socket implementation, e.g. posix
-recv_buf_size           | Optional | number      | Size of socket receive buffer in bytes
-send_buf_size           | Optional | number      | Size of socket send buffer in bytes
-enable_recv_pipe        | Optional | boolean     | Enable or disable receive pipe
-enable_zerocopy_send    | Optional | boolean     | Enable or disable zero copy on send
-enable_quick_ack        | Optional | boolean     | Enable or disable quick ACK
-enable_placement_id     | Optional | number      | Enable or disable placement_id. 0:disable,1:incoming_napi,2:incoming_cpu
+Name                        | Optional | Type        | Description
+--------------------------- | -------- | ----------- | -----------
+impl_name                   | Required | string      | Name of socket implementation, e.g. posix
+recv_buf_size               | Optional | number      | Size of socket receive buffer in bytes
+send_buf_size               | Optional | number      | Size of socket send buffer in bytes
+enable_recv_pipe            | Optional | boolean     | Enable or disable receive pipe
+enable_quick_ack            | Optional | boolean     | Enable or disable quick ACK
+enable_placement_id         | Optional | number      | Enable or disable placement_id. 0:disable,1:incoming_napi,2:incoming_cpu
+enable_zerocopy_send_server | Optional | boolean     | Enable or disable zero copy on send for server sockets
+enable_zerocopy_send_client | Optional | boolean     | Enable or disable zero copy on send for client sockets
 
 ### Response
 
@@ -8630,9 +8928,10 @@ Example request:
     "recv_buf_size": 2097152,
     "send_buf_size": 2097152,
     "enable_recv_pipe": false,
-    "enable_zerocopy_send": true,
     "enable_quick_ack": false,
-    "enable_placement_id": 0
+    "enable_placement_id": 0,
+    "enable_zerocopy_send_server": true,
+    "enable_zerocopy_send_client": false
   }
 }
 ~~~

doc/misc.md

@@ -2,3 +2,4 @@
 
 - @subpage peer_2_peer
 - @subpage containers
+- @subpage rpms

doc/nvmf_tracing.md

@@ -174,11 +174,13 @@ SPDK_TRACE_REGISTER_FN(nvmf_trace)
 	spdk_trace_register_object(OBJECT_NVMF_RDMA_IO, 'r');
 	spdk_trace_register_description("RDMA_REQ_NEW", "",
 					TRACE_RDMA_REQUEST_STATE_NEW,
-					OWNER_NONE, OBJECT_NVMF_RDMA_IO, 1, 1, "cmid:	");
+					OWNER_NONE, OBJECT_NVMF_RDMA_IO, 1,
+					SPDK_TRACE_ARG_TYPE_PTR, "cmid:	");
 	...
 	spdk_trace_register_description("NEW_RDMA_REQ_NAME", "",
 					NEW_TRACE_POINT_NAME,
-					OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid:	");
+					OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0,
+					SPDK_TRACE_ARG_TYPE_PTR, "cmid:	");
 }
 ~~~
 

doc/performance_reports.md

@@ -1,5 +1,12 @@
 # Performance Reports {#performance_reports}
 
+## Release 21.04
+
+- [SPDK 21.04 NVMe Bdev Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_nvme_bdev_perf_report_2104.pdf)
+- [SPDK 21.04 NVMe-oF TCP Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_tcp_perf_report_2104.pdf)
+- [SPDK 21.04 NVMe-oF RDMA Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_rdma_perf_report_2104.pdf)
+- [SPDK 21.04 Vhost Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_vhost_perf_report_2104.pdf)
+
 ## Release 21.01
 
 - [SPDK 21.01 NVMe Bdev Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_nvme_bdev_perf_report_2101.pdf)

doc/spdk_top.md

@@ -17,13 +17,10 @@ Run the spdk_top application
 Menu at the bottom of SPDK top window shows many options for changing displayed data. Each menu item has a key associated with it in square brackets.
 
 * Quit - quits the SPDK top application.
-* TAB selection - allows to select THREADS/POLLERS/CORES tabs.
+* Switch tab - allows to select THREADS/POLLERS/CORES tabs.
 * Previous page/Next page - scrolls up/down to the next set of rows displayed. Indicator in the bottom-left corner shows current page and number of all available pages.
-* Columns - enables/disables chosen columns in a column pop-up window.
-* Sorting - allows to sort displayed data by column in a sorting pop-up.
-* Refresh rate - takes user input from 0 to 255 and changes refresh rate to that value in seconds.
 * Item details - displays details pop-up window for highlighted data row. Selection is changed by pressing UP and DOWN arrow keys.
-* Total/Interval - changes displayed values in all tabs to either Total time (measured since start of SPDK application) or Interval time (measured since last refresh).
+* Help - displays help pop-up window.
 
 # Threads Tab
 The threads tab displays a line item for each spdk thread. The information displayed shows:
@@ -63,3 +60,6 @@ The cores tab provides insights into how the application is using the CPU cores
 
 \n
 Pressing ENTER key makes a pop-up window appear, showing above information, along with a list of threads running on selected core. Cores details window allows to select a thread and display thread details pop-up on top of it. To close both pop-ups use ESC key.
+
+# Help Window
+Help window pop-up can be invoked by pressing H key inside any tab. It contains explanations for each key used inside the spdk_top application.

doc/usdt.md

@@ -0,0 +1,204 @@
+# Userspace DTrace (USDT) {#usdt}
+
+## Package Dependencies
+
+These dependencies are needed for building bpftrace and
+the sys/sdt.h header file that SPDK libraries will include
+for DTRACE_PROBE macro definitions.
+
+Fedora:
+libbpf
+gtest-devel
+gmock-devel
+bcc-devel
+systemtap-sdt-devel
+llvm-devel
+bison
+flex
+
+Ubuntu:
+systemtap-sdt-dev
+libbpfcc-dev
+libclang-7-dev
+bison
+flex
+
+## Building bpftrace
+
+We have found issues with the packaged bpftrace on both Ubuntu 20.04
+and Fedora 33.  So bpftrace should be built and installed from source.
+
+```
+git clone https://github.com/iovisor/bpftrace.git
+mkdir bpftrace/build
+cd bpftrace/build
+cmake -DCMAKE_BUILD_TYPE=Release ..
+make
+sudo make install
+```
+
+## bpftrace.sh
+
+bpftrace.sh is a helper script that facilitates running bpftrace scripts
+against a running SPDK application.  Here is a typical usage:
+
+```
+scripts/bpftrace.sh `pidof spdk_tgt` scripts/bpf/nvmf.bt
+```
+
+Attaching to USDT probes requires the full path of the binary in the
+probe description. SPDK bpftrace scripts can be written with an __EXE__
+marker instead of a full path name, and bpftrace.sh will dynamically
+replace that string with the full path name using information from procfs.
+
+It is also useful to filter certain kernel events (such as system calls)
+based on the PID of the SPDK application.  SPDK bpftrace scripts can be
+written with a __PID__ marker, and bpftrace.sh will dynamically replace
+that string with the PID provided to the script.
+
+## Configuring SPDK Build
+
+```
+./configure --with-usdt
+```
+
+## Start SPDK application and bpftrace script
+
+From first terminal:
+
+```
+build/bin/spdk_tgt -m 0xC
+```
+
+From second terminal:
+
+```
+scripts/bpftrace.sh `pidof spdk_tgt` scripts/bpf/nvmf.bt
+```
+
+nvmf.bt will print information about nvmf subsystem and poll
+group info state transitions.
+
+From third terminal:
+
+```
+scripts/rpc.py <<EOF
+nvmf_create_transport -t tcp
+nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 -m 10
+nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t tcp -a 127.0.0.1 -s 4420
+bdev_null_create null0 1000 512
+nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 null0
+EOF
+```
+
+This creates the nvmf tcp transport, a new nvmf subsystem that listens on a tcp
+port, and a null bdev which is added as a namespace to the new nvmf subsystem.
+
+You will see output from the second terminal that looks like this:
+
+```
+2110.935735: nvmf_tgt reached state NONE
+2110.954316: nvmf_tgt reached state CREATE_TARGET
+2110.967905: nvmf_tgt reached state CREATE_POLL_GROUPS
+2111.235982: nvmf_tgt reached state START_SUBSYSTEMS
+2111.253560: nqn.2014-08.org.nvmexpress.discovery change state from INACTIVE to ACTIVE start
+2111.260278: nqn.2014-08.org.nvmexpress.discovery on thread 2 state to ACTIVE start
+2111.264281: nqn.2014-08.org.nvmexpress.discovery on thread 2 state to ACTIVE done
+2111.284083: nqn.2014-08.org.nvmexpress.discovery change state from INACTIVE to ACTIVE done
+2111.289197: nvmf_tgt reached state RUNNING
+2111.271573: nqn.2014-08.org.nvmexpress.discovery on thread 3 state to ACTIVE start
+2111.279787: nqn.2014-08.org.nvmexpress.discovery on thread 3 state to ACTIVE done
+2189.921492: nqn.2016-06.io.spdk:cnode1 change state from INACTIVE to ACTIVE start
+2189.952508: nqn.2016-06.io.spdk:cnode1 on thread 2 state to ACTIVE start
+2189.959125: nqn.2016-06.io.spdk:cnode1 on thread 2 state to ACTIVE done
+2190.005832: nqn.2016-06.io.spdk:cnode1 change state from INACTIVE to ACTIVE done
+2189.969058: nqn.2016-06.io.spdk:cnode1 on thread 3 state to ACTIVE start
+2189.999889: nqn.2016-06.io.spdk:cnode1 on thread 3 state to ACTIVE done
+2197.859104: nqn.2016-06.io.spdk:cnode1 change state from ACTIVE to PAUSED start
+2197.879199: nqn.2016-06.io.spdk:cnode1 on thread 2 state to PAUSED start
+2197.883416: nqn.2016-06.io.spdk:cnode1 on thread 2 state to PAUSED done
+2197.902291: nqn.2016-06.io.spdk:cnode1 change state from ACTIVE to PAUSED done
+2197.908939: nqn.2016-06.io.spdk:cnode1 change state from PAUSED to ACTIVE start
+2197.912644: nqn.2016-06.io.spdk:cnode1 on thread 2 state to ACTIVE start
+2197.927409: nqn.2016-06.io.spdk:cnode1 on thread 2 state to ACTIVE done
+2197.949742: nqn.2016-06.io.spdk:cnode1 change state from PAUSED to ACTIVE done
+2197.890812: nqn.2016-06.io.spdk:cnode1 on thread 3 state to PAUSED start
+2197.897233: nqn.2016-06.io.spdk:cnode1 on thread 3 state to PAUSED done
+2197.931278: nqn.2016-06.io.spdk:cnode1 on thread 3 state to ACTIVE start
+2197.946124: nqn.2016-06.io.spdk:cnode1 on thread 3 state to ACTIVE done
+2205.859904: nqn.2016-06.io.spdk:cnode1 change state from ACTIVE to PAUSED start
+2205.891392: nqn.2016-06.io.spdk:cnode1 on thread 2 state to PAUSED start
+2205.896588: nqn.2016-06.io.spdk:cnode1 on thread 2 state to PAUSED done
+2205.920133: nqn.2016-06.io.spdk:cnode1 change state from ACTIVE to PAUSED done
+2205.905900: nqn.2016-06.io.spdk:cnode1 on thread 3 state to PAUSED start
+2205.914856: nqn.2016-06.io.spdk:cnode1 on thread 3 state to PAUSED done
+2206.091084: nqn.2016-06.io.spdk:cnode1 change state from PAUSED to ACTIVE start
+2206.099222: nqn.2016-06.io.spdk:cnode1 on thread 2 state to ACTIVE start
+2206.105445: nqn.2016-06.io.spdk:cnode1 on thread 2 state to ACTIVE done
+2206.119271: nqn.2016-06.io.spdk:cnode1 change state from PAUSED to ACTIVE done
+2206.109144: nqn.2016-06.io.spdk:cnode1 on thread 3 state to ACTIVE start
+2206.115636: nqn.2016-06.io.spdk:cnode1 on thread 3 state to ACTIVE done
+```
+
+Now stop the bpftrace.sh running in the second terminal, and start
+it again with the send_msg.bt script.  This script keeps a count of
+functions executed as part of an spdk_for_each_channel or
+spdk_thread_send_msg function call.
+
+```
+scripts/bpftrace.sh `pidof spdk_tgt` scripts/bpf/send_msg.bt
+```
+
+From the third terminal, create another null bdev and add it as a
+namespace to the cnode1 subsystem.
+
+```
+scripts/rpc.py <<EOF
+bdev_null_create null1 1000 512
+nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 null1
+EOF
+```
+
+Now Ctrl-C the bpftrace.sh in the second terminal, and it will
+print the final results of the maps.
+
+```
+@for_each_channel[subsystem_state_change_on_pg]: 2
+
+@send_msg[_finish_unregister]: 1
+@send_msg[_call_completion]: 2
+@send_msg[put_io_channel]: 4
+@send_msg[_call_channel]: 4
+```
+
+## TODOs and known limitations
+
+- add definitions for common nvmf data structures - then we can pass the subsystem pointer
+  itself as a probe parameter and let the script decide which fields it wants to access
+  (Note: these would need to be kept up-to-date with the C definitions of the struct - it is
+  not possible to include the header files in a bpftrace script)
+
+- investigate using pahole to generate data structure definitions that can be included in
+  bpftrace scripts; this would allow us to pass the subsystem pointer itself as a probe
+  argument, and let the script decide which fields it wants to access; for example,
+  `pahole -E -C spdk_nvmf_subsystem build/bin/spdk_tgt` gets us close to what we need,
+  but there are some limiters:
+
+	- our structures have char arrays (not char pointers) for things like subnqn; large
+	  arrays like these cannot currently be passed to bpftrace printf without generating
+	  a stack space error (probe points are limited to 512 bytes of stack); we could
+	  modify SPDK to have the char array for storage, and a char pointer that points to
+	  that storage, the latter could easily then be used in bpftrace scripts
+	- our structures include fields with their enum types instead of int; bpftrace will
+	  complain it does not know about the enum (pahole doesn't print out enum
+	  descriptions); information on enums can be found in the applications .debug_info
+	  section, but we would need something that can convert that into a file we can
+	  include in a bpftrace script
+
+- Note that bpftrace prints are not always printed in exact chronological order; this can
+  be seen especially with spdk_for_each_channel iterations, where we execute trace points
+  on multiple threads in a very short period of time, and those may not get printed to the
+  console in exact order; this is why the nvmf.bt script prints out a msec.nsec timestamp
+  so that the user can understand the ordering and even pipe through sort if desired
+
+- flesh out more DTrace probes in the nvmf code

doc/user_guides.md

@@ -11,3 +11,4 @@
 - @subpage blobfs
 - @subpage jsonrpc
 - @subpage jsonrpc_proxy
+- @subpage usdt

doc/vhost.md

@@ -280,9 +280,9 @@ host:~# taskset -c 2,3 qemu-system-x86_64 \
   -drive file=guest_os_image.qcow2,if=none,id=disk \
   -device ide-hd,drive=disk,bootindex=0 \
   -chardev socket,id=spdk_vhost_scsi0,path=/var/tmp/vhost.0 \
-  -device vhost-user-scsi-pci,id=scsi0,chardev=spdk_vhost_scsi0,num_queues=4 \
+  -device vhost-user-scsi-pci,id=scsi0,chardev=spdk_vhost_scsi0,num_queues=2 \
   -chardev socket,id=spdk_vhost_blk0,path=/var/tmp/vhost.1 \
-  -device vhost-user-blk-pci,chardev=spdk_vhost_blk0,num-queues=4
+  -device vhost-user-blk-pci,chardev=spdk_vhost_blk0,num-queues=2
 ~~~
 
 Please note the following two commands are run on the guest VM.
@@ -331,6 +331,10 @@ to set `num_queues=4` to saturate physical device. Adding too many queues might
 vhost performance degradation if many vhost devices are used because each device will require
 additional `num_queues` to be polled.
 
+Some Linux distributions report a kernel panic when starting the VM if the number of I/O queues
+specified via the `num-queues` parameter is greater than number of vCPUs. If you need to use
+more I/O queues than vCPUs, check that your OS image supports that configuration.
+
 ## Hot-attach/hot-detach {#vhost_hotattach}
 
 Hotplug/hotremove within a vhost controller is called hot-attach/detach. This is to

dpdk

@@ -1 +1 @@
-Subproject commit eb16786836e3a8380bb86fde67efa2ee0d9d3852
+Subproject commit bad3c0e51d7a34e3188d75d94f15a36c8f5e8301

dpdkbuild/Makefile

@@ -137,7 +137,7 @@ ifeq ($(MAKE_PID),)
 MAKE_PID := $(shell echo $$PPID)
 endif
 
-MAKE_NUMJOBS := $(shell ps T | sed -nE 's/\s*$(MAKE_PID)\s.* (-j|--jobs=)( *[0-9]+).*/\1\2/p')
+MAKE_NUMJOBS := $(shell ps T | sed -nE 's/[[:space:]]*$(MAKE_PID)[[:space:]].* (-j|--jobs=)( *[0-9]+).*/\1\2/p')
 
 all: $(SPDK_ROOT_DIR)/dpdk/build-tmp
 	$(Q)# DPDK doesn't handle nested make calls, so unset MAKEFLAGS

examples/Makefile

@@ -34,7 +34,7 @@
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-DIRS-y += accel bdev blob ioat nvme sock vmd nvmf
+DIRS-y += accel bdev blob ioat nvme sock vmd nvmf util
 
 ifeq ($(OS),Linux)
 DIRS-$(CONFIG_VHOST) += interrupt_tgt

examples/accel/perf/Makefile

@@ -39,6 +39,6 @@ APP = accel_perf
 
 C_SRCS := accel_perf.c
 
-SPDK_LIB_LIST = $(ACCEL_MODULES_LIST) event_accel
+SPDK_LIB_LIST = $(ACCEL_MODULES_LIST) event event_accel
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

examples/accel/perf/accel_perf.c

@@ -44,11 +44,16 @@
 #define DATA_PATTERN 0x5a
 #define ALIGN_4K 0x1000
 
+static bool g_using_sw_engine = false;
 static uint64_t	g_tsc_rate;
 static uint64_t g_tsc_end;
 static int g_rc;
 static int g_xfer_size_bytes = 4096;
 static int g_queue_depth = 32;
+/* g_allocate_depth indicates how many tasks we allocate per worker. It will
+ * be at least as much as the queue depth.
+ */
+static int g_allocate_depth = 0;
 static int g_ops_per_batch = 0;
 static int g_threads_per_core = 1;
 static int g_time_in_sec = 5;
@@ -62,7 +67,6 @@ static enum accel_capability g_workload_selection;
 static struct worker_thread *g_workers = NULL;
 static int g_num_workers = 0;
 static pthread_mutex_t g_workers_lock = PTHREAD_MUTEX_INITIALIZER;
-uint64_t g_capabilites;
 
 struct worker_thread;
 static void accel_done(void *ref, int status);
@@ -78,6 +82,7 @@ struct ap_task {
 	uint32_t		iov_cnt;
 	void			*dst;
 	void			*dst2;
+	uint32_t		crc_dst;
 	struct worker_thread	*worker;
 	int			status;
 	int			expected_status; /* used for the compare operation */
@@ -120,16 +125,22 @@ dump_user_config(struct spdk_app_opts *opts)
 	printf("Core mask:      %s\n\n", opts->reactor_mask);
 	printf("Accel Perf Configuration:\n");
 	printf("Workload Type:  %s\n", g_workload_type);
-	if (g_workload_selection == ACCEL_CRC32C) {
+	if (g_workload_selection == ACCEL_CRC32C || g_workload_selection == ACCEL_COPY_CRC32C) {
 		printf("CRC-32C seed:   %u\n", g_crc32c_seed);
-		printf("vector size:    %u\n", g_crc32c_chained_count);
+		printf("vector count    %u\n", g_crc32c_chained_count);
 	} else if (g_workload_selection == ACCEL_FILL) {
 		printf("Fill pattern:   0x%x\n", g_fill_pattern);
 	} else if ((g_workload_selection == ACCEL_COMPARE) && g_fail_percent_goal > 0) {
 		printf("Failure inject: %u percent\n", g_fail_percent_goal);
 	}
-	printf("Transfer size:  %u bytes\n", g_xfer_size_bytes);
+	if (g_workload_selection == ACCEL_COPY_CRC32C) {
+		printf("Vector size:    %u bytes\n", g_xfer_size_bytes);
+		printf("Transfer size:  %u bytes\n", g_xfer_size_bytes * g_crc32c_chained_count);
+	} else {
+		printf("Transfer size:  %u bytes\n", g_xfer_size_bytes);
+	}
 	printf("Queue depth:    %u\n", g_queue_depth);
+	printf("Allocate depth: %u\n", g_allocate_depth);
 	printf("# threads/core: %u\n", g_threads_per_core);
 	printf("Run time:       %u seconds\n", g_time_in_sec);
 	if (g_ops_per_batch > 0) {
@@ -146,49 +157,78 @@ usage(void)
 	printf("accel_perf options:\n");
 	printf("\t[-h help message]\n");
 	printf("\t[-q queue depth per core]\n");
-	printf("\t[-C for crc32c workload, use this value to configre the io vector size to test (default 1)\n");
+	printf("\t[-C for crc32c workload, use this value to configure the io vector size to test (default 1)\n");
 	printf("\t[-T number of threads per core\n");
 	printf("\t[-n number of channels]\n");
 	printf("\t[-o transfer size in bytes]\n");
 	printf("\t[-t time in seconds]\n");
-	printf("\t[-w workload type must be one of these: copy, fill, crc32c, compare, dualcast\n");
+	printf("\t[-w workload type must be one of these: copy, fill, crc32c, copy_crc32c, compare, dualcast\n");
 	printf("\t[-s for crc32c workload, use this seed value (default 0)\n");
 	printf("\t[-P for compare workload, percentage of operations that should miscompare (percent, default 0)\n");
 	printf("\t[-f for fill workload, use this BYTE value (default 255)\n");
 	printf("\t[-y verify result if this switch is on]\n");
 	printf("\t[-b batch this number of operations at a time (default 0 = disabled)]\n");
+	printf("\t[-a tasks to allocate per core (default: same value as -q)]\n");
+	printf("\t\tCan be used to spread operations across a wider range of memory.\n");
 }
 
 static int
 parse_args(int argc, char *argv)
 {
+	int argval = 0;
+
 	switch (argc) {
+	case 'a':
 	case 'b':
-		g_ops_per_batch = spdk_strtol(optarg, 10);
+	case 'C':
+	case 'f':
+	case 'T':
+	case 'o':
+	case 'P':
+	case 'q':
+	case 's':
+	case 't':
+		argval = spdk_strtol(optarg, 10);
+		if (argval < 0) {
+			fprintf(stderr, "-%c option must be non-negative.\n", argc);
+			usage();
+			return 1;
+		}
+		break;
+	default:
+		break;
+	};
+
+	switch (argc) {
+	case 'a':
+		g_allocate_depth = argval;
+		break;
+	case 'b':
+		g_ops_per_batch = argval;
 		break;
 	case 'C':
-		g_crc32c_chained_count = spdk_strtol(optarg, 10);
+		g_crc32c_chained_count = argval;
 		break;
 	case 'f':
-		g_fill_pattern = (uint8_t)spdk_strtol(optarg, 10);
+		g_fill_pattern = (uint8_t)argval;
 		break;
 	case 'T':
-		g_threads_per_core = spdk_strtol(optarg, 10);
+		g_threads_per_core = argval;
 		break;
 	case 'o':
-		g_xfer_size_bytes = spdk_strtol(optarg, 10);
+		g_xfer_size_bytes = argval;
 		break;
 	case 'P':
-		g_fail_percent_goal = spdk_strtol(optarg, 10);
+		g_fail_percent_goal = argval;
 		break;
 	case 'q':
-		g_queue_depth = spdk_strtol(optarg, 10);
+		g_queue_depth = argval;
 		break;
 	case 's':
-		g_crc32c_seed = spdk_strtol(optarg, 10);
+		g_crc32c_seed = argval;
 		break;
 	case 't':
-		g_time_in_sec = spdk_strtol(optarg, 10);
+		g_time_in_sec = argval;
 		break;
 	case 'y':
 		g_verify = true;
@@ -201,6 +241,8 @@ parse_args(int argc, char *argv)
 			g_workload_selection = ACCEL_FILL;
 		} else if (!strcmp(g_workload_type, "crc32c")) {
 			g_workload_selection = ACCEL_CRC32C;
+		} else if (!strcmp(g_workload_type, "copy_crc32c")) {
+			g_workload_selection = ACCEL_COPY_CRC32C;
 		} else if (!strcmp(g_workload_type, "compare")) {
 			g_workload_selection = ACCEL_COMPARE;
 		} else if (!strcmp(g_workload_type, "dualcast")) {
@@ -239,6 +281,7 @@ _get_task_data_bufs(struct ap_task *task)
 {
 	uint32_t align = 0;
 	uint32_t i = 0;
+	int dst_buff_len = g_xfer_size_bytes;
 
 	/* For dualcast, the DSA HW requires 4K alignment on destination addresses but
 	 * we do this for all engines to keep it simple.
@@ -247,7 +290,7 @@ _get_task_data_bufs(struct ap_task *task)
 		align = ALIGN_4K;
 	}
 
-	if (g_workload_selection == ACCEL_CRC32C) {
+	if (g_workload_selection == ACCEL_CRC32C || g_workload_selection == ACCEL_COPY_CRC32C) {
 		assert(g_crc32c_chained_count > 0);
 		task->iov_cnt = g_crc32c_chained_count;
 		task->iovs = calloc(task->iov_cnt, sizeof(struct iovec));
@@ -256,6 +299,10 @@ _get_task_data_bufs(struct ap_task *task)
 			return -ENOMEM;
 		}
 
+		if (g_workload_selection == ACCEL_COPY_CRC32C) {
+			dst_buff_len = g_xfer_size_bytes * g_crc32c_chained_count;
+		}
+
 		for (i = 0; i < task->iov_cnt; i++) {
 			task->iovs[i].iov_base = spdk_dma_zmalloc(g_xfer_size_bytes, 0, NULL);
 			if (task->iovs[i].iov_base == NULL) {
@@ -280,17 +327,19 @@ _get_task_data_bufs(struct ap_task *task)
 		}
 	}
 
-	task->dst = spdk_dma_zmalloc(g_xfer_size_bytes, align, NULL);
-	if (task->dst == NULL) {
-		fprintf(stderr, "Unable to alloc dst buffer\n");
-		return -ENOMEM;
-	}
+	if (g_workload_selection != ACCEL_COPY_CRC32C) {
+		task->dst = spdk_dma_zmalloc(dst_buff_len, align, NULL);
+		if (task->dst == NULL) {
+			fprintf(stderr, "Unable to alloc dst buffer\n");
+			return -ENOMEM;
+		}
 
-	/* For compare we want the buffers to match, otherwise not. */
-	if (g_workload_selection == ACCEL_COMPARE) {
-		memset(task->dst, DATA_PATTERN, g_xfer_size_bytes);
-	} else {
-		memset(task->dst, ~DATA_PATTERN, g_xfer_size_bytes);
+		/* For compare we want the buffers to match, otherwise not. */
+		if (g_workload_selection == ACCEL_COMPARE) {
+			memset(task->dst, DATA_PATTERN, dst_buff_len);
+		} else {
+			memset(task->dst, ~DATA_PATTERN, dst_buff_len);
+		}
 	}
 
 	if (g_workload_selection == ACCEL_DUALCAST) {
@@ -318,8 +367,6 @@ _get_task(struct worker_thread *worker)
 		return NULL;
 	}
 
-	task->worker = worker;
-	task->worker->current_queue_depth++;
 	return task;
 }
 
@@ -343,10 +390,14 @@ _submit_single(struct worker_thread *worker, struct ap_task *task)
 					    g_xfer_size_bytes, accel_done, task);
 		break;
 	case ACCEL_CRC32C:
-		rc = spdk_accel_submit_crc32cv(worker->ch, (uint32_t *)task->dst,
+		rc = spdk_accel_submit_crc32cv(worker->ch, &task->crc_dst,
 					       task->iovs, task->iov_cnt, g_crc32c_seed,
 					       accel_done, task);
 		break;
+	case ACCEL_COPY_CRC32C:
+		rc = spdk_accel_submit_copy_crc32cv(worker->ch, task->dst, task->iovs, task->iov_cnt,
+						    &task->crc_dst, g_crc32c_seed, accel_done, task);
+		break;
 	case ACCEL_COMPARE:
 		random_num = rand() % 100;
 		if (random_num < g_fail_percent_goal) {
@@ -402,8 +453,12 @@ _batch_prep_cmd(struct worker_thread *worker, struct ap_task *task,
 						*(uint8_t *)task->src,
 						g_xfer_size_bytes, accel_done, task);
 		break;
+	case ACCEL_COPY_CRC32C:
+		rc = spdk_accel_batch_prep_copy_crc32c(worker->ch, batch, task->dst, task->src, &task->crc_dst,
+						       g_crc32c_seed, g_xfer_size_bytes, accel_done, task);
+		break;
 	case ACCEL_CRC32C:
-		rc = spdk_accel_batch_prep_crc32cv(worker->ch, batch, (uint32_t *)task->dst,
+		rc = spdk_accel_batch_prep_crc32cv(worker->ch, batch, &task->crc_dst,
 						   task->iovs, task->iov_cnt, g_crc32c_seed, accel_done, task);
 		break;
 	default:
@@ -559,18 +614,26 @@ batch_done(void *cb_arg, int status)
 	spdk_thread_send_msg(worker_batch->worker->thread, _batch_done, worker_batch);
 }
 
-static uint32_t
-_update_crc32c_iov(struct iovec *iov, int iovcnt, uint32_t crc32c)
+static int
+_vector_memcmp(void *_dst, struct iovec *src_iovs, uint32_t iovcnt)
 {
-	int i;
+	uint32_t i;
+	uint32_t ttl_len = 0;
+	uint8_t *dst = (uint8_t *)_dst;
 
 	for (i = 0; i < iovcnt; i++) {
-		assert(iov[i].iov_base != NULL);
-		assert(iov[i].iov_len != 0);
-		crc32c = spdk_crc32c_update(iov[i].iov_base, iov[i].iov_len, crc32c);
-
+		if (memcmp(dst, src_iovs[i].iov_base, src_iovs[i].iov_len)) {
+			return -1;
+		}
+		dst += src_iovs[i].iov_len;
+		ttl_len += src_iovs[i].iov_len;
 	}
-	return crc32c;
+
+	if (ttl_len != iovcnt * g_xfer_size_bytes) {
+		return -1;
+	}
+
+	return 0;
 }
 
 static void
@@ -585,9 +648,20 @@ _accel_done(void *arg1)
 
 	if (g_verify && task->status == 0) {
 		switch (g_workload_selection) {
+		case ACCEL_COPY_CRC32C:
+			sw_crc32c = spdk_crc32c_iov_update(task->iovs, task->iov_cnt, ~g_crc32c_seed);
+			if (task->crc_dst != sw_crc32c) {
+				SPDK_NOTICELOG("CRC-32C miscompare\n");
+				worker->xfer_failed++;
+			}
+			if (_vector_memcmp(task->dst, task->iovs, task->iov_cnt)) {
+				SPDK_NOTICELOG("Data miscompare\n");
+				worker->xfer_failed++;
+			}
+			break;
 		case ACCEL_CRC32C:
-			sw_crc32c = _update_crc32c_iov(task->iovs, task->iov_cnt, ~g_crc32c_seed);
-			if (*(uint32_t *)task->dst != sw_crc32c) {
+			sw_crc32c = spdk_crc32c_iov_update(task->iovs, task->iov_cnt, ~g_crc32c_seed);
+			if (task->crc_dst != sw_crc32c) {
 				SPDK_NOTICELOG("CRC-32C miscompare\n");
 				worker->xfer_failed++;
 			}
@@ -634,6 +708,8 @@ _accel_done(void *arg1)
 	worker->current_queue_depth--;
 
 	if (!worker->is_draining) {
+		TAILQ_INSERT_TAIL(&worker->tasks_pool, task, link);
+		task = _get_task(worker);
 		if (g_ops_per_batch == 0) {
 			_submit_single(worker, task);
 			worker->current_queue_depth++;
@@ -740,10 +816,11 @@ _init_thread(void *arg1)
 	int i, rc, num_batches;
 	int max_per_batch;
 	int remaining = g_queue_depth;
-	int num_tasks = g_queue_depth;
+	int num_tasks = g_allocate_depth;
 	struct accel_batch *tmp;
 	struct accel_batch *worker_batch = NULL;
 	struct display_info *display = arg1;
+	uint64_t capabilities;
 
 	worker = calloc(1, sizeof(*worker));
 	if (worker == NULL) {
@@ -764,6 +841,15 @@ _init_thread(void *arg1)
 	pthread_mutex_unlock(&g_workers_lock);
 	worker->ch = spdk_accel_engine_get_io_channel();
 
+	if (g_num_workers == 1) {
+		capabilities = spdk_accel_get_capabilities(worker->ch);
+		if ((capabilities & g_workload_selection) != g_workload_selection) {
+			g_using_sw_engine = true;
+			SPDK_WARNLOG("The selected workload is not natively supported by the current engine\n");
+			SPDK_WARNLOG("The software engine will be used instead.\n\n");
+		}
+	}
+
 	TAILQ_INIT(&worker->tasks_pool);
 
 	if (g_ops_per_batch > 0) {
@@ -818,6 +904,7 @@ _init_thread(void *arg1)
 	task = worker->task_base;
 	for (i = 0; i < num_tasks; i++) {
 		TAILQ_INSERT_TAIL(&worker->tasks_pool, task, link);
+		task->worker = worker;
 		if (_get_task_data_bufs(task)) {
 			fprintf(stderr, "Unable to get data bufs\n");
 			goto error;
@@ -848,6 +935,7 @@ _init_thread(void *arg1)
 
 			for (i = 0; i < g_ops_per_batch; i++) {
 				task = _get_task(worker);
+				worker->current_queue_depth++;
 				if (task == NULL) {
 					goto error;
 				}
@@ -877,6 +965,7 @@ _init_thread(void *arg1)
 	/* Submit as singles when no batching is enabled or we ran out of batches. */
 	for (i = 0; i < remaining; i++) {
 		task = _get_task(worker);
+		worker->current_queue_depth++;
 		if (task == NULL) {
 			goto error;
 		}
@@ -908,13 +997,16 @@ accel_done(void *cb_arg, int status)
 	assert(worker);
 
 	task->status = status;
-	spdk_thread_send_msg(worker->thread, _accel_done, task);
+	if (g_using_sw_engine == false) {
+		_accel_done(task);
+	} else {
+		spdk_thread_send_msg(worker->thread, _accel_done, task);
+	}
 }
 
 static void
 accel_perf_start(void *arg1)
 {
-	struct spdk_io_channel *accel_ch;
 	struct spdk_cpuset tmp_cpumask = {};
 	char thread_name[32];
 	uint32_t i;
@@ -922,15 +1014,6 @@ accel_perf_start(void *arg1)
 	struct spdk_thread *thread;
 	struct display_info *display;
 
-	accel_ch = spdk_accel_engine_get_io_channel();
-	g_capabilites = spdk_accel_get_capabilities(accel_ch);
-	spdk_put_io_channel(accel_ch);
-
-	if ((g_capabilites & g_workload_selection) != g_workload_selection) {
-		SPDK_WARNLOG("The selected workload is not natively supported by the current engine\n");
-		SPDK_WARNLOG("The software engine will be used instead.\n\n");
-	}
-
 	g_tsc_rate = spdk_get_ticks_hz();
 	g_tsc_end = spdk_get_ticks() + g_time_in_sec * g_tsc_rate;
 
@@ -966,7 +1049,7 @@ main(int argc, char **argv)
 	pthread_mutex_init(&g_workers_lock, NULL);
 	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.reactor_mask = "0x1";
-	if (spdk_app_parse_args(argc, argv, &opts, "C:o:q:t:yw:P:f:b:T:", NULL, parse_args,
+	if (spdk_app_parse_args(argc, argv, &opts, "a:C:o:q:t:yw:P:f:b:T:", NULL, parse_args,
 				usage) != SPDK_APP_PARSE_ARGS_SUCCESS) {
 		g_rc = -1;
 		goto cleanup;
@@ -975,6 +1058,7 @@ main(int argc, char **argv)
 	if ((g_workload_selection != ACCEL_COPY) &&
 	    (g_workload_selection != ACCEL_FILL) &&
 	    (g_workload_selection != ACCEL_CRC32C) &&
+	    (g_workload_selection != ACCEL_COPY_CRC32C) &&
 	    (g_workload_selection != ACCEL_COMPARE) &&
 	    (g_workload_selection != ACCEL_DUALCAST)) {
 		usage();
@@ -989,7 +1073,18 @@ main(int argc, char **argv)
 		goto cleanup;
 	}
 
-	if (g_workload_selection == ACCEL_CRC32C &&
+	if (g_allocate_depth > 0 && g_queue_depth > g_allocate_depth) {
+		fprintf(stdout, "allocate depth must be at least as big as queue depth\n");
+		usage();
+		g_rc = -1;
+		goto cleanup;
+	}
+
+	if (g_allocate_depth == 0) {
+		g_allocate_depth = g_queue_depth;
+	}
+
+	if ((g_workload_selection == ACCEL_CRC32C || g_workload_selection == ACCEL_COPY_CRC32C) &&
 	    g_crc32c_chained_count == 0) {
 		usage();
 		g_rc = -1;

examples/bdev/fio_plugin/Makefile

@@ -40,6 +40,6 @@ FIO_PLUGIN := spdk_bdev
 
 C_SRCS = fio_plugin.c
 
-SPDK_LIB_LIST = $(ALL_MODULES_LIST) event_bdev
+SPDK_LIB_LIST = $(ALL_MODULES_LIST) event event_bdev
 
 include $(SPDK_ROOT_DIR)/mk/spdk.fio.mk

examples/bdev/fio_plugin/README.md

@@ -71,3 +71,52 @@ engine's full path via the ioengine parameter - LD_PRELOAD is recommended to avo
 When testing random workloads, it is recommended to set norandommap=1.  fio's random map
 processing consumes extra CPU cycles which will degrade performance over time with
 the fio_plugin since all I/O are submitted and completed on a single CPU core.
+
+# Zoned Block Devices
+
+SPDK has a zoned block device API (bdev_zone.h) which currently supports Open-channel SSDs,
+NVMe Zoned Namespaces (ZNS), and the virtual zoned block device SPDK module.
+
+If you wish to run fio against a SPDK zoned block device, you can use the fio option:
+
+    zonemode=zbd
+
+It is recommended to use a fio version newer than version 3.26, if using --numjobs > 1.
+If using --numjobs=1, fio version >= 3.23 should suffice.
+
+See zbd_example.fio in this directory for a zoned block device example config.
+
+## Maximum Open Zones
+
+Most zoned block devices have a resource constraint on the amount of zones which can be in an opened
+state at any point in time. It is very important to not exceed this limit.
+
+You can control how many zones fio will keep in an open state by using the
+``--max_open_zones`` option.
+
+If you use a fio version newer than 3.26, fio will automatically detect and set the proper value.
+If you use an old version of fio, make sure to provide the proper --max_open_zones value yourself.
+
+## Maximum Active Zones
+
+Zoned block devices may also have a resource constraint on the number of zones that can be active at
+any point in time. Unlike ``max_open_zones``, fio currently does not manage this constraint, and
+there is thus no option to limit it either.
+
+Since the max active zones limit (by definition) has to be greater than or equal to the max open
+zones limit, the easiest way to work around that fio does not manage this constraint, is to start
+with a clean state each run (except for read-only workloads), by resetting all zones before fio
+starts running its jobs by using the engine option:
+
+    --initial_zone_reset=1
+
+## Zone Append
+
+When running fio against a zoned block device you need to specify --iodepth=1 to avoid
+"Zone Invalid Write: The write to a zone was not at the write pointer." I/O errors.
+However, if your zoned block device supports Zone Append, you can use the engine option:
+
+    --zone_append=1
+
+To send zone append commands instead of write commands to the zoned block device.
+When using zone append, you will be able to specify a --iodepth greater than 1.

examples/bdev/fio_plugin/bdev_zoned.json

@@ -0,0 +1,26 @@
+{
+  "subsystems": [
+    {
+      "subsystem": "bdev",
+      "config": [
+        {
+          "method": "bdev_malloc_create",
+          "params": {
+            "name": "Malloc0",
+            "num_blocks": 2097152,
+            "block_size": 512
+          }
+        },
+        {
+          "method": "bdev_zone_block_create",
+          "params": {
+            "base_bdev": "Malloc0",
+            "name": "Zone0",
+            "zone_capacity": 262144,
+            "optimal_open_zones": 8
+          }
+        }
+      ]
+    }
+  ]
+}

examples/bdev/fio_plugin/fio_plugin.c

@@ -34,8 +34,10 @@
 #include "spdk/stdinc.h"
 
 #include "spdk/bdev.h"
+#include "spdk/bdev_zone.h"
 #include "spdk/accel_engine.h"
 #include "spdk/env.h"
+#include "spdk/init.h"
 #include "spdk/thread.h"
 #include "spdk/log.h"
 #include "spdk/string.h"
@@ -48,6 +50,12 @@
 #include "fio.h"
 #include "optgroup.h"
 
+#ifdef for_each_rw_ddir
+#define FIO_HAS_ZBD (FIO_IOOPS_VERSION >= 26)
+#else
+#define FIO_HAS_ZBD (0)
+#endif
+
 /* FreeBSD is missing CLOCK_MONOTONIC_RAW,
  * so alternative is provided. */
 #ifndef CLOCK_MONOTONIC_RAW /* Defined in glibc bits/time.h */
@@ -60,6 +68,8 @@ struct spdk_fio_options {
 	char *json_conf;
 	unsigned mem_mb;
 	int mem_single_seg;
+	int initial_zone_reset;
+	int zone_append;
 };
 
 struct spdk_fio_request {
@@ -71,6 +81,7 @@ struct spdk_fio_target {
 	struct spdk_bdev	*bdev;
 	struct spdk_bdev_desc	*desc;
 	struct spdk_io_channel	*ch;
+	bool zone_append_enabled;
 
 	TAILQ_ENTRY(spdk_fio_target) link;
 };
@@ -89,12 +100,24 @@ struct spdk_fio_thread {
 	TAILQ_ENTRY(spdk_fio_thread)	link;
 };
 
+struct spdk_fio_zone_cb_arg {
+	struct spdk_fio_target *target;
+	struct spdk_bdev_zone_info *spdk_zones;
+	int completed;
+	uint64_t offset_blocks;
+	struct zbd_zone *fio_zones;
+	unsigned int nr_zones;
+};
+
 static bool g_spdk_env_initialized = false;
 static const char *g_json_config_file = NULL;
 
 static int spdk_fio_init(struct thread_data *td);
 static void spdk_fio_cleanup(struct thread_data *td);
 static size_t spdk_fio_poll_thread(struct spdk_fio_thread *fio_thread);
+static int spdk_fio_handle_options(struct thread_data *td, struct fio_file *f,
+				   struct spdk_bdev *bdev);
+static int spdk_fio_handle_options_per_target(struct thread_data *td, struct fio_file *f);
 
 static pthread_t g_init_thread_id = 0;
 static pthread_mutex_t g_init_mtx = PTHREAD_MUTEX_INITIALIZER;
@@ -196,8 +219,8 @@ spdk_fio_bdev_init_start(void *arg)
 {
 	bool *done = arg;
 
-	spdk_app_json_config_load(g_json_config_file, SPDK_DEFAULT_RPC_ADDR,
-				  spdk_fio_bdev_init_done, done, true);
+	spdk_subsystem_init_from_json_config(g_json_config_file, SPDK_DEFAULT_RPC_ADDR,
+					     spdk_fio_bdev_init_done, done, true);
 }
 
 static void
@@ -455,6 +478,7 @@ spdk_fio_setup(struct thread_data *td)
 
 	for_each_file(td, f, i) {
 		struct spdk_bdev *bdev;
+		int rc;
 
 		if (strcmp(f->file_name, "*") == 0) {
 			continue;
@@ -468,7 +492,13 @@ spdk_fio_setup(struct thread_data *td)
 
 		f->real_file_size = spdk_bdev_get_num_blocks(bdev) *
 				    spdk_bdev_get_block_size(bdev);
+		f->filetype = FIO_TYPE_BLOCK;
+		fio_file_set_size_known(f);
 
+		rc = spdk_fio_handle_options(td, f, bdev);
+		if (rc) {
+			return rc;
+		}
 	}
 
 	return 0;
@@ -528,21 +558,48 @@ spdk_fio_bdev_open(void *arg)
 
 		f->engine_data = target;
 
+		rc = spdk_fio_handle_options_per_target(td, f);
+		if (rc) {
+			SPDK_ERRLOG("Failed to handle options for: %s\n", f->file_name);
+			f->engine_data = NULL;
+			spdk_put_io_channel(target->ch);
+			spdk_bdev_close(target->desc);
+			free(target);
+			fio_thread->failed = true;
+			return;
+		}
+
 		TAILQ_INSERT_TAIL(&fio_thread->targets, target, link);
 	}
 }
 
 /* Called for each thread, on that thread, shortly after the thread
  * starts.
+ *
+ * Also called by spdk_fio_report_zones(), since we need an I/O channel
+ * in order to get the zone report. (fio calls the .report_zones callback
+ * before it calls the .init callback.)
+ * Therefore, if fio was run with --zonemode=zbd, the thread will already
+ * be initialized by the time that fio calls the .init callback.
  */
 static int
 spdk_fio_init(struct thread_data *td)
 {
 	struct spdk_fio_thread *fio_thread;
+	int rc;
 
-	spdk_fio_init_thread(td);
+	/* If thread has already been initialized, do nothing. */
+	if (td->io_ops_data) {
+		return 0;
+	}
+
+	rc = spdk_fio_init_thread(td);
+	if (rc) {
+		return rc;
+	}
 
 	fio_thread = td->io_ops_data;
+	assert(fio_thread);
 	fio_thread->failed = false;
 
 	spdk_thread_send_msg(fio_thread->thread, spdk_fio_bdev_open, td);
@@ -644,6 +701,17 @@ typedef enum fio_q_status fio_q_status_t;
 typedef int fio_q_status_t;
 #endif
 
+static uint64_t
+spdk_fio_zone_bytes_to_blocks(struct spdk_bdev *bdev, uint64_t offset_bytes, uint64_t *zone_start,
+			      uint64_t num_bytes, uint64_t *num_blocks)
+{
+	uint32_t block_size = spdk_bdev_get_block_size(bdev);
+	*zone_start = (offset_bytes / (spdk_bdev_get_zone_size(bdev) * block_size)) *
+		      spdk_bdev_get_zone_size(bdev);
+	*num_blocks = num_bytes / block_size;
+	return (offset_bytes % block_size) | (num_bytes % block_size);
+}
+
 static fio_q_status_t
 spdk_fio_queue(struct thread_data *td, struct io_u *io_u)
 {
@@ -666,9 +734,21 @@ spdk_fio_queue(struct thread_data *td, struct io_u *io_u)
 				    spdk_fio_completion_cb, fio_req);
 		break;
 	case DDIR_WRITE:
-		rc = spdk_bdev_write(target->desc, target->ch,
-				     io_u->buf, io_u->offset, io_u->xfer_buflen,
-				     spdk_fio_completion_cb, fio_req);
+		if (!target->zone_append_enabled) {
+			rc = spdk_bdev_write(target->desc, target->ch,
+					     io_u->buf, io_u->offset, io_u->xfer_buflen,
+					     spdk_fio_completion_cb, fio_req);
+		} else {
+			uint64_t zone_start, num_blocks;
+			if (spdk_fio_zone_bytes_to_blocks(target->bdev, io_u->offset, &zone_start,
+							  io_u->xfer_buflen, &num_blocks) != 0) {
+				rc = -EINVAL;
+				break;
+			}
+			rc = spdk_bdev_zone_append(target->desc, target->ch, io_u->buf,
+						   zone_start, num_blocks, spdk_fio_completion_cb,
+						   fio_req);
+		}
 		break;
 	case DDIR_TRIM:
 		rc = spdk_bdev_unmap(target->desc, target->ch,
@@ -755,6 +835,308 @@ spdk_fio_invalidate(struct thread_data *td, struct fio_file *f)
 	return 0;
 }
 
+#if FIO_HAS_ZBD
+static int
+spdk_fio_get_zoned_model(struct thread_data *td, struct fio_file *f, enum zbd_zoned_model *model)
+{
+	struct spdk_bdev *bdev;
+
+	if (f->filetype != FIO_TYPE_BLOCK) {
+		SPDK_ERRLOG("Unsupported filetype: %d\n", f->filetype);
+		return -EINVAL;
+	}
+
+	bdev = spdk_bdev_get_by_name(f->file_name);
+	if (!bdev) {
+		SPDK_ERRLOG("Cannot get zoned model, no bdev with name: %s\n", f->file_name);
+		return -ENODEV;
+	}
+
+	if (spdk_bdev_is_zoned(bdev)) {
+		*model = ZBD_HOST_MANAGED;
+	} else {
+		*model = ZBD_NONE;
+	}
+
+	return 0;
+}
+
+static void
+spdk_fio_bdev_get_zone_info_done(struct spdk_bdev_io *bdev_io, bool success, void *arg)
+{
+	struct spdk_fio_zone_cb_arg *cb_arg = arg;
+	unsigned int i;
+	int handled_zones = 0;
+
+	if (!success) {
+		spdk_bdev_free_io(bdev_io);
+		cb_arg->completed = -EIO;
+		return;
+	}
+
+	for (i = 0; i < cb_arg->nr_zones; i++) {
+		struct spdk_bdev_zone_info *zone_src = &cb_arg->spdk_zones[handled_zones];
+		struct zbd_zone *zone_dest = &cb_arg->fio_zones[handled_zones];
+		uint32_t block_size = spdk_bdev_get_block_size(cb_arg->target->bdev);
+
+		zone_dest->type = ZBD_ZONE_TYPE_SWR;
+		zone_dest->len = spdk_bdev_get_zone_size(cb_arg->target->bdev) * block_size;
+		zone_dest->capacity = zone_src->capacity * block_size;
+		zone_dest->start = zone_src->zone_id * block_size;
+		zone_dest->wp = zone_src->write_pointer * block_size;
+
+		switch (zone_src->state) {
+		case SPDK_BDEV_ZONE_STATE_EMPTY:
+			zone_dest->cond = ZBD_ZONE_COND_EMPTY;
+			break;
+		case SPDK_BDEV_ZONE_STATE_IMP_OPEN:
+			zone_dest->cond = ZBD_ZONE_COND_IMP_OPEN;
+			break;
+		case SPDK_BDEV_ZONE_STATE_EXP_OPEN:
+			zone_dest->cond = ZBD_ZONE_COND_EXP_OPEN;
+			break;
+		case SPDK_BDEV_ZONE_STATE_FULL:
+			zone_dest->cond = ZBD_ZONE_COND_FULL;
+			break;
+		case SPDK_BDEV_ZONE_STATE_CLOSED:
+			zone_dest->cond = ZBD_ZONE_COND_CLOSED;
+			break;
+		case SPDK_BDEV_ZONE_STATE_READ_ONLY:
+			zone_dest->cond = ZBD_ZONE_COND_READONLY;
+			break;
+		case SPDK_BDEV_ZONE_STATE_OFFLINE:
+			zone_dest->cond = ZBD_ZONE_COND_OFFLINE;
+			break;
+		default:
+			spdk_bdev_free_io(bdev_io);
+			cb_arg->completed = -EIO;
+			return;
+		}
+		handled_zones++;
+	}
+
+	spdk_bdev_free_io(bdev_io);
+	cb_arg->completed = handled_zones;
+}
+
+static void
+spdk_fio_bdev_get_zone_info(void *arg)
+{
+	struct spdk_fio_zone_cb_arg *cb_arg = arg;
+	struct spdk_fio_target *target = cb_arg->target;
+	int rc;
+
+	rc = spdk_bdev_get_zone_info(target->desc, target->ch, cb_arg->offset_blocks,
+				     cb_arg->nr_zones, cb_arg->spdk_zones,
+				     spdk_fio_bdev_get_zone_info_done, cb_arg);
+	if (rc < 0) {
+		cb_arg->completed = rc;
+	}
+}
+
+static int
+spdk_fio_report_zones(struct thread_data *td, struct fio_file *f, uint64_t offset,
+		      struct zbd_zone *zones, unsigned int nr_zones)
+{
+	struct spdk_fio_target *target;
+	struct spdk_fio_thread *fio_thread;
+	struct spdk_fio_zone_cb_arg cb_arg;
+	uint32_t block_size;
+	int rc;
+
+	if (nr_zones == 0) {
+		return 0;
+	}
+
+	/* spdk_fio_report_zones() is only called before the bdev I/O channels have been created.
+	 * Since we need an I/O channel for report_zones(), call spdk_fio_init() to initialize
+	 * the thread early.
+	 * spdk_fio_report_zones() might be called several times by fio, if e.g. the zone report
+	 * for all zones does not fit in the buffer that fio has allocated for the zone report.
+	 * It is safe to call spdk_fio_init(), even if the thread has already been initialized.
+	 */
+	rc = spdk_fio_init(td);
+	if (rc) {
+		return rc;
+	}
+	fio_thread = td->io_ops_data;
+	target = f->engine_data;
+
+	assert(fio_thread);
+	assert(target);
+
+	block_size = spdk_bdev_get_block_size(target->bdev);
+
+	cb_arg.target = target;
+	cb_arg.completed = 0;
+	cb_arg.offset_blocks = offset / block_size;
+	cb_arg.fio_zones = zones;
+	cb_arg.nr_zones = spdk_min(nr_zones, spdk_bdev_get_num_zones(target->bdev));
+
+	cb_arg.spdk_zones = calloc(1, sizeof(*cb_arg.spdk_zones) * cb_arg.nr_zones);
+	if (!cb_arg.spdk_zones) {
+		SPDK_ERRLOG("Could not allocate memory for zone report!\n");
+		rc = -ENOMEM;
+		goto cleanup_thread;
+	}
+
+	spdk_thread_send_msg(fio_thread->thread, spdk_fio_bdev_get_zone_info, &cb_arg);
+	do {
+		spdk_fio_poll_thread(fio_thread);
+	} while (!cb_arg.completed);
+
+	/* Free cb_arg.spdk_zones. The report in fio format is stored in cb_arg.fio_zones/zones. */
+	free(cb_arg.spdk_zones);
+
+	rc = cb_arg.completed;
+	if (rc < 0) {
+		SPDK_ERRLOG("Failed to get zone info: %d\n", rc);
+		goto cleanup_thread;
+	}
+
+	/* Return the amount of zones successfully copied. */
+	return rc;
+
+cleanup_thread:
+	spdk_fio_cleanup(td);
+
+	return rc;
+}
+
+static void
+spdk_fio_bdev_zone_reset_done(struct spdk_bdev_io *bdev_io, bool success, void *arg)
+{
+	struct spdk_fio_zone_cb_arg *cb_arg = arg;
+
+	spdk_bdev_free_io(bdev_io);
+
+	if (!success) {
+		cb_arg->completed = -EIO;
+	} else {
+		cb_arg->completed = 1;
+	}
+}
+
+static void
+spdk_fio_bdev_zone_reset(void *arg)
+{
+	struct spdk_fio_zone_cb_arg *cb_arg = arg;
+	struct spdk_fio_target *target = cb_arg->target;
+	int rc;
+
+	rc = spdk_bdev_zone_management(target->desc, target->ch, cb_arg->offset_blocks,
+				       SPDK_BDEV_ZONE_RESET,
+				       spdk_fio_bdev_zone_reset_done, cb_arg);
+	if (rc < 0) {
+		cb_arg->completed = rc;
+	}
+}
+
+static int
+spdk_fio_reset_zones(struct spdk_fio_thread *fio_thread, struct spdk_fio_target *target,
+		     uint64_t offset, uint64_t length)
+{
+	uint64_t zone_size_bytes;
+	uint32_t block_size;
+	int rc;
+
+	assert(fio_thread);
+	assert(target);
+
+	block_size = spdk_bdev_get_block_size(target->bdev);
+	zone_size_bytes = spdk_bdev_get_zone_size(target->bdev) * block_size;
+
+	for (uint64_t cur = offset; cur < offset + length; cur += zone_size_bytes) {
+		struct spdk_fio_zone_cb_arg cb_arg = {
+			.target = target,
+			.completed = 0,
+			.offset_blocks = cur / block_size,
+		};
+
+		spdk_thread_send_msg(fio_thread->thread, spdk_fio_bdev_zone_reset, &cb_arg);
+		do {
+			spdk_fio_poll_thread(fio_thread);
+		} while (!cb_arg.completed);
+
+		rc = cb_arg.completed;
+		if (rc < 0) {
+			SPDK_ERRLOG("Failed to reset zone: %d\n", rc);
+			return rc;
+		}
+	}
+
+	return 0;
+}
+
+static int
+spdk_fio_reset_wp(struct thread_data *td, struct fio_file *f, uint64_t offset, uint64_t length)
+{
+	return spdk_fio_reset_zones(td->io_ops_data, f->engine_data, offset, length);
+}
+#endif
+
+#if FIO_IOOPS_VERSION >= 30
+static int spdk_fio_get_max_open_zones(struct thread_data *td, struct fio_file *f,
+				       unsigned int *max_open_zones)
+{
+	struct spdk_bdev *bdev;
+
+	bdev = spdk_bdev_get_by_name(f->file_name);
+	if (!bdev) {
+		SPDK_ERRLOG("Cannot get max open zones, no bdev with name: %s\n", f->file_name);
+		return -ENODEV;
+	}
+
+	*max_open_zones = spdk_bdev_get_max_open_zones(bdev);
+
+	return 0;
+}
+#endif
+
+static int
+spdk_fio_handle_options(struct thread_data *td, struct fio_file *f, struct spdk_bdev *bdev)
+{
+	struct spdk_fio_options *fio_options = td->eo;
+
+	if (fio_options->initial_zone_reset && spdk_bdev_is_zoned(bdev)) {
+#if FIO_HAS_ZBD
+		int rc = spdk_fio_init(td);
+		if (rc) {
+			return rc;
+		}
+		rc = spdk_fio_reset_zones(td->io_ops_data, f->engine_data, 0, f->real_file_size);
+		if (rc) {
+			spdk_fio_cleanup(td);
+			return rc;
+		}
+#else
+		SPDK_ERRLOG("fio version is too old to support zoned block devices\n");
+#endif
+	}
+
+	return 0;
+}
+
+static int
+spdk_fio_handle_options_per_target(struct thread_data *td, struct fio_file *f)
+{
+	struct spdk_fio_target *target = f->engine_data;
+	struct spdk_fio_options *fio_options = td->eo;
+
+	if (fio_options->zone_append && spdk_bdev_is_zoned(target->bdev)) {
+		if (spdk_bdev_io_type_supported(target->bdev, SPDK_BDEV_IO_TYPE_ZONE_APPEND)) {
+			SPDK_DEBUGLOG(fio_bdev, "Using zone appends instead of writes on: '%s'\n",
+				      f->file_name);
+			target->zone_append_enabled = true;
+		} else {
+			SPDK_WARNLOG("Falling back to writes on: '%s' - bdev lacks zone append cmd\n",
+				     f->file_name);
+		}
+	}
+
+	return 0;
+}
+
 static struct fio_option options[] = {
 	{
 		.name		= "spdk_conf",
@@ -793,6 +1175,26 @@ static struct fio_option options[] = {
 		.category	= FIO_OPT_C_ENGINE,
 		.group		= FIO_OPT_G_INVALID,
 	},
+	{
+		.name		= "initial_zone_reset",
+		.lname		= "Reset Zones on initialization",
+		.type		= FIO_OPT_INT,
+		.off1		= offsetof(struct spdk_fio_options, initial_zone_reset),
+		.def		= "0",
+		.help		= "Reset Zones on initialization (0=disable, 1=Reset All Zones)",
+		.category	= FIO_OPT_C_ENGINE,
+		.group		= FIO_OPT_G_INVALID,
+	},
+	{
+		.name		= "zone_append",
+		.lname		= "Use zone append instead of write",
+		.type		= FIO_OPT_INT,
+		.off1		= offsetof(struct spdk_fio_options, zone_append),
+		.def		= "0",
+		.help		= "Use zone append instead of write (1=zone append, 0=write)",
+		.category	= FIO_OPT_C_ENGINE,
+		.group		= FIO_OPT_G_INVALID,
+	},
 	{
 		.name		= NULL,
 	},
@@ -823,6 +1225,14 @@ struct ioengine_ops ioengine = {
 	.iomem_free		= spdk_fio_iomem_free,
 	.io_u_init		= spdk_fio_io_u_init,
 	.io_u_free		= spdk_fio_io_u_free,
+#if FIO_HAS_ZBD
+	.get_zoned_model	= spdk_fio_get_zoned_model,
+	.report_zones		= spdk_fio_report_zones,
+	.reset_wp		= spdk_fio_reset_wp,
+#endif
+#if FIO_IOOPS_VERSION >= 30
+	.get_max_open_zones	= spdk_fio_get_max_open_zones,
+#endif
 	.option_struct_size	= sizeof(struct spdk_fio_options),
 	.options		= options,
 };
@@ -852,3 +1262,5 @@ static void fio_exit spdk_fio_unregister(void)
 	}
 	unregister_ioengine(&ioengine);
 }
+
+SPDK_LOG_REGISTER_COMPONENT(fio_bdev)

examples/bdev/fio_plugin/zbd_example.fio

@@ -0,0 +1,18 @@
+[global]
+ioengine=spdk_bdev
+spdk_json_conf=./examples/bdev/fio_plugin/bdev_zoned.json
+thread=1
+direct=1
+time_based
+runtime=10
+rw=randwrite
+bs=16K
+zonemode=zbd
+max_open_zones=8
+initial_zone_reset=1
+zone_append=1
+iodepth=64
+
+[test]
+filename=Zone0
+numjobs=1

examples/bdev/hello_world/Makefile

@@ -37,6 +37,6 @@ APP = hello_bdev
 
 C_SRCS := hello_bdev.c
 
-SPDK_LIB_LIST = $(ALL_MODULES_LIST) event_bdev
+SPDK_LIB_LIST = $(ALL_MODULES_LIST) event event_bdev
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

examples/blob/cli/Makefile

@@ -38,6 +38,6 @@ APP = blobcli
 C_SRCS := blobcli.c
 
 # Don't link bdev_lvol in blobcli - otherwise this utility cannot operate on an lvolstore
-SPDK_LIB_LIST = $(filter-out bdev_lvol,$(ALL_MODULES_LIST)) event_bdev
+SPDK_LIB_LIST = $(filter-out bdev_lvol,$(ALL_MODULES_LIST)) event event_bdev
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

examples/blob/hello_world/Makefile

@@ -37,6 +37,6 @@ APP = hello_blob
 
 C_SRCS := hello_blob.c
 
-SPDK_LIB_LIST = $(ALL_MODULES_LIST) event_bdev
+SPDK_LIB_LIST = $(ALL_MODULES_LIST) event event_bdev
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

examples/blob/hello_world/hello_blob.c

@@ -415,7 +415,7 @@ hello_start(void *arg1)
 
 	/*
 	 * In this example, use our malloc (RAM) disk configured via
-	 * hello_blob.conf that was passed in when we started the
+	 * hello_blob.json that was passed in when we started the
 	 * SPDK app framework.
 	 *
 	 * spdk_bs_init() requires us to fill out the structure

examples/interrupt_tgt/Makefile

@@ -39,7 +39,7 @@ APP = interrupt_tgt
 
 C_SRCS := interrupt_tgt.c
 
-SPDK_LIB_LIST = $(INTR_BLOCKDEV_MODULES_LIST) event_bdev conf
+SPDK_LIB_LIST = $(INTR_BLOCKDEV_MODULES_LIST) event event_bdev conf
 
 SPDK_LIB_LIST += event_nbd
 SPDK_LIB_LIST += event_vhost

examples/interrupt_tgt/interrupt_tgt.c

@@ -78,6 +78,14 @@ rpc_reactor_set_interrupt_mode(struct spdk_jsonrpc_request *request,
 		return;
 	}
 
+	if (!spdk_interrupt_mode_is_enabled()) {
+		SPDK_ERRLOG("Interrupt mode is not set when staring the application\n");
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+						 "spdk_json_decode_object failed");
+		return;
+	}
+
+
 	SPDK_NOTICELOG("RPC Start to %s interrupt mode on reactor %d.\n",
 		       req.disable_interrupt ? "disable" : "enable", req.lcore);
 	if (req.lcore >= (int64_t)spdk_env_get_first_core() &&

examples/nvme/abort/abort.c

@@ -912,8 +912,8 @@ unregister_controllers(void)
 		free(entry);
 	}
 
-	while (detach_ctx && spdk_nvme_detach_poll_async(detach_ctx) == -EAGAIN) {
-		;
+	if (detach_ctx) {
+		spdk_nvme_detach_poll(detach_ctx);
 	}
 }
 

examples/nvme/fio_plugin/README.md

@@ -125,7 +125,8 @@ Zoned Namespaces has a resource constraint on the amount of zones which can be i
 any point in time. You can control how many zones fio will keep in an open state by using the
 ``--max_open_zones`` option.
 
-The SPDK/NVMe fio io-engine will set a default value if you do not provide one.
+If you use a fio version newer than 3.26, fio will automatically detect and set the proper value.
+If you use an old version of fio, make sure to provide the proper --max_open_zones value yourself.
 
 ## Maximum Active Zones
 

examples/nvme/fio_plugin/fio_plugin.c

@@ -241,7 +241,6 @@ get_fio_ctrlr(const struct spdk_nvme_transport_id *trid)
 	return NULL;
 }
 
-#if FIO_HAS_ZBD
 /**
  * Returns the fio_qpair matching the given fio_file and has an associated ns
  */
@@ -259,6 +258,7 @@ get_fio_qpair(struct spdk_fio_thread *fio_thread, struct fio_file *f)
 	return NULL;
 }
 
+#if FIO_HAS_ZBD
 /**
  * Callback function to use while processing completions until completion-indicator turns non-zero
  */
@@ -426,12 +426,12 @@ attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
 
 	if (fio_options->zone_append && spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS) {
 		if (spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ZONE_APPEND_SUPPORTED) {
-			fprintf(stdout, "Using zone appends instead of writes on: '%s'\n",
-				fio_qpair->f->file_name);
+			SPDK_DEBUGLOG(fio_nvme, "Using zone appends instead of writes on: '%s'\n",
+				      f->file_name);
 			fio_qpair->zone_append_enabled = true;
 		} else {
 			SPDK_WARNLOG("Falling back to writes on: '%s' - ns lacks zone append cmd\n",
-				     fio_qpair->f->file_name);
+				     f->file_name);
 		}
 	}
 
@@ -988,16 +988,12 @@ spdk_fio_queue(struct thread_data *td, struct io_u *io_u)
 	uint64_t		lba;
 	uint32_t		lba_count;
 
-	/* Find the namespace that corresponds to the file in the io_u */
-	TAILQ_FOREACH(fio_qpair, &fio_thread->fio_qpair, link) {
-		if (fio_qpair->f == io_u->file) {
-			ns = fio_qpair->ns;
-			break;
-		}
-	}
-	if (fio_qpair == NULL || ns == NULL) {
+	fio_qpair = get_fio_qpair(fio_thread, io_u->file);
+	if (fio_qpair == NULL) {
 		return -ENXIO;
 	}
+	ns = fio_qpair->ns;
+
 	if (fio_qpair->nvme_pi_enabled && !fio_qpair->extended_lba) {
 		md_buf = fio_req->md_buf;
 	}
@@ -1041,7 +1037,7 @@ spdk_fio_queue(struct thread_data *td, struct io_u *io_u)
 								    spdk_fio_completion_cb, fio_req,
 								    fio_qpair->io_flags, dif_ctx->apptag_mask, dif_ctx->app_tag);
 			} else {
-				uint64_t zslba = fio_offset_to_zslba(io_u->offset, fio_qpair->ns);
+				uint64_t zslba = fio_offset_to_zslba(io_u->offset, ns);
 				rc = spdk_nvme_zns_zone_append_with_md(ns, fio_qpair->qpair, io_u->buf, md_buf, zslba,
 								       lba_count,
 								       spdk_fio_completion_cb, fio_req,
@@ -1054,7 +1050,7 @@ spdk_fio_queue(struct thread_data *td, struct io_u *io_u)
 								     spdk_nvme_io_reset_sgl, spdk_nvme_io_next_sge, md_buf,
 								     dif_ctx->apptag_mask, dif_ctx->app_tag);
 			} else {
-				uint64_t zslba = fio_offset_to_zslba(io_u->offset, fio_qpair->ns);
+				uint64_t zslba = fio_offset_to_zslba(io_u->offset, ns);
 				rc = spdk_nvme_zns_zone_appendv_with_md(ns, fio_qpair->qpair, zslba,
 									lba_count, spdk_fio_completion_cb, fio_req, fio_qpair->io_flags,
 									spdk_nvme_io_reset_sgl, spdk_nvme_io_next_sge, md_buf,
@@ -1157,13 +1153,9 @@ spdk_fio_get_zoned_model(struct thread_data *td, struct fio_file *f, enum zbd_zo
 	struct spdk_fio_qpair *fio_qpair = NULL;
 	const struct spdk_nvme_zns_ns_data *zns_data = NULL;
 
-	*model = ZBD_IGNORE;
-
-	if (f->filetype != FIO_TYPE_FILE && \
-	    f->filetype != FIO_TYPE_BLOCK && \
-	    f->filetype != FIO_TYPE_CHAR) {
-		log_info("spdk/nvme: ignoring filetype: %d\n", f->filetype);
-		return 0;
+	if (f->filetype != FIO_TYPE_BLOCK) {
+		log_info("spdk/nvme: unsupported filetype: %d\n", f->filetype);
+		return -EINVAL;
 	}
 
 	fio_qpair = get_fio_qpair(fio_thread, f);
@@ -1190,29 +1182,6 @@ spdk_fio_get_zoned_model(struct thread_data *td, struct fio_file *f, enum zbd_zo
 
 		*model = ZBD_HOST_MANAGED;
 
-		/** Unlimited open resources, skip checking 'max_open_zones' */
-		if (0xFFFFFFFF == zns_data->mor) {
-			return 0;
-		}
-
-		if (!td->o.max_open_zones) {
-			td->o.max_open_zones = spdk_min(ZBD_MAX_OPEN_ZONES, zns_data->mor + 1);
-			log_info("spdk/nvme: parameter 'max_open_zones' was unset; assigned: %d.\n",
-				 td->o.max_open_zones);
-		} else if (td->o.max_open_zones < 0) {
-			log_err("spdk/nvme: invalid parameter 'max_open_zones': %d\n",
-				td->o.max_open_zones);
-			return -EINVAL;
-		} else if (td->o.max_open_zones > ZBD_MAX_OPEN_ZONES) {
-			log_err("spdk/nvme: parameter 'max_open_zones': %d exceeds fio-limit: %d\n",
-				td->o.max_open_zones, ZBD_MAX_OPEN_ZONES);
-			return -EINVAL;
-		} else if ((uint32_t)td->o.max_open_zones > (zns_data->mor + 1)) {
-			log_err("spdk/nvme: parameter 'max_open_zones': %d exceeds dev-limit: %u\n",
-				td->o.max_open_zones, zns_data->mor + 1);
-			return -EINVAL;
-		}
-
 		return 0;
 	}
 
@@ -1382,6 +1351,25 @@ spdk_fio_reset_wp(struct thread_data *td, struct fio_file *f, uint64_t offset, u
 }
 #endif
 
+#if FIO_IOOPS_VERSION >= 30
+static int spdk_fio_get_max_open_zones(struct thread_data *td, struct fio_file *f,
+				       unsigned int *max_open_zones)
+{
+	struct spdk_fio_thread *fio_thread = td->io_ops_data;
+	struct spdk_fio_qpair *fio_qpair = NULL;
+
+	fio_qpair = get_fio_qpair(fio_thread, f);
+	if (!fio_qpair) {
+		log_err("spdk/nvme: no ns/qpair or file_name: '%s'\n", f->file_name);
+		return -ENODEV;
+	}
+
+	*max_open_zones = spdk_nvme_zns_ns_get_max_open_zones(fio_qpair->ns);
+
+	return 0;
+}
+#endif
+
 static void spdk_fio_cleanup(struct thread_data *td)
 {
 	struct spdk_fio_thread	*fio_thread = td->io_ops_data;
@@ -1409,8 +1397,8 @@ static void spdk_fio_cleanup(struct thread_data *td)
 			free(fio_ctrlr);
 		}
 
-		while (detach_ctx && spdk_nvme_detach_poll_async(detach_ctx) == -EAGAIN) {
-			;
+		if (detach_ctx) {
+			spdk_nvme_detach_poll(detach_ctx);
 		}
 
 		if (fio_options->enable_vmd) {
@@ -1634,7 +1622,7 @@ static struct fio_option options[] = {
 		.type		= FIO_OPT_INT,
 		.off1		= offsetof(struct spdk_fio_options, zone_append),
 		.def		= "0",
-		.help		= "Use zone append instead of write (zone_append=1 or zone_append=0)",
+		.help		= "Use zone append instead of write (1=zone append, 0=write)",
 		.category	= FIO_OPT_C_ENGINE,
 		.group		= FIO_OPT_G_INVALID,
 	},
@@ -1673,6 +1661,9 @@ struct ioengine_ops ioengine = {
 	.get_zoned_model	= spdk_fio_get_zoned_model,
 	.report_zones		= spdk_fio_report_zones,
 	.reset_wp		= spdk_fio_reset_wp,
+#endif
+#if FIO_IOOPS_VERSION >= 30
+	.get_max_open_zones	= spdk_fio_get_max_open_zones,
 #endif
 	.flags			= FIO_RAWIO | FIO_NOEXTEND | FIO_NODISKUTIL | FIO_MEMALIGN,
 	.options		= options,
@@ -1688,3 +1679,5 @@ static void fio_exit fio_spdk_unregister(void)
 {
 	unregister_ioengine(&ioengine);
 }
+
+SPDK_LOG_REGISTER_COMPONENT(fio_nvme)

examples/nvme/hello_world/hello_world.c

@@ -386,8 +386,8 @@ cleanup(void)
 		free(ctrlr_entry);
 	}
 
-	while (detach_ctx && spdk_nvme_detach_poll_async(detach_ctx) == -EAGAIN) {
-		;
+	if (detach_ctx) {
+		spdk_nvme_detach_poll(detach_ctx);
 	}
 }
 

examples/nvme/hotplug/hotplug.c

@@ -73,6 +73,7 @@ static int g_insert_times;
 static int g_removal_times;
 static int g_shm_id = -1;
 static uint64_t g_timeout_in_us = SPDK_SEC_TO_USEC;
+static struct spdk_nvme_detach_ctx *g_detach_ctx;
 
 static void
 task_complete(struct perf_task *task);
@@ -100,7 +101,8 @@ register_dev(struct spdk_nvme_ctrlr *ctrlr)
 	dev->is_removed = false;
 	dev->is_draining = false;
 
-	spdk_nvme_ctrlr_register_timeout_callback(ctrlr, g_timeout_in_us, timeout_cb, NULL);
+	spdk_nvme_ctrlr_register_timeout_callback(ctrlr, g_timeout_in_us, g_timeout_in_us, timeout_cb,
+			NULL);
 
 	dev->ns = spdk_nvme_ctrlr_get_ns(ctrlr, 1);
 
@@ -142,7 +144,7 @@ unregister_dev(struct dev_ctx *dev)
 	fprintf(stderr, "unregister_dev: %s\n", dev->name);
 
 	spdk_nvme_ctrlr_free_io_qpair(dev->qpair);
-	spdk_nvme_detach(dev->ctrlr);
+	spdk_nvme_detach_async(dev->ctrlr, &g_detach_ctx);
 
 	TAILQ_REMOVE(&g_devs, dev, tailq);
 	free(dev);
@@ -319,7 +321,7 @@ remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr)
 	 * in g_devs (for example, because we skipped it during register_dev),
 	 * so immediately detach it.
 	 */
-	spdk_nvme_detach(ctrlr);
+	spdk_nvme_detach_async(ctrlr, &g_detach_ctx);
 }
 
 static void
@@ -338,6 +340,7 @@ io_loop(void)
 	struct dev_ctx *dev, *dev_tmp;
 	uint64_t tsc_end;
 	uint64_t next_stats_tsc;
+	int rc;
 
 	tsc_end = spdk_get_ticks() + g_time_in_sec * g_tsc_rate;
 	next_stats_tsc = spdk_get_ticks();
@@ -382,6 +385,13 @@ io_loop(void)
 			}
 		}
 
+		if (g_detach_ctx) {
+			rc = spdk_nvme_detach_poll_async(g_detach_ctx);
+			if (rc == 0) {
+				g_detach_ctx = NULL;
+			}
+		}
+
 		now = spdk_get_ticks();
 		if (now > tsc_end) {
 			break;
@@ -400,6 +410,10 @@ io_loop(void)
 		drain_io(dev);
 		unregister_dev(dev);
 	}
+
+	if (g_detach_ctx) {
+		spdk_nvme_detach_poll(g_detach_ctx);
+	}
 }
 
 static void usage(char *program_name)

examples/nvme/identify/identify.c

@@ -70,6 +70,8 @@ static struct spdk_nvme_firmware_page firmware_page;
 
 static struct spdk_nvme_ana_page *g_ana_log_page;
 
+static struct spdk_nvme_ana_group_descriptor *g_copied_ana_desc;
+
 static size_t g_ana_log_page_size;
 
 static struct spdk_nvme_cmds_and_effect_log_page cmd_effects_log_page;
@@ -100,7 +102,7 @@ static bool g_dpdk_mem_single_seg = false;
 
 static int g_main_core = 0;
 
-static char g_core_mask[16] = "0x1";
+static char g_core_mask[20] = "0x1";
 
 static struct spdk_nvme_transport_id g_trid;
 static char g_hostnqn[SPDK_NVMF_NQN_MAX_LEN + 1];
@@ -111,6 +113,8 @@ static bool g_vmd = false;
 
 static bool g_ocssd_verbose = false;
 
+static struct spdk_nvme_detach_ctx *g_detach_ctx = NULL;
+
 static void
 hex_dump(const void *data, size_t size)
 {
@@ -201,7 +205,7 @@ get_zns_zone_report_completion(void *cb_arg, const struct spdk_nvme_cpl *cpl)
 }
 
 static int
-get_feature(struct spdk_nvme_ctrlr *ctrlr, uint8_t fid)
+get_feature(struct spdk_nvme_ctrlr *ctrlr, uint8_t fid, uint32_t nsid)
 {
 	struct spdk_nvme_cmd cmd = {};
 	struct feature *feature = &features[fid];
@@ -210,35 +214,28 @@ get_feature(struct spdk_nvme_ctrlr *ctrlr, uint8_t fid)
 
 	cmd.opc = SPDK_NVME_OPC_GET_FEATURES;
 	cmd.cdw10_bits.get_features.fid = fid;
+	cmd.nsid = nsid;
 
 	return spdk_nvme_ctrlr_cmd_admin_raw(ctrlr, &cmd, NULL, 0, get_feature_completion, feature);
 }
 
 static void
-get_features(struct spdk_nvme_ctrlr *ctrlr)
+get_features(struct spdk_nvme_ctrlr *ctrlr, uint8_t *features_to_get, size_t num_features,
+	     uint32_t nsid)
 {
 	size_t i;
 
-	uint8_t features_to_get[] = {
-		SPDK_NVME_FEAT_ARBITRATION,
-		SPDK_NVME_FEAT_POWER_MANAGEMENT,
-		SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD,
-		SPDK_NVME_FEAT_ERROR_RECOVERY,
-		SPDK_NVME_FEAT_NUMBER_OF_QUEUES,
-		SPDK_OCSSD_FEAT_MEDIA_FEEDBACK,
-	};
-
 	/* Submit only one GET FEATURES at a time. There is a known issue #1799
 	 * with Google Cloud Platform NVMe SSDs that do not handle overlapped
 	 * GET FEATURES commands correctly.
 	 */
 	outstanding_commands = 0;
-	for (i = 0; i < SPDK_COUNTOF(features_to_get); i++) {
+	for (i = 0; i < num_features; i++) {
 		if (!spdk_nvme_ctrlr_is_ocssd_supported(ctrlr) &&
 		    features_to_get[i] == SPDK_OCSSD_FEAT_MEDIA_FEEDBACK) {
 			continue;
 		}
-		if (get_feature(ctrlr, features_to_get[i]) == 0) {
+		if (get_feature(ctrlr, features_to_get[i], nsid) == 0) {
 			outstanding_commands++;
 		} else {
 			printf("get_feature(0x%02X) failed to submit command\n", features_to_get[i]);
@@ -251,6 +248,30 @@ get_features(struct spdk_nvme_ctrlr *ctrlr)
 
 }
 
+static void
+get_ctrlr_features(struct spdk_nvme_ctrlr *ctrlr)
+{
+	uint8_t features_to_get[] = {
+		SPDK_NVME_FEAT_ARBITRATION,
+		SPDK_NVME_FEAT_POWER_MANAGEMENT,
+		SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD,
+		SPDK_NVME_FEAT_NUMBER_OF_QUEUES,
+		SPDK_OCSSD_FEAT_MEDIA_FEEDBACK,
+	};
+
+	get_features(ctrlr, features_to_get, SPDK_COUNTOF(features_to_get), 0);
+}
+
+static void
+get_ns_features(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid)
+{
+	uint8_t features_to_get[] = {
+		SPDK_NVME_FEAT_ERROR_RECOVERY,
+	};
+
+	get_features(ctrlr, features_to_get, SPDK_COUNTOF(features_to_get), nsid);
+}
+
 static int
 get_error_log_page(struct spdk_nvme_ctrlr *ctrlr)
 {
@@ -496,6 +517,10 @@ get_log_pages(struct spdk_nvme_ctrlr *ctrlr)
 		if (g_ana_log_page == NULL) {
 			exit(1);
 		}
+		g_copied_ana_desc = calloc(1, g_ana_log_page_size);
+		if (g_copied_ana_desc == NULL) {
+			exit(1);
+		}
 		if (get_ana_log_page(ctrlr) == 0) {
 			outstanding_commands++;
 		} else {
@@ -942,6 +967,16 @@ print_namespace(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns)
 	/* This function is only called for active namespaces. */
 	assert(spdk_nvme_ns_is_active(ns));
 
+	if (features[SPDK_NVME_FEAT_ERROR_RECOVERY].valid) {
+		unsigned tler = features[SPDK_NVME_FEAT_ERROR_RECOVERY].result & 0xFFFF;
+		printf("Error Recovery Timeout:                ");
+		if (tler == 0) {
+			printf("Unlimited\n");
+		} else {
+			printf("%u milliseconds\n", tler * 100);
+		}
+	}
+
 	printf("Command Set Identifier:                %s (%02Xh)\n",
 	       csi_name(spdk_nvme_ns_get_csi(ns)), spdk_nvme_ns_get_csi(ns));
 	printf("Deallocate:                            %s\n",
@@ -1014,6 +1049,10 @@ print_namespace(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns)
 			printf("  Atomic Write Unit (PFail):           %d\n", nsdata->nawupf + 1);
 		}
 
+		if (nsdata->npwg) {
+			printf("  Preferred Write Granularity:         %d\n", nsdata->npwg + 1);
+		}
+
 		if (nsdata->nacwu) {
 			printf("  Atomic Compare & Write Unit:         %d\n", nsdata->nacwu + 1);
 		}
@@ -1023,6 +1062,12 @@ print_namespace(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns)
 		printf("  Atomic Boundary Offset:              %d\n", nsdata->nabo);
 	}
 
+	if (cdata->oncs.copy) {
+		printf("Maximum Single Source Range Length:    %d\n", nsdata->mssrl);
+		printf("Maximum Copy Length:                   %d\n", nsdata->mcl);
+		printf("Maximum Source Range Count:            %d\n", nsdata->msrc + 1);
+	}
+
 	printf("NGUID/EUI64 Never Reused:              %s\n",
 	       nsdata->nsfeat.guid_never_reused ? "Yes" : "No");
 
@@ -1179,7 +1224,10 @@ print_controller(struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_transport
 	struct spdk_pci_id			pci_id;
 	uint32_t				nsid;
 	uint64_t				pmrsz;
-	struct spdk_nvme_ana_group_descriptor	*desc;
+	uint8_t					*orig_desc;
+	struct spdk_nvme_ana_group_descriptor	*copied_desc;
+	uint32_t				desc_size, copy_len;
+
 
 	cap = spdk_nvme_ctrlr_get_regs_cap(ctrlr);
 	vs = spdk_nvme_ctrlr_get_regs_vs(ctrlr);
@@ -1194,7 +1242,7 @@ print_controller(struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_transport
 		 * attempt GET_FEATURES when NOT targeting a
 		 * Discovery Controller.
 		 */
-		get_features(ctrlr);
+		get_ctrlr_features(ctrlr);
 	}
 	get_log_pages(ctrlr);
 
@@ -1246,7 +1294,7 @@ print_controller(struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_transport
 	       cdata->ieee[0], cdata->ieee[1], cdata->ieee[2]);
 	printf("Multi-path I/O\n");
 	printf("  May have multiple subsystem ports:   %s\n", cdata->cmic.multi_port ? "Yes" : "No");
-	printf("  May be connected to multiple hosts:  %s\n", cdata->cmic.multi_host ? "Yes" : "No");
+	printf("  May have multiple controllers:       %s\n", cdata->cmic.multi_ctrlr ? "Yes" : "No");
 	printf("  Associated with SR-IOV VF:           %s\n", cdata->cmic.sr_iov ? "Yes" : "No");
 	printf("Max Data Transfer Size:                ");
 	if (cdata->mdts == 0) {
@@ -1255,15 +1303,6 @@ print_controller(struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_transport
 		printf("%" PRIu64 "\n", (uint64_t)1 << (12 + cap.bits.mpsmin + cdata->mdts));
 	}
 	printf("Max Number of Namespaces:              %d\n", cdata->nn);
-	if (features[SPDK_NVME_FEAT_ERROR_RECOVERY].valid) {
-		unsigned tler = features[SPDK_NVME_FEAT_ERROR_RECOVERY].result & 0xFFFF;
-		printf("Error Recovery Timeout:                ");
-		if (tler == 0) {
-			printf("Unlimited\n");
-		} else {
-			printf("%u milliseconds\n", tler * 100);
-		}
-	}
 	printf("NVMe Specification Version (VS):       %u.%u", vs.bits.mjr, vs.bits.mnr);
 	if (vs.bits.ter) {
 		printf(".%u", vs.bits.ter);
@@ -1459,6 +1498,8 @@ print_controller(struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_transport
 	       cdata->oncs.reservations ? "Supported" : "Not Supported");
 	printf("Timestamp:                   %s\n",
 	       cdata->oncs.timestamp ? "Supported" : "Not Supported");
+	printf("Copy:                        %s\n",
+	       cdata->oncs.copy ? "Supported" : "Not Supported");
 	printf("Volatile Write Cache:        %s\n",
 	       cdata->vwc.present ? "Present" : "Not Present");
 	printf("Atomic Write Unit (Normal):  %d\n", cdata->awun + 1);
@@ -1527,21 +1568,29 @@ print_controller(struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_transport
 		printf("Change Count                    : %" PRIx64 "\n", g_ana_log_page->change_count);
 		printf("Number of ANA Group Descriptors : %u\n", g_ana_log_page->num_ana_group_desc);
 
-		desc = (void *)((uint8_t *)g_ana_log_page + sizeof(struct spdk_nvme_ana_page));
+		copied_desc = g_copied_ana_desc;
+		orig_desc = (uint8_t *)g_ana_log_page + sizeof(struct spdk_nvme_ana_page);
+		copy_len = g_ana_log_page_size - sizeof(struct spdk_nvme_ana_page);
 
 		for (i = 0; i < g_ana_log_page->num_ana_group_desc; i++) {
+			memcpy(copied_desc, orig_desc, copy_len);
+
 			printf("ANA Group Descriptor            : %u\n", i);
-			printf("  ANA Group ID                  : %u\n", desc->ana_group_id);
-			printf("  Number of NSID Values         : %u\n", desc->num_of_nsid);
-			printf("  Change Count                  : %" PRIx64 "\n", desc->change_count);
-			printf("  ANA State                     : %u\n", desc->ana_state);
-			for (j = 0; j < desc->num_of_nsid; j++) {
-				printf("  Namespace Identifier          : %u\n", desc->nsid[j]);
+			printf("  ANA Group ID                  : %u\n", copied_desc->ana_group_id);
+			printf("  Number of NSID Values         : %u\n", copied_desc->num_of_nsid);
+			printf("  Change Count                  : %" PRIx64 "\n", copied_desc->change_count);
+			printf("  ANA State                     : %u\n", copied_desc->ana_state);
+			for (j = 0; j < copied_desc->num_of_nsid; j++) {
+				printf("  Namespace Identifier          : %u\n", copied_desc->nsid[j]);
 			}
-			desc = (void *)((uint8_t *)desc + sizeof(struct spdk_nvme_ana_group_descriptor) +
-					desc->num_of_nsid * sizeof(uint32_t));
+
+			desc_size = sizeof(struct spdk_nvme_ana_group_descriptor) +
+				    copied_desc->num_of_nsid * sizeof(uint32_t);
+			orig_desc += desc_size;
+			copy_len -= desc_size;
 		}
 		free(g_ana_log_page);
+		free(g_copied_ana_desc);
 	}
 
 	printf("\n");
@@ -1647,18 +1696,18 @@ print_controller(struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_transport
 		printf("Number of Power States:      %u\n", cdata->npss + 1);
 		printf("Current Power State:         Power State #%u\n", ps);
 		for (i = 0; i <= cdata->npss; i++) {
-			const struct spdk_nvme_power_state *psd = &cdata->psd[i];
+			const struct spdk_nvme_power_state psd = cdata->psd[i];
 			printf("Power State #%u:  ", i);
-			if (psd->mps) {
+			if (psd.mps) {
 				/* MP scale is 0.0001 W */
 				printf("Max Power: %u.%04u W\n",
-				       psd->mp / 10000,
-				       psd->mp % 10000);
+				       psd.mp / 10000,
+				       psd.mp % 10000);
 			} else {
 				/* MP scale is 0.01 W */
 				printf("Max Power: %3u.%02u W\n",
-				       psd->mp / 100,
-				       psd->mp % 100);
+				       psd.mp / 100,
+				       psd.mp % 100);
 			}
 			/* TODO: print other power state descriptor fields */
 		}
@@ -1943,6 +1992,7 @@ print_controller(struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_transport
 	printf("=================\n");
 	for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
 	     nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) {
+		get_ns_features(ctrlr, nsid);
 		print_namespace(ctrlr, spdk_nvme_ctrlr_get_ns(ctrlr, nsid));
 	}
 
@@ -2174,7 +2224,7 @@ attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
 {
 	g_controllers_found++;
 	print_controller(ctrlr, trid);
-	spdk_nvme_detach(ctrlr);
+	spdk_nvme_detach_async(ctrlr, &g_detach_ctx);
 }
 
 int main(int argc, char **argv)
@@ -2223,12 +2273,16 @@ int main(int argc, char **argv)
 
 		g_controllers_found++;
 		print_controller(ctrlr, &g_trid);
-		spdk_nvme_detach(ctrlr);
+		spdk_nvme_detach_async(ctrlr, &g_detach_ctx);
 	} else if (spdk_nvme_probe(&g_trid, NULL, probe_cb, attach_cb, NULL) != 0) {
 		fprintf(stderr, "spdk_nvme_probe() failed\n");
 		return 1;
 	}
 
+	if (g_detach_ctx) {
+		spdk_nvme_detach_poll(g_detach_ctx);
+	}
+
 	if (g_controllers_found == 0) {
 		fprintf(stderr, "No NVMe controllers found.\n");
 	}

examples/nvme/nvme_manage/nvme_manage.c

@@ -1700,8 +1700,8 @@ int main(int argc, char **argv)
 		spdk_nvme_detach_async(dev->ctrlr, &detach_ctx);
 	}
 
-	while (detach_ctx && spdk_nvme_detach_poll_async(detach_ctx) == -EAGAIN) {
-		;
+	if (detach_ctx) {
+		spdk_nvme_detach_poll(detach_ctx);
 	}
 
 	return 0;

examples/nvme/perf/perf.c

@@ -5,6 +5,7 @@
  *   All rights reserved.
  *
  *   Copyright (c) 2019-2021 Mellanox Technologies LTD. All rights reserved.
+ *   Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
@@ -49,6 +50,7 @@
 #include "spdk/log.h"
 #include "spdk/likely.h"
 #include "spdk/sock.h"
+#include "spdk/zipf.h"
 
 #ifdef SPDK_CONFIG_URING
 #include <liburing.h>
@@ -105,6 +107,8 @@ struct ns_entry {
 	uint32_t		block_size;
 	uint32_t		md_size;
 	bool			md_interleave;
+	unsigned int		seed;
+	struct spdk_zipf	*zipf;
 	bool			pi_loc;
 	enum spdk_nvme_pi_type	pi_type;
 	uint32_t		io_flags;
@@ -267,6 +271,7 @@ static bool g_exit;
 /* Default to 10 seconds for the keep alive value. This value is arbitrary. */
 static uint32_t g_keep_alive_timeout_in_ms = 10000;
 static uint32_t g_quiet_count = 1;
+static double g_zipf_theta;
 
 /* When user specifies -Q, some error messages are rate limited.  When rate
  * limited, we only print the error message every g_quiet_count times the
@@ -334,7 +339,7 @@ perf_set_sock_zcopy(const char *impl_name, bool enable)
 		opts_size = sizeof(sock_opts);
 	}
 
-	sock_opts.enable_zerocopy_send = enable;
+	sock_opts.enable_zerocopy_send_client = enable;
 
 	if (spdk_sock_impl_set_opts(impl_name, &sock_opts, opts_size)) {
 		fprintf(stderr, "Failed to %s zcopy send for sock impl %s: error %d (%s)\n",
@@ -725,6 +730,10 @@ register_file(const char *path)
 	entry->size_in_ios = size / g_io_size_bytes;
 	entry->io_size_blocks = g_io_size_bytes / blklen;
 
+	if (g_is_random && g_zipf_theta > 0) {
+		entry->zipf = spdk_zipf_create(entry->size_in_ios, g_zipf_theta, 0);
+	}
+
 	snprintf(entry->name, sizeof(entry->name), "%s", path);
 
 	g_num_namespaces++;
@@ -1212,6 +1221,10 @@ register_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns)
 	entry->size_in_ios = ns_size / g_io_size_bytes;
 	entry->io_size_blocks = g_io_size_bytes / sector_size;
 
+	if (g_is_random && g_zipf_theta > 0) {
+		entry->zipf = spdk_zipf_create(entry->size_in_ios, g_zipf_theta, 0);
+	}
+
 	entry->block_size = spdk_nvme_ns_get_extended_sector_size(ns);
 	entry->md_size = spdk_nvme_ns_get_md_size(ns);
 	entry->md_interleave = spdk_nvme_ns_supports_extended_lba(ns);
@@ -1252,6 +1265,7 @@ unregister_namespaces(void)
 
 	TAILQ_FOREACH_SAFE(entry, &g_namespaces, link, tmp) {
 		TAILQ_REMOVE(&g_namespaces, entry, link);
+		spdk_zipf_free(&entry->zipf);
 		free(entry);
 	}
 }
@@ -1340,8 +1354,6 @@ register_ctrlr(struct spdk_nvme_ctrlr *ctrlr, struct trid_entry *trid_entry)
 	}
 }
 
-static __thread unsigned int seed = 0;
-
 static inline void
 submit_single_io(struct perf_task *task)
 {
@@ -1350,8 +1362,10 @@ submit_single_io(struct perf_task *task)
 	struct ns_worker_ctx	*ns_ctx = task->ns_ctx;
 	struct ns_entry		*entry = ns_ctx->entry;
 
-	if (g_is_random) {
-		offset_in_ios = rand_r(&seed) % entry->size_in_ios;
+	if (entry->zipf) {
+		offset_in_ios = spdk_zipf_generate(entry->zipf);
+	} else if (g_is_random) {
+		offset_in_ios = rand_r(&entry->seed) % entry->size_in_ios;
 	} else {
 		offset_in_ios = ns_ctx->offset_in_ios++;
 		if (ns_ctx->offset_in_ios == entry->size_in_ios) {
@@ -1362,7 +1376,7 @@ submit_single_io(struct perf_task *task)
 	task->submit_tsc = spdk_get_ticks();
 
 	if ((g_rw_percentage == 100) ||
-	    (g_rw_percentage != 0 && ((rand_r(&seed) % 100) < g_rw_percentage))) {
+	    (g_rw_percentage != 0 && ((rand_r(&entry->seed) % 100) < g_rw_percentage))) {
 		task->is_read = true;
 	} else {
 		task->is_read = false;
@@ -1526,13 +1540,13 @@ print_periodic_performance(bool warmup)
 		if (g_monitor_perf_cores) {
 			core_busy_tsc += busy_tsc;
 			core_idle_tsc += idle_tsc;
-			core_busy_perc += (double)core_busy_tsc / (core_idle_tsc + core_busy_tsc) * 100;
 		}
 	}
 	mb_this_second = (double)io_this_second * g_io_size_bytes / (1024 * 1024);
 
 	printf("%s%9ju IOPS, %8.2f MiB/s", warmup ? "[warmup] " : "", io_this_second, mb_this_second);
 	if (g_monitor_perf_cores) {
+		core_busy_perc = (double)core_busy_tsc / (core_idle_tsc + core_busy_tsc) * 100;
 		printf("%3d Core(s): %6.2f%% Busy", g_num_workers, core_busy_perc);
 	}
 	printf("\r");
@@ -1698,6 +1712,7 @@ static void usage(char *program_name)
 	printf("\t[-w, --io-pattern <pattern> io pattern type, must be one of\n");
 	printf("\t\t(read, write, randread, randwrite, rw, randrw)]\n");
 	printf("\t[-M, --rwmixread <0-100> rwmixread (100 for reads, 0 for writes)]\n");
+	printf("\t[-F, --zipf <theta> use zipf distribution for random I/O\n");
 	printf("\t[-L, --enable-sw-latency-tracking enable latency tracking via sw, default: disabled]\n");
 	printf("\t\t-L for latency summary, -LL for detailed histogram\n");
 	printf("\t[-l, --enable-ssd-latency-tracking enable latency tracking via ssd (if supported), default: disabled]\n");
@@ -1751,9 +1766,9 @@ static void usage(char *program_name)
 	printf("\t[-G, --enable-debug enable debug logging]\n");
 #else
 	printf("\t[-G, --enable-debug enable debug logging (flag disabled, must reconfigure with --enable-debug)\n");
+#endif
 	printf("\t[--transport-stats dump transport statistics]\n");
 	printf("\t[--iova-mode <mode> specify DPDK IOVA mode: va|pa]\n");
-#endif
 }
 
 static void
@@ -2170,7 +2185,7 @@ parse_metadata(const char *metacfg_str)
 	return 0;
 }
 
-#define PERF_GETOPT_SHORT "a:b:c:e:gi:lmo:q:r:k:s:t:w:z:A:C:DGHILM:NO:P:Q:RS:T:U:VZ:"
+#define PERF_GETOPT_SHORT "a:b:c:e:gi:lmo:q:r:k:s:t:w:z:A:C:DF:GHILM:NO:P:Q:RS:T:U:VZ:"
 
 static const struct option g_perf_cmdline_opts[] = {
 #define PERF_WARMUP_TIME	'a'
@@ -2211,6 +2226,8 @@ static const struct option g_perf_cmdline_opts[] = {
 	{"max-completion-per-poll",			required_argument,	NULL, PERF_MAX_COMPLETIONS_PER_POLL},
 #define PERF_DISABLE_SQ_CMB	'D'
 	{"disable-sq-cmb",			no_argument,	NULL, PERF_DISABLE_SQ_CMB},
+#define PERF_ZIPF		'F'
+	{"zipf",				required_argument,	NULL, PERF_ZIPF},
 #define PERF_ENABLE_DEBUG	'G'
 	{"enable-debug",			no_argument,	NULL, PERF_ENABLE_DEBUG},
 #define PERF_ENABLE_TCP_HDGST	'H'
@@ -2255,6 +2272,7 @@ parse_args(int argc, char **argv, struct spdk_env_opts *env_opts)
 	int op, long_idx;
 	long int val;
 	int rc;
+	char *endptr;
 
 	while ((op = getopt_long(argc, argv, PERF_GETOPT_SHORT, g_perf_cmdline_opts, &long_idx)) != -1) {
 		switch (op) {
@@ -2330,6 +2348,14 @@ parse_args(int argc, char **argv, struct spdk_env_opts *env_opts)
 				break;
 			}
 			break;
+		case PERF_ZIPF:
+			errno = 0;
+			g_zipf_theta = strtod(optarg, &endptr);
+			if (errno || optarg == endptr || g_zipf_theta < 0) {
+				fprintf(stderr, "Illegal zipf theta value %s\n", optarg);
+				return 1;
+			}
+			break;
 		case PERF_ALLOWED_PCI_ADDR:
 			if (add_allowed_pci_device(optarg, env_opts)) {
 				usage(argv[0]);
@@ -2575,6 +2601,11 @@ probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
 		}
 	}
 
+	if (trid->trtype != trid_entry->trid.trtype &&
+	    strcasecmp(trid->trstring, trid_entry->trid.trstring)) {
+		return false;
+	}
+
 	/* Set io_queue_size to UINT16_MAX, NVMe driver
 	 * will then reduce this to MQES to maximize
 	 * the io_queue_size as much as possible.
@@ -2675,8 +2706,8 @@ unregister_controllers(void)
 		free(entry);
 	}
 
-	while (detach_ctx && spdk_nvme_detach_poll_async(detach_ctx) == -EAGAIN) {
-		;
+	if (detach_ctx) {
+		spdk_nvme_detach_poll(detach_ctx);
 	}
 
 	if (g_vmd) {

examples/nvme/pmr_persistence/pmr_persistence.c

@@ -351,8 +351,8 @@ cleanup(void)
 
 	spdk_nvme_detach_async(g_config.pmr_dev.ctrlr, &detach_ctx);
 
-	while (detach_ctx && spdk_nvme_detach_poll_async(detach_ctx) == -EAGAIN) {
-		;
+	if (detach_ctx) {
+		spdk_nvme_detach_poll(detach_ctx);
 	}
 }
 

examples/nvme/reconnect/reconnect.c

@@ -970,8 +970,8 @@ unregister_controllers(void)
 		free(entry);
 	}
 
-	while (detach_ctx && spdk_nvme_detach_poll_async(detach_ctx) == -EAGAIN) {
-		;
+	if (detach_ctx) {
+		spdk_nvme_detach_poll(detach_ctx);
 	}
 }
 

examples/nvmf/nvmf/Makefile

@@ -38,6 +38,6 @@ include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
 APP := nvmf
 
 C_SRCS := nvmf.c
-SPDK_LIB_LIST = $(ALL_MODULES_LIST) event_bdev nvmf
+SPDK_LIB_LIST = $(ALL_MODULES_LIST) event event_bdev nvmf
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

examples/nvmf/nvmf/nvmf.c

@@ -32,8 +32,10 @@
  */
 
 #include "spdk/stdinc.h"
+
 #include "spdk/env.h"
 #include "spdk/event.h"
+#include "spdk/init.h"
 #include "spdk/string.h"
 #include "spdk/thread.h"
 #include "spdk/bdev.h"

examples/sock/hello_world/Makefile

@@ -38,6 +38,6 @@ APP = hello_sock
 C_SRCS := hello_sock.c
 
 SPDK_LIB_LIST = $(SOCK_MODULES_LIST)
-SPDK_LIB_LIST += event_net sock
+SPDK_LIB_LIST += event sock
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

examples/sock/hello_world/hello_sock.c

@@ -39,7 +39,6 @@
 #include "spdk/string.h"
 
 #include "spdk/sock.h"
-#include "spdk/net.h"
 
 #define ACCEPT_TIMEOUT_US 1000
 #define CLOSE_TIMEOUT_US 1000000

examples/util/Makefile

@@ -0,0 +1,47 @@
+#
+#  BSD LICENSE
+#
+#  Copyright (c) Intel Corporation.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+DIRS-y += zipf
+
+.PHONY: all clean $(DIRS-y)
+
+all: $(DIRS-y)
+	@:
+
+clean: $(DIRS-y)
+	@:
+
+include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk

examples/util/zipf/Makefile

@@ -0,0 +1,42 @@
+#
+#  Copyright (c) Intel Corporation.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
+
+APP = zipf
+
+C_SRCS := zipf.c
+
+SPDK_LIB_LIST = util
+
+include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

examples/util/zipf/zipf.c

@@ -0,0 +1,102 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+#include "spdk/zipf.h"
+#include "spdk/histogram_data.h"
+#include "spdk/string.h"
+
+static void
+usage(const char *prog)
+{
+	printf("usage: %s <theta> <range> <count>\n", prog);
+}
+
+static void
+print_bucket(void *ctx, uint64_t start, uint64_t end, uint64_t count,
+	     uint64_t total, uint64_t so_far)
+{
+	double so_far_pct;
+	char range[64];
+
+	if (count == 0) {
+		return;
+	}
+
+	so_far_pct = (double)so_far * 100 / total;
+	snprintf(range, sizeof(range), "[%ju, %ju)", start, end);
+	printf("%24s: %9.4f%%  (%9ju)\n", range, so_far_pct, count);
+}
+
+int
+main(int argc, char **argv)
+{
+	struct spdk_zipf *zipf;
+	struct spdk_histogram_data *h;
+	float theta;
+	int range, count, i;
+
+	if (argc < 4) {
+		usage(argv[0]);
+		return 1;
+	}
+
+	theta = atof(argv[1]);
+	range = spdk_strtol(argv[2], 10);
+	count = spdk_strtol(argv[3], 10);
+
+	if (range <= 0 || count <= 0) {
+		printf("range and count must be positive integers\n");
+		usage(argv[0]);
+		return 1;
+	}
+
+	zipf = spdk_zipf_create(range, theta, time(NULL));
+	h = spdk_histogram_data_alloc();
+	if (zipf == NULL || h == NULL) {
+		spdk_zipf_free(&zipf);
+		spdk_histogram_data_free(h);
+		printf("out of resource\n");
+		return 1;
+	}
+
+	for (i = 0; i < count; i++) {
+		spdk_histogram_data_tally(h, spdk_zipf_generate(zipf));
+	}
+
+	spdk_histogram_data_iterate(h, print_bucket, NULL);
+	spdk_histogram_data_free(h);
+	spdk_zipf_free(&zipf);
+
+	return 0;
+}

include/spdk/accel_engine.h

@@ -51,6 +51,7 @@ enum accel_capability {
 	ACCEL_COMPARE		= 1 << 3,
 	ACCEL_CRC32C		= 1 << 4,
 	ACCEL_DIF		= 1 << 5,
+	ACCEL_COPY_CRC32C	= 1 << 6,
 };
 
 /**
@@ -302,7 +303,7 @@ int spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, uint8_t fill,
  *
  * \param ch I/O channel associated with this call.
  * \param batch Handle provided when the batch was started with spdk_accel_batch_create().
- * \param dst Destination to write the CRC-32C to.
+ * \param crc_dst Destination to write the CRC-32C to.
  * \param src The source address for the data.
  * \param seed Four byte seed value.
  * \param nbytes Length in bytes.
@@ -312,7 +313,7 @@ int spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, uint8_t fill,
  * \return 0 on success, negative errno on failure.
  */
 int spdk_accel_batch_prep_crc32c(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
-				 uint32_t *dst, void *src, uint32_t seed, uint64_t nbytes,
+				 uint32_t *crc_dst, void *src, uint32_t seed, uint64_t nbytes,
 				 spdk_accel_completion_cb cb_fn, void *cb_arg);
 
 /**
@@ -323,7 +324,7 @@ int spdk_accel_batch_prep_crc32c(struct spdk_io_channel *ch, struct spdk_accel_b
  *
  * \param ch I/O channel associated with this call.
  * \param batch Handle provided when the batch was started with spdk_accel_batch_create().
- * \param dst Destination to write the CRC-32C to.
+ * \param crc_dst Destination to write the CRC-32C to.
  * \param iovs The io vector array which stores the src data and len.
  * \param iovcnt The size of the iov.
  * \param seed Four byte seed value.
@@ -333,16 +334,38 @@ int spdk_accel_batch_prep_crc32c(struct spdk_io_channel *ch, struct spdk_accel_b
  * \return 0 on success, negative errno on failure.
  */
 int spdk_accel_batch_prep_crc32cv(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
-				  uint32_t *dst, struct iovec *iovs, uint32_t iovcnt, uint32_t seed,
+				  uint32_t *crc_dst, struct iovec *iovs, uint32_t iovcnt, uint32_t seed,
 				  spdk_accel_completion_cb cb_fn, void *cb_arg);
 
+/**
+ * Synchronous call to prepare a copy + crc32c request into a previously initialized batch
+ *  created with spdk_accel_batch_create(). The callback will be called when the operation
+ *  completes after the batch has been submitted by an asynchronous call to
+ *  spdk_accel_batch_submit().
+ *
+ * \param ch I/O channel associated with this call.
+ * \param batch Handle provided when the batch was started with spdk_accel_batch_create().
+ * \param dst Destination to write the data to.
+ * \param src The source address for the data.
+ * \param crc_dst Destination to write the CRC-32C to.
+ * \param seed Four byte seed value.
+ * \param nbytes Length in bytes.
+ * \param cb_fn Called when this operation completes.
+ * \param cb_arg Callback argument.
+ *
+ * \return 0 on success, negative errno on failure.
+ */
+int spdk_accel_batch_prep_copy_crc32c(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
+				      void *dst, void *src, uint32_t *crc_dst, uint32_t seed, uint64_t nbytes,
+				      spdk_accel_completion_cb cb_fn, void *cb_arg);
+
 /**
  * Submit a CRC-32C calculation request.
  *
  * This operation will calculate the 4 byte CRC32-C for the given data.
  *
  * \param ch I/O channel associated with this call.
- * \param dst Destination to write the CRC-32C to.
+ * \param crc_dst Destination to write the CRC-32C to.
  * \param src The source address for the data.
  * \param seed Four byte seed value.
  * \param nbytes Length in bytes.
@@ -351,7 +374,8 @@ int spdk_accel_batch_prep_crc32cv(struct spdk_io_channel *ch, struct spdk_accel_
  *
  * \return 0 on success, negative errno on failure.
  */
-int spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *dst, void *src, uint32_t seed,
+int spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst, void *src,
+			     uint32_t seed,
 			     uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
 
 /**
@@ -360,7 +384,7 @@ int spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *dst, void *sr
  * This operation will calculate the 4 byte CRC32-C for the given data.
  *
  * \param ch I/O channel associated with this call.
- * \param dst Destination to write the CRC-32C to.
+ * \param crc_dst Destination to write the CRC-32C to.
  * \param iovs The io vector array which stores the src data and len.
  * \param iovcnt The size of the iov.
  * \param seed Four byte seed value.
@@ -369,9 +393,49 @@ int spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *dst, void *sr
  *
  * \return 0 on success, negative errno on failure.
  */
-int spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *dst, struct iovec *iovs,
+int spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst, struct iovec *iovs,
 			      uint32_t iovcnt, uint32_t seed, spdk_accel_completion_cb cb_fn, void *cb_arg);
 
+/**
+ * Submit a copy with CRC-32C calculation request.
+ *
+ * This operation will copy data and calculate the 4 byte CRC32-C for the given data.
+ *
+ * \param ch I/O channel associated with this call.
+ * \param dst Destination to write the data to.
+ * \param src The source address for the data.
+ * \param crc_dst Destination to write the CRC-32C to.
+ * \param seed Four byte seed value.
+ * \param nbytes Length in bytes.
+ * \param cb_fn Called when this CRC-32C operation completes.
+ * \param cb_arg Callback argument.
+ *
+ * \return 0 on success, negative errno on failure.
+ */
+int spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst, void *src,
+				  uint32_t *crc_dst, uint32_t seed, uint64_t nbytes,
+				  spdk_accel_completion_cb cb_fn, void *cb_arg);
+
+/**
+ * Submit a chained copy + CRC-32C calculation request.
+ *
+ * This operation will calculate the 4 byte CRC32-C for the given data.
+ *
+ * \param ch I/O channel associated with this call.
+ * \param dst Destination to write the data to.
+ * \param src_iovs The io vector array which stores the src data and len.
+ * \param iovcnt The size of the io vectors.
+ * \param crc_dst Destination to write the CRC-32C to.
+ * \param seed Four byte seed value.
+ * \param cb_fn Called when this CRC-32C operation completes.
+ * \param cb_arg Callback argument.
+ *
+ * \return 0 on success, negative errno on failure.
+ */
+int spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, struct iovec *src_iovs,
+				   uint32_t iovcnt, uint32_t *crc_dst, uint32_t seed, spdk_accel_completion_cb cb_fn, void *cb_arg);
+
+
 struct spdk_json_write_ctx;
 
 /**

include/spdk/bdev.h

@@ -590,6 +590,14 @@ bool spdk_bdev_is_zoned(const struct spdk_bdev *bdev);
  */
 uint32_t spdk_bdev_get_data_block_size(const struct spdk_bdev *bdev);
 
+/**
+ * Get block device physical block size.
+ *
+ * \param bdev Block device to query.
+ * \return Size of physical block size for this bdev in bytes.
+ */
+uint32_t spdk_bdev_get_physical_block_size(const struct spdk_bdev *bdev);
+
 /**
  * Get DIF type of the block device.
  *
@@ -795,7 +803,7 @@ int spdk_bdev_read_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *c
  *   * -ENOMEM - spdk_bdev_io buffer cannot be allocated
  */
 int spdk_bdev_read_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
-				  void *buf, void *md, int64_t offset_blocks, uint64_t num_blocks,
+				  void *buf, void *md, uint64_t offset_blocks, uint64_t num_blocks,
 				  spdk_bdev_io_completion_cb cb, void *cb_arg);
 
 /**
@@ -1218,6 +1226,8 @@ int spdk_bdev_comparev_and_writev_blocks(struct spdk_bdev_desc *desc, struct spd
  *
  * \param desc Block device descriptor
  * \param ch I/O channel. Obtained by calling spdk_bdev_get_io_channel().
+ * \param iov A scatter gather list to be populated with the buffers
+ * \param iovcnt The maximum number of elements in iov.
  * \param offset_blocks The offset, in blocks, from the start of the block device.
  * \param num_blocks The number of blocks.
  * \param populate Whether the data buffer should be populated with the
@@ -1231,11 +1241,11 @@ int spdk_bdev_comparev_and_writev_blocks(struct spdk_bdev_desc *desc, struct spd
  * negated errno on failure, in which case the callback will not be called.
  */
 int spdk_bdev_zcopy_start(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
+			  struct iovec *iov, int iovcnt,
 			  uint64_t offset_blocks, uint64_t num_blocks,
 			  bool populate,
 			  spdk_bdev_io_completion_cb cb, void *cb_arg);
 
-
 /**
  * Submit a request to release a data buffer representing a range of blocks.
  *

include/spdk/bdev_module.h

@@ -47,6 +47,7 @@
 #include "spdk/queue.h"
 #include "spdk/scsi_spec.h"
 #include "spdk/thread.h"
+#include "spdk/tree.h"
 #include "spdk/util.h"
 #include "spdk/uuid.h"
 
@@ -161,6 +162,36 @@ struct spdk_bdev_module {
 	} internal;
 };
 
+/**
+ * Called by a bdev module to lay exclusive write claim to a bdev.
+ *
+ * Also upgrades that bdev's descriptor to have write access.
+ *
+ * \param bdev Block device to be claimed.
+ * \param desc Descriptor for the above block device.
+ * \param module Bdev module attempting to claim bdev.
+ *
+ * \return 0 on success
+ * \return -EPERM if the bdev is already claimed by another module.
+ */
+int spdk_bdev_module_claim_bdev(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
+				struct spdk_bdev_module *module);
+
+/**
+ * Called to release a write claim on a block device.
+ *
+ * \param bdev Block device to be released.
+ */
+void spdk_bdev_module_release_bdev(struct spdk_bdev *bdev);
+
+/* Libraries may define __SPDK_BDEV_MODULE_ONLY so that they include
+ * only the struct spdk_bdev_module definition, and the relevant APIs
+ * to claim/release a bdev. This may be useful in some cases to avoid
+ * abidiff errors related to including the struct spdk_bdev structure
+ * unnecessarily.
+ */
+#ifndef __SPDK_BDEV_MODULE_ONLY
+
 typedef void (*spdk_bdev_unregister_cb)(void *cb_arg, int rc);
 
 /**
@@ -237,8 +268,14 @@ enum spdk_bdev_io_status {
 	SPDK_BDEV_IO_STATUS_SUCCESS = 1,
 };
 
+struct spdk_bdev_name {
+	char *name;
+	struct spdk_bdev *bdev;
+	RB_ENTRY(spdk_bdev_name) node;
+};
+
 struct spdk_bdev_alias {
-	char *alias;
+	struct spdk_bdev_name alias;
 	TAILQ_ENTRY(spdk_bdev_alias) tailq;
 };
 
@@ -265,6 +302,9 @@ struct spdk_bdev {
 	/** Size in bytes of a logical block for the backend */
 	uint32_t blocklen;
 
+	/** Size in bytes of a physical block for the backend */
+	uint32_t phys_blocklen;
+
 	/** Number of blocks */
 	uint64_t blockcnt;
 
@@ -310,6 +350,15 @@ struct spdk_bdev {
 	/* Maximum number of segments in a I/O */
 	uint32_t max_num_segments;
 
+	/* Maximum unmap in unit of logical block */
+	uint32_t max_unmap;
+
+	/* Maximum unmap block segments */
+	uint32_t max_unmap_segments;
+
+	/* Maximum write zeroes in unit of logical block */
+	uint32_t max_write_zeroes;
+
 	/**
 	 * UUID for this bdev.
 	 *
@@ -462,6 +511,9 @@ struct spdk_bdev {
 		 *  locked due to overlapping with another locked range.
 		 */
 		lba_range_tailq_t pending_locked_ranges;
+
+		/** Bdev name used for quick lookup */
+		struct spdk_bdev_name bdev_name;
 	} internal;
 };
 
@@ -766,28 +818,6 @@ void spdk_bdev_module_init_done(struct spdk_bdev_module *module);
  */
 void spdk_bdev_module_finish_done(void);
 
-/**
- * Called by a bdev module to lay exclusive write claim to a bdev.
- *
- * Also upgrades that bdev's descriptor to have write access.
- *
- * \param bdev Block device to be claimed.
- * \param desc Descriptor for the above block device.
- * \param module Bdev module attempting to claim bdev.
- *
- * \return 0 on success
- * \return -EPERM if the bdev is already claimed by another module.
- */
-int spdk_bdev_module_claim_bdev(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
-				struct spdk_bdev_module *module);
-
-/**
- * Called to release a write claim on a block device.
- *
- * \param bdev Block device to be released.
- */
-void spdk_bdev_module_release_bdev(struct spdk_bdev *bdev);
-
 /**
  * Add alias to block device names list.
  * Aliases can be add only to registered bdev.
@@ -1227,4 +1257,6 @@ static void __attribute__((constructor)) _spdk_bdev_module_register_##name(void)
 	spdk_bdev_module_list_add(module); \
 } \
 
+#endif /* __SPDK_BDEV_MODULE_ONLY */
+
 #endif /* SPDK_BDEV_MODULE_H */

include/spdk/crc32.h

@@ -66,6 +66,16 @@ uint32_t spdk_crc32_ieee_update(const void *buf, size_t len, uint32_t crc);
  */
 uint32_t spdk_crc32c_update(const void *buf, size_t len, uint32_t crc);
 
+/**
+ * Calculate a partial CRC-32C checksum.
+ *
+ * \param iov Data buffer vectors to checksum.
+ * \param iovcnt size of iov parameter.
+ * \param crc32c Previous CRC-32C value.
+ * \return Updated CRC-32C value.
+ */
+uint32_t spdk_crc32c_iov_update(struct iovec *iov, int iovcnt, uint32_t crc32c);
+
 #ifdef __cplusplus
 }
 #endif

include/spdk/env.h

@@ -1343,7 +1343,7 @@ struct spdk_pci_event {
 	struct spdk_pci_addr traddr;
 };
 
-typedef void (*spdk_pci_error_handler)(siginfo_t *info, void *ctx);
+typedef void (*spdk_pci_error_handler)(const void *failure_addr, void *ctx);
 
 /**
  * Begin listening for PCI bus events. This is used to detect hot-insert and

include/spdk/event.h

@@ -44,6 +44,7 @@
 #include "spdk/stdinc.h"
 
 #include "spdk/cpuset.h"
+#include "spdk/init.h"
 #include "spdk/queue.h"
 #include "spdk/log.h"
 #include "spdk/thread.h"
@@ -82,8 +83,6 @@ typedef void (*spdk_app_shutdown_cb)(void);
  */
 typedef void (*spdk_sighandler_t)(int signal);
 
-#define SPDK_DEFAULT_RPC_ADDR "/var/tmp/spdk.sock"
-
 /**
  * \brief Event framework initialization options
  */

include/spdk/histogram_data.h

@@ -123,9 +123,7 @@ __spdk_histogram_data_get_bucket_range(struct spdk_histogram_data *h, uint64_t d
 {
 	uint32_t clz, range;
 
-	assert(datapoint != 0);
-
-	clz = __builtin_clzll(datapoint);
+	clz = datapoint > 0 ? __builtin_clzll(datapoint) : 64;
 
 	if (clz <= SPDK_HISTOGRAM_BUCKET_LSB(h)) {
 		range = SPDK_HISTOGRAM_BUCKET_LSB(h) - clz;

include/spdk/idxd.h

@@ -69,14 +69,6 @@ struct idxd_batch;
  */
 int spdk_idxd_configure_chan(struct spdk_idxd_io_channel *chan);
 
-/**
- * Reconfigures this channel based on how many current channels there are.
- *
- * \param chan IDXD channel to be set.
- * \return 0 on success, negative errno on failure.
- */
-int spdk_idxd_reconfigure_chan(struct spdk_idxd_io_channel *chan);
-
 /**
  * Signature for callback function invoked when a request is completed.
  *
@@ -86,26 +78,14 @@ int spdk_idxd_reconfigure_chan(struct spdk_idxd_io_channel *chan);
  */
 typedef void (*spdk_idxd_req_cb)(void *arg, int status);
 
-/**
- * Callback for spdk_idxd_probe() enumeration.
- *
- * \param cb_ctx User-specified opaque value corresponding to cb_ctx from spdk_idxd_probe().
- * \param pci_dev PCI device that is being probed.
- *
- * \return true to attach to this device.
- */
-typedef bool (*spdk_idxd_probe_cb)(void *cb_ctx, struct spdk_pci_device *pci_dev);
-
 /**
  * Callback for spdk_idxd_probe() to report a device that has been attached to
  * the userspace IDXD driver.
  *
  * \param cb_ctx User-specified opaque value corresponding to cb_ctx from spdk_idxd_probe().
- * \param pci_dev PCI device that was attached to the driver.
  * \param idxd IDXD device that was attached to the driver.
  */
-typedef void (*spdk_idxd_attach_cb)(void *cb_ctx, struct spdk_pci_device *pci_dev,
-				    struct spdk_idxd_device *idxd);
+typedef void (*spdk_idxd_attach_cb)(void *cb_ctx, struct spdk_idxd_device *idxd);
 
 /**
  * Enumerate the IDXD devices attached to the system and attach the userspace
@@ -119,13 +99,12 @@ typedef void (*spdk_idxd_attach_cb)(void *cb_ctx, struct spdk_pci_device *pci_de
  *
  * \param cb_ctx Opaque value which will be passed back in cb_ctx parameter of
  * the callbacks.
- * \param probe_cb will be called once per IDXD device found in the system.
  * \param attach_cb will be called for devices for which probe_cb returned true
  * once the IDXD controller has been attached to the userspace driver.
  *
  * \return 0 on success, -1 on failure.
  */
-int spdk_idxd_probe(void *cb_ctx, spdk_idxd_probe_cb probe_cb, spdk_idxd_attach_cb attach_cb);
+int spdk_idxd_probe(void *cb_ctx, spdk_idxd_attach_cb attach_cb);
 
 /**
  * Detach specified device returned by spdk_idxd_probe() from the IDXD driver.
@@ -138,8 +117,9 @@ void spdk_idxd_detach(struct spdk_idxd_device *idxd);
  * Sets the IDXD configuration.
  *
  * \param config_number the configuration number for a valid IDXD config.
+ * \param kernel_mode true if using kernel driver.
   */
-void spdk_idxd_set_config(uint32_t config_number);
+void spdk_idxd_set_config(uint32_t config_number, bool kernel_mode);
 
 /**
  * Return the max number of descriptors per batch for IDXD.
@@ -352,7 +332,7 @@ int spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan,
  *
  * \param chan IDXD channel to submit request.
  * \param batch Handle provided when the batch was started with spdk_idxd_batch_create().
- * \param dst Resulting calculation.
+ * \param crc_dst Resulting calculation.
  * \param src Source virtual address.
  * \param seed Four byte CRC-32C seed value.
  * \param nbytes Number of bytes to calculate on.
@@ -363,7 +343,7 @@ int spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan,
  * \return 0 on success, negative errno on failure.
  */
 int spdk_idxd_batch_prep_crc32c(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch,
-				uint32_t *dst, void *src, uint32_t seed, uint64_t nbytes,
+				uint32_t *crc_dst, void *src, uint32_t seed, uint64_t nbytes,
 				spdk_idxd_req_cb cb_fn, void *cb_arg);
 
 /**
@@ -373,7 +353,7 @@ int spdk_idxd_batch_prep_crc32c(struct spdk_idxd_io_channel *chan, struct idxd_b
  * by writing to the proper device portal.
  *
  * \param chan IDXD channel to submit request.
- * \param dst Resulting calculation.
+ * \param crc_dst Resulting calculation.
  * \param src Source virtual address.
  * \param seed Four byte CRC-32C seed value.
  * \param nbytes Number of bytes to calculate on.
@@ -383,10 +363,55 @@ int spdk_idxd_batch_prep_crc32c(struct spdk_idxd_io_channel *chan, struct idxd_b
  *
  * \return 0 on success, negative errno on failure.
  */
-int spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan, uint32_t *dst, void *src,
+int spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan, uint32_t *crc_dst, void *src,
 			    uint32_t seed, uint64_t nbytes,
 			    spdk_idxd_req_cb cb_fn, void *cb_arg);
 
+/**
+ * Synchronous call to prepare a copy combined with crc32c request into a previously
+ *  initialized batch created with spdk_idxd_batch_create(). The callback will be called
+ *  when the copy + crc32c completes after the batch has been submitted by an asynchronous
+ *  call to spdk_idxd_batch_submit().
+ *
+ * \param chan IDXD channel to submit request.
+ * \param batch Handle provided when the batch was started with spdk_idxd_batch_create().
+ * \param dst Destination virtual address.
+ * \param src Source virtual address.
+ * \param crc_dst Resulting calculation.
+ * \param seed Four byte CRC-32C seed value.
+ * \param nbytes Number of bytes to calculate on.
+ * \param cb_fn Callback function which will be called when the request is complete.
+ * \param cb_arg Opaque value which will be passed back as the arg parameter in
+ * the completion callback.
+ *
+ * \return 0 on success, negative errno on failure.
+ */
+int spdk_idxd_batch_prep_copy_crc32c(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch,
+				     void *dst, void *src, uint32_t *crc_dst, uint32_t seed, uint64_t nbytes,
+				     spdk_idxd_req_cb cb_fn, void *cb_arg);
+
+/**
+ * Build and submit a copy combined with CRC32-C request.
+ *
+ * This function will build the descriptor for copy plus CRC32-C and then immediately
+ * submit by writing to the proper device portal.
+ *
+ * \param chan IDXD channel to submit request.
+ * \param dst Destination virtual address.
+ * \param src Source virtual address.
+ * \param crc_dst Resulting calculation.
+ * \param seed Four byte CRC-32C seed value.
+ * \param nbytes Number of bytes to calculate on.
+ * \param cb_fn Callback function which will be called when the request is complete.
+ * \param cb_arg Opaque value which will be passed back as the cb_arg parameter
+ * in the completion callback.
+ *
+ * \return 0 on success, negative errno on failure.
+ */
+int spdk_idxd_submit_copy_crc32c(struct spdk_idxd_io_channel *chan, void *dst, void *src,
+				 uint32_t *crc_dst, uint32_t seed, uint64_t nbytes,
+				 spdk_idxd_req_cb cb_fn, void *cb_arg);
+
 /**
  * Check for completed requests on an IDXD channel.
  *
@@ -399,7 +424,6 @@ int spdk_idxd_process_events(struct spdk_idxd_io_channel *chan);
  * Returns an IDXD channel for a given IDXD device.
  *
  * \param idxd IDXD device to get a channel for.
- *
  * \return pointer to an IDXD channel.
  */
 struct spdk_idxd_io_channel *spdk_idxd_get_channel(struct spdk_idxd_device *idxd);
@@ -408,17 +432,16 @@ struct spdk_idxd_io_channel *spdk_idxd_get_channel(struct spdk_idxd_device *idxd
  * Free an IDXD channel.
  *
  * \param chan IDXD channel to free.
- * \return true if the underlying device needs a rebalance
  */
-bool spdk_idxd_put_channel(struct spdk_idxd_io_channel *chan);
+void spdk_idxd_put_channel(struct spdk_idxd_io_channel *chan);
 
 /**
- * Determine if the idxd device needs rebalancing.
+ * Get the max number of outstanding operations supported by this channel.
  *
- * \param idxd IDXD device.
- * \return true if rebalance is needed, false if not.
+ * \param chan IDXD channel to communicate on.
+ * \return max number of operations supported.
  */
-bool spdk_idxd_device_needs_rebalance(struct spdk_idxd_device *idxd);
+int spdk_idxd_chan_get_max_operations(struct spdk_idxd_io_channel *chan);
 
 #ifdef __cplusplus
 }

include/spdk/init.h

@@ -0,0 +1,107 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.  All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ * SPDK Initialization Helper
+ */
+
+#ifndef SPDK_INIT_H
+#define SPDK_INIT_H
+
+#include "spdk/stdinc.h"
+#include "spdk/queue.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define SPDK_DEFAULT_RPC_ADDR "/var/tmp/spdk.sock"
+
+/**
+ * Create the SPDK JSON-RPC server and listen at the provided address. The RPC server is optional and is
+ * independent of subsystem initialization. The RPC server can be started and stopped at any time.
+ *
+ * \param listen_addr Path to a unix domain socket to listen on
+ *
+ * \return Negated errno on failure. 0 on success.
+ */
+int spdk_rpc_initialize(const char *listen_addr);
+
+/**
+ * Shut down the SPDK JSON-RPC target
+ */
+void spdk_rpc_finish(void);
+
+typedef void (*spdk_subsystem_init_fn)(int rc, void *ctx);
+
+/**
+ * Begin the initialization process for all SPDK subsystems. SPDK is divided into subsystems at a macro-level
+ * and each subsystem automatically registers itself with this library at start up using a C
+ * constructor. Further, each subsystem can declare other subsystems that it depends on.
+ * Calling this function will correctly initialize all subsystems that are present, in the
+ * required order.
+ *
+ * \param cb_fn Function called when the process is complete.
+ * \param cb_arg User context passed to cb_fn.
+ */
+void spdk_subsystem_init(spdk_subsystem_init_fn cb_fn, void *cb_arg);
+
+/**
+ * Like spdk_subsystem_init, but additionally configure each subsystem using the provided JSON config
+ * file. This will automatically start a JSON RPC server and then stop it.
+ *
+ * \param json_config_file Path to a JSON config file.
+ * \param rpc_addr Path to a unix domain socket to send configuration RPCs to.
+ * \param cb_fn Function called when the process is complete.
+ * \param cb_arg User context passed to cb_fn.
+ * \param stop_on_error Whether to stop initialization if one of the JSON RPCs fails.
+ */
+void spdk_subsystem_init_from_json_config(const char *json_config_file, const char *rpc_addr,
+		spdk_subsystem_init_fn cb_fn, void *cb_arg,
+		bool stop_on_error);
+
+typedef void (*spdk_subsystem_fini_fn)(void *ctx);
+
+/**
+ * Tear down all of the subsystems in the correct order.
+ *
+ * \param cb_fn Function called when the process is complete.
+ * \param cb_arg User context passed to cb_fn
+ */
+void spdk_subsystem_fini(spdk_subsystem_fini_fn cb_fn, void *cb_arg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/spdk/json.h

@@ -208,6 +208,7 @@ int spdk_json_write_int32(struct spdk_json_write_ctx *w, int32_t val);
 int spdk_json_write_uint32(struct spdk_json_write_ctx *w, uint32_t val);
 int spdk_json_write_int64(struct spdk_json_write_ctx *w, int64_t val);
 int spdk_json_write_uint64(struct spdk_json_write_ctx *w, uint64_t val);
+int spdk_json_write_uint128(struct spdk_json_write_ctx *w, uint64_t low_val, uint64_t high_val);
 int spdk_json_write_string(struct spdk_json_write_ctx *w, const char *val);
 int spdk_json_write_string_raw(struct spdk_json_write_ctx *w, const char *val, size_t len);
 
@@ -257,6 +258,8 @@ int spdk_json_write_named_bool(struct spdk_json_write_ctx *w, const char *name,
 int spdk_json_write_named_int32(struct spdk_json_write_ctx *w, const char *name, int32_t val);
 int spdk_json_write_named_uint32(struct spdk_json_write_ctx *w, const char *name, uint32_t val);
 int spdk_json_write_named_uint64(struct spdk_json_write_ctx *w, const char *name, uint64_t val);
+int spdk_json_write_named_uint128(struct spdk_json_write_ctx *w, const char *name,
+				  uint64_t low_val, uint64_t high_val);
 int spdk_json_write_named_int64(struct spdk_json_write_ctx *w, const char *name, int64_t val);
 int spdk_json_write_named_string(struct spdk_json_write_ctx *w, const char *name, const char *val);
 int spdk_json_write_named_string_fmt(struct spdk_json_write_ctx *w, const char *name,

include/spdk/nvme.h

@@ -256,6 +256,14 @@ struct spdk_nvme_ctrlr_opts {
 	 * this controller in microseconds.
 	 */
 	uint64_t fabrics_connect_timeout_us;
+
+	/**
+	 * Disable reading ANA log page. The upper layer should reading ANA log page instead
+	 * if set to true.
+	 *
+	 * Default is `false` (ANA log page is read).
+	 */
+	bool disable_read_ana_log_page;
 };
 
 /**
@@ -297,6 +305,15 @@ struct spdk_nvme_accel_fn_table {
  */
 bool spdk_nvme_ctrlr_is_discovery(struct spdk_nvme_ctrlr *ctrlr);
 
+/**
+ * Indicate whether a ctrlr handle is associated with a fabrics controller.
+ *
+ * \param ctrlr Opaque handle to NVMe controller.
+ *
+ * \return true if a fabrics controller, else false.
+ */
+bool spdk_nvme_ctrlr_is_fabrics(struct spdk_nvme_ctrlr *ctrlr);
+
 /**
  * Get the default options for the creation of a specific NVMe controller.
  *
@@ -941,6 +958,13 @@ int spdk_nvme_detach_async(struct spdk_nvme_ctrlr *ctrlr,
  */
 int spdk_nvme_detach_poll_async(struct spdk_nvme_detach_ctx *detach_ctx);
 
+/**
+ * Continue calling spdk_nvme_detach_poll_async() internally until it returns 0.
+ *
+ * \param detach_ctx Context to track the detachment.
+ */
+void spdk_nvme_detach_poll(struct spdk_nvme_detach_ctx *detach_ctx);
+
 /**
  * Update the transport ID for a given controller.
  *
@@ -993,6 +1017,41 @@ void spdk_nvme_ctrlr_set_remove_cb(struct spdk_nvme_ctrlr *ctrlr,
  */
 int spdk_nvme_ctrlr_reset(struct spdk_nvme_ctrlr *ctrlr);
 
+struct spdk_nvme_ctrlr_reset_ctx;
+
+/**
+ * Create a context object that can be polled to perform a full hardware reset of the NVMe controller.
+ *
+ * The function will set the controller reset context on success, user must call
+ * spdk_nvme_ctrlr_reset_poll_async() until it returns a value other than -EAGAIN.
+ *
+ * \param ctrlr Opaque handle to NVMe controller.
+ * \param reset_ctx Double pointer to reset context.
+ *
+ * \return 0 on success.
+ * \return -ENOMEM if context could not be allocated.
+ * \return -ENXIO if controller has been removed.
+ *
+ */
+int spdk_nvme_ctrlr_reset_async(struct spdk_nvme_ctrlr *ctrlr,
+				struct spdk_nvme_ctrlr_reset_ctx **reset_ctx);
+
+/**
+ * Proceed with resetting contoller associated with the controller reset context.
+ *
+ * The controller reset context is one returned from a previous call to
+ * spdk_nvme_ctrlr_reset_async().  Users must call this function on the
+ * controller reset context until it returns a value other than -EAGAIN.
+ *
+ * \param ctrlr_reset_ctx Context used to track controller reset actions.
+ *
+ * \return 0 if all controller reset operations are complete; the ctrlr_reset_ctx
+ * is also freed and no longer valid.
+ * \return -EAGAIN if there are still pending controller reset operations; user must call
+ * spdk_nvme_ctrlr_reset_poll_async again to continue progress.
+ */
+int spdk_nvme_ctrlr_reset_poll_async(struct spdk_nvme_ctrlr_reset_ctx *ctrlr_reset_ctx);
+
 /**
  * Perform a NVMe subsystem reset.
  *
@@ -1278,12 +1337,14 @@ typedef void (*spdk_nvme_timeout_cb)(void *cb_arg,
  * for timeout callback.
  *
  * \param ctrlr NVMe controller on which to monitor for timeout.
- * \param timeout_us Timeout value in microseconds.
+ * \param timeout_io_us Timeout value in microseconds for io commands.
+ * \param timeout_admin_us Timeout value in microseconds for admin commands.
  * \param cb_fn A function pointer that points to the callback function.
  * \param cb_arg Argument to the callback function.
  */
 void spdk_nvme_ctrlr_register_timeout_callback(struct spdk_nvme_ctrlr *ctrlr,
-		uint64_t timeout_us, spdk_nvme_timeout_cb cb_fn, void *cb_arg);
+		uint64_t timeout_io_us, uint64_t timeout_admin_us,
+		spdk_nvme_timeout_cb cb_fn, void *cb_arg);
 
 /**
  * NVMe I/O queue pair initialization options.
@@ -1485,10 +1546,11 @@ spdk_nvme_qp_failure_reason spdk_nvme_ctrlr_get_admin_qp_failure_reason(
 /**
  * Free an I/O queue pair that was allocated by spdk_nvme_ctrlr_alloc_io_qpair().
  *
+ * The qpair must not be accessed after calling this function.
+ *
  * \param qpair I/O queue pair to free.
  *
- * \return 0 on success, -1 on failure.  On failure, the caller should reset
- * the controller and try to free the io qpair again after the reset.
+ * \return 0 on success.  This function will never return any value other than 0.
  */
 int spdk_nvme_ctrlr_free_io_qpair(struct spdk_nvme_qpair *qpair);
 
@@ -2622,6 +2684,15 @@ enum spdk_nvme_dealloc_logical_block_read_value spdk_nvme_ns_get_dealloc_logical
  */
 uint32_t spdk_nvme_ns_get_optimal_io_boundary(struct spdk_nvme_ns *ns);
 
+/**
+ * Get the NGUID for the given namespace.
+ *
+ * \param ns Namespace to query.
+ *
+ * \return a pointer to namespace NGUID, or NULL if ns does not have a NGUID.
+ */
+const uint8_t *spdk_nvme_ns_get_nguid(const struct spdk_nvme_ns *ns);
+
 /**
  * Get the UUID for the given namespace.
  *
@@ -3062,6 +3133,40 @@ int spdk_nvme_ns_cmd_dataset_management(struct spdk_nvme_ns *ns, struct spdk_nvm
 					spdk_nvme_cmd_cb cb_fn,
 					void *cb_arg);
 
+/**
+ * Submit a simple copy command request to the specified NVMe namespace.
+ *
+ * The command is submitted to a qpair allocated by spdk_nvme_ctrlr_alloc_io_qpair().
+ * The user must ensure that only one thread submits I/O on a given qpair at any
+ * given time.
+ *
+ * This is a convenience wrapper that will automatically allocate and construct
+ * the correct data buffers. Therefore, ranges does not need to be allocated from
+ * pinned memory and can be placed on the stack. If a higher performance, zero-copy
+ * version of SCC is required, simply build and submit a raw command using
+ * spdk_nvme_ctrlr_cmd_io_raw().
+ *
+ * \param ns NVMe namespace to submit the SCC request
+ * \param qpair I/O queue pair to submit the request
+ * \param ranges An array of \ref spdk_nvme_scc_source_range elements describing the LBAs
+ * to operate on.
+ * \param num_ranges The number of elements in the ranges array.
+ * \param dest_lba Destination LBA to copy the data.
+ * \param cb_fn Callback function to invoke when the I/O is completed
+ * \param cb_arg Argument to pass to the callback function
+ *
+ * \return 0 if successfully submitted, negated errnos on the following error conditions:
+ * -ENOMEM: The request cannot be allocated.
+ * -EINVAL: Invalid ranges.
+ * -ENXIO: The qpair is failed at the transport level.
+ */
+int spdk_nvme_ns_cmd_copy(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+			  const struct spdk_nvme_scc_source_range *ranges,
+			  uint16_t num_ranges,
+			  uint64_t dest_lba,
+			  spdk_nvme_cmd_cb cb_fn,
+			  void *cb_arg);
+
 /**
  * Submit a flush request to the specified NVMe namespace.
  *
@@ -3527,25 +3632,6 @@ int spdk_nvme_cuse_register(struct spdk_nvme_ctrlr *ctrlr);
  */
 int spdk_nvme_cuse_unregister(struct spdk_nvme_ctrlr *ctrlr);
 
-int spdk_nvme_map_prps(void *prv, struct spdk_nvme_cmd *cmd, struct iovec *iovs,
-		       uint32_t len, size_t mps,
-		       void *(*gpa_to_vva)(void *prv, uint64_t addr, uint64_t len));
-
-/**
- * Map NVMe command data buffers sent from Virtual Machine to virtual addresses
- *
- *\param prv Opaque handle to gpa_to_vva callback
- *\param cmd NVMe command
- *\param iovs IO vectors used to point the data buffers in NVMe command
- *\param max_iovcnt Maximum IO vectors that can be used
- *\param len Total buffer length for the NVMe command
- *\param mps Memory page size
- *\param gpa_to_vva Callback to map memory from Guest Physical address to Virtual address
- */
-int spdk_nvme_map_cmd(void *prv, struct spdk_nvme_cmd *cmd, struct iovec *iovs, uint32_t max_iovcnt,
-		      uint32_t len, size_t mps,
-		      void *(*gpa_to_vva)(void *prv, uint64_t addr, uint64_t len));
-
 /**
  * Opaque handle for a transport poll group. Used by the transport function table.
  */

include/spdk/nvme_spec.h

@@ -72,6 +72,11 @@ extern "C" {
  */
 #define SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS	0xFFFFFFFFu
 
+/**
+ * Maximum number of entries in the log page of Changed Namespace List.
+ */
+#define SPDK_NVME_MAX_CHANGED_NAMESPACES 1024
+
 union spdk_nvme_cap_register {
 	uint64_t	raw;
 	struct {
@@ -461,6 +466,14 @@ union spdk_nvme_bpinfo_register	{
 };
 SPDK_STATIC_ASSERT(sizeof(union spdk_nvme_bpinfo_register) == 4, "Incorrect size");
 
+/** Boot read status values */
+enum spdk_nvme_brs_value {
+	SPDK_NVME_BRS_NO_READ		= 0x0,
+	SPDK_NVME_BRS_READ_IN_PROGRESS	= 0x1,
+	SPDK_NVME_BRS_READ_SUCCESS	= 0x2,
+	SPDK_NVME_BRS_READ_ERROR	= 0x3,
+};
+
 /** Boot partition read select */
 union spdk_nvme_bprsel_register {
 	uint32_t	raw;
@@ -1338,6 +1351,21 @@ struct spdk_nvme_dsm_range {
 };
 SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_dsm_range) == 16, "Incorrect size");
 
+/**
+ * Simple Copy Command source range
+ */
+struct spdk_nvme_scc_source_range {
+	uint64_t reserved0;
+	uint64_t slba;
+	uint16_t nlb;
+	uint16_t reserved18;
+	uint32_t reserved20;
+	uint32_t eilbrt;
+	uint16_t elbat;
+	uint16_t elbatm;
+};
+SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_scc_source_range) == 32, "Incorrect size");
+
 /**
  * Status code types
  */
@@ -1386,6 +1414,9 @@ enum spdk_nvme_generic_command_status_code {
 	SPDK_NVME_SC_SANITIZE_IN_PROGRESS		= 0x1d,
 	SPDK_NVME_SC_SGL_DATA_BLOCK_GRANULARITY_INVALID	= 0x1e,
 	SPDK_NVME_SC_COMMAND_INVALID_IN_CMB		= 0x1f,
+	SPDK_NVME_SC_COMMAND_NAMESPACE_IS_PROTECTED	= 0x20,
+	SPDK_NVME_SC_COMMAND_INTERRUPTED		= 0x21,
+	SPDK_NVME_SC_COMMAND_TRANSIENT_TRANSPORT_ERROR	= 0x22,
 
 	SPDK_NVME_SC_LBA_OUT_OF_RANGE			= 0x80,
 	SPDK_NVME_SC_CAPACITY_EXCEEDED			= 0x81,
@@ -1443,6 +1474,7 @@ enum spdk_nvme_command_specific_status_code {
 	SPDK_NVME_SC_CONFLICTING_ATTRIBUTES		= 0x80,
 	SPDK_NVME_SC_INVALID_PROTECTION_INFO		= 0x81,
 	SPDK_NVME_SC_ATTEMPTED_WRITE_TO_RO_RANGE	= 0x82,
+	SPDK_NVME_SC_CMD_SIZE_LIMIT_SIZE_EXCEEDED	= 0x83,
 };
 
 /**
@@ -1539,6 +1571,8 @@ enum spdk_nvme_nvm_opcode {
 
 	SPDK_NVME_OPC_RESERVATION_ACQUIRE		= 0x11,
 	SPDK_NVME_OPC_RESERVATION_RELEASE		= 0x15,
+
+	SPDK_NVME_OPC_COPY				= 0x19,
 };
 
 /**
@@ -1822,7 +1856,7 @@ struct spdk_nvme_cdata_sgls {
 	uint32_t	reserved2 : 10;
 };
 
-struct __attribute__((packed)) __attribute__((aligned)) spdk_nvme_ctrlr_data {
+struct __attribute__((packed)) spdk_nvme_ctrlr_data {
 	/* bytes 0-255: controller capabilities and features */
 
 	/** pci vendor id */
@@ -1849,7 +1883,7 @@ struct __attribute__((packed)) __attribute__((aligned)) spdk_nvme_ctrlr_data {
 	/** controller multi-path I/O and namespace sharing capabilities */
 	struct {
 		uint8_t multi_port	: 1;
-		uint8_t multi_host	: 1;
+		uint8_t multi_ctrlr	: 1;
 		uint8_t sr_iov		: 1;
 		uint8_t ana_reporting	: 1;
 		uint8_t reserved	: 4;
@@ -2149,7 +2183,9 @@ struct __attribute__((packed)) __attribute__((aligned)) spdk_nvme_ctrlr_data {
 		uint16_t	set_features_save: 1;
 		uint16_t	reservations: 1;
 		uint16_t	timestamp: 1;
-		uint16_t	reserved: 9;
+		uint16_t	verify: 1;
+		uint16_t	copy: 1;
+		uint16_t	reserved9: 7;
 	} oncs;
 
 	/** fused operation support */
@@ -2187,7 +2223,12 @@ struct __attribute__((packed)) __attribute__((aligned)) spdk_nvme_ctrlr_data {
 	/** atomic compare & write unit */
 	uint16_t		acwu;
 
-	uint16_t		reserved534;
+	/** optional copy formats supported */
+	struct {
+		uint16_t	copy_format0 : 1;
+		uint16_t	reserved1: 15;
+	} ocfs;
+
 
 	struct spdk_nvme_cdata_sgls sgls;
 
@@ -2479,7 +2520,16 @@ struct spdk_nvme_ns_data {
 	/** Namespace Optimal Write Size */
 	uint16_t                nows;
 
-	uint8_t			reserved64[18];
+	/** Maximum Single Source Range Length */
+	uint16_t                mssrl;
+
+	/** Maximum Copy Length */
+	uint32_t                mcl;
+
+	/** Maximum Source Range Count */
+	uint8_t	                msrc;
+
+	uint8_t			reserved64[11];
 
 	/** ANA group identifier */
 	uint32_t		anagrpid;
@@ -2794,8 +2844,12 @@ enum spdk_nvme_log_page {
 	/* 0x81-0xBF - I/O command set specific */
 
 	/* 0xC0-0xFF - vendor specific */
+	SPDK_NVME_LOG_VENDOR_SPECIFIC_START	= 0xc0,
+	SPDK_NVME_LOG_VENDOR_SPECIFIC_END	= 0xff,
 };
 
+#define spdk_nvme_log_page_is_vendor_specific(lid) ((lid) >= SPDK_NVME_LOG_VENDOR_SPECIFIC_START)
+
 /**
  * Error information log page (\ref SPDK_NVME_LOG_ERROR)
  */
@@ -3284,6 +3338,16 @@ enum spdk_nvme_fw_commit_action {
 	 * requested to be activated immediately without reset.
 	 */
 	SPDK_NVME_FW_COMMIT_RUN_IMG			= 0x3,
+	/**
+	 * Downloaded image replaces the Boot Partition specified by
+	 * the Boot Partition ID field.
+	 */
+	SPDK_NVME_FW_COMMIT_REPLACE_BOOT_PARTITION	= 0x6,
+	/**
+	 * Mark the Boot Partition specified in the BPID field as Active
+	 * and update BPINFO.ABPID.
+	 */
+	SPDK_NVME_FW_COMMIT_ACTIVATE_BOOT_PARTITION	= 0x7,
 };
 
 /** Parameters for SPDK_NVME_OPC_FIRMWARE_COMMIT cdw10 */
@@ -3300,7 +3364,12 @@ struct spdk_nvme_fw_commit {
 	 * placed image.
 	 */
 	uint32_t	ca		: 3;
-	uint32_t	reserved	: 26;
+	uint32_t	reserved	: 25;
+	/**
+	 * Boot Partition ID. Specifies the boot partition that shall be used for the
+	 * Commit Action.
+	 */
+	uint32_t	bpid		: 1;
 };
 SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_fw_commit) == 4, "Incorrect size");
 

include/spdk/nvmf.h

@@ -70,6 +70,7 @@ struct spdk_nvmf_target_opts {
 	char		name[NVMF_TGT_NAME_MAX_LENGTH];
 	uint32_t	max_subsystems;
 	uint32_t	acceptor_poll_rate;
+	uint16_t	crdt[3];
 };
 
 struct spdk_nvmf_transport_opts {
@@ -120,46 +121,17 @@ struct spdk_nvmf_listen_opts {
 void spdk_nvmf_listen_opts_init(struct spdk_nvmf_listen_opts *opts, size_t opts_size);
 
 struct spdk_nvmf_poll_group_stat {
+	/* cumulative admin qpair count */
 	uint32_t admin_qpairs;
+	/* cumulative io qpair count */
 	uint32_t io_qpairs;
+	/* current admin qpair count */
+	uint32_t current_admin_qpairs;
+	/* current io qpair count */
+	uint32_t current_io_qpairs;
 	uint64_t pending_bdev_io;
 };
 
-/* Deprecated.
- * Please use the flow with spdk_nvmf_poll_group_dump_stat,
- * which hides statistics structures within the transport.
- */
-struct spdk_nvmf_rdma_device_stat {
-	const char *name;
-	uint64_t polls;
-	uint64_t idle_polls;
-	uint64_t completions;
-	uint64_t requests;
-	uint64_t request_latency;
-	uint64_t pending_free_request;
-	uint64_t pending_rdma_read;
-	uint64_t pending_rdma_write;
-	uint64_t total_send_wrs;
-	uint64_t send_doorbell_updates;
-	uint64_t total_recv_wrs;
-	uint64_t recv_doorbell_updates;
-};
-
-/* Deprecated.
- * Please use the flow with spdk_nvmf_poll_group_dump_stat,
- * which hides statistics structures within the transport.
- */
-struct spdk_nvmf_transport_poll_group_stat {
-	spdk_nvme_transport_type_t trtype;
-	union {
-		struct {
-			uint64_t pending_data_buffer;
-			uint64_t num_devices;
-			struct spdk_nvmf_rdma_device_stat *devices;
-		} rdma;
-	};
-};
-
 /**
  * Function to be called once asynchronous listen add and remove
  * operations are completed. See spdk_nvmf_subsystem_add_listener()
@@ -313,18 +285,6 @@ void spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group,
 int spdk_nvmf_poll_group_add(struct spdk_nvmf_poll_group *group,
 			     struct spdk_nvmf_qpair *qpair);
 
-/**
- * Get current poll group statistics. (deprecated)
- *
- * \param tgt The NVMf target.
- * \param stat Pointer to allocated statistics structure to fill with values.
- *
- * \return 0 upon success.
- * \return -EINVAL if either group or stat is NULL.
- */
-int spdk_nvmf_poll_group_get_stat(struct spdk_nvmf_tgt *tgt,
-				  struct spdk_nvmf_poll_group_stat *stat);
-
 typedef void (*nvmf_qpair_disconnect_cb)(void *ctx);
 
 /**
@@ -844,6 +804,24 @@ struct spdk_nvmf_ns *spdk_nvmf_subsystem_get_ns(struct spdk_nvmf_subsystem *subs
  */
 uint32_t spdk_nvmf_subsystem_get_max_namespaces(const struct spdk_nvmf_subsystem *subsystem);
 
+/**
+ * Get the minimum controller ID allowed in a subsystem.
+ *
+ * \param subsystem Subsystem to query.
+ *
+ * \return Minimum controller ID allowed in the subsystem.
+ */
+uint16_t spdk_nvmf_subsystem_get_min_cntlid(const struct spdk_nvmf_subsystem *subsystem);
+
+/**
+ * Get the maximum controller ID allowed in a subsystem.
+ *
+ * \param subsystem Subsystem to query.
+ *
+ * \return Maximum controller ID allowed in the subsystem.
+ */
+uint16_t spdk_nvmf_subsystem_get_max_cntlid(const struct spdk_nvmf_subsystem *subsystem);
+
 /**
  * Get a namespace's NSID.
  *
@@ -1105,44 +1083,6 @@ int spdk_nvmf_transport_stop_listen_async(struct spdk_nvmf_transport *transport,
 		spdk_nvmf_tgt_subsystem_listen_done_fn cb_fn,
 		void *cb_arg);
 
-
-/**
- * \brief Get current transport poll group statistics. (deprecated)
- *
- * Please use the flow with spdk_nvmf_poll_group_dump_stat.
- *
- * This function allocates memory for statistics and returns it
- * in \p stat parameter. Caller must free this memory with
- * spdk_nvmf_transport_poll_group_free_stat() when it is not needed
- * anymore.
- *
- * \param tgt The NVMf target.
- * \param transport The NVMf transport.
- * \param stat Output parameter that will contain pointer to allocated statistics structure.
- *
- * \return 0 upon success.
- * \return -ENOTSUP if transport does not support statistics.
- * \return -EINVAL if any of parameters is NULL.
- * \return -ENOENT if transport poll group is not found.
- * \return -ENOMEM if memory allocation failed.
- */
-int
-spdk_nvmf_transport_poll_group_get_stat(struct spdk_nvmf_tgt *tgt,
-					struct spdk_nvmf_transport *transport,
-					struct spdk_nvmf_transport_poll_group_stat **stat);
-
-/**
- * Free statistics memory previously allocated with spdk_nvmf_transport_poll_group_get_stat(). (deprecated)
- *
- * Please use the flow with spdk_nvmf_poll_group_dump_stat.
- *
- * \param transport The NVMf transport.
- * \param stat Pointer to transport poll group statistics structure.
- */
-void
-spdk_nvmf_transport_poll_group_free_stat(struct spdk_nvmf_transport *transport,
-		struct spdk_nvmf_transport_poll_group_stat *stat);
-
 /**
  * Dump poll group statistics into JSON.
  *

include/spdk/nvmf_transport.h

@@ -2,7 +2,7 @@
  *   BSD LICENSE
  *
  *   Copyright (c) Intel Corporation. All rights reserved.
- *   Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved.
+ *   Copyright (c) 2019, 2021 Mellanox Technologies LTD. All rights reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
@@ -74,31 +74,51 @@ SPDK_STATIC_ASSERT(sizeof(union nvmf_c2h_msg) == 16, "Incorrect size");
 
 struct spdk_nvmf_dif_info {
 	struct spdk_dif_ctx			dif_ctx;
-	bool					dif_insert_or_strip;
 	uint32_t				elba_length;
 	uint32_t				orig_length;
 };
 
+enum spdk_nvmf_zcopy_phase {
+	NVMF_ZCOPY_PHASE_NONE,        /* Request is not using ZCOPY */
+	NVMF_ZCOPY_PHASE_INIT,        /* Requesting Buffers */
+	NVMF_ZCOPY_PHASE_EXECUTE,     /* Got buffers processing commands */
+	NVMF_ZCOPY_PHASE_END_PENDING, /* Releasing buffers */
+	NVMF_ZCOPY_PHASE_COMPLETE,    /* Buffers Released */
+	NVMF_ZCOPY_PHASE_INIT_FAILED  /* Failed to get the buffers */
+};
+
+static inline bool
+spdk_nvmf_using_zcopy(enum spdk_nvmf_zcopy_phase phase)
+{
+	return (phase != NVMF_ZCOPY_PHASE_NONE);
+}
+
 struct spdk_nvmf_request {
 	struct spdk_nvmf_qpair		*qpair;
 	uint32_t			length;
-	enum spdk_nvme_data_transfer	xfer;
+	uint8_t				xfer; /* type enum spdk_nvme_data_transfer */
+	bool				data_from_pool;
+	bool				dif_enabled;
 	void				*data;
 	union nvmf_h2c_msg		*cmd;
 	union nvmf_c2h_msg		*rsp;
-	void				*buffers[NVMF_REQ_MAX_BUFFERS];
-	struct iovec			iov[NVMF_REQ_MAX_BUFFERS];
+	STAILQ_ENTRY(spdk_nvmf_request)	buf_link;
+	uint64_t			timeout_tsc;
+
 	uint32_t			iovcnt;
-	bool				data_from_pool;
-	struct spdk_bdev_io_wait_entry	bdev_io_wait;
+	struct iovec			iov[NVMF_REQ_MAX_BUFFERS];
+	void				*buffers[NVMF_REQ_MAX_BUFFERS];
+
 	struct spdk_nvmf_dif_info	dif;
+
+	struct spdk_bdev_io_wait_entry	bdev_io_wait;
 	spdk_nvmf_nvme_passthru_cmd_cb	cmd_cb_fn;
 	struct spdk_nvmf_request	*first_fused_req;
 	struct spdk_nvmf_request	*req_to_abort;
 	struct spdk_poller		*poller;
-	uint64_t			timeout_tsc;
+	struct spdk_bdev_io		*zcopy_bdev_io; /* Contains the bdev_io when using ZCOPY */
+	enum spdk_nvmf_zcopy_phase	zcopy_phase;
 
-	STAILQ_ENTRY(spdk_nvmf_request)	buf_link;
 	TAILQ_ENTRY(spdk_nvmf_request)	link;
 };
 
@@ -367,19 +387,6 @@ struct spdk_nvmf_transport_ops {
 	void (*qpair_abort_request)(struct spdk_nvmf_qpair *qpair,
 				    struct spdk_nvmf_request *req);
 
-	/*
-	 * Get transport poll group statistics. (deprecated)
-	 * Please use the flow with spdk_nvmf_poll_group_dump_stat.
-	 */
-	int (*poll_group_get_stat)(struct spdk_nvmf_tgt *tgt,
-				   struct spdk_nvmf_transport_poll_group_stat **stat);
-
-	/*
-	 * Free transport poll group statistics previously allocated with poll_group_get_stat(). (deprecated)
-	 * Please use the flow with spdk_nvmf_poll_group_dump_stat.
-	 */
-	void (*poll_group_free_stat)(struct spdk_nvmf_transport_poll_group_stat *stat);
-
 	/*
 	 * Dump transport poll group statistics into JSON.
 	 */
@@ -429,10 +436,6 @@ int spdk_nvmf_request_get_buffers(struct spdk_nvmf_request *req,
 				  struct spdk_nvmf_transport_poll_group *group,
 				  struct spdk_nvmf_transport *transport,
 				  uint32_t length);
-int spdk_nvmf_request_get_buffers_multi(struct spdk_nvmf_request *req,
-					struct spdk_nvmf_transport_poll_group *group,
-					struct spdk_nvmf_transport *transport,
-					uint32_t *lengths, uint32_t num_lengths);
 
 bool spdk_nvmf_request_get_dif_ctx(struct spdk_nvmf_request *req, struct spdk_dif_ctx *dif_ctx);
 
@@ -440,6 +443,8 @@ void spdk_nvmf_request_exec(struct spdk_nvmf_request *req);
 void spdk_nvmf_request_exec_fabrics(struct spdk_nvmf_request *req);
 int spdk_nvmf_request_free(struct spdk_nvmf_request *req);
 int spdk_nvmf_request_complete(struct spdk_nvmf_request *req);
+int spdk_nvmf_request_zcopy_start(struct spdk_nvmf_request *req);
+int spdk_nvmf_request_zcopy_end(struct spdk_nvmf_request *req, bool commit);
 
 /**
  * Remove the given qpair from the poll group.

include/spdk/pci_ids.h

@@ -52,6 +52,7 @@ extern "C" {
 #define SPDK_PCI_VID_VIRTIO		0x1af4
 #define SPDK_PCI_VID_CNEXLABS		0x1d1d
 #define SPDK_PCI_VID_VMWARE		0x15ad
+#define SPDK_PCI_VID_REDHAT		0x1b36
 
 #define SPDK_PCI_CLASS_ANY_ID		0xffffff
 /**

include/spdk/sock.h

@@ -111,6 +111,7 @@ struct spdk_sock_impl_opts {
 	bool enable_recv_pipe;
 
 	/**
+	 * **Deprecated, please use enable_zerocopy_send_server or enable_zerocopy_send_client instead**
 	 * Enable or disable use of zero copy flow on send. Used by posix socket module.
 	 */
 	bool enable_zerocopy_send;
@@ -126,6 +127,15 @@ struct spdk_sock_impl_opts {
 	 */
 	uint32_t enable_placement_id;
 
+	/**
+	 * Enable or disable use of zero copy flow on send for server sockets. Used by posix socket module.
+	 */
+	bool enable_zerocopy_send_server;
+
+	/**
+	 * Enable or disable use of zero copy flow on send for client sockets. Used by posix socket module.
+	 */
+	bool enable_zerocopy_send_client;
 };
 
 /**

include/spdk/stdinc.h

@@ -50,6 +50,7 @@ extern "C" {
 #include <errno.h>
 #include <inttypes.h>
 #include <limits.h>
+#include <math.h>
 #include <stdarg.h>
 #include <stdbool.h>
 #include <stddef.h>

include/spdk/thread.h

@@ -39,9 +39,7 @@
 #define SPDK_THREAD_H_
 
 #include "spdk/stdinc.h"
-
 #include "spdk/cpuset.h"
-#include "spdk/queue.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -210,30 +208,7 @@ typedef void (*spdk_channel_msg)(struct spdk_io_channel_iter *i);
  */
 typedef void (*spdk_channel_for_each_cpl)(struct spdk_io_channel_iter *i, int status);
 
-/**
- * \brief Represents a per-thread channel for accessing an I/O device.
- *
- * An I/O device may be a physical entity (i.e. NVMe controller) or a software
- *  entity (i.e. a blobstore).
- *
- * This structure is not part of the API - all accesses should be done through
- *  spdk_io_channel function calls.
- */
-struct spdk_io_channel {
-	struct spdk_thread		*thread;
-	struct io_device		*dev;
-	uint32_t			ref;
-	uint32_t			destroy_ref;
-	TAILQ_ENTRY(spdk_io_channel)	tailq;
-	spdk_io_channel_destroy_cb	destroy_cb;
-
-	/*
-	 * Modules will allocate extra memory off the end of this structure
-	 *  to store references to hardware-specific references (i.e. NVMe queue
-	 *  pairs, or references to child device spdk_io_channels (i.e.
-	 *  virtual bdevs).
-	 */
-};
+#define SPDK_IO_CHANNEL_STRUCT_SIZE	96
 
 /**
  * Initialize the threading library. Must be called once prior to allocating any threads.
@@ -688,7 +663,7 @@ void spdk_put_io_channel(struct spdk_io_channel *ch);
 static inline void *
 spdk_io_channel_get_ctx(struct spdk_io_channel *ch)
 {
-	return (uint8_t *)ch + sizeof(*ch);
+	return (uint8_t *)ch + SPDK_IO_CHANNEL_STRUCT_SIZE;
 }
 
 /**

include/spdk/trace.h

@@ -40,6 +40,7 @@
 #define _SPDK_TRACE_H_
 
 #include "spdk/stdinc.h"
+#include "spdk/assert.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -53,9 +54,18 @@ struct spdk_trace_entry {
 	uint16_t	poller_id;
 	uint32_t	size;
 	uint64_t	object_id;
-	uint64_t	arg1;
+	uint8_t		args[8];
 };
 
+struct spdk_trace_entry_buffer {
+	uint64_t	tsc;
+	uint16_t	tpoint_id;
+	uint8_t		data[22];
+};
+
+SPDK_STATIC_ASSERT(sizeof(struct spdk_trace_entry_buffer) == sizeof(struct spdk_trace_entry),
+		   "Invalid size of trace entry buffer");
+
 /* If type changes from a uint8_t, change this value. */
 #define SPDK_TRACE_MAX_OWNER (UCHAR_MAX + 1)
 
@@ -80,15 +90,22 @@ struct spdk_trace_object {
 #define SPDK_TRACE_ARG_TYPE_PTR 1
 #define SPDK_TRACE_ARG_TYPE_STR 2
 
+#define SPDK_TRACE_MAX_ARGS_COUNT 5
+
+struct spdk_trace_argument {
+	char	name[14];
+	uint8_t	type;
+	uint8_t	size;
+};
+
 struct spdk_trace_tpoint {
-	char		name[24];
-	uint16_t	tpoint_id;
-	uint8_t		owner_type;
-	uint8_t		object_type;
-	uint8_t		new_object;
-	uint8_t		arg1_type;
-	uint8_t		reserved;
-	char		arg1_name[8];
+	char				name[24];
+	uint16_t			tpoint_id;
+	uint8_t				owner_type;
+	uint8_t				object_type;
+	uint8_t				new_object;
+	uint8_t				num_args;
+	struct spdk_trace_argument	args[SPDK_TRACE_MAX_ARGS_COUNT];
 };
 
 struct spdk_trace_history {
@@ -175,37 +192,24 @@ spdk_get_per_lcore_history(struct spdk_trace_histories *trace_histories, unsigne
 }
 
 void _spdk_trace_record(uint64_t tsc, uint16_t tpoint_id, uint16_t poller_id,
-			uint32_t size, uint64_t object_id, uint64_t arg1);
+			uint32_t size, uint64_t object_id, int num_args, ...);
 
-/**
- * Record the current trace state for tracing tpoints. Debug tool can read the
- * information from shared memory to post-process the tpoint entries and display
- * in a human-readable format. This function will call spdk_get_ticks() to get
- * the current tsc to save in the tracepoint.
- *
- * \param tpoint_id Tracepoint id to record.
- * \param poller_id Poller id to record.
- * \param size Size to record.
- * \param object_id Object id to record.
- * \param arg1 Argument to record.
- */
-static inline
-void spdk_trace_record(uint16_t tpoint_id, uint16_t poller_id, uint32_t size,
-		       uint64_t object_id, uint64_t arg1)
-{
-	/*
-	 * Tracepoint group ID is encoded in the tpoint_id.  Lower 6 bits determine the tracepoint
-	 *  within the group, the remaining upper bits determine the tracepoint group.  Each
-	 *  tracepoint group has its own tracepoint mask.
-	 */
-	assert(tpoint_id < SPDK_TRACE_MAX_TPOINT_ID);
-	if (g_trace_histories == NULL ||
-	    !((1ULL << (tpoint_id & 0x3F)) & g_trace_histories->flags.tpoint_mask[tpoint_id >> 6])) {
-		return;
-	}
+#define _spdk_trace_record_tsc(tsc, tpoint_id, poller_id, size, object_id, num_args, ...)	\
+	do {											\
+		assert(tpoint_id < SPDK_TRACE_MAX_TPOINT_ID);					\
+		if (g_trace_histories == NULL ||						\
+		    !((1ULL << (tpoint_id & 0x3F)) &						\
+		      g_trace_histories->flags.tpoint_mask[tpoint_id >> 6])) {			\
+			break;									\
+		}										\
+		_spdk_trace_record(tsc, tpoint_id, poller_id, size, object_id,			\
+				   num_args, ## __VA_ARGS__);					\
+	} while (0)
 
-	_spdk_trace_record(0, tpoint_id, poller_id, size, object_id, arg1);
-}
+/* Return the number of variable arguments. */
+#define spdk_trace_num_args(...) _spdk_trace_num_args(, ## __VA_ARGS__)
+#define _spdk_trace_num_args(...) __spdk_trace_num_args(__VA_ARGS__, 8, 7, 6, 5, 4, 3, 2, 1, 0)
+#define __spdk_trace_num_args(v, a1, a2, a3, a4, a5, a6, a7, a8, count, ...) count
 
 /**
  * Record the current trace state for tracing tpoints. Debug tool can read the
@@ -217,25 +221,28 @@ void spdk_trace_record(uint16_t tpoint_id, uint16_t poller_id, uint32_t size,
  * \param poller_id Poller id to record.
  * \param size Size to record.
  * \param object_id Object id to record.
- * \param arg1 Argument to record.
+ * \param ... Extra tracepoint arguments. The number, types, and order of the arguments
+ *	      must match the definition of the tracepoint.
  */
-static inline
-void spdk_trace_record_tsc(uint64_t tsc, uint16_t tpoint_id, uint16_t poller_id,
-			   uint32_t size, uint64_t object_id, uint64_t arg1)
-{
-	/*
-	 * Tracepoint group ID is encoded in the tpoint_id.  Lower 6 bits determine the tracepoint
-	 *  within the group, the remaining upper bits determine the tracepoint group.  Each
-	 *  tracepoint group has its own tracepoint mask.
-	 */
-	assert(tpoint_id < SPDK_TRACE_MAX_TPOINT_ID);
-	if (g_trace_histories == NULL ||
-	    !((1ULL << (tpoint_id & 0x3F)) & g_trace_histories->flags.tpoint_mask[tpoint_id >> 6])) {
-		return;
-	}
+#define spdk_trace_record_tsc(tsc, tpoint_id, poller_id, size, object_id, ...)	\
+	_spdk_trace_record_tsc(tsc, tpoint_id, poller_id, size, object_id,	\
+			       spdk_trace_num_args(__VA_ARGS__), ## __VA_ARGS__)
 
-	_spdk_trace_record(tsc, tpoint_id, poller_id, size, object_id, arg1);
-}
+/**
+ * Record the current trace state for tracing tpoints. Debug tool can read the
+ * information from shared memory to post-process the tpoint entries and display
+ * in a human-readable format. This macro will call spdk_get_ticks() to get
+ * the current tsc to save in the tracepoint.
+ *
+ * \param tpoint_id Tracepoint id to record.
+ * \param poller_id Poller id to record.
+ * \param size Size to record.
+ * \param object_id Object id to record.
+ * \param ... Extra tracepoint arguments. The number, types, and order of the arguments
+ *	      must match the definition of the tracepoint.
+ */
+#define spdk_trace_record(tpoint_id, poller_id, size, object_id, ...) \
+	spdk_trace_record_tsc(0, tpoint_id, poller_id, size, object_id, ## __VA_ARGS__)
 
 /**
  * Get the current tpoint mask of the given tpoint group.
@@ -326,7 +333,7 @@ void spdk_trace_register_owner(uint8_t type, char id_prefix);
 void spdk_trace_register_object(uint8_t type, char id_prefix);
 
 /**
- * Register the description for the tpoint.
+ * Register the description for a tpoint with a single argument.
  *
  * \param name Name for the tpoint.
  * \param tpoint_id Id for the tpoint.
@@ -340,6 +347,29 @@ void spdk_trace_register_description(const char *name, uint16_t tpoint_id, uint8
 				     uint8_t object_type, uint8_t new_object,
 				     uint8_t arg1_type, const char *arg1_name);
 
+struct spdk_trace_tpoint_opts {
+	const char	*name;
+	uint16_t	tpoint_id;
+	uint8_t		owner_type;
+	uint8_t		object_type;
+	uint8_t		new_object;
+	struct {
+		const char	*name;
+		uint8_t		type;
+		uint8_t		size;
+	} args[SPDK_TRACE_MAX_ARGS_COUNT];
+};
+
+/**
+ * Register the description for a number of tpoints. This function allows the user to register
+ * tracepoints with multiple arguments (up to 5).
+ *
+ * \param opts Array of structures describing tpoints and their arguments.
+ * \param num_opts Number of tpoints to register (size of the opts array).
+ */
+void spdk_trace_register_description_ext(const struct spdk_trace_tpoint_opts *opts,
+		size_t num_opts);
+
 struct spdk_trace_register_fn *spdk_trace_get_first_register_fn(void);
 
 struct spdk_trace_register_fn *spdk_trace_get_next_register_fn(struct spdk_trace_register_fn

include/spdk/tree.h

@@ -0,0 +1,834 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright 2002 Niels Provos <provos@citi.umich.edu>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef	SPDK_TREE_H
+#define SPDK_TREE_H
+
+#include <sys/cdefs.h>
+
+/*
+ * This file defines data structures for different types of trees:
+ * splay trees and rank-balanced trees.
+ *
+ * A splay tree is a self-organizing data structure.  Every operation
+ * on the tree causes a splay to happen.  The splay moves the requested
+ * node to the root of the tree and partly rebalances it.
+ *
+ * This has the benefit that request locality causes faster lookups as
+ * the requested nodes move to the top of the tree.  On the other hand,
+ * every lookup causes memory writes.
+ *
+ * The Balance Theorem bounds the total access time for m operations
+ * and n inserts on an initially empty tree as O((m + n)lg n).  The
+ * amortized cost for a sequence of m accesses to a splay tree is O(lg n);
+ *
+ * A rank-balanced tree is a binary search tree with an integer
+ * rank-difference as an attribute of each pointer from parent to child.
+ * The sum of the rank-differences on any path from a node down to null is
+ * the same, and defines the rank of that node. The rank of the null node
+ * is -1.
+ *
+ * Different additional conditions define different sorts of balanced
+ * trees, including "red-black" and "AVL" trees.  The set of conditions
+ * applied here are the "weak-AVL" conditions of Haeupler, Sen and Tarjan:
+ *	- every rank-difference is 1 or 2.
+ *	- the rank of any leaf is 1.
+ *
+ * For historical reasons, rank differences that are even are associated
+ * with the color red (Rank-Even-Difference), and the child that a red edge
+ * points to is called a red child.
+ *
+ * Every operation on a rank-balanced tree is bounded as O(lg n).
+ * The maximum height of a rank-balanced tree is 2lg (n+1).
+ */
+
+#define SPLAY_HEAD(name, type)						\
+struct name {								\
+	struct type *sph_root; /* root of the tree */			\
+}
+
+#define SPLAY_INITIALIZER(root)						\
+	{ NULL }
+
+#define SPLAY_INIT(root) do {						\
+	(root)->sph_root = NULL;					\
+} while (/* CONSTCOND */ 0)
+
+#define SPLAY_ENTRY(type)						\
+struct {								\
+	struct type *spe_left; /* left element */			\
+	struct type *spe_right; /* right element */			\
+}
+
+#define SPLAY_LEFT(elm, field)		(elm)->field.spe_left
+#define SPLAY_RIGHT(elm, field)		(elm)->field.spe_right
+#define SPLAY_ROOT(head)		(head)->sph_root
+#define SPLAY_EMPTY(head)		(SPLAY_ROOT(head) == NULL)
+
+/* SPLAY_ROTATE_{LEFT,RIGHT} expect that tmp hold SPLAY_{RIGHT,LEFT} */
+#define SPLAY_ROTATE_RIGHT(head, tmp, field) do {			\
+	SPLAY_LEFT((head)->sph_root, field) = SPLAY_RIGHT(tmp, field);	\
+	SPLAY_RIGHT(tmp, field) = (head)->sph_root;			\
+	(head)->sph_root = tmp;						\
+} while (/* CONSTCOND */ 0)
+
+#define SPLAY_ROTATE_LEFT(head, tmp, field) do {			\
+	SPLAY_RIGHT((head)->sph_root, field) = SPLAY_LEFT(tmp, field);	\
+	SPLAY_LEFT(tmp, field) = (head)->sph_root;			\
+	(head)->sph_root = tmp;						\
+} while (/* CONSTCOND */ 0)
+
+#define SPLAY_LINKLEFT(head, tmp, field) do {				\
+	SPLAY_LEFT(tmp, field) = (head)->sph_root;			\
+	tmp = (head)->sph_root;						\
+	(head)->sph_root = SPLAY_LEFT((head)->sph_root, field);		\
+} while (/* CONSTCOND */ 0)
+
+#define SPLAY_LINKRIGHT(head, tmp, field) do {				\
+	SPLAY_RIGHT(tmp, field) = (head)->sph_root;			\
+	tmp = (head)->sph_root;						\
+	(head)->sph_root = SPLAY_RIGHT((head)->sph_root, field);	\
+} while (/* CONSTCOND */ 0)
+
+#define SPLAY_ASSEMBLE(head, node, left, right, field) do {		\
+	SPLAY_RIGHT(left, field) = SPLAY_LEFT((head)->sph_root, field);	\
+	SPLAY_LEFT(right, field) = SPLAY_RIGHT((head)->sph_root, field);\
+	SPLAY_LEFT((head)->sph_root, field) = SPLAY_RIGHT(node, field);	\
+	SPLAY_RIGHT((head)->sph_root, field) = SPLAY_LEFT(node, field);	\
+} while (/* CONSTCOND */ 0)
+
+/* Generates prototypes and inline functions */
+
+#define SPLAY_PROTOTYPE(name, type, field, cmp)				\
+void name##_SPLAY(struct name *, struct type *);			\
+void name##_SPLAY_MINMAX(struct name *, int);				\
+struct type *name##_SPLAY_INSERT(struct name *, struct type *);		\
+struct type *name##_SPLAY_REMOVE(struct name *, struct type *);		\
+									\
+/* Finds the node with the same key as elm */				\
+static __attribute__((unused)) __inline struct type *					\
+name##_SPLAY_FIND(struct name *head, struct type *elm)			\
+{									\
+	if (SPLAY_EMPTY(head))						\
+		return(NULL);						\
+	name##_SPLAY(head, elm);					\
+	if ((cmp)(elm, (head)->sph_root) == 0)				\
+		return (head->sph_root);				\
+	return (NULL);							\
+}									\
+									\
+static __attribute__((unused)) __inline struct type *					\
+name##_SPLAY_NEXT(struct name *head, struct type *elm)			\
+{									\
+	name##_SPLAY(head, elm);					\
+	if (SPLAY_RIGHT(elm, field) != NULL) {				\
+		elm = SPLAY_RIGHT(elm, field);				\
+		while (SPLAY_LEFT(elm, field) != NULL) {		\
+			elm = SPLAY_LEFT(elm, field);			\
+		}							\
+	} else								\
+		elm = NULL;						\
+	return (elm);							\
+}									\
+									\
+static __attribute__((unused)) __inline struct type *					\
+name##_SPLAY_MIN_MAX(struct name *head, int val)			\
+{									\
+	name##_SPLAY_MINMAX(head, val);					\
+        return (SPLAY_ROOT(head));					\
+}
+
+/* Main splay operation.
+ * Moves node close to the key of elm to top
+ */
+#define SPLAY_GENERATE(name, type, field, cmp)				\
+struct type *								\
+name##_SPLAY_INSERT(struct name *head, struct type *elm)		\
+{									\
+    if (SPLAY_EMPTY(head)) {						\
+	    SPLAY_LEFT(elm, field) = SPLAY_RIGHT(elm, field) = NULL;	\
+    } else {								\
+	    int __comp;							\
+	    name##_SPLAY(head, elm);					\
+	    __comp = (cmp)(elm, (head)->sph_root);			\
+	    if (__comp < 0) {						\
+		    SPLAY_LEFT(elm, field) = SPLAY_LEFT((head)->sph_root, field);\
+		    SPLAY_RIGHT(elm, field) = (head)->sph_root;		\
+		    SPLAY_LEFT((head)->sph_root, field) = NULL;		\
+	    } else if (__comp > 0) {					\
+		    SPLAY_RIGHT(elm, field) = SPLAY_RIGHT((head)->sph_root, field);\
+		    SPLAY_LEFT(elm, field) = (head)->sph_root;		\
+		    SPLAY_RIGHT((head)->sph_root, field) = NULL;	\
+	    } else							\
+		    return ((head)->sph_root);				\
+    }									\
+    (head)->sph_root = (elm);						\
+    return (NULL);							\
+}									\
+									\
+struct type *								\
+name##_SPLAY_REMOVE(struct name *head, struct type *elm)		\
+{									\
+	struct type *__tmp;						\
+	if (SPLAY_EMPTY(head))						\
+		return (NULL);						\
+	name##_SPLAY(head, elm);					\
+	if ((cmp)(elm, (head)->sph_root) == 0) {			\
+		if (SPLAY_LEFT((head)->sph_root, field) == NULL) {	\
+			(head)->sph_root = SPLAY_RIGHT((head)->sph_root, field);\
+		} else {						\
+			__tmp = SPLAY_RIGHT((head)->sph_root, field);	\
+			(head)->sph_root = SPLAY_LEFT((head)->sph_root, field);\
+			name##_SPLAY(head, elm);			\
+			SPLAY_RIGHT((head)->sph_root, field) = __tmp;	\
+		}							\
+		return (elm);						\
+	}								\
+	return (NULL);							\
+}									\
+									\
+void									\
+name##_SPLAY(struct name *head, struct type *elm)			\
+{									\
+	struct type __node, *__left, *__right, *__tmp;			\
+	int __comp;							\
+\
+	SPLAY_LEFT(&__node, field) = SPLAY_RIGHT(&__node, field) = NULL;\
+	__left = __right = &__node;					\
+\
+	while ((__comp = (cmp)(elm, (head)->sph_root)) != 0) {		\
+		if (__comp < 0) {					\
+			__tmp = SPLAY_LEFT((head)->sph_root, field);	\
+			if (__tmp == NULL)				\
+				break;					\
+			if ((cmp)(elm, __tmp) < 0){			\
+				SPLAY_ROTATE_RIGHT(head, __tmp, field);	\
+				if (SPLAY_LEFT((head)->sph_root, field) == NULL)\
+					break;				\
+			}						\
+			SPLAY_LINKLEFT(head, __right, field);		\
+		} else if (__comp > 0) {				\
+			__tmp = SPLAY_RIGHT((head)->sph_root, field);	\
+			if (__tmp == NULL)				\
+				break;					\
+			if ((cmp)(elm, __tmp) > 0){			\
+				SPLAY_ROTATE_LEFT(head, __tmp, field);	\
+				if (SPLAY_RIGHT((head)->sph_root, field) == NULL)\
+					break;				\
+			}						\
+			SPLAY_LINKRIGHT(head, __left, field);		\
+		}							\
+	}								\
+	SPLAY_ASSEMBLE(head, &__node, __left, __right, field);		\
+}									\
+									\
+/* Splay with either the minimum or the maximum element			\
+ * Used to find minimum or maximum element in tree.			\
+ */									\
+void name##_SPLAY_MINMAX(struct name *head, int __comp) \
+{									\
+	struct type __node, *__left, *__right, *__tmp;			\
+\
+	SPLAY_LEFT(&__node, field) = SPLAY_RIGHT(&__node, field) = NULL;\
+	__left = __right = &__node;					\
+\
+	while (1) {							\
+		if (__comp < 0) {					\
+			__tmp = SPLAY_LEFT((head)->sph_root, field);	\
+			if (__tmp == NULL)				\
+				break;					\
+			if (__comp < 0){				\
+				SPLAY_ROTATE_RIGHT(head, __tmp, field);	\
+				if (SPLAY_LEFT((head)->sph_root, field) == NULL)\
+					break;				\
+			}						\
+			SPLAY_LINKLEFT(head, __right, field);		\
+		} else if (__comp > 0) {				\
+			__tmp = SPLAY_RIGHT((head)->sph_root, field);	\
+			if (__tmp == NULL)				\
+				break;					\
+			if (__comp > 0) {				\
+				SPLAY_ROTATE_LEFT(head, __tmp, field);	\
+				if (SPLAY_RIGHT((head)->sph_root, field) == NULL)\
+					break;				\
+			}						\
+			SPLAY_LINKRIGHT(head, __left, field);		\
+		}							\
+	}								\
+	SPLAY_ASSEMBLE(head, &__node, __left, __right, field);		\
+}
+
+#define SPLAY_NEGINF	-1
+#define SPLAY_INF	1
+
+#define SPLAY_INSERT(name, x, y)	name##_SPLAY_INSERT(x, y)
+#define SPLAY_REMOVE(name, x, y)	name##_SPLAY_REMOVE(x, y)
+#define SPLAY_FIND(name, x, y)		name##_SPLAY_FIND(x, y)
+#define SPLAY_NEXT(name, x, y)		name##_SPLAY_NEXT(x, y)
+#define SPLAY_MIN(name, x)		(SPLAY_EMPTY(x) ? NULL	\
+					: name##_SPLAY_MIN_MAX(x, SPLAY_NEGINF))
+#define SPLAY_MAX(name, x)		(SPLAY_EMPTY(x) ? NULL	\
+					: name##_SPLAY_MIN_MAX(x, SPLAY_INF))
+
+#define SPLAY_FOREACH(x, name, head)					\
+	for ((x) = SPLAY_MIN(name, head);				\
+	     (x) != NULL;						\
+	     (x) = SPLAY_NEXT(name, head, x))
+
+/* Macros that define a rank-balanced tree */
+#define RB_HEAD(name, type)						\
+struct name {								\
+	struct type *rbh_root; /* root of the tree */			\
+}
+
+#define RB_INITIALIZER(root)						\
+	{ NULL }
+
+#define RB_INIT(root) do {						\
+	(root)->rbh_root = NULL;					\
+} while (/* CONSTCOND */ 0)
+
+#define RB_ENTRY(type)							\
+struct {								\
+	struct type *rbe_left;		/* left element */		\
+	struct type *rbe_right;		/* right element */		\
+	struct type *rbe_parent;	/* parent element */		\
+}
+
+#define RB_LEFT(elm, field)		(elm)->field.rbe_left
+#define RB_RIGHT(elm, field)		(elm)->field.rbe_right
+
+/*
+ * With the expectation that any object of struct type has an
+ * address that is a multiple of 4, and that therefore the
+ * 2 least significant bits of a pointer to struct type are
+ * always zero, this implementation sets those bits to indicate
+ * that the left or right child of the tree node is "red".
+ */
+#define RB_UP(elm, field)		(elm)->field.rbe_parent
+#define RB_BITS(elm, field)		(*(uintptr_t *)&RB_UP(elm, field))
+#define RB_RED_L			((uintptr_t)1)
+#define RB_RED_R			((uintptr_t)2)
+#define RB_RED_MASK			((uintptr_t)3)
+#define RB_FLIP_LEFT(elm, field)	(RB_BITS(elm, field) ^= RB_RED_L)
+#define RB_FLIP_RIGHT(elm, field)	(RB_BITS(elm, field) ^= RB_RED_R)
+#define RB_RED_LEFT(elm, field)		((RB_BITS(elm, field) & RB_RED_L) != 0)
+#define RB_RED_RIGHT(elm, field)	((RB_BITS(elm, field) & RB_RED_R) != 0)
+#define RB_PARENT(elm, field)		((__typeof(RB_UP(elm, field)))	\
+					 (RB_BITS(elm, field) & ~RB_RED_MASK))
+/*
+ * _RB_ROOT starts with an underscore. This is a workaround for the issue that
+ * RB_ROOT() had a name conflict with the SPDK FIO plugin. The SPDK FIO plugin
+ * includes FIO and FIO defines RB_ROOT() itself.
+ */
+#define _RB_ROOT(head)			(head)->rbh_root
+#define RB_EMPTY(head)			(_RB_ROOT(head) == NULL)
+
+#define RB_SET_PARENT(dst, src, field) do {				\
+	RB_BITS(dst, field) &= RB_RED_MASK;				\
+	RB_BITS(dst, field) |= (uintptr_t)src;			\
+} while (/* CONSTCOND */ 0)
+
+#define RB_SET(elm, parent, field) do {					\
+	RB_UP(elm, field) = parent;					\
+	RB_LEFT(elm, field) = RB_RIGHT(elm, field) = NULL;		\
+} while (/* CONSTCOND */ 0)
+
+#define RB_COLOR(elm, field)	(RB_PARENT(elm, field) == NULL ? 0 :	\
+				RB_LEFT(RB_PARENT(elm, field), field) == elm ? \
+				RB_RED_LEFT(RB_PARENT(elm, field), field) : \
+				RB_RED_RIGHT(RB_PARENT(elm, field), field))
+
+/*
+ * Something to be invoked in a loop at the root of every modified subtree,
+ * from the bottom up to the root, to update augmented node data.
+ */
+#ifndef RB_AUGMENT
+#define RB_AUGMENT(x)	break
+#endif
+
+#define RB_SWAP_CHILD(head, out, in, field) do {			\
+	if (RB_PARENT(out, field) == NULL)				\
+		_RB_ROOT(head) = (in);					\
+	else if ((out) == RB_LEFT(RB_PARENT(out, field), field))	\
+		RB_LEFT(RB_PARENT(out, field), field) = (in);		\
+	else								\
+		RB_RIGHT(RB_PARENT(out, field), field) = (in);		\
+} while (/* CONSTCOND */ 0)
+
+#define RB_ROTATE_LEFT(head, elm, tmp, field) do {			\
+	(tmp) = RB_RIGHT(elm, field);					\
+	if ((RB_RIGHT(elm, field) = RB_LEFT(tmp, field)) != NULL) {	\
+		RB_SET_PARENT(RB_RIGHT(elm, field), elm, field);	\
+	}								\
+	RB_SET_PARENT(tmp, RB_PARENT(elm, field), field);		\
+	RB_SWAP_CHILD(head, elm, tmp, field);				\
+	RB_LEFT(tmp, field) = (elm);					\
+	RB_SET_PARENT(elm, tmp, field);					\
+	RB_AUGMENT(elm);						\
+} while (/* CONSTCOND */ 0)
+
+#define RB_ROTATE_RIGHT(head, elm, tmp, field) do {			\
+	(tmp) = RB_LEFT(elm, field);					\
+	if ((RB_LEFT(elm, field) = RB_RIGHT(tmp, field)) != NULL) {	\
+		RB_SET_PARENT(RB_LEFT(elm, field), elm, field);		\
+	}								\
+	RB_SET_PARENT(tmp, RB_PARENT(elm, field), field);		\
+	RB_SWAP_CHILD(head, elm, tmp, field);				\
+	RB_RIGHT(tmp, field) = (elm);					\
+	RB_SET_PARENT(elm, tmp, field);					\
+	RB_AUGMENT(elm);						\
+} while (/* CONSTCOND */ 0)
+
+/* Generates prototypes and inline functions */
+#define	RB_PROTOTYPE(name, type, field, cmp)				\
+	RB_PROTOTYPE_INTERNAL(name, type, field, cmp,)
+#define	RB_PROTOTYPE_STATIC(name, type, field, cmp)			\
+	RB_PROTOTYPE_INTERNAL(name, type, field, cmp, __attribute__((unused)) static)
+#define RB_PROTOTYPE_INTERNAL(name, type, field, cmp, attr)		\
+	RB_PROTOTYPE_INSERT_COLOR(name, type, attr);			\
+	RB_PROTOTYPE_REMOVE_COLOR(name, type, attr);			\
+	RB_PROTOTYPE_INSERT(name, type, attr);				\
+	RB_PROTOTYPE_REMOVE(name, type, attr);				\
+	RB_PROTOTYPE_FIND(name, type, attr);				\
+	RB_PROTOTYPE_NFIND(name, type, attr);				\
+	RB_PROTOTYPE_NEXT(name, type, attr);				\
+	RB_PROTOTYPE_PREV(name, type, attr);				\
+	RB_PROTOTYPE_MINMAX(name, type, attr);				\
+	RB_PROTOTYPE_REINSERT(name, type, attr);
+#define RB_PROTOTYPE_INSERT_COLOR(name, type, attr)			\
+	attr void name##_RB_INSERT_COLOR(struct name *, struct type *)
+#define RB_PROTOTYPE_REMOVE_COLOR(name, type, attr)			\
+	attr void name##_RB_REMOVE_COLOR(struct name *,			\
+	    struct type *, struct type *)
+#define RB_PROTOTYPE_REMOVE(name, type, attr)				\
+	attr struct type *name##_RB_REMOVE(struct name *, struct type *)
+#define RB_PROTOTYPE_INSERT(name, type, attr)				\
+	attr struct type *name##_RB_INSERT(struct name *, struct type *)
+#define RB_PROTOTYPE_FIND(name, type, attr)				\
+	attr struct type *name##_RB_FIND(struct name *, struct type *)
+#define RB_PROTOTYPE_NFIND(name, type, attr)				\
+	attr struct type *name##_RB_NFIND(struct name *, struct type *)
+#define RB_PROTOTYPE_NEXT(name, type, attr)				\
+	attr struct type *name##_RB_NEXT(struct type *)
+#define RB_PROTOTYPE_PREV(name, type, attr)				\
+	attr struct type *name##_RB_PREV(struct type *)
+#define RB_PROTOTYPE_MINMAX(name, type, attr)				\
+	attr struct type *name##_RB_MINMAX(struct name *, int)
+#define RB_PROTOTYPE_REINSERT(name, type, attr)			\
+	attr struct type *name##_RB_REINSERT(struct name *, struct type *)
+
+/* Main rb operation.
+ * Moves node close to the key of elm to top
+ */
+#define	RB_GENERATE(name, type, field, cmp)				\
+	RB_GENERATE_INTERNAL(name, type, field, cmp,)
+#define	RB_GENERATE_STATIC(name, type, field, cmp)			\
+	RB_GENERATE_INTERNAL(name, type, field, cmp, __attribute__((unused)) static)
+#define RB_GENERATE_INTERNAL(name, type, field, cmp, attr)		\
+	RB_GENERATE_INSERT_COLOR(name, type, field, attr)		\
+	RB_GENERATE_REMOVE_COLOR(name, type, field, attr)		\
+	RB_GENERATE_INSERT(name, type, field, cmp, attr)		\
+	RB_GENERATE_REMOVE(name, type, field, attr)			\
+	RB_GENERATE_FIND(name, type, field, cmp, attr)			\
+	RB_GENERATE_NFIND(name, type, field, cmp, attr)			\
+	RB_GENERATE_NEXT(name, type, field, attr)			\
+	RB_GENERATE_PREV(name, type, field, attr)			\
+	RB_GENERATE_MINMAX(name, type, field, attr)			\
+	RB_GENERATE_REINSERT(name, type, field, cmp, attr)
+
+#define RB_GENERATE_INSERT_COLOR(name, type, field, attr)		\
+attr void								\
+name##_RB_INSERT_COLOR(struct name *head, struct type *elm)		\
+{									\
+	struct type *child, *parent;					\
+	while ((parent = RB_PARENT(elm, field)) != NULL) {		\
+		if (RB_LEFT(parent, field) == elm) {			\
+			if (RB_RED_LEFT(parent, field)) {		\
+				RB_FLIP_LEFT(parent, field);		\
+				return;					\
+			}						\
+			RB_FLIP_RIGHT(parent, field);			\
+			if (RB_RED_RIGHT(parent, field)) {		\
+				elm = parent;				\
+				continue;				\
+			}						\
+			if (!RB_RED_RIGHT(elm, field)) {		\
+				RB_FLIP_LEFT(elm, field);		\
+				RB_ROTATE_LEFT(head, elm, child, field);\
+				if (RB_RED_LEFT(child, field))		\
+					RB_FLIP_RIGHT(elm, field);	\
+				else if (RB_RED_RIGHT(child, field))	\
+					RB_FLIP_LEFT(parent, field);	\
+				elm = child;				\
+			}						\
+			RB_ROTATE_RIGHT(head, parent, elm, field);	\
+		} else {						\
+			if (RB_RED_RIGHT(parent, field)) {		\
+				RB_FLIP_RIGHT(parent, field);		\
+				return;					\
+			}						\
+			RB_FLIP_LEFT(parent, field);			\
+			if (RB_RED_LEFT(parent, field)) {		\
+				elm = parent;				\
+				continue;				\
+			}						\
+			if (!RB_RED_LEFT(elm, field)) {			\
+				RB_FLIP_RIGHT(elm, field);		\
+				RB_ROTATE_RIGHT(head, elm, child, field);\
+				if (RB_RED_RIGHT(child, field))		\
+					RB_FLIP_LEFT(elm, field);	\
+				else if (RB_RED_LEFT(child, field))	\
+					RB_FLIP_RIGHT(parent, field);	\
+				elm = child;				\
+			}						\
+			RB_ROTATE_LEFT(head, parent, elm, field);	\
+		}							\
+		RB_BITS(elm, field) &= ~RB_RED_MASK;			\
+		break;							\
+	}								\
+}
+
+#define RB_GENERATE_REMOVE_COLOR(name, type, field, attr)		\
+attr void								\
+name##_RB_REMOVE_COLOR(struct name *head,				\
+    struct type *parent, struct type *elm)				\
+{									\
+	struct type *sib;						\
+	if (RB_LEFT(parent, field) == elm &&				\
+	    RB_RIGHT(parent, field) == elm) {				\
+		RB_BITS(parent, field) &= ~RB_RED_MASK;			\
+		elm = parent;						\
+		parent = RB_PARENT(elm, field);				\
+		if (parent == NULL)					\
+			return;						\
+	}								\
+	do  {								\
+		if (RB_LEFT(parent, field) == elm) {			\
+			if (!RB_RED_LEFT(parent, field)) {		\
+				RB_FLIP_LEFT(parent, field);		\
+				return;					\
+			}						\
+			if (RB_RED_RIGHT(parent, field)) {		\
+				RB_FLIP_RIGHT(parent, field);		\
+				elm = parent;				\
+				continue;				\
+			}						\
+			sib = RB_RIGHT(parent, field);			\
+			if ((~RB_BITS(sib, field) & RB_RED_MASK) == 0) {\
+				RB_BITS(sib, field) &= ~RB_RED_MASK;	\
+				elm = parent;				\
+				continue;				\
+			}						\
+			RB_FLIP_RIGHT(sib, field);			\
+			if (RB_RED_LEFT(sib, field))			\
+				RB_FLIP_LEFT(parent, field);		\
+			else if (!RB_RED_RIGHT(sib, field)) {		\
+				RB_FLIP_LEFT(parent, field);		\
+				RB_ROTATE_RIGHT(head, sib, elm, field);	\
+				if (RB_RED_RIGHT(elm, field))		\
+					RB_FLIP_LEFT(sib, field);	\
+				if (RB_RED_LEFT(elm, field))		\
+					RB_FLIP_RIGHT(parent, field);	\
+				RB_BITS(elm, field) |= RB_RED_MASK;	\
+				sib = elm;				\
+			}						\
+			RB_ROTATE_LEFT(head, parent, sib, field);	\
+		} else {						\
+			if (!RB_RED_RIGHT(parent, field)) {		\
+				RB_FLIP_RIGHT(parent, field);		\
+				return;					\
+			}						\
+			if (RB_RED_LEFT(parent, field)) {		\
+				RB_FLIP_LEFT(parent, field);		\
+				elm = parent;				\
+				continue;				\
+			}						\
+			sib = RB_LEFT(parent, field);			\
+			if ((~RB_BITS(sib, field) & RB_RED_MASK) == 0) {\
+				RB_BITS(sib, field) &= ~RB_RED_MASK;	\
+				elm = parent;				\
+				continue;				\
+			}						\
+			RB_FLIP_LEFT(sib, field);			\
+			if (RB_RED_RIGHT(sib, field))			\
+				RB_FLIP_RIGHT(parent, field);		\
+			else if (!RB_RED_LEFT(sib, field)) {		\
+				RB_FLIP_RIGHT(parent, field);		\
+				RB_ROTATE_LEFT(head, sib, elm, field);	\
+				if (RB_RED_LEFT(elm, field))		\
+					RB_FLIP_RIGHT(sib, field);	\
+				if (RB_RED_RIGHT(elm, field))		\
+					RB_FLIP_LEFT(parent, field);	\
+				RB_BITS(elm, field) |= RB_RED_MASK;	\
+				sib = elm;				\
+			}						\
+			RB_ROTATE_RIGHT(head, parent, sib, field);	\
+		}							\
+		break;							\
+	} while ((parent = RB_PARENT(elm, field)) != NULL);		\
+}
+
+#define RB_GENERATE_REMOVE(name, type, field, attr)			\
+attr struct type *							\
+name##_RB_REMOVE(struct name *head, struct type *elm)			\
+{									\
+	struct type *child, *old, *parent, *right;			\
+									\
+	old = elm;							\
+	parent = RB_PARENT(elm, field);					\
+	right = RB_RIGHT(elm, field);					\
+	if (RB_LEFT(elm, field) == NULL)				\
+		elm = child = right;					\
+	else if (right == NULL)						\
+		elm = child = RB_LEFT(elm, field);			\
+	else {								\
+		if ((child = RB_LEFT(right, field)) == NULL) {		\
+			child = RB_RIGHT(right, field);			\
+			RB_RIGHT(old, field) = child;			\
+			parent = elm = right;				\
+		} else {						\
+			do						\
+				elm = child;				\
+			while ((child = RB_LEFT(elm, field)) != NULL);	\
+			child = RB_RIGHT(elm, field);			\
+			parent = RB_PARENT(elm, field);			\
+			RB_LEFT(parent, field) = child;			\
+			RB_SET_PARENT(RB_RIGHT(old, field), elm, field);\
+		}							\
+		RB_SET_PARENT(RB_LEFT(old, field), elm, field);		\
+		elm->field = old->field;				\
+	}								\
+	RB_SWAP_CHILD(head, old, elm, field);				\
+	if (child != NULL)						\
+		RB_SET_PARENT(child, parent, field);			\
+	if (parent != NULL)						\
+		name##_RB_REMOVE_COLOR(head, parent, child);		\
+	while (parent != NULL) {					\
+		RB_AUGMENT(parent);					\
+		parent = RB_PARENT(parent, field);			\
+	}								\
+	return (old);							\
+}
+
+#define RB_GENERATE_INSERT(name, type, field, cmp, attr)		\
+/* Inserts a node into the RB tree */					\
+attr struct type *							\
+name##_RB_INSERT(struct name *head, struct type *elm)			\
+{									\
+	struct type *tmp;						\
+	struct type *parent = NULL;					\
+	int comp = 0;							\
+	tmp = _RB_ROOT(head);						\
+	while (tmp) {							\
+		parent = tmp;						\
+		comp = (cmp)(elm, parent);				\
+		if (comp < 0)						\
+			tmp = RB_LEFT(tmp, field);			\
+		else if (comp > 0)					\
+			tmp = RB_RIGHT(tmp, field);			\
+		else							\
+			return (tmp);					\
+	}								\
+	RB_SET(elm, parent, field);					\
+	if (parent == NULL)						\
+		_RB_ROOT(head) = elm;					\
+	else if (comp < 0)						\
+		RB_LEFT(parent, field) = elm;				\
+	else								\
+		RB_RIGHT(parent, field) = elm;				\
+	name##_RB_INSERT_COLOR(head, elm);				\
+	while (elm != NULL) {						\
+		RB_AUGMENT(elm);					\
+		elm = RB_PARENT(elm, field);				\
+	}								\
+	return (NULL);							\
+}
+
+#define RB_GENERATE_FIND(name, type, field, cmp, attr)			\
+/* Finds the node with the same key as elm */				\
+attr struct type *							\
+name##_RB_FIND(struct name *head, struct type *elm)			\
+{									\
+	struct type *tmp = _RB_ROOT(head);				\
+	int comp;							\
+	while (tmp) {							\
+		comp = cmp(elm, tmp);					\
+		if (comp < 0)						\
+			tmp = RB_LEFT(tmp, field);			\
+		else if (comp > 0)					\
+			tmp = RB_RIGHT(tmp, field);			\
+		else							\
+			return (tmp);					\
+	}								\
+	return (NULL);							\
+}
+
+#define RB_GENERATE_NFIND(name, type, field, cmp, attr)			\
+/* Finds the first node greater than or equal to the search key */	\
+attr struct type *							\
+name##_RB_NFIND(struct name *head, struct type *elm)			\
+{									\
+	struct type *tmp = _RB_ROOT(head);				\
+	struct type *res = NULL;					\
+	int comp;							\
+	while (tmp) {							\
+		comp = cmp(elm, tmp);					\
+		if (comp < 0) {						\
+			res = tmp;					\
+			tmp = RB_LEFT(tmp, field);			\
+		}							\
+		else if (comp > 0)					\
+			tmp = RB_RIGHT(tmp, field);			\
+		else							\
+			return (tmp);					\
+	}								\
+	return (res);							\
+}
+
+#define RB_GENERATE_NEXT(name, type, field, attr)			\
+/* ARGSUSED */								\
+attr struct type *							\
+name##_RB_NEXT(struct type *elm)					\
+{									\
+	if (RB_RIGHT(elm, field)) {					\
+		elm = RB_RIGHT(elm, field);				\
+		while (RB_LEFT(elm, field))				\
+			elm = RB_LEFT(elm, field);			\
+	} else {							\
+		if (RB_PARENT(elm, field) &&				\
+		    (elm == RB_LEFT(RB_PARENT(elm, field), field)))	\
+			elm = RB_PARENT(elm, field);			\
+		else {							\
+			while (RB_PARENT(elm, field) &&			\
+			    (elm == RB_RIGHT(RB_PARENT(elm, field), field)))\
+				elm = RB_PARENT(elm, field);		\
+			elm = RB_PARENT(elm, field);			\
+		}							\
+	}								\
+	return (elm);							\
+}
+
+#define RB_GENERATE_PREV(name, type, field, attr)			\
+/* ARGSUSED */								\
+attr struct type *							\
+name##_RB_PREV(struct type *elm)					\
+{									\
+	if (RB_LEFT(elm, field)) {					\
+		elm = RB_LEFT(elm, field);				\
+		while (RB_RIGHT(elm, field))				\
+			elm = RB_RIGHT(elm, field);			\
+	} else {							\
+		if (RB_PARENT(elm, field) &&				\
+		    (elm == RB_RIGHT(RB_PARENT(elm, field), field)))	\
+			elm = RB_PARENT(elm, field);			\
+		else {							\
+			while (RB_PARENT(elm, field) &&			\
+			    (elm == RB_LEFT(RB_PARENT(elm, field), field)))\
+				elm = RB_PARENT(elm, field);		\
+			elm = RB_PARENT(elm, field);			\
+		}							\
+	}								\
+	return (elm);							\
+}
+
+#define RB_GENERATE_MINMAX(name, type, field, attr)			\
+attr struct type *							\
+name##_RB_MINMAX(struct name *head, int val)				\
+{									\
+	struct type *tmp = _RB_ROOT(head);				\
+	struct type *parent = NULL;					\
+	while (tmp) {							\
+		parent = tmp;						\
+		if (val < 0)						\
+			tmp = RB_LEFT(tmp, field);			\
+		else							\
+			tmp = RB_RIGHT(tmp, field);			\
+	}								\
+	return (parent);						\
+}
+
+#define	RB_GENERATE_REINSERT(name, type, field, cmp, attr)		\
+attr struct type *							\
+name##_RB_REINSERT(struct name *head, struct type *elm)			\
+{									\
+	struct type *cmpelm;						\
+	if (((cmpelm = RB_PREV(name, head, elm)) != NULL &&		\
+	    cmp(cmpelm, elm) >= 0) ||					\
+	    ((cmpelm = RB_NEXT(name, head, elm)) != NULL &&		\
+	    cmp(elm, cmpelm) >= 0)) {					\
+		/* XXXLAS: Remove/insert is heavy handed. */		\
+		RB_REMOVE(name, head, elm);				\
+		return (RB_INSERT(name, head, elm));			\
+	}								\
+	return (NULL);							\
+}									\
+
+#define RB_NEGINF	-1
+#define RB_INF	1
+
+#define RB_INSERT(name, x, y)	name##_RB_INSERT(x, y)
+#define RB_REMOVE(name, x, y)	name##_RB_REMOVE(x, y)
+#define RB_FIND(name, x, y)	name##_RB_FIND(x, y)
+#define RB_NFIND(name, x, y)	name##_RB_NFIND(x, y)
+#define RB_NEXT(name, x, y)	name##_RB_NEXT(y)
+#define RB_PREV(name, x, y)	name##_RB_PREV(y)
+#define RB_MIN(name, x)		name##_RB_MINMAX(x, RB_NEGINF)
+#define RB_MAX(name, x)		name##_RB_MINMAX(x, RB_INF)
+#define RB_REINSERT(name, x, y)	name##_RB_REINSERT(x, y)
+
+#define RB_FOREACH(x, name, head)					\
+	for ((x) = RB_MIN(name, head);					\
+	     (x) != NULL;						\
+	     (x) = name##_RB_NEXT(x))
+
+#define RB_FOREACH_FROM(x, name, y)					\
+	for ((x) = (y);							\
+	    ((x) != NULL) && ((y) = name##_RB_NEXT(x), (x) != NULL);	\
+	     (x) = (y))
+
+#define RB_FOREACH_SAFE(x, name, head, y)				\
+	for ((x) = RB_MIN(name, head);					\
+	    ((x) != NULL) && ((y) = name##_RB_NEXT(x), (x) != NULL);	\
+	     (x) = (y))
+
+#define RB_FOREACH_REVERSE(x, name, head)				\
+	for ((x) = RB_MAX(name, head);					\
+	     (x) != NULL;						\
+	     (x) = name##_RB_PREV(x))
+
+#define RB_FOREACH_REVERSE_FROM(x, name, y)				\
+	for ((x) = (y);							\
+	    ((x) != NULL) && ((y) = name##_RB_PREV(x), (x) != NULL);	\
+	     (x) = (y))
+
+#define RB_FOREACH_REVERSE_SAFE(x, name, head, y)			\
+	for ((x) = RB_MAX(name, head);					\
+	    ((x) != NULL) && ((y) = name##_RB_PREV(x), (x) != NULL);	\
+	     (x) = (y))
+
+#endif	/* SPDK_TREE_H */

include/spdk/util.h

@@ -53,6 +53,11 @@ extern "C" {
 
 #define SPDK_CONTAINEROF(ptr, type, member) ((type *)((uintptr_t)ptr - offsetof(type, member)))
 
+/**
+ * Get the size of a member of a struct.
+ */
+#define SPDK_SIZEOF_MEMBER(type, member) (sizeof(((type *)0)->member))
+
 #define SPDK_SEC_TO_USEC 1000000ULL
 #define SPDK_SEC_TO_NSEC 1000000000ULL
 
@@ -66,7 +71,7 @@ extern "C" {
  * power-of-two value.
  */
 #define SPDK_ALIGN_FLOOR(val, align) \
-	(typeof(val))((val) & (~((typeof(val))((align) - 1))))
+	(__typeof__(val))((val) & (~((__typeof__(val))((align) - 1))))
 /**
  * Macro to align a value to a given power-of-two. The resultant value
  * will be of the same type as the first parameter, and will be no lower
@@ -74,7 +79,7 @@ extern "C" {
  * value.
  */
 #define SPDK_ALIGN_CEIL(val, align) \
-	SPDK_ALIGN_FLOOR(((val) + ((typeof(val)) (align) - 1)), align)
+	SPDK_ALIGN_FLOOR(((val) + ((__typeof__(val)) (align) - 1)), align)
 
 uint32_t spdk_u32log2(uint32_t x);
 

include/spdk/version.h

@@ -46,7 +46,7 @@
 /**
  * Minor version number (month of original release).
  */
-#define SPDK_VERSION_MINOR	4
+#define SPDK_VERSION_MINOR	7
 
 /**
  * Patch level.
@@ -54,7 +54,7 @@
  * Patch level is incremented on maintenance branch releases and reset to 0 for each
  * new major.minor release.
  */
-#define SPDK_VERSION_PATCH	0
+#define SPDK_VERSION_PATCH	1
 
 /**
  * Version string suffix.

include/spdk/vfio_user_spec.h

@@ -36,29 +36,34 @@
 
 #include "spdk/stdinc.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 enum vfio_user_command {
 	VFIO_USER_VERSION			= 1,
 	VFIO_USER_DMA_MAP			= 2,
 	VFIO_USER_DMA_UNMAP			= 3,
 	VFIO_USER_DEVICE_GET_INFO		= 4,
 	VFIO_USER_DEVICE_GET_REGION_INFO	= 5,
-	VFIO_USER_DEVICE_GET_IRQ_INFO		= 6,
-	VFIO_USER_DEVICE_SET_IRQS		= 7,
-	VFIO_USER_REGION_READ			= 8,
-	VFIO_USER_REGION_WRITE			= 9,
-	VFIO_USER_DMA_READ			= 10,
-	VFIO_USER_DMA_WRITE			= 11,
-	VFIO_USER_VM_INTERRUPT			= 12,
+	VFIO_USER_DEVICE_GET_REGION_IO_FDS	= 6,
+	VFIO_USER_DEVICE_GET_IRQ_INFO		= 7,
+	VFIO_USER_DEVICE_SET_IRQS		= 8,
+	VFIO_USER_REGION_READ			= 9,
+	VFIO_USER_REGION_WRITE			= 10,
+	VFIO_USER_DMA_READ			= 11,
+	VFIO_USER_DMA_WRITE			= 12,
 	VFIO_USER_DEVICE_RESET			= 13,
+	VFIO_USER_DIRTY_PAGES			= 14,
 	VFIO_USER_MAX,
 };
 
 enum vfio_user_message_type {
-	VFIO_USER_MESSAGE_COMMAND		= 0,
-	VFIO_USER_MESSAGE_REPLY			= 1,
+	VFIO_USER_MESSAGE_COMMAND	= 0,
+	VFIO_USER_MESSAGE_REPLY		= 1,
 };
 
-#define VFIO_USER_FLAGS_NO_REPLY		(0x1)
+#define VFIO_USER_FLAGS_NO_REPLY	(0x1)
 
 struct vfio_user_header {
 	uint16_t	msg_id;
@@ -66,8 +71,8 @@ struct vfio_user_header {
 	uint32_t	msg_size;
 	struct {
 		uint32_t	type     : 4;
-#define VFIO_USER_F_TYPE_COMMAND    0
-#define VFIO_USER_F_TYPE_REPLY      1
+#define VFIO_USER_F_TYPE_COMMAND	0
+#define VFIO_USER_F_TYPE_REPLY		1
 		uint32_t	no_reply : 1;
 		uint32_t	error    : 1;
 		uint32_t	resvd    : 26;
@@ -81,15 +86,47 @@ struct vfio_user_version {
 	uint8_t		data[];
 } __attribute__((packed));
 
-struct vfio_user_dma_region {
+/*
+ * Similar to vfio_device_info, but without caps (yet).
+ */
+struct vfio_user_device_info {
+	uint32_t	argsz;
+	/* VFIO_DEVICE_FLAGS_* */
+	uint32_t	flags;
+	uint32_t	num_regions;
+	uint32_t	num_irqs;
+} __attribute__((packed));
+
+/* based on struct vfio_bitmap */
+struct vfio_user_bitmap {
+	uint64_t	pgsize;
+	uint64_t	size;
+	char		data[];
+} __attribute__((packed));
+
+/* based on struct vfio_iommu_type1_dma_map */
+struct vfio_user_dma_map {
+	uint32_t	argsz;
+#define VFIO_USER_F_DMA_REGION_READ	(1 << 0)
+#define VFIO_USER_F_DMA_REGION_WRITE	(1 << 1)
+	uint32_t	flags;
+	uint64_t	offset;
 	uint64_t	addr;
 	uint64_t	size;
-	uint64_t	offset;
-	uint32_t	prot;
-	uint32_t	flags;
-#define VFIO_USER_F_DMA_REGION_MAPPABLE (1 << 0)
 } __attribute__((packed));
 
+/* based on struct vfio_iommu_type1_dma_unmap */
+struct vfio_user_dma_unmap {
+	uint32_t	argsz;
+#ifndef VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP
+#define VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP	(1 << 0)
+#endif
+	uint32_t	flags;
+	uint64_t	addr;
+	uint64_t	size;
+	struct vfio_user_bitmap	bitmap[];
+};
+
 struct vfio_user_region_access {
 	uint64_t	offset;
 	uint32_t	region;
@@ -98,13 +135,25 @@ struct vfio_user_region_access {
 } __attribute__((packed));
 
 struct vfio_user_dma_region_access {
-	uint64_t    addr;
-	uint32_t    count;
-	uint8_t     data[];
+	uint64_t	addr;
+	uint64_t	count;
+	uint8_t		data[];
 } __attribute__((packed));
 
 struct vfio_user_irq_info {
-	uint32_t    subindex;
+	uint32_t	subindex;
 } __attribute__((packed));
 
+/* based on struct vfio_iommu_type1_dirty_bitmap_get */
+struct vfio_user_bitmap_range {
+	uint64_t	iova;
+	uint64_t	size;
+	struct vfio_user_bitmap	bitmap;
+} __attribute__((packed));
+
+
+#ifdef __cplusplus
+}
+#endif
+
 #endif

include/spdk/zipf.h

@@ -31,49 +31,53 @@
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#ifndef SPDK_NET_INTERNAL_H
-#define SPDK_NET_INTERNAL_H
+/** \file
+ * Zipf random number distribution
+ */
+
+#ifndef SPDK_ZIPF_H
+#define SPDK_ZIPF_H
 
 #include "spdk/stdinc.h"
 
-#include "spdk/queue.h"
+#ifdef __cplusplus
+extern "C" {
+#endif
 
-#define SPDK_IFNAMSIZE		32
-#define SPDK_MAX_IP_PER_IFC	32
-
-struct spdk_interface {
-	char name[SPDK_IFNAMSIZE];
-	uint32_t index;
-	uint32_t num_ip_addresses; /* number of IP addresses defined */
-	uint32_t ip_address[SPDK_MAX_IP_PER_IFC];
-	TAILQ_ENTRY(spdk_interface)	tailq;
-};
+struct spdk_zipf;
 
 /**
- * Add an ip address to the network interface.
+ * Create a zipf random number generator.
  *
- * \param ifc_index Index of the network interface.
- * \param ip_addr Ip address to add.
+ * Numbers from [0, range) will be returned by the generator when
+ * calling \ref spdk_zipf_generate.
  *
- * \return 0 on success, -1 on failure.
+ * \param range Range of values for the zipf distribution.
+ * \param theta Theta distribution parameter.
+ * \param seed Seed value for the random number generator.
+ *
+ * \return a pointer to the new zipf generator.
  */
-int interface_net_interface_add_ip_address(int ifc_index, char *ip_addr);
+struct spdk_zipf *spdk_zipf_create(uint64_t range, double theta, uint32_t seed);
 
 /**
- * Delete an ip address from the network interface.
+ * Free a zipf generator and set the pointer to NULL.
  *
- * \param ifc_index Index of the network interface.
- * \param ip_addr Ip address to delete.
- *
- * \return 0 on success, -1 on failure.
+ * \param zipfp Zipf generator to free.
  */
-int interface_net_interface_delete_ip_address(int ifc_index, char *ip_addr);
+void spdk_zipf_free(struct spdk_zipf **zipfp);
 
 /**
- * Get the list of all the network interfaces.
+ * Generate a value from the zipf generator.
  *
- * \return a pointer to the head of the linked list of all the network interfaces.
+ * \param zipf Zipf generator to generate the value from.
+ *
+ * \return value in the range [0, range)
  */
-void *interface_get_list(void);
+uint64_t spdk_zipf_generate(struct spdk_zipf *zipf);
 
-#endif /* SPDK_NET_INTERNAL_H */
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/spdk_internal/accel_engine.h

@@ -67,12 +67,13 @@ struct spdk_accel_batch {
 };
 
 enum accel_opcode {
-	ACCEL_OPCODE_MEMMOVE	= 0,
-	ACCEL_OPCODE_MEMFILL	= 1,
-	ACCEL_OPCODE_COMPARE	= 2,
-	ACCEL_OPCODE_BATCH	= 3,
-	ACCEL_OPCODE_CRC32C	= 4,
-	ACCEL_OPCODE_DUALCAST	= 5,
+	ACCEL_OPCODE_MEMMOVE		= 0,
+	ACCEL_OPCODE_MEMFILL		= 1,
+	ACCEL_OPCODE_COMPARE		= 2,
+	ACCEL_OPCODE_BATCH		= 3,
+	ACCEL_OPCODE_CRC32C		= 4,
+	ACCEL_OPCODE_DUALCAST		= 5,
+	ACCEL_OPCODE_COPY_CRC32C	= 6,
 };
 
 struct spdk_accel_task {
@@ -80,6 +81,10 @@ struct spdk_accel_task {
 	struct spdk_accel_batch		*batch;
 	spdk_accel_completion_cb	cb_fn;
 	void				*cb_arg;
+	struct {
+		spdk_accel_completion_cb	cb_fn;
+		void				*cb_arg;
+	} chained;
 	union {
 		struct {
 			struct iovec			*iovs; /* iovs passed by the caller */
@@ -92,14 +97,11 @@ struct spdk_accel_task {
 		void			*src2;
 	};
 	union {
-		struct {
-			spdk_accel_completion_cb	cb_fn;
-			void				*cb_arg;
-		} chained;
 		void				*dst2;
 		uint32_t			seed;
 		uint64_t			fill_pattern;
 	};
+	uint32_t			*crc_dst;
 	enum accel_opcode		op_code;
 	uint64_t			nbytes;
 	TAILQ_ENTRY(spdk_accel_task)	link;

include/spdk_internal/event.h

@@ -64,11 +64,11 @@ struct spdk_lw_thread {
 	TAILQ_ENTRY(spdk_lw_thread)	link;
 	uint64_t			tsc_start;
 	uint32_t                        lcore;
-	uint32_t                        new_lcore;
 	bool				resched;
+	/* stats over a lifetime of a thread */
+	struct spdk_thread_stats	total_stats;
+	/* stats during the last scheduling period */
 	struct spdk_thread_stats	current_stats;
-	struct spdk_thread_stats	snapshot_stats;
-	struct spdk_thread_stats	last_stats;
 };
 
 /**
@@ -88,8 +88,7 @@ struct spdk_reactor {
 
 	struct {
 		uint32_t				is_valid : 1;
-		uint32_t				is_scheduling : 1;
-		uint32_t				reserved : 30;
+		uint32_t				reserved : 31;
 	} flags;
 
 	uint64_t					tsc_last;
@@ -125,7 +124,7 @@ void spdk_reactors_stop(void *arg1);
 
 struct spdk_reactor *spdk_reactor_get(uint32_t lcore);
 
-struct spdk_reactor *_spdk_get_scheduling_reactor(void);
+extern bool g_scheduling_in_progress;
 
 /**
  * Allocate and pass an event to each reactor, serially.
@@ -164,68 +163,7 @@ int spdk_reactor_set_interrupt_mode(uint32_t lcore, bool new_in_interrupt,
  */
 struct spdk_thread *_spdk_get_app_thread(void);
 
-struct spdk_subsystem {
-	const char *name;
-	/* User must call spdk_subsystem_init_next() when they are done with their initialization. */
-	void (*init)(void);
-	void (*fini)(void);
-
-	/**
-	 * Write JSON configuration handler.
-	 *
-	 * \param w JSON write context
-	 */
-	void (*write_config_json)(struct spdk_json_write_ctx *w);
-	TAILQ_ENTRY(spdk_subsystem) tailq;
-};
-
-struct spdk_subsystem *spdk_subsystem_find(const char *name);
-struct spdk_subsystem *spdk_subsystem_get_first(void);
-struct spdk_subsystem *spdk_subsystem_get_next(struct spdk_subsystem *cur_subsystem);
-
-struct spdk_subsystem_depend {
-	const char *name;
-	const char *depends_on;
-	TAILQ_ENTRY(spdk_subsystem_depend) tailq;
-};
-
-struct spdk_subsystem_depend *spdk_subsystem_get_first_depend(void);
-struct spdk_subsystem_depend *spdk_subsystem_get_next_depend(struct spdk_subsystem_depend
-		*cur_depend);
-
-void spdk_add_subsystem(struct spdk_subsystem *subsystem);
-void spdk_add_subsystem_depend(struct spdk_subsystem_depend *depend);
-
-typedef void (*spdk_subsystem_init_fn)(int rc, void *ctx);
-void spdk_subsystem_init(spdk_subsystem_init_fn cb_fn, void *cb_arg);
-void spdk_subsystem_fini(spdk_msg_fn cb_fn, void *cb_arg);
-void spdk_subsystem_init_next(int rc);
-void spdk_subsystem_fini_next(void);
-void spdk_app_json_config_load(const char *json_config_file, const char *rpc_addr,
-			       spdk_subsystem_init_fn cb_fn, void *cb_arg,
-			       bool stop_on_error);
-
-/**
- * Save pointed \c subsystem configuration to the JSON write context \c w. In case of
- * error \c null is written to the JSON context.
- *
- * \param w JSON write context
- * \param subsystem the subsystem to query
- */
-void spdk_subsystem_config_json(struct spdk_json_write_ctx *w, struct spdk_subsystem *subsystem);
-
-int spdk_rpc_initialize(const char *listen_addr);
-void spdk_rpc_finish(void);
-
 struct spdk_governor_capabilities {
-	bool freq_change;
-	bool freq_getset;
-	bool freq_up;
-	bool freq_down;
-	bool freq_max;
-	bool freq_min;
-	bool turbo_set;
-	bool turbo_available;
 	bool priority;
 };
 
@@ -233,37 +171,14 @@ struct spdk_governor_capabilities {
 struct spdk_governor {
 	char *name;
 
-	/* freqs - the buffer array to save the frequencies; num - the number of frequencies to get; return - the number of available frequencies */
-	uint32_t (*get_core_freqs)(uint32_t lcore_id, uint32_t *freqs, uint32_t num);
-
 	/* return - current frequency on success, 0 on failure */
 	uint32_t (*get_core_curr_freq)(uint32_t lcore_id);
 
-	/**
-	 * freq_index - index of available frequencies returned from get_core_freqs call
-	 *
-	 * return
-	 *  - 1 on success with frequency changed.
-	 *  - 0 on success without frequency changed.
-	 *  - Negative on error.
-	 */
-	int (*set_core_freq)(uint32_t lcore_id, uint32_t freq_index);
 	int (*core_freq_up)(uint32_t lcore_id);
 	int (*core_freq_down)(uint32_t lcore_id);
 	int (*set_core_freq_max)(uint32_t lcore_id);
 	int (*set_core_freq_min)(uint32_t lcore_id);
 
-	/**
-	 * return
-	 *  - 1 Turbo Boost is enabled for this lcore.
-	 *  - 0 Turbo Boost is disabled for this lcore.
-	 *  - Negative on error.
-	 */
-	int (*get_core_turbo_status)(uint32_t lcore_id);
-
-	/* return - 0 on success; negative on error */
-	int (*enable_core_turbo)(uint32_t lcore_id);
-	int (*disable_core_turbo)(uint32_t lcore_id);
 	int (*get_core_capabilities)(uint32_t lcore_id, struct spdk_governor_capabilities *capabilities);
 	int (*init_core)(uint32_t lcore_id);
 	int (*deinit_core)(uint32_t lcore_id);
@@ -308,17 +223,33 @@ struct spdk_governor *_spdk_governor_get(void);
 		_spdk_governor_list_add(governor); \
 	} \
 
+/**
+ * Structure representing thread used for scheduling.
+ */
+struct spdk_scheduler_thread_info {
+	uint32_t lcore;
+	uint64_t thread_id;
+	/* stats over a lifetime of a thread */
+	struct spdk_thread_stats total_stats;
+	/* stats during the last scheduling period */
+	struct spdk_thread_stats current_stats;
+};
+
 /**
  * A list of cores and threads which is used for scheduling.
  */
 struct spdk_scheduler_core_info {
-	uint64_t core_idle_tsc;
-	uint64_t core_busy_tsc;
+	/* stats over a lifetime of a core */
+	uint64_t total_idle_tsc;
+	uint64_t total_busy_tsc;
+	/* stats during the last scheduling period */
+	uint64_t current_idle_tsc;
+	uint64_t current_busy_tsc;
+
 	uint32_t lcore;
 	uint32_t threads_count;
-	uint32_t pending_threads_count;
 	bool interrupt_mode;
-	struct spdk_lw_thread **threads;
+	struct spdk_scheduler_thread_info *thread_infos;
 };
 
 /**
@@ -380,11 +311,6 @@ struct spdk_scheduler *_spdk_scheduler_get(void);
  */
 void _spdk_scheduler_period_set(uint64_t period);
 
-/**
- * Disable the scheduler.
- */
-void _spdk_scheduler_disable(void);
-
 /**
  * Get period of currently set scheduler.
  */
@@ -408,37 +334,6 @@ static void __attribute__((constructor)) _spdk_scheduler_register_ ## scheduler
  */
 void _spdk_lw_thread_set_core(struct spdk_lw_thread *thread, uint32_t lcore);
 
-/**
- * Get threads stats
- *
- * \param thread thread that stats regards to.
- * \param stats Output parameter for accumulated TSC counts while the thread was busy.
- */
-void _spdk_lw_thread_get_current_stats(struct spdk_lw_thread *thread,
-				       struct spdk_thread_stats *stats);
-
-/**
- * \brief Register a new subsystem
- */
-#define SPDK_SUBSYSTEM_REGISTER(_name) \
-	__attribute__((constructor)) static void _name ## _register(void)	\
-	{									\
-		spdk_add_subsystem(&_name);					\
-	}
-
-/**
- * \brief Declare that a subsystem depends on another subsystem.
- */
-#define SPDK_SUBSYSTEM_DEPEND(_name, _depends_on)						\
-	static struct spdk_subsystem_depend __subsystem_ ## _name ## _depend_on ## _depends_on = { \
-	.name = #_name,										\
-	.depends_on = #_depends_on,								\
-	};											\
-	__attribute__((constructor)) static void _name ## _depend_on ## _depends_on(void)	\
-	{											\
-		spdk_add_subsystem_depend(&__subsystem_ ## _name ## _depend_on ## _depends_on); \
-	}
-
 #ifdef __cplusplus
 }
 #endif

include/spdk_internal/init.h

@@ -0,0 +1,90 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.  All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef SPDK_INIT_INTERNAL_H
+#define SPDK_INIT_INTERNAL_H
+
+#include "spdk/stdinc.h"
+#include "spdk/queue.h"
+
+struct spdk_json_write_ctx;
+
+struct spdk_subsystem {
+	const char *name;
+	/* User must call spdk_subsystem_init_next() when they are done with their initialization. */
+	void (*init)(void);
+	void (*fini)(void);
+
+	/**
+	 * Write JSON configuration handler.
+	 *
+	 * \param w JSON write context
+	 */
+	void (*write_config_json)(struct spdk_json_write_ctx *w);
+	TAILQ_ENTRY(spdk_subsystem) tailq;
+};
+
+struct spdk_subsystem_depend {
+	const char *name;
+	const char *depends_on;
+	TAILQ_ENTRY(spdk_subsystem_depend) tailq;
+};
+
+void spdk_add_subsystem(struct spdk_subsystem *subsystem);
+void spdk_add_subsystem_depend(struct spdk_subsystem_depend *depend);
+
+void spdk_subsystem_init_next(int rc);
+void spdk_subsystem_fini_next(void);
+
+/**
+ * \brief Register a new subsystem
+ */
+#define SPDK_SUBSYSTEM_REGISTER(_name) \
+	__attribute__((constructor)) static void _name ## _register(void)	\
+	{									\
+		spdk_add_subsystem(&_name);					\
+	}
+
+/**
+ * \brief Declare that a subsystem depends on another subsystem.
+ */
+#define SPDK_SUBSYSTEM_DEPEND(_name, _depends_on)						\
+	static struct spdk_subsystem_depend __subsystem_ ## _name ## _depend_on ## _depends_on = { \
+	.name = #_name,										\
+	.depends_on = #_depends_on,								\
+	};											\
+	__attribute__((constructor)) static void _name ## _depend_on ## _depends_on(void)	\
+	{											\
+		spdk_add_subsystem_depend(&__subsystem_ ## _name ## _depend_on ## _depends_on); \
+	}
+
+#endif

include/spdk_internal/lvolstore.h

@@ -36,8 +36,8 @@
 
 #include "spdk/blob.h"
 #include "spdk/lvol.h"
+#include "spdk/queue.h"
 #include "spdk/uuid.h"
-#include "spdk/bdev_module.h"
 
 /* Default size of blobstore cluster */
 #define SPDK_LVS_OPTS_CLUSTER_SZ (4 * 1024 * 1024)

include/spdk_internal/nvme_tcp.h

@@ -38,6 +38,8 @@
 #include "spdk/sock.h"
 #include "spdk/dif.h"
 
+#include "sgl.h"
+
 #define SPDK_CRC32C_XOR				0xffffffffUL
 #define SPDK_NVME_TCP_DIGEST_LEN		4
 #define SPDK_NVME_TCP_DIGEST_ALIGNMENT		4
@@ -76,13 +78,6 @@
 
 typedef void (*nvme_tcp_qpair_xfer_complete_cb)(void *cb_arg);
 
-struct _nvme_tcp_sgl {
-	struct iovec	*iov;
-	int		iovcnt;
-	uint32_t	iov_offset;
-	uint32_t	total_size;
-};
-
 struct nvme_tcp_pdu {
 	union {
 		/* to hold error pdu data */
@@ -101,7 +96,6 @@ struct nvme_tcp_pdu {
 
 	bool						has_hdgst;
 	bool						ddgst_enable;
-	uint32_t					header_digest_crc32;
 	uint32_t					data_digest_crc32;
 	uint8_t						data_digest[SPDK_NVME_TCP_DIGEST_LEN];
 
@@ -126,7 +120,7 @@ struct nvme_tcp_pdu {
 	TAILQ_ENTRY(nvme_tcp_pdu)			tailq;
 	uint32_t					remaining;
 	uint32_t					padding_len;
-	struct _nvme_tcp_sgl				sgl;
+	struct spdk_iov_sgl				sgl;
 
 	struct spdk_dif_ctx				*dif_ctx;
 
@@ -206,20 +200,6 @@ nvme_tcp_pdu_calc_header_digest(struct nvme_tcp_pdu *pdu)
 	return crc32c;
 }
 
-static uint32_t
-_update_crc32c_iov(struct iovec *iov, int iovcnt, uint32_t crc32c)
-{
-	int i;
-
-	for (i = 0; i < iovcnt; i++) {
-		assert(iov[i].iov_base != NULL);
-		assert(iov[i].iov_len != 0);
-		crc32c = spdk_crc32c_update(iov[i].iov_base, iov[i].iov_len, crc32c);
-	}
-
-	return crc32c;
-}
-
 static uint32_t
 nvme_tcp_pdu_calc_data_digest(struct nvme_tcp_pdu *pdu)
 {
@@ -229,7 +209,7 @@ nvme_tcp_pdu_calc_data_digest(struct nvme_tcp_pdu *pdu)
 	assert(pdu->data_len != 0);
 
 	if (spdk_likely(!pdu->dif_ctx)) {
-		crc32c = _update_crc32c_iov(pdu->data_iov, pdu->data_iovcnt, crc32c);
+		crc32c = spdk_crc32c_iov_update(pdu->data_iov, pdu->data_iovcnt, crc32c);
 	} else {
 		spdk_dif_update_crc32c_stream(pdu->data_iov, pdu->data_iovcnt,
 					      0, pdu->data_len, &crc32c, pdu->dif_ctx);
@@ -249,32 +229,7 @@ nvme_tcp_pdu_calc_data_digest(struct nvme_tcp_pdu *pdu)
 }
 
 static inline void
-_nvme_tcp_sgl_init(struct _nvme_tcp_sgl *s, struct iovec *iov, int iovcnt,
-		   uint32_t iov_offset)
-{
-	s->iov = iov;
-	s->iovcnt = iovcnt;
-	s->iov_offset = iov_offset;
-	s->total_size = 0;
-}
-
-static inline void
-_nvme_tcp_sgl_advance(struct _nvme_tcp_sgl *s, uint32_t step)
-{
-	s->iov_offset += step;
-	while (s->iovcnt > 0) {
-		if (s->iov_offset < s->iov->iov_len) {
-			break;
-		}
-
-		s->iov_offset -= s->iov->iov_len;
-		s->iov++;
-		s->iovcnt--;
-	}
-}
-
-static inline void
-_nvme_tcp_sgl_get_buf(struct _nvme_tcp_sgl *s, void **_buf, uint32_t *_buf_len)
+_nvme_tcp_sgl_get_buf(struct spdk_iov_sgl *s, void **_buf, uint32_t *_buf_len)
 {
 	if (_buf != NULL) {
 		*_buf = s->iov->iov_base + s->iov_offset;
@@ -285,33 +240,12 @@ _nvme_tcp_sgl_get_buf(struct _nvme_tcp_sgl *s, void **_buf, uint32_t *_buf_len)
 }
 
 static inline bool
-_nvme_tcp_sgl_append(struct _nvme_tcp_sgl *s, uint8_t *data, uint32_t data_len)
-{
-	if (s->iov_offset >= data_len) {
-		s->iov_offset -= data_len;
-	} else {
-		assert(s->iovcnt > 0);
-		s->iov->iov_base = data + s->iov_offset;
-		s->iov->iov_len = data_len - s->iov_offset;
-		s->total_size += data_len - s->iov_offset;
-		s->iov_offset = 0;
-		s->iov++;
-		s->iovcnt--;
-		if (s->iovcnt == 0) {
-			return false;
-		}
-	}
-
-	return true;
-}
-
-static inline bool
-_nvme_tcp_sgl_append_multi(struct _nvme_tcp_sgl *s, struct iovec *iov, int iovcnt)
+_nvme_tcp_sgl_append_multi(struct spdk_iov_sgl *s, struct iovec *iov, int iovcnt)
 {
 	int i;
 
 	for (i = 0; i < iovcnt; i++) {
-		if (!_nvme_tcp_sgl_append(s, iov[i].iov_base, iov[i].iov_len)) {
+		if (!spdk_iov_sgl_append(s, iov[i].iov_base, iov[i].iov_len)) {
 			return false;
 		}
 	}
@@ -333,7 +267,7 @@ _get_iov_array_size(struct iovec *iov, int iovcnt)
 }
 
 static inline bool
-_nvme_tcp_sgl_append_multi_with_md(struct _nvme_tcp_sgl *s, struct iovec *iov, int iovcnt,
+_nvme_tcp_sgl_append_multi_with_md(struct spdk_iov_sgl *s, struct iovec *iov, int iovcnt,
 				   uint32_t data_len, const struct spdk_dif_ctx *dif_ctx)
 {
 	int rc;
@@ -369,14 +303,14 @@ nvme_tcp_build_iovs(struct iovec *iov, int iovcnt, struct nvme_tcp_pdu *pdu,
 		    bool hdgst_enable, bool ddgst_enable, uint32_t *_mapped_length)
 {
 	uint32_t hlen, plen;
-	struct _nvme_tcp_sgl *sgl;
+	struct spdk_iov_sgl *sgl;
 
 	if (iovcnt == 0) {
 		return 0;
 	}
 
 	sgl = &pdu->sgl;
-	_nvme_tcp_sgl_init(sgl, iov, iovcnt, 0);
+	spdk_iov_sgl_init(sgl, iov, iovcnt, 0);
 	hlen = pdu->hdr.common.hlen;
 
 	/* Header Digest */
@@ -387,7 +321,7 @@ nvme_tcp_build_iovs(struct iovec *iov, int iovcnt, struct nvme_tcp_pdu *pdu,
 	plen = hlen;
 	if (!pdu->data_len) {
 		/* PDU header + possible header digest */
-		_nvme_tcp_sgl_append(sgl, (uint8_t *)&pdu->hdr.raw, hlen);
+		spdk_iov_sgl_append(sgl, (uint8_t *)&pdu->hdr.raw, hlen);
 		goto end;
 	}
 
@@ -397,7 +331,7 @@ nvme_tcp_build_iovs(struct iovec *iov, int iovcnt, struct nvme_tcp_pdu *pdu,
 		plen = hlen;
 	}
 
-	if (!_nvme_tcp_sgl_append(sgl, (uint8_t *)&pdu->hdr.raw, hlen)) {
+	if (!spdk_iov_sgl_append(sgl, (uint8_t *)&pdu->hdr.raw, hlen)) {
 		goto end;
 	}
 
@@ -417,7 +351,7 @@ nvme_tcp_build_iovs(struct iovec *iov, int iovcnt, struct nvme_tcp_pdu *pdu,
 	/* Data Digest */
 	if (g_nvme_tcp_ddgst[pdu->hdr.common.pdu_type] && ddgst_enable) {
 		plen += SPDK_NVME_TCP_DIGEST_LEN;
-		_nvme_tcp_sgl_append(sgl, pdu->data_digest, SPDK_NVME_TCP_DIGEST_LEN);
+		spdk_iov_sgl_append(sgl, pdu->data_digest, SPDK_NVME_TCP_DIGEST_LEN);
 	}
 
 	assert(plen == pdu->hdr.common.plen);
@@ -434,14 +368,14 @@ static int
 nvme_tcp_build_payload_iovs(struct iovec *iov, int iovcnt, struct nvme_tcp_pdu *pdu,
 			    bool ddgst_enable, uint32_t *_mapped_length)
 {
-	struct _nvme_tcp_sgl *sgl;
+	struct spdk_iov_sgl *sgl;
 
 	if (iovcnt == 0) {
 		return 0;
 	}
 
 	sgl = &pdu->sgl;
-	_nvme_tcp_sgl_init(sgl, iov, iovcnt, pdu->rw_offset);
+	spdk_iov_sgl_init(sgl, iov, iovcnt, pdu->rw_offset);
 
 	if (spdk_likely(!pdu->dif_ctx)) {
 		if (!_nvme_tcp_sgl_append_multi(sgl, pdu->data_iov, pdu->data_iovcnt)) {
@@ -456,7 +390,7 @@ nvme_tcp_build_payload_iovs(struct iovec *iov, int iovcnt, struct nvme_tcp_pdu *
 
 	/* Data Digest */
 	if (ddgst_enable) {
-		_nvme_tcp_sgl_append(sgl, pdu->data_digest, SPDK_NVME_TCP_DIGEST_LEN);
+		spdk_iov_sgl_append(sgl, pdu->data_digest, SPDK_NVME_TCP_DIGEST_LEN);
 	}
 
 end:
@@ -566,7 +500,7 @@ nvme_tcp_pdu_set_data_buf(struct nvme_tcp_pdu *pdu,
 {
 	uint32_t buf_offset, buf_len, remain_len, len;
 	uint8_t *buf;
-	struct _nvme_tcp_sgl *pdu_sgl, buf_sgl;
+	struct spdk_iov_sgl *pdu_sgl, buf_sgl;
 
 	pdu->data_len = data_len;
 
@@ -584,20 +518,20 @@ nvme_tcp_pdu_set_data_buf(struct nvme_tcp_pdu *pdu,
 	} else {
 		pdu_sgl = &pdu->sgl;
 
-		_nvme_tcp_sgl_init(pdu_sgl, pdu->data_iov, NVME_TCP_MAX_SGL_DESCRIPTORS, 0);
-		_nvme_tcp_sgl_init(&buf_sgl, iov, iovcnt, 0);
+		spdk_iov_sgl_init(pdu_sgl, pdu->data_iov, NVME_TCP_MAX_SGL_DESCRIPTORS, 0);
+		spdk_iov_sgl_init(&buf_sgl, iov, iovcnt, 0);
 
-		_nvme_tcp_sgl_advance(&buf_sgl, buf_offset);
+		spdk_iov_sgl_advance(&buf_sgl, buf_offset);
 		remain_len = buf_len;
 
 		while (remain_len > 0) {
 			_nvme_tcp_sgl_get_buf(&buf_sgl, (void *)&buf, &len);
 			len = spdk_min(len, remain_len);
 
-			_nvme_tcp_sgl_advance(&buf_sgl, len);
+			spdk_iov_sgl_advance(&buf_sgl, len);
 			remain_len -= len;
 
-			if (!_nvme_tcp_sgl_append(pdu_sgl, buf, len)) {
+			if (!spdk_iov_sgl_append(pdu_sgl, buf, len)) {
 				break;
 			}
 		}

include/spdk_internal/sgl.h

@@ -0,0 +1,127 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __SGL_INTERNAL_H__
+#define __SGL_INTERNAL_H__
+
+#include "spdk/stdinc.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct spdk_iov_sgl {
+	struct iovec    *iov;
+	int             iovcnt;
+	uint32_t        iov_offset;
+	uint32_t        total_size;
+};
+
+/**
+ * Initialize struct spdk_iov_sgl with iov, iovcnt and iov_offset.
+ *
+ * \param s the spdk_iov_sgl to be filled.
+ * \param iov the io vector to fill the s
+ * \param iovcnt the size the iov
+ * \param iov_offset the current filled iov_offset for s.
+ */
+
+static inline void
+spdk_iov_sgl_init(struct spdk_iov_sgl *s, struct iovec *iov, int iovcnt,
+		  uint32_t iov_offset)
+{
+	s->iov = iov;
+	s->iovcnt = iovcnt;
+	s->iov_offset = iov_offset;
+	s->total_size = 0;
+}
+
+/**
+ * Consume the iovs in spdk_iov_sgl with passed bytes
+ *
+ * \param s the spdk_iov_sgl which contains the iov
+ * \param step the bytes_size consumed.
+ */
+
+static inline void
+spdk_iov_sgl_advance(struct spdk_iov_sgl *s, uint32_t step)
+{
+	s->iov_offset += step;
+	while (s->iovcnt > 0) {
+		assert(s->iov != NULL);
+		if (s->iov_offset < s->iov->iov_len) {
+			break;
+		}
+
+		s->iov_offset -= s->iov->iov_len;
+		s->iov++;
+		s->iovcnt--;
+	}
+}
+
+/**
+ * Append the data to the struct spdk_iov_sgl pointed by s
+ *
+ * \param s the address of the struct spdk_iov_sgl
+ * \param data the data buffer to be appended
+ * \param data_len the length of the data.
+ *
+ * \return true if all the data is appended.
+ */
+
+static inline bool
+spdk_iov_sgl_append(struct spdk_iov_sgl *s, uint8_t *data, uint32_t data_len)
+{
+	if (s->iov_offset >= data_len) {
+		s->iov_offset -= data_len;
+	} else {
+		assert(s->iovcnt > 0);
+		s->iov->iov_base = data + s->iov_offset;
+		s->iov->iov_len = data_len - s->iov_offset;
+		s->total_size += data_len - s->iov_offset;
+		s->iov_offset = 0;
+		s->iov++;
+		s->iovcnt--;
+		if (s->iovcnt == 0) {
+			return false;
+		}
+	}
+
+	return true;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __SGL_INTERNAL_H__ */

include/spdk_internal/thread.h

@@ -31,115 +31,40 @@
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#ifndef SPDK_THREAD_INTERNAL_H_
-#define SPDK_THREAD_INTERNAL_H_
+#ifndef SPDK_INTERNAL_THREAD_H_
+#define SPDK_INTERNAL_THREAD_H_
 
 #include "spdk/stdinc.h"
 #include "spdk/thread.h"
 
-#define SPDK_MAX_POLLER_NAME_LEN	256
-#define SPDK_MAX_THREAD_NAME_LEN	256
+struct spdk_poller;
 
-enum spdk_poller_state {
-	/* The poller is registered with a thread but not currently executing its fn. */
-	SPDK_POLLER_STATE_WAITING,
-
-	/* The poller is currently running its fn. */
-	SPDK_POLLER_STATE_RUNNING,
-
-	/* The poller was unregistered during the execution of its fn. */
-	SPDK_POLLER_STATE_UNREGISTERED,
-
-	/* The poller is in the process of being paused.  It will be paused
-	 * during the next time it's supposed to be executed.
-	 */
-	SPDK_POLLER_STATE_PAUSING,
-
-	/* The poller is registered but currently paused.  It's on the
-	 * paused_pollers list.
-	 */
-	SPDK_POLLER_STATE_PAUSED,
+struct spdk_poller_stats {
+	uint64_t	run_count;
+	uint64_t	busy_count;
 };
 
-struct spdk_poller {
-	TAILQ_ENTRY(spdk_poller)	tailq;
+struct io_device;
+struct spdk_thread;
 
-	/* Current state of the poller; should only be accessed from the poller's thread. */
-	enum spdk_poller_state		state;
+const char *spdk_poller_get_name(struct spdk_poller *poller);
+const char *spdk_poller_get_state_str(struct spdk_poller *poller);
+uint64_t spdk_poller_get_period_ticks(struct spdk_poller *poller);
+void spdk_poller_get_stats(struct spdk_poller *poller, struct spdk_poller_stats *stats);
 
-	uint64_t			period_ticks;
-	uint64_t			next_run_tick;
-	uint64_t			run_count;
-	uint64_t			busy_count;
-	spdk_poller_fn			fn;
-	void				*arg;
-	struct spdk_thread		*thread;
-	int				interruptfd;
-	spdk_poller_set_interrupt_mode_cb set_intr_cb_fn;
-	void				*set_intr_cb_arg;
-
-	char				name[SPDK_MAX_POLLER_NAME_LEN + 1];
-};
-
-enum spdk_thread_state {
-	/* The thread is pocessing poller and message by spdk_thread_poll(). */
-	SPDK_THREAD_STATE_RUNNING,
-
-	/* The thread is in the process of termination. It reaps unregistering
-	 * poller are releasing I/O channel.
-	 */
-	SPDK_THREAD_STATE_EXITING,
-
-	/* The thread is exited. It is ready to call spdk_thread_destroy(). */
-	SPDK_THREAD_STATE_EXITED,
-};
-
-struct spdk_thread {
-	uint64_t			tsc_last;
-	struct spdk_thread_stats	stats;
-	/*
-	 * Contains pollers actively running on this thread.  Pollers
-	 *  are run round-robin. The thread takes one poller from the head
-	 *  of the ring, executes it, then puts it back at the tail of
-	 *  the ring.
-	 */
-	TAILQ_HEAD(active_pollers_head, spdk_poller)	active_pollers;
-	/**
-	 * Contains pollers running on this thread with a periodic timer.
-	 */
-	TAILQ_HEAD(timed_pollers_head, spdk_poller)	timed_pollers;
-	/*
-	 * Contains paused pollers.  Pollers on this queue are waiting until
-	 * they are resumed (in which case they're put onto the active/timer
-	 * queues) or unregistered.
-	 */
-	TAILQ_HEAD(paused_pollers_head, spdk_poller)	paused_pollers;
-	struct spdk_ring		*messages;
-	int				msg_fd;
-	SLIST_HEAD(, spdk_msg)		msg_cache;
-	size_t				msg_cache_count;
-	spdk_msg_fn			critical_msg;
-	uint64_t			id;
-	enum spdk_thread_state		state;
-	int				pending_unregister_count;
-
-	TAILQ_HEAD(, spdk_io_channel)	io_channels;
-	TAILQ_ENTRY(spdk_thread)	tailq;
-
-	char				name[SPDK_MAX_THREAD_NAME_LEN + 1];
-	struct spdk_cpuset		cpumask;
-	uint64_t			exit_timeout_tsc;
-
-	/* Indicates whether this spdk_thread currently runs in interrupt. */
-	bool				in_interrupt;
-	struct spdk_fd_group		*fgrp;
-
-	/* User context allocated at the end */
-	uint8_t				ctx[0];
-};
-
-const char *spdk_poller_state_str(enum spdk_poller_state state);
+const char *spdk_io_channel_get_io_device_name(struct spdk_io_channel *ch);
+int spdk_io_channel_get_ref_count(struct spdk_io_channel *ch);
 
 const char *spdk_io_device_get_name(struct io_device *dev);
 
-#endif /* SPDK_THREAD_INTERNAL_H_ */
+struct spdk_poller *spdk_thread_get_first_active_poller(struct spdk_thread *thread);
+struct spdk_poller *spdk_thread_get_next_active_poller(struct spdk_poller *prev);
+struct spdk_poller *spdk_thread_get_first_timed_poller(struct spdk_thread *thread);
+struct spdk_poller *spdk_thread_get_next_timed_poller(struct spdk_poller *prev);
+struct spdk_poller *spdk_thread_get_first_paused_poller(struct spdk_thread *thread);
+struct spdk_poller *spdk_thread_get_next_paused_poller(struct spdk_poller *prev);
+
+struct spdk_io_channel *spdk_thread_get_first_io_channel(struct spdk_thread *thread);
+struct spdk_io_channel *spdk_thread_get_next_io_channel(struct spdk_io_channel *prev);
+
+#endif /* SPDK_INTERNAL_THREAD_H_ */

include/spdk_internal/usdt.h

@@ -31,33 +31,30 @@
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include "spdk/stdinc.h"
+#ifndef SPDK_INTERNAL_USDT_H
+#define SPDK_INTERNAL_USDT_H
 
-#include "spdk/net.h"
+#include "spdk/config.h"
+#include "spdk/env.h"
 
-#include "spdk_internal/event.h"
+#if defined(SPDK_CONFIG_USDT) && !defined(SPDK_UNIT_TEST)
 
-static void
-interface_subsystem_init(void)
-{
-	int rc;
+#include <sys/sdt.h>
 
-	rc = spdk_interface_init();
+#define SPDK_DTRACE_PROBE(name)			DTRACE_PROBE1(spdk,name,spdk_get_ticks())
+#define SPDK_DTRACE_PROBE1(name,a1)		DTRACE_PROBE2(spdk,name,spdk_get_ticks(),a1)
+#define SPDK_DTRACE_PROBE2(name,a1,a2)		DTRACE_PROBE3(spdk,name,spdk_get_ticks(),a1,a2)
+#define SPDK_DTRACE_PROBE3(name,a1,a2,a3)	DTRACE_PROBE4(spdk,name,spdk_get_ticks(),a1,a2,a3)
+#define SPDK_DTRACE_PROBE4(name,a1,a2,a3,a4)	DTRACE_PROBE5(spdk,name,spdk_get_ticks(),a1,a2,a3,a4)
 
-	spdk_subsystem_init_next(rc);
-}
+#else
 
-static void
-interface_subsystem_destroy(void)
-{
-	spdk_interface_destroy();
-	spdk_subsystem_fini_next();
-}
+#define SPDK_DTRACE_PROBE(...)
+#define SPDK_DTRACE_PROBE1(...)
+#define SPDK_DTRACE_PROBE2(...)
+#define SPDK_DTRACE_PROBE3(...)
+#define SPDK_DTRACE_PROBE4(...)
 
-static struct spdk_subsystem g_spdk_subsystem_interface = {
-	.name = "interface",
-	.init = interface_subsystem_init,
-	.fini = interface_subsystem_destroy,
-};
+#endif
 
-SPDK_SUBSYSTEM_REGISTER(g_spdk_subsystem_interface);
+#endif /* SPDK_INTERNAL_USDT_H */

lib/Makefile

@@ -36,8 +36,8 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 include $(SPDK_ROOT_DIR)/mk/spdk.lib_deps.mk
 
 DIRS-y += bdev blob blobfs conf accel event json jsonrpc \
-          log lvol net rpc sock thread trace util nvme vmd nvmf scsi \
-          ioat ut_mock iscsi notify
+          log lvol rpc sock thread trace util nvme vmd nvmf scsi \
+          ioat ut_mock iscsi notify init
 ifeq ($(OS),Linux)
 DIRS-y += nbd ftl
 endif

lib/accel/Makefile

@@ -34,7 +34,7 @@
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-SO_VER := 5
+SO_VER := 6
 SO_MINOR := 0
 SO_SUFFIX := $(SO_VER).$(SO_MINOR)
 

lib/accel/accel_engine.c

@@ -301,7 +301,7 @@ spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, uint8_t fill, uint
 
 /* Accel framework public API for CRC-32C function */
 int
-spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *dst, void *src, uint32_t seed,
+spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst, void *src, uint32_t seed,
 			 uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
 {
 	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
@@ -312,7 +312,7 @@ spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *dst, void *src, u
 		return -ENOMEM;
 	}
 
-	accel_task->dst = (void *)dst;
+	accel_task->crc_dst = crc_dst;
 	accel_task->src = src;
 	accel_task->v.iovcnt = 0;
 	accel_task->seed = seed;
@@ -322,7 +322,7 @@ spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *dst, void *src, u
 	if (_is_supported(accel_ch->engine, ACCEL_CRC32C)) {
 		return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);
 	} else {
-		_sw_accel_crc32c(dst, src, seed, nbytes);
+		_sw_accel_crc32c(crc_dst, src, seed, nbytes);
 		spdk_accel_task_complete(accel_task, 0);
 		return 0;
 	}
@@ -338,9 +338,18 @@ crc32cv_done(void *cb_arg, int status)
 	assert(accel_task->chained.cb_arg != NULL);
 
 	if (spdk_likely(!status)) {
-		status = spdk_accel_submit_crc32cv(ch, accel_task->dst, ++accel_task->v.iovs,
-						   accel_task->v.iovcnt - 1, ~(*((uint32_t *)accel_task->dst)),
-						   accel_task->chained.cb_fn, accel_task->chained.cb_arg);
+		if (accel_task->op_code == ACCEL_OPCODE_COPY_CRC32C) {
+			accel_task->dst = (char *)accel_task->dst + accel_task->nbytes;
+			status = spdk_accel_submit_copy_crc32cv(ch, accel_task->dst, ++accel_task->v.iovs,
+								accel_task->v.iovcnt - 1, accel_task->crc_dst,
+								~(*((uint32_t *)accel_task->crc_dst)),
+								accel_task->chained.cb_fn, accel_task->chained.cb_arg);
+		} else {
+			status = spdk_accel_submit_crc32cv(ch, accel_task->crc_dst, ++accel_task->v.iovs,
+							   accel_task->v.iovcnt - 1, ~(*((uint32_t *)accel_task->crc_dst)),
+							   accel_task->chained.cb_fn, accel_task->chained.cb_arg);
+		}
+
 		if (spdk_likely(!status)) {
 			return;
 		}
@@ -351,7 +360,7 @@ crc32cv_done(void *cb_arg, int status)
 
 /* Accel framework public API for chained CRC-32C function */
 int
-spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *dst, struct iovec *iov,
+spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst, struct iovec *iov,
 			  uint32_t iov_cnt, uint32_t seed, spdk_accel_completion_cb cb_fn, void *cb_arg)
 {
 	struct accel_io_channel *accel_ch;
@@ -368,7 +377,7 @@ spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *dst, struct iove
 	}
 
 	if (iov_cnt == 1) {
-		return spdk_accel_submit_crc32c(ch, dst, iov[0].iov_base, seed, iov[0].iov_len, cb_fn, cb_arg);
+		return spdk_accel_submit_crc32c(ch, crc_dst, iov[0].iov_base, seed, iov[0].iov_len, cb_fn, cb_arg);
 	}
 
 	accel_ch = spdk_io_channel_get_ctx(ch);
@@ -381,7 +390,8 @@ spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *dst, struct iove
 
 	accel_task->v.iovs = iov;
 	accel_task->v.iovcnt = iov_cnt;
-	accel_task->dst = (void *)dst;
+	accel_task->crc_dst = crc_dst;
+	accel_task->seed = seed;
 	accel_task->op_code = ACCEL_OPCODE_CRC32C;
 
 	if (_is_supported(accel_ch->engine, ACCEL_CRC32C)) {
@@ -394,12 +404,101 @@ spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *dst, struct iove
 
 		return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);
 	} else {
-		_sw_accel_crc32cv(dst, iov, iov_cnt, seed);
+		_sw_accel_crc32cv(crc_dst, iov, iov_cnt, seed);
 		spdk_accel_task_complete(accel_task, 0);
 		return 0;
 	}
 }
 
+/* Accel framework public API for copy with CRC-32C function */
+int
+spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst, void *src,
+			      uint32_t *crc_dst, uint32_t seed, uint64_t nbytes,
+			      spdk_accel_completion_cb cb_fn, void *cb_arg)
+{
+	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
+	struct spdk_accel_task *accel_task;
+
+	accel_task = _get_task(accel_ch, NULL, cb_fn, cb_arg);
+	if (accel_task == NULL) {
+		return -ENOMEM;
+	}
+
+	accel_task->dst = dst;
+	accel_task->src = src;
+	accel_task->crc_dst = crc_dst;
+	accel_task->v.iovcnt = 0;
+	accel_task->seed = seed;
+	accel_task->nbytes = nbytes;
+	accel_task->op_code = ACCEL_OPCODE_COPY_CRC32C;
+
+	if (_is_supported(accel_ch->engine, ACCEL_COPY_CRC32C)) {
+		return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);
+	} else {
+		_sw_accel_copy(dst, src, nbytes);
+		_sw_accel_crc32c(crc_dst, src, seed, nbytes);
+		spdk_accel_task_complete(accel_task, 0);
+		return 0;
+	}
+}
+
+/* Accel framework public API for chained copy + CRC-32C function */
+int
+spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, struct iovec *src_iovs,
+			       uint32_t iov_cnt, uint32_t *crc_dst, uint32_t seed,
+			       spdk_accel_completion_cb cb_fn, void *cb_arg)
+{
+	struct accel_io_channel *accel_ch;
+	struct spdk_accel_task *accel_task;
+
+	if (src_iovs == NULL) {
+		SPDK_ERRLOG("iov should not be NULL");
+		return -EINVAL;
+	}
+
+	if (!iov_cnt) {
+		SPDK_ERRLOG("iovcnt should not be zero value\n");
+		return -EINVAL;
+	}
+
+	if (iov_cnt == 1) {
+		return spdk_accel_submit_copy_crc32c(ch, dst, src_iovs[0].iov_base, crc_dst, seed,
+						     src_iovs[0].iov_len, cb_fn, cb_arg);
+	}
+
+	accel_ch = spdk_io_channel_get_ctx(ch);
+	accel_task = _get_task(accel_ch, NULL, cb_fn, cb_arg);
+	if (accel_task == NULL) {
+		SPDK_ERRLOG("no memory\n");
+		assert(0);
+		return -ENOMEM;
+	}
+
+	accel_task->v.iovs = src_iovs;
+	accel_task->v.iovcnt = iov_cnt;
+	accel_task->dst = (void *)dst;
+	accel_task->crc_dst = crc_dst;
+	accel_task->seed = seed;
+	accel_task->op_code = ACCEL_OPCODE_COPY_CRC32C;
+
+	if (_is_supported(accel_ch->engine, ACCEL_COPY_CRC32C)) {
+		accel_task->cb_fn = crc32cv_done;
+		accel_task->cb_arg = accel_task;
+		accel_task->chained.cb_fn = cb_fn;
+		accel_task->chained.cb_arg = cb_arg;
+
+		accel_task->nbytes = src_iovs[0].iov_len;
+
+		return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);
+	} else {
+		_sw_accel_copy(dst, src_iovs[0].iov_base, src_iovs[0].iov_len);
+		_sw_accel_crc32cv(crc_dst, src_iovs, iov_cnt, seed);
+		spdk_accel_task_complete(accel_task, 0);
+		return 0;
+	}
+}
+
+
 /* Accel framework public API for getting max operations for a batch. */
 uint32_t
 spdk_accel_batch_get_max(struct spdk_io_channel *ch)
@@ -525,7 +624,7 @@ spdk_accel_batch_prep_fill(struct spdk_io_channel *ch, struct spdk_accel_batch *
 
 int
 spdk_accel_batch_prep_crc32c(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
-			     uint32_t *dst, void *src, uint32_t seed, uint64_t nbytes,
+			     uint32_t *crc_dst, void *src, uint32_t seed, uint64_t nbytes,
 			     spdk_accel_completion_cb cb_fn, void *cb_arg)
 {
 	struct spdk_accel_task *accel_task;
@@ -536,7 +635,7 @@ spdk_accel_batch_prep_crc32c(struct spdk_io_channel *ch, struct spdk_accel_batch
 		return -ENOMEM;
 	}
 
-	accel_task->dst = dst;
+	accel_task->crc_dst = crc_dst;
 	accel_task->src = src;
 	accel_task->v.iovcnt = 0;
 	accel_task->seed = seed;
@@ -565,8 +664,9 @@ batched_crc32cv_done(void *cb_arg, int status)
 	assert(accel_task->chained.cb_arg != NULL);
 
 	if (spdk_likely(!status)) {
-		status = spdk_accel_batch_prep_crc32cv(ch, batch, accel_task->dst,
-						       ++accel_task->v.iovs, accel_task->v.iovcnt - 1,  ~(*((uint32_t *)accel_task->dst)),
+		status = spdk_accel_batch_prep_crc32cv(ch, batch, accel_task->crc_dst,
+						       ++accel_task->v.iovs, accel_task->v.iovcnt - 1,
+						       ~(*((uint32_t *)accel_task->crc_dst)),
 						       accel_task->chained.cb_fn, accel_task->chained.cb_arg);
 		if (spdk_likely(!status)) {
 			return;
@@ -578,7 +678,7 @@ batched_crc32cv_done(void *cb_arg, int status)
 
 int
 spdk_accel_batch_prep_crc32cv(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
-			      uint32_t *dst, struct iovec *iovs, uint32_t iov_cnt, uint32_t seed,
+			      uint32_t *crc_dst, struct iovec *iovs, uint32_t iov_cnt, uint32_t seed,
 			      spdk_accel_completion_cb cb_fn, void *cb_arg)
 {
 	struct accel_io_channel *accel_ch;
@@ -595,7 +695,8 @@ spdk_accel_batch_prep_crc32cv(struct spdk_io_channel *ch, struct spdk_accel_batc
 	}
 
 	if (iov_cnt == 1) {
-		return spdk_accel_batch_prep_crc32c(ch, batch, dst, iovs[0].iov_base, seed, iovs[0].iov_len, cb_fn,
+		return spdk_accel_batch_prep_crc32c(ch, batch, crc_dst, iovs[0].iov_base, seed, iovs[0].iov_len,
+						    cb_fn,
 						    cb_arg);
 	}
 
@@ -607,7 +708,7 @@ spdk_accel_batch_prep_crc32cv(struct spdk_io_channel *ch, struct spdk_accel_batc
 
 	accel_task->v.iovs = iovs;
 	accel_task->v.iovcnt = iov_cnt;
-	accel_task->dst = dst;
+	accel_task->crc_dst = crc_dst;
 	accel_task->seed = seed;
 	accel_task->op_code = ACCEL_OPCODE_CRC32C;
 
@@ -628,6 +729,36 @@ spdk_accel_batch_prep_crc32cv(struct spdk_io_channel *ch, struct spdk_accel_batc
 	return 0;
 }
 
+int
+spdk_accel_batch_prep_copy_crc32c(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
+				  void *dst, void *src, uint32_t *crc_dst,  uint32_t seed, uint64_t nbytes,
+				  spdk_accel_completion_cb cb_fn, void *cb_arg)
+{
+	struct spdk_accel_task *accel_task;
+	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
+
+	accel_task = _get_task(accel_ch, batch, cb_fn, cb_arg);
+	if (accel_task == NULL) {
+		return -ENOMEM;
+	}
+
+	accel_task->dst = dst;
+	accel_task->src = src;
+	accel_task->crc_dst = crc_dst;
+	accel_task->v.iovcnt = 0;
+	accel_task->seed = seed;
+	accel_task->nbytes = nbytes;
+	accel_task->op_code = ACCEL_OPCODE_COPY_CRC32C;
+
+	if (_is_supported(accel_ch->engine, ACCEL_COPY_CRC32C)) {
+		TAILQ_INSERT_TAIL(&batch->hw_tasks, accel_task, link);
+	} else {
+		TAILQ_INSERT_TAIL(&batch->sw_tasks, accel_task, link);
+	}
+
+	return 0;
+}
+
 /* Accel framework public API for batch_create function. */
 struct spdk_accel_batch *
 spdk_accel_batch_create(struct spdk_io_channel *ch)
@@ -715,6 +846,11 @@ spdk_accel_batch_submit(struct spdk_io_channel *ch, struct spdk_accel_batch *bat
 			}
 			spdk_accel_task_complete(accel_task, 0);
 			break;
+		case ACCEL_OPCODE_COPY_CRC32C:
+			_sw_accel_copy(accel_task->dst, accel_task->src, accel_task->nbytes);
+			_sw_accel_crc32c(accel_task->crc_dst, accel_task->src, accel_task->seed, accel_task->nbytes);
+			spdk_accel_task_complete(accel_task, 0);
+			break;
 		case ACCEL_OPCODE_DUALCAST:
 			_sw_accel_dualcast(accel_task->dst, accel_task->dst2, accel_task->src,
 					   accel_task->nbytes);
@@ -959,23 +1095,15 @@ _sw_accel_fill(void *dst, uint8_t fill, uint64_t nbytes)
 }
 
 static void
-_sw_accel_crc32c(uint32_t *dst, void *src, uint32_t seed, uint64_t nbytes)
+_sw_accel_crc32c(uint32_t *crc_dst, void *src, uint32_t seed, uint64_t nbytes)
 {
-	*dst = spdk_crc32c_update(src, nbytes, ~seed);
+	*crc_dst = spdk_crc32c_update(src, nbytes, ~seed);
 }
 
 static void
-_sw_accel_crc32cv(uint32_t *dst, struct iovec *iov, uint32_t iovcnt, uint32_t seed)
+_sw_accel_crc32cv(uint32_t *crc_dst, struct iovec *iov, uint32_t iovcnt, uint32_t seed)
 {
-	uint32_t i, crc32c = ~seed;
-
-	for (i = 0; i < iovcnt; i++) {
-		assert(iov[i].iov_base != NULL);
-		assert(iov[i].iov_len != 0);
-		crc32c = spdk_crc32c_update(iov[i].iov_base, iov[i].iov_len, crc32c);
-	}
-
-	*dst = crc32c;
+	*crc_dst = spdk_crc32c_iov_update(iov, iovcnt, ~seed);
 }
 
 static struct spdk_io_channel *sw_accel_get_io_channel(void);

lib/accel/spdk_accel.map

@@ -15,6 +15,7 @@
 	spdk_accel_batch_prep_fill;
 	spdk_accel_batch_prep_crc32c;
 	spdk_accel_batch_prep_crc32cv;
+	spdk_accel_batch_prep_copy_crc32c;
 	spdk_accel_batch_submit;
 	spdk_accel_batch_cancel;
 	spdk_accel_submit_copy;
@@ -23,6 +24,8 @@
 	spdk_accel_submit_fill;
 	spdk_accel_submit_crc32c;
 	spdk_accel_submit_crc32cv;
+	spdk_accel_submit_copy_crc32c;
+	spdk_accel_submit_copy_crc32cv;
 	spdk_accel_write_config_json;
 
 	# functions needed by modules

lib/bdev/Makefile

@@ -34,7 +34,7 @@
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-SO_VER := 6
+SO_VER := 7
 SO_MINOR := 0
 
 ifeq ($(CONFIG_VTUNE),y)

lib/bdev/bdev.c

@@ -84,12 +84,27 @@ int __itt_init_ittlib(const char *, __itt_group_id);
 
 #define SPDK_BDEV_POOL_ALIGNMENT 512
 
+/* The maximum number of children requests for a UNMAP or WRITE ZEROES command
+ * when splitting into children requests at a time.
+ */
+#define SPDK_BDEV_MAX_CHILDREN_UNMAP_WRITE_ZEROES_REQS (8)
+
 static const char *qos_rpc_type[] = {"rw_ios_per_sec",
 				     "rw_mbytes_per_sec", "r_mbytes_per_sec", "w_mbytes_per_sec"
 				    };
 
 TAILQ_HEAD(spdk_bdev_list, spdk_bdev);
 
+RB_HEAD(bdev_name_tree, spdk_bdev_name);
+
+static int
+bdev_name_cmp(struct spdk_bdev_name *name1, struct spdk_bdev_name *name2)
+{
+	return strcmp(name1->name, name2->name);
+}
+
+RB_GENERATE_STATIC(bdev_name_tree, spdk_bdev_name, node, bdev_name_cmp);
+
 struct spdk_bdev_mgr {
 	struct spdk_mempool *bdev_io_pool;
 
@@ -101,6 +116,7 @@ struct spdk_bdev_mgr {
 	TAILQ_HEAD(bdev_module_list, spdk_bdev_module) bdev_modules;
 
 	struct spdk_bdev_list bdevs;
+	struct bdev_name_tree bdev_names;
 
 	bool init_complete;
 	bool module_init_complete;
@@ -115,6 +131,7 @@ struct spdk_bdev_mgr {
 static struct spdk_bdev_mgr g_bdev_mgr = {
 	.bdev_modules = TAILQ_HEAD_INITIALIZER(g_bdev_mgr.bdev_modules),
 	.bdevs = TAILQ_HEAD_INITIALIZER(g_bdev_mgr.bdevs),
+	.bdev_names = RB_INITIALIZER(g_bdev_mgr.bdev_names),
 	.init_complete = false,
 	.module_init_complete = false,
 	.mutex = PTHREAD_MUTEX_INITIALIZER,
@@ -458,6 +475,33 @@ spdk_bdev_set_opts(struct spdk_bdev_opts *opts)
 	return 0;
 }
 
+static struct spdk_bdev *
+bdev_get_by_name(const char *bdev_name)
+{
+	struct spdk_bdev_name find;
+	struct spdk_bdev_name *res;
+
+	find.name = (char *)bdev_name;
+	res = RB_FIND(bdev_name_tree, &g_bdev_mgr.bdev_names, &find);
+	if (res != NULL) {
+		return res->bdev;
+	}
+
+	return NULL;
+}
+
+struct spdk_bdev *
+spdk_bdev_get_by_name(const char *bdev_name)
+{
+	struct spdk_bdev *bdev;
+
+	pthread_mutex_lock(&g_bdev_mgr.mutex);
+	bdev = bdev_get_by_name(bdev_name);
+	pthread_mutex_unlock(&g_bdev_mgr.mutex);
+
+	return bdev;
+}
+
 struct spdk_bdev_wait_for_examine_ctx {
 	struct spdk_poller              *poller;
 	spdk_bdev_wait_for_examine_cb	cb_fn;
@@ -541,7 +585,7 @@ bdev_in_examine_allowlist(struct spdk_bdev *bdev)
 		return true;
 	}
 	TAILQ_FOREACH(tmp, &bdev->aliases, tailq) {
-		if (bdev_examine_allowlist_check(tmp->alias)) {
+		if (bdev_examine_allowlist_check(tmp->alias.name)) {
 			return true;
 		}
 	}
@@ -708,29 +752,6 @@ spdk_bdev_next_leaf(struct spdk_bdev *prev)
 	return bdev;
 }
 
-struct spdk_bdev *
-spdk_bdev_get_by_name(const char *bdev_name)
-{
-	struct spdk_bdev_alias *tmp;
-	struct spdk_bdev *bdev = spdk_bdev_first();
-
-	while (bdev != NULL) {
-		if (strcmp(bdev_name, bdev->name) == 0) {
-			return bdev;
-		}
-
-		TAILQ_FOREACH(tmp, &bdev->aliases, tailq) {
-			if (strcmp(bdev_name, tmp->alias) == 0) {
-				return bdev;
-			}
-		}
-
-		bdev = spdk_bdev_next(bdev);
-	}
-
-	return NULL;
-}
-
 void
 spdk_bdev_io_set_buf(struct spdk_bdev_io *bdev_io, void *buf, size_t len)
 {
@@ -1953,26 +1974,7 @@ bdev_queue_io_wait_with_cb(struct spdk_bdev_io *bdev_io, spdk_bdev_io_wait_cb cb
 }
 
 static bool
-bdev_io_type_can_split(uint8_t type)
-{
-	assert(type != SPDK_BDEV_IO_TYPE_INVALID);
-	assert(type < SPDK_BDEV_NUM_IO_TYPES);
-
-	/* Only split READ and WRITE I/O.  Theoretically other types of I/O like
-	 * UNMAP could be split, but these types of I/O are typically much larger
-	 * in size (sometimes the size of the entire block device), and the bdev
-	 * module can more efficiently split these types of I/O.  Plus those types
-	 * of I/O do not have a payload, which makes the splitting process simpler.
-	 */
-	if (type == SPDK_BDEV_IO_TYPE_READ || type == SPDK_BDEV_IO_TYPE_WRITE) {
-		return true;
-	} else {
-		return false;
-	}
-}
-
-static bool
-bdev_io_should_split(struct spdk_bdev_io *bdev_io)
+bdev_rw_should_split(struct spdk_bdev_io *bdev_io)
 {
 	uint32_t io_boundary = bdev_io->bdev->optimal_io_boundary;
 	uint32_t max_size = bdev_io->bdev->max_segment_size;
@@ -1984,10 +1986,6 @@ bdev_io_should_split(struct spdk_bdev_io *bdev_io)
 		return false;
 	}
 
-	if (!bdev_io_type_can_split(bdev_io->type)) {
-		return false;
-	}
-
 	if (io_boundary) {
 		uint64_t start_stripe, end_stripe;
 
@@ -2024,6 +2022,52 @@ bdev_io_should_split(struct spdk_bdev_io *bdev_io)
 	return false;
 }
 
+static bool
+bdev_unmap_should_split(struct spdk_bdev_io *bdev_io)
+{
+	uint32_t num_unmap_segments;
+
+	if (!bdev_io->bdev->max_unmap || !bdev_io->bdev->max_unmap_segments) {
+		return false;
+	}
+	num_unmap_segments = spdk_divide_round_up(bdev_io->u.bdev.num_blocks, bdev_io->bdev->max_unmap);
+	if (num_unmap_segments > bdev_io->bdev->max_unmap_segments) {
+		return true;
+	}
+
+	return false;
+}
+
+static bool
+bdev_write_zeroes_should_split(struct spdk_bdev_io *bdev_io)
+{
+	if (!bdev_io->bdev->max_write_zeroes) {
+		return false;
+	}
+
+	if (bdev_io->u.bdev.num_blocks > bdev_io->bdev->max_write_zeroes) {
+		return true;
+	}
+
+	return false;
+}
+
+static bool
+bdev_io_should_split(struct spdk_bdev_io *bdev_io)
+{
+	switch (bdev_io->type) {
+	case SPDK_BDEV_IO_TYPE_READ:
+	case SPDK_BDEV_IO_TYPE_WRITE:
+		return bdev_rw_should_split(bdev_io);
+	case SPDK_BDEV_IO_TYPE_UNMAP:
+		return bdev_unmap_should_split(bdev_io);
+	case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
+		return bdev_write_zeroes_should_split(bdev_io);
+	default:
+		return false;
+	}
+}
+
 static uint32_t
 _to_next_boundary(uint64_t offset, uint32_t boundary)
 {
@@ -2034,7 +2078,104 @@ static void
 bdev_io_split_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg);
 
 static void
-_bdev_io_split(void *_bdev_io)
+_bdev_rw_split(void *_bdev_io);
+
+static void
+bdev_unmap_split(struct spdk_bdev_io *bdev_io);
+
+static void
+_bdev_unmap_split(void *_bdev_io)
+{
+	return bdev_unmap_split((struct spdk_bdev_io *)_bdev_io);
+}
+
+static void
+bdev_write_zeroes_split(struct spdk_bdev_io *bdev_io);
+
+static void
+_bdev_write_zeroes_split(void *_bdev_io)
+{
+	return bdev_write_zeroes_split((struct spdk_bdev_io *)_bdev_io);
+}
+
+static int
+bdev_io_split_submit(struct spdk_bdev_io *bdev_io, struct iovec *iov, int iovcnt, void *md_buf,
+		     uint64_t num_blocks, uint64_t *offset, uint64_t *remaining)
+{
+	int rc;
+	uint64_t current_offset, current_remaining;
+	spdk_bdev_io_wait_cb io_wait_fn;
+
+	current_offset = *offset;
+	current_remaining = *remaining;
+
+	bdev_io->u.bdev.split_outstanding++;
+
+	io_wait_fn = _bdev_rw_split;
+	switch (bdev_io->type) {
+	case SPDK_BDEV_IO_TYPE_READ:
+		rc = bdev_readv_blocks_with_md(bdev_io->internal.desc,
+					       spdk_io_channel_from_ctx(bdev_io->internal.ch),
+					       iov, iovcnt, md_buf, current_offset,
+					       num_blocks,
+					       bdev_io_split_done, bdev_io);
+		break;
+	case SPDK_BDEV_IO_TYPE_WRITE:
+		rc = bdev_writev_blocks_with_md(bdev_io->internal.desc,
+						spdk_io_channel_from_ctx(bdev_io->internal.ch),
+						iov, iovcnt, md_buf, current_offset,
+						num_blocks,
+						bdev_io_split_done, bdev_io);
+		break;
+	case SPDK_BDEV_IO_TYPE_UNMAP:
+		io_wait_fn = _bdev_unmap_split;
+		rc = spdk_bdev_unmap_blocks(bdev_io->internal.desc,
+					    spdk_io_channel_from_ctx(bdev_io->internal.ch),
+					    current_offset, num_blocks,
+					    bdev_io_split_done, bdev_io);
+		break;
+	case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
+		io_wait_fn = _bdev_write_zeroes_split;
+		rc = spdk_bdev_write_zeroes_blocks(bdev_io->internal.desc,
+						   spdk_io_channel_from_ctx(bdev_io->internal.ch),
+						   current_offset, num_blocks,
+						   bdev_io_split_done, bdev_io);
+		break;
+	default:
+		assert(false);
+		rc = -EINVAL;
+		break;
+	}
+
+	if (rc == 0) {
+		current_offset += num_blocks;
+		current_remaining -= num_blocks;
+		bdev_io->u.bdev.split_current_offset_blocks = current_offset;
+		bdev_io->u.bdev.split_remaining_num_blocks = current_remaining;
+		*offset = current_offset;
+		*remaining = current_remaining;
+	} else {
+		bdev_io->u.bdev.split_outstanding--;
+		if (rc == -ENOMEM) {
+			if (bdev_io->u.bdev.split_outstanding == 0) {
+				/* No I/O is outstanding. Hence we should wait here. */
+				bdev_queue_io_wait_with_cb(bdev_io, io_wait_fn);
+			}
+		} else {
+			bdev_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;
+			if (bdev_io->u.bdev.split_outstanding == 0) {
+				spdk_trace_record(TRACE_BDEV_IO_DONE, 0, 0, (uintptr_t)bdev_io);
+				TAILQ_REMOVE(&bdev_io->internal.ch->io_submitted, bdev_io, internal.ch_link);
+				bdev_io->internal.cb(bdev_io, false, bdev_io->internal.caller_ctx);
+			}
+		}
+	}
+
+	return rc;
+}
+
+static void
+_bdev_rw_split(void *_bdev_io)
 {
 	struct iovec *parent_iov, *iov;
 	struct spdk_bdev_io *bdev_io = _bdev_io;
@@ -2138,8 +2279,7 @@ _bdev_io_split(void *_bdev_io)
 							if (bdev_io->u.bdev.split_outstanding == 0) {
 								SPDK_ERRLOG("The first child io was less than a block size\n");
 								bdev_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;
-								spdk_trace_record_tsc(spdk_get_ticks(), TRACE_BDEV_IO_DONE, 0, 0,
-										      (uintptr_t)bdev_io, 0);
+								spdk_trace_record(TRACE_BDEV_IO_DONE, 0, 0, (uintptr_t)bdev_io);
 								TAILQ_REMOVE(&bdev_io->internal.ch->io_submitted, bdev_io, internal.ch_link);
 								bdev_io->internal.cb(bdev_io, false, bdev_io->internal.caller_ctx);
 							}
@@ -2162,44 +2302,56 @@ _bdev_io_split(void *_bdev_io)
 			to_next_boundary -= to_next_boundary_bytes / blocklen;
 		}
 
-		bdev_io->u.bdev.split_outstanding++;
-
-		if (bdev_io->type == SPDK_BDEV_IO_TYPE_READ) {
-			rc = bdev_readv_blocks_with_md(bdev_io->internal.desc,
-						       spdk_io_channel_from_ctx(bdev_io->internal.ch),
-						       iov, iovcnt, md_buf, current_offset,
-						       to_next_boundary,
-						       bdev_io_split_done, bdev_io);
-		} else {
-			rc = bdev_writev_blocks_with_md(bdev_io->internal.desc,
-							spdk_io_channel_from_ctx(bdev_io->internal.ch),
-							iov, iovcnt, md_buf, current_offset,
-							to_next_boundary,
-							bdev_io_split_done, bdev_io);
+		rc = bdev_io_split_submit(bdev_io, iov, iovcnt, md_buf, to_next_boundary,
+					  &current_offset, &remaining);
+		if (spdk_unlikely(rc)) {
+			return;
 		}
+	}
+}
 
-		if (rc == 0) {
-			current_offset += to_next_boundary;
-			remaining -= to_next_boundary;
-			bdev_io->u.bdev.split_current_offset_blocks = current_offset;
-			bdev_io->u.bdev.split_remaining_num_blocks = remaining;
+static void
+bdev_unmap_split(struct spdk_bdev_io *bdev_io)
+{
+	uint64_t offset, unmap_blocks, remaining, max_unmap_blocks;
+	uint32_t num_children_reqs = 0;
+	int rc;
+
+	offset = bdev_io->u.bdev.split_current_offset_blocks;
+	remaining = bdev_io->u.bdev.split_remaining_num_blocks;
+	max_unmap_blocks = bdev_io->bdev->max_unmap * bdev_io->bdev->max_unmap_segments;
+
+	while (remaining && (num_children_reqs < SPDK_BDEV_MAX_CHILDREN_UNMAP_WRITE_ZEROES_REQS)) {
+		unmap_blocks = spdk_min(remaining, max_unmap_blocks);
+
+		rc = bdev_io_split_submit(bdev_io, NULL, 0, NULL, unmap_blocks,
+					  &offset, &remaining);
+		if (spdk_likely(rc == 0)) {
+			num_children_reqs++;
 		} else {
-			bdev_io->u.bdev.split_outstanding--;
-			if (rc == -ENOMEM) {
-				if (bdev_io->u.bdev.split_outstanding == 0) {
-					/* No I/O is outstanding. Hence we should wait here. */
-					bdev_queue_io_wait_with_cb(bdev_io, _bdev_io_split);
-				}
-			} else {
-				bdev_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;
-				if (bdev_io->u.bdev.split_outstanding == 0) {
-					spdk_trace_record_tsc(spdk_get_ticks(), TRACE_BDEV_IO_DONE, 0, 0,
-							      (uintptr_t)bdev_io, 0);
-					TAILQ_REMOVE(&bdev_io->internal.ch->io_submitted, bdev_io, internal.ch_link);
-					bdev_io->internal.cb(bdev_io, false, bdev_io->internal.caller_ctx);
-				}
-			}
+			return;
+		}
+	}
+}
 
+static void
+bdev_write_zeroes_split(struct spdk_bdev_io *bdev_io)
+{
+	uint64_t offset, write_zeroes_blocks, remaining;
+	uint32_t num_children_reqs = 0;
+	int rc;
+
+	offset = bdev_io->u.bdev.split_current_offset_blocks;
+	remaining = bdev_io->u.bdev.split_remaining_num_blocks;
+
+	while (remaining && (num_children_reqs < SPDK_BDEV_MAX_CHILDREN_UNMAP_WRITE_ZEROES_REQS)) {
+		write_zeroes_blocks = spdk_min(remaining, bdev_io->bdev->max_write_zeroes);
+
+		rc = bdev_io_split_submit(bdev_io, NULL, 0, NULL, write_zeroes_blocks,
+					  &offset, &remaining);
+		if (spdk_likely(rc == 0)) {
+			num_children_reqs++;
+		} else {
 			return;
 		}
 	}
@@ -2228,8 +2380,7 @@ bdev_io_split_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
 	 */
 	if (parent_io->u.bdev.split_remaining_num_blocks == 0) {
 		assert(parent_io->internal.cb != bdev_io_split_done);
-		spdk_trace_record_tsc(spdk_get_ticks(), TRACE_BDEV_IO_DONE, 0, 0,
-				      (uintptr_t)parent_io, 0);
+		spdk_trace_record(TRACE_BDEV_IO_DONE, 0, 0, (uintptr_t)parent_io);
 		TAILQ_REMOVE(&parent_io->internal.ch->io_submitted, parent_io, internal.ch_link);
 		parent_io->internal.cb(parent_io, parent_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS,
 				       parent_io->internal.caller_ctx);
@@ -2240,40 +2391,66 @@ bdev_io_split_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
 	 * Continue with the splitting process.  This function will complete the parent I/O if the
 	 * splitting is done.
 	 */
-	_bdev_io_split(parent_io);
+	switch (parent_io->type) {
+	case SPDK_BDEV_IO_TYPE_READ:
+	case SPDK_BDEV_IO_TYPE_WRITE:
+		_bdev_rw_split(parent_io);
+		break;
+	case SPDK_BDEV_IO_TYPE_UNMAP:
+		bdev_unmap_split(parent_io);
+		break;
+	case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
+		bdev_write_zeroes_split(parent_io);
+		break;
+	default:
+		assert(false);
+		break;
+	}
 }
 
 static void
-bdev_io_split_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, bool success);
+bdev_rw_split_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, bool success);
 
 static void
 bdev_io_split(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
 {
-	assert(bdev_io_type_can_split(bdev_io->type));
-
 	bdev_io->u.bdev.split_current_offset_blocks = bdev_io->u.bdev.offset_blocks;
 	bdev_io->u.bdev.split_remaining_num_blocks = bdev_io->u.bdev.num_blocks;
 	bdev_io->u.bdev.split_outstanding = 0;
 	bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS;
 
-	if (_is_buf_allocated(bdev_io->u.bdev.iovs)) {
-		_bdev_io_split(bdev_io);
-	} else {
-		assert(bdev_io->type == SPDK_BDEV_IO_TYPE_READ);
-		spdk_bdev_io_get_buf(bdev_io, bdev_io_split_get_buf_cb,
-				     bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen);
+	switch (bdev_io->type) {
+	case SPDK_BDEV_IO_TYPE_READ:
+	case SPDK_BDEV_IO_TYPE_WRITE:
+		if (_is_buf_allocated(bdev_io->u.bdev.iovs)) {
+			_bdev_rw_split(bdev_io);
+		} else {
+			assert(bdev_io->type == SPDK_BDEV_IO_TYPE_READ);
+			spdk_bdev_io_get_buf(bdev_io, bdev_rw_split_get_buf_cb,
+					     bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen);
+		}
+		break;
+	case SPDK_BDEV_IO_TYPE_UNMAP:
+		bdev_unmap_split(bdev_io);
+		break;
+	case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
+		bdev_write_zeroes_split(bdev_io);
+		break;
+	default:
+		assert(false);
+		break;
 	}
 }
 
 static void
-bdev_io_split_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, bool success)
+bdev_rw_split_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, bool success)
 {
 	if (!success) {
 		spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
 		return;
 	}
 
-	_bdev_io_split(bdev_io);
+	_bdev_rw_split(bdev_io);
 }
 
 /* Explicitly mark this inline, since it's used as a function pointer and otherwise won't
@@ -2467,11 +2644,6 @@ spdk_bdev_io_type_supported(struct spdk_bdev *bdev, enum spdk_bdev_io_type io_ty
 			/* The bdev layer will emulate write zeroes as long as write is supported. */
 			supported = bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_WRITE);
 			break;
-		case SPDK_BDEV_IO_TYPE_ZCOPY:
-			/* Zero copy can be emulated with regular read and write */
-			supported = bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_READ) &&
-				    bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_WRITE);
-			break;
 		default:
 			break;
 		}
@@ -3073,32 +3245,64 @@ bdev_channel_destroy(void *io_device, void *ctx_buf)
 	bdev_channel_destroy_resource(ch);
 }
 
+/*
+ * If the name already exists in the global bdev name tree, RB_INSERT() returns a pointer
+ * to it. Hence we do not have to call bdev_get_by_name() when using this function.
+ */
+static int
+bdev_name_add(struct spdk_bdev_name *bdev_name, struct spdk_bdev *bdev, const char *name)
+{
+	struct spdk_bdev_name *tmp;
+
+	bdev_name->name = strdup(name);
+	if (bdev_name->name == NULL) {
+		SPDK_ERRLOG("Unable to allocate bdev name\n");
+		return -ENOMEM;
+	}
+
+	bdev_name->bdev = bdev;
+
+	pthread_mutex_lock(&g_bdev_mgr.mutex);
+	tmp = RB_INSERT(bdev_name_tree, &g_bdev_mgr.bdev_names, bdev_name);
+	pthread_mutex_unlock(&g_bdev_mgr.mutex);
+
+	if (tmp != NULL) {
+		SPDK_ERRLOG("Bdev name %s already exists\n", name);
+		free(bdev_name->name);
+		return -EEXIST;
+	}
+
+	return 0;
+}
+
+static void
+bdev_name_del(struct spdk_bdev_name *bdev_name)
+{
+	RB_REMOVE(bdev_name_tree, &g_bdev_mgr.bdev_names, bdev_name);
+	free(bdev_name->name);
+}
+
 int
 spdk_bdev_alias_add(struct spdk_bdev *bdev, const char *alias)
 {
 	struct spdk_bdev_alias *tmp;
+	int ret;
 
 	if (alias == NULL) {
 		SPDK_ERRLOG("Empty alias passed\n");
 		return -EINVAL;
 	}
 
-	if (spdk_bdev_get_by_name(alias)) {
-		SPDK_ERRLOG("Bdev name/alias: %s already exists\n", alias);
-		return -EEXIST;
-	}
-
 	tmp = calloc(1, sizeof(*tmp));
 	if (tmp == NULL) {
 		SPDK_ERRLOG("Unable to allocate alias\n");
 		return -ENOMEM;
 	}
 
-	tmp->alias = strdup(alias);
-	if (tmp->alias == NULL) {
+	ret = bdev_name_add(&tmp->alias, bdev, alias);
+	if (ret != 0) {
 		free(tmp);
-		SPDK_ERRLOG("Unable to allocate alias\n");
-		return -ENOMEM;
+		return ret;
 	}
 
 	TAILQ_INSERT_TAIL(&bdev->aliases, tmp, tailq);
@@ -3112,9 +3316,11 @@ spdk_bdev_alias_del(struct spdk_bdev *bdev, const char *alias)
 	struct spdk_bdev_alias *tmp;
 
 	TAILQ_FOREACH(tmp, &bdev->aliases, tailq) {
-		if (strcmp(alias, tmp->alias) == 0) {
+		if (strcmp(alias, tmp->alias.name) == 0) {
 			TAILQ_REMOVE(&bdev->aliases, tmp, tailq);
-			free(tmp->alias);
+			pthread_mutex_lock(&g_bdev_mgr.mutex);
+			bdev_name_del(&tmp->alias);
+			pthread_mutex_unlock(&g_bdev_mgr.mutex);
 			free(tmp);
 			return 0;
 		}
@@ -3132,7 +3338,9 @@ spdk_bdev_alias_del_all(struct spdk_bdev *bdev)
 
 	TAILQ_FOREACH_SAFE(p, &bdev->aliases, tailq, tmp) {
 		TAILQ_REMOVE(&bdev->aliases, p, tailq);
-		free(p->alias);
+		pthread_mutex_lock(&g_bdev_mgr.mutex);
+		bdev_name_del(&p->alias);
+		pthread_mutex_unlock(&g_bdev_mgr.mutex);
 		free(p);
 	}
 }
@@ -3291,6 +3499,12 @@ spdk_bdev_get_data_block_size(const struct spdk_bdev *bdev)
 	}
 }
 
+uint32_t
+spdk_bdev_get_physical_block_size(const struct spdk_bdev *bdev)
+{
+	return bdev->phys_blocklen;
+}
+
 static uint32_t
 _bdev_get_block_size_with_md(const struct spdk_bdev *bdev)
 {
@@ -3524,7 +3738,7 @@ _bdev_io_check_md_buf(const struct iovec *iovs, const void *md_buf)
 
 static int
 bdev_read_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, void *buf,
-			 void *md_buf, int64_t offset_blocks, uint64_t num_blocks,
+			 void *md_buf, uint64_t offset_blocks, uint64_t num_blocks,
 			 spdk_bdev_io_completion_cb cb, void *cb_arg)
 {
 	struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
@@ -3581,7 +3795,7 @@ spdk_bdev_read_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
 
 int
 spdk_bdev_read_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
-			      void *buf, void *md_buf, int64_t offset_blocks, uint64_t num_blocks,
+			      void *buf, void *md_buf, uint64_t offset_blocks, uint64_t num_blocks,
 			      spdk_bdev_io_completion_cb cb, void *cb_arg)
 {
 	struct iovec iov = {
@@ -4184,31 +4398,9 @@ spdk_bdev_comparev_and_writev_blocks(struct spdk_bdev_desc *desc, struct spdk_io
 				   bdev_comparev_and_writev_blocks_locked, bdev_io);
 }
 
-static void
-bdev_zcopy_get_buf(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, bool success)
-{
-	if (!success) {
-		/* Don't use spdk_bdev_io_complete here - this bdev_io was never actually submitted. */
-		bdev_io->internal.status = SPDK_BDEV_IO_STATUS_NOMEM;
-		bdev_io->internal.cb(bdev_io, success, bdev_io->internal.caller_ctx);
-		return;
-	}
-
-	if (bdev_io->u.bdev.zcopy.populate) {
-		/* Read the real data into the buffer */
-		bdev_io->type = SPDK_BDEV_IO_TYPE_READ;
-		bdev_io->internal.status = SPDK_BDEV_IO_STATUS_PENDING;
-		bdev_io_submit(bdev_io);
-		return;
-	}
-
-	/* Don't use spdk_bdev_io_complete here - this bdev_io was never actually submitted. */
-	bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS;
-	bdev_io->internal.cb(bdev_io, success, bdev_io->internal.caller_ctx);
-}
-
 int
 spdk_bdev_zcopy_start(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
+		      struct iovec *iov, int iovcnt,
 		      uint64_t offset_blocks, uint64_t num_blocks,
 		      bool populate,
 		      spdk_bdev_io_completion_cb cb, void *cb_arg)
@@ -4239,21 +4431,15 @@ spdk_bdev_zcopy_start(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
 	bdev_io->type = SPDK_BDEV_IO_TYPE_ZCOPY;
 	bdev_io->u.bdev.num_blocks = num_blocks;
 	bdev_io->u.bdev.offset_blocks = offset_blocks;
-	bdev_io->u.bdev.iovs = NULL;
-	bdev_io->u.bdev.iovcnt = 0;
+	bdev_io->u.bdev.iovs = iov;
+	bdev_io->u.bdev.iovcnt = iovcnt;
 	bdev_io->u.bdev.md_buf = NULL;
 	bdev_io->u.bdev.zcopy.populate = populate ? 1 : 0;
 	bdev_io->u.bdev.zcopy.commit = 0;
 	bdev_io->u.bdev.zcopy.start = 1;
 	bdev_io_init(bdev_io, bdev, cb_arg, cb);
 
-	if (bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_ZCOPY)) {
-		bdev_io_submit(bdev_io);
-	} else {
-		/* Emulate zcopy by allocating a buffer */
-		spdk_bdev_io_get_buf(bdev_io, bdev_zcopy_get_buf,
-				     bdev_io->u.bdev.num_blocks * bdev->blocklen);
-	}
+	bdev_io_submit(bdev_io);
 
 	return 0;
 }
@@ -4262,16 +4448,6 @@ int
 spdk_bdev_zcopy_end(struct spdk_bdev_io *bdev_io, bool commit,
 		    spdk_bdev_io_completion_cb cb, void *cb_arg)
 {
-	struct spdk_bdev *bdev = bdev_io->bdev;
-
-	if (bdev_io->type == SPDK_BDEV_IO_TYPE_READ) {
-		/* This can happen if the zcopy was emulated in start */
-		if (bdev_io->u.bdev.zcopy.start != 1) {
-			return -EINVAL;
-		}
-		bdev_io->type = SPDK_BDEV_IO_TYPE_ZCOPY;
-	}
-
 	if (bdev_io->type != SPDK_BDEV_IO_TYPE_ZCOPY) {
 		return -EINVAL;
 	}
@@ -4282,19 +4458,6 @@ spdk_bdev_zcopy_end(struct spdk_bdev_io *bdev_io, bool commit,
 	bdev_io->internal.cb = cb;
 	bdev_io->internal.status = SPDK_BDEV_IO_STATUS_PENDING;
 
-	if (bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_ZCOPY)) {
-		bdev_io_submit(bdev_io);
-		return 0;
-	}
-
-	if (!bdev_io->u.bdev.zcopy.commit) {
-		/* Don't use spdk_bdev_io_complete here - this bdev_io was never actually submitted. */
-		bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS;
-		bdev_io->internal.cb(bdev_io, true, bdev_io->internal.caller_ctx);
-		return 0;
-	}
-
-	bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE;
 	bdev_io_submit(bdev_io);
 
 	return 0;
@@ -5062,7 +5225,7 @@ bdev_io_complete(void *ctx)
 
 	tsc = spdk_get_ticks();
 	tsc_diff = tsc - bdev_io->internal.submit_tsc;
-	spdk_trace_record_tsc(tsc, TRACE_BDEV_IO_DONE, 0, 0, (uintptr_t)bdev_io, 0);
+	spdk_trace_record_tsc(tsc, TRACE_BDEV_IO_DONE, 0, 0, (uintptr_t)bdev_io);
 
 	TAILQ_REMOVE(&bdev_ch->io_submitted, bdev_io, internal.ch_link);
 
@@ -5404,9 +5567,10 @@ spdk_bdev_io_get_io_channel(struct spdk_bdev_io *bdev_io)
 }
 
 static int
-bdev_init(struct spdk_bdev *bdev)
+bdev_register(struct spdk_bdev *bdev)
 {
 	char *bdev_name;
+	int ret;
 
 	assert(bdev->module != NULL);
 
@@ -5420,11 +5584,6 @@ bdev_init(struct spdk_bdev *bdev)
 		return -EINVAL;
 	}
 
-	if (spdk_bdev_get_by_name(bdev->name)) {
-		SPDK_ERRLOG("Bdev name:%s already exists\n", bdev->name);
-		return -EEXIST;
-	}
-
 	/* Users often register their own I/O devices using the bdev name. In
 	 * order to avoid conflicts, prepend bdev_. */
 	bdev_name = spdk_sprintf_alloc("bdev_%s", bdev->name);
@@ -5439,6 +5598,12 @@ bdev_init(struct spdk_bdev *bdev)
 	bdev->internal.qd_poller = NULL;
 	bdev->internal.qos = NULL;
 
+	ret = bdev_name_add(&bdev->internal.bdev_name, bdev, bdev->name);
+	if (ret != 0) {
+		free(bdev_name);
+		return ret;
+	}
+
 	/* If the user didn't specify a uuid, generate one. */
 	if (spdk_mem_all_zero(&bdev->uuid, sizeof(bdev->uuid))) {
 		spdk_uuid_generate(&bdev->uuid);
@@ -5464,6 +5629,10 @@ bdev_init(struct spdk_bdev *bdev)
 		bdev->acwu = 1;
 	}
 
+	if (bdev->phys_blocklen == 0) {
+		bdev->phys_blocklen = spdk_bdev_get_data_block_size(bdev);
+	}
+
 	TAILQ_INIT(&bdev->internal.open_descs);
 	TAILQ_INIT(&bdev->internal.locked_ranges);
 	TAILQ_INIT(&bdev->internal.pending_locked_ranges);
@@ -5480,6 +5649,10 @@ bdev_init(struct spdk_bdev *bdev)
 	free(bdev_name);
 
 	pthread_mutex_init(&bdev->internal.mutex, NULL);
+
+	SPDK_DEBUGLOG(bdev, "Inserting bdev %s into list\n", bdev->name);
+	TAILQ_INSERT_TAIL(&g_bdev_mgr.bdevs, bdev, internal.link);
+
 	return 0;
 }
 
@@ -5508,32 +5681,23 @@ bdev_destroy_cb(void *io_device)
 }
 
 static void
-bdev_start_finished(void *arg)
+bdev_register_finished(void *arg)
 {
 	struct spdk_bdev *bdev = arg;
 
 	spdk_notify_send("bdev_register", spdk_bdev_get_name(bdev));
 }
 
-static void
-bdev_start(struct spdk_bdev *bdev)
-{
-	SPDK_DEBUGLOG(bdev, "Inserting bdev %s into list\n", bdev->name);
-	TAILQ_INSERT_TAIL(&g_bdev_mgr.bdevs, bdev, internal.link);
-
-	/* Examine configuration before initializing I/O */
-	bdev_examine(bdev);
-
-	spdk_bdev_wait_for_examine(bdev_start_finished, bdev);
-}
-
 int
 spdk_bdev_register(struct spdk_bdev *bdev)
 {
-	int rc = bdev_init(bdev);
+	int rc = bdev_register(bdev);
 
 	if (rc == 0) {
-		bdev_start(bdev);
+		/* Examine configuration before initializing I/O */
+		bdev_examine(bdev);
+
+		spdk_bdev_wait_for_examine(bdev_register_finished, bdev);
 	}
 
 	return rc;
@@ -5599,6 +5763,7 @@ bdev_unregister_unsafe(struct spdk_bdev *bdev)
 	if (rc == 0) {
 		TAILQ_REMOVE(&g_bdev_mgr.bdevs, bdev, internal.link);
 		SPDK_DEBUGLOG(bdev, "Removing bdev %s from list done\n", bdev->name);
+		bdev_name_del(&bdev->internal.bdev_name);
 		spdk_notify_send("bdev_unregister", spdk_bdev_get_name(bdev));
 	}
 
@@ -5729,7 +5894,7 @@ spdk_bdev_open_ext(const char *bdev_name, bool write, spdk_bdev_event_cb_t event
 
 	pthread_mutex_lock(&g_bdev_mgr.mutex);
 
-	bdev = spdk_bdev_get_by_name(bdev_name);
+	bdev = bdev_get_by_name(bdev_name);
 
 	if (bdev == NULL) {
 		SPDK_NOTICELOG("Currently unable to find bdev with name: %s\n", bdev_name);
@@ -5793,6 +5958,7 @@ spdk_bdev_close(struct spdk_bdev_desc *desc)
 
 	spdk_poller_unregister(&desc->io_timeout_poller);
 
+	pthread_mutex_lock(&g_bdev_mgr.mutex);
 	pthread_mutex_lock(&bdev->internal.mutex);
 	pthread_mutex_lock(&desc->mutex);
 
@@ -5825,12 +5991,14 @@ spdk_bdev_close(struct spdk_bdev_desc *desc)
 	if (bdev->internal.status == SPDK_BDEV_STATUS_REMOVING && TAILQ_EMPTY(&bdev->internal.open_descs)) {
 		rc = bdev_unregister_unsafe(bdev);
 		pthread_mutex_unlock(&bdev->internal.mutex);
+		pthread_mutex_unlock(&g_bdev_mgr.mutex);
 
 		if (rc == 0) {
 			spdk_io_device_unregister(__bdev_to_io_dev(bdev), bdev_destroy_cb);
 		}
 	} else {
 		pthread_mutex_unlock(&bdev->internal.mutex);
+		pthread_mutex_unlock(&g_bdev_mgr.mutex);
 	}
 }
 
@@ -6637,7 +6805,7 @@ bdev_lock_lba_range_get_channel(struct spdk_io_channel_iter *i)
 static void
 bdev_lock_lba_range_ctx(struct spdk_bdev *bdev, struct locked_lba_range_ctx *ctx)
 {
-	assert(spdk_get_thread() == ctx->range.owner_ch->channel->thread);
+	assert(spdk_get_thread() == spdk_io_channel_get_thread(ctx->range.owner_ch->channel));
 
 	/* We will add a copy of this range to each channel now. */
 	spdk_for_each_channel(__bdev_to_io_dev(bdev), bdev_lock_lba_range_get_channel, ctx,
@@ -6728,7 +6896,7 @@ bdev_unlock_lba_range_cb(struct spdk_io_channel_iter *i, int status)
 			TAILQ_REMOVE(&bdev->internal.pending_locked_ranges, range, tailq);
 			pending_ctx = SPDK_CONTAINEROF(range, struct locked_lba_range_ctx, range);
 			TAILQ_INSERT_TAIL(&bdev->internal.locked_ranges, range, tailq);
-			spdk_thread_send_msg(pending_ctx->range.owner_ch->channel->thread,
+			spdk_thread_send_msg(spdk_io_channel_get_thread(pending_ctx->range.owner_ch->channel),
 					     bdev_lock_lba_range_ctx_msg, pending_ctx);
 		}
 	}
@@ -6846,7 +7014,9 @@ SPDK_TRACE_REGISTER_FN(bdev_trace, "bdev", TRACE_GROUP_BDEV)
 	spdk_trace_register_owner(OWNER_BDEV, 'b');
 	spdk_trace_register_object(OBJECT_BDEV_IO, 'i');
 	spdk_trace_register_description("BDEV_IO_START", TRACE_BDEV_IO_START, OWNER_BDEV,
-					OBJECT_BDEV_IO, 1, 0, "type:   ");
+					OBJECT_BDEV_IO, 1,
+					SPDK_TRACE_ARG_TYPE_INT, "type");
 	spdk_trace_register_description("BDEV_IO_DONE", TRACE_BDEV_IO_DONE, OWNER_BDEV,
-					OBJECT_BDEV_IO, 0, 0, "");
+					OBJECT_BDEV_IO, 0,
+					SPDK_TRACE_ARG_TYPE_INT, "");
 }