dpdkbuild: add support for DPDK 20.05

EAL got a new dependency in 20.05: rte_telemetry.

Signed-off-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>

(cherry picked from commit 3b99a376d5595f4d6a458e8221bfd0b6b6f07b83)
Change-Id: I43df7afe9a84e88f034a7f87fc6a299f0bbd8bac
Signed-off-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2899
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>

.githooks/pre-push

@@ -30,7 +30,6 @@ fi
 echo "Running make with $COMP ..."
 echo "${MAKE} clean " > make.log
 $MAKE clean  >> make.log 2>&1
-
 echo "${MAKE} CONFIG_DEBUG=n CONFIG_WERROR=y " >> make.log
 $MAKE CONFIG_DEBUG=n CONFIG_WERROR=y  >> make.log 2>&1
 rc=$?
@@ -76,6 +75,64 @@ fi
 echo "$MAKE clean " >> make.log
 $MAKE clean  >> make.log 2>&1
 
+if [ "$SYSTEM" = "FreeBSD" ]; then
+	echo
+        echo "Pushing to $1 $2"
+        exit $rc
+fi
+
+if ! hash clang 2>/dev/null; then
+	echo "clang not found; skipping the clang tests"
+	echo
+	echo "Pushing to $1 $2"
+	exit $rc
+fi
+
+echo "Running make with clang ..."
+echo "make CONFIG_DEBUG=n CONFIG_WERROR=y CC=clang CXX=clang++ " >> make.log
+$MAKE CONFIG_DEBUG=n CONFIG_WERROR=y CC=clang CXX=clang++  >> make.log 2>&1
+rc=$?
+if [ $rc -ne 0 ]; then
+tail -20 make.log
+	echo ""
+	echo "ERROR make CC=clang CXX=clang++ returned errors!"
+	echo "ERROR Fix the problem and use 'git commit' to update your changes."
+	echo "ERROR See `pwd`/make.log for more information."
+	echo ""
+	exit $rc
+fi
+
+echo "make clean CC=clang CXX=clang++ SKIP_DPDK_BUILD=1 " >> make.log
+$MAKE clean CC=clang CXX=clang++ SKIP_DPDK_BUILD=1 >> make.log 2>&1
+echo "make CONFIG_DEBUG=y CONFIG_WERROR=y CC=clang CXX=clang++ SKIP_DPDK_BUILD=1 " >> make.log
+$MAKE CONFIG_DEBUG=y CONFIG_WERROR=y CC=clang CXX=clang++ SKIP_DPDK_BUILD=1 >> make.log 2>&1
+rc=$?
+if [ $rc -ne 0 ]; then
+	tail -20 make.log
+	echo ""
+	echo "ERROR make CC=clang CXX=clang++ returned errors!"
+	echo "ERROR Fix the problem and use 'git commit' to update your changes."
+	echo "ERROR See `pwd`/make.log for more information."
+	echo ""
+	exit $rc
+fi
+
+echo "Running unittest.sh ..."
+echo "./test/unit/unittest.sh" >> make.log
+"./test/unit/unittest.sh" >> make.log 2>&1
+rc=$?
+if [ $rc -ne 0 ]; then
+	tail -20 make.log
+	echo ""
+	echo "ERROR unittest returned errors!"
+	echo "ERROR Fix the problem and use 'git commit' to update your changes."
+	echo "ERROR See `pwd`/make.log for more information."
+	echo ""
+	exit $rc
+fi
+
+${MAKE} clean CC=clang CXX=clang++ 2> /dev/null
+
 echo "Pushing to $1 $2"
 
 exit $rc

.github/mistaken-pull-closer.yml

@@ -1,10 +0,0 @@
-filters:
-  - true
-
-commentBody: |
-  Thanks for your contribution!  Unfortunately, we don't use GitHub pull
-  requests to manage code contributions to this repository.  Instead, please
-  see https://spdk.io/development which provides instructions on how to
-  submit patches to the SPDK Gerrit instance.
-
-addLabel: false

.gitignore

@@ -2,23 +2,17 @@
 *.a
 *.cmd
 *.d
-*.dll
-*.exe
 *.gcda
 *.gcno
 *.kdev4
 *.ko
-*.lib
 *.log
 *.o
-*.obj
-*.pdb
 *.pyc
 *.so
 *.so.*
 *.swp
 *.DS_Store
-build/
 ut_coverage/
 tags
 cscope.out
@@ -31,7 +25,6 @@ CONFIG.local
 .project
 .cproject
 .settings
-.gitreview
 mk/cc.mk
 mk/config.mk
 mk/cc.flags.mk

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "dpdk"]
 	path = dpdk
-	url = https://git.quacker.org/d/numam-dpdk.git
+	url = https://github.com/spdk/dpdk.git
 [submodule "intel-ipsec-mb"]
 	path = intel-ipsec-mb
 	url = https://github.com/spdk/intel-ipsec-mb.git
@@ -10,6 +10,3 @@
 [submodule "ocf"]
 	path = ocf
 	url = https://github.com/Open-CAS/ocf.git
-[submodule "libvfio-user"]
-	path = libvfio-user
-	url = https://github.com/nutanix/libvfio-user.git

CHANGELOG.md

@@ -1,771 +1,15 @@
 # Changelog
 
-## v21.04.1: (Upcoming Release)
+## v20.04.2: (Upcoming Release)
 
-## v21.04: ZNS NVMe bdev, PMR, ADQ initiator, RPM
-
-### accel
-
-Two new accelerated crc32 functions `spdk_accel_submit_crc32cv` and
-`spdk_accel_batch_prep_crc32cv` are added in order to provide the
-chained accelerated CRC32 computation support.
-
-### bdev
-
-For `bdev_ocssd_create` RPC, the optional parameter `range` was removed.
-Only one OCSSD bdev can be created for one OCSSD namespace.
-
-Removed the `spdk_bdev_open` from bdev library API.
-Removed the `spdk_vbdev_register` and `spdk_bdev_part_base_construct` from bdev module API.
-Removed the `config_text` function for bdev modules to report legacy config.
-
-Added `spdk_bdev_get_max_active_zones` API to display maximum number active zones of a given bdev.
-
-Added `spdk_bdev_get_max_zone_append_size` API to display maximum zone append data transfer size.
-
-### bdev_nvme
-
-Added support for zoned namespaces.
-
-### blobstore
-
-Removed the `spdk_bdev_create_bs_dev_from_desc` and `spdk_bdev_create_bs_dev` API.
-
-### env
-
-Added `spdk_pci_device_allow` API to allow applications to add PCI addresses to
-the allowed list after the application has started.
-
-Removed the `pci_whitelist`, `pci_blacklist` and `master_core` members of struct `spdk_env_opts`.
-
-Added hotplug support based on uevent in `pci_event.c`. Added usage of this functionality in
-nvme, virtio-scsi and virtio_blk libraries. Please see the new API `spdk_pci_event_listen`,
-`spdk_pci_get_event`, `spdk_pci_register_error_handler` and `spdk_pci_unregister_error_handler`.
-
-### event
-
-Removed the `config_file`, `max_delay_us`, `pci_whitelist`
-and `pci_blacklist` members of struct `spdk_app_opts`.
-
-### idxd
-
-A new API `spdk_idxd_device_needs_rebalance` was added so that users of the library
-can know whether they need to rebalance the flow control for the channel
-that was just added/removed.  This is based on how the low level library
-shares devices amongst channels.
-
-The API `spdk_idxd_reconfigure_chan` had the `num_channels` removed as this
-is now tracked in the library.  The app makes use the new API above to
-determine whether to rebalance or not. This applies to `spdk_idxd_configure_chan`
-as well.
-
-The API `spdk_idxd_put_channel` now returns the rebalance state for the
-underlying device.
-
-### iscsi
-
-A security vulnerability has been identified and fixed in the SPDK iSCSI target.
-A TEXT PDU with no data, but CONTINUE flag set, would result in a NULL pointer dereference
-and crash the SPDK iSCSI target process. All users of the SPDK iSCSI target
-are recommended to update. All SPDK versions <= v21.01 are affected.
-
-### net
-
-The net library is deprecated and will be removed in the 21.07 release.
-
-### nvme
-
-Added a new function `spdk_nvme_ctrlr_get_regs_pmrcap` to get the PMR capabilities.
-
-Added an accelerated table pointer in `spdk_nvme_poll_group`
-which can be used to provide the accelerated functions by users with
-hardware engine, such as crc32c accelerated function.
-
-Added new functions `spdk_nvme_ctrlr_get_pmrsz`, `spdk_nvme_ctrlr_enable_pmr`,
-`spdk_nvme_ctrlr_disable_pmr`, `spdk_nvme_ctrlr_map_pmr` and `spdk_nvme_ctrlr_unmap_pmr`.
-
-Added NVMe transport operations to enable, disable, map and unmap the PMR.
-
-Added `spdk_nvme_qpair_get_optimal_poll_group` function and `qpair_get_optimal_poll_group`
-function pointer in spdk_nvmf_transport_ops structure in order to add the qpair to the most
-suitable polling group.
-
-Added OPTPERF and namespace optimal performance fields to nvme_spec.h.
-
-Added `spdk_nvme_set_hotplug_filter` API to allow applications to choose which
-hot-inserted SSDs should be probed. This is useful for use cases where multiple
-independent SPDK processes are running on one node. The filter function can
-then be implemented in these processes to decide which SSDs to probe based on
-the new SSD's PCI address.
-
-New functions `spdk_nvme_poll_group_get_stats` and `spdk_nvme_poll_group_free_stats`
-were added. These functions allow to get transport statistics per NVME poll group.
-
-Added `spdk_nvme_map_cmd` API to map the NVMe command with SGL cases.
-
-Added support for vector variant of ZNS zone append commands with new API
-`spdk_nvme_zns_zone_appendv` and `spdk_nvme_zns_zone_appendv_with_md`.
-
-Added `spdk_nvme_zns_ns_get_max_open_zones` and `spdk_nvme_zns_ns_get_max_active_zones` API,
-to display maximum number of open and active zones of the given namespace.
-
-Added `spdk_nvme_zns_ns_get_zone_size_sectors` API to provide size of zone in number of
-sectors.
-
-Added `spdk_nvme_qpair_get_id` API to display the ID of the specified qpair.
-
-### nvmf
-
-Removed the `spdk_nvmf_tgt_listen` and `spdk_nvmf_subsystem_add_ns` API.
-
-Added new APIs:
-
-- `spdk_nvmf_poll_group_dump_stat` (function in `nvmf.h`).
-- `poll_group_dump_stat` (transport op in `nvmf_transport.h`).
-
-The following APIs have been deprecated and will be removed in SPDK 21.07:
-
-- `spdk_nvmf_poll_group_get_stat` (function in `nvmf.h`),
-- `spdk_nvmf_transport_poll_group_get_stat` (function in `nvmf.h`),
-- `spdk_nvmf_transport_poll_group_free_stat`(function in `nvmf.h`),
-- `spdk_nvmf_rdma_device_stat` (struct in `nvmf.h`),
-- `spdk_nvmf_transport_poll_group_stat` (struct in `nvmf.h`),
-- `poll_group_get_stat` (transport op in `nvmf_transport.h`),
-- `poll_group_free_stat` (transport op in `nvmf_transport.h`).
-
-See header files for details.
-
-The `trtype` field in JSON returned by `nvmf_get_stats` RPC contains now the name of the transport,
-which is the same as the type for defined transports and more informative for a custom transport.
-
-Added `hdgst` and `ddgst` parameters to `bdev_nvme_attach_controller` RPC in order change
-state of TCP header and data digest.
-
-Added `num_cqe` parameter to `nvmf_create_transport` RPC to set number of completion queues (CQ)
-for RDMA transport. Useful when CQ resize operation is not supported, for example iWARP.
-
-### ocf
-
-Updated OCF submodule to v20.12.2
-
-Added `bdev_ocf_set_cache_mode` RPC to dynamically switch cache mode of OCF bdev.
-
-### opal
-
-Removed the `spdk_opal_supported` API.
-
-### raid
-
-For `bdev_raid_create` RPC, the deprecated parameter `strip_size` was removed.
-
-### rpc
-
-New RPC `bdev_nvme_get_transport_statistics` was added, it allows to get transport statistics
-of nvme poll groups.
-
-Parameter `enable-zerocopy-send` of RPC `sock_impl_set_options` is deprecated and will be removed in SPDK 21.07,
-use `enable-zerocopy-send-server` or `enable-zerocopy-send-client` instead.
-Parameter `disable-zerocopy-send` of RPC `sock_impl_set_options` is deprecated and will be removed in SPDK 21.07,
-use `disable-zerocopy-send-server` or `disable-zerocopy-send-client` instead.
-
-### rpm
-
-Added support for new RPM spec, rpmbuild/spdk.spec, which can be used for packaging the
-SPDK. The pkg/spdk.spec is considered to be deprecated and scheduled for removal in SPDK 21.07.
-See [RPM documentation](https://spdk.io/doc/rpm.html) for more details.
-
-### sock
-
-The type of `enable_placement_id` in struct `spdk_sock_impl_opts` is changed from
-bool to int. We can use RPC to configure different value of `enable_placement_id`.
-Then we can leverage SO_INCOMING_CPU to get placement_id, which aims to utilize
-CPU cache locality, enabled by setting enable_placement_id=2.
-
-A new socket placement mode called PLACEMENT_MARK has been added. Some NICs allow
-sockets to be marked using the SO_MARK socket option as a hint for which hardware
-queue they should be associated with. This mode leverages that by setting the same
-value for all sockets within a poll group.
-
-New parameters `enable_zerocopy_send_server` and `enable_zerocopy_send_client` were added
-to struct spdk_sock_impl_opts, these parameters enable or disable zero copy send for server
-and client sockets which are created using `spdk_sock_listen` and `spdk_sock_listen_ext` (server);
-`spdk_sock_connect` and `spdk_sock_connect_ext` (client) functions. Existing parameter
-`enable_zerocopy_send`  enables or disables zero copy send for both server and client sockets.
-
-### thread
-
-A new API `spdk_io_channel_get_io_device` was added to get the io_device for the specified
-I/O channel.
-
-Added `spdk_thread_set_interrupt_mode` function in order to set present spdk_thread into
-interrupt mode or back to poll mode. It is valid only when thread interrupt facility is
-enabled by `spdk_interrupt_mode_enable`.
-
-Added `spdk_poller_register_interrupt` function to mark that the poller is capable of
-entering interrupt mode. Callback function will be called when the poller must transition
-into or out of interrupt mode.
-
-### virtio
-
-Added the `bdev_virtio_blk_set_hotplug` RPC for the virtio blk pci device.
-
-## v21.01:
-
-### bdev
-
-An `opts_size` element was added in the `spdk_bdev_opts` structure to solve the
-ABI compatibility issue between different SPDK version. And also add `opts_size`
-parameter in spdk_bdev_get_opts function. Two fields `small_buf_pool_size` and
-`large_buf_pool_size` were added into spdk_bdev_opts, which were used to determine
-the small and large buffer pool size of the whole bdev module.
-
-A new API `spdk_bdev_wait_for_examine` was added to allow for checking state of
-examine process. Along with corresponding `bdev_wait_for_examine` RPC, which
-is now always called during `spdk_bdev_subsystem_config_json` making sure
-every bdev is ready to be used.
-
-A new API `spdk_bdev_io_get_aio_status` was added for getting the status of
-bdev_io as Linux AIO errno. Also `spdk_bdev_io_complete_aio_status` function
-and `SPDK_BDEV_IO_STATUS_AIO_ERROR` were added for bdev module to complete
-a bdev_io with Linux AIO errno.
-
-A new API `spdk_bdev_get_module_name` was added for getting block device module name.
-Also `spdk_bdev_get_module_ctx` was added to obtain a bdev module context for
-the block device opened by the specified descriptor.
-
-Added `max_segment_size` and `max_num_segments` to `spdk_bdev` structure for
-bdev modules to specify splitting requirements.
-
-### blob
-
-An `opts_size` element was added in the `spdk_bs_opts` structure to solve the
-ABI compatibility issue between different SPDK version. And also add `opts_size`
-parameter in `spdk_bs_opts_init` function.
-
-An `opts_size` element was added in the `spdk_blob_opts` structure to solve the
-ABI compatibility issue between different SPDK version. And also add `opts_size`
-parameter in `spdk_blob_opts_init` function.
-
-An `opts_size` element was added in the `spdk_blob_open_opts` structure to solve the
-ABI compatibility issue between different SPDK version. And also add `opts_size`
-parameter in `spdk_blob_open_opts_init` function.
-
-### build
-
-SPDK now generates pkg-config files to simplify the process of determining which
-libraries must be linked into an SPDK application.
+## v20.04.1:
 
 ### dpdk
 
-Updated DPDK submodule to DPDK 20.11.
+Updated DPDK submodule to DPDK 19.11.2, which includes fixes for DPDK vulnerabilities:
+CVE-2020-10722, CVE-2020-10723, CVE-2020-10724, CVE-2020-10725, CVE-2020-10724.
 
-Removed `--with-igb-uio-driver` configure option. Since DPDK 20.11 igb_uio driver
-was moved to separate dpdk-kmods repository. If required, users need to refer to
-dpdk-kmods repository to build the driver.
-
-### env
-
-The `pci_whitelist`, `pci_blacklist` and `master_core` members of struct `spdk_env_opts`
-have been deprecated.  The new members are named `pci_allowed`, `pci_blocked` and
-`main_core` respectively.
-
-An `opts_size` element was added in the `spdk_app_opts` structure
-to solve the ABI compatibility issue between different SPDK version. An `opts_size`
-parameter is added into `spdk_app_opts_init` function.
-
-### event
-
-The `pci_whitelist` and `pci_blacklist` members of struct `spdk_app_opts` have been
-deprecated.  The new members are named `pci_allowed` and `pci_blocked` respectively.
-
-The `--pci-blacklist` command line option has been deprecated, replaced with
-`--pci-blocked`.
-
-The `--pci-whitelist/-W` command line options have been deprecated, replaced with
-`--pci-allowed/-A`.
-
-Added new experimental `dynamic` scheduler that rebalances idle threads, adjusts CPU frequency
-using dpdk_governor and turns idle reactor cores to interrupt mode. Please see
-[scheduler documentation](https://www.spdk.io/doc/scheduler.html) for details.
-
-## ioat
-
-The PCI BDF whitelist option has been removed from the `ioat_scan_accel_engine` RPC.
-IOAT PCI functions can still be allowed or denied using SPDK application command
-line options.
-
-### isa-l
-
-Updated ISA-L submodule to v2.30.0.
-
-### json
-
-A new API `spdk_jsonrpc_send_bool_response` was added to allow sending response for
-writing json bool results into one function.
-Update API `bdev_nvme_set_options` and add a `keep_alive_timeout_ms` parameter. Now you
-can specify the `keep_alive_timeout` before creating NVMe bdev.
-
-### nbd
-
-Change the return type of function `spdk_nbd_stop` from void to int. And update the
-`spdk_nbd_fini` with two parameters to make its behavior from sync to async.
-
-### nvme
-
-Directives support was added to the NVMe driver.
-
-Two async APIs `spdk_nvme_ctrlr_cmd_directive_receive` and `spdk_nvme_ctrlr_cmd_directive_send`
-are added for Directive Send and Directive Receive command, respectively.
-
-Added a new function `spdk_nvme_ctrlr_reset_subsystem` to perform a NVMe
-subsystem reset. Note: The NVMe-oF target does not support the subsystem reset yet.
-
-Add a new function `spdk_nvme_bytes_to_numd` to transfer bytes to number of
-dwords.
-
-Added a new custom transport `SPDK_NVME_TRANSPORT_VFIOUSER` to enable NVMe
-driver running with NVMe over vfio-user target.
-
-Added the vfio-user custom transport implementation in NVMe driver which can connect
-to NVMe over vfio-user target via vfio-user transport.
-
-Added a new function `spdk_nvme_ctrlr_set_remove_cb` to remove or override
-`remove_cb` and `remove_ctx` specified when the controller was probed.
-
-Added support for ZNS zone append command with new API `spdk_nvme_zns_zone_append` and
-`spdk_nvme_zns_zone_append_with_md`.
-
-### nvmf
-
-`nvmf_fc_lld_fini` now takes callback and hence updating FC Broadcom LLD driver
-to the latest is required.
-
-FC transport supported primary tagging and VMID.
-
-Broadcom FC LLD driver and SPDK NVMe-oF FC transport consolidated one LLD API,
-`nvmf_fc_init_poller_queues` into another LLD API `nvmf_fc_init_q`.
-Hence updating Broadcom FC LLD driver to the latest is required.
-
-The functions `destroy` and `qpair_fini` in the transport interface now accept a
-`cb_fn` and `cb_arg` to call upon completion, and their execution can be asynchronous.
-
-The SPDK NVMe-oF target now supports async event notification for discovery log changes.
-This allows the initiator to create persistent connection to discovery controller and
-be notified of any discovery log changes.
-
-An `opts_size` element was added in the `spdk_nvmf_transport_opts` structure
-to solve the ABI compatiblity issue between different SPDK version. And also add
-`opts_size` parameter in `spdk_nvmf_transport_opts_init` function.
-
-Added a new custom vfio-user transport implementation in NVMe-oF which can provide
-emulated NVMe devices to QEMU and SPDK NVMe driver.
-
-Added new API `spdk_nvmf_tgt_listen_ext` that uses `spdk_nvmf_listen_opts` to allow
-passing generic options to specific transport. This functionality replaces one provided
-with `nvmf_subsystem_set_options`. Also removed `nvmf_subsystem_set_options` RPC
-that allowed transport specific options within subsystem.
-
-The `trsvcid` in `nvmf_subsystem_add_listener` RPC is now optional.
-
-Pausing a subsystem now only pauses all admin queues. This allows for the
-management of hosts and listeners, plus the addition of namespaces without a
-full subsystem pause. Additionally, the target now allows for pausing
-individual namespaces within a subsystem. To remove a namespace from a
-subsystem, only the specific namespace must be paused. I/O will continue to
-other namespaces while these operations execute.
-
-### rpc
-
-An new optional parameter `wait` was added to the RPC `iscsi_create_portal_group`,
-and an new RPC `iscsi_start_portal_group` was added. They will be used not to start
-listening on portals for a portal group until all associated target nodes are created
-at startup, otherwise some iSCSI initiators may fail to re-login when SPDK iSCSI
-target application restarts.
-
-Two optional parameter `--small-buf-pool-size` and `--large-buf-pool-size` were added
-into `bdev_set_options` function.
-
-### vhost
-
-Added optional `packed_ring_recovery` parameter to `vhost_create_blk_controller` RPC
-enabling packed ring live recovery.
-
-## v20.10:
-
-### accel
-
-New APIs `spdk_accel_batch_cancel` and `spdk_idxd_batch_cancel` were added to allow
-cancelling a batch sequence.
-
-### bdev
-
-A new `spdk_bdev_part_base_construct_ext` function has been added and the
-`spdk_bdev_part_base_construct` has been deprecated.  The
-`spdk_bdev_part_base_construct_ext` function takes bdev name as an argument instead
-of bdev structure to avoid a race condition that can happen when the bdev is being
-removed between a call to get its structure based on a name and actually opening it.
-
-Removed `spdk_bdev_config_text` function for bdev modules to report legacy config.
-
-### blobstore
-
-A new `spdk_bdev_create_bs_dev_ext` function has been added and `spdk_bdev_create_bs_dev_from_desc`
-function has been deprecated.
-
-### dpdk
-
-Updated DPDK submodule to DPDK 20.08.
-
-### env
-
-A new API `spdk_mem_get_fd_and_offset` was added to get file descriptor and offset for
-a given virtual address.
-
-### event
-
-Removed `spdk_subsystem_config` callback for submodules as part of legacy config removal.
-
-Removed `spdk_app_get_running_config` function that printed configuration in legacy format,
-and removed `usr1_handler` from `struct spdk_app_opts` callback that was used to call it.
-
-Added SPDK thread scheduler framework which is used to rebalance load among CPU cores.
-The scheduler implementation is pluggable and two schedulers are added first, `static` scheduler
-and `gscheduler` scheduler.
-
-`static` scheduler is used by default and does not reschedule threads and keeps the initial
-assignments.
-
-CPU frequency governor framework and implementation are also added.
-`dpdk_governor` is the first implementation and is based on rte_power library of DPDK.
-`gscheduler` scheduler utilizes the CPU frequency governor.
-
-Scheduler and governor frameworks are experimental features.
-
-### fio
-
-Bdev fio_plugin no longer supports legacy configuration files. Options `spdk_conf` and
-`spdk_json_conf` have to point to valid JSON configuration.
-
-### gpt
-
-Removed option to disable GPT probing via legacy configuration. Please use generic
-bdev functionality to replicate the behavior. See `bdev_set_options --disable-auto-examine`
-and `bdev_examine` RPC.
-
-### intel-ipsec-mb
-
-Updated intel-ipsec-mb submodule to v0.54
-
-### isa-l
-
-Updated ISA-L submodule to v2.29.0.
-
-### iscsi
-
-The iSCSI target supports iSCSI login redirection feature to achieve fault tolerance
-and load balancing particularly in scale out iSCSI target system, which runs multiple
-SPDK iSCSI targets.
-
-### json
-
-A new API `spdk_json_decode_object_relaxed` was added to allow parsing json values
-even if decoder for a given key is not found.
-
-A new API `spdk_json_free_object` was added to free memory allocated by `spdk_json_decode_object`.
-
-### log
-
-A log flag structure, `spdk_log_register_flag`, `spdk_log_get_first_flag`,
-`spdk_log_get_next_flag` and macro `SPDK_LOG_REGISTER_COMPONENT`, have been added which
-enable registering and interacting with log flags. Added `SPDK_INFOLOG`, `SPDK_DEBUGLOG`
-and `SPDK_LOGDUMP` macros to display logs for specified log flag.
-
-The log flags in the event framework `-L` and several SPDK applications has been enabled on
-release builds of SPDK. On debug builds this option will additionally set
-log print level to `SPDK_LOG_DEBUG`.
-
-A new API `spdk_log_enable_timestamps` and new RPC `log_enable_timestamps` was added to
-toggle log timestamps.
-
-### miscellaneous
-
-The contents of the log_rpc library have been moved to the event library. The log_rpc
-library now no longer exists.
-
-The contents of the app_rpc library have been moved to the event library. The app_rpc
-library now no longer exists.
-
-The contents of the bdev_rpc library have been moved to the bdev library. The app_rpc
-library now no longer exists.
-
-The bdevperf application now disables the zcopy API by default. Prior to this change,
-bdevperf enabled using the zcopy API by default which caused a performance impact of
-up to 25% on bdevs that don't natively support zcopy because the API emulates zero-copy
-by allocating a buffer. The bdevperf `-x` param was renamed to `-Z` and the default
-value changed to false. For bdevs that support zcopy, use the -Z flag to enable
-using zcopy API.
-
-### net
-
-Net framework intended for use by user-space TCP stacks was removed.
-
-### nvme
-
-Zoned Namespace Command Set support has been added to NVMe driver, see `nvme_zns.h` for details.
-
-New APIs, `spdk_nvme_detach_async` and `spdk_nvme_detach_poll_async`, have been added to
-detach multiple controllers in parallel to mitigate lengthy shutdown notification of
-a few NVMe SSDs.
-
-New APIs, `spdk_nvme_ns_get_ana_group_id` and `spdk_nvme_ns_get_ana_state`, have been
-added to get ANA group ID and ANA state of the given namespace, respectively.
-
-If no specific command set is requested (by setting the command_set member in the
-`spdk_nvme_ctrlr_opts` structure), SPDK will automatically select the most appropriate
-command set based on what the controller supports.
-
-### nvmf
-
-A new API `spdk_nvmf_subsystem_add_ns_ext` has been added and the API `spdk_nvmf_subsystem_add_ns`
-has been deprecated.
-
-The NVMe-oF target now supports Asymmetric Namespace Access (ANA) Reporting to provide
-multipath to NVMe-oF initiator.
-
-Add `no_wr_batching` parameter to `spdk_nvmf_transport_opts` struct to disable
-Work Requests batching in RDMA transport.
-
-NVMf Target transports can now parse any additional JSON params in the nvmf_create_transport RPC
-through the JSON context provided via spdk_nvmf_target_opts->transport_specific.
-
-A new RPC `nvmf_subsystem_set_options` was added to allow transport specific options within subsystem.
-
-A new API `spdk_nvmf_transport_stop_listen_async` was added to stop accepting new connections
-at the provided address. `nvmf_subsystem_remove_listener` RPC now uses this function.
-
-A new API `spdk_nvmf_subsystem_disconnect_host` was added to disconnect all connections
-originating from the provided hostnqn. `nvmf_subsystem_remove_host` RPC now uses this function.
-
-### ocf
-
-Updated OCF submodule to v20.03.1
-
-### rpc
-
-New optional parameters, `enable_placement_id` and `enable_quickack` were added to the
-`sock_impl_set_options` RPC.
-
-A new RPC `bdev_examine_bdev` was added to allow users to examine a bdev explicitly.
-It can be used only if bdev_auto_examine is set to false by the RPC `bdev_set_options`.
-
-New optional parameters `no_wr_batching` and `control_msg_num` were added to the RPC
-'nvmf_create_transport'.
-
-New RPCs, `iscsi_target_node_set_redirect` and `iscsi_target_node_request_logout`, have
-been added, and a new optional parameter `private` has been added to the RPC
-`iscsi_create_portal_group` to support iSCSI login redirection feature.
-
-A new optional parameter `ana_reporting` was added to the RPC `nvmf_create_subsystem`, and
-a new RPC `nvmf_subsystem_listener_set_ana_state` was added for ANA reporting.
-
-New RPCs, `nvmf_subsystem_get_listeners` and `nvmf_subsystem_get_qpairs`, were added to
-retrieve configuration of the NVMe-oF subsystem.
-
-### scsi
-
-Two new APIs have been added `spdk_scsi_dev_construct_ext` and
-`spdk_scsi_dev_add_lun_ext` that allow the upper layer(e.g. vhost-scsi) to
-receive the notification when the scsi bdev has been resized.
-
-The `spdk_scsi_dev_construct` and `spdk_scsi_dev_add_lun` eventually may be
-deprecated and removed.
-
-### sock
-
-The `enable_placement_id` field was added in the `struct spdk_sock_impl_opts` to
-make the placement_id feature configurable by users. The default setting is
-not enabled.
-
-The `enable_quick_ack` field was added in the `struct spdk_sock_impl_opts` to enable
-or disable quick ack for the POSIX sock module. The default setting is not enabled.
-
-### thread
-
-`fd_group` is applied to support interrupt mode.
-New APIs were added to support an experimental interrupt mode.  This allows modules or
-libraries to selectively register file descriptors that the spdk_thread can wait on,
-as an alternative to polling. In v20.10, this functionality is enabled in a very small
-subset of SPDK libraries and modules.
-
-### util
-
-A new utility named `fd_group` was add. It is now
-implemented by epoll on Linux platform. It can be used by
-spdk_thread and reactor to implement interrupt mode.
-
-Added `spdk_bit_pool` utility, see `bit_pool.h` for details.
-
-### vhost
-
-SPDK has switched to DPDK's rte_vhost library since 19.07 release, removed the internal
-rte_vhost library which is used for DPDK older than 19.05, removed the experimental vhost
-nvme target which depends on the internal rte_vhost library.
-
-### vpp
-
-Removed socket abstraction for VPP.
-
-## v20.07: SPDK CSI driver, new accel_fw commands, I/O abort support
-
-spdk_json_free_object() was added to free memory allocated by spdk_json_decode_object().
-
-### accel
-
-A new API was added `spdk_accel_get_capabilities` that allows applications to
-query the capabilities of the currently enabled accel engine back-end.
-
-A new capability, CRC-32C, was added via `spdk_accel_submit_crc32c`.
-
-The software accel engine implemenation has added support for CRC-32C.
-
-A new capability, compare, was added via `spdk_accel_submit_compare`.
-
-The software accel engine implemenation has added support for compare.
-
-Several APIs were added to `accel_engine.h` to support batched submission
-of operations.
-
-Several APIs were added to `accel_engine.h` to support dualcast operations.
-
-### accel_fw
-
-The accel_fw was updated to no longer require the app to allocate an
-accel_task on its behalf. All public APIs now take a callback arg as
-the parameter that used to be the accel_task.
-
-The accel_fw API `spdk_accel_task_size` was removed as it is no longer
-required.
-
-The accel_fw was updated to support compare, dualcast, crc32c.
-
-The accel_fw introduced batching support for all commands in all plug-ins.
-See docs for detailed information.
-
-### bdev
-
-A new API `spdk_bdev_abort` has been added to submit abort requests to abort all I/Os
-whose callback context match to the bdev on the given channel.
-
-### build
-
-The fio plugins now compile to `build/fio` and are named `spdk_bdev` and `spdk_nvme`.
-Existing fio configuration files will need to be updated.
-
-### dpdk
-
-Updated DPDK submodule to DPDK 20.05.
-
-### env
-
-Several new APIs have been added to provide greater flexibility in registering and
-accessing polled mode PCI drivers. See `env.h` for more details.
-
-### idxd
-
-The idxd library and plug-in module for the accel_fw were updated to support
-all accel_fw commands as well as batching.  Batching is supported both
-through the library and the plug-in module.
-
-IDXD engine support for CRC-32C has been added.
-
-### ioat
-
-A new API `spdk_ioat_get_max_descriptors` was added.
-
-### nvme
-
-An `opts_size`element was added in the  `spdk_nvme_ctrlr_opts` structure
-to solve the ABI compatiblity issue between different SPDK version.
-
-A new API `spdk_nvme_ctrlr_cmd_abort_ext` has been added to abort previously submitted
-commands whose callback argument match.
-
-Convenience functions, `spdk_nvme_print_command` and `spdk_nvme-print_completion` were added
-to the public API.
-
-A new function, `spdk_nvmf_cuse_update_namespaces`, updates the cuse representation of an NVMe
-controller.
-
-A new function `qpair_iterate_requests` has been added to the nvme transport interface. ALl
-implementations of the transport interface will have to implement that function.
-
-### nvmf
-
-The NVMe-oF target no longer supports connecting scheduling configuration and instead
-always uses what was previously called "transport" scheduling.
-
-`spdk_nvmf_tgt_accept` no longer exists. The accept process now occurs automatically after
-the creation of an nvmf target and queue pairs are assigned to poll groups by the underlying
-transport. Further, `spdk_nvmf_transport_ops` has changed such that the accept function
-pointer no longer takes a function pointer as an argument. Instead, transports should call
-`spdk_nvmf_tgt_new_qpair` whenever they previously would have called that callback.
-
-The NVMe-oF target now supports aborting any submitted NVM or Admin command. Previously,
-the NVMe-oF target could abort only Asynchronous Event Request commands.
-
-### rdma
-
-A new `rdma` library has been added. It is an abstraction layer over different RDMA providers.
-Two providers are available - verbs (used by default when RDMA is enabled or enabled explicitly
-using --with-rdma=verbs) and mlx5 Direct Verbs aka DV (enabled by --with-rdma=mlx5_dv).
-Using mlx5_dv requires libmlx5 installed on the system.
-
-### rpc
-
-Parameter `-p` or `--max-qpairs-per-ctrlr` of `nvmf_create_transport` RPC command accepted by the
-rpc.py script is deprecated, new parameter `-m` or `--max-io-qpairs-per-ctrlr` was added.
-
-Added `sock_impl_get_options` and `sock_impl_set_options` RPC methods.
-
-Command line parameters `-r` and `--rpc-socket` will longer accept TCP ports. RPC server
-must now be started on a Unix domain socket. Exposing RPC on the network, as well as providing
-proper authentication (if needed) is now a responsibility of the user.
-
-The `bdev_set_options` RPC has a new option, `bdev_auto_examine` to control the auto examine function
-of bdev modules.
-
-New RPCs `sock_impl_get_options` and `sock_impl_set_options` been added to expose new socket features.
-See `sock` section for more details.
-
-### sock
-
-Added `spdk_sock_impl_get_opts` and `spdk_sock_impl_set_opts` functions to set/get socket layer configuration
-options. Options can be set independently for each implementation.
-
-Added `recv_buf_size` and 'send_buf_size' socket layer options. They are used only in posix implementation.
-
-Added `uring` based socket implementation, the code is located in module/sock/uring. This feature is only
-available in Linux which requires kernel version is greater than 5.4.3. Currently, our CI pool added the uring
-based socket tests for iSCSI target and also the tests for SPDK NVMe-oF tcp transport.
-
-Added `enable_recv_pipe` socket layer option to allow disabling of double buffering on receive.
-New option is used only in posix implementation.
-
-Added `enable_zerocopy_send` socket layer option to allow disabling of zero copy flow on send.
-New option is used only in posix implementation.
-
-### util
-
-Some previously exposed CRC32 functions have been removed from the public API -
-`spdk_crc32_update`, `spdk_crc32_table_init`, and the `spdk_crc32_table` struct.
-
-### vhost
-
-The function `spdk_vhost_blk_get_dev` has been removed.
-
-## v20.04: SPDK Top, IDXD, NVMe qpair groups
-
-IDXD engine support for compare has been added.
+## v20.04:
 
 ### configuration
 
@@ -966,7 +210,7 @@ Poll groups per session have been replaced by SPDK threads per vhost controller.
 A new function, `spdk_vmd_fini`, has been added. It releases all resources acquired by the VMD
 library through the `spdk_vmd_init` call.
 
-## v20.01: Optimized thin provisioning, FTL bdev, VMD hot plug, FUSED support
+## v20.01
 
 ### bdev
 
@@ -1156,7 +400,7 @@ code for fused compare-and-write operation.
 
 Added spdk_bdev_get_acwu function for getting block device atomic compare and write unit size.
 
-## v19.10: Zoned bdev API, Opal bdev, NVMe character devices
+## v19.10
 
 ### rpc
 
@@ -1386,7 +630,7 @@ New cache modes added to use via RPC, wi - write invalidate and wa - write aroun
 
 New version of OCF provides fully asynchronous management API.
 
-## v19.07: NVMe-oF FC Transport, VMD, NVMe-oF Persistent reservations, Bdev I/O with separate metadata
+## v19.07
 
 ### ftl
 
@@ -1662,7 +906,7 @@ with SPDK thread when necessary.
 Added spdk_thread_destroy() to allow framework polling the thread to
 release resources associated with that thread.
 
-## v19.04: Compression bdev, Notification library, NVMe Opal support
+## v19.04
 
 ### nvme
 
@@ -1883,7 +1127,7 @@ Added "reduce" block compression scheme based on using SSDs for storing
 compressed blocks of storage and presistent memory for metadata. Please see
 [compression](https://spdk.io/doc/bdev.html) for more details.
 
-## v19.01: NVMe-oF TCP/IP Transport, Open Channel SSD Flash Translation Layer, Caching bdev based on OCF, ISA-L Support, DIF/DIX library
+## v19.01
 
 ### ocf bdev
 
@@ -2138,7 +1382,7 @@ JSON RPC client is now running in non-blocking mode. Requests are sent and recei
 JSON RPC server can now recieve a callback on connection termination or server shutdown using `spdk_jsonrpc_conn_add_close_cb`
 and `spdk_jsonrpc_conn_del_close_cb`.
 
-## v18.10: Dynamic memory allocation, Crypto Virtual Bdev, jsonrpc-client, SPDKCLI iSCSI and NVMe-oF support
+## v18.10
 
 ### nvme
 
@@ -2335,7 +1579,7 @@ in QEMU.
 The SPDKCLI interactive command tool for managing SPDK is no longer considered experimental.
 Support for the iSCSI and NVMe-oF targets has been added.
 
-## v18.07: Raid, Infrastructure Improvements, Bug Fixes
+## v18.07
 
 ### bdev
 
@@ -2639,7 +1883,7 @@ may be set via the configure `--prefix` option.  Example: `make install prefix=/
 
 ### RPC
 
-A JSON RPC listener is now enabled by default using a UNIX domain socket at /var/tmp/spdk.sock.
+A JSON RPC listener is now enabled by default using a UNIX domain socket at /var/run/spdk.sock.
 A -r option command line option has been added to enable an alternative UNIX domain socket location,
 or a TCP port in the format ip_addr:tcp_port (i.e. 127.0.0.1:5260).  The Rpc configuration file
 section is now deprecated and will be removed in the v18.04 release.

CONFIG

@@ -43,6 +43,9 @@ CONFIG_CROSS_PREFIX=
 # Build with debug logging. Turn off for performance testing and normal usage
 CONFIG_DEBUG=n
 
+# Build with support of backtrace printing in log messages. Requires libunwind.
+CONFIG_LOG_BACKTRACE=n
+
 # Treat warnings as errors (fail the build on any warning).
 CONFIG_WERROR=n
 
@@ -76,9 +79,6 @@ CONFIG_UNIT_TESTS=y
 # Build examples
 CONFIG_EXAMPLES=y
 
-# Build with Control-flow Enforcement Technology (CET)
-CONFIG_CET=n
-
 # Directory that contains the desired SPDK environment library.
 # By default, this is implemented using DPDK.
 CONFIG_ENV=
@@ -87,9 +87,6 @@ CONFIG_ENV=
 # installation.
 CONFIG_DPDK_DIR=
 
-# This directory should contain 'include' and 'lib' directories for WPDK.
-CONFIG_WPDK_DIR=
-
 # Build SPDK FIO plugin. Requires CONFIG_FIO_SOURCE_DIR set to a valid
 # fio source code directory.
 CONFIG_FIO_PLUGIN=n
@@ -103,7 +100,6 @@ CONFIG_FIO_SOURCE_DIR=/usr/src/fio
 CONFIG_RDMA=n
 CONFIG_RDMA_SEND_WITH_INVAL=n
 CONFIG_RDMA_SET_ACK_TIMEOUT=n
-CONFIG_RDMA_PROV=verbs
 
 # Enable NVMe Character Devices.
 CONFIG_NVME_CUSE=n
@@ -119,14 +115,11 @@ CONFIG_RBD=n
 
 # Build vhost library.
 CONFIG_VHOST=y
+CONFIG_VHOST_INTERNAL_LIB=n
 
 # Build vhost initiator (Virtio) driver.
 CONFIG_VIRTIO=y
 
-# Build custom vfio-user transport for NVMf target and NVMe initiator.
-CONFIG_VFIO_USER=n
-CONFIG_VFIO_USER_DIR=
-
 # Build with PMDK backends
 CONFIG_PMDK=n
 CONFIG_PMDK_DIR=
@@ -134,6 +127,10 @@ CONFIG_PMDK_DIR=
 # Enable the dependencies for building the compress vbdev
 CONFIG_REDUCE=n
 
+# Build with VPP
+CONFIG_VPP=n
+CONFIG_VPP_DIR=
+
 # Requires libiscsi development libraries.
 CONFIG_ISCSI_INITIATOR=n
 
@@ -147,6 +144,9 @@ CONFIG_SHARED=n
 CONFIG_VTUNE=n
 CONFIG_VTUNE_DIR=
 
+# Build the dpdk igb_uio driver
+CONFIG_IGB_UIO_DRIVER=n
+
 # Build Intel IPSEC_MB library
 CONFIG_IPSEC_MB=n
 

LICENSE

@@ -1,16 +1,3 @@
-The SPDK repo contains multiple git submodules each with its own
-license info. Unless otherwise noted all other code in this repo
-is BSD as stated below.
-
-Submodule license info:
-dpdk: see dpdk/license
-intel-ipsec-mb: see intel-ipsec-mb/LICENSE
-isa-l: see isa-l/LICENSE
-libvfio-user: see libvfio-user/LICENSE
-ocf: see ocf/LICENSE
-
-The rest of the SPDK repo:
-
 BSD LICENSE
 
 Copyright (c) Intel Corporation.

Makefile

@@ -2,7 +2,6 @@
 #  BSD LICENSE
 #
 #  Copyright (c) Intel Corporation.
-#  Copyright (c) 2020, Mellanox Corporation.
 #  All rights reserved.
 #
 #  Redistribution and use in source and binary forms, with or without
@@ -45,15 +44,11 @@ DIRS-$(CONFIG_EXAMPLES) += examples
 DIRS-y += test
 DIRS-$(CONFIG_IPSEC_MB) += ipsecbuild
 DIRS-$(CONFIG_ISAL) += isalbuild
-DIRS-$(CONFIG_VFIO_USER) += vfiouserbuild
 
 .PHONY: all clean $(DIRS-y) include/spdk/config.h mk/config.mk \
 	cc_version cxx_version .libs_only_other .ldflags ldflags install \
 	uninstall
 
-# Workaround for ninja. See dpdkbuild/Makefile
-export MAKE_PID := $(shell echo $$PPID)
-
 ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV))
 ifeq ($(CURDIR)/dpdk/build,$(CONFIG_DPDK_DIR))
 ifneq ($(SKIP_DPDK_BUILD),1)
@@ -63,13 +58,6 @@ endif
 endif
 endif
 
-ifeq ($(OS),Windows)
-ifeq ($(CURDIR)/wpdk/build,$(CONFIG_WPDK_DIR))
-WPDK = wpdk
-DIRS-y += wpdk
-endif
-endif
-
 ifeq ($(CONFIG_SHARED),y)
 LIB = shared_lib
 else
@@ -86,20 +74,9 @@ LIB += isalbuild
 DPDK_DEPS += isalbuild
 endif
 
-ifeq ($(CONFIG_VFIO_USER),y)
-VFIOUSERBUILD = vfiouserbuild
-LIB += vfiouserbuild
-endif
-
 all: mk/cc.mk $(DIRS-y)
 clean: $(DIRS-y)
 	$(Q)rm -f include/spdk/config.h
-	$(Q)rm -rf build/bin
-	$(Q)rm -rf build/fio
-	$(Q)rm -rf build/examples
-	$(Q)rm -rf build/include
-	$(Q)rm -rf build/lib/pkgconfig
-	$(Q)find build/lib ! -name .gitignore -type f -delete
 
 install: all
 	$(Q)echo "Installed to $(DESTDIR)$(CONFIG_PREFIX)"
@@ -108,11 +85,10 @@ uninstall: $(DIRS-y)
 	$(Q)echo "Uninstalled spdk"
 
 ifneq ($(SKIP_DPDK_BUILD),1)
-dpdkdeps $(DPDK_DEPS): $(WPDK)
-dpdkbuild: $(WPDK) $(DPDK_DEPS)
+dpdkbuild: $(DPDK_DEPS)
 endif
 
-lib: $(WPDK) $(DPDKBUILD) $(VFIOUSERBUILD)
+lib: $(DPDKBUILD)
 module: lib
 shared_lib: module
 app: $(LIB)
@@ -121,19 +97,12 @@ examples: $(LIB)
 pkgdep:
 	sh ./scripts/pkgdep.sh
 
-$(DIRS-y): mk/cc.mk build_dir include/spdk/config.h
+$(DIRS-y): include/spdk/config.h
 
 mk/cc.mk:
 	$(Q)echo "Please run configure prior to make"
 	false
 
-build_dir: mk/cc.mk
-	$(Q)mkdir -p build/lib/pkgconfig/tmp
-	$(Q)mkdir -p build/bin
-	$(Q)mkdir -p build/fio
-	$(Q)mkdir -p build/examples
-	$(Q)mkdir -p build/include/spdk
-
 include/spdk/config.h: mk/config.mk scripts/genconfig.py
 	$(Q)echo "#ifndef SPDK_CONFIG_H" > $@.tmp; \
 	echo "#define SPDK_CONFIG_H" >> $@.tmp; \

README.md

@@ -190,16 +190,13 @@ to do the following steps:
 - run ldconfig specifying the directory containing SPDK shared libraries
 - provide proper `LD_LIBRARY_PATH`
 
-If DPDK shared libraries are used, you may also need to add DPDK shared
-libraries to `LD_LIBRARY_PATH`
-
 Linux:
 
 ~~~{.sh}
 ./configure --with-shared
 make
 ldconfig -v -n ./build/lib
-LD_LIBRARY_PATH=./build/lib/:./dpdk/build/lib/ ./build/bin/spdk_tgt
+LD_LIBRARY_PATH=./build/lib/ ./app/spdk_tgt/spdk_tgt
 ~~~
 
 <a id="huge"></a>

app/Makefile

@@ -41,13 +41,9 @@ DIRS-y += iscsi_top
 DIRS-y += iscsi_tgt
 DIRS-y += spdk_tgt
 DIRS-y += spdk_lspci
-ifneq ($(OS),Windows)
-# TODO - currently disabled on Windows due to lack of support for curses
 DIRS-y += spdk_top
-endif
 ifeq ($(OS),Linux)
 DIRS-$(CONFIG_VHOST) += vhost
-DIRS-y += spdk_dd
 endif
 
 .PHONY: all clean $(DIRS-y)

app/iscsi_tgt/Makefile

@@ -43,14 +43,18 @@ CFLAGS += -I$(SPDK_ROOT_DIR)/lib
 
 C_SRCS := iscsi_tgt.c
 
-SPDK_LIB_LIST = $(ALL_MODULES_LIST) event_iscsi event_net
+SPDK_LIB_LIST = $(ALL_MODULES_LIST)
+SPDK_LIB_LIST += event_bdev event_accel event_iscsi event_net event_scsi event_vmd event
+SPDK_LIB_LIST += jsonrpc json rpc bdev_rpc bdev iscsi scsi accel trace conf
+SPDK_LIB_LIST += thread util log log_rpc app_rpc net sock notify
 
 ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV))
 SPDK_LIB_LIST += env_dpdk_rpc
 endif
 
+
 ifeq ($(OS),Linux)
-SPDK_LIB_LIST += event_nbd
+SPDK_LIB_LIST += event_nbd nbd
 endif
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

app/iscsi_tgt/iscsi_tgt.c

@@ -41,6 +41,21 @@
 
 static int g_daemon_mode = 0;
 
+static void
+spdk_sigusr1(int signo __attribute__((__unused__)))
+{
+	char *config_str = NULL;
+	if (spdk_app_get_running_config(&config_str, "iscsi.conf") < 0) {
+		fprintf(stderr, "Error getting config\n");
+	} else {
+		fprintf(stdout, "============================\n");
+		fprintf(stdout, " iSCSI target running config\n");
+		fprintf(stdout, "=============================\n");
+		fprintf(stdout, "%s", config_str);
+	}
+	free(config_str);
+}
+
 static void
 iscsi_usage(void)
 {
@@ -75,7 +90,7 @@ main(int argc, char **argv)
 	int rc;
 	struct spdk_app_opts opts = {};
 
-	spdk_app_opts_init(&opts, sizeof(opts));
+	spdk_app_opts_init(&opts);
 	opts.name = "iscsi";
 	if ((rc = spdk_app_parse_args(argc, argv, &opts, "b", NULL,
 				      iscsi_parse_arg, iscsi_usage)) !=
@@ -91,6 +106,7 @@ main(int argc, char **argv)
 	}
 
 	opts.shutdown_cb = NULL;
+	opts.usr1_handler = spdk_sigusr1;
 
 	/* Blocks until the application is exiting */
 	rc = spdk_app_start(&opts, spdk_startup, NULL);

app/iscsi_top/Makefile

@@ -33,14 +33,21 @@
 
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
-include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
+include $(SPDK_ROOT_DIR)/mk/spdk.app_cxx.mk
+
+CXXFLAGS += $(ENV_CXXFLAGS)
+CXXFLAGS += -I$(SPDK_ROOT_DIR)/lib
+CXX_SRCS = iscsi_top.cpp
 
 APP = iscsi_top
 
-SPDK_LIB_LIST = rpc
+all: $(APP)
+	@:
 
-CFLAGS += -I$(SPDK_ROOT_DIR)/lib
+$(APP) : $(OBJS)
+	$(LINK_CXX)
 
-C_SRCS := iscsi_top.c
+clean:
+	$(CLEAN_C) $(APP)
 
-include $(SPDK_ROOT_DIR)/mk/spdk.app.mk
+include $(SPDK_ROOT_DIR)/mk/spdk.deps.mk

app/iscsi_top/iscsi_top.cpp

@@ -33,106 +33,93 @@
 
 #include "spdk/stdinc.h"
 
-#include "spdk/event.h"
-#include "spdk/jsonrpc.h"
-#include "spdk/rpc.h"
-#include "spdk/string.h"
-#include "spdk/trace.h"
-#include "spdk/util.h"
+#include <algorithm>
+#include <map>
+#include <vector>
 
+extern "C" {
+#include "spdk/trace.h"
 #include "iscsi/conn.h"
+}
 
 static char *exe_name;
 static int g_shm_id = 0;
 
-struct spdk_jsonrpc_client *g_rpc_client;
-
 static void usage(void)
 {
 	fprintf(stderr, "usage:\n");
 	fprintf(stderr, "   %s <option>\n", exe_name);
 	fprintf(stderr, "        option = '-i' to specify the shared memory ID,"
 		" (required)\n");
-	fprintf(stderr, " -r <path>  RPC listen address (default: /var/tmp/spdk.sock\n");
 }
 
-struct rpc_conn_info {
-	uint32_t	id;
-	uint32_t	cid;
-	uint32_t	tsih;
-	uint32_t	lcore_id;
-	char		*initiator_addr;
-	char		*target_addr;
-	char		*target_node_name;
-};
-
-static struct rpc_conn_info g_conn_info[1024];
-
-static const struct spdk_json_object_decoder rpc_conn_info_decoders[] = {
-	{"id", offsetof(struct rpc_conn_info, id), spdk_json_decode_uint32},
-	{"cid", offsetof(struct rpc_conn_info, cid), spdk_json_decode_uint32},
-	{"tsih", offsetof(struct rpc_conn_info, tsih), spdk_json_decode_uint32},
-	{"lcore_id", offsetof(struct rpc_conn_info, lcore_id), spdk_json_decode_uint32},
-	{"initiator_addr", offsetof(struct rpc_conn_info, initiator_addr), spdk_json_decode_string},
-	{"target_addr", offsetof(struct rpc_conn_info, target_addr), spdk_json_decode_string},
-	{"target_node_name", offsetof(struct rpc_conn_info, target_node_name), spdk_json_decode_string},
-};
-
-static int
-rpc_decode_conn_object(const struct spdk_json_val *val, void *out)
+/* Group by poll group */
+static bool
+conns_compare(struct spdk_iscsi_conn *first, struct spdk_iscsi_conn *second)
 {
-	struct rpc_conn_info *info = (struct rpc_conn_info *)out;
+	if ((uintptr_t)first->pg < (uintptr_t)second->pg) {
+		return true;
+	}
 
-	return spdk_json_decode_object(val, rpc_conn_info_decoders,
-				       SPDK_COUNTOF(rpc_conn_info_decoders), info);
+	if ((uintptr_t)first->pg > (uintptr_t)second->pg) {
+		return false;
+	}
+
+	if (first->id < second->id) {
+		return true;
+	}
+
+	return false;
 }
 
 static void
 print_connections(void)
 {
-	struct spdk_jsonrpc_client_response *json_resp = NULL;
-	struct spdk_json_write_ctx *w;
-	struct spdk_jsonrpc_client_request *request;
-	int rc;
-	size_t conn_count, i;
-	struct rpc_conn_info *conn;
+	std::vector<struct spdk_iscsi_conn *>		v;
+	std::vector<struct spdk_iscsi_conn *>::iterator	iter;
+	size_t			conns_size;
+	struct spdk_iscsi_conn	*conns, *conn;
+	void			*conns_ptr;
+	int			fd, i;
+	char			shm_name[64];
 
-	request = spdk_jsonrpc_client_create_request();
-	if (request == NULL) {
-		return;
+	snprintf(shm_name, sizeof(shm_name), "/spdk_iscsi_conns.%d", g_shm_id);
+	fd = shm_open(shm_name, O_RDONLY, 0600);
+	if (fd < 0) {
+		fprintf(stderr, "Cannot open shared memory: %s\n", shm_name);
+		usage();
+		exit(1);
 	}
 
-	w = spdk_jsonrpc_begin_request(request, 1, "iscsi_get_connections");
-	spdk_jsonrpc_end_request(request, w);
-	spdk_jsonrpc_client_send_request(g_rpc_client, request);
+	conns_size = sizeof(*conns) * MAX_ISCSI_CONNECTIONS;
 
-	do {
-		rc = spdk_jsonrpc_client_poll(g_rpc_client, 1);
-	} while (rc == 0 || rc == -ENOTCONN);
-
-	if (rc <= 0) {
-		goto end;
+	conns_ptr = mmap(NULL, conns_size, PROT_READ, MAP_SHARED, fd, 0);
+	if (conns_ptr == MAP_FAILED) {
+		fprintf(stderr, "Cannot mmap shared memory (%d)\n", errno);
+		exit(1);
 	}
 
-	json_resp = spdk_jsonrpc_client_get_response(g_rpc_client);
-	if (json_resp == NULL) {
-		goto end;
+	conns = (struct spdk_iscsi_conn *)conns_ptr;
+
+	for (i = 0; i < MAX_ISCSI_CONNECTIONS; i++) {
+		if (!conns[i].is_valid) {
+			continue;
+		}
+		v.push_back(&conns[i]);
 	}
 
-	if (spdk_json_decode_array(json_resp->result, rpc_decode_conn_object, g_conn_info,
-				   SPDK_COUNTOF(g_conn_info), &conn_count, sizeof(struct rpc_conn_info))) {
-		goto end;
+	stable_sort(v.begin(), v.end(), conns_compare);
+	for (iter = v.begin(); iter != v.end(); iter++) {
+		conn = *iter;
+		printf("pg %p conn %3d T:%-8s I:%s (%s)\n",
+		       conn->pg, conn->id,
+		       conn->target_short_name, conn->initiator_name,
+		       conn->initiator_addr);
 	}
 
-	for (i = 0; i < conn_count; i++) {
-		conn = &g_conn_info[i];
-
-		printf("Connection: %u CID: %u TSIH: %u Initiator Address: %s Target Address: %s Target Node Name: %s\n",
-		       conn->id, conn->cid, conn->tsih, conn->initiator_addr, conn->target_addr, conn->target_node_name);
-	}
-
-end:
-	spdk_jsonrpc_client_free_request(request);
+	printf("\n");
+	munmap(conns, conns_size);
+	close(fd);
 }
 
 int main(int argc, char **argv)
@@ -140,7 +127,6 @@ int main(int argc, char **argv)
 	void			*history_ptr;
 	struct spdk_trace_histories *histories;
 	struct spdk_trace_history *history;
-	const char *rpc_socket_path = SPDK_DEFAULT_RPC_ADDR;
 
 	uint64_t		tasks_done, last_tasks_done[SPDK_TRACE_MAX_LCORE];
 	int			delay, old_delay, history_fd, i, quit, rc;
@@ -154,13 +140,10 @@ int main(int argc, char **argv)
 	int			op;
 
 	exe_name = argv[0];
-	while ((op = getopt(argc, argv, "i:r:")) != -1) {
+	while ((op = getopt(argc, argv, "i:")) != -1) {
 		switch (op) {
 		case 'i':
-			g_shm_id = spdk_strtol(optarg, 10);
-			break;
-		case 'r':
-			rpc_socket_path = optarg;
+			g_shm_id = atoi(optarg);
 			break;
 		default:
 			usage();
@@ -168,12 +151,6 @@ int main(int argc, char **argv)
 		}
 	}
 
-	g_rpc_client = spdk_jsonrpc_client_connect(rpc_socket_path, AF_UNIX);
-	if (!g_rpc_client) {
-		fprintf(stderr, "spdk_jsonrpc_client_connect() failed: %d\n", errno);
-		return 1;
-	}
-
 	snprintf(spdk_trace_shm_name, sizeof(spdk_trace_shm_name), "/iscsi_trace.%d", g_shm_id);
 	history_fd = shm_open(spdk_trace_shm_name, O_RDONLY, 0600);
 	if (history_fd < 0) {
@@ -271,7 +248,5 @@ cleanup:
 	munmap(history_ptr, sizeof(*histories));
 	close(history_fd);
 
-	spdk_jsonrpc_client_close(g_rpc_client);
-
 	return (0);
 }

app/nvmf_tgt/Makefile

@@ -39,14 +39,24 @@ APP = nvmf_tgt
 
 C_SRCS := nvmf_main.c
 
-SPDK_LIB_LIST = $(ALL_MODULES_LIST) event_nvmf
+SPDK_LIB_LIST = $(ALL_MODULES_LIST)
+SPDK_LIB_LIST += event_bdev event_accel event_nvmf event_net event_vmd
+SPDK_LIB_LIST += nvmf event log trace conf thread util bdev accel rpc jsonrpc json net sock
+SPDK_LIB_LIST += app_rpc log_rpc bdev_rpc notify
 
 ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV))
 SPDK_LIB_LIST += env_dpdk_rpc
 endif
 
 ifeq ($(OS),Linux)
-SPDK_LIB_LIST += event_nbd
+SPDK_LIB_LIST += event_nbd nbd
+endif
+
+ifeq ($(CONFIG_FC),y)
+ifneq ($(strip $(CONFIG_FC_PATH)),)
+SYS_LIBS += -L$(CONFIG_FC_PATH)
+endif
+SYS_LIBS += -lufc
 endif
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

app/nvmf_tgt/nvmf_main.c

@@ -63,7 +63,7 @@ main(int argc, char **argv)
 	struct spdk_app_opts opts = {};
 
 	/* default value in opts */
-	spdk_app_opts_init(&opts, sizeof(opts));
+	spdk_app_opts_init(&opts);
 	opts.name = "nvmf";
 	if ((rc = spdk_app_parse_args(argc, argv, &opts, "", NULL,
 				      nvmf_parse_arg, nvmf_usage)) !=

app/spdk_dd/.gitignore

@@ -1 +0,0 @@
-spdk_dd

app/spdk_dd/Makefile

@@ -1,44 +0,0 @@
-#
-#  BSD LICENSE
-#
-#  Copyright (c) Intel Corporation.
-#  All rights reserved.
-#
-#  Redistribution and use in source and binary forms, with or without
-#  modification, are permitted provided that the following conditions
-#  are met:
-#
-#    * Redistributions of source code must retain the above copyright
-#      notice, this list of conditions and the following disclaimer.
-#    * Redistributions in binary form must reproduce the above copyright
-#      notice, this list of conditions and the following disclaimer in
-#      the documentation and/or other materials provided with the
-#      distribution.
-#    * Neither the name of Intel Corporation nor the names of its
-#      contributors may be used to endorse or promote products derived
-#      from this software without specific prior written permission.
-#
-#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
-
-SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
-include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
-include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
-
-APP = spdk_dd
-
-C_SRCS := spdk_dd.c
-
-SPDK_LIB_LIST = $(ALL_MODULES_LIST) event_bdev
-
-include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

app/spdk_lspci/Makefile

@@ -33,12 +33,11 @@
 
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
-include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
 
 APP = spdk_lspci
 
 C_SRCS := spdk_lspci.c
 
-SPDK_LIB_LIST = $(SOCK_MODULES_LIST) nvme vmd
+SPDK_LIB_LIST = vmd log
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

app/spdk_tgt/Makefile

@@ -41,17 +41,32 @@ C_SRCS := spdk_tgt.c
 
 SPDK_LIB_LIST = $(ALL_MODULES_LIST)
 
-SPDK_LIB_LIST += event_iscsi event_nvmf
+ifeq ($(OS),Linux)
+ifeq ($(CONFIG_VHOST),y)
+SPDK_LIB_LIST += vhost event_vhost
+ifeq ($(CONFIG_VHOST_INTERNAL_LIB),y)
+SPDK_LIB_LIST += rte_vhost
+endif
+endif
+endif
+
+SPDK_LIB_LIST += event_bdev event_accel event_iscsi event_net event_scsi event_nvmf event_vmd event
+SPDK_LIB_LIST += nvmf trace log conf thread util bdev iscsi scsi accel rpc jsonrpc json
+SPDK_LIB_LIST += app_rpc log_rpc bdev_rpc net sock notify
 
 ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV))
 SPDK_LIB_LIST += env_dpdk_rpc
 endif
 
 ifeq ($(OS),Linux)
-SPDK_LIB_LIST += event_nbd
-ifeq ($(CONFIG_VHOST),y)
-SPDK_LIB_LIST += event_vhost
+SPDK_LIB_LIST += event_nbd nbd
 endif
+
+ifeq ($(CONFIG_FC),y)
+ifneq ($(strip $(CONFIG_FC_PATH)),)
+SYS_LIBS += -L$(CONFIG_FC_PATH)
+endif
+SYS_LIBS += -lufc
 endif
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

app/spdk_tgt/spdk_tgt.c

@@ -109,7 +109,7 @@ main(int argc, char **argv)
 	struct spdk_app_opts opts = {};
 	int rc;
 
-	spdk_app_opts_init(&opts, sizeof(opts));
+	spdk_app_opts_init(&opts);
 	opts.name = "spdk_tgt";
 	if ((rc = spdk_app_parse_args(argc, argv, &opts, g_spdk_tgt_get_opts_string,
 				      NULL, spdk_tgt_parse_arg, spdk_tgt_usage)) !=

app/spdk_top/Makefile

@@ -38,7 +38,7 @@ APP = spdk_top
 
 C_SRCS := spdk_top.c
 
-SPDK_LIB_LIST = rpc
+SPDK_LIB_LIST = jsonrpc json rpc log util
 LIBS=-lncurses -lpanel -lmenu
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

app/trace/Makefile

@@ -33,11 +33,19 @@
 
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
-include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
-
-APP = spdk_trace
-SPDK_NO_LINK_ENV = 1
+include $(SPDK_ROOT_DIR)/mk/spdk.app_cxx.mk
 
 CXX_SRCS := trace.cpp
 
-include $(SPDK_ROOT_DIR)/mk/spdk.app_cxx.mk
+APP = spdk_trace
+
+all: $(APP)
+	@:
+
+$(APP): $(OBJS) $(SPDK_LIBS)
+	$(LINK_CXX)
+
+clean:
+	$(CLEAN_C) $(APP)
+
+include $(SPDK_ROOT_DIR)/mk/spdk.deps.mk

app/trace_record/trace_record.c

@@ -613,8 +613,6 @@ int main(int argc, char **argv)
 			file_name = optarg;
 			break;
 		case 'h':
-			usage();
-			exit(EXIT_SUCCESS);
 		default:
 			usage();
 			exit(1);

app/vhost/Makefile

@@ -39,7 +39,17 @@ APP = vhost
 
 C_SRCS := vhost.c
 
-SPDK_LIB_LIST = $(ALL_MODULES_LIST) event_vhost event_nbd
+SPDK_LIB_LIST = $(ALL_MODULES_LIST)
+SPDK_LIB_LIST += vhost event_vhost
+
+ifeq ($(CONFIG_VHOST_INTERNAL_LIB),y)
+SPDK_LIB_LIST += rte_vhost
+endif
+
+SPDK_LIB_LIST += event_bdev event_accel event_net event_scsi event_vmd event
+SPDK_LIB_LIST += jsonrpc json rpc bdev_rpc bdev scsi accel trace conf
+SPDK_LIB_LIST += thread util log log_rpc app_rpc
+SPDK_LIB_LIST += event_nbd nbd net sock notify
 
 ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV))
 SPDK_LIB_LIST += env_dpdk_rpc

app/vhost/vhost.c

@@ -33,6 +33,7 @@
 
 #include "spdk/stdinc.h"
 
+#include "spdk/conf.h"
 #include "spdk/event.h"
 
 #include "spdk/vhost.h"
@@ -88,7 +89,7 @@ main(int argc, char *argv[])
 	struct spdk_app_opts opts = {};
 	int rc;
 
-	spdk_app_opts_init(&opts, sizeof(opts));
+	spdk_app_opts_init(&opts);
 	opts.name = "vhost";
 
 	if ((rc = spdk_app_parse_args(argc, argv, &opts, "f:S:", NULL,

autobuild.sh

@@ -12,30 +12,27 @@ rootdir=$(readlink -f $(dirname $0))
 
 source "$1"
 source "$rootdir/test/common/autotest_common.sh"
-source "$rootdir/scripts/common.sh"
 
 out=$output_dir
-if [ -n "$SPDK_TEST_NATIVE_DPDK" ]; then
-	scanbuild_exclude=" --exclude $(dirname $SPDK_RUN_EXTERNAL_DPDK)"
-else
-	scanbuild_exclude="--exclude $rootdir/dpdk/"
-fi
-scanbuild="scan-build -o $output_dir/scan-build-tmp $scanbuild_exclude --status-bugs"
+scanbuild="scan-build -o $output_dir/scan-build-tmp --status-bugs"
 config_params=$(get_config_params)
 
-trap '[[ -d $SPDK_WORKSPACE ]] && rm -rf "$SPDK_WORKSPACE"' 0
-
-SPDK_WORKSPACE=$(mktemp -dt "spdk_$(date +%s).XXXXXX")
-export SPDK_WORKSPACE
-
+rm -rf /tmp/spdk
+mkdir /tmp/spdk
 umask 022
 cd $rootdir
 
 # Print some test system info out for the log
 date -u
 git describe --tags
+./configure $config_params
+echo "** START ** Info for Hostname: $HOSTNAME"
+uname -a
+$MAKE cc_version
+$MAKE cxx_version
+echo "** END ** Info for Hostname: $HOSTNAME"
 
-function ocf_precompile() {
+function ocf_precompile {
 	# We compile OCF sources ourselves
 	# They don't need to be checked with scanbuild and code coverage is not applicable
 	# So we precompile OCF now for further use as standalone static library
@@ -48,105 +45,7 @@ function ocf_precompile() {
 	./configure $config_params
 }
 
-function build_native_dpdk() {
-	local external_dpdk_dir
-	local external_dpdk_base_dir
-
-	external_dpdk_dir="$SPDK_RUN_EXTERNAL_DPDK"
-	external_dpdk_base_dir="$(dirname $external_dpdk_dir)"
-
-	if [[ ! -d "$external_dpdk_base_dir" ]]; then
-		sudo mkdir -p "$external_dpdk_base_dir"
-		sudo chown -R $(whoami) "$external_dpdk_base_dir"/..
-	fi
-	orgdir=$PWD
-
-	rm -rf "$external_dpdk_base_dir"
-	git clone --branch $SPDK_TEST_NATIVE_DPDK --depth 1 http://dpdk.org/git/dpdk "$external_dpdk_base_dir"
-	git -C "$external_dpdk_base_dir" log --oneline -n 5
-
-	dpdk_cflags="-fPIC -g -Werror -fcommon"
-	dpdk_ldflags=""
-
-	# the drivers we use
-	# net/i40e driver is not really needed by us, but it's built as a workaround
-	# for DPDK issue: https://bugs.dpdk.org/show_bug.cgi?id=576
-	DPDK_DRIVERS=("bus" "bus/pci" "bus/vdev" "mempool/ring" "net/i40e" "net/i40e/base")
-	# all possible DPDK drivers
-	DPDK_ALL_DRIVERS=($(find "$external_dpdk_base_dir/drivers" -mindepth 1 -type d | sed -n "s#^$external_dpdk_base_dir/drivers/##p"))
-
-	if [[ "$SPDK_TEST_CRYPTO" -eq 1 ]]; then
-		git clone --branch v0.54 --depth 1 https://github.com/intel/intel-ipsec-mb.git "$external_dpdk_base_dir/intel-ipsec-mb"
-		cd "$external_dpdk_base_dir/intel-ipsec-mb"
-		$MAKE $MAKEFLAGS all SHARED=y EXTRA_CFLAGS=-fPIC
-		DPDK_DRIVERS+=("crypto")
-		DPDK_DRIVERS+=("crypto/aesni_mb")
-		DPDK_DRIVERS+=("crypto/qat")
-		DPDK_DRIVERS+=("compress/qat")
-		DPDK_DRIVERS+=("common/qat")
-		dpdk_cflags+=" -I$external_dpdk_base_dir/intel-ipsec-mb"
-		dpdk_ldflags+=" -L$external_dpdk_base_dir/intel-ipsec-mb"
-		export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$external_dpdk_base_dir/intel-ipsec-mb
-	fi
-
-	if [[ "$SPDK_TEST_REDUCE" -eq 1 ]]; then
-		isal_dir="$external_dpdk_base_dir/isa-l"
-		git clone --branch v2.29.0 --depth 1 https://github.com/intel/isa-l.git "$isal_dir"
-
-		cd $isal_dir
-		./autogen.sh
-		./configure CFLAGS="-fPIC -g -O2" --enable-shared=yes --prefix="$isal_dir/build"
-		ln -s $PWD/include $PWD/isa-l
-		$MAKE $MAKEFLAGS all
-		$MAKE install
-		DPDK_DRIVERS+=("compress")
-		DPDK_DRIVERS+=("compress/isal")
-		DPDK_DRIVERS+=("compress/qat")
-		DPDK_DRIVERS+=("common/qat")
-		export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$isal_dir/build/lib/pkgconfig"
-		export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$isal_dir/build/lib"
-	fi
-
-	# Use difference between DPDK_ALL_DRIVERS and DPDK_DRIVERS as a set of DPDK drivers we don't want or
-	# don't need to build.
-	DPDK_DISABLED_DRIVERS=($(sort <(printf "%s\n" "${DPDK_DRIVERS[@]}") <(printf "%s\n" "${DPDK_ALL_DRIVERS[@]}") | uniq -u))
-
-	cd $external_dpdk_base_dir
-	if [ "$(uname -s)" = "Linux" ]; then
-		dpdk_cflags+=" -Wno-stringop-overflow"
-		# Fix for freeing device if not kernel driver configured.
-		# TODO: Remove once this is merged in upstream DPDK
-		if grep "20.08.0" $external_dpdk_base_dir/VERSION; then
-			wget https://github.com/spdk/dpdk/commit/64f1ced13f974e8b3d46b87c361a09eca68126f9.patch -O dpdk-pci.patch
-			wget https://github.com/spdk/dpdk/commit/c2c273d5c8fbf673623b427f8f4ab5af5ddf0e08.patch -O dpdk-qat.patch
-		elif grep "20.11\|21.02" $external_dpdk_base_dir/VERSION; then
-			wget https://github.com/karlatec/dpdk/commit/3219c0cfc38803aec10c809dde16e013b370bda9.patch -O dpdk-pci.patch
-			wget https://github.com/karlatec/dpdk/commit/adf8f7638de29bc4bf9ba3faf12bbdae73acda0c.patch -O dpdk-qat.patch
-		else
-			wget https://github.com/karlatec/dpdk/commit/f95e331be3a1f856b816948990dd2afc67ea4020.patch -O dpdk-pci.patch
-			wget https://github.com/karlatec/dpdk/commit/6fd2fa906ffdcee04e6ce5da40e61cb841be9827.patch -O dpdk-qat.patch
-		fi
-		git config --local user.name "spdk"
-		git config --local user.email "nomail@all.com"
-		git am dpdk-pci.patch
-		git am dpdk-qat.patch
-	fi
-
-	meson build-tmp --prefix="$external_dpdk_dir" --libdir lib \
-		-Denable_docs=false -Denable_kmods=false -Dtests=false \
-		-Dc_link_args="$dpdk_ldflags" -Dc_args="$dpdk_cflags" \
-		-Dmachine=native -Ddisable_drivers=$(printf "%s," "${DPDK_DISABLED_DRIVERS[@]}")
-	ninja -C "$external_dpdk_base_dir/build-tmp" $MAKEFLAGS
-	ninja -C "$external_dpdk_base_dir/build-tmp" $MAKEFLAGS install
-
-	# Save this path. In tests are run using autorun.sh then autotest.sh
-	# script will be unaware of LD_LIBRARY_PATH and will fail tests.
-	echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" > /tmp/spdk-ld-path
-
-	cd "$orgdir"
-}
-
-function make_fail_cleanup() {
+function make_fail_cleanup {
 	if [ -d $out/scan-build-tmp ]; then
 		scanoutput=$(ls -1 $out/scan-build-tmp/)
 		mv $out/scan-build-tmp/$scanoutput $out/scan-build
@@ -156,7 +55,7 @@ function make_fail_cleanup() {
 	false
 }
 
-function scanbuild_make() {
+function scanbuild_make {
 	pass=true
 	$scanbuild $MAKE $MAKEFLAGS > $out/build_output.txt && rm -rf $out/scan-build-tmp || make_fail_cleanup
 	xtrace_disable
@@ -190,7 +89,7 @@ function scanbuild_make() {
 	$pass
 }
 
-function porcelain_check() {
+function porcelain_check {
 	if [ $(git status --porcelain --ignore-submodules | wc -l) -ne 0 ]; then
 		echo "Generated files missing from .gitignore:"
 		git status --porcelain --ignore-submodules
@@ -201,12 +100,12 @@ function porcelain_check() {
 # Check that header file dependencies are working correctly by
 #  capturing a binary's stat data before and after touching a
 #  header file and re-making.
-function header_dependency_check() {
-	STAT1=$(stat $SPDK_BIN_DIR/spdk_tgt)
+function header_dependency_check {
+	STAT1=$(stat app/spdk_tgt/spdk_tgt)
 	sleep 1
 	touch lib/nvme/nvme_internal.h
 	$MAKE $MAKEFLAGS
-	STAT2=$(stat $SPDK_BIN_DIR/spdk_tgt)
+	STAT2=$(stat app/spdk_tgt/spdk_tgt)
 
 	if [ "$STAT1" == "$STAT2" ]; then
 		echo "Header dependency check failed"
@@ -214,29 +113,28 @@ function header_dependency_check() {
 	fi
 }
 
-function test_make_uninstall() {
+function test_make_uninstall {
 	# Create empty file to check if it is not deleted by target uninstall
-	touch "$SPDK_WORKSPACE/usr/lib/sample_xyz.a"
-	$MAKE $MAKEFLAGS uninstall DESTDIR="$SPDK_WORKSPACE" prefix=/usr
-	if [[ $(find "$SPDK_WORKSPACE/usr" -maxdepth 1 -mindepth 1 | wc -l) -ne 2 ]] || [[ $(find "$SPDK_WORKSPACE/usr/lib/" -maxdepth 1 -mindepth 1 | wc -l) -ne 1 ]]; then
-		ls -lR "$SPDK_WORKSPACE"
+	touch /tmp/spdk/usr/lib/sample_xyz.a
+	$MAKE $MAKEFLAGS uninstall DESTDIR=/tmp/spdk prefix=/usr
+	if [[ $(find /tmp/spdk/usr -maxdepth 1 -mindepth 1 | wc -l) -ne 2 ]] || [[ $(find /tmp/spdk/usr/lib/ -maxdepth 1 -mindepth 1 | wc -l) -ne 1 ]]; then
+		ls -lR /tmp/spdk
+		rm -rf /tmp/spdk
 		echo "Make uninstall failed"
 		exit 1
+	else
+		rm -rf /tmp/spdk
 	fi
 }
 
-function build_doc() {
-	local doxygenv
-	doxygenv=$(doxygen --version)
-
+function build_doc {
 	$MAKE -C "$rootdir"/doc --no-print-directory $MAKEFLAGS &> "$out"/doxygen.log
 	if [ -s "$out"/doxygen.log ]; then
 		cat "$out"/doxygen.log
 		echo "Doxygen errors found!"
-		eq "$doxygenv" 1.8.20 || exit 1
-		echo "Doxygen $doxygenv detected, all warnings are potentially false positives, continuing the test"
+		exit 1
 	fi
-	if hash pdflatex 2> /dev/null; then
+	if hash pdflatex 2>/dev/null; then
 		$MAKE -C "$rootdir"/doc/output/latex --no-print-directory $MAKEFLAGS &>> "$out"/doxygen.log
 	fi
 	mkdir -p "$out"/doc
@@ -246,24 +144,22 @@ function build_doc() {
 	fi
 	$MAKE -C "$rootdir"/doc --no-print-directory $MAKEFLAGS clean &>> "$out"/doxygen.log
 	if [ -s "$out"/doxygen.log ]; then
-		# Save the log as an artifact in case we are working with potentially broken version
-		eq "$doxygenv" 1.8.20 || rm "$out"/doxygen.log
+		rm "$out"/doxygen.log
 	fi
 	rm -rf "$rootdir"/doc/output
 }
 
-function autobuild_test_suite() {
+function autobuild_test_suite {
 	run_test "autobuild_check_format" ./scripts/check_format.sh
-	run_test "autobuild_external_code" sudo -E --preserve-env=PATH LD_LIBRARY_PATH=$LD_LIBRARY_PATH $rootdir/test/external_code/test_make.sh $rootdir
+	run_test "autobuild_external_code" sudo -E $rootdir/test/external_code/test_make.sh $rootdir
 	if [ "$SPDK_TEST_OCF" -eq 1 ]; then
 		run_test "autobuild_ocf_precompile" ocf_precompile
 	fi
-	run_test "autobuild_check_so_deps" $rootdir/test/make/check_so_deps.sh $1
-	./configure $config_params --without-shared
+	run_test "autobuild_check_so_deps" $rootdir/test/make/check_so_deps.sh
 	run_test "scanbuild_make" scanbuild_make
 	run_test "autobuild_generated_files_check" porcelain_check
 	run_test "autobuild_header_dependency_check" header_dependency_check
-	run_test "autobuild_make_install" $MAKE $MAKEFLAGS install DESTDIR="$SPDK_WORKSPACE" prefix=/usr
+	run_test "autobuild_make_install" $MAKE $MAKEFLAGS install DESTDIR=/tmp/spdk prefix=/usr
 	run_test "autobuild_make_uninstall" test_make_uninstall
 	run_test "autobuild_build_doc" build_doc
 }
@@ -280,24 +176,11 @@ if [ $SPDK_RUN_UBSAN -eq 1 ]; then
 	run_test "ubsan" echo "using ubsan"
 fi
 
-if [ -n "$SPDK_TEST_NATIVE_DPDK" ]; then
-	run_test "build_native_dpdk" build_native_dpdk
-fi
-
-./configure $config_params
-echo "** START ** Info for Hostname: $HOSTNAME"
-uname -a
-$MAKE cc_version
-$MAKE cxx_version
-echo "** END ** Info for Hostname: $HOSTNAME"
-
 if [ "$SPDK_TEST_AUTOBUILD" -eq 1 ]; then
-	run_test "autobuild" autobuild_test_suite $1
+	run_test "autobuild" autobuild_test_suite
 else
 	if [ "$SPDK_TEST_OCF" -eq 1 ]; then
 		run_test "autobuild_ocf_precompile" ocf_precompile
 	fi
-	# if we aren't testing the unittests, build with shared objects.
-	./configure $config_params --with-shared
 	run_test "make" $MAKE $MAKEFLAGS
 fi

autopackage.sh

@@ -13,37 +13,6 @@ source "$1"
 rootdir=$(readlink -f $(dirname $0))
 source "$rootdir/test/common/autotest_common.sh"
 
-function build_rpms() (
-	local version rpms
-
-	# Make sure linker will not attempt to look under DPDK's repo dir to get the libs
-	unset -v LD_LIBRARY_PATH
-
-	install_uninstall_rpms() {
-		rpms=("$HOME/rpmbuild/RPMS/x86_64/"spdk{,-devel,{,-dpdk}-libs}-$version-1.x86_64.rpm)
-
-		sudo rpm -i "${rpms[@]}"
-		rpms=("${rpms[@]##*/}") rpms=("${rpms[@]%.rpm}")
-		# Check if we can find one of the apps in the PATH now and verify if it doesn't miss
-		# any libs.
-		LIST_LIBS=yes "$rootdir/rpmbuild/rpm-deps.sh" "${SPDK_APP[@]##*/}"
-		sudo rpm -e "${rpms[@]}"
-	}
-
-	build_rpm() {
-		MAKEFLAGS="$MAKEFLAGS" SPDK_VERSION="$version" DEPS=no "$rootdir/rpmbuild/rpm.sh" "$@"
-		install_uninstall_rpms
-	}
-
-	version="test_shared"
-	run_test "build_shared_rpm" build_rpm --with-shared
-
-	if [[ -n $SPDK_TEST_NATIVE_DPDK ]]; then
-		version="test_shared_native_dpdk"
-		run_test "build_shared_native_dpdk_rpm" build_rpm --with-shared --with-dpdk="$SPDK_RUN_EXTERNAL_DPDK"
-	fi
-)
-
 out=$PWD
 
 MAKEFLAGS=${MAKEFLAGS:--j16}
@@ -59,11 +28,6 @@ if [ $(git status --porcelain --ignore-submodules | wc -l) -ne 0 ]; then
 fi
 timing_exit porcelain_check
 
-if [[ $SPDK_TEST_RELEASE_BUILD -eq 1 ]]; then
-	run_test "build_rpms" build_rpms
-	$MAKE clean
-fi
-
 if [[ $RUN_NIGHTLY -eq 0 ]]; then
 	timing_finish
 	exit 0
@@ -71,12 +35,11 @@ fi
 
 timing_enter build_release
 
-config_params="$(get_config_params | sed 's/--enable-debug//g')"
 if [ $(uname -s) = Linux ]; then
-	./configure $config_params --enable-lto
+	./configure $(get_config_params) --disable-debug --enable-lto --disable-unit-tests
 else
 	# LTO needs a special compiler to work on BSD.
-	./configure $config_params
+	./configure $(get_config_params) --disable-debug
 fi
 $MAKE ${MAKEFLAGS}
 $MAKE ${MAKEFLAGS} clean

autorun.sh

@@ -4,8 +4,7 @@ set -e
 
 rootdir=$(readlink -f $(dirname $0))
 
-default_conf=~/autorun-spdk.conf
-conf=${1:-${default_conf}}
+conf=~/autorun-spdk.conf
 
 # If the configuration of tests is not provided, no tests will be carried out.
 if [[ ! -f $conf ]]; then
@@ -18,5 +17,5 @@ cat "$conf"
 
 # Runs agent scripts
 $rootdir/autobuild.sh "$conf"
-sudo -E $rootdir/autotest.sh "$conf"
+sudo -E WITH_DPDK_DIR="$WITH_DPDK_DIR" $rootdir/autotest.sh "$conf"
 $rootdir/autopackage.sh "$conf"

autorun_post.py

@@ -176,7 +176,6 @@ def aggregateCompletedTests(output_dir, repo_dir, skip_confirm=False):
 
 
 def main(output_dir, repo_dir, skip_confirm=False):
-    print("-----Begin Post Process Script------")
     generateCoverageReport(output_dir, repo_dir)
     collectOne(output_dir, 'doc')
     collectOne(output_dir, 'ut_coverage')

autotest.sh

@@ -9,16 +9,6 @@ if [[ ! -f $1 ]]; then
 	exit 1
 fi
 
-# always test with SPDK shared objects.
-export SPDK_LIB_DIR="$rootdir/build/lib"
-
-# Autotest.sh, as part of autorun.sh, runs in a different
-# shell process than autobuild.sh. Use helper file to pass
-# over env variable containing libraries paths.
-if [[ -e /tmp/spdk-ld-path ]]; then
-	source /tmp/spdk-ld-path
-fi
-
 source "$1"
 source "$rootdir/test/common/autotest_common.sh"
 source "$rootdir/test/nvmf/common.sh"
@@ -29,25 +19,12 @@ if [ $EUID -ne 0 ]; then
 fi
 
 if [ $(uname -s) = Linux ]; then
-	old_core_pattern=$(< /proc/sys/kernel/core_pattern)
-	mkdir -p "$output_dir/coredumps"
 	# set core_pattern to a known value to avoid ABRT, systemd-coredump, etc.
-	echo "|$rootdir/scripts/core-collector.sh %P %s %t $output_dir/coredumps" > /proc/sys/kernel/core_pattern
-	echo 2 > /proc/sys/kernel/core_pipe_limit
-
-	# Make sure that the hugepage state for our VM is fresh so we don't fail
-	# hugepage allocation. Allow time for this action to complete.
-	echo 1 > /proc/sys/vm/drop_caches
-	sleep 3
+	echo "core" > /proc/sys/kernel/core_pattern
 
 	# make sure nbd (network block device) driver is loaded if it is available
 	# this ensures that when tests need to use nbd, it will be fully initialized
 	modprobe nbd || true
-
-	if udevadm=$(type -P udevadm); then
-		"$udevadm" monitor --property &> "$output_dir/udev.log" &
-		udevadm_pid=$!
-	fi
 fi
 
 trap "process_core; autotest_cleanup; exit 1" SIGINT SIGTERM EXIT
@@ -60,13 +37,11 @@ src=$(readlink -f $(dirname $0))
 out=$output_dir
 cd $src
 
+./scripts/setup.sh status
+
 freebsd_update_contigmem_mod
 
-# lcov takes considerable time to process clang coverage.
-# Disabling lcov allow us to do this.
-# More information: https://github.com/spdk/spdk/issues/1693
-CC_TYPE=$(grep CC_TYPE mk/cc.mk)
-if hash lcov && ! [[ "$CC_TYPE" == *"clang"* ]]; then
+if hash lcov; then
 	# setup output dir for unittest.sh
 	export UT_COVERAGE=$out/ut_coverage
 	export LCOV_OPTS="
@@ -92,47 +67,44 @@ rm -f /var/tmp/spdk*.sock
 # Load the kernel driver
 ./scripts/setup.sh reset
 
+# Let the kernel discover any filesystems or partitions
+sleep 10
+
 if [ $(uname -s) = Linux ]; then
 	# OCSSD devices drivers don't support IO issues by kernel so
-	# detect OCSSD devices and block them (unbind from any driver).
+	# detect OCSSD devices and blacklist them (unbind from any driver).
 	# If test scripts want to use this device it needs to do this explicitly.
 	#
 	# If some OCSSD device is bound to other driver than nvme we won't be able to
 	# discover if it is OCSSD or not so load the kernel driver first.
 
-	while IFS= read -r -d '' dev; do
+
+	while IFS= read -r -d '' dev
+	do
 		# Send Open Channel 2.0 Geometry opcode "0xe2" - not supported by NVMe device.
-		if nvme admin-passthru $dev --namespace-id=1 --data-len=4096 --opcode=0xe2 --read > /dev/null; then
+		if nvme admin-passthru $dev --namespace-id=1 --data-len=4096  --opcode=0xe2 --read >/dev/null; then
 			bdf="$(basename $(readlink -e /sys/class/nvme/${dev#/dev/}/device))"
-			echo "INFO: blocking OCSSD device: $dev ($bdf)"
-			PCI_BLOCKED+=" $bdf"
+			echo "INFO: blacklisting OCSSD device: $dev ($bdf)"
+			PCI_BLACKLIST+=" $bdf"
 			OCSSD_PCI_DEVICES+=" $bdf"
 		fi
-	done < <(find /dev -maxdepth 1 -regex '/dev/nvme[0-9]+' -print0)
+	done <   <(find /dev -maxdepth 1 -regex '/dev/nvme[0-9]+' -print0)
 
 	export OCSSD_PCI_DEVICES
 
-	# Now, bind blocked devices to pci-stub module. This will prevent
+	# Now, bind blacklisted devices to pci-stub module. This will prevent
 	# automatic grabbing these devices when we add device/vendor ID to
 	# proper driver.
-	if [[ -n "$PCI_BLOCKED" ]]; then
+	if [[ -n "$PCI_BLACKLIST" ]]; then
 		# shellcheck disable=SC2097,SC2098
-		PCI_ALLOWED="$PCI_BLOCKED" \
-			PCI_BLOCKED="" \
-			DRIVER_OVERRIDE="pci-stub" \
+		PCI_WHITELIST="$PCI_BLACKLIST" \
+		PCI_BLACKLIST="" \
+		DRIVER_OVERRIDE="pci-stub" \
 			./scripts/setup.sh
 
-		# Export our blocked list so it will take effect during next setup.sh
-		export PCI_BLOCKED
+		# Export our blacklist so it will take effect during next setup.sh
+		export PCI_BLACKLIST
 	fi
-	run_test "setup.sh" "$rootdir/test/setup/test-setup.sh"
-fi
-
-./scripts/setup.sh status
-
-if [[ $(uname -s) == Linux ]]; then
-	# Revert NVMe namespaces to default state
-	nvme_namespace_revert
 fi
 
 # Delete all leftover lvols and gpt partitions
@@ -151,13 +123,12 @@ timing_enter afterboot
 ./scripts/setup.sh
 timing_exit afterboot
 
+timing_enter nvmf_setup
+rdma_device_init
+timing_exit nvmf_setup
+
 if [[ $SPDK_TEST_CRYPTO -eq 1 || $SPDK_TEST_REDUCE -eq 1 ]]; then
-	# Make sure that memory is distributed across all NUMA nodes - by default, all goes to
-	# node0, but if QAT devices are attached to a different node, all of their VFs will end
-	# up under that node too and memory needs to be available there for the tests.
-	CLEAR_HUGE=yes HUGE_EVEN_ALLOC=yes ./scripts/setup.sh
-	./scripts/setup.sh status
-	if [[ $SPDK_TEST_USE_IGB_UIO -eq 1 ]]; then
+	if grep -q '#define SPDK_CONFIG_IGB_UIO_DRIVER 1' $rootdir/include/spdk/config.h; then
 		./scripts/qat_setup.sh igb_uio
 	else
 		./scripts/qat_setup.sh
@@ -166,7 +137,8 @@ fi
 
 # Revert existing OPAL to factory settings that may have been left from earlier failed tests.
 # This ensures we won't hit any unexpected failures due to NVMe SSDs being locked.
-opal_revert_cleanup
+# Disable this for now as we don't have opal test running
+# opal_revert_cleanup
 
 #####################
 # Unit Tests
@@ -174,29 +146,23 @@ opal_revert_cleanup
 
 if [ $SPDK_TEST_UNITTEST -eq 1 ]; then
 	run_test "unittest" ./test/unit/unittest.sh
-	run_test "env" test/env/env.sh
 fi
 
 if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
 	timing_enter lib
 
+	run_test "env" test/env/env.sh
 	run_test "rpc" test/rpc/rpc.sh
 	run_test "rpc_client" test/rpc_client/rpc_client.sh
 	run_test "json_config" ./test/json_config/json_config.sh
 	run_test "alias_rpc" test/json_config/alias_rpc/alias_rpc.sh
 	run_test "spdkcli_tcp" test/spdkcli/tcp.sh
-	run_test "dpdk_mem_utility" test/dpdk_memory_utility/test_dpdk_mem_info.sh
+        run_test "dpdk_mem_utility" test/dpdk_memory_utility/test_dpdk_mem_info.sh
 	run_test "event" test/event/event.sh
-	run_test "accel_engine" test/accel_engine/accel_engine.sh
 
 	if [ $SPDK_TEST_BLOCKDEV -eq 1 ]; then
 		run_test "blockdev_general" test/bdev/blockdev.sh
 		run_test "bdev_raid" test/bdev/bdev_raid.sh
-		run_test "bdevperf_config" test/bdev/bdevperf/test_config.sh
-		if [[ $(uname -s) == Linux ]]; then
-			run_test "spdk_dd" test/dd/dd.sh
-			run_test "reactor_set_interrupt" test/interrupt/reactor_set_interrupt.sh
-		fi
 	fi
 
 	if [ $SPDK_TEST_JSON -eq 1 ]; then
@@ -207,18 +173,20 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
 		run_test "blockdev_nvme" test/bdev/blockdev.sh "nvme"
 		run_test "blockdev_nvme_gpt" test/bdev/blockdev.sh "gpt"
 		run_test "nvme" test/nvme/nvme.sh
-		if [[ $SPDK_TEST_NVME_PMR -eq 1 ]]; then
-			run_test "nvme_pmr" test/nvme/nvme_pmr.sh
+		if [[ $SPDK_TEST_NVME_CLI -eq 1 ]]; then
+			run_test "nvme_cli" test/nvme/spdk_nvme_cli.sh
 		fi
 		if [[ $SPDK_TEST_NVME_CUSE -eq 1 ]]; then
 			run_test "nvme_cuse" test/nvme/cuse/nvme_cuse.sh
+			run_test "nvme_cli_cuse" test/nvme/spdk_nvme_cli_cuse.sh
+			run_test "nvme_smartctl_cuse" test/nvme/spdk_smartctl_cuse.sh
 		fi
 		run_test "nvme_rpc" test/nvme/nvme_rpc.sh
 		# Only test hotplug without ASAN enabled. Since if it is
 		# enabled, it catches SEGV earlier than our handler which
 		# breaks the hotplug logic.
 		if [ $SPDK_RUN_ASAN -eq 0 ]; then
-			run_test "nvme_hotplug" test/nvme/hotplug.sh root
+			run_test "nvme_hotplug" test/nvme/hotplug.sh intel
 		fi
 	fi
 
@@ -229,41 +197,38 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
 	timing_exit lib
 
 	if [ $SPDK_TEST_ISCSI -eq 1 ]; then
-		run_test "iscsi_tgt" ./test/iscsi_tgt/iscsi_tgt.sh
+		run_test "iscsi_tgt_posix" ./test/iscsi_tgt/iscsi_tgt.sh posix
 		run_test "spdkcli_iscsi" ./test/spdkcli/iscsi.sh
 
 		# Run raid spdkcli test under iSCSI since blockdev tests run on systems that can't run spdkcli yet
 		run_test "spdkcli_raid" test/spdkcli/raid.sh
 	fi
 
+	if [ $SPDK_TEST_VPP -eq 1 ]; then
+		run_test "iscsi_tgt_vpp" ./test/iscsi_tgt/iscsi_tgt.sh vpp
+	fi
+
 	if [ $SPDK_TEST_BLOBFS -eq 1 ]; then
 		run_test "rocksdb" ./test/blobfs/rocksdb/rocksdb.sh
 		run_test "blobstore" ./test/blobstore/blobstore.sh
 		run_test "blobfs" ./test/blobfs/blobfs.sh
-		run_test "hello_blob" $SPDK_EXAMPLE_DIR/hello_blob \
-			examples/blob/hello_world/hello_blob.json
+		run_test "hello_blob" ./examples/blob/hello_world/hello_blob \
+			examples/blob/hello_world/hello_blob.conf
 	fi
 
 	if [ $SPDK_TEST_NVMF -eq 1 ]; then
 		# The NVMe-oF run test cases are split out like this so that the parser that compiles the
 		# list of all tests can properly differentiate them. Please do not merge them into one line.
 		if [ "$SPDK_TEST_NVMF_TRANSPORT" = "rdma" ]; then
-			timing_enter rdma_setup
-			rdma_device_init
-			timing_exit rdma_setup
 			run_test "nvmf_rdma" ./test/nvmf/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
-			run_test "spdkcli_nvmf_rdma" ./test/spdkcli/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
+			run_test "spdkcli_nvmf_rdma" ./test/spdkcli/nvmf.sh
 		elif [ "$SPDK_TEST_NVMF_TRANSPORT" = "tcp" ]; then
-			timing_enter tcp_setup
-			tcp_device_init
-			timing_exit tcp_setup
 			run_test "nvmf_tcp" ./test/nvmf/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
-			run_test "spdkcli_nvmf_tcp" ./test/spdkcli/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
+			run_test "spdkcli_nvmf_tcp" ./test/spdkcli/nvmf.sh
 			run_test "nvmf_identify_passthru" test/nvmf/target/identify_passthru.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
-			run_test "nvmf_dif" test/nvmf/target/dif.sh
 		elif [ "$SPDK_TEST_NVMF_TRANSPORT" = "fc" ]; then
-			run_test "nvmf_fc" ./test/nvmf/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
-			run_test "spdkcli_nvmf_fc" ./test/spdkcli/nvmf.sh
+				run_test "nvmf_fc" ./test/nvmf/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
+				run_test "spdkcli_nvmf_fc" ./test/spdkcli/nvmf.sh
 		else
 			echo "unknown NVMe transport, please specify rdma, tcp, or fc."
 			exit 1
@@ -275,7 +240,7 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
 	fi
 
 	if [ $SPDK_TEST_LVOL -eq 1 ]; then
-		run_test "lvol" ./test/lvol/lvol.sh
+		run_test "lvol2" ./test/lvol/lvol2.sh
 		run_test "blob_io_wait" ./test/blobstore/blob_io_wait/blob_io_wait.sh
 	fi
 
@@ -311,10 +276,10 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
 		run_test "vmd" ./test/vmd/vmd.sh
 	fi
 
-	if [ $SPDK_TEST_REDUCE -eq 1 ]; then
-		run_test "compress_qat" ./test/compress/compress.sh "qat"
-		run_test "compress_isal" ./test/compress/compress.sh "isal"
-	fi
+        if [ $SPDK_TEST_REDUCE -eq 1 ]; then
+                run_test "compress_qat" ./test/compress/compress.sh "qat"
+                run_test "compress_isal" ./test/compress/compress.sh "isal"
+        fi
 
 	if [ $SPDK_TEST_OPAL -eq 1 ]; then
 		run_test "nvme_opal" ./test/nvme/nvme_opal.sh
@@ -327,10 +292,6 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
 			run_test "blockdev_crypto_qat" ./test/bdev/blockdev.sh "crypto_qat"
 		fi
 	fi
-
-	if [[ $SPDK_TEST_SCHEDULER -eq 1 ]]; then
-		run_test "scheduler" ./test/scheduler/scheduler.sh
-	fi
 fi
 
 timing_enter cleanup
@@ -345,7 +306,7 @@ trap - SIGINT SIGTERM EXIT
 # catch any stray core files
 process_core
 
-if hash lcov && ! [[ "$CC_TYPE" == *"clang"* ]]; then
+if hash lcov; then
 	# generate coverage data and combine with baseline
 	$LCOV -q -c -d $src -t "$(hostname)" -o $out/cov_test.info
 	$LCOV -q -a $out/cov_base.info -a $out/cov_test.info -o $out/cov_total.info

configure

@@ -5,9 +5,9 @@ set -e
 trap 'echo -e "\n\nConfiguration failed\n\n" >&2' ERR
 
 rootdir=$(readlink -f $(dirname $0))
-source "$rootdir/scripts/common.sh"
 
-function usage() {
+function usage()
+{
 	echo "'configure' configures SPDK to compile on supported platforms."
 	echo ""
 	echo "Usage: ./configure [OPTION]..."
@@ -24,6 +24,7 @@ function usage() {
 	echo "                           example: aarch64-linux-gnu"
 	echo ""
 	echo " --enable-debug            Configure for debug builds"
+	echo " --enable-log-bt           Enable support of backtrace printing in SPDK logs (requires libunwind)."
 	echo " --enable-werror           Treat compiler warnings as errors"
 	echo " --enable-asan             Enable address sanitizer"
 	echo " --enable-ubsan            Enable undefined behavior sanitizer"
@@ -31,7 +32,6 @@ function usage() {
 	echo " --enable-lto              Enable link-time optimization"
 	echo " --enable-pgo-capture      Enable generation of profile guided optimization data"
 	echo " --enable-pgo-use          Use previously captured profile guided optimization data"
-	echo " --enable-cet              Enable Intel Control-flow Enforcement Technology (CET)"
 	echo " --disable-tests           Disable building of functional tests"
 	echo " --disable-unit-tests      Disable building of unit tests"
 	echo " --disable-examples        Disable building of examples"
@@ -48,6 +48,10 @@ function usage() {
 	echo "                           example: /usr/share/dpdk/x86_64-default-linuxapp-gcc"
 	echo " env                       Use an alternate environment implementation instead of DPDK."
 	echo "                           Implies --without-dpdk."
+	echo " igb-uio-driver            Build DPDK's igb-uio driver."
+	echo "                           Required on some systems to use qat devices. This flag is"
+	echo "                           effective only with the default dpdk submodule."
+	echo "                           No path required"
 	echo " idxd                      Build the IDXD library and accel framework plug-in module."
 	echo "                           Disabled while experimental. Only built for x86 when enabled."
 	echo " crypto                    Build vbdev crypto module."
@@ -56,19 +60,21 @@ function usage() {
 	echo "                           default: /usr/src/fio"
 	echo " vhost                     Build vhost target. Enabled by default."
 	echo "                           No path required."
+	echo " internal-vhost-lib        Use the internal copy of rte_vhost. By default, the upstream"
+	echo "                           rte_vhost from DPDK will be used."
+	echo "                           No path required."
 	echo " virtio                    Build vhost initiator and virtio-pci bdev modules."
 	echo "                           No path required."
-	echo " vfio-user                 Build custom vfio-user transport for NVMf target and NVMe initiator."
-	echo "                           example: /usr/src/libvfio-user"
 	echo " pmdk                      Build persistent memory bdev."
 	echo "                           example: /usr/share/pmdk"
 	echo " reduce                    Build vbdev compression module."
 	echo "                           No path required."
+	echo " vpp                       Build VPP net module."
+	echo "                           example: /vpp_repo/build-root/rpmbuild/vpp-18.01.1.0/build-root/install-vpp-native/vpp"
 	echo " rbd                       Build Ceph RBD bdev module."
 	echo "                           No path required."
 	echo " rdma                      Build RDMA transport for NVMf target and initiator."
-	echo "                           Accepts optional RDMA provider name. Can be \"verbs\" or \"mlx5_dv\"."
-	echo "                           If no provider specified, \"verbs\" provider is used by default."
+	echo "                           No path required."
 	echo " fc                        Build FC transport for NVMf target."
 	echo "                           If an argument is provided, it is considered a directory containing"
 	echo "                           libufc.a and fc_lld.h. Otherwise the regular system paths will"
@@ -96,8 +102,6 @@ function usage() {
 	echo "                           No path required."
 	echo " raid5                     Build with bdev_raid module RAID5 support."
 	echo "                           No path required."
-	echo " wpdk                      Build using WPDK to provide support for Windows (experimental)."
-	echo "                           The argument must be a directory containing lib and include."
 	echo ""
 	echo "Environment variables:"
 	echo ""
@@ -133,18 +137,14 @@ for i in "$@"; do
 done
 
 # Detect the compiler toolchain
-$rootdir/scripts/detect_cc.sh --cc="$CC" --cxx="$CXX" --lto="${CONFIG[LTO]}" --ld="$LD" --cross-prefix="${CONFIG[CROSS_PREFIX]}" > $rootdir/mk/cc.mk
+$rootdir/scripts/detect_cc.sh --cc="$CC" --cxx="$CXX" --lto="${CONFIG[LTO]}" --ld="$LD" --cross-prefix="${CONFIG[CROSS_PREFIX]}"  > $rootdir/mk/cc.mk
 
-CC=$(grep "DEFAULT_CC=" "$rootdir/mk/cc.mk" | sed s/DEFAULT_CC=//)
-CC_TYPE=$(grep "CC_TYPE=" "$rootdir/mk/cc.mk" | cut -d "=" -f 2)
+CC=$(cat $rootdir/mk/cc.mk | grep "DEFAULT_CC=" | cut -d "=" -f 2)
+CC_TYPE=$(cat $rootdir/mk/cc.mk | grep "CC_TYPE=" | cut -d "=" -f 2)
 
 arch=$($CC -dumpmachine)
 sys_name=$(uname -s)
 
-if [[ $arch == *mingw* ]] || [[ $arch == *windows* ]]; then
-	sys_name=Windows
-fi
-
 # Sanitize default configuration. All parameters set by user explicit should fail
 # Force no ISA-L if non-x86 or non-aarch64 architecture
 if [[ "${CONFIG[ISAL]}" = "y" ]]; then
@@ -154,24 +154,25 @@ if [[ "${CONFIG[ISAL]}" = "y" ]]; then
 	fi
 fi
 
-if [[ $sys_name != "Linux" ]]; then
+if [[ $sys_name == "FreeBSD" ]]; then
 	# Vhost, rte_vhost library and virtio are only supported on Linux.
 	CONFIG[VHOST]="n"
+	CONFIG[VHOST_INTERNAL_LIB]="n"
 	CONFIG[VIRTIO]="n"
 	echo "Notice: Vhost, rte_vhost library and virtio are only supported on Linux. Turning off default feature."
 fi
 
 #check nasm only on x86
 if [[ $arch == x86_64* ]]; then
-	ver=$(nasm -v 2> /dev/null | awk '{print $3}')
-	if lt "$ver" 2.14; then
-		# ISA-L, compression & crypto require NASM version 2.14 or newer.
+	ver=$(nasm -v 2>/dev/null | awk '{print $3}' | sed 's/[^0-9]*//g')
+	if [[ "${ver:0:1}" -le "2" ]] && [[ "${ver:0:3}" -le "213" ]] && [[ "${ver:0:5}" -lt "21303" ]]; then
+		# ISA-L, compression & crypto require NASM version 2.13.03 or newer.
 		CONFIG[ISAL]=n
 		CONFIG[CRYPTO]=n
 		CONFIG[IPSEC_MB]=n
 		CONFIG[REDUCE]=n
 		HAVE_NASM=n
-		echo "Notice: ISA-L, compression & crypto require NASM version 2.14 or newer. Turning off default ISA-L and crypto features."
+		echo "Notice: ISA-L, compression & crypto require NASM version 2.13.03 or newer. Turning off default ISA-L and crypto features."
 	else
 		HAVE_NASM=y
 	fi
@@ -188,7 +189,7 @@ function check_dir() {
 
 for i in "$@"; do
 	case "$i" in
-		-h | --help)
+		-h|--help)
 			usage
 			exit 0
 			;;
@@ -209,6 +210,12 @@ for i in "$@"; do
 		--disable-debug)
 			CONFIG[DEBUG]=n
 			;;
+		--enable-log-bt)
+			CONFIG[LOG_BACKTRACE]=y
+			;;
+		--disable-log-bt)
+			CONFIG[LOG_BACKTRACE]=n
+			;;
 		--enable-asan)
 			CONFIG[ASAN]=y
 			;;
@@ -269,12 +276,6 @@ for i in "$@"; do
 		--disable-werror)
 			CONFIG[WERROR]=n
 			;;
-		--enable-cet)
-			CONFIG[CET]=y
-			;;
-		--disable-cet)
-			CONFIG[CET]=n
-			;;
 		--with-dpdk=*)
 			check_dir "$i"
 			CONFIG[DPDK_DIR]=$(readlink -f ${i#*=})
@@ -282,10 +283,6 @@ for i in "$@"; do
 		--without-dpdk)
 			CONFIG[DPDK_DIR]=
 			;;
-		--with-wpdk=*)
-			check_dir "$i"
-			CONFIG[WPDK_DIR]=$(readlink -f ${i#*=})
-			;;
 		--with-env=*)
 			CONFIG[ENV]="${i#*=}"
 			;;
@@ -295,13 +292,8 @@ for i in "$@"; do
 		--without-rbd)
 			CONFIG[RBD]=n
 			;;
-		--with-rdma=*)
-			CONFIG[RDMA]=y
-			CONFIG[RDMA_PROV]=${i#*=}
-			;;
 		--with-rdma)
 			CONFIG[RDMA]=y
-			CONFIG[RDMA_PROV]="verbs"
 			;;
 		--without-rdma)
 			CONFIG[RDMA]=n
@@ -342,24 +334,18 @@ for i in "$@"; do
 		--without-vhost)
 			CONFIG[VHOST]=n
 			;;
+		--with-internal-vhost-lib)
+			CONFIG[VHOST_INTERNAL_LIB]=y
+			;;
+		--without-internal-vhost-lib)
+			CONFIG[VHOST_INTERNAL_LIB]=n
+			;;
 		--with-virtio)
 			CONFIG[VIRTIO]=y
 			;;
 		--without-virtio)
 			CONFIG[VIRTIO]=n
 			;;
-		--with-vfio-user)
-			CONFIG[VFIO_USER]=y
-			CONFIG[VFIO_USER_DIR]=""
-			;;
-		--with-vfio-user=*)
-			CONFIG[VFIO_USER]=y
-			check_dir "$i"
-			CONFIG[VFIO_USER_DIR]=$(readlink -f ${i#*=})
-			;;
-		--without-vfio-user)
-			CONFIG[VFIO_USER]=n
-			;;
 		--with-pmdk)
 			CONFIG[PMDK]=y
 			CONFIG[PMDK_DIR]=""
@@ -378,6 +364,17 @@ for i in "$@"; do
 		--without-reduce)
 			CONFIG[REDUCE]=n
 			;;
+		--with-vpp)
+			CONFIG[VPP]=y
+			;;
+		--with-vpp=*)
+			CONFIG[VPP]=y
+			check_dir "$i"
+			CONFIG[VPP_DIR]=$(readlink -f ${i#*=})
+			;;
+		--without-vpp)
+			CONFIG[VPP]=n
+			;;
 		--with-fio) ;&
 		--with-fio=*)
 			if [[ ${i#*=} != "$i" ]]; then
@@ -398,6 +395,12 @@ for i in "$@"; do
 			CONFIG[VTUNE_DIR]=
 			CONFIG[VTUNE]=n
 			;;
+		--with-igb-uio-driver)
+			CONFIG[IGB_UIO_DRIVER]=y
+			;;
+		--without-igb-uio-driver)
+			CONFIG[IGB_UIO_DRIVER]=n
+			;;
 		--with-ocf)
 			CONFIG[OCF]=y
 			CONFIG[OCF_PATH]=$(readlink -f "./ocf")
@@ -459,45 +462,19 @@ for i in "$@"; do
 			echo "Unrecognized option $i"
 			usage
 			exit 1
-			;;
 	esac
 done
 
 if [[ $arch == x86_64* ]]; then
-	BUILD_CMD=("$CC" -o /dev/null -x c $CPPFLAGS $CFLAGS $LDFLAGS "-march=native")
+	BUILD_CMD=($CC -o /dev/null -x c $CPPFLAGS $CFLAGS $LDFLAGS -march=native)
 else
-	BUILD_CMD=("$CC" -o /dev/null -x c $CPPFLAGS $CFLAGS $LDFLAGS)
-fi
-BUILD_CMD+=(-I/usr/local/include -L/usr/local/lib)
-
-if [[ "${CONFIG[VFIO_USER]}" = "y" ]]; then
-
-	if ! hash cmake; then
-		echo "ERROR: --with-vfio-user requires cmake"
-		echo "Please install then re-run this script"
-		exit 1
-	fi
-	if [[ ! -d /usr/include/json-c ]] && [[ ! -d /usr/local/include/json-c ]]; then
-		echo "ERROR: --with-vfio-user requires json-c-devel"
-		echo "Please install then re-run this script"
-		exit 1
-	fi
-	if [[ ! -e /usr/include/cmocka.h ]] && [[ ! -e /usr/local/include/cmocka.h ]]; then
-		echo "ERROR: --with-vfio-user requires libcmocka-devel"
-		echo "Please install then re-run this script"
-		exit 1
-	fi
+	BUILD_CMD=($CC -o /dev/null -x c $CPPFLAGS $CFLAGS $LDFLAGS)
 fi
 
 # IDXD uses Intel specific instructions.
 if [[ "${CONFIG[IDXD]}" = "y" ]]; then
-	if [ $(uname -s) == "FreeBSD" ]; then
-		intel="hw.model: Intel"
-		cpu_vendor=$(sysctl -a | grep hw.model | cut -c 1-15)
-	else
-		intel="GenuineIntel"
-		cpu_vendor=$(grep -i 'vendor' /proc/cpuinfo --max-count=1)
-	fi
+	intel="GenuineIntel"
+	cpu_vendor=$(grep -i 'vendor' /proc/cpuinfo  --max-count=1)
 	if [[ "$cpu_vendor" != *"$intel"* ]]; then
 		echo "ERROR: IDXD cannot be used due to CPU incompatiblity."
 		exit 1
@@ -513,15 +490,20 @@ if [[ "${CONFIG[ISAL]}" = "y" ]]; then
 fi
 
 if [[ "${CONFIG[ISAL]}" = "n" ]] && [[ "${CONFIG[REDUCE]}" = "y" ]]; then
-	echo "ERROR Conflicting options: --with-reduce is not compatible with --without-isal."
-	exit 1
+		echo "ERROR Conflicting options: --with-reduce is not compatible with --without-isal."
+		exit 1
+fi
+
+if [[ "${CONFIG[LTO]}" = "y" ]] && [[ "${CONFIG[UNIT_TESTS]}" = "y"  ]]; then
+		echo "ERROR Conflicting options: --enable-lto is not compatible with --enable-unit-tests."
+		exit 1
 fi
 
 if [ -z "${CONFIG[ENV]}" ]; then
 	CONFIG[ENV]=$rootdir/lib/env_dpdk
 	echo "Using default SPDK env in ${CONFIG[ENV]}"
 	if [ -z "${CONFIG[DPDK_DIR]}" ]; then
-		if [ ! -f "$rootdir"/dpdk/config/meson.build ]; then
+		if [ ! -f "$rootdir"/dpdk/config/common_base ]; then
 			echo "DPDK not found; please specify --with-dpdk=<path> or run:"
 			echo
 			echo "  git submodule update --init"
@@ -530,6 +512,31 @@ if [ -z "${CONFIG[ENV]}" ]; then
 			CONFIG[DPDK_DIR]="${rootdir}/dpdk/build"
 			echo "Using default DPDK in ${CONFIG[DPDK_DIR]}"
 		fi
+
+		if [[ "${CONFIG[VHOST]}" = "y" ]] && [[ "${CONFIG[VHOST_INTERNAL_LIB]}" = "n" ]]; then
+			# We lookup "common_linux" file to check if DPDK version is >= 19.05.
+			# "common_linux" is available since exactly DPDK 19.05 - it was renamed
+			# from "common_linuxapp".
+			if [ ! -f "$rootdir"/dpdk/config/common_linux ]; then
+				echo "Notice: Using internal, legacy rte_vhost library due to DPDK" \
+				     "version < 19.05"
+				CONFIG[VHOST_INTERNAL_LIB]=y
+			fi
+		fi
+	else
+		if [[ "${CONFIG[VHOST]}" = "y" ]] && [[ "${CONFIG[VHOST_INTERNAL_LIB]}" = "n" ]]; then
+			# DPDK must be already built, so we can simply try to use the new rte_vhost.
+			# It has a number of internal dependencies though, so don't try to link the
+			# program, just compile it
+			if ! echo -e '#include <rte_vhost.h>\n' \
+				'int main(void) { return rte_vhost_extern_callback_register(0, NULL, NULL); }\n' \
+				| ${BUILD_CMD[@]} -c -Wno-deprecated-declarations -Werror \
+				-I"${CONFIG[DPDK_DIR]}/include" - &>/dev/null; then
+				echo "Notice: DPDK's rte_vhost not found or version < 19.05, using internal," \
+				     "legacy rte_vhost library."
+				CONFIG[VHOST_INTERNAL_LIB]=y
+			fi
+		fi
 	fi
 else
 	if [ -n "${CONFIG[DPDK_DIR]}" ]; then
@@ -549,23 +556,6 @@ else
 	CONFIG[VIRTIO]="n"
 fi
 
-if [[ $sys_name == "Windows" ]]; then
-	if [ -z "${CONFIG[WPDK_DIR]}" ]; then
-		if [ ! -f "$rootdir"/wpdk/Makefile ]; then
-			echo "WPDK not found; please specify --with-wpdk=<path>. See https://wpdk.github.io."
-			exit 1
-		else
-			CONFIG[WPDK_DIR]="${rootdir}/wpdk/build"
-			echo "Using default WPDK in ${CONFIG[WPDK_DIR]}"
-		fi
-	fi
-else
-	if [ -n "${CONFIG[WPDK_DIR]}" ]; then
-		echo "ERROR: --with-wpdk is only supported for Windows"
-		exit 1
-	fi
-fi
-
 if [ "${CONFIG[VTUNE]}" = "y" ]; then
 	if [ -z "${CONFIG[VTUNE_DIR]}" ]; then
 		echo "When VTune is enabled, you must specify the VTune directory using --with-vtune=path"
@@ -573,7 +563,7 @@ if [ "${CONFIG[VTUNE]}" = "y" ]; then
 	fi
 fi
 
-if [[ "${CONFIG[ASAN]}" = "y" && "${CONFIG[TSAN]}" = "y" ]]; then
+if [ "${CONFIG[ASAN]}" = "y" -a "${CONFIG[TSAN]}" = "y" ]; then
 	echo "ERROR: ASAN and TSAN cannot be enabled at the same time."
 	exit 1
 fi
@@ -586,11 +576,15 @@ if [[ $sys_name == "FreeBSD" ]]; then
 	fi
 fi
 
-if [[ $sys_name != "Linux" ]]; then
+if [[ $sys_name == "FreeBSD" ]]; then
 	if [[ "${CONFIG[VHOST]}" == "y" ]]; then
 		echo "Vhost is only supported on Linux."
 		exit 1
 	fi
+	if [[ "${CONFIG[VHOST_INTERNAL_LIB]}" == "y" ]]; then
+		echo "Internal rte_vhost library is only supported on Linux."
+		exit 1
+	fi
 	if [[ "${CONFIG[VIRTIO]}" == "y" ]]; then
 		echo "Virtio is only supported on Linux."
 		exit 1
@@ -598,22 +592,17 @@ if [[ $sys_name != "Linux" ]]; then
 fi
 
 if [ "${CONFIG[RDMA]}" = "y" ]; then
-	if [[ ! "${CONFIG[RDMA_PROV]}" == "verbs" ]] && [[ ! "${CONFIG[RDMA_PROV]}" == "mlx5_dv" ]]; then
-		echo "Invalid RDMA provider specified, must be \"verbs\" or \"mlx5_dv\""
-		exit 1
-	fi
-
 	if ! echo -e '#include <infiniband/verbs.h>\n#include <rdma/rdma_verbs.h>\n' \
 		'int main(void) { return 0; }\n' \
-		| "${BUILD_CMD[@]}" -libverbs -lrdmacm - 2> /dev/null; then
-		echo "--with-rdma requires libverbs and librdmacm."
-		echo "Please install then re-run this script."
+		| ${BUILD_CMD[@]} -libverbs -lrdmacm - 2>/dev/null; then
+		echo --with-rdma requires libverbs and librdmacm.
+		echo Please install then re-run this script.
 		exit 1
 	fi
 
 	if echo -e '#include <infiniband/verbs.h>\n' \
 		'int main(void) { return !!IBV_WR_SEND_WITH_INV; }\n' \
-		| "${BUILD_CMD[@]}" -c - 2> /dev/null; then
+		| ${BUILD_CMD[@]} -c - 2>/dev/null; then
 		CONFIG[RDMA_SEND_WITH_INVAL]="y"
 	else
 		CONFIG[RDMA_SEND_WITH_INVAL]="n"
@@ -635,26 +624,13 @@ than or equal to 4.14 will see significantly reduced performance.
 
 	if echo -e '#include <rdma/rdma_cma.h>\n' \
 		'int main(void) { return !!RDMA_OPTION_ID_ACK_TIMEOUT; }\n' \
-		| "${BUILD_CMD[@]}" -c - 2> /dev/null; then
+		| ${BUILD_CMD[@]} -c - 2>/dev/null; then
 		CONFIG[RDMA_SET_ACK_TIMEOUT]="y"
 	else
 		CONFIG[RDMA_SET_ACK_TIMEOUT]="n"
 		echo "RDMA_OPTION_ID_ACK_TIMEOUT is not supported"
 	fi
 
-	if [ "${CONFIG[RDMA_PROV]}" == "mlx5_dv" ]; then
-		if ! echo -e '#include <spdk/stdinc.h>\n' \
-			'#include <infiniband/mlx5dv.h>\n' \
-			'#include <rdma/rdma_cma.h>\n' \
-			'int main(void) { return rdma_establish(NULL) || ' \
-			'!!IBV_QP_INIT_ATTR_SEND_OPS_FLAGS || !!MLX5_OPCODE_RDMA_WRITE; }\n' \
-			| "${BUILD_CMD[@]}" -lmlx5 -I${rootdir}/include -c - 2> /dev/null; then
-			echo "mlx5_dv provider is not supported"
-			exit 1
-		fi
-	fi
-
-	echo "Using '${CONFIG[RDMA_PROV]}' RDMA provider"
 fi
 
 if [[ "${CONFIG[FC]}" = "y" ]]; then
@@ -668,7 +644,7 @@ fi
 
 if [[ "${CONFIG[ISAL]}" = "y" ]] || [[ "${CONFIG[CRYPTO]}" = "y" ]]; then
 	if [[ "${HAVE_NASM}" = "n" ]] && [[ $arch == x86_64* ]]; then
-		echo "ERROR: ISA-L, compression & crypto require NASM version 2.14 or newer."
+		echo "ERROR: ISA-L, compression & crypto require NASM version 2.13.03 or newer."
 		echo "Please install or upgrade them re-run this script."
 		exit 1
 	else
@@ -678,29 +654,57 @@ if [[ "${CONFIG[ISAL]}" = "y" ]] || [[ "${CONFIG[CRYPTO]}" = "y" ]]; then
 	fi
 fi
 
+if [[ "${CONFIG[ISAL]}" = "y" ]]; then
+	if [ ! -f "$rootdir"/isa-l/autogen.sh ]; then
+		echo "ISA-L was not found; To install ISA-L run:"
+		echo "  git submodule update --init"
+		exit 1
+	fi
+
+	cd $rootdir/isa-l
+	ISAL_LOG=$rootdir/isa-l/spdk-isal.log
+	echo -n "Configuring ISA-L (logfile: $ISAL_LOG)..."
+	./autogen.sh &> $ISAL_LOG
+	./configure CFLAGS="-fPIC -g -O2" --enable-shared=no >> $ISAL_LOG 2>&1
+	echo "done."
+	cd $rootdir
+fi
+
 if [[ "${CONFIG[PMDK]}" = "y" ]]; then
 	if ! echo -e '#include <libpmemblk.h>\nint main(void) { return 0; }\n' \
-		| "${BUILD_CMD[@]}" -lpmemblk - 2> /dev/null; then
-		echo "--with-pmdk requires libpmemblk."
-		echo "Please install then re-run this script."
+		| ${BUILD_CMD[@]} -lpmemblk - 2>/dev/null; then
+		echo --with-pmdk requires libpmemblk.
+		echo Please install then re-run this script.
 		exit 1
 	fi
 fi
 
 if [[ "${CONFIG[REDUCE]}" = "y" ]]; then
 	if ! echo -e '#include <libpmem.h>\nint main(void) { return 0; }\n' \
-		| "${BUILD_CMD[@]}" -lpmem - 2> /dev/null; then
-		echo "--with-reduce requires libpmem."
-		echo "Please install then re-run this script."
+		| ${BUILD_CMD[@]} -lpmem - 2>/dev/null; then
+		echo --with-reduce requires libpmem.
+		echo Please install then re-run this script.
+		exit 1
+	fi
+fi
+
+if [[ "${CONFIG[VPP]}" = "y" ]]; then
+	if [ ! -z "${CONFIG[VPP_DIR]}" ]; then
+		VPP_CFLAGS="-L${CONFIG[VPP_DIR]}/lib -I${CONFIG[VPP_DIR]}/include"
+	fi
+	if ! echo -e '#include <vnet/session/application_interface.h>\nint main(void) { return 0; }\n' \
+		| ${BUILD_CMD[@]} ${VPP_CFLAGS} -lvppinfra -lsvm -lvlibmemoryclient - 2>/dev/null; then
+		echo --with-vpp requires installed vpp.
+		echo Please install then re-run this script.
 		exit 1
 	fi
 fi
 
 if [[ "${CONFIG[NVME_CUSE]}" = "y" ]]; then
 	if ! echo -e '#define FUSE_USE_VERSION 31\n#include <fuse3/cuse_lowlevel.h>\n#include <fuse3/fuse_lowlevel.h>\n#include <fuse3/fuse_opt.h>\nint main(void) { return 0; }\n' \
-		| "${BUILD_CMD[@]}" -lfuse3 -D_FILE_OFFSET_BITS=64 - 2> /dev/null; then
-		echo "--with-cuse requires libfuse3."
-		echo "Please install then re-run this script."
+		| ${BUILD_CMD[@]} -lfuse3 -D_FILE_OFFSET_BITS=64 - 2>/dev/null; then
+		echo --with-cuse requires libfuse3.
+		echo Please install then re-run this script.
 		exit 1
 	fi
 fi
@@ -708,9 +712,9 @@ fi
 if [[ "${CONFIG[RBD]}" = "y" ]]; then
 	if ! echo -e '#include <rbd/librbd.h>\n#include <rados/librados.h>\n' \
 		'int main(void) { return 0; }\n' \
-		| "${BUILD_CMD[@]}" -lrados -lrbd - 2> /dev/null; then
-		echo "--with-rbd requires librados and librbd."
-		echo "Please install then re-run this script."
+		| ${BUILD_CMD[@]} -lrados -lrbd - 2>/dev/null; then
+		echo --with-rbd requires librados and librbd.
+		echo Please install then re-run this script.
 		exit 1
 	fi
 fi
@@ -722,39 +726,46 @@ if [[ "${CONFIG[ISCSI_INITIATOR]}" = "y" ]]; then
 		'#error\n' \
 		'#endif\n' \
 		'int main(void) { return 0; }\n' \
-		| "${BUILD_CMD[@]}" -L/usr/lib64/iscsi -liscsi - 2> /dev/null; then
-		echo "--with-iscsi-initiator requires libiscsi with"
-		echo "LIBISCSI_API_VERSION >= 20150621."
-		echo "Please install then re-run this script."
+		| ${BUILD_CMD[@]} -L/usr/lib64/iscsi -liscsi - 2>/dev/null; then
+		echo --with-iscsi-initiator requires libiscsi with
+		echo 'LIBISCSI_API_VERSION >= 20150621.'
+		echo Please install then re-run this script.
+		exit 1
+	fi
+fi
+
+if [[ "${CONFIG[LOG_BACKTRACE]}" = "y" ]]; then
+	if ! echo -e '#include <libunwind.h>\nint main(void) { return 0; }\n' \
+		| ${BUILD_CMD[@]} -lunwind - 2>/dev/null; then
+		echo --enable-log-bt requires libunwind.
+		echo Please install then re-run this script.
 		exit 1
 	fi
 fi
 
 if [[ "${CONFIG[ASAN]}" = "y" ]]; then
 	if ! echo -e 'int main(void) { return 0; }\n' \
-		| "${BUILD_CMD[@]}" -fsanitize=address - 2> /dev/null; then
-		echo "--enable-asan requires libasan."
-		echo "Please install then re-run this script."
+		| ${BUILD_CMD[@]} -fsanitize=address - 2>/dev/null; then
+		echo --enable-asan requires libasan.
+		echo Please install then re-run this script.
 		exit 1
 	fi
 fi
 
 if [[ "${CONFIG[UBSAN]}" = "y" ]]; then
 	if ! echo -e 'int main(void) { return 0; }\n' \
-		| "${BUILD_CMD[@]}" -fsanitize=undefined - 2> /dev/null; then
-		echo "--enable-ubsan requires libubsan."
-		echo "Please install then re-run this script."
-		echo "If installed, please check that the GCC version is at least 6.4"
-		echo "and synchronize CC accordingly."
+		| ${BUILD_CMD[@]} -fsanitize=undefined - 2>/dev/null; then
+		echo --enable-ubsan requires libubsan.
+		echo Please install then re-run this script.
 		exit 1
 	fi
 fi
 
 if [[ "${CONFIG[TSAN]}" = "y" ]]; then
 	if ! echo -e 'int main(void) { return 0; }\n' \
-		| "${BUILD_CMD[@]}" -fsanitize=thread - 2> /dev/null; then
-		echo "--enable-tsan requires libtsan."
-		echo "Please install then re-run this script."
+		| ${BUILD_CMD[@]} -fsanitize=thread - 2>/dev/null; then
+		echo --enable-tsan requires libtsan.
+		echo Please install then re-run this script.
 		exit 1
 	fi
 fi
@@ -785,11 +796,6 @@ if [[ "${CONFIG[URING]}" = "y" ]]; then
 			echo "${CONFIG[URING_PATH]}: directory not found"
 			exit 1
 		fi
-	elif ! echo -e '#include <liburing.h>\nint main(void) { return 0; }\n' \
-		| "${BUILD_CMD[@]}" -luring - 2> /dev/null; then
-		echo "--with-uring requires liburing."
-		echo "Please build and install then re-run this script."
-		exit 1
 	fi
 fi
 
@@ -801,51 +807,22 @@ if [[ "${CONFIG[FUSE]}" = "y" ]]; then
 	fi
 fi
 
-if [ "${CONFIG[CET]}" = "y" ]; then
-	if ! echo -e 'int main(void) { return 0; }\n' | "${BUILD_CMD[@]}" -fcf-protection - 2> /dev/null; then
-		echo "--enable-cet requires compiler/linker that supports CET."
-		echo "Please install then re-run this script."
-		exit 1
-	fi
-fi
-
-if [[ "${CONFIG[ISAL]}" = "y" ]]; then
-	if [ ! -f "$rootdir"/isa-l/autogen.sh ]; then
-		echo "ISA-L was not found; To install ISA-L run:"
-		echo "  git submodule update --init"
-		exit 1
-	fi
-
-	cd $rootdir/isa-l
-	ISAL_LOG=$rootdir/isa-l/spdk-isal.log
-	if [[ -n "${CONFIG[CROSS_PREFIX]}" ]]; then
-		ISAL_OPTS=("--host=${CONFIG[CROSS_PREFIX]}")
-	else
-		ISAL_OPTS=()
-	fi
-	echo -n "Configuring ISA-L (logfile: $ISAL_LOG)..."
-	./autogen.sh &> $ISAL_LOG
-	./configure CFLAGS="-fPIC -g -O2" "${ISAL_OPTS[@]}" --enable-shared=no >> $ISAL_LOG 2>&1
-	echo "done."
-	cd $rootdir
-fi
-
 # We are now ready to generate final configuration. But first do sanity
 # check to see if all keys in CONFIG array have its reflection in CONFIG file.
-if (($(grep -cE "^\s*CONFIG_[[:alnum:]_]+=" "$rootdir/CONFIG") != ${#CONFIG[@]})); then
+if [ $(egrep -c "^\s*CONFIG_[[:alnum:]_]+=" $rootdir/CONFIG) -ne ${#CONFIG[@]} ]; then
 	echo ""
 	echo "BUG: Some configuration options are not present in CONFIG file. Please update this file."
 	echo "Missing options in CONFIG (+) file and in current config (-): "
 	diff -u --label "CONFIG file" --label "CONFIG[@]" \
 		<(sed -r -e '/^\s*$/d; /^\s*#.*/d; s/(CONFIG_[[:alnum:]_]+)=.*/\1/g' CONFIG | sort) \
-		<(printf "CONFIG_%s\n" "${!CONFIG[@]}" | sort)
+		<(printf "CONFIG_%s\n" ${!CONFIG[@]} | sort)
 	exit 1
 fi
 
 echo -n "Creating mk/config.mk..."
 cp -f $rootdir/CONFIG $rootdir/mk/config.mk
-for key in "${!CONFIG[@]}"; do
-	sed -i.bak -r "s#[[:space:]]*CONFIG_${key}=.*#CONFIG_${key}\?=${CONFIG[$key]}#g" $rootdir/mk/config.mk
+for key in ${!CONFIG[@]}; do
+	sed -i.bak -r "s#^\s*CONFIG_${key}=.*#CONFIG_${key}\?=${CONFIG[$key]}#g" $rootdir/mk/config.mk
 done
 # On FreeBSD sed -i 'SUFFIX' - SUFFIX is mandatory. So no way but to delete the backed file.
 rm -f $rootdir/mk/config.mk.bak
@@ -863,7 +840,7 @@ echo "done."
 # Create .sh with build config for easy sourcing|lookup during the tests.
 for conf in "${!CONFIG[@]}"; do
 	echo "CONFIG_$conf=${CONFIG[$conf]}"
-done > "$rootdir/test/common/build_config.sh"
+done >"$rootdir/test/common/build_config.sh"
 
 if [[ $sys_name == "FreeBSD" ]]; then
 	echo "Type 'gmake' to build."

deprecation.md

@@ -1,42 +0,0 @@
-# ABI and API Deprecation {#deprecation}
-
-This document details the policy for maintaining stability of SPDK ABI and API.
-
-Major ABI version can change at most once for each quarterly SPDK release.
-ABI versions are managed separately for each library and follow [Semantic Versoning](https://semver.org/).
-
-API and ABI deprecation notices shall be posted in the next section.
-Each entry must describe what will be removed and can suggest the future use or alternative.
-Specific future SPDK release for the removal must be provided.
-ABI cannot be removed without providing deprecation notice for at least single SPDK release.
-
-# Deprecation Notices {#deprecation-notices}
-
-## net
-
-The net library is deprecated and will be removed in the 21.07 release.
-
-## nvmf
-
-The following APIs have been deprecated and will be removed in SPDK 21.07:
-- `spdk_nvmf_poll_group_get_stat` (function in `nvmf.h`),
-- `spdk_nvmf_transport_poll_group_get_stat` (function in `nvmf.h`),
-- `spdk_nvmf_transport_poll_group_free_stat`(function in `nvmf.h`),
-- `spdk_nvmf_rdma_device_stat` (struct in `nvmf.h`),
-- `spdk_nvmf_transport_poll_group_stat` (struct in `nvmf.h`),
-- `poll_group_get_stat` (transport op in `nvmf_transport.h`),
-- `poll_group_free_stat` (transport op in `nvmf_transport.h`).
-Please use `spdk_nvmf_poll_group_dump_stat` and `poll_group_dump_stat` instead.
-
-## rpc
-
-Parameter `enable-zerocopy-send` of RPC `sock_impl_set_options` is deprecated and will be removed in SPDK 21.07,
-use `enable-zerocopy-send-server` or `enable-zerocopy-send-client` instead.
-Parameter `disable-zerocopy-send` of RPC `sock_impl_set_options` is deprecated and will be removed in SPDK 21.07,
-use `disable-zerocopy-send-server` or `disable-zerocopy-send-client` instead.
-
-## rpm
-
-`pkg/spdk.spec` is considered to be deprecated and scheduled for removal in SPDK 21.07.
-Please use `rpmbuild/spdk.spec` instead and see
-[RPM documentation](https://spdk.io/doc/rpm.html) for more details.

doc/Doxyfile

@@ -234,7 +234,7 @@ ALIASES                =
 # A mapping has the form "name=value". For example adding "class=itcl::class"
 # will allow you to use the command class in the itcl::class meaning.
 
-# TCL_SUBST              =
+TCL_SUBST              =
 
 # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
 # only. Doxygen will then generate output that is more tailored for C. For
@@ -795,16 +795,13 @@ INPUT                  += \
                          misc.md \
                          driver_modules.md \
                          tools.md \
-                         ci_tools.md \
                          performance_reports.md \
 
 # All remaining pages are listed here in alphabetical order by filename.
 INPUT                  += \
                          about.md \
-                         accel_fw.md \
                          applications.md \
                          bdev.md \
-                         bdevperf.md \
                          bdev_module.md \
                          bdev_pg.md \
                          blob.md \
@@ -813,7 +810,6 @@ INPUT                  += \
                          compression.md \
                          concurrency.md \
                          containers.md \
-                         ../deprecation.md \
                          event.md \
                          ftl.md \
                          gdb_macros.md \
@@ -823,24 +819,19 @@ INPUT                  += \
                          iscsi.md \
                          jsonrpc.md \
                          jsonrpc_proxy.md \
-                         libraries.md \
                          lvol.md \
                          memory.md \
                          notify.md \
                          nvme.md \
+                         nvme-cli.md \
                          nvme_spec.md \
                          nvmf.md \
                          nvmf_tgt_pg.md \
                          nvmf_tracing.md \
                          overview.md \
                          peer_2_peer.md \
-                         pkgconfig.md \
                          porting.md \
-                         rpm.md \
-                         scheduler.md \
-                         shfmt.md \
                          spdkcli.md \
-                         spdk_top.md \
                          ssd_internals.md \
                          system_configuration.md \
                          userspace.md \
@@ -848,7 +839,8 @@ INPUT                  += \
                          vhost.md \
                          vhost_processing.md \
                          virtio.md \
-                         vmd.md
+                         vmd.md \
+                         vpp_integration.md
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
@@ -1105,7 +1097,7 @@ ALPHABETICAL_INDEX     = YES
 # Minimum value: 1, maximum value: 20, default value: 5.
 # This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
 
-# COLS_IN_ALPHA_INDEX    = 5
+COLS_IN_ALPHA_INDEX    = 5
 
 # In case all classes in a project start with a common prefix, all classes will
 # be put under the same header in the alphabetical index. The IGNORE_PREFIX tag
@@ -1666,7 +1658,7 @@ EXTRA_SEARCH_MAPPINGS  =
 # If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
 # The default value is: YES.
 
-GENERATE_LATEX         = NO
+GENERATE_LATEX         = YES
 
 # The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a
 # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
@@ -2170,7 +2162,7 @@ EXTERNAL_PAGES         = YES
 # interpreter (i.e. the result of 'which perl').
 # The default file (with absolute path) is: /usr/bin/perl.
 
-# PERL_PATH              = /usr/bin/perl
+PERL_PATH              = /usr/bin/perl
 
 #---------------------------------------------------------------------------
 # Configuration options related to the dot tool
@@ -2192,7 +2184,7 @@ CLASS_DIAGRAMS         = YES
 # the mscgen tool resides. If left empty the tool is assumed to be found in the
 # default search path.
 
-# MSCGEN_PATH            =
+MSCGEN_PATH            =
 
 # You can include diagrams made with dia in doxygen documentation. Doxygen will
 # then run dia to produce the diagram and insert it in the documentation. The

doc/accel_fw.md

@@ -1,107 +0,0 @@
-# Acceleration Framework {#accel_fw}
-
-SPDK provides a framework for abstracting general acceleration capabilities
-that can be implemented through plug-in modules and low-level libraries. These
-plug-in modules include support for hardware acceleration engines such as
-the Intel(R) I/O Acceleration Technology (IOAT) engine and the Intel(R) Data
-Streaming Accelerator (DSA) engine. Additionally, a software plug-in module
-exists to enable use of the framework in environments without hardware
-acceleration capabilities. ISA/L is used for optimized CRC32C calculation within
-the software module.
-
-The framework includes an API for getting the current capabilities of the
-selected module. See [`spdk_accel_get_capabilities`](https://spdk.io/doc/accel__engine_8h.html) for more details. For the software module, all capabilities will be reported as supported. For the hardware modules, only functions accelerated by hardware will be reported however any function can still be called, it will just be backed by software if it is not reported as a supported capability.
-
-# Acceleration Framework Functions {#accel_functions}
-
-Functions implemented via the framework can be found in the DoxyGen documentation of the
-framework public header file here [accel_engine.h](https://spdk.io/doc/accel__engine_8h.html)
-
-# Acceleration Framework Design Considerations {#accel_dc}
-
-The general interface is defined by `/include/accel_engine.h` and implemented
-in `/lib/accel`.  These functions may be called by an SPDK application and in
-most cases, except where otherwise documented, are asynchronous and follow the
-standard SPDK model for callbacks with a callback argument.
-
-If the acceleration framework is started without initializing a hardware module,
-optimized software implementations of the functions will back the public API.
-Additionally, if any hardware module does not support a specific function and that
-hardware module is initialized, the specific function will fallback to a software
-optimized implementation.  For example, IOAT does not support the dualcast function
-in hardware but if the IOAT module has been initialized and the public dualcast API
-is called, it will actually be done via software behind the scenes.
-
-# Acceleration Low Level Libraries {#accel_libs}
-
-Low level libraries provide only the most basic functions that are specific to
-the hardware. Low level libraries are located in the '/lib' directory with the
-exception of the software implementation which is implemented as part of the
-framework itself. The software low level library does not expose a public API.
-Applications may choose to interact directly with a low level library if there are
-specific needs/considerations not met via accessing the library through the
-framework/module. Note that when using the low level libraries directly, the
-framework abstracted interface is bypassed as the application will call the public
-functions exposed by the individual low level libraries. Thus, code written this
-way needs to be certain that the underlying hardware exists everywhere that it runs.
-
-The low level library for IOAT is located in `/lib/ioat`.  The low level library
-for DSA is in `/liv/idxd` (IDXD stands for Intel(R) Data Acceleration Driver).
-
-# Acceleration Plug-In Modules {#accel_modules}
-
-Plug-in modules depend on low level libraries to interact with the hardware and
-add additional functionality such as queueing during busy conditions or flow
-control in some cases. The framework in turn depends on the modules to provide
-the complete implementation of the acceleration component. A module must be
-selected via startup RPC when the application is started. Otherwise, if no startup
-RPC is provided, the framework is available and will use the software plug-in module.
-
-## IOAT Module {#accel_ioat}
-
-To use the IOAT engine, use the RPC [`ioat_scan_accel_engine`](https://spdk.io/doc/jsonrpc.html) before starting the application.
-
-## IDXD Module {#accel_idxd}
-
-To use the DSA engine, use the RPC [`idxd_scan_accel_engine`](https://spdk.io/doc/jsonrpc.html) with an optional parameter of `-c` and provide a configuration number of either 0 or 1. These pre-defined configurations determine how the DSA engine will be setup in terms
-of work queues and engines.  The DSA engine is very flexible allowing for various configurations of these elements to either account for different quality of service requirements or to isolate hardware paths where the back end media is of varying latency (i.e. persistent memory vs DRAM).  The pre-defined configurations are as follows:
-
-0: A single work queue backed with four DSA engines.  This is a generic configuration
-that enables the hardware to best determine which engine to use as it pulls in new
-operations.
-
-1: Two separate work queues each backed with two DSA engines. This is another
-generic configuration that is documented in the specification and allows the
-application to partition submissions across two work queues. This would be useful
-when different priorities might be desired per group.
-
-There are several other configurations that are possible that include quality
-of service parameters on the work queues that are not currently utilized by
-the module. Specialized use of DSA may require different configurations that
-can be added to the module as needed.
-
-## Software Module {#accel_sw}
-
-The software module is enabled by default. If no hardware engine is explicitly
-enabled via startup RPC as discussed earlier, the software module will use ISA-L
-if available for functions such as CRC32C. Otherwise, standard glibc calls are
-used to back the framework API.
-
-## Batching {#batching}
-
-Batching is exposed by the acceleration framework and provides an interface to
-batch sets of commands up and then submit them with a single command.  The public
-API is consistent with the implementation however each plug-in module behaves
-differently depending on its capabilities.
-
-The DSA engine has complete support for batching all supported commands together
-into one submission. This is advantageous as it reduces the overhead incurred in
-the submission process to the hardware.
-
-The software engine supports batching only to be consistent with the framework API.
-In software there is no savings by batching sets of commands versus submitting them
-individually.
-
-The IOAT engine supports batching but it is only beneficial for `memmove` and `memfill`
-as these are supported by the hardware.  All other commands can be batched and the
-framework will manage all other commands via software.

doc/applications.md

@@ -35,22 +35,25 @@ Param    | Long Param             | Type     | Default                | Descript
 -i       | --shm-id               | integer  |                        | @ref cmd_arg_multi_process
 -m       | --cpumask              | CPU mask | 0x1                    | application @ref cpu_mask
 -n       | --mem-channels         | integer  | all channels           | number of memory channels used for DPDK
--p       | --main-core            | integer  | first core in CPU mask | main (primary) core for DPDK
+-p       | --master-core          | integer  | first core in CPU mask | master (primary) core for DPDK
 -r       | --rpc-socket           | string   | /var/tmp/spdk.sock     | RPC listen address
 -s       | --mem-size             | integer  | all hugepage memory    | @ref cmd_arg_memory_size
 |        | --silence-noticelog    | flag     |                        | disable notice level logging to `stderr`
 -u       | --no-pci               | flag     |                        | @ref cmd_arg_disable_pci_access.
 |        | --wait-for-rpc         | flag     |                        | @ref cmd_arg_deferred_initialization
--B       | --pci-blocked          | B:D:F    |                        | @ref cmd_arg_pci_blocked_allowed.
--A       | --pci-allowed          | B:D:F    |                        | @ref cmd_arg_pci_blocked_allowed.
+-B       | --pci-blacklist        | B:D:F    |                        | @ref cmd_arg_pci_blacklist_whitelist.
+-W       | --pci-whitelist        | B:D:F    |                        | @ref cmd_arg_pci_blacklist_whitelist.
 -R       | --huge-unlink          | flag     |                        | @ref cmd_arg_huge_unlink
 |        | --huge-dir             | string   | the first discovered   | allocate hugepages from a specific mount
--L       | --logflag              | string   |                        | @ref cmd_arg_log_flags
+-L       | --logflag              | string   |                        | @ref cmd_arg_debug_log_flags
 
 ### Configuration file {#cmd_arg_config_file}
 
-SPDK applications are configured using a JSON RPC configuration file.
-See @ref jsonrpc for details.
+Historically, the SPDK applications were configured using a configuration file.
+This is still supported, but is considered deprecated in favor of JSON RPC
+configuration. See @ref jsonrpc for details.
+
+Note that `--config` and `--wait-for-rpc` cannot be used at the same time.
 
 ### Limit coredump {#cmd_arg_limit_coredump}
 
@@ -121,12 +124,12 @@ If SPDK is run with PCI access disabled it won't detect any PCI devices. This
 includes primarily NVMe and IOAT devices. Also, the VFIO and UIO kernel modules
 are not required in this mode.
 
-### PCI address blocked and allowed lists {#cmd_arg_pci_blocked_allowed}
+### PCI address blacklist and whitelist {#cmd_arg_pci_blacklist_whitelist}
 
-If blocked list is used, then all devices with the provided PCI address will be
-ignored. If an allowed list is used, only allowed devices will be probed.
-`-B` or `-A` can be used more than once, but cannot be mixed together. That is,
-`-B` and `-A` cannot be used at the same time.
+If blacklist is used, then all devices with the provided PCI address will be
+ignored. If a whitelist is used, only whitelisted devices will be probed.
+`-B` or `-W` can be used more than once, but cannot be mixed together. That is,
+`-B` and `-W` cannot be used at the same time.
 
 ### Unlink hugepage files after initialization {#cmd_arg_huge_unlink}
 
@@ -134,11 +137,11 @@ By default, each DPDK-based application tries to remove any orphaned hugetlbfs
 files during its initialization. This option removes hugetlbfs files of the current
 process as soon as they're created, but is not compatible with `--shm-id`.
 
-### Log flag {#cmd_arg_log_flags}
+### Debug log {#cmd_arg_debug_log_flags}
 
-Enable a specific log type. This option can be used more than once. A list of
+Enable a specific debug log type. This option can be used more than once. A list of
 all available types is provided in the `--help` output, with `--logflag all`
-enabling all of them. Additionally enables debug print level in debug builds of SPDK.
+enabling all of them. Debug logs are only available in debug builds of SPDK.
 
 ## CPU mask {#cpu_mask}
 

doc/bdev.md

@@ -1,9 +1,5 @@
 # Block Device User Guide {#bdev}
 
-# Target Audience {#bdev_ug_targetaudience}
-
-This user guide is intended for software developers who have knowledge of block storage, storage drivers, issuing JSON-RPC commands and storage services such as RAID, compression, crypto, and others.
-
 # Introduction {#bdev_ug_introduction}
 
 The SPDK block device layer, often simply called *bdev*, is a C library
@@ -39,12 +35,72 @@ directly from SPDK application by running `scripts/rpc.py rpc_get_methods`.
 Detailed help for each command can be displayed by adding `-h` flag as a
 command parameter.
 
-# Configuring Block Device Modules {#bdev_ug_general_rpcs}
+# General Purpose RPCs {#bdev_ug_general_rpcs}
 
-Block devices can be configured using JSON RPCs. A complete list of available RPC commands
-with detailed information can be found on the @ref jsonrpc_components_bdev page.
+## bdev_get_bdevs {#bdev_ug_get_bdevs}
 
-# Common Block Device Configuration Examples
+List of currently available block devices including detailed information about
+them can be get by using `bdev_get_bdevs` RPC command. User can add optional
+parameter `name` to get details about specified by that name bdev.
+
+Example response
+
+~~~
+{
+  "num_blocks": 32768,
+  "assigned_rate_limits": {
+    "rw_ios_per_sec": 10000,
+    "rw_mbytes_per_sec": 20
+  },
+  "supported_io_types": {
+    "reset": true,
+    "nvme_admin": false,
+    "unmap": true,
+    "read": true,
+    "write_zeroes": true,
+    "write": true,
+    "flush": true,
+    "nvme_io": false
+  },
+  "driver_specific": {},
+  "claimed": false,
+  "block_size": 4096,
+  "product_name": "Malloc disk",
+  "name": "Malloc0"
+}
+~~~
+
+## bdev_set_qos_limit {#bdev_set_qos_limit}
+
+Users can use the `bdev_set_qos_limit` RPC command to enable, adjust, and disable
+rate limits on an existing bdev.  Two types of rate limits are supported:
+IOPS and bandwidth.  The rate limits can be enabled, adjusted, and disabled at any
+time for the specified bdev.  The bdev name is a required parameter for this
+RPC command and at least one of `rw_ios_per_sec` and `rw_mbytes_per_sec` must be
+specified.  When both rate limits are enabled, the first met limit will
+take effect.  The value 0 may be specified to disable the corresponding rate
+limit. Users can run this command with `-h` or `--help` for more information.
+
+## Histograms {#rpc_bdev_histogram}
+
+The `bdev_enable_histogram` RPC command allows to enable or disable gathering
+latency data for specified bdev. Histogram can be downloaded by the user by
+calling `bdev_get_histogram` and parsed using scripts/histogram.py script.
+
+Example command
+
+`rpc.py bdev_enable_histogram Nvme0n1 --enable`
+
+The command will enable gathering data for histogram on Nvme0n1 device.
+
+`rpc.py bdev_get_histogram Nvme0n1 | histogram.py`
+
+The command will download gathered histogram data. The script will parse
+the data and show table containing IO count for latency ranges.
+
+`rpc.py bdev_enable_histogram Nvme0n1 --disable`
+
+The command will disable histogram on Nvme0n1 device.
 
 # Ceph RBD {#bdev_config_rbd}
 
@@ -205,7 +261,7 @@ possibly multiple virtual bdevs.
 ## SPDK GPT partition table {#bdev_ug_gpt}
 
 The SPDK partition type GUID is `7c5222bd-8f5d-4087-9c00-bf9843c7b58c`. Existing SPDK bdevs
-can be exposed as Linux block devices via NBD and then can be partitioned with
+can be exposed as Linux block devices via NBD and then ca be partitioned with
 standard partitioning tools. After partitioning, the bdevs will need to be deleted and
 attached again for the GPT bdev module to see any changes. NBD kernel module must be
 loaded first. To create NBD bdev user should use `nbd_start_disk` RPC command.
@@ -280,9 +336,9 @@ Example commands
 
 This command will create `aio0` device from /dev/sda.
 
-`rpc.py bdev_aio_create /tmp/file file 4096`
+`rpc.py bdev_aio_create /tmp/file file 8192`
 
-This command will create `file` device with block size 4096 from /tmp/file.
+This command will create `file` device with block size 8192 from /tmp/file.
 
 To delete an aio bdev use the bdev_aio_delete command.
 
@@ -312,22 +368,16 @@ To remove `Cache1`:
 
 During removal OCF-cache will be stopped and all cached data will be written to the core device.
 
-Note that OCF has a per-device RAM requirement. More details can be found in the
-[OCF documentation](https://open-cas.github.io/guide_system_requirements.html).
+Note that OCF has a per-device RAM requirement
+of about 56000 + _cache device size_ * 58 / _cache line size_ (in bytes).
+To get more information on OCF
+please visit [OCF documentation](https://open-cas.github.io/).
 
 # Malloc bdev {#bdev_config_malloc}
 
 Malloc bdevs are ramdisks. Because of its nature they are volatile. They are created from hugepage memory given to SPDK
 application.
 
-Example command for creating malloc bdev:
-
-`rpc.py bdev_malloc_create -b Malloc0 64 512`
-
-Example command for removing malloc bdev:
-
-`rpc.py bdev_malloc_delete Malloc0`
-
 # Null {#bdev_config_null}
 
 The SPDK null bdev driver is a dummy block I/O target that discards all writes and returns undefined
@@ -369,22 +419,15 @@ This command will remove NVMe bdev named Nvme0.
 
 ## NVMe bdev character device {#bdev_config_nvme_cuse}
 
-This feature is considered as experimental. You must configure with --with-nvme-cuse
-option to enable this RPC.
+This feature is considered as experimental.
 
 Example commands
 
-`rpc.py bdev_nvme_cuse_register -n Nvme3
+`rpc.py bdev_nvme_cuse_register -n Nvme0 -p spdk/nvme0`
 
-This command will register a character device under /dev/spdk associated with Nvme3
-controller. If there are namespaces created on Nvme3 controller, a namespace
-character device is also created for each namespace.
-
-For example, the first controller registered will have a character device path of
-/dev/spdk/nvmeX, where X is replaced with a unique integer to differentiate it from
-other controllers.  Note that this 'nvmeX' name here has no correlation to the name
-associated with the controller in SPDK.  Namespace character devices will have a path
-of /dev/spdk/nvmeXnY, where Y is the namespace ID.
+This command will register /dev/spdk/nvme0 character device associated with Nvme0
+controller. If there are namespaces created on Nvme0 controller, for each namespace
+device /dev/spdk/nvme0nX is created.
 
 Cuse devices are removed from system, when NVMe controller is detached or unregistered
 with command:
@@ -417,6 +460,7 @@ User can get list of available lvol stores using `bdev_lvol_get_lvstores` RPC co
 parameters available).
 
 Example response
+
 ~~~
 {
   "uuid": "330a6ab2-f468-11e7-983e-001e67edf35d",
@@ -448,6 +492,26 @@ Example commands
 
 `rpc.py bdev_lvol_create lvol2 25 -u 330a6ab2-f468-11e7-983e-001e67edf35d`
 
+# RAID {#bdev_ug_raid}
+
+RAID virtual bdev module provides functionality to combine any SPDK bdevs into
+one RAID bdev. Currently SPDK supports only RAID 0. RAID functionality does not
+store on-disk metadata on the member disks, so user must recreate the RAID
+volume when restarting application. User may specify member disks to create RAID
+volume event if they do not exists yet - as the member disks are registered at
+a later time, the RAID module will claim them and will surface the RAID volume
+after all of the member disks are available. It is allowed to use disks of
+different sizes - the smallest disk size will be the amount of space used on
+each member disk.
+
+Example commands
+
+`rpc.py bdev_raid_create -n Raid0 -z 64 -r 0 -b "lvol0 lvol1 lvol2 lvol3"`
+
+`rpc.py bdev_raid_get_bdevs`
+
+`rpc.py bdev_raid_delete Raid0`
+
 # Passthru {#bdev_config_passthru}
 
 The SPDK Passthru virtual block device module serves as an example of how to write a
@@ -497,65 +561,6 @@ To remove a block device representation use the bdev_pmem_delete command.
 
 `rpc.py bdev_pmem_delete pmem`
 
-# RAID {#bdev_ug_raid}
-
-RAID virtual bdev module provides functionality to combine any SPDK bdevs into
-one RAID bdev. Currently SPDK supports only RAID 0. RAID functionality does not
-store on-disk metadata on the member disks, so user must recreate the RAID
-volume when restarting application. User may specify member disks to create RAID
-volume event if they do not exists yet - as the member disks are registered at
-a later time, the RAID module will claim them and will surface the RAID volume
-after all of the member disks are available. It is allowed to use disks of
-different sizes - the smallest disk size will be the amount of space used on
-each member disk.
-
-Example commands
-
-`rpc.py bdev_raid_create -n Raid0 -z 64 -r 0 -b "lvol0 lvol1 lvol2 lvol3"`
-
-`rpc.py bdev_raid_get_bdevs`
-
-`rpc.py bdev_raid_delete Raid0`
-
-# Split {#bdev_ug_split}
-
-The split block device module takes an underlying block device and splits it into
-several smaller equal-sized virtual block devices. This serves as an example to create
-more vbdevs on a given base bdev for user testing.
-
-Example commands
-
-To create four split bdevs with base bdev_b0 use the `bdev_split_create` command.
-Each split bdev will be one fourth the size of the base bdev.
-
-`rpc.py bdev_split_create bdev_b0 4`
-
-The `split_size_mb`(-s) parameter restricts the size of each split bdev.
-The total size of all split bdevs must not exceed the base bdev size.
-
-`rpc.py bdev_split_create bdev_b0 4 -s 128`
-
-To remove the split bdevs, use the `bdev_split_delete` command with the base bdev name.
-
-`rpc.py bdev_split_delete bdev_b0`
-
-# Uring {#bdev_ug_uring}
-
-The uring bdev module issues I/O to kernel block devices using the io_uring Linux kernel API. This module requires liburing.
-For more information on io_uring refer to kernel [IO_uring] (https://kernel.dk/io_uring.pdf)
-
-The user needs to configure SPDK to include io_uring support:
-
-`configure --with-uring`
-
-To create a uring bdev with given filename, bdev name and block size use the `bdev_uring_create` RPC.
-
-`rpc.py  bdev_uring_create /path/to/device bdev_u0 512`
-
-To remove a uring bdev use the `bdev_uring_delete` RPC.
-
-`rpc.py bdev_uring_delete bdev_u0`
-
 # Virtio Block {#bdev_config_virtio_blk}
 
 The Virtio-Block driver allows creating SPDK bdevs from Virtio-Block devices.

doc/bdev_module.md

@@ -18,7 +18,7 @@ how to write a module.
 
 ## Creating A New Module
 
-Block device modules are located in subdirectories under module/bdev today. It is not
+Block device modules are located in subdirectories under lib/bdev today. It is not
 currently possible to place the code for a bdev module elsewhere, but updates
 to the build system could be made to enable this in the future. To create a
 module, add a new directory with a single C file and a Makefile. A great
@@ -137,15 +137,6 @@ block device. Once the I/O request is completed, the module must call
 spdk_bdev_io_complete(). The I/O does not have to finish within the calling
 context of `submit_request`.
 
-Integrating a new bdev module into the build system requires updates to various
-files in the /mk directory.
-
-## Creating Bdevs in an External Repository
-
-A User can build their own bdev module and application on top of existing SPDK libraries. The example in
-test/external_code serves as a template for creating, building and linking an external
-bdev module. Refer to test/external_code/README.md and @ref so_linking for further information.
-
 ## Creating Virtual Bdevs
 
 Block devices are considered virtual if they handle I/O requests by routing
@@ -153,7 +144,7 @@ the I/O to other block devices. The canonical example would be a bdev module
 that implements RAID. Virtual bdevs are created in the same way as regular
 bdevs, but take one additional step. The module can look up the underlying
 bdevs it wishes to route I/O to using spdk_bdev_get_by_name(), where the string
-name is provided by the user via an RPC. The module
+name is provided by the user in a configuration file or via an RPC. The module
 then may proceed is normal by opening the bdev to obtain a descriptor, and
 creating I/O channels for the bdev (probably in response to the
 `get_io_channel` callback). The final step is to have the module use its open

doc/bdevperf.md

@@ -1,86 +0,0 @@
-# Using bdevperf application {#bdevperf}
-
-## Introduction
-
-bdevperf is an SPDK application that is used for performance testing
-of block devices (bdevs) exposed by the SPDK bdev layer.  It is an
-alternative to the SPDK bdev fio plugin for benchmarking SPDK bdevs.
-In some cases, bdevperf can provide much lower overhead than the fio
-plugin, resulting in much better performance for tests using a limited
-number of CPU cores.
-
-bdevperf exposes command line interface that allows to specify
-SPDK framework options as well as testing options.
-Since SPDK 20.07, bdevperf supports configuration file that is similar
-to FIO. It allows user to create jobs parameterized by
-filename, cpumask, blocksize, queuesize, etc.
-
-## Config file
-
-Bdevperf's config file is similar to FIO's config file format.
-
-Below is an example config file that uses all available parameters:
-
-~~~{.ini}
-[global]
-filename=Malloc0:Malloc1
-bs=1024
-iosize=256
-rw=randrw
-rwmixread=90
-
-[A]
-cpumask=0xff
-
-[B]
-cpumask=[0-128]
-filename=Malloc1
-
-[global]
-filename=Malloc0
-rw=write
-
-[C]
-bs=4096
-iosize=128
-offset=1000000
-length=1000000
-~~~
-
-Jobs `[A]` `[B]` or `[C]`, inherit default values from `[global]`
-section residing above them. So in the example, job `[A]` inherits
-`filename` value and uses both `Malloc0` and `Malloc1` bdevs as targets,
-job `[B]` overrides its `filename` value and uses `Malloc1` and
-job `[C]` inherits value `Malloc0` for its `filename`.
-
-Interaction with CLI arguments is not the same as in FIO however.
-If bdevperf receives CLI argument, it overrides values
-of corresponding parameter for all `[global]` sections of config file.
-So if example config is used, specifying `-q` argument
-will make jobs `[A]` and `[B]` use its value.
-
-Below is a full list of supported parameters with descriptions.
-
-Param     | Default           | Description
---------- | ----------------- | -----------
-filename  |                   | Bdevs to use, separated by ":"
-cpumask   | Maximum available | CPU mask. Format is defined at @ref cpu_mask
-bs        |                   | Block size (io size)
-iodepth   |                   | Queue depth
-rwmixread | `50`              | Percentage of a mixed workload that should be reads
-offset    | `0`               | Start I/O at the provided offset on the bdev
-length    | 100% of bdev size | End I/O at `offset`+`length` on the bdev
-rw        |                   | Type of I/O pattern
-
-Available rw types:
-- read
-- randread
-- write
-- randwrite
-- verify
-- reset
-- unmap
-- write_zeroes
-- flush
-- rw
-- randrw

doc/blob.md

@@ -225,7 +225,7 @@ with SPDK API.
 ### Error Handling
 
 Asynchronous Blobstore callbacks all include an error number that should be checked; non-zero values
-indicate an error. Synchronous calls will typically return an error value if applicable.
+indicate and error. Synchronous calls will typically return an error value if applicable.
 
 ### Asynchronous API
 

doc/blobfs.md

@@ -14,30 +14,30 @@ make
 ~~~
 
 Clone the RocksDB repository from the SPDK GitHub fork into a separate directory.
-Make sure you check out the `6.15.fb` branch.
+Make sure you check out the `spdk-v5.14.3` branch.
 
 ~~~{.sh}
 cd ..
-git clone -b 6.15.fb https://github.com/spdk/rocksdb.git
+git clone -b spdk-v5.14.3 https://github.com/spdk/rocksdb.git
 ~~~
 
 Build RocksDB.  Only the `db_bench` benchmarking tool is integrated with BlobFS.
 
 ~~~{.sh}
 cd rocksdb
-make db_bench SPDK_DIR=relative_path/to/spdk
+make db_bench SPDK_DIR=path/to/spdk
 ~~~
 
 Or you can also add `DEBUG_LEVEL=0` for a release build (need to turn on `USE_RTTI`).
 
 ~~~{.sh}
-export USE_RTTI=1 && make db_bench DEBUG_LEVEL=0 SPDK_DIR=relative_path/to/spdk
+export USE_RTTI=1 && make db_bench DEBUG_LEVEL=0 SPDK_DIR=path/to/spdk
 ~~~
 
 Create an NVMe section in the configuration file using SPDK's `gen_nvme.sh` script.
 
 ~~~{.sh}
-scripts/gen_nvme.sh --json-with-subsystems > /usr/local/etc/spdk/rocksdb.json
+scripts/gen_nvme.sh > /usr/local/etc/spdk/rocksdb.conf
 ~~~
 
 Verify the configuration file has specified the correct NVMe SSD.
@@ -54,7 +54,7 @@ HUGEMEM=5120 scripts/setup.sh
 Create an empty SPDK blobfs for testing.
 
 ~~~{.sh}
-test/blobfs/mkfs/mkfs /usr/local/etc/spdk/rocksdb.json Nvme0n1
+test/blobfs/mkfs/mkfs /usr/local/etc/spdk/rocksdb.conf Nvme0n1
 ~~~
 
 At this point, RocksDB is ready for testing with SPDK.  Three `db_bench` parameters are used to configure SPDK:
@@ -74,7 +74,7 @@ BlobFS provides a FUSE plug-in to mount an SPDK BlobFS as a kernel filesystem fo
 The FUSE plug-in requires fuse3 and will be built automatically when fuse3 is detected on the system.
 
 ~~~{.sh}
-test/blobfs/fuse/fuse /usr/local/etc/spdk/rocksdb.json Nvme0n1 /mnt/fuse
+test/blobfs/fuse/fuse /usr/local/etc/spdk/rocksdb.conf Nvme0n1 /mnt/fuse
 ~~~
 
 Note that the FUSE plug-in has some limitations - see the list below.

doc/ci_tools.md

@@ -1,6 +0,0 @@
-# CI Tools {#ci_tools}
-
-Section describing tools used by CI to verify integrity of the submitted
-patches ([status](https://ci.spdk.io)).
-
-- @subpage shfmt

doc/concurrency.md

@@ -117,10 +117,14 @@ framework for all of the example applications it shipped with, in the interest
 of supporting the widest variety of frameworks possible. But the applications do
 of course require something that implements an asynchronous event loop in order
 to run, so enter the `event` framework located in `lib/event`. This framework
-includes things like polling and scheduling the lightweight threads, installing
-signal handlers to cleanly shutdown, and basic command line option parsing.
-Only established applications should consider directly integrating the lower
-level libraries.
+includes things like spawning one thread per core, pinning each thread to a
+unique core, polling and scheduling the lightweight threads, installing signal
+handlers to cleanly shutdown, and basic command line option parsing. When
+started through spdk_app_start(), the library automatically spawns all of the
+threads requested, pins them, and is ready for lightweight threads to be
+created. This makes it much easier to implement a brand new SPDK application and
+is the recommended method for those starting out. Only established applications
+should consider directly integrating the lower level libraries.
 
 # Limitations of the C Language
 

doc/containers.md

@@ -16,7 +16,7 @@ runtime with lightweight virtual machines that feel and perform like
 containers, but provide stronger workload isolation using hardware
 virtualization technology as a second layer of defense.
 
-From Kata Containers [1.11.0](https://github.com/kata-containers/runtime/releases/tag/1.11.0),
+From Kata Containers [1.11.0-alpha1](https://github.com/kata-containers/runtime/releases/tag/1.11.0-alpha1),
 vhost-user-blk support is enabled in `kata-containers/runtime`. That is to say
 SPDK vhost target can be used to provide volume service to Kata Containers directly.
 In addition, a container manager like Docker, can be configured easily to launch

doc/ftl.md

@@ -213,7 +213,7 @@ To verify that the drive is emulated correctly, one can check the output of the
 device):
 
 ```
-$ build/examples/identify
+$ examples/nvme/identify/identify
 =====================================================
 NVMe Controller at 0000:00:0a.0 [1d1d:1f1f]
 =====================================================

doc/general.md

@@ -1,6 +1,5 @@
 # General Information {#general}
 
 - @subpage event
-- @subpage scheduler
 - @subpage logical_volumes
-- @subpage accel_fw
+- @subpage vpp_integration

doc/getting_started.md

@@ -110,7 +110,7 @@ with no arguments to see the help output. If your system has its IOMMU
 enabled you can run the examples as your regular user. If it doesn't, you'll
 need to run as a privileged user (root).
 
-A good example to start with is `build/examples/identify`, which prints
+A good example to start with is `examples/nvme/identify/identify`, which prints
 out information about all of the NVMe devices on your system.
 
 Larger, more fully functional applications are available in the `app`

doc/idxd.md

@@ -9,14 +9,7 @@
 Function                                | Description
 --------------------------------------- | -----------
 spdk_idxd_probe()                       | @copybrief spdk_idxd_probe()
-spdk_idxd_batch_get_max()               | @copybrief spdk_idxd_batch_get_max()
-spdk_idxd_batch_create()                | @copybrief spdk_idxd_batch_create()
-spdk_idxd_batch_prep_copy()             | @copybrief spdk_idxd_batch_prep_copy()
-spdk_idxd_batch_submit()                | @copybrief spdk_idxd_batch_submit()
 spdk_idxd_submit_copy()                 | @copybrief spdk_idxd_submit_copy()
-spdk_idxd_submit_compare()              | @copybrief spdk_idxd_submit_compare()
-spdk_idxd_submit_crc32c()               | @copybrief spdk_idxd_submit_crc32c()
-spdk_idxd_submit_dualcast               | @copybrief spdk_idxd_submit_dualcast()
 spdk_idxd_submit_fill()                 | @copybrief spdk_idxd_submit_fill()
 
 # Pre-defined configurations {#idxd_configs}

doc/index.md

@@ -1,4 +1,4 @@
-# Storage Performance Development Kit {#mainpage}
+# Storage Performance Development Kit {#index}
 
 # Introduction
 
@@ -32,10 +32,6 @@
 
 @copydoc tools
 
-# CI Tools
-
-@copydoc ci_tools
-
 # Performance Reports
 
 @copydoc performance_reports

doc/intro.md

@@ -4,5 +4,4 @@
 - @subpage getting_started
 - @subpage vagrant
 - @subpage changelog
-- @subpage deprecation
 - [Source Code (GitHub)](https://github.com/spdk/spdk)

doc/iscsi.md

@@ -10,7 +10,7 @@ This following section describes how to run iscsi from your cloned package.
 This guide starts by assuming that you can already build the standard SPDK distribution on your
 platform.
 
-Once built, the binary will be in `build/bin`.
+Once built, the binary will be in `app/iscsi_tgt`.
 
 If you want to kill the application by using signal, make sure use the SIGTERM, then the application
 will release all the shared memory resource before exit, the SIGKILL will make the shared memory
@@ -23,6 +23,24 @@ document.
 
 ![iSCSI structure](iscsi.svg)
 
+## Configuring iSCSI Target via config file {#iscsi_config}
+
+A `iscsi_tgt` specific configuration file is used to configure the iSCSI target. A fully documented
+example configuration file is located at `etc/spdk/iscsi.conf.in`.
+
+The configuration file is used to configure the SPDK iSCSI target. This file defines the following:
+TCP ports to use as iSCSI portals; general iSCSI parameters; initiator names and addresses to allow
+access to iSCSI target nodes; number and types of storage backends to export over iSCSI LUNs; iSCSI
+target node mappings between portal groups, initiator groups, and LUNs.
+
+You should make a copy of the example configuration file, modify it to suit your environment, and
+then run the iscsi_tgt application and pass it the configuration file using the -c option. Right now,
+the target requires elevated privileges (root) to run.
+
+~~~
+app/iscsi_tgt/iscsi_tgt -c /path/to/iscsi.conf
+~~~
+
 ### Assigning CPU Cores to the iSCSI Target {#iscsi_config_lcore}
 
 SPDK uses the [DPDK Environment Abstraction Layer](http://dpdk.org/doc/guides/prog_guide/env_abstraction_layer.html)
@@ -39,9 +57,26 @@ command line option is used to configure the SPDK iSCSI target:
 This is a hexadecimal bit mask of the CPU cores where the iSCSI target will start polling threads.
 In this example, CPU cores 24, 25, 26 and 27 would be used.
 
+### Configuring a LUN in the iSCSI Target {#iscsi_lun}
+
+Each LUN in an iSCSI target node is associated with an SPDK block device.  See @ref bdev
+for details on configuring SPDK block devices.  The block device to LUN mappings are specified in the
+configuration file as:
+
+~~~~
+[TargetNodeX]
+  LUN0 Malloc0
+  LUN1 Nvme0n1
+~~~~
+
+This exports a malloc'd target. The disk is a RAM disk that is a chunk of memory allocated by iscsi in
+user space. It will use offload engine to do the copy job instead of memcpy if the system has enough DMA
+channels.
+
 ## Configuring iSCSI Target via RPC method {#iscsi_rpc}
 
-The iSCSI target is configured via JSON-RPC calls. See @ref jsonrpc for details.
+In addition to the configuration file, the iSCSI target may also be configured via JSON-RPC calls. See
+@ref jsonrpc for details.
 
 ### Portal groups
 
@@ -183,7 +218,7 @@ echo "1024" > /sys/block/sdc/queue/nr_requests
 
 ### Example: Configure simple iSCSI Target with one portal and two LUNs
 
-Assuming we have one iSCSI Target server with portal at 10.0.0.1:3200, two LUNs (Malloc0 and Malloc1),
+Assuming we have one iSCSI Target server with portal at 10.0.0.1:3200, two LUNs (Malloc0 and Malloc),
  and accepting initiators on 10.0.0.2/32, like on diagram below:
 
 ![Sample iSCSI configuration](iscsi_example.svg)
@@ -192,7 +227,7 @@ Assuming we have one iSCSI Target server with portal at 10.0.0.1:3200, two LUNs
 
 Start iscsi_tgt application:
 ```
-./build/bin/iscsi_tgt
+./app/iscsi_tgt/iscsi_tgt
 ```
 
 Construct two 64MB Malloc block devices with 512B sector size "Malloc0" and "Malloc1":
@@ -289,39 +324,11 @@ At the iSCSI level, we provide the following support for Hotplug:
   wait for all the commands which have already been submitted to block device to
   return back; after all the commands return back, the LUN will be deleted.
 
+## Known bugs and limitations {#iscsi_hotplug_bugs}
+
+For write command, if you want to test hotplug with write command which will
+cause r2t, for example 1M size IO, it will crash the iscsi tgt.
+For read command, if you want to test hotplug with large read IO, for example 1M
+size IO, it will probably crash the iscsi tgt.
+
 @sa spdk_nvme_probe
-
-# iSCSI Login Redirection {#iscsi_login_redirection}
-
-The SPDK iSCSI target application supports iSCSI login redirection feature.
-
-A portal refers to an IP address and TCP port number pair, and a portal group
-contains a set of portals. Users for the SPDK iSCSI target application configure
-portals through portal groups.
-
-To support login redirection feature, we utilize two types of portal groups,
-public portal group and private portal group.
-
-The SPDK iSCSI target application usually has a discovery portal. The discovery
-portal is connected by an initiator to get a list of targets, as well as the list
-of portals on which these target may be accessed, by a discovery session.
-
-Public portal groups have their portals returned by a discovery session. Private
-portal groups do not have their portals returned by a discovery session. A public
-portal group may optionally have a redirect portal for non-discovery logins for
-each associated target. This redirect portal must be from a private portal group.
-
-Initiators configure portals in public portal groups as target portals. When an
-initator logs in to a target through a portal in an associated public portal group,
-the target sends a temporary redirection response with a redirect portal. Then the
-initiator logs in to the target again through the redirect portal.
-
-Users set a portal group to public or private at creation using the
-`iscsi_create_portal_group` RPC, associate portal groups with a target using the
-`iscsi_create_target_node` RPC or the `iscsi_target_node_add_pg_ig_maps` RPC,
-specify a up-to-date redirect portal in a public portal group for a target using
-the `iscsi_target_node_set_redirect` RPC, and terminate the corresponding connections
-by asynchronous logout request using the `iscsi_target_node_request_logout` RPC.
-
-Typically users will use the login redirection feature in scale out iSCSI target
-system, which runs multiple SPDK iSCSI target applications.

doc/libraries.md

@@ -1,213 +0,0 @@
-# SPDK Libraries {#libraries}
-
-The SPDK repository is, first and foremost, a collection of high-performance
-storage-centric software libraries. With this in mind, much care has been taken
-to ensure that these libraries have consistent and robust naming and versioning
-conventions. The libraries themselves are also divided across two directories
-(`lib` and `module`) inside of the SPDK repository in a deliberate way to prevent
-mixing of SPDK event framework dependent code and lower level libraries. This document
-is aimed at explaining the structure, naming conventions, versioning scheme, and use cases
-of the libraries contained in these two directories.
-
-# Directory Structure {#structure}
-
-The SPDK libraries are divided into two directories. The `lib` directory contains the base libraries that
-compose SPDK. Some of these base libraries define plug-in systems. Instances of those plug-ins are called
-modules and are located in the `module` directory. For example, the `spdk_sock` library is contained in the
-`lib` directory while the implementations of socket abstractions, `sock_posix` and `sock_uring`
-are contained in the `module` directory.
-
-## lib {#lib}
-
-The libraries in the `lib` directory can be readily divided into four categories:
-
-- Utility Libraries: These libraries contain basic, commonly used functions that make more complex
-libraries easier to implement. For example, `spdk_log` contains macro definitions that provide a
-consistent logging paradigm and `spdk_json` is a general purpose JSON parsing library.
-- Protocol Libraries: These libraries contain the building blocks for a specific service. For example,
-`spdk_nvmf` and `spdk_vhost` each define the storage protocols after which they are named.
-- Storage Service Libraries: These libraries provide a specific abstraction that can be mapped to somewhere
-between the physical drive and the filesystem level of your typical storage stack. For example `spdk_bdev`
-provides a general block device abstraction layer, `spdk_lvol` provides a logical volume abstraction,
-`spdk_blobfs` provides a filesystem abstraction, and `spdk_ftl` provides a flash translation layer
-abstraction.
-- System Libraries: These libraries provide system level services such as a JSON based RPC service
-(see `spdk_jsonrpc`) and thread abstractions (see `spdk_thread`). The most notable library in this category
-is the `spdk_env_dpdk` library which provides a shim for the underlying Data Plane Development Kit (DPDK)
-environment and provides services like memory management.
-
-The one library in the `lib` directory that doesn't fit into the above classification is the `spdk_event` library.
-This library defines a framework used by the applications contained in the `app` and `example` directories. Much
-care has been taken to keep the SPDK libraries independent from this framework. The libraries in `lib` are engineered
-to allow plugging directly into independent application frameworks such as Seastar or libuv with minimal effort.
-
-Currently there are two exceptions in the `lib` directory which still rely on `spdk_event`, `spdk_vhost` and `spdk_iscsi`.
-There are efforts underway to remove all remaining dependencies these libraries have on the `spdk_event` library.
-
-Much like the `spdk_event` library, the `spdk_env_dpdk` library has been architected in such a way that it
-can be readily replaced by an alternate environment shim. More information on replacing the `spdk_env_dpdk`
-module and the underlying `dpdk` environment can be found in the [environment](#env_replacement) section.
-
-## module {#module}
-
-The component libraries in the `module` directory represent specific implementations of the base libraries in
-the `lib` directory. As with the `lib` directory, much care has been taken to avoid dependencies on the
-`spdk_event` framework except for those libraries which directly implement the `spdk_event` module plugin system.
-
-There are seven sub-directories in the `module` directory which each hold a different class of libraries. These
-sub-directories can be divided into two types.
-
-- plug-in libraries: These libraries are explicitly tied to one of the libraries in the `lib` directory and
-are registered with that library at runtime by way of a specific constructor function. The parent library in
-the `lib` directory then manages the module directly. These types of libraries each implement a function table
-defined by their parent library. The following table shows these directories and their corresponding parent
-libraries:
-
-<center>
-| module directory | parent library | dependent on event library |
-|------------------|----------------|----------------------------|
-| module/accel     | spdk_accel     | no                         |
-| module/bdev      | spdk_bdev      | no                         |
-| module/event     | spdk_event     | yes                        |
-| module/sock      | spdk_sock      | no                         |
-</center>
-
-- Free libraries: These libraries are highly dependent upon a library in the `lib` directory but are not
-explicitly registered to that library via a constructor. The libraries in the `blob`, `blobfs`, and `env_dpdk`
-directories fall into this category. None of the libraries in this category depend explicitly on the
-`spdk_event` library.
-
-# Library Conventions {#conventions}
-
-The SPDK libraries follow strict conventions for naming functions, logging, versioning, and header files.
-
-## Headers {#headers}
-
-All public SPDK header files exist in the `include` directory of the SPDK repository. These headers
-are divided into two sub-directories.
-
-`include/spdk` contains headers intended to be used by consumers of the SPDK libraries. All of the
-functions, variables, and types in these functions are intended for public consumption. Multiple headers
-in this directory may depend upon the same underlying library and work together to expose different facets
-of the library. The `spdk_bdev` library, for example, is exposed in three different headers. `bdev_module.h`
-defines the interfaces a bdev module library would need to implement, `bdev.h` contains general block device
-functions that would be used by an application consuming block devices exposed by SPDK, and `bdev_zone.h`
-exposes zoned bdev specific functions. Many of the other libraries exhibit a similar behavior of splitting
-headers between consumers of the library and those wishing to register a module with that library.
-
-`include/spdk_internal`, as its name suggests contains header files intended to be consumed only by other
-libraries inside of the SPDK repository. These headers are typically used for sharing lower level functions
-between two libraries that both require similar functions. For example `spdk_internal/nvme_tcp.h` contains
-low level tcp functions used by both the `spdk_nvme` and `spdk_nvmf` libraries. These headers are *NOT*
-intended for general consumption.
-
-Other header files contained directly in the `lib` and `module` directories are intended to be consumed *only*
-by source files of their corresponding library. Any symbols intended to be used across libraries need to be
-included in a header in the `include/spdk_internal` directory.
-
-## Naming Conventions {#naming}
-
-All public types and functions in SPDK libraries begin with the prefix `spdk_`. They are also typically
-further namespaced using the spdk library name. The rest of the function or type name describes its purpose.
-
-There are no internal library functions that begin with the `spdk_` prefix. This naming convention is
-enforced by the SPDK continuous Integration testing. Functions not intended for use outside of their home
-library should be namespaced with the name of the library only.
-
-## Map Files {#map}
-
-SPDK libraries can be built as both static and shared object files. To facilitate building libraries as shared
-objects, each one has a corresponding map file (e.g. `spdk_nvmf` relies on `spdk_nvmf.map`). SPDK libraries
-not exporting any symbols rely on a blank map file located at `mk/spdk_blank.map`.
-
-# SPDK Shared Objects {#shared_objects}
-
-## Shared Object Versioning {#versioning}
-
-SPDK shared objects follow a semantic versioning pattern with a major and minor version. Any changes which
-break backwards compatibility (symbol removal or change) will cause a shared object major increment and
-backwards compatible changes will cause a minor version increment; i.e. an application that relies on
-`libspdk_nvmf.so.3.0` will be compatible with `libspdk_nvmf.so.3.1` but not with `libspdk_nvmf.so.4.0`.
-
-Shared object versions are incremented only once between each release cycle. This means that at most, the
-major version of each SPDK shared library will increment only once between each SPDK release.
-
-There are currently no guarantees in SPDK of ABI compatibility between two major SPDK releases.
-
-The point releases of an LTS release will be ABI compatible with the corresponding LTS major release.
-
-Shared objects are versioned independently of one another. This means that `libspdk_nvme.so.3.0` and
-`libspdk_bdev.so.3.0` do not necessarily belong to the same release. This also means that shared objects
-with the same suffix are not necessarily compatible with each other. It is important to source all of your
-SPDK libraries from the same repository and version to ensure inter-library compatibility.
-
-## Linking to Shared Objects {#so_linking}
-
-Shared objects in SPDK are created on a per-library basis. There is a top level `libspdk.so` object
-which is a linker script. It simply contains references to all of the other spdk shared objects.
-
-There are essentially two ways of linking to SPDK libraries.
-
-1. An application can link to the top level shared object library as follows:
-~~~{.sh}
-	gcc -o my_app ./my_app.c -lspdk -lspdk_env_dpdk -ldpdk
-~~~
-
-2. An application can link to only a subset of libraries by linking directly to the ones it relies on:
-~~~{.sh}
-	gcc -o my_app ./my_app.c -lpassthru_external -lspdk_event_bdev -lspdk_bdev -lspdk_bdev_malloc
-	-lspdk_log -lspdk_thread -lspdk_util -lspdk_event -lspdk_env_dpdk -ldpdk
-~~~
-
-In the second instance, please note that applications need only link to the libraries upon which they
-directly depend. All SPDK libraries have their dependencies specified at object compile time. This means
-that when linking to `spdk_net`, one does not also have to specify `spdk_log`, `spdk_util`, `spdk_json`,
-`spdk_jsonrpc`, and `spdk_rpc`. However, this dependency inclusion does not extend to the application
-itself; i.e. if an application directly uses symbols from both `spdk_bdev` and `spdk_log`, both libraries
-will need to be supplied to the linker when linking the application even though `spdk_log` is a dependency
-of `spdk_bdev`.
-
-Please also note that when linking to SPDK libraries, both the spdk_env shim library and the env library
-itself need to be supplied to the linker. In the examples above, these are `spdk_env_dpdk` and `dpdk`
-respectively. This was intentional and allows one to easily swap out both the environment and the
-environment shim.
-
-## Replacing the env abstraction {#env_replacement}
-
-SPDK depends on an environment abstraction that provides crucial pinned memory management and PCIe
-bus management operations. The interface for this environment abstraction is defined in the
-`include/env.h` header file. The default implementation of this environment is located in `spdk_env_dpdk`.
-This abstraction in turn relies upon the DPDK libraries. This two part implementation was deliberate
-and allows for easily swapping out the dpdk version upon which the spdk libraries rely without making
-modifications to the spdk source directly.
-
-Any environment can replace the `spdk_env_dpdk` environment by implementing the `include/env.h` header
-file. The environment can either be implemented wholesale in a single library or as a two-part
-shim/implementation library system.
-~~~{.sh}
-	# single library
-	gcc -o my_app ./my_app.c -lspdk -lcustom_env_implementation
-
-	# two libraries
-	gcc -o my_app ./my_app.c -lspdk -lcustom_env_shim -lcustom_env_implementation
-~~~
-
-# SPDK Static Objects {#static_objects}
-
-SPDK static objects are compiled by default even when no parameters are supplied to the build system.
-Unlike SPDK shared objects, the filename does not contain any versioning semantics. Linking against
-static objects is similar to shared objects but will always require the use of `-Wl,--whole-archive`
-as argument. This is due to the use of constructor functions in SPDK such as those to register
-NVMe transports.
-
-Due to the lack of versioning semantics, it is not recommended to install static libraries system wide.
-Instead the path to these static libraries should be added as argument at compile time using
-`-L/path/to/static/libs`. The use of static objects instead of shared objects can also be forced
-through `-Wl,-Bsatic`, otherwise some compilers might prefer to use the shared objects if both
-are available.
-
-~~~{.sh}
-	gcc -o my_app ./my_app.c -L/path/to/static/libs -Wl,--whole-archive -Wl,-Bstatic -lpassthru_external
-	-lspdk_event_bdev -lspdk_bdev -lspdk_bdev_malloc -lspdk_log -lspdk_thread -lspdk_util -lspdk_event
-	-lspdk_env_dpdk -Wl,--no-whole-archive -Wl,-Bdynamic -pthread -ldpdk
-~~~

doc/nvme-cli.md

@@ -0,0 +1,96 @@
+# nvme-cli {#nvme-cli}
+
+# nvme-cli with SPDK Getting Started Guide
+
+Now nvme-cli can support both kernel driver and SPDK user mode driver for most of its available commands and
+Intel specific commands.
+
+1. Clone the nvme-cli repository from the SPDK GitHub fork. Make sure you check out the spdk-1.6 branch.
+
+  ~~~{.sh}
+  git clone -b spdk-1.6 https://github.com/spdk/nvme-cli.git
+  ~~~
+
+2. Clone the SPDK repository from https://github.com/spdk/spdk under the nvme-cli folder.
+
+3. Refer to the "README.md" under SPDK folder to properly build SPDK.
+
+4. Refer to the "README.md" under nvme-cli folder to properly build nvme-cli.
+
+5. Execute "<spdk_folder>/scripts/setup.sh" with the "root" account.
+
+6. Update the "spdk.conf" file under nvme-cli folder to properly configure the SPDK. Notes as following:
+
+  ~~~{.sh}
+  spdk=1
+  Indicates whether or not to use spdk. Can be 0 (off) or 1 (on).
+  Defaults to 1 which assumes that you have run "<spdk_folder>/scripts/setup.sh", unbinding your drives from the kernel.
+
+  core_mask=0x1
+  A bitmask representing which core(s) to use for nvme-cli operations.
+  Defaults to core 0.
+
+  mem_size=512
+  The amount of reserved hugepage memory to use for nvme-cli (in MB).
+  Defaults to 512MB.
+
+  shm_id=0
+  Indicates the shared memory ID for the spdk application with which your NVMe drives are associated,
+  and should be adjusted accordingly.
+  Defaults to 0.
+~~~
+
+7. Run the "./nvme list" command to get the domain:bus:device.function for each found NVMe SSD.
+
+8. Run the other nvme commands with domain:bus:device.function instead of "/dev/nvmeX" for the specified device.
+
+  ~~~{.sh}
+  Example: ./nvme smart-log 0000:01:00.0
+  ~~~
+
+9. Run the "./nvme intel" commands for Intel specific commands against Intel NVMe SSD.
+
+  ~~~{.sh}
+  Example: ./nvme intel internal-log 0000:08:00.0
+  ~~~
+
+10. Execute "<spdk_folder>/scripts/setup.sh reset" with the "root" account and update "spdk=0" in spdk.conf to
+    use the kernel driver if wanted.
+
+## Use scenarios
+
+### Run as the only SPDK application on the system
+
+1. Modify the spdk to 1 in spdk.conf. If the system has fewer cores or less memory, update the spdk.conf accordingly.
+
+### Run together with other running SPDK applications on shared NVMe SSDs
+
+1. For the other running SPDK application, start with the parameter like "-i 1" to have the same "shm_id".
+
+2. Use the default spdk.conf setting where "shm_id=1" to start the nvme-cli.
+
+3. If other SPDK applications run with different shm_id parameter, update the "spdk.conf" accordingly.
+
+### Run with other running SPDK applications on non-shared NVMe SSDs
+
+1. Properly configure the other running SPDK applications.
+
+  ~~~{.sh}
+  a. Only access the NVMe SSDs it wants.
+  b. Allocate a fixed number of memory instead of all available memory.
+  ~~~
+
+2. Properly configure the spdk.conf setting for nvme-cli.
+
+  ~~~{.sh}
+  a. Not access the NVMe SSDs from other SPDK applications.
+  b. Change the mem_size to a proper size.
+  ~~~
+
+## Note
+
+1. To run the newly built nvme-cli, either explicitly run as "./nvme" or added it into the $PATH to avoid
+   invoke other already installed version.
+
+2. To run the newly built nvme-cli with SPDK support in arbitrary directory, copy "spdk.conf" to that
+   directory from the nvme cli folder and update the configuration as suggested.

doc/nvme.md

@@ -304,8 +304,6 @@ At the NVMe driver level, we provide the following support for Hotplug:
 
 This feature is considered as experimental.
 
-## Design
-
 ![NVMe character devices processing diagram](nvme_cuse.svg)
 
 For each controller as well as namespace, character devices are created in the
@@ -319,67 +317,35 @@ Where X is unique SPDK NVMe controller index and Y is namespace id.
 
 Requests from CUSE are handled by pthreads when controller and namespaces are created.
 Those pass the I/O or admin commands via a ring to a thread that processes them using
-nvme_io_msg_process().
+spdk_nvme_io_msg_process().
 
 Ioctls that request information attained when attaching NVMe controller receive an
 immediate response, without passing them through the ring.
 
-This interface reserves one additional qpair for sending down the I/O for each controller.
+This interface reserves one qpair for sending down the I/O for each controller.
 
-## Usage
+## Enabling cuse support for NVMe
 
-### Enabling cuse support for NVMe
+Cuse support is disabled by default. To enable support for NVMe devices SPDK
+must be compiled with "./configure --with-nvme-cuse".
 
-Cuse support is disabled by default. To enable support for NVMe-CUSE devices first
-install required dependencies
-~~~{.sh}
-sudo scripts/pkgdep.sh --fuse
-~~~
-Then compile SPDK with "./configure --with-nvme-cuse".
+## Limitations
 
-### Creating NVMe-CUSE device
+NVMe CUSE presents character device for controller and namespaces only at the time
+the controller is being attached. Dynamic creation/deletion of namespaces is not
+supported yet.
 
-First make sure to prepare the environment (see @ref getting_started).
-This includes loading CUSE kernel module.
-Any NVMe controller attached to a running SPDK application can be
-exposed via NVMe-CUSE interface. When closing SPDK application,
-the NVMe-CUSE devices are unregistered.
+NVMe namespaces are created as character devices and their use may be limited for
+tools expecting block devices.
 
-~~~{.sh}
-$ sudo scripts/setup.sh
-$ sudo modprobe cuse
-$ sudo build/bin/spdk_tgt
-# Continue in another session
-$ sudo scripts/rpc.py bdev_nvme_attach_controller -b Nvme0 -t PCIe -a 0000:82:00.0
-Nvme0n1
-$ sudo scripts/rpc.py bdev_nvme_get_controllers
-[
-  {
-    "name": "Nvme0",
-    "trid": {
-      "trtype": "PCIe",
-      "traddr": "0000:82:00.0"
-    }
-  }
-]
-$ sudo scripts/rpc.py bdev_nvme_cuse_register -n Nvme0
-$ ls /dev/spdk/
-nvme0  nvme0n1
-~~~
+Sysfs is not updated by SPDK.
 
-### Example of using nvme-cli
+SPDK NVMe CUSE creates nodes in "/dev/spdk/" directory to explicitly differentiate
+from other devices. Tools that only search in the "/dev" directory might not work
+with SPDK NVMe CUSE.
 
-Most nvme-cli commands can point to specific controller or namespace by providing a path to it.
-This can be leveraged to issue commands to the SPDK NVMe-CUSE devices.
-
-~~~{.sh}
-sudo nvme id-ctrl /dev/spdk/nvme0
-sudo nvme smart-log /dev/spdk/nvme0
-sudo nvme id-ns /dev/spdk/nvme0n1
-~~~
-
-Note: `nvme list` command does not display SPDK NVMe-CUSE devices,
-see nvme-cli [PR #773](https://github.com/linux-nvme/nvme-cli/pull/773).
+SCSI to NVMe Translation Layer is not implemented. Tools that are using this layer to
+identify, manage or operate device might not work properly or their use may be limited.
 
 ### Examples of using smartctl
 
@@ -394,17 +360,3 @@ the NVMe device.
     smartctl -d nvme -H /dev/spdk/nvme1
     ...
 ~~~
-
-## Limitations
-
-NVMe namespaces are created as character devices and their use may be limited for
-tools expecting block devices.
-
-Sysfs is not updated by SPDK.
-
-SPDK NVMe CUSE creates nodes in "/dev/spdk/" directory to explicitly differentiate
-from other devices. Tools that only search in the "/dev" directory might not work
-with SPDK NVMe CUSE.
-
-SCSI to NVMe Translation Layer is not implemented. Tools that are using this layer to
-identify, manage or operate device might not work properly or their use may be limited.

doc/nvme_spec.md

@@ -20,8 +20,8 @@ registers involved that are called doorbells.
 
 An I/O is submitted to an NVMe device by constructing a 64 byte command, placing
 it into the submission queue at the current location of the submission queue
-tail index, and then writing the new index of the submission queue tail to the
-submission queue tail doorbell register. It's actually valid to copy a whole set
+head index, and then writing the new index of the submission queue head to the
+submission queue head doorbell register. It's actually valid to copy a whole set
 of commands into open slots in the ring and then write the doorbell just one
 time to submit the whole batch.
 

doc/nvmf.md

@@ -43,7 +43,7 @@ Then build SPDK with RDMA enabled:
 make
 ~~~
 
-Once built, the binary will be in `build/bin`.
+Once built, the binary will be in `app/nvmf_tgt`.
 
 ### Prerequisites for InfiniBand/RDMA Verbs {#nvmf_prereqs_verbs}
 
@@ -106,14 +106,20 @@ using 1GB hugepages or by pre-reserving memory at application startup with `--me
 option. All pre-reserved memory will be registered as a single region, but won't be returned to the
 system until the SPDK application is terminated.
 
-Another known issue occurs when using the E810 NICs in RoCE mode. Specifically, the NVMe-oF target
-sometimes cannot destroy a qpair, because its posted work requests don't get flushed.  It can cause
-the NVMe-oF target application unable to terminate cleanly.
-
 ## TCP transport support {#nvmf_tcp_transport}
 
 The transport is built into the nvmf_tgt by default, and it does not need any special libraries.
 
+## Configuring the SPDK NVMe over Fabrics Target {#nvmf_config}
+
+An NVMe over Fabrics target can be configured using JSON RPCs.
+The basic RPCs needed to configure the NVMe-oF subsystem are detailed below. More information about
+working with NVMe over Fabrics specific RPCs can be found on the @ref jsonrpc_components_nvmf_tgt RPC page.
+
+Using .ini style configuration files for configuration of the NVMe-oF target is deprecated and should
+be replaced with JSON based RPCs. .ini style configuration files can be converted to json format by way
+of the new script `scripts/config_converter.py`.
+
 ## FC transport support {#nvmf_fc_transport}
 
 To build nvmf_tgt with the FC transport, there is an additional FC LLD (Low Level Driver) code dependency.
@@ -141,12 +147,6 @@ cd ../spdk
 make
 ~~~
 
-## Configuring the SPDK NVMe over Fabrics Target {#nvmf_config}
-
-An NVMe over Fabrics target can be configured using JSON RPCs.
-The basic RPCs needed to configure the NVMe-oF subsystem are detailed below. More information about
-working with NVMe over Fabrics specific RPCs can be found on the @ref jsonrpc_components_nvmf_tgt RPC page.
-
 ### Using RPCs {#nvmf_config_rpc}
 
 Start the nvmf_tgt application with elevated privileges. Once the target is started,
@@ -157,9 +157,9 @@ and an in capsule data size of 0 bytes. The TCP transport is configured with an
 16384 bytes, 8 max qpairs per controller, and an in capsule data size of 8192 bytes.
 
 ~~~{.sh}
-build/bin/nvmf_tgt
-scripts/rpc.py nvmf_create_transport -t RDMA -u 8192 -m 4 -c 0
-scripts/rpc.py nvmf_create_transport -t TCP -u 16384 -m 8 -c 8192
+app/nvmf_tgt/nvmf_tgt
+scripts/rpc.py nvmf_create_transport -t RDMA -u 8192 -p 4 -c 0
+scripts/rpc.py nvmf_create_transport -t TCP -u 16384 -p 8 -c 8192
 ~~~
 
 Below is an example of creating a malloc bdev and assigning it to a subsystem. Adjust the bdevs,
@@ -231,7 +231,7 @@ The `-m` core mask option specifies a bit mask of the CPU cores that
 SPDK is allowed to execute work items on.
 For example, to allow SPDK to use cores 24, 25, 26 and 27:
 ~~~{.sh}
-build/bin/nvmf_tgt -m 0xF000000
+app/nvmf_tgt/nvmf_tgt -m 0xF000000
 ~~~
 
 ## Configuring the Linux NVMe over Fabrics Host {#nvmf_host}

doc/nvmf_tgt_pg.md

@@ -68,7 +68,7 @@ system. This is used for access control.
 
 A user of the NVMe-oF target library begins by creating a target using
 spdk_nvmf_tgt_create(), setting up a set of addresses on which to accept
-connections by calling spdk_nvmf_tgt_listen_ext(), then creating a subsystem
+connections by calling spdk_nvmf_tgt_listen(), then creating a subsystem
 using spdk_nvmf_subsystem_create().
 
 Subsystems begin in an inactive state and must be activated by calling
@@ -78,13 +78,12 @@ calling spdk_nvmf_subsystem_pause() and resumed by calling
 spdk_nvmf_subsystem_resume().
 
 Namespaces may be added to the subsystem by calling
-spdk_nvmf_subsystem_add_ns_ext() when the subsystem is inactive or paused.
+spdk_nvmf_subsystem_add_ns() when the subsystem is inactive or paused.
 Namespaces are bdevs. See @ref bdev for more information about the SPDK bdev
 layer. A bdev may be obtained by calling spdk_bdev_get_by_name().
 
 Once a subsystem exists and the target is listening on an address, new
-connections will be automatically assigned to poll groups as they are
-detected.
+connections may be accepted by polling spdk_nvmf_tgt_accept().
 
 All I/O to a subsystem is driven by a poll group, which polls for incoming
 network I/O. Poll groups may be created by calling
@@ -92,6 +91,14 @@ spdk_nvmf_poll_group_create(). They automatically request to begin polling
 upon creation on the thread from which they were created. Most importantly, *a
 poll group may only be accessed from the thread on which it was created.*
 
+When spdk_nvmf_tgt_accept() detects a new connection, it will construct a new
+struct spdk_nvmf_qpair object and call the user provided `new_qpair_fn`
+callback for each new qpair. In response to this callback, the user must
+assign the qpair to a poll group by calling spdk_nvmf_poll_group_add().
+Remember, a poll group may only be accessed from the thread on which it was created,
+so making a call to spdk_nvmf_poll_group_add() may require passing a message
+to the appropriate thread.
+
 ## Access Control
 
 Access control is performed at the subsystem level by adding allowed listen
@@ -104,7 +111,9 @@ and hosts may only be added to inactive or paused subsystems.
 
 A discovery subsystem, as defined by the NVMe-oF specification, is
 automatically created for each NVMe-oF target constructed. Connections to the
-discovery subsystem are handled in the same way as any other subsystem.
+discovery subsystem are handled in the same way as any other subsystem - new
+qpairs are created in response to spdk_nvmf_tgt_accept() and they must be
+assigned to a poll group.
 
 ## Transports
 
@@ -123,7 +132,15 @@ fabrics simultaneously.
 The SPDK NVMe-oF target library does not strictly dictate threading model, but
 poll groups do all of their polling and I/O processing on the thread they are
 created on. Given that, it almost always makes sense to create one poll group
-per thread used in the application.
+per thread used in the application. New qpairs created in response to
+spdk_nvmf_tgt_accept() can be handed out round-robin to the poll groups. This
+is how the SPDK NVMe-oF target application currently functions.
+
+More advanced algorithms for distributing qpairs to poll groups are possible.
+For instance, a NUMA-aware algorithm would be an improvement over basic
+round-robin, where NUMA-aware means assigning qpairs to poll groups running on
+CPU cores that are on the same NUMA node as the network adapter and storage
+device. Load-aware algorithms also may have benefits.
 
 ## Scaling Across CPU Cores
 

doc/nvmf_tracing.md

@@ -16,14 +16,14 @@ the instrumentation of all the tracepoints group in an SPDK target application,
 target with -e parameter set to 0xFFFF:
 
 ~~~
-build/bin/nvmf_tgt -e 0xFFFF
+app/nvmf_tgt/nvmf_tgt -e 0xFFFF
 ~~~
 
 To enable the instrumentation of just the NVMe-oF RDMA tracepoints in an SPDK target
 application, start the target with the -e parameter set to 0x10:
 
 ~~~
-build/bin/nvmf_tgt -e 0x10
+app/nvmf_tgt/nvmf_tgt -e 0x10
 ~~~
 
 When the target starts, a message is logged with the information you need to view
@@ -55,7 +55,7 @@ The spdk_trace program can be found in the app/trace directory.  To analyze the
 system running the NVMe-oF target, simply execute the command line shown in the log:
 
 ~~~{.sh}
-build/bin/spdk_trace -s nvmf -p 24147
+app/trace/spdk_trace -s nvmf -p 24147
 ~~~
 
 To analyze the tracepoints on a different system, first prepare the tracepoint file for transfer.  The
@@ -70,7 +70,7 @@ After transferring the /tmp/trace.bz2 tracepoint file to a different system:
 
 ~~~{.sh}
 bunzip2 /tmp/trace.bz2
-build/bin/spdk_trace -f /tmp/trace
+app/trace/spdk_trace -f /tmp/trace
 ~~~
 
 The following is sample trace capture showing the cumulative time that each
@@ -134,7 +134,7 @@ and store all entries into specified output file at its shutdown on SIGINT or SI
 After SPDK nvmf target is launched, simply execute the command line shown in the log:
 
 ~~~{.sh}
-build/bin/spdk_trace_record -q -s nvmf -p 24147 -f /tmp/spdk_nvmf_record.trace
+app/trace_record/spdk_trace_record -q -s nvmf -p 24147 -f /tmp/spdk_nvmf_record.trace
 ~~~
 
 Also send I/Os to the SPDK target application to generate events by previous perf example for 10 minutes.
@@ -147,7 +147,7 @@ After the completion of perf example, shut down spdk_trace_record by signal SIGI
 To analyze the tracepoints output file from spdk_trace_record, simply run spdk_trace program by:
 
 ~~~{.sh}
-build/bin/spdk_trace -f /tmp/spdk_nvmf_record.trace
+app/trace/spdk_trace -f /tmp/spdk_nvmf_record.trace
 ~~~
 
 # Adding New Tracepoints {#add_tracepoints}
@@ -202,4 +202,4 @@ record the current trace state of several tracepoints.
 		...
 ~~~
 
-All the tracing functions are documented in the [Tracepoint library documentation](https://spdk.io/doc/trace_8h.html)
+All the tracing functions are documented in the [Tracepoint library documentation](https://www.spdk.io/doc/trace_8h.html)

doc/peer_2_peer.md

@@ -39,7 +39,7 @@ SPDK's identify example application displays whether a device has a controller
 memory buffer and which operations it supports. Run it as follows:
 
 ~~~{.sh}
-./build/examples/identify -r traddr:<pci id of ssd>
+./examples/nvme/identify/identify -r traddr:<pci id of ssd>
 ~~~
 
 # cmb_copy: An example P2P Application {#p2p_cmb_copy}
@@ -47,7 +47,7 @@ memory buffer and which operations it supports. Run it as follows:
 Run the cmb_copy example application.
 
 ~~~{.sh}
-./build/examples/cmb_copy -r <pci id of write ssd>-1-0-1 -w <pci id of write ssd>-1-0-1 -c <pci id of the ssd with cmb>
+./examples/nvme/cmb_copy/cmb_copy -r <pci id of write ssd>-1-0-1 -w <pci id of write ssd>-1-0-1 -c <pci id of the ssd with cmb>
 ~~~
 This should copy a single LBA (LBA 0) from namespace 1 on the read
 NVMe SSD to LBA 0 on namespace 1 on the write SSD using the CMB as the

doc/performance_reports.md

@@ -1,36 +1,8 @@
 # Performance Reports {#performance_reports}
 
-## Release 21.01
-
-- [SPDK 21.01 NVMe Bdev Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_nvme_bdev_perf_report_2101.pdf)
-- [SPDK 21.01 NVMe-oF TCP Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_tcp_perf_report_2101.pdf)
-- [SPDK 21.01 NVMe-oF RDMA Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_rdma_perf_report_2101.pdf)
-- [SPDK 21.01 Vhost Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_vhost_perf_report_2101.pdf)
-
-## Release 20.10
-
-- [SPDK 20.10 NVMe Bdev Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_nvme_bdev_perf_report_2010.pdf)
-- [SPDK 20.10 NVMe-oF TCP Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_tcp_perf_report_2010.pdf)
-- [SPDK 20.10 NVMe-oF RDMA Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_rdma_perf_report_2010.pdf)
-- [SPDK 20.10 Vhost Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_vhost_perf_report_2010.pdf)
-
-## Release 20.07
-
-- [SPDK 20.07 NVMe-oF TCP Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_tcp_perf_report_2007.pdf)
-- [SPDK 20.07 NVMe-oF RDMA Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_rdma_perf_report_2007.pdf)
-- [SPDK 20.07 Vhost Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_vhost_perf_report_2007.pdf)
-
-## Release 20.04
-
-- [SPDK 20.04 NVMe-oF TCP Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_tcp_perf_report_2004.pdf)
-- [SPDK 20.04 NVMe-oF RDMA Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_rdma_perf_report_2004.pdf)
-- [SPDK 20.04 Vhost Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_vhost_perf_report_2004.pdf)
-
 ## Release 20.01
 
 - [SPDK 20.01 Vhost Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_vhost_perf_report_2001.pdf)
-- [SPDK 20.01 NVMe-oF TCP Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_tcp_perf_report_2001.pdf)
-- [SPDK 20.01 NVMe-oF RDMA Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_rdma_perf_report_2001.pdf)
 
 ## Release 19.10
 

doc/pkgconfig.md

@@ -1,56 +0,0 @@
-# Linking SPDK applications with pkg-config {#pkgconfig}
-
-The SPDK build system generates pkg-config files to facilitate linking
-applications with the correct set of SPDK and DPDK libraries. Using pkg-config
-in your build system will ensure you do not need to make modifications
-when SPDK adds or modifies library dependencies.
-
-If your application is using the SPDK nvme library, you would use the following
-to get the list of required SPDK libraries:
-
-~~~
-PKG_CONFIG_PATH=/path/to/spdk/build/lib/pkgconfig pkg-config --libs spdk_nvme
-~~~
-
-To get the list of required SPDK and DPDK libraries to use the DPDK-based
-environment layer:
-
-~~~
-PKG_CONFIG_PATH=/path/to/spdk/build/lib/pkgconfig pkg-config --libs spdk_env_dpdk
-~~~
-
-When linking with static libraries, the dependent system libraries must also be
-specified. To get the list of required system libraries:
-
-~~~
-PKG_CONFIG_PATH=/path/to/spdk/build/lib/pkgconfig pkg-config --libs spdk_syslibs
-~~~
-
-Note that SPDK libraries use constructor functions liberally, so you must surround
-the library list with extra linker options to ensure these functions are not dropped
-from the resulting application binary. With shared libraries this is achieved through
-the `-Wl,--no-as-needed` parameters while with static libraries `-Wl,--whole-archive`
-is used. Here is an example Makefile snippet that shows how to use pkg-config to link
-an application that uses the SPDK nvme shared library:
-
-~~~
-PKG_CONFIG_PATH = $(SPDK_DIR)/build/lib/pkgconfig
-SPDK_LIB := $(shell PKG_CONFIG_PATH="$(PKG_CONFIG_PATH)" pkg-config --libs spdk_nvme
-DPDK_LIB := $(shell PKG_CONFIG_PATH="$(PKG_CONFIG_PATH)" pkg-config --libs spdk_env_dpdk
-
-app:
-	$(CC) -o app app.o -pthread -Wl,--no-as-needed $(SPDK_LIB) $(DPDK_LIB) -Wl,--as-needed
-~~~
-
-If using the SPDK nvme static library:
-
-~~~
-PKG_CONFIG_PATH = $(SPDK_DIR)/build/lib/pkgconfig
-SPDK_LIB := $(shell PKG_CONFIG_PATH="$(PKG_CONFIG_PATH)" pkg-config --libs spdk_nvme
-DPDK_LIB := $(shell PKG_CONFIG_PATH="$(PKG_CONFIG_PATH)" pkg-config --libs spdk_env_dpdk
-SYS_LIB := $(shell PKG_CONFIG_PATH="$(PKG_CONFIG_PATH)" pkg-config --libs --static spdk_syslibs
-
-app:
-	$(CC) -o app app.o -pthread -Wl,--whole-archive $(SPDK_LIB) $(DPDK_LIB) -Wl,--no-whole-archive \
-		$(SYS_LIB)
-~~~

doc/rpm.md

@@ -1,49 +0,0 @@
-# RPMs {#rpms}
-
-# In this document {#rpms_toc}
-
-* @ref building_rpms
-
-# Building SPDK RPMs {#building_rpms}
-
-To build basic set of RPM packages out of the SPDK repo simply run:
-
-~~~{.sh}
-# rpmbuild/rpm.sh
-~~~
-
-Additional configuration options can be passed directly as arguments:
-
-~~~{.sh}
-# rpmbuild/rpm.sh --with-shared --with-dpdk=/path/to/dpdk/build
-~~~
-
-There are several options that may be passed via environment as well:
-
-- DEPS          - Install all needed dependencies for building RPM packages.
-                Default: "yes"
-- MAKEFLAGS     - Flags passed to make
-- RPM_RELEASE   - Target release version of the RPM packages. Default: 1
-- REQUIREMENTS  - Extra set of RPM dependencies if deemed as needed
-- SPDK_VERSION  - SPDK version. Default: currently checked out tag
-
-~~~{.sh}
-# DEPS=no MAKEFLAGS="-d -j1" rpmbuild/rpm.sh --with-shared
-~~~
-
-By default, all RPM packages should be created under $HOME directory of the
-target user:
-
-~~~{.sh}
-# printf '%s\n' /root/rpmbuild/RPMS/x86_64/*
-/root/rpmbuild/RPMS/x86_64/spdk-devel-v21.01-1.x86_64.rpm
-/root/rpmbuild/RPMS/x86_64/spdk-dpdk-libs-v21.01-1.x86_64.rpm
-/root/rpmbuild/RPMS/x86_64/spdk-libs-v21.01-1.x86_64.rpm
-/root/rpmbuild/RPMS/x86_64/spdk-v21.01-1.x86_64.rpm
-#
-~~~
-
-- spdk            - provides all the binaries, common tooling, etc.
-- spdk-devel      - provides development files
-- spdk-libs       - provides target lib, .pc files (--with-shared)
-- spdk-dpdk-libs  - provides dpdk lib files (--with-shared|--with-dpdk)

doc/scheduler.md

@@ -1,82 +0,0 @@
-# Scheduler {#scheduler}
-
-SPDK's event/application framework (`lib/event`) now supports scheduling of
-lightweight threads. Schedulers are provided as plugins, called
-implementations. A default implementation is provided, but users may wish to
-write their own scheduler to integrate into broader code frameworks or meet
-their performance needs.
-
-This feature should be considered experimental and is disabled by default. When
-enabled, the scheduler framework gathers data for each spdk thread and reactor
-and passes it to a scheduler implementation to perform one of the following
-actions.
-
-## Actions
-
-### Move a thread
-
-`spdk_thread`s can be moved to another reactor. Schedulers can examine the
-suggested cpu_mask value for each lightweight thread to see if the user has
-requested specific reactors, or choose a reactor using whatever algorithm they
-deem fit.
-
-### Switch reactor mode
-
-Reactors by default run in a mode that constantly polls for new actions for the
-most efficient processing. Schedulers can switch a reactor into a mode that
-instead waits for an event on a file descriptor. On Linux, this is implemented
-using epoll. This results in reduced CPU usage but may be less responsive when
-events occur. A reactor cannot enter this mode if any `spdk_threads` are
-currently scheduled to it. This limitation is expected to be lifted in the
-future, allowing `spdk_threads` to enter interrupt mode.
-
-### Set frequency of CPU core
-
-The frequency of CPU cores can be modified by the scheduler in response to
-load. Only CPU cores that match the application cpu_mask may be modified. The
-mechanism for controlling CPU frequency is pluggable and the default provided
-implementation is called `dpdk_governor`, based on the `rte_power` library from
-DPDK.
-
-#### Known limitation
-
-When SMT (Hyperthreading) is enabled the two logical CPU cores sharing a single
-physical CPU core must run at the same frequency. If one of two of such logical
-CPU cores is outside the application cpu_mask, the policy and frequency on that
-core has to be managed by the administrator.
-
-## Scheduler implementations
-
-The scheduler in use may be controlled by JSON-RPC. Please use the
-[framework_set_scheduler](jsonrpc.md/#rpc_framework_set_scheduler) RPC to
-switch between schedulers or change their options.
-
-[spdk_top](spdk_top.md#spdk_top) is a useful tool to observe the behavior of
-schedulers in different scenarios and workloads.
-
-### static [default]
-
-The `static` scheduler is the default scheduler and does no dynamic scheduling.
-Lightweight threads are distributed round-robin among reactors, respecting
-their requested cpu_mask, and then they are never moved. This is equivalent to
-the previous behavior of the SPDK event/application framework.
-
-### dynamic
-
-The `dynamic` scheduler is designed for power saving and reduction of CPU
-utilization, especially in cases where workloads show large variations over
-time.
-
-Active threads are distributed equally among reactors, taking cpu_mask into
-account. All idle threads are moved to the main core. Once an idle thread becomes
-active, it is redistributed again.
-
-When a reactor has no scheduled `spdk_thread`s it is switched into interrupt
-mode and stops actively polling. After enough threads become active, the
-reactor is switched back into poll mode and threads are assigned to it again.
-
-The main core can contain active threads only when their execution time does
-not exceed the sum of all idle threads. When no active threads are present on
-the main core, the frequency of that CPU core will decrease as the load
-decreases. All CPU cores corresponding to the other reactors remain at maximum
-frequency.

doc/shfmt.md

@@ -1,146 +0,0 @@
-# shfmt {#shfmt}
-
-# In this document {#shfmt_toc}
-
-* @ref shfmt_overview
-* @ref shfmt_usage
-* @ref shfmt_installation
-* @ref shfmt_examples
-
-# Overview {#shfmt_overview}
-
-The majority of tests (and scripts overall) in the SPDK repo are written
-in Bash (with a quite significant emphasis on "Bashism"), thus a style
-formatter, shfmt, was introduced to help keep the .sh code consistent
-across the entire repo. For more details on the tool itself, please see
-[shfmt](https://github.com/mvdan/sh).
-
-# Usage {#shfmt_usage}
-
-On the CI pool, the shfmt is run against all the updated .sh files that
-have been committed but not merged yet. Additionally, shfmt will pick
-all .sh present in the staging area when run locally from our pre-commit
-hook (via check_format.sh). In case any style errors are detected, a
-patch with needed changes is going to be generated and either build (CI)
-or the commit will be aborted. Said patch can be then easily applied:
-
-~~~{.sh}
-# Run from the root of the SPDK repo
-patch --merge -p0 <shfmt-3.1.0.patch
-~~~
-
-The name of the patch is derived from the version of shfmt that is
-currently in use (3.1.0 is currently supported).
-
-Please, see ./scripts/check_format.sh for all the arguments the shfmt
-is run with. Additionally, @ref shfmt_examples has more details on how
-each of the arguments behave.
-
-# Installation {#shfmt_installation}
-
-The shfmt can be easily installed via pkgdep.sh:
-
-~~~{.sh}
-./scripts/pkgdep.sh -d
-~~~
-
-This will install all the developers tools, including shfmt, on the
-local system. The precompiled binary will be saved, by default, to
-/opt/shfmt and then linked under /usr/bin. Both paths can be changed
-by setting SHFMT_DIR and SHFMT_DIR_OUT in the environment. Example:
-
-~~~{.sh}
-SHFMT_DIR=/keep_the_binary_here \
-SHFMT_DIR_OUT=/and_link_it_here \
-  ./scripts/pkgdep.sh -d
-~~~
-
-# Examples {#shfmt_examples}
-
-~~~{.sh}
-#######################################
-if foo=$(bar); then
-  echo "$foo"
-fi
-
-exec "$foo" \
-  --bar \
-  --foo
-
-# indent_style = tab
-
-if foo=$(bar); then
-        echo "$foo"
-fi
-
-exec foobar \
-        --bar \
-        --foo
-######################################
-if foo=$(bar); then
-        echo "$foo" && \
-        echo "$(bar)"
-fi
-# binary_next_line = true
-if foo=$(bar); then
-        echo "$foo" \
-                && echo "$(bar)"
-fi
-
-# Note that each break line is also being indented:
-
-if [[ -v foo ]] \
-&& [[ -v bar ]] \
-&& [[ -v foobar ]]; then
-	echo "This is foo"
-fi
-# ->
-if [[ -v foo ]] \
-        && [[ -v bar ]] \
-        && [[ -v foobar ]]; then
-        echo "This is foo"
-fi
-
-# Currently, newlines are being escaped even if syntax-wise
-# they are not needed, thus watch for the following:
-if [[ -v foo
-        && -v bar
-        && -v foobar ]]; then
-        echo "This is foo"
-fi
-#->
-if [[ -v foo && -v \
-        bar && -v \
-        foobar ]]; then
-        echo "This is foo"
-fi
-# This, unfortunately, also breaks the -bn behavior.
-# (see https://github.com/mvdan/sh/issues/565) for details.
-######################################
-case "$FOO" in
-        BAR)
-        echo "$FOO" ;;
-esac
-# switch_case_indent = true
-case "$FOO" in
-        BAR)
-                echo "$FOO"
-                ;;
-esac
-######################################
-exec {foo}>bar
-:>foo
-exec {bar}<foo
-# -sr
-exec {foo}> bar
-: > foo
-exec {bar}< foo
-######################################
-# miscellaneous, enforced by shfmt
-(( no_spacing_at_the_beginning & ~and_no_spacing_at_the_end ))
-: $(( no_spacing_at_the_beginning & ~and_no_spacing_at_the_end ))
-
-# ->
-((no_spacing_at_the_beginning & ~and_no_spacing_at_the_end))
-: $((no_spacing_at_the_beginning & ~and_no_spacing_at_the_end))
-~~~

doc/spdk_top.md

@@ -1,65 +0,0 @@
-# spdk_top {#spdk_top}
-
-The spdk_top application is designed to resemble the standard top in that it provides a real-time insights into CPU cores usage by SPDK lightweight threads and pollers. Have you ever wondered which CPU core is used most by your SPDK instance? Are you building your own bdev or library and want to know if your code is running efficiently? Are your new pollers busy most of the time? The spdk_top application uses RPC calls to collect performance metrics and displays them in a report that you can analyze and determine if your code is running efficiently so that you can tune your implementation and get more from SPDK.
-
-Why doesn't the classic top utility work for SPDK? SPDK uses a polled-mode design; a reactor thread running on each CPU core assigned to an SPDK application schedules SPDK lightweight threads and pollers to run on the CPU core. Therefore, the standard Linux top utility is not effective for analyzing the CPU usage for polled-mode applications like SPDK because it just reports that they are using 100% of the CPU resources assigned to them. The spdk_top utility was developed to analyze and report the CPU cycles used to do real work vs just polling for work. The utility relies on instrumentation added to pollers to track when they are doing work vs. polling for work. The spdk_top utility gets the fine grained metrics from the pollers, analyzes and report the metrics on a per poller, thread and core basis. This information enables users to identify CPU cores that are busy doing real work so that they can determine if the application needs more or less CPU resources.
-
-# Run spdk_top
-Before running spdk_top you need to run the SPDK application whose performance you want to analyze using spdk_top.
-
-Run the spdk_top application
-
-~~~{.sh}
-./build/bin/spdk_top
-~~~
-
-# Bottom menu
-Menu at the bottom of SPDK top window shows many options for changing displayed data. Each menu item has a key associated with it in square brackets.
-
-* Quit - quits the SPDK top application.
-* TAB selection - allows to select THREADS/POLLERS/CORES tabs.
-* Previous page/Next page - scrolls up/down to the next set of rows displayed. Indicator in the bottom-left corner shows current page and number of all available pages.
-* Columns - enables/disables chosen columns in a column pop-up window.
-* Sorting - allows to sort displayed data by column in a sorting pop-up.
-* Refresh rate - takes user input from 0 to 255 and changes refresh rate to that value in seconds.
-* Item details - displays details pop-up window for highlighted data row. Selection is changed by pressing UP and DOWN arrow keys.
-* Total/Interval - changes displayed values in all tabs to either Total time (measured since start of SPDK application) or Interval time (measured since last refresh).
-
-# Threads Tab
-The threads tab displays a line item for each spdk thread. The information displayed shows:
-
-* Thread name - name of SPDK thread.
-* Core - core on which the thread is currently running.
-* Active/Timed/Paused pollers - number of pollers grouped by type on this thread.
-* Idle/Busy - how many microseconds the thread was idle/busy.
-
-\n
-By pressing ENTER key a pop-up window appears, showing above and a list of pollers running on selected thread (with poller name, type, run count and period).
-Pop-up then can be closed by pressing ESC key.
-
-To learn more about spdk threads see @ref concurrency.
-
-# Pollers Tab
-The pollers tab displays a line item for each poller. The information displayed shows:
-
-* Poller name - name of currently selected poller.
-* Type - type of poller (Active/Paused/Timed).
-* On thread - thread on which the poller is running.
-* Run count - how many times poller was run.
-* Period - poller period in microseconds. If period equals 0 then it is not displayed.
-* Status - whether poller is currently Busy (red color) or Idle (blue color).
-
-\n
-Poller pop-up window can be displayed by pressing ENTER on a selected data row and displays above information.
-Pop-up can be closed by pressing ESC key.
-
-# Cores Tab
-The cores tab provides insights into how the application is using the CPU cores assigned to it. The information displayed for each core shows:
-
-* Core - core number.
-* Thread count - number of threads currently running on core.
-* Poller count - total number of pollers running on core.
-* Idle/Busy - how many microseconds core was idle (including time when core ran pollers but did not find any work) or doing actual work.
-
-\n
-Pressing ENTER key makes a pop-up window appear, showing above information, along with a list of threads running on selected core. Cores details window allows to select a thread and display thread details pop-up on top of it. To close both pop-ups use ESC key.

doc/spdkcli.md

@@ -18,7 +18,7 @@ Package dependencies at the moment include:
 
 ~~~{.sh}
 ./scripts/setup.sh
-./build/bin/vhost -c vhost.json
+./app/vhost/vhost -c vhost.conf
 ~~~
 
 ### Run SPDK CLI

doc/system_configuration.md

@@ -18,118 +18,7 @@ x86_64 system, or add `iommu.passthrough=1` on arm64 systems.
 
 There are also some instances where a user may not want to use `uio_pci_generic` or the kernel
 version they are using has a bug where `uio_pci_generic` [fails to bind to NVMe drives](https://github.com/spdk/spdk/issues/399).
-In these cases, users can build the `igb_uio` kernel module which can be found in dpdk-kmods repository.
-To ensure that the driver is properly bound, users should specify `DRIVER_OVERRIDE=/path/to/igb_uio.ko`.
-
-# Running SPDK as non-priviledged user {#system_configuration_nonroot}
-
-One of the benefits of using the `VFIO` Linux kernel driver is the ability to
-perform DMA operations with peripheral devices as unprivileged user. The
-permissions to access particular devices still need to be granted by the system
-administrator, but only on a one-time basis. Note that this functionality
-is supported with DPDK starting from version 18.11.
-
-## Hugetlbfs access
-
-Make sure the target user has RW access to at least one hugepage mount.
-A good idea is to create a new mount specifically for SPDK:
-
-~~~{.sh}
-# mkdir /mnt/spdk_hugetlbfs
-# mount -t hugetlbfs -o uid=spdk,size=<value> none /mnt/spdk_hugetlbfs
-~~~
-
-Then start SPDK applications with an additional parameter `--huge-dir /mnt/spdk_hugetlbfs`
-
-Full guide on configuring hugepage mounts is available in the
-[Linux Hugetlbpage Documentation](https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt)
-
-## Device access {#system_configuration_nonroot_device_access}
-
-`VFIO` device access is protected with sysfs file permissions and can be
-configured with chown/chmod.
-
-Please note that the VFIO device isolation is based around IOMMU groups and it's
-only possible to change permissions of the entire group, which might possibly
-consist of more than one device. (You could also apply a custom kernel patch to
-further isolate those devices in the kernel, but it comes with potential risks
-as described on
-[Alex Williamson's VFIO blog](https://vfio.blogspot.com/2014/08/iommu-groups-inside-and-out.html),
-with the patch in question available here:
-[[PATCH] pci: Enable overrides for missing ACS capabilities](https://lkml.org/lkml/2013/5/30/513))
-
-Let's assume we want to use PCI device `0000:04:00.0`. First of all, verify
-that it has an IOMMU group assigned:
-
-~~~{.sh}
-$ readlink "/sys/bus/pci/devices/0000:00:04.0/iommu_group"
-~~~
-
-The output should be e.g.
-`../../../kernel/iommu_groups/5`
-
-Which means that the device is a part of the IOMMU group 5. We can check if
-there are any other devices in that group.
-
-~~~{.sh}
-$ ls /sys/kernel/iommu_groups/5/devices/
-0000:00:04.0  0000:00:04.1  0000:00:04.2  0000:00:04.3  0000:00:04.4  0000:00:04.5  0000:00:04.6  0000:00:04.7
-~~~
-
-In this case `0000:04:00.0` is an I/OAT channel which comes with 7 different
-channels associated with the same IOMMU group.
-
-To give the user `spdk` full access to the VFIO IOMMU group 5 and all its
-devices, use the following:
-
-~~~{.sh}
-# chown spdk /dev/vfio/5
-~~~
-
-## Memory constraints {#system_configuration_nonroot_memory_constraints}
-
-As soon as the first device is attached to SPDK, all of SPDK memory will be
-mapped to the IOMMU through the VFIO APIs. VFIO will try to mlock that memory and
-will likely exceed user ulimit on locked memory. Besides having various
-SPDK errors and failures, this would also pollute the syslog with the following
-entries:
-
-`vfio_pin_pages: RLIMIT_MEMLOCK`
-
-The limit can be checked by running the following command as target user:
-(output in kilobytes)
-
-~~~{.sh}
-$ ulimit -l
-~~~
-
-On Ubuntu 18.04 this returns 16384 (16MB) by default, which is way below
-what SPDK needs.
-
-The limit can be increased with one of the methods below. Keep in mind SPDK will
-try to map not only its reserved hugepages, but also all the memory that's
-shared by its vhost clients as described in the
-[Vhost processing guide](https://spdk.io/doc/vhost_processing.html#vhost_processing_init).
-
-### Increasing the memlock limit permanently
-
-Open the `/etc/security/limits.conf` file as root and append the following:
-
-```
-spdk     hard   memlock           unlimited
-spdk     soft   memlock           unlimited
-```
-
-Then logout from the target user account. The changes will take effect after the next login.
-
-### Increasing the memlock for a specific process
-
-Linux offers a `prlimit` utility that can override limits of any given process.
-On Ubuntu, it is a part of the `util-linux` package.
-
-~~~{.sh}
-# prlimit --pid <pid> --memlock=<soft>:<hard>
-~~~
-
-Note that the above needs to be executed before the first device is attached to
-the SPDK application.
+In these cases, users building with the DPDK submodule can build the `igb_uio` kernel module by
+supplying `--with-igb-uio-driver` to `./configure`. Upon a successful make, the file will be
+located at `dpdk/build/build/kmod/igb_uio.ko`. To ensure that the driver is properly bound, users
+should specify `DRIVER_OVERRIDE=/path/to/igb_uio.ko`.

doc/tools.md

@@ -1,5 +1,4 @@
 # Tools {#tools}
 
 - @subpage spdkcli
-- @subpage bdevperf
-- @subpage spdk_top
+- @subpage nvme-cli

doc/user_guides.md

@@ -1,8 +1,6 @@
 # User Guides {#user_guides}
 
 - @subpage system_configuration
-- @subpage libraries
-- @subpage pkgconfig
 - @subpage app_overview
 - @subpage iscsi
 - @subpage nvmf

doc/vagrant.md

@@ -147,7 +147,7 @@ vagrant@vagrant:~/spdk_repo/spdk$ make
 vagrant@vagrant:~/spdk_repo/spdk$ sudo ./scripts/setup.sh
 0000:00:0e.0 (80ee 4e56): nvme -> uio_pci_generic
 
-vagrant@vagrant:~/spdk_repo/spdk$ sudo build/examples/hello_world
+vagrant@vagrant:~/spdk_repo/spdk$ sudo examples/nvme/hello_world/hello_world
 Starting SPDK v18.10-pre / DPDK 18.05.0 initialization...
 [ DPDK EAL parameters: hello_world -c 0x1 --legacy-mem --file-prefix=spdk0 --base-virtaddr=0x200000000000 --proc-type=auto ]
 EAL: Detected 4 lcore(s)

doc/vhost.md

@@ -91,13 +91,13 @@ be restricted to run on a subset of these CPU cores. See @ref vhost_vdev_create
 details.
 
 ~~~{.sh}
-build/bin/vhost -S /var/tmp -m 0x3
+app/vhost/vhost -S /var/tmp -m 0x3
 ~~~
 
 To list all available vhost options use the following command.
 
 ~~~{.sh}
-build/bin/vhost -h
+app/vhost/vhost -h
 ~~~
 
 # SPDK Configuration {#vhost_config}
@@ -105,7 +105,7 @@ build/bin/vhost -h
 ## Create bdev (block device) {#vhost_bdev_create}
 
 SPDK bdevs are block devices which will be exposed to the guest OS.
-For vhost-scsi, bdevs are exposed as SCSI LUNs on SCSI devices attached to the
+For vhost-scsi, bdevs are exposed as as SCSI LUNs on SCSI devices attached to the
 vhost-scsi controller in the guest OS.
 For vhost-blk, bdevs are exposed directly as block devices in the guest OS and are
 not associated at all with SCSI.
@@ -171,6 +171,26 @@ extra `-r` or `--readonly` parameter.
 scripts/rpc.py vhost_create_blk_controller --cpumask 0x1 -r vhost.1 Malloc0
 ~~~
 
+### Vhost-NVMe (experimental)
+
+The following RPC will attach the Malloc0 bdev to the vhost.0 vhost-nvme
+controller. Malloc0 will appear as Namespace 1 of vhost.0 controller. Users
+can use `--cpumask` parameter to specify which cores should be used for this
+controller. Users must specify the maximum I/O queues supported for the
+controller, at least 1 Namespace is required for each controller.
+
+~~~{.sh}
+$rpc_py vhost_create_nvme_controller --cpumask 0x1 vhost.2 16
+$rpc_py vhost_nvme_controller_add_ns vhost.2 Malloc0
+~~~
+
+Users can use the following command to remove the controller, all the block
+devices attached to controller's Namespace will be removed automatically.
+
+~~~{.sh}
+$rpc_py vhost_delete_controller vhost.2
+~~~
+
 ## QEMU {#vhost_qemu_config}
 
 Now the virtual machine can be started with QEMU.  The following command-line
@@ -209,6 +229,13 @@ Finally, specify the SPDK vhost devices:
 -device vhost-user-blk-pci,id=blk0,chardev=char1
 ~~~
 
+### Vhost-NVMe (experimental)
+
+~~~{.sh}
+-chardev socket,id=char2,path=/var/tmp/vhost.2
+-device vhost-user-nvme,id=nvme0,chardev=char2,num_io_queues=4
+~~~
+
 ## Example output {#vhost_example}
 
 This example uses an NVMe bdev alongside Mallocs. SPDK vhost application is started
@@ -220,9 +247,9 @@ host:~# HUGEMEM=2048 ./scripts/setup.sh
 ~~~
 
 ~~~{.sh}
-host:~# ./build/bin/vhost -S /var/tmp -s 1024 -m 0x3 &
+host:~# ./app/vhost/vhost -S /var/tmp -s 1024 -m 0x3 &
 Starting DPDK 17.11.0 initialization...
-[ DPDK EAL parameters: vhost -c 3 -m 1024 --main-lcore=1 --file-prefix=spdk_pid156014 ]
+[ DPDK EAL parameters: vhost -c 3 -m 1024 --master-lcore=1 --file-prefix=spdk_pid156014 ]
 EAL: Detected 48 lcore(s)
 EAL: Probing VFIO support...
 EAL: VFIO support initialized
@@ -346,7 +373,7 @@ be aborted - possibly flooding a VM with syslog warnings and errors.
 
 ### Hot-attach
 
-Hot-attach is done by simply attaching a bdev to a vhost controller with a QEMU VM
+Hot-attach is is done by simply attaching a bdev to a vhost controller with a QEMU VM
 already started. No other extra action is necessary.
 
 ~~~{.sh}

doc/vmd.md

@@ -47,18 +47,18 @@ $ 5d:05.5 RAID bus controller: Intel Corporation Device 201d (rev 04)
 $ d7:05.5 RAID bus controller: Intel Corporation Device 201d (rev 04)
 ```
 
-Run setup.sh script with VMD devices set in PCI_ALLOWED.
+Run setup.sh script with VMD devices set in PCI_WHITELIST.
 
 Example:
 ```
-$ PCI_ALLOWED="0000:5d:05.5 0000:d7:05.5" scripts/setup.sh
+$ PCI_WHITELIST="0000:5d:05.5 0000:d7:05.5" scripts/setup.sh
 ```
 
 Check for available devices behind the VMD with spdk_lspci.
 
 Example:
 ```
-$ ./build/bin/spdk_lspci
+$ ./app/spdk_lspci/spdk_lspci
 
  5d0505:01:00.0 (8086 a54) (NVMe disk behind VMD)
  5d0505:03:00.0 (8086 a54) (NVMe disk behind VMD)
@@ -79,6 +79,12 @@ $ ./scripts/rpc.py bdev_nvme_attach_controller -b NVMe1 -t PCIe -a 5d0505:01:00.
 
 When application framework is used, VMD section needs to be added to the configuration file:
 
+Text config:
+```
+[VMD]
+  Enable True
+```
+
 JSON config:
 ```
 {
@@ -94,7 +100,7 @@ JSON config:
 
 or use RPC call before framework starts e.g.
 ```
-$ ./build/bin/spdk_tgt --wait_for_rpc
+$ ./app/spdk_tgt/spdk_tgt --wait_for_rpc
 $ ./scripts/rpc.py enable_vmd
 $ ./scripts/rpc.py framework_start_init
 ```

doc/vpp_integration.md

@@ -0,0 +1,237 @@
+# Vector Packet Processing {#vpp_integration}
+
+VPP (part of [Fast Data - Input/Output](https://fd.io/) project) is an extensible
+userspace framework providing networking functionality. It is built around the concept of
+packet processing graph (see [What is VPP?](https://wiki.fd.io/view/VPP/What_is_VPP?)).
+
+Detailed instructions for **simplified steps 1-3** below, can be found on
+VPP [Quick Start Guide](https://wiki.fd.io/view/VPP).
+
+*SPDK supports VPP version 19.04.2.*
+
+# 1. Building VPP (optional) {#vpp_build}
+
+*Please skip this step if using already built packages.*
+
+Clone and checkout VPP
+~~~
+git clone https://gerrit.fd.io/r/vpp && cd vpp
+git checkout v19.04.2
+~~~
+
+Install VPP build dependencies
+~~~
+make install-dep
+~~~
+
+Build and create .rpm packages
+~~~
+make pkg-rpm
+~~~
+
+Alternatively, build and create .deb packages
+~~~
+make bootstrap && make pkg-deb
+~~~
+
+Packages can be found in `vpp/build-root/` directory.
+
+For more in depth instructions please see Building section in
+[VPP documentation](https://wiki.fd.io/view/VPP/Pulling,_Building,_Running,_Hacking_and_Pushing_VPP_Code#Building)
+
+# 2. Installing VPP {#vpp_install}
+
+Packages can be installed from a distribution repository or built in previous step.
+Minimal set of packages consists of `vpp`, `vpp-lib` and `vpp-devel`.
+
+*Note: Please remove or modify /etc/sysctl.d/80-vpp.conf file with appropriate values
+dependent on number of hugepages that will be used on system.*
+
+# 3. Running VPP {#vpp_run}
+
+VPP takes over any network interfaces that were bound to userspace driver,
+for details please see DPDK guide on
+[Binding and Unbinding Network Ports to/from the Kernel Modules](http://dpdk.org/doc/guides/linux_gsg/linux_drivers.html#binding-and-unbinding-network-ports-to-from-the-kernel-modules).
+
+VPP is installed as service and disabled by default. To start VPP with default config:
+~~~
+sudo systemctl start vpp
+~~~
+
+Alternatively, use `vpp` binary directly
+~~~
+sudo vpp unix {cli-listen /run/vpp/cli.sock} session { evt_qs_memfd_seg } socksvr { socket-name /run/vpp-api.sock }
+~~~
+
+# 4. Configure VPP {#vpp_config}
+
+VPP can be configured using a VPP startup file and the `vppctl` command; By default, the VPP startup file is `/etc/vpp/startup.conf`, however, you can pass any file with the `-c` vpp command argument.
+
+## Startup configuration
+
+Some key values from iSCSI point of view includes:
+
+CPU section (`cpu`):
+
+- `main-core <lcore>` -- logical CPU core used for main thread.
+- `corelist-workers <lcore list>` -- logical CPU cores where worker threads are running.
+
+DPDK section (`dpdk`):
+
+- `num-rx-queues <num>` -- number of receive queues.
+- `num-tx-queues <num>` -- number of transmit queues.
+- `dev <PCI address>` -- whitelisted device.
+
+Session section (`session`):
+
+- `evt_qs_memfd_seg` -- uses a memfd segment for event queues. This is required for SPDK.
+
+Socket server session (`socksvr`):
+
+- `socket-name <path>` -- configure API socket filename (curently SPDK uses default path `/run/vpp-api.sock`).
+
+Plugins section (`plugins`):
+
+- `plugin <plugin name> { [enable|disable] }` -- enable or disable VPP plugin.
+
+### Example
+
+~~~
+unix {
+	nodaemon
+	cli-listen /run/vpp/cli.sock
+}
+cpu {
+	main-core 1
+}
+session {
+	evt_qs_memfd_seg
+}
+socksvr {
+	socket-name /run/vpp-api.sock
+}
+plugins {
+	plugin default { disable }
+	plugin dpdk_plugin.so { enable }
+}
+~~~
+
+## vppctl command tool
+
+The `vppctl` command tool allows users to control VPP at runtime via a command prompt
+~~~
+sudo vppctl
+~~~
+
+Or, by sending single command directly. For example to display interfaces within VPP:
+~~~
+sudo vppctl show interface
+~~~
+
+Useful commands:
+
+- `show interface` -- show interfaces settings, state and some basic statistics.
+- `show interface address` -- show interfaces state and assigned addresses.
+
+- `set interface ip address <VPP interface> <Address>` -- set interfaces IP address.
+- `set interface state <VPP interface> [up|down]` -- bring interface up or down.
+
+- `show errors` -- show error counts.
+
+## Example: Configure two interfaces to be available via VPP
+
+We want to configure two DPDK ports with PCI addresses 0000:09:00.1 and 0000:0b:00.1
+to be used as portals 10.0.0.1/24 and 10.10.0.1/24.
+
+In the VPP startup file (e.g. `/etc/vpp/startup.conf`), whitelist the interfaces
+by specifying PCI addresses in section dpdk:
+~~~
+	dev 0000:09:00.1
+	dev 0000:0b:00.1
+~~~
+
+Bind PCI NICs to UIO driver (`igb_uio` or `uio_pci_generic`).
+
+Restart vpp and use vppctl tool to verify interfaces.
+~~~
+$ vppctl show interface
+              Name               Idx    State  MTU (L3/IP4/IP6/MPLS)     Counter          Count
+
+FortyGigabitEthernet9/0/1         1     down         9000/0/0/0
+FortyGigabitEthernetb/0/1         2     down         9000/0/0/0
+~~~
+
+Set appropriate addresses and bring interfaces up:
+~~~
+$ vppctl set interface ip address FortyGigabitEthernet9/0/1 10.0.0.1/24
+$ vppctl set interface state FortyGigabitEthernet9/0/1 up
+$ vppctl set interface ip address FortyGigabitEthernetb/0/1 10.10.0.1/24
+$ vppctl set interface state FortyGigabitEthernetb/0/1 up
+~~~
+
+Verify configuration:
+~~~
+$ vppctl show interface address
+FortyGigabitEthernet9/0/1 (up):
+  L3 10.0.0.1/24
+FortyGigabitEthernetb/0/1 (up):
+  L3 10.10.0.1/24
+~~~
+
+Now, both interfaces are ready to use. To verify conectivity you can ping
+10.0.0.1 and 10.10.0.1 addresses from another machine.
+
+## Example: Tap interfaces on single host
+
+For functional test purposes a virtual tap interface can be created,
+so no additional network hardware is required.
+This will allow network communication between SPDK iSCSI target using VPP end of tap
+and kernel iSCSI initiator using the kernel part of tap. A single host is used in this scenario.
+
+Create tap interface via VPP
+~~~
+    vppctl tap connect tap0
+    vppctl set interface state tapcli-0 up
+    vppctl set interface ip address tapcli-0 10.0.0.1/24
+    vppctl show int addr
+~~~
+
+Assign address on kernel interface
+~~~
+    sudo ip addr add 10.0.0.2/24 dev tap0
+    sudo ip link set tap0 up
+~~~
+
+To verify connectivity
+~~~
+    ping 10.0.0.1
+~~~
+
+# 5. Building SPDK with VPP {#vpp_built_into_spdk}
+
+Support for VPP can be built into SPDK by using configuration option.
+~~~
+configure --with-vpp
+~~~
+
+Alternatively, directory with built libraries can be pointed at
+and will be used for compilation instead of installed packages.
+~~~
+configure --with-vpp=/path/to/vpp/repo/build-root/install-vpp-native/vpp
+~~~
+
+# 6. Running SPDK with VPP {#vpp_running_with_spdk}
+
+VPP application has to be started before SPDK application, in order to enable
+usage of network interfaces. For example, if you use SPDK iSCSI target or
+NVMe-oF target, after the initialization finishes, interfaces configured within
+VPP will be available to be configured as portal addresses.
+
+Moreover, you do not need to specifiy which TCP sock implementation (e.g., posix,
+VPP) to be used through configuration file or RPC call. Since SPDK program
+automatically determines the protocol according to the configured portal addresses
+info. For example, you can specify a Listen address in NVMe-oF subsystem
+configuration such as "Listen TCP 10.0.0.1:4420". SPDK programs automatically
+uses different implemenation to listen this provided portal info via posix or
+vpp implemenation(if compiled in SPDK program), and only one implementation can
+successfully listen on the provided portal.

dpdk

@@ -1 +1 @@
-Subproject commit 4f93dbc0c0ab3804abaa20123030ad7fccf78709
+Subproject commit ef71bfaface10cc19b75e45d3158ab71a788e3a9

dpdkbuild/Makefile

@@ -36,64 +36,101 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
 .PHONY: all clean install uninstall
 
-DPDK_OPTS = -Denable_docs=false
+DPDK_FRAMEWORK = n
+DPDK_OPTS =
 DPDK_CFLAGS =
+DPDK_OPTS += CONFIG_RTE_BUILD_SHARED_LIB=n
+DPDK_OPTS += CONFIG_RTE_LIBRTE_PMD_AESNI_MB=n
+DPDK_OPTS += CONFIG_RTE_LIBRTE_REORDER=n
+DPDK_OPTS += CONFIG_RTE_LIBRTE_ETHER=n
+DPDK_OPTS += CONFIG_RTE_LIBRTE_CMDLINE=n
+DPDK_OPTS += CONFIG_RTE_LIBRTE_METER=n
+DPDK_OPTS += CONFIG_RTE_LIBRTE_HASH=n
+DPDK_OPTS += CONFIG_RTE_LIBRTE_VHOST=n
+DPDK_OPTS += CONFIG_RTE_EAL_IGB_UIO=n
+DPDK_OPTS += CONFIG_RTE_LIBRTE_PMD_QAT=n
+DPDK_OPTS += CONFIG_RTE_LIBRTE_PMD_QAT_SYM=n
+DPDK_OPTS += CONFIG_RTE_LIBRTE_PMD_ISAL=n
 
-DPDK_KMODS = false
-ifeq ($(OS),FreeBSD)
-DPDK_KMODS = true
-endif
-DPDK_OPTS += -Denable_kmods=$(DPDK_KMODS)
-
-ifeq ($(CONFIG_DEBUG),y)
-DPDK_OPTS += --buildtype=debug
-endif
-
-# the drivers we use
-DPDK_DRIVERS = bus bus/pci bus/vdev mempool/ring
-
-# common crypto/reduce drivers
-ifeq ($(findstring y,$(CONFIG_CRYPTO)$(CONFIG_REDUCE)),y)
-DPDK_DRIVERS += crypto/qat compress/qat common/qat
+ifeq ($(CONFIG_SHARED),y)
+DPDK_OPTS += CONFIG_RTE_BUILD_SHARED_LIB=y
+DPDK_LDFLAGS+= -rpath $(SPDK_ROOT_DIR)/dpdk/build/lib
 endif
 
 ifeq ($(CONFIG_CRYPTO),y)
-# crypto/qat is just a stub, the compress/qat pmd is used instead
-DPDK_DRIVERS += crypto crypto/aesni_mb
+DPDK_FRAMEWORK = y
+DPDK_OPTS += CONFIG_RTE_LIBRTE_PMD_AESNI_MB=y
+DPDK_OPTS += CONFIG_RTE_LIBRTE_REORDER=y
 DPDK_CFLAGS += -I$(IPSEC_MB_DIR)
 DPDK_LDFLAGS += -L$(IPSEC_MB_DIR)
 endif
 
 ifeq ($(CONFIG_REDUCE),y)
-DPDK_DRIVERS += compress compress/isal
+DPDK_FRAMEWORK = y
+DPDK_OPTS += CONFIG_RTE_LIBRTE_PMD_ISAL=y
 DPDK_CFLAGS += -I$(ISAL_DIR)
-DPDK_LDFLAGS += -L$(ISAL_DIR)/.libs -lisal
+DPDK_LDFLAGS += -L$(ISAL_DIR)/.libs
 endif
 
-DPDK_OPTS += -Dmachine=$(TARGET_ARCHITECTURE)
+ifeq ($(CONFIG_VHOST),y)
+DPDK_OPTS += CONFIG_RTE_LIBRTE_ETHER=y
+DPDK_OPTS += CONFIG_RTE_LIBRTE_CMDLINE=y
+DPDK_OPTS += CONFIG_RTE_LIBRTE_METER=y
+DPDK_OPTS += CONFIG_RTE_LIBRTE_HASH=y
+DPDK_OPTS += CONFIG_RTE_LIBRTE_VHOST=y
+endif
+
+ifeq ($(CONFIG_IGB_UIO_DRIVER),y)
+DPDK_OPTS += CONFIG_RTE_EAL_IGB_UIO=y
+endif
+
+ifeq ($(CONFIG_RAID5),y)
+DPDK_OPTS += CONFIG_RTE_LIBRTE_HASH=y
+endif
+
+ifeq ($(DPDK_FRAMEWORK),y)
+DPDK_OPTS += CONFIG_RTE_LIBRTE_PMD_QAT=y
+DPDK_OPTS += CONFIG_RTE_LIBRTE_PMD_QAT_SYM=y
+endif
+
+ifeq ($(TARGET_MACHINE),aarch64)
+DPDK_CONFIG := arm64-armv8a
+else
+DPDK_CONFIG := $(TARGET_MACHINE)-native
+endif
 
 ifneq ($(CONFIG_CROSS_PREFIX),)
-ifeq ($(findstring mingw,$(CONFIG_CROSS_PREFIX)),mingw)
-DPDK_OPTS += --cross-file $(SPDK_ROOT_DIR)/dpdk/config/x86/cross-mingw
-else
-$(error Automatic DPDK cross build is not supported. Please compile DPDK manually \
-with e.g. `meson build --cross-file config/arm/arm64_armv8_linux_gcc`)
+DPDK_OPTS += CROSS=$(CONFIG_CROSS_PREFIX)-
 endif
+
+ifeq ($(OS),Linux)
+DPDK_CONFIG := $(DPDK_CONFIG)-linuxapp
+NPROC := $(shell nproc)
+else
+ifeq ($(OS),FreeBSD)
+DPDK_CONFIG := $(DPDK_CONFIG)-bsdapp
+NPROC := $(shell sysctl hw.ncpu | awk '{print $$NF}')
+endif
+endif
+
+ifeq ($(CC_TYPE),clang)
+DPDK_CONFIG := $(DPDK_CONFIG)-clang
+else
+DPDK_CONFIG := $(DPDK_CONFIG)-gcc
 endif
 
 DPDK_CFLAGS += -fPIC
 
+ifeq ($(CONFIG_DEBUG),y)
+DPDK_CFLAGS += -O0 -g
+endif
+
 ifeq ($(CONFIG_WERROR),y)
 DPDK_CFLAGS += -Werror
 else
 DPDK_CFLAGS += -Wno-error
 endif
 
-ifeq ($(CONFIG_CET),y)
-DPDK_CFLAGS += -fcf-protection
-DPDK_LDFLAGS += -fcf-protection
-endif
-
 ifdef EXTRA_DPDK_CFLAGS
 $(warning EXTRA_DPDK_CFLAGS defined, possibly to work around an unsupported compiler version)
 $(shell sleep 1)
@@ -102,80 +139,20 @@ endif
 # Allow users to specify EXTRA_DPDK_CFLAGS if they want to build DPDK using unsupported compiler versions
 DPDK_CFLAGS += $(EXTRA_DPDK_CFLAGS)
 
-ifeq ($(CC_TYPE),gcc)
-GCC_MAJOR = $(shell echo __GNUC__ | $(CC) -E -x c - | tail -n 1)
-ifeq ($(shell test $(GCC_MAJOR) -ge 10 && echo 1), 1)
-#1. gcc 10 complains on operations with zero size arrays in rte_cryptodev.c, so
-#disable this warning
-#2. gcc 10 disables fcommon by default and complains on multiple definition of
-#aesni_mb_logtype_driver symbol which is defined in header file and presented in sevral
-#translation units
-DPDK_CFLAGS += -Wno-stringop-overflow -fcommon
-endif
-endif
-
 # Force-disable scan-build
 SUB_CC = $(patsubst %ccc-analyzer,$(DEFAULT_CC),$(CC))
 
-DPDK_ALL_DRIVER_DIRS = $(shell find $(SPDK_ROOT_DIR)/dpdk/drivers -mindepth 1 -type d)
-DPDK_ALL_DRIVERS = $(DPDK_ALL_DRIVER_DIRS:$(SPDK_ROOT_DIR)/dpdk/drivers/%=%)
-DPDK_DISABLED_DRVERS = $(filter-out $(DPDK_DRIVERS),$(DPDK_ALL_DRIVERS))
+$(SPDK_ROOT_DIR)/dpdk/build: $(SPDK_ROOT_DIR)/mk/cc.mk $(SPDK_ROOT_DIR)/include/spdk/config.h
+	$(Q)rm -rf $(SPDK_ROOT_DIR)/dpdk/build
+	$(Q)$(MAKE) -C $(SPDK_ROOT_DIR)/dpdk config T=$(DPDK_CONFIG) $(DPDK_OPTS)
 
-ifneq ($(OS),FreeBSD)
-SED_INPLACE_FLAG = "-i"
-MESON_PREFIX = $(SPDK_ROOT_DIR)/dpdk/build
-else
-SED_INPLACE_FLAG = "-i ''"
-MESON_PREFIX = "/"
-endif
-
-# Some ninja versions come with a (broken?) jobserver which defaults to use
-# only 1 thread for the build. We workaround this by specifying -j to ninja
-# with the same value as top-makefile. This is OK as long as DPDK is not built
-# in parralel with anything else, which is the case for now.
-ifeq ($(MAKE_PID),)
-MAKE_PID := $(shell echo $$PPID)
-endif
-
-MAKE_NUMJOBS := $(shell ps T | sed -nE 's/[[:space:]]*$(MAKE_PID)[[:space:]].* (-j|--jobs=)( *[0-9]+).*/\1\2/p')
-
-all: $(SPDK_ROOT_DIR)/dpdk/build-tmp
-	$(Q)# DPDK doesn't handle nested make calls, so unset MAKEFLAGS
-	$(Q)env -u MAKEFLAGS ninja -C $(SPDK_ROOT_DIR)/dpdk/build-tmp $(MAKE_NUMJOBS)
-	$(Q) \
-	# Meson on FreeBSD sometimes appends --prefix value to the default DESTDIR (which is e.g. \
-	# /usr/local) instead of replacing it. --prefix needs to be an absolute path, so we set \
-	# it to / and then set DESTDIR directly, so libs and headers are copied to "DESTDIR//". \
-	# DPDK kernel modules are set to install in $DESTDIR/boot/modules, but we move them \
-	# to DESTDIR/kmod to be consistent with the makefile build. \
-	# \
-	# Also use meson install --only-changed instead of ninja install so that the shared \
-	# libraries don't get reinstalled when they haven't been rebuilt - this avoids all of \
-	# our applications getting relinked even when nothing has changed.
-	$(Q)if [ "$(OS)" = "FreeBSD" ]; then \
-		env -u MAKEFLAGS DESTDIR=$(SPDK_ROOT_DIR)/dpdk/build ninja -C $(SPDK_ROOT_DIR)/dpdk/build-tmp $(MAKE_NUMJOBS) install > /dev/null && \
-		mv $(SPDK_ROOT_DIR)/dpdk/build/boot/modules $(SPDK_ROOT_DIR)/dpdk/build/kmod; \
-	else \
-		env -u MAKEFLAGS meson install -C $(SPDK_ROOT_DIR)/dpdk/build-tmp --only-changed > /dev/null; \
-	fi
-
-$(SPDK_ROOT_DIR)/dpdk/build-tmp: $(SPDK_ROOT_DIR)/mk/cc.mk $(SPDK_ROOT_DIR)/include/spdk/config.h
-	$(Q)rm -rf $(SPDK_ROOT_DIR)/dpdk/build $(SPDK_ROOT_DIR)/dpdk/build-tmp
-	$(Q)cd "$(SPDK_ROOT_DIR)/dpdk"; CC="$(SUB_CC)" meson --prefix="$(MESON_PREFIX)" --libdir lib -Dc_args="$(DPDK_CFLAGS)" -Dc_link_args="$(DPDK_LDFLAGS)" $(DPDK_OPTS) -Ddisable_drivers="$(shell echo $(DPDK_DISABLED_DRVERS) | sed -E "s/ +/,/g")" build-tmp
-	$(Q)sed $(SED_INPLACE_FLAG) 's/#define RTE_EAL_PMD_PATH .*/#define RTE_EAL_PMD_PATH ""/g' $(SPDK_ROOT_DIR)/dpdk/build-tmp/rte_build_config.h
-	$(Q) \
-	# TODO Meson build adds libbsd dependency when it's available. This means any app will be \
-	# forced to link with -lbsd, but only if it's available on the system. The clean way to \
-	# handle this would be to rely on DPDK's pkg-config file which will contain the -lbsd when \
-	# required. For now just remove the libbsd dependency. DPDK will fallback to its internal \
-	# functions.
-	$(Q)sed $(SED_INPLACE_FLAG) 's/#define RTE_USE_LIBBSD .*//g' $(SPDK_ROOT_DIR)/dpdk/build-tmp/rte_build_config.h
+all: $(SPDK_ROOT_DIR)/dpdk/build
+	$(Q)$(MAKE) -C $(SPDK_ROOT_DIR)/dpdk/build EXTRA_CFLAGS="$(DPDK_CFLAGS)" EXTRA_LDFLAGS="$(DPDK_LDFLAGS)" CC=$(SUB_CC) T="$(DPDK_CONFIG)" $(DPDK_OPTS)
 
 clean:
-	$(Q)rm -rf $(SPDK_ROOT_DIR)/dpdk/build $(SPDK_ROOT_DIR)/dpdk/build-tmp
+	$(Q)rm -rf $(SPDK_ROOT_DIR)/dpdk/build
 
-install:
-	@:
+install: all
 
 uninstall:
 	@:

etc/spdk/iscsi.conf.in

@@ -0,0 +1,230 @@
+# iSCSI target configuration file
+#
+# Please write all parameters using ASCII.
+# The parameter must be quoted if it includes whitespace.
+#
+# Configuration syntax:
+# Leading whitespace is ignored.
+# Lines starting with '#' are comments.
+# Lines ending with '\' are concatenated with the next line.
+# Bracketed ([]) names define sections
+
+[Global]
+  # Shared Memory Group ID. SPDK applications with the same ID will share memory.
+  # Default: <the process PID>
+  #SharedMemoryID 0
+
+  # Disable PCI access. PCI is enabled by default. Setting this
+  # option will hide any PCI device from all SPDK modules, making
+  # SPDK act as if they don't exist.
+  #NoPci Yes
+
+  # Tracepoint group mask for spdk trace buffers
+  # Default: 0x0 (all tracepoint groups disabled)
+  # Set to 0xFFFF to enable all tracepoint groups.
+  #TpointGroupMask 0x0
+
+# Users may activate entries in this section to override default values for
+# global parameters in the block device (bdev) subsystem.
+[Bdev]
+  # Number of spdk_bdev_io structures allocated in the global bdev subsystem pool.
+  #BdevIoPoolSize 65536
+
+  # Maximum number of spdk_bdev_io structures to cache per thread.
+  #BdevIoCacheSize 256
+
+[iSCSI]
+  # node name (not include optional part)
+  # Users can optionally change this to fit their environment.
+  NodeBase "iqn.2016-06.io.spdk"
+
+  AuthFile /usr/local/etc/spdk/auth.conf
+
+  # Socket I/O timeout sec. (0 is infinite)
+  Timeout 30
+
+  # authentication information for discovery session
+  # Options:
+  # None, Auto, CHAP and Mutual.  Note that Mutual infers CHAP.
+  DiscoveryAuthMethod Auto
+
+  #MaxSessions 128
+  #MaxConnectionsPerSession 2
+
+  # iSCSI initial parameters negotiate with initiators
+  # NOTE: incorrect values might crash
+  DefaultTime2Wait 2
+  DefaultTime2Retain 60
+
+  # Maximum amount in bytes of unsolicited data the iSCSI
+  # initiator may send to the target during the execution of
+  # a single SCSI command.
+  FirstBurstLength 8192
+
+  ImmediateData Yes
+  ErrorRecoveryLevel 0
+
+# Users must change the PortalGroup section(s) to match the IP addresses
+#  for their environment.
+# PortalGroup sections define which network portals the iSCSI target
+#  will use to listen for incoming connections.  These are also used to
+#  determine which targets are accessible over each portal group.
+# Up to 1024 portal directives are allowed.  These define the network
+#  portals of the portal group. The user must specify a IP address
+#  for each network portal, and may optionally specify a port.
+#  If the port is omitted, 3260 will be used.
+#  Syntax:
+#    Portal <Name> <IP address>[:<port>]
+[PortalGroup1]
+  Portal DA1 192.168.2.21:3260
+  Portal DA2 192.168.2.22:3260
+
+# Users must change the InitiatorGroup section(s) to match the IP
+#  addresses and initiator configuration in their environment.
+# Netmask can be used to specify a single IP address or a range of IP addresses
+#  Netmask 192.168.1.20   <== single IP address
+#  Netmask 192.168.1.0/24 <== IP range 192.168.1.*
+[InitiatorGroup1]
+  InitiatorName ANY
+  Netmask 192.168.2.0/24
+
+# NVMe configuration options
+[Nvme]
+  # NVMe Device Whitelist
+  # Users may specify which NVMe devices to claim by their transport id.
+  # See spdk_nvme_transport_id_parse() in spdk/nvme.h for the correct format.
+  # The second argument is the assigned name, which can be referenced from
+  # other sections in the configuration file. For NVMe devices, a namespace
+  # is automatically appended to each name in the format <YourName>nY, where
+  # Y is the NSID (starts at 1).
+  TransportID "trtype:PCIe traddr:0000:00:00.0" Nvme0
+  TransportID "trtype:PCIe traddr:0000:01:00.0" Nvme1
+
+  # The number of attempts per I/O when an I/O fails. Do not include
+  # this key to get the default behavior.
+  RetryCount 4
+  # Timeout for each command, in microseconds. If 0, don't track timeouts.
+  TimeoutUsec 0
+  # Action to take on command time out. Only valid when Timeout is greater
+  # than 0. This may be 'Reset' to reset the controller, 'Abort' to abort
+  # the command, or 'None' to just print a message but do nothing.
+  # Admin command timeouts will always result in a reset.
+  ActionOnTimeout None
+  # Set how often the admin queue is polled for asynchronous events.
+  # Units in microseconds.
+  AdminPollRate 100000
+  # Set how often I/O queues are polled from completions.
+  # Units in microseconds.
+  IOPollRate 0
+
+  # Disable handling of hotplug (runtime insert and remove) events,
+  # users can set to Yes if want to enable it.
+  # Default: No
+  HotplugEnable No
+
+  # Set how often the hotplug is processed for insert and remove events.
+  # Units in microseconds.
+  HotplugPollRate 0
+
+# Users may change this section to create a different number or size of
+#  malloc LUNs.
+# If the system has hardware DMA engine, it can use an IOAT
+# (i.e. Crystal Beach DMA) channel to do the copy instead of memcpy
+# by specifying "Enable Yes" in [Ioat] section.
+# Offload is disabled by default even it is available.
+[Malloc]
+  # Number of Malloc targets
+  NumberOfLuns 5
+  # Malloc targets are 128M
+  LunSizeInMB 128
+  # Block size. Default is 512 bytes.
+  BlockSize 4096
+
+# Users can use offload by specifying "Enable Yes" in this section
+# if it is available.
+# Users may use the whitelist to initialize specified devices, IDS
+#  uses BUS:DEVICE.FUNCTION to identify each Ioat channel.
+[Ioat]
+  Enable No
+  Whitelist 00:04.0
+  Whitelist 00:04.1
+
+# Users must change this section to match the /dev/sdX devices to be
+# exported as iSCSI LUNs. The devices are accessed using Linux AIO.
+# The format is:
+# AIO <file name> <bdev name> [<block size>]
+# The file name is the backing device
+# The bdev name can be referenced from elsewhere in the configuration file.
+# Block size may be omitted to automatically detect the block size of a disk.
+[AIO]
+  AIO /dev/sdb AIO0
+  AIO /dev/sdc AIO1
+  AIO /tmp/myfile AIO2 4096
+
+# PMDK libpmemblk-based block device
+[Pmem]
+  # Syntax:
+  #   Blk <pmemblk pool file name> <bdev name>
+  Blk /path/to/pmem-pool Pmem0
+
+# The Split virtual block device slices block devices into multiple smaller bdevs.
+[Split]
+  # Syntax:
+  #   Split <bdev> <count> [<size_in_megabytes>]
+
+  # Split Malloc1 into two equally-sized portions, Malloc1p0 and Malloc1p1
+  Split Malloc1 2
+
+  # Split Malloc2 into eight 1-megabyte portions, Malloc2p0 ... Malloc2p7,
+  # leaving the rest of the device inaccessible
+  Split Malloc2 8 1
+
+# The RAID virtual block device based on pre-configured block device.
+[RAID1]
+  # Unique name of this RAID device.
+  Name Raid0
+  # RAID level, only raid level 0 is supported.
+  RaidLevel 0
+  # Strip size in KB.
+  StripSize 64
+  # Number of pre-configured bdevs.
+  NumDevices 2
+  # Pre-configured bdevs name with Nvme.
+  #Devices Nvme0n1 Nvme1n1
+  # Pre-configured bdevs name with Malloc.
+  Devices Malloc3 Malloc4
+  # Pre-configured bdevs name with AIO.
+  #Devices AIO0 AIO1
+
+# Users should change the TargetNode section(s) below to match the
+#  desired iSCSI target node configuration.
+# TargetName, Mapping, LUN0 are minimum required
+[TargetNode1]
+  TargetName disk1
+  TargetAlias "Data Disk1"
+  Mapping PortalGroup1 InitiatorGroup1
+  AuthMethod Auto
+  AuthGroup AuthGroup1
+  # Enable header and data digest
+  # UseDigest Header Data
+  UseDigest Auto
+  # Use the first malloc target
+  LUN0 Malloc0
+  # Using the first AIO target
+  LUN1 AIO0
+  # Using the second storage target
+  LUN2 AIO1
+  # Using the third storage target
+  LUN3 AIO2
+  QueueDepth 128
+
+[TargetNode2]
+  TargetName disk2
+  TargetAlias "Data Disk2"
+  Mapping PortalGroup1 InitiatorGroup1
+  AuthMethod Auto
+  AuthGroup AuthGroup1
+  UseDigest Auto
+  LUN0 Nvme0n1
+  LUN1 Raid0
+  QueueDepth 32

etc/spdk/nvmf.conf.in

@@ -0,0 +1,295 @@
+# NVMf Target Configuration File
+#
+# Please write all parameters using ASCII.
+# The parameter must be quoted if it includes whitespace.
+#
+# Configuration syntax:
+# Leading whitespace is ignored.
+# Lines starting with '#' are comments.
+# Lines ending with '\' are concatenated with the next line.
+# Bracketed ([]) names define sections
+
+[Global]
+  # Tracepoint group mask for spdk trace buffers
+  # Default: 0x0 (all tracepoint groups disabled)
+  # Set to 0xFFFF to enable all tracepoint groups.
+  #TpointGroupMask 0x0
+
+  # PciBlacklist and PciWhitelist cannot be used at the same time
+  #PciBlacklist 0000:01:00.0
+  #PciBlacklist 0000:02:00.0
+  #PciWhitelist 0000:03:00.0
+  #PciWhitelist 0000:04:00.0
+
+# Users may activate entries in this section to override default values for
+# global parameters in the block device (bdev) subsystem.
+[Bdev]
+  # Number of spdk_bdev_io structures allocated in the global bdev subsystem pool.
+  #BdevIoPoolSize 65536
+
+  # Maximum number of spdk_bdev_io structures to cache per thread.
+  #BdevIoCacheSize 256
+
+# Users may change this section to create a different number or size of
+#  malloc LUNs.
+# This will generate 8 LUNs with a malloc-allocated backend.
+# Each LUN will be size 64MB and these will be named
+# Malloc0 through Malloc7.  Not all LUNs defined here are necessarily
+#  used below.
+[Malloc]
+  NumberOfLuns 8
+  LunSizeInMB 64
+
+# Users must change this section to match the /dev/sdX devices to be
+# exported as iSCSI LUNs. The devices are accessed using Linux AIO.
+# The format is:
+# AIO <file name> <bdev name>
+# The file name is the backing device
+# The bdev name can be referenced from elsewhere in the configuration file.
+# Block size may be omitted to automatically detect the block size of a disk.
+[AIO]
+  AIO /dev/sdb AIO0
+  AIO /dev/sdc AIO1
+  AIO /tmp/myfile AIO2 4096
+
+# PMDK libpmemblk-based block device
+[Pmem]
+  # Syntax:
+  #   Blk <pmemblk pool file name> <bdev name>
+  Blk /path/to/pmem-pool Pmem0
+
+# Define NVMf protocol global options
+[Nvmf]
+  # Set how often the acceptor polls for incoming connections. The acceptor is also
+  # responsible for polling existing connections that have gone idle. 0 means continuously
+  # poll. Units in microseconds.
+  AcceptorPollRate 10000
+
+  # Set how the connection is scheduled among multiple threads, current supported string value are
+  # "RoundRobin", "Host", "Transport".
+  # RoundRobin: Schedule the connection with roundrobin manner.
+  # Host: Schedule the connection according to host IP.
+  # Transport: Schedule the connection according to the transport characteristics.
+  #  For example, for  TCP transport, we can schedule the connection according to socket NAPI_ID info.
+  #  The connection which has the same socket NAPI_ID info will be grouped in the same polling group.
+  ConnectionScheduler RoundRobin
+
+# One valid transport type must be set in each [Transport].
+# The first is the case of RDMA transport and the second is the case of TCP transport.
+[Transport]
+  # Set RDMA transport type.
+  Type RDMA
+
+  # Set the maximum number of outstanding I/O per queue.
+  #MaxQueueDepth 128
+
+  # Set the maximum number of submission and completion queues per session.
+  # Setting this to '8', for example, allows for 8 submission and 8 completion queues
+  # per session.
+  #MaxQueuesPerSession 4
+
+  # Set the maximum in-capsule data size. Must be a multiple of 16.
+  # 0 is a valid choice.
+  #InCapsuleDataSize 4096
+
+  # Set the maximum I/O size. Must be a multiple of 4096.
+  #MaxIOSize 131072
+
+  # Set the I/O unit size, and this value should not be larger than MaxIOSize
+  #IOUnitSize 131072
+
+  # Set the maximum number of IO for admin queue
+  #MaxAQDepth 32
+
+  # Set the number of pooled data buffers available to the transport
+  # It is used to provide the read/write data buffers for the qpairs on this transport.
+  #NumSharedBuffers 512
+
+  # Set the number of shared buffers to be cached per poll group
+  #BufCacheSize 32
+
+  # Set the maximum number outstanding I/O per shared receive queue. Relevant only for RDMA transport
+  #MaxSRQDepth 4096
+
+[Transport]
+  # Set TCP transport type.
+  Type TCP
+
+  # Set the maximum number of outstanding I/O per queue.
+  #MaxQueueDepth 128
+
+  # Set the maximum number of submission and completion queues per session.
+  # Setting this to '8', for example, allows for 8 submission and 8 completion queues
+  # per session.
+  #MaxQueuesPerSession 4
+
+  # Set the maximum in-capsule data size. Must be a multiple of 16.
+  # 0 is a valid choice.
+  #InCapsuleDataSize 4096
+
+  # Set the maximum I/O size. Must be a multiple of 4096.
+  #MaxIOSize 131072
+
+  # Set the I/O unit size, and this value should not be larger than MaxIOSize
+  #IOUnitSize 131072
+
+  # Set the maximum number of IO for admin queue
+  #MaxAQDepth 32
+
+  # Set the number of pooled data buffers available to the transport
+  # It is used to provide the read/write data buffers for the qpairs on this transport.
+  #NumSharedBuffers 512
+
+  # Set the number of shared buffers to be cached per poll group
+  #BufCacheSize 32
+
+  # Set whether to use the C2H Success optimization, only used for TCP transport.
+  # C2HSuccess true
+
+  # Set whether to use different priority for socket, only used for TCP transport.
+  # SockPriority 0
+
+# Define FC transport
+#[Transport]
+  # Set FC transport type.
+  #Type FC
+
+  # Set the maximum number of submission and completion queues per session.
+  # Setting this to '8', for example, allows for 8 submission and 8 completion queues
+  # per session.
+  #MaxQueuesPerSession 5
+
+  # Set the maximum number of outstanding I/O per queue.
+  #MaxQueueDepth 128
+
+  # Set the maximum I/O size. Must be a multiple of 4096.
+  #MaxIOSize 65536
+
+[Nvme]
+  # NVMe Device Whitelist
+  # Users may specify which NVMe devices to claim by their transport id.
+  # See spdk_nvme_transport_id_parse() in spdk/nvme.h for the correct format.
+  # The second argument is the assigned name, which can be referenced from
+  # other sections in the configuration file. For NVMe devices, a namespace
+  # is automatically appended to each name in the format <YourName>nY, where
+  # Y is the NSID (starts at 1).
+  TransportID "trtype:PCIe traddr:0000:00:00.0" Nvme0
+  TransportID "trtype:PCIe traddr:0000:01:00.0" Nvme1
+  TransportID "trtype:PCIe traddr:0000:02:00.0" Nvme2
+  TransportID "trtype:PCIe traddr:0000:03:00.0" Nvme3
+  TransportID "trtype:RDMA adrfam:IPv4 traddr:192.168.100.8 trsvcid:4420 hostaddr:192.168.100.9 subnqn:nqn.2016-06.io.spdk:cnode1" Nvme4
+  TransportID "trtype:TCP adrfam:IPv4 traddr:192.168.100.3 trsvcid:4420 hostaddr:192.168.100.4 subnqn:nqn.2016-06.io.spdk:cnode2" Nvme5
+
+  # The number of attempts per I/O when an I/O fails. Do not include
+  # this key to get the default behavior.
+  RetryCount 4
+  # Timeout for each command, in microseconds. If 0, don't track timeouts.
+  TimeoutUsec 0
+  # Action to take on command time out. Only valid when Timeout is greater
+  # than 0. This may be 'Reset' to reset the controller, 'Abort' to abort
+  # the command, or 'None' to just print a message but do nothing.
+  # Admin command timeouts will always result in a reset.
+  ActionOnTimeout None
+  # Set how often the admin queue is polled for asynchronous events.
+  # Units in microseconds.
+  AdminPollRate 100000
+  # Set how often I/O queues are polled from completions.
+  # Units in microseconds.
+  IOPollRate 0
+
+  # Disable handling of hotplug (runtime insert and remove) events,
+  # users can set to Yes if want to enable it.
+  # Default: No
+  HotplugEnable No
+
+  # Enable/disable delayed NVMe command submission.
+  # Default: True.
+  DelayCmdSubmit True
+
+# The Split virtual block device slices block devices into multiple smaller bdevs.
+[Split]
+  # Syntax:
+  #   Split <bdev> <count> [<size_in_megabytes>]
+
+  # Split Malloc2 into two equally-sized portions, Malloc2p0 and Malloc2p1
+  Split Malloc2 2
+
+  # Split Malloc3 into eight 1-megabyte portions, Malloc3p0 ... Malloc3p7,
+  # leaving the rest of the device inaccessible
+  Split Malloc3 8 1
+
+# The RAID virtual block device based on pre-configured block device.
+[RAID1]
+  # Unique name of this RAID device.
+  Name Raid0
+  # RAID level, only raid level 0 is supported.
+  RaidLevel 0
+  # Strip size in KB.
+  StripSize 64
+  # Number of pre-configured bdevs.
+  NumDevices 2
+  # Pre-configured bdevs name with Nvme.
+  Devices Nvme2n1 Nvme3n1
+  # Pre-configured bdevs name with Malloc.
+  #Devices Malloc0 Malloc1
+  # Pre-configured bdevs name with AIO.
+  #Devices AIO0 AIO1
+
+# Define an NVMf Subsystem.
+# - NQN is required and must be unique.
+# - Between 1 and 255 Listen directives are allowed. This defines
+#   the addresses on which new connections may be accepted. The format
+#   is Listen <type> <address> where type can be RDMA, TCP or FC.
+# - Between 0 and 255 Host directives are allowed. This defines the
+#   NQNs of allowed hosts. If no Host directive is specified, all hosts
+#   are allowed to connect.
+# - Between 0 and 255 Namespace directives are allowed. These define the
+#   namespaces accessible from this subsystem.
+#   The user must specify MaxNamespaces to allow for adding namespaces
+#   during active connection. By default it is 0
+#   The user must specify a bdev name for each namespace, and may optionally
+#   specify a namespace ID. If nsid is omitted, the namespace will be
+#   assigned the next available NSID. The NSID must be unique within the
+#   subsystem. An optional namespace UUID may also be specified.
+#   Syntax:
+#     Namespace <bdev_name> [<nsid> [<uuid>]]
+
+# Namespaces backed by physical NVMe devices
+[Subsystem1]
+  NQN nqn.2016-06.io.spdk:cnode1
+  Listen TCP 15.15.15.2:4420
+  AllowAnyHost No
+  Host nqn.2016-06.io.spdk:init
+  SN SPDK00000000000001
+  MN SPDK_Controller1
+  MaxNamespaces 20
+  Namespace Nvme0n1 1
+  Namespace Nvme1n1 2
+  Namespace Raid0
+
+# Multiple subsystems are allowed.
+# Namespaces backed by non-NVMe devices
+[Subsystem2]
+  NQN nqn.2016-06.io.spdk:cnode2
+  Listen RDMA 192.168.2.21:4420
+  AllowAnyHost No
+  Host nqn.2016-06.io.spdk:init
+  SN SPDK00000000000002
+  MN SPDK_Controller2
+  Namespace Malloc0
+  Namespace Malloc1
+  Namespace AIO0
+  Namespace AIO1
+
+# Subsystem with FC listen address directive
+# - Listen option allows subsystem access on specific FC ports identified
+#   by WWNN-WWPN. Each subsystem allows 0 - 255 listen directives.
+#   If no listen directive is provided, subsystem can be accessed on all
+#   avialable FC links permitted by FC zoning rules.
+#
+# [Subsystem3]
+  #NQN nqn.2016-06.io.spdk:cnode3
+  #Listen FC "nn-0x20000090fac7ca5c:pn-0x10000090fac7ca5c"
+  #AllowAnyHost Yes
+  #SN SPDK00000000000003
+  #Namespace Malloc4

etc/spdk/vhost.conf.in

@@ -0,0 +1,187 @@
+# SPDK vhost configuration file
+#
+# Please write all parameters using ASCII.
+# The parameter must be quoted if it includes whitespace.
+
+# Configuration syntax:
+# Leading whitespace is ignored.
+# Lines starting with '#' are comments.
+# Lines ending with '\' are concatenated with the next line.
+# Bracketed ([]) names define sections
+
+[Global]
+  # Instance ID for multi-process support
+  # Default: 0
+  #InstanceID 0
+
+  # Disable PCI access. PCI is enabled by default. Setting this
+  # option will hide any PCI device from all SPDK modules, making
+  # SPDK act as if they don't exist.
+  #NoPci Yes
+
+  # Tracepoint group mask for spdk trace buffers
+  # Default: 0x0 (all tracepoint groups disabled)
+  # Set to 0xFFFF to enable all tracepoint groups.
+  #TpointGroupMask 0x0
+
+# Users may activate entries in this section to override default values for
+# global parameters in the block device (bdev) subsystem.
+[Bdev]
+  # Number of spdk_bdev_io structures allocated in the global bdev subsystem pool.
+  #BdevIoPoolSize 65536
+
+  # Maximum number of spdk_bdev_io structures to cache per thread.
+  #BdevIoCacheSize 256
+
+# Users may not want to use offload even it is available.
+# Users can use offload by specifying "Enable Yes" in this section
+# if it is available.
+# Users may use the whitelist to initialize specified devices, IDS
+#  uses BUS:DEVICE.FUNCTION to identify each Ioat channel.
+[Ioat]
+  Enable No
+  #Whitelist 00:04.0
+  #Whitelist 00:04.1
+
+# Users must change this section to match the /dev/sdX devices to be
+#  exported as vhost scsi drives. The devices are accessed using Linux AIO.
+[AIO]
+  #AIO /dev/sdb AIO0
+  #AIO /dev/sdc AIO1
+
+# PMDK libpmemblk-based block device
+[Pmem]
+  # Syntax:
+  #   Blk <pmemblk pool file name> <bdev name>
+  Blk /path/to/pmem-pool Pmem0
+
+# Users may change this section to create a different number or size of
+#  malloc LUNs.
+# If the system has hardware DMA engine, it can use an IOAT
+# (i.e. Crystal Beach DMA) channel to do the copy instead of memcpy
+# by specifying "Enable Yes" in [Ioat] section.
+# Offload is disabled by default even it is available.
+[Malloc]
+  # Number of Malloc targets
+  NumberOfLuns 3
+  # Malloc targets are 128M
+  LunSizeInMB 128
+  # Block size. Default is 512 bytes.
+  BlockSize 4096
+
+# NVMe configuration options
+[Nvme]
+  # NVMe Device Whitelist
+  # Users may specify which NVMe devices to claim by their transport id.
+  # See spdk_nvme_transport_id_parse() in spdk/nvme.h for the correct format.
+  # The second argument is the assigned name, which can be referenced from
+  # other sections in the configuration file. For NVMe devices, a namespace
+  # is automatically appended to each name in the format <YourName>nY, where
+  # Y is the NSID (starts at 1).
+  TransportID "trtype:PCIe traddr:0000:00:00.0" Nvme0
+  TransportID "trtype:PCIe traddr:0000:01:00.0" Nvme1
+
+  # The number of attempts per I/O when an I/O fails. Do not include
+  # this key to get the default behavior.
+  RetryCount 4
+  # Timeout for each command, in microseconds. If 0, don't track timeouts.
+  TimeoutUsec 0
+  # Action to take on command time out. Only valid when Timeout is greater
+  # than 0. This may be 'Reset' to reset the controller, 'Abort' to abort
+  # the command, or 'None' to just print a message but do nothing.
+  # Admin command timeouts will always result in a reset.
+  ActionOnTimeout None
+  # Set how often the admin queue is polled for asynchronous events.
+  # Units in microseconds.
+  AdminPollRate 100000
+  # Set how often I/O queues are polled from completions.
+  # Units in microseconds.
+  IOPollRate 0
+
+# The Split virtual block device slices block devices into multiple smaller bdevs.
+[Split]
+  # Syntax:
+  #   Split <bdev> <count> [<size_in_megabytes>]
+  #
+  # Split Nvme1n1 into two equally-sized portions, Nvme1n1p0 and Nvme1n1p1
+  #Split Nvme1n1 2
+
+  # Split Malloc2 into eight 1-megabyte portions, Malloc2p0 ... Malloc2p7,
+  # leaving the rest of the device inaccessible
+  #Split Malloc2 8 1
+
+# The RAID virtual block device based on pre-configured block device.
+[RAID1]
+  # Unique name of this RAID device.
+  Name Raid0
+  # RAID level, only raid level 0 is supported.
+  RaidLevel 0
+  # Strip size in KB.
+  StripSize 64
+  # Number of pre-configured bdevs.
+  NumDevices 2
+  # Pre-configured bdevs name with Nvme.
+  #Devices Nvme0n1 Nvme1n1
+  # Pre-configured bdevs name with Malloc.
+  Devices Malloc1 Malloc2
+  # Pre-configured bdevs name with AIO.
+  #Devices AIO0 AIO1
+
+# Vhost scsi controller configuration
+# Users should change the VhostScsi section(s) below to match the desired
+# vhost configuration.
+# Name is minimum required
+[VhostScsi0]
+  # Define name for controller
+  Name vhost.0
+  # Assign devices from backend
+  # Use the first malloc device
+  Target 0 Malloc0
+  # Use the first AIO device
+  #Target 1 AIO0
+  # Use the frist Nvme device
+  #Target 2 Nvme0n1
+  # Use the third partition from second Nvme device
+  #Target 3 Nvme1n1p2
+
+  # Start the poller for this vhost controller on one of the cores in
+  #  this cpumask.  By default, it not specified, will use any core in the
+  #  SPDK process.
+  #Cpumask 0x1
+
+#[VhostScsi1]
+#  Name vhost.1
+#  Target 0 AIO1
+#  Cpumask 0x1
+
+#[VhostBlk0]
+  # Define name for controller
+  #Name vhost.2
+  # Use first partition from the second Malloc device
+  #Dev Malloc2p0
+  # Put controller in read-only mode
+  #ReadOnly no
+  # Start the poller for this vhost controller on one of the cores in
+  #  this cpumask.  By default, it not specified, will use any core in the
+  #  SPDK process.
+  #Cpumask 0x1
+
+#[VhostBlk1]
+  # Define name for controller
+  #Name vhost.2
+  # Use device which named Raid0
+  #Dev Raid0
+
+#[VhostNvme0]
+  # Define name for controller
+  #Name vhost.0
+  #NumberOfQueues 2
+  # Use first partition from the first NVMe device
+  #Namespace Nvme0n1p0
+  # Use first partition from the first NVMe device
+  #Namespace Nvme0n1p1
+
+  # Start the poller for this vhost controller on one of the cores in
+  #  this cpumask.  By default, it not specified, will use any core in the
+  #  SPDK process.
+  #Cpumask 0x1

examples/Makefile

@@ -36,10 +36,6 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
 DIRS-y += accel bdev blob ioat nvme sock vmd nvmf
 
-ifeq ($(OS),Linux)
-DIRS-$(CONFIG_VHOST) += interrupt_tgt
-endif
-
 .PHONY: all clean $(DIRS-y)
 
 all: $(DIRS-y)

examples/accel/perf/Makefile

@@ -39,6 +39,9 @@ APP = accel_perf
 
 C_SRCS := accel_perf.c
 
-SPDK_LIB_LIST = $(ACCEL_MODULES_LIST) event_accel
+SPDK_LIB_LIST = $(ALL_MODULES_LIST)
+SPDK_LIB_LIST += event_bdev event_accel event_vmd
+SPDK_LIB_LIST += bdev accel event thread util conf trace \
+		log jsonrpc json rpc sock notify
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

examples/bdev/fio_plugin/Makefile

@@ -36,10 +36,15 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
 
-FIO_PLUGIN := spdk_bdev
+APP := fio_plugin
 
 C_SRCS = fio_plugin.c
+CFLAGS += -I$(CONFIG_FIO_SOURCE_DIR)
+LDFLAGS += -shared -rdynamic -Wl,-z,nodelete
 
-SPDK_LIB_LIST = $(ALL_MODULES_LIST) event_bdev
+# Unable to combine the FIO plugin and the VPP socket abstraction (license incompatibility)
+SPDK_LIB_LIST = $(filter-out sock_vpp,$(ALL_MODULES_LIST))
+SPDK_LIB_LIST += thread util bdev bdev_rpc conf accel rpc jsonrpc json log sock trace notify
+SPDK_LIB_LIST += event event_bdev event_accel event_vmd
 
-include $(SPDK_ROOT_DIR)/mk/spdk.fio.mk
+include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

examples/bdev/fio_plugin/README.md

@@ -45,11 +45,14 @@ To use the SPDK fio plugin with fio, specify the plugin binary using LD_PRELOAD
 fio and set ioengine=spdk_bdev in the fio configuration file (see example_config.fio in the same
 directory as this README).
 
-    LD_PRELOAD=<path to spdk repo>/build/fio/spdk_bdev fio
+    LD_PRELOAD=<path to spdk repo>/examples/bdev/fio_plugin/fio_plugin fio
 
 The fio configuration file must contain one new parameter:
 
-    spdk_json_conf=./examples/bdev/fio_plugin/bdev.json
+    spdk_conf=./examples/bdev/fio_plugin/bdev.conf
+
+This must point at an SPDK configuration file. There are a number of example configuration
+files in the SPDK repository under etc/spdk.
 
 You can specify which block device to run against by setting the filename parameter
 to the block device name:
@@ -60,10 +63,8 @@ Or for NVMe devices:
 
     filename=Nvme0n1
 
-fio by default forks a separate process for every job. It also supports just spawning a separate
-thread in the same process for every job. The SPDK fio plugin is limited to this latter thread
-usage model, so fio jobs must also specify thread=1 when using the SPDK fio plugin. The SPDK fio
-plugin supports multiple threads - in this case, the "1" just means "use thread mode".
+Currently the SPDK fio plugin is limited to the thread usage model, so fio jobs must also specify thread=1
+when using the SPDK fio plugin.
 
 fio also currently has a race condition on shutdown if dynamically loading the ioengine by specifying the
 engine's full path via the ioengine parameter - LD_PRELOAD is recommended to avoid this race condition.

examples/bdev/fio_plugin/bdev.conf.in

@@ -0,0 +1,3 @@
+[Malloc]
+  NumberOfLuns 1
+  LunSizeInMB 128

examples/bdev/fio_plugin/bdev.json

@@ -1,17 +0,0 @@
-{
-  "subsystems": [
-    {
-      "subsystem": "bdev",
-      "config": [
-        {
-          "params": {
-            "block_size": 512,
-            "num_blocks": 262144,
-            "name": "Malloc0"
-          },
-          "method": "bdev_malloc_create"
-        }
-      ]
-    }
-  ]
-}

examples/bdev/fio_plugin/example_config.fio

@@ -1,6 +1,6 @@
 [global]
 ioengine=spdk_bdev
-spdk_json_conf=./examples/bdev/fio_plugin/bdev.json
+spdk_conf=./examples/bdev/fio_plugin/bdev.conf.in
 thread=1
 group_reporting=1
 direct=1
@@ -13,5 +13,4 @@ rw=randrw
 bs=4k
 
 [test]
-filename=Malloc0
 numjobs=1

examples/bdev/fio_plugin/fio_plugin.c

@@ -35,6 +35,7 @@
 
 #include "spdk/bdev.h"
 #include "spdk/accel_engine.h"
+#include "spdk/conf.h"
 #include "spdk/env.h"
 #include "spdk/thread.h"
 #include "spdk/log.h"
@@ -42,6 +43,7 @@
 #include "spdk/queue.h"
 #include "spdk/util.h"
 
+#include "spdk_internal/thread.h"
 #include "spdk_internal/event.h"
 
 #include "config-host.h"
@@ -59,7 +61,7 @@ struct spdk_fio_options {
 	char *conf;
 	char *json_conf;
 	unsigned mem_mb;
-	int mem_single_seg;
+	bool mem_single_seg;
 };
 
 struct spdk_fio_request {
@@ -85,8 +87,6 @@ struct spdk_fio_thread {
 	struct io_u		**iocq;		/* io completion queue */
 	unsigned int		iocq_count;	/* number of iocq entries filled by last getevents */
 	unsigned int		iocq_size;	/* number of iocq entries allocated */
-
-	TAILQ_ENTRY(spdk_fio_thread)	link;
 };
 
 static bool g_spdk_env_initialized = false;
@@ -96,12 +96,6 @@ static int spdk_fio_init(struct thread_data *td);
 static void spdk_fio_cleanup(struct thread_data *td);
 static size_t spdk_fio_poll_thread(struct spdk_fio_thread *fio_thread);
 
-static pthread_t g_init_thread_id = 0;
-static pthread_mutex_t g_init_mtx = PTHREAD_MUTEX_INITIALIZER;
-static pthread_cond_t g_init_cond;
-static bool g_poll_loop = true;
-static TAILQ_HEAD(, spdk_fio_thread) g_threads = TAILQ_HEAD_INITIALIZER(g_threads);
-
 /* Default polling timeout (ns) */
 #define SPDK_FIO_POLLING_TIMEOUT 1000000000ULL
 
@@ -155,9 +149,19 @@ spdk_fio_cleanup_thread(struct spdk_fio_thread *fio_thread)
 {
 	spdk_thread_send_msg(fio_thread->thread, spdk_fio_bdev_close_targets, fio_thread);
 
-	pthread_mutex_lock(&g_init_mtx);
-	TAILQ_INSERT_TAIL(&g_threads, fio_thread, link);
-	pthread_mutex_unlock(&g_init_mtx);
+	while (!spdk_thread_is_idle(fio_thread->thread)) {
+		spdk_fio_poll_thread(fio_thread);
+	}
+
+	spdk_set_thread(fio_thread->thread);
+
+	spdk_thread_exit(fio_thread->thread);
+	while (!spdk_thread_is_exited(fio_thread->thread)) {
+		spdk_thread_poll(fio_thread->thread, 0, 0);
+	}
+	spdk_thread_destroy(fio_thread->thread);
+	free(fio_thread->iocq);
+	free(fio_thread);
 }
 
 static void
@@ -185,6 +189,11 @@ spdk_fio_calc_timeout(struct spdk_fio_thread *fio_thread, struct timespec *ts)
 	}
 }
 
+static pthread_t g_init_thread_id = 0;
+static pthread_mutex_t g_init_mtx = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t g_init_cond;
+static bool g_poll_loop = true;
+
 static void
 spdk_fio_bdev_init_done(int rc, void *cb_arg)
 {
@@ -196,8 +205,12 @@ spdk_fio_bdev_init_start(void *arg)
 {
 	bool *done = arg;
 
-	spdk_app_json_config_load(g_json_config_file, SPDK_DEFAULT_RPC_ADDR,
-				  spdk_fio_bdev_init_done, done, true);
+	if (g_json_config_file != NULL) {
+		spdk_app_json_config_load(g_json_config_file, SPDK_DEFAULT_RPC_ADDR,
+					  spdk_fio_bdev_init_done, done, true);
+	} else {
+		spdk_subsystem_init(spdk_fio_bdev_init_done, done);
+	}
 }
 
 static void
@@ -219,7 +232,7 @@ spdk_init_thread_poll(void *arg)
 {
 	struct spdk_fio_options		*eo = arg;
 	struct spdk_fio_thread		*fio_thread;
-	struct spdk_fio_thread		*thread, *tmp;
+	struct spdk_conf		*config = NULL;
 	struct spdk_env_opts		opts;
 	bool				done;
 	int				rc;
@@ -238,7 +251,26 @@ spdk_init_thread_poll(void *arg)
 		rc = EINVAL;
 		goto err_exit;
 	} else if (eo->conf && strlen(eo->conf)) {
-		g_json_config_file = eo->conf;
+		config = spdk_conf_allocate();
+		if (!config) {
+			SPDK_ERRLOG("Unable to allocate configuration file\n");
+			rc = ENOMEM;
+			goto err_exit;
+		}
+
+		rc = spdk_conf_read(config, eo->conf);
+		if (rc != 0) {
+			SPDK_ERRLOG("Invalid configuration file format\n");
+			spdk_conf_free(config);
+			goto err_exit;
+		}
+		if (spdk_conf_first_section(config) == NULL) {
+			SPDK_ERRLOG("Invalid configuration file format\n");
+			spdk_conf_free(config);
+			rc = EINVAL;
+			goto err_exit;
+		}
+		spdk_conf_set_as_default(config);
 	} else if (eo->json_conf && strlen(eo->json_conf)) {
 		g_json_config_file = eo->json_conf;
 	} else {
@@ -258,6 +290,7 @@ spdk_init_thread_poll(void *arg)
 
 	if (spdk_env_init(&opts) < 0) {
 		SPDK_ERRLOG("Unable to initialize SPDK env\n");
+		spdk_conf_free(config);
 		rc = EINVAL;
 		goto err_exit;
 	}
@@ -292,37 +325,19 @@ spdk_init_thread_poll(void *arg)
 	pthread_mutex_lock(&g_init_mtx);
 	pthread_cond_signal(&g_init_cond);
 
-	pthread_mutex_unlock(&g_init_mtx);
-
 	while (g_poll_loop) {
 		spdk_fio_poll_thread(fio_thread);
 
-		pthread_mutex_lock(&g_init_mtx);
-		if (!TAILQ_EMPTY(&g_threads)) {
-			TAILQ_FOREACH_SAFE(thread, &g_threads, link, tmp) {
-				spdk_fio_poll_thread(thread);
-			}
-
-			/* If there are exiting threads to poll, don't sleep. */
-			pthread_mutex_unlock(&g_init_mtx);
-			continue;
-		}
-
-		/* Figure out how long to sleep. */
 		clock_gettime(CLOCK_MONOTONIC, &ts);
 		spdk_fio_calc_timeout(fio_thread, &ts);
 
 		rc = pthread_cond_timedwait(&g_init_cond, &g_init_mtx, &ts);
-		pthread_mutex_unlock(&g_init_mtx);
-
 		if (rc != ETIMEDOUT) {
 			break;
 		}
-
-
 	}
 
-	spdk_fio_cleanup_thread(fio_thread);
+	pthread_mutex_unlock(&g_init_mtx);
 
 	/* Finalize the bdev layer */
 	done = false;
@@ -330,32 +345,9 @@ spdk_init_thread_poll(void *arg)
 
 	do {
 		spdk_fio_poll_thread(fio_thread);
+	} while (!done && !spdk_thread_is_idle(fio_thread->thread));
 
-		TAILQ_FOREACH_SAFE(thread, &g_threads, link, tmp) {
-			spdk_fio_poll_thread(thread);
-		}
-	} while (!done);
-
-	/* Now exit all the threads */
-	TAILQ_FOREACH(thread, &g_threads, link) {
-		spdk_set_thread(thread->thread);
-		spdk_thread_exit(thread->thread);
-		spdk_set_thread(NULL);
-	}
-
-	/* And wait for them to gracefully exit */
-	while (!TAILQ_EMPTY(&g_threads)) {
-		TAILQ_FOREACH_SAFE(thread, &g_threads, link, tmp) {
-			if (spdk_thread_is_exited(thread->thread)) {
-				TAILQ_REMOVE(&g_threads, thread, link);
-				spdk_thread_destroy(thread->thread);
-				free(thread->iocq);
-				free(thread);
-			} else {
-				spdk_thread_poll(thread->thread, 0, 0);
-			}
-		}
-	}
+	spdk_fio_cleanup_thread(fio_thread);
 
 	pthread_exit(NULL);
 
@@ -474,13 +466,6 @@ spdk_fio_setup(struct thread_data *td)
 	return 0;
 }
 
-static void
-fio_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev,
-		  void *event_ctx)
-{
-	SPDK_WARNLOG("Unsupported bdev event: type %d\n", type);
-}
-
 static void
 spdk_fio_bdev_open(void *arg)
 {
@@ -506,8 +491,15 @@ spdk_fio_bdev_open(void *arg)
 			return;
 		}
 
-		rc = spdk_bdev_open_ext(f->file_name, true, fio_bdev_event_cb, NULL,
-					&target->desc);
+		target->bdev = spdk_bdev_get_by_name(f->file_name);
+		if (!target->bdev) {
+			SPDK_ERRLOG("Unable to find bdev with name %s\n", f->file_name);
+			free(target);
+			fio_thread->failed = true;
+			return;
+		}
+
+		rc = spdk_bdev_open(target->bdev, true, NULL, NULL, &target->desc);
 		if (rc) {
 			SPDK_ERRLOG("Unable to open bdev %s\n", f->file_name);
 			free(target);
@@ -515,8 +507,6 @@ spdk_fio_bdev_open(void *arg)
 			return;
 		}
 
-		target->bdev = spdk_bdev_desc_get_bdev(target->desc);
-
 		target->ch = spdk_bdev_get_io_channel(target->desc);
 		if (!target->ch) {
 			SPDK_ERRLOG("Unable to get I/O channel for bdev.\n");
@@ -675,11 +665,6 @@ spdk_fio_queue(struct thread_data *td, struct io_u *io_u)
 				     io_u->offset, io_u->xfer_buflen,
 				     spdk_fio_completion_cb, fio_req);
 		break;
-	case DDIR_SYNC:
-		rc = spdk_bdev_flush(target->desc, target->ch,
-				     io_u->offset, io_u->xfer_buflen,
-				     spdk_fio_completion_cb, fio_req);
-		break;
 	default:
 		assert(false);
 		break;
@@ -761,7 +746,7 @@ static struct fio_option options[] = {
 		.lname		= "SPDK configuration file",
 		.type		= FIO_OPT_STR_STORE,
 		.off1		= offsetof(struct spdk_fio_options, conf),
-		.help		= "A SPDK JSON configuration file",
+		.help		= "A SPDK configuration file",
 		.category	= FIO_OPT_C_ENGINE,
 		.group		= FIO_OPT_G_INVALID,
 	},
@@ -789,7 +774,6 @@ static struct fio_option options[] = {
 		.type		= FIO_OPT_BOOL,
 		.off1		= offsetof(struct spdk_fio_options, mem_single_seg),
 		.help		= "If set to 1, SPDK will use just a single hugetlbfs file",
-		.def            = "0",
 		.category	= FIO_OPT_C_ENGINE,
 		.group		= FIO_OPT_G_INVALID,
 	},

examples/bdev/fio_plugin/full_bench.fio

@@ -1,6 +1,6 @@
 [global]
 ioengine=spdk_bdev
-spdk_conf=./examples/bdev/fio_plugin/bdev.json
+spdk_conf=./examples/bdev/fio_plugin/bdev.conf.in
 thread=1
 group_reporting=1
 direct=1

examples/bdev/hello_world/Makefile

@@ -37,6 +37,8 @@ APP = hello_bdev
 
 C_SRCS := hello_bdev.c
 
-SPDK_LIB_LIST = $(ALL_MODULES_LIST) event_bdev
+SPDK_LIB_LIST = $(ALL_MODULES_LIST)
+SPDK_LIB_LIST += event_bdev event_accel event_vmd
+SPDK_LIB_LIST += bdev_rpc bdev accel event thread util conf trace log jsonrpc json rpc sock notify
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

examples/bdev/hello_world/bdev.conf

@@ -0,0 +1,3 @@
+[Malloc]
+  NumberOfLuns 1
+  LunSizeInMB  32

examples/bdev/hello_world/bdev.json

@@ -1,17 +0,0 @@
-{
-  "subsystems": [
-    {
-      "subsystem": "bdev",
-      "config": [
-        {
-          "method": "bdev_malloc_create",
-          "params": {
-            "name": "Malloc0",
-            "num_blocks": 32768,
-            "block_size": 512
-          }
-        }
-      ]
-    }
-  ]
-}

examples/bdev/hello_world/hello_bdev.c

@@ -38,7 +38,7 @@
 #include "spdk/event.h"
 #include "spdk/log.h"
 #include "spdk/string.h"
-#include "spdk/bdev_zone.h"
+#include "spdk/bdev_module.h"
 
 static char *g_bdev_name = "Malloc0";
 
@@ -184,57 +184,6 @@ hello_write(void *arg)
 	}
 }
 
-static void
-hello_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev,
-		    void *event_ctx)
-{
-	SPDK_NOTICELOG("Unsupported bdev event: type %d\n", type);
-}
-
-static void
-reset_zone_complete(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
-{
-	struct hello_context_t *hello_context = cb_arg;
-
-	/* Complete the I/O */
-	spdk_bdev_free_io(bdev_io);
-
-	if (!success) {
-		SPDK_ERRLOG("bdev io reset zone error: %d\n", EIO);
-		spdk_put_io_channel(hello_context->bdev_io_channel);
-		spdk_bdev_close(hello_context->bdev_desc);
-		spdk_app_stop(-1);
-		return;
-	}
-
-	hello_write(hello_context);
-}
-
-static void
-hello_reset_zone(void *arg)
-{
-	struct hello_context_t *hello_context = arg;
-	int rc = 0;
-
-	rc = spdk_bdev_zone_management(hello_context->bdev_desc, hello_context->bdev_io_channel,
-				       0, SPDK_BDEV_ZONE_RESET, reset_zone_complete, hello_context);
-
-	if (rc == -ENOMEM) {
-		SPDK_NOTICELOG("Queueing io\n");
-		/* In case we cannot perform I/O now, queue I/O */
-		hello_context->bdev_io_wait.bdev = hello_context->bdev;
-		hello_context->bdev_io_wait.cb_fn = hello_reset_zone;
-		hello_context->bdev_io_wait.cb_arg = hello_context;
-		spdk_bdev_queue_io_wait(hello_context->bdev, hello_context->bdev_io_channel,
-					&hello_context->bdev_io_wait);
-	} else if (rc) {
-		SPDK_ERRLOG("%s error while resetting zone: %d\n", spdk_strerror(-rc), rc);
-		spdk_put_io_channel(hello_context->bdev_io_channel);
-		spdk_bdev_close(hello_context->bdev_desc);
-		spdk_app_stop(-1);
-	}
-}
-
 /*
  * Our initial event that kicks off everything from main().
  */
@@ -250,25 +199,29 @@ hello_start(void *arg1)
 	SPDK_NOTICELOG("Successfully started the application\n");
 
 	/*
-	 * There can be many bdevs configured, but this application will only use
-	 * the one input by the user at runtime.
-	 *
-	 * Open the bdev by calling spdk_bdev_open_ext() with its name.
+	 * Get the bdev. There can be many bdevs configured, but this
+	 * application will only use the one input by the user at runtime so
+	 * we get it via its name.
+	 */
+	hello_context->bdev = spdk_bdev_get_by_name(hello_context->bdev_name);
+	if (hello_context->bdev == NULL) {
+		SPDK_ERRLOG("Could not find the bdev: %s\n", hello_context->bdev_name);
+		spdk_app_stop(-1);
+		return;
+	}
+
+	/*
+	 * Open the bdev by calling spdk_bdev_open()
 	 * The function will return a descriptor
 	 */
 	SPDK_NOTICELOG("Opening the bdev %s\n", hello_context->bdev_name);
-	rc = spdk_bdev_open_ext(hello_context->bdev_name, true, hello_bdev_event_cb, NULL,
-				&hello_context->bdev_desc);
+	rc = spdk_bdev_open(hello_context->bdev, true, NULL, NULL, &hello_context->bdev_desc);
 	if (rc) {
 		SPDK_ERRLOG("Could not open bdev: %s\n", hello_context->bdev_name);
 		spdk_app_stop(-1);
 		return;
 	}
 
-	/* A bdev pointer is valid while the bdev is opened. */
-	hello_context->bdev = spdk_bdev_desc_get_bdev(hello_context->bdev_desc);
-
-
 	SPDK_NOTICELOG("Opening io channel\n");
 	/* Open I/O channel */
 	hello_context->bdev_io_channel = spdk_bdev_get_io_channel(hello_context->bdev_desc);
@@ -294,12 +247,6 @@ hello_start(void *arg1)
 	}
 	snprintf(hello_context->buff, blk_size, "%s", "Hello World!\n");
 
-	if (spdk_bdev_is_zoned(hello_context->bdev)) {
-		hello_reset_zone(hello_context);
-		/* If bdev is zoned, the callback, reset_zone_complete, will call hello_write() */
-		return;
-	}
-
 	hello_write(hello_context);
 }
 
@@ -311,7 +258,7 @@ main(int argc, char **argv)
 	struct hello_context_t hello_context = {};
 
 	/* Set default values in opts structure. */
-	spdk_app_opts_init(&opts, sizeof(opts));
+	spdk_app_opts_init(&opts);
 	opts.name = "hello_bdev";
 
 	/*

examples/blob/cli/Makefile

@@ -38,6 +38,9 @@ APP = blobcli
 C_SRCS := blobcli.c
 
 # Don't link bdev_lvol in blobcli - otherwise this utility cannot operate on an lvolstore
-SPDK_LIB_LIST = $(filter-out bdev_lvol,$(ALL_MODULES_LIST)) event_bdev
+SPDK_LIB_LIST = $(filter-out bdev_lvol,$(ALL_MODULES_LIST))
+SPDK_LIB_LIST += event_bdev event_accel event_vmd
+SPDK_LIB_LIST += bdev accel event thread util conf trace \
+		log jsonrpc json rpc sock notify
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

examples/blob/cli/blobcli.c

@@ -52,8 +52,8 @@
 static void cli_start(void *arg1);
 
 static const char *program_name = "blobcli";
-/* default name for .json file, any name can be used however with -j switch */
-static const char *program_conf = "blobcli.json";
+/* default name for .conf file, any name can be used however with -c switch */
+static const char *program_conf = "blobcli.conf";
 
 /*
  * CMD mode runs one command at a time which can be annoying as the init takes
@@ -179,7 +179,7 @@ usage(struct cli_context_t *cli_context, char *msg)
 
 	if (!cli_context || cli_context->cli_mode == CLI_MODE_CMD) {
 		printf("Version %s\n", SPDK_VERSION_STRING);
-		printf("Usage: %s [-j SPDK josn_config_file] Command\n", program_name);
+		printf("Usage: %s [-c SPDK config_file] Command\n", program_name);
 		printf("\n%s is a command line tool for interacting with blobstore\n",
 		       program_name);
 		printf("on the underlying device specified in the conf file passed\n");
@@ -857,26 +857,25 @@ load_bs_cb(void *arg1, struct spdk_blob_store *bs, int bserrno)
 	}
 }
 
-static void
-base_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev,
-		   void *event_ctx)
-{
-	printf("Unsupported bdev event: type %d\n", type);
-}
-
 /*
  * Load the blobstore.
  */
 static void
 load_bs(struct cli_context_t *cli_context)
 {
+	struct spdk_bdev *bdev = NULL;
 	struct spdk_bs_dev *bs_dev = NULL;
-	int rc;
 
-	rc = spdk_bdev_create_bs_dev_ext(cli_context->bdev_name, base_bdev_event_cb,
-					 NULL, &bs_dev);
-	if (rc != 0) {
-		printf("Could not create blob bdev, %s!!\n", spdk_strerror(-rc));
+	bdev = spdk_bdev_get_by_name(cli_context->bdev_name);
+	if (bdev == NULL) {
+		printf("Could not find a bdev\n");
+		spdk_app_stop(-1);
+		return;
+	}
+
+	bs_dev = spdk_bdev_create_bs_dev(bdev, NULL, NULL);
+	if (bs_dev == NULL) {
+		printf("Could not create blob bdev!!\n");
 		spdk_app_stop(-1);
 		return;
 	}
@@ -940,14 +939,20 @@ bs_init_cb(void *cb_arg, struct spdk_blob_store *bs,
 static void
 init_bs(struct cli_context_t *cli_context)
 {
-	int rc;
+	struct spdk_bdev *bdev = NULL;
 
-	printf("Init blobstore using bdev Name: %s\n", cli_context->bdev_name);
+	bdev = spdk_bdev_get_by_name(cli_context->bdev_name);
+	if (bdev == NULL) {
+		printf("Could not find a bdev\n");
+		spdk_app_stop(-1);
+		return;
+	}
+	printf("Init blobstore using bdev Product Name: %s\n",
+	       spdk_bdev_get_product_name(bdev));
 
-	rc = spdk_bdev_create_bs_dev_ext(cli_context->bdev_name, base_bdev_event_cb, NULL,
-					 &cli_context->bs_dev);
-	if (rc != 0) {
-		printf("Could not create blob bdev, %s!!\n", spdk_strerror(-rc));
+	cli_context->bs_dev = spdk_bdev_create_bs_dev(bdev, NULL, NULL);
+	if (cli_context->bs_dev == NULL) {
+		printf("Could not create blob bdev!!\n");
 		spdk_app_stop(-1);
 		return;
 	}
@@ -1006,14 +1011,20 @@ bsdump_print_xattr(FILE *fp, const char *bstype, const char *name, const void *v
 static void
 dump_bs(struct cli_context_t *cli_context)
 {
-	int rc;
+	struct spdk_bdev *bdev = NULL;
 
-	printf("Init blobstore using bdev Name: %s\n", cli_context->bdev_name);
+	bdev = spdk_bdev_get_by_name(cli_context->bdev_name);
+	if (bdev == NULL) {
+		printf("Could not find a bdev\n");
+		spdk_app_stop(-1);
+		return;
+	}
+	printf("Init blobstore using bdev Product Name: %s\n",
+	       spdk_bdev_get_product_name(bdev));
 
-	rc = spdk_bdev_create_bs_dev_ext(cli_context->bdev_name, base_bdev_event_cb, NULL,
-					 &cli_context->bs_dev);
-	if (rc != 0) {
-		printf("Could not create blob bdev, %s!!\n", spdk_strerror(-rc));
+	cli_context->bs_dev = spdk_bdev_create_bs_dev(bdev, NULL, NULL);
+	if (cli_context->bs_dev == NULL) {
+		printf("Could not create blob bdev!!\n");
 		spdk_app_stop(-1);
 		return;
 	}
@@ -1031,7 +1042,7 @@ cmd_parser(int argc, char **argv, struct cli_context_t *cli_context)
 	int cmd_chosen = 0;
 	char resp;
 
-	while ((op = getopt(argc, argv, "b:d:f:hij:l:m:n:p:r:s:DST:Xx:")) != -1) {
+	while ((op = getopt(argc, argv, "b:c:d:f:hil:m:n:p:r:s:DST:Xx:")) != -1) {
 		switch (op) {
 		case 'b':
 			if (strcmp(cli_context->bdev_name, "") == 0) {
@@ -1041,6 +1052,13 @@ cmd_parser(int argc, char **argv, struct cli_context_t *cli_context)
 				usage(cli_context, "ERROR: -b option can only be set once.\n");
 			}
 			break;
+		case 'c':
+			if (cli_context->app_started == false) {
+				cli_context->config_file = optarg;
+			} else {
+				usage(cli_context, "ERROR: -c option not valid during shell mode.\n");
+			}
+			break;
 		case 'D':
 			cmd_chosen++;
 			cli_context->action = CLI_DUMP_BS;
@@ -1088,13 +1106,6 @@ cmd_parser(int argc, char **argv, struct cli_context_t *cli_context)
 				cli_context->action = CLI_INIT_BS;
 			}
 			break;
-		case 'j':
-			if (cli_context->app_started == false) {
-				cli_context->config_file = optarg;
-			} else {
-				usage(cli_context, "ERROR: -j option not valid during shell mode.\n");
-			}
-			break;
 		case 'r':
 			if (argv[optind] != NULL) {
 				cmd_chosen++;
@@ -1522,8 +1533,8 @@ main(int argc, char **argv)
 	/* if the config file doesn't exist, tell them how to make one */
 	if (access(cli_context->config_file, F_OK) == -1) {
 		printf("Error: No config file found.\n");
-		printf("To create a config file named 'blobcli.json' for your NVMe device:\n");
-		printf("   <path to spdk>/scripts/gen_nvme.sh --json-with-subsystems > blobcli.json\n");
+		printf("To create a config file named 'blobcli.conf' for your NVMe device:\n");
+		printf("   <path to spdk>/scripts/gen_nvme.sh > blobcli.conf\n");
 		printf("and then re-run the cli tool.\n");
 		exit(-1);
 	}
@@ -1542,9 +1553,9 @@ main(int argc, char **argv)
 	}
 
 	/* Set default values in opts struct along with name and conf file. */
-	spdk_app_opts_init(&opts, sizeof(opts));
+	spdk_app_opts_init(&opts);
 	opts.name = "blobcli";
-	opts.json_config_file = cli_context->config_file;
+	opts.config_file = cli_context->config_file;
 
 	cli_context->app_started = true;
 	rc = spdk_app_start(&opts, cli_start, cli_context);

examples/blob/hello_world/Makefile

@@ -37,6 +37,9 @@ APP = hello_blob
 
 C_SRCS := hello_blob.c
 
-SPDK_LIB_LIST = $(ALL_MODULES_LIST) event_bdev
+SPDK_LIB_LIST = $(ALL_MODULES_LIST)
+SPDK_LIB_LIST += event_bdev event_accel event_vmd
+SPDK_LIB_LIST += bdev accel event thread util conf trace \
+		log jsonrpc json rpc sock notify
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

examples/blob/hello_world/hello_blob.c

@@ -39,7 +39,6 @@
 #include "spdk/blob_bdev.h"
 #include "spdk/blob.h"
 #include "spdk/log.h"
-#include "spdk/string.h"
 
 /*
  * We'll use this struct to gather housekeeping hello_context to pass between
@@ -394,13 +393,6 @@ bs_init_complete(void *cb_arg, struct spdk_blob_store *bs,
 	create_blob(hello_context);
 }
 
-static void
-base_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev,
-		   void *event_ctx)
-{
-	SPDK_WARNLOG("Unsupported bdev event: type %d\n", type);
-}
-
 /*
  * Our initial event that kicks off everything from main().
  */
@@ -408,16 +400,24 @@ static void
 hello_start(void *arg1)
 {
 	struct hello_context_t *hello_context = arg1;
+	struct spdk_bdev *bdev = NULL;
 	struct spdk_bs_dev *bs_dev = NULL;
-	int rc;
 
 	SPDK_NOTICELOG("entry\n");
+	/*
+	 * Get the bdev. For this example it is our malloc (RAM)
+	 * disk configured via hello_blob.conf that was passed
+	 * in when we started the SPDK app framework so we can
+	 * get it via its name.
+	 */
+	bdev = spdk_bdev_get_by_name("Malloc0");
+	if (bdev == NULL) {
+		SPDK_ERRLOG("Could not find a bdev\n");
+		spdk_app_stop(-1);
+		return;
+	}
 
 	/*
-	 * In this example, use our malloc (RAM) disk configured via
-	 * hello_blob.conf that was passed in when we started the
-	 * SPDK app framework.
-	 *
 	 * spdk_bs_init() requires us to fill out the structure
 	 * spdk_bs_dev with a set of callbacks. These callbacks
 	 * implement read, write, and other operations on the
@@ -430,10 +430,9 @@ hello_start(void *arg1)
 	 * However blobstore can be more tightly integrated into
 	 * any lower layer, such as NVMe for example.
 	 */
-	rc = spdk_bdev_create_bs_dev_ext("Malloc0", base_bdev_event_cb, NULL, &bs_dev);
-	if (rc != 0) {
-		SPDK_ERRLOG("Could not create blob bdev, %s!!\n",
-			    spdk_strerror(-rc));
+	bs_dev = spdk_bdev_create_bs_dev(bdev, NULL, NULL);
+	if (bs_dev == NULL) {
+		SPDK_ERRLOG("Could not create blob bdev!!\n");
 		spdk_app_stop(-1);
 		return;
 	}
@@ -451,7 +450,7 @@ main(int argc, char **argv)
 	SPDK_NOTICELOG("entry\n");
 
 	/* Set default values in opts structure. */
-	spdk_app_opts_init(&opts, sizeof(opts));
+	spdk_app_opts_init(&opts);
 
 	/*
 	 * Setup a few specifics before we init, for most SPDK cmd line
@@ -460,7 +459,7 @@ main(int argc, char **argv)
 	 * specify a name for the app.
 	 */
 	opts.name = "hello_blob";
-	opts.json_config_file = argv[1];
+	opts.config_file = argv[1];
 
 
 	/*

examples/blob/hello_world/hello_blob.conf

@@ -0,0 +1,3 @@
+[Malloc]
+  NumberOfLuns 1
+  LunSizeInMB  16