accel/idxd: add and respect flag to support writes to PMEM

Plumbing for flags was added in prior pathces.  This patch
introduces and respects the relevant flags for use with PMEM
aka durable memory through the accel_fw, IDXD, IOAT and SW
modules.

Signed-off-by: paul luse <paul.e.luse@intel.com>
Change-Id: I792f31459e061d220965feced60e0c236d819a68
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9455
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,6 +7,8 @@ assignees: ''
 
 ---
 
+# Bug report
+
 <!--- Provide a general summary of the issue in the Title above -->
 
 ## Expected Behavior

.github/ISSUE_TEMPLATE/intermittent_failure.md

@@ -7,6 +7,8 @@ assignees: ''
 
 ---
 
+# CI Intermittent Failure
+
 <!--- Provide a [test_name] where the issue occurred and brief description in the Title above. -->
 <!--- Name of the test can be found by last occurrence of: -->
 <!--- ************************************	-->

CHANGELOG.md

@@ -1,6 +1,490 @@
 # Changelog
 
-## v21.04: (Upcoming Release)
+## v22.05: (Upcoming Release)
+
+### bdev
+
+Removed deprecated spdk_bdev_module_finish_done(). Use spdk_bdev_module_fini_done() instead.
+
+### idxd
+
+A new parameter `flags` was added to all low level submission and preparation
+APIs to enable the caller to pass operation flags per the DSA specification.
+
+A new flag 'SPDK_IDXD_FLAG_PERSISTENT' was added to let DSA know that
+the destination is persistent.
+
+### accel_fw
+
+A new parameter `flags` was added to accel API.
+The APIs include:
+`spdk_accel_submit_copy`
+`spdk_accel_submit_dualcast`
+`spdk_accel_submit_fill`
+`spdk_accel_submit_copy_crc32c`
+`spdk_accel_submit_copy_crc32cv`
+
+A new flag `ACCEL_FLAG_PERSISTENT` was added to indicate the target memory is PMEM.
+
+### bdev_nvme
+
+Added `bdev_nvme_add_error_injection` and `bdev_nvme_remove_error_injection` RPCs to add and
+remove NVMe error injections.
+
+### nvmf
+
+Removed deprecated max_qpairs_per_ctrlr parameter from nvmf_create_transport RPC. Use
+max_io_qpairs_per_ctrlr instead.
+
+Deprecated the ability for hosts to connect to the discovery subsystem automatically on any
+existing listener. Users should now explicitly add listeners for the discovery subsystem.
+Host can still connect to the discovery subsystem as before, but a warning message will be
+emitted if no listener was configured for the transport ID of the incoming connection.
+
+### thread
+
+Added `spdk_thread_exec_msg()` API.
+
+## v22.01
+
+### accel
+
+The batching capability was removed. Batching is now considered an implementation
+detail of the low level drivers.
+
+### bdev
+
+The NVMe bdev module supports multipath and improved I/O error resiliency.
+
+The parameter `retry_count` of the RPC `bdev_nvme_set_options` was deprecated and will be
+removed in SPDK 22.04, and the parameter `transport_retry_count` is added and used instead.
+
+An new parameter `bdev_retry_count` is added to the RPC `bdev_nvme_set_options`.
+
+New parameters, `ctrlr_loss_timeout_sec`, `reconnect_delay_sec`, and `fast_io_fail_timeout_sec`, are
+added to the RPC `bdev_nvme_attach_controller`.
+
+An new parameter `num_io_queues` is added to `bdev_nvme_attach_controller` RPC to allow specifying amount
+of requested IO queues.
+
+Added `key_file` parameter to the `rbd_register_cluster` RPC.  It is an optional parameter to
+specify a keyring file to connect to a RADOS cluster.
+
+Added `spdk_bdev_get_zone_id` API to retrieve zone_id for a certain LBA.
+
+### bdev_aio
+
+Added `bdev_aio_rescan` RPC to allow rescanning the size of aio bdev.
+
+### bdev_nvme
+
+Added discovery service to bdev_nvme module, which now can connect to a discovery controller.
+bdev_nvme will connect to all current and future subsystems in the discovery controller.
+Added `bdev_nvme_start_discovery` and `bdev_nvme_stop_discovery` RPC.
+
+### dpdk
+
+Updated DPDK submodule to DPDK 21.11.
+
+### env
+
+Added `spdk_pci_for_each_device`.
+
+Removed `spdk_pci_get_first_device` and `spdk_pci_get_next_device`.  These APIs were unsafe, because
+they did not account for PCI devices being inserted or removed while the caller was using handles
+returned from these APIs.  Existing users of these APIs should switch to `spdk_pci_for_each_device`
+instead.
+
+Added 3 experimental APIs to handle PCI device interrupts (`spdk_pci_device_enable_interrupt`,
+`spdk_pci_device_disable_interrupt`, `spdk_pci_device_get_interrupt_efd`).
+
+Added `framework_get_pci_devices` RPC to list PCIe devices attached to an SPDK application.
+
+### idxd
+
+Many APIs are now vectored rather than scalar, meaning they take iovecs instead of individual pointers.
+
+### json
+
+Added `spdk_json_write_bytearray` API to serialize a buffer as a hex string.
+
+### nvme
+
+API `spdk_nvme_trtype_is_fabrics` was added to return existing transport type
+is fabric or not.
+
+API `spdk_nvme_poll_group_remove` was limited to be available only for a
+disconnected qpair in the group.
+
+New APIs, `spdk_nvme_ctrlr_disconnect`, `spdk_nvme_ctrlr_reconnect_async`, and
+`spdk_nvme_ctrlr_reconnect_poll_async`, have been added to improve error recovery, and
+the existing APIs,`spdk_nvme_ctrlr_reset_async` and `spdk_nvme_ctrlr_reset_poll_async`
+were deprecated.
+
+Added `spdk_nvme_ctrlr_get_discovery_log_page` API for getting the full discovery log page
+from a discovery controller.
+
+### nvmf
+
+Added support for zero-copy operations in the NVMe-oF TCP target. It can be enabled via
+the `zcopy` parameter when creating a transport. The zero-copy operations are only used
+for requests not using in-capsule data.
+
+Added a `subsystem` parameter to `spdk_nvmf_transport_stop_listen_async`. When not NULL,
+it will only disconnect qpairs for controllers associated with the specified subsystem.
+
+Removed accept poller from transport layer. Each transport can have its own policy of
+handling new connections. To notify transport layer `spdk_nvmf_poll_group_add` and
+`spdk_nvmf_tgt_new_qpair` can be used.
+
+### scsi
+
+Structure `spdk_scsi_lun` has been extended with new member `resizing` so that SCSI layer now reports
+unit attention for disk resize.
+
+### trace
+
+Added `spdk_trace_create_tpoint_group_mask` to return tracepoint group mask from
+a tracepoint group name.
+
+Added `trace_set_tpoint_mask` and `trace_clear_tpoint_mask` RPC to allow enabling
+individual traces.
+
+### util
+
+Added `spdk_ioviter_first` and `spdk_ioviter_next` to iterate over two iovecs and
+yield pointers to common length segments.
+
+## v21.10
+
+Structure `spdk_nvmf_target_opts` has been extended with new member `discovery_filter` which allows to specify
+filtering rules applied during discovery log generation. Refer to `enum spdk_nvmf_tgt_discovery_filter` for more info.
+
+### bdev
+
+New API `spdk_bdev_get_memory_domains` has been added, it allows to get SPDK memory domains used by bdev.
+
+New API functions `spdk_bdev_readv_blocks_ext` and `spdk_bdev_writev_blocks_ext` have been added.
+These functions accept `spdk_bdev_ext_io_opts` structure with extended IO request
+options, e.g. DMA memory domain which describes data that may belong to another memory domain and
+can't be accessed directly.
+
+Added `async_fini_start` to allow bdev modules to complete the `fini_start` asynchronously,
+with new `spdk_bdev_module_fini_start_done` API.
+
+Deprecated `spdk_bdev_module_finish_done()` API, which will be removed in SPDK 22.01.
+Bdev modules should use `spdk_bdev_module_fini_done()` instead.
+
+The `ocssd` bdev has been removed. The Open Channel specification has been largely superseded by
+zoned namespaces, few if any devices have been brought to market, and there is little reason to
+continue to support this. OCSSD support in the nvme driver will remain for now.
+
+### dma
+
+A new library, lib/dma, has been added. This library provides the necessary infrastructure for
+handling systems and devices with multiple memory domains. For example, a PCIe add-in card with an
+SoC may be running SPDK on the SoC. That SoC has its own local memory, but SPDK may be controlling
+devices that can also access the host system memory. This library provides infrastructure to enumerate
+the memory domains and request hardware perform DMA transfers between them.
+
+### dpdk
+
+Updated DPDK submodule to DPDK 21.08.
+
+### event
+
+Added the `disable_signal_handlers` flag to the `spdk_app_opts` struct.
+
+### idxd
+
+Added `spdk_idxd_get_socket` to query the socket that the idxd device
+is on.
+
+### json
+
+Added API to allow for writing and decoding of new types:
+
+- `spdk_json_write_named_uint16`
+- `spdk_json_write_named_uint8`
+- `spdk_json_write_uint16`
+- `spdk_json_write_uint8`
+- `spdk_json_number_to_uint8`
+- `spdk_json_decode_uint8`
+
+### log
+
+Added API `spdk_log_to_syslog_level` to return syslog level based on SPDK's
+log level.
+
+### nvme
+
+Added new functions `spdk_nvme_zns_set_zone_desc_ext` and `spdk_nvme_zns_ext_report_zones`
+to set zone descriptor extension and to get extended zone report respectively.
+
+New API `spdk_nvme_ctrlr_get_memory_domains` has been added, it allows to get SPDK memory domains used by nvme controller.
+
+New API functions `spdk_nvme_ns_cmd_readv_ext` and `spdk_nvme_ns_cmd_writev_ext`
+have been added. These functions accept `spdk_nvme_ns_cmd_ext_io_opts` structure with extended IO request
+options, e.g. DMA memory domain which describes data that may belong to another memory domain and
+can't be accessed directly.
+
+Added a new function `spdk_nvme_ctrlr_get_regs_bpinfo` to get boot partition info of a controller.
+Added new functions `spdk_nvme_ctrlr_write_boot_partition`,
+`spdk_nvme_ctrlr_read_boot_partition_start` and `spdk_nvme_ctrlr_read_boot_partition_poll`
+to write and read the boot partitions of a controller.
+
+Added `spdk_nvme_ctrlr_get_opts` to retrieve the current controller options.
+
+Added `async_mode` to `spdk_nvme_io_qpair_opts` to enable creation of submission and completion
+queues asynchronously. This mode is currently supported at PCIe layer,
+which tracks the qpair creation with state machine and returns to the user immediately.
+Default mode is set to false to create io qpairs synchronously.
+
+Added `spdk_nvme_ctrlr_get_regs_cc` to retrieve NVMe controller CC (Configuration) register.
+
+Added `spdk_nvme_ctrlr_prepare_for_reset` to inform the driver that the application is preparing
+to reset the specified NVMe controller. This function allows the driver to make decisions
+knowing that a reset is about to happen.
+
+### nvmf
+
+Added `oncs` to `struct spdk_nvmf_ctrlr_data` so that the transport layer
+can decide support RESERVATION feature or not.
+
+An `opts_size` element was added in the `spdk_nvmf_ns_opts` structure to solve the
+ABI compatibility issue between different SPDK version. An new option `anagrpid` was
+added in the `spdk_nvmf_ns_opts` structure.
+
+An new parameter `anagrpid` was added to the RPC `nvmf_subsystem_add_ns`.
+
+An new parameter `anagrpid` was added to the RPC `nvmf_subsystem_listener_set_ana_state`.
+
+`spdk_nvmf_subsystem_destroy` is now can be asynchronous, it accepts a callback and callback argument.
+
+A new parameter, `poll_groups_mask` was added to the `nvmf_set_config` RPC that allows specifying
+a subset of cores for the nvmf poll groups. This helps to avoid imbalances when some cores are
+busy with periodic timer tasks that run very frequently.
+
+A new parameter, `commit` was added to `nvmf_bdev_ctrlr_end_zcopy` that allows indicating whether
+the buffers should be committed.
+
+Added `listen_dump_opts` to `spdk_nvmf_transport_ops` so that the transport layer can display
+listen opts.
+
+Added `fabrics_connect_timeout_us` to `bdev_nvme_attach_controller` RPC to allow specifying
+timeout for connect operation.
+
+Added `multipath` to `bdev_nvme_attach_controller` RPC to allow specifying multipath behavior.
+
+### scheduler
+
+New API for implementing schedulers and governors. Please see `include/spdk/scheduler.h` for details.
+
+### scsi
+
+New functions, `spdk_scsi_dev_get_first_lun` and `spdk_scsi_dev_get_next_lun`
+have been added to iterate LUNs of a SCSI device.
+
+Each SCSI device supports 256 LUNs at the maximum now and the macro constant
+`SPDK_SCSI_DEV_MAX_LUN` was removed.
+
+### trace_parser
+
+A new library, lib/trace_parser, has been added. This library provides functions that parse traces
+recorded by an SPDK application. That includes merging traces from multiple cores,
+sorting them by their timestamp and constructing trace entries spanning across multiple buffers.
+
+### util
+
+The `spdk_fd_group_add` API now takes a `name` parameter.
+
+## v21.07
+
+### accel_fw
+
+Added API `spdk_accel_submit_copy_crc32c` to perform a CRC32C while copying data.
+
+Added API `spdk_accel_batch_prep_copy_crc32c` to batch CRC32C + copy commands.
+
+Added API `spdk_accel_submit_copy_crc32cv` to submit chained CRC32C + copy commands.
+
+### bdev
+
+Change `spdk_bdev_read_blocks_with_md` arg offset definition from int64_t to uint64_t.
+
+Red-black tree has been used to organize the bdev names and aliases uniformly
+to provide faster lookup.
+
+Removed ZCOPY emulation: The bdev module can be checked to see if it supports ZCOPY
+and if not supported then use existing READ/WRITE commands.
+
+Added iov to spdk_bdev_zcopy_start.
+
+### dpdk
+
+Updated DPDK submodule to DPDK 21.05.
+
+### idxd
+
+Remove the `probe_cb` parameter in `spdk_idxd_probe` function. And remove the definition
+of `spdk_idxd_probe_cb` function pointer. It should be implemented in idxd_user.c.
+
+Added API `spdk_idxd_submit_copy_crc32c` to perform a CRC32C while copying data.
+
+Added API `spdk_idxd_batch_prep_copy_crc32c` to prepare a batch operation to perform
+a CRC32C while copying data.
+
+We have userspace idxd driver under accel_engine library (module/accel/idxd/accel_engine).
+And kernel idxd driver usage is also added under the accel_engine library. This approach
+implemented a wrapper library to use IDXD device by leveraging the kernel DSA driver in
+SPDK idxd library (lib/idxd). Then users can leverage the RPC later to configure how to
+use the DSA device by user space driver or kernel driver.
+
+### init
+
+Added new `init` library that initializes the SPDK subsystems, which previously was
+internal to application framework. That functionality has been made public and for
+use by applications which don't leverage SPDK's application framework.
+
+### iscsi
+
+New parameters, `pdu_pool_size`, `immediate_data_pool_size`, and `data_out_pool_size`,
+were added to the RPC `iscsi_set_options` to run iSCSI target with varying amount of
+available memory.
+
+### json
+
+Added API `spdk_json_write_named_uint128` and `spdk_json_write_uint128` to perform
+the uint128 related data.
+
+### net
+
+Removed deprecated `net` library.
+
+Added support for ZCOPY.  ZCOPY is used in preference to READ and WRITE if the
+bdev module supports ZCOPY and the bdev module has ZCOPY enabled.
+
+### nvme
+
+`spdk_nvme_map_prps` and `spdk_nvme_map_cmd` were moved to nvmf/vfio-user as internal APIs
+as vfio-user is the only user for the above two APIs.
+
+Added a new function `spdk_nvme_ns_cmd_copy` to submit a Simple Copy Command to a Namespace.
+
+Update the `spdk_nvme_generic_command_status_code` structure with new status code
+according to the definition in NVMe 1.4 spec.
+
+`spdk_nvme_ctrlr_get_default_ctrlr_opts` now sets `use_cmb_sqs` to false. This means
+that if a controller has a CMB and supports SQs in the CMB, SPDK will not use
+the CMB for SQs by default - the user must set `use_cmb_sqs` to true in
+the `spdk_nvme_ctrlr_opts` structure prior to controller attach.
+
+Add a new function `spdk_nvme_detach_poll` to simplify a common use case to continue
+polling until all detachments complete.
+
+Added new argument `timeout_admin_us` to `spdk_nvme_ctrlr_register_timeout_callback` so callers
+can specify a different timeout for admin commands vs. io commands.
+
+An existing function `spdk_nvme_detach_async` was updated to add one or more detachments
+to an active context while it is being polled.
+
+Rename a variable in the member `cmic` of the struct `spdk_nvme_ctrlr_data` from
+`multi_host` to `multi_ctrlr`. The variable means that the NVM subsystem may have two or
+more controllers if set to 1. However `multi_host` had indicated a particular use case
+such that the NVM subsystem is used by multiple hosts.
+
+A new option `disable_read_ana_log_page` was added to struct `spdk_nvme_ctrlr_opts` to disable
+reading ANA log page. The upper layer is expected to read ANA log page instead if `true`.
+The default value is `false`.
+
+New APIs, `spdk_nvme_ctrlr_reset_async` and `spdk_nvme_ctrlr_reset_poll_async`, have been added to
+reset a controller asynchronously.
+
+New RPC `bdev_nvme_reset_controller` was added, to reset an NVMe controller.
+
+Added `spdk_nvme_ns_get_nguid` function to get NGUID for the given namespace.
+
+Added `spdk_nvme_ctrlr_is_fabrics` function to indicate whether a ctrlr handle
+is associated with a fabrics controller.
+
+### nvmf
+
+Added `min_cntlid` and `max_cntlid` to `nvmf_create_subsystem` to limit the controller ID range.
+Added `spdk_nvmf_subsystem_get_min_cntlid` and `spdk_nvmf_subsystem_get_max_cntlid` to request those values.
+
+`spdk_nvmf_request_get_buffers_multi` API is removed.
+
+Added the `nvmf_set_crdt` RPC for setting command retry delay times.
+
+Expanded `spdk_nvmf_poll_group_stat` with current qpair count statistics.
+
+Removed following deprecated APIs:
+
+- `spdk_nvmf_poll_group_get_stat` (function in `nvmf.h`),
+- `spdk_nvmf_transport_poll_group_get_stat` (function in `nvmf.h`),
+- `spdk_nvmf_transport_poll_group_free_stat`(function in `nvmf.h`),
+- `spdk_nvmf_rdma_device_stat` (struct in `nvmf.h`),
+- `spdk_nvmf_transport_poll_group_stat` (struct in `nvmf.h`),
+- `poll_group_get_stat` (transport op in `nvmf_transport.h`),
+- `poll_group_free_stat` (transport op in `nvmf_transport.h`).
+
+### rpc
+
+New RPC `bdev_rbd_register_cluster` and `bdev_rbd_unregister_cluster` was added, it allows to create
+and delete the rados object cluster, then users can choose the cluster to create related rbd
+device.
+
+Revised `bdev_rbd_create` parameter, it allows to use an optional parameter `--cluster-name`
+to create a rbd bdev with  an already registered Rados Cluster Object.
+
+New RPC `bdev_rbd_get_clusters_info` was added, it allows to get the info of the registered
+Rados Cluster names.
+
+New optional parameter, `timeout_admin_us`, added to the bdev_nvme_set_options RPC.
+
+Revised a parameter `--stripe-size_kb` to `--stripe-size-kb` of `bdev_raid_create` method
+provided in `scripts/rpc.py` for consistency.
+
+An new optional parameter `config_kernel_mode` was added to the RPC `idxd_scan_accel_engine`,
+and this is used to enable using the kernel mode IDXD driver.
+
+Removed deprecated parameter `enable-zerocopy-send` of RPC `sock_impl_set_options`,
+use `enable-zerocopy-send-server` or `enable-zerocopy-send-client` instead.
+Removed deprecated parameter `disable-zerocopy-send` of RPC `sock_impl_set_options`,
+use `disable-zerocopy-send-server` or `disable-zerocopy-send-client` instead.
+
+### rpm
+
+Removed deprecated `pkg/spdk.spec`, please use `rpmbuild/spdk.spec` instead.
+See [RPM documentation](https://spdk.io/doc/rpm.html) for more details.
+
+### thread
+
+Red-black tree has been used for timed pollers to provide faster insertion and deletion
+and for io_devices to provide faster lookup.
+
+Removed `spdk_io_channel` structure from public header and moved it to thread_internal.h.
+
+### trace
+
+Added `spdk_trace_register_description_ext` function to register variable number of tracepoint arguments.
+
+Added ability to chain multiple trace entries together to extend the size of the argument buffer.
+
+### util
+
+Red-black tree macros has been added by using the macros provided by the FreeBSD operating system
+under the same BSD license.
+
+Add an new macro `SPDK_SIZEOF_MEMBER` to get the size of a member of a struct.
+
+`spdk_crc32c_iov_update` function was added to support calculating the crc32c of the iovs.
+
+Added zipf random number generator with power law probability distribution.
+When applied to performance testing of block devices, it will select blocks over
+the full range of LBAs, but will more frequently select lower-numbered LBAs.
+
+## v21.04
 
 ### accel
 
@@ -161,6 +645,17 @@ For `bdev_raid_create` RPC, the deprecated parameter `strip_size` was removed.
 New RPC `bdev_nvme_get_transport_statistics` was added, it allows to get transport statistics
 of nvme poll groups.
 
+Parameter `enable-zerocopy-send` of RPC `sock_impl_set_options` is deprecated and will be removed in SPDK 21.07,
+use `enable-zerocopy-send-server` or `enable-zerocopy-send-client` instead.
+Parameter `disable-zerocopy-send` of RPC `sock_impl_set_options` is deprecated and will be removed in SPDK 21.07,
+use `disable-zerocopy-send-server` or `disable-zerocopy-send-client` instead.
+
+Added cmd_parser.py used to parse the args from argparse. There are
+two benefit to use command parser:
+
+- Simplify the definition of rpc method. It will reduce the rpc method code.
+- Make the rpc call more versatile. User can add private args into rpc method.
+
 ### rpm
 
 Added support for new RPM spec, rpmbuild/spdk.spec, which can be used for packaging the
@@ -179,6 +674,12 @@ sockets to be marked using the SO_MARK socket option as a hint for which hardwar
 queue they should be associated with. This mode leverages that by setting the same
 value for all sockets within a poll group.
 
+New parameters `enable_zerocopy_send_server` and `enable_zerocopy_send_client` were added
+to struct spdk_sock_impl_opts, these parameters enable or disable zero copy send for server
+and client sockets which are created using `spdk_sock_listen` and `spdk_sock_listen_ext` (server);
+`spdk_sock_connect` and `spdk_sock_connect_ext` (client) functions. Existing parameter
+`enable_zerocopy_send`  enables or disables zero copy send for both server and client sockets.
+
 ### thread
 
 A new API `spdk_io_channel_get_io_device` was added to get the io_device for the specified
@@ -196,7 +697,7 @@ into or out of interrupt mode.
 
 Added the `bdev_virtio_blk_set_hotplug` RPC for the virtio blk pci device.
 
-## v21.01:
+## v21.01
 
 ### bdev
 
@@ -341,7 +842,7 @@ This allows the initiator to create persistent connection to discovery controlle
 be notified of any discovery log changes.
 
 An `opts_size` element was added in the `spdk_nvmf_transport_opts` structure
-to solve the ABI compatiblity issue between different SPDK version. And also add
+to solve the ABI compatibility issue between different SPDK version. And also add
 `opts_size` parameter in `spdk_nvmf_transport_opts_init` function.
 
 Added a new custom vfio-user transport implementation in NVMe-oF which can provide
@@ -377,7 +878,7 @@ into `bdev_set_options` function.
 Added optional `packed_ring_recovery` parameter to `vhost_create_blk_controller` RPC
 enabling packed ring live recovery.
 
-## v20.10:
+## v20.10
 
 ### accel
 
@@ -613,11 +1114,11 @@ query the capabilities of the currently enabled accel engine back-end.
 
 A new capability, CRC-32C, was added via `spdk_accel_submit_crc32c`.
 
-The software accel engine implemenation has added support for CRC-32C.
+The software accel engine implementation has added support for CRC-32C.
 
 A new capability, compare, was added via `spdk_accel_submit_compare`.
 
-The software accel engine implemenation has added support for compare.
+The software accel engine implementation has added support for compare.
 
 Several APIs were added to `accel_engine.h` to support batched submission
 of operations.
@@ -672,7 +1173,7 @@ A new API `spdk_ioat_get_max_descriptors` was added.
 ### nvme
 
 An `opts_size`element was added in the  `spdk_nvme_ctrlr_opts` structure
-to solve the ABI compatiblity issue between different SPDK version.
+to solve the ABI compatibility issue between different SPDK version.
 
 A new API `spdk_nvme_ctrlr_cmd_abort_ext` has been added to abort previously submitted
 commands whose callback argument match.
@@ -1053,7 +1554,7 @@ log page requests.
 ### nvmf
 
 Add SockPriority option in [Transport] section, this can be used for NVMe-oF target
-on TCP transport to set sock priority for the incomming TCP connections.
+on TCP transport to set sock priority for the incoming TCP connections.
 
 The NVMe-oF target now supports plugging out of tree NVMe-oF transports. In order
 to facilitate this feature, several small API changes have been made:
@@ -1171,18 +1672,18 @@ to blobfs.
 Added new parameter `cdw0` to `spdk_bdev_io_complete_nvme_status()` and
 `spdk_bdev_io_get_nvme_status()` that allows setting/getting
 the NVMe completion queue DW0 entry. This allows vendor specific IO commands
-to return commmand specific completion info back to the initiator.
+to return command specific completion info back to the initiator.
 
 Added `spdk_bdev_get_write_unit_size()` function for retrieving required number
 of logical blocks for write operation.
 
 New zone-related fields were added to the result of the `get_bdevs` RPC call:
 
- - `zoned`: indicates whether the device is zoned or a regular
-   block device
- - `zone_size`: number of blocks in a single zone
- - `max_open_zones`: maximum number of open zones
- - `optimal_open_zones`: optimal number of open zones
+- `zoned`: indicates whether the device is zoned or a regular
+  block device
+- `zone_size`: number of blocks in a single zone
+- `max_open_zones`: maximum number of open zones
+- `optimal_open_zones`: optimal number of open zones
 
 The `zoned` field is a boolean and is always present, while the rest is only available for zoned
 bdevs.
@@ -1405,8 +1906,8 @@ A new file API `spdk_posix_file_load` was added to load file content into a data
 New APIs `spdk_dif_ctx_set_data_offset`, `spdk_dif_verify_stream`,
 `spdk_dif_get_range_with_md`, `spdk_dif_get_length_with_md` have been added,
 and existing APIs `spdk_dif_ctx_init`, `spdk_dif_set_md_interleave_iovs`, and
-`spdk_dif_generate_stream` have been refined to insert or strip DIF by stream
-fasion with any alignment.
+`spdk_dif_generate_stream` have been refined to insert or strip DIF into iovs
+windows of any alignment.
 
 New APIs `spdk_dif_ctx_set_remapped_init_ref_tag`, `spdk_dif_remap_ref_tag`,
 and `spdk_dix_remap_ref_tag` have been added to remap DIF reference tag.
@@ -1443,7 +1944,7 @@ while some compatibility fixes make their way down the pipeline for 5.1.x kernel
 
 The sock priority setting of the TCP connection owned by the tcp transport is added. It is
 used to optimize the TCP connection performance under designated traffic classes. And the
-priority is used to differeniate the sock priority between SPDK NVMe-oF TCP target application
+priority is used to differentiate the sock priority between SPDK NVMe-oF TCP target application
 and other TCP based applications.
 
 Shared receive queue can now be disabled even for NICs that support it using the
@@ -1495,7 +1996,7 @@ modules. When using partition type virtual bdevs, block address space is
 remapped during I/O processing and DIF reference tag is remapped accordingly.
 
 Added spdk_bdev_*_with_md() functions allowing for IO with metadata being transferred in
-separate buffer. To check support for separatate metadata, use spdk_bdev_is_md_separate().
+separate buffer. To check support for separate metadata, use spdk_bdev_is_md_separate().
 
 All bdevs now have a UUID. For devices whose backing hardware does not provide a UUID,
 one is automatically generated. Across runs of SPDK, bdevs whose UUID is automatically
@@ -1514,7 +2015,7 @@ limited requests in NVMe driver layer.
 Added spdk_nvme_ctrlr_get_transport_id() to get the transport ID from a
 previously attached controller.
 
-Nvme Opal library spdk_opal_cmd deprecated. Adding seperate command APIs.
+Nvme Opal library spdk_opal_cmd deprecated. Adding separate command APIs.
 NVMe Opal library add support for activating locking SP which will make the transaction
 from "Manufactured-Inactive" state to "Manufactured" state. Upon successfully invoking
 of this method, lock and unlock features will be enabled.
@@ -1582,7 +2083,7 @@ spdk_sock_set_priority() is added to set the priority of the socket.
 
 Added thread_get_stats RPC method to retrieve existing statistics.
 
-Added nvmf_get_stats RPC method to retrieve NVMf susbsystem statistics.
+Added nvmf_get_stats RPC method to retrieve NVMf subsystem statistics.
 
 Response buffers for RPC requests are now always pre-allocated, which implies
 that all spdk_jsonrpc_begin_result() calls always succeed and return a valid
@@ -1606,7 +2107,7 @@ Preliminary support for cross compilation is now available. Targeting an older
 CPU on the same architecture using your native compiler can be accomplished by
 using the `--target-arch` option to `configure` as follows:
 
-~~~
+~~~bash
 ./configure --target-arch=broadwell
 ~~~
 
@@ -1614,7 +2115,7 @@ Additionally, some support for cross-compiling to other architectures has been
 added via the `--cross-prefix` argument to `configure`. To cross-compile, set CC
 and CXX to the cross compilers, then run configure as follows:
 
-~~~
+~~~bash
 ./configure --target-arch=aarm64 --cross-prefix=aarch64-linux-gnu
 ~~~
 
@@ -1867,7 +2368,7 @@ be combined with link time optimization which has been available under the
 ### compression bdev/reduce library
 
 Added "reduce" block compression scheme based on using SSDs for storing
-compressed blocks of storage and presistent memory for metadata. Please see
+compressed blocks of storage and persistent memory for metadata. Please see
 [compression](https://spdk.io/doc/bdev.html) for more details.
 
 ## v19.01: NVMe-oF TCP/IP Transport, Open Channel SSD Flash Translation Layer, Caching bdev based on OCF, ISA-L Support, DIF/DIX library
@@ -1902,19 +2403,19 @@ for which the memory is contiguous in the physical memory address space.
 
 The following functions were removed:
 
- - spdk_pci_nvme_device_attach()
- - spdk_pci_nvme_enumerate()
- - spdk_pci_ioat_device_attach()
- - spdk_pci_ioat_enumerate()
- - spdk_pci_virtio_device_attach()
- - spdk_pci_virtio_enumerate()
+- spdk_pci_nvme_device_attach()
+- spdk_pci_nvme_enumerate()
+- spdk_pci_ioat_device_attach()
+- spdk_pci_ioat_enumerate()
+- spdk_pci_virtio_device_attach()
+- spdk_pci_virtio_enumerate()
 
 They were replaced with generic spdk_pci_device_attach() and spdk_pci_enumerate() which
 require a new spdk_pci_driver object to be provided. It can be one of the following:
 
- - spdk_pci_nvme_get_driver()
- - spdk_pci_ioat_get_driver()
- - spdk_pci_virtio_get_driver()
+- spdk_pci_nvme_get_driver()
+- spdk_pci_ioat_get_driver()
+- spdk_pci_virtio_get_driver()
 
 spdk_pci_hook_device() and spdk_pci_unhook_device() were added. Those allow adding a virtual
 spdk_pci_device into the SPDK PCI subsystem. A virtual device calls provided callbacks for
@@ -1995,7 +2496,7 @@ configure the shared buffer numbers of the transport used by RDMA or TCP transpo
 
 ### nvmf
 
-Add a new TCP/IP transport (located in lib/nvmf/tcp.c). With this tranport,
+Add a new TCP/IP transport (located in lib/nvmf/tcp.c). With this transport,
 the SPDK NVMe-oF target can have a new transport, and can serve the NVMe-oF
 protocol via TCP/IP from the host.
 
@@ -2023,7 +2524,7 @@ for the bdev type to be removed (i.e. delete_malloc_bdev).
 
 Added support for separate bandwidth rate limits for read and write to QoS in bdev layer.
 
-Bdev I/O statistics now track unmap opertations.
+Bdev I/O statistics now track unmap operations.
 
 ### logical volumes
 
@@ -2093,12 +2594,12 @@ Dropped support for DPDK 16.07 and earlier, which SPDK won't even compile with r
 
 The following RPC commands deprecated in the previous release are now removed:
 
- - construct_virtio_user_scsi_bdev
- - construct_virtio_pci_scsi_bdev
- - construct_virtio_user_blk_bdev
- - construct_virtio_pci_blk_bdev
- - remove_virtio_scsi_bdev
- - construct_nvmf_subsystem
+- construct_virtio_user_scsi_bdev
+- construct_virtio_pci_scsi_bdev
+- construct_virtio_user_blk_bdev
+- construct_virtio_pci_blk_bdev
+- remove_virtio_scsi_bdev
+- construct_nvmf_subsystem
 
 ### Miscellaneous
 
@@ -2122,7 +2623,7 @@ RPC `wait_subsystem_init` has been added to allow clients to block untill all su
 ### json rpc
 
 JSON RPC client is now running in non-blocking mode. Requests are sent and received during spdk_jsonrpc_client_poll.
-JSON RPC server can now recieve a callback on connection termination or server shutdown using `spdk_jsonrpc_conn_add_close_cb`
+JSON RPC server can now receive a callback on connection termination or server shutdown using `spdk_jsonrpc_conn_add_close_cb`
 and `spdk_jsonrpc_conn_del_close_cb`.
 
 ## v18.10: Dynamic memory allocation, Crypto Virtual Bdev, jsonrpc-client, SPDKCLI iSCSI and NVMe-oF support
@@ -2263,7 +2764,7 @@ for Python 2 up to the EOL which is year 2020.
 Invoking interpreter explicitly is forbidden for executable scripts. There
 is no need to use syntax like "python ./scripts/rpc.py". All executable
 scripts must contain proper shebang pointing to the right interpreter.
-Scripts without shebang musn't be executable.
+Scripts without shebang mustn't be executable.
 
 A Python script has been added to enable conversion of old INI config file
 to new JSON-RPC config file format. This script can be found at
@@ -2282,11 +2783,11 @@ respectively.
 
 The following RPC commands have been deprecated:
 
- - construct_virtio_user_scsi_bdev
- - construct_virtio_pci_scsi_bdev
- - construct_virtio_user_blk_bdev
- - construct_virtio_pci_blk_bdev
- - remove_virtio_scsi_bdev
+- construct_virtio_user_scsi_bdev
+- construct_virtio_pci_scsi_bdev
+- construct_virtio_user_blk_bdev
+- construct_virtio_pci_blk_bdev
+- remove_virtio_scsi_bdev
 
 The `construct_virtio_*` ones were replaced with a single `construct_virtio_dev`
 command that can create any type of Virtio bdev(s). `remove_virtio_scsi_bdev`
@@ -2303,12 +2804,12 @@ Added jsonrpc-client C library intended for issuing RPC commands from applicatio
 
 Added API enabling iteration over JSON object:
 
- - spdk_json_find()
- - spdk_json_find_string()
- - spdk_json_find_array()
- - spdk_json_object_first()
- - spdk_json_array_first()
- - spdk_json_next()
+- spdk_json_find()
+- spdk_json_find_string()
+- spdk_json_find_array()
+- spdk_json_object_first()
+- spdk_json_array_first()
+- spdk_json_next()
 
 ### Blobstore
 
@@ -2639,7 +3140,7 @@ framework (include/spdk/event.h) to the I/O channel library
 framework to request registration and unregistration of pollers.
 
 spdk_for_each_channel() now allows asynchronous operations during iteration.
-Instead of immediately continuing the interation upon returning from the iteration
+Instead of immediately continuing the interaction upon returning from the iteration
 callback, the user must call spdk_for_each_channel_continue() to resume iteration.
 
 ### Block Device Abstraction Layer (bdev)
@@ -2783,7 +3284,7 @@ The NVMe-oF target no longer requires any in-capsule data buffers to run, and
 the feature is now entirely optional. Previously, at least 4 KiB in-capsule
 data buffers were required.
 
-NVMe-oF subsytems have a new configuration option, AllowAnyHost, to control
+NVMe-oF subsystems have a new configuration option, AllowAnyHost, to control
 whether the host NQN whitelist is enforced when accepting new connections.
 If no Host options have been specified and AllowAnyHost is disabled, the
 connection will be denied; this is a behavior change from previous releases,
@@ -2848,7 +3349,8 @@ See the [Pmem Block Device](http://www.spdk.io/doc/bdev.html#bdev_config_pmem) d
 A userspace driver for Virtio SCSI devices has been added.
 The driver is capable of creating block devices on top of LUNs exposed by another SPDK vhost-scsi application.
 
-See the [Virtio SCSI](http://www.spdk.io/doc/virtio.html) documentation and [Getting Started](http://www.spdk.io/doc/bdev.html#bdev_config_virtio_scsi) guide for more information.
+See the [Virtio SCSI](http://www.spdk.io/doc/virtio.html) documentation and
+[Getting Started](http://www.spdk.io/doc/bdev.html#bdev_config_virtio_scsi) guide for more information.
 
 ### Vhost target
 
@@ -3074,33 +3576,33 @@ in app/iscsi_tgt and a documented configuration file can be found at etc/spdk/sp
 
 This release also significantly improves the existing NVMe over Fabrics target.
 
-  - The configuration file format was changed, which will require updates to
-    any existing nvmf.conf files (see `etc/spdk/nvmf.conf.in`):
-    - `SubsystemGroup` was renamed to `Subsystem`.
-    - `AuthFile` was removed (it was unimplemented).
-    - `nvmf_tgt` was updated to correctly recognize NQN (NVMe Qualified Names)
-      when naming subsystems.  The default node name was changed to reflect this;
-      it is now "nqn.2016-06.io.spdk".
-    - `Port` and `Host` sections were merged into the `Subsystem` section
-    - Global options to control max queue depth, number of queues, max I/O
-      size, and max in-capsule data size were added.
-    - The Nvme section was removed. Now a list of devices is specified by
-      bus/device/function directly in the Subsystem section.
-    - Subsystems now have a Mode, which can be Direct or Virtual. This is an attempt
-      to future-proof the interface, so the only mode supported by this release
-      is "Direct".
-  - Many bug fixes and cleanups were applied to the `nvmf_tgt` app and library.
-  - The target now supports discovery.
+- The configuration file format was changed, which will require updates to
+  any existing nvmf.conf files (see `etc/spdk/nvmf.conf.in`):
+  - `SubsystemGroup` was renamed to `Subsystem`.
+  - `AuthFile` was removed (it was unimplemented).
+  - `nvmf_tgt` was updated to correctly recognize NQN (NVMe Qualified Names)
+    when naming subsystems.  The default node name was changed to reflect this;
+    it is now "nqn.2016-06.io.spdk".
+  - `Port` and `Host` sections were merged into the `Subsystem` section
+  - Global options to control max queue depth, number of queues, max I/O
+    size, and max in-capsule data size were added.
+  - The Nvme section was removed. Now a list of devices is specified by
+    bus/device/function directly in the Subsystem section.
+  - Subsystems now have a Mode, which can be Direct or Virtual. This is an attempt
+    to future-proof the interface, so the only mode supported by this release
+    is "Direct".
+- Many bug fixes and cleanups were applied to the `nvmf_tgt` app and library.
+- The target now supports discovery.
 
 This release also adds one new feature and provides some better examples and tools
 for the NVMe driver.
 
-  - The Weighted Round Robin arbitration method is now supported. This allows
-    the user to specify different priorities on a per-I/O-queue basis.  To
-    enable WRR, set the `arb_mechanism` field during `spdk_nvme_probe()`.
-  - A simplified "Hello World" example was added to show the proper way to use
-    the NVMe library API; see `examples/nvme/hello_world/hello_world.c`.
-  - A test for measuring software overhead was added. See `test/lib/nvme/overhead`.
+- The Weighted Round Robin arbitration method is now supported. This allows
+  the user to specify different priorities on a per-I/O-queue basis.  To
+  enable WRR, set the `arb_mechanism` field during `spdk_nvme_probe()`.
+- A simplified "Hello World" example was added to show the proper way to use
+  the NVMe library API; see `examples/nvme/hello_world/hello_world.c`.
+- A test for measuring software overhead was added. See `test/lib/nvme/overhead`.
 
 ## v16.06: NVMf userspace target
 
@@ -3128,13 +3630,13 @@ user code.
   supported on any attached controller.
   - Added support for the Write Zeroes command.
   - `examples/nvme/perf` can now report I/O command latency from the
-   the controller's viewpoint using the Intel vendor-specific read/write latency
-   log page.
+  the controller's viewpoint using the Intel vendor-specific read/write latency
+  log page.
   - Added namespace reservation command support, which can be used to coordinate
   sharing of a namespace between multiple hosts.
   - Added hardware SGL support, which enables use of scattered buffers that
-   don't conform to the PRP list alignment and length requirements on supported
-   NVMe controllers.
+  don't conform to the PRP list alignment and length requirements on supported
+  NVMe controllers.
   - Added end-to-end data protection support, including the ability to write and
   read metadata in extended LBA (metadata appended to each block of data in the
   buffer) and separate metadata buffer modes.
@@ -3155,7 +3657,8 @@ user code.
 
 ## v1.2.0: IOAT user-space driver
 
-This release adds a user-space driver with support for the Intel I/O Acceleration Technology (I/OAT, also known as "Crystal Beach") DMA offload engine.
+This release adds a user-space driver with support for the Intel I/O Acceleration Technology
+(I/OAT, also known as "Crystal Beach") DMA offload engine.
 
 - IOAT
   - New user-space driver supporting DMA memory copy offload

CONFIG

@@ -3,6 +3,7 @@
 #
 #  Copyright (c) Intel Corporation.
 #  All rights reserved.
+#  Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 #  Redistribution and use in source and binary forms, with or without
 #  modification, are permitted provided that the following conditions
@@ -64,6 +65,10 @@ CONFIG_ASAN=n
 # Build with Undefined Behavior Sanitizer enabled
 CONFIG_UBSAN=n
 
+# Build with LLVM fuzzing enabled
+CONFIG_FUZZER=n
+CONFIG_FUZZER_LIB=
+
 # Build with Thread Sanitizer enabled
 CONFIG_TSAN=n
 
@@ -76,6 +81,9 @@ CONFIG_UNIT_TESTS=y
 # Build examples
 CONFIG_EXAMPLES=y
 
+# Build apps
+CONFIG_APPS=y
+
 # Build with Control-flow Enforcement Technology (CET)
 CONFIG_CET=n
 
@@ -86,6 +94,10 @@ CONFIG_ENV=
 # This directory should contain 'include' and 'lib' directories for your DPDK
 # installation.
 CONFIG_DPDK_DIR=
+# Automatically set via pkg-config when bare --with-dpdk is set
+CONFIG_DPDK_LIB_DIR=
+CONFIG_DPDK_INC_DIR=
+CONFIG_DPDK_PKG_CONFIG=n
 
 # This directory should contain 'include' and 'lib' directories for WPDK.
 CONFIG_WPDK_DIR=
@@ -134,6 +146,12 @@ CONFIG_PMDK_DIR=
 # Enable the dependencies for building the compress vbdev
 CONFIG_REDUCE=n
 
+# Enable mlx5_pci dpdk compress PMD, enabled automatically if CONFIG_REDUCE=y and libmlx5 exists
+CONFIG_REDUCE_MLX5=n
+
+# Enable mlx5_pci dpdk crypto PMD, enabled automatically if CONFIG_CRYPTO=y and libmlx5 exists
+CONFIG_CRYPTO_MLX5=n
+
 # Requires libiscsi development libraries.
 CONFIG_ISCSI_INITIATOR=n
 
@@ -171,4 +189,18 @@ CONFIG_FUSE=n
 CONFIG_RAID5=n
 
 # Build with IDXD support
+# In this mode, SPDK fully controls the DSA device.
 CONFIG_IDXD=n
+
+# Build with USDT support
+CONFIG_USDT=n
+
+# Build with IDXD kernel support.
+# In this mode, SPDK shares the DSA device with the kernel.
+CONFIG_IDXD_KERNEL=n
+
+# Is DPDK using libbsd?
+CONFIG_HAVE_LIBBSD=n
+
+# Path to IPSEC_MB used by DPDK
+CONFIG_IPSEC_MB_DIR=

Makefile

@@ -40,8 +40,9 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 DIRS-y += lib
 DIRS-y += module
 DIRS-$(CONFIG_SHARED) += shared_lib
-DIRS-y += app include
+DIRS-y += include
 DIRS-$(CONFIG_EXAMPLES) += examples
+DIRS-$(CONFIG_APPS) += app
 DIRS-y += test
 DIRS-$(CONFIG_IPSEC_MB) += ipsecbuild
 DIRS-$(CONFIG_ISAL) += isalbuild
@@ -57,11 +58,13 @@ export MAKE_PID := $(shell echo $$PPID)
 ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV))
 ifeq ($(CURDIR)/dpdk/build,$(CONFIG_DPDK_DIR))
 ifneq ($(SKIP_DPDK_BUILD),1)
+ifneq ($(CONFIG_DPDK_PKG_CONFIG),y)
 DPDKBUILD = dpdkbuild
 DIRS-y += dpdkbuild
 endif
 endif
 endif
+endif
 
 ifeq ($(OS),Windows)
 ifeq ($(CURDIR)/wpdk/build,$(CONFIG_WPDK_DIR))
@@ -94,12 +97,7 @@ endif
 all: mk/cc.mk $(DIRS-y)
 clean: $(DIRS-y)
 	$(Q)rm -f include/spdk/config.h
-	$(Q)rm -rf build/bin
-	$(Q)rm -rf build/fio
-	$(Q)rm -rf build/examples
-	$(Q)rm -rf build/include
-	$(Q)rm -rf build/lib/pkgconfig
-	$(Q)find build/lib ! -name .gitignore -type f -delete
+	$(Q)rm -rf build
 
 install: all
 	$(Q)echo "Installed to $(DESTDIR)$(CONFIG_PREFIX)"

README.md

@@ -18,7 +18,7 @@ The development kit currently includes:
 * [vhost target](http://www.spdk.io/doc/vhost.html)
 * [Virtio-SCSI driver](http://www.spdk.io/doc/virtio.html)
 
-# In this readme
+## In this readme
 
 * [Documentation](#documentation)
 * [Prerequisites](#prerequisites)
@@ -129,7 +129,9 @@ Boolean (on/off) options are configured with a 'y' (yes) or 'n' (no). For
 example, this line of `CONFIG` controls whether the optional RDMA (libibverbs)
 support is enabled:
 
-	CONFIG_RDMA?=n
+~~~{.sh}
+CONFIG_RDMA?=n
+~~~
 
 To enable RDMA, this line may be added to `mk/config.mk` with a 'y' instead of
 'n'. For the majority of options this can be done using the `configure` script.
@@ -221,6 +223,13 @@ configuring 8192MB memory.
 sudo HUGEMEM=8192 scripts/setup.sh
 ~~~
 
+There are a lot of other environment variables that can be set to configure
+setup.sh for advanced users. To see the full list, run:
+
+~~~{.sh}
+scripts/setup.sh --help
+~~~
+
 <a id="examples"></a>
 ## Example Code
 

app/iscsi_tgt/Makefile

@@ -43,7 +43,7 @@ CFLAGS += -I$(SPDK_ROOT_DIR)/lib
 
 C_SRCS := iscsi_tgt.c
 
-SPDK_LIB_LIST = $(ALL_MODULES_LIST) event_iscsi event_net
+SPDK_LIB_LIST = $(ALL_MODULES_LIST) event event_iscsi
 
 ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV))
 SPDK_LIB_LIST += env_dpdk_rpc

app/iscsi_tgt/iscsi_tgt.c

@@ -37,7 +37,6 @@
 #include "spdk/event.h"
 #include "iscsi/iscsi.h"
 #include "spdk/log.h"
-#include "spdk/net.h"
 
 static int g_daemon_mode = 0;
 

app/nvmf_tgt/Makefile

@@ -39,7 +39,7 @@ APP = nvmf_tgt
 
 C_SRCS := nvmf_main.c
 
-SPDK_LIB_LIST = $(ALL_MODULES_LIST) event_nvmf
+SPDK_LIB_LIST = $(ALL_MODULES_LIST) event event_nvmf
 
 ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV))
 SPDK_LIB_LIST += env_dpdk_rpc

app/spdk_dd/Makefile

@@ -39,6 +39,6 @@ APP = spdk_dd
 
 C_SRCS := spdk_dd.c
 
-SPDK_LIB_LIST = $(ALL_MODULES_LIST) event_bdev
+SPDK_LIB_LIST = $(ALL_MODULES_LIST) event event_bdev
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

app/spdk_dd/spdk_dd.c

@@ -227,7 +227,7 @@ dd_show_progress(uint64_t offset, uint64_t length, bool finish)
 		return;
 	}
 
-	/* Find the rigth unit for size displaying (B vs kB vs MB vs GB vs TB) */
+	/* Find the right unit for size displaying (B vs kB vs MB vs GB vs TB) */
 	while (size > 1024 * 10) {
 		size >>= 10;
 		size_unit <<= 10;
@@ -247,7 +247,7 @@ dd_show_progress(uint64_t offset, uint64_t length, bool finish)
 
 	i = 0;
 
-	/* Find the rigth unit for speed displaying (Bps vs kBps vs MBps vs GBps vs TBps) */
+	/* Find the right unit for speed displaying (Bps vs kBps vs MBps vs GBps vs TBps) */
 	while (tmp_speed > 1024) {
 		tmp_speed >>= 10;
 		speed_unit <<= 10;
@@ -1000,7 +1000,7 @@ usage(void)
 	printf(" --of Output file. Must specify either --of or --ob.\n");
 	printf(" --ob Output bdev. Must specify either --of or --ob.\n");
 	printf(" --iflag Input file flags.\n");
-	printf(" --oflag Onput file flags.\n");
+	printf(" --oflag Output file flags.\n");
 	printf(" --bs I/O unit size (default: %" PRId64 ")\n", g_opts.io_unit_size);
 	printf(" --qd Queue depth (default: %d)\n", g_opts.queue_depth);
 	printf(" --count I/O unit count. The number of I/O units to copy. (default: all)\n");
@@ -1167,7 +1167,7 @@ main(int argc, char **argv)
 
 	rc = spdk_app_start(&opts, dd_run, NULL);
 	if (rc) {
-		SPDK_ERRLOG("Error occured while performing copy\n");
+		SPDK_ERRLOG("Error occurred while performing copy\n");
 	}
 
 	dd_free();

app/spdk_lspci/spdk_lspci.c

@@ -49,7 +49,7 @@ pci_enum_cb(void *ctx, struct spdk_pci_device *dev)
 }
 
 static void
-print_pci_dev(struct spdk_pci_device *dev)
+print_pci_dev(void *ctx, struct spdk_pci_device *dev)
 {
 	struct spdk_pci_addr pci_addr = spdk_pci_device_get_addr(dev);
 	char addr[32] = { 0 };
@@ -74,9 +74,8 @@ print_pci_dev(struct spdk_pci_device *dev)
 int
 main(int argc, char **argv)
 {
-	int op;
+	int op, rc = 0;
 	struct spdk_env_opts opts;
-	struct spdk_pci_device *dev;
 
 	while ((op = getopt(argc, argv, "h")) != -1) {
 		switch (op) {
@@ -103,21 +102,16 @@ main(int argc, char **argv)
 
 	if (spdk_pci_enumerate(spdk_pci_nvme_get_driver(), pci_enum_cb, NULL)) {
 		printf("Unable to enumerate PCI nvme driver\n");
-		return 1;
-	}
-
-	dev = spdk_pci_get_first_device();
-	if (!dev) {
-		printf("\nLack of PCI devices available for SPDK!\n");
+		rc = 1;
+		goto exit;
 	}
 
 	printf("\nList of available PCI devices:\n");
-	while (dev) {
-		print_pci_dev(dev);
-		dev = spdk_pci_get_next_device(dev);
-	}
+	spdk_pci_for_each_device(NULL, print_pci_dev);
 
+exit:
 	spdk_vmd_fini();
+	spdk_env_fini();
 
-	return 0;
+	return rc;
 }

app/spdk_tgt/Makefile

@@ -41,7 +41,7 @@ C_SRCS := spdk_tgt.c
 
 SPDK_LIB_LIST = $(ALL_MODULES_LIST)
 
-SPDK_LIB_LIST += event_iscsi event_nvmf
+SPDK_LIB_LIST += event event_iscsi event_nvmf
 
 ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV))
 SPDK_LIB_LIST += env_dpdk_rpc

app/spdk_top/Makefile

@@ -39,6 +39,6 @@ APP = spdk_top
 C_SRCS := spdk_top.c
 
 SPDK_LIB_LIST = rpc
-LIBS=-lncurses -lpanel -lmenu
+LIBS=-lpanel -lmenu -lncurses
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

app/trace/Makefile

@@ -38,6 +38,8 @@ include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
 APP = spdk_trace
 SPDK_NO_LINK_ENV = 1
 
+SPDK_LIB_LIST += json trace_parser
+
 CXX_SRCS := trace.cpp
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app_cxx.mk

app/trace/trace.cpp

@@ -32,60 +32,47 @@
  */
 
 #include "spdk/stdinc.h"
+#include "spdk/env.h"
+#include "spdk/json.h"
+#include "spdk/likely.h"
+#include "spdk/string.h"
+#include "spdk/util.h"
 
 #include <map>
 
 extern "C" {
-#include "spdk/trace.h"
+#include "spdk/trace_parser.h"
 #include "spdk/util.h"
 }
 
-static struct spdk_trace_histories *g_histories;
+static struct spdk_trace_parser *g_parser;
+static const struct spdk_trace_flags *g_flags;
+static struct spdk_json_write_ctx *g_json;
 static bool g_print_tsc = false;
 
+/* This is a bit ugly, but we don't want to include env_dpdk in the app, while spdk_util, which we
+ * do need, uses some of the functions implemented there.  We're not actually using the functions
+ * that depend on those, so just define them as no-ops to allow the app to link.
+ */
+extern "C" {
+	void *
+	spdk_realloc(void *buf, size_t size, size_t align)
+	{
+		assert(false);
+
+		return NULL;
+	}
+
+	void
+	spdk_free(void *buf)
+	{
+		assert(false);
+	}
+} /* extern "C" */
+
 static void usage(void);
 
-struct entry_key {
-	entry_key(uint16_t _lcore, uint64_t _tsc) : lcore(_lcore), tsc(_tsc) {}
-	uint16_t lcore;
-	uint64_t tsc;
-};
-
-class compare_entry_key
-{
-public:
-	bool operator()(const entry_key &first, const entry_key &second) const
-	{
-		if (first.tsc == second.tsc) {
-			return first.lcore < second.lcore;
-		} else {
-			return first.tsc < second.tsc;
-		}
-	}
-};
-
-typedef std::map<entry_key, spdk_trace_entry *, compare_entry_key> entry_map;
-
-entry_map g_entry_map;
-
-struct object_stats {
-
-	std::map<uint64_t, uint64_t> start;
-	std::map<uint64_t, uint64_t> index;
-	std::map<uint64_t, uint64_t> size;
-	std::map<uint64_t, uint64_t> tpoint_id;
-	uint64_t counter;
-
-	object_stats() : start(), index(), size(), tpoint_id(), counter(0) {}
-};
-
-struct object_stats g_stats[SPDK_TRACE_MAX_OBJECT];
-
 static char *g_exe_name;
-static int g_verbose = 1;
-
-static uint64_t g_tsc_rate;
-static uint64_t g_first_tsc = 0x0;
 
 static float
 get_us_from_tsc(uint64_t tsc, uint64_t tsc_rate)
@@ -93,10 +80,19 @@ get_us_from_tsc(uint64_t tsc, uint64_t tsc_rate)
 	return ((float)tsc) * 1000 * 1000 / tsc_rate;
 }
 
+static const char *
+format_argname(const char *name)
+{
+	static char namebuf[16];
+
+	snprintf(namebuf, sizeof(namebuf), "%s: ", name);
+	return namebuf;
+}
+
 static void
 print_ptr(const char *arg_string, uint64_t arg)
 {
-	printf("%-7.7s0x%-14jx ", arg_string, arg);
+	printf("%-7.7s0x%-14jx ", format_argname(arg_string), arg);
 }
 
 static void
@@ -107,14 +103,13 @@ print_uint64(const char *arg_string, uint64_t arg)
 	 *  for FLUSH WRITEBUF when writev() returns -1 due to full
 	 *  socket buffer.
 	 */
-	printf("%-7.7s%-16jd ", arg_string, arg);
+	printf("%-7.7s%-16jd ", format_argname(arg_string), arg);
 }
 
 static void
-print_string(const char *arg_string, uint64_t arg)
+print_string(const char *arg_string, const char *arg)
 {
-	char *str = (char *)&arg;
-	printf("%-7.7s%.8s ", arg_string, str);
+	printf("%-7.7s%-16.16s ", format_argname(arg_string), arg);
 }
 
 static void
@@ -128,64 +123,46 @@ print_size(uint32_t size)
 }
 
 static void
-print_object_id(uint8_t type, uint64_t id)
+print_object_id(const struct spdk_trace_tpoint *d, struct spdk_trace_parser_entry *entry)
 {
-	printf("id:    %c%-15jd ", g_histories->flags.object[type].id_prefix, id);
+	/* Set size to 128 and 256 bytes to make sure we can fit all the characters we need */
+	char related_id[128] = {'\0'};
+	char ids[256] = {'\0'};
+
+	if (entry->related_type != OBJECT_NONE) {
+		snprintf(related_id, sizeof(related_id), " (%c%jd)",
+			 g_flags->object[entry->related_type].id_prefix,
+			 entry->related_index);
+	}
+
+	snprintf(ids, sizeof(ids), "%c%jd%s", g_flags->object[d->object_type].id_prefix,
+		 entry->object_index, related_id);
+	printf("id:    %-17s", ids);
 }
 
 static void
 print_float(const char *arg_string, float arg)
 {
-	printf("%-7s%-16.3f ", arg_string, arg);
+	printf("%-7s%-16.3f ", format_argname(arg_string), arg);
 }
 
 static void
-print_arg(uint8_t arg_type, const char *arg_string, uint64_t arg)
+print_event(struct spdk_trace_parser_entry *entry, uint64_t tsc_rate, uint64_t tsc_offset)
 {
-	if (arg_string[0] == 0) {
-		printf("%24s", "");
-		return;
-	}
-
-	switch (arg_type) {
-	case SPDK_TRACE_ARG_TYPE_PTR:
-		print_ptr(arg_string, arg);
-		break;
-	case SPDK_TRACE_ARG_TYPE_INT:
-		print_uint64(arg_string, arg);
-		break;
-	case SPDK_TRACE_ARG_TYPE_STR:
-		print_string(arg_string, arg);
-		break;
-	}
-}
-
-static void
-print_event(struct spdk_trace_entry *e, uint64_t tsc_rate,
-	    uint64_t tsc_offset, uint16_t lcore)
-{
-	struct spdk_trace_tpoint	*d;
-	struct object_stats		*stats;
+	struct spdk_trace_entry		*e = entry->entry;
+	const struct spdk_trace_tpoint	*d;
 	float				us;
+	size_t				i;
 
-	d = &g_histories->flags.tpoint[e->tpoint_id];
-	stats = &g_stats[d->object_type];
-
-	if (d->new_object) {
-		stats->index[e->object_id] = stats->counter++;
-		stats->tpoint_id[e->object_id] = e->tpoint_id;
-		stats->start[e->object_id] = e->tsc;
-		stats->size[e->object_id] = e->size;
-	}
-
+	d = &g_flags->tpoint[e->tpoint_id];
 	us = get_us_from_tsc(e->tsc - tsc_offset, tsc_rate);
 
-	printf("%2d: %10.3f ", lcore, us);
+	printf("%2d: %10.3f ", entry->lcore, us);
 	if (g_print_tsc) {
 		printf("(%9ju) ", e->tsc - tsc_offset);
 	}
-	if (g_histories->flags.owner[d->owner_type].id_prefix) {
-		printf("%c%02d ", g_histories->flags.owner[d->owner_type].id_prefix, e->poller_id);
+	if (g_flags->owner[d->owner_type].id_prefix) {
+		printf("%c%02d ", g_flags->owner[d->owner_type].id_prefix, e->poller_id);
 	} else {
 		printf("%4s", " ");
 	}
@@ -193,94 +170,180 @@ print_event(struct spdk_trace_entry *e, uint64_t tsc_rate,
 	printf("%-*s ", (int)sizeof(d->name), d->name);
 	print_size(e->size);
 
-	print_arg(d->arg1_type, d->arg1_name, e->arg1);
 	if (d->new_object) {
-		print_object_id(d->object_type, stats->index[e->object_id]);
+		print_object_id(d, entry);
 	} else if (d->object_type != OBJECT_NONE) {
-		if (stats->start.find(e->object_id) != stats->start.end()) {
-			us = get_us_from_tsc(e->tsc - stats->start[e->object_id],
-					     tsc_rate);
-			print_object_id(d->object_type, stats->index[e->object_id]);
-			print_float("time:", us);
+		if (entry->object_index != UINT64_MAX) {
+			us = get_us_from_tsc(e->tsc - entry->object_start, tsc_rate);
+			print_object_id(d, entry);
+			print_float("time", us);
 		} else {
 			printf("id:    N/A");
 		}
 	} else if (e->object_id != 0) {
-		print_arg(SPDK_TRACE_ARG_TYPE_PTR, "object: ", e->object_id);
+		print_ptr("object", e->object_id);
+	}
+
+	for (i = 0; i < d->num_args; ++i) {
+		switch (d->args[i].type) {
+		case SPDK_TRACE_ARG_TYPE_PTR:
+			print_ptr(d->args[i].name, (uint64_t)entry->args[i].pointer);
+			break;
+		case SPDK_TRACE_ARG_TYPE_INT:
+			print_uint64(d->args[i].name, entry->args[i].integer);
+			break;
+		case SPDK_TRACE_ARG_TYPE_STR:
+			print_string(d->args[i].name, entry->args[i].string);
+			break;
+		}
 	}
 	printf("\n");
 }
 
 static void
-process_event(struct spdk_trace_entry *e, uint64_t tsc_rate,
-	      uint64_t tsc_offset, uint16_t lcore)
+print_event_json(struct spdk_trace_parser_entry *entry, uint64_t tsc_rate, uint64_t tsc_offset)
 {
-	if (g_verbose) {
-		print_event(e, tsc_rate, tsc_offset, lcore);
+	struct spdk_trace_entry *e = entry->entry;
+	const struct spdk_trace_tpoint *d;
+	size_t i;
+
+	d = &g_flags->tpoint[e->tpoint_id];
+
+	spdk_json_write_object_begin(g_json);
+	spdk_json_write_named_uint64(g_json, "lcore", entry->lcore);
+	spdk_json_write_named_uint64(g_json, "tpoint", e->tpoint_id);
+	spdk_json_write_named_uint64(g_json, "tsc", e->tsc);
+
+	if (g_flags->owner[d->owner_type].id_prefix) {
+		spdk_json_write_named_string_fmt(g_json, "poller", "%c%02d",
+						 g_flags->owner[d->owner_type].id_prefix,
+						 e->poller_id);
+	}
+	if (e->size != 0) {
+		spdk_json_write_named_uint32(g_json, "size", e->size);
+	}
+	if (d->new_object || d->object_type != OBJECT_NONE || e->object_id != 0) {
+		char object_type;
+
+		spdk_json_write_named_object_begin(g_json, "object");
+		if (d->new_object) {
+			object_type =  g_flags->object[d->object_type].id_prefix;
+			spdk_json_write_named_string_fmt(g_json, "id", "%c%" PRIu64, object_type,
+							 entry->object_index);
+		} else if (d->object_type != OBJECT_NONE) {
+			object_type =  g_flags->object[d->object_type].id_prefix;
+			if (entry->object_index != UINT64_MAX) {
+				spdk_json_write_named_string_fmt(g_json, "id", "%c%" PRIu64,
+								 object_type,
+								 entry->object_index);
+				spdk_json_write_named_uint64(g_json, "time",
+							     e->tsc - entry->object_start);
+			}
+		}
+		spdk_json_write_named_uint64(g_json, "value", e->object_id);
+		spdk_json_write_object_end(g_json);
+	}
+
+	/* Print related objects array */
+	if (entry->related_index != UINT64_MAX) {
+		spdk_json_write_named_string_fmt(g_json, "related", "%c%" PRIu64,
+						 g_flags->object[entry->related_type].id_prefix,
+						 entry->related_index);
+	}
+
+	if (d->num_args > 0) {
+		spdk_json_write_named_array_begin(g_json, "args");
+		for (i = 0; i < d->num_args; ++i) {
+			switch (d->args[i].type) {
+			case SPDK_TRACE_ARG_TYPE_PTR:
+				spdk_json_write_uint64(g_json, (uint64_t)entry->args[i].pointer);
+				break;
+			case SPDK_TRACE_ARG_TYPE_INT:
+				spdk_json_write_uint64(g_json, entry->args[i].integer);
+				break;
+			case SPDK_TRACE_ARG_TYPE_STR:
+				spdk_json_write_string(g_json, entry->args[i].string);
+				break;
+			}
+		}
+		spdk_json_write_array_end(g_json);
+	}
+
+	spdk_json_write_object_end(g_json);
+}
+
+static void
+process_event(struct spdk_trace_parser_entry *e, uint64_t tsc_rate, uint64_t tsc_offset)
+{
+	if (g_json == NULL) {
+		print_event(e, tsc_rate, tsc_offset);
+	} else {
+		print_event_json(e, tsc_rate, tsc_offset);
 	}
 }
 
-static int
-populate_events(struct spdk_trace_history *history, int num_entries)
+static void
+print_tpoint_definitions(void)
 {
-	int i, num_entries_filled;
-	struct spdk_trace_entry *e;
-	int first, last, lcore;
+	const struct spdk_trace_tpoint *tpoint;
+	size_t i, j;
 
-	lcore = history->lcore;
-
-	e = history->entries;
-
-	num_entries_filled = num_entries;
-	while (e[num_entries_filled - 1].tsc == 0) {
-		num_entries_filled--;
+	/* We only care about these when printing JSON */
+	if (!g_json) {
+		return;
 	}
 
-	if (num_entries == num_entries_filled) {
-		first = last = 0;
-		for (i = 1; i < num_entries; i++) {
-			if (e[i].tsc < e[first].tsc) {
-				first = i;
-			}
-			if (e[i].tsc > e[last].tsc) {
-				last = i;
-			}
+	spdk_json_write_named_uint64(g_json, "tsc_rate", g_flags->tsc_rate);
+	spdk_json_write_named_array_begin(g_json, "tpoints");
+
+	for (i = 0; i < SPDK_COUNTOF(g_flags->tpoint); ++i) {
+		tpoint = &g_flags->tpoint[i];
+		if (tpoint->tpoint_id == 0) {
+			continue;
 		}
-	} else {
-		first = 0;
-		last = num_entries_filled - 1;
-	}
 
-	/*
-	 * We keep track of the highest first TSC out of all reactors.
-	 *  We will ignore any events that occured before this TSC on any
-	 *  other reactors.  This will ensure we only print data for the
-	 *  subset of time where we have data across all reactors.
-	 */
-	if (e[first].tsc > g_first_tsc) {
-		g_first_tsc = e[first].tsc;
-	}
+		spdk_json_write_object_begin(g_json);
+		spdk_json_write_named_string(g_json, "name", tpoint->name);
+		spdk_json_write_named_uint32(g_json, "id", tpoint->tpoint_id);
+		spdk_json_write_named_bool(g_json, "new_object", tpoint->new_object);
 
-	i = first;
-	while (1) {
-		g_entry_map[entry_key(lcore, e[i].tsc)] = &e[i];
-		if (i == last) {
-			break;
-		}
-		i++;
-		if (i == num_entries_filled) {
-			i = 0;
+		spdk_json_write_named_array_begin(g_json, "args");
+		for (j = 0; j < tpoint->num_args; ++j) {
+			spdk_json_write_object_begin(g_json);
+			spdk_json_write_named_string(g_json, "name", tpoint->args[j].name);
+			spdk_json_write_named_uint32(g_json, "type", tpoint->args[j].type);
+			spdk_json_write_named_uint32(g_json, "size", tpoint->args[j].size);
+			spdk_json_write_object_end(g_json);
 		}
+		spdk_json_write_array_end(g_json);
+		spdk_json_write_object_end(g_json);
 	}
 
-	return (0);
+	spdk_json_write_array_end(g_json);
+}
+
+static int
+print_json(void *cb_ctx, const void *data, size_t size)
+{
+	ssize_t rc;
+
+	while (size > 0) {
+		rc = write(STDOUT_FILENO, data, size);
+		if (rc < 0) {
+			fprintf(stderr, "%s: %s\n", g_exe_name, spdk_strerror(errno));
+			abort();
+		}
+
+		size -= rc;
+	}
+
+	return 0;
 }
 
 static void usage(void)
 {
 	fprintf(stderr, "usage:\n");
 	fprintf(stderr, "   %s <option> <lcore#>\n", g_exe_name);
-	fprintf(stderr, "        option = '-q' to disable verbose mode\n");
 	fprintf(stderr, "                 '-c' to display single lcore history\n");
 	fprintf(stderr, "                 '-t' to display TSC offset for each event\n");
 	fprintf(stderr, "                 '-s' to specify spdk_trace shm name for a\n");
@@ -291,25 +354,24 @@ static void usage(void)
 	fprintf(stderr, "                       -i or -p must be specified)\n");
 	fprintf(stderr, "                 '-f' to specify a tracepoint file name\n");
 	fprintf(stderr, "                      (-s and -f are mutually exclusive)\n");
+	fprintf(stderr, "                 '-j' to use JSON to format the output\n");
 }
 
 int main(int argc, char **argv)
 {
-	void			*history_ptr;
-	struct spdk_trace_history *history;
-	int			fd, i, rc;
-	int			lcore = SPDK_TRACE_MAX_LCORE;
-	uint64_t		tsc_offset;
-	const char		*app_name = NULL;
-	const char		*file_name = NULL;
-	int			op;
-	char			shm_name[64];
-	int			shm_id = -1, shm_pid = -1;
-	uint64_t		trace_histories_size;
-	struct stat		_stat;
+	struct spdk_trace_parser_opts	opts;
+	struct spdk_trace_parser_entry	entry;
+	int				lcore = SPDK_TRACE_MAX_LCORE;
+	uint64_t			tsc_offset, entry_count;
+	const char			*app_name = NULL;
+	const char			*file_name = NULL;
+	int				op, i;
+	char				shm_name[64];
+	int				shm_id = -1, shm_pid = -1;
+	bool				json = false;
 
 	g_exe_name = argv[0];
-	while ((op = getopt(argc, argv, "c:f:i:p:qs:t")) != -1) {
+	while ((op = getopt(argc, argv, "c:f:i:jp:s:t")) != -1) {
 		switch (op) {
 		case 'c':
 			lcore = atoi(optarg);
@@ -326,9 +388,6 @@ int main(int argc, char **argv)
 		case 'p':
 			shm_pid = atoi(optarg);
 			break;
-		case 'q':
-			g_verbose = 0;
-			break;
 		case 's':
 			app_name = optarg;
 			break;
@@ -338,6 +397,9 @@ int main(int argc, char **argv)
 		case 't':
 			g_print_tsc = true;
 			break;
+		case 'j':
+			json = true;
+			break;
 		default:
 			usage();
 			exit(1);
@@ -356,107 +418,65 @@ int main(int argc, char **argv)
 		exit(1);
 	}
 
-	if (file_name) {
-		fd = open(file_name, O_RDONLY);
-	} else {
+	if (json) {
+		g_json = spdk_json_write_begin(print_json, NULL, 0);
+		if (g_json == NULL) {
+			fprintf(stderr, "Failed to allocate JSON write context\n");
+			exit(1);
+		}
+	}
+
+	if (!file_name) {
 		if (shm_id >= 0) {
 			snprintf(shm_name, sizeof(shm_name), "/%s_trace.%d", app_name, shm_id);
 		} else {
 			snprintf(shm_name, sizeof(shm_name), "/%s_trace.pid%d", app_name, shm_pid);
 		}
-		fd = shm_open(shm_name, O_RDONLY, 0600);
 		file_name = shm_name;
 	}
-	if (fd < 0) {
-		fprintf(stderr, "Could not open %s.\n", file_name);
-		usage();
-		exit(-1);
+
+	opts.filename = file_name;
+	opts.lcore = lcore;
+	opts.mode = app_name == NULL ? SPDK_TRACE_PARSER_MODE_FILE : SPDK_TRACE_PARSER_MODE_SHM;
+	g_parser = spdk_trace_parser_init(&opts);
+	if (g_parser == NULL) {
+		fprintf(stderr, "Failed to initialize trace parser\n");
+		exit(1);
 	}
 
-	rc = fstat(fd, &_stat);
-	if (rc < 0) {
-		fprintf(stderr, "Could not get size of %s.\n", file_name);
-		usage();
-		exit(-1);
-	}
-	if ((size_t)_stat.st_size < sizeof(*g_histories)) {
-		fprintf(stderr, "%s is not a valid trace file\n", file_name);
-		usage();
-		exit(-1);
-	}
-
-	/* Map the header of trace file */
-	history_ptr = mmap(NULL, sizeof(*g_histories), PROT_READ, MAP_SHARED, fd, 0);
-	if (history_ptr == MAP_FAILED) {
-		fprintf(stderr, "Could not mmap %s.\n", file_name);
-		usage();
-		exit(-1);
-	}
-
-	g_histories = (struct spdk_trace_histories *)history_ptr;
-
-	g_tsc_rate = g_histories->flags.tsc_rate;
-	if (g_tsc_rate == 0) {
-		fprintf(stderr, "Invalid tsc_rate %ju\n", g_tsc_rate);
-		usage();
-		exit(-1);
-	}
-
-	if (g_verbose) {
-		printf("TSC Rate: %ju\n", g_tsc_rate);
-	}
-
-	/* Remap the entire trace file */
-	trace_histories_size = spdk_get_trace_histories_size(g_histories);
-	munmap(history_ptr, sizeof(*g_histories));
-	if ((size_t)_stat.st_size < trace_histories_size) {
-		fprintf(stderr, "%s is not a valid trace file\n", file_name);
-		usage();
-		exit(-1);
-	}
-	history_ptr = mmap(NULL, trace_histories_size, PROT_READ, MAP_SHARED, fd, 0);
-	if (history_ptr == MAP_FAILED) {
-		fprintf(stderr, "Could not mmap %s.\n", file_name);
-		usage();
-		exit(-1);
-	}
-
-	g_histories = (struct spdk_trace_histories *)history_ptr;
-
-	if (lcore == SPDK_TRACE_MAX_LCORE) {
-		for (i = 0; i < SPDK_TRACE_MAX_LCORE; i++) {
-			history = spdk_get_per_lcore_history(g_histories, i);
-			if (history->num_entries == 0 || history->entries[0].tsc == 0) {
-				continue;
-			}
-
-			if (g_verbose && history->num_entries) {
-				printf("Trace Size of lcore (%d): %ju\n", i, history->num_entries);
-			}
-
-			populate_events(history, history->num_entries);
-		}
+	g_flags = spdk_trace_parser_get_flags(g_parser);
+	if (!g_json) {
+		printf("TSC Rate: %ju\n", g_flags->tsc_rate);
 	} else {
-		history = spdk_get_per_lcore_history(g_histories, lcore);
-		if (history->num_entries > 0 && history->entries[0].tsc != 0) {
-			if (g_verbose && history->num_entries) {
-				printf("Trace Size of lcore (%d): %ju\n", lcore, history->num_entries);
-			}
+		spdk_json_write_object_begin(g_json);
+		print_tpoint_definitions();
+		spdk_json_write_named_array_begin(g_json, "entries");
+	}
 
-			populate_events(history, history->num_entries);
+	for (i = 0; i < SPDK_TRACE_MAX_LCORE; ++i) {
+		if (lcore == SPDK_TRACE_MAX_LCORE || i == lcore) {
+			entry_count = spdk_trace_parser_get_entry_count(g_parser, i);
+			if (entry_count > 0) {
+				printf("Trace Size of lcore (%d): %ju\n", i, entry_count);
+			}
 		}
 	}
 
-	tsc_offset = g_first_tsc;
-	for (entry_map::iterator it = g_entry_map.begin(); it != g_entry_map.end(); it++) {
-		if (it->first.tsc < g_first_tsc) {
+	tsc_offset = spdk_trace_parser_get_tsc_offset(g_parser);
+	while (spdk_trace_parser_next_entry(g_parser, &entry)) {
+		if (entry.entry->tsc < tsc_offset) {
 			continue;
 		}
-		process_event(it->second, g_tsc_rate, tsc_offset, it->first.lcore);
+		process_event(&entry, g_flags->tsc_rate, tsc_offset);
 	}
 
-	munmap(history_ptr, trace_histories_size);
-	close(fd);
+	if (g_json != NULL) {
+		spdk_json_write_array_end(g_json);
+		spdk_json_write_object_end(g_json);
+		spdk_json_write_end(g_json);
+	}
+
+	spdk_trace_parser_cleanup(g_parser);
 
 	return (0);
 }

app/vhost/Makefile

@@ -39,7 +39,7 @@ APP = vhost
 
 C_SRCS := vhost.c
 
-SPDK_LIB_LIST = $(ALL_MODULES_LIST) event_vhost event_nbd
+SPDK_LIB_LIST = $(ALL_MODULES_LIST) event event_vhost event_nbd
 
 ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV))
 SPDK_LIB_LIST += env_dpdk_rpc

autobuild.sh

@@ -14,6 +14,10 @@ source "$1"
 source "$rootdir/test/common/autotest_common.sh"
 source "$rootdir/scripts/common.sh"
 
+if [[ -n $EXTERNAL_MAKE_HUGEMEM ]]; then
+	export EXTERNAL_MAKE_HUGEMEM
+fi
+
 out=$output_dir
 if [ -n "$SPDK_TEST_NATIVE_DPDK" ]; then
 	scanbuild_exclude=" --exclude $(dirname $SPDK_RUN_EXTERNAL_DPDK)"
@@ -41,9 +45,9 @@ function ocf_precompile() {
 	# So we precompile OCF now for further use as standalone static library
 	./configure $(echo $config_params | sed 's/--enable-coverage//g')
 	$MAKE $MAKEFLAGS include/spdk/config.h
-	CC=gcc CCAR=ar $MAKE $MAKEFLAGS -C lib/env_ocf exportlib O=$rootdir/build/ocf.a
+	CC=gcc CCAR=ar $MAKE $MAKEFLAGS -C lib/env_ocf exportlib O=$rootdir/ocf.a
 	# Set config to use precompiled library
-	config_params="$config_params --with-ocf=/$rootdir/build/ocf.a"
+	config_params="$config_params --with-ocf=/$rootdir/ocf.a"
 	# need to reconfigure to avoid clearing ocf related files on future make clean.
 	./configure $config_params
 }
@@ -51,7 +55,25 @@ function ocf_precompile() {
 function build_native_dpdk() {
 	local external_dpdk_dir
 	local external_dpdk_base_dir
+	local compiler_version
+	local compiler
+	local dpdk_kmods
 
+	compiler=${CC:-gcc}
+
+	# Export CC to be absolutely sure it's set.
+	# If CC was not set and we defaulted to "gcc" then we need to do the export
+	# so that "meson build" command a few lines below is aware of which compiler
+	# to use.
+	export CC="$compiler"
+
+	if [[ $compiler != *clang* && $compiler != *gcc* ]]; then
+		echo "Unsupported compiler detected ($compiler), failing the test" >&2
+		return 1
+	fi
+
+	compiler_version=$("$compiler" -dumpversion)
+	compiler_version=${compiler_version%%.*}
 	external_dpdk_dir="$SPDK_RUN_EXTERNAL_DPDK"
 	external_dpdk_base_dir="$(dirname $external_dpdk_dir)"
 
@@ -65,28 +87,58 @@ function build_native_dpdk() {
 	git clone --branch $SPDK_TEST_NATIVE_DPDK --depth 1 http://dpdk.org/git/dpdk "$external_dpdk_base_dir"
 	git -C "$external_dpdk_base_dir" log --oneline -n 5
 
-	dpdk_cflags="-fPIC -g -Werror -fcommon"
+	dpdk_cflags="-fPIC -g -fcommon"
 	dpdk_ldflags=""
+	dpdk_ver=$(< "$external_dpdk_base_dir/VERSION")
+
+	if [[ $compiler == *gcc* && $compiler_version -ge 5 ]]; then
+		dpdk_cflags+=" -Werror"
+	fi
+
+	if [[ $compiler == *gcc* && $compiler_version -ge 10 ]]; then
+		dpdk_cflags+=" -Wno-stringop-overflow"
+	fi
 
 	# the drivers we use
 	# net/i40e driver is not really needed by us, but it's built as a workaround
 	# for DPDK issue: https://bugs.dpdk.org/show_bug.cgi?id=576
 	DPDK_DRIVERS=("bus" "bus/pci" "bus/vdev" "mempool/ring" "net/i40e" "net/i40e/base")
-	# all possible DPDK drivers
-	DPDK_ALL_DRIVERS=($(find "$external_dpdk_base_dir/drivers" -mindepth 1 -type d | sed -n "s#^$external_dpdk_base_dir/drivers/##p"))
 
+	local mlx5_libs_added="n"
 	if [[ "$SPDK_TEST_CRYPTO" -eq 1 ]]; then
-		git clone --branch v0.54 --depth 1 https://github.com/intel/intel-ipsec-mb.git "$external_dpdk_base_dir/intel-ipsec-mb"
+		intel_ipsec_mb_ver=v0.54
+		intel_ipsec_mb_drv=crypto/aesni_mb
+		intel_ipsec_lib=""
+		if ge "$dpdk_ver" 21.11.0; then
+			# Minimum supported version of intel-ipsec-mb, for DPDK >= 21.11, is 1.0.
+			# Source of the aesni_mb driver was moved to ipsec_mb. .{h,so,a} were moved
+			# to ./lib.
+			# https://github.com/dpdk/dpdk/commit/918fd2f1466b0e3b21a033df7012a77a83665582.
+			intel_ipsec_mb_ver=v1.0
+			intel_ipsec_mb_drv=crypto/ipsec_mb
+			intel_ipsec_lib=lib
+		fi
+		git clone --branch "$intel_ipsec_mb_ver" --depth 1 https://github.com/intel/intel-ipsec-mb.git "$external_dpdk_base_dir/intel-ipsec-mb"
 		cd "$external_dpdk_base_dir/intel-ipsec-mb"
 		$MAKE $MAKEFLAGS all SHARED=y EXTRA_CFLAGS=-fPIC
 		DPDK_DRIVERS+=("crypto")
-		DPDK_DRIVERS+=("crypto/aesni_mb")
+		DPDK_DRIVERS+=("$intel_ipsec_mb_drv")
 		DPDK_DRIVERS+=("crypto/qat")
 		DPDK_DRIVERS+=("compress/qat")
 		DPDK_DRIVERS+=("common/qat")
-		dpdk_cflags+=" -I$external_dpdk_base_dir/intel-ipsec-mb"
-		dpdk_ldflags+=" -L$external_dpdk_base_dir/intel-ipsec-mb"
-		export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$external_dpdk_base_dir/intel-ipsec-mb
+		# 22.03.0 is version of DPDK with stable support for mlx5 crypto.
+		if ge "$dpdk_ver" 22.03.0; then
+			# SPDK enables CRYPTO_MLX in case supported version of DPDK is detected
+			# so make sure proper libs are built.
+			DPDK_DRIVERS+=("bus/auxiliary")
+			DPDK_DRIVERS+=("common/mlx5")
+			DPDK_DRIVERS+=("common/mlx5/linux")
+			DPDK_DRIVERS+=("crypto/mlx5")
+			mlx5_libs_added="y"
+		fi
+		dpdk_cflags+=" -I$external_dpdk_base_dir/intel-ipsec-mb/$intel_ipsec_lib"
+		dpdk_ldflags+=" -L$external_dpdk_base_dir/intel-ipsec-mb/$intel_ipsec_lib"
+		export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$external_dpdk_base_dir/intel-ipsec-mb/$intel_ipsec_lib"
 	fi
 
 	if [[ "$SPDK_TEST_REDUCE" -eq 1 ]]; then
@@ -103,39 +155,41 @@ function build_native_dpdk() {
 		DPDK_DRIVERS+=("compress/isal")
 		DPDK_DRIVERS+=("compress/qat")
 		DPDK_DRIVERS+=("common/qat")
+		if ge "$dpdk_ver" 21.02.0; then
+			# SPDK enables REDUCE_MLX in case supported version of DPDK is detected
+			# so make sure proper libs are built.
+			if test $mlx5_libs_added = "n"; then
+				DPDK_DRIVERS+=("bus/auxiliary")
+				DPDK_DRIVERS+=("common/mlx5")
+				DPDK_DRIVERS+=("common/mlx5/linux")
+			fi
+			DPDK_DRIVERS+=("compress/mlx5")
+		fi
 		export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$isal_dir/build/lib/pkgconfig"
 		export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$isal_dir/build/lib"
 	fi
 
-	# Use difference between DPDK_ALL_DRIVERS and DPDK_DRIVERS as a set of DPDK drivers we don't want or
-	# don't need to build.
-	DPDK_DISABLED_DRIVERS=($(sort <(printf "%s\n" "${DPDK_DRIVERS[@]}") <(printf "%s\n" "${DPDK_ALL_DRIVERS[@]}") | uniq -u))
-
 	cd $external_dpdk_base_dir
 	if [ "$(uname -s)" = "Linux" ]; then
-		dpdk_cflags+=" -Wno-stringop-overflow"
-		# Fix for freeing device if not kernel driver configured.
-		# TODO: Remove once this is merged in upstream DPDK
-		if grep "20.08.0" $external_dpdk_base_dir/VERSION; then
-			wget https://github.com/spdk/dpdk/commit/64f1ced13f974e8b3d46b87c361a09eca68126f9.patch -O dpdk-pci.patch
-			wget https://github.com/spdk/dpdk/commit/c2c273d5c8fbf673623b427f8f4ab5af5ddf0e08.patch -O dpdk-qat.patch
-		elif grep "20.11\|21.02" $external_dpdk_base_dir/VERSION; then
-			wget https://github.com/karlatec/dpdk/commit/3219c0cfc38803aec10c809dde16e013b370bda9.patch -O dpdk-pci.patch
-			wget https://github.com/karlatec/dpdk/commit/adf8f7638de29bc4bf9ba3faf12bbdae73acda0c.patch -O dpdk-qat.patch
-		else
-			wget https://github.com/karlatec/dpdk/commit/eac05db0580091ef8e4d338aa5d2210695521894.patch -O dpdk-pci.patch
-			wget https://github.com/karlatec/dpdk/commit/d649d5efb7bb404ce59dea81768adeb994b284f7.patch -O dpdk-qat.patch
+		if [[ $dpdk_ver == 20.11* ]]; then
+			patch -p1 < "$rootdir/test/common/config/pkgdep/patches/dpdk/20.11/dpdk_pci.patch"
+			patch -p1 < "$rootdir/test/common/config/pkgdep/patches/dpdk/20.11/dpdk_qat.patch"
+		elif [[ $dpdk_ver == 21.11* ]]; then
+			patch -p1 < "$rootdir/test/common/config/pkgdep/patches/dpdk/21.11/dpdk_qat.patch"
+		elif [[ $dpdk_ver == 22.03* ]]; then
+			patch -p1 < "$rootdir/test/common/config/pkgdep/patches/dpdk/22.03/dpdk_qat.patch"
 		fi
-		git config --local user.name "spdk"
-		git config --local user.email "nomail@all.com"
-		git am dpdk-pci.patch
-		git am dpdk-qat.patch
+	fi
+
+	dpdk_kmods="false"
+	if [ "$(uname -s)" = "FreeBSD" ]; then
+		dpdk_kmods="true"
 	fi
 
 	meson build-tmp --prefix="$external_dpdk_dir" --libdir lib \
-		-Denable_docs=false -Denable_kmods=false -Dtests=false \
+		-Denable_docs=false -Denable_kmods="$dpdk_kmods" -Dtests=false \
 		-Dc_link_args="$dpdk_ldflags" -Dc_args="$dpdk_cflags" \
-		-Dmachine=native -Ddisable_drivers=$(printf "%s," "${DPDK_DISABLED_DRIVERS[@]}")
+		-Dmachine=native -Denable_drivers=$(printf "%s," "${DPDK_DRIVERS[@]}")
 	ninja -C "$external_dpdk_base_dir/build-tmp" $MAKEFLAGS
 	ninja -C "$external_dpdk_base_dir/build-tmp" $MAKEFLAGS install
 
@@ -158,6 +212,7 @@ function make_fail_cleanup() {
 
 function scanbuild_make() {
 	pass=true
+	"$rootdir/configure" $config_params --without-shared
 	$scanbuild $MAKE $MAKEFLAGS > $out/build_output.txt && rm -rf $out/scan-build-tmp || make_fail_cleanup
 	xtrace_disable
 
@@ -231,10 +286,18 @@ function build_doc() {
 
 	$MAKE -C "$rootdir"/doc --no-print-directory $MAKEFLAGS &> "$out"/doxygen.log
 	if [ -s "$out"/doxygen.log ]; then
-		cat "$out"/doxygen.log
-		echo "Doxygen errors found!"
-		eq "$doxygenv" 1.8.20 || exit 1
-		echo "Doxygen $doxygenv detected, all warnings are potentially false positives, continuing the test"
+		if [[ "$doxygenv" != "1.8.20" ]]; then
+			cat "$out"/doxygen.log
+			echo "Doxygen errors found!"
+			exit 1
+		fi
+		# Doxygen 1.8.20 produces false positives, see:
+		# https://github.com/doxygen/doxygen/issues/7948
+		if grep -v "\ilinebr" "$out"/doxygen.log; then
+			echo "Doxygen errors found!"
+			exit 1
+		fi
+		echo "Doxygen $doxygenv detected. No warnings except false positives, continuing the test"
 	fi
 	if hash pdflatex 2> /dev/null; then
 		$MAKE -C "$rootdir"/doc/output/latex --no-print-directory $MAKEFLAGS &>> "$out"/doxygen.log
@@ -255,12 +318,9 @@ function build_doc() {
 function autobuild_test_suite() {
 	run_test "autobuild_check_format" ./scripts/check_format.sh
 	run_test "autobuild_external_code" sudo -E --preserve-env=PATH LD_LIBRARY_PATH=$LD_LIBRARY_PATH $rootdir/test/external_code/test_make.sh $rootdir
-	if [ "$SPDK_TEST_OCF" -eq 1 ]; then
-		run_test "autobuild_ocf_precompile" ocf_precompile
-	fi
 	run_test "autobuild_check_so_deps" $rootdir/test/make/check_so_deps.sh $1
 	./configure $config_params --without-shared
-	run_test "scanbuild_make" scanbuild_make
+	$MAKE $MAKEFLAGS
 	run_test "autobuild_generated_files_check" porcelain_check
 	run_test "autobuild_header_dependency_check" header_dependency_check
 	run_test "autobuild_make_install" $MAKE $MAKEFLAGS install DESTDIR="$SPDK_WORKSPACE" prefix=/usr
@@ -268,6 +328,11 @@ function autobuild_test_suite() {
 	run_test "autobuild_build_doc" build_doc
 }
 
+function unittest_build() {
+	"$rootdir/configure" $config_params --without-shared
+	$MAKE $MAKEFLAGS
+}
+
 if [ $SPDK_RUN_VALGRIND -eq 1 ]; then
 	run_test "valgrind" echo "using valgrind"
 fi
@@ -291,12 +356,17 @@ $MAKE cc_version
 $MAKE cxx_version
 echo "** END ** Info for Hostname: $HOSTNAME"
 
-if [ "$SPDK_TEST_AUTOBUILD" -eq 1 ]; then
+if [[ $SPDK_TEST_OCF -eq 1 ]]; then
+	run_test "autobuild_ocf_precompile" ocf_precompile
+fi
+
+if [[ $SPDK_TEST_AUTOBUILD -eq 1 ]]; then
 	run_test "autobuild" autobuild_test_suite $1
+elif [[ $SPDK_TEST_UNITTEST -eq 1 ]]; then
+	run_test "unittest_build" unittest_build
+elif [[ $SPDK_TEST_SCANBUILD -eq 1 ]]; then
+	run_test "scanbuild_make" scanbuild_make
 else
-	if [ "$SPDK_TEST_OCF" -eq 1 ]; then
-		run_test "autobuild_ocf_precompile" ocf_precompile
-	fi
 	# if we aren't testing the unittests, build with shared objects.
 	./configure $config_params --with-shared
 	run_test "make" $MAKE $MAKEFLAGS

autopackage.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -xe
+set -e
 
 # If the configuration of tests is not provided, no tests will be carried out.
 if [[ ! -f $1 ]]; then
@@ -11,6 +11,7 @@ fi
 source "$1"
 
 rootdir=$(readlink -f $(dirname $0))
+testdir=$rootdir # to get the storage space for tests
 source "$rootdir/test/common/autotest_common.sh"
 
 function build_rpms() (
@@ -20,27 +21,66 @@ function build_rpms() (
 	unset -v LD_LIBRARY_PATH
 
 	install_uninstall_rpms() {
-		rpms=("$HOME/rpmbuild/RPMS/x86_64/"spdk{,-devel,{,-dpdk}-libs}-$version-1.x86_64.rpm)
+		rpms=("${1:-$builddir/rpm/}/x86_64/"*.rpm)
 
 		sudo rpm -i "${rpms[@]}"
-		rpms=("${rpms[@]##*/}") rpms=("${rpms[@]%.rpm}")
 		# Check if we can find one of the apps in the PATH now and verify if it doesn't miss
 		# any libs.
 		LIST_LIBS=yes "$rootdir/rpmbuild/rpm-deps.sh" "${SPDK_APP[@]##*/}"
+		rm "${rpms[@]}"
+		rpms=("${rpms[@]##*/}") rpms=("${rpms[@]%.rpm}")
 		sudo rpm -e "${rpms[@]}"
 	}
 
 	build_rpm() {
-		MAKEFLAGS="$MAKEFLAGS" SPDK_VERSION="$version" DEPS=no "$rootdir/rpmbuild/rpm.sh" "$@"
+		# Separate run to see the final .spec in use
+		GEN_SPEC=yes BUILDDIR=$builddir MAKEFLAGS="$MAKEFLAGS" SPDK_VERSION="$version" DEPS=no "$rootdir/rpmbuild/rpm.sh" "$@"
+		# Actual build
+		BUILDDIR=$builddir MAKEFLAGS="$MAKEFLAGS" SPDK_VERSION="$version" DEPS=no "$rootdir/rpmbuild/rpm.sh" "$@"
 		install_uninstall_rpms
 	}
 
-	version="test_shared"
-	run_test "build_shared_rpm" build_rpm --with-shared
+	build_rpm_with_rpmed_dpdk() {
+		sudo dnf install -y dpdk-devel
+		build_rpm --with-shared --with-dpdk
+	}
 
-	if [[ -n $SPDK_TEST_NATIVE_DPDK ]]; then
-		version="test_shared_native_dpdk"
-		run_test "build_shared_native_dpdk_rpm" build_rpm --with-shared --with-dpdk="$SPDK_RUN_EXTERNAL_DPDK"
+	build_rpm_from_gen_spec() {
+		GEN_SPEC=yes \
+			USE_DEFAULT_DIRS=yes \
+			MAKEFLAGS="$MAKEFLAGS" \
+			SPDK_VERSION="$version" \
+			DEPS=no \
+			"$rootdir/rpmbuild/rpm.sh" --with-shared > "$builddir/gen-spdk.spec"
+
+		# Default locations should be in use.
+		sourcedir=$(rpm --eval "%{_sourcedir}") rpmdir=$(rpm --eval "%{_rpmdir}")
+		mkdir -p "$sourcedir" "$rpmdir"
+
+		# Prepare the source at the default location - default %prep step requires
+		# extra dir inside the source package hence the dance with symlinking to
+		# the repo (after the extraction source should be under spdk-$version/) -
+		# make sure symlinking is done outside of the repo to avoid nasty loops.
+		ln -s "$rootdir" "/tmp/spdk-$version"
+		tar -czhf "$sourcedir/spdk-$version.tar.gz" -C /tmp "spdk-$version"
+
+		# See rpm.sh for details on the PYTHONPATH HACK
+		PYTHONPATH="$(python3 -c "import sys; print('%s' % ':'.join(sys.path)[1:])")" \
+			rpmbuild -ba "$builddir/gen-spdk.spec"
+		install_uninstall_rpms "$rpmdir"
+	}
+
+	version="test_shared"
+	builddir=$SPDK_TEST_STORAGE/test-rpm
+
+	run_test "build_shared_rpm" build_rpm --with-shared
+	if [[ $RUN_NIGHTLY -eq 1 ]]; then
+		run_test "build_shared_rpm_with_rpmed_dpdk" build_rpm_with_rpmed_dpdk
+		run_test "build_rpm_from_gen_spec" build_rpm_from_gen_spec
+		if [[ -n $SPDK_TEST_NATIVE_DPDK ]]; then
+			version="test_shared_native_dpdk"
+			run_test "build_shared_native_dpdk_rpm" build_rpm --with-shared --with-dpdk="$SPDK_RUN_EXTERNAL_DPDK"
+		fi
 	fi
 )
 
@@ -73,6 +113,11 @@ timing_enter build_release
 
 config_params="$(get_config_params | sed 's/--enable-debug//g')"
 if [ $(uname -s) = Linux ]; then
+	# LTO needs a special compiler to work under clang. See detect_cc.sh for details.
+	if [[ $CC == *clang* ]]; then
+		LD=$(type -P ld.gold)
+		export LD
+	fi
 	./configure $config_params --enable-lto
 else
 	# LTO needs a special compiler to work on BSD.

autorun.sh

@@ -12,11 +12,14 @@ if [[ ! -f $conf ]]; then
 	echo "ERROR: $conf doesn't exist"
 	exit 1
 fi
+source "$conf"
 
 echo "Test configuration:"
 cat "$conf"
 
 # Runs agent scripts
 $rootdir/autobuild.sh "$conf"
-sudo -E $rootdir/autotest.sh "$conf"
+if ((SPDK_TEST_UNITTEST == 1 || SPDK_RUN_FUNCTIONAL_TEST == 1)); then
+	sudo -E $rootdir/autotest.sh "$conf"
+fi
 $rootdir/autopackage.sh "$conf"

autotest.sh

@@ -9,9 +9,6 @@ if [[ ! -f $1 ]]; then
 	exit 1
 fi
 
-# always test with SPDK shared objects.
-export SPDK_LIB_DIR="$rootdir/build/lib"
-
 # Autotest.sh, as part of autorun.sh, runs in a different
 # shell process than autobuild.sh. Use helper file to pass
 # over env variable containing libraries paths.
@@ -32,13 +29,7 @@ if [ $(uname -s) = Linux ]; then
 	old_core_pattern=$(< /proc/sys/kernel/core_pattern)
 	mkdir -p "$output_dir/coredumps"
 	# set core_pattern to a known value to avoid ABRT, systemd-coredump, etc.
-	echo "|$rootdir/scripts/core-collector.sh %P %s %t $output_dir/coredumps" > /proc/sys/kernel/core_pattern
-	echo 2 > /proc/sys/kernel/core_pipe_limit
-
-	# Make sure that the hugepage state for our VM is fresh so we don't fail
-	# hugepage allocation. Allow time for this action to complete.
-	echo 1 > /proc/sys/vm/drop_caches
-	sleep 3
+	echo "|$rootdir/scripts/core-collector.sh %P %s %t %c $output_dir/coredumps" > /proc/sys/kernel/core_pattern
 
 	# make sure nbd (network block device) driver is loaded if it is available
 	# this ensures that when tests need to use nbd, it will be fully initialized
@@ -50,7 +41,7 @@ if [ $(uname -s) = Linux ]; then
 	fi
 fi
 
-trap "process_core; autotest_cleanup; exit 1" SIGINT SIGTERM EXIT
+trap "autotest_cleanup || :; revert_soft_roce; exit 1" SIGINT SIGTERM EXIT
 
 timing_enter autotest
 
@@ -92,39 +83,31 @@ rm -f /var/tmp/spdk*.sock
 # Load the kernel driver
 ./scripts/setup.sh reset
 
-if [ $(uname -s) = Linux ]; then
-	# OCSSD devices drivers don't support IO issues by kernel so
-	# detect OCSSD devices and block them (unbind from any driver).
-	# If test scripts want to use this device it needs to do this explicitly.
-	#
-	# If some OCSSD device is bound to other driver than nvme we won't be able to
-	# discover if it is OCSSD or not so load the kernel driver first.
+get_zoned_devs
 
-	while IFS= read -r -d '' dev; do
-		# Send Open Channel 2.0 Geometry opcode "0xe2" - not supported by NVMe device.
-		if nvme admin-passthru $dev --namespace-id=1 --data-len=4096 --opcode=0xe2 --read > /dev/null; then
-			bdf="$(basename $(readlink -e /sys/class/nvme/${dev#/dev/}/device))"
-			echo "INFO: blocking OCSSD device: $dev ($bdf)"
-			PCI_BLOCKED+=" $bdf"
-			OCSSD_PCI_DEVICES+=" $bdf"
-		fi
-	done < <(find /dev -maxdepth 1 -regex '/dev/nvme[0-9]+' -print0)
+if ((${#zoned_devs[@]} > 0)); then
+	# FIXME: For now make sure zoned devices are tested on-demand by
+	# a designated tests instead of falling into any other. The main
+	# concern here are fio workloads where specific configuration
+	# must be in place for it to work with the zoned device.
+	export PCI_BLOCKED="${zoned_devs[*]}"
+	export PCI_ZONED="${zoned_devs[*]}"
+fi
 
-	export OCSSD_PCI_DEVICES
-
-	# Now, bind blocked devices to pci-stub module. This will prevent
-	# automatic grabbing these devices when we add device/vendor ID to
-	# proper driver.
-	if [[ -n "$PCI_BLOCKED" ]]; then
-		# shellcheck disable=SC2097,SC2098
-		PCI_ALLOWED="$PCI_BLOCKED" \
-			PCI_BLOCKED="" \
-			DRIVER_OVERRIDE="pci-stub" \
-			./scripts/setup.sh
-
-		# Export our blocked list so it will take effect during next setup.sh
-		export PCI_BLOCKED
+# Delete all leftover lvols and gpt partitions
+# Matches both /dev/nvmeXnY on Linux and /dev/nvmeXnsY on BSD
+# Filter out nvme with partitions - the "p*" suffix
+for dev in $(ls /dev/nvme*n* | grep -v p || true); do
+	# Skip zoned devices as non-sequential IO will always fail
+	[[ -z ${zoned_devs["${dev##*/}"]} ]] || continue
+	if ! block_in_use "$dev"; then
+		dd if=/dev/zero of="$dev" bs=1M count=1
 	fi
+done
+
+sync
+
+if [ $(uname -s) = Linux ]; then
 	run_test "setup.sh" "$rootdir/test/setup/test-setup.sh"
 fi
 
@@ -135,15 +118,6 @@ if [[ $(uname -s) == Linux ]]; then
 	nvme_namespace_revert
 fi
 
-# Delete all leftover lvols and gpt partitions
-# Matches both /dev/nvmeXnY on Linux and /dev/nvmeXnsY on BSD
-# Filter out nvme with partitions - the "p*" suffix
-for dev in $(ls /dev/nvme*n* | grep -v p || true); do
-	dd if=/dev/zero of="$dev" bs=1M count=1
-done
-
-sync
-
 timing_exit cleanup
 
 # set up huge pages
@@ -151,19 +125,6 @@ timing_enter afterboot
 ./scripts/setup.sh
 timing_exit afterboot
 
-if [[ $SPDK_TEST_CRYPTO -eq 1 || $SPDK_TEST_REDUCE -eq 1 ]]; then
-	# Make sure that memory is distributed across all NUMA nodes - by default, all goes to
-	# node0, but if QAT devices are attached to a different node, all of their VFs will end
-	# up under that node too and memory needs to be available there for the tests.
-	CLEAR_HUGE=yes HUGE_EVEN_ALLOC=yes ./scripts/setup.sh
-	./scripts/setup.sh status
-	if [[ $SPDK_TEST_USE_IGB_UIO -eq 1 ]]; then
-		./scripts/qat_setup.sh igb_uio
-	else
-		./scripts/qat_setup.sh
-	fi
-fi
-
 # Revert existing OPAL to factory settings that may have been left from earlier failed tests.
 # This ensures we won't hit any unexpected failures due to NVMe SSDs being locked.
 opal_revert_cleanup
@@ -178,15 +139,24 @@ if [ $SPDK_TEST_UNITTEST -eq 1 ]; then
 fi
 
 if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
+	if [[ $SPDK_TEST_CRYPTO -eq 1 || $SPDK_TEST_REDUCE -eq 1 ]]; then
+		if [[ $SPDK_TEST_USE_IGB_UIO -eq 1 ]]; then
+			./scripts/qat_setup.sh igb_uio
+		else
+			./scripts/qat_setup.sh
+		fi
+	fi
 	timing_enter lib
 
 	run_test "rpc" test/rpc/rpc.sh
 	run_test "rpc_client" test/rpc_client/rpc_client.sh
 	run_test "json_config" ./test/json_config/json_config.sh
+	run_test "json_config_extra_key" ./test/json_config/json_config_extra_key.sh
 	run_test "alias_rpc" test/json_config/alias_rpc/alias_rpc.sh
 	run_test "spdkcli_tcp" test/spdkcli/tcp.sh
 	run_test "dpdk_mem_utility" test/dpdk_memory_utility/test_dpdk_mem_info.sh
 	run_test "event" test/event/event.sh
+	run_test "thread" test/thread/thread.sh
 	run_test "accel_engine" test/accel_engine/accel_engine.sh
 
 	if [ $SPDK_TEST_BLOCKDEV -eq 1 ]; then
@@ -194,30 +164,56 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
 		run_test "bdev_raid" test/bdev/bdev_raid.sh
 		run_test "bdevperf_config" test/bdev/bdevperf/test_config.sh
 		if [[ $(uname -s) == Linux ]]; then
-			run_test "spdk_dd" test/dd/dd.sh
+			run_test "reactor_set_interrupt" test/interrupt/reactor_set_interrupt.sh
+			run_test "reap_unregistered_poller" test/interrupt/reap_unregistered_poller.sh
 		fi
 	fi
 
-	if [ $SPDK_TEST_JSON -eq 1 ]; then
-		run_test "test_converter" test/config_converter/test_converter.sh
+	if [[ $(uname -s) == Linux ]]; then
+		if [[ $SPDK_TEST_BLOCKDEV -eq 1 || $SPDK_TEST_URING -eq 1 ]]; then
+			# The crypto job also includes the SPDK_TEST_BLOCKDEV in its configuration hence the
+			# dd tests are executed there as well. However, these tests can take a significant
+			# amount of time to complete (up to 4min) on a physical system leading to a potential
+			# job timeout. Avoid that by skipping these tests - this should not affect the coverage
+			# since dd tests are still run as part of the vg jobs.
+			if [[ $SPDK_TEST_CRYPTO -eq 0 ]]; then
+				run_test "spdk_dd" test/dd/dd.sh
+			fi
+		fi
 	fi
 
 	if [ $SPDK_TEST_NVME -eq 1 ]; then
 		run_test "blockdev_nvme" test/bdev/blockdev.sh "nvme"
-		run_test "blockdev_nvme_gpt" test/bdev/blockdev.sh "gpt"
+		if [[ $(uname -s) == Linux ]]; then
+			run_test "blockdev_nvme_gpt" test/bdev/blockdev.sh "gpt"
+		fi
 		run_test "nvme" test/nvme/nvme.sh
 		if [[ $SPDK_TEST_NVME_PMR -eq 1 ]]; then
 			run_test "nvme_pmr" test/nvme/nvme_pmr.sh
 		fi
+		if [[ $SPDK_TEST_NVME_SCC -eq 1 ]]; then
+			run_test "nvme_scc" test/nvme/nvme_scc.sh
+		fi
+		if [[ $SPDK_TEST_NVME_BP -eq 1 ]]; then
+			run_test "nvme_bp" test/nvme/nvme_bp.sh
+		fi
 		if [[ $SPDK_TEST_NVME_CUSE -eq 1 ]]; then
 			run_test "nvme_cuse" test/nvme/cuse/nvme_cuse.sh
 		fi
+		if [[ $SPDK_TEST_NVME_CMB -eq 1 ]]; then
+			run_test "nvme_cmb" test/nvme/cmb/cmb.sh
+		fi
+
+		if [[ $SPDK_TEST_NVME_ZNS -eq 1 ]]; then
+			run_test "nvme_zns" test/nvme/zns/zns.sh
+		fi
+
 		run_test "nvme_rpc" test/nvme/nvme_rpc.sh
 		# Only test hotplug without ASAN enabled. Since if it is
 		# enabled, it catches SEGV earlier than our handler which
 		# breaks the hotplug logic.
 		if [ $SPDK_RUN_ASAN -eq 0 ]; then
-			run_test "nvme_hotplug" test/nvme/hotplug.sh root
+			run_test "nvme_hotplug" test/nvme/hotplug.sh
 		fi
 	fi
 
@@ -247,18 +243,14 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
 		# The NVMe-oF run test cases are split out like this so that the parser that compiles the
 		# list of all tests can properly differentiate them. Please do not merge them into one line.
 		if [ "$SPDK_TEST_NVMF_TRANSPORT" = "rdma" ]; then
-			timing_enter rdma_setup
-			rdma_device_init
-			timing_exit rdma_setup
 			run_test "nvmf_rdma" ./test/nvmf/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
 			run_test "spdkcli_nvmf_rdma" ./test/spdkcli/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
 		elif [ "$SPDK_TEST_NVMF_TRANSPORT" = "tcp" ]; then
-			timing_enter tcp_setup
-			tcp_device_init
-			timing_exit tcp_setup
 			run_test "nvmf_tcp" ./test/nvmf/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
-			run_test "spdkcli_nvmf_tcp" ./test/spdkcli/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
-			run_test "nvmf_identify_passthru" test/nvmf/target/identify_passthru.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
+			if [[ $SPDK_TEST_URING -eq 0 ]]; then
+				run_test "spdkcli_nvmf_tcp" ./test/spdkcli/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
+				run_test "nvmf_identify_passthru" test/nvmf/target/identify_passthru.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
+			fi
 			run_test "nvmf_dif" test/nvmf/target/dif.sh
 		elif [ "$SPDK_TEST_NVMF_TRANSPORT" = "fc" ]; then
 			run_test "nvmf_fc" ./test/nvmf/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
@@ -273,6 +265,10 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
 		run_test "vhost" ./test/vhost/vhost.sh
 	fi
 
+	if [ $SPDK_TEST_VFIOUSER_QEMU -eq 1 ]; then
+		run_test "vfio_user_qemu" ./test/vfio_user/vfio_user.sh
+	fi
+
 	if [ $SPDK_TEST_LVOL -eq 1 ]; then
 		run_test "lvol" ./test/lvol/lvol.sh
 		run_test "blob_io_wait" ./test/blobstore/blob_io_wait/blob_io_wait.sh
@@ -334,6 +330,7 @@ fi
 
 timing_enter cleanup
 autotest_cleanup
+revert_soft_roce
 timing_exit cleanup
 
 timing_exit autotest
@@ -341,8 +338,7 @@ chmod a+r $output_dir/timing.txt
 
 trap - SIGINT SIGTERM EXIT
 
-# catch any stray core files
-process_core
+[[ -f "$output_dir/udev.log" ]] && rm -f "$output_dir/udev.log"
 
 if hash lcov && ! [[ "$CC_TYPE" == *"clang"* ]]; then
 	# generate coverage data and combine with baseline

build/lib/.gitignore

@@ -1 +0,0 @@
-# Placeholder

configure

@@ -35,6 +35,7 @@ function usage() {
 	echo " --disable-tests           Disable building of functional tests"
 	echo " --disable-unit-tests      Disable building of unit tests"
 	echo " --disable-examples        Disable building of examples"
+	echo " --disable-apps            Disable building of apps"
 	echo ""
 	echo "Specifying Dependencies:"
 	echo "--with-DEPENDENCY[=path]   Use the given dependency. Optionally, provide the"
@@ -43,61 +44,67 @@ function usage() {
 	echo "                           disable features and components."
 	echo ""
 	echo "Valid dependencies are listed below."
-	echo " dpdk                      Build against a custom dpdk version. By default, the dpdk"
-	echo "                           submodule in spdk tree will be used."
+	echo " --with-dpdk[=DIR]         Build against a custom dpdk version. By default, the dpdk"
+	echo " --without-dpdk            submodule in spdk tree will be used."
 	echo "                           example: /usr/share/dpdk/x86_64-default-linuxapp-gcc"
-	echo " env                       Use an alternate environment implementation instead of DPDK."
+	echo " --with-env=DIR            Use an alternate environment implementation instead of DPDK."
 	echo "                           Implies --without-dpdk."
-	echo " idxd                      Build the IDXD library and accel framework plug-in module."
-	echo "                           Disabled while experimental. Only built for x86 when enabled."
-	echo " crypto                    Build vbdev crypto module."
-	echo "                           No path required."
-	echo " fio                       Build fio_plugin."
-	echo "                           default: /usr/src/fio"
-	echo " vhost                     Build vhost target. Enabled by default."
-	echo "                           No path required."
-	echo " virtio                    Build vhost initiator and virtio-pci bdev modules."
-	echo "                           No path required."
-	echo " vfio-user                 Build custom vfio-user transport for NVMf target and NVMe initiator."
-	echo "                           example: /usr/src/libvfio-user"
-	echo " pmdk                      Build persistent memory bdev."
-	echo "                           example: /usr/share/pmdk"
-	echo " reduce                    Build vbdev compression module."
-	echo "                           No path required."
-	echo " rbd                       Build Ceph RBD bdev module."
-	echo "                           No path required."
-	echo " rdma                      Build RDMA transport for NVMf target and initiator."
-	echo "                           Accepts optional RDMA provider name. Can be \"verbs\" or \"mlx5_dv\"."
+	echo " --with-idxd               Build the IDXD library and accel framework plug-in module."
+	echo " --without-idxd            Disabled while experimental. Only built for x86 when enabled."
+	echo " --with-crypto             Build vbdev crypto module."
+	echo " --without-crypto          No path required."
+	echo " --with-fio[=DIR]          Build fio_plugin."
+	echo " --without-fio             default: /usr/src/fio"
+	echo " --with-vhost              Build vhost target. Enabled by default."
+	echo " --without-vhost           No path required."
+	echo " --with-virtio             Build vhost initiator and virtio-pci bdev modules."
+	echo " --without-virtio          No path required."
+	echo " --with-vfio-user[=DIR]    Build custom vfio-user transport for NVMf target and NVMe initiator."
+	echo " --without-vfio-user       example: /usr/src/libvfio-user"
+	echo " --with-pmdk[=DIR]         Build persistent memory bdev."
+	echo " --without-pmdk            example: /usr/share/pmdk"
+	echo " --with-reduce             Build vbdev compression module."
+	echo " --without-reduce          No path required."
+	echo " --with-rbd                Build Ceph RBD bdev module."
+	echo " --without-rbd             No path required."
+	echo " --with-rdma[=DIR]         Build RDMA transport for NVMf target and initiator."
+	echo " --without-rdma            Accepts optional RDMA provider name. Can be \"verbs\" or \"mlx5_dv\"."
 	echo "                           If no provider specified, \"verbs\" provider is used by default."
-	echo " fc                        Build FC transport for NVMf target."
-	echo "                           If an argument is provided, it is considered a directory containing"
+	echo " --with-fc[=DIR]           Build FC transport for NVMf target."
+	echo " --without-fc              If an argument is provided, it is considered a directory containing"
 	echo "                           libufc.a and fc_lld.h. Otherwise the regular system paths will"
 	echo "                           be searched."
-	echo " shared                    Build spdk shared libraries."
-	echo "                           No path required."
-	echo " iscsi-initiator           Build with iscsi bdev module."
-	echo "                           No path required."
-	echo " vtune                     Required to profile I/O under Intel VTune Amplifier XE."
-	echo "                           example: /opt/intel/vtune_amplifier_xe_version"
-	echo " ocf                       Build OCF library and bdev module."
-	echo "                           If argument is directory, interpret it as root of OCF repo"
+	echo " --with-shared             Build spdk shared libraries."
+	echo " --without-shared          No path required."
+	echo " --with-iscsi-initiator    Build with iscsi bdev module."
+	echo " --without-iscsi-initiator No path required."
+	echo " --with-vtune=DIR          Required to profile I/O under Intel VTune Amplifier XE."
+	echo " --without-vtune           example: /opt/intel/vtune_amplifier_xe_version"
+	echo " --with-ocf[=DIR]          Build OCF library and bdev module."
+	echo " --without-ocf             If argument is directory, interpret it as root of OCF repo"
 	echo "                           If argument is file, interpret it as compiled OCF lib"
 	echo "                           If no argument is specified, OCF git submodule is used by default"
 	echo "                           example: /usr/src/ocf/"
-	echo " isal                      Build with ISA-L. Enabled by default on x86 and aarch64 architectures."
-	echo "                           No path required."
-	echo " uring                     Build I/O uring bdev or socket module."
-	echo "                           If an argument is provided, it is considered a directory containing"
+	echo " --with-isal               Build with ISA-L. Enabled by default on x86 and aarch64 architectures."
+	echo " --without-isal            No path required."
+	echo " --with-uring[=DIR]        Build I/O uring bdev or socket module."
+	echo " --without-uring           If an argument is provided, it is considered a directory containing"
 	echo "                           liburing.a and io_uring.h. Otherwise the regular system paths will"
 	echo "                           be searched."
-	echo " fuse                      Build FUSE components for mounting a blobfs filesystem."
-	echo "                           No path required."
-	echo " nvme-cuse                 Build NVMe driver with support for CUSE-based character devices."
-	echo "                           No path required."
-	echo " raid5                     Build with bdev_raid module RAID5 support."
-	echo "                           No path required."
-	echo " wpdk                      Build using WPDK to provide support for Windows (experimental)."
-	echo "                           The argument must be a directory containing lib and include."
+	echo " --with-fuse               Build FUSE components for mounting a blobfs filesystem."
+	echo " --without-fuse            No path required."
+	echo " --with-nvme-cuse          Build NVMe driver with support for CUSE-based character devices."
+	echo " --without-nvme-cuse       No path required."
+	echo " --with-raid5              Build with bdev_raid module RAID5 support."
+	echo " --without-raid5           No path required."
+	echo " --with-wpdk=DIR           Build using WPDK to provide support for Windows (experimental)."
+	echo " --without-wpdk            The argument must be a directory containing lib and include."
+	echo " --with-usdt               Build with userspace DTrace probes enabled."
+	echo " --without-usdt            No path required."
+	echo " --with-fuzzer             Build with LLVM fuzzing enabled."
+	echo "                           Path to clang_rt.fuzzer_no_main library required."
+	echo "                           Requires setting CC and CXX to clang."
+	echo "                           (Typically /usr/lib/llvm-VER/lib/clang/VER/lib/linux/libclang_rt.fuzzer_no_main-ARCH.a)"
 	echo ""
 	echo "Environment variables:"
 	echo ""
@@ -112,7 +119,7 @@ function usage() {
 }
 
 # Load default values
-# Convert config to sourcable configuration file
+# Convert config to sourceable configuration file
 sed -r 's/CONFIG_([[:alnum:]_]+)=(.*)/CONFIG[\1]=\2/g' $rootdir/CONFIG > $rootdir/CONFIG.sh
 declare -A CONFIG
 source $rootdir/CONFIG.sh
@@ -163,7 +170,7 @@ fi
 
 #check nasm only on x86
 if [[ $arch == x86_64* ]]; then
-	ver=$(nasm -v 2> /dev/null | awk '{print $3}')
+	ver=$(nasm -v 2> /dev/null | awk '{print $3}' | awk -Fr '{print $1}')
 	if lt "$ver" 2.14; then
 		# ISA-L, compression & crypto require NASM version 2.14 or newer.
 		CONFIG[ISAL]=n
@@ -186,6 +193,86 @@ function check_dir() {
 	fi
 }
 
+# On x86_64 'clang -dumpmachine' produces x86_64-pc-linux-gnu
+# whereas the dpdk might be built with gcc and its libs lie in
+# x86_64-linux-gnu. Let's find the right libdir for dpdkd libs.
+function find_dpdk_arch_libdir() {
+	local dpdk_dir=$1
+
+	# Checking first what we have with $arch, then clang
+	# variant of arch.
+	arches=("$arch" "$(echo $arch | sed 's/-pc//g')")
+	for a in "${arches[@]}"; do
+		local libdir="$dpdk_dir/lib/$a"
+		if [[ -d $libdir ]]; then
+			echo $libdir
+			return
+		fi
+	done
+
+	# Fallback to the libdir without arch component
+	echo "$dpdk_dir/lib"
+}
+
+function check_IPSec_mb() {
+	local mode=$1
+	local dpdk_libdir=$2
+	local dpdk_incdir=$3
+	local have_ipsec_mb=n
+
+	if [[ $mode = "pkg-config" ]]; then
+		local dpdk_libs
+
+		# Request libdpdk pkg-config settings to figure out if the IPSec_MB is used
+		# as a dependency.
+		# Due to some reason pkg-config shows -lIPSec_MB only with --static option
+		dpdk_libs=$(PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$dpdk_libdir/pkgconfig" pkg-config --libs --static libdpdk)
+		if echo "$dpdk_libs" | grep "\-lIPSec_MB" > /dev/null 2>&1; then
+			have_ipsec_mb=y
+		fi
+	elif [[ $mode = "build-config" ]]; then
+		# Use dpdk build config header to check if the IPSec_MB was used.
+		if grep -F "define RTE_CRYPTO_IPSEC_MB 1" "$dpdk_incdir/rte_build_config.h" > /dev/null 2>&1; then
+			have_ipsec_mb=y
+		fi
+	else
+		echo "ERROR: Invalid IPSec_MB checking mode $mode."
+		echo "ERROR: Only \"pkg-config\" and \"build-config\" available."
+		exit 1
+	fi
+	if [[ $have_ipsec_mb = "n" ]]; then
+		CONFIG[IPSEC_MB]=n
+		return
+	fi
+
+	# Since we don't know the library path where the IPSec_MB is located
+	# let's find it out with the ldd utility. This can be a standard location
+	# or a custom build.
+	local librte_crypto_ipsec_mb="$dpdk_libdir/librte_crypto_ipsec_mb.so"
+	if [[ -f "$librte_crypto_ipsec_mb" ]]; then
+		local ipsec_mb_libdir
+
+		ipsec_mb_libdir=$(ldd "$librte_crypto_ipsec_mb" | grep "libIPSec_MB.so" \
+			| sed -e 's/\s*libIPSec_MB.so.*=>\s//' -e 's/\/libIPSec_MB.so.*$//')
+		if [[ -d $ipsec_mb_libdir ]]; then
+			CONFIG[IPSEC_MB]=y
+			CONFIG[IPSEC_MB_DIR]="$ipsec_mb_libdir"
+		elif [[ $ipsec_mb_libdir = "not found" ]]; then
+			# ldconfig cache is broken, old build with refs to non-existing libs, etc.
+			echo "ERROR: Invalid IPSec_MB installation. Library is not found and/or ldconfig cache is broken!"
+			exit 1
+		else
+			# Failed to check for IPSec_MB lib path. Let's just assume it is lives
+			# in one of the standard locations (/usr/lib, etc.).
+			CONFIG[IPSEC_MB]=y
+		fi
+	else
+		# pkg-config says there is IPSec_mb and dpdk lib does not have it. Let's just
+		# assume it is installed in the system in one of the standard locations.
+		CONFIG[IPSEC_MB]=y
+	fi
+}
+
 for i in "$@"; do
 	case "$i" in
 		-h | --help)
@@ -263,6 +350,12 @@ for i in "$@"; do
 		--disable-examples)
 			CONFIG[EXAMPLES]=n
 			;;
+		--enable-apps)
+			CONFIG[APPS]=y
+			;;
+		--disable-apps)
+			CONFIG[APPS]=N
+			;;
 		--enable-werror)
 			CONFIG[WERROR]=y
 			;;
@@ -275,9 +368,63 @@ for i in "$@"; do
 		--disable-cet)
 			CONFIG[CET]=n
 			;;
+		--with-dpdk)
+			# Can we use pkg-config?
+			if command -v "pkg-config" > /dev/null 2>&1 && pkg-config --exists libdpdk; then
+				dpdk_libdir=$(pkg-config --variable=libdir libdpdk)
+				dpdk_libdir=$(readlink -f $dpdk_libdir)
+				dpdk_incdir=$(pkg-config --variable=includedir libdpdk)
+				echo "Using DPDK lib dir $dpdk_libdir"
+				CONFIG[DPDK_LIB_DIR]=$dpdk_libdir
+				CONFIG[DPDK_INC_DIR]=$dpdk_incdir
+				CONFIG[DPDK_PKG_CONFIG]=y
+				if pkg-config --print-requires libdpdk | grep "libbsd" > /dev/null 2>&1; then
+					CONFIG[HAVE_LIBBSD]=y
+				fi
+				CFLAGS="${CFLAGS:+$CFLAGS }$(pkg-config --cflags libdpdk)"
+				check_IPSec_mb "pkg-config" "$dpdk_libdir" "$dpdk_incdir"
+			else
+				echo "libdpdk.pc not found, aborting"
+				exit 1
+			fi
+			;;
 		--with-dpdk=*)
 			check_dir "$i"
-			CONFIG[DPDK_DIR]=$(readlink -f ${i#*=})
+			dpdk_dir=$(readlink -f ${i#*=})
+			dpdk_libdir=$(find_dpdk_arch_libdir $dpdk_dir)
+			dpdk_incdir="$dpdk_dir/include"
+
+			# Can we use pkg-config?
+			if command -v "pkg-config" > /dev/null 2>&1 && PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$dpdk_libdir/pkgconfig" pkg-config --exists libdpdk; then
+				echo "Using $dpdk_libdir/pkgconfig for additional libs..."
+				dpdk_libdir=$(PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$dpdk_libdir/pkgconfig" pkg-config --variable=libdir libdpdk)
+				dpdk_libdir=$(readlink -f $dpdk_libdir)
+				if ! echo $dpdk_libdir | grep $dpdk_dir > /dev/null 2>&1; then
+					echo "ERROR: pkg-config reported DPDK libdir $dpdk_libdir is out of the directory specified with --with-dpdk="
+					echo "ERROR: do you have another DPDK installed in the system?"
+					exit 1
+				fi
+				dpdk_reqs=$(PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$dpdk_libdir/pkgconfig" pkg-config --print-requires libdpdk)
+				if echo $dpdk_reqs | grep "libbsd" > /dev/null 2>&1; then
+					CONFIG[HAVE_LIBBSD]=y
+				fi
+				CFLAGS="${CFLAGS:+$CFLAGS }$(PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$dpdk_libdir/pkgconfig" pkg-config --cflags libdpdk)"
+				dpdk_incdir=$(PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$dpdk_libdir/pkgconfig" pkg-config --variable=includedir libdpdk)
+				check_IPSec_mb "pkg-config" "$dpdk_libdir" "$dpdk_incdir"
+			else
+				echo "Using $dpdk_incdir/rte_build_config.h for additional libs..."
+
+				if grep -F "define RTE_USE_LIBBSD 1" $dpdk_incdir/rte_build_config.h > /dev/null 2>&1; then
+					CONFIG[HAVE_LIBBSD]=y
+				fi
+				check_IPSec_mb "build-config" "$dpdk_libdir" "$dpdk_incdir"
+			fi
+			echo "DPDK libraries: $dpdk_libdir"
+			echo "DPDK includes: $dpdk_incdir"
+			CONFIG[DPDK_DIR]=$dpdk_dir
+			CONFIG[DPDK_LIB_DIR]="$dpdk_libdir"
+			CONFIG[DPDK_INC_DIR]="$dpdk_incdir"
+			CONFIG[DPDK_PKG_CONFIG]=n
 			;;
 		--without-dpdk)
 			CONFIG[DPDK_DIR]=
@@ -286,6 +433,9 @@ for i in "$@"; do
 			check_dir "$i"
 			CONFIG[WPDK_DIR]=$(readlink -f ${i#*=})
 			;;
+		--without-wpdk)
+			CONFIG[WPDK_DIR]=
+			;;
 		--with-env=*)
 			CONFIG[ENV]="${i#*=}"
 			;;
@@ -448,10 +598,30 @@ for i in "$@"; do
 			;;
 		--with-idxd)
 			CONFIG[IDXD]=y
+			CONFIG[IDXD_KERNEL]=n
 			;;
 		--without-idxd)
 			CONFIG[IDXD]=n
 			;;
+		--with-usdt)
+			CONFIG[USDT]=y
+			;;
+		--without-usdt)
+			CONFIG[USDT]=n
+			;;
+		--with-fuzzer)
+			echo "Must specify fuzzer library path with --with-fuzzer"
+			usage
+			exit 1
+			;;
+		--with-fuzzer=*)
+			CONFIG[FUZZER]=y
+			CONFIG[FUZZER_LIB]=$(readlink -f ${i#*=})
+			;;
+		--without-fuzzer)
+			CONFIG[FUZZER]=n
+			CONFIG[FUZZER_LIB]=
+			;;
 		--)
 			break
 			;;
@@ -472,7 +642,7 @@ BUILD_CMD+=(-I/usr/local/include -L/usr/local/lib)
 
 if [[ "${CONFIG[VFIO_USER]}" = "y" ]]; then
 
-	if ! hash cmake; then
+	if ! bash -c "command -v cmake3 cmake" > /dev/null; then
 		echo "ERROR: --with-vfio-user requires cmake"
 		echo "Please install then re-run this script"
 		exit 1
@@ -499,15 +669,19 @@ if [[ "${CONFIG[IDXD]}" = "y" ]]; then
 		cpu_vendor=$(grep -i 'vendor' /proc/cpuinfo --max-count=1)
 	fi
 	if [[ "$cpu_vendor" != *"$intel"* ]]; then
-		echo "ERROR: IDXD cannot be used due to CPU incompatiblity."
+		echo "ERROR: IDXD cannot be used due to CPU incompatibility."
 		exit 1
 	fi
+	if [ -e /usr/include/accel-config/libaccel_config.h ]; then
+		CONFIG[IDXD_KERNEL]=y
+	fi
+
 fi
 
 # Detect architecture and force no ISA-L if non-x86 or non-aarch64 architecture
 if [[ "${CONFIG[ISAL]}" = "y" ]]; then
 	if [[ $arch != x86_64* ]] && [[ $arch != aarch64* ]]; then
-		echo "ERROR: ISA-L cannot be used due to CPU incompatiblity."
+		echo "ERROR: ISA-L cannot be used due to CPU incompatibility."
 		exit 1
 	fi
 fi
@@ -520,7 +694,7 @@ fi
 if [ -z "${CONFIG[ENV]}" ]; then
 	CONFIG[ENV]=$rootdir/lib/env_dpdk
 	echo "Using default SPDK env in ${CONFIG[ENV]}"
-	if [ -z "${CONFIG[DPDK_DIR]}" ]; then
+	if [[ -z "${CONFIG[DPDK_DIR]}" && "${CONFIG[DPDK_PKG_CONFIG]}" == n ]]; then
 		if [ ! -f "$rootdir"/dpdk/config/meson.build ]; then
 			echo "DPDK not found; please specify --with-dpdk=<path> or run:"
 			echo
@@ -528,11 +702,16 @@ if [ -z "${CONFIG[ENV]}" ]; then
 			exit 1
 		else
 			CONFIG[DPDK_DIR]="${rootdir}/dpdk/build"
+			# Default ipsec libs
+			if [[ "${CONFIG[CRYPTO]}" = "y" ]]; then
+				CONFIG[IPSEC_MB]=y
+				CONFIG[IPSEC_MB_DIR]="${rootdir}/intel-ipsec-mb/lib"
+			fi
 			echo "Using default DPDK in ${CONFIG[DPDK_DIR]}"
 		fi
 	fi
 else
-	if [ -n "${CONFIG[DPDK_DIR]}" ]; then
+	if [[ -n "${CONFIG[DPDK_DIR]}" || "${CONFIG[DPDK_PKG_CONFIG]}" == y ]]; then
 		echo "--with-env and --with-dpdk are mutually exclusive."
 		exit 1
 	fi
@@ -549,6 +728,14 @@ else
 	CONFIG[VIRTIO]="n"
 fi
 
+if [[ "${CONFIG[DPDK_PKG_CONFIG]}" == y ]]; then
+	if [[ "${CONFIG[SHARED]}" == n ]]; then
+		# dpdk-devel doesn't provide static libs
+		echo "Build against packaged DPDK requested, enabling shared libraries"
+		CONFIG[SHARED]=y
+	fi
+fi
+
 if [[ $sys_name == "Windows" ]]; then
 	if [ -z "${CONFIG[WPDK_DIR]}" ]; then
 		if [ ! -f "$rootdir"/wpdk/Makefile ]; then
@@ -671,10 +858,6 @@ if [[ "${CONFIG[ISAL]}" = "y" ]] || [[ "${CONFIG[CRYPTO]}" = "y" ]]; then
 		echo "ERROR: ISA-L, compression & crypto require NASM version 2.14 or newer."
 		echo "Please install or upgrade them re-run this script."
 		exit 1
-	else
-		if [[ "${CONFIG[CRYPTO]}" = "y" ]]; then
-			CONFIG[IPSEC_MB]=y
-		fi
 	fi
 fi
 
@@ -687,6 +870,31 @@ if [[ "${CONFIG[PMDK]}" = "y" ]]; then
 	fi
 fi
 
+function dpdk_version() {
+	# Check DPDK version to determine if mlx5_pci driver is supported
+	local dpdk_ver="none"
+	if [[ "${CONFIG[DPDK_DIR]}" == "$rootdir/dpdk/build" ]]; then
+		# DPDK_DIR points at our submodule so ./build may not exist yet. Use
+		# absolute path to lookup the version.
+		dpdk_ver=$(< "$rootdir/dpdk/VERSION")
+	elif [[ -f "${CONFIG[DPDK_DIR]}"/../VERSION ]]; then
+		dpdk_ver=$(< "${CONFIG[DPDK_DIR]}"/../VERSION)
+	fi
+	echo $dpdk_ver
+}
+
+function mlx5_build() {
+	# Check if libmlx5 exists to enable mlx5_pci compress/crypto PMD
+	if ! echo -e '#include <spdk/stdinc.h>\n' \
+		'#include <infiniband/mlx5dv.h>\n' \
+		'#include <infiniband/verbs.h>\n' \
+		'int main(void) { return 0; }\n' \
+		| "${BUILD_CMD[@]}" -lmlx5 -libverbs -I${rootdir}/include -c - 2> /dev/null; then
+		return 1
+	fi
+	return 0
+}
+
 if [[ "${CONFIG[REDUCE]}" = "y" ]]; then
 	if ! echo -e '#include <libpmem.h>\nint main(void) { return 0; }\n' \
 		| "${BUILD_CMD[@]}" -lpmem - 2> /dev/null; then
@@ -694,6 +902,71 @@ if [[ "${CONFIG[REDUCE]}" = "y" ]]; then
 		echo "Please install then re-run this script."
 		exit 1
 	fi
+	# Try to enable mlx5 compress
+	CONFIG[REDUCE_MLX5]="y"
+
+	# Check if libmlx5 exists to enable mlx5_pci compress PMD
+	if ! mlx5_build; then
+		echo "libmlx5 is not found, so disabling DPDK mlx5_pci compress PMD"
+		CONFIG[REDUCE_MLX5]="n"
+	else
+		if [[ "${CONFIG[DPDK_PKG_CONFIG]}" = "y" ]]; then
+			# Check if librte_compress_mlx5 exists in DPDK package
+			if [ ! -f "${CONFIG[DPDK_LIB_DIR]}"/librte_compress_mlx5.so ]; then
+				echo "librte_compress_mlx5 is not found, so disabling DPDK mlx5_pci compress PMD"
+				CONFIG[REDUCE_MLX5]="n"
+			fi
+		else
+			# Check DPDK version to determine if mlx5_pci driver is supported
+			dpdk_ver=$(dpdk_version)
+			if [[ $dpdk_ver = "none" ]]; then
+				echo "Cannot get DPDK version, so disabling DPDK mlx5_pci compress PMD"
+				CONFIG[REDUCE_MLX5]="n"
+			elif [[ -n $dpdk_ver ]] && lt "$dpdk_ver" 21.02.0; then
+				# mlx5_pci for compress is supported by DPDK >- 21.02.0
+				echo "DPDK version ${dpdk_ver} doesn't support mlx5_pci compress PMD"
+				CONFIG[REDUCE_MLX5]="n"
+			elif [[ -n ${CONFIG[DPDK_LIB_DIR]} ]] && [ ! -f "${CONFIG[DPDK_LIB_DIR]}"/librte_compress_mlx5.so ]; then
+				# This is only checked when --with-dpdk or --with-dpdk=* is used
+				echo "librte_compress_mlx5 is not found, so disabling DPDK mlx5_pci compress PMD"
+				CONFIG[REDUCE_MLX5]="n"
+			fi
+		fi
+	fi
+fi
+
+if [[ "${CONFIG[CRYPTO]}" = "y" ]]; then
+	# Try to enable mlx5 crypto
+	CONFIG[CRYPTO_MLX5]="y"
+
+	# Check if libmlx5 exists to enable mlx5_pci compress PMD
+	if ! mlx5_build; then
+		echo "libmlx5 is not found, so disabling DPDK mlx5_pci crypto PMD"
+		CONFIG[CRYPTO_MLX5]="n"
+	else
+		if [[ "${CONFIG[DPDK_PKG_CONFIG]}" = "y" ]]; then
+			# Check if librte_crypto_mlx5 exists in DPDK package
+			if [ ! -f "${CONFIG[DPDK_LIB_DIR]}"/librte_crypto_mlx5.so ]; then
+				echo "librte_crypto_mlx5 is not found, so disabling DPDK mlx5_pci crypto PMD"
+				CONFIG[CRYPTO_MLX5]="n"
+			fi
+		else
+			# Check DPDK version to determine if mlx5_pci driver is supported
+			dpdk_ver=$(dpdk_version)
+			if [[ $dpdk_ver = "none" ]]; then
+				echo "Cannot get DPDK version, so disabling DPDK mlx5_pci crypto PMD"
+				CONFIG[CRYPTO_MLX5]="n"
+			elif [[ -n $dpdk_ver ]] && lt "$dpdk_ver" 22.03.0; then
+				# mlx5_pci for crypto is supported by DPDK >- 22.03.0
+				echo "DPDK version ${dpdk_ver} doesn't support mlx5_pci crypto PMD"
+				CONFIG[CRYPTO_MLX5]="n"
+			elif [[ -n ${CONFIG[DPDK_LIB_DIR]} ]] && [ ! -f "${CONFIG[DPDK_LIB_DIR]}"/librte_crypto_mlx5.so ]; then
+				# This is only checked when --with-dpdk or --with-dpdk=* is used
+				echo "librte_crypto_mlx5 is not found, so disabling DPDK mlx5_pci crypto PMD"
+				CONFIG[CRYPTO_MLX5]="n"
+			fi
+		fi
+	fi
 fi
 
 if [[ "${CONFIG[NVME_CUSE]}" = "y" ]]; then
@@ -809,6 +1082,11 @@ if [ "${CONFIG[CET]}" = "y" ]; then
 	fi
 fi
 
+if [[ "${CONFIG[FUZZER]}" = "y" && "$CC_TYPE" != "clang" ]]; then
+	echo "--with-fuzzer requires setting CC and CXX to clang."
+	exit 1
+fi
+
 if [[ "${CONFIG[ISAL]}" = "y" ]]; then
 	if [ ! -f "$rootdir"/isa-l/autogen.sh ]; then
 		echo "ISA-L was not found; To install ISA-L run:"
@@ -830,6 +1108,20 @@ if [[ "${CONFIG[ISAL]}" = "y" ]]; then
 	cd $rootdir
 fi
 
+# For ARM Neoverse-N1 platform, debug build needs gcc version newer than 8.4
+if [[ "${CONFIG[DEBUG]}" = "y" && $arch = aarch64* && "$CC_TYPE" = "gcc" ]]; then
+	GCC_VERSION=$($CC -dumpfullversion)
+	PART_NUM=$(grep -i -m 1 "CPU part" /proc/cpuinfo | awk '{print $4}')
+
+	if [[ "$(printf '%s\n' "8.4.0" "$GCC_VERSION" | sort -V | head -n1)" != "8.4.0" ]]; then
+		if [[ $PART_NUM = 0xd0c ]]; then
+			echo "WARNING: For ARM Neoverse-N1 platform, debug build needs GCC version newer than 8.4."
+			echo "         Will work around this by using armv8.2-a+crypto as target architecture for now."
+			CONFIG[ARCH]=armv8.2-a+crypto
+		fi
+	fi
+fi
+
 # We are now ready to generate final configuration. But first do sanity
 # check to see if all keys in CONFIG array have its reflection in CONFIG file.
 if (($(grep -cE "^\s*CONFIG_[[:alnum:]_]+=" "$rootdir/CONFIG") != ${#CONFIG[@]})); then
@@ -837,7 +1129,7 @@ if (($(grep -cE "^\s*CONFIG_[[:alnum:]_]+=" "$rootdir/CONFIG") != ${#CONFIG[@]})
 	echo "BUG: Some configuration options are not present in CONFIG file. Please update this file."
 	echo "Missing options in CONFIG (+) file and in current config (-): "
 	diff -u --label "CONFIG file" --label "CONFIG[@]" \
-		<(sed -r -e '/^\s*$/d; /^\s*#.*/d; s/(CONFIG_[[:alnum:]_]+)=.*/\1/g' CONFIG | sort) \
+		<(sed -r -e '/^[[:space:]]*$/d; /^[[:space:]]*#.*/d; s/(CONFIG_[[:alnum:]_]+)=.*/\1/g' CONFIG | sort) \
 		<(printf "CONFIG_%s\n" "${!CONFIG[@]}" | sort)
 	exit 1
 fi
@@ -845,7 +1137,7 @@ fi
 echo -n "Creating mk/config.mk..."
 cp -f $rootdir/CONFIG $rootdir/mk/config.mk
 for key in "${!CONFIG[@]}"; do
-	sed -i.bak -r "s#^\s*CONFIG_${key}=.*#CONFIG_${key}\?=${CONFIG[$key]}#g" $rootdir/mk/config.mk
+	sed -i.bak -r "s#[[:space:]]*CONFIG_${key}=.*#CONFIG_${key}\?=${CONFIG[$key]}#g" $rootdir/mk/config.mk
 done
 # On FreeBSD sed -i 'SUFFIX' - SUFFIX is mandatory. So no way but to delete the backed file.
 rm -f $rootdir/mk/config.mk.bak

deprecation.md

@@ -1,35 +1,21 @@
-# ABI and API Deprecation {#deprecation}
+# Deprecation
+
+## ABI and API Deprecation {#deprecation}
 
 This document details the policy for maintaining stability of SPDK ABI and API.
 
 Major ABI version can change at most once for each quarterly SPDK release.
-ABI versions are managed separately for each library and follow [Semantic Versoning](https://semver.org/).
+ABI versions are managed separately for each library and follow [Semantic Versioning](https://semver.org/).
 
 API and ABI deprecation notices shall be posted in the next section.
 Each entry must describe what will be removed and can suggest the future use or alternative.
 Specific future SPDK release for the removal must be provided.
 ABI cannot be removed without providing deprecation notice for at least single SPDK release.
 
-# Deprecation Notices {#deprecation-notices}
+## Deprecation Notices {#deprecation-notices}
 
-## net
+### nvme
 
-The net library is deprecated and will be removed in the 21.07 release.
-
-## nvmf
-
-The following APIs have been deprecated and will be removed in SPDK 21.07:
-- `spdk_nvmf_poll_group_get_stat` (function in `nvmf.h`),
-- `spdk_nvmf_transport_poll_group_get_stat` (function in `nvmf.h`),
-- `spdk_nvmf_transport_poll_group_free_stat`(function in `nvmf.h`),
-- `spdk_nvmf_rdma_device_stat` (struct in `nvmf.h`),
-- `spdk_nvmf_transport_poll_group_stat` (struct in `nvmf.h`),
-- `poll_group_get_stat` (transport op in `nvmf_transport.h`),
-- `poll_group_free_stat` (transport op in `nvmf_transport.h`).
-Please use `spdk_nvmf_poll_group_dump_stat` and `poll_group_dump_stat` instead.
-
-## rpm
-
-`pkg/spdk.spec` is considered to be deprecated and scheduled for removal in SPDK 21.07.
-Please use `rpmbuild/spdk.spec` instead and see
-[RPM documentation](https://spdk.io/doc/rpm.html) for more details.
+Deprecated `spdk_nvme_ctrlr_reset_async` and `spdk_nvme_ctrlr_reset_poll_async` APIs,
+which will be removed in SPDK 22.01. `spdk_nvme_ctrlr_disconnect`, `spdk_nvme_ctrlr_reconnect_async`,
+and `spdk_nvme_ctrlr_reconnect_poll_async` should be used instead.

doc/Doxyfile

@@ -746,7 +746,7 @@ WARN_IF_DOC_ERROR      = YES
 # parameter documentation, but not about the absence of documentation.
 # The default value is: NO.
 
-WARN_NO_PARAMDOC       = NO
+WARN_NO_PARAMDOC       = YES
 
 # If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when
 # a warning is encountered.
@@ -814,6 +814,7 @@ INPUT                  += \
                          concurrency.md \
                          containers.md \
                          ../deprecation.md \
+                         distributions.md \
                          event.md \
                          ftl.md \
                          gdb_macros.md \
@@ -843,6 +844,7 @@ INPUT                  += \
                          spdk_top.md \
                          ssd_internals.md \
                          system_configuration.md \
+                         usdt.md \
                          userspace.md \
                          vagrant.md \
                          vhost.md \
@@ -1513,7 +1515,7 @@ EXT_LINKS_IN_WINDOW    = NO
 
 FORMULA_FONTSIZE       = 10
 
-# Use the FORMULA_TRANPARENT tag to determine whether or not the images
+# Use the FORMULA_TRANSPARENT tag to determine whether or not the images
 # generated for formulas are transparent PNGs. Transparent PNGs are not
 # supported properly for IE 6.0, but are supported on all modern browsers.
 #

doc/README.md

@@ -1,11 +1,9 @@
-SPDK Documentation
-==================
+# SPDK Documentation
 
 The current version of the SPDK documentation can be found online at
 http://www.spdk.io/doc/
 
-Building the Documentation
-==========================
+## Building the Documentation
 
 To convert the documentation into HTML run `make` in the `doc`
 directory.  The output will be located in `doc/output/html`. Before

doc/accel_fw.md

@@ -10,14 +10,17 @@ acceleration capabilities. ISA/L is used for optimized CRC32C calculation within
 the software module.
 
 The framework includes an API for getting the current capabilities of the
-selected module. See [`spdk_accel_get_capabilities`](https://spdk.io/doc/accel__engine_8h.html) for more details. For the software module, all capabilities will be reported as supported. For the hardware modules, only functions accelerated by hardware will be reported however any function can still be called, it will just be backed by software if it is not reported as a supported capability.
+selected module. See [`spdk_accel_get_capabilities`](https://spdk.io/doc/accel__engine_8h.html) for more details.
+For the software module, all capabilities will be reported as supported. For the hardware modules, only functions
+accelerated by hardware will be reported however any function can still be called, it will just be backed by
+software if it is not reported as a supported capability.
 
-# Acceleration Framework Functions {#accel_functions}
+## Acceleration Framework Functions {#accel_functions}
 
 Functions implemented via the framework can be found in the DoxyGen documentation of the
 framework public header file here [accel_engine.h](https://spdk.io/doc/accel__engine_8h.html)
 
-# Acceleration Framework Design Considerations {#accel_dc}
+## Acceleration Framework Design Considerations {#accel_dc}
 
 The general interface is defined by `/include/accel_engine.h` and implemented
 in `/lib/accel`.  These functions may be called by an SPDK application and in
@@ -32,7 +35,7 @@ optimized implementation.  For example, IOAT does not support the dualcast funct
 in hardware but if the IOAT module has been initialized and the public dualcast API
 is called, it will actually be done via software behind the scenes.
 
-# Acceleration Low Level Libraries {#accel_libs}
+## Acceleration Low Level Libraries {#accel_libs}
 
 Low level libraries provide only the most basic functions that are specific to
 the hardware. Low level libraries are located in the '/lib' directory with the
@@ -46,9 +49,21 @@ functions exposed by the individual low level libraries. Thus, code written this
 way needs to be certain that the underlying hardware exists everywhere that it runs.
 
 The low level library for IOAT is located in `/lib/ioat`.  The low level library
-for DSA is in `/liv/idxd` (IDXD stands for Intel(R) Data Acceleration Driver).
+for DSA is in `/lib/idxd` (IDXD stands for Intel(R) Data Acceleration Driver).
+In `/lib/idxd` folder, SPDK supports to leverage both user space and kernel space driver
+to drive DSA devices. And the following describes each usage scenario:
 
-# Acceleration Plug-In Modules {#accel_modules}
+Leveraging user space idxd driver: The DSA devices are managed by the user space
+driver in a dedicated SPDK process, then the device cannot be shared by another
+process. The benefit of this usage is no kernel dependency.
+
+Leveraging kernel space driver: The DSA devices are managed by kernel
+space drivers. And the Work queues inside the DSA device can be shared among
+different processes. Naturally, it can be used in cloud native scenario. The drawback of
+this usage is the kernel dependency, i.e., idxd driver must be supported and loaded
+in the kernel.
+
+## Acceleration Plug-In Modules {#accel_modules}
 
 Plug-in modules depend on low level libraries to interact with the hardware and
 add additional functionality such as queueing during busy conditions or flow
@@ -57,14 +72,18 @@ the complete implementation of the acceleration component. A module must be
 selected via startup RPC when the application is started. Otherwise, if no startup
 RPC is provided, the framework is available and will use the software plug-in module.
 
-## IOAT Module {#accel_ioat}
+### IOAT Module {#accel_ioat}
 
 To use the IOAT engine, use the RPC [`ioat_scan_accel_engine`](https://spdk.io/doc/jsonrpc.html) before starting the application.
 
-## IDXD Module {#accel_idxd}
+### IDXD Module {#accel_idxd}
 
-To use the DSA engine, use the RPC [`idxd_scan_accel_engine`](https://spdk.io/doc/jsonrpc.html) with an optional parameter of `-c` and provide a configuration number of either 0 or 1. These pre-defined configurations determine how the DSA engine will be setup in terms
-of work queues and engines.  The DSA engine is very flexible allowing for various configurations of these elements to either account for different quality of service requirements or to isolate hardware paths where the back end media is of varying latency (i.e. persistent memory vs DRAM).  The pre-defined configurations are as follows:
+To use the DSA engine, use the RPC [`idxd_scan_accel_engine`](https://spdk.io/doc/jsonrpc.html). With an optional parameter
+of `-c` and providing a configuration number of either 0 or 1, users can determine which pre-defined configuration can be used.
+With an optional parameter of `-k` to use kernel or user space driver. These pre-defined configurations determine how the DSA engine
+will be setup in terms of work queues and engines.  The DSA engine is very flexible allowing for various configurations of
+these elements to either account for different quality of service requirements or to isolate hardware paths where the back
+end media is of varying latency (i.e. persistent memory vs DRAM).  The pre-defined configurations are as follows:
 
 0: A single work queue backed with four DSA engines.  This is a generic configuration
 that enables the hardware to best determine which engine to use as it pulls in new
@@ -80,28 +99,45 @@ of service parameters on the work queues that are not currently utilized by
 the module. Specialized use of DSA may require different configurations that
 can be added to the module as needed.
 
-## Software Module {#accel_sw}
+When a new channel starts, a DSA device will be assigned to the channel. The accel
+idxd module has been tuned for the most likely best performance case. The result
+is that there is a limited number of channels that can be supported based on the
+number of DSA devices in the system.  Additionally, for best performance, the accel
+idxd module will only use DSA devices on the same socket as the requesting
+channel/thread.  If an error occurs on initialization indicating that there are no
+more DSA devices available either try fewer threads or, if on a 2 socket system,
+try spreading threads across cores if possible.
+
+### How to use kernel idxd driver {#accel_idxd_kernel}
+
+There are several dependencies to leverage kernel idxd driver for driving DSA devices.
+
+1 Linux kernel support: To leverage kernel space idxd driver, you need to have a Linux kernel with
+`idxd` driver loaded with scalable mode. And currently SPDK uses the character device while `idxd` driver is
+enabled in the kernel. So when booting the machine, we need to add additional configuration in
+the grub, i.e, revise the kernel boot commandline `intel_iommu=on,sm_on` with VT-d turned on in BIOS.
+
+2 User library dependency: Users need to install `idxd-config` library. For example, users can
+download the library from [idxd-config repo](https://github.com/intel/idxd-config). After the
+library is installed, users can use the `accel-config` command to configure the work queues(WQs)
+of the idxd devices managed by the kernel with the following steps:
+
+```bash
+accel-config disable-wq dsa0/wq0.1
+accel-config disable-device dsa0
+accel-config config-wq --group-id=0 --mode=dedicated --wq-size=16 --type=user --name="MyApp1"
+ --priority=10 --block-on-fault=1 dsa0/wq0.1
+accel-config config-engine dsa0/engine0.1 --group-id=0
+accel-config enable-device dsa0
+accel-config enable-wq dsa0/wq0.1
+```
+
+For more details on the usage of `idxd-config`, please refer to
+[idxd-config usage](https://github.com/intel/idxd-config/tree/master/Documentation/accfg).
+
+### Software Module {#accel_sw}
 
 The software module is enabled by default. If no hardware engine is explicitly
 enabled via startup RPC as discussed earlier, the software module will use ISA-L
 if available for functions such as CRC32C. Otherwise, standard glibc calls are
 used to back the framework API.
-
-## Batching {#batching}
-
-Batching is exposed by the acceleration framework and provides an interface to
-batch sets of commands up and then submit them with a single command.  The public
-API is consistent with the implementation however each plug-in module behaves
-differently depending on its capabilities.
-
-The DSA engine has complete support for batching all supported commands together
-into one submission. This is advantageous as it reduces the overhead incurred in
-the submission process to the hardware.
-
-The software engine supports batching only to be consistent with the framework API.
-In software there is no savings by batching sets of commands versus submitting them
-individually.
-
-The IOAT engine supports batching but it is only beneficial for `memmove` and `memfill`
-as these are supported by the hardware.  All other commands can be batched and the
-framework will manage all other commands via software.

doc/applications.md

@@ -151,7 +151,7 @@ Whenever the `CPU mask` is mentioned it is a string in one of the following form
 
 The following CPU masks are equal and correspond to CPUs 0, 1, 2, 8, 9, 10, 11 and 12:
 
-~~~
+~~~bash
 0x1f07
 0x1F07
 1f07

doc/bdev.md

@@ -1,10 +1,11 @@
 # Block Device User Guide {#bdev}
 
-# Target Audience {#bdev_ug_targetaudience}
+## Target Audience {#bdev_ug_targetaudience}
 
-This user guide is intended for software developers who have knowledge of block storage, storage drivers, issuing JSON-RPC commands and storage services such as RAID, compression, crypto, and others.
+This user guide is intended for software developers who have knowledge of block storage, storage drivers, issuing JSON-RPC
+commands and storage services such as RAID, compression, crypto, and others.
 
-# Introduction {#bdev_ug_introduction}
+## Introduction {#bdev_ug_introduction}
 
 The SPDK block device layer, often simply called *bdev*, is a C library
 intended to be equivalent to the operating system block storage layer that
@@ -26,7 +27,7 @@ device underneath (please refer to @ref bdev_module for details). SPDK
 provides also vbdev modules which creates block devices on existing bdev. For
 example @ref bdev_ug_logical_volumes or @ref bdev_ug_gpt
 
-# Prerequisites {#bdev_ug_prerequisites}
+## Prerequisites {#bdev_ug_prerequisites}
 
 This guide assumes that you can already build the standard SPDK distribution
 on your platform. The block device layer is a C library with a single public
@@ -39,14 +40,14 @@ directly from SPDK application by running `scripts/rpc.py rpc_get_methods`.
 Detailed help for each command can be displayed by adding `-h` flag as a
 command parameter.
 
-# Configuring Block Device Modules {#bdev_ug_general_rpcs}
+## Configuring Block Device Modules {#bdev_ug_general_rpcs}
 
 Block devices can be configured using JSON RPCs. A complete list of available RPC commands
 with detailed information can be found on the @ref jsonrpc_components_bdev page.
 
-# Common Block Device Configuration Examples
+## Common Block Device Configuration Examples
 
-# Ceph RBD {#bdev_config_rbd}
+## Ceph RBD {#bdev_config_rbd}
 
 The SPDK RBD bdev driver provides SPDK block layer access to Ceph RADOS block
 devices (RBD). Ceph RBD devices are accessed via librbd and librados libraries
@@ -69,7 +70,7 @@ To resize a bdev use the bdev_rbd_resize command.
 
 This command will resize the Rbd0 bdev to 4096 MiB.
 
-# Compression Virtual Bdev Module {#bdev_config_compress}
+## Compression Virtual Bdev Module {#bdev_config_compress}
 
 The compression bdev module can be configured to provide compression/decompression
 services for an underlying thinly provisioned logical volume. Although the underlying
@@ -81,7 +82,12 @@ for detailed information.
 The vbdev module relies on the DPDK CompressDev Framework to provide all compression
 functionality. The framework provides support for many different software only
 compression modules as well as hardware assisted support for Intel QAT. At this
-time the vbdev module supports the DPDK drivers for ISAL and QAT.
+time the vbdev module supports the DPDK drivers for ISAL, QAT and mlx5_pci.
+
+mlx5_pci driver works with BlueField 2 SmartNIC and requires additional configuration of DPDK
+environment to enable compression function. It can be done via SPDK event library by configuring
+`env_context` member of `spdk_app_opts` structure or by passing corresponding CLI arguments in the
+following form: `--allow=BDF,class=compress`, e.g. `--allow=0000:01:00.0,class=compress`.
 
 Persistent memory is used to store metadata associated with the layout of the data on the
 backing device. SPDK relies on [PMDK](http://pmem.io/pmdk/) to interface persistent memory so any hardware
@@ -128,7 +134,7 @@ all volumes, if used it will return the name or an error that the device does no
 
 `rpc.py bdev_compress_get_orphans --name COMP_Nvme0n1`
 
-# Crypto Virtual Bdev Module {#bdev_config_crypto}
+## Crypto Virtual Bdev Module {#bdev_config_crypto}
 
 The crypto virtual bdev module can be configured to provide at rest data encryption
 for any underlying bdev. The module relies on the DPDK CryptoDev Framework to provide
@@ -165,7 +171,7 @@ To remove the vbdev use the bdev_crypto_delete command.
 
 `rpc.py bdev_crypto_delete CryNvmeA`
 
-# Delay Bdev Module {#bdev_config_delay}
+## Delay Bdev Module {#bdev_config_delay}
 
 The delay vbdev module is intended to apply a predetermined additional latency on top of a lower
 level bdev. This enables the simulation of the latency characteristics of a device during the functional
@@ -196,13 +202,13 @@ Example command:
 
 `rpc.py bdev_delay_delete delay0`
 
-# GPT (GUID Partition Table) {#bdev_config_gpt}
+## GPT (GUID Partition Table) {#bdev_config_gpt}
 
 The GPT virtual bdev driver is enabled by default and does not require any configuration.
 It will automatically detect @ref bdev_ug_gpt on any attached bdev and will create
 possibly multiple virtual bdevs.
 
-## SPDK GPT partition table {#bdev_ug_gpt}
+### SPDK GPT partition table {#bdev_ug_gpt}
 
 The SPDK partition type GUID is `7c5222bd-8f5d-4087-9c00-bf9843c7b58c`. Existing SPDK bdevs
 can be exposed as Linux block devices via NBD and then can be partitioned with
@@ -228,9 +234,9 @@ Example command
 
 `rpc.py nbd_stop_disk -n /dev/nbd0`
 
-## Creating a GPT partition table using NBD {#bdev_ug_gpt_create_part}
+### Creating a GPT partition table using NBD {#bdev_ug_gpt_create_part}
 
-~~~
+~~~bash
 # Expose bdev Nvme0n1 as kernel block device /dev/nbd0 by JSON-RPC
 rpc.py nbd_start_disk Nvme0n1 /dev/nbd0
 
@@ -252,7 +258,7 @@ rpc.py nbd_stop_disk /dev/nbd0
 # Nvme0n1p1 in SPDK applications.
 ~~~
 
-# iSCSI bdev {#bdev_config_iscsi}
+## iSCSI bdev {#bdev_config_iscsi}
 
 The SPDK iSCSI bdev driver depends on libiscsi and hence is not enabled by default.
 In order to use it, build SPDK with an extra `--with-iscsi-initiator` configure option.
@@ -265,7 +271,7 @@ with `iqn.2016-06.io.spdk:init` as the reported initiator IQN.
 The URL is in the following format:
 `iscsi://[<username>[%<password>]@]<host>[:<port>]/<target-iqn>/<lun>`
 
-# Linux AIO bdev {#bdev_config_aio}
+## Linux AIO bdev {#bdev_config_aio}
 
 The SPDK AIO bdev driver provides SPDK block layer access to Linux kernel block
 devices or a file on a Linux filesystem via Linux AIO. Note that O_DIRECT is
@@ -288,7 +294,7 @@ To delete an aio bdev use the bdev_aio_delete command.
 
 `rpc.py bdev_aio_delete aio0`
 
-# OCF Virtual bdev {#bdev_config_cas}
+## OCF Virtual bdev {#bdev_config_cas}
 
 OCF virtual bdev module is based on [Open CAS Framework](https://github.com/Open-CAS/ocf) - a
 high performance block storage caching meta-library.
@@ -315,7 +321,7 @@ During removal OCF-cache will be stopped and all cached data will be written to
 Note that OCF has a per-device RAM requirement. More details can be found in the
 [OCF documentation](https://open-cas.github.io/guide_system_requirements.html).
 
-# Malloc bdev {#bdev_config_malloc}
+## Malloc bdev {#bdev_config_malloc}
 
 Malloc bdevs are ramdisks. Because of its nature they are volatile. They are created from hugepage memory given to SPDK
 application.
@@ -328,7 +334,7 @@ Example command for removing malloc bdev:
 
 `rpc.py bdev_malloc_delete Malloc0`
 
-# Null {#bdev_config_null}
+## Null {#bdev_config_null}
 
 The SPDK null bdev driver is a dummy block I/O target that discards all writes and returns undefined
 data for reads.  It is useful for benchmarking the rest of the bdev I/O stack with minimal block
@@ -345,7 +351,7 @@ To delete a null bdev use the bdev_null_delete command.
 
 `rpc.py bdev_null_delete Null0`
 
-# NVMe bdev {#bdev_config_nvme}
+## NVMe bdev {#bdev_config_nvme}
 
 There are two ways to create block device based on NVMe device in SPDK. First
 way is to connect local PCIe drive and second one is to connect NVMe-oF device.
@@ -367,7 +373,7 @@ To remove an NVMe controller use the bdev_nvme_detach_controller command.
 
 This command will remove NVMe bdev named Nvme0.
 
-## NVMe bdev character device {#bdev_config_nvme_cuse}
+### NVMe bdev character device {#bdev_config_nvme_cuse}
 
 This feature is considered as experimental. You must configure with --with-nvme-cuse
 option to enable this RPC.
@@ -391,14 +397,14 @@ with command:
 
 `rpc.py bdev_nvme_cuse_unregister -n Nvme0`
 
-# Logical volumes {#bdev_ug_logical_volumes}
+## Logical volumes {#bdev_ug_logical_volumes}
 
 The Logical Volumes library is a flexible storage space management system. It allows
 creating and managing virtual block devices with variable size on top of other bdevs.
 The SPDK Logical Volume library is built on top of @ref blob. For detailed description
 please refer to @ref lvol.
 
-## Logical volume store {#bdev_ug_lvol_store}
+### Logical volume store {#bdev_ug_lvol_store}
 
 Before creating any logical volumes (lvols), an lvol store has to be created first on
 selected block device. Lvol store is lvols vessel responsible for managing underlying
@@ -437,7 +443,7 @@ Example commands
 
 `rpc.py bdev_lvol_delete_lvstore -l lvs`
 
-## Lvols {#bdev_ug_lvols}
+### Lvols {#bdev_ug_lvols}
 
 To create lvols on existing lvol store user should use `bdev_lvol_create` RPC command.
 Each created lvol will be represented by new bdev.
@@ -448,7 +454,7 @@ Example commands
 
 `rpc.py bdev_lvol_create lvol2 25 -u 330a6ab2-f468-11e7-983e-001e67edf35d`
 
-# Passthru {#bdev_config_passthru}
+## Passthru {#bdev_config_passthru}
 
 The SPDK Passthru virtual block device module serves as an example of how to write a
 virtual block device module. It implements the required functionality of a vbdev module
@@ -460,7 +466,7 @@ Example commands
 
 `rpc.py bdev_passthru_delete pt`
 
-# Pmem {#bdev_config_pmem}
+## Pmem {#bdev_config_pmem}
 
 The SPDK pmem bdev driver uses pmemblk pool as the target for block I/O operations. For
 details on Pmem memory please refer to PMDK documentation on http://pmem.io website.
@@ -486,7 +492,7 @@ Example command
 
 `rpc.py bdev_pmem_delete_pool /path/to/pmem_pool`
 
-To create bdev based on pmemblk pool file user should use `bdev_pmem_create ` RPC
+To create bdev based on pmemblk pool file user should use `bdev_pmem_create` RPC
 command.
 
 Example command
@@ -497,7 +503,7 @@ To remove a block device representation use the bdev_pmem_delete command.
 
 `rpc.py bdev_pmem_delete pmem`
 
-# RAID {#bdev_ug_raid}
+## RAID {#bdev_ug_raid}
 
 RAID virtual bdev module provides functionality to combine any SPDK bdevs into
 one RAID bdev. Currently SPDK supports only RAID 0. RAID functionality does not
@@ -517,7 +523,7 @@ Example commands
 
 `rpc.py bdev_raid_delete Raid0`
 
-# Split {#bdev_ug_split}
+## Split {#bdev_ug_split}
 
 The split block device module takes an underlying block device and splits it into
 several smaller equal-sized virtual block devices. This serves as an example to create
@@ -539,7 +545,7 @@ To remove the split bdevs, use the `bdev_split_delete` command with th
 
 `rpc.py bdev_split_delete bdev_b0`
 
-# Uring {#bdev_ug_uring}
+## Uring {#bdev_ug_uring}
 
 The uring bdev module issues I/O to kernel block devices using the io_uring Linux kernel API. This module requires liburing.
 For more information on io_uring refer to kernel [IO_uring] (https://kernel.dk/io_uring.pdf)
@@ -556,7 +562,7 @@ To remove a uring bdev use the `bdev_uring_delete` RPC.
 
 `rpc.py bdev_uring_delete bdev_u0`
 
-# Virtio Block {#bdev_config_virtio_blk}
+## Virtio Block {#bdev_config_virtio_blk}
 
 The Virtio-Block driver allows creating SPDK bdevs from Virtio-Block devices.
 
@@ -577,7 +583,7 @@ Virtio-Block devices can be removed with the following command
 
 `rpc.py bdev_virtio_detach_controller VirtioBlk0`
 
-# Virtio SCSI {#bdev_config_virtio_scsi}
+## Virtio SCSI {#bdev_config_virtio_scsi}
 
 The Virtio-SCSI driver allows creating SPDK block devices from Virtio-SCSI LUNs.
 

doc/bdevperf.md

@@ -73,6 +73,7 @@ length    | 100% of bdev size | End I/O at `offset`+`length` on the bdev
 rw        |                   | Type of I/O pattern
 
 Available rw types:
+
 - read
 - randread
 - write

doc/blob.md

@@ -1,6 +1,6 @@
 # Blobstore Programmer's Guide {#blob}
 
-# In this document {#blob_pg_toc}
+## In this document {#blob_pg_toc}
 
 * @ref blob_pg_audience
 * @ref blob_pg_intro
@@ -57,56 +57,17 @@ The Blobstore defines a hierarchy of storage abstractions as follows.
   Blobstore owns the entire underlying device which is made up of a private Blobstore metadata region and the collection of
   blobs as managed by the application.
 
-@htmlonly
-
-  <div id="blob_hierarchy"></div>
-
-  <script>
-    let elem = document.getElementById('blob_hierarchy');
-
-    let canvasWidth = 800;
-    let canvasHeight = 200;
-    var two = new Two({ width: 800, height: 200 }).appendTo(elem);
-
-    var blobRect = two.makeRectangle(canvasWidth / 2, canvasHeight / 2, canvasWidth, canvasWidth);
-    blobRect.fill = '#7ED3F7';
-
-    var blobText = two.makeText('Blob', canvasWidth / 2, 10, { alignment: 'center'});
-
-    for (var i = 0; i < 2; i++) {
-        let clusterWidth = 400;
-        let clusterHeight = canvasHeight;
-        var clusterRect = two.makeRectangle((clusterWidth / 2) + (i * clusterWidth),
-                                            clusterHeight / 2,
-                                            clusterWidth - 10,
-                                            clusterHeight - 50);
-        clusterRect.fill = '#00AEEF';
-
-        var clusterText =  two.makeText('Cluster',
-                                        (clusterWidth / 2) + (i * clusterWidth),
-                                        35,
-                                        { alignment: 'center', fill: 'white' });
-
-        for (var j = 0; j < 4; j++) {
-            let pageWidth = 100;
-            let pageHeight = canvasHeight;
-            var pageRect = two.makeRectangle((pageWidth / 2) + (j * pageWidth) + (i * clusterWidth),
-                                             pageHeight / 2,
-                                             pageWidth - 20,
-                                             pageHeight - 100);
-            pageRect.fill = '#003C71';
-
-            var pageText =  two.makeText('Page',
-                                         (pageWidth / 2) + (j * pageWidth) + (i * clusterWidth),
-                                         pageHeight / 2,
-                                         { alignment: 'center', fill: 'white' });
-        }
-    }
-
-    two.update();
-  </script>
-
-@endhtmlonly
+```text
++-----------------------------------------------------------------+
+|                              Blob                               |
+| +-----------------------------+ +-----------------------------+ |
+| |           Cluster           | |           Cluster           | |
+| | +----+ +----+ +----+ +----+ | | +----+ +----+ +----+ +----+ | |
+| | |Page| |Page| |Page| |Page| | | |Page| |Page| |Page| |Page| | |
+| | +----+ +----+ +----+ +----+ | | +----+ +----+ +----+ +----+ | |
+| +-----------------------------+ +-----------------------------+ |
++-----------------------------------------------------------------+
+```
 
 ### Atomicity
 
@@ -295,19 +256,23 @@ contribute to the Blobstore effort itself.
 The Blobstore owns the entire storage device. The device is divided into clusters starting from the beginning, such
 that cluster 0 begins at the first logical block.
 
-    LBA 0                                   LBA N
-    +-----------+-----------+-----+-----------+
-    | Cluster 0 | Cluster 1 | ... | Cluster N |
-    +-----------+-----------+-----+-----------+
+```text
+LBA 0                                   LBA N
++-----------+-----------+-----+-----------+
+| Cluster 0 | Cluster 1 | ... | Cluster N |
++-----------+-----------+-----+-----------+
+```
 
 Cluster 0 is special and has the following format, where page 0 is the first page of the cluster:
 
-    +--------+-------------------+
-    | Page 0 | Page 1 ... Page N |
-    +--------+-------------------+
-    | Super  |  Metadata Region  |
-    | Block  |                   |
-    +--------+-------------------+
+```text
++--------+-------------------+
+| Page 0 | Page 1 ... Page N |
++--------+-------------------+
+| Super  |  Metadata Region  |
+| Block  |                   |
++--------+-------------------+
+```
 
 The super block is a single page located at the beginning of the partition. It contains basic information about
 the Blobstore. The metadata region is the remainder of cluster 0 and may extend to additional clusters. Refer

doc/blobfs.md

@@ -1,8 +1,8 @@
 # BlobFS (Blobstore Filesystem) {#blobfs}
 
-# BlobFS Getting Started Guide {#blobfs_getting_started}
+## BlobFS Getting Started Guide {#blobfs_getting_started}
 
-# RocksDB Integration {#blobfs_rocksdb}
+## RocksDB Integration {#blobfs_rocksdb}
 
 Clone and build the SPDK repository as per https://github.com/spdk/spdk
 
@@ -68,7 +68,7 @@ At this point, RocksDB is ready for testing with SPDK.  Three `db_bench` paramet
 SPDK has a set of scripts which will run `db_bench` against a variety of workloads and capture performance and profiling
 data.  The primary script is `test/blobfs/rocksdb/rocksdb.sh`.
 
-# FUSE
+## FUSE
 
 BlobFS provides a FUSE plug-in to mount an SPDK BlobFS as a kernel filesystem for inspection or debug purposes.
 The FUSE plug-in requires fuse3 and will be built automatically when fuse3 is detected on the system.
@@ -79,7 +79,7 @@ test/blobfs/fuse/fuse /usr/local/etc/spdk/rocksdb.json Nvme0n1 /mnt/fuse
 
 Note that the FUSE plug-in has some limitations - see the list below.
 
-# Limitations
+## Limitations
 
 * BlobFS has primarily been tested with RocksDB so far, so any use cases different from how RocksDB uses a filesystem
   may run into issues.  BlobFS will be tested in a broader range of use cases after this initial release.

doc/ci_tools.md

@@ -4,3 +4,4 @@ Section describing tools used by CI to verify integrity of the submitted
 patches ([status](https://ci.spdk.io)).
 
 - @subpage shfmt
+- @subpage distributions

doc/compression.md

@@ -88,7 +88,7 @@ In these examples, the value "X" will represent the special value (2^64-1) descr
 
 ### Initial Creation
 
-```
+```text
                   +--------------------+
   Backing Device  |                    |
                   +--------------------+
@@ -123,7 +123,7 @@ In these examples, the value "X" will represent the special value (2^64-1) descr
   store the 16KB of data.
 * Write the chunk map index to entry 2 in the logical map.
 
-```
+```text
                   +--------------------+
   Backing Device  |01                  |
                   +--------------------+
@@ -157,7 +157,7 @@ In these examples, the value "X" will represent the special value (2^64-1) descr
 * Write (2, X, X, X) to the chunk map.
 * Write the chunk map index to entry 0 in the logical map.
 
-```
+```text
                   +--------------------+
   Backing Device  |012                 |
                   +--------------------+
@@ -205,7 +205,7 @@ In these examples, the value "X" will represent the special value (2^64-1) descr
 * Free chunk map 1 back to the free chunk map list.
 * Free backing IO unit 2 back to the free backing IO unit list.
 
-```
+```text
                   +--------------------+
   Backing Device  |01 34               |
                   +--------------------+

doc/concurrency.md

@@ -1,6 +1,6 @@
 # Message Passing and Concurrency {#concurrency}
 
-# Theory
+## Theory
 
 One of the primary aims of SPDK is to scale linearly with the addition of
 hardware. This can mean many things in practice. For instance, moving from one
@@ -56,7 +56,7 @@ data isn't mutated very often, but is read very frequently, and is often
 employed in the I/O path. This of course trades memory size for computational
 efficiency, so it is used in only the most critical code paths.
 
-# Message Passing Infrastructure
+## Message Passing Infrastructure
 
 SPDK provides several layers of message passing infrastructure. The most
 fundamental libraries in SPDK, for instance, don't do any message passing on
@@ -110,7 +110,7 @@ repeatedly call `spdk_thread_poll()` on each `spdk_thread()` that exists. This
 makes SPDK very portable to a wide variety of asynchronous, event-based
 frameworks such as [Seastar](https://www.seastar.io) or [libuv](https://libuv.org/).
 
-# The event Framework
+## The event Framework
 
 The SPDK project didn't want to officially pick an asynchronous, event-based
 framework for all of the example applications it shipped with, in the interest
@@ -122,7 +122,7 @@ signal handlers to cleanly shutdown, and basic command line option parsing.
 Only established applications should consider directly integrating the lower
 level libraries.
 
-# Limitations of the C Language
+## Limitations of the C Language
 
 Message passing is efficient, but it results in asynchronous code.
 Unfortunately, asynchronous code is a challenge in C. It's often implemented by
@@ -140,6 +140,7 @@ function `foo` performs some asynchronous operation and when that completes
 function `bar` is called, then function `bar` performs some operation that
 calls function `baz` on completion, a good way to write it is as such:
 
+```c
     void baz(void *ctx) {
             ...
     }
@@ -151,6 +152,7 @@ calls function `baz` on completion, a good way to write it is as such:
     void foo(void *ctx) {
             async_op(bar, ctx);
     }
+```
 
 Don't split these functions up - keep them as a nice unit that can be read from bottom to top.
 
@@ -162,6 +164,7 @@ them in C we can still write them out by hand. As an example, here's a
 callback chain that performs `foo` 5 times and then calls `bar` - effectively
 an asynchronous for loop.
 
+```c
     enum states {
             FOO_START = 0,
             FOO_END,
@@ -244,6 +247,7 @@ an asynchronous for loop.
 
             run_state_machine(sm);
     }
+```
 
 This is complex, of course, but the `run_state_machine` function can be read
 from top to bottom to get a clear overview of what's happening in the code

doc/containers.md

@@ -4,12 +4,12 @@ This is a living document as there are many ways to use containers with
 SPDK. As new usages are identified and tested, they will be documented
 here.
 
-# In this document {#containers_toc}
+## In this document {#containers_toc}
 
 * @ref kata_containers_with_spdk_vhost
 * @ref spdk_in_docker
 
-# Using SPDK vhost target to provide volume service to Kata Containers and Docker {#kata_containers_with_spdk_vhost}
+## Using SPDK vhost target to provide volume service to Kata Containers and Docker {#kata_containers_with_spdk_vhost}
 
 [Kata Containers](https://katacontainers.io) can build a secure container
 runtime with lightweight virtual machines that feel and perform like
@@ -23,7 +23,7 @@ In addition, a container manager like Docker, can be configured easily to launch
 a Kata container with an SPDK vhost-user block device. For operating details, visit
 Kata containers use-case [Setup to run SPDK vhost-user devices with Kata Containers and Docker](https://github.com/kata-containers/documentation/blob/master/use-cases/using-SPDK-vhostuser-and-kata.md#host-setup-for-vhost-user-devices)
 
-# Containerizing an SPDK Application for Docker {#spdk_in_docker}
+## Containerizing an SPDK Application for Docker {#spdk_in_docker}
 
 There are no SPDK specific changes needed to run an SPDK based application in
 a docker container, however this quick start guide should help you as you
@@ -78,7 +78,8 @@ Your output should look something like this:
 ~~~{.sh}
 $ sudo docker run --privileged -v //dev//hugepages://dev//hugepages hello:1.0
 Starting SPDK v20.01-pre git sha1 80da95481 // DPDK 19.11.0 initialization...
-[ DPDK EAL parameters: hello_world -c 0x1 --log-level=lib.eal:6 --log-level=lib.cryptodev:5 --log-level=user1:6 --iova-mode=pa --base-virtaddr=0x200000000000 --match-allocations --file-prefix=spdk0 --proc-type=auto ]
+[ DPDK EAL parameters: hello_world -c 0x1 --log-level=lib.eal:6 --log-level=lib.cryptodev:5 --log-level=user1:6 --iova-mode=pa
+--base-virtaddr=0x200000000000 --match-allocations --file-prefix=spdk0 --proc-type=auto ]
 EAL: No available hugepages reported in hugepages-1048576kB
 Initializing NVMe Controllers
 Attaching to 0000:06:00.0

doc/distributions.md

@@ -0,0 +1,64 @@
+# distributions {#distributions}
+
+## In this document {#distros_toc}
+
+* @ref distros_overview
+* @ref linux_list
+* @ref freebsd_list
+
+## Overview {#distros_overview}
+
+CI pool uses different flavors of `Linux` and `FreeBSD` distributions which are
+used as a base for all the tests run against submitted patches. Below is the
+listing which covers all currently supported versions and the related CI
+jobs (see [status](https://ci.spdk.io) as a reference).
+
+## Linux distributions {#linux_list}
+
+* Fedora: Trying to follow new release as per the release cycle whenever
+          possible. Currently at `Fedora33`.
+
+```list
+- autobuild-vg-autotest
+- clang-vg-autotest
+- iscsi*-vg-autotest
+- nvme-vg-autotest
+- nvmf*-vg-autotest
+- scanbuild-vg-autotest
+- unittest-vg-autotest
+- vhost-initiator-vg-autotest
+```
+
+Jobs listed below are run on bare-metal systems where version of
+Fedora may vary. In the future these will be aligned with the
+`vg` jobs.
+
+```list
+- BlobFS-autotest
+- crypto-autotest
+- nvme-phy-autotest
+- nvmf*-phy-autotest
+- vhost-autotest
+```
+
+* Ubuntu: Last two LTS releases. Currently `20.04` and `18.04`.
+
+```list
+- ubuntu18-vg-autotest
+- ubuntu20-vg-autotest
+```
+
+* CentOS: Maintained releases. Currently `7.9` and `8.2`.
+
+```list
+- centos7-vg-autotest
+- centos8-vg-autotest
+```
+
+## FreeBSD distributions {#freebsd_list}
+
+* FreeBSD: Production release. Currently `12.2`.
+
+```list
+- freebsd-vg-autotest
+```

doc/event.md

@@ -14,7 +14,7 @@ concurrency.
 
 The event framework public interface is defined in event.h.
 
-# Event Framework Design Considerations {#event_design}
+## Event Framework Design Considerations {#event_design}
 
 Simple server applications can be written in a single-threaded fashion. This
 allows for straightforward code that can maintain state without any locking or
@@ -27,9 +27,9 @@ synchronization. Unfortunately, in many real-world cases, the connections are
 not entirely independent and cross-thread shared state is necessary. SPDK
 provides an event framework to help solve this problem.
 
-# SPDK Event Framework Components {#event_components}
+## SPDK Event Framework Components {#event_components}
 
-## Events {#event_component_events}
+### Events {#event_component_events}
 
 To accomplish cross-thread communication while minimizing synchronization
 overhead, the framework provides message passing in the form of events. The
@@ -45,7 +45,7 @@ asynchronous operations to achieve concurrency. Asynchronous I/O may be issued
 with a non-blocking function call, and completion is typically signaled using
 a callback function.
 
-## Reactors {#event_component_reactors}
+### Reactors {#event_component_reactors}
 
 Each reactor has a lock-free queue for incoming events to that core, and
 threads from any core may insert events into the queue of any other core. The
@@ -54,7 +54,7 @@ in first-in, first-out order as they are received. Event functions should
 never block and should preferably execute very quickly, since they are called
 directly from the event loop on the destination core.
 
-## Pollers {#event_component_pollers}
+### Pollers {#event_component_pollers}
 
 The framework also defines another type of function called a poller. Pollers
 may be registered with the spdk_poller_register() function. Pollers, like
@@ -66,10 +66,18 @@ intended to poll hardware as a replacement for interrupts. Normally, pollers
 are executed on every iteration of the main event loop. Pollers may also be
 scheduled to execute periodically on a timer if low latency is not required.
 
-## Application Framework {#event_component_app}
+### Application Framework {#event_component_app}
 
 The framework itself is bundled into a higher level abstraction called an "app". Once
 spdk_app_start() is called, it will block the current thread until the application
 terminates by calling spdk_app_stop() or an error condition occurs during the
 initialization code within spdk_app_start(), itself, before invoking the caller's
 supplied function.
+
+### Custom shutdown callback {#event_component_shutdown}
+
+When creating SPDK based application user may add custom shutdown callback which
+will be called before the application framework starts the shutdown process.
+To do that set shutdown_cb function callback in spdk_app_opts structure passed
+to spdk_app_start(). Custom shutdown callback should call spdk_app_stop() before
+returning to continue application shutdown process.

doc/ftl.md

@@ -5,11 +5,11 @@ implementing bdev_zone interface.
 It handles the logical to physical address mapping, responds to the asynchronous
 media management events, and manages the defragmentation process.
 
-# Terminology {#ftl_terminology}
+## Terminology {#ftl_terminology}
 
-## Logical to physical address map
+### Logical to physical address map
 
- * Shorthand: L2P
+- Shorthand: L2P
 
 Contains the mapping of the logical addresses (LBA) to their on-disk physical location. The LBAs
 are contiguous and in range from 0 to the number of surfaced blocks (the number of spare blocks
@@ -17,7 +17,7 @@ are calculated during device formation and are subtracted from the available add
 spare blocks account for zones going offline throughout the lifespan of the device as well as
 provide necessary buffer for data [defragmentation](#ftl_reloc).
 
-## Band {#ftl_band}
+### Band {#ftl_band}
 
 A band describes a collection of zones, each belonging to a different parallel unit. All writes to
 a band follow the same pattern - a batch of logical blocks is written to one zone, another batch
@@ -27,6 +27,7 @@ well as their validity, as some of the data will be invalidated by subsequent wr
 logical address. The L2P mapping can be restored from the SSD by reading this information in order
 from the oldest band to the youngest.
 
+```text
              +--------------+        +--------------+                        +--------------+
     band 1   |   zone 1     +--------+    zone 1    +---- --- --- --- --- ---+     zone 1   |
              +--------------+        +--------------+                        +--------------+
@@ -42,20 +43,23 @@ from the oldest band to the youngest.
              +--------------+        +--------------+                        +--------------+
 
               parallel unit 1              pu 2                                    pu n
+```
 
 The address map and valid map are, along with a several other things (e.g. UUID of the device it's
 part of, number of surfaced LBAs, band's sequence number, etc.), parts of the band's metadata. The
 metadata is split in two parts:
 
+```text
        head metadata               band's data               tail metadata
     +-------------------+-------------------------------+------------------------+
     |zone 1 |...|zone n |...|...|zone 1 |...|           | ... |zone  m-1 |zone  m|
     |block 1|   |block 1|   |   |block x|   |           |     |block y   |block y|
     +-------------------+-------------+-----------------+------------------------+
+```
 
- * the head part, containing information already known when opening the band (device's UUID, band's
-   sequence number, etc.), located at the beginning blocks of the band,
- * the tail part, containing the address map and the valid map, located at the end of the band.
+- the head part, containing information already known when opening the band (device's UUID, band's
+  sequence number, etc.), located at the beginning blocks of the band,
+- the tail part, containing the address map and the valid map, located at the end of the band.
 
 Bands are written sequentially (in a way that was described earlier). Before a band can be written
 to, all of its zones need to be erased. During that time, the band is considered to be in a `PREP`
@@ -64,15 +68,16 @@ is being written. Then the band moves to the `OPEN` state and actual user data c
 band. Once the whole available space is filled, tail metadata is written and the band transitions to
 `CLOSING` state. When that finishes the band becomes `CLOSED`.
 
-## Ring write buffer {#ftl_rwb}
+### Ring write buffer {#ftl_rwb}
 
- * Shorthand: RWB
+- Shorthand: RWB
 
 Because the smallest write size the SSD may support can be a multiple of block size, in order to
 support writes to a single block, the data needs to be buffered. The write buffer is the solution to
 this problem. It consists of a number of pre-allocated buffers called batches, each of size allowing
 for a single transfer to the SSD. A single batch is divided into block-sized buffer entries.
 
+```text
                  write buffer
     +-----------------------------------+
     |batch 1                            |
@@ -89,6 +94,7 @@ for a single transfer to the SSD. A single batch is divided into block-sized buf
     |   |entry 1|entry 2|     |entry n| |
     |   +-----------------------------+ |
     +-----------------------------------+
+```
 
 When a write is scheduled, it needs to acquire an entry for each of its blocks and copy the data
 onto this buffer. Once all blocks are copied, the write can be signalled as completed to the user.
@@ -97,9 +103,9 @@ After that operation is completed the whole batch can be freed. For the whole ti
 the `rwb`, the L2P points at the buffer entry instead of a location on the SSD. This allows for
 servicing read requests from the buffer.
 
-## Defragmentation and relocation {#ftl_reloc}
+### Defragmentation and relocation {#ftl_reloc}
 
- * Shorthand: defrag, reloc
+- Shorthand: defrag, reloc
 
 Since a write to the same LBA invalidates its previous physical location, some of the blocks on a
 band might contain old data that basically wastes space. As there is no way to overwrite an already
@@ -108,12 +114,14 @@ situation in which all of the bands contain some valid data and no band can be e
 can be executed anymore. Therefore a mechanism is needed to move valid data and invalidate whole
 bands, so that they can be reused.
 
+```text
                     band                                             band
     +-----------------------------------+            +-----------------------------------+
     | ** *    * ***      *    *** * *   |            |                                   |
     |**  *       *    *    * *     *   *|   +---->   |                                   |
     |*     ***  *      *            *   |            |                                   |
     +-----------------------------------+            +-----------------------------------+
+```
 
 Valid blocks are marked with an asterisk '\*'.
 
@@ -133,78 +141,22 @@ index of its zones (3) (how many times the band was written to). The lower the r
 higher its age (2) and the lower its write count (3), the higher the chance the band will be chosen
 for defrag.
 
-# Usage {#ftl_usage}
+## Usage {#ftl_usage}
 
-## Prerequisites {#ftl_prereq}
+### Prerequisites {#ftl_prereq}
 
 In order to use the FTL module, a device capable of zoned interface is required e.g. `zone_block`
 bdev or OCSSD `nvme` bdev.
 
-## FTL bdev creation {#ftl_create}
+### FTL bdev creation {#ftl_create}
 
 Similar to other bdevs, the FTL bdevs can be created either based on JSON config files or via RPC.
 Both interfaces require the same arguments which are described by the `--help` option of the
 `bdev_ftl_create` RPC call, which are:
 
- - bdev's name
- - base bdev's name (base bdev must implement bdev_zone API)
- - UUID of the FTL device (if the FTL is to be restored from the SSD)
-
-## FTL usage with OCSSD nvme bdev {#ftl_ocssd}
-
-This option requires an Open Channel SSD, which can be emulated using QEMU.
-
-The QEMU with the patches providing Open Channel support can be found on the SPDK's QEMU fork
-on [spdk-3.0.0](https://github.com/spdk/qemu/tree/spdk-3.0.0) branch.
-
-## Configuring QEMU {#ftl_qemu_config}
-
-To emulate an Open Channel device, QEMU expects parameters describing the characteristics and
-geometry of the SSD:
-
- - `serial` - serial number,
- - `lver` - version of the OCSSD standard (0 - disabled, 1 - "1.2", 2 - "2.0"), libftl only supports
-   2.0,
- - `lba_index` - default LBA format. Possible values can be found in the table below (libftl only supports lba_index >= 3):
- - `lnum_ch` - number of groups,
- - `lnum_lun` - number of parallel units
- - `lnum_pln` - number of planes (logical blocks from all planes constitute a chunk)
- - `lpgs_per_blk` - number of pages (smallest programmable unit) per chunk
- - `lsecs_per_pg` - number of sectors in a page
- - `lblks_per_pln` - number of chunks in a parallel unit
- - `laer_thread_sleep` - timeout in ms between asynchronous events requesting the host to relocate
-   the data based on media feedback
- - `lmetadata` - metadata file
-
-        |lba_index| data| metadata|
-        |---------|-----|---------|
-        |    0    | 512B|    0B   |
-        |    1    | 512B|    8B   |
-        |    2    | 512B|   16B   |
-        |    3    |4096B|    0B   |
-        |    4    |4096B|   64B   |
-        |    5    |4096B|  128B   |
-        |    6    |4096B|   16B   |
-
-For more detailed description of the available options, consult the `hw/block/nvme.c` file in
-the QEMU repository.
-
-Example:
-
-```
-$ /path/to/qemu [OTHER PARAMETERS] -drive format=raw,file=/path/to/data/file,if=none,id=myocssd0
-        -device nvme,drive=myocssd0,serial=deadbeef,lver=2,lba_index=3,lnum_ch=1,lnum_lun=8,lnum_pln=4,
-        lpgs_per_blk=1536,lsecs_per_pg=4,lblks_per_pln=512,lmetadata=/path/to/md/file
-```
-
-In the above example, a device is created with 1 channel, 8 parallel units, 512 chunks per parallel
-unit, 24576 (`lnum_pln` * `lpgs_per_blk` * `lsecs_per_pg`) logical blocks in each chunk with logical
-block being 4096B. Therefore the data file needs to be at least 384G (8 * 512 * 24576 * 4096B) of
-size and can be created with the following command:
-
-```
-fallocate -l 384G /path/to/data/file
-```
+- bdev's name
+- base bdev's name (base bdev must implement bdev_zone API)
+- UUID of the FTL device (if the FTL is to be restored from the SSD)
 
 ## Configuring SPDK {#ftl_spdk_config}
 
@@ -212,7 +164,7 @@ To verify that the drive is emulated correctly, one can check the output of the
 (assuming that `scripts/setup.sh` was called before and the driver has been changed for that
 device):
 
-```
+```bash
 $ build/examples/identify
 =====================================================
 NVMe Controller at 0000:00:0a.0 [1d1d:1f1f]
@@ -226,40 +178,6 @@ Model Number:                          QEMU NVMe Ctrl
 
 ... other info ...
 
-Namespace OCSSD Geometry
-=======================
-OC version: maj:2 min:0
-
-... other info ...
-
-Groups (channels): 1
-PUs (LUNs) per group: 8
-Chunks per LUN: 512
-Logical blks per chunk: 24576
-
-... other info ...
-
-```
-
-In order to create FTL on top Open Channel SSD, the following steps are required:
-
-1) Attach OCSSD NVMe controller
-2) Create OCSSD bdev on the controller attached in step 1 (user could specify parallel unit range
-and create multiple OCSSD bdevs on single OCSSD NVMe controller)
-3) Create FTL bdev on top of bdev created in step 2
-
-Example:
-```
-$ scripts/rpc.py bdev_nvme_attach_controller -b nvme0 -a 00:0a.0 -t pcie
-
-$ scripts/rpc.py bdev_ocssd_create -c nvme0 -b nvme0n1
-	nvme0n1
-
-$ scripts/rpc.py bdev_ftl_create -b ftl0 -d nvme0n1
-{
-	"name": "ftl0",
-	"uuid": "3b469565-1fa5-4bfb-8341-747ec9fca9b9"
-}
 ```
 
 ## FTL usage with zone block bdev {#ftl_zone_block}

doc/gdb_macros.md

@@ -1,6 +1,6 @@
 # GDB Macros User Guide {#gdb_macros}
 
-# Introduction
+## Introduction
 
 When debugging an spdk application using gdb we may need to view data structures
 in lists, e.g. information about bdevs or threads.
@@ -125,7 +125,7 @@ nqn "nqn.2016-06.io.spdk.umgmt:cnode1", '\000' <repeats 191 times>
 ID 1
 ~~~
 
-# Loading The gdb Macros
+## Loading The gdb Macros
 
 Copy the gdb macros to the host where you are about to debug.
 It is best to copy the file either to somewhere within the PYTHONPATH, or to add
@@ -146,7 +146,7 @@ the PYTHONPATH, so I had to manually add the directory to the path.
 (gdb) spdk_load_macros
 ~~~
 
-# Using the gdb Data Directory
+## Using the gdb Data Directory
 
 On most systems, the data directory is /usr/share/gdb. The python script should
 be copied into the python/gdb/function (or python/gdb/command) directory under
@@ -155,12 +155,12 @@ the data directory, e.g. /usr/share/gdb/python/gdb/function.
 If the python script is in there, then the only thing you need to do when
 starting gdb is type "spdk_load_macros".
 
-# Using .gdbinit To Load The Macros
+## Using .gdbinit To Load The Macros
 
 .gdbinit can also be used in order to run automatically run the manual steps
 above prior to starting gdb.
 
-Exmaple .gdbinit:
+Example .gdbinit:
 
 ~~~{.sh}
 source /opt/km/install/tools/gdb_macros/gdb_macros.py
@@ -168,7 +168,7 @@ source /opt/km/install/tools/gdb_macros/gdb_macros.py
 
 When starting gdb you still have to call spdk_load_macros.
 
-# Why Do We Need to Explicitly Call spdk_load_macros
+## Why Do We Need to Explicitly Call spdk_load_macros
 
 The reason is that the macros need to use globals provided by spdk in order to
 iterate the spdk lists and build iterable representations of the list objects.
@@ -196,7 +196,7 @@ Error occurred in Python command: No symbol table is loaded.  Use the "file"
 command.
 ~~~
 
-# Macros available
+## Macros available
 
 - spdk_load_macros: load the macros (use --reload in order to reload them)
 - spdk_print_bdevs: information about bdevs
@@ -205,7 +205,7 @@ command.
 - spdk_print_nvmf_subsystems: information about nvmf subsystems
 - spdk_print_threads: information about threads
 
-# Adding New Macros
+## Adding New Macros
 
 The list iteration macros are usually built from 3 layers:
 

doc/getting_started.md

@@ -1,6 +1,6 @@
 # Getting Started {#getting_started}
 
-# Getting the Source Code {#getting_started_source}
+## Getting the Source Code {#getting_started_source}
 
 ~~~{.sh}
 git clone https://github.com/spdk/spdk
@@ -8,7 +8,7 @@ cd spdk
 git submodule update --init
 ~~~
 
-# Installing Prerequisites {#getting_started_prerequisites}
+## Installing Prerequisites {#getting_started_prerequisites}
 
 The `scripts/pkgdep.sh` script will automatically install the bare minimum
 dependencies required to build SPDK.
@@ -24,7 +24,7 @@ Option --all will install all dependencies needed by SPDK features.
 sudo scripts/pkgdep.sh --all
 ~~~
 
-# Building {#getting_started_building}
+## Building {#getting_started_building}
 
 Linux:
 
@@ -57,7 +57,7 @@ can enable it by doing the following:
 make
 ~~~
 
-# Running the Unit Tests {#getting_started_unittests}
+## Running the Unit Tests {#getting_started_unittests}
 
 It's always a good idea to confirm your build worked by running the
 unit tests.
@@ -70,7 +70,7 @@ You will see several error messages when running the unit tests, but they are
 part of the test suite. The final message at the end of the script indicates
 success or failure.
 
-# Running the Example Applications {#getting_started_examples}
+## Running the Example Applications {#getting_started_examples}
 
 Before running an SPDK application, some hugepages must be allocated and
 any NVMe and I/OAT devices must be unbound from the native kernel drivers.

doc/idxd.md

@@ -1,25 +1,21 @@
 # IDXD Driver {#idxd}
 
-# Public Interface {#idxd_interface}
+## Public Interface {#idxd_interface}
 
 - spdk/idxd.h
 
-# Key Functions {#idxd_key_functions}
+## Key Functions {#idxd_key_functions}
 
 Function                                | Description
 --------------------------------------- | -----------
 spdk_idxd_probe()                       | @copybrief spdk_idxd_probe()
-spdk_idxd_batch_get_max()               | @copybrief spdk_idxd_batch_get_max()
-spdk_idxd_batch_create()                | @copybrief spdk_idxd_batch_create()
-spdk_idxd_batch_prep_copy()             | @copybrief spdk_idxd_batch_prep_copy()
-spdk_idxd_batch_submit()                | @copybrief spdk_idxd_batch_submit()
 spdk_idxd_submit_copy()                 | @copybrief spdk_idxd_submit_copy()
 spdk_idxd_submit_compare()              | @copybrief spdk_idxd_submit_compare()
 spdk_idxd_submit_crc32c()               | @copybrief spdk_idxd_submit_crc32c()
 spdk_idxd_submit_dualcast               | @copybrief spdk_idxd_submit_dualcast()
 spdk_idxd_submit_fill()                 | @copybrief spdk_idxd_submit_fill()
 
-# Pre-defined configurations {#idxd_configs}
+## Pre-defined configurations {#idxd_configs}
 
 The RPC `idxd_scan_accel_engine` is used to both enable IDXD and set it's
 configuration to one of two pre-defined configs:

doc/index.md

@@ -1,41 +1,41 @@
 # Storage Performance Development Kit {#mainpage}
 
-# Introduction
+## Introduction
 
 @copydoc intro
 
-# Concepts
+## Concepts
 
 @copydoc concepts
 
-# User Guides
+## User Guides
 
 @copydoc user_guides
 
-# Programmer Guides
+## Programmer Guides
 
 @copydoc prog_guides
 
-# General Information
+## General Information
 
 @copydoc general
 
-# Miscellaneous
+## Miscellaneous
 
 @copydoc misc
 
-# Driver Modules
+## Driver Modules
 
 @copydoc driver_modules
 
-# Tools
+## Tools
 
 @copydoc tools
 
-# CI Tools
+## CI Tools
 
 @copydoc ci_tools
 
-# Performance Reports
+## Performance Reports
 
 @copydoc performance_reports

doc/ioat.md

@@ -1,10 +1,10 @@
 # I/OAT Driver {#ioat}
 
-# Public Interface {#ioat_interface}
+## Public Interface {#ioat_interface}
 
 - spdk/ioat.h
 
-# Key Functions {#ioat_key_functions}
+## Key Functions {#ioat_key_functions}
 
 Function                                | Description
 --------------------------------------- | -----------

doc/iscsi.md

@@ -1,6 +1,6 @@
 # iSCSI Target {#iscsi}
 
-# iSCSI Target Getting Started Guide {#iscsi_getting_started}
+## iSCSI Target Getting Started Guide {#iscsi_getting_started}
 
 The Storage Performance Development Kit iSCSI target application is named `iscsi_tgt`.
 This following section describes how to run iscsi from your cloned package.
@@ -32,7 +32,7 @@ To ensure the SPDK iSCSI target has the best performance, place the NICs and the
 same NUMA node and configure the target to run on CPU cores associated with that node. The following
 command line option is used to configure the SPDK iSCSI target:
 
-~~~
+~~~bash
 -m 0xF000000
 ~~~
 
@@ -45,35 +45,35 @@ The iSCSI target is configured via JSON-RPC calls. See @ref jsonrpc for details.
 
 ### Portal groups
 
- - iscsi_create_portal_group -- Add a portal group.
- - iscsi_delete_portal_group -- Delete an existing portal group.
- - iscsi_target_node_add_pg_ig_maps -- Add initiator group to portal group mappings to an existing iSCSI target node.
- - iscsi_target_node_remove_pg_ig_maps -- Delete initiator group to portal group mappings from an existing iSCSI target node.
- - iscsi_get_portal_groups -- Show information about all available portal groups.
+- iscsi_create_portal_group -- Add a portal group.
+- iscsi_delete_portal_group -- Delete an existing portal group.
+- iscsi_target_node_add_pg_ig_maps -- Add initiator group to portal group mappings to an existing iSCSI target node.
+- iscsi_target_node_remove_pg_ig_maps -- Delete initiator group to portal group mappings from an existing iSCSI target node.
+- iscsi_get_portal_groups -- Show information about all available portal groups.
 
-~~~
+~~~bash
 /path/to/spdk/scripts/rpc.py iscsi_create_portal_group 1 10.0.0.1:3260
 ~~~
 
 ### Initiator groups
 
- - iscsi_create_initiator_group -- Add an initiator group.
- - iscsi_delete_initiator_group -- Delete an existing initiator group.
- - iscsi_initiator_group_add_initiators -- Add initiators to an existing initiator group.
- - iscsi_get_initiator_groups -- Show information about all available initiator groups.
+- iscsi_create_initiator_group -- Add an initiator group.
+- iscsi_delete_initiator_group -- Delete an existing initiator group.
+- iscsi_initiator_group_add_initiators -- Add initiators to an existing initiator group.
+- iscsi_get_initiator_groups -- Show information about all available initiator groups.
 
-~~~
+~~~bash
 /path/to/spdk/scripts/rpc.py iscsi_create_initiator_group 2 ANY 10.0.0.2/32
 ~~~
 
 ### Target nodes
 
- - iscsi_create_target_node -- Add an iSCSI target node.
- - iscsi_delete_target_node -- Delete an iSCSI target node.
- - iscsi_target_node_add_lun -- Add a LUN to an existing iSCSI target node.
- - iscsi_get_target_nodes -- Show information about all available iSCSI target nodes.
+- iscsi_create_target_node -- Add an iSCSI target node.
+- iscsi_delete_target_node -- Delete an iSCSI target node.
+- iscsi_target_node_add_lun -- Add a LUN to an existing iSCSI target node.
+- iscsi_get_target_nodes -- Show information about all available iSCSI target nodes.
 
-~~~
+~~~bash
 /path/to/spdk/scripts/rpc.py iscsi_create_target_node Target3 Target3_alias MyBdev:0 1:2 64 -d
 ~~~
 
@@ -83,30 +83,30 @@ The Linux initiator is open-iscsi.
 
 Installing open-iscsi package
 Fedora:
-~~~
+~~~bash
 yum install -y iscsi-initiator-utils
 ~~~
 
 Ubuntu:
-~~~
+~~~bash
 apt-get install -y open-iscsi
 ~~~
 
 ### Setup
 
 Edit /etc/iscsi/iscsid.conf
-~~~
+~~~bash
 node.session.cmds_max = 4096
 node.session.queue_depth = 128
 ~~~
 
 iscsid must be restarted or receive SIGHUP for changes to take effect. To send SIGHUP, run:
-~~~
+~~~bash
 killall -HUP iscsid
 ~~~
 
 Recommended changes to /etc/sysctl.conf
-~~~
+~~~bash
 net.ipv4.tcp_timestamps = 1
 net.ipv4.tcp_sack = 0
 
@@ -124,13 +124,14 @@ net.core.netdev_max_backlog = 300000
 ### Discovery
 
 Assume target is at 10.0.0.1
-~~~
+
+~~~bash
 iscsiadm -m discovery -t sendtargets -p 10.0.0.1
 ~~~
 
 ### Connect to target
 
-~~~
+~~~bash
 iscsiadm -m node --login
 ~~~
 
@@ -139,13 +140,13 @@ they came up as.
 
 ### Disconnect from target
 
-~~~
+~~~bash
 iscsiadm -m node --logout
 ~~~
 
 ### Deleting target node cache
 
-~~~
+~~~bash
 iscsiadm -m node -o delete
 ~~~
 
@@ -153,7 +154,7 @@ This will cause the initiator to forget all previously discovered iSCSI target n
 
 ### Finding /dev/sdX nodes for iSCSI LUNs
 
-~~~
+~~~bash
 iscsiadm -m session -P 3 | grep "Attached scsi disk" | awk '{print $4}'
 ~~~
 
@@ -165,19 +166,19 @@ After the targets are connected, they can be tuned. For example if /dev/sdc is
 an iSCSI disk then the following can be done:
 Set noop to scheduler
 
-~~~
+~~~bash
 echo noop > /sys/block/sdc/queue/scheduler
 ~~~
 
 Disable merging/coalescing (can be useful for precise workload measurements)
 
-~~~
+~~~bash
 echo "2" > /sys/block/sdc/queue/nomerges
 ~~~
 
 Increase requests for block queue
 
-~~~
+~~~bash
 echo "1024" > /sys/block/sdc/queue/nr_requests
 ~~~
 
@@ -191,33 +192,34 @@ Assuming we have one iSCSI Target server with portal at 10.0.0.1:3200, two LUNs
 #### Configure iSCSI Target
 
 Start iscsi_tgt application:
-```
+
+```bash
 ./build/bin/iscsi_tgt
 ```
 
 Construct two 64MB Malloc block devices with 512B sector size "Malloc0" and "Malloc1":
 
-```
+```bash
 ./scripts/rpc.py bdev_malloc_create -b Malloc0 64 512
 ./scripts/rpc.py bdev_malloc_create -b Malloc1 64 512
 ```
 
 Create new portal group with id 1, and address 10.0.0.1:3260:
 
-```
+```bash
 ./scripts/rpc.py iscsi_create_portal_group 1 10.0.0.1:3260
 ```
 
 Create one initiator group with id 2 to accept any connection from 10.0.0.2/32:
 
-```
+```bash
 ./scripts/rpc.py iscsi_create_initiator_group 2 ANY 10.0.0.2/32
 ```
 
 Finally construct one target using previously created bdevs as LUN0 (Malloc0) and LUN1 (Malloc1)
 with a name "disk1" and alias "Data Disk1" using portal group 1 and initiator group 2.
 
-```
+```bash
 ./scripts/rpc.py iscsi_create_target_node disk1 "Data Disk1" "Malloc0:0 Malloc1:1" 1:2 64 -d
 ```
 
@@ -225,14 +227,14 @@ with a name "disk1" and alias "Data Disk1" using portal group 1 and initiator gr
 
 Discover target
 
-~~~
+~~~bash
 $ iscsiadm -m discovery -t sendtargets -p 10.0.0.1
 10.0.0.1:3260,1 iqn.2016-06.io.spdk:disk1
 ~~~
 
 Connect to the target
 
-~~~
+~~~bash
 iscsiadm -m node --login
 ~~~
 
@@ -240,7 +242,7 @@ At this point the iSCSI target should show up as SCSI disks.
 
 Check dmesg to see what they came up as. In this example it can look like below:
 
-~~~
+~~~bash
 ...
 [630111.860078] scsi host68: iSCSI Initiator over TCP/IP
 [630112.124743] scsi 68:0:0:0: Direct-Access     INTEL    Malloc disk      0001 PQ: 0 ANSI: 5
@@ -263,35 +265,37 @@ Check dmesg to see what they came up as. In this example it can look like below:
 You may also use simple bash command to find /dev/sdX nodes for each iSCSI LUN
 in all logged iSCSI sessions:
 
-~~~
+~~~bash
 $ iscsiadm -m session -P 3 | grep "Attached scsi disk" | awk '{print $4}'
 sdd
 sde
 ~~~
 
-# iSCSI Hotplug {#iscsi_hotplug}
+## iSCSI Hotplug {#iscsi_hotplug}
 
 At the iSCSI level, we provide the following support for Hotplug:
 
 1. bdev/nvme:
-  At the bdev/nvme level, we start one hotplug monitor which will call
-  spdk_nvme_probe() periodically to get the hotplug events. We provide the
-  private attach_cb and remove_cb for spdk_nvme_probe(). For the attach_cb,
-  we will create the block device base on the NVMe device attached, and for the
-  remove_cb, we will unregister the block device, which will also notify the
-  upper level stack (for iSCSI target, the upper level stack is scsi/lun) to
-  handle the hot-remove event.
+
+At the bdev/nvme level, we start one hotplug monitor which will call
+spdk_nvme_probe() periodically to get the hotplug events. We provide the
+private attach_cb and remove_cb for spdk_nvme_probe(). For the attach_cb,
+we will create the block device base on the NVMe device attached, and for the
+remove_cb, we will unregister the block device, which will also notify the
+upper level stack (for iSCSI target, the upper level stack is scsi/lun) to
+handle the hot-remove event.
 
 2. scsi/lun:
-  When the LUN receive the hot-remove notification from block device layer,
-  the LUN will be marked as removed, and all the IOs after this point will
-  return with check condition status. Then the LUN starts one poller which will
-  wait for all the commands which have already been submitted to block device to
-  return back; after all the commands return back, the LUN will be deleted.
+
+When the LUN receive the hot-remove notification from block device layer,
+the LUN will be marked as removed, and all the IOs after this point will
+return with check condition status. Then the LUN starts one poller which will
+wait for all the commands which have already been submitted to block device to
+return back; after all the commands return back, the LUN will be deleted.
 
 @sa spdk_nvme_probe
 
-# iSCSI Login Redirection {#iscsi_login_redirection}
+## iSCSI Login Redirection {#iscsi_login_redirection}
 
 The SPDK iSCSI target application supports iSCSI login redirection feature.
 
@@ -312,7 +316,7 @@ portal group may optionally have a redirect portal for non-discovery logins for
 each associated target. This redirect portal must be from a private portal group.
 
 Initiators configure portals in public portal groups as target portals. When an
-initator logs in to a target through a portal in an associated public portal group,
+initiator logs in to a target through a portal in an associated public portal group,
 the target sends a temporary redirection response with a redirect portal. Then the
 initiator logs in to the target again through the redirect portal.
 

doc/jsonrpc_proxy.md

@@ -1,6 +1,8 @@
 # JSON-RPC Remote access {#jsonrpc_proxy}
 
-SPDK provides a sample python script `rpc_http_proxy.py`, that provides http server which listens for JSON objects from users. It uses HTTP POST method to receive JSON objects including methods and parameters described in this chapter.
+SPDK provides a sample python script `rpc_http_proxy.py`, that provides http server which listens for JSON
+objects from users. It uses HTTP POST method to receive JSON objects including methods and parameters
+described in this chapter.
 
 ## Parameters
 
@@ -26,9 +28,10 @@ Status 200 with resultant JSON object included on success.
 
 ## Client side
 
-Below is a sample python script acting as a client side. It sends `bdev_get_bdevs` method with optional `name` parameter and prints JSON object returned from remote_rpc script.
+Below is a sample python script acting as a client side. It sends `bdev_get_bdevs` method with optional `name`
+parameter and prints JSON object returned from remote_rpc script.
 
-~~~
+~~~python
 import json
 import requests
 
@@ -45,7 +48,10 @@ if __name__ == '__main__':
 
 Output:
 
-~~~
+~~~python
 python client.py
-[{u'num_blocks': 2621440, u'name': u'Malloc0', u'uuid': u'fb57e59c-599d-42f1-8b89-3e46dbe12641', u'claimed': True, u'driver_specific': {}, u'supported_io_types': {u'reset': True, u'nvme_admin': False, u'unmap': True, u'read': True, u'nvme_io': False, u'write': True, u'flush': True, u'write_zeroes': True}, u'qos_ios_per_sec': 0, u'block_size': 4096, u'product_name': u'Malloc disk', u'aliases': []}]
+[{u'num_blocks': 2621440, u'name': u'Malloc0', u'uuid': u'fb57e59c-599d-42f1-8b89-3e46dbe12641', u'claimed': True,
+u'driver_specific': {}, u'supported_io_types': {u'reset': True, u'nvme_admin': False, u'unmap': True, u'read': True,
+u'nvme_io': False, u'write': True, u'flush': True, u'write_zeroes': True}, u'qos_ios_per_sec': 0, u'block_size': 4096,
+u'product_name': u'Malloc disk', u'aliases': []}]
 ~~~

doc/libraries.md

@@ -9,7 +9,7 @@ mixing of SPDK event framework dependent code and lower level libraries. This do
 is aimed at explaining the structure, naming conventions, versioning scheme, and use cases
 of the libraries contained in these two directories.
 
-# Directory Structure {#structure}
+## Directory Structure {#structure}
 
 The SPDK libraries are divided into two directories. The `lib` directory contains the base libraries that
 compose SPDK. Some of these base libraries define plug-in systems. Instances of those plug-ins are called
@@ -17,24 +17,24 @@ modules and are located in the `module` directory. For example, the `spdk_sock`
 `lib` directory while the implementations of socket abstractions, `sock_posix` and `sock_uring`
 are contained in the `module` directory.
 
-## lib {#lib}
+### lib {#lib}
 
 The libraries in the `lib` directory can be readily divided into four categories:
 
 - Utility Libraries: These libraries contain basic, commonly used functions that make more complex
-libraries easier to implement. For example, `spdk_log` contains macro definitions that provide a
-consistent logging paradigm and `spdk_json` is a general purpose JSON parsing library.
+  libraries easier to implement. For example, `spdk_log` contains macro definitions that provide a
+  consistent logging paradigm and `spdk_json` is a general purpose JSON parsing library.
 - Protocol Libraries: These libraries contain the building blocks for a specific service. For example,
-`spdk_nvmf` and `spdk_vhost` each define the storage protocols after which they are named.
+  `spdk_nvmf` and `spdk_vhost` each define the storage protocols after which they are named.
 - Storage Service Libraries: These libraries provide a specific abstraction that can be mapped to somewhere
-between the physical drive and the filesystem level of your typical storage stack. For example `spdk_bdev`
-provides a general block device abstraction layer, `spdk_lvol` provides a logical volume abstraction,
-`spdk_blobfs` provides a filesystem abstraction, and `spdk_ftl` provides a flash translation layer
-abstraction.
+  between the physical drive and the filesystem level of your typical storage stack. For example `spdk_bdev`
+  provides a general block device abstraction layer, `spdk_lvol` provides a logical volume abstraction,
+  `spdk_blobfs` provides a filesystem abstraction, and `spdk_ftl` provides a flash translation layer
+  abstraction.
 - System Libraries: These libraries provide system level services such as a JSON based RPC service
-(see `spdk_jsonrpc`) and thread abstractions (see `spdk_thread`). The most notable library in this category
-is the `spdk_env_dpdk` library which provides a shim for the underlying Data Plane Development Kit (DPDK)
-environment and provides services like memory management.
+  (see `spdk_jsonrpc`) and thread abstractions (see `spdk_thread`). The most notable library in this category
+  is the `spdk_env_dpdk` library which provides a shim for the underlying Data Plane Development Kit (DPDK)
+  environment and provides services like memory management.
 
 The one library in the `lib` directory that doesn't fit into the above classification is the `spdk_event` library.
 This library defines a framework used by the applications contained in the `app` and `example` directories. Much
@@ -48,7 +48,7 @@ Much like the `spdk_event` library, the `spdk_env_dpdk` library has been archite
 can be readily replaced by an alternate environment shim. More information on replacing the `spdk_env_dpdk`
 module and the underlying `dpdk` environment can be found in the [environment](#env_replacement) section.
 
-## module {#module}
+### module {#module}
 
 The component libraries in the `module` directory represent specific implementations of the base libraries in
 the `lib` directory. As with the `lib` directory, much care has been taken to avoid dependencies on the
@@ -58,10 +58,10 @@ There are seven sub-directories in the `module` directory which each hold a diff
 sub-directories can be divided into two types.
 
 - plug-in libraries: These libraries are explicitly tied to one of the libraries in the `lib` directory and
-are registered with that library at runtime by way of a specific constructor function. The parent library in
-the `lib` directory then manages the module directly. These types of libraries each implement a function table
-defined by their parent library. The following table shows these directories and their corresponding parent
-libraries:
+  are registered with that library at runtime by way of a specific constructor function. The parent library in
+  the `lib` directory then manages the module directly. These types of libraries each implement a function table
+  defined by their parent library. The following table shows these directories and their corresponding parent
+  libraries:
 
 <center>
 | module directory | parent library | dependent on event library |
@@ -73,15 +73,15 @@ libraries:
 </center>
 
 - Free libraries: These libraries are highly dependent upon a library in the `lib` directory but are not
-explicitly registered to that library via a constructor. The libraries in the `blob`, `blobfs`, and `env_dpdk`
-directories fall into this category. None of the libraries in this category depend explicitly on the
-`spdk_event` library.
+  explicitly registered to that library via a constructor. The libraries in the `blob`, `blobfs`, and `env_dpdk`
+  directories fall into this category. None of the libraries in this category depend explicitly on the
+  `spdk_event` library.
 
-# Library Conventions {#conventions}
+## Library Conventions {#conventions}
 
 The SPDK libraries follow strict conventions for naming functions, logging, versioning, and header files.
 
-## Headers {#headers}
+### Headers {#headers}
 
 All public SPDK header files exist in the `include` directory of the SPDK repository. These headers
 are divided into two sub-directories.
@@ -105,7 +105,7 @@ Other header files contained directly in the `lib` and `module` directories are
 by source files of their corresponding library. Any symbols intended to be used across libraries need to be
 included in a header in the `include/spdk_internal` directory.
 
-## Naming Conventions {#naming}
+### Naming Conventions {#naming}
 
 All public types and functions in SPDK libraries begin with the prefix `spdk_`. They are also typically
 further namespaced using the spdk library name. The rest of the function or type name describes its purpose.
@@ -114,15 +114,15 @@ There are no internal library functions that begin with the `spdk_` prefix. This
 enforced by the SPDK continuous Integration testing. Functions not intended for use outside of their home
 library should be namespaced with the name of the library only.
 
-## Map Files {#map}
+### Map Files {#map}
 
 SPDK libraries can be built as both static and shared object files. To facilitate building libraries as shared
 objects, each one has a corresponding map file (e.g. `spdk_nvmf` relies on `spdk_nvmf.map`). SPDK libraries
 not exporting any symbols rely on a blank map file located at `mk/spdk_blank.map`.
 
-# SPDK Shared Objects {#shared_objects}
+## SPDK Shared Objects {#shared_objects}
 
-## Shared Object Versioning {#versioning}
+### Shared Object Versioning {#versioning}
 
 SPDK shared objects follow a semantic versioning pattern with a major and minor version. Any changes which
 break backwards compatibility (symbol removal or change) will cause a shared object major increment and
@@ -141,7 +141,7 @@ Shared objects are versioned independently of one another. This means that `libs
 with the same suffix are not necessarily compatible with each other. It is important to source all of your
 SPDK libraries from the same repository and version to ensure inter-library compatibility.
 
-## Linking to Shared Objects {#so_linking}
+### Linking to Shared Objects {#so_linking}
 
 Shared objects in SPDK are created on a per-library basis. There is a top level `libspdk.so` object
 which is a linker script. It simply contains references to all of the other spdk shared objects.
@@ -149,15 +149,15 @@ which is a linker script. It simply contains references to all of the other spdk
 There are essentially two ways of linking to SPDK libraries.
 
 1. An application can link to the top level shared object library as follows:
-~~~{.sh}
+   ~~~{.sh}
 	gcc -o my_app ./my_app.c -lspdk -lspdk_env_dpdk -ldpdk
-~~~
+   ~~~
 
 2. An application can link to only a subset of libraries by linking directly to the ones it relies on:
-~~~{.sh}
+   ~~~{.sh}
 	gcc -o my_app ./my_app.c -lpassthru_external -lspdk_event_bdev -lspdk_bdev -lspdk_bdev_malloc
 	-lspdk_log -lspdk_thread -lspdk_util -lspdk_event -lspdk_env_dpdk -ldpdk
-~~~
+   ~~~
 
 In the second instance, please note that applications need only link to the libraries upon which they
 directly depend. All SPDK libraries have their dependencies specified at object compile time. This means
@@ -172,7 +172,7 @@ itself need to be supplied to the linker. In the examples above, these are `spdk
 respectively. This was intentional and allows one to easily swap out both the environment and the
 environment shim.
 
-## Replacing the env abstraction {#env_replacement}
+### Replacing the env abstraction {#env_replacement}
 
 SPDK depends on an environment abstraction that provides crucial pinned memory management and PCIe
 bus management operations. The interface for this environment abstraction is defined in the
@@ -184,6 +184,7 @@ modifications to the spdk source directly.
 Any environment can replace the `spdk_env_dpdk` environment by implementing the `include/env.h` header
 file. The environment can either be implemented wholesale in a single library or as a two-part
 shim/implementation library system.
+
 ~~~{.sh}
 	# single library
 	gcc -o my_app ./my_app.c -lspdk -lcustom_env_implementation
@@ -192,7 +193,7 @@ shim/implementation library system.
 	gcc -o my_app ./my_app.c -lspdk -lcustom_env_shim -lcustom_env_implementation
 ~~~
 
-# SPDK Static Objects {#static_objects}
+## SPDK Static Objects {#static_objects}
 
 SPDK static objects are compiled by default even when no parameters are supplied to the build system.
 Unlike SPDK shared objects, the filename does not contain any versioning semantics. Linking against
@@ -203,7 +204,7 @@ NVMe transports.
 Due to the lack of versioning semantics, it is not recommended to install static libraries system wide.
 Instead the path to these static libraries should be added as argument at compile time using
 `-L/path/to/static/libs`. The use of static objects instead of shared objects can also be forced
-through `-Wl,-Bsatic`, otherwise some compilers might prefer to use the shared objects if both
+through `-Wl,-Bstatic`, otherwise some compilers might prefer to use the shared objects if both
 are available.
 
 ~~~{.sh}

doc/lvol.md

@@ -1,38 +1,48 @@
 # Logical Volumes {#logical_volumes}
 
-The Logical Volumes library is a flexible storage space management system. It provides creating and managing virtual block devices with variable size. The SPDK Logical Volume library is built on top of @ref blob.
+The Logical Volumes library is a flexible storage space management system. It provides creating and managing virtual
+block devices with variable size. The SPDK Logical Volume library is built on top of @ref blob.
 
-# Terminology {#lvol_terminology}
+## Terminology {#lvol_terminology}
 
-## Logical volume store {#lvs}
+### Logical volume store {#lvs}
 
 * Shorthand:  lvolstore, lvs
 * Type name:  struct spdk_lvol_store
 
-A logical volume store uses the super blob feature of blobstore to hold uuid (and in future other metadata). Blobstore types are implemented in blobstore itself, and saved on disk. An lvolstore will generate a UUID on creation, so that it can be uniquely identified from other lvolstores.
-By default when creating lvol store data region is unmapped. Optional --clear-method parameter can be passed on creation to change that behavior to writing zeroes or performing no operation.
+A logical volume store uses the super blob feature of blobstore to hold uuid (and in future other metadata).
+Blobstore types are implemented in blobstore itself, and saved on disk. An lvolstore will generate a UUID on
+creation, so that it can be uniquely identified from other lvolstores.
+By default when creating lvol store data region is unmapped. Optional --clear-method parameter can be passed
+on creation to change that behavior to writing zeroes or performing no operation.
 
-## Logical volume {#lvol}
+### Logical volume {#lvol}
 
 * Shorthand: lvol
 * Type name: struct spdk_lvol
 
-A logical volume is implemented as an SPDK blob created from an lvolstore. An lvol is uniquely identified by its UUID. Lvol additional can have alias name.
+A logical volume is implemented as an SPDK blob created from an lvolstore. An lvol is uniquely identified by
+its UUID. Lvol additional can have alias name.
 
-## Logical volume block device {#lvol_bdev}
+### Logical volume block device {#lvol_bdev}
 
 * Shorthand: lvol_bdev
 * Type name: struct spdk_lvol_bdev
 
 Representation of an SPDK block device (spdk_bdev) with an lvol implementation.
-A logical volume block device translates generic SPDK block device I/O (spdk_bdev_io) operations into the equivalent SPDK blob operations. Combination of lvol name and lvolstore name gives lvol_bdev alias name in a form "lvs_name/lvol_name". block_size of the created bdev is always 4096, due to blobstore page size. Cluster_size is configurable by parameter.
-Size of the new bdev will be rounded up to nearest multiple of cluster_size.
-By default lvol bdevs claim part of lvol store equal to their set size. When thin provision option is enabled, no space is taken from lvol store until data is written to lvol bdev.
-By default when deleting lvol bdev or resizing down, allocated clusters are unmapped. Optional --clear-method parameter can be passed on creation to change that behavior to writing zeroes or performing no operation.
+A logical volume block device translates generic SPDK block device I/O (spdk_bdev_io) operations into the
+equivalent SPDK blob operations. Combination of lvol name and lvolstore name gives lvol_bdev alias name in
+a form "lvs_name/lvol_name". block_size of the created bdev is always 4096, due to blobstore page size.
+Cluster_size is configurable by parameter. Size of the new bdev will be rounded up to nearest multiple of
+cluster_size. By default lvol bdevs claim part of lvol store equal to their set size. When thin provision
+option is enabled, no space is taken from lvol store until data is written to lvol bdev.
+By default when deleting lvol bdev or resizing down, allocated clusters are unmapped. Optional --clear-method
+parameter can be passed on creation to change that behavior to writing zeroes or performing no operation.
 
-## Thin provisioning {#lvol_thin_provisioning}
+### Thin provisioning {#lvol_thin_provisioning}
 
-Thin provisioned lvols rely on dynamic cluster allocation (e.g. when the first write operation on a cluster is performed), only space required to store data is used and unallocated clusters are obtained from underlying device (e.g. zeroes_dev).
+Thin provisioned lvols rely on dynamic cluster allocation (e.g. when the first write operation on a cluster is performed), only space
+required to store data is used and unallocated clusters are obtained from underlying device (e.g. zeroes_dev).
 
 Sample write operations of thin provisioned blob are shown on the diagram below:
 
@@ -42,11 +52,13 @@ Sample read operations and the structure of thin provisioned blob are shown on t
 
 ![Reading clusters from thin provisioned blob](lvol_thin_provisioning.svg)
 
-## Snapshots and clone {#lvol_snapshots}
+### Snapshots and clone {#lvol_snapshots}
 
-Logical volumes support snapshots and clones functionality. User may at any given time create snapshot of existing logical volume to save a backup of current volume state.
-When creating snapshot original volume becomes thin provisioned and saves only incremental differences from its underlying snapshot. This means that every read from unallocated cluster is actually a read from the snapshot and
-every write to unallocated cluster triggers new cluster allocation and data copy from corresponding cluster in snapshot to the new cluster in logical volume before the actual write occurs.
+Logical volumes support snapshots and clones functionality. User may at any given time create snapshot of existing
+logical volume to save a backup of current volume state. When creating snapshot original volume becomes thin provisioned
+and saves only incremental differences from its underlying snapshot. This means that every read from unallocated cluster
+is actually a read from the snapshot and every write to unallocated cluster triggers new cluster allocation and data copy
+from corresponding cluster in snapshot to the new cluster in logical volume before the actual write occurs.
 
 The read operation is performed as shown in the diagram below:
 ![Reading cluster from clone](lvol_clone_snapshot_read.svg)
@@ -54,32 +66,38 @@ The read operation is performed as shown in the diagram below:
 The write operation is performed as shown in the diagram below:
 ![Writing cluster to the clone](lvol_clone_snapshot_write.svg)
 
-User may also create clone of existing snapshot that will be thin provisioned and it will behave in the same way as logical volume from which snapshot is created.
-There is no limit of clones and snapshots that may be created as long as there is enough space on logical volume store. Snapshots are read only. Clones may be created only from snapshots or read only logical volumes.
+User may also create clone of existing snapshot that will be thin provisioned and it will behave in the same way as logical volume
+from which snapshot is created. There is no limit of clones and snapshots that may be created as long as there is enough space on
+logical volume store. Snapshots are read only. Clones may be created only from snapshots or read only logical volumes.
 
-A snapshot can be removed only if there is a single clone on top of it. The relation chain will be updated accordingly. The cluster map of clone and snapshot will be merged and entries for unallocated clusters in the clone
-will be updated with addresses from the snapshot cluster map. The entire operation modifies metadata only - no data is copied during this process.
+A snapshot can be removed only if there is a single clone on top of it. The relation chain will be updated accordingly.
+The cluster map of clone and snapshot will be merged and entries for unallocated clusters in the clone will be updated with
+addresses from the snapshot cluster map. The entire operation modifies metadata only - no data is copied during this process.
 
-## Inflation {#lvol_inflation}
+### Inflation {#lvol_inflation}
 
-Blobs can be inflated to copy data from backing devices (e.g. snapshots) and allocate all remaining clusters. As a result of this operation all dependencies for the blob are removed.
+Blobs can be inflated to copy data from backing devices (e.g. snapshots) and allocate all remaining clusters. As a result of this
+operation all dependencies for the blob are removed.
 
 ![Removing backing blob and bdevs relations using inflate call](lvol_inflate_clone_snapshot.svg)
 
-## Decoupling {#lvol_decoupling}
+### Decoupling {#lvol_decoupling}
 
-Blobs can be decoupled from their parent blob by copying data from backing devices (e.g. snapshots) for all allocated clusters. Remaining unallocated clusters are kept thin provisioned.
-Note: When decouple is performed, only single dependency is removed. To remove all dependencies in a chain of blobs depending on each other, multiple calls need to be issued.
+Blobs can be decoupled from their parent blob by copying data from backing devices (e.g. snapshots) for all allocated clusters.
+Remaining unallocated clusters are kept thin provisioned.
+Note: When decouple is performed, only single dependency is removed. To remove all dependencies in a chain of blobs depending
+on each other, multiple calls need to be issued.
 
-# Configuring Logical Volumes
+## Configuring Logical Volumes
 
-There is no static configuration available for logical volumes. All configuration is done trough RPC. Information about logical volumes is kept on block devices.
+There is no static configuration available for logical volumes. All configuration is done trough RPC. Information about
+logical volumes is kept on block devices.
 
-# RPC overview {#lvol_rpc}
+## RPC overview {#lvol_rpc}
 
 RPC regarding lvolstore:
 
-```
+```bash
 bdev_lvol_create_lvstore [-h] [-c CLUSTER_SZ] bdev_name lvs_name
     Constructs lvolstore on specified bdev with specified name. During
     construction bdev is unmapped at initialization and all data is
@@ -111,7 +129,7 @@ bdev_lvol_rename_lvstore [-h] old_name new_name
 
 RPC regarding lvol and spdk bdev:
 
-```
+```bash
 bdev_lvol_create [-h] [-u UUID] [-l LVS_NAME] [-t] [-c CLEAR_METHOD] lvol_name size
     Creates lvol with specified size and name on lvolstore specified by its uuid
     or name. Then constructs spdk bdev on top of that lvol and presents it as spdk bdev.

doc/memory.md

@@ -92,7 +92,7 @@ SPDK must be allocated using spdk_dma_malloc() or its siblings. The buffers
 must be allocated specifically so that they are pinned and so that physical
 addresses are known.
 
-# IOMMU Support
+## IOMMU Support
 
 Many platforms contain an extra piece of hardware called an I/O Memory
 Management Unit (IOMMU). An IOMMU is much like a regular MMU, except it

doc/misc.md

@@ -2,3 +2,4 @@
 
 - @subpage peer_2_peer
 - @subpage containers
+- @subpage rpms

doc/notify.md

@@ -9,32 +9,32 @@ do not poll frequently enough, events may be lost. All events are identified by
 monotonically increasing integer, so missing events may be detected, although
 not recovered.
 
-# Register event types {#notify_register}
+## Register event types {#notify_register}
 
 During initialization the sender library should register its own event types using
 `spdk_notify_type_register(const char *type)`. Parameter 'type' is the name of
 notification type.
 
-# Get info about events {#notify_get_info}
+## Get info about events {#notify_get_info}
 
 A consumer can get information about the available event types during runtime using
 `spdk_notify_foreach_type`, which iterates over registered notification types and
 calls a callback on each of them, so that user can produce detailed information
 about notification.
 
-# Get new events {#notify_listen}
+## Get new events {#notify_listen}
 
 A consumer can get events by calling function `spdk_notify_foreach_event`.
 The caller should specify last received event and the maximum number of invocations.
 There might be multiple consumers of each event. The event bus is implemented as a
 circular buffer, so older events may be overwritten by newer ones.
 
-# Send events {#notify_send}
+## Send events {#notify_send}
 
 When an event occurs, a library can invoke `spdk_notify_send` with two strings.
 One containing the type of the event, like "spdk_bdev_register", second with context,
 for example "Nvme0n1"
 
-# RPC Calls {#rpc_calls}
+## RPC Calls {#rpc_calls}
 
 See [JSON-RPC documentation](jsonrpc.md/#rpc_notify_get_types)

doc/nvme.md

@@ -1,17 +1,17 @@
 # NVMe Driver {#nvme}
 
-# In this document {#nvme_toc}
+## In this document {#nvme_toc}
 
-* @ref nvme_intro
-* @ref nvme_examples
-* @ref nvme_interface
-* @ref nvme_design
-* @ref nvme_fabrics_host
-* @ref nvme_multi_process
-* @ref nvme_hotplug
-* @ref nvme_cuse
+- @ref nvme_intro
+- @ref nvme_examples
+- @ref nvme_interface
+- @ref nvme_design
+- @ref nvme_fabrics_host
+- @ref nvme_multi_process
+- @ref nvme_hotplug
+- @ref nvme_cuse
 
-# Introduction {#nvme_intro}
+## Introduction {#nvme_intro}
 
 The NVMe driver is a C library that may be linked directly into an application
 that provides direct, zero-copy data transfer to and from
@@ -29,23 +29,23 @@ devices via NVMe over Fabrics. Users may now call spdk_nvme_probe() on both
 local PCI busses and on remote NVMe over Fabrics discovery services. The API is
 otherwise unchanged.
 
-# Examples {#nvme_examples}
+## Examples {#nvme_examples}
 
-## Getting Start with Hello World {#nvme_helloworld}
+### Getting Start with Hello World {#nvme_helloworld}
 
 There are a number of examples provided that demonstrate how to use the NVMe
 library. They are all in the [examples/nvme](https://github.com/spdk/spdk/tree/master/examples/nvme)
 directory in the repository. The best place to start is
 [hello_world](https://github.com/spdk/spdk/blob/master/examples/nvme/hello_world/hello_world.c).
 
-## Running Benchmarks with Fio Plugin {#nvme_fioplugin}
+### Running Benchmarks with Fio Plugin {#nvme_fioplugin}
 
 SPDK provides a plugin to the very popular [fio](https://github.com/axboe/fio)
 tool for running some basic benchmarks. See the fio start up
 [guide](https://github.com/spdk/spdk/blob/master/examples/nvme/fio_plugin/)
 for more details.
 
-## Running Benchmarks with Perf Tool {#nvme_perf}
+### Running Benchmarks with Perf Tool {#nvme_perf}
 
 NVMe perf utility in the [examples/nvme/perf](https://github.com/spdk/spdk/tree/master/examples/nvme/perf)
 is one of the examples which also can be used for performance tests. The fio
@@ -79,7 +79,7 @@ perf -q 1 -o 4096 -w write -r 'trtype:PCIe traddr:0000:04:00.0' -t 300 -e 'PRACT
 perf -q 1 -o 4096 -w read -r 'trtype:PCIe traddr:0000:04:00.0' -t 200 -e 'PRACT=0,PRCKH=GUARD'
 ~~~
 
-# Public Interface {#nvme_interface}
+## Public Interface {#nvme_interface}
 
 - spdk/nvme.h
 
@@ -103,9 +103,9 @@ spdk_nvme_ctrlr_process_admin_completions() | @copybrief spdk_nvme_ctrlr_process
 spdk_nvme_ctrlr_cmd_io_raw()                | @copybrief spdk_nvme_ctrlr_cmd_io_raw()
 spdk_nvme_ctrlr_cmd_io_raw_with_md()        | @copybrief spdk_nvme_ctrlr_cmd_io_raw_with_md()
 
-# NVMe Driver Design {#nvme_design}
+## NVMe Driver Design {#nvme_design}
 
-## NVMe I/O Submission {#nvme_io_submission}
+### NVMe I/O Submission {#nvme_io_submission}
 
 I/O is submitted to an NVMe namespace using nvme_ns_cmd_xxx functions. The NVMe
 driver submits the I/O request as an NVMe submission queue entry on the queue
@@ -117,21 +117,21 @@ spdk_nvme_qpair_process_completions().
 @sa spdk_nvme_ns_cmd_read, spdk_nvme_ns_cmd_write, spdk_nvme_ns_cmd_dataset_management,
 spdk_nvme_ns_cmd_flush, spdk_nvme_qpair_process_completions
 
-### Fused operations {#nvme_fuses}
+#### Fused operations {#nvme_fuses}
 
 To "fuse" two commands, the first command should have the SPDK_NVME_IO_FLAGS_FUSE_FIRST
 io flag set, and the next one should have the SPDK_NVME_IO_FLAGS_FUSE_SECOND.
 
 In addition, the following rules must be met to execute two commands as an atomic unit:
 
- - The commands shall be inserted next to each other in the same submission queue.
- - The LBA range, should be the same for the two commands.
+- The commands shall be inserted next to each other in the same submission queue.
+- The LBA range, should be the same for the two commands.
 
 E.g. To send fused compare and write operation user must call spdk_nvme_ns_cmd_compare
 followed with spdk_nvme_ns_cmd_write and make sure no other operations are submitted
 in between on the same queue, like in example below:
 
-~~~
+~~~c
 	rc = spdk_nvme_ns_cmd_compare(ns, qpair, cmp_buf, 0, 1, nvme_fused_first_cpl_cb,
 			NULL, SPDK_NVME_CMD_FUSE_FIRST);
 	if (rc != 0) {
@@ -149,7 +149,7 @@ The NVMe specification currently defines compare-and-write as a fused operation.
 Support for compare-and-write is reported by the controller flag
 SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED.
 
-### Scaling Performance {#nvme_scaling}
+#### Scaling Performance {#nvme_scaling}
 
 NVMe queue pairs (struct spdk_nvme_qpair) provide parallel submission paths for
 I/O. I/O may be submitted on multiple queue pairs simultaneously from different
@@ -182,7 +182,7 @@ require that data should be done by sending a request to the owning thread.
 This results in a message passing architecture, as opposed to a locking
 architecture, and will result in superior scaling across CPU cores.
 
-## NVMe Driver Internal Memory Usage {#nvme_memory_usage}
+### NVMe Driver Internal Memory Usage {#nvme_memory_usage}
 
 The SPDK NVMe driver provides a zero-copy data transfer path, which means that
 there are no data buffers for I/O commands. However, some Admin commands have
@@ -202,12 +202,12 @@ Each submission queue entry (SQE) and completion queue entry (CQE) consumes 64 b
 and 16 bytes respectively. Therefore, the maximum memory used for each I/O queue
 pair is (MQES + 1) * (64 + 16) Bytes.
 
-# NVMe over Fabrics Host Support {#nvme_fabrics_host}
+## NVMe over Fabrics Host Support {#nvme_fabrics_host}
 
 The NVMe driver supports connecting to remote NVMe-oF targets and
 interacting with them in the same manner as local NVMe SSDs.
 
-## Specifying Remote NVMe over Fabrics Targets {#nvme_fabrics_trid}
+### Specifying Remote NVMe over Fabrics Targets {#nvme_fabrics_trid}
 
 The method for connecting to a remote NVMe-oF target is very similar
 to the normal enumeration process for local PCIe-attached NVMe devices.
@@ -228,11 +228,11 @@ single NVM subsystem directly, the NVMe library will call `probe_cb`
 for just that subsystem; this allows the user to skip the discovery step
 and connect directly to a subsystem with a known address.
 
-## RDMA Limitations
+### RDMA Limitations
 
 Please refer to NVMe-oF target's @ref nvmf_rdma_limitations
 
-# NVMe Multi Process {#nvme_multi_process}
+## NVMe Multi Process {#nvme_multi_process}
 
 This capability enables the SPDK NVMe driver to support multiple processes accessing the
 same NVMe device. The NVMe driver allocates critical structures from shared memory, so
@@ -243,7 +243,7 @@ The primary motivation for this feature is to support management tools that can
 to long running applications, perform some maintenance work or gather information, and
 then detach.
 
-## Configuration {#nvme_multi_process_configuration}
+### Configuration {#nvme_multi_process_configuration}
 
 DPDK EAL allows different types of processes to be spawned, each with different permissions
 on the hugepage memory used by the applications.
@@ -269,7 +269,7 @@ Example: identical shm_id and non-overlapping core masks
 ./perf -q 8 -o 131072 -w write -c 0x10 -t 60 -i 1
 ~~~
 
-## Limitations {#nvme_multi_process_limitations}
+### Limitations {#nvme_multi_process_limitations}
 
 1. Two processes sharing memory may not share any cores in their core mask.
 2. If a primary process exits while secondary processes are still running, those processes
@@ -280,7 +280,7 @@ Example: identical shm_id and non-overlapping core masks
 
 @sa spdk_nvme_probe, spdk_nvme_ctrlr_process_admin_completions
 
-# NVMe Hotplug {#nvme_hotplug}
+## NVMe Hotplug {#nvme_hotplug}
 
 At the NVMe driver level, we provide the following support for Hotplug:
 
@@ -300,11 +300,11 @@ At the NVMe driver level, we provide the following support for Hotplug:
 
 @sa spdk_nvme_probe
 
-# NVMe Character Devices {#nvme_cuse}
+## NVMe Character Devices {#nvme_cuse}
 
 This feature is considered as experimental.
 
-## Design
+### Design
 
 ![NVMe character devices processing diagram](nvme_cuse.svg)
 
@@ -326,9 +326,9 @@ immediate response, without passing them through the ring.
 
 This interface reserves one additional qpair for sending down the I/O for each controller.
 
-## Usage
+### Usage
 
-### Enabling cuse support for NVMe
+#### Enabling cuse support for NVMe
 
 Cuse support is disabled by default. To enable support for NVMe-CUSE devices first
 install required dependencies
@@ -337,7 +337,7 @@ sudo scripts/pkgdep.sh --fuse
 ~~~
 Then compile SPDK with "./configure --with-nvme-cuse".
 
-### Creating NVMe-CUSE device
+#### Creating NVMe-CUSE device
 
 First make sure to prepare the environment (see @ref getting_started).
 This includes loading CUSE kernel module.
@@ -367,7 +367,7 @@ $ ls /dev/spdk/
 nvme0  nvme0n1
 ~~~
 
-### Example of using nvme-cli
+#### Example of using nvme-cli
 
 Most nvme-cli commands can point to specific controller or namespace by providing a path to it.
 This can be leveraged to issue commands to the SPDK NVMe-CUSE devices.
@@ -381,7 +381,7 @@ sudo nvme id-ns /dev/spdk/nvme0n1
 Note: `nvme list` command does not display SPDK NVMe-CUSE devices,
 see nvme-cli [PR #773](https://github.com/linux-nvme/nvme-cli/pull/773).
 
-### Examples of using smartctl
+#### Examples of using smartctl
 
 smartctl tool recognizes device type based on the device path. If none of expected
 patterns match, SCSI translation layer is used to identify device.
@@ -395,7 +395,7 @@ the NVMe device.
     ...
 ~~~
 
-## Limitations
+### Limitations
 
 NVMe namespaces are created as character devices and their use may be limited for
 tools expecting block devices.

doc/nvmf.md

@@ -3,7 +3,7 @@
 @sa @ref nvme_fabrics_host
 @sa @ref nvmf_tgt_tracepoints
 
-# NVMe-oF Target Getting Started Guide {#nvmf_getting_started}
+## NVMe-oF Target Getting Started Guide {#nvmf_getting_started}
 
 The SPDK NVMe over Fabrics target is a user space application that presents block devices over a fabrics
 such as Ethernet, Infiniband or Fibre Channel. SPDK currently supports RDMA and TCP transports.
@@ -152,13 +152,13 @@ working with NVMe over Fabrics specific RPCs can be found on the @ref jsonrpc_co
 Start the nvmf_tgt application with elevated privileges. Once the target is started,
 the nvmf_create_transport rpc can be used to initialize a given transport. Below is an
 example where the target is started and configured with two different transports.
-The RDMA transport is configured with an I/O unit size of 8192 bytes, 4 max qpairs per controller,
-and an in capsule data size of 0 bytes. The TCP transport is configured with an I/O unit size of
+The RDMA transport is configured with an I/O unit size of 8192 bytes, max I/O size 131072 and an
+in capsule data size of 8192 bytes. The TCP transport is configured with an I/O unit size of
 16384 bytes, 8 max qpairs per controller, and an in capsule data size of 8192 bytes.
 
 ~~~{.sh}
 build/bin/nvmf_tgt
-scripts/rpc.py nvmf_create_transport -t RDMA -u 8192 -m 4 -c 0
+scripts/rpc.py nvmf_create_transport -t RDMA -u 8192 -i 131072 -c 8192
 scripts/rpc.py nvmf_create_transport -t TCP -u 16384 -m 8 -c 8192
 ~~~
 
@@ -186,7 +186,8 @@ Basic Types
 year = 4 * digit ;
 month = '01' | '02' | '03' | '04' | '05' | '06' | '07' | '08' | '09' | '10' | '11' | '12' ;
 digit = '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' ;
-hex digit = 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' ;
+hex digit = 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | '0' |
+'1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' ;
 
 NQN Definition
 NVMe Qualified Name = ( NVMe-oF Discovery NQN | NVMe UUID NQN | NVMe Domain NQN ), '\0' ;

doc/nvmf_tracing.md

@@ -1,36 +1,81 @@
 # NVMe-oF Target Tracepoints {#nvmf_tgt_tracepoints}
 
-# Introduction {#tracepoints_intro}
+## Introduction {#tracepoints_intro}
 
 SPDK has a tracing framework for capturing low-level event information at runtime.
 Tracepoints provide a high-performance tracing mechanism that is accessible at runtime.
 They are implemented as a circular buffer in shared memory that is accessible from other
 processes. The NVMe-oF target is instrumented with tracepoints to enable analysis of
-both performance and application crashes. (Note: the SPDK tracing framework should still
-be considered experimental.  Work to formalize and document the framework is in progress.)
+both performance and application crashes and it has to be configured beforehand using
+this [guide](https://spdk.io/doc/nvmf.html).
+(Note: the SPDK tracing framework should still be considered experimental.
+Work to formalize and document the framework is in progress.)
 
-# Enabling Tracepoints {#enable_tracepoints}
+## Enabling Tracepoints {#enable_tracepoints}
 
-Tracepoints are placed in groups. They are enabled and disabled as a group. To enable
-the instrumentation of all the tracepoints group in an SPDK target application, start the
-target with -e parameter set to 0xFFFF:
+Tracepoints are placed in groups. They are enabled and disabled as a group or individually
+inside a group.
 
-~~~
+### Enabling Tracepoints in Groups
+
+To enable the instrumentation of all the tracepoints groups in an SPDK target
+application, start the target with `-e` parameter set to `0xFFFF` or `all`:
+
+~~~bash
 build/bin/nvmf_tgt -e 0xFFFF
 ~~~
 
-To enable the instrumentation of just the NVMe-oF RDMA tracepoints in an SPDK target
-application, start the target with the -e parameter set to 0x10:
+or
 
+~~~bash
+build/bin/nvmf_tgt -e all
 ~~~
+
+To enable the instrumentation of just the `NVMe-oF RDMA` tracepoints in an SPDK target
+application, start the target with the `-e` parameter set to `0x10`:
+
+~~~bash
 build/bin/nvmf_tgt -e 0x10
 ~~~
 
+### Enabling Individual Tracepoints
+
+To enable individual tracepoints inside a group:
+
+~~~bash
+build/bin/nvmf_tgt -e 0x10:B
+~~~
+
+or
+
+~~~bash
+build/bin/nvmf_tgt -e nvmf_rdma:B
+~~~
+
+where `:` is a separator and `B` is the tracepoint mask. This will enable only the first, second and fourth (binary: 1011) tracepoint inside `NVMe-oF RDMA` group.
+
+### Combining Tracepoint Masks
+
+It is also possible to combine enabling whole groups of tpoints and individual ones:
+
+~~~bash
+build/bin/nvmf_tgt -e 0x10:2,0x400
+~~~
+
+This will enable the second tracepoint inside `NVMe-oF RDMA` group (0x10) and all of the tracepoints defined by the `thread` group (0x400).
+
+### Tracepoint Group Values
+
+iscsi (0x2), scsi (0x4), bdev (0x8), nvmf_rdma (0x10), nvmf_tcp (0x20), ftl (0x40), blobfs (0x80), nvmf_fc (0x100),
+idxd (0x200), thread (0x400), nvme_pcie (0x800)
+
+### Starting the SPDK Target
+
 When the target starts, a message is logged with the information you need to view
 the tracepoints in a human-readable format using the spdk_trace application. The target
 will also log information about the shared memory file.
 
-~~~{.sh}
+~~~bash
 app.c: 527:spdk_app_setup_trace: *NOTICE*: Tracepoint Group Mask 0xFFFF specified.
 app.c: 531:spdk_app_setup_trace: *NOTICE*: Use 'spdk_trace -s nvmf -p 24147' to capture a snapshot of events at runtime.
 app.c: 533:spdk_app_setup_trace: *NOTICE*: Or copy /dev/shm/nvmf_trace.pid24147 for offline analysis/debug.
@@ -41,20 +86,20 @@ exits.  This ensures the file can be used for analysis after the application exi
 shared memory files are in /dev/shm, and can be deleted manually to free shm space if needed.  A system
 reboot will also free all of the /dev/shm files.
 
-# Capturing a snapshot of events {#capture_tracepoints}
+## Capturing a snapshot of events {#capture_tracepoints}
 
 Send I/Os to the SPDK target application to generate events. The following is
 an example usage of perf to send I/Os to the NVMe-oF target over an RDMA network
 interface for 10 minutes.
 
-~~~
-./perf -q 128 -s 4096 -w randread -t 600 -r 'trtype:RDMA adrfam:IPv4 traddr:192.168.100.2 trsvcid:4420'
+~~~bash
+./perf -q 128 -o 4096 -w randread -t 600 -r 'trtype:RDMA adrfam:IPv4 traddr:192.168.100.2 trsvcid:4420'
 ~~~
 
 The spdk_trace program can be found in the app/trace directory.  To analyze the tracepoints on the same
 system running the NVMe-oF target, simply execute the command line shown in the log:
 
-~~~{.sh}
+~~~bash
 build/bin/spdk_trace -s nvmf -p 24147
 ~~~
 
@@ -62,13 +107,13 @@ To analyze the tracepoints on a different system, first prepare the tracepoint f
 tracepoint file can be large, but usually compresses very well.  This step can also be used to prepare
 a tracepoint file to attach to a GitHub issue for debugging NVMe-oF application crashes.
 
-~~~{.sh}
+~~~bash
 bzip2 -c /dev/shm/nvmf_trace.pid24147 > /tmp/trace.bz2
 ~~~
 
 After transferring the /tmp/trace.bz2 tracepoint file to a different system:
 
-~~~{.sh}
+~~~bash
 bunzip2 /tmp/trace.bz2
 build/bin/spdk_trace -f /tmp/trace
 ~~~
@@ -77,7 +122,7 @@ The following is sample trace capture showing the cumulative time that each
 I/O spends at each RDMA state. All the trace captures with the same id are for
 the same I/O.
 
-~~~
+~~~bash
 28:   6026.658 ( 12656064)     RDMA_REQ_NEED_BUFFER                                      id:    r3622            time:  0.019
 28:   6026.694 ( 12656140)     RDMA_REQ_RDY_TO_EXECUTE                                   id:    r3622            time:  0.055
 28:   6026.820 ( 12656406)     RDMA_REQ_EXECUTING                                        id:    r3622            time:  0.182
@@ -124,7 +169,7 @@ the same I/O.
 28:   6033.056 ( 12669500)     RDMA_REQ_COMPLETED                                        id:    r3564            time:  100.211
 ~~~
 
-# Capturing sufficient trace events {#capture_trace_events}
+## Capturing sufficient trace events {#capture_trace_events}
 
 Since the tracepoint file generated directly by SPDK application is a circular buffer in shared memory,
 the trace events captured by it may be insufficient for further analysis.
@@ -133,31 +178,31 @@ spdk_trace_record is used to poll the spdk tracepoint shared memory, record new
 and store all entries into specified output file at its shutdown on SIGINT or SIGTERM.
 After SPDK nvmf target is launched, simply execute the command line shown in the log:
 
-~~~{.sh}
+~~~bash
 build/bin/spdk_trace_record -q -s nvmf -p 24147 -f /tmp/spdk_nvmf_record.trace
 ~~~
 
 Also send I/Os to the SPDK target application to generate events by previous perf example for 10 minutes.
 
-~~~{.sh}
-./perf -q 128 -s 4096 -w randread -t 600 -r 'trtype:RDMA adrfam:IPv4 traddr:192.168.100.2 trsvcid:4420'
+~~~bash
+./perf -q 128 -o 4096 -w randread -t 600 -r 'trtype:RDMA adrfam:IPv4 traddr:192.168.100.2 trsvcid:4420'
 ~~~
 
 After the completion of perf example, shut down spdk_trace_record by signal SIGINT (Ctrl + C).
 To analyze the tracepoints output file from spdk_trace_record, simply run spdk_trace program by:
 
-~~~{.sh}
+~~~bash
 build/bin/spdk_trace -f /tmp/spdk_nvmf_record.trace
 ~~~
 
-# Adding New Tracepoints {#add_tracepoints}
+## Adding New Tracepoints {#add_tracepoints}
 
 SPDK applications and libraries provide several trace points. You can add new
 tracepoints to the existing trace groups. For example, to add a new tracepoints
 to the SPDK RDMA library (lib/nvmf/rdma.c) trace group TRACE_GROUP_NVMF_RDMA,
 define the tracepoints and assigning them a unique ID using the SPDK_TPOINT_ID macro:
 
-~~~
+~~~c
 #define	TRACE_GROUP_NVMF_RDMA	0x4
 #define TRACE_RDMA_REQUEST_STATE_NEW	SPDK_TPOINT_ID(TRACE_GROUP_NVMF_RDMA, 0x0)
 ...
@@ -168,17 +213,19 @@ You also need to register the new trace points in the SPDK_TRACE_REGISTER_FN mac
 within the application/library using the spdk_trace_register_description function
 as shown below:
 
-~~~
+~~~c
 SPDK_TRACE_REGISTER_FN(nvmf_trace)
 {
 	spdk_trace_register_object(OBJECT_NVMF_RDMA_IO, 'r');
 	spdk_trace_register_description("RDMA_REQ_NEW", "",
 					TRACE_RDMA_REQUEST_STATE_NEW,
-					OWNER_NONE, OBJECT_NVMF_RDMA_IO, 1, 1, "cmid:	");
+					OWNER_NONE, OBJECT_NVMF_RDMA_IO, 1,
+					SPDK_TRACE_ARG_TYPE_PTR, "cmid:	");
 	...
 	spdk_trace_register_description("NEW_RDMA_REQ_NAME", "",
 					NEW_TRACE_POINT_NAME,
-					OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0, 1, "cmid:	");
+					OWNER_NONE, OBJECT_NVMF_RDMA_IO, 0,
+					SPDK_TRACE_ARG_TYPE_PTR, "cmid:	");
 }
 ~~~
 
@@ -187,7 +234,7 @@ application/library to record the current trace state for the new trace points.
 The following example shows the usage of the spdk_trace_record function to
 record the current trace state of several tracepoints.
 
-~~~
+~~~c
 	case RDMA_REQUEST_STATE_NEW:
 		spdk_trace_record(TRACE_RDMA_REQUEST_STATE_NEW, 0, 0, (uintptr_t)rdma_req, (uintptr_t)rqpair->cm_id);
 		...

doc/overview.md

@@ -1,6 +1,6 @@
 # SPDK Structural Overview {#overview}
 
-# Overview {#dir_overview}
+## Overview {#dir_overview}
 
 SPDK is composed of a set of C libraries residing in `lib` with public interface
 header files in `include/spdk`, plus a set of applications built out of those
@@ -77,13 +77,13 @@ directory and include the headers by prefixing `spdk/` like this:
 Most of the headers here correspond with a library in the `lib` directory. There
 are a few headers that stand alone, however. They are:
 
- - `assert.h`
- - `barrier.h`
- - `endian.h`
- - `fd.h`
- - `mmio.h`
- - `queue.h` and `queue_extras.h`
- - `string.h`
+- `assert.h`
+- `barrier.h`
+- `endian.h`
+- `fd.h`
+- `mmio.h`
+- `queue.h` and `queue_extras.h`
+- `string.h`
 
 There is also an `spdk_internal` directory that contains header files widely included
 by libraries within SPDK, but that are not part of the public API and would not be

doc/peer_2_peer.md

@@ -3,14 +3,14 @@
 Please note that the functionality discussed in this document is
 currently tagged as experimental.
 
-# In this document {#p2p_toc}
+## In this document {#p2p_toc}
 
 * @ref p2p_overview
 * @ref p2p_nvme_api
 * @ref p2p_cmb_copy
 * @ref p2p_issues
 
-# Overview {#p2p_overview}
+## Overview {#p2p_overview}
 
 Peer-2-Peer (P2P) is the concept of DMAing data directly from one PCI
 End Point (EP) to another without using a system memory buffer. The
@@ -22,7 +22,7 @@ In this section of documentation we outline how to perform P2P
 operations in SPDK and outline some of the issues that can occur when
 performing P2P operations.
 
-# The P2P API for NVMe {#p2p_nvme_api}
+## The P2P API for NVMe {#p2p_nvme_api}
 
 The functions that provide access to the NVMe CMBs for P2P
 capabilities are given in the table below.
@@ -33,7 +33,7 @@ spdk_nvme_ctrlr_map_cmb()                   | @copybrief spdk_nvme_ctrlr_map_cmb
 spdk_nvme_ctrlr_unmap_cmb()                 | @copybrief spdk_nvme_ctrlr_unmap_cmb()
 spdk_nvme_ctrlr_get_regs_cmbsz()            | @copybrief spdk_nvme_ctrlr_get_regs_cmbsz()
 
-# Determining device support {#p2p_support}
+## Determining device support {#p2p_support}
 
 SPDK's identify example application displays whether a device has a controller
 memory buffer and which operations it supports. Run it as follows:
@@ -42,7 +42,7 @@ memory buffer and which operations it supports. Run it as follows:
 ./build/examples/identify -r traddr:<pci id of ssd>
 ~~~
 
-# cmb_copy: An example P2P Application {#p2p_cmb_copy}
+## cmb_copy: An example P2P Application {#p2p_cmb_copy}
 
 Run the cmb_copy example application.
 
@@ -53,7 +53,7 @@ This should copy a single LBA (LBA 0) from namespace 1 on the read
 NVMe SSD to LBA 0 on namespace 1 on the write SSD using the CMB as the
 DMA buffer.
 
-# Issues with P2P {#p2p_issues}
+## Issues with P2P {#p2p_issues}
 
 * In some systems when performing peer-2-peer DMAs between PCIe EPs
   that are directly connected to the Root Complex (RC) the DMA may

doc/performance_reports.md

@@ -1,5 +1,33 @@
 # Performance Reports {#performance_reports}
 
+## Release 22.01
+
+- [SPDK 22.01 NVMe Bdev Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_nvme_bdev_perf_report_2201.pdf)
+- [SPDK 22.01 NVMe-oF TCP Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_tcp_perf_report_2201.pdf)
+- [SPDK 22.01 NVMe-oF RDMA Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_rdma_perf_report_2201.pdf)
+- [SPDK 22.01 Vhost Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_vhost_perf_report_2201.pdf)
+
+## Release 21.10
+
+- [SPDK 21.10 NVMe Bdev Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_nvme_bdev_perf_report_2110.pdf)
+- [SPDK 21.10 NVMe-oF TCP Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_tcp_perf_report_2110.pdf)
+- [SPDK 21.10 NVMe-oF RDMA Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_rdma_perf_report_2110.pdf)
+- [SPDK 21.10 Vhost Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_vhost_perf_report_2110.pdf)
+
+## Release 21.07
+
+- [SPDK 21.07 NVMe Bdev Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_nvme_bdev_perf_report_2107.pdf)
+- [SPDK 21.07 NVMe-oF TCP Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_tcp_perf_report_2107.pdf)
+- [SPDK 21.07 NVMe-oF RDMA Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_rdma_perf_report_2107.pdf)
+- [SPDK 21.07 Vhost Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_vhost_perf_report_2107.pdf)
+
+## Release 21.04
+
+- [SPDK 21.04 NVMe Bdev Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_nvme_bdev_perf_report_2104.pdf)
+- [SPDK 21.04 NVMe-oF TCP Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_tcp_perf_report_2104.pdf)
+- [SPDK 21.04 NVMe-oF RDMA Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_rdma_perf_report_2104.pdf)
+- [SPDK 21.04 Vhost Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_vhost_perf_report_2104.pdf)
+
 ## Release 21.01
 
 - [SPDK 21.01 NVMe Bdev Performance Report](https://ci.spdk.io/download/performance-reports/SPDK_nvme_bdev_perf_report_2101.pdf)

doc/pkgconfig.md

@@ -8,21 +8,21 @@ when SPDK adds or modifies library dependencies.
 If your application is using the SPDK nvme library, you would use the following
 to get the list of required SPDK libraries:
 
-~~~
+~~~bash
 PKG_CONFIG_PATH=/path/to/spdk/build/lib/pkgconfig pkg-config --libs spdk_nvme
 ~~~
 
 To get the list of required SPDK and DPDK libraries to use the DPDK-based
 environment layer:
 
-~~~
+~~~bash
 PKG_CONFIG_PATH=/path/to/spdk/build/lib/pkgconfig pkg-config --libs spdk_env_dpdk
 ~~~
 
 When linking with static libraries, the dependent system libraries must also be
 specified. To get the list of required system libraries:
 
-~~~
+~~~bash
 PKG_CONFIG_PATH=/path/to/spdk/build/lib/pkgconfig pkg-config --libs spdk_syslibs
 ~~~
 
@@ -33,7 +33,7 @@ the `-Wl,--no-as-needed` parameters while with static libraries `-Wl,--whole-arc
 is used. Here is an example Makefile snippet that shows how to use pkg-config to link
 an application that uses the SPDK nvme shared library:
 
-~~~
+~~~bash
 PKG_CONFIG_PATH = $(SPDK_DIR)/build/lib/pkgconfig
 SPDK_LIB := $(shell PKG_CONFIG_PATH="$(PKG_CONFIG_PATH)" pkg-config --libs spdk_nvme
 DPDK_LIB := $(shell PKG_CONFIG_PATH="$(PKG_CONFIG_PATH)" pkg-config --libs spdk_env_dpdk
@@ -44,7 +44,7 @@ app:
 
 If using the SPDK nvme static library:
 
-~~~
+~~~bash
 PKG_CONFIG_PATH = $(SPDK_DIR)/build/lib/pkgconfig
 SPDK_LIB := $(shell PKG_CONFIG_PATH="$(PKG_CONFIG_PATH)" pkg-config --libs spdk_nvme
 DPDK_LIB := $(shell PKG_CONFIG_PATH="$(PKG_CONFIG_PATH)" pkg-config --libs spdk_env_dpdk

doc/porting.md

@@ -18,4 +18,6 @@ a new version of the *env* library.  The new implementation can be
 integrated into the SPDK build by updating the following line
 in CONFIG:
 
-    CONFIG_ENV?=$(SPDK_ROOT_DIR)/lib/env_dpdk
+```bash
+CONFIG_ENV?=$(SPDK_ROOT_DIR)/lib/env_dpdk
+```

doc/rpm.md

@@ -1,10 +1,11 @@
 # RPMs {#rpms}
 
-# In this document {#rpms_toc}
+## In this document {#rpms_toc}
 
 * @ref building_rpms
+* @ref dpdk_devel
 
-# Building SPDK RPMs {#building_rpms}
+## Building SPDK RPMs {#building_rpms}
 
 To build basic set of RPM packages out of the SPDK repo simply run:
 
@@ -26,6 +27,13 @@ There are several options that may be passed via environment as well:
 - RPM_RELEASE   - Target release version of the RPM packages. Default: 1
 - REQUIREMENTS  - Extra set of RPM dependencies if deemed as needed
 - SPDK_VERSION  - SPDK version. Default: currently checked out tag
+- GEN_SPEC      - Orders rpm.sh to only generate a valid .spec and print
+                it on stdout. The content of the .spec is determined based
+                mainly on the ./configure cmdline passed to rpm.sh.
+- USE_DEFAULT_DIRS - Normally, rpm.sh will order rpmbuild to build under
+                   customizable set of directories. Since this may be not
+                   desired, especially when used together with GEN_SPEC,
+                   this option will preserve the default set of directories.
 
 ~~~{.sh}
 # DEPS=no MAKEFLAGS="-d -j1" rpmbuild/rpm.sh --with-shared
@@ -47,3 +55,13 @@ target user:
 - spdk-devel      - provides development files
 - spdk-libs       - provides target lib, .pc files (--with-shared)
 - spdk-dpdk-libs  - provides dpdk lib files (--with-shared|--with-dpdk)
+
+## Special case for dpdk-devel {#dpdk_devel}
+
+When rpm.sh finds a bare --with-dpdk argument on the cmdline it will try to
+adjust the behavior of the rpmbuild to make sure only SPDK RPMs are built.
+Since this argument requests SPDK to be built against installed DPDK (e.g.
+dpdk-devel package) the spdk-dpdk-libs RPM won't be included.  Moreover, the
+.spec will be armed with a build requirement to make sure dpdk-devel is
+present on the building system. The minimum required version of dpdk-devel
+is set to 19.11.

doc/shfmt.md

@@ -1,13 +1,13 @@
 # shfmt {#shfmt}
 
-# In this document {#shfmt_toc}
+## In this document {#shfmt_toc}
 
 * @ref shfmt_overview
 * @ref shfmt_usage
 * @ref shfmt_installation
 * @ref shfmt_examples
 
-# Overview {#shfmt_overview}
+## Overview {#shfmt_overview}
 
 The majority of tests (and scripts overall) in the SPDK repo are written
 in Bash (with a quite significant emphasis on "Bashism"), thus a style
@@ -15,7 +15,10 @@ formatter, shfmt, was introduced to help keep the .sh code consistent
 across the entire repo. For more details on the tool itself, please see
 [shfmt](https://github.com/mvdan/sh).
 
-# Usage {#shfmt_usage}
+We also advise to use 4.4 Bash as a minimum version to make sure scripts
+across the whole repo work as intended.
+
+## Usage {#shfmt_usage}
 
 On the CI pool, the shfmt is run against all the updated .sh files that
 have been committed but not merged yet. Additionally, shfmt will pick
@@ -36,7 +39,7 @@ Please, see ./scripts/check_format.sh for all the arguments the shfmt
 is run with. Additionally, @ref shfmt_examples has more details on how
 each of the arguments behave.
 
-# Installation {#shfmt_installation}
+## Installation {#shfmt_installation}
 
 The shfmt can be easily installed via pkgdep.sh:
 
@@ -55,7 +58,7 @@ SHFMT_DIR_OUT=/and_link_it_here \
   ./scripts/pkgdep.sh -d
 ~~~
 
-# Examples {#shfmt_examples}
+## Examples {#shfmt_examples}
 
 ~~~{.sh}
 #######################################

doc/spdk_top.md

@@ -1,10 +1,22 @@
 # spdk_top {#spdk_top}
 
-The spdk_top application is designed to resemble the standard top in that it provides a real-time insights into CPU cores usage by SPDK lightweight threads and pollers. Have you ever wondered which CPU core is used most by your SPDK instance? Are you building your own bdev or library and want to know if your code is running efficiently? Are your new pollers busy most of the time? The spdk_top application uses RPC calls to collect performance metrics and displays them in a report that you can analyze and determine if your code is running efficiently so that you can tune your implementation and get more from SPDK.
+The spdk_top application is designed to resemble the standard top in that it provides a real-time insights into CPU cores usage by SPDK
+lightweight threads and pollers. Have you ever wondered which CPU core is used most by your SPDK instance? Are you building your own bdev
+or library and want to know if your code is running efficiently? Are your new pollers busy most of the time? The spdk_top application uses
+RPC calls to collect performance metrics and displays them in a report that you can analyze and determine if your code is running efficiently
+so that you can tune your implementation and get more from SPDK.
 
-Why doesn't the classic top utility work for SPDK? SPDK uses a polled-mode design; a reactor thread running on each CPU core assigned to an SPDK application schedules SPDK lightweight threads and pollers to run on the CPU core. Therefore, the standard Linux top utility is not effective for analyzing the CPU usage for polled-mode applications like SPDK because it just reports that they are using 100% of the CPU resources assigned to them. The spdk_top utility was developed to analyze and report the CPU cycles used to do real work vs just polling for work. The utility relies on instrumentation added to pollers to track when they are doing work vs. polling for work. The spdk_top utility gets the fine grained metrics from the pollers, analyzes and report the metrics on a per poller, thread and core basis. This information enables users to identify CPU cores that are busy doing real work so that they can determine if the application needs more or less CPU resources.
+Why doesn't the classic top utility work for SPDK? SPDK uses a polled-mode design; a reactor thread running on each CPU core assigned to
+an SPDK application schedules SPDK lightweight threads and pollers to run on the CPU core. Therefore, the standard Linux top utility is
+not effective for analyzing the CPU usage for polled-mode applications like SPDK because it just reports that they are using 100% of the
+CPU resources assigned to them. The spdk_top utility was developed to analyze and report the CPU cycles used to do real work vs just
+polling for work. The utility relies on instrumentation added to pollers to track when they are doing work vs. polling for work. The
+spdk_top utility gets the fine grained metrics from the pollers, analyzes and report the metrics on a per poller, thread and core basis.
+This information enables users to identify CPU cores that are busy doing real work so that they can determine if the application
+needs more or less CPU resources.
+
+## Run spdk_top
 
-# Run spdk_top
 Before running spdk_top you need to run the SPDK application whose performance you want to analyze using spdk_top.
 
 Run the spdk_top application
@@ -13,19 +25,19 @@ Run the spdk_top application
 ./build/bin/spdk_top
 ~~~
 
-# Bottom menu
+## Bottom menu
+
 Menu at the bottom of SPDK top window shows many options for changing displayed data. Each menu item has a key associated with it in square brackets.
 
 * Quit - quits the SPDK top application.
-* TAB selection - allows to select THREADS/POLLERS/CORES tabs.
-* Previous page/Next page - scrolls up/down to the next set of rows displayed. Indicator in the bottom-left corner shows current page and number of all available pages.
-* Columns - enables/disables chosen columns in a column pop-up window.
-* Sorting - allows to sort displayed data by column in a sorting pop-up.
-* Refresh rate - takes user input from 0 to 255 and changes refresh rate to that value in seconds.
+* Switch tab - allows to select THREADS/POLLERS/CORES tabs.
+* Previous page/Next page - scrolls up/down to the next set of rows displayed. Indicator in the bottom-left corner shows current page and number
+  of all available pages.
 * Item details - displays details pop-up window for highlighted data row. Selection is changed by pressing UP and DOWN arrow keys.
-* Total/Interval - changes displayed values in all tabs to either Total time (measured since start of SPDK application) or Interval time (measured since last refresh).
+* Help - displays help pop-up window.
+
+## Threads Tab
 
-# Threads Tab
 The threads tab displays a line item for each spdk thread. The information displayed shows:
 
 * Thread name - name of SPDK thread.
@@ -34,12 +46,14 @@ The threads tab displays a line item for each spdk thread. The information displ
 * Idle/Busy - how many microseconds the thread was idle/busy.
 
 \n
-By pressing ENTER key a pop-up window appears, showing above and a list of pollers running on selected thread (with poller name, type, run count and period).
+By pressing ENTER key a pop-up window appears, showing above and a list of pollers running on selected
+thread (with poller name, type, run count and period).
 Pop-up then can be closed by pressing ESC key.
 
 To learn more about spdk threads see @ref concurrency.
 
-# Pollers Tab
+## Pollers Tab
+
 The pollers tab displays a line item for each poller. The information displayed shows:
 
 * Poller name - name of currently selected poller.
@@ -53,13 +67,20 @@ The pollers tab displays a line item for each poller. The information displayed
 Poller pop-up window can be displayed by pressing ENTER on a selected data row and displays above information.
 Pop-up can be closed by pressing ESC key.
 
-# Cores Tab
+## Cores Tab
+
 The cores tab provides insights into how the application is using the CPU cores assigned to it. The information displayed for each core shows:
 
 * Core - core number.
 * Thread count - number of threads currently running on core.
 * Poller count - total number of pollers running on core.
 * Idle/Busy - how many microseconds core was idle (including time when core ran pollers but did not find any work) or doing actual work.
+* Intr - whether this core is in interrupt mode or not.
 
 \n
-Pressing ENTER key makes a pop-up window appear, showing above information, along with a list of threads running on selected core. Cores details window allows to select a thread and display thread details pop-up on top of it. To close both pop-ups use ESC key.
+Pressing ENTER key makes a pop-up window appear, showing above information, along with a list of threads running on selected core. Cores details
+window allows to select a thread and display thread details pop-up on top of it. To close both pop-ups use ESC key.
+
+## Help Window
+
+Help window pop-up can be invoked by pressing H key inside any tab. It contains explanations for each key used inside the spdk_top application.

doc/spdkcli.md

@@ -7,21 +7,21 @@ This experimental version was added for v18.04 to get early feedback
 that can be incorporated as spdkcli becomes more fully-featured
 for the next SPDK release.
 
-### Install needed dependencies
+## Install needed dependencies
 
 All dependencies should be handled by scripts/pkgdep.sh script.
 Package dependencies at the moment include:
 
- - configshell
+- configshell
 
-### Run SPDK application instance
+## Run SPDK application instance
 
 ~~~{.sh}
 ./scripts/setup.sh
 ./build/bin/vhost -c vhost.json
 ~~~
 
-### Run SPDK CLI
+## Run SPDK CLI
 
 Spdkcli should be run with the same privileges as SPDK application.
 In order to use SPDK CLI in interactive mode please use:
@@ -33,11 +33,11 @@ Use "help" command to get a list of available commands for each tree node.
 It is also possible to use SPDK CLI to run just a single command,
 just use the command as an argument to the application.
 For example, to view current configuration and immediately exit:
- ~~~{.sh}
+~~~{.sh}
 scripts/spdkcli.py ls
 ~~~
 
-### Optional - create Python virtual environment
+## Optional - create Python virtual environment
 
 You can use Python virtual environment if you don't want to litter your
 system Python installation.

doc/system_configuration.md

@@ -2,7 +2,7 @@
 
 This system configuration guide describes how to configure a system for use with SPDK.
 
-# IOMMU configuration {#iommu_config}
+## IOMMU configuration {#iommu_config}
 
 An IOMMU may be present and enabled on many platforms. When an IOMMU is present and enabled, it is
 recommended that SPDK applications are deployed with the `vfio-pci` kernel driver. SPDK's
@@ -21,7 +21,7 @@ version they are using has a bug where `uio_pci_generic` [fails to bind to NVMe
 In these cases, users can build the `igb_uio` kernel module which can be found in dpdk-kmods repository.
 To ensure that the driver is properly bound, users should specify `DRIVER_OVERRIDE=/path/to/igb_uio.ko`.
 
-# Running SPDK as non-priviledged user {#system_configuration_nonroot}
+## Running SPDK as non-privileged user {#system_configuration_nonroot}
 
 One of the benefits of using the `VFIO` Linux kernel driver is the ability to
 perform DMA operations with peripheral devices as unprivileged user. The
@@ -29,7 +29,7 @@ permissions to access particular devices still need to be granted by the system
 administrator, but only on a one-time basis. Note that this functionality
 is supported with DPDK starting from version 18.11.
 
-## Hugetlbfs access
+### Hugetlbfs access
 
 Make sure the target user has RW access to at least one hugepage mount.
 A good idea is to create a new mount specifically for SPDK:
@@ -44,7 +44,7 @@ Then start SPDK applications with an additional parameter `--huge-dir /mnt/spdk_
 Full guide on configuring hugepage mounts is available in the
 [Linux Hugetlbpage Documentation](https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt)
 
-## Device access {#system_configuration_nonroot_device_access}
+### Device access {#system_configuration_nonroot_device_access}
 
 `VFIO` device access is protected with sysfs file permissions and can be
 configured with chown/chmod.
@@ -62,7 +62,7 @@ Let's assume we want to use PCI device `0000:04:00.0`. First of all, verify
 that it has an IOMMU group assigned:
 
 ~~~{.sh}
-$ readlink "/sys/bus/pci/devices/0000:00:04.0/iommu_group"
+readlink "/sys/bus/pci/devices/0000:00:04.0/iommu_group"
 ~~~
 
 The output should be e.g.
@@ -86,7 +86,7 @@ devices, use the following:
 # chown spdk /dev/vfio/5
 ~~~
 
-## Memory constraints {#system_configuration_nonroot_memory_constraints}
+### Memory constraints {#system_configuration_nonroot_memory_constraints}
 
 As soon as the first device is attached to SPDK, all of SPDK memory will be
 mapped to the IOMMU through the VFIO APIs. VFIO will try to mlock that memory and
@@ -100,7 +100,7 @@ The limit can be checked by running the following command as target user:
 (output in kilobytes)
 
 ~~~{.sh}
-$ ulimit -l
+ulimit -l
 ~~~
 
 On Ubuntu 18.04 this returns 16384 (16MB) by default, which is way below
@@ -111,18 +111,18 @@ try to map not only its reserved hugepages, but also all the memory that's
 shared by its vhost clients as described in the
 [Vhost processing guide](https://spdk.io/doc/vhost_processing.html#vhost_processing_init).
 
-### Increasing the memlock limit permanently
+#### Increasing the memlock limit permanently
 
 Open the `/etc/security/limits.conf` file as root and append the following:
 
-```
+```bash
 spdk     hard   memlock           unlimited
 spdk     soft   memlock           unlimited
 ```
 
 Then logout from the target user account. The changes will take effect after the next login.
 
-### Increasing the memlock for a specific process
+#### Increasing the memlock for a specific process
 
 Linux offers a `prlimit` utility that can override limits of any given process.
 On Ubuntu, it is a part of the `util-linux` package.

doc/template_pg.md

@@ -1,6 +1,6 @@
 # ComponentName Programmer's Guide {#componentname_pg}
 
-# In this document {#componentname_pg_toc}
+## In this document {#componentname_pg_toc}
 
 @ref componentname_pg_audience
 @ref componentname_pg_intro
@@ -47,7 +47,7 @@ Some questions to consider when authoring this section:
 
 Here is where you want to highlight things they need to think about in *their* design. If you have written test code
 for this module think about the things that you needed to go learn about to properly interact with this module. Think
-about how they need to consider initialization options, threading, limitations, any sort of quirky or non-obious
+about how they need to consider initialization options, threading, limitations, any sort of quirky or non-obvious
 interactions or module behaviors that might save them some time and effort by thinking about before they start their
 design.
 

doc/usdt.md

@@ -0,0 +1,200 @@
+# Userspace DTrace (USDT) {#usdt}
+
+## Package Dependencies
+
+These dependencies are needed for building bpftrace and
+the sys/sdt.h header file that SPDK libraries will include
+for DTRACE_PROBE macro definitions.
+
+Fedora:
+libbpf
+gtest-devel
+gmock-devel
+bcc-devel
+systemtap-sdt-devel
+llvm-devel
+bison
+flex
+
+Ubuntu:
+systemtap-sdt-dev
+libbpfcc-dev
+libclang-7-dev
+bison
+flex
+
+## Building bpftrace
+
+We have found issues with the packaged bpftrace on both Ubuntu 20.04
+and Fedora 33.  So bpftrace should be built and installed from source.
+
+```bash
+git clone https://github.com/iovisor/bpftrace.git
+mkdir bpftrace/build
+cd bpftrace/build
+cmake -DCMAKE_BUILD_TYPE=Release ..
+make
+sudo make install
+```
+
+## bpftrace.sh
+
+bpftrace.sh is a helper script that facilitates running bpftrace scripts
+against a running SPDK application.  Here is a typical usage:
+
+```bash
+scripts/bpftrace.sh `pidof spdk_tgt` scripts/bpf/nvmf.bt
+```
+
+Attaching to USDT probes requires the full path of the binary in the
+probe description. SPDK bpftrace scripts can be written with an __EXE__
+marker instead of a full path name, and bpftrace.sh will dynamically
+replace that string with the full path name using information from procfs.
+
+It is also useful to filter certain kernel events (such as system calls)
+based on the PID of the SPDK application.  SPDK bpftrace scripts can be
+written with a __PID__ marker, and bpftrace.sh will dynamically replace
+that string with the PID provided to the script.
+
+## Configuring SPDK Build
+
+```bash
+./configure --with-usdt
+```
+
+## Start SPDK application and bpftrace script
+
+From first terminal:
+
+```bash
+build/bin/spdk_tgt -m 0xC
+```
+
+From second terminal:
+
+```bash
+scripts/bpftrace.sh `pidof spdk_tgt` scripts/bpf/nvmf.bt
+```
+
+nvmf.bt will print information about nvmf subsystem and poll
+group info state transitions.
+
+From third terminal:
+
+```bash
+scripts/rpc.py <<EOF
+nvmf_create_transport -t tcp
+nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 -m 10
+nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t tcp -a 127.0.0.1 -s 4420
+bdev_null_create null0 1000 512
+nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 null0
+EOF
+```
+
+This creates the nvmf tcp transport, a new nvmf subsystem that listens on a tcp
+port, and a null bdev which is added as a namespace to the new nvmf subsystem.
+
+You will see output from the second terminal that looks like this:
+
+```bash
+2110.935735: nvmf_tgt reached state NONE
+2110.954316: nvmf_tgt reached state CREATE_TARGET
+2110.967905: nvmf_tgt reached state CREATE_POLL_GROUPS
+2111.235982: nvmf_tgt reached state START_SUBSYSTEMS
+2111.253560: nqn.2014-08.org.nvmexpress.discovery change state from INACTIVE to ACTIVE start
+2111.260278: nqn.2014-08.org.nvmexpress.discovery on thread 2 state to ACTIVE start
+2111.264281: nqn.2014-08.org.nvmexpress.discovery on thread 2 state to ACTIVE done
+2111.284083: nqn.2014-08.org.nvmexpress.discovery change state from INACTIVE to ACTIVE done
+2111.289197: nvmf_tgt reached state RUNNING
+2111.271573: nqn.2014-08.org.nvmexpress.discovery on thread 3 state to ACTIVE start
+2111.279787: nqn.2014-08.org.nvmexpress.discovery on thread 3 state to ACTIVE done
+2189.921492: nqn.2016-06.io.spdk:cnode1 change state from INACTIVE to ACTIVE start
+2189.952508: nqn.2016-06.io.spdk:cnode1 on thread 2 state to ACTIVE start
+2189.959125: nqn.2016-06.io.spdk:cnode1 on thread 2 state to ACTIVE done
+2190.005832: nqn.2016-06.io.spdk:cnode1 change state from INACTIVE to ACTIVE done
+2189.969058: nqn.2016-06.io.spdk:cnode1 on thread 3 state to ACTIVE start
+2189.999889: nqn.2016-06.io.spdk:cnode1 on thread 3 state to ACTIVE done
+2197.859104: nqn.2016-06.io.spdk:cnode1 change state from ACTIVE to PAUSED start
+2197.879199: nqn.2016-06.io.spdk:cnode1 on thread 2 state to PAUSED start
+2197.883416: nqn.2016-06.io.spdk:cnode1 on thread 2 state to PAUSED done
+2197.902291: nqn.2016-06.io.spdk:cnode1 change state from ACTIVE to PAUSED done
+2197.908939: nqn.2016-06.io.spdk:cnode1 change state from PAUSED to ACTIVE start
+2197.912644: nqn.2016-06.io.spdk:cnode1 on thread 2 state to ACTIVE start
+2197.927409: nqn.2016-06.io.spdk:cnode1 on thread 2 state to ACTIVE done
+2197.949742: nqn.2016-06.io.spdk:cnode1 change state from PAUSED to ACTIVE done
+2197.890812: nqn.2016-06.io.spdk:cnode1 on thread 3 state to PAUSED start
+2197.897233: nqn.2016-06.io.spdk:cnode1 on thread 3 state to PAUSED done
+2197.931278: nqn.2016-06.io.spdk:cnode1 on thread 3 state to ACTIVE start
+2197.946124: nqn.2016-06.io.spdk:cnode1 on thread 3 state to ACTIVE done
+2205.859904: nqn.2016-06.io.spdk:cnode1 change state from ACTIVE to PAUSED start
+2205.891392: nqn.2016-06.io.spdk:cnode1 on thread 2 state to PAUSED start
+2205.896588: nqn.2016-06.io.spdk:cnode1 on thread 2 state to PAUSED done
+2205.920133: nqn.2016-06.io.spdk:cnode1 change state from ACTIVE to PAUSED done
+2205.905900: nqn.2016-06.io.spdk:cnode1 on thread 3 state to PAUSED start
+2205.914856: nqn.2016-06.io.spdk:cnode1 on thread 3 state to PAUSED done
+2206.091084: nqn.2016-06.io.spdk:cnode1 change state from PAUSED to ACTIVE start
+2206.099222: nqn.2016-06.io.spdk:cnode1 on thread 2 state to ACTIVE start
+2206.105445: nqn.2016-06.io.spdk:cnode1 on thread 2 state to ACTIVE done
+2206.119271: nqn.2016-06.io.spdk:cnode1 change state from PAUSED to ACTIVE done
+2206.109144: nqn.2016-06.io.spdk:cnode1 on thread 3 state to ACTIVE start
+2206.115636: nqn.2016-06.io.spdk:cnode1 on thread 3 state to ACTIVE done
+```
+
+Now stop the bpftrace.sh running in the second terminal, and start
+it again with the send_msg.bt script.  This script keeps a count of
+functions executed as part of an spdk_for_each_channel or
+spdk_thread_send_msg function call.
+
+```bash
+scripts/bpftrace.sh `pidof spdk_tgt` scripts/bpf/send_msg.bt
+```
+
+From the third terminal, create another null bdev and add it as a
+namespace to the cnode1 subsystem.
+
+```bash
+scripts/rpc.py <<EOF
+bdev_null_create null1 1000 512
+nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 null1
+EOF
+```
+
+Now Ctrl-C the bpftrace.sh in the second terminal, and it will
+print the final results of the maps.
+
+```bash
+@for_each_channel[subsystem_state_change_on_pg]: 2
+
+@send_msg[_finish_unregister]: 1
+@send_msg[_call_completion]: 2
+@send_msg[put_io_channel]: 4
+@send_msg[_call_channel]: 4
+```
+
+## TODOs and known limitations
+
+- add definitions for common nvmf data structures - then we can pass the subsystem pointer
+  itself as a probe parameter and let the script decide which fields it wants to access
+  (Note: these would need to be kept up-to-date with the C definitions of the struct - it is
+  not possible to include the header files in a bpftrace script)
+- investigate using pahole to generate data structure definitions that can be included in
+  bpftrace scripts; this would allow us to pass the subsystem pointer itself as a probe
+  argument, and let the script decide which fields it wants to access; for example,
+  `pahole -E -C spdk_nvmf_subsystem build/bin/spdk_tgt` gets us close to what we need,
+  but there are some limiters:
+  - our structures have char arrays (not char pointers) for things like subnqn; large
+    arrays like these cannot currently be passed to bpftrace printf without generating
+    a stack space error (probe points are limited to 512 bytes of stack); we could
+    modify SPDK to have the char array for storage, and a char pointer that points to
+    that storage, the latter could easily then be used in bpftrace scripts
+  - our structures include fields with their enum types instead of int; bpftrace will
+    complain it does not know about the enum (pahole doesn't print out enum
+    descriptions); information on enums can be found in the applications .debug_info
+    section, but we would need something that can convert that into a file we can
+    include in a bpftrace script
+- Note that bpftrace prints are not always printed in exact chronological order; this can
+  be seen especially with spdk_for_each_channel iterations, where we execute trace points
+  on multiple threads in a very short period of time, and those may not get printed to the
+  console in exact order; this is why the nvmf.bt script prints out a msec.nsec timestamp
+  so that the user can understand the ordering and even pipe through sort if desired
+- flesh out more DTrace probes in the nvmf code

doc/user_guides.md

@@ -11,3 +11,4 @@
 - @subpage blobfs
 - @subpage jsonrpc
 - @subpage jsonrpc_proxy
+- @subpage usdt

doc/userspace.md

@@ -1,6 +1,6 @@
 # User Space Drivers {#userspace}
 
-# Controlling Hardware From User Space {#userspace_control}
+## Controlling Hardware From User Space {#userspace_control}
 
 Much of the documentation for SPDK talks about _user space drivers_, so it's
 important to understand what that means at a technical level. First and
@@ -53,7 +53,7 @@ with the
 [NVMe Specification](http://nvmexpress.org/wp-content/uploads/NVM_Express_Revision_1.3.pdf)
 to initialize the device, create queue pairs, and ultimately send I/O.
 
-# Interrupts {#userspace_interrupts}
+## Interrupts {#userspace_interrupts}
 
 SPDK polls devices for completions instead of waiting for interrupts. There
 are a number of reasons for doing this: 1) practically speaking, routing an
@@ -69,7 +69,7 @@ technologies such as Intel's
 will ensure that the host memory being checked is present in the CPU cache
 after an update by the device.
 
-# Threading {#userspace_threading}
+## Threading {#userspace_threading}
 
 NVMe devices expose multiple queues for submitting requests to the hardware.
 Separate queues can be accessed without coordination, so software can send

doc/vagrant.md

@@ -1,6 +1,6 @@
 # Vagrant Development Environment {#vagrant}
 
-# Introduction {#vagrant_intro}
+## Introduction {#vagrant_intro}
 
 [Vagrant](https://www.vagrantup.com/) provides a quick way to get a basic
 NVMe enabled virtual machine sandbox running without the need for any
@@ -22,7 +22,7 @@ vagrant plugin install vagrant-proxyconf
 
 In case you want use kvm/libvirt you should also install `vagrant-libvirt`
 
-# VM Configuration {#vagrant_config}
+## VM Configuration {#vagrant_config}
 
 To create a configured VM with vagrant you need to run `create_vbox.sh` script.
 
@@ -47,7 +47,7 @@ world example application.
     vagrant --help
 ~~~
 
-# Running An Example {#vagrant_example}
+## Running An Example {#vagrant_example}
 
 The following shows sample output from starting up a Ubuntu18 VM,
 compiling SPDK on it and running the NVMe sample application `hello_world`.

doc/vhost.md

@@ -1,6 +1,6 @@
 # vhost Target {#vhost}
 
-# Table of Contents {#vhost_toc}
+## Table of Contents {#vhost_toc}
 
 - @ref vhost_intro
 - @ref vhost_prereqs
@@ -11,7 +11,7 @@
 - @ref vhost_advanced_topics
 - @ref vhost_bugs
 
-# Introduction {#vhost_intro}
+## Introduction {#vhost_intro}
 
 A vhost target provides a local storage service as a process running on a local machine.
 It is capable of exposing virtualized block devices to QEMU instances or other arbitrary
@@ -28,12 +28,12 @@ techniques as other components in SPDK.  Since SPDK is polling for vhost submiss
 it can signal the VM to skip notifications on submission.  This avoids VMEXITs on I/O
 submission and can significantly reduce CPU usage in the VM on heavy I/O workloads.
 
-# Prerequisites {#vhost_prereqs}
+## Prerequisites {#vhost_prereqs}
 
 This guide assumes the SPDK has been built according to the instructions in @ref
 getting_started.  The SPDK vhost target is built with the default configure options.
 
-## Vhost Command Line Parameters {#vhost_cmd_line_args}
+### Vhost Command Line Parameters {#vhost_cmd_line_args}
 
 Additional command line flags are available for Vhost target.
 
@@ -41,7 +41,7 @@ Param    | Type     | Default                | Description
 -------- | -------- | ---------------------- | -----------
 -S       | string   | $PWD                   | directory where UNIX domain sockets will be created
 
-## Supported Guest Operating Systems
+### Supported Guest Operating Systems
 
 The guest OS must contain virtio-scsi or virtio-blk drivers.  Most Linux and FreeBSD
 distributions include virtio drivers.
@@ -49,7 +49,7 @@ distributions include virtio drivers.
 installed separately.  The SPDK vhost target has been tested with recent versions of Ubuntu,
 Fedora, and Windows
 
-## QEMU
+### QEMU
 
 Userspace vhost-scsi target support was added to upstream QEMU in v2.10.0.  Run
 the following command to confirm your QEMU supports userspace vhost-scsi.
@@ -65,16 +65,7 @@ the following command to confirm your QEMU supports userspace vhost-blk.
 qemu-system-x86_64 -device vhost-user-blk-pci,help
 ~~~
 
-Userspace vhost-nvme target was added as experimental feature for SPDK 18.04
-release, patches for QEMU are available in SPDK's QEMU repository only.
-
-Run the following command to confirm your QEMU supports userspace vhost-nvme.
-
-~~~{.sh}
-qemu-system-x86_64 -device vhost-user-nvme,help
-~~~
-
-# Starting SPDK vhost target {#vhost_start}
+## Starting SPDK vhost target {#vhost_start}
 
 First, run the SPDK setup.sh script to setup some hugepages for the SPDK vhost target
 application.  This will allocate 4096MiB (4GiB) of hugepages, enough for the SPDK
@@ -100,9 +91,9 @@ To list all available vhost options use the following command.
 build/bin/vhost -h
 ~~~
 
-# SPDK Configuration {#vhost_config}
+## SPDK Configuration {#vhost_config}
 
-## Create bdev (block device) {#vhost_bdev_create}
+### Create bdev (block device) {#vhost_bdev_create}
 
 SPDK bdevs are block devices which will be exposed to the guest OS.
 For vhost-scsi, bdevs are exposed as SCSI LUNs on SCSI devices attached to the
@@ -121,9 +112,9 @@ will create a 64MB malloc bdev with 512-byte block size.
 scripts/rpc.py bdev_malloc_create 64 512 -b Malloc0
 ~~~
 
-## Create a vhost device {#vhost_vdev_create}
+### Create a vhost device {#vhost_vdev_create}
 
-### Vhost-SCSI
+#### Vhost-SCSI
 
 The following RPC will create a vhost-scsi controller which can be accessed
 by QEMU via /var/tmp/vhost.0. At the time of creation the controller will be
@@ -152,7 +143,7 @@ To remove a bdev from a vhost-scsi controller use the following RPC:
 scripts/rpc.py vhost_scsi_controller_remove_target vhost.0 0
 ~~~
 
-### Vhost-BLK
+#### Vhost-BLK
 
 The following RPC will create a vhost-blk device exposing Malloc0 bdev.
 The device will be accessible to QEMU via /var/tmp/vhost.1. All the I/O polling
@@ -171,7 +162,7 @@ extra `-r` or `--readonly` parameter.
 scripts/rpc.py vhost_create_blk_controller --cpumask 0x1 -r vhost.1 Malloc0
 ~~~
 
-## QEMU {#vhost_qemu_config}
+### QEMU {#vhost_qemu_config}
 
 Now the virtual machine can be started with QEMU.  The following command-line
 parameters must be added to connect the virtual machine to its vhost controller.
@@ -195,21 +186,21 @@ SPDK malloc block device by specifying bootindex=0 for the boot image.
 
 Finally, specify the SPDK vhost devices:
 
-### Vhost-SCSI
+#### Vhost-SCSI
 
 ~~~{.sh}
 -chardev socket,id=char0,path=/var/tmp/vhost.0
 -device vhost-user-scsi-pci,id=scsi0,chardev=char0
 ~~~
 
-### Vhost-BLK
+#### Vhost-BLK
 
 ~~~{.sh}
 -chardev socket,id=char1,path=/var/tmp/vhost.1
 -device vhost-user-blk-pci,id=blk0,chardev=char1
 ~~~
 
-## Example output {#vhost_example}
+### Example output {#vhost_example}
 
 This example uses an NVMe bdev alongside Mallocs. SPDK vhost application is started
 on CPU cores 0 and 1, QEMU on cores 2 and 3.
@@ -280,9 +271,9 @@ host:~# taskset -c 2,3 qemu-system-x86_64 \
   -drive file=guest_os_image.qcow2,if=none,id=disk \
   -device ide-hd,drive=disk,bootindex=0 \
   -chardev socket,id=spdk_vhost_scsi0,path=/var/tmp/vhost.0 \
-  -device vhost-user-scsi-pci,id=scsi0,chardev=spdk_vhost_scsi0,num_queues=4 \
+  -device vhost-user-scsi-pci,id=scsi0,chardev=spdk_vhost_scsi0,num_queues=2 \
   -chardev socket,id=spdk_vhost_blk0,path=/var/tmp/vhost.1 \
-  -device vhost-user-blk-pci,chardev=spdk_vhost_blk0,num-queues=4
+  -device vhost-user-blk-pci,chardev=spdk_vhost_blk0,num-queues=2
 ~~~
 
 Please note the following two commands are run on the guest VM.
@@ -310,9 +301,9 @@ vhost.c:1006:session_shutdown: *NOTICE*: Exiting
 We can see that `sdb` and `sdc` are SPDK vhost-scsi LUNs, and `vda` is SPDK a
 vhost-blk disk.
 
-# Advanced Topics {#vhost_advanced_topics}
+## Advanced Topics {#vhost_advanced_topics}
 
-## Multi-Queue Block Layer (blk-mq) {#vhost_multiqueue}
+### Multi-Queue Block Layer (blk-mq) {#vhost_multiqueue}
 
 For best performance use the Linux kernel block multi-queue feature with vhost.
 To enable it on Linux, it is required to modify kernel options inside the
@@ -331,7 +322,11 @@ to set `num_queues=4` to saturate physical device. Adding too many queues might
 vhost performance degradation if many vhost devices are used because each device will require
 additional `num_queues` to be polled.
 
-## Hot-attach/hot-detach {#vhost_hotattach}
+Some Linux distributions report a kernel panic when starting the VM if the number of I/O queues
+specified via the `num-queues` parameter is greater than number of vCPUs. If you need to use
+more I/O queues than vCPUs, check that your OS image supports that configuration.
+
+### Hot-attach/hot-detach {#vhost_hotattach}
 
 Hotplug/hotremove within a vhost controller is called hot-attach/detach. This is to
 distinguish it from SPDK bdev hotplug/hotremove. E.g. if an NVMe bdev is attached
@@ -344,7 +339,7 @@ to hot-attach/detach the bdev from a Vhost-BLK device. If Vhost-BLK device expos
 an NVMe bdev that is hotremoved, all the I/O traffic on that Vhost-BLK device will
 be aborted - possibly flooding a VM with syslog warnings and errors.
 
-### Hot-attach
+#### Hot-attach
 
 Hot-attach is done by simply attaching a bdev to a vhost controller with a QEMU VM
 already started. No other extra action is necessary.
@@ -353,7 +348,7 @@ already started. No other extra action is necessary.
 scripts/rpc.py vhost_scsi_controller_add_target vhost.0 0 Malloc0
 ~~~
 
-### Hot-detach
+#### Hot-detach
 
 Just like hot-attach, the hot-detach is done by simply removing bdev from a controller
 when QEMU VM is already started.
@@ -368,22 +363,16 @@ Removing an entire bdev will hot-detach it from a controller as well.
 scripts/rpc.py bdev_malloc_delete Malloc0
 ~~~
 
-# Known bugs and limitations {#vhost_bugs}
+## Known bugs and limitations {#vhost_bugs}
 
-## Vhost-NVMe (experimental) can only be supported with latest Linux kernel
-
-Vhost-NVMe target was designed for one new feature of NVMe 1.3 specification, Doorbell
-Buffer Config Admin command, which is used for emulated NVMe controller only. Linux 4.12
-added this feature, so a new Guest kernel later than 4.12 is required to test this feature.
-
-## Windows virtio-blk driver before version 0.1.130-1 only works with 512-byte sectors
+### Windows virtio-blk driver before version 0.1.130-1 only works with 512-byte sectors
 
 The Windows `viostor` driver before version 0.1.130-1 is buggy and does not
 correctly support vhost-blk devices with non-512-byte block size.
 See the [bug report](https://bugzilla.redhat.com/show_bug.cgi?id=1411092) for
 more information.
 
-## QEMU vhost-user-blk
+### QEMU vhost-user-blk
 
 QEMU [vhost-user-blk](https://git.qemu.org/?p=qemu.git;a=commit;h=00343e4b54ba) is
 supported from version 2.12.

doc/vhost_processing.md

@@ -1,6 +1,6 @@
 # Virtualized I/O with Vhost-user {#vhost_processing}
 
-# Table of Contents {#vhost_processing_toc}
+## Table of Contents {#vhost_processing_toc}
 
 - @ref vhost_processing_intro
 - @ref vhost_processing_qemu
@@ -8,7 +8,7 @@
 - @ref vhost_processing_io_path
 - @ref vhost_spdk_optimizations
 
-# Introduction {#vhost_processing_intro}
+## Introduction {#vhost_processing_intro}
 
 This document is intended to provide an overview of how Vhost works behind the
 scenes. Code snippets used in this document might have been simplified for the
@@ -18,12 +18,10 @@ reference.
 Reading from the
 [Virtio specification](http://docs.oasis-open.org/virtio/virtio/v1.0/virtio-v1.0.html):
 
-```
-The purpose of virtio and [virtio] specification is that virtual environments
-and guests should have a straightforward, efficient, standard and extensible
-mechanism for virtual devices, rather than boutique per-environment or per-OS
-mechanisms.
-```
+> The purpose of virtio and [virtio] specification is that virtual environments
+> and guests should have a straightforward, efficient, standard and extensible
+> mechanism for virtual devices, rather than boutique per-environment or per-OS
+> mechanisms.
 
 Virtio devices use virtqueues to transport data efficiently. Virtqueue is a set
 of three different single-producer, single-consumer ring structures designed to
@@ -47,28 +45,26 @@ SPDK to expose a vhost device is Vhost-user protocol.
 The [Vhost-user specification](https://git.qemu.org/?p=qemu.git;a=blob_plain;f=docs/interop/vhost-user.txt;hb=HEAD)
 describes the protocol as follows:
 
-```
-[Vhost-user protocol] is aiming to complement the ioctl interface used to
-control the vhost implementation in the Linux kernel. It implements the control
-plane needed to establish virtqueue sharing with a user space process on the
-same host. It uses communication over a Unix domain socket to share file
-descriptors in the ancillary data of the message.
-
-The protocol defines 2 sides of the communication, master and slave. Master is
-the application that shares its virtqueues, in our case QEMU. Slave is the
-consumer of the virtqueues.
-
-In the current implementation QEMU is the Master, and the Slave is intended to
-be a software Ethernet switch running in user space, such as Snabbswitch.
-
-Master and slave can be either a client (i.e. connecting) or server (listening)
-in the socket communication.
-```
+> [Vhost-user protocol] is aiming to complement the ioctl interface used to
+> control the vhost implementation in the Linux kernel. It implements the control
+> plane needed to establish virtqueue sharing with a user space process on the
+> same host. It uses communication over a Unix domain socket to share file
+> descriptors in the ancillary data of the message.
+>
+> The protocol defines 2 sides of the communication, master and slave. Master is
+> the application that shares its virtqueues, in our case QEMU. Slave is the
+> consumer of the virtqueues.
+>
+> In the current implementation QEMU is the Master, and the Slave is intended to
+> be a software Ethernet switch running in user space, such as Snabbswitch.
+>
+> Master and slave can be either a client (i.e. connecting) or server (listening)
+> in the socket communication.
 
 SPDK vhost is a Vhost-user slave server. It exposes Unix domain sockets and
 allows external applications to connect.
 
-# QEMU {#vhost_processing_qemu}
+## QEMU {#vhost_processing_qemu}
 
 One of major Vhost-user use cases is networking (DPDK) or storage (SPDK)
 offload in QEMU. The following diagram presents how QEMU-based VM
@@ -76,7 +72,7 @@ communicates with SPDK Vhost-SCSI device.
 
 ![QEMU/SPDK vhost data flow](img/qemu_vhost_data_flow.svg)
 
-# Device initialization {#vhost_processing_init}
+## Device initialization {#vhost_processing_init}
 
 All initialization and management information is exchanged using Vhost-user
 messages. The connection always starts with the feature negotiation. Both
@@ -90,13 +86,13 @@ physically-discontiguous regions and Vhost-user specification puts a limit on
 their number - currently 8. The driver sends a single message for each region with
 the following data:
 
- * file descriptor - for mmap
- * user address - for memory translations in Vhost-user messages (e.g.
-   translating vring addresses)
- * guest address - for buffers addresses translations in vrings (for QEMU this
-   is a physical address inside the guest)
- * user offset - positive offset for the mmap
- * size
+- file descriptor - for mmap
+- user address - for memory translations in Vhost-user messages (e.g.
+  translating vring addresses)
+- guest address - for buffers addresses translations in vrings (for QEMU this
+  is a physical address inside the guest)
+- user offset - positive offset for the mmap
+- size
 
 The Master will send new memory regions after each memory change - usually
 hotplug/hotremove. The previous mappings will be removed.
@@ -108,24 +104,24 @@ as they use common SCSI I/O to inquiry the underlying disk(s).
 Afterwards, the driver requests the number of maximum supported queues and
 starts sending virtqueue data, which consists of:
 
- * unique virtqueue id
- * index of the last processed vring descriptor
- * vring addresses (from user address space)
- * call descriptor (for interrupting the driver after I/O completions)
- * kick descriptor (to listen for I/O requests - unused by SPDK)
+- unique virtqueue id
+- index of the last processed vring descriptor
+- vring addresses (from user address space)
+- call descriptor (for interrupting the driver after I/O completions)
+- kick descriptor (to listen for I/O requests - unused by SPDK)
 
 If multiqueue feature has been negotiated, the driver has to send a specific
 *ENABLE* message for each extra queue it wants to be polled. Other queues are
 polled as soon as they're initialized.
 
-# I/O path {#vhost_processing_io_path}
+## I/O path {#vhost_processing_io_path}
 
 The Master sends I/O by allocating proper buffers in shared memory, filling
 the request data, and putting guest addresses of those buffers into virtqueues.
 
 A Virtio-Block request looks as follows.
 
-```
+```c
 struct virtio_blk_req {
         uint32_t type; // READ, WRITE, FLUSH (read-only)
         uint64_t offset; // offset in the disk (read-only)
@@ -135,7 +131,7 @@ struct virtio_blk_req {
 ```
 And a Virtio-SCSI request as follows.
 
-```
+```c
 struct virtio_scsi_req_cmd {
   struct virtio_scsi_cmd_req *req; // request data (read-only)
   struct iovec read_only_buffers[]; // scatter-gatter list for WRITE I/Os
@@ -149,7 +145,7 @@ to be converted into a chain of such descriptors. A single descriptor can be
 either readable or writable, so each I/O request consists of at least two
 (request + response).
 
-```
+```c
 struct virtq_desc {
         /* Address (guest-physical). */
         le64 addr;
@@ -186,7 +182,7 @@ proper data and interrupts the guest by doing an eventfd_write on the call
 descriptor for proper virtqueue. There are multiple interrupt coalescing
 features involved, but they are not be discussed in this document.
 
-## SPDK optimizations {#vhost_spdk_optimizations}
+### SPDK optimizations {#vhost_spdk_optimizations}
 
 Due to its poll-mode nature, SPDK vhost removes the requirement for I/O submission
 notifications, drastically increasing the vhost server throughput and decreasing

doc/virtio.md

@@ -1,6 +1,6 @@
 # Virtio driver {#virtio}
 
-# Introduction {#virtio_intro}
+## Introduction {#virtio_intro}
 
 SPDK Virtio driver is a C library that allows communicating with Virtio devices.
 It allows any SPDK application to become an initiator for (SPDK) vhost targets.
@@ -20,7 +20,7 @@ This Virtio library is currently used to implement two bdev modules:
 @ref bdev_config_virtio_scsi and @ref bdev_config_virtio_blk.
 These modules will export generic SPDK block devices usable by any SPDK application.
 
-# 2MB hugepages {#virtio_2mb}
+## 2MB hugepages {#virtio_2mb}
 
 vhost-user specification puts a limitation on the number of "memory regions" used (8).
 Each region corresponds to one file descriptor, and DPDK - as SPDK's memory allocator -

doc/vmd.md

@@ -1,6 +1,6 @@
 # VMD driver {#vmd}
 
-# In this document {#vmd_toc}
+## In this document {#vmd_toc}
 
 * @ref vmd_intro
 * @ref vmd_interface
@@ -10,7 +10,7 @@
 * @ref vmd_app
 * @ref vmd_led
 
-# Introduction {#vmd_intro}
+## Introduction {#vmd_intro}
 
 Intel Volume Management Device is a hardware logic inside processor's Root Complex
 responsible for management of PCIe NVMe SSDs. It provides robust Hot Plug support
@@ -19,11 +19,11 @@ and Status LED management.
 The driver is responsible for enumeration and hooking NVMe devices behind VMD
 into SPDK PCIe subsystem. It also provides API for LED management and hot plug.
 
-# Public Interface {#vmd_interface}
+## Public Interface {#vmd_interface}
 
 - spdk/vmd.h
 
-# Key Functions {#vmd_key_functions}
+## Key Functions {#vmd_key_functions}
 
 Function                                | Description
 --------------------------------------- | -----------
@@ -33,7 +33,7 @@ spdk_vmd_set_led_state()                | @copybrief spdk_vmd_set_led_state()
 spdk_vmd_get_led_state()                | @copybrief spdk_vmd_get_led_state()
 spdk_vmd_hotplug_monitor()              | @copybrief spdk_vmd_hotplug_monitor()
 
-# Configuration {#vmd_config}
+## Configuration {#vmd_config}
 
 To enable VMD driver enumeration, the following steps are required:
 
@@ -75,7 +75,7 @@ Example:
 $ ./scripts/rpc.py bdev_nvme_attach_controller -b NVMe1 -t PCIe -a 5d0505:01:00.0
 ```
 
-# Application framework {#vmd_app_frame}
+## Application framework {#vmd_app_frame}
 
 When application framework is used, VMD section needs to be added to the configuration file:
 
@@ -98,7 +98,7 @@ $ ./build/bin/spdk_tgt --wait_for_rpc
 $ ./scripts/rpc.py enable_vmd
 $ ./scripts/rpc.py framework_start_init
 ```
-# Applications w/o application framework {#vmd_app}
+## Applications w/o application framework {#vmd_app}
 
 To enable VMD enumeration in SPDK application that are not using application framework
 e.g nvme/perf, nvme/identify -V flag is required - please refer to app help if it supports VMD.
@@ -107,7 +107,7 @@ Applications need to call spdk_vmd_init() to enumerate NVMe devices behind the V
 spdk_nvme_(probe|connect).
 To support hot plugs spdk_vmd_hotplug_monitor() needs to be called periodically.
 
-# LED management {#vmd_led}
+## LED management {#vmd_led}
 
 VMD LED utility in the [examples/vmd/led](https://github.com/spdk/spdk/tree/master/examples/vmd/led)
 could be used to set LED states.

docker/README.md

@@ -0,0 +1,99 @@
+# SPDK Docker suite
+
+This suite is meant to serve as an example of how SPDK can be encapsulated
+into docker container images. The example containers consist of SPDK NVMe-oF
+target sharing devices to another SPDK NVMe-oF application. Which serves
+as both initiator and target. Finally a traffic generator based on FIO
+issues I/O to the connected devices.
+
+## Prerequisites
+
+docker: We recommend version 20.10 and above because it supports cgroups v2 for
+customization of host resources like CPUs, memory, and block I/O.
+
+docker-compose: We recommend using 1.29.2 version or newer.
+
+kernel: Hugepages must be allocated prior running the containers and hugetlbfs
+mount must be available under /dev/hugepages. Also, tmpfs should be mounted
+under /dev/shm. Depending on the use-case, some kernel modules should be also
+loaded into the kernel prior running the containers.
+
+proxy: If you are working behind firewall make sure dockerd is aware of the
+proxy. Please refer to:
+[docker-proxy](https://docs.docker.com/config/daemon/systemd/#httphttps-proxy)
+
+To pass `$http_proxy` to docker-compose build use:
+~~~{.sh}
+docker-compose build --build-arg PROXY=$http_proxy
+~~~
+
+## How-To
+
+`docker-compose.yaml` shows an example deployment of the storage containers based on SPDK.
+Running `docker-compose build` creates 5 docker images:
+
+- build_base
+- storage-target
+- proxy-container
+- traffic-generator-nvme
+- traffic-generator-virtio
+
+The `build_base` image provides the core components required to containerize SPDK
+applications. The fedora:33 image from the Fedora Container Registry is used and then SPDK is installed. SPDK is installed out of `build_base/spdk.tar.gz` provided.
+See `build_base` folder for details on what's included in the final image.
+
+Running `docker-compose up` creates 3 docker containers:
+
+-- storage-target: Contains SPDK NVMe-oF target exposing single subsystem to
+`proxy-container` based on malloc bdev.
+-- proxy-container: Contains SPDK NVMe-oF target connecting to `storage-target`
+and then exposing the same devices to `traffic-generator-nvme` using NVMe-oF and
+to `traffic-generator-virtio` using Virtio.
+-- traffic-generator-nvme: Contains FIO using SPDK plugin to connect to `proxy-container`
+and runs a sample workload.
+-- traffic-generator-virtio: Contains FIO using SPDK plugin to connect to `proxy-container`
+and runs a sample workload.
+
+Each container is connected to a separate "spdk" network which is created before
+deploying the containers. See `docker-compose.yaml` for the network's detailed setup and ip assignment.
+
+All the above boils down to:
+
+~~~{.sh}
+cd docker
+tar -czf build_base/spdk.tar.gz --exclude='docker/*' -C .. .
+docker-compose build
+docker-compose up
+~~~
+
+The `storage-target` and `proxy-container` can be started as services.
+Allowing for multiple traffic generator containers to connect.
+
+~~~{.sh}
+docker-compose up -d proxy-container
+docker-compose run traffic-generator-nvme
+docker-compose run traffic-generator-virtio
+~~~
+
+Enviroment variables to containers can be passed as shown in
+[docs](https://docs.docker.com/compose/environment-variables/).
+For example extra arguments to fio can be passed as so:
+
+~~~{.sh}
+docker-compose run -e FIO_ARGS="--minimal" traffic-generator-nvme
+~~~
+
+As each container includes SPDK installation it is possible to use rpc.py to
+examine the final setup. E.g.:
+
+~~~{.sh}
+docker-compose exec storage-target rpc.py bdev_get_bdevs
+docker-compose exec proxy-container rpc.py nvmf_get_subsystems
+~~~
+
+## Caveats
+
+- If you run docker < 20.10 under distro which switched fully to cgroups2
+  (e.g. f33) make sure that /sys/fs/cgroup/systemd exists otherwise docker/build
+  will simply fail.
+- Each SPDK app inside the containers is limited to single, separate CPU.

docker/build_base/Dockerfile

@@ -0,0 +1,40 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) Intel Corporation
+
+FROM fedora:33 AS base
+
+# Generic args
+ARG PROXY
+ARG NO_PROXY
+
+ENV http_proxy=$PROXY
+ENV https_proxy=$PROXY
+ENV no_proxy=$NO_PROXY
+
+
+COPY spdk.tar.gz /spdk.tar.gz
+COPY pre-install /install
+RUN /install
+
+# We are doing a multi-stage build here. This means that previous image,
+# base, is going to end up as an intermediate one, untagged, <none> - this
+# image can be then manually removed (--force-rm doesn't work here. Go
+# figure).
+FROM fedora:33 AS spdk
+
+LABEL maintainer=spdk.io
+
+# Proxy configuration must be set for each build separately...
+ARG PROXY
+ARG NO_PROXY
+
+ENV http_proxy=$PROXY
+ENV https_proxy=$PROXY
+ENV no_proxy=$NO_PROXY
+
+# Copy SPDK's RPMs built during pre-install step.
+COPY --from=base /tmp/*.rpm /tmp/
+COPY --from=base /tmp/fio /tmp/
+# Wrap up the image
+COPY post-install /install
+RUN /install

docker/build_base/post-install

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -e
+
+dnf install -y /tmp/*.rpm
+
+# Be nice for docker exec and link SPDK scripts|binaries under common PATH
+# location like /usr/bin.
+ln -sf $(ls -1dpA /usr/local/bin/* | grep -v "/$") /usr/bin
+ln -sf $(ls -1dpA /usr/local/bin/fio/* | grep -v "/$") /usr/bin
+ln -s /usr/libexec/spdk/scripts/rpc.py /usr/bin
+ln -s /usr/libexec/spdk/scripts/rpc_http_proxy.py /usr/bin
+ln -s /usr/libexec/spdk/scripts/setup.sh /usr/bin
+ln -s /usr/libexec/spdk/include/spdk /usr/include
+ln -s /usr/libexec/spdk/scripts/ /usr
+
+mkdir -p /usr/src/fio
+mv /tmp/fio /usr/src/fio
+
+dnf clean all
+rm -f /tmp/*.rpm

docker/build_base/pre-install

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+set -e
+
+spdk_repo=$(mktemp -dt "spdk.XXXXXX")
+spdk_tar=/spdk.tar.gz
+
+cleanup() {
+
+	rm -f "$HOME/rpmbuild/rpm/x86_64/"*.rpm
+	rm -f "$spdk_tar"
+	rm -rf "$spdk_repo"
+}
+
+trap 'cleanup' EXIT
+
+if [[ ! -e $spdk_tar ]]; then
+	printf 'Missing %s\n' "$spdk_tar" >&2
+	exit 1
+fi
+
+tar -C "$spdk_repo" -xf "$spdk_tar"
+
+# Required for building RPM
+dnf install -y rpm-build
+
+# Spice it a bit with supported sources
+"$spdk_repo/scripts/pkgdep.sh" -d
+"$spdk_repo/test/common/config/vm_setup.sh" --test-conf=fio
+
+# HACK: In case we received a .tar with built SPDK we need to overwrite the
+# configuration to update all the paths make would need to lookup - this is
+# needed since we execute inside a different mount namespace so we won't be
+# able to find any absolute paths that were used prior creating the .tar.
+"$spdk_repo/configure" > /dev/null
+
+# Deploy SPDK inside the container
+DEPS="no" "$spdk_repo/rpmbuild/rpm.sh" \
+	--with-shared \
+	--with-virtio \
+	--with-fio
+
+mv "$HOME/rpmbuild/rpm/x86_64/"*.rpm /tmp
+mv "/usr/src/fio/fio" /tmp
+dnf clean all

docker/docker-compose.yaml

@@ -0,0 +1,80 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) Intel Corporation
+version: "3.8"
+services:
+  build_base:
+    image: spdk
+    build:
+      context: build_base
+    container_name: build_base
+  storage-target:
+    image: spdk-app
+    build:
+      context: spdk-app
+    container_name: storage-target
+    depends_on:
+      - build_base
+    networks:
+      spdk:
+        ipv4_address: 192.168.42.2
+    volumes:
+      - /dev/hugepages:/dev/hugepages
+      - ./spdk-app/storage-target.conf:/config
+    environment:
+      - SPDK_ARGS=-m 0x2
+    privileged: true
+  proxy-container:
+    image: spdk-app
+    build:
+      context: spdk-app
+    container_name: proxy-container
+    depends_on:
+      - storage-target
+    networks:
+      spdk:
+        ipv4_address: 192.168.42.3
+    volumes:
+      - /dev/hugepages:/dev/hugepages
+      - ./spdk-app/proxy-container.conf:/config
+      - vhost-user:/vhost-user
+    environment:
+      - SPDK_ARGS=-m 0x4 -S /vhost-user
+    privileged: true
+  traffic-generator-virtio:
+    image: traffic-generator
+    build:
+      context: traffic-generator
+    container_name: traffic-generator-virtio
+    depends_on:
+      - proxy-container
+    networks:
+      spdk:
+    volumes:
+      - /dev/hugepages:/dev/hugepages
+      - ./traffic-generator/conf-virtio:/config
+      - vhost-user:/vhost-user
+      - ./traffic-generator/fio-virtio.conf:/fio.conf
+    privileged: true
+  traffic-generator-nvme:
+    image: traffic-generator
+    build:
+      context: traffic-generator
+    container_name: traffic-generator-nvme
+    depends_on:
+      - proxy-container
+    networks:
+      spdk:
+    volumes:
+      - /dev/hugepages:/dev/hugepages
+      - ./traffic-generator/conf-nvme:/config
+      - ./traffic-generator/fio-nvme.conf:/fio.conf
+    privileged: true
+networks:
+  spdk:
+    name: "spdk"
+    ipam:
+      config:
+        - subnet: 192.168.42.0/29
+          gateway: 192.168.42.1
+volumes:
+  vhost-user:

docker/spdk-app/Dockerfile

@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) Intel Corporation
+
+FROM spdk
+
+# Generic args
+ARG PROXY
+ARG NO_PROXY
+
+ENV http_proxy=$PROXY
+ENV https_proxy=$PROXY
+ENV no_proxy=$NO_PROXY
+
+
+COPY init /init
+
+ENTRYPOINT ["/init"]

docker/spdk-app/init

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+app=spdk_tgt args=() limit_args=()
+
+# Override default app
+if [[ -n $SPDK_APP ]]; then
+	app=$SPDK_APP
+fi
+
+# Define extra arguments to the app
+if [[ -n $SPDK_ARGS ]]; then
+	args=($SPDK_ARGS)
+fi
+
+# Limit the app with to following options,
+# to allow for minimal impact on the host.
+limit_args+=("--no-pci")
+limit_args+=("--num-trace-entries" 0)
+
+# if set, don't include limit_args[] on the cmdline
+if [[ ! -v SPDK_NO_LIMIT ]]; then
+	args+=("${limit_args[@]}")
+fi
+
+if [[ -e /config ]]; then
+	args+=("--json" "/config")
+fi
+
+# Wait a bit to make sure ip is in place
+sleep 2s
+
+exec "$app" "${args[@]}"

docker/spdk-app/proxy-container.conf

@@ -0,0 +1,80 @@
+{
+  "subsystems": [
+    {
+      "subsystem": "bdev",
+      "config": [
+        {
+          "method": "bdev_nvme_attach_controller",
+          "params": {
+            "name": "Nvme0",
+            "trtype": "TCP",
+            "adrfam": "IPv4",
+            "traddr": "192.168.42.2",
+            "trsvcid": "4420",
+            "subnqn": "nqn.2016-06.io.spdk:cnode1",
+            "prchk_reftag": false,
+            "prchk_guard": false
+          }
+        }
+      ]
+    },
+    {
+      "subsystem": "nvmf",
+      "config": [
+        {
+          "method": "nvmf_create_transport",
+          "params": {
+            "trtype": "TCP",
+            "io_unit_size": 8192
+          }
+        },
+        {
+          "method": "nvmf_create_subsystem",
+          "params": {
+            "nqn": "nqn.2016-06.io.spdk:cnode1",
+            "allow_any_host": true,
+            "serial_number": "SPDK00000000000001",
+            "model_number": "SPDK bdev Controller",
+            "max_namespaces": 32,
+            "min_cntlid": 1,
+            "max_cntlid": 65519
+          }
+        },
+        {
+          "method": "nvmf_subsystem_add_ns",
+          "params": {
+            "nqn": "nqn.2016-06.io.spdk:cnode1",
+            "namespace": {
+              "nsid": 1,
+              "bdev_name": "Nvme0n1"
+            }
+          }
+        },
+        {
+          "method": "nvmf_subsystem_add_listener",
+          "params": {
+            "nqn": "nqn.2016-06.io.spdk:cnode1",
+            "listen_address": {
+              "trtype": "TCP",
+              "adrfam": "IPv4",
+              "traddr": "192.168.42.3",
+              "trsvcid": "4420"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "subsystem": "vhost",
+      "config": [
+        {
+          "method": "vhost_create_blk_controller",
+          "params": {
+            "ctrlr": "VirtioBlk0",
+            "dev_name": "Nvme0n2"
+          }
+        }
+      ]
+    }
+  ]
+}

docker/spdk-app/storage-target.conf

@@ -0,0 +1,81 @@
+{
+  "subsystems": [
+    {
+      "subsystem": "bdev",
+      "config": [
+        {
+          "method": "bdev_malloc_create",
+          "params": {
+            "name": "Malloc0",
+            "num_blocks": 131072,
+            "block_size": 512
+          }
+        },
+        {
+          "method": "bdev_malloc_create",
+          "params": {
+            "name": "Malloc1",
+            "num_blocks": 131072,
+            "block_size": 512
+          }
+        }
+      ]
+    },
+    {
+      "subsystem": "nvmf",
+      "config": [
+        {
+          "method": "nvmf_create_transport",
+          "params": {
+            "trtype": "TCP",
+            "io_unit_size": 8192
+          }
+        },
+        {
+          "method": "nvmf_create_subsystem",
+          "params": {
+            "nqn": "nqn.2016-06.io.spdk:cnode1",
+            "allow_any_host": true,
+            "serial_number": "SPDK00000000000001",
+            "model_number": "SPDK bdev Controller",
+            "max_namespaces": 32,
+            "min_cntlid": 1,
+            "max_cntlid": 65519
+          }
+        },
+        {
+          "method": "nvmf_subsystem_add_ns",
+          "params": {
+            "nqn": "nqn.2016-06.io.spdk:cnode1",
+            "namespace": {
+              "nsid": 1,
+              "bdev_name": "Malloc0"
+            }
+          }
+        },
+        {
+          "method": "nvmf_subsystem_add_ns",
+          "params": {
+            "nqn": "nqn.2016-06.io.spdk:cnode1",
+            "namespace": {
+              "nsid": 2,
+              "bdev_name": "Malloc1"
+            }
+          }
+        },
+        {
+          "method": "nvmf_subsystem_add_listener",
+          "params": {
+            "nqn": "nqn.2016-06.io.spdk:cnode1",
+            "listen_address": {
+              "trtype": "TCP",
+              "adrfam": "IPv4",
+              "traddr": "192.168.42.2",
+              "trsvcid": "4420"
+            }
+          }
+        }
+      ]
+    }
+  ]
+}

docker/traffic-generator/Dockerfile

@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) Intel Corporation
+
+FROM spdk
+
+# Generic args
+ARG PROXY
+ARG NO_PROXY
+
+ENV http_proxy=$PROXY
+ENV https_proxy=$PROXY
+ENV no_proxy=$NO_PROXY
+
+
+COPY init /init
+
+ENTRYPOINT ["/init"]

docker/traffic-generator/conf-nvme

@@ -0,0 +1,22 @@
+{
+  "subsystems": [
+    {
+      "subsystem": "bdev",
+      "config": [
+        {
+          "method": "bdev_nvme_attach_controller",
+          "params": {
+            "name": "Nvme0",
+            "trtype": "TCP",
+            "adrfam": "IPv4",
+            "traddr": "192.168.42.3",
+            "trsvcid": "4420",
+            "subnqn": "nqn.2016-06.io.spdk:cnode1",
+            "prchk_reftag": false,
+            "prchk_guard": false
+          }
+        }
+      ]
+    }
+  ]
+}

docker/traffic-generator/conf-virtio

@@ -6,12 +6,10 @@
         {
           "method": "bdev_virtio_attach_controller",
           "params": {
-            "vq_count": 2,
-            "traddr": "Malloc.0",
-            "dev_type": "blk",
-            "vq_size": 512,
             "name": "VirtioBlk0",
-            "trtype": "user"
+            "trtype": "user",
+            "traddr": "/vhost-user/VirtioBlk0",
+            "dev_type": "blk"
           }
         }
       ]

docker/traffic-generator/fio-nvme.conf

@@ -0,0 +1,17 @@
+[global]
+ioengine=spdk_bdev
+spdk_json_conf=/config
+spdk_mem=1024
+thread=1
+direct=1
+rw=randread
+ramp_time=0
+norandommap=1
+time_based=1
+bs=4k
+numjobs=1
+runtime=10
+
+[filename0]
+filename=Nvme0n1
+iodepth=128

docker/traffic-generator/fio-virtio.conf

@@ -0,0 +1,18 @@
+[global]
+ioengine=spdk_bdev
+spdk_json_conf=/config
+spdk_single_seg=1
+spdk_mem=1024
+thread=1
+direct=1
+rw=randread
+ramp_time=0
+norandommap=1
+time_based=1
+bs=4k
+numjobs=1
+runtime=10
+
+[filename0]
+filename=VirtioBlk0
+iodepth=128

docker/traffic-generator/init

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+args=()
+
+# Define extra arguments to the app
+if [[ -n $FIO_ARGS ]]; then
+	args+=($FIO_ARGS)
+fi
+
+# Wait a bit to make sure ip is in place
+sleep 2s
+
+export LD_PRELOAD=/usr/local/bin/fio/spdk_bdev
+exec /usr/src/fio/fio "${args[@]}" /fio.conf

dpdk

@@ -1 +1 @@
-Subproject commit eb16786836e3a8380bb86fde67efa2ee0d9d3852
+Subproject commit 2e2390492155e15f506ca4b3e4456c7e53ed1f68

dpdkbuild/Makefile

@@ -3,6 +3,7 @@
 #
 #  Copyright (c) Intel Corporation.
 #  All rights reserved.
+#  Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 #  Redistribution and use in source and binary forms, with or without
 #  modification, are permitted provided that the following conditions
@@ -37,6 +38,7 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 .PHONY: all clean install uninstall
 
 DPDK_OPTS = -Denable_docs=false
+DPDK_OPTS += -Dtests=false
 DPDK_CFLAGS =
 
 DPDK_KMODS = false
@@ -52,24 +54,53 @@ endif
 # the drivers we use
 DPDK_DRIVERS = bus bus/pci bus/vdev mempool/ring
 
+# Core DPDK libs
+DPDK_LIBS = eal ring mempool pci
+DPDK_LIBS += kvargs telemetry
+# Governor required libs
+DPDK_LIBS += power timer ethdev net
+
 # common crypto/reduce drivers
 ifeq ($(findstring y,$(CONFIG_CRYPTO)$(CONFIG_REDUCE)),y)
 DPDK_DRIVERS += crypto/qat compress/qat common/qat
 endif
 
+# common mlx5 libs
+ifeq ($(findstring y,$(CONFIG_CRYPTO_MLX5)$(CONFIG_REDUCE_MLX5)),y)
+DPDK_DRIVERS += common/mlx5 bus/auxiliary
+endif
+
 ifeq ($(CONFIG_CRYPTO),y)
-# crypto/qat is just a stub, the compress/qat pmd is used instead
-DPDK_DRIVERS += crypto crypto/aesni_mb
+DPDK_DRIVERS += crypto crypto/ipsec_mb
+# aesni_mb is name of the PMD in DPDK 21.08 and earlier
+DPDK_DRIVERS += crypto/aesni_mb
+ifeq ($(CONFIG_CRYPTO_MLX5),y)
+DPDK_DRIVERS += crypto/mlx5
+endif
 DPDK_CFLAGS += -I$(IPSEC_MB_DIR)
 DPDK_LDFLAGS += -L$(IPSEC_MB_DIR)
 endif
 
 ifeq ($(CONFIG_REDUCE),y)
 DPDK_DRIVERS += compress compress/isal
+ifeq ($(CONFIG_REDUCE_MLX5),y)
+DPDK_DRIVERS += compress/mlx5
+endif
 DPDK_CFLAGS += -I$(ISAL_DIR)
 DPDK_LDFLAGS += -L$(ISAL_DIR)/.libs -lisal
 endif
 
+# crypto & compress deps
+DPDK_LIBS += reorder cryptodev
+DPDK_LIBS += compressdev
+DPDK_LIBS += security
+
+# vhost and deps
+DPDK_LIBS += cryptodev mbuf cmdline meter hash vhost
+
+# raid5 deps
+DPDK_LIBS += hash rcu
+
 DPDK_OPTS += -Dmachine=$(TARGET_ARCHITECTURE)
 
 ifneq ($(CONFIG_CROSS_PREFIX),)
@@ -108,7 +139,7 @@ ifeq ($(shell test $(GCC_MAJOR) -ge 10 && echo 1), 1)
 #1. gcc 10 complains on operations with zero size arrays in rte_cryptodev.c, so
 #disable this warning
 #2. gcc 10 disables fcommon by default and complains on multiple definition of
-#aesni_mb_logtype_driver symbol which is defined in header file and presented in sevral
+#aesni_mb_logtype_driver symbol which is defined in header file and presented in several
 #translation units
 DPDK_CFLAGS += -Wno-stringop-overflow -fcommon
 endif
@@ -117,9 +148,9 @@ endif
 # Force-disable scan-build
 SUB_CC = $(patsubst %ccc-analyzer,$(DEFAULT_CC),$(CC))
 
-DPDK_ALL_DRIVER_DIRS = $(shell find $(SPDK_ROOT_DIR)/dpdk/drivers -mindepth 1 -type d)
-DPDK_ALL_DRIVERS = $(DPDK_ALL_DRIVER_DIRS:$(SPDK_ROOT_DIR)/dpdk/drivers/%=%)
-DPDK_DISABLED_DRVERS = $(filter-out $(DPDK_DRIVERS),$(DPDK_ALL_DRIVERS))
+DPDK_ALL_LIB_DIRS = $(shell find $(SPDK_ROOT_DIR)/dpdk/lib -mindepth 1 -maxdepth 1 -type d)
+DPDK_ALL_LIBS = $(DPDK_ALL_LIB_DIRS:$(SPDK_ROOT_DIR)/dpdk/lib/%=%)
+DPDK_DISABLED_LIBS = $(filter-out $(DPDK_LIBS),$(DPDK_ALL_LIBS))
 
 ifneq ($(OS),FreeBSD)
 SED_INPLACE_FLAG = "-i"
@@ -132,12 +163,12 @@ endif
 # Some ninja versions come with a (broken?) jobserver which defaults to use
 # only 1 thread for the build. We workaround this by specifying -j to ninja
 # with the same value as top-makefile. This is OK as long as DPDK is not built
-# in parralel with anything else, which is the case for now.
+# in parallel with anything else, which is the case for now.
 ifeq ($(MAKE_PID),)
 MAKE_PID := $(shell echo $$PPID)
 endif
 
-MAKE_NUMJOBS := $(shell ps T | sed -nE 's/\s*$(MAKE_PID)\s.* (-j|--jobs=)( *[0-9]+).*/\1\2/p')
+MAKE_NUMJOBS := $(shell ps T | sed -nE 's/[[:space:]]*$(MAKE_PID)[[:space:]].* (-j|--jobs=)( *[0-9]+).*/\1\2/p')
 
 all: $(SPDK_ROOT_DIR)/dpdk/build-tmp
 	$(Q)# DPDK doesn't handle nested make calls, so unset MAKEFLAGS
@@ -161,15 +192,15 @@ all: $(SPDK_ROOT_DIR)/dpdk/build-tmp
 
 $(SPDK_ROOT_DIR)/dpdk/build-tmp: $(SPDK_ROOT_DIR)/mk/cc.mk $(SPDK_ROOT_DIR)/include/spdk/config.h
 	$(Q)rm -rf $(SPDK_ROOT_DIR)/dpdk/build $(SPDK_ROOT_DIR)/dpdk/build-tmp
-	$(Q)cd "$(SPDK_ROOT_DIR)/dpdk"; CC="$(SUB_CC)" meson --prefix="$(MESON_PREFIX)" --libdir lib -Dc_args="$(DPDK_CFLAGS)" -Dc_link_args="$(DPDK_LDFLAGS)" $(DPDK_OPTS) -Ddisable_drivers="$(shell echo $(DPDK_DISABLED_DRVERS) | sed -E "s/ +/,/g")" build-tmp
+	$(Q)cd "$(SPDK_ROOT_DIR)/dpdk"; CC="$(SUB_CC)" meson --prefix="$(MESON_PREFIX)" --libdir lib -Dc_args="$(DPDK_CFLAGS)" -Dc_link_args="$(DPDK_LDFLAGS)" $(DPDK_OPTS) -Denable_drivers="$(shell echo $(DPDK_DRIVERS) | sed -E "s/ +/,/g")" -Ddisable_libs="$(shell echo $(DPDK_DISABLED_LIBS) | sed -E "s/ +/,/g")" build-tmp
 	$(Q)sed $(SED_INPLACE_FLAG) 's/#define RTE_EAL_PMD_PATH .*/#define RTE_EAL_PMD_PATH ""/g' $(SPDK_ROOT_DIR)/dpdk/build-tmp/rte_build_config.h
 	$(Q) \
-	# TODO Meson build adds libbsd dependency when it's available. This means any app will be \
-	# forced to link with -lbsd, but only if it's available on the system. The clean way to \
-	# handle this would be to rely on DPDK's pkg-config file which will contain the -lbsd when \
-	# required. For now just remove the libbsd dependency. DPDK will fallback to its internal \
-	# functions.
+	# TODO Meson build adds libbsd and/or libarchive dependency when it's available. This means any app will be \
+	# forced to link with -lbsd and/or -larchive, but only if it's available on the system. The clean way to \
+	# handle this would be to rely on DPDK's pkg-config file which will contain the -lbsd/-larchive when \
+	# required. For now just remove the dependencies. DPDK will fallback to its internal functions.
 	$(Q)sed $(SED_INPLACE_FLAG) 's/#define RTE_USE_LIBBSD .*//g' $(SPDK_ROOT_DIR)/dpdk/build-tmp/rte_build_config.h
+	$(Q)sed $(SED_INPLACE_FLAG) 's/#define RTE_HAS_LIBARCHIVE .*//g' $(SPDK_ROOT_DIR)/dpdk/build-tmp/rte_build_config.h
 
 clean:
 	$(Q)rm -rf $(SPDK_ROOT_DIR)/dpdk/build $(SPDK_ROOT_DIR)/dpdk/build-tmp

examples/Makefile

@@ -34,7 +34,8 @@
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-DIRS-y += accel bdev blob ioat nvme sock vmd nvmf
+DIRS-y += accel bdev blob ioat nvme sock vmd nvmf util
+DIRS-$(CONFIG_IDXD) += idxd
 
 ifeq ($(OS),Linux)
 DIRS-$(CONFIG_VHOST) += interrupt_tgt

examples/accel/perf/Makefile

@@ -39,6 +39,6 @@ APP = accel_perf
 
 C_SRCS := accel_perf.c
 
-SPDK_LIB_LIST = $(ACCEL_MODULES_LIST) event_accel
+SPDK_LIB_LIST = $(ACCEL_MODULES_LIST) event event_accel
 
 include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

examples/accel/perf/accel_perf.c

@@ -49,7 +49,10 @@ static uint64_t g_tsc_end;
 static int g_rc;
 static int g_xfer_size_bytes = 4096;
 static int g_queue_depth = 32;
-static int g_ops_per_batch = 0;
+/* g_allocate_depth indicates how many tasks we allocate per worker. It will
+ * be at least as much as the queue depth.
+ */
+static int g_allocate_depth = 0;
 static int g_threads_per_core = 1;
 static int g_time_in_sec = 5;
 static uint32_t g_crc32c_seed = 0;
@@ -62,7 +65,6 @@ static enum accel_capability g_workload_selection;
 static struct worker_thread *g_workers = NULL;
 static int g_num_workers = 0;
 static pthread_mutex_t g_workers_lock = PTHREAD_MUTEX_INITIALIZER;
-uint64_t g_capabilites;
 
 struct worker_thread;
 static void accel_done(void *ref, int status);
@@ -78,20 +80,12 @@ struct ap_task {
 	uint32_t		iov_cnt;
 	void			*dst;
 	void			*dst2;
+	uint32_t		crc_dst;
 	struct worker_thread	*worker;
-	int			status;
 	int			expected_status; /* used for the compare operation */
 	TAILQ_ENTRY(ap_task)	link;
 };
 
-struct accel_batch {
-	int				status;
-	int				cmd_count;
-	struct spdk_accel_batch		*batch;
-	struct worker_thread		*worker;
-	TAILQ_ENTRY(accel_batch)	link;
-};
-
 struct worker_thread {
 	struct spdk_io_channel		*ch;
 	uint64_t			xfer_completed;
@@ -106,11 +100,7 @@ struct worker_thread {
 	struct spdk_poller		*is_draining_poller;
 	struct spdk_poller		*stop_poller;
 	void				*task_base;
-	struct accel_batch		*batch_base;
 	struct display_info		display;
-	TAILQ_HEAD(, accel_batch)	in_prep_batches;
-	TAILQ_HEAD(, accel_batch)	in_use_batches;
-	TAILQ_HEAD(, accel_batch)	to_submit_batches;
 };
 
 static void
@@ -120,23 +110,24 @@ dump_user_config(struct spdk_app_opts *opts)
 	printf("Core mask:      %s\n\n", opts->reactor_mask);
 	printf("Accel Perf Configuration:\n");
 	printf("Workload Type:  %s\n", g_workload_type);
-	if (g_workload_selection == ACCEL_CRC32C) {
+	if (g_workload_selection == ACCEL_CRC32C || g_workload_selection == ACCEL_COPY_CRC32C) {
 		printf("CRC-32C seed:   %u\n", g_crc32c_seed);
-		printf("vector size:    %u\n", g_crc32c_chained_count);
+		printf("vector count    %u\n", g_crc32c_chained_count);
 	} else if (g_workload_selection == ACCEL_FILL) {
 		printf("Fill pattern:   0x%x\n", g_fill_pattern);
 	} else if ((g_workload_selection == ACCEL_COMPARE) && g_fail_percent_goal > 0) {
 		printf("Failure inject: %u percent\n", g_fail_percent_goal);
 	}
-	printf("Transfer size:  %u bytes\n", g_xfer_size_bytes);
+	if (g_workload_selection == ACCEL_COPY_CRC32C) {
+		printf("Vector size:    %u bytes\n", g_xfer_size_bytes);
+		printf("Transfer size:  %u bytes\n", g_xfer_size_bytes * g_crc32c_chained_count);
+	} else {
+		printf("Transfer size:  %u bytes\n", g_xfer_size_bytes);
+	}
 	printf("Queue depth:    %u\n", g_queue_depth);
+	printf("Allocate depth: %u\n", g_allocate_depth);
 	printf("# threads/core: %u\n", g_threads_per_core);
 	printf("Run time:       %u seconds\n", g_time_in_sec);
-	if (g_ops_per_batch > 0) {
-		printf("Batching:       %u operations\n", g_ops_per_batch);
-	} else {
-		printf("Batching:       Disabled\n");
-	}
 	printf("Verify:         %s\n\n", g_verify ? "Yes" : "No");
 }
 
@@ -146,49 +137,73 @@ usage(void)
 	printf("accel_perf options:\n");
 	printf("\t[-h help message]\n");
 	printf("\t[-q queue depth per core]\n");
-	printf("\t[-C for crc32c workload, use this value to configre the io vector size to test (default 1)\n");
+	printf("\t[-C for crc32c workload, use this value to configure the io vector size to test (default 1)\n");
 	printf("\t[-T number of threads per core\n");
 	printf("\t[-n number of channels]\n");
 	printf("\t[-o transfer size in bytes]\n");
 	printf("\t[-t time in seconds]\n");
-	printf("\t[-w workload type must be one of these: copy, fill, crc32c, compare, dualcast\n");
+	printf("\t[-w workload type must be one of these: copy, fill, crc32c, copy_crc32c, compare, dualcast\n");
 	printf("\t[-s for crc32c workload, use this seed value (default 0)\n");
 	printf("\t[-P for compare workload, percentage of operations that should miscompare (percent, default 0)\n");
 	printf("\t[-f for fill workload, use this BYTE value (default 255)\n");
 	printf("\t[-y verify result if this switch is on]\n");
-	printf("\t[-b batch this number of operations at a time (default 0 = disabled)]\n");
+	printf("\t[-a tasks to allocate per core (default: same value as -q)]\n");
+	printf("\t\tCan be used to spread operations across a wider range of memory.\n");
 }
 
 static int
 parse_args(int argc, char *argv)
 {
+	int argval = 0;
+
 	switch (argc) {
-	case 'b':
-		g_ops_per_batch = spdk_strtol(optarg, 10);
+	case 'a':
+	case 'C':
+	case 'f':
+	case 'T':
+	case 'o':
+	case 'P':
+	case 'q':
+	case 's':
+	case 't':
+		argval = spdk_strtol(optarg, 10);
+		if (argval < 0) {
+			fprintf(stderr, "-%c option must be non-negative.\n", argc);
+			usage();
+			return 1;
+		}
+		break;
+	default:
+		break;
+	};
+
+	switch (argc) {
+	case 'a':
+		g_allocate_depth = argval;
 		break;
 	case 'C':
-		g_crc32c_chained_count = spdk_strtol(optarg, 10);
+		g_crc32c_chained_count = argval;
 		break;
 	case 'f':
-		g_fill_pattern = (uint8_t)spdk_strtol(optarg, 10);
+		g_fill_pattern = (uint8_t)argval;
 		break;
 	case 'T':
-		g_threads_per_core = spdk_strtol(optarg, 10);
+		g_threads_per_core = argval;
 		break;
 	case 'o':
-		g_xfer_size_bytes = spdk_strtol(optarg, 10);
+		g_xfer_size_bytes = argval;
 		break;
 	case 'P':
-		g_fail_percent_goal = spdk_strtol(optarg, 10);
+		g_fail_percent_goal = argval;
 		break;
 	case 'q':
-		g_queue_depth = spdk_strtol(optarg, 10);
+		g_queue_depth = argval;
 		break;
 	case 's':
-		g_crc32c_seed = spdk_strtol(optarg, 10);
+		g_crc32c_seed = argval;
 		break;
 	case 't':
-		g_time_in_sec = spdk_strtol(optarg, 10);
+		g_time_in_sec = argval;
 		break;
 	case 'y':
 		g_verify = true;
@@ -201,6 +216,8 @@ parse_args(int argc, char *argv)
 			g_workload_selection = ACCEL_FILL;
 		} else if (!strcmp(g_workload_type, "crc32c")) {
 			g_workload_selection = ACCEL_CRC32C;
+		} else if (!strcmp(g_workload_type, "copy_crc32c")) {
+			g_workload_selection = ACCEL_COPY_CRC32C;
 		} else if (!strcmp(g_workload_type, "compare")) {
 			g_workload_selection = ACCEL_COMPARE;
 		} else if (!strcmp(g_workload_type, "dualcast")) {
@@ -222,7 +239,6 @@ unregister_worker(void *arg1)
 	struct worker_thread *worker = arg1;
 
 	free(worker->task_base);
-	free(worker->batch_base);
 	spdk_put_io_channel(worker->ch);
 	pthread_mutex_lock(&g_workers_lock);
 	assert(g_num_workers >= 1);
@@ -239,6 +255,7 @@ _get_task_data_bufs(struct ap_task *task)
 {
 	uint32_t align = 0;
 	uint32_t i = 0;
+	int dst_buff_len = g_xfer_size_bytes;
 
 	/* For dualcast, the DSA HW requires 4K alignment on destination addresses but
 	 * we do this for all engines to keep it simple.
@@ -247,7 +264,7 @@ _get_task_data_bufs(struct ap_task *task)
 		align = ALIGN_4K;
 	}
 
-	if (g_workload_selection == ACCEL_CRC32C) {
+	if (g_workload_selection == ACCEL_CRC32C || g_workload_selection == ACCEL_COPY_CRC32C) {
 		assert(g_crc32c_chained_count > 0);
 		task->iov_cnt = g_crc32c_chained_count;
 		task->iovs = calloc(task->iov_cnt, sizeof(struct iovec));
@@ -256,6 +273,10 @@ _get_task_data_bufs(struct ap_task *task)
 			return -ENOMEM;
 		}
 
+		if (g_workload_selection == ACCEL_COPY_CRC32C) {
+			dst_buff_len = g_xfer_size_bytes * g_crc32c_chained_count;
+		}
+
 		for (i = 0; i < task->iov_cnt; i++) {
 			task->iovs[i].iov_base = spdk_dma_zmalloc(g_xfer_size_bytes, 0, NULL);
 			if (task->iovs[i].iov_base == NULL) {
@@ -280,17 +301,19 @@ _get_task_data_bufs(struct ap_task *task)
 		}
 	}
 
-	task->dst = spdk_dma_zmalloc(g_xfer_size_bytes, align, NULL);
-	if (task->dst == NULL) {
-		fprintf(stderr, "Unable to alloc dst buffer\n");
-		return -ENOMEM;
-	}
+	if (g_workload_selection != ACCEL_CRC32C) {
+		task->dst = spdk_dma_zmalloc(dst_buff_len, align, NULL);
+		if (task->dst == NULL) {
+			fprintf(stderr, "Unable to alloc dst buffer\n");
+			return -ENOMEM;
+		}
 
-	/* For compare we want the buffers to match, otherwise not. */
-	if (g_workload_selection == ACCEL_COMPARE) {
-		memset(task->dst, DATA_PATTERN, g_xfer_size_bytes);
-	} else {
-		memset(task->dst, ~DATA_PATTERN, g_xfer_size_bytes);
+		/* For compare we want the buffers to match, otherwise not. */
+		if (g_workload_selection == ACCEL_COMPARE) {
+			memset(task->dst, DATA_PATTERN, dst_buff_len);
+		} else {
+			memset(task->dst, ~DATA_PATTERN, dst_buff_len);
+		}
 	}
 
 	if (g_workload_selection == ACCEL_DUALCAST) {
@@ -318,8 +341,6 @@ _get_task(struct worker_thread *worker)
 		return NULL;
 	}
 
-	task->worker = worker;
-	task->worker->current_queue_depth++;
 	return task;
 }
 
@@ -329,24 +350,29 @@ _submit_single(struct worker_thread *worker, struct ap_task *task)
 {
 	int random_num;
 	int rc = 0;
+	int flags = 0;
 
 	assert(worker);
 
 	switch (g_workload_selection) {
 	case ACCEL_COPY:
 		rc = spdk_accel_submit_copy(worker->ch, task->dst, task->src,
-					    g_xfer_size_bytes, accel_done, task);
+					    g_xfer_size_bytes, flags, accel_done, task);
 		break;
 	case ACCEL_FILL:
 		/* For fill use the first byte of the task->dst buffer */
 		rc = spdk_accel_submit_fill(worker->ch, task->dst, *(uint8_t *)task->src,
-					    g_xfer_size_bytes, accel_done, task);
+					    g_xfer_size_bytes, flags, accel_done, task);
 		break;
 	case ACCEL_CRC32C:
-		rc = spdk_accel_submit_crc32cv(worker->ch, (uint32_t *)task->dst,
+		rc = spdk_accel_submit_crc32cv(worker->ch, &task->crc_dst,
 					       task->iovs, task->iov_cnt, g_crc32c_seed,
 					       accel_done, task);
 		break;
+	case ACCEL_COPY_CRC32C:
+		rc = spdk_accel_submit_copy_crc32cv(worker->ch, task->dst, task->iovs, task->iov_cnt,
+						    &task->crc_dst, g_crc32c_seed, flags, accel_done, task);
+		break;
 	case ACCEL_COMPARE:
 		random_num = rand() % 100;
 		if (random_num < g_fail_percent_goal) {
@@ -361,7 +387,7 @@ _submit_single(struct worker_thread *worker, struct ap_task *task)
 		break;
 	case ACCEL_DUALCAST:
 		rc = spdk_accel_submit_dualcast(worker->ch, task->dst, task->dst2,
-						task->src, g_xfer_size_bytes, accel_done, task);
+						task->src, g_xfer_size_bytes, flags, accel_done, task);
 		break;
 	default:
 		assert(false);
@@ -374,52 +400,12 @@ _submit_single(struct worker_thread *worker, struct ap_task *task)
 	}
 }
 
-static int
-_batch_prep_cmd(struct worker_thread *worker, struct ap_task *task,
-		struct accel_batch *worker_batch)
-{
-	struct spdk_accel_batch *batch = worker_batch->batch;
-	int rc = 0;
-
-	worker_batch->cmd_count++;
-	assert(worker_batch->cmd_count <= g_ops_per_batch);
-
-	switch (g_workload_selection) {
-	case ACCEL_COPY:
-		rc = spdk_accel_batch_prep_copy(worker->ch, batch, task->dst,
-						task->src, g_xfer_size_bytes, accel_done, task);
-		break;
-	case ACCEL_DUALCAST:
-		rc = spdk_accel_batch_prep_dualcast(worker->ch, batch, task->dst, task->dst2,
-						    task->src, g_xfer_size_bytes, accel_done, task);
-		break;
-	case ACCEL_COMPARE:
-		rc = spdk_accel_batch_prep_compare(worker->ch, batch, task->dst, task->src,
-						   g_xfer_size_bytes, accel_done, task);
-		break;
-	case ACCEL_FILL:
-		rc = spdk_accel_batch_prep_fill(worker->ch, batch, task->dst,
-						*(uint8_t *)task->src,
-						g_xfer_size_bytes, accel_done, task);
-		break;
-	case ACCEL_CRC32C:
-		rc = spdk_accel_batch_prep_crc32cv(worker->ch, batch, (uint32_t *)task->dst,
-						   task->iovs, task->iov_cnt, g_crc32c_seed, accel_done, task);
-		break;
-	default:
-		assert(false);
-		break;
-	}
-
-	return rc;
-}
-
 static void
 _free_task_buffers(struct ap_task *task)
 {
 	uint32_t i;
 
-	if (g_workload_selection == ACCEL_CRC32C) {
+	if (g_workload_selection == ACCEL_CRC32C || g_workload_selection == ACCEL_COPY_CRC32C) {
 		if (task->iovs) {
 			for (i = 0; i < task->iov_cnt; i++) {
 				if (task->iovs[i].iov_base) {
@@ -438,143 +424,30 @@ _free_task_buffers(struct ap_task *task)
 	}
 }
 
-static void _batch_done(void *cb_arg);
-static void
-_build_batch(struct worker_thread *worker, struct ap_task *task)
+static int
+_vector_memcmp(void *_dst, struct iovec *src_iovs, uint32_t iovcnt)
 {
-	struct accel_batch *worker_batch = NULL;
-	int rc;
-
-	assert(!TAILQ_EMPTY(&worker->in_prep_batches));
-
-	worker_batch = TAILQ_FIRST(&worker->in_prep_batches);
-
-	/* If an accel batch hasn't been created yet do so now. */
-	if (worker_batch->batch == NULL) {
-		worker_batch->batch = spdk_accel_batch_create(worker->ch);
-		if (worker_batch->batch == NULL) {
-			fprintf(stderr, "error unable to create new batch\n");
-			return;
-		}
-	}
-
-	/* Prep the command re-using the last completed command's task */
-	rc = _batch_prep_cmd(worker, task, worker_batch);
-	if (rc) {
-		fprintf(stderr, "error preping command for batch\n");
-		goto error;
-	}
-
-	/* If this batch is full move it to the to_submit list so it gets
-	 * submitted as batches complete.
-	 */
-	if (worker_batch->cmd_count == g_ops_per_batch) {
-		TAILQ_REMOVE(&worker->in_prep_batches, worker_batch, link);
-		TAILQ_INSERT_TAIL(&worker->to_submit_batches, worker_batch, link);
-	}
-
-	return;
-error:
-	spdk_accel_batch_cancel(worker->ch, worker_batch->batch);
-
-}
-
-static void batch_done(void *cb_arg, int status);
-static void
-_drain_batch(struct worker_thread *worker)
-{
-	struct accel_batch *worker_batch, *tmp;
-	int rc;
-
-	/* submit any batches that were being built up. */
-	TAILQ_FOREACH_SAFE(worker_batch, &worker->in_prep_batches, link, tmp) {
-		if (worker_batch->cmd_count == 0) {
-			continue;
-		}
-		worker->current_queue_depth += worker_batch->cmd_count + 1;
-
-		TAILQ_REMOVE(&worker->in_prep_batches, worker_batch, link);
-		TAILQ_INSERT_TAIL(&worker->in_use_batches, worker_batch, link);
-		rc = spdk_accel_batch_submit(worker->ch, worker_batch->batch, batch_done, worker_batch);
-		if (rc == 0) {
-			worker_batch->cmd_count = 0;
-		} else {
-			fprintf(stderr, "error sending final batch\n");
-			worker->current_queue_depth -= worker_batch->cmd_count + 1;
-			break;
-		}
-	}
-}
-
-static void
-_batch_done(void *cb_arg)
-{
-	struct accel_batch *worker_batch = (struct accel_batch *)cb_arg;
-	struct worker_thread *worker = worker_batch->worker;
-	int rc;
-
-	assert(TAILQ_EMPTY(&worker->in_use_batches) == 0);
-
-	if (worker_batch->status) {
-		SPDK_ERRLOG("error %d\n", worker_batch->status);
-	}
-
-	worker->current_queue_depth--;
-	TAILQ_REMOVE(&worker->in_use_batches, worker_batch, link);
-	TAILQ_INSERT_TAIL(&worker->in_prep_batches, worker_batch, link);
-	worker_batch->batch = NULL;
-	worker_batch->cmd_count = 0;
-
-	if (!worker->is_draining) {
-		worker_batch = TAILQ_FIRST(&worker->to_submit_batches);
-		if (worker_batch != NULL) {
-
-			assert(worker_batch->cmd_count == g_ops_per_batch);
-
-			/* Add one for the batch command itself. */
-			worker->current_queue_depth += g_ops_per_batch + 1;
-			TAILQ_REMOVE(&worker->to_submit_batches, worker_batch, link);
-			TAILQ_INSERT_TAIL(&worker->in_use_batches, worker_batch, link);
-
-			rc = spdk_accel_batch_submit(worker->ch, worker_batch->batch, batch_done, worker_batch);
-			if (rc) {
-				fprintf(stderr, "error ending batch\n");
-				worker->current_queue_depth -= g_ops_per_batch + 1;
-				return;
-			}
-		}
-	} else {
-		_drain_batch(worker);
-	}
-}
-
-static void
-batch_done(void *cb_arg, int status)
-{
-	struct accel_batch *worker_batch = (struct accel_batch *)cb_arg;
-
-	assert(worker_batch->worker);
-
-	worker_batch->status = status;
-	spdk_thread_send_msg(worker_batch->worker->thread, _batch_done, worker_batch);
-}
-
-static uint32_t
-_update_crc32c_iov(struct iovec *iov, int iovcnt, uint32_t crc32c)
-{
-	int i;
+	uint32_t i;
+	uint32_t ttl_len = 0;
+	uint8_t *dst = (uint8_t *)_dst;
 
 	for (i = 0; i < iovcnt; i++) {
-		assert(iov[i].iov_base != NULL);
-		assert(iov[i].iov_len != 0);
-		crc32c = spdk_crc32c_update(iov[i].iov_base, iov[i].iov_len, crc32c);
-
+		if (memcmp(dst, src_iovs[i].iov_base, src_iovs[i].iov_len)) {
+			return -1;
+		}
+		dst += src_iovs[i].iov_len;
+		ttl_len += src_iovs[i].iov_len;
 	}
-	return crc32c;
+
+	if (ttl_len != iovcnt * g_xfer_size_bytes) {
+		return -1;
+	}
+
+	return 0;
 }
 
 static void
-_accel_done(void *arg1)
+accel_done(void *arg1, int status)
 {
 	struct ap_task *task = arg1;
 	struct worker_thread *worker = task->worker;
@@ -583,11 +456,22 @@ _accel_done(void *arg1)
 	assert(worker);
 	assert(worker->current_queue_depth > 0);
 
-	if (g_verify && task->status == 0) {
+	if (g_verify && status == 0) {
 		switch (g_workload_selection) {
+		case ACCEL_COPY_CRC32C:
+			sw_crc32c = spdk_crc32c_iov_update(task->iovs, task->iov_cnt, ~g_crc32c_seed);
+			if (task->crc_dst != sw_crc32c) {
+				SPDK_NOTICELOG("CRC-32C miscompare\n");
+				worker->xfer_failed++;
+			}
+			if (_vector_memcmp(task->dst, task->iovs, task->iov_cnt)) {
+				SPDK_NOTICELOG("Data miscompare\n");
+				worker->xfer_failed++;
+			}
+			break;
 		case ACCEL_CRC32C:
-			sw_crc32c = _update_crc32c_iov(task->iovs, task->iov_cnt, ~g_crc32c_seed);
-			if (*(uint32_t *)task->dst != sw_crc32c) {
+			sw_crc32c = spdk_crc32c_iov_update(task->iovs, task->iov_cnt, ~g_crc32c_seed);
+			if (task->crc_dst != sw_crc32c) {
 				SPDK_NOTICELOG("CRC-32C miscompare\n");
 				worker->xfer_failed++;
 			}
@@ -623,9 +507,9 @@ _accel_done(void *arg1)
 	}
 
 	if (task->expected_status == -EILSEQ) {
-		assert(task->status != 0);
+		assert(status != 0);
 		worker->injected_miscompares++;
-	} else if (task->status) {
+	} else if (status) {
 		/* Expected to pass but the accel engine reported an error (ex: COMPARE operation). */
 		worker->xfer_failed++;
 	}
@@ -634,14 +518,10 @@ _accel_done(void *arg1)
 	worker->current_queue_depth--;
 
 	if (!worker->is_draining) {
-		if (g_ops_per_batch == 0) {
-			_submit_single(worker, task);
-			worker->current_queue_depth++;
-		} else {
-			_build_batch(worker, task);
-		}
-	} else if (g_ops_per_batch > 0) {
-		_drain_batch(worker);
+		TAILQ_INSERT_TAIL(&worker->tasks_pool, task, link);
+		task = _get_task(worker);
+		_submit_single(worker, task);
+		worker->current_queue_depth++;
 	} else {
 		TAILQ_INSERT_TAIL(&worker->tasks_pool, task, link);
 	}
@@ -713,7 +593,7 @@ _check_draining(void *arg)
 		unregister_worker(worker);
 	}
 
-	return -1;
+	return SPDK_POLLER_BUSY;
 }
 
 static int
@@ -729,7 +609,20 @@ _worker_stop(void *arg)
 	worker->is_draining = true;
 	worker->is_draining_poller = SPDK_POLLER_REGISTER(_check_draining, worker, 0);
 
-	return 0;
+	return SPDK_POLLER_BUSY;
+}
+
+static inline void
+identify_accel_engine_usage(struct spdk_io_channel *ch)
+{
+	uint64_t capabilities;
+
+	assert(ch != NULL);
+	capabilities = spdk_accel_get_capabilities(ch);
+	if ((capabilities & g_workload_selection) != g_workload_selection) {
+		SPDK_WARNLOG("The selected workload is not natively supported by the current engine\n");
+		SPDK_WARNLOG("The software engine will be used instead.\n\n");
+	}
 }
 
 static void
@@ -737,12 +630,7 @@ _init_thread(void *arg1)
 {
 	struct worker_thread *worker;
 	struct ap_task *task;
-	int i, rc, num_batches;
-	int max_per_batch;
-	int remaining = g_queue_depth;
-	int num_tasks = g_queue_depth;
-	struct accel_batch *tmp;
-	struct accel_batch *worker_batch = NULL;
+	int i, num_tasks = g_allocate_depth;
 	struct display_info *display = arg1;
 
 	worker = calloc(1, sizeof(*worker));
@@ -758,57 +646,19 @@ _init_thread(void *arg1)
 	worker->core = spdk_env_get_current_core();
 	worker->thread = spdk_get_thread();
 	pthread_mutex_lock(&g_workers_lock);
+	i = g_num_workers;
 	g_num_workers++;
 	worker->next = g_workers;
 	g_workers = worker;
 	pthread_mutex_unlock(&g_workers_lock);
 	worker->ch = spdk_accel_engine_get_io_channel();
 
-	TAILQ_INIT(&worker->tasks_pool);
-
-	if (g_ops_per_batch > 0) {
-
-		max_per_batch = spdk_accel_batch_get_max(worker->ch);
-		assert(max_per_batch > 0);
-
-		if (g_ops_per_batch > max_per_batch) {
-			fprintf(stderr, "Reducing requested batch amount to max supported of %d\n", max_per_batch);
-			g_ops_per_batch = max_per_batch;
-		}
-
-		if (g_ops_per_batch > g_queue_depth) {
-			fprintf(stderr, "Batch amount > queue depth, resetting to %d\n", g_queue_depth);
-			g_ops_per_batch = g_queue_depth;
-		}
-
-		TAILQ_INIT(&worker->in_prep_batches);
-		TAILQ_INIT(&worker->to_submit_batches);
-		TAILQ_INIT(&worker->in_use_batches);
-
-		/* A worker_batch will live on one of 3 lists:
-		 * IN_PREP: as individual IOs complete new ones are built on on a
-		 *          worker_batch on this list until it reaches g_ops_per_batch.
-		 * TO_SUBMIT: as batches are built up on IO completion they are moved
-		 *	      to this list once they are full.  This list is used in
-		 *	      batch completion to start new batches.
-		 * IN_USE: the worker_batch is outstanding and will be moved to in prep
-		 *         list when the batch is completed.
-		 *
-		 * So we need enough to cover Q depth loading and then one to replace
-		 * each one of those and for when everything is outstanding there needs
-		 * to be one extra batch to build up while the last batch is completing
-		 * IO but before it's completed the batch command.
-		 */
-		num_batches = (g_queue_depth / g_ops_per_batch * 2) + 1;
-		worker->batch_base = calloc(num_batches, sizeof(struct accel_batch));
-		worker_batch = worker->batch_base;
-		for (i = 0; i < num_batches; i++) {
-			worker_batch->worker = worker;
-			TAILQ_INSERT_TAIL(&worker->in_prep_batches, worker_batch, link);
-			worker_batch++;
-		}
+	if (i == 0) {
+		identify_accel_engine_usage(worker->ch);
 	}
 
+	TAILQ_INIT(&worker->tasks_pool);
+
 	worker->task_base = calloc(num_tasks, sizeof(struct ap_task));
 	if (worker->task_base == NULL) {
 		fprintf(stderr, "Could not allocate task base.\n");
@@ -818,6 +668,7 @@ _init_thread(void *arg1)
 	task = worker->task_base;
 	for (i = 0; i < num_tasks; i++) {
 		TAILQ_INSERT_TAIL(&worker->tasks_pool, task, link);
+		task->worker = worker;
 		if (_get_task_data_bufs(task)) {
 			fprintf(stderr, "Unable to get data bufs\n");
 			goto error;
@@ -829,54 +680,10 @@ _init_thread(void *arg1)
 	worker->stop_poller = SPDK_POLLER_REGISTER(_worker_stop, worker,
 			      g_time_in_sec * 1000000ULL);
 
-	/* If batching is enabled load up to the full Q depth before
-	 * processing any completions, then ping pong between two batches,
-	 * one processing and one being built up for when the other completes.
-	 */
-	if (g_ops_per_batch > 0) {
-		do {
-			worker_batch = TAILQ_FIRST(&worker->in_prep_batches);
-			if (worker_batch == NULL) {
-				goto error;
-			}
-
-			worker_batch->batch = spdk_accel_batch_create(worker->ch);
-			if (worker_batch->batch == NULL) {
-				raise(SIGINT);
-				break;
-			}
-
-			for (i = 0; i < g_ops_per_batch; i++) {
-				task = _get_task(worker);
-				if (task == NULL) {
-					goto error;
-				}
-
-				rc = _batch_prep_cmd(worker, task, worker_batch);
-				if (rc) {
-					fprintf(stderr, "error preping command\n");
-					goto error;
-				}
-			}
-
-			/* for the batch operation itself. */
-			task->worker->current_queue_depth++;
-			TAILQ_REMOVE(&worker->in_prep_batches, worker_batch, link);
-			TAILQ_INSERT_TAIL(&worker->in_use_batches, worker_batch, link);
-
-			rc = spdk_accel_batch_submit(worker->ch, worker_batch->batch, batch_done, worker_batch);
-			if (rc) {
-				fprintf(stderr, "error ending batch\n");
-				goto error;
-			}
-			assert(remaining >= g_ops_per_batch);
-			remaining -= g_ops_per_batch;
-		} while (remaining > 0);
-	}
-
-	/* Submit as singles when no batching is enabled or we ran out of batches. */
-	for (i = 0; i < remaining; i++) {
+	/* Load up queue depth worth of operations. */
+	for (i = 0; i < g_queue_depth; i++) {
 		task = _get_task(worker);
+		worker->current_queue_depth++;
 		if (task == NULL) {
 			goto error;
 		}
@@ -885,36 +692,16 @@ _init_thread(void *arg1)
 	}
 	return;
 error:
-	if (worker_batch && worker_batch->batch) {
-		TAILQ_FOREACH_SAFE(worker_batch, &worker->in_use_batches, link, tmp) {
-			spdk_accel_batch_cancel(worker->ch, worker_batch->batch);
-			TAILQ_REMOVE(&worker->in_use_batches, worker_batch, link);
-		}
-	}
 
 	_free_task_buffers_in_pool(worker);
-	free(worker->batch_base);
 	free(worker->task_base);
 	free(worker);
 	spdk_app_stop(-1);
 }
 
-static void
-accel_done(void *cb_arg, int status)
-{
-	struct ap_task *task = (struct ap_task *)cb_arg;
-	struct worker_thread *worker = task->worker;
-
-	assert(worker);
-
-	task->status = status;
-	spdk_thread_send_msg(worker->thread, _accel_done, task);
-}
-
 static void
 accel_perf_start(void *arg1)
 {
-	struct spdk_io_channel *accel_ch;
 	struct spdk_cpuset tmp_cpumask = {};
 	char thread_name[32];
 	uint32_t i;
@@ -922,15 +709,6 @@ accel_perf_start(void *arg1)
 	struct spdk_thread *thread;
 	struct display_info *display;
 
-	accel_ch = spdk_accel_engine_get_io_channel();
-	g_capabilites = spdk_accel_get_capabilities(accel_ch);
-	spdk_put_io_channel(accel_ch);
-
-	if ((g_capabilites & g_workload_selection) != g_workload_selection) {
-		SPDK_WARNLOG("The selected workload is not natively supported by the current engine\n");
-		SPDK_WARNLOG("The software engine will be used instead.\n\n");
-	}
-
 	g_tsc_rate = spdk_get_ticks_hz();
 	g_tsc_end = spdk_get_ticks() + g_time_in_sec * g_tsc_rate;
 
@@ -966,7 +744,7 @@ main(int argc, char **argv)
 	pthread_mutex_init(&g_workers_lock, NULL);
 	spdk_app_opts_init(&opts, sizeof(opts));
 	opts.reactor_mask = "0x1";
-	if (spdk_app_parse_args(argc, argv, &opts, "C:o:q:t:yw:P:f:b:T:", NULL, parse_args,
+	if (spdk_app_parse_args(argc, argv, &opts, "a:C:o:q:t:yw:P:f:T:", NULL, parse_args,
 				usage) != SPDK_APP_PARSE_ARGS_SUCCESS) {
 		g_rc = -1;
 		goto cleanup;
@@ -975,6 +753,7 @@ main(int argc, char **argv)
 	if ((g_workload_selection != ACCEL_COPY) &&
 	    (g_workload_selection != ACCEL_FILL) &&
 	    (g_workload_selection != ACCEL_CRC32C) &&
+	    (g_workload_selection != ACCEL_COPY_CRC32C) &&
 	    (g_workload_selection != ACCEL_COMPARE) &&
 	    (g_workload_selection != ACCEL_DUALCAST)) {
 		usage();
@@ -982,14 +761,18 @@ main(int argc, char **argv)
 		goto cleanup;
 	}
 
-	if (g_ops_per_batch > 0 && (g_queue_depth % g_ops_per_batch > 0)) {
-		fprintf(stdout, "batch size must be a multiple of queue depth\n");
+	if (g_allocate_depth > 0 && g_queue_depth > g_allocate_depth) {
+		fprintf(stdout, "allocate depth must be at least as big as queue depth\n");
 		usage();
 		g_rc = -1;
 		goto cleanup;
 	}
 
-	if (g_workload_selection == ACCEL_CRC32C &&
+	if (g_allocate_depth == 0) {
+		g_allocate_depth = g_queue_depth;
+	}
+
+	if ((g_workload_selection == ACCEL_CRC32C || g_workload_selection == ACCEL_COPY_CRC32C) &&
 	    g_crc32c_chained_count == 0) {
 		usage();
 		g_rc = -1;

examples/bdev/fio_plugin/Makefile

@@ -40,6 +40,6 @@ FIO_PLUGIN := spdk_bdev
 
 C_SRCS = fio_plugin.c
 
-SPDK_LIB_LIST = $(ALL_MODULES_LIST) event_bdev
+SPDK_LIB_LIST = $(ALL_MODULES_LIST) event event_bdev
 
 include $(SPDK_ROOT_DIR)/mk/spdk.fio.mk

examples/bdev/fio_plugin/README.md

@@ -1,64 +1,83 @@
 # Introduction
 
-This directory contains a plug-in module for fio to enable use
-with SPDK. Fio is free software published under version 2 of
-the GPL license.
+This directory contains a plug-in module for fio to enable use with SPDK. Fio is free software
+published under version 2 of the GPL license.
 
-# Compiling fio
+## Compiling fio
 
-Clone the fio source repository from https://github.com/axboe/fio
+Clone the [fio source repository](https://github.com/axboe/fio)
 
+```bash
     git clone https://github.com/axboe/fio
     cd fio
+```
 
 Compile the fio code and install:
 
+```bash
     make
     make install
+```
 
-# Compiling SPDK
+## Compiling SPDK
 
-Clone the SPDK source repository from https://github.com/spdk/spdk
+Clone the [SPDK source repository](https://github.com/spdk/spdk)
 
+```bash
     git clone https://github.com/spdk/spdk
     cd spdk
     git submodule update --init
+```
 
 Then, run the SPDK configure script to enable fio (point it to the root of the fio repository):
 
+```bash
     cd spdk
     ./configure --with-fio=/path/to/fio/repo <other configuration options>
+```
 
 Finally, build SPDK:
 
+```bash
     make
+```
 
 **Note to advanced users**: These steps assume you're using the DPDK submodule. If you are using your
 own version of DPDK, the fio plugin requires that DPDK be compiled with -fPIC. You can compile DPDK
 with -fPIC by modifying your DPDK configuration file and adding the line:
 
-    EXTRA_CFLAGS=-fPIC
+```bash
+EXTRA_CFLAGS=-fPIC
+```
 
-# Usage
+## Usage
 
 To use the SPDK fio plugin with fio, specify the plugin binary using LD_PRELOAD when running
 fio and set ioengine=spdk_bdev in the fio configuration file (see example_config.fio in the same
-directory as this README).
+directory as this README). Following example command assumes `fio` is in your system `$PATH` environment variable.
 
-    LD_PRELOAD=<path to spdk repo>/build/fio/spdk_bdev fio
+```bash
+LD_PRELOAD=<path to spdk repo>/build/fio/spdk_bdev fio
+```
 
-The fio configuration file must contain one new parameter:
+The fio configuration file must contain parameter pointing to a JSON configuration file containing SPDK bdev configuration:
 
-    spdk_json_conf=./examples/bdev/fio_plugin/bdev.json
+```bash
+spdk_json_conf=./examples/bdev/fio_plugin/bdev.json
+```
 
 You can specify which block device to run against by setting the filename parameter
 to the block device name:
 
-    filename=Malloc0
+```bash
+filename=Malloc0
+```
 
 Or for NVMe devices:
 
-    filename=Nvme0n1
+```bash
+filename=Nvme0n1
+```
 
 fio by default forks a separate process for every job. It also supports just spawning a separate
 thread in the same process for every job. The SPDK fio plugin is limited to this latter thread
@@ -71,3 +90,206 @@ engine's full path via the ioengine parameter - LD_PRELOAD is recommended to avo
 When testing random workloads, it is recommended to set norandommap=1.  fio's random map
 processing consumes extra CPU cycles which will degrade performance over time with
 the fio_plugin since all I/O are submitted and completed on a single CPU core.
+
+### Step-by-step usage examples
+
+These examples assume you have built fio and SPDK with `--with-fio` option enabled.
+
+#### Using fio bdev plugin with local NVMe storage
+
+- Bind local NVMe drives to userspace driver
+
+- Run gen_nvme.sh script to create a JSON file with bdev subsystem configuration
+
+    ```bash
+    scripts/gen_nvme.sh --json-with-subsystems > /tmp/bdev.json
+
+    cat /tmp/bdev_local.json  | jq
+    {
+        "subsystems": [
+        {
+            "subsystem": "bdev",
+            "config": [
+            {
+                "method": "bdev_nvme_attach_controller",
+                "params": {
+                "trtype": "PCIe",
+                "name": "Nvme0",
+                "traddr": "0000:0a:00.0"
+                }
+            },
+            {
+                "method": "bdev_nvme_attach_controller",
+                "params": {
+                "trtype": "PCIe",
+                "name": "Nvme1",
+                "traddr": "0000:85:00.0"
+                }
+            }
+            ]
+        }
+        ]
+    }
+    ```
+
+- Prepare fio configuration file
+
+    ```bash
+    cat /tmp/fio.conf
+
+    [global]
+    ioengine=/spdk/build/fio/spdk_bdev
+    spdk_json_conf=/tmp/bdev.json
+
+    thread=1
+    direct=1
+    group_reporting=1
+
+    bs=4k
+    rw=randread
+    rwmixread=70
+    time_based=1
+    runtime=10
+    norandommap=1
+
+    [filename0]
+    filename=Nvme0n1
+    filename=Nvme1n1
+    iodepth=8
+    ```
+
+- Run fio with spdk bdev plugin
+
+    ```bash
+    /usr/src/fio/fio /tmp/fio.conf
+    ```
+
+#### Using fio bdev plugin as NVMe-oF initiator with remote storage
+
+- Start SPDK NVMe-oF Target process and configure it with block devices and NVMe-oF subsystems
+
+    ```bash
+    build/bin/nvmf_tgt &
+    sleep 3
+    scripts/rpc.py bdev_malloc_create 10 512 -b Malloc0
+    scripts/rpc.py nvmf_create_transport -t TCP
+    scripts/rpc.py nvmf_create_subsystem nqn.2018-09.io.spdk:cnode1 -a -s S000001
+    scripts/rpc.py nvmf_subsystem_add_listener nqn.2018-09.io.spdk:cnode1 -t tcp -f ipv4 -s 4420 -a 10.0.0.1
+    scripts/rpc.py nvmf_subsystem_add_ns nqn.2018-09.io.spdk:cnode1 Malloc0
+    ```
+
+- Run gen_nvme.sh script to prepare a JSON file containing bdev subsystem configuration
+  for initiator which will allow it to connect to target
+
+    ```bash
+    scripts/gen_nvme.sh --json-with-subsystems --mode=remote \
+    --trid=tcp:10.0.0.1:4420:nqn.2018-09.io.spdk:cnode1 > /tmp/bdev.json
+
+    cat /tmp/bdev.json | jq
+    {
+        "subsystems": [
+        {
+            "subsystem": "bdev",
+            "config": [
+            {
+                "method": "bdev_nvme_attach_controller",
+                "params": {
+                "trtype": "tcp",
+                "adrfam": "IPv4",
+                "name": "Nvme0",
+                "subnqn": "nqn.2018-09.io.spdk:cnode1",
+                "traddr": "10.0.0.1",
+                "trsvcid": "4420"
+                }
+            }
+            ]
+        }
+        ]
+    }
+    ```
+
+- Prepare fio configuration file
+
+    ```bash
+    cat /tmp/fio.conf
+
+    [global]
+    ioengine=/spdk/build/fio/spdk_bdev
+    spdk_json_conf=/tmp/bdev.json
+
+    thread=1
+    direct=1
+    group_reporting=1
+
+    bs=4k
+    rw=randread
+    rwmixread=70
+    time_based=1
+    runtime=10
+    norandommap=1
+
+    [filename0]
+    filename=Nvme0n1
+    iodepth=8
+    ```
+
+- Run fio bdev plugin as initiator
+
+    ```bash
+    /usr/src/fio/fio /tmp/fio.conf
+    ```
+
+## Zoned Block Devices
+
+SPDK has a zoned block device API (bdev_zone.h) which currently supports Open-channel SSDs,
+NVMe Zoned Namespaces (ZNS), and the virtual zoned block device SPDK module.
+
+If you wish to run fio against a SPDK zoned block device, you can use the fio option:
+
+```bash
+zonemode=zbd
+```
+
+It is recommended to use a fio version newer than version 3.26, if using --numjobs > 1.
+If using --numjobs=1, fio version >= 3.23 should suffice.
+
+See zbd_example.fio in this directory for a zoned block device example config.
+
+### Maximum Open Zones
+
+Most zoned block devices have a resource constraint on the amount of zones which can be in an opened
+state at any point in time. It is very important to not exceed this limit.
+
+You can control how many zones fio will keep in an open state by using the
+`--max_open_zones` option.
+
+If you use a fio version newer than 3.26, fio will automatically detect and set the proper value.
+If you use an old version of fio, make sure to provide the proper --max_open_zones value yourself.
+
+### Maximum Active Zones
+
+Zoned block devices may also have a resource constraint on the number of zones that can be active at
+any point in time. Unlike `max_open_zones`, fio currently does not manage this constraint, and
+there is thus no option to limit it either.
+
+Since the max active zones limit (by definition) has to be greater than or equal to the max open
+zones limit, the easiest way to work around that fio does not manage this constraint, is to start
+with a clean state each run (except for read-only workloads), by resetting all zones before fio
+starts running its jobs by using the engine option:
+
+```bash
+--initial_zone_reset=1
+```
+
+### Zone Append
+
+When running fio against a zoned block device you need to specify --iodepth=1 to avoid
+"Zone Invalid Write: The write to a zone was not at the write pointer." I/O errors.
+However, if your zoned block device supports Zone Append, you can use the engine option:
+
+```bash
+--zone_append=1
+```
+
+To send zone append commands instead of write commands to the zoned block device.
+When using zone append, you will be able to specify a --iodepth greater than 1.

examples/bdev/fio_plugin/bdev_zoned.json

@@ -0,0 +1,26 @@
+{
+  "subsystems": [
+    {
+      "subsystem": "bdev",
+      "config": [
+        {
+          "method": "bdev_malloc_create",
+          "params": {
+            "name": "Malloc0",
+            "num_blocks": 2097152,
+            "block_size": 512
+          }
+        },
+        {
+          "method": "bdev_zone_block_create",
+          "params": {
+            "base_bdev": "Malloc0",
+            "name": "Zone0",
+            "zone_capacity": 262144,
+            "optimal_open_zones": 8
+          }
+        }
+      ]
+    }
+  ]
+}

examples/bdev/fio_plugin/fio_plugin.c

@@ -34,8 +34,10 @@
 #include "spdk/stdinc.h"
 
 #include "spdk/bdev.h"
+#include "spdk/bdev_zone.h"
 #include "spdk/accel_engine.h"
 #include "spdk/env.h"
+#include "spdk/init.h"
 #include "spdk/thread.h"
 #include "spdk/log.h"
 #include "spdk/string.h"
@@ -48,6 +50,12 @@
 #include "fio.h"
 #include "optgroup.h"
 
+#ifdef for_each_rw_ddir
+#define FIO_HAS_ZBD (FIO_IOOPS_VERSION >= 26)
+#else
+#define FIO_HAS_ZBD (0)
+#endif
+
 /* FreeBSD is missing CLOCK_MONOTONIC_RAW,
  * so alternative is provided. */
 #ifndef CLOCK_MONOTONIC_RAW /* Defined in glibc bits/time.h */
@@ -58,8 +66,11 @@ struct spdk_fio_options {
 	void *pad;
 	char *conf;
 	char *json_conf;
+	char *log_flags;
 	unsigned mem_mb;
 	int mem_single_seg;
+	int initial_zone_reset;
+	int zone_append;
 };
 
 struct spdk_fio_request {
@@ -71,6 +82,7 @@ struct spdk_fio_target {
 	struct spdk_bdev	*bdev;
 	struct spdk_bdev_desc	*desc;
 	struct spdk_io_channel	*ch;
+	bool zone_append_enabled;
 
 	TAILQ_ENTRY(spdk_fio_target) link;
 };
@@ -89,12 +101,24 @@ struct spdk_fio_thread {
 	TAILQ_ENTRY(spdk_fio_thread)	link;
 };
 
+struct spdk_fio_zone_cb_arg {
+	struct spdk_fio_target *target;
+	struct spdk_bdev_zone_info *spdk_zones;
+	int completed;
+	uint64_t offset_blocks;
+	struct zbd_zone *fio_zones;
+	unsigned int nr_zones;
+};
+
 static bool g_spdk_env_initialized = false;
 static const char *g_json_config_file = NULL;
 
 static int spdk_fio_init(struct thread_data *td);
 static void spdk_fio_cleanup(struct thread_data *td);
 static size_t spdk_fio_poll_thread(struct spdk_fio_thread *fio_thread);
+static int spdk_fio_handle_options(struct thread_data *td, struct fio_file *f,
+				   struct spdk_bdev *bdev);
+static int spdk_fio_handle_options_per_target(struct thread_data *td, struct fio_file *f);
 
 static pthread_t g_init_thread_id = 0;
 static pthread_mutex_t g_init_mtx = PTHREAD_MUTEX_INITIALIZER;
@@ -196,8 +220,8 @@ spdk_fio_bdev_init_start(void *arg)
 {
 	bool *done = arg;
 
-	spdk_app_json_config_load(g_json_config_file, SPDK_DEFAULT_RPC_ADDR,
-				  spdk_fio_bdev_init_done, done, true);
+	spdk_subsystem_init_from_json_config(g_json_config_file, SPDK_DEFAULT_RPC_ADDR,
+					     spdk_fio_bdev_init_done, done, true);
 }
 
 static void
@@ -263,6 +287,21 @@ spdk_init_thread_poll(void *arg)
 	}
 	spdk_unaffinitize_thread();
 
+	if (eo->log_flags) {
+		char *tok = strtok(eo->log_flags, ",");
+		do {
+			rc = spdk_log_set_flag(tok);
+			if (rc < 0) {
+				SPDK_ERRLOG("unknown spdk log flag %s\n", tok);
+				rc = EINVAL;
+				goto err_exit;
+			}
+		} while ((tok = strtok(NULL, ",")) != NULL);
+#ifdef DEBUG
+		spdk_log_set_print_level(SPDK_LOG_DEBUG);
+#endif
+	}
+
 	spdk_thread_lib_init(NULL, 0);
 
 	/* Create an SPDK thread temporarily */
@@ -403,6 +442,27 @@ out:
 	return rc;
 }
 
+static bool
+fio_redirected_to_dev_null(void)
+{
+	char path[PATH_MAX] = "";
+	ssize_t ret;
+
+	ret = readlink("/proc/self/fd/1", path, sizeof(path));
+
+	if (ret == -1 || strcmp(path, "/dev/null") != 0) {
+		return false;
+	}
+
+	ret = readlink("/proc/self/fd/2", path, sizeof(path));
+
+	if (ret == -1 || strcmp(path, "/dev/null") != 0) {
+		return false;
+	}
+
+	return true;
+}
+
 /* Called for each thread to fill in the 'real_file_size' member for
  * each file associated with this thread. This is called prior to
  * the init operation (spdk_fio_init()) below. This call will occur
@@ -416,16 +476,18 @@ spdk_fio_setup(struct thread_data *td)
 	unsigned int i;
 	struct fio_file *f;
 
-	/* we might be running in a daemonized FIO instance where standard
-	 * input and output were closed and fds 0, 1, and 2 are reused
-	 * for something important by FIO. We can't ensure we won't print
-	 * anything (and so will our dependencies, e.g. DPDK), so abort early.
-	 * (is_backend is an fio global variable)
+	/*
+	 * If we're running in a daemonized FIO instance, it's possible
+	 * fd 1/2 were re-used for something important by FIO. Newer fio
+	 * versions are careful to redirect those to /dev/null, but if we're
+	 * not, we'll abort early, so we don't accidentally write messages to
+	 * an important file, etc.
 	 */
-	if (is_backend) {
+	if (is_backend && !fio_redirected_to_dev_null()) {
 		char buf[1024];
 		snprintf(buf, sizeof(buf),
-			 "SPDK FIO plugin won't work with daemonized FIO server.");
+			 "SPDK FIO plugin is in daemon mode, but stdout/stderr "
+			 "aren't redirected to /dev/null. Aborting.");
 		fio_server_text_output(FIO_LOG_ERR, buf, sizeof(buf));
 		return -1;
 	}
@@ -455,6 +517,7 @@ spdk_fio_setup(struct thread_data *td)
 
 	for_each_file(td, f, i) {
 		struct spdk_bdev *bdev;
+		int rc;
 
 		if (strcmp(f->file_name, "*") == 0) {
 			continue;
@@ -468,7 +531,13 @@ spdk_fio_setup(struct thread_data *td)
 
 		f->real_file_size = spdk_bdev_get_num_blocks(bdev) *
 				    spdk_bdev_get_block_size(bdev);
+		f->filetype = FIO_TYPE_BLOCK;
+		fio_file_set_size_known(f);
 
+		rc = spdk_fio_handle_options(td, f, bdev);
+		if (rc) {
+			return rc;
+		}
 	}
 
 	return 0;
@@ -528,21 +597,48 @@ spdk_fio_bdev_open(void *arg)
 
 		f->engine_data = target;
 
+		rc = spdk_fio_handle_options_per_target(td, f);
+		if (rc) {
+			SPDK_ERRLOG("Failed to handle options for: %s\n", f->file_name);
+			f->engine_data = NULL;
+			spdk_put_io_channel(target->ch);
+			spdk_bdev_close(target->desc);
+			free(target);
+			fio_thread->failed = true;
+			return;
+		}
+
 		TAILQ_INSERT_TAIL(&fio_thread->targets, target, link);
 	}
 }
 
 /* Called for each thread, on that thread, shortly after the thread
  * starts.
+ *
+ * Also called by spdk_fio_report_zones(), since we need an I/O channel
+ * in order to get the zone report. (fio calls the .report_zones callback
+ * before it calls the .init callback.)
+ * Therefore, if fio was run with --zonemode=zbd, the thread will already
+ * be initialized by the time that fio calls the .init callback.
  */
 static int
 spdk_fio_init(struct thread_data *td)
 {
 	struct spdk_fio_thread *fio_thread;
+	int rc;
 
-	spdk_fio_init_thread(td);
+	/* If thread has already been initialized, do nothing. */
+	if (td->io_ops_data) {
+		return 0;
+	}
+
+	rc = spdk_fio_init_thread(td);
+	if (rc) {
+		return rc;
+	}
 
 	fio_thread = td->io_ops_data;
+	assert(fio_thread);
 	fio_thread->failed = false;
 
 	spdk_thread_send_msg(fio_thread->thread, spdk_fio_bdev_open, td);
@@ -644,6 +740,16 @@ typedef enum fio_q_status fio_q_status_t;
 typedef int fio_q_status_t;
 #endif
 
+static uint64_t
+spdk_fio_zone_bytes_to_blocks(struct spdk_bdev *bdev, uint64_t offset_bytes, uint64_t *zone_start,
+			      uint64_t num_bytes, uint64_t *num_blocks)
+{
+	uint32_t block_size = spdk_bdev_get_block_size(bdev);
+	*zone_start = spdk_bdev_get_zone_id(bdev, offset_bytes / block_size);
+	*num_blocks = num_bytes / block_size;
+	return (offset_bytes % block_size) | (num_bytes % block_size);
+}
+
 static fio_q_status_t
 spdk_fio_queue(struct thread_data *td, struct io_u *io_u)
 {
@@ -666,9 +772,21 @@ spdk_fio_queue(struct thread_data *td, struct io_u *io_u)
 				    spdk_fio_completion_cb, fio_req);
 		break;
 	case DDIR_WRITE:
-		rc = spdk_bdev_write(target->desc, target->ch,
-				     io_u->buf, io_u->offset, io_u->xfer_buflen,
-				     spdk_fio_completion_cb, fio_req);
+		if (!target->zone_append_enabled) {
+			rc = spdk_bdev_write(target->desc, target->ch,
+					     io_u->buf, io_u->offset, io_u->xfer_buflen,
+					     spdk_fio_completion_cb, fio_req);
+		} else {
+			uint64_t zone_start, num_blocks;
+			if (spdk_fio_zone_bytes_to_blocks(target->bdev, io_u->offset, &zone_start,
+							  io_u->xfer_buflen, &num_blocks) != 0) {
+				rc = -EINVAL;
+				break;
+			}
+			rc = spdk_bdev_zone_append(target->desc, target->ch, io_u->buf,
+						   zone_start, num_blocks, spdk_fio_completion_cb,
+						   fio_req);
+		}
 		break;
 	case DDIR_TRIM:
 		rc = spdk_bdev_unmap(target->desc, target->ch,
@@ -755,6 +873,308 @@ spdk_fio_invalidate(struct thread_data *td, struct fio_file *f)
 	return 0;
 }
 
+#if FIO_HAS_ZBD
+static int
+spdk_fio_get_zoned_model(struct thread_data *td, struct fio_file *f, enum zbd_zoned_model *model)
+{
+	struct spdk_bdev *bdev;
+
+	if (f->filetype != FIO_TYPE_BLOCK) {
+		SPDK_ERRLOG("Unsupported filetype: %d\n", f->filetype);
+		return -EINVAL;
+	}
+
+	bdev = spdk_bdev_get_by_name(f->file_name);
+	if (!bdev) {
+		SPDK_ERRLOG("Cannot get zoned model, no bdev with name: %s\n", f->file_name);
+		return -ENODEV;
+	}
+
+	if (spdk_bdev_is_zoned(bdev)) {
+		*model = ZBD_HOST_MANAGED;
+	} else {
+		*model = ZBD_NONE;
+	}
+
+	return 0;
+}
+
+static void
+spdk_fio_bdev_get_zone_info_done(struct spdk_bdev_io *bdev_io, bool success, void *arg)
+{
+	struct spdk_fio_zone_cb_arg *cb_arg = arg;
+	unsigned int i;
+	int handled_zones = 0;
+
+	if (!success) {
+		spdk_bdev_free_io(bdev_io);
+		cb_arg->completed = -EIO;
+		return;
+	}
+
+	for (i = 0; i < cb_arg->nr_zones; i++) {
+		struct spdk_bdev_zone_info *zone_src = &cb_arg->spdk_zones[handled_zones];
+		struct zbd_zone *zone_dest = &cb_arg->fio_zones[handled_zones];
+		uint32_t block_size = spdk_bdev_get_block_size(cb_arg->target->bdev);
+
+		zone_dest->type = ZBD_ZONE_TYPE_SWR;
+		zone_dest->len = spdk_bdev_get_zone_size(cb_arg->target->bdev) * block_size;
+		zone_dest->capacity = zone_src->capacity * block_size;
+		zone_dest->start = zone_src->zone_id * block_size;
+		zone_dest->wp = zone_src->write_pointer * block_size;
+
+		switch (zone_src->state) {
+		case SPDK_BDEV_ZONE_STATE_EMPTY:
+			zone_dest->cond = ZBD_ZONE_COND_EMPTY;
+			break;
+		case SPDK_BDEV_ZONE_STATE_IMP_OPEN:
+			zone_dest->cond = ZBD_ZONE_COND_IMP_OPEN;
+			break;
+		case SPDK_BDEV_ZONE_STATE_EXP_OPEN:
+			zone_dest->cond = ZBD_ZONE_COND_EXP_OPEN;
+			break;
+		case SPDK_BDEV_ZONE_STATE_FULL:
+			zone_dest->cond = ZBD_ZONE_COND_FULL;
+			break;
+		case SPDK_BDEV_ZONE_STATE_CLOSED:
+			zone_dest->cond = ZBD_ZONE_COND_CLOSED;
+			break;
+		case SPDK_BDEV_ZONE_STATE_READ_ONLY:
+			zone_dest->cond = ZBD_ZONE_COND_READONLY;
+			break;
+		case SPDK_BDEV_ZONE_STATE_OFFLINE:
+			zone_dest->cond = ZBD_ZONE_COND_OFFLINE;
+			break;
+		default:
+			spdk_bdev_free_io(bdev_io);
+			cb_arg->completed = -EIO;
+			return;
+		}
+		handled_zones++;
+	}
+
+	spdk_bdev_free_io(bdev_io);
+	cb_arg->completed = handled_zones;
+}
+
+static void
+spdk_fio_bdev_get_zone_info(void *arg)
+{
+	struct spdk_fio_zone_cb_arg *cb_arg = arg;
+	struct spdk_fio_target *target = cb_arg->target;
+	int rc;
+
+	rc = spdk_bdev_get_zone_info(target->desc, target->ch, cb_arg->offset_blocks,
+				     cb_arg->nr_zones, cb_arg->spdk_zones,
+				     spdk_fio_bdev_get_zone_info_done, cb_arg);
+	if (rc < 0) {
+		cb_arg->completed = rc;
+	}
+}
+
+static int
+spdk_fio_report_zones(struct thread_data *td, struct fio_file *f, uint64_t offset,
+		      struct zbd_zone *zones, unsigned int nr_zones)
+{
+	struct spdk_fio_target *target;
+	struct spdk_fio_thread *fio_thread;
+	struct spdk_fio_zone_cb_arg cb_arg;
+	uint32_t block_size;
+	int rc;
+
+	if (nr_zones == 0) {
+		return 0;
+	}
+
+	/* spdk_fio_report_zones() is only called before the bdev I/O channels have been created.
+	 * Since we need an I/O channel for report_zones(), call spdk_fio_init() to initialize
+	 * the thread early.
+	 * spdk_fio_report_zones() might be called several times by fio, if e.g. the zone report
+	 * for all zones does not fit in the buffer that fio has allocated for the zone report.
+	 * It is safe to call spdk_fio_init(), even if the thread has already been initialized.
+	 */
+	rc = spdk_fio_init(td);
+	if (rc) {
+		return rc;
+	}
+	fio_thread = td->io_ops_data;
+	target = f->engine_data;
+
+	assert(fio_thread);
+	assert(target);
+
+	block_size = spdk_bdev_get_block_size(target->bdev);
+
+	cb_arg.target = target;
+	cb_arg.completed = 0;
+	cb_arg.offset_blocks = offset / block_size;
+	cb_arg.fio_zones = zones;
+	cb_arg.nr_zones = spdk_min(nr_zones, spdk_bdev_get_num_zones(target->bdev));
+
+	cb_arg.spdk_zones = calloc(1, sizeof(*cb_arg.spdk_zones) * cb_arg.nr_zones);
+	if (!cb_arg.spdk_zones) {
+		SPDK_ERRLOG("Could not allocate memory for zone report!\n");
+		rc = -ENOMEM;
+		goto cleanup_thread;
+	}
+
+	spdk_thread_send_msg(fio_thread->thread, spdk_fio_bdev_get_zone_info, &cb_arg);
+	do {
+		spdk_fio_poll_thread(fio_thread);
+	} while (!cb_arg.completed);
+
+	/* Free cb_arg.spdk_zones. The report in fio format is stored in cb_arg.fio_zones/zones. */
+	free(cb_arg.spdk_zones);
+
+	rc = cb_arg.completed;
+	if (rc < 0) {
+		SPDK_ERRLOG("Failed to get zone info: %d\n", rc);
+		goto cleanup_thread;
+	}
+
+	/* Return the amount of zones successfully copied. */
+	return rc;
+
+cleanup_thread:
+	spdk_fio_cleanup(td);
+
+	return rc;
+}
+
+static void
+spdk_fio_bdev_zone_reset_done(struct spdk_bdev_io *bdev_io, bool success, void *arg)
+{
+	struct spdk_fio_zone_cb_arg *cb_arg = arg;
+
+	spdk_bdev_free_io(bdev_io);
+
+	if (!success) {
+		cb_arg->completed = -EIO;
+	} else {
+		cb_arg->completed = 1;
+	}
+}
+
+static void
+spdk_fio_bdev_zone_reset(void *arg)
+{
+	struct spdk_fio_zone_cb_arg *cb_arg = arg;
+	struct spdk_fio_target *target = cb_arg->target;
+	int rc;
+
+	rc = spdk_bdev_zone_management(target->desc, target->ch, cb_arg->offset_blocks,
+				       SPDK_BDEV_ZONE_RESET,
+				       spdk_fio_bdev_zone_reset_done, cb_arg);
+	if (rc < 0) {
+		cb_arg->completed = rc;
+	}
+}
+
+static int
+spdk_fio_reset_zones(struct spdk_fio_thread *fio_thread, struct spdk_fio_target *target,
+		     uint64_t offset, uint64_t length)
+{
+	uint64_t zone_size_bytes;
+	uint32_t block_size;
+	int rc;
+
+	assert(fio_thread);
+	assert(target);
+
+	block_size = spdk_bdev_get_block_size(target->bdev);
+	zone_size_bytes = spdk_bdev_get_zone_size(target->bdev) * block_size;
+
+	for (uint64_t cur = offset; cur < offset + length; cur += zone_size_bytes) {
+		struct spdk_fio_zone_cb_arg cb_arg = {
+			.target = target,
+			.completed = 0,
+			.offset_blocks = cur / block_size,
+		};
+
+		spdk_thread_send_msg(fio_thread->thread, spdk_fio_bdev_zone_reset, &cb_arg);
+		do {
+			spdk_fio_poll_thread(fio_thread);
+		} while (!cb_arg.completed);
+
+		rc = cb_arg.completed;
+		if (rc < 0) {
+			SPDK_ERRLOG("Failed to reset zone: %d\n", rc);
+			return rc;
+		}
+	}
+
+	return 0;
+}
+
+static int
+spdk_fio_reset_wp(struct thread_data *td, struct fio_file *f, uint64_t offset, uint64_t length)
+{
+	return spdk_fio_reset_zones(td->io_ops_data, f->engine_data, offset, length);
+}
+#endif
+
+#if FIO_IOOPS_VERSION >= 30
+static int spdk_fio_get_max_open_zones(struct thread_data *td, struct fio_file *f,
+				       unsigned int *max_open_zones)
+{
+	struct spdk_bdev *bdev;
+
+	bdev = spdk_bdev_get_by_name(f->file_name);
+	if (!bdev) {
+		SPDK_ERRLOG("Cannot get max open zones, no bdev with name: %s\n", f->file_name);
+		return -ENODEV;
+	}
+
+	*max_open_zones = spdk_bdev_get_max_open_zones(bdev);
+
+	return 0;
+}
+#endif
+
+static int
+spdk_fio_handle_options(struct thread_data *td, struct fio_file *f, struct spdk_bdev *bdev)
+{
+	struct spdk_fio_options *fio_options = td->eo;
+
+	if (fio_options->initial_zone_reset && spdk_bdev_is_zoned(bdev)) {
+#if FIO_HAS_ZBD
+		int rc = spdk_fio_init(td);
+		if (rc) {
+			return rc;
+		}
+		rc = spdk_fio_reset_zones(td->io_ops_data, f->engine_data, 0, f->real_file_size);
+		if (rc) {
+			spdk_fio_cleanup(td);
+			return rc;
+		}
+#else
+		SPDK_ERRLOG("fio version is too old to support zoned block devices\n");
+#endif
+	}
+
+	return 0;
+}
+
+static int
+spdk_fio_handle_options_per_target(struct thread_data *td, struct fio_file *f)
+{
+	struct spdk_fio_target *target = f->engine_data;
+	struct spdk_fio_options *fio_options = td->eo;
+
+	if (fio_options->zone_append && spdk_bdev_is_zoned(target->bdev)) {
+		if (spdk_bdev_io_type_supported(target->bdev, SPDK_BDEV_IO_TYPE_ZONE_APPEND)) {
+			SPDK_DEBUGLOG(fio_bdev, "Using zone appends instead of writes on: '%s'\n",
+				      f->file_name);
+			target->zone_append_enabled = true;
+		} else {
+			SPDK_WARNLOG("Falling back to writes on: '%s' - bdev lacks zone append cmd\n",
+				     f->file_name);
+		}
+	}
+
+	return 0;
+}
+
 static struct fio_option options[] = {
 	{
 		.name		= "spdk_conf",
@@ -793,6 +1213,35 @@ static struct fio_option options[] = {
 		.category	= FIO_OPT_C_ENGINE,
 		.group		= FIO_OPT_G_INVALID,
 	},
+	{
+		.name           = "log_flags",
+		.lname          = "log flags",
+		.type           = FIO_OPT_STR_STORE,
+		.off1           = offsetof(struct spdk_fio_options, log_flags),
+		.help           = "SPDK log flags to enable",
+		.category       = FIO_OPT_C_ENGINE,
+		.group          = FIO_OPT_G_INVALID,
+	},
+	{
+		.name		= "initial_zone_reset",
+		.lname		= "Reset Zones on initialization",
+		.type		= FIO_OPT_INT,
+		.off1		= offsetof(struct spdk_fio_options, initial_zone_reset),
+		.def		= "0",
+		.help		= "Reset Zones on initialization (0=disable, 1=Reset All Zones)",
+		.category	= FIO_OPT_C_ENGINE,
+		.group		= FIO_OPT_G_INVALID,
+	},
+	{
+		.name		= "zone_append",
+		.lname		= "Use zone append instead of write",
+		.type		= FIO_OPT_INT,
+		.off1		= offsetof(struct spdk_fio_options, zone_append),
+		.def		= "0",
+		.help		= "Use zone append instead of write (1=zone append, 0=write)",
+		.category	= FIO_OPT_C_ENGINE,
+		.group		= FIO_OPT_G_INVALID,
+	},
 	{
 		.name		= NULL,
 	},
@@ -823,6 +1272,14 @@ struct ioengine_ops ioengine = {
 	.iomem_free		= spdk_fio_iomem_free,
 	.io_u_init		= spdk_fio_io_u_init,
 	.io_u_free		= spdk_fio_io_u_free,
+#if FIO_HAS_ZBD
+	.get_zoned_model	= spdk_fio_get_zoned_model,
+	.report_zones		= spdk_fio_report_zones,
+	.reset_wp		= spdk_fio_reset_wp,
+#endif
+#if FIO_IOOPS_VERSION >= 30
+	.get_max_open_zones	= spdk_fio_get_max_open_zones,
+#endif
 	.option_struct_size	= sizeof(struct spdk_fio_options),
 	.options		= options,
 };
@@ -842,6 +1299,7 @@ spdk_fio_finish_env(void)
 	pthread_join(g_init_thread_id, NULL);
 
 	spdk_thread_lib_fini();
+	spdk_env_fini();
 }
 
 static void fio_exit spdk_fio_unregister(void)
@@ -852,3 +1310,5 @@ static void fio_exit spdk_fio_unregister(void)
 	}
 	unregister_ioengine(&ioengine);
 }
+
+SPDK_LOG_REGISTER_COMPONENT(fio_bdev)

examples/bdev/fio_plugin/zbd_example.fio

@@ -0,0 +1,18 @@
+[global]
+ioengine=spdk_bdev
+spdk_json_conf=./examples/bdev/fio_plugin/bdev_zoned.json
+thread=1
+direct=1
+time_based
+runtime=10
+rw=randwrite
+bs=16K
+zonemode=zbd
+max_open_zones=8
+initial_zone_reset=1
+zone_append=1
+iodepth=64
+
+[test]
+filename=Zone0
+numjobs=1