diff --git a/CHANGELOG.md b/CHANGELOG.md index 74306a2e8f..2030e66d48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ Command line parameters `-r` and `--rpc-socket` will longer accept TCP ports. RP must now be started on a Unix domain socket. Exposing RPC on the network, as well as providing proper authentication (if needed) is now a responsibility of the user. +### build + +The fio plugins now compile to `build/fio` and are named `spdk_bdev` and `spdk_nvme`. +Existing fio configuration files will need to be updated. + ### accel A new API was added `spdk_accel_get_capabilities` that allows applications to diff --git a/Makefile b/Makefile index b81074766c..72b239ce42 100644 --- a/Makefile +++ b/Makefile @@ -78,6 +78,7 @@ all: mk/cc.mk $(DIRS-y) clean: $(DIRS-y) $(Q)rm -f include/spdk/config.h $(Q)rm -rf build/bin + $(Q)rm -rf build/fio install: all $(Q)echo "Installed to $(DESTDIR)$(CONFIG_PREFIX)" @@ -107,6 +108,7 @@ mk/cc.mk: build_dir: mk/cc.mk $(Q)mkdir -p build/lib $(Q)mkdir -p build/bin + $(Q)mkdir -p build/fio include/spdk/config.h: mk/config.mk scripts/genconfig.py $(Q)echo "#ifndef SPDK_CONFIG_H" > $@.tmp; \ diff --git a/examples/bdev/fio_plugin/Makefile b/examples/bdev/fio_plugin/Makefile index f28aeef280..d43221402f 100644 --- a/examples/bdev/fio_plugin/Makefile +++ b/examples/bdev/fio_plugin/Makefile @@ -36,15 +36,13 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..) include $(SPDK_ROOT_DIR)/mk/spdk.common.mk include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk -APP := fio_plugin +FIO_PLUGIN := spdk_bdev C_SRCS = fio_plugin.c -CFLAGS += -I$(CONFIG_FIO_SOURCE_DIR) -LDFLAGS += -shared -rdynamic -Wl,-z,nodelete # Unable to combine the FIO plugin and the VPP socket abstraction (license incompatibility) SPDK_LIB_LIST = $(filter-out sock_vpp,$(ALL_MODULES_LIST)) SPDK_LIB_LIST += thread util bdev bdev_rpc conf accel rpc jsonrpc json log sock trace notify SPDK_LIB_LIST += event event_bdev event_accel event_vmd -include $(SPDK_ROOT_DIR)/mk/spdk.app.mk +include $(SPDK_ROOT_DIR)/mk/spdk.fio.mk diff --git a/examples/bdev/fio_plugin/README.md b/examples/bdev/fio_plugin/README.md index 2913b73888..369756fe31 100644 --- a/examples/bdev/fio_plugin/README.md +++ b/examples/bdev/fio_plugin/README.md @@ -45,7 +45,7 @@ To use the SPDK fio plugin with fio, specify the plugin binary using LD_PRELOAD fio and set ioengine=spdk_bdev in the fio configuration file (see example_config.fio in the same directory as this README). - LD_PRELOAD=/examples/bdev/fio_plugin/fio_plugin fio + LD_PRELOAD=/build/fio/spdk_bdev fio The fio configuration file must contain one new parameter: diff --git a/examples/nvme/fio_plugin/Makefile b/examples/nvme/fio_plugin/Makefile index 5032efdc88..5ffdac8010 100644 --- a/examples/nvme/fio_plugin/Makefile +++ b/examples/nvme/fio_plugin/Makefile @@ -32,11 +32,20 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -SPDK_ROOT_DIR := $(CURDIR)/../../.. +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..) +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk +include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk -APP := fio_plugin +FIO_PLUGIN := spdk_nvme -CFLAGS += -I$(CONFIG_FIO_SOURCE_DIR) -LDFLAGS += -shared -rdynamic +C_SRCS = fio_plugin.c -include $(SPDK_ROOT_DIR)/mk/nvme.libtest.mk +# Unable to combine the FIO plugin and the VPP socket abstraction (license incompatibility) +SPDK_LIB_LIST = $(filter-out sock_vpp,$(SOCK_MODULES_LIST)) +SPDK_LIB_LIST += nvme thread util log sock vmd + +ifeq ($(CONFIG_RDMA),y) +SPDK_LIB_LIST += rdma +endif + +include $(SPDK_ROOT_DIR)/mk/spdk.fio.mk diff --git a/examples/nvme/fio_plugin/README.md b/examples/nvme/fio_plugin/README.md index da4b554256..e7a8b7c016 100644 --- a/examples/nvme/fio_plugin/README.md +++ b/examples/nvme/fio_plugin/README.md @@ -36,7 +36,7 @@ To use the SPDK fio plugin with fio, specify the plugin binary using LD_PRELOAD fio and set ioengine=spdk in the fio configuration file (see example_config.fio in the same directory as this README). - LD_PRELOAD=/examples/nvme/fio_plugin/fio_plugin fio + LD_PRELOAD=/build/fio/spdk_nvme fio To select NVMe devices, you pass an SPDK Transport Identifier string as the filename. These are in the form: diff --git a/mk/spdk.fio.mk b/mk/spdk.fio.mk new file mode 100644 index 0000000000..6aff4a0a73 --- /dev/null +++ b/mk/spdk.fio.mk @@ -0,0 +1,63 @@ +# +# BSD LICENSE +# +# Copyright (c) Intel Corporation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(SPDK_ROOT_DIR)/mk/spdk.app_vars.mk + +# Plugins go into build/example/ +FIO_PLUGIN := $(SPDK_ROOT_DIR)/build/fio/$(notdir $(FIO_PLUGIN)) + +LIBS += $(SPDK_LIB_LINKER_ARGS) + +CFLAGS += -I$(CONFIG_FIO_SOURCE_DIR) +LDFLAGS += -shared -rdynamic -Wl,-z,nodelete + +CLEAN_FILES = $(FIO_PLUGIN) + +all : $(FIO_PLUGIN) + @: + +install: empty_rule + +uninstall: empty_rule + +# To avoid overwriting warning +empty_rule: + @: + +$(FIO_PLUGIN) : $(OBJS) $(SPDK_LIB_FILES) $(ENV_LIBS) + $(LINK_C) + +clean : + $(CLEAN_C) $(CLEAN_FILES) + +include $(SPDK_ROOT_DIR)/mk/spdk.deps.mk diff --git a/scripts/perf/nvme/run_fio_test.sh b/scripts/perf/nvme/run_fio_test.sh index 454ea555f2..bc888d0f48 100755 --- a/scripts/perf/nvme/run_fio_test.sh +++ b/scripts/perf/nvme/run_fio_test.sh @@ -7,13 +7,13 @@ rootdir=$(readlink -f $testdir/../../..) $rootdir/scripts/setup.sh # Run Performance Test with 1 SSD -$testdir/run_fio_test.py $testdir/fio_test.conf $rootdir/examples/nvme/fio_plugin/fio_plugin 1 +$testdir/run_fio_test.py $testdir/fio_test.conf $rootdir/build/fio/spdk_nvme 1 # 2 SSDs test run -$testdir/run_fio_test.py $testdir/fio_test.conf $rootdir/examples/nvme/fio_plugin/fio_plugin 2 +$testdir/run_fio_test.py $testdir/fio_test.conf $rootdir/build/fio/spdk_nvme 2 # 4 SSDs test run -$testdir/run_fio_test.py $testdir/fio_test.conf $rootdir/examples/nvme/fio_plugin/fio_plugin 4 +$testdir/run_fio_test.py $testdir/fio_test.conf $rootdir/build/fio/spdk_nvme 4 # 8 SSDs test run -$testdir/run_fio_test.py $testdir/fio_test.conf $rootdir/examples/nvme/fio_plugin/fio_plugin 8 +$testdir/run_fio_test.py $testdir/fio_test.conf $rootdir/build/fio/spdk_nvme 8 diff --git a/scripts/perf/nvmf/run_nvmf.py b/scripts/perf/nvmf/run_nvmf.py index 447b33ae97..7cf0e78cbf 100755 --- a/scripts/perf/nvmf/run_nvmf.py +++ b/scripts/perf/nvmf/run_nvmf.py @@ -326,7 +326,7 @@ runtime={run_time} subsystems = self.discover_subsystems(self.nic_ips, subsys_no) bdev_conf = self.gen_spdk_bdev_conf(subsystems) self.remote_call("echo '%s' > %s/bdev.conf" % (bdev_conf, self.spdk_dir)) - ioengine = "%s/examples/bdev/fio_plugin/fio_plugin" % self.spdk_dir + ioengine = "%s/build/fio/spdk_bdev" % self.spdk_dir spdk_conf = "spdk_conf=%s/bdev.conf" % self.spdk_dir else: ioengine = "libaio" diff --git a/test/common/autotest_common.sh b/test/common/autotest_common.sh index ea48051dac..01e93d9d34 100755 --- a/test/common/autotest_common.sh +++ b/test/common/autotest_common.sh @@ -1074,7 +1074,7 @@ EOL function fio_bdev() { # Setup fio binary cmd line local fio_dir=$CONFIG_FIO_SOURCE_DIR - local bdev_plugin="$rootdir/examples/bdev/fio_plugin/fio_plugin" + local bdev_plugin="$rootdir/build/fio/spdk_bdev" # Preload AddressSanitizer library to fio if fio_plugin was compiled with it local asan_lib @@ -1086,7 +1086,7 @@ function fio_bdev() { function fio_nvme() { # Setup fio binary cmd line local fio_dir=$CONFIG_FIO_SOURCE_DIR - local nvme_plugin="$rootdir/examples/nvme/fio_plugin/fio_plugin" + local nvme_plugin="$rootdir/build/fio/spdk_nvme" # Preload AddressSanitizer library to fio if fio_plugin was compiled with it asan_lib=$(ldd $nvme_plugin | grep libasan | awk '{print $3}') diff --git a/test/nvme/nvme.sh b/test/nvme/nvme.sh index 8e8fa94be9..cb50e8662e 100755 --- a/test/nvme/nvme.sh +++ b/test/nvme/nvme.sh @@ -24,10 +24,9 @@ function nvme_perf() { } function nvme_fio_test() { - PLUGIN_DIR=$rootdir/examples/nvme/fio_plugin for bdf in $(get_nvme_bdfs); do for blkname in $(get_nvme_name_from_bdf $bdf); do - fio_nvme $PLUGIN_DIR/example_config.fio --filename="trtype=PCIe traddr=${bdf//:/.} ns=${blkname##*n}" + fio_nvme $rootdir/examples/nvme/fio_plugin/example_config.fio --filename="trtype=PCIe traddr=${bdf//:/.} ns=${blkname##*n}" done done } diff --git a/test/nvme/perf/common.sh b/test/nvme/perf/common.sh index 805c1c0b85..d12c9d1e45 100755 --- a/test/nvme/perf/common.sh +++ b/test/nvme/perf/common.sh @@ -4,8 +4,7 @@ set -e BASE_DIR=$(readlink -f $(dirname $0)) ROOT_DIR=$(readlink -f $BASE_DIR/../../..) rootdir=$ROOT_DIR -PLUGIN_DIR_NVME=$ROOT_DIR/examples/nvme/fio_plugin -PLUGIN_DIR_BDEV=$ROOT_DIR/examples/bdev/fio_plugin +PLUGIN_DIR=$ROOT_DIR/build/fio BDEVPERF_DIR=$ROOT_DIR/test/bdev/bdevperf NVMEPERF_DIR=$ROOT_DIR/examples/nvme/perf . $ROOT_DIR/scripts/common.sh || exit 1 @@ -350,9 +349,9 @@ function run_spdk_nvme_fio() { local plugin=$1 echo "** Running fio test, this can take a while, depending on the run-time and ramp-time setting." if [[ "$plugin" = "spdk-plugin-nvme" ]]; then - LD_PRELOAD=$PLUGIN_DIR_NVME/fio_plugin $FIO_BIN $BASE_DIR/config.fio --output-format=json "${@:2}" --ioengine=spdk + LD_PRELOAD=$PLUGIN_DIR/spdk_nvme $FIO_BIN $BASE_DIR/config.fio --output-format=json "${@:2}" --ioengine=spdk elif [[ "$plugin" = "spdk-plugin-bdev" ]]; then - LD_PRELOAD=$PLUGIN_DIR_BDEV/fio_plugin $FIO_BIN $BASE_DIR/config.fio --output-format=json "${@:2}" --ioengine=spdk_bdev --spdk_json_conf=$BASE_DIR/bdev.conf --spdk_mem=4096 + LD_PRELOAD=$PLUGIN_DIR/spdk_bdev $FIO_BIN $BASE_DIR/config.fio --output-format=json "${@:2}" --ioengine=spdk_bdev --spdk_json_conf=$BASE_DIR/bdev.conf --spdk_mem=4096 fi sleep 1 diff --git a/test/vhost/perf_bench/vhost_perf.sh b/test/vhost/perf_bench/vhost_perf.sh index 1b06cf7271..e03197e5a6 100755 --- a/test/vhost/perf_bench/vhost_perf.sh +++ b/test/vhost/perf_bench/vhost_perf.sh @@ -212,7 +212,7 @@ if [[ $run_precondition == true ]]; then fio_filename=$(printf ":%sn1" "${nvmes[@]}") fio_filename=${fio_filename:1} $precond_fio_bin --name="precondition" \ - --ioengine="${rootdir}/examples/bdev/fio_plugin/fio_plugin" \ + --ioengine="${rootdir}/build/fio/spdk_bdev" \ --rw="write" --spdk_json_conf="$json_cfg" --thread="1" \ --group_reporting --direct="1" --size="100%" --loops="2" --bs="256k" \ --iodepth=32 --filename="${fio_filename}" || true diff --git a/test/vmd/vmd.sh b/test/vmd/vmd.sh index fd3c7642c7..f07c41f32d 100755 --- a/test/vmd/vmd.sh +++ b/test/vmd/vmd.sh @@ -21,7 +21,6 @@ function vmd_perf() { } function vmd_fio() { - PLUGIN_DIR=$rootdir/examples/nvme/fio_plugin for bdf in $pci_devs; do fio_nvme $testdir/config/config.fio --filename="trtype=PCIe traddr=${bdf//:/.} ns=1" done