From 5c9a503990babdc3478ed065eaff8b41da18f9b2 Mon Sep 17 00:00:00 2001 From: Konrad Sztyber Date: Tue, 18 Jun 2019 15:19:06 +0200 Subject: [PATCH] test/ftl: use non-volatile cache in functional tests This patch adds non-volatile cache to some of the test configurations if required device exists in the system (regular NVMe disk with separate metadata support). Change-Id: I0ea43990b360712361f34aeeb1982755f48b4dc5 Signed-off-by: Konrad Sztyber Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/459624 Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Ben Walker --- test/ftl/common.sh | 35 ++++++++++++++++++ test/ftl/dirty_shutdown.sh | 75 ++++++++++++++++---------------------- test/ftl/ftl.sh | 28 ++++++++++++-- test/ftl/restore.sh | 28 +++++++++++--- 4 files changed, 114 insertions(+), 52 deletions(-) create mode 100644 test/ftl/common.sh diff --git a/test/ftl/common.sh b/test/ftl/common.sh new file mode 100644 index 0000000000..721bb62193 --- /dev/null +++ b/test/ftl/common.sh @@ -0,0 +1,35 @@ +# Common utility functions to be sourced by the libftl test scripts + +function get_chunk_size() { + echo $($rootdir/examples/nvme/identify/identify -r "trtype:PCIe traddr:$1" | \ + grep 'Logical blks per chunk' | sed 's/[^0-9]//g') +} + +function has_separate_md() { + local md_type=$($rootdir/examples/nvme/identify/identify -r "trtype:PCIe traddr:$1" | \ + grep 'Metadata Transferred' | cut -d: -f2) + if [[ "$md_type" =~ Separate ]]; then + return 0 + else + return 1 + fi +} + +function create_nv_cache_bdev() { + local name=$1 + local ocssd_bdf=$2 + local cache_bdf=$3 + local num_punits=$4 + + local bytes_to_mb=$[1024 * 1024] + local chunk_size=$(get_chunk_size $ocssd_bdf) + + # We need at least 2 bands worth of data + 1 block + local size=$[2 * 4096 * $chunk_size * $num_punits + 1] + # Round the size up to the nearest megabyte + local size=$[($size + $bytes_to_mb) / $bytes_to_mb] + + # Create NVMe bdev on specified device and split it so that it has the desired size + local nvc_bdev=$($rootdir/scripts/rpc.py construct_nvme_bdev -b $name -t PCIe -a $cache_bdf) + $rootdir/scripts/rpc.py construct_split_vbdev $nvc_bdev -s $size 1 +} diff --git a/test/ftl/dirty_shutdown.sh b/test/ftl/dirty_shutdown.sh index b1e082074d..86cf2d3291 100755 --- a/test/ftl/dirty_shutdown.sh +++ b/test/ftl/dirty_shutdown.sh @@ -3,19 +3,25 @@ testdir=$(readlink -f $(dirname $0)) rootdir=$(readlink -f $testdir/../..) source $rootdir/test/common/autotest_common.sh +source $testdir/common.sh rpc_py=$rootdir/scripts/rpc.py pu_start=0 pu_end=3 -additional_blocks=16 + +while getopts ':u:c:' opt; do + case $opt in + u) uuid=$OPTARG ;; + c) nv_cache=$OPTARG ;; + ?) echo "Usage: $0 [-u UUID] [-c NV_CACHE_PCI_BDF] OCSSD_PCI_BDF" && exit 1 ;; + esac +done +shift $((OPTIND -1)) device=$1 -uuid=$2 restore_kill() { rm -f $testdir/config/ftl.json - rm -f $testdir/empty - rm -f $testdir/testblock rm -f $testdir/testfile.md5 rm -f $testdir/testfile2.md5 @@ -26,22 +32,26 @@ restore_kill() { trap "restore_kill; exit 1" SIGINT SIGTERM EXIT -# Extract chunk size -chunk_size=$($rootdir/examples/nvme/identify/identify -r "trtype:PCIe traddr:$device" | -grep 'Logical blks per chunk' | sed 's/[^0-9]//g') +chunk_size=$(get_chunk_size $device) +pu_count=$(($pu_end - $pu_start + 1)) -band_size=$(($chunk_size*($pu_end-$pu_start+1))) +# Write one band worth of data + one extra chunk +data_size=$(($chunk_size * ($pu_count + 1))) $rootdir/app/spdk_tgt/spdk_tgt & svcpid=$! -# Wait until spdk_tgt starts waitforlisten $svcpid -if [ -n "$uuid" ]; then - $rpc_py construct_ftl_bdev -b nvme0 -a $device -l $pu_start-$pu_end -u $uuid -o -else - $rpc_py construct_ftl_bdev -b nvme0 -a $device -l $pu_start-$pu_end -o +if [ -n "$nv_cache" ]; then + nvc_bdev=$(create_nv_cache_bdev nvc0 $device $nv_cache $pu_count) fi +ftl_construct_args="construct_ftl_bdev -b nvme0 -a $device -l $pu_start-$pu_end -o" + +[ -n "$nvc_bdev" ] && ftl_construct_args+=" -c $nvc_bdev" +[ -n "$uuid" ] && ftl_construct_args+=" -u $uuid" + +$rpc_py $ftl_construct_args + # Load the nbd driver modprobe nbd $rpc_py start_nbd_disk nvme0 /dev/nbd0 @@ -49,53 +59,32 @@ waitfornbd nbd0 $rpc_py save_config > $testdir/config/ftl.json -# Send band worth of data in 2 steps (some data should be written to 2nd band due to metadata overhead) -dd if=/dev/urandom of=/dev/nbd0 bs=4K count=$(($band_size - $chunk_size)) oflag=dsync -offset=$(($band_size - $chunk_size)) - -dd if=/dev/urandom of=/dev/nbd0 bs=4K count=$chunk_size seek=$offset oflag=dsync -offset=$(($offset + $chunk_size)) - -# Save md5 data of first batch (which should be fully on a closed band and recoverable) -dd if=/dev/nbd0 bs=4K count=$(($band_size - $chunk_size)) | md5sum > $testdir/testfile.md5 - -# Make sure the third batch of written data is fully on the second band -dd if=/dev/urandom of=/dev/nbd0 bs=4K count=$additional_blocks seek=$offset oflag=dsync -offset=$(($offset + $additional_blocks)) - +dd if=/dev/urandom of=/dev/nbd0 bs=4K count=$data_size oflag=dsync +# Calculate checksum of the data written +dd if=/dev/nbd0 bs=4K count=$data_size | md5sum > $testdir/testfile.md5 $rpc_py stop_nbd_disk /dev/nbd0 # Force kill bdev service (dirty shutdown) and start it again kill -9 $svcpid rm -f /dev/shm/spdk_tgt_trace.pid$svcpid -# TODO Adapt this after waitforlisten is expanded -sleep 5 -$rootdir/app/spdk_tgt/spdk_tgt & svcpid=$! -# Wait until spdk_tgt starts +$rootdir/app/spdk_tgt/spdk_tgt -L ftl_init & svcpid=$! waitforlisten $svcpid -# Ftl should recover, though with a loss of data (-o config option) $rpc_py load_config < $testdir/config/ftl.json waitfornbd nbd0 # Write extra data after restore -dd if=/dev/urandom of=/dev/nbd0 bs=4K count=$chunk_size seek=$offset oflag=dsync +dd if=/dev/urandom of=/dev/nbd0 bs=4K count=$chunk_size seek=$data_size oflag=dsync # Save md5 data -dd if=/dev/nbd0 bs=4K count=$chunk_size skip=$offset | md5sum > $testdir/testfile2.md5 +dd if=/dev/nbd0 bs=4K count=$chunk_size skip=$data_size | md5sum > $testdir/testfile2.md5 # Make sure all data will be read from disk echo 3 > /proc/sys/vm/drop_caches -# Without persistent cache, first batch of data should be recoverable -dd if=/dev/nbd0 bs=4K count=$(($band_size - $chunk_size)) | md5sum -c $testdir/testfile.md5 - -dd if=/dev/nbd0 of=$testdir/testblock bs=4k count=$additional_blocks skip=$band_size -# Last 4k blocks from before restore should be on second band, and return as 0s -dd if=/dev/zero of=$testdir/empty bs=4k count=$additional_blocks -cmp $testdir/empty $testdir/testblock -# Verify data written after restore is still there -dd if=/dev/nbd0 bs=4K count=$chunk_size skip=$(($band_size + $additional_blocks)) | md5sum -c $testdir/testfile2.md5 +# Verify that the checksum matches and the data is consistent +dd if=/dev/nbd0 bs=4K count=$data_size | md5sum -c $testdir/testfile.md5 +dd if=/dev/nbd0 bs=4K count=$chunk_size skip=$data_size | md5sum -c $testdir/testfile2.md5 report_test_completion ftl_dirty_shutdown diff --git a/test/ftl/ftl.sh b/test/ftl/ftl.sh index e210056d52..30473409d8 100755 --- a/test/ftl/ftl.sh +++ b/test/ftl/ftl.sh @@ -3,6 +3,7 @@ testdir=$(readlink -f $(dirname $0)) rootdir=$(readlink -f $testdir/../..) source $rootdir/test/common/autotest_common.sh +source $testdir/common.sh rpc_py=$rootdir/scripts/rpc.py @@ -28,6 +29,22 @@ trap "at_ftl_exit" SIGINT SIGTERM EXIT # OCSSD is blacklisted so bind it to vfio/uio driver before testing PCI_WHITELIST="$device" PCI_BLACKLIST="" DRIVER_OVERRIDE="" ./scripts/setup.sh +# Use first regular NVMe disk (non-OC) as non-volatile cache +nvme_disks=$($rootdir/scripts/gen_nvme.sh --json | jq -r \ + ".config[] | select(.params.traddr != \"$device\").params.traddr") + +for disk in $nvme_disks; do + if has_separate_md $disk; then + nv_cache=$disk + break + fi +done + +if [ -z "$nv_cache" ]; then + # TODO: once CI has devices with separate metadata support fail the test here + echo "Couldn't find NVMe device to be used as non-volatile cache" +fi + timing_enter ftl timing_enter bdevperf @@ -37,11 +54,16 @@ timing_exit bdevperf timing_enter restore run_test suite $testdir/restore.sh $device +if [ -n "$nv_cache" ]; then + run_test suite $testdir/restore.sh -c $nv_cache $device +fi timing_exit restore -timing_enter dirty_shutdown -run_test suite $testdir/dirty_shutdown.sh $device -timing_exit dirty_shutdown +if [ -n "$nv_cache" ]; then + timing_enter dirty_shutdown + run_test suite $testdir/dirty_shutdown.sh -c $nv_cache $device + timing_exit dirty_shutdown +fi timing_enter json run_test suite $testdir/json.sh $device diff --git a/test/ftl/restore.sh b/test/ftl/restore.sh index 16ec3db54c..839d04cefd 100755 --- a/test/ftl/restore.sh +++ b/test/ftl/restore.sh @@ -3,12 +3,23 @@ testdir=$(readlink -f $(dirname $0)) rootdir=$(readlink -f $testdir/../..) source $rootdir/test/common/autotest_common.sh +source $testdir/common.sh rpc_py=$rootdir/scripts/rpc.py mount_dir=$(mktemp -d) +pu_start=0 +pu_end=3 + +while getopts ':u:c:' opt; do + case $opt in + u) uuid=$OPTARG ;; + c) nv_cache=$OPTARG ;; + ?) echo "Usage: $0 [-u UUID] [-c NV_CACHE_PCI_BDF] OCSSD_PCI_BDF" && exit 1 ;; + esac +done +shift $((OPTIND -1)) device=$1 -uuid=$2 restore_kill() { if mount | grep $mount_dir; then @@ -30,12 +41,17 @@ $rootdir/app/spdk_tgt/spdk_tgt & svcpid=$! # Wait until spdk_tgt starts waitforlisten $svcpid -if [ -n "$uuid" ]; then - $rpc_py construct_ftl_bdev -b nvme0 -a $device -l 0-3 -u $uuid -else - $rpc_py construct_ftl_bdev -b nvme0 -a $device -l 0-3 +if [ -n "$nv_cache" ]; then + nvc_bdev=$(create_nv_cache_bdev nvc0 $device $nv_cache $(($pu_end - $pu_start + 1))) fi +ftl_construct_args="construct_ftl_bdev -b nvme0 -a $device -l ${pu_start}-${pu_end}" + +[ -n "$uuid" ] && ftl_construct_args+=" -u $uuid" +[ -n "$nv_cache" ] && ftl_construct_args+=" -c $nvc_bdev" + +$rpc_py $ftl_construct_args + # Load the nbd driver modprobe nbd $rpc_py start_nbd_disk nvme0 /dev/nbd0 @@ -55,7 +71,7 @@ md5sum $mount_dir/testfile > $testdir/testfile.md5 umount $mount_dir killprocess $svcpid -$rootdir/app/spdk_tgt/spdk_tgt & svcpid=$! +$rootdir/app/spdk_tgt/spdk_tgt -L ftl_init & svcpid=$! # Wait until spdk_tgt starts waitforlisten $svcpid