From 5470a0584f4dc84d8e69c81de041b7ef4e0d46f2 Mon Sep 17 00:00:00 2001 From: Seth Howell Date: Fri, 24 May 2019 15:58:19 -0700 Subject: [PATCH] scripts/fio.py: add argparse for command line parameters This script was getting a bit unwieldy. adding argparse will make it easier to add more options in the future or to set default values and validate arguments going forward. Change-Id: I1ffdbdf2082287ceb8a88cd3eb6ecf9bbd6c9e11 Signed-off-by: Seth Howell Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/455724 Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Shuhei Matsumoto Reviewed-by: Darek Stojaczyk --- scripts/fio.py | 41 ++++++++----------- test/iscsi_tgt/digests/digests.sh | 4 +- test/iscsi_tgt/fio/fio.sh | 12 +++--- test/iscsi_tgt/ip_migration/ip_migration.sh | 2 +- test/iscsi_tgt/lvol/iscsi_lvol.sh | 2 +- .../multiconnection/multiconnection.sh | 4 +- test/iscsi_tgt/nvme_remote/fio_remote_nvme.sh | 4 +- test/iscsi_tgt/pmem/iscsi_pmem.sh | 2 +- test/iscsi_tgt/qos/qos.sh | 2 +- test/iscsi_tgt/rbd/rbd.sh | 4 +- test/iscsi_tgt/reset/reset.sh | 2 +- test/iscsi_tgt/trace_record/trace_record.sh | 2 +- test/nvmf/target/fio.sh | 10 ++--- test/nvmf/target/multiconnection.sh | 4 +- test/nvmf/target/nmic.sh | 2 +- test/nvmf/target/srq_overwhelm.sh | 2 +- 16 files changed, 47 insertions(+), 52 deletions(-) diff --git a/scripts/fio.py b/scripts/fio.py index fc10031fe3..56816436a5 100755 --- a/scripts/fio.py +++ b/scripts/fio.py @@ -6,6 +6,7 @@ import sys import signal import os.path import time +import argparse fio_template = """ [global] @@ -44,31 +45,12 @@ def interrupt_handler(signum, frame): sys.exit(0) -def main(): - +def main(io_size, protocol, queue_depth, test_type, runtime, num_jobs, verify): global fio - if (len(sys.argv) < 7): - print("usage:") - print(" " + sys.argv[0] + " ") - print("advanced usage:") - print("If you want to run fio with verify, please add verify string after runtime.") - print("Currently fio.py only support write rw randwrite randrw with verify enabled.") - sys.exit(1) - app = str(sys.argv[1]) - io_size = int(sys.argv[2]) - queue_depth = int(sys.argv[3]) - test_type = sys.argv[4] - runtime = sys.argv[5] - num_jobs = sys.argv[6] - - verify = False - if len(sys.argv) > 7: - verify = True - - if app == "nvmf": + if protocol == "nvmf": devices = get_nvmf_target_devices() - elif app == "iscsi": + elif protocol == "iscsi": devices = get_iscsi_target_devices() configure_devices(devices) @@ -166,4 +148,17 @@ def configure_devices(devices): if __name__ == "__main__": - main() + parser = argparse.ArgumentParser(description="fio.py") + parser.add_argument("-i", "--io-size", type=int, help="The desired I/O size in bytes.", required=True) + parser.add_argument("-p", "--protocol", type=str, help="The protocol we are testing against. One of iscsi or nvmf.", required=True) + parser.add_argument("-d", "--queue-depth", type=int, help="The desired queue depth for each job.", required=True) + parser.add_argument("-t", "--test-type", type=str, help="The fio I/O pattern to run. e.g. read, randwrite, randrw.", required=True) + parser.add_argument("-r", "--runtime", type=int, help="Time in seconds to run the workload.", required=True) + parser.add_argument("-n", "--num-jobs", type=int, help="The number of fio jobs to run in your workload. default 1.", default=1) + parser.add_argument("-v", "--verify", action="store_true", help="Supply this argument to verify the I/O.", default=False) + args = parser.parse_args() + + if args.protocol.lower() != "nvmf" and args.protocol.lower() != "iscsi": + parser.error("Protocol must be one of the following: nvmf, iscsi.") + + main(args.io_size, args.protocol, args.queue_depth, args.test_type, args.runtime, args.num_jobs, args.verify) diff --git a/test/iscsi_tgt/digests/digests.sh b/test/iscsi_tgt/digests/digests.sh index 63154f70ae..40422269ef 100755 --- a/test/iscsi_tgt/digests/digests.sh +++ b/test/iscsi_tgt/digests/digests.sh @@ -14,8 +14,8 @@ function node_login_fio_logout() { iscsiadm -m node -p $TARGET_IP:$ISCSI_PORT -o update -n node.conn[0].iscsi.$arg done iscsiadm -m node --login -p $TARGET_IP:$ISCSI_PORT - $fio_py iscsi 512 1 write 2 1 - $fio_py iscsi 512 1 read 2 1 + $fio_py -p iscsi -i 512 -d 1 -t write -r 2 + $fio_py -p iscsi -i 512 -d 1 -t read -r 2 iscsiadm -m node --logout -p $TARGET_IP:$ISCSI_PORT sleep 1 } diff --git a/test/iscsi_tgt/fio/fio.sh b/test/iscsi_tgt/fio/fio.sh index 00f26c0b57..87e4040383 100755 --- a/test/iscsi_tgt/fio/fio.sh +++ b/test/iscsi_tgt/fio/fio.sh @@ -40,7 +40,7 @@ function running_config() { timing_exit start_iscsi_tgt2 sleep 1 - $fio_py iscsi 4096 1 randrw 5 1 + $fio_py -p iscsi -i 4096 -d 1 -t randrw -r 5 } if [ -z "$TARGET_IP" ]; then @@ -95,12 +95,12 @@ iscsiadm -m node --login -p $TARGET_IP:$ISCSI_PORT trap "iscsicleanup; killprocess $pid; iscsitestfini $1 $2; delete_tmp_files; exit 1" SIGINT SIGTERM EXIT -$fio_py iscsi 4096 1 randrw 1 1 verify -$fio_py iscsi 131072 32 randrw 1 1 verify -$fio_py iscsi 524288 128 randrw 1 1 verify +$fio_py -p iscsi -i 4096 -d 1 -t randrw -r 1 -v +$fio_py -p iscsi -i 131072 -d 32 -t randrw -r 1 -v +$fio_py -p iscsi -i 524288 -d 128 -t randrw -r 1 -v if [ $RUN_NIGHTLY -eq 1 ]; then - $fio_py iscsi 4096 1 write 300 1 verify + $fio_py -p iscsi -i 4096 -d 1 -t write -r 300 -v # Run the running_config test which will generate a config file from the # running iSCSI target, then kill and restart the iSCSI target using the @@ -110,7 +110,7 @@ if [ $RUN_NIGHTLY -eq 1 ]; then fi # Start hotplug test case. -$fio_py iscsi 1048576 128 rw 10 1 & +$fio_py -p iscsi -i 1048576 -d 128 -t rw -r 10 & fio_pid=$! sleep 3 diff --git a/test/iscsi_tgt/ip_migration/ip_migration.sh b/test/iscsi_tgt/ip_migration/ip_migration.sh index 28b9e7d8ab..cb6a3178a9 100755 --- a/test/iscsi_tgt/ip_migration/ip_migration.sh +++ b/test/iscsi_tgt/ip_migration/ip_migration.sh @@ -74,7 +74,7 @@ sleep 1 iscsiadm -m node --login -p $MIGRATION_ADDRESS:$ISCSI_PORT # fio tests for multi-process -$fio_py iscsi 4096 32 randrw 10 1 & +$fio_py -p iscsi -i 4096 -d 32 -t randrw -r 10 & fiopid=$! sleep 5 diff --git a/test/iscsi_tgt/lvol/iscsi_lvol.sh b/test/iscsi_tgt/lvol/iscsi_lvol.sh index e7892f17dd..e8e1d3be15 100755 --- a/test/iscsi_tgt/lvol/iscsi_lvol.sh +++ b/test/iscsi_tgt/lvol/iscsi_lvol.sh @@ -74,7 +74,7 @@ iscsiadm -m node --login -p $TARGET_IP:$ISCSI_PORT timing_exit discovery timing_enter fio -$fio_py iscsi 131072 8 randwrite 10 1 verify +$fio_py -p iscsi -i 131072 -d 8 -t randwrite -r 10 -v timing_exit fio rm -f ./local-job0-0-verify.state diff --git a/test/iscsi_tgt/multiconnection/multiconnection.sh b/test/iscsi_tgt/multiconnection/multiconnection.sh index 4b49f87350..53531ee6b0 100755 --- a/test/iscsi_tgt/multiconnection/multiconnection.sh +++ b/test/iscsi_tgt/multiconnection/multiconnection.sh @@ -73,8 +73,8 @@ iscsiadm -m discovery -t sendtargets -p $TARGET_IP:$ISCSI_PORT iscsiadm -m node --login -p $TARGET_IP:$ISCSI_PORT echo "Running FIO" -$fio_py iscsi 131072 64 randrw 5 1 -$fio_py iscsi 262144 16 randwrite 10 1 +$fio_py -p iscsi -i 131072 -d 64 -t randrw -r 5 +$fio_py -p iscsi -i 262144 -d 16 -t randwrite -r 10 sync trap - SIGINT SIGTERM EXIT diff --git a/test/iscsi_tgt/nvme_remote/fio_remote_nvme.sh b/test/iscsi_tgt/nvme_remote/fio_remote_nvme.sh index b43949bc69..7218977c1a 100755 --- a/test/iscsi_tgt/nvme_remote/fio_remote_nvme.sh +++ b/test/iscsi_tgt/nvme_remote/fio_remote_nvme.sh @@ -85,7 +85,7 @@ trap "iscsicleanup; killprocess $iscsipid; \ rm -f ./local-job0-0-verify.state; iscsitestfini $1 $2; nvmftestfini; exit 1" SIGINT SIGTERM EXIT echo "Running FIO" -$fio_py iscsi 4096 1 randrw 1 1 verify +$fio_py -p iscsi -i 4096 -d 1 -t randrw -r 1 -v rm -f ./local-job0-0-verify.state iscsicleanup @@ -94,7 +94,7 @@ killprocess $iscsipid run_nvme_remote "remote" echo "Running FIO" -$fio_py iscsi 4096 1 randrw 1 1 verify +$fio_py -p iscsi -i 4096 -d 1 -t randrw -r 1 -v rm -f ./local-job0-0-verify.state trap - SIGINT SIGTERM EXIT diff --git a/test/iscsi_tgt/pmem/iscsi_pmem.sh b/test/iscsi_tgt/pmem/iscsi_pmem.sh index de50c0f7c9..9ef7102f40 100755 --- a/test/iscsi_tgt/pmem/iscsi_pmem.sh +++ b/test/iscsi_tgt/pmem/iscsi_pmem.sh @@ -54,7 +54,7 @@ iscsiadm -m node --login -p $TARGET_IP:$ISCSI_PORT timing_exit discovery timing_enter fio_test -$fio_py iscsi $BLOCKSIZE 64 randwrite $RUNTIME 1 verify +$fio_py -p iscsi -i $BLOCKSIZE -d 64 -t randwrite -r $RUNTIME -v timing_exit fio_test iscsicleanup diff --git a/test/iscsi_tgt/qos/qos.sh b/test/iscsi_tgt/qos/qos.sh index ac7f1840f9..daca46c61e 100755 --- a/test/iscsi_tgt/qos/qos.sh +++ b/test/iscsi_tgt/qos/qos.sh @@ -17,7 +17,7 @@ function run_fio() { local start_io_count=$(jq -r '.bdevs[0].num_read_ops' <<< "$iostats") local start_bytes_read=$(jq -r '.bdevs[0].bytes_read' <<< "$iostats") - $fio_py iscsi 1024 128 randread $run_time 1 + $fio_py -p iscsi -i 1024 -d 128 -t randread -r $run_time iostats=$($rpc_py get_bdevs_iostat -b $bdev_name) local end_io_count=$(jq -r '.bdevs[0].num_read_ops' <<< "$iostats") diff --git a/test/iscsi_tgt/rbd/rbd.sh b/test/iscsi_tgt/rbd/rbd.sh index 571be6ca99..8eadae9714 100755 --- a/test/iscsi_tgt/rbd/rbd.sh +++ b/test/iscsi_tgt/rbd/rbd.sh @@ -54,8 +54,8 @@ iscsiadm -m node --login -p $TARGET_IP:$ISCSI_PORT trap "iscsicleanup; killprocess $pid; rbd_cleanup; exit 1" SIGINT SIGTERM EXIT -$fio_py iscsi 4096 1 randrw 1 1 verify -$fio_py iscsi 131072 32 randrw 1 1 verify +$fio_py -p iscsi -i 4096 -d 1 -t randrw -r 1 -v +$fio_py -p iscsi -i 131072 -d 32 -t randrw -r 1 -v rm -f ./local-job0-0-verify.state diff --git a/test/iscsi_tgt/reset/reset.sh b/test/iscsi_tgt/reset/reset.sh index 433b4a8050..89e1d74936 100755 --- a/test/iscsi_tgt/reset/reset.sh +++ b/test/iscsi_tgt/reset/reset.sh @@ -53,7 +53,7 @@ iscsiadm -m node --login -p $TARGET_IP:$ISCSI_PORT sleep 1 dev=$(iscsiadm -m session -P 3 | grep "Attached scsi disk" | awk '{print $4}') -$fio_py iscsi 512 1 read 60 1 & +$fio_py -p iscsi -i 512 -d 1 -t read -r 60 & fiopid=$! echo "FIO pid: $fiopid" diff --git a/test/iscsi_tgt/trace_record/trace_record.sh b/test/iscsi_tgt/trace_record/trace_record.sh index a3e853200f..f810f2ea4d 100755 --- a/test/iscsi_tgt/trace_record/trace_record.sh +++ b/test/iscsi_tgt/trace_record/trace_record.sh @@ -77,7 +77,7 @@ echo "Trace record pid: $record_pid" trap "iscsicleanup; killprocess $iscsi_pid; killprocess $record_pid; delete_tmp_files; iscsitestfini $1 $2; exit 1" SIGINT SIGTERM EXIT echo "Running FIO" -$fio_py iscsi 131072 32 randrw 1 1 +$fio_py -p iscsi -i 131072 -d 32 -t randrw -r 1 iscsicleanup diff --git a/test/nvmf/target/fio.sh b/test/nvmf/target/fio.sh index 1ef3f72f01..f58b32b070 100755 --- a/test/nvmf/target/fio.sh +++ b/test/nvmf/target/fio.sh @@ -40,15 +40,15 @@ waitforblk "nvme0n1" waitforblk "nvme0n2" waitforblk "nvme0n3" -$rootdir/scripts/fio.py nvmf 4096 1 write 1 1 verify -$rootdir/scripts/fio.py nvmf 4096 1 randwrite 1 1 verify -$rootdir/scripts/fio.py nvmf 4096 128 write 1 1 verify -$rootdir/scripts/fio.py nvmf 4096 128 randwrite 1 1 verify +$rootdir/scripts/fio.py -p nvmf -i 4096 -d 1 -t write -r 1 -v +$rootdir/scripts/fio.py -p nvmf -i 4096 -d 1 -t randwrite -r 1 -v +$rootdir/scripts/fio.py -p nvmf -i 4096 -d 128 -t write -r 1 -v +$rootdir/scripts/fio.py -p nvmf -i 4096 -d 128 -t randwrite -r 1 -v sync #start hotplug test case -$rootdir/scripts/fio.py nvmf 4096 1 read 10 1 & +$rootdir/scripts/fio.py -p nvmf -i 4096 -d 1 -t read -r 10 & fio_pid=$! sleep 3 diff --git a/test/nvmf/target/multiconnection.sh b/test/nvmf/target/multiconnection.sh index df16bbab24..d1997efae9 100755 --- a/test/nvmf/target/multiconnection.sh +++ b/test/nvmf/target/multiconnection.sh @@ -43,8 +43,8 @@ for i in `seq 1 $NVMF_SUBSYS`; do waitforblk "nvme${k}n1" done -$rootdir/scripts/fio.py nvmf 262144 64 read 10 1 -$rootdir/scripts/fio.py nvmf 262144 64 randwrite 10 1 +$rootdir/scripts/fio.py -p nvmf -i 262144 -d 64 -t read -r 10 +$rootdir/scripts/fio.py -p nvmf -i 262144 -d 64 -t randwrite -r 10 sync for i in `seq 1 $NVMF_SUBSYS`; do diff --git a/test/nvmf/target/nmic.sh b/test/nvmf/target/nmic.sh index 806979fefa..742e334eeb 100755 --- a/test/nvmf/target/nmic.sh +++ b/test/nvmf/target/nmic.sh @@ -52,7 +52,7 @@ if [ ! -z $NVMF_SECOND_TARGET_IP ]; then waitforblk "nvme0n1" - $rootdir/scripts/fio.py nvmf 4096 1 write 1 1 verify + $rootdir/scripts/fio.py -p nvmf -i 4096 -d 1 -t write -r 1 -v fi nvme disconnect -n "nqn.2016-06.io.spdk:cnode1" || true diff --git a/test/nvmf/target/srq_overwhelm.sh b/test/nvmf/target/srq_overwhelm.sh index 18e715c151..64d0825cca 100755 --- a/test/nvmf/target/srq_overwhelm.sh +++ b/test/nvmf/target/srq_overwhelm.sh @@ -39,7 +39,7 @@ done # working even at very high queue depths because the rdma qpair doesn't fail. # It is normal to see the initiator timeout and reconnect waiting for completions from an overwhelmmed target, # but the connection should come up and FIO should complete without errors. -$rootdir/scripts/fio.py nvmf 1048576 128 read 10 13 +$rootdir/scripts/fio.py -p nvmf -i 1048576 -d 128 -t read -r 10 -n 13 sync