autotest: add test completion tracking

Change-Id: I1ca0578a010db2ff8535505bfd981cd1c368e403
Signed-off-by: Seth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/392240
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

autobuild.sh

@@ -39,7 +39,22 @@ make_timing_label='make'
 if [ $SPDK_RUN_SCANBUILD -eq 1 ] && hash scan-build; then
 	scanbuild="scan-build -o $out/scan-build-tmp --status-bugs"
 	make_timing_label='scanbuild_make'
+	report_test_completion "scanbuild"
+
 fi
+
+if [ $SPDK_RUN_VALGRIND -eq 1 ]; then
+	report_test_completion "valgrind"
+fi
+
+if [ $SPDK_RUN_ASAN -eq 1 ]; then
+	report_test_completion "asan"
+fi
+
+if [ $SPDK_RUN_UBSAN -eq 1 ]; then
+	report_test_completion "ubsan"
+fi
+
 echo $scanbuild
 $MAKE $MAKEFLAGS clean
 

autorun_post.py

@@ -63,9 +63,69 @@ def prepDocumentation(output_dir, repo_dir):
     shutil.move(docDir, os.path.join(output_dir, 'doc'))
 
 
+def aggregateCompletedTests(output_dir, repo_dir):
+    test_list = {}
+    test_with_asan = {}
+    test_with_ubsan = {}
+    asan_enabled = False
+    ubsan_enabled = False
+    test_unit_with_valgrind = False
+    testFilePath = os.path.join(output_dir, '**', 'all_tests.txt')
+    completionFilePath = os.path.join(output_dir, '**', 'test_completions.txt')
+    testFiles = glob.glob(testFilePath, recursive=True)
+    completionFiles = glob.glob(completionFilePath, recursive=True)
+
+    if len(testFiles) == 0:
+        print("Unable to perform test completion aggregator. No input files.")
+        return 0
+    for item in testFiles:
+        with open(item, 'r') as raw_test_list:
+            for line in raw_test_list:
+                test_list[line.strip()] = (False, False, False)
+    for item in completionFiles:
+        with open(item, 'r') as completion_list:
+            completions = completion_list.read()
+
+            if "asan" not in completions:
+                asan_enabled = False
+            else:
+                asan_enabled = True
+
+            if "ubsan" not in completions:
+                ubsan_enabled = False
+            else:
+                ubsan_enabled = True
+
+            if "valgrind" in completions and "unittest" in completions:
+                test_unit_with_valgrind = True
+            for line in completions.split('\n'):
+                try:
+                    test_list[line.strip()] = (True, asan_enabled | test_list[line.strip()][1], ubsan_enabled | test_list[line.strip()][1])
+                except KeyError:
+                    continue
+
+    print("\n\n-----Tests Missing From Build------")
+    if not test_unit_with_valgrind:
+        print("UNITTEST_WITH_VALGRIND\n")
+    for item in sorted(test_list):
+        if test_list[item][0] is False:
+            print(item)
+
+    print("\n\n-----Tests Missing ASAN------")
+    for item in sorted(test_list):
+        if test_list[item][1] is False:
+            print(item)
+
+    print("\n\n-----Tests Missing UBSAN------")
+    for item in sorted(test_list):
+        if test_list[item][2] is False:
+            print(item)
+
+
 def main(output_dir, repo_dir):
     generateCoverageReport(output_dir, repo_dir)
     prepDocumentation(output_dir, repo_dir)
+    aggregateCompletedTests(output_dir, repo_dir)
 
 
 if __name__ == "__main__":

autotest.sh

@@ -20,6 +20,8 @@ trap "process_core; $rootdir/scripts/setup.sh reset; exit 1" SIGINT SIGTERM EXIT
 
 timing_enter autotest
 
+create_test_list
+
 src=$(readlink -f $(dirname $0))
 out=$PWD
 cd $src
@@ -88,6 +90,7 @@ fi
 if [ $SPDK_TEST_UNITTEST -eq 1 ]; then
 	timing_enter unittest
 	run_test ./unittest.sh
+	report_test_completion "unittest"
 	timing_exit unittest
 fi
 
@@ -189,6 +192,7 @@ if [ $SPDK_TEST_LVOL -eq 1 ]; then
 	test_cases+="300,301,450,451,452,550,600,601,650,651,652,654,655,"
 	test_cases+="700,701,10000"
 	run_test ./test/lvol/lvol.sh --test-cases=$test_cases
+	report_test_completion "lvol"
 	timing_exit lvol
 fi
 

test/blobfs/rocksdb/rocksdb.sh

@@ -127,4 +127,5 @@ trap - SIGINT SIGTERM EXIT
 
 rm -f $ROCKSDB_CONF
 
+report_test_completion "blobfs"
 timing_exit rocksdb

test/common/autotest_common.sh

@@ -170,6 +170,14 @@ function timing_finish() {
 	fi
 }
 
+function create_test_list() {
+	grep -rsh --exclude="autotest_common.sh" --exclude="$rootdir/test/common/autotest_common.sh" -e "report_test_completion" $rootdir | sed 's/report_test_completion//g; s/[[:blank:]]//g; s/"//g;' > $output_dir/all_tests.txt || true
+}
+
+function report_test_completion() {
+	echo "$1" >> $output_dir/test_completions.txt
+}
+
 function process_core() {
 	ret=0
 	for core in $(find . -type f \( -name 'core*' -o -name '*.core' \)); do

test/iscsi_tgt/ext4test/ext4test.sh

@@ -123,4 +123,5 @@ trap - SIGINT SIGTERM EXIT
 rm -f $testdir/iscsi.conf
 iscsicleanup
 killprocess $pid
+report_test_completion "nightly_iscsi_ext4test"
 timing_exit ext4test

test/iscsi_tgt/ip_migration/ip_migration.sh

@@ -88,4 +88,5 @@ trap - SIGINT SIGTERM EXIT
 iscsicleanup
 
 $rpc_py -s $rpc_second_addr kill_instance SIGTERM
+report_test_completion "nightly_iscsi_ip_migration"
 timing_exit ip_migration

test/iscsi_tgt/nvme_remote/fio_remote_nvme.sh

@@ -80,4 +80,5 @@ rm -f $testdir/iscsi.conf.tmp
 $rpc_py delete_nvmf_subsystem nqn.2016-06.io.spdk:cnode1
 killprocess $nvmfpid
 
+report_test_completion "iscsi_nvme_remote"
 timing_exit nvme_remote

test/iscsi_tgt/pmem/iscsi_pmem.sh

@@ -80,4 +80,5 @@ trap - SIGINT SIGTERM EXIT
 rm -f ./local-job*
 rm -f /tmp/pool_file*
 killprocess $pid
+report_test_completion "nightly_iscsi_pmem"
 timing_exit iscsi_pmem

test/iscsi_tgt/rbd/rbd.sh

@@ -60,4 +60,5 @@ trap - SIGINT SIGTERM EXIT
 iscsicleanup
 killprocess $pid
 
+report_test_completion "iscsi_rbd"
 timing_exit rbd

test/lib/bdev/blockdev.sh

@@ -107,7 +107,7 @@ if [ -d /usr/src/fio ] && [ $SPDK_RUN_ASAN -eq 0 ]; then
 	rm -f *.state
 	rm -f $testdir/bdev.fio
 	timing_exit fio_trim
-
+	report_test_completion "bdev_fio"
 	timing_exit fio
 fi
 
@@ -129,6 +129,7 @@ if [ $RUN_NIGHTLY -eq 1 ]; then
 	timing_enter reset
 	#$testdir/bdevperf/bdevperf -c $testdir/bdev.conf -q 16 -w reset -s 4096 -t 60
 	timing_exit reset
+	report_test_completion "nightly_bdev_reset"
 fi
 
 
@@ -138,4 +139,5 @@ fi
 
 rm -f /tmp/aiofile
 rm -f $testdir/bdev.conf
+report_test_completion "bdev"
 timing_exit bdev

test/lib/env/env.sh

@@ -20,4 +20,5 @@ timing_enter pci
 $testdir/pci/pci_ut
 timing_exit pci
 
+report_test_completion "env"
 timing_exit env

test/lib/event/event.sh

@@ -8,4 +8,5 @@ timing_enter event
 $testdir/event_perf/event_perf -m 0xF -t 1
 $testdir/reactor/reactor -t 1
 $testdir/reactor_perf/reactor_perf -t 1
+report_test_completion "event"
 timing_exit event

test/lib/ioat/ioat.sh

@@ -23,7 +23,9 @@ if [ `uname` = Linux ]; then
 	$rootdir/examples/ioat/kperf/ioat_kperf -n 4 -q 4 -s 12 -t 32
 	rmmod dmaperf.ko
 	$rootdir/scripts/setup.sh
+	report_test_completion "ioat_kperf"
 	timing_exit kperf
 fi
 
+report_test_completion "ioat"
 timing_exit ioat

test/lib/nvme/hotplug.sh

@@ -136,6 +136,7 @@ kill -9 $qemupid
 rm "$qemu_pidfile"
 rm "$test_img"
 
+report_test_completion "nvme_hotplug"
 timing_exit hotplug_test
 
 timing_exit hotplug

test/lib/nvme/nvme.sh

@@ -99,6 +99,7 @@ if [ $RUN_NIGHTLY -eq 1 ]; then
 
 	timing_enter reset
 	$testdir/reset/reset -q 64 -w write -s 4096 -t 2
+	report_test_completion "nightly_nvme_reset"
 	timing_exit reset
 fi
 
@@ -114,6 +115,7 @@ $rootdir/examples/nvme/perf/perf -q 128 -w read -s 12288 -t 1 -LL -i 0
 if [ -b /dev/ram0 ]; then
 	# Test perf with AIO device
 	$rootdir/examples/nvme/perf/perf /dev/ram0 -q 128 -w read -s 12288 -t 1 -LL -i 0
+	report_test_completion "nvme_perf"
 fi
 timing_exit perf
 
@@ -154,6 +156,7 @@ if [ `uname` = Linux ]; then
 	$rootdir/examples/nvme/perf/perf -i 0 -q 16 -w read -s 4096 -t 3 -c 0x4
 	wait $pid0
 	wait $pid1
+	report_test_completion "nvme_multi_secondary"
 	timing_exit multi_secondary
 fi
 
@@ -169,6 +172,7 @@ if [ -d /usr/src/fio ]; then
 		# Only test when ASAN is not enabled. If ASAN is enabled, we cannot test.
 		if [ $SPDK_RUN_ASAN -eq 0 ]; then
 			LD_PRELOAD=$PLUGIN_DIR/fio_plugin /usr/src/fio/fio $PLUGIN_DIR/example_config.fio --filename="trtype=PCIe traddr=${bdf//:/.} ns=1"
+			report_test_completion "bdev_fio"
 		fi
 		break
 	done

test/nvmf/nvmf.sh

@@ -32,8 +32,10 @@ run_test test/nvmf/shutdown/shutdown.sh
 if [ $SPDK_TEST_NVML -eq 1 ]; then
 	if [ $RUN_NIGHTLY -eq 1 ]; then
 		run_test test/nvmf/pmem/nvmf_pmem.sh 30
+		report_test_completion "nightly_nvmf_pmem"
 	else
 		run_test test/nvmf/pmem/nvmf_pmem.sh 10
+		report_test_completion "nvmf_pmem"
 	fi
 fi
 
@@ -63,4 +65,6 @@ kill_stub
 run_test test/nvmf/rpc/rpc.sh
 run_test test/nvmf/fio/fio.sh
 revert_soft_roce
+
+report_test_completion "nvmf"
 timing_exit nvmf_tgt

test/pmem/pmem.sh

@@ -65,6 +65,7 @@ if [[ $EUID -ne 0 ]]; then
 fi
 
 source $TEST_DIR/test/pmem/common.sh
+source $TEST_DIR/test/common/autotest_common.sh
 
 #================================================
 # pmem_pool_info tests
@@ -694,4 +695,5 @@ if $test_delete_bdev || $test_all; then
 fi
 
 pmem_clean_pool_file
+report_test_completion "pmem"
 vhost_kill

test/vhost/initiator/blockdev.sh

@@ -95,6 +95,7 @@ timing_exit create_bdev_config
 timing_enter run_spdk_fio
 run_spdk_fio $BASE_DIR/bdev.fio --filename=$virtio_bdevs --section=job_randwrite --section=job_randrw \
 	--section=job_write --section=job_rw --spdk_conf=$BASE_DIR/bdev.conf
+report_test_completion "vhost_run_spdk_fio"
 timing_exit run_spdk_fio
 
 timing_enter run_spdk_fio_unmap

test/vhost/spdk_vhost.sh

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+rootdir=$(readlink -f $(dirname $0))/../..
+source "$rootdir/test/common/autotest_common.sh"
 
 set -e
 
@@ -61,6 +63,7 @@ case $1 in
 	-n|--negative)
 		echo 'Negative tests suite...'
 		$WORKDIR/other/negative.sh
+		report_test_completion "vhost_negative"
 		;;
 	-p|--performance)
 		echo 'Running performance suite...'
@@ -68,6 +71,7 @@ case $1 in
 		--vm=0,$VM_IMAGE,Nvme0n1p0 \
 		--test-type=spdk_vhost_scsi \
 		--fio-job=$WORKDIR/common/fio_jobs/default_performance.job
+		report_test_completion "vhost_perf"
 		;;
 	-pb|--performance-blk)
 		echo 'Running blk performance suite...'
@@ -75,6 +79,7 @@ case $1 in
 		--vm=0,$VM_IMAGE,Nvme0n1p0 \
 		--test-type=spdk_vhost_blk \
 		--fio-job=$WORKDIR/common/fio_jobs/default_performance.job
+		report_test_completion "vhost_perf_blk"
 		;;
 	-m|--migration)
 		echo 'Running migration suite...'
@@ -87,6 +92,7 @@ case $1 in
 		--vm=0,$VM_IMAGE,Nvme0n1p0:Nvme0n1p1:Nvme0n1p2:Nvme0n1p3 \
 		--test-type=spdk_vhost_scsi \
 		--fio-job=$WORKDIR/common/fio_jobs/default_integrity.job
+		report_test_completion "nightly_vhost_integrity"
 		;;
 	-ib|--integrity-blk)
 		echo 'Running blk integrity suite...'
@@ -94,24 +100,29 @@ case $1 in
 		--vm=0,$VM_IMAGE,Nvme0n1p0:Nvme0n1p1:Nvme0n1p2:Nvme0n1p3 \
 		--test-type=spdk_vhost_blk \
 		--fio-job=$WORKDIR/common/fio_jobs/default_integrity.job
+		report_test_completion "nightly_vhost_integrity_blk"
 		;;
 	-fs|--fs-integrity-scsi)
 		echo 'Running filesystem integrity suite...'
 		$WORKDIR/integrity/integrity_start.sh -i $VM_IMAGE -m scsi -f "xfs ntfs btrfs ext4"
+		report_test_completion "vhost_fs_integrity_scsi"
 		;;
 	-fb|--fs-integrity-blk)
 		echo 'Running filesystem integrity suite...'
 		$WORKDIR/integrity/integrity_start.sh -i $VM_IMAGE -m blk -f "xfs ntfs btrfs ext4"
+		report_test_completion "vhost_fs_integrity_blk"
 		;;
 	-ils|--integrity-lvol-scsi)
 		echo 'Running lvol integrity suite...'
 		$WORKDIR/lvol/lvol_test.sh -x --fio-bin=$FIO_BIN \
 		--ctrl-type=spdk_vhost_scsi --thin-provisioning
+		report_test_completion "vhost_integrity_lvol_scsi"
 		;;
 	-ilb|--integrity-lvol-blk)
 		echo 'Running lvol integrity suite...'
 		$WORKDIR/lvol/lvol_test.sh -x --fio-bin=$FIO_BIN \
 		--ctrl-type=spdk_vhost_blk
+		report_test_completion "vhost_integrity_lvol_blk"
 		;;
 	-ilsn|--integrity-lvol-scsi-nightly)
 		if [[ $DISKS_NUMBER -ge 2 ]]; then
@@ -160,6 +171,7 @@ case $1 in
 			--vm=3,$VM_IMAGE,Nvme0n1p6:Nvme0n1p7 \
 			--test-type=spdk_vhost_scsi \
 			--fio-jobs=$WORKDIR/hotplug/fio_jobs/default_integrity.job -x
+		report_test_completion "vhost_hotplug"
 		;;
 	-shr|--scsi-hot-remove)
 		echo 'Running scsi hotremove tests suite...'
@@ -173,6 +185,7 @@ case $1 in
 	-ro|--readonly)
 		echo 'Running readonly tests suite...'
 		$WORKDIR/readonly/readonly.sh --vm_image=$VM_IMAGE --disk=Nvme0n1
+		report_test_completion "vhost_readonly"
 		;;
 	*)
 		echo "unknown test type: $1"