test/vhost: fix vhost migration tests

Bug introduced in:
9fc706f test/vhost: improve run_fio --local option

The original "nohup + no-wait" for spawned local VM fio
processes was probably as expected. Turns out we can't
"wait" for fio processes in some cases, and migration
is one of them.

run_fio() must start fio and exit immediately, so that
parent script can have a chance to do the migration
process while fio is still running.

Signed-off-by: Karol Latecki <karol.latecki@intel.com>
Change-Id: I3c0ab65e96bfa0b020eb42df76b14deda2b43e83
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/1937
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Seth Howell <seth.howell@intel.com>
Reviewed-by: Maciej Wawryk <maciejx.wawryk@intel.com>
Reviewed-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>

test/vhost/common.sh

@@ -1070,6 +1070,7 @@ function run_fio()
 	local run_plugin_mode=false
 	local fio_start_cmd
 	local fio_output_format="normal"
+	local wait_for_fio=true
 
 	for arg in "$@"; do
 		case "$arg" in
@@ -1087,6 +1088,7 @@ function run_fio()
 				run_server_mode=false ;;
 			--json) fio_output_format="json" ;;
 			--hide-results) hide_results=true ;;
+			--no-wait-for-fio) wait_for_fio=false ;;
 		*)
 			error "Invalid argument '$arg'"
 			return 1
@@ -1134,12 +1136,15 @@ function run_fio()
 			fi
 
 			notice "Running local fio on VM $vm_num"
-			vm_exec $vm_num "$vm_fio_bin --output=/root/$log_fname --output-format=$fio_output_format /root/$job_fname" &
+			vm_exec $vm_num "$vm_fio_bin --output=/root/$log_fname --output-format=$fio_output_format /root/$job_fname & echo \$! > /root/fio.pid" &
 			vm_exec_pids+=("$!")
 		fi
 	done
 
 	if ! $run_server_mode; then
+		if ! $wait_for_fio; then
+			return 0
+		fi
 		echo "Waiting for guest fio instances to finish.."
 		wait "${vm_exec_pids[@]}"
 

test/vhost/migration/migration-tc1.sh

@@ -55,6 +55,8 @@ function migration_tc1()
 	# incoming VM - the one we want to migrate
 	# targe VM - the one which will accept migration
 	local job_file="$testdir/migration-tc1.job"
+	local log_file
+	log_file="/root/$(basename ${job_file%%.*}).log"
 
 	# Run vhost
 	vhost_run 0
@@ -74,14 +76,14 @@ function migration_tc1()
 	notice "Starting FIO"
 
 	vm_check_scsi_location $incoming_vm
-	run_fio $fio_bin --job-file="$job_file" --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)"
+	run_fio $fio_bin --job-file="$job_file" --no-wait-for-fio --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)"
 
 	# Wait a while to let the FIO time to issue some IO
 	sleep 5
 
 	# Check if fio is still running before migration
 	if ! is_fio_running $incoming_vm; then
-		vm_exec $incoming_vm "cat /root/$(basename ${job_file}).out"
+		vm_exec $incoming_vm "cat $log_file"
 		error "FIO is not running before migration: process crashed or finished too early"
 	fi
 
@@ -90,7 +92,7 @@ function migration_tc1()
 
 	# Check if fio is still running after migration
 	if ! is_fio_running $target_vm; then
-		vm_exec $target_vm "cat /root/$(basename ${job_file}).out"
+		vm_exec $target_vm "cat $log_file"
 		error "FIO is not running after migration: process crashed or finished too early"
 	fi
 
@@ -105,7 +107,7 @@ function migration_tc1()
 	done
 
 	notice "Fio result is:"
-	vm_exec $target_vm "cat /root/$(basename ${job_file}).out"
+	vm_exec $target_vm "cat $log_file"
 
 	notice "Migration DONE"
 

test/vhost/migration/migration-tc2.sh

@@ -162,20 +162,22 @@ function migration_tc2()
 	# incoming VM - the one we want to migrate
 	# targe VM - the one which will accept migration
 	local job_file="$testdir/migration-tc2.job"
+	local log_file
+	log_file="/root/$(basename ${job_file%%.*}).log"
 
 	migration_tc2_configure_vhost
 
 	# Run fio before migration
 	notice "Starting FIO"
 	vm_check_scsi_location $incoming_vm
-	run_fio $fio_bin --job-file="$job_file" --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)"
+	run_fio $fio_bin --job-file="$job_file" --no-wait-for-fio --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)"
 
 	# Wait a while to let the FIO time to issue some IO
 	sleep 5
 
 	# Check if fio is still running before migration
 	if ! is_fio_running $incoming_vm; then
-		vm_exec $incoming_vm "cat /root/$(basename ${job_file}).out"
+		vm_exec $incoming_vm "cat $log_file"
 		error "FIO is not running before migration: process crashed or finished too early"
 	fi
 
@@ -184,7 +186,7 @@ function migration_tc2()
 
 	# Check if fio is still running after migration
 	if ! is_fio_running $target_vm; then
-		vm_exec $target_vm "cat /root/$(basename ${job_file}).out"
+		vm_exec $target_vm "cat $log_file"
 		error "FIO is not running after migration: process crashed or finished too early"
 	fi
 
@@ -199,7 +201,7 @@ function migration_tc2()
 	done
 
 	notice "Fio result is:"
-	vm_exec $target_vm "cat /root/$(basename ${job_file}).out"
+	vm_exec $target_vm "cat $log_file"
 
 	migration_tc2_cleanup_vhost_config
 	notice "Migration TC2 SUCCESS"

test/vhost/migration/migration-tc3a.sh

@@ -8,6 +8,7 @@ target_vm_ctrlr=naa.VhostScsi0.$target_vm
 share_dir=$TEST_DIR/share
 spdk_repo_share_dir=$TEST_DIR/share_spdk
 job_file=$testdir/migration-tc3.job
+log_file="/root/$(basename ${job_file%%.*}).log"
 
 if [ -z "$MGMT_TARGET_IP" ]; then
 	error "No IP address of target is given"
@@ -206,11 +207,11 @@ function migration_tc3()
 	notice "Starting fio on local VM"
 	vm_check_scsi_location $incoming_vm
 
-	run_fio $fio_bin --job-file="$job_file" --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)"
+	run_fio $fio_bin --job-file="$job_file" --no-wait-for-fio --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)"
 	sleep 5
 
 	if ! is_fio_running $incoming_vm; then
-		vh_ssh $incoming_vm "cat /root/$(basename ${job_file}).out"
+		vm_exec $incoming_vm "cat $log_file"
 		error "Fio not running on local VM before starting migration!"
 	fi
 

test/vhost/migration/migration-tc3b.sh

@@ -58,7 +58,7 @@ if ! vm_os_booted $target_vm; then
 fi
 
 if ! is_fio_running $target_vm; then
-	vm_exec $target_vm "cat /root/migration-tc3.job.out"
+	vm_exec $target_vm "cat /root/migration-tc3.log"
 	error "FIO is not running on remote server after migration!"
 fi
 
@@ -73,7 +73,7 @@ while is_fio_running $target_vm; do
 done
 
 notice "FIO result after migration:"
-vm_exec $target_vm "cat /root/migration-tc3.job.out"
+vm_exec $target_vm "cat /root/migration-tc3.log"
 
 host_2_cleanup_vhost
 echo "DONE" > $share_dir/DONE