test: add functional test for reactor_set_intr

test script 'test/interrupt/reactor_set_intr.sh' will
do various reactor set intr operations on interrupt_tgt
without spdk_thread and with spdk_thread.

Signed-off-by: Liu Xiaodong <xiaodong.liu@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/7348 (master)

(cherry picked from commit ac0c36d72a)
Change-Id: Ie5af1dc68b0272c34a91e8a66b78088c3794907c
Signed-off-by: Krzysztof Karas <krzysztof.karas@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/7678
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: <dongx.yi@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

autotest.sh

@@ -195,6 +195,7 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
 		run_test "bdevperf_config" test/bdev/bdevperf/test_config.sh
 		if [[ $(uname -s) == Linux ]]; then
 			run_test "spdk_dd" test/dd/dd.sh
+			run_test "reactor_set_interrupt" test/interrupt/reactor_set_interrupt.sh
 		fi
 	fi
 

test/interrupt/interrupt_common.sh

@@ -0,0 +1,98 @@
+testdir=$(readlink -f $(dirname $0))
+rootdir=$(readlink -f $testdir/../..)
+source $rootdir/test/common/autotest_common.sh
+
+rpc_py="$rootdir/scripts/rpc.py"
+
+r0_mask=0x1
+r1_mask=0x2
+r2_mask=0x4
+
+cpu_server_mask=0x07
+rpc_server_addr="/var/tmp/spdk.sock"
+
+function cleanup() {
+	rm -f "$SPDK_TEST_STORAGE/aiofile"
+}
+
+function start_intr_tgt() {
+	local rpc_addr="${1:-$rpc_server_addr}"
+	local cpu_mask="${2:-$cpu_server_mask}"
+
+	"$SPDK_EXAMPLE_DIR/interrupt_tgt" -m $cpu_mask -r $rpc_addr -E -g &
+	intr_tgt_pid=$!
+	trap 'killprocess "$intr_tgt_pid"; cleanup; exit 1' SIGINT SIGTERM EXIT
+	waitforlisten "$intr_tgt_pid" $rpc_addr
+}
+
+function reactor_is_busy_or_idle() {
+	local pid=$1
+	local idx=$2
+	local state=$3
+
+	if [[ $state != "busy" ]] && [[ $state != "idle" ]]; then
+		return 1
+	fi
+
+	if ! hash top; then
+		# Fail this test if top is missing from system.
+		return 1
+	fi
+
+	for ((j = 10; j != 0; j--)); do
+		top_reactor=$(top -bHn 1 -p $pid -w 256 | grep reactor_$idx)
+		cpu_rate=$(echo $top_reactor | sed -e 's/^\s*//g' | awk '{print $9}')
+		cpu_rate=${cpu_rate%.*}
+
+		if [[ $state = "busy" ]] && [[ $cpu_rate -lt 70 ]]; then
+			sleep 1
+		elif [[ $state = "idle" ]] && [[ $cpu_rate -gt 30 ]]; then
+			sleep 1
+		else
+			return 0
+		fi
+	done
+
+	if [[ $state = "busy" ]]; then
+		echo "cpu rate ${cpu_rate} of reactor $i probably is not busy polling"
+	else
+		echo "cpu rate ${cpu_rate} of reactor $i probably is not idle interrupt"
+	fi
+
+	return 1
+}
+
+function reactor_is_busy() {
+	reactor_is_busy_or_idle $1 $2 "busy"
+}
+
+function reactor_is_idle() {
+	reactor_is_busy_or_idle $1 $2 "idle"
+}
+
+function reactor_get_thread_ids() {
+	local reactor_cpumask=$1
+	local grep_str
+
+	reactor_cpumask=$((reactor_cpumask))
+	jq_str='.threads|.[]|select(.cpumask == $reactor_cpumask)|.id'
+
+	# shellcheck disable=SC2005
+	echo "$($rpc_py thread_get_stats | jq --arg reactor_cpumask "$reactor_cpumask" "$jq_str")"
+
+}
+
+function setup_bdev_mem() {
+	"$rpc_py" <<- RPC
+		bdev_malloc_create -b Malloc0 32 512
+		bdev_malloc_create -b Malloc1 32 512
+		bdev_malloc_create -b Malloc2 32 512
+	RPC
+}
+
+function setup_bdev_aio() {
+	if [[ $(uname -s) != "FreeBSD" ]]; then
+		dd if=/dev/zero of="$SPDK_TEST_STORAGE/aiofile" bs=2048 count=5000
+		"$rpc_py" bdev_aio_create "$SPDK_TEST_STORAGE/aiofile" AIO0 2048
+	fi
+}

test/interrupt/reactor_set_interrupt.sh

@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+
+testdir=$(readlink -f $(dirname $0))
+rootdir=$(readlink -f $testdir/../..)
+source $rootdir/test/common/autotest_common.sh
+source $testdir/interrupt_common.sh
+
+export PYTHONPATH=$rootdir/examples/interrupt_tgt
+
+function reactor_set_intr_mode() {
+	local spdk_pid=$1
+	local without_thd=$2
+
+	thd0_ids=($(reactor_get_thread_ids $r0_mask))
+	thd2_ids=($(reactor_get_thread_ids $r2_mask))
+
+	# Nubmer of thd0_ids shouldn't be zero
+	if [[ ${#thd0_ids[*]} -eq 0 ]]; then
+		echo "spdk_thread is expected in reactor 0."
+		return 1
+	else
+		echo "spdk_thread ids are ${thd0_ids[*]} on reactor0."
+	fi
+
+	# CPU utilization of reactor 0~2 should be idle
+	for i in {0..2}; do
+		reactor_is_idle $spdk_pid $i
+	done
+
+	if [ "$without_thd"x = x ]; then
+		# Schedule all spdk_threads to reactor 1
+		for i in ${thd0_ids[*]}; do
+			$rpc_py thread_set_cpumask -i $i -m $r1_mask
+		done
+		for i in ${thd2_ids[*]}; do
+			$rpc_py thread_set_cpumask -i $i -m $r1_mask
+		done
+	fi
+	# Set reactor 0 and 2 to be poll mode
+	$rpc_py --plugin interrupt_plugin reactor_set_interrupt_mode 0 -d
+	$rpc_py --plugin interrupt_plugin reactor_set_interrupt_mode 2 -d
+	# CPU utilization of reactor 0 and 2 should be busy
+	for i in 0 2; do
+		reactor_is_busy $spdk_pid $i
+	done
+
+	# Set reactor 2 back to intr mode
+	$rpc_py --plugin interrupt_plugin reactor_set_interrupt_mode 2
+	if [ "$without_thd"x = x ]; then
+		# Schedule spdk_threads in thd2_ids back to reactor 2
+		for i in ${thd2_ids[*]}; do
+			$rpc_py thread_set_cpumask -i $i -m $r2_mask
+		done
+	fi
+	# CPU utilization of reactor 2 should be idle
+	reactor_is_idle $spdk_pid 2
+
+	# Set reactor 0 back to intr mode
+	$rpc_py --plugin interrupt_plugin reactor_set_interrupt_mode 0
+	if [ "$without_thd"x = x ]; then
+		# Schedule spdk_threads in thd2_ids back to reactor 0
+		for i in ${thd0_ids[*]}; do
+			$rpc_py thread_set_cpumask -i $i -m $r0_mask
+		done
+	fi
+	# CPU utilization of reactor 0 should be idle
+	reactor_is_idle $spdk_pid 0
+
+	return 0
+}
+
+function reactor_set_mode_without_threads() {
+	reactor_set_intr_mode $1 "without_thd"
+	return 0
+}
+
+function reactor_set_mode_with_threads() {
+	reactor_set_intr_mode $1
+	return 0
+}
+
+# Set reactors with intr_tgt without spdk_thread
+start_intr_tgt
+setup_bdev_mem
+setup_bdev_aio
+
+reactor_set_mode_without_threads $intr_tgt_pid
+
+trap - SIGINT SIGTERM EXIT
+killprocess $intr_tgt_pid
+cleanup
+
+# Set reactors with intr_tgt with spdk_thread
+start_intr_tgt
+setup_bdev_mem
+setup_bdev_aio
+
+reactor_set_mode_with_threads $intr_tgt_pid
+
+trap - SIGINT SIGTERM EXIT
+killprocess $intr_tgt_pid
+cleanup