diff --git a/autotest.sh b/autotest.sh index 76063015e2..1b2f2f6cb6 100755 --- a/autotest.sh +++ b/autotest.sh @@ -195,6 +195,7 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then run_test "bdevperf_config" test/bdev/bdevperf/test_config.sh if [[ $(uname -s) == Linux ]]; then run_test "spdk_dd" test/dd/dd.sh + run_test "reactor_set_interrupt" test/interrupt/reactor_set_interrupt.sh fi fi diff --git a/test/interrupt/interrupt_common.sh b/test/interrupt/interrupt_common.sh new file mode 100644 index 0000000000..b6402d8ac2 --- /dev/null +++ b/test/interrupt/interrupt_common.sh @@ -0,0 +1,98 @@ +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../..) +source $rootdir/test/common/autotest_common.sh + +rpc_py="$rootdir/scripts/rpc.py" + +r0_mask=0x1 +r1_mask=0x2 +r2_mask=0x4 + +cpu_server_mask=0x07 +rpc_server_addr="/var/tmp/spdk.sock" + +function cleanup() { + rm -f "$SPDK_TEST_STORAGE/aiofile" +} + +function start_intr_tgt() { + local rpc_addr="${1:-$rpc_server_addr}" + local cpu_mask="${2:-$cpu_server_mask}" + + "$SPDK_EXAMPLE_DIR/interrupt_tgt" -m $cpu_mask -r $rpc_addr -E -g & + intr_tgt_pid=$! + trap 'killprocess "$intr_tgt_pid"; cleanup; exit 1' SIGINT SIGTERM EXIT + waitforlisten "$intr_tgt_pid" $rpc_addr +} + +function reactor_is_busy_or_idle() { + local pid=$1 + local idx=$2 + local state=$3 + + if [[ $state != "busy" ]] && [[ $state != "idle" ]]; then + return 1 + fi + + if ! hash top; then + # Fail this test if top is missing from system. + return 1 + fi + + for ((j = 10; j != 0; j--)); do + top_reactor=$(top -bHn 1 -p $pid -w 256 | grep reactor_$idx) + cpu_rate=$(echo $top_reactor | sed -e 's/^\s*//g' | awk '{print $9}') + cpu_rate=${cpu_rate%.*} + + if [[ $state = "busy" ]] && [[ $cpu_rate -lt 70 ]]; then + sleep 1 + elif [[ $state = "idle" ]] && [[ $cpu_rate -gt 30 ]]; then + sleep 1 + else + return 0 + fi + done + + if [[ $state = "busy" ]]; then + echo "cpu rate ${cpu_rate} of reactor $i probably is not busy polling" + else + echo "cpu rate ${cpu_rate} of reactor $i probably is not idle interrupt" + fi + + return 1 +} + +function reactor_is_busy() { + reactor_is_busy_or_idle $1 $2 "busy" +} + +function reactor_is_idle() { + reactor_is_busy_or_idle $1 $2 "idle" +} + +function reactor_get_thread_ids() { + local reactor_cpumask=$1 + local grep_str + + reactor_cpumask=$((reactor_cpumask)) + jq_str='.threads|.[]|select(.cpumask == $reactor_cpumask)|.id' + + # shellcheck disable=SC2005 + echo "$($rpc_py thread_get_stats | jq --arg reactor_cpumask "$reactor_cpumask" "$jq_str")" + +} + +function setup_bdev_mem() { + "$rpc_py" <<- RPC + bdev_malloc_create -b Malloc0 32 512 + bdev_malloc_create -b Malloc1 32 512 + bdev_malloc_create -b Malloc2 32 512 + RPC +} + +function setup_bdev_aio() { + if [[ $(uname -s) != "FreeBSD" ]]; then + dd if=/dev/zero of="$SPDK_TEST_STORAGE/aiofile" bs=2048 count=5000 + "$rpc_py" bdev_aio_create "$SPDK_TEST_STORAGE/aiofile" AIO0 2048 + fi +} diff --git a/test/interrupt/reactor_set_interrupt.sh b/test/interrupt/reactor_set_interrupt.sh new file mode 100755 index 0000000000..e1a4f88929 --- /dev/null +++ b/test/interrupt/reactor_set_interrupt.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../..) +source $rootdir/test/common/autotest_common.sh +source $testdir/interrupt_common.sh + +export PYTHONPATH=$rootdir/examples/interrupt_tgt + +function reactor_set_intr_mode() { + local spdk_pid=$1 + local without_thd=$2 + + thd0_ids=($(reactor_get_thread_ids $r0_mask)) + thd2_ids=($(reactor_get_thread_ids $r2_mask)) + + # Nubmer of thd0_ids shouldn't be zero + if [[ ${#thd0_ids[*]} -eq 0 ]]; then + echo "spdk_thread is expected in reactor 0." + return 1 + else + echo "spdk_thread ids are ${thd0_ids[*]} on reactor0." + fi + + # CPU utilization of reactor 0~2 should be idle + for i in {0..2}; do + reactor_is_idle $spdk_pid $i + done + + if [ "$without_thd"x = x ]; then + # Schedule all spdk_threads to reactor 1 + for i in ${thd0_ids[*]}; do + $rpc_py thread_set_cpumask -i $i -m $r1_mask + done + for i in ${thd2_ids[*]}; do + $rpc_py thread_set_cpumask -i $i -m $r1_mask + done + fi + # Set reactor 0 and 2 to be poll mode + $rpc_py --plugin interrupt_plugin reactor_set_interrupt_mode 0 -d + $rpc_py --plugin interrupt_plugin reactor_set_interrupt_mode 2 -d + # CPU utilization of reactor 0 and 2 should be busy + for i in 0 2; do + reactor_is_busy $spdk_pid $i + done + + # Set reactor 2 back to intr mode + $rpc_py --plugin interrupt_plugin reactor_set_interrupt_mode 2 + if [ "$without_thd"x = x ]; then + # Schedule spdk_threads in thd2_ids back to reactor 2 + for i in ${thd2_ids[*]}; do + $rpc_py thread_set_cpumask -i $i -m $r2_mask + done + fi + # CPU utilization of reactor 2 should be idle + reactor_is_idle $spdk_pid 2 + + # Set reactor 0 back to intr mode + $rpc_py --plugin interrupt_plugin reactor_set_interrupt_mode 0 + if [ "$without_thd"x = x ]; then + # Schedule spdk_threads in thd2_ids back to reactor 0 + for i in ${thd0_ids[*]}; do + $rpc_py thread_set_cpumask -i $i -m $r0_mask + done + fi + # CPU utilization of reactor 0 should be idle + reactor_is_idle $spdk_pid 0 + + return 0 +} + +function reactor_set_mode_without_threads() { + reactor_set_intr_mode $1 "without_thd" + return 0 +} + +function reactor_set_mode_with_threads() { + reactor_set_intr_mode $1 + return 0 +} + +# Set reactors with intr_tgt without spdk_thread +start_intr_tgt +setup_bdev_mem +setup_bdev_aio + +reactor_set_mode_without_threads $intr_tgt_pid + +trap - SIGINT SIGTERM EXIT +killprocess $intr_tgt_pid +cleanup + +# Set reactors with intr_tgt with spdk_thread +start_intr_tgt +setup_bdev_mem +setup_bdev_aio + +reactor_set_mode_with_threads $intr_tgt_pid + +trap - SIGINT SIGTERM EXIT +killprocess $intr_tgt_pid +cleanup