tests: Create HW hotplug test

This patch introduces new hotplug test which uses
PCIe interposer capable of physically connecting and
disconnecting NVMe drive from the system. It uses
custom build HW based on mircrocontroller to which
we connect and through which we send command to NVMe
interposer to connect or disconnect NVMe drive from
test machine. Scenario of this test is similar to the
scenario we are using for software hotplug test with VM.
Parameters for the test are IP address of the microcontroller
and GPIO pin to which the interposer is connected.

Signed-off-by: Maciej Szwed <maciej.szwed@intel.com>
Change-Id: I3610dadfd23521da2c90fd83e6895d942b3f66df
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/462470
Reviewed-by: Karol Latecki <karol.latecki@intel.com>
Reviewed-by: Paul Luse <paul.e.luse@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

test/nvme/hw_hotplug.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+
+testdir=$(readlink -f $(dirname $0))
+rootdir=$(readlink -f $testdir/../..)
+source $rootdir/test/common/autotest_common.sh
+
+function insert_device() {
+	ssh root@$ip 'Beetle --SetGpio "$gpio" HIGH'
+	waitforblk $name
+	DRIVER_OVERRIDE=$driver $rootdir/scripts/setup.sh
+}
+
+function remove_device() {
+	ssh root@$ip 'Beetle --SetGpio "$gpio" LOW'
+}
+
+ip=$1
+gpio=$2
+driver=$3
+declare -i io_time=5
+declare -i kernel_hotplug_time=7
+
+timing_enter hotplug_hw
+
+timing_enter hotplug_hw_cfg
+
+# Configure microcontroller
+ssh root@$ip 'Beetle --SetGpioDirection "$gpio" OUT'
+
+# Get blk dev name connected to interposer
+ssh root@$ip 'Beetle --SetGpio "$gpio" HIGH'
+sleep $kernel_hotplug_time
+$rootdir/scripts/setup.sh reset
+blk_list1=$(lsblk -d --output NAME | grep "^nvme")
+remove_device
+sleep $kernel_hotplug_time
+blk_list2=$(lsblk -d --output NAME | grep "^nvme") || true
+name=${blk_list1#"$blk_list2"}
+
+insert_device
+
+timing_exit hotplug_hw_cfg
+
+timing_enter hotplug_hw_test
+
+$rootdir/examples/nvme/hotplug/hotplug -i 0 -t 100 -n 2 -r 2 2>&1 | tee -a log.txt &
+example_pid=$!
+trap "killprocess $example_pid; exit 1" SIGINT SIGTERM EXIT
+
+i=0
+while ! grep "Starting I/O" log.txt; do
+	[ $i -lt 20 ] || break
+	i=$(($i+1))
+	sleep 1
+done
+
+if ! grep "Starting I/O" log.txt; then
+	return 1
+fi
+
+# Add and remove NVMe with delays between to give some time for IO to proceed
+remove_device
+sleep $io_time
+insert_device
+sleep $io_time
+remove_device
+sleep $io_time
+insert_device
+sleep $io_time
+
+timing_enter wait_for_example
+wait $example_pid
+timing_exit wait_for_example
+
+trap - SIGINT SIGTERM EXIT
+
+report_test_completion "nvme_hotplug_hw"
+timing_exit hotplug_hw_test
+
+timing_exit hotplug_hw