test/nvmf: Refactor hw detection routines

Merge detection functions into a single one where all supported
devices are picked up and exposed in the environment. Be verbose
about what's been found for given tests.

Signed-off-by: Michal Berger <michalx.berger@intel.com>
Change-Id: I780c5e480c301dbb92092074f927a731ed2680b1
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/7849
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Pawel Piatek <pawelx.piatek@intel.com>

autotest.sh

@@ -243,15 +243,9 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
 		# The NVMe-oF run test cases are split out like this so that the parser that compiles the
 		# list of all tests can properly differentiate them. Please do not merge them into one line.
 		if [ "$SPDK_TEST_NVMF_TRANSPORT" = "rdma" ]; then
-			timing_enter rdma_setup
-			rdma_device_init
-			timing_exit rdma_setup
 			run_test "nvmf_rdma" ./test/nvmf/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
 			run_test "spdkcli_nvmf_rdma" ./test/spdkcli/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
 		elif [ "$SPDK_TEST_NVMF_TRANSPORT" = "tcp" ]; then
-			timing_enter tcp_setup
-			tcp_device_init
-			timing_exit tcp_setup
 			run_test "nvmf_tcp" ./test/nvmf/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
 			if [[ $SPDK_TEST_URING -eq 0 ]]; then
 				run_test "spdkcli_nvmf_tcp" ./test/spdkcli/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT

test/common/autotest_common.sh

@@ -155,6 +155,8 @@ export SPDK_TEST_USE_IGB_UIO
 export SPDK_TEST_SCHEDULER
 : ${SPDK_TEST_SCANBUILD:=0}
 export SPDK_TEST_SCANBUILD
+: ${SPDK_TEST_NVMF_DRIVER:=}
+export SPDK_TEST_NVMF_DRIVER
 
 # always test with SPDK shared objects.
 export SPDK_LIB_DIR="$rootdir/build/lib"

test/json_config/json_config.sh

@@ -286,8 +286,7 @@ function create_nvmf_subsystem_config() {
 
 	NVMF_FIRST_TARGET_IP="127.0.0.1"
 	if [[ $SPDK_TEST_NVMF_TRANSPORT == "rdma" ]]; then
-		rdma_device_init
+		TEST_TRANSPORT=$SPDK_TEST_NVMF_TRANSPORT nvmftestinit
-		NVMF_FIRST_TARGET_IP=$(get_available_rdma_ips | head -n 1)
 	fi
 
 	if [[ -z $NVMF_FIRST_TARGET_IP ]]; then

test/nvmf/common.sh

@@ -26,6 +26,8 @@ function build_nvmf_app_args() {
 	fi
 }
 
+source "$rootdir/scripts/common.sh"
+
 : ${NVMF_APP_SHM_ID="0"}
 export NVMF_APP_SHM_ID
 build_nvmf_app_args
@@ -57,140 +59,6 @@ function detect_soft_roce_nics() {
 	rxe_cfg start
 }
 
-# args 1 and 2 represent the grep filters for finding our NICS.
-# subsequent args are all drivers that should be loaded if we find these NICs.
-# Those drivers should be supplied in the correct order.
-function detect_nics_and_probe_drivers() {
-	NIC_VENDOR="$1"
-	NIC_CLASS="$2"
-
-	nvmf_nic_bdfs=$(lspci | grep Ethernet | grep "$NIC_VENDOR" | grep "$NIC_CLASS" | awk -F ' ' '{print "0000:"$1}')
-
-	if [ -z "$nvmf_nic_bdfs" ]; then
-		return 0
-	fi
-
-	have_pci_nics=1
-	if [ $# -ge 2 ]; then
-		# shift out the first two positional arguments.
-		shift 2
-		# Iterate through the remaining arguments.
-		for i; do
-			if [[ $i == irdma ]]; then
-				# Our tests don't play well with iWARP protocol. Make sure we use RoCEv2 instead.
-				if [[ -e /sys/module/irdma/parameters/roce_ena ]]; then
-					# reload the module to re-init the rdma devices
-					(($(< /sys/module/irdma/parameters/roce_ena) != 1)) && modprobe -r irdma
-				fi
-				modprobe "$i" roce_ena=1
-			else
-				modprobe "$i"
-			fi
-		done
-	fi
-}
-
-function pci_rdma_switch() {
-	local driver=$1
-
-	local -a driver_args=()
-	driver_args+=("Mellanox ConnectX-4 mlx5_core mlx5_ib")
-	driver_args+=("Mellanox ConnectX-5 mlx5_core mlx5_ib")
-	driver_args+=("Intel E810 ice irdma")
-	driver_args+=("Intel X722 i40e i40iw")
-	driver_args+=("Chelsio \"Unified Wire\" cxgb4 iw_cxgb4")
-
-	case $driver in
-		mlx5_ib)
-			detect_nics_and_probe_drivers ${driver_args[0]}
-			detect_nics_and_probe_drivers ${driver_args[1]}
-			;;
-		irdma)
-			detect_nics_and_probe_drivers ${driver_args[2]}
-			;;
-		i40iw)
-			detect_nics_and_probe_drivers ${driver_args[3]}
-			;;
-		iw_cxgb4)
-			detect_nics_and_probe_drivers ${driver_args[4]}
-			;;
-		*)
-			for d in "${driver_args[@]}"; do
-				detect_nics_and_probe_drivers $d
-			done
-			;;
-	esac
-}
-
-function pci_tcp_switch() {
-	local driver=$1
-
-	local -a driver_args=()
-	driver_args+=("Intel E810 ice")
-
-	case $driver in
-		ice)
-			detect_nics_and_probe_drivers ${driver_args[0]}
-			;;
-		*)
-			for d in "${driver_args[@]}"; do
-				detect_nics_and_probe_drivers $d
-			done
-			;;
-	esac
-}
-
-function detect_pci_nics() {
-
-	if ! hash lspci; then
-		return 0
-	fi
-
-	local nic_drivers
-	local found_drivers
-
-	if [[ -z "$TEST_TRANSPORT" ]]; then
-		TEST_TRANSPORT=$SPDK_TEST_NVMF_TRANSPORT
-	fi
-
-	if [[ "$TEST_TRANSPORT" == "rdma" ]]; then
-		nic_drivers="mlx5_ib|irdma|i40iw|iw_cxgb4"
-
-		# Try to find RDMA drivers which are already loded and try to
-		# use only it's associated NICs, without probing all drivers.
-		found_drivers=$(lsmod | grep -Eo $nic_drivers | sort -u)
-		for d in $found_drivers; do
-			pci_rdma_switch $d
-		done
-
-		# In case lsmod reported driver, but lspci does not report
-		# physical NICs - fall back to old approach any try to
-		# probe all compatible NICs.
-		((have_pci_nics == 0)) && pci_rdma_switch "default"
-
-	elif [[ "$TEST_TRANSPORT" == "tcp" ]]; then
-		nic_drivers="ice"
-		found_drivers=$(lsmod | grep -Eo $nic_drivers | sort -u)
-		for d in $found_drivers; do
-			pci_tcp_switch $d
-		done
-		((have_pci_nics == 0)) && pci_tcp_switch "default"
-	fi
-
-	# Use softroce if everything else failed.
-	((have_pci_nics == 0)) && return 0
-
-	# Provide time for drivers to properly load.
-	sleep 5
-}
-
-function detect_transport_nics() {
-	detect_pci_nics
-	if [ "$have_pci_nics" -eq "0" ]; then
-		detect_soft_roce_nics
-	fi
-}
-
 function allocate_nic_ips() {
 	((count = NVMF_IP_LEAST_ADDR))
 	for nic_name in $(get_rdma_if_list); do
@@ -212,17 +80,25 @@ function get_available_rdma_ips() {
 }
 
 function get_rdma_if_list() {
-	rxe_cfg rxe-net
+	local net_dev rxe_net_dev rxe_net_devs
-}
 
-function get_tcp_if_list_by_driver() {
+	mapfile -t rxe_net_devs < <(rxe_cfg rxe-net)
-	local driver
-	driver=${1:-ice}
 
-	shopt -s nullglob
+	if ((${#net_devs[@]} == 0)); then
-	tcp_if_list=(/sys/bus/pci/drivers/$driver/0000*/net/*)
+		# No rdma-capable nics on board, using soft-RoCE
-	shopt -u nullglob
+		printf '%s\n' "${rxe_net_devs[@]}"
-	printf '%s\n' "${tcp_if_list[@]##*/}"
+		return 0
+	fi
+
+	# Pick only these devices which were found during gather_supported_nvmf_pci_devs() run
+	for net_dev in "${net_devs[@]}"; do
+		for rxe_net_dev in "${rxe_net_devs[@]}"; do
+			if [[ $net_dev == "$rxe_net_dev" ]]; then
+				echo "$net_dev"
+				continue 2
+			fi
+		done
+	done
 }
 
 function get_ip_address() {
@@ -345,8 +221,8 @@ function nvmf_veth_fini() {
 function nvmf_tcp_init() {
 	NVMF_INITIATOR_IP=10.0.0.1
 	NVMF_FIRST_TARGET_IP=10.0.0.2
-	TCP_INTERFACE_LIST=($(get_tcp_if_list_by_driver))
+	TCP_INTERFACE_LIST=("${net_devs[@]}")
-	if ((${#TCP_INTERFACE_LIST[@]} == 0)) || [ "$TEST_MODE" == "iso" ]; then
+	if ((${#TCP_INTERFACE_LIST[@]} == 0)); then
 		nvmf_veth_init
 		return 0
 	fi
@@ -400,6 +276,103 @@ function nvmf_tcp_fini() {
 	ip -4 addr flush $NVMF_INITIATOR_INTERFACE || :
 }
 
+function gather_supported_nvmf_pci_devs() {
+	# Go through the entire pci bus and gather all ethernet controllers we support for the nvmf tests.
+	# Focus on the hardware that's currently being tested by the CI.
+	xtrace_disable
+	cache_pci_bus_sysfs
+	xtrace_restore
+
+	local intel=0x8086 mellanox=0x15b3 pci
+
+	local -a pci_devs=()
+	local -a pci_net_devs=()
+	local -A pci_drivers=()
+
+	local -ga net_devs=()
+	local -ga e810=()
+	local -ga x722=()
+	local -ga mlx=()
+
+	# E810-XXV
+	e810+=(${pci_bus_cache["$intel:0x1592"]})
+	e810+=(${pci_bus_cache["$intel:0x159b"]})
+	# X722 10G
+	x722+=(${pci_bus_cache["$intel:0x37d2"]})
+	# ConnectX-5
+	mlx+=(${pci_bus_cache["$mellanox:0x1017"]})
+	# ConnectX-4
+	mlx+=(${pci_bus_cache["$mellanox:0x1015"]})
+	mlx+=(${pci_bus_cache["$mellanox:0x1013"]})
+
+	pci_devs+=("${e810[@]}")
+	if [[ $TEST_TRANSPORT == rdma ]]; then
+		pci_devs+=("${x722[@]}")
+		pci_devs+=("${mlx[@]}")
+	fi
+
+	# Try to respect what CI wants to test and override pci_devs[]
+	if [[ $SPDK_TEST_NVMF_DRIVER == mlx5_ib ]]; then
+		pci_devs=("${mlx[@]}")
+	elif [[ $SPDK_TEST_NVMF_DRIVER == ice ]]; then
+		pci_devs=("${e810[@]}")
+	fi
+
+	if ((${#pci_devs[@]} == 0)); then
+		if [[ $TEST_TRANSPORT == rdma ]]; then
+			echo "WARNING: No pci devices found for the $TEST_TRANSPORT test, falling back to Soft-RoCE"
+			detect_soft_roce_nics
+		fi
+		# tcp fallbacks to veth setup
+		return 0
+	fi
+
+	# Load proper kernel modules if necessary
+	for pci in "${pci_devs[@]}"; do
+		echo "Found $pci (${pci_ids_vendor["$pci"]} - ${pci_ids_device["$pci"]})"
+		if [[ ${pci_mod_resolved["$pci"]} == unknown ]]; then
+			echo "Unresolved modalias for $pci (${pci_mod_driver["$pci"]}). Driver not installed|builtin?"
+			continue
+		fi
+		if [[ ${pci_bus_driver["$pci"]} == unbound ]]; then
+			echo "$pci not bound, needs ${pci_mod_resolved["$pci"]}"
+			pci_drivers["${pci_mod_resolved["$pci"]}"]=1
+		fi
+	done
+
+	if ((${#pci_drivers[@]} > 0)); then
+		echo "Loading kernel modules: ${!pci_drivers[*]}"
+		modprobe -a "${!pci_drivers[@]}"
+	fi
+
+	# E810 cards also need irdma driver to be around.
+	if ((${#e810[@]} > 0)) && [[ $TEST_TRANSPORT == rdma ]]; then
+		if [[ -e /sys/module/irdma/parameters/roce_ena ]]; then
+			# Our tests don't play well with iWARP protocol. Make sure we use RoCEv2 instead.
+			(($(< /sys/module/irdma/parameters/roce_ena) != 1)) && modprobe -r irdma
+		fi
+		modinfo irdma && modprobe irdma roce_ena=1
+	fi > /dev/null
+
+	# All devices detected, kernel modules loaded. Now look under net class to see if there
+	# are any net devices bound to the controllers.
+	for pci in "${pci_devs[@]}"; do
+		if [[ ! -e /sys/bus/pci/devices/$pci/net ]]; then
+			echo "No net devices associated with $pci"
+			continue
+		fi
+		pci_net_devs=("/sys/bus/pci/devices/$pci/net/"*)
+		pci_net_devs=("${pci_net_devs[@]##*/}")
+		echo "Found net devices under $pci: ${pci_net_devs[*]}"
+		net_devs+=("${pci_net_devs[@]}")
+	done
+
+	if ((${#net_devs[@]} == 0)); then
+		echo "ERROR: No net devices were found for: ${pci_devs[*]}. Cannot run the $TEST_TRANSPORT test"
+		return 1
+	fi
+}
+
 function nvmftestinit() {
 	if [ -z $TEST_TRANSPORT ]; then
 		echo "transport not specified - use --transport= to specify"
@@ -408,18 +381,15 @@ function nvmftestinit() {
 
 	trap 'process_shm --id $NVMF_APP_SHM_ID || :; nvmftestfini' SIGINT SIGTERM EXIT
 
+	gather_supported_nvmf_pci_devs
+
 	if [ "$TEST_MODE" == "iso" ]; then
 		$rootdir/scripts/setup.sh
-		if [[ "$TEST_TRANSPORT" == "rdma" ]]; then
-			rdma_device_init
-		fi
-		if [[ "$TEST_TRANSPORT" == "tcp" ]]; then
-			tcp_device_init
-		fi
 	fi
 
 	NVMF_TRANSPORT_OPTS="-t $TEST_TRANSPORT"
 	if [[ "$TEST_TRANSPORT" == "rdma" ]]; then
+		rdma_device_init
 		RDMA_IP_LIST=$(get_available_rdma_ips)
 		NVMF_FIRST_TARGET_IP=$(echo "$RDMA_IP_LIST" | head -n 1)
 		NVMF_SECOND_TARGET_IP=$(echo "$RDMA_IP_LIST" | tail -n +2 | head -n 1)
@@ -458,9 +428,6 @@ function nvmftestfini() {
 	fi
 	if [ "$TEST_MODE" == "iso" ]; then
 		$rootdir/scripts/setup.sh reset
-		if [[ "$TEST_TRANSPORT" == "rdma" ]]; then
-			rdma_device_init
-		fi
 	fi
 	if [[ "$TEST_TRANSPORT" == "tcp" ]]; then
 		nvmf_tcp_fini
@@ -469,14 +436,9 @@ function nvmftestfini() {
 
 function rdma_device_init() {
 	load_ib_rdma_modules
-	detect_transport_nics
 	allocate_nic_ips
 }
 
-function tcp_device_init() {
-	detect_transport_nics
-}
-
 function revert_soft_roce() {
 	rxe_cfg stop
 }

test/spdkcli/nvmf.sh

@@ -17,8 +17,8 @@ timing_exit run_nvmf_tgt
 
 NVMF_TARGET_IP="127.0.0.1"
 if [[ $TEST_TRANSPORT == "rdma" ]]; then
-	rdma_device_init
+	nvmftestinit
-	NVMF_TARGET_IP=$(get_available_rdma_ips | head -n 1)
+	NVMF_TARGET_IP=$NVMF_FIRST_TARGET_IP
 fi
 
 timing_enter spdkcli_create_nvmf_config