d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

test/nvmf: Refactor hw detection routines

Browse Source 
Merge detection functions into a single one where all supported
devices are picked up and exposed in the environment. Be verbose
about what's been found for given tests.

Signed-off-by: Michal Berger <michalx.berger@intel.com>
Change-Id: I780c5e480c301dbb92092074f927a731ed2680b1
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/7849
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Pawel Piatek <pawelx.piatek@intel.com>
This commit is contained in:
Michal Berger 2021-05-11 14:23:27 +02:00 committed by Jim Harris
parent 27470646c5
commit e8a5e965de
5 changed files with 126 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
autotest.sh
View File

@ -243,15 +243,9 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
# The NVMe-oF run test cases are split out like this so that the parser that compiles the
# list of all tests can properly differentiate them. Please do not merge them into one line.
if [ "$SPDK_TEST_NVMF_TRANSPORT" = "rdma" ]; then
timing_enter rdma_setup
rdma_device_init
timing_exit rdma_setup
run_test "nvmf_rdma" ./test/nvmf/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
run_test "spdkcli_nvmf_rdma" ./test/spdkcli/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
elif [ "$SPDK_TEST_NVMF_TRANSPORT" = "tcp" ]; then
timing_enter tcp_setup
tcp_device_init
timing_exit tcp_setup
run_test "nvmf_tcp" ./test/nvmf/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT
if [[ $SPDK_TEST_URING -eq 0 ]]; then
run_test "spdkcli_nvmf_tcp" ./test/spdkcli/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT

2
test/common/autotest_common.sh
View File

@ -155,6 +155,8 @@ export SPDK_TEST_USE_IGB_UIO
export SPDK_TEST_SCHEDULER
: ${SPDK_TEST_SCANBUILD:=0}
export SPDK_TEST_SCANBUILD
: ${SPDK_TEST_NVMF_DRIVER:=}
export SPDK_TEST_NVMF_DRIVER
# always test with SPDK shared objects.
export SPDK_LIB_DIR="$rootdir/build/lib"

3
test/json_config/json_config.sh
View File

@ -286,8 +286,7 @@ function create_nvmf_subsystem_config() {
NVMF_FIRST_TARGET_IP="127.0.0.1"
if [[ $SPDK_TEST_NVMF_TRANSPORT == "rdma" ]]; then
rdma_device_init
NVMF_FIRST_TARGET_IP=$(get_available_rdma_ips | head -n 1)
TEST_TRANSPORT=$SPDK_TEST_NVMF_TRANSPORT nvmftestinit
fi
if [[ -z $NVMF_FIRST_TARGET_IP ]]; then

280
test/nvmf/common.sh
View File

@ -26,6 +26,8 @@ function build_nvmf_app_args() {
fi
}
source "$rootdir/scripts/common.sh"
: ${NVMF_APP_SHM_ID="0"}
export NVMF_APP_SHM_ID
build_nvmf_app_args
@ -57,140 +59,6 @@ function detect_soft_roce_nics() {
rxe_cfg start
}
# args 1 and 2 represent the grep filters for finding our NICS.
# subsequent args are all drivers that should be loaded if we find these NICs.
# Those drivers should be supplied in the correct order.
function detect_nics_and_probe_drivers() {
NIC_VENDOR="$1"
NIC_CLASS="$2"
nvmf_nic_bdfs=$(lspci | grep Ethernet | grep "$NIC_VENDOR" | grep "$NIC_CLASS" | awk -F ' ' '{print "0000:"$1}')
if [ -z "$nvmf_nic_bdfs" ]; then
return 0
fi
have_pci_nics=1
if [ $# -ge 2 ]; then
# shift out the first two positional arguments.
shift 2
# Iterate through the remaining arguments.
for i; do
if [[ $i == irdma ]]; then
# Our tests don't play well with iWARP protocol. Make sure we use RoCEv2 instead.
if [[ -e /sys/module/irdma/parameters/roce_ena ]]; then
# reload the module to re-init the rdma devices
(($(< /sys/module/irdma/parameters/roce_ena) != 1)) && modprobe -r irdma
fi
modprobe "$i" roce_ena=1
else
modprobe "$i"
fi
done
fi
}
function pci_rdma_switch() {
local driver=$1
local -a driver_args=()
driver_args+=("Mellanox ConnectX-4 mlx5_core mlx5_ib")
driver_args+=("Mellanox ConnectX-5 mlx5_core mlx5_ib")
driver_args+=("Intel E810 ice irdma")
driver_args+=("Intel X722 i40e i40iw")
driver_args+=("Chelsio \"Unified Wire\" cxgb4 iw_cxgb4")
case $driver in
mlx5_ib)
detect_nics_and_probe_drivers ${driver_args[0]}
detect_nics_and_probe_drivers ${driver_args[1]}
;;
irdma)
detect_nics_and_probe_drivers ${driver_args[2]}
;;
i40iw)
detect_nics_and_probe_drivers ${driver_args[3]}
;;
iw_cxgb4)
detect_nics_and_probe_drivers ${driver_args[4]}
;;
*)
for d in "${driver_args[@]}"; do
detect_nics_and_probe_drivers $d
done
;;
esac
}
function pci_tcp_switch() {
local driver=$1
local -a driver_args=()
driver_args+=("Intel E810 ice")
case $driver in
ice)
detect_nics_and_probe_drivers ${driver_args[0]}
;;
*)
for d in "${driver_args[@]}"; do
detect_nics_and_probe_drivers $d
done
;;
esac
}
function detect_pci_nics() {
if ! hash lspci; then
return 0
fi
local nic_drivers
local found_drivers
if [[ -z "$TEST_TRANSPORT" ]]; then
TEST_TRANSPORT=$SPDK_TEST_NVMF_TRANSPORT
fi
if [[ "$TEST_TRANSPORT" == "rdma" ]]; then
nic_drivers="mlx5_ib|irdma|i40iw|iw_cxgb4"
# Try to find RDMA drivers which are already loded and try to
# use only it's associated NICs, without probing all drivers.
found_drivers=$(lsmod | grep -Eo $nic_drivers | sort -u)
for d in $found_drivers; do
pci_rdma_switch $d
done
# In case lsmod reported driver, but lspci does not report
# physical NICs - fall back to old approach any try to
# probe all compatible NICs.
((have_pci_nics == 0)) && pci_rdma_switch "default"
elif [[ "$TEST_TRANSPORT" == "tcp" ]]; then
nic_drivers="ice"
found_drivers=$(lsmod | grep -Eo $nic_drivers | sort -u)
for d in $found_drivers; do
pci_tcp_switch $d
done
((have_pci_nics == 0)) && pci_tcp_switch "default"
fi
# Use softroce if everything else failed.
((have_pci_nics == 0)) && return 0
# Provide time for drivers to properly load.
sleep 5
}
function detect_transport_nics() {
detect_pci_nics
if [ "$have_pci_nics" -eq "0" ]; then
detect_soft_roce_nics
fi
}
function allocate_nic_ips() {
((count = NVMF_IP_LEAST_ADDR))
for nic_name in $(get_rdma_if_list); do
@ -212,17 +80,25 @@ function get_available_rdma_ips() {
}
function get_rdma_if_list() {
rxe_cfg rxe-net
}
local net_dev rxe_net_dev rxe_net_devs
function get_tcp_if_list_by_driver() {
local driver
driver=${1:-ice}
mapfile -t rxe_net_devs < <(rxe_cfg rxe-net)
shopt -s nullglob
tcp_if_list=(/sys/bus/pci/drivers/$driver/0000*/net/*)
shopt -u nullglob
printf '%s\n' "${tcp_if_list[@]##*/}"
if ((${#net_devs[@]} == 0)); then
# No rdma-capable nics on board, using soft-RoCE
printf '%s\n' "${rxe_net_devs[@]}"
return 0
fi
# Pick only these devices which were found during gather_supported_nvmf_pci_devs() run
for net_dev in "${net_devs[@]}"; do
for rxe_net_dev in "${rxe_net_devs[@]}"; do
if [[ $net_dev == "$rxe_net_dev" ]]; then
echo "$net_dev"
continue 2
fi
done
done
}
function get_ip_address() {
@ -345,8 +221,8 @@ function nvmf_veth_fini() {
function nvmf_tcp_init() {
NVMF_INITIATOR_IP=10.0.0.1
NVMF_FIRST_TARGET_IP=10.0.0.2
TCP_INTERFACE_LIST=($(get_tcp_if_list_by_driver))
if ((${#TCP_INTERFACE_LIST[@]} == 0)) || [ "$TEST_MODE" == "iso" ]; then
TCP_INTERFACE_LIST=("${net_devs[@]}")
if ((${#TCP_INTERFACE_LIST[@]} == 0)); then
nvmf_veth_init
return 0
fi
@ -400,6 +276,103 @@ function nvmf_tcp_fini() {
ip -4 addr flush $NVMF_INITIATOR_INTERFACE || :
}
function gather_supported_nvmf_pci_devs() {
# Go through the entire pci bus and gather all ethernet controllers we support for the nvmf tests.
# Focus on the hardware that's currently being tested by the CI.
xtrace_disable
cache_pci_bus_sysfs
xtrace_restore
local intel=0x8086 mellanox=0x15b3 pci
local -a pci_devs=()
local -a pci_net_devs=()
local -A pci_drivers=()
local -ga net_devs=()
local -ga e810=()
local -ga x722=()
local -ga mlx=()
# E810-XXV
e810+=(${pci_bus_cache["$intel:0x1592"]})
e810+=(${pci_bus_cache["$intel:0x159b"]})
# X722 10G
x722+=(${pci_bus_cache["$intel:0x37d2"]})
# ConnectX-5
mlx+=(${pci_bus_cache["$mellanox:0x1017"]})
# ConnectX-4
mlx+=(${pci_bus_cache["$mellanox:0x1015"]})
mlx+=(${pci_bus_cache["$mellanox:0x1013"]})
pci_devs+=("${e810[@]}")
if [[ $TEST_TRANSPORT == rdma ]]; then
pci_devs+=("${x722[@]}")
pci_devs+=("${mlx[@]}")
fi
# Try to respect what CI wants to test and override pci_devs[]
if [[ $SPDK_TEST_NVMF_DRIVER == mlx5_ib ]]; then
pci_devs=("${mlx[@]}")
elif [[ $SPDK_TEST_NVMF_DRIVER == ice ]]; then
pci_devs=("${e810[@]}")
fi
if ((${#pci_devs[@]} == 0)); then
if [[ $TEST_TRANSPORT == rdma ]]; then
echo "WARNING: No pci devices found for the $TEST_TRANSPORT test, falling back to Soft-RoCE"
detect_soft_roce_nics
fi
# tcp fallbacks to veth setup
return 0
fi
# Load proper kernel modules if necessary
for pci in "${pci_devs[@]}"; do
echo "Found $pci (${pci_ids_vendor["$pci"]} - ${pci_ids_device["$pci"]})"
if [[ ${pci_mod_resolved["$pci"]} == unknown ]]; then
echo "Unresolved modalias for $pci (${pci_mod_driver["$pci"]}). Driver not installed|builtin?"
continue
fi
if [[ ${pci_bus_driver["$pci"]} == unbound ]]; then
echo "$pci not bound, needs ${pci_mod_resolved["$pci"]}"
pci_drivers["${pci_mod_resolved["$pci"]}"]=1
fi
done
if ((${#pci_drivers[@]} > 0)); then
echo "Loading kernel modules: ${!pci_drivers[*]}"
modprobe -a "${!pci_drivers[@]}"
fi
# E810 cards also need irdma driver to be around.
if ((${#e810[@]} > 0)) && [[ $TEST_TRANSPORT == rdma ]]; then
if [[ -e /sys/module/irdma/parameters/roce_ena ]]; then
# Our tests don't play well with iWARP protocol. Make sure we use RoCEv2 instead.
(($(< /sys/module/irdma/parameters/roce_ena) != 1)) && modprobe -r irdma
fi
modinfo irdma && modprobe irdma roce_ena=1
fi > /dev/null
# All devices detected, kernel modules loaded. Now look under net class to see if there
# are any net devices bound to the controllers.
for pci in "${pci_devs[@]}"; do
if [[ ! -e /sys/bus/pci/devices/$pci/net ]]; then
echo "No net devices associated with $pci"
continue
fi
pci_net_devs=("/sys/bus/pci/devices/$pci/net/"*)
pci_net_devs=("${pci_net_devs[@]##*/}")
echo "Found net devices under $pci: ${pci_net_devs[*]}"
net_devs+=("${pci_net_devs[@]}")
done
if ((${#net_devs[@]} == 0)); then
echo "ERROR: No net devices were found for: ${pci_devs[*]}. Cannot run the $TEST_TRANSPORT test"
return 1
fi
}
function nvmftestinit() {
if [ -z $TEST_TRANSPORT ]; then
echo "transport not specified - use --transport= to specify"
@ -408,18 +381,15 @@ function nvmftestinit() {
trap 'process_shm --id $NVMF_APP_SHM_ID || :; nvmftestfini' SIGINT SIGTERM EXIT
gather_supported_nvmf_pci_devs
if [ "$TEST_MODE" == "iso" ]; then
$rootdir/scripts/setup.sh
if [[ "$TEST_TRANSPORT" == "rdma" ]]; then
rdma_device_init
fi
if [[ "$TEST_TRANSPORT" == "tcp" ]]; then
tcp_device_init
fi
fi
NVMF_TRANSPORT_OPTS="-t $TEST_TRANSPORT"
if [[ "$TEST_TRANSPORT" == "rdma" ]]; then
rdma_device_init
RDMA_IP_LIST=$(get_available_rdma_ips)
NVMF_FIRST_TARGET_IP=$(echo "$RDMA_IP_LIST" | head -n 1)
NVMF_SECOND_TARGET_IP=$(echo "$RDMA_IP_LIST" | tail -n +2 | head -n 1)
@ -458,9 +428,6 @@ function nvmftestfini() {
fi
if [ "$TEST_MODE" == "iso" ]; then
$rootdir/scripts/setup.sh reset
if [[ "$TEST_TRANSPORT" == "rdma" ]]; then
rdma_device_init
fi
fi
if [[ "$TEST_TRANSPORT" == "tcp" ]]; then
nvmf_tcp_fini
@ -469,14 +436,9 @@ function nvmftestfini() {
function rdma_device_init() {
load_ib_rdma_modules
detect_transport_nics
allocate_nic_ips
}
function tcp_device_init() {
detect_transport_nics
}
function revert_soft_roce() {
rxe_cfg stop
}

4
test/spdkcli/nvmf.sh
View File

@ -17,8 +17,8 @@ timing_exit run_nvmf_tgt
NVMF_TARGET_IP="127.0.0.1"
if [[ $TEST_TRANSPORT == "rdma" ]]; then
rdma_device_init
NVMF_TARGET_IP=$(get_available_rdma_ips | head -n 1)
nvmftestinit
NVMF_TARGET_IP=$NVMF_FIRST_TARGET_IP
fi
timing_enter spdkcli_create_nvmf_config