vhost: add live migration support
This patch adds support for live migration for vhost-scsi and vhost-blk backends. Change-Id: Ibfc8a713dbba14ba8cb38377a71e28fd340b1487 Signed-off-by: Pawel Wodkowski <pawelx.wodkowski@intel.com> Reviewed-on: https://review.gerrithub.io/394203 Tested-by: SPDK Automated Test System <sys_sgsw@intel.com> Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com> Reviewed-by: Jim Harris <james.r.harris@intel.com>
This commit is contained in:
Pawel Wodkowski
Jim Harris
parent
f69503f155
commit
efb18b9b46
@ -1076,8 +1076,8 @@ vhost_user_msg_handler(int vid, int fd)
|
||||
return -1;
|
||||
}
|
||||
|
||||
RTE_LOG(INFO, VHOST_CONFIG, "read message %s\n",
|
||||
vhost_message_str[msg.request]);
|
||||
RTE_LOG(INFO, VHOST_CONFIG, "%s: read message %s\n",
|
||||
dev->ifname, vhost_message_str[msg.request]);
|
||||
|
||||
ret = vhost_user_check_and_alloc_queue_pair(dev, &msg);
|
||||
if (ret < 0) {
|
||||
|
||||
@ -91,6 +91,71 @@ void *spdk_vhost_gpa_to_vva(struct spdk_vhost_dev *vdev, uint64_t addr)
|
||||
return (void *)rte_vhost_gpa_to_vva(vdev->mem, addr);
|
||||
}
|
||||
|
||||
static void
|
||||
spdk_vhost_log_req_desc(struct spdk_vhost_dev *vdev, struct spdk_vhost_virtqueue *virtqueue,
|
||||
uint16_t req_id)
|
||||
{
|
||||
struct vring_desc *desc, *desc_table;
|
||||
uint32_t desc_table_size;
|
||||
int rc;
|
||||
|
||||
if (spdk_likely(!spdk_vhost_dev_has_feature(vdev, VHOST_F_LOG_ALL))) {
|
||||
return;
|
||||
}
|
||||
|
||||
rc = spdk_vhost_vq_get_desc(vdev, virtqueue, req_id, &desc, &desc_table, &desc_table_size);
|
||||
if (spdk_unlikely(rc != 0)) {
|
||||
SPDK_ERRLOG("Can't log used ring descriptors!\n");
|
||||
return;
|
||||
}
|
||||
|
||||
do {
|
||||
if (spdk_vhost_vring_desc_is_wr(desc)) {
|
||||
/* To be honest, only pages realy touched should be logged, but
|
||||
* doing so would require tracking those changes in each backed.
|
||||
* Also backend most likely will touch all/most of those pages so
|
||||
* for lets assume we touched all pages passed to as writeable buffers. */
|
||||
rte_vhost_log_write(vdev->vid, desc->addr, desc->len);
|
||||
}
|
||||
spdk_vhost_vring_desc_get_next(&desc, desc_table, desc_table_size);
|
||||
} while (desc);
|
||||
}
|
||||
|
||||
static void
|
||||
spdk_vhost_log_used_vring_elem(struct spdk_vhost_dev *vdev, struct spdk_vhost_virtqueue *virtqueue,
|
||||
uint16_t idx)
|
||||
{
|
||||
uint64_t offset, len;
|
||||
uint16_t vq_idx;
|
||||
|
||||
if (spdk_likely(!spdk_vhost_dev_has_feature(vdev, VHOST_F_LOG_ALL))) {
|
||||
return;
|
||||
}
|
||||
|
||||
offset = offsetof(struct vring_used, ring[idx]);
|
||||
len = sizeof(virtqueue->vring.used->ring[idx]);
|
||||
vq_idx = virtqueue - vdev->virtqueue;
|
||||
|
||||
rte_vhost_log_used_vring(vdev->vid, vq_idx, offset, len);
|
||||
}
|
||||
|
||||
static void
|
||||
spdk_vhost_log_used_vring_idx(struct spdk_vhost_dev *vdev, struct spdk_vhost_virtqueue *virtqueue)
|
||||
{
|
||||
uint64_t offset, len;
|
||||
uint16_t vq_idx;
|
||||
|
||||
if (spdk_likely(!spdk_vhost_dev_has_feature(vdev, VHOST_F_LOG_ALL))) {
|
||||
return;
|
||||
}
|
||||
|
||||
offset = offsetof(struct vring_used, idx);
|
||||
len = sizeof(virtqueue->vring.used->idx);
|
||||
vq_idx = virtqueue - vdev->virtqueue;
|
||||
|
||||
rte_vhost_log_used_vring(vdev->vid, vq_idx, offset, len);
|
||||
}
|
||||
|
||||
/*
|
||||
* Get available requests from avail ring.
|
||||
*/
|
||||
@ -290,13 +355,17 @@ spdk_vhost_vq_used_ring_enqueue(struct spdk_vhost_dev *vdev, struct spdk_vhost_v
|
||||
"Queue %td - USED RING: last_idx=%"PRIu16" req id=%"PRIu16" len=%"PRIu32"\n",
|
||||
virtqueue - vdev->virtqueue, vring->last_used_idx, id, len);
|
||||
|
||||
spdk_vhost_log_req_desc(vdev, virtqueue, id);
|
||||
|
||||
vring->last_used_idx++;
|
||||
used->ring[last_idx].id = id;
|
||||
used->ring[last_idx].len = len;
|
||||
spdk_vhost_log_used_vring_elem(vdev, virtqueue, last_idx);
|
||||
|
||||
/* Ensure the used ring is updated before we increment used->idx. */
|
||||
spdk_smp_wmb();
|
||||
* (volatile uint16_t *) &used->idx = vring->last_used_idx;
|
||||
spdk_vhost_log_used_vring_idx(vdev, virtqueue);
|
||||
|
||||
/* Ensure all our used ring changes are visible to the guest at the time
|
||||
* of interrupt.
|
||||
@ -970,7 +1039,6 @@ start_device(int vid)
|
||||
SPDK_ERRLOG("vhost device %d: Failed to disable guest notification on queue %"PRIu16"\n", vid, i);
|
||||
goto out;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
vdev->num_queues = num_queues;
|
||||
|
||||
@ -90,8 +90,7 @@
|
||||
(1ULL << VIRTIO_RING_F_EVENT_IDX) | \
|
||||
(1ULL << VIRTIO_RING_F_INDIRECT_DESC))
|
||||
|
||||
#define SPDK_VHOST_DISABLED_FEATURES ((1ULL << VHOST_F_LOG_ALL) | \
|
||||
(1ULL << VIRTIO_RING_F_EVENT_IDX) | \
|
||||
#define SPDK_VHOST_DISABLED_FEATURES ((1ULL << VIRTIO_RING_F_EVENT_IDX) | \
|
||||
(1ULL << VIRTIO_RING_F_INDIRECT_DESC))
|
||||
|
||||
enum spdk_vhost_dev_type {
|
||||
|
||||
@ -81,6 +81,9 @@ DEFINE_STUB(rte_vhost_driver_callback_register, int,
|
||||
DEFINE_STUB(rte_vhost_driver_disable_features, int, (const char *path, uint64_t features), 0);
|
||||
DEFINE_STUB(rte_vhost_driver_set_features, int, (const char *path, uint64_t features), 0);
|
||||
DEFINE_STUB(rte_vhost_driver_register, int, (const char *path, uint64_t flags), 0);
|
||||
DEFINE_STUB_V(rte_vhost_log_used_vring, (int vid, uint16_t vring_idx, uint64_t offset,
|
||||
uint64_t len));
|
||||
DEFINE_STUB_V(rte_vhost_log_write, (int vid, uint64_t addr, uint64_t len));
|
||||
DEFINE_STUB(spdk_vhost_scsi_controller_construct, int, (void), 0);
|
||||
DEFINE_STUB(spdk_vhost_blk_controller_construct, int, (void), 0);
|
||||
DEFINE_STUB(rte_vhost_set_vhost_vring_last_idx, int,
|
||||
|
||||
@ -388,7 +388,7 @@ function vm_shutdown_all()
|
||||
while [[ $timeo -gt 0 ]]; do
|
||||
all_vms_down=1
|
||||
for vm in $VM_BASE_DIR/[0-9]*; do
|
||||
if /bin/kill -0 "$(cat $vm/qemu.pid)"; then
|
||||
if [[ -r $vm/qemu.pid ]] && pkill -0 -F "$vm/qemu.pid"; then
|
||||
all_vms_down=0
|
||||
break
|
||||
fi
|
||||
@ -412,13 +412,16 @@ function vm_shutdown_all()
|
||||
function vm_setup()
|
||||
{
|
||||
local shell_restore_x="$( [[ "$-" =~ x ]] && echo 'set -x' )"
|
||||
local OPTIND optchar a
|
||||
local OPTIND optchar vm_num
|
||||
|
||||
local os=""
|
||||
local os_mode=""
|
||||
local qemu_args=""
|
||||
local disk_type=NOT_DEFINED
|
||||
local disks=""
|
||||
local raw_cache=""
|
||||
local vm_incoming=""
|
||||
local vm_migrate_to=""
|
||||
local force_vm=""
|
||||
local guest_memory=1024
|
||||
local queue_number=""
|
||||
@ -435,6 +438,8 @@ function vm_setup()
|
||||
force=*) local force_vm=${OPTARG#*=} ;;
|
||||
memory=*) local guest_memory=${OPTARG#*=} ;;
|
||||
queue_num=*) local queue_number=${OPTARG#*=} ;;
|
||||
incoming=*) local vm_incoming="${OPTARG#*=}" ;;
|
||||
migrate-to=*) local vm_migrate_to="${OPTARG#*=}" ;;
|
||||
*)
|
||||
error "unknown argument $OPTARG"
|
||||
return 1
|
||||
@ -454,8 +459,6 @@ function vm_setup()
|
||||
vm_num_is_valid $vm_num || return 1
|
||||
local vm_dir="$VM_BASE_DIR/$vm_num"
|
||||
[[ -d $vm_dir ]] && warning "removing existing VM in '$vm_dir'"
|
||||
# FIXME: why this is just echo???
|
||||
echo "rm -rf $vm_dir"
|
||||
else
|
||||
local vm_dir=""
|
||||
|
||||
@ -474,11 +477,43 @@ function vm_setup()
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ ! -z "$vm_migrate_to" && ! -z "$vm_incoming" ]]; then
|
||||
error "'--incoming' and '--migrate-to' cannot be used together"
|
||||
return 1
|
||||
elif [[ ! -z "$vm_incoming" ]]; then
|
||||
if [[ ! -z "$os_mode" || ! -z "$os_img" ]]; then
|
||||
error "'--incoming' can't be used together with '--os' nor '--os-mode'"
|
||||
return 1
|
||||
fi
|
||||
|
||||
os_mode="original"
|
||||
os="$VM_BASE_DIR/$vm_incoming/os.qcow2"
|
||||
elif [[ ! -z "$vm_migrate_to" ]]; then
|
||||
[[ "$os_mode" != "backing" ]] && warning "Using 'backing' mode for OS since '--migrate-to' is used"
|
||||
os_mode=backing
|
||||
fi
|
||||
|
||||
notice "Creating new VM in $vm_dir"
|
||||
mkdir -p $vm_dir
|
||||
if [[ ! -r $os ]]; then
|
||||
error "file not found: $os"
|
||||
return 1
|
||||
|
||||
if [[ "$os_mode" == "backing" ]]; then
|
||||
notice "Creating backing file for OS image file: $os"
|
||||
if ! $INSTALL_DIR/bin/qemu-img create -f qcow2 -b $os $vm_dir/os.qcow2; then
|
||||
error "Failed to create OS backing file in '$vm_dir/os.qcow2' using '$os'"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local os=$vm_dir/os.qcow2
|
||||
elif [[ "$os_mode" == "original" ]]; then
|
||||
warning "Using original OS image file: $os"
|
||||
elif [[ "$os_mode" != "snapshot" ]]; then
|
||||
if [[ -z "$os_mode" ]]; then
|
||||
notice "No '--os-mode' parameter provided - using 'snapshot'"
|
||||
os_mode="snapshot"
|
||||
else
|
||||
error "Invalid '--os-mode=$os_mode'"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# WARNING:
|
||||
@ -501,8 +536,8 @@ function vm_setup()
|
||||
|
||||
local ssh_socket=$(( vm_socket_offset + 0 ))
|
||||
local fio_socket=$(( vm_socket_offset + 1 ))
|
||||
# vm_socket_offset + 2 - can be reused
|
||||
# vm_socket_offset + 3 - can be reused
|
||||
local monitor_port=$(( vm_socket_offset + 2 ))
|
||||
local migration_port=$(( vm_socket_offset + 3 ))
|
||||
local gdbserver_socket=$(( vm_socket_offset + 4 ))
|
||||
local vnc_socket=$(( 100 + vm_num ))
|
||||
local qemu_pid_file="$vm_dir/qemu.pid"
|
||||
@ -520,18 +555,19 @@ function vm_setup()
|
||||
|
||||
$shell_restore_x
|
||||
|
||||
#-cpu host
|
||||
local node_num=${!qemu_numa_node_param}
|
||||
notice "NUMA NODE: $node_num"
|
||||
cmd+="-m $guest_memory --enable-kvm -cpu host -smp $cpu_num -vga std -vnc :$vnc_socket -daemonize -snapshot ${eol}"
|
||||
cmd+="-m $guest_memory --enable-kvm -cpu host -smp $cpu_num -vga std -vnc :$vnc_socket -daemonize ${eol}"
|
||||
cmd+="-object memory-backend-file,id=mem,size=${guest_memory}M,mem-path=/dev/hugepages,share=on,prealloc=yes,host-nodes=$node_num,policy=bind ${eol}"
|
||||
[[ $os_mode == snapshot ]] && cmd+="-snapshot ${eol}"
|
||||
[[ ! -z "$vm_incoming" ]] && cmd+=" -incoming tcp:0:$migration_port ${eol}"
|
||||
cmd+="-monitor telnet:127.0.0.1:$monitor_port,server,nowait ${eol}"
|
||||
cmd+="-numa node,memdev=mem ${eol}"
|
||||
cmd+="-pidfile $qemu_pid_file ${eol}"
|
||||
cmd+="-serial file:$vm_dir/serial.log ${eol}"
|
||||
cmd+="-D $vm_dir/qemu.log ${eol}"
|
||||
cmd+="-net user,hostfwd=tcp::$ssh_socket-:22,hostfwd=tcp::$fio_socket-:8765 ${eol}"
|
||||
cmd+="-net nic ${eol}"
|
||||
|
||||
cmd+="-drive file=$os,if=none,id=os_disk ${eol}"
|
||||
cmd+="-device ide-hd,drive=os_disk,bootindex=0 ${eol}"
|
||||
|
||||
@ -608,7 +644,7 @@ function vm_setup()
|
||||
# remove last $eol
|
||||
cmd="${cmd%\\\\\\n }"
|
||||
|
||||
notice "Saving to $vm_dir/run.sh:"
|
||||
notice "Saving to $vm_dir/run.sh"
|
||||
(
|
||||
echo '#!/bin/bash'
|
||||
echo 'if [[ $EUID -ne 0 ]]; then '
|
||||
@ -638,8 +674,16 @@ function vm_setup()
|
||||
# Save generated sockets redirection
|
||||
echo $ssh_socket > $vm_dir/ssh_socket
|
||||
echo $fio_socket > $vm_dir/fio_socket
|
||||
echo $monitor_port > $vm_dir/monitor_port
|
||||
|
||||
rm -f $vm_dir/migration_port
|
||||
[[ -z $vm_incoming ]] || echo $migration_port > $vm_dir/migration_port
|
||||
|
||||
echo $gdbserver_socket > $vm_dir/gdbserver_socket
|
||||
echo $vnc_socket >> $vm_dir/vnc_socket
|
||||
|
||||
[[ -z $vm_incoming ]] || ln -fs $VM_BASE_DIR/$vm_incoming $vm_dir/vm_incoming
|
||||
[[ -z $vm_migrate_to ]] || ln -fs $VM_BASE_DIR/$vm_migrate_to $vm_dir/vm_migrate_to
|
||||
}
|
||||
|
||||
function vm_run()
|
||||
@ -840,16 +884,18 @@ function run_fio()
|
||||
local out=""
|
||||
local fio_disks=""
|
||||
local vm
|
||||
local run_server_mode=true
|
||||
|
||||
for arg in $@; do
|
||||
case "$arg" in
|
||||
--job-file=*) local job_file="${arg#*=}" ;;
|
||||
--fio-bin=*) local fio_bin="--fio-bin=${arg#*=}" ;;
|
||||
--fio-bin=*) local fio_bin="${arg#*=}" ;;
|
||||
--vm=*) vms+=( "${arg#*=}" ) ;;
|
||||
--out=*)
|
||||
local out="$arg"
|
||||
mkdir -p ${out#*=}
|
||||
;;
|
||||
local out="${arg#*=}"
|
||||
mkdir -p $out
|
||||
;;
|
||||
--local) run_server_mode=false ;;
|
||||
*)
|
||||
error "Invalid argument '$arg'"
|
||||
return 1
|
||||
@ -857,8 +903,17 @@ function run_fio()
|
||||
esac
|
||||
done
|
||||
|
||||
local job_fname=$(basename "$job_file")
|
||||
if [[ ! -z "$fio_bin" && ! -r "$fio_bin" ]]; then
|
||||
error "FIO binary '$fio_bin' does not exist"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ ! -r "$job_file" ]]; then
|
||||
error "Fio job '$job_file' does not exist"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local job_fname=$(basename "$job_file")
|
||||
# prepare job file for each VM
|
||||
for vm in ${vms[@]}; do
|
||||
local vm_num=${vm%%:*}
|
||||
@ -868,9 +923,25 @@ function run_fio()
|
||||
fio_disks+="127.0.0.1:$(vm_fio_socket $vm_num):$vmdisks,"
|
||||
|
||||
vm_ssh $vm_num cat /root/$job_fname
|
||||
if ! $run_server_mode; then
|
||||
if [[ ! -z "$fio_bin" ]]; then
|
||||
cat $fio_bin | vm_ssh $vm_num 'cat > /root/fio; chmod +x /root/fio'
|
||||
fi
|
||||
|
||||
notice "Running local fio on VM $vm_num"
|
||||
vm_ssh $vm_num "nohup /root/fio /root/$job_fname 1>/root/$job_fname.out 2>/root/$job_fname.out </dev/null & echo \$! > /root/fio.pid"
|
||||
fi
|
||||
done
|
||||
|
||||
python $SPDK_BUILD_DIR/test/vhost/common/run_fio.py --job-file=/root/$job_fname $fio_bin $out ${fio_disks%,}
|
||||
if ! $run_server_mode; then
|
||||
# Give FIO time to run
|
||||
sleep 0.5
|
||||
return 0
|
||||
fi
|
||||
|
||||
python $SPDK_BUILD_DIR/test/vhost/common/run_fio.py --job-file=/root/$job_fname \
|
||||
$([[ ! -z "$fio_bin" ]] && echo "--fio-bin=$fio_bin") \
|
||||
--out=$out ${fio_disks%,}
|
||||
}
|
||||
|
||||
# Shutdown or kill any running VM and SPDK APP.
|
||||
|
||||
@ -20,7 +20,7 @@ def show_help():
|
||||
Options:
|
||||
-h, --help Show this message.
|
||||
-j, --job-file Paths to file with FIO job configuration on remote host.
|
||||
-f, --fio-bin Location of FIO binary on remote host (Default "fio")
|
||||
-f, --fio-bin Location of FIO binary on local host (Default "fio")
|
||||
-o, --out Directory used to save generated job files and
|
||||
files with test results
|
||||
-p, --perf-vmex Enable aggregating statistic for VMEXITS for VMs
|
||||
@ -112,6 +112,7 @@ def main():
|
||||
|
||||
vms = []
|
||||
fio_cfg = None
|
||||
out_dir = None
|
||||
perf_vmex = False
|
||||
|
||||
try:
|
||||
@ -143,7 +144,7 @@ def main():
|
||||
print("ERROR! No FIO job provided!")
|
||||
sys.exit(1)
|
||||
|
||||
if not os.path.exists(out_dir):
|
||||
if out_dir is None or not os.path.exists(out_dir):
|
||||
print("ERROR! Folder {out_dir} does not exist ".format(out_dir=out_dir))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
@ -11,70 +11,63 @@ function usage()
|
||||
echo "Usage: $(basename $1) [OPTIONS] VM_NUM"
|
||||
echo
|
||||
echo "-h, --help print help and exit"
|
||||
echo "-f VM_NUM Force VM_NUM reconfiguration if already exist"
|
||||
echo " --work-dir=WORK_DIR Where to find build file. Must exit. (default: $TEST_DIR)"
|
||||
echo " --test-type=TYPE Perform specified test:"
|
||||
echo " --force=VM_NUM Force VM_NUM reconfiguration if already exist"
|
||||
echo " --disk-type=TYPE Perform specified test:"
|
||||
echo " virtio - test host virtio-scsi-pci using file as disk image"
|
||||
echo " kernel_vhost - use kernel driver vhost-scsi"
|
||||
echo " spdk_vhost_scsi - use spdk vhost scsi"
|
||||
echo " spdk_vhost_blk - use spdk vhost block"
|
||||
echo " ---cache=CACHE Use CACHE for virtio test: "
|
||||
echo " --raw-cache=CACHE Use CACHE for virtio test: "
|
||||
echo " writethrough, writeback, none, unsafe or directsyns"
|
||||
echo " Default is writethrough"
|
||||
echo " --disk=PATH Disk to use in test. test specific meaning:"
|
||||
echo " virtio - disk path (file or block device ex: /dev/nvme0n1)"
|
||||
echo " kernel_vhost - the WWN number to be used"
|
||||
echo " spdk_vhost - the socket path. Default is WORK_DIR/vhost/usvhost"
|
||||
echo " spdk_vhost_[scsi|blk] - the socket path."
|
||||
echo " --os=OS_QCOW2 Custom OS qcow2 image file"
|
||||
echo " --os-mode=MODE MODE how to use provided image: default: backing"
|
||||
echo " backing - create new image but use provided backing file"
|
||||
echo " copy - copy provided image and use a copy"
|
||||
echo " orginal - use file directly. Will modify the provided file"
|
||||
echo " --incoming=VM_NUM Use VM_NUM as source migration VM."
|
||||
echo " --migrate-to=VM_NUM Use VM_NUM as target migration VM."
|
||||
echo "-x Turn on script debug (set -x)"
|
||||
echo "-v Be more verbose"
|
||||
exit 0
|
||||
}
|
||||
disk=""
|
||||
raw_cache=""
|
||||
img_mode=""
|
||||
os=""
|
||||
while getopts 'xf:h-:' optchar; do
|
||||
case "$optchar" in
|
||||
-)
|
||||
case "$OPTARG" in
|
||||
help) usage $0 ;;
|
||||
work-dir=*) TEST_DIR="${OPTARG#*=}" ;;
|
||||
raw-cache=*) raw_cache="--raw-cache=${OPTARG#*=}" ;;
|
||||
test-type=*) test_type="${OPTARG#*=}" ;;
|
||||
spdk-vhost-mode=*) spdk_vhost_mode="${OPTARG#*=}" ;;
|
||||
disk=*) disk="${OPTARG#*=}" ;;
|
||||
os=*) os="${OPTARG#*=}"
|
||||
if [[ ! -r "$os" ]]; then
|
||||
echo "ERROR: can't read '$os'"
|
||||
usage $0
|
||||
fi
|
||||
os="$(readlink -f $os)"
|
||||
;;
|
||||
os-mode=*) os_mode="--os-mode=${OPTARG#*=}" ;;
|
||||
*) usage $0 "Invalid argument '$OPTARG'" ;;
|
||||
esac
|
||||
;;
|
||||
h) usage $0 ;;
|
||||
x) set -x ;;
|
||||
f) force_vm_num="--force=${OPTARG#*=}" ;;
|
||||
*) usage $0 "Invalid argument '$OPTARG'" ;;
|
||||
|
||||
setup_params=()
|
||||
for param in "$@"; do
|
||||
case "$param" in
|
||||
--help|-h) usage $0 ;;
|
||||
--work-dir=*)
|
||||
TEST_DIR="${param#*=}"
|
||||
continue
|
||||
;;
|
||||
--raw-cache=*) ;;
|
||||
--disk-type=*) ;;
|
||||
--disks=*) ;;
|
||||
--os=*) ;;
|
||||
--os-mode=*) ;;
|
||||
--force=*) ;;
|
||||
--incoming=*) ;;
|
||||
--migrate-to=*) ;;
|
||||
-x)
|
||||
set -x
|
||||
continue
|
||||
;;
|
||||
-v)
|
||||
SPDK_VHOST_VERBOSE=true
|
||||
continue
|
||||
;;
|
||||
*) usage $0 "Invalid argument '$param'" ;;
|
||||
esac
|
||||
|
||||
setup_params+=( "$param" )
|
||||
done
|
||||
|
||||
. $COMMON_DIR/common.sh
|
||||
|
||||
[[ -z "$os" ]] && os="$TEST_DIR/debian.qcow2"
|
||||
[[ $test_type =~ "spdk_vhost" ]] && [[ -z "$disk" ]] && disk="$SPDK_VHOST_SCSI_TEST_DIR/usvhost"
|
||||
if [[ $test_type == "kernel_vhost" ]] && [[ -z "$disk" ]]; then
|
||||
fail "for $test_type '--disk=WWN' is mandatory"
|
||||
fi
|
||||
vm_setup ${setup_params[@]}
|
||||
|
||||
vm_setup \
|
||||
--os=$os \
|
||||
--disk-type=$test_type \
|
||||
--disks=$disk \
|
||||
$wwn $raw_cache $force_vm_num $os_mode
|
||||
trap -- ERR
|
||||
|
||||
@ -107,10 +107,10 @@ vm_no="0"
|
||||
vm_setup --disk-type=spdk_vhost_scsi --force=$vm_no --os=$os_image --disks="Nvme0n1:Malloc0:Malloc1" --queue_num=18 --memory=6144
|
||||
vm_run $vm_no
|
||||
vm_wait_for_boot 600 $vm_no
|
||||
vm_scp $vm_num -r $ROOT_DIR "127.0.0.1:/root/spdk"
|
||||
vm_ssh $vm_num " cd spdk ; make clean ; ./configure --with-fio=/root/fio_src ; make -j2"
|
||||
vm_ssh $vm_num "/root/spdk/scripts/setup.sh"
|
||||
vbdevs=$(vm_ssh $vm_num ". /root/spdk/scripts/autotest_common.sh && discover_bdevs /root/spdk \
|
||||
vm_scp $vm_no -r $ROOT_DIR "127.0.0.1:/root/spdk"
|
||||
vm_ssh $vm_no " cd spdk ; make clean ; ./configure --with-fio=/root/fio_src ; make -j2"
|
||||
vm_ssh $vm_no "/root/spdk/scripts/setup.sh"
|
||||
vbdevs=$(vm_ssh $vm_no ". /root/spdk/scripts/autotest_common.sh && discover_bdevs /root/spdk \
|
||||
/root/spdk/test/vhost/initiator/bdev_pci.conf")
|
||||
virtio_bdevs=$(jq -r '[.[].name] | join(":")' <<< $vbdevs)
|
||||
virtio_with_unmap=$(jq -r '[.[] | select(.supported_io_types.unmap==true).name]
|
||||
@ -118,14 +118,14 @@ virtio_with_unmap=$(jq -r '[.[] | select(.supported_io_types.unmap==true).name]
|
||||
timing_exit setup_vm
|
||||
|
||||
timing_enter run_spdk_fio_pci
|
||||
vm_ssh $vm_num "LD_PRELOAD=/root/spdk/examples/bdev/fio_plugin/fio_plugin /root/fio_src/fio --ioengine=spdk_bdev \
|
||||
vm_ssh $vm_no "LD_PRELOAD=/root/spdk/examples/bdev/fio_plugin/fio_plugin /root/fio_src/fio --ioengine=spdk_bdev \
|
||||
/root/spdk/test/vhost/initiator/bdev.fio --filename=$virtio_bdevs --section=job_randwrite \
|
||||
--section=job_randrw --section=job_write --section=job_rw \
|
||||
--spdk_conf=/root/spdk/test/vhost/initiator/bdev_pci.conf --spdk_mem=1024"
|
||||
timing_exit run_spdk_fio_pci
|
||||
|
||||
timing_enter run_spdk_fio_pci_unmap
|
||||
vm_ssh $vm_num "LD_PRELOAD=/root/spdk/examples/bdev/fio_plugin/fio_plugin /root/fio_src/fio --ioengine=spdk_bdev \
|
||||
vm_ssh $vm_no "LD_PRELOAD=/root/spdk/examples/bdev/fio_plugin/fio_plugin /root/fio_src/fio --ioengine=spdk_bdev \
|
||||
/root/spdk/test/vhost/initiator/bdev.fio --filename=$virtio_with_unmap \
|
||||
--spdk_conf=/root/spdk/test/vhost/initiator/bdev_pci.conf --spdk_mem=1024"
|
||||
timing_exit run_spdk_fio_pci_unmap
|
||||
|
||||
25
test/vhost/migration/migration-malloc.job
Normal file
25
test/vhost/migration/migration-malloc.job
Normal file
@ -0,0 +1,25 @@
|
||||
[global]
|
||||
blocksize_range=4k-512k
|
||||
#bs=512k
|
||||
iodepth=128
|
||||
ioengine=libaio
|
||||
filename=
|
||||
group_reporting
|
||||
thread
|
||||
numjobs=1
|
||||
direct=1
|
||||
do_verify=1
|
||||
verify=md5
|
||||
verify_fatal=1
|
||||
verify_dump=1
|
||||
size=100%
|
||||
|
||||
[write]
|
||||
rw=write
|
||||
stonewall
|
||||
|
||||
[randread]
|
||||
rw=randread
|
||||
runtime=10
|
||||
time_based
|
||||
stonewall
|
||||
231
test/vhost/migration/migration-malloc.sh
Executable file
231
test/vhost/migration/migration-malloc.sh
Executable file
@ -0,0 +1,231 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
vms=()
|
||||
declare -A vms_os
|
||||
declare -A vms_raw_disks
|
||||
declare -A vms_ctrlrs
|
||||
declare -A vms_ctrlrs_disks
|
||||
|
||||
# By default use Guest fio
|
||||
fio_bin=""
|
||||
|
||||
function usage()
|
||||
{
|
||||
[[ ! -z $2 ]] && ( echo "$2"; echo ""; )
|
||||
echo "Shortcut script for doing automated test of live migration."
|
||||
echo "Usage: $(basename $1) [OPTIONS]"
|
||||
echo
|
||||
echo " --work-dir=WORK_DIR Where to find build file. Must exist. [default: $TEST_DIR]"
|
||||
echo " --os ARGS VM configuration. This parameter might be used more than once:"
|
||||
echo " --fio-bin=FIO Use specific fio binary (will be uploaded to VM)"
|
||||
echo " --fio-job= Fio config to use for test."
|
||||
echo " num=NUM - VM number"
|
||||
echo " os=OS - VM os disk path"
|
||||
echo " bdevs=DISKS - VM test disks/devices path separated by ':'"
|
||||
echo " incoming - set this VM to wait for incoming migration"
|
||||
echo " If test-type=spdk_vhost_blk then each disk size is 20G e.g."
|
||||
echo " --vm num=X,os=os.qcow,bdevs=Malloc0:Nvme0n1:Malloc1"
|
||||
echo "-x set -x for script debug"
|
||||
}
|
||||
|
||||
for param in "$@"; do
|
||||
case "$param" in
|
||||
--help|-h)
|
||||
usage $0
|
||||
exit 0
|
||||
;;
|
||||
--work-dir=*) TEST_DIR="${param#*=}" ;;
|
||||
--os=*) os_image="${param#*=}" ;;
|
||||
--fio-bin=*) fio_bin="${param}" ;;
|
||||
-x) set -x ;;
|
||||
-v) SPDK_VHOST_VERBOSE=true ;;
|
||||
*)
|
||||
usage $0 "Invalid argument '$param'"
|
||||
exit 1;;
|
||||
esac
|
||||
done
|
||||
|
||||
. $(readlink -e "$(dirname $0)/../common/common.sh") || exit 1
|
||||
|
||||
job_file="$BASE_DIR/migration-malloc.job"
|
||||
|
||||
trap 'error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
|
||||
|
||||
function vm_monitor_send()
|
||||
{
|
||||
local vm_num=$1
|
||||
local cmd_result_file="$2"
|
||||
local vm_dir="$VM_BASE_DIR/$1"
|
||||
local vm_monitor_port=$(cat $vm_dir/monitor_port)
|
||||
|
||||
[[ ! -z "$vm_monitor_port" ]] || fail "No monitor port!"
|
||||
|
||||
shift 2
|
||||
nc 127.0.0.1 $vm_monitor_port "$@" > $cmd_result_file
|
||||
}
|
||||
|
||||
# Migrate VM $1
|
||||
function vm_migrate()
|
||||
{
|
||||
local from_vm_dir="$VM_BASE_DIR/$1"
|
||||
local target_vm_dir="$(readlink -e $from_vm_dir/vm_migrate_to)"
|
||||
local target_vm="$(basename $target_vm_dir)"
|
||||
local target_vm_migration_port="$(cat $target_vm_dir/migration_port)"
|
||||
|
||||
# Sanity check if target VM (QEMU) is configured to accept source VM (QEMU) migration
|
||||
if [[ "$(readlink -e ${target_vm_dir}/vm_incoming)" != "$(readlink -e ${from_vm_dir})" ]]; then
|
||||
fail "source VM $1 or destination VM is not properly configured for live migration"
|
||||
fi
|
||||
|
||||
notice "Migrating VM $1 to VM "$(basename $target_vm_dir)
|
||||
echo -e \
|
||||
"migrate_set_speed 1g\n" \
|
||||
"migrate tcp:127.0.0.1:$target_vm_migration_port\n" \
|
||||
"info migrate\n" \
|
||||
"quit" | vm_monitor_send $1 "$from_vm_dir/migration_result"
|
||||
|
||||
# Post migration checks:
|
||||
if ! grep "Migration status: completed" $from_vm_dir/migration_result -q; then
|
||||
cat $from_vm_dir/migration_result
|
||||
fail "Migration failed:\n"
|
||||
fi
|
||||
|
||||
if ! vm_os_booted $target_vm; then
|
||||
fail "VM$target_vm is not running"
|
||||
cat $target_vm $target_vm_dir/cont_result
|
||||
fi
|
||||
|
||||
notice "Migration complete"
|
||||
}
|
||||
|
||||
# FIXME: this shoul'd not be needed
|
||||
vm_kill_all
|
||||
|
||||
rpc="python $SPDK_BUILD_DIR/scripts/rpc.py "
|
||||
|
||||
# Use 2 VMs:
|
||||
# incoming VM - the one we want to migrate
|
||||
# targe VM - the one which will accept migration
|
||||
incoming_vm=0
|
||||
target_vm=1
|
||||
incoming_vm_ctrlr=naa.Malloc0.$incoming_vm
|
||||
target_vm_ctrlr=naa.Malloc0.$target_vm
|
||||
|
||||
vm_setup --os="$os_image" --force=$incoming_vm --disk-type=spdk_vhost_scsi --disks=Malloc0 --migrate-to=$target_vm
|
||||
vm_setup --force=$target_vm --disk-type=spdk_vhost_scsi --disks=Malloc0 --incoming=$incoming_vm
|
||||
|
||||
spdk_vhost_run $BASE_DIR
|
||||
|
||||
notice "==============="
|
||||
notice ""
|
||||
notice "Setting up VMs"
|
||||
notice ""
|
||||
|
||||
declare -A vm_ctrlr
|
||||
|
||||
function clean_vhost_config()
|
||||
{
|
||||
notice "Removing vhost devices & controllers via RPC ..."
|
||||
# Delete bdev first to remove all LUNs and SCSI targets
|
||||
$rpc delete_bdev Malloc0
|
||||
|
||||
# Delete controllers
|
||||
$rpc remove_vhost_controller $incoming_vm_ctrlr
|
||||
$rpc remove_vhost_controller $target_vm_ctrlr
|
||||
}
|
||||
|
||||
function error_migration_clean()
|
||||
{
|
||||
trap - SIGINT ERR EXIT
|
||||
set -x
|
||||
|
||||
vm_kill_all
|
||||
clean_vhost_config
|
||||
}
|
||||
|
||||
function is_fio_running()
|
||||
{
|
||||
local shell_restore_x="$( [[ "$-" =~ x ]] && echo 'set -x' )"
|
||||
set +x
|
||||
|
||||
if vm_ssh $1 'kill -0 $(cat /root/fio.pid)'; then
|
||||
local ret=0
|
||||
else
|
||||
local ret=1
|
||||
fi
|
||||
|
||||
$shell_restore_x
|
||||
return $ret
|
||||
}
|
||||
|
||||
trap 'error_migration_clean; error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
|
||||
|
||||
# Construct shared Malloc Bdev
|
||||
$rpc construct_malloc_bdev -b Malloc0 128 4096
|
||||
|
||||
# And two controllers - one for each VM. Both are using the same Malloc Bdev as LUN 0
|
||||
$rpc construct_vhost_scsi_controller $incoming_vm_ctrlr
|
||||
$rpc add_vhost_scsi_lun $incoming_vm_ctrlr 0 Malloc0
|
||||
|
||||
$rpc construct_vhost_scsi_controller $target_vm_ctrlr
|
||||
$rpc add_vhost_scsi_lun $target_vm_ctrlr 0 Malloc0
|
||||
|
||||
# Run everything
|
||||
vm_run $incoming_vm $target_vm
|
||||
|
||||
# Wait only for incoming VM, as target is waiting for migration
|
||||
vm_wait_for_boot 600 $incoming_vm
|
||||
|
||||
# Run fio before migration
|
||||
notice "Starting FIO"
|
||||
|
||||
vm_check_scsi_location $incoming_vm
|
||||
run_fio $fio_bin --job-file="$job_file" --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)"
|
||||
|
||||
# Wait a while to let the FIO time to issue some IO
|
||||
sleep 5
|
||||
|
||||
# Check if fio is still running before migration
|
||||
if ! is_fio_running $incoming_vm; then
|
||||
vm_ssh $incoming_vm "cat /root/$(basename ${job_file}).out"
|
||||
error "FIO is not running before migration: process crashed or finished too early"
|
||||
fi
|
||||
|
||||
vm_migrate $incoming_vm
|
||||
sleep 3
|
||||
|
||||
# Check if fio is still running after migration
|
||||
if ! is_fio_running $target_vm; then
|
||||
vm_ssh $target_vm "cat /root/$(basename ${job_file}).out"
|
||||
error "FIO is not running after migration: process crashed or finished too early"
|
||||
fi
|
||||
|
||||
notice "Waiting for fio to finish"
|
||||
timeout=40
|
||||
while is_fio_running $target_vm; do
|
||||
sleep 1
|
||||
echo -n "."
|
||||
if (( timeout-- == 0 )); then
|
||||
error "timeout while waiting for FIO!"
|
||||
fi
|
||||
done
|
||||
|
||||
notice "Fio result is:"
|
||||
vm_ssh $target_vm "cat /root/$(basename ${job_file}).out"
|
||||
|
||||
notice "Migration DONE"
|
||||
|
||||
|
||||
notice "Shutting down all VMs"
|
||||
vm_shutdown_all
|
||||
clean_vhost_config
|
||||
|
||||
notice "killing vhost app"
|
||||
spdk_vhost_kill
|
||||
|
||||
notice "Migration Test SUCCESS"
|
||||
notice "==============="
|
||||
|
||||
trap - SIGINT ERR EXIT
|
||||
5
test/vhost/migration/vhost.conf.in
Normal file
5
test/vhost/migration/vhost.conf.in
Normal file
@ -0,0 +1,5 @@
|
||||
[Global]
|
||||
NoPci Yes
|
||||
|
||||
[Ioat]
|
||||
Disable Yes
|
||||
@ -74,6 +74,11 @@ case $1 in
|
||||
--test-type=spdk_vhost_blk \
|
||||
--fio-job=$WORKDIR/common/fio_jobs/default_performance.job
|
||||
;;
|
||||
-m|--migration)
|
||||
echo 'Running migration suite...'
|
||||
./migration/migration-malloc.sh -x \
|
||||
--fio-bin=$FIO_BIN --os=$VM_IMAGE
|
||||
;;
|
||||
-i|--integrity)
|
||||
echo 'Running SCSI integrity suite...'
|
||||
./fiotest/autotest.sh -x --fio-bin=$FIO_BIN \
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user