From 679e2831bd3319e8a71a93f55379b7ec94c4f3d8 Mon Sep 17 00:00:00 2001 From: Dariusz Stojaczyk Date: Wed, 15 Mar 2017 17:00:55 +0100 Subject: [PATCH] vhost: added rpc commands to remove vhost controllers and devices Added new rpc commands together with underlying vhost API and tests. Change-Id: Ib9c6a530d0909193ea5115aaac4920c44f39613c Signed-off-by: Dariusz Stojaczyk --- include/spdk/vhost.h | 3 + lib/vhost/vhost.c | 75 +++++++++++++++++++- lib/vhost/vhost_rpc.c | 126 ++++++++++++++++++++++++++++++++- scripts/rpc.py | 20 ++++++ test/vhost/fiotest/autotest.sh | 51 ++++++++++++- 5 files changed, 270 insertions(+), 5 deletions(-) diff --git a/include/spdk/vhost.h b/include/spdk/vhost.h index 995caf8df5..c4e2ca6d66 100644 --- a/include/spdk/vhost.h +++ b/include/spdk/vhost.h @@ -63,10 +63,13 @@ struct spdk_vhost_scsi_ctrlr *spdk_vhost_scsi_ctrlr_next(struct spdk_vhost_scsi_ const char *spdk_vhost_scsi_ctrlr_get_name(struct spdk_vhost_scsi_ctrlr *ctrl); uint64_t spdk_vhost_scsi_ctrlr_get_cpumask(struct spdk_vhost_scsi_ctrlr *ctrl); +struct spdk_vhost_scsi_ctrlr *spdk_vhost_scsi_ctrlr_find(const char *ctrlr_name); int spdk_vhost_scsi_ctrlr_construct(const char *name, uint64_t cpumask); +int spdk_vhost_scsi_ctrlr_remove(struct spdk_vhost_scsi_ctrlr *vdev); int spdk_vhost_parse_core_mask(const char *mask, uint64_t *cpumask); struct spdk_scsi_dev *spdk_vhost_scsi_ctrlr_get_dev(struct spdk_vhost_scsi_ctrlr *ctrl, uint8_t num); int spdk_vhost_scsi_ctrlr_add_dev(const char *name, unsigned scsi_dev_num, const char *lun_name); +int spdk_vhost_scsi_ctrlr_remove_dev(struct spdk_vhost_scsi_ctrlr *vdev, unsigned scsi_dev_num); #endif /* SPDK_VHOST_H */ diff --git a/lib/vhost/vhost.c b/lib/vhost/vhost.c index 9baa6bbe37..5ad3184757 100644 --- a/lib/vhost/vhost.c +++ b/lib/vhost/vhost.c @@ -881,7 +881,7 @@ destroy_device(int vid) static struct spdk_vhost_scsi_ctrlr *spdk_vhost_ctrlrs[MAX_SCSI_CTRLRS]; -static struct spdk_vhost_scsi_ctrlr * +struct spdk_vhost_scsi_ctrlr * spdk_vhost_scsi_ctrlr_find(const char *ctrlr_name) { unsigned i; @@ -1005,6 +1005,57 @@ spdk_vhost_scsi_ctrlr_construct(const char *name, uint64_t cpumask) return 0; } +int +spdk_vhost_scsi_ctrlr_remove(struct spdk_vhost_scsi_ctrlr *vdev) +{ + unsigned ctrlr_num; + char path[PATH_MAX]; + int i; + + if (vdev->lcore != -1) { + SPDK_ERRLOG("Controller %s is in use and hotplug is not supported\n", vdev->name); + return -ENODEV; + } + + for (ctrlr_num = 0; ctrlr_num < MAX_SCSI_CTRLRS; ctrlr_num++) { + if (spdk_vhost_ctrlrs[ctrlr_num] == vdev) { + break; + } + } + + if (ctrlr_num == MAX_SCSI_CTRLRS) { + SPDK_ERRLOG("Trying to remove invalid controller: %s.\n", vdev->name); + return -ENOSPC; + } + + if (snprintf(path, sizeof(path), "%s%s", dev_dirname, vdev->name) >= (int)sizeof(path)) { + SPDK_ERRLOG("Resulting socket path for controller %s is too long: %s%s\n", vdev->name, dev_dirname, + vdev->name); + return -EINVAL; + } + + for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; ++i) { + if (vdev->scsi_dev[i]) { + SPDK_ERRLOG("Trying to remove non-empty controller: %s.\n", vdev->name); + return -EBUSY; + } + } + + if (rte_vhost_driver_unregister(path) != 0) { + SPDK_ERRLOG("Could not unregister controller %s with vhost library\n" + "Check if domain socket %s still exists\n", vdev->name, path); + return -EIO; + } + + SPDK_NOTICELOG("Controller %s: removed\n", vdev->name); + + free(vdev->name); + spdk_free(spdk_vhost_ctrlrs[ctrlr_num]); + spdk_vhost_ctrlrs[ctrlr_num] = NULL; + + return 0; +} + int spdk_vhost_parse_core_mask(const char *mask, uint64_t *cpumask) { @@ -1099,6 +1150,28 @@ spdk_vhost_scsi_ctrlr_add_dev(const char *ctrlr_name, unsigned scsi_dev_num, con return 0; } + +int +spdk_vhost_scsi_ctrlr_remove_dev(struct spdk_vhost_scsi_ctrlr *vdev, unsigned scsi_dev_num) +{ + if (vdev->lcore != -1) { + SPDK_ERRLOG("Controller %s is in use and hotremove is not supported\n", vdev->name); + return -EBUSY; + } + + if (vdev->scsi_dev[scsi_dev_num] == NULL) { + SPDK_ERRLOG("Controller %s dev %u is not occupied\n", vdev->name, scsi_dev_num); + return -ENODEV; + } + + spdk_scsi_dev_destruct(vdev->scsi_dev[scsi_dev_num]); + vdev->scsi_dev[scsi_dev_num] = NULL; + + SPDK_NOTICELOG("Controller %s: removed device 'Dev %u'\n", + vdev->name, scsi_dev_num); + return 0; +} + struct spdk_vhost_scsi_ctrlr * spdk_vhost_scsi_ctrlr_next(struct spdk_vhost_scsi_ctrlr *prev) { diff --git a/lib/vhost/vhost_rpc.c b/lib/vhost/vhost_rpc.c index 6b20bb5d68..39b377265d 100644 --- a/lib/vhost/vhost_rpc.c +++ b/lib/vhost/vhost_rpc.c @@ -186,6 +186,64 @@ invalid: } SPDK_RPC_REGISTER("construct_vhost_scsi_controller", spdk_rpc_construct_vhost_scsi_controller) +struct rpc_remove_vhost_scsi_ctrlr { + char *ctrlr; +}; + +static void +free_rpc_remove_vhost_scsi_ctrlr(struct rpc_remove_vhost_scsi_ctrlr *req) +{ + free(req->ctrlr); +} + +static const struct spdk_json_object_decoder rpc_remove_vhost_ctrlr[] = { + {"ctrlr", offsetof(struct rpc_remove_vhost_scsi_ctrlr, ctrlr), spdk_json_decode_string }, +}; + +static void +spdk_rpc_remove_vhost_scsi_controller(struct spdk_jsonrpc_server_conn *conn, + const struct spdk_json_val *params, + const struct spdk_json_val *id) +{ + struct rpc_remove_vhost_scsi_ctrlr req = {NULL}; + struct spdk_json_write_ctx *w; + struct spdk_vhost_scsi_ctrlr *vdev; + int rc; + + if (spdk_json_decode_object(params, rpc_remove_vhost_ctrlr, + SPDK_COUNTOF(rpc_remove_vhost_ctrlr), + &req)) { + SPDK_TRACELOG(SPDK_TRACE_DEBUG, "spdk_json_decode_object failed\n"); + rc = -EINVAL; + goto invalid; + } + + if (!(vdev = spdk_vhost_scsi_ctrlr_find(req.ctrlr))) { + rc = -ENODEV; + goto invalid; + } + + rc = spdk_vhost_scsi_ctrlr_remove(vdev); + if (rc < 0) { + goto invalid; + } + + free_rpc_remove_vhost_scsi_ctrlr(&req); + + if (id != NULL) { + w = spdk_jsonrpc_begin_result(conn, id); + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(conn, w); + } + + return; +invalid: + free_rpc_remove_vhost_scsi_ctrlr(&req); + spdk_jsonrpc_send_error_response(conn, id, SPDK_JSONRPC_ERROR_INVALID_PARAMS, strerror(-rc)); +} +SPDK_RPC_REGISTER("remove_vhost_scsi_controller", spdk_rpc_remove_vhost_scsi_controller) + + struct rpc_add_vhost_scsi_ctrlr_lun { char *ctrlr; uint32_t scsi_dev_num; @@ -229,12 +287,74 @@ spdk_rpc_add_vhost_scsi_lun(struct spdk_jsonrpc_server_conn *conn, free_rpc_add_vhost_scsi_ctrlr_lun(&req); - w = spdk_jsonrpc_begin_result(conn, id); - spdk_json_write_bool(w, true); - spdk_jsonrpc_end_result(conn, w); + if (id != NULL) { + w = spdk_jsonrpc_begin_result(conn, id); + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(conn, w); + } + return; invalid: free_rpc_add_vhost_scsi_ctrlr_lun(&req); spdk_jsonrpc_send_error_response(conn, id, SPDK_JSONRPC_ERROR_INVALID_PARAMS, strerror(-rc)); } SPDK_RPC_REGISTER("add_vhost_scsi_lun", spdk_rpc_add_vhost_scsi_lun) + +struct rpc_remove_vhost_scsi_ctrlr_dev { + char *ctrlr; + uint32_t scsi_dev_num; +}; + +static void +free_rpc_remove_vhost_scsi_ctrlr_dev(struct rpc_remove_vhost_scsi_ctrlr_dev *req) +{ + free(req->ctrlr); +} + +static const struct spdk_json_object_decoder rpc_vhost_remove_dev[] = { + {"ctrlr", offsetof(struct rpc_remove_vhost_scsi_ctrlr_dev, ctrlr), spdk_json_decode_string }, + {"scsi_dev_num", offsetof(struct rpc_remove_vhost_scsi_ctrlr_dev, scsi_dev_num), spdk_json_decode_uint32}, +}; + +static void +spdk_rpc_remove_vhost_scsi_dev(struct spdk_jsonrpc_server_conn *conn, + const struct spdk_json_val *params, + const struct spdk_json_val *id) +{ + struct rpc_remove_vhost_scsi_ctrlr_dev req = {0}; + struct spdk_json_write_ctx *w; + struct spdk_vhost_scsi_ctrlr *vdev; + int rc; + + if (spdk_json_decode_object(params, rpc_vhost_remove_dev, + SPDK_COUNTOF(rpc_vhost_remove_dev), + &req)) { + SPDK_TRACELOG(SPDK_TRACE_DEBUG, "spdk_json_decode_object failed\n"); + rc = -EINVAL; + goto invalid; + } + + if (!(vdev = spdk_vhost_scsi_ctrlr_find(req.ctrlr))) { + rc = -ENODEV; + goto invalid; + } + + rc = spdk_vhost_scsi_ctrlr_remove_dev(vdev, req.scsi_dev_num); + if (rc < 0) { + goto invalid; + } + + free_rpc_remove_vhost_scsi_ctrlr_dev(&req); + + if (id != NULL) { + w = spdk_jsonrpc_begin_result(conn, id); + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(conn, w); + } + + return; +invalid: + free_rpc_remove_vhost_scsi_ctrlr_dev(&req); + spdk_jsonrpc_send_error_response(conn, id, SPDK_JSONRPC_ERROR_INVALID_PARAMS, strerror(-rc)); +} +SPDK_RPC_REGISTER("remove_vhost_scsi_dev", spdk_rpc_remove_vhost_scsi_dev) diff --git a/scripts/rpc.py b/scripts/rpc.py index 0deaec67b9..7673dc2630 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -473,6 +473,14 @@ p.add_argument('ctrlr', help='controller name') p.add_argument('--cpumask', help='cpu mask for this controller') p.set_defaults(func=construct_vhost_scsi_controller) +def remove_vhost_scsi_controller(args): + params = {'ctrlr': args.ctrlr} + jsonrpc_call('remove_vhost_scsi_controller', params) + +p = subparsers.add_parser('remove_vhost_scsi_controller', help='Remove vhost controller') +p.add_argument('ctrlr', help='controller name') +p.set_defaults(func=remove_vhost_scsi_controller) + def add_vhost_scsi_lun(args): params = { 'ctrlr': args.ctrlr, @@ -487,5 +495,17 @@ p.add_argument('scsi_dev_num', help='scsi_dev_num', type=int) p.add_argument('lun_name', help='lun name') p.set_defaults(func=add_vhost_scsi_lun) +def remove_vhost_scsi_dev(args): + params = { + 'ctrlr': args.ctrlr, + 'scsi_dev_num': args.scsi_dev_num, + } + jsonrpc_call('remove_vhost_scsi_dev', params) + +p = subparsers.add_parser('remove_vhost_scsi_dev', help='Remove device from vhost controller') +p.add_argument('ctrlr', help='controller name to remove device from') +p.add_argument('scsi_dev_num', help='scsi_dev_num', type=int) +p.set_defaults(func=remove_vhost_scsi_dev) + args = parser.parse_args() args.func(args) diff --git a/test/vhost/fiotest/autotest.sh b/test/vhost/fiotest/autotest.sh index 2f32a8f7dc..62ea346018 100755 --- a/test/vhost/fiotest/autotest.sh +++ b/test/vhost/fiotest/autotest.sh @@ -152,12 +152,39 @@ for vm_conf in ${vms[@]}; do [[ x"${conf[2]}" != x"" ]] && setup_cmd+=" --disk=${conf[2]}" if [[ $test_type == "spdk_vhost" ]]; then + echo "INFO: Trying to remove inexistent controller" + if $rpc_py remove_vhost_scsi_controller unk0 > /dev/null; then + echo "ERROR: Removing inexistent controller succeeded, but it shouldn't" + false + fi + echo "INFO: Adding device via RPC ..." echo "" while IFS=':' read -ra disks; do for disk in "${disks[@]}"; do + echo "INFO: Creating controller naa.$disk.${conf[0]}" $rpc_py construct_vhost_scsi_controller naa.$disk.${conf[0]} + + echo "INFO: Adding initial device (0) to naa.$disk.${conf[0]}" + $rpc_py add_vhost_scsi_lun naa.$disk.${conf[0]} 0 $disk + + echo "INFO: Trying to remove inexistent device on existing controller" + if $rpc_py remove_vhost_scsi_dev naa.$disk.${conf[0]} 1 > /dev/null; then + echo "ERROR: Removing inexistent device (1) from controller naa.$disk.${conf[0]} succeeded, but it shouldn't" + false + fi + + echo "INFO: Trying to remove existing device from a controller" + $rpc_py remove_vhost_scsi_dev naa.$disk.${conf[0]} 0 + + echo "INFO: Trying to remove a just-deleted device from a controller again" + if $rpc_py remove_vhost_scsi_dev naa.$disk.${conf[0]} 0 > /dev/null; then + echo "ERROR: Removing device 0 from controller naa.$disk.${conf[0]} succeeded, but it shouldn't" + false + fi + + echo "INFO: Re-adding device 0 to naa.$disk.${conf[0]}" $rpc_py add_vhost_scsi_lun naa.$disk.${conf[0]} 0 $disk done done <<< "${conf[2]}" @@ -236,8 +263,30 @@ done if ! $no_shutdown; then echo "===============" + echo "INFO: APP EXITING" + echo "INFO: killing all VMs" + vm_kill_all + echo "INFO: waiting 2 seconds to let all VMs die" + sleep 2 + if [[ $test_type == "spdk_vhost" ]]; then + echo "INFO: Removing vhost devices & controllers via RPC ..." + for vm_conf in ${vms[@]}; do + IFS=',' read -ra conf <<< "$vm_conf" + + while IFS=':' read -ra disks; do + for disk in "${disks[@]}"; do + echo "INFO: Removing all vhost devices from controller naa.$disk.${conf[0]}" + $rpc_py remove_vhost_scsi_dev naa.$disk.${conf[0]} 0 + $rpc_py remove_vhost_scsi_controller naa.$disk.${conf[0]} + done + done <<< "${conf[2]}" + done + fi echo "INFO: Testing done -> shutting down" - at_app_exit + echo "INFO: killing vhost app" + spdk_vhost_kill + + echo "INFO: EXIT DONE" echo "===============" else echo "==============="