From 6b7e9d0af2bdc62d79116a4eb0cffc33e33d8471 Mon Sep 17 00:00:00 2001 From: GangCao Date: Fri, 14 Jan 2022 11:32:15 -0500 Subject: [PATCH] Lib/iSCSI: add the LUN Resize support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From SAM-4, section 5.13 (Sense Data); “When a command terminates with a CHECK CONDITION status, sense data shall be returned in the same I_T_L_Q nexus transaction (see 3.1.50) as the CHECK CONDITION status. After the sense data is returned, it shall be cleared except when it is associated with a unit attention condition and the UA_INTLCK_CTRL field in the Control mode page (see SPC-4) contains 10b or 11b.” SPDK does not set UA_INTLCK_CTRL to 10b or 11b, so we set the unit attention condition immediately against a single IO or Admin IO after reporting it via a CHECK CONDITION. Once the failed IO received at iSCSI initiator side, it will be retried. In the case of resize operation, if there is no IO from iSCSI initiator side, the unit attention condition will be delayed to report until the first IO is received at the iSCSI target side. Meanwhile, we clear the resizing (newly added) flag on our SCSI LUN structure after first time we report the resize unit attention condition. The kernel initiator won’t actually resize the corresponding block device automatically. It will report a uevent, and then you can set up udev rules to trigger a rescan. SPDK iSCSI initiator will automatically report the LUN size change. Change-Id: Ifc85b8d4d3fbea13e76fb5d1faf1ac6c8f662e6c Signed-off-by: GangCao Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/11086 Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Xiaodong Liu Reviewed-by: Changpeng Liu Community-CI: Broadcom CI --- CHANGELOG.md | 5 ++ include/spdk/scsi_spec.h | 2 + lib/scsi/lun.c | 34 +++++++-- lib/scsi/scsi_internal.h | 3 + module/bdev/iscsi/bdev_iscsi.c | 110 +++++++++++++++++++++++++++++- test/iscsi_tgt/iscsi_tgt.sh | 1 + test/iscsi_tgt/resize/resize.sh | 73 ++++++++++++++++++++ test/unit/lib/scsi/lun.c/lun_ut.c | 39 +++++++++++ 8 files changed, 262 insertions(+), 5 deletions(-) create mode 100755 test/iscsi_tgt/resize/resize.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 975b2ec7f9..64c97d4b64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,11 @@ Added 3 experimental APIs to handle PCI device interrupts (`spdk_pci_device_enab Added a 'subsystem' parameter to spdk_nvmf_transport_stop_listen_async. When not NULL, it will only disconnect qpairs for controllers associated with the specified subsystem. +### scsi + +Structure `spdk_scsi_lun` has been extended with new member `resizing` so that SCSI layer now reports +unit attention for disk resize. + ## v21.10 Structure `spdk_nvmf_target_opts` has been extended with new member `discovery_filter` which allows to specify diff --git a/include/spdk/scsi_spec.h b/include/spdk/scsi_spec.h index 2711c8ea44..8150cac7e2 100644 --- a/include/spdk/scsi_spec.h +++ b/include/spdk/scsi_spec.h @@ -101,6 +101,7 @@ enum spdk_scsi_asc { SPDK_SCSI_ASC_INVALID_FIELD_IN_CDB = 0x24, SPDK_SCSI_ASC_LOGICAL_UNIT_NOT_SUPPORTED = 0x25, SPDK_SCSI_ASC_WRITE_PROTECTED = 0x27, + SPDK_SCSI_ASC_CAPACITY_DATA_HAS_CHANGED = 0x2a, SPDK_SCSI_ASC_FORMAT_COMMAND_FAILED = 0x31, SPDK_SCSI_ASC_SAVING_PARAMETERS_NOT_SUPPORTED = 0x39, SPDK_SCSI_ASC_INTERNAL_TARGET_FAILURE = 0x44, @@ -116,6 +117,7 @@ enum spdk_scsi_ascq { SPDK_SCSI_ASCQ_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED = 0x03, SPDK_SCSI_ASCQ_POWER_LOSS_EXPECTED = 0x08, SPDK_SCSI_ASCQ_INVALID_LU_IDENTIFIER = 0x09, + SPDK_SCSI_ASCQ_CAPACITY_DATA_HAS_CHANGED = 0x09, }; enum spdk_spc_opcode { diff --git a/lib/scsi/lun.c b/lib/scsi/lun.c index 4690507099..e8e81fe083 100644 --- a/lib/scsi/lun.c +++ b/lib/scsi/lun.c @@ -125,6 +125,23 @@ scsi_lun_append_mgmt_task(struct spdk_scsi_lun *lun, TAILQ_INSERT_TAIL(&lun->pending_mgmt_tasks, task, scsi_link); } +static bool +_scsi_lun_handle_unit_attention(struct spdk_scsi_task *task) +{ + uint8_t *cdb = task->cdb; + + assert(task->cdb); + + switch (cdb[0]) { + case SPDK_SPC_INQUIRY: + case SPDK_SPC_REPORT_LUNS: + case SPDK_SPC_REQUEST_SENSE: + return false; + default: + return true; + } +} + static void _scsi_lun_execute_mgmt_task(struct spdk_scsi_lun *lun) { @@ -195,7 +212,17 @@ _scsi_lun_execute_task(struct spdk_scsi_lun *lun, struct spdk_scsi_task *task) task->status = SPDK_SCSI_STATUS_GOOD; spdk_trace_record(TRACE_SCSI_TASK_START, lun->dev->id, task->length, (uintptr_t)task); TAILQ_INSERT_TAIL(&lun->tasks, task, scsi_link); - if (!lun->removed) { + if (spdk_unlikely(lun->removed)) { + spdk_scsi_task_process_abort(task); + rc = SPDK_SCSI_TASK_COMPLETE; + } else if (spdk_unlikely(lun->resizing) && _scsi_lun_handle_unit_attention(task)) { + spdk_scsi_task_set_status(task, SPDK_SCSI_STATUS_CHECK_CONDITION, + SPDK_SCSI_SENSE_UNIT_ATTENTION, + SPDK_SCSI_ASC_CAPACITY_DATA_HAS_CHANGED, + SPDK_SCSI_ASCQ_CAPACITY_DATA_HAS_CHANGED); + lun->resizing = false; + rc = SPDK_SCSI_TASK_COMPLETE; + } else { /* Check the command is allowed or not when reservation is exist */ if (spdk_unlikely(lun->reservation.flags & SCSI_SPC2_RESERVE)) { rc = scsi2_reserve_check(task); @@ -208,9 +235,6 @@ _scsi_lun_execute_task(struct spdk_scsi_lun *lun, struct spdk_scsi_task *task) } else { rc = bdev_scsi_execute(task); } - } else { - spdk_scsi_task_process_abort(task); - rc = SPDK_SCSI_TASK_COMPLETE; } switch (rc) { @@ -405,6 +429,7 @@ bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, break; case SPDK_BDEV_EVENT_RESIZE: SPDK_NOTICELOG("bdev name (%s) received event(SPDK_BDEV_EVENT_RESIZE)\n", spdk_bdev_get_name(bdev)); + lun->resizing = true; if (lun->resize_cb) { lun->resize_cb(lun, lun->resize_ctx); } @@ -466,6 +491,7 @@ struct spdk_scsi_lun *scsi_lun_construct(const char *bdev_name, lun->resize_cb = resize_cb; lun->resize_ctx = resize_ctx; + lun->resizing = false; TAILQ_INIT(&lun->open_descs); TAILQ_INIT(&lun->reg_head); diff --git a/lib/scsi/scsi_internal.h b/lib/scsi/scsi_internal.h index cf7249507a..ac8d7ca5df 100644 --- a/lib/scsi/scsi_internal.h +++ b/lib/scsi/scsi_internal.h @@ -178,6 +178,9 @@ struct spdk_scsi_lun { /** A structure to connect LUNs in a list. */ TAILQ_ENTRY(spdk_scsi_lun) tailq; + + /** The LUN is resizing */ + bool resizing; }; struct spdk_scsi_lun *scsi_lun_construct(const char *bdev_name, diff --git a/module/bdev/iscsi/bdev_iscsi.c b/module/bdev/iscsi/bdev_iscsi.c index d8e288440a..6307b58275 100644 --- a/module/bdev/iscsi/bdev_iscsi.c +++ b/module/bdev/iscsi/bdev_iscsi.c @@ -72,12 +72,16 @@ struct bdev_iscsi_lun; #define BDEV_ISCSI_MAX_UNMAP_BLOCK_DESCS_COUNT (1) static int bdev_iscsi_initialize(void); +static void bdev_iscsi_readcapacity16(struct iscsi_context *context, struct bdev_iscsi_lun *lun); +static void _bdev_iscsi_submit_request(void *_bdev_io); + static TAILQ_HEAD(, bdev_iscsi_conn_req) g_iscsi_conn_req = TAILQ_HEAD_INITIALIZER( g_iscsi_conn_req); static struct spdk_poller *g_conn_poller = NULL; struct bdev_iscsi_io { struct spdk_thread *submit_td; + struct bdev_iscsi_lun *lun; enum spdk_bdev_io_status status; int scsi_status; enum spdk_scsi_sense sk; @@ -217,12 +221,26 @@ bdev_iscsi_io_complete(struct bdev_iscsi_io *iscsi_io, enum spdk_bdev_io_status } } +static bool +_bdev_iscsi_is_size_change(int status, struct scsi_task *task) +{ + if (status == SPDK_SCSI_STATUS_CHECK_CONDITION && + (uint8_t)task->sense.key == SPDK_SCSI_SENSE_UNIT_ATTENTION && + task->sense.ascq == 0x2a09) { + /* ASCQ: SCSI_SENSE_ASCQ_CAPACITY_DATA_HAS_CHANGED (0x2a09) */ + return true; + } + + return false; +} + /* Common call back function for read/write/flush command */ static void bdev_iscsi_command_cb(struct iscsi_context *context, int status, void *_task, void *_iscsi_io) { struct scsi_task *task = _task; struct bdev_iscsi_io *iscsi_io = _iscsi_io; + struct spdk_bdev_io *bdev_io; iscsi_io->scsi_status = status; iscsi_io->sk = (uint8_t)task->sense.key; @@ -230,7 +248,86 @@ bdev_iscsi_command_cb(struct iscsi_context *context, int status, void *_task, vo iscsi_io->ascq = task->sense.ascq & 0xFF; scsi_free_scsi_task(task); - bdev_iscsi_io_complete(iscsi_io, SPDK_BDEV_IO_STATUS_SUCCESS); + + if (_bdev_iscsi_is_size_change(status, task)) { + bdev_iscsi_readcapacity16(context, iscsi_io->lun); + + /* Retry this failed IO immediately */ + bdev_io = spdk_bdev_io_from_ctx(iscsi_io); + if (iscsi_io->submit_td != NULL) { + spdk_thread_send_msg(iscsi_io->lun->main_td, + _bdev_iscsi_submit_request, bdev_io); + } else { + _bdev_iscsi_submit_request(bdev_io); + } + } else { + bdev_iscsi_io_complete(iscsi_io, SPDK_BDEV_IO_STATUS_SUCCESS); + } +} + +static int +bdev_iscsi_resize(struct spdk_bdev *bdev, const uint64_t new_size_in_block) +{ + int rc; + + assert(bdev->module == &g_iscsi_bdev_module); + + if (new_size_in_block <= bdev->blockcnt) { + SPDK_ERRLOG("The new bdev size must be larger than current bdev size.\n"); + return -EINVAL; + } + + rc = spdk_bdev_notify_blockcnt_change(bdev, new_size_in_block); + if (rc != 0) { + SPDK_ERRLOG("failed to notify block cnt change.\n"); + return rc; + } + + return 0; +} + +static void +bdev_iscsi_readcapacity16_cb(struct iscsi_context *context, int status, void *_task, + void *private_data) +{ + struct bdev_iscsi_lun *lun = private_data; + struct scsi_readcapacity16 *readcap16; + struct scsi_task *task = _task; + uint64_t size_in_block = 0; + int rc; + + if (status != SPDK_SCSI_STATUS_GOOD) { + SPDK_ERRLOG("iSCSI error: %s\n", iscsi_get_error(context)); + goto ret; + } + + readcap16 = scsi_datain_unmarshall(task); + if (!readcap16) { + SPDK_ERRLOG("Read capacity error\n"); + goto ret; + } + + size_in_block = readcap16->returned_lba + 1; + + rc = bdev_iscsi_resize(&lun->bdev, size_in_block); + if (rc != 0) { + SPDK_ERRLOG("Bdev (%s) resize error: %d\n", lun->bdev.name, rc); + } + +ret: + scsi_free_scsi_task(task); +} + +static void +bdev_iscsi_readcapacity16(struct iscsi_context *context, struct bdev_iscsi_lun *lun) +{ + struct scsi_task *task; + + task = iscsi_readcapacity16_task(context, lun->lun_id, + bdev_iscsi_readcapacity16_cb, lun); + if (task == NULL) { + SPDK_ERRLOG("failed to get readcapacity16_task\n"); + } } static void @@ -510,6 +607,8 @@ static void bdev_iscsi_submit_request(struct spdk_io_channel *_ch, struct spdk_b struct bdev_iscsi_io *iscsi_io = (struct bdev_iscsi_io *)bdev_io->driver_ctx; struct bdev_iscsi_lun *lun = (struct bdev_iscsi_lun *)bdev_io->bdev->ctxt; + iscsi_io->lun = lun; + if (lun->main_td != submit_td) { iscsi_io->submit_td = submit_td; spdk_thread_send_msg(lun->main_td, _bdev_iscsi_submit_request, bdev_io); @@ -722,9 +821,18 @@ iscsi_readcapacity16_cb(struct iscsi_context *iscsi, int status, struct scsi_readcapacity16 *readcap16; struct spdk_bdev *bdev = NULL; struct scsi_task *task = command_data; + struct scsi_task *retry_task = NULL; if (status != SPDK_SCSI_STATUS_GOOD) { SPDK_ERRLOG("iSCSI error: %s\n", iscsi_get_error(iscsi)); + if (_bdev_iscsi_is_size_change(status, task)) { + scsi_free_scsi_task(task); + retry_task = iscsi_readcapacity16_task(iscsi, req->lun, + iscsi_readcapacity16_cb, req); + if (retry_task) { + return; + } + } goto ret; } diff --git a/test/iscsi_tgt/iscsi_tgt.sh b/test/iscsi_tgt/iscsi_tgt.sh index e3ee13254c..10297458c9 100755 --- a/test/iscsi_tgt/iscsi_tgt.sh +++ b/test/iscsi_tgt/iscsi_tgt.sh @@ -64,6 +64,7 @@ fi if [ $SPDK_TEST_ISCSI_INITIATOR -eq 1 ]; then run_test "iscsi_tgt_initiator" ./test/iscsi_tgt/initiator/initiator.sh run_test "iscsi_tgt_bdev_io_wait" ./test/iscsi_tgt/bdev_io_wait/bdev_io_wait.sh + run_test "iscsi_tgt_resize" ./test/iscsi_tgt/resize/resize.sh fi cleanup_veth_interfaces diff --git a/test/iscsi_tgt/resize/resize.sh b/test/iscsi_tgt/resize/resize.sh new file mode 100755 index 0000000000..fa4dbc45bf --- /dev/null +++ b/test/iscsi_tgt/resize/resize.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/iscsi_tgt/common.sh + +iscsitestinit + +BDEV_SIZE=64 +BDEV_NEW_SIZE=128 +BLOCK_SIZE=512 +RESIZE_SOCK="/var/tmp/spdk-resize.sock" + +rpc_py="$rootdir/scripts/rpc.py" + +timing_enter start_iscsi_tgt + +# Remove the sock file first +rm -f $RESIZE_SOCK + +"${ISCSI_APP[@]}" -m 0x2 -p 1 -s 512 --wait-for-rpc & +pid=$! +echo "iSCSI target launched. pid: $pid" +trap 'killprocess $pid; iscsitestfini; exit 1' SIGINT SIGTERM EXIT +waitforlisten $pid +$rpc_py framework_start_init +echo "iscsi_tgt is listening. Running tests..." + +timing_exit start_iscsi_tgt + +$rpc_py iscsi_create_portal_group $PORTAL_TAG $TARGET_IP:$ISCSI_PORT +$rpc_py iscsi_create_initiator_group $INITIATOR_TAG $INITIATOR_NAME $NETMASK +$rpc_py bdev_null_create Null0 $BDEV_SIZE $BLOCK_SIZE +# "Null0:0" ==> use Null0 blockdev for LUN0 +# "1:2" ==> map PortalGroup1 to InitiatorGroup2 +# "64" ==> iSCSI queue depth 64 +# "-d" ==> disable CHAP authentication +$rpc_py iscsi_create_target_node disk1 disk1_alias 'Null0:0' $PORTAL_TAG:$INITIATOR_TAG 256 -d +sleep 1 +trap 'killprocess $pid; iscsitestfini; exit 1' SIGINT SIGTERM EXIT + +# Start bdevperf with another sock file and iSCSI initiator +"$rootdir/test/bdev/bdevperf/bdevperf" -r $RESIZE_SOCK --json <(initiator_json_config) -q 16 -o 4096 -w read -t 5 -R -s 128 -z & +bdevperf_pid=$! +waitforlisten $bdevperf_pid $RESIZE_SOCK +# Resize the Bdev from iSCSI target +$rpc_py bdev_null_resize Null0 $BDEV_NEW_SIZE +# Obtain the Bdev from bdevperf with iSCSI initiator +num_block=$($rpc_py -s $RESIZE_SOCK bdev_get_bdevs | grep num_blocks | sed 's/[^[:digit:]]//g') +# Size is not changed as no IO sent yet and resize notification is deferred. +total_size=$((num_block * BLOCK_SIZE / 1048576)) +if [ $total_size != $BDEV_SIZE ]; then + echo "resize failed" + exit 1 +fi +sleep 2 +# Start the bdevperf IO +$rootdir/test/bdev/bdevperf/bdevperf.py -s $RESIZE_SOCK perform_tests +# Obtain the Bdev from bdevperf with iSCSI initiator +num_block=$($rpc_py -s $RESIZE_SOCK bdev_get_bdevs | grep num_blocks | sed 's/[^[:digit:]]//g') +# Get the new bdev size in MiB. +total_size=$((num_block * BLOCK_SIZE / 1048576)) +if [ $total_size != $BDEV_NEW_SIZE ]; then + echo "resize failed" + exit 1 +fi + +trap - SIGINT SIGTERM EXIT +killprocess $bdevperf_pid +killprocess $pid + +iscsitestfini diff --git a/test/unit/lib/scsi/lun.c/lun_ut.c b/test/unit/lib/scsi/lun.c/lun_ut.c index 8acc7654f0..fad3871522 100644 --- a/test/unit/lib/scsi/lun.c/lun_ut.c +++ b/test/unit/lib/scsi/lun.c/lun_ut.c @@ -431,6 +431,44 @@ lun_execute_scsi_task_complete(void) CU_ASSERT_EQUAL(g_task_count, 0); } +static void +lun_execute_scsi_task_resize(void) +{ + struct spdk_scsi_lun *lun; + struct spdk_scsi_task task = { 0 }; + struct spdk_scsi_dev dev = { 0 }; + uint8_t cdb = SPDK_SBC_READ_16; + + lun = lun_construct(); + + ut_init_task(&task); + task.lun = lun; + task.cdb = &cdb; + lun->dev = &dev; + lun->resizing = true; + + /* the tasks list should still be empty since it has not been + executed yet + */ + CU_ASSERT(TAILQ_EMPTY(&lun->tasks)); + + scsi_lun_execute_task(lun, &task); + CU_ASSERT_EQUAL(task.status, SPDK_SCSI_STATUS_CHECK_CONDITION); + /* SENSE KEY */ + CU_ASSERT_EQUAL(task.sense_data[2], SPDK_SCSI_SENSE_UNIT_ATTENTION); + /* SCSI_SENSE_ASCQ_CAPACITY_DATA_HAS_CHANGED: 0x2a09 */ + CU_ASSERT_EQUAL(task.sense_data[12], SPDK_SCSI_ASC_CAPACITY_DATA_HAS_CHANGED); + CU_ASSERT_EQUAL(task.sense_data[13], SPDK_SCSI_ASCQ_CAPACITY_DATA_HAS_CHANGED); + CU_ASSERT(lun->resizing == false); + + /* Assert the task has not been added to the tasks queue */ + CU_ASSERT(TAILQ_EMPTY(&lun->tasks)); + + lun_destruct(lun); + + CU_ASSERT_EQUAL(g_task_count, 0); +} + static void lun_destruct_success(void) { @@ -727,6 +765,7 @@ main(int argc, char **argv) CU_ADD_TEST(suite, lun_append_task_null_lun_not_supported); CU_ADD_TEST(suite, lun_execute_scsi_task_pending); CU_ADD_TEST(suite, lun_execute_scsi_task_complete); + CU_ADD_TEST(suite, lun_execute_scsi_task_resize); CU_ADD_TEST(suite, lun_destruct_success); CU_ADD_TEST(suite, lun_construct_null_ctx); CU_ADD_TEST(suite, lun_construct_success);