diff --git a/CHANGELOG.md b/CHANGELOG.md index 975b2ec7f9..64c97d4b64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,11 @@ Added 3 experimental APIs to handle PCI device interrupts (`spdk_pci_device_enab Added a 'subsystem' parameter to spdk_nvmf_transport_stop_listen_async. When not NULL, it will only disconnect qpairs for controllers associated with the specified subsystem. +### scsi + +Structure `spdk_scsi_lun` has been extended with new member `resizing` so that SCSI layer now reports +unit attention for disk resize. + ## v21.10 Structure `spdk_nvmf_target_opts` has been extended with new member `discovery_filter` which allows to specify diff --git a/include/spdk/scsi_spec.h b/include/spdk/scsi_spec.h index 2711c8ea44..8150cac7e2 100644 --- a/include/spdk/scsi_spec.h +++ b/include/spdk/scsi_spec.h @@ -101,6 +101,7 @@ enum spdk_scsi_asc { SPDK_SCSI_ASC_INVALID_FIELD_IN_CDB = 0x24, SPDK_SCSI_ASC_LOGICAL_UNIT_NOT_SUPPORTED = 0x25, SPDK_SCSI_ASC_WRITE_PROTECTED = 0x27, + SPDK_SCSI_ASC_CAPACITY_DATA_HAS_CHANGED = 0x2a, SPDK_SCSI_ASC_FORMAT_COMMAND_FAILED = 0x31, SPDK_SCSI_ASC_SAVING_PARAMETERS_NOT_SUPPORTED = 0x39, SPDK_SCSI_ASC_INTERNAL_TARGET_FAILURE = 0x44, @@ -116,6 +117,7 @@ enum spdk_scsi_ascq { SPDK_SCSI_ASCQ_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED = 0x03, SPDK_SCSI_ASCQ_POWER_LOSS_EXPECTED = 0x08, SPDK_SCSI_ASCQ_INVALID_LU_IDENTIFIER = 0x09, + SPDK_SCSI_ASCQ_CAPACITY_DATA_HAS_CHANGED = 0x09, }; enum spdk_spc_opcode { diff --git a/lib/scsi/lun.c b/lib/scsi/lun.c index 4690507099..e8e81fe083 100644 --- a/lib/scsi/lun.c +++ b/lib/scsi/lun.c @@ -125,6 +125,23 @@ scsi_lun_append_mgmt_task(struct spdk_scsi_lun *lun, TAILQ_INSERT_TAIL(&lun->pending_mgmt_tasks, task, scsi_link); } +static bool +_scsi_lun_handle_unit_attention(struct spdk_scsi_task *task) +{ + uint8_t *cdb = task->cdb; + + assert(task->cdb); + + switch (cdb[0]) { + case SPDK_SPC_INQUIRY: + case SPDK_SPC_REPORT_LUNS: + case SPDK_SPC_REQUEST_SENSE: + return false; + default: + return true; + } +} + static void _scsi_lun_execute_mgmt_task(struct spdk_scsi_lun *lun) { @@ -195,7 +212,17 @@ _scsi_lun_execute_task(struct spdk_scsi_lun *lun, struct spdk_scsi_task *task) task->status = SPDK_SCSI_STATUS_GOOD; spdk_trace_record(TRACE_SCSI_TASK_START, lun->dev->id, task->length, (uintptr_t)task); TAILQ_INSERT_TAIL(&lun->tasks, task, scsi_link); - if (!lun->removed) { + if (spdk_unlikely(lun->removed)) { + spdk_scsi_task_process_abort(task); + rc = SPDK_SCSI_TASK_COMPLETE; + } else if (spdk_unlikely(lun->resizing) && _scsi_lun_handle_unit_attention(task)) { + spdk_scsi_task_set_status(task, SPDK_SCSI_STATUS_CHECK_CONDITION, + SPDK_SCSI_SENSE_UNIT_ATTENTION, + SPDK_SCSI_ASC_CAPACITY_DATA_HAS_CHANGED, + SPDK_SCSI_ASCQ_CAPACITY_DATA_HAS_CHANGED); + lun->resizing = false; + rc = SPDK_SCSI_TASK_COMPLETE; + } else { /* Check the command is allowed or not when reservation is exist */ if (spdk_unlikely(lun->reservation.flags & SCSI_SPC2_RESERVE)) { rc = scsi2_reserve_check(task); @@ -208,9 +235,6 @@ _scsi_lun_execute_task(struct spdk_scsi_lun *lun, struct spdk_scsi_task *task) } else { rc = bdev_scsi_execute(task); } - } else { - spdk_scsi_task_process_abort(task); - rc = SPDK_SCSI_TASK_COMPLETE; } switch (rc) { @@ -405,6 +429,7 @@ bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, break; case SPDK_BDEV_EVENT_RESIZE: SPDK_NOTICELOG("bdev name (%s) received event(SPDK_BDEV_EVENT_RESIZE)\n", spdk_bdev_get_name(bdev)); + lun->resizing = true; if (lun->resize_cb) { lun->resize_cb(lun, lun->resize_ctx); } @@ -466,6 +491,7 @@ struct spdk_scsi_lun *scsi_lun_construct(const char *bdev_name, lun->resize_cb = resize_cb; lun->resize_ctx = resize_ctx; + lun->resizing = false; TAILQ_INIT(&lun->open_descs); TAILQ_INIT(&lun->reg_head); diff --git a/lib/scsi/scsi_internal.h b/lib/scsi/scsi_internal.h index cf7249507a..ac8d7ca5df 100644 --- a/lib/scsi/scsi_internal.h +++ b/lib/scsi/scsi_internal.h @@ -178,6 +178,9 @@ struct spdk_scsi_lun { /** A structure to connect LUNs in a list. */ TAILQ_ENTRY(spdk_scsi_lun) tailq; + + /** The LUN is resizing */ + bool resizing; }; struct spdk_scsi_lun *scsi_lun_construct(const char *bdev_name, diff --git a/module/bdev/iscsi/bdev_iscsi.c b/module/bdev/iscsi/bdev_iscsi.c index d8e288440a..6307b58275 100644 --- a/module/bdev/iscsi/bdev_iscsi.c +++ b/module/bdev/iscsi/bdev_iscsi.c @@ -72,12 +72,16 @@ struct bdev_iscsi_lun; #define BDEV_ISCSI_MAX_UNMAP_BLOCK_DESCS_COUNT (1) static int bdev_iscsi_initialize(void); +static void bdev_iscsi_readcapacity16(struct iscsi_context *context, struct bdev_iscsi_lun *lun); +static void _bdev_iscsi_submit_request(void *_bdev_io); + static TAILQ_HEAD(, bdev_iscsi_conn_req) g_iscsi_conn_req = TAILQ_HEAD_INITIALIZER( g_iscsi_conn_req); static struct spdk_poller *g_conn_poller = NULL; struct bdev_iscsi_io { struct spdk_thread *submit_td; + struct bdev_iscsi_lun *lun; enum spdk_bdev_io_status status; int scsi_status; enum spdk_scsi_sense sk; @@ -217,12 +221,26 @@ bdev_iscsi_io_complete(struct bdev_iscsi_io *iscsi_io, enum spdk_bdev_io_status } } +static bool +_bdev_iscsi_is_size_change(int status, struct scsi_task *task) +{ + if (status == SPDK_SCSI_STATUS_CHECK_CONDITION && + (uint8_t)task->sense.key == SPDK_SCSI_SENSE_UNIT_ATTENTION && + task->sense.ascq == 0x2a09) { + /* ASCQ: SCSI_SENSE_ASCQ_CAPACITY_DATA_HAS_CHANGED (0x2a09) */ + return true; + } + + return false; +} + /* Common call back function for read/write/flush command */ static void bdev_iscsi_command_cb(struct iscsi_context *context, int status, void *_task, void *_iscsi_io) { struct scsi_task *task = _task; struct bdev_iscsi_io *iscsi_io = _iscsi_io; + struct spdk_bdev_io *bdev_io; iscsi_io->scsi_status = status; iscsi_io->sk = (uint8_t)task->sense.key; @@ -230,7 +248,86 @@ bdev_iscsi_command_cb(struct iscsi_context *context, int status, void *_task, vo iscsi_io->ascq = task->sense.ascq & 0xFF; scsi_free_scsi_task(task); - bdev_iscsi_io_complete(iscsi_io, SPDK_BDEV_IO_STATUS_SUCCESS); + + if (_bdev_iscsi_is_size_change(status, task)) { + bdev_iscsi_readcapacity16(context, iscsi_io->lun); + + /* Retry this failed IO immediately */ + bdev_io = spdk_bdev_io_from_ctx(iscsi_io); + if (iscsi_io->submit_td != NULL) { + spdk_thread_send_msg(iscsi_io->lun->main_td, + _bdev_iscsi_submit_request, bdev_io); + } else { + _bdev_iscsi_submit_request(bdev_io); + } + } else { + bdev_iscsi_io_complete(iscsi_io, SPDK_BDEV_IO_STATUS_SUCCESS); + } +} + +static int +bdev_iscsi_resize(struct spdk_bdev *bdev, const uint64_t new_size_in_block) +{ + int rc; + + assert(bdev->module == &g_iscsi_bdev_module); + + if (new_size_in_block <= bdev->blockcnt) { + SPDK_ERRLOG("The new bdev size must be larger than current bdev size.\n"); + return -EINVAL; + } + + rc = spdk_bdev_notify_blockcnt_change(bdev, new_size_in_block); + if (rc != 0) { + SPDK_ERRLOG("failed to notify block cnt change.\n"); + return rc; + } + + return 0; +} + +static void +bdev_iscsi_readcapacity16_cb(struct iscsi_context *context, int status, void *_task, + void *private_data) +{ + struct bdev_iscsi_lun *lun = private_data; + struct scsi_readcapacity16 *readcap16; + struct scsi_task *task = _task; + uint64_t size_in_block = 0; + int rc; + + if (status != SPDK_SCSI_STATUS_GOOD) { + SPDK_ERRLOG("iSCSI error: %s\n", iscsi_get_error(context)); + goto ret; + } + + readcap16 = scsi_datain_unmarshall(task); + if (!readcap16) { + SPDK_ERRLOG("Read capacity error\n"); + goto ret; + } + + size_in_block = readcap16->returned_lba + 1; + + rc = bdev_iscsi_resize(&lun->bdev, size_in_block); + if (rc != 0) { + SPDK_ERRLOG("Bdev (%s) resize error: %d\n", lun->bdev.name, rc); + } + +ret: + scsi_free_scsi_task(task); +} + +static void +bdev_iscsi_readcapacity16(struct iscsi_context *context, struct bdev_iscsi_lun *lun) +{ + struct scsi_task *task; + + task = iscsi_readcapacity16_task(context, lun->lun_id, + bdev_iscsi_readcapacity16_cb, lun); + if (task == NULL) { + SPDK_ERRLOG("failed to get readcapacity16_task\n"); + } } static void @@ -510,6 +607,8 @@ static void bdev_iscsi_submit_request(struct spdk_io_channel *_ch, struct spdk_b struct bdev_iscsi_io *iscsi_io = (struct bdev_iscsi_io *)bdev_io->driver_ctx; struct bdev_iscsi_lun *lun = (struct bdev_iscsi_lun *)bdev_io->bdev->ctxt; + iscsi_io->lun = lun; + if (lun->main_td != submit_td) { iscsi_io->submit_td = submit_td; spdk_thread_send_msg(lun->main_td, _bdev_iscsi_submit_request, bdev_io); @@ -722,9 +821,18 @@ iscsi_readcapacity16_cb(struct iscsi_context *iscsi, int status, struct scsi_readcapacity16 *readcap16; struct spdk_bdev *bdev = NULL; struct scsi_task *task = command_data; + struct scsi_task *retry_task = NULL; if (status != SPDK_SCSI_STATUS_GOOD) { SPDK_ERRLOG("iSCSI error: %s\n", iscsi_get_error(iscsi)); + if (_bdev_iscsi_is_size_change(status, task)) { + scsi_free_scsi_task(task); + retry_task = iscsi_readcapacity16_task(iscsi, req->lun, + iscsi_readcapacity16_cb, req); + if (retry_task) { + return; + } + } goto ret; } diff --git a/test/iscsi_tgt/iscsi_tgt.sh b/test/iscsi_tgt/iscsi_tgt.sh index e3ee13254c..10297458c9 100755 --- a/test/iscsi_tgt/iscsi_tgt.sh +++ b/test/iscsi_tgt/iscsi_tgt.sh @@ -64,6 +64,7 @@ fi if [ $SPDK_TEST_ISCSI_INITIATOR -eq 1 ]; then run_test "iscsi_tgt_initiator" ./test/iscsi_tgt/initiator/initiator.sh run_test "iscsi_tgt_bdev_io_wait" ./test/iscsi_tgt/bdev_io_wait/bdev_io_wait.sh + run_test "iscsi_tgt_resize" ./test/iscsi_tgt/resize/resize.sh fi cleanup_veth_interfaces diff --git a/test/iscsi_tgt/resize/resize.sh b/test/iscsi_tgt/resize/resize.sh new file mode 100755 index 0000000000..fa4dbc45bf --- /dev/null +++ b/test/iscsi_tgt/resize/resize.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/iscsi_tgt/common.sh + +iscsitestinit + +BDEV_SIZE=64 +BDEV_NEW_SIZE=128 +BLOCK_SIZE=512 +RESIZE_SOCK="/var/tmp/spdk-resize.sock" + +rpc_py="$rootdir/scripts/rpc.py" + +timing_enter start_iscsi_tgt + +# Remove the sock file first +rm -f $RESIZE_SOCK + +"${ISCSI_APP[@]}" -m 0x2 -p 1 -s 512 --wait-for-rpc & +pid=$! +echo "iSCSI target launched. pid: $pid" +trap 'killprocess $pid; iscsitestfini; exit 1' SIGINT SIGTERM EXIT +waitforlisten $pid +$rpc_py framework_start_init +echo "iscsi_tgt is listening. Running tests..." + +timing_exit start_iscsi_tgt + +$rpc_py iscsi_create_portal_group $PORTAL_TAG $TARGET_IP:$ISCSI_PORT +$rpc_py iscsi_create_initiator_group $INITIATOR_TAG $INITIATOR_NAME $NETMASK +$rpc_py bdev_null_create Null0 $BDEV_SIZE $BLOCK_SIZE +# "Null0:0" ==> use Null0 blockdev for LUN0 +# "1:2" ==> map PortalGroup1 to InitiatorGroup2 +# "64" ==> iSCSI queue depth 64 +# "-d" ==> disable CHAP authentication +$rpc_py iscsi_create_target_node disk1 disk1_alias 'Null0:0' $PORTAL_TAG:$INITIATOR_TAG 256 -d +sleep 1 +trap 'killprocess $pid; iscsitestfini; exit 1' SIGINT SIGTERM EXIT + +# Start bdevperf with another sock file and iSCSI initiator +"$rootdir/test/bdev/bdevperf/bdevperf" -r $RESIZE_SOCK --json <(initiator_json_config) -q 16 -o 4096 -w read -t 5 -R -s 128 -z & +bdevperf_pid=$! +waitforlisten $bdevperf_pid $RESIZE_SOCK +# Resize the Bdev from iSCSI target +$rpc_py bdev_null_resize Null0 $BDEV_NEW_SIZE +# Obtain the Bdev from bdevperf with iSCSI initiator +num_block=$($rpc_py -s $RESIZE_SOCK bdev_get_bdevs | grep num_blocks | sed 's/[^[:digit:]]//g') +# Size is not changed as no IO sent yet and resize notification is deferred. +total_size=$((num_block * BLOCK_SIZE / 1048576)) +if [ $total_size != $BDEV_SIZE ]; then + echo "resize failed" + exit 1 +fi +sleep 2 +# Start the bdevperf IO +$rootdir/test/bdev/bdevperf/bdevperf.py -s $RESIZE_SOCK perform_tests +# Obtain the Bdev from bdevperf with iSCSI initiator +num_block=$($rpc_py -s $RESIZE_SOCK bdev_get_bdevs | grep num_blocks | sed 's/[^[:digit:]]//g') +# Get the new bdev size in MiB. +total_size=$((num_block * BLOCK_SIZE / 1048576)) +if [ $total_size != $BDEV_NEW_SIZE ]; then + echo "resize failed" + exit 1 +fi + +trap - SIGINT SIGTERM EXIT +killprocess $bdevperf_pid +killprocess $pid + +iscsitestfini diff --git a/test/unit/lib/scsi/lun.c/lun_ut.c b/test/unit/lib/scsi/lun.c/lun_ut.c index 8acc7654f0..fad3871522 100644 --- a/test/unit/lib/scsi/lun.c/lun_ut.c +++ b/test/unit/lib/scsi/lun.c/lun_ut.c @@ -431,6 +431,44 @@ lun_execute_scsi_task_complete(void) CU_ASSERT_EQUAL(g_task_count, 0); } +static void +lun_execute_scsi_task_resize(void) +{ + struct spdk_scsi_lun *lun; + struct spdk_scsi_task task = { 0 }; + struct spdk_scsi_dev dev = { 0 }; + uint8_t cdb = SPDK_SBC_READ_16; + + lun = lun_construct(); + + ut_init_task(&task); + task.lun = lun; + task.cdb = &cdb; + lun->dev = &dev; + lun->resizing = true; + + /* the tasks list should still be empty since it has not been + executed yet + */ + CU_ASSERT(TAILQ_EMPTY(&lun->tasks)); + + scsi_lun_execute_task(lun, &task); + CU_ASSERT_EQUAL(task.status, SPDK_SCSI_STATUS_CHECK_CONDITION); + /* SENSE KEY */ + CU_ASSERT_EQUAL(task.sense_data[2], SPDK_SCSI_SENSE_UNIT_ATTENTION); + /* SCSI_SENSE_ASCQ_CAPACITY_DATA_HAS_CHANGED: 0x2a09 */ + CU_ASSERT_EQUAL(task.sense_data[12], SPDK_SCSI_ASC_CAPACITY_DATA_HAS_CHANGED); + CU_ASSERT_EQUAL(task.sense_data[13], SPDK_SCSI_ASCQ_CAPACITY_DATA_HAS_CHANGED); + CU_ASSERT(lun->resizing == false); + + /* Assert the task has not been added to the tasks queue */ + CU_ASSERT(TAILQ_EMPTY(&lun->tasks)); + + lun_destruct(lun); + + CU_ASSERT_EQUAL(g_task_count, 0); +} + static void lun_destruct_success(void) { @@ -727,6 +765,7 @@ main(int argc, char **argv) CU_ADD_TEST(suite, lun_append_task_null_lun_not_supported); CU_ADD_TEST(suite, lun_execute_scsi_task_pending); CU_ADD_TEST(suite, lun_execute_scsi_task_complete); + CU_ADD_TEST(suite, lun_execute_scsi_task_resize); CU_ADD_TEST(suite, lun_destruct_success); CU_ADD_TEST(suite, lun_construct_null_ctx); CU_ADD_TEST(suite, lun_construct_success);