Lib/iSCSI: add the LUN Resize support

From SAM-4, section 5.13 (Sense Data); “When a command terminates with a CHECK CONDITION status, sense data shall be returned in the same I_T_L_Q nexus transaction (see 3.1.50) as the CHECK CONDITION status. After the sense data is returned, it shall be cleared except when it is associated with a unit attention condition and the UA_INTLCK_CTRL field in the Control mode page (see SPC-4) contains 10b or 11b.” SPDK does not set UA_INTLCK_CTRL to 10b or 11b, so we set the unit attention condition immediately against a single IO or Admin IO after reporting it via a CHECK CONDITION. Once the failed IO received at iSCSI initiator side, it will be retried. In the case of resize operation, if there is no IO from iSCSI initiator side, the unit attention condition will be delayed to report until the first IO is received at the iSCSI target side. Meanwhile, we clear the resizing (newly added) flag on our SCSI LUN structure after first time we report the resize unit attention condition. The kernel initiator won’t actually resize the corresponding block device automatically. It will report a uevent, and then you can set up udev rules to trigger a rescan. SPDK iSCSI initiator will automatically report the LUN size change. Change-Id: Ifc85b8d4d3fbea13e76fb5d1faf1ac6c8f662e6c Signed-off-by: GangCao <gang.cao@intel.com> Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/11086 Tested-by: SPDK CI Jenkins <sys_sgci@intel.com> Reviewed-by: Jim Harris <james.r.harris@intel.com> Reviewed-by: Xiaodong Liu <xiaodong.liu@intel.com> Reviewed-by: Changpeng Liu <changpeng.liu@intel.com> Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
2022-01-14 11:32:15 -05:00 · 2022-01-14 11:32:15 -05:00 · 6b7e9d0af2
commit 6b7e9d0af2
parent 2da24e2c1b
8 changed files with 262 additions and 5 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -58,6 +58,11 @@ Added 3 experimental APIs to handle PCI device interrupts (`spdk_pci_device_enab
 Added a 'subsystem' parameter to spdk_nvmf_transport_stop_listen_async. When not NULL,
 it will only disconnect qpairs for controllers associated with the specified subsystem.

+### scsi
+
+Structure `spdk_scsi_lun` has been extended with new member `resizing` so that SCSI layer now reports
+unit attention for disk resize.
+
 ## v21.10

 Structure `spdk_nvmf_target_opts` has been extended with new member `discovery_filter` which allows to specify
--- a/include/spdk/scsi_spec.h
+++ b/include/spdk/scsi_spec.h
@ -101,6 +101,7 @@ enum spdk_scsi_asc {
 	SPDK_SCSI_ASC_INVALID_FIELD_IN_CDB = 0x24,
 	SPDK_SCSI_ASC_LOGICAL_UNIT_NOT_SUPPORTED = 0x25,
 	SPDK_SCSI_ASC_WRITE_PROTECTED = 0x27,
+	SPDK_SCSI_ASC_CAPACITY_DATA_HAS_CHANGED = 0x2a,
 	SPDK_SCSI_ASC_FORMAT_COMMAND_FAILED = 0x31,
 	SPDK_SCSI_ASC_SAVING_PARAMETERS_NOT_SUPPORTED = 0x39,
 	SPDK_SCSI_ASC_INTERNAL_TARGET_FAILURE = 0x44,
@ -116,6 +117,7 @@ enum spdk_scsi_ascq {
 	SPDK_SCSI_ASCQ_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED = 0x03,
 	SPDK_SCSI_ASCQ_POWER_LOSS_EXPECTED = 0x08,
 	SPDK_SCSI_ASCQ_INVALID_LU_IDENTIFIER = 0x09,
+	SPDK_SCSI_ASCQ_CAPACITY_DATA_HAS_CHANGED = 0x09,
 };

 enum spdk_spc_opcode {
--- a/lib/scsi/lun.c
+++ b/lib/scsi/lun.c
@ -125,6 +125,23 @@ scsi_lun_append_mgmt_task(struct spdk_scsi_lun *lun,
 	TAILQ_INSERT_TAIL(&lun->pending_mgmt_tasks, task, scsi_link);
 }

+static bool
+_scsi_lun_handle_unit_attention(struct spdk_scsi_task *task)
+{
+	uint8_t *cdb = task->cdb;
+
+	assert(task->cdb);
+
+	switch (cdb[0]) {
+	case SPDK_SPC_INQUIRY:
+	case SPDK_SPC_REPORT_LUNS:
+	case SPDK_SPC_REQUEST_SENSE:
+		return false;
+	default:
+		return true;
+	}
+}
+
 static void
 _scsi_lun_execute_mgmt_task(struct spdk_scsi_lun *lun)
 {
@ -195,7 +212,17 @@ _scsi_lun_execute_task(struct spdk_scsi_lun *lun, struct spdk_scsi_task *task)
 	task->status = SPDK_SCSI_STATUS_GOOD;
 	spdk_trace_record(TRACE_SCSI_TASK_START, lun->dev->id, task->length, (uintptr_t)task);
 	TAILQ_INSERT_TAIL(&lun->tasks, task, scsi_link);
-	if (!lun->removed) {
+	if (spdk_unlikely(lun->removed)) {
+		spdk_scsi_task_process_abort(task);
+		rc = SPDK_SCSI_TASK_COMPLETE;
+	} else if (spdk_unlikely(lun->resizing) && _scsi_lun_handle_unit_attention(task)) {
+		spdk_scsi_task_set_status(task, SPDK_SCSI_STATUS_CHECK_CONDITION,
+					  SPDK_SCSI_SENSE_UNIT_ATTENTION,
+					  SPDK_SCSI_ASC_CAPACITY_DATA_HAS_CHANGED,
+					  SPDK_SCSI_ASCQ_CAPACITY_DATA_HAS_CHANGED);
+		lun->resizing = false;
+		rc = SPDK_SCSI_TASK_COMPLETE;
+	} else {
 		/* Check the command is allowed or not when reservation is exist */
 		if (spdk_unlikely(lun->reservation.flags & SCSI_SPC2_RESERVE)) {
 			rc = scsi2_reserve_check(task);
@ -208,9 +235,6 @@ _scsi_lun_execute_task(struct spdk_scsi_lun *lun, struct spdk_scsi_task *task)
 		} else {
 			rc = bdev_scsi_execute(task);
 		}
-	} else {
-		spdk_scsi_task_process_abort(task);
-		rc = SPDK_SCSI_TASK_COMPLETE;
 	}

 	switch (rc) {
@ -405,6 +429,7 @@ bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev,
 		break;
 	case SPDK_BDEV_EVENT_RESIZE:
 		SPDK_NOTICELOG("bdev name (%s) received event(SPDK_BDEV_EVENT_RESIZE)\n", spdk_bdev_get_name(bdev));
+		lun->resizing = true;
 		if (lun->resize_cb) {
 			lun->resize_cb(lun, lun->resize_ctx);
 		}
@ -466,6 +491,7 @@ struct spdk_scsi_lun *scsi_lun_construct(const char *bdev_name,

 	lun->resize_cb = resize_cb;
 	lun->resize_ctx = resize_ctx;
+	lun->resizing = false;

 	TAILQ_INIT(&lun->open_descs);
 	TAILQ_INIT(&lun->reg_head);
--- a/lib/scsi/scsi_internal.h
+++ b/lib/scsi/scsi_internal.h
@ -178,6 +178,9 @@ struct spdk_scsi_lun {

 	/** A structure to connect LUNs in a list. */
 	TAILQ_ENTRY(spdk_scsi_lun) tailq;
+
+	/** The LUN is resizing */
+	bool resizing;
 };

 struct spdk_scsi_lun *scsi_lun_construct(const char *bdev_name,
--- a/module/bdev/iscsi/bdev_iscsi.c
+++ b/module/bdev/iscsi/bdev_iscsi.c
@ -72,12 +72,16 @@ struct bdev_iscsi_lun;
 #define BDEV_ISCSI_MAX_UNMAP_BLOCK_DESCS_COUNT (1)

 static int bdev_iscsi_initialize(void);
+static void bdev_iscsi_readcapacity16(struct iscsi_context *context, struct bdev_iscsi_lun *lun);
+static void _bdev_iscsi_submit_request(void *_bdev_io);
+
 static TAILQ_HEAD(, bdev_iscsi_conn_req) g_iscsi_conn_req = TAILQ_HEAD_INITIALIZER(
 			g_iscsi_conn_req);
 static struct spdk_poller *g_conn_poller = NULL;

 struct bdev_iscsi_io {
 	struct spdk_thread *submit_td;
+	struct bdev_iscsi_lun *lun;
 	enum spdk_bdev_io_status status;
 	int scsi_status;
 	enum spdk_scsi_sense sk;
@ -217,12 +221,26 @@ bdev_iscsi_io_complete(struct bdev_iscsi_io *iscsi_io, enum spdk_bdev_io_status
 	}
 }

+static bool
+_bdev_iscsi_is_size_change(int status, struct scsi_task *task)
+{
+	if (status == SPDK_SCSI_STATUS_CHECK_CONDITION &&
+	    (uint8_t)task->sense.key == SPDK_SCSI_SENSE_UNIT_ATTENTION &&
+	    task->sense.ascq == 0x2a09) {
+		/* ASCQ: SCSI_SENSE_ASCQ_CAPACITY_DATA_HAS_CHANGED (0x2a09) */
+		return true;
+	}
+
+	return false;
+}
+
 /* Common call back function for read/write/flush command */
 static void
 bdev_iscsi_command_cb(struct iscsi_context *context, int status, void *_task, void *_iscsi_io)
 {
 	struct scsi_task *task = _task;
 	struct bdev_iscsi_io *iscsi_io = _iscsi_io;
+	struct spdk_bdev_io *bdev_io;

 	iscsi_io->scsi_status = status;
 	iscsi_io->sk = (uint8_t)task->sense.key;
@ -230,7 +248,86 @@ bdev_iscsi_command_cb(struct iscsi_context *context, int status, void *_task, vo
 	iscsi_io->ascq = task->sense.ascq & 0xFF;

 	scsi_free_scsi_task(task);
-	bdev_iscsi_io_complete(iscsi_io, SPDK_BDEV_IO_STATUS_SUCCESS);
+
+	if (_bdev_iscsi_is_size_change(status, task)) {
+		bdev_iscsi_readcapacity16(context, iscsi_io->lun);
+
+		/* Retry this failed IO immediately */
+		bdev_io = spdk_bdev_io_from_ctx(iscsi_io);
+		if (iscsi_io->submit_td != NULL) {
+			spdk_thread_send_msg(iscsi_io->lun->main_td,
+					     _bdev_iscsi_submit_request, bdev_io);
+		} else {
+			_bdev_iscsi_submit_request(bdev_io);
+		}
+	} else {
+		bdev_iscsi_io_complete(iscsi_io, SPDK_BDEV_IO_STATUS_SUCCESS);
+	}
+}
+
+static int
+bdev_iscsi_resize(struct spdk_bdev *bdev, const uint64_t new_size_in_block)
+{
+	int rc;
+
+	assert(bdev->module == &g_iscsi_bdev_module);
+
+	if (new_size_in_block <= bdev->blockcnt) {
+		SPDK_ERRLOG("The new bdev size must be larger than current bdev size.\n");
+		return -EINVAL;
+	}
+
+	rc = spdk_bdev_notify_blockcnt_change(bdev, new_size_in_block);
+	if (rc != 0) {
+		SPDK_ERRLOG("failed to notify block cnt change.\n");
+		return rc;
+	}
+
+	return 0;
+}
+
+static void
+bdev_iscsi_readcapacity16_cb(struct iscsi_context *context, int status, void *_task,
+			     void *private_data)
+{
+	struct bdev_iscsi_lun *lun = private_data;
+	struct scsi_readcapacity16 *readcap16;
+	struct scsi_task *task = _task;
+	uint64_t size_in_block = 0;
+	int rc;
+
+	if (status != SPDK_SCSI_STATUS_GOOD) {
+		SPDK_ERRLOG("iSCSI error: %s\n", iscsi_get_error(context));
+		goto ret;
+	}
+
+	readcap16 = scsi_datain_unmarshall(task);
+	if (!readcap16) {
+		SPDK_ERRLOG("Read capacity error\n");
+		goto ret;
+	}
+
+	size_in_block = readcap16->returned_lba + 1;
+
+	rc = bdev_iscsi_resize(&lun->bdev, size_in_block);
+	if (rc != 0) {
+		SPDK_ERRLOG("Bdev (%s) resize error: %d\n", lun->bdev.name, rc);
+	}
+
+ret:
+	scsi_free_scsi_task(task);
+}
+
+static void
+bdev_iscsi_readcapacity16(struct iscsi_context *context, struct bdev_iscsi_lun *lun)
+{
+	struct scsi_task *task;
+
+	task = iscsi_readcapacity16_task(context, lun->lun_id,
+					 bdev_iscsi_readcapacity16_cb, lun);
+	if (task == NULL) {
+		SPDK_ERRLOG("failed to get readcapacity16_task\n");
+	}
 }

 static void
@ -510,6 +607,8 @@ static void bdev_iscsi_submit_request(struct spdk_io_channel *_ch, struct spdk_b
 	struct bdev_iscsi_io *iscsi_io = (struct bdev_iscsi_io *)bdev_io->driver_ctx;
 	struct bdev_iscsi_lun *lun = (struct bdev_iscsi_lun *)bdev_io->bdev->ctxt;

+	iscsi_io->lun = lun;
+
 	if (lun->main_td != submit_td) {
 		iscsi_io->submit_td = submit_td;
 		spdk_thread_send_msg(lun->main_td, _bdev_iscsi_submit_request, bdev_io);
@ -722,9 +821,18 @@ iscsi_readcapacity16_cb(struct iscsi_context *iscsi, int status,
 	struct scsi_readcapacity16 *readcap16;
 	struct spdk_bdev *bdev = NULL;
 	struct scsi_task *task = command_data;
+	struct scsi_task *retry_task = NULL;

 	if (status != SPDK_SCSI_STATUS_GOOD) {
 		SPDK_ERRLOG("iSCSI error: %s\n", iscsi_get_error(iscsi));
+		if (_bdev_iscsi_is_size_change(status, task)) {
+			scsi_free_scsi_task(task);
+			retry_task = iscsi_readcapacity16_task(iscsi, req->lun,
+							       iscsi_readcapacity16_cb, req);
+			if (retry_task) {
+				return;
+			}
+		}
 		goto ret;
 	}

--- a/test/iscsi_tgt/iscsi_tgt.sh
+++ b/test/iscsi_tgt/iscsi_tgt.sh
@ -64,6 +64,7 @@ fi
 if [ $SPDK_TEST_ISCSI_INITIATOR -eq 1 ]; then
 	run_test "iscsi_tgt_initiator" ./test/iscsi_tgt/initiator/initiator.sh
 	run_test "iscsi_tgt_bdev_io_wait" ./test/iscsi_tgt/bdev_io_wait/bdev_io_wait.sh
+	run_test "iscsi_tgt_resize" ./test/iscsi_tgt/resize/resize.sh
 fi

 cleanup_veth_interfaces
--- a/test/iscsi_tgt/resize/resize.sh
+++ b/test/iscsi_tgt/resize/resize.sh
@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+
+testdir=$(readlink -f $(dirname $0))
+rootdir=$(readlink -f $testdir/../../..)
+source $rootdir/test/common/autotest_common.sh
+source $rootdir/test/iscsi_tgt/common.sh
+
+iscsitestinit
+
+BDEV_SIZE=64
+BDEV_NEW_SIZE=128
+BLOCK_SIZE=512
+RESIZE_SOCK="/var/tmp/spdk-resize.sock"
+
+rpc_py="$rootdir/scripts/rpc.py"
+
+timing_enter start_iscsi_tgt
+
+# Remove the sock file first
+rm -f $RESIZE_SOCK
+
+"${ISCSI_APP[@]}" -m 0x2 -p 1 -s 512 --wait-for-rpc &
+pid=$!
+echo "iSCSI target launched. pid: $pid"
+trap 'killprocess $pid; iscsitestfini; exit 1' SIGINT SIGTERM EXIT
+waitforlisten $pid
+$rpc_py framework_start_init
+echo "iscsi_tgt is listening. Running tests..."
+
+timing_exit start_iscsi_tgt
+
+$rpc_py iscsi_create_portal_group $PORTAL_TAG $TARGET_IP:$ISCSI_PORT
+$rpc_py iscsi_create_initiator_group $INITIATOR_TAG $INITIATOR_NAME $NETMASK
+$rpc_py bdev_null_create Null0 $BDEV_SIZE $BLOCK_SIZE
+# "Null0:0" ==> use Null0 blockdev for LUN0
+# "1:2" ==> map PortalGroup1 to InitiatorGroup2
+# "64" ==> iSCSI queue depth 64
+# "-d" ==> disable CHAP authentication
+$rpc_py iscsi_create_target_node disk1 disk1_alias 'Null0:0' $PORTAL_TAG:$INITIATOR_TAG 256 -d
+sleep 1
+trap 'killprocess $pid; iscsitestfini; exit 1' SIGINT SIGTERM EXIT
+
+# Start bdevperf with another sock file and iSCSI initiator
+"$rootdir/test/bdev/bdevperf/bdevperf" -r $RESIZE_SOCK --json <(initiator_json_config) -q 16 -o 4096 -w read -t 5 -R -s 128 -z &
+bdevperf_pid=$!
+waitforlisten $bdevperf_pid $RESIZE_SOCK
+# Resize the Bdev from iSCSI target
+$rpc_py bdev_null_resize Null0 $BDEV_NEW_SIZE
+# Obtain the Bdev from bdevperf with iSCSI initiator
+num_block=$($rpc_py -s $RESIZE_SOCK bdev_get_bdevs | grep num_blocks | sed 's/[^[:digit:]]//g')
+# Size is not changed as no IO sent yet and resize notification is deferred.
+total_size=$((num_block * BLOCK_SIZE / 1048576))
+if [ $total_size != $BDEV_SIZE ]; then
+	echo "resize failed"
+	exit 1
+fi
+sleep 2
+# Start the bdevperf IO
+$rootdir/test/bdev/bdevperf/bdevperf.py -s $RESIZE_SOCK perform_tests
+# Obtain the Bdev from bdevperf with iSCSI initiator
+num_block=$($rpc_py -s $RESIZE_SOCK bdev_get_bdevs | grep num_blocks | sed 's/[^[:digit:]]//g')
+# Get the new bdev size in MiB.
+total_size=$((num_block * BLOCK_SIZE / 1048576))
+if [ $total_size != $BDEV_NEW_SIZE ]; then
+	echo "resize failed"
+	exit 1
+fi
+
+trap - SIGINT SIGTERM EXIT
+killprocess $bdevperf_pid
+killprocess $pid
+
+iscsitestfini
--- a/test/unit/lib/scsi/lun.c/lun_ut.c
+++ b/test/unit/lib/scsi/lun.c/lun_ut.c
@ -431,6 +431,44 @@ lun_execute_scsi_task_complete(void)
 	CU_ASSERT_EQUAL(g_task_count, 0);
 }

+static void
+lun_execute_scsi_task_resize(void)
+{
+	struct spdk_scsi_lun *lun;
+	struct spdk_scsi_task task = { 0 };
+	struct spdk_scsi_dev dev = { 0 };
+	uint8_t cdb = SPDK_SBC_READ_16;
+
+	lun = lun_construct();
+
+	ut_init_task(&task);
+	task.lun = lun;
+	task.cdb = &cdb;
+	lun->dev = &dev;
+	lun->resizing = true;
+
+	/* the tasks list should still be empty since it has not been
+	   executed yet
+	 */
+	CU_ASSERT(TAILQ_EMPTY(&lun->tasks));
+
+	scsi_lun_execute_task(lun, &task);
+	CU_ASSERT_EQUAL(task.status, SPDK_SCSI_STATUS_CHECK_CONDITION);
+	/* SENSE KEY */
+	CU_ASSERT_EQUAL(task.sense_data[2], SPDK_SCSI_SENSE_UNIT_ATTENTION);
+	/* SCSI_SENSE_ASCQ_CAPACITY_DATA_HAS_CHANGED: 0x2a09 */
+	CU_ASSERT_EQUAL(task.sense_data[12], SPDK_SCSI_ASC_CAPACITY_DATA_HAS_CHANGED);
+	CU_ASSERT_EQUAL(task.sense_data[13], SPDK_SCSI_ASCQ_CAPACITY_DATA_HAS_CHANGED);
+	CU_ASSERT(lun->resizing == false);
+
+	/* Assert the task has not been added to the tasks queue */
+	CU_ASSERT(TAILQ_EMPTY(&lun->tasks));
+
+	lun_destruct(lun);
+
+	CU_ASSERT_EQUAL(g_task_count, 0);
+}
+
 static void
 lun_destruct_success(void)
 {
@ -727,6 +765,7 @@ main(int argc, char **argv)
 	CU_ADD_TEST(suite, lun_append_task_null_lun_not_supported);
 	CU_ADD_TEST(suite, lun_execute_scsi_task_pending);
 	CU_ADD_TEST(suite, lun_execute_scsi_task_complete);
+	CU_ADD_TEST(suite, lun_execute_scsi_task_resize);
 	CU_ADD_TEST(suite, lun_destruct_success);
 	CU_ADD_TEST(suite, lun_construct_null_ctx);
 	CU_ADD_TEST(suite, lun_construct_success);