dif: Copy data and generate DIF for extended LBA payload

This patch adds APIs to copy data and generate and verify DIF
for internally created extended LBA payload.

This patch is for read strip and write insert operation to make
DIF transparent to the upper layer.

Change-Id: I025f35571490aa197aac5e7572549e31c2479b41
Signed-off-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-on: https://review.gerrithub.io/c/432987
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: wuzhouhui <wuzhouhui@kingsoft.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>

include/spdk/dif.h

@@ -150,6 +150,34 @@ int spdk_dif_generate(struct iovec *iovs, int iovcnt, uint32_t num_blocks,
 int spdk_dif_verify(struct iovec *iovs, int iovcnt, uint32_t num_blocks,
 		    const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err_blk);
 
+/**
+ * Copy data and generate DIF for extended LBA payload.
+ *
+ * \param iovs iovec array describing the LBA payload.
+ * \param iovcnt Number of elements in the iovec array.
+ * \param bounce_iov A contiguous buffer forming extended LBA payload.
+ * \param num_blocks Number of blocks of the LBA payload.
+ * \param ctx DIF context.
+ *
+ * \return 0 on success and negated errno otherwise.
+ */
+int spdk_dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov,
+			   uint32_t num_blocks, const struct spdk_dif_ctx *ctx);
+
+/**
+ * Verify DIF and copy data for extended LBA payload.
+ *
+ * \param iovs iovec array describing the LBA payload.
+ * \param iovcnt Number of elements in the iovec array.
+ * \param bounce_iov A contiguous buffer forming extended LBA payload.
+ * \param num_blocks Number of blocks of the LBA payload.
+ * \param ctx DIF context.
+ *
+ * \return 0 on success and negated errno otherwise.
+ */
+int spdk_dif_verify_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov,
+			 uint32_t num_blocks, const struct spdk_dif_ctx *ctx);
+
 /**
  * Inject bit flip error to extended LBA payload.
  *

lib/util/dif.c

@@ -577,6 +577,268 @@ spdk_dif_verify(struct iovec *iovs, int iovcnt, uint32_t num_blocks,
 	}
 }
 
+static void
+dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov,
+		  uint32_t num_blocks, const struct spdk_dif_ctx *ctx)
+{
+	struct _iov_iter src_iter, dst_iter;
+	uint32_t offset_blocks, data_block_size;
+	void *src, *dst;
+	uint16_t guard;
+
+	offset_blocks = 0;
+	_iov_iter_init(&src_iter, iovs, iovcnt);
+	_iov_iter_init(&dst_iter, bounce_iov, 1);
+
+	data_block_size = ctx->block_size - ctx->md_size;
+
+	while (offset_blocks < num_blocks &&
+	       _iov_iter_cont(&src_iter) && _iov_iter_cont(&dst_iter)) {
+
+		_iov_iter_get_buf(&src_iter, &src, NULL);
+		_iov_iter_get_buf(&dst_iter, &dst, NULL);
+
+		guard = 0;
+		if (ctx->dif_flags & SPDK_DIF_GUARD_CHECK) {
+			guard = spdk_crc16_t10dif_copy(0, dst, src, data_block_size);
+			guard = spdk_crc16_t10dif(guard, dst + data_block_size,
+						  ctx->guard_interval - data_block_size);
+		} else {
+			memcpy(dst, src, data_block_size);
+		}
+
+		_dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx);
+
+		_iov_iter_advance(&src_iter, data_block_size);
+		_iov_iter_advance(&dst_iter, ctx->block_size);
+		offset_blocks++;
+	}
+}
+
+static void
+_dif_generate_copy_split(struct _iov_iter *src_iter, struct _iov_iter *dst_iter,
+			 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx)
+{
+	uint32_t offset_in_block, src_len, data_block_size;
+	uint16_t guard;
+	void *src, *dst;
+
+	_iov_iter_get_buf(dst_iter, &dst, NULL);
+
+	data_block_size = ctx->block_size - ctx->md_size;
+
+	guard = 0;
+	offset_in_block = 0;
+
+	while (offset_in_block < data_block_size && _iov_iter_cont(src_iter)) {
+		/* Compute CRC over split logical block data and copy
+		 * data to bounce buffer.
+		 */
+		_iov_iter_get_buf(src_iter, &src, &src_len);
+		src_len = spdk_min(src_len, data_block_size - offset_in_block);
+
+		if (ctx->dif_flags & SPDK_DIF_GUARD_CHECK) {
+			guard = spdk_crc16_t10dif_copy(guard, dst + offset_in_block,
+						       src, src_len);
+		} else {
+			memcpy(dst + offset_in_block, src, src_len);
+		}
+
+		_iov_iter_advance(src_iter, src_len);
+		offset_in_block += src_len;
+	}
+
+	if (ctx->dif_flags & SPDK_DIF_GUARD_CHECK) {
+		guard = spdk_crc16_t10dif(guard, dst + data_block_size,
+					  ctx->guard_interval - data_block_size);
+	}
+
+	_iov_iter_advance(dst_iter, ctx->block_size);
+
+	_dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx);
+}
+
+static void
+dif_generate_copy_split(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov,
+			uint32_t num_blocks, const struct spdk_dif_ctx *ctx)
+{
+	struct _iov_iter src_iter, dst_iter;
+	uint32_t offset_blocks;
+
+	offset_blocks = 0;
+	_iov_iter_init(&src_iter, iovs, iovcnt);
+	_iov_iter_init(&dst_iter, bounce_iov, 1);
+
+	while (offset_blocks < num_blocks &&
+	       _iov_iter_cont(&src_iter) && _iov_iter_cont(&dst_iter)) {
+		_dif_generate_copy_split(&src_iter, &dst_iter, offset_blocks, ctx);
+		offset_blocks++;
+	}
+}
+
+int
+spdk_dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov,
+		       uint32_t num_blocks, const struct spdk_dif_ctx *ctx)
+{
+	uint32_t data_block_size;
+
+	data_block_size = ctx->block_size - ctx->md_size;
+
+	if (!_are_iovs_valid(iovs, iovcnt, data_block_size * num_blocks) ||
+	    !_are_iovs_valid(bounce_iov, 1, ctx->block_size * num_blocks)) {
+		SPDK_ERRLOG("Size of iovec arrays are not valid.\n");
+		return -EINVAL;
+	}
+
+	if (_dif_is_disabled(ctx->dif_type)) {
+		return 0;
+	}
+
+	if (_are_iovs_bytes_multiple(iovs, iovcnt, data_block_size)) {
+		dif_generate_copy(iovs, iovcnt, bounce_iov, num_blocks, ctx);
+	} else {
+		dif_generate_copy_split(iovs, iovcnt, bounce_iov, num_blocks, ctx);
+	}
+
+	return 0;
+}
+
+static int
+dif_verify_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov,
+		uint32_t num_blocks, const struct spdk_dif_ctx *ctx)
+{
+	struct _iov_iter src_iter, dst_iter;
+	uint32_t offset_blocks, data_block_size;
+	void *src, *dst;
+	int rc;
+	uint16_t guard;
+
+	offset_blocks = 0;
+	_iov_iter_init(&src_iter, bounce_iov, 1);
+	_iov_iter_init(&dst_iter, iovs, iovcnt);
+
+	data_block_size = ctx->block_size - ctx->md_size;
+
+	while (offset_blocks < num_blocks && _iov_iter_cont(&src_iter) &&
+	       _iov_iter_cont(&dst_iter)) {
+
+		_iov_iter_get_buf(&src_iter, &src, NULL);
+		_iov_iter_get_buf(&dst_iter, &dst, NULL);
+
+		guard = 0;
+		if (ctx->dif_flags & SPDK_DIF_GUARD_CHECK) {
+			guard = spdk_crc16_t10dif_copy(0, dst, src, data_block_size);
+			guard = spdk_crc16_t10dif(guard, src + data_block_size,
+						  ctx->guard_interval - data_block_size);
+		} else {
+			memcpy(dst, src, data_block_size);
+		}
+
+		rc = _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, NULL);
+		if (rc != 0) {
+			return rc;
+		}
+
+		_iov_iter_advance(&src_iter, ctx->block_size);
+		_iov_iter_advance(&dst_iter, data_block_size);
+		offset_blocks++;
+	}
+
+	return 0;
+}
+
+static int
+_dif_verify_copy_split(struct _iov_iter *src_iter, struct _iov_iter *dst_iter,
+		       uint32_t offset_blocks, const struct spdk_dif_ctx *ctx)
+{
+	uint32_t offset_in_block, dst_len, data_block_size;
+	uint16_t guard;
+	void *src, *dst;
+
+	_iov_iter_get_buf(src_iter, &src, NULL);
+
+	data_block_size = ctx->block_size - ctx->md_size;
+
+	guard = 0;
+	offset_in_block = 0;
+
+	while (offset_in_block < data_block_size) {
+		/* Compute CRC over split logical block data and copy
+		 * data to bounce buffer.
+		 */
+		_iov_iter_get_buf(dst_iter, &dst, &dst_len);
+		dst_len = spdk_min(dst_len, data_block_size - offset_in_block);
+
+		if (ctx->dif_flags & SPDK_DIF_GUARD_CHECK) {
+			guard = spdk_crc16_t10dif_copy(guard, dst,
+						       src + offset_in_block, dst_len);
+		} else {
+			memcpy(dst, src + offset_in_block, dst_len);
+		}
+
+		_iov_iter_advance(dst_iter, dst_len);
+		offset_in_block += dst_len;
+	}
+
+	if (ctx->dif_flags & SPDK_DIF_GUARD_CHECK) {
+		guard = spdk_crc16_t10dif(guard, src + data_block_size,
+					  ctx->guard_interval - data_block_size);
+	}
+
+	_iov_iter_advance(src_iter, ctx->block_size);
+
+	return _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, NULL);
+}
+
+static int
+dif_verify_copy_split(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov,
+		      uint32_t num_blocks, const struct spdk_dif_ctx *ctx)
+{
+	struct _iov_iter src_iter, dst_iter;
+	uint32_t offset_blocks;
+	int rc;
+
+	offset_blocks = 0;
+	_iov_iter_init(&src_iter, bounce_iov, 1);
+	_iov_iter_init(&dst_iter, iovs, iovcnt);
+
+	while (offset_blocks < num_blocks &&
+	       _iov_iter_cont(&src_iter) && _iov_iter_cont(&dst_iter)) {
+		rc = _dif_verify_copy_split(&src_iter, &dst_iter, offset_blocks, ctx);
+		if (rc != 0) {
+			return rc;
+		}
+		offset_blocks++;
+	}
+
+	return 0;
+}
+
+int
+spdk_dif_verify_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov,
+		     uint32_t num_blocks, const struct spdk_dif_ctx *ctx)
+{
+	uint32_t data_block_size;
+
+	data_block_size = ctx->block_size - ctx->md_size;
+
+	if (!_are_iovs_valid(iovs, iovcnt, data_block_size * num_blocks) ||
+	    !_are_iovs_valid(bounce_iov, 1, ctx->block_size * num_blocks)) {
+		SPDK_ERRLOG("Size of iovec arrays are not valid\n");
+		return -EINVAL;
+	}
+
+	if (_dif_is_disabled(ctx->dif_type)) {
+		return 0;
+	}
+
+	if (_are_iovs_bytes_multiple(iovs, iovcnt, data_block_size)) {
+		return dif_verify_copy(iovs, iovcnt, bounce_iov, num_blocks, ctx);
+	} else {
+		return dif_verify_copy_split(iovs, iovcnt, bounce_iov, num_blocks, ctx);
+	}
+}
+
 static void
 _bit_flip(uint8_t *buf, uint32_t flip_bit)
 {

test/unit/lib/util/dif.c/dif_ut.c

@@ -728,6 +728,170 @@ dif_sec_4096_md_128_inject_1_2_4_8_multi_iovs_split_reftag_test(void)
 	_iov_free_buf(&iovs[1]);
 }
 
+static void
+dif_copy_gen_and_verify(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov,
+			uint32_t block_size, uint32_t md_size, uint32_t num_blocks,
+			bool dif_loc, enum spdk_dif_type dif_type, uint32_t dif_flags,
+			uint32_t init_ref_tag, uint16_t apptag_mask, uint16_t app_tag)
+{
+	struct spdk_dif_ctx ctx = {};
+	int rc;
+
+	rc = ut_data_pattern_generate(iovs, iovcnt, block_size - md_size, 0, num_blocks);
+	CU_ASSERT(rc == 0);
+
+	rc = spdk_dif_ctx_init(&ctx, block_size, md_size, dif_loc, dif_type, dif_flags,
+			       init_ref_tag, apptag_mask, app_tag);
+	CU_ASSERT(rc == 0);
+
+	rc = spdk_dif_generate_copy(iovs, iovcnt, bounce_iov, num_blocks, &ctx);
+	CU_ASSERT(rc == 0);
+
+	rc = spdk_dif_verify_copy(iovs, iovcnt, bounce_iov, num_blocks, &ctx);
+	CU_ASSERT(rc == 0);
+
+	rc = ut_data_pattern_verify(iovs, iovcnt, block_size - md_size, 0, num_blocks);
+	CU_ASSERT(rc == 0);
+}
+
+static void
+dif_copy_sec_512_md_8_prchk_0_single_iov(void)
+{
+	struct iovec iov, bounce_iov;
+
+	_iov_alloc_buf(&iov, 512 * 4);
+	_iov_alloc_buf(&bounce_iov, (512 + 8) * 4);
+
+	dif_copy_gen_and_verify(&iov, 1, &bounce_iov, 512 + 8, 8, 4,
+				false, SPDK_DIF_TYPE1, 0, 0, 0, 0);
+	dif_copy_gen_and_verify(&iov, 1, &bounce_iov, 512 + 8, 8, 4,
+				true, SPDK_DIF_TYPE1, 0, 0, 0, 0);
+
+	_iov_free_buf(&iov);
+	_iov_free_buf(&bounce_iov);
+}
+
+static void
+dif_copy_sec_512_md_8_prchk_0_1_2_4_multi_iovs(void)
+{
+	struct iovec iovs[4], bounce_iov;
+	int i, num_blocks;
+
+	num_blocks = 0;
+
+	for (i = 0; i < 4; i++) {
+		_iov_alloc_buf(&iovs[i], 512 * (i + 1));
+		num_blocks += i + 1;
+	}
+
+	_iov_alloc_buf(&bounce_iov, (512 + 8) * num_blocks);
+
+	dif_copy_gen_and_verify(iovs, 4, &bounce_iov, 512 + 8, 8, num_blocks,
+				false, SPDK_DIF_TYPE1, 0, 22, 0xFFFF, 0x22);
+
+	dif_copy_gen_and_verify(iovs, 4, &bounce_iov, 512 + 8, 8, num_blocks,
+				false, SPDK_DIF_TYPE1, SPDK_DIF_GUARD_CHECK, 22, 0xFFFF, 0x22);
+
+	dif_copy_gen_and_verify(iovs, 4, &bounce_iov, 512 + 8, 8, num_blocks,
+				false, SPDK_DIF_TYPE1, SPDK_DIF_APPTAG_CHECK, 22, 0xFFFF, 0x22);
+
+	dif_copy_gen_and_verify(iovs, 4, &bounce_iov, 512 + 8, 8, num_blocks,
+				false, SPDK_DIF_TYPE1, SPDK_DIF_REFTAG_CHECK, 22, 0xFFFF, 0x22);
+
+	for (i = 0; i < 4; i++) {
+		_iov_free_buf(&iovs[i]);
+	}
+	_iov_free_buf(&bounce_iov);
+}
+
+static void
+dif_copy_sec_4096_md_128_prchk_7_multi_iovs(void)
+{
+	struct iovec iovs[4], bounce_iov;
+	uint32_t dif_flags;
+	int i, num_blocks;
+
+	dif_flags = SPDK_DIF_GUARD_CHECK | SPDK_DIF_APPTAG_CHECK | SPDK_DIF_REFTAG_CHECK;
+
+	num_blocks = 0;
+
+	for (i = 0; i < 4; i++) {
+		_iov_alloc_buf(&iovs[i], 4096 * (i + 1));
+		num_blocks += i + 1;
+	}
+
+	_iov_alloc_buf(&bounce_iov, (4096 + 128) * num_blocks);
+
+	dif_copy_gen_and_verify(iovs, 4, &bounce_iov, 4096 + 128, 128, num_blocks,
+				false, SPDK_DIF_TYPE1, dif_flags, 22, 0xFFFF, 0x22);
+	dif_copy_gen_and_verify(iovs, 4, &bounce_iov, 4096 + 128, 128, num_blocks,
+				true, SPDK_DIF_TYPE1, dif_flags, 22, 0xFFFF, 0x22);
+
+	for (i = 0; i < 4; i++) {
+		_iov_free_buf(&iovs[i]);
+	}
+	_iov_free_buf(&bounce_iov);
+}
+
+static void
+dif_copy_sec_512_md_8_prchk_7_multi_iovs_split_data(void)
+{
+	struct iovec iovs[2], bounce_iov;
+	uint32_t dif_flags;
+
+	dif_flags = SPDK_DIF_GUARD_CHECK | SPDK_DIF_APPTAG_CHECK | SPDK_DIF_REFTAG_CHECK;
+
+	_iov_alloc_buf(&iovs[0], 256);
+	_iov_alloc_buf(&iovs[1], 256);
+
+	_iov_alloc_buf(&bounce_iov, 512 + 8);
+
+	dif_copy_gen_and_verify(iovs, 2, &bounce_iov, 512 + 8, 8, 1,
+				false, SPDK_DIF_TYPE1, dif_flags, 22, 0xFFFF, 0x22);
+
+	_iov_free_buf(&iovs[0]);
+	_iov_free_buf(&iovs[1]);
+	_iov_free_buf(&bounce_iov);
+}
+
+static void
+dif_copy_sec_512_md_8_prchk_7_multi_iovs_complex_splits(void)
+{
+	struct iovec iovs[6], bounce_iov;
+	uint32_t dif_flags;
+	int i;
+
+	dif_flags = SPDK_DIF_GUARD_CHECK | SPDK_DIF_APPTAG_CHECK | SPDK_DIF_REFTAG_CHECK;
+
+	/* data[0][255:0] */
+	_iov_alloc_buf(&iovs[0], 256);
+
+	/* data[0][511:256], data[1][255:0] */
+	_iov_alloc_buf(&iovs[1], 256 + 256);
+
+	/* data[1][382:256] */
+	_iov_alloc_buf(&iovs[2], 128);
+
+	/* data[1][383] */
+	_iov_alloc_buf(&iovs[3], 1);
+
+	/* data[1][510:384] */
+	_iov_alloc_buf(&iovs[4], 126);
+
+	/* data[1][511], data[2][511:0], data[3][511:0] */
+	_iov_alloc_buf(&iovs[5], 1 + 512 * 2);
+
+	_iov_alloc_buf(&bounce_iov, (512 + 8) * 4);
+
+	dif_copy_gen_and_verify(iovs, 6, &bounce_iov, 512 + 8, 8, 4,
+				true, SPDK_DIF_TYPE1, dif_flags, 22, 0xFFFF, 0x22);
+
+	for (i = 0; i < 6; i++) {
+		_iov_free_buf(&iovs[i]);
+	}
+	_iov_free_buf(&bounce_iov);
+}
+
 int
 main(int argc, char **argv)
 {
@@ -781,7 +945,17 @@ main(int argc, char **argv)
 		CU_add_test(suite, "dif_sec_4096_md_128_inject_1_2_4_8__multi_iovs_split_apptag_test",
 			    dif_sec_4096_md_128_inject_1_2_4_8_multi_iovs_split_apptag_test) == NULL ||
 		CU_add_test(suite, "dif_sec_4096_md_128_inject_1_2_4_8_multi_iovs_split_reftag_test",
-			    dif_sec_4096_md_128_inject_1_2_4_8_multi_iovs_split_reftag_test) == NULL
+			    dif_sec_4096_md_128_inject_1_2_4_8_multi_iovs_split_reftag_test) == NULL ||
+		CU_add_test(suite, "dif_copy_sec_512_md_8_prchk_0_single_iov",
+			    dif_copy_sec_512_md_8_prchk_0_single_iov) == NULL ||
+		CU_add_test(suite, "dif_copy_sec_512_md_8_prchk_0_1_2_4_multi_iovs",
+			    dif_copy_sec_512_md_8_prchk_0_1_2_4_multi_iovs) == NULL ||
+		CU_add_test(suite, "dif_copy_sec_4096_md_128_prchk_7_multi_iovs",
+			    dif_copy_sec_4096_md_128_prchk_7_multi_iovs) == NULL ||
+		CU_add_test(suite, "dif_copy_sec_512_md_8_prchk_7_multi_iovs_split_data",
+			    dif_copy_sec_512_md_8_prchk_7_multi_iovs_split_data) == NULL ||
+		CU_add_test(suite, "dif_copy_sec_512_md_8_prchk_7_multi_iovs_complex_splits",
+			    dif_copy_sec_512_md_8_prchk_7_multi_iovs_complex_splits) == NULL
 	) {
 		CU_cleanup_registry();
 		return CU_get_error();