d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

nvme/tcp: Support unaligned extended LBA payload in nvme_tcp_pdu_set_data_buf

Browse Source 
When DIF is inserted or stripped, nvme_tcp_pdu_set_data_buf must
shift the range of the buffer from LBA based to extended LBA based
because the extended LBA based range must be passed to create a special
temporary iovec array to leave a space of metadata for each block then.

This patch do the following:
- Add a pointer to the current DIF context to struct nvme_tcp_pdu.
  This pointer is set when DIF is inserted or stripped to the current
  NVMf request.
- When the pointer to the current DIF context in the PDU is set, get
  the extended LBA based offset and length by calling
  spdk_dif_get_range_with_md() and use them in nvme_tcp_pdu_set_data_buf().
- Original data length is set in the PDU and original data offset is
  set in the current DIF context in nvme_tcp_pdu_set_data_buf().
- Add UT code.

Change-Id: Ic1f4fd326b5b3e3d4c74677592bda34d6949a11e
Signed-off-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/458920
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>
This commit is contained in:
Shuhei Matsumoto 2019-07-05 14:15:27 +09:00 committed by Changpeng Liu
parent 457afd77b1
commit a7a32ea052
2 changed files with 126 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
include/spdk_internal/nvme_tcp.h
View File

@ -35,6 +35,7 @@
#define SPDK_INTERNAL_NVME_TCP_H
#include "spdk/sock.h"
#include "spdk/dif.h"
#define SPDK_CRC32C_XOR 0xffffffffUL
#define SPDK_NVME_TCP_DIGEST_LEN 4
@ -118,6 +119,8 @@ struct nvme_tcp_pdu {
uint32_t padding_len;
struct _nvme_tcp_sgl sgl;
struct spdk_dif_ctx *dif_ctx;
void *ctx; /* data tied to a tcp request */
};
@ -446,32 +449,50 @@ nvme_tcp_read_payload_data(struct spdk_sock *sock, struct nvme_tcp_pdu *pdu)
}
static void
nvme_tcp_pdu_set_data(struct nvme_tcp_pdu *pdu, void *data, uint32_t data_len)
_nvme_tcp_pdu_set_data(struct nvme_tcp_pdu *pdu, void *data, uint32_t data_len)
{
pdu->data_iov[0].iov_base = data;
pdu->data_iov[0].iov_len = pdu->data_len = data_len;
pdu->data_iov[0].iov_len = data_len;
pdu->data_iovcnt = 1;
}
static void
nvme_tcp_pdu_set_data(struct nvme_tcp_pdu *pdu, void *data, uint32_t data_len)
{
_nvme_tcp_pdu_set_data(pdu, data, data_len);
pdu->data_len = data_len;
}
static void
nvme_tcp_pdu_set_data_buf(struct nvme_tcp_pdu *pdu,
struct iovec *iov, int iovcnt,
uint32_t data_offset, uint32_t data_len)
{
uint32_t remain_len, len;
uint32_t buf_offset, buf_len, remain_len, len;
uint8_t *buf;
struct _nvme_tcp_sgl *pdu_sgl, buf_sgl;
pdu->data_len = data_len;
if (spdk_likely(!pdu->dif_ctx)) {
buf_offset = data_offset;
buf_len = data_len;
} else {
spdk_dif_ctx_set_data_offset(pdu->dif_ctx, data_offset);
spdk_dif_get_range_with_md(data_offset, data_len,
&buf_offset, &buf_len, pdu->dif_ctx);
}
if (iovcnt == 1) {
nvme_tcp_pdu_set_data(pdu, (void *)((uint64_t)iov[0].iov_base + data_offset), data_len);
_nvme_tcp_pdu_set_data(pdu, (void *)((uint64_t)iov[0].iov_base + buf_offset), buf_len);
} else {
pdu_sgl = &pdu->sgl;
_nvme_tcp_sgl_init(pdu_sgl, pdu->data_iov, NVME_TCP_MAX_SGL_DESCRIPTORS, 0);
_nvme_tcp_sgl_init(&buf_sgl, iov, iovcnt, 0);
_nvme_tcp_sgl_advance(&buf_sgl, data_offset);
remain_len = data_len;
_nvme_tcp_sgl_advance(&buf_sgl, buf_offset);
remain_len = buf_len;
while (remain_len > 0) {
_nvme_tcp_sgl_get_buf(&buf_sgl, (void *)&buf, &len);
@ -486,10 +507,9 @@ nvme_tcp_pdu_set_data_buf(struct nvme_tcp_pdu *pdu,
}
assert(remain_len == 0);
assert(pdu_sgl->total_size == data_len);
assert(pdu_sgl->total_size == buf_len);
pdu->data_iovcnt = NVME_TCP_MAX_SGL_DESCRIPTORS - pdu_sgl->iovcnt;
pdu->data_len = data_len;
}
}

99
test/unit/lib/nvme/nvme_tcp.c/nvme_tcp_ut.c
View File

@ -291,6 +291,101 @@ test_nvme_tcp_build_sgl_request(void)
}
}
static void
test_nvme_tcp_pdu_set_data_buf_with_md(void)
{
struct nvme_tcp_pdu pdu = {};
struct iovec iovs[7] = {};
struct spdk_dif_ctx dif_ctx = {};
int rc;
pdu.dif_ctx = &dif_ctx;
rc = spdk_dif_ctx_init(&dif_ctx, 520, 8, true, false, SPDK_DIF_DISABLE, 0,
0, 0, 0, 0, 0);
CU_ASSERT(rc == 0);
/* Single iovec case */
iovs[0].iov_base = (void *)0xDEADBEEF;
iovs[0].iov_len = 2080;
nvme_tcp_pdu_set_data_buf(&pdu, iovs, 1, 0, 500);
CU_ASSERT(dif_ctx.data_offset == 0);
CU_ASSERT(pdu.data_len == 500);
CU_ASSERT(pdu.data_iovcnt == 1);
CU_ASSERT(pdu.data_iov[0].iov_base == (void *)0xDEADBEEF);
CU_ASSERT(pdu.data_iov[0].iov_len == 500);
nvme_tcp_pdu_set_data_buf(&pdu, iovs, 1, 500, 1000);
CU_ASSERT(dif_ctx.data_offset == 500);
CU_ASSERT(pdu.data_len == 1000);
CU_ASSERT(pdu.data_iovcnt == 1);
CU_ASSERT(pdu.data_iov[0].iov_base == (void *)(0xDEADBEEF + 500));
CU_ASSERT(pdu.data_iov[0].iov_len == 1016);
nvme_tcp_pdu_set_data_buf(&pdu, iovs, 1, 1500, 548);
CU_ASSERT(dif_ctx.data_offset == 1500);
CU_ASSERT(pdu.data_len == 548);
CU_ASSERT(pdu.data_iovcnt == 1);
CU_ASSERT(pdu.data_iov[0].iov_base == (void *)(0xDEADBEEF + 1516));
CU_ASSERT(pdu.data_iov[0].iov_len == 564);
/* Multiple iovecs case */
iovs[0].iov_base = (void *)0xDEADBEEF;
iovs[0].iov_len = 256;
iovs[1].iov_base = (void *)((uint8_t *)(0xDEADBEEF + 0x1000));
iovs[1].iov_len = 256 + 1;
iovs[2].iov_base = (void *)((uint8_t *)(0xDEADBEEF + 0x2000));
iovs[2].iov_len = 4;
iovs[3].iov_base = (void *)((uint8_t *)(0xDEADBEEF + 0x3000));
iovs[3].iov_len = 3 + 123;
iovs[4].iov_base = (void *)((uint8_t *)(0xDEADBEEF + 0x4000));
iovs[4].iov_len = 389 + 6;
iovs[5].iov_base = (void *)((uint8_t *)(0xDEADBEEF + 0x5000));
iovs[5].iov_len = 2 + 512 + 8 + 432;
iovs[6].iov_base = (void *)((uint8_t *)(0xDEADBEEF + 0x6000));
iovs[6].iov_len = 80 + 8;
nvme_tcp_pdu_set_data_buf(&pdu, iovs, 7, 0, 500);
CU_ASSERT(dif_ctx.data_offset == 0);
CU_ASSERT(pdu.data_len == 500);
CU_ASSERT(pdu.data_iovcnt == 2);
CU_ASSERT(pdu.data_iov[0].iov_base == (void *)0xDEADBEEF);
CU_ASSERT(pdu.data_iov[0].iov_len == 256);
CU_ASSERT(pdu.data_iov[1].iov_base == (void *)(0xDEADBEEF + 0x1000));
CU_ASSERT(pdu.data_iov[1].iov_len == 244);
nvme_tcp_pdu_set_data_buf(&pdu, iovs, 7, 500, 1000);
CU_ASSERT(dif_ctx.data_offset == 500);
CU_ASSERT(pdu.data_len == 1000);
CU_ASSERT(pdu.data_iovcnt == 5);
CU_ASSERT(pdu.data_iov[0].iov_base == (void *)(0xDEADBEEF + 0x1000 + 244));
CU_ASSERT(pdu.data_iov[0].iov_len == 13);
CU_ASSERT(pdu.data_iov[1].iov_base == (void *)(0xDEADBEEF + 0x2000));
CU_ASSERT(pdu.data_iov[1].iov_len == 4);
CU_ASSERT(pdu.data_iov[2].iov_base == (void *)(0xDEADBEEF + 0x3000));
CU_ASSERT(pdu.data_iov[2].iov_len == 3 + 123);
CU_ASSERT(pdu.data_iov[3].iov_base == (void *)(0xDEADBEEF + 0x4000));
CU_ASSERT(pdu.data_iov[3].iov_len == 395);
CU_ASSERT(pdu.data_iov[4].iov_base == (void *)(0xDEADBEEF + 0x5000));
CU_ASSERT(pdu.data_iov[4].iov_len == 478);
nvme_tcp_pdu_set_data_buf(&pdu, iovs, 7, 1500, 548);
CU_ASSERT(dif_ctx.data_offset == 1500);
CU_ASSERT(pdu.data_len == 548);
CU_ASSERT(pdu.data_iovcnt == 2);
CU_ASSERT(pdu.data_iov[0].iov_base == (void *)(0xDEADBEEF + 0x5000 + 478));
CU_ASSERT(pdu.data_iov[0].iov_len == 476);
CU_ASSERT(pdu.data_iov[1].iov_base == (void *)(0xDEADBEEF + 0x6000));
CU_ASSERT(pdu.data_iov[1].iov_len == 88);
}
int main(int argc, char **argv)
{
CU_pSuite suite = NULL;
@ -311,7 +406,9 @@ int main(int argc, char **argv)
CU_add_test(suite, "nvme_tcp_build_iovs",
test_nvme_tcp_build_iovs) == NULL ||
CU_add_test(suite, "build_sgl_request",
test_nvme_tcp_build_sgl_request) == NULL
test_nvme_tcp_build_sgl_request) == NULL ||
CU_add_test(suite, "nvme_tcp_pdu_set_data_buf_with_md",
test_nvme_tcp_pdu_set_data_buf_with_md) == NULL
) {
CU_cleanup_registry();
return CU_get_error();