From c501d2b37ce524e50417701bc45056bccef7ec66 Mon Sep 17 00:00:00 2001 From: paul luse Date: Tue, 4 Jan 2022 14:16:51 -0700 Subject: [PATCH] idxd: fix issue w/multiple WQ config Found via inspection during spec review of latest HW. We were using the wrong stride for the WQCFG regsiter when configuring but it just so happened to be the right value for the current DSA version. We were mixing up the size of the WQCFG register with the stride value used to configure the next WQCFG regsiter as they are not contiguous in HW, we need to read another capabilities bit to determine the address of the next wqcfg to configure.. Signed-off-by: paul luse Change-Id: I14d1ff95e0131fd30121aa955bfbc8c8fb3fc512 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/10968 Reviewed-by: Jim Harris Reviewed-by: Ben Walker Community-CI: Broadcom CI Tested-by: SPDK CI Jenkins --- include/spdk/idxd_spec.h | 1 + lib/idxd/idxd.h | 1 - lib/idxd/idxd_user.c | 9 +++++---- test/unit/lib/idxd/idxd_user.c/idxd_user_ut.c | 12 ++++++------ 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/include/spdk/idxd_spec.h b/include/spdk/idxd_spec.h index 783b58ad56..8c7f5814ba 100644 --- a/include/spdk/idxd_spec.h +++ b/include/spdk/idxd_spec.h @@ -52,6 +52,7 @@ extern "C" { #define WQ_TOTAL_PORTAL_SIZE (PORTAL_SIZE * 4) #define PORTAL_STRIDE 0x40 #define PORTAL_MASK (PORTAL_SIZE - 1) +#define WQCFG_SHIFT 5 #define CFG_ENGINE_OFFSET 0x20 #define CFG_FLAG_OFFSET 0x28 diff --git a/lib/idxd/idxd.h b/lib/idxd/idxd.h index eae3f1efcb..c7216b7913 100644 --- a/lib/idxd/idxd.h +++ b/lib/idxd/idxd.h @@ -66,7 +66,6 @@ static inline void movdir64b(void *dst, const void *src) #define DESC_PER_BATCH (1 << LOG2_WQ_MAX_BATCH) #define LOG2_WQ_MAX_XFER 30 /* 2^30 = 1073741824 */ -#define WQCFG_NUM_DWORDS 8 #define WQ_PRIORITY_1 1 #define IDXD_MAX_QUEUES 64 diff --git a/lib/idxd/idxd_user.c b/lib/idxd/idxd_user.c index d0f8b938e3..9e4e406dbf 100644 --- a/lib/idxd/idxd_user.c +++ b/lib/idxd/idxd_user.c @@ -261,10 +261,11 @@ idxd_group_config(struct spdk_idxd_device *idxd) static int idxd_wq_config(struct spdk_user_idxd_device *user_idxd) { - int i, j; + uint32_t i, j; struct idxd_wq *queue; struct spdk_idxd_device *idxd = &user_idxd->idxd; - u_int32_t wq_size = user_idxd->registers.wqcap.total_wq_size / g_user_dev_cfg.total_wqs; + uint32_t wq_size = user_idxd->registers.wqcap.total_wq_size / g_user_dev_cfg.total_wqs; + uint32_t wqcap_size = 1 << (WQCFG_SHIFT + user_idxd->registers.wqcap.wqcfg_size); SPDK_DEBUGLOG(idxd, "Total ring slots available space 0x%x, so per work queue is 0x%x\n", user_idxd->registers.wqcap.total_wq_size, wq_size); @@ -303,8 +304,8 @@ idxd_wq_config(struct spdk_user_idxd_device *user_idxd) */ for (i = 0 ; i < user_idxd->registers.wqcap.num_wqs; i++) { queue = &idxd->queues[i]; - for (j = 0 ; j < WQCFG_NUM_DWORDS; j++) { - _idxd_write_4(idxd, user_idxd->wqcfg_offset + i * 32 + j * 4, + for (j = 0 ; j < (sizeof(union idxd_wqcfg) / sizeof(uint32_t)); j++) { + _idxd_write_4(idxd, user_idxd->wqcfg_offset + i * wqcap_size + j * sizeof(uint32_t), queue->wqcfg.raw[j]); } } diff --git a/test/unit/lib/idxd/idxd_user.c/idxd_user_ut.c b/test/unit/lib/idxd/idxd_user.c/idxd_user_ut.c index 8650ed1399..d2b2bafec1 100644 --- a/test/unit/lib/idxd/idxd_user.c/idxd_user_ut.c +++ b/test/unit/lib/idxd/idxd_user.c/idxd_user_ut.c @@ -108,8 +108,9 @@ test_idxd_wq_config(void) struct spdk_idxd_device *idxd = &user_idxd.idxd; union idxd_wqcfg wqcfg = {}; uint32_t expected[8] = {0x40, 0, 0x11, 0xbe, 0, 0, 0x40000000, 0}; - uint32_t wq_size; - int rc, i, j; + uint32_t wq_size, i, j; + uint32_t wqcap_size = 32; + int rc; user_idxd.reg_base = calloc(1, FAKE_REG_SIZE); SPDK_CU_ASSERT_FATAL(user_idxd.reg_base != NULL); @@ -139,10 +140,9 @@ test_idxd_wq_config(void) } for (i = 0 ; i < user_idxd.registers.wqcap.num_wqs; i++) { - for (j = 0 ; j < WQCFG_NUM_DWORDS; j++) { - wqcfg.raw[j] = spdk_mmio_read_4((uint32_t *)(user_idxd.reg_base + user_idxd.wqcfg_offset + i * 32 + - j * - 4)); + for (j = 0 ; j < (sizeof(union idxd_wqcfg) / sizeof(uint32_t)); j++) { + wqcfg.raw[j] = spdk_mmio_read_4((uint32_t *)(user_idxd.reg_base + + user_idxd.wqcfg_offset + i * wqcap_size + j * sizeof(uint32_t))); CU_ASSERT(wqcfg.raw[j] == expected[j]); } }