d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

idxd: fix issue w/multiple WQ config

Browse Source 
Found via inspection during spec review of latest HW. We were using the
wrong stride for the WQCFG regsiter when configuring but it just so
happened to be the right value for the current DSA version.  We were
mixing up the size of the WQCFG register with the stride value used to
configure the next WQCFG regsiter as they are not contiguous in HW, we
need to read another capabilities bit to determine the address of the
next wqcfg to configure..

Signed-off-by: paul luse <paul.e.luse@intel.com>
Change-Id: I14d1ff95e0131fd30121aa955bfbc8c8fb3fc512
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/10968
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
This commit is contained in:
paul luse 2022-01-04 14:16:51 -07:00 committed by Jim Harris
parent 026f003154
commit c501d2b37c
4 changed files with 12 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/spdk/idxd_spec.h
View File

@ -52,6 +52,7 @@ extern "C" {
#define WQ_TOTAL_PORTAL_SIZE (PORTAL_SIZE * 4)
#define PORTAL_STRIDE 0x40
#define PORTAL_MASK (PORTAL_SIZE - 1)
#define WQCFG_SHIFT 5
#define CFG_ENGINE_OFFSET 0x20
#define CFG_FLAG_OFFSET 0x28

1
lib/idxd/idxd.h
View File

@ -66,7 +66,6 @@ static inline void movdir64b(void *dst, const void *src)
#define DESC_PER_BATCH (1 << LOG2_WQ_MAX_BATCH)
#define LOG2_WQ_MAX_XFER 30 /* 2^30 = 1073741824 */
#define WQCFG_NUM_DWORDS 8
#define WQ_PRIORITY_1 1
#define IDXD_MAX_QUEUES 64

9
lib/idxd/idxd_user.c
View File

@ -261,10 +261,11 @@ idxd_group_config(struct spdk_idxd_device *idxd)
static int
idxd_wq_config(struct spdk_user_idxd_device *user_idxd)
{
int i, j;
uint32_t i, j;
struct idxd_wq *queue;
struct spdk_idxd_device *idxd = &user_idxd->idxd;
u_int32_t wq_size = user_idxd->registers.wqcap.total_wq_size / g_user_dev_cfg.total_wqs;
uint32_t wq_size = user_idxd->registers.wqcap.total_wq_size / g_user_dev_cfg.total_wqs;
uint32_t wqcap_size = 1 << (WQCFG_SHIFT + user_idxd->registers.wqcap.wqcfg_size);
SPDK_DEBUGLOG(idxd, "Total ring slots available space 0x%x, so per work queue is 0x%x\n",
user_idxd->registers.wqcap.total_wq_size, wq_size);
@ -303,8 +304,8 @@ idxd_wq_config(struct spdk_user_idxd_device *user_idxd)
*/
for (i = 0 ; i < user_idxd->registers.wqcap.num_wqs; i++) {
queue = &idxd->queues[i];
for (j = 0 ; j < WQCFG_NUM_DWORDS; j++) {
_idxd_write_4(idxd, user_idxd->wqcfg_offset + i * 32 + j * 4,
for (j = 0 ; j < (sizeof(union idxd_wqcfg) / sizeof(uint32_t)); j++) {
_idxd_write_4(idxd, user_idxd->wqcfg_offset + i * wqcap_size + j * sizeof(uint32_t),
queue->wqcfg.raw[j]);
}
}

12
test/unit/lib/idxd/idxd_user.c/idxd_user_ut.c
View File

@ -108,8 +108,9 @@ test_idxd_wq_config(void)
struct spdk_idxd_device *idxd = &user_idxd.idxd;
union idxd_wqcfg wqcfg = {};
uint32_t expected[8] = {0x40, 0, 0x11, 0xbe, 0, 0, 0x40000000, 0};
uint32_t wq_size;
int rc, i, j;
uint32_t wq_size, i, j;
uint32_t wqcap_size = 32;
int rc;
user_idxd.reg_base = calloc(1, FAKE_REG_SIZE);
SPDK_CU_ASSERT_FATAL(user_idxd.reg_base != NULL);
@ -139,10 +140,9 @@ test_idxd_wq_config(void)
}
for (i = 0 ; i < user_idxd.registers.wqcap.num_wqs; i++) {
for (j = 0 ; j < WQCFG_NUM_DWORDS; j++) {
wqcfg.raw[j] = spdk_mmio_read_4((uint32_t *)(user_idxd.reg_base + user_idxd.wqcfg_offset + i * 32 +
j *
4));
for (j = 0 ; j < (sizeof(union idxd_wqcfg) / sizeof(uint32_t)); j++) {
wqcfg.raw[j] = spdk_mmio_read_4((uint32_t *)(user_idxd.reg_base +
user_idxd.wqcfg_offset + i * wqcap_size + j * sizeof(uint32_t)));
CU_ASSERT(wqcfg.raw[j] == expected[j]);
}
}