d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

idxd: Split the idxd library into common and user space part.

Browse Source 
Purpose: This patch is used to prepare to add the kernel
idxd support later.

Signed-off-by: Ziye Yang <ziye.yang@intel.com>
Change-Id: If89665f95d622c7342ab75050664158ec6fc615a
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/7330
Community-CI: Broadcom CI
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Paul Luse <paul.e.luse@intel.com>
This commit is contained in:
Ziye Yang 2021-04-13 19:02:46 +08:00 committed by Tomasz Zawadzki
parent 6cebe9d06b
commit 20698a4a8d
10 changed files with 936 additions and 632 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lib/idxd/Makefile
View File

@ -37,7 +37,8 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
SO_VER := 4
SO_MINOR := 0
C_SRCS = idxd.c
C_SRCS = idxd.c idxd_user.c
LIBNAME = idxd
SPDK_MAP_FILE = $(abspath $(CURDIR)/spdk_idxd.map)

469
lib/idxd/idxd.c
View File

@ -45,8 +45,10 @@
#include "idxd.h"
#define ALIGN_4K 0x1000
#define USERSPACE_DRIVER_NAME "user"
pthread_mutex_t g_driver_lock = PTHREAD_MUTEX_INITIALIZER;
static STAILQ_HEAD(, spdk_idxd_impl) g_idxd_impls = STAILQ_HEAD_INITIALIZER(g_idxd_impls);
static struct spdk_idxd_impl *g_idxd_impl;
/*
* g_dev_cfg gives us 2 pre-set configurations of DSA to choose from
@ -73,36 +75,18 @@ struct device_config g_dev_cfg1 = {
.total_engines = 4,
};
static uint32_t
_idxd_read_4(struct spdk_idxd_device *idxd, uint32_t offset)
{
return spdk_mmio_read_4((uint32_t *)(idxd->reg_base + offset));
}
static void
_idxd_write_4(struct spdk_idxd_device *idxd, uint32_t offset, uint32_t value)
{
spdk_mmio_write_4((uint32_t *)(idxd->reg_base + offset), value);
}
static uint64_t
_idxd_read_8(struct spdk_idxd_device *idxd, uint32_t offset)
{
return spdk_mmio_read_8((uint64_t *)(idxd->reg_base + offset));
}
static void
_idxd_write_8(struct spdk_idxd_device *idxd, uint32_t offset, uint64_t value)
{
spdk_mmio_write_8((uint64_t *)(idxd->reg_base + offset), value);
}
bool
spdk_idxd_device_needs_rebalance(struct spdk_idxd_device *idxd)
{
return idxd->needs_rebalance;
}
static uint64_t
idxd_read_8(struct spdk_idxd_device *idxd, void *portal, uint32_t offset)
{
return idxd->impl->read_8(idxd, portal, offset);
}
struct spdk_idxd_io_channel *
spdk_idxd_get_channel(struct spdk_idxd_device *idxd)
{
@ -243,8 +227,7 @@ spdk_idxd_configure_chan(struct spdk_idxd_io_channel *chan)
}
}
/* Assign portal based on work queue chosen earlier. */
chan->portal = (char *)chan->idxd->portals + chan->idxd->wq_id * PORTAL_SIZE;
chan->portal = chan->idxd->impl->portal_get_addr(chan->idxd);
return 0;
@ -264,34 +247,6 @@ err_desc:
return rc;
}
/* Used for control commands, not for descriptor submission. */
static int
idxd_wait_cmd(struct spdk_idxd_device *idxd, int _timeout)
{
uint32_t timeout = _timeout;
union idxd_cmdsts_reg cmd_status = {};
cmd_status.raw = _idxd_read_4(idxd, IDXD_CMDSTS_OFFSET);
while (cmd_status.active && --timeout) {
usleep(1);
cmd_status.raw = _idxd_read_4(idxd, IDXD_CMDSTS_OFFSET);
}
/* Check for timeout */
if (timeout == 0 && cmd_status.active) {
SPDK_ERRLOG("Command timeout, waited %u\n", _timeout);
return -EBUSY;
}
/* Check for error */
if (cmd_status.err) {
SPDK_ERRLOG("Command status reg reports error 0x%x\n", cmd_status.err);
return -EINVAL;
}
return 0;
}
static void
_idxd_drain(struct spdk_idxd_io_channel *chan)
{
@ -347,10 +302,32 @@ spdk_idxd_reconfigure_chan(struct spdk_idxd_io_channel *chan)
return rc;
}
static inline struct spdk_idxd_impl *
idxd_get_impl_by_name(const char *impl_name)
{
struct spdk_idxd_impl *impl;
assert(impl_name != NULL);
STAILQ_FOREACH(impl, &g_idxd_impls, link) {
if (0 == strcmp(impl_name, impl->name)) {
return impl;
}
}
return NULL;
}
/* Called via RPC to select a pre-defined configuration. */
void
spdk_idxd_set_config(uint32_t config_num)
{
g_idxd_impl = idxd_get_impl_by_name(USERSPACE_DRIVER_NAME);
if (g_idxd_impl == NULL) {
SPDK_ERRLOG("Cannot set the idxd implementation");
return;
}
switch (config_num) {
case 0:
g_dev_cfg = &g_dev_cfg0;
@ -363,378 +340,27 @@ spdk_idxd_set_config(uint32_t config_num)
SPDK_ERRLOG("Invalid config, using default\n");
break;
}
}
static int
idxd_unmap_pci_bar(struct spdk_idxd_device *idxd, int bar)
{
int rc = 0;
void *addr = NULL;
if (bar == IDXD_MMIO_BAR) {
addr = (void *)idxd->reg_base;
} else if (bar == IDXD_WQ_BAR) {
addr = (void *)idxd->portals;
}
if (addr) {
rc = spdk_pci_device_unmap_bar(idxd->device, 0, addr);
}
return rc;
}
static int
idxd_map_pci_bars(struct spdk_idxd_device *idxd)
{
int rc;
void *addr;
uint64_t phys_addr, size;
rc = spdk_pci_device_map_bar(idxd->device, IDXD_MMIO_BAR, &addr, &phys_addr, &size);
if (rc != 0 || addr == NULL) {
SPDK_ERRLOG("pci_device_map_range failed with error code %d\n", rc);
return -1;
}
idxd->reg_base = addr;
rc = spdk_pci_device_map_bar(idxd->device, IDXD_WQ_BAR, &addr, &phys_addr, &size);
if (rc != 0 || addr == NULL) {
SPDK_ERRLOG("pci_device_map_range failed with error code %d\n", rc);
rc = idxd_unmap_pci_bar(idxd, IDXD_MMIO_BAR);
if (rc) {
SPDK_ERRLOG("unable to unmap MMIO bar\n");
}
return -EINVAL;
}
idxd->portals = addr;
return 0;
}
static int
idxd_reset_dev(struct spdk_idxd_device *idxd)
{
int rc;
_idxd_write_4(idxd, IDXD_CMD_OFFSET, IDXD_RESET_DEVICE << IDXD_CMD_SHIFT);
rc = idxd_wait_cmd(idxd, IDXD_REGISTER_TIMEOUT_US);
if (rc < 0) {
SPDK_ERRLOG("Error resetting device %u\n", rc);
}
return rc;
}
/*
* Build group config based on getting info from the device combined
* with the defined configuration. Once built, it is written to the
* device.
*/
static int
idxd_group_config(struct spdk_idxd_device *idxd)
{
int i;
uint64_t base_offset;
struct idxd_grpcfg *grpcfg;
assert(g_dev_cfg->num_groups <= idxd->registers.groupcap.num_groups);
idxd->groups = calloc(idxd->registers.groupcap.num_groups, sizeof(struct idxd_group));
if (idxd->groups == NULL) {
SPDK_ERRLOG("Failed to allocate group memory\n");
return -ENOMEM;
}
assert(g_dev_cfg->total_engines <= idxd->registers.enginecap.num_engines);
for (i = 0; i < g_dev_cfg->total_engines; i++) {
idxd->groups[i % g_dev_cfg->num_groups].grpcfg.engines |= (1 << i);
}
assert(g_dev_cfg->total_wqs <= idxd->registers.wqcap.num_wqs);
for (i = 0; i < g_dev_cfg->total_wqs; i++) {
idxd->groups[i % g_dev_cfg->num_groups].grpcfg.wqs[0] |= (1 << i);
}
for (i = 0; i < g_dev_cfg->num_groups; i++) {
idxd->groups[i].idxd = idxd;
idxd->groups[i].id = i;
/* Divide BW tokens evenly */
idxd->groups[i].grpcfg.flags.tokens_allowed =
idxd->registers.groupcap.total_tokens / g_dev_cfg->num_groups;
}
/*
* Now write the group config to the device for all groups. We write
* to the max number of groups in order to 0 out the ones we didn't
* configure.
*/
for (i = 0 ; i < idxd->registers.groupcap.num_groups; i++) {
grpcfg = &idxd->groups[i].grpcfg;
if (i < g_dev_cfg->num_groups) {
SPDK_DEBUGLOG(idxd, "Group #%u: wqueue_cfg 0x%lx, engine_cfg 0x%lx, flags 0x%x\n", i,
grpcfg->wqs[0], grpcfg->engines, grpcfg->flags.raw);
}
base_offset = idxd->grpcfg_offset + i * 64;
/* GRPWQCFG, work queues config */
_idxd_write_8(idxd, base_offset, grpcfg->wqs[0]);
/* GRPENGCFG, engine config */
_idxd_write_8(idxd, base_offset + CFG_ENGINE_OFFSET, grpcfg->engines);
/* GRPFLAGS, flags config */
_idxd_write_8(idxd, base_offset + CFG_FLAG_OFFSET, grpcfg->flags.raw);
}
return 0;
}
/*
* Build work queue (WQ) config based on getting info from the device combined
* with the defined configuration. Once built, it is written to the device.
*/
static int
idxd_wq_config(struct spdk_idxd_device *idxd)
{
int i, j;
struct idxd_wq *queue;
u_int32_t wq_size = idxd->registers.wqcap.total_wq_size / g_dev_cfg->total_wqs;
SPDK_NOTICELOG("Total ring slots available space 0x%x, so per work queue is 0x%x\n",
idxd->registers.wqcap.total_wq_size, wq_size);
assert(g_dev_cfg->total_wqs <= IDXD_MAX_QUEUES);
assert(g_dev_cfg->total_wqs <= idxd->registers.wqcap.num_wqs);
assert(LOG2_WQ_MAX_BATCH <= idxd->registers.gencap.max_batch_shift);
assert(LOG2_WQ_MAX_XFER <= idxd->registers.gencap.max_xfer_shift);
idxd->queues = calloc(1, idxd->registers.wqcap.num_wqs * sizeof(struct idxd_wq));
if (idxd->queues == NULL) {
SPDK_ERRLOG("Failed to allocate queue memory\n");
return -ENOMEM;
}
for (i = 0; i < g_dev_cfg->total_wqs; i++) {
queue = &idxd->queues[i];
queue->wqcfg.wq_size = wq_size;
queue->wqcfg.mode = WQ_MODE_DEDICATED;
queue->wqcfg.max_batch_shift = LOG2_WQ_MAX_BATCH;
queue->wqcfg.max_xfer_shift = LOG2_WQ_MAX_XFER;
queue->wqcfg.wq_state = WQ_ENABLED;
queue->wqcfg.priority = WQ_PRIORITY_1;
/* Not part of the config struct */
queue->idxd = idxd;
queue->group = &idxd->groups[i % g_dev_cfg->num_groups];
}
/*
* Now write the work queue config to the device for all wq space
*/
for (i = 0 ; i < idxd->registers.wqcap.num_wqs; i++) {
queue = &idxd->queues[i];
for (j = 0 ; j < WQCFG_NUM_DWORDS; j++) {
_idxd_write_4(idxd, idxd->wqcfg_offset + i * 32 + j * 4,
queue->wqcfg.raw[j]);
}
}
return 0;
}
static int
idxd_device_configure(struct spdk_idxd_device *idxd)
{
int i, rc = 0;
union idxd_offsets_register offsets_reg;
union idxd_genstatus_register genstatus_reg;
/*
* Map BAR0 and BAR2
*/
rc = idxd_map_pci_bars(idxd);
if (rc) {
return rc;
}
/*
* Reset the device
*/
rc = idxd_reset_dev(idxd);
if (rc) {
goto err_reset;
}
/*
* Read in config registers
*/
idxd->registers.version = _idxd_read_4(idxd, IDXD_VERSION_OFFSET);
idxd->registers.gencap.raw = _idxd_read_8(idxd, IDXD_GENCAP_OFFSET);
idxd->registers.wqcap.raw = _idxd_read_8(idxd, IDXD_WQCAP_OFFSET);
idxd->registers.groupcap.raw = _idxd_read_8(idxd, IDXD_GRPCAP_OFFSET);
idxd->registers.enginecap.raw = _idxd_read_8(idxd, IDXD_ENGCAP_OFFSET);
for (i = 0; i < IDXD_OPCAP_WORDS; i++) {
idxd->registers.opcap.raw[i] =
_idxd_read_8(idxd, i * sizeof(uint64_t) + IDXD_OPCAP_OFFSET);
}
offsets_reg.raw[0] = _idxd_read_8(idxd, IDXD_TABLE_OFFSET);
offsets_reg.raw[1] = _idxd_read_8(idxd, IDXD_TABLE_OFFSET + sizeof(uint64_t));
idxd->grpcfg_offset = offsets_reg.grpcfg * IDXD_TABLE_OFFSET_MULT;
idxd->wqcfg_offset = offsets_reg.wqcfg * IDXD_TABLE_OFFSET_MULT;
idxd->ims_offset = offsets_reg.ims * IDXD_TABLE_OFFSET_MULT;
idxd->msix_perm_offset = offsets_reg.msix_perm * IDXD_TABLE_OFFSET_MULT;
idxd->perfmon_offset = offsets_reg.perfmon * IDXD_TABLE_OFFSET_MULT;
/*
* Configure groups and work queues.
*/
rc = idxd_group_config(idxd);
if (rc) {
goto err_group_cfg;
}
rc = idxd_wq_config(idxd);
if (rc) {
goto err_wq_cfg;
}
/*
* Enable the device
*/
genstatus_reg.raw = _idxd_read_4(idxd, IDXD_GENSTATUS_OFFSET);
assert(genstatus_reg.state == IDXD_DEVICE_STATE_DISABLED);
_idxd_write_4(idxd, IDXD_CMD_OFFSET, IDXD_ENABLE_DEV << IDXD_CMD_SHIFT);
rc = idxd_wait_cmd(idxd, IDXD_REGISTER_TIMEOUT_US);
genstatus_reg.raw = _idxd_read_4(idxd, IDXD_GENSTATUS_OFFSET);
if ((rc < 0) || (genstatus_reg.state != IDXD_DEVICE_STATE_ENABLED)) {
rc = -EINVAL;
SPDK_ERRLOG("Error enabling device %u\n", rc);
goto err_device_enable;
}
genstatus_reg.raw = spdk_mmio_read_4((uint32_t *)(idxd->reg_base + IDXD_GENSTATUS_OFFSET));
assert(genstatus_reg.state == IDXD_DEVICE_STATE_ENABLED);
/*
* Enable the work queues that we've configured
*/
for (i = 0; i < g_dev_cfg->total_wqs; i++) {
_idxd_write_4(idxd, IDXD_CMD_OFFSET,
(IDXD_ENABLE_WQ << IDXD_CMD_SHIFT) | i);
rc = idxd_wait_cmd(idxd, IDXD_REGISTER_TIMEOUT_US);
if (rc < 0) {
SPDK_ERRLOG("Error enabling work queues 0x%x\n", rc);
goto err_wq_enable;
}
}
if ((rc == 0) && (genstatus_reg.state == IDXD_DEVICE_STATE_ENABLED)) {
SPDK_NOTICELOG("Device enabled, version 0x%x gencap: 0x%lx\n",
idxd->registers.version,
idxd->registers.gencap.raw);
}
return rc;
err_wq_enable:
err_device_enable:
free(idxd->queues);
err_wq_cfg:
free(idxd->groups);
err_group_cfg:
err_reset:
idxd_unmap_pci_bar(idxd, IDXD_MMIO_BAR);
idxd_unmap_pci_bar(idxd, IDXD_MMIO_BAR);
return rc;
g_idxd_impl->set_config(g_dev_cfg, config_num);
}
static void
idxd_device_destruct(struct spdk_idxd_device *idxd)
{
idxd_unmap_pci_bar(idxd, IDXD_MMIO_BAR);
idxd_unmap_pci_bar(idxd, IDXD_WQ_BAR);
free(idxd->groups);
free(idxd->queues);
free(idxd);
}
assert(idxd->impl != NULL);
/* Caller must hold g_driver_lock */
static struct spdk_idxd_device *
idxd_attach(struct spdk_pci_device *device)
{
struct spdk_idxd_device *idxd;
uint32_t cmd_reg;
int rc;
idxd = calloc(1, sizeof(struct spdk_idxd_device));
if (idxd == NULL) {
SPDK_ERRLOG("Failed to allocate memory for idxd device.\n");
return NULL;
}
idxd->device = device;
pthread_mutex_init(&idxd->num_channels_lock, NULL);
/* Enable PCI busmaster. */
spdk_pci_device_cfg_read32(device, &cmd_reg, 4);
cmd_reg |= 0x4;
spdk_pci_device_cfg_write32(device, cmd_reg, 4);
rc = idxd_device_configure(idxd);
if (rc) {
goto err;
}
return idxd;
err:
idxd_device_destruct(idxd);
return NULL;
}
struct idxd_enum_ctx {
spdk_idxd_probe_cb probe_cb;
spdk_idxd_attach_cb attach_cb;
void *cb_ctx;
};
/* This function must only be called while holding g_driver_lock */
static int
idxd_enum_cb(void *ctx, struct spdk_pci_device *pci_dev)
{
struct idxd_enum_ctx *enum_ctx = ctx;
struct spdk_idxd_device *idxd;
if (enum_ctx->probe_cb(enum_ctx->cb_ctx, pci_dev)) {
idxd = idxd_attach(pci_dev);
if (idxd == NULL) {
SPDK_ERRLOG("idxd_attach() failed\n");
return -EINVAL;
}
enum_ctx->attach_cb(enum_ctx->cb_ctx, pci_dev, idxd);
}
return 0;
idxd->impl->destruct(idxd);
}
int
spdk_idxd_probe(void *cb_ctx, spdk_idxd_probe_cb probe_cb, spdk_idxd_attach_cb attach_cb)
{
int rc;
struct idxd_enum_ctx enum_ctx;
if (g_idxd_impl == NULL) {
SPDK_ERRLOG("No idxd impl is selected\n");
return -1;
}
enum_ctx.probe_cb = probe_cb;
enum_ctx.attach_cb = attach_cb;
enum_ctx.cb_ctx = cb_ctx;
pthread_mutex_lock(&g_driver_lock);
rc = spdk_pci_enumerate(spdk_pci_idxd_get_driver(), idxd_enum_cb, &enum_ctx);
pthread_mutex_unlock(&g_driver_lock);
return rc;
return g_idxd_impl->probe(cb_ctx, probe_cb, attach_cb);
}
void
@ -1180,11 +806,10 @@ _idxd_batch_prep_nop(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch
/* Command specific. */
desc->opcode = IDXD_OPCODE_NOOP;
/* TODO: temp workaround for simulator. Remove when fixed or w/silicon. */
if (chan->idxd->registers.gencap.raw == 0x1833f011f) {
if (chan->idxd->impl->nop_check && chan->idxd->impl->nop_check(chan->idxd)) {
desc->xfer_size = 1;
}
return 0;
}
@ -1369,7 +994,7 @@ _dump_error_reg(struct spdk_idxd_io_channel *chan)
uint64_t sw_error_0;
uint16_t i;
sw_error_0 = _idxd_read_8(chan->idxd, IDXD_SWERR_OFFSET);
sw_error_0 = idxd_read_8(chan->idxd, chan->portal, IDXD_SWERR_OFFSET);
SPDK_NOTICELOG("SW Error bits set:");
for (i = 0; i < CHAR_BIT; i++) {
@ -1409,7 +1034,7 @@ spdk_idxd_process_events(struct spdk_idxd_io_channel *chan)
rc++;
if (spdk_unlikely(IDXD_FAILURE(comp_ctx->hw.status))) {
sw_error_0 = _idxd_read_8(chan->idxd, IDXD_SWERR_OFFSET);
sw_error_0 = idxd_read_8(chan->idxd, chan->portal, IDXD_SWERR_OFFSET);
if (IDXD_SW_ERROR(sw_error_0)) {
_dump_error_reg(chan);
status = -EINVAL;
@ -1453,4 +1078,10 @@ spdk_idxd_process_events(struct spdk_idxd_io_channel *chan)
return rc;
}
void
idxd_impl_register(struct spdk_idxd_impl *impl)
{
STAILQ_INSERT_HEAD(&g_idxd_impls, impl, link);
}
SPDK_LOG_REGISTER_COMPONENT(idxd)

30
lib/idxd/idxd.h
View File

@ -178,26 +178,40 @@ struct idxd_wq {
union idxd_wqcfg wqcfg;
};
struct spdk_idxd_impl {
const char *name;
void (*set_config)(struct device_config *g_dev_cfg, uint32_t config_num);
int (*probe)(void *cb_ctx, spdk_idxd_probe_cb probe_cb, spdk_idxd_attach_cb attach_cb);
void (*destruct)(struct spdk_idxd_device *idxd);
uint64_t (*read_8)(struct spdk_idxd_device *idxd, void *portal, uint32_t offset);
char *(*portal_get_addr)(struct spdk_idxd_device *idxd);
/* It is a workround for simulator */
bool (*nop_check)(struct spdk_idxd_device *idxd);
STAILQ_ENTRY(spdk_idxd_impl) link;
};
struct spdk_idxd_device {
struct spdk_pci_device *device;
void *reg_base;
struct spdk_idxd_impl *impl;
void *portals;
int socket_id;
int wq_id;
uint32_t num_channels;
bool needs_rebalance;
pthread_mutex_t num_channels_lock;
struct idxd_registers registers;
uint32_t ims_offset;
uint32_t msix_perm_offset;
uint32_t wqcfg_offset;
uint32_t grpcfg_offset;
uint32_t perfmon_offset;
struct idxd_group *groups;
struct idxd_wq *queues;
};
void idxd_impl_register(struct spdk_idxd_impl *impl);
#define SPDK_IDXD_IMPL_REGISTER(name, impl) \
static void __attribute__((constructor)) idxd_impl_register_##name(void) \
{ \
idxd_impl_register(impl); \
}
#ifdef __cplusplus
}
#endif

541
lib/idxd/idxd_user.c Normal file
View File

@ -0,0 +1,541 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation.
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "spdk/stdinc.h"
#include "spdk/env.h"
#include "spdk/util.h"
#include "spdk/memory.h"
#include "spdk/likely.h"
#include "spdk/log.h"
#include "spdk_internal/idxd.h"
#include "idxd.h"
struct spdk_user_idxd_device {
struct spdk_idxd_device idxd;
int sock_id;
struct idxd_registers registers;
void *reg_base;
uint32_t wqcfg_offset;
uint32_t grpcfg_offset;
uint32_t ims_offset;
uint32_t msix_perm_offset;
uint32_t perfmon_offset;
};
#define __user_idxd(idxd) (struct spdk_user_idxd_device *)idxd
pthread_mutex_t g_driver_lock = PTHREAD_MUTEX_INITIALIZER;
static struct device_config g_user_dev_cfg = {};
static struct spdk_idxd_device *idxd_attach(struct spdk_pci_device *device);
static uint32_t
_idxd_read_4(struct spdk_idxd_device *idxd, uint32_t offset)
{
struct spdk_user_idxd_device *user_idxd = __user_idxd(idxd);
return spdk_mmio_read_4((uint32_t *)(user_idxd->reg_base + offset));
}
static void
_idxd_write_4(struct spdk_idxd_device *idxd, uint32_t offset, uint32_t value)
{
struct spdk_user_idxd_device *user_idxd = __user_idxd(idxd);
spdk_mmio_write_4((uint32_t *)(user_idxd->reg_base + offset), value);
}
static uint64_t
_idxd_read_8(struct spdk_idxd_device *idxd, uint32_t offset)
{
struct spdk_user_idxd_device *user_idxd = __user_idxd(idxd);
return spdk_mmio_read_8((uint64_t *)(user_idxd->reg_base + offset));
}
static uint64_t
idxd_read_8(struct spdk_idxd_device *idxd, void *portal, uint32_t offset)
{
return _idxd_read_8(idxd, offset);
}
static void
_idxd_write_8(struct spdk_idxd_device *idxd, uint32_t offset, uint64_t value)
{
struct spdk_user_idxd_device *user_idxd = __user_idxd(idxd);
spdk_mmio_write_8((uint64_t *)(user_idxd->reg_base + offset), value);
}
static void
user_idxd_set_config(struct device_config *dev_cfg, uint32_t config_num)
{
g_user_dev_cfg = *dev_cfg;
}
/* Used for control commands, not for descriptor submission. */
static int
idxd_wait_cmd(struct spdk_idxd_device *idxd, int _timeout)
{
uint32_t timeout = _timeout;
union idxd_cmdsts_reg cmd_status = {};
cmd_status.raw = _idxd_read_4(idxd, IDXD_CMDSTS_OFFSET);
while (cmd_status.active && --timeout) {
usleep(1);
cmd_status.raw = _idxd_read_4(idxd, IDXD_CMDSTS_OFFSET);
}
/* Check for timeout */
if (timeout == 0 && cmd_status.active) {
SPDK_ERRLOG("Command timeout, waited %u\n", _timeout);
return -EBUSY;
}
/* Check for error */
if (cmd_status.err) {
SPDK_ERRLOG("Command status reg reports error 0x%x\n", cmd_status.err);
return -EINVAL;
}
return 0;
}
static int
idxd_unmap_pci_bar(struct spdk_idxd_device *idxd, int bar)
{
int rc = 0;
void *addr = NULL;
struct spdk_user_idxd_device *user_idxd = __user_idxd(idxd);
if (bar == IDXD_MMIO_BAR) {
addr = (void *)user_idxd->reg_base;
} else if (bar == IDXD_WQ_BAR) {
addr = (void *)idxd->portals;
}
if (addr) {
rc = spdk_pci_device_unmap_bar(idxd->device, 0, addr);
}
return rc;
}
static int
idxd_map_pci_bars(struct spdk_idxd_device *idxd)
{
int rc;
void *addr;
uint64_t phys_addr, size;
struct spdk_user_idxd_device *user_idxd = __user_idxd(idxd);
rc = spdk_pci_device_map_bar(idxd->device, IDXD_MMIO_BAR, &addr, &phys_addr, &size);
if (rc != 0 || addr == NULL) {
SPDK_ERRLOG("pci_device_map_range failed with error code %d\n", rc);
return -1;
}
user_idxd->reg_base = addr;
rc = spdk_pci_device_map_bar(idxd->device, IDXD_WQ_BAR, &addr, &phys_addr, &size);
if (rc != 0 || addr == NULL) {
SPDK_ERRLOG("pci_device_map_range failed with error code %d\n", rc);
rc = idxd_unmap_pci_bar(idxd, IDXD_MMIO_BAR);
if (rc) {
SPDK_ERRLOG("unable to unmap MMIO bar\n");
}
return -EINVAL;
}
idxd->portals = addr;
return 0;
}
static int
idxd_reset_dev(struct spdk_idxd_device *idxd)
{
int rc;
_idxd_write_4(idxd, IDXD_CMD_OFFSET, IDXD_RESET_DEVICE << IDXD_CMD_SHIFT);
rc = idxd_wait_cmd(idxd, IDXD_REGISTER_TIMEOUT_US);
if (rc < 0) {
SPDK_ERRLOG("Error resetting device %u\n", rc);
}
return rc;
}
/*
* Build group config based on getting info from the device combined
* with the defined configuration. Once built, it is written to the
* device.
*/
static int
idxd_group_config(struct spdk_idxd_device *idxd)
{
int i;
uint64_t base_offset;
struct spdk_user_idxd_device *user_idxd = __user_idxd(idxd);
assert(g_user_dev_cfg.num_groups <= user_idxd->registers.groupcap.num_groups);
idxd->groups = calloc(user_idxd->registers.groupcap.num_groups, sizeof(struct idxd_group));
if (idxd->groups == NULL) {
SPDK_ERRLOG("Failed to allocate group memory\n");
return -ENOMEM;
}
assert(g_user_dev_cfg.total_engines <= user_idxd->registers.enginecap.num_engines);
for (i = 0; i < g_user_dev_cfg.total_engines; i++) {
idxd->groups[i % g_user_dev_cfg.num_groups].grpcfg.engines |= (1 << i);
}
assert(g_user_dev_cfg.total_wqs <= user_idxd->registers.wqcap.num_wqs);
for (i = 0; i < g_user_dev_cfg.total_wqs; i++) {
idxd->groups[i % g_user_dev_cfg.num_groups].grpcfg.wqs[0] |= (1 << i);
}
for (i = 0; i < g_user_dev_cfg.num_groups; i++) {
idxd->groups[i].idxd = idxd;
idxd->groups[i].id = i;
/* Divide BW tokens evenly */
idxd->groups[i].grpcfg.flags.tokens_allowed =
user_idxd->registers.groupcap.total_tokens / g_user_dev_cfg.num_groups;
}
/*
* Now write the group config to the device for all groups. We write
* to the max number of groups in order to 0 out the ones we didn't
* configure.
*/
for (i = 0 ; i < user_idxd->registers.groupcap.num_groups; i++) {
base_offset = user_idxd->grpcfg_offset + i * 64;
/* GRPWQCFG, work queues config */
_idxd_write_8(idxd, base_offset, idxd->groups[i].grpcfg.wqs[0]);
/* GRPENGCFG, engine config */
_idxd_write_8(idxd, base_offset + CFG_ENGINE_OFFSET, idxd->groups[i].grpcfg.engines);
/* GRPFLAGS, flags config */
_idxd_write_8(idxd, base_offset + CFG_FLAG_OFFSET, idxd->groups[i].grpcfg.flags.raw);
}
return 0;
}
/*
* Build work queue (WQ) config based on getting info from the device combined
* with the defined configuration. Once built, it is written to the device.
*/
static int
idxd_wq_config(struct spdk_user_idxd_device *user_idxd)
{
int i, j;
struct idxd_wq *queue;
struct spdk_idxd_device *idxd = &user_idxd->idxd;
u_int32_t wq_size = user_idxd->registers.wqcap.total_wq_size / g_user_dev_cfg.total_wqs;
SPDK_NOTICELOG("Total ring slots available space 0x%x, so per work queue is 0x%x\n",
user_idxd->registers.wqcap.total_wq_size, wq_size);
assert(g_user_dev_cfg.total_wqs <= IDXD_MAX_QUEUES);
assert(g_user_dev_cfg.total_wqs <= user_idxd->registers.wqcap.num_wqs);
assert(LOG2_WQ_MAX_BATCH <= user_idxd->registers.gencap.max_batch_shift);
assert(LOG2_WQ_MAX_XFER <= user_idxd->registers.gencap.max_xfer_shift);
idxd->queues = calloc(1, user_idxd->registers.wqcap.num_wqs * sizeof(struct idxd_wq));
if (idxd->queues == NULL) {
SPDK_ERRLOG("Failed to allocate queue memory\n");
return -ENOMEM;
}
for (i = 0; i < g_user_dev_cfg.total_wqs; i++) {
queue = &user_idxd->idxd.queues[i];
queue->wqcfg.wq_size = wq_size;
queue->wqcfg.mode = WQ_MODE_DEDICATED;
queue->wqcfg.max_batch_shift = LOG2_WQ_MAX_BATCH;
queue->wqcfg.max_xfer_shift = LOG2_WQ_MAX_XFER;
queue->wqcfg.wq_state = WQ_ENABLED;
queue->wqcfg.priority = WQ_PRIORITY_1;
/* Not part of the config struct */
queue->idxd = &user_idxd->idxd;
queue->group = &idxd->groups[i % g_user_dev_cfg.num_groups];
}
/*
* Now write the work queue config to the device for all wq space
*/
for (i = 0 ; i < user_idxd->registers.wqcap.num_wqs; i++) {
queue = &idxd->queues[i];
for (j = 0 ; j < WQCFG_NUM_DWORDS; j++) {
_idxd_write_4(idxd, user_idxd->wqcfg_offset + i * 32 + j * 4,
queue->wqcfg.raw[j]);
}
}
return 0;
}
static int
idxd_device_configure(struct spdk_user_idxd_device *user_idxd)
{
int i, rc = 0;
union idxd_offsets_register offsets_reg;
union idxd_genstatus_register genstatus_reg;
struct spdk_idxd_device *idxd = &user_idxd->idxd;
/*
* Map BAR0 and BAR2
*/
rc = idxd_map_pci_bars(idxd);
if (rc) {
return rc;
}
/*
* Reset the device
*/
rc = idxd_reset_dev(idxd);
if (rc) {
goto err_reset;
}
/*
* Read in config registers
*/
user_idxd->registers.version = _idxd_read_4(idxd, IDXD_VERSION_OFFSET);
user_idxd->registers.gencap.raw = _idxd_read_8(idxd, IDXD_GENCAP_OFFSET);
user_idxd->registers.wqcap.raw = _idxd_read_8(idxd, IDXD_WQCAP_OFFSET);
user_idxd->registers.groupcap.raw = _idxd_read_8(idxd, IDXD_GRPCAP_OFFSET);
user_idxd->registers.enginecap.raw = _idxd_read_8(idxd, IDXD_ENGCAP_OFFSET);
for (i = 0; i < IDXD_OPCAP_WORDS; i++) {
user_idxd->registers.opcap.raw[i] =
_idxd_read_8(idxd, i * sizeof(uint64_t) + IDXD_OPCAP_OFFSET);
}
offsets_reg.raw[0] = _idxd_read_8(idxd, IDXD_TABLE_OFFSET);
offsets_reg.raw[1] = _idxd_read_8(idxd, IDXD_TABLE_OFFSET + sizeof(uint64_t));
user_idxd->grpcfg_offset = offsets_reg.grpcfg * IDXD_TABLE_OFFSET_MULT;
user_idxd->wqcfg_offset = offsets_reg.wqcfg * IDXD_TABLE_OFFSET_MULT;
user_idxd->ims_offset = offsets_reg.ims * IDXD_TABLE_OFFSET_MULT;
user_idxd->msix_perm_offset = offsets_reg.msix_perm * IDXD_TABLE_OFFSET_MULT;
user_idxd->perfmon_offset = offsets_reg.perfmon * IDXD_TABLE_OFFSET_MULT;
/*
* Configure groups and work queues.
*/
rc = idxd_group_config(idxd);
if (rc) {
goto err_group_cfg;
}
rc = idxd_wq_config(user_idxd);
if (rc) {
goto err_wq_cfg;
}
/*
* Enable the device
*/
genstatus_reg.raw = _idxd_read_4(idxd, IDXD_GENSTATUS_OFFSET);
assert(genstatus_reg.state == IDXD_DEVICE_STATE_DISABLED);
_idxd_write_4(idxd, IDXD_CMD_OFFSET, IDXD_ENABLE_DEV << IDXD_CMD_SHIFT);
rc = idxd_wait_cmd(idxd, IDXD_REGISTER_TIMEOUT_US);
genstatus_reg.raw = _idxd_read_4(idxd, IDXD_GENSTATUS_OFFSET);
if ((rc < 0) || (genstatus_reg.state != IDXD_DEVICE_STATE_ENABLED)) {
rc = -EINVAL;
SPDK_ERRLOG("Error enabling device %u\n", rc);
goto err_device_enable;
}
genstatus_reg.raw = spdk_mmio_read_4((uint32_t *)(user_idxd->reg_base + IDXD_GENSTATUS_OFFSET));
assert(genstatus_reg.state == IDXD_DEVICE_STATE_ENABLED);
/*
* Enable the work queues that we've configured
*/
for (i = 0; i < g_user_dev_cfg.total_wqs; i++) {
_idxd_write_4(idxd, IDXD_CMD_OFFSET,
(IDXD_ENABLE_WQ << IDXD_CMD_SHIFT) | i);
rc = idxd_wait_cmd(idxd, IDXD_REGISTER_TIMEOUT_US);
if (rc < 0) {
SPDK_ERRLOG("Error enabling work queues 0x%x\n", rc);
goto err_wq_enable;
}
}
if ((rc == 0) && (genstatus_reg.state == IDXD_DEVICE_STATE_ENABLED)) {
SPDK_NOTICELOG("Device enabled, version 0x%x gencap: 0x%lx\n",
user_idxd->registers.version,
user_idxd->registers.gencap.raw);
}
return rc;
err_wq_enable:
err_device_enable:
free(idxd->queues);
err_wq_cfg:
free(idxd->groups);
err_group_cfg:
err_reset:
idxd_unmap_pci_bar(idxd, IDXD_MMIO_BAR);
idxd_unmap_pci_bar(idxd, IDXD_MMIO_BAR);
return rc;
}
static void
user_idxd_device_destruct(struct spdk_idxd_device *idxd)
{
idxd_unmap_pci_bar(idxd, IDXD_MMIO_BAR);
idxd_unmap_pci_bar(idxd, IDXD_WQ_BAR);
free(idxd->groups);
free(idxd->queues);
free(idxd);
}
struct idxd_enum_ctx {
spdk_idxd_probe_cb probe_cb;
spdk_idxd_attach_cb attach_cb;
void *cb_ctx;
};
/* This function must only be called while holding g_driver_lock */
static int
idxd_enum_cb(void *ctx, struct spdk_pci_device *pci_dev)
{
struct idxd_enum_ctx *enum_ctx = ctx;
struct spdk_idxd_device *idxd;
if (enum_ctx->probe_cb(enum_ctx->cb_ctx, pci_dev)) {
idxd = idxd_attach(pci_dev);
if (idxd == NULL) {
SPDK_ERRLOG("idxd_attach() failed\n");
return -EINVAL;
}
enum_ctx->attach_cb(enum_ctx->cb_ctx, pci_dev, idxd);
}
return 0;
}
static int
user_idxd_probe(void *cb_ctx, spdk_idxd_probe_cb probe_cb, spdk_idxd_attach_cb attach_cb)
{
int rc;
struct idxd_enum_ctx enum_ctx;
enum_ctx.probe_cb = probe_cb;
enum_ctx.attach_cb = attach_cb;
enum_ctx.cb_ctx = cb_ctx;
pthread_mutex_lock(&g_driver_lock);
rc = spdk_pci_enumerate(spdk_pci_idxd_get_driver(), idxd_enum_cb, &enum_ctx);
pthread_mutex_unlock(&g_driver_lock);
return rc;
}
static char *
user_idxd_portal_get_addr(struct spdk_idxd_device *idxd)
{
return (char *)idxd->portals + idxd->wq_id * PORTAL_SIZE;
}
static bool
user_idxd_nop_check(struct spdk_idxd_device *idxd)
{
struct spdk_user_idxd_device *user_idxd = __user_idxd(idxd);
/* TODO: temp workaround for simulator. Remove this function when fixed or w/silicon. */
if (user_idxd->registers.gencap.raw == 0x1833f011f) {
return true;
}
return false;
}
static struct spdk_idxd_impl g_user_idxd_impl = {
.name = "user",
.set_config = user_idxd_set_config,
.probe = user_idxd_probe,
.destruct = user_idxd_device_destruct,
.read_8 = idxd_read_8,
.portal_get_addr = user_idxd_portal_get_addr,
.nop_check = user_idxd_nop_check,
};
/* Caller must hold g_driver_lock */
static struct spdk_idxd_device *
idxd_attach(struct spdk_pci_device *device)
{
struct spdk_user_idxd_device *user_idxd;
struct spdk_idxd_device *idxd;
uint32_t cmd_reg;
int rc;
user_idxd = calloc(1, sizeof(struct spdk_user_idxd_device));
if (user_idxd == NULL) {
SPDK_ERRLOG("Failed to allocate memory for user_idxd device.\n");
return NULL;
}
idxd = &user_idxd->idxd;
idxd->impl = &g_user_idxd_impl;
idxd->device = device;
pthread_mutex_init(&idxd->num_channels_lock, NULL);
/* Enable PCI busmaster. */
spdk_pci_device_cfg_read32(device, &cmd_reg, 4);
cmd_reg |= 0x4;
spdk_pci_device_cfg_write32(device, cmd_reg, 4);
rc = idxd_device_configure(user_idxd);
if (rc) {
goto err;
}
return idxd;
err:
user_idxd_device_destruct(idxd);
return NULL;
}
SPDK_IDXD_IMPL_REGISTER(user, &g_user_idxd_impl);

2
test/unit/lib/idxd/Makefile
View File

@ -34,7 +34,7 @@
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
DIRS-y = idxd.c
DIRS-y = idxd.c idxd_user.c
.PHONY: all clean $(DIRS-y)

211
test/unit/lib/idxd/idxd.c/idxd_ut.c
View File

@ -37,210 +37,17 @@
#include "common/lib/test_env.c"
#include "idxd/idxd.h"
#define FAKE_REG_SIZE 0x800
#define GRP_CFG_OFFSET 0x400
#define MAX_TOKENS 0x40
#define MAX_ARRAY_SIZE 0x20
DEFINE_STUB(spdk_pci_idxd_get_driver, struct spdk_pci_driver *, (void), NULL);
int
spdk_pci_enumerate(struct spdk_pci_driver *driver, spdk_pci_enum_cb enum_cb, void *enum_ctx)
{
return -1;
}
int
spdk_pci_device_map_bar(struct spdk_pci_device *dev, uint32_t bar,
void **mapped_addr, uint64_t *phys_addr, uint64_t *size)
{
*mapped_addr = NULL;
*phys_addr = 0;
*size = 0;
return 0;
}
int
spdk_pci_device_unmap_bar(struct spdk_pci_device *dev, uint32_t bar, void *addr)
{
return 0;
}
int
spdk_pci_device_cfg_read32(struct spdk_pci_device *dev, uint32_t *value,
uint32_t offset)
{
*value = 0xFFFFFFFFu;
return 0;
}
int
spdk_pci_device_cfg_write32(struct spdk_pci_device *dev, uint32_t value,
uint32_t offset)
{
return 0;
}
#define movdir64b mock_movdir64b
static inline void
mock_movdir64b(void *dst, const void *src)
{
return;
}
#include "idxd/idxd.c"
#define WQ_CFG_OFFSET 0x500
#define TOTAL_WQE_SIZE 0x40
static int
test_idxd_wq_config(void)
static void
user_idxd_set_config(struct device_config *dev_cfg, uint32_t config_num)
{
struct spdk_idxd_device idxd = {};
union idxd_wqcfg wqcfg = {};
uint32_t expected[8] = {0x40, 0, 0x11, 0x9e, 0, 0, 0x40000000, 0};
uint32_t wq_size;
int rc, i, j;
idxd.reg_base = calloc(1, FAKE_REG_SIZE);
SPDK_CU_ASSERT_FATAL(idxd.reg_base != NULL);
SPDK_CU_ASSERT_FATAL(g_dev_cfg->num_groups <= MAX_ARRAY_SIZE);
idxd.groups = calloc(g_dev_cfg->num_groups, sizeof(struct idxd_group));
SPDK_CU_ASSERT_FATAL(idxd.groups != NULL);
idxd.registers.wqcap.total_wq_size = TOTAL_WQE_SIZE;
idxd.registers.wqcap.num_wqs = g_dev_cfg->total_wqs;
idxd.registers.gencap.max_batch_shift = LOG2_WQ_MAX_BATCH;
idxd.registers.gencap.max_xfer_shift = LOG2_WQ_MAX_XFER;
idxd.wqcfg_offset = WQ_CFG_OFFSET;
wq_size = idxd.registers.wqcap.total_wq_size / g_dev_cfg->total_wqs;
rc = idxd_wq_config(&idxd);
CU_ASSERT(rc == 0);
for (i = 0; i < g_dev_cfg->total_wqs; i++) {
CU_ASSERT(idxd.queues[i].wqcfg.wq_size == wq_size);
CU_ASSERT(idxd.queues[i].wqcfg.mode == WQ_MODE_DEDICATED);
CU_ASSERT(idxd.queues[i].wqcfg.max_batch_shift == LOG2_WQ_MAX_BATCH);
CU_ASSERT(idxd.queues[i].wqcfg.max_xfer_shift == LOG2_WQ_MAX_XFER);
CU_ASSERT(idxd.queues[i].wqcfg.wq_state == WQ_ENABLED);
CU_ASSERT(idxd.queues[i].wqcfg.priority == WQ_PRIORITY_1);
CU_ASSERT(idxd.queues[i].idxd == &idxd);
CU_ASSERT(idxd.queues[i].group == &idxd.groups[i % g_dev_cfg->num_groups]);
}
for (i = 0 ; i < idxd.registers.wqcap.num_wqs; i++) {
for (j = 0 ; j < WQCFG_NUM_DWORDS; j++) {
wqcfg.raw[j] = spdk_mmio_read_4((uint32_t *)(idxd.reg_base + idxd.wqcfg_offset + i * 32 + j *
4));
CU_ASSERT(wqcfg.raw[j] == expected[j]);
}
}
free(idxd.queues);
free(idxd.reg_base);
free(idxd.groups);
return 0;
}
static int
test_idxd_group_config(void)
{
struct spdk_idxd_device idxd = {};
uint64_t wqs[MAX_ARRAY_SIZE] = {};
uint64_t engines[MAX_ARRAY_SIZE] = {};
union idxd_group_flags flags[MAX_ARRAY_SIZE] = {};
int rc, i;
uint64_t base_offset;
idxd.reg_base = calloc(1, FAKE_REG_SIZE);
SPDK_CU_ASSERT_FATAL(idxd.reg_base != NULL);
SPDK_CU_ASSERT_FATAL(g_dev_cfg->num_groups <= MAX_ARRAY_SIZE);
idxd.registers.groupcap.num_groups = g_dev_cfg->num_groups;
idxd.registers.enginecap.num_engines = g_dev_cfg->total_engines;
idxd.registers.wqcap.num_wqs = g_dev_cfg->total_wqs;
idxd.registers.groupcap.total_tokens = MAX_TOKENS;
idxd.grpcfg_offset = GRP_CFG_OFFSET;
rc = idxd_group_config(&idxd);
CU_ASSERT(rc == 0);
for (i = 0 ; i < idxd.registers.groupcap.num_groups; i++) {
base_offset = idxd.grpcfg_offset + i * 64;
wqs[i] = spdk_mmio_read_8((uint64_t *)(idxd.reg_base + base_offset));
engines[i] = spdk_mmio_read_8((uint64_t *)(idxd.reg_base + base_offset + CFG_ENGINE_OFFSET));
flags[i].raw = spdk_mmio_read_8((uint64_t *)(idxd.reg_base + base_offset + CFG_FLAG_OFFSET));
}
/* wqe and engine arrays are indexed by group id and are bitmaps of assigned elements. */
CU_ASSERT(wqs[0] == 0x1);
CU_ASSERT(engines[0] == 0xf);
CU_ASSERT(flags[0].tokens_allowed == MAX_TOKENS / g_dev_cfg->num_groups);
/* groups allocated by code under test. */
free(idxd.groups);
free(idxd.reg_base);
return 0;
}
static int
test_idxd_reset_dev(void)
{
struct spdk_idxd_device idxd = {};
union idxd_cmdsts_reg *fake_cmd_status_reg;
int rc;
idxd.reg_base = calloc(1, FAKE_REG_SIZE);
SPDK_CU_ASSERT_FATAL(idxd.reg_base != NULL);
fake_cmd_status_reg = idxd.reg_base + IDXD_CMDSTS_OFFSET;
/* Test happy path */
rc = idxd_reset_dev(&idxd);
CU_ASSERT(rc == 0);
/* Test error reported path */
fake_cmd_status_reg->err = 1;
rc = idxd_reset_dev(&idxd);
CU_ASSERT(rc == -EINVAL);
free(idxd.reg_base);
return 0;
}
static int
test_idxd_wait_cmd(void)
{
struct spdk_idxd_device idxd = {};
int timeout = 1;
union idxd_cmdsts_reg *fake_cmd_status_reg;
int rc;
idxd.reg_base = calloc(1, FAKE_REG_SIZE);
SPDK_CU_ASSERT_FATAL(idxd.reg_base != NULL);
fake_cmd_status_reg = idxd.reg_base + IDXD_CMDSTS_OFFSET;
/* Test happy path. */
rc = idxd_wait_cmd(&idxd, timeout);
CU_ASSERT(rc == 0);
/* Setup up our fake register to set the error bit. */
fake_cmd_status_reg->err = 1;
rc = idxd_wait_cmd(&idxd, timeout);
CU_ASSERT(rc == -EINVAL);
fake_cmd_status_reg->err = 0;
/* Setup up our fake register to set the active bit. */
fake_cmd_status_reg->active = 1;
rc = idxd_wait_cmd(&idxd, timeout);
CU_ASSERT(rc == -EBUSY);
free(idxd.reg_base);
return 0;
}
static struct spdk_idxd_impl g_user_idxd_impl = {
.name = "user",
.set_config = user_idxd_set_config,
};
static int
test_spdk_idxd_set_config(void)
@ -283,6 +90,8 @@ test_spdk_idxd_reconfigure_chan(void)
static int
test_setup(void)
{
idxd_impl_register(&g_user_idxd_impl);
g_dev_cfg = &g_dev_cfg0;
return 0;
}
@ -299,10 +108,6 @@ int main(int argc, char **argv)
CU_ADD_TEST(suite, test_spdk_idxd_reconfigure_chan);
CU_ADD_TEST(suite, test_spdk_idxd_set_config);
CU_ADD_TEST(suite, test_idxd_wait_cmd);
CU_ADD_TEST(suite, test_idxd_reset_dev);
CU_ADD_TEST(suite, test_idxd_group_config);
CU_ADD_TEST(suite, test_idxd_wq_config);
CU_basic_set_mode(CU_BRM_VERBOSE);
CU_basic_run_tests();

1
test/unit/lib/idxd/idxd_user.c/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
idxd_user_ut

38
test/unit/lib/idxd/idxd_user.c/Makefile Normal file
View File

@ -0,0 +1,38 @@
#
# BSD LICENSE
#
# Copyright (c) Intel Corporation.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../../..)
TEST_FILE = idxd_user_ut.c
include $(SPDK_ROOT_DIR)/mk/spdk.unittest.mk

272
test/unit/lib/idxd/idxd_user.c/idxd_user_ut.c Normal file
View File

@ -0,0 +1,272 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "spdk_cunit.h"
#include "spdk_internal/mock.h"
#include "spdk_internal/idxd.h"
#include "common/lib/test_env.c"
#include "idxd/idxd.h"
#include "idxd/idxd_user.c"
#define FAKE_REG_SIZE 0x800
#define GRP_CFG_OFFSET 0x400
#define MAX_TOKENS 0x40
#define MAX_ARRAY_SIZE 0x20
DEFINE_STUB(spdk_pci_idxd_get_driver, struct spdk_pci_driver *, (void), NULL);
DEFINE_STUB_V(idxd_impl_register, (struct spdk_idxd_impl *impl));
int
spdk_pci_enumerate(struct spdk_pci_driver *driver, spdk_pci_enum_cb enum_cb, void *enum_ctx)
{
return -1;
}
int
spdk_pci_device_map_bar(struct spdk_pci_device *dev, uint32_t bar,
void **mapped_addr, uint64_t *phys_addr, uint64_t *size)
{
*mapped_addr = NULL;
*phys_addr = 0;
*size = 0;
return 0;
}
int
spdk_pci_device_unmap_bar(struct spdk_pci_device *dev, uint32_t bar, void *addr)
{
return 0;
}
int
spdk_pci_device_cfg_read32(struct spdk_pci_device *dev, uint32_t *value,
uint32_t offset)
{
*value = 0xFFFFFFFFu;
return 0;
}
int
spdk_pci_device_cfg_write32(struct spdk_pci_device *dev, uint32_t value,
uint32_t offset)
{
return 0;
}
#define WQ_CFG_OFFSET 0x500
#define TOTAL_WQE_SIZE 0x40
static int
test_idxd_wq_config(void)
{
struct spdk_user_idxd_device user_idxd = {};
struct spdk_idxd_device *idxd = &user_idxd.idxd;
union idxd_wqcfg wqcfg = {};
uint32_t expected[8] = {0x40, 0, 0x11, 0x9e, 0, 0, 0x40000000, 0};
uint32_t wq_size;
int rc, i, j;
user_idxd.reg_base = calloc(1, FAKE_REG_SIZE);
SPDK_CU_ASSERT_FATAL(user_idxd.reg_base != NULL);
SPDK_CU_ASSERT_FATAL(g_user_dev_cfg.num_groups <= MAX_ARRAY_SIZE);
idxd->groups = calloc(g_user_dev_cfg.num_groups, sizeof(struct idxd_group));
SPDK_CU_ASSERT_FATAL(idxd->groups != NULL);
user_idxd.registers.wqcap.total_wq_size = TOTAL_WQE_SIZE;
user_idxd.registers.wqcap.num_wqs = g_user_dev_cfg.total_wqs;
user_idxd.registers.gencap.max_batch_shift = LOG2_WQ_MAX_BATCH;
user_idxd.registers.gencap.max_xfer_shift = LOG2_WQ_MAX_XFER;
user_idxd.wqcfg_offset = WQ_CFG_OFFSET;
wq_size = user_idxd.registers.wqcap.total_wq_size / g_user_dev_cfg.total_wqs;
rc = idxd_wq_config(&user_idxd);
CU_ASSERT(rc == 0);
for (i = 0; i < g_user_dev_cfg.total_wqs; i++) {
CU_ASSERT(idxd->queues[i].wqcfg.wq_size == wq_size);
CU_ASSERT(idxd->queues[i].wqcfg.mode == WQ_MODE_DEDICATED);
CU_ASSERT(idxd->queues[i].wqcfg.max_batch_shift == LOG2_WQ_MAX_BATCH);
CU_ASSERT(idxd->queues[i].wqcfg.max_xfer_shift == LOG2_WQ_MAX_XFER);
CU_ASSERT(idxd->queues[i].wqcfg.wq_state == WQ_ENABLED);
CU_ASSERT(idxd->queues[i].wqcfg.priority == WQ_PRIORITY_1);
CU_ASSERT(idxd->queues[i].idxd == idxd);
CU_ASSERT(idxd->queues[i].group == &idxd->groups[i % g_user_dev_cfg.num_groups]);
}
for (i = 0 ; i < user_idxd.registers.wqcap.num_wqs; i++) {
for (j = 0 ; j < WQCFG_NUM_DWORDS; j++) {
wqcfg.raw[j] = spdk_mmio_read_4((uint32_t *)(user_idxd.reg_base + user_idxd.wqcfg_offset + i * 32 +
j *
4));
CU_ASSERT(wqcfg.raw[j] == expected[j]);
}
}
free(idxd->queues);
free(user_idxd.reg_base);
free(idxd->groups);
return 0;
}
static int
test_idxd_group_config(void)
{
struct spdk_user_idxd_device user_idxd = {};
struct spdk_idxd_device *idxd = &user_idxd.idxd;
uint64_t wqs[MAX_ARRAY_SIZE] = {};
uint64_t engines[MAX_ARRAY_SIZE] = {};
union idxd_group_flags flags[MAX_ARRAY_SIZE] = {};
int rc, i;
uint64_t base_offset;
user_idxd.reg_base = calloc(1, FAKE_REG_SIZE);
SPDK_CU_ASSERT_FATAL(user_idxd.reg_base != NULL);
SPDK_CU_ASSERT_FATAL(g_user_dev_cfg.num_groups <= MAX_ARRAY_SIZE);
user_idxd.registers.groupcap.num_groups = g_user_dev_cfg.num_groups;
user_idxd.registers.enginecap.num_engines = g_user_dev_cfg.total_engines;
user_idxd.registers.wqcap.num_wqs = g_user_dev_cfg.total_wqs;
user_idxd.registers.groupcap.total_tokens = MAX_TOKENS;
user_idxd.grpcfg_offset = GRP_CFG_OFFSET;
rc = idxd_group_config(idxd);
CU_ASSERT(rc == 0);
for (i = 0 ; i < user_idxd.registers.groupcap.num_groups; i++) {
base_offset = user_idxd.grpcfg_offset + i * 64;
wqs[i] = spdk_mmio_read_8((uint64_t *)(user_idxd.reg_base + base_offset));
engines[i] = spdk_mmio_read_8((uint64_t *)(user_idxd.reg_base + base_offset + CFG_ENGINE_OFFSET));
flags[i].raw = spdk_mmio_read_8((uint64_t *)(user_idxd.reg_base + base_offset + CFG_FLAG_OFFSET));
}
/* wqe and engine arrays are indexed by group id and are bitmaps of assigned elements. */
CU_ASSERT(wqs[0] == 0x1);
CU_ASSERT(engines[0] == 0xf);
CU_ASSERT(flags[0].tokens_allowed == MAX_TOKENS / g_user_dev_cfg.num_groups);
/* groups allocated by code under test. */
free(idxd->groups);
free(user_idxd.reg_base);
return 0;
}
static int
test_idxd_reset_dev(void)
{
struct spdk_user_idxd_device user_idxd = {};
union idxd_cmdsts_reg *fake_cmd_status_reg;
int rc;
user_idxd.reg_base = calloc(1, FAKE_REG_SIZE);
SPDK_CU_ASSERT_FATAL(user_idxd.reg_base != NULL);
fake_cmd_status_reg = user_idxd.reg_base + IDXD_CMDSTS_OFFSET;
/* Test happy path */
rc = idxd_reset_dev(&user_idxd.idxd);
CU_ASSERT(rc == 0);
/* Test error reported path */
fake_cmd_status_reg->err = 1;
rc = idxd_reset_dev(&user_idxd.idxd);
CU_ASSERT(rc == -EINVAL);
free(user_idxd.reg_base);
return 0;
}
static int
test_idxd_wait_cmd(void)
{
struct spdk_user_idxd_device user_idxd = {};
int timeout = 1;
union idxd_cmdsts_reg *fake_cmd_status_reg;
int rc;
user_idxd.reg_base = calloc(1, FAKE_REG_SIZE);
SPDK_CU_ASSERT_FATAL(user_idxd.reg_base != NULL);
fake_cmd_status_reg = user_idxd.reg_base + IDXD_CMDSTS_OFFSET;
/* Test happy path. */
rc = idxd_wait_cmd(&user_idxd.idxd, timeout);
CU_ASSERT(rc == 0);
/* Setup up our fake register to set the error bit. */
fake_cmd_status_reg->err = 1;
rc = idxd_wait_cmd(&user_idxd.idxd, timeout);
CU_ASSERT(rc == -EINVAL);
fake_cmd_status_reg->err = 0;
/* Setup up our fake register to set the active bit. */
fake_cmd_status_reg->active = 1;
rc = idxd_wait_cmd(&user_idxd.idxd, timeout);
CU_ASSERT(rc == -EBUSY);
free(user_idxd.reg_base);
return 0;
}
static int
test_setup(void)
{
g_user_dev_cfg.config_num = 0;
g_user_dev_cfg.num_groups = 1;
g_user_dev_cfg.total_wqs = 1;
g_user_dev_cfg.total_engines = 4;
return 0;
}
int main(int argc, char **argv)
{
CU_pSuite suite = NULL;
unsigned int num_failures;
CU_set_error_action(CUEA_ABORT);
CU_initialize_registry();
suite = CU_add_suite("idxd_user", test_setup, NULL);
CU_ADD_TEST(suite, test_idxd_wait_cmd);
CU_ADD_TEST(suite, test_idxd_reset_dev);
CU_ADD_TEST(suite, test_idxd_group_config);
CU_ADD_TEST(suite, test_idxd_wq_config);
CU_basic_set_mode(CU_BRM_VERBOSE);
CU_basic_run_tests();
num_failures = CU_get_number_of_failures();
CU_cleanup_registry();
return num_failures;
}

1
test/unit/unittest.sh
View File

@ -201,6 +201,7 @@ run_test "unittest_accel" $valgrind $testdir/lib/accel/accel.c/accel_engine_ut
run_test "unittest_ioat" $valgrind $testdir/lib/ioat/ioat.c/ioat_ut
if grep -q '#define SPDK_CONFIG_IDXD 1' $rootdir/include/spdk/config.h; then
run_test "unittest_idxd" $valgrind $testdir/lib/idxd/idxd.c/idxd_ut
run_test "unittest_idxd_user" $valgrind $testdir/lib/idxd/idxd_user.c/idxd_user_ut
fi
run_test "unittest_iscsi" unittest_iscsi
run_test "unittest_json" unittest_json