module/accel/ioat: add batching suport for sw operations
This patch has the basic infrastructure to support the accel framework batching API but only for commands not HW accelerated by IOAT, that will come in the next patch... Signed-off-by: paul luse <paul.e.luse@intel.com> Change-Id: I6168831ac5698a9e58a81ef35ce919d75a72d0f5 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/3153 Tested-by: SPDK CI Jenkins <sys_sgci@intel.com> Reviewed-by: Jim Harris <james.r.harris@intel.com> Reviewed-by: Ben Walker <benjamin.walker@intel.com> Community-CI: Mellanox Build Bot
This commit is contained in:
paul luse
Jim Harris
parent
6a9e923da2
commit
be34c31e16
@ -110,7 +110,7 @@ DEPDIRS-blob_bdev := log thread bdev
|
||||
DEPDIRS-blobfs_bdev := $(BDEV_DEPS_THREAD) blob_bdev blobfs
|
||||
|
||||
# module/accel
|
||||
DEPDIRS-accel_ioat := log ioat conf thread $(JSON_LIBS) accel
|
||||
DEPDIRS-accel_ioat := log ioat conf thread $(JSON_LIBS) accel util
|
||||
DEPDIRS-accel_idxd := log idxd thread $(JSON_LIBS) accel
|
||||
|
||||
# module/env_dpdk
|
||||
|
||||
@ -43,7 +43,36 @@
|
||||
#include "spdk/event.h"
|
||||
#include "spdk/thread.h"
|
||||
#include "spdk/ioat.h"
|
||||
#include "spdk/crc32.h"
|
||||
|
||||
#define ALIGN_4K 0x1000
|
||||
|
||||
enum ioat_accel_opcode {
|
||||
IOAT_ACCEL_OPCODE_MEMMOVE = 0,
|
||||
IOAT_ACCEL_OPCODE_MEMFILL = 1,
|
||||
IOAT_ACCEL_OPCODE_COMPARE = 2,
|
||||
IOAT_ACCEL_OPCODE_CRC32C = 3,
|
||||
IOAT_ACCEL_OPCODE_DUALCAST = 4,
|
||||
};
|
||||
|
||||
struct ioat_accel_op {
|
||||
struct ioat_io_channel *ioat_ch;
|
||||
void *cb_arg;
|
||||
spdk_accel_completion_cb cb_fn;
|
||||
void *src;
|
||||
union {
|
||||
void *dst;
|
||||
void *src2;
|
||||
};
|
||||
void *dst2;
|
||||
uint32_t seed;
|
||||
uint64_t fill_pattern;
|
||||
enum ioat_accel_opcode op_code;
|
||||
uint64_t nbytes;
|
||||
TAILQ_ENTRY(ioat_accel_op) link;
|
||||
};
|
||||
|
||||
static int g_batch_size;
|
||||
static bool g_ioat_enable = false;
|
||||
static bool g_ioat_initialized = false;
|
||||
|
||||
@ -71,11 +100,13 @@ static pthread_mutex_t g_ioat_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
|
||||
static TAILQ_HEAD(, pci_device) g_pci_devices = TAILQ_HEAD_INITIALIZER(g_pci_devices);
|
||||
|
||||
|
||||
struct ioat_io_channel {
|
||||
struct spdk_ioat_chan *ioat_ch;
|
||||
struct ioat_device *ioat_dev;
|
||||
struct spdk_poller *poller;
|
||||
struct spdk_ioat_chan *ioat_ch;
|
||||
struct ioat_device *ioat_dev;
|
||||
struct spdk_poller *poller;
|
||||
TAILQ_HEAD(, ioat_accel_op) op_pool;
|
||||
TAILQ_HEAD(, ioat_accel_op) sw_batch; /* for operations not hw accelerated */
|
||||
bool hw_batch; /* for operations that are hw accelerated */
|
||||
};
|
||||
|
||||
static int
|
||||
@ -191,19 +222,224 @@ ioat_poll(void *arg)
|
||||
static struct spdk_io_channel *ioat_get_io_channel(void);
|
||||
|
||||
/*
|
||||
* The IOAT engine has more capabilities than this but these are
|
||||
* the only ones we expose via the accel engine.
|
||||
* The IOAT engine only supports these capabilities as hardware
|
||||
* accelerated. The accel fw will handle unsupported functions
|
||||
* by calling the software implementations of the functions.
|
||||
*/
|
||||
static uint64_t
|
||||
ioat_get_capabilities(void)
|
||||
{
|
||||
return ACCEL_COPY | ACCEL_FILL;
|
||||
return ACCEL_COPY | ACCEL_FILL | ACCEL_BATCH;
|
||||
}
|
||||
|
||||
/* The IOAT batch functions exposed by the accel fw do not match up 1:1
|
||||
* with the functions in the IOAT library. The IOAT library directly only
|
||||
* supports construction of accelerated functions via the IOAT native
|
||||
* interface. The accel_fw batch capabilities are implemented here in the
|
||||
* plug-in and rely on either the IOAT library for accelerated commands
|
||||
* or software functions for non-accelerated.
|
||||
*/
|
||||
static uint32_t
|
||||
ioat_batch_get_max(void)
|
||||
{
|
||||
return g_batch_size;
|
||||
}
|
||||
|
||||
static struct spdk_accel_batch *
|
||||
ioat_batch_create(struct spdk_io_channel *ch)
|
||||
{
|
||||
struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch);
|
||||
|
||||
if (!TAILQ_EMPTY(&ioat_ch->sw_batch) || (ioat_ch->hw_batch == true)) {
|
||||
SPDK_ERRLOG("IOAT accel engine only supports one batch at a time.\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return (struct spdk_accel_batch *)&ioat_ch->hw_batch;
|
||||
}
|
||||
|
||||
static struct ioat_accel_op *
|
||||
_prep_op(void *cb_arg, struct ioat_io_channel *ioat_ch, struct spdk_accel_batch *batch,
|
||||
spdk_accel_completion_cb cb)
|
||||
{
|
||||
struct ioat_accel_op *op;
|
||||
|
||||
if ((struct spdk_accel_batch *)&ioat_ch->hw_batch != batch) {
|
||||
SPDK_ERRLOG("Invalid batch\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (!TAILQ_EMPTY(&ioat_ch->op_pool)) {
|
||||
op = TAILQ_FIRST(&ioat_ch->op_pool);
|
||||
TAILQ_REMOVE(&ioat_ch->op_pool, op, link);
|
||||
} else {
|
||||
SPDK_ERRLOG("Ran out of operations for batch\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
op->cb_arg = cb_arg;
|
||||
op->cb_fn = cb;
|
||||
op->ioat_ch = ioat_ch;
|
||||
|
||||
return op;
|
||||
}
|
||||
|
||||
static int
|
||||
ioat_batch_prep_copy(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
|
||||
void *dst, void *src, uint64_t nbytes, spdk_accel_completion_cb cb)
|
||||
{
|
||||
/* TODO - HW ACCELERATED */
|
||||
return 0;;
|
||||
}
|
||||
|
||||
static int
|
||||
ioat_batch_prep_fill(void *cb_arg, struct spdk_io_channel *ch,
|
||||
struct spdk_accel_batch *batch, void *dst, uint8_t fill,
|
||||
uint64_t nbytes, spdk_accel_completion_cb cb)
|
||||
{
|
||||
/* TODO - HW ACCELERATED */
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
ioat_batch_prep_dualcast(void *cb_arg, struct spdk_io_channel *ch,
|
||||
struct spdk_accel_batch *batch, void *dst1, void *dst2,
|
||||
void *src, uint64_t nbytes, spdk_accel_completion_cb cb)
|
||||
{
|
||||
struct ioat_accel_op *op;
|
||||
struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch);
|
||||
|
||||
if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) {
|
||||
SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
op = _prep_op(cb_arg, ioat_ch, batch, cb);
|
||||
if (op == NULL) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
/* Command specific. */
|
||||
op->src = src;
|
||||
op->dst = dst1;
|
||||
op->dst2 = dst2;
|
||||
op->nbytes = nbytes;
|
||||
op->op_code = IOAT_ACCEL_OPCODE_DUALCAST;
|
||||
TAILQ_INSERT_TAIL(&ioat_ch->sw_batch, op, link);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
ioat_batch_prep_compare(void *cb_arg, struct spdk_io_channel *ch,
|
||||
struct spdk_accel_batch *batch, void *src1,
|
||||
void *src2, uint64_t nbytes, spdk_accel_completion_cb cb)
|
||||
{
|
||||
struct ioat_accel_op *op;
|
||||
struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch);
|
||||
|
||||
op = _prep_op(cb_arg, ioat_ch, batch, cb);
|
||||
if (op == NULL) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
/* Command specific. */
|
||||
op->src = src1;
|
||||
op->src2 = src2;
|
||||
op->nbytes = nbytes;
|
||||
op->op_code = IOAT_ACCEL_OPCODE_COMPARE;
|
||||
TAILQ_INSERT_TAIL(&ioat_ch->sw_batch, op, link);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
ioat_batch_prep_crc32c(void *cb_arg, struct spdk_io_channel *ch,
|
||||
struct spdk_accel_batch *batch, uint32_t *dst, void *src,
|
||||
uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb)
|
||||
{
|
||||
struct ioat_accel_op *op;
|
||||
struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch);
|
||||
|
||||
op = _prep_op(cb_arg, ioat_ch, batch, cb);
|
||||
if (op == NULL) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
/* Command specific. */
|
||||
op->dst = (void *)dst;
|
||||
op->src = src;
|
||||
op->seed = seed;
|
||||
op->nbytes = nbytes;
|
||||
op->op_code = IOAT_ACCEL_OPCODE_CRC32C;
|
||||
TAILQ_INSERT_TAIL(&ioat_ch->sw_batch, op, link);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
ioat_batch_submit(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
|
||||
spdk_accel_completion_cb cb)
|
||||
{
|
||||
struct ioat_accel_op *op;
|
||||
struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch);
|
||||
struct spdk_accel_task *accel_req;
|
||||
int batch_status = 0, cmd_status = 0;
|
||||
|
||||
if ((struct spdk_accel_batch *)&ioat_ch->hw_batch != batch) {
|
||||
SPDK_ERRLOG("Invalid batch\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
/* TODO submit the batched HW items first. */
|
||||
|
||||
/* Complete the batched software items. */
|
||||
while ((op = TAILQ_FIRST(&ioat_ch->sw_batch))) {
|
||||
TAILQ_REMOVE(&ioat_ch->sw_batch, op, link);
|
||||
accel_req = (struct spdk_accel_task *)((uintptr_t)op->cb_arg -
|
||||
offsetof(struct spdk_accel_task, offload_ctx));
|
||||
|
||||
switch (op->op_code) {
|
||||
case IOAT_ACCEL_OPCODE_DUALCAST:
|
||||
memcpy(op->dst, op->src, op->nbytes);
|
||||
memcpy(op->dst2, op->src, op->nbytes);
|
||||
break;
|
||||
case IOAT_ACCEL_OPCODE_COMPARE:
|
||||
cmd_status = memcmp(op->src, op->src2, op->nbytes);
|
||||
break;
|
||||
case IOAT_ACCEL_OPCODE_CRC32C:
|
||||
*(uint32_t *)op->dst = spdk_crc32c_update(op->src, op->nbytes, ~op->seed);
|
||||
break;
|
||||
default:
|
||||
assert(false);
|
||||
break;
|
||||
}
|
||||
|
||||
batch_status |= cmd_status;
|
||||
op->cb_fn(accel_req, cmd_status);
|
||||
TAILQ_INSERT_TAIL(&ioat_ch->op_pool, op, link);
|
||||
}
|
||||
|
||||
/* Now complete the batch request itself. */
|
||||
accel_req = (struct spdk_accel_task *)((uintptr_t)cb_arg -
|
||||
offsetof(struct spdk_accel_task, offload_ctx));
|
||||
cb(accel_req, batch_status);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static struct spdk_accel_engine ioat_accel_engine = {
|
||||
.get_capabilities = ioat_get_capabilities,
|
||||
.copy = ioat_submit_copy,
|
||||
.fill = ioat_submit_fill,
|
||||
.batch_get_max = ioat_batch_get_max,
|
||||
.batch_create = ioat_batch_create,
|
||||
.batch_prep_copy = ioat_batch_prep_copy,
|
||||
.batch_prep_dualcast = ioat_batch_prep_dualcast,
|
||||
.batch_prep_compare = ioat_batch_prep_compare,
|
||||
.batch_prep_fill = ioat_batch_prep_fill,
|
||||
.batch_prep_crc32c = ioat_batch_prep_crc32c,
|
||||
.batch_submit = ioat_batch_submit,
|
||||
.get_io_channel = ioat_get_io_channel,
|
||||
};
|
||||
|
||||
@ -212,12 +448,32 @@ ioat_create_cb(void *io_device, void *ctx_buf)
|
||||
{
|
||||
struct ioat_io_channel *ch = ctx_buf;
|
||||
struct ioat_device *ioat_dev;
|
||||
struct ioat_accel_op *op;
|
||||
int i;
|
||||
|
||||
ioat_dev = ioat_allocate_device();
|
||||
if (ioat_dev == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
TAILQ_INIT(&ch->sw_batch);
|
||||
ch->hw_batch = false;
|
||||
TAILQ_INIT(&ch->op_pool);
|
||||
|
||||
g_batch_size = spdk_ioat_get_max_descriptors(ioat_dev->ioat);
|
||||
for (i = 0 ; i < g_batch_size ; i++) {
|
||||
op = calloc(1, sizeof(struct ioat_accel_op));
|
||||
if (op == NULL) {
|
||||
SPDK_ERRLOG("Failed to allocate operation for batch.\n");
|
||||
while ((op = TAILQ_FIRST(&ch->op_pool))) {
|
||||
TAILQ_REMOVE(&ch->op_pool, op, link);
|
||||
free(op);
|
||||
}
|
||||
return -ENOMEM;
|
||||
}
|
||||
TAILQ_INSERT_TAIL(&ch->op_pool, op, link);
|
||||
}
|
||||
|
||||
ch->ioat_dev = ioat_dev;
|
||||
ch->ioat_ch = ioat_dev->ioat;
|
||||
ch->poller = SPDK_POLLER_REGISTER(ioat_poll, ch->ioat_ch, 0);
|
||||
@ -228,6 +484,12 @@ static void
|
||||
ioat_destroy_cb(void *io_device, void *ctx_buf)
|
||||
{
|
||||
struct ioat_io_channel *ch = ctx_buf;
|
||||
struct ioat_accel_op *op;
|
||||
|
||||
while ((op = TAILQ_FIRST(&ch->op_pool))) {
|
||||
TAILQ_REMOVE(&ch->op_pool, op, link);
|
||||
free(op);
|
||||
}
|
||||
|
||||
ioat_free_device(ch->ioat_dev);
|
||||
spdk_poller_unregister(&ch->poller);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user