d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

accel/idxd: add and respect flag to support writes to PMEM

Browse Source 
Plumbing for flags was added in prior pathces.  This patch
introduces and respects the relevant flags for use with PMEM
aka durable memory through the accel_fw, IDXD, IOAT and SW
modules.

Signed-off-by: paul luse <paul.e.luse@intel.com>
Change-Id: I792f31459e061d220965feced60e0c236d819a68
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9455
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
This commit is contained in:
paul luse 2021-09-08 17:29:35 -04:00 committed by Jim Harris
parent 12c40f05e2
commit 8951c15759
8 changed files with 160 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@ -11,6 +11,9 @@ Removed deprecated spdk_bdev_module_finish_done(). Use spdk_bdev_module_fini_don
A new parameter `flags` was added to all low level submission and preparation
APIs to enable the caller to pass operation flags per the DSA specification.
A new flag 'SPDK_IDXD_FLAG_PERSISTENT' was added to let DSA know that
the destination is persistent.
### accel_fw
A new parameter `flags` was added to accel API.
@ -21,6 +24,8 @@ The APIs include:
`spdk_accel_submit_copy_crc32c`
`spdk_accel_submit_copy_crc32cv`
A new flag `ACCEL_FLAG_PERSISTENT` was added to indicate the target memory is PMEM.
### bdev_nvme
Added `bdev_nvme_add_error_injection` and `bdev_nvme_remove_error_injection` RPCs to add and

3
include/spdk/accel_engine.h
View File

@ -44,6 +44,9 @@
extern "C" {
#endif
/* Flags for accel operations */
#define ACCEL_FLAG_PERSISTENT (1 << 0)
enum accel_capability {
ACCEL_COPY = 1 << 0,
ACCEL_FILL = 1 << 1,

5
include/spdk/idxd.h
View File

@ -63,6 +63,11 @@ extern "C" {
*/
#define SPDK_IDXD_FLAG_NONTEMPORAL IDXD_FLAG_CACHE_CONTROL
/* The following flag is optional and specifies that the destination is persistent memory. The
* low level library will not set this flag.
*/
#define SPDK_IDXD_FLAG_PERSISTENT IDXD_FLAG_DEST_STEERING_TAG
/**
* Opaque handle for a single IDXD channel.
*/

2
include/spdk/idxd_spec.h
View File

@ -84,7 +84,7 @@ extern "C" {
#define IDXD_FLAG_COMPLETION_ADDR_VALID (1 << 2)
#define IDXD_FLAG_REQUEST_COMPLETION (1 << 3)
#define IDXD_FLAG_CACHE_CONTROL (1 << 8)
#define IDXD_FLAG_DEST_STEERING_TAG (1 << 15)
#define IDXD_FLAG_CRC_READ_CRC_SEED (1 << 16)
/*

141
lib/accel/accel_engine.c
View File

@ -43,6 +43,10 @@
#include "spdk/crc32.h"
#include "spdk/util.h"
#ifdef SPDK_CONFIG_PMDK
#include "libpmem.h"
#endif
/* Accelerator Engine Framework: The following provides a top level
* generic API for the accelerator functions defined here. Modules,
* such as the one in /module/accel/ioat, supply the implementation
@ -66,12 +70,12 @@ static void *g_fini_cb_arg = NULL;
static TAILQ_HEAD(, spdk_accel_module_if) spdk_accel_module_list =
TAILQ_HEAD_INITIALIZER(spdk_accel_module_list);
static void _sw_accel_dualcast(void *dst1, void *dst2, void *src, uint64_t nbytes);
static void _sw_accel_copy(void *dst, void *src, uint64_t nbytes);
static void _sw_accel_copyv(void *dst, struct iovec *iov, uint32_t iovcnt);
static int _sw_accel_compare(void *src1, void *src2, uint64_t nbytes);
static void _sw_accel_fill(void *dst, uint8_t fill, uint64_t nbytes);
static void _sw_accel_crc32c(uint32_t *dst, void *src, uint32_t seed, uint64_t nbytes);
static void _sw_accel_dualcast(void *dst1, void *dst2, void *src, size_t nbytes, int flags);
static void _sw_accel_copy(void *dst, void *src, size_t nbytes, int flags);
static void _sw_accel_copyv(void *dst, struct iovec *iov, uint32_t iovcnt, int flags);
static int _sw_accel_compare(void *src1, void *src2, size_t nbytes);
static void _sw_accel_fill(void *dst, uint8_t fill, size_t nbytes, int flags);
static void _sw_accel_crc32c(uint32_t *dst, void *src, uint32_t seed, size_t nbytes);
static void _sw_accel_crc32cv(uint32_t *dst, struct iovec *iov, uint32_t iovcnt, uint32_t seed);
/* Registration of hw modules (currently supports only 1 at a time) */
@ -165,6 +169,20 @@ _add_to_comp_list(struct accel_io_channel *accel_ch, struct spdk_accel_task *acc
TAILQ_INSERT_TAIL(&sw_ch->tasks_to_complete, accel_task, link);
}
/* Used when the SW engine is selected and the durable flag is set. */
inline static int
_check_flags(int flags)
{
if (flags & ACCEL_FLAG_PERSISTENT) {
#ifndef SPDK_CONFIG_PMDK
/* PMDK is required to use this flag. */
SPDK_ERRLOG("ACCEL_FLAG_PERSISTENT set but PMDK not configured. Configure PMDK or do not use this flag.\n");
return -EINVAL;
#endif
}
return 0;
}
/* Accel framework public API for copy function */
int
spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, uint64_t nbytes,
@ -172,6 +190,7 @@ spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, uint64_
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
int rc;
accel_task = _get_task(accel_ch, cb_fn, cb_arg);
if (accel_task == NULL) {
@ -187,7 +206,11 @@ spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, uint64_
if (_is_supported(accel_ch->engine, ACCEL_COPY)) {
return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);
} else {
_sw_accel_copy(dst, src, nbytes);
rc = _check_flags(flags);
if (rc) {
return rc;
}
_sw_accel_copy(dst, src, (size_t)nbytes, flags);
_add_to_comp_list(accel_ch, accel_task, 0);
return 0;
}
@ -200,6 +223,7 @@ spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, void *dst2, v
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
int rc;
if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) {
SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n");
@ -221,7 +245,11 @@ spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, void *dst2, v
if (_is_supported(accel_ch->engine, ACCEL_DUALCAST)) {
return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);
} else {
_sw_accel_dualcast(dst1, dst2, src, nbytes);
rc = _check_flags(flags);
if (rc) {
return rc;
}
_sw_accel_dualcast(dst1, dst2, src, (size_t)nbytes, flags);
_add_to_comp_list(accel_ch, accel_task, 0);
return 0;
}
@ -249,7 +277,7 @@ spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1, void *src2, ui
if (_is_supported(accel_ch->engine, ACCEL_COMPARE)) {
return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);
} else {
rc = _sw_accel_compare(src1, src2, nbytes);
rc = _sw_accel_compare(src1, src2, (size_t)nbytes);
_add_to_comp_list(accel_ch, accel_task, rc);
return 0;
}
@ -262,6 +290,7 @@ spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, uint8_t fill, uint
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
int rc;
accel_task = _get_task(accel_ch, cb_fn, cb_arg);
if (accel_task == NULL) {
@ -277,7 +306,11 @@ spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, uint8_t fill, uint
if (_is_supported(accel_ch->engine, ACCEL_FILL)) {
return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);
} else {
_sw_accel_fill(dst, fill, nbytes);
rc = _check_flags(flags);
if (rc) {
return rc;
}
_sw_accel_fill(dst, fill, (size_t)nbytes, flags);
_add_to_comp_list(accel_ch, accel_task, 0);
return 0;
}
@ -306,7 +339,7 @@ spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst, void *sr
if (_is_supported(accel_ch->engine, ACCEL_CRC32C)) {
return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);
} else {
_sw_accel_crc32c(crc_dst, src, seed, nbytes);
_sw_accel_crc32c(crc_dst, src, seed, (size_t)nbytes);
_add_to_comp_list(accel_ch, accel_task, 0);
return 0;
}
@ -361,6 +394,7 @@ spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst, void *src,
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
int rc;
accel_task = _get_task(accel_ch, cb_fn, cb_arg);
if (accel_task == NULL) {
@ -379,8 +413,12 @@ spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst, void *src,
if (_is_supported(accel_ch->engine, ACCEL_COPY_CRC32C)) {
return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);
} else {
_sw_accel_copy(dst, src, nbytes);
_sw_accel_crc32c(crc_dst, src, seed, nbytes);
rc = _check_flags(flags);
if (rc) {
return rc;
}
_sw_accel_copy(dst, src, (size_t)nbytes, flags);
_sw_accel_crc32c(crc_dst, src, seed, (size_t)nbytes);
_add_to_comp_list(accel_ch, accel_task, 0);
return 0;
}
@ -394,6 +432,7 @@ spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, struct iov
{
struct accel_io_channel *accel_ch;
struct spdk_accel_task *accel_task;
int rc;
if (src_iovs == NULL) {
SPDK_ERRLOG("iov should not be NULL");
@ -424,7 +463,11 @@ spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, struct iov
if (_is_supported(accel_ch->engine, ACCEL_COPY_CRC32C)) {
return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);
} else {
_sw_accel_copyv(dst, src_iovs, iov_cnt);
rc = _check_flags(flags);
if (rc) {
return rc;
}
_sw_accel_copyv(dst, src_iovs, iov_cnt, flags);
_sw_accel_crc32cv(crc_dst, src_iovs, iov_cnt, seed);
_add_to_comp_list(accel_ch, accel_task, 0);
return 0;
@ -596,41 +639,89 @@ sw_accel_get_capabilities(void)
return 0;
}
static void
_sw_accel_dualcast(void *dst1, void *dst2, void *src, uint64_t nbytes)
static inline void
_pmem_memcpy(void *dst, const void *src, size_t len)
{
memcpy(dst1, src, (size_t)nbytes);
memcpy(dst2, src, (size_t)nbytes);
#ifdef SPDK_CONFIG_PMDK
int is_pmem = pmem_is_pmem(dst, len);
if (is_pmem) {
pmem_memcpy_persist(dst, src, len);
} else {
memcpy(dst, src, len);
pmem_msync(dst, len);
}
#else
SPDK_ERRLOG("Function not defined without SPDK_CONFIG_PMDK enabled.\n");
assert(0);
#endif
}
static void
_sw_accel_copy(void *dst, void *src, uint64_t nbytes)
_sw_accel_dualcast(void *dst1, void *dst2, void *src, size_t nbytes, int flags)
{
memcpy(dst, src, (size_t)nbytes);
if (flags & ACCEL_FLAG_PERSISTENT) {
_pmem_memcpy(dst1, src, nbytes);
_pmem_memcpy(dst2, src, nbytes);
} else {
memcpy(dst1, src, nbytes);
memcpy(dst2, src, nbytes);
}
}
static void
_sw_accel_copyv(void *dst, struct iovec *iov, uint32_t iovcnt)
_sw_accel_copy(void *dst, void *src, size_t nbytes, int flags)
{
if (flags & ACCEL_FLAG_PERSISTENT) {
_pmem_memcpy(dst, src, nbytes);
} else {
memcpy(dst, src, nbytes);
}
}
static void
_sw_accel_copyv(void *dst, struct iovec *iov, uint32_t iovcnt, int flags)
{
uint32_t i;
for (i = 0; i < iovcnt; i++) {
assert(iov[i].iov_base != NULL);
memcpy(dst, iov[i].iov_base, iov[i].iov_len);
if (flags & ACCEL_FLAG_PERSISTENT) {
_pmem_memcpy(dst, iov[i].iov_base, (size_t)iov[i].iov_len);
} else {
memcpy(dst, iov[i].iov_base, (size_t)iov[i].iov_len);
}
dst += iov[i].iov_len;
}
}
static int
_sw_accel_compare(void *src1, void *src2, uint64_t nbytes)
_sw_accel_compare(void *src1, void *src2, size_t nbytes)
{
return memcmp(src1, src2, (size_t)nbytes);
return memcmp(src1, src2, nbytes);
}
static void
_sw_accel_fill(void *dst, uint8_t fill, uint64_t nbytes)
_sw_accel_fill(void *dst, uint8_t fill, size_t nbytes, int flags)
{
if (flags & ACCEL_FLAG_PERSISTENT) {
#ifdef SPDK_CONFIG_PMDK
int is_pmem = pmem_is_pmem(dst, nbytes);
if (is_pmem) {
pmem_memset_persist(dst, fill, nbytes);
} else {
memset(dst, fill, nbytes);
pmem_msync(dst, nbytes);
}
#else
SPDK_ERRLOG("Function not defined without SPDK_CONFIG_PMDK enabled.\n");
assert(0);
#endif
} else {
memset(dst, fill, nbytes);
}
}
static void

16
module/accel/idxd/accel_engine_idxd.c
View File

@ -158,9 +158,17 @@ _process_single_task(struct spdk_io_channel *ch, struct spdk_accel_task *task)
siov.iov_len = task->nbytes;
diov.iov_base = task->dst;
diov.iov_len = task->nbytes;
if (task->flags & ACCEL_FLAG_PERSISTENT) {
flags |= SPDK_IDXD_FLAG_PERSISTENT;
flags |= SPDK_IDXD_FLAG_NONTEMPORAL;
}
rc = spdk_idxd_submit_copy(chan->chan, &diov, 1, &siov, 1, flags, idxd_done, task);
break;
case ACCEL_OPCODE_DUALCAST:
if (task->flags & ACCEL_FLAG_PERSISTENT) {
flags |= SPDK_IDXD_FLAG_PERSISTENT;
flags |= SPDK_IDXD_FLAG_NONTEMPORAL;
}
rc = spdk_idxd_submit_dualcast(chan->chan, task->dst, task->dst2, task->src, task->nbytes,
flags, idxd_done, task);
break;
@ -175,6 +183,10 @@ _process_single_task(struct spdk_io_channel *ch, struct spdk_accel_task *task)
memset(&task->fill_pattern, fill_pattern, sizeof(uint64_t));
diov.iov_base = task->dst;
diov.iov_len = task->nbytes;
if (task->flags & ACCEL_FLAG_PERSISTENT) {
flags |= SPDK_IDXD_FLAG_PERSISTENT;
flags |= SPDK_IDXD_FLAG_NONTEMPORAL;
}
rc = spdk_idxd_submit_fill(chan->chan, &diov, 1, task->fill_pattern, flags, idxd_done,
task);
break;
@ -203,6 +215,10 @@ _process_single_task(struct spdk_io_channel *ch, struct spdk_accel_task *task)
}
diov.iov_base = task->dst;
diov.iov_len = task->nbytes;
if (task->flags & ACCEL_FLAG_PERSISTENT) {
flags |= SPDK_IDXD_FLAG_PERSISTENT;
flags |= SPDK_IDXD_FLAG_NONTEMPORAL;
}
rc = spdk_idxd_submit_copy_crc32c(chan->chan, &diov, 1, iov, iovcnt,
task->seed, task->crc_dst, flags,
idxd_done, task);

5
module/accel/ioat/accel_engine_ioat.c
View File

@ -139,6 +139,11 @@ ioat_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task
struct spdk_accel_task *tmp;
int rc = 0;
if (accel_task->flags == ACCEL_FLAG_PERSISTENT) {
SPDK_ERRLOG("IOAT does not support durable destinations.\n");
return -EINVAL;
}
do {
switch (accel_task->op_code) {
case ACCEL_OPCODE_MEMFILL:

7
test/unit/lib/accel/accel.c/accel_engine_ut.c
View File

@ -39,6 +39,13 @@
#include "accel/accel_engine.c"
#include "unit/lib/json_mock.c"
#ifdef SPDK_CONFIG_PMDK
DEFINE_STUB(pmem_msync, int, (const void *addr, size_t len), 0);
DEFINE_STUB(pmem_memcpy_persist, void *, (void *pmemdest, const void *src, size_t len), NULL);
DEFINE_STUB(pmem_is_pmem, int, (const void *addr, size_t len), 0);
DEFINE_STUB(pmem_memset_persist, void *, (void *pmemdest, int c, size_t len), NULL);
#endif
/* global vars and setup/cleanup functions used for all test functions */
struct spdk_accel_engine g_accel_engine = {};
struct spdk_io_channel *g_ch = NULL;