d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

raw/ioat: add local API to perform copies

Browse Source 
Add local APIs to trigger data copies, and retrieve handle values once
those copies are completed. Included are unit tests to validate the data
is copies correctly.

Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
Acked-by: Jiayu Hu <jiayu.hu@intel.com>
Tested-by: Harry van Haaren <harry.van.haaren@intel.com>
This commit is contained in:
Bruce Richardson 2019-07-02 15:12:30 +01:00 committed by Thomas Monjalon
parent 39e4701f0b
commit 0a92e63fc4
5 changed files with 422 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
doc/guides/rawdevs/ioat_rawdev.rst
View File

@ -150,6 +150,106 @@ The following code shows how the device is configured in
Once configured, the device can then be made ready for use by calling the
``rte_rawdev_start()`` API.
Performing Data Copies
~~~~~~~~~~~~~~~~~~~~~~~
To perform data copies using IOAT rawdev devices, the functions
``rte_ioat_enqueue_copy()`` and ``rte_ioat_do_copies()`` should be used.
Once copies have been completed, the completion will be reported back when
the application calls ``rte_ioat_completed_copies()``.
The ``rte_ioat_enqueue_copy()`` function enqueues a single copy to the
device ring for copying at a later point. The parameters to that function
include the physical addresses of both the source and destination buffers,
as well as two "handles" to be returned to the user when the copy is
completed. These handles can be arbitrary values, but two are provided so
that the library can track handles for both source and destination on
behalf of the user, e.g. virtual addresses for the buffers, or mbuf
pointers if packet data is being copied.
While the ``rte_ioat_enqueue_copy()`` function enqueues a copy operation on
the device ring, the copy will not actually be performed until after the
application calls the ``rte_ioat_do_copies()`` function. This function
informs the device hardware of the elements enqueued on the ring, and the
device will begin to process them. It is expected that, for efficiency
reasons, a burst of operations will be enqueued to the device via multiple
enqueue calls between calls to the ``rte_ioat_do_copies()`` function.
The following code from ``test_ioat_rawdev.c`` demonstrates how to enqueue
a burst of copies to the device and start the hardware processing of them:
.. code-block:: C
struct rte_mbuf *srcs[32], *dsts[32];
unsigned int j;
for (i = 0; i < RTE_DIM(srcs); i++) {
char *src_data;
srcs[i] = rte_pktmbuf_alloc(pool);
dsts[i] = rte_pktmbuf_alloc(pool);
srcs[i]->data_len = srcs[i]->pkt_len = length;
dsts[i]->data_len = dsts[i]->pkt_len = length;
src_data = rte_pktmbuf_mtod(srcs[i], char *);
for (j = 0; j < length; j++)
src_data[j] = rand() & 0xFF;
if (rte_ioat_enqueue_copy(dev_id,
srcs[i]->buf_iova + srcs[i]->data_off,
dsts[i]->buf_iova + dsts[i]->data_off,
length,
(uintptr_t)srcs[i],
(uintptr_t)dsts[i],
0 /* nofence */) != 1) {
printf("Error with rte_ioat_enqueue_copy for buffer %u\n",
i);
return -1;
}
}
rte_ioat_do_copies(dev_id);
To retrieve information about completed copies, the API
``rte_ioat_completed_copies()`` should be used. This API will return to the
application a set of completion handles passed in when the relevant copies
were enqueued.
The following code from ``test_ioat_rawdev.c`` shows the test code
retrieving information about the completed copies and validating the data
is correct before freeing the data buffers using the returned handles:
.. code-block:: C
if (rte_ioat_completed_copies(dev_id, 64, (void *)completed_src,
(void *)completed_dst) != RTE_DIM(srcs)) {
printf("Error with rte_ioat_completed_copies\n");
return -1;
}
for (i = 0; i < RTE_DIM(srcs); i++) {
char *src_data, *dst_data;
if (completed_src[i] != srcs[i]) {
printf("Error with source pointer %u\n", i);
return -1;
}
if (completed_dst[i] != dsts[i]) {
printf("Error with dest pointer %u\n", i);
return -1;
}
src_data = rte_pktmbuf_mtod(srcs[i], char *);
dst_data = rte_pktmbuf_mtod(dsts[i], char *);
for (j = 0; j < length; j++)
if (src_data[j] != dst_data[j]) {
printf("Error with copy of packet %u, byte %u\n",
i, j);
return -1;
}
rte_pktmbuf_free(srcs[i]);
rte_pktmbuf_free(dsts[i]);
}
Querying Device Statistics
~~~~~~~~~~~~~~~~~~~~~~~~~~~

1
drivers/raw/ioat/Makefile
View File

@ -12,6 +12,7 @@ CFLAGS += $(WERROR_FLAGS)
LDLIBS += -lrte_eal -lrte_rawdev
LDLIBS += -lrte_pci -lrte_bus_pci
LDLIBS += -lrte_mbuf -lrte_mempool
# library version
LIBABIVER := 1

155
drivers/raw/ioat/ioat_rawdev_test.c
View File

@ -2,12 +2,139 @@
* Copyright(c) 2019 Intel Corporation
*/
#include <unistd.h>
#include <inttypes.h>
#include <rte_mbuf.h>
#include "rte_rawdev.h"
#include "rte_ioat_rawdev.h"
int ioat_rawdev_test(uint16_t dev_id); /* pre-define to keep compiler happy */
static struct rte_mempool *pool;
static int
test_enqueue_copies(int dev_id)
{
const unsigned int length = 1024;
unsigned int i;
do {
struct rte_mbuf *src, *dst;
char *src_data, *dst_data;
struct rte_mbuf *completed[2] = {0};
/* test doing a single copy */
src = rte_pktmbuf_alloc(pool);
dst = rte_pktmbuf_alloc(pool);
src->data_len = src->pkt_len = length;
dst->data_len = dst->pkt_len = length;
src_data = rte_pktmbuf_mtod(src, char *);
dst_data = rte_pktmbuf_mtod(dst, char *);
for (i = 0; i < length; i++)
src_data[i] = rand() & 0xFF;
if (rte_ioat_enqueue_copy(dev_id,
src->buf_iova + src->data_off,
dst->buf_iova + dst->data_off,
length,
(uintptr_t)src,
(uintptr_t)dst,
0 /* no fence */) != 1) {
printf("Error with rte_ioat_enqueue_copy\n");
return -1;
}
rte_ioat_do_copies(dev_id);
usleep(10);
if (rte_ioat_completed_copies(dev_id, 1, (void *)&completed[0],
(void *)&completed[1]) != 1) {
printf("Error with rte_ioat_completed_copies\n");
return -1;
}
if (completed[0] != src || completed[1] != dst) {
printf("Error with completions: got (%p, %p), not (%p,%p)\n",
completed[0], completed[1], src, dst);
return -1;
}
for (i = 0; i < length; i++)
if (dst_data[i] != src_data[i]) {
printf("Data mismatch at char %u\n", i);
return -1;
}
rte_pktmbuf_free(src);
rte_pktmbuf_free(dst);
} while (0);
/* test doing multiple copies */
do {
struct rte_mbuf *srcs[32], *dsts[32];
struct rte_mbuf *completed_src[64];
struct rte_mbuf *completed_dst[64];
unsigned int j;
for (i = 0; i < RTE_DIM(srcs); i++) {
char *src_data;
srcs[i] = rte_pktmbuf_alloc(pool);
dsts[i] = rte_pktmbuf_alloc(pool);
srcs[i]->data_len = srcs[i]->pkt_len = length;
dsts[i]->data_len = dsts[i]->pkt_len = length;
src_data = rte_pktmbuf_mtod(srcs[i], char *);
for (j = 0; j < length; j++)
src_data[j] = rand() & 0xFF;
if (rte_ioat_enqueue_copy(dev_id,
srcs[i]->buf_iova + srcs[i]->data_off,
dsts[i]->buf_iova + dsts[i]->data_off,
length,
(uintptr_t)srcs[i],
(uintptr_t)dsts[i],
0 /* nofence */) != 1) {
printf("Error with rte_ioat_enqueue_copy for buffer %u\n",
i);
return -1;
}
}
rte_ioat_do_copies(dev_id);
usleep(100);
if (rte_ioat_completed_copies(dev_id, 64, (void *)completed_src,
(void *)completed_dst) != RTE_DIM(srcs)) {
printf("Error with rte_ioat_completed_copies\n");
return -1;
}
for (i = 0; i < RTE_DIM(srcs); i++) {
char *src_data, *dst_data;
if (completed_src[i] != srcs[i]) {
printf("Error with source pointer %u\n", i);
return -1;
}
if (completed_dst[i] != dsts[i]) {
printf("Error with dest pointer %u\n", i);
return -1;
}
src_data = rte_pktmbuf_mtod(srcs[i], char *);
dst_data = rte_pktmbuf_mtod(dsts[i], char *);
for (j = 0; j < length; j++)
if (src_data[j] != dst_data[j]) {
printf("Error with copy of packet %u, byte %u\n",
i, j);
return -1;
}
rte_pktmbuf_free(srcs[i]);
rte_pktmbuf_free(dsts[i]);
}
} while (0);
return 0;
}
int
ioat_rawdev_test(uint16_t dev_id)
{
@ -44,6 +171,17 @@ ioat_rawdev_test(uint16_t dev_id)
return -1;
}
pool = rte_pktmbuf_pool_create("TEST_IOAT_POOL",
256, /* n == num elements */
32, /* cache size */
0, /* priv size */
2048, /* data room size */
info.socket_id);
if (pool == NULL) {
printf("Error with mempool creation\n");
return -1;
}
/* allocate memory for xstats names and values */
nb_xstats = rte_rawdev_xstats_names_get(dev_id, NULL, 0);
@ -68,17 +206,28 @@ ioat_rawdev_test(uint16_t dev_id)
goto err;
}
rte_rawdev_xstats_get(dev_id, ids, stats, nb_xstats);
for (i = 0; i < nb_xstats; i++)
printf("%s: %"PRIu64" ", snames[i].name, stats[i]);
/* run the test cases */
for (i = 0; i < 100; i++) {
unsigned int j;
if (test_enqueue_copies(dev_id) != 0)
goto err;
rte_rawdev_xstats_get(dev_id, ids, stats, nb_xstats);
for (j = 0; j < nb_xstats; j++)
printf("%s: %"PRIu64" ", snames[j].name, stats[j]);
printf("\r");
}
printf("\n");
rte_mempool_free(pool);
free(snames);
free(stats);
free(ids);
return 0;
err:
rte_mempool_free(pool);
free(snames);
free(stats);
free(ids);

2
drivers/raw/ioat/meson.build
View File

@ -4,7 +4,7 @@
build = dpdk_conf.has('RTE_ARCH_X86')
sources = files('ioat_rawdev.c',
'ioat_rawdev_test.c')
deps += ['rawdev', 'bus_pci']
deps += ['rawdev', 'bus_pci', 'mbuf']
install_headers('rte_ioat_rawdev.h',
'rte_ioat_spec.h')

169
drivers/raw/ioat/rte_ioat_rawdev.h
View File

@ -15,8 +15,10 @@
*/
#include <x86intrin.h>
#include <rte_atomic.h>
#include <rte_memory.h>
#include <rte_memzone.h>
#include <rte_prefetch.h>
#include <rte_ioat_spec.h>
/** Name of the device driver */
@ -54,6 +56,10 @@ struct rte_ioat_rawdev {
struct rte_ioat_generic_hw_desc *desc_ring;
__m128i *hdls; /* completion handles for returning to user */
unsigned short next_read;
unsigned short next_write;
/* some statistics for tracking, if added/changed update xstats fns*/
uint64_t enqueue_failed __rte_cache_aligned;
uint64_t enqueued;
@ -64,4 +70,165 @@ struct rte_ioat_rawdev {
volatile uint64_t status __rte_cache_aligned;
};
#endif
/**
* Enqueue a copy operation onto the ioat device
*
* This queues up a copy operation to be performed by hardware, but does not
* trigger hardware to begin that operation.
*
* @param dev_id
* The rawdev device id of the ioat instance
* @param src
* The physical address of the source buffer
* @param dst
* The physical address of the destination buffer
* @param length
* The length of the data to be copied
* @param src_hdl
* An opaque handle for the source data, to be returned when this operation
* has been completed and the user polls for the completion details
* @param dst_hdl
* An opaque handle for the destination data, to be returned when this
* operation has been completed and the user polls for the completion details
* @param fence
* A flag parameter indicating that hardware should not begin to perform any
* subsequently enqueued copy operations until after this operation has
* completed
* @return
* Number of operations enqueued, either 0 or 1
*/
static inline int
rte_ioat_enqueue_copy(int dev_id, phys_addr_t src, phys_addr_t dst,
unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl,
int fence)
{
struct rte_ioat_rawdev *ioat = rte_rawdevs[dev_id].dev_private;
unsigned short read = ioat->next_read;
unsigned short write = ioat->next_write;
unsigned short mask = ioat->ring_size - 1;
unsigned short space = mask + read - write;
struct rte_ioat_generic_hw_desc *desc;
if (space == 0) {
ioat->enqueue_failed++;
return 0;
}
ioat->next_write = write + 1;
write &= mask;
desc = &ioat->desc_ring[write];
desc->size = length;
/* set descriptor write-back every 16th descriptor */
desc->u.control_raw = (uint32_t)((!!fence << 4) | (!(write & 0xF)) << 3);
desc->src_addr = src;
desc->dest_addr = dst;
ioat->hdls[write] = _mm_set_epi64((__m64)((uint64_t)dst_hdl),
(__m64)((uint64_t)src_hdl));
rte_prefetch0(&ioat->desc_ring[ioat->next_write & mask]);
ioat->enqueued++;
return 1;
}
/**
* Trigger hardware to begin performing enqueued copy operations
*
* This API is used to write the "doorbell" to the hardware to trigger it
* to begin the copy operations previously enqueued by rte_ioat_enqueue_copy()
*
* @param dev_id
* The rawdev device id of the ioat instance
*/
static inline void
rte_ioat_do_copies(int dev_id)
{
struct rte_ioat_rawdev *ioat = rte_rawdevs[dev_id].dev_private;
ioat->desc_ring[(ioat->next_write - 1) & (ioat->ring_size - 1)].u
.control.completion_update = 1;
rte_compiler_barrier();
ioat->regs->dmacount = ioat->next_write;
ioat->started = ioat->enqueued;
}
/**
* @internal
* Returns the index of the last completed operation.
*/
static inline int
rte_ioat_get_last_completed(struct rte_ioat_rawdev *ioat, int *error)
{
uint64_t status = ioat->status;
/* lower 3 bits indicate "transfer status" : active, idle, halted.
* We can ignore bit 0.
*/
*error = status & (RTE_IOAT_CHANSTS_SUSPENDED | RTE_IOAT_CHANSTS_ARMED);
return (status - ioat->ring_addr) >> 6;
}
/**
* Returns details of copy operations that have been completed
*
* Returns to the caller the user-provided "handles" for the copy operations
* which have been completed by the hardware, and not already returned by
* a previous call to this API.
*
* @param dev_id
* The rawdev device id of the ioat instance
* @param max_copies
* The number of entries which can fit in the src_hdls and dst_hdls
* arrays, i.e. max number of completed operations to report
* @param src_hdls
* Array to hold the source handle parameters of the completed copies
* @param dst_hdls
* Array to hold the destination handle parameters of the completed copies
* @return
* -1 on error, with rte_errno set appropriately.
* Otherwise number of completed operations i.e. number of entries written
* to the src_hdls and dst_hdls array parameters.
*/
static inline int
rte_ioat_completed_copies(int dev_id, uint8_t max_copies,
uintptr_t *src_hdls, uintptr_t *dst_hdls)
{
struct rte_ioat_rawdev *ioat = rte_rawdevs[dev_id].dev_private;
unsigned short mask = (ioat->ring_size - 1);
unsigned short read = ioat->next_read;
unsigned short end_read, count;
int error;
int i = 0;
end_read = (rte_ioat_get_last_completed(ioat, &error) + 1) & mask;
count = (end_read - (read & mask)) & mask;
if (error) {
rte_errno = EIO;
return -1;
}
if (count > max_copies)
count = max_copies;
for (; i < count - 1; i += 2, read += 2) {
__m128i hdls0 = _mm_load_si128(&ioat->hdls[read & mask]);
__m128i hdls1 = _mm_load_si128(&ioat->hdls[(read + 1) & mask]);
_mm_storeu_si128((void *)&src_hdls[i],
_mm_unpacklo_epi64(hdls0, hdls1));
_mm_storeu_si128((void *)&dst_hdls[i],
_mm_unpackhi_epi64(hdls0, hdls1));
}
for (; i < count; i++, read++) {
uintptr_t *hdls = (void *)&ioat->hdls[read & mask];
src_hdls[i] = hdls[0];
dst_hdls[i] = hdls[1];
}
ioat->next_read = read;
ioat->completed += count;
return count;
}
#endif /* _RTE_IOAT_RAWDEV_H_ */