idxd: Add support for vectored copy operations

Change-Id: Icb650129488b3cea76cf9082c02667f5b13b5ab4
Signed-off-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/10286
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>

CHANGELOG.md

@@ -7,6 +7,10 @@
 The batching capability was removed. Batching is now considered an implementation
 detail of the low level drivers.
 
+### idxd
+
+Many APIs are now vectored rather than scalar, meaning they take iovecs instead of individual pointers.
+
 ### nvme
 
 API `spdk_nvme_trtype_is_fabrics` was added to return existing transport type

examples/idxd/perf/perf.c

@@ -609,6 +609,8 @@ _submit_ops(struct idxd_chan_entry *t, struct idxd_task *task)
 {
 	int random_num;
 	int rc = 0;
+	struct iovec siov = {};
+	struct iovec diov = {};
 
 	assert(t);
 	t->current_queue_depth++;
@@ -621,8 +623,12 @@ _submit_ops(struct idxd_chan_entry *t, struct idxd_task *task)
 
 		switch (g_workload_selection) {
 		case IDXD_COPY:
-			rc = spdk_idxd_submit_copy(t->ch, task->dst, task->src,
+			siov.iov_base = task->src;
-						   g_xfer_size_bytes, idxd_done, task);
+			siov.iov_len = g_xfer_size_bytes;
+			diov.iov_base = task->dst;
+			diov.iov_len = g_xfer_size_bytes;
+			rc = spdk_idxd_submit_copy(t->ch, &diov, 1, &siov, 1,
+						   idxd_done, task);
 			break;
 		case IDXD_FILL:
 			/* For fill use the first byte of the task->dst buffer */

include/spdk/idxd.h

@@ -217,9 +217,10 @@ int spdk_idxd_batch_prep_dualcast(struct spdk_idxd_io_channel *chan, struct idxd
  * by writing to the proper device portal.
  *
  * \param chan IDXD channel to submit request.
- * \param dst Destination virtual address.
+ * \param diov Destination iovec
- * \param src Source virtual address.
+ * \param diovcnt Number of elements in diov
- * \param nbytes Number of bytes to copy.
+ * \param siov Source iovec
+ * \param siovcnt Number of elements in siov
  * \param cb_fn Callback function which will be called when the request is complete.
  * \param cb_arg Opaque value which will be passed back as the arg parameter in
  * the completion callback.
@@ -227,7 +228,8 @@ int spdk_idxd_batch_prep_dualcast(struct spdk_idxd_io_channel *chan, struct idxd
  * \return 0 on success, negative errno on failure.
  */
 int spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan,
-			  void *dst, const void *src, uint64_t nbytes,
+			  struct iovec *diov, uint32_t diovcnt,
+			  struct iovec *siov, uint32_t siovcnt,
 			  spdk_idxd_req_cb cb_fn, void *cb_arg);
 
 /**

lib/idxd/Makefile

@@ -34,7 +34,7 @@
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-SO_VER := 6
+SO_VER := 7
 SO_MINOR := 0
 
 C_SRCS = idxd.c idxd_user.c

lib/idxd/idxd.c

@@ -456,9 +456,8 @@ _idxd_batch_prep_nop(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch
 	return 0;
 }
 
-
+static inline int
-int
+_idxd_submit_copy_single(struct spdk_idxd_io_channel *chan, void *dst, const void *src,
-spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan, void *dst, const void *src,
 			 uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg)
 {
 	struct idxd_hw_desc *desc;
@@ -502,6 +501,72 @@ error:
 	return rc;
 }
 
+int
+spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan,
+		      struct iovec *diov, uint32_t diovcnt,
+		      struct iovec *siov, uint32_t siovcnt,
+		      spdk_idxd_req_cb cb_fn, void *cb_arg)
+{
+	struct idxd_hw_desc *desc;
+	struct idxd_ops *op;
+	void *src, *dst;
+	uint64_t src_addr, dst_addr;
+	int rc;
+	uint64_t len;
+	struct idxd_batch *batch;
+	struct spdk_ioviter iter;
+
+	assert(chan != NULL);
+	assert(diov != NULL);
+	assert(siov != NULL);
+
+	if (diovcnt == 1 && siovcnt == 1) {
+		/* Simple case - copying one buffer to another */
+		if (diov[0].iov_len < siov[0].iov_len) {
+			return -EINVAL;
+		}
+
+		return _idxd_submit_copy_single(chan, diov[0].iov_base,
+						siov[0].iov_base, siov[0].iov_len,
+						cb_fn, cb_arg);
+	}
+
+	batch = spdk_idxd_batch_create(chan);
+	if (!batch) {
+		return -EBUSY;
+	}
+
+	for (len = spdk_ioviter_first(&iter, siov, siovcnt, diov, diovcnt, &src, &dst);
+	     len > 0;
+	     len = spdk_ioviter_next(&iter, &src, &dst)) {
+		rc = _idxd_prep_batch_cmd(chan, NULL, NULL, batch, &desc, &op);
+		if (rc) {
+			goto err;
+		}
+
+		rc = _vtophys(src, &src_addr, len);
+		if (rc) {
+			goto err;
+		}
+
+		rc = _vtophys(dst, &dst_addr, len);
+		if (rc) {
+			goto err;
+		}
+
+		desc->opcode = IDXD_OPCODE_MEMMOVE;
+		desc->src_addr = src_addr;
+		desc->dst_addr = dst_addr;
+		desc->xfer_size = len;
+	}
+
+	return spdk_idxd_batch_submit(chan, batch, cb_fn, cb_arg);
+
+err:
+	spdk_idxd_batch_cancel(chan, batch);
+	return rc;
+}
+
 /* Dual-cast copies the same source to two separate destination buffers. */
 int
 spdk_idxd_submit_dualcast(struct spdk_idxd_io_channel *chan, void *dst1, void *dst2,

mk/spdk.lib_deps.mk

@@ -47,7 +47,7 @@ DEPDIRS-rte_vhost :=
 DEPDIRS-env_dpdk := log util
 
 DEPDIRS-ioat := log
-DEPDIRS-idxd := log
+DEPDIRS-idxd := log util
 DEPDIRS-sock := log $(JSON_LIBS)
 DEPDIRS-util := log
 DEPDIRS-vmd := log

module/accel/idxd/accel_engine_idxd.c

@@ -152,6 +152,8 @@ _process_single_task(struct spdk_io_channel *ch, struct spdk_accel_task *task)
 	int rc = 0;
 	uint8_t fill_pattern = (uint8_t)task->fill_pattern;
 	void *src;
+	struct iovec siov = {};
+	struct iovec diov = {};
 
 	if (chan->num_outstanding == chan->max_outstanding) {
 		chan->state = IDXD_CHANNEL_PAUSED;
@@ -160,7 +162,11 @@ _process_single_task(struct spdk_io_channel *ch, struct spdk_accel_task *task)
 
 	switch (task->op_code) {
 	case ACCEL_OPCODE_MEMMOVE:
-		rc = spdk_idxd_submit_copy(chan->chan, task->dst, task->src, task->nbytes, idxd_done, task);
+		siov.iov_base = task->src;
+		siov.iov_len = task->nbytes;
+		diov.iov_base = task->dst;
+		diov.iov_len = task->nbytes;
+		rc = spdk_idxd_submit_copy(chan->chan, &diov, 1, &siov, 1, idxd_done, task);
 		break;
 	case ACCEL_OPCODE_DUALCAST:
 		rc = spdk_idxd_submit_dualcast(chan->chan, task->dst, task->dst2, task->src, task->nbytes,