raw/ioat: add datapath data structures for idxd devices

Add in the relevant data structures for the data path for DSA devices. Also
include a device dump function to output the status of each device.

Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
Reviewed-by: Kevin Laatz <kevin.laatz@intel.com>
Acked-by: Radu Nicolau <radu.nicolau@intel.com>

drivers/raw/ioat/idxd_pci.c

@@ -54,6 +54,7 @@ idxd_is_wq_enabled(struct idxd_rawdev *idxd)
 static const struct rte_rawdev_ops idxd_pci_ops = {
 		.dev_close = idxd_rawdev_close,
 		.dev_selftest = idxd_rawdev_test,
+		.dump = idxd_dev_dump,
 };
 
 /* each portal uses 4 x 4k pages */

drivers/raw/ioat/idxd_vdev.c

@@ -33,6 +33,7 @@ struct idxd_vdev_args {
 static const struct rte_rawdev_ops idxd_vdev_ops = {
 		.dev_close = idxd_rawdev_close,
 		.dev_selftest = idxd_rawdev_test,
+		.dump = idxd_dev_dump,
 };
 
 static void *

drivers/raw/ioat/ioat_common.c

@@ -14,6 +14,36 @@ idxd_rawdev_close(struct rte_rawdev *dev __rte_unused)
 	return 0;
 }
 
+int
+idxd_dev_dump(struct rte_rawdev *dev, FILE *f)
+{
+	struct idxd_rawdev *idxd = dev->dev_private;
+	struct rte_idxd_rawdev *rte_idxd = &idxd->public;
+	int i;
+
+	fprintf(f, "Raw Device #%d\n", dev->dev_id);
+	fprintf(f, "Driver: %s\n\n", dev->driver_name);
+
+	fprintf(f, "Portal: %p\n", rte_idxd->portal);
+	fprintf(f, "Batch Ring size: %u\n", rte_idxd->batch_ring_sz);
+	fprintf(f, "Comp Handle Ring size: %u\n\n", rte_idxd->hdl_ring_sz);
+
+	fprintf(f, "Next batch: %u\n", rte_idxd->next_batch);
+	fprintf(f, "Next batch to be completed: %u\n", rte_idxd->next_completed);
+	for (i = 0; i < rte_idxd->batch_ring_sz; i++) {
+		struct rte_idxd_desc_batch *b = &rte_idxd->batch_ring[i];
+		fprintf(f, "Batch %u @%p: submitted=%u, op_count=%u, hdl_end=%u\n",
+				i, b, b->submitted, b->op_count, b->hdl_end);
+	}
+
+	fprintf(f, "\n");
+	fprintf(f, "Next free hdl: %u\n", rte_idxd->next_free_hdl);
+	fprintf(f, "Last completed hdl: %u\n", rte_idxd->last_completed_hdl);
+	fprintf(f, "Next returned hdl: %u\n", rte_idxd->next_ret_hdl);
+
+	return 0;
+}
+
 int
 idxd_rawdev_create(const char *name, struct rte_device *dev,
 		   const struct idxd_rawdev *base_idxd,
@@ -25,6 +55,10 @@ idxd_rawdev_create(const char *name, struct rte_device *dev,
 	char mz_name[RTE_MEMZONE_NAMESIZE];
 	int ret = 0;
 
+	RTE_BUILD_BUG_ON(sizeof(struct rte_idxd_hw_desc) != 64);
+	RTE_BUILD_BUG_ON(offsetof(struct rte_idxd_hw_desc, size) != 32);
+	RTE_BUILD_BUG_ON(sizeof(struct rte_idxd_completion) != 32);
+
 	if (!name) {
 		IOAT_PMD_ERR("Invalid name of the device!");
 		ret = -EINVAL;

drivers/raw/ioat/ioat_private.h

@@ -61,4 +61,6 @@ extern int idxd_rawdev_close(struct rte_rawdev *dev);
 
 extern int idxd_rawdev_test(uint16_t dev_id);
 
+extern int idxd_dev_dump(struct rte_rawdev *dev, FILE *f);
+
 #endif /* _IOAT_PRIVATE_H_ */

drivers/raw/ioat/ioat_rawdev_test.c

@@ -270,7 +270,8 @@ err:
 }
 
 int
-idxd_rawdev_test(uint16_t dev_id __rte_unused)
+idxd_rawdev_test(uint16_t dev_id)
 {
+	rte_rawdev_dump(dev_id, stdout);
 	return 0;
 }

drivers/raw/ioat/rte_ioat_rawdev_fns.h

@@ -90,6 +90,86 @@ struct rte_ioat_rawdev {
 #define RTE_IOAT_CHANSTS_HALTED			0x3
 #define RTE_IOAT_CHANSTS_ARMED			0x4
 
+/*
+ * Defines used in the data path for interacting with hardware.
+ */
+#define IDXD_CMD_OP_SHIFT 24
+enum rte_idxd_ops {
+	idxd_op_nop = 0,
+	idxd_op_batch,
+	idxd_op_drain,
+	idxd_op_memmove,
+	idxd_op_fill
+};
+
+#define IDXD_FLAG_FENCE                 (1 << 0)
+#define IDXD_FLAG_COMPLETION_ADDR_VALID (1 << 2)
+#define IDXD_FLAG_REQUEST_COMPLETION    (1 << 3)
+#define IDXD_FLAG_CACHE_CONTROL         (1 << 8)
+
+/**
+ * Hardware descriptor used by DSA hardware, for both bursts and
+ * for individual operations.
+ */
+struct rte_idxd_hw_desc {
+	uint32_t pasid;
+	uint32_t op_flags;
+	rte_iova_t completion;
+
+	RTE_STD_C11
+	union {
+		rte_iova_t src;      /* source address for copy ops etc. */
+		rte_iova_t desc_addr; /* descriptor pointer for batch */
+	};
+	rte_iova_t dst;
+
+	uint32_t size;    /* length of data for op, or batch size */
+
+	/* 28 bytes of padding here */
+} __rte_aligned(64);
+
+/**
+ * Completion record structure written back by DSA
+ */
+struct rte_idxd_completion {
+	uint8_t status;
+	uint8_t result;
+	/* 16-bits pad here */
+	uint32_t completed_size; /* data length, or descriptors for batch */
+
+	rte_iova_t fault_address;
+	uint32_t invalid_flags;
+} __rte_aligned(32);
+
+#define BATCH_SIZE 64
+
+/**
+ * Structure used inside the driver for building up and submitting
+ * a batch of operations to the DSA hardware.
+ */
+struct rte_idxd_desc_batch {
+	struct rte_idxd_completion comp; /* the completion record for batch */
+
+	uint16_t submitted;
+	uint16_t op_count;
+	uint16_t hdl_end;
+
+	struct rte_idxd_hw_desc batch_desc;
+
+	/* batches must always have 2 descriptors, so put a null at the start */
+	struct rte_idxd_hw_desc null_desc;
+	struct rte_idxd_hw_desc ops[BATCH_SIZE];
+};
+
+/**
+ * structure used to save the "handles" provided by the user to be
+ * returned to the user on job completion.
+ */
+struct rte_idxd_user_hdl {
+	uint64_t src;
+	uint64_t dst;
+};
+
 /**
  * @internal
  * Structure representing an IDXD device instance