ioat(4): Implement CRC and MOVECRC APIs

And document them in ioat.4.

Sponsored by:	EMC / Isilon Storage Division

share/man/man4/ioat.4

@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd January 14, 2016
+.Dd May 3, 2016
 .Dt IOAT 4
 .Os
 .Sh NAME
@@ -99,6 +99,29 @@ In
 .Fa "uint32_t flags"
 .Fc
 .Ft struct bus_dmadesc *
+.Fo ioat_copy_crc
+.Fa "bus_dmaengine_t dmaengine"
+.Fa "bus_addr_t dst"
+.Fa "bus_addr_t src"
+.Fa "bus_size_t len"
+.Fa "uint32_t *initialseed"
+.Fa "bus_addr_t crcptr"
+.Fa "bus_dmaengine_callback_t callback_fn"
+.Fa "void *callback_arg"
+.Fa "uint32_t flags"
+.Fc
+.Ft struct bus_dmadesc *
+.Fo ioat_crc
+.Fa "bus_dmaengine_t dmaengine"
+.Fa "bus_addr_t src"
+.Fa "bus_size_t len"
+.Fa "uint32_t *initialseed"
+.Fa "bus_addr_t crcptr"
+.Fa "bus_dmaengine_callback_t callback_fn"
+.Fa "void *callback_arg"
+.Fa "uint32_t flags"
+.Fc
+.Ft struct bus_dmadesc *
 .Fo ioat_blockfill
 .Fa "bus_dmaengine_t dmaengine"
 .Fa "bus_addr_t dst"
@@ -183,6 +206,43 @@ from.
 It is invalid to attempt to submit new DMA operations in a
 .Fa bus_dmaengine_callback_t
 context.
+.Pp
+The CRC operations have three distinct modes.
+The default mode is to accumulate.
+By accumulating over multiple descriptors, a user may gather a CRC over several
+chunks of memory and only write out the result once.
+.Pp
+The
+.Ar DMA_CRC_STORE
+flag causes the operation to emit the CRC32C result.
+If
+.Ar DMA_CRC_INLINE
+is set, the result is written inline with the destination data (or source in
+.Fn ioat_crc
+mode).
+If
+.Ar DMA_CRC_INLINE
+is not set, the result is written to the provided
+.Fa crcptr .
+.Pp
+Similarly, the
+.Ar DMA_CRC_TEST
+flag causes the operation to compare the CRC32C result to an existing checksum.
+If
+.Ar DMA_CRC_INLINE
+is set, the result is compared against the inline four bytes trailing the
+source data.
+If it is not set, the result is compared against the value pointed to by
+.Fa crcptr .
+.Pp
+.Fn ioat_copy_crc
+calculates a CRC32C while copying data.
+.Fn ioat_crc
+only computes a CRC32C of some data.
+If the
+.Fa initialseed
+argument to either routine is non-NULL, the CRC32C engine is initialized with
+the value it points to.
 .Sh USAGE
 A typical user will lookup the DMA engine object for a given channel with
 .Fn ioat_get_dmaengine .
@@ -199,9 +259,13 @@ If
 succeeds, there is guaranteed to be room for
 .Fa N
 new operations in the internal ring buffer.
+.Pp
 Then, they will submit one or more operations using
 .Fn ioat_blockfill ,
 .Fn ioat_copy ,
+.Fn ioat_copy_8k_aligned ,
+.Fn ioat_copy_crc ,
+.Fn ioat_crc ,
 or
 .Fn ioat_null .
 After queuing one or more individual DMA operations, they will

sys/dev/ioat/ioat.c

@@ -882,8 +882,8 @@ ioat_op_generic(struct ioat_softc *ioat, uint8_t op,
 
 	mtx_assert(&ioat->submit_lock, MA_OWNED);
 
-	KASSERT((flags & ~DMA_ALL_FLAGS) == 0, ("Unrecognized flag(s): %#x",
+	KASSERT((flags & ~_DMA_GENERIC_FLAGS) == 0,
-		flags & ~DMA_ALL_FLAGS));
+	    ("Unrecognized flag(s): %#x", flags & ~_DMA_GENERIC_FLAGS));
 	if ((flags & DMA_NO_WAIT) != 0)
 		mflags = M_NOWAIT;
 	else
@@ -1017,6 +1017,164 @@ ioat_copy_8k_aligned(bus_dmaengine_t dmaengine, bus_addr_t dst1,
 	return (&desc->bus_dmadesc);
 }
 
+struct bus_dmadesc *
+ioat_copy_crc(bus_dmaengine_t dmaengine, bus_addr_t dst, bus_addr_t src,
+    bus_size_t len, uint32_t *initialseed, bus_addr_t crcptr,
+    bus_dmaengine_callback_t callback_fn, void *callback_arg, uint32_t flags)
+{
+	struct ioat_crc32_hw_descriptor *hw_desc;
+	struct ioat_descriptor *desc;
+	struct ioat_softc *ioat;
+	uint32_t teststore;
+	uint8_t op;
+
+	CTR0(KTR_IOAT, __func__);
+	ioat = to_ioat_softc(dmaengine);
+
+	if ((ioat->capabilities & IOAT_DMACAP_MOVECRC) == 0) {
+		ioat_log_message(0, "%s: Device lacks MOVECRC capability\n",
+		    __func__);
+		return (NULL);
+	}
+	if (((src | dst) & (0xffffffull << 40)) != 0) {
+		ioat_log_message(0, "%s: High 24 bits of src/dst invalid\n",
+		    __func__);
+		return (NULL);
+	}
+	teststore = (flags & _DMA_CRC_TESTSTORE);
+	if (teststore == _DMA_CRC_TESTSTORE) {
+		ioat_log_message(0, "%s: TEST and STORE invalid\n", __func__);
+		return (NULL);
+	}
+	if (teststore == 0 && (flags & DMA_CRC_INLINE) != 0) {
+		ioat_log_message(0, "%s: INLINE invalid without TEST or STORE\n",
+		    __func__);
+		return (NULL);
+	}
+
+	switch (teststore) {
+	case DMA_CRC_STORE:
+		op = IOAT_OP_MOVECRC_STORE;
+		break;
+	case DMA_CRC_TEST:
+		op = IOAT_OP_MOVECRC_TEST;
+		break;
+	default:
+		KASSERT(teststore == 0, ("bogus"));
+		op = IOAT_OP_MOVECRC;
+		break;
+	}
+
+	if ((flags & DMA_CRC_INLINE) == 0 &&
+	    (crcptr & (0xffffffull << 40)) != 0) {
+		ioat_log_message(0,
+		    "%s: High 24 bits of crcptr invalid\n", __func__);
+		return (NULL);
+	}
+
+	desc = ioat_op_generic(ioat, op, len, src, dst, callback_fn,
+	    callback_arg, flags & ~_DMA_CRC_FLAGS);
+	if (desc == NULL)
+		return (NULL);
+
+	hw_desc = desc->u.crc32;
+
+	if ((flags & DMA_CRC_INLINE) == 0)
+		hw_desc->crc_address = crcptr;
+	else
+		hw_desc->u.control.crc_location = 1;
+
+	if (initialseed != NULL) {
+		hw_desc->u.control.use_seed = 1;
+		hw_desc->seed = *initialseed;
+	}
+
+	if (g_ioat_debug_level >= 3)
+		dump_descriptor(hw_desc);
+
+	ioat_submit_single(ioat);
+	return (&desc->bus_dmadesc);
+}
+
+struct bus_dmadesc *
+ioat_crc(bus_dmaengine_t dmaengine, bus_addr_t src, bus_size_t len,
+    uint32_t *initialseed, bus_addr_t crcptr,
+    bus_dmaengine_callback_t callback_fn, void *callback_arg, uint32_t flags)
+{
+	struct ioat_crc32_hw_descriptor *hw_desc;
+	struct ioat_descriptor *desc;
+	struct ioat_softc *ioat;
+	uint32_t teststore;
+	uint8_t op;
+
+	CTR0(KTR_IOAT, __func__);
+	ioat = to_ioat_softc(dmaengine);
+
+	if ((ioat->capabilities & IOAT_DMACAP_CRC) == 0) {
+		ioat_log_message(0, "%s: Device lacks CRC capability\n",
+		    __func__);
+		return (NULL);
+	}
+	if ((src & (0xffffffull << 40)) != 0) {
+		ioat_log_message(0, "%s: High 24 bits of src invalid\n",
+		    __func__);
+		return (NULL);
+	}
+	teststore = (flags & _DMA_CRC_TESTSTORE);
+	if (teststore == _DMA_CRC_TESTSTORE) {
+		ioat_log_message(0, "%s: TEST and STORE invalid\n", __func__);
+		return (NULL);
+	}
+	if (teststore == 0 && (flags & DMA_CRC_INLINE) != 0) {
+		ioat_log_message(0, "%s: INLINE invalid without TEST or STORE\n",
+		    __func__);
+		return (NULL);
+	}
+
+	switch (teststore) {
+	case DMA_CRC_STORE:
+		op = IOAT_OP_CRC_STORE;
+		break;
+	case DMA_CRC_TEST:
+		op = IOAT_OP_CRC_TEST;
+		break;
+	default:
+		KASSERT(teststore == 0, ("bogus"));
+		op = IOAT_OP_CRC;
+		break;
+	}
+
+	if ((flags & DMA_CRC_INLINE) == 0 &&
+	    (crcptr & (0xffffffull << 40)) != 0) {
+		ioat_log_message(0,
+		    "%s: High 24 bits of crcptr invalid\n", __func__);
+		return (NULL);
+	}
+
+	desc = ioat_op_generic(ioat, op, len, src, 0, callback_fn,
+	    callback_arg, flags & ~_DMA_CRC_FLAGS);
+	if (desc == NULL)
+		return (NULL);
+
+	hw_desc = desc->u.crc32;
+
+	if ((flags & DMA_CRC_INLINE) == 0)
+		hw_desc->crc_address = crcptr;
+	else
+		hw_desc->u.control.crc_location = 1;
+
+	if (initialseed != NULL) {
+		hw_desc->u.control.use_seed = 1;
+		hw_desc->seed = *initialseed;
+	}
+
+	if (g_ioat_debug_level >= 3)
+		dump_descriptor(hw_desc);
+
+	ioat_submit_single(ioat);
+	return (&desc->bus_dmadesc);
+}
+
 struct bus_dmadesc *
 ioat_blockfill(bus_dmaengine_t dmaengine, bus_addr_t dst, uint64_t fillpattern,
     bus_size_t len, bus_dmaengine_callback_t callback_fn, void *callback_arg,

sys/dev/ioat/ioat.h

@@ -52,7 +52,26 @@ __FBSDID("$FreeBSD$");
  * may be prefetched before operation 1 completes.
  */
 #define	DMA_FENCE	0x4
-#define	DMA_ALL_FLAGS	(DMA_INT_EN | DMA_NO_WAIT | DMA_FENCE)
+#define	_DMA_GENERIC_FLAGS	(DMA_INT_EN | DMA_NO_WAIT | DMA_FENCE)
+
+/*
+ * Emit a CRC32C as the result of a ioat_copy_crc() or ioat_crc().
+ */
+#define	DMA_CRC_STORE	0x8
+
+/*
+ * Compare the CRC32C of a ioat_copy_crc() or ioat_crc() against an expeceted
+ * value.  It is invalid to specify both TEST and STORE.
+ */
+#define	DMA_CRC_TEST	0x10
+#define	_DMA_CRC_TESTSTORE	(DMA_CRC_STORE | DMA_CRC_TEST)
+
+/*
+ * Use an inline comparison CRC32C or emit an inline CRC32C result.  Invalid
+ * without one of STORE or TEST.
+ */
+#define	DMA_CRC_INLINE	0x20
+#define	_DMA_CRC_FLAGS	(DMA_CRC_STORE | DMA_CRC_TEST | DMA_CRC_INLINE)
 
 /*
  * Hardware revision number.  Different hardware revisions support different
@@ -151,6 +170,42 @@ struct bus_dmadesc *ioat_copy_8k_aligned(bus_dmaengine_t dmaengine,
     bus_addr_t dst1, bus_addr_t dst2, bus_addr_t src1, bus_addr_t src2,
     bus_dmaengine_callback_t callback_fn, void *callback_arg, uint32_t flags);
 
+/*
+ * Copy len bytes from dst to src, like ioat_copy().
+ *
+ * Additionally, accumulate a CRC32C of the data.
+ *
+ * If initialseed is not NULL, the value it points to is used to seed the
+ * initial value of the CRC32C.
+ *
+ * If flags include DMA_CRC_STORE and not DMA_CRC_INLINE, crcptr is written
+ * with the 32-bit CRC32C result (in wire format).
+ *
+ * If flags include DMA_CRC_TEST and not DMA_CRC_INLINE, the computed CRC32C is
+ * compared with the 32-bit CRC32C pointed to by crcptr.  If they do not match,
+ * a channel error is raised.
+ *
+ * If the DMA_CRC_INLINE flag is set, crcptr is ignored and the DMA engine uses
+ * the 4 bytes trailing the source data (TEST) or the destination data (STORE).
+ */
+struct bus_dmadesc *ioat_copy_crc(bus_dmaengine_t dmaengine, bus_addr_t dst,
+    bus_addr_t src, bus_size_t len, uint32_t *initialseed, bus_addr_t crcptr,
+    bus_dmaengine_callback_t callback_fn, void *callback_arg, uint32_t flags);
+
+/*
+ * ioat_crc() is nearly identical to ioat_copy_crc(), but does not actually
+ * move data around.
+ *
+ * Like ioat_copy_crc, ioat_crc computes a CRC32C over len bytes pointed to by
+ * src.  The flags affect its operation in the same way, with one exception:
+ *
+ * If flags includes both DMA_CRC_STORE and DMA_CRC_INLINE, the computed CRC32C
+ * is written to the 4 bytes trailing the *source* data.
+ */
+struct bus_dmadesc *ioat_crc(bus_dmaengine_t dmaengine, bus_addr_t src,
+    bus_size_t len, uint32_t *initialseed, bus_addr_t crcptr,
+    bus_dmaengine_callback_t callback_fn, void *callback_arg, uint32_t flags);
+
 /*
  * Issues a null operation. This issues the operation to the hardware, but the
  * hardware doesn't do anything with it.