Generalize AES iov optimization

Right now, aesni_cipher_alloc does a bit of special-casing
for CRYPTO_F_IOV, to not do any allocation if the first uio
is large enough for the requested size. While working on ZFS
crypto port, I ran into horrible performance because the code
uses scatter-gather, and many of the times the data to encrypt
was in the second entry. This code looks through the list, and
tries to see if there is a single uio that can contain the
requested data, and, if so, uses that.

This has a slight impact on the current consumers, in that the
check is a little more complicated for the ones that use
CRYPTO_F_IOV -- but none of them meet the criteria for testing
more than one.

Submitted by:	sef at ixsystems.com
Reviewed by:	cem@
MFC after:	3 days
Sponsored by:	iX Systems
Differential Revision:	https://reviews.freebsd.org/D18522

sys/crypto/aesni/aesni.c

@@ -403,29 +403,14 @@ static uint8_t *
 aesni_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp,
     bool *allocated)
 {
-	struct mbuf *m;
-	struct uio *uio;
-	struct iovec *iov;
 	uint8_t *addr;
 
-	if (crp->crp_flags & CRYPTO_F_IMBUF) {
-		m = (struct mbuf *)crp->crp_buf;
-		if (m->m_next != NULL)
-			goto alloc;
-		addr = mtod(m, uint8_t *);
-	} else if (crp->crp_flags & CRYPTO_F_IOV) {
-		uio = (struct uio *)crp->crp_buf;
-		if (uio->uio_iovcnt != 1)
-			goto alloc;
-		iov = uio->uio_iov;
-		addr = (uint8_t *)iov->iov_base;
-	} else
-		addr = (uint8_t *)crp->crp_buf;
+	addr = crypto_contiguous_subsegment(crp->crp_flags,
+	    crp->crp_buf, enccrd->crd_skip, enccrd->crd_len);
+	if (addr != NULL) {
 		*allocated = false;
-	addr += enccrd->crd_skip;
 		return (addr);
-
-alloc:
+	}
 	addr = malloc(enccrd->crd_len, M_AESNI, M_NOWAIT);
 	if (addr != NULL) {
 		*allocated = true;

sys/opencrypto/criov.c

@@ -38,6 +38,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/mbuf.h>
 #include <sys/uio.h>
+#include <sys/limits.h>
+#include <sys/lock.h>
 
 #include <opencrypto/cryptodev.h>
 
@@ -239,3 +241,55 @@ crypto_mbuftoiov(struct mbuf *mbuf, struct iovec **iovptr, int *cnt,
 	*cnt = i;
 	return 0;
 }
+
+static inline void *
+m_contiguous_subsegment(struct mbuf *m, size_t skip, size_t len)
+{
+	int rel_off;
+
+	MPASS(skip <= INT_MAX);
+
+	m = m_getptr(m, (int)skip, &rel_off);
+	if (m == NULL)
+		return (NULL);
+
+	MPASS(rel_off >= 0);
+	skip = rel_off;
+	if (skip + len > m->m_len)
+		return (NULL);
+
+	return (mtod(m, char*) + skip);
+}
+
+static inline void *
+cuio_contiguous_segment(struct uio *uio, size_t skip, size_t len)
+{
+	int rel_off, idx;
+
+	MPASS(skip <= INT_MAX);
+	idx = cuio_getptr(uio, (int)skip, &rel_off);
+	if (idx < 0)
+		return (NULL);
+
+	MPASS(rel_off >= 0);
+	skip = rel_off;
+	if (skip + len > uio->uio_iov[idx].iov_len)
+		return (NULL);
+	return ((char *)uio->uio_iov[idx].iov_base + skip);
+}
+
+void *
+crypto_contiguous_subsegment(int crp_flags, void *crpbuf,
+    size_t skip, size_t len)
+{
+	if ((crp_flags & CRYPTO_F_IMBUF) != 0)
+		return (m_contiguous_subsegment(crpbuf, skip, len));
+	else if ((crp_flags & CRYPTO_F_IOV) != 0)
+		return (cuio_contiguous_segment(crpbuf, skip, len));
+	else {
+		MPASS((crp_flags & (CRYPTO_F_IMBUF | CRYPTO_F_IOV)) !=
+		    (CRYPTO_F_IMBUF | CRYPTO_F_IOV));
+		return ((char*)crpbuf + skip);
+	}
+}
+

sys/opencrypto/cryptodev.h

@@ -564,5 +564,7 @@ extern	void crypto_copydata(int flags, caddr_t buf, int off, int size,
 extern	int crypto_apply(int flags, caddr_t buf, int off, int len,
 	    int (*f)(void *, void *, u_int), void *arg);
 
+extern void *crypto_contiguous_subsegment(int, void *, size_t, size_t);
+
 #endif /* _KERNEL */
 #endif /* _CRYPTO_CRYPTO_H_ */