d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

crypto(9): add CRYPTO_BUF_VMPAGE

Browse Source 
crypto(9) functions can now be used on buffers composed of an array of
vm_page_t structures, such as those stored in an unmapped struct bio.  It
requires the running to kernel to support the direct memory map, so not all
architectures can use it.

Reviewed by:	markj, kib, jhb, mjg, mat, bcr (manpages)
MFC after:	1 week
Sponsored by:	Axcient
Differential Revision:	https://reviews.freebsd.org/D25671
This commit is contained in:
Alan Somers 2020-08-26 02:37:42 +00:00
parent d4d2928a3f
commit e6f6d0c9bc
10 changed files with 338 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
share/man/man9/crypto_buffer.9
View File

@ -30,7 +30,7 @@
.\"
.\" $FreeBSD$
.\"
.Dd May 25, 2020
.Dd August 12, 2020
.Dt CRYPTO_BUFFER 9
.Os
.Sh NAME
@ -197,10 +197,17 @@ A scatter/gather list of kernel buffers as described in
.It Dv CRYPTO_BUF_MBUF
A network memory buffer as described in
.Xr mbuf 9 .
.It Dv CRYPTO_BUF_VMPAGE
A scatter/gather list of
.Vt vm_page_t
structures describing pages in the kernel's address space.
This buffer type is only available if
.Dv CRYPTO_HAS_VMPAGE
is true.
.El
.Pp
The structure also contains the following type-specific fields:
.Bl -tag -width " cb_buf_len"
.Bl -tag -width " cb_vm_page_offset"
.It Fa cb_buf
A pointer to the start of a
.Dv CRYPTO_BUF_CONTIG
@ -219,6 +226,19 @@ A pointer to a
.Vt struct uio
for
.Dv CRYPTO_BUF_UIO .
.It Fa cb_vm_page
A pointer to an array of
.Vt struct vm_page
for
.Dv CRYPTO_BUF_VMPAGE .
.It Fa cb_vm_page_len
The total amount of data included in the
.Fa cb_vm_page
array, in bytes.
.It Fa cb_vm_page_offset
Offset in bytes in the first page of
.Fa cb_vm_page
where valid data begins.
.El
.Ss Cursors
Cursors provide a mechanism for iterating over a data buffer.

18
share/man/man9/crypto_request.9
View File

@ -30,7 +30,7 @@
.\"
.\" $FreeBSD$
.\"
.Dd July 16, 2020
.Dd August 12, 2020
.Dt CRYPTO_REQUEST 9
.Os
.Sh NAME
@ -55,11 +55,15 @@
.Ft void
.Fn crypto_use_uio "struct cryptop *crp" "struct uio *uio"
.Ft void
.Fn crypto_use_vmpage "struct cryptop *crp" "vm_page_t *pages" "int len" "int offset"
.Ft void
.Fn crypto_use_output_buf "struct cryptop *crp" "void *buf" "int len"
.Ft void
.Fn crypto_use_output_mbuf "struct cryptop *crp" "struct mbuf *m"
.Ft void
.Fn crypto_use_output_uio "struct cryptop *crp" "struct uio *uio"
.Ft void
.Fn crypto_use_output_vmpage "struct cryptop *crp" "vm_page_t *pages" "int len" "int offset"
.Sh DESCRIPTION
Each symmetric cryptographic operation in the kernel is described by
an instance of
@ -141,7 +145,7 @@ mode requests.
All requests must have a valid
.Fa crp_buf
initialized by one of the following functions:
.Bl -tag -width "Fn crypto_use_mbuf"
.Bl -tag -width "Fn crypto_use_vmpage"
.It Fn crypto_use_buf
Uses an array of
.Fa len
@ -156,12 +160,16 @@ as the data buffer.
Uses the scatter/gather list
.Fa uio
as the data buffer.
.It Fn crypto_use_vmpage
Uses the array of
.Vt vm_page_t
structures as the data buffer.
.El
.Pp
One of the following functions should be used to initialize
.Fa crp_obuf
for requests that use separate input and output buffers:
.Bl -tag -width "Fn crypto_use_output_mbuf"
.Bl -tag -width "Fn crypto_use_output_vmpage"
.It Fn crypto_use_output_buf
Uses an array of
.Fa len
@ -176,6 +184,10 @@ as the output buffer.
Uses the scatter/gather list
.Fa uio
as the output buffer.
.It Fn crypto_use_output_vmpage
Uses the array of
.Vt vm_page_t
structures as the output buffer.
.El
.Ss Request Regions
Each request describes one or more regions in the data buffers.

4
sys/crypto/ccp/ccp.c
View File

@ -107,6 +107,10 @@ ccp_populate_sglist(struct sglist *sg, struct crypto_buffer *cb)
case CRYPTO_BUF_CONTIG:
error = sglist_append(sg, cb->cb_buf, cb->cb_buf_len);
break;
case CRYPTO_BUF_VMPAGE:
error = sglist_append_vmpages(sg, cb->cb_vm_page,
cb->cb_vm_page_len, cb->cb_vm_page_offset);
break;
default:
error = EINVAL;
}

4
sys/dev/cxgbe/crypto/t4_crypto.c
View File

@ -272,6 +272,10 @@ ccr_populate_sglist(struct sglist *sg, struct crypto_buffer *cb)
case CRYPTO_BUF_CONTIG:
error = sglist_append(sg, cb->cb_buf, cb->cb_buf_len);
break;
case CRYPTO_BUF_VMPAGE:
error = sglist_append_vmpages(sg, cb->cb_vm_page,
cb->cb_vm_page_len, cb->cb_vm_page_offset);
break;
default:
error = EINVAL;
}

3
sys/dev/sec/sec.c
View File

@ -851,6 +851,9 @@ sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem,
case CRYPTO_BUF_MBUF:
size = m_length(crp->crp_buf.cb_mbuf, NULL);
break;
case CRYPTO_BUF_VMPAGE:
size = PAGE_SIZE - cb->cb_vm_page_offset;
break;
default:
return (EINVAL);
}

5
sys/kern/subr_bus_dma.c
View File

@ -661,6 +661,11 @@ bus_dmamap_load_crp_buffer(bus_dma_tag_t dmat, bus_dmamap_t map,
error = _bus_dmamap_load_uio(dmat, map, cb->cb_uio, &nsegs,
flags);
break;
case CRYPTO_BUF_VMPAGE:
error = _bus_dmamap_load_ma(dmat, map, cb->cb_vm_page,
cb->cb_vm_page_len, cb->cb_vm_page_offset, flags, NULL,
&nsegs);
break;
default:
error = EINVAL;
}

215
sys/opencrypto/criov.c
View File

@ -40,12 +40,21 @@ __FBSDID("$FreeBSD$");
#include <sys/uio.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/sdt.h>
#include <machine/vmparam.h>
#include <vm/vm.h>
#include <vm/vm_page.h>
#include <vm/pmap.h>
#include <opencrypto/cryptodev.h>
SDT_PROVIDER_DECLARE(opencrypto);
/*
* This macro is only for avoiding code duplication, as we need to skip
* given number of bytes in the same way in three functions below.
* These macros are only for avoiding code duplication, as we need to skip
* given number of bytes in the same way in several functions below.
*/
#define CUIO_SKIP() do { \
KASSERT(off >= 0, ("%s: off %d < 0", __func__, off)); \
@ -60,6 +69,18 @@ __FBSDID("$FreeBSD$");
} \
} while (0)
#define CVM_PAGE_SKIP() do { \
KASSERT(off >= 0, ("%s: off %d < 0", __func__, off)); \
KASSERT(len >= 0, ("%s: len %d < 0", __func__, len)); \
while (off > 0) { \
if (off < PAGE_SIZE) \
break; \
processed += PAGE_SIZE - off; \
off -= PAGE_SIZE - off; \
pages++; \
} \
} while (0)
static void
cuio_copydata(struct uio* uio, int off, int len, caddr_t cp)
{
@ -128,6 +149,96 @@ cuio_getptr(struct uio *uio, int loc, int *off)
return (-1);
}
#if CRYPTO_MAY_HAVE_VMPAGE
/*
* Apply function f to the data in a vm_page_t list starting "off" bytes from
* the beginning, continuing for "len" bytes.
*/
static int
cvm_page_apply(vm_page_t *pages, int off, int len,
int (*f)(void *, const void *, u_int), void *arg)
{
int processed = 0;
unsigned count;
int rval;
CVM_PAGE_SKIP();
while (len > 0) {
char *kaddr = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages));
count = min(PAGE_SIZE - off, len);
rval = (*f)(arg, kaddr + off, count);
if (rval)
return (rval);
len -= count;
processed += count;
off = 0;
pages++;
}
return (0);
}
static inline void *
cvm_page_contiguous_segment(vm_page_t *pages, size_t skip, int len)
{
if ((skip + len - 1) / PAGE_SIZE > skip / PAGE_SIZE)
return (NULL);
pages += (skip / PAGE_SIZE);
skip -= rounddown(skip, PAGE_SIZE);
return (((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages))) + skip);
}
/*
* Copy len bytes of data from the vm_page_t array, skipping the first off
* bytes, into the pointer cp. Return the number of bytes skipped and copied.
* Does not verify the length of the array.
*/
static int
cvm_page_copyback(vm_page_t *pages, int off, int len, c_caddr_t cp)
{
int processed = 0;
unsigned count;
CVM_PAGE_SKIP();
while (len > 0) {
count = min(PAGE_SIZE - off, len);
bcopy(cp, (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages)) + off,
count);
len -= count;
cp += count;
processed += count;
off = 0;
pages++;
}
return (processed);
}
/*
* Copy len bytes of data from the pointer cp into the vm_page_t array,
* skipping the first off bytes, Return the number of bytes skipped and copied.
* Does not verify the length of the array.
*/
static int
cvm_page_copydata(vm_page_t *pages, int off, int len, caddr_t cp)
{
int processed = 0;
unsigned count;
CVM_PAGE_SKIP();
while (len > 0) {
count = min(PAGE_SIZE - off, len);
bcopy(((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages)) + off), cp,
count);
len -= count;
cp += count;
processed += count;
off = 0;
pages++;
}
return processed;
}
#endif /* CRYPTO_MAY_HAVE_VMPAGE */
void
crypto_cursor_init(struct crypto_buffer_cursor *cc,
const struct crypto_buffer *cb)
@ -142,6 +253,11 @@ crypto_cursor_init(struct crypto_buffer_cursor *cc,
case CRYPTO_BUF_MBUF:
cc->cc_mbuf = cb->cb_mbuf;
break;
case CRYPTO_BUF_VMPAGE:
cc->cc_vmpage = cb->cb_vm_page;
cc->cc_buf_len = cb->cb_vm_page_len;
cc->cc_offset = cb->cb_vm_page_offset;
break;
case CRYPTO_BUF_UIO:
cc->cc_iov = cb->cb_uio->uio_iov;
break;
@ -153,6 +269,8 @@ crypto_cursor_init(struct crypto_buffer_cursor *cc,
}
}
SDT_PROBE_DEFINE2(opencrypto, criov, cursor_advance, vmpage, "struct crypto_buffer_cursor*", "size_t");
void
crypto_cursor_advance(struct crypto_buffer_cursor *cc, size_t amount)
{
@ -178,6 +296,24 @@ crypto_cursor_advance(struct crypto_buffer_cursor *cc, size_t amount)
break;
}
break;
case CRYPTO_BUF_VMPAGE:
for (;;) {
SDT_PROBE2(opencrypto, criov, cursor_advance, vmpage,
cc, amount);
remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len);
if (amount < remain) {
cc->cc_buf_len -= amount;
cc->cc_offset += amount;
break;
}
cc->cc_buf_len -= remain;
amount -= remain;
cc->cc_vmpage++;
cc->cc_offset = 0;
if (amount == 0 || cc->cc_buf_len == 0)
break;
}
break;
case CRYPTO_BUF_UIO:
for (;;) {
remain = cc->cc_iov->iov_len - cc->cc_offset;
@ -212,6 +348,9 @@ crypto_cursor_segbase(struct crypto_buffer_cursor *cc)
KASSERT((cc->cc_mbuf->m_flags & M_EXTPG) == 0,
("%s: not supported for unmapped mbufs", __func__));
return (mtod(cc->cc_mbuf, char *) + cc->cc_offset);
case CRYPTO_BUF_VMPAGE:
return ((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
*cc->cc_vmpage)) + cc->cc_offset);
case CRYPTO_BUF_UIO:
return ((char *)cc->cc_iov->iov_base + cc->cc_offset);
default:
@ -228,6 +367,8 @@ crypto_cursor_seglen(struct crypto_buffer_cursor *cc)
switch (cc->cc_type) {
case CRYPTO_BUF_CONTIG:
return (cc->cc_buf_len);
case CRYPTO_BUF_VMPAGE:
return (PAGE_SIZE - cc->cc_offset);
case CRYPTO_BUF_MBUF:
if (cc->cc_mbuf == NULL)
return (0);
@ -278,6 +419,26 @@ crypto_cursor_copyback(struct crypto_buffer_cursor *cc, int size,
break;
}
break;
case CRYPTO_BUF_VMPAGE:
for (;;) {
dst = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
*cc->cc_vmpage)) + cc->cc_offset;
remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len);
todo = MIN(remain, size);
memcpy(dst, src, todo);
src += todo;
cc->cc_buf_len -= todo;
if (todo < remain) {
cc->cc_offset += todo;
break;
}
size -= todo;
cc->cc_vmpage++;
cc->cc_offset = 0;
if (size == 0)
break;
}
break;
case CRYPTO_BUF_UIO:
for (;;) {
dst = (char *)cc->cc_iov->iov_base + cc->cc_offset;
@ -339,6 +500,26 @@ crypto_cursor_copydata(struct crypto_buffer_cursor *cc, int size, void *vdst)
break;
}
break;
case CRYPTO_BUF_VMPAGE:
for (;;) {
src = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
*cc->cc_vmpage)) + cc->cc_offset;
remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len);
todo = MIN(remain, size);
memcpy(dst, src, todo);
src += todo;
cc->cc_buf_len -= todo;
if (todo < remain) {
cc->cc_offset += todo;
break;
}
size -= todo;
cc->cc_vmpage++;
cc->cc_offset = 0;
if (size == 0)
break;
}
break;
case CRYPTO_BUF_UIO:
for (;;) {
src = (const char *)cc->cc_iov->iov_base +
@ -421,6 +602,15 @@ crypto_copyback(struct cryptop *crp, int off, int size, const void *src)
case CRYPTO_BUF_MBUF:
m_copyback(cb->cb_mbuf, off, size, src);
break;
#if CRYPTO_MAY_HAVE_VMPAGE
case CRYPTO_BUF_VMPAGE:
MPASS(size <= cb->cb_vm_page_len);
MPASS(size + off <=
cb->cb_vm_page_len + cb->cb_vm_page_offset);
cvm_page_copyback(cb->cb_vm_page,
off + cb->cb_vm_page_offset, size, src);
break;
#endif /* CRYPTO_MAY_HAVE_VMPAGE */
case CRYPTO_BUF_UIO:
cuio_copyback(cb->cb_uio, off, size, src);
break;
@ -444,6 +634,15 @@ crypto_copydata(struct cryptop *crp, int off, int size, void *dst)
case CRYPTO_BUF_MBUF:
m_copydata(crp->crp_buf.cb_mbuf, off, size, dst);
break;
#if CRYPTO_MAY_HAVE_VMPAGE
case CRYPTO_BUF_VMPAGE:
MPASS(size <= crp->crp_buf.cb_vm_page_len);
MPASS(size + off <= crp->crp_buf.cb_vm_page_len +
crp->crp_buf.cb_vm_page_offset);
cvm_page_copydata(crp->crp_buf.cb_vm_page,
off + crp->crp_buf.cb_vm_page_offset, size, dst);
break;
#endif /* CRYPTO_MAY_HAVE_VMPAGE */
case CRYPTO_BUF_UIO:
cuio_copydata(crp->crp_buf.cb_uio, off, size, dst);
break;
@ -473,6 +672,12 @@ crypto_apply_buf(struct crypto_buffer *cb, int off, int len,
case CRYPTO_BUF_UIO:
error = cuio_apply(cb->cb_uio, off, len, f, arg);
break;
#if CRYPTO_MAY_HAVE_VMPAGE
case CRYPTO_BUF_VMPAGE:
error = cvm_page_apply(cb->cb_vm_page,
off + cb->cb_vm_page_offset, len, f, arg);
break;
#endif /* CRYPTO_MAY_HAVE_VMPAGE */
case CRYPTO_BUF_CONTIG:
MPASS(off + len <= cb->cb_buf_len);
error = (*f)(arg, cb->cb_buf + off, len);
@ -540,6 +745,12 @@ crypto_buffer_contiguous_subsegment(struct crypto_buffer *cb, size_t skip,
return (m_contiguous_subsegment(cb->cb_mbuf, skip, len));
case CRYPTO_BUF_UIO:
return (cuio_contiguous_segment(cb->cb_uio, skip, len));
#if CRYPTO_MAY_HAVE_VMPAGE
case CRYPTO_BUF_VMPAGE:
MPASS(skip + len <= cb->cb_vm_page_len);
return (cvm_page_contiguous_segment(cb->cb_vm_page,
skip + cb->cb_vm_page_offset, len));
#endif /* CRYPTO_MAY_HAVE_VMPAGE */
case CRYPTO_BUF_CONTIG:
MPASS(skip + len <= cb->cb_buf_len);
return (cb->cb_buf + skip);

22
sys/opencrypto/crypto.c
View File

@ -78,7 +78,9 @@ __FBSDID("$FreeBSD$");
#include <ddb/ddb.h>
#include <machine/vmparam.h>
#include <vm/uma.h>
#include <crypto/intake.h>
#include <opencrypto/cryptodev.h>
#include <opencrypto/xform_auth.h>
@ -1218,6 +1220,8 @@ crypto_buffer_len(struct crypto_buffer *cb)
if (cb->cb_mbuf->m_flags & M_PKTHDR)
return (cb->cb_mbuf->m_pkthdr.len);
return (m_length(cb->cb_mbuf, NULL));
case CRYPTO_BUF_VMPAGE:
return (cb->cb_vm_page_len);
case CRYPTO_BUF_UIO:
return (cb->cb_uio->uio_resid);
default:
@ -1232,9 +1236,25 @@ cb_sanity(struct crypto_buffer *cb, const char *name)
{
KASSERT(cb->cb_type > CRYPTO_BUF_NONE && cb->cb_type <= CRYPTO_BUF_LAST,
("incoming crp with invalid %s buffer type", name));
if (cb->cb_type == CRYPTO_BUF_CONTIG)
switch (cb->cb_type) {
case CRYPTO_BUF_CONTIG:
KASSERT(cb->cb_buf_len >= 0,
("incoming crp with -ve %s buffer length", name));
break;
case CRYPTO_BUF_VMPAGE:
KASSERT(CRYPTO_HAS_VMPAGE,
("incoming crp uses dmap on supported arch"));
KASSERT(cb->cb_vm_page_len >= 0,
("incoming crp with -ve %s buffer length", name));
KASSERT(cb->cb_vm_page_offset >= 0,
("incoming crp with -ve %s buffer offset", name));
KASSERT(cb->cb_vm_page_offset < PAGE_SIZE,
("incoming crp with %s buffer offset greater than page size"
, name));
break;
default:
break;
}
}
static void

52
sys/opencrypto/cryptodev.h
View File

@ -205,6 +205,15 @@
#define CRYPTO_FLAG_HARDWARE 0x01000000 /* hardware accelerated */
#define CRYPTO_FLAG_SOFTWARE 0x02000000 /* software implementation */
/* Does the kernel support vmpage buffers on this platform? */
#ifdef __powerpc__
#define CRYPTO_MAY_HAVE_VMPAGE 1
#else
#define CRYPTO_MAY_HAVE_VMPAGE ( PMAP_HAS_DMAP )
#endif
/* Does the currently running system support vmpage buffers on this platform? */
#define CRYPTO_HAS_VMPAGE ( PMAP_HAS_DMAP )
/* NB: deprecated */
struct session_op {
u_int32_t cipher; /* ie. CRYPTO_AES_CBC */
@ -387,7 +396,8 @@ enum crypto_buffer_type {
CRYPTO_BUF_CONTIG,
CRYPTO_BUF_UIO,
CRYPTO_BUF_MBUF,
CRYPTO_BUF_LAST = CRYPTO_BUF_MBUF
CRYPTO_BUF_VMPAGE,
CRYPTO_BUF_LAST = CRYPTO_BUF_VMPAGE
};
/*
@ -402,6 +412,11 @@ struct crypto_buffer {
int cb_buf_len;
};
struct mbuf *cb_mbuf;
struct {
vm_page_t *cb_vm_page;
int cb_vm_page_len;
int cb_vm_page_offset;
};
struct uio *cb_uio;
};
enum crypto_buffer_type cb_type;
@ -415,11 +430,15 @@ struct crypto_buffer_cursor {
char *cc_buf;
struct mbuf *cc_mbuf;
struct iovec *cc_iov;
vm_page_t *cc_vmpage;
};
union {
int cc_buf_len;
size_t cc_offset;
};
/* Optional bytes of valid data remaining */
int cc_buf_len;
/*
* Optional offset within the current buffer segment where
* valid data begins
*/
size_t cc_offset;
enum crypto_buffer_type cc_type;
};
@ -508,6 +527,16 @@ _crypto_use_mbuf(struct crypto_buffer *cb, struct mbuf *m)
cb->cb_type = CRYPTO_BUF_MBUF;
}
static __inline void
_crypto_use_vmpage(struct crypto_buffer *cb, vm_page_t *pages, int len,
int offset)
{
cb->cb_vm_page = pages;
cb->cb_vm_page_len = len;
cb->cb_vm_page_offset = offset;
cb->cb_type = CRYPTO_BUF_VMPAGE;
}
static __inline void
_crypto_use_uio(struct crypto_buffer *cb, struct uio *uio)
{
@ -527,6 +556,12 @@ crypto_use_mbuf(struct cryptop *crp, struct mbuf *m)
_crypto_use_mbuf(&crp->crp_buf, m);
}
static __inline void
crypto_use_vmpage(struct cryptop *crp, vm_page_t *pages, int len, int offset)
{
_crypto_use_vmpage(&crp->crp_buf, pages, len, offset);
}
static __inline void
crypto_use_uio(struct cryptop *crp, struct uio *uio)
{
@ -545,6 +580,13 @@ crypto_use_output_mbuf(struct cryptop *crp, struct mbuf *m)
_crypto_use_mbuf(&crp->crp_obuf, m);
}
static __inline void
crypto_use_output_vmpage(struct cryptop *crp, vm_page_t *pages, int len,
int offset)
{
_crypto_use_vmpage(&crp->crp_obuf, pages, len, offset);
}
static __inline void
crypto_use_output_uio(struct cryptop *crp, struct uio *uio)
{

4
sys/opencrypto/cryptosoft.c
View File

@ -980,6 +980,10 @@ swcr_compdec(struct swcr_session *ses, struct cryptop *crp)
}
}
break;
case CRYPTO_BUF_VMPAGE:
adj = crp->crp_payload_length - result;
crp->crp_buf.cb_vm_page_len -= adj;
break;
default:
break;
}