d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

crypto: Add a new type of crypto buffer for a single mbuf.

Browse Source 
This is intended for use in KTLS transmit where each TLS record is
described by a single mbuf that is itself queued in the socket buffer.
Using the existing CRYPTO_BUF_MBUF would result in
bus_dmamap_load_crp() walking additional mbufs in the socket buffer
that are not relevant, but generating a S/G list that potentially
exceeds the limit of the tag (while also wasting CPU cycles).

Reviewed by:	markj
Sponsored by:	Netflix
Differential Revision:	https://reviews.freebsd.org/D30136
This commit is contained in:
John Baldwin 2021-05-25 16:59:18 -07:00
parent 6663f8a23e
commit 883a0196b6
11 changed files with 93 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
share/man/man9/crypto_buffer.9
View File

@ -30,7 +30,7 @@
.\"
.\" $FreeBSD$
.\"
.Dd September 24, 2020
.Dd May 25, 2021
.Dt CRYPTO_BUFFER 9
.Os
.Sh NAME
@ -195,7 +195,10 @@ An array of bytes mapped into the kernel's address space.
A scatter/gather list of kernel buffers as described in
.Xr uio 9 .
.It Dv CRYPTO_BUF_MBUF
A network memory buffer as described in
A chain of network memory buffers as described in
.Xr mbuf 9 .
.It Dv CRYPTO_BUF_SINGLE_MBUF
A single network memory buffer as described in
.Xr mbuf 9 .
.It Dv CRYPTO_BUF_VMPAGE
A scatter/gather list of
@ -220,7 +223,9 @@ data buffer
A pointer to a
.Vt struct mbuf
for
.Dv CRYPTO_BUF_MBUF .
.Dv CRYPTO_BUF_MBUF
and
.Dv CRYPTO_BUF_SINGLE_MBUF .
.It Fa cb_uio
A pointer to a
.Vt struct uio

3
sys/crypto/ccp/ccp.c
View File

@ -101,6 +101,9 @@ ccp_populate_sglist(struct sglist *sg, struct crypto_buffer *cb)
case CRYPTO_BUF_MBUF:
error = sglist_append_mbuf(sg, cb->cb_mbuf);
break;
case CRYPTO_BUF_SINGLE_MBUF:
error = sglist_append_single_mbuf(sg, cb->cb_mbuf);
break;
case CRYPTO_BUF_UIO:
error = sglist_append_uio(sg, cb->cb_uio);
break;

3
sys/dev/cxgbe/crypto/t4_crypto.c
View File

@ -271,6 +271,9 @@ ccr_populate_sglist(struct sglist *sg, struct crypto_buffer *cb)
case CRYPTO_BUF_MBUF:
error = sglist_append_mbuf(sg, cb->cb_mbuf);
break;
case CRYPTO_BUF_SINGLE_MBUF:
error = sglist_append_single_mbuf(sg, cb->cb_mbuf);
break;
case CRYPTO_BUF_UIO:
error = sglist_append_uio(sg, cb->cb_uio);
break;

3
sys/dev/sec/sec.c
View File

@ -850,6 +850,9 @@ sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem,
case CRYPTO_BUF_MBUF:
size = m_length(crp->crp_buf.cb_mbuf, NULL);
break;
case CRYPTO_BUF_SINGLE_MBUF:
size = crp->crp_buf.cb_mbuf->m_len;
break;
case CRYPTO_BUF_VMPAGE:
size = PAGE_SIZE - crp->crp_buf.cb_vm_page_offset;
break;

25
sys/kern/subr_bus_dma.c
View File

@ -172,6 +172,27 @@ _bus_dmamap_load_mbuf_epg(bus_dma_tag_t dmat, bus_dmamap_t map,
return (error);
}
/*
* Load a single mbuf.
*/
static int
_bus_dmamap_load_single_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map,
struct mbuf *m, bus_dma_segment_t *segs, int *nsegs, int flags)
{
int error;
error = 0;
if ((m->m_flags & M_EXTPG) != 0)
error = _bus_dmamap_load_mbuf_epg(dmat, map, m, segs, nsegs,
flags);
else
error = _bus_dmamap_load_buffer(dmat, map, m->m_data, m->m_len,
kernel_pmap, flags | BUS_DMA_LOAD_MBUF, segs, nsegs);
CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
__func__, dmat, flags, error, *nsegs);
return (error);
}
/*
* Load an mbuf chain.
*/
@ -658,6 +679,10 @@ bus_dmamap_load_crp_buffer(bus_dma_tag_t dmat, bus_dmamap_t map,
error = _bus_dmamap_load_mbuf_sg(dmat, map, cb->cb_mbuf,
NULL, &nsegs, flags);
break;
case CRYPTO_BUF_SINGLE_MBUF:
error = _bus_dmamap_load_single_mbuf(dmat, map, cb->cb_mbuf,
NULL, &nsegs, flags);
break;
case CRYPTO_BUF_UIO:
error = _bus_dmamap_load_uio(dmat, map, cb->cb_uio, &nsegs,
flags);

3
sys/mips/nlm/dev/sec/nlmsec.c
View File

@ -468,12 +468,15 @@ xlp_get_nsegs(struct cryptop *crp, unsigned int *nsegs)
switch (crp->crp_buf.cb_type) {
case CRYPTO_BUF_MBUF:
case CRYPTO_BUF_SINGLE_MBUF:
{
struct mbuf *m = NULL;
m = crp->crp_buf.cb_mbuf;
while (m != NULL) {
*nsegs += NLM_CRYPTO_NUM_SEGS_REQD(m->m_len);
if (crp->crp_buf.cb_type == CRYPTO_BUF_SINGLE_MBUF)
break;
m = m->m_next;
}
break;

3
sys/mips/nlm/dev/sec/nlmseclib.c
View File

@ -112,6 +112,7 @@ nlm_crypto_form_srcdst_segs(struct xlp_sec_command *cmd,
switch (crp->crp_buf.cb_type) {
case CRYPTO_BUF_MBUF:
case CRYPTO_BUF_SINGLE_MBUF:
{
struct mbuf *m = NULL;
@ -123,6 +124,8 @@ nlm_crypto_form_srcdst_segs(struct xlp_sec_command *cmd,
dstseg = nlm_crypto_fill_dst_seg(cmd->paramp,
dstseg, mtod(m,caddr_t), m->m_len);
}
if (crp->crp_buf.cb_type == CRYPTO_BUF_SINGLE_MBUF)
break;
m = m->m_next;
}
break;

21
sys/opencrypto/criov.c
View File

@ -344,6 +344,7 @@ crypto_cursor_init(struct crypto_buffer_cursor *cc,
cc->cc_buf_len = cb->cb_buf_len;
break;
case CRYPTO_BUF_MBUF:
case CRYPTO_BUF_SINGLE_MBUF:
cc->cc_mbuf = cb->cb_mbuf;
break;
case CRYPTO_BUF_VMPAGE:
@ -389,6 +390,10 @@ crypto_cursor_advance(struct crypto_buffer_cursor *cc, size_t amount)
break;
}
break;
case CRYPTO_BUF_SINGLE_MBUF:
MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + amount);
cc->cc_offset += amount;
break;
case CRYPTO_BUF_VMPAGE:
for (;;) {
SDT_PROBE2(opencrypto, criov, cursor_advance, vmpage,
@ -436,6 +441,7 @@ crypto_cursor_segbase(struct crypto_buffer_cursor *cc)
case CRYPTO_BUF_CONTIG:
return (cc->cc_buf);
case CRYPTO_BUF_MBUF:
case CRYPTO_BUF_SINGLE_MBUF:
if (cc->cc_mbuf == NULL)
return (NULL);
if (cc->cc_mbuf->m_flags & M_EXTPG)
@ -463,6 +469,7 @@ crypto_cursor_seglen(struct crypto_buffer_cursor *cc)
case CRYPTO_BUF_VMPAGE:
return (PAGE_SIZE - cc->cc_offset);
case CRYPTO_BUF_MBUF:
case CRYPTO_BUF_SINGLE_MBUF:
if (cc->cc_mbuf == NULL)
return (0);
if (cc->cc_mbuf->m_flags & M_EXTPG)
@ -516,6 +523,11 @@ crypto_cursor_copyback(struct crypto_buffer_cursor *cc, int size,
break;
}
break;
case CRYPTO_BUF_SINGLE_MBUF:
MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + size);
m_copyback(cc->cc_mbuf, cc->cc_offset, size, src);
cc->cc_offset += size;
break;
case CRYPTO_BUF_VMPAGE:
for (;;) {
dst = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
@ -599,6 +611,11 @@ crypto_cursor_copydata(struct crypto_buffer_cursor *cc, int size, void *vdst)
break;
}
break;
case CRYPTO_BUF_SINGLE_MBUF:
MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + size);
m_copydata(cc->cc_mbuf, cc->cc_offset, size, dst);
cc->cc_offset += size;
break;
case CRYPTO_BUF_VMPAGE:
for (;;) {
src = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
@ -699,6 +716,7 @@ crypto_copyback(struct cryptop *crp, int off, int size, const void *src)
cb = &crp->crp_buf;
switch (cb->cb_type) {
case CRYPTO_BUF_MBUF:
case CRYPTO_BUF_SINGLE_MBUF:
m_copyback(cb->cb_mbuf, off, size, src);
break;
#if CRYPTO_MAY_HAVE_VMPAGE
@ -731,6 +749,7 @@ crypto_copydata(struct cryptop *crp, int off, int size, void *dst)
switch (crp->crp_buf.cb_type) {
case CRYPTO_BUF_MBUF:
case CRYPTO_BUF_SINGLE_MBUF:
m_copydata(crp->crp_buf.cb_mbuf, off, size, dst);
break;
#if CRYPTO_MAY_HAVE_VMPAGE
@ -765,6 +784,7 @@ crypto_apply_buf(struct crypto_buffer *cb, int off, int len,
switch (cb->cb_type) {
case CRYPTO_BUF_MBUF:
case CRYPTO_BUF_SINGLE_MBUF:
error = m_apply(cb->cb_mbuf, off, len,
(int (*)(void *, void *, u_int))f, arg);
break;
@ -843,6 +863,7 @@ crypto_buffer_contiguous_subsegment(struct crypto_buffer *cb, size_t skip,
switch (cb->cb_type) {
case CRYPTO_BUF_MBUF:
case CRYPTO_BUF_SINGLE_MBUF:
return (m_contiguous_subsegment(cb->cb_mbuf, skip, len));
case CRYPTO_BUF_UIO:
return (cuio_contiguous_segment(cb->cb_uio, skip, len));

2
sys/opencrypto/crypto.c
View File

@ -1155,6 +1155,8 @@ crypto_buffer_len(struct crypto_buffer *cb)
if (cb->cb_mbuf->m_flags & M_PKTHDR)
return (cb->cb_mbuf->m_pkthdr.len);
return (m_length(cb->cb_mbuf, NULL));
case CRYPTO_BUF_SINGLE_MBUF:
return (cb->cb_mbuf->m_len);
case CRYPTO_BUF_VMPAGE:
return (cb->cb_vm_page_len);
case CRYPTO_BUF_UIO:

22
sys/opencrypto/cryptodev.h
View File

@ -354,7 +354,8 @@ enum crypto_buffer_type {
CRYPTO_BUF_UIO,
CRYPTO_BUF_MBUF,
CRYPTO_BUF_VMPAGE,
CRYPTO_BUF_LAST = CRYPTO_BUF_VMPAGE
CRYPTO_BUF_SINGLE_MBUF,
CRYPTO_BUF_LAST = CRYPTO_BUF_SINGLE_MBUF
};
/*
@ -480,6 +481,13 @@ _crypto_use_mbuf(struct crypto_buffer *cb, struct mbuf *m)
cb->cb_type = CRYPTO_BUF_MBUF;
}
static __inline void
_crypto_use_single_mbuf(struct crypto_buffer *cb, struct mbuf *m)
{
cb->cb_mbuf = m;
cb->cb_type = CRYPTO_BUF_SINGLE_MBUF;
}
static __inline void
_crypto_use_vmpage(struct crypto_buffer *cb, vm_page_t *pages, int len,
int offset)
@ -509,6 +517,12 @@ crypto_use_mbuf(struct cryptop *crp, struct mbuf *m)
_crypto_use_mbuf(&crp->crp_buf, m);
}
static __inline void
crypto_use_single_mbuf(struct cryptop *crp, struct mbuf *m)
{
_crypto_use_single_mbuf(&crp->crp_buf, m);
}
static __inline void
crypto_use_vmpage(struct cryptop *crp, vm_page_t *pages, int len, int offset)
{
@ -533,6 +547,12 @@ crypto_use_output_mbuf(struct cryptop *crp, struct mbuf *m)
_crypto_use_mbuf(&crp->crp_obuf, m);
}
static __inline void
crypto_use_output_single_mbuf(struct cryptop *crp, struct mbuf *m)
{
_crypto_use_single_mbuf(&crp->crp_obuf, m);
}
static __inline void
crypto_use_output_vmpage(struct cryptop *crp, vm_page_t *pages, int len,
int offset)

1
sys/opencrypto/cryptosoft.c
View File

@ -1105,6 +1105,7 @@ swcr_compdec(struct swcr_session *ses, struct cryptop *crp)
if (result < crp->crp_payload_length) {
switch (crp->crp_buf.cb_type) {
case CRYPTO_BUF_MBUF:
case CRYPTO_BUF_SINGLE_MBUF:
adj = result - crp->crp_payload_length;
m_adj(crp->crp_buf.cb_mbuf, adj);
break;