From 883a0196b629a07e52562b4103cc0f6391083080 Mon Sep 17 00:00:00 2001 From: John Baldwin Date: Tue, 25 May 2021 16:59:18 -0700 Subject: [PATCH] crypto: Add a new type of crypto buffer for a single mbuf. This is intended for use in KTLS transmit where each TLS record is described by a single mbuf that is itself queued in the socket buffer. Using the existing CRYPTO_BUF_MBUF would result in bus_dmamap_load_crp() walking additional mbufs in the socket buffer that are not relevant, but generating a S/G list that potentially exceeds the limit of the tag (while also wasting CPU cycles). Reviewed by: markj Sponsored by: Netflix Differential Revision: https://reviews.freebsd.org/D30136 --- share/man/man9/crypto_buffer.9 | 11 ++++++++--- sys/crypto/ccp/ccp.c | 3 +++ sys/dev/cxgbe/crypto/t4_crypto.c | 3 +++ sys/dev/sec/sec.c | 3 +++ sys/kern/subr_bus_dma.c | 25 +++++++++++++++++++++++++ sys/mips/nlm/dev/sec/nlmsec.c | 3 +++ sys/mips/nlm/dev/sec/nlmseclib.c | 3 +++ sys/opencrypto/criov.c | 21 +++++++++++++++++++++ sys/opencrypto/crypto.c | 2 ++ sys/opencrypto/cryptodev.h | 22 +++++++++++++++++++++- sys/opencrypto/cryptosoft.c | 1 + 11 files changed, 93 insertions(+), 4 deletions(-) diff --git a/share/man/man9/crypto_buffer.9 b/share/man/man9/crypto_buffer.9 index 333fc5ec449f..df6128650cd4 100644 --- a/share/man/man9/crypto_buffer.9 +++ b/share/man/man9/crypto_buffer.9 @@ -30,7 +30,7 @@ .\" .\" $FreeBSD$ .\" -.Dd September 24, 2020 +.Dd May 25, 2021 .Dt CRYPTO_BUFFER 9 .Os .Sh NAME @@ -195,7 +195,10 @@ An array of bytes mapped into the kernel's address space. A scatter/gather list of kernel buffers as described in .Xr uio 9 . .It Dv CRYPTO_BUF_MBUF -A network memory buffer as described in +A chain of network memory buffers as described in +.Xr mbuf 9 . +.It Dv CRYPTO_BUF_SINGLE_MBUF +A single network memory buffer as described in .Xr mbuf 9 . .It Dv CRYPTO_BUF_VMPAGE A scatter/gather list of @@ -220,7 +223,9 @@ data buffer A pointer to a .Vt struct mbuf for -.Dv CRYPTO_BUF_MBUF . +.Dv CRYPTO_BUF_MBUF +and +.Dv CRYPTO_BUF_SINGLE_MBUF . .It Fa cb_uio A pointer to a .Vt struct uio diff --git a/sys/crypto/ccp/ccp.c b/sys/crypto/ccp/ccp.c index 63967d7f8e3c..7cc38b14f3fd 100644 --- a/sys/crypto/ccp/ccp.c +++ b/sys/crypto/ccp/ccp.c @@ -101,6 +101,9 @@ ccp_populate_sglist(struct sglist *sg, struct crypto_buffer *cb) case CRYPTO_BUF_MBUF: error = sglist_append_mbuf(sg, cb->cb_mbuf); break; + case CRYPTO_BUF_SINGLE_MBUF: + error = sglist_append_single_mbuf(sg, cb->cb_mbuf); + break; case CRYPTO_BUF_UIO: error = sglist_append_uio(sg, cb->cb_uio); break; diff --git a/sys/dev/cxgbe/crypto/t4_crypto.c b/sys/dev/cxgbe/crypto/t4_crypto.c index cdd14fcee2f9..0fc806dc2eb8 100644 --- a/sys/dev/cxgbe/crypto/t4_crypto.c +++ b/sys/dev/cxgbe/crypto/t4_crypto.c @@ -271,6 +271,9 @@ ccr_populate_sglist(struct sglist *sg, struct crypto_buffer *cb) case CRYPTO_BUF_MBUF: error = sglist_append_mbuf(sg, cb->cb_mbuf); break; + case CRYPTO_BUF_SINGLE_MBUF: + error = sglist_append_single_mbuf(sg, cb->cb_mbuf); + break; case CRYPTO_BUF_UIO: error = sglist_append_uio(sg, cb->cb_uio); break; diff --git a/sys/dev/sec/sec.c b/sys/dev/sec/sec.c index fa3e20806124..f485d932727b 100644 --- a/sys/dev/sec/sec.c +++ b/sys/dev/sec/sec.c @@ -850,6 +850,9 @@ sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, case CRYPTO_BUF_MBUF: size = m_length(crp->crp_buf.cb_mbuf, NULL); break; + case CRYPTO_BUF_SINGLE_MBUF: + size = crp->crp_buf.cb_mbuf->m_len; + break; case CRYPTO_BUF_VMPAGE: size = PAGE_SIZE - crp->crp_buf.cb_vm_page_offset; break; diff --git a/sys/kern/subr_bus_dma.c b/sys/kern/subr_bus_dma.c index 6248dd520511..a7ad30e77715 100644 --- a/sys/kern/subr_bus_dma.c +++ b/sys/kern/subr_bus_dma.c @@ -172,6 +172,27 @@ _bus_dmamap_load_mbuf_epg(bus_dma_tag_t dmat, bus_dmamap_t map, return (error); } +/* + * Load a single mbuf. + */ +static int +_bus_dmamap_load_single_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, + struct mbuf *m, bus_dma_segment_t *segs, int *nsegs, int flags) +{ + int error; + + error = 0; + if ((m->m_flags & M_EXTPG) != 0) + error = _bus_dmamap_load_mbuf_epg(dmat, map, m, segs, nsegs, + flags); + else + error = _bus_dmamap_load_buffer(dmat, map, m->m_data, m->m_len, + kernel_pmap, flags | BUS_DMA_LOAD_MBUF, segs, nsegs); + CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", + __func__, dmat, flags, error, *nsegs); + return (error); +} + /* * Load an mbuf chain. */ @@ -658,6 +679,10 @@ bus_dmamap_load_crp_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, error = _bus_dmamap_load_mbuf_sg(dmat, map, cb->cb_mbuf, NULL, &nsegs, flags); break; + case CRYPTO_BUF_SINGLE_MBUF: + error = _bus_dmamap_load_single_mbuf(dmat, map, cb->cb_mbuf, + NULL, &nsegs, flags); + break; case CRYPTO_BUF_UIO: error = _bus_dmamap_load_uio(dmat, map, cb->cb_uio, &nsegs, flags); diff --git a/sys/mips/nlm/dev/sec/nlmsec.c b/sys/mips/nlm/dev/sec/nlmsec.c index 9318206a04fd..fb3e0b65b5a1 100644 --- a/sys/mips/nlm/dev/sec/nlmsec.c +++ b/sys/mips/nlm/dev/sec/nlmsec.c @@ -468,12 +468,15 @@ xlp_get_nsegs(struct cryptop *crp, unsigned int *nsegs) switch (crp->crp_buf.cb_type) { case CRYPTO_BUF_MBUF: + case CRYPTO_BUF_SINGLE_MBUF: { struct mbuf *m = NULL; m = crp->crp_buf.cb_mbuf; while (m != NULL) { *nsegs += NLM_CRYPTO_NUM_SEGS_REQD(m->m_len); + if (crp->crp_buf.cb_type == CRYPTO_BUF_SINGLE_MBUF) + break; m = m->m_next; } break; diff --git a/sys/mips/nlm/dev/sec/nlmseclib.c b/sys/mips/nlm/dev/sec/nlmseclib.c index bb154b5d3d79..6e5e715c28f8 100644 --- a/sys/mips/nlm/dev/sec/nlmseclib.c +++ b/sys/mips/nlm/dev/sec/nlmseclib.c @@ -112,6 +112,7 @@ nlm_crypto_form_srcdst_segs(struct xlp_sec_command *cmd, switch (crp->crp_buf.cb_type) { case CRYPTO_BUF_MBUF: + case CRYPTO_BUF_SINGLE_MBUF: { struct mbuf *m = NULL; @@ -123,6 +124,8 @@ nlm_crypto_form_srcdst_segs(struct xlp_sec_command *cmd, dstseg = nlm_crypto_fill_dst_seg(cmd->paramp, dstseg, mtod(m,caddr_t), m->m_len); } + if (crp->crp_buf.cb_type == CRYPTO_BUF_SINGLE_MBUF) + break; m = m->m_next; } break; diff --git a/sys/opencrypto/criov.c b/sys/opencrypto/criov.c index bf7965032084..df053b314d41 100644 --- a/sys/opencrypto/criov.c +++ b/sys/opencrypto/criov.c @@ -344,6 +344,7 @@ crypto_cursor_init(struct crypto_buffer_cursor *cc, cc->cc_buf_len = cb->cb_buf_len; break; case CRYPTO_BUF_MBUF: + case CRYPTO_BUF_SINGLE_MBUF: cc->cc_mbuf = cb->cb_mbuf; break; case CRYPTO_BUF_VMPAGE: @@ -389,6 +390,10 @@ crypto_cursor_advance(struct crypto_buffer_cursor *cc, size_t amount) break; } break; + case CRYPTO_BUF_SINGLE_MBUF: + MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + amount); + cc->cc_offset += amount; + break; case CRYPTO_BUF_VMPAGE: for (;;) { SDT_PROBE2(opencrypto, criov, cursor_advance, vmpage, @@ -436,6 +441,7 @@ crypto_cursor_segbase(struct crypto_buffer_cursor *cc) case CRYPTO_BUF_CONTIG: return (cc->cc_buf); case CRYPTO_BUF_MBUF: + case CRYPTO_BUF_SINGLE_MBUF: if (cc->cc_mbuf == NULL) return (NULL); if (cc->cc_mbuf->m_flags & M_EXTPG) @@ -463,6 +469,7 @@ crypto_cursor_seglen(struct crypto_buffer_cursor *cc) case CRYPTO_BUF_VMPAGE: return (PAGE_SIZE - cc->cc_offset); case CRYPTO_BUF_MBUF: + case CRYPTO_BUF_SINGLE_MBUF: if (cc->cc_mbuf == NULL) return (0); if (cc->cc_mbuf->m_flags & M_EXTPG) @@ -516,6 +523,11 @@ crypto_cursor_copyback(struct crypto_buffer_cursor *cc, int size, break; } break; + case CRYPTO_BUF_SINGLE_MBUF: + MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + size); + m_copyback(cc->cc_mbuf, cc->cc_offset, size, src); + cc->cc_offset += size; + break; case CRYPTO_BUF_VMPAGE: for (;;) { dst = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS( @@ -599,6 +611,11 @@ crypto_cursor_copydata(struct crypto_buffer_cursor *cc, int size, void *vdst) break; } break; + case CRYPTO_BUF_SINGLE_MBUF: + MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + size); + m_copydata(cc->cc_mbuf, cc->cc_offset, size, dst); + cc->cc_offset += size; + break; case CRYPTO_BUF_VMPAGE: for (;;) { src = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS( @@ -699,6 +716,7 @@ crypto_copyback(struct cryptop *crp, int off, int size, const void *src) cb = &crp->crp_buf; switch (cb->cb_type) { case CRYPTO_BUF_MBUF: + case CRYPTO_BUF_SINGLE_MBUF: m_copyback(cb->cb_mbuf, off, size, src); break; #if CRYPTO_MAY_HAVE_VMPAGE @@ -731,6 +749,7 @@ crypto_copydata(struct cryptop *crp, int off, int size, void *dst) switch (crp->crp_buf.cb_type) { case CRYPTO_BUF_MBUF: + case CRYPTO_BUF_SINGLE_MBUF: m_copydata(crp->crp_buf.cb_mbuf, off, size, dst); break; #if CRYPTO_MAY_HAVE_VMPAGE @@ -765,6 +784,7 @@ crypto_apply_buf(struct crypto_buffer *cb, int off, int len, switch (cb->cb_type) { case CRYPTO_BUF_MBUF: + case CRYPTO_BUF_SINGLE_MBUF: error = m_apply(cb->cb_mbuf, off, len, (int (*)(void *, void *, u_int))f, arg); break; @@ -843,6 +863,7 @@ crypto_buffer_contiguous_subsegment(struct crypto_buffer *cb, size_t skip, switch (cb->cb_type) { case CRYPTO_BUF_MBUF: + case CRYPTO_BUF_SINGLE_MBUF: return (m_contiguous_subsegment(cb->cb_mbuf, skip, len)); case CRYPTO_BUF_UIO: return (cuio_contiguous_segment(cb->cb_uio, skip, len)); diff --git a/sys/opencrypto/crypto.c b/sys/opencrypto/crypto.c index b56d0fb70698..b6c4441b4284 100644 --- a/sys/opencrypto/crypto.c +++ b/sys/opencrypto/crypto.c @@ -1155,6 +1155,8 @@ crypto_buffer_len(struct crypto_buffer *cb) if (cb->cb_mbuf->m_flags & M_PKTHDR) return (cb->cb_mbuf->m_pkthdr.len); return (m_length(cb->cb_mbuf, NULL)); + case CRYPTO_BUF_SINGLE_MBUF: + return (cb->cb_mbuf->m_len); case CRYPTO_BUF_VMPAGE: return (cb->cb_vm_page_len); case CRYPTO_BUF_UIO: diff --git a/sys/opencrypto/cryptodev.h b/sys/opencrypto/cryptodev.h index 862ad1d4f6e5..1adc81f2e4d3 100644 --- a/sys/opencrypto/cryptodev.h +++ b/sys/opencrypto/cryptodev.h @@ -354,7 +354,8 @@ enum crypto_buffer_type { CRYPTO_BUF_UIO, CRYPTO_BUF_MBUF, CRYPTO_BUF_VMPAGE, - CRYPTO_BUF_LAST = CRYPTO_BUF_VMPAGE + CRYPTO_BUF_SINGLE_MBUF, + CRYPTO_BUF_LAST = CRYPTO_BUF_SINGLE_MBUF }; /* @@ -480,6 +481,13 @@ _crypto_use_mbuf(struct crypto_buffer *cb, struct mbuf *m) cb->cb_type = CRYPTO_BUF_MBUF; } +static __inline void +_crypto_use_single_mbuf(struct crypto_buffer *cb, struct mbuf *m) +{ + cb->cb_mbuf = m; + cb->cb_type = CRYPTO_BUF_SINGLE_MBUF; +} + static __inline void _crypto_use_vmpage(struct crypto_buffer *cb, vm_page_t *pages, int len, int offset) @@ -509,6 +517,12 @@ crypto_use_mbuf(struct cryptop *crp, struct mbuf *m) _crypto_use_mbuf(&crp->crp_buf, m); } +static __inline void +crypto_use_single_mbuf(struct cryptop *crp, struct mbuf *m) +{ + _crypto_use_single_mbuf(&crp->crp_buf, m); +} + static __inline void crypto_use_vmpage(struct cryptop *crp, vm_page_t *pages, int len, int offset) { @@ -533,6 +547,12 @@ crypto_use_output_mbuf(struct cryptop *crp, struct mbuf *m) _crypto_use_mbuf(&crp->crp_obuf, m); } +static __inline void +crypto_use_output_single_mbuf(struct cryptop *crp, struct mbuf *m) +{ + _crypto_use_single_mbuf(&crp->crp_obuf, m); +} + static __inline void crypto_use_output_vmpage(struct cryptop *crp, vm_page_t *pages, int len, int offset) diff --git a/sys/opencrypto/cryptosoft.c b/sys/opencrypto/cryptosoft.c index a09f085a1eef..4040a014881e 100644 --- a/sys/opencrypto/cryptosoft.c +++ b/sys/opencrypto/cryptosoft.c @@ -1105,6 +1105,7 @@ swcr_compdec(struct swcr_session *ses, struct cryptop *crp) if (result < crp->crp_payload_length) { switch (crp->crp_buf.cb_type) { case CRYPTO_BUF_MBUF: + case CRYPTO_BUF_SINGLE_MBUF: adj = result - crp->crp_payload_length; m_adj(crp->crp_buf.cb_mbuf, adj); break;