diff --git a/share/man/man9/sglist.9 b/share/man/man9/sglist.9 index 9ff875e5acba..b1d781b84d63 100644 --- a/share/man/man9/sglist.9 +++ b/share/man/man9/sglist.9 @@ -26,7 +26,7 @@ .\" .\" $FreeBSD$ .\" -.Dd June 28, 2019 +.Dd April 24, 2020 .Dt SGLIST 9 .Os .Sh NAME @@ -34,9 +34,8 @@ .Nm sglist_alloc , .Nm sglist_append , .Nm sglist_append_bio , -.Nm sglist_append_ext_pgs, -.Nm sglist_append_mb_ext_pgs, .Nm sglist_append_mbuf , +.Nm sglist_append_mbuf_epg, .Nm sglist_append_phys , .Nm sglist_append_sglist , .Nm sglist_append_uio , @@ -46,8 +45,7 @@ .Nm sglist_clone , .Nm sglist_consume_uio , .Nm sglist_count , -.Nm sglist_count_ext_pgs , -.Nm sglist_count_mb_ext_pgs , +.Nm sglist_count_mbuf_epg , .Nm sglist_count_vmpages , .Nm sglist_free , .Nm sglist_hold , @@ -68,9 +66,7 @@ .Ft int .Fn sglist_append_bio "struct sglist *sg" "struct bio *bp" .Ft int -.Fn sglist_append_ext_pgs "struct sglist *sg" "struct mbuf_ext_pgs *ext_pgs" "size_t offset" "size_t len" -.Ft int -.Fn sglist_append_mb_ext_pgs "struct sglist *sg" "struct mbuf *m" +.Fn sglist_append_mbuf_epg "struct sglist *sg" "struct mbuf *m" "size_t offset" "size_t len" .Ft int .Fn sglist_append_mbuf "struct sglist *sg" "struct mbuf *m" .Ft int @@ -92,9 +88,7 @@ .Ft int .Fn sglist_count "void *buf" "size_t len" .Ft int -.Fn sglist_count_ext_pgs "struct mbuf_ext_pgs *ext_pgs" "size_t offset" "size_t len" -.Ft int -.Fn sglist_count_mb_ext_pgs "struct mbuf *m" +.Fn sglist_count_mbuf_epg "struct mbuf *m" "size_t offset" "size_t len" .Ft int .Fn sglist_count_vmpages "vm_page_t *m" "size_t pgoff" "size_t len" .Ft void @@ -158,20 +152,15 @@ and is bytes long. .Pp The -.Nm sglist_count_ext_pgs +.Nm sglist_count_mbuf_epg function returns the number of scatter/gather list elements needed to describe -the unmapped external mbuf buffer -.Fa ext_pgs . +the external multipage mbuf buffer +.Fa m . The ranges start at an offset of .Fa offset relative to the start of the buffer and is .Fa len bytes long. -The -.Nm sglist_count_mb_ext_pgs -function returns the number of scatter/gather list elements needed to describe -the physical address ranges of a single unmapped mbuf -.Fa m . .Pp The .Nm sglist_count_vmpages @@ -265,9 +254,11 @@ to the scatter/gather list .Fa sg . .Pp The -.Nm sglist_append_ext_pgs -function appends the physical address ranges described by the unmapped -external mbuf buffer +.Nm sglist_append_mbuf_epg +function appends the physical address ranges described by the +external multipage +.Xr mbuf 9 +buffer .Fa ext_pgs to the scatter/gather list .Fa sg . @@ -278,17 +269,9 @@ within and continue for .Fa len bytes. -.Pp -The -.Nm sglist_append_mb_ext_pgs -function appends the physical address ranges described by the unmapped -mbuf -.Fa m -to the scatter/gather list -.Fa sg . Note that unlike .Nm sglist_append_mbuf , -.Nm sglist_append_mb_ext_pgs +.Nm sglist_append_mbuf_epg only adds ranges for a single mbuf, not an entire mbuf chain. .Pp diff --git a/sys/dev/cxgbe/crypto/t4_kern_tls.c b/sys/dev/cxgbe/crypto/t4_kern_tls.c index 1d44b58b0dd3..1cc3e6ad2f1d 100644 --- a/sys/dev/cxgbe/crypto/t4_kern_tls.c +++ b/sys/dev/cxgbe/crypto/t4_kern_tls.c @@ -1064,7 +1064,7 @@ ktls_wr_len(struct tlspcb *tlsp, struct mbuf *m, struct mbuf *m_tls, wr_len += roundup2(imm_len, 16); /* TLS record payload via DSGL. */ - *nsegsp = sglist_count_ext_pgs(m_tls, ext_pgs->hdr_len + offset, + *nsegsp = sglist_count_mbuf_epg(m_tls, ext_pgs->hdr_len + offset, plen - (ext_pgs->hdr_len + offset)); wr_len += ktls_sgl_size(*nsegsp); @@ -1799,7 +1799,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq, /* Recalculate 'nsegs' if cached value is not available. */ if (nsegs == 0) - nsegs = sglist_count_ext_pgs(m_tls, ext_pgs->hdr_len + + nsegs = sglist_count_mbuf_epg(m_tls, ext_pgs->hdr_len + offset, plen - (ext_pgs->hdr_len + offset)); /* Calculate the size of the TLS work request. */ @@ -2067,7 +2067,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq, /* SGL for record payload */ sglist_reset(txq->gl); - if (sglist_append_ext_pgs(txq->gl, m_tls, ext_pgs->hdr_len + offset, + if (sglist_append_mbuf_epg(txq->gl, m_tls, ext_pgs->hdr_len + offset, plen - (ext_pgs->hdr_len + offset)) != 0) { #ifdef INVARIANTS panic("%s: failed to append sglist", __func__); diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c index 02891dc17751..739de819b61f 100644 --- a/sys/dev/cxgbe/t4_sge.c +++ b/sys/dev/cxgbe/t4_sge.c @@ -2413,23 +2413,21 @@ m_advance(struct mbuf **pm, int *poffset, int len) static inline int count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_t *nextaddr) { - struct mbuf_ext_pgs *ext_pgs; vm_paddr_t paddr; int i, len, off, pglen, pgoff, seglen, segoff; int nsegs = 0; MBUF_EXT_PGS_ASSERT(m); - ext_pgs = &m->m_ext_pgs; off = mtod(m, vm_offset_t); len = m->m_len; off += skip; len -= skip; - if (ext_pgs->hdr_len != 0) { - if (off >= ext_pgs->hdr_len) { - off -= ext_pgs->hdr_len; + if (m->m_ext_pgs.hdr_len != 0) { + if (off >= m->m_ext_pgs.hdr_len) { + off -= m->m_ext_pgs.hdr_len; } else { - seglen = ext_pgs->hdr_len - off; + seglen = m->m_ext_pgs.hdr_len - off; segoff = off; seglen = min(seglen, len); off = 0; @@ -2441,9 +2439,9 @@ count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_t *nextaddr) *nextaddr = paddr + seglen; } } - pgoff = ext_pgs->first_pg_off; - for (i = 0; i < ext_pgs->npgs && len > 0; i++) { - pglen = mbuf_ext_pg_len(ext_pgs, i, pgoff); + pgoff = m->m_ext_pgs.first_pg_off; + for (i = 0; i < m->m_ext_pgs.npgs && len > 0; i++) { + pglen = mbuf_ext_pg_len(&m->m_ext_pgs, i, pgoff); if (off >= pglen) { off -= pglen; pgoff = 0; @@ -2461,7 +2459,7 @@ count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_t *nextaddr) pgoff = 0; }; if (len != 0) { - seglen = min(len, ext_pgs->trail_len - off); + seglen = min(len, m->m_ext_pgs.trail_len - off); len -= seglen; paddr = pmap_kextract((vm_offset_t)&m->m_epg_trail[off]); if (*nextaddr != paddr) @@ -5838,9 +5836,12 @@ write_ethofld_wr(struct cxgbe_rate_tag *cst, struct fw_eth_tx_eo_wr *wr, immhdrs -= m0->m_len; continue; } - - sglist_append(&sg, mtod(m0, char *) + immhdrs, - m0->m_len - immhdrs); + if (m0->m_flags & M_NOMAP) + sglist_append_mbuf_epg(&sg, m0, + mtod(m0, vm_offset_t), m0->m_len); + else + sglist_append(&sg, mtod(m0, char *) + immhdrs, + m0->m_len - immhdrs); immhdrs = 0; } MPASS(sg.sg_nseg == nsegs); diff --git a/sys/dev/cxgbe/tom/t4_cpl_io.c b/sys/dev/cxgbe/tom/t4_cpl_io.c index 043e6a111024..4f5506b551df 100644 --- a/sys/dev/cxgbe/tom/t4_cpl_io.c +++ b/sys/dev/cxgbe/tom/t4_cpl_io.c @@ -611,7 +611,8 @@ write_tx_sgl(void *dst, struct mbuf *start, struct mbuf *stop, int nsegs, int n) i = -1; for (m = start; m != stop; m = m->m_next) { if (m->m_flags & M_NOMAP) - rc = sglist_append_mb_ext_pgs(&sg, m); + rc = sglist_append_mbuf_epg(&sg, m, + mtod(m, vm_offset_t), m->m_len); else rc = sglist_append(&sg, mtod(m, void *), m->m_len); if (__predict_false(rc != 0)) @@ -742,7 +743,8 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) break; } #endif - n = sglist_count_mb_ext_pgs(m); + n = sglist_count_mbuf_epg(m, + mtod(m, vm_offset_t), m->m_len); } else n = sglist_count(mtod(m, void *), m->m_len); diff --git a/sys/kern/subr_bus_dma.c b/sys/kern/subr_bus_dma.c index bfdcc91ff21c..f47e37a9123a 100644 --- a/sys/kern/subr_bus_dma.c +++ b/sys/kern/subr_bus_dma.c @@ -116,14 +116,12 @@ _bus_dmamap_load_plist(bus_dma_tag_t dmat, bus_dmamap_t map, * Load an unmapped mbuf */ static int -_bus_dmamap_load_unmapped_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, +_bus_dmamap_load_mbuf_epg(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m, bus_dma_segment_t *segs, int *nsegs, int flags) { - struct mbuf_ext_pgs *ext_pgs; int error, i, off, len, pglen, pgoff, seglen, segoff; MBUF_EXT_PGS_ASSERT(m); - ext_pgs = &m->m_ext_pgs; len = m->m_len; error = 0; @@ -131,11 +129,11 @@ _bus_dmamap_load_unmapped_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, /* Skip over any data removed from the front. */ off = mtod(m, vm_offset_t); - if (ext_pgs->hdr_len != 0) { - if (off >= ext_pgs->hdr_len) { - off -= ext_pgs->hdr_len; + if (m->m_ext_pgs.hdr_len != 0) { + if (off >= m->m_ext_pgs.hdr_len) { + off -= m->m_ext_pgs.hdr_len; } else { - seglen = ext_pgs->hdr_len - off; + seglen = m->m_ext_pgs.hdr_len - off; segoff = off; seglen = min(seglen, len); off = 0; @@ -145,9 +143,9 @@ _bus_dmamap_load_unmapped_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, flags, segs, nsegs); } } - pgoff = ext_pgs->first_pg_off; - for (i = 0; i < ext_pgs->npgs && error == 0 && len > 0; i++) { - pglen = mbuf_ext_pg_len(ext_pgs, i, pgoff); + pgoff = m->m_ext_pgs.first_pg_off; + for (i = 0; i < m->m_ext_pgs.npgs && error == 0 && len > 0; i++) { + pglen = mbuf_ext_pg_len(&m->m_ext_pgs, i, pgoff); if (off >= pglen) { off -= pglen; pgoff = 0; @@ -163,9 +161,9 @@ _bus_dmamap_load_unmapped_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, pgoff = 0; }; if (len != 0 && error == 0) { - KASSERT((off + len) <= ext_pgs->trail_len, + KASSERT((off + len) <= m->m_ext_pgs.trail_len, ("off + len > trail (%d + %d > %d)", off, len, - ext_pgs->trail_len)); + m->m_ext_pgs.trail_len)); error = _bus_dmamap_load_buffer(dmat, map, &m->m_epg_trail[off], len, kernel_pmap, flags, segs, nsegs); @@ -187,7 +185,7 @@ _bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, for (m = m0; m != NULL && error == 0; m = m->m_next) { if (m->m_len > 0) { if ((m->m_flags & M_NOMAP) != 0) - error = _bus_dmamap_load_unmapped_mbuf_sg(dmat, + error = _bus_dmamap_load_mbuf_epg(dmat, map, m, segs, nsegs, flags); else error = _bus_dmamap_load_buffer(dmat, map, diff --git a/sys/kern/subr_sglist.c b/sys/kern/subr_sglist.c index 31967c5a9bcc..28d7bb19733e 100644 --- a/sys/kern/subr_sglist.c +++ b/sys/kern/subr_sglist.c @@ -223,9 +223,8 @@ sglist_count_vmpages(vm_page_t *m, size_t pgoff, size_t len) * describe an EXT_PGS buffer. */ int -sglist_count_ext_pgs(struct mbuf *m, size_t off, size_t len) +sglist_count_mbuf_epg(struct mbuf *m, size_t off, size_t len) { - struct mbuf_ext_pgs *ext_pgs = &m->m_ext_pgs; vm_paddr_t nextaddr, paddr; size_t seglen, segoff; int i, nsegs, pglen, pgoff; @@ -234,11 +233,11 @@ sglist_count_ext_pgs(struct mbuf *m, size_t off, size_t len) return (0); nsegs = 0; - if (ext_pgs->hdr_len != 0) { - if (off >= ext_pgs->hdr_len) { - off -= ext_pgs->hdr_len; + if (m->m_ext_pgs.hdr_len != 0) { + if (off >= m->m_ext_pgs.hdr_len) { + off -= m->m_ext_pgs.hdr_len; } else { - seglen = ext_pgs->hdr_len - off; + seglen = m->m_ext_pgs.hdr_len - off; segoff = off; seglen = MIN(seglen, len); off = 0; @@ -248,9 +247,9 @@ sglist_count_ext_pgs(struct mbuf *m, size_t off, size_t len) } } nextaddr = 0; - pgoff = ext_pgs->first_pg_off; - for (i = 0; i < ext_pgs->npgs && len > 0; i++) { - pglen = mbuf_ext_pg_len(ext_pgs, i, pgoff); + pgoff = m->m_ext_pgs.first_pg_off; + for (i = 0; i < m->m_ext_pgs.npgs && len > 0; i++) { + pglen = mbuf_ext_pg_len(&m->m_ext_pgs, i, pgoff); if (off >= pglen) { off -= pglen; pgoff = 0; @@ -268,7 +267,7 @@ sglist_count_ext_pgs(struct mbuf *m, size_t off, size_t len) pgoff = 0; }; if (len != 0) { - seglen = MIN(len, ext_pgs->trail_len - off); + seglen = MIN(len, m->m_ext_pgs.trail_len - off); len -= seglen; nsegs += sglist_count(&m->m_epg_trail[off], seglen); } @@ -276,18 +275,6 @@ sglist_count_ext_pgs(struct mbuf *m, size_t off, size_t len) return (nsegs); } -/* - * Determine the number of scatter/gather list elements needed to - * describe an EXT_PGS mbuf. - */ -int -sglist_count_mb_ext_pgs(struct mbuf *m) -{ - - MBUF_EXT_PGS_ASSERT(m); - return (sglist_count_ext_pgs(m, mtod(m, vm_offset_t), m->m_len)); -} - /* * Allocate a scatter/gather list along with 'nsegs' segments. The * 'mflags' parameters are the same as passed to malloc(9). The caller @@ -390,24 +377,25 @@ sglist_append_phys(struct sglist *sg, vm_paddr_t paddr, size_t len) } /* - * Append the segments to describe an EXT_PGS buffer to a - * scatter/gather list. If there are insufficient segments, then this - * fails with EFBIG. + * Append the segments of single multi-page mbuf. + * If there are insufficient segments, then this fails with EFBIG. */ int -sglist_append_ext_pgs(struct sglist *sg, struct mbuf *m, size_t off, size_t len) +sglist_append_mbuf_epg(struct sglist *sg, struct mbuf *m, size_t off, + size_t len) { - struct mbuf_ext_pgs *ext_pgs = &m->m_ext_pgs; size_t seglen, segoff; vm_paddr_t paddr; int error, i, pglen, pgoff; + MBUF_EXT_PGS_ASSERT(m); + error = 0; - if (ext_pgs->hdr_len != 0) { - if (off >= ext_pgs->hdr_len) { - off -= ext_pgs->hdr_len; + if (m->m_ext_pgs.hdr_len != 0) { + if (off >= m->m_ext_pgs.hdr_len) { + off -= m->m_ext_pgs.hdr_len; } else { - seglen = ext_pgs->hdr_len - off; + seglen = m->m_ext_pgs.hdr_len - off; segoff = off; seglen = MIN(seglen, len); off = 0; @@ -416,9 +404,9 @@ sglist_append_ext_pgs(struct sglist *sg, struct mbuf *m, size_t off, size_t len) &m->m_epg_hdr[segoff], seglen); } } - pgoff = ext_pgs->first_pg_off; - for (i = 0; i < ext_pgs->npgs && error == 0 && len > 0; i++) { - pglen = mbuf_ext_pg_len(ext_pgs, i, pgoff); + pgoff = m->m_ext_pgs.first_pg_off; + for (i = 0; i < m->m_ext_pgs.npgs && error == 0 && len > 0; i++) { + pglen = mbuf_ext_pg_len(&m->m_ext_pgs, i, pgoff); if (off >= pglen) { off -= pglen; pgoff = 0; @@ -434,7 +422,7 @@ sglist_append_ext_pgs(struct sglist *sg, struct mbuf *m, size_t off, size_t len) pgoff = 0; }; if (error == 0 && len > 0) { - seglen = MIN(len, ext_pgs->trail_len - off); + seglen = MIN(len, m->m_ext_pgs.trail_len - off); len -= seglen; error = sglist_append(sg, &m->m_epg_trail[off], seglen); @@ -444,20 +432,6 @@ sglist_append_ext_pgs(struct sglist *sg, struct mbuf *m, size_t off, size_t len) return (error); } -/* - * Append the segments to describe an EXT_PGS mbuf to a scatter/gather - * list. If there are insufficient segments, then this fails with - * EFBIG. - */ -int -sglist_append_mb_ext_pgs(struct sglist *sg, struct mbuf *m) -{ - - /* for now, all unmapped mbufs are assumed to be EXT_PGS */ - MBUF_EXT_PGS_ASSERT(m); - return (sglist_append_ext_pgs(sg, m, mtod(m, vm_offset_t), m->m_len)); -} - /* * Append the segments that describe a single mbuf chain to a * scatter/gather list. If there are insufficient segments, then this @@ -478,7 +452,8 @@ sglist_append_mbuf(struct sglist *sg, struct mbuf *m0) for (m = m0; m != NULL; m = m->m_next) { if (m->m_len > 0) { if ((m->m_flags & M_NOMAP) != 0) - error = sglist_append_mb_ext_pgs(sg, m); + error = sglist_append_mbuf_epg(sg, m, + mtod(m, vm_offset_t), m->m_len); else error = sglist_append(sg, m->m_data, m->m_len); diff --git a/sys/sys/sglist.h b/sys/sys/sglist.h index a1086786e30d..82a3ac6ed6ac 100644 --- a/sys/sys/sglist.h +++ b/sys/sys/sglist.h @@ -87,10 +87,9 @@ sglist_hold(struct sglist *sg) struct sglist *sglist_alloc(int nsegs, int mflags); int sglist_append(struct sglist *sg, void *buf, size_t len); int sglist_append_bio(struct sglist *sg, struct bio *bp); -int sglist_append_ext_pgs(struct sglist *sg, struct mbuf *m, size_t off, - size_t len); -int sglist_append_mb_ext_pgs(struct sglist *sg, struct mbuf *m); int sglist_append_mbuf(struct sglist *sg, struct mbuf *m0); +int sglist_append_mbuf_epg(struct sglist *sg, struct mbuf *m0, size_t off, + size_t len); int sglist_append_phys(struct sglist *sg, vm_paddr_t paddr, size_t len); int sglist_append_sglist(struct sglist *sg, struct sglist *source, @@ -104,8 +103,7 @@ struct sglist *sglist_build(void *buf, size_t len, int mflags); struct sglist *sglist_clone(struct sglist *sg, int mflags); int sglist_consume_uio(struct sglist *sg, struct uio *uio, size_t resid); int sglist_count(void *buf, size_t len); -int sglist_count_ext_pgs(struct mbuf *m, size_t off, size_t len); -int sglist_count_mb_ext_pgs(struct mbuf *m); +int sglist_count_mbuf_epg(struct mbuf *m, size_t off, size_t len); int sglist_count_vmpages(vm_page_t *m, size_t pgoff, size_t len); void sglist_free(struct sglist *sg); int sglist_join(struct sglist *first, struct sglist *second);