d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Continuation of multi page mbuf redesign from r359919.

Browse Source 
The following series of patches addresses three things:

Now that array of pages is embedded into mbuf, we no longer need
separate structure to pass around, so struct mbuf_ext_pgs is an
artifact of the first implementation. And struct mbuf_ext_pgs_data
is a crutch to accomodate the main idea r359919 with minimal churn.

Also, M_EXT of type EXT_PGS are just a synonym of M_NOMAP.

The namespace for the newfeature is somewhat inconsistent and
sometimes has a lengthy prefixes. In these patches we will
gradually bring the namespace to "m_epg" prefix for all mbuf
fields and most functions.

Step 1 of 4:

 o Anonymize mbuf_ext_pgs_data, embed in m_ext
 o Embed mbuf_ext_pgs
 o Start documenting all this entanglement

Reviewed by:	gallatin
Differential Revision:	https://reviews.freebsd.org/D24598
This commit is contained in:
Gleb Smirnoff 2020-05-02 22:39:26 +00:00
parent 4053f8ac4d
commit 0c1032665c
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=360569
12 changed files with 184 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
sys/dev/cxgbe/crypto/t4_kern_tls.c
View File

@ -906,7 +906,7 @@ ktls_tcp_payload_length(struct tlspcb *tlsp, struct mbuf *m_tls)
MBUF_EXT_PGS_ASSERT(m_tls);
ext_pgs = &m_tls->m_ext_pgs;
hdr = (void *)ext_pgs->m_epg_hdr;
hdr = (void *)m_tls->m_epg_hdr;
plen = ntohs(hdr->tls_length);
/*
@ -962,7 +962,7 @@ ktls_payload_offset(struct tlspcb *tlsp, struct mbuf *m_tls)
MBUF_EXT_PGS_ASSERT(m_tls);
ext_pgs = &m_tls->m_ext_pgs;
hdr = (void *)ext_pgs->m_epg_hdr;
hdr = (void *)m_tls->m_epg_hdr;
plen = ntohs(hdr->tls_length);
#ifdef INVARIANTS
mlen = mtod(m_tls, vm_offset_t) + m_tls->m_len;
@ -1040,7 +1040,7 @@ ktls_wr_len(struct tlspcb *tlsp, struct mbuf *m, struct mbuf *m_tls,
return (wr_len);
}
hdr = (void *)ext_pgs->m_epg_hdr;
hdr = (void *)m_tls->m_epg_hdr;
plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - ext_pgs->trail_len;
if (tlen < plen) {
plen = tlen;
@ -1064,7 +1064,7 @@ ktls_wr_len(struct tlspcb *tlsp, struct mbuf *m, struct mbuf *m_tls,
wr_len += roundup2(imm_len, 16);
/* TLS record payload via DSGL. */
*nsegsp = sglist_count_ext_pgs(ext_pgs, ext_pgs->hdr_len + offset,
*nsegsp = sglist_count_ext_pgs(m_tls, ext_pgs->hdr_len + offset,
plen - (ext_pgs->hdr_len + offset));
wr_len += ktls_sgl_size(*nsegsp);
@ -1543,7 +1543,7 @@ ktls_write_tunnel_packet(struct sge_txq *txq, void *dst, struct mbuf *m,
(m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp)));
/* Copy the subset of the TLS header requested. */
copy_to_txd(&txq->eq, (char *)ext_pgs->m_epg_hdr +
copy_to_txd(&txq->eq, (char *)m_tls->m_epg_hdr +
mtod(m_tls, vm_offset_t), &out, m_tls->m_len);
txq->imm_wrs++;
@ -1604,7 +1604,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
/* Locate the TLS header. */
MBUF_EXT_PGS_ASSERT(m_tls);
ext_pgs = &m_tls->m_ext_pgs;
hdr = (void *)ext_pgs->m_epg_hdr;
hdr = (void *)m_tls->m_epg_hdr;
plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - ext_pgs->trail_len;
/* Determine how much of the TLS record to send. */
@ -1799,7 +1799,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
/* Recalculate 'nsegs' if cached value is not available. */
if (nsegs == 0)
nsegs = sglist_count_ext_pgs(ext_pgs, ext_pgs->hdr_len +
nsegs = sglist_count_ext_pgs(m_tls, ext_pgs->hdr_len +
offset, plen - (ext_pgs->hdr_len + offset));
/* Calculate the size of the TLS work request. */
@ -2031,7 +2031,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
/* Populate the TLS header */
out = (void *)(tx_data + 1);
if (offset == 0) {
memcpy(out, ext_pgs->m_epg_hdr, ext_pgs->hdr_len);
memcpy(out, m_tls->m_epg_hdr, ext_pgs->hdr_len);
out += ext_pgs->hdr_len;
}
@ -2067,7 +2067,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
/* SGL for record payload */
sglist_reset(txq->gl);
if (sglist_append_ext_pgs(txq->gl, ext_pgs, ext_pgs->hdr_len + offset,
if (sglist_append_ext_pgs(txq->gl, m_tls, ext_pgs->hdr_len + offset,
plen - (ext_pgs->hdr_len + offset)) != 0) {
#ifdef INVARIANTS
panic("%s: failed to append sglist", __func__);

6
sys/dev/cxgbe/t4_sge.c
View File

@ -2435,7 +2435,7 @@ count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_t *nextaddr)
off = 0;
len -= seglen;
paddr = pmap_kextract(
(vm_offset_t)&ext_pgs->m_epg_hdr[segoff]);
(vm_offset_t)&m->m_epg_hdr[segoff]);
if (*nextaddr != paddr)
nsegs++;
*nextaddr = paddr + seglen;
@ -2454,7 +2454,7 @@ count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_t *nextaddr)
off = 0;
seglen = min(seglen, len);
len -= seglen;
paddr = ext_pgs->m_epg_pa[i] + segoff;
paddr = m->m_epg_pa[i] + segoff;
if (*nextaddr != paddr)
nsegs++;
*nextaddr = paddr + seglen;
@ -2463,7 +2463,7 @@ count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_t *nextaddr)
if (len != 0) {
seglen = min(len, ext_pgs->trail_len - off);
len -= seglen;
paddr = pmap_kextract((vm_offset_t)&ext_pgs->m_epg_trail[off]);
paddr = pmap_kextract((vm_offset_t)&m->m_epg_trail[off]);
if (*nextaddr != paddr)
nsegs++;
*nextaddr = paddr + seglen;

4
sys/dev/cxgbe/tom/t4_cpl_io.c
View File

@ -1935,7 +1935,7 @@ aiotx_free_pgs(struct mbuf *m)
#endif
for (int i = 0; i < ext_pgs->npgs; i++) {
pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
vm_page_unwire(pg, PQ_ACTIVE);
}
@ -2003,7 +2003,7 @@ alloc_aiotx_mbuf(struct kaiocb *job, int len)
(npages - 2) * PAGE_SIZE;
}
for (i = 0; i < npages; i++)
ext_pgs->m_epg_pa[i] = VM_PAGE_TO_PHYS(pgs[i]);
m->m_epg_pa[i] = VM_PAGE_TO_PHYS(pgs[i]);
m->m_len = mlen;
m->m_ext.ext_size = npages * PAGE_SIZE;

49
sys/dev/cxgbe/tom/t4_tls.c
View File

@ -1623,26 +1623,24 @@ t4_push_tls_records(struct adapter *sc, struct toepcb *toep, int drop)
#ifdef KERN_TLS
static int
count_ext_pgs_segs(struct mbuf_ext_pgs *ext_pgs,
struct mbuf_ext_pgs_data *ext_pgs_data)
count_ext_pgs_segs(struct mbuf *m)
{
vm_paddr_t nextpa;
u_int i, nsegs;
MPASS(ext_pgs->npgs > 0);
MPASS(m->m_ext_pgs.npgs > 0);
nsegs = 1;
nextpa = ext_pgs_data->pa[0] + PAGE_SIZE;
for (i = 1; i < ext_pgs->npgs; i++) {
if (nextpa != ext_pgs_data->pa[i])
nextpa = m->m_epg_pa[0] + PAGE_SIZE;
for (i = 1; i < m->m_ext_pgs.npgs; i++) {
if (nextpa != m->m_epg_pa[i])
nsegs++;
nextpa = ext_pgs_data->pa[i] + PAGE_SIZE;
nextpa = m->m_epg_pa[i] + PAGE_SIZE;
}
return (nsegs);
}
static void
write_ktlstx_sgl(void *dst, struct mbuf_ext_pgs *ext_pgs,
struct mbuf_ext_pgs_data *ext_pgs_data, int nsegs)
write_ktlstx_sgl(void *dst, struct mbuf *m, int nsegs)
{
struct ulptx_sgl *usgl = dst;
vm_paddr_t pa;
@ -1655,15 +1653,15 @@ write_ktlstx_sgl(void *dst, struct mbuf_ext_pgs *ext_pgs,
V_ULPTX_NSGE(nsegs));
/* Figure out the first S/G length. */
pa = ext_pgs_data->pa[0] + ext_pgs->first_pg_off;
pa = m->m_epg_pa[0] + m->m_ext_pgs.first_pg_off;
usgl->addr0 = htobe64(pa);
len = mbuf_ext_pg_len(ext_pgs, 0, ext_pgs->first_pg_off);
len = mbuf_ext_pg_len(&m->m_ext_pgs, 0, m->m_ext_pgs.first_pg_off);
pa += len;
for (i = 1; i < ext_pgs->npgs; i++) {
if (ext_pgs_data->pa[i] != pa)
for (i = 1; i < m->m_ext_pgs.npgs; i++) {
if (m->m_epg_pa[i] != pa)
break;
len += mbuf_ext_pg_len(ext_pgs, i, 0);
pa += mbuf_ext_pg_len(ext_pgs, i, 0);
len += mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
pa += mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
}
usgl->len0 = htobe32(len);
#ifdef INVARIANTS
@ -1671,21 +1669,21 @@ write_ktlstx_sgl(void *dst, struct mbuf_ext_pgs *ext_pgs,
#endif
j = -1;
for (; i < ext_pgs->npgs; i++) {
if (j == -1 || ext_pgs_data->pa[i] != pa) {
for (; i < m->m_ext_pgs.npgs; i++) {
if (j == -1 || m->m_epg_pa[i] != pa) {
if (j >= 0)
usgl->sge[j / 2].len[j & 1] = htobe32(len);
j++;
#ifdef INVARIANTS
nsegs--;
#endif
pa = ext_pgs_data->pa[i];
pa = m->m_epg_pa[i];
usgl->sge[j / 2].addr[j & 1] = htobe64(pa);
len = mbuf_ext_pg_len(ext_pgs, i, 0);
len = mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
pa += len;
} else {
len += mbuf_ext_pg_len(ext_pgs, i, 0);
pa += mbuf_ext_pg_len(ext_pgs, i, 0);
len += mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
pa += mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
}
}
if (j >= 0) {
@ -1694,8 +1692,7 @@ write_ktlstx_sgl(void *dst, struct mbuf_ext_pgs *ext_pgs,
if ((j & 1) == 0)
usgl->sge[j / 2].len[1] = htobe32(0);
}
KASSERT(nsegs == 0, ("%s: nsegs %d, ext_pgs %p", __func__, nsegs,
ext_pgs));
KASSERT(nsegs == 0, ("%s: nsegs %d, m %p", __func__, nsegs, m));
}
/*
@ -1813,8 +1810,7 @@ t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop)
wr_len += AES_BLOCK_LEN;
/* Account for SGL in work request length. */
nsegs = count_ext_pgs_segs(&m->m_ext_pgs,
&m->m_ext.ext_pgs);
nsegs = count_ext_pgs_segs(m);
wr_len += sizeof(struct ulptx_sgl) +
((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
@ -1892,8 +1888,7 @@ t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop)
memcpy(buf, thdr + 1, toep->tls.iv_len);
buf += AES_BLOCK_LEN;
write_ktlstx_sgl(buf, &m->m_ext_pgs, &m->m_ext.ext_pgs,
nsegs);
write_ktlstx_sgl(buf, m, nsegs);
KASSERT(toep->tx_credits >= credits,
("%s: not enough credits", __func__));

18
sys/kern/kern_mbuf.c
View File

@ -311,9 +311,6 @@ static void mb_reclaim(uma_zone_t, int);
/* Ensure that MSIZE is a power of 2. */
CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
_Static_assert(offsetof(struct mbuf, m_ext) ==
offsetof(struct mbuf, m_ext_pgs.m_ext),
"m_ext offset mismatch between mbuf and ext_pgs");
_Static_assert(sizeof(struct mbuf) <= MSIZE,
"size of mbuf exceeds MSIZE");
/*
@ -984,7 +981,7 @@ _mb_unmapped_to_ext(struct mbuf *m)
goto fail;
m_new->m_len = seglen;
prev = top = m_new;
memcpy(mtod(m_new, void *), &ext_pgs->m_epg_hdr[segoff],
memcpy(mtod(m_new, void *), &m->m_epg_hdr[segoff],
seglen);
}
}
@ -1002,7 +999,7 @@ _mb_unmapped_to_ext(struct mbuf *m)
seglen = min(seglen, len);
len -= seglen;
pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
m_new = m_get(M_NOWAIT, MT_DATA);
if (m_new == NULL)
goto fail;
@ -1036,7 +1033,7 @@ _mb_unmapped_to_ext(struct mbuf *m)
else
prev->m_next = m_new;
m_new->m_len = len;
memcpy(mtod(m_new, void *), &ext_pgs->m_epg_trail[off], len);
memcpy(mtod(m_new, void *), &m->m_epg_trail[off], len);
}
if (ref_inc != 0) {
@ -1154,8 +1151,9 @@ mb_alloc_ext_pgs(int how, m_ext_free_t ext_free)
#ifdef INVARIANT_SUPPORT
void
mb_ext_pgs_check(struct mbuf_ext_pgs *ext_pgs)
mb_ext_pgs_check(struct mbuf *m)
{
struct mbuf_ext_pgs *ext_pgs = &m->m_ext_pgs;
/*
* NB: This expects a non-empty buffer (npgs > 0 and
@ -1163,7 +1161,7 @@ mb_ext_pgs_check(struct mbuf_ext_pgs *ext_pgs)
*/
KASSERT(ext_pgs->npgs > 0,
("ext_pgs with no valid pages: %p", ext_pgs));
KASSERT(ext_pgs->npgs <= nitems(ext_pgs->m_epg_pa),
KASSERT(ext_pgs->npgs <= nitems(m->m_epg_pa),
("ext_pgs with too many pages: %p", ext_pgs));
KASSERT(ext_pgs->nrdy <= ext_pgs->npgs,
("ext_pgs with too many ready pages: %p", ext_pgs));
@ -1178,9 +1176,9 @@ mb_ext_pgs_check(struct mbuf_ext_pgs *ext_pgs)
PAGE_SIZE, ("ext_pgs with single page too large: %p",
ext_pgs));
}
KASSERT(ext_pgs->hdr_len <= sizeof(ext_pgs->m_epg_hdr),
KASSERT(ext_pgs->hdr_len <= sizeof(m->m_epg_hdr),
("ext_pgs with too large header length: %p", ext_pgs));
KASSERT(ext_pgs->trail_len <= sizeof(ext_pgs->m_epg_trail),
KASSERT(ext_pgs->trail_len <= sizeof(m->m_epg_trail),
("ext_pgs with too large header length: %p", ext_pgs));
}
#endif

6
sys/kern/kern_sendfile.c
View File

@ -203,7 +203,7 @@ sendfile_free_mext_pg(struct mbuf *m)
for (i = 0; i < ext_pgs->npgs; i++) {
if (cache_last && i == ext_pgs->npgs - 1)
flags = 0;
pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
vm_page_release(pg, flags);
}
@ -1046,11 +1046,11 @@ vn_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
ext_pgs->nrdy++;
}
ext_pgs->m_epg_pa[ext_pgs_idx] = VM_PAGE_TO_PHYS(pga);
m0->m_epg_pa[ext_pgs_idx] = VM_PAGE_TO_PHYS(pga);
ext_pgs->npgs++;
xfs = xfsize(i, npages, off, space);
ext_pgs->last_pg_len = xfs;
MBUF_EXT_PGS_ASSERT_SANITY(ext_pgs);
MBUF_EXT_PGS_ASSERT_SANITY(m0);
mtail->m_len += xfs;
mtail->m_ext.ext_size += PAGE_SIZE;
continue;

6
sys/kern/subr_bus_dma.c
View File

@ -141,7 +141,7 @@ _bus_dmamap_load_unmapped_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
off = 0;
len -= seglen;
error = _bus_dmamap_load_buffer(dmat, map,
&ext_pgs->m_epg_hdr[segoff], seglen, kernel_pmap,
&m->m_epg_hdr[segoff], seglen, kernel_pmap,
flags, segs, nsegs);
}
}
@ -159,7 +159,7 @@ _bus_dmamap_load_unmapped_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
seglen = min(seglen, len);
len -= seglen;
error = _bus_dmamap_load_phys(dmat, map,
ext_pgs->m_epg_pa[i] + segoff, seglen, flags, segs, nsegs);
m->m_epg_pa[i] + segoff, seglen, flags, segs, nsegs);
pgoff = 0;
};
if (len != 0 && error == 0) {
@ -167,7 +167,7 @@ _bus_dmamap_load_unmapped_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
("off + len > trail (%d + %d > %d)", off, len,
ext_pgs->trail_len));
error = _bus_dmamap_load_buffer(dmat, map,
&ext_pgs->m_epg_trail[off], len, kernel_pmap, flags, segs,
&m->m_epg_trail[off], len, kernel_pmap, flags, segs,
nsegs);
}
return (error);

25
sys/kern/subr_sglist.c
View File

@ -223,8 +223,9 @@ sglist_count_vmpages(vm_page_t *m, size_t pgoff, size_t len)
* describe an EXT_PGS buffer.
*/
int
sglist_count_ext_pgs(struct mbuf_ext_pgs *ext_pgs, size_t off, size_t len)
sglist_count_ext_pgs(struct mbuf *m, size_t off, size_t len)
{
struct mbuf_ext_pgs *ext_pgs = &m->m_ext_pgs;
vm_paddr_t nextaddr, paddr;
size_t seglen, segoff;
int i, nsegs, pglen, pgoff;
@ -242,7 +243,7 @@ sglist_count_ext_pgs(struct mbuf_ext_pgs *ext_pgs, size_t off, size_t len)
seglen = MIN(seglen, len);
off = 0;
len -= seglen;
nsegs += sglist_count(&ext_pgs->m_epg_hdr[segoff],
nsegs += sglist_count(&m->m_epg_hdr[segoff],
seglen);
}
}
@ -260,7 +261,7 @@ sglist_count_ext_pgs(struct mbuf_ext_pgs *ext_pgs, size_t off, size_t len)
off = 0;
seglen = MIN(seglen, len);
len -= seglen;
paddr = ext_pgs->m_epg_pa[i] + segoff;
paddr = m->m_epg_pa[i] + segoff;
if (paddr != nextaddr)
nsegs++;
nextaddr = paddr + seglen;
@ -269,7 +270,7 @@ sglist_count_ext_pgs(struct mbuf_ext_pgs *ext_pgs, size_t off, size_t len)
if (len != 0) {
seglen = MIN(len, ext_pgs->trail_len - off);
len -= seglen;
nsegs += sglist_count(&ext_pgs->m_epg_trail[off], seglen);
nsegs += sglist_count(&m->m_epg_trail[off], seglen);
}
KASSERT(len == 0, ("len != 0"));
return (nsegs);
@ -284,8 +285,7 @@ sglist_count_mb_ext_pgs(struct mbuf *m)
{
MBUF_EXT_PGS_ASSERT(m);
return (sglist_count_ext_pgs(&m->m_ext_pgs, mtod(m, vm_offset_t),
m->m_len));
return (sglist_count_ext_pgs(m, mtod(m, vm_offset_t), m->m_len));
}
/*
@ -395,9 +395,9 @@ sglist_append_phys(struct sglist *sg, vm_paddr_t paddr, size_t len)
* fails with EFBIG.
*/
int
sglist_append_ext_pgs(struct sglist *sg, struct mbuf_ext_pgs *ext_pgs,
size_t off, size_t len)
sglist_append_ext_pgs(struct sglist *sg, struct mbuf *m, size_t off, size_t len)
{
struct mbuf_ext_pgs *ext_pgs = &m->m_ext_pgs;
size_t seglen, segoff;
vm_paddr_t paddr;
int error, i, pglen, pgoff;
@ -413,7 +413,7 @@ sglist_append_ext_pgs(struct sglist *sg, struct mbuf_ext_pgs *ext_pgs,
off = 0;
len -= seglen;
error = sglist_append(sg,
&ext_pgs->m_epg_hdr[segoff], seglen);
&m->m_epg_hdr[segoff], seglen);
}
}
pgoff = ext_pgs->first_pg_off;
@ -429,7 +429,7 @@ sglist_append_ext_pgs(struct sglist *sg, struct mbuf_ext_pgs *ext_pgs,
off = 0;
seglen = MIN(seglen, len);
len -= seglen;
paddr = ext_pgs->m_epg_pa[i] + segoff;
paddr = m->m_epg_pa[i] + segoff;
error = sglist_append_phys(sg, paddr, seglen);
pgoff = 0;
};
@ -437,7 +437,7 @@ sglist_append_ext_pgs(struct sglist *sg, struct mbuf_ext_pgs *ext_pgs,
seglen = MIN(len, ext_pgs->trail_len - off);
len -= seglen;
error = sglist_append(sg,
&ext_pgs->m_epg_trail[off], seglen);
&m->m_epg_trail[off], seglen);
}
if (error == 0)
KASSERT(len == 0, ("len != 0"));
@ -455,8 +455,7 @@ sglist_append_mb_ext_pgs(struct sglist *sg, struct mbuf *m)
/* for now, all unmapped mbufs are assumed to be EXT_PGS */
MBUF_EXT_PGS_ASSERT(m);
return (sglist_append_ext_pgs(sg, &m->m_ext_pgs,
mtod(m, vm_offset_t), m->m_len));
return (sglist_append_ext_pgs(sg, m, mtod(m, vm_offset_t), m->m_len));
}
/*

12
sys/kern/uipc_ktls.c
View File

@ -1374,7 +1374,7 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls, int *enq_cnt,
m->m_len += pgs->hdr_len + pgs->trail_len;
/* Populate the TLS header. */
tlshdr = (void *)pgs->m_epg_hdr;
tlshdr = (void *)m->m_epg_hdr;
tlshdr->tls_vmajor = tls->params.tls_vmajor;
/*
@ -1387,7 +1387,7 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls, int *enq_cnt,
tlshdr->tls_type = TLS_RLTYPE_APP;
/* save the real record type for later */
pgs->record_type = record_type;
pgs->m_epg_trail[0] = record_type;
m->m_epg_trail[0] = record_type;
} else {
tlshdr->tls_vminor = tls->params.tls_vminor;
tlshdr->tls_type = record_type;
@ -1552,7 +1552,7 @@ ktls_encrypt(struct mbuf_ext_pgs *pgs)
len = mbuf_ext_pg_len(pgs, i, off);
src_iov[i].iov_len = len;
src_iov[i].iov_base =
(char *)(void *)PHYS_TO_DMAP(pgs->m_epg_pa[i]) +
(char *)(void *)PHYS_TO_DMAP(m->m_epg_pa[i]) +
off;
if (is_anon) {
@ -1576,8 +1576,8 @@ ktls_encrypt(struct mbuf_ext_pgs *pgs)
npages += i;
error = (*tls->sw_encrypt)(tls,
(const struct tls_record_layer *)pgs->m_epg_hdr,
pgs->m_epg_trail, src_iov, dst_iov, i, pgs->seqno,
(const struct tls_record_layer *)m->m_epg_hdr,
m->m_epg_trail, src_iov, dst_iov, i, pgs->seqno,
pgs->record_type);
if (error) {
counter_u64_add(ktls_offload_failed_crypto, 1);
@ -1595,7 +1595,7 @@ ktls_encrypt(struct mbuf_ext_pgs *pgs)
/* Replace them with the new pages. */
for (i = 0; i < pgs->npgs; i++)
pgs->m_epg_pa[i] = parray[i];
m->m_epg_pa[i] = parray[i];
/* Use the basic free routine. */
m->m_ext.ext_free = mb_free_mext_pgs;

47
sys/kern/uipc_mbuf.c
View File

@ -163,11 +163,11 @@ CTASSERT(offsetof(struct mbuf, m_pktdat) % 8 == 0);
#if defined(__LP64__)
CTASSERT(offsetof(struct mbuf, m_dat) == 32);
CTASSERT(sizeof(struct pkthdr) == 56);
CTASSERT(sizeof(struct m_ext) == 168);
CTASSERT(sizeof(struct m_ext) == 160);
#else
CTASSERT(offsetof(struct mbuf, m_dat) == 24);
CTASSERT(sizeof(struct pkthdr) == 48);
CTASSERT(sizeof(struct m_ext) == 184);
CTASSERT(sizeof(struct m_ext) == 180);
#endif
/*
@ -195,19 +195,30 @@ mb_dupcl(struct mbuf *n, struct mbuf *m)
KASSERT(!(n->m_flags & M_EXT), ("%s: M_EXT set on %p", __func__, n));
/*
* Cache access optimization. For most kinds of external
* storage we don't need full copy of m_ext, since the
* holder of the 'ext_count' is responsible to carry the
* free routine and its arguments. Exclusion is EXT_EXTREF,
* where 'ext_cnt' doesn't point into mbuf at all.
* Cache access optimization.
*
* o Regular M_EXT storage doesn't need full copy of m_ext, since
* the holder of the 'ext_count' is responsible to carry the free
* routine and its arguments.
* o EXT_PGS data is split between main part of mbuf and m_ext, the
* main part is copied in full, the m_ext part is similar to M_EXT.
* o EXT_EXTREF, where 'ext_cnt' doesn't point into mbuf at all, is
* special - it needs full copy of m_ext into each mbuf, since any
* copy could end up as the last to free.
*/
if (m->m_ext.ext_type == EXT_EXTREF)
bcopy(&m->m_ext, &n->m_ext, sizeof(struct m_ext));
else if (m->m_ext.ext_type == EXT_PGS)
switch (m->m_ext.ext_type) {
case EXT_PGS:
bcopy(&m->m_ext, &n->m_ext, m_epg_copylen);
bcopy(&m->m_ext_pgs, &n->m_ext_pgs,
sizeof(struct mbuf_ext_pgs));
else
break;
case EXT_EXTREF:
bcopy(&m->m_ext, &n->m_ext, sizeof(struct m_ext));
break;
default:
bcopy(&m->m_ext, &n->m_ext, m_ext_copylen);
}
n->m_flags |= M_EXT;
n->m_flags |= m->m_flags & (M_RDONLY | M_NOMAP);
@ -1623,7 +1634,7 @@ mb_free_mext_pgs(struct mbuf *m)
MBUF_EXT_PGS_ASSERT(m);
ext_pgs = &m->m_ext_pgs;
for (int i = 0; i < ext_pgs->npgs; i++) {
pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
vm_page_unwire_noq(pg);
vm_page_free(pg);
}
@ -1681,11 +1692,11 @@ m_uiotombuf_nomap(struct uio *uio, int how, int len, int maxseg, int flags)
}
}
pg_array[i]->flags &= ~PG_ZERO;
pgs->m_epg_pa[i] = VM_PAGE_TO_PHYS(pg_array[i]);
mb->m_epg_pa[i] = VM_PAGE_TO_PHYS(pg_array[i]);
pgs->npgs++;
}
pgs->last_pg_len = length - PAGE_SIZE * (pgs->npgs - 1);
MBUF_EXT_PGS_ASSERT_SANITY(pgs);
MBUF_EXT_PGS_ASSERT_SANITY(mb);
total -= length;
error = uiomove_fromphys(pg_array, 0, length, uio);
if (error != 0)
@ -1788,7 +1799,8 @@ m_unmappedtouio(const struct mbuf *m, int m_off, struct uio *uio, int len)
seglen = min(seglen, len);
off = 0;
len -= seglen;
error = uiomove(&ext_pgs->m_epg_hdr[segoff], seglen, uio);
error = uiomove(__DECONST(void *,
&m->m_epg_hdr[segoff]), seglen, uio);
}
}
pgoff = ext_pgs->first_pg_off;
@ -1804,7 +1816,7 @@ m_unmappedtouio(const struct mbuf *m, int m_off, struct uio *uio, int len)
off = 0;
seglen = min(seglen, len);
len -= seglen;
pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
error = uiomove_fromphys(&pg, segoff, seglen, uio);
pgoff = 0;
};
@ -1812,7 +1824,8 @@ m_unmappedtouio(const struct mbuf *m, int m_off, struct uio *uio, int len)
KASSERT((off + len) <= ext_pgs->trail_len,
("off + len > trail (%d + %d > %d, m_off = %d)", off, len,
ext_pgs->trail_len, m_off));
error = uiomove(&ext_pgs->m_epg_trail[off], len, uio);
error = uiomove(__DECONST(void *, &m->m_epg_trail[off]),
len, uio);
}
return (error);
}

140
sys/sys/mbuf.h
View File

@ -231,13 +231,6 @@ struct pkthdr {
#define MBUF_PEXT_FLAG_ANON 1 /* Data can be encrypted in place. */
struct mbuf_ext_pgs_data {
vm_paddr_t pa[MBUF_PEXT_MAX_PGS]; /* phys addrs of pgs */
char trail[MBUF_PEXT_TRAIL_LEN]; /* TLS trailer */
char hdr[MBUF_PEXT_HDR_LEN]; /* TLS header */
};
struct ktls_session;
struct socket;
@ -266,49 +259,49 @@ struct m_ext {
uint32_t ext_size; /* size of buffer, for ext_free */
uint32_t ext_type:8, /* type of external storage */
ext_flags:24; /* external storage mbuf flags */
char *ext_buf; /* start of buffer */
/*
* Fields below store the free context for the external storage.
* They are valid only in the refcount carrying mbuf, the one with
* EXT_FLAG_EMBREF flag, with exclusion for EXT_EXTREF type, where
* the free context is copied into all mbufs that use same external
* storage.
*/
#define m_ext_copylen offsetof(struct m_ext, ext_free)
m_ext_free_t *ext_free; /* free routine if not the usual */
void *ext_arg1; /* optional argument pointer */
union {
void *ext_arg2; /* optional argument pointer */
struct mbuf_ext_pgs_data ext_pgs;
struct {
/*
* Regular M_EXT mbuf:
* o ext_buf always points to the external buffer.
* o ext_free (below) and two optional arguments
* ext_arg1 and ext_arg2 store the free context for
* the external storage. They are set only in the
* refcount carrying mbuf, the one with
* EXT_FLAG_EMBREF flag, with exclusion for
* EXT_EXTREF type, where the free context is copied
* into all mbufs that use same external storage.
*/
char *ext_buf; /* start of buffer */
#define m_ext_copylen offsetof(struct m_ext, ext_arg2)
void *ext_arg2;
};
struct {
/*
* Multi-page M_EXTPG mbuf:
* o extpg_pa - page vector.
* o extpg_trail and extpg_hdr - TLS trailer and
* header.
* Uses ext_free and may also use ext_arg1.
*/
vm_paddr_t extpg_pa[MBUF_PEXT_MAX_PGS];
char extpg_trail[MBUF_PEXT_TRAIL_LEN];
char extpg_hdr[MBUF_PEXT_HDR_LEN];
/* Pretend these 3 fields are part of mbuf itself. */
#define m_epg_pa m_ext.extpg_pa
#define m_epg_trail m_ext.extpg_trail
#define m_epg_hdr m_ext.extpg_hdr
#define m_epg_copylen offsetof(struct m_ext, ext_free)
};
};
/*
* Free method and optional argument pointer, both
* used by M_EXT and M_EXTPG.
*/
m_ext_free_t *ext_free;
void *ext_arg1;
};
struct mbuf_ext_pgs {
uint8_t npgs; /* Number of attached pages */
uint8_t nrdy; /* Pages with I/O pending */
uint8_t hdr_len; /* TLS header length */
uint8_t trail_len; /* TLS trailer length */
uint16_t first_pg_off; /* Offset into 1st page */
uint16_t last_pg_len; /* Length of last page */
uint8_t flags; /* Flags */
uint8_t record_type;
uint8_t spare[2];
int enc_cnt;
struct ktls_session *tls; /* TLS session */
struct socket *so;
uint64_t seqno;
struct mbuf *mbuf;
STAILQ_ENTRY(mbuf_ext_pgs) stailq;
#if !defined(__LP64__)
uint8_t pad[8]; /* pad to size of pkthdr */
#endif
struct m_ext m_ext;
};
#define m_epg_hdr m_ext.ext_pgs.hdr
#define m_epg_trail m_ext.ext_pgs.trail
#define m_epg_pa m_ext.ext_pgs.pa
/*
* The core of the mbuf object along with some shortcut defines for practical
* purposes.
@ -347,15 +340,48 @@ struct mbuf {
* order to support future work on variable-size mbufs.
*/
union {
union {
struct {
struct pkthdr m_pkthdr; /* M_PKTHDR set */
union {
struct m_ext m_ext; /* M_EXT set */
char m_pktdat[0];
};
struct {
union {
/* M_PKTHDR set. */
struct pkthdr m_pkthdr;
/* M_EXTPG set.
* Multi-page M_EXTPG mbuf has its meta data
* split between the mbuf_ext_pgs structure
* and m_ext. It carries vector of pages,
* optional header and trailer char vectors
* and pointers to socket/TLS data.
*/
struct mbuf_ext_pgs {
/* Overall count of pages and count of
* pages with I/O pending. */
uint8_t npgs;
uint8_t nrdy;
/* TLS header and trailer lengths.
* The data itself resides in m_ext. */
uint8_t hdr_len;
uint8_t trail_len;
/* Offset into 1st page and lenght of
* data in the last page. */
uint16_t first_pg_off;
uint16_t last_pg_len;
uint8_t flags;
uint8_t record_type;
uint8_t spare[2];
int enc_cnt;
struct ktls_session *tls;
struct socket *so;
uint64_t seqno;
struct mbuf *mbuf;
STAILQ_ENTRY(mbuf_ext_pgs) stailq;
} m_ext_pgs;
};
union {
/* M_EXT or M_EXTPG set. */
struct m_ext m_ext;
/* M_PKTHDR set, neither M_EXT nor M_EXTPG. */
char m_pktdat[0];
};
struct mbuf_ext_pgs m_ext_pgs;
};
char m_dat[0]; /* !M_PKTHDR, !M_EXT */
};
@ -375,12 +401,12 @@ mbuf_ext_pg_len(struct mbuf_ext_pgs *ext_pgs, int pidx, int pgoff)
}
#ifdef INVARIANT_SUPPORT
void mb_ext_pgs_check(struct mbuf_ext_pgs *ext_pgs);
void mb_ext_pgs_check(struct mbuf *m);
#endif
#ifdef INVARIANTS
#define MBUF_EXT_PGS_ASSERT_SANITY(ext_pgs) mb_ext_pgs_check((ext_pgs))
#define MBUF_EXT_PGS_ASSERT_SANITY(m) mb_ext_pgs_check((m))
#else
#define MBUF_EXT_PGS_ASSERT_SANITY(ext_pgs)
#define MBUF_EXT_PGS_ASSERT_SANITY(m)
#endif
#endif

8
sys/sys/sglist.h
View File

@ -57,7 +57,6 @@ struct sglist {
struct bio;
struct mbuf;
struct mbuf_ext_pgs;
struct uio;
static __inline void
@ -88,8 +87,8 @@ sglist_hold(struct sglist *sg)
struct sglist *sglist_alloc(int nsegs, int mflags);
int sglist_append(struct sglist *sg, void *buf, size_t len);
int sglist_append_bio(struct sglist *sg, struct bio *bp);
int sglist_append_ext_pgs(struct sglist *sg, struct mbuf_ext_pgs *ext_pgs,
size_t off, size_t len);
int sglist_append_ext_pgs(struct sglist *sg, struct mbuf *m, size_t off,
size_t len);
int sglist_append_mb_ext_pgs(struct sglist *sg, struct mbuf *m);
int sglist_append_mbuf(struct sglist *sg, struct mbuf *m0);
int sglist_append_phys(struct sglist *sg, vm_paddr_t paddr,
@ -105,8 +104,7 @@ struct sglist *sglist_build(void *buf, size_t len, int mflags);
struct sglist *sglist_clone(struct sglist *sg, int mflags);
int sglist_consume_uio(struct sglist *sg, struct uio *uio, size_t resid);
int sglist_count(void *buf, size_t len);
int sglist_count_ext_pgs(struct mbuf_ext_pgs *ext_pgs, size_t off,
size_t len);
int sglist_count_ext_pgs(struct mbuf *m, size_t off, size_t len);
int sglist_count_mb_ext_pgs(struct mbuf *m);
int sglist_count_vmpages(vm_page_t *m, size_t pgoff, size_t len);
void sglist_free(struct sglist *sg);