Continuation of multi page mbuf redesign from r359919.
The following series of patches addresses three things: Now that array of pages is embedded into mbuf, we no longer need separate structure to pass around, so struct mbuf_ext_pgs is an artifact of the first implementation. And struct mbuf_ext_pgs_data is a crutch to accomodate the main idea r359919 with minimal churn. Also, M_EXT of type EXT_PGS are just a synonym of M_NOMAP. The namespace for the newfeature is somewhat inconsistent and sometimes has a lengthy prefixes. In these patches we will gradually bring the namespace to "m_epg" prefix for all mbuf fields and most functions. Step 1 of 4: o Anonymize mbuf_ext_pgs_data, embed in m_ext o Embed mbuf_ext_pgs o Start documenting all this entanglement Reviewed by: gallatin Differential Revision: https://reviews.freebsd.org/D24598
This commit is contained in:
Gleb Smirnoff
parent
4053f8ac4d
commit
0c1032665c
@ -906,7 +906,7 @@ ktls_tcp_payload_length(struct tlspcb *tlsp, struct mbuf *m_tls)
|
||||
|
||||
MBUF_EXT_PGS_ASSERT(m_tls);
|
||||
ext_pgs = &m_tls->m_ext_pgs;
|
||||
hdr = (void *)ext_pgs->m_epg_hdr;
|
||||
hdr = (void *)m_tls->m_epg_hdr;
|
||||
plen = ntohs(hdr->tls_length);
|
||||
|
||||
/*
|
||||
@ -962,7 +962,7 @@ ktls_payload_offset(struct tlspcb *tlsp, struct mbuf *m_tls)
|
||||
|
||||
MBUF_EXT_PGS_ASSERT(m_tls);
|
||||
ext_pgs = &m_tls->m_ext_pgs;
|
||||
hdr = (void *)ext_pgs->m_epg_hdr;
|
||||
hdr = (void *)m_tls->m_epg_hdr;
|
||||
plen = ntohs(hdr->tls_length);
|
||||
#ifdef INVARIANTS
|
||||
mlen = mtod(m_tls, vm_offset_t) + m_tls->m_len;
|
||||
@ -1040,7 +1040,7 @@ ktls_wr_len(struct tlspcb *tlsp, struct mbuf *m, struct mbuf *m_tls,
|
||||
return (wr_len);
|
||||
}
|
||||
|
||||
hdr = (void *)ext_pgs->m_epg_hdr;
|
||||
hdr = (void *)m_tls->m_epg_hdr;
|
||||
plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - ext_pgs->trail_len;
|
||||
if (tlen < plen) {
|
||||
plen = tlen;
|
||||
@ -1064,7 +1064,7 @@ ktls_wr_len(struct tlspcb *tlsp, struct mbuf *m, struct mbuf *m_tls,
|
||||
wr_len += roundup2(imm_len, 16);
|
||||
|
||||
/* TLS record payload via DSGL. */
|
||||
*nsegsp = sglist_count_ext_pgs(ext_pgs, ext_pgs->hdr_len + offset,
|
||||
*nsegsp = sglist_count_ext_pgs(m_tls, ext_pgs->hdr_len + offset,
|
||||
plen - (ext_pgs->hdr_len + offset));
|
||||
wr_len += ktls_sgl_size(*nsegsp);
|
||||
|
||||
@ -1543,7 +1543,7 @@ ktls_write_tunnel_packet(struct sge_txq *txq, void *dst, struct mbuf *m,
|
||||
(m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp)));
|
||||
|
||||
/* Copy the subset of the TLS header requested. */
|
||||
copy_to_txd(&txq->eq, (char *)ext_pgs->m_epg_hdr +
|
||||
copy_to_txd(&txq->eq, (char *)m_tls->m_epg_hdr +
|
||||
mtod(m_tls, vm_offset_t), &out, m_tls->m_len);
|
||||
txq->imm_wrs++;
|
||||
|
||||
@ -1604,7 +1604,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
|
||||
/* Locate the TLS header. */
|
||||
MBUF_EXT_PGS_ASSERT(m_tls);
|
||||
ext_pgs = &m_tls->m_ext_pgs;
|
||||
hdr = (void *)ext_pgs->m_epg_hdr;
|
||||
hdr = (void *)m_tls->m_epg_hdr;
|
||||
plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - ext_pgs->trail_len;
|
||||
|
||||
/* Determine how much of the TLS record to send. */
|
||||
@ -1799,7 +1799,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
|
||||
|
||||
/* Recalculate 'nsegs' if cached value is not available. */
|
||||
if (nsegs == 0)
|
||||
nsegs = sglist_count_ext_pgs(ext_pgs, ext_pgs->hdr_len +
|
||||
nsegs = sglist_count_ext_pgs(m_tls, ext_pgs->hdr_len +
|
||||
offset, plen - (ext_pgs->hdr_len + offset));
|
||||
|
||||
/* Calculate the size of the TLS work request. */
|
||||
@ -2031,7 +2031,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
|
||||
/* Populate the TLS header */
|
||||
out = (void *)(tx_data + 1);
|
||||
if (offset == 0) {
|
||||
memcpy(out, ext_pgs->m_epg_hdr, ext_pgs->hdr_len);
|
||||
memcpy(out, m_tls->m_epg_hdr, ext_pgs->hdr_len);
|
||||
out += ext_pgs->hdr_len;
|
||||
}
|
||||
|
||||
@ -2067,7 +2067,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
|
||||
|
||||
/* SGL for record payload */
|
||||
sglist_reset(txq->gl);
|
||||
if (sglist_append_ext_pgs(txq->gl, ext_pgs, ext_pgs->hdr_len + offset,
|
||||
if (sglist_append_ext_pgs(txq->gl, m_tls, ext_pgs->hdr_len + offset,
|
||||
plen - (ext_pgs->hdr_len + offset)) != 0) {
|
||||
#ifdef INVARIANTS
|
||||
panic("%s: failed to append sglist", __func__);
|
||||
|
||||
@ -2435,7 +2435,7 @@ count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_t *nextaddr)
|
||||
off = 0;
|
||||
len -= seglen;
|
||||
paddr = pmap_kextract(
|
||||
(vm_offset_t)&ext_pgs->m_epg_hdr[segoff]);
|
||||
(vm_offset_t)&m->m_epg_hdr[segoff]);
|
||||
if (*nextaddr != paddr)
|
||||
nsegs++;
|
||||
*nextaddr = paddr + seglen;
|
||||
@ -2454,7 +2454,7 @@ count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_t *nextaddr)
|
||||
off = 0;
|
||||
seglen = min(seglen, len);
|
||||
len -= seglen;
|
||||
paddr = ext_pgs->m_epg_pa[i] + segoff;
|
||||
paddr = m->m_epg_pa[i] + segoff;
|
||||
if (*nextaddr != paddr)
|
||||
nsegs++;
|
||||
*nextaddr = paddr + seglen;
|
||||
@ -2463,7 +2463,7 @@ count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_t *nextaddr)
|
||||
if (len != 0) {
|
||||
seglen = min(len, ext_pgs->trail_len - off);
|
||||
len -= seglen;
|
||||
paddr = pmap_kextract((vm_offset_t)&ext_pgs->m_epg_trail[off]);
|
||||
paddr = pmap_kextract((vm_offset_t)&m->m_epg_trail[off]);
|
||||
if (*nextaddr != paddr)
|
||||
nsegs++;
|
||||
*nextaddr = paddr + seglen;
|
||||
|
||||
@ -1935,7 +1935,7 @@ aiotx_free_pgs(struct mbuf *m)
|
||||
#endif
|
||||
|
||||
for (int i = 0; i < ext_pgs->npgs; i++) {
|
||||
pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
|
||||
pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
|
||||
vm_page_unwire(pg, PQ_ACTIVE);
|
||||
}
|
||||
|
||||
@ -2003,7 +2003,7 @@ alloc_aiotx_mbuf(struct kaiocb *job, int len)
|
||||
(npages - 2) * PAGE_SIZE;
|
||||
}
|
||||
for (i = 0; i < npages; i++)
|
||||
ext_pgs->m_epg_pa[i] = VM_PAGE_TO_PHYS(pgs[i]);
|
||||
m->m_epg_pa[i] = VM_PAGE_TO_PHYS(pgs[i]);
|
||||
|
||||
m->m_len = mlen;
|
||||
m->m_ext.ext_size = npages * PAGE_SIZE;
|
||||
|
||||
@ -1623,26 +1623,24 @@ t4_push_tls_records(struct adapter *sc, struct toepcb *toep, int drop)
|
||||
|
||||
#ifdef KERN_TLS
|
||||
static int
|
||||
count_ext_pgs_segs(struct mbuf_ext_pgs *ext_pgs,
|
||||
struct mbuf_ext_pgs_data *ext_pgs_data)
|
||||
count_ext_pgs_segs(struct mbuf *m)
|
||||
{
|
||||
vm_paddr_t nextpa;
|
||||
u_int i, nsegs;
|
||||
|
||||
MPASS(ext_pgs->npgs > 0);
|
||||
MPASS(m->m_ext_pgs.npgs > 0);
|
||||
nsegs = 1;
|
||||
nextpa = ext_pgs_data->pa[0] + PAGE_SIZE;
|
||||
for (i = 1; i < ext_pgs->npgs; i++) {
|
||||
if (nextpa != ext_pgs_data->pa[i])
|
||||
nextpa = m->m_epg_pa[0] + PAGE_SIZE;
|
||||
for (i = 1; i < m->m_ext_pgs.npgs; i++) {
|
||||
if (nextpa != m->m_epg_pa[i])
|
||||
nsegs++;
|
||||
nextpa = ext_pgs_data->pa[i] + PAGE_SIZE;
|
||||
nextpa = m->m_epg_pa[i] + PAGE_SIZE;
|
||||
}
|
||||
return (nsegs);
|
||||
}
|
||||
|
||||
static void
|
||||
write_ktlstx_sgl(void *dst, struct mbuf_ext_pgs *ext_pgs,
|
||||
struct mbuf_ext_pgs_data *ext_pgs_data, int nsegs)
|
||||
write_ktlstx_sgl(void *dst, struct mbuf *m, int nsegs)
|
||||
{
|
||||
struct ulptx_sgl *usgl = dst;
|
||||
vm_paddr_t pa;
|
||||
@ -1655,15 +1653,15 @@ write_ktlstx_sgl(void *dst, struct mbuf_ext_pgs *ext_pgs,
|
||||
V_ULPTX_NSGE(nsegs));
|
||||
|
||||
/* Figure out the first S/G length. */
|
||||
pa = ext_pgs_data->pa[0] + ext_pgs->first_pg_off;
|
||||
pa = m->m_epg_pa[0] + m->m_ext_pgs.first_pg_off;
|
||||
usgl->addr0 = htobe64(pa);
|
||||
len = mbuf_ext_pg_len(ext_pgs, 0, ext_pgs->first_pg_off);
|
||||
len = mbuf_ext_pg_len(&m->m_ext_pgs, 0, m->m_ext_pgs.first_pg_off);
|
||||
pa += len;
|
||||
for (i = 1; i < ext_pgs->npgs; i++) {
|
||||
if (ext_pgs_data->pa[i] != pa)
|
||||
for (i = 1; i < m->m_ext_pgs.npgs; i++) {
|
||||
if (m->m_epg_pa[i] != pa)
|
||||
break;
|
||||
len += mbuf_ext_pg_len(ext_pgs, i, 0);
|
||||
pa += mbuf_ext_pg_len(ext_pgs, i, 0);
|
||||
len += mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
|
||||
pa += mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
|
||||
}
|
||||
usgl->len0 = htobe32(len);
|
||||
#ifdef INVARIANTS
|
||||
@ -1671,21 +1669,21 @@ write_ktlstx_sgl(void *dst, struct mbuf_ext_pgs *ext_pgs,
|
||||
#endif
|
||||
|
||||
j = -1;
|
||||
for (; i < ext_pgs->npgs; i++) {
|
||||
if (j == -1 || ext_pgs_data->pa[i] != pa) {
|
||||
for (; i < m->m_ext_pgs.npgs; i++) {
|
||||
if (j == -1 || m->m_epg_pa[i] != pa) {
|
||||
if (j >= 0)
|
||||
usgl->sge[j / 2].len[j & 1] = htobe32(len);
|
||||
j++;
|
||||
#ifdef INVARIANTS
|
||||
nsegs--;
|
||||
#endif
|
||||
pa = ext_pgs_data->pa[i];
|
||||
pa = m->m_epg_pa[i];
|
||||
usgl->sge[j / 2].addr[j & 1] = htobe64(pa);
|
||||
len = mbuf_ext_pg_len(ext_pgs, i, 0);
|
||||
len = mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
|
||||
pa += len;
|
||||
} else {
|
||||
len += mbuf_ext_pg_len(ext_pgs, i, 0);
|
||||
pa += mbuf_ext_pg_len(ext_pgs, i, 0);
|
||||
len += mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
|
||||
pa += mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
|
||||
}
|
||||
}
|
||||
if (j >= 0) {
|
||||
@ -1694,8 +1692,7 @@ write_ktlstx_sgl(void *dst, struct mbuf_ext_pgs *ext_pgs,
|
||||
if ((j & 1) == 0)
|
||||
usgl->sge[j / 2].len[1] = htobe32(0);
|
||||
}
|
||||
KASSERT(nsegs == 0, ("%s: nsegs %d, ext_pgs %p", __func__, nsegs,
|
||||
ext_pgs));
|
||||
KASSERT(nsegs == 0, ("%s: nsegs %d, m %p", __func__, nsegs, m));
|
||||
}
|
||||
|
||||
/*
|
||||
@ -1813,8 +1810,7 @@ t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop)
|
||||
wr_len += AES_BLOCK_LEN;
|
||||
|
||||
/* Account for SGL in work request length. */
|
||||
nsegs = count_ext_pgs_segs(&m->m_ext_pgs,
|
||||
&m->m_ext.ext_pgs);
|
||||
nsegs = count_ext_pgs_segs(m);
|
||||
wr_len += sizeof(struct ulptx_sgl) +
|
||||
((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
|
||||
|
||||
@ -1892,8 +1888,7 @@ t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop)
|
||||
memcpy(buf, thdr + 1, toep->tls.iv_len);
|
||||
buf += AES_BLOCK_LEN;
|
||||
|
||||
write_ktlstx_sgl(buf, &m->m_ext_pgs, &m->m_ext.ext_pgs,
|
||||
nsegs);
|
||||
write_ktlstx_sgl(buf, m, nsegs);
|
||||
|
||||
KASSERT(toep->tx_credits >= credits,
|
||||
("%s: not enough credits", __func__));
|
||||
|
||||
@ -311,9 +311,6 @@ static void mb_reclaim(uma_zone_t, int);
|
||||
/* Ensure that MSIZE is a power of 2. */
|
||||
CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
|
||||
|
||||
_Static_assert(offsetof(struct mbuf, m_ext) ==
|
||||
offsetof(struct mbuf, m_ext_pgs.m_ext),
|
||||
"m_ext offset mismatch between mbuf and ext_pgs");
|
||||
_Static_assert(sizeof(struct mbuf) <= MSIZE,
|
||||
"size of mbuf exceeds MSIZE");
|
||||
/*
|
||||
@ -984,7 +981,7 @@ _mb_unmapped_to_ext(struct mbuf *m)
|
||||
goto fail;
|
||||
m_new->m_len = seglen;
|
||||
prev = top = m_new;
|
||||
memcpy(mtod(m_new, void *), &ext_pgs->m_epg_hdr[segoff],
|
||||
memcpy(mtod(m_new, void *), &m->m_epg_hdr[segoff],
|
||||
seglen);
|
||||
}
|
||||
}
|
||||
@ -1002,7 +999,7 @@ _mb_unmapped_to_ext(struct mbuf *m)
|
||||
seglen = min(seglen, len);
|
||||
len -= seglen;
|
||||
|
||||
pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
|
||||
pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
|
||||
m_new = m_get(M_NOWAIT, MT_DATA);
|
||||
if (m_new == NULL)
|
||||
goto fail;
|
||||
@ -1036,7 +1033,7 @@ _mb_unmapped_to_ext(struct mbuf *m)
|
||||
else
|
||||
prev->m_next = m_new;
|
||||
m_new->m_len = len;
|
||||
memcpy(mtod(m_new, void *), &ext_pgs->m_epg_trail[off], len);
|
||||
memcpy(mtod(m_new, void *), &m->m_epg_trail[off], len);
|
||||
}
|
||||
|
||||
if (ref_inc != 0) {
|
||||
@ -1154,8 +1151,9 @@ mb_alloc_ext_pgs(int how, m_ext_free_t ext_free)
|
||||
|
||||
#ifdef INVARIANT_SUPPORT
|
||||
void
|
||||
mb_ext_pgs_check(struct mbuf_ext_pgs *ext_pgs)
|
||||
mb_ext_pgs_check(struct mbuf *m)
|
||||
{
|
||||
struct mbuf_ext_pgs *ext_pgs = &m->m_ext_pgs;
|
||||
|
||||
/*
|
||||
* NB: This expects a non-empty buffer (npgs > 0 and
|
||||
@ -1163,7 +1161,7 @@ mb_ext_pgs_check(struct mbuf_ext_pgs *ext_pgs)
|
||||
*/
|
||||
KASSERT(ext_pgs->npgs > 0,
|
||||
("ext_pgs with no valid pages: %p", ext_pgs));
|
||||
KASSERT(ext_pgs->npgs <= nitems(ext_pgs->m_epg_pa),
|
||||
KASSERT(ext_pgs->npgs <= nitems(m->m_epg_pa),
|
||||
("ext_pgs with too many pages: %p", ext_pgs));
|
||||
KASSERT(ext_pgs->nrdy <= ext_pgs->npgs,
|
||||
("ext_pgs with too many ready pages: %p", ext_pgs));
|
||||
@ -1178,9 +1176,9 @@ mb_ext_pgs_check(struct mbuf_ext_pgs *ext_pgs)
|
||||
PAGE_SIZE, ("ext_pgs with single page too large: %p",
|
||||
ext_pgs));
|
||||
}
|
||||
KASSERT(ext_pgs->hdr_len <= sizeof(ext_pgs->m_epg_hdr),
|
||||
KASSERT(ext_pgs->hdr_len <= sizeof(m->m_epg_hdr),
|
||||
("ext_pgs with too large header length: %p", ext_pgs));
|
||||
KASSERT(ext_pgs->trail_len <= sizeof(ext_pgs->m_epg_trail),
|
||||
KASSERT(ext_pgs->trail_len <= sizeof(m->m_epg_trail),
|
||||
("ext_pgs with too large header length: %p", ext_pgs));
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -203,7 +203,7 @@ sendfile_free_mext_pg(struct mbuf *m)
|
||||
for (i = 0; i < ext_pgs->npgs; i++) {
|
||||
if (cache_last && i == ext_pgs->npgs - 1)
|
||||
flags = 0;
|
||||
pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
|
||||
pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
|
||||
vm_page_release(pg, flags);
|
||||
}
|
||||
|
||||
@ -1046,11 +1046,11 @@ vn_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
|
||||
ext_pgs->nrdy++;
|
||||
}
|
||||
|
||||
ext_pgs->m_epg_pa[ext_pgs_idx] = VM_PAGE_TO_PHYS(pga);
|
||||
m0->m_epg_pa[ext_pgs_idx] = VM_PAGE_TO_PHYS(pga);
|
||||
ext_pgs->npgs++;
|
||||
xfs = xfsize(i, npages, off, space);
|
||||
ext_pgs->last_pg_len = xfs;
|
||||
MBUF_EXT_PGS_ASSERT_SANITY(ext_pgs);
|
||||
MBUF_EXT_PGS_ASSERT_SANITY(m0);
|
||||
mtail->m_len += xfs;
|
||||
mtail->m_ext.ext_size += PAGE_SIZE;
|
||||
continue;
|
||||
|
||||
@ -141,7 +141,7 @@ _bus_dmamap_load_unmapped_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
|
||||
off = 0;
|
||||
len -= seglen;
|
||||
error = _bus_dmamap_load_buffer(dmat, map,
|
||||
&ext_pgs->m_epg_hdr[segoff], seglen, kernel_pmap,
|
||||
&m->m_epg_hdr[segoff], seglen, kernel_pmap,
|
||||
flags, segs, nsegs);
|
||||
}
|
||||
}
|
||||
@ -159,7 +159,7 @@ _bus_dmamap_load_unmapped_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
|
||||
seglen = min(seglen, len);
|
||||
len -= seglen;
|
||||
error = _bus_dmamap_load_phys(dmat, map,
|
||||
ext_pgs->m_epg_pa[i] + segoff, seglen, flags, segs, nsegs);
|
||||
m->m_epg_pa[i] + segoff, seglen, flags, segs, nsegs);
|
||||
pgoff = 0;
|
||||
};
|
||||
if (len != 0 && error == 0) {
|
||||
@ -167,7 +167,7 @@ _bus_dmamap_load_unmapped_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
|
||||
("off + len > trail (%d + %d > %d)", off, len,
|
||||
ext_pgs->trail_len));
|
||||
error = _bus_dmamap_load_buffer(dmat, map,
|
||||
&ext_pgs->m_epg_trail[off], len, kernel_pmap, flags, segs,
|
||||
&m->m_epg_trail[off], len, kernel_pmap, flags, segs,
|
||||
nsegs);
|
||||
}
|
||||
return (error);
|
||||
|
||||
@ -223,8 +223,9 @@ sglist_count_vmpages(vm_page_t *m, size_t pgoff, size_t len)
|
||||
* describe an EXT_PGS buffer.
|
||||
*/
|
||||
int
|
||||
sglist_count_ext_pgs(struct mbuf_ext_pgs *ext_pgs, size_t off, size_t len)
|
||||
sglist_count_ext_pgs(struct mbuf *m, size_t off, size_t len)
|
||||
{
|
||||
struct mbuf_ext_pgs *ext_pgs = &m->m_ext_pgs;
|
||||
vm_paddr_t nextaddr, paddr;
|
||||
size_t seglen, segoff;
|
||||
int i, nsegs, pglen, pgoff;
|
||||
@ -242,7 +243,7 @@ sglist_count_ext_pgs(struct mbuf_ext_pgs *ext_pgs, size_t off, size_t len)
|
||||
seglen = MIN(seglen, len);
|
||||
off = 0;
|
||||
len -= seglen;
|
||||
nsegs += sglist_count(&ext_pgs->m_epg_hdr[segoff],
|
||||
nsegs += sglist_count(&m->m_epg_hdr[segoff],
|
||||
seglen);
|
||||
}
|
||||
}
|
||||
@ -260,7 +261,7 @@ sglist_count_ext_pgs(struct mbuf_ext_pgs *ext_pgs, size_t off, size_t len)
|
||||
off = 0;
|
||||
seglen = MIN(seglen, len);
|
||||
len -= seglen;
|
||||
paddr = ext_pgs->m_epg_pa[i] + segoff;
|
||||
paddr = m->m_epg_pa[i] + segoff;
|
||||
if (paddr != nextaddr)
|
||||
nsegs++;
|
||||
nextaddr = paddr + seglen;
|
||||
@ -269,7 +270,7 @@ sglist_count_ext_pgs(struct mbuf_ext_pgs *ext_pgs, size_t off, size_t len)
|
||||
if (len != 0) {
|
||||
seglen = MIN(len, ext_pgs->trail_len - off);
|
||||
len -= seglen;
|
||||
nsegs += sglist_count(&ext_pgs->m_epg_trail[off], seglen);
|
||||
nsegs += sglist_count(&m->m_epg_trail[off], seglen);
|
||||
}
|
||||
KASSERT(len == 0, ("len != 0"));
|
||||
return (nsegs);
|
||||
@ -284,8 +285,7 @@ sglist_count_mb_ext_pgs(struct mbuf *m)
|
||||
{
|
||||
|
||||
MBUF_EXT_PGS_ASSERT(m);
|
||||
return (sglist_count_ext_pgs(&m->m_ext_pgs, mtod(m, vm_offset_t),
|
||||
m->m_len));
|
||||
return (sglist_count_ext_pgs(m, mtod(m, vm_offset_t), m->m_len));
|
||||
}
|
||||
|
||||
/*
|
||||
@ -395,9 +395,9 @@ sglist_append_phys(struct sglist *sg, vm_paddr_t paddr, size_t len)
|
||||
* fails with EFBIG.
|
||||
*/
|
||||
int
|
||||
sglist_append_ext_pgs(struct sglist *sg, struct mbuf_ext_pgs *ext_pgs,
|
||||
size_t off, size_t len)
|
||||
sglist_append_ext_pgs(struct sglist *sg, struct mbuf *m, size_t off, size_t len)
|
||||
{
|
||||
struct mbuf_ext_pgs *ext_pgs = &m->m_ext_pgs;
|
||||
size_t seglen, segoff;
|
||||
vm_paddr_t paddr;
|
||||
int error, i, pglen, pgoff;
|
||||
@ -413,7 +413,7 @@ sglist_append_ext_pgs(struct sglist *sg, struct mbuf_ext_pgs *ext_pgs,
|
||||
off = 0;
|
||||
len -= seglen;
|
||||
error = sglist_append(sg,
|
||||
&ext_pgs->m_epg_hdr[segoff], seglen);
|
||||
&m->m_epg_hdr[segoff], seglen);
|
||||
}
|
||||
}
|
||||
pgoff = ext_pgs->first_pg_off;
|
||||
@ -429,7 +429,7 @@ sglist_append_ext_pgs(struct sglist *sg, struct mbuf_ext_pgs *ext_pgs,
|
||||
off = 0;
|
||||
seglen = MIN(seglen, len);
|
||||
len -= seglen;
|
||||
paddr = ext_pgs->m_epg_pa[i] + segoff;
|
||||
paddr = m->m_epg_pa[i] + segoff;
|
||||
error = sglist_append_phys(sg, paddr, seglen);
|
||||
pgoff = 0;
|
||||
};
|
||||
@ -437,7 +437,7 @@ sglist_append_ext_pgs(struct sglist *sg, struct mbuf_ext_pgs *ext_pgs,
|
||||
seglen = MIN(len, ext_pgs->trail_len - off);
|
||||
len -= seglen;
|
||||
error = sglist_append(sg,
|
||||
&ext_pgs->m_epg_trail[off], seglen);
|
||||
&m->m_epg_trail[off], seglen);
|
||||
}
|
||||
if (error == 0)
|
||||
KASSERT(len == 0, ("len != 0"));
|
||||
@ -455,8 +455,7 @@ sglist_append_mb_ext_pgs(struct sglist *sg, struct mbuf *m)
|
||||
|
||||
/* for now, all unmapped mbufs are assumed to be EXT_PGS */
|
||||
MBUF_EXT_PGS_ASSERT(m);
|
||||
return (sglist_append_ext_pgs(sg, &m->m_ext_pgs,
|
||||
mtod(m, vm_offset_t), m->m_len));
|
||||
return (sglist_append_ext_pgs(sg, m, mtod(m, vm_offset_t), m->m_len));
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
@ -1374,7 +1374,7 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls, int *enq_cnt,
|
||||
m->m_len += pgs->hdr_len + pgs->trail_len;
|
||||
|
||||
/* Populate the TLS header. */
|
||||
tlshdr = (void *)pgs->m_epg_hdr;
|
||||
tlshdr = (void *)m->m_epg_hdr;
|
||||
tlshdr->tls_vmajor = tls->params.tls_vmajor;
|
||||
|
||||
/*
|
||||
@ -1387,7 +1387,7 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls, int *enq_cnt,
|
||||
tlshdr->tls_type = TLS_RLTYPE_APP;
|
||||
/* save the real record type for later */
|
||||
pgs->record_type = record_type;
|
||||
pgs->m_epg_trail[0] = record_type;
|
||||
m->m_epg_trail[0] = record_type;
|
||||
} else {
|
||||
tlshdr->tls_vminor = tls->params.tls_vminor;
|
||||
tlshdr->tls_type = record_type;
|
||||
@ -1552,7 +1552,7 @@ ktls_encrypt(struct mbuf_ext_pgs *pgs)
|
||||
len = mbuf_ext_pg_len(pgs, i, off);
|
||||
src_iov[i].iov_len = len;
|
||||
src_iov[i].iov_base =
|
||||
(char *)(void *)PHYS_TO_DMAP(pgs->m_epg_pa[i]) +
|
||||
(char *)(void *)PHYS_TO_DMAP(m->m_epg_pa[i]) +
|
||||
off;
|
||||
|
||||
if (is_anon) {
|
||||
@ -1576,8 +1576,8 @@ ktls_encrypt(struct mbuf_ext_pgs *pgs)
|
||||
npages += i;
|
||||
|
||||
error = (*tls->sw_encrypt)(tls,
|
||||
(const struct tls_record_layer *)pgs->m_epg_hdr,
|
||||
pgs->m_epg_trail, src_iov, dst_iov, i, pgs->seqno,
|
||||
(const struct tls_record_layer *)m->m_epg_hdr,
|
||||
m->m_epg_trail, src_iov, dst_iov, i, pgs->seqno,
|
||||
pgs->record_type);
|
||||
if (error) {
|
||||
counter_u64_add(ktls_offload_failed_crypto, 1);
|
||||
@ -1595,7 +1595,7 @@ ktls_encrypt(struct mbuf_ext_pgs *pgs)
|
||||
|
||||
/* Replace them with the new pages. */
|
||||
for (i = 0; i < pgs->npgs; i++)
|
||||
pgs->m_epg_pa[i] = parray[i];
|
||||
m->m_epg_pa[i] = parray[i];
|
||||
|
||||
/* Use the basic free routine. */
|
||||
m->m_ext.ext_free = mb_free_mext_pgs;
|
||||
|
||||
@ -163,11 +163,11 @@ CTASSERT(offsetof(struct mbuf, m_pktdat) % 8 == 0);
|
||||
#if defined(__LP64__)
|
||||
CTASSERT(offsetof(struct mbuf, m_dat) == 32);
|
||||
CTASSERT(sizeof(struct pkthdr) == 56);
|
||||
CTASSERT(sizeof(struct m_ext) == 168);
|
||||
CTASSERT(sizeof(struct m_ext) == 160);
|
||||
#else
|
||||
CTASSERT(offsetof(struct mbuf, m_dat) == 24);
|
||||
CTASSERT(sizeof(struct pkthdr) == 48);
|
||||
CTASSERT(sizeof(struct m_ext) == 184);
|
||||
CTASSERT(sizeof(struct m_ext) == 180);
|
||||
#endif
|
||||
|
||||
/*
|
||||
@ -195,19 +195,30 @@ mb_dupcl(struct mbuf *n, struct mbuf *m)
|
||||
KASSERT(!(n->m_flags & M_EXT), ("%s: M_EXT set on %p", __func__, n));
|
||||
|
||||
/*
|
||||
* Cache access optimization. For most kinds of external
|
||||
* storage we don't need full copy of m_ext, since the
|
||||
* holder of the 'ext_count' is responsible to carry the
|
||||
* free routine and its arguments. Exclusion is EXT_EXTREF,
|
||||
* where 'ext_cnt' doesn't point into mbuf at all.
|
||||
* Cache access optimization.
|
||||
*
|
||||
* o Regular M_EXT storage doesn't need full copy of m_ext, since
|
||||
* the holder of the 'ext_count' is responsible to carry the free
|
||||
* routine and its arguments.
|
||||
* o EXT_PGS data is split between main part of mbuf and m_ext, the
|
||||
* main part is copied in full, the m_ext part is similar to M_EXT.
|
||||
* o EXT_EXTREF, where 'ext_cnt' doesn't point into mbuf at all, is
|
||||
* special - it needs full copy of m_ext into each mbuf, since any
|
||||
* copy could end up as the last to free.
|
||||
*/
|
||||
if (m->m_ext.ext_type == EXT_EXTREF)
|
||||
bcopy(&m->m_ext, &n->m_ext, sizeof(struct m_ext));
|
||||
else if (m->m_ext.ext_type == EXT_PGS)
|
||||
switch (m->m_ext.ext_type) {
|
||||
case EXT_PGS:
|
||||
bcopy(&m->m_ext, &n->m_ext, m_epg_copylen);
|
||||
bcopy(&m->m_ext_pgs, &n->m_ext_pgs,
|
||||
sizeof(struct mbuf_ext_pgs));
|
||||
else
|
||||
break;
|
||||
case EXT_EXTREF:
|
||||
bcopy(&m->m_ext, &n->m_ext, sizeof(struct m_ext));
|
||||
break;
|
||||
default:
|
||||
bcopy(&m->m_ext, &n->m_ext, m_ext_copylen);
|
||||
}
|
||||
|
||||
n->m_flags |= M_EXT;
|
||||
n->m_flags |= m->m_flags & (M_RDONLY | M_NOMAP);
|
||||
|
||||
@ -1623,7 +1634,7 @@ mb_free_mext_pgs(struct mbuf *m)
|
||||
MBUF_EXT_PGS_ASSERT(m);
|
||||
ext_pgs = &m->m_ext_pgs;
|
||||
for (int i = 0; i < ext_pgs->npgs; i++) {
|
||||
pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
|
||||
pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
|
||||
vm_page_unwire_noq(pg);
|
||||
vm_page_free(pg);
|
||||
}
|
||||
@ -1681,11 +1692,11 @@ m_uiotombuf_nomap(struct uio *uio, int how, int len, int maxseg, int flags)
|
||||
}
|
||||
}
|
||||
pg_array[i]->flags &= ~PG_ZERO;
|
||||
pgs->m_epg_pa[i] = VM_PAGE_TO_PHYS(pg_array[i]);
|
||||
mb->m_epg_pa[i] = VM_PAGE_TO_PHYS(pg_array[i]);
|
||||
pgs->npgs++;
|
||||
}
|
||||
pgs->last_pg_len = length - PAGE_SIZE * (pgs->npgs - 1);
|
||||
MBUF_EXT_PGS_ASSERT_SANITY(pgs);
|
||||
MBUF_EXT_PGS_ASSERT_SANITY(mb);
|
||||
total -= length;
|
||||
error = uiomove_fromphys(pg_array, 0, length, uio);
|
||||
if (error != 0)
|
||||
@ -1788,7 +1799,8 @@ m_unmappedtouio(const struct mbuf *m, int m_off, struct uio *uio, int len)
|
||||
seglen = min(seglen, len);
|
||||
off = 0;
|
||||
len -= seglen;
|
||||
error = uiomove(&ext_pgs->m_epg_hdr[segoff], seglen, uio);
|
||||
error = uiomove(__DECONST(void *,
|
||||
&m->m_epg_hdr[segoff]), seglen, uio);
|
||||
}
|
||||
}
|
||||
pgoff = ext_pgs->first_pg_off;
|
||||
@ -1804,7 +1816,7 @@ m_unmappedtouio(const struct mbuf *m, int m_off, struct uio *uio, int len)
|
||||
off = 0;
|
||||
seglen = min(seglen, len);
|
||||
len -= seglen;
|
||||
pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
|
||||
pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
|
||||
error = uiomove_fromphys(&pg, segoff, seglen, uio);
|
||||
pgoff = 0;
|
||||
};
|
||||
@ -1812,7 +1824,8 @@ m_unmappedtouio(const struct mbuf *m, int m_off, struct uio *uio, int len)
|
||||
KASSERT((off + len) <= ext_pgs->trail_len,
|
||||
("off + len > trail (%d + %d > %d, m_off = %d)", off, len,
|
||||
ext_pgs->trail_len, m_off));
|
||||
error = uiomove(&ext_pgs->m_epg_trail[off], len, uio);
|
||||
error = uiomove(__DECONST(void *, &m->m_epg_trail[off]),
|
||||
len, uio);
|
||||
}
|
||||
return (error);
|
||||
}
|
||||
|
||||
140
sys/sys/mbuf.h
140
sys/sys/mbuf.h
@ -231,13 +231,6 @@ struct pkthdr {
|
||||
|
||||
#define MBUF_PEXT_FLAG_ANON 1 /* Data can be encrypted in place. */
|
||||
|
||||
|
||||
struct mbuf_ext_pgs_data {
|
||||
vm_paddr_t pa[MBUF_PEXT_MAX_PGS]; /* phys addrs of pgs */
|
||||
char trail[MBUF_PEXT_TRAIL_LEN]; /* TLS trailer */
|
||||
char hdr[MBUF_PEXT_HDR_LEN]; /* TLS header */
|
||||
};
|
||||
|
||||
struct ktls_session;
|
||||
struct socket;
|
||||
|
||||
@ -266,49 +259,49 @@ struct m_ext {
|
||||
uint32_t ext_size; /* size of buffer, for ext_free */
|
||||
uint32_t ext_type:8, /* type of external storage */
|
||||
ext_flags:24; /* external storage mbuf flags */
|
||||
char *ext_buf; /* start of buffer */
|
||||
/*
|
||||
* Fields below store the free context for the external storage.
|
||||
* They are valid only in the refcount carrying mbuf, the one with
|
||||
* EXT_FLAG_EMBREF flag, with exclusion for EXT_EXTREF type, where
|
||||
* the free context is copied into all mbufs that use same external
|
||||
* storage.
|
||||
*/
|
||||
#define m_ext_copylen offsetof(struct m_ext, ext_free)
|
||||
m_ext_free_t *ext_free; /* free routine if not the usual */
|
||||
void *ext_arg1; /* optional argument pointer */
|
||||
union {
|
||||
void *ext_arg2; /* optional argument pointer */
|
||||
struct mbuf_ext_pgs_data ext_pgs;
|
||||
struct {
|
||||
/*
|
||||
* Regular M_EXT mbuf:
|
||||
* o ext_buf always points to the external buffer.
|
||||
* o ext_free (below) and two optional arguments
|
||||
* ext_arg1 and ext_arg2 store the free context for
|
||||
* the external storage. They are set only in the
|
||||
* refcount carrying mbuf, the one with
|
||||
* EXT_FLAG_EMBREF flag, with exclusion for
|
||||
* EXT_EXTREF type, where the free context is copied
|
||||
* into all mbufs that use same external storage.
|
||||
*/
|
||||
char *ext_buf; /* start of buffer */
|
||||
#define m_ext_copylen offsetof(struct m_ext, ext_arg2)
|
||||
void *ext_arg2;
|
||||
};
|
||||
struct {
|
||||
/*
|
||||
* Multi-page M_EXTPG mbuf:
|
||||
* o extpg_pa - page vector.
|
||||
* o extpg_trail and extpg_hdr - TLS trailer and
|
||||
* header.
|
||||
* Uses ext_free and may also use ext_arg1.
|
||||
*/
|
||||
vm_paddr_t extpg_pa[MBUF_PEXT_MAX_PGS];
|
||||
char extpg_trail[MBUF_PEXT_TRAIL_LEN];
|
||||
char extpg_hdr[MBUF_PEXT_HDR_LEN];
|
||||
/* Pretend these 3 fields are part of mbuf itself. */
|
||||
#define m_epg_pa m_ext.extpg_pa
|
||||
#define m_epg_trail m_ext.extpg_trail
|
||||
#define m_epg_hdr m_ext.extpg_hdr
|
||||
#define m_epg_copylen offsetof(struct m_ext, ext_free)
|
||||
};
|
||||
};
|
||||
/*
|
||||
* Free method and optional argument pointer, both
|
||||
* used by M_EXT and M_EXTPG.
|
||||
*/
|
||||
m_ext_free_t *ext_free;
|
||||
void *ext_arg1;
|
||||
};
|
||||
|
||||
struct mbuf_ext_pgs {
|
||||
uint8_t npgs; /* Number of attached pages */
|
||||
uint8_t nrdy; /* Pages with I/O pending */
|
||||
uint8_t hdr_len; /* TLS header length */
|
||||
uint8_t trail_len; /* TLS trailer length */
|
||||
uint16_t first_pg_off; /* Offset into 1st page */
|
||||
uint16_t last_pg_len; /* Length of last page */
|
||||
uint8_t flags; /* Flags */
|
||||
uint8_t record_type;
|
||||
uint8_t spare[2];
|
||||
int enc_cnt;
|
||||
struct ktls_session *tls; /* TLS session */
|
||||
struct socket *so;
|
||||
uint64_t seqno;
|
||||
struct mbuf *mbuf;
|
||||
STAILQ_ENTRY(mbuf_ext_pgs) stailq;
|
||||
#if !defined(__LP64__)
|
||||
uint8_t pad[8]; /* pad to size of pkthdr */
|
||||
#endif
|
||||
struct m_ext m_ext;
|
||||
};
|
||||
|
||||
#define m_epg_hdr m_ext.ext_pgs.hdr
|
||||
#define m_epg_trail m_ext.ext_pgs.trail
|
||||
#define m_epg_pa m_ext.ext_pgs.pa
|
||||
|
||||
/*
|
||||
* The core of the mbuf object along with some shortcut defines for practical
|
||||
* purposes.
|
||||
@ -347,15 +340,48 @@ struct mbuf {
|
||||
* order to support future work on variable-size mbufs.
|
||||
*/
|
||||
union {
|
||||
union {
|
||||
struct {
|
||||
struct pkthdr m_pkthdr; /* M_PKTHDR set */
|
||||
union {
|
||||
struct m_ext m_ext; /* M_EXT set */
|
||||
char m_pktdat[0];
|
||||
};
|
||||
struct {
|
||||
union {
|
||||
/* M_PKTHDR set. */
|
||||
struct pkthdr m_pkthdr;
|
||||
|
||||
/* M_EXTPG set.
|
||||
* Multi-page M_EXTPG mbuf has its meta data
|
||||
* split between the mbuf_ext_pgs structure
|
||||
* and m_ext. It carries vector of pages,
|
||||
* optional header and trailer char vectors
|
||||
* and pointers to socket/TLS data.
|
||||
*/
|
||||
struct mbuf_ext_pgs {
|
||||
/* Overall count of pages and count of
|
||||
* pages with I/O pending. */
|
||||
uint8_t npgs;
|
||||
uint8_t nrdy;
|
||||
/* TLS header and trailer lengths.
|
||||
* The data itself resides in m_ext. */
|
||||
uint8_t hdr_len;
|
||||
uint8_t trail_len;
|
||||
/* Offset into 1st page and lenght of
|
||||
* data in the last page. */
|
||||
uint16_t first_pg_off;
|
||||
uint16_t last_pg_len;
|
||||
uint8_t flags;
|
||||
uint8_t record_type;
|
||||
uint8_t spare[2];
|
||||
int enc_cnt;
|
||||
struct ktls_session *tls;
|
||||
struct socket *so;
|
||||
uint64_t seqno;
|
||||
struct mbuf *mbuf;
|
||||
STAILQ_ENTRY(mbuf_ext_pgs) stailq;
|
||||
} m_ext_pgs;
|
||||
};
|
||||
union {
|
||||
/* M_EXT or M_EXTPG set. */
|
||||
struct m_ext m_ext;
|
||||
/* M_PKTHDR set, neither M_EXT nor M_EXTPG. */
|
||||
char m_pktdat[0];
|
||||
};
|
||||
struct mbuf_ext_pgs m_ext_pgs;
|
||||
};
|
||||
char m_dat[0]; /* !M_PKTHDR, !M_EXT */
|
||||
};
|
||||
@ -375,12 +401,12 @@ mbuf_ext_pg_len(struct mbuf_ext_pgs *ext_pgs, int pidx, int pgoff)
|
||||
}
|
||||
|
||||
#ifdef INVARIANT_SUPPORT
|
||||
void mb_ext_pgs_check(struct mbuf_ext_pgs *ext_pgs);
|
||||
void mb_ext_pgs_check(struct mbuf *m);
|
||||
#endif
|
||||
#ifdef INVARIANTS
|
||||
#define MBUF_EXT_PGS_ASSERT_SANITY(ext_pgs) mb_ext_pgs_check((ext_pgs))
|
||||
#define MBUF_EXT_PGS_ASSERT_SANITY(m) mb_ext_pgs_check((m))
|
||||
#else
|
||||
#define MBUF_EXT_PGS_ASSERT_SANITY(ext_pgs)
|
||||
#define MBUF_EXT_PGS_ASSERT_SANITY(m)
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
||||
@ -57,7 +57,6 @@ struct sglist {
|
||||
|
||||
struct bio;
|
||||
struct mbuf;
|
||||
struct mbuf_ext_pgs;
|
||||
struct uio;
|
||||
|
||||
static __inline void
|
||||
@ -88,8 +87,8 @@ sglist_hold(struct sglist *sg)
|
||||
struct sglist *sglist_alloc(int nsegs, int mflags);
|
||||
int sglist_append(struct sglist *sg, void *buf, size_t len);
|
||||
int sglist_append_bio(struct sglist *sg, struct bio *bp);
|
||||
int sglist_append_ext_pgs(struct sglist *sg, struct mbuf_ext_pgs *ext_pgs,
|
||||
size_t off, size_t len);
|
||||
int sglist_append_ext_pgs(struct sglist *sg, struct mbuf *m, size_t off,
|
||||
size_t len);
|
||||
int sglist_append_mb_ext_pgs(struct sglist *sg, struct mbuf *m);
|
||||
int sglist_append_mbuf(struct sglist *sg, struct mbuf *m0);
|
||||
int sglist_append_phys(struct sglist *sg, vm_paddr_t paddr,
|
||||
@ -105,8 +104,7 @@ struct sglist *sglist_build(void *buf, size_t len, int mflags);
|
||||
struct sglist *sglist_clone(struct sglist *sg, int mflags);
|
||||
int sglist_consume_uio(struct sglist *sg, struct uio *uio, size_t resid);
|
||||
int sglist_count(void *buf, size_t len);
|
||||
int sglist_count_ext_pgs(struct mbuf_ext_pgs *ext_pgs, size_t off,
|
||||
size_t len);
|
||||
int sglist_count_ext_pgs(struct mbuf *m, size_t off, size_t len);
|
||||
int sglist_count_mb_ext_pgs(struct mbuf *m);
|
||||
int sglist_count_vmpages(vm_page_t *m, size_t pgoff, size_t len);
|
||||
void sglist_free(struct sglist *sg);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user