Rename m_getm() to m_getm2() and rewrite it to allocate up to page sized
mbuf clusters. Add a flags parameter to accept M_PKTHDR and M_EOR mbuf chain flags. Provide compatibility macro for m_getm() calling m_getm2() with M_PKTHDR set. Rewrite m_uiotombuf() to use m_getm2() for mbuf allocation and do the uiomove() in a tight loop over the mbuf chain. Add a flags parameter to accept mbuf flags to be passed to m_getm2(). Adjust all callers for the extra parameter. Sponsored by: TCP/IP Optimization Fundraise 2005 MFC after: 3 month
This commit is contained in:
parent
593bbd2195
commit
5e20f43d31
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=163915
@ -94,61 +94,61 @@ SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragrandomfailures, CTLFLAG_RW,
|
||||
* chain.
|
||||
*/
|
||||
struct mbuf *
|
||||
m_getm(struct mbuf *m, int len, int how, short type)
|
||||
m_getm2(struct mbuf *m, int len, int how, short type, int flags)
|
||||
{
|
||||
struct mbuf *mb, *top, *cur, *mtail;
|
||||
int num, rem;
|
||||
int i;
|
||||
struct mbuf *mb, *nm = NULL, *mtail = NULL;
|
||||
|
||||
KASSERT(len >= 0, ("m_getm(): len is < 0"));
|
||||
KASSERT(len >= 0, ("%s: len is < 0", __func__));
|
||||
|
||||
/* If m != NULL, we will append to the end of that chain. */
|
||||
if (m != NULL)
|
||||
for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next);
|
||||
else
|
||||
mtail = NULL;
|
||||
/* Validate flags. */
|
||||
flags &= (M_PKTHDR | M_EOR);
|
||||
|
||||
/*
|
||||
* Calculate how many mbufs+clusters ("packets") we need and how much
|
||||
* leftover there is after that and allocate the first mbuf+cluster
|
||||
* if required.
|
||||
*/
|
||||
num = len / MCLBYTES;
|
||||
rem = len % MCLBYTES;
|
||||
top = cur = NULL;
|
||||
if (num > 0) {
|
||||
if ((top = cur = m_getcl(how, type, 0)) == NULL)
|
||||
goto failed;
|
||||
top->m_len = 0;
|
||||
}
|
||||
num--;
|
||||
/* Packet header mbuf must be first in chain. */
|
||||
if ((flags & M_PKTHDR) && m != NULL)
|
||||
flags &= ~M_PKTHDR;
|
||||
|
||||
for (i = 0; i < num; i++) {
|
||||
mb = m_getcl(how, type, 0);
|
||||
if (mb == NULL)
|
||||
goto failed;
|
||||
mb->m_len = 0;
|
||||
cur = (cur->m_next = mb);
|
||||
}
|
||||
if (rem > 0) {
|
||||
mb = (rem >= MINCLSIZE) ?
|
||||
m_getcl(how, type, 0) : m_get(how, type);
|
||||
if (mb == NULL)
|
||||
goto failed;
|
||||
mb->m_len = 0;
|
||||
if (cur == NULL)
|
||||
top = mb;
|
||||
/* Loop and append maximum sized mbufs to the chain tail. */
|
||||
while (len > 0) {
|
||||
if (len > MCLBYTES)
|
||||
mb = m_getjcl(how, type, (flags & M_PKTHDR),
|
||||
MJUMPAGESIZE);
|
||||
else if (len >= MINCLSIZE)
|
||||
mb = m_getcl(how, type, (flags & M_PKTHDR));
|
||||
else if (flags & M_PKTHDR)
|
||||
mb = m_gethdr(how, type);
|
||||
else
|
||||
cur->m_next = mb;
|
||||
}
|
||||
mb = m_get(how, type);
|
||||
|
||||
if (mtail != NULL)
|
||||
mtail->m_next = top;
|
||||
return top;
|
||||
failed:
|
||||
if (top != NULL)
|
||||
m_freem(top);
|
||||
return NULL;
|
||||
/* Fail the whole operation if one mbuf can't be allocated. */
|
||||
if (mb == NULL) {
|
||||
if (nm != NULL)
|
||||
m_freem(nm);
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
/* Book keeping. */
|
||||
len -= (mb->m_flags & M_EXT) ? mb->m_ext.ext_size :
|
||||
((mb->m_flags & M_PKTHDR) ? MHLEN : MLEN);
|
||||
if (mtail != NULL)
|
||||
mtail->m_next = mb;
|
||||
else
|
||||
nm = mb;
|
||||
mtail = mb;
|
||||
flags &= ~M_PKTHDR; /* Only valid on the first mbuf. */
|
||||
}
|
||||
if (flags & M_EOR)
|
||||
mtail->m_flags |= M_EOR; /* Only valid on the last mbuf. */
|
||||
|
||||
/* If mbuf was supplied, append new chain to the end of it. */
|
||||
if (m != NULL) {
|
||||
for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next)
|
||||
;
|
||||
mtail->m_next = nm;
|
||||
mtail->m_flags &= ~M_EOR;
|
||||
} else
|
||||
m = nm;
|
||||
|
||||
return (m);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -1610,55 +1610,58 @@ m_fragment(struct mbuf *m0, int how, int length)
|
||||
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Copy the contents of uio into a properly sized mbuf chain.
|
||||
*/
|
||||
struct mbuf *
|
||||
m_uiotombuf(struct uio *uio, int how, int len, int align)
|
||||
m_uiotombuf(struct uio *uio, int how, int len, int align, int flags)
|
||||
{
|
||||
struct mbuf *m_new = NULL, *m_final = NULL;
|
||||
int progress = 0, error = 0, length, total;
|
||||
struct mbuf *m, *mb;
|
||||
int error, length, total;
|
||||
int progress = 0;
|
||||
|
||||
/*
|
||||
* len can be zero or an arbitrary large value bound by
|
||||
* the total data supplied by the uio.
|
||||
*/
|
||||
if (len > 0)
|
||||
total = min(uio->uio_resid, len);
|
||||
else
|
||||
total = uio->uio_resid;
|
||||
|
||||
/*
|
||||
* The smallest unit returned by m_getm2() is a single mbuf
|
||||
* with pkthdr. We can't align past it. Align align itself.
|
||||
*/
|
||||
if (align)
|
||||
align &= ~(sizeof(long) - 1);
|
||||
if (align >= MHLEN)
|
||||
goto nospace;
|
||||
if (total + align > MHLEN)
|
||||
m_final = m_getcl(how, MT_DATA, M_PKTHDR);
|
||||
else
|
||||
m_final = m_gethdr(how, MT_DATA);
|
||||
if (m_final == NULL)
|
||||
goto nospace;
|
||||
m_final->m_data += align;
|
||||
m_new = m_final;
|
||||
while (progress < total) {
|
||||
length = total - progress;
|
||||
if (length > MCLBYTES)
|
||||
length = MCLBYTES;
|
||||
if (m_new == NULL) {
|
||||
if (length > MLEN)
|
||||
m_new = m_getcl(how, MT_DATA, 0);
|
||||
else
|
||||
m_new = m_get(how, MT_DATA);
|
||||
if (m_new == NULL)
|
||||
goto nospace;
|
||||
return (NULL);
|
||||
|
||||
/* Give us all or nothing. */
|
||||
m = m_getm2(NULL, total + align, how, MT_DATA, flags);
|
||||
if (m == NULL)
|
||||
return (NULL);
|
||||
m->m_data += align;
|
||||
|
||||
/* Fill all mbufs with uio data and update header information. */
|
||||
for (mb = m; mb != NULL; mb = mb->m_next) {
|
||||
length = min(M_TRAILINGSPACE(mb), total - progress);
|
||||
|
||||
error = uiomove(mtod(mb, void *), length, uio);
|
||||
if (error) {
|
||||
m_freem(m);
|
||||
return (NULL);
|
||||
}
|
||||
error = uiomove(mtod(m_new, void *), length, uio);
|
||||
if (error)
|
||||
goto nospace;
|
||||
|
||||
mb->m_len = length;
|
||||
progress += length;
|
||||
m_new->m_len = length;
|
||||
if (m_new != m_final)
|
||||
m_cat(m_final, m_new);
|
||||
m_new = NULL;
|
||||
if (flags & M_PKTHDR)
|
||||
m->m_pkthdr.len += length;
|
||||
}
|
||||
m_fixhdr(m_final);
|
||||
return (m_final);
|
||||
nospace:
|
||||
if (m_new)
|
||||
m_free(m_new);
|
||||
if (m_final)
|
||||
m_freem(m_final);
|
||||
return (NULL);
|
||||
KASSERT(progress == total, ("%s: progress != total", __func__));
|
||||
|
||||
return (m);
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -1966,13 +1966,11 @@ kern_sendfile(struct thread *td, struct sendfile_args *uap,
|
||||
hdr_uio->uio_rw = UIO_WRITE;
|
||||
if (hdr_uio->uio_resid > 0) {
|
||||
m = m_uiotombuf(hdr_uio, (mnw ? M_NOWAIT : M_WAITOK),
|
||||
0, 0);
|
||||
0, 0, 0);
|
||||
if (m == NULL) {
|
||||
error = mnw ? EAGAIN : ENOBUFS;
|
||||
goto done;
|
||||
}
|
||||
/* XXX: This should not be a header mbuf. */
|
||||
m_demote(m, 0);
|
||||
headersize = hdr_uio->uio_resid;
|
||||
if (compat)
|
||||
sbytes += headersize;
|
||||
|
@ -827,7 +827,8 @@ tapwrite(struct cdev *dev, struct uio *uio, int flag)
|
||||
return (EIO);
|
||||
}
|
||||
|
||||
if ((m = m_uiotombuf(uio, M_DONTWAIT, 0, ETHER_ALIGN)) == NULL) {
|
||||
if ((m = m_uiotombuf(uio, M_DONTWAIT, 0, ETHER_ALIGN,
|
||||
M_PKTHDR)) == NULL) {
|
||||
ifp->if_ierrors ++;
|
||||
return (error);
|
||||
}
|
||||
|
@ -790,7 +790,7 @@ tunwrite(struct cdev *dev, struct uio *uio, int flag)
|
||||
return (EIO);
|
||||
}
|
||||
|
||||
if ((m = m_uiotombuf(uio, M_DONTWAIT, 0, 0)) == NULL) {
|
||||
if ((m = m_uiotombuf(uio, M_DONTWAIT, 0, 0, M_PKTHDR)) == NULL) {
|
||||
ifp->if_ierrors++;
|
||||
return (error);
|
||||
}
|
||||
|
@ -384,7 +384,7 @@ pppwrite(tp, uio, flag)
|
||||
return (EMSGSIZE);
|
||||
|
||||
s = spltty();
|
||||
if ((m = m_uiotombuf(uio, M_DONTWAIT, 0, 0)) == NULL) {
|
||||
if ((m = m_uiotombuf(uio, M_DONTWAIT, 0, 0, M_PKTHDR)) == NULL) {
|
||||
splx(s);
|
||||
return (ENOBUFS);
|
||||
}
|
||||
|
@ -466,7 +466,7 @@ ngdwrite(struct cdev *dev, struct uio *uio, int flag)
|
||||
if (uio->uio_resid < 0 || uio->uio_resid > IP_MAXPACKET)
|
||||
return (EIO);
|
||||
|
||||
if ((m = m_uiotombuf(uio, M_DONTWAIT, 0, 0)) == NULL)
|
||||
if ((m = m_uiotombuf(uio, M_DONTWAIT, 0, 0, M_PKTHDR)) == NULL)
|
||||
return (ENOBUFS);
|
||||
|
||||
NG_SEND_DATA_ONLY(error, priv->hook, m);
|
||||
|
@ -517,6 +517,8 @@ m_chtype(struct mbuf *m, short new_type)
|
||||
#define MCLGET(m, how) m_clget((m), (how))
|
||||
#define MEXTADD(m, buf, size, free, args, flags, type) \
|
||||
m_extadd((m), (caddr_t)(buf), (size), (free), (args), (flags), (type))
|
||||
#define m_getm(m, len, how, type) \
|
||||
m_getm2((m), (len), (how), (type), M_PKTHDR)
|
||||
|
||||
/*
|
||||
* Evaluate TRUE if it's safe to write to the mbuf m's data region (this
|
||||
@ -657,7 +659,7 @@ int m_dup_pkthdr(struct mbuf *, struct mbuf *, int);
|
||||
u_int m_fixhdr(struct mbuf *);
|
||||
struct mbuf *m_fragment(struct mbuf *, int, int);
|
||||
void m_freem(struct mbuf *);
|
||||
struct mbuf *m_getm(struct mbuf *, int, int, short);
|
||||
struct mbuf *m_getm2(struct mbuf *, int, int, short, int);
|
||||
struct mbuf *m_getptr(struct mbuf *, int, int *);
|
||||
u_int m_length(struct mbuf *, struct mbuf **);
|
||||
void m_move_pkthdr(struct mbuf *, struct mbuf *);
|
||||
@ -667,7 +669,7 @@ struct mbuf *m_pulldown(struct mbuf *, int, int, int *);
|
||||
struct mbuf *m_pullup(struct mbuf *, int);
|
||||
int m_sanity(struct mbuf *, int);
|
||||
struct mbuf *m_split(struct mbuf *, int, int);
|
||||
struct mbuf *m_uiotombuf(struct uio *, int, int, int);
|
||||
struct mbuf *m_uiotombuf(struct uio *, int, int, int, int);
|
||||
struct mbuf *m_unshare(struct mbuf *, int how);
|
||||
|
||||
/*-
|
||||
|
Loading…
Reference in New Issue
Block a user