Rename m_getm() to m_getm2() and rewrite it to allocate up to page sized

mbuf clusters.  Add a flags parameter to accept M_PKTHDR and M_EOR mbuf
chain flags.  Provide compatibility macro for m_getm() calling m_getm2()
with M_PKTHDR set.

Rewrite m_uiotombuf() to use m_getm2() for mbuf allocation and do the
uiomove() in a tight loop over the mbuf chain.  Add a flags parameter to
accept mbuf flags to be passed to m_getm2().  Adjust all callers for the
extra parameter.

Sponsored by:	TCP/IP Optimization Fundraise 2005
MFC after:	3 month

sys/kern/uipc_mbuf.c

@@ -94,61 +94,61 @@ SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragrandomfailures, CTLFLAG_RW,
  * chain.
  */
 struct mbuf *
-m_getm(struct mbuf *m, int len, int how, short type)
+m_getm2(struct mbuf *m, int len, int how, short type, int flags)
 {
-	struct mbuf *mb, *top, *cur, *mtail;
-	int num, rem;
-	int i;
+	struct mbuf *mb, *nm = NULL, *mtail = NULL;
 
-	KASSERT(len >= 0, ("m_getm(): len is < 0"));
+	KASSERT(len >= 0, ("%s: len is < 0", __func__));
 
-	/* If m != NULL, we will append to the end of that chain. */
-	if (m != NULL)
-		for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next);
-	else
-		mtail = NULL;
+	/* Validate flags. */
+	flags &= (M_PKTHDR | M_EOR);
 
-	/*
-	 * Calculate how many mbufs+clusters ("packets") we need and how much
-	 * leftover there is after that and allocate the first mbuf+cluster
-	 * if required.
-	 */
-	num = len / MCLBYTES;
-	rem = len % MCLBYTES;
-	top = cur = NULL;
-	if (num > 0) {
-		if ((top = cur = m_getcl(how, type, 0)) == NULL)
-			goto failed;
-		top->m_len = 0;
-	}
-	num--;
+	/* Packet header mbuf must be first in chain. */
+	if ((flags & M_PKTHDR) && m != NULL)
+		flags &= ~M_PKTHDR;
 
-	for (i = 0; i < num; i++) {
-		mb = m_getcl(how, type, 0);
-		if (mb == NULL)
-			goto failed;
-		mb->m_len = 0;
-		cur = (cur->m_next = mb);
-	}
-	if (rem > 0) {
-		mb = (rem >= MINCLSIZE) ?
-		    m_getcl(how, type, 0) : m_get(how, type);
-		if (mb == NULL)
-			goto failed;
-		mb->m_len = 0;
-		if (cur == NULL)
-			top = mb;
+	/* Loop and append maximum sized mbufs to the chain tail. */
+	while (len > 0) {
+		if (len > MCLBYTES)
+			mb = m_getjcl(how, type, (flags & M_PKTHDR),
+			    MJUMPAGESIZE);
+		else if (len >= MINCLSIZE)
+			mb = m_getcl(how, type, (flags & M_PKTHDR));
+		else if (flags & M_PKTHDR)
+			mb = m_gethdr(how, type);
 		else
-			cur->m_next = mb;
-	}
+			mb = m_get(how, type);
 
-	if (mtail != NULL)
-		mtail->m_next = top;
-	return top;
-failed:
-	if (top != NULL)
-		m_freem(top);
-	return NULL;
+		/* Fail the whole operation if one mbuf can't be allocated. */
+		if (mb == NULL) {
+			if (nm != NULL)
+				m_freem(nm);
+			return (NULL);
+		}
+
+		/* Book keeping. */
+		len -= (mb->m_flags & M_EXT) ? mb->m_ext.ext_size :
+			((mb->m_flags & M_PKTHDR) ? MHLEN : MLEN);
+		if (mtail != NULL)
+			mtail->m_next = mb;
+		else
+			nm = mb;
+		mtail = mb;
+		flags &= ~M_PKTHDR;	/* Only valid on the first mbuf. */
+	}
+	if (flags & M_EOR)
+		mtail->m_flags |= M_EOR;  /* Only valid on the last mbuf. */
+
+	/* If mbuf was supplied, append new chain to the end of it. */
+	if (m != NULL) {
+		for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next)
+			;
+		mtail->m_next = nm;
+		mtail->m_flags &= ~M_EOR;
+	} else
+		m = nm;
+
+	return (m);
 }
 
 /*
@@ -1610,55 +1610,58 @@ m_fragment(struct mbuf *m0, int how, int length)
 
 #endif
 
+/*
+ * Copy the contents of uio into a properly sized mbuf chain.
+ */
 struct mbuf *
-m_uiotombuf(struct uio *uio, int how, int len, int align)
+m_uiotombuf(struct uio *uio, int how, int len, int align, int flags)
 {
-	struct mbuf *m_new = NULL, *m_final = NULL;
-	int progress = 0, error = 0, length, total;
+	struct mbuf *m, *mb;
+	int error, length, total;
+	int progress = 0;
 
+	/*
+	 * len can be zero or an arbitrary large value bound by
+	 * the total data supplied by the uio.
+	 */
 	if (len > 0)
 		total = min(uio->uio_resid, len);
 	else
 		total = uio->uio_resid;
+
+	/*
+	 * The smallest unit returned by m_getm2() is a single mbuf
+	 * with pkthdr.  We can't align past it.  Align align itself.
+	 */
+	if (align)
+		align &= ~(sizeof(long) - 1);
 	if (align >= MHLEN)
-		goto nospace;
-	if (total + align > MHLEN)
-		m_final = m_getcl(how, MT_DATA, M_PKTHDR);
-	else
-		m_final = m_gethdr(how, MT_DATA);
-	if (m_final == NULL)
-		goto nospace;
-	m_final->m_data += align;
-	m_new = m_final;
-	while (progress < total) {
-		length = total - progress;
-		if (length > MCLBYTES)
-			length = MCLBYTES;
-		if (m_new == NULL) {
-			if (length > MLEN)
-				m_new = m_getcl(how, MT_DATA, 0);
-			else
-				m_new = m_get(how, MT_DATA);
-			if (m_new == NULL)
-				goto nospace;
+		return (NULL);
+
+	/* Give us all or nothing. */
+	m = m_getm2(NULL, total + align, how, MT_DATA, flags);
+	if (m == NULL)
+		return (NULL);
+	m->m_data += align;
+
+	/* Fill all mbufs with uio data and update header information. */
+	for (mb = m; mb != NULL; mb = mb->m_next) {
+		length = min(M_TRAILINGSPACE(mb), total - progress);
+
+		error = uiomove(mtod(mb, void *), length, uio);
+		if (error) {
+			m_freem(m);
+			return (NULL);
 		}
-		error = uiomove(mtod(m_new, void *), length, uio);
-		if (error)
-			goto nospace;
+
+		mb->m_len = length;
 		progress += length;
-		m_new->m_len = length;
-		if (m_new != m_final)
-			m_cat(m_final, m_new);
-		m_new = NULL;
+		if (flags & M_PKTHDR)
+			m->m_pkthdr.len += length;
 	}
-	m_fixhdr(m_final);
-	return (m_final);
-nospace:
-	if (m_new)
-		m_free(m_new);
-	if (m_final)
-		m_freem(m_final);
-	return (NULL);
+	KASSERT(progress == total, ("%s: progress != total", __func__));
+
+	return (m);
 }
 
 /*

sys/kern/uipc_syscalls.c

@@ -1966,13 +1966,11 @@ kern_sendfile(struct thread *td, struct sendfile_args *uap,
 		hdr_uio->uio_rw = UIO_WRITE;
 		if (hdr_uio->uio_resid > 0) {
 			m = m_uiotombuf(hdr_uio, (mnw ? M_NOWAIT : M_WAITOK),
-			    0, 0);
+			    0, 0, 0);
 			if (m == NULL) {
 				error = mnw ? EAGAIN : ENOBUFS;
 				goto done;
 			}
-			/* XXX: This should not be a header mbuf. */
-			m_demote(m, 0);
 			headersize = hdr_uio->uio_resid;
 			if (compat)
 				sbytes += headersize;

sys/net/if_tap.c

@@ -827,7 +827,8 @@ tapwrite(struct cdev *dev, struct uio *uio, int flag)
 		return (EIO);
 	}
 
-	if ((m = m_uiotombuf(uio, M_DONTWAIT, 0, ETHER_ALIGN)) == NULL) {
+	if ((m = m_uiotombuf(uio, M_DONTWAIT, 0, ETHER_ALIGN,
+	    M_PKTHDR)) == NULL) {
 		ifp->if_ierrors ++;
 		return (error);
 	}

sys/net/if_tun.c

@@ -790,7 +790,7 @@ tunwrite(struct cdev *dev, struct uio *uio, int flag)
 		return (EIO);
 	}
 
-	if ((m = m_uiotombuf(uio, M_DONTWAIT, 0, 0)) == NULL) {
+	if ((m = m_uiotombuf(uio, M_DONTWAIT, 0, 0, M_PKTHDR)) == NULL) {
 		ifp->if_ierrors++;
 		return (error);
 	}

sys/net/ppp_tty.c

@@ -384,7 +384,7 @@ pppwrite(tp, uio, flag)
 	return (EMSGSIZE);
 
     s = spltty();
-    if ((m = m_uiotombuf(uio, M_DONTWAIT, 0, 0)) == NULL) {
+    if ((m = m_uiotombuf(uio, M_DONTWAIT, 0, 0, M_PKTHDR)) == NULL) {
 	splx(s);
 	return (ENOBUFS);
     }

sys/netgraph/ng_device.c

@@ -466,7 +466,7 @@ ngdwrite(struct cdev *dev, struct uio *uio, int flag)
 	if (uio->uio_resid < 0 || uio->uio_resid > IP_MAXPACKET)
 		return (EIO);
 
-	if ((m = m_uiotombuf(uio, M_DONTWAIT, 0, 0)) == NULL)
+	if ((m = m_uiotombuf(uio, M_DONTWAIT, 0, 0, M_PKTHDR)) == NULL)
 		return (ENOBUFS);
 
 	NG_SEND_DATA_ONLY(error, priv->hook, m);

sys/sys/mbuf.h

@@ -517,6 +517,8 @@ m_chtype(struct mbuf *m, short new_type)
 #define	MCLGET(m, how)		m_clget((m), (how))
 #define	MEXTADD(m, buf, size, free, args, flags, type) 			\
     m_extadd((m), (caddr_t)(buf), (size), (free), (args), (flags), (type))
+#define	m_getm(m, len, how, type)					\
+    m_getm2((m), (len), (how), (type), M_PKTHDR)
 
 /*
  * Evaluate TRUE if it's safe to write to the mbuf m's data region (this
@@ -657,7 +659,7 @@ int		 m_dup_pkthdr(struct mbuf *, struct mbuf *, int);
 u_int		 m_fixhdr(struct mbuf *);
 struct	mbuf	*m_fragment(struct mbuf *, int, int);
 void		 m_freem(struct mbuf *);
-struct	mbuf	*m_getm(struct mbuf *, int, int, short);
+struct	mbuf	*m_getm2(struct mbuf *, int, int, short, int);
 struct	mbuf	*m_getptr(struct mbuf *, int, int *);
 u_int		 m_length(struct mbuf *, struct mbuf **);
 void		 m_move_pkthdr(struct mbuf *, struct mbuf *);
@@ -667,7 +669,7 @@ struct	mbuf	*m_pulldown(struct mbuf *, int, int, int *);
 struct	mbuf	*m_pullup(struct mbuf *, int);
 int		m_sanity(struct mbuf *, int);
 struct	mbuf	*m_split(struct mbuf *, int, int);
-struct	mbuf	*m_uiotombuf(struct uio *, int, int, int);
+struct	mbuf	*m_uiotombuf(struct uio *, int, int, int, int);
 struct	mbuf	*m_unshare(struct mbuf *, int how);
 
 /*-