Add support for ext_pgs mbufs to nfsrv_adj().

This patch uses a slightly different algorithm for nfsrv_adj()
since ext_pgs mbuf lists are not permitted to have m_len == 0 mbufs.
As such, the code now frees mbufs after the adjustment in the list instead
of setting their m_len field to 0.
Since mbuf(s) may be trimmed off the tail of the list, the function now
returns a pointer to the last mbuf in the list.  This saves the caller
from needing to use m_last() to find the last mbuf.
It also implies that it might return a nul list, which required a check for
that in nfsrvd_readlink().

This is another in the series of commits that add support to the NFS client
and server for building RPC messages in ext_pgs mbufs with anonymous pages.
This is useful so that the entire mbuf list does not need to be
copied before calling sosend() when NFS over TLS is enabled.

Use of ext_pgs mbufs will not be enabled until the kernel RPC is updated
to handle TLS.

sys/fs/nfs/nfs_var.h

@@ -391,7 +391,7 @@ int nfsv4_fillattr(struct nfsrv_descript *, struct mount *, vnode_t, NFSACL_T *,
     struct vattr *, fhandle_t *, int, nfsattrbit_t *,
     struct ucred *, NFSPROC_T *, int, int, int, int, uint64_t, struct statfs *);
 void nfsrv_fillattr(struct nfsrv_descript *, struct nfsvattr *);
-void nfsrv_adj(struct mbuf *, int, int);
+struct mbuf *nfsrv_adj(struct mbuf *, int, int);
 void nfsrv_postopattr(struct nfsrv_descript *, int, struct nfsvattr *);
 int nfsd_errmap(struct nfsrv_descript *);
 void nfsv4_uidtostr(uid_t, u_char **, int *);

sys/fs/nfsserver/nfs_nfsdport.c

@@ -757,7 +757,12 @@ nfsvno_readlink(struct vnode *vp, struct ucred *cred, struct thread *p,
 	if (uiop->uio_resid > 0) {
 		len -= uiop->uio_resid;
 		tlen = NFSM_RNDUP(len);
-		nfsrv_adj(mp3, NFS_MAXPATHLEN - tlen, tlen - len);
+		if (tlen == 0) {
+			m_freem(mp3);
+			mp3 = mp = NULL;
+		} else if (tlen != NFS_MAXPATHLEN || tlen != len)
+			mp = nfsrv_adj(mp3, NFS_MAXPATHLEN - tlen,
+			    tlen - len);
 	}
 	*lenp = len;
 	*mpp = mp3;
@@ -872,9 +877,9 @@ nfsvno_read(struct vnode *vp, off_t off, int cnt, struct ucred *cred,
 	tlen = NFSM_RNDUP(cnt);
 	if (tlen == 0) {
 		m_freem(m3);
-		m3 = NULL;
+		m3 = m = NULL;
 	} else if (len != tlen || tlen != cnt)
-		nfsrv_adj(m3, len - tlen, tlen - cnt);
+		m = nfsrv_adj(m3, len - tlen, tlen - cnt);
 	*mpp = m3;
 	*mpendp = m;
 
@@ -6247,7 +6252,11 @@ nfsvno_getxattr(struct vnode *vp, char *name, uint32_t maxresp,
 		tlen = NFSM_RNDUP(len);
 		if (alen != tlen)
 			printf("nfsvno_getxattr: weird size read\n");
-		nfsrv_adj(m, alen - tlen, tlen - len);
+		if (tlen == 0) {
+			m_freem(m);
+			m = m2 = NULL;
+		} else if (alen != tlen || tlen != len)
+			m2 = nfsrv_adj(m, alen - tlen, tlen - len);
 	}
 	*lenp = len;
 	*mpp = m;

sys/fs/nfsserver/nfs_nfsdserv.c

@@ -690,9 +690,11 @@ nfsrvd_readlink(struct nfsrv_descript *nd, __unused int isdgram,
 		goto out;
 	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
 	*tl = txdr_unsigned(len);
-	nd->nd_mb->m_next = mp;
+	if (mp != NULL) {
-	nd->nd_mb = mpend;
+		nd->nd_mb->m_next = mp;
-	nd->nd_bpos = mtod(mpend, caddr_t) + mpend->m_len;
+		nd->nd_mb = mpend;
+		nd->nd_bpos = mtod(mpend, caddr_t) + mpend->m_len;
+	}
 
 out:
 	NFSEXITCODE2(0, nd);

sys/fs/nfsserver/nfs_nfsdsubs.c

@@ -1268,62 +1268,100 @@ static short *nfsrv_v4errmap[] = {
 };
 
 /*
- * A fiddled version of m_adj() that ensures null fill to a long
+ * Trim tlen bytes off the end of the mbuf list and then ensure
- * boundary and only trims off the back end
+ * the end of the last mbuf is nul filled to a long boundary,
+ * as indicated by the value of "nul".
+ * Return the last mbuf in the updated list and free and mbufs
+ * that follow it in the original list.
+ * This is somewhat different than the old nfsrv_adj() with
+ * support for ext_pgs mbufs.  It frees the remaining mbufs
+ * instead of setting them 0 length, since lists of ext_pgs
+ * mbufs are all expected to be non-empty.
  */
-void
+struct mbuf *
 nfsrv_adj(struct mbuf *mp, int len, int nul)
 {
-	struct mbuf *m;
+	struct mbuf *m, *m2;
-	int count, i;
+	vm_page_t pg;
+	int i, lastlen, pgno, plen, tlen, trim;
+	uint16_t off;
 	char *cp;
 
 	/*
-	 * Trim from tail.  Scan the mbuf chain,
+	 * Find the last mbuf after adjustment and
-	 * calculating its length and finding the last mbuf.
+	 * how much it needs to be adjusted by.
-	 * If the adjustment only affects this mbuf, then just
-	 * adjust and return.  Otherwise, rescan and truncate
-	 * after the remaining size.
 	 */
-	count = 0;
+	tlen = 0;
 	m = mp;
 	for (;;) {
-		count += m->m_len;
+		tlen += m->m_len;
 		if (m->m_next == NULL)
 			break;
 		m = m->m_next;
 	}
-	if (m->m_len > len) {
+	/* m is now the last mbuf and tlen the total length. */
-		m->m_len -= len;
+
-		if (nul > 0) {
+	if (len >= m->m_len) {
-			cp = mtod(m, caddr_t) + m->m_len - nul;
+		/* Need to trim away the last mbuf(s). */
-			for (i = 0; i < nul; i++)
+		i = tlen - len;
-				*cp++ = '\0';
+		m = mp;
+		for (;;) {
+			if (m->m_len >= i)
+				break;
+			i -= m->m_len;
+			m = m->m_next;
 		}
-		return;
+		lastlen = i;
-	}
+	} else
-	count -= len;
+		lastlen = m->m_len - len;
-	if (count < 0)
+
-		count = 0;
 	/*
-	 * Correct length for chain is "count".
+	 * m is now the last mbuf after trimming and its length needs to
-	 * Find the mbuf with last data, adjust its length,
+	 * be lastlen.
-	 * and toss data from remaining mbufs on chain.
+	 * Adjust the last mbuf and set cp to point to where nuls must be
+	 * written.
 	 */
-	for (m = mp; m; m = m->m_next) {
+	if ((m->m_flags & M_EXTPG) != 0) {
-		if (m->m_len >= count) {
+		pgno = m->m_epg_npgs - 1;
-			m->m_len = count;
+		off = (pgno == 0) ? m->m_epg_1st_off : 0;
-			if (nul > 0) {
+		plen = m_epg_pagelen(m, pgno, off);
-				cp = mtod(m, caddr_t) + m->m_len - nul;
+		if (m->m_len > lastlen) {
-				for (i = 0; i < nul; i++)
+			/* Trim this mbuf. */
-					*cp++ = '\0';
+			trim = m->m_len - lastlen;
+			while (trim >= plen) {
+				KASSERT(pgno > 0,
+				    ("nfsrv_adj: freeing page 0"));
+				/* Free page. */
+				pg = PHYS_TO_VM_PAGE(m->m_epg_pa[pgno]);
+				vm_page_unwire_noq(pg);
+				vm_page_free(pg);
+				trim -= plen;
+				m->m_epg_npgs--;
+				pgno--;
+				off = (pgno == 0) ? m->m_epg_1st_off : 0;
+				plen = m_epg_pagelen(m, pgno, off);
 			}
-			break;
+			plen -= trim;
+			m->m_epg_last_len = plen;
+			m->m_len = lastlen;
 		}
-		count -= m->m_len;
+		cp = (char *)(void *)PHYS_TO_DMAP(m->m_epg_pa[pgno]);
+		cp += off + plen - nul;
+	} else {
+		m->m_len = lastlen;
+		cp = mtod(m, char *) + m->m_len - nul;
 	}
-	for (m = m->m_next; m; m = m->m_next)
+
-		m->m_len = 0;
+	/* Write the nul bytes. */
+	for (i = 0; i < nul; i++)
+		*cp++ = '\0';
+
+	/* Free up any mbufs past "m". */
+	m2 = m->m_next;
+	m->m_next = NULL;
+	if (m2 != NULL)
+		m_freem(m2);
+	return (m);
 }
 
 /*