Rewrite of the NFS client's reply handling. We now have NFS socket

upcalls which do RPC header parsing and match up the reply with the
request. NFS calls now sleep on the nfsreq structure. This enables
us to eliminate the NFS recvlock.

Submitted by:	Mohan Srinivasan mohans at yahoo-inc dot com

sys/nfs4client/nfs4_vfsops.c

@@ -82,6 +82,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysctl.h>
 #include <sys/unistd.h>
 #include <sys/vnode.h>
+#include <sys/signalvar.h>
 
 #include <vm/vm.h>
 #include <vm/vm_extern.h>

sys/nfs4client/nfs4_vnops.c

@@ -87,6 +87,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/stat.h>
 #include <sys/sysctl.h>
 #include <sys/lockmgr.h>
+#include <sys/signalvar.h>
 
 #include <vm/vm.h>
 #include <vm/vm_extern.h>

sys/nfsclient/nfs.h

@@ -90,8 +90,6 @@
 #define NFSSTA_GOTFSINFO	0x00100000  /* Got the V3 fsinfo */
 #define	NFSSTA_SNDLOCK		0x01000000  /* Send socket lock */
 #define	NFSSTA_WANTSND		0x02000000  /* Want above */
-#define	NFSSTA_RCVLOCK		0x04000000  /* Rcv socket lock */
-#define	NFSSTA_WANTRCV		0x08000000  /* Want above */
 #define	NFSSTA_TIMEO		0x10000000  /* Experiencing a timeout */
 
 
@@ -150,19 +148,6 @@ struct buf;
 struct vattr;
 struct nameidata;
 
-/*
- * The set of signals that interrupt an I/O in progress for NFSMNT_INT mounts.
- * What should be in this set is open to debate, but I believe that since
- * I/O system calls on ufs are never interrupted by signals the set should
- * be minimal. My reasoning is that many current programs that use signals
- * such as SIGALRM will not expect file I/O system calls to be interrupted
- * by them and break.
- */
-#define	NFSINT_SIGMASK(set) 						\
-	(SIGISMEMBER(set, SIGINT) || SIGISMEMBER(set, SIGTERM) ||	\
-	 SIGISMEMBER(set, SIGHUP) || SIGISMEMBER(set, SIGKILL) ||	\
-	 SIGISMEMBER(set, SIGQUIT))
-
 /*
  * Socket errors ignored for connectionless sockets??
  * For now, ignore them all
@@ -321,6 +306,13 @@ int	nfs_fsinfo(struct nfsmount *, struct vnode *, struct ucred *,
 int	nfs_meta_setsize (struct vnode *, struct ucred *,
 	    struct thread *, u_quad_t);
 
+void    nfs_set_sigmask __P((struct thread *td, sigset_t *oldset));
+void    nfs_restore_sigmask __P((struct thread *td, sigset_t *set));
+int     nfs_tsleep __P((struct thread *td, void *ident, int priority, char *wmesg, 
+			int timo));
+int     nfs_msleep __P((struct thread *td, void *ident, struct mtx *mtx, int priority, 
+			char *wmesg, int timo));
+
 #endif	/* _KERNEL */
 
 #endif

sys/nfsclient/nfs_bio.c

@@ -1036,7 +1036,11 @@ nfs_getcacheblk(struct vnode *vp, daddr_t bn, int size, struct thread *td)
 	nmp = VFSTONFS(mp);
 
 	if (nmp->nm_flag & NFSMNT_INT) {
+ 		sigset_t oldset;
+
+ 		nfs_set_sigmask(td, &oldset);
 		bp = getblk(vp, bn, size, PCATCH, 0, 0);
+ 		nfs_restore_sigmask(td, &oldset);
 		while (bp == NULL) {
 			if (nfs_sigintr(nmp, NULL, td))
 				return (NULL);
@@ -1208,8 +1212,8 @@ nfs_asyncio(struct nfsmount *nmp, struct buf *bp, struct ucred *cred, struct thr
 			NFS_DPF(ASYNCIO,
 				("nfs_asyncio: waiting for mount %p queue to drain\n", nmp));
 			nmp->nm_bufqwant = TRUE;
-			error = tsleep(&nmp->nm_bufq, slpflag | PRIBIO,
-				       "nfsaio", slptimeo);
+ 			error = nfs_tsleep(td, &nmp->nm_bufq, slpflag | PRIBIO,
+ 					   "nfsaio", slptimeo);
 			if (error) {
 				error2 = nfs_sigintr(nmp, NULL, td);
 				if (error2)
@@ -1511,6 +1515,8 @@ nfs_meta_setsize(struct vnode *vp, struct ucred *cred, struct thread *td, u_quad
 		lbn = nsize / biosize;
 		bufsize = nsize & (biosize - 1);
 		bp = nfs_getcacheblk(vp, lbn, bufsize, td);
+ 		if (!bp)
+ 			return EINTR;
 		if (bp->b_dirtyoff > bp->b_bcount)
 			bp->b_dirtyoff = bp->b_bcount;
 		if (bp->b_dirtyend > bp->b_bcount)

sys/nfsclient/nfs_subs.c

@@ -94,6 +94,8 @@ int		nfs_ticks;
 int		nfs_pbuf_freecnt = -1;	/* start out unlimited */
 
 struct nfs_reqq	nfs_reqq;
+struct mtx nfs_reqq_mtx;
+struct mtx nfs_reply_mtx;
 struct nfs_bufq	nfs_bufq;
 
 /*
@@ -412,6 +414,8 @@ nfs_init(struct vfsconf *vfsp)
 	 */
 	TAILQ_INIT(&nfs_reqq);
 	callout_init(&nfs_callout, 0);
+	mtx_init(&nfs_reqq_mtx, "NFS reqq lock", NULL, MTX_DEF);
+	mtx_init(&nfs_reply_mtx, "Synch NFS reply posting", NULL, MTX_DEF);
 
 	nfs_pbuf_freecnt = nswbuf / 2 + 1;
 

sys/nfsclient/nfs_vfsops.c

@@ -817,6 +817,10 @@ mountnfs(struct nfs_args *argp, struct mount *mp, struct sockaddr *nam,
 
 	nfs_decode_args(nmp, argp);
 
+	if (nmp->nm_sotype == SOCK_STREAM)
+		mtx_init(&nmp->nm_nfstcpstate.mtx, "NFS/TCP state lock", 
+			 NULL, MTX_DEF);		
+
 	/*
 	 * For Connection based sockets (TCP,...) defer the connect until
 	 * the first request, in case the server is not responding.
@@ -862,6 +866,8 @@ mountnfs(struct nfs_args *argp, struct mount *mp, struct sockaddr *nam,
 
 	return (0);
 bad:
+	if (nmp->nm_sotype == SOCK_STREAM)
+		mtx_destroy(&nmp->nm_nfstcpstate.mtx);
 	nfs_disconnect(nmp);
 	uma_zfree(nfsmount_zone, nmp);
 	FREE(nam, M_SONAME);
@@ -903,6 +909,9 @@ nfs_unmount(struct mount *mp, int mntflags, struct thread *td)
 	nfs_disconnect(nmp);
 	FREE(nmp->nm_nam, M_SONAME);
 
+	if (nmp->nm_sotype == SOCK_STREAM)
+		mtx_destroy(&nmp->nm_nfstcpstate.mtx);
+	
 	uma_zfree(nfsmount_zone, nmp);
 	return (0);
 }

sys/nfsclient/nfs_vnops.c

@@ -59,6 +59,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/lockf.h>
 #include <sys/stat.h>
 #include <sys/sysctl.h>
+#include <sys/signalvar.h>
 
 #include <vm/vm.h>
 #include <vm/vm_object.h>

sys/nfsclient/nfsm_subs.h

@@ -134,7 +134,10 @@ do { \
 
 #define	nfsm_request(v, t, p, c) \
 do { \
+	sigset_t oldset; \
+	nfs_set_sigmask(p, &oldset); \
 	error = nfs_request((v), mreq, (t), (p), (c), &mrep, &md, &dpos); \
+	nfs_restore_sigmask(p, &oldset); \
 	if (error != 0) { \
 		if (error & NFSERR_RETERR) \
 			error &= ~NFSERR_RETERR; \

sys/nfsclient/nfsmount.h

@@ -36,6 +36,14 @@
 #ifndef _NFSCLIENT_NFSMOUNT_H_
 #define _NFSCLIENT_NFSMOUNT_H_
 
+struct nfs_tcp_mountstate {
+ 	int rpcresid;
+#define NFS_TCP_EXPECT_RPCMARKER 	0x0001 /* Expect to see a RPC/TCP marker next */
+#define NFS_TCP_FORCE_RECONNECT 	0x0002 /* Force a TCP reconnect */
+ 	int flags;
+	struct mtx mtx;
+};
+
 /*
  * Mount structure.
  * One allocated on every NFS mount.
@@ -79,6 +87,7 @@ struct	nfsmount {
 	struct nfs_rpcops *nm_rpcops;
 	int	nm_tprintf_initial_delay;	/* initial delay */
 	int	nm_tprintf_delay;		/* interval for messages */
+	struct nfs_tcp_mountstate nm_nfstcpstate;
 
 	/* NFSv4 */
 	uint64_t nm_clientid;