Clean up properly when unloading NFS client module.

This includes a modified form of some code from Thomas Moestl (tmm@)
to properly clean up the UMA zone and the "nfsnodehashtbl" hash
table.

Reviewed By:	iedowse
PR:		16299

sys/nfsclient/nfs.h

@@ -136,6 +136,7 @@ extern struct callout nfs_callout;
 extern struct nfsstats nfsstats;
 
 extern int nfs_numasync;
+extern unsigned int nfs_iodmax;
 extern int nfs_pbuf_freecnt;
 extern int nfs_ticks;
 
@@ -295,6 +296,7 @@ int	nfs_loadattrcache(struct vnode **, struct mbuf **, caddr_t *,
 	    struct vattr *, int);
 int	nfsm_mbuftouio(struct mbuf **, struct uio *, int, caddr_t *);
 void	nfs_nhinit(void);
+void	nfs_nhuninit(void);
 int	nfs_nmcancelreqs(struct nfsmount *);
 void	nfs_timer(void*);
 

sys/nfsclient/nfs_nfsiod.c

@@ -87,7 +87,7 @@ static unsigned int nfs_iodmaxidle = 120;
 SYSCTL_UINT(_vfs_nfs, OID_AUTO, iodmaxidle, CTLFLAG_RW, &nfs_iodmaxidle, 0, "");
 
 /* Maximum number of nfsiod kthreads */
-static unsigned int nfs_iodmax = 20;
+unsigned int nfs_iodmax = 20;
 
 /* Minimum number of nfsiod kthreads to keep as spares */
 static unsigned int nfs_iodmin = 4;
@@ -280,7 +280,9 @@ finish:
 	    nmp->nm_bufqiods--;
 	nfs_iodwant[myiod] = NULL;
 	nfs_iodmount[myiod] = NULL;
-	nfs_numasync--;
+	/* Someone may be waiting for the last nfsiod to terminate. */
+	if (--nfs_numasync == 0)
+		wakeup(&nfs_numasync);
 	if ((error == 0) || (error == EWOULDBLOCK))
 		kthread_exit(0);
 	/* Abnormal termination */

sys/nfsclient/nfs_node.c

@@ -160,6 +160,16 @@ nfs_nhinit(void)
 	nfsnodehashtbl = hashinit(desiredvnodes, M_NFSHASH, &nfsnodehash);
 }
 
+/*
+ * Release hash table resources
+ */
+void
+nfs_nhuninit(void)
+{
+	hashdestroy(nfsnodehashtbl, M_NFSHASH, nfsnodehash);
+	uma_zdestroy(nfsnode_zone);
+}
+
 /*
  * Look up a vnode/nfsnode by file handle.
  * Callers must check for mount points!!

sys/nfsclient/nfs_subs.c

@@ -429,10 +429,29 @@ nfs_init(struct vfsconf *vfsp)
 int
 nfs_uninit(struct vfsconf *vfsp)
 {
+	int i;
 
 	callout_stop(&nfs_callout);
 	sysent[SYS_nfsclnt].sy_narg = nfs_prev_nfsclnt_sy_narg;
 	sysent[SYS_nfsclnt].sy_call = nfs_prev_nfsclnt_sy_call;
+
+	KASSERT(TAILQ_ISEMPTY(&nfs_reqq),
+	    ("nfs_uninit: request queue not empty"));
+
+	/*
+	 * Tell all nfsiod processes to exit. Clear nfs_iodmax, and wakeup
+	 * any sleeping nfsiods so they check nfs_iodmax and exit.
+	 */
+	nfs_iodmax = 0;
+	for (i = 0; i < nfs_numasync; i++)
+		if (nfs_iodwant[i])
+			wakeup(&nfs_iodwant[i]);
+	/* The last nfsiod to exit will wake us up when nfs_numasync hits 0 */
+	while (nfs_numasync)
+		tsleep(&nfs_numasync, PWAIT, "ioddie", 0);
+
+	nfs_nhuninit();
+	uma_zdestroy(nfsmount_zone);
 	return (0);
 }