Add flags to enable NFS over TLS to the NFS client and server.

An Internet Draft titled "Towards Remote Procedure Call Encryption By Default"
(soon to be an RFC I think) describes how Sun RPC is to use TLS with NFS
as a specific application case.
Various commits prepared the NFS code to use KERN_TLS, mainly enabling use
of ext_pgs mbufs for large RPC messages.
r364475 added TLS support to the kernel RPC.

This commit (which is the final one for kernel changes required to do
NFS over TLS) adds support for three export flags:
MNT_EXTLS - Requires a TLS connection.
MNT_EXTLSCERT - Requires a TLS connection where the client presents a valid
            X.509 certificate during TLS handshake.
MNT_EXTLSCERTUSER - Requires a TLS connection where the client presents a
            valid X.509 certificate with "user@domain" in the otherName
            field of the SubjectAltName during TLS handshake.
Without these export options, clients are permitted, but not required, to
use TLS.

For the client, a new nmount(2) option called "tls" makes the client do
a STARTTLS Null RPC and TLS handshake for all TCP connections used for the
mount. The CLSET_TLS client control option is used to indicate to the kernel RPC
that this should be done.

Unless the above export flags or "tls" option is used, semantics should
not change for the NFS client nor server.

For NFS over TLS to work, the userspace daemons rpctlscd(8) { for client }
or rpctlssd(8) daemon { for server } must be running.

sys/fs/nfs/nfs_commonkrpc.c

@@ -281,6 +281,8 @@ newnfs_connect(struct nfsmount *nmp, struct nfssockreq *nrp,
 			CLNT_CONTROL(client, CLSET_INTERRUPTIBLE, &one);
 		if ((nmp->nm_flag & NFSMNT_RESVPORT))
 			CLNT_CONTROL(client, CLSET_PRIVPORT, &one);
+		if (NFSHASTLS(nmp))
+			CLNT_CONTROL(client, CLSET_TLS, &one);
 		if (NFSHASSOFT(nmp)) {
 			if (nmp->nm_sotype == SOCK_DGRAM)
 				/*

sys/fs/nfs/nfsdport.h

@@ -81,6 +81,9 @@ struct nfsexstuff {
 #define	NFSVNO_EXPORTANON(e)		((e)->nes_exflag & MNT_EXPORTANON)
 #define	NFSVNO_EXSTRICTACCESS(e)	((e)->nes_exflag & MNT_EXSTRICTACCESS)
 #define	NFSVNO_EXV4ONLY(e)		((e)->nes_exflag & MNT_EXV4ONLY)
+#define	NFSVNO_EXTLS(e)			((e)->nes_exflag & MNT_EXTLS)
+#define	NFSVNO_EXTLSCERT(e)		((e)->nes_exflag & MNT_EXTLSCERT)
+#define	NFSVNO_EXTLSCERTUSER(e)		((e)->nes_exflag & MNT_EXTLSCERTUSER)
 
 #define	NFSVNO_SETEXRDONLY(e)	((e)->nes_exflag = (MNT_EXPORTED|MNT_EXRDONLY))
 

sys/fs/nfs/nfsport.h

@@ -1055,6 +1055,7 @@ bool ncl_pager_setsize(struct vnode *vp, u_quad_t *nsizep);
 #define	NFSHASOPENMODE(n)	((n)->nm_state & NFSSTA_OPENMODE)
 #define	NFSHASONEOPENOWN(n)	(((n)->nm_flag & NFSMNT_ONEOPENOWN) != 0 &&	\
 				    (n)->nm_minorvers > 0)
+#define	NFSHASTLS(n)		(((n)->nm_newflag & NFSMNT_TLS) != 0)
 
 /*
  * Set boottime.

sys/fs/nfsclient/nfs_clkrpc.c

@@ -37,12 +37,14 @@
 __FBSDID("$FreeBSD$");
 
 #include "opt_kgssapi.h"
+#include "opt_kern_tls.h"
 
 #include <fs/nfs/nfsport.h>
 
 #include <rpc/rpc.h>
-#include <rpc/rpcsec_gss.h>
 #include <rpc/replay.h>
+#include <rpc/rpcsec_gss.h>
+#include <rpc/rpcsec_tls.h>
 
 
 NFSDLOCKMUTEX;
@@ -67,6 +69,9 @@ nfscb_program(struct svc_req *rqst, SVCXPRT *xprt)
 {
 	struct nfsrv_descript nd;
 	int cacherep, credflavor;
+#ifdef KERN_TLS
+	u_int maxlen;
+#endif
 
 	memset(&nd, 0, sizeof(nd));
 	if (rqst->rq_proc != NFSPROC_NULL &&
@@ -107,6 +112,13 @@ nfscb_program(struct svc_req *rqst, SVCXPRT *xprt)
 #ifdef MAC
 		mac_cred_associate_nfsd(nd.nd_cred);
 #endif
+#endif
+#ifdef KERN_TLS
+		if ((xprt->xp_tls & RPCTLS_FLAGS_HANDSHAKE) != 0 &&
+		    rpctls_getinfo(&maxlen, false, false)) {
+			nd.nd_flag |= ND_EXTPG;
+			nd.nd_maxextsiz = maxlen;
+		}
 #endif
 		cacherep = nfs_cbproc(&nd, rqst->rq_xid);
 	} else {

sys/fs/nfsclient/nfs_clvfsops.c

@@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
 
 #include "opt_bootp.h"
 #include "opt_nfsroot.h"
+#include "opt_kern_tls.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -77,6 +78,8 @@ __FBSDID("$FreeBSD$");
 #include <fs/nfsclient/nfs.h>
 #include <nfs/nfsdiskless.h>
 
+#include <rpc/rpcsec_tls.h>
+
 FEATURE(nfscl, "NFSv4 client");
 
 extern int nfscl_ticks;
@@ -117,7 +120,7 @@ static void	nfs_decode_args(struct mount *mp, struct nfsmount *nmp,
 static int	mountnfs(struct nfs_args *, struct mount *,
 		    struct sockaddr *, char *, u_char *, int, u_char *, int,
 		    u_char *, int, struct vnode **, struct ucred *,
-		    struct thread *, int, int, int);
+		    struct thread *, int, int, int, uint32_t);
 static void	nfs_getnlminfo(struct vnode *, uint8_t *, size_t *,
 		    struct sockaddr_storage *, int *, off_t *,
 		    struct timeval *);
@@ -544,7 +547,7 @@ nfs_mountdiskless(char *path,
 	nam = sodupsockaddr((struct sockaddr *)sin, M_WAITOK);
 	if ((error = mountnfs(args, mp, nam, path, NULL, 0, dirpath, dirlen,
 	    NULL, 0, vpp, td->td_ucred, td, NFS_DEFAULT_NAMETIMEO, 
-	    NFS_DEFAULT_NEGNAMETIMEO, 0)) != 0) {
+	    NFS_DEFAULT_NEGNAMETIMEO, 0, 0)) != 0) {
 		printf("nfs_mountroot: mount %s on /: %d\n", path, error);
 		return (error);
 	}
@@ -746,7 +749,7 @@ static const char *nfs_opts[] = { "from", "nfs_args",
     "resvport", "readahead", "hostname", "timeo", "timeout", "addr", "fh",
     "nfsv3", "sec", "principal", "nfsv4", "gssname", "allgssname", "dirpath",
     "minorversion", "nametimeo", "negnametimeo", "nocto", "noncontigwr",
-    "pnfs", "wcommitsize", "oneopenown",
+    "pnfs", "wcommitsize", "oneopenown", "tls",
     NULL };
 
 /*
@@ -897,9 +900,11 @@ nfs_mount(struct mount *mp)
 	int dirlen, has_nfs_args_opt, has_nfs_from_opt,
 	    krbnamelen, srvkrbnamelen;
 	size_t hstlen;
+	uint32_t newflag;
 
 	has_nfs_args_opt = 0;
 	has_nfs_from_opt = 0;
+	newflag = 0;
 	hst = malloc(MNAMELEN, M_TEMP, M_WAITOK);
 	if (vfs_filteropt(mp->mnt_optnew, nfs_opts)) {
 		error = EINVAL;
@@ -983,6 +988,8 @@ nfs_mount(struct mount *mp)
 		args.flags |= NFSMNT_PNFS;
 	if (vfs_getopt(mp->mnt_optnew, "oneopenown", NULL, NULL) == 0)
 		args.flags |= NFSMNT_ONEOPENOWN;
+	if (vfs_getopt(mp->mnt_optnew, "tls", NULL, NULL) == 0)
+		newflag |= NFSMNT_TLS;
 	if (vfs_getopt(mp->mnt_optnew, "readdirsize", (void **)&opt, NULL) == 0) {
 		if (opt == NULL) { 
 			vfs_mount_error(mp, "illegal readdirsize");
@@ -1337,7 +1344,7 @@ nfs_mount(struct mount *mp)
 	args.fh = nfh;
 	error = mountnfs(&args, mp, nam, hst, krbname, krbnamelen, dirpath,
 	    dirlen, srvkrbname, srvkrbnamelen, &vp, td->td_ucred, td,
-	    nametimeo, negnametimeo, minvers);
+	    nametimeo, negnametimeo, minvers, newflag);
 out:
 	if (!error) {
 		MNT_ILOCK(mp);
@@ -1386,7 +1393,7 @@ mountnfs(struct nfs_args *argp, struct mount *mp, struct sockaddr *nam,
     char *hst, u_char *krbname, int krbnamelen, u_char *dirpath, int dirlen,
     u_char *srvkrbname, int srvkrbnamelen, struct vnode **vpp,
     struct ucred *cred, struct thread *td, int nametimeo, int negnametimeo,
-    int minvers)
+    int minvers, uint32_t newflag)
 {
 	struct nfsmount *nmp;
 	struct nfsnode *np;
@@ -1396,6 +1403,9 @@ mountnfs(struct nfs_args *argp, struct mount *mp, struct sockaddr *nam,
 	struct nfsclds *dsp, *tdsp;
 	uint32_t lease;
 	static u_int64_t clval = 0;
+#ifdef KERN_TLS
+	u_int maxlen;
+#endif
 
 	NFSCL_DEBUG(3, "in mnt\n");
 	clp = NULL;
@@ -1405,9 +1415,22 @@ mountnfs(struct nfs_args *argp, struct mount *mp, struct sockaddr *nam,
 		free(nam, M_SONAME);
 		return (0);
 	} else {
+		/* NFS-over-TLS requires that rpctls be functioning. */
+		if ((newflag & NFSMNT_TLS) != 0) {
+			error = EINVAL;
+#ifdef KERN_TLS
+			if (rpctls_getinfo(&maxlen, true, false))
+				error = 0;
+#endif
+			if (error != 0) {
+				free(nam, M_SONAME);
+				return (error);
+			}
+		}
 		nmp = malloc(sizeof (struct nfsmount) +
 		    krbnamelen + dirlen + srvkrbnamelen + 2,
 		    M_NEWNFSMNT, M_WAITOK | M_ZERO);
+		nmp->nm_newflag = newflag;
 		TAILQ_INIT(&nmp->nm_bufq);
 		TAILQ_INIT(&nmp->nm_sess);
 		if (clval == 0)
@@ -2011,6 +2034,8 @@ void nfscl_retopts(struct nfsmount *nmp, char *buffer, size_t buflen)
 	nfscl_printopt(nmp, nmp->nm_sotype != SOCK_STREAM, ",udp", &buf, &blen);
 	nfscl_printopt(nmp, (nmp->nm_flag & NFSMNT_RESVPORT) != 0, ",resvport",
 	    &buf, &blen);
+	nfscl_printopt(nmp, (nmp->nm_newflag & NFSMNT_TLS) != 0, ",tls", &buf,
+	    &blen);
 	nfscl_printopt(nmp, (nmp->nm_flag & NFSMNT_NOCONN) != 0, ",noconn",
 	    &buf, &blen);
 	nfscl_printopt(nmp, (nmp->nm_flag & NFSMNT_SOFT) == 0, ",hard", &buf,

sys/fs/nfsclient/nfsmount.h

@@ -47,6 +47,7 @@
 struct	nfsmount {
 	struct	nfsmount_common nm_com;	/* Common fields for nlm */
 	uint32_t nm_privflag;		/* Private flags */
+	uint32_t nm_newflag;		/* New mount flags */
 	int	nm_numgrps;		/* Max. size of groupslist */
 	u_char	nm_fh[NFSX_FHMAX];	/* File handle of root dir */
 	int	nm_fhsize;		/* Size of root file handle */
@@ -114,6 +115,9 @@ struct	nfsmount {
 #define	NFSMNTP_NOADVISE	0x00000100
 #define	NFSMNTP_NOALLOCATE	0x00000200
 
+/* New mount flags only used by the kernel via nmount(2). */
+#define	NFSMNT_TLS		0x00000001
+
 #define	NFSMNT_DIRPATH(m)	(&((m)->nm_name[(m)->nm_krbnamelen + 1]))
 #define	NFSMNT_SRVKRBNAME(m)						\
 	(&((m)->nm_name[(m)->nm_krbnamelen + (m)->nm_dirpathlen + 2]))

sys/fs/nfsserver/nfs_nfsdkrpc.c

@@ -38,11 +38,13 @@ __FBSDID("$FreeBSD$");
 
 #include "opt_inet6.h"
 #include "opt_kgssapi.h"
+#include "opt_kern_tls.h"
 
 #include <fs/nfs/nfsport.h>
 
 #include <rpc/rpc.h>
 #include <rpc/rpcsec_gss.h>
+#include <rpc/rpcsec_tls.h>
 
 #include <fs/nfsserver/nfs_fha_new.h>
 
@@ -120,6 +122,9 @@ nfssvc_program(struct svc_req *rqst, SVCXPRT *xprt)
 	struct nfsrv_descript nd;
 	struct nfsrvcache *rp = NULL;
 	int cacherep, credflavor;
+#ifdef KERN_TLS
+	u_int maxlen;
+#endif
 
 	memset(&nd, 0, sizeof(nd));
 	if (rqst->rq_vers == NFS_VER2) {
@@ -234,6 +239,14 @@ nfssvc_program(struct svc_req *rqst, SVCXPRT *xprt)
 			goto out;
 		}
 
+		if ((xprt->xp_tls & RPCTLS_FLAGS_HANDSHAKE) != 0) {
+			nd.nd_flag |= ND_TLS;
+			if ((xprt->xp_tls & RPCTLS_FLAGS_VERIFIED) != 0)
+				nd.nd_flag |= ND_TLSCERT;
+			if ((xprt->xp_tls & RPCTLS_FLAGS_CERTUSER) != 0)
+				nd.nd_flag |= ND_TLSCERTUSER;
+		}
+		nd.nd_maxextsiz = 16384;
 #ifdef MAC
 		mac_cred_associate_nfsd(nd.nd_cred);
 #endif
@@ -268,6 +281,11 @@ nfssvc_program(struct svc_req *rqst, SVCXPRT *xprt)
 			}
 		}
 
+#ifdef KERN_TLS
+		if ((xprt->xp_tls & RPCTLS_FLAGS_HANDSHAKE) != 0 &&
+		    rpctls_getinfo(&maxlen, false, false))
+			nd.nd_maxextsiz = maxlen;
+#endif
 		cacherep = nfs_proc(&nd, rqst->rq_xid, xprt, &rp);
 		NFSLOCKV4ROOTMUTEX();
 		nfsv4_relref(&nfsd_suspend_lock);

sys/fs/nfsserver/nfs_nfsdport.c

@@ -3283,6 +3283,19 @@ nfsd_fhtovp(struct nfsrv_descript *nd, struct nfsrvfh *nfp, int lktype,
 		nd->nd_repstat = EACCES;
 	}
 
+	/*
+	 * If TLS is required by the export, check the flags in nd_flag.
+	 */
+	if (nd->nd_repstat == 0 && ((NFSVNO_EXTLS(exp) &&
+	    (nd->nd_flag & ND_TLS) == 0) ||
+	     (NFSVNO_EXTLSCERT(exp) &&
+	      (nd->nd_flag & ND_TLSCERT) == 0) ||
+	     (NFSVNO_EXTLSCERTUSER(exp) &&
+	      (nd->nd_flag & ND_TLSCERTUSER) == 0))) {
+		vput(*vpp);
+		nd->nd_repstat = NFSERR_ACCES;
+	}
+
 	/*
 	 * Personally, I've never seen any point in requiring a
 	 * reserved port#, since only in the rare case where the
@@ -3547,6 +3560,15 @@ nfsvno_v4rootexport(struct nfsrv_descript *nd)
 			nd->nd_flag |= ND_EXGSSPRIVACY;
 	}
 
+	/* And set ND_EXxx flags for TLS. */
+	if ((exflags & MNT_EXTLS) != 0) {
+		nd->nd_flag |= ND_EXTLS;
+		if ((exflags & MNT_EXTLSCERT) != 0)
+			nd->nd_flag |= ND_EXTLSCERT;
+		if ((exflags & MNT_EXTLSCERTUSER) != 0)
+			nd->nd_flag |= ND_EXTLSCERTUSER;
+	}
+
 out:
 	NFSEXITCODE(error);
 	return (error);

sys/fs/nfsserver/nfs_nfsdserv.c

@@ -3816,6 +3816,11 @@ nfsrvd_setclientid(struct nfsrv_descript *nd, __unused int isdgram,
 		clp->lc_uid = nd->nd_cred->cr_uid;
 		clp->lc_gid = nd->nd_cred->cr_gid;
 	}
+
+	/* If the client is using TLS, do so for the callback connection. */
+	if (nd->nd_flag & ND_TLS)
+		clp->lc_flags |= LCL_TLSCB;
+
 	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
 	clp->lc_program = fxdr_unsigned(u_int32_t, *tl);
 	error = nfsrv_getclientipaddr(nd, clp);

sys/fs/nfsserver/nfs_nfsdsubs.c

@@ -2114,15 +2114,28 @@ nfsd_checkrootexp(struct nfsrv_descript *nd)
 {
 
 	if ((nd->nd_flag & (ND_GSS | ND_EXAUTHSYS)) == ND_EXAUTHSYS)
-		return (0);
+		goto checktls;
 	if ((nd->nd_flag & (ND_GSSINTEGRITY | ND_EXGSSINTEGRITY)) ==
 	    (ND_GSSINTEGRITY | ND_EXGSSINTEGRITY))
-		return (0);
+		goto checktls;
 	if ((nd->nd_flag & (ND_GSSPRIVACY | ND_EXGSSPRIVACY)) ==
 	    (ND_GSSPRIVACY | ND_EXGSSPRIVACY))
-		return (0);
+		goto checktls;
 	if ((nd->nd_flag & (ND_GSS | ND_GSSINTEGRITY | ND_GSSPRIVACY |
 	     ND_EXGSS)) == (ND_GSS | ND_EXGSS))
+		goto checktls;
+	return (1);
+checktls:
+	if ((nd->nd_flag & ND_EXTLS) == 0)
+		return (0);
+	if ((nd->nd_flag & (ND_TLSCERTUSER | ND_EXTLSCERTUSER)) ==
+	    (ND_TLSCERTUSER | ND_EXTLSCERTUSER))
+		return (0);
+	if ((nd->nd_flag & (ND_TLSCERT | ND_EXTLSCERT | ND_EXTLSCERTUSER)) ==
+	    (ND_TLSCERT | ND_EXTLSCERT))
+		return (0);
+	if ((nd->nd_flag & (ND_TLS | ND_EXTLSCERTUSER | ND_EXTLSCERT)) ==
+	    ND_TLS)
 		return (0);
 	return (1);
 }