Partially decompose priv_check by adding priv_check_cred_vfs_generation

During buildkernel there are very frequent calls to priv_check and they
all are for PRIV_VFS_GENERATION (coming from stat/fstat).

This results in branching on several potential privileges checking if
perhaps that's the one which has to be evaluated.

Instead of the kitchen-sink approach provide a way to have commonly used
privs directly evaluated.

sys/kern/kern_jail.c

@@ -2998,6 +2998,16 @@ int
 prison_priv_check(struct ucred *cred, int priv)
 {
 
+	/*
+	 * Some policies have custom handlers. This routine should not be
+	 * called for them. See priv_check_cred().
+	 */
+	switch (priv) {
+	case PRIV_VFS_GENERATION:
+		KASSERT(0, ("prison_priv_check instead of a custom handler "
+		    "called for %d\n", priv));
+	}
+
 	if (!jailed(cred))
 		return (0);
 

sys/kern/kern_priv.c

@@ -71,6 +71,51 @@ SDT_PROVIDER_DEFINE(priv);
 SDT_PROBE_DEFINE1(priv, kernel, priv_check, priv__ok, "int");
 SDT_PROBE_DEFINE1(priv, kernel, priv_check, priv__err, "int");
 
+static __always_inline int
+priv_check_cred_pre(struct ucred *cred, int priv)
+{
+	int error;
+
+#ifdef MAC
+	error = mac_priv_check(cred, priv);
+#else
+	error = 0;
+#endif
+	return (error);
+}
+
+static __always_inline int
+priv_check_cred_post(struct ucred *cred, int priv, int error, bool handled)
+{
+
+	if (__predict_true(handled))
+		goto out;
+	/*
+	 * Now check with MAC, if enabled, to see if a policy module grants
+	 * privilege.
+	 */
+#ifdef MAC
+	if (mac_priv_grant(cred, priv) == 0) {
+		error = 0;
+		goto out;
+	}
+#endif
+
+	/*
+	 * The default is deny, so if no policies have granted it, reject
+	 * with a privilege error here.
+	 */
+	error = EPERM;
+out:
+	if (SDT_PROBES_ENABLED()) {
+		if (error)
+			SDT_PROBE1(priv, kernel, priv_check, priv__err, priv);
+		else
+			SDT_PROBE1(priv, kernel, priv_check, priv__ok, priv);
+	}
+	return (error);
+}
+
 /*
  * Check a credential for privilege.  Lots of good reasons to deny privilege;
  * only a few to grant it.
@@ -83,15 +128,18 @@ priv_check_cred(struct ucred *cred, int priv)
 	KASSERT(PRIV_VALID(priv), ("priv_check_cred: invalid privilege %d",
 	    priv));
 
+	switch (priv) {
+	case PRIV_VFS_GENERATION:
+		return (priv_check_cred_vfs_generation(cred));
+	}
+
 	/*
 	 * We first evaluate policies that may deny the granting of
 	 * privilege unilaterally.
 	 */
-#ifdef MAC
-	error = mac_priv_check(cred, priv);
+	error = priv_check_cred_pre(cred, priv);
 	if (error)
 		goto out;
-#endif
 
 	/*
 	 * Jail policy will restrict certain privileges that may otherwise be
@@ -177,30 +225,9 @@ priv_check_cred(struct ucred *cred, int priv)
 		}
 	}
 
-	/*
-	 * Now check with MAC, if enabled, to see if a policy module grants
-	 * privilege.
-	 */
-#ifdef MAC
-	if (mac_priv_grant(cred, priv) == 0) {
-		error = 0;
-		goto out;
-	}
-#endif
-
-	/*
-	 * The default is deny, so if no policies have granted it, reject
-	 * with a privilege error here.
-	 */
-	error = EPERM;
+	return (priv_check_cred_post(cred, priv, error, false));
 out:
-	if (SDT_PROBES_ENABLED()) {
-		if (error)
-			SDT_PROBE1(priv, kernel, priv_check, priv__err, priv);
-		else
-			SDT_PROBE1(priv, kernel, priv_check, priv__ok, priv);
-	}
-	return (error);
+	return (priv_check_cred_post(cred, priv, error, true));
 }
 
 int
@@ -211,3 +238,28 @@ priv_check(struct thread *td, int priv)
 
 	return (priv_check_cred(td->td_ucred, priv));
 }
+
+int
+priv_check_cred_vfs_generation(struct ucred *cred)
+{
+	int error;
+
+	error = priv_check_cred_pre(cred, PRIV_VFS_GENERATION);
+	if (error)
+		goto out;
+
+	if (jailed(cred)) {
+		error = EPERM;
+		goto out;
+	}
+
+	if (cred->cr_uid == 0 && suser_enabled) {
+		error = 0;
+		goto out;
+	}
+
+	return (priv_check_cred_post(cred, PRIV_VFS_GENERATION, error, false));
+out:
+	return (priv_check_cred_post(cred, PRIV_VFS_GENERATION, error, true));
+
+}

sys/kern/vfs_syscalls.c

@@ -273,7 +273,7 @@ kern_do_statfs(struct thread *td, struct mount *mp, struct statfs *buf)
 	error = VFS_STATFS(mp, buf);
 	if (error != 0)
 		goto out;
-	if (priv_check(td, PRIV_VFS_GENERATION)) {
+	if (priv_check_cred_vfs_generation(td->td_ucred)) {
 		buf->f_fsid.val[0] = buf->f_fsid.val[1] = 0;
 		prison_enforce_statfs(td->td_ucred, mp, buf);
 	}
@@ -488,7 +488,7 @@ kern_getfsstat(struct thread *td, struct statfs **buf, size_t bufsize,
 					continue;
 				}
 			}
-			if (priv_check(td, PRIV_VFS_GENERATION)) {
+			if (priv_check_cred_vfs_generation(td->td_ucred)) {
 				sptmp = malloc(sizeof(struct statfs), M_STATFS,
 				    M_WAITOK);
 				*sptmp = *sp;

sys/kern/vfs_vnops.c

@@ -1477,7 +1477,7 @@ vn_stat(struct vnode *vp, struct stat *sb, struct ucred *active_cred,
 	sb->st_blksize = max(PAGE_SIZE, vap->va_blocksize);
 	
 	sb->st_flags = vap->va_flags;
-	if (priv_check(td, PRIV_VFS_GENERATION))
+	if (priv_check_cred_vfs_generation(td->td_ucred))
 		sb->st_gen = 0;
 	else
 		sb->st_gen = vap->va_gen;

sys/sys/priv.h

@@ -533,6 +533,7 @@ struct thread;
 struct ucred;
 int	priv_check(struct thread *td, int priv);
 int	priv_check_cred(struct ucred *cred, int priv);
+int	priv_check_cred_vfs_generation(struct ucred *cred);
 #endif
 
 #endif /* !_SYS_PRIV_H_ */