vfs: use mac fastpath for lookup, open, read, write, mmap

svn path=/head/; revision=357889

sys/kern/vfs_lookup.c

@@ -932,12 +932,9 @@ lookup(struct nameidata *ndp)
 	 */
 unionlookup:
 #ifdef MAC
-	if ((cnp->cn_flags & NOMACCHECK) == 0) {
+	error = mac_vnode_check_lookup(cnp->cn_thread->td_ucred, dp, cnp);
-		error = mac_vnode_check_lookup(cnp->cn_thread->td_ucred, dp,
+	if (error)
-		    cnp);
+		goto bad;
-		if (error)
-			goto bad;
-	}
 #endif
 	ndp->ni_dvp = dp;
 	ndp->ni_vp = NULL;

sys/security/mac/mac_framework.c

@@ -125,6 +125,12 @@ bool __read_frequently mac_##f##_fp_flag
 
 FPFLAG(priv_check);
 FPFLAG(priv_grant);
+FPFLAG(vnode_check_lookup);
+FPFLAG(vnode_check_open);
+FPFLAG(vnode_check_stat);
+FPFLAG(vnode_check_read);
+FPFLAG(vnode_check_write);
+FPFLAG(vnode_check_mmap);
 
 #undef FPFLAG
 
@@ -403,6 +409,18 @@ struct mac_policy_fastpath_elem {
 struct mac_policy_fastpath_elem mac_policy_fastpath_array[] = {
 	{ .offset = FPO(priv_check), .flag = &mac_priv_check_fp_flag },
 	{ .offset = FPO(priv_grant), .flag = &mac_priv_grant_fp_flag },
+	{ .offset = FPO(vnode_check_lookup),
+		.flag = &mac_vnode_check_lookup_fp_flag },
+	{ .offset = FPO(vnode_check_open),
+		.flag = &mac_vnode_check_open_fp_flag },
+	{ .offset = FPO(vnode_check_stat),
+		.flag = &mac_vnode_check_stat_fp_flag },
+	{ .offset = FPO(vnode_check_read),
+		.flag = &mac_vnode_check_read_fp_flag },
+	{ .offset = FPO(vnode_check_write),
+		.flag = &mac_vnode_check_write_fp_flag },
+	{ .offset = FPO(vnode_check_mmap),
+		.flag = &mac_vnode_check_mmap_fp_flag },
 };
 
 static void

sys/security/mac/mac_framework.h

@@ -390,6 +390,12 @@ void	mac_sysvshm_init(struct shmid_kernel *);
 
 void	mac_thread_userret(struct thread *td);
 
+#ifdef DEBUG_VFS_LOCKS
+void	mac_vnode_assert_locked(struct vnode *vp, const char *func);
+#else
+#define mac_vnode_assert_locked(vp, func) do { } while (0)
+#endif
+
 int	mac_vnode_associate_extattr(struct mount *mp, struct vnode *vp);
 void	mac_vnode_associate_singlelabel(struct mount *mp, struct vnode *vp);
 int	mac_vnode_check_access(struct ucred *cred, struct vnode *vp,
@@ -412,18 +418,53 @@ int	mac_vnode_check_link(struct ucred *cred, struct vnode *dvp,
 	    struct vnode *vp, struct componentname *cnp);
 int	mac_vnode_check_listextattr(struct ucred *cred, struct vnode *vp,
 	    int attrnamespace);
-int	mac_vnode_check_lookup(struct ucred *cred, struct vnode *dvp,
+
+int	mac_vnode_check_lookup_impl(struct ucred *cred, struct vnode *dvp,
  	    struct componentname *cnp);
-int	mac_vnode_check_mmap(struct ucred *cred, struct vnode *vp, int prot,
+extern bool mac_vnode_check_lookup_fp_flag;
+static inline int
+mac_vnode_check_lookup(struct ucred *cred, struct vnode *dvp,
+    struct componentname *cnp)
+{
+
+	mac_vnode_assert_locked(dvp, "mac_vnode_check_lookup");
+	if (__predict_false(mac_vnode_check_lookup_fp_flag))
+                return (mac_vnode_check_lookup_impl(cred, dvp, cnp));
+	return (0);
+}
+
+int	mac_vnode_check_mmap_impl(struct ucred *cred, struct vnode *vp, int prot,
 	    int flags);
+extern bool mac_vnode_check_mmap_fp_flag;
+static inline int
+mac_vnode_check_mmap(struct ucred *cred, struct vnode *vp, int prot,
+    int flags)
+{
+
+	mac_vnode_assert_locked(vp, "mac_vnode_check_mmap");
+	if (__predict_false(mac_vnode_check_mmap_fp_flag))
+		return (mac_vnode_check_mmap_impl(cred, vp, prot, flags));
+	return (0);
+}
+
+int	mac_vnode_check_open_impl(struct ucred *cred, struct vnode *vp,
+	    accmode_t accmode);
+extern bool mac_vnode_check_open_fp_flag;
+static inline int
+mac_vnode_check_open(struct ucred *cred, struct vnode *vp,
+    accmode_t accmode)
+{
+
+	mac_vnode_assert_locked(vp, "mac_vnode_check_open");
+	if (__predict_false(mac_vnode_check_open_fp_flag))
+		return (mac_vnode_check_open_impl(cred, vp, accmode));
+	return (0);
+}
+
 int	mac_vnode_check_mprotect(struct ucred *cred, struct vnode *vp,
 	    int prot);
-int	mac_vnode_check_open(struct ucred *cred, struct vnode *vp,
-	    accmode_t accmode);
 int	mac_vnode_check_poll(struct ucred *active_cred,
 	    struct ucred *file_cred, struct vnode *vp);
-int	mac_vnode_check_read(struct ucred *active_cred,
-	    struct ucred *file_cred, struct vnode *vp);
 int	mac_vnode_check_readdir(struct ucred *cred, struct vnode *vp);
 int	mac_vnode_check_readlink(struct ucred *cred, struct vnode *vp);
 int	mac_vnode_check_rename_from(struct ucred *cred, struct vnode *dvp,
@@ -443,12 +484,51 @@ int	mac_vnode_check_setowner(struct ucred *cred, struct vnode *vp,
 	    uid_t uid, gid_t gid);
 int	mac_vnode_check_setutimes(struct ucred *cred, struct vnode *vp,
 	    struct timespec atime, struct timespec mtime);
-int	mac_vnode_check_stat(struct ucred *active_cred,
+
+int	mac_vnode_check_stat_impl(struct ucred *active_cred,
 	    struct ucred *file_cred, struct vnode *vp);
+extern bool mac_vnode_check_stat_fp_flag;
+static inline int
+mac_vnode_check_stat(struct ucred *active_cred, struct ucred *file_cred,
+    struct vnode *vp)
+{
+
+	mac_vnode_assert_locked(vp, "mac_vnode_check_stat");
+	if (__predict_false(mac_vnode_check_stat_fp_flag))
+		return (mac_vnode_check_stat_impl(active_cred, file_cred, vp));
+	return (0);
+}
+
+int	mac_vnode_check_read_impl(struct ucred *active_cred,
+	    struct ucred *file_cred, struct vnode *vp);
+extern bool mac_vnode_check_read_fp_flag;
+static inline int
+mac_vnode_check_read(struct ucred *active_cred, struct ucred *file_cred,
+    struct vnode *vp)
+{
+
+	mac_vnode_assert_locked(vp, "mac_vnode_check_read");
+	if (__predict_false(mac_vnode_check_read_fp_flag))
+		return (mac_vnode_check_read_impl(active_cred, file_cred, vp));
+	return (0);
+}
+
+int	mac_vnode_check_write_impl(struct ucred *active_cred,
+	    struct ucred *file_cred, struct vnode *vp);
+extern bool mac_vnode_check_write_fp_flag;
+static inline int
+mac_vnode_check_write(struct ucred *active_cred, struct ucred *file_cred,
+    struct vnode *vp)
+{
+
+	mac_vnode_assert_locked(vp, "mac_vnode_check_write");
+	if (__predict_false(mac_vnode_check_write_fp_flag))
+		return (mac_vnode_check_write_impl(active_cred, file_cred, vp));
+	return (0);
+}
+
 int	mac_vnode_check_unlink(struct ucred *cred, struct vnode *dvp,
 	    struct vnode *vp, struct componentname *cnp);
-int	mac_vnode_check_write(struct ucred *active_cred,
-	    struct ucred *file_cred, struct vnode *vp);
 void	mac_vnode_copy_label(struct label *, struct label *);
 void	mac_vnode_init(struct vnode *);
 int	mac_vnode_create_extattr(struct ucred *cred, struct mount *mp,

sys/security/mac/mac_vfs.c

@@ -565,13 +565,15 @@ MAC_CHECK_PROBE_DEFINE3(vnode_check_lookup, "struct ucred *",
     "struct vnode *", "struct componentname *");
 
 int
-mac_vnode_check_lookup(struct ucred *cred, struct vnode *dvp,
+mac_vnode_check_lookup_impl(struct ucred *cred, struct vnode *dvp,
     struct componentname *cnp)
 {
 	int error;
 
 	ASSERT_VOP_LOCKED(dvp, "mac_vnode_check_lookup");
 
+	if ((cnp->cn_flags & NOMACCHECK) != 0)
+		return (0);
 	MAC_POLICY_CHECK(vnode_check_lookup, cred, dvp, dvp->v_label, cnp);
 	MAC_CHECK_PROBE3(vnode_check_lookup, error, cred, dvp, cnp);
 
@@ -582,7 +584,7 @@ MAC_CHECK_PROBE_DEFINE4(vnode_check_mmap, "struct ucred *", "struct vnode *",
     "int", "int");
 
 int
-mac_vnode_check_mmap(struct ucred *cred, struct vnode *vp, int prot,
+mac_vnode_check_mmap_impl(struct ucred *cred, struct vnode *vp, int prot,
     int flags)
 {
 	int error;
@@ -629,7 +631,7 @@ MAC_CHECK_PROBE_DEFINE3(vnode_check_open, "struct ucred *", "struct vnode *",
     "accmode_t");
 
 int
-mac_vnode_check_open(struct ucred *cred, struct vnode *vp, accmode_t accmode)
+mac_vnode_check_open_impl(struct ucred *cred, struct vnode *vp, accmode_t accmode)
 {
 	int error;
 
@@ -664,7 +666,7 @@ MAC_CHECK_PROBE_DEFINE3(vnode_check_read, "struct ucred *", "struct ucred *",
     "struct vnode *");
 
 int
-mac_vnode_check_read(struct ucred *active_cred, struct ucred *file_cred,
+mac_vnode_check_read_impl(struct ucred *active_cred, struct ucred *file_cred,
     struct vnode *vp)
 {
 	int error;
@@ -889,7 +891,7 @@ MAC_CHECK_PROBE_DEFINE3(vnode_check_stat, "struct ucred *", "struct ucred *",
     "struct vnode *");
 
 int
-mac_vnode_check_stat(struct ucred *active_cred, struct ucred *file_cred,
+mac_vnode_check_stat_impl(struct ucred *active_cred, struct ucred *file_cred,
     struct vnode *vp)
 {
 	int error;
@@ -927,7 +929,7 @@ MAC_CHECK_PROBE_DEFINE3(vnode_check_write, "struct ucred *",
     "struct ucred *", "struct vnode *");
 
 int
-mac_vnode_check_write(struct ucred *active_cred, struct ucred *file_cred,
+mac_vnode_check_write_impl(struct ucred *active_cred, struct ucred *file_cred,
     struct vnode *vp)
 {
 	int error;
@@ -1068,3 +1070,12 @@ vn_setlabel(struct vnode *vp, struct label *intlabel, struct ucred *cred)
 
 	return (0);
 }
+
+#ifdef DEBUG_VFS_LOCKS
+void
+mac_vnode_assert_locked(struct vnode *vp, const char *func)
+{
+
+	ASSERT_VOP_LOCKED(vp, func);
+}
+#endif