mac: implement fast path for checks

All checking routines walk a linked list of all modules in order to determine
if given hook is installed. This became a significant problem after mac_ntpd
started being loaded by default.

Implement a way perform checks for select hooks by testing a boolean.

Use it for priv_check and priv_grant, which are constantly called from priv_check.

The real fix would use hotpatching, but the above provides a way to know when
to do it.

sys/security/mac/mac_framework.c

@@ -117,6 +117,17 @@ static unsigned int	mac_version = MAC_VERSION;
 SYSCTL_UINT(_security_mac, OID_AUTO, version, CTLFLAG_RD, &mac_version, 0,
     "");
 
+/*
+ * Flags for inlined checks.
+ */
+#define FPFLAG(f)	\
+bool __read_frequently mac_##f##_fp_flag
+
+FPFLAG(priv_check);
+FPFLAG(priv_grant);
+
+#undef FPFLAG
+
 /*
  * Labels consist of a indexed set of "slots", which are allocated policies
  * as required.  The MAC Framework maintains a bitmask of slots allocated so
@@ -376,6 +387,84 @@ mac_policy_update(void)
 	}
 }
 
+/*
+ * There are frequently used code paths which check for rarely installed
+ * policies. Gross hack below enables doing it in a cheap manner.
+ */
+
+#define FPO(f)	(offsetof(struct mac_policy_ops, mpo_##f) / sizeof(uintptr_t))
+
+struct mac_policy_fastpath_elem {
+	int	count;
+	bool	*flag;
+	size_t	offset;
+};
+
+struct mac_policy_fastpath_elem mac_policy_fastpath_array[] = {
+	{ .offset = FPO(priv_check), .flag = &mac_priv_check_fp_flag },
+	{ .offset = FPO(priv_grant), .flag = &mac_priv_grant_fp_flag },
+};
+
+static void
+mac_policy_fastpath_enable(struct mac_policy_fastpath_elem *mpfe)
+{
+
+	MPASS(mpfe->count >= 0);
+	mpfe->count++;
+	if (mpfe->count == 1) {
+		MPASS(*mpfe->flag == false);
+		*mpfe->flag = true;
+	}
+}
+
+static void
+mac_policy_fastpath_disable(struct mac_policy_fastpath_elem *mpfe)
+{
+
+	MPASS(mpfe->count >= 1);
+	mpfe->count--;
+	if (mpfe->count == 0) {
+		MPASS(*mpfe->flag == true);
+		*mpfe->flag = false;
+	}
+}
+
+static void
+mac_policy_fastpath_register(struct mac_policy_conf *mpc)
+{
+	struct mac_policy_fastpath_elem *mpfe;
+	uintptr_t **ops;
+	int i;
+
+	mac_policy_xlock_assert();
+
+	ops = (uintptr_t **)mpc->mpc_ops;
+	for (i = 0; i < nitems(mac_policy_fastpath_array); i++) {
+		mpfe = &mac_policy_fastpath_array[i];
+		if (ops[mpfe->offset] != NULL)
+			mac_policy_fastpath_enable(mpfe);
+	}
+}
+
+static void
+mac_policy_fastpath_unregister(struct mac_policy_conf *mpc)
+{
+	struct mac_policy_fastpath_elem *mpfe;
+	uintptr_t **ops;
+	int i;
+
+	mac_policy_xlock_assert();
+
+	ops = (uintptr_t **)mpc->mpc_ops;
+	for (i = 0; i < nitems(mac_policy_fastpath_array); i++) {
+		mpfe = &mac_policy_fastpath_array[i];
+		if (ops[mpfe->offset] != NULL)
+			mac_policy_fastpath_disable(mpfe);
+	}
+}
+
+#undef FPO
+
 static int
 mac_policy_register(struct mac_policy_conf *mpc)
 {
@@ -446,6 +535,9 @@ mac_policy_register(struct mac_policy_conf *mpc)
 	 */
 	if (mpc->mpc_ops->mpo_init != NULL)
 		(*(mpc->mpc_ops->mpo_init))(mpc);
+
+	mac_policy_fastpath_register(mpc);
+
 	mac_policy_update();
 
 	SDT_PROBE1(mac, , policy, register, mpc);
@@ -487,6 +579,9 @@ mac_policy_unregister(struct mac_policy_conf *mpc)
 		mac_policy_xunlock();
 		return (EBUSY);
 	}
+
+	mac_policy_fastpath_unregister(mpc);
+
 	if (mpc->mpc_ops->mpo_destroy != NULL)
 		(*(mpc->mpc_ops->mpo_destroy))(mpc);
 

sys/security/mac/mac_framework.h

@@ -258,8 +258,27 @@ void 	mac_posixshm_create(struct ucred *cred, struct shmfd *shmfd);
 void	mac_posixshm_destroy(struct shmfd *);
 void	mac_posixshm_init(struct shmfd *);
 
-int	mac_priv_check(struct ucred *cred, int priv);
-int	mac_priv_grant(struct ucred *cred, int priv);
+int	mac_priv_check_impl(struct ucred *cred, int priv);
+extern bool mac_priv_check_fp_flag;
+static inline int
+mac_priv_check(struct ucred *cred, int priv)
+{
+
+	if (__predict_false(mac_priv_check_fp_flag))
+		return (mac_priv_check_impl(cred, priv));
+	return (0);
+}
+
+int	mac_priv_grant_impl(struct ucred *cred, int priv);
+extern bool mac_priv_grant_fp_flag;
+static inline int
+mac_priv_grant(struct ucred *cred, int priv)
+{
+
+	if (__predict_false(mac_priv_grant_fp_flag))
+		return (mac_priv_grant_impl(cred, priv));
+	return (EPERM);
+}
 
 int	mac_proc_check_debug(struct ucred *cred, struct proc *p);
 int	mac_proc_check_sched(struct ucred *cred, struct proc *p);

sys/security/mac/mac_priv.c

@@ -67,7 +67,7 @@ MAC_CHECK_PROBE_DEFINE2(priv_check, "struct ucred *", "int");
  * policy denies access.
  */
 int
-mac_priv_check(struct ucred *cred, int priv)
+mac_priv_check_impl(struct ucred *cred, int priv)
 {
 	int error;
 
@@ -84,7 +84,7 @@ MAC_GRANT_PROBE_DEFINE2(priv_grant, "struct ucred *", "int");
  * policy grants access.
  */
 int
-mac_priv_grant(struct ucred *cred, int priv)
+mac_priv_grant_impl(struct ucred *cred, int priv)
 {
 	int error;