Virtualize the pfil hooks so that different jails may chose different

packet filters. ALso allows ipfw to be enabled on on ejail and disabled
on another. In 8.0 it's a global setting.

Sitting aroung in tree waiting to commit for: 2 months
MFC after:	2 months

sys/net/if_bridge.c

@@ -109,6 +109,7 @@ __FBSDID("$FreeBSD$");
 #include <net/if_types.h>
 #include <net/if_var.h>
 #include <net/pfil.h>
+#include <net/vnet.h>
 
 #include <netinet/in.h> /* for struct arpcom */
 #include <netinet/in_systm.h>
@@ -1800,9 +1801,9 @@ bridge_dummynet(struct mbuf *m, struct ifnet *ifp)
 		return;
 	}
 
-	if (PFIL_HOOKED(&inet_pfil_hook)
+	if (PFIL_HOOKED(&V_inet_pfil_hook)
 #ifdef INET6
-	    || PFIL_HOOKED(&inet6_pfil_hook)
+	    || PFIL_HOOKED(&V_inet6_pfil_hook)
 #endif
 	    ) {
 		if (bridge_pfil(&m, sc->sc_ifp, ifp, PFIL_OUT) != 0)
@@ -2062,9 +2063,9 @@ bridge_forward(struct bridge_softc *sc, struct bridge_iflist *sbif,
 		ETHER_BPF_MTAP(ifp, m);
 
 	/* run the packet filter */
-	if (PFIL_HOOKED(&inet_pfil_hook)
+	if (PFIL_HOOKED(&V_inet_pfil_hook)
 #ifdef INET6
-	    || PFIL_HOOKED(&inet6_pfil_hook)
+	    || PFIL_HOOKED(&V_inet6_pfil_hook)
 #endif
 	    ) {
 		BRIDGE_UNLOCK(sc);
@@ -2102,9 +2103,9 @@ bridge_forward(struct bridge_softc *sc, struct bridge_iflist *sbif,
 
 	BRIDGE_UNLOCK(sc);
 
-	if (PFIL_HOOKED(&inet_pfil_hook)
+	if (PFIL_HOOKED(&V_inet_pfil_hook)
 #ifdef INET6
-	    || PFIL_HOOKED(&inet6_pfil_hook)
+	    || PFIL_HOOKED(&V_inet6_pfil_hook)
 #endif
 	    ) {
 		if (bridge_pfil(&m, ifp, dst_if, PFIL_OUT) != 0)
@@ -2243,7 +2244,7 @@ bridge_input(struct ifnet *ifp, struct mbuf *m)
 
 #ifdef INET6
 #   define OR_PFIL_HOOKED_INET6 \
-	|| PFIL_HOOKED(&inet6_pfil_hook)
+	|| PFIL_HOOKED(&V_inet6_pfil_hook)
 #else
 #   define OR_PFIL_HOOKED_INET6
 #endif
@@ -2260,7 +2261,7 @@ bridge_input(struct ifnet *ifp, struct mbuf *m)
 			iface->if_ipackets++;				\
 			/* Filter on the physical interface. */		\
 			if (pfil_local_phys &&				\
-			    (PFIL_HOOKED(&inet_pfil_hook)		\
+			    (PFIL_HOOKED(&V_inet_pfil_hook)		\
 			     OR_PFIL_HOOKED_INET6)) {			\
 				if (bridge_pfil(&m, NULL, ifp,		\
 				    PFIL_IN) != 0 || m == NULL) {	\
@@ -2349,9 +2350,9 @@ bridge_broadcast(struct bridge_softc *sc, struct ifnet *src_if,
 	}
 
 	/* Filter on the bridge interface before broadcasting */
-	if (runfilt && (PFIL_HOOKED(&inet_pfil_hook)
+	if (runfilt && (PFIL_HOOKED(&V_inet_pfil_hook)
 #ifdef INET6
-	    || PFIL_HOOKED(&inet6_pfil_hook)
+	    || PFIL_HOOKED(&V_inet6_pfil_hook)
 #endif
 	    )) {
 		if (bridge_pfil(&m, sc->sc_ifp, NULL, PFIL_OUT) != 0)
@@ -2396,9 +2397,9 @@ bridge_broadcast(struct bridge_softc *sc, struct ifnet *src_if,
 		 * pointer so we do not redundantly filter on the bridge for
 		 * each interface we broadcast on.
 		 */
-		if (runfilt && (PFIL_HOOKED(&inet_pfil_hook)
+		if (runfilt && (PFIL_HOOKED(&V_inet_pfil_hook)
 #ifdef INET6
-		    || PFIL_HOOKED(&inet6_pfil_hook)
+		    || PFIL_HOOKED(&V_inet6_pfil_hook)
 #endif
 		    )) {
 			if (used == 0) {
@@ -3037,7 +3038,7 @@ bridge_pfil(struct mbuf **mp, struct ifnet *bifp, struct ifnet *ifp, int dir)
 			goto bad;
 	}
 
-	if (ip_fw_chk_ptr && pfil_ipfw != 0 && dir == PFIL_OUT && ifp != NULL) {
+	if (V_ip_fw_chk_ptr && pfil_ipfw != 0 && dir == PFIL_OUT && ifp != NULL) {
 		struct dn_pkt_tag *dn_tag;
 
 		error = -1;
@@ -3057,7 +3058,7 @@ bridge_pfil(struct mbuf **mp, struct ifnet *bifp, struct ifnet *ifp, int dir)
 		args.next_hop = NULL;
 		args.eh = &eh2;
 		args.inp = NULL;	/* used by ipfw uid/gid/jail rules */
-		i = ip_fw_chk_ptr(&args);
+		i = V_ip_fw_chk_ptr(&args);
 		*mp = args.m;
 
 		if (*mp == NULL)
@@ -3109,21 +3110,21 @@ ipfwpass:
 		 *   in_if -> bridge_if -> out_if
 		 */
 		if (pfil_bridge && dir == PFIL_OUT && bifp != NULL)
-			error = pfil_run_hooks(&inet_pfil_hook, mp, bifp,
+			error = pfil_run_hooks(&V_inet_pfil_hook, mp, bifp,
 					dir, NULL);
 
 		if (*mp == NULL || error != 0) /* filter may consume */
 			break;
 
 		if (pfil_member && ifp != NULL)
-			error = pfil_run_hooks(&inet_pfil_hook, mp, ifp,
+			error = pfil_run_hooks(&V_inet_pfil_hook, mp, ifp,
 					dir, NULL);
 
 		if (*mp == NULL || error != 0) /* filter may consume */
 			break;
 
 		if (pfil_bridge && dir == PFIL_IN && bifp != NULL)
-			error = pfil_run_hooks(&inet_pfil_hook, mp, bifp,
+			error = pfil_run_hooks(&V_inet_pfil_hook, mp, bifp,
 					dir, NULL);
 
 		if (*mp == NULL || error != 0) /* filter may consume */
@@ -3163,21 +3164,21 @@ ipfwpass:
 #ifdef INET6
 	case ETHERTYPE_IPV6:
 		if (pfil_bridge && dir == PFIL_OUT && bifp != NULL)
-			error = pfil_run_hooks(&inet6_pfil_hook, mp, bifp,
+			error = pfil_run_hooks(&V_inet6_pfil_hook, mp, bifp,
 					dir, NULL);
 
 		if (*mp == NULL || error != 0) /* filter may consume */
 			break;
 
 		if (pfil_member && ifp != NULL)
-			error = pfil_run_hooks(&inet6_pfil_hook, mp, ifp,
+			error = pfil_run_hooks(&V_inet6_pfil_hook, mp, ifp,
 					dir, NULL);
 
 		if (*mp == NULL || error != 0) /* filter may consume */
 			break;
 
 		if (pfil_bridge && dir == PFIL_IN && bifp != NULL)
-			error = pfil_run_hooks(&inet6_pfil_hook, mp, bifp,
+			error = pfil_run_hooks(&V_inet6_pfil_hook, mp, bifp,
 					dir, NULL);
 		break;
 #endif

sys/net/if_ethersubr.c

@@ -434,7 +434,7 @@ ether_output_frame(struct ifnet *ifp, struct mbuf *m)
 {
 #if defined(INET) || defined(INET6)
 
-	if (ip_fw_chk_ptr && V_ether_ipfw != 0) {
+	if (V_ip_fw_chk_ptr && V_ether_ipfw != 0) {
 		if (ether_ipfw_chk(&m, ifp, 0) == 0) {
 			if (m) {
 				m_freem(m);
@@ -502,7 +502,7 @@ ether_ipfw_chk(struct mbuf **m0, struct ifnet *dst, int shared)
 	args.next_hop = NULL;	/* we do not support forward yet	*/
 	args.eh = &save_eh;	/* MAC header for bridged/MAC packets	*/
 	args.inp = NULL;	/* used by ipfw uid/gid/jail rules	*/
-	i = ip_fw_chk_ptr(&args);
+	i = V_ip_fw_chk_ptr(&args);
 	m = args.m;
 	if (m != NULL) {
 		/*
@@ -775,7 +775,7 @@ ether_demux(struct ifnet *ifp, struct mbuf *m)
 	 * Allow dummynet and/or ipfw to claim the frame.
 	 * Do not do this for PROMISC frames in case we are re-entered.
 	 */
-	if (ip_fw_chk_ptr && V_ether_ipfw != 0 && !(m->m_flags & M_PROMISC)) {
+	if (V_ip_fw_chk_ptr && V_ether_ipfw != 0 && !(m->m_flags & M_PROMISC)) {
 		if (ether_ipfw_chk(&m, NULL, 0) == 0) {
 			if (m)
 				m_freem(m);	/* dropped; free mbuf chain */

sys/net/pfil.c

@@ -56,8 +56,9 @@ static int pfil_list_add(pfil_list_t *, struct packet_filter_hook *, int);
 static int pfil_list_remove(pfil_list_t *,
     int (*)(void *, struct mbuf **, struct ifnet *, int, struct inpcb *), void *);
 
-LIST_HEAD(, pfil_head) pfil_head_list =
+LIST_HEAD(pfilheadhead, pfil_head);
-    LIST_HEAD_INITIALIZER(&pfil_head_list);
+VNET_DEFINE(struct pfilheadhead, pfil_head_list);
+#define	V_pfil_head_list	VNET(pfil_head_list)
 
 /*
  * pfil_run_hooks() runs the specified packet filter hooks.
@@ -97,7 +98,7 @@ pfil_head_register(struct pfil_head *ph)
 	struct pfil_head *lph;
 
 	PFIL_LIST_LOCK();
-	LIST_FOREACH(lph, &pfil_head_list, ph_list) {
+	LIST_FOREACH(lph, &V_pfil_head_list, ph_list) {
 		if (ph->ph_type == lph->ph_type &&
 		    ph->ph_un.phu_val == lph->ph_un.phu_val) {
 			PFIL_LIST_UNLOCK();
@@ -108,7 +109,7 @@ pfil_head_register(struct pfil_head *ph)
 	ph->ph_nhooks = 0;
 	TAILQ_INIT(&ph->ph_in);
 	TAILQ_INIT(&ph->ph_out);
-	LIST_INSERT_HEAD(&pfil_head_list, ph, ph_list);
+	LIST_INSERT_HEAD(&V_pfil_head_list, ph, ph_list);
 	PFIL_LIST_UNLOCK();
 	return (0);
 }
@@ -143,7 +144,7 @@ pfil_head_get(int type, u_long val)
 	struct pfil_head *ph;
 
 	PFIL_LIST_LOCK();
-	LIST_FOREACH(ph, &pfil_head_list, ph_list)
+	LIST_FOREACH(ph, &V_pfil_head_list, ph_list)
 		if (ph->ph_type == type && ph->ph_un.phu_val == val)
 			break;
 	PFIL_LIST_UNLOCK();
@@ -284,3 +285,45 @@ pfil_list_remove(pfil_list_t *list,
 		}
 	return ENOENT;
 }
+
+/****************
+ * Stuff that must be initialized for every instance
+ * (including the first of course).
+ */
+static int
+vnet_pfil_init(const void *unused)
+{
+	LIST_INIT(&V_pfil_head_list);
+	return (0);
+}
+
+/***********************
+ * Called for the removal of each instance.
+ */
+static int
+vnet_pfil_uninit(const void *unused)
+{
+	/*  XXX should panic if list is not empty */
+	return 0;
+}
+
+/* Define startup order. */
+#define	PFIL_SYSINIT_ORDER	SI_SUB_PROTO_BEGIN
+#define	PFIL_MODEVENT_ORDER	(SI_ORDER_FIRST) /* On boot slot in here. */
+#define	PFIL_VNET_ORDER		(PFIL_MODEVENT_ORDER + 2) /* Later still. */
+
+/*
+ * Starting up. 
+ * VNET_SYSINIT is called for each existing vnet and each new vnet.
+ */
+VNET_SYSINIT(vnet_pfil_init, PFIL_SYSINIT_ORDER, PFIL_VNET_ORDER,
+	    vnet_pfil_init, NULL);
+ 
+/*
+ * Closing up shop. These are done in REVERSE ORDER, 
+ * Not called on reboot.
+ * VNET_SYSUNINIT is called for each exiting vnet as it exits.
+ */
+VNET_SYSUNINIT(vnet_pfil_uninit, PFIL_SYSINIT_ORDER, PFIL_VNET_ORDER,
+	    vnet_pfil_uninit, NULL);
+

sys/netgraph/ng_bridge.c

@@ -634,7 +634,7 @@ ng_bridge_rcvdata(hook_p hook, item_p item)
 
 	/* Run packet through ipfw processing, if enabled */
 #if 0
-	if (priv->conf.ipfw[linkNum] && V_fw_enable && ip_fw_chk_ptr != NULL) {
+	if (priv->conf.ipfw[linkNum] && V_fw_enable && V_ip_fw_chk_ptr != NULL) {
 		/* XXX not implemented yet */
 	}
 #endif

sys/netinet/ip_fastfwd.c

@@ -351,10 +351,11 @@ ip_fastforward(struct mbuf *m)
 	/*
 	 * Run through list of ipfilter hooks for input packets
 	 */
-	if (!PFIL_HOOKED(&inet_pfil_hook))
+	if (!PFIL_HOOKED(&V_inet_pfil_hook))
 		goto passin;
 
-	if (pfil_run_hooks(&inet_pfil_hook, &m, m->m_pkthdr.rcvif, PFIL_IN, NULL) ||
+	if (pfil_run_hooks(
+	    &V_inet_pfil_hook, &m, m->m_pkthdr.rcvif, PFIL_IN, NULL) ||
 	    m == NULL)
 		goto drop;
 
@@ -438,10 +439,10 @@ passin:
 	/*
 	 * Run through list of hooks for output packets.
 	 */
-	if (!PFIL_HOOKED(&inet_pfil_hook))
+	if (!PFIL_HOOKED(&V_inet_pfil_hook))
 		goto passout;
 
-	if (pfil_run_hooks(&inet_pfil_hook, &m, ifp, PFIL_OUT, NULL) || m == NULL) {
+	if (pfil_run_hooks(&V_inet_pfil_hook, &m, ifp, PFIL_OUT, NULL) || m == NULL) {
 		goto drop;
 	}
 

sys/netinet/ip_input.c

@@ -170,7 +170,7 @@ SYSCTL_VNET_INT(_net_inet_ip, OID_AUTO, check_interface, CTLFLAG_RW,
     &VNET_NAME(ip_checkinterface), 0,
     "Verify packet arrives on correct interface");
 
-struct pfil_head inet_pfil_hook;	/* Packet filter hooks */
+VNET_DEFINE(struct pfil_head, inet_pfil_hook);	/* Packet filter hooks */
 
 static struct netisr_handler ip_nh = {
 	.nh_name = "ip",
@@ -318,6 +318,13 @@ ip_init(void)
 	    NULL, UMA_ALIGN_PTR, 0);
 	maxnipq_update();
 
+	/* Initialize packet filter hooks. */
+	V_inet_pfil_hook.ph_type = PFIL_TYPE_AF;
+	V_inet_pfil_hook.ph_af = AF_INET;
+	if ((i = pfil_head_register(&V_inet_pfil_hook)) != 0)
+		printf("%s: WARNING: unable to register pfil hook, "
+			"error %d\n", __func__, i);
+
 #ifdef FLOWTABLE
 	TUNABLE_INT_FETCH("net.inet.ip.output_flowtable_size",
 	    &V_ip_output_flowtable_size);
@@ -348,13 +355,6 @@ ip_init(void)
 				ip_protox[pr->pr_protocol] = pr - inetsw;
 		}
 
-	/* Initialize packet filter hooks. */
-	inet_pfil_hook.ph_type = PFIL_TYPE_AF;
-	inet_pfil_hook.ph_af = AF_INET;
-	if ((i = pfil_head_register(&inet_pfil_hook)) != 0)
-		printf("%s: WARNING: unable to register pfil hook, "
-			"error %d\n", __func__, i);
-
 	/* Start ipport_tick. */
 	callout_init(&ipport_tick_callout, CALLOUT_MPSAFE);
 	callout_reset(&ipport_tick_callout, 1, ipport_tick, NULL);
@@ -510,11 +510,11 @@ tooshort:
 	 */
 
 	/* Jump over all PFIL processing if hooks are not active. */
-	if (!PFIL_HOOKED(&inet_pfil_hook))
+	if (!PFIL_HOOKED(&V_inet_pfil_hook))
 		goto passin;
 
 	odst = ip->ip_dst;
-	if (pfil_run_hooks(&inet_pfil_hook, &m, ifp, PFIL_IN, NULL) != 0)
+	if (pfil_run_hooks(&V_inet_pfil_hook, &m, ifp, PFIL_IN, NULL) != 0)
 		return;
 	if (m == NULL)			/* consumed by filter */
 		return;

sys/netinet/ip_output.c

@@ -489,12 +489,12 @@ sendit:
 #endif /* IPSEC */
 
 	/* Jump over all PFIL processing if hooks are not active. */
-	if (!PFIL_HOOKED(&inet_pfil_hook))
+	if (!PFIL_HOOKED(&V_inet_pfil_hook))
 		goto passout;
 
 	/* Run through list of hooks for output packets. */
 	odst.s_addr = ip->ip_dst.s_addr;
-	error = pfil_run_hooks(&inet_pfil_hook, &m, ifp, PFIL_OUT, inp);
+	error = pfil_run_hooks(&V_inet_pfil_hook, &m, ifp, PFIL_OUT, inp);
 	if (error != 0 || m == NULL)
 		goto done;
 

sys/netinet/ip_var.h

@@ -244,14 +244,20 @@ extern int	(*ip_rsvp_vif)(struct socket *, struct sockopt *);
 extern void	(*ip_rsvp_force_done)(struct socket *);
 extern void	(*rsvp_input_p)(struct mbuf *m, int off);
 
-extern	struct pfil_head inet_pfil_hook;	/* packet filter hooks */
+VNET_DECLARE(struct pfil_head, inet_pfil_hook);	/* packet filter hooks */
+#define	V_inet_pfil_hook	VNET(inet_pfil_hook)
 
 void	in_delayed_cksum(struct mbuf *m);
 
 /* ipfw and dummynet hooks. Most are declared in raw_ip.c */
 struct ip_fw_args;
-extern int	(*ip_fw_chk_ptr)(struct ip_fw_args *args);
+typedef int	(*ip_fw_chk_ptr_t)(struct ip_fw_args *args);
-extern int	(*ip_fw_ctl_ptr)(struct sockopt *);
+typedef int	(*ip_fw_ctl_ptr_t)(struct sockopt *);
+VNET_DECLARE(ip_fw_chk_ptr_t, ip_fw_chk_ptr);
+VNET_DECLARE(ip_fw_ctl_ptr_t, ip_fw_ctl_ptr);
+#define	V_ip_fw_chk_ptr		VNET(ip_fw_chk_ptr)
+#define	V_ip_fw_ctl_ptr		VNET(ip_fw_ctl_ptr)
+
 extern int	(*ip_dn_ctl_ptr)(struct sockopt *);
 extern int	(*ip_dn_io_ptr)(struct mbuf **m, int dir, struct ip_fw_args *fwa);
 extern void	(*ip_dn_ruledel_ptr)(void *);		/* in ip_fw2.c */

sys/netinet/ipfw/ip_fw2.c

@@ -2495,6 +2495,10 @@ do {									\
 	}
 
 	IPFW_RLOCK(chain);
+	if (! V_ipfw_vnet_ready) { /* shutting down, leave NOW. */
+		IPFW_RUNLOCK(chain);
+		return (IP_FW_PASS);	/* accept */
+	}
 	mtag = m_tag_find(m, PACKET_TAG_DIVERT, NULL);
 	if (args->rule) {
 		/*
@@ -4637,29 +4641,21 @@ ipfw_init(void)
 		printf("limited to %d packets/entry by default\n",
 		    V_verbose_limit);
 
-	/*
-	 * Hook us up to pfil.
-	 * Eventually pfil will be per vnet.
-	 */
-	if ((error = ipfw_hook()) != 0) {
-		printf("ipfw_hook() error\n");
-		return (error);
-	}
-#ifdef INET6
-	if ((error = ipfw6_hook()) != 0) {
-		printf("ipfw6_hook() error\n");
-		return (error);
-	}
-#endif
-	/*
-	 * Other things that are only done the first time.
-	 * (now that we are guaranteed of success).
-	 */
-	ip_fw_ctl_ptr = ipfw_ctl;
-	ip_fw_chk_ptr = ipfw_chk;
 	return (error);
 }
 
+/**********************
+ * Called for the removal of the last instance only on module unload.
+ */
+static void
+ipfw_destroy(void)
+{
+
+	uma_zdestroy(ipfw_dyn_rule_zone);
+	IPFW_DYN_LOCK_DESTROY();
+	printf("IP firewall unloaded\n");
+}
+
 /****************
  * Stuff that must be initialized for every instance
  * (including the first of course).
@@ -4743,21 +4739,32 @@ vnet_ipfw_init(const void *unused)
 
 	/* First set up some values that are compile time options */
 	V_ipfw_vnet_ready = 1;		/* Open for business */
+
+	/* Hook up the raw inputs */
+	V_ip_fw_ctl_ptr = ipfw_ctl;
+	V_ip_fw_chk_ptr = ipfw_chk;
+
+	/*
+	 * Hook us up to pfil.
+	 */
+	if (V_fw_enable) {
+		if ((error = ipfw_hook()) != 0) {
+			printf("ipfw_hook() error\n");
+			return (error);
+		}
+	}
+#ifdef INET6
+	if (V_fw6_enable) {
+		if ((error = ipfw6_hook()) != 0) {
+			printf("ipfw6_hook() error\n");
+			/* XXX should we unhook everything else? */
+			return (error);
+		}
+	}
+#endif
 	return (0);
 }
 
-/**********************
- * Called for the removal of the last instance only on module unload.
- */
-static void
-ipfw_destroy(void)
-{
-
-	uma_zdestroy(ipfw_dyn_rule_zone);
-	IPFW_DYN_LOCK_DESTROY();
-	printf("IP firewall unloaded\n");
-}
-
 /***********************
  * Called for the removal of each instance.
  */
@@ -4767,9 +4774,18 @@ vnet_ipfw_uninit(const void *unused)
 	struct ip_fw *reap;
 
 	V_ipfw_vnet_ready = 0; /* tell new callers to go away */
-	callout_drain(&V_ipfw_timeout);
+	ipfw_unhook();
-	/* We wait on the wlock here until the last user leaves */
+#ifdef INET6
+	ipfw6_unhook();
+#endif
+	/* layer2 and other entrypoints still come in this way. */
+	V_ip_fw_chk_ptr = NULL;
+	V_ip_fw_ctl_ptr = NULL;
 	IPFW_WLOCK(&V_layer3_chain);
+	/* We wait on the wlock here until the last user leaves */
+	IPFW_WUNLOCK(&V_layer3_chain);
+	IPFW_WLOCK(&V_layer3_chain);
+	callout_drain(&V_ipfw_timeout);
 	flush_tables(&V_layer3_chain);
 	V_layer3_chain.reap = NULL;
 	free_chain(&V_layer3_chain, 1 /* kill default rule */);
@@ -4803,21 +4819,10 @@ ipfw_modevent(module_t mod, int type, void *unused)
 		/* Called once at module load or
 	 	 * system boot if compiled in. */
 		break;
-	case MOD_UNLOAD:
-		break;
 	case MOD_QUIESCE:
-		/* Yes, the unhooks can return errors, we can safely ignore
+		/* Called before unload. May veto unloading. */
-		 * them. Eventually these will be done per jail as they
+		break;
-		 * shut down. We will wait on each vnet's l3 lock as existing
+	case MOD_UNLOAD:
-		 * callers go away.
-		 */
-		ipfw_unhook();
-#ifdef INET6
-		ipfw6_unhook();
-#endif
-		/* layer2 and other entrypoints still come in this way. */
-		ip_fw_chk_ptr = NULL;
-		ip_fw_ctl_ptr = NULL;
 		/* Called during unload. */
 		break;
 	case MOD_SHUTDOWN:
@@ -4866,4 +4871,3 @@ SYSUNINIT(ipfw_destroy, IPFW_SI_SUB_FIREWALL, IPFW_MODULE_ORDER,
 VNET_SYSUNINIT(vnet_ipfw_uninit, IPFW_SI_SUB_FIREWALL, IPFW_VNET_ORDER,
 	    vnet_ipfw_uninit, NULL);
 
- 

sys/netinet/ipfw/ip_fw_pfil.c

@@ -515,42 +515,54 @@ ipfw6_unhook(void)
 int
 ipfw_chg_hook(SYSCTL_HANDLER_ARGS)
 {
-	int enable = *(int *)arg1;
+	int enable;
+	int oldenable;
 	int error;
 
-#ifdef VIMAGE /* Since enabling is global, only let base do it. */
+	if (arg1 == &VNET_NAME(fw_enable)) {
-	if (! IS_DEFAULT_VNET(curvnet))
+		enable = V_fw_enable;
-		return (EPERM);
+	}
+#ifdef INET6
+	else if (arg1 == &VNET_NAME(fw6_enable)) {
+		enable = V_fw6_enable;
+	}
 #endif
+	else 
+		return (EINVAL);
+
+	oldenable = enable;
+
 	error = sysctl_handle_int(oidp, &enable, 0, req);
+
 	if (error)
 		return (error);
 
 	enable = (enable) ? 1 : 0;
 
-	if (enable == *(int *)arg1)
+	if (enable == oldenable)
 		return (0);
 
-	if (arg1 == &V_fw_enable) {
+	if (arg1 == &VNET_NAME(fw_enable)) {
 		if (enable)
 			error = ipfw_hook();
 		else
 			error = ipfw_unhook();
+		if (error)
+			return (error);
+		V_fw_enable = enable;
 	}
 #ifdef INET6
-	if (arg1 == &V_fw6_enable) {
+	else if (arg1 == &VNET_NAME(fw6_enable)) {
 		if (enable)
 			error = ipfw6_hook();
 		else
 			error = ipfw6_unhook();
+		if (error)
+			return (error);
+		V_fw6_enable = enable;
 	}
 #endif
 
-	if (error)
-		return (error);
-
-	*(int *)arg1 = enable;
-
 	return (0);
 }
 

sys/netinet/raw_ip.c

@@ -84,9 +84,9 @@ VNET_DEFINE(struct inpcbinfo, ripcbinfo);
  * The data hooks are not used here but it is convenient
  * to keep them all in one place.
  */
-int (*ip_fw_ctl_ptr)(struct sockopt *) = NULL;
+VNET_DEFINE(ip_fw_chk_ptr_t, ip_fw_chk_ptr) = NULL;
+VNET_DEFINE(ip_fw_ctl_ptr_t, ip_fw_ctl_ptr) = NULL;
 int (*ip_dn_ctl_ptr)(struct sockopt *) = NULL;
-int (*ip_fw_chk_ptr)(struct ip_fw_args *args) = NULL;
 int (*ip_dn_io_ptr)(struct mbuf **m, int dir, struct ip_fw_args *fwa) = NULL;
 
 /*
@@ -523,8 +523,8 @@ rip_ctloutput(struct socket *so, struct sockopt *sopt)
 		case IP_FW_TABLE_LIST:
 		case IP_FW_NAT_GET_CONFIG:
 		case IP_FW_NAT_GET_LOG:
-			if (ip_fw_ctl_ptr != NULL)
+			if (V_ip_fw_ctl_ptr != NULL)
-				error = ip_fw_ctl_ptr(sopt);
+				error = V_ip_fw_ctl_ptr(sopt);
 			else
 				error = ENOPROTOOPT;
 			break;
@@ -584,8 +584,8 @@ rip_ctloutput(struct socket *so, struct sockopt *sopt)
 		case IP_FW_TABLE_FLUSH:
 		case IP_FW_NAT_CFG:
 		case IP_FW_NAT_DEL:
-			if (ip_fw_ctl_ptr != NULL)
+			if (V_ip_fw_ctl_ptr != NULL)
-				error = ip_fw_ctl_ptr(sopt);
+				error = V_ip_fw_ctl_ptr(sopt);
 			else
 				error = ENOPROTOOPT;
 			break;

sys/netinet6/ip6_forward.c

@@ -551,11 +551,11 @@ skip_routing:
 	in6_clearscope(&ip6->ip6_dst);
 
 	/* Jump over all PFIL processing if hooks are not active. */
-	if (!PFIL_HOOKED(&inet6_pfil_hook))
+	if (!PFIL_HOOKED(&V_inet6_pfil_hook))
 		goto pass;
 
 	/* Run through list of hooks for output packets. */
-	error = pfil_run_hooks(&inet6_pfil_hook, &m, rt->rt_ifp, PFIL_OUT, NULL);
+	error = pfil_run_hooks(&V_inet6_pfil_hook, &m, rt->rt_ifp, PFIL_OUT, NULL);
 	if (error != 0)
 		goto senderr;
 	if (m == NULL)

sys/netinet6/ip6_input.c

@@ -152,7 +152,7 @@ VNET_DECLARE(int, udp6_recvspace);
 struct rwlock in6_ifaddr_lock;
 RW_SYSINIT(in6_ifaddr_lock, &in6_ifaddr_lock, "in6_ifaddr_lock");
 
-struct pfil_head inet6_pfil_hook;
+VNET_DEFINE (struct pfil_head, inet6_pfil_hook);
 
 static void ip6_init2(void *);
 static struct ip6aux *ip6_setdstifaddr(struct mbuf *, struct in6_ifaddr *);
@@ -247,6 +247,13 @@ ip6_init(void)
 
 	V_ip6_desync_factor = arc4random() % MAX_TEMP_DESYNC_FACTOR;
 
+	/* Initialize packet filter hooks. */
+	V_inet6_pfil_hook.ph_type = PFIL_TYPE_AF;
+	V_inet6_pfil_hook.ph_af = AF_INET6;
+	if ((i = pfil_head_register(&V_inet6_pfil_hook)) != 0)
+		printf("%s: WARNING: unable to register pfil hook, "
+			"error %d\n", __func__, i);
+
 	/* Skip global initialization stuff for non-default instances. */
 	if (!IS_DEFAULT_VNET(curvnet))
 		return;
@@ -275,13 +282,6 @@ ip6_init(void)
 				ip6_protox[pr->pr_protocol] = pr - inet6sw;
 		}
 
-	/* Initialize packet filter hooks. */
-	inet6_pfil_hook.ph_type = PFIL_TYPE_AF;
-	inet6_pfil_hook.ph_af = AF_INET6;
-	if ((i = pfil_head_register(&inet6_pfil_hook)) != 0)
-		printf("%s: WARNING: unable to register pfil hook, "
-			"error %d\n", __func__, i);
-
 	netisr_register(&ip6_nh);
 }
 
@@ -515,10 +515,11 @@ ip6_input(struct mbuf *m)
 	odst = ip6->ip6_dst;
 
 	/* Jump over all PFIL processing if hooks are not active. */
-	if (!PFIL_HOOKED(&inet6_pfil_hook))
+	if (!PFIL_HOOKED(&V_inet6_pfil_hook))
 		goto passin;
 
-	if (pfil_run_hooks(&inet6_pfil_hook, &m, m->m_pkthdr.rcvif, PFIL_IN, NULL))
+	if (pfil_run_hooks(&V_inet6_pfil_hook, &m,
+	    m->m_pkthdr.rcvif, PFIL_IN, NULL))
 		return;
 	if (m == NULL)			/* consumed by filter */
 		return;

sys/netinet6/ip6_output.c

@@ -805,12 +805,12 @@ again:
 	}
 
 	/* Jump over all PFIL processing if hooks are not active. */
-	if (!PFIL_HOOKED(&inet6_pfil_hook))
+	if (!PFIL_HOOKED(&V_inet6_pfil_hook))
 		goto passout;
 
 	odst = ip6->ip6_dst;
 	/* Run through list of hooks for output packets. */
-	error = pfil_run_hooks(&inet6_pfil_hook, &m, ifp, PFIL_OUT, inp);
+	error = pfil_run_hooks(&V_inet6_pfil_hook, &m, ifp, PFIL_OUT, inp);
 	if (error != 0 || m == NULL)
 		goto done;
 	ip6 = mtod(m, struct ip6_hdr *);

sys/netinet6/ip6_var.h

@@ -358,7 +358,8 @@ VNET_DECLARE(int, ip6_use_defzone);	/* Whether to use the default scope
 #endif
 #define	V_ip6_use_defzone		VNET(ip6_use_defzone)
 
-extern	struct pfil_head inet6_pfil_hook;	/* packet filter hooks */
+VNET_DECLARE (struct pfil_head, inet6_pfil_hook);	/* packet filter hooks */
+#define	V_inet6_pfil_hook	VNET(inet6_pfil_hook)
 
 extern struct	pr_usrreqs rip6_usrreqs;
 struct sockopt;