Overhaul if_enc(4) and make it loadable in run-time.

Use hhook(9) framework to achieve ability of loading and unloading
if_enc(4) kernel module. INET and INET6 code on initialization registers
two helper hooks points in the kernel. if_enc(4) module uses these helper
hook points and registers its hooks. IPSEC code uses these hhook points
to call helper hooks implemented in if_enc(4).

sys/conf/files

@@ -3466,7 +3466,7 @@ net/if_dead.c			standard
 net/if_debug.c			optional ddb
 net/if_disc.c			optional disc
 net/if_edsc.c			optional edsc
-net/if_enc.c			optional enc ipsec inet | enc ipsec inet6
+net/if_enc.c			optional enc inet | enc inet6
 net/if_epair.c			optional epair
 net/if_ethersubr.c		optional ether
 net/if_fddisubr.c		optional fddi

sys/conf/options

@@ -707,7 +707,6 @@ ISAPNP			opt_isa.h
 # various 'device presence' options.
 DEV_BPF			opt_bpf.h
 DEV_CARP		opt_carp.h
-DEV_ENC			opt_enc.h
 DEV_MCA			opt_mca.h
 DEV_NETMAP		opt_global.h
 DEV_PCI			opt_pci.h

sys/modules/Makefile

@@ -142,6 +142,7 @@ SUBDIR=	\
 	if_bridge \
 	if_disc \
 	if_edsc \
+	${_if_enc} \
 	if_epair \
 	${_if_gif} \
 	${_if_gre} \
@@ -429,6 +430,7 @@ SUBDIR+=	cuse
 	defined(ALL_MODULES)
 _carp=		carp
 _toecore=	toecore
+_if_enc=	if_enc
 _if_gif=	if_gif
 _if_gre=	if_gre
 .endif

sys/modules/if_enc/Makefile

@@ -0,0 +1,11 @@
+# $FreeBSD$
+
+SYSDIR?=${.CURDIR}/../..
+.include "${SYSDIR}/conf/kern.opts.mk"
+
+.PATH: ${SYSDIR}/net
+
+KMOD=	if_enc
+SRCS=	if_enc.c opt_inet.h opt_inet6.h
+
+.include <bsd.kmod.mk>

sys/net/if.c

@@ -183,6 +183,10 @@ static void	if_detach_internal(struct ifnet *, int, struct if_clone **);
 extern void	nd6_setmtu(struct ifnet *);
 #endif
 
+/* ipsec helper hooks */
+VNET_DEFINE(struct hhook_head *, ipsec_hhh_in[HHOOK_IPSEC_COUNT]);
+VNET_DEFINE(struct hhook_head *, ipsec_hhh_out[HHOOK_IPSEC_COUNT]);
+
 VNET_DEFINE(int, if_index);
 int	ifqmaxlen = IFQ_MAXLEN;
 VNET_DEFINE(struct ifnethead, ifnet);	/* depend on static init XXX */

sys/net/if_enc.c

@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2006 The FreeBSD Project.
+ * Copyright (c) 2015 Andrey V. Elsukov <ae@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -79,56 +80,67 @@ struct enchdr {
 	u_int32_t spi;
 	u_int32_t flags;
 };
-
-struct ifnet	*encif;
-static struct mtx	enc_mtx;
-
 struct enc_softc {
 	struct	ifnet *sc_ifp;
 };
+static VNET_DEFINE(struct enc_softc *, enc_sc);
+#define	V_enc_sc	VNET(enc_sc)
+static VNET_DEFINE(struct if_clone *, enc_cloner);
+#define	V_enc_cloner	VNET(enc_cloner)
 
 static int	enc_ioctl(struct ifnet *, u_long, caddr_t);
-static int	enc_output(struct ifnet *ifp, struct mbuf *m,
+static int	enc_output(struct ifnet *, struct mbuf *,
-		    const struct sockaddr *dst, struct route *ro);
+    const struct sockaddr *, struct route *);
 static int	enc_clone_create(struct if_clone *, int, caddr_t);
 static void	enc_clone_destroy(struct ifnet *);
-static struct if_clone *enc_cloner;
+static int	enc_add_hhooks(struct enc_softc *);
-static const char encname[] = "enc";
+static void	enc_remove_hhooks(struct enc_softc *);
 
-/*
+static const char encname[] = "enc";
- * Sysctls.
- */
 
 /*
  * Before and after are relative to when we are stripping the
  * outer IP header.
  */
+static VNET_DEFINE(int, filter_mask_in) = IPSEC_ENC_BEFORE;
+static VNET_DEFINE(int, bpf_mask_in) = IPSEC_ENC_BEFORE;
+static VNET_DEFINE(int, filter_mask_out) = IPSEC_ENC_BEFORE;
+static VNET_DEFINE(int, bpf_mask_out) = IPSEC_ENC_BEFORE | IPSEC_ENC_AFTER;
+#define	V_filter_mask_in	VNET(filter_mask_in)
+#define	V_bpf_mask_in		VNET(bpf_mask_in)
+#define	V_filter_mask_out	VNET(filter_mask_out)
+#define	V_bpf_mask_out		VNET(bpf_mask_out)
+
 static SYSCTL_NODE(_net, OID_AUTO, enc, CTLFLAG_RW, 0, "enc sysctl");
-
 static SYSCTL_NODE(_net_enc, OID_AUTO, in, CTLFLAG_RW, 0, "enc input sysctl");
-static int ipsec_filter_mask_in = ENC_BEFORE;
-SYSCTL_INT(_net_enc_in, OID_AUTO, ipsec_filter_mask, CTLFLAG_RW,
-	&ipsec_filter_mask_in, 0, "IPsec input firewall filter mask");
-static int ipsec_bpf_mask_in = ENC_BEFORE;
-SYSCTL_INT(_net_enc_in, OID_AUTO, ipsec_bpf_mask, CTLFLAG_RW,
-	&ipsec_bpf_mask_in, 0, "IPsec input bpf mask");
-
 static SYSCTL_NODE(_net_enc, OID_AUTO, out, CTLFLAG_RW, 0, "enc output sysctl");
-static int ipsec_filter_mask_out = ENC_BEFORE;
+SYSCTL_INT(_net_enc_in, OID_AUTO, ipsec_filter_mask,
-SYSCTL_INT(_net_enc_out, OID_AUTO, ipsec_filter_mask, CTLFLAG_RW,
+    CTLFLAG_RW | CTLFLAG_VNET, &VNET_NAME(filter_mask_in), 0,
-	&ipsec_filter_mask_out, 0, "IPsec output firewall filter mask");
+    "IPsec input firewall filter mask");
-static int ipsec_bpf_mask_out = ENC_BEFORE|ENC_AFTER;
+SYSCTL_INT(_net_enc_in, OID_AUTO, ipsec_bpf_mask,
-SYSCTL_INT(_net_enc_out, OID_AUTO, ipsec_bpf_mask, CTLFLAG_RW,
+    CTLFLAG_RW | CTLFLAG_VNET, &VNET_NAME(bpf_mask_in), 0,
-	&ipsec_bpf_mask_out, 0, "IPsec output bpf mask");
+    "IPsec input bpf mask");
+SYSCTL_INT(_net_enc_out, OID_AUTO, ipsec_filter_mask,
+    CTLFLAG_RW | CTLFLAG_VNET, &VNET_NAME(filter_mask_out), 0,
+    "IPsec output firewall filter mask");
+SYSCTL_INT(_net_enc_out, OID_AUTO, ipsec_bpf_mask,
+    CTLFLAG_RW | CTLFLAG_VNET, &VNET_NAME(bpf_mask_out), 0,
+    "IPsec output bpf mask");
 
 static void
 enc_clone_destroy(struct ifnet *ifp)
 {
-	KASSERT(ifp != encif, ("%s: destroying encif", __func__));
+	struct enc_softc *sc;
 
+	sc = ifp->if_softc;
+	KASSERT(sc == V_enc_sc, ("sc != ifp->if_softc"));
+
+	enc_remove_hhooks(sc);
 	bpfdetach(ifp);
 	if_detach(ifp);
 	if_free(ifp);
+	free(sc, M_DEVBUF);
+	V_enc_sc = NULL;
 }
 
 static int
@@ -137,43 +149,245 @@ enc_clone_create(struct if_clone *ifc, int unit, caddr_t params)
 	struct ifnet *ifp;
 	struct enc_softc *sc;
 
-	sc = malloc(sizeof(*sc), M_DEVBUF, M_WAITOK|M_ZERO);
+	sc = malloc(sizeof(struct enc_softc), M_DEVBUF,
+	    M_WAITOK | M_ZERO);
 	ifp = sc->sc_ifp = if_alloc(IFT_ENC);
 	if (ifp == NULL) {
 		free(sc, M_DEVBUF);
 		return (ENOSPC);
 	}
-
+	if (V_enc_sc != NULL) {
+		if_free(ifp);
+		free(sc, M_DEVBUF);
+		return (EEXIST);
+	}
+	V_enc_sc = sc;
 	if_initname(ifp, encname, unit);
 	ifp->if_mtu = ENCMTU;
 	ifp->if_ioctl = enc_ioctl;
 	ifp->if_output = enc_output;
-	ifp->if_snd.ifq_maxlen = ifqmaxlen;
 	ifp->if_softc = sc;
 	if_attach(ifp);
 	bpfattach(ifp, DLT_ENC, sizeof(struct enchdr));
-
+	if (enc_add_hhooks(sc) != 0) {
-	mtx_lock(&enc_mtx);
+		enc_clone_destroy(ifp);
-	/* grab a pointer to enc0, ignore the rest */
+		return (ENXIO);
-	if (encif == NULL)
+	}
-		encif = ifp;
-	mtx_unlock(&enc_mtx);
-
 	return (0);
 }
 
+static int
+enc_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
+    struct route *ro)
+{
+
+	m_freem(m);
+	return (0);
+}
+
+static int
+enc_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
+{
+
+	if (cmd != SIOCSIFFLAGS)
+		return (EINVAL);
+	if (ifp->if_flags & IFF_UP)
+		ifp->if_drv_flags |= IFF_DRV_RUNNING;
+	else
+		ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+	return (0);
+}
+
+/*
+ * One helper hook function is used by any hook points.
+ * + from hhook_type we can determine the packet direction:
+ *   HHOOK_TYPE_IPSEC_IN or HHOOK_TYPE_IPSEC_OUT;
+ * + from hhook_id we can determine address family: AF_INET or AF_INET6;
+ * + udata contains pointer to enc_softc;
+ * + ctx_data contains pointer to struct ipsec_ctx_data.
+ */
+static int
+enc_hhook(int32_t hhook_type, int32_t hhook_id, void *udata, void *ctx_data,
+    void *hdata, struct osd *hosd)
+{
+	struct enchdr hdr;
+	struct ipsec_ctx_data *ctx;
+	struct enc_softc *sc;
+	struct ifnet *ifp, *rcvif;
+	struct pfil_head *ph;
+	int pdir;
+
+	sc = (struct enc_softc *)udata;
+	ifp = sc->sc_ifp;
+	if ((ifp->if_flags & IFF_UP) == 0)
+		return (0);
+
+	ctx = (struct ipsec_ctx_data *)ctx_data;
+	/* XXX: wrong hook point was used by caller? */
+	if (ctx->af != hhook_id)
+		return (EPFNOSUPPORT);
+
+	if (((hhook_type == HHOOK_TYPE_IPSEC_IN &&
+	    (ctx->enc & V_bpf_mask_in) != 0) ||
+	    (hhook_type == HHOOK_TYPE_IPSEC_OUT &&
+	    (ctx->enc & V_bpf_mask_out) != 0)) &&
+	    bpf_peers_present(ifp->if_bpf) != 0) {
+		hdr.af = ctx->af;
+		hdr.spi = ctx->sav->spi;
+		hdr.flags = 0;
+		if (ctx->sav->alg_enc != SADB_EALG_NONE)
+			hdr.flags |= M_CONF;
+		if (ctx->sav->alg_auth != SADB_AALG_NONE)
+			hdr.flags |= M_AUTH;
+		bpf_mtap2(ifp->if_bpf, &hdr, sizeof(hdr), *ctx->mp);
+	}
+
+	switch (hhook_type) {
+	case HHOOK_TYPE_IPSEC_IN:
+		if (ctx->enc == IPSEC_ENC_BEFORE) {
+			/* Do accounting only once */
+			if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
+			if_inc_counter(ifp, IFCOUNTER_IBYTES,
+			    (*ctx->mp)->m_pkthdr.len);
+		}
+		if ((ctx->enc & V_filter_mask_in) == 0)
+			return (0); /* skip pfil processing */
+		pdir = PFIL_IN;
+		break;
+	case HHOOK_TYPE_IPSEC_OUT:
+		if (ctx->enc == IPSEC_ENC_BEFORE) {
+			/* Do accounting only once */
+			if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
+			if_inc_counter(ifp, IFCOUNTER_OBYTES,
+			    (*ctx->mp)->m_pkthdr.len);
+		}
+		if ((ctx->enc & V_filter_mask_out) == 0)
+			return (0); /* skip pfil processing */
+		pdir = PFIL_OUT;
+		break;
+	default:
+		return (EINVAL);
+	}
+
+	switch (hhook_id) {
+#ifdef INET
+	case AF_INET:
+		ph = &V_inet_pfil_hook;
+		break;
+#endif
+#ifdef INET6
+	case AF_INET6:
+		ph = &V_inet6_pfil_hook;
+		break;
+#endif
+	default:
+		ph = NULL;
+	}
+	if (ph == NULL || !PFIL_HOOKED(ph))
+		return (0);
+	/* Make a packet looks like it was received on enc(4) */
+	rcvif = (*ctx->mp)->m_pkthdr.rcvif;
+	(*ctx->mp)->m_pkthdr.rcvif = ifp;
+	if (pfil_run_hooks(ph, ctx->mp, ifp, pdir, NULL) != 0 ||
+	    *ctx->mp == NULL) {
+		*ctx->mp = NULL; /* consumed by filter */
+		return (EACCES);
+	}
+	(*ctx->mp)->m_pkthdr.rcvif = rcvif;
+	return (0);
+}
+
+static int
+enc_add_hhooks(struct enc_softc *sc)
+{
+	struct hookinfo hki;
+	int error;
+
+	error = EPFNOSUPPORT;
+	hki.hook_func = enc_hhook;
+	hki.hook_helper = NULL;
+	hki.hook_udata = sc;
+#ifdef INET
+	hki.hook_id = AF_INET;
+	hki.hook_type = HHOOK_TYPE_IPSEC_IN;
+	error = hhook_add_hook(V_ipsec_hhh_in[HHOOK_IPSEC_INET],
+	    &hki, HHOOK_WAITOK);
+	if (error != 0)
+		return (error);
+	hki.hook_type = HHOOK_TYPE_IPSEC_OUT;
+	error = hhook_add_hook(V_ipsec_hhh_out[HHOOK_IPSEC_INET],
+	    &hki, HHOOK_WAITOK);
+	if (error != 0)
+		return (error);
+#endif
+#ifdef INET6
+	hki.hook_id = AF_INET6;
+	hki.hook_type = HHOOK_TYPE_IPSEC_IN;
+	error = hhook_add_hook(V_ipsec_hhh_in[HHOOK_IPSEC_INET6],
+	    &hki, HHOOK_WAITOK);
+	if (error != 0)
+		return (error);
+	hki.hook_type = HHOOK_TYPE_IPSEC_OUT;
+	error = hhook_add_hook(V_ipsec_hhh_out[HHOOK_IPSEC_INET6],
+	    &hki, HHOOK_WAITOK);
+	if (error != 0)
+		return (error);
+#endif
+	return (error);
+}
+
+static void
+enc_remove_hhooks(struct enc_softc *sc)
+{
+	struct hookinfo hki;
+
+	hki.hook_func = enc_hhook;
+	hki.hook_helper = NULL;
+	hki.hook_udata = sc;
+#ifdef INET
+	hki.hook_id = AF_INET;
+	hki.hook_type = HHOOK_TYPE_IPSEC_IN;
+	hhook_remove_hook(V_ipsec_hhh_in[HHOOK_IPSEC_INET], &hki);
+	hki.hook_type = HHOOK_TYPE_IPSEC_OUT;
+	hhook_remove_hook(V_ipsec_hhh_out[HHOOK_IPSEC_INET], &hki);
+#endif
+#ifdef INET6
+	hki.hook_id = AF_INET6;
+	hki.hook_type = HHOOK_TYPE_IPSEC_IN;
+	hhook_remove_hook(V_ipsec_hhh_in[HHOOK_IPSEC_INET6], &hki);
+	hki.hook_type = HHOOK_TYPE_IPSEC_OUT;
+	hhook_remove_hook(V_ipsec_hhh_out[HHOOK_IPSEC_INET6], &hki);
+#endif
+}
+
+static void
+vnet_enc_init(const void *unused __unused)
+{
+
+	V_enc_sc = NULL;
+	V_enc_cloner = if_clone_simple(encname, enc_clone_create,
+	    enc_clone_destroy, 1);
+}
+VNET_SYSINIT(vnet_enc_init, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY,
+    vnet_enc_init, NULL);
+
+static void
+vnet_enc_uninit(const void *unused __unused)
+{
+
+	if_clone_detach(V_enc_cloner);
+}
+VNET_SYSUNINIT(vnet_enc_uninit, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY,
+    vnet_enc_uninit, NULL);
+
 static int
 enc_modevent(module_t mod, int type, void *data)
 {
+
 	switch (type) {
 	case MOD_LOAD:
-		mtx_init(&enc_mtx, "enc mtx", NULL, MTX_DEF);
-		enc_cloner = if_clone_simple(encname, enc_clone_create,
-		    enc_clone_destroy, 1);
-		break;
 	case MOD_UNLOAD:
-		printf("enc module unload - not possible for this module\n");
+		break;
-		return (EINVAL);
 	default:
 		return (EOPNOTSUPP);
 	}
@@ -187,184 +401,3 @@ static moduledata_t enc_mod = {
 };
 
 DECLARE_MODULE(if_enc, enc_mod, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY);
-
-static int
-enc_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
-    struct route *ro)
-{
-	m_freem(m);
-	return (0);
-}
-
-/*
- * Process an ioctl request.
- */
-/* ARGSUSED */
-static int
-enc_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
-{
-	int error = 0;
-
-	mtx_lock(&enc_mtx);
-
-	switch (cmd) {
-
-	case SIOCSIFFLAGS:
-		if (ifp->if_flags & IFF_UP)
-			ifp->if_drv_flags |= IFF_DRV_RUNNING;
-		else
-			ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
-
-		break;
-
-	default:
-		error = EINVAL;
-	}
-
-	mtx_unlock(&enc_mtx);
-	return (error);
-}
-
-int
-ipsec_filter(struct mbuf **mp, int dir, int flags)
-{
-	int error, i;
-	struct ip *ip;
-	struct ifnet *rcvif;
-
-	KASSERT(encif != NULL, ("%s: encif is null", __func__));
-	KASSERT(flags & (ENC_IN|ENC_OUT),
-		("%s: invalid flags: %04x", __func__, flags));
-
-	if ((encif->if_drv_flags & IFF_DRV_RUNNING) == 0)
-		return (0);
-
-	if (flags & ENC_IN) {
-		if ((flags & ipsec_filter_mask_in) == 0)
-			return (0);
-	} else {
-		if ((flags & ipsec_filter_mask_out) == 0)
-			return (0);
-	}
-
-	/* Skip pfil(9) if no filters are loaded */
-	if (1
-#ifdef INET
-	    && !PFIL_HOOKED(&V_inet_pfil_hook)
-#endif
-#ifdef INET6
-	    && !PFIL_HOOKED(&V_inet6_pfil_hook)
-#endif
-	    ) {
-		return (0);
-	}
-
-	i = min((*mp)->m_pkthdr.len, max_protohdr);
-	if ((*mp)->m_len < i) {
-		*mp = m_pullup(*mp, i);
-		if (*mp == NULL) {
-			printf("%s: m_pullup failed\n", __func__);
-			return (-1);
-		}
-	}
-
-	error = 0;
-	rcvif = (*mp)->m_pkthdr.rcvif;
-	(*mp)->m_pkthdr.rcvif = encif;
-	ip = mtod(*mp, struct ip *);
-	switch (ip->ip_v) {
-#ifdef INET
-		case 4:
-			error = pfil_run_hooks(&V_inet_pfil_hook, mp,
-			    encif, dir, NULL);
-			break;
-#endif
-#ifdef INET6
-		case 6:
-			error = pfil_run_hooks(&V_inet6_pfil_hook, mp,
-			    encif, dir, NULL);
-			break;
-#endif
-		default:
-			printf("%s: unknown IP version\n", __func__);
-	}
-
-	/*
-	 * If the mbuf was consumed by the filter for requeueing (dummynet, etc)
-	 * then error will be zero but we still want to return an error to our
-	 * caller so the null mbuf isn't forwarded further.
-	 */
-	if (*mp == NULL && error == 0)
-		return (-1);	/* Consumed by the filter */
-	if (*mp == NULL)
-		return (error);
-	if (error != 0)
-		goto bad;
-
-	(*mp)->m_pkthdr.rcvif = rcvif;
-	return (error);
-
-bad:
-	m_freem(*mp);
-	*mp = NULL;
-	return (error);
-}
-
-void
-ipsec_bpf(struct mbuf *m, struct secasvar *sav, int af, int flags)
-{
-	int mflags;
-	struct enchdr hdr;
-
-	KASSERT(encif != NULL, ("%s: encif is null", __func__));
-	KASSERT(flags & (ENC_IN|ENC_OUT),
-		("%s: invalid flags: %04x", __func__, flags));
-
-	if ((encif->if_drv_flags & IFF_DRV_RUNNING) == 0)
-		return;
-
-	if (flags & ENC_IN) {
-		if ((flags & ipsec_bpf_mask_in) == 0)
-			return;
-	} else {
-		if ((flags & ipsec_bpf_mask_out) == 0)
-			return;
-	}
-
-	if (bpf_peers_present(encif->if_bpf)) {
-		mflags = 0;
-		hdr.spi = 0;
-		if (!sav) {
-			struct m_tag *mtag;
-			mtag = m_tag_find(m, PACKET_TAG_IPSEC_IN_DONE, NULL);
-			if (mtag != NULL) {
-				struct tdb_ident *tdbi;
-				tdbi = (struct tdb_ident *) (mtag + 1);
-				if (tdbi->alg_enc != SADB_EALG_NONE)
-					mflags |= M_CONF;
-				if (tdbi->alg_auth != SADB_AALG_NONE)
-					mflags |= M_AUTH;
-				hdr.spi = tdbi->spi;
-			}
-		} else {
-			if (sav->alg_enc != SADB_EALG_NONE)
-				mflags |= M_CONF;
-			if (sav->alg_auth != SADB_AALG_NONE)
-				mflags |= M_AUTH;
-			hdr.spi = sav->spi;
-		}
-
-		/*
-		 * We need to prepend the address family as a four byte
-		 * field.  Cons up a dummy header to pacify bpf.  This
-		 * is safe because bpf will only read from the mbuf
-		 * (i.e., it won't try to free it or keep a pointer a
-		 * to it).
-		 */
-		hdr.af = af;
-		/* hdr.spi already set above */
-		hdr.flags = mflags;
-
-		bpf_mtap2(encif->if_bpf, &hdr, sizeof(hdr), m);
-	}
-}

sys/net/if_enc.h

@@ -30,6 +30,13 @@
 #ifndef _NET_IF_ENC_H
 #define _NET_IF_ENC_H
 
-extern struct ifnet	*encif;
+struct ipsec_ctx_data {
+	struct mbuf	**mp;
+	struct secasvar	*sav;
+	uint8_t		af;
+#define	IPSEC_ENC_BEFORE	0x01
+#define	IPSEC_ENC_AFTER		0x02
+	uint8_t		enc;
+};
 
 #endif /* _NET_IF_ENC_H */

sys/net/if_var.h

@@ -93,6 +93,14 @@ TAILQ_HEAD(ifgrouphead, ifg_group);
 #ifdef _KERNEL
 VNET_DECLARE(struct pfil_head, link_pfil_hook);	/* packet filter hooks */
 #define	V_link_pfil_hook	VNET(link_pfil_hook)
+
+#define	HHOOK_IPSEC_INET	0
+#define	HHOOK_IPSEC_INET6	1
+#define	HHOOK_IPSEC_COUNT	2
+VNET_DECLARE(struct hhook_head *, ipsec_hhh_in[HHOOK_IPSEC_COUNT]);
+VNET_DECLARE(struct hhook_head *, ipsec_hhh_out[HHOOK_IPSEC_COUNT]);
+#define	V_ipsec_hhh_in	VNET(ipsec_hhh_in)
+#define	V_ipsec_hhh_out	VNET(ipsec_hhh_out)
 #endif /* _KERNEL */
 
 typedef enum {

sys/netinet/ip_input.c

@@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <sys/hhook.h>
 #include <sys/mbuf.h>
 #include <sys/malloc.h>
 #include <sys/domain.h>
@@ -318,6 +319,17 @@ ip_init(void)
 		printf("%s: WARNING: unable to register pfil hook, "
 			"error %d\n", __func__, i);
 
+	if (hhook_head_register(HHOOK_TYPE_IPSEC_IN, AF_INET,
+	    &V_ipsec_hhh_in[HHOOK_IPSEC_INET],
+	    HHOOK_WAITOK | HHOOK_HEADISINVNET) != 0)
+		printf("%s: WARNING: unable to register input helper hook\n",
+		    __func__);
+	if (hhook_head_register(HHOOK_TYPE_IPSEC_OUT, AF_INET,
+	    &V_ipsec_hhh_out[HHOOK_IPSEC_INET],
+	    HHOOK_WAITOK | HHOOK_HEADISINVNET) != 0)
+		printf("%s: WARNING: unable to register output helper hook\n",
+		    __func__);
+
 	/* Skip initialization of globals for non-default instances. */
 	if (!IS_DEFAULT_VNET(curvnet))
 		return;
@@ -352,12 +364,24 @@ ip_init(void)
 void
 ip_destroy(void)
 {
-	int i;
+	int error;
 
-	if ((i = pfil_head_unregister(&V_inet_pfil_hook)) != 0)
+	if ((error = pfil_head_unregister(&V_inet_pfil_hook)) != 0)
 		printf("%s: WARNING: unable to unregister pfil hook, "
-		    "error %d\n", __func__, i);
+		    "error %d\n", __func__, error);
 
+	error = hhook_head_deregister(V_ipsec_hhh_in[HHOOK_IPSEC_INET]);
+	if (error != 0) {
+		printf("%s: WARNING: unable to deregister input helper hook "
+		    "type HHOOK_TYPE_IPSEC_IN, id HHOOK_IPSEC_INET: "
+		    "error %d returned\n", __func__, error);
+	}
+	error = hhook_head_deregister(V_ipsec_hhh_out[HHOOK_IPSEC_INET]);
+	if (error != 0) {
+		printf("%s: WARNING: unable to deregister output helper hook "
+		    "type HHOOK_TYPE_IPSEC_OUT, id HHOOK_IPSEC_INET: "
+		    "error %d returned\n", __func__, error);
+	}
 	/* Cleanup in_ifaddr hash table; should be empty. */
 	hashdestroy(V_in_ifaddrhashtbl, M_IFADDR, V_in_ifaddrhmask);
 

sys/netinet6/ip6_input.c

@@ -72,6 +72,7 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <sys/hhook.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <sys/proc.h>
@@ -201,6 +202,17 @@ ip6_init(void)
 		printf("%s: WARNING: unable to register pfil hook, "
 			"error %d\n", __func__, i);
 
+	if (hhook_head_register(HHOOK_TYPE_IPSEC_IN, AF_INET6,
+	    &V_ipsec_hhh_in[HHOOK_IPSEC_INET6],
+	    HHOOK_WAITOK | HHOOK_HEADISINVNET) != 0)
+		printf("%s: WARNING: unable to register input helper hook\n",
+		    __func__);
+	if (hhook_head_register(HHOOK_TYPE_IPSEC_OUT, AF_INET6,
+	    &V_ipsec_hhh_out[HHOOK_IPSEC_INET6],
+	    HHOOK_WAITOK | HHOOK_HEADISINVNET) != 0)
+		printf("%s: WARNING: unable to register output helper hook\n",
+		    __func__);
+
 	scope6_init();
 	addrsel_policy_init();
 	nd6_init();
@@ -300,11 +312,23 @@ ip6proto_unregister(short ip6proto)
 void
 ip6_destroy()
 {
-	int i;
+	int error;
 
-	if ((i = pfil_head_unregister(&V_inet6_pfil_hook)) != 0)
+	if ((error = pfil_head_unregister(&V_inet6_pfil_hook)) != 0)
 		printf("%s: WARNING: unable to unregister pfil hook, "
-		    "error %d\n", __func__, i);
+		    "error %d\n", __func__, error);
+	error = hhook_head_deregister(V_ipsec_hhh_in[HHOOK_IPSEC_INET6]);
+	if (error != 0) {
+		printf("%s: WARNING: unable to deregister input helper hook "
+		    "type HHOOK_TYPE_IPSEC_IN, id HHOOK_IPSEC_INET6: "
+		    "error %d returned\n", __func__, error);
+	}
+	error = hhook_head_deregister(V_ipsec_hhh_out[HHOOK_IPSEC_INET6]);
+	if (error != 0) {
+		printf("%s: WARNING: unable to deregister output helper hook "
+		    "type HHOOK_TYPE_IPSEC_OUT, id HHOOK_IPSEC_INET6: "
+		    "error %d returned\n", __func__, error);
+	}
 	hashdestroy(V_in6_ifaddrhashtbl, M_IFADDR, V_in6_ifaddrhmask);
 	nd6_destroy();
 	callout_drain(&V_in6_tmpaddrtimer_ch);

sys/netipsec/ipsec.c

@@ -48,6 +48,7 @@
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/errno.h>
+#include <sys/hhook.h>
 #include <sys/time.h>
 #include <sys/kernel.h>
 #include <sys/syslog.h>
@@ -55,6 +56,7 @@
 #include <sys/proc.h>
 
 #include <net/if.h>
+#include <net/if_enc.h>
 #include <net/if_var.h>
 #include <net/vnet.h>
 
@@ -806,6 +808,34 @@ ipsec6_setspidx_ipaddr(struct mbuf *m, struct secpolicyindex *spidx)
 }
 #endif
 
+int
+ipsec_run_hhooks(struct ipsec_ctx_data *ctx, int type)
+{
+	int idx;
+
+	switch (ctx->af) {
+#ifdef INET
+	case AF_INET:
+		idx = HHOOK_IPSEC_INET;
+		break;
+#endif
+#ifdef INET6
+	case AF_INET6:
+		idx = HHOOK_IPSEC_INET6;
+		break;
+#endif
+	default:
+		return (EPFNOSUPPORT);
+	}
+	if (type == HHOOK_TYPE_IPSEC_IN)
+		HHOOKS_RUN_IF(V_ipsec_hhh_in[idx], ctx, NULL);
+	else
+		HHOOKS_RUN_IF(V_ipsec_hhh_out[idx], ctx, NULL);
+	if (*ctx->mp == NULL)
+		return (EACCES);
+	return (0);
+}
+
 static void
 ipsec_delpcbpolicy(struct inpcbpolicy *p)
 {

sys/netipsec/ipsec.h

@@ -258,6 +258,15 @@ struct ipsecstat {
 #ifdef _KERNEL
 #include <sys/counter.h>
 
+struct ipsec_ctx_data;
+#define	IPSEC_INIT_CTX(_ctx, _mp, _sav, _af, _enc) do {	\
+	(_ctx)->mp = (_mp);				\
+	(_ctx)->sav = (_sav);				\
+	(_ctx)->af = (_af);				\
+	(_ctx)->enc = (_enc);				\
+} while(0)
+int	ipsec_run_hhooks(struct ipsec_ctx_data *ctx, int direction);
+
 VNET_DECLARE(int, ipsec_debug);
 #define	V_ipsec_debug		VNET(ipsec_debug)
 
@@ -352,14 +361,6 @@ extern	struct mbuf *m_makespace(struct mbuf *m0, int skip, int hlen, int *off);
 extern	caddr_t m_pad(struct mbuf *m, int n);
 extern	int m_striphdr(struct mbuf *m, int skip, int hlen);
 
-#ifdef DEV_ENC
-#define	ENC_BEFORE	0x0001
-#define	ENC_AFTER	0x0002
-#define	ENC_IN		0x0100
-#define	ENC_OUT		0x0200
-extern	int ipsec_filter(struct mbuf **, int, int);
-extern	void ipsec_bpf(struct mbuf *, struct secasvar *, int, int);
-#endif
 #endif /* _KERNEL */
 
 #ifndef _KERNEL

sys/netipsec/ipsec_input.c

@@ -43,7 +43,6 @@
 #include "opt_inet.h"
 #include "opt_inet6.h"
 #include "opt_ipsec.h"
-#include "opt_enc.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -53,11 +52,12 @@
 #include <sys/protosw.h>
 #include <sys/socket.h>
 #include <sys/errno.h>
+#include <sys/hhook.h>
 #include <sys/syslog.h>
 
 #include <net/if.h>
 #include <net/if_var.h>
-#include <net/pfil.h>
+#include <net/if_enc.h>
 #include <net/netisr.h>
 #include <net/vnet.h>
 
@@ -94,10 +94,6 @@
 #include <machine/in_cksum.h>
 #include <machine/stdarg.h>
 
-#ifdef DEV_ENC
-#include <net/if_enc.h>
-#endif
-
 
 #define	IPSEC_ISTAT(proto, name)	do {	\
 	if ((proto) == IPPROTO_ESP)		\
@@ -314,6 +310,7 @@ ipsec4_common_input_cb(struct mbuf *m, struct secasvar *sav, int skip,
     int protoff)
 {
 	char buf[INET6_ADDRSTRLEN];
+	struct ipsec_ctx_data ctx;
 	int prot, af, sproto, isr_prot;
 	struct ip *ip;
 	struct m_tag *mtag;
@@ -368,16 +365,10 @@ ipsec4_common_input_cb(struct mbuf *m, struct secasvar *sav, int skip,
 	}
 	prot = ip->ip_p;
 
-#ifdef DEV_ENC
+	IPSEC_INIT_CTX(&ctx, &m, sav, AF_INET, IPSEC_ENC_BEFORE);
-	if_inc_counter(encif, IFCOUNTER_IPACKETS, 1);
+	if ((error = ipsec_run_hhooks(&ctx, HHOOK_TYPE_IPSEC_IN)) != 0)
-	if_inc_counter(encif, IFCOUNTER_IBYTES, m->m_pkthdr.len);
+		goto bad;
-
-	/* Pass the mbuf to enc0 for bpf and pfil. */
-	ipsec_bpf(m, sav, AF_INET, ENC_IN|ENC_BEFORE);
-	if ((error = ipsec_filter(&m, PFIL_IN, ENC_IN|ENC_BEFORE)) != 0)
-		return (error);
 	ip = mtod(m, struct ip *);
-#endif /* DEV_ENC */
 
 	/* IP-in-IP encapsulation */
 	if (prot == IPPROTO_IPIP &&
@@ -501,32 +492,18 @@ ipsec4_common_input_cb(struct mbuf *m, struct secasvar *sav, int skip,
 	 */
 	if (saidx->mode == IPSEC_MODE_TRANSPORT)
 		prot = IPPROTO_IPIP;
-#ifdef DEV_ENC
-	/*
-	 * Pass the mbuf to enc0 for bpf and pfil.
-	 */
-	if (prot == IPPROTO_IPIP)
-		ipsec_bpf(m, sav, AF_INET, ENC_IN|ENC_AFTER);
-#ifdef INET6
-	if (prot == IPPROTO_IPV6)
-		ipsec_bpf(m, sav, AF_INET6, ENC_IN|ENC_AFTER);
-#endif
-
-	if ((error = ipsec_filter(&m, PFIL_IN, ENC_IN|ENC_AFTER)) != 0)
-		return (error);
-#endif /* DEV_ENC */
-
 	/*
 	 * Re-dispatch via software interrupt.
 	 */
-
 	switch (prot) {
 	case IPPROTO_IPIP:
 		isr_prot = NETISR_IP;
+		af = AF_INET;
 		break;
 #ifdef INET6
 	case IPPROTO_IPV6:
 		isr_prot = NETISR_IPV6;
+		af = AF_INET6;
 		break;
 #endif
 	default:
@@ -537,6 +514,9 @@ ipsec4_common_input_cb(struct mbuf *m, struct secasvar *sav, int skip,
 		goto bad;
 	}
 
+	IPSEC_INIT_CTX(&ctx, &m, sav, af, IPSEC_ENC_AFTER);
+	if ((error = ipsec_run_hhooks(&ctx, HHOOK_TYPE_IPSEC_IN)) != 0)
+		goto bad;
 	error = netisr_queue_src(isr_prot, (uintptr_t)sav->spi, m);
 	if (error) {
 		IPSEC_ISTAT(sproto, qfull);
@@ -611,6 +591,7 @@ ipsec6_common_input_cb(struct mbuf *m, struct secasvar *sav, int skip,
     int protoff)
 {
 	char buf[INET6_ADDRSTRLEN];
+	struct ipsec_ctx_data ctx;
 	int prot, af, sproto;
 	struct ip6_hdr *ip6;
 	struct m_tag *mtag;
@@ -658,20 +639,13 @@ ipsec6_common_input_cb(struct mbuf *m, struct secasvar *sav, int skip,
 	ip6 = mtod(m, struct ip6_hdr *);
 	ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(struct ip6_hdr));
 
+	IPSEC_INIT_CTX(&ctx, &m, sav, af, IPSEC_ENC_BEFORE);
+	if ((error = ipsec_run_hhooks(&ctx, HHOOK_TYPE_IPSEC_IN)) != 0)
+		goto bad;
 	/* Save protocol */
 	m_copydata(m, protoff, 1, &nxt8);
 	prot = nxt8;
 
-#ifdef DEV_ENC
-	if_inc_counter(encif, IFCOUNTER_IPACKETS, 1);
-	if_inc_counter(encif, IFCOUNTER_IBYTES, m->m_pkthdr.len);
-
-	/* Pass the mbuf to enc0 for bpf and pfil. */
-	ipsec_bpf(m, sav, AF_INET6, ENC_IN|ENC_BEFORE);
-	if ((error = ipsec_filter(&m, PFIL_IN, ENC_IN|ENC_BEFORE)) != 0)
-		return (error);
-#endif /* DEV_ENC */
-
 	/* IPv6-in-IP encapsulation */
 	if (prot == IPPROTO_IPV6 &&
 	    saidx->mode != IPSEC_MODE_TRANSPORT) {
@@ -778,20 +752,16 @@ ipsec6_common_input_cb(struct mbuf *m, struct secasvar *sav, int skip,
 
 	key_sa_recordxfer(sav, m);
 
-#ifdef DEV_ENC
+
-	/*
-	 * Pass the mbuf to enc0 for bpf and pfil.
-	 */
 #ifdef INET
 	if (prot == IPPROTO_IPIP)
-		ipsec_bpf(m, sav, AF_INET, ENC_IN|ENC_AFTER);
+		af = AF_INET;
+	else
 #endif
-	if (prot == IPPROTO_IPV6)
+		af = AF_INET6;
-		ipsec_bpf(m, sav, AF_INET6, ENC_IN|ENC_AFTER);
+	IPSEC_INIT_CTX(&ctx, &m, sav, af, IPSEC_ENC_AFTER);
-
+	if ((error = ipsec_run_hhooks(&ctx, HHOOK_TYPE_IPSEC_IN)) != 0)
-	if ((error = ipsec_filter(&m, PFIL_IN, ENC_IN|ENC_AFTER)) != 0)
+		goto bad;
-		return (error);
-#endif /* DEV_ENC */
 	if (skip == 0) {
 		/*
 		 * We stripped outer IPv6 header.

sys/netipsec/ipsec_output.c

@@ -32,7 +32,6 @@
 #include "opt_inet.h"
 #include "opt_inet6.h"
 #include "opt_ipsec.h"
-#include "opt_enc.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -41,11 +40,12 @@
 #include <sys/protosw.h>
 #include <sys/socket.h>
 #include <sys/errno.h>
+#include <sys/hhook.h>
 #include <sys/syslog.h>
 
 #include <net/if.h>
+#include <net/if_enc.h>
 #include <net/if_var.h>
-#include <net/pfil.h>
 #include <net/vnet.h>
 
 #include <netinet/in.h>
@@ -88,11 +88,6 @@
 #include <netinet/udp.h>
 #endif
 
-#ifdef DEV_ENC
-#include <net/if_enc.h>
-#endif
-
-
 int
 ipsec_process_done(struct mbuf *m, struct ipsecrequest *isr)
 {
@@ -531,6 +526,7 @@ int
 ipsec4_process_packet(struct mbuf *m, struct ipsecrequest *isr)
 {
 	char sbuf[INET6_ADDRSTRLEN], dbuf[INET6_ADDRSTRLEN];
+	struct ipsec_ctx_data ctx;
 	union sockaddr_union *dst;
 	struct secasindex saidx;
 	struct secasvar *sav;
@@ -555,19 +551,13 @@ ipsec4_process_packet(struct mbuf *m, struct ipsecrequest *isr)
 		error = ENOBUFS;
 		goto bad;
 	}
+
+	IPSEC_INIT_CTX(&ctx, &m, sav, AF_INET, IPSEC_ENC_BEFORE);
+	if ((error = ipsec_run_hhooks(&ctx, HHOOK_TYPE_IPSEC_OUT)) != 0)
+		goto bad;
+
 	ip = mtod(m, struct ip *);
 	dst = &sav->sah->saidx.dst;
-#ifdef DEV_ENC
-	if_inc_counter(encif, IFCOUNTER_OPACKETS, 1);
-	if_inc_counter(encif, IFCOUNTER_OBYTES, m->m_pkthdr.len);
-
-	/* pass the mbuf to enc0 for bpf processing */
-	ipsec_bpf(m, sav, AF_INET, ENC_OUT|ENC_BEFORE);
-	/* pass the mbuf to enc0 for packet filtering */
-	if ((error = ipsec_filter(&m, PFIL_OUT, ENC_OUT|ENC_BEFORE)) != 0)
-		goto bad;
-	ip = mtod(m, struct ip *);
-#endif
 	/* Do the appropriate encapsulation, if necessary */
 	if (isr->saidx.mode == IPSEC_MODE_TUNNEL || /* Tunnel requ'd */
 	    dst->sa.sa_family != AF_INET ||	    /* PF mismatch */
@@ -589,13 +579,10 @@ ipsec4_process_packet(struct mbuf *m, struct ipsecrequest *isr)
 			goto bad;
 		}
 	}
-#ifdef DEV_ENC
+
-	/* pass the mbuf to enc0 for bpf processing */
+	IPSEC_INIT_CTX(&ctx, &m, sav, dst->sa.sa_family, IPSEC_ENC_AFTER);
-	ipsec_bpf(m, sav, sav->sah->saidx.dst.sa.sa_family, ENC_OUT|ENC_AFTER);
+	if ((error = ipsec_run_hhooks(&ctx, HHOOK_TYPE_IPSEC_OUT)) != 0)
-	/* pass the mbuf to enc0 for packet filtering */
-	if ((error = ipsec_filter(&m, PFIL_OUT, ENC_OUT|ENC_AFTER)) != 0)
 		goto bad;
-#endif
 
 	/*
 	 * Dispatch to the appropriate IPsec transform logic.  The
@@ -657,6 +644,7 @@ int
 ipsec6_process_packet(struct mbuf *m, struct ipsecrequest *isr)
 {
 	char sbuf[INET6_ADDRSTRLEN], dbuf[INET6_ADDRSTRLEN];
+	struct ipsec_ctx_data ctx;
 	struct secasindex saidx;
 	struct secasvar *sav;
 	struct ip6_hdr *ip6;
@@ -677,19 +665,12 @@ ipsec6_process_packet(struct mbuf *m, struct ipsecrequest *isr)
 	sav = isr->sav;
 	dst = &sav->sah->saidx.dst;
 
+	IPSEC_INIT_CTX(&ctx, &m, sav, AF_INET6, IPSEC_ENC_BEFORE);
+	if ((error = ipsec_run_hhooks(&ctx, HHOOK_TYPE_IPSEC_OUT)) != 0)
+		goto bad;
+
 	ip6 = mtod(m, struct ip6_hdr *);
 	ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6));
-#ifdef DEV_ENC
-	if_inc_counter(encif, IFCOUNTER_OPACKETS, 1);
-	if_inc_counter(encif, IFCOUNTER_OBYTES, m->m_pkthdr.len);
-
-	/* pass the mbuf to enc0 for bpf processing */
-	ipsec_bpf(m, isr->sav, AF_INET6, ENC_OUT|ENC_BEFORE);
-	/* pass the mbuf to enc0 for packet filtering */
-	if ((error = ipsec_filter(&m, PFIL_OUT, ENC_OUT|ENC_BEFORE)) != 0)
-		goto bad;
-	ip6 = mtod(m, struct ip6_hdr *);
-#endif /* DEV_ENC */
 
 	/* Do the appropriate encapsulation, if necessary */
 	if (isr->saidx.mode == IPSEC_MODE_TUNNEL || /* Tunnel requ'd */
@@ -715,12 +696,9 @@ ipsec6_process_packet(struct mbuf *m, struct ipsecrequest *isr)
 		}
 	}
 
-#ifdef DEV_ENC
+	IPSEC_INIT_CTX(&ctx, &m, sav, dst->sa.sa_family, IPSEC_ENC_AFTER);
-	ipsec_bpf(m, isr->sav, dst->sa.sa_family, ENC_OUT|ENC_AFTER);
+	if ((error = ipsec_run_hhooks(&ctx, HHOOK_TYPE_IPSEC_OUT)) != 0)
-	/* pass the mbuf to enc0 for packet filtering */
-	if ((error = ipsec_filter(&m, PFIL_OUT, ENC_OUT|ENC_AFTER)) != 0)
 		goto bad;
-#endif /* DEV_ENC */
 
 	switch(dst->sa.sa_family) {
 #ifdef INET
@@ -741,14 +719,13 @@ ipsec6_process_packet(struct mbuf *m, struct ipsecrequest *isr)
 		DPRINTF(("%s: unsupported protocol family %u\n",
 				 __func__, dst->sa.sa_family));
 		error = EPFNOSUPPORT;
-		IPSEC6STAT_INC(ips_out_inval);
 		goto bad;
 	}
 	error = (*sav->tdb_xform->xf_output)(m, isr, NULL, i, off);
 	IPSECREQUEST_UNLOCK(isr);
 	return error;
 bad:
-
+	IPSEC6STAT_INC(ips_out_inval);
 	if (isr)
 		IPSECREQUEST_UNLOCK(isr);
 	if (m)

sys/sys/hhook.h

@@ -65,6 +65,8 @@
 /* Helper hook types. */
 #define	HHOOK_TYPE_TCP		1
 #define	HHOOK_TYPE_SOCKET	2
+#define	HHOOK_TYPE_IPSEC_IN	3
+#define	HHOOK_TYPE_IPSEC_OUT	4
 
 struct helper;
 struct osd;