diff --git a/sys/netinet/ip_ipsec.c b/sys/netinet/ip_ipsec.c
index 1550018a8062..133fa7c62edd 100644
--- a/sys/netinet/ip_ipsec.c
+++ b/sys/netinet/ip_ipsec.c
@@ -230,6 +230,10 @@ ip_ipsec_output(struct mbuf **m, struct inpcb *inp, int *flags, int *error)
 	struct secpolicy *sp = NULL;
 	struct tdb_ident *tdbi;
 	struct m_tag *mtag;
+
+	if (!key_havesp(IPSEC_DIR_OUTBOUND))
+		return 0;
+
 	/*
 	 * Check the security policy (SP) for the packet and, if
 	 * required, do IPsec-related processing.  There are two
diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c
index ba2d8347fb29..7bdd573aaeb0 100644
--- a/sys/netinet/tcp_subr.c
+++ b/sys/netinet/tcp_subr.c
@@ -1947,7 +1947,8 @@ ipsec_hdrsiz_tcp(struct tcpcb *tp)
 #endif
 	struct tcphdr *th;
 
-	if ((tp == NULL) || ((inp = tp->t_inpcb) == NULL))
+	if ((tp == NULL) || ((inp = tp->t_inpcb) == NULL) ||
+		(!key_havesp(IPSEC_DIR_OUTBOUND)))
 		return (0);
 	m = m_gethdr(M_NOWAIT, MT_DATA);
 	if (!m)
diff --git a/sys/netipsec/ipsec.c b/sys/netipsec/ipsec.c
index f27019d31df6..93e37e817b9e 100644
--- a/sys/netipsec/ipsec.c
+++ b/sys/netipsec/ipsec.c
@@ -1273,6 +1273,9 @@ ipsec46_in_reject(struct mbuf *m, struct inpcb *inp)
 	int error;
 	int result;
 
+	if (!key_havesp(IPSEC_DIR_INBOUND))
+		return 0;
+
 	IPSEC_ASSERT(m != NULL, ("null mbuf"));
 
 	/*
@@ -1405,6 +1408,9 @@ ipsec_hdrsiz(struct mbuf *m, u_int dir, struct inpcb *inp)
 	int error;
 	size_t size;
 
+	if (!key_havesp(dir))
+		return 0;
+
 	IPSEC_ASSERT(m != NULL, ("null mbuf"));
 
 	/* Get SP for this packet.