Add support for IPsec ESN and pass relevant information to crypto layer

Implement support for including IPsec ESN (Extended Sequence Number) to
both encrypt and authenticate mode (eg. AES-CBC and SHA256) and combined
mode (eg. AES-GCM). Both ESP and AH protocols are updated. Additionally
pass relevant information about ESN to crypto layer.

For the ETA mode the ESN is stored in separate crp_esn buffer because
the high-order 32 bits of the sequence number are appended after the
Next Header (RFC 4303).

For the AEAD modes the high-order 32 bits of the sequence number
[e.g.  RFC 4106, Chapter 5 AAD Construction] are included as part of
crp_aad (SPI + ESN (32 high order bits) + Seq nr (32 low order bits)).

Submitted by:           Grzegorz Jaszczyk <jaz@semihalf.com>
                        Patryk Duda <pdk@semihalf.com>
Reviewed by:            jhb, gnn
Differential revision:  https://reviews.freebsd.org/D22369
Obtained from:          Semihalf
Sponsored by:           Stormshield

sys/netipsec/keydb.h

@@ -197,6 +197,8 @@ struct secasvar {
 #define	SAV_ISCTR(_sav) ((_sav)->alg_enc == SADB_X_EALG_AESCTR)
 #define SAV_ISCTRORGCM(_sav)	(SAV_ISCTR((_sav)) || SAV_ISGCM((_sav)))
 
+#define	IPSEC_SEQH_SHIFT	32
+
 /* Replay prevention, protected by SECASVAR_LOCK:
  *  (m) locked by mtx
  *  (c) read only except during creation / free

sys/netipsec/xform_ah.c

@@ -236,6 +236,10 @@ ah_init(struct secasvar *sav, struct xformsw *xsp)
 
 	memset(&csp, 0, sizeof(csp));
 	csp.csp_mode = CSP_MODE_DIGEST;
+
+	if (sav->flags & SADB_X_SAFLAGS_ESN)
+		csp.csp_flags |= CSP_F_ESN;
+
 	error = ah_init0(sav, xsp, &csp);
 	return error ? error :
 		 crypto_newsession(&sav->tdb_cryptoid, &csp, V_crypto_support);
@@ -654,6 +658,12 @@ ah_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 	crp->crp_callback = ah_input_cb;
 	crp->crp_opaque = xd;
 
+	if (sav->flags & SADB_X_SAFLAGS_ESN &&
+	    sav->replay != NULL && sav->replay->wsize != 0) {
+		seqh = htonl(seqh);
+		memcpy(crp->crp_esn, &seqh, sizeof(seqh));
+	}
+
 	/* These are passed as-is to the callback. */
 	xd->sav = sav;
 	xd->nxt = hl;
@@ -834,6 +844,7 @@ ah_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 	uint16_t iplen;
 	int error, rplen, authsize, ahsize, maxpacketsize, roff;
 	uint8_t prot;
+	uint32_t seqh;
 
 	IPSEC_ASSERT(sav != NULL, ("null SA"));
 	ahx = sav->tdb_authalgxform;
@@ -1031,6 +1042,11 @@ ah_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 	crp->crp_callback = ah_output_cb;
 	crp->crp_opaque = xd;
 
+	if (sav->flags & SADB_X_SAFLAGS_ESN && sav->replay != NULL) {
+		seqh = htonl((uint32_t)(sav->replay->count >> IPSEC_SEQH_SHIFT));
+		memcpy(crp->crp_esn, &seqh, sizeof(seqh));
+	}
+
 	/* These are passed as-is to the callback. */
 	xd->sp = sp;
 	xd->sav = sav;

sys/netipsec/xform_esp.c

@@ -80,6 +80,8 @@
 #include <opencrypto/cryptodev.h>
 #include <opencrypto/xform.h>
 
+#define SPI_SIZE	4
+
 VNET_DEFINE(int, esp_enable) = 1;
 VNET_DEFINE_STATIC(int, esp_ctr_compatibility) = 1;
 #define V_esp_ctr_compatibility VNET(esp_ctr_compatibility)
@@ -219,9 +221,13 @@ esp_init(struct secasvar *sav, struct xformsw *xsp)
 			return EINVAL;
 		}
 		csp.csp_mode = CSP_MODE_AEAD;
-	} else if (sav->alg_auth != 0)
+		if (sav->flags & SADB_X_SAFLAGS_ESN)
+			csp.csp_flags |= CSP_F_SEPARATE_AAD;
+	} else if (sav->alg_auth != 0) {
 		csp.csp_mode = CSP_MODE_ETA;
-	else
+		if (sav->flags & SADB_X_SAFLAGS_ESN)
+			csp.csp_flags |= CSP_F_ESN;
+	} else
 		csp.csp_mode = CSP_MODE_CIPHER;
 
 	/* Initialize crypto session. */
@@ -263,6 +269,7 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 	crypto_session_t cryptoid;
 	int alen, error, hlen, plen;
 	uint32_t seqh;
+	const struct crypto_session_params *csp;
 
 	IPSEC_ASSERT(sav != NULL, ("null SA"));
 	IPSEC_ASSERT(sav->tdb_encalgxform != NULL, ("null encoding xform"));
@@ -329,6 +336,7 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 			error = EACCES;
 			goto bad;
 		}
+		seqh = htonl(seqh);
 	}
 	cryptoid = sav->tdb_cryptoid;
 	SECASVAR_UNLOCK(sav);
@@ -350,19 +358,49 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 	xd = malloc(sizeof(*xd), M_XDATA, M_NOWAIT | M_ZERO);
 	if (xd == NULL) {
 		DPRINTF(("%s: failed to allocate xform_data\n", __func__));
-		ESPSTAT_INC(esps_crypto);
+		goto xd_fail;
-		crypto_freereq(crp);
-		error = ENOBUFS;
-		goto bad;
 	}
 
 	if (esph != NULL) {
 		crp->crp_op = CRYPTO_OP_VERIFY_DIGEST;
-		crp->crp_aad_start = skip;
 		if (SAV_ISGCM(sav))
 			crp->crp_aad_length = 8; /* RFC4106 5, SPI + SN */
 		else
 			crp->crp_aad_length = hlen;
+
+		csp = crypto_get_params(crp->crp_session);
+		if ((csp->csp_flags & CSP_F_SEPARATE_AAD) &&
+		    (sav->replay != NULL) && (sav->replay->wsize != 0)) {
+			int aad_skip;
+
+			crp->crp_aad_length += sizeof(seqh);
+			crp->crp_aad = malloc(crp->crp_aad_length, M_XDATA, M_NOWAIT);
+			if (crp->crp_aad == NULL) {
+				DPRINTF(("%s: failed to allocate xform_data\n",
+					 __func__));
+				goto crp_aad_fail;
+			}
+
+			/* SPI */
+			m_copydata(m, skip, SPI_SIZE, crp->crp_aad);
+			aad_skip = SPI_SIZE;
+
+			/* ESN */
+			bcopy(&seqh, (char *)crp->crp_aad + aad_skip, sizeof(seqh));
+			aad_skip += sizeof(seqh);
+
+			/* Rest of aad */
+			if (crp->crp_aad_length - aad_skip > 0)
+				m_copydata(m, skip + SPI_SIZE,
+					   crp->crp_aad_length - aad_skip,
+					   (char *)crp->crp_aad + aad_skip);
+		} else
+			crp->crp_aad_start = skip;
+
+		if (csp->csp_flags & CSP_F_ESN &&
+			   sav->replay != NULL && sav->replay->wsize != 0)
+			memcpy(crp->crp_esn, &seqh, sizeof(seqh));
+
 		crp->crp_digest_start = m->m_pkthdr.len - alen;
 	}
 
@@ -423,6 +461,13 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 		crp->crp_iv_start = skip + hlen - sav->ivlen;
 
 	return (crypto_dispatch(crp));
+
+crp_aad_fail:
+	free(xd, M_XDATA);
+xd_fail:
+	crypto_freereq(crp);
+	ESPSTAT_INC(esps_crypto);
+	error = ENOBUFS;
 bad:
 	m_freem(m);
 	key_freesav(&sav);
@@ -505,6 +550,7 @@ esp_input_cb(struct cryptop *crp)
 
 	/* Release the crypto descriptors */
 	free(xd, M_XDATA), xd = NULL;
+	free(crp->crp_aad, M_XDATA), crp->crp_aad = NULL;
 	crypto_freereq(crp), crp = NULL;
 
 	/*
@@ -614,8 +660,10 @@ esp_input_cb(struct cryptop *crp)
 		m_freem(m);
 	if (xd != NULL)
 		free(xd, M_XDATA);
-	if (crp != NULL)
+	if (crp != NULL) {
+		free(crp->crp_aad, M_XDATA);
 		crypto_freereq(crp);
+	}
 	return error;
 }
 /*
@@ -639,6 +687,8 @@ esp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 	int hlen, rlen, padding, blks, alen, i, roff;
 	int error, maxpacketsize;
 	uint8_t prot;
+	uint32_t seqh;
+	const struct crypto_session_params *csp;
 
 	IPSEC_ASSERT(sav != NULL, ("null SA"));
 	esph = sav->tdb_authalgxform;
@@ -745,6 +795,8 @@ esp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 
 		bcopy((caddr_t) &replay, mtod(mo, caddr_t) + roff +
 		    sizeof(uint32_t), sizeof(uint32_t));
+
+		seqh = htonl((uint32_t)(sav->replay->count >> IPSEC_SEQH_SHIFT));
 	}
 	cryptoid = sav->tdb_cryptoid;
 	if (SAV_ISCTRORGCM(sav))
@@ -801,13 +853,10 @@ esp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 	}
 
 	/* IPsec-specific opaque crypto info. */
-	xd =  malloc(sizeof(struct xform_data), M_XDATA, M_NOWAIT | M_ZERO);
+	xd = malloc(sizeof(struct xform_data), M_XDATA, M_NOWAIT | M_ZERO);
 	if (xd == NULL) {
-		crypto_freereq(crp);
 		DPRINTF(("%s: failed to allocate xform_data\n", __func__));
-		ESPSTAT_INC(esps_crypto);
+		goto xd_fail;
-		error = ENOBUFS;
-		goto bad;
 	}
 
 	/* Encryption descriptor. */
@@ -855,15 +904,54 @@ esp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 	if (esph) {
 		/* Authentication descriptor. */
 		crp->crp_op |= CRYPTO_OP_COMPUTE_DIGEST;
-		crp->crp_aad_start = skip;
 		if (SAV_ISGCM(sav))
 			crp->crp_aad_length = 8; /* RFC4106 5, SPI + SN */
 		else
 			crp->crp_aad_length = hlen;
+
+		csp = crypto_get_params(crp->crp_session);
+		if (csp->csp_flags & CSP_F_SEPARATE_AAD &&
+		    sav->replay != NULL) {
+			int aad_skip;
+
+			crp->crp_aad_length += sizeof(seqh);
+			crp->crp_aad = malloc(crp->crp_aad_length, M_XDATA, M_NOWAIT);
+			if (crp->crp_aad == NULL) {
+				DPRINTF(("%s: failed to allocate xform_data\n",
+					 __func__));
+				goto crp_aad_fail;
+			}
+
+			/* SPI */
+			m_copydata(m, skip, SPI_SIZE, crp->crp_aad);
+			aad_skip = SPI_SIZE;
+
+			/* ESN */
+			bcopy(&seqh, (char *)crp->crp_aad + aad_skip, sizeof(seqh));
+			aad_skip += sizeof(seqh);
+
+			/* Rest of aad */
+			if (crp->crp_aad_length - aad_skip > 0)
+				m_copydata(m, skip + SPI_SIZE,
+					   crp->crp_aad_length - aad_skip,
+					   (char *)crp->crp_aad + aad_skip);
+		} else
+			crp->crp_aad_start = skip;
+
+		if (csp->csp_flags & CSP_F_ESN && sav->replay != NULL)
+			memcpy(crp->crp_esn, &seqh, sizeof(seqh));
+
 		crp->crp_digest_start = m->m_pkthdr.len - alen;
 	}
 
 	return crypto_dispatch(crp);
+
+crp_aad_fail:
+	free(xd, M_XDATA);
+xd_fail:
+	crypto_freereq(crp);
+	ESPSTAT_INC(esps_crypto);
+	error = ENOBUFS;
 bad:
 	if (m)
 		m_freem(m);
@@ -918,6 +1006,7 @@ esp_output_cb(struct cryptop *crp)
 		goto bad;
 	}
 	free(xd, M_XDATA);
+	free(crp->crp_aad, M_XDATA);
 	crypto_freereq(crp);
 	ESPSTAT_INC(esps_hist[sav->alg_enc]);
 	if (sav->tdb_authalgxform != NULL)
@@ -951,6 +1040,7 @@ esp_output_cb(struct cryptop *crp)
 bad:
 	CURVNET_RESTORE();
 	free(xd, M_XDATA);
+	free(crp->crp_aad, M_XDATA);
 	crypto_freereq(crp);
 	key_freesav(&sav);
 	key_freesp(&sp);