Fixed IPsec's HMAC_SHA256-512 support to be RFC4868 compliant.

This will break interoperability with all older versions of
FreeBSD for those algorithms.

Reviewed by:	bz, gnn
Obtained from:	NETASQ
MFC after:	1w

UPDATING

@@ -9,6 +9,16 @@ handbook.
 Items affecting the ports and packages system can be found in
 /usr/ports/UPDATING.  Please read that file before running portupgrade.
 
+20110218:
+	IPsec's HMAC_SHA256-512 support has been fixed to be RFC4868
+	compliant, and will now use half of hash for authentication.
+	This will break interoperability with all stacks (including all
+	actual FreeBSD versions) who implement
+	draft-ietf-ipsec-ciph-sha-256-00 (they use 96 bits of hash for
+	authentication).
+	The only workaround with such peers is to use another HMAC
+	algorithm for IPsec ("phase 2") authentication.
+
 NOTE TO PEOPLE WHO THINK THAT FreeBSD 9.x IS SLOW:
 	FreeBSD 9.x has many debugging features turned on, in both the kernel
 	and userland.  These features attempt to detect incorrect use of

sys/netipsec/key.c

@@ -6095,6 +6095,9 @@ key_getsizes_ah(
 		case SADB_X_AALG_MD5:	*min = *max = 16; break;
 		case SADB_X_AALG_SHA:	*min = *max = 20; break;
 		case SADB_X_AALG_NULL:	*min = 1; *max = 256; break;
+		case SADB_X_AALG_SHA2_256: *min = *max = 32; break;
+		case SADB_X_AALG_SHA2_384: *min = *max = 48; break;
+		case SADB_X_AALG_SHA2_512: *min = *max = 64; break;
 		default:
 			DPRINTF(("%s: unknown AH algorithm %u\n",
 				__func__, alg));
@@ -6120,7 +6123,11 @@ key_getcomb_ah()
 	for (i = 1; i <= SADB_AALG_MAX; i++) {
 #if 1
 		/* we prefer HMAC algorithms, not old algorithms */
-		if (i != SADB_AALG_SHA1HMAC && i != SADB_AALG_MD5HMAC)
+		if (i != SADB_AALG_SHA1HMAC &&
+		    i != SADB_AALG_MD5HMAC  &&
+		    i != SADB_X_AALG_SHA2_256 &&
+		    i != SADB_X_AALG_SHA2_384 &&
+		    i != SADB_X_AALG_SHA2_512)
 			continue;
 #endif
 		algo = ah_algorithm_lookup(i);

sys/netipsec/xform.h

@@ -46,6 +46,7 @@
 #include <opencrypto/xform.h>
 
 #define	AH_HMAC_HASHLEN		12	/* 96 bits of authenticator */
+#define	AH_HMAC_MAXHASHLEN	(SHA2_512_HASH_LEN/2)	/* Keep this updated */
 #define	AH_HMAC_INITIAL_RPL	1	/* replay counter initial value */
 
 /*

sys/netipsec/xform_ah.c

@@ -85,8 +85,7 @@
  * to use a fixed 16-byte authenticator.  The new algorithm use 12-byte
  * authenticator.
  */
-#define	AUTHSIZE(sav) \
+#define	AUTHSIZE(sav)	ah_authsize(sav)
-	((sav->flags & SADB_X_EXT_OLD) ? 16 : AH_HMAC_HASHLEN)
 
 VNET_DEFINE(int, ah_enable) = 1;	/* control flow of packets with AH */
 VNET_DEFINE(int, ah_cleartos) = 1;	/* clear ip_tos when doing AH calc */
@@ -105,6 +104,27 @@ static unsigned char ipseczeroes[256];	/* larger than an ip6 extension hdr */
 static int ah_input_cb(struct cryptop*);
 static int ah_output_cb(struct cryptop*);
 
+static int
+ah_authsize(struct secasvar *sav)
+{
+
+	IPSEC_ASSERT(sav != NULL, ("%s: sav == NULL", __func__));
+
+	if (sav->flags & SADB_X_EXT_OLD)
+		return 16;
+
+	switch (sav->alg_auth) {
+	case SADB_X_AALG_SHA2_256:
+		return 16;
+	case SADB_X_AALG_SHA2_384:
+		return 24;
+	case SADB_X_AALG_SHA2_512:
+		return 32;
+	default:
+		return AH_HMAC_HASHLEN;
+	}
+	/* NOTREACHED */
+}
 /*
  * NB: this is public for use by the PF_KEY support.
  */

sys/netipsec/xform_esp.c

@@ -303,7 +303,19 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 	else
 		hlen = sizeof (struct newesp) + sav->ivlen;
 	/* Authenticator hash size */
-	alen = esph ? AH_HMAC_HASHLEN : 0;
+	if (esph != NULL) {
+		switch (esph->type) {
+		case CRYPTO_SHA2_256_HMAC:
+		case CRYPTO_SHA2_384_HMAC:
+		case CRYPTO_SHA2_512_HMAC:
+			alen = esph->hashsize/2;
+			break;
+		default:
+			alen = AH_HMAC_HASHLEN;
+			break;
+		}
+	}else
+		alen = 0;
 
 	/*
 	 * Verify payload length is multiple of encryption algorithm
@@ -456,8 +468,8 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 static int
 esp_input_cb(struct cryptop *crp)
 {
-	u_int8_t lastthree[3], aalg[AH_HMAC_HASHLEN];
+	u_int8_t lastthree[3], aalg[AH_HMAC_MAXHASHLEN];
-	int hlen, skip, protoff, error;
+	int hlen, skip, protoff, error, alen;
 	struct mbuf *m;
 	struct cryptodesc *crd;
 	struct auth_hash *esph;
@@ -525,6 +537,16 @@ esp_input_cb(struct cryptop *crp)
 
 	/* If authentication was performed, check now. */
 	if (esph != NULL) {
+		switch (esph->type) {
+		case CRYPTO_SHA2_256_HMAC:
+		case CRYPTO_SHA2_384_HMAC:
+		case CRYPTO_SHA2_512_HMAC:
+			alen = esph->hashsize/2;
+			break;
+		default:
+			alen = AH_HMAC_HASHLEN;
+			break;
+		}
 		/*
 		 * If we have a tag, it means an IPsec-aware NIC did
 		 * the verification for us.  Otherwise we need to
@@ -533,13 +555,13 @@ esp_input_cb(struct cryptop *crp)
 		V_ahstat.ahs_hist[sav->alg_auth]++;
 		if (mtag == NULL) {
 			/* Copy the authenticator from the packet */
-			m_copydata(m, m->m_pkthdr.len - AH_HMAC_HASHLEN,
+			m_copydata(m, m->m_pkthdr.len - alen,
-				AH_HMAC_HASHLEN, aalg);
+				alen, aalg);
 
 			ptr = (caddr_t) (tc + 1);
 
 			/* Verify authenticator */
-			if (bcmp(ptr, aalg, AH_HMAC_HASHLEN) != 0) {
+			if (bcmp(ptr, aalg, alen) != 0) {
 				DPRINTF(("%s: "
 		    "authentication hash mismatch for packet in SA %s/%08lx\n",
 				    __func__,
@@ -552,7 +574,7 @@ esp_input_cb(struct cryptop *crp)
 		}
 
 		/* Remove trailing authenticator */
-		m_adj(m, -AH_HMAC_HASHLEN);
+		m_adj(m, -alen);
 	}
 
 	/* Release the crypto descriptors */
@@ -696,7 +718,16 @@ esp_output(
 	plen = rlen + padding;		/* Padded payload length. */
 
 	if (esph)
+		switch (esph->type) {
+		case CRYPTO_SHA2_256_HMAC:
+		case CRYPTO_SHA2_384_HMAC:
+		case CRYPTO_SHA2_512_HMAC:
+			alen = esph->hashsize/2;
+			break;
+		default:
 		alen = AH_HMAC_HASHLEN;
+			break;
+		}
 	else
 		alen = 0;
 
@@ -950,7 +981,7 @@ esp_output_cb(struct cryptop *crp)
 #ifdef REGRESSION
 	/* Emulate man-in-the-middle attack when ipsec_integrity is TRUE. */
 	if (V_ipsec_integrity) {
-		static unsigned char ipseczeroes[AH_HMAC_HASHLEN];
+		static unsigned char ipseczeroes[AH_HMAC_MAXHASHLEN];
 		struct auth_hash *esph;
 
 		/*
@@ -959,8 +990,20 @@ esp_output_cb(struct cryptop *crp)
 		 */
 		esph = sav->tdb_authalgxform;
 		if (esph !=  NULL) {
-			m_copyback(m, m->m_pkthdr.len - AH_HMAC_HASHLEN,
+			int alen;
-			    AH_HMAC_HASHLEN, ipseczeroes);
+
+			switch (esph->type) {
+			case CRYPTO_SHA2_256_HMAC:
+			case CRYPTO_SHA2_384_HMAC:
+			case CRYPTO_SHA2_512_HMAC:
+				alen = esph->hashsize/2;
+				break;
+			default:
+				alen = AH_HMAC_HASHLEN;
+				break;
+			}
+			m_copyback(m, m->m_pkthdr.len - alen,
+			    alen, ipseczeroes);
 		}
 	}
 #endif