From b89af8e16db66225520b80a277fe02ede951ab70 Mon Sep 17 00:00:00 2001
From: Michael Tuexen <tuexen@FreeBSD.org>
Date: Tue, 14 Apr 2020 16:35:05 +0000
Subject: [PATCH] Improve the TCP blackhole detection. The principle is to
 reduce the MSS in two steps and try each candidate two times. However, if two
 candidates are the same (which is the case in TCP/IPv6), this candidate was
 tested four times. This patch ensures that each candidate actually reduced
 the MSS and is only tested 2 times. This reduces the time window of
 missclassifying a temporary outage as an MTU issue.

Reviewed by:		jtl
MFC after:		1 week
Sponsored by:		Netflix, Inc.
Differential Revision:	https://reviews.freebsd.org/D24308
---
 sys/netinet/tcp_timer.c | 50 ++++++++++++++++++++++++++++++-----------
 sys/netinet/tcp_var.h   |  2 ++
 2 files changed, 39 insertions(+), 13 deletions(-)

diff --git a/sys/netinet/tcp_timer.c b/sys/netinet/tcp_timer.c
index bd5f51c5bb90..139e32172531 100644
--- a/sys/netinet/tcp_timer.c
+++ b/sys/netinet/tcp_timer.c
@@ -723,16 +723,40 @@ tcp_timer_rexmt(void * xtp)
 	    (V_tcp_pmtud_blackhole_detect == 3 && isipv6)) &&
 	    ((tp->t_state == TCPS_ESTABLISHED) ||
 	    (tp->t_state == TCPS_FIN_WAIT_1))) {
-		/*
-		 * Idea here is that at each stage of mtu probe (usually, 1448
-		 * -> 1188 -> 524) should be given 2 chances to recover before
-		 *  further clamping down. 'tp->t_rxtshift % 2 == 0' should
-		 *  take care of that.
-		 */
+		if (tp->t_rxtshift == 1) {
+			/*
+			 * We enter blackhole detection after the first
+			 * unsuccessful timer based retransmission.
+			 * Then we reduce up to two times the MSS, each
+			 * candidate giving two tries of retransmissions.
+			 * But we give a candidate only two tries, if it
+			 * actually reduces the MSS.
+			 */
+			tp->t_blackhole_enter = 2;
+			tp->t_blackhole_exit = tp->t_blackhole_enter;
+			if (isipv6) {
+#ifdef INET6
+				if (tp->t_maxseg > V_tcp_v6pmtud_blackhole_mss)
+					tp->t_blackhole_exit += 2;
+				if (tp->t_maxseg > V_tcp_v6mssdflt &&
+				    V_tcp_v6pmtud_blackhole_mss > V_tcp_v6mssdflt)
+					tp->t_blackhole_exit += 2;
+#endif
+			} else {
+#ifdef INET
+				if (tp->t_maxseg > V_tcp_pmtud_blackhole_mss)
+					tp->t_blackhole_exit += 2;
+				if (tp->t_maxseg > V_tcp_mssdflt &&
+				    V_tcp_pmtud_blackhole_mss > V_tcp_mssdflt)
+					tp->t_blackhole_exit += 2;
+#endif
+			}
+		}
 		if (((tp->t_flags2 & (TF2_PLPMTU_PMTUD|TF2_PLPMTU_MAXSEGSNT)) ==
 		    (TF2_PLPMTU_PMTUD|TF2_PLPMTU_MAXSEGSNT)) &&
-		    (tp->t_rxtshift >= 2 && tp->t_rxtshift < 6 &&
-		    tp->t_rxtshift % 2 == 0)) {
+		    (tp->t_rxtshift >= tp->t_blackhole_enter &&
+		    tp->t_rxtshift < tp->t_blackhole_exit &&
+		    (tp->t_rxtshift - tp->t_blackhole_enter) % 2 == 0)) {
 			/*
 			 * Enter Path MTU Black-hole Detection mechanism:
 			 * - Disable Path MTU Discovery (IP "DF" bit).
@@ -752,7 +776,8 @@ tcp_timer_rexmt(void * xtp)
 			 */
 #ifdef INET6
 			if (isipv6 &&
-			    tp->t_maxseg > V_tcp_v6pmtud_blackhole_mss) {
+			    tp->t_maxseg > V_tcp_v6pmtud_blackhole_mss &&
+			    V_tcp_v6pmtud_blackhole_mss > V_tcp_v6mssdflt) {
 				/* Use the sysctl tuneable blackhole MSS. */
 				tp->t_maxseg = V_tcp_v6pmtud_blackhole_mss;
 				TCPSTAT_INC(tcps_pmtud_blackhole_activated);
@@ -771,7 +796,8 @@ tcp_timer_rexmt(void * xtp)
 			else
 #endif
 #ifdef INET
-			if (tp->t_maxseg > V_tcp_pmtud_blackhole_mss) {
+			if (tp->t_maxseg > V_tcp_pmtud_blackhole_mss &&
+			    V_tcp_pmtud_blackhole_mss > V_tcp_mssdflt) {
 				/* Use the sysctl tuneable blackhole MSS. */
 				tp->t_maxseg = V_tcp_pmtud_blackhole_mss;
 				TCPSTAT_INC(tcps_pmtud_blackhole_activated);
@@ -798,11 +824,9 @@ tcp_timer_rexmt(void * xtp)
 			 * with a lowered MTU, maybe this isn't a blackhole and
 			 * we restore the previous MSS and blackhole detection
 			 * flags.
-			 * The limit '6' is determined by giving each probe
-			 * stage (1448, 1188, 524) 2 chances to recover.
 			 */
 			if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) &&
-			    (tp->t_rxtshift >= 6)) {
+			    (tp->t_rxtshift >= tp->t_blackhole_exit)) {
 				tp->t_flags2 |= TF2_PLPMTU_PMTUD;
 				tp->t_flags2 &= ~TF2_PLPMTU_BLACKHOLE;
 				tp->t_maxseg = tp->t_pmtud_saved_maxseg;
diff --git a/sys/netinet/tcp_var.h b/sys/netinet/tcp_var.h
index 82b6a9dc4e4b..10b1b6586dfd 100644
--- a/sys/netinet/tcp_var.h
+++ b/sys/netinet/tcp_var.h
@@ -169,6 +169,8 @@ struct tcpcb {
 	u_int	t_starttime;		/* time connection was established */
 
 	u_int	t_pmtud_saved_maxseg;	/* pre-blackhole MSS */
+	int	t_blackhole_enter;	/* when to enter blackhole detection */
+	int	t_blackhole_exit;	/* when to exit blackhole detection */
 	u_int	t_rttmin;		/* minimum rtt allowed */
 
 	u_int	t_rttbest;		/* best rtt we've seen */