From 35e4e998d8187c1d4d413bdc13a79a6415a30a18 Mon Sep 17 00:00:00 2001
From: Stephen Hurd <shurd@FreeBSD.org>
Date: Mon, 6 Nov 2017 16:23:21 +0000
Subject: [PATCH] Only chain non-LRO mbufs when LRO is not possible

Preserve packet order between tcp_lro_rx() and if_input() to avoid
creating extra corner cases. If no packets can be LROed, combine them
into one chain for submission via if_input(). If any packet can
potentially be LROed however, retain old behaviour and call if_input()
for each packet.

This should keep the 12% improvement for small packet forwarding intact,
but mostly avoids impacting the LRO case.

Reviewed by:	cem, sbruno
Approved by:	sbruno (mentor)
Sponsored by:	Limelight Networks
Differential Revision:	https://reviews.freebsd.org/D12876
---
 sys/net/iflib.c | 70 ++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 64 insertions(+), 6 deletions(-)

diff --git a/sys/net/iflib.c b/sys/net/iflib.c
index 9d879c498c0e..8cb6bb46e3bc 100644
--- a/sys/net/iflib.c
+++ b/sys/net/iflib.c
@@ -59,6 +59,7 @@ __FBSDID("$FreeBSD$");
 #include <net/bpf.h>
 #include <net/ethernet.h>
 #include <net/mp_ring.h>
+#include <net/vnet.h>
 
 #include <netinet/in.h>
 #include <netinet/in_pcb.h>
@@ -68,6 +69,8 @@ __FBSDID("$FreeBSD$");
 #include <netinet/ip.h>
 #include <netinet/ip6.h>
 #include <netinet/tcp.h>
+#include <netinet/ip_var.h>
+#include <netinet6/ip6_var.h>
 
 #include <machine/bus.h>
 #include <machine/in_cksum.h>
@@ -2463,6 +2466,47 @@ iflib_rxd_pkt_get(iflib_rxq_t rxq, if_rxd_info_t ri)
 	return (m);
 }
 
+#if defined(INET6) || defined(INET)
+/*
+ * Returns true if it's possible this packet could be LROed.
+ * if it returns false, it is guaranteed that tcp_lro_rx()
+ * would not return zero.
+ */
+static bool
+iflib_check_lro_possible(struct lro_ctrl *lc, struct mbuf *m)
+{
+	struct ether_header *eh;
+	uint16_t eh_type;
+
+	eh = mtod(m, struct ether_header *);
+	eh_type = ntohs(eh->ether_type);
+	switch (eh_type) {
+		case ETHERTYPE_IPV6:
+		{
+			CURVNET_SET(lc->ifp->if_vnet);
+			if (VNET(ip6_forwarding) == 0) {
+				CURVNET_RESTORE();
+				return true;
+			}
+			CURVNET_RESTORE();
+			break;
+		}
+		case ETHERTYPE_IP:
+		{
+			CURVNET_SET(lc->ifp->if_vnet);
+			if (VNET(ipforwarding) == 0) {
+				CURVNET_RESTORE();
+				return true;
+			}
+			CURVNET_RESTORE();
+			break;
+		}
+	}
+
+	return false;
+}
+#endif
+
 static bool
 iflib_rxeof(iflib_rxq_t rxq, qidx_t budget)
 {
@@ -2476,6 +2520,7 @@ iflib_rxeof(iflib_rxq_t rxq, qidx_t budget)
 	iflib_fl_t fl;
 	struct ifnet *ifp;
 	int lro_enabled;
+	bool lro_possible = false;
 
 	/*
 	 * XXX early demux data packets so that if_input processing only handles
@@ -2555,8 +2600,6 @@ iflib_rxeof(iflib_rxq_t rxq, qidx_t budget)
 	mt = mf = NULL;
 	while (mh != NULL) {
 		m = mh;
-		if (mf == NULL)
-			mf = m;
 		mh = mh->m_nextpkt;
 		m->m_nextpkt = NULL;
 #ifndef __NO_STRICT_ALIGNMENT
@@ -2566,12 +2609,27 @@ iflib_rxeof(iflib_rxq_t rxq, qidx_t budget)
 		rx_bytes += m->m_pkthdr.len;
 		rx_pkts++;
 #if defined(INET6) || defined(INET)
-		if (lro_enabled && tcp_lro_rx(&rxq->ifr_lc, m, 0) == 0) {
-			if (mf == m)
-				mf = NULL;
-			continue;
+		if (lro_enabled) {
+			if (!lro_possible) {
+				lro_possible = iflib_check_lro_possible(&rxq->ifr_lc, m);
+				if (lro_possible && mf != NULL) {
+					ifp->if_input(ifp, mf);
+					DBG_COUNTER_INC(rx_if_input);
+					mt = mf = NULL;
+				}
+			}
+			if (lro_possible && tcp_lro_rx(&rxq->ifr_lc, m, 0) == 0)
+				continue;
 		}
 #endif
+		if (lro_possible) {
+			ifp->if_input(ifp, m);
+			DBG_COUNTER_INC(rx_if_input);
+			continue;
+		}
+
+		if (mf == NULL)
+			mf = m;
 		if (mt != NULL)
 			mt->m_nextpkt = m;
 		mt = m;