Update the IPv4 input path to handle reassembled frames and incoming frames

with no RSS hash.

When doing RSS:

* Create a new IPv4 netisr which expects the frames to have been verified;
  it just directly dispatches to the IPv4 input path.
* Once IPv4 reassembly is done, re-calculate the RSS hash with the new
  IP and L3 header; then reinject it as appropriate.
* Update the IPv4 netisr to be a CPU affinity netisr with the RSS hash
  function (rss_soft_m2cpuid) - this will do a software hash if the
  hardware doesn't provide one.

NICs that don't implement hardware RSS hashing will now benefit from RSS
distribution - it'll inject into the correct destination netisr.

Note: the netisr distribution doesn't work out of the box - netisr doesn't
query RSS for how many CPUs and the affinity setup.  Yes, netisr likely
shouldn't really be doing CPU stuff anymore and should be "some kind of
'thing' that is a workqueue that may or may not have any CPU affinity";
that's for a later commit.

Differential Revision:	https://reviews.freebsd.org/D527
Reviewed by:	grehan

sys/net/netisr.h

@@ -57,6 +57,8 @@
 #define	NETISR_IPV6	6
 #define	NETISR_NATM	7
 #define	NETISR_EPAIR	8		/* if_epair(4) */
+#define	NETISR_IP_DIRECT	9	/* direct-dispatch IPv4 */
+#define	NETISR_IPV6_DIRECT	10	/* direct-dispatch IPv6 */
 
 /*
  * Protocol ordering and affinity policy constants.  See the detailed

sys/netinet/in_var.h

@@ -411,6 +411,7 @@ void	in_rtqdrain(void);
 int	in_addprefix(struct in_ifaddr *, int);
 int	in_scrubprefix(struct in_ifaddr *, u_int);
 void	ip_input(struct mbuf *);
+void	ip_direct_input(struct mbuf *);
 void	in_ifadown(struct ifaddr *ifa, int);
 struct	mbuf	*ip_fastforward(struct mbuf *);
 void	*in_domifattach(struct ifnet *);

sys/netinet/ip_input.c

@@ -37,6 +37,7 @@ __FBSDID("$FreeBSD$");
 #include "opt_ipstealth.h"
 #include "opt_ipsec.h"
 #include "opt_route.h"
+#include "opt_rss.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -77,6 +78,7 @@ __FBSDID("$FreeBSD$");
 #ifdef IPSEC
 #include <netinet/ip_ipsec.h>
 #endif /* IPSEC */
+#include <netinet/in_rss.h>
 
 #include <sys/socketvar.h>
 
@@ -144,9 +146,33 @@ static struct netisr_handler ip_nh = {
 	.nh_name = "ip",
 	.nh_handler = ip_input,
 	.nh_proto = NETISR_IP,
+#ifdef	RSS
+	.nh_m2cpuid = rss_soft_m2cpuid,
+	.nh_policy = NETISR_POLICY_CPU,
+	.nh_dispatch = NETISR_DISPATCH_HYBRID,
+#else
 	.nh_policy = NETISR_POLICY_FLOW,
+#endif
 };
 
+#ifdef	RSS
+/*
+ * Directly dispatched frames are currently assumed
+ * to have a flowid already calculated.
+ *
+ * It should likely have something that assert it
+ * actually has valid flow details.
+ */
+static struct netisr_handler ip_direct_nh = {
+	.nh_name = "ip_direct",
+	.nh_handler = ip_direct_input,
+	.nh_proto = NETISR_IP_DIRECT,
+	.nh_m2cpuid = rss_m2cpuid,
+	.nh_policy = NETISR_POLICY_CPU,
+	.nh_dispatch = NETISR_DISPATCH_HYBRID,
+};
+#endif
+
 extern	struct domain inetdomain;
 extern	struct protosw inetsw[];
 u_char	ip_protox[IPPROTO_MAX];
@@ -266,6 +292,46 @@ SYSCTL_PROC(_net_inet_ip, IPCTL_INTRQDROPS, intr_queue_drops,
     CTLTYPE_INT|CTLFLAG_RD, 0, 0, sysctl_netinet_intr_queue_drops, "I",
     "Number of packets dropped from the IP input queue");
 
+#ifdef	RSS
+static int
+sysctl_netinet_intr_direct_queue_maxlen(SYSCTL_HANDLER_ARGS)
+{
+	int error, qlimit;
+
+	netisr_getqlimit(&ip_direct_nh, &qlimit);
+	error = sysctl_handle_int(oidp, &qlimit, 0, req);
+	if (error || !req->newptr)
+		return (error);
+	if (qlimit < 1)
+		return (EINVAL);
+	return (netisr_setqlimit(&ip_direct_nh, qlimit));
+}
+SYSCTL_PROC(_net_inet_ip, IPCTL_INTRQMAXLEN, intr_direct_queue_maxlen,
+    CTLTYPE_INT|CTLFLAG_RW, 0, 0, sysctl_netinet_intr_direct_queue_maxlen, "I",
+    "Maximum size of the IP direct input queue");
+
+static int
+sysctl_netinet_intr_direct_queue_drops(SYSCTL_HANDLER_ARGS)
+{
+	u_int64_t qdrops_long;
+	int error, qdrops;
+
+	netisr_getqdrops(&ip_direct_nh, &qdrops_long);
+	qdrops = qdrops_long;
+	error = sysctl_handle_int(oidp, &qdrops, 0, req);
+	if (error || !req->newptr)
+		return (error);
+	if (qdrops != 0)
+		return (EINVAL);
+	netisr_clearqdrops(&ip_direct_nh);
+	return (0);
+}
+
+SYSCTL_PROC(_net_inet_ip, IPCTL_INTRQDROPS, intr_direct_queue_drops,
+    CTLTYPE_INT|CTLFLAG_RD, 0, 0, sysctl_netinet_intr_direct_queue_drops, "I",
+    "Number of packets dropped from the IP direct input queue");
+#endif	/* RSS */
+
 /*
  * IP initialization: fill in IP protocol switch table.
  * All protocols not implemented in kernel go to raw IP protocol handler.
@@ -327,6 +393,9 @@ ip_init(void)
 	/* Initialize various other remaining things. */
 	IPQ_LOCK_INIT();
 	netisr_register(&ip_nh);
+#ifdef	RSS
+	netisr_register(&ip_direct_nh);
+#endif
 }
 
 #ifdef VIMAGE
@@ -350,6 +419,28 @@ ip_destroy(void)
 }
 #endif
 
+#ifdef	RSS
+/*
+ * IP direct input routine.
+ *
+ * This is called when reinjecting completed fragments where
+ * all of the previous checking and book-keeping has been done.
+ */
+void
+ip_direct_input(struct mbuf *m)
+{
+	struct ip *ip;
+	int hlen;
+
+	ip = mtod(m, struct ip *);
+	hlen = ip->ip_hl << 2;
+
+	IPSTAT_INC(ips_delivered);
+	(*inetsw[ip_protox[ip->ip_p]].pr_input)(&m, &hlen, ip->ip_p);
+	return;
+}
+#endif
+
 /*
  * Ip input routine.  Checksum and byte swap header.  If fragmented
  * try to reassemble.  Process options.  Pass to next level.
@@ -463,6 +554,7 @@ ip_input(struct mbuf *m)
 		} else
 			m_adj(m, ip_len - m->m_pkthdr.len);
 	}
+
 #ifdef IPSEC
 	/*
 	 * Bypass packet filtering for packets previously handled by IPsec.
@@ -817,6 +909,9 @@ ip_reass(struct mbuf *m)
 	int i, hlen, next;
 	u_int8_t ecn, ecn0;
 	u_short hash;
+#ifdef	RSS
+	uint32_t rss_hash, rss_type;
+#endif
 
 	/* If maxnipq or maxfragsperpacket are 0, never accept fragments. */
 	if (V_maxnipq == 0 || V_maxfragsperpacket == 0) {
@@ -1106,6 +1201,42 @@ ip_reass(struct mbuf *m)
 		m_fixhdr(m);
 	IPSTAT_INC(ips_reassembled);
 	IPQ_UNLOCK();
+
+#ifdef	RSS
+	/*
+	 * Query the RSS layer for the flowid / flowtype for the
+	 * mbuf payload.
+	 *
+	 * For now, just assume we have to calculate a new one.
+	 * Later on we should check to see if the assigned flowid matches
+	 * what RSS wants for the given IP protocol and if so, just keep it.
+	 *
+	 * We then queue into the relevant netisr so it can be dispatched
+	 * to the correct CPU.
+	 *
+	 * Note - this may return 1, which means the flowid in the mbuf
+	 * is correct for the configured RSS hash types and can be used.
+	 */
+	if (rss_mbuf_software_hash_v4(m, 0, &rss_hash, &rss_type) == 0) {
+		m->m_pkthdr.flowid = rss_hash;
+		M_HASHTYPE_SET(m, rss_type);
+		m->m_flags |= M_FLOWID;
+	}
+#endif
+
+#ifdef	RSS
+	/*
+	 * Queue/dispatch for reprocessing.
+	 *
+	 * Note: this is much slower than just handling the frame in the
+	 * current receive context.  It's likely worth investigating
+	 * why this is.
+	 */
+	netisr_dispatch(NETISR_IP_DIRECT, m);
+	return (NULL);
+#endif
+
+	/* Handle in-line */
 	return (m);
 
 dropfrag: