Update the IPv4 input path to handle reassembled frames and incoming frames
with no RSS hash. When doing RSS: * Create a new IPv4 netisr which expects the frames to have been verified; it just directly dispatches to the IPv4 input path. * Once IPv4 reassembly is done, re-calculate the RSS hash with the new IP and L3 header; then reinject it as appropriate. * Update the IPv4 netisr to be a CPU affinity netisr with the RSS hash function (rss_soft_m2cpuid) - this will do a software hash if the hardware doesn't provide one. NICs that don't implement hardware RSS hashing will now benefit from RSS distribution - it'll inject into the correct destination netisr. Note: the netisr distribution doesn't work out of the box - netisr doesn't query RSS for how many CPUs and the affinity setup. Yes, netisr likely shouldn't really be doing CPU stuff anymore and should be "some kind of 'thing' that is a workqueue that may or may not have any CPU affinity"; that's for a later commit. Differential Revision: https://reviews.freebsd.org/D527 Reviewed by: grehan
This commit is contained in:
parent
e18f344b9b
commit
b8bc95cd49
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=271300
@ -57,6 +57,8 @@
|
||||
#define NETISR_IPV6 6
|
||||
#define NETISR_NATM 7
|
||||
#define NETISR_EPAIR 8 /* if_epair(4) */
|
||||
#define NETISR_IP_DIRECT 9 /* direct-dispatch IPv4 */
|
||||
#define NETISR_IPV6_DIRECT 10 /* direct-dispatch IPv6 */
|
||||
|
||||
/*
|
||||
* Protocol ordering and affinity policy constants. See the detailed
|
||||
|
@ -411,6 +411,7 @@ void in_rtqdrain(void);
|
||||
int in_addprefix(struct in_ifaddr *, int);
|
||||
int in_scrubprefix(struct in_ifaddr *, u_int);
|
||||
void ip_input(struct mbuf *);
|
||||
void ip_direct_input(struct mbuf *);
|
||||
void in_ifadown(struct ifaddr *ifa, int);
|
||||
struct mbuf *ip_fastforward(struct mbuf *);
|
||||
void *in_domifattach(struct ifnet *);
|
||||
|
@ -37,6 +37,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include "opt_ipstealth.h"
|
||||
#include "opt_ipsec.h"
|
||||
#include "opt_route.h"
|
||||
#include "opt_rss.h"
|
||||
|
||||
#include <sys/param.h>
|
||||
#include <sys/systm.h>
|
||||
@ -77,6 +78,7 @@ __FBSDID("$FreeBSD$");
|
||||
#ifdef IPSEC
|
||||
#include <netinet/ip_ipsec.h>
|
||||
#endif /* IPSEC */
|
||||
#include <netinet/in_rss.h>
|
||||
|
||||
#include <sys/socketvar.h>
|
||||
|
||||
@ -144,9 +146,33 @@ static struct netisr_handler ip_nh = {
|
||||
.nh_name = "ip",
|
||||
.nh_handler = ip_input,
|
||||
.nh_proto = NETISR_IP,
|
||||
#ifdef RSS
|
||||
.nh_m2cpuid = rss_soft_m2cpuid,
|
||||
.nh_policy = NETISR_POLICY_CPU,
|
||||
.nh_dispatch = NETISR_DISPATCH_HYBRID,
|
||||
#else
|
||||
.nh_policy = NETISR_POLICY_FLOW,
|
||||
#endif
|
||||
};
|
||||
|
||||
#ifdef RSS
|
||||
/*
|
||||
* Directly dispatched frames are currently assumed
|
||||
* to have a flowid already calculated.
|
||||
*
|
||||
* It should likely have something that assert it
|
||||
* actually has valid flow details.
|
||||
*/
|
||||
static struct netisr_handler ip_direct_nh = {
|
||||
.nh_name = "ip_direct",
|
||||
.nh_handler = ip_direct_input,
|
||||
.nh_proto = NETISR_IP_DIRECT,
|
||||
.nh_m2cpuid = rss_m2cpuid,
|
||||
.nh_policy = NETISR_POLICY_CPU,
|
||||
.nh_dispatch = NETISR_DISPATCH_HYBRID,
|
||||
};
|
||||
#endif
|
||||
|
||||
extern struct domain inetdomain;
|
||||
extern struct protosw inetsw[];
|
||||
u_char ip_protox[IPPROTO_MAX];
|
||||
@ -266,6 +292,46 @@ SYSCTL_PROC(_net_inet_ip, IPCTL_INTRQDROPS, intr_queue_drops,
|
||||
CTLTYPE_INT|CTLFLAG_RD, 0, 0, sysctl_netinet_intr_queue_drops, "I",
|
||||
"Number of packets dropped from the IP input queue");
|
||||
|
||||
#ifdef RSS
|
||||
static int
|
||||
sysctl_netinet_intr_direct_queue_maxlen(SYSCTL_HANDLER_ARGS)
|
||||
{
|
||||
int error, qlimit;
|
||||
|
||||
netisr_getqlimit(&ip_direct_nh, &qlimit);
|
||||
error = sysctl_handle_int(oidp, &qlimit, 0, req);
|
||||
if (error || !req->newptr)
|
||||
return (error);
|
||||
if (qlimit < 1)
|
||||
return (EINVAL);
|
||||
return (netisr_setqlimit(&ip_direct_nh, qlimit));
|
||||
}
|
||||
SYSCTL_PROC(_net_inet_ip, IPCTL_INTRQMAXLEN, intr_direct_queue_maxlen,
|
||||
CTLTYPE_INT|CTLFLAG_RW, 0, 0, sysctl_netinet_intr_direct_queue_maxlen, "I",
|
||||
"Maximum size of the IP direct input queue");
|
||||
|
||||
static int
|
||||
sysctl_netinet_intr_direct_queue_drops(SYSCTL_HANDLER_ARGS)
|
||||
{
|
||||
u_int64_t qdrops_long;
|
||||
int error, qdrops;
|
||||
|
||||
netisr_getqdrops(&ip_direct_nh, &qdrops_long);
|
||||
qdrops = qdrops_long;
|
||||
error = sysctl_handle_int(oidp, &qdrops, 0, req);
|
||||
if (error || !req->newptr)
|
||||
return (error);
|
||||
if (qdrops != 0)
|
||||
return (EINVAL);
|
||||
netisr_clearqdrops(&ip_direct_nh);
|
||||
return (0);
|
||||
}
|
||||
|
||||
SYSCTL_PROC(_net_inet_ip, IPCTL_INTRQDROPS, intr_direct_queue_drops,
|
||||
CTLTYPE_INT|CTLFLAG_RD, 0, 0, sysctl_netinet_intr_direct_queue_drops, "I",
|
||||
"Number of packets dropped from the IP direct input queue");
|
||||
#endif /* RSS */
|
||||
|
||||
/*
|
||||
* IP initialization: fill in IP protocol switch table.
|
||||
* All protocols not implemented in kernel go to raw IP protocol handler.
|
||||
@ -327,6 +393,9 @@ ip_init(void)
|
||||
/* Initialize various other remaining things. */
|
||||
IPQ_LOCK_INIT();
|
||||
netisr_register(&ip_nh);
|
||||
#ifdef RSS
|
||||
netisr_register(&ip_direct_nh);
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef VIMAGE
|
||||
@ -350,6 +419,28 @@ ip_destroy(void)
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef RSS
|
||||
/*
|
||||
* IP direct input routine.
|
||||
*
|
||||
* This is called when reinjecting completed fragments where
|
||||
* all of the previous checking and book-keeping has been done.
|
||||
*/
|
||||
void
|
||||
ip_direct_input(struct mbuf *m)
|
||||
{
|
||||
struct ip *ip;
|
||||
int hlen;
|
||||
|
||||
ip = mtod(m, struct ip *);
|
||||
hlen = ip->ip_hl << 2;
|
||||
|
||||
IPSTAT_INC(ips_delivered);
|
||||
(*inetsw[ip_protox[ip->ip_p]].pr_input)(&m, &hlen, ip->ip_p);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Ip input routine. Checksum and byte swap header. If fragmented
|
||||
* try to reassemble. Process options. Pass to next level.
|
||||
@ -463,6 +554,7 @@ ip_input(struct mbuf *m)
|
||||
} else
|
||||
m_adj(m, ip_len - m->m_pkthdr.len);
|
||||
}
|
||||
|
||||
#ifdef IPSEC
|
||||
/*
|
||||
* Bypass packet filtering for packets previously handled by IPsec.
|
||||
@ -817,6 +909,9 @@ ip_reass(struct mbuf *m)
|
||||
int i, hlen, next;
|
||||
u_int8_t ecn, ecn0;
|
||||
u_short hash;
|
||||
#ifdef RSS
|
||||
uint32_t rss_hash, rss_type;
|
||||
#endif
|
||||
|
||||
/* If maxnipq or maxfragsperpacket are 0, never accept fragments. */
|
||||
if (V_maxnipq == 0 || V_maxfragsperpacket == 0) {
|
||||
@ -1106,6 +1201,42 @@ ip_reass(struct mbuf *m)
|
||||
m_fixhdr(m);
|
||||
IPSTAT_INC(ips_reassembled);
|
||||
IPQ_UNLOCK();
|
||||
|
||||
#ifdef RSS
|
||||
/*
|
||||
* Query the RSS layer for the flowid / flowtype for the
|
||||
* mbuf payload.
|
||||
*
|
||||
* For now, just assume we have to calculate a new one.
|
||||
* Later on we should check to see if the assigned flowid matches
|
||||
* what RSS wants for the given IP protocol and if so, just keep it.
|
||||
*
|
||||
* We then queue into the relevant netisr so it can be dispatched
|
||||
* to the correct CPU.
|
||||
*
|
||||
* Note - this may return 1, which means the flowid in the mbuf
|
||||
* is correct for the configured RSS hash types and can be used.
|
||||
*/
|
||||
if (rss_mbuf_software_hash_v4(m, 0, &rss_hash, &rss_type) == 0) {
|
||||
m->m_pkthdr.flowid = rss_hash;
|
||||
M_HASHTYPE_SET(m, rss_type);
|
||||
m->m_flags |= M_FLOWID;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef RSS
|
||||
/*
|
||||
* Queue/dispatch for reprocessing.
|
||||
*
|
||||
* Note: this is much slower than just handling the frame in the
|
||||
* current receive context. It's likely worth investigating
|
||||
* why this is.
|
||||
*/
|
||||
netisr_dispatch(NETISR_IP_DIRECT, m);
|
||||
return (NULL);
|
||||
#endif
|
||||
|
||||
/* Handle in-line */
|
||||
return (m);
|
||||
|
||||
dropfrag:
|
||||
|
Loading…
Reference in New Issue
Block a user