Implement RSS hashing/re-hashing for IPv6 ingress packets.
This mirrors the basic IPv4 implementation - IPv6 packets under RSS now are checked for a correct RSS hash and if one isn't provided, it's done in software. This only handles the initial receive - it doesn't yet handle reinjecting / rehashing packets after being decapsulated from various tunneling setups. That'll come in some follow-up work. For non-RSS users, this is almost a giant no-op. It does change a couple of ipv6 methods to use const mbuf * instead of mbuf * but it doesn't have any functional changes. So, the following now occurs: * If the NIC doesn't do any RSS hashing, it's all done in software. Single-queue, non-RSS NICs will now have the RX path distributed into multiple receive netisr queues. * If the NIC provides the wrong hash (eg only IPv6 hash when we needed an IPv6 TCP hash, or IPv6 UDP hash when we expected IPv6 hash) then the hash is recalculated. * .. if the hash is recalculated, it'll end up being injected into the correct netisr queue for v6 processing. Submitted by: Tiwei Bie <btw@mail.ustc.edu.cn> Differential Revision: https://reviews.freebsd.org/D3504
This commit is contained in:
Adrian Chadd
parent
2527ccad2d
commit
0be189151f
@ -58,7 +58,8 @@ __FBSDID("$FreeBSD$");
|
||||
#include <netinet/in_var.h>
|
||||
|
||||
/* for software rss hash support */
|
||||
#include <netinet/ip.h>
|
||||
#include <netinet/ip6.h>
|
||||
#include <netinet6/ip6_var.h>
|
||||
#include <netinet/tcp.h>
|
||||
#include <netinet/udp.h>
|
||||
|
||||
@ -150,3 +151,207 @@ rss_proto_software_hash_v6(const struct in6_addr *s, const struct in6_addr *d,
|
||||
RSS_DEBUG("no available hashtypes!\n");
|
||||
return (-1);
|
||||
}
|
||||
|
||||
/*
|
||||
* Do a software calculation of the RSS for the given mbuf.
|
||||
*
|
||||
* This is typically used by the input path to recalculate the RSS after
|
||||
* some form of packet processing (eg de-capsulation, IP fragment reassembly.)
|
||||
*
|
||||
* dir is the packet direction - RSS_HASH_PKT_INGRESS for incoming and
|
||||
* RSS_HASH_PKT_EGRESS for outgoing.
|
||||
*
|
||||
* Returns 0 if a hash was done, -1 if no hash was done, +1 if
|
||||
* the mbuf already had a valid RSS flowid.
|
||||
*
|
||||
* This function doesn't modify the mbuf. It's up to the caller to
|
||||
* assign flowid/flowtype as appropriate.
|
||||
*/
|
||||
int
|
||||
rss_mbuf_software_hash_v6(const struct mbuf *m, int dir, uint32_t *hashval,
|
||||
uint32_t *hashtype)
|
||||
{
|
||||
const struct ip6_hdr *ip6;
|
||||
const struct tcphdr *th;
|
||||
const struct udphdr *uh;
|
||||
uint32_t flowtype;
|
||||
uint8_t proto;
|
||||
int off, newoff;
|
||||
int nxt;
|
||||
|
||||
/*
|
||||
* XXX For now this only handles hashing on incoming mbufs.
|
||||
*/
|
||||
if (dir != RSS_HASH_PKT_INGRESS) {
|
||||
RSS_DEBUG("called on EGRESS packet!\n");
|
||||
return (-1);
|
||||
}
|
||||
|
||||
off = sizeof(struct ip6_hdr);
|
||||
|
||||
/*
|
||||
* First, validate that the mbuf we have is long enough
|
||||
* to have an IPv6 header in it.
|
||||
*/
|
||||
if (m->m_pkthdr.len < off) {
|
||||
RSS_DEBUG("short mbuf pkthdr\n");
|
||||
return (-1);
|
||||
}
|
||||
if (m->m_len < off) {
|
||||
RSS_DEBUG("short mbuf len\n");
|
||||
return (-1);
|
||||
}
|
||||
|
||||
/* Ok, let's dereference that */
|
||||
ip6 = mtod(m, struct ip6_hdr *);
|
||||
proto = ip6->ip6_nxt;
|
||||
|
||||
/*
|
||||
* Find the beginning of the TCP/UDP header.
|
||||
*
|
||||
* If this is a fragment then it shouldn't be four-tuple
|
||||
* hashed just yet. Once it's reassembled into a full
|
||||
* frame it should be re-hashed.
|
||||
*/
|
||||
while (proto != IPPROTO_FRAGMENT) {
|
||||
newoff = ip6_nexthdr(m, off, proto, &nxt);
|
||||
if (newoff < 0)
|
||||
break;
|
||||
off = newoff;
|
||||
proto = nxt;
|
||||
}
|
||||
|
||||
/*
|
||||
* If the mbuf flowid/flowtype matches the packet type,
|
||||
* and we don't support the 4-tuple version of the given protocol,
|
||||
* then signal to the owner that it can trust the flowid/flowtype
|
||||
* details.
|
||||
*
|
||||
* This is a little picky - eg, if TCPv6 / UDPv6 hashing
|
||||
* is supported but we got a TCP/UDP frame only 2-tuple hashed,
|
||||
* then we shouldn't just "trust" the 2-tuple hash. We need
|
||||
* a 4-tuple hash.
|
||||
*/
|
||||
flowtype = M_HASHTYPE_GET(m);
|
||||
|
||||
if (flowtype != M_HASHTYPE_NONE) {
|
||||
switch (proto) {
|
||||
case IPPROTO_UDP:
|
||||
if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6) &&
|
||||
(flowtype == M_HASHTYPE_RSS_UDP_IPV6)) {
|
||||
return (1);
|
||||
}
|
||||
/*
|
||||
* Only allow 2-tuple for UDP frames if we don't also
|
||||
* support 4-tuple for UDP.
|
||||
*/
|
||||
if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV6) &&
|
||||
((rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6) == 0) &&
|
||||
flowtype == M_HASHTYPE_RSS_IPV6) {
|
||||
return (1);
|
||||
}
|
||||
break;
|
||||
case IPPROTO_TCP:
|
||||
if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6) &&
|
||||
(flowtype == M_HASHTYPE_RSS_TCP_IPV6)) {
|
||||
return (1);
|
||||
}
|
||||
/*
|
||||
* Only allow 2-tuple for TCP frames if we don't also
|
||||
* support 4-tuple for TCP.
|
||||
*/
|
||||
if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV6) &&
|
||||
((rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6) == 0) &&
|
||||
flowtype == M_HASHTYPE_RSS_IPV6) {
|
||||
return (1);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV6) &&
|
||||
flowtype == M_HASHTYPE_RSS_IPV6) {
|
||||
return (1);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Decode enough information to make a hash decision.
|
||||
*/
|
||||
if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6) &&
|
||||
(proto == IPPROTO_TCP)) {
|
||||
if (m->m_len < off + sizeof(struct tcphdr)) {
|
||||
RSS_DEBUG("short TCP frame?\n");
|
||||
return (-1);
|
||||
}
|
||||
th = (const struct tcphdr *)((c_caddr_t)ip6 + off);
|
||||
return rss_proto_software_hash_v6(&ip6->ip6_src, &ip6->ip6_dst,
|
||||
th->th_sport,
|
||||
th->th_dport,
|
||||
proto,
|
||||
hashval,
|
||||
hashtype);
|
||||
} else if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6) &&
|
||||
(proto == IPPROTO_UDP)) {
|
||||
if (m->m_len < off + sizeof(struct udphdr)) {
|
||||
RSS_DEBUG("short UDP frame?\n");
|
||||
return (-1);
|
||||
}
|
||||
uh = (const struct udphdr *)((c_caddr_t)ip6 + off);
|
||||
return rss_proto_software_hash_v6(&ip6->ip6_src, &ip6->ip6_dst,
|
||||
uh->uh_sport,
|
||||
uh->uh_dport,
|
||||
proto,
|
||||
hashval,
|
||||
hashtype);
|
||||
} else if (rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV6) {
|
||||
/* Default to 2-tuple hash */
|
||||
return rss_proto_software_hash_v6(&ip6->ip6_src, &ip6->ip6_dst,
|
||||
0, /* source port */
|
||||
0, /* destination port */
|
||||
0, /* IPPROTO_IP */
|
||||
hashval,
|
||||
hashtype);
|
||||
} else {
|
||||
RSS_DEBUG("no available hashtypes!\n");
|
||||
return (-1);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Similar to rss_m2cpuid, but designed to be used by the IPv6 NETISR
|
||||
* on incoming frames.
|
||||
*
|
||||
* If an existing RSS hash exists and it matches what the configured
|
||||
* hashing is, then use it.
|
||||
*
|
||||
* If there's an existing RSS hash but the desired hash is different,
|
||||
* or if there's no useful RSS hash, then calculate it via
|
||||
* the software path.
|
||||
*
|
||||
* XXX TODO: definitely want statistics here!
|
||||
*/
|
||||
struct mbuf *
|
||||
rss_soft_m2cpuid_v6(struct mbuf *m, uintptr_t source, u_int *cpuid)
|
||||
{
|
||||
uint32_t hash_val, hash_type;
|
||||
int ret;
|
||||
|
||||
M_ASSERTPKTHDR(m);
|
||||
|
||||
ret = rss_mbuf_software_hash_v6(m, RSS_HASH_PKT_INGRESS,
|
||||
&hash_val, &hash_type);
|
||||
if (ret > 0) {
|
||||
/* mbuf has a valid hash already; don't need to modify it */
|
||||
*cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m));
|
||||
} else if (ret == 0) {
|
||||
/* hash was done; update */
|
||||
m->m_pkthdr.flowid = hash_val;
|
||||
M_HASHTYPE_SET(m, hash_type);
|
||||
*cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m));
|
||||
} else { /* ret < 0 */
|
||||
/* no hash was done */
|
||||
*cpuid = NETISR_CPUID_NONE;
|
||||
}
|
||||
return (m);
|
||||
}
|
||||
|
||||
@ -46,8 +46,13 @@ uint32_t rss_hash_ip6_2tuple(const struct in6_addr *src,
|
||||
* Functions to calculate a software RSS hash for a given mbuf or
|
||||
* packet detail.
|
||||
*/
|
||||
int rss_mbuf_software_hash_v6(const struct mbuf *m, int dir,
|
||||
uint32_t *hashval, uint32_t *hashtype);
|
||||
int rss_proto_software_hash_v6(const struct in6_addr *src,
|
||||
const struct in6_addr *dst, u_short src_port,
|
||||
u_short dst_port, int proto, uint32_t *hashval,
|
||||
uint32_t *hashtype);
|
||||
struct mbuf * rss_soft_m2cpuid_v6(struct mbuf *m, uintptr_t source,
|
||||
u_int *cpuid);
|
||||
|
||||
#endif /* !_NETINET6_IN6_RSS_H_ */
|
||||
|
||||
@ -68,6 +68,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include "opt_ipfw.h"
|
||||
#include "opt_ipsec.h"
|
||||
#include "opt_route.h"
|
||||
#include "opt_rss.h"
|
||||
|
||||
#include <sys/param.h>
|
||||
#include <sys/systm.h>
|
||||
@ -112,6 +113,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include <netinet6/scope6_var.h>
|
||||
#include <netinet6/in6_ifattach.h>
|
||||
#include <netinet6/nd6.h>
|
||||
#include <netinet6/in6_rss.h>
|
||||
|
||||
#ifdef IPSEC
|
||||
#include <netipsec/ipsec.h>
|
||||
@ -132,7 +134,13 @@ static struct netisr_handler ip6_nh = {
|
||||
.nh_name = "ip6",
|
||||
.nh_handler = ip6_input,
|
||||
.nh_proto = NETISR_IPV6,
|
||||
#ifdef RSS
|
||||
.nh_m2cpuid = rss_soft_m2cpuid_v6,
|
||||
.nh_policy = NETISR_POLICY_CPU,
|
||||
.nh_dispatch = NETISR_DISPATCH_HYBRID,
|
||||
#else
|
||||
.nh_policy = NETISR_POLICY_FLOW,
|
||||
#endif
|
||||
};
|
||||
|
||||
VNET_DECLARE(struct callout, in6_tmpaddrtimer_ch);
|
||||
@ -1440,7 +1448,7 @@ ip6_pullexthdr(struct mbuf *m, size_t off, int nxt)
|
||||
* we develop `neater' mechanism to process extension headers.
|
||||
*/
|
||||
char *
|
||||
ip6_get_prevhdr(struct mbuf *m, int off)
|
||||
ip6_get_prevhdr(const struct mbuf *m, int off)
|
||||
{
|
||||
struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
|
||||
|
||||
@ -1479,7 +1487,7 @@ ip6_get_prevhdr(struct mbuf *m, int off)
|
||||
* get next header offset. m will be retained.
|
||||
*/
|
||||
int
|
||||
ip6_nexthdr(struct mbuf *m, int off, int proto, int *nxtp)
|
||||
ip6_nexthdr(const struct mbuf *m, int off, int proto, int *nxtp)
|
||||
{
|
||||
struct ip6_hdr ip6;
|
||||
struct ip6_ext ip6e;
|
||||
@ -1547,14 +1555,14 @@ ip6_nexthdr(struct mbuf *m, int off, int proto, int *nxtp)
|
||||
return -1;
|
||||
}
|
||||
|
||||
return -1;
|
||||
/* NOTREACHED */
|
||||
}
|
||||
|
||||
/*
|
||||
* get offset for the last header in the chain. m will be kept untainted.
|
||||
*/
|
||||
int
|
||||
ip6_lasthdr(struct mbuf *m, int off, int proto, int *nxtp)
|
||||
ip6_lasthdr(const struct mbuf *m, int off, int proto, int *nxtp)
|
||||
{
|
||||
int newoff;
|
||||
int nxt;
|
||||
|
||||
@ -356,9 +356,9 @@ void ip6_input(struct mbuf *);
|
||||
void ip6_freepcbopts(struct ip6_pktopts *);
|
||||
|
||||
int ip6_unknown_opt(u_int8_t *, struct mbuf *, int);
|
||||
char * ip6_get_prevhdr(struct mbuf *, int);
|
||||
int ip6_nexthdr(struct mbuf *, int, int, int *);
|
||||
int ip6_lasthdr(struct mbuf *, int, int, int *);
|
||||
char * ip6_get_prevhdr(const struct mbuf *, int);
|
||||
int ip6_nexthdr(const struct mbuf *, int, int, int *);
|
||||
int ip6_lasthdr(const struct mbuf *, int, int, int *);
|
||||
|
||||
extern int (*ip6_mforward)(struct ip6_hdr *, struct ifnet *,
|
||||
struct mbuf *);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user