Refactor / restructure the RSS code into generic, IPv4 and IPv6 specific
bits. The motivation here is to eventually teach netisr and potentially other networking subsystems a bit more about how RSS work queues / buckets are configured so things have a hope of auto-configuring in the future. * net/rss_config.[ch] takes care of the generic bits for doing configuration, hash function selection, etc; * topelitz.[ch] is now in net/ rather than netinet/; * (and would be in libkern if it didn't directly include RSS_KEYSIZE; that's a later thing to fix up.) * netinet/in_rss.[ch] now just contains the IPv4 specific methods; * and netinet/in6_rss.[ch] now just contains the IPv6 specific methods. This should have no functional impact on anyone currently using the RSS support. Differential Revision: D1383 Reviewed by: gnn, jfv (intel driver bits)
This commit is contained in:
Adrian Chadd
parent
6ae52b27b6
commit
b2bdc62a95
@ -3270,9 +3270,11 @@ net/radix_mpath.c standard
|
||||
net/raw_cb.c standard
|
||||
net/raw_usrreq.c standard
|
||||
net/route.c standard
|
||||
net/rss_config.c optional inet rss | inet6 rss
|
||||
net/rtsock.c standard
|
||||
net/slcompress.c optional netgraph_vjc | sppp | \
|
||||
netgraph_sppp
|
||||
net/toeplitz.c optional inet rss | inet6 rss
|
||||
net/vnet.c optional vimage
|
||||
net/zlib.c optional crypto | geom_uzip | ipsec | \
|
||||
mxge | netgraph_deflate | \
|
||||
@ -3422,7 +3424,7 @@ netinet/in_pcb.c optional inet | inet6
|
||||
netinet/in_pcbgroup.c optional inet pcbgroup | inet6 pcbgroup
|
||||
netinet/in_proto.c optional inet | inet6
|
||||
netinet/in_rmx.c optional inet
|
||||
netinet/in_rss.c optional inet rss | inet6 rss
|
||||
netinet/in_rss.c optional inet rss
|
||||
netinet/ip_divert.c optional inet ipdivert ipfirewall
|
||||
netinet/ip_ecn.c optional inet | inet6
|
||||
netinet/ip_encap.c optional inet | inet6
|
||||
@ -3465,7 +3467,6 @@ netinet/tcp_syncache.c optional inet | inet6
|
||||
netinet/tcp_timer.c optional inet | inet6
|
||||
netinet/tcp_timewait.c optional inet | inet6
|
||||
netinet/tcp_usrreq.c optional inet | inet6
|
||||
netinet/toeplitz.c optional inet rss | inet6 rss
|
||||
netinet/udp_usrreq.c optional inet | inet6
|
||||
netinet/libalias/alias.c optional libalias inet | netgraph_nat inet
|
||||
netinet/libalias/alias_db.c optional libalias inet | netgraph_nat inet
|
||||
@ -3485,6 +3486,7 @@ netinet6/in6_pcb.c optional inet6
|
||||
netinet6/in6_pcbgroup.c optional inet6 pcbgroup
|
||||
netinet6/in6_proto.c optional inet6
|
||||
netinet6/in6_rmx.c optional inet6
|
||||
netinet6/in6_rss.c optional inet6 rss
|
||||
netinet6/in6_src.c optional inet6
|
||||
netinet6/ip6_forward.c optional inet6
|
||||
netinet6/ip6_gre.c optional gre inet6
|
||||
|
||||
@ -73,6 +73,9 @@
|
||||
#include <net/if_arp.h>
|
||||
#include <net/if_dl.h>
|
||||
#include <net/if_media.h>
|
||||
#ifdef RSS
|
||||
#include <net/rss_config.h>
|
||||
#endif
|
||||
|
||||
#include <net/if_types.h>
|
||||
#include <net/if_vlan_var.h>
|
||||
@ -85,9 +88,6 @@
|
||||
#include <netinet/tcp.h>
|
||||
#include <netinet/tcp_lro.h>
|
||||
#include <netinet/udp.h>
|
||||
#ifdef RSS
|
||||
#include <netinet/in_rss.h>
|
||||
#endif
|
||||
|
||||
#include <machine/in_cksum.h>
|
||||
#include <dev/led/led.h>
|
||||
|
||||
@ -39,7 +39,7 @@
|
||||
#include "ixgbe.h"
|
||||
|
||||
#ifdef RSS
|
||||
#include <netinet/in_rss.h>
|
||||
#include <net/rss_config.h>
|
||||
#endif
|
||||
|
||||
/*********************************************************************
|
||||
|
||||
@ -1378,7 +1378,7 @@ static inline int
|
||||
ixl_ptype_to_hash(u8 ptype)
|
||||
{
|
||||
struct i40e_rx_ptype_decoded decoded;
|
||||
u8 ex = 0
|
||||
u8 ex = 0;
|
||||
|
||||
decoded = decode_rx_desc_ptype(ptype);
|
||||
ex = decoded.outer_frag;
|
||||
|
||||
@ -63,6 +63,7 @@
|
||||
#include <net/if_vlan_var.h>
|
||||
#include <net/if_llatbl.h>
|
||||
#include <net/pfil.h>
|
||||
#include <net/rss_config.h>
|
||||
#include <net/vnet.h>
|
||||
|
||||
#include <netpfil/pf/pf_mtag.h>
|
||||
@ -71,7 +72,6 @@
|
||||
#include <netinet/in.h>
|
||||
#include <netinet/in_var.h>
|
||||
#include <netinet/if_ether.h>
|
||||
#include <netinet/in_rss.h>
|
||||
#include <netinet/ip_carp.h>
|
||||
#include <netinet/ip_var.h>
|
||||
#endif
|
||||
|
||||
558
sys/net/rss_config.c
Normal file
558
sys/net/rss_config.c
Normal file
@ -0,0 +1,558 @@
|
||||
/*-
|
||||
* Copyright (c) 2010-2011 Juniper Networks, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This software was developed by Robert N. M. Watson under contract
|
||||
* to Juniper Networks, Inc.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
|
||||
__FBSDID("$FreeBSD$");
|
||||
|
||||
#include "opt_inet6.h"
|
||||
#include "opt_pcbgroup.h"
|
||||
|
||||
#ifndef PCBGROUP
|
||||
#error "options RSS depends on options PCBGROUP"
|
||||
#endif
|
||||
|
||||
#include <sys/param.h>
|
||||
#include <sys/mbuf.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/priv.h>
|
||||
#include <sys/kernel.h>
|
||||
#include <sys/smp.h>
|
||||
#include <sys/sysctl.h>
|
||||
#include <sys/sbuf.h>
|
||||
|
||||
#include <net/if.h>
|
||||
#include <net/if_var.h>
|
||||
#include <net/netisr.h>
|
||||
#include <net/rss_config.h>
|
||||
#include <net/toeplitz.h>
|
||||
|
||||
#if 0
|
||||
#include <netinet/in.h>
|
||||
#include <netinet/in_pcb.h>
|
||||
#include <netinet/in_rss.h>
|
||||
#include <netinet/in_var.h>
|
||||
|
||||
/* for software rss hash support */
|
||||
#include <netinet/ip.h>
|
||||
#include <netinet/tcp.h>
|
||||
#include <netinet/udp.h>
|
||||
#endif
|
||||
|
||||
/*-
|
||||
* Operating system parts of receiver-side scaling (RSS), which allows
|
||||
* network cards to direct flows to particular receive queues based on hashes
|
||||
* of header tuples. This implementation aligns RSS buckets with connection
|
||||
* groups at the TCP/IP layer, so each bucket is associated with exactly one
|
||||
* group. As a result, the group lookup structures (and lock) should have an
|
||||
* effective affinity with exactly one CPU.
|
||||
*
|
||||
* Network device drivers needing to configure RSS will query this framework
|
||||
* for parameters, such as the current RSS key, hashing policies, number of
|
||||
* bits, and indirection table mapping hashes to buckets and CPUs. They may
|
||||
* provide their own supplementary information, such as queue<->CPU bindings.
|
||||
* It is the responsibility of the network device driver to inject packets
|
||||
* into the stack on as close to the right CPU as possible, if playing by RSS
|
||||
* rules.
|
||||
*
|
||||
* TODO:
|
||||
*
|
||||
* - Synchronization for rss_key and other future-configurable parameters.
|
||||
* - Event handler drivers can register to pick up RSS configuration changes.
|
||||
* - Should we allow rss_basecpu to be configured?
|
||||
* - Randomize key on boot.
|
||||
* - IPv6 support.
|
||||
* - Statistics on how often there's a misalignment between hardware
|
||||
* placement and pcbgroup expectations.
|
||||
*/
|
||||
|
||||
SYSCTL_DECL(_net_inet);
|
||||
SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW, 0, "Receive-side steering");
|
||||
|
||||
/*
|
||||
* Toeplitz is the only required hash function in the RSS spec, so use it by
|
||||
* default.
|
||||
*/
|
||||
static u_int rss_hashalgo = RSS_HASH_TOEPLITZ;
|
||||
SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0,
|
||||
"RSS hash algorithm");
|
||||
|
||||
/*
|
||||
* Size of the indirection table; at most 128 entries per the RSS spec. We
|
||||
* size it to at least 2 times the number of CPUs by default to allow useful
|
||||
* rebalancing. If not set explicitly with a loader tunable, we tune based
|
||||
* on the number of CPUs present.
|
||||
*
|
||||
* XXXRW: buckets might be better to use for the tunable than bits.
|
||||
*/
|
||||
static u_int rss_bits;
|
||||
SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0,
|
||||
"RSS bits");
|
||||
|
||||
static u_int rss_mask;
|
||||
SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0,
|
||||
"RSS mask");
|
||||
|
||||
static const u_int rss_maxbits = RSS_MAXBITS;
|
||||
SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD,
|
||||
__DECONST(int *, &rss_maxbits), 0, "RSS maximum bits");
|
||||
|
||||
/*
|
||||
* RSS's own count of the number of CPUs it could be using for processing.
|
||||
* Bounded to 64 by RSS constants.
|
||||
*/
|
||||
static u_int rss_ncpus;
|
||||
SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0,
|
||||
"Number of CPUs available to RSS");
|
||||
|
||||
#define RSS_MAXCPUS (1 << (RSS_MAXBITS - 1))
|
||||
static const u_int rss_maxcpus = RSS_MAXCPUS;
|
||||
SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD,
|
||||
__DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used");
|
||||
|
||||
/*
|
||||
* Variable exists just for reporting rss_bits in a user-friendly way.
|
||||
*/
|
||||
static u_int rss_buckets;
|
||||
SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0,
|
||||
"RSS buckets");
|
||||
|
||||
/*
|
||||
* Base CPU number; devices will add this to all CPU numbers returned by the
|
||||
* RSS indirection table. Currently unmodifable in FreeBSD.
|
||||
*/
|
||||
static const u_int rss_basecpu;
|
||||
SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD,
|
||||
__DECONST(int *, &rss_basecpu), 0, "RSS base CPU");
|
||||
|
||||
/*
|
||||
* RSS secret key, intended to prevent attacks on load-balancing. Its
|
||||
* effectiveness may be limited by algorithm choice and available entropy
|
||||
* during the boot.
|
||||
*
|
||||
* XXXRW: And that we don't randomize it yet!
|
||||
*
|
||||
* This is the default Microsoft RSS specification key which is also
|
||||
* the Chelsio T5 firmware default key.
|
||||
*/
|
||||
static uint8_t rss_key[RSS_KEYSIZE] = {
|
||||
0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
|
||||
0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
|
||||
0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
|
||||
0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
|
||||
0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa,
|
||||
};
|
||||
|
||||
/*
|
||||
* RSS hash->CPU table, which maps hashed packet headers to particular CPUs.
|
||||
* Drivers may supplement this table with a seperate CPU<->queue table when
|
||||
* programming devices.
|
||||
*/
|
||||
struct rss_table_entry {
|
||||
uint8_t rte_cpu; /* CPU affinity of bucket. */
|
||||
};
|
||||
static struct rss_table_entry rss_table[RSS_TABLE_MAXLEN];
|
||||
|
||||
static void
|
||||
rss_init(__unused void *arg)
|
||||
{
|
||||
u_int i;
|
||||
u_int cpuid;
|
||||
|
||||
/*
|
||||
* Validate tunables, coerce to sensible values.
|
||||
*/
|
||||
switch (rss_hashalgo) {
|
||||
case RSS_HASH_TOEPLITZ:
|
||||
case RSS_HASH_NAIVE:
|
||||
break;
|
||||
|
||||
default:
|
||||
printf("%s: invalid RSS hashalgo %u, coercing to %u",
|
||||
__func__, rss_hashalgo, RSS_HASH_TOEPLITZ);
|
||||
rss_hashalgo = RSS_HASH_TOEPLITZ;
|
||||
}
|
||||
|
||||
/*
|
||||
* Count available CPUs.
|
||||
*
|
||||
* XXXRW: Note incorrect assumptions regarding contiguity of this set
|
||||
* elsewhere.
|
||||
*/
|
||||
rss_ncpus = 0;
|
||||
for (i = 0; i <= mp_maxid; i++) {
|
||||
if (CPU_ABSENT(i))
|
||||
continue;
|
||||
rss_ncpus++;
|
||||
}
|
||||
if (rss_ncpus > RSS_MAXCPUS)
|
||||
rss_ncpus = RSS_MAXCPUS;
|
||||
|
||||
/*
|
||||
* Tune RSS table entries to be no less than 2x the number of CPUs
|
||||
* -- unless we're running uniprocessor, in which case there's not
|
||||
* much point in having buckets to rearrange for load-balancing!
|
||||
*/
|
||||
if (rss_ncpus > 1) {
|
||||
if (rss_bits == 0)
|
||||
rss_bits = fls(rss_ncpus - 1) + 1;
|
||||
|
||||
/*
|
||||
* Microsoft limits RSS table entries to 128, so apply that
|
||||
* limit to both auto-detected CPU counts and user-configured
|
||||
* ones.
|
||||
*/
|
||||
if (rss_bits == 0 || rss_bits > RSS_MAXBITS) {
|
||||
printf("%s: RSS bits %u not valid, coercing to %u",
|
||||
__func__, rss_bits, RSS_MAXBITS);
|
||||
rss_bits = RSS_MAXBITS;
|
||||
}
|
||||
|
||||
/*
|
||||
* Figure out how many buckets to use; warn if less than the
|
||||
* number of configured CPUs, although this is not a fatal
|
||||
* problem.
|
||||
*/
|
||||
rss_buckets = (1 << rss_bits);
|
||||
if (rss_buckets < rss_ncpus)
|
||||
printf("%s: WARNING: rss_buckets (%u) less than "
|
||||
"rss_ncpus (%u)\n", __func__, rss_buckets,
|
||||
rss_ncpus);
|
||||
rss_mask = rss_buckets - 1;
|
||||
} else {
|
||||
rss_bits = 0;
|
||||
rss_buckets = 1;
|
||||
rss_mask = 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Set up initial CPU assignments: round-robin by default.
|
||||
*/
|
||||
cpuid = CPU_FIRST();
|
||||
for (i = 0; i < rss_buckets; i++) {
|
||||
rss_table[i].rte_cpu = cpuid;
|
||||
cpuid = CPU_NEXT(cpuid);
|
||||
}
|
||||
|
||||
/*
|
||||
* Randomize rrs_key.
|
||||
*
|
||||
* XXXRW: Not yet. If nothing else, will require an rss_isbadkey()
|
||||
* loop to check for "bad" RSS keys.
|
||||
*/
|
||||
}
|
||||
SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL);
|
||||
|
||||
static uint32_t
|
||||
rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen,
|
||||
const uint8_t *data)
|
||||
{
|
||||
uint32_t v;
|
||||
u_int i;
|
||||
|
||||
v = 0;
|
||||
for (i = 0; i < keylen; i++)
|
||||
v += key[i];
|
||||
for (i = 0; i < datalen; i++)
|
||||
v += data[i];
|
||||
return (v);
|
||||
}
|
||||
|
||||
uint32_t
|
||||
rss_hash(u_int datalen, const uint8_t *data)
|
||||
{
|
||||
|
||||
switch (rss_hashalgo) {
|
||||
case RSS_HASH_TOEPLITZ:
|
||||
return (toeplitz_hash(sizeof(rss_key), rss_key, datalen,
|
||||
data));
|
||||
|
||||
case RSS_HASH_NAIVE:
|
||||
return (rss_naive_hash(sizeof(rss_key), rss_key, datalen,
|
||||
data));
|
||||
|
||||
default:
|
||||
panic("%s: unsupported/unknown hashalgo %d", __func__,
|
||||
rss_hashalgo);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Query the number of RSS bits in use.
|
||||
*/
|
||||
u_int
|
||||
rss_getbits(void)
|
||||
{
|
||||
|
||||
return (rss_bits);
|
||||
}
|
||||
|
||||
/*
|
||||
* Query the RSS bucket associated with an RSS hash.
|
||||
*/
|
||||
u_int
|
||||
rss_getbucket(u_int hash)
|
||||
{
|
||||
|
||||
return (hash & rss_mask);
|
||||
}
|
||||
|
||||
/*
|
||||
* Query the RSS layer bucket associated with the given
|
||||
* entry in the RSS hash space.
|
||||
*
|
||||
* The RSS indirection table is 0 .. rss_buckets-1,
|
||||
* covering the low 'rss_bits' of the total 128 slot
|
||||
* RSS indirection table. So just mask off rss_bits and
|
||||
* return that.
|
||||
*
|
||||
* NIC drivers can then iterate over the 128 slot RSS
|
||||
* indirection table and fetch which RSS bucket to
|
||||
* map it to. This will typically be a CPU queue
|
||||
*/
|
||||
u_int
|
||||
rss_get_indirection_to_bucket(u_int index)
|
||||
{
|
||||
|
||||
return (index & rss_mask);
|
||||
}
|
||||
|
||||
/*
|
||||
* Query the RSS CPU associated with an RSS bucket.
|
||||
*/
|
||||
u_int
|
||||
rss_getcpu(u_int bucket)
|
||||
{
|
||||
|
||||
return (rss_table[bucket].rte_cpu);
|
||||
}
|
||||
|
||||
/*
|
||||
* netisr CPU affinity lookup given just the hash and hashtype.
|
||||
*/
|
||||
u_int
|
||||
rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type)
|
||||
{
|
||||
|
||||
switch (hash_type) {
|
||||
case M_HASHTYPE_RSS_IPV4:
|
||||
case M_HASHTYPE_RSS_TCP_IPV4:
|
||||
case M_HASHTYPE_RSS_UDP_IPV4:
|
||||
case M_HASHTYPE_RSS_IPV6:
|
||||
case M_HASHTYPE_RSS_TCP_IPV6:
|
||||
case M_HASHTYPE_RSS_UDP_IPV6:
|
||||
return (rss_getcpu(rss_getbucket(hash_val)));
|
||||
default:
|
||||
return (NETISR_CPUID_NONE);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Query the RSS bucket associated with the given hash value and
|
||||
* type.
|
||||
*/
|
||||
int
|
||||
rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id)
|
||||
{
|
||||
|
||||
switch (hash_type) {
|
||||
case M_HASHTYPE_RSS_IPV4:
|
||||
case M_HASHTYPE_RSS_TCP_IPV4:
|
||||
case M_HASHTYPE_RSS_UDP_IPV4:
|
||||
case M_HASHTYPE_RSS_IPV6:
|
||||
case M_HASHTYPE_RSS_TCP_IPV6:
|
||||
case M_HASHTYPE_RSS_UDP_IPV6:
|
||||
*bucket_id = rss_getbucket(hash_val);
|
||||
return (0);
|
||||
default:
|
||||
return (-1);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* netisr CPU affinity lookup routine for use by protocols.
|
||||
*/
|
||||
struct mbuf *
|
||||
rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid)
|
||||
{
|
||||
|
||||
M_ASSERTPKTHDR(m);
|
||||
*cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m));
|
||||
return (m);
|
||||
}
|
||||
|
||||
int
|
||||
rss_m2bucket(struct mbuf *m, uint32_t *bucket_id)
|
||||
{
|
||||
|
||||
M_ASSERTPKTHDR(m);
|
||||
|
||||
return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m),
|
||||
bucket_id));
|
||||
}
|
||||
|
||||
/*
|
||||
* Query the RSS hash algorithm.
|
||||
*/
|
||||
u_int
|
||||
rss_gethashalgo(void)
|
||||
{
|
||||
|
||||
return (rss_hashalgo);
|
||||
}
|
||||
|
||||
/*
|
||||
* Query the current RSS key; likely to be used by device drivers when
|
||||
* configuring hardware RSS. Caller must pass an array of size RSS_KEYSIZE.
|
||||
*
|
||||
* XXXRW: Perhaps we should do the accept-a-length-and-truncate thing?
|
||||
*/
|
||||
void
|
||||
rss_getkey(uint8_t *key)
|
||||
{
|
||||
|
||||
bcopy(rss_key, key, sizeof(rss_key));
|
||||
}
|
||||
|
||||
/*
|
||||
* Query the number of buckets; this may be used by both network device
|
||||
* drivers, which will need to populate hardware shadows of the software
|
||||
* indirection table, and the network stack itself (such as when deciding how
|
||||
* many connection groups to allocate).
|
||||
*/
|
||||
u_int
|
||||
rss_getnumbuckets(void)
|
||||
{
|
||||
|
||||
return (rss_buckets);
|
||||
}
|
||||
|
||||
/*
|
||||
* Query the number of CPUs in use by RSS; may be useful to device drivers
|
||||
* trying to figure out how to map a larger number of CPUs into a smaller
|
||||
* number of receive queues.
|
||||
*/
|
||||
u_int
|
||||
rss_getnumcpus(void)
|
||||
{
|
||||
|
||||
return (rss_ncpus);
|
||||
}
|
||||
|
||||
/*
|
||||
* Return the supported RSS hash configuration.
|
||||
*
|
||||
* NICs should query this to determine what to configure in their redirection
|
||||
* matching table.
|
||||
*/
|
||||
inline u_int
|
||||
rss_gethashconfig(void)
|
||||
{
|
||||
|
||||
/* Return 4-tuple for TCP; 2-tuple for others */
|
||||
/*
|
||||
* UDP may fragment more often than TCP and thus we'll end up with
|
||||
* NICs returning 2-tuple fragments.
|
||||
* udp_init() and udplite_init() both currently initialise things
|
||||
* as 2-tuple.
|
||||
* So for now disable UDP 4-tuple hashing until all of the other
|
||||
* pieces are in place.
|
||||
*/
|
||||
return (
|
||||
RSS_HASHTYPE_RSS_IPV4
|
||||
| RSS_HASHTYPE_RSS_TCP_IPV4
|
||||
| RSS_HASHTYPE_RSS_IPV6
|
||||
| RSS_HASHTYPE_RSS_TCP_IPV6
|
||||
| RSS_HASHTYPE_RSS_IPV6_EX
|
||||
| RSS_HASHTYPE_RSS_TCP_IPV6_EX
|
||||
#if 0
|
||||
| RSS_HASHTYPE_RSS_UDP_IPV4
|
||||
| RSS_HASHTYPE_RSS_UDP_IPV4_EX
|
||||
| RSS_HASHTYPE_RSS_UDP_IPV6
|
||||
| RSS_HASHTYPE_RSS_UDP_IPV6_EX
|
||||
#endif
|
||||
);
|
||||
}
|
||||
|
||||
/*
|
||||
* XXXRW: Confirm that sysctl -a won't dump this keying material, don't want
|
||||
* it appearing in debugging output unnecessarily.
|
||||
*/
|
||||
static int
|
||||
sysctl_rss_key(SYSCTL_HANDLER_ARGS)
|
||||
{
|
||||
uint8_t temp_rss_key[RSS_KEYSIZE];
|
||||
int error;
|
||||
|
||||
error = priv_check(req->td, PRIV_NETINET_HASHKEY);
|
||||
if (error)
|
||||
return (error);
|
||||
|
||||
bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key));
|
||||
error = sysctl_handle_opaque(oidp, temp_rss_key,
|
||||
sizeof(temp_rss_key), req);
|
||||
if (error)
|
||||
return (error);
|
||||
if (req->newptr != NULL) {
|
||||
/* XXXRW: Not yet. */
|
||||
return (EINVAL);
|
||||
}
|
||||
return (0);
|
||||
}
|
||||
SYSCTL_PROC(_net_inet_rss, OID_AUTO, key,
|
||||
CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key,
|
||||
"", "RSS keying material");
|
||||
|
||||
static int
|
||||
sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS)
|
||||
{
|
||||
struct sbuf *sb;
|
||||
int error;
|
||||
int i;
|
||||
|
||||
error = 0;
|
||||
error = sysctl_wire_old_buffer(req, 0);
|
||||
if (error != 0)
|
||||
return (error);
|
||||
sb = sbuf_new_for_sysctl(NULL, NULL, 512, req);
|
||||
if (sb == NULL)
|
||||
return (ENOMEM);
|
||||
for (i = 0; i < rss_buckets; i++) {
|
||||
sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ",
|
||||
i,
|
||||
rss_getcpu(i));
|
||||
}
|
||||
error = sbuf_finish(sb);
|
||||
sbuf_delete(sb);
|
||||
|
||||
return (error);
|
||||
}
|
||||
SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping,
|
||||
CTLTYPE_STRING | CTLFLAG_RD, NULL, 0,
|
||||
sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping");
|
||||
123
sys/net/rss_config.h
Normal file
123
sys/net/rss_config.h
Normal file
@ -0,0 +1,123 @@
|
||||
/*-
|
||||
* Copyright (c) 2010-2011 Juniper Networks, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This software was developed by Robert N. M. Watson under contract
|
||||
* to Juniper Networks, Inc.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*
|
||||
* $FreeBSD$
|
||||
*/
|
||||
|
||||
#ifndef _NET_RSS_CONFIG_H_
|
||||
#define _NET_RSS_CONFIG_H_
|
||||
|
||||
#include <netinet/in.h> /* in_addr_t */
|
||||
|
||||
/*
|
||||
* Supported RSS hash functions.
|
||||
*/
|
||||
#define RSS_HASH_NAIVE 0x00000001 /* Poor but fast hash. */
|
||||
#define RSS_HASH_TOEPLITZ 0x00000002 /* Required by RSS. */
|
||||
#define RSS_HASH_CRC32 0x00000004 /* Future; some NICs do it. */
|
||||
|
||||
#define RSS_HASH_MASK (RSS_HASH_NAIVE | RSS_HASH_TOEPLITZ)
|
||||
|
||||
/*
|
||||
* Instances of struct inpcbinfo declare an RSS hash type indicating what
|
||||
* header fields are covered.
|
||||
*/
|
||||
#define RSS_HASHFIELDS_NONE 0
|
||||
#define RSS_HASHFIELDS_4TUPLE 1
|
||||
#define RSS_HASHFIELDS_2TUPLE 2
|
||||
|
||||
/*
|
||||
* Define RSS representations of the M_HASHTYPE_* values, representing
|
||||
* which particular bits are supported. The NICs can then use this to
|
||||
* calculate which hash types to enable and which not to enable.
|
||||
*
|
||||
* The fact that these line up with M_HASHTYPE_* is not to be relied
|
||||
* upon.
|
||||
*/
|
||||
#define RSS_HASHTYPE_RSS_IPV4 (1 << 1) /* IPv4 2-tuple */
|
||||
#define RSS_HASHTYPE_RSS_TCP_IPV4 (1 << 2) /* TCPv4 4-tuple */
|
||||
#define RSS_HASHTYPE_RSS_IPV6 (1 << 3) /* IPv6 2-tuple */
|
||||
#define RSS_HASHTYPE_RSS_TCP_IPV6 (1 << 4) /* TCPv6 4-tuple */
|
||||
#define RSS_HASHTYPE_RSS_IPV6_EX (1 << 5) /* IPv6 2-tuple + ext hdrs */
|
||||
#define RSS_HASHTYPE_RSS_TCP_IPV6_EX (1 << 6) /* TCPv6 4-tiple + ext hdrs */
|
||||
#define RSS_HASHTYPE_RSS_UDP_IPV4 (1 << 7) /* IPv4 UDP 4-tuple */
|
||||
#define RSS_HASHTYPE_RSS_UDP_IPV4_EX (1 << 8) /* IPv4 UDP 4-tuple + ext hdrs */
|
||||
#define RSS_HASHTYPE_RSS_UDP_IPV6 (1 << 9) /* IPv6 UDP 4-tuple */
|
||||
#define RSS_HASHTYPE_RSS_UDP_IPV6_EX (1 << 10) /* IPv6 UDP 4-tuple + ext hdrs */
|
||||
|
||||
/*
|
||||
* Compile-time limits on the size of the indirection table.
|
||||
*/
|
||||
#define RSS_MAXBITS 7
|
||||
#define RSS_TABLE_MAXLEN (1 << RSS_MAXBITS)
|
||||
|
||||
/*
|
||||
* Maximum key size used throughout. It's OK for hardware to use only the
|
||||
* first 16 bytes, which is all that's required for IPv4.
|
||||
*/
|
||||
#define RSS_KEYSIZE 40
|
||||
|
||||
/*
|
||||
* For RSS hash methods that do a software hash on an mbuf, the packet
|
||||
* direction (ingress / egress) is required.
|
||||
*
|
||||
* The default direction (INGRESS) is the "receive into the NIC" - ie,
|
||||
* what the hardware is hashing on.
|
||||
*/
|
||||
#define RSS_HASH_PKT_INGRESS 0
|
||||
#define RSS_HASH_PKT_EGRESS 1
|
||||
|
||||
/*
|
||||
* Device driver interfaces to query RSS properties that must be programmed
|
||||
* into hardware.
|
||||
*/
|
||||
u_int rss_getbits(void);
|
||||
u_int rss_getbucket(u_int hash);
|
||||
u_int rss_get_indirection_to_bucket(u_int index);
|
||||
u_int rss_getcpu(u_int bucket);
|
||||
void rss_getkey(uint8_t *key);
|
||||
u_int rss_gethashalgo(void);
|
||||
u_int rss_getnumbuckets(void);
|
||||
u_int rss_getnumcpus(void);
|
||||
u_int rss_gethashconfig(void);
|
||||
|
||||
/*
|
||||
* Hash calculation functions.
|
||||
*/
|
||||
uint32_t rss_hash(u_int datalen, const uint8_t *data);
|
||||
|
||||
/*
|
||||
* Network stack interface to query desired CPU affinity of a packet.
|
||||
*/
|
||||
struct mbuf * rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid);
|
||||
u_int rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type);
|
||||
int rss_hash2bucket(uint32_t hash_val, uint32_t hash_type,
|
||||
uint32_t *bucket_id);
|
||||
int rss_m2bucket(struct mbuf *m, uint32_t *bucket_id);
|
||||
|
||||
#endif /* !_NET_RSS_CONFIG_H_ */
|
||||
@ -29,8 +29,8 @@ __FBSDID("$FreeBSD$");
|
||||
|
||||
#include <sys/types.h>
|
||||
|
||||
#include <netinet/in_rss.h>
|
||||
#include <netinet/toeplitz.h>
|
||||
#include <net/rss_config.h>
|
||||
#include <net/toeplitz.h>
|
||||
|
||||
#include <sys/systm.h>
|
||||
|
||||
@ -71,12 +71,12 @@ __FBSDID("$FreeBSD$");
|
||||
#include <net/if_var.h>
|
||||
#include <net/if_types.h>
|
||||
#include <net/route.h>
|
||||
#include <net/rss_config.h>
|
||||
#include <net/vnet.h>
|
||||
|
||||
#if defined(INET) || defined(INET6)
|
||||
#include <netinet/in.h>
|
||||
#include <netinet/in_pcb.h>
|
||||
#include <netinet/in_rss.h>
|
||||
#include <netinet/ip_var.h>
|
||||
#include <netinet/tcp_var.h>
|
||||
#include <netinet/udp.h>
|
||||
|
||||
@ -42,7 +42,10 @@ __FBSDID("$FreeBSD$");
|
||||
#include <sys/smp.h>
|
||||
#include <sys/socketvar.h>
|
||||
|
||||
#include <net/rss_config.h>
|
||||
|
||||
#include <netinet/in.h>
|
||||
|
||||
#include <netinet/in_pcb.h>
|
||||
#include <netinet/in_rss.h>
|
||||
#ifdef INET6
|
||||
|
||||
@ -50,257 +50,18 @@ __FBSDID("$FreeBSD$");
|
||||
#include <net/if.h>
|
||||
#include <net/if_var.h>
|
||||
#include <net/netisr.h>
|
||||
#include <net/rss_config.h>
|
||||
|
||||
#include <netinet/in.h>
|
||||
#include <netinet/in_pcb.h>
|
||||
#include <netinet/in_rss.h>
|
||||
#include <netinet/in_var.h>
|
||||
#include <netinet/toeplitz.h>
|
||||
|
||||
/* for software rss hash support */
|
||||
#include <netinet/ip.h>
|
||||
#include <netinet/tcp.h>
|
||||
#include <netinet/udp.h>
|
||||
|
||||
/*-
|
||||
* Operating system parts of receiver-side scaling (RSS), which allows
|
||||
* network cards to direct flows to particular receive queues based on hashes
|
||||
* of header tuples. This implementation aligns RSS buckets with connection
|
||||
* groups at the TCP/IP layer, so each bucket is associated with exactly one
|
||||
* group. As a result, the group lookup structures (and lock) should have an
|
||||
* effective affinity with exactly one CPU.
|
||||
*
|
||||
* Network device drivers needing to configure RSS will query this framework
|
||||
* for parameters, such as the current RSS key, hashing policies, number of
|
||||
* bits, and indirection table mapping hashes to buckets and CPUs. They may
|
||||
* provide their own supplementary information, such as queue<->CPU bindings.
|
||||
* It is the responsibility of the network device driver to inject packets
|
||||
* into the stack on as close to the right CPU as possible, if playing by RSS
|
||||
* rules.
|
||||
*
|
||||
* TODO:
|
||||
*
|
||||
* - Synchronization for rss_key and other future-configurable parameters.
|
||||
* - Event handler drivers can register to pick up RSS configuration changes.
|
||||
* - Should we allow rss_basecpu to be configured?
|
||||
* - Randomize key on boot.
|
||||
* - IPv6 support.
|
||||
* - Statistics on how often there's a misalignment between hardware
|
||||
* placement and pcbgroup expectations.
|
||||
*/
|
||||
|
||||
SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW, 0, "Receive-side steering");
|
||||
|
||||
/*
|
||||
* Toeplitz is the only required hash function in the RSS spec, so use it by
|
||||
* default.
|
||||
*/
|
||||
static u_int rss_hashalgo = RSS_HASH_TOEPLITZ;
|
||||
SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0,
|
||||
"RSS hash algorithm");
|
||||
|
||||
/*
|
||||
* Size of the indirection table; at most 128 entries per the RSS spec. We
|
||||
* size it to at least 2 times the number of CPUs by default to allow useful
|
||||
* rebalancing. If not set explicitly with a loader tunable, we tune based
|
||||
* on the number of CPUs present.
|
||||
*
|
||||
* XXXRW: buckets might be better to use for the tunable than bits.
|
||||
*/
|
||||
static u_int rss_bits;
|
||||
SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0,
|
||||
"RSS bits");
|
||||
|
||||
static u_int rss_mask;
|
||||
SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0,
|
||||
"RSS mask");
|
||||
|
||||
static const u_int rss_maxbits = RSS_MAXBITS;
|
||||
SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD,
|
||||
__DECONST(int *, &rss_maxbits), 0, "RSS maximum bits");
|
||||
|
||||
/*
|
||||
* RSS's own count of the number of CPUs it could be using for processing.
|
||||
* Bounded to 64 by RSS constants.
|
||||
*/
|
||||
static u_int rss_ncpus;
|
||||
SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0,
|
||||
"Number of CPUs available to RSS");
|
||||
|
||||
#define RSS_MAXCPUS (1 << (RSS_MAXBITS - 1))
|
||||
static const u_int rss_maxcpus = RSS_MAXCPUS;
|
||||
SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD,
|
||||
__DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used");
|
||||
|
||||
/*
|
||||
* Variable exists just for reporting rss_bits in a user-friendly way.
|
||||
*/
|
||||
static u_int rss_buckets;
|
||||
SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0,
|
||||
"RSS buckets");
|
||||
|
||||
/*
|
||||
* Base CPU number; devices will add this to all CPU numbers returned by the
|
||||
* RSS indirection table. Currently unmodifable in FreeBSD.
|
||||
*/
|
||||
static const u_int rss_basecpu;
|
||||
SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD,
|
||||
__DECONST(int *, &rss_basecpu), 0, "RSS base CPU");
|
||||
|
||||
/*
|
||||
* RSS secret key, intended to prevent attacks on load-balancing. Its
|
||||
* effectiveness may be limited by algorithm choice and available entropy
|
||||
* during the boot.
|
||||
*
|
||||
* XXXRW: And that we don't randomize it yet!
|
||||
*
|
||||
* This is the default Microsoft RSS specification key which is also
|
||||
* the Chelsio T5 firmware default key.
|
||||
*/
|
||||
static uint8_t rss_key[RSS_KEYSIZE] = {
|
||||
0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
|
||||
0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
|
||||
0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
|
||||
0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
|
||||
0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa,
|
||||
};
|
||||
|
||||
/*
|
||||
* RSS hash->CPU table, which maps hashed packet headers to particular CPUs.
|
||||
* Drivers may supplement this table with a seperate CPU<->queue table when
|
||||
* programming devices.
|
||||
*/
|
||||
struct rss_table_entry {
|
||||
uint8_t rte_cpu; /* CPU affinity of bucket. */
|
||||
};
|
||||
static struct rss_table_entry rss_table[RSS_TABLE_MAXLEN];
|
||||
|
||||
static inline u_int rss_gethashconfig_local(void);
|
||||
|
||||
static void
|
||||
rss_init(__unused void *arg)
|
||||
{
|
||||
u_int i;
|
||||
u_int cpuid;
|
||||
|
||||
/*
|
||||
* Validate tunables, coerce to sensible values.
|
||||
*/
|
||||
switch (rss_hashalgo) {
|
||||
case RSS_HASH_TOEPLITZ:
|
||||
case RSS_HASH_NAIVE:
|
||||
break;
|
||||
|
||||
default:
|
||||
printf("%s: invalid RSS hashalgo %u, coercing to %u",
|
||||
__func__, rss_hashalgo, RSS_HASH_TOEPLITZ);
|
||||
rss_hashalgo = RSS_HASH_TOEPLITZ;
|
||||
}
|
||||
|
||||
/*
|
||||
* Count available CPUs.
|
||||
*
|
||||
* XXXRW: Note incorrect assumptions regarding contiguity of this set
|
||||
* elsewhere.
|
||||
*/
|
||||
rss_ncpus = 0;
|
||||
for (i = 0; i <= mp_maxid; i++) {
|
||||
if (CPU_ABSENT(i))
|
||||
continue;
|
||||
rss_ncpus++;
|
||||
}
|
||||
if (rss_ncpus > RSS_MAXCPUS)
|
||||
rss_ncpus = RSS_MAXCPUS;
|
||||
|
||||
/*
|
||||
* Tune RSS table entries to be no less than 2x the number of CPUs
|
||||
* -- unless we're running uniprocessor, in which case there's not
|
||||
* much point in having buckets to rearrange for load-balancing!
|
||||
*/
|
||||
if (rss_ncpus > 1) {
|
||||
if (rss_bits == 0)
|
||||
rss_bits = fls(rss_ncpus - 1) + 1;
|
||||
|
||||
/*
|
||||
* Microsoft limits RSS table entries to 128, so apply that
|
||||
* limit to both auto-detected CPU counts and user-configured
|
||||
* ones.
|
||||
*/
|
||||
if (rss_bits == 0 || rss_bits > RSS_MAXBITS) {
|
||||
printf("%s: RSS bits %u not valid, coercing to %u",
|
||||
__func__, rss_bits, RSS_MAXBITS);
|
||||
rss_bits = RSS_MAXBITS;
|
||||
}
|
||||
|
||||
/*
|
||||
* Figure out how many buckets to use; warn if less than the
|
||||
* number of configured CPUs, although this is not a fatal
|
||||
* problem.
|
||||
*/
|
||||
rss_buckets = (1 << rss_bits);
|
||||
if (rss_buckets < rss_ncpus)
|
||||
printf("%s: WARNING: rss_buckets (%u) less than "
|
||||
"rss_ncpus (%u)\n", __func__, rss_buckets,
|
||||
rss_ncpus);
|
||||
rss_mask = rss_buckets - 1;
|
||||
} else {
|
||||
rss_bits = 0;
|
||||
rss_buckets = 1;
|
||||
rss_mask = 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Set up initial CPU assignments: round-robin by default.
|
||||
*/
|
||||
cpuid = CPU_FIRST();
|
||||
for (i = 0; i < rss_buckets; i++) {
|
||||
rss_table[i].rte_cpu = cpuid;
|
||||
cpuid = CPU_NEXT(cpuid);
|
||||
}
|
||||
|
||||
/*
|
||||
* Randomize rrs_key.
|
||||
*
|
||||
* XXXRW: Not yet. If nothing else, will require an rss_isbadkey()
|
||||
* loop to check for "bad" RSS keys.
|
||||
*/
|
||||
}
|
||||
SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL);
|
||||
|
||||
static uint32_t
|
||||
rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen,
|
||||
const uint8_t *data)
|
||||
{
|
||||
uint32_t v;
|
||||
u_int i;
|
||||
|
||||
v = 0;
|
||||
for (i = 0; i < keylen; i++)
|
||||
v += key[i];
|
||||
for (i = 0; i < datalen; i++)
|
||||
v += data[i];
|
||||
return (v);
|
||||
}
|
||||
|
||||
static uint32_t
|
||||
rss_hash(u_int datalen, const uint8_t *data)
|
||||
{
|
||||
|
||||
switch (rss_hashalgo) {
|
||||
case RSS_HASH_TOEPLITZ:
|
||||
return (toeplitz_hash(sizeof(rss_key), rss_key, datalen,
|
||||
data));
|
||||
|
||||
case RSS_HASH_NAIVE:
|
||||
return (rss_naive_hash(sizeof(rss_key), rss_key, datalen,
|
||||
data));
|
||||
|
||||
default:
|
||||
panic("%s: unsupported/unknown hashalgo %d", __func__,
|
||||
rss_hashalgo);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Hash an IPv4 2-tuple.
|
||||
*/
|
||||
@ -341,162 +102,6 @@ rss_hash_ip4_4tuple(struct in_addr src, u_short srcport, struct in_addr dst,
|
||||
return (rss_hash(datalen, data));
|
||||
}
|
||||
|
||||
#ifdef INET6
|
||||
/*
|
||||
* Hash an IPv6 2-tuple.
|
||||
*/
|
||||
uint32_t
|
||||
rss_hash_ip6_2tuple(const struct in6_addr *src, const struct in6_addr *dst)
|
||||
{
|
||||
uint8_t data[sizeof(*src) + sizeof(*dst)];
|
||||
u_int datalen;
|
||||
|
||||
datalen = 0;
|
||||
bcopy(src, &data[datalen], sizeof(*src));
|
||||
datalen += sizeof(*src);
|
||||
bcopy(dst, &data[datalen], sizeof(*dst));
|
||||
datalen += sizeof(*dst);
|
||||
return (rss_hash(datalen, data));
|
||||
}
|
||||
|
||||
/*
|
||||
* Hash an IPv6 4-tuple.
|
||||
*/
|
||||
uint32_t
|
||||
rss_hash_ip6_4tuple(const struct in6_addr *src, u_short srcport,
|
||||
const struct in6_addr *dst, u_short dstport)
|
||||
{
|
||||
uint8_t data[sizeof(*src) + sizeof(*dst) + sizeof(srcport) +
|
||||
sizeof(dstport)];
|
||||
u_int datalen;
|
||||
|
||||
datalen = 0;
|
||||
bcopy(src, &data[datalen], sizeof(*src));
|
||||
datalen += sizeof(*src);
|
||||
bcopy(dst, &data[datalen], sizeof(*dst));
|
||||
datalen += sizeof(*dst);
|
||||
bcopy(&srcport, &data[datalen], sizeof(srcport));
|
||||
datalen += sizeof(srcport);
|
||||
bcopy(&dstport, &data[datalen], sizeof(dstport));
|
||||
datalen += sizeof(dstport);
|
||||
return (rss_hash(datalen, data));
|
||||
}
|
||||
#endif /* INET6 */
|
||||
|
||||
/*
|
||||
* Query the number of RSS bits in use.
|
||||
*/
|
||||
u_int
|
||||
rss_getbits(void)
|
||||
{
|
||||
|
||||
return (rss_bits);
|
||||
}
|
||||
|
||||
/*
|
||||
* Query the RSS bucket associated with an RSS hash.
|
||||
*/
|
||||
u_int
|
||||
rss_getbucket(u_int hash)
|
||||
{
|
||||
|
||||
return (hash & rss_mask);
|
||||
}
|
||||
|
||||
/*
|
||||
* Query the RSS layer bucket associated with the given
|
||||
* entry in the RSS hash space.
|
||||
*
|
||||
* The RSS indirection table is 0 .. rss_buckets-1,
|
||||
* covering the low 'rss_bits' of the total 128 slot
|
||||
* RSS indirection table. So just mask off rss_bits and
|
||||
* return that.
|
||||
*
|
||||
* NIC drivers can then iterate over the 128 slot RSS
|
||||
* indirection table and fetch which RSS bucket to
|
||||
* map it to. This will typically be a CPU queue
|
||||
*/
|
||||
u_int
|
||||
rss_get_indirection_to_bucket(u_int index)
|
||||
{
|
||||
|
||||
return (index & rss_mask);
|
||||
}
|
||||
|
||||
/*
|
||||
* Query the RSS CPU associated with an RSS bucket.
|
||||
*/
|
||||
u_int
|
||||
rss_getcpu(u_int bucket)
|
||||
{
|
||||
|
||||
return (rss_table[bucket].rte_cpu);
|
||||
}
|
||||
|
||||
/*
|
||||
* netisr CPU affinity lookup given just the hash and hashtype.
|
||||
*/
|
||||
u_int
|
||||
rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type)
|
||||
{
|
||||
|
||||
switch (hash_type) {
|
||||
case M_HASHTYPE_RSS_IPV4:
|
||||
case M_HASHTYPE_RSS_TCP_IPV4:
|
||||
case M_HASHTYPE_RSS_UDP_IPV4:
|
||||
case M_HASHTYPE_RSS_IPV6:
|
||||
case M_HASHTYPE_RSS_TCP_IPV6:
|
||||
case M_HASHTYPE_RSS_UDP_IPV6:
|
||||
return (rss_getcpu(rss_getbucket(hash_val)));
|
||||
default:
|
||||
return (NETISR_CPUID_NONE);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Query the RSS bucket associated with the given hash value and
|
||||
* type.
|
||||
*/
|
||||
int
|
||||
rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id)
|
||||
{
|
||||
|
||||
switch (hash_type) {
|
||||
case M_HASHTYPE_RSS_IPV4:
|
||||
case M_HASHTYPE_RSS_TCP_IPV4:
|
||||
case M_HASHTYPE_RSS_UDP_IPV4:
|
||||
case M_HASHTYPE_RSS_IPV6:
|
||||
case M_HASHTYPE_RSS_TCP_IPV6:
|
||||
case M_HASHTYPE_RSS_UDP_IPV6:
|
||||
*bucket_id = rss_getbucket(hash_val);
|
||||
return (0);
|
||||
default:
|
||||
return (-1);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* netisr CPU affinity lookup routine for use by protocols.
|
||||
*/
|
||||
struct mbuf *
|
||||
rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid)
|
||||
{
|
||||
|
||||
M_ASSERTPKTHDR(m);
|
||||
*cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m));
|
||||
return (m);
|
||||
}
|
||||
|
||||
int
|
||||
rss_m2bucket(struct mbuf *m, uint32_t *bucket_id)
|
||||
{
|
||||
|
||||
M_ASSERTPKTHDR(m);
|
||||
|
||||
return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m),
|
||||
bucket_id));
|
||||
}
|
||||
|
||||
/*
|
||||
* Calculate an appropriate ipv4 2-tuple or 4-tuple given the given
|
||||
* IPv4 source/destination address, UDP or TCP source/destination ports
|
||||
@ -522,18 +127,18 @@ rss_proto_software_hash_v4(struct in_addr s, struct in_addr d,
|
||||
* identifier.
|
||||
*/
|
||||
if ((proto == IPPROTO_TCP) &&
|
||||
(rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4)) {
|
||||
(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4)) {
|
||||
hash = rss_hash_ip4_4tuple(s, sp, d, dp);
|
||||
*hashval = hash;
|
||||
*hashtype = M_HASHTYPE_RSS_TCP_IPV4;
|
||||
return (0);
|
||||
} else if ((proto == IPPROTO_UDP) &&
|
||||
(rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4)) {
|
||||
(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4)) {
|
||||
hash = rss_hash_ip4_4tuple(s, sp, d, dp);
|
||||
*hashval = hash;
|
||||
*hashtype = M_HASHTYPE_RSS_UDP_IPV4;
|
||||
return (0);
|
||||
} else if (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) {
|
||||
} else if (rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) {
|
||||
/* RSS doesn't hash on other protocols like SCTP; so 2-tuple */
|
||||
hash = rss_hash_ip4_2tuple(s, d);
|
||||
*hashval = hash;
|
||||
@ -625,7 +230,7 @@ rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, uint32_t *hashval,
|
||||
if (flowtype != M_HASHTYPE_NONE) {
|
||||
switch (proto) {
|
||||
case IPPROTO_UDP:
|
||||
if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4) &&
|
||||
if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4) &&
|
||||
(flowtype == M_HASHTYPE_RSS_UDP_IPV4) &&
|
||||
(is_frag == 0)) {
|
||||
return (1);
|
||||
@ -634,14 +239,14 @@ rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, uint32_t *hashval,
|
||||
* Only allow 2-tuple for UDP frames if we don't also
|
||||
* support 4-tuple for UDP.
|
||||
*/
|
||||
if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) &&
|
||||
((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4) == 0) &&
|
||||
if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) &&
|
||||
((rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4) == 0) &&
|
||||
flowtype == M_HASHTYPE_RSS_IPV4) {
|
||||
return (1);
|
||||
}
|
||||
break;
|
||||
case IPPROTO_TCP:
|
||||
if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4) &&
|
||||
if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4) &&
|
||||
(flowtype == M_HASHTYPE_RSS_TCP_IPV4) &&
|
||||
(is_frag == 0)) {
|
||||
return (1);
|
||||
@ -650,14 +255,14 @@ rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, uint32_t *hashval,
|
||||
* Only allow 2-tuple for TCP frames if we don't also
|
||||
* support 2-tuple for TCP.
|
||||
*/
|
||||
if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) &&
|
||||
((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4) == 0) &&
|
||||
if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) &&
|
||||
((rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4) == 0) &&
|
||||
flowtype == M_HASHTYPE_RSS_IPV4) {
|
||||
return (1);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) &&
|
||||
if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) &&
|
||||
flowtype == M_HASHTYPE_RSS_IPV4) {
|
||||
return (1);
|
||||
}
|
||||
@ -671,7 +276,7 @@ rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, uint32_t *hashval,
|
||||
* XXX TODO: does the hardware hash on 4-tuple if IP
|
||||
* options are present?
|
||||
*/
|
||||
if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4) &&
|
||||
if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4) &&
|
||||
(proto == IPPROTO_TCP) &&
|
||||
(is_frag == 0)) {
|
||||
if (m->m_len < iphlen + sizeof(struct tcphdr)) {
|
||||
@ -685,7 +290,7 @@ rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, uint32_t *hashval,
|
||||
proto,
|
||||
hashval,
|
||||
hashtype);
|
||||
} else if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4) &&
|
||||
} else if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4) &&
|
||||
(proto == IPPROTO_UDP) &&
|
||||
(is_frag == 0)) {
|
||||
uh = (struct udphdr *)((caddr_t)ip + iphlen);
|
||||
@ -699,7 +304,7 @@ rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, uint32_t *hashval,
|
||||
proto,
|
||||
hashval,
|
||||
hashtype);
|
||||
} else if (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) {
|
||||
} else if (rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) {
|
||||
/* Default to 2-tuple hash */
|
||||
return rss_proto_software_hash_v4(ip->ip_src, ip->ip_dst,
|
||||
0, /* source port */
|
||||
@ -750,150 +355,3 @@ rss_soft_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid)
|
||||
}
|
||||
return (m);
|
||||
}
|
||||
|
||||
/*
|
||||
* Query the RSS hash algorithm.
|
||||
*/
|
||||
u_int
|
||||
rss_gethashalgo(void)
|
||||
{
|
||||
|
||||
return (rss_hashalgo);
|
||||
}
|
||||
|
||||
/*
|
||||
* Query the current RSS key; likely to be used by device drivers when
|
||||
* configuring hardware RSS. Caller must pass an array of size RSS_KEYSIZE.
|
||||
*
|
||||
* XXXRW: Perhaps we should do the accept-a-length-and-truncate thing?
|
||||
*/
|
||||
void
|
||||
rss_getkey(uint8_t *key)
|
||||
{
|
||||
|
||||
bcopy(rss_key, key, sizeof(rss_key));
|
||||
}
|
||||
|
||||
/*
|
||||
* Query the number of buckets; this may be used by both network device
|
||||
* drivers, which will need to populate hardware shadows of the software
|
||||
* indirection table, and the network stack itself (such as when deciding how
|
||||
* many connection groups to allocate).
|
||||
*/
|
||||
u_int
|
||||
rss_getnumbuckets(void)
|
||||
{
|
||||
|
||||
return (rss_buckets);
|
||||
}
|
||||
|
||||
/*
|
||||
* Query the number of CPUs in use by RSS; may be useful to device drivers
|
||||
* trying to figure out how to map a larger number of CPUs into a smaller
|
||||
* number of receive queues.
|
||||
*/
|
||||
u_int
|
||||
rss_getnumcpus(void)
|
||||
{
|
||||
|
||||
return (rss_ncpus);
|
||||
}
|
||||
|
||||
static inline u_int
|
||||
rss_gethashconfig_local(void)
|
||||
{
|
||||
|
||||
/* Return 4-tuple for TCP; 2-tuple for others */
|
||||
/*
|
||||
* UDP may fragment more often than TCP and thus we'll end up with
|
||||
* NICs returning 2-tuple fragments.
|
||||
* udp_init() and udplite_init() both currently initialise things
|
||||
* as 2-tuple.
|
||||
* So for now disable UDP 4-tuple hashing until all of the other
|
||||
* pieces are in place.
|
||||
*/
|
||||
return (
|
||||
RSS_HASHTYPE_RSS_IPV4
|
||||
| RSS_HASHTYPE_RSS_TCP_IPV4
|
||||
| RSS_HASHTYPE_RSS_IPV6
|
||||
| RSS_HASHTYPE_RSS_TCP_IPV6
|
||||
| RSS_HASHTYPE_RSS_IPV6_EX
|
||||
| RSS_HASHTYPE_RSS_TCP_IPV6_EX
|
||||
#if 0
|
||||
| RSS_HASHTYPE_RSS_UDP_IPV4
|
||||
| RSS_HASHTYPE_RSS_UDP_IPV4_EX
|
||||
| RSS_HASHTYPE_RSS_UDP_IPV6
|
||||
| RSS_HASHTYPE_RSS_UDP_IPV6_EX
|
||||
#endif
|
||||
);
|
||||
}
|
||||
|
||||
/*
|
||||
* Return the supported RSS hash configuration.
|
||||
*
|
||||
* NICs should query this to determine what to configure in their redirection
|
||||
* matching table.
|
||||
*/
|
||||
u_int
|
||||
rss_gethashconfig(void)
|
||||
{
|
||||
|
||||
return (rss_gethashconfig_local());
|
||||
}
|
||||
|
||||
/*
|
||||
* XXXRW: Confirm that sysctl -a won't dump this keying material, don't want
|
||||
* it appearing in debugging output unnecessarily.
|
||||
*/
|
||||
static int
|
||||
sysctl_rss_key(SYSCTL_HANDLER_ARGS)
|
||||
{
|
||||
uint8_t temp_rss_key[RSS_KEYSIZE];
|
||||
int error;
|
||||
|
||||
error = priv_check(req->td, PRIV_NETINET_HASHKEY);
|
||||
if (error)
|
||||
return (error);
|
||||
|
||||
bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key));
|
||||
error = sysctl_handle_opaque(oidp, temp_rss_key,
|
||||
sizeof(temp_rss_key), req);
|
||||
if (error)
|
||||
return (error);
|
||||
if (req->newptr != NULL) {
|
||||
/* XXXRW: Not yet. */
|
||||
return (EINVAL);
|
||||
}
|
||||
return (0);
|
||||
}
|
||||
SYSCTL_PROC(_net_inet_rss, OID_AUTO, key,
|
||||
CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key,
|
||||
"", "RSS keying material");
|
||||
|
||||
static int
|
||||
sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS)
|
||||
{
|
||||
struct sbuf *sb;
|
||||
int error;
|
||||
int i;
|
||||
|
||||
error = 0;
|
||||
error = sysctl_wire_old_buffer(req, 0);
|
||||
if (error != 0)
|
||||
return (error);
|
||||
sb = sbuf_new_for_sysctl(NULL, NULL, 512, req);
|
||||
if (sb == NULL)
|
||||
return (ENOMEM);
|
||||
for (i = 0; i < rss_buckets; i++) {
|
||||
sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ",
|
||||
i,
|
||||
rss_getcpu(i));
|
||||
}
|
||||
error = sbuf_finish(sb);
|
||||
sbuf_delete(sb);
|
||||
|
||||
return (error);
|
||||
}
|
||||
SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping,
|
||||
CTLTYPE_STRING | CTLFLAG_RD, NULL, 0,
|
||||
sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping");
|
||||
|
||||
@ -34,97 +34,12 @@
|
||||
|
||||
#include <netinet/in.h> /* in_addr_t */
|
||||
|
||||
/*
|
||||
* Supported RSS hash functions.
|
||||
*/
|
||||
#define RSS_HASH_NAIVE 0x00000001 /* Poor but fast hash. */
|
||||
#define RSS_HASH_TOEPLITZ 0x00000002 /* Required by RSS. */
|
||||
#define RSS_HASH_CRC32 0x00000004 /* Future; some NICs do it. */
|
||||
|
||||
#define RSS_HASH_MASK (RSS_HASH_NAIVE | RSS_HASH_TOEPLITZ)
|
||||
|
||||
/*
|
||||
* Instances of struct inpcbinfo declare an RSS hash type indicating what
|
||||
* header fields are covered.
|
||||
*/
|
||||
#define RSS_HASHFIELDS_NONE 0
|
||||
#define RSS_HASHFIELDS_4TUPLE 1
|
||||
#define RSS_HASHFIELDS_2TUPLE 2
|
||||
|
||||
/*
|
||||
* Define RSS representations of the M_HASHTYPE_* values, representing
|
||||
* which particular bits are supported. The NICs can then use this to
|
||||
* calculate which hash types to enable and which not to enable.
|
||||
*
|
||||
* The fact that these line up with M_HASHTYPE_* is not to be relied
|
||||
* upon.
|
||||
*/
|
||||
#define RSS_HASHTYPE_RSS_IPV4 (1 << 1) /* IPv4 2-tuple */
|
||||
#define RSS_HASHTYPE_RSS_TCP_IPV4 (1 << 2) /* TCPv4 4-tuple */
|
||||
#define RSS_HASHTYPE_RSS_IPV6 (1 << 3) /* IPv6 2-tuple */
|
||||
#define RSS_HASHTYPE_RSS_TCP_IPV6 (1 << 4) /* TCPv6 4-tuple */
|
||||
#define RSS_HASHTYPE_RSS_IPV6_EX (1 << 5) /* IPv6 2-tuple + ext hdrs */
|
||||
#define RSS_HASHTYPE_RSS_TCP_IPV6_EX (1 << 6) /* TCPv6 4-tiple + ext hdrs */
|
||||
#define RSS_HASHTYPE_RSS_UDP_IPV4 (1 << 7) /* IPv4 UDP 4-tuple */
|
||||
#define RSS_HASHTYPE_RSS_UDP_IPV4_EX (1 << 8) /* IPv4 UDP 4-tuple + ext hdrs */
|
||||
#define RSS_HASHTYPE_RSS_UDP_IPV6 (1 << 9) /* IPv6 UDP 4-tuple */
|
||||
#define RSS_HASHTYPE_RSS_UDP_IPV6_EX (1 << 10) /* IPv6 UDP 4-tuple + ext hdrs */
|
||||
|
||||
/*
|
||||
* Compile-time limits on the size of the indirection table.
|
||||
*/
|
||||
#define RSS_MAXBITS 7
|
||||
#define RSS_TABLE_MAXLEN (1 << RSS_MAXBITS)
|
||||
|
||||
/*
|
||||
* Maximum key size used throughout. It's OK for hardware to use only the
|
||||
* first 16 bytes, which is all that's required for IPv4.
|
||||
*/
|
||||
#define RSS_KEYSIZE 40
|
||||
|
||||
/*
|
||||
* For RSS hash methods that do a software hash on an mbuf, the packet
|
||||
* direction (ingress / egress) is required.
|
||||
*
|
||||
* The default direction (INGRESS) is the "receive into the NIC" - ie,
|
||||
* what the hardware is hashing on.
|
||||
*/
|
||||
#define RSS_HASH_PKT_INGRESS 0
|
||||
#define RSS_HASH_PKT_EGRESS 1
|
||||
|
||||
/*
|
||||
* Device driver interfaces to query RSS properties that must be programmed
|
||||
* into hardware.
|
||||
*/
|
||||
u_int rss_getbits(void);
|
||||
u_int rss_getbucket(u_int hash);
|
||||
u_int rss_get_indirection_to_bucket(u_int index);
|
||||
u_int rss_getcpu(u_int bucket);
|
||||
void rss_getkey(uint8_t *key);
|
||||
u_int rss_gethashalgo(void);
|
||||
u_int rss_getnumbuckets(void);
|
||||
u_int rss_getnumcpus(void);
|
||||
u_int rss_gethashconfig(void);
|
||||
|
||||
/*
|
||||
* Network stack interface to generate a hash for a protocol tuple.
|
||||
*/
|
||||
uint32_t rss_hash_ip4_4tuple(struct in_addr src, u_short srcport,
|
||||
struct in_addr dst, u_short dstport);
|
||||
uint32_t rss_hash_ip4_2tuple(struct in_addr src, struct in_addr dst);
|
||||
uint32_t rss_hash_ip6_4tuple(const struct in6_addr *src, u_short srcport,
|
||||
const struct in6_addr *dst, u_short dstport);
|
||||
uint32_t rss_hash_ip6_2tuple(const struct in6_addr *src,
|
||||
const struct in6_addr *dst);
|
||||
|
||||
/*
|
||||
* Network stack interface to query desired CPU affinity of a packet.
|
||||
*/
|
||||
struct mbuf *rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid);
|
||||
u_int rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type);
|
||||
int rss_hash2bucket(uint32_t hash_val, uint32_t hash_type,
|
||||
uint32_t *bucket_id);
|
||||
int rss_m2bucket(struct mbuf *m, uint32_t *bucket_id);
|
||||
|
||||
/*
|
||||
* Functions to calculate a software RSS hash for a given mbuf or
|
||||
|
||||
@ -61,6 +61,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include <net/if_dl.h>
|
||||
#include <net/route.h>
|
||||
#include <net/netisr.h>
|
||||
#include <net/rss_config.h>
|
||||
#include <net/vnet.h>
|
||||
|
||||
#include <netinet/in.h>
|
||||
|
||||
@ -65,6 +65,7 @@ __FBSDID("$FreeBSD$");
|
||||
#ifdef RADIX_MPATH
|
||||
#include <net/radix_mpath.h>
|
||||
#endif
|
||||
#include <net/rss_config.h>
|
||||
#include <net/vnet.h>
|
||||
|
||||
#include <netinet/in.h>
|
||||
|
||||
@ -51,6 +51,7 @@ __FBSDID("$FreeBSD$");
|
||||
|
||||
#include <net/if.h>
|
||||
#include <net/route.h>
|
||||
#include <net/rss_config.h>
|
||||
#include <net/vnet.h>
|
||||
#include <net/netisr.h>
|
||||
|
||||
|
||||
@ -70,6 +70,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include <net/if.h>
|
||||
#include <net/if_var.h>
|
||||
#include <net/route.h>
|
||||
#include <net/rss_config.h>
|
||||
|
||||
#include <netinet/in.h>
|
||||
#include <netinet/in_kdtrace.h>
|
||||
|
||||
@ -37,11 +37,13 @@ __FBSDID("$FreeBSD$");
|
||||
#include <sys/param.h>
|
||||
#include <sys/mbuf.h>
|
||||
|
||||
#include <net/rss_config.h>
|
||||
|
||||
#include <netinet/in.h>
|
||||
#include <netinet/in_pcb.h>
|
||||
#include <netinet/in_rss.h>
|
||||
#ifdef INET6
|
||||
#include <netinet6/in6_pcb.h>
|
||||
#include <netinet6/in6_rss.h>
|
||||
#endif /* INET6 */
|
||||
|
||||
/*
|
||||
|
||||
103
sys/netinet6/in6_rss.c
Normal file
103
sys/netinet6/in6_rss.c
Normal file
@ -0,0 +1,103 @@
|
||||
/*-
|
||||
* Copyright (c) 2010-2011 Juniper Networks, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This software was developed by Robert N. M. Watson under contract
|
||||
* to Juniper Networks, Inc.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
|
||||
__FBSDID("$FreeBSD$");
|
||||
|
||||
#include "opt_inet6.h"
|
||||
#include "opt_pcbgroup.h"
|
||||
|
||||
#ifndef PCBGROUP
|
||||
#error "options RSS depends on options PCBGROUP"
|
||||
#endif
|
||||
|
||||
#include <sys/param.h>
|
||||
#include <sys/mbuf.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/priv.h>
|
||||
#include <sys/kernel.h>
|
||||
#include <sys/smp.h>
|
||||
#include <sys/sysctl.h>
|
||||
#include <sys/sbuf.h>
|
||||
|
||||
#include <net/if.h>
|
||||
#include <net/if_var.h>
|
||||
#include <net/netisr.h>
|
||||
#include <net/rss_config.h>
|
||||
|
||||
#include <netinet/in.h>
|
||||
#include <netinet/in_pcb.h>
|
||||
#include <netinet6/in6_rss.h>
|
||||
#include <netinet/in_var.h>
|
||||
|
||||
/* for software rss hash support */
|
||||
#include <netinet/ip.h>
|
||||
#include <netinet/tcp.h>
|
||||
#include <netinet/udp.h>
|
||||
|
||||
/*
|
||||
* Hash an IPv6 2-tuple.
|
||||
*/
|
||||
uint32_t
|
||||
rss_hash_ip6_2tuple(const struct in6_addr *src, const struct in6_addr *dst)
|
||||
{
|
||||
uint8_t data[sizeof(*src) + sizeof(*dst)];
|
||||
u_int datalen;
|
||||
|
||||
datalen = 0;
|
||||
bcopy(src, &data[datalen], sizeof(*src));
|
||||
datalen += sizeof(*src);
|
||||
bcopy(dst, &data[datalen], sizeof(*dst));
|
||||
datalen += sizeof(*dst);
|
||||
return (rss_hash(datalen, data));
|
||||
}
|
||||
|
||||
/*
|
||||
* Hash an IPv6 4-tuple.
|
||||
*/
|
||||
uint32_t
|
||||
rss_hash_ip6_4tuple(const struct in6_addr *src, u_short srcport,
|
||||
const struct in6_addr *dst, u_short dstport)
|
||||
{
|
||||
uint8_t data[sizeof(*src) + sizeof(*dst) + sizeof(srcport) +
|
||||
sizeof(dstport)];
|
||||
u_int datalen;
|
||||
|
||||
datalen = 0;
|
||||
bcopy(src, &data[datalen], sizeof(*src));
|
||||
datalen += sizeof(*src);
|
||||
bcopy(dst, &data[datalen], sizeof(*dst));
|
||||
datalen += sizeof(*dst);
|
||||
bcopy(&srcport, &data[datalen], sizeof(srcport));
|
||||
datalen += sizeof(srcport);
|
||||
bcopy(&dstport, &data[datalen], sizeof(dstport));
|
||||
datalen += sizeof(dstport);
|
||||
return (rss_hash(datalen, data));
|
||||
}
|
||||
45
sys/netinet6/in6_rss.h
Normal file
45
sys/netinet6/in6_rss.h
Normal file
@ -0,0 +1,45 @@
|
||||
/*-
|
||||
* Copyright (c) 2010-2011 Juniper Networks, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This software was developed by Robert N. M. Watson under contract
|
||||
* to Juniper Networks, Inc.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*
|
||||
* $FreeBSD$
|
||||
*/
|
||||
|
||||
#ifndef _NETINET6_IN6_RSS_H_
|
||||
#define _NETINET6_IN6_RSS_H_
|
||||
|
||||
#include <netinet/in.h> /* in_addr_t */
|
||||
|
||||
/*
|
||||
* Network stack interface to generate a hash for a protocol tuple.
|
||||
*/
|
||||
uint32_t rss_hash_ip6_4tuple(const struct in6_addr *src, u_short srcport,
|
||||
const struct in6_addr *dst, u_short dstport);
|
||||
uint32_t rss_hash_ip6_2tuple(const struct in6_addr *src,
|
||||
const struct in6_addr *dst);
|
||||
|
||||
#endif /* !_NETINET6_IN6_RSS_H_ */
|
||||
@ -91,6 +91,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include <net/netisr.h>
|
||||
#include <net/route.h>
|
||||
#include <net/pfil.h>
|
||||
#include <net/rss_config.h>
|
||||
#include <net/vnet.h>
|
||||
|
||||
#include <netinet/in.h>
|
||||
@ -103,7 +104,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include <netinet/in_pcb.h>
|
||||
#include <netinet/tcp_var.h>
|
||||
#include <netinet6/nd6.h>
|
||||
#include <netinet/in_rss.h>
|
||||
#include <netinet6/in6_rss.h>
|
||||
|
||||
#ifdef IPSEC
|
||||
#include <netipsec/ipsec.h>
|
||||
|
||||
@ -97,6 +97,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include <net/if_var.h>
|
||||
#include <net/if_types.h>
|
||||
#include <net/route.h>
|
||||
#include <net/rss_config.h>
|
||||
|
||||
#include <netinet/in.h>
|
||||
#include <netinet/in_kdtrace.h>
|
||||
@ -112,11 +113,11 @@ __FBSDID("$FreeBSD$");
|
||||
#include <netinet/udp.h>
|
||||
#include <netinet/udp_var.h>
|
||||
#include <netinet/udplite.h>
|
||||
#include <netinet/in_rss.h>
|
||||
|
||||
#include <netinet6/ip6protosw.h>
|
||||
#include <netinet6/ip6_var.h>
|
||||
#include <netinet6/in6_pcb.h>
|
||||
#include <netinet6/in6_rss.h>
|
||||
#include <netinet6/udp6_var.h>
|
||||
#include <netinet6/scope6_var.h>
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user