d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Filter TCP connections to SO_REUSEPORT_LB listen sockets by NUMA domain

Browse Source 
In order to efficiently serve web traffic on a NUMA
machine, one must avoid as many NUMA domain crossings as
possible. With SO_REUSEPORT_LB, a number of workers can share a
listen socket. However, even if a worker sets affinity to a core
or set of cores on a NUMA domain, it will receive connections
associated with all NUMA domains in the system. This will lead to
cross-domain traffic when the server writes to the socket or
calls sendfile(), and memory is allocated on the server's local
NUMA node, but transmitted on the NUMA node associated with the
TCP connection. Similarly, when the server reads from the socket,
he will likely be reading memory allocated on the NUMA domain
associated with the TCP connection.

This change provides a new socket ioctl, TCP_REUSPORT_LB_NUMA. A
server can now tell the kernel to filter traffic so that only
incoming connections associated with the desired NUMA domain are
given to the server. (Of course, in the case where there are no
servers sharing the listen socket on some domain, then as a
fallback, traffic will be hashed as normal to all servers sharing
the listen socket regardless of domain). This allows a server to
deal only with traffic that is local to its NUMA domain, and
avoids cross-domain traffic in most cases.

This patch, and a corresponding small patch to nginx to use
TCP_REUSPORT_LB_NUMA allows us to serve 190Gb/s of kTLS encrypted
https media content from dual-socket Xeons with only 13% (as
measured by pcm.x) cross domain traffic on the memory controller.

Reviewed by:	jhb, bz (earlier version), bcr (man page)
Tested by: gonzo
Sponsored by:	Netfix
Differential Revision:	https://reviews.freebsd.org/D21636
This commit is contained in:
Andrew Gallatin 2020-12-19 22:04:46 +00:00
parent 02bc3865aa
commit a034518ac8
7 changed files with 150 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
share/man/man4/tcp.4
View File

@ -34,7 +34,7 @@
.\" From: @(#)tcp.4 8.1 (Berkeley) 6/5/93
.\" $FreeBSD$
.\"
.Dd November 25, 2020
.Dd December 19, 2020
.Dt TCP 4
.Os
.Sh NAME
@ -314,6 +314,21 @@ Enable in-kernel TLS for data read from this socket.
See
.Xr ktls 4
for more details.
.It Dv TCP_REUSPORT_LB_NUMA
Changes NUMA affinity filtering for an established TCP listen
socket.
This option takes a single integer argument which specifies
the NUMA domain to filter on for this listen socket.
The argument can also have the follwing special values:
.Bl -tag -width "Dv TCP_REUSPORT_LB_NUMA"
.It Dv TCP_REUSPORT_LB_NUMA_NODOM
Remove NUMA filtering for this listen socket.
.It Dv TCP_REUSPORT_LB_NUMA_CURDOM
Filter traffic associated with the domain where the calling thread is
currently executing.
This is typically used after a process or thread inherits a listen
socket from its parent, and sets its CPU affinity to a particular core.
.El
.El
.Pp
The option level for the

112
sys/netinet/in_pcb.c
View File

@ -75,6 +75,7 @@ __FBSDID("$FreeBSD$");
#endif
#include <vm/uma.h>
#include <vm/vm.h>
#include <net/if.h>
#include <net/if_var.h>
@ -150,7 +151,8 @@ static void in_pcbremlists(struct inpcb *inp);
static struct inpcb *in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
struct in_addr faddr, u_int fport_arg,
struct in_addr laddr, u_int lport_arg,
int lookupflags, struct ifnet *ifp);
int lookupflags, struct ifnet *ifp,
uint8_t numa_domain);
#define RANGECHK(var, min, max) \
if ((var) < (min)) { (var) = (min); } \
@ -248,7 +250,8 @@ SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, set_ok, CTLFLAG_RD,
static struct inpcblbgroup *
in_pcblbgroup_alloc(struct inpcblbgrouphead *hdr, u_char vflag,
uint16_t port, const union in_dependaddr *addr, int size)
uint16_t port, const union in_dependaddr *addr, int size,
uint8_t numa_domain)
{
struct inpcblbgroup *grp;
size_t bytes;
@ -259,6 +262,7 @@ in_pcblbgroup_alloc(struct inpcblbgrouphead *hdr, u_char vflag,
return (NULL);
grp->il_vflag = vflag;
grp->il_lport = port;
grp->il_numa_domain = numa_domain;
grp->il_dependladdr = *addr;
grp->il_inpsiz = size;
CK_LIST_INSERT_HEAD(hdr, grp, il_list);
@ -290,7 +294,8 @@ in_pcblbgroup_resize(struct inpcblbgrouphead *hdr,
int i;
grp = in_pcblbgroup_alloc(hdr, old_grp->il_vflag,
old_grp->il_lport, &old_grp->il_dependladdr, size);
old_grp->il_lport, &old_grp->il_dependladdr, size,
old_grp->il_numa_domain);
if (grp == NULL)
return (NULL);
@ -333,7 +338,7 @@ in_pcblbgroup_reorder(struct inpcblbgrouphead *hdr, struct inpcblbgroup **grpp,
* Add PCB to load balance group for SO_REUSEPORT_LB option.
*/
static int
in_pcbinslbgrouphash(struct inpcb *inp)
in_pcbinslbgrouphash(struct inpcb *inp, uint8_t numa_domain)
{
const static struct timeval interval = { 60, 0 };
static struct timeval lastprint;
@ -369,6 +374,7 @@ in_pcbinslbgrouphash(struct inpcb *inp)
CK_LIST_FOREACH(grp, hdr, il_list) {
if (grp->il_vflag == inp->inp_vflag &&
grp->il_lport == inp->inp_lport &&
grp->il_numa_domain == numa_domain &&
memcmp(&grp->il_dependladdr,
&inp->inp_inc.inc_ie.ie_dependladdr,
sizeof(grp->il_dependladdr)) == 0)
@ -378,7 +384,7 @@ in_pcbinslbgrouphash(struct inpcb *inp)
/* Create new load balance group. */
grp = in_pcblbgroup_alloc(hdr, inp->inp_vflag,
inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
INPCBLBGROUP_SIZMIN);
INPCBLBGROUP_SIZMIN, numa_domain);
if (grp == NULL)
return (ENOBUFS);
} else if (grp->il_inpcnt == grp->il_inpsiz) {
@ -439,6 +445,57 @@ in_pcbremlbgrouphash(struct inpcb *inp)
}
}
int
in_pcblbgroup_numa(struct inpcb *inp, int arg)
{
struct inpcbinfo *pcbinfo;
struct inpcblbgrouphead *hdr;
struct inpcblbgroup *grp;
int err, i;
uint8_t numa_domain;
switch (arg) {
case TCP_REUSPORT_LB_NUMA_NODOM:
numa_domain = M_NODOM;
break;
case TCP_REUSPORT_LB_NUMA_CURDOM:
numa_domain = PCPU_GET(domain);
break;
default:
if (arg < 0 || arg >= vm_ndomains)
return (EINVAL);
numa_domain = arg;
}
err = 0;
pcbinfo = inp->inp_pcbinfo;
INP_WLOCK_ASSERT(inp);
INP_HASH_WLOCK(pcbinfo);
hdr = &pcbinfo->ipi_lbgrouphashbase[
INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];
CK_LIST_FOREACH(grp, hdr, il_list) {
for (i = 0; i < grp->il_inpcnt; ++i) {
if (grp->il_inp[i] != inp)
continue;
if (grp->il_numa_domain == numa_domain) {
goto abort_with_hash_wlock;
}
/* Remove it from the old group. */
in_pcbremlbgrouphash(inp);
/* Add it to the new group based on numa domain. */
in_pcbinslbgrouphash(inp, numa_domain);
goto abort_with_hash_wlock;
}
}
err = ENOENT;
abort_with_hash_wlock:
INP_HASH_WUNLOCK(pcbinfo);
return (err);
}
/*
* Different protocols initialize their inpcbs differently - giving
* different name to the lock. But they all are disposed the same.
@ -731,14 +788,14 @@ in_pcb_lport_dest(struct inpcb *inp, struct sockaddr *lsa, u_short *lportp,
if (lsa->sa_family == AF_INET) {
tmpinp = in_pcblookup_hash_locked(pcbinfo,
faddr, fport, laddr, lport, lookupflags,
NULL);
NULL, M_NODOM);
}
#endif
#ifdef INET6
if (lsa->sa_family == AF_INET6) {
tmpinp = in6_pcblookup_hash_locked(pcbinfo,
faddr6, fport, laddr6, lport, lookupflags,
NULL);
NULL, M_NODOM);
}
#endif
} else {
@ -1399,9 +1456,10 @@ in_pcbconnect_setup(struct inpcb *inp, struct sockaddr *nam,
if (error)
return (error);
}
if (lport != 0) {
oinp = in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr,
fport, laddr, lport, 0, NULL);
fport, laddr, lport, 0, NULL, M_NODOM);
if (oinp != NULL) {
if (oinpp != NULL)
*oinpp = oinp;
@ -2019,9 +2077,9 @@ in_pcblookup_local(struct inpcbinfo *pcbinfo, struct in_addr laddr,
static struct inpcb *
in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
const struct in_addr *laddr, uint16_t lport, const struct in_addr *faddr,
uint16_t fport, int lookupflags)
uint16_t fport, int lookupflags, int numa_domain)
{
struct inpcb *local_wild;
struct inpcb *local_wild, *numa_wild;
const struct inpcblbgrouphead *hdr;
struct inpcblbgroup *grp;
uint32_t idx;
@ -2041,6 +2099,7 @@ in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
* - Load balanced group does not contain IPv4 mapped INET6 wild sockets
*/
local_wild = NULL;
numa_wild = NULL;
CK_LIST_FOREACH(grp, hdr, il_list) {
#ifdef INET6
if (!(grp->il_vflag & INP_IPV4))
@ -2051,12 +2110,24 @@ in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
idx = INP_PCBLBGROUP_PKTHASH(faddr->s_addr, lport, fport) %
grp->il_inpcnt;
if (grp->il_laddr.s_addr == laddr->s_addr)
return (grp->il_inp[idx]);
if (grp->il_laddr.s_addr == laddr->s_addr) {
if (numa_domain == M_NODOM ||
grp->il_numa_domain == numa_domain) {
return (grp->il_inp[idx]);
} else {
numa_wild = grp->il_inp[idx];
}
}
if (grp->il_laddr.s_addr == INADDR_ANY &&
(lookupflags & INPLOOKUP_WILDCARD) != 0)
(lookupflags & INPLOOKUP_WILDCARD) != 0 &&
(local_wild == NULL || numa_domain == M_NODOM ||
grp->il_numa_domain == numa_domain)) {
local_wild = grp->il_inp[idx];
}
}
if (numa_wild != NULL)
return (numa_wild);
return (local_wild);
}
@ -2303,7 +2374,7 @@ in_pcblookup_group(struct inpcbinfo *pcbinfo, struct inpcbgroup *pcbgroup,
static struct inpcb *
in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,
u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
struct ifnet *ifp)
struct ifnet *ifp, uint8_t numa_domain)
{
struct inpcbhead *head;
struct inpcb *inp, *tmpinp;
@ -2348,7 +2419,7 @@ in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,
*/
if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
inp = in_pcblookup_lbgroup(pcbinfo, &laddr, lport, &faddr,
fport, lookupflags);
fport, lookupflags, numa_domain);
if (inp != NULL)
return (inp);
}
@ -2435,12 +2506,13 @@ in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,
static struct inpcb *
in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
struct ifnet *ifp)
struct ifnet *ifp, uint8_t numa_domain)
{
struct inpcb *inp;
inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
(lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp);
(lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp,
numa_domain);
if (inp != NULL) {
if (lookupflags & INPLOOKUP_WLOCKPCB) {
INP_WLOCK(inp);
@ -2507,7 +2579,7 @@ in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport,
}
#endif
return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
lookupflags, ifp));
lookupflags, ifp, M_NODOM));
}
struct inpcb *
@ -2549,7 +2621,7 @@ in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr,
}
#endif
return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
lookupflags, ifp));
lookupflags, ifp, m->m_pkthdr.numa_domain));
}
#endif /* INET */
@ -2591,7 +2663,7 @@ in_pcbinshash_internal(struct inpcb *inp, struct mbuf *m)
*/
so_options = inp_so_options(inp);
if (so_options & SO_REUSEPORT_LB) {
int ret = in_pcbinslbgrouphash(inp);
int ret = in_pcbinslbgrouphash(inp, M_NODOM);
if (ret) {
/* pcb lb group malloc fail (ret=ENOBUFS). */
return (ret);

3
sys/netinet/in_pcb.h
View File

@ -565,7 +565,7 @@ struct inpcblbgroup {
struct epoch_context il_epoch_ctx;
uint16_t il_lport; /* (c) */
u_char il_vflag; /* (c) */
u_char il_pad;
u_int8_t il_numa_domain;
uint32_t il_pad2;
union in_dependaddr il_dependladdr; /* (c) */
#define il_laddr il_dependladdr.id46_addr.ia46_addr4
@ -852,6 +852,7 @@ int in_pcbinshash(struct inpcb *);
int in_pcbinshash_mbuf(struct inpcb *, struct mbuf *);
int in_pcbladdr(struct inpcb *, struct in_addr *, struct in_addr *,
struct ucred *);
int in_pcblbgroup_numa(struct inpcb *, int arg);
struct inpcb *
in_pcblookup_local(struct inpcbinfo *,
struct in_addr, u_short, int, struct ucred *);

4
sys/netinet/tcp.h
View File

@ -196,6 +196,7 @@ struct tcphdr {
#define TCP_PCAP_IN 4096 /* number of input packets to keep */
#define TCP_FUNCTION_BLK 8192 /* Set the tcp function pointers to the specified stack */
/* Options for Rack and BBR */
#define TCP_REUSPORT_LB_NUMA 1026 /* set listen socket numa domain */
#define TCP_RACK_MBUF_QUEUE 1050 /* Do we allow mbuf queuing if supported */
#define TCP_RACK_PROP 1051 /* RACK proportional rate reduction (bool) */
#define TCP_RACK_TLP_REDUCE 1052 /* RACK TLP cwnd reduction (bool) */
@ -406,4 +407,7 @@ struct tcp_function_set {
#define VOI_TCP_GPUT_ND 8 /* Goodput normalised delta */
#define VOI_TCP_ACKLEN 9 /* Average ACKed bytes per ACK */
#define TCP_REUSPORT_LB_NUMA_NODOM (-2) /* remove numa binding */
#define TCP_REUSPORT_LB_NUMA_CURDOM (-1) /* bind to current domain */
#endif /* !_NETINET_TCP_H_ */

10
sys/netinet/tcp_usrreq.c
View File

@ -2143,6 +2143,16 @@ tcp_default_ctloutput(struct socket *so, struct sockopt *sopt, struct inpcb *inp
INP_WUNLOCK(inp);
break;
case TCP_REUSPORT_LB_NUMA:
INP_WUNLOCK(inp);
error = sooptcopyin(sopt, &optval, sizeof(optval),
sizeof(optval));
INP_WLOCK_RECHECK(inp);
if (!error)
error = in_pcblbgroup_numa(inp, optval);
INP_WUNLOCK(inp);
break;
#ifdef KERN_TLS
case TCP_TXTLS_ENABLE:
INP_WUNLOCK(inp);

37
sys/netinet6/in6_pcb.c
View File

@ -446,7 +446,7 @@ in6_pcbconnect_mbuf(struct inpcb *inp, struct sockaddr *nam,
sin6->sin6_port,
IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)
? &laddr6.sin6_addr : &inp->in6p_laddr,
inp->inp_lport, 0, NULL) != NULL) {
inp->inp_lport, 0, NULL, M_NODOM) != NULL) {
return (EADDRINUSE);
}
if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) {
@ -903,9 +903,9 @@ in6_rtchange(struct inpcb *inp, int errno __unused)
static struct inpcb *
in6_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
const struct in6_addr *laddr, uint16_t lport, const struct in6_addr *faddr,
uint16_t fport, int lookupflags)
uint16_t fport, int lookupflags, uint8_t numa_domain)
{
struct inpcb *local_wild;
struct inpcb *local_wild, *numa_wild;
const struct inpcblbgrouphead *hdr;
struct inpcblbgroup *grp;
uint32_t idx;
@ -925,6 +925,7 @@ in6_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
* - Load balanced does not contain IPv4 mapped INET6 wild sockets.
*/
local_wild = NULL;
numa_wild = NULL;
CK_LIST_FOREACH(grp, hdr, il_list) {
#ifdef INET
if (!(grp->il_vflag & INP_IPV6))
@ -935,12 +936,23 @@ in6_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
idx = INP_PCBLBGROUP_PKTHASH(INP6_PCBHASHKEY(faddr), lport,
fport) % grp->il_inpcnt;
if (IN6_ARE_ADDR_EQUAL(&grp->il6_laddr, laddr))
return (grp->il_inp[idx]);
if (IN6_ARE_ADDR_EQUAL(&grp->il6_laddr, laddr)) {
if (numa_domain == M_NODOM ||
grp->il_numa_domain == numa_domain) {
return (grp->il_inp[idx]);
}
else
numa_wild = grp->il_inp[idx];
}
if (IN6_IS_ADDR_UNSPECIFIED(&grp->il6_laddr) &&
(lookupflags & INPLOOKUP_WILDCARD) != 0)
(lookupflags & INPLOOKUP_WILDCARD) != 0 &&
(local_wild == NULL || numa_domain == M_NODOM ||
grp->il_numa_domain == numa_domain)) {
local_wild = grp->il_inp[idx];
}
}
if (numa_wild != NULL)
return (numa_wild);
return (local_wild);
}
@ -1151,7 +1163,7 @@ in6_pcblookup_group(struct inpcbinfo *pcbinfo, struct inpcbgroup *pcbgroup,
struct inpcb *
in6_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in6_addr *faddr,
u_int fport_arg, struct in6_addr *laddr, u_int lport_arg,
int lookupflags, struct ifnet *ifp)
int lookupflags, struct ifnet *ifp, uint8_t numa_domain)
{
struct inpcbhead *head;
struct inpcb *inp, *tmpinp;
@ -1195,7 +1207,7 @@ in6_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in6_addr *faddr,
*/
if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
inp = in6_pcblookup_lbgroup(pcbinfo, laddr, lport, faddr,
fport, lookupflags);
fport, lookupflags, numa_domain);
if (inp != NULL)
return (inp);
}
@ -1273,12 +1285,13 @@ in6_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in6_addr *faddr,
static struct inpcb *
in6_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in6_addr *faddr,
u_int fport, struct in6_addr *laddr, u_int lport, int lookupflags,
struct ifnet *ifp)
struct ifnet *ifp, uint8_t numa_domain)
{
struct inpcb *inp;
inp = in6_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
(lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp);
(lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp,
numa_domain);
if (inp != NULL) {
if (lookupflags & INPLOOKUP_WLOCKPCB) {
INP_WLOCK(inp);
@ -1344,7 +1357,7 @@ in6_pcblookup(struct inpcbinfo *pcbinfo, struct in6_addr *faddr, u_int fport,
}
#endif
return (in6_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
lookupflags, ifp));
lookupflags, ifp, M_NODOM));
}
struct inpcb *
@ -1386,7 +1399,7 @@ in6_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in6_addr *faddr,
}
#endif
return (in6_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
lookupflags, ifp));
lookupflags, ifp, m->m_pkthdr.numa_domain));
}
void

2
sys/netinet6/in6_pcb.h
View File

@ -95,7 +95,7 @@ struct inpcb *
struct inpcb *
in6_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
struct in6_addr *faddr, u_int fport_arg, struct in6_addr *laddr,
u_int lport_arg, int lookupflags, struct ifnet *ifp);
u_int lport_arg, int lookupflags, struct ifnet *ifp, uint8_t);
struct inpcb *
in6_pcblookup(struct inpcbinfo *, struct in6_addr *,
u_int, struct in6_addr *, u_int, int,