2005-01-07 02:30:35 +00:00
|
|
|
/*-
|
2017-11-20 19:43:44 +00:00
|
|
|
* SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
|
*
|
1999-11-22 02:45:11 +00:00
|
|
|
* Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
|
|
|
|
|
* All rights reserved.
|
|
|
|
|
*
|
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
|
* are met:
|
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
|
|
|
|
* 3. Neither the name of the project nor the names of its contributors
|
|
|
|
|
* may be used to endorse or promote products derived from this software
|
|
|
|
|
* without specific prior written permission.
|
|
|
|
|
*
|
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
|
|
|
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
|
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
|
|
|
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
|
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
|
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
|
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
|
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
|
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
|
|
|
* SUCH DAMAGE.
|
|
|
|
|
*
|
2007-12-10 16:03:40 +00:00
|
|
|
* $KAME: in6_pcb.h,v 1.13 2001/02/06 09:16:53 itojun Exp $
|
1999-11-22 02:45:11 +00:00
|
|
|
*/
|
|
|
|
|
|
2005-01-07 02:30:35 +00:00
|
|
|
/*-
|
1999-11-22 02:45:11 +00:00
|
|
|
* Copyright (c) 1982, 1986, 1990, 1993
|
|
|
|
|
* The Regents of the University of California. All rights reserved.
|
|
|
|
|
*
|
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
|
* are met:
|
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
2017-02-28 23:42:47 +00:00
|
|
|
* 3. Neither the name of the University nor the names of its contributors
|
1999-11-22 02:45:11 +00:00
|
|
|
* may be used to endorse or promote products derived from this software
|
|
|
|
|
* without specific prior written permission.
|
|
|
|
|
*
|
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
|
|
|
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
|
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
|
|
|
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
|
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
|
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
|
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
|
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
|
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
|
|
|
* SUCH DAMAGE.
|
|
|
|
|
*
|
|
|
|
|
* @(#)in_pcb.h 8.1 (Berkeley) 6/10/93
|
2007-12-10 16:03:40 +00:00
|
|
|
* $FreeBSD$
|
1999-11-22 02:45:11 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#ifndef _NETINET6_IN6_PCB_H_
|
|
|
|
|
#define _NETINET6_IN6_PCB_H_
|
|
|
|
|
|
1999-12-29 04:46:21 +00:00
|
|
|
#ifdef _KERNEL
|
1999-11-22 02:45:11 +00:00
|
|
|
#define satosin6(sa) ((struct sockaddr_in6 *)(sa))
|
|
|
|
|
#define sin6tosa(sin6) ((struct sockaddr *)(sin6))
|
|
|
|
|
#define ifatoia6(ifa) ((struct in6_ifaddr *)(ifa))
|
|
|
|
|
|
2012-10-22 21:49:56 +00:00
|
|
|
void in6_pcbpurgeif0(struct inpcbinfo *, struct ifnet *);
|
|
|
|
|
void in6_losing(struct inpcb *);
|
|
|
|
|
int in6_pcbbind(struct inpcb *, struct sockaddr *, struct ucred *);
|
|
|
|
|
int in6_pcbconnect(struct inpcb *, struct sockaddr *, struct ucred *);
|
|
|
|
|
int in6_pcbconnect_mbuf(struct inpcb *, struct sockaddr *,
|
2020-01-12 17:52:32 +00:00
|
|
|
struct ucred *, struct mbuf *, bool);
|
2012-10-22 21:49:56 +00:00
|
|
|
void in6_pcbdisconnect(struct inpcb *);
|
1999-11-22 02:45:11 +00:00
|
|
|
struct inpcb *
|
2012-10-22 21:49:56 +00:00
|
|
|
in6_pcblookup_local(struct inpcbinfo *,
|
2008-07-10 13:31:11 +00:00
|
|
|
struct in6_addr *, u_short, int,
|
2012-10-22 21:49:56 +00:00
|
|
|
struct ucred *);
|
2020-05-18 22:53:12 +00:00
|
|
|
struct inpcb *
|
|
|
|
|
in6_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
|
|
|
|
|
struct in6_addr *faddr, u_int fport_arg, struct in6_addr *laddr,
|
Filter TCP connections to SO_REUSEPORT_LB listen sockets by NUMA domain
In order to efficiently serve web traffic on a NUMA
machine, one must avoid as many NUMA domain crossings as
possible. With SO_REUSEPORT_LB, a number of workers can share a
listen socket. However, even if a worker sets affinity to a core
or set of cores on a NUMA domain, it will receive connections
associated with all NUMA domains in the system. This will lead to
cross-domain traffic when the server writes to the socket or
calls sendfile(), and memory is allocated on the server's local
NUMA node, but transmitted on the NUMA node associated with the
TCP connection. Similarly, when the server reads from the socket,
he will likely be reading memory allocated on the NUMA domain
associated with the TCP connection.
This change provides a new socket ioctl, TCP_REUSPORT_LB_NUMA. A
server can now tell the kernel to filter traffic so that only
incoming connections associated with the desired NUMA domain are
given to the server. (Of course, in the case where there are no
servers sharing the listen socket on some domain, then as a
fallback, traffic will be hashed as normal to all servers sharing
the listen socket regardless of domain). This allows a server to
deal only with traffic that is local to its NUMA domain, and
avoids cross-domain traffic in most cases.
This patch, and a corresponding small patch to nginx to use
TCP_REUSPORT_LB_NUMA allows us to serve 190Gb/s of kTLS encrypted
https media content from dual-socket Xeons with only 13% (as
measured by pcm.x) cross domain traffic on the memory controller.
Reviewed by: jhb, bz (earlier version), bcr (man page)
Tested by: gonzo
Sponsored by: Netfix
Differential Revision: https://reviews.freebsd.org/D21636
2020-12-19 22:04:46 +00:00
|
|
|
u_int lport_arg, int lookupflags, struct ifnet *ifp, uint8_t);
|
1999-11-22 02:45:11 +00:00
|
|
|
struct inpcb *
|
2012-10-22 21:49:56 +00:00
|
|
|
in6_pcblookup(struct inpcbinfo *, struct in6_addr *,
|
Decompose the current single inpcbinfo lock into two locks:
- The existing ipi_lock continues to protect the global inpcb list and
inpcb counter. This lock is now relegated to a small number of
allocation and free operations, and occasional operations that walk
all connections (including, awkwardly, certain UDP multicast receive
operations -- something to revisit).
- A new ipi_hash_lock protects the two inpcbinfo hash tables for
looking up connections and bound sockets, manipulated using new
INP_HASH_*() macros. This lock, combined with inpcb locks, protects
the 4-tuple address space.
Unlike the current ipi_lock, ipi_hash_lock follows the individual inpcb
connection locks, so may be acquired while manipulating a connection on
which a lock is already held, avoiding the need to acquire the inpcbinfo
lock preemptively when a binding change might later be required. As a
result, however, lookup operations necessarily go through a reference
acquire while holding the lookup lock, later acquiring an inpcb lock --
if required.
A new function in_pcblookup() looks up connections, and accepts flags
indicating how to return the inpcb. Due to lock order changes, callers
no longer need acquire locks before performing a lookup: the lookup
routine will acquire the ipi_hash_lock as needed. In the future, it will
also be able to use alternative lookup and locking strategies
transparently to callers, such as pcbgroup lookup. New lookup flags are,
supplementing the existing INPLOOKUP_WILDCARD flag:
INPLOOKUP_RLOCKPCB - Acquire a read lock on the returned inpcb
INPLOOKUP_WLOCKPCB - Acquire a write lock on the returned inpcb
Callers must pass exactly one of these flags (for the time being).
Some notes:
- All protocols are updated to work within the new regime; especially,
TCP, UDPv4, and UDPv6. pcbinfo ipi_lock acquisitions are largely
eliminated, and global hash lock hold times are dramatically reduced
compared to previous locking.
- The TCP syncache still relies on the pcbinfo lock, something that we
may want to revisit.
- Support for reverting to the FreeBSD 7.x locking strategy in TCP input
is no longer available -- hash lookup locks are now held only very
briefly during inpcb lookup, rather than for potentially extended
periods. However, the pcbinfo ipi_lock will still be acquired if a
connection state might change such that a connection is added or
removed.
- Raw IP sockets continue to use the pcbinfo ipi_lock for protection,
due to maintaining their own hash tables.
- The interface in6_pcblookup_hash_locked() is maintained, which allows
callers to acquire hash locks and perform one or more lookups atomically
with 4-tuple allocation: this is required only for TCPv6, as there is no
in6_pcbconnect_setup(), which there should be.
- UDPv6 locking remains significantly more conservative than UDPv4
locking, which relates to source address selection. This needs
attention, as it likely significantly reduces parallelism in this code
for multithreaded socket use (such as in BIND).
- In the UDPv4 and UDPv6 multicast cases, we need to revisit locking
somewhat, as they relied on ipi_lock to stablise 4-tuple matches, which
is no longer sufficient. A second check once the inpcb lock is held
should do the trick, keeping the general case from requiring the inpcb
lock for every inpcb visited.
- This work reminds us that we need to revisit locking of the v4/v6 flags,
which may be accessed lock-free both before and after this change.
- Right now, a single lock name is used for the pcbhash lock -- this is
undesirable, and probably another argument is required to take care of
this (or a char array name field in the pcbinfo?).
This is not an MFC candidate for 8.x due to its impact on lookup and
locking semantics. It's possible some of these issues could be worked
around with compatibility wrappers, if necessary.
Reviewed by: bz
Sponsored by: Juniper Networks, Inc.
2011-05-30 09:43:55 +00:00
|
|
|
u_int, struct in6_addr *, u_int, int,
|
2012-10-22 21:49:56 +00:00
|
|
|
struct ifnet *);
|
Add _mbuf() variants of various inpcb-related interfaces, including lookup,
hash install, etc. For now, these are arguments are unused, but as we add
RSS support, we will want to use hashes extracted from mbufs, rather than
manually calculated hashes of header fields, due to the expensive of the
software version of Toeplitz (and similar hashes).
Add notes that it would be nice to be able to pass mbufs into lookup
routines in pf(4), optimising firewall lookup in the same way, but the
code structure there doesn't facilitate that currently.
(In principle there is no reason this couldn't be MFCed -- the change
extends rather than modifies the KBI. However, it won't be useful without
other previous possibly less MFCable changes.)
Reviewed by: bz
Sponsored by: Juniper Networks, Inc.
2011-06-04 16:33:06 +00:00
|
|
|
struct inpcb *
|
2012-10-22 21:49:56 +00:00
|
|
|
in6_pcblookup_mbuf(struct inpcbinfo *, struct in6_addr *,
|
Add _mbuf() variants of various inpcb-related interfaces, including lookup,
hash install, etc. For now, these are arguments are unused, but as we add
RSS support, we will want to use hashes extracted from mbufs, rather than
manually calculated hashes of header fields, due to the expensive of the
software version of Toeplitz (and similar hashes).
Add notes that it would be nice to be able to pass mbufs into lookup
routines in pf(4), optimising firewall lookup in the same way, but the
code structure there doesn't facilitate that currently.
(In principle there is no reason this couldn't be MFCed -- the change
extends rather than modifies the KBI. However, it won't be useful without
other previous possibly less MFCable changes.)
Reviewed by: bz
Sponsored by: Juniper Networks, Inc.
2011-06-04 16:33:06 +00:00
|
|
|
u_int, struct in6_addr *, u_int, int,
|
2012-10-22 21:49:56 +00:00
|
|
|
struct ifnet *ifp, struct mbuf *);
|
|
|
|
|
void in6_pcbnotify(struct inpcbinfo *, struct sockaddr *,
|
2004-02-13 14:50:01 +00:00
|
|
|
u_int, const struct sockaddr *, u_int, int, void *,
|
2012-10-22 21:49:56 +00:00
|
|
|
struct inpcb *(*)(struct inpcb *, int));
|
2002-06-14 08:35:21 +00:00
|
|
|
struct inpcb *
|
2012-10-22 21:49:56 +00:00
|
|
|
in6_rtchange(struct inpcb *, int);
|
2002-08-21 11:57:12 +00:00
|
|
|
struct sockaddr *
|
2012-10-22 21:49:56 +00:00
|
|
|
in6_sockaddr(in_port_t port, struct in6_addr *addr_p);
|
2002-08-21 11:57:12 +00:00
|
|
|
struct sockaddr *
|
2012-10-22 21:49:56 +00:00
|
|
|
in6_v4mapsin6_sockaddr(in_port_t port, struct in_addr *addr_p);
|
|
|
|
|
int in6_getpeeraddr(struct socket *so, struct sockaddr **nam);
|
|
|
|
|
int in6_getsockaddr(struct socket *so, struct sockaddr **nam);
|
|
|
|
|
int in6_mapped_sockaddr(struct socket *so, struct sockaddr **nam);
|
|
|
|
|
int in6_mapped_peeraddr(struct socket *so, struct sockaddr **nam);
|
2019-08-02 07:41:36 +00:00
|
|
|
int in6_selecthlim(struct inpcb *, struct ifnet *);
|
2017-02-10 05:58:16 +00:00
|
|
|
int in6_pcbsetport(struct in6_addr *, struct inpcb *, struct ucred *);
|
2017-03-06 04:01:58 +00:00
|
|
|
void init_sin6(struct sockaddr_in6 *sin6, struct mbuf *m, int);
|
1999-12-29 04:46:21 +00:00
|
|
|
#endif /* _KERNEL */
|
1999-11-22 02:45:11 +00:00
|
|
|
|
|
|
|
|
#endif /* !_NETINET6_IN6_PCB_H_ */
|
