freebsd-nq/sys/netinet6/in6.c

2598 lines
67 KiB
C
Raw Normal View History

/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the project nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
2007-12-10 16:03:40 +00:00
*
* $KAME: in6.c,v 1.259 2002/01/21 11:37:50 keiichi Exp $
*/
/*-
* Copyright (c) 1982, 1986, 1991, 1993
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)in.c 8.2 (Berkeley) 11/15/93
*/
2007-12-10 16:03:40 +00:00
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_inet.h"
#include "opt_inet6.h"
#include <sys/param.h>
#include <sys/eventhandler.h>
#include <sys/errno.h>
#include <sys/jail.h>
#include <sys/malloc.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/sockio.h>
#include <sys/systm.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/protosw.h>
#include <sys/time.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/rmlock.h>
#include <sys/sysctl.h>
#include <sys/syslog.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/if_types.h>
#include <net/route.h>
Split rtinit() into multiple functions. rtinit[1]() is a function used to add or remove interface address prefix routes, similar to ifa_maintain_loopback_route(). It was intended to be family-agnostic. There is a problem with this approach in reality. 1) IPv6 code does not use it for the ifa routes. There is a separate layer, nd6_prelist_(), providing interface for maintaining interface routes. Its part, responsible for the actual route table interaction, mimics rtenty() code. 2) rtinit tries to combine multiple actions in the same function: constructing proper route attributes and handling iterations over multiple fibs, for the non-zero net.add_addr_allfibs use case. It notably increases the code complexity. 3) dstaddr handling. flags parameter re-uses RTF_ flags. As there is no special flag for p2p connections, host routes and p2p routes are handled in the same way. Additionally, mapping IFA flags to RTF flags makes the interface pretty messy. It make rtinit() to clash with ifa_mainain_loopback_route() for IPV4 interface aliases. 4) rtinit() is the last customer passing non-masked prefixes to rib_action(), complicating rib_action() implementation. 5) rtinit() coupled ifa announce/withdrawal notifications, producing "false positive" ifa messages in certain corner cases. To address all these points, the following has been done: * rtinit() has been split into multiple functions: - Route attribute construction were moved to the per-address-family functions, dealing with (2), (3) and (4). - funnction providing net.add_addr_allfibs handling and route rtsock notificaions is the new routing table inteface. - rtsock ifa notificaion has been moved out as well. resulting set of funcion are only responsible for the actual route notifications. Side effects: * /32 alias does not result in interface routes (/32 route and "host" route) * RTF_PINNED is now set for IPv6 prefixes corresponding to the interface addresses Differential revision: https://reviews.freebsd.org/D28186
2021-01-09 00:19:25 +00:00
#include <net/route/route_ctl.h>
#include <net/route/nhop.h>
#include <net/if_dl.h>
#include <net/vnet.h>
#include <netinet/in.h>
#include <netinet/in_var.h>
#include <net/if_llatbl.h>
#include <netinet/if_ether.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>
#include <netinet/in_pcb.h>
#include <netinet/ip_carp.h>
#include <netinet/ip6.h>
#include <netinet6/ip6_var.h>
#include <netinet6/nd6.h>
#include <netinet6/mld6_var.h>
#include <netinet6/ip6_mroute.h>
#include <netinet6/in6_ifattach.h>
#include <netinet6/scope6_var.h>
#include <netinet6/in6_fib.h>
#include <netinet6/in6_pcb.h>
/*
* struct in6_ifreq and struct ifreq must be type punnable for common members
* of ifr_ifru to allow accessors to be shared.
*/
_Static_assert(offsetof(struct in6_ifreq, ifr_ifru) ==
offsetof(struct ifreq, ifr_ifru),
"struct in6_ifreq and struct ifreq are not type punnable");
VNET_DECLARE(int, icmp6_nodeinfo_oldmcprefix);
#define V_icmp6_nodeinfo_oldmcprefix VNET(icmp6_nodeinfo_oldmcprefix)
/*
* Definitions of some costant IP6 addresses.
*/
const struct in6_addr in6addr_any = IN6ADDR_ANY_INIT;
const struct in6_addr in6addr_loopback = IN6ADDR_LOOPBACK_INIT;
const struct in6_addr in6addr_nodelocal_allnodes =
IN6ADDR_NODELOCAL_ALLNODES_INIT;
const struct in6_addr in6addr_linklocal_allnodes =
IN6ADDR_LINKLOCAL_ALLNODES_INIT;
const struct in6_addr in6addr_linklocal_allrouters =
IN6ADDR_LINKLOCAL_ALLROUTERS_INIT;
Bite the bullet, and make the IPv6 SSM and MLDv2 mega-commit: import from p4 bms_netdev. Summary of changes: * Connect netinet6/in6_mcast.c to build. The legacy KAME KPIs are mostly preserved. * Eliminate now dead code from ip6_output.c. Don't do mbuf bingo, we are not going to do RFC 2292 style CMSG tricks for multicast options as they are not required by any current IPv6 normative reference. * Refactor transports (UDP, raw_ip6) to do own mcast filtering. SCTP, TCP unaffected by this change. * Add ip6_msource, in6_msource structs to in6_var.h. * Hookup mld_ifinfo state to in6_ifextra, allocate from domifattach path. * Eliminate IN6_LOOKUP_MULTI(), it is no longer referenced. Kernel consumers which need this should use in6m_lookup(). * Refactor IPv6 socket group memberships to use a vector (like IPv4). * Update ifmcstat(8) for IPv6 SSM. * Add witness lock order for IN6_MULTI_LOCK. * Move IN6_MULTI_LOCK out of lower ip6_output()/ip6_input() paths. * Introduce IP6STAT_ADD/SUB/INC/DEC as per rwatson's IPv4 cleanup. * Update carp(4) for new IPv6 SSM KPIs. * Virtualize ip6_mrouter socket. Changes mostly localized to IPv6 MROUTING. * Don't do a local group lookup in MROUTING. * Kill unused KAME prototypes in6_purgemkludge(), in6_restoremkludge(). * Preserve KAME DAD timer jitter behaviour in MLDv1 compatibility mode. * Bump __FreeBSD_version to 800084. * Update UPDATING. NOTE WELL: * This code hasn't been tested against real MLDv2 queriers (yet), although the on-wire protocol has been verified in Wireshark. * There are a few unresolved issues in the socket layer APIs to do with scope ID propagation. * There is a LOR present in ip6_output()'s use of in6_setscope() which needs to be resolved. See comments in mld6.c. This is believed to be benign and can't be avoided for the moment without re-introducing an indirect netisr. This work was mostly derived from the IGMPv3 implementation, and has been sponsored by a third party.
2009-04-29 19:19:13 +00:00
const struct in6_addr in6addr_linklocal_allv2routers =
IN6ADDR_LINKLOCAL_ALLV2ROUTERS_INIT;
const struct in6_addr in6mask0 = IN6MASK0;
const struct in6_addr in6mask32 = IN6MASK32;
const struct in6_addr in6mask64 = IN6MASK64;
const struct in6_addr in6mask96 = IN6MASK96;
const struct in6_addr in6mask128 = IN6MASK128;
const struct sockaddr_in6 sa6_any =
{ sizeof(sa6_any), AF_INET6, 0, 0, IN6ADDR_ANY_INIT, 0 };
static int in6_notify_ifa(struct ifnet *, struct in6_ifaddr *,
struct in6_aliasreq *, int);
2008-01-08 19:08:58 +00:00
static void in6_unlink_ifa(struct in6_ifaddr *, struct ifnet *);
static int in6_validate_ifra(struct ifnet *, struct in6_aliasreq *,
struct in6_ifaddr *, int);
static struct in6_ifaddr *in6_alloc_ifa(struct ifnet *,
struct in6_aliasreq *, int flags);
static int in6_update_ifa_internal(struct ifnet *, struct in6_aliasreq *,
struct in6_ifaddr *, int, int);
static int in6_broadcast_ifa(struct ifnet *, struct in6_aliasreq *,
struct in6_ifaddr *, int);
#define ifa2ia6(ifa) ((struct in6_ifaddr *)(ifa))
#define ia62ifa(ia6) (&((ia6)->ia_ifa))
void
in6_newaddrmsg(struct in6_ifaddr *ia, int cmd)
{
struct rt_addrinfo info;
struct ifaddr *ifa;
struct sockaddr_dl gateway;
Constrain IPv6 routes to single FIBs when net.add_addr_allfibs=0 sys/netinet6/icmp6.c Use the interface's FIB for source address selection in ICMPv6 error responses. sys/netinet6/in6.c In in6_newaddrmsg, announce arrival of local addresses on the interface's FIB only. In in6_lltable_rtcheck, use a per-fib ND6 cache instead of a single cache. sys/netinet6/in6_src.c In in6_selectsrc, use the caller's fib instead of the default fib. In in6_selectsrc_socket, remove a superfluous check. sys/netinet6/nd6.c In nd6_lle_event, use the interface's fib for routing socket messages. In nd6_is_new_addr_neighbor, check all FIBs when trying to determine whether an address is a neighbor. Also, simplify the code for point to point interfaces. sys/netinet6/nd6.h sys/netinet6/nd6.c sys/netinet6/nd6_rtr.c Make defrouter_select fib-aware, and make all of its callers pass in the interface fib. sys/netinet6/nd6_nbr.c When inputting a Neighbor Solicitation packet, consider the interface fib instead of the default fib for DAD. Output NS and Neighbor Advertisement packets on the correct fib. sys/netinet6/nd6_rtr.c Allow installing the same host route on different interfaces in different FIBs. If rt_add_addr_allfibs=0, only install or delete the prefix route on the interface fib. tests/sys/netinet/fibs_test.sh Clear some expected failures, but add a skip for the newly revealed BUG217871. PR: 196361 Submitted by: Erick Turnquist <jhujhiti@adjectivism.org> Reported by: Jason Healy <jhealy@logn.net> Reviewed by: asomers MFC after: 3 weeks Sponsored by: Spectra Logic Corp Differential Revision: https://reviews.freebsd.org/D9451
2017-03-17 16:50:37 +00:00
int fibnum;
ifa = &ia->ia_ifa;
/*
* Prepare info data for the host route.
* This code mimics one from ifa_maintain_loopback_route().
*/
bzero(&info, sizeof(struct rt_addrinfo));
info.rti_flags = ifa->ifa_flags | RTF_HOST | RTF_STATIC | RTF_PINNED;
info.rti_info[RTAX_DST] = ifa->ifa_addr;
info.rti_info[RTAX_GATEWAY] = (struct sockaddr *)&gateway;
link_init_sdl(ifa->ifa_ifp, (struct sockaddr *)&gateway, ifa->ifa_ifp->if_type);
if (cmd != RTM_DELETE)
info.rti_ifp = V_loif;
fibnum = ia62ifa(ia)->ifa_ifp->if_fib;
if (cmd == RTM_ADD) {
rt_addrmsg(cmd, &ia->ia_ifa, fibnum);
rt_routemsg_info(cmd, &info, fibnum);
} else if (cmd == RTM_DELETE) {
rt_routemsg_info(cmd, &info, fibnum);
rt_addrmsg(cmd, &ia->ia_ifa, fibnum);
}
}
int
in6_mask2len(struct in6_addr *mask, u_char *lim0)
{
int x = 0, y;
u_char *lim = lim0, *p;
/* ignore the scope_id part */
if (lim0 == NULL || lim0 - (u_char *)mask > sizeof(*mask))
lim = (u_char *)mask + sizeof(*mask);
for (p = (u_char *)mask; p < lim; x++, p++) {
if (*p != 0xff)
break;
}
y = 0;
if (p < lim) {
for (y = 0; y < 8; y++) {
if ((*p & (0x80 >> y)) == 0)
break;
}
}
/*
* when the limit pointer is given, do a stricter check on the
* remaining bits.
*/
if (p < lim) {
if (y != 0 && (*p & (0x00ff >> y)) != 0)
return (-1);
for (p = p + 1; p < lim; p++)
if (*p != 0)
return (-1);
}
return x * 8 + y;
}
#ifdef COMPAT_FREEBSD32
struct in6_ndifreq32 {
char ifname[IFNAMSIZ];
uint32_t ifindex;
};
#define SIOCGDEFIFACE32_IN6 _IOWR('i', 86, struct in6_ndifreq32)
#endif
int
in6_control(struct socket *so, u_long cmd, caddr_t data,
struct ifnet *ifp, struct thread *td)
{
struct in6_ifreq *ifr = (struct in6_ifreq *)data;
struct in6_ifaddr *ia = NULL;
struct in6_aliasreq *ifra = (struct in6_aliasreq *)data;
struct sockaddr_in6 *sa6;
int carp_attached = 0;
int error;
u_long ocmd = cmd;
/*
* Compat to make pre-10.x ifconfig(8) operable.
*/
if (cmd == OSIOCAIFADDR_IN6)
cmd = SIOCAIFADDR_IN6;
switch (cmd) {
case SIOCGETSGCNT_IN6:
case SIOCGETMIFCNT_IN6:
2012-08-01 09:00:26 +00:00
/*
* XXX mrt_ioctl has a 3rd, unused, FIB argument in route.c.
* We cannot see how that would be needed, so do not adjust the
* KPI blindly; more likely should clean up the IPv4 variant.
*/
return (mrt6_ioctl ? mrt6_ioctl(cmd, data) : EOPNOTSUPP);
}
switch (cmd) {
case SIOCAADDRCTL_POLICY:
case SIOCDADDRCTL_POLICY:
if (td != NULL) {
error = priv_check(td, PRIV_NETINET_ADDRCTRL6);
if (error)
return (error);
}
return (in6_src_ioctl(cmd, data));
}
if (ifp == NULL)
return (EOPNOTSUPP);
switch (cmd) {
case SIOCSNDFLUSH_IN6:
case SIOCSPFXFLUSH_IN6:
case SIOCSRTRFLUSH_IN6:
case SIOCSDEFIFACE_IN6:
case SIOCSIFINFO_FLAGS:
case SIOCSIFINFO_IN6:
if (td != NULL) {
error = priv_check(td, PRIV_NETINET_ND6);
if (error)
return (error);
}
/* FALLTHROUGH */
case OSIOCGIFINFO_IN6:
case SIOCGIFINFO_IN6:
case SIOCGNBRINFO_IN6:
case SIOCGDEFIFACE_IN6:
return (nd6_ioctl(cmd, data, ifp));
#ifdef COMPAT_FREEBSD32
case SIOCGDEFIFACE32_IN6:
{
struct in6_ndifreq ndif;
struct in6_ndifreq32 *ndif32;
error = nd6_ioctl(SIOCGDEFIFACE_IN6, (caddr_t)&ndif,
ifp);
if (error)
return (error);
ndif32 = (struct in6_ndifreq32 *)data;
ndif32->ifindex = ndif.ifindex;
return (0);
}
#endif
}
switch (cmd) {
case SIOCSIFPREFIX_IN6:
case SIOCDIFPREFIX_IN6:
case SIOCAIFPREFIX_IN6:
case SIOCCIFPREFIX_IN6:
case SIOCSGIFPREFIX_IN6:
case SIOCGIFPREFIX_IN6:
log(LOG_NOTICE,
"prefix ioctls are now invalidated. "
"please use ifconfig.\n");
return (EOPNOTSUPP);
}
switch (cmd) {
case SIOCSSCOPE6:
if (td != NULL) {
error = priv_check(td, PRIV_NETINET_SCOPE6);
if (error)
return (error);
}
/* FALLTHROUGH */
case SIOCGSCOPE6:
case SIOCGSCOPE6DEF:
return (scope6_ioctl(cmd, data, ifp));
}
/*
* Find address for this interface, if it exists.
*
* In netinet code, we have checked ifra_addr in SIOCSIF*ADDR operation
* only, and used the first interface address as the target of other
* operations (without checking ifra_addr). This was because netinet
* code/API assumed at most 1 interface address per interface.
* Since IPv6 allows a node to assign multiple addresses
* on a single interface, we almost always look and check the
* presence of ifra_addr, and reject invalid ones here.
* It also decreases duplicated code among SIOC*_IN6 operations.
*/
switch (cmd) {
case SIOCAIFADDR_IN6:
case SIOCSIFPHYADDR_IN6:
sa6 = &ifra->ifra_addr;
break;
case SIOCSIFADDR_IN6:
case SIOCGIFADDR_IN6:
case SIOCSIFDSTADDR_IN6:
case SIOCSIFNETMASK_IN6:
case SIOCGIFDSTADDR_IN6:
case SIOCGIFNETMASK_IN6:
case SIOCDIFADDR_IN6:
case SIOCGIFPSRCADDR_IN6:
case SIOCGIFPDSTADDR_IN6:
case SIOCGIFAFLAG_IN6:
case SIOCSNDFLUSH_IN6:
case SIOCSPFXFLUSH_IN6:
case SIOCSRTRFLUSH_IN6:
case SIOCGIFALIFETIME_IN6:
case SIOCGIFSTAT_IN6:
case SIOCGIFSTAT_ICMP6:
sa6 = &ifr->ifr_addr;
break;
case SIOCSIFADDR:
case SIOCSIFBRDADDR:
case SIOCSIFDSTADDR:
case SIOCSIFNETMASK:
/*
* Although we should pass any non-INET6 ioctl requests
* down to driver, we filter some legacy INET requests.
* Drivers trust SIOCSIFADDR et al to come from an already
* privileged layer, and do not perform any credentials
* checks or input validation.
*/
return (EINVAL);
default:
sa6 = NULL;
break;
}
if (sa6 && sa6->sin6_family == AF_INET6) {
if (sa6->sin6_scope_id != 0)
error = sa6_embedscope(sa6, 0);
else
error = in6_setscope(&sa6->sin6_addr, ifp, NULL);
if (error != 0)
return (error);
if (td != NULL && (error = prison_check_ip6(td->td_ucred,
&sa6->sin6_addr)) != 0)
return (error);
ia = in6ifa_ifpwithaddr(ifp, &sa6->sin6_addr);
} else
ia = NULL;
switch (cmd) {
case SIOCSIFADDR_IN6:
case SIOCSIFDSTADDR_IN6:
case SIOCSIFNETMASK_IN6:
/*
* Since IPv6 allows a node to assign multiple addresses
* on a single interface, SIOCSIFxxx ioctls are deprecated.
*/
/* we decided to obsolete this command (20000704) */
error = EINVAL;
goto out;
case SIOCDIFADDR_IN6:
/*
* for IPv4, we look for existing in_ifaddr here to allow
* "ifconfig if0 delete" to remove the first IPv4 address on
* the interface. For IPv6, as the spec allows multiple
* interface address from the day one, we consider "remove the
* first one" semantics to be not preferable.
*/
if (ia == NULL) {
error = EADDRNOTAVAIL;
goto out;
}
/* FALLTHROUGH */
case SIOCAIFADDR_IN6:
/*
* We always require users to specify a valid IPv6 address for
* the corresponding operation.
*/
if (ifra->ifra_addr.sin6_family != AF_INET6 ||
ifra->ifra_addr.sin6_len != sizeof(struct sockaddr_in6)) {
error = EAFNOSUPPORT;
goto out;
}
if (td != NULL) {
error = priv_check(td, (cmd == SIOCDIFADDR_IN6) ?
PRIV_NET_DELIFADDR : PRIV_NET_ADDIFADDR);
if (error)
goto out;
}
/* FALLTHROUGH */
case SIOCGIFSTAT_IN6:
case SIOCGIFSTAT_ICMP6:
if (ifp->if_afdata[AF_INET6] == NULL) {
error = EPFNOSUPPORT;
goto out;
}
break;
case SIOCGIFADDR_IN6:
/* This interface is basically deprecated. use SIOCGIFCONF. */
/* FALLTHROUGH */
case SIOCGIFAFLAG_IN6:
case SIOCGIFNETMASK_IN6:
case SIOCGIFDSTADDR_IN6:
case SIOCGIFALIFETIME_IN6:
/* must think again about its semantics */
if (ia == NULL) {
error = EADDRNOTAVAIL;
goto out;
}
break;
}
switch (cmd) {
case SIOCGIFADDR_IN6:
ifr->ifr_addr = ia->ia_addr;
if ((error = sa6_recoverscope(&ifr->ifr_addr)) != 0)
goto out;
break;
case SIOCGIFDSTADDR_IN6:
if ((ifp->if_flags & IFF_POINTOPOINT) == 0) {
error = EINVAL;
goto out;
}
ifr->ifr_dstaddr = ia->ia_dstaddr;
if ((error = sa6_recoverscope(&ifr->ifr_dstaddr)) != 0)
goto out;
break;
case SIOCGIFNETMASK_IN6:
ifr->ifr_addr = ia->ia_prefixmask;
break;
case SIOCGIFAFLAG_IN6:
ifr->ifr_ifru.ifru_flags6 = ia->ia6_flags;
break;
case SIOCGIFSTAT_IN6:
COUNTER_ARRAY_COPY(((struct in6_ifextra *)
ifp->if_afdata[AF_INET6])->in6_ifstat,
&ifr->ifr_ifru.ifru_stat,
sizeof(struct in6_ifstat) / sizeof(uint64_t));
break;
case SIOCGIFSTAT_ICMP6:
COUNTER_ARRAY_COPY(((struct in6_ifextra *)
ifp->if_afdata[AF_INET6])->icmp6_ifstat,
&ifr->ifr_ifru.ifru_icmp6stat,
sizeof(struct icmp6_ifstat) / sizeof(uint64_t));
break;
case SIOCGIFALIFETIME_IN6:
ifr->ifr_ifru.ifru_lifetime = ia->ia6_lifetime;
if (ia->ia6_lifetime.ia6t_vltime != ND6_INFINITE_LIFETIME) {
time_t maxexpire;
struct in6_addrlifetime *retlt =
&ifr->ifr_ifru.ifru_lifetime;
/*
* XXX: adjust expiration time assuming time_t is
* signed.
*/
maxexpire = (-1) &
~((time_t)1 << ((sizeof(maxexpire) * 8) - 1));
if (ia->ia6_lifetime.ia6t_vltime <
maxexpire - ia->ia6_updatetime) {
retlt->ia6t_expire = ia->ia6_updatetime +
ia->ia6_lifetime.ia6t_vltime;
} else
retlt->ia6t_expire = maxexpire;
}
if (ia->ia6_lifetime.ia6t_pltime != ND6_INFINITE_LIFETIME) {
time_t maxexpire;
struct in6_addrlifetime *retlt =
&ifr->ifr_ifru.ifru_lifetime;
/*
* XXX: adjust expiration time assuming time_t is
* signed.
*/
maxexpire = (-1) &
~((time_t)1 << ((sizeof(maxexpire) * 8) - 1));
if (ia->ia6_lifetime.ia6t_pltime <
maxexpire - ia->ia6_updatetime) {
retlt->ia6t_preferred = ia->ia6_updatetime +
ia->ia6_lifetime.ia6t_pltime;
} else
retlt->ia6t_preferred = maxexpire;
}
break;
case SIOCAIFADDR_IN6:
{
struct nd_prefixctl pr0;
struct nd_prefix *pr;
/*
* first, make or update the interface address structure,
* and link it to the list.
*/
if ((error = in6_update_ifa(ifp, ifra, ia, 0)) != 0)
goto out;
if (ia != NULL) {
if (ia->ia_ifa.ifa_carp)
(*carp_detach_p)(&ia->ia_ifa, true);
ifa_free(&ia->ia_ifa);
}
if ((ia = in6ifa_ifpwithaddr(ifp, &ifra->ifra_addr.sin6_addr))
== NULL) {
/*
* this can happen when the user specify the 0 valid
* lifetime.
*/
break;
}
if (cmd == ocmd && ifra->ifra_vhid > 0) {
if (carp_attach_p != NULL)
error = (*carp_attach_p)(&ia->ia_ifa,
ifra->ifra_vhid);
else
error = EPROTONOSUPPORT;
if (error)
goto out;
else
carp_attached = 1;
}
/*
* then, make the prefix on-link on the interface.
* XXX: we'd rather create the prefix before the address, but
* we need at least one address to install the corresponding
* interface route, so we configure the address first.
*/
/*
* convert mask to prefix length (prefixmask has already
* been validated in in6_update_ifa().
*/
bzero(&pr0, sizeof(pr0));
pr0.ndpr_ifp = ifp;
pr0.ndpr_plen = in6_mask2len(&ifra->ifra_prefixmask.sin6_addr,
NULL);
if (pr0.ndpr_plen == 128) {
/* we don't need to install a host route. */
goto aifaddr_out;
}
pr0.ndpr_prefix = ifra->ifra_addr;
/* apply the mask for safety. */
IN6_MASK_ADDR(&pr0.ndpr_prefix.sin6_addr,
&ifra->ifra_prefixmask.sin6_addr);
/*
* XXX: since we don't have an API to set prefix (not address)
* lifetimes, we just use the same lifetimes as addresses.
* The (temporarily) installed lifetimes can be overridden by
* later advertised RAs (when accept_rtadv is non 0), which is
* an intended behavior.
*/
pr0.ndpr_raf_onlink = 1; /* should be configurable? */
pr0.ndpr_raf_auto =
((ifra->ifra_flags & IN6_IFF_AUTOCONF) != 0);
pr0.ndpr_vltime = ifra->ifra_lifetime.ia6t_vltime;
pr0.ndpr_pltime = ifra->ifra_lifetime.ia6t_pltime;
/* add the prefix if not yet. */
if ((pr = nd6_prefix_lookup(&pr0)) == NULL) {
/*
* nd6_prelist_add will install the corresponding
* interface route.
*/
if ((error = nd6_prelist_add(&pr0, NULL, &pr)) != 0) {
if (carp_attached)
(*carp_detach_p)(&ia->ia_ifa, false);
goto out;
}
}
/* relate the address to the prefix */
if (ia->ia6_ndpr == NULL) {
ia->ia6_ndpr = pr;
pr->ndpr_addrcnt++;
/*
* If this is the first autoconf address from the
* prefix, create a temporary address as well
* (when required).
*/
if ((ia->ia6_flags & IN6_IFF_AUTOCONF) &&
V_ip6_use_tempaddr && pr->ndpr_addrcnt == 1) {
int e;
if ((e = in6_tmpifadd(ia, 1, 0)) != 0) {
log(LOG_NOTICE, "in6_control: failed "
"to create a temporary address, "
"errno=%d\n", e);
}
}
}
nd6_prefix_rele(pr);
/*
* this might affect the status of autoconfigured addresses,
* that is, this address might make other addresses detached.
*/
pfxlist_onlink_check();
aifaddr_out:
/*
* Try to clear the flag when a new IPv6 address is added
* onto an IFDISABLED interface and it succeeds.
*/
if (ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED) {
struct in6_ndireq nd;
memset(&nd, 0, sizeof(nd));
nd.ndi.flags = ND_IFINFO(ifp)->flags;
nd.ndi.flags &= ~ND6_IFF_IFDISABLED;
if (nd6_ioctl(SIOCSIFINFO_FLAGS, (caddr_t)&nd, ifp) < 0)
log(LOG_NOTICE, "SIOCAIFADDR_IN6: "
"SIOCSIFINFO_FLAGS for -ifdisabled "
"failed.");
/*
* Ignore failure of clearing the flag intentionally.
* The failure means address duplication was detected.
*/
}
break;
}
case SIOCDIFADDR_IN6:
in6_purgeifaddr(ia);
EVENTHANDLER_INVOKE(ifaddr_event_ext, ifp, &ia->ia_ifa,
IFADDR_EVENT_DEL);
break;
default:
if (ifp->if_ioctl == NULL) {
error = EOPNOTSUPP;
goto out;
}
error = (*ifp->if_ioctl)(ifp, cmd, data);
goto out;
}
error = 0;
out:
if (ia != NULL)
ifa_free(&ia->ia_ifa);
return (error);
}
static struct in6_multi_mship *
in6_joingroup_legacy(struct ifnet *ifp, const struct in6_addr *mcaddr,
int *errorp, int delay)
{
struct in6_multi_mship *imm;
int error;
imm = malloc(sizeof(*imm), M_IP6MADDR, M_NOWAIT);
if (imm == NULL) {
*errorp = ENOBUFS;
return (NULL);
}
delay = (delay * PR_FASTHZ) / hz;
error = in6_joingroup(ifp, mcaddr, NULL, &imm->i6mm_maddr, delay);
if (error) {
*errorp = error;
free(imm, M_IP6MADDR);
return (NULL);
}
return (imm);
}
/*
* Join necessary multicast groups. Factored out from in6_update_ifa().
* This entire work should only be done once, for the default FIB.
*/
static int
in6_update_ifa_join_mc(struct ifnet *ifp, struct in6_aliasreq *ifra,
struct in6_ifaddr *ia, int flags, struct in6_multi **in6m_sol)
{
char ip6buf[INET6_ADDRSTRLEN];
struct in6_addr mltaddr;
struct in6_multi_mship *imm;
int delay, error;
KASSERT(in6m_sol != NULL, ("%s: in6m_sol is NULL", __func__));
/* Join solicited multicast addr for new host id. */
bzero(&mltaddr, sizeof(struct in6_addr));
mltaddr.s6_addr32[0] = IPV6_ADDR_INT32_MLL;
mltaddr.s6_addr32[2] = htonl(1);
mltaddr.s6_addr32[3] = ifra->ifra_addr.sin6_addr.s6_addr32[3];
mltaddr.s6_addr8[12] = 0xff;
if ((error = in6_setscope(&mltaddr, ifp, NULL)) != 0) {
/* XXX: should not happen */
log(LOG_ERR, "%s: in6_setscope failed\n", __func__);
goto cleanup;
}
delay = error = 0;
if ((flags & IN6_IFAUPDATE_DADDELAY)) {
/*
* We need a random delay for DAD on the address being
* configured. It also means delaying transmission of the
* corresponding MLD report to avoid report collision.
* [RFC 4861, Section 6.3.7]
*/
delay = arc4random() % (MAX_RTR_SOLICITATION_DELAY * hz);
}
imm = in6_joingroup_legacy(ifp, &mltaddr, &error, delay);
if (imm == NULL) {
nd6log((LOG_WARNING, "%s: in6_joingroup failed for %s on %s "
"(errno=%d)\n", __func__, ip6_sprintf(ip6buf, &mltaddr),
if_name(ifp), error));
goto cleanup;
}
LIST_INSERT_HEAD(&ia->ia6_memberships, imm, i6mm_chain);
*in6m_sol = imm->i6mm_maddr;
/*
* Join link-local all-nodes address.
*/
mltaddr = in6addr_linklocal_allnodes;
if ((error = in6_setscope(&mltaddr, ifp, NULL)) != 0)
goto cleanup; /* XXX: should not fail */
imm = in6_joingroup_legacy(ifp, &mltaddr, &error, 0);
if (imm == NULL) {
nd6log((LOG_WARNING, "%s: in6_joingroup failed for %s on %s "
"(errno=%d)\n", __func__, ip6_sprintf(ip6buf, &mltaddr),
if_name(ifp), error));
goto cleanup;
}
LIST_INSERT_HEAD(&ia->ia6_memberships, imm, i6mm_chain);
/*
* Join node information group address.
*/
delay = 0;
if ((flags & IN6_IFAUPDATE_DADDELAY)) {
/*
* The spec does not say anything about delay for this group,
* but the same logic should apply.
*/
delay = arc4random() % (MAX_RTR_SOLICITATION_DELAY * hz);
}
if (in6_nigroup(ifp, NULL, -1, &mltaddr) == 0) {
/* XXX jinmei */
imm = in6_joingroup_legacy(ifp, &mltaddr, &error, delay);
if (imm == NULL)
nd6log((LOG_WARNING,
"%s: in6_joingroup failed for %s on %s "
"(errno=%d)\n", __func__, ip6_sprintf(ip6buf,
&mltaddr), if_name(ifp), error));
/* XXX not very fatal, go on... */
else
LIST_INSERT_HEAD(&ia->ia6_memberships, imm, i6mm_chain);
}
if (V_icmp6_nodeinfo_oldmcprefix &&
in6_nigroup_oldmcprefix(ifp, NULL, -1, &mltaddr) == 0) {
imm = in6_joingroup_legacy(ifp, &mltaddr, &error, delay);
if (imm == NULL)
nd6log((LOG_WARNING,
"%s: in6_joingroup failed for %s on %s "
"(errno=%d)\n", __func__, ip6_sprintf(ip6buf,
&mltaddr), if_name(ifp), error));
/* XXX not very fatal, go on... */
else
LIST_INSERT_HEAD(&ia->ia6_memberships, imm, i6mm_chain);
}
/*
* Join interface-local all-nodes address.
* (ff01::1%ifN, and ff01::%ifN/32)
*/
mltaddr = in6addr_nodelocal_allnodes;
if ((error = in6_setscope(&mltaddr, ifp, NULL)) != 0)
goto cleanup; /* XXX: should not fail */
imm = in6_joingroup_legacy(ifp, &mltaddr, &error, 0);
if (imm == NULL) {
nd6log((LOG_WARNING, "%s: in6_joingroup failed for %s on %s "
"(errno=%d)\n", __func__, ip6_sprintf(ip6buf,
&mltaddr), if_name(ifp), error));
goto cleanup;
}
LIST_INSERT_HEAD(&ia->ia6_memberships, imm, i6mm_chain);
cleanup:
return (error);
}
/*
* Update parameters of an IPv6 interface address.
* If necessary, a new entry is created and linked into address chains.
* This function is separated from in6_control().
*/
int
in6_update_ifa(struct ifnet *ifp, struct in6_aliasreq *ifra,
struct in6_ifaddr *ia, int flags)
{
int error, hostIsNew = 0;
if ((error = in6_validate_ifra(ifp, ifra, ia, flags)) != 0)
return (error);
if (ia == NULL) {
hostIsNew = 1;
if ((ia = in6_alloc_ifa(ifp, ifra, flags)) == NULL)
return (ENOBUFS);
}
error = in6_update_ifa_internal(ifp, ifra, ia, hostIsNew, flags);
if (error != 0) {
if (hostIsNew != 0) {
in6_unlink_ifa(ia, ifp);
ifa_free(&ia->ia_ifa);
}
return (error);
}
if (hostIsNew)
error = in6_broadcast_ifa(ifp, ifra, ia, flags);
return (error);
}
/*
* Fill in basic IPv6 address request info.
*/
void
in6_prepare_ifra(struct in6_aliasreq *ifra, const struct in6_addr *addr,
const struct in6_addr *mask)
{
memset(ifra, 0, sizeof(struct in6_aliasreq));
ifra->ifra_addr.sin6_family = AF_INET6;
ifra->ifra_addr.sin6_len = sizeof(struct sockaddr_in6);
if (addr != NULL)
ifra->ifra_addr.sin6_addr = *addr;
ifra->ifra_prefixmask.sin6_family = AF_INET6;
ifra->ifra_prefixmask.sin6_len = sizeof(struct sockaddr_in6);
if (mask != NULL)
ifra->ifra_prefixmask.sin6_addr = *mask;
}
static int
in6_validate_ifra(struct ifnet *ifp, struct in6_aliasreq *ifra,
struct in6_ifaddr *ia, int flags)
{
int plen = -1;
struct sockaddr_in6 dst6;
struct in6_addrlifetime *lt;
char ip6buf[INET6_ADDRSTRLEN];
/* Validate parameters */
if (ifp == NULL || ifra == NULL) /* this maybe redundant */
return (EINVAL);
/*
* The destination address for a p2p link must have a family
* of AF_UNSPEC or AF_INET6.
*/
if ((ifp->if_flags & IFF_POINTOPOINT) != 0 &&
ifra->ifra_dstaddr.sin6_family != AF_INET6 &&
ifra->ifra_dstaddr.sin6_family != AF_UNSPEC)
return (EAFNOSUPPORT);
/*
* Validate address
*/
if (ifra->ifra_addr.sin6_len != sizeof(struct sockaddr_in6) ||
ifra->ifra_addr.sin6_family != AF_INET6)
return (EINVAL);
/*
* validate ifra_prefixmask. don't check sin6_family, netmask
* does not carry fields other than sin6_len.
*/
if (ifra->ifra_prefixmask.sin6_len > sizeof(struct sockaddr_in6))
return (EINVAL);
/*
* Because the IPv6 address architecture is classless, we require
* users to specify a (non 0) prefix length (mask) for a new address.
* We also require the prefix (when specified) mask is valid, and thus
* reject a non-consecutive mask.
*/
if (ia == NULL && ifra->ifra_prefixmask.sin6_len == 0)
return (EINVAL);
if (ifra->ifra_prefixmask.sin6_len != 0) {
plen = in6_mask2len(&ifra->ifra_prefixmask.sin6_addr,
(u_char *)&ifra->ifra_prefixmask +
ifra->ifra_prefixmask.sin6_len);
if (plen <= 0)
return (EINVAL);
} else {
/*
* In this case, ia must not be NULL. We just use its prefix
* length.
*/
plen = in6_mask2len(&ia->ia_prefixmask.sin6_addr, NULL);
}
/*
* If the destination address on a p2p interface is specified,
* and the address is a scoped one, validate/set the scope
* zone identifier.
*/
dst6 = ifra->ifra_dstaddr;
if ((ifp->if_flags & (IFF_POINTOPOINT|IFF_LOOPBACK)) != 0 &&
(dst6.sin6_family == AF_INET6)) {
struct in6_addr in6_tmp;
u_int32_t zoneid;
in6_tmp = dst6.sin6_addr;
if (in6_setscope(&in6_tmp, ifp, &zoneid))
return (EINVAL); /* XXX: should be impossible */
if (dst6.sin6_scope_id != 0) {
if (dst6.sin6_scope_id != zoneid)
return (EINVAL);
} else /* user omit to specify the ID. */
dst6.sin6_scope_id = zoneid;
/* convert into the internal form */
if (sa6_embedscope(&dst6, 0))
return (EINVAL); /* XXX: should be impossible */
}
/* Modify original ifra_dstaddr to reflect changes */
ifra->ifra_dstaddr = dst6;
/*
* The destination address can be specified only for a p2p or a
* loopback interface. If specified, the corresponding prefix length
* must be 128.
*/
if (ifra->ifra_dstaddr.sin6_family == AF_INET6) {
if ((ifp->if_flags & (IFF_POINTOPOINT|IFF_LOOPBACK)) == 0) {
/* XXX: noisy message */
2003-11-04 14:09:37 +00:00
nd6log((LOG_INFO, "in6_update_ifa: a destination can "
"be specified for a p2p or a loopback IF only\n"));
return (EINVAL);
}
if (plen != 128) {
2003-11-04 14:09:37 +00:00
nd6log((LOG_INFO, "in6_update_ifa: prefixlen should "
"be 128 when dstaddr is specified\n"));
return (EINVAL);
}
}
/* lifetime consistency check */
lt = &ifra->ifra_lifetime;
if (lt->ia6t_pltime > lt->ia6t_vltime)
return (EINVAL);
if (lt->ia6t_vltime == 0) {
/*
* the following log might be noisy, but this is a typical
* configuration mistake or a tool's bug.
*/
2003-11-04 14:09:37 +00:00
nd6log((LOG_INFO,
"in6_update_ifa: valid lifetime is 0 for %s\n",
ip6_sprintf(ip6buf, &ifra->ifra_addr.sin6_addr)));
if (ia == NULL)
return (0); /* there's nothing to do */
}
/* Check prefix mask */
if (ia != NULL && ifra->ifra_prefixmask.sin6_len != 0) {
/*
* We prohibit changing the prefix length of an existing
* address, because
* + such an operation should be rare in IPv6, and
* + the operation would confuse prefix management.
*/
if (ia->ia_prefixmask.sin6_len != 0 &&
in6_mask2len(&ia->ia_prefixmask.sin6_addr, NULL) != plen) {
nd6log((LOG_INFO, "in6_validate_ifa: the prefix length "
"of an existing %s address should not be changed\n",
ip6_sprintf(ip6buf, &ia->ia_addr.sin6_addr)));
return (EINVAL);
}
}
return (0);
}
/*
* Allocate a new ifaddr and link it into chains.
*/
static struct in6_ifaddr *
in6_alloc_ifa(struct ifnet *ifp, struct in6_aliasreq *ifra, int flags)
{
struct in6_ifaddr *ia;
/*
* When in6_alloc_ifa() is called in a process of a received
* RA, it is called under an interrupt context. So, we should
* call malloc with M_NOWAIT.
*/
ia = (struct in6_ifaddr *)ifa_alloc(sizeof(*ia), M_NOWAIT);
if (ia == NULL)
return (NULL);
LIST_INIT(&ia->ia6_memberships);
/* Initialize the address and masks, and put time stamp */
ia->ia_ifa.ifa_addr = (struct sockaddr *)&ia->ia_addr;
ia->ia_addr.sin6_family = AF_INET6;
ia->ia_addr.sin6_len = sizeof(ia->ia_addr);
/* XXX: Can we assign ,sin6_addr and skip the rest? */
ia->ia_addr = ifra->ifra_addr;
ia->ia6_createtime = time_uptime;
if ((ifp->if_flags & (IFF_POINTOPOINT | IFF_LOOPBACK)) != 0) {
/*
* Some functions expect that ifa_dstaddr is not
* NULL for p2p interfaces.
*/
ia->ia_ifa.ifa_dstaddr =
(struct sockaddr *)&ia->ia_dstaddr;
} else {
ia->ia_ifa.ifa_dstaddr = NULL;
}
/* set prefix mask if any */
ia->ia_ifa.ifa_netmask = (struct sockaddr *)&ia->ia_prefixmask;
if (ifra->ifra_prefixmask.sin6_len != 0) {
ia->ia_prefixmask.sin6_family = AF_INET6;
ia->ia_prefixmask.sin6_len = ifra->ifra_prefixmask.sin6_len;
ia->ia_prefixmask.sin6_addr = ifra->ifra_prefixmask.sin6_addr;
}
ia->ia_ifp = ifp;
ifa_ref(&ia->ia_ifa); /* if_addrhead */
IF_ADDR_WLOCK(ifp);
ifnet: Replace if_addr_lock rwlock with epoch + mutex Run on LLNW canaries and tested by pho@ gallatin: Using a 14-core, 28-HTT single socket E5-2697 v3 with a 40GbE MLX5 based ConnectX 4-LX NIC, I see an almost 12% improvement in received packet rate, and a larger improvement in bytes delivered all the way to userspace. When the host receiving 64 streams of netperf -H $DUT -t UDP_STREAM -- -m 1, I see, using nstat -I mce0 1 before the patch: InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 4.98 0.00 4.42 0.00 4235592 33 83.80 4720653 2149771 1235 247.32 4.73 0.00 4.20 0.00 4025260 33 82.99 4724900 2139833 1204 247.32 4.72 0.00 4.20 0.00 4035252 33 82.14 4719162 2132023 1264 247.32 4.71 0.00 4.21 0.00 4073206 33 83.68 4744973 2123317 1347 247.32 4.72 0.00 4.21 0.00 4061118 33 80.82 4713615 2188091 1490 247.32 4.72 0.00 4.21 0.00 4051675 33 85.29 4727399 2109011 1205 247.32 4.73 0.00 4.21 0.00 4039056 33 84.65 4724735 2102603 1053 247.32 After the patch InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 5.43 0.00 4.20 0.00 3313143 33 84.96 5434214 1900162 2656 245.51 5.43 0.00 4.20 0.00 3308527 33 85.24 5439695 1809382 2521 245.51 5.42 0.00 4.19 0.00 3316778 33 87.54 5416028 1805835 2256 245.51 5.42 0.00 4.19 0.00 3317673 33 90.44 5426044 1763056 2332 245.51 5.42 0.00 4.19 0.00 3314839 33 88.11 5435732 1792218 2499 245.52 5.44 0.00 4.19 0.00 3293228 33 91.84 5426301 1668597 2121 245.52 Similarly, netperf reports 230Mb/s before the patch, and 270Mb/s after the patch Reviewed by: gallatin Sponsored by: Limelight Networks Differential Revision: https://reviews.freebsd.org/D15366
2018-05-18 20:13:34 +00:00
CK_STAILQ_INSERT_TAIL(&ifp->if_addrhead, &ia->ia_ifa, ifa_link);
IF_ADDR_WUNLOCK(ifp);
ifa_ref(&ia->ia_ifa); /* in6_ifaddrhead */
IN6_IFADDR_WLOCK();
ifnet: Replace if_addr_lock rwlock with epoch + mutex Run on LLNW canaries and tested by pho@ gallatin: Using a 14-core, 28-HTT single socket E5-2697 v3 with a 40GbE MLX5 based ConnectX 4-LX NIC, I see an almost 12% improvement in received packet rate, and a larger improvement in bytes delivered all the way to userspace. When the host receiving 64 streams of netperf -H $DUT -t UDP_STREAM -- -m 1, I see, using nstat -I mce0 1 before the patch: InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 4.98 0.00 4.42 0.00 4235592 33 83.80 4720653 2149771 1235 247.32 4.73 0.00 4.20 0.00 4025260 33 82.99 4724900 2139833 1204 247.32 4.72 0.00 4.20 0.00 4035252 33 82.14 4719162 2132023 1264 247.32 4.71 0.00 4.21 0.00 4073206 33 83.68 4744973 2123317 1347 247.32 4.72 0.00 4.21 0.00 4061118 33 80.82 4713615 2188091 1490 247.32 4.72 0.00 4.21 0.00 4051675 33 85.29 4727399 2109011 1205 247.32 4.73 0.00 4.21 0.00 4039056 33 84.65 4724735 2102603 1053 247.32 After the patch InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 5.43 0.00 4.20 0.00 3313143 33 84.96 5434214 1900162 2656 245.51 5.43 0.00 4.20 0.00 3308527 33 85.24 5439695 1809382 2521 245.51 5.42 0.00 4.19 0.00 3316778 33 87.54 5416028 1805835 2256 245.51 5.42 0.00 4.19 0.00 3317673 33 90.44 5426044 1763056 2332 245.51 5.42 0.00 4.19 0.00 3314839 33 88.11 5435732 1792218 2499 245.52 5.44 0.00 4.19 0.00 3293228 33 91.84 5426301 1668597 2121 245.52 Similarly, netperf reports 230Mb/s before the patch, and 270Mb/s after the patch Reviewed by: gallatin Sponsored by: Limelight Networks Differential Revision: https://reviews.freebsd.org/D15366
2018-05-18 20:13:34 +00:00
CK_STAILQ_INSERT_TAIL(&V_in6_ifaddrhead, ia, ia_link);
CK_LIST_INSERT_HEAD(IN6ADDR_HASH(&ia->ia_addr.sin6_addr), ia, ia6_hash);
IN6_IFADDR_WUNLOCK();
return (ia);
}
/*
* Update/configure interface address parameters:
*
* 1) Update lifetime
* 2) Update interface metric ad flags
* 3) Notify other subsystems
*/
static int
in6_update_ifa_internal(struct ifnet *ifp, struct in6_aliasreq *ifra,
struct in6_ifaddr *ia, int hostIsNew, int flags)
{
int error;
/* update timestamp */
ia->ia6_updatetime = time_uptime;
/*
* Set lifetimes. We do not refer to ia6t_expire and ia6t_preferred
* to see if the address is deprecated or invalidated, but initialize
* these members for applications.
*/
ia->ia6_lifetime = ifra->ifra_lifetime;
if (ia->ia6_lifetime.ia6t_vltime != ND6_INFINITE_LIFETIME) {
ia->ia6_lifetime.ia6t_expire =
time_uptime + ia->ia6_lifetime.ia6t_vltime;
} else
ia->ia6_lifetime.ia6t_expire = 0;
if (ia->ia6_lifetime.ia6t_pltime != ND6_INFINITE_LIFETIME) {
ia->ia6_lifetime.ia6t_preferred =
time_uptime + ia->ia6_lifetime.ia6t_pltime;
} else
ia->ia6_lifetime.ia6t_preferred = 0;
/*
* backward compatibility - if IN6_IFF_DEPRECATED is set from the
* userland, make it deprecated.
*/
if ((ifra->ifra_flags & IN6_IFF_DEPRECATED) != 0) {
ia->ia6_lifetime.ia6t_pltime = 0;
ia->ia6_lifetime.ia6t_preferred = time_uptime;
}
/*
* configure address flags.
*/
ia->ia6_flags = ifra->ifra_flags;
/*
* Make the address tentative before joining multicast addresses,
* so that corresponding MLD responses would not have a tentative
* source address.
*/
ia->ia6_flags &= ~IN6_IFF_DUPLICATED; /* safety */
/*
* DAD should be performed for an new address or addresses on
* an interface with ND6_IFF_IFDISABLED.
*/
if (in6if_do_dad(ifp) &&
(hostIsNew || (ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED)))
ia->ia6_flags |= IN6_IFF_TENTATIVE;
/* notify other subsystems */
error = in6_notify_ifa(ifp, ia, ifra, hostIsNew);
return (error);
}
/*
* Do link-level ifa job:
* 1) Add lle entry for added address
* 2) Notifies routing socket users about new address
* 3) join appropriate multicast group
* 4) start DAD if enabled
*/
static int
in6_broadcast_ifa(struct ifnet *ifp, struct in6_aliasreq *ifra,
struct in6_ifaddr *ia, int flags)
{
struct in6_multi *in6m_sol;
int error = 0;
/* Add local address to lltable, if necessary (ex. on p2p link). */
if ((error = nd6_add_ifa_lle(ia)) != 0) {
in6_purgeaddr(&ia->ia_ifa);
ifa_free(&ia->ia_ifa);
return (error);
}
/* Join necessary multicast groups. */
in6m_sol = NULL;
if ((ifp->if_flags & IFF_MULTICAST) != 0) {
error = in6_update_ifa_join_mc(ifp, ifra, ia, flags, &in6m_sol);
if (error != 0) {
in6_purgeaddr(&ia->ia_ifa);
ifa_free(&ia->ia_ifa);
return (error);
}
}
/* Perform DAD, if the address is TENTATIVE. */
if ((ia->ia6_flags & IN6_IFF_TENTATIVE)) {
int delay, mindelay, maxdelay;
delay = 0;
if ((flags & IN6_IFAUPDATE_DADDELAY)) {
/*
* We need to impose a delay before sending an NS
* for DAD. Check if we also needed a delay for the
* corresponding MLD message. If we did, the delay
* should be larger than the MLD delay (this could be
* relaxed a bit, but this simple logic is at least
* safe).
Bite the bullet, and make the IPv6 SSM and MLDv2 mega-commit: import from p4 bms_netdev. Summary of changes: * Connect netinet6/in6_mcast.c to build. The legacy KAME KPIs are mostly preserved. * Eliminate now dead code from ip6_output.c. Don't do mbuf bingo, we are not going to do RFC 2292 style CMSG tricks for multicast options as they are not required by any current IPv6 normative reference. * Refactor transports (UDP, raw_ip6) to do own mcast filtering. SCTP, TCP unaffected by this change. * Add ip6_msource, in6_msource structs to in6_var.h. * Hookup mld_ifinfo state to in6_ifextra, allocate from domifattach path. * Eliminate IN6_LOOKUP_MULTI(), it is no longer referenced. Kernel consumers which need this should use in6m_lookup(). * Refactor IPv6 socket group memberships to use a vector (like IPv4). * Update ifmcstat(8) for IPv6 SSM. * Add witness lock order for IN6_MULTI_LOCK. * Move IN6_MULTI_LOCK out of lower ip6_output()/ip6_input() paths. * Introduce IP6STAT_ADD/SUB/INC/DEC as per rwatson's IPv4 cleanup. * Update carp(4) for new IPv6 SSM KPIs. * Virtualize ip6_mrouter socket. Changes mostly localized to IPv6 MROUTING. * Don't do a local group lookup in MROUTING. * Kill unused KAME prototypes in6_purgemkludge(), in6_restoremkludge(). * Preserve KAME DAD timer jitter behaviour in MLDv1 compatibility mode. * Bump __FreeBSD_version to 800084. * Update UPDATING. NOTE WELL: * This code hasn't been tested against real MLDv2 queriers (yet), although the on-wire protocol has been verified in Wireshark. * There are a few unresolved issues in the socket layer APIs to do with scope ID propagation. * There is a LOR present in ip6_output()'s use of in6_setscope() which needs to be resolved. See comments in mld6.c. This is believed to be benign and can't be avoided for the moment without re-introducing an indirect netisr. This work was mostly derived from the IGMPv3 implementation, and has been sponsored by a third party.
2009-04-29 19:19:13 +00:00
* XXX: Break data hiding guidelines and look at
* state for the solicited multicast group.
*/
mindelay = 0;
if (in6m_sol != NULL &&
Bite the bullet, and make the IPv6 SSM and MLDv2 mega-commit: import from p4 bms_netdev. Summary of changes: * Connect netinet6/in6_mcast.c to build. The legacy KAME KPIs are mostly preserved. * Eliminate now dead code from ip6_output.c. Don't do mbuf bingo, we are not going to do RFC 2292 style CMSG tricks for multicast options as they are not required by any current IPv6 normative reference. * Refactor transports (UDP, raw_ip6) to do own mcast filtering. SCTP, TCP unaffected by this change. * Add ip6_msource, in6_msource structs to in6_var.h. * Hookup mld_ifinfo state to in6_ifextra, allocate from domifattach path. * Eliminate IN6_LOOKUP_MULTI(), it is no longer referenced. Kernel consumers which need this should use in6m_lookup(). * Refactor IPv6 socket group memberships to use a vector (like IPv4). * Update ifmcstat(8) for IPv6 SSM. * Add witness lock order for IN6_MULTI_LOCK. * Move IN6_MULTI_LOCK out of lower ip6_output()/ip6_input() paths. * Introduce IP6STAT_ADD/SUB/INC/DEC as per rwatson's IPv4 cleanup. * Update carp(4) for new IPv6 SSM KPIs. * Virtualize ip6_mrouter socket. Changes mostly localized to IPv6 MROUTING. * Don't do a local group lookup in MROUTING. * Kill unused KAME prototypes in6_purgemkludge(), in6_restoremkludge(). * Preserve KAME DAD timer jitter behaviour in MLDv1 compatibility mode. * Bump __FreeBSD_version to 800084. * Update UPDATING. NOTE WELL: * This code hasn't been tested against real MLDv2 queriers (yet), although the on-wire protocol has been verified in Wireshark. * There are a few unresolved issues in the socket layer APIs to do with scope ID propagation. * There is a LOR present in ip6_output()'s use of in6_setscope() which needs to be resolved. See comments in mld6.c. This is believed to be benign and can't be avoided for the moment without re-introducing an indirect netisr. This work was mostly derived from the IGMPv3 implementation, and has been sponsored by a third party.
2009-04-29 19:19:13 +00:00
in6m_sol->in6m_state == MLD_REPORTING_MEMBER) {
mindelay = in6m_sol->in6m_timer;
}
maxdelay = MAX_RTR_SOLICITATION_DELAY * hz;
if (maxdelay - mindelay == 0)
delay = 0;
else {
delay =
(arc4random() % (maxdelay - mindelay)) +
mindelay;
}
}
nd6_dad_start((struct ifaddr *)ia, delay);
}
in6_newaddrmsg(ia, RTM_ADD);
ifa_free(&ia->ia_ifa);
return (error);
}
Split rtinit() into multiple functions. rtinit[1]() is a function used to add or remove interface address prefix routes, similar to ifa_maintain_loopback_route(). It was intended to be family-agnostic. There is a problem with this approach in reality. 1) IPv6 code does not use it for the ifa routes. There is a separate layer, nd6_prelist_(), providing interface for maintaining interface routes. Its part, responsible for the actual route table interaction, mimics rtenty() code. 2) rtinit tries to combine multiple actions in the same function: constructing proper route attributes and handling iterations over multiple fibs, for the non-zero net.add_addr_allfibs use case. It notably increases the code complexity. 3) dstaddr handling. flags parameter re-uses RTF_ flags. As there is no special flag for p2p connections, host routes and p2p routes are handled in the same way. Additionally, mapping IFA flags to RTF flags makes the interface pretty messy. It make rtinit() to clash with ifa_mainain_loopback_route() for IPV4 interface aliases. 4) rtinit() is the last customer passing non-masked prefixes to rib_action(), complicating rib_action() implementation. 5) rtinit() coupled ifa announce/withdrawal notifications, producing "false positive" ifa messages in certain corner cases. To address all these points, the following has been done: * rtinit() has been split into multiple functions: - Route attribute construction were moved to the per-address-family functions, dealing with (2), (3) and (4). - funnction providing net.add_addr_allfibs handling and route rtsock notificaions is the new routing table inteface. - rtsock ifa notificaion has been moved out as well. resulting set of funcion are only responsible for the actual route notifications. Side effects: * /32 alias does not result in interface routes (/32 route and "host" route) * RTF_PINNED is now set for IPv6 prefixes corresponding to the interface addresses Differential revision: https://reviews.freebsd.org/D28186
2021-01-09 00:19:25 +00:00
/*
* Adds or deletes interface route for p2p ifa.
* Returns 0 on success or errno.
*/
static int
in6_handle_dstaddr_rtrequest(int cmd, struct in6_ifaddr *ia)
{
struct epoch_tracker et;
struct ifaddr *ifa = &ia->ia_ifa;
int error;
/* Prepare gateway */
struct sockaddr_dl_short sdl = {
.sdl_family = AF_LINK,
.sdl_len = sizeof(struct sockaddr_dl_short),
.sdl_type = ifa->ifa_ifp->if_type,
.sdl_index = ifa->ifa_ifp->if_index,
};
struct sockaddr_in6 dst = {
.sin6_family = AF_INET6,
.sin6_len = sizeof(struct sockaddr_in6),
.sin6_addr = ia->ia_dstaddr.sin6_addr,
};
struct rt_addrinfo info = {
.rti_ifa = ifa,
.rti_flags = RTF_PINNED | RTF_HOST,
.rti_info = {
[RTAX_DST] = (struct sockaddr *)&dst,
[RTAX_GATEWAY] = (struct sockaddr *)&sdl,
},
};
/* Don't set additional per-gw filters on removal */
NET_EPOCH_ENTER(et);
error = rib_handle_ifaddr_info(ifa->ifa_ifp->if_fib, cmd, &info);
NET_EPOCH_EXIT(et);
return (error);
}
static bool
ifa_is_p2p(struct in6_ifaddr *ia)
{
int plen;
plen = in6_mask2len(&ia->ia_prefixmask.sin6_addr, NULL); /* XXX */
if ((plen == 128) && (ia->ia_dstaddr.sin6_family == AF_INET6) &&
!IN6_ARE_ADDR_EQUAL(&ia->ia_addr.sin6_addr, &ia->ia_dstaddr.sin6_addr))
return (true);
return (false);
}
void
in6_purgeaddr(struct ifaddr *ifa)
{
struct ifnet *ifp = ifa->ifa_ifp;
struct in6_ifaddr *ia = (struct in6_ifaddr *) ifa;
struct in6_multi_mship *imm;
int error;
if (ifa->ifa_carp)
(*carp_detach_p)(ifa, false);
/*
* Remove the loopback route to the interface address.
* The check for the current setting of "nd6_useloopback"
* is not needed.
*/
if (ia->ia_flags & IFA_RTSELF) {
error = ifa_del_loopback_route((struct ifaddr *)ia,
(struct sockaddr *)&ia->ia_addr);
if (error == 0)
ia->ia_flags &= ~IFA_RTSELF;
}
/* stop DAD processing */
nd6_dad_stop(ifa);
/* Leave multicast groups. */
while ((imm = LIST_FIRST(&ia->ia6_memberships)) != NULL) {
LIST_REMOVE(imm, i6mm_chain);
if (imm->i6mm_maddr != NULL)
in6_leavegroup(imm->i6mm_maddr, NULL);
free(imm, M_IP6MADDR);
}
Split rtinit() into multiple functions. rtinit[1]() is a function used to add or remove interface address prefix routes, similar to ifa_maintain_loopback_route(). It was intended to be family-agnostic. There is a problem with this approach in reality. 1) IPv6 code does not use it for the ifa routes. There is a separate layer, nd6_prelist_(), providing interface for maintaining interface routes. Its part, responsible for the actual route table interaction, mimics rtenty() code. 2) rtinit tries to combine multiple actions in the same function: constructing proper route attributes and handling iterations over multiple fibs, for the non-zero net.add_addr_allfibs use case. It notably increases the code complexity. 3) dstaddr handling. flags parameter re-uses RTF_ flags. As there is no special flag for p2p connections, host routes and p2p routes are handled in the same way. Additionally, mapping IFA flags to RTF flags makes the interface pretty messy. It make rtinit() to clash with ifa_mainain_loopback_route() for IPV4 interface aliases. 4) rtinit() is the last customer passing non-masked prefixes to rib_action(), complicating rib_action() implementation. 5) rtinit() coupled ifa announce/withdrawal notifications, producing "false positive" ifa messages in certain corner cases. To address all these points, the following has been done: * rtinit() has been split into multiple functions: - Route attribute construction were moved to the per-address-family functions, dealing with (2), (3) and (4). - funnction providing net.add_addr_allfibs handling and route rtsock notificaions is the new routing table inteface. - rtsock ifa notificaion has been moved out as well. resulting set of funcion are only responsible for the actual route notifications. Side effects: * /32 alias does not result in interface routes (/32 route and "host" route) * RTF_PINNED is now set for IPv6 prefixes corresponding to the interface addresses Differential revision: https://reviews.freebsd.org/D28186
2021-01-09 00:19:25 +00:00
/* Check if we need to remove p2p route */
if ((ia->ia_flags & IFA_ROUTE) && ifa_is_p2p(ia)) {
Split rtinit() into multiple functions. rtinit[1]() is a function used to add or remove interface address prefix routes, similar to ifa_maintain_loopback_route(). It was intended to be family-agnostic. There is a problem with this approach in reality. 1) IPv6 code does not use it for the ifa routes. There is a separate layer, nd6_prelist_(), providing interface for maintaining interface routes. Its part, responsible for the actual route table interaction, mimics rtenty() code. 2) rtinit tries to combine multiple actions in the same function: constructing proper route attributes and handling iterations over multiple fibs, for the non-zero net.add_addr_allfibs use case. It notably increases the code complexity. 3) dstaddr handling. flags parameter re-uses RTF_ flags. As there is no special flag for p2p connections, host routes and p2p routes are handled in the same way. Additionally, mapping IFA flags to RTF flags makes the interface pretty messy. It make rtinit() to clash with ifa_mainain_loopback_route() for IPV4 interface aliases. 4) rtinit() is the last customer passing non-masked prefixes to rib_action(), complicating rib_action() implementation. 5) rtinit() coupled ifa announce/withdrawal notifications, producing "false positive" ifa messages in certain corner cases. To address all these points, the following has been done: * rtinit() has been split into multiple functions: - Route attribute construction were moved to the per-address-family functions, dealing with (2), (3) and (4). - funnction providing net.add_addr_allfibs handling and route rtsock notificaions is the new routing table inteface. - rtsock ifa notificaion has been moved out as well. resulting set of funcion are only responsible for the actual route notifications. Side effects: * /32 alias does not result in interface routes (/32 route and "host" route) * RTF_PINNED is now set for IPv6 prefixes corresponding to the interface addresses Differential revision: https://reviews.freebsd.org/D28186
2021-01-09 00:19:25 +00:00
error = in6_handle_dstaddr_rtrequest(RTM_DELETE, ia);
if (error != 0)
log(LOG_INFO, "%s: err=%d, destination address delete "
"failed\n", __func__, error);
ia->ia_flags &= ~IFA_ROUTE;
}
in6_newaddrmsg(ia, RTM_DELETE);
in6_unlink_ifa(ia, ifp);
}
/*
* Removes @ia from the corresponding interfaces and unlinks corresponding
* prefix if no addresses are using it anymore.
*/
void
in6_purgeifaddr(struct in6_ifaddr *ia)
{
struct nd_prefix *pr;
/*
* If the address being deleted is the only one that owns
* the corresponding prefix, expire the prefix as well.
* XXX: theoretically, we don't have to worry about such
* relationship, since we separate the address management
* and the prefix management. We do this, however, to provide
* as much backward compatibility as possible in terms of
* the ioctl operation.
* Note that in6_purgeaddr() will decrement ndpr_addrcnt.
*/
pr = ia->ia6_ndpr;
in6_purgeaddr(&ia->ia_ifa);
if (pr != NULL && pr->ndpr_addrcnt == 0) {
ND6_WLOCK();
nd6_prefix_unlink(pr, NULL);
ND6_WUNLOCK();
nd6_prefix_del(pr);
}
}
static void
in6_unlink_ifa(struct in6_ifaddr *ia, struct ifnet *ifp)
{
char ip6buf[INET6_ADDRSTRLEN];
int remove_lle;
IF_ADDR_WLOCK(ifp);
ifnet: Replace if_addr_lock rwlock with epoch + mutex Run on LLNW canaries and tested by pho@ gallatin: Using a 14-core, 28-HTT single socket E5-2697 v3 with a 40GbE MLX5 based ConnectX 4-LX NIC, I see an almost 12% improvement in received packet rate, and a larger improvement in bytes delivered all the way to userspace. When the host receiving 64 streams of netperf -H $DUT -t UDP_STREAM -- -m 1, I see, using nstat -I mce0 1 before the patch: InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 4.98 0.00 4.42 0.00 4235592 33 83.80 4720653 2149771 1235 247.32 4.73 0.00 4.20 0.00 4025260 33 82.99 4724900 2139833 1204 247.32 4.72 0.00 4.20 0.00 4035252 33 82.14 4719162 2132023 1264 247.32 4.71 0.00 4.21 0.00 4073206 33 83.68 4744973 2123317 1347 247.32 4.72 0.00 4.21 0.00 4061118 33 80.82 4713615 2188091 1490 247.32 4.72 0.00 4.21 0.00 4051675 33 85.29 4727399 2109011 1205 247.32 4.73 0.00 4.21 0.00 4039056 33 84.65 4724735 2102603 1053 247.32 After the patch InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 5.43 0.00 4.20 0.00 3313143 33 84.96 5434214 1900162 2656 245.51 5.43 0.00 4.20 0.00 3308527 33 85.24 5439695 1809382 2521 245.51 5.42 0.00 4.19 0.00 3316778 33 87.54 5416028 1805835 2256 245.51 5.42 0.00 4.19 0.00 3317673 33 90.44 5426044 1763056 2332 245.51 5.42 0.00 4.19 0.00 3314839 33 88.11 5435732 1792218 2499 245.52 5.44 0.00 4.19 0.00 3293228 33 91.84 5426301 1668597 2121 245.52 Similarly, netperf reports 230Mb/s before the patch, and 270Mb/s after the patch Reviewed by: gallatin Sponsored by: Limelight Networks Differential Revision: https://reviews.freebsd.org/D15366
2018-05-18 20:13:34 +00:00
CK_STAILQ_REMOVE(&ifp->if_addrhead, &ia->ia_ifa, ifaddr, ifa_link);
IF_ADDR_WUNLOCK(ifp);
ifa_free(&ia->ia_ifa); /* if_addrhead */
/*
* Defer the release of what might be the last reference to the
* in6_ifaddr so that it can't be freed before the remainder of the
* cleanup.
*/
IN6_IFADDR_WLOCK();
ifnet: Replace if_addr_lock rwlock with epoch + mutex Run on LLNW canaries and tested by pho@ gallatin: Using a 14-core, 28-HTT single socket E5-2697 v3 with a 40GbE MLX5 based ConnectX 4-LX NIC, I see an almost 12% improvement in received packet rate, and a larger improvement in bytes delivered all the way to userspace. When the host receiving 64 streams of netperf -H $DUT -t UDP_STREAM -- -m 1, I see, using nstat -I mce0 1 before the patch: InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 4.98 0.00 4.42 0.00 4235592 33 83.80 4720653 2149771 1235 247.32 4.73 0.00 4.20 0.00 4025260 33 82.99 4724900 2139833 1204 247.32 4.72 0.00 4.20 0.00 4035252 33 82.14 4719162 2132023 1264 247.32 4.71 0.00 4.21 0.00 4073206 33 83.68 4744973 2123317 1347 247.32 4.72 0.00 4.21 0.00 4061118 33 80.82 4713615 2188091 1490 247.32 4.72 0.00 4.21 0.00 4051675 33 85.29 4727399 2109011 1205 247.32 4.73 0.00 4.21 0.00 4039056 33 84.65 4724735 2102603 1053 247.32 After the patch InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 5.43 0.00 4.20 0.00 3313143 33 84.96 5434214 1900162 2656 245.51 5.43 0.00 4.20 0.00 3308527 33 85.24 5439695 1809382 2521 245.51 5.42 0.00 4.19 0.00 3316778 33 87.54 5416028 1805835 2256 245.51 5.42 0.00 4.19 0.00 3317673 33 90.44 5426044 1763056 2332 245.51 5.42 0.00 4.19 0.00 3314839 33 88.11 5435732 1792218 2499 245.52 5.44 0.00 4.19 0.00 3293228 33 91.84 5426301 1668597 2121 245.52 Similarly, netperf reports 230Mb/s before the patch, and 270Mb/s after the patch Reviewed by: gallatin Sponsored by: Limelight Networks Differential Revision: https://reviews.freebsd.org/D15366
2018-05-18 20:13:34 +00:00
CK_STAILQ_REMOVE(&V_in6_ifaddrhead, ia, in6_ifaddr, ia_link);
CK_LIST_REMOVE(ia, ia6_hash);
IN6_IFADDR_WUNLOCK();
/*
* Release the reference to the base prefix. There should be a
* positive reference.
*/
remove_lle = 0;
if (ia->ia6_ndpr == NULL) {
nd6log((LOG_NOTICE,
"in6_unlink_ifa: autoconf'ed address "
"%s has no prefix\n", ip6_sprintf(ip6buf, IA6_IN6(ia))));
} else {
ia->ia6_ndpr->ndpr_addrcnt--;
/* Do not delete lles within prefix if refcont != 0 */
if (ia->ia6_ndpr->ndpr_addrcnt == 0)
remove_lle = 1;
ia->ia6_ndpr = NULL;
}
nd6_rem_ifa_lle(ia, remove_lle);
/*
* Also, if the address being removed is autoconf'ed, call
* pfxlist_onlink_check() since the release might affect the status of
* other (detached) addresses.
*/
if ((ia->ia6_flags & IN6_IFF_AUTOCONF)) {
pfxlist_onlink_check();
}
ifa_free(&ia->ia_ifa); /* in6_ifaddrhead */
}
/*
* Notifies other subsystems about address change/arrival:
* 1) Notifies device handler on the first IPv6 address assignment
* 2) Handle routing table changes for P2P links and route
* 3) Handle routing table changes for address host route
*/
static int
in6_notify_ifa(struct ifnet *ifp, struct in6_ifaddr *ia,
struct in6_aliasreq *ifra, int hostIsNew)
{
int error = 0, ifacount = 0;
struct ifaddr *ifa;
struct sockaddr_in6 *pdst;
char ip6buf[INET6_ADDRSTRLEN];
/*
* Give the interface a chance to initialize
* if this is its first address,
*/
if (hostIsNew != 0) {
struct epoch_tracker et;
NET_EPOCH_ENTER(et);
ifnet: Replace if_addr_lock rwlock with epoch + mutex Run on LLNW canaries and tested by pho@ gallatin: Using a 14-core, 28-HTT single socket E5-2697 v3 with a 40GbE MLX5 based ConnectX 4-LX NIC, I see an almost 12% improvement in received packet rate, and a larger improvement in bytes delivered all the way to userspace. When the host receiving 64 streams of netperf -H $DUT -t UDP_STREAM -- -m 1, I see, using nstat -I mce0 1 before the patch: InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 4.98 0.00 4.42 0.00 4235592 33 83.80 4720653 2149771 1235 247.32 4.73 0.00 4.20 0.00 4025260 33 82.99 4724900 2139833 1204 247.32 4.72 0.00 4.20 0.00 4035252 33 82.14 4719162 2132023 1264 247.32 4.71 0.00 4.21 0.00 4073206 33 83.68 4744973 2123317 1347 247.32 4.72 0.00 4.21 0.00 4061118 33 80.82 4713615 2188091 1490 247.32 4.72 0.00 4.21 0.00 4051675 33 85.29 4727399 2109011 1205 247.32 4.73 0.00 4.21 0.00 4039056 33 84.65 4724735 2102603 1053 247.32 After the patch InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 5.43 0.00 4.20 0.00 3313143 33 84.96 5434214 1900162 2656 245.51 5.43 0.00 4.20 0.00 3308527 33 85.24 5439695 1809382 2521 245.51 5.42 0.00 4.19 0.00 3316778 33 87.54 5416028 1805835 2256 245.51 5.42 0.00 4.19 0.00 3317673 33 90.44 5426044 1763056 2332 245.51 5.42 0.00 4.19 0.00 3314839 33 88.11 5435732 1792218 2499 245.52 5.44 0.00 4.19 0.00 3293228 33 91.84 5426301 1668597 2121 245.52 Similarly, netperf reports 230Mb/s before the patch, and 270Mb/s after the patch Reviewed by: gallatin Sponsored by: Limelight Networks Differential Revision: https://reviews.freebsd.org/D15366
2018-05-18 20:13:34 +00:00
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family != AF_INET6)
continue;
ifacount++;
}
NET_EPOCH_EXIT(et);
}
if (ifacount <= 1 && ifp->if_ioctl) {
error = (*ifp->if_ioctl)(ifp, SIOCSIFADDR, (caddr_t)ia);
if (error)
goto done;
}
/*
* If a new destination address is specified, scrub the old one and
* install the new destination. Note that the interface must be
* p2p or loopback.
*/
pdst = &ifra->ifra_dstaddr;
if (pdst->sin6_family == AF_INET6 &&
!IN6_ARE_ADDR_EQUAL(&pdst->sin6_addr, &ia->ia_dstaddr.sin6_addr)) {
if ((ia->ia_flags & IFA_ROUTE) != 0 &&
Split rtinit() into multiple functions. rtinit[1]() is a function used to add or remove interface address prefix routes, similar to ifa_maintain_loopback_route(). It was intended to be family-agnostic. There is a problem with this approach in reality. 1) IPv6 code does not use it for the ifa routes. There is a separate layer, nd6_prelist_(), providing interface for maintaining interface routes. Its part, responsible for the actual route table interaction, mimics rtenty() code. 2) rtinit tries to combine multiple actions in the same function: constructing proper route attributes and handling iterations over multiple fibs, for the non-zero net.add_addr_allfibs use case. It notably increases the code complexity. 3) dstaddr handling. flags parameter re-uses RTF_ flags. As there is no special flag for p2p connections, host routes and p2p routes are handled in the same way. Additionally, mapping IFA flags to RTF flags makes the interface pretty messy. It make rtinit() to clash with ifa_mainain_loopback_route() for IPV4 interface aliases. 4) rtinit() is the last customer passing non-masked prefixes to rib_action(), complicating rib_action() implementation. 5) rtinit() coupled ifa announce/withdrawal notifications, producing "false positive" ifa messages in certain corner cases. To address all these points, the following has been done: * rtinit() has been split into multiple functions: - Route attribute construction were moved to the per-address-family functions, dealing with (2), (3) and (4). - funnction providing net.add_addr_allfibs handling and route rtsock notificaions is the new routing table inteface. - rtsock ifa notificaion has been moved out as well. resulting set of funcion are only responsible for the actual route notifications. Side effects: * /32 alias does not result in interface routes (/32 route and "host" route) * RTF_PINNED is now set for IPv6 prefixes corresponding to the interface addresses Differential revision: https://reviews.freebsd.org/D28186
2021-01-09 00:19:25 +00:00
(in6_handle_dstaddr_rtrequest(RTM_DELETE, ia) != 0)) {
nd6log((LOG_ERR, "in6_update_ifa_internal: failed to "
"remove a route to the old destination: %s\n",
ip6_sprintf(ip6buf, &ia->ia_addr.sin6_addr)));
/* proceed anyway... */
} else
ia->ia_flags &= ~IFA_ROUTE;
ia->ia_dstaddr = *pdst;
}
/*
* If a new destination address is specified for a point-to-point
* interface, install a route to the destination as an interface
* direct route.
* XXX: the logic below rejects assigning multiple addresses on a p2p
* interface that share the same destination.
*/
if (!(ia->ia_flags & IFA_ROUTE) && ifa_is_p2p(ia)) {
Split rtinit() into multiple functions. rtinit[1]() is a function used to add or remove interface address prefix routes, similar to ifa_maintain_loopback_route(). It was intended to be family-agnostic. There is a problem with this approach in reality. 1) IPv6 code does not use it for the ifa routes. There is a separate layer, nd6_prelist_(), providing interface for maintaining interface routes. Its part, responsible for the actual route table interaction, mimics rtenty() code. 2) rtinit tries to combine multiple actions in the same function: constructing proper route attributes and handling iterations over multiple fibs, for the non-zero net.add_addr_allfibs use case. It notably increases the code complexity. 3) dstaddr handling. flags parameter re-uses RTF_ flags. As there is no special flag for p2p connections, host routes and p2p routes are handled in the same way. Additionally, mapping IFA flags to RTF flags makes the interface pretty messy. It make rtinit() to clash with ifa_mainain_loopback_route() for IPV4 interface aliases. 4) rtinit() is the last customer passing non-masked prefixes to rib_action(), complicating rib_action() implementation. 5) rtinit() coupled ifa announce/withdrawal notifications, producing "false positive" ifa messages in certain corner cases. To address all these points, the following has been done: * rtinit() has been split into multiple functions: - Route attribute construction were moved to the per-address-family functions, dealing with (2), (3) and (4). - funnction providing net.add_addr_allfibs handling and route rtsock notificaions is the new routing table inteface. - rtsock ifa notificaion has been moved out as well. resulting set of funcion are only responsible for the actual route notifications. Side effects: * /32 alias does not result in interface routes (/32 route and "host" route) * RTF_PINNED is now set for IPv6 prefixes corresponding to the interface addresses Differential revision: https://reviews.freebsd.org/D28186
2021-01-09 00:19:25 +00:00
error = in6_handle_dstaddr_rtrequest(RTM_ADD, ia);
if (error)
goto done;
ia->ia_flags |= IFA_ROUTE;
}
/*
* add a loopback route to self if not exists
*/
if (!(ia->ia_flags & IFA_RTSELF) && V_nd6_useloopback) {
error = ifa_add_loopback_route((struct ifaddr *)ia,
(struct sockaddr *)&ia->ia_addr);
if (error == 0)
ia->ia_flags |= IFA_RTSELF;
}
done:
WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
"Invoking IPv6 network device address event may sleep");
ifa_ref(&ia->ia_ifa);
EVENTHANDLER_INVOKE(ifaddr_event_ext, ifp, &ia->ia_ifa,
IFADDR_EVENT_ADD);
ifa_free(&ia->ia_ifa);
return (error);
}
/*
* Find an IPv6 interface link-local address specific to an interface.
* ifaddr is returned referenced.
*/
struct in6_ifaddr *
in6ifa_ifpforlinklocal(struct ifnet *ifp, int ignoreflags)
{
struct ifaddr *ifa;
Widen NET_EPOCH coverage. When epoch(9) was introduced to network stack, it was basically dropped in place of existing locking, which was mutexes and rwlocks. For the sake of performance mutex covered areas were as small as possible, so became epoch covered areas. However, epoch doesn't introduce any contention, it just delays memory reclaim. So, there is no point to minimise epoch covered areas in sense of performance. Meanwhile entering/exiting epoch also has non-zero CPU usage, so doing this less often is a win. Not the least is also code maintainability. In the new paradigm we can assume that at any stage of processing a packet, we are inside network epoch. This makes coding both input and output path way easier. On output path we already enter epoch quite early - in the ip_output(), in the ip6_output(). This patch does the same for the input path. All ISR processing, network related callouts, other ways of packet injection to the network stack shall be performed in net_epoch. Any leaf function that walks network configuration now asserts epoch. Tricky part is configuration code paths - ioctls, sysctls. They also call into leaf functions, so some need to be changed. This patch would introduce more epoch recursions (see EPOCH_TRACE) than we had before. They will be cleaned up separately, as several of them aren't trivial. Note, that unlike a lock recursion the epoch recursion is safe and just wastes a bit of resources. Reviewed by: gallatin, hselasky, cy, adrian, kristof Differential Revision: https://reviews.freebsd.org/D19111
2019-10-07 22:40:05 +00:00
NET_EPOCH_ASSERT();
ifnet: Replace if_addr_lock rwlock with epoch + mutex Run on LLNW canaries and tested by pho@ gallatin: Using a 14-core, 28-HTT single socket E5-2697 v3 with a 40GbE MLX5 based ConnectX 4-LX NIC, I see an almost 12% improvement in received packet rate, and a larger improvement in bytes delivered all the way to userspace. When the host receiving 64 streams of netperf -H $DUT -t UDP_STREAM -- -m 1, I see, using nstat -I mce0 1 before the patch: InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 4.98 0.00 4.42 0.00 4235592 33 83.80 4720653 2149771 1235 247.32 4.73 0.00 4.20 0.00 4025260 33 82.99 4724900 2139833 1204 247.32 4.72 0.00 4.20 0.00 4035252 33 82.14 4719162 2132023 1264 247.32 4.71 0.00 4.21 0.00 4073206 33 83.68 4744973 2123317 1347 247.32 4.72 0.00 4.21 0.00 4061118 33 80.82 4713615 2188091 1490 247.32 4.72 0.00 4.21 0.00 4051675 33 85.29 4727399 2109011 1205 247.32 4.73 0.00 4.21 0.00 4039056 33 84.65 4724735 2102603 1053 247.32 After the patch InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 5.43 0.00 4.20 0.00 3313143 33 84.96 5434214 1900162 2656 245.51 5.43 0.00 4.20 0.00 3308527 33 85.24 5439695 1809382 2521 245.51 5.42 0.00 4.19 0.00 3316778 33 87.54 5416028 1805835 2256 245.51 5.42 0.00 4.19 0.00 3317673 33 90.44 5426044 1763056 2332 245.51 5.42 0.00 4.19 0.00 3314839 33 88.11 5435732 1792218 2499 245.52 5.44 0.00 4.19 0.00 3293228 33 91.84 5426301 1668597 2121 245.52 Similarly, netperf reports 230Mb/s before the patch, and 270Mb/s after the patch Reviewed by: gallatin Sponsored by: Limelight Networks Differential Revision: https://reviews.freebsd.org/D15366
2018-05-18 20:13:34 +00:00
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family != AF_INET6)
continue;
if (IN6_IS_ADDR_LINKLOCAL(IFA_IN6(ifa))) {
if ((((struct in6_ifaddr *)ifa)->ia6_flags &
ignoreflags) != 0)
continue;
ifa_ref(ifa);
break;
}
}
return ((struct in6_ifaddr *)ifa);
}
/*
* find the interface address corresponding to a given IPv6 address.
* ifaddr is returned referenced if @referenced flag is set.
*/
struct in6_ifaddr *
in6ifa_ifwithaddr(const struct in6_addr *addr, uint32_t zoneid, bool referenced)
{
struct rm_priotracker in6_ifa_tracker;
struct in6_ifaddr *ia;
IN6_IFADDR_RLOCK(&in6_ifa_tracker);
CK_LIST_FOREACH(ia, IN6ADDR_HASH(addr), ia6_hash) {
if (IN6_ARE_ADDR_EQUAL(IA6_IN6(ia), addr)) {
if (zoneid != 0 &&
zoneid != ia->ia_addr.sin6_scope_id)
continue;
if (referenced)
ifa_ref(&ia->ia_ifa);
break;
}
}
IN6_IFADDR_RUNLOCK(&in6_ifa_tracker);
return (ia);
}
/*
* find the internet address corresponding to a given interface and address.
* ifaddr is returned referenced.
*/
struct in6_ifaddr *
in6ifa_ifpwithaddr(struct ifnet *ifp, const struct in6_addr *addr)
{
struct epoch_tracker et;
struct ifaddr *ifa;
NET_EPOCH_ENTER(et);
ifnet: Replace if_addr_lock rwlock with epoch + mutex Run on LLNW canaries and tested by pho@ gallatin: Using a 14-core, 28-HTT single socket E5-2697 v3 with a 40GbE MLX5 based ConnectX 4-LX NIC, I see an almost 12% improvement in received packet rate, and a larger improvement in bytes delivered all the way to userspace. When the host receiving 64 streams of netperf -H $DUT -t UDP_STREAM -- -m 1, I see, using nstat -I mce0 1 before the patch: InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 4.98 0.00 4.42 0.00 4235592 33 83.80 4720653 2149771 1235 247.32 4.73 0.00 4.20 0.00 4025260 33 82.99 4724900 2139833 1204 247.32 4.72 0.00 4.20 0.00 4035252 33 82.14 4719162 2132023 1264 247.32 4.71 0.00 4.21 0.00 4073206 33 83.68 4744973 2123317 1347 247.32 4.72 0.00 4.21 0.00 4061118 33 80.82 4713615 2188091 1490 247.32 4.72 0.00 4.21 0.00 4051675 33 85.29 4727399 2109011 1205 247.32 4.73 0.00 4.21 0.00 4039056 33 84.65 4724735 2102603 1053 247.32 After the patch InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 5.43 0.00 4.20 0.00 3313143 33 84.96 5434214 1900162 2656 245.51 5.43 0.00 4.20 0.00 3308527 33 85.24 5439695 1809382 2521 245.51 5.42 0.00 4.19 0.00 3316778 33 87.54 5416028 1805835 2256 245.51 5.42 0.00 4.19 0.00 3317673 33 90.44 5426044 1763056 2332 245.51 5.42 0.00 4.19 0.00 3314839 33 88.11 5435732 1792218 2499 245.52 5.44 0.00 4.19 0.00 3293228 33 91.84 5426301 1668597 2121 245.52 Similarly, netperf reports 230Mb/s before the patch, and 270Mb/s after the patch Reviewed by: gallatin Sponsored by: Limelight Networks Differential Revision: https://reviews.freebsd.org/D15366
2018-05-18 20:13:34 +00:00
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family != AF_INET6)
continue;
if (IN6_ARE_ADDR_EQUAL(addr, IFA_IN6(ifa))) {
ifa_ref(ifa);
break;
}
}
NET_EPOCH_EXIT(et);
return ((struct in6_ifaddr *)ifa);
}
/*
* Find a link-local scoped address on ifp and return it if any.
*/
struct in6_ifaddr *
in6ifa_llaonifp(struct ifnet *ifp)
{
struct epoch_tracker et;
struct sockaddr_in6 *sin6;
struct ifaddr *ifa;
if (ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED)
return (NULL);
NET_EPOCH_ENTER(et);
ifnet: Replace if_addr_lock rwlock with epoch + mutex Run on LLNW canaries and tested by pho@ gallatin: Using a 14-core, 28-HTT single socket E5-2697 v3 with a 40GbE MLX5 based ConnectX 4-LX NIC, I see an almost 12% improvement in received packet rate, and a larger improvement in bytes delivered all the way to userspace. When the host receiving 64 streams of netperf -H $DUT -t UDP_STREAM -- -m 1, I see, using nstat -I mce0 1 before the patch: InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 4.98 0.00 4.42 0.00 4235592 33 83.80 4720653 2149771 1235 247.32 4.73 0.00 4.20 0.00 4025260 33 82.99 4724900 2139833 1204 247.32 4.72 0.00 4.20 0.00 4035252 33 82.14 4719162 2132023 1264 247.32 4.71 0.00 4.21 0.00 4073206 33 83.68 4744973 2123317 1347 247.32 4.72 0.00 4.21 0.00 4061118 33 80.82 4713615 2188091 1490 247.32 4.72 0.00 4.21 0.00 4051675 33 85.29 4727399 2109011 1205 247.32 4.73 0.00 4.21 0.00 4039056 33 84.65 4724735 2102603 1053 247.32 After the patch InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 5.43 0.00 4.20 0.00 3313143 33 84.96 5434214 1900162 2656 245.51 5.43 0.00 4.20 0.00 3308527 33 85.24 5439695 1809382 2521 245.51 5.42 0.00 4.19 0.00 3316778 33 87.54 5416028 1805835 2256 245.51 5.42 0.00 4.19 0.00 3317673 33 90.44 5426044 1763056 2332 245.51 5.42 0.00 4.19 0.00 3314839 33 88.11 5435732 1792218 2499 245.52 5.44 0.00 4.19 0.00 3293228 33 91.84 5426301 1668597 2121 245.52 Similarly, netperf reports 230Mb/s before the patch, and 270Mb/s after the patch Reviewed by: gallatin Sponsored by: Limelight Networks Differential Revision: https://reviews.freebsd.org/D15366
2018-05-18 20:13:34 +00:00
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family != AF_INET6)
continue;
sin6 = (struct sockaddr_in6 *)ifa->ifa_addr;
if (IN6_IS_SCOPE_LINKLOCAL(&sin6->sin6_addr) ||
IN6_IS_ADDR_MC_INTFACELOCAL(&sin6->sin6_addr) ||
IN6_IS_ADDR_MC_NODELOCAL(&sin6->sin6_addr))
break;
}
NET_EPOCH_EXIT(et);
return ((struct in6_ifaddr *)ifa);
}
/*
* Convert IP6 address to printable (loggable) representation. Caller
* has to make sure that ip6buf is at least INET6_ADDRSTRLEN long.
*/
static char digits[] = "0123456789abcdef";
char *
ip6_sprintf(char *ip6buf, const struct in6_addr *addr)
{
int i, cnt = 0, maxcnt = 0, idx = 0, index = 0;
char *cp;
const u_int16_t *a = (const u_int16_t *)addr;
const u_int8_t *d;
int dcolon = 0, zero = 0;
cp = ip6buf;
for (i = 0; i < 8; i++) {
if (*(a + i) == 0) {
cnt++;
if (cnt == 1)
idx = i;
}
else if (maxcnt < cnt) {
maxcnt = cnt;
index = idx;
cnt = 0;
}
}
if (maxcnt < cnt) {
maxcnt = cnt;
index = idx;
}
for (i = 0; i < 8; i++) {
if (dcolon == 1) {
if (*a == 0) {
if (i == 7)
*cp++ = ':';
a++;
continue;
} else
dcolon = 2;
}
if (*a == 0) {
if (dcolon == 0 && *(a + 1) == 0 && i == index) {
if (i == 0)
*cp++ = ':';
*cp++ = ':';
dcolon = 1;
} else {
*cp++ = '0';
*cp++ = ':';
}
a++;
continue;
}
d = (const u_char *)a;
/* Try to eliminate leading zeros in printout like in :0001. */
zero = 1;
*cp = digits[*d >> 4];
if (*cp != '0') {
zero = 0;
cp++;
}
*cp = digits[*d++ & 0xf];
if (zero == 0 || (*cp != '0')) {
zero = 0;
cp++;
}
*cp = digits[*d >> 4];
if (zero == 0 || (*cp != '0')) {
zero = 0;
cp++;
}
*cp++ = digits[*d & 0xf];
*cp++ = ':';
a++;
}
*--cp = '\0';
return (ip6buf);
}
int
in6_localaddr(struct in6_addr *in6)
{
struct rm_priotracker in6_ifa_tracker;
struct in6_ifaddr *ia;
if (IN6_IS_ADDR_LOOPBACK(in6) || IN6_IS_ADDR_LINKLOCAL(in6))
return 1;
IN6_IFADDR_RLOCK(&in6_ifa_tracker);
ifnet: Replace if_addr_lock rwlock with epoch + mutex Run on LLNW canaries and tested by pho@ gallatin: Using a 14-core, 28-HTT single socket E5-2697 v3 with a 40GbE MLX5 based ConnectX 4-LX NIC, I see an almost 12% improvement in received packet rate, and a larger improvement in bytes delivered all the way to userspace. When the host receiving 64 streams of netperf -H $DUT -t UDP_STREAM -- -m 1, I see, using nstat -I mce0 1 before the patch: InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 4.98 0.00 4.42 0.00 4235592 33 83.80 4720653 2149771 1235 247.32 4.73 0.00 4.20 0.00 4025260 33 82.99 4724900 2139833 1204 247.32 4.72 0.00 4.20 0.00 4035252 33 82.14 4719162 2132023 1264 247.32 4.71 0.00 4.21 0.00 4073206 33 83.68 4744973 2123317 1347 247.32 4.72 0.00 4.21 0.00 4061118 33 80.82 4713615 2188091 1490 247.32 4.72 0.00 4.21 0.00 4051675 33 85.29 4727399 2109011 1205 247.32 4.73 0.00 4.21 0.00 4039056 33 84.65 4724735 2102603 1053 247.32 After the patch InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 5.43 0.00 4.20 0.00 3313143 33 84.96 5434214 1900162 2656 245.51 5.43 0.00 4.20 0.00 3308527 33 85.24 5439695 1809382 2521 245.51 5.42 0.00 4.19 0.00 3316778 33 87.54 5416028 1805835 2256 245.51 5.42 0.00 4.19 0.00 3317673 33 90.44 5426044 1763056 2332 245.51 5.42 0.00 4.19 0.00 3314839 33 88.11 5435732 1792218 2499 245.52 5.44 0.00 4.19 0.00 3293228 33 91.84 5426301 1668597 2121 245.52 Similarly, netperf reports 230Mb/s before the patch, and 270Mb/s after the patch Reviewed by: gallatin Sponsored by: Limelight Networks Differential Revision: https://reviews.freebsd.org/D15366
2018-05-18 20:13:34 +00:00
CK_STAILQ_FOREACH(ia, &V_in6_ifaddrhead, ia_link) {
if (IN6_ARE_MASKED_ADDR_EQUAL(in6, &ia->ia_addr.sin6_addr,
&ia->ia_prefixmask.sin6_addr)) {
IN6_IFADDR_RUNLOCK(&in6_ifa_tracker);
return 1;
}
}
IN6_IFADDR_RUNLOCK(&in6_ifa_tracker);
return (0);
}
/*
* Return 1 if an internet address is for the local host and configured
* on one of its interfaces.
*/
int
in6_localip(struct in6_addr *in6)
{
struct rm_priotracker in6_ifa_tracker;
struct in6_ifaddr *ia;
IN6_IFADDR_RLOCK(&in6_ifa_tracker);
CK_LIST_FOREACH(ia, IN6ADDR_HASH(in6), ia6_hash) {
if (IN6_ARE_ADDR_EQUAL(in6, &ia->ia_addr.sin6_addr)) {
IN6_IFADDR_RUNLOCK(&in6_ifa_tracker);
return (1);
}
}
IN6_IFADDR_RUNLOCK(&in6_ifa_tracker);
return (0);
}
/*
* Return 1 if an internet address is configured on an interface.
*/
int
in6_ifhasaddr(struct ifnet *ifp, struct in6_addr *addr)
{
struct in6_addr in6;
struct ifaddr *ifa;
struct in6_ifaddr *ia6;
Widen NET_EPOCH coverage. When epoch(9) was introduced to network stack, it was basically dropped in place of existing locking, which was mutexes and rwlocks. For the sake of performance mutex covered areas were as small as possible, so became epoch covered areas. However, epoch doesn't introduce any contention, it just delays memory reclaim. So, there is no point to minimise epoch covered areas in sense of performance. Meanwhile entering/exiting epoch also has non-zero CPU usage, so doing this less often is a win. Not the least is also code maintainability. In the new paradigm we can assume that at any stage of processing a packet, we are inside network epoch. This makes coding both input and output path way easier. On output path we already enter epoch quite early - in the ip_output(), in the ip6_output(). This patch does the same for the input path. All ISR processing, network related callouts, other ways of packet injection to the network stack shall be performed in net_epoch. Any leaf function that walks network configuration now asserts epoch. Tricky part is configuration code paths - ioctls, sysctls. They also call into leaf functions, so some need to be changed. This patch would introduce more epoch recursions (see EPOCH_TRACE) than we had before. They will be cleaned up separately, as several of them aren't trivial. Note, that unlike a lock recursion the epoch recursion is safe and just wastes a bit of resources. Reviewed by: gallatin, hselasky, cy, adrian, kristof Differential Revision: https://reviews.freebsd.org/D19111
2019-10-07 22:40:05 +00:00
NET_EPOCH_ASSERT();
in6 = *addr;
if (in6_clearscope(&in6))
return (0);
in6_setscope(&in6, ifp, NULL);
ifnet: Replace if_addr_lock rwlock with epoch + mutex Run on LLNW canaries and tested by pho@ gallatin: Using a 14-core, 28-HTT single socket E5-2697 v3 with a 40GbE MLX5 based ConnectX 4-LX NIC, I see an almost 12% improvement in received packet rate, and a larger improvement in bytes delivered all the way to userspace. When the host receiving 64 streams of netperf -H $DUT -t UDP_STREAM -- -m 1, I see, using nstat -I mce0 1 before the patch: InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 4.98 0.00 4.42 0.00 4235592 33 83.80 4720653 2149771 1235 247.32 4.73 0.00 4.20 0.00 4025260 33 82.99 4724900 2139833 1204 247.32 4.72 0.00 4.20 0.00 4035252 33 82.14 4719162 2132023 1264 247.32 4.71 0.00 4.21 0.00 4073206 33 83.68 4744973 2123317 1347 247.32 4.72 0.00 4.21 0.00 4061118 33 80.82 4713615 2188091 1490 247.32 4.72 0.00 4.21 0.00 4051675 33 85.29 4727399 2109011 1205 247.32 4.73 0.00 4.21 0.00 4039056 33 84.65 4724735 2102603 1053 247.32 After the patch InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 5.43 0.00 4.20 0.00 3313143 33 84.96 5434214 1900162 2656 245.51 5.43 0.00 4.20 0.00 3308527 33 85.24 5439695 1809382 2521 245.51 5.42 0.00 4.19 0.00 3316778 33 87.54 5416028 1805835 2256 245.51 5.42 0.00 4.19 0.00 3317673 33 90.44 5426044 1763056 2332 245.51 5.42 0.00 4.19 0.00 3314839 33 88.11 5435732 1792218 2499 245.52 5.44 0.00 4.19 0.00 3293228 33 91.84 5426301 1668597 2121 245.52 Similarly, netperf reports 230Mb/s before the patch, and 270Mb/s after the patch Reviewed by: gallatin Sponsored by: Limelight Networks Differential Revision: https://reviews.freebsd.org/D15366
2018-05-18 20:13:34 +00:00
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family != AF_INET6)
continue;
ia6 = (struct in6_ifaddr *)ifa;
Widen NET_EPOCH coverage. When epoch(9) was introduced to network stack, it was basically dropped in place of existing locking, which was mutexes and rwlocks. For the sake of performance mutex covered areas were as small as possible, so became epoch covered areas. However, epoch doesn't introduce any contention, it just delays memory reclaim. So, there is no point to minimise epoch covered areas in sense of performance. Meanwhile entering/exiting epoch also has non-zero CPU usage, so doing this less often is a win. Not the least is also code maintainability. In the new paradigm we can assume that at any stage of processing a packet, we are inside network epoch. This makes coding both input and output path way easier. On output path we already enter epoch quite early - in the ip_output(), in the ip6_output(). This patch does the same for the input path. All ISR processing, network related callouts, other ways of packet injection to the network stack shall be performed in net_epoch. Any leaf function that walks network configuration now asserts epoch. Tricky part is configuration code paths - ioctls, sysctls. They also call into leaf functions, so some need to be changed. This patch would introduce more epoch recursions (see EPOCH_TRACE) than we had before. They will be cleaned up separately, as several of them aren't trivial. Note, that unlike a lock recursion the epoch recursion is safe and just wastes a bit of resources. Reviewed by: gallatin, hselasky, cy, adrian, kristof Differential Revision: https://reviews.freebsd.org/D19111
2019-10-07 22:40:05 +00:00
if (IN6_ARE_ADDR_EQUAL(&ia6->ia_addr.sin6_addr, &in6))
return (1);
}
return (0);
}
int
in6_is_addr_deprecated(struct sockaddr_in6 *sa6)
{
struct rm_priotracker in6_ifa_tracker;
struct in6_ifaddr *ia;
IN6_IFADDR_RLOCK(&in6_ifa_tracker);
CK_LIST_FOREACH(ia, IN6ADDR_HASH(&sa6->sin6_addr), ia6_hash) {
if (IN6_ARE_ADDR_EQUAL(IA6_IN6(ia), &sa6->sin6_addr)) {
if (ia->ia6_flags & IN6_IFF_DEPRECATED) {
IN6_IFADDR_RUNLOCK(&in6_ifa_tracker);
return (1); /* true */
}
break;
}
}
IN6_IFADDR_RUNLOCK(&in6_ifa_tracker);
return (0); /* false */
}
/*
* return length of part which dst and src are equal
* hard coding...
*/
int
in6_matchlen(struct in6_addr *src, struct in6_addr *dst)
{
int match = 0;
u_char *s = (u_char *)src, *d = (u_char *)dst;
u_char *lim = s + 16, r;
while (s < lim)
if ((r = (*d++ ^ *s++)) != 0) {
while (r < 128) {
match++;
r <<= 1;
}
break;
} else
match += 8;
return match;
}
/* XXX: to be scope conscious */
int
in6_are_prefix_equal(struct in6_addr *p1, struct in6_addr *p2, int len)
{
int bytelen, bitlen;
/* sanity check */
if (0 > len || len > 128) {
log(LOG_ERR, "in6_are_prefix_equal: invalid prefix length(%d)\n",
len);
return (0);
}
bytelen = len / 8;
bitlen = len % 8;
if (bcmp(&p1->s6_addr, &p2->s6_addr, bytelen))
return (0);
if (bitlen != 0 &&
p1->s6_addr[bytelen] >> (8 - bitlen) !=
p2->s6_addr[bytelen] >> (8 - bitlen))
return (0);
return (1);
}
void
in6_prefixlen2mask(struct in6_addr *maskp, int len)
{
u_char maskarray[8] = {0x80, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc, 0xfe, 0xff};
int bytelen, bitlen, i;
/* sanity check */
if (0 > len || len > 128) {
log(LOG_ERR, "in6_prefixlen2mask: invalid prefix length(%d)\n",
len);
return;
}
bzero(maskp, sizeof(*maskp));
bytelen = len / 8;
bitlen = len % 8;
for (i = 0; i < bytelen; i++)
maskp->s6_addr[i] = 0xff;
if (bitlen)
maskp->s6_addr[bytelen] = maskarray[bitlen - 1];
}
/*
* return the best address out of the same scope. if no address was
* found, return the first valid address from designated IF.
*/
struct in6_ifaddr *
in6_ifawithifp(struct ifnet *ifp, struct in6_addr *dst)
{
int dst_scope = in6_addrscope(dst), blen = -1, tlen;
struct ifaddr *ifa;
struct in6_ifaddr *besta = NULL;
struct in6_ifaddr *dep[2]; /* last-resort: deprecated */
Widen NET_EPOCH coverage. When epoch(9) was introduced to network stack, it was basically dropped in place of existing locking, which was mutexes and rwlocks. For the sake of performance mutex covered areas were as small as possible, so became epoch covered areas. However, epoch doesn't introduce any contention, it just delays memory reclaim. So, there is no point to minimise epoch covered areas in sense of performance. Meanwhile entering/exiting epoch also has non-zero CPU usage, so doing this less often is a win. Not the least is also code maintainability. In the new paradigm we can assume that at any stage of processing a packet, we are inside network epoch. This makes coding both input and output path way easier. On output path we already enter epoch quite early - in the ip_output(), in the ip6_output(). This patch does the same for the input path. All ISR processing, network related callouts, other ways of packet injection to the network stack shall be performed in net_epoch. Any leaf function that walks network configuration now asserts epoch. Tricky part is configuration code paths - ioctls, sysctls. They also call into leaf functions, so some need to be changed. This patch would introduce more epoch recursions (see EPOCH_TRACE) than we had before. They will be cleaned up separately, as several of them aren't trivial. Note, that unlike a lock recursion the epoch recursion is safe and just wastes a bit of resources. Reviewed by: gallatin, hselasky, cy, adrian, kristof Differential Revision: https://reviews.freebsd.org/D19111
2019-10-07 22:40:05 +00:00
NET_EPOCH_ASSERT();
dep[0] = dep[1] = NULL;
/*
* We first look for addresses in the same scope.
* If there is one, return it.
* If two or more, return one which matches the dst longest.
* If none, return one of global addresses assigned other ifs.
*/
ifnet: Replace if_addr_lock rwlock with epoch + mutex Run on LLNW canaries and tested by pho@ gallatin: Using a 14-core, 28-HTT single socket E5-2697 v3 with a 40GbE MLX5 based ConnectX 4-LX NIC, I see an almost 12% improvement in received packet rate, and a larger improvement in bytes delivered all the way to userspace. When the host receiving 64 streams of netperf -H $DUT -t UDP_STREAM -- -m 1, I see, using nstat -I mce0 1 before the patch: InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 4.98 0.00 4.42 0.00 4235592 33 83.80 4720653 2149771 1235 247.32 4.73 0.00 4.20 0.00 4025260 33 82.99 4724900 2139833 1204 247.32 4.72 0.00 4.20 0.00 4035252 33 82.14 4719162 2132023 1264 247.32 4.71 0.00 4.21 0.00 4073206 33 83.68 4744973 2123317 1347 247.32 4.72 0.00 4.21 0.00 4061118 33 80.82 4713615 2188091 1490 247.32 4.72 0.00 4.21 0.00 4051675 33 85.29 4727399 2109011 1205 247.32 4.73 0.00 4.21 0.00 4039056 33 84.65 4724735 2102603 1053 247.32 After the patch InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 5.43 0.00 4.20 0.00 3313143 33 84.96 5434214 1900162 2656 245.51 5.43 0.00 4.20 0.00 3308527 33 85.24 5439695 1809382 2521 245.51 5.42 0.00 4.19 0.00 3316778 33 87.54 5416028 1805835 2256 245.51 5.42 0.00 4.19 0.00 3317673 33 90.44 5426044 1763056 2332 245.51 5.42 0.00 4.19 0.00 3314839 33 88.11 5435732 1792218 2499 245.52 5.44 0.00 4.19 0.00 3293228 33 91.84 5426301 1668597 2121 245.52 Similarly, netperf reports 230Mb/s before the patch, and 270Mb/s after the patch Reviewed by: gallatin Sponsored by: Limelight Networks Differential Revision: https://reviews.freebsd.org/D15366
2018-05-18 20:13:34 +00:00
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family != AF_INET6)
continue;
if (((struct in6_ifaddr *)ifa)->ia6_flags & IN6_IFF_ANYCAST)
continue; /* XXX: is there any case to allow anycast? */
if (((struct in6_ifaddr *)ifa)->ia6_flags & IN6_IFF_NOTREADY)
continue; /* don't use this interface */
if (((struct in6_ifaddr *)ifa)->ia6_flags & IN6_IFF_DETACHED)
continue;
if (((struct in6_ifaddr *)ifa)->ia6_flags & IN6_IFF_DEPRECATED) {
if (V_ip6_use_deprecated)
dep[0] = (struct in6_ifaddr *)ifa;
continue;
}
if (dst_scope == in6_addrscope(IFA_IN6(ifa))) {
/*
* call in6_matchlen() as few as possible
*/
if (besta) {
if (blen == -1)
blen = in6_matchlen(&besta->ia_addr.sin6_addr, dst);
tlen = in6_matchlen(IFA_IN6(ifa), dst);
if (tlen > blen) {
blen = tlen;
besta = (struct in6_ifaddr *)ifa;
}
} else
besta = (struct in6_ifaddr *)ifa;
}
}
if (besta)
return (besta);
ifnet: Replace if_addr_lock rwlock with epoch + mutex Run on LLNW canaries and tested by pho@ gallatin: Using a 14-core, 28-HTT single socket E5-2697 v3 with a 40GbE MLX5 based ConnectX 4-LX NIC, I see an almost 12% improvement in received packet rate, and a larger improvement in bytes delivered all the way to userspace. When the host receiving 64 streams of netperf -H $DUT -t UDP_STREAM -- -m 1, I see, using nstat -I mce0 1 before the patch: InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 4.98 0.00 4.42 0.00 4235592 33 83.80 4720653 2149771 1235 247.32 4.73 0.00 4.20 0.00 4025260 33 82.99 4724900 2139833 1204 247.32 4.72 0.00 4.20 0.00 4035252 33 82.14 4719162 2132023 1264 247.32 4.71 0.00 4.21 0.00 4073206 33 83.68 4744973 2123317 1347 247.32 4.72 0.00 4.21 0.00 4061118 33 80.82 4713615 2188091 1490 247.32 4.72 0.00 4.21 0.00 4051675 33 85.29 4727399 2109011 1205 247.32 4.73 0.00 4.21 0.00 4039056 33 84.65 4724735 2102603 1053 247.32 After the patch InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 5.43 0.00 4.20 0.00 3313143 33 84.96 5434214 1900162 2656 245.51 5.43 0.00 4.20 0.00 3308527 33 85.24 5439695 1809382 2521 245.51 5.42 0.00 4.19 0.00 3316778 33 87.54 5416028 1805835 2256 245.51 5.42 0.00 4.19 0.00 3317673 33 90.44 5426044 1763056 2332 245.51 5.42 0.00 4.19 0.00 3314839 33 88.11 5435732 1792218 2499 245.52 5.44 0.00 4.19 0.00 3293228 33 91.84 5426301 1668597 2121 245.52 Similarly, netperf reports 230Mb/s before the patch, and 270Mb/s after the patch Reviewed by: gallatin Sponsored by: Limelight Networks Differential Revision: https://reviews.freebsd.org/D15366
2018-05-18 20:13:34 +00:00
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family != AF_INET6)
continue;
if (((struct in6_ifaddr *)ifa)->ia6_flags & IN6_IFF_ANYCAST)
continue; /* XXX: is there any case to allow anycast? */
if (((struct in6_ifaddr *)ifa)->ia6_flags & IN6_IFF_NOTREADY)
continue; /* don't use this interface */
if (((struct in6_ifaddr *)ifa)->ia6_flags & IN6_IFF_DETACHED)
continue;
if (((struct in6_ifaddr *)ifa)->ia6_flags & IN6_IFF_DEPRECATED) {
if (V_ip6_use_deprecated)
dep[1] = (struct in6_ifaddr *)ifa;
continue;
}
return (struct in6_ifaddr *)ifa;
}
/* use the last-resort values, that are, deprecated addresses */
if (dep[0])
return dep[0];
if (dep[1])
return dep[1];
return NULL;
}
/*
* perform DAD when interface becomes IFF_UP.
*/
void
in6_if_up(struct ifnet *ifp)
{
struct epoch_tracker et;
struct ifaddr *ifa;
struct in6_ifaddr *ia;
NET_EPOCH_ENTER(et);
ifnet: Replace if_addr_lock rwlock with epoch + mutex Run on LLNW canaries and tested by pho@ gallatin: Using a 14-core, 28-HTT single socket E5-2697 v3 with a 40GbE MLX5 based ConnectX 4-LX NIC, I see an almost 12% improvement in received packet rate, and a larger improvement in bytes delivered all the way to userspace. When the host receiving 64 streams of netperf -H $DUT -t UDP_STREAM -- -m 1, I see, using nstat -I mce0 1 before the patch: InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 4.98 0.00 4.42 0.00 4235592 33 83.80 4720653 2149771 1235 247.32 4.73 0.00 4.20 0.00 4025260 33 82.99 4724900 2139833 1204 247.32 4.72 0.00 4.20 0.00 4035252 33 82.14 4719162 2132023 1264 247.32 4.71 0.00 4.21 0.00 4073206 33 83.68 4744973 2123317 1347 247.32 4.72 0.00 4.21 0.00 4061118 33 80.82 4713615 2188091 1490 247.32 4.72 0.00 4.21 0.00 4051675 33 85.29 4727399 2109011 1205 247.32 4.73 0.00 4.21 0.00 4039056 33 84.65 4724735 2102603 1053 247.32 After the patch InMpps OMpps InGbs OGbs err TCP Est %CPU syscalls csw irq GBfree 5.43 0.00 4.20 0.00 3313143 33 84.96 5434214 1900162 2656 245.51 5.43 0.00 4.20 0.00 3308527 33 85.24 5439695 1809382 2521 245.51 5.42 0.00 4.19 0.00 3316778 33 87.54 5416028 1805835 2256 245.51 5.42 0.00 4.19 0.00 3317673 33 90.44 5426044 1763056 2332 245.51 5.42 0.00 4.19 0.00 3314839 33 88.11 5435732 1792218 2499 245.52 5.44 0.00 4.19 0.00 3293228 33 91.84 5426301 1668597 2121 245.52 Similarly, netperf reports 230Mb/s before the patch, and 270Mb/s after the patch Reviewed by: gallatin Sponsored by: Limelight Networks Differential Revision: https://reviews.freebsd.org/D15366
2018-05-18 20:13:34 +00:00
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family != AF_INET6)
continue;
ia = (struct in6_ifaddr *)ifa;
if (ia->ia6_flags & IN6_IFF_TENTATIVE) {
/*
* The TENTATIVE flag was likely set by hand
* beforehand, implicitly indicating the need for DAD.
* We may be able to skip the random delay in this
* case, but we impose delays just in case.
*/
nd6_dad_start(ifa,
arc4random() % (MAX_RTR_SOLICITATION_DELAY * hz));
}
}
NET_EPOCH_EXIT(et);
/*
* special cases, like 6to4, are handled in in6_ifattach
*/
in6_ifattach(ifp, NULL);
}
int
in6if_do_dad(struct ifnet *ifp)
{
if ((ifp->if_flags & IFF_LOOPBACK) != 0)
return (0);
if ((ifp->if_flags & IFF_MULTICAST) == 0)
return (0);
if ((ND_IFINFO(ifp)->flags &
(ND6_IFF_IFDISABLED | ND6_IFF_NO_DAD)) != 0)
return (0);
return (1);
}
/*
* Calculate max IPv6 MTU through all the interfaces and store it
* to in6_maxmtu.
*/
void
in6_setmaxmtu(void)
{
struct epoch_tracker et;
unsigned long maxmtu = 0;
struct ifnet *ifp;
NET_EPOCH_ENTER(et);
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
/* this function can be called during ifnet initialization */
if (!ifp->if_afdata[AF_INET6])
continue;
if ((ifp->if_flags & IFF_LOOPBACK) == 0 &&
IN6_LINKMTU(ifp) > maxmtu)
maxmtu = IN6_LINKMTU(ifp);
}
NET_EPOCH_EXIT(et);
if (maxmtu) /* update only when maxmtu is positive */
V_in6_maxmtu = maxmtu;
}
/*
* Provide the length of interface identifiers to be used for the link attached
* to the given interface. The length should be defined in "IPv6 over
* xxx-link" document. Note that address architecture might also define
* the length for a particular set of address prefixes, regardless of the
* link type. As clarified in rfc2462bis, those two definitions should be
* consistent, and those really are as of August 2004.
*/
int
in6_if2idlen(struct ifnet *ifp)
{
switch (ifp->if_type) {
case IFT_ETHER: /* RFC2464 */
case IFT_PROPVIRTUAL: /* XXX: no RFC. treat it as ether */
case IFT_L2VLAN: /* ditto */
case IFT_BRIDGE: /* bridge(4) only does Ethernet-like links */
case IFT_INFINIBAND:
return (64);
case IFT_PPP: /* RFC2472 */
return (64);
case IFT_FRELAY: /* RFC2590 */
return (64);
case IFT_IEEE1394: /* RFC3146 */
return (64);
case IFT_GIF:
return (64); /* draft-ietf-v6ops-mech-v2-07 */
case IFT_LOOP:
return (64); /* XXX: is this really correct? */
default:
/*
* Unknown link type:
* It might be controversial to use the today's common constant
* of 64 for these cases unconditionally. For full compliance,
* we should return an error in this case. On the other hand,
* if we simply miss the standard for the link type or a new
* standard is defined for a new link type, the IFID length
* is very likely to be the common constant. As a compromise,
* we always use the constant, but make an explicit notice
* indicating the "unknown" case.
*/
printf("in6_if2idlen: unknown link type (%d)\n", ifp->if_type);
return (64);
}
}
struct in6_llentry {
struct llentry base;
};
#define IN6_LLTBL_DEFAULT_HSIZE 32
#define IN6_LLTBL_HASH(k, h) \
(((((((k >> 8) ^ k) >> 8) ^ k) >> 8) ^ k) & ((h) - 1))
/*
* Do actual deallocation of @lle.
*/
static void
in6_lltable_destroy_lle_unlocked(epoch_context_t ctx)
{
struct llentry *lle;
lle = __containerof(ctx, struct llentry, lle_epoch_ctx);
LLE_LOCK_DESTROY(lle);
LLE_REQ_DESTROY(lle);
free(lle, M_LLTABLE);
}
/*
* Called by LLE_FREE_LOCKED when number of references
* drops to zero.
*/
static void
in6_lltable_destroy_lle(struct llentry *lle)
{
LLE_WUNLOCK(lle);
NET_EPOCH_CALL(in6_lltable_destroy_lle_unlocked, &lle->lle_epoch_ctx);
}
static struct llentry *
in6_lltable_new(const struct in6_addr *addr6, u_int flags)
{
struct in6_llentry *lle;
lle = malloc(sizeof(struct in6_llentry), M_LLTABLE, M_NOWAIT | M_ZERO);
if (lle == NULL) /* NB: caller generates msg */
return NULL;
lle->base.r_l3addr.addr6 = *addr6;
lle->base.lle_refcnt = 1;
lle->base.lle_free = in6_lltable_destroy_lle;
LLE_LOCK_INIT(&lle->base);
Remove LLE read lock from IPv6 fast path. LLE structure is mostly unchanged during its lifecycle: there are only 2 things relevant for fast path lookup code: 1) link-level address change. Since r286722, these updates are performed under AFDATA WLOCK. 2) Some sort of feedback indicating that this particular entry is used so we send NS to perform reachability verification instead of expiring entry. The only signal that is needed from fast path is something like binary yes/no. The latter is solved by the following changes: Special r_skip_req (introduced in D3688) value is used for fast path feedback. It is read lockless by fast path, but updated under req_mutex mutex. If this field is non-zero, then fast path will acquire lock and set it back to 0. After transitioning to STALE state, callout timer is armed to run each V_nd6_delay seconds to make sure that if packet was transmitted at the start of given interval, we would be able to switch to PROBE state in V_nd6_delay seconds as user expects. (in STALE state) timer is rescheduled until original V_nd6_gctimer expires keeping lle in STALE state (remaining timer value stored in lle_remtime). (in STALE state) timer is rescheduled if packet was transmitted less that V_nd6_delay seconds ago to make sure we transition to PROBE state exactly after V_n6_delay seconds. As a result, all packets towards lle in REACHABLE/STALE/PROBE states are handled by fast path without acquiring lle read lock. Differential Revision: https://reviews.freebsd.org/D3780
2015-12-13 07:39:49 +00:00
LLE_REQ_INIT(&lle->base);
callout_init(&lle->base.lle_timer, 1);
return (&lle->base);
}
static int
in6_lltable_match_prefix(const struct sockaddr *saddr,
const struct sockaddr *smask, u_int flags, struct llentry *lle)
{
const struct in6_addr *addr, *mask, *lle_addr;
addr = &((const struct sockaddr_in6 *)saddr)->sin6_addr;
mask = &((const struct sockaddr_in6 *)smask)->sin6_addr;
lle_addr = &lle->r_l3addr.addr6;
if (IN6_ARE_MASKED_ADDR_EQUAL(lle_addr, addr, mask) == 0)
return (0);
if (lle->la_flags & LLE_IFADDR) {
/*
* Delete LLE_IFADDR records IFF address & flag matches.
* Note that addr is the interface address within prefix
* being matched.
*/
if (IN6_ARE_ADDR_EQUAL(addr, lle_addr) &&
(flags & LLE_STATIC) != 0)
return (1);
return (0);
}
/* flags & LLE_STATIC means deleting both dynamic and static entries */
if ((flags & LLE_STATIC) || !(lle->la_flags & LLE_STATIC))
return (1);
return (0);
}
static void
in6_lltable_free_entry(struct lltable *llt, struct llentry *lle)
{
struct ifnet *ifp;
LLE_WLOCK_ASSERT(lle);
KASSERT(llt != NULL, ("lltable is NULL"));
/* Unlink entry from table */
if ((lle->la_flags & LLE_LINKED) != 0) {
ifp = llt->llt_ifp;
IF_AFDATA_WLOCK_ASSERT(ifp);
lltable_unlink_entry(llt, lle);
}
llentry_free(lle);
}
static int
in6_lltable_rtcheck(struct ifnet *ifp,
u_int flags,
const struct sockaddr *l3addr)
{
const struct sockaddr_in6 *sin6;
struct nhop_object *nh;
struct in6_addr dst;
uint32_t scopeid;
char ip6buf[INET6_ADDRSTRLEN];
Constrain IPv6 routes to single FIBs when net.add_addr_allfibs=0 sys/netinet6/icmp6.c Use the interface's FIB for source address selection in ICMPv6 error responses. sys/netinet6/in6.c In in6_newaddrmsg, announce arrival of local addresses on the interface's FIB only. In in6_lltable_rtcheck, use a per-fib ND6 cache instead of a single cache. sys/netinet6/in6_src.c In in6_selectsrc, use the caller's fib instead of the default fib. In in6_selectsrc_socket, remove a superfluous check. sys/netinet6/nd6.c In nd6_lle_event, use the interface's fib for routing socket messages. In nd6_is_new_addr_neighbor, check all FIBs when trying to determine whether an address is a neighbor. Also, simplify the code for point to point interfaces. sys/netinet6/nd6.h sys/netinet6/nd6.c sys/netinet6/nd6_rtr.c Make defrouter_select fib-aware, and make all of its callers pass in the interface fib. sys/netinet6/nd6_nbr.c When inputting a Neighbor Solicitation packet, consider the interface fib instead of the default fib for DAD. Output NS and Neighbor Advertisement packets on the correct fib. sys/netinet6/nd6_rtr.c Allow installing the same host route on different interfaces in different FIBs. If rt_add_addr_allfibs=0, only install or delete the prefix route on the interface fib. tests/sys/netinet/fibs_test.sh Clear some expected failures, but add a skip for the newly revealed BUG217871. PR: 196361 Submitted by: Erick Turnquist <jhujhiti@adjectivism.org> Reported by: Jason Healy <jhealy@logn.net> Reviewed by: asomers MFC after: 3 weeks Sponsored by: Spectra Logic Corp Differential Revision: https://reviews.freebsd.org/D9451
2017-03-17 16:50:37 +00:00
int fibnum;
Widen NET_EPOCH coverage. When epoch(9) was introduced to network stack, it was basically dropped in place of existing locking, which was mutexes and rwlocks. For the sake of performance mutex covered areas were as small as possible, so became epoch covered areas. However, epoch doesn't introduce any contention, it just delays memory reclaim. So, there is no point to minimise epoch covered areas in sense of performance. Meanwhile entering/exiting epoch also has non-zero CPU usage, so doing this less often is a win. Not the least is also code maintainability. In the new paradigm we can assume that at any stage of processing a packet, we are inside network epoch. This makes coding both input and output path way easier. On output path we already enter epoch quite early - in the ip_output(), in the ip6_output(). This patch does the same for the input path. All ISR processing, network related callouts, other ways of packet injection to the network stack shall be performed in net_epoch. Any leaf function that walks network configuration now asserts epoch. Tricky part is configuration code paths - ioctls, sysctls. They also call into leaf functions, so some need to be changed. This patch would introduce more epoch recursions (see EPOCH_TRACE) than we had before. They will be cleaned up separately, as several of them aren't trivial. Note, that unlike a lock recursion the epoch recursion is safe and just wastes a bit of resources. Reviewed by: gallatin, hselasky, cy, adrian, kristof Differential Revision: https://reviews.freebsd.org/D19111
2019-10-07 22:40:05 +00:00
NET_EPOCH_ASSERT();
KASSERT(l3addr->sa_family == AF_INET6,
("sin_family %d", l3addr->sa_family));
sin6 = (const struct sockaddr_in6 *)l3addr;
in6_splitscope(&sin6->sin6_addr, &dst, &scopeid);
Constrain IPv6 routes to single FIBs when net.add_addr_allfibs=0 sys/netinet6/icmp6.c Use the interface's FIB for source address selection in ICMPv6 error responses. sys/netinet6/in6.c In in6_newaddrmsg, announce arrival of local addresses on the interface's FIB only. In in6_lltable_rtcheck, use a per-fib ND6 cache instead of a single cache. sys/netinet6/in6_src.c In in6_selectsrc, use the caller's fib instead of the default fib. In in6_selectsrc_socket, remove a superfluous check. sys/netinet6/nd6.c In nd6_lle_event, use the interface's fib for routing socket messages. In nd6_is_new_addr_neighbor, check all FIBs when trying to determine whether an address is a neighbor. Also, simplify the code for point to point interfaces. sys/netinet6/nd6.h sys/netinet6/nd6.c sys/netinet6/nd6_rtr.c Make defrouter_select fib-aware, and make all of its callers pass in the interface fib. sys/netinet6/nd6_nbr.c When inputting a Neighbor Solicitation packet, consider the interface fib instead of the default fib for DAD. Output NS and Neighbor Advertisement packets on the correct fib. sys/netinet6/nd6_rtr.c Allow installing the same host route on different interfaces in different FIBs. If rt_add_addr_allfibs=0, only install or delete the prefix route on the interface fib. tests/sys/netinet/fibs_test.sh Clear some expected failures, but add a skip for the newly revealed BUG217871. PR: 196361 Submitted by: Erick Turnquist <jhujhiti@adjectivism.org> Reported by: Jason Healy <jhealy@logn.net> Reviewed by: asomers MFC after: 3 weeks Sponsored by: Spectra Logic Corp Differential Revision: https://reviews.freebsd.org/D9451
2017-03-17 16:50:37 +00:00
fibnum = V_rt_add_addr_allfibs ? RT_DEFAULT_FIB : ifp->if_fib;
nh = fib6_lookup(fibnum, &dst, scopeid, NHR_NONE, 0);
if (nh && ((nh->nh_flags & NHF_GATEWAY) || nh->nh_ifp != ifp)) {
struct ifaddr *ifa;
/*
* Create an ND6 cache for an IPv6 neighbor
* that is not covered by our own prefix.
*/
ifa = ifaof_ifpforaddr(l3addr, ifp);
if (ifa != NULL) {
return 0;
}
log(LOG_INFO, "IPv6 address: \"%s\" is not on the network\n",
ip6_sprintf(ip6buf, &sin6->sin6_addr));
return EINVAL;
}
return 0;
}
/*
* Called by the datapath to indicate that the entry was used.
*/
static void
in6_lltable_mark_used(struct llentry *lle)
{
LLE_REQ_LOCK(lle);
lle->r_skip_req = 0;
/*
* Set the hit time so the callback function
* can determine the remaining time before
* transiting to the DELAY state.
*/
lle->lle_hittime = time_uptime;
LLE_REQ_UNLOCK(lle);
}
static inline uint32_t
in6_lltable_hash_dst(const struct in6_addr *dst, uint32_t hsize)
{
return (IN6_LLTBL_HASH(dst->s6_addr32[3], hsize));
}
static uint32_t
in6_lltable_hash(const struct llentry *lle, uint32_t hsize)
{
return (in6_lltable_hash_dst(&lle->r_l3addr.addr6, hsize));
}
static void
in6_lltable_fill_sa_entry(const struct llentry *lle, struct sockaddr *sa)
{
struct sockaddr_in6 *sin6;
sin6 = (struct sockaddr_in6 *)sa;
bzero(sin6, sizeof(*sin6));
sin6->sin6_family = AF_INET6;
sin6->sin6_len = sizeof(*sin6);
sin6->sin6_addr = lle->r_l3addr.addr6;
}
static inline struct llentry *
in6_lltable_find_dst(struct lltable *llt, const struct in6_addr *dst)
{
struct llentry *lle;
struct llentries *lleh;
u_int hashidx;
hashidx = in6_lltable_hash_dst(dst, llt->llt_hsize);
lleh = &llt->lle_head[hashidx];
CK_LIST_FOREACH(lle, lleh, lle_next) {
if (lle->la_flags & LLE_DELETED)
continue;
if (IN6_ARE_ADDR_EQUAL(&lle->r_l3addr.addr6, dst))
break;
}
return (lle);
}
static void
in6_lltable_delete_entry(struct lltable *llt, struct llentry *lle)
{
lle->la_flags |= LLE_DELETED;
EVENTHANDLER_INVOKE(lle_event, lle, LLENTRY_DELETED);
#ifdef DIAGNOSTIC
log(LOG_INFO, "ifaddr cache = %p is deleted\n", lle);
2012-08-01 09:00:26 +00:00
#endif
llentry_free(lle);
}
static struct llentry *
in6_lltable_alloc(struct lltable *llt, u_int flags,
const struct sockaddr *l3addr)
{
const struct sockaddr_in6 *sin6 = (const struct sockaddr_in6 *)l3addr;
struct ifnet *ifp = llt->llt_ifp;
struct llentry *lle;
Implement interface link header precomputation API. Add if_requestencap() interface method which is capable of calculating various link headers for given interface. Right now there is support for INET/INET6/ARP llheader calculation (IFENCAP_LL type request). Other types are planned to support more complex calculation (L2 multipath lagg nexthops, tunnel encap nexthops, etc..). Reshape 'struct route' to be able to pass additional data (with is length) to prepend to mbuf. These two changes permits routing code to pass pre-calculated nexthop data (like L2 header for route w/gateway) down to the stack eliminating the need for other lookups. It also brings us closer to more complex scenarios like transparently handling MPLS nexthops and tunnel interfaces. Last, but not least, it removes layering violation introduced by flowtable code (ro_lle) and simplifies handling of existing if_output consumers. ARP/ND changes: Make arp/ndp stack pre-calculate link header upon installing/updating lle record. Interface link address change are handled by re-calculating headers for all lles based on if_lladdr event. After these changes, arpresolve()/nd6_resolve() returns full pre-calculated header for supported interfaces thus simplifying if_output(). Move these lookups to separate ether_resolve_addr() function which ether returs error or fully-prepared link header. Add <arp|nd6_>resolve_addr() compat versions to return link addresses instead of pre-calculated data. BPF changes: Raw bpf writes occupied _two_ cases: AF_UNSPEC and pseudo_AF_HDRCMPLT. Despite the naming, both of there have ther header "complete". The only difference is that interface source mac has to be filled by OS for AF_UNSPEC (controlled via BIOCGHDRCMPLT). This logic has to stay inside BPF and not pollute if_output() routines. Convert BPF to pass prepend data via new 'struct route' mechanism. Note that it does not change non-optimized if_output(): ro_prepend handling is purely optional. Side note: hackish pseudo_AF_HDRCMPLT is supported for ethernet and FDDI. It is not needed for ethernet anymore. The only remaining FDDI user is dev/pdq mostly untouched since 2007. FDDI support was eliminated from OpenBSD in 2013 (sys/net/if_fddisubr.c rev 1.65). Flowtable changes: Flowtable violates layering by saving (and not correctly managing) rtes/lles. Instead of passing lle pointer, pass pointer to pre-calculated header data from that lle. Differential Revision: https://reviews.freebsd.org/D4102
2015-12-31 05:03:27 +00:00
char linkhdr[LLE_MAX_LINKHDR];
size_t linkhdrsize;
int lladdr_off;
KASSERT(l3addr->sa_family == AF_INET6,
("sin_family %d", l3addr->sa_family));
/*
* A route that covers the given address must have
* been installed 1st because we are doing a resolution,
* verify this.
*/
if (!(flags & LLE_IFADDR) &&
in6_lltable_rtcheck(ifp, flags, l3addr) != 0)
return (NULL);
lle = in6_lltable_new(&sin6->sin6_addr, flags);
if (lle == NULL) {
log(LOG_INFO, "lla_lookup: new lle malloc failed\n");
return (NULL);
}
lle->la_flags = flags;
if ((flags & LLE_IFADDR) == LLE_IFADDR) {
Implement interface link header precomputation API. Add if_requestencap() interface method which is capable of calculating various link headers for given interface. Right now there is support for INET/INET6/ARP llheader calculation (IFENCAP_LL type request). Other types are planned to support more complex calculation (L2 multipath lagg nexthops, tunnel encap nexthops, etc..). Reshape 'struct route' to be able to pass additional data (with is length) to prepend to mbuf. These two changes permits routing code to pass pre-calculated nexthop data (like L2 header for route w/gateway) down to the stack eliminating the need for other lookups. It also brings us closer to more complex scenarios like transparently handling MPLS nexthops and tunnel interfaces. Last, but not least, it removes layering violation introduced by flowtable code (ro_lle) and simplifies handling of existing if_output consumers. ARP/ND changes: Make arp/ndp stack pre-calculate link header upon installing/updating lle record. Interface link address change are handled by re-calculating headers for all lles based on if_lladdr event. After these changes, arpresolve()/nd6_resolve() returns full pre-calculated header for supported interfaces thus simplifying if_output(). Move these lookups to separate ether_resolve_addr() function which ether returs error or fully-prepared link header. Add <arp|nd6_>resolve_addr() compat versions to return link addresses instead of pre-calculated data. BPF changes: Raw bpf writes occupied _two_ cases: AF_UNSPEC and pseudo_AF_HDRCMPLT. Despite the naming, both of there have ther header "complete". The only difference is that interface source mac has to be filled by OS for AF_UNSPEC (controlled via BIOCGHDRCMPLT). This logic has to stay inside BPF and not pollute if_output() routines. Convert BPF to pass prepend data via new 'struct route' mechanism. Note that it does not change non-optimized if_output(): ro_prepend handling is purely optional. Side note: hackish pseudo_AF_HDRCMPLT is supported for ethernet and FDDI. It is not needed for ethernet anymore. The only remaining FDDI user is dev/pdq mostly untouched since 2007. FDDI support was eliminated from OpenBSD in 2013 (sys/net/if_fddisubr.c rev 1.65). Flowtable changes: Flowtable violates layering by saving (and not correctly managing) rtes/lles. Instead of passing lle pointer, pass pointer to pre-calculated header data from that lle. Differential Revision: https://reviews.freebsd.org/D4102
2015-12-31 05:03:27 +00:00
linkhdrsize = LLE_MAX_LINKHDR;
if (lltable_calc_llheader(ifp, AF_INET6, IF_LLADDR(ifp),
linkhdr, &linkhdrsize, &lladdr_off) != 0) {
NET_EPOCH_CALL(in6_lltable_destroy_lle_unlocked, &lle->lle_epoch_ctx);
Implement interface link header precomputation API. Add if_requestencap() interface method which is capable of calculating various link headers for given interface. Right now there is support for INET/INET6/ARP llheader calculation (IFENCAP_LL type request). Other types are planned to support more complex calculation (L2 multipath lagg nexthops, tunnel encap nexthops, etc..). Reshape 'struct route' to be able to pass additional data (with is length) to prepend to mbuf. These two changes permits routing code to pass pre-calculated nexthop data (like L2 header for route w/gateway) down to the stack eliminating the need for other lookups. It also brings us closer to more complex scenarios like transparently handling MPLS nexthops and tunnel interfaces. Last, but not least, it removes layering violation introduced by flowtable code (ro_lle) and simplifies handling of existing if_output consumers. ARP/ND changes: Make arp/ndp stack pre-calculate link header upon installing/updating lle record. Interface link address change are handled by re-calculating headers for all lles based on if_lladdr event. After these changes, arpresolve()/nd6_resolve() returns full pre-calculated header for supported interfaces thus simplifying if_output(). Move these lookups to separate ether_resolve_addr() function which ether returs error or fully-prepared link header. Add <arp|nd6_>resolve_addr() compat versions to return link addresses instead of pre-calculated data. BPF changes: Raw bpf writes occupied _two_ cases: AF_UNSPEC and pseudo_AF_HDRCMPLT. Despite the naming, both of there have ther header "complete". The only difference is that interface source mac has to be filled by OS for AF_UNSPEC (controlled via BIOCGHDRCMPLT). This logic has to stay inside BPF and not pollute if_output() routines. Convert BPF to pass prepend data via new 'struct route' mechanism. Note that it does not change non-optimized if_output(): ro_prepend handling is purely optional. Side note: hackish pseudo_AF_HDRCMPLT is supported for ethernet and FDDI. It is not needed for ethernet anymore. The only remaining FDDI user is dev/pdq mostly untouched since 2007. FDDI support was eliminated from OpenBSD in 2013 (sys/net/if_fddisubr.c rev 1.65). Flowtable changes: Flowtable violates layering by saving (and not correctly managing) rtes/lles. Instead of passing lle pointer, pass pointer to pre-calculated header data from that lle. Differential Revision: https://reviews.freebsd.org/D4102
2015-12-31 05:03:27 +00:00
return (NULL);
}
Implement interface link header precomputation API. Add if_requestencap() interface method which is capable of calculating various link headers for given interface. Right now there is support for INET/INET6/ARP llheader calculation (IFENCAP_LL type request). Other types are planned to support more complex calculation (L2 multipath lagg nexthops, tunnel encap nexthops, etc..). Reshape 'struct route' to be able to pass additional data (with is length) to prepend to mbuf. These two changes permits routing code to pass pre-calculated nexthop data (like L2 header for route w/gateway) down to the stack eliminating the need for other lookups. It also brings us closer to more complex scenarios like transparently handling MPLS nexthops and tunnel interfaces. Last, but not least, it removes layering violation introduced by flowtable code (ro_lle) and simplifies handling of existing if_output consumers. ARP/ND changes: Make arp/ndp stack pre-calculate link header upon installing/updating lle record. Interface link address change are handled by re-calculating headers for all lles based on if_lladdr event. After these changes, arpresolve()/nd6_resolve() returns full pre-calculated header for supported interfaces thus simplifying if_output(). Move these lookups to separate ether_resolve_addr() function which ether returs error or fully-prepared link header. Add <arp|nd6_>resolve_addr() compat versions to return link addresses instead of pre-calculated data. BPF changes: Raw bpf writes occupied _two_ cases: AF_UNSPEC and pseudo_AF_HDRCMPLT. Despite the naming, both of there have ther header "complete". The only difference is that interface source mac has to be filled by OS for AF_UNSPEC (controlled via BIOCGHDRCMPLT). This logic has to stay inside BPF and not pollute if_output() routines. Convert BPF to pass prepend data via new 'struct route' mechanism. Note that it does not change non-optimized if_output(): ro_prepend handling is purely optional. Side note: hackish pseudo_AF_HDRCMPLT is supported for ethernet and FDDI. It is not needed for ethernet anymore. The only remaining FDDI user is dev/pdq mostly untouched since 2007. FDDI support was eliminated from OpenBSD in 2013 (sys/net/if_fddisubr.c rev 1.65). Flowtable changes: Flowtable violates layering by saving (and not correctly managing) rtes/lles. Instead of passing lle pointer, pass pointer to pre-calculated header data from that lle. Differential Revision: https://reviews.freebsd.org/D4102
2015-12-31 05:03:27 +00:00
lltable_set_entry_addr(ifp, lle, linkhdr, linkhdrsize,
lladdr_off);
2015-11-07 11:12:00 +00:00
lle->la_flags |= LLE_STATIC;
}
if ((lle->la_flags & LLE_STATIC) != 0)
lle->ln_state = ND6_LLINFO_REACHABLE;
return (lle);
}
static struct llentry *
in6_lltable_lookup(struct lltable *llt, u_int flags,
const struct sockaddr *l3addr)
{
const struct sockaddr_in6 *sin6 = (const struct sockaddr_in6 *)l3addr;
struct llentry *lle;
IF_AFDATA_LOCK_ASSERT(llt->llt_ifp);
KASSERT(l3addr->sa_family == AF_INET6,
("sin_family %d", l3addr->sa_family));
KASSERT((flags & (LLE_UNLOCKED | LLE_EXCLUSIVE)) !=
(LLE_UNLOCKED | LLE_EXCLUSIVE),
("wrong lle request flags: %#x", flags));
lle = in6_lltable_find_dst(llt, &sin6->sin6_addr);
if (lle == NULL)
return (NULL);
Remove LLE read lock from IPv6 fast path. LLE structure is mostly unchanged during its lifecycle: there are only 2 things relevant for fast path lookup code: 1) link-level address change. Since r286722, these updates are performed under AFDATA WLOCK. 2) Some sort of feedback indicating that this particular entry is used so we send NS to perform reachability verification instead of expiring entry. The only signal that is needed from fast path is something like binary yes/no. The latter is solved by the following changes: Special r_skip_req (introduced in D3688) value is used for fast path feedback. It is read lockless by fast path, but updated under req_mutex mutex. If this field is non-zero, then fast path will acquire lock and set it back to 0. After transitioning to STALE state, callout timer is armed to run each V_nd6_delay seconds to make sure that if packet was transmitted at the start of given interval, we would be able to switch to PROBE state in V_nd6_delay seconds as user expects. (in STALE state) timer is rescheduled until original V_nd6_gctimer expires keeping lle in STALE state (remaining timer value stored in lle_remtime). (in STALE state) timer is rescheduled if packet was transmitted less that V_nd6_delay seconds ago to make sure we transition to PROBE state exactly after V_n6_delay seconds. As a result, all packets towards lle in REACHABLE/STALE/PROBE states are handled by fast path without acquiring lle read lock. Differential Revision: https://reviews.freebsd.org/D3780
2015-12-13 07:39:49 +00:00
if (flags & LLE_UNLOCKED)
return (lle);
if (flags & LLE_EXCLUSIVE)
LLE_WLOCK(lle);
else
LLE_RLOCK(lle);
/*
* If the afdata lock is not held, the LLE may have been unlinked while
* we were blocked on the LLE lock. Check for this case.
*/
if (__predict_false((lle->la_flags & LLE_LINKED) == 0)) {
if (flags & LLE_EXCLUSIVE)
LLE_WUNLOCK(lle);
else
LLE_RUNLOCK(lle);
return (NULL);
}
return (lle);
}
static int
in6_lltable_dump_entry(struct lltable *llt, struct llentry *lle,
struct sysctl_req *wr)
{
struct ifnet *ifp = llt->llt_ifp;
/* XXX stack use */
struct {
struct rt_msghdr rtm;
struct sockaddr_in6 sin6;
/*
* ndp.c assumes that sdl is word aligned
*/
#ifdef __LP64__
uint32_t pad;
#endif
struct sockaddr_dl sdl;
} ndpc;
struct sockaddr_dl *sdl;
int error;
bzero(&ndpc, sizeof(ndpc));
/* skip deleted entries */
if ((lle->la_flags & LLE_DELETED) == LLE_DELETED)
return (0);
/* Skip if jailed and not a valid IP of the prison. */
lltable_fill_sa_entry(lle, (struct sockaddr *)&ndpc.sin6);
if (prison_if(wr->td->td_ucred, (struct sockaddr *)&ndpc.sin6) != 0)
return (0);
/*
* produce a msg made of:
* struct rt_msghdr;
* struct sockaddr_in6 (IPv6)
* struct sockaddr_dl;
*/
ndpc.rtm.rtm_msglen = sizeof(ndpc);
ndpc.rtm.rtm_version = RTM_VERSION;
ndpc.rtm.rtm_type = RTM_GET;
ndpc.rtm.rtm_flags = RTF_UP;
ndpc.rtm.rtm_addrs = RTA_DST | RTA_GATEWAY;
sa6_recoverscope(&ndpc.sin6);
/* publish */
if (lle->la_flags & LLE_PUB)
ndpc.rtm.rtm_flags |= RTF_ANNOUNCE;
sdl = &ndpc.sdl;
sdl->sdl_family = AF_LINK;
sdl->sdl_len = sizeof(*sdl);
sdl->sdl_index = ifp->if_index;
sdl->sdl_type = ifp->if_type;
if ((lle->la_flags & LLE_VALID) == LLE_VALID) {
sdl->sdl_alen = ifp->if_addrlen;
bcopy(lle->ll_addr, LLADDR(sdl), ifp->if_addrlen);
} else {
sdl->sdl_alen = 0;
bzero(LLADDR(sdl), ifp->if_addrlen);
}
if (lle->la_expire != 0)
ndpc.rtm.rtm_rmx.rmx_expire = lle->la_expire +
lle->lle_remtime / hz + time_second - time_uptime;
ndpc.rtm.rtm_flags |= (RTF_HOST | RTF_LLDATA);
if (lle->la_flags & LLE_STATIC)
ndpc.rtm.rtm_flags |= RTF_STATIC;
if (lle->la_flags & LLE_IFADDR)
ndpc.rtm.rtm_flags |= RTF_PINNED;
if (lle->ln_router != 0)
ndpc.rtm.rtm_flags |= RTF_GATEWAY;
ndpc.rtm.rtm_rmx.rmx_pksent = lle->la_asked;
/* Store state in rmx_weight value */
ndpc.rtm.rtm_rmx.rmx_state = lle->ln_state;
ndpc.rtm.rtm_index = ifp->if_index;
error = SYSCTL_OUT(wr, &ndpc, sizeof(ndpc));
return (error);
}
static struct lltable *
in6_lltattach(struct ifnet *ifp)
{
struct lltable *llt;
llt = lltable_allocate_htbl(IN6_LLTBL_DEFAULT_HSIZE);
llt->llt_af = AF_INET6;
llt->llt_ifp = ifp;
llt->llt_lookup = in6_lltable_lookup;
llt->llt_alloc_entry = in6_lltable_alloc;
llt->llt_delete_entry = in6_lltable_delete_entry;
llt->llt_dump_entry = in6_lltable_dump_entry;
llt->llt_hash = in6_lltable_hash;
llt->llt_fill_sa_entry = in6_lltable_fill_sa_entry;
llt->llt_free_entry = in6_lltable_free_entry;
llt->llt_match_prefix = in6_lltable_match_prefix;
llt->llt_mark_used = in6_lltable_mark_used;
lltable_link(llt);
return (llt);
}
void *
in6_domifattach(struct ifnet *ifp)
{
struct in6_ifextra *ext;
/* There are not IPv6-capable interfaces. */
switch (ifp->if_type) {
case IFT_PFLOG:
case IFT_PFSYNC:
case IFT_USB:
return (NULL);
}
ext = (struct in6_ifextra *)malloc(sizeof(*ext), M_IFADDR, M_WAITOK);
bzero(ext, sizeof(*ext));
ext->in6_ifstat = malloc(sizeof(counter_u64_t) *
sizeof(struct in6_ifstat) / sizeof(uint64_t), M_IFADDR, M_WAITOK);
COUNTER_ARRAY_ALLOC(ext->in6_ifstat,
sizeof(struct in6_ifstat) / sizeof(uint64_t), M_WAITOK);
ext->icmp6_ifstat = malloc(sizeof(counter_u64_t) *
sizeof(struct icmp6_ifstat) / sizeof(uint64_t), M_IFADDR,
M_WAITOK);
COUNTER_ARRAY_ALLOC(ext->icmp6_ifstat,
sizeof(struct icmp6_ifstat) / sizeof(uint64_t), M_WAITOK);
ext->nd_ifinfo = nd6_ifattach(ifp);
ext->scope6_id = scope6_ifattach(ifp);
ext->lltable = in6_lltattach(ifp);
Bite the bullet, and make the IPv6 SSM and MLDv2 mega-commit: import from p4 bms_netdev. Summary of changes: * Connect netinet6/in6_mcast.c to build. The legacy KAME KPIs are mostly preserved. * Eliminate now dead code from ip6_output.c. Don't do mbuf bingo, we are not going to do RFC 2292 style CMSG tricks for multicast options as they are not required by any current IPv6 normative reference. * Refactor transports (UDP, raw_ip6) to do own mcast filtering. SCTP, TCP unaffected by this change. * Add ip6_msource, in6_msource structs to in6_var.h. * Hookup mld_ifinfo state to in6_ifextra, allocate from domifattach path. * Eliminate IN6_LOOKUP_MULTI(), it is no longer referenced. Kernel consumers which need this should use in6m_lookup(). * Refactor IPv6 socket group memberships to use a vector (like IPv4). * Update ifmcstat(8) for IPv6 SSM. * Add witness lock order for IN6_MULTI_LOCK. * Move IN6_MULTI_LOCK out of lower ip6_output()/ip6_input() paths. * Introduce IP6STAT_ADD/SUB/INC/DEC as per rwatson's IPv4 cleanup. * Update carp(4) for new IPv6 SSM KPIs. * Virtualize ip6_mrouter socket. Changes mostly localized to IPv6 MROUTING. * Don't do a local group lookup in MROUTING. * Kill unused KAME prototypes in6_purgemkludge(), in6_restoremkludge(). * Preserve KAME DAD timer jitter behaviour in MLDv1 compatibility mode. * Bump __FreeBSD_version to 800084. * Update UPDATING. NOTE WELL: * This code hasn't been tested against real MLDv2 queriers (yet), although the on-wire protocol has been verified in Wireshark. * There are a few unresolved issues in the socket layer APIs to do with scope ID propagation. * There is a LOR present in ip6_output()'s use of in6_setscope() which needs to be resolved. See comments in mld6.c. This is believed to be benign and can't be avoided for the moment without re-introducing an indirect netisr. This work was mostly derived from the IGMPv3 implementation, and has been sponsored by a third party.
2009-04-29 19:19:13 +00:00
ext->mld_ifinfo = mld_domifattach(ifp);
return ext;
}
Make checks for rt_mtu generic: Some virtual if drivers has (ab)used ifa ifa_rtrequest hook to enforce route MTU to be not bigger that interface MTU. While ifa_rtrequest hooking might be an option in some situation, it is not feasible to do MTU checks there: generic (or per-domain) routing code is perfectly capable of doing this. We currrently have 3 places where MTU is altered: 1) route addition. In this case domain overrides radix _addroute callback (in[6]_addroute) and all necessary checks/fixes are/can be done there. 2) route change (especially, GW change). In this case, there are no explicit per-domain calls, but one can override rte by setting ifa_rtrequest hook to domain handler (inet6 does this). 3) ifconfig ifaceX mtu YYYY In this case, we have no callbacks, but ip[6]_output performes runtime checks and decreases rt_mtu if necessary. Generally, the goals are to be able to handle all MTU changes in control plane, not in runtime part, and properly deal with increased interface MTU. This commit changes the following: * removes hooks setting MTU from drivers side * adds proper per-doman MTU checks for case 1) * adds generic MTU check for case 2) * The latter is done by using new dom_ifmtu callback since if_mtu denotes L3 interface MTU, e.g. maximum trasmitted _packet_ size. However, IPv6 mtu might be different from if_mtu one (e.g. default 1280) for some cases, so we need an abstract way to know maximum MTU size for given interface and domain. * moves rt_setmetrics() before MTU/ifa_rtrequest hooks since it copies user-supplied data which must be checked. * removes RT_LOCK_ASSERT() from other ifa_rtrequest hooks to be able to use this functions on new non-inserted rte. More changes will follow soon. MFC after: 1 month Sponsored by: Yandex LLC
2014-11-06 13:13:09 +00:00
int
in6_domifmtu(struct ifnet *ifp)
{
if (ifp->if_afdata[AF_INET6] == NULL)
return ifp->if_mtu;
Make checks for rt_mtu generic: Some virtual if drivers has (ab)used ifa ifa_rtrequest hook to enforce route MTU to be not bigger that interface MTU. While ifa_rtrequest hooking might be an option in some situation, it is not feasible to do MTU checks there: generic (or per-domain) routing code is perfectly capable of doing this. We currrently have 3 places where MTU is altered: 1) route addition. In this case domain overrides radix _addroute callback (in[6]_addroute) and all necessary checks/fixes are/can be done there. 2) route change (especially, GW change). In this case, there are no explicit per-domain calls, but one can override rte by setting ifa_rtrequest hook to domain handler (inet6 does this). 3) ifconfig ifaceX mtu YYYY In this case, we have no callbacks, but ip[6]_output performes runtime checks and decreases rt_mtu if necessary. Generally, the goals are to be able to handle all MTU changes in control plane, not in runtime part, and properly deal with increased interface MTU. This commit changes the following: * removes hooks setting MTU from drivers side * adds proper per-doman MTU checks for case 1) * adds generic MTU check for case 2) * The latter is done by using new dom_ifmtu callback since if_mtu denotes L3 interface MTU, e.g. maximum trasmitted _packet_ size. However, IPv6 mtu might be different from if_mtu one (e.g. default 1280) for some cases, so we need an abstract way to know maximum MTU size for given interface and domain. * moves rt_setmetrics() before MTU/ifa_rtrequest hooks since it copies user-supplied data which must be checked. * removes RT_LOCK_ASSERT() from other ifa_rtrequest hooks to be able to use this functions on new non-inserted rte. More changes will follow soon. MFC after: 1 month Sponsored by: Yandex LLC
2014-11-06 13:13:09 +00:00
return (IN6_LINKMTU(ifp));
}
void
in6_domifdetach(struct ifnet *ifp, void *aux)
{
struct in6_ifextra *ext = (struct in6_ifextra *)aux;
Bite the bullet, and make the IPv6 SSM and MLDv2 mega-commit: import from p4 bms_netdev. Summary of changes: * Connect netinet6/in6_mcast.c to build. The legacy KAME KPIs are mostly preserved. * Eliminate now dead code from ip6_output.c. Don't do mbuf bingo, we are not going to do RFC 2292 style CMSG tricks for multicast options as they are not required by any current IPv6 normative reference. * Refactor transports (UDP, raw_ip6) to do own mcast filtering. SCTP, TCP unaffected by this change. * Add ip6_msource, in6_msource structs to in6_var.h. * Hookup mld_ifinfo state to in6_ifextra, allocate from domifattach path. * Eliminate IN6_LOOKUP_MULTI(), it is no longer referenced. Kernel consumers which need this should use in6m_lookup(). * Refactor IPv6 socket group memberships to use a vector (like IPv4). * Update ifmcstat(8) for IPv6 SSM. * Add witness lock order for IN6_MULTI_LOCK. * Move IN6_MULTI_LOCK out of lower ip6_output()/ip6_input() paths. * Introduce IP6STAT_ADD/SUB/INC/DEC as per rwatson's IPv4 cleanup. * Update carp(4) for new IPv6 SSM KPIs. * Virtualize ip6_mrouter socket. Changes mostly localized to IPv6 MROUTING. * Don't do a local group lookup in MROUTING. * Kill unused KAME prototypes in6_purgemkludge(), in6_restoremkludge(). * Preserve KAME DAD timer jitter behaviour in MLDv1 compatibility mode. * Bump __FreeBSD_version to 800084. * Update UPDATING. NOTE WELL: * This code hasn't been tested against real MLDv2 queriers (yet), although the on-wire protocol has been verified in Wireshark. * There are a few unresolved issues in the socket layer APIs to do with scope ID propagation. * There is a LOR present in ip6_output()'s use of in6_setscope() which needs to be resolved. See comments in mld6.c. This is believed to be benign and can't be avoided for the moment without re-introducing an indirect netisr. This work was mostly derived from the IGMPv3 implementation, and has been sponsored by a third party.
2009-04-29 19:19:13 +00:00
mld_domifdetach(ifp);
scope6_ifdetach(ext->scope6_id);
Get closer to a VIMAGE network stack teardown from top to bottom rather than removing the network interfaces first. This change is rather larger and convoluted as the ordering requirements cannot be separated. Move the pfil(9) framework to SI_SUB_PROTO_PFIL, move Firewalls and related modules to their own SI_SUB_PROTO_FIREWALL. Move initialization of "physical" interfaces to SI_SUB_DRIVERS, move virtual (cloned) interfaces to SI_SUB_PSEUDO. Move Multicast to SI_SUB_PROTO_MC. Re-work parts of multicast initialisation and teardown, not taking the huge amount of memory into account if used as a module yet. For interface teardown we try to do as many of them as we can on SI_SUB_INIT_IF, but for some this makes no sense, e.g., when tunnelling over a higher layer protocol such as IP. In that case the interface has to go along (or before) the higher layer protocol is shutdown. Kernel hhooks need to go last on teardown as they may be used at various higher layers and we cannot remove them before we cleaned up the higher layers. For interface teardown there are multiple paths: (a) a cloned interface is destroyed (inside a VIMAGE or in the base system), (b) any interface is moved from a virtual network stack to a different network stack ("vmove"), or (c) a virtual network stack is being shut down. All code paths go through if_detach_internal() where we, depending on the vmove flag or the vnet state, make a decision on how much to shut down; in case we are destroying a VNET the individual protocol layers will cleanup their own parts thus we cannot do so again for each interface as we end up with, e.g., double-frees, destroying locks twice or acquiring already destroyed locks. When calling into protocol cleanups we equally have to tell them whether they need to detach upper layer protocols ("ulp") or not (e.g., in6_ifdetach()). Provide or enahnce helper functions to do proper cleanup at a protocol rather than at an interface level. Approved by: re (hrs) Obtained from: projects/vnet Reviewed by: gnn, jhb Sponsored by: The FreeBSD Foundation MFC after: 2 weeks Differential Revision: https://reviews.freebsd.org/D6747
2016-06-21 13:48:49 +00:00
nd6_ifdetach(ifp, ext->nd_ifinfo);
lltable_free(ext->lltable);
COUNTER_ARRAY_FREE(ext->in6_ifstat,
sizeof(struct in6_ifstat) / sizeof(uint64_t));
free(ext->in6_ifstat, M_IFADDR);
COUNTER_ARRAY_FREE(ext->icmp6_ifstat,
sizeof(struct icmp6_ifstat) / sizeof(uint64_t));
free(ext->icmp6_ifstat, M_IFADDR);
free(ext, M_IFADDR);
}
/*
* Convert sockaddr_in6 to sockaddr_in. Original sockaddr_in6 must be
* v4 mapped addr or v4 compat addr
*/
void
in6_sin6_2_sin(struct sockaddr_in *sin, struct sockaddr_in6 *sin6)
{
bzero(sin, sizeof(*sin));
sin->sin_len = sizeof(struct sockaddr_in);
sin->sin_family = AF_INET;
sin->sin_port = sin6->sin6_port;
sin->sin_addr.s_addr = sin6->sin6_addr.s6_addr32[3];
}
/* Convert sockaddr_in to sockaddr_in6 in v4 mapped addr format. */
void
in6_sin_2_v4mapsin6(struct sockaddr_in *sin, struct sockaddr_in6 *sin6)
{
bzero(sin6, sizeof(*sin6));
sin6->sin6_len = sizeof(struct sockaddr_in6);
sin6->sin6_family = AF_INET6;
sin6->sin6_port = sin->sin_port;
sin6->sin6_addr.s6_addr32[0] = 0;
sin6->sin6_addr.s6_addr32[1] = 0;
sin6->sin6_addr.s6_addr32[2] = IPV6_ADDR_INT32_SMP;
sin6->sin6_addr.s6_addr32[3] = sin->sin_addr.s_addr;
}
/* Convert sockaddr_in6 into sockaddr_in. */
void
in6_sin6_2_sin_in_sock(struct sockaddr *nam)
{
struct sockaddr_in *sin_p;
struct sockaddr_in6 sin6;
/*
* Save original sockaddr_in6 addr and convert it
* to sockaddr_in.
*/
sin6 = *(struct sockaddr_in6 *)nam;
sin_p = (struct sockaddr_in *)nam;
in6_sin6_2_sin(sin_p, &sin6);
}
/* Convert sockaddr_in into sockaddr_in6 in v4 mapped addr format. */
void
in6_sin_2_v4mapsin6_in_sock(struct sockaddr **nam)
{
struct sockaddr_in *sin_p;
struct sockaddr_in6 *sin6_p;
sin6_p = malloc(sizeof *sin6_p, M_SONAME, M_WAITOK);
sin_p = (struct sockaddr_in *)*nam;
in6_sin_2_v4mapsin6(sin_p, sin6_p);
free(*nam, M_SONAME);
*nam = (struct sockaddr *)sin6_p;
}