bridge: epoch-ification
Run the bridge datapath under epoch, rather than under the BRIDGE_LOCK(). We still take the BRIDGE_LOCK() whenever we insert or delete items in the relevant lists, but we use epoch callbacks to free items so that it's safe to iterate the lists without the BRIDGE_LOCK. Tests on mercat5/6 shows this increases bridge throughput significantly, from 3.7Mpps to 18.6Mpps. Reviewed by: emaste, philip, melifaro MFC after: 2 months Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D24250
This commit is contained in:
Kristof Provost
parent
a7c8533634
commit
fffd27e5f3
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=360345
@ -189,41 +189,14 @@ extern void nd6_setmtu(struct ifnet *);
|
||||
*/
|
||||
#define BRIDGE_LOCK_INIT(_sc) do { \
|
||||
mtx_init(&(_sc)->sc_mtx, "if_bridge", NULL, MTX_DEF); \
|
||||
cv_init(&(_sc)->sc_cv, "if_bridge_cv"); \
|
||||
} while (0)
|
||||
#define BRIDGE_LOCK_DESTROY(_sc) do { \
|
||||
mtx_destroy(&(_sc)->sc_mtx); \
|
||||
cv_destroy(&(_sc)->sc_cv); \
|
||||
} while (0)
|
||||
#define BRIDGE_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx)
|
||||
#define BRIDGE_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx)
|
||||
#define BRIDGE_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->sc_mtx, MA_OWNED)
|
||||
#define BRIDGE_UNLOCK_ASSERT(_sc) mtx_assert(&(_sc)->sc_mtx, MA_NOTOWNED)
|
||||
#define BRIDGE_LOCK2REF(_sc, _err) do { \
|
||||
mtx_assert(&(_sc)->sc_mtx, MA_OWNED); \
|
||||
if ((_sc)->sc_iflist_xcnt > 0) \
|
||||
(_err) = EBUSY; \
|
||||
else \
|
||||
(_sc)->sc_iflist_ref++; \
|
||||
mtx_unlock(&(_sc)->sc_mtx); \
|
||||
} while (0)
|
||||
#define BRIDGE_UNREF(_sc) do { \
|
||||
mtx_lock(&(_sc)->sc_mtx); \
|
||||
(_sc)->sc_iflist_ref--; \
|
||||
if (((_sc)->sc_iflist_xcnt > 0) && ((_sc)->sc_iflist_ref == 0)) \
|
||||
cv_broadcast(&(_sc)->sc_cv); \
|
||||
mtx_unlock(&(_sc)->sc_mtx); \
|
||||
} while (0)
|
||||
#define BRIDGE_XLOCK(_sc) do { \
|
||||
mtx_assert(&(_sc)->sc_mtx, MA_OWNED); \
|
||||
(_sc)->sc_iflist_xcnt++; \
|
||||
while ((_sc)->sc_iflist_ref > 0) \
|
||||
cv_wait(&(_sc)->sc_cv, &(_sc)->sc_mtx); \
|
||||
} while (0)
|
||||
#define BRIDGE_XDROP(_sc) do { \
|
||||
mtx_assert(&(_sc)->sc_mtx, MA_OWNED); \
|
||||
(_sc)->sc_iflist_xcnt--; \
|
||||
} while (0)
|
||||
|
||||
/*
|
||||
* Bridge interface list entry.
|
||||
@ -237,6 +210,7 @@ struct bridge_iflist {
|
||||
uint32_t bif_addrmax; /* max # of addresses */
|
||||
uint32_t bif_addrcnt; /* cur. # of addresses */
|
||||
uint32_t bif_addrexceeded;/* # of address violations */
|
||||
struct epoch_context bif_epoch_ctx;
|
||||
};
|
||||
|
||||
/*
|
||||
@ -250,6 +224,8 @@ struct bridge_rtnode {
|
||||
uint8_t brt_flags; /* address flags */
|
||||
uint8_t brt_addr[ETHER_ADDR_LEN];
|
||||
uint16_t brt_vlan; /* vlan id */
|
||||
struct vnet *brt_vnet;
|
||||
struct epoch_context brt_epoch_ctx;
|
||||
};
|
||||
#define brt_ifp brt_dst->bif_ifp
|
||||
|
||||
@ -260,13 +236,10 @@ struct bridge_softc {
|
||||
struct ifnet *sc_ifp; /* make this an interface */
|
||||
LIST_ENTRY(bridge_softc) sc_list;
|
||||
struct mtx sc_mtx;
|
||||
struct cv sc_cv;
|
||||
uint32_t sc_brtmax; /* max # of addresses */
|
||||
uint32_t sc_brtcnt; /* cur. # of addresses */
|
||||
uint32_t sc_brttimeout; /* rt timeout in seconds */
|
||||
struct callout sc_brcallout; /* bridge callout */
|
||||
uint32_t sc_iflist_ref; /* refcount for sc_iflist */
|
||||
uint32_t sc_iflist_xcnt; /* refcount for sc_iflist */
|
||||
CK_LIST_HEAD(, bridge_iflist) sc_iflist; /* member interface list */
|
||||
CK_LIST_HEAD(, bridge_rtnode) *sc_rthash; /* our forwarding table */
|
||||
CK_LIST_HEAD(, bridge_rtnode) sc_rtlist; /* list version of above */
|
||||
@ -276,6 +249,7 @@ struct bridge_softc {
|
||||
uint32_t sc_brtexceeded; /* # of cache drops */
|
||||
struct ifnet *sc_ifaddr; /* member mac copied from */
|
||||
struct ether_addr sc_defaddr; /* Default MAC address */
|
||||
struct epoch_context sc_epoch_ctx;
|
||||
};
|
||||
|
||||
VNET_DEFINE_STATIC(struct mtx, bridge_list_mtx);
|
||||
@ -596,6 +570,10 @@ vnet_bridge_uninit(const void *unused __unused)
|
||||
if_clone_detach(V_bridge_cloner);
|
||||
V_bridge_cloner = NULL;
|
||||
BRIDGE_LIST_LOCK_DESTROY();
|
||||
|
||||
/* Callbacks may use the UMA zone. */
|
||||
epoch_drain_callbacks(net_epoch_preempt);
|
||||
|
||||
uma_zdestroy(V_bridge_rtnode_zone);
|
||||
}
|
||||
VNET_SYSUNINIT(vnet_bridge_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY,
|
||||
@ -722,6 +700,17 @@ bridge_clone_create(struct if_clone *ifc, int unit, caddr_t params)
|
||||
return (0);
|
||||
}
|
||||
|
||||
static void
|
||||
bridge_clone_destroy_cb(struct epoch_context *ctx)
|
||||
{
|
||||
struct bridge_softc *sc;
|
||||
|
||||
sc = __containerof(ctx, struct bridge_softc, sc_epoch_ctx);
|
||||
|
||||
BRIDGE_LOCK_DESTROY(sc);
|
||||
free(sc, M_DEVBUF);
|
||||
}
|
||||
|
||||
/*
|
||||
* bridge_clone_destroy:
|
||||
*
|
||||
@ -732,7 +721,9 @@ bridge_clone_destroy(struct ifnet *ifp)
|
||||
{
|
||||
struct bridge_softc *sc = ifp->if_softc;
|
||||
struct bridge_iflist *bif;
|
||||
struct epoch_tracker et;
|
||||
|
||||
NET_EPOCH_ENTER(et);
|
||||
BRIDGE_LOCK(sc);
|
||||
|
||||
bridge_stop(ifp, 1);
|
||||
@ -757,11 +748,12 @@ bridge_clone_destroy(struct ifnet *ifp)
|
||||
BRIDGE_LIST_UNLOCK();
|
||||
|
||||
bstp_detach(&sc->sc_stp);
|
||||
NET_EPOCH_EXIT(et);
|
||||
|
||||
ether_ifdetach(ifp);
|
||||
if_free(ifp);
|
||||
|
||||
BRIDGE_LOCK_DESTROY(sc);
|
||||
free(sc, M_DEVBUF);
|
||||
NET_EPOCH_CALL(bridge_clone_destroy_cb, &sc->sc_epoch_ctx);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -787,6 +779,9 @@ bridge_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
|
||||
struct ifdrv *ifd = (struct ifdrv *) data;
|
||||
const struct bridge_control *bc;
|
||||
int error = 0, oldmtu;
|
||||
struct epoch_tracker et;
|
||||
|
||||
NET_EPOCH_ENTER(et);
|
||||
|
||||
switch (cmd) {
|
||||
|
||||
@ -908,6 +903,8 @@ bridge_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
|
||||
break;
|
||||
}
|
||||
|
||||
NET_EPOCH_EXIT(et);
|
||||
|
||||
return (error);
|
||||
}
|
||||
|
||||
@ -922,6 +919,8 @@ bridge_mutecaps(struct bridge_softc *sc)
|
||||
struct bridge_iflist *bif;
|
||||
int enabled, mask;
|
||||
|
||||
BRIDGE_LOCK_ASSERT(sc);
|
||||
|
||||
/* Initial bitmask of capabilities to test */
|
||||
mask = BRIDGE_IFCAPS_MASK;
|
||||
|
||||
@ -930,7 +929,6 @@ bridge_mutecaps(struct bridge_softc *sc)
|
||||
mask &= bif->bif_savedcaps;
|
||||
}
|
||||
|
||||
BRIDGE_XLOCK(sc);
|
||||
CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) {
|
||||
enabled = bif->bif_ifp->if_capenable;
|
||||
enabled &= ~BRIDGE_IFCAPS_STRIP;
|
||||
@ -941,8 +939,6 @@ bridge_mutecaps(struct bridge_softc *sc)
|
||||
bridge_set_ifcap(sc, bif, enabled);
|
||||
BRIDGE_LOCK(sc);
|
||||
}
|
||||
BRIDGE_XDROP(sc);
|
||||
|
||||
}
|
||||
|
||||
static void
|
||||
@ -983,7 +979,7 @@ bridge_lookup_member(struct bridge_softc *sc, const char *name)
|
||||
struct bridge_iflist *bif;
|
||||
struct ifnet *ifp;
|
||||
|
||||
BRIDGE_LOCK_ASSERT(sc);
|
||||
NET_EPOCH_ASSERT();
|
||||
|
||||
CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) {
|
||||
ifp = bif->bif_ifp;
|
||||
@ -1004,7 +1000,7 @@ bridge_lookup_member_if(struct bridge_softc *sc, struct ifnet *member_ifp)
|
||||
{
|
||||
struct bridge_iflist *bif;
|
||||
|
||||
BRIDGE_LOCK_ASSERT(sc);
|
||||
NET_EPOCH_ASSERT();
|
||||
|
||||
CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) {
|
||||
if (bif->bif_ifp == member_ifp)
|
||||
@ -1014,6 +1010,16 @@ bridge_lookup_member_if(struct bridge_softc *sc, struct ifnet *member_ifp)
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
static void
|
||||
bridge_delete_member_cb(struct epoch_context *ctx)
|
||||
{
|
||||
struct bridge_iflist *bif;
|
||||
|
||||
bif = __containerof(ctx, struct bridge_iflist, bif_epoch_ctx);
|
||||
|
||||
free(bif, M_DEVBUF);
|
||||
}
|
||||
|
||||
/*
|
||||
* bridge_delete_member:
|
||||
*
|
||||
@ -1033,9 +1039,7 @@ bridge_delete_member(struct bridge_softc *sc, struct bridge_iflist *bif,
|
||||
bstp_disable(&bif->bif_stp);
|
||||
|
||||
ifs->if_bridge = NULL;
|
||||
BRIDGE_XLOCK(sc);
|
||||
CK_LIST_REMOVE(bif, bif_next);
|
||||
BRIDGE_XDROP(sc);
|
||||
|
||||
/*
|
||||
* If removing the interface that gave the bridge its mac address, set
|
||||
@ -1094,7 +1098,8 @@ bridge_delete_member(struct bridge_softc *sc, struct bridge_iflist *bif,
|
||||
}
|
||||
bstp_destroy(&bif->bif_stp); /* prepare to free */
|
||||
BRIDGE_LOCK(sc);
|
||||
free(bif, M_DEVBUF);
|
||||
|
||||
NET_EPOCH_CALL(bridge_delete_member_cb, &bif->bif_epoch_ctx);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -1111,7 +1116,8 @@ bridge_delete_span(struct bridge_softc *sc, struct bridge_iflist *bif)
|
||||
("%s: not a span interface", __func__));
|
||||
|
||||
CK_LIST_REMOVE(bif, bif_next);
|
||||
free(bif, M_DEVBUF);
|
||||
|
||||
NET_EPOCH_CALL(bridge_delete_member_cb, &bif->bif_epoch_ctx);
|
||||
}
|
||||
|
||||
static int
|
||||
@ -1167,7 +1173,6 @@ bridge_ioctl_add(struct bridge_softc *sc, void *arg)
|
||||
* If any, remove all inet6 addresses from the member
|
||||
* interfaces.
|
||||
*/
|
||||
BRIDGE_XLOCK(sc);
|
||||
CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) {
|
||||
if (in6ifa_llaonifp(bif->bif_ifp)) {
|
||||
BRIDGE_UNLOCK(sc);
|
||||
@ -1180,7 +1185,6 @@ bridge_ioctl_add(struct bridge_softc *sc, void *arg)
|
||||
bif->bif_ifp->if_xname);
|
||||
}
|
||||
}
|
||||
BRIDGE_XDROP(sc);
|
||||
if (in6ifa_llaonifp(ifs)) {
|
||||
BRIDGE_UNLOCK(sc);
|
||||
in6_ifdetach(ifs);
|
||||
@ -1494,12 +1498,17 @@ bridge_ioctl_saddr(struct bridge_softc *sc, void *arg)
|
||||
struct bridge_iflist *bif;
|
||||
int error;
|
||||
|
||||
NET_EPOCH_ASSERT();
|
||||
|
||||
bif = bridge_lookup_member(sc, req->ifba_ifsname);
|
||||
if (bif == NULL)
|
||||
return (ENOENT);
|
||||
|
||||
/* bridge_rtupdate() may acquire the lock. */
|
||||
BRIDGE_UNLOCK(sc);
|
||||
error = bridge_rtupdate(sc, req->ifba_dst, req->ifba_vlan, bif, 1,
|
||||
req->ifba_flags);
|
||||
BRIDGE_LOCK(sc);
|
||||
|
||||
return (error);
|
||||
}
|
||||
@ -1838,6 +1847,7 @@ bridge_ifdetach(void *arg __unused, struct ifnet *ifp)
|
||||
{
|
||||
struct bridge_softc *sc = ifp->if_bridge;
|
||||
struct bridge_iflist *bif;
|
||||
struct epoch_tracker et;
|
||||
|
||||
if (ifp->if_flags & IFF_RENAMING)
|
||||
return;
|
||||
@ -1848,6 +1858,7 @@ bridge_ifdetach(void *arg __unused, struct ifnet *ifp)
|
||||
*/
|
||||
return;
|
||||
}
|
||||
NET_EPOCH_ENTER(et);
|
||||
/* Check if the interface is a bridge member */
|
||||
if (sc != NULL) {
|
||||
BRIDGE_LOCK(sc);
|
||||
@ -1857,6 +1868,7 @@ bridge_ifdetach(void *arg __unused, struct ifnet *ifp)
|
||||
bridge_delete_member(sc, bif, 1);
|
||||
|
||||
BRIDGE_UNLOCK(sc);
|
||||
NET_EPOCH_EXIT(et);
|
||||
return;
|
||||
}
|
||||
|
||||
@ -1873,6 +1885,7 @@ bridge_ifdetach(void *arg __unused, struct ifnet *ifp)
|
||||
BRIDGE_UNLOCK(sc);
|
||||
}
|
||||
BRIDGE_LIST_UNLOCK();
|
||||
NET_EPOCH_EXIT(et);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -1909,6 +1922,7 @@ bridge_stop(struct ifnet *ifp, int disable)
|
||||
{
|
||||
struct bridge_softc *sc = ifp->if_softc;
|
||||
|
||||
NET_EPOCH_ASSERT();
|
||||
BRIDGE_LOCK_ASSERT(sc);
|
||||
|
||||
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
|
||||
@ -2032,6 +2046,8 @@ bridge_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa,
|
||||
struct bridge_softc *sc;
|
||||
uint16_t vlan;
|
||||
|
||||
NET_EPOCH_ASSERT();
|
||||
|
||||
if (m->m_len < ETHER_HDR_LEN) {
|
||||
m = m_pullup(m, ETHER_HDR_LEN);
|
||||
if (m == NULL)
|
||||
@ -2042,7 +2058,6 @@ bridge_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa,
|
||||
sc = ifp->if_bridge;
|
||||
vlan = VLANTAGOF(m);
|
||||
|
||||
BRIDGE_LOCK(sc);
|
||||
bifp = sc->sc_ifp;
|
||||
|
||||
/*
|
||||
@ -2069,16 +2084,10 @@ bridge_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa,
|
||||
if (dst_if == NULL) {
|
||||
struct bridge_iflist *bif;
|
||||
struct mbuf *mc;
|
||||
int error = 0, used = 0;
|
||||
int used = 0;
|
||||
|
||||
bridge_span(sc, m);
|
||||
|
||||
BRIDGE_LOCK2REF(sc, error);
|
||||
if (error) {
|
||||
m_freem(m);
|
||||
return (0);
|
||||
}
|
||||
|
||||
CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) {
|
||||
dst_if = bif->bif_ifp;
|
||||
|
||||
@ -2112,7 +2121,6 @@ bridge_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa,
|
||||
}
|
||||
if (used == 0)
|
||||
m_freem(m);
|
||||
BRIDGE_UNREF(sc);
|
||||
return (0);
|
||||
}
|
||||
|
||||
@ -2124,11 +2132,9 @@ bridge_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa,
|
||||
bridge_span(sc, m);
|
||||
if ((dst_if->if_drv_flags & IFF_DRV_RUNNING) == 0) {
|
||||
m_freem(m);
|
||||
BRIDGE_UNLOCK(sc);
|
||||
return (0);
|
||||
}
|
||||
|
||||
BRIDGE_UNLOCK(sc);
|
||||
bridge_enqueue(sc, dst_if, m);
|
||||
return (0);
|
||||
}
|
||||
@ -2153,10 +2159,8 @@ bridge_transmit(struct ifnet *ifp, struct mbuf *m)
|
||||
|
||||
eh = mtod(m, struct ether_header *);
|
||||
|
||||
BRIDGE_LOCK(sc);
|
||||
if (((m->m_flags & (M_BCAST|M_MCAST)) == 0) &&
|
||||
(dst_if = bridge_rtlookup(sc, eh->ether_dhost, 1)) != NULL) {
|
||||
BRIDGE_UNLOCK(sc);
|
||||
error = bridge_enqueue(sc, dst_if, m);
|
||||
} else
|
||||
bridge_broadcast(sc, ifp, m, 0);
|
||||
@ -2190,6 +2194,8 @@ bridge_forward(struct bridge_softc *sc, struct bridge_iflist *sbif,
|
||||
uint8_t *dst;
|
||||
int error;
|
||||
|
||||
NET_EPOCH_ASSERT();
|
||||
|
||||
src_if = m->m_pkthdr.rcvif;
|
||||
ifp = sc->sc_ifp;
|
||||
|
||||
@ -2268,12 +2274,10 @@ bridge_forward(struct bridge_softc *sc, struct bridge_iflist *sbif,
|
||||
|| PFIL_HOOKED_IN(V_inet6_pfil_head)
|
||||
#endif
|
||||
) {
|
||||
BRIDGE_UNLOCK(sc);
|
||||
if (bridge_pfil(&m, ifp, src_if, PFIL_IN) != 0)
|
||||
return;
|
||||
if (m == NULL)
|
||||
return;
|
||||
BRIDGE_LOCK(sc);
|
||||
}
|
||||
|
||||
if (dst_if == NULL) {
|
||||
@ -2301,8 +2305,6 @@ bridge_forward(struct bridge_softc *sc, struct bridge_iflist *sbif,
|
||||
dbif->bif_stp.bp_state == BSTP_IFSTATE_DISCARDING)
|
||||
goto drop;
|
||||
|
||||
BRIDGE_UNLOCK(sc);
|
||||
|
||||
if (PFIL_HOOKED_OUT(V_inet_pfil_head)
|
||||
#ifdef INET6
|
||||
|| PFIL_HOOKED_OUT(V_inet6_pfil_head)
|
||||
@ -2318,7 +2320,6 @@ bridge_forward(struct bridge_softc *sc, struct bridge_iflist *sbif,
|
||||
return;
|
||||
|
||||
drop:
|
||||
BRIDGE_UNLOCK(sc);
|
||||
m_freem(m);
|
||||
}
|
||||
|
||||
@ -2339,6 +2340,8 @@ bridge_input(struct ifnet *ifp, struct mbuf *m)
|
||||
uint16_t vlan;
|
||||
int error;
|
||||
|
||||
NET_EPOCH_ASSERT();
|
||||
|
||||
if ((sc->sc_ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
|
||||
return (m);
|
||||
|
||||
@ -2359,10 +2362,8 @@ bridge_input(struct ifnet *ifp, struct mbuf *m)
|
||||
m_freem(m);
|
||||
return (NULL);
|
||||
}
|
||||
BRIDGE_LOCK(sc);
|
||||
bif = bridge_lookup_member_if(sc, ifp);
|
||||
if (bif == NULL) {
|
||||
BRIDGE_UNLOCK(sc);
|
||||
return (m);
|
||||
}
|
||||
|
||||
@ -2375,13 +2376,11 @@ bridge_input(struct ifnet *ifp, struct mbuf *m)
|
||||
if (memcmp(eh->ether_dhost, bstp_etheraddr,
|
||||
ETHER_ADDR_LEN) == 0) {
|
||||
bstp_input(&bif->bif_stp, ifp, m); /* consumes mbuf */
|
||||
BRIDGE_UNLOCK(sc);
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
if ((bif->bif_flags & IFBIF_STP) &&
|
||||
bif->bif_stp.bp_state == BSTP_IFSTATE_DISCARDING) {
|
||||
BRIDGE_UNLOCK(sc);
|
||||
return (m);
|
||||
}
|
||||
|
||||
@ -2392,7 +2391,6 @@ bridge_input(struct ifnet *ifp, struct mbuf *m)
|
||||
*/
|
||||
mc = m_dup(m, M_NOWAIT);
|
||||
if (mc == NULL) {
|
||||
BRIDGE_UNLOCK(sc);
|
||||
return (m);
|
||||
}
|
||||
|
||||
@ -2424,7 +2422,6 @@ bridge_input(struct ifnet *ifp, struct mbuf *m)
|
||||
|
||||
if ((bif->bif_flags & IFBIF_STP) &&
|
||||
bif->bif_stp.bp_state == BSTP_IFSTATE_DISCARDING) {
|
||||
BRIDGE_UNLOCK(sc);
|
||||
return (m);
|
||||
}
|
||||
|
||||
@ -2458,7 +2455,6 @@ bridge_input(struct ifnet *ifp, struct mbuf *m)
|
||||
error = bridge_rtupdate(sc, eh->ether_shost, \
|
||||
vlan, bif, 0, IFBAF_DYNAMIC); \
|
||||
if (error && bif->bif_addrmax) { \
|
||||
BRIDGE_UNLOCK(sc); \
|
||||
m_freem(m); \
|
||||
return (NULL); \
|
||||
} \
|
||||
@ -2466,7 +2462,6 @@ bridge_input(struct ifnet *ifp, struct mbuf *m)
|
||||
m->m_pkthdr.rcvif = iface; \
|
||||
if ((iface) == ifp) { \
|
||||
/* Skip bridge processing... src == dest */ \
|
||||
BRIDGE_UNLOCK(sc); \
|
||||
return (m); \
|
||||
} \
|
||||
/* It's passing over or to the bridge, locally. */ \
|
||||
@ -2478,13 +2473,11 @@ bridge_input(struct ifnet *ifp, struct mbuf *m)
|
||||
OR_PFIL_HOOKED_INET6)) { \
|
||||
if (bridge_pfil(&m, NULL, ifp, \
|
||||
PFIL_IN) != 0 || m == NULL) { \
|
||||
BRIDGE_UNLOCK(sc); \
|
||||
return (NULL); \
|
||||
} \
|
||||
} \
|
||||
if ((iface) != bifp) \
|
||||
ETHER_BPF_MTAP(iface, m); \
|
||||
BRIDGE_UNLOCK(sc); \
|
||||
return (m); \
|
||||
} \
|
||||
\
|
||||
@ -2492,7 +2485,6 @@ bridge_input(struct ifnet *ifp, struct mbuf *m)
|
||||
if (memcmp(IF_LLADDR((iface)), eh->ether_shost, ETHER_ADDR_LEN) == 0 \
|
||||
OR_CARP_CHECK_WE_ARE_SRC((iface)) \
|
||||
) { \
|
||||
BRIDGE_UNLOCK(sc); \
|
||||
m_freem(m); \
|
||||
return (NULL); \
|
||||
}
|
||||
@ -2543,16 +2535,12 @@ bridge_broadcast(struct bridge_softc *sc, struct ifnet *src_if,
|
||||
struct bridge_iflist *dbif, *sbif;
|
||||
struct mbuf *mc;
|
||||
struct ifnet *dst_if;
|
||||
int error = 0, used = 0, i;
|
||||
int used = 0, i;
|
||||
|
||||
NET_EPOCH_ASSERT();
|
||||
|
||||
sbif = bridge_lookup_member_if(sc, src_if);
|
||||
|
||||
BRIDGE_LOCK2REF(sc, error);
|
||||
if (error) {
|
||||
m_freem(m);
|
||||
return;
|
||||
}
|
||||
|
||||
/* Filter on the bridge interface before broadcasting */
|
||||
if (runfilt && (PFIL_HOOKED_OUT(V_inet_pfil_head)
|
||||
#ifdef INET6
|
||||
@ -2560,9 +2548,9 @@ bridge_broadcast(struct bridge_softc *sc, struct ifnet *src_if,
|
||||
#endif
|
||||
)) {
|
||||
if (bridge_pfil(&m, sc->sc_ifp, NULL, PFIL_OUT) != 0)
|
||||
goto out;
|
||||
return;
|
||||
if (m == NULL)
|
||||
goto out;
|
||||
return;
|
||||
}
|
||||
|
||||
CK_LIST_FOREACH(dbif, &sc->sc_iflist, bif_next) {
|
||||
@ -2625,9 +2613,6 @@ bridge_broadcast(struct bridge_softc *sc, struct ifnet *src_if,
|
||||
}
|
||||
if (used == 0)
|
||||
m_freem(m);
|
||||
|
||||
out:
|
||||
BRIDGE_UNREF(sc);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -2643,6 +2628,8 @@ bridge_span(struct bridge_softc *sc, struct mbuf *m)
|
||||
struct ifnet *dst_if;
|
||||
struct mbuf *mc;
|
||||
|
||||
NET_EPOCH_ASSERT();
|
||||
|
||||
if (CK_LIST_EMPTY(&sc->sc_spanlist))
|
||||
return;
|
||||
|
||||
@ -2674,7 +2661,8 @@ bridge_rtupdate(struct bridge_softc *sc, const uint8_t *dst, uint16_t vlan,
|
||||
struct bridge_rtnode *brt;
|
||||
int error;
|
||||
|
||||
BRIDGE_LOCK_ASSERT(sc);
|
||||
NET_EPOCH_ASSERT();
|
||||
BRIDGE_UNLOCK_ASSERT(sc);
|
||||
|
||||
/* Check the source address is valid and not multicast. */
|
||||
if (ETHER_IS_MULTICAST(dst) ||
|
||||
@ -2691,13 +2679,24 @@ bridge_rtupdate(struct bridge_softc *sc, const uint8_t *dst, uint16_t vlan,
|
||||
* update it, otherwise create a new one.
|
||||
*/
|
||||
if ((brt = bridge_rtnode_lookup(sc, dst, vlan)) == NULL) {
|
||||
BRIDGE_LOCK(sc);
|
||||
|
||||
/* Check again, now that we have the lock. There could have
|
||||
* been a race and we only want to insert this once. */
|
||||
if ((brt = bridge_rtnode_lookup(sc, dst, vlan)) != NULL) {
|
||||
BRIDGE_UNLOCK(sc);
|
||||
return (0);
|
||||
}
|
||||
|
||||
if (sc->sc_brtcnt >= sc->sc_brtmax) {
|
||||
sc->sc_brtexceeded++;
|
||||
BRIDGE_UNLOCK(sc);
|
||||
return (ENOSPC);
|
||||
}
|
||||
/* Check per interface address limits (if enabled) */
|
||||
if (bif->bif_addrmax && bif->bif_addrcnt >= bif->bif_addrmax) {
|
||||
bif->bif_addrexceeded++;
|
||||
BRIDGE_UNLOCK(sc);
|
||||
return (ENOSPC);
|
||||
}
|
||||
|
||||
@ -2707,8 +2706,11 @@ bridge_rtupdate(struct bridge_softc *sc, const uint8_t *dst, uint16_t vlan,
|
||||
* address.
|
||||
*/
|
||||
brt = uma_zalloc(V_bridge_rtnode_zone, M_NOWAIT | M_ZERO);
|
||||
if (brt == NULL)
|
||||
if (brt == NULL) {
|
||||
BRIDGE_UNLOCK(sc);
|
||||
return (ENOMEM);
|
||||
}
|
||||
brt->brt_vnet = curvnet;
|
||||
|
||||
if (bif->bif_flags & IFBIF_STICKY)
|
||||
brt->brt_flags = IFBAF_STICKY;
|
||||
@ -2720,17 +2722,22 @@ bridge_rtupdate(struct bridge_softc *sc, const uint8_t *dst, uint16_t vlan,
|
||||
|
||||
if ((error = bridge_rtnode_insert(sc, brt)) != 0) {
|
||||
uma_zfree(V_bridge_rtnode_zone, brt);
|
||||
BRIDGE_UNLOCK(sc);
|
||||
return (error);
|
||||
}
|
||||
brt->brt_dst = bif;
|
||||
bif->bif_addrcnt++;
|
||||
|
||||
BRIDGE_UNLOCK(sc);
|
||||
}
|
||||
|
||||
if ((brt->brt_flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC &&
|
||||
brt->brt_dst != bif) {
|
||||
BRIDGE_LOCK(sc);
|
||||
brt->brt_dst->bif_addrcnt--;
|
||||
brt->brt_dst = bif;
|
||||
brt->brt_dst->bif_addrcnt++;
|
||||
BRIDGE_UNLOCK(sc);
|
||||
}
|
||||
|
||||
if ((flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC)
|
||||
@ -2751,7 +2758,7 @@ bridge_rtlookup(struct bridge_softc *sc, const uint8_t *addr, uint16_t vlan)
|
||||
{
|
||||
struct bridge_rtnode *brt;
|
||||
|
||||
BRIDGE_LOCK_ASSERT(sc);
|
||||
NET_EPOCH_ASSERT();
|
||||
|
||||
if ((brt = bridge_rtnode_lookup(sc, addr, vlan)) == NULL)
|
||||
return (NULL);
|
||||
@ -2771,6 +2778,7 @@ bridge_rttrim(struct bridge_softc *sc)
|
||||
{
|
||||
struct bridge_rtnode *brt, *nbrt;
|
||||
|
||||
NET_EPOCH_ASSERT();
|
||||
BRIDGE_LOCK_ASSERT(sc);
|
||||
|
||||
/* Make sure we actually need to do this. */
|
||||
@ -2800,7 +2808,9 @@ static void
|
||||
bridge_timer(void *arg)
|
||||
{
|
||||
struct bridge_softc *sc = arg;
|
||||
struct epoch_tracker et;
|
||||
|
||||
NET_EPOCH_ENTER(et);
|
||||
BRIDGE_LOCK_ASSERT(sc);
|
||||
|
||||
/* Destruction of rtnodes requires a proper vnet context */
|
||||
@ -2811,6 +2821,7 @@ bridge_timer(void *arg)
|
||||
callout_reset(&sc->sc_brcallout,
|
||||
bridge_rtable_prune_period * hz, bridge_timer, sc);
|
||||
CURVNET_RESTORE();
|
||||
NET_EPOCH_EXIT(et);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -2823,6 +2834,7 @@ bridge_rtage(struct bridge_softc *sc)
|
||||
{
|
||||
struct bridge_rtnode *brt, *nbrt;
|
||||
|
||||
NET_EPOCH_ASSERT();
|
||||
BRIDGE_LOCK_ASSERT(sc);
|
||||
|
||||
CK_LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) {
|
||||
@ -2843,6 +2855,7 @@ bridge_rtflush(struct bridge_softc *sc, int full)
|
||||
{
|
||||
struct bridge_rtnode *brt, *nbrt;
|
||||
|
||||
NET_EPOCH_ASSERT();
|
||||
BRIDGE_LOCK_ASSERT(sc);
|
||||
|
||||
CK_LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) {
|
||||
@ -2862,6 +2875,7 @@ bridge_rtdaddr(struct bridge_softc *sc, const uint8_t *addr, uint16_t vlan)
|
||||
struct bridge_rtnode *brt;
|
||||
int found = 0;
|
||||
|
||||
NET_EPOCH_ASSERT();
|
||||
BRIDGE_LOCK_ASSERT(sc);
|
||||
|
||||
/*
|
||||
@ -2886,6 +2900,7 @@ bridge_rtdelete(struct bridge_softc *sc, struct ifnet *ifp, int full)
|
||||
{
|
||||
struct bridge_rtnode *brt, *nbrt;
|
||||
|
||||
NET_EPOCH_ASSERT();
|
||||
BRIDGE_LOCK_ASSERT(sc);
|
||||
|
||||
CK_LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) {
|
||||
@ -2990,7 +3005,7 @@ bridge_rtnode_lookup(struct bridge_softc *sc, const uint8_t *addr, uint16_t vlan
|
||||
uint32_t hash;
|
||||
int dir;
|
||||
|
||||
BRIDGE_LOCK_ASSERT(sc);
|
||||
NET_EPOCH_ASSERT();
|
||||
|
||||
hash = bridge_rthash(sc, addr);
|
||||
CK_LIST_FOREACH(brt, &sc->sc_rthash[hash], brt_hash) {
|
||||
@ -3053,6 +3068,18 @@ bridge_rtnode_insert(struct bridge_softc *sc, struct bridge_rtnode *brt)
|
||||
return (0);
|
||||
}
|
||||
|
||||
static void
|
||||
bridge_rtnode_destroy_cb(struct epoch_context *ctx)
|
||||
{
|
||||
struct bridge_rtnode *brt;
|
||||
|
||||
brt = __containerof(ctx, struct bridge_rtnode, brt_epoch_ctx);
|
||||
|
||||
CURVNET_SET(brt->brt_vnet);
|
||||
uma_zfree(V_bridge_rtnode_zone, brt);
|
||||
CURVNET_RESTORE();
|
||||
}
|
||||
|
||||
/*
|
||||
* bridge_rtnode_destroy:
|
||||
*
|
||||
@ -3061,6 +3088,7 @@ bridge_rtnode_insert(struct bridge_softc *sc, struct bridge_rtnode *brt)
|
||||
static void
|
||||
bridge_rtnode_destroy(struct bridge_softc *sc, struct bridge_rtnode *brt)
|
||||
{
|
||||
NET_EPOCH_ASSERT();
|
||||
BRIDGE_LOCK_ASSERT(sc);
|
||||
|
||||
CK_LIST_REMOVE(brt, brt_hash);
|
||||
@ -3068,7 +3096,8 @@ bridge_rtnode_destroy(struct bridge_softc *sc, struct bridge_rtnode *brt)
|
||||
CK_LIST_REMOVE(brt, brt_list);
|
||||
sc->sc_brtcnt--;
|
||||
brt->brt_dst->bif_addrcnt--;
|
||||
uma_zfree(V_bridge_rtnode_zone, brt);
|
||||
|
||||
NET_EPOCH_CALL(bridge_rtnode_destroy_cb, &brt->brt_epoch_ctx);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -3081,7 +3110,9 @@ bridge_rtable_expire(struct ifnet *ifp, int age)
|
||||
{
|
||||
struct bridge_softc *sc = ifp->if_bridge;
|
||||
struct bridge_rtnode *brt;
|
||||
struct epoch_tracker et;
|
||||
|
||||
NET_EPOCH_ENTER(et);
|
||||
CURVNET_SET(ifp->if_vnet);
|
||||
BRIDGE_LOCK(sc);
|
||||
|
||||
@ -3102,6 +3133,7 @@ bridge_rtable_expire(struct ifnet *ifp, int age)
|
||||
}
|
||||
BRIDGE_UNLOCK(sc);
|
||||
CURVNET_RESTORE();
|
||||
NET_EPOCH_EXIT(et);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -3607,17 +3639,20 @@ bridge_linkstate(struct ifnet *ifp)
|
||||
{
|
||||
struct bridge_softc *sc = ifp->if_bridge;
|
||||
struct bridge_iflist *bif;
|
||||
struct epoch_tracker et;
|
||||
|
||||
NET_EPOCH_ENTER(et);
|
||||
|
||||
BRIDGE_LOCK(sc);
|
||||
bif = bridge_lookup_member_if(sc, ifp);
|
||||
if (bif == NULL) {
|
||||
BRIDGE_UNLOCK(sc);
|
||||
NET_EPOCH_EXIT(et);
|
||||
return;
|
||||
}
|
||||
bridge_linkcheck(sc);
|
||||
BRIDGE_UNLOCK(sc);
|
||||
|
||||
bstp_linkstate(&bif->bif_stp);
|
||||
|
||||
NET_EPOCH_EXIT(et);
|
||||
}
|
||||
|
||||
static void
|
||||
@ -3626,7 +3661,8 @@ bridge_linkcheck(struct bridge_softc *sc)
|
||||
struct bridge_iflist *bif;
|
||||
int new_link, hasls;
|
||||
|
||||
BRIDGE_LOCK_ASSERT(sc);
|
||||
NET_EPOCH_ASSERT();
|
||||
|
||||
new_link = LINK_STATE_DOWN;
|
||||
hasls = 0;
|
||||
/* Our link is considered up if at least one of our ports is active */
|
||||
|
||||
Loading…
Reference in New Issue
Block a user