d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use less-invasive approach for IF_AFDATA lock: convert into 2 locks:

Browse Source 
use rwlock accessible via external functions
    (IF_AFDATA_CFG_* -> if_afdata_cfg_*()) for all control plane tasks
  use rmlock (IF_AFDATA_RUN_*) for fast-path lookups.
This commit is contained in:
Alexander V. Chernikov 2014-11-22 19:53:36 +00:00
parent 27688dfe1d
commit 7c066c18db
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/projects/routing/; revision=274887
6 changed files with 142 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
sys/net/if.c
View File

@ -51,6 +51,7 @@
#include <sys/lock.h>
#include <sys/refcount.h>
#include <sys/module.h>
#include <sys/rmlock.h>
#include <sys/rwlock.h>
#include <sys/sockio.h>
#include <sys/syslog.h>
@ -773,7 +774,7 @@ if_attachdomain1(struct ifnet *ifp)
* Since dp->dom_ifattach calls malloc() with M_WAITOK, we
* cannot lock ifp->if_afdata initialization, entirely.
*/
if (IF_AFDATA_TRYLOCK(ifp) == 0)
if (IF_AFDATA_TRY_WLOCK(ifp) == 0)
return;
if (ifp->if_afdata_initialized >= domain_init_status) {
IF_AFDATA_UNLOCK(ifp);
@ -3937,3 +3938,65 @@ drbr_enqueue_drv(if_t ifh, struct buf_ring *br, struct mbuf *m)
return drbr_enqueue(ifh, br, m);
}
void
if_afdata_cfg_rlock(struct ifnet *ifp)
{
rw_rlock(&ifp->if_afdata_cfg_lock);
}
void
if_afdata_cfg_runlock(struct ifnet *ifp)
{
rw_runlock(&ifp->if_afdata_cfg_lock);
}
void
if_afdata_cfg_wlock(struct ifnet *ifp)
{
rw_wlock(&ifp->if_afdata_cfg_lock);
}
void
if_afdata_cfg_wunlock(struct ifnet *ifp)
{
rw_wunlock(&ifp->if_afdata_cfg_lock);
}
void
if_afdata_cfg_lock_assert(struct ifnet *ifp, int what)
{
rw_assert(&ifp->if_afdata_cfg_lock, what);
}
void
if_afdata_wlock(struct ifnet *ifp)
{
if_afdata_cfg_wlock(ifp);
IF_AFDATA_RUN_WLOCK(ifp);
}
void
if_afdata_wunlock(struct ifnet *ifp)
{
if_afdata_cfg_wunlock(ifp);
IF_AFDATA_RUN_WUNLOCK(ifp);
}
int
if_afdata_try_wlock(struct ifnet *ifp)
{
if (rw_try_wlock(&ifp->if_afdata_cfg_lock) == 0)
return (0);
IF_AFDATA_RUN_WLOCK(ifp);
return (1);
}

64
sys/net/if_var.h
View File

@ -192,7 +192,7 @@ struct ifnet {
int if_amcount; /* number of all-multicast requests */
struct ifaddr *if_addr; /* pointer to link-level address */
const u_int8_t *if_broadcastaddr; /* linklevel broadcast bytestring */
struct rwlock if_afdata_lock;
struct rmlock if_afdata_run_lock;
void *if_afdata[AF_MAX];
int if_afdata_initialized;
@ -253,6 +253,7 @@ struct ifnet {
u_int if_hw_tsomaxsegcount; /* TSO maximum segment count */
u_int if_hw_tsomaxsegsize; /* TSO maximum segment size in bytes */
struct rwlock if_afdata_cfg_lock;
/*
* Spare fields to be added before branching a stable branch, so
* that structure can be enhanced without changing the kernel
@ -339,22 +340,59 @@ typedef void (*group_change_event_handler_t)(void *, const char *);
EVENTHANDLER_DECLARE(group_change_event, group_change_event_handler_t);
#endif /* _SYS_EVENTHANDLER_H_ */
#define IF_AFDATA_LOCK_INIT(ifp) \
rw_init(&(ifp)->if_afdata_lock, "if_afdata")
#define IF_AFDATA_LOCK_INIT(ifp) do { \
rw_init(&(ifp)->if_afdata_cfg_lock, "if_afdata_cfg"); \
rm_init(&(ifp)->if_afdata_run_lock, "if_afdata_run"); \
} while (0)
#define IF_AFDATA_WLOCK(ifp) rw_wlock(&(ifp)->if_afdata_lock)
#define IF_AFDATA_RLOCK(ifp) rw_rlock(&(ifp)->if_afdata_lock)
#define IF_AFDATA_WUNLOCK(ifp) rw_wunlock(&(ifp)->if_afdata_lock)
#define IF_AFDATA_RUNLOCK(ifp) rw_runlock(&(ifp)->if_afdata_lock)
#define IF_AFDATA_DESTROY(ifp) do { \
rw_destroy(&(ifp)->if_afdata_cfg_lock); \
rm_destroy(&(ifp)->if_afdata_run_lock); \
} while(0)
/* if_afdata lock: control plane functions */
#define IF_AFDATA_CFG_RLOCK(ifp) if_afdata_cfg_rlock(ifp)
#define IF_AFDATA_CFG_RUNLOCK(ifp) if_afdata_cfg_runlock(ifp)
#define IF_AFDATA_CFG_WLOCK(ifp) if_afdata_cfg_wlock(ifp)
#define IF_AFDATA_CFG_WUNLOCK(ifp) if_afdata_cfg_wunlock(ifp)
#define IF_AFDATA_CFG_LOCK_ASSERT(i) if_afdata_cfg_lock_assert(i, RA_LOCKED)
#define IF_AFDATA_CFG_RLOCK_ASSERT(i) if_afdata_cfg_lock_assert(i, RA_RLOCKED)
#define IF_AFDATA_CFG_WLOCK_ASSERT(i) if_afdata_cfg_lock_assert(i, RA_WLOCKED)
#define IF_AFDATA_CFG_UNLOCK_ASSERT(i) if_afdata_cfg_lock_assert(i,RA_UNLOCKED)
void if_afdata_cfg_rlock(struct ifnet *ifp);
void if_afdata_cfg_runlock(struct ifnet *ifp);
void if_afdata_cfg_wlock(struct ifnet *ifp);
void if_afdata_cfg_wunlock(struct ifnet *ifp);
void if_afdata_cfg_lock_assert(struct ifnet *ifp, int what);
/* if_afdata lock: fast path */
#define IF_AFDATA_RUN_WLOCK(ifp) rm_wlock(&(ifp)->if_afdata_run_lock)
#define IF_AFDATA_RUN_WUNLOCK(ifp) rm_wunlock(&(ifp)->if_afdata_run_lock)
#define IF_AFDATA_RUN_RLOCK(ifp) \
rm_rlock(&(ifp)->if_afdata_run_lock, &if_afdata_tracker)
#define IF_AFDATA_RUN_RUNLOCK(ifp) \
rm_runlock(&(ifp)->if_afdata_run_lock, &if_afdata_tracker)
#define IF_AFDATA_RUN_TRACKER struct rm_priotracker if_afdata_tracker
/* Common wrappers */
#define IF_AFDATA_RLOCK(ifp) IF_AFDATA_CFG_RLOCK(ifp)
#define IF_AFDATA_RUNLOCK(ifp) IF_AFDATA_CFG_RUNLOCK(ifp)
#define IF_AFDATA_WLOCK(ifp) if_afdata_wlock(ifp)
#define IF_AFDATA_WUNLOCK(ifp) if_afdata_wunlock(ifp)
#define IF_AFDATA_TRY_WLOCK(ifp) if_afdata_try_wlock(ifp)
#define IF_AFDATA_LOCK(ifp) IF_AFDATA_WLOCK(ifp)
#define IF_AFDATA_UNLOCK(ifp) IF_AFDATA_WUNLOCK(ifp)
#define IF_AFDATA_TRYLOCK(ifp) rw_try_wlock(&(ifp)->if_afdata_lock)
#define IF_AFDATA_DESTROY(ifp) rw_destroy(&(ifp)->if_afdata_lock)
void if_afdata_wlock(struct ifnet *ifp);
void if_afdata_wunlock(struct ifnet *ifp);
int if_afdata_try_wlock(struct ifnet *ifp);
#define IF_AFDATA_LOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_LOCKED)
#define IF_AFDATA_RLOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_RLOCKED)
#define IF_AFDATA_WLOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_WLOCKED)
#define IF_AFDATA_UNLOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_UNLOCKED)
#define IF_AFDATA_LOCK_ASSERT(ifp) IF_AFDATA_CFG_LOCK_ASSERT(ifp)
#define IF_AFDATA_RLOCK_ASSERT(ifp) IF_AFDATA_CFG_RLOCK_ASSERT(ifp)
#define IF_AFDATA_WLOCK_ASSERT(ifp) IF_AFDATA_CFG_WLOCK_ASSERT(ifp)
#define IF_AFDATA_UNLOCK_ASSERT(ifp) IF_AFDATA_CFG_UNLOCK_ASSERT(ifp)
/*
* 72 was chosen below because it is the size of a TCP/IP

7
sys/net/rt_nhops.c
View File

@ -681,6 +681,7 @@ fib6_storelladdr(struct ifnet *ifp, struct in6_addr *dst, int mm_flags,
{
struct llentry *ln;
struct sockaddr_in6 dst_sa;
IF_AFDATA_RUN_TRACKER;
if (mm_flags & M_MCAST) {
ETHER_MAP_IPV6_MULTICAST(&dst, desten);
@ -697,7 +698,7 @@ fib6_storelladdr(struct ifnet *ifp, struct in6_addr *dst, int mm_flags,
/*
* the entry should have been created in nd6_store_lladdr
*/
IF_AFDATA_RLOCK(ifp);
IF_AFDATA_RUN_RLOCK(ifp);
ln = lla_lookup(LLTABLE6(ifp), 0, (struct sockaddr *)&dst_sa);
/*
@ -712,12 +713,12 @@ fib6_storelladdr(struct ifnet *ifp, struct in6_addr *dst, int mm_flags,
ln->ln_state != ND6_LLINFO_DELAY)) {
if (ln != NULL)
LLE_RUNLOCK(ln);
IF_AFDATA_RUNLOCK(ifp);
IF_AFDATA_RUN_RUNLOCK(ifp);
return (1);
}
bcopy(&ln->ll_addr, desten, ifp->if_addrlen);
LLE_RUNLOCK(ln);
IF_AFDATA_RUNLOCK(ifp);
IF_AFDATA_RUN_RUNLOCK(ifp);
return (0);
}

16
sys/netinet/if_ether.c
View File

@ -47,6 +47,8 @@ __FBSDID("$FreeBSD$");
#include <sys/systm.h>
#include <sys/mbuf.h>
#include <sys/malloc.h>
#include <sys/lock.h>
#include <sys/rmlock.h>
#include <sys/proc.h>
#include <sys/socket.h>
#include <sys/syslog.h>
@ -354,6 +356,7 @@ arpresolve_fast(struct ifnet *ifp, struct in_addr dst, u_int mflags,
struct llentry *la;
struct sockaddr_in sin;
const struct sockaddr *sa_dst;
IF_AFDATA_RUN_TRACKER;
if (mflags & M_BCAST) {
memcpy(dst_addr, ifp->if_broadcastaddr, ifp->if_addrlen);
@ -370,17 +373,17 @@ arpresolve_fast(struct ifnet *ifp, struct in_addr dst, u_int mflags,
sin.sin_len = sizeof(sin);
sa_dst = (const struct sockaddr *)&sin;
IF_AFDATA_RLOCK(ifp);
IF_AFDATA_RUN_RLOCK(ifp);
la = lla_lookup(LLTABLE(ifp), LLE_UNLOCKED, sa_dst);
if (la != NULL && (la->r_flags & RLLE_VALID) != 0) {
/* Entry found, let's copy lle info */
bcopy(&la->ll_addr, dst_addr, ifp->if_addrlen);
if (la->r_kick != 0)
la->r_kick = 0; /* Notify that entry was used */
IF_AFDATA_RUNLOCK(ifp);
IF_AFDATA_RUN_RUNLOCK(ifp);
return (0);
}
IF_AFDATA_RUNLOCK(ifp);
IF_AFDATA_RUN_RUNLOCK(ifp);
return (EAGAIN);
@ -438,6 +441,7 @@ arpresolve(struct ifnet *ifp, struct rtentry *rt0, struct mbuf *m,
{
struct llentry *la = NULL;
int is_gw;
IF_AFDATA_RUN_TRACKER;
*lle = NULL;
if (m != NULL) {
@ -454,18 +458,18 @@ arpresolve(struct ifnet *ifp, struct rtentry *rt0, struct mbuf *m,
}
}
IF_AFDATA_RLOCK(ifp);
IF_AFDATA_RUN_RLOCK(ifp);
la = lla_lookup(LLTABLE(ifp), LLE_UNLOCKED, dst);
if (la != NULL && (la->r_flags & RLLE_VALID) != 0) {
/* Entry found, let's copy lle info */
bcopy(&la->ll_addr, desten, ifp->if_addrlen);
if (la->r_kick != 0)
la->r_kick = 0; /* Notify that entry was used */
IF_AFDATA_RUNLOCK(ifp);
IF_AFDATA_RUN_RUNLOCK(ifp);
*lle = la;
return (0);
}
IF_AFDATA_RUNLOCK(ifp);
IF_AFDATA_RUN_RUNLOCK(ifp);
is_gw = (rt0 != NULL && (rt0->rt_flags & RTF_GATEWAY)) ? 1 : 0;
return (arpresolve_slow(ifp, is_gw, m, dst, desten, lle));

9
sys/netinet/in.c
View File

@ -41,6 +41,8 @@ __FBSDID("$FreeBSD$");
#include <sys/sockio.h>
#include <sys/malloc.h>
#include <sys/priv.h>
#include <sys/lock.h>
#include <sys/rmlock.h>
#include <sys/socket.h>
#include <sys/jail.h>
#include <sys/kernel.h>
@ -1249,11 +1251,14 @@ in_lltable_unlink(struct llentry *lle)
static struct llentry *
in_lltable_lookup(struct lltable *llt, u_int flags, const struct sockaddr *l3addr)
{
struct ifnet *ifp = llt->llt_ifp;
struct llentry *lle;
struct in_addr dst;
IF_AFDATA_LOCK_ASSERT(ifp);
/*
* Do not check for AFDATA lock since search can be protected
* by different locks.
* IF_AFDATA_LOCK_ASSERT(llt->llt_ifp);
*/
KASSERT(l3addr->sa_family == AF_INET,
("sin_family %d", l3addr->sa_family));

10
sys/netinet6/in6.c
View File

@ -2180,10 +2180,9 @@ in6_lltable_delete(struct lltable *llt, u_int flags,
const struct sockaddr *l3addr)
{
const struct sockaddr_in6 *sin6 = (const struct sockaddr_in6 *)l3addr;
struct ifnet *ifp = llt->llt_ifp;
struct llentry *lle;
IF_AFDATA_LOCK_ASSERT(ifp);
IF_AFDATA_LOCK_ASSERT(llt->llt_ifp);
KASSERT(l3addr->sa_family == AF_INET6,
("sin_family %d", l3addr->sa_family));
@ -2287,10 +2286,13 @@ in6_lltable_lookup(struct lltable *llt, u_int flags,
const struct sockaddr *l3addr)
{
const struct sockaddr_in6 *sin6 = (const struct sockaddr_in6 *)l3addr;
struct ifnet *ifp = llt->llt_ifp;
struct llentry *lle;
IF_AFDATA_LOCK_ASSERT(ifp);
/*
* Do not check for AFDATA lock since search can be protected
* by different locks.
* IF_AFDATA_LOCK_ASSERT(llt->llt_ifp);
*/
KASSERT(l3addr->sa_family == AF_INET6,
("sin_family %d", l3addr->sa_family));