d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Switch RIB and RADIX_NODE_HEAD lock from rwlock(9) to rmlock(9).

Browse Source 
Using of rwlock with multiqueue NICs for IP forwarding on high pps
produces high lock contention and inefficient. Rmlock fits better for
such workloads.

Reviewed by:	melifaro, olivier
Obtained from:	Yandex LLC
Sponsored by:	Yandex LLC
Differential Revision:	https://reviews.freebsd.org/D15789
This commit is contained in:
Andrey V. Elsukov 2018-06-16 08:26:23 +00:00
parent bcc0decdc9
commit 20efcfc602
12 changed files with 46 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
sys/kern/subr_witness.c
View File

@ -524,7 +524,7 @@ static struct witness_order_list_entry order_lists[] = {
* Routing * Routing
*/ */
{ "so_rcv", &lock_class_mtx_sleep }, { "so_rcv", &lock_class_mtx_sleep },
{ "radix node head", &lock_class_rw }, { "radix node head", &lock_class_rm },
{ "rtentry", &lock_class_mtx_sleep }, { "rtentry", &lock_class_mtx_sleep },
{ "ifaddr", &lock_class_mtx_sleep }, { "ifaddr", &lock_class_mtx_sleep },
{ NULL, NULL }, { NULL, NULL },

3
sys/kern/vfs_export.c
View File

@ -51,7 +51,7 @@ __FBSDID("$FreeBSD$");
#include <sys/mbuf.h> #include <sys/mbuf.h>
#include <sys/mount.h> #include <sys/mount.h>
#include <sys/mutex.h> #include <sys/mutex.h>
#include <sys/rwlock.h> #include <sys/rmlock.h>
#include <sys/refcount.h> #include <sys/refcount.h>
#include <sys/signalvar.h> #include <sys/signalvar.h>
#include <sys/socket.h> #include <sys/socket.h>
@ -449,6 +449,7 @@ vfs_setpublicfs(struct mount *mp, struct netexport *nep,
static struct netcred * static struct netcred *
vfs_export_lookup(struct mount *mp, struct sockaddr *nam) vfs_export_lookup(struct mount *mp, struct sockaddr *nam)
{ {
RADIX_NODE_HEAD_RLOCK_TRACKER;
struct netexport *nep; struct netexport *nep;
struct netcred *np = NULL; struct netcred *np = NULL;
struct radix_node_head *rnh; struct radix_node_head *rnh;

2
sys/net/radix.c
View File

@ -39,7 +39,7 @@
#ifdef _KERNEL #ifdef _KERNEL
#include <sys/lock.h> #include <sys/lock.h>
#include <sys/mutex.h> #include <sys/mutex.h>
#include <sys/rwlock.h> #include <sys/rmlock.h>
#include <sys/systm.h> #include <sys/systm.h>
#include <sys/malloc.h> #include <sys/malloc.h>
#include <sys/syslog.h> #include <sys/syslog.h>

26
sys/net/radix.h
View File

@ -38,7 +38,7 @@
#ifdef _KERNEL #ifdef _KERNEL
#include <sys/_lock.h> #include <sys/_lock.h>
#include <sys/_mutex.h> #include <sys/_mutex.h>
#include <sys/_rwlock.h> #include <sys/_rmlock.h>
#endif #endif
#ifdef MALLOC_DECLARE #ifdef MALLOC_DECLARE
@ -138,7 +138,7 @@ struct radix_node_head {
rn_close_t *rnh_close; /*do something when the last ref drops*/ rn_close_t *rnh_close; /*do something when the last ref drops*/
struct radix_node rnh_nodes[3]; /* empty tree for common case */ struct radix_node rnh_nodes[3]; /* empty tree for common case */
#ifdef _KERNEL #ifdef _KERNEL
struct rwlock rnh_lock; /* locks entire radix tree */ struct rmlock rnh_lock; /* locks entire radix tree */
#endif #endif
}; };
@ -159,18 +159,18 @@ void rn_inithead_internal(struct radix_head *rh, struct radix_node *base_nodes,
#define R_Zalloc(p, t, n) (p = (t) malloc((unsigned long)(n), M_RTABLE, M_NOWAIT | M_ZERO)) #define R_Zalloc(p, t, n) (p = (t) malloc((unsigned long)(n), M_RTABLE, M_NOWAIT | M_ZERO))
#define R_Free(p) free((caddr_t)p, M_RTABLE); #define R_Free(p) free((caddr_t)p, M_RTABLE);
#define RADIX_NODE_HEAD_RLOCK_TRACKER struct rm_priotracker _rhn_tracker
#define RADIX_NODE_HEAD_LOCK_INIT(rnh) \ #define RADIX_NODE_HEAD_LOCK_INIT(rnh) \
rw_init_flags(&(rnh)->rnh_lock, "radix node head", 0) rm_init(&(rnh)->rnh_lock, "radix node head")
#define RADIX_NODE_HEAD_LOCK(rnh) rw_wlock(&(rnh)->rnh_lock) #define RADIX_NODE_HEAD_LOCK(rnh) rm_wlock(&(rnh)->rnh_lock)
#define RADIX_NODE_HEAD_UNLOCK(rnh) rw_wunlock(&(rnh)->rnh_lock) #define RADIX_NODE_HEAD_UNLOCK(rnh) rm_wunlock(&(rnh)->rnh_lock)
#define RADIX_NODE_HEAD_RLOCK(rnh) rw_rlock(&(rnh)->rnh_lock) #define RADIX_NODE_HEAD_RLOCK(rnh) rm_rlock(&(rnh)->rnh_lock,\
#define RADIX_NODE_HEAD_RUNLOCK(rnh) rw_runlock(&(rnh)->rnh_lock) &_rhn_tracker)
#define RADIX_NODE_HEAD_LOCK_TRY_UPGRADE(rnh) rw_try_upgrade(&(rnh)->rnh_lock) #define RADIX_NODE_HEAD_RUNLOCK(rnh) rm_runlock(&(rnh)->rnh_lock,\
&_rhn_tracker)
#define RADIX_NODE_HEAD_DESTROY(rnh) rm_destroy(&(rnh)->rnh_lock)
#define RADIX_NODE_HEAD_DESTROY(rnh) rw_destroy(&(rnh)->rnh_lock) #define RADIX_NODE_HEAD_LOCK_ASSERT(rnh) rm_assert(&(rnh)->rnh_lock, RA_LOCKED)
#define RADIX_NODE_HEAD_LOCK_ASSERT(rnh) rw_assert(&(rnh)->rnh_lock, RA_LOCKED) #define RADIX_NODE_HEAD_WLOCK_ASSERT(rnh) rm_assert(&(rnh)->rnh_lock, RA_WLOCKED)
#define RADIX_NODE_HEAD_WLOCK_ASSERT(rnh) rw_assert(&(rnh)->rnh_lock, RA_WLOCKED)
#endif /* _KERNEL */ #endif /* _KERNEL */
int rn_inithead(void **, int); int rn_inithead(void **, int);

3
sys/net/radix_mpath.c
View File

@ -43,12 +43,15 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h> #include <sys/param.h>
#include <sys/systm.h> #include <sys/systm.h>
#include <sys/lock.h>
#include <sys/malloc.h> #include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/socket.h> #include <sys/socket.h>
#include <sys/domain.h> #include <sys/domain.h>
#include <sys/syslog.h> #include <sys/syslog.h>
#include <net/radix.h> #include <net/radix.h>
#include <net/radix_mpath.h> #include <net/radix_mpath.h>
#include <sys/rmlock.h>
#include <net/route.h> #include <net/route.h>
#include <net/route_var.h> #include <net/route_var.h>
#include <net/if.h> #include <net/if.h>

5
sys/net/route.c
View File

@ -54,6 +54,8 @@
#include <sys/proc.h> #include <sys/proc.h>
#include <sys/domain.h> #include <sys/domain.h>
#include <sys/kernel.h> #include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/rmlock.h>
#include <net/if.h> #include <net/if.h>
#include <net/if_var.h> #include <net/if_var.h>
@ -440,6 +442,7 @@ struct rtentry *
rtalloc1_fib(struct sockaddr *dst, int report, u_long ignflags, rtalloc1_fib(struct sockaddr *dst, int report, u_long ignflags,
u_int fibnum) u_int fibnum)
{ {
RIB_RLOCK_TRACKER;
struct rib_head *rh; struct rib_head *rh;
struct radix_node *rn; struct radix_node *rn;
struct rtentry *newrt; struct rtentry *newrt;
@ -923,6 +926,7 @@ int
rib_lookup_info(uint32_t fibnum, const struct sockaddr *dst, uint32_t flags, rib_lookup_info(uint32_t fibnum, const struct sockaddr *dst, uint32_t flags,
uint32_t flowid, struct rt_addrinfo *info) uint32_t flowid, struct rt_addrinfo *info)
{ {
RIB_RLOCK_TRACKER;
struct rib_head *rh; struct rib_head *rh;
struct radix_node *rn; struct radix_node *rn;
struct rtentry *rt; struct rtentry *rt;
@ -1944,6 +1948,7 @@ rt_maskedcopy(struct sockaddr *src, struct sockaddr *dst, struct sockaddr *netma
static inline int static inline int
rtinit1(struct ifaddr *ifa, int cmd, int flags, int fibnum) rtinit1(struct ifaddr *ifa, int cmd, int flags, int fibnum)
{ {
RIB_RLOCK_TRACKER;
struct sockaddr *dst; struct sockaddr *dst;
struct sockaddr *netmask; struct sockaddr *netmask;
struct rtentry *rt = NULL; struct rtentry *rt = NULL;

19
sys/net/route_var.h
View File

@ -44,18 +44,19 @@ struct rib_head {
rt_gen_t rnh_gen; /* generation counter */ rt_gen_t rnh_gen; /* generation counter */
int rnh_multipath; /* multipath capable ? */ int rnh_multipath; /* multipath capable ? */
struct radix_node rnh_nodes[3]; /* empty tree for common case */ struct radix_node rnh_nodes[3]; /* empty tree for common case */
struct rwlock rib_lock; /* config/data path lock */ struct rmlock rib_lock; /* config/data path lock */
struct radix_mask_head rmhead; /* masks radix head */ struct radix_mask_head rmhead; /* masks radix head */
}; };
#define RIB_LOCK_INIT(rh) rw_init(&(rh)->rib_lock, "rib head lock") #define RIB_RLOCK_TRACKER struct rm_priotracker _rib_tracker
#define RIB_LOCK_DESTROY(rh) rw_destroy(&(rh)->rib_lock) #define RIB_LOCK_INIT(rh) rm_init(&(rh)->rib_lock, "rib head lock")
#define RIB_RLOCK(rh) rw_rlock(&(rh)->rib_lock) #define RIB_LOCK_DESTROY(rh) rm_destroy(&(rh)->rib_lock)
#define RIB_RUNLOCK(rh) rw_runlock(&(rh)->rib_lock) #define RIB_RLOCK(rh) rm_rlock(&(rh)->rib_lock, &_rib_tracker)
#define RIB_WLOCK(rh) rw_wlock(&(rh)->rib_lock) #define RIB_RUNLOCK(rh) rm_runlock(&(rh)->rib_lock, &_rib_tracker)
#define RIB_WUNLOCK(rh) rw_wunlock(&(rh)->rib_lock) #define RIB_WLOCK(rh) rm_wlock(&(rh)->rib_lock)
#define RIB_LOCK_ASSERT(rh) rw_assert(&(rh)->rib_lock, RA_LOCKED) #define RIB_WUNLOCK(rh) rm_wunlock(&(rh)->rib_lock)
#define RIB_WLOCK_ASSERT(rh) rw_assert(&(rh)->rib_lock, RA_WLOCKED) #define RIB_LOCK_ASSERT(rh) rm_assert(&(rh)->rib_lock, RA_LOCKED)
#define RIB_WLOCK_ASSERT(rh) rm_assert(&(rh)->rib_lock, RA_WLOCKED)
struct rib_head *rt_tables_get_rnh(int fib, int family); struct rib_head *rt_tables_get_rnh(int fib, int family);

3
sys/net/rtsock.c
View File

@ -45,6 +45,7 @@
#include <sys/priv.h> #include <sys/priv.h>
#include <sys/proc.h> #include <sys/proc.h>
#include <sys/protosw.h> #include <sys/protosw.h>
#include <sys/rmlock.h>
#include <sys/rwlock.h> #include <sys/rwlock.h>
#include <sys/signalvar.h> #include <sys/signalvar.h>
#include <sys/socket.h> #include <sys/socket.h>
@ -542,6 +543,7 @@ rtm_get_jailed(struct rt_addrinfo *info, struct ifnet *ifp,
static int static int
route_output(struct mbuf *m, struct socket *so, ...) route_output(struct mbuf *m, struct socket *so, ...)
{ {
RIB_RLOCK_TRACKER;
struct rt_msghdr *rtm = NULL; struct rt_msghdr *rtm = NULL;
struct rtentry *rt = NULL; struct rtentry *rt = NULL;
struct rib_head *rnh; struct rib_head *rnh;
@ -1850,6 +1852,7 @@ sysctl_ifmalist(int af, struct walkarg *w)
static int static int
sysctl_rtsock(SYSCTL_HANDLER_ARGS) sysctl_rtsock(SYSCTL_HANDLER_ARGS)
{ {
RIB_RLOCK_TRACKER;
int *name = (int *)arg1; int *name = (int *)arg1;
u_int namelen = arg2; u_int namelen = arg2;
struct rib_head *rnh = NULL; /* silence compiler. */ struct rib_head *rnh = NULL; /* silence compiler. */

4
sys/netinet/in_fib.c
View File

@ -37,7 +37,7 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h> #include <sys/param.h>
#include <sys/systm.h> #include <sys/systm.h>
#include <sys/lock.h> #include <sys/lock.h>
#include <sys/rwlock.h> #include <sys/rmlock.h>
#include <sys/malloc.h> #include <sys/malloc.h>
#include <sys/mbuf.h> #include <sys/mbuf.h>
#include <sys/socket.h> #include <sys/socket.h>
@ -134,6 +134,7 @@ int
fib4_lookup_nh_basic(uint32_t fibnum, struct in_addr dst, uint32_t flags, fib4_lookup_nh_basic(uint32_t fibnum, struct in_addr dst, uint32_t flags,
uint32_t flowid, struct nhop4_basic *pnh4) uint32_t flowid, struct nhop4_basic *pnh4)
{ {
RIB_RLOCK_TRACKER;
struct rib_head *rh; struct rib_head *rh;
struct radix_node *rn; struct radix_node *rn;
struct sockaddr_in sin; struct sockaddr_in sin;
@ -182,6 +183,7 @@ int
fib4_lookup_nh_ext(uint32_t fibnum, struct in_addr dst, uint32_t flags, fib4_lookup_nh_ext(uint32_t fibnum, struct in_addr dst, uint32_t flags,
uint32_t flowid, struct nhop4_extended *pnh4) uint32_t flowid, struct nhop4_extended *pnh4)
{ {
RIB_RLOCK_TRACKER;
struct rib_head *rh; struct rib_head *rh;
struct radix_node *rn; struct radix_node *rn;
struct sockaddr_in sin; struct sockaddr_in sin;

4
sys/netinet6/in6_fib.c
View File

@ -38,7 +38,7 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h> #include <sys/param.h>
#include <sys/systm.h> #include <sys/systm.h>
#include <sys/lock.h> #include <sys/lock.h>
#include <sys/rwlock.h> #include <sys/rmlock.h>
#include <sys/malloc.h> #include <sys/malloc.h>
#include <sys/mbuf.h> #include <sys/mbuf.h>
#include <sys/socket.h> #include <sys/socket.h>
@ -171,6 +171,7 @@ int
fib6_lookup_nh_basic(uint32_t fibnum, const struct in6_addr *dst, uint32_t scopeid, fib6_lookup_nh_basic(uint32_t fibnum, const struct in6_addr *dst, uint32_t scopeid,
uint32_t flags, uint32_t flowid, struct nhop6_basic *pnh6) uint32_t flags, uint32_t flowid, struct nhop6_basic *pnh6)
{ {
RIB_RLOCK_TRACKER;
struct rib_head *rh; struct rib_head *rh;
struct radix_node *rn; struct radix_node *rn;
struct sockaddr_in6 sin6; struct sockaddr_in6 sin6;
@ -220,6 +221,7 @@ int
fib6_lookup_nh_ext(uint32_t fibnum, const struct in6_addr *dst,uint32_t scopeid, fib6_lookup_nh_ext(uint32_t fibnum, const struct in6_addr *dst,uint32_t scopeid,
uint32_t flags, uint32_t flowid, struct nhop6_extended *pnh6) uint32_t flags, uint32_t flowid, struct nhop6_extended *pnh6)
{ {
RIB_RLOCK_TRACKER;
struct rib_head *rh; struct rib_head *rh;
struct radix_node *rn; struct radix_node *rn;
struct sockaddr_in6 sin6; struct sockaddr_in6 sin6;

1
sys/netpfil/ipfw/ip_fw_table_algo.c
View File

@ -4047,6 +4047,7 @@ static void
ta_foreach_kfib(void *ta_state, struct table_info *ti, ta_foreach_f *f, ta_foreach_kfib(void *ta_state, struct table_info *ti, ta_foreach_f *f,
void *arg) void *arg)
{ {
RIB_RLOCK_TRACKER;
struct rib_head *rh; struct rib_head *rh;
int error; int error;

1
sys/nfs/bootp_subr.c
View File

@ -376,6 +376,7 @@ bootpboot_p_tree(struct radix_node *rn)
void void
bootpboot_p_rtlist(void) bootpboot_p_rtlist(void)
{ {
RIB_RLOCK_TRACKER;
struct rib_head *rnh; struct rib_head *rnh;
printf("Routing table:\n"); printf("Routing table:\n");