d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rework locking in BPF code to remove rwlock from fast path.

Browse Source 
On high packets rate the contention on rwlock in bpf_*tap*() functions
can lead to packets dropping. To avoid this, migrate this code to use
epoch(9) KPI and ConcurrencyKit's lists.

* all lists changed to use CK_LIST;
* reference counting added to bpf_if and bpf_d;
* now bpf_if references ifnet and releases this reference on destroy;
* each bpf_d descriptor references bpf_if when it is attached;
* new struct bpf_program_buffer introduced to keep BPF filter programs;
* bpf_program_buffer, bpf_d and bpf_if structures are freed by
  epoch_call();
* bpf_freelist and ifnet_departure event are no longer needed, thus
  both are removed;

Reviewed by:	melifaro
Sponsored by:	Yandex LLC
Differential Revision:	https://reviews.freebsd.org/D20224
This commit is contained in:
Andrey V. Elsukov 2019-05-13 13:45:28 +00:00
parent eb4c63f731
commit 699281b545
3 changed files with 378 additions and 377 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

734
sys/net/bpf.c
View File

File diff suppressed because it is too large Load Diff

13
sys/net/bpf.h
View File

@ -42,6 +42,9 @@
#ifndef _NET_BPF_H_
#define _NET_BPF_H_
#include <sys/ck.h>
#include <net/dlt.h>
/* BSD style release date */
#define BPF_RELEASE 199606
@ -233,9 +236,6 @@ struct bpf_zbuf_header {
u_int _bzh_pad[5];
};
/* Pull in data-link level type codes. */
#include <net/dlt.h>
/*
* The instruction encodings.
*
@ -409,10 +409,11 @@ SYSCTL_DECL(_net_bpf);
* bpf_peers_present() calls.
*/
struct bpf_if;
CK_LIST_HEAD(bpfd_list, bpf_d);
struct bpf_if_ext {
LIST_ENTRY(bpf_if) bif_next; /* list of all interfaces */
LIST_HEAD(, bpf_d) bif_dlist; /* descriptor list */
CK_LIST_ENTRY(bpf_if) bif_next; /* list of all interfaces */
struct bpfd_list bif_dlist; /* descriptor list */
};
void bpf_bufheld(struct bpf_d *d);
@ -436,7 +437,7 @@ bpf_peers_present(struct bpf_if *bpf)
struct bpf_if_ext *ext;
ext = (struct bpf_if_ext *)bpf;
if (!LIST_EMPTY(&ext->bif_dlist))
if (!CK_LIST_EMPTY(&ext->bif_dlist))
return (1);
return (0);
}

8
sys/net/bpfdesc.h
View File

@ -43,9 +43,10 @@
#include <sys/callout.h>
#include <sys/selinfo.h>
#include <sys/queue.h>
#include <sys/ck.h>
#include <sys/conf.h>
#include <sys/counter.h>
#include <sys/epoch.h>
#include <net/if.h>
/*
@ -53,7 +54,7 @@
*/
struct zbuf;
struct bpf_d {
LIST_ENTRY(bpf_d) bd_next; /* Linked list of descriptors */
CK_LIST_ENTRY(bpf_d) bd_next; /* Linked list of descriptors */
/*
* Buffer slots: two memory buffers store the incoming packets.
* The model has three slots. Sbuf is always occupied.
@ -104,6 +105,9 @@ struct bpf_d {
counter_u64_t bd_wdcount; /* number of packets dropped during a write */
counter_u64_t bd_zcopy; /* number of zero copy operations */
u_char bd_compat32; /* 32-bit stream on LP64 system */
volatile u_int bd_refcnt;
struct epoch_context epoch_ctx;
};
/* Values for bd_state */