Rework locking in BPF code to remove rwlock from fast path.
On high packets rate the contention on rwlock in bpf_*tap*() functions can lead to packets dropping. To avoid this, migrate this code to use epoch(9) KPI and ConcurrencyKit's lists. * all lists changed to use CK_LIST; * reference counting added to bpf_if and bpf_d; * now bpf_if references ifnet and releases this reference on destroy; * each bpf_d descriptor references bpf_if when it is attached; * new struct bpf_program_buffer introduced to keep BPF filter programs; * bpf_program_buffer, bpf_d and bpf_if structures are freed by epoch_call(); * bpf_freelist and ifnet_departure event are no longer needed, thus both are removed; Reviewed by: melifaro Sponsored by: Yandex LLC Differential Revision: https://reviews.freebsd.org/D20224
This commit is contained in:
parent
eb4c63f731
commit
699281b545
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=347526
702
sys/net/bpf.c
702
sys/net/bpf.c
File diff suppressed because it is too large
Load Diff
@ -42,6 +42,9 @@
|
|||||||
#ifndef _NET_BPF_H_
|
#ifndef _NET_BPF_H_
|
||||||
#define _NET_BPF_H_
|
#define _NET_BPF_H_
|
||||||
|
|
||||||
|
#include <sys/ck.h>
|
||||||
|
#include <net/dlt.h>
|
||||||
|
|
||||||
/* BSD style release date */
|
/* BSD style release date */
|
||||||
#define BPF_RELEASE 199606
|
#define BPF_RELEASE 199606
|
||||||
|
|
||||||
@ -233,9 +236,6 @@ struct bpf_zbuf_header {
|
|||||||
u_int _bzh_pad[5];
|
u_int _bzh_pad[5];
|
||||||
};
|
};
|
||||||
|
|
||||||
/* Pull in data-link level type codes. */
|
|
||||||
#include <net/dlt.h>
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* The instruction encodings.
|
* The instruction encodings.
|
||||||
*
|
*
|
||||||
@ -409,10 +409,11 @@ SYSCTL_DECL(_net_bpf);
|
|||||||
* bpf_peers_present() calls.
|
* bpf_peers_present() calls.
|
||||||
*/
|
*/
|
||||||
struct bpf_if;
|
struct bpf_if;
|
||||||
|
CK_LIST_HEAD(bpfd_list, bpf_d);
|
||||||
|
|
||||||
struct bpf_if_ext {
|
struct bpf_if_ext {
|
||||||
LIST_ENTRY(bpf_if) bif_next; /* list of all interfaces */
|
CK_LIST_ENTRY(bpf_if) bif_next; /* list of all interfaces */
|
||||||
LIST_HEAD(, bpf_d) bif_dlist; /* descriptor list */
|
struct bpfd_list bif_dlist; /* descriptor list */
|
||||||
};
|
};
|
||||||
|
|
||||||
void bpf_bufheld(struct bpf_d *d);
|
void bpf_bufheld(struct bpf_d *d);
|
||||||
@ -436,7 +437,7 @@ bpf_peers_present(struct bpf_if *bpf)
|
|||||||
struct bpf_if_ext *ext;
|
struct bpf_if_ext *ext;
|
||||||
|
|
||||||
ext = (struct bpf_if_ext *)bpf;
|
ext = (struct bpf_if_ext *)bpf;
|
||||||
if (!LIST_EMPTY(&ext->bif_dlist))
|
if (!CK_LIST_EMPTY(&ext->bif_dlist))
|
||||||
return (1);
|
return (1);
|
||||||
return (0);
|
return (0);
|
||||||
}
|
}
|
||||||
|
@ -43,9 +43,10 @@
|
|||||||
|
|
||||||
#include <sys/callout.h>
|
#include <sys/callout.h>
|
||||||
#include <sys/selinfo.h>
|
#include <sys/selinfo.h>
|
||||||
#include <sys/queue.h>
|
#include <sys/ck.h>
|
||||||
#include <sys/conf.h>
|
#include <sys/conf.h>
|
||||||
#include <sys/counter.h>
|
#include <sys/counter.h>
|
||||||
|
#include <sys/epoch.h>
|
||||||
#include <net/if.h>
|
#include <net/if.h>
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@ -53,7 +54,7 @@
|
|||||||
*/
|
*/
|
||||||
struct zbuf;
|
struct zbuf;
|
||||||
struct bpf_d {
|
struct bpf_d {
|
||||||
LIST_ENTRY(bpf_d) bd_next; /* Linked list of descriptors */
|
CK_LIST_ENTRY(bpf_d) bd_next; /* Linked list of descriptors */
|
||||||
/*
|
/*
|
||||||
* Buffer slots: two memory buffers store the incoming packets.
|
* Buffer slots: two memory buffers store the incoming packets.
|
||||||
* The model has three slots. Sbuf is always occupied.
|
* The model has three slots. Sbuf is always occupied.
|
||||||
@ -104,6 +105,9 @@ struct bpf_d {
|
|||||||
counter_u64_t bd_wdcount; /* number of packets dropped during a write */
|
counter_u64_t bd_wdcount; /* number of packets dropped during a write */
|
||||||
counter_u64_t bd_zcopy; /* number of zero copy operations */
|
counter_u64_t bd_zcopy; /* number of zero copy operations */
|
||||||
u_char bd_compat32; /* 32-bit stream on LP64 system */
|
u_char bd_compat32; /* 32-bit stream on LP64 system */
|
||||||
|
|
||||||
|
volatile u_int bd_refcnt;
|
||||||
|
struct epoch_context epoch_ctx;
|
||||||
};
|
};
|
||||||
|
|
||||||
/* Values for bd_state */
|
/* Values for bd_state */
|
||||||
|
Loading…
Reference in New Issue
Block a user