epoch: add non-preemptible "critical" variant
adds: - epoch_enter_critical() - can be called inside a different epoch, starts a section that will acquire any MTX_DEF mutexes or do anything that might sleep. - epoch_exit_critical() - corresponding exit call - epoch_wait_critical() - wait variant that is guaranteed that any threads in a section are running. - epoch_global_critical - an epoch_wait_critical safe epoch instance Requested by: markj Approved by: sbruno
This commit is contained in:
parent
dedc82ae26
commit
5e68a3dfe3
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=333775
@ -83,6 +83,12 @@ SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, ncontended, CTLFLAG_RW,
|
|||||||
static counter_u64_t switch_count;
|
static counter_u64_t switch_count;
|
||||||
SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, switches, CTLFLAG_RW,
|
SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, switches, CTLFLAG_RW,
|
||||||
&switch_count, "# of times a thread voluntarily context switched in epoch_wait");
|
&switch_count, "# of times a thread voluntarily context switched in epoch_wait");
|
||||||
|
static counter_u64_t epoch_call_count;
|
||||||
|
SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, epoch_calls, CTLFLAG_RW,
|
||||||
|
&epoch_call_count, "# of times a callback was deferred");
|
||||||
|
static counter_u64_t epoch_call_task_count;
|
||||||
|
SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, epoch_call_tasks, CTLFLAG_RW,
|
||||||
|
&epoch_call_task_count, "# of times a callback task was run");
|
||||||
|
|
||||||
TAILQ_HEAD(threadlist, thread);
|
TAILQ_HEAD(threadlist, thread);
|
||||||
|
|
||||||
@ -103,9 +109,8 @@ struct epoch_pcpu_state {
|
|||||||
struct epoch {
|
struct epoch {
|
||||||
struct ck_epoch e_epoch __aligned(EPOCH_ALIGN);
|
struct ck_epoch e_epoch __aligned(EPOCH_ALIGN);
|
||||||
struct epoch_pcpu_state *e_pcpu_dom[MAXMEMDOM] __aligned(EPOCH_ALIGN);
|
struct epoch_pcpu_state *e_pcpu_dom[MAXMEMDOM] __aligned(EPOCH_ALIGN);
|
||||||
counter_u64_t e_frees;
|
|
||||||
uint64_t e_free_last;
|
|
||||||
int e_idx;
|
int e_idx;
|
||||||
|
int e_flags;
|
||||||
struct epoch_pcpu_state *e_pcpu[0];
|
struct epoch_pcpu_state *e_pcpu[0];
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -119,7 +124,7 @@ static __read_mostly int domoffsets[MAXMEMDOM];
|
|||||||
static __read_mostly int inited;
|
static __read_mostly int inited;
|
||||||
static __read_mostly int epoch_count;
|
static __read_mostly int epoch_count;
|
||||||
__read_mostly epoch_t global_epoch;
|
__read_mostly epoch_t global_epoch;
|
||||||
static __read_mostly epoch_t private_epoch;
|
__read_mostly epoch_t global_epoch_critical;
|
||||||
|
|
||||||
static void epoch_call_task(void *context __unused);
|
static void epoch_call_task(void *context __unused);
|
||||||
|
|
||||||
@ -161,8 +166,8 @@ epoch_init(void *arg __unused)
|
|||||||
taskqgroup_attach_cpu(qgroup_softirq, DPCPU_ID_PTR(cpu, epoch_cb_task), NULL, cpu, -1, "epoch call task");
|
taskqgroup_attach_cpu(qgroup_softirq, DPCPU_ID_PTR(cpu, epoch_cb_task), NULL, cpu, -1, "epoch call task");
|
||||||
}
|
}
|
||||||
inited = 1;
|
inited = 1;
|
||||||
global_epoch = epoch_alloc();
|
global_epoch = epoch_alloc(0);
|
||||||
private_epoch = epoch_alloc();
|
global_epoch_critical = epoch_alloc(EPOCH_CRITICAL);
|
||||||
}
|
}
|
||||||
SYSINIT(epoch, SI_SUB_TASKQ + 1, SI_ORDER_FIRST, epoch_init, NULL);
|
SYSINIT(epoch, SI_SUB_TASKQ + 1, SI_ORDER_FIRST, epoch_init, NULL);
|
||||||
|
|
||||||
@ -206,7 +211,7 @@ epoch_init_legacy(epoch_t epoch)
|
|||||||
}
|
}
|
||||||
|
|
||||||
epoch_t
|
epoch_t
|
||||||
epoch_alloc(void)
|
epoch_alloc(int flags)
|
||||||
{
|
{
|
||||||
epoch_t epoch;
|
epoch_t epoch;
|
||||||
|
|
||||||
@ -215,12 +220,12 @@ epoch_alloc(void)
|
|||||||
epoch = malloc(sizeof(struct epoch) + mp_ncpus*sizeof(void*),
|
epoch = malloc(sizeof(struct epoch) + mp_ncpus*sizeof(void*),
|
||||||
M_EPOCH, M_ZERO|M_WAITOK);
|
M_EPOCH, M_ZERO|M_WAITOK);
|
||||||
ck_epoch_init(&epoch->e_epoch);
|
ck_epoch_init(&epoch->e_epoch);
|
||||||
epoch->e_frees = counter_u64_alloc(M_WAITOK);
|
|
||||||
if (usedomains)
|
if (usedomains)
|
||||||
epoch_init_numa(epoch);
|
epoch_init_numa(epoch);
|
||||||
else
|
else
|
||||||
epoch_init_legacy(epoch);
|
epoch_init_legacy(epoch);
|
||||||
MPASS(epoch_count < MAX_EPOCHS-2);
|
MPASS(epoch_count < MAX_EPOCHS-2);
|
||||||
|
epoch->e_flags = flags;
|
||||||
epoch->e_idx = epoch_count;
|
epoch->e_idx = epoch_count;
|
||||||
allepochs[epoch_count++] = epoch;
|
allepochs[epoch_count++] = epoch;
|
||||||
return (epoch);
|
return (epoch);
|
||||||
@ -240,11 +245,7 @@ epoch_free(epoch_t epoch)
|
|||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
allepochs[epoch->e_idx] = NULL;
|
allepochs[epoch->e_idx] = NULL;
|
||||||
epoch_wait(private_epoch);
|
epoch_wait_critical(global_epoch_critical);
|
||||||
/*
|
|
||||||
* Execute any lingering callbacks
|
|
||||||
*/
|
|
||||||
counter_u64_free(epoch->e_frees);
|
|
||||||
if (usedomains)
|
if (usedomains)
|
||||||
for (domain = 0; domain < vm_ndomains; domain++)
|
for (domain = 0; domain < vm_ndomains; domain++)
|
||||||
free_domain(epoch->e_pcpu_dom[domain], M_EPOCH);
|
free_domain(epoch->e_pcpu_dom[domain], M_EPOCH);
|
||||||
@ -289,14 +290,21 @@ epoch_enter_internal(epoch_t epoch, struct thread *td)
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static void
|
void
|
||||||
epoch_enter_private(ck_epoch_section_t *section)
|
epoch_enter_critical(epoch_t epoch)
|
||||||
{
|
{
|
||||||
struct epoch_pcpu_state *eps;
|
ck_epoch_record_t *record;
|
||||||
|
ck_epoch_section_t *section;
|
||||||
|
struct thread *td;
|
||||||
|
|
||||||
MPASS(curthread->td_critnest);
|
section = NULL;
|
||||||
eps = private_epoch->e_pcpu[curcpu];
|
td = curthread;
|
||||||
ck_epoch_begin(&eps->eps_record.er_record, section);
|
critical_enter();
|
||||||
|
if (__predict_true(td->td_epochnest++ == 0))
|
||||||
|
section = (ck_epoch_section_t*)&td->td_epoch_section;
|
||||||
|
|
||||||
|
record = &epoch->e_pcpu[curcpu]->eps_record.er_record;
|
||||||
|
ck_epoch_begin(record, section);
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
@ -321,14 +329,21 @@ epoch_exit_internal(epoch_t epoch, struct thread *td)
|
|||||||
critical_exit();
|
critical_exit();
|
||||||
}
|
}
|
||||||
|
|
||||||
static void
|
void
|
||||||
epoch_exit_private(ck_epoch_section_t *section)
|
epoch_exit_critical(epoch_t epoch)
|
||||||
{
|
{
|
||||||
struct epoch_pcpu_state *eps;
|
ck_epoch_record_t *record;
|
||||||
|
ck_epoch_section_t *section;
|
||||||
|
struct thread *td;
|
||||||
|
|
||||||
MPASS(curthread->td_critnest);
|
section = NULL;
|
||||||
eps = private_epoch->e_pcpu[curcpu];
|
td = curthread;
|
||||||
ck_epoch_end(&eps->eps_record.er_record, section);
|
MPASS(td->td_critnest);
|
||||||
|
if (__predict_true(td->td_epochnest-- == 1))
|
||||||
|
section = (ck_epoch_section_t*)&td->td_epoch_section;
|
||||||
|
record = &epoch->e_pcpu[curcpu]->eps_record.er_record;
|
||||||
|
ck_epoch_end(record, section);
|
||||||
|
critical_exit();
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@ -517,6 +532,23 @@ epoch_wait(epoch_t epoch)
|
|||||||
("%d residual locks held", td->td_locks - locks));
|
("%d residual locks held", td->td_locks - locks));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
epoch_block_handler_critical(struct ck_epoch *g __unused, ck_epoch_record_t *c __unused,
|
||||||
|
void *arg __unused)
|
||||||
|
{
|
||||||
|
cpu_spinwait();
|
||||||
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
epoch_wait_critical(epoch_t epoch)
|
||||||
|
{
|
||||||
|
|
||||||
|
MPASS(epoch->e_flags & EPOCH_CRITICAL);
|
||||||
|
critical_enter();
|
||||||
|
ck_epoch_synchronize_wait(&epoch->e_epoch, epoch_block_handler_critical, NULL);
|
||||||
|
critical_exit();
|
||||||
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t))
|
epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t))
|
||||||
{
|
{
|
||||||
@ -530,8 +562,6 @@ epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t
|
|||||||
if (__predict_false(epoch == NULL))
|
if (__predict_false(epoch == NULL))
|
||||||
goto boottime;
|
goto boottime;
|
||||||
|
|
||||||
counter_u64_add(epoch->e_frees, 1);
|
|
||||||
|
|
||||||
critical_enter();
|
critical_enter();
|
||||||
*DPCPU_PTR(epoch_cb_count) += 1;
|
*DPCPU_PTR(epoch_cb_count) += 1;
|
||||||
eps = epoch->e_pcpu[curcpu];
|
eps = epoch->e_pcpu[curcpu];
|
||||||
@ -542,20 +572,18 @@ epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t
|
|||||||
callback(ctx);
|
callback(ctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static void
|
static void
|
||||||
epoch_call_task(void *arg __unused)
|
epoch_call_task(void *arg __unused)
|
||||||
{
|
{
|
||||||
ck_stack_entry_t *cursor, *head, *next;
|
ck_stack_entry_t *cursor, *head, *next;
|
||||||
ck_epoch_record_t *record;
|
ck_epoch_record_t *record;
|
||||||
ck_epoch_section_t section;
|
|
||||||
epoch_t epoch;
|
epoch_t epoch;
|
||||||
ck_stack_t cb_stack;
|
ck_stack_t cb_stack;
|
||||||
int i, npending, total;
|
int i, npending, total;
|
||||||
|
|
||||||
ck_stack_init(&cb_stack);
|
ck_stack_init(&cb_stack);
|
||||||
critical_enter();
|
critical_enter();
|
||||||
epoch_enter_private(§ion);
|
epoch_enter_critical(global_epoch_critical);
|
||||||
for (total = i = 0; i < epoch_count; i++) {
|
for (total = i = 0; i < epoch_count; i++) {
|
||||||
if (__predict_false((epoch = allepochs[i]) == NULL))
|
if (__predict_false((epoch = allepochs[i]) == NULL))
|
||||||
continue;
|
continue;
|
||||||
@ -565,10 +593,13 @@ epoch_call_task(void *arg __unused)
|
|||||||
ck_epoch_poll_deferred(record, &cb_stack);
|
ck_epoch_poll_deferred(record, &cb_stack);
|
||||||
total += npending - record->n_pending;
|
total += npending - record->n_pending;
|
||||||
}
|
}
|
||||||
epoch_exit_private(§ion);
|
epoch_exit_critical(global_epoch_critical);
|
||||||
*DPCPU_PTR(epoch_cb_count) -= total;
|
*DPCPU_PTR(epoch_cb_count) -= total;
|
||||||
critical_exit();
|
critical_exit();
|
||||||
|
|
||||||
|
counter_u64_add(epoch_call_count, total);
|
||||||
|
counter_u64_add(epoch_call_task_count, 1);
|
||||||
|
|
||||||
head = ck_stack_batch_pop_npsc(&cb_stack);
|
head = ck_stack_batch_pop_npsc(&cb_stack);
|
||||||
for (cursor = head; cursor != NULL; cursor = next) {
|
for (cursor = head; cursor != NULL; cursor = next) {
|
||||||
struct ck_epoch_entry *entry =
|
struct ck_epoch_entry *entry =
|
||||||
|
@ -903,7 +903,7 @@ if_attachdomain(void *dummy)
|
|||||||
{
|
{
|
||||||
struct ifnet *ifp;
|
struct ifnet *ifp;
|
||||||
|
|
||||||
net_epoch = epoch_alloc();
|
net_epoch = epoch_alloc(0);
|
||||||
TAILQ_FOREACH(ifp, &V_ifnet, if_link)
|
TAILQ_FOREACH(ifp, &V_ifnet, if_link)
|
||||||
if_attachdomain1(ifp);
|
if_attachdomain1(ifp);
|
||||||
}
|
}
|
||||||
|
@ -35,7 +35,10 @@
|
|||||||
struct epoch;
|
struct epoch;
|
||||||
typedef struct epoch *epoch_t;
|
typedef struct epoch *epoch_t;
|
||||||
|
|
||||||
|
#define EPOCH_CRITICAL 0x1
|
||||||
|
|
||||||
extern epoch_t global_epoch;
|
extern epoch_t global_epoch;
|
||||||
|
extern epoch_t global_epoch_critical;
|
||||||
DPCPU_DECLARE(int, epoch_cb_count);
|
DPCPU_DECLARE(int, epoch_cb_count);
|
||||||
DPCPU_DECLARE(struct grouptask, epoch_cb_task);
|
DPCPU_DECLARE(struct grouptask, epoch_cb_task);
|
||||||
|
|
||||||
@ -45,11 +48,14 @@ struct epoch_context {
|
|||||||
|
|
||||||
typedef struct epoch_context *epoch_context_t;
|
typedef struct epoch_context *epoch_context_t;
|
||||||
|
|
||||||
epoch_t epoch_alloc(void);
|
epoch_t epoch_alloc(int flags);
|
||||||
void epoch_free(epoch_t epoch);
|
void epoch_free(epoch_t epoch);
|
||||||
|
void epoch_enter_critical(epoch_t epoch);
|
||||||
void epoch_enter_internal(epoch_t epoch, struct thread *td);
|
void epoch_enter_internal(epoch_t epoch, struct thread *td);
|
||||||
|
void epoch_exit_critical(epoch_t epoch);
|
||||||
void epoch_exit_internal(epoch_t epoch, struct thread *td);
|
void epoch_exit_internal(epoch_t epoch, struct thread *td);
|
||||||
void epoch_wait(epoch_t epoch);
|
void epoch_wait(epoch_t epoch);
|
||||||
|
void epoch_wait_critical(epoch_t epoch);
|
||||||
void epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t));
|
void epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t));
|
||||||
int in_epoch(void);
|
int in_epoch(void);
|
||||||
|
|
||||||
|
@ -29,6 +29,7 @@ __FBSDID("$FreeBSD$");
|
|||||||
|
|
||||||
#include <sys/param.h>
|
#include <sys/param.h>
|
||||||
#include <sys/types.h>
|
#include <sys/types.h>
|
||||||
|
#include <sys/proc.h>
|
||||||
#include <sys/counter.h>
|
#include <sys/counter.h>
|
||||||
#include <sys/epoch.h>
|
#include <sys/epoch.h>
|
||||||
#include <sys/gtaskqueue.h>
|
#include <sys/gtaskqueue.h>
|
||||||
@ -138,7 +139,7 @@ test_modinit(void)
|
|||||||
int i, error, pri_range, pri_off;
|
int i, error, pri_range, pri_off;
|
||||||
|
|
||||||
pri_range = PRI_MIN_TIMESHARE - PRI_MIN_REALTIME;
|
pri_range = PRI_MIN_TIMESHARE - PRI_MIN_REALTIME;
|
||||||
test_epoch = epoch_alloc();
|
test_epoch = epoch_alloc(0);
|
||||||
for (i = 0; i < mp_ncpus*2; i++) {
|
for (i = 0; i < mp_ncpus*2; i++) {
|
||||||
etilist[i].threadid = i;
|
etilist[i].threadid = i;
|
||||||
error = kthread_add(testloop, &etilist[i], NULL, &testthreads[i],
|
error = kthread_add(testloop, &etilist[i], NULL, &testthreads[i],
|
||||||
|
Loading…
Reference in New Issue
Block a user