d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

epoch(9): eliminate the need to wait when polling for callbacks to run

Browse Source 
by using ck's own callback handling mechanism we can simply check which
callbacks have had a grace period elapse

Approved by:	sbruno
This commit is contained in:
Matt Macy 2018-05-17 19:50:55 +00:00
parent d1bcb409f6
commit 2a45e8282a
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=333759
1 changed files with 20 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
sys/kern/subr_epoch.c
View File

@ -64,6 +64,7 @@ static MALLOC_DEFINE(M_EPOCH, "epoch", "epoch based reclamation");
#endif
CTASSERT(sizeof(epoch_section_t) == sizeof(ck_epoch_section_t));
CTASSERT(sizeof(ck_epoch_entry_t) == sizeof(struct epoch_context));
SYSCTL_NODE(_kern, OID_AUTO, epoch, CTLFLAG_RW, 0, "epoch information");
SYSCTL_NODE(_kern_epoch, OID_AUTO, stats, CTLFLAG_RW, 0, "epoch stats");
@ -84,13 +85,11 @@ static counter_u64_t switch_count;
SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, switches, CTLFLAG_RW,
&switch_count, "# of times a thread voluntarily context switched in epoch_wait");
typedef struct epoch_cb {
void (*ec_callback)(epoch_context_t);
STAILQ_ENTRY(epoch_cb) ec_link;
} *epoch_cb_t;
TAILQ_HEAD(threadlist, thread);
CK_STACK_CONTAINER(struct ck_epoch_entry, stack_entry,
ck_epoch_entry_container)
typedef struct epoch_record {
ck_epoch_record_t er_record;
volatile struct threadlist er_tdlist;
@ -100,7 +99,6 @@ typedef struct epoch_record {
struct epoch_pcpu_state {
struct epoch_record eps_record;
STAILQ_HEAD(, epoch_cb) eps_cblist;
} __aligned(EPOCH_ALIGN);
struct epoch {
@ -179,7 +177,6 @@ epoch_init_numa(epoch_t epoch)
for (int i = 0; i < domcount[domain]; i++, eps++) {
epoch->e_pcpu[cpu_offset + i] = eps;
er = &eps->eps_record;
STAILQ_INIT(&eps->eps_cblist);
ck_epoch_register(&epoch->e_epoch, &er->er_record, NULL);
TAILQ_INIT((struct threadlist *)(uintptr_t)&er->er_tdlist);
er->er_cpuid = cpu_offset + i;
@ -200,7 +197,6 @@ epoch_init_legacy(epoch_t epoch)
er = &eps->eps_record;
ck_epoch_register(&epoch->e_epoch, &er->er_record, NULL);
TAILQ_INIT((struct threadlist *)(uintptr_t)&er->er_tdlist);
STAILQ_INIT(&eps->eps_cblist);
er->er_cpuid = i;
}
}
@ -525,25 +521,24 @@ void
epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t))
{
struct epoch_pcpu_state *eps;
epoch_cb_t cb;
ck_epoch_entry_t *cb;
cb = (void *)ctx;
MPASS(callback);
/* too early in boot to have epoch set up */
if (__predict_false(epoch == NULL)) {
callback(ctx);
return;
}
MPASS(cb->ec_callback == NULL);
MPASS(cb->ec_link.stqe_next == NULL);
cb->ec_callback = callback;
if (__predict_false(epoch == NULL))
goto boottime;
counter_u64_add(epoch->e_frees, 1);
critical_enter();
eps = epoch->e_pcpu[curcpu];
STAILQ_INSERT_HEAD(&eps->eps_cblist, cb, ec_link);
ck_epoch_call(&eps->eps_record.er_record, cb, (ck_epoch_cb_t*)callback);
critical_exit();
return;
boottime:
callback(ctx);
}
static void
@ -551,28 +546,26 @@ epoch_call_task(void *context)
{
struct epoch_pcpu_state *eps;
epoch_t epoch;
epoch_cb_t cb;
struct thread *td;
ck_stack_entry_t *cursor;
ck_stack_t deferred;
int cpu;
STAILQ_HEAD(, epoch_cb) tmp_head;
epoch = context;
STAILQ_INIT(&tmp_head);
td = curthread;
ck_stack_init(&deferred);
thread_lock(td);
CPU_FOREACH(cpu) {
sched_bind(td, cpu);
eps = epoch->e_pcpu[cpu];
if (!STAILQ_EMPTY(&eps->eps_cblist))
STAILQ_CONCAT(&tmp_head, &eps->eps_cblist);
ck_epoch_poll_deferred(&eps->eps_record.er_record, &deferred);
}
sched_unbind(td);
thread_unlock(td);
epoch_wait(epoch);
while ((cb = STAILQ_FIRST(&tmp_head)) != NULL) {
STAILQ_REMOVE_HEAD(&tmp_head, ec_link);
cb->ec_callback((void*)cb);
while((cursor = ck_stack_pop_npsc(&deferred)) != NULL) {
struct ck_epoch_entry *entry =
ck_epoch_entry_container(cursor);
entry->function(entry);
}
}