Implement per-cpu callout threads, wheels, and locks.
- Move callout thread creation from kern_intr.c to kern_timeout.c - Call callout_tick() on every processor via hardclock_cpu() rather than inspecting callout internal details in kern_clock.c. - Remove callout implementation details from callout.h - Package up all of the global variables into a per-cpu callout structure. - Start one thread per-cpu. Threads are not strictly bound. They prefer to execute on the native cpu but may migrate temporarily if interrupts are starving callout processing. - Run all callouts by default in the thread for cpu0 to maintain current ordering and concurrency guarantees. Many consumers may not properly handle concurrent execution. - The new callout_reset_on() api allows specifying a particular cpu to execute the callout on. This may migrate a callout to a new cpu. callout_reset() schedules on the last assigned cpu while callout_reset_curcpu() schedules on the current cpu. Reviewed by: phk Sponsored by: Nokia
This commit is contained in:
parent
35b450291a
commit
8d809d5061
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=177859
@ -292,6 +292,7 @@ hardclock_cpu(int usermode)
|
||||
if (PMC_CPU_HAS_SAMPLES(PCPU_GET(cpuid)))
|
||||
PMC_CALL_HOOK_UNLOCKED(curthread, PMC_FN_DO_SAMPLES, NULL);
|
||||
#endif
|
||||
callout_tick();
|
||||
}
|
||||
|
||||
/*
|
||||
@ -300,10 +301,9 @@ hardclock_cpu(int usermode)
|
||||
void
|
||||
hardclock(int usermode, uintfptr_t pc)
|
||||
{
|
||||
int need_softclock = 0;
|
||||
|
||||
atomic_add_int((volatile int *)&ticks, 1);
|
||||
hardclock_cpu(usermode);
|
||||
|
||||
tc_ticktock();
|
||||
/*
|
||||
* If no separate statistics clock is available, run it from here.
|
||||
@ -314,30 +314,9 @@ hardclock(int usermode, uintfptr_t pc)
|
||||
profclock(usermode, pc);
|
||||
statclock(usermode);
|
||||
}
|
||||
|
||||
#ifdef DEVICE_POLLING
|
||||
hardclock_device_poll(); /* this is very short and quick */
|
||||
#endif /* DEVICE_POLLING */
|
||||
|
||||
/*
|
||||
* Process callouts at a very low cpu priority, so we don't keep the
|
||||
* relatively high clock interrupt priority any longer than necessary.
|
||||
*/
|
||||
mtx_lock_spin_flags(&callout_lock, MTX_QUIET);
|
||||
ticks++;
|
||||
if (!TAILQ_EMPTY(&callwheel[ticks & callwheelmask])) {
|
||||
need_softclock = 1;
|
||||
} else if (softticks + 1 == ticks)
|
||||
++softticks;
|
||||
mtx_unlock_spin_flags(&callout_lock, MTX_QUIET);
|
||||
|
||||
/*
|
||||
* swi_sched acquires the thread lock, so we don't want to call it
|
||||
* with callout_lock held; incorrect locking order.
|
||||
*/
|
||||
if (need_softclock)
|
||||
swi_sched(softclock_ih, 0);
|
||||
|
||||
#ifdef SW_WATCHDOG
|
||||
if (watchdog_enabled > 0 && --watchdog_ticks <= 0)
|
||||
watchdog_fire();
|
||||
|
@ -79,7 +79,6 @@ struct intr_entropy {
|
||||
|
||||
struct intr_event *clk_intr_event;
|
||||
struct intr_event *tty_intr_event;
|
||||
void *softclock_ih;
|
||||
void *vm_ih;
|
||||
struct proc *intrproc;
|
||||
|
||||
@ -917,6 +916,18 @@ swi_add(struct intr_event **eventp, const char *name, driver_intr_t handler,
|
||||
}
|
||||
return (intr_event_add_handler(ie, name, NULL, handler, arg,
|
||||
(pri * RQ_PPQ) + PI_SOFT, flags, cookiep));
|
||||
error = intr_event_add_handler(ie, name, NULL, handler, arg,
|
||||
(pri * RQ_PPQ) + PI_SOFT, flags, cookiep);
|
||||
if (error)
|
||||
return (error);
|
||||
if (pri == SWI_CLOCK) {
|
||||
struct proc *p;
|
||||
p = ie->ie_thread->it_thread->td_proc;
|
||||
PROC_LOCK(p);
|
||||
p->p_flag |= P_NOLOAD;
|
||||
PROC_UNLOCK(p);
|
||||
}
|
||||
return (0);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -1548,17 +1559,9 @@ DB_SHOW_COMMAND(intr, db_show_intr)
|
||||
static void
|
||||
start_softintr(void *dummy)
|
||||
{
|
||||
struct proc *p;
|
||||
|
||||
if (swi_add(&clk_intr_event, "clock", softclock, NULL, SWI_CLOCK,
|
||||
INTR_MPSAFE, &softclock_ih) ||
|
||||
swi_add(NULL, "vm", swi_vm, NULL, SWI_VM, INTR_MPSAFE, &vm_ih))
|
||||
panic("died while creating standard software ithreads");
|
||||
|
||||
p = clk_intr_event->ie_thread->it_thread->td_proc;
|
||||
PROC_LOCK(p);
|
||||
p->p_flag |= P_NOLOAD;
|
||||
PROC_UNLOCK(p);
|
||||
if (swi_add(NULL, "vm", swi_vm, NULL, SWI_VM, INTR_MPSAFE, &vm_ih))
|
||||
panic("died while creating vm swi ithread");
|
||||
}
|
||||
SYSINIT(start_softintr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softintr,
|
||||
NULL);
|
||||
|
@ -39,15 +39,19 @@ __FBSDID("$FreeBSD$");
|
||||
|
||||
#include <sys/param.h>
|
||||
#include <sys/systm.h>
|
||||
#include <sys/bus.h>
|
||||
#include <sys/callout.h>
|
||||
#include <sys/condvar.h>
|
||||
#include <sys/interrupt.h>
|
||||
#include <sys/kernel.h>
|
||||
#include <sys/ktr.h>
|
||||
#include <sys/lock.h>
|
||||
#include <sys/malloc.h>
|
||||
#include <sys/mutex.h>
|
||||
#include <sys/proc.h>
|
||||
#include <sys/sleepqueue.h>
|
||||
#include <sys/sysctl.h>
|
||||
#include <sys/smp.h>
|
||||
|
||||
static int avg_depth;
|
||||
SYSCTL_INT(_debug, OID_AUTO, to_avg_depth, CTLFLAG_RD, &avg_depth, 0,
|
||||
@ -65,36 +69,53 @@ SYSCTL_INT(_debug, OID_AUTO, to_avg_mpcalls, CTLFLAG_RD, &avg_mpcalls, 0,
|
||||
* TODO:
|
||||
* allocate more timeout table slots when table overflows.
|
||||
*/
|
||||
|
||||
/* Exported to machdep.c and/or kern_clock.c. */
|
||||
struct callout *callout;
|
||||
struct callout_list callfree;
|
||||
int callwheelsize, callwheelbits, callwheelmask;
|
||||
struct callout_tailq *callwheel;
|
||||
int softticks; /* Like ticks, but for softclock(). */
|
||||
struct mtx callout_lock;
|
||||
|
||||
static struct callout *nextsoftcheck; /* Next callout to be checked. */
|
||||
struct callout_cpu {
|
||||
struct mtx cc_lock;
|
||||
struct callout *cc_callout;
|
||||
struct callout_tailq *cc_callwheel;
|
||||
struct callout_list cc_callfree;
|
||||
struct callout *cc_next;
|
||||
struct callout *cc_curr;
|
||||
void *cc_cookie;
|
||||
int cc_softticks;
|
||||
int cc_cancel;
|
||||
int cc_waiting;
|
||||
};
|
||||
|
||||
#ifdef SMP
|
||||
struct callout_cpu cc_cpu[MAXCPU];
|
||||
#define CC_CPU(cpu) (&cc_cpu[(cpu)])
|
||||
#define CC_SELF() CC_CPU(PCPU_GET(cpuid))
|
||||
#else
|
||||
struct callout_cpu cc_cpu;
|
||||
#define CC_CPU(cpu) &cc_cpu
|
||||
#define CC_SELF() &cc_cpu
|
||||
#endif
|
||||
#define CC_LOCK(cc) mtx_lock_spin(&(cc)->cc_lock)
|
||||
#define CC_UNLOCK(cc) mtx_unlock_spin(&(cc)->cc_lock)
|
||||
|
||||
static int timeout_cpu;
|
||||
|
||||
MALLOC_DEFINE(M_CALLOUT, "callout", "Callout datastructures");
|
||||
|
||||
/**
|
||||
* Locked by callout_lock:
|
||||
* curr_callout - If a callout is in progress, it is curr_callout.
|
||||
* Locked by cc_lock:
|
||||
* cc_curr - If a callout is in progress, it is curr_callout.
|
||||
* If curr_callout is non-NULL, threads waiting in
|
||||
* callout_drain() will be woken up as soon as the
|
||||
* callout_drain() will be woken up as soon as the
|
||||
* relevant callout completes.
|
||||
* curr_cancelled - Changing to 1 with both callout_lock and c_lock held
|
||||
* cc_cancel - Changing to 1 with both callout_lock and c_lock held
|
||||
* guarantees that the current callout will not run.
|
||||
* The softclock() function sets this to 0 before it
|
||||
* drops callout_lock to acquire c_lock, and it calls
|
||||
* the handler only if curr_cancelled is still 0 after
|
||||
* c_lock is successfully acquired.
|
||||
* callout_wait - If a thread is waiting in callout_drain(), then
|
||||
* cc_waiting - If a thread is waiting in callout_drain(), then
|
||||
* callout_wait is nonzero. Set only when
|
||||
* curr_callout is non-NULL.
|
||||
*/
|
||||
static struct callout *curr_callout;
|
||||
static int curr_cancelled;
|
||||
static int callout_wait;
|
||||
|
||||
/*
|
||||
* kern_timeout_callwheel_alloc() - kernel low level callwheel initialization
|
||||
@ -105,6 +126,10 @@ static int callout_wait;
|
||||
caddr_t
|
||||
kern_timeout_callwheel_alloc(caddr_t v)
|
||||
{
|
||||
struct callout_cpu *cc;
|
||||
|
||||
timeout_cpu = PCPU_GET(cpuid);
|
||||
cc = CC_CPU(timeout_cpu);
|
||||
/*
|
||||
* Calculate callout wheel size
|
||||
*/
|
||||
@ -114,13 +139,34 @@ kern_timeout_callwheel_alloc(caddr_t v)
|
||||
;
|
||||
callwheelmask = callwheelsize - 1;
|
||||
|
||||
callout = (struct callout *)v;
|
||||
v = (caddr_t)(callout + ncallout);
|
||||
callwheel = (struct callout_tailq *)v;
|
||||
v = (caddr_t)(callwheel + callwheelsize);
|
||||
cc->cc_callout = (struct callout *)v;
|
||||
v = (caddr_t)(cc->cc_callout + ncallout);
|
||||
cc->cc_callwheel = (struct callout_tailq *)v;
|
||||
v = (caddr_t)(cc->cc_callwheel + callwheelsize);
|
||||
return(v);
|
||||
}
|
||||
|
||||
static void
|
||||
callout_cpu_init(struct callout_cpu *cc)
|
||||
{
|
||||
struct callout *c;
|
||||
int i;
|
||||
|
||||
mtx_init(&cc->cc_lock, "callout", NULL, MTX_SPIN | MTX_RECURSE);
|
||||
SLIST_INIT(&cc->cc_callfree);
|
||||
for (i = 0; i < callwheelsize; i++) {
|
||||
TAILQ_INIT(&cc->cc_callwheel[i]);
|
||||
}
|
||||
if (cc->cc_callout == NULL)
|
||||
return;
|
||||
for (i = 0; i < ncallout; i++) {
|
||||
c = &cc->cc_callout[i];
|
||||
callout_init(c, 0);
|
||||
c->c_flags = CALLOUT_LOCAL_ALLOC;
|
||||
SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* kern_timeout_callwheel_init() - initialize previously reserved callwheel
|
||||
* space.
|
||||
@ -131,18 +177,88 @@ kern_timeout_callwheel_alloc(caddr_t v)
|
||||
void
|
||||
kern_timeout_callwheel_init(void)
|
||||
{
|
||||
int i;
|
||||
callout_cpu_init(CC_CPU(timeout_cpu));
|
||||
}
|
||||
|
||||
SLIST_INIT(&callfree);
|
||||
for (i = 0; i < ncallout; i++) {
|
||||
callout_init(&callout[i], 0);
|
||||
callout[i].c_flags = CALLOUT_LOCAL_ALLOC;
|
||||
SLIST_INSERT_HEAD(&callfree, &callout[i], c_links.sle);
|
||||
/*
|
||||
* Start standard softclock thread.
|
||||
*/
|
||||
void *softclock_ih;
|
||||
|
||||
static void
|
||||
start_softclock(void *dummy)
|
||||
{
|
||||
struct callout_cpu *cc;
|
||||
#ifdef SMP
|
||||
int cpu;
|
||||
#endif
|
||||
|
||||
cc = CC_CPU(timeout_cpu);
|
||||
if (swi_add(&clk_intr_event, "clock", softclock, cc, SWI_CLOCK,
|
||||
INTR_MPSAFE, &softclock_ih))
|
||||
panic("died while creating standard software ithreads");
|
||||
cc->cc_cookie = softclock_ih;
|
||||
#ifdef SMP
|
||||
for (cpu = 0; cpu <= mp_maxid; cpu++) {
|
||||
if (cpu == timeout_cpu)
|
||||
continue;
|
||||
if (CPU_ABSENT(cpu))
|
||||
continue;
|
||||
cc = CC_CPU(cpu);
|
||||
if (swi_add(NULL, "clock", softclock, cc, SWI_CLOCK,
|
||||
INTR_MPSAFE, &cc->cc_cookie))
|
||||
panic("died while creating standard software ithreads");
|
||||
cc->cc_callout = NULL; /* Only cpu0 handles timeout(). */
|
||||
cc->cc_callwheel = malloc(
|
||||
sizeof(struct callout_tailq) * callwheelsize, M_CALLOUT,
|
||||
M_WAITOK);
|
||||
callout_cpu_init(cc);
|
||||
}
|
||||
for (i = 0; i < callwheelsize; i++) {
|
||||
TAILQ_INIT(&callwheel[i]);
|
||||
#endif
|
||||
}
|
||||
|
||||
SYSINIT(start_softclock, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softclock, NULL);
|
||||
|
||||
void
|
||||
callout_tick(void)
|
||||
{
|
||||
int need_softclock = 0;
|
||||
struct callout_cpu *cc;
|
||||
|
||||
/*
|
||||
* Process callouts at a very low cpu priority, so we don't keep the
|
||||
* relatively high clock interrupt priority any longer than necessary.
|
||||
*/
|
||||
cc = CC_SELF();
|
||||
mtx_lock_spin_flags(&cc->cc_lock, MTX_QUIET);
|
||||
if (!TAILQ_EMPTY(&cc->cc_callwheel[ticks & callwheelmask])) {
|
||||
need_softclock = 1;
|
||||
} else if (cc->cc_softticks + 1 == ticks)
|
||||
++cc->cc_softticks;
|
||||
mtx_unlock_spin_flags(&cc->cc_lock, MTX_QUIET);
|
||||
/*
|
||||
* swi_sched acquires the thread lock, so we don't want to call it
|
||||
* with cc_lock held; incorrect locking order.
|
||||
*/
|
||||
if (need_softclock)
|
||||
swi_sched(cc->cc_cookie, 0);
|
||||
}
|
||||
|
||||
static struct callout_cpu *
|
||||
callout_lock(struct callout *c)
|
||||
{
|
||||
struct callout_cpu *cc;
|
||||
int cpu;
|
||||
|
||||
for (;;) {
|
||||
cpu = c->c_cpu;
|
||||
cc = CC_CPU(cpu);
|
||||
CC_LOCK(cc);
|
||||
if (cpu == c->c_cpu)
|
||||
break;
|
||||
CC_UNLOCK(cc);
|
||||
}
|
||||
mtx_init(&callout_lock, "callout", NULL, MTX_SPIN | MTX_RECURSE);
|
||||
return (cc);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -162,8 +278,9 @@ kern_timeout_callwheel_init(void)
|
||||
* Run periodic events from timeout queue.
|
||||
*/
|
||||
void
|
||||
softclock(void *dummy)
|
||||
softclock(void *arg)
|
||||
{
|
||||
struct callout_cpu *cc;
|
||||
struct callout *c;
|
||||
struct callout_tailq *bucket;
|
||||
int curticks;
|
||||
@ -188,15 +305,16 @@ softclock(void *dummy)
|
||||
gcalls = 0;
|
||||
depth = 0;
|
||||
steps = 0;
|
||||
mtx_lock_spin(&callout_lock);
|
||||
while (softticks != ticks) {
|
||||
softticks++;
|
||||
cc = (struct callout_cpu *)arg;
|
||||
CC_LOCK(cc);
|
||||
while (cc->cc_softticks != ticks) {
|
||||
cc->cc_softticks++;
|
||||
/*
|
||||
* softticks may be modified by hard clock, so cache
|
||||
* cc_softticks may be modified by hard clock, so cache
|
||||
* it while we work on a given bucket.
|
||||
*/
|
||||
curticks = softticks;
|
||||
bucket = &callwheel[curticks & callwheelmask];
|
||||
curticks = cc->cc_softticks;
|
||||
bucket = &cc->cc_callwheel[curticks & callwheelmask];
|
||||
c = TAILQ_FIRST(bucket);
|
||||
while (c) {
|
||||
depth++;
|
||||
@ -204,12 +322,12 @@ softclock(void *dummy)
|
||||
c = TAILQ_NEXT(c, c_links.tqe);
|
||||
++steps;
|
||||
if (steps >= MAX_SOFTCLOCK_STEPS) {
|
||||
nextsoftcheck = c;
|
||||
cc->cc_next = c;
|
||||
/* Give interrupts a chance. */
|
||||
mtx_unlock_spin(&callout_lock);
|
||||
CC_UNLOCK(cc);
|
||||
; /* nothing */
|
||||
mtx_lock_spin(&callout_lock);
|
||||
c = nextsoftcheck;
|
||||
CC_LOCK(cc);
|
||||
c = cc->cc_next;
|
||||
steps = 0;
|
||||
}
|
||||
} else {
|
||||
@ -219,7 +337,7 @@ softclock(void *dummy)
|
||||
struct lock_object *c_lock;
|
||||
int c_flags, sharedlock;
|
||||
|
||||
nextsoftcheck = TAILQ_NEXT(c, c_links.tqe);
|
||||
cc->cc_next = TAILQ_NEXT(c, c_links.tqe);
|
||||
TAILQ_REMOVE(bucket, c, c_links.tqe);
|
||||
class = (c->c_lock != NULL) ?
|
||||
LOCK_CLASS(c->c_lock) : NULL;
|
||||
@ -231,26 +349,25 @@ softclock(void *dummy)
|
||||
c_flags = c->c_flags;
|
||||
if (c->c_flags & CALLOUT_LOCAL_ALLOC) {
|
||||
c->c_flags = CALLOUT_LOCAL_ALLOC;
|
||||
curr_callout = c;
|
||||
} else {
|
||||
c->c_flags =
|
||||
(c->c_flags & ~CALLOUT_PENDING);
|
||||
curr_callout = c;
|
||||
}
|
||||
curr_cancelled = 0;
|
||||
mtx_unlock_spin(&callout_lock);
|
||||
cc->cc_curr = c;
|
||||
cc->cc_cancel = 0;
|
||||
CC_UNLOCK(cc);
|
||||
if (c_lock != NULL) {
|
||||
class->lc_lock(c_lock, sharedlock);
|
||||
/*
|
||||
* The callout may have been cancelled
|
||||
* while we switched locks.
|
||||
*/
|
||||
if (curr_cancelled) {
|
||||
if (cc->cc_cancel) {
|
||||
class->lc_unlock(c_lock);
|
||||
goto skip;
|
||||
}
|
||||
/* The callout cannot be stopped now. */
|
||||
curr_cancelled = 1;
|
||||
cc->cc_cancel = 1;
|
||||
|
||||
if (c_lock == &Giant.lock_object) {
|
||||
gcalls++;
|
||||
@ -295,7 +412,7 @@ softclock(void *dummy)
|
||||
if ((c_flags & CALLOUT_RETURNUNLOCKED) == 0)
|
||||
class->lc_unlock(c_lock);
|
||||
skip:
|
||||
mtx_lock_spin(&callout_lock);
|
||||
CC_LOCK(cc);
|
||||
/*
|
||||
* If the current callout is locally
|
||||
* allocated (from timeout(9))
|
||||
@ -311,22 +428,22 @@ softclock(void *dummy)
|
||||
CALLOUT_LOCAL_ALLOC,
|
||||
("corrupted callout"));
|
||||
c->c_func = NULL;
|
||||
SLIST_INSERT_HEAD(&callfree, c,
|
||||
SLIST_INSERT_HEAD(&cc->cc_callfree, c,
|
||||
c_links.sle);
|
||||
}
|
||||
curr_callout = NULL;
|
||||
if (callout_wait) {
|
||||
cc->cc_curr = NULL;
|
||||
if (cc->cc_waiting) {
|
||||
/*
|
||||
* There is someone waiting
|
||||
* for the callout to complete.
|
||||
*/
|
||||
callout_wait = 0;
|
||||
mtx_unlock_spin(&callout_lock);
|
||||
wakeup(&callout_wait);
|
||||
mtx_lock_spin(&callout_lock);
|
||||
cc->cc_waiting = 0;
|
||||
CC_UNLOCK(cc);
|
||||
wakeup(&cc->cc_waiting);
|
||||
CC_LOCK(cc);
|
||||
}
|
||||
steps = 0;
|
||||
c = nextsoftcheck;
|
||||
c = cc->cc_next;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -334,8 +451,8 @@ softclock(void *dummy)
|
||||
avg_mpcalls += (mpcalls * 1000 - avg_mpcalls) >> 8;
|
||||
avg_lockcalls += (lockcalls * 1000 - avg_lockcalls) >> 8;
|
||||
avg_gcalls += (gcalls * 1000 - avg_gcalls) >> 8;
|
||||
nextsoftcheck = NULL;
|
||||
mtx_unlock_spin(&callout_lock);
|
||||
cc->cc_next = NULL;
|
||||
CC_UNLOCK(cc);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -360,22 +477,22 @@ timeout(ftn, arg, to_ticks)
|
||||
void *arg;
|
||||
int to_ticks;
|
||||
{
|
||||
struct callout_cpu *cc;
|
||||
struct callout *new;
|
||||
struct callout_handle handle;
|
||||
|
||||
mtx_lock_spin(&callout_lock);
|
||||
|
||||
cc = CC_CPU(timeout_cpu);
|
||||
CC_LOCK(cc);
|
||||
/* Fill in the next free callout structure. */
|
||||
new = SLIST_FIRST(&callfree);
|
||||
new = SLIST_FIRST(&cc->cc_callfree);
|
||||
if (new == NULL)
|
||||
/* XXX Attempt to malloc first */
|
||||
panic("timeout table full");
|
||||
SLIST_REMOVE_HEAD(&callfree, c_links.sle);
|
||||
|
||||
SLIST_REMOVE_HEAD(&cc->cc_callfree, c_links.sle);
|
||||
callout_reset(new, to_ticks, ftn, arg);
|
||||
|
||||
handle.callout = new;
|
||||
mtx_unlock_spin(&callout_lock);
|
||||
CC_UNLOCK(cc);
|
||||
|
||||
return (handle);
|
||||
}
|
||||
|
||||
@ -385,6 +502,7 @@ untimeout(ftn, arg, handle)
|
||||
void *arg;
|
||||
struct callout_handle handle;
|
||||
{
|
||||
struct callout_cpu *cc;
|
||||
|
||||
/*
|
||||
* Check for a handle that was initialized
|
||||
@ -394,10 +512,10 @@ untimeout(ftn, arg, handle)
|
||||
if (handle.callout == NULL)
|
||||
return;
|
||||
|
||||
mtx_lock_spin(&callout_lock);
|
||||
cc = callout_lock(handle.callout);
|
||||
if (handle.callout->c_func == ftn && handle.callout->c_arg == arg)
|
||||
callout_stop(handle.callout);
|
||||
mtx_unlock_spin(&callout_lock);
|
||||
CC_UNLOCK(cc);
|
||||
}
|
||||
|
||||
void
|
||||
@ -423,24 +541,29 @@ callout_handle_init(struct callout_handle *handle)
|
||||
* callout_deactivate() - marks the callout as having been serviced
|
||||
*/
|
||||
int
|
||||
callout_reset(c, to_ticks, ftn, arg)
|
||||
struct callout *c;
|
||||
int to_ticks;
|
||||
void (*ftn)(void *);
|
||||
void *arg;
|
||||
callout_reset_on(struct callout *c, int to_ticks, void (*ftn)(void *),
|
||||
void *arg, int cpu)
|
||||
{
|
||||
struct callout_cpu *cc;
|
||||
int cancelled = 0;
|
||||
|
||||
mtx_lock_spin(&callout_lock);
|
||||
if (c == curr_callout) {
|
||||
/*
|
||||
* Don't allow migration of pre-allocated callouts lest they
|
||||
* become unbalanced.
|
||||
*/
|
||||
if (c->c_flags & CALLOUT_LOCAL_ALLOC)
|
||||
cpu = c->c_cpu;
|
||||
retry:
|
||||
cc = callout_lock(c);
|
||||
if (cc->cc_curr == c) {
|
||||
/*
|
||||
* We're being asked to reschedule a callout which is
|
||||
* currently in progress. If there is a lock then we
|
||||
* can cancel the callout if it has not really started.
|
||||
*/
|
||||
if (c->c_lock != NULL && !curr_cancelled)
|
||||
cancelled = curr_cancelled = 1;
|
||||
if (callout_wait) {
|
||||
if (c->c_lock != NULL && !cc->cc_cancel)
|
||||
cancelled = cc->cc_cancel = 1;
|
||||
if (cc->cc_waiting) {
|
||||
/*
|
||||
* Someone has called callout_drain to kill this
|
||||
* callout. Don't reschedule.
|
||||
@ -448,34 +571,30 @@ callout_reset(c, to_ticks, ftn, arg)
|
||||
CTR4(KTR_CALLOUT, "%s %p func %p arg %p",
|
||||
cancelled ? "cancelled" : "failed to cancel",
|
||||
c, c->c_func, c->c_arg);
|
||||
mtx_unlock_spin(&callout_lock);
|
||||
CC_UNLOCK(cc);
|
||||
return (cancelled);
|
||||
}
|
||||
}
|
||||
if (c->c_flags & CALLOUT_PENDING) {
|
||||
if (nextsoftcheck == c) {
|
||||
nextsoftcheck = TAILQ_NEXT(c, c_links.tqe);
|
||||
if (cc->cc_next == c) {
|
||||
cc->cc_next = TAILQ_NEXT(c, c_links.tqe);
|
||||
}
|
||||
TAILQ_REMOVE(&callwheel[c->c_time & callwheelmask], c,
|
||||
TAILQ_REMOVE(&cc->cc_callwheel[c->c_time & callwheelmask], c,
|
||||
c_links.tqe);
|
||||
|
||||
cancelled = 1;
|
||||
|
||||
/*
|
||||
* Part of the normal "stop a pending callout" process
|
||||
* is to clear the CALLOUT_ACTIVE and CALLOUT_PENDING
|
||||
* flags. We're not going to bother doing that here,
|
||||
* because we're going to be setting those flags ten lines
|
||||
* after this point, and we're holding callout_lock
|
||||
* between now and then.
|
||||
*/
|
||||
c->c_flags &= ~(CALLOUT_ACTIVE | CALLOUT_PENDING);
|
||||
}
|
||||
/*
|
||||
* If the lock must migrate we have to check the state again as
|
||||
* we can't hold both the new and old locks simultaneously.
|
||||
*/
|
||||
if (c->c_cpu != cpu) {
|
||||
c->c_cpu = cpu;
|
||||
CC_UNLOCK(cc);
|
||||
goto retry;
|
||||
}
|
||||
|
||||
/*
|
||||
* We could unlock callout_lock here and lock it again before the
|
||||
* TAILQ_INSERT_TAIL, but there's no point since doing this setup
|
||||
* doesn't take much time.
|
||||
*/
|
||||
if (to_ticks <= 0)
|
||||
to_ticks = 1;
|
||||
|
||||
@ -483,11 +602,11 @@ callout_reset(c, to_ticks, ftn, arg)
|
||||
c->c_flags |= (CALLOUT_ACTIVE | CALLOUT_PENDING);
|
||||
c->c_func = ftn;
|
||||
c->c_time = ticks + to_ticks;
|
||||
TAILQ_INSERT_TAIL(&callwheel[c->c_time & callwheelmask],
|
||||
TAILQ_INSERT_TAIL(&cc->cc_callwheel[c->c_time & callwheelmask],
|
||||
c, c_links.tqe);
|
||||
CTR5(KTR_CALLOUT, "%sscheduled %p func %p arg %p in %d",
|
||||
cancelled ? "re" : "", c, c->c_func, c->c_arg, to_ticks);
|
||||
mtx_unlock_spin(&callout_lock);
|
||||
CC_UNLOCK(cc);
|
||||
|
||||
return (cancelled);
|
||||
}
|
||||
@ -497,6 +616,7 @@ _callout_stop_safe(c, safe)
|
||||
struct callout *c;
|
||||
int safe;
|
||||
{
|
||||
struct callout_cpu *cc;
|
||||
struct lock_class *class;
|
||||
int use_lock, sq_locked;
|
||||
|
||||
@ -517,7 +637,7 @@ _callout_stop_safe(c, safe)
|
||||
|
||||
sq_locked = 0;
|
||||
again:
|
||||
mtx_lock_spin(&callout_lock);
|
||||
cc = callout_lock(c);
|
||||
/*
|
||||
* If the callout isn't pending, it's not on the queue, so
|
||||
* don't attempt to remove it from the queue. We can try to
|
||||
@ -530,12 +650,12 @@ _callout_stop_safe(c, safe)
|
||||
* If it wasn't on the queue and it isn't the current
|
||||
* callout, then we can't stop it, so just bail.
|
||||
*/
|
||||
if (c != curr_callout) {
|
||||
if (cc->cc_curr != c) {
|
||||
CTR3(KTR_CALLOUT, "failed to stop %p func %p arg %p",
|
||||
c, c->c_func, c->c_arg);
|
||||
mtx_unlock_spin(&callout_lock);
|
||||
CC_UNLOCK(cc);
|
||||
if (sq_locked)
|
||||
sleepq_release(&callout_wait);
|
||||
sleepq_release(&cc->cc_curr);
|
||||
return (0);
|
||||
}
|
||||
|
||||
@ -546,19 +666,19 @@ _callout_stop_safe(c, safe)
|
||||
* just wait for the current invocation to
|
||||
* finish.
|
||||
*/
|
||||
while (c == curr_callout) {
|
||||
while (cc->cc_curr == c) {
|
||||
|
||||
/*
|
||||
* Use direct calls to sleepqueue interface
|
||||
* instead of cv/msleep in order to avoid
|
||||
* a LOR between callout_lock and sleepqueue
|
||||
* a LOR between cc_lock and sleepqueue
|
||||
* chain spinlocks. This piece of code
|
||||
* emulates a msleep_spin() call actually.
|
||||
*
|
||||
* If we already have the sleepqueue chain
|
||||
* locked, then we can safely block. If we
|
||||
* don't already have it locked, however,
|
||||
* we have to drop the callout_lock to lock
|
||||
* we have to drop the cc_lock to lock
|
||||
* it. This opens several races, so we
|
||||
* restart at the beginning once we have
|
||||
* both locks. If nothing has changed, then
|
||||
@ -566,26 +686,25 @@ _callout_stop_safe(c, safe)
|
||||
* set.
|
||||
*/
|
||||
if (!sq_locked) {
|
||||
mtx_unlock_spin(&callout_lock);
|
||||
sleepq_lock(&callout_wait);
|
||||
CC_UNLOCK(cc);
|
||||
sleepq_lock(&cc->cc_curr);
|
||||
sq_locked = 1;
|
||||
goto again;
|
||||
}
|
||||
|
||||
callout_wait = 1;
|
||||
cc->cc_waiting = 1;
|
||||
DROP_GIANT();
|
||||
mtx_unlock_spin(&callout_lock);
|
||||
sleepq_add(&callout_wait,
|
||||
&callout_lock.lock_object, "codrain",
|
||||
CC_UNLOCK(cc);
|
||||
sleepq_add(&cc->cc_curr,
|
||||
&cc->cc_lock.lock_object, "codrain",
|
||||
SLEEPQ_SLEEP, 0);
|
||||
sleepq_wait(&callout_wait, 0);
|
||||
sleepq_wait(&cc->cc_curr, 0);
|
||||
sq_locked = 0;
|
||||
|
||||
/* Reacquire locks previously released. */
|
||||
PICKUP_GIANT();
|
||||
mtx_lock_spin(&callout_lock);
|
||||
CC_LOCK(cc);
|
||||
}
|
||||
} else if (use_lock && !curr_cancelled) {
|
||||
} else if (use_lock && !cc->cc_cancel) {
|
||||
/*
|
||||
* The current callout is waiting for its
|
||||
* lock which we hold. Cancel the callout
|
||||
@ -593,37 +712,38 @@ _callout_stop_safe(c, safe)
|
||||
* lock, the callout will be skipped in
|
||||
* softclock().
|
||||
*/
|
||||
curr_cancelled = 1;
|
||||
cc->cc_cancel = 1;
|
||||
CTR3(KTR_CALLOUT, "cancelled %p func %p arg %p",
|
||||
c, c->c_func, c->c_arg);
|
||||
mtx_unlock_spin(&callout_lock);
|
||||
CC_UNLOCK(cc);
|
||||
KASSERT(!sq_locked, ("sleepqueue chain locked"));
|
||||
return (1);
|
||||
}
|
||||
CTR3(KTR_CALLOUT, "failed to stop %p func %p arg %p",
|
||||
c, c->c_func, c->c_arg);
|
||||
mtx_unlock_spin(&callout_lock);
|
||||
CC_UNLOCK(cc);
|
||||
KASSERT(!sq_locked, ("sleepqueue chain still locked"));
|
||||
return (0);
|
||||
}
|
||||
if (sq_locked)
|
||||
sleepq_release(&callout_wait);
|
||||
sleepq_release(&cc->cc_curr);
|
||||
|
||||
c->c_flags &= ~(CALLOUT_ACTIVE | CALLOUT_PENDING);
|
||||
|
||||
if (nextsoftcheck == c) {
|
||||
nextsoftcheck = TAILQ_NEXT(c, c_links.tqe);
|
||||
if (cc->cc_next == c) {
|
||||
cc->cc_next = TAILQ_NEXT(c, c_links.tqe);
|
||||
}
|
||||
TAILQ_REMOVE(&callwheel[c->c_time & callwheelmask], c, c_links.tqe);
|
||||
TAILQ_REMOVE(&cc->cc_callwheel[c->c_time & callwheelmask], c,
|
||||
c_links.tqe);
|
||||
|
||||
CTR3(KTR_CALLOUT, "cancelled %p func %p arg %p",
|
||||
c, c->c_func, c->c_arg);
|
||||
|
||||
if (c->c_flags & CALLOUT_LOCAL_ALLOC) {
|
||||
c->c_func = NULL;
|
||||
SLIST_INSERT_HEAD(&callfree, c, c_links.sle);
|
||||
SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle);
|
||||
}
|
||||
mtx_unlock_spin(&callout_lock);
|
||||
CC_UNLOCK(cc);
|
||||
return (1);
|
||||
}
|
||||
|
||||
@ -640,6 +760,7 @@ callout_init(c, mpsafe)
|
||||
c->c_lock = &Giant.lock_object;
|
||||
c->c_flags = 0;
|
||||
}
|
||||
c->c_cpu = timeout_cpu;
|
||||
}
|
||||
|
||||
void
|
||||
@ -658,6 +779,7 @@ _callout_init_lock(c, lock, flags)
|
||||
(LC_SPINLOCK | LC_SLEEPABLE)), ("%s: invalid lock class",
|
||||
__func__));
|
||||
c->c_flags = flags & (CALLOUT_RETURNUNLOCKED | CALLOUT_SHAREDLOCK);
|
||||
c->c_cpu = timeout_cpu;
|
||||
}
|
||||
|
||||
#ifdef APM_FIXUP_CALLTODO
|
||||
@ -709,7 +831,7 @@ adjust_timeout_calltodo(time_change)
|
||||
*/
|
||||
|
||||
/* don't collide with softclock() */
|
||||
mtx_lock_spin(&callout_lock);
|
||||
CC_LOCK(cc);
|
||||
for (p = calltodo.c_next; p != NULL; p = p->c_next) {
|
||||
p->c_time -= delta_ticks;
|
||||
|
||||
@ -720,7 +842,7 @@ adjust_timeout_calltodo(time_change)
|
||||
/* take back the ticks the timer didn't use (p->c_time <= 0) */
|
||||
delta_ticks = -p->c_time;
|
||||
}
|
||||
mtx_unlock_spin(&callout_lock);
|
||||
CC_UNLOCK(cc);
|
||||
|
||||
return;
|
||||
}
|
||||
|
@ -52,9 +52,10 @@ struct callout {
|
||||
} c_links;
|
||||
int c_time; /* ticks to the event */
|
||||
void *c_arg; /* function argument */
|
||||
void (*c_func)(void *); /* function to call */
|
||||
void (*c_func)(void *); /* function to call */
|
||||
struct lock_object *c_lock; /* lock to handle */
|
||||
int c_flags; /* state of this entry */
|
||||
volatile int c_cpu; /* CPU we're scheduled on */
|
||||
};
|
||||
|
||||
#define CALLOUT_LOCAL_ALLOC 0x0001 /* was allocated from callfree */
|
||||
@ -69,12 +70,7 @@ struct callout_handle {
|
||||
};
|
||||
|
||||
#ifdef _KERNEL
|
||||
extern struct callout_list callfree;
|
||||
extern struct callout *callout;
|
||||
extern int ncallout;
|
||||
extern struct callout_tailq *callwheel;
|
||||
extern int callwheelsize, callwheelbits, callwheelmask, softticks;
|
||||
extern struct mtx callout_lock;
|
||||
|
||||
#define callout_active(c) ((c)->c_flags & CALLOUT_ACTIVE)
|
||||
#define callout_deactivate(c) ((c)->c_flags &= ~CALLOUT_ACTIVE)
|
||||
@ -88,9 +84,15 @@ void _callout_init_lock(struct callout *, struct lock_object *, int);
|
||||
_callout_init_lock((c), ((rw) != NULL) ? &(rw)->lock_object : \
|
||||
NULL, (flags))
|
||||
#define callout_pending(c) ((c)->c_flags & CALLOUT_PENDING)
|
||||
int callout_reset(struct callout *, int, void (*)(void *), void *);
|
||||
int callout_reset_on(struct callout *, int, void (*)(void *), void *, int);
|
||||
#define callout_reset(c, on_tick, fn, arg) \
|
||||
callout_reset_on((c), (on_tick), (fn), (arg), (c)->c_cpu)
|
||||
#define callout_reset_curcpu(c, on_tick, fn, arg) \
|
||||
callout_reset_on((c), (on_tick), (fn), (arg), PCPU_GET(cpuid))
|
||||
#define callout_stop(c) _callout_stop_safe(c, 0)
|
||||
int _callout_stop_safe(struct callout *, int);
|
||||
void callout_tick(void);
|
||||
|
||||
|
||||
#endif
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user