If a thread that is swapped out is made runnable, then the setrunnable()
routine wakes up proc0 so that proc0 can swap the thread back in. Historically, this has been done by waking up proc0 directly from setrunnable() itself via a wakeup(). When waking up a sleeping thread that was swapped out (the usual case when waking proc0 since only sleeping threads are eligible to be swapped out), this resulted in a bit of recursion (e.g. wakeup() -> setrunnable() -> wakeup()). With sleep queues having separate locks in 6.x and later, this caused a spin lock LOR (sleepq lock -> sched_lock/thread lock -> sleepq lock). An attempt was made to fix this in 7.0 by making the proc0 wakeup use the ithread mechanism for doing the wakeup. However, this required grabbing proc0's thread lock to perform the wakeup. If proc0 was asleep elsewhere in the kernel (e.g. waiting for disk I/O), then this degenerated into the same LOR since the thread lock would be some other sleepq lock. Fix this by deferring the wakeup of the swapper until after the sleepq lock held by the upper layer has been locked. The setrunnable() routine now returns a boolean value to indicate whether or not proc0 needs to be woken up. The end result is that consumers of the sleepq API such as *sleep/wakeup, condition variables, sx locks, and lockmgr, have to wakeup proc0 if they get a non-zero return value from sleepq_abort(), sleepq_broadcast(), or sleepq_signal(). Discussed with: jeff Glanced at by: sam Tested by: Jurgen Weber jurgen - ish com au MFC after: 2 weeks
This commit is contained in:
parent
e29bfa9ed9
commit
da7bbd2c08
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=181334
@ -389,13 +389,17 @@ _cv_timedwait_sig(struct cv *cvp, struct lock_object *lock, int timo)
|
||||
void
|
||||
cv_signal(struct cv *cvp)
|
||||
{
|
||||
int wakeup_swapper;
|
||||
|
||||
wakeup_swapper = 0;
|
||||
sleepq_lock(cvp);
|
||||
if (cvp->cv_waiters > 0) {
|
||||
cvp->cv_waiters--;
|
||||
sleepq_signal(cvp, SLEEPQ_CONDVAR, 0, 0);
|
||||
wakeup_swapper = sleepq_signal(cvp, SLEEPQ_CONDVAR, 0, 0);
|
||||
}
|
||||
sleepq_release(cvp);
|
||||
if (wakeup_swapper)
|
||||
kick_proc0();
|
||||
}
|
||||
|
||||
/*
|
||||
@ -405,16 +409,21 @@ cv_signal(struct cv *cvp)
|
||||
void
|
||||
cv_broadcastpri(struct cv *cvp, int pri)
|
||||
{
|
||||
int wakeup_swapper;
|
||||
|
||||
/*
|
||||
* XXX sleepq_broadcast pri argument changed from -1 meaning
|
||||
* no pri to 0 meaning no pri.
|
||||
*/
|
||||
wakeup_swapper = 0;
|
||||
if (pri == -1)
|
||||
pri = 0;
|
||||
sleepq_lock(cvp);
|
||||
if (cvp->cv_waiters > 0) {
|
||||
cvp->cv_waiters = 0;
|
||||
sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0);
|
||||
wakeup_swapper = sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0);
|
||||
}
|
||||
sleepq_release(cvp);
|
||||
if (wakeup_swapper)
|
||||
kick_proc0();
|
||||
}
|
||||
|
@ -196,17 +196,18 @@ sleeplk(struct lock *lk, u_int flags, struct lock_object *ilk,
|
||||
return (error);
|
||||
}
|
||||
|
||||
static __inline void
|
||||
static __inline int
|
||||
wakeupshlk(struct lock *lk, const char *file, int line)
|
||||
{
|
||||
uintptr_t v, x;
|
||||
int queue;
|
||||
int queue, wakeup_swapper;
|
||||
|
||||
TD_LOCKS_DEC(curthread);
|
||||
TD_SLOCKS_DEC(curthread);
|
||||
WITNESS_UNLOCK(&lk->lock_object, 0, file, line);
|
||||
LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line);
|
||||
|
||||
wakeup_swapper = 0;
|
||||
for (;;) {
|
||||
x = lk->lk_lock;
|
||||
|
||||
@ -261,12 +262,14 @@ wakeupshlk(struct lock *lk, const char *file, int line)
|
||||
LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
|
||||
__func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
|
||||
"exclusive");
|
||||
sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue);
|
||||
wakeup_swapper = sleepq_broadcast(&lk->lock_object, SLEEPQ_LK,
|
||||
0, queue);
|
||||
sleepq_release(&lk->lock_object);
|
||||
break;
|
||||
}
|
||||
|
||||
lock_profile_release_lock(&lk->lock_object);
|
||||
return (wakeup_swapper);
|
||||
}
|
||||
|
||||
static void
|
||||
@ -335,7 +338,7 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
|
||||
const char *iwmesg;
|
||||
uintptr_t tid, v, x;
|
||||
u_int op;
|
||||
int contested, error, ipri, itimo, queue;
|
||||
int contested, error, ipri, itimo, queue, wakeup_swapper;
|
||||
|
||||
contested = 0;
|
||||
error = 0;
|
||||
@ -367,6 +370,7 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
|
||||
if (op == LK_SHARED && (lk->lock_object.lo_flags & LK_NOSHARE))
|
||||
op = LK_EXCLUSIVE;
|
||||
|
||||
wakeup_swapper = 0;
|
||||
switch (op) {
|
||||
case LK_SHARED:
|
||||
if (LK_CAN_WITNESS(flags))
|
||||
@ -495,7 +499,7 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
|
||||
* We have been unable to succeed in upgrading, so just
|
||||
* give up the shared lock.
|
||||
*/
|
||||
wakeupshlk(lk, file, line);
|
||||
wakeup_swapper += wakeupshlk(lk, file, line);
|
||||
|
||||
/* FALLTHROUGH */
|
||||
case LK_EXCLUSIVE:
|
||||
@ -710,11 +714,12 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
|
||||
__func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
|
||||
"exclusive");
|
||||
atomic_store_rel_ptr(&lk->lk_lock, v);
|
||||
sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue);
|
||||
wakeup_swapper = sleepq_broadcast(&lk->lock_object,
|
||||
SLEEPQ_LK, 0, queue);
|
||||
sleepq_release(&lk->lock_object);
|
||||
break;
|
||||
} else
|
||||
wakeupshlk(lk, file, line);
|
||||
wakeup_swapper = wakeupshlk(lk, file, line);
|
||||
break;
|
||||
case LK_DRAIN:
|
||||
if (LK_CAN_WITNESS(flags))
|
||||
@ -782,8 +787,8 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
|
||||
"%s: %p waking up all threads on the %s queue",
|
||||
__func__, lk, queue == SQ_SHARED_QUEUE ?
|
||||
"shared" : "exclusive");
|
||||
sleepq_broadcast(&lk->lock_object, SLEEPQ_LK,
|
||||
0, queue);
|
||||
wakeup_swapper += sleepq_broadcast(
|
||||
&lk->lock_object, SLEEPQ_LK, 0, queue);
|
||||
|
||||
/*
|
||||
* If shared waiters have been woken up we need
|
||||
@ -850,6 +855,8 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
|
||||
|
||||
if (flags & LK_INTERLOCK)
|
||||
class->lc_unlock(ilk);
|
||||
if (wakeup_swapper)
|
||||
kick_proc0();
|
||||
|
||||
return (error);
|
||||
}
|
||||
|
@ -1957,6 +1957,7 @@ tdsignal(struct proc *p, struct thread *td, int sig, ksiginfo_t *ksi)
|
||||
struct sigacts *ps;
|
||||
int intrval;
|
||||
int ret = 0;
|
||||
int wakeup_swapper;
|
||||
|
||||
PROC_LOCK_ASSERT(p, MA_OWNED);
|
||||
|
||||
@ -2165,12 +2166,15 @@ tdsignal(struct proc *p, struct thread *td, int sig, ksiginfo_t *ksi)
|
||||
* the PROCESS runnable, leave it stopped.
|
||||
* It may run a bit until it hits a thread_suspend_check().
|
||||
*/
|
||||
wakeup_swapper = 0;
|
||||
PROC_SLOCK(p);
|
||||
thread_lock(td);
|
||||
if (TD_ON_SLEEPQ(td) && (td->td_flags & TDF_SINTR))
|
||||
sleepq_abort(td, intrval);
|
||||
wakeup_swapper = sleepq_abort(td, intrval);
|
||||
thread_unlock(td);
|
||||
PROC_SUNLOCK(p);
|
||||
if (wakeup_swapper)
|
||||
kick_proc0();
|
||||
goto out;
|
||||
/*
|
||||
* Mutexes are short lived. Threads waiting on them will
|
||||
@ -2237,7 +2241,9 @@ tdsigwakeup(struct thread *td, int sig, sig_t action, int intrval)
|
||||
{
|
||||
struct proc *p = td->td_proc;
|
||||
register int prop;
|
||||
int wakeup_swapper;
|
||||
|
||||
wakeup_swapper = 0;
|
||||
PROC_LOCK_ASSERT(p, MA_OWNED);
|
||||
prop = sigprop(sig);
|
||||
|
||||
@ -2281,7 +2287,7 @@ tdsigwakeup(struct thread *td, int sig, sig_t action, int intrval)
|
||||
if (td->td_priority > PUSER)
|
||||
sched_prio(td, PUSER);
|
||||
|
||||
sleepq_abort(td, intrval);
|
||||
wakeup_swapper = sleepq_abort(td, intrval);
|
||||
} else {
|
||||
/*
|
||||
* Other states do nothing with the signal immediately,
|
||||
@ -2296,6 +2302,8 @@ tdsigwakeup(struct thread *td, int sig, sig_t action, int intrval)
|
||||
out:
|
||||
PROC_SUNLOCK(p);
|
||||
thread_unlock(td);
|
||||
if (wakeup_swapper)
|
||||
kick_proc0();
|
||||
}
|
||||
|
||||
static void
|
||||
|
@ -361,6 +361,7 @@ void
|
||||
_sx_downgrade(struct sx *sx, const char *file, int line)
|
||||
{
|
||||
uintptr_t x;
|
||||
int wakeup_swapper;
|
||||
|
||||
KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
|
||||
("sx_downgrade() of destroyed sx @ %s:%d", file, line));
|
||||
@ -401,15 +402,19 @@ _sx_downgrade(struct sx *sx, const char *file, int line)
|
||||
* Preserve SX_LOCK_EXCLUSIVE_WAITERS while downgraded to a single
|
||||
* shared lock. If there are any shared waiters, wake them up.
|
||||
*/
|
||||
wakeup_swapper = 0;
|
||||
x = sx->sx_lock;
|
||||
atomic_store_rel_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) |
|
||||
(x & SX_LOCK_EXCLUSIVE_WAITERS));
|
||||
if (x & SX_LOCK_SHARED_WAITERS)
|
||||
sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0,
|
||||
SQ_SHARED_QUEUE);
|
||||
wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX,
|
||||
0, SQ_SHARED_QUEUE);
|
||||
sleepq_release(&sx->lock_object);
|
||||
|
||||
LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line);
|
||||
|
||||
if (wakeup_swapper)
|
||||
kick_proc0();
|
||||
}
|
||||
|
||||
/*
|
||||
@ -589,7 +594,7 @@ void
|
||||
_sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line)
|
||||
{
|
||||
uintptr_t x;
|
||||
int queue;
|
||||
int queue, wakeup_swapper;
|
||||
|
||||
MPASS(!(sx->sx_lock & SX_LOCK_SHARED));
|
||||
|
||||
@ -627,8 +632,11 @@ _sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line)
|
||||
__func__, sx, queue == SQ_SHARED_QUEUE ? "shared" :
|
||||
"exclusive");
|
||||
atomic_store_rel_ptr(&sx->sx_lock, x);
|
||||
sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0, queue);
|
||||
wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0,
|
||||
queue);
|
||||
sleepq_release(&sx->lock_object);
|
||||
if (wakeup_swapper)
|
||||
kick_proc0();
|
||||
}
|
||||
|
||||
/*
|
||||
@ -795,6 +803,7 @@ void
|
||||
_sx_sunlock_hard(struct sx *sx, const char *file, int line)
|
||||
{
|
||||
uintptr_t x;
|
||||
int wakeup_swapper;
|
||||
|
||||
for (;;) {
|
||||
x = sx->sx_lock;
|
||||
@ -862,9 +871,11 @@ _sx_sunlock_hard(struct sx *sx, const char *file, int line)
|
||||
if (LOCK_LOG_TEST(&sx->lock_object, 0))
|
||||
CTR2(KTR_LOCK, "%s: %p waking up all thread on"
|
||||
"exclusive queue", __func__, sx);
|
||||
sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0,
|
||||
SQ_EXCLUSIVE_QUEUE);
|
||||
wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX,
|
||||
0, SQ_EXCLUSIVE_QUEUE);
|
||||
sleepq_release(&sx->lock_object);
|
||||
if (wakeup_swapper)
|
||||
kick_proc0();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
@ -327,10 +327,13 @@ pause(const char *wmesg, int timo)
|
||||
void
|
||||
wakeup(void *ident)
|
||||
{
|
||||
int wakeup_swapper;
|
||||
|
||||
sleepq_lock(ident);
|
||||
sleepq_broadcast(ident, SLEEPQ_SLEEP, 0, 0);
|
||||
wakeup_swapper = sleepq_broadcast(ident, SLEEPQ_SLEEP, 0, 0);
|
||||
sleepq_release(ident);
|
||||
if (wakeup_swapper)
|
||||
kick_proc0();
|
||||
}
|
||||
|
||||
/*
|
||||
@ -341,10 +344,13 @@ wakeup(void *ident)
|
||||
void
|
||||
wakeup_one(void *ident)
|
||||
{
|
||||
int wakeup_swapper;
|
||||
|
||||
sleepq_lock(ident);
|
||||
sleepq_signal(ident, SLEEPQ_SLEEP, 0, 0);
|
||||
wakeup_swapper = sleepq_signal(ident, SLEEPQ_SLEEP, 0, 0);
|
||||
sleepq_release(ident);
|
||||
if (wakeup_swapper)
|
||||
kick_proc0();
|
||||
}
|
||||
|
||||
static void
|
||||
@ -440,11 +446,11 @@ mi_switch(int flags, struct thread *newtd)
|
||||
}
|
||||
|
||||
/*
|
||||
* Change process state to be runnable,
|
||||
* placing it on the run queue if it is in memory,
|
||||
* and awakening the swapper if it isn't in memory.
|
||||
* Change thread state to be runnable, placing it on the run queue if
|
||||
* it is in memory. If it is swapped out, return true so our caller
|
||||
* will know to awaken the swapper.
|
||||
*/
|
||||
void
|
||||
int
|
||||
setrunnable(struct thread *td)
|
||||
{
|
||||
|
||||
@ -454,15 +460,15 @@ setrunnable(struct thread *td)
|
||||
switch (td->td_state) {
|
||||
case TDS_RUNNING:
|
||||
case TDS_RUNQ:
|
||||
return;
|
||||
return (0);
|
||||
case TDS_INHIBITED:
|
||||
/*
|
||||
* If we are only inhibited because we are swapped out
|
||||
* then arange to swap in this process. Otherwise just return.
|
||||
*/
|
||||
if (td->td_inhibitors != TDI_SWAPPED)
|
||||
return;
|
||||
/* XXX: intentional fall-through ? */
|
||||
return (0);
|
||||
/* FALLTHROUGH */
|
||||
case TDS_CAN_RUN:
|
||||
break;
|
||||
default:
|
||||
@ -472,15 +478,11 @@ setrunnable(struct thread *td)
|
||||
if ((td->td_flags & TDF_INMEM) == 0) {
|
||||
if ((td->td_flags & TDF_SWAPINREQ) == 0) {
|
||||
td->td_flags |= TDF_SWAPINREQ;
|
||||
/*
|
||||
* due to a LOR between the thread lock and
|
||||
* the sleepqueue chain locks, use
|
||||
* lower level scheduling functions.
|
||||
*/
|
||||
kick_proc0();
|
||||
return (1);
|
||||
}
|
||||
} else
|
||||
sched_wakeup(td);
|
||||
return (0);
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -508,7 +508,7 @@ thread_single(int mode)
|
||||
struct thread *td;
|
||||
struct thread *td2;
|
||||
struct proc *p;
|
||||
int remaining;
|
||||
int remaining, wakeup_swapper;
|
||||
|
||||
td = curthread;
|
||||
p = td->td_proc;
|
||||
@ -545,6 +545,7 @@ thread_single(int mode)
|
||||
while (remaining != 1) {
|
||||
if (P_SHOULDSTOP(p) != P_STOPPED_SINGLE)
|
||||
goto stopme;
|
||||
wakeup_swapper = 0;
|
||||
FOREACH_THREAD_IN_PROC(p, td2) {
|
||||
if (td2 == td)
|
||||
continue;
|
||||
@ -559,7 +560,8 @@ thread_single(int mode)
|
||||
thread_unsuspend_one(td2);
|
||||
if (TD_ON_SLEEPQ(td2) &&
|
||||
(td2->td_flags & TDF_SINTR))
|
||||
sleepq_abort(td2, EINTR);
|
||||
wakeup_swapper =
|
||||
sleepq_abort(td2, EINTR);
|
||||
break;
|
||||
case SINGLE_BOUNDARY:
|
||||
break;
|
||||
@ -585,6 +587,8 @@ thread_single(int mode)
|
||||
#endif
|
||||
thread_unlock(td2);
|
||||
}
|
||||
if (wakeup_swapper)
|
||||
kick_proc0();
|
||||
if (mode == SINGLE_EXIT)
|
||||
remaining = p->p_numthreads;
|
||||
else if (mode == SINGLE_BOUNDARY)
|
||||
@ -787,7 +791,11 @@ thread_unsuspend_one(struct thread *td)
|
||||
KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended"));
|
||||
TD_CLR_SUSPENDED(td);
|
||||
p->p_suspcount--;
|
||||
setrunnable(td);
|
||||
if (setrunnable(td)) {
|
||||
#ifdef INVARIANTS
|
||||
panic("not waking up swapper");
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -160,7 +160,7 @@ static int sleepq_check_timeout(void);
|
||||
static void sleepq_dtor(void *mem, int size, void *arg);
|
||||
#endif
|
||||
static int sleepq_init(void *mem, int size, int flags);
|
||||
static void sleepq_resume_thread(struct sleepqueue *sq, struct thread *td,
|
||||
static int sleepq_resume_thread(struct sleepqueue *sq, struct thread *td,
|
||||
int pri);
|
||||
static void sleepq_switch(void *wchan, int pri);
|
||||
static void sleepq_timeout(void *arg);
|
||||
@ -434,7 +434,15 @@ sleepq_catch_signals(void *wchan, int pri)
|
||||
*/
|
||||
if (TD_ON_SLEEPQ(td)) {
|
||||
sq = sleepq_lookup(wchan);
|
||||
sleepq_resume_thread(sq, td, 0);
|
||||
if (sleepq_resume_thread(sq, td, 0)) {
|
||||
#ifdef INVARIANTS
|
||||
/*
|
||||
* This thread hasn't gone to sleep yet, so it
|
||||
* should not be swapped out.
|
||||
*/
|
||||
panic("not waking up swapper");
|
||||
#endif
|
||||
}
|
||||
}
|
||||
mtx_unlock_spin(&sc->sc_lock);
|
||||
MPASS(td->td_lock != &sc->sc_lock);
|
||||
@ -474,7 +482,15 @@ sleepq_switch(void *wchan, int pri)
|
||||
if (td->td_flags & TDF_TIMEOUT) {
|
||||
MPASS(TD_ON_SLEEPQ(td));
|
||||
sq = sleepq_lookup(wchan);
|
||||
sleepq_resume_thread(sq, td, 0);
|
||||
if (sleepq_resume_thread(sq, td, 0)) {
|
||||
#ifdef INVARIANTS
|
||||
/*
|
||||
* This thread hasn't gone to sleep yet, so it
|
||||
* should not be swapped out.
|
||||
*/
|
||||
panic("not waking up swapper");
|
||||
#endif
|
||||
}
|
||||
mtx_unlock_spin(&sc->sc_lock);
|
||||
return;
|
||||
}
|
||||
@ -631,7 +647,7 @@ sleepq_timedwait_sig(void *wchan, int pri)
|
||||
* Removes a thread from a sleep queue and makes it
|
||||
* runnable.
|
||||
*/
|
||||
static void
|
||||
static int
|
||||
sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, int pri)
|
||||
{
|
||||
struct sleepqueue_chain *sc;
|
||||
@ -683,7 +699,7 @@ sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, int pri)
|
||||
MPASS(pri == 0 || (pri >= PRI_MIN && pri <= PRI_MAX));
|
||||
if (pri != 0 && td->td_priority > pri)
|
||||
sched_prio(td, pri);
|
||||
setrunnable(td);
|
||||
return (setrunnable(td));
|
||||
}
|
||||
|
||||
#ifdef INVARIANTS
|
||||
@ -722,18 +738,19 @@ sleepq_init(void *mem, int size, int flags)
|
||||
/*
|
||||
* Find the highest priority thread sleeping on a wait channel and resume it.
|
||||
*/
|
||||
void
|
||||
int
|
||||
sleepq_signal(void *wchan, int flags, int pri, int queue)
|
||||
{
|
||||
struct sleepqueue *sq;
|
||||
struct thread *td, *besttd;
|
||||
int wakeup_swapper;
|
||||
|
||||
CTR2(KTR_PROC, "sleepq_signal(%p, %d)", wchan, flags);
|
||||
KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
|
||||
MPASS((queue >= 0) && (queue < NR_SLEEPQS));
|
||||
sq = sleepq_lookup(wchan);
|
||||
if (sq == NULL)
|
||||
return;
|
||||
return (0);
|
||||
KASSERT(sq->sq_type == (flags & SLEEPQ_TYPE),
|
||||
("%s: mismatch between sleep/wakeup and cv_*", __func__));
|
||||
|
||||
@ -750,35 +767,40 @@ sleepq_signal(void *wchan, int flags, int pri, int queue)
|
||||
}
|
||||
MPASS(besttd != NULL);
|
||||
thread_lock(besttd);
|
||||
sleepq_resume_thread(sq, besttd, pri);
|
||||
wakeup_swapper = sleepq_resume_thread(sq, besttd, pri);
|
||||
thread_unlock(besttd);
|
||||
return (wakeup_swapper);
|
||||
}
|
||||
|
||||
/*
|
||||
* Resume all threads sleeping on a specified wait channel.
|
||||
*/
|
||||
void
|
||||
int
|
||||
sleepq_broadcast(void *wchan, int flags, int pri, int queue)
|
||||
{
|
||||
struct sleepqueue *sq;
|
||||
struct thread *td;
|
||||
int wakeup_swapper;
|
||||
|
||||
CTR2(KTR_PROC, "sleepq_broadcast(%p, %d)", wchan, flags);
|
||||
KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
|
||||
MPASS((queue >= 0) && (queue < NR_SLEEPQS));
|
||||
sq = sleepq_lookup(wchan);
|
||||
if (sq == NULL)
|
||||
return;
|
||||
return (0);
|
||||
KASSERT(sq->sq_type == (flags & SLEEPQ_TYPE),
|
||||
("%s: mismatch between sleep/wakeup and cv_*", __func__));
|
||||
|
||||
/* Resume all blocked threads on the sleep queue. */
|
||||
wakeup_swapper = 0;
|
||||
while (!TAILQ_EMPTY(&sq->sq_blocked[queue])) {
|
||||
td = TAILQ_FIRST(&sq->sq_blocked[queue]);
|
||||
thread_lock(td);
|
||||
sleepq_resume_thread(sq, td, pri);
|
||||
if (sleepq_resume_thread(sq, td, pri))
|
||||
wakeup_swapper = 1;
|
||||
thread_unlock(td);
|
||||
}
|
||||
return (wakeup_swapper);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -792,8 +814,10 @@ sleepq_timeout(void *arg)
|
||||
struct sleepqueue *sq;
|
||||
struct thread *td;
|
||||
void *wchan;
|
||||
int wakeup_swapper;
|
||||
|
||||
td = arg;
|
||||
wakeup_swapper = 0;
|
||||
CTR3(KTR_PROC, "sleepq_timeout: thread %p (pid %ld, %s)",
|
||||
(void *)td, (long)td->td_proc->p_pid, (void *)td->td_name);
|
||||
|
||||
@ -809,8 +833,10 @@ sleepq_timeout(void *arg)
|
||||
sq = sleepq_lookup(wchan);
|
||||
MPASS(sq != NULL);
|
||||
td->td_flags |= TDF_TIMEOUT;
|
||||
sleepq_resume_thread(sq, td, 0);
|
||||
wakeup_swapper = sleepq_resume_thread(sq, td, 0);
|
||||
thread_unlock(td);
|
||||
if (wakeup_swapper)
|
||||
kick_proc0();
|
||||
return;
|
||||
}
|
||||
|
||||
@ -839,10 +865,12 @@ sleepq_timeout(void *arg)
|
||||
MPASS(TD_IS_SLEEPING(td));
|
||||
td->td_flags &= ~TDF_TIMEOUT;
|
||||
TD_CLR_SLEEPING(td);
|
||||
setrunnable(td);
|
||||
wakeup_swapper = setrunnable(td);
|
||||
} else
|
||||
td->td_flags |= TDF_TIMOFAIL;
|
||||
thread_unlock(td);
|
||||
if (wakeup_swapper)
|
||||
kick_proc0();
|
||||
}
|
||||
|
||||
/*
|
||||
@ -853,6 +881,7 @@ void
|
||||
sleepq_remove(struct thread *td, void *wchan)
|
||||
{
|
||||
struct sleepqueue *sq;
|
||||
int wakeup_swapper;
|
||||
|
||||
/*
|
||||
* Look up the sleep queue for this wait channel, then re-check
|
||||
@ -876,16 +905,18 @@ sleepq_remove(struct thread *td, void *wchan)
|
||||
thread_lock(td);
|
||||
MPASS(sq != NULL);
|
||||
MPASS(td->td_wchan == wchan);
|
||||
sleepq_resume_thread(sq, td, 0);
|
||||
wakeup_swapper = sleepq_resume_thread(sq, td, 0);
|
||||
thread_unlock(td);
|
||||
sleepq_release(wchan);
|
||||
if (wakeup_swapper)
|
||||
kick_proc0();
|
||||
}
|
||||
|
||||
/*
|
||||
* Abort a thread as if an interrupt had occurred. Only abort
|
||||
* interruptible waits (unfortunately it isn't safe to abort others).
|
||||
*/
|
||||
void
|
||||
int
|
||||
sleepq_abort(struct thread *td, int intrval)
|
||||
{
|
||||
struct sleepqueue *sq;
|
||||
@ -901,7 +932,7 @@ sleepq_abort(struct thread *td, int intrval)
|
||||
* timeout is scheduled anyhow.
|
||||
*/
|
||||
if (td->td_flags & TDF_TIMEOUT)
|
||||
return;
|
||||
return (0);
|
||||
|
||||
CTR3(KTR_PROC, "sleepq_abort: thread %p (pid %ld, %s)",
|
||||
(void *)td, (long)td->td_proc->p_pid, (void *)td->td_name);
|
||||
@ -913,14 +944,14 @@ sleepq_abort(struct thread *td, int intrval)
|
||||
* we have to do it here.
|
||||
*/
|
||||
if (!TD_IS_SLEEPING(td))
|
||||
return;
|
||||
return (0);
|
||||
wchan = td->td_wchan;
|
||||
MPASS(wchan != NULL);
|
||||
sq = sleepq_lookup(wchan);
|
||||
MPASS(sq != NULL);
|
||||
|
||||
/* Thread is asleep on sleep queue sq, so wake it up. */
|
||||
sleepq_resume_thread(sq, td, 0);
|
||||
return (sleepq_resume_thread(sq, td, 0));
|
||||
}
|
||||
|
||||
#ifdef SLEEPQUEUE_PROFILING
|
||||
|
@ -808,7 +808,7 @@ void pstats_free(struct pstats *ps);
|
||||
int securelevel_ge(struct ucred *cr, int level);
|
||||
int securelevel_gt(struct ucred *cr, int level);
|
||||
void sessrele(struct session *);
|
||||
void setrunnable(struct thread *);
|
||||
int setrunnable(struct thread *);
|
||||
void setsugid(struct proc *p);
|
||||
int sigonstack(size_t sp);
|
||||
void sleepinit(void);
|
||||
|
@ -91,17 +91,17 @@ struct thread;
|
||||
#define SLEEPQ_INTERRUPTIBLE 0x100 /* Sleep is interruptible. */
|
||||
|
||||
void init_sleepqueues(void);
|
||||
void sleepq_abort(struct thread *td, int intrval);
|
||||
int sleepq_abort(struct thread *td, int intrval);
|
||||
void sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg,
|
||||
int flags, int queue);
|
||||
struct sleepqueue *sleepq_alloc(void);
|
||||
void sleepq_broadcast(void *wchan, int flags, int pri, int queue);
|
||||
int sleepq_broadcast(void *wchan, int flags, int pri, int queue);
|
||||
void sleepq_free(struct sleepqueue *sq);
|
||||
void sleepq_lock(void *wchan);
|
||||
struct sleepqueue *sleepq_lookup(void *wchan);
|
||||
void sleepq_release(void *wchan);
|
||||
void sleepq_remove(struct thread *td, void *wchan);
|
||||
void sleepq_signal(void *wchan, int flags, int pri, int queue);
|
||||
int sleepq_signal(void *wchan, int flags, int pri, int queue);
|
||||
void sleepq_set_timeout(void *wchan, int timo);
|
||||
int sleepq_timedwait(void *wchan, int pri);
|
||||
int sleepq_timedwait_sig(void *wchan, int pri);
|
||||
|
@ -116,10 +116,6 @@ static int swapout(struct proc *);
|
||||
static void swapclear(struct proc *);
|
||||
#endif
|
||||
|
||||
|
||||
static volatile int proc0_rescan;
|
||||
|
||||
|
||||
/*
|
||||
* MPSAFE
|
||||
*
|
||||
@ -683,9 +679,6 @@ scheduler(dummy)
|
||||
loop:
|
||||
if (vm_page_count_min()) {
|
||||
VM_WAIT;
|
||||
thread_lock(&thread0);
|
||||
proc0_rescan = 0;
|
||||
thread_unlock(&thread0);
|
||||
goto loop;
|
||||
}
|
||||
|
||||
@ -732,13 +725,7 @@ scheduler(dummy)
|
||||
* Nothing to do, back to sleep.
|
||||
*/
|
||||
if ((p = pp) == NULL) {
|
||||
thread_lock(&thread0);
|
||||
if (!proc0_rescan) {
|
||||
TD_SET_IWAIT(&thread0);
|
||||
mi_switch(SW_VOL | SWT_IWAIT, NULL);
|
||||
}
|
||||
proc0_rescan = 0;
|
||||
thread_unlock(&thread0);
|
||||
tsleep(&proc0, PVM, "sched", maxslp * hz / 2);
|
||||
goto loop;
|
||||
}
|
||||
PROC_LOCK(p);
|
||||
@ -750,9 +737,6 @@ scheduler(dummy)
|
||||
*/
|
||||
if (p->p_flag & (P_INMEM | P_SWAPPINGOUT | P_SWAPPINGIN)) {
|
||||
PROC_UNLOCK(p);
|
||||
thread_lock(&thread0);
|
||||
proc0_rescan = 0;
|
||||
thread_unlock(&thread0);
|
||||
goto loop;
|
||||
}
|
||||
|
||||
@ -762,32 +746,16 @@ scheduler(dummy)
|
||||
*/
|
||||
faultin(p);
|
||||
PROC_UNLOCK(p);
|
||||
thread_lock(&thread0);
|
||||
proc0_rescan = 0;
|
||||
thread_unlock(&thread0);
|
||||
goto loop;
|
||||
}
|
||||
|
||||
void kick_proc0(void)
|
||||
void
|
||||
kick_proc0(void)
|
||||
{
|
||||
struct thread *td = &thread0;
|
||||
|
||||
/* XXX This will probably cause a LOR in some cases */
|
||||
thread_lock(td);
|
||||
if (TD_AWAITING_INTR(td)) {
|
||||
CTR2(KTR_INTR, "%s: sched_add %d", __func__, 0);
|
||||
TD_CLR_IWAIT(td);
|
||||
sched_add(td, SRQ_INTR);
|
||||
} else {
|
||||
proc0_rescan = 1;
|
||||
CTR2(KTR_INTR, "%s: state %d",
|
||||
__func__, td->td_state);
|
||||
}
|
||||
thread_unlock(td);
|
||||
|
||||
wakeup(&proc0);
|
||||
}
|
||||
|
||||
|
||||
#ifndef NO_SWAPPING
|
||||
|
||||
/*
|
||||
@ -980,7 +948,16 @@ swapclear(p)
|
||||
td->td_flags &= ~TDF_SWAPINREQ;
|
||||
TD_CLR_SWAPPED(td);
|
||||
if (TD_CAN_RUN(td))
|
||||
setrunnable(td);
|
||||
if (setrunnable(td)) {
|
||||
#ifdef INVARIANTS
|
||||
/*
|
||||
* XXX: We just cleared TDI_SWAPPED
|
||||
* above and set TDF_INMEM, so this
|
||||
* should never happen.
|
||||
*/
|
||||
panic("not waking up swapper");
|
||||
#endif
|
||||
}
|
||||
thread_unlock(td);
|
||||
}
|
||||
p->p_flag &= ~(P_SWAPPINGIN|P_SWAPPINGOUT);
|
||||
|
Loading…
Reference in New Issue
Block a user