Add second sleep queue so that sx and lockmgr can have separate sleep

queues for shared and exclusive acquisitions

Submitted by: Attilio Rao
Approved by: jhb
This commit is contained in:
Kip Macy 2006-12-16 06:54:09 +00:00
parent 8c3bc2c180
commit 6cbb70e2cc
Notes: svn2git 2020-12-20 02:59:44 +00:00
svn path=/head/; revision=165272
5 changed files with 71 additions and 45 deletions

View File

@ -149,7 +149,8 @@ cv_wait_unlock(struct cv *cvp, struct mtx *mp)
DROP_GIANT();
mtx_unlock(mp);
sleepq_add(cvp, &mp->mtx_object, cvp->cv_description, SLEEPQ_CONDVAR);
sleepq_add(cvp, &mp->mtx_object, cvp->cv_description, SLEEPQ_CONDVAR,
0);
sleepq_wait(cvp);
PICKUP_GIANT();
@ -197,7 +198,7 @@ cv_wait_sig(struct cv *cvp, struct mtx *mp)
mtx_unlock(mp);
sleepq_add(cvp, &mp->mtx_object, cvp->cv_description, SLEEPQ_CONDVAR |
SLEEPQ_INTERRUPTIBLE);
SLEEPQ_INTERRUPTIBLE, 0);
rval = sleepq_wait_sig(cvp);
#ifdef KTRACE
@ -250,7 +251,8 @@ cv_timedwait(struct cv *cvp, struct mtx *mp, int timo)
DROP_GIANT();
mtx_unlock(mp);
sleepq_add(cvp, &mp->mtx_object, cvp->cv_description, SLEEPQ_CONDVAR);
sleepq_add(cvp, &mp->mtx_object, cvp->cv_description, SLEEPQ_CONDVAR,
0);
sleepq_set_timeout(cvp, timo);
rval = sleepq_timedwait(cvp);
@ -308,7 +310,7 @@ cv_timedwait_sig(struct cv *cvp, struct mtx *mp, int timo)
mtx_unlock(mp);
sleepq_add(cvp, &mp->mtx_object, cvp->cv_description, SLEEPQ_CONDVAR |
SLEEPQ_INTERRUPTIBLE);
SLEEPQ_INTERRUPTIBLE, 0);
sleepq_set_timeout(cvp, timo);
rval = sleepq_timedwait_sig(cvp);
@ -337,7 +339,7 @@ cv_signal(struct cv *cvp)
sleepq_lock(cvp);
if (cvp->cv_waiters > 0) {
cvp->cv_waiters--;
sleepq_signal(cvp, SLEEPQ_CONDVAR, -1);
sleepq_signal(cvp, SLEEPQ_CONDVAR, -1, 0);
} else
sleepq_release(cvp);
}
@ -353,7 +355,7 @@ cv_broadcastpri(struct cv *cvp, int pri)
sleepq_lock(cvp);
if (cvp->cv_waiters > 0) {
cvp->cv_waiters = 0;
sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri);
sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0);
} else
sleepq_release(cvp);
}

View File

@ -189,7 +189,7 @@ msleep(ident, mtx, priority, wmesg, timo)
* return from cursig().
*/
sleepq_add(ident, ident == &lbolt ? NULL : &mtx->mtx_object, wmesg,
flags);
flags, 0);
if (timo)
sleepq_set_timeout(ident, timo);
@ -267,7 +267,7 @@ msleep_spin(ident, mtx, wmesg, timo)
/*
* We put ourselves on the sleep queue and start our timeout.
*/
sleepq_add(ident, &mtx->mtx_object, wmesg, SLEEPQ_MSLEEP);
sleepq_add(ident, &mtx->mtx_object, wmesg, SLEEPQ_MSLEEP, 0);
if (timo)
sleepq_set_timeout(ident, timo);
@ -316,7 +316,7 @@ wakeup(ident)
{
sleepq_lock(ident);
sleepq_broadcast(ident, SLEEPQ_MSLEEP, -1);
sleepq_broadcast(ident, SLEEPQ_MSLEEP, -1, 0);
}
/*
@ -330,7 +330,7 @@ wakeup_one(ident)
{
sleepq_lock(ident);
sleepq_signal(ident, SLEEPQ_MSLEEP, -1);
sleepq_signal(ident, SLEEPQ_MSLEEP, -1, 0);
}
/*

View File

@ -94,7 +94,7 @@ __FBSDID("$FreeBSD$");
#define SC_SHIFT 8
#define SC_HASH(wc) (((uintptr_t)(wc) >> SC_SHIFT) & SC_MASK)
#define SC_LOOKUP(wc) &sleepq_chains[SC_HASH(wc)]
#define NR_SLEEPQS 2
/*
* There two different lists of sleep queues. Both lists are connected
* via the sq_hash entries. The first list is the sleep queue chain list
@ -114,7 +114,7 @@ __FBSDID("$FreeBSD$");
* c - sleep queue chain lock
*/
struct sleepqueue {
TAILQ_HEAD(, thread) sq_blocked; /* (c) Blocked threads. */
TAILQ_HEAD(, thread) sq_blocked[NR_SLEEPQS]; /* (c) Blocked threads. */
LIST_ENTRY(sleepqueue) sq_hash; /* (c) Chain and free list. */
LIST_HEAD(, sleepqueue) sq_free; /* (c) Free queues. */
void *sq_wchan; /* (c) Wait channel. */
@ -148,6 +148,8 @@ static MALLOC_DEFINE(M_SLEEPQUEUE, "sleepqueue", "sleep queues");
/*
* Prototypes for non-exported routines.
*/
static int sleepq_catch_signals(void *wchan);
static int sleepq_check_signals(void);
static int sleepq_check_timeout(void);
static void sleepq_switch(void *wchan);
static void sleepq_timeout(void *arg);
@ -192,9 +194,11 @@ struct sleepqueue *
sleepq_alloc(void)
{
struct sleepqueue *sq;
int i;
sq = malloc(sizeof(struct sleepqueue), M_SLEEPQUEUE, M_WAITOK | M_ZERO);
TAILQ_INIT(&sq->sq_blocked);
for (i = 0; i < NR_SLEEPQS; i++)
TAILQ_INIT(&sq->sq_blocked[i]);
LIST_INIT(&sq->sq_free);
return (sq);
}
@ -205,9 +209,11 @@ sleepq_alloc(void)
void
sleepq_free(struct sleepqueue *sq)
{
int i;
MPASS(sq != NULL);
MPASS(TAILQ_EMPTY(&sq->sq_blocked));
for (i = 0; i < NR_SLEEPQS; i++)
MPASS(TAILQ_EMPTY(&sq->sq_blocked[i]));
free(sq, M_SLEEPQUEUE);
}
@ -262,7 +268,8 @@ sleepq_release(void *wchan)
* woken up.
*/
void
sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg, int flags)
sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg, int flags,
int queue)
{
struct sleepqueue_chain *sc;
struct sleepqueue *sq;
@ -273,6 +280,7 @@ sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg, int flags)
mtx_assert(&sc->sc_lock, MA_OWNED);
MPASS(td->td_sleepqueue != NULL);
MPASS(wchan != NULL);
MPASS((queue >= 0) && (queue < NR_SLEEPQS));
/* If this thread is not allowed to sleep, die a horrible death. */
KASSERT(!(td->td_pflags & TDP_NOSLEEPING),
@ -287,6 +295,16 @@ sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg, int flags)
* into the sleep queue already in use by this wait channel.
*/
if (sq == NULL) {
#ifdef INVARIANTS
int i;
for (i = 0; i < NR_SLEEP_QUEUEUS; i++)
KASSERT(TAILQ_EMPTY(&sq->sq_blocked[i]),
("thread's sleep queue %d is not empty", i));
KASSERT(LIST_EMPTY(&sq->sq_free),
("thread's sleep queue has a non-empty free list"));
KASSERT(sq->sq_wchan == NULL, ("stale sq_wchan pointer"));
#endif
#ifdef SLEEPQUEUE_PROFILING
sc->sc_depth++;
if (sc->sc_depth > sc->sc_max_depth) {
@ -297,11 +315,6 @@ sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg, int flags)
#endif
sq = td->td_sleepqueue;
LIST_INSERT_HEAD(&sc->sc_queues, sq, sq_hash);
KASSERT(TAILQ_EMPTY(&sq->sq_blocked),
("thread's sleep queue has a non-empty queue"));
KASSERT(LIST_EMPTY(&sq->sq_free),
("thread's sleep queue has a non-empty free list"));
KASSERT(sq->sq_wchan == NULL, ("stale sq_wchan pointer"));
sq->sq_wchan = wchan;
#ifdef INVARIANTS
sq->sq_lock = lock;
@ -313,9 +326,10 @@ sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg, int flags)
MPASS((flags & SLEEPQ_TYPE) == sq->sq_type);
LIST_INSERT_HEAD(&sq->sq_free, td->td_sleepqueue, sq_hash);
}
TAILQ_INSERT_TAIL(&sq->sq_blocked, td, td_slpq);
TAILQ_INSERT_TAIL(&sq->sq_blocked[queue], td, td_slpq);
td->td_sleepqueue = NULL;
mtx_lock_spin(&sched_lock);
td->td_sqqueue = queue;
td->td_wchan = wchan;
td->td_wmesg = wmesg;
if (flags & SLEEPQ_INTERRUPTIBLE) {
@ -606,12 +620,13 @@ sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, int pri)
MPASS(td != NULL);
MPASS(sq->sq_wchan != NULL);
MPASS(td->td_wchan == sq->sq_wchan);
MPASS(td->td_sqqueue < NR_SLEEPQS && td->td_sqqueue >= 0);
sc = SC_LOOKUP(sq->sq_wchan);
mtx_assert(&sc->sc_lock, MA_OWNED);
mtx_assert(&sched_lock, MA_OWNED);
/* Remove the thread from the queue. */
TAILQ_REMOVE(&sq->sq_blocked, td, td_slpq);
TAILQ_REMOVE(&sq->sq_blocked[td->td_sqqueue], td, td_slpq);
/*
* Get a sleep queue for this thread. If this is the last waiter,
@ -656,13 +671,14 @@ sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, int pri)
* Find the highest priority thread sleeping on a wait channel and resume it.
*/
void
sleepq_signal(void *wchan, int flags, int pri)
sleepq_signal(void *wchan, int flags, int pri, int queue)
{
struct sleepqueue *sq;
struct thread *td, *besttd;
CTR2(KTR_PROC, "sleepq_signal(%p, %d)", wchan, flags);
KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
MPASS((queue >= 0) && (queue < NR_SLEEPQS));
sq = sleepq_lookup(wchan);
if (sq == NULL) {
sleepq_release(wchan);
@ -678,7 +694,7 @@ sleepq_signal(void *wchan, int flags, int pri)
* the tail of sleep queues.
*/
besttd = NULL;
TAILQ_FOREACH(td, &sq->sq_blocked, td_slpq) {
TAILQ_FOREACH(td, &sq->sq_blocked[queue], td_slpq) {
if (besttd == NULL || td->td_priority < besttd->td_priority)
besttd = td;
}
@ -693,12 +709,13 @@ sleepq_signal(void *wchan, int flags, int pri)
* Resume all threads sleeping on a specified wait channel.
*/
void
sleepq_broadcast(void *wchan, int flags, int pri)
sleepq_broadcast(void *wchan, int flags, int pri, int queue)
{
struct sleepqueue *sq;
CTR2(KTR_PROC, "sleepq_broadcast(%p, %d)", wchan, flags);
KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
MPASS((queue >= 0) && (queue < NR_SLEEPQS));
sq = sleepq_lookup(wchan);
if (sq == NULL) {
sleepq_release(wchan);
@ -709,8 +726,9 @@ sleepq_broadcast(void *wchan, int flags, int pri)
/* Resume all blocked threads on the sleep queue. */
mtx_lock_spin(&sched_lock);
while (!TAILQ_EMPTY(&sq->sq_blocked))
sleepq_resume_thread(sq, TAILQ_FIRST(&sq->sq_blocked), pri);
while (!TAILQ_EMPTY(&sq->sq_blocked[queue]))
sleepq_resume_thread(sq, TAILQ_FIRST(&sq->sq_blocked[queue]),
pri);
mtx_unlock_spin(&sched_lock);
sleepq_release(wchan);
}
@ -901,15 +919,19 @@ DB_SHOW_COMMAND(sleepq, db_show_sleepqueue)
}
#endif
db_printf("Blocked threads:\n");
if (TAILQ_EMPTY(&sq->sq_blocked))
db_printf("\tempty\n");
else
TAILQ_FOREACH(td, &sq->sq_blocked, td_slpq) {
db_printf("\t%p (tid %d, pid %d, \"%s\")\n", td,
td->td_tid, td->td_proc->p_pid,
td->td_name[0] != '\0' ? td->td_name :
td->td_proc->p_comm);
}
for (i = 0; i < NR_SLEEPQS; i++) {
db_printf("\nQueue[%d]:\n", i);
if (TAILQ_EMPTY(&sq->sq_blocked[i]))
db_printf("\tempty\n");
else
TAILQ_FOREACH(td, &sq->sq_blocked[0],
td_slpq) {
db_printf("\t%p (tid %d, pid %d, \"%s\")\n", td,
td->td_tid, td->td_proc->p_pid,
td->td_name[i] != '\0' ? td->td_name :
td->td_proc->p_comm);
}
}
}
/* Alias 'show sleepqueue' to 'show sleepq'. */

View File

@ -235,6 +235,7 @@ struct thread {
int td_inhibitors; /* (j) Why can not run. */
int td_pflags; /* (k) Private thread (TDP_*) flags. */
int td_dupfd; /* (k) Ret value from fdopen. XXX */
int td_sqqueue; /* (j) Sleepqueue queue blocked on. */
void *td_wchan; /* (j) Sleep address. */
const char *td_wmesg; /* (j) Reason for sleep. */
u_char td_lastcpu; /* (j) Last cpu we were on. */

View File

@ -88,19 +88,20 @@ struct thread;
void init_sleepqueues(void);
void sleepq_abort(struct thread *td, int intrval);
void sleepq_add(void *, struct lock_object *, const char *, int);
void sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg,
int flags, int queue);
struct sleepqueue *sleepq_alloc(void);
void sleepq_broadcast(void *, int, int);
void sleepq_free(struct sleepqueue *);
void sleepq_lock(void *);
struct sleepqueue *sleepq_lookup(void *);
void sleepq_release(void *);
void sleepq_remove(struct thread *, void *);
void sleepq_signal(void *, int, int);
void sleepq_broadcast(void *wchan, int flags, int pri, int queue);
void sleepq_free(struct sleepqueue *sq);
void sleepq_lock(void *wchan);
struct sleepqueue *sleepq_lookup(void *wchan);
void sleepq_release(void *wchan);
void sleepq_remove(struct thread *td, void *wchan);
void sleepq_signal(void *wchan, int flags, int pri, int queue);
void sleepq_set_timeout(void *wchan, int timo);
int sleepq_timedwait(void *wchan);
int sleepq_timedwait_sig(void *wchan);
void sleepq_wait(void *);
void sleepq_wait(void *wchan);
int sleepq_wait_sig(void *wchan);
#endif /* _KERNEL */