Add second sleep queue so that sx and lockmgr can have separate sleep
queues for shared and exclusive acquisitions Submitted by: Attilio Rao Approved by: jhb
This commit is contained in:
parent
8c3bc2c180
commit
6cbb70e2cc
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=165272
@ -149,7 +149,8 @@ cv_wait_unlock(struct cv *cvp, struct mtx *mp)
|
||||
DROP_GIANT();
|
||||
mtx_unlock(mp);
|
||||
|
||||
sleepq_add(cvp, &mp->mtx_object, cvp->cv_description, SLEEPQ_CONDVAR);
|
||||
sleepq_add(cvp, &mp->mtx_object, cvp->cv_description, SLEEPQ_CONDVAR,
|
||||
0);
|
||||
sleepq_wait(cvp);
|
||||
|
||||
PICKUP_GIANT();
|
||||
@ -197,7 +198,7 @@ cv_wait_sig(struct cv *cvp, struct mtx *mp)
|
||||
mtx_unlock(mp);
|
||||
|
||||
sleepq_add(cvp, &mp->mtx_object, cvp->cv_description, SLEEPQ_CONDVAR |
|
||||
SLEEPQ_INTERRUPTIBLE);
|
||||
SLEEPQ_INTERRUPTIBLE, 0);
|
||||
rval = sleepq_wait_sig(cvp);
|
||||
|
||||
#ifdef KTRACE
|
||||
@ -250,7 +251,8 @@ cv_timedwait(struct cv *cvp, struct mtx *mp, int timo)
|
||||
DROP_GIANT();
|
||||
mtx_unlock(mp);
|
||||
|
||||
sleepq_add(cvp, &mp->mtx_object, cvp->cv_description, SLEEPQ_CONDVAR);
|
||||
sleepq_add(cvp, &mp->mtx_object, cvp->cv_description, SLEEPQ_CONDVAR,
|
||||
0);
|
||||
sleepq_set_timeout(cvp, timo);
|
||||
rval = sleepq_timedwait(cvp);
|
||||
|
||||
@ -308,7 +310,7 @@ cv_timedwait_sig(struct cv *cvp, struct mtx *mp, int timo)
|
||||
mtx_unlock(mp);
|
||||
|
||||
sleepq_add(cvp, &mp->mtx_object, cvp->cv_description, SLEEPQ_CONDVAR |
|
||||
SLEEPQ_INTERRUPTIBLE);
|
||||
SLEEPQ_INTERRUPTIBLE, 0);
|
||||
sleepq_set_timeout(cvp, timo);
|
||||
rval = sleepq_timedwait_sig(cvp);
|
||||
|
||||
@ -337,7 +339,7 @@ cv_signal(struct cv *cvp)
|
||||
sleepq_lock(cvp);
|
||||
if (cvp->cv_waiters > 0) {
|
||||
cvp->cv_waiters--;
|
||||
sleepq_signal(cvp, SLEEPQ_CONDVAR, -1);
|
||||
sleepq_signal(cvp, SLEEPQ_CONDVAR, -1, 0);
|
||||
} else
|
||||
sleepq_release(cvp);
|
||||
}
|
||||
@ -353,7 +355,7 @@ cv_broadcastpri(struct cv *cvp, int pri)
|
||||
sleepq_lock(cvp);
|
||||
if (cvp->cv_waiters > 0) {
|
||||
cvp->cv_waiters = 0;
|
||||
sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri);
|
||||
sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0);
|
||||
} else
|
||||
sleepq_release(cvp);
|
||||
}
|
||||
|
@ -189,7 +189,7 @@ msleep(ident, mtx, priority, wmesg, timo)
|
||||
* return from cursig().
|
||||
*/
|
||||
sleepq_add(ident, ident == &lbolt ? NULL : &mtx->mtx_object, wmesg,
|
||||
flags);
|
||||
flags, 0);
|
||||
if (timo)
|
||||
sleepq_set_timeout(ident, timo);
|
||||
|
||||
@ -267,7 +267,7 @@ msleep_spin(ident, mtx, wmesg, timo)
|
||||
/*
|
||||
* We put ourselves on the sleep queue and start our timeout.
|
||||
*/
|
||||
sleepq_add(ident, &mtx->mtx_object, wmesg, SLEEPQ_MSLEEP);
|
||||
sleepq_add(ident, &mtx->mtx_object, wmesg, SLEEPQ_MSLEEP, 0);
|
||||
if (timo)
|
||||
sleepq_set_timeout(ident, timo);
|
||||
|
||||
@ -316,7 +316,7 @@ wakeup(ident)
|
||||
{
|
||||
|
||||
sleepq_lock(ident);
|
||||
sleepq_broadcast(ident, SLEEPQ_MSLEEP, -1);
|
||||
sleepq_broadcast(ident, SLEEPQ_MSLEEP, -1, 0);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -330,7 +330,7 @@ wakeup_one(ident)
|
||||
{
|
||||
|
||||
sleepq_lock(ident);
|
||||
sleepq_signal(ident, SLEEPQ_MSLEEP, -1);
|
||||
sleepq_signal(ident, SLEEPQ_MSLEEP, -1, 0);
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -94,7 +94,7 @@ __FBSDID("$FreeBSD$");
|
||||
#define SC_SHIFT 8
|
||||
#define SC_HASH(wc) (((uintptr_t)(wc) >> SC_SHIFT) & SC_MASK)
|
||||
#define SC_LOOKUP(wc) &sleepq_chains[SC_HASH(wc)]
|
||||
|
||||
#define NR_SLEEPQS 2
|
||||
/*
|
||||
* There two different lists of sleep queues. Both lists are connected
|
||||
* via the sq_hash entries. The first list is the sleep queue chain list
|
||||
@ -114,7 +114,7 @@ __FBSDID("$FreeBSD$");
|
||||
* c - sleep queue chain lock
|
||||
*/
|
||||
struct sleepqueue {
|
||||
TAILQ_HEAD(, thread) sq_blocked; /* (c) Blocked threads. */
|
||||
TAILQ_HEAD(, thread) sq_blocked[NR_SLEEPQS]; /* (c) Blocked threads. */
|
||||
LIST_ENTRY(sleepqueue) sq_hash; /* (c) Chain and free list. */
|
||||
LIST_HEAD(, sleepqueue) sq_free; /* (c) Free queues. */
|
||||
void *sq_wchan; /* (c) Wait channel. */
|
||||
@ -148,6 +148,8 @@ static MALLOC_DEFINE(M_SLEEPQUEUE, "sleepqueue", "sleep queues");
|
||||
/*
|
||||
* Prototypes for non-exported routines.
|
||||
*/
|
||||
static int sleepq_catch_signals(void *wchan);
|
||||
static int sleepq_check_signals(void);
|
||||
static int sleepq_check_timeout(void);
|
||||
static void sleepq_switch(void *wchan);
|
||||
static void sleepq_timeout(void *arg);
|
||||
@ -192,9 +194,11 @@ struct sleepqueue *
|
||||
sleepq_alloc(void)
|
||||
{
|
||||
struct sleepqueue *sq;
|
||||
int i;
|
||||
|
||||
sq = malloc(sizeof(struct sleepqueue), M_SLEEPQUEUE, M_WAITOK | M_ZERO);
|
||||
TAILQ_INIT(&sq->sq_blocked);
|
||||
for (i = 0; i < NR_SLEEPQS; i++)
|
||||
TAILQ_INIT(&sq->sq_blocked[i]);
|
||||
LIST_INIT(&sq->sq_free);
|
||||
return (sq);
|
||||
}
|
||||
@ -205,9 +209,11 @@ sleepq_alloc(void)
|
||||
void
|
||||
sleepq_free(struct sleepqueue *sq)
|
||||
{
|
||||
int i;
|
||||
|
||||
MPASS(sq != NULL);
|
||||
MPASS(TAILQ_EMPTY(&sq->sq_blocked));
|
||||
for (i = 0; i < NR_SLEEPQS; i++)
|
||||
MPASS(TAILQ_EMPTY(&sq->sq_blocked[i]));
|
||||
free(sq, M_SLEEPQUEUE);
|
||||
}
|
||||
|
||||
@ -262,7 +268,8 @@ sleepq_release(void *wchan)
|
||||
* woken up.
|
||||
*/
|
||||
void
|
||||
sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg, int flags)
|
||||
sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg, int flags,
|
||||
int queue)
|
||||
{
|
||||
struct sleepqueue_chain *sc;
|
||||
struct sleepqueue *sq;
|
||||
@ -273,6 +280,7 @@ sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg, int flags)
|
||||
mtx_assert(&sc->sc_lock, MA_OWNED);
|
||||
MPASS(td->td_sleepqueue != NULL);
|
||||
MPASS(wchan != NULL);
|
||||
MPASS((queue >= 0) && (queue < NR_SLEEPQS));
|
||||
|
||||
/* If this thread is not allowed to sleep, die a horrible death. */
|
||||
KASSERT(!(td->td_pflags & TDP_NOSLEEPING),
|
||||
@ -287,6 +295,16 @@ sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg, int flags)
|
||||
* into the sleep queue already in use by this wait channel.
|
||||
*/
|
||||
if (sq == NULL) {
|
||||
#ifdef INVARIANTS
|
||||
int i;
|
||||
for (i = 0; i < NR_SLEEP_QUEUEUS; i++)
|
||||
KASSERT(TAILQ_EMPTY(&sq->sq_blocked[i]),
|
||||
("thread's sleep queue %d is not empty", i));
|
||||
|
||||
KASSERT(LIST_EMPTY(&sq->sq_free),
|
||||
("thread's sleep queue has a non-empty free list"));
|
||||
KASSERT(sq->sq_wchan == NULL, ("stale sq_wchan pointer"));
|
||||
#endif
|
||||
#ifdef SLEEPQUEUE_PROFILING
|
||||
sc->sc_depth++;
|
||||
if (sc->sc_depth > sc->sc_max_depth) {
|
||||
@ -297,11 +315,6 @@ sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg, int flags)
|
||||
#endif
|
||||
sq = td->td_sleepqueue;
|
||||
LIST_INSERT_HEAD(&sc->sc_queues, sq, sq_hash);
|
||||
KASSERT(TAILQ_EMPTY(&sq->sq_blocked),
|
||||
("thread's sleep queue has a non-empty queue"));
|
||||
KASSERT(LIST_EMPTY(&sq->sq_free),
|
||||
("thread's sleep queue has a non-empty free list"));
|
||||
KASSERT(sq->sq_wchan == NULL, ("stale sq_wchan pointer"));
|
||||
sq->sq_wchan = wchan;
|
||||
#ifdef INVARIANTS
|
||||
sq->sq_lock = lock;
|
||||
@ -313,9 +326,10 @@ sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg, int flags)
|
||||
MPASS((flags & SLEEPQ_TYPE) == sq->sq_type);
|
||||
LIST_INSERT_HEAD(&sq->sq_free, td->td_sleepqueue, sq_hash);
|
||||
}
|
||||
TAILQ_INSERT_TAIL(&sq->sq_blocked, td, td_slpq);
|
||||
TAILQ_INSERT_TAIL(&sq->sq_blocked[queue], td, td_slpq);
|
||||
td->td_sleepqueue = NULL;
|
||||
mtx_lock_spin(&sched_lock);
|
||||
td->td_sqqueue = queue;
|
||||
td->td_wchan = wchan;
|
||||
td->td_wmesg = wmesg;
|
||||
if (flags & SLEEPQ_INTERRUPTIBLE) {
|
||||
@ -606,12 +620,13 @@ sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, int pri)
|
||||
MPASS(td != NULL);
|
||||
MPASS(sq->sq_wchan != NULL);
|
||||
MPASS(td->td_wchan == sq->sq_wchan);
|
||||
MPASS(td->td_sqqueue < NR_SLEEPQS && td->td_sqqueue >= 0);
|
||||
sc = SC_LOOKUP(sq->sq_wchan);
|
||||
mtx_assert(&sc->sc_lock, MA_OWNED);
|
||||
mtx_assert(&sched_lock, MA_OWNED);
|
||||
|
||||
/* Remove the thread from the queue. */
|
||||
TAILQ_REMOVE(&sq->sq_blocked, td, td_slpq);
|
||||
TAILQ_REMOVE(&sq->sq_blocked[td->td_sqqueue], td, td_slpq);
|
||||
|
||||
/*
|
||||
* Get a sleep queue for this thread. If this is the last waiter,
|
||||
@ -656,13 +671,14 @@ sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, int pri)
|
||||
* Find the highest priority thread sleeping on a wait channel and resume it.
|
||||
*/
|
||||
void
|
||||
sleepq_signal(void *wchan, int flags, int pri)
|
||||
sleepq_signal(void *wchan, int flags, int pri, int queue)
|
||||
{
|
||||
struct sleepqueue *sq;
|
||||
struct thread *td, *besttd;
|
||||
|
||||
CTR2(KTR_PROC, "sleepq_signal(%p, %d)", wchan, flags);
|
||||
KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
|
||||
MPASS((queue >= 0) && (queue < NR_SLEEPQS));
|
||||
sq = sleepq_lookup(wchan);
|
||||
if (sq == NULL) {
|
||||
sleepq_release(wchan);
|
||||
@ -678,7 +694,7 @@ sleepq_signal(void *wchan, int flags, int pri)
|
||||
* the tail of sleep queues.
|
||||
*/
|
||||
besttd = NULL;
|
||||
TAILQ_FOREACH(td, &sq->sq_blocked, td_slpq) {
|
||||
TAILQ_FOREACH(td, &sq->sq_blocked[queue], td_slpq) {
|
||||
if (besttd == NULL || td->td_priority < besttd->td_priority)
|
||||
besttd = td;
|
||||
}
|
||||
@ -693,12 +709,13 @@ sleepq_signal(void *wchan, int flags, int pri)
|
||||
* Resume all threads sleeping on a specified wait channel.
|
||||
*/
|
||||
void
|
||||
sleepq_broadcast(void *wchan, int flags, int pri)
|
||||
sleepq_broadcast(void *wchan, int flags, int pri, int queue)
|
||||
{
|
||||
struct sleepqueue *sq;
|
||||
|
||||
CTR2(KTR_PROC, "sleepq_broadcast(%p, %d)", wchan, flags);
|
||||
KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
|
||||
MPASS((queue >= 0) && (queue < NR_SLEEPQS));
|
||||
sq = sleepq_lookup(wchan);
|
||||
if (sq == NULL) {
|
||||
sleepq_release(wchan);
|
||||
@ -709,8 +726,9 @@ sleepq_broadcast(void *wchan, int flags, int pri)
|
||||
|
||||
/* Resume all blocked threads on the sleep queue. */
|
||||
mtx_lock_spin(&sched_lock);
|
||||
while (!TAILQ_EMPTY(&sq->sq_blocked))
|
||||
sleepq_resume_thread(sq, TAILQ_FIRST(&sq->sq_blocked), pri);
|
||||
while (!TAILQ_EMPTY(&sq->sq_blocked[queue]))
|
||||
sleepq_resume_thread(sq, TAILQ_FIRST(&sq->sq_blocked[queue]),
|
||||
pri);
|
||||
mtx_unlock_spin(&sched_lock);
|
||||
sleepq_release(wchan);
|
||||
}
|
||||
@ -901,15 +919,19 @@ DB_SHOW_COMMAND(sleepq, db_show_sleepqueue)
|
||||
}
|
||||
#endif
|
||||
db_printf("Blocked threads:\n");
|
||||
if (TAILQ_EMPTY(&sq->sq_blocked))
|
||||
db_printf("\tempty\n");
|
||||
else
|
||||
TAILQ_FOREACH(td, &sq->sq_blocked, td_slpq) {
|
||||
db_printf("\t%p (tid %d, pid %d, \"%s\")\n", td,
|
||||
td->td_tid, td->td_proc->p_pid,
|
||||
td->td_name[0] != '\0' ? td->td_name :
|
||||
td->td_proc->p_comm);
|
||||
}
|
||||
for (i = 0; i < NR_SLEEPQS; i++) {
|
||||
db_printf("\nQueue[%d]:\n", i);
|
||||
if (TAILQ_EMPTY(&sq->sq_blocked[i]))
|
||||
db_printf("\tempty\n");
|
||||
else
|
||||
TAILQ_FOREACH(td, &sq->sq_blocked[0],
|
||||
td_slpq) {
|
||||
db_printf("\t%p (tid %d, pid %d, \"%s\")\n", td,
|
||||
td->td_tid, td->td_proc->p_pid,
|
||||
td->td_name[i] != '\0' ? td->td_name :
|
||||
td->td_proc->p_comm);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Alias 'show sleepqueue' to 'show sleepq'. */
|
||||
|
@ -235,6 +235,7 @@ struct thread {
|
||||
int td_inhibitors; /* (j) Why can not run. */
|
||||
int td_pflags; /* (k) Private thread (TDP_*) flags. */
|
||||
int td_dupfd; /* (k) Ret value from fdopen. XXX */
|
||||
int td_sqqueue; /* (j) Sleepqueue queue blocked on. */
|
||||
void *td_wchan; /* (j) Sleep address. */
|
||||
const char *td_wmesg; /* (j) Reason for sleep. */
|
||||
u_char td_lastcpu; /* (j) Last cpu we were on. */
|
||||
|
@ -88,19 +88,20 @@ struct thread;
|
||||
|
||||
void init_sleepqueues(void);
|
||||
void sleepq_abort(struct thread *td, int intrval);
|
||||
void sleepq_add(void *, struct lock_object *, const char *, int);
|
||||
void sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg,
|
||||
int flags, int queue);
|
||||
struct sleepqueue *sleepq_alloc(void);
|
||||
void sleepq_broadcast(void *, int, int);
|
||||
void sleepq_free(struct sleepqueue *);
|
||||
void sleepq_lock(void *);
|
||||
struct sleepqueue *sleepq_lookup(void *);
|
||||
void sleepq_release(void *);
|
||||
void sleepq_remove(struct thread *, void *);
|
||||
void sleepq_signal(void *, int, int);
|
||||
void sleepq_broadcast(void *wchan, int flags, int pri, int queue);
|
||||
void sleepq_free(struct sleepqueue *sq);
|
||||
void sleepq_lock(void *wchan);
|
||||
struct sleepqueue *sleepq_lookup(void *wchan);
|
||||
void sleepq_release(void *wchan);
|
||||
void sleepq_remove(struct thread *td, void *wchan);
|
||||
void sleepq_signal(void *wchan, int flags, int pri, int queue);
|
||||
void sleepq_set_timeout(void *wchan, int timo);
|
||||
int sleepq_timedwait(void *wchan);
|
||||
int sleepq_timedwait_sig(void *wchan);
|
||||
void sleepq_wait(void *);
|
||||
void sleepq_wait(void *wchan);
|
||||
int sleepq_wait_sig(void *wchan);
|
||||
|
||||
#endif /* _KERNEL */
|
||||
|
Loading…
Reference in New Issue
Block a user