d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Commit 3/14 of sched_lock decomposition.

Browse Source 
- Add a per-turnstile spinlock to solve potential priority propagation
   deadlocks that are possible with thread_lock().
 - The turnstile lock order is defined as the exact opposite of the
   lock order used with the sleep locks they represent.  This allows us
   to walk in reverse order in priority_propagate and this is the only
   place we wish to multiply acquire turnstile locks.
 - Use the turnstile_chain lock to protect assigning mutexes to turnstiles.
 - Change the turnstile interface to pass back turnstile pointers to the
   consumers.  This allows us to reduce some locking and makes it easier
   to cancel turnstile assignment while the turnstile chain lock is held.

Tested by:      kris, current@
Tested on:      i386, amd64, ULE, 4BSD, libthr, libkse, PREEMPTION, etc.
Discussed with: kris, attilio, kmacy, jhb, julian, bde (small parts each)
This commit is contained in:
Jeff Roberson 2007-06-04 23:51:44 +00:00
parent d72e80f09a
commit 2502c107ba
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=170295
4 changed files with 305 additions and 192 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

149
sys/kern/kern_mutex.c
View File

@ -127,6 +127,7 @@ struct lock_class lock_class_mtx_spin = {
/*
* System-wide mutexes
*/
struct mtx blocked_lock;
struct mtx sched_lock;
struct mtx Giant;
@ -305,6 +306,7 @@ void
_mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file,
int line)
{
struct turnstile *ts;
#ifdef ADAPTIVE_MUTEXES
volatile struct thread *owner;
#endif
@ -334,7 +336,7 @@ _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file,
m->lock_object.lo_name, (void *)m->mtx_lock, file, line);
while (!_obtain_lock(m, tid)) {
turnstile_lock(&m->lock_object);
ts = turnstile_trywait(&m->lock_object);
v = m->mtx_lock;
/*
@ -342,7 +344,7 @@ _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file,
* the turnstile chain lock.
*/
if (v == MTX_UNOWNED) {
turnstile_release(&m->lock_object);
turnstile_cancel(ts);
cpu_spinwait();
continue;
}
@ -358,7 +360,7 @@ _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file,
*/
if (v == MTX_CONTESTED) {
m->mtx_lock = tid | MTX_CONTESTED;
turnstile_claim(&m->lock_object);
turnstile_claim(ts);
break;
}
#endif
@ -370,7 +372,7 @@ _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file,
*/
if ((v & MTX_CONTESTED) == 0 &&
!atomic_cmpset_ptr(&m->mtx_lock, v, v | MTX_CONTESTED)) {
turnstile_release(&m->lock_object);
turnstile_cancel(ts);
cpu_spinwait();
continue;
}
@ -387,7 +389,7 @@ _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file,
if (m != &Giant && TD_IS_RUNNING(owner))
#endif
{
turnstile_release(&m->lock_object);
turnstile_cancel(ts);
while (mtx_owner(m) == owner && TD_IS_RUNNING(owner)) {
cpu_spinwait();
}
@ -414,8 +416,7 @@ _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file,
/*
* Block on the turnstile.
*/
turnstile_wait(&m->lock_object, mtx_owner(m),
TS_EXCLUSIVE_QUEUE);
turnstile_wait(ts, mtx_owner(m), TS_EXCLUSIVE_QUEUE);
}
#ifdef KTR
if (cont_logged) {
@ -428,7 +429,25 @@ _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file,
waittime, (file), (line));
}
static void
_mtx_lock_spin_failed(struct mtx *m)
{
struct thread *td;
td = mtx_owner(m);
/* If the mutex is unlocked, try again. */
if (td == NULL)
return;
#ifdef SMP
printf( "spin lock %p (%s) held by %p (tid %d) too long\n",
m, m->lock_object.lo_name, td, td->td_tid);
#ifdef WITNESS
witness_display_spinlock(&m->lock_object, td);
#endif
panic("spin lock held too long");
}
/*
* _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock.
*
@ -440,7 +459,6 @@ _mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file,
int line)
{
int i = 0, contested = 0;
struct thread *td;
uint64_t waittime = 0;
if (LOCK_LOG_TEST(&m->lock_object, opts))
@ -458,20 +476,8 @@ _mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file,
}
if (i < 60000000 || kdb_active || panicstr != NULL)
DELAY(1);
else {
td = mtx_owner(m);
/* If the mutex is unlocked, try again. */
if (td == NULL)
continue;
printf(
"spin lock %p (%s) held by %p (tid %d) too long\n",
m, m->lock_object.lo_name, td, td->td_tid);
#ifdef WITNESS
witness_display_spinlock(&m->lock_object, td);
#endif
panic("spin lock held too long");
}
else
_mtx_lock_spin_failed(m);
cpu_spinwait();
}
spinlock_enter();
@ -482,10 +488,87 @@ _mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file,
lock_profile_obtain_lock_success(&m->lock_object, contested,
waittime, (file), (line));
}
#endif /* SMP */
void
_thread_lock_flags(struct thread *td, int opts, const char *file, int line)
{
struct mtx *m;
uintptr_t tid;
int i;
i = 0;
tid = (uintptr_t)curthread;
for (;;) {
retry:
spinlock_enter();
m = __DEVOLATILE(struct mtx *, td->td_lock);
WITNESS_CHECKORDER(&m->lock_object,
opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line);
while (!_obtain_lock(m, tid)) {
if (m->mtx_lock == tid) {
m->mtx_recurse++;
break;
}
/* Give interrupts a chance while we spin. */
spinlock_exit();
while (m->mtx_lock != MTX_UNOWNED) {
if (i++ < 10000000)
cpu_spinwait();
else if (i < 60000000 ||
kdb_active || panicstr != NULL)
DELAY(1);
else
_mtx_lock_spin_failed(m);
cpu_spinwait();
if (m != td->td_lock)
goto retry;
}
spinlock_enter();
}
if (m == td->td_lock)
break;
_rel_spin_lock(m); /* does spinlock_exit() */
}
WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line);
}
struct mtx *
thread_lock_block(struct thread *td)
{
struct mtx *lock;
spinlock_enter();
THREAD_LOCK_ASSERT(td, MA_OWNED);
lock = __DEVOLATILE(struct mtx *, td->td_lock);
td->td_lock = &blocked_lock;
mtx_unlock_spin(lock);
return (lock);
}
void
thread_lock_unblock(struct thread *td, struct mtx *new)
{
mtx_assert(new, MA_OWNED);
MPASS(td->td_lock == &blocked_lock);
atomic_store_rel_ptr((void *)&td->td_lock, (uintptr_t)new);
spinlock_exit();
}
void
thread_lock_set(struct thread *td, struct mtx *new)
{
struct mtx *lock;
mtx_assert(new, MA_OWNED);
THREAD_LOCK_ASSERT(td, MA_OWNED);
lock = __DEVOLATILE(struct mtx *, td->td_lock);
td->td_lock = new;
mtx_unlock_spin(lock);
}
/*
* _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock.
*
@ -508,7 +591,11 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line)
return;
}
turnstile_lock(&m->lock_object);
/*
* We have to lock the chain before the turnstile so this turnstile
* can be removed from the hash list if it is empty.
*/
turnstile_chain_lock(&m->lock_object);
ts = turnstile_lookup(&m->lock_object);
if (LOCK_LOG_TEST(&m->lock_object, opts))
CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m);
@ -518,7 +605,7 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line)
_release_lock_quick(m);
if (LOCK_LOG_TEST(&m->lock_object, opts))
CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p no sleepers", m);
turnstile_release(&m->lock_object);
turnstile_chain_unlock(&m->lock_object);
return;
}
#else
@ -543,7 +630,12 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line)
m);
}
#endif
/*
* This turnstile is now no longer associated with the mutex. We can
* unlock the chain lock so a new turnstile may take it's place.
*/
turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
turnstile_chain_unlock(&m->lock_object);
#ifndef PREEMPTION
/*
@ -557,7 +649,7 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line)
if (td->td_critnest > 0 || td1->td_priority >= td->td_priority)
return;
mtx_lock_spin(&sched_lock);
thread_lock(td1);
if (!TD_IS_RUNNING(td1)) {
#ifdef notyet
if (td->td_ithd != NULL) {
@ -582,7 +674,7 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line)
CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p",
m, (void *)m->mtx_lock);
}
mtx_unlock_spin(&sched_lock);
thread_unlock(td1);
#endif
}
@ -761,7 +853,10 @@ mutex_init(void)
*/
mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE);
mtx_init(&sched_lock, "sched lock", NULL, MTX_SPIN | MTX_RECURSE);
mtx_init(&blocked_lock, "blocked lock", NULL, MTX_SPIN);
blocked_lock.mtx_lock = 0xdeadc0de; /* Always blocked. */
mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK);
mtx_init(&proc0.p_slock, "process slock", NULL, MTX_SPIN | MTX_RECURSE);
mtx_init(&devmtx, "cdev", NULL, MTX_DEF);
mtx_lock(&Giant);

54
sys/kern/kern_rwlock.c
View File

@ -187,6 +187,7 @@ _rw_wunlock(struct rwlock *rw, const char *file, int line)
void
_rw_rlock(struct rwlock *rw, const char *file, int line)
{
struct turnstile *ts;
#ifdef ADAPTIVE_RWLOCKS
volatile struct thread *owner;
#endif
@ -256,7 +257,7 @@ _rw_rlock(struct rwlock *rw, const char *file, int line)
* has a write lock, so acquire the turnstile lock so we can
* begin the process of blocking.
*/
turnstile_lock(&rw->lock_object);
ts = turnstile_trywait(&rw->lock_object);
/*
* The lock might have been released while we spun, so
@ -265,7 +266,7 @@ _rw_rlock(struct rwlock *rw, const char *file, int line)
*/
x = rw->rw_lock;
if (x & RW_LOCK_READ) {
turnstile_release(&rw->lock_object);
turnstile_cancel(ts);
cpu_spinwait();
continue;
}
@ -279,7 +280,7 @@ _rw_rlock(struct rwlock *rw, const char *file, int line)
if (!(x & RW_LOCK_READ_WAITERS)) {
if (!atomic_cmpset_ptr(&rw->rw_lock, x,
x | RW_LOCK_READ_WAITERS)) {
turnstile_release(&rw->lock_object);
turnstile_cancel(ts);
cpu_spinwait();
continue;
}
@ -296,7 +297,7 @@ _rw_rlock(struct rwlock *rw, const char *file, int line)
*/
owner = (struct thread *)RW_OWNER(x);
if (TD_IS_RUNNING(owner)) {
turnstile_release(&rw->lock_object);
turnstile_cancel(ts);
if (LOCK_LOG_TEST(&rw->lock_object, 0))
CTR3(KTR_LOCK, "%s: spinning on %p held by %p",
__func__, rw, owner);
@ -314,7 +315,7 @@ _rw_rlock(struct rwlock *rw, const char *file, int line)
if (LOCK_LOG_TEST(&rw->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__,
rw);
turnstile_wait(&rw->lock_object, rw_owner(rw), TS_SHARED_QUEUE);
turnstile_wait(ts, rw_owner(rw), TS_SHARED_QUEUE);
if (LOCK_LOG_TEST(&rw->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p resuming from turnstile",
__func__, rw);
@ -407,7 +408,7 @@ _rw_runlock(struct rwlock *rw, const char *file, int line)
* Ok, we know we have a waiting writer and we think we
* are the last reader, so grab the turnstile lock.
*/
turnstile_lock(&rw->lock_object);
turnstile_chain_lock(&rw->lock_object);
/*
* Try to drop our lock leaving the lock in a unlocked
@ -427,7 +428,7 @@ _rw_runlock(struct rwlock *rw, const char *file, int line)
*/
if (!atomic_cmpset_ptr(&rw->rw_lock,
RW_READERS_LOCK(1) | RW_LOCK_WRITE_WAITERS, RW_UNLOCKED)) {
turnstile_release(&rw->lock_object);
turnstile_chain_unlock(&rw->lock_object);
continue;
}
if (LOCK_LOG_TEST(&rw->lock_object, 0))
@ -445,6 +446,7 @@ _rw_runlock(struct rwlock *rw, const char *file, int line)
MPASS(ts != NULL);
turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE);
turnstile_unpend(ts, TS_SHARED_LOCK);
turnstile_chain_unlock(&rw->lock_object);
break;
}
lock_profile_release_lock(&rw->lock_object);
@ -458,6 +460,7 @@ _rw_runlock(struct rwlock *rw, const char *file, int line)
void
_rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
{
struct turnstile *ts;
#ifdef ADAPTIVE_RWLOCKS
volatile struct thread *owner;
#endif
@ -468,7 +471,7 @@ _rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
rw->lock_object.lo_name, (void *)rw->rw_lock, file, line);
while (!_rw_write_lock(rw, tid)) {
turnstile_lock(&rw->lock_object);
ts = turnstile_trywait(&rw->lock_object);
v = rw->rw_lock;
/*
@ -476,7 +479,7 @@ _rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
* turnstile chain lock, try again.
*/
if (v == RW_UNLOCKED) {
turnstile_release(&rw->lock_object);
turnstile_cancel(ts);
cpu_spinwait();
continue;
}
@ -495,12 +498,12 @@ _rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
if (atomic_cmpset_acq_ptr(&rw->rw_lock,
RW_UNLOCKED | RW_LOCK_WRITE_WAITERS,
tid | RW_LOCK_WRITE_WAITERS)) {
turnstile_claim(&rw->lock_object);
turnstile_claim(ts);
CTR2(KTR_LOCK, "%s: %p claimed by new writer",
__func__, rw);
break;
}
turnstile_release(&rw->lock_object);
turnstile_cancel(ts);
cpu_spinwait();
continue;
}
@ -513,7 +516,7 @@ _rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
if (!(v & RW_LOCK_WRITE_WAITERS)) {
if (!atomic_cmpset_ptr(&rw->rw_lock, v,
v | RW_LOCK_WRITE_WAITERS)) {
turnstile_release(&rw->lock_object);
turnstile_cancel(ts);
cpu_spinwait();
continue;
}
@ -530,7 +533,7 @@ _rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
*/
owner = (struct thread *)RW_OWNER(v);
if (!(v & RW_LOCK_READ) && TD_IS_RUNNING(owner)) {
turnstile_release(&rw->lock_object);
turnstile_cancel(ts);
if (LOCK_LOG_TEST(&rw->lock_object, 0))
CTR3(KTR_LOCK, "%s: spinning on %p held by %p",
__func__, rw, owner);
@ -548,8 +551,7 @@ _rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
if (LOCK_LOG_TEST(&rw->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__,
rw);
turnstile_wait(&rw->lock_object, rw_owner(rw),
TS_EXCLUSIVE_QUEUE);
turnstile_wait(ts, rw_owner(rw), TS_EXCLUSIVE_QUEUE);
if (LOCK_LOG_TEST(&rw->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p resuming from turnstile",
__func__, rw);
@ -574,7 +576,7 @@ _rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
if (LOCK_LOG_TEST(&rw->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p contested", __func__, rw);
turnstile_lock(&rw->lock_object);
turnstile_chain_lock(&rw->lock_object);
ts = turnstile_lookup(&rw->lock_object);
#ifdef ADAPTIVE_RWLOCKS
@ -587,7 +589,7 @@ _rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
atomic_store_rel_ptr(&rw->rw_lock, RW_UNLOCKED);
if (LOCK_LOG_TEST(&rw->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p no sleepers", __func__, rw);
turnstile_release(&rw->lock_object);
turnstile_chain_unlock(&rw->lock_object);
return;
}
#else
@ -640,6 +642,7 @@ _rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
CTR2(KTR_LOCK, "%s: %p no sleepers 2", __func__, rw);
atomic_store_rel_ptr(&rw->rw_lock, v);
turnstile_disown(ts);
turnstile_chain_unlock(&rw->lock_object);
return;
}
#endif
@ -651,6 +654,7 @@ _rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
turnstile_broadcast(ts, queue);
atomic_store_rel_ptr(&rw->rw_lock, v);
turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
turnstile_chain_unlock(&rw->lock_object);
}
/*
@ -662,6 +666,7 @@ int
_rw_try_upgrade(struct rwlock *rw, const char *file, int line)
{
uintptr_t v, tid;
struct turnstile *ts;
int success;
KASSERT(rw->rw_lock != RW_DESTROYED,
@ -686,7 +691,7 @@ _rw_try_upgrade(struct rwlock *rw, const char *file, int line)
* Ok, we think we have write waiters, so lock the
* turnstile.
*/
turnstile_lock(&rw->lock_object);
ts = turnstile_trywait(&rw->lock_object);
/*
* Try to switch from one reader to a writer again. This time
@ -705,9 +710,9 @@ _rw_try_upgrade(struct rwlock *rw, const char *file, int line)
#else
if (success && v)
#endif
turnstile_claim(&rw->lock_object);
turnstile_claim(ts);
else
turnstile_release(&rw->lock_object);
turnstile_cancel(ts);
out:
LOCK_LOG_TRY("WUPGRADE", &rw->lock_object, 0, success, file, line);
if (success)
@ -745,7 +750,7 @@ _rw_downgrade(struct rwlock *rw, const char *file, int line)
* Ok, we think we have waiters, so lock the turnstile so we can
* read the waiter flags without any races.
*/
turnstile_lock(&rw->lock_object);
turnstile_chain_lock(&rw->lock_object);
v = rw->rw_lock;
MPASS(v & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS));
@ -779,12 +784,9 @@ _rw_downgrade(struct rwlock *rw, const char *file, int line)
(v & RW_LOCK_WRITE_WAITERS));
if (v & RW_LOCK_READ_WAITERS)
turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
#ifdef ADAPTIVE_RWLOCKS
else if (ts == NULL)
turnstile_release(&rw->lock_object);
#endif
else
else if (ts)
turnstile_disown(ts);
turnstile_chain_unlock(&rw->lock_object);
out:
LOCK_LOG_LOCK("WDOWNGRADE", &rw->lock_object, 0, 0, file, line);
}

284
sys/kern/subr_turnstile.c
View File

@ -116,6 +116,7 @@ __FBSDID("$FreeBSD$");
* q - td_contested lock
*/
struct turnstile {
struct mtx ts_lock; /* Spin lock for self. */
struct threadqueue ts_blocked[2]; /* (c + q) Blocked threads. */
struct threadqueue ts_pending; /* (c) Pending threads. */
LIST_ENTRY(turnstile) ts_hash; /* (c) Chain and free list. */
@ -162,6 +163,7 @@ static void turnstile_setowner(struct turnstile *ts, struct thread *owner);
static void turnstile_dtor(void *mem, int size, void *arg);
#endif
static int turnstile_init(void *mem, int size, int flags);
static void turnstile_fini(void *mem, int size);
/*
* Walks the chain of turnstiles and their owners to propagate the priority
@ -171,13 +173,20 @@ static int turnstile_init(void *mem, int size, int flags);
static void
propagate_priority(struct thread *td)
{
struct turnstile_chain *tc;
struct turnstile *ts;
int pri;
mtx_assert(&sched_lock, MA_OWNED);
THREAD_LOCK_ASSERT(td, MA_OWNED);
pri = td->td_priority;
ts = td->td_blocked;
MPASS(td->td_lock == &ts->ts_lock);
/*
* Grab a recursive lock on this turnstile chain so it stays locked
* for the whole operation. The caller expects us to return with
* the original lock held. We only ever lock down the chain so
* the lock order is constant.
*/
mtx_lock_spin(&ts->ts_lock);
for (;;) {
td = ts->ts_owner;
@ -186,9 +195,12 @@ propagate_priority(struct thread *td)
* This might be a read lock with no owner. There's
* not much we can do, so just bail.
*/
mtx_unlock_spin(&ts->ts_lock);
return;
}
thread_lock_flags(td, MTX_DUPOK);
mtx_unlock_spin(&ts->ts_lock);
MPASS(td->td_proc != NULL);
MPASS(td->td_proc->p_magic == P_MAGIC);
@ -213,8 +225,10 @@ propagate_priority(struct thread *td)
* If this thread already has higher priority than the
* thread that is being blocked, we are finished.
*/
if (td->td_priority <= pri)
if (td->td_priority <= pri) {
thread_unlock(td);
return;
}
/*
* Bump this thread's priority.
@ -227,6 +241,7 @@ propagate_priority(struct thread *td)
*/
if (TD_IS_RUNNING(td) || TD_ON_RUNQ(td)) {
MPASS(td->td_blocked == NULL);
thread_unlock(td);
return;
}
@ -251,15 +266,13 @@ propagate_priority(struct thread *td)
*/
ts = td->td_blocked;
MPASS(ts != NULL);
tc = TC_LOOKUP(ts->ts_lockobj);
mtx_lock_spin(&tc->tc_lock);
MPASS(td->td_lock == &ts->ts_lock);
/* Resort td on the list if needed. */
if (!turnstile_adjust_thread(ts, td)) {
mtx_unlock_spin(&tc->tc_lock);
mtx_unlock_spin(&ts->ts_lock);
return;
}
mtx_unlock_spin(&tc->tc_lock);
/* The thread lock is released as ts lock above. */
}
}
@ -270,17 +283,16 @@ propagate_priority(struct thread *td)
static int
turnstile_adjust_thread(struct turnstile *ts, struct thread *td)
{
struct turnstile_chain *tc;
struct thread *td1, *td2;
int queue;
mtx_assert(&sched_lock, MA_OWNED);
THREAD_LOCK_ASSERT(td, MA_OWNED);
MPASS(TD_ON_LOCK(td));
/*
* This thread may not be blocked on this turnstile anymore
* but instead might already be woken up on another CPU
* that is waiting on sched_lock in turnstile_unpend() to
* that is waiting on the thread lock in turnstile_unpend() to
* finish waking this thread up. We can detect this case
* by checking to see if this thread has been given a
* turnstile by either turnstile_signal() or
@ -295,8 +307,7 @@ turnstile_adjust_thread(struct turnstile *ts, struct thread *td)
* It needs to be moved if either its priority is lower than
* the previous thread or higher than the next thread.
*/
tc = TC_LOOKUP(ts->ts_lockobj);
mtx_assert(&tc->tc_lock, MA_OWNED);
MPASS(td->td_lock == &ts->ts_lock);
td1 = TAILQ_PREV(td, threadqueue, td_lockq);
td2 = TAILQ_NEXT(td, td_lockq);
if ((td1 != NULL && td->td_priority < td1->td_priority) ||
@ -385,9 +396,10 @@ init_turnstile0(void *dummy)
turnstile_zone = uma_zcreate("TURNSTILE", sizeof(struct turnstile),
#ifdef INVARIANTS
NULL, turnstile_dtor, turnstile_init, NULL, UMA_ALIGN_CACHE, 0);
NULL, turnstile_dtor, turnstile_init, turnstile_fini,
UMA_ALIGN_CACHE, 0);
#else
NULL, NULL, turnstile_init, NULL, UMA_ALIGN_CACHE, 0);
NULL, NULL, turnstile_init, turnstile_fini, UMA_ALIGN_CACHE, 0);
#endif
thread0.td_turnstile = turnstile_alloc();
}
@ -400,10 +412,8 @@ SYSINIT(turnstile0, SI_SUB_LOCK, SI_ORDER_ANY, init_turnstile0, NULL);
void
turnstile_adjust(struct thread *td, u_char oldpri)
{
struct turnstile_chain *tc;
struct turnstile *ts;
mtx_assert(&sched_lock, MA_OWNED);
MPASS(TD_ON_LOCK(td));
/*
@ -411,15 +421,12 @@ turnstile_adjust(struct thread *td, u_char oldpri)
*/
ts = td->td_blocked;
MPASS(ts != NULL);
tc = TC_LOOKUP(ts->ts_lockobj);
mtx_lock_spin(&tc->tc_lock);
MPASS(td->td_lock == &ts->ts_lock);
mtx_assert(&ts->ts_lock, MA_OWNED);
/* Resort the turnstile on the list. */
if (!turnstile_adjust_thread(ts, td)) {
mtx_unlock_spin(&tc->tc_lock);
if (!turnstile_adjust_thread(ts, td))
return;
}
/*
* If our priority was lowered and we are at the head of the
* turnstile, then propagate our new priority up the chain.
@ -430,12 +437,8 @@ turnstile_adjust(struct thread *td, u_char oldpri)
td->td_tsqueue == TS_SHARED_QUEUE);
if (td == TAILQ_FIRST(&ts->ts_blocked[td->td_tsqueue]) &&
td->td_priority < oldpri) {
mtx_unlock_spin(&tc->tc_lock);
critical_enter();
propagate_priority(td);
critical_exit();
} else
mtx_unlock_spin(&tc->tc_lock);
}
}
/*
@ -487,9 +490,19 @@ turnstile_init(void *mem, int size, int flags)
TAILQ_INIT(&ts->ts_blocked[TS_SHARED_QUEUE]);
TAILQ_INIT(&ts->ts_pending);
LIST_INIT(&ts->ts_free);
mtx_init(&ts->ts_lock, "turnstile lock", NULL, MTX_SPIN | MTX_RECURSE);
return (0);
}
static void
turnstile_fini(void *mem, int size)
{
struct turnstile *ts;
ts = mem;
mtx_destroy(&ts->ts_lock);
}
/*
* Get a turnstile for a new thread.
*/
@ -514,7 +527,7 @@ turnstile_free(struct turnstile *ts)
* Lock the turnstile chain associated with the specified lock.
*/
void
turnstile_lock(struct lock_object *lock)
turnstile_chain_lock(struct lock_object *lock)
{
struct turnstile_chain *tc;
@ -522,6 +535,45 @@ turnstile_lock(struct lock_object *lock)
mtx_lock_spin(&tc->tc_lock);
}
struct turnstile *
turnstile_trywait(struct lock_object *lock)
{
struct turnstile_chain *tc;
struct turnstile *ts;
tc = TC_LOOKUP(lock);
mtx_lock_spin(&tc->tc_lock);
LIST_FOREACH(ts, &tc->tc_turnstiles, ts_hash)
if (ts->ts_lockobj == lock) {
mtx_lock_spin(&ts->ts_lock);
return (ts);
}
ts = curthread->td_turnstile;
MPASS(ts != NULL);
mtx_lock_spin(&ts->ts_lock);
KASSERT(ts->ts_lockobj == NULL, ("stale ts_lockobj pointer"));
ts->ts_lockobj = lock;
return (ts);
}
void
turnstile_cancel(struct turnstile *ts)
{
struct turnstile_chain *tc;
struct lock_object *lock;
mtx_assert(&ts->ts_lock, MA_OWNED);
mtx_unlock_spin(&ts->ts_lock);
lock = ts->ts_lockobj;
if (ts == curthread->td_turnstile)
ts->ts_lockobj = NULL;
tc = TC_LOOKUP(lock);
mtx_unlock_spin(&tc->tc_lock);
}
/*
* Look up the turnstile for a lock in the hash table locking the associated
* turnstile chain along the way. If no turnstile is found in the hash
@ -536,8 +588,10 @@ turnstile_lookup(struct lock_object *lock)
tc = TC_LOOKUP(lock);
mtx_assert(&tc->tc_lock, MA_OWNED);
LIST_FOREACH(ts, &tc->tc_turnstiles, ts_hash)
if (ts->ts_lockobj == lock)
if (ts->ts_lockobj == lock) {
mtx_lock_spin(&ts->ts_lock);
return (ts);
}
return (NULL);
}
@ -545,7 +599,7 @@ turnstile_lookup(struct lock_object *lock)
* Unlock the turnstile chain associated with a given lock.
*/
void
turnstile_release(struct lock_object *lock)
turnstile_chain_unlock(struct lock_object *lock)
{
struct turnstile_chain *tc;
@ -574,16 +628,13 @@ turnstile_first_waiter(struct turnstile *ts)
* owner appropriately.
*/
void
turnstile_claim(struct lock_object *lock)
turnstile_claim(struct turnstile *ts)
{
struct turnstile_chain *tc;
struct turnstile *ts;
struct thread *td, *owner;
struct turnstile_chain *tc;
tc = TC_LOOKUP(lock);
mtx_assert(&tc->tc_lock, MA_OWNED);
ts = turnstile_lookup(lock);
MPASS(ts != NULL);
mtx_assert(&ts->ts_lock, MA_OWNED);
MPASS(ts != curthread->td_turnstile);
owner = curthread;
mtx_lock_spin(&td_contested_lock);
@ -593,15 +644,18 @@ turnstile_claim(struct lock_object *lock)
td = turnstile_first_waiter(ts);
MPASS(td != NULL);
MPASS(td->td_proc->p_magic == P_MAGIC);
mtx_unlock_spin(&tc->tc_lock);
MPASS(td->td_lock == &ts->ts_lock);
/*
* Update the priority of the new owner if needed.
*/
mtx_lock_spin(&sched_lock);
thread_lock(owner);
if (td->td_priority < owner->td_priority)
sched_lend_prio(owner, td->td_priority);
mtx_unlock_spin(&sched_lock);
thread_unlock(owner);
tc = TC_LOOKUP(ts->ts_lockobj);
mtx_unlock_spin(&ts->ts_lock);
mtx_unlock_spin(&tc->tc_lock);
}
/*
@ -611,31 +665,28 @@ turnstile_claim(struct lock_object *lock)
* turnstile chain locked and will return with it unlocked.
*/
void
turnstile_wait(struct lock_object *lock, struct thread *owner, int queue)
turnstile_wait(struct turnstile *ts, struct thread *owner, int queue)
{
struct turnstile_chain *tc;
struct turnstile *ts;
struct thread *td, *td1;
struct lock_object *lock;
td = curthread;
tc = TC_LOOKUP(lock);
mtx_assert(&tc->tc_lock, MA_OWNED);
MPASS(td->td_turnstile != NULL);
mtx_assert(&ts->ts_lock, MA_OWNED);
if (queue == TS_SHARED_QUEUE)
MPASS(owner != NULL);
if (owner)
MPASS(owner->td_proc->p_magic == P_MAGIC);
MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE);
/* Look up the turnstile associated with the lock 'lock'. */
ts = turnstile_lookup(lock);
/*
* If the lock does not already have a turnstile, use this thread's
* turnstile. Otherwise insert the current thread into the
* turnstile already in use by this lock.
*/
if (ts == NULL) {
tc = TC_LOOKUP(ts->ts_lockobj);
if (ts == td->td_turnstile) {
mtx_assert(&tc->tc_lock, MA_OWNED);
#ifdef TURNSTILE_PROFILING
tc->tc_depth++;
if (tc->tc_depth > tc->tc_max_depth) {
@ -644,7 +695,7 @@ turnstile_wait(struct lock_object *lock, struct thread *owner, int queue)
turnstile_max_depth = tc->tc_max_depth;
}
#endif
ts = td->td_turnstile;
tc = TC_LOOKUP(ts->ts_lockobj);
LIST_INSERT_HEAD(&tc->tc_turnstiles, ts, ts_hash);
KASSERT(TAILQ_EMPTY(&ts->ts_pending),
("thread's turnstile has pending threads"));
@ -654,8 +705,7 @@ turnstile_wait(struct lock_object *lock, struct thread *owner, int queue)
("thread's turnstile has shared waiters"));
KASSERT(LIST_EMPTY(&ts->ts_free),
("thread's turnstile has a non-empty free list"));
KASSERT(ts->ts_lockobj == NULL, ("stale ts_lockobj pointer"));
ts->ts_lockobj = lock;
MPASS(ts->ts_lockobj != NULL);
mtx_lock_spin(&td_contested_lock);
TAILQ_INSERT_TAIL(&ts->ts_blocked[queue], td, td_lockq);
turnstile_setowner(ts, owner);
@ -674,58 +724,31 @@ turnstile_wait(struct lock_object *lock, struct thread *owner, int queue)
MPASS(td->td_turnstile != NULL);
LIST_INSERT_HEAD(&ts->ts_free, td->td_turnstile, ts_hash);
}
thread_lock(td);
thread_lock_set(td, &ts->ts_lock);
td->td_turnstile = NULL;
mtx_unlock_spin(&tc->tc_lock);
mtx_lock_spin(&sched_lock);
/*
* Handle race condition where a thread on another CPU that owns
* lock 'lock' could have woken us in between us dropping the
* turnstile chain lock and acquiring the sched_lock.
*/
if (td->td_flags & TDF_TSNOBLOCK) {
td->td_flags &= ~TDF_TSNOBLOCK;
mtx_unlock_spin(&sched_lock);
return;
}
#ifdef notyet
/*
* If we're borrowing an interrupted thread's VM context, we
* must clean up before going to sleep.
*/
if (td->td_ithd != NULL) {
struct ithd *it = td->td_ithd;
if (it->it_interrupted) {
if (LOCK_LOG_TEST(lock, 0))
CTR3(KTR_LOCK, "%s: %p interrupted %p",
__func__, it, it->it_interrupted);
intr_thd_fixup(it);
}
}
#endif
/* Save who we are blocked on and switch. */
lock = ts->ts_lockobj;
td->td_tsqueue = queue;
td->td_blocked = ts;
td->td_lockname = lock->lo_name;
TD_SET_LOCK(td);
critical_enter();
mtx_unlock_spin(&tc->tc_lock);
propagate_priority(td);
critical_exit();
if (LOCK_LOG_TEST(lock, 0))
CTR4(KTR_LOCK, "%s: td %d blocked on [%p] %s", __func__,
td->td_tid, lock, lock->lo_name);
MPASS(td->td_lock == &ts->ts_lock);
SCHED_STAT_INC(switch_turnstile);
mi_switch(SW_VOL, NULL);
if (LOCK_LOG_TEST(lock, 0))
CTR4(KTR_LOCK, "%s: td %d free from blocked on [%p] %s",
__func__, td->td_tid, lock, lock->lo_name);
mtx_unlock_spin(&sched_lock);
thread_unlock(td);
}
/*
@ -740,11 +763,10 @@ turnstile_signal(struct turnstile *ts, int queue)
int empty;
MPASS(ts != NULL);
mtx_assert(&ts->ts_lock, MA_OWNED);
MPASS(curthread->td_proc->p_magic == P_MAGIC);
MPASS(ts->ts_owner == curthread ||
(queue == TS_EXCLUSIVE_QUEUE && ts->ts_owner == NULL));
tc = TC_LOOKUP(ts->ts_lockobj);
mtx_assert(&tc->tc_lock, MA_OWNED);
MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE);
/*
@ -766,6 +788,8 @@ turnstile_signal(struct turnstile *ts, int queue)
empty = TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]) &&
TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE]);
if (empty) {
tc = TC_LOOKUP(ts->ts_lockobj);
mtx_assert(&tc->tc_lock, MA_OWNED);
MPASS(LIST_EMPTY(&ts->ts_free));
#ifdef TURNSTILE_PROFILING
tc->tc_depth--;
@ -791,9 +815,14 @@ turnstile_broadcast(struct turnstile *ts, int queue)
struct thread *td;
MPASS(ts != NULL);
mtx_assert(&ts->ts_lock, MA_OWNED);
MPASS(curthread->td_proc->p_magic == P_MAGIC);
MPASS(ts->ts_owner == curthread ||
(queue == TS_EXCLUSIVE_QUEUE && ts->ts_owner == NULL));
/*
* We must have the chain locked so that we can remove the empty
* turnstile from the hash queue.
*/
tc = TC_LOOKUP(ts->ts_lockobj);
mtx_assert(&tc->tc_lock, MA_OWNED);
MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE);
@ -833,15 +862,14 @@ void
turnstile_unpend(struct turnstile *ts, int owner_type)
{
TAILQ_HEAD( ,thread) pending_threads;
struct turnstile_chain *tc;
struct turnstile *nts;
struct thread *td;
u_char cp, pri;
MPASS(ts != NULL);
mtx_assert(&ts->ts_lock, MA_OWNED);
MPASS(ts->ts_owner == curthread ||
(owner_type == TS_SHARED_LOCK && ts->ts_owner == NULL));
tc = TC_LOOKUP(ts->ts_lockobj);
mtx_assert(&tc->tc_lock, MA_OWNED);
MPASS(!TAILQ_EMPTY(&ts->ts_pending));
/*
@ -855,7 +883,15 @@ turnstile_unpend(struct turnstile *ts, int owner_type)
TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE]))
ts->ts_lockobj = NULL;
#endif
/*
* Adjust the priority of curthread based on other contested
* locks it owns. Don't lower the priority below the base
* priority however.
*/
td = curthread;
pri = PRI_MAX;
thread_lock(td);
mtx_lock_spin(&td_contested_lock);
/*
* Remove the turnstile from this thread's list of contested locks
* since this thread doesn't own it anymore. New threads will
@ -864,31 +900,17 @@ turnstile_unpend(struct turnstile *ts, int owner_type)
* lock.
*/
if (ts->ts_owner != NULL) {
mtx_lock_spin(&td_contested_lock);
ts->ts_owner = NULL;
LIST_REMOVE(ts, ts_link);
mtx_unlock_spin(&td_contested_lock);
}
critical_enter();
mtx_unlock_spin(&tc->tc_lock);
/*
* Adjust the priority of curthread based on other contested
* locks it owns. Don't lower the priority below the base
* priority however.
*/
td = curthread;
pri = PRI_MAX;
mtx_lock_spin(&sched_lock);
mtx_lock_spin(&td_contested_lock);
LIST_FOREACH(ts, &td->td_contested, ts_link) {
cp = turnstile_first_waiter(ts)->td_priority;
LIST_FOREACH(nts, &td->td_contested, ts_link) {
cp = turnstile_first_waiter(nts)->td_priority;
if (cp < pri)
pri = cp;
}
mtx_unlock_spin(&td_contested_lock);
sched_unlend_prio(td, pri);
thread_unlock(td);
/*
* Wake up all the pending threads. If a thread is not blocked
* on a lock, then it is currently executing on another CPU in
@ -899,23 +921,21 @@ turnstile_unpend(struct turnstile *ts, int owner_type)
while (!TAILQ_EMPTY(&pending_threads)) {
td = TAILQ_FIRST(&pending_threads);
TAILQ_REMOVE(&pending_threads, td, td_lockq);
thread_lock(td);
MPASS(td->td_lock == &ts->ts_lock);
MPASS(td->td_proc->p_magic == P_MAGIC);
if (TD_ON_LOCK(td)) {
td->td_blocked = NULL;
td->td_lockname = NULL;
MPASS(TD_ON_LOCK(td));
TD_CLR_LOCK(td);
MPASS(TD_CAN_RUN(td));
td->td_blocked = NULL;
td->td_lockname = NULL;
#ifdef INVARIANTS
td->td_tsqueue = 0xff;
td->td_tsqueue = 0xff;
#endif
TD_CLR_LOCK(td);
MPASS(TD_CAN_RUN(td));
sched_add(td, SRQ_BORING);
} else {
td->td_flags |= TDF_TSNOBLOCK;
MPASS(TD_IS_RUNNING(td) || TD_ON_RUNQ(td));
}
sched_add(td, SRQ_BORING);
thread_unlock(td);
}
critical_exit();
mtx_unlock_spin(&sched_lock);
mtx_unlock_spin(&ts->ts_lock);
}
/*
@ -925,14 +945,12 @@ turnstile_unpend(struct turnstile *ts, int owner_type)
void
turnstile_disown(struct turnstile *ts)
{
struct turnstile_chain *tc;
struct thread *td;
u_char cp, pri;
MPASS(ts != NULL);
mtx_assert(&ts->ts_lock, MA_OWNED);
MPASS(ts->ts_owner == curthread);
tc = TC_LOOKUP(ts->ts_lockobj);
mtx_assert(&tc->tc_lock, MA_OWNED);
MPASS(TAILQ_EMPTY(&ts->ts_pending));
MPASS(!TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]) ||
!TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE]));
@ -947,7 +965,6 @@ turnstile_disown(struct turnstile *ts)
ts->ts_owner = NULL;
LIST_REMOVE(ts, ts_link);
mtx_unlock_spin(&td_contested_lock);
mtx_unlock_spin(&tc->tc_lock);
/*
* Adjust the priority of curthread based on other contested
@ -956,7 +973,8 @@ turnstile_disown(struct turnstile *ts)
*/
td = curthread;
pri = PRI_MAX;
mtx_lock_spin(&sched_lock);
thread_lock(td);
mtx_unlock_spin(&ts->ts_lock);
mtx_lock_spin(&td_contested_lock);
LIST_FOREACH(ts, &td->td_contested, ts_link) {
cp = turnstile_first_waiter(ts)->td_priority;
@ -965,7 +983,7 @@ turnstile_disown(struct turnstile *ts)
}
mtx_unlock_spin(&td_contested_lock);
sched_unlend_prio(td, pri);
mtx_unlock_spin(&sched_lock);
thread_unlock(td);
}
/*
@ -975,12 +993,10 @@ struct thread *
turnstile_head(struct turnstile *ts, int queue)
{
#ifdef INVARIANTS
struct turnstile_chain *tc;
MPASS(ts != NULL);
MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE);
tc = TC_LOOKUP(ts->ts_lockobj);
mtx_assert(&tc->tc_lock, MA_OWNED);
mtx_assert(&ts->ts_lock, MA_OWNED);
#endif
return (TAILQ_FIRST(&ts->ts_blocked[queue]));
}
@ -992,12 +1008,10 @@ int
turnstile_empty(struct turnstile *ts, int queue)
{
#ifdef INVARIANTS
struct turnstile_chain *tc;
MPASS(ts != NULL);
MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE);
tc = TC_LOOKUP(ts->ts_lockobj);
mtx_assert(&tc->tc_lock, MA_OWNED);
mtx_assert(&ts->ts_lock, MA_OWNED);
#endif
return (TAILQ_EMPTY(&ts->ts_blocked[queue]));
}

10
sys/sys/turnstile.h
View File

@ -91,17 +91,19 @@ void init_turnstiles(void);
void turnstile_adjust(struct thread *, u_char);
struct turnstile *turnstile_alloc(void);
void turnstile_broadcast(struct turnstile *, int);
void turnstile_claim(struct lock_object *);
void turnstile_cancel(struct turnstile *);
void turnstile_chain_lock(struct lock_object *);
void turnstile_chain_unlock(struct lock_object *);
void turnstile_claim(struct turnstile *);
void turnstile_disown(struct turnstile *);
int turnstile_empty(struct turnstile *ts, int queue);
void turnstile_free(struct turnstile *);
struct thread *turnstile_head(struct turnstile *, int);
void turnstile_lock(struct lock_object *);
struct turnstile *turnstile_lookup(struct lock_object *);
void turnstile_release(struct lock_object *);
int turnstile_signal(struct turnstile *, int);
struct turnstile *turnstile_trywait(struct lock_object *);
void turnstile_unpend(struct turnstile *, int);
void turnstile_wait(struct lock_object *, struct thread *, int);
void turnstile_wait(struct turnstile *, struct thread *, int);
#endif /* _KERNEL */
#endif /* _SYS_TURNSTILE_H_ */