From 3c2e44364ebaaf0f7f109a41a647609085d51d66 Mon Sep 17 00:00:00 2001 From: Jeff Roberson Date: Mon, 4 Jun 2007 23:54:50 +0000 Subject: [PATCH] Commit 8/14 of sched_lock decomposition. - Use a global umtx spinlock to protect the sleep queues now that there is no global scheduler lock. - Use thread_lock() to protect thread state. Tested by: kris, current@ Tested on: i386, amd64, ULE, 4BSD, libthr, libkse, PREEMPTION, etc. Discussed with: kris, attilio, kmacy, jhb, julian, bde (small parts each) --- sys/kern/kern_umtx.c | 85 +++++++++++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 33 deletions(-) diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c index f46c8fa234ec..2640278a7d16 100644 --- a/sys/kern/kern_umtx.c +++ b/sys/kern/kern_umtx.c @@ -124,8 +124,8 @@ struct umtx_q { /* * Blocked on PI mutex. read can use chain lock - * or sched_lock, write must have both chain lock and - * sched_lock being hold. + * or umtx_lock, write must have both chain lock and + * umtx_lock being hold. */ struct umtx_pi *uq_pi_blocked; @@ -225,6 +225,8 @@ static void umtx_exec_hook(void *arg __unused, struct proc *p __unused, struct image_params *imgp __unused); SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL); +static struct mtx umtx_lock; + static void umtxq_sysinit(void *arg __unused) { @@ -240,6 +242,7 @@ umtxq_sysinit(void *arg __unused) umtxq_chains[i].uc_busy = 0; umtxq_chains[i].uc_waiters = 0; } + mtx_init(&umtx_lock, "umtx lock", NULL, MTX_SPIN); EVENTHANDLER_REGISTER(process_exec, umtx_exec_hook, NULL, EVENTHANDLER_PRI_ANY); } @@ -1270,7 +1273,7 @@ umtx_pi_adjust_thread(struct umtx_pi *pi, struct thread *td) struct umtx_q *uq, *uq1, *uq2; struct thread *td1; - mtx_assert(&sched_lock, MA_OWNED); + mtx_assert(&umtx_lock, MA_OWNED); if (pi == NULL) return (0); @@ -1316,7 +1319,7 @@ umtx_propagate_priority(struct thread *td) struct umtx_pi *pi; int pri; - mtx_assert(&sched_lock, MA_OWNED); + mtx_assert(&umtx_lock, MA_OWNED); pri = UPRI(td); uq = td->td_umtxq; pi = uq->uq_pi_blocked; @@ -1334,7 +1337,9 @@ umtx_propagate_priority(struct thread *td) if (UPRI(td) <= pri) return; + thread_lock(td); sched_lend_user_prio(td, pri); + thread_unlock(td); /* * Pick up the lock that td is blocked on. @@ -1358,7 +1363,7 @@ umtx_unpropagate_priority(struct umtx_pi *pi) struct umtx_pi *pi2; int pri; - mtx_assert(&sched_lock, MA_OWNED); + mtx_assert(&umtx_lock, MA_OWNED); while (pi != NULL && pi->pi_owner != NULL) { pri = PRI_MAX; @@ -1374,7 +1379,9 @@ umtx_unpropagate_priority(struct umtx_pi *pi) if (pri > uq_owner->uq_inherited_pri) pri = uq_owner->uq_inherited_pri; + thread_lock(pi->pi_owner); sched_unlend_user_prio(pi->pi_owner, pri); + thread_unlock(pi->pi_owner); pi = uq_owner->uq_pi_blocked; } } @@ -1388,7 +1395,7 @@ umtx_pi_setowner(struct umtx_pi *pi, struct thread *owner) struct umtx_q *uq_owner; uq_owner = owner->td_umtxq; - mtx_assert(&sched_lock, MA_OWNED); + mtx_assert(&umtx_lock, MA_OWNED); if (pi->pi_owner != NULL) panic("pi_ower != NULL"); pi->pi_owner = owner; @@ -1404,9 +1411,9 @@ umtx_pi_claim(struct umtx_pi *pi, struct thread *owner) struct umtx_q *uq, *uq_owner; uq_owner = owner->td_umtxq; - mtx_lock_spin(&sched_lock); + mtx_lock_spin(&umtx_lock); if (pi->pi_owner == owner) { - mtx_unlock_spin(&sched_lock); + mtx_unlock_spin(&umtx_lock); return (0); } @@ -1414,7 +1421,7 @@ umtx_pi_claim(struct umtx_pi *pi, struct thread *owner) /* * userland may have already messed the mutex, sigh. */ - mtx_unlock_spin(&sched_lock); + mtx_unlock_spin(&umtx_lock); return (EPERM); } umtx_pi_setowner(pi, owner); @@ -1423,10 +1430,12 @@ umtx_pi_claim(struct umtx_pi *pi, struct thread *owner) int pri; pri = UPRI(uq->uq_thread); + thread_lock(owner); if (pri < UPRI(owner)) sched_lend_user_prio(owner, pri); + thread_unlock(owner); } - mtx_unlock_spin(&sched_lock); + mtx_unlock_spin(&umtx_lock); return (0); } @@ -1442,7 +1451,7 @@ umtx_pi_adjust(struct thread *td, u_char oldpri) uq = td->td_umtxq; - mtx_assert(&sched_lock, MA_OWNED); + mtx_assert(&umtx_lock, MA_OWNED); MPASS(TD_ON_UPILOCK(td)); /* @@ -1493,14 +1502,14 @@ umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, */ PROC_LOCK(curproc); td1 = thread_find(curproc, owner); - mtx_lock_spin(&sched_lock); + mtx_lock_spin(&umtx_lock); if (td1 != NULL && pi->pi_owner == NULL) { uq1 = td1->td_umtxq; umtx_pi_setowner(pi, td1); } PROC_UNLOCK(curproc); } else { - mtx_lock_spin(&sched_lock); + mtx_lock_spin(&umtx_lock); } TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) { @@ -1516,12 +1525,12 @@ umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, uq->uq_pi_blocked = pi; td->td_flags |= TDF_UPIBLOCKED; - mtx_unlock_spin(&sched_lock); + mtx_unlock_spin(&umtx_lock); umtxq_unlock(&uq->uq_key); - mtx_lock_spin(&sched_lock); + mtx_lock_spin(&umtx_lock); umtx_propagate_priority(td); - mtx_unlock_spin(&sched_lock); + mtx_unlock_spin(&umtx_lock); umtxq_lock(&uq->uq_key); if (uq->uq_flags & UQF_UMTXQ) { @@ -1536,12 +1545,12 @@ umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, } umtxq_unlock(&uq->uq_key); - mtx_lock_spin(&sched_lock); + mtx_lock_spin(&umtx_lock); uq->uq_pi_blocked = NULL; td->td_flags &= ~TDF_UPIBLOCKED; TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq); umtx_unpropagate_priority(pi); - mtx_unlock_spin(&sched_lock); + mtx_unlock_spin(&umtx_lock); umtxq_lock(&uq->uq_key); @@ -1575,7 +1584,7 @@ umtx_pi_unref(struct umtx_pi *pi) UMTXQ_LOCKED_ASSERT(uc); KASSERT(pi->pi_refcount > 0, ("invalid reference count")); if (--pi->pi_refcount == 0) { - mtx_lock_spin(&sched_lock); + mtx_lock_spin(&umtx_lock); if (pi->pi_owner != NULL) { TAILQ_REMOVE(&pi->pi_owner->td_umtxq->uq_pi_contested, pi, pi_link); @@ -1583,7 +1592,7 @@ umtx_pi_unref(struct umtx_pi *pi) } KASSERT(TAILQ_EMPTY(&pi->pi_blocked), ("blocked queue not empty")); - mtx_unlock_spin(&sched_lock); + mtx_unlock_spin(&umtx_lock); TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink); free = 1; } @@ -1822,7 +1831,7 @@ do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags) return (EPERM); } uq_me = curthread->td_umtxq; - mtx_lock_spin(&sched_lock); + mtx_lock_spin(&umtx_lock); pi->pi_owner = NULL; TAILQ_REMOVE(&uq_me->uq_pi_contested, pi, pi_link); uq_first = TAILQ_FIRST(&pi->pi_blocked); @@ -1834,8 +1843,10 @@ do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags) pri = UPRI(uq_first2->uq_thread); } } + thread_lock(curthread); sched_unlend_user_prio(curthread, pri); - mtx_unlock_spin(&sched_lock); + thread_unlock(curthread); + mtx_unlock_spin(&umtx_lock); } umtxq_unlock(&key); @@ -1891,18 +1902,20 @@ _do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, int timo, goto out; } - mtx_lock_spin(&sched_lock); + mtx_lock_spin(&umtx_lock); if (UPRI(td) < PRI_MIN_REALTIME + ceiling) { - mtx_unlock_spin(&sched_lock); + mtx_unlock_spin(&umtx_lock); error = EINVAL; goto out; } if (su && PRI_MIN_REALTIME + ceiling < uq->uq_inherited_pri) { uq->uq_inherited_pri = PRI_MIN_REALTIME + ceiling; + thread_lock(td); if (uq->uq_inherited_pri < UPRI(td)) sched_lend_user_prio(td, uq->uq_inherited_pri); + thread_unlock(td); } - mtx_unlock_spin(&sched_lock); + mtx_unlock_spin(&umtx_lock); owner = casuword32(&m->m_owner, UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); @@ -1943,7 +1956,7 @@ _do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, int timo, umtxq_remove(uq); umtxq_unlock(&uq->uq_key); - mtx_lock_spin(&sched_lock); + mtx_lock_spin(&umtx_lock); uq->uq_inherited_pri = old_inherited_pri; pri = PRI_MAX; TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { @@ -1955,12 +1968,14 @@ _do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, int timo, } if (pri > uq->uq_inherited_pri) pri = uq->uq_inherited_pri; + thread_lock(td); sched_unlend_user_prio(td, pri); - mtx_unlock_spin(&sched_lock); + thread_unlock(td); + mtx_unlock_spin(&umtx_lock); } if (error != 0) { - mtx_lock_spin(&sched_lock); + mtx_lock_spin(&umtx_lock); uq->uq_inherited_pri = old_inherited_pri; pri = PRI_MAX; TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { @@ -1972,8 +1987,10 @@ _do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, int timo, } if (pri > uq->uq_inherited_pri) pri = uq->uq_inherited_pri; + thread_lock(td); sched_unlend_user_prio(td, pri); - mtx_unlock_spin(&sched_lock); + thread_unlock(td); + mtx_unlock_spin(&umtx_lock); } out: @@ -2048,7 +2065,7 @@ do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags) if (error == -1) error = EFAULT; else { - mtx_lock_spin(&sched_lock); + mtx_lock_spin(&umtx_lock); if (su != 0) uq->uq_inherited_pri = new_inherited_pri; pri = PRI_MAX; @@ -2061,8 +2078,10 @@ do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags) } if (pri > uq->uq_inherited_pri) pri = uq->uq_inherited_pri; + thread_lock(td); sched_unlend_user_prio(td, pri); - mtx_unlock_spin(&sched_lock); + thread_unlock(td); + mtx_unlock_spin(&umtx_lock); } umtx_key_release(&key); return (error); @@ -2748,12 +2767,12 @@ umtx_thread_cleanup(struct thread *td) if ((uq = td->td_umtxq) == NULL) return; - mtx_lock_spin(&sched_lock); + mtx_lock_spin(&umtx_lock); uq->uq_inherited_pri = PRI_MAX; while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) { pi->pi_owner = NULL; TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link); } td->td_flags &= ~TDF_UBORROWING; - mtx_unlock_spin(&sched_lock); + mtx_unlock_spin(&umtx_lock); }