diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c index c815e3616356..58e76bc90b64 100644 --- a/sys/kern/kern_umtx.c +++ b/sys/kern/kern_umtx.c @@ -510,6 +510,15 @@ umtxq_unbusy(struct umtx_key *key) wakeup_one(uc); } +static inline void +umtxq_unbusy_unlocked(struct umtx_key *key) +{ + + umtxq_lock(key); + umtxq_unbusy(key); + umtxq_unlock(key); +} + static struct umtxq_queue * umtxq_queue_lookup(struct umtx_key *key, int q) { @@ -847,6 +856,7 @@ do_wait(struct thread *td, void *addr, u_long id, struct abs_timeout timo; struct umtx_q *uq; u_long tmp; + uint32_t tmp32; int error = 0; uq = td->td_umtxq; @@ -860,18 +870,29 @@ do_wait(struct thread *td, void *addr, u_long id, umtxq_lock(&uq->uq_key); umtxq_insert(uq); umtxq_unlock(&uq->uq_key); - if (compat32 == 0) - tmp = fuword(addr); - else - tmp = (unsigned int)fuword32(addr); + if (compat32 == 0) { + error = fueword(addr, &tmp); + if (error != 0) + error = EFAULT; + } else { + error = fueword32(addr, &tmp32); + if (error == 0) + tmp = tmp32; + else + error = EFAULT; + } umtxq_lock(&uq->uq_key); - if (tmp == id) - error = umtxq_sleep(uq, "uwait", timeout == NULL ? - NULL : &timo); - if ((uq->uq_flags & UQF_UMTXQ) == 0) - error = 0; - else + if (error == 0) { + if (tmp == id) + error = umtxq_sleep(uq, "uwait", timeout == NULL ? + NULL : &timo); + if ((uq->uq_flags & UQF_UMTXQ) == 0) + error = 0; + else + umtxq_remove(uq); + } else if ((uq->uq_flags & UQF_UMTXQ) != 0) { umtxq_remove(uq); + } umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); if (error == ERESTART) @@ -908,11 +929,11 @@ do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, struct abs_timeout timo; struct umtx_q *uq; uint32_t owner, old, id; - int error = 0; + int error, rv; id = td->td_tid; uq = td->td_umtxq; - + error = 0; if (timeout != NULL) abs_timeout_init2(&timo, timeout); @@ -921,7 +942,9 @@ do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, * can fault on any access. */ for (;;) { - owner = fuword32(__DEVOLATILE(void *, &m->m_owner)); + rv = fueword32(__DEVOLATILE(void *, &m->m_owner), &owner); + if (rv == -1) + return (EFAULT); if (mode == _UMUTEX_WAIT) { if (owner == UMUTEX_UNOWNED || owner == UMUTEX_CONTESTED) return (0); @@ -929,31 +952,31 @@ do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, /* * Try the uncontested case. This should be done in userland. */ - owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id); + rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, + &owner, id); + /* The address was invalid. */ + if (rv == -1) + return (EFAULT); /* The acquire succeeded. */ if (owner == UMUTEX_UNOWNED) return (0); - /* The address was invalid. */ - if (owner == -1) - return (EFAULT); - /* If no one owns it but it is contested try to acquire it. */ if (owner == UMUTEX_CONTESTED) { - owner = casuword32(&m->m_owner, - UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); + rv = casueword32(&m->m_owner, + UMUTEX_CONTESTED, &owner, + id | UMUTEX_CONTESTED); + /* The address was invalid. */ + if (rv == -1) + return (EFAULT); if (owner == UMUTEX_CONTESTED) return (0); - /* The address was invalid. */ - if (owner == -1) - return (EFAULT); - - error = umtxq_check_susp(td); - if (error != 0) - return (error); + rv = umtxq_check_susp(td); + if (rv != 0) + return (rv); /* If this failed the lock has changed, restart. */ continue; @@ -985,10 +1008,11 @@ do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, * either some one else has acquired the lock or it has been * released. */ - old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED); + rv = casueword32(&m->m_owner, owner, &old, + owner | UMUTEX_CONTESTED); /* The address was invalid. */ - if (old == -1) { + if (rv == -1) { umtxq_lock(&uq->uq_key); umtxq_remove(uq); umtxq_unbusy(&uq->uq_key); @@ -1033,16 +1057,16 @@ do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags) /* * Make sure we own this mtx. */ - owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); - if (owner == -1) + error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner), &owner); + if (error == -1) return (EFAULT); if ((owner & ~UMUTEX_CONTESTED) != id) return (EPERM); if ((owner & UMUTEX_CONTESTED) == 0) { - old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED); - if (old == -1) + error = casueword32(&m->m_owner, owner, &old, UMUTEX_UNOWNED); + if (error == -1) return (EFAULT); if (old == owner) return (0); @@ -1064,14 +1088,14 @@ do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags) * there is zero or one thread only waiting for it. * Otherwise, it must be marked as contested. */ - old = casuword32(&m->m_owner, owner, - count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); + error = casueword32(&m->m_owner, owner, &old, + count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); umtxq_lock(&key); umtxq_signal(&key,1); umtxq_unbusy(&key); umtxq_unlock(&key); umtx_key_release(&key); - if (old == -1) + if (error == -1) return (EFAULT); if (old != owner) return (EINVAL); @@ -1091,14 +1115,16 @@ do_wake_umutex(struct thread *td, struct umutex *m) int error; int count; - owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); - if (owner == -1) + error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner), &owner); + if (error == -1) return (EFAULT); if ((owner & ~UMUTEX_CONTESTED) != 0) return (0); - flags = fuword32(&m->m_flags); + error = fueword32(&m->m_flags, &flags); + if (error == -1) + return (EFAULT); /* We should only ever be in here for contested locks */ if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), @@ -1110,16 +1136,20 @@ do_wake_umutex(struct thread *td, struct umutex *m) count = umtxq_count(&key); umtxq_unlock(&key); - if (count <= 1) - owner = casuword32(&m->m_owner, UMUTEX_CONTESTED, UMUTEX_UNOWNED); + if (count <= 1) { + error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, + UMUTEX_UNOWNED); + if (error == -1) + error = EFAULT; + } umtxq_lock(&key); - if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0) + if (error == 0 && count != 0 && (owner & ~UMUTEX_CONTESTED) == 0) umtxq_signal(&key, 1); umtxq_unbusy(&key); umtxq_unlock(&key); umtx_key_release(&key); - return (0); + return (error); } /* @@ -1162,41 +1192,49 @@ do_wake2_umutex(struct thread *td, struct umutex *m, uint32_t flags) * any memory. */ if (count > 1) { - owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); - while ((owner & UMUTEX_CONTESTED) ==0) { - old = casuword32(&m->m_owner, owner, - owner|UMUTEX_CONTESTED); + error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner), + &owner); + if (error == -1) + error = EFAULT; + while (error == 0 && (owner & UMUTEX_CONTESTED) == 0) { + error = casueword32(&m->m_owner, owner, &old, + owner | UMUTEX_CONTESTED); + if (error == -1) { + error = EFAULT; + break; + } if (old == owner) break; owner = old; - if (old == -1) - break; error = umtxq_check_susp(td); if (error != 0) break; } } else if (count == 1) { - owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); - while ((owner & ~UMUTEX_CONTESTED) != 0 && + error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner), + &owner); + if (error == -1) + error = EFAULT; + while (error == 0 && (owner & ~UMUTEX_CONTESTED) != 0 && (owner & UMUTEX_CONTESTED) == 0) { - old = casuword32(&m->m_owner, owner, - owner|UMUTEX_CONTESTED); + error = casueword32(&m->m_owner, owner, &old, + owner | UMUTEX_CONTESTED); + if (error == -1) { + error = EFAULT; + break; + } if (old == owner) break; owner = old; - if (old == -1) - break; error = umtxq_check_susp(td); if (error != 0) break; } } umtxq_lock(&key); - if (owner == -1) { - error = EFAULT; + if (error == EFAULT) { umtxq_signal(&key, INT_MAX); - } - else if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0) + } else if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0) umtxq_signal(&key, 1); umtxq_unbusy(&key); umtxq_unlock(&key); @@ -1576,7 +1614,7 @@ do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, struct umtx_q *uq; struct umtx_pi *pi, *new_pi; uint32_t id, owner, old; - int error; + int error, rv; id = td->td_tid; uq = td->td_umtxq; @@ -1619,7 +1657,12 @@ do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, /* * Try the uncontested case. This should be done in userland. */ - owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id); + rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, &owner, id); + /* The address was invalid. */ + if (rv == -1) { + error = EFAULT; + break; + } /* The acquire succeeded. */ if (owner == UMUTEX_UNOWNED) { @@ -1627,16 +1670,15 @@ do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, break; } - /* The address was invalid. */ - if (owner == -1) { - error = EFAULT; - break; - } - /* If no one owns it but it is contested try to acquire it. */ if (owner == UMUTEX_CONTESTED) { - owner = casuword32(&m->m_owner, - UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); + rv = casueword32(&m->m_owner, + UMUTEX_CONTESTED, &owner, id | UMUTEX_CONTESTED); + /* The address was invalid. */ + if (rv == -1) { + error = EFAULT; + break; + } if (owner == UMUTEX_CONTESTED) { umtxq_lock(&uq->uq_key); @@ -1647,12 +1689,6 @@ do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, break; } - /* The address was invalid. */ - if (owner == -1) { - error = EFAULT; - break; - } - error = umtxq_check_susp(td); if (error != 0) break; @@ -1683,13 +1719,12 @@ do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, * either some one else has acquired the lock or it has been * released. */ - old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED); + rv = casueword32(&m->m_owner, owner, &old, + owner | UMUTEX_CONTESTED); /* The address was invalid. */ - if (old == -1) { - umtxq_lock(&uq->uq_key); - umtxq_unbusy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + if (rv == -1) { + umtxq_unbusy_unlocked(&uq->uq_key); error = EFAULT; break; } @@ -1741,8 +1776,8 @@ do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags) /* * Make sure we own this mtx. */ - owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); - if (owner == -1) + error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner), &owner); + if (error == -1) return (EFAULT); if ((owner & ~UMUTEX_CONTESTED) != id) @@ -1750,8 +1785,8 @@ do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags) /* This should be done in userland */ if ((owner & UMUTEX_CONTESTED) == 0) { - old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED); - if (old == -1) + error = casueword32(&m->m_owner, owner, &old, UMUTEX_UNOWNED); + if (error == -1) return (EFAULT); if (old == owner) return (0); @@ -1809,14 +1844,12 @@ do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags) * there is zero or one thread only waiting for it. * Otherwise, it must be marked as contested. */ - old = casuword32(&m->m_owner, owner, - count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); + error = casueword32(&m->m_owner, owner, &old, + count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); - umtxq_lock(&key); - umtxq_unbusy(&key); - umtxq_unlock(&key); + umtxq_unbusy_unlocked(&key); umtx_key_release(&key); - if (old == -1) + if (error == -1) return (EFAULT); if (old != owner) return (EINVAL); @@ -1835,7 +1868,7 @@ do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, struct umtx_pi *pi; uint32_t ceiling; uint32_t owner, id; - int error, pri, old_inherited_pri, su; + int error, pri, old_inherited_pri, su, rv; id = td->td_tid; uq = td->td_umtxq; @@ -1853,7 +1886,12 @@ do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, umtxq_busy(&uq->uq_key); umtxq_unlock(&uq->uq_key); - ceiling = RTP_PRIO_MAX - fuword32(&m->m_ceilings[0]); + rv = fueword32(&m->m_ceilings[0], &ceiling); + if (rv == -1) { + error = EFAULT; + goto out; + } + ceiling = RTP_PRIO_MAX - ceiling; if (ceiling > RTP_PRIO_MAX) { error = EINVAL; goto out; @@ -1874,17 +1912,16 @@ do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, } mtx_unlock_spin(&umtx_lock); - owner = casuword32(&m->m_owner, - UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); - - if (owner == UMUTEX_CONTESTED) { - error = 0; + rv = casueword32(&m->m_owner, + UMUTEX_CONTESTED, &owner, id | UMUTEX_CONTESTED); + /* The address was invalid. */ + if (rv == -1) { + error = EFAULT; break; } - /* The address was invalid. */ - if (owner == -1) { - error = EFAULT; + if (owner == UMUTEX_CONTESTED) { + error = 0; break; } @@ -1946,9 +1983,7 @@ do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, } out: - umtxq_lock(&uq->uq_key); - umtxq_unbusy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_unbusy_unlocked(&uq->uq_key); umtx_key_release(&uq->uq_key); return (error); } @@ -1973,8 +2008,8 @@ do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags) /* * Make sure we own this mtx. */ - owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); - if (owner == -1) + error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner), &owner); + if (error == -1) return (EFAULT); if ((owner & ~UMUTEX_CONTESTED) != id) @@ -2047,9 +2082,11 @@ do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling, uint32_t save_ceiling; uint32_t owner, id; uint32_t flags; - int error; + int error, rv; - flags = fuword32(&m->m_flags); + error = fueword32(&m->m_flags, &flags); + if (error == -1) + return (EFAULT); if ((flags & UMUTEX_PRIO_PROTECT) == 0) return (EINVAL); if (ceiling > RTP_PRIO_MAX) @@ -2064,10 +2101,18 @@ do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling, umtxq_busy(&uq->uq_key); umtxq_unlock(&uq->uq_key); - save_ceiling = fuword32(&m->m_ceilings[0]); + rv = fueword32(&m->m_ceilings[0], &save_ceiling); + if (rv == -1) { + error = EFAULT; + break; + } - owner = casuword32(&m->m_owner, - UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); + rv = casueword32(&m->m_owner, + UMUTEX_CONTESTED, &owner, id | UMUTEX_CONTESTED); + if (rv == -1) { + error = EFAULT; + break; + } if (owner == UMUTEX_CONTESTED) { suword32(&m->m_ceilings[0], ceiling); @@ -2077,12 +2122,6 @@ do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling, break; } - /* The address was invalid. */ - if (owner == -1) { - error = EFAULT; - break; - } - if ((owner & ~UMUTEX_CONTESTED) == id) { suword32(&m->m_ceilings[0], ceiling); error = 0; @@ -2129,8 +2168,8 @@ do_lock_umutex(struct thread *td, struct umutex *m, uint32_t flags; int error; - flags = fuword32(&m->m_flags); - if (flags == -1) + error = fueword32(&m->m_flags, &flags); + if (error == -1) return (EFAULT); switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { @@ -2164,9 +2203,10 @@ static int do_unlock_umutex(struct thread *td, struct umutex *m) { uint32_t flags; + int error; - flags = fuword32(&m->m_flags); - if (flags == -1) + error = fueword32(&m->m_flags, &flags); + if (error == -1) return (EFAULT); switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { @@ -2187,21 +2227,27 @@ do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m, { struct abs_timeout timo; struct umtx_q *uq; - uint32_t flags; - uint32_t clockid; + uint32_t flags, clockid, hasw; int error; uq = td->td_umtxq; - flags = fuword32(&cv->c_flags); + error = fueword32(&cv->c_flags, &flags); + if (error == -1) + return (EFAULT); error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key); if (error != 0) return (error); if ((wflags & CVWAIT_CLOCKID) != 0) { - clockid = fuword32(&cv->c_clockid); + error = fueword32(&cv->c_clockid, &clockid); + if (error == -1) { + umtx_key_release(&uq->uq_key); + return (EFAULT); + } if (clockid < CLOCK_REALTIME || clockid >= CLOCK_THREAD_CPUTIME_ID) { /* hmm, only HW clock id will work. */ + umtx_key_release(&uq->uq_key); return (EINVAL); } } else { @@ -2217,12 +2263,12 @@ do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m, * Set c_has_waiters to 1 before releasing user mutex, also * don't modify cache line when unnecessary. */ - if (fuword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters)) == 0) + error = fueword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), + &hasw); + if (error == 0 && hasw == 0) suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 1); - umtxq_lock(&uq->uq_key); - umtxq_unbusy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_unbusy_unlocked(&uq->uq_key); error = do_unlock_umutex(td, m); @@ -2276,7 +2322,9 @@ do_cv_signal(struct thread *td, struct ucond *cv) int error, cnt, nwake; uint32_t flags; - flags = fuword32(&cv->c_flags); + error = fueword32(&cv->c_flags, &flags); + if (error == -1) + return (EFAULT); if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0) return (error); umtxq_lock(&key); @@ -2287,6 +2335,8 @@ do_cv_signal(struct thread *td, struct ucond *cv) umtxq_unlock(&key); error = suword32( __DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0); + if (error == -1) + error = EFAULT; umtxq_lock(&key); } umtxq_unbusy(&key); @@ -2302,7 +2352,9 @@ do_cv_broadcast(struct thread *td, struct ucond *cv) int error; uint32_t flags; - flags = fuword32(&cv->c_flags); + error = fueword32(&cv->c_flags, &flags); + if (error == -1) + return (EFAULT); if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0) return (error); @@ -2312,10 +2364,10 @@ do_cv_broadcast(struct thread *td, struct ucond *cv) umtxq_unlock(&key); error = suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0); + if (error == -1) + error = EFAULT; - umtxq_lock(&key); - umtxq_unbusy(&key); - umtxq_unlock(&key); + umtxq_unbusy_unlocked(&key); umtx_key_release(&key); return (error); @@ -2329,10 +2381,12 @@ do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, struct _umtx uint32_t flags, wrflags; int32_t state, oldstate; int32_t blocked_readers; - int error; + int error, rv; uq = td->td_umtxq; - flags = fuword32(&rwlock->rw_flags); + error = fueword32(&rwlock->rw_flags, &flags); + if (error == -1) + return (EFAULT); error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); if (error != 0) return (error); @@ -2345,15 +2399,22 @@ do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, struct _umtx wrflags |= URWLOCK_WRITE_WAITERS; for (;;) { - state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); + rv = fueword32(__DEVOLATILE(int32_t *, &rwlock->rw_state), + &state); + if (rv == -1) { + umtx_key_release(&uq->uq_key); + return (EFAULT); + } + /* try to lock it */ while (!(state & wrflags)) { if (__predict_false(URWLOCK_READER_COUNT(state) == URWLOCK_MAX_READERS)) { umtx_key_release(&uq->uq_key); return (EAGAIN); } - oldstate = casuword32(&rwlock->rw_state, state, state + 1); - if (oldstate == -1) { + rv = casueword32(&rwlock->rw_state, state, + &oldstate, state + 1); + if (rv == -1) { umtx_key_release(&uq->uq_key); return (EFAULT); } @@ -2379,12 +2440,17 @@ do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, struct _umtx * re-read the state, in case it changed between the try-lock above * and the check below */ - state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); + rv = fueword32(__DEVOLATILE(int32_t *, &rwlock->rw_state), + &state); + if (rv == -1) + error = EFAULT; /* set read contention bit */ - while ((state & wrflags) && !(state & URWLOCK_READ_WAITERS)) { - oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_READ_WAITERS); - if (oldstate == -1) { + while (error == 0 && (state & wrflags) && + !(state & URWLOCK_READ_WAITERS)) { + rv = casueword32(&rwlock->rw_state, state, + &oldstate, state | URWLOCK_READ_WAITERS); + if (rv == -1) { error = EFAULT; break; } @@ -2396,17 +2462,13 @@ do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, struct _umtx break; } if (error != 0) { - umtxq_lock(&uq->uq_key); - umtxq_unbusy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_unbusy_unlocked(&uq->uq_key); break; } /* state is changed while setting flags, restart */ if (!(state & wrflags)) { - umtxq_lock(&uq->uq_key); - umtxq_unbusy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_unbusy_unlocked(&uq->uq_key); error = umtxq_check_susp(td); if (error != 0) break; @@ -2415,7 +2477,13 @@ do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, struct _umtx sleep: /* contention bit is set, before sleeping, increase read waiter count */ - blocked_readers = fuword32(&rwlock->rw_blocked_readers); + rv = fueword32(&rwlock->rw_blocked_readers, + &blocked_readers); + if (rv == -1) { + umtxq_unbusy_unlocked(&uq->uq_key); + error = EFAULT; + break; + } suword32(&rwlock->rw_blocked_readers, blocked_readers+1); while (state & wrflags) { @@ -2431,18 +2499,32 @@ do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, struct _umtx umtxq_unlock(&uq->uq_key); if (error) break; - state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); + rv = fueword32(__DEVOLATILE(int32_t *, + &rwlock->rw_state), &state); + if (rv == -1) { + error = EFAULT; + break; + } } /* decrease read waiter count, and may clear read contention bit */ - blocked_readers = fuword32(&rwlock->rw_blocked_readers); + rv = fueword32(&rwlock->rw_blocked_readers, + &blocked_readers); + if (rv == -1) { + umtxq_unbusy_unlocked(&uq->uq_key); + error = EFAULT; + break; + } suword32(&rwlock->rw_blocked_readers, blocked_readers-1); if (blocked_readers == 1) { - state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); - for (;;) { - oldstate = casuword32(&rwlock->rw_state, state, - state & ~URWLOCK_READ_WAITERS); - if (oldstate == -1) { + rv = fueword32(__DEVOLATILE(int32_t *, + &rwlock->rw_state), &state); + if (rv == -1) + error = EFAULT; + while (error == 0) { + rv = casueword32(&rwlock->rw_state, state, + &oldstate, state & ~URWLOCK_READ_WAITERS); + if (rv == -1) { error = EFAULT; break; } @@ -2450,14 +2532,10 @@ do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, struct _umtx break; state = oldstate; error = umtxq_check_susp(td); - if (error != 0) - break; } } - umtxq_lock(&uq->uq_key); - umtxq_unbusy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_unbusy_unlocked(&uq->uq_key); if (error != 0) break; } @@ -2476,10 +2554,12 @@ do_rw_wrlock(struct thread *td, struct urwlock *rwlock, struct _umtx_time *timeo int32_t state, oldstate; int32_t blocked_writers; int32_t blocked_readers; - int error; + int error, rv; uq = td->td_umtxq; - flags = fuword32(&rwlock->rw_flags); + error = fueword32(&rwlock->rw_flags, &flags); + if (error == -1) + return (EFAULT); error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); if (error != 0) return (error); @@ -2489,10 +2569,16 @@ do_rw_wrlock(struct thread *td, struct urwlock *rwlock, struct _umtx_time *timeo blocked_readers = 0; for (;;) { - state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); + rv = fueword32(__DEVOLATILE(int32_t *, &rwlock->rw_state), + &state); + if (rv == -1) { + umtx_key_release(&uq->uq_key); + return (EFAULT); + } while (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) { - oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_WRITE_OWNER); - if (oldstate == -1) { + rv = casueword32(&rwlock->rw_state, state, + &oldstate, state | URWLOCK_WRITE_OWNER); + if (rv == -1) { umtx_key_release(&uq->uq_key); return (EFAULT); } @@ -2528,12 +2614,17 @@ do_rw_wrlock(struct thread *td, struct urwlock *rwlock, struct _umtx_time *timeo * re-read the state, in case it changed between the try-lock above * and the check below */ - state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); + rv = fueword32(__DEVOLATILE(int32_t *, &rwlock->rw_state), + &state); + if (rv == -1) + error = EFAULT; - while (((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) && - (state & URWLOCK_WRITE_WAITERS) == 0) { - oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_WRITE_WAITERS); - if (oldstate == -1) { + while (error == 0 && ((state & URWLOCK_WRITE_OWNER) || + URWLOCK_READER_COUNT(state) != 0) && + (state & URWLOCK_WRITE_WAITERS) == 0) { + rv = casueword32(&rwlock->rw_state, state, + &oldstate, state | URWLOCK_WRITE_WAITERS); + if (rv == -1) { error = EFAULT; break; } @@ -2545,23 +2636,25 @@ do_rw_wrlock(struct thread *td, struct urwlock *rwlock, struct _umtx_time *timeo break; } if (error != 0) { - umtxq_lock(&uq->uq_key); - umtxq_unbusy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_unbusy_unlocked(&uq->uq_key); break; } if (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) { - umtxq_lock(&uq->uq_key); - umtxq_unbusy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_unbusy_unlocked(&uq->uq_key); error = umtxq_check_susp(td); if (error != 0) break; continue; } sleep: - blocked_writers = fuword32(&rwlock->rw_blocked_writers); + rv = fueword32(&rwlock->rw_blocked_writers, + &blocked_writers); + if (rv == -1) { + umtxq_unbusy_unlocked(&uq->uq_key); + error = EFAULT; + break; + } suword32(&rwlock->rw_blocked_writers, blocked_writers+1); while ((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) { @@ -2577,17 +2670,34 @@ do_rw_wrlock(struct thread *td, struct urwlock *rwlock, struct _umtx_time *timeo umtxq_unlock(&uq->uq_key); if (error) break; - state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); + rv = fueword32(__DEVOLATILE(int32_t *, + &rwlock->rw_state), &state); + if (rv == -1) { + error = EFAULT; + break; + } } - blocked_writers = fuword32(&rwlock->rw_blocked_writers); + rv = fueword32(&rwlock->rw_blocked_writers, + &blocked_writers); + if (rv == -1) { + umtxq_unbusy_unlocked(&uq->uq_key); + error = EFAULT; + break; + } suword32(&rwlock->rw_blocked_writers, blocked_writers-1); if (blocked_writers == 1) { - state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); + rv = fueword32(__DEVOLATILE(int32_t *, + &rwlock->rw_state), &state); + if (rv == -1) { + umtxq_unbusy_unlocked(&uq->uq_key); + error = EFAULT; + break; + } for (;;) { - oldstate = casuword32(&rwlock->rw_state, state, - state & ~URWLOCK_WRITE_WAITERS); - if (oldstate == -1) { + rv = casueword32(&rwlock->rw_state, state, + &oldstate, state & ~URWLOCK_WRITE_WAITERS); + if (rv == -1) { error = EFAULT; break; } @@ -2603,13 +2713,17 @@ do_rw_wrlock(struct thread *td, struct urwlock *rwlock, struct _umtx_time *timeo if (error != 0) break; } - blocked_readers = fuword32(&rwlock->rw_blocked_readers); + rv = fueword32(&rwlock->rw_blocked_readers, + &blocked_readers); + if (rv == -1) { + umtxq_unbusy_unlocked(&uq->uq_key); + error = EFAULT; + break; + } } else blocked_readers = 0; - umtxq_lock(&uq->uq_key); - umtxq_unbusy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_unbusy_unlocked(&uq->uq_key); } umtx_key_release(&uq->uq_key); @@ -2624,20 +2738,26 @@ do_rw_unlock(struct thread *td, struct urwlock *rwlock) struct umtx_q *uq; uint32_t flags; int32_t state, oldstate; - int error, q, count; + int error, rv, q, count; uq = td->td_umtxq; - flags = fuword32(&rwlock->rw_flags); + error = fueword32(&rwlock->rw_flags, &flags); + if (error == -1) + return (EFAULT); error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); if (error != 0) return (error); - state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); + error = fueword32(__DEVOLATILE(int32_t *, &rwlock->rw_state), &state); + if (error == -1) { + error = EFAULT; + goto out; + } if (state & URWLOCK_WRITE_OWNER) { for (;;) { - oldstate = casuword32(&rwlock->rw_state, state, - state & ~URWLOCK_WRITE_OWNER); - if (oldstate == -1) { + rv = casueword32(&rwlock->rw_state, state, + &oldstate, state & ~URWLOCK_WRITE_OWNER); + if (rv == -1) { error = EFAULT; goto out; } @@ -2655,9 +2775,9 @@ do_rw_unlock(struct thread *td, struct urwlock *rwlock) } } else if (URWLOCK_READER_COUNT(state) != 0) { for (;;) { - oldstate = casuword32(&rwlock->rw_state, state, - state - 1); - if (oldstate == -1) { + rv = casueword32(&rwlock->rw_state, state, + &oldstate, state - 1); + if (rv == -1) { error = EFAULT; goto out; } @@ -2716,11 +2836,13 @@ do_sem_wait(struct thread *td, struct _usem *sem, struct _umtx_time *timeout) { struct abs_timeout timo; struct umtx_q *uq; - uint32_t flags, count; - int error; + uint32_t flags, count, count1; + int error, rv; uq = td->td_umtxq; - flags = fuword32(&sem->_flags); + error = fueword32(&sem->_flags, &flags); + if (error == -1) + return (EFAULT); error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key); if (error != 0) return (error); @@ -2732,15 +2854,16 @@ do_sem_wait(struct thread *td, struct _usem *sem, struct _umtx_time *timeout) umtxq_busy(&uq->uq_key); umtxq_insert(uq); umtxq_unlock(&uq->uq_key); - casuword32(&sem->_has_waiters, 0, 1); - count = fuword32(__DEVOLATILE(uint32_t *, &sem->_count)); - if (count != 0) { + rv = casueword32(&sem->_has_waiters, 0, &count1, 1); + if (rv == 0) + rv = fueword32(__DEVOLATILE(uint32_t *, &sem->_count), &count); + if (rv == -1 || count != 0) { umtxq_lock(&uq->uq_key); umtxq_unbusy(&uq->uq_key); umtxq_remove(uq); umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); - return (0); + return (rv == -1 ? EFAULT : 0); } umtxq_lock(&uq->uq_key); umtxq_unbusy(&uq->uq_key); @@ -2771,7 +2894,9 @@ do_sem_wake(struct thread *td, struct _usem *sem) int error, cnt; uint32_t flags; - flags = fuword32(&sem->_flags); + error = fueword32(&sem->_flags, &flags); + if (error == -1) + return (EFAULT); if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0) return (error); umtxq_lock(&key); @@ -2789,6 +2914,8 @@ do_sem_wake(struct thread *td, struct _usem *sem) error = suword32( __DEVOLATILE(uint32_t *, &sem->_has_waiters), 0); umtxq_lock(&key); + if (error == -1) + error = EFAULT; } } umtxq_unbusy(&key); @@ -2804,7 +2931,7 @@ do_sem2_wait(struct thread *td, struct _usem2 *sem, struct _umtx_time *timeout) struct abs_timeout timo; struct umtx_q *uq; uint32_t count, flags; - int error; + int error, rv; uq = td->td_umtxq; flags = fuword32(&sem->_flags); @@ -2819,8 +2946,8 @@ do_sem2_wait(struct thread *td, struct _usem2 *sem, struct _umtx_time *timeout) umtxq_busy(&uq->uq_key); umtxq_insert(uq); umtxq_unlock(&uq->uq_key); - count = fuword32(__DEVOLATILE(uint32_t *, &sem->_count)); - if (count == -1) { + rv = fueword32(__DEVOLATILE(uint32_t *, &sem->_count), &count); + if (rv == -1) { umtxq_lock(&uq->uq_key); umtxq_unbusy(&uq->uq_key); umtxq_remove(uq); @@ -2839,8 +2966,8 @@ do_sem2_wait(struct thread *td, struct _usem2 *sem, struct _umtx_time *timeout) } if (count == USEM_HAS_WAITERS) break; - count = casuword32(&sem->_count, 0, USEM_HAS_WAITERS); - if (count == -1) { + rv = casueword32(&sem->_count, 0, &count, USEM_HAS_WAITERS); + if (rv == -1) { umtxq_lock(&uq->uq_key); umtxq_unbusy(&uq->uq_key); umtxq_remove(uq); @@ -2877,10 +3004,12 @@ static int do_sem2_wake(struct thread *td, struct _usem2 *sem) { struct umtx_key key; - int error, cnt; + int error, cnt, rv; uint32_t count, flags; - flags = fuword32(&sem->_flags); + rv = fueword32(&sem->_flags, &flags); + if (rv == -1) + return (EFAULT); if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0) return (error); umtxq_lock(&key); @@ -2895,12 +3024,12 @@ do_sem2_wake(struct thread *td, struct _usem2 *sem) */ if (cnt == 1) { umtxq_unlock(&key); - count = fuword32(__DEVOLATILE(uint32_t *, - &sem->_count)); - while (count != -1 && count & USEM_HAS_WAITERS) - count = casuword32(&sem->_count, count, + rv = fueword32(__DEVOLATILE(uint32_t *, &sem->_count), + &count); + while (rv != -1 && count & USEM_HAS_WAITERS) + rv = casueword32(&sem->_count, count, &count, count & ~USEM_HAS_WAITERS); - if (count == -1) + if (rv == -1) error = EFAULT; umtxq_lock(&key); }