d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

rwlock: don't check for curthread's read lock count in the fast path

Browse Source
This commit is contained in:
mjg 2017-11-22 23:52:05 +00:00
parent 5713e2d563
commit 41e06ccaac
1 changed files with 17 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
sys/kern/kern_rwlock.c
View File

@ -364,12 +364,20 @@ _rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line)
* is unlocked and has no writer waiters or spinners. Failing otherwise * is unlocked and has no writer waiters or spinners. Failing otherwise
* prioritizes writers before readers. * prioritizes writers before readers.
*/ */
#define RW_CAN_READ(td, _rw) \ static bool __always_inline
(((_rw) & (RW_LOCK_READ | RW_LOCK_WRITE_WAITERS | RW_LOCK_WRITE_SPINNER)) ==\ __rw_can_read(struct thread *td, uintptr_t v, bool fp)
RW_LOCK_READ || ((td)->td_rw_rlocks && (_rw) & RW_LOCK_READ)) {
if ((v & (RW_LOCK_READ | RW_LOCK_WRITE_WAITERS | RW_LOCK_WRITE_SPINNER))
== RW_LOCK_READ)
return (true);
if (!fp && td->td_rw_rlocks && (v & RW_LOCK_READ))
return (true);
return (false);
}
static bool __always_inline static bool __always_inline
__rw_rlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp __rw_rlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp, bool fp
LOCK_FILE_LINE_ARG_DEF) LOCK_FILE_LINE_ARG_DEF)
{ {
@ -383,7 +391,7 @@ __rw_rlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp
* completely unlocked rwlock since such a lock is encoded * completely unlocked rwlock since such a lock is encoded
* as a read lock with no waiters. * as a read lock with no waiters.
*/ */
while (RW_CAN_READ(td, *vp)) { while (__rw_can_read(td, *vp, fp)) {
if (atomic_fcmpset_acq_ptr(&rw->rw_lock, vp, if (atomic_fcmpset_acq_ptr(&rw->rw_lock, vp,
*vp + RW_ONE_READER)) { *vp + RW_ONE_READER)) {
if (LOCK_LOG_TEST(&rw->lock_object, 0)) if (LOCK_LOG_TEST(&rw->lock_object, 0))
@ -452,7 +460,7 @@ __rw_rlock_hard(struct rwlock *rw, struct thread *td, uintptr_t v
#endif #endif
for (;;) { for (;;) {
if (__rw_rlock_try(rw, td, &v LOCK_FILE_LINE_ARG)) if (__rw_rlock_try(rw, td, &v, false LOCK_FILE_LINE_ARG))
break; break;
#ifdef KDTRACE_HOOKS #ifdef KDTRACE_HOOKS
lda.spin_cnt++; lda.spin_cnt++;
@ -492,7 +500,7 @@ __rw_rlock_hard(struct rwlock *rw, struct thread *td, uintptr_t v
n = RW_READERS(v); n = RW_READERS(v);
lock_delay_spin(n); lock_delay_spin(n);
v = RW_READ_VALUE(rw); v = RW_READ_VALUE(rw);
if ((v & RW_LOCK_READ) == 0 || RW_CAN_READ(td, v)) if ((v & RW_LOCK_READ) == 0 || __rw_can_read(td, v, false))
break; break;
} }
#ifdef KDTRACE_HOOKS #ifdef KDTRACE_HOOKS
@ -518,7 +526,7 @@ __rw_rlock_hard(struct rwlock *rw, struct thread *td, uintptr_t v
* recheck its state and restart the loop if needed. * recheck its state and restart the loop if needed.
*/ */
v = RW_READ_VALUE(rw); v = RW_READ_VALUE(rw);
if (RW_CAN_READ(td, v)) { if (__rw_can_read(td, v, false)) {
turnstile_cancel(ts); turnstile_cancel(ts);
continue; continue;
} }
@ -630,7 +638,7 @@ __rw_rlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
v = RW_READ_VALUE(rw); v = RW_READ_VALUE(rw);
if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(rw__acquire) || if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(rw__acquire) ||
!__rw_rlock_try(rw, td, &v LOCK_FILE_LINE_ARG))) !__rw_rlock_try(rw, td, &v, true LOCK_FILE_LINE_ARG)))
__rw_rlock_hard(rw, td, v LOCK_FILE_LINE_ARG); __rw_rlock_hard(rw, td, v LOCK_FILE_LINE_ARG);
LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line); LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line);