d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

rwlock: move lockstat handling out of inline primitives

Browse Source 
See r313275 for details.

One difference here is that recursion handling was removed from the fallback
routine. As it is it was never supposed to see a recursed lock in the first
place. Future changes will move it out of inline variants, but right now
there is no easy to way to test if the lock is recursed without reading
additional words.
This commit is contained in:
Mateusz Guzik 2017-02-05 13:37:23 +00:00
parent 96ee43103d
commit 993ddec44d
2 changed files with 25 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
sys/kern/kern_rwlock.c
View File

@ -283,6 +283,7 @@ void
_rw_wlock_cookie(volatile uintptr_t *c, const char *file, int line)
{
struct rwlock *rw;
uintptr_t tid, v;
if (SCHEDULER_STOPPED())
return;
@ -296,7 +297,14 @@ _rw_wlock_cookie(volatile uintptr_t *c, const char *file, int line)
("rw_wlock() of destroyed rwlock @ %s:%d", file, line));
WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
line, NULL);
__rw_wlock(rw, curthread, file, line);
tid = (uintptr_t)curthread;
v = RW_UNLOCKED;
if (!_rw_write_lock_fetch(rw, &v, tid))
_rw_wlock_hard(rw, v, tid, file, line);
else
LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw,
0, 0, file, line, LOCKSTAT_WRITER);
LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line);
WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
TD_LOCKS_INC(curthread);
@ -355,7 +363,11 @@ _rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line)
WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file,
line);
__rw_wunlock(rw, curthread, file, line);
if (rw->rw_recurse)
rw->rw_recurse--;
else
_rw_wunlock_hard(rw, (uintptr_t)curthread, file, line);
TD_LOCKS_DEC(curthread);
}
@ -998,13 +1010,12 @@ __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
return;
rw = rwlock2rw(c);
MPASS(!rw_recursed(rw));
if (rw_wlocked(rw) && rw_recursed(rw)) {
rw->rw_recurse--;
if (LOCK_LOG_TEST(&rw->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw);
LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw,
LOCKSTAT_WRITER);
if (_rw_write_unlock(rw, tid))
return;
}
KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS),
("%s: neither of the waiter flags are set", __func__));

19
sys/sys/rwlock.h
View File

@ -84,10 +84,8 @@
#define _rw_write_lock(rw, tid) \
atomic_cmpset_acq_ptr(&(rw)->rw_lock, RW_UNLOCKED, (tid))
#define _rw_write_lock_fetch(rw, vp, tid) ({ \
*vp = RW_UNLOCKED; \
atomic_fcmpset_acq_ptr(&(rw)->rw_lock, vp, (tid)); \
})
#define _rw_write_lock_fetch(rw, vp, tid) \
atomic_fcmpset_acq_ptr(&(rw)->rw_lock, vp, (tid))
/* Release a write lock quickly if there are no waiters. */
#define _rw_write_unlock(rw, tid) \
@ -102,13 +100,11 @@
/* Acquire a write lock. */
#define __rw_wlock(rw, tid, file, line) do { \
uintptr_t _tid = (uintptr_t)(tid); \
uintptr_t _v; \
uintptr_t _v = RW_UNLOCKED; \
\
if (!_rw_write_lock_fetch((rw), &_v, _tid)) \
if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__acquire) || \
!_rw_write_lock_fetch((rw), &_v, _tid))) \
_rw_wlock_hard((rw), _v, _tid, (file), (line)); \
else \
LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, \
0, 0, file, line, LOCKSTAT_WRITER); \
} while (0)
/* Release a write lock. */
@ -118,9 +114,8 @@
if ((rw)->rw_recurse) \
(rw)->rw_recurse--; \
else { \
LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, \
LOCKSTAT_WRITER); \
if (!_rw_write_unlock((rw), _tid)) \
if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__release) ||\
!_rw_write_unlock((rw), _tid))) \
_rw_wunlock_hard((rw), _tid, (file), (line)); \
} \
} while (0)