d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

rwlock: switch to fcmpset

Browse Source 
Discussed with:	jhb
Tested by:	pho
This commit is contained in:
Mateusz Guzik 2017-02-05 04:53:13 +00:00
parent 90836c3270
commit c84f347985
2 changed files with 20 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
sys/kern/kern_rwlock.c
View File

@ -440,7 +440,7 @@ __rw_rlock(volatile uintptr_t *c, const char *file, int line)
* if the lock has been unlocked and write waiters
* were present.
*/
if (atomic_cmpset_acq_ptr(&rw->rw_lock, v,
if (atomic_fcmpset_acq_ptr(&rw->rw_lock, &v,
v + RW_ONE_READER)) {
if (LOCK_LOG_TEST(&rw->lock_object, 0))
CTR4(KTR_LOCK,
@ -449,7 +449,6 @@ __rw_rlock(volatile uintptr_t *c, const char *file, int line)
(void *)(v + RW_ONE_READER));
break;
}
v = RW_READ_VALUE(rw);
continue;
}
#ifdef KDTRACE_HOOKS
@ -675,7 +674,7 @@ _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line)
* just drop one and return.
*/
if (RW_READERS(x) > 1) {
if (atomic_cmpset_rel_ptr(&rw->rw_lock, x,
if (atomic_fcmpset_rel_ptr(&rw->rw_lock, &x,
x - RW_ONE_READER)) {
if (LOCK_LOG_TEST(&rw->lock_object, 0))
CTR4(KTR_LOCK,
@ -684,7 +683,6 @@ _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line)
(void *)(x - RW_ONE_READER));
break;
}
x = RW_READ_VALUE(rw);
continue;
}
/*
@ -694,14 +692,13 @@ _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line)
if (!(x & RW_LOCK_WAITERS)) {
MPASS((x & ~RW_LOCK_WRITE_SPINNER) ==
RW_READERS_LOCK(1));
if (atomic_cmpset_rel_ptr(&rw->rw_lock, x,
if (atomic_fcmpset_rel_ptr(&rw->rw_lock, &x,
RW_UNLOCKED)) {
if (LOCK_LOG_TEST(&rw->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p last succeeded",
__func__, rw);
break;
}
x = RW_READ_VALUE(rw);
continue;
}
/*
@ -769,8 +766,8 @@ _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line)
* read or write lock.
*/
void
__rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
int line)
__rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, uintptr_t tid,
const char *file, int line)
{
struct rwlock *rw;
struct turnstile *ts;
@ -779,7 +776,7 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
int spintries = 0;
int i;
#endif
uintptr_t v, x;
uintptr_t x;
#ifdef LOCK_PROFILING
uint64_t waittime = 0;
int contested = 0;
@ -803,7 +800,6 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
lock_delay_arg_init(&lda, NULL);
#endif
rw = rwlock2rw(c);
v = RW_READ_VALUE(rw);
if (__predict_false(lv_rw_wowner(v) == (struct thread *)tid)) {
KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE,
@ -825,9 +821,8 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
#endif
for (;;) {
if (v == RW_UNLOCKED) {
if (_rw_write_lock(rw, tid))
if (_rw_write_lock_fetch(rw, &v, tid))
break;
v = RW_READ_VALUE(rw);
continue;
}
#ifdef KDTRACE_HOOKS

20
sys/sys/rwlock.h
View File

@ -84,6 +84,11 @@
#define _rw_write_lock(rw, tid) \
atomic_cmpset_acq_ptr(&(rw)->rw_lock, RW_UNLOCKED, (tid))
#define _rw_write_lock_fetch(rw, vp, tid) ({ \
*vp = RW_UNLOCKED; \
atomic_fcmpset_acq_ptr(&(rw)->rw_lock, vp, (tid)); \
})
/* Release a write lock quickly if there are no waiters. */
#define _rw_write_unlock(rw, tid) \
atomic_cmpset_rel_ptr(&(rw)->rw_lock, (tid), RW_UNLOCKED)
@ -97,9 +102,10 @@
/* Acquire a write lock. */
#define __rw_wlock(rw, tid, file, line) do { \
uintptr_t _tid = (uintptr_t)(tid); \
uintptr_t _v; \
\
if ((rw)->rw_lock != RW_UNLOCKED || !_rw_write_lock((rw), _tid))\
_rw_wlock_hard((rw), _tid, (file), (line)); \
if (!_rw_write_lock_fetch((rw), &_v, _tid)) \
_rw_wlock_hard((rw), _v, _tid, (file), (line)); \
else \
LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, \
0, 0, file, line, LOCKSTAT_WRITER); \
@ -114,7 +120,7 @@
else { \
LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, \
LOCKSTAT_WRITER); \
if ((rw)->rw_lock != _tid || !_rw_write_unlock((rw), _tid))\
if (!_rw_write_unlock((rw), _tid)) \
_rw_wunlock_hard((rw), _tid, (file), (line)); \
} \
} while (0)
@ -135,8 +141,8 @@ void _rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line);
void __rw_rlock(volatile uintptr_t *c, const char *file, int line);
int __rw_try_rlock(volatile uintptr_t *c, const char *file, int line);
void _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line);
void __rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
int line);
void __rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, uintptr_t tid,
const char *file, int line);
void __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid,
const char *file, int line);
int __rw_try_upgrade(volatile uintptr_t *c, const char *file, int line);
@ -171,8 +177,8 @@ void __rw_assert(const volatile uintptr_t *c, int what, const char *file,
__rw_try_rlock(&(rw)->rw_lock, f, l)
#define _rw_runlock(rw, f, l) \
_rw_runlock_cookie(&(rw)->rw_lock, f, l)
#define _rw_wlock_hard(rw, t, f, l) \
__rw_wlock_hard(&(rw)->rw_lock, t, f, l)
#define _rw_wlock_hard(rw, v, t, f, l) \
__rw_wlock_hard(&(rw)->rw_lock, v, t, f, l)
#define _rw_wunlock_hard(rw, t, f, l) \
__rw_wunlock_hard(&(rw)->rw_lock, t, f, l)
#define _rw_try_upgrade(rw, f, l) \