d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
6ab76844d8
freebsd-skq/lib/libthr/thread/thr_mutex.c
pfg 9da7bdde06 spdx: initial adoption of licensing ID tags. 
The Software Package Data Exchange (SPDX) group provides a specification
to make it easier for automated tools to detect and summarize well known
opensource licenses. We are gradually adopting the specification, noting
that the tags are considered only advisory and do not, in any way,
superceed or replace the license texts.

Special thanks to Wind River for providing access to "The Duke of
Highlander" tool: an older (2014) run over FreeBSD tree was useful as a
starting point.

Initially, only tag files that use BSD 4-Clause "Original" license.

RelNotes:	yes
Differential Revision:	https://reviews.freebsd.org/D13133
2017-11-18 14:26:50 +00:00

1191 lines
30 KiB
C
Raw Blame History

/*-
* SPDX-License-Identifier: BSD-4-Clause
*
* Copyright (c) 1995 John Birrell <jb@cimlogic.com.au>.
* Copyright (c) 2006 David Xu <davidxu@freebsd.org>.
* Copyright (c) 2015, 2016 The FreeBSD Foundation
*
* All rights reserved.
*
* Portions of this software were developed by Konstantin Belousov
* under sponsorship from the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed by John Birrell.
* 4. Neither the name of the author nor the names of any co-contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "namespace.h"
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <sys/param.h>
#include <sys/queue.h>
#include <pthread.h>
#include <pthread_np.h>
#include "un-namespace.h"
#include "thr_private.h"
_Static_assert(sizeof(struct pthread_mutex) <= PAGE_SIZE,
"pthread_mutex is too large for off-page");
/*
* For adaptive mutexes, how many times to spin doing trylock2
* before entering the kernel to block
*/
#define MUTEX_ADAPTIVE_SPINS 2000
/*
* Prototypes
*/
int __pthread_mutex_consistent(pthread_mutex_t *mutex);
int __pthread_mutex_init(pthread_mutex_t *mutex,
const pthread_mutexattr_t *mutex_attr);
int __pthread_mutex_trylock(pthread_mutex_t *mutex);
int __pthread_mutex_lock(pthread_mutex_t *mutex);
int __pthread_mutex_timedlock(pthread_mutex_t *mutex,
const struct timespec *abstime);
int _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count);
int _pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count);
int __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count);
int _pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count);
int _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count);
int __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count);
static int mutex_self_trylock(pthread_mutex_t);
static int mutex_self_lock(pthread_mutex_t,
const struct timespec *abstime);
static int mutex_unlock_common(struct pthread_mutex *, bool, int *);
static int mutex_lock_sleep(struct pthread *, pthread_mutex_t,
const struct timespec *);
static void mutex_init_robust(struct pthread *curthread);
static int mutex_qidx(struct pthread_mutex *m);
static bool is_robust_mutex(struct pthread_mutex *m);
static bool is_pshared_mutex(struct pthread_mutex *m);
__weak_reference(__pthread_mutex_init, pthread_mutex_init);
__strong_reference(__pthread_mutex_init, _pthread_mutex_init);
__weak_reference(__pthread_mutex_lock, pthread_mutex_lock);
__strong_reference(__pthread_mutex_lock, _pthread_mutex_lock);
__weak_reference(__pthread_mutex_timedlock, pthread_mutex_timedlock);
__strong_reference(__pthread_mutex_timedlock, _pthread_mutex_timedlock);
__weak_reference(__pthread_mutex_trylock, pthread_mutex_trylock);
__strong_reference(__pthread_mutex_trylock, _pthread_mutex_trylock);
__weak_reference(_pthread_mutex_consistent, pthread_mutex_consistent);
__strong_reference(_pthread_mutex_consistent, __pthread_mutex_consistent);
/* Single underscore versions provided for libc internal usage: */
/* No difference between libc and application usage of these: */
__weak_reference(_pthread_mutex_destroy, pthread_mutex_destroy);
__weak_reference(_pthread_mutex_unlock, pthread_mutex_unlock);
__weak_reference(_pthread_mutex_getprioceiling, pthread_mutex_getprioceiling);
__weak_reference(_pthread_mutex_setprioceiling, pthread_mutex_setprioceiling);
__weak_reference(__pthread_mutex_setspinloops_np, pthread_mutex_setspinloops_np);
__strong_reference(__pthread_mutex_setspinloops_np, _pthread_mutex_setspinloops_np);
__weak_reference(_pthread_mutex_getspinloops_np, pthread_mutex_getspinloops_np);
__weak_reference(__pthread_mutex_setyieldloops_np, pthread_mutex_setyieldloops_np);
__strong_reference(__pthread_mutex_setyieldloops_np, _pthread_mutex_setyieldloops_np);
__weak_reference(_pthread_mutex_getyieldloops_np, pthread_mutex_getyieldloops_np);
__weak_reference(_pthread_mutex_isowned_np, pthread_mutex_isowned_np);
static void
mutex_init_link(struct pthread_mutex *m)
{
#if defined(_PTHREADS_INVARIANTS)
m->m_qe.tqe_prev = NULL;
m->m_qe.tqe_next = NULL;
m->m_pqe.tqe_prev = NULL;
m->m_pqe.tqe_next = NULL;
#endif
}
static void
mutex_assert_is_owned(struct pthread_mutex *m __unused)
{
#if defined(_PTHREADS_INVARIANTS)
if (__predict_false(m->m_qe.tqe_prev == NULL))
PANIC("mutex %p own %#x is not on list %p %p",
m, m->m_lock.m_owner, m->m_qe.tqe_prev, m->m_qe.tqe_next);
#endif
}
static void
mutex_assert_not_owned(struct pthread *curthread __unused,
struct pthread_mutex *m __unused)
{
#if defined(_PTHREADS_INVARIANTS)
if (__predict_false(m->m_qe.tqe_prev != NULL ||
m->m_qe.tqe_next != NULL))
PANIC("mutex %p own %#x is on list %p %p",
m, m->m_lock.m_owner, m->m_qe.tqe_prev, m->m_qe.tqe_next);
if (__predict_false(is_robust_mutex(m) &&
(m->m_lock.m_rb_lnk != 0 || m->m_rb_prev != NULL ||
(is_pshared_mutex(m) && curthread->robust_list ==
(uintptr_t)&m->m_lock) ||
(!is_pshared_mutex(m) && curthread->priv_robust_list ==
(uintptr_t)&m->m_lock))))
PANIC(
"mutex %p own %#x is on robust linkage %p %p head %p phead %p",
m, m->m_lock.m_owner, (void *)m->m_lock.m_rb_lnk,
m->m_rb_prev, (void *)curthread->robust_list,
(void *)curthread->priv_robust_list);
#endif
}
static bool
is_pshared_mutex(struct pthread_mutex *m)
{
return ((m->m_lock.m_flags & USYNC_PROCESS_SHARED) != 0);
}
static bool
is_robust_mutex(struct pthread_mutex *m)
{
return ((m->m_lock.m_flags & UMUTEX_ROBUST) != 0);
}
int
_mutex_enter_robust(struct pthread *curthread, struct pthread_mutex *m)
{
#if defined(_PTHREADS_INVARIANTS)
if (__predict_false(curthread->inact_mtx != 0))
PANIC("inact_mtx enter");
#endif
if (!is_robust_mutex(m))
return (0);
mutex_init_robust(curthread);
curthread->inact_mtx = (uintptr_t)&m->m_lock;
return (1);
}
void
_mutex_leave_robust(struct pthread *curthread, struct pthread_mutex *m __unused)
{
#if defined(_PTHREADS_INVARIANTS)
if (__predict_false(curthread->inact_mtx != (uintptr_t)&m->m_lock))
PANIC("inact_mtx leave");
#endif
curthread->inact_mtx = 0;
}
static int
mutex_check_attr(const struct pthread_mutex_attr *attr)
{
if (attr->m_type < PTHREAD_MUTEX_ERRORCHECK ||
attr->m_type >= PTHREAD_MUTEX_TYPE_MAX)
return (EINVAL);
if (attr->m_protocol < PTHREAD_PRIO_NONE ||
attr->m_protocol > PTHREAD_PRIO_PROTECT)
return (EINVAL);
return (0);
}
static void
mutex_init_robust(struct pthread *curthread)
{
struct umtx_robust_lists_params rb;
if (curthread == NULL)
curthread = _get_curthread();
if (curthread->robust_inited)
return;
rb.robust_list_offset = (uintptr_t)&curthread->robust_list;
rb.robust_priv_list_offset = (uintptr_t)&curthread->priv_robust_list;
rb.robust_inact_offset = (uintptr_t)&curthread->inact_mtx;
_umtx_op(NULL, UMTX_OP_ROBUST_LISTS, sizeof(rb), &rb, NULL);
curthread->robust_inited = 1;
}
static void
mutex_init_body(struct pthread_mutex *pmutex,
const struct pthread_mutex_attr *attr)
{
pmutex->m_flags = attr->m_type;
pmutex->m_count = 0;
pmutex->m_spinloops = 0;
pmutex->m_yieldloops = 0;
mutex_init_link(pmutex);
switch (attr->m_protocol) {
case PTHREAD_PRIO_NONE:
pmutex->m_lock.m_owner = UMUTEX_UNOWNED;
pmutex->m_lock.m_flags = 0;
break;
case PTHREAD_PRIO_INHERIT:
pmutex->m_lock.m_owner = UMUTEX_UNOWNED;
pmutex->m_lock.m_flags = UMUTEX_PRIO_INHERIT;
break;
case PTHREAD_PRIO_PROTECT:
pmutex->m_lock.m_owner = UMUTEX_CONTESTED;
pmutex->m_lock.m_flags = UMUTEX_PRIO_PROTECT;
pmutex->m_lock.m_ceilings[0] = attr->m_ceiling;
break;
}
if (attr->m_pshared == PTHREAD_PROCESS_SHARED)
pmutex->m_lock.m_flags |= USYNC_PROCESS_SHARED;
if (attr->m_robust == PTHREAD_MUTEX_ROBUST) {
mutex_init_robust(NULL);
pmutex->m_lock.m_flags |= UMUTEX_ROBUST;
}
if (PMUTEX_TYPE(pmutex->m_flags) == PTHREAD_MUTEX_ADAPTIVE_NP) {
pmutex->m_spinloops =
_thr_spinloops ? _thr_spinloops: MUTEX_ADAPTIVE_SPINS;
pmutex->m_yieldloops = _thr_yieldloops;
}
}
static int
mutex_init(pthread_mutex_t *mutex,
const struct pthread_mutex_attr *mutex_attr,
void *(calloc_cb)(size_t, size_t))
{
const struct pthread_mutex_attr *attr;
struct pthread_mutex *pmutex;
int error;
if (mutex_attr == NULL) {
attr = &_pthread_mutexattr_default;
} else {
attr = mutex_attr;
error = mutex_check_attr(attr);
if (error != 0)
return (error);
}
if ((pmutex = (pthread_mutex_t)
calloc_cb(1, sizeof(struct pthread_mutex))) == NULL)
return (ENOMEM);
mutex_init_body(pmutex, attr);
*mutex = pmutex;
return (0);
}
static int
init_static(struct pthread *thread, pthread_mutex_t *mutex)
{
int ret;
THR_LOCK_ACQUIRE(thread, &_mutex_static_lock);
if (*mutex == THR_MUTEX_INITIALIZER)
ret = mutex_init(mutex, &_pthread_mutexattr_default, calloc);
else if (*mutex == THR_ADAPTIVE_MUTEX_INITIALIZER)
ret = mutex_init(mutex, &_pthread_mutexattr_adaptive_default,
calloc);
else
ret = 0;
THR_LOCK_RELEASE(thread, &_mutex_static_lock);
return (ret);
}
static void
set_inherited_priority(struct pthread *curthread, struct pthread_mutex *m)
{
struct pthread_mutex *m2;
m2 = TAILQ_LAST(&curthread->mq[mutex_qidx(m)], mutex_queue);
if (m2 != NULL)
m->m_lock.m_ceilings[1] = m2->m_lock.m_ceilings[0];
else
m->m_lock.m_ceilings[1] = -1;
}
static void
shared_mutex_init(struct pthread_mutex *pmtx, const struct
pthread_mutex_attr *mutex_attr)
{
static const struct pthread_mutex_attr foobar_mutex_attr = {
.m_type = PTHREAD_MUTEX_DEFAULT,
.m_protocol = PTHREAD_PRIO_NONE,
.m_ceiling = 0,
.m_pshared = PTHREAD_PROCESS_SHARED,
.m_robust = PTHREAD_MUTEX_STALLED,
};
bool done;
/*
* Hack to allow multiple pthread_mutex_init() calls on the
* same process-shared mutex. We rely on kernel allocating
* zeroed offpage for the mutex, i.e. the
* PMUTEX_INITSTAGE_ALLOC value must be zero.
*/
for (done = false; !done;) {
switch (pmtx->m_ps) {
case PMUTEX_INITSTAGE_DONE:
atomic_thread_fence_acq();
done = true;
break;
case PMUTEX_INITSTAGE_ALLOC:
if (atomic_cmpset_int(&pmtx->m_ps,
PMUTEX_INITSTAGE_ALLOC, PMUTEX_INITSTAGE_BUSY)) {
if (mutex_attr == NULL)
mutex_attr = &foobar_mutex_attr;
mutex_init_body(pmtx, mutex_attr);
atomic_store_rel_int(&pmtx->m_ps,
PMUTEX_INITSTAGE_DONE);
done = true;
}
break;
case PMUTEX_INITSTAGE_BUSY:
_pthread_yield();
break;
default:
PANIC("corrupted offpage");
break;
}
}
}
int
__pthread_mutex_init(pthread_mutex_t *mutex,
const pthread_mutexattr_t *mutex_attr)
{
struct pthread_mutex *pmtx;
int ret;
if (mutex_attr != NULL) {
ret = mutex_check_attr(*mutex_attr);
if (ret != 0)
return (ret);
}
if (mutex_attr == NULL ||
(*mutex_attr)->m_pshared == PTHREAD_PROCESS_PRIVATE) {
return (mutex_init(mutex, mutex_attr ? *mutex_attr : NULL,
calloc));
}
pmtx = __thr_pshared_offpage(mutex, 1);
if (pmtx == NULL)
return (EFAULT);
*mutex = THR_PSHARED_PTR;
shared_mutex_init(pmtx, *mutex_attr);
return (0);
}
/* This function is used internally by malloc. */
int
_pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
void *(calloc_cb)(size_t, size_t))
{
static const struct pthread_mutex_attr attr = {
.m_type = PTHREAD_MUTEX_NORMAL,
.m_protocol = PTHREAD_PRIO_NONE,
.m_ceiling = 0,
.m_pshared = PTHREAD_PROCESS_PRIVATE,
.m_robust = PTHREAD_MUTEX_STALLED,
};
int ret;
ret = mutex_init(mutex, &attr, calloc_cb);
if (ret == 0)
(*mutex)->m_flags |= PMUTEX_FLAG_PRIVATE;
return (ret);
}
/*
* Fix mutex ownership for child process.
*
* Process private mutex ownership is transmitted from the forking
* thread to the child process.
*
* Process shared mutex should not be inherited because owner is
* forking thread which is in parent process, they are removed from
* the owned mutex list.
*/
static void
queue_fork(struct pthread *curthread, struct mutex_queue *q,
struct mutex_queue *qp, uint bit)
{
struct pthread_mutex *m;
TAILQ_INIT(q);
TAILQ_FOREACH(m, qp, m_pqe) {
TAILQ_INSERT_TAIL(q, m, m_qe);
m->m_lock.m_owner = TID(curthread) | bit;
}
}
void
_mutex_fork(struct pthread *curthread)
{
queue_fork(curthread, &curthread->mq[TMQ_NORM],
&curthread->mq[TMQ_NORM_PRIV], 0);
queue_fork(curthread, &curthread->mq[TMQ_NORM_PP],
&curthread->mq[TMQ_NORM_PP_PRIV], UMUTEX_CONTESTED);
queue_fork(curthread, &curthread->mq[TMQ_ROBUST_PP],
&curthread->mq[TMQ_ROBUST_PP_PRIV], UMUTEX_CONTESTED);
curthread->robust_list = 0;
}
int
_pthread_mutex_destroy(pthread_mutex_t *mutex)
{
pthread_mutex_t m, m1;
int ret;
m = *mutex;
if (m < THR_MUTEX_DESTROYED) {
ret = 0;
} else if (m == THR_MUTEX_DESTROYED) {
ret = EINVAL;
} else {
if (m == THR_PSHARED_PTR) {
m1 = __thr_pshared_offpage(mutex, 0);
if (m1 != NULL) {
mutex_assert_not_owned(_get_curthread(), m1);
__thr_pshared_destroy(mutex);
}
*mutex = THR_MUTEX_DESTROYED;
return (0);
}
if (PMUTEX_OWNER_ID(m) != 0 &&
(uint32_t)m->m_lock.m_owner != UMUTEX_RB_NOTRECOV) {
ret = EBUSY;
} else {
*mutex = THR_MUTEX_DESTROYED;
mutex_assert_not_owned(_get_curthread(), m);
free(m);
ret = 0;
}
}
return (ret);
}
static int
mutex_qidx(struct pthread_mutex *m)
{
if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
return (TMQ_NORM);
return (is_robust_mutex(m) ? TMQ_ROBUST_PP : TMQ_NORM_PP);
}
/*
* Both enqueue_mutex() and dequeue_mutex() operate on the
* thread-private linkage of the locked mutexes and on the robust
* linkage.
*
* Robust list, as seen by kernel, must be consistent even in the case
* of thread termination at arbitrary moment. Since either enqueue or
* dequeue for list walked by kernel consists of rewriting a single
* forward pointer, it is safe. On the other hand, rewrite of the
* back pointer is not atomic WRT the forward one, but kernel does not
* care.
*/
static void
enqueue_mutex(struct pthread *curthread, struct pthread_mutex *m,
int error)
{
struct pthread_mutex *m1;
uintptr_t *rl;
int qidx;
/* Add to the list of owned mutexes: */
if (error != EOWNERDEAD)
mutex_assert_not_owned(curthread, m);
qidx = mutex_qidx(m);
TAILQ_INSERT_TAIL(&curthread->mq[qidx], m, m_qe);
if (!is_pshared_mutex(m))
TAILQ_INSERT_TAIL(&curthread->mq[qidx + 1], m, m_pqe);
if (is_robust_mutex(m)) {
rl = is_pshared_mutex(m) ? &curthread->robust_list :
&curthread->priv_robust_list;
m->m_rb_prev = NULL;
if (*rl != 0) {
m1 = __containerof((void *)*rl,
struct pthread_mutex, m_lock);
m->m_lock.m_rb_lnk = (uintptr_t)&m1->m_lock;
m1->m_rb_prev = m;
} else {
m1 = NULL;
m->m_lock.m_rb_lnk = 0;
}
*rl = (uintptr_t)&m->m_lock;
}
}
static void
dequeue_mutex(struct pthread *curthread, struct pthread_mutex *m)
{
struct pthread_mutex *mp, *mn;
int qidx;
mutex_assert_is_owned(m);
qidx = mutex_qidx(m);
if (is_robust_mutex(m)) {
mp = m->m_rb_prev;
if (mp == NULL) {
if (is_pshared_mutex(m)) {
curthread->robust_list = m->m_lock.m_rb_lnk;
} else {
curthread->priv_robust_list =
m->m_lock.m_rb_lnk;
}
} else {
mp->m_lock.m_rb_lnk = m->m_lock.m_rb_lnk;
}
if (m->m_lock.m_rb_lnk != 0) {
mn = __containerof((void *)m->m_lock.m_rb_lnk,
struct pthread_mutex, m_lock);
mn->m_rb_prev = m->m_rb_prev;
}
m->m_lock.m_rb_lnk = 0;
m->m_rb_prev = NULL;
}
TAILQ_REMOVE(&curthread->mq[qidx], m, m_qe);
if (!is_pshared_mutex(m))
TAILQ_REMOVE(&curthread->mq[qidx + 1], m, m_pqe);
if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) != 0)
set_inherited_priority(curthread, m);
mutex_init_link(m);
}
static int
check_and_init_mutex(pthread_mutex_t *mutex, struct pthread_mutex **m)
{
int ret;
*m = *mutex;
ret = 0;
if (*m == THR_PSHARED_PTR) {
*m = __thr_pshared_offpage(mutex, 0);
if (*m == NULL)
ret = EINVAL;
else
shared_mutex_init(*m, NULL);
} else if (__predict_false(*m <= THR_MUTEX_DESTROYED)) {
if (*m == THR_MUTEX_DESTROYED) {
ret = EINVAL;
} else {
ret = init_static(_get_curthread(), mutex);
if (ret == 0)
*m = *mutex;
}
}
return (ret);
}
int
__pthread_mutex_trylock(pthread_mutex_t *mutex)
{
struct pthread *curthread;
struct pthread_mutex *m;
uint32_t id;
int ret, robust;
ret = check_and_init_mutex(mutex, &m);
if (ret != 0)
return (ret);
curthread = _get_curthread();
id = TID(curthread);
if (m->m_flags & PMUTEX_FLAG_PRIVATE)
THR_CRITICAL_ENTER(curthread);
robust = _mutex_enter_robust(curthread, m);
ret = _thr_umutex_trylock(&m->m_lock, id);
if (__predict_true(ret == 0) || ret == EOWNERDEAD) {
enqueue_mutex(curthread, m, ret);
if (ret == EOWNERDEAD)
m->m_lock.m_flags |= UMUTEX_NONCONSISTENT;
} else if (PMUTEX_OWNER_ID(m) == id) {
ret = mutex_self_trylock(m);
} /* else {} */
if (robust)
_mutex_leave_robust(curthread, m);
if (ret != 0 && ret != EOWNERDEAD &&
(m->m_flags & PMUTEX_FLAG_PRIVATE) != 0)
THR_CRITICAL_LEAVE(curthread);
return (ret);
}
static int
mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m,
const struct timespec *abstime)
{
uint32_t id, owner;
int count, ret;
id = TID(curthread);
if (PMUTEX_OWNER_ID(m) == id)
return (mutex_self_lock(m, abstime));
/*
* For adaptive mutexes, spin for a bit in the expectation
* that if the application requests this mutex type then
* the lock is likely to be released quickly and it is
* faster than entering the kernel
*/
if (__predict_false((m->m_lock.m_flags & (UMUTEX_PRIO_PROTECT |
UMUTEX_PRIO_INHERIT | UMUTEX_ROBUST | UMUTEX_NONCONSISTENT)) != 0))
goto sleep_in_kernel;
if (!_thr_is_smp)
goto yield_loop;
count = m->m_spinloops;
while (count--) {
owner = m->m_lock.m_owner;
if ((owner & ~UMUTEX_CONTESTED) == 0) {
if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner,
id | owner)) {
ret = 0;
goto done;
}
}
CPU_SPINWAIT;
}
yield_loop:
count = m->m_yieldloops;
while (count--) {
_sched_yield();
owner = m->m_lock.m_owner;
if ((owner & ~UMUTEX_CONTESTED) == 0) {
if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner,
id | owner)) {
ret = 0;
goto done;
}
}
}
sleep_in_kernel:
if (abstime == NULL)
ret = __thr_umutex_lock(&m->m_lock, id);
else if (__predict_false(abstime->tv_nsec < 0 ||
abstime->tv_nsec >= 1000000000))
ret = EINVAL;
else
ret = __thr_umutex_timedlock(&m->m_lock, id, abstime);
done:
if (ret == 0 || ret == EOWNERDEAD) {
enqueue_mutex(curthread, m, ret);
if (ret == EOWNERDEAD)
m->m_lock.m_flags |= UMUTEX_NONCONSISTENT;
}
return (ret);
}
static inline int
mutex_lock_common(struct pthread_mutex *m, const struct timespec *abstime,
bool cvattach, bool rb_onlist)
{
struct pthread *curthread;
int ret, robust;
robust = 0; /* pacify gcc */
curthread = _get_curthread();
if (!cvattach && m->m_flags & PMUTEX_FLAG_PRIVATE)
THR_CRITICAL_ENTER(curthread);
if (!rb_onlist)
robust = _mutex_enter_robust(curthread, m);
ret = _thr_umutex_trylock2(&m->m_lock, TID(curthread));
if (ret == 0 || ret == EOWNERDEAD) {
enqueue_mutex(curthread, m, ret);
if (ret == EOWNERDEAD)
m->m_lock.m_flags |= UMUTEX_NONCONSISTENT;
} else {
ret = mutex_lock_sleep(curthread, m, abstime);
}
if (!rb_onlist && robust)
_mutex_leave_robust(curthread, m);
if (ret != 0 && ret != EOWNERDEAD &&
(m->m_flags & PMUTEX_FLAG_PRIVATE) != 0 && !cvattach)
THR_CRITICAL_LEAVE(curthread);
return (ret);
}
int
__pthread_mutex_lock(pthread_mutex_t *mutex)
{
struct pthread_mutex *m;
int ret;
_thr_check_init();
ret = check_and_init_mutex(mutex, &m);
if (ret == 0)
ret = mutex_lock_common(m, NULL, false, false);
return (ret);
}
int
__pthread_mutex_timedlock(pthread_mutex_t *mutex,
const struct timespec *abstime)
{
struct pthread_mutex *m;
int ret;
_thr_check_init();
ret = check_and_init_mutex(mutex, &m);
if (ret == 0)
ret = mutex_lock_common(m, abstime, false, false);
return (ret);
}
int
_pthread_mutex_unlock(pthread_mutex_t *mutex)
{
struct pthread_mutex *mp;
if (*mutex == THR_PSHARED_PTR) {
mp = __thr_pshared_offpage(mutex, 0);
if (mp == NULL)
return (EINVAL);
shared_mutex_init(mp, NULL);
} else {
mp = *mutex;
}
return (mutex_unlock_common(mp, false, NULL));
}
int
_mutex_cv_lock(struct pthread_mutex *m, int count, bool rb_onlist)
{
int error;
error = mutex_lock_common(m, NULL, true, rb_onlist);
if (error == 0 || error == EOWNERDEAD)
m->m_count = count;
return (error);
}
int
_mutex_cv_unlock(struct pthread_mutex *m, int *count, int *defer)
{
/*
* Clear the count in case this is a recursive mutex.
*/
*count = m->m_count;
m->m_count = 0;
(void)mutex_unlock_common(m, true, defer);
return (0);
}
int
_mutex_cv_attach(struct pthread_mutex *m, int count)
{
struct pthread *curthread;
curthread = _get_curthread();
enqueue_mutex(curthread, m, 0);
m->m_count = count;
return (0);
}
int
_mutex_cv_detach(struct pthread_mutex *mp, int *recurse)
{
struct pthread *curthread;
int deferred, error;
curthread = _get_curthread();
if ((error = _mutex_owned(curthread, mp)) != 0)
return (error);
/*
* Clear the count in case this is a recursive mutex.
*/
*recurse = mp->m_count;
mp->m_count = 0;
dequeue_mutex(curthread, mp);
/* Will this happen in real-world ? */
if ((mp->m_flags & PMUTEX_FLAG_DEFERRED) != 0) {
deferred = 1;
mp->m_flags &= ~PMUTEX_FLAG_DEFERRED;
} else
deferred = 0;
if (deferred) {
_thr_wake_all(curthread->defer_waiters,
curthread->nwaiter_defer);
curthread->nwaiter_defer = 0;
}
return (0);
}
static int
mutex_self_trylock(struct pthread_mutex *m)
{
int ret;
switch (PMUTEX_TYPE(m->m_flags)) {
case PTHREAD_MUTEX_ERRORCHECK:
case PTHREAD_MUTEX_NORMAL:
case PTHREAD_MUTEX_ADAPTIVE_NP:
ret = EBUSY;
break;
case PTHREAD_MUTEX_RECURSIVE:
/* Increment the lock count: */
if (m->m_count + 1 > 0) {
m->m_count++;
ret = 0;
} else
ret = EAGAIN;
break;
default:
/* Trap invalid mutex types; */
ret = EINVAL;
}
return (ret);
}
static int
mutex_self_lock(struct pthread_mutex *m, const struct timespec *abstime)
{
struct timespec ts1, ts2;
int ret;
switch (PMUTEX_TYPE(m->m_flags)) {
case PTHREAD_MUTEX_ERRORCHECK:
case PTHREAD_MUTEX_ADAPTIVE_NP:
if (abstime) {
if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 ||
abstime->tv_nsec >= 1000000000) {
ret = EINVAL;
} else {
clock_gettime(CLOCK_REALTIME, &ts1);
TIMESPEC_SUB(&ts2, abstime, &ts1);
__sys_nanosleep(&ts2, NULL);
ret = ETIMEDOUT;
}
} else {
/*
* POSIX specifies that mutexes should return
* EDEADLK if a recursive lock is detected.
*/
ret = EDEADLK;
}
break;
case PTHREAD_MUTEX_NORMAL:
/*
* What SS2 define as a 'normal' mutex. Intentionally
* deadlock on attempts to get a lock you already own.
*/
ret = 0;
if (abstime) {
if (abstime->tv_sec < 0 || abstime->tv_nsec < 0 ||
abstime->tv_nsec >= 1000000000) {
ret = EINVAL;
} else {
clock_gettime(CLOCK_REALTIME, &ts1);
TIMESPEC_SUB(&ts2, abstime, &ts1);
__sys_nanosleep(&ts2, NULL);
ret = ETIMEDOUT;
}
} else {
ts1.tv_sec = 30;
ts1.tv_nsec = 0;
for (;;)
__sys_nanosleep(&ts1, NULL);
}
break;
case PTHREAD_MUTEX_RECURSIVE:
/* Increment the lock count: */
if (m->m_count + 1 > 0) {
m->m_count++;
ret = 0;
} else
ret = EAGAIN;
break;
default:
/* Trap invalid mutex types; */
ret = EINVAL;
}
return (ret);
}
static int
mutex_unlock_common(struct pthread_mutex *m, bool cv, int *mtx_defer)
{
struct pthread *curthread;
uint32_t id;
int deferred, error, robust;
if (__predict_false(m <= THR_MUTEX_DESTROYED)) {
if (m == THR_MUTEX_DESTROYED)
return (EINVAL);
return (EPERM);
}
curthread = _get_curthread();
id = TID(curthread);
/*
* Check if the running thread is not the owner of the mutex.
*/
if (__predict_false(PMUTEX_OWNER_ID(m) != id))
return (EPERM);
error = 0;
if (__predict_false(PMUTEX_TYPE(m->m_flags) ==
PTHREAD_MUTEX_RECURSIVE && m->m_count > 0)) {
m->m_count--;
} else {
if ((m->m_flags & PMUTEX_FLAG_DEFERRED) != 0) {
deferred = 1;
m->m_flags &= ~PMUTEX_FLAG_DEFERRED;
} else
deferred = 0;
robust = _mutex_enter_robust(curthread, m);
dequeue_mutex(curthread, m);
error = _thr_umutex_unlock2(&m->m_lock, id, mtx_defer);
if (deferred) {
if (mtx_defer == NULL) {
_thr_wake_all(curthread->defer_waiters,
curthread->nwaiter_defer);
curthread->nwaiter_defer = 0;
} else
*mtx_defer = 1;
}
if (robust)
_mutex_leave_robust(curthread, m);
}
if (!cv && m->m_flags & PMUTEX_FLAG_PRIVATE)
THR_CRITICAL_LEAVE(curthread);
return (error);
}
int
_pthread_mutex_getprioceiling(pthread_mutex_t *mutex,
int *prioceiling)
{
struct pthread_mutex *m;
if (*mutex == THR_PSHARED_PTR) {
m = __thr_pshared_offpage(mutex, 0);
if (m == NULL)
return (EINVAL);
shared_mutex_init(m, NULL);
} else {
m = *mutex;
if (m <= THR_MUTEX_DESTROYED)
return (EINVAL);
}
if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
return (EINVAL);
*prioceiling = m->m_lock.m_ceilings[0];
return (0);
}
int
_pthread_mutex_setprioceiling(pthread_mutex_t *mutex,
int ceiling, int *old_ceiling)
{
struct pthread *curthread;
struct pthread_mutex *m, *m1, *m2;
struct mutex_queue *q, *qp;
int qidx, ret;
if (*mutex == THR_PSHARED_PTR) {
m = __thr_pshared_offpage(mutex, 0);
if (m == NULL)
return (EINVAL);
shared_mutex_init(m, NULL);
} else {
m = *mutex;
if (m <= THR_MUTEX_DESTROYED)
return (EINVAL);
}
if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
return (EINVAL);
ret = __thr_umutex_set_ceiling(&m->m_lock, ceiling, old_ceiling);
if (ret != 0)
return (ret);
curthread = _get_curthread();
if (PMUTEX_OWNER_ID(m) == TID(curthread)) {
mutex_assert_is_owned(m);
m1 = TAILQ_PREV(m, mutex_queue, m_qe);
m2 = TAILQ_NEXT(m, m_qe);
if ((m1 != NULL && m1->m_lock.m_ceilings[0] > (u_int)ceiling) ||
(m2 != NULL && m2->m_lock.m_ceilings[0] < (u_int)ceiling)) {
qidx = mutex_qidx(m);
q = &curthread->mq[qidx];
qp = &curthread->mq[qidx + 1];
TAILQ_REMOVE(q, m, m_qe);
if (!is_pshared_mutex(m))
TAILQ_REMOVE(qp, m, m_pqe);
TAILQ_FOREACH(m2, q, m_qe) {
if (m2->m_lock.m_ceilings[0] > (u_int)ceiling) {
TAILQ_INSERT_BEFORE(m2, m, m_qe);
if (!is_pshared_mutex(m)) {
while (m2 != NULL &&
is_pshared_mutex(m2)) {
m2 = TAILQ_PREV(m2,
mutex_queue, m_qe);
}
if (m2 == NULL) {
TAILQ_INSERT_HEAD(qp,
m, m_pqe);
} else {
TAILQ_INSERT_BEFORE(m2,
m, m_pqe);
}
}
return (0);
}
}
TAILQ_INSERT_TAIL(q, m, m_qe);
if (!is_pshared_mutex(m))
TAILQ_INSERT_TAIL(qp, m, m_pqe);
}
}
return (0);
}
int
_pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count)
{
struct pthread_mutex *m;
int ret;
ret = check_and_init_mutex(mutex, &m);
if (ret == 0)
*count = m->m_spinloops;
return (ret);
}
int
__pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count)
{
struct pthread_mutex *m;
int ret;
ret = check_and_init_mutex(mutex, &m);
if (ret == 0)
m->m_spinloops = count;
return (ret);
}
int
_pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count)
{
struct pthread_mutex *m;
int ret;
ret = check_and_init_mutex(mutex, &m);
if (ret == 0)
*count = m->m_yieldloops;
return (ret);
}
int
__pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count)
{
struct pthread_mutex *m;
int ret;
ret = check_and_init_mutex(mutex, &m);
if (ret == 0)
m->m_yieldloops = count;
return (0);
}
int
_pthread_mutex_isowned_np(pthread_mutex_t *mutex)
{
struct pthread_mutex *m;
if (*mutex == THR_PSHARED_PTR) {
m = __thr_pshared_offpage(mutex, 0);
if (m == NULL)
return (0);
shared_mutex_init(m, NULL);
} else {
m = *mutex;
if (m <= THR_MUTEX_DESTROYED)
return (0);
}
return (PMUTEX_OWNER_ID(m) == TID(_get_curthread()));
}
int
_mutex_owned(struct pthread *curthread, const struct pthread_mutex *mp)
{
if (__predict_false(mp <= THR_MUTEX_DESTROYED)) {
if (mp == THR_MUTEX_DESTROYED)
return (EINVAL);
return (EPERM);
}
if (PMUTEX_OWNER_ID(mp) != TID(curthread))
return (EPERM);
return (0);
}
int
_pthread_mutex_consistent(pthread_mutex_t *mutex)
{
struct pthread_mutex *m;
struct pthread *curthread;
if (*mutex == THR_PSHARED_PTR) {
m = __thr_pshared_offpage(mutex, 0);
if (m == NULL)
return (EINVAL);
shared_mutex_init(m, NULL);
} else {
m = *mutex;
if (m <= THR_MUTEX_DESTROYED)
return (EINVAL);
}
curthread = _get_curthread();
if ((m->m_lock.m_flags & (UMUTEX_ROBUST | UMUTEX_NONCONSISTENT)) !=
(UMUTEX_ROBUST | UMUTEX_NONCONSISTENT))
return (EINVAL);
if (PMUTEX_OWNER_ID(m) != TID(curthread))
return (EPERM);
m->m_lock.m_flags &= ~UMUTEX_NONCONSISTENT;
return (0);
}
Reference in New Issue View Git Blame Copy Permalink