d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
0aad0f2282
freebsd-dev/lib/libkse/thread/thr_mutex.c
Kris Kennaway 2017a7cdfe Add a new "non-portable" mutex type, PTHREAD_MUTEX_ADAPTIVE_NP. This 
is also implemented in glibc and is used by a number of existing
applications (mysql, firefox, etc).

This mutex type is a default mutex with the additional property that
it spins briefly when attempting to acquire a contested lock, doing
trylock operations in userland before entering the kernel to block if
eventually unsuccessful.

The expectation is that applications requesting this mutex type know
that the mutex is likely to be only held for very brief periods, so it
is faster to spin in userland and probably succeed in acquiring the
mutex, than to enter the kernel and sleep, only to be woken up almost
immediately.  This can help significantly in certain cases when
pthread mutexes are heavily contended and held for brief durations
(such as mysql).

Spin up to 200 times before entering the kernel, which represents only
a few us on modern CPUs.  No performance degradation was observed with
this value and it is sufficient to avoid a large performance drop in
mysql performance in the heavily contended pthread mutex case.

The libkse implementation is a NOP.

Reviewed by:      jeff
MFC after:        3 days
2007-10-29 21:01:47 +00:00

1833 lines
49 KiB
C
Raw Blame History

/*
* Copyright (c) 1995 John Birrell <jb@cimlogic.com.au>.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the author nor the names of any co-contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <sys/param.h>
#include <sys/queue.h>
#include <pthread.h>
#include "thr_private.h"
#if defined(_PTHREADS_INVARIANTS)
#define MUTEX_INIT_LINK(m) do { \
(m)->m_qe.tqe_prev = NULL; \
(m)->m_qe.tqe_next = NULL; \
} while (0)
#define MUTEX_ASSERT_IS_OWNED(m) do { \
if ((m)->m_qe.tqe_prev == NULL) \
PANIC("mutex is not on list"); \
} while (0)
#define MUTEX_ASSERT_NOT_OWNED(m) do { \
if (((m)->m_qe.tqe_prev != NULL) || \
((m)->m_qe.tqe_next != NULL)) \
PANIC("mutex is on list"); \
} while (0)
#define THR_ASSERT_NOT_IN_SYNCQ(thr) do { \
THR_ASSERT(((thr)->sflags & THR_FLAGS_IN_SYNCQ) == 0, \
"thread in syncq when it shouldn't be."); \
} while (0);
#else
#define MUTEX_INIT_LINK(m)
#define MUTEX_ASSERT_IS_OWNED(m)
#define MUTEX_ASSERT_NOT_OWNED(m)
#define THR_ASSERT_NOT_IN_SYNCQ(thr)
#endif
#define THR_IN_MUTEXQ(thr) (((thr)->sflags & THR_FLAGS_IN_SYNCQ) != 0)
#define MUTEX_DESTROY(m) do { \
_lock_destroy(&(m)->m_lock); \
free(m); \
} while (0)
/*
* Prototypes
*/
static struct kse_mailbox *mutex_handoff(struct pthread *,
struct pthread_mutex *);
static inline int mutex_self_trylock(struct pthread *, pthread_mutex_t);
static inline int mutex_self_lock(struct pthread *, pthread_mutex_t);
static int mutex_unlock_common(pthread_mutex_t *, int);
static void mutex_priority_adjust(struct pthread *, pthread_mutex_t);
static void mutex_rescan_owned (struct pthread *, struct pthread *,
struct pthread_mutex *);
static inline pthread_t mutex_queue_deq(pthread_mutex_t);
static inline void mutex_queue_remove(pthread_mutex_t, pthread_t);
static inline void mutex_queue_enq(pthread_mutex_t, pthread_t);
static void mutex_lock_backout(void *arg);
static struct pthread_mutex_attr static_mutex_attr =
PTHREAD_MUTEXATTR_STATIC_INITIALIZER;
static pthread_mutexattr_t static_mattr = &static_mutex_attr;
LT10_COMPAT_PRIVATE(__pthread_mutex_init);
LT10_COMPAT_PRIVATE(_pthread_mutex_init);
LT10_COMPAT_DEFAULT(pthread_mutex_init);
LT10_COMPAT_PRIVATE(__pthread_mutex_lock);
LT10_COMPAT_PRIVATE(_pthread_mutex_lock);
LT10_COMPAT_DEFAULT(pthread_mutex_lock);
LT10_COMPAT_PRIVATE(__pthread_mutex_timedlock);
LT10_COMPAT_PRIVATE(_pthread_mutex_timedlock);
LT10_COMPAT_DEFAULT(pthread_mutex_timedlock);
LT10_COMPAT_PRIVATE(__pthread_mutex_trylock);
LT10_COMPAT_PRIVATE(_pthread_mutex_trylock);
LT10_COMPAT_DEFAULT(pthread_mutex_trylock);
LT10_COMPAT_PRIVATE(_pthread_mutex_destroy);
LT10_COMPAT_DEFAULT(pthread_mutex_destroy);
LT10_COMPAT_PRIVATE(_pthread_mutex_unlock);
LT10_COMPAT_DEFAULT(pthread_mutex_unlock);
/* Single underscore versions provided for libc internal usage: */
__weak_reference(__pthread_mutex_init, pthread_mutex_init);
__weak_reference(__pthread_mutex_lock, pthread_mutex_lock);
__weak_reference(__pthread_mutex_timedlock, pthread_mutex_timedlock);
__weak_reference(__pthread_mutex_trylock, pthread_mutex_trylock);
/* No difference between libc and application usage of these: */
__weak_reference(_pthread_mutex_destroy, pthread_mutex_destroy);
__weak_reference(_pthread_mutex_unlock, pthread_mutex_unlock);
int
__pthread_mutex_init(pthread_mutex_t *mutex,
const pthread_mutexattr_t *mutex_attr)
{
struct pthread_mutex *pmutex;
enum pthread_mutextype type;
int protocol;
int ceiling;
int flags;
int ret = 0;
if (mutex == NULL)
ret = EINVAL;
/* Check if default mutex attributes: */
else if (mutex_attr == NULL || *mutex_attr == NULL) {
/* Default to a (error checking) POSIX mutex: */
type = PTHREAD_MUTEX_ERRORCHECK;
protocol = PTHREAD_PRIO_NONE;
ceiling = THR_MAX_PRIORITY;
flags = 0;
}
/* Check mutex type: */
else if (((*mutex_attr)->m_type < PTHREAD_MUTEX_ERRORCHECK) ||
((*mutex_attr)->m_type >= PTHREAD_MUTEX_TYPE_MAX))
/* Return an invalid argument error: */
ret = EINVAL;
/* Check mutex protocol: */
else if (((*mutex_attr)->m_protocol < PTHREAD_PRIO_NONE) ||
((*mutex_attr)->m_protocol > PTHREAD_MUTEX_RECURSIVE))
/* Return an invalid argument error: */
ret = EINVAL;
else {
/* Use the requested mutex type and protocol: */
type = (*mutex_attr)->m_type;
protocol = (*mutex_attr)->m_protocol;
ceiling = (*mutex_attr)->m_ceiling;
flags = (*mutex_attr)->m_flags;
}
/* Check no errors so far: */
if (ret == 0) {
if ((pmutex = (pthread_mutex_t)
malloc(sizeof(struct pthread_mutex))) == NULL)
ret = ENOMEM;
else if (_lock_init(&pmutex->m_lock, LCK_ADAPTIVE,
_thr_lock_wait, _thr_lock_wakeup) != 0) {
free(pmutex);
*mutex = NULL;
ret = ENOMEM;
} else {
/* Set the mutex flags: */
pmutex->m_flags = flags;
/* Process according to mutex type: */
switch (type) {
/* case PTHREAD_MUTEX_DEFAULT: */
case PTHREAD_MUTEX_ERRORCHECK:
case PTHREAD_MUTEX_NORMAL:
case PTHREAD_MUTEX_ADAPTIVE_NP:
/* Nothing to do here. */
break;
/* Single UNIX Spec 2 recursive mutex: */
case PTHREAD_MUTEX_RECURSIVE:
/* Reset the mutex count: */
pmutex->m_count = 0;
break;
/* Trap invalid mutex types: */
default:
/* Return an invalid argument error: */
ret = EINVAL;
break;
}
if (ret == 0) {
/* Initialise the rest of the mutex: */
TAILQ_INIT(&pmutex->m_queue);
pmutex->m_flags |= MUTEX_FLAGS_INITED;
pmutex->m_owner = NULL;
pmutex->m_type = type;
pmutex->m_protocol = protocol;
pmutex->m_refcount = 0;
if (protocol == PTHREAD_PRIO_PROTECT)
pmutex->m_prio = ceiling;
else
pmutex->m_prio = -1;
pmutex->m_saved_prio = 0;
MUTEX_INIT_LINK(pmutex);
*mutex = pmutex;
} else {
/* Free the mutex lock structure: */
MUTEX_DESTROY(pmutex);
*mutex = NULL;
}
}
}
/* Return the completion status: */
return (ret);
}
int
_pthread_mutex_init(pthread_mutex_t *mutex,
const pthread_mutexattr_t *mutex_attr)
{
struct pthread_mutex_attr mattr, *mattrp;
if ((mutex_attr == NULL) || (*mutex_attr == NULL))
return (__pthread_mutex_init(mutex, &static_mattr));
else {
mattr = **mutex_attr;
mattr.m_flags |= MUTEX_FLAGS_PRIVATE;
mattrp = &mattr;
return (__pthread_mutex_init(mutex, &mattrp));
}
}
void
_thr_mutex_reinit(pthread_mutex_t *mutex)
{
_lock_reinit(&(*mutex)->m_lock, LCK_ADAPTIVE,
_thr_lock_wait, _thr_lock_wakeup);
TAILQ_INIT(&(*mutex)->m_queue);
(*mutex)->m_owner = NULL;
(*mutex)->m_count = 0;
(*mutex)->m_refcount = 0;
(*mutex)->m_prio = 0;
(*mutex)->m_saved_prio = 0;
}
int
_pthread_mutex_destroy(pthread_mutex_t *mutex)
{
struct pthread *curthread = _get_curthread();
pthread_mutex_t m;
int ret = 0;
if (mutex == NULL || *mutex == NULL)
ret = EINVAL;
else {
/* Lock the mutex structure: */
THR_LOCK_ACQUIRE(curthread, &(*mutex)->m_lock);
/*
* Check to see if this mutex is in use:
*/
if (((*mutex)->m_owner != NULL) ||
(!TAILQ_EMPTY(&(*mutex)->m_queue)) ||
((*mutex)->m_refcount != 0)) {
ret = EBUSY;
/* Unlock the mutex structure: */
THR_LOCK_RELEASE(curthread, &(*mutex)->m_lock);
} else {
/*
* Save a pointer to the mutex so it can be free'd
* and set the caller's pointer to NULL:
*/
m = *mutex;
*mutex = NULL;
/* Unlock the mutex structure: */
THR_LOCK_RELEASE(curthread, &m->m_lock);
/*
* Free the memory allocated for the mutex
* structure:
*/
MUTEX_ASSERT_NOT_OWNED(m);
MUTEX_DESTROY(m);
}
}
/* Return the completion status: */
return (ret);
}
static int
init_static(struct pthread *thread, pthread_mutex_t *mutex)
{
int ret;
THR_LOCK_ACQUIRE(thread, &_mutex_static_lock);
if (*mutex == NULL)
ret = pthread_mutex_init(mutex, NULL);
else
ret = 0;
THR_LOCK_RELEASE(thread, &_mutex_static_lock);
return (ret);
}
static int
init_static_private(struct pthread *thread, pthread_mutex_t *mutex)
{
int ret;
THR_LOCK_ACQUIRE(thread, &_mutex_static_lock);
if (*mutex == NULL)
ret = pthread_mutex_init(mutex, &static_mattr);
else
ret = 0;
THR_LOCK_RELEASE(thread, &_mutex_static_lock);
return (ret);
}
static int
mutex_trylock_common(struct pthread *curthread, pthread_mutex_t *mutex)
{
int private;
int ret = 0;
THR_ASSERT((mutex != NULL) && (*mutex != NULL),
"Uninitialized mutex in pthread_mutex_trylock_basic");
/* Lock the mutex structure: */
THR_LOCK_ACQUIRE(curthread, &(*mutex)->m_lock);
private = (*mutex)->m_flags & MUTEX_FLAGS_PRIVATE;
/*
* If the mutex was statically allocated, properly
* initialize the tail queue.
*/
if (((*mutex)->m_flags & MUTEX_FLAGS_INITED) == 0) {
TAILQ_INIT(&(*mutex)->m_queue);
MUTEX_INIT_LINK(*mutex);
(*mutex)->m_flags |= MUTEX_FLAGS_INITED;
}
/* Process according to mutex type: */
switch ((*mutex)->m_protocol) {
/* Default POSIX mutex: */
case PTHREAD_PRIO_NONE:
/* Check if this mutex is not locked: */
if ((*mutex)->m_owner == NULL) {
/* Lock the mutex for the running thread: */
(*mutex)->m_owner = curthread;
/* Add to the list of owned mutexes: */
MUTEX_ASSERT_NOT_OWNED(*mutex);
TAILQ_INSERT_TAIL(&curthread->mutexq,
(*mutex), m_qe);
} else if ((*mutex)->m_owner == curthread)
ret = mutex_self_trylock(curthread, *mutex);
else
/* Return a busy error: */
ret = EBUSY;
break;
/* POSIX priority inheritence mutex: */
case PTHREAD_PRIO_INHERIT:
/* Check if this mutex is not locked: */
if ((*mutex)->m_owner == NULL) {
/* Lock the mutex for the running thread: */
(*mutex)->m_owner = curthread;
THR_SCHED_LOCK(curthread, curthread);
/* Track number of priority mutexes owned: */
curthread->priority_mutex_count++;
/*
* The mutex takes on the attributes of the
* running thread when there are no waiters.
*/
(*mutex)->m_prio = curthread->active_priority;
(*mutex)->m_saved_prio =
curthread->inherited_priority;
curthread->inherited_priority = (*mutex)->m_prio;
THR_SCHED_UNLOCK(curthread, curthread);
/* Add to the list of owned mutexes: */
MUTEX_ASSERT_NOT_OWNED(*mutex);
TAILQ_INSERT_TAIL(&curthread->mutexq,
(*mutex), m_qe);
} else if ((*mutex)->m_owner == curthread)
ret = mutex_self_trylock(curthread, *mutex);
else
/* Return a busy error: */
ret = EBUSY;
break;
/* POSIX priority protection mutex: */
case PTHREAD_PRIO_PROTECT:
/* Check for a priority ceiling violation: */
if (curthread->active_priority > (*mutex)->m_prio)
ret = EINVAL;
/* Check if this mutex is not locked: */
else if ((*mutex)->m_owner == NULL) {
/* Lock the mutex for the running thread: */
(*mutex)->m_owner = curthread;
THR_SCHED_LOCK(curthread, curthread);
/* Track number of priority mutexes owned: */
curthread->priority_mutex_count++;
/*
* The running thread inherits the ceiling
* priority of the mutex and executes at that
* priority.
*/
curthread->active_priority = (*mutex)->m_prio;
(*mutex)->m_saved_prio =
curthread->inherited_priority;
curthread->inherited_priority =
(*mutex)->m_prio;
THR_SCHED_UNLOCK(curthread, curthread);
/* Add to the list of owned mutexes: */
MUTEX_ASSERT_NOT_OWNED(*mutex);
TAILQ_INSERT_TAIL(&curthread->mutexq,
(*mutex), m_qe);
} else if ((*mutex)->m_owner == curthread)
ret = mutex_self_trylock(curthread, *mutex);
else
/* Return a busy error: */
ret = EBUSY;
break;
/* Trap invalid mutex types: */
default:
/* Return an invalid argument error: */
ret = EINVAL;
break;
}
if (ret == 0 && private)
THR_CRITICAL_ENTER(curthread);
/* Unlock the mutex structure: */
THR_LOCK_RELEASE(curthread, &(*mutex)->m_lock);
/* Return the completion status: */
return (ret);
}
int
__pthread_mutex_trylock(pthread_mutex_t *mutex)
{
struct pthread *curthread = _get_curthread();
int ret = 0;
if (mutex == NULL)
ret = EINVAL;
/*
* If the mutex is statically initialized, perform the dynamic
* initialization:
*/
else if ((*mutex != NULL) ||
((ret = init_static(curthread, mutex)) == 0))
ret = mutex_trylock_common(curthread, mutex);
return (ret);
}
int
_pthread_mutex_trylock(pthread_mutex_t *mutex)
{
struct pthread *curthread = _get_curthread();
int ret = 0;
if (mutex == NULL)
ret = EINVAL;
/*
* If the mutex is statically initialized, perform the dynamic
* initialization marking the mutex private (delete safe):
*/
else if ((*mutex != NULL) ||
((ret = init_static_private(curthread, mutex)) == 0))
ret = mutex_trylock_common(curthread, mutex);
return (ret);
}
static int
mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m,
const struct timespec * abstime)
{
int private;
int ret = 0;
THR_ASSERT((m != NULL) && (*m != NULL),
"Uninitialized mutex in pthread_mutex_trylock_basic");
if (abstime != NULL && (abstime->tv_sec < 0 || abstime->tv_nsec < 0 ||
abstime->tv_nsec >= 1000000000))
return (EINVAL);
/* Reset the interrupted flag: */
curthread->interrupted = 0;
curthread->timeout = 0;
curthread->wakeup_time.tv_sec = -1;
private = (*m)->m_flags & MUTEX_FLAGS_PRIVATE;
/*
* Enter a loop waiting to become the mutex owner. We need a
* loop in case the waiting thread is interrupted by a signal
* to execute a signal handler. It is not (currently) possible
* to remain in the waiting queue while running a handler.
* Instead, the thread is interrupted and backed out of the
* waiting queue prior to executing the signal handler.
*/
do {
/* Lock the mutex structure: */
THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock);
/*
* If the mutex was statically allocated, properly
* initialize the tail queue.
*/
if (((*m)->m_flags & MUTEX_FLAGS_INITED) == 0) {
TAILQ_INIT(&(*m)->m_queue);
(*m)->m_flags |= MUTEX_FLAGS_INITED;
MUTEX_INIT_LINK(*m);
}
/* Process according to mutex type: */
switch ((*m)->m_protocol) {
/* Default POSIX mutex: */
case PTHREAD_PRIO_NONE:
if ((*m)->m_owner == NULL) {
/* Lock the mutex for this thread: */
(*m)->m_owner = curthread;
/* Add to the list of owned mutexes: */
MUTEX_ASSERT_NOT_OWNED(*m);
TAILQ_INSERT_TAIL(&curthread->mutexq,
(*m), m_qe);
if (private)
THR_CRITICAL_ENTER(curthread);
/* Unlock the mutex structure: */
THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
} else if ((*m)->m_owner == curthread) {
ret = mutex_self_lock(curthread, *m);
/* Unlock the mutex structure: */
THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
} else {
/*
* Join the queue of threads waiting to lock
* the mutex and save a pointer to the mutex.
*/
mutex_queue_enq(*m, curthread);
curthread->data.mutex = *m;
curthread->sigbackout = mutex_lock_backout;
/*
* This thread is active and is in a critical
* region (holding the mutex lock); we should
* be able to safely set the state.
*/
THR_SCHED_LOCK(curthread, curthread);
/* Set the wakeup time: */
if (abstime) {
curthread->wakeup_time.tv_sec =
abstime->tv_sec;
curthread->wakeup_time.tv_nsec =
abstime->tv_nsec;
}
THR_SET_STATE(curthread, PS_MUTEX_WAIT);
THR_SCHED_UNLOCK(curthread, curthread);
/* Unlock the mutex structure: */
THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
/* Schedule the next thread: */
_thr_sched_switch(curthread);
if (THR_IN_MUTEXQ(curthread)) {
THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock);
mutex_queue_remove(*m, curthread);
THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
}
/*
* Only clear these after assuring the
* thread is dequeued.
*/
curthread->data.mutex = NULL;
curthread->sigbackout = NULL;
}
break;
/* POSIX priority inheritence mutex: */
case PTHREAD_PRIO_INHERIT:
/* Check if this mutex is not locked: */
if ((*m)->m_owner == NULL) {
/* Lock the mutex for this thread: */
(*m)->m_owner = curthread;
THR_SCHED_LOCK(curthread, curthread);
/* Track number of priority mutexes owned: */
curthread->priority_mutex_count++;
/*
* The mutex takes on attributes of the
* running thread when there are no waiters.
* Make sure the thread's scheduling lock is
* held while priorities are adjusted.
*/
(*m)->m_prio = curthread->active_priority;
(*m)->m_saved_prio =
curthread->inherited_priority;
curthread->inherited_priority = (*m)->m_prio;
THR_SCHED_UNLOCK(curthread, curthread);
/* Add to the list of owned mutexes: */
MUTEX_ASSERT_NOT_OWNED(*m);
TAILQ_INSERT_TAIL(&curthread->mutexq,
(*m), m_qe);
if (private)
THR_CRITICAL_ENTER(curthread);
/* Unlock the mutex structure: */
THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
} else if ((*m)->m_owner == curthread) {
ret = mutex_self_lock(curthread, *m);
/* Unlock the mutex structure: */
THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
} else {
/*
* Join the queue of threads waiting to lock
* the mutex and save a pointer to the mutex.
*/
mutex_queue_enq(*m, curthread);
curthread->data.mutex = *m;
curthread->sigbackout = mutex_lock_backout;
/*
* This thread is active and is in a critical
* region (holding the mutex lock); we should
* be able to safely set the state.
*/
if (curthread->active_priority > (*m)->m_prio)
/* Adjust priorities: */
mutex_priority_adjust(curthread, *m);
THR_SCHED_LOCK(curthread, curthread);
/* Set the wakeup time: */
if (abstime) {
curthread->wakeup_time.tv_sec =
abstime->tv_sec;
curthread->wakeup_time.tv_nsec =
abstime->tv_nsec;
}
THR_SET_STATE(curthread, PS_MUTEX_WAIT);
THR_SCHED_UNLOCK(curthread, curthread);
/* Unlock the mutex structure: */
THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
/* Schedule the next thread: */
_thr_sched_switch(curthread);
if (THR_IN_MUTEXQ(curthread)) {
THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock);
mutex_queue_remove(*m, curthread);
THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
}
/*
* Only clear these after assuring the
* thread is dequeued.
*/
curthread->data.mutex = NULL;
curthread->sigbackout = NULL;
}
break;
/* POSIX priority protection mutex: */
case PTHREAD_PRIO_PROTECT:
/* Check for a priority ceiling violation: */
if (curthread->active_priority > (*m)->m_prio) {
/* Unlock the mutex structure: */
THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
ret = EINVAL;
}
/* Check if this mutex is not locked: */
else if ((*m)->m_owner == NULL) {
/*
* Lock the mutex for the running
* thread:
*/
(*m)->m_owner = curthread;
THR_SCHED_LOCK(curthread, curthread);
/* Track number of priority mutexes owned: */
curthread->priority_mutex_count++;
/*
* The running thread inherits the ceiling
* priority of the mutex and executes at that
* priority. Make sure the thread's
* scheduling lock is held while priorities
* are adjusted.
*/
curthread->active_priority = (*m)->m_prio;
(*m)->m_saved_prio =
curthread->inherited_priority;
curthread->inherited_priority = (*m)->m_prio;
THR_SCHED_UNLOCK(curthread, curthread);
/* Add to the list of owned mutexes: */
MUTEX_ASSERT_NOT_OWNED(*m);
TAILQ_INSERT_TAIL(&curthread->mutexq,
(*m), m_qe);
if (private)
THR_CRITICAL_ENTER(curthread);
/* Unlock the mutex structure: */
THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
} else if ((*m)->m_owner == curthread) {
ret = mutex_self_lock(curthread, *m);
/* Unlock the mutex structure: */
THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
} else {
/*
* Join the queue of threads waiting to lock
* the mutex and save a pointer to the mutex.
*/
mutex_queue_enq(*m, curthread);
curthread->data.mutex = *m;
curthread->sigbackout = mutex_lock_backout;
/* Clear any previous error: */
curthread->error = 0;
/*
* This thread is active and is in a critical
* region (holding the mutex lock); we should
* be able to safely set the state.
*/
THR_SCHED_LOCK(curthread, curthread);
/* Set the wakeup time: */
if (abstime) {
curthread->wakeup_time.tv_sec =
abstime->tv_sec;
curthread->wakeup_time.tv_nsec =
abstime->tv_nsec;
}
THR_SET_STATE(curthread, PS_MUTEX_WAIT);
THR_SCHED_UNLOCK(curthread, curthread);
/* Unlock the mutex structure: */
THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
/* Schedule the next thread: */
_thr_sched_switch(curthread);
if (THR_IN_MUTEXQ(curthread)) {
THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock);
mutex_queue_remove(*m, curthread);
THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
}
/*
* Only clear these after assuring the
* thread is dequeued.
*/
curthread->data.mutex = NULL;
curthread->sigbackout = NULL;
/*
* The threads priority may have changed while
* waiting for the mutex causing a ceiling
* violation.
*/
ret = curthread->error;
curthread->error = 0;
}
break;
/* Trap invalid mutex types: */
default:
/* Unlock the mutex structure: */
THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
/* Return an invalid argument error: */
ret = EINVAL;
break;
}
} while (((*m)->m_owner != curthread) && (ret == 0) &&
(curthread->interrupted == 0) && (curthread->timeout == 0));
if (ret == 0 && (*m)->m_owner != curthread && curthread->timeout)
ret = ETIMEDOUT;
/*
* Check to see if this thread was interrupted and
* is still in the mutex queue of waiting threads:
*/
if (curthread->interrupted != 0) {
/* Remove this thread from the mutex queue. */
THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock);
if (THR_IN_SYNCQ(curthread))
mutex_queue_remove(*m, curthread);
THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
/* Check for asynchronous cancellation. */
if (curthread->continuation != NULL)
curthread->continuation((void *) curthread);
}
/* Return the completion status: */
return (ret);
}
int
__pthread_mutex_lock(pthread_mutex_t *m)
{
struct pthread *curthread;
int ret = 0;
if (_thr_initial == NULL)
_libpthread_init(NULL);
curthread = _get_curthread();
if (m == NULL)
ret = EINVAL;
/*
* If the mutex is statically initialized, perform the dynamic
* initialization:
*/
else if ((*m != NULL) || ((ret = init_static(curthread, m)) == 0))
ret = mutex_lock_common(curthread, m, NULL);
return (ret);
}
__strong_reference(__pthread_mutex_lock, _thr_mutex_lock);
int
_pthread_mutex_lock(pthread_mutex_t *m)
{
struct pthread *curthread;
int ret = 0;
if (_thr_initial == NULL)
_libpthread_init(NULL);
curthread = _get_curthread();
if (m == NULL)
ret = EINVAL;
/*
* If the mutex is statically initialized, perform the dynamic
* initialization marking it private (delete safe):
*/
else if ((*m != NULL) ||
((ret = init_static_private(curthread, m)) == 0))
ret = mutex_lock_common(curthread, m, NULL);
return (ret);
}
int
__pthread_mutex_timedlock(pthread_mutex_t *m,
const struct timespec *abs_timeout)
{
struct pthread *curthread;
int ret = 0;
if (_thr_initial == NULL)
_libpthread_init(NULL);
curthread = _get_curthread();
if (m == NULL)
ret = EINVAL;
/*
* If the mutex is statically initialized, perform the dynamic
* initialization:
*/
else if ((*m != NULL) || ((ret = init_static(curthread, m)) == 0))
ret = mutex_lock_common(curthread, m, abs_timeout);
return (ret);
}
int
_pthread_mutex_timedlock(pthread_mutex_t *m,
const struct timespec *abs_timeout)
{
struct pthread *curthread;
int ret = 0;
if (_thr_initial == NULL)
_libpthread_init(NULL);
curthread = _get_curthread();
if (m == NULL)
ret = EINVAL;
/*
* If the mutex is statically initialized, perform the dynamic
* initialization marking it private (delete safe):
*/
else if ((*m != NULL) ||
((ret = init_static_private(curthread, m)) == 0))
ret = mutex_lock_common(curthread, m, abs_timeout);
return (ret);
}
int
_pthread_mutex_unlock(pthread_mutex_t *m)
{
return (mutex_unlock_common(m, /* add reference */ 0));
}
__strong_reference(_pthread_mutex_unlock, _thr_mutex_unlock);
int
_mutex_cv_unlock(pthread_mutex_t *m)
{
return (mutex_unlock_common(m, /* add reference */ 1));
}
int
_mutex_cv_lock(pthread_mutex_t *m)
{
struct pthread *curthread;
int ret;
curthread = _get_curthread();
if ((ret = _pthread_mutex_lock(m)) == 0) {
THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock);
(*m)->m_refcount--;
THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
}
return (ret);
}
static inline int
mutex_self_trylock(struct pthread *curthread, pthread_mutex_t m)
{
int ret = 0;
switch (m->m_type) {
/* case PTHREAD_MUTEX_DEFAULT: */
case PTHREAD_MUTEX_ERRORCHECK:
case PTHREAD_MUTEX_NORMAL:
case PTHREAD_MUTEX_ADAPTIVE_NP:
ret = EBUSY;
break;
case PTHREAD_MUTEX_RECURSIVE:
/* Increment the lock count: */
m->m_count++;
break;
default:
/* Trap invalid mutex types; */
ret = EINVAL;
}
return (ret);
}
static inline int
mutex_self_lock(struct pthread *curthread, pthread_mutex_t m)
{
int ret = 0;
/*
* Don't allow evil recursive mutexes for private use
* in libc and libpthread.
*/
if (m->m_flags & MUTEX_FLAGS_PRIVATE)
PANIC("Recurse on a private mutex.");
switch (m->m_type) {
/* case PTHREAD_MUTEX_DEFAULT: */
case PTHREAD_MUTEX_ERRORCHECK:
case PTHREAD_MUTEX_ADAPTIVE_NP:
/*
* POSIX specifies that mutexes should return EDEADLK if a
* recursive lock is detected.
*/
ret = EDEADLK;
break;
case PTHREAD_MUTEX_NORMAL:
/*
* What SS2 define as a 'normal' mutex. Intentionally
* deadlock on attempts to get a lock you already own.
*/
THR_SCHED_LOCK(curthread, curthread);
THR_SET_STATE(curthread, PS_DEADLOCK);
THR_SCHED_UNLOCK(curthread, curthread);
/* Unlock the mutex structure: */
THR_LOCK_RELEASE(curthread, &m->m_lock);
/* Schedule the next thread: */
_thr_sched_switch(curthread);
break;
case PTHREAD_MUTEX_RECURSIVE:
/* Increment the lock count: */
m->m_count++;
break;
default:
/* Trap invalid mutex types; */
ret = EINVAL;
}
return (ret);
}
static int
mutex_unlock_common(pthread_mutex_t *m, int add_reference)
{
struct pthread *curthread = _get_curthread();
struct kse_mailbox *kmbx = NULL;
int ret = 0;
if (m == NULL || *m == NULL)
ret = EINVAL;
else {
/* Lock the mutex structure: */
THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock);
/* Process according to mutex type: */
switch ((*m)->m_protocol) {
/* Default POSIX mutex: */
case PTHREAD_PRIO_NONE:
/*
* Check if the running thread is not the owner of the
* mutex:
*/
if ((*m)->m_owner != curthread)
ret = EPERM;
else if (((*m)->m_type == PTHREAD_MUTEX_RECURSIVE) &&
((*m)->m_count > 0))
/* Decrement the count: */
(*m)->m_count--;
else {
/*
* Clear the count in case this is a recursive
* mutex.
*/
(*m)->m_count = 0;
/* Remove the mutex from the threads queue. */
MUTEX_ASSERT_IS_OWNED(*m);
TAILQ_REMOVE(&(*m)->m_owner->mutexq,
(*m), m_qe);
MUTEX_INIT_LINK(*m);
/*
* Hand off the mutex to the next waiting
* thread:
*/
kmbx = mutex_handoff(curthread, *m);
}
break;
/* POSIX priority inheritence mutex: */
case PTHREAD_PRIO_INHERIT:
/*
* Check if the running thread is not the owner of the
* mutex:
*/
if ((*m)->m_owner != curthread)
ret = EPERM;
else if (((*m)->m_type == PTHREAD_MUTEX_RECURSIVE) &&
((*m)->m_count > 0))
/* Decrement the count: */
(*m)->m_count--;
else {
/*
* Clear the count in case this is recursive
* mutex.
*/
(*m)->m_count = 0;
/*
* Restore the threads inherited priority and
* recompute the active priority (being careful
* not to override changes in the threads base
* priority subsequent to locking the mutex).
*/
THR_SCHED_LOCK(curthread, curthread);
curthread->inherited_priority =
(*m)->m_saved_prio;
curthread->active_priority =
MAX(curthread->inherited_priority,
curthread->base_priority);
/*
* This thread now owns one less priority mutex.
*/
curthread->priority_mutex_count--;
THR_SCHED_UNLOCK(curthread, curthread);
/* Remove the mutex from the threads queue. */
MUTEX_ASSERT_IS_OWNED(*m);
TAILQ_REMOVE(&(*m)->m_owner->mutexq,
(*m), m_qe);
MUTEX_INIT_LINK(*m);
/*
* Hand off the mutex to the next waiting
* thread:
*/
kmbx = mutex_handoff(curthread, *m);
}
break;
/* POSIX priority ceiling mutex: */
case PTHREAD_PRIO_PROTECT:
/*
* Check if the running thread is not the owner of the
* mutex:
*/
if ((*m)->m_owner != curthread)
ret = EPERM;
else if (((*m)->m_type == PTHREAD_MUTEX_RECURSIVE) &&
((*m)->m_count > 0))
/* Decrement the count: */
(*m)->m_count--;
else {
/*
* Clear the count in case this is a recursive
* mutex.
*/
(*m)->m_count = 0;
/*
* Restore the threads inherited priority and
* recompute the active priority (being careful
* not to override changes in the threads base
* priority subsequent to locking the mutex).
*/
THR_SCHED_LOCK(curthread, curthread);
curthread->inherited_priority =
(*m)->m_saved_prio;
curthread->active_priority =
MAX(curthread->inherited_priority,
curthread->base_priority);
/*
* This thread now owns one less priority mutex.
*/
curthread->priority_mutex_count--;
THR_SCHED_UNLOCK(curthread, curthread);
/* Remove the mutex from the threads queue. */
MUTEX_ASSERT_IS_OWNED(*m);
TAILQ_REMOVE(&(*m)->m_owner->mutexq,
(*m), m_qe);
MUTEX_INIT_LINK(*m);
/*
* Hand off the mutex to the next waiting
* thread:
*/
kmbx = mutex_handoff(curthread, *m);
}
break;
/* Trap invalid mutex types: */
default:
/* Return an invalid argument error: */
ret = EINVAL;
break;
}
if ((ret == 0) && (add_reference != 0))
/* Increment the reference count: */
(*m)->m_refcount++;
/* Leave the critical region if this is a private mutex. */
if ((ret == 0) && ((*m)->m_flags & MUTEX_FLAGS_PRIVATE))
THR_CRITICAL_LEAVE(curthread);
/* Unlock the mutex structure: */
THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
if (kmbx != NULL)
kse_wakeup(kmbx);
}
/* Return the completion status: */
return (ret);
}
/*
* This function is called when a change in base priority occurs for
* a thread that is holding or waiting for a priority protection or
* inheritence mutex. A change in a threads base priority can effect
* changes to active priorities of other threads and to the ordering
* of mutex locking by waiting threads.
*
* This must be called without the target thread's scheduling lock held.
*/
void
_mutex_notify_priochange(struct pthread *curthread, struct pthread *pthread,
int propagate_prio)
{
struct pthread_mutex *m;
/* Adjust the priorites of any owned priority mutexes: */
if (pthread->priority_mutex_count > 0) {
/*
* Rescan the mutexes owned by this thread and correct
* their priorities to account for this threads change
* in priority. This has the side effect of changing
* the threads active priority.
*
* Be sure to lock the first mutex in the list of owned
* mutexes. This acts as a barrier against another
* simultaneous call to change the threads priority
* and from the owning thread releasing the mutex.
*/
m = TAILQ_FIRST(&pthread->mutexq);
if (m != NULL) {
THR_LOCK_ACQUIRE(curthread, &m->m_lock);
/*
* Make sure the thread still owns the lock.
*/
if (m == TAILQ_FIRST(&pthread->mutexq))
mutex_rescan_owned(curthread, pthread,
/* rescan all owned */ NULL);
THR_LOCK_RELEASE(curthread, &m->m_lock);
}
}
/*
* If this thread is waiting on a priority inheritence mutex,
* check for priority adjustments. A change in priority can
* also cause a ceiling violation(*) for a thread waiting on
* a priority protection mutex; we don't perform the check here
* as it is done in pthread_mutex_unlock.
*
* (*) It should be noted that a priority change to a thread
* _after_ taking and owning a priority ceiling mutex
* does not affect ownership of that mutex; the ceiling
* priority is only checked before mutex ownership occurs.
*/
if (propagate_prio != 0) {
/*
* Lock the thread's scheduling queue. This is a bit
* convoluted; the "in synchronization queue flag" can
* only be cleared with both the thread's scheduling and
* mutex locks held. The thread's pointer to the wanted
* mutex is guaranteed to be valid during this time.
*/
THR_SCHED_LOCK(curthread, pthread);
if (((pthread->sflags & THR_FLAGS_IN_SYNCQ) == 0) ||
((m = pthread->data.mutex) == NULL))
THR_SCHED_UNLOCK(curthread, pthread);
else {
/*
* This thread is currently waiting on a mutex; unlock
* the scheduling queue lock and lock the mutex. We
* can't hold both at the same time because the locking
* order could cause a deadlock.
*/
THR_SCHED_UNLOCK(curthread, pthread);
THR_LOCK_ACQUIRE(curthread, &m->m_lock);
/*
* Check to make sure this thread is still in the
* same state (the lock above can yield the CPU to
* another thread or the thread may be running on
* another CPU).
*/
if (((pthread->sflags & THR_FLAGS_IN_SYNCQ) != 0) &&
(pthread->data.mutex == m)) {
/*
* Remove and reinsert this thread into
* the list of waiting threads to preserve
* decreasing priority order.
*/
mutex_queue_remove(m, pthread);
mutex_queue_enq(m, pthread);
if (m->m_protocol == PTHREAD_PRIO_INHERIT)
/* Adjust priorities: */
mutex_priority_adjust(curthread, m);
}
/* Unlock the mutex structure: */
THR_LOCK_RELEASE(curthread, &m->m_lock);
}
}
}
/*
* Called when a new thread is added to the mutex waiting queue or
* when a threads priority changes that is already in the mutex
* waiting queue.
*
* This must be called with the mutex locked by the current thread.
*/
static void
mutex_priority_adjust(struct pthread *curthread, pthread_mutex_t mutex)
{
pthread_mutex_t m = mutex;
struct pthread *pthread_next, *pthread = mutex->m_owner;
int done, temp_prio;
/*
* Calculate the mutex priority as the maximum of the highest
* active priority of any waiting threads and the owning threads
* active priority(*).
*
* (*) Because the owning threads current active priority may
* reflect priority inherited from this mutex (and the mutex
* priority may have changed) we must recalculate the active
* priority based on the threads saved inherited priority
* and its base priority.
*/
pthread_next = TAILQ_FIRST(&m->m_queue); /* should never be NULL */
temp_prio = MAX(pthread_next->active_priority,
MAX(m->m_saved_prio, pthread->base_priority));
/* See if this mutex really needs adjusting: */
if (temp_prio == m->m_prio)
/* No need to propagate the priority: */
return;
/* Set new priority of the mutex: */
m->m_prio = temp_prio;
/*
* Don't unlock the mutex passed in as an argument. It is
* expected to be locked and unlocked by the caller.
*/
done = 1;
do {
/*
* Save the threads priority before rescanning the
* owned mutexes:
*/
temp_prio = pthread->active_priority;
/*
* Fix the priorities for all mutexes held by the owning
* thread since taking this mutex. This also has a
* potential side-effect of changing the threads priority.
*
* At this point the mutex is locked by the current thread.
* The owning thread can't release the mutex until it is
* unlocked, so we should be able to safely walk its list
* of owned mutexes.
*/
mutex_rescan_owned(curthread, pthread, m);
/*
* If this isn't the first time through the loop,
* the current mutex needs to be unlocked.
*/
if (done == 0)
THR_LOCK_RELEASE(curthread, &m->m_lock);
/* Assume we're done unless told otherwise: */
done = 1;
/*
* If the thread is currently waiting on a mutex, check
* to see if the threads new priority has affected the
* priority of the mutex.
*/
if ((temp_prio != pthread->active_priority) &&
((pthread->sflags & THR_FLAGS_IN_SYNCQ) != 0) &&
((m = pthread->data.mutex) != NULL) &&
(m->m_protocol == PTHREAD_PRIO_INHERIT)) {
/* Lock the mutex structure: */
THR_LOCK_ACQUIRE(curthread, &m->m_lock);
/*
* Make sure the thread is still waiting on the
* mutex:
*/
if (((pthread->sflags & THR_FLAGS_IN_SYNCQ) != 0) &&
(m == pthread->data.mutex)) {
/*
* The priority for this thread has changed.
* Remove and reinsert this thread into the
* list of waiting threads to preserve
* decreasing priority order.
*/
mutex_queue_remove(m, pthread);
mutex_queue_enq(m, pthread);
/*
* Grab the waiting thread with highest
* priority:
*/
pthread_next = TAILQ_FIRST(&m->m_queue);
/*
* Calculate the mutex priority as the maximum
* of the highest active priority of any
* waiting threads and the owning threads
* active priority.
*/
temp_prio = MAX(pthread_next->active_priority,
MAX(m->m_saved_prio,
m->m_owner->base_priority));
if (temp_prio != m->m_prio) {
/*
* The priority needs to be propagated
* to the mutex this thread is waiting
* on and up to the owner of that mutex.
*/
m->m_prio = temp_prio;
pthread = m->m_owner;
/* We're not done yet: */
done = 0;
}
}
/* Only release the mutex if we're done: */
if (done != 0)
THR_LOCK_RELEASE(curthread, &m->m_lock);
}
} while (done == 0);
}
static void
mutex_rescan_owned(struct pthread *curthread, struct pthread *pthread,
struct pthread_mutex *mutex)
{
struct pthread_mutex *m;
struct pthread *pthread_next;
int active_prio, inherited_prio;
/*
* Start walking the mutexes the thread has taken since
* taking this mutex.
*/
if (mutex == NULL) {
/*
* A null mutex means start at the beginning of the owned
* mutex list.
*/
m = TAILQ_FIRST(&pthread->mutexq);
/* There is no inherited priority yet. */
inherited_prio = 0;
} else {
/*
* The caller wants to start after a specific mutex. It
* is assumed that this mutex is a priority inheritence
* mutex and that its priority has been correctly
* calculated.
*/
m = TAILQ_NEXT(mutex, m_qe);
/* Start inheriting priority from the specified mutex. */
inherited_prio = mutex->m_prio;
}
active_prio = MAX(inherited_prio, pthread->base_priority);
for (; m != NULL; m = TAILQ_NEXT(m, m_qe)) {
/*
* We only want to deal with priority inheritence
* mutexes. This might be optimized by only placing
* priority inheritence mutexes into the owned mutex
* list, but it may prove to be useful having all
* owned mutexes in this list. Consider a thread
* exiting while holding mutexes...
*/
if (m->m_protocol == PTHREAD_PRIO_INHERIT) {
/*
* Fix the owners saved (inherited) priority to
* reflect the priority of the previous mutex.
*/
m->m_saved_prio = inherited_prio;
if ((pthread_next = TAILQ_FIRST(&m->m_queue)) != NULL)
/* Recalculate the priority of the mutex: */
m->m_prio = MAX(active_prio,
pthread_next->active_priority);
else
m->m_prio = active_prio;
/* Recalculate new inherited and active priorities: */
inherited_prio = m->m_prio;
active_prio = MAX(m->m_prio, pthread->base_priority);
}
}
/*
* Fix the threads inherited priority and recalculate its
* active priority.
*/
pthread->inherited_priority = inherited_prio;
active_prio = MAX(inherited_prio, pthread->base_priority);
if (active_prio != pthread->active_priority) {
/* Lock the thread's scheduling queue: */
THR_SCHED_LOCK(curthread, pthread);
if ((pthread->flags & THR_FLAGS_IN_RUNQ) == 0) {
/*
* This thread is not in a run queue. Just set
* its active priority.
*/
pthread->active_priority = active_prio;
}
else {
/*
* This thread is in a run queue. Remove it from
* the queue before changing its priority:
*/
THR_RUNQ_REMOVE(pthread);
/*
* POSIX states that if the priority is being
* lowered, the thread must be inserted at the
* head of the queue for its priority if it owns
* any priority protection or inheritence mutexes.
*/
if ((active_prio < pthread->active_priority) &&
(pthread->priority_mutex_count > 0)) {
/* Set the new active priority. */
pthread->active_priority = active_prio;
THR_RUNQ_INSERT_HEAD(pthread);
} else {
/* Set the new active priority. */
pthread->active_priority = active_prio;
THR_RUNQ_INSERT_TAIL(pthread);
}
}
THR_SCHED_UNLOCK(curthread, pthread);
}
}
void
_mutex_unlock_private(pthread_t pthread)
{
struct pthread_mutex *m, *m_next;
for (m = TAILQ_FIRST(&pthread->mutexq); m != NULL; m = m_next) {
m_next = TAILQ_NEXT(m, m_qe);
if ((m->m_flags & MUTEX_FLAGS_PRIVATE) != 0)
pthread_mutex_unlock(&m);
}
}
/*
* This is called by the current thread when it wants to back out of a
* mutex_lock in order to run a signal handler.
*/
static void
mutex_lock_backout(void *arg)
{
struct pthread *curthread = (struct pthread *)arg;
struct pthread_mutex *m;
if ((curthread->sflags & THR_FLAGS_IN_SYNCQ) != 0) {
/*
* Any other thread may clear the "in sync queue flag",
* but only the current thread can clear the pointer
* to the mutex. So if the flag is set, we can
* guarantee that the pointer to the mutex is valid.
* The only problem may be if the mutex is destroyed
* out from under us, but that should be considered
* an application bug.
*/
m = curthread->data.mutex;
/* Lock the mutex structure: */
THR_LOCK_ACQUIRE(curthread, &m->m_lock);
/*
* Check to make sure this thread doesn't already own
* the mutex. Since mutexes are unlocked with direct
* handoffs, it is possible the previous owner gave it
* to us after we checked the sync queue flag and before
* we locked the mutex structure.
*/
if (m->m_owner == curthread) {
THR_LOCK_RELEASE(curthread, &m->m_lock);
mutex_unlock_common(&m, /* add_reference */ 0);
} else {
/*
* Remove ourselves from the mutex queue and
* clear the pointer to the mutex. We may no
* longer be in the mutex queue, but the removal
* function will DTRT.
*/
mutex_queue_remove(m, curthread);
curthread->data.mutex = NULL;
THR_LOCK_RELEASE(curthread, &m->m_lock);
}
}
/* No need to call this again. */
curthread->sigbackout = NULL;
}
/*
* Dequeue a waiting thread from the head of a mutex queue in descending
* priority order.
*
* In order to properly dequeue a thread from the mutex queue and
* make it runnable without the possibility of errant wakeups, it
* is necessary to lock the thread's scheduling queue while also
* holding the mutex lock.
*/
static struct kse_mailbox *
mutex_handoff(struct pthread *curthread, struct pthread_mutex *mutex)
{
struct kse_mailbox *kmbx = NULL;
struct pthread *pthread;
/* Keep dequeueing until we find a valid thread: */
mutex->m_owner = NULL;
pthread = TAILQ_FIRST(&mutex->m_queue);
while (pthread != NULL) {
/* Take the thread's scheduling lock: */
THR_SCHED_LOCK(curthread, pthread);
/* Remove the thread from the mutex queue: */
TAILQ_REMOVE(&mutex->m_queue, pthread, sqe);
pthread->sflags &= ~THR_FLAGS_IN_SYNCQ;
/*
* Only exit the loop if the thread hasn't been
* cancelled.
*/
switch (mutex->m_protocol) {
case PTHREAD_PRIO_NONE:
/*
* Assign the new owner and add the mutex to the
* thread's list of owned mutexes.
*/
mutex->m_owner = pthread;
TAILQ_INSERT_TAIL(&pthread->mutexq, mutex, m_qe);
break;
case PTHREAD_PRIO_INHERIT:
/*
* Assign the new owner and add the mutex to the
* thread's list of owned mutexes.
*/
mutex->m_owner = pthread;
TAILQ_INSERT_TAIL(&pthread->mutexq, mutex, m_qe);
/* Track number of priority mutexes owned: */
pthread->priority_mutex_count++;
/*
* Set the priority of the mutex. Since our waiting
* threads are in descending priority order, the
* priority of the mutex becomes the active priority
* of the thread we just dequeued.
*/
mutex->m_prio = pthread->active_priority;
/* Save the owning threads inherited priority: */
mutex->m_saved_prio = pthread->inherited_priority;
/*
* The owning threads inherited priority now becomes
* his active priority (the priority of the mutex).
*/
pthread->inherited_priority = mutex->m_prio;
break;
case PTHREAD_PRIO_PROTECT:
if (pthread->active_priority > mutex->m_prio) {
/*
* Either the mutex ceiling priority has
* been lowered and/or this threads priority
* has been raised subsequent to the thread
* being queued on the waiting list.
*/
pthread->error = EINVAL;
}
else {
/*
* Assign the new owner and add the mutex
* to the thread's list of owned mutexes.
*/
mutex->m_owner = pthread;
TAILQ_INSERT_TAIL(&pthread->mutexq,
mutex, m_qe);
/* Track number of priority mutexes owned: */
pthread->priority_mutex_count++;
/*
* Save the owning threads inherited
* priority:
*/
mutex->m_saved_prio =
pthread->inherited_priority;
/*
* The owning thread inherits the ceiling
* priority of the mutex and executes at
* that priority:
*/
pthread->inherited_priority = mutex->m_prio;
pthread->active_priority = mutex->m_prio;
}
break;
}
/* Make the thread runnable and unlock the scheduling queue: */
kmbx = _thr_setrunnable_unlocked(pthread);
/* Add a preemption point. */
if ((curthread->kseg == pthread->kseg) &&
(pthread->active_priority > curthread->active_priority))
curthread->critical_yield = 1;
if (mutex->m_owner == pthread) {
/* We're done; a valid owner was found. */
if (mutex->m_flags & MUTEX_FLAGS_PRIVATE)
THR_CRITICAL_ENTER(pthread);
THR_SCHED_UNLOCK(curthread, pthread);
break;
}
THR_SCHED_UNLOCK(curthread, pthread);
/* Get the next thread from the waiting queue: */
pthread = TAILQ_NEXT(pthread, sqe);
}
if ((pthread == NULL) && (mutex->m_protocol == PTHREAD_PRIO_INHERIT))
/* This mutex has no priority: */
mutex->m_prio = 0;
return (kmbx);
}
/*
* Dequeue a waiting thread from the head of a mutex queue in descending
* priority order.
*/
static inline pthread_t
mutex_queue_deq(struct pthread_mutex *mutex)
{
pthread_t pthread;
while ((pthread = TAILQ_FIRST(&mutex->m_queue)) != NULL) {
TAILQ_REMOVE(&mutex->m_queue, pthread, sqe);
pthread->sflags &= ~THR_FLAGS_IN_SYNCQ;
/*
* Only exit the loop if the thread hasn't been
* cancelled.
*/
if (pthread->interrupted == 0)
break;
}
return (pthread);
}
/*
* Remove a waiting thread from a mutex queue in descending priority order.
*/
static inline void
mutex_queue_remove(pthread_mutex_t mutex, pthread_t pthread)
{
if ((pthread->sflags & THR_FLAGS_IN_SYNCQ) != 0) {
TAILQ_REMOVE(&mutex->m_queue, pthread, sqe);
pthread->sflags &= ~THR_FLAGS_IN_SYNCQ;
}
}
/*
* Enqueue a waiting thread to a queue in descending priority order.
*/
static inline void
mutex_queue_enq(pthread_mutex_t mutex, pthread_t pthread)
{
pthread_t tid = TAILQ_LAST(&mutex->m_queue, mutex_head);
THR_ASSERT_NOT_IN_SYNCQ(pthread);
/*
* For the common case of all threads having equal priority,
* we perform a quick check against the priority of the thread
* at the tail of the queue.
*/
if ((tid == NULL) || (pthread->active_priority <= tid->active_priority))
TAILQ_INSERT_TAIL(&mutex->m_queue, pthread, sqe);
else {
tid = TAILQ_FIRST(&mutex->m_queue);
while (pthread->active_priority <= tid->active_priority)
tid = TAILQ_NEXT(tid, sqe);
TAILQ_INSERT_BEFORE(tid, pthread, sqe);
}
pthread->sflags |= THR_FLAGS_IN_SYNCQ;
}
Reference in New Issue View Git Blame Copy Permalink