/* * Copyright (c) 1995 John Birrell . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by John Birrell. * 4. Neither the name of the author nor the names of any co-contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include #include #include #include #include "thr_private.h" #if defined(_PTHREADS_INVARIANTS) #define MUTEX_INIT_LINK(m) do { \ (m)->m_qe.tqe_prev = NULL; \ (m)->m_qe.tqe_next = NULL; \ } while (0) #define MUTEX_ASSERT_IS_OWNED(m) do { \ if ((m)->m_qe.tqe_prev == NULL) \ PANIC("mutex is not on list"); \ } while (0) #define MUTEX_ASSERT_NOT_OWNED(m) do { \ if (((m)->m_qe.tqe_prev != NULL) || \ ((m)->m_qe.tqe_next != NULL)) \ PANIC("mutex is on list"); \ } while (0) #define THR_ASSERT_NOT_IN_SYNCQ(thr) do { \ THR_ASSERT(((thr)->sflags & THR_FLAGS_IN_SYNCQ) == 0, \ "thread in syncq when it shouldn't be."); \ } while (0); #else #define MUTEX_INIT_LINK(m) #define MUTEX_ASSERT_IS_OWNED(m) #define MUTEX_ASSERT_NOT_OWNED(m) #define THR_ASSERT_NOT_IN_SYNCQ(thr) #endif #define THR_IN_MUTEXQ(thr) (((thr)->sflags & THR_FLAGS_IN_SYNCQ) != 0) #define MUTEX_DESTROY(m) do { \ _lock_destroy(&(m)->m_lock); \ free(m); \ } while (0) /* * Prototypes */ static struct kse_mailbox *mutex_handoff(struct pthread *, struct pthread_mutex *); static inline int mutex_self_trylock(struct pthread *, pthread_mutex_t); static inline int mutex_self_lock(struct pthread *, pthread_mutex_t); static int mutex_unlock_common(pthread_mutex_t *, int); static void mutex_priority_adjust(struct pthread *, pthread_mutex_t); static void mutex_rescan_owned (struct pthread *, struct pthread *, struct pthread_mutex *); static inline pthread_t mutex_queue_deq(pthread_mutex_t); static inline void mutex_queue_remove(pthread_mutex_t, pthread_t); static inline void mutex_queue_enq(pthread_mutex_t, pthread_t); static struct pthread_mutex_attr static_mutex_attr = PTHREAD_MUTEXATTR_STATIC_INITIALIZER; static pthread_mutexattr_t static_mattr = &static_mutex_attr; /* Single underscore versions provided for libc internal usage: */ __weak_reference(__pthread_mutex_lock, pthread_mutex_lock); __weak_reference(__pthread_mutex_timedlock, pthread_mutex_timedlock); __weak_reference(__pthread_mutex_trylock, pthread_mutex_trylock); /* No difference between libc and application usage of these: */ __weak_reference(_pthread_mutex_init, pthread_mutex_init); __weak_reference(_pthread_mutex_destroy, pthread_mutex_destroy); __weak_reference(_pthread_mutex_unlock, pthread_mutex_unlock); int _pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *mutex_attr) { struct pthread_mutex *pmutex; enum pthread_mutextype type; int protocol; int ceiling; int flags; int ret = 0; if (mutex == NULL) ret = EINVAL; /* Check if default mutex attributes: */ else if (mutex_attr == NULL || *mutex_attr == NULL) { /* Default to a (error checking) POSIX mutex: */ type = PTHREAD_MUTEX_ERRORCHECK; protocol = PTHREAD_PRIO_NONE; ceiling = THR_MAX_PRIORITY; flags = 0; } /* Check mutex type: */ else if (((*mutex_attr)->m_type < PTHREAD_MUTEX_ERRORCHECK) || ((*mutex_attr)->m_type >= MUTEX_TYPE_MAX)) /* Return an invalid argument error: */ ret = EINVAL; /* Check mutex protocol: */ else if (((*mutex_attr)->m_protocol < PTHREAD_PRIO_NONE) || ((*mutex_attr)->m_protocol > PTHREAD_MUTEX_RECURSIVE)) /* Return an invalid argument error: */ ret = EINVAL; else { /* Use the requested mutex type and protocol: */ type = (*mutex_attr)->m_type; protocol = (*mutex_attr)->m_protocol; ceiling = (*mutex_attr)->m_ceiling; flags = (*mutex_attr)->m_flags; } /* Check no errors so far: */ if (ret == 0) { if ((pmutex = (pthread_mutex_t) malloc(sizeof(struct pthread_mutex))) == NULL) ret = ENOMEM; else if (_lock_init(&pmutex->m_lock, LCK_ADAPTIVE, _thr_lock_wait, _thr_lock_wakeup) != 0) { free(pmutex); *mutex = NULL; ret = ENOMEM; } else { /* Set the mutex flags: */ pmutex->m_flags = flags; /* Process according to mutex type: */ switch (type) { /* case PTHREAD_MUTEX_DEFAULT: */ case PTHREAD_MUTEX_ERRORCHECK: case PTHREAD_MUTEX_NORMAL: /* Nothing to do here. */ break; /* Single UNIX Spec 2 recursive mutex: */ case PTHREAD_MUTEX_RECURSIVE: /* Reset the mutex count: */ pmutex->m_count = 0; break; /* Trap invalid mutex types: */ default: /* Return an invalid argument error: */ ret = EINVAL; break; } if (ret == 0) { /* Initialise the rest of the mutex: */ TAILQ_INIT(&pmutex->m_queue); pmutex->m_flags |= MUTEX_FLAGS_INITED; pmutex->m_owner = NULL; pmutex->m_type = type; pmutex->m_protocol = protocol; pmutex->m_refcount = 0; if (protocol == PTHREAD_PRIO_PROTECT) pmutex->m_prio = ceiling; else pmutex->m_prio = -1; pmutex->m_saved_prio = 0; MUTEX_INIT_LINK(pmutex); *mutex = pmutex; } else { /* Free the mutex lock structure: */ MUTEX_DESTROY(pmutex); *mutex = NULL; } } } /* Return the completion status: */ return (ret); } void _thr_mutex_reinit(pthread_mutex_t *mutex) { _lock_reinit(&(*mutex)->m_lock, LCK_ADAPTIVE, _thr_lock_wait, _thr_lock_wakeup); TAILQ_INIT(&(*mutex)->m_queue); (*mutex)->m_owner = NULL; (*mutex)->m_count = 0; (*mutex)->m_refcount = 0; (*mutex)->m_prio = 0; (*mutex)->m_saved_prio = 0; } int _pthread_mutex_destroy(pthread_mutex_t *mutex) { struct pthread *curthread = _get_curthread(); pthread_mutex_t m; int ret = 0; if (mutex == NULL || *mutex == NULL) ret = EINVAL; else { /* Lock the mutex structure: */ THR_LOCK_ACQUIRE(curthread, &(*mutex)->m_lock); /* * Check to see if this mutex is in use: */ if (((*mutex)->m_owner != NULL) || (TAILQ_FIRST(&(*mutex)->m_queue) != NULL) || ((*mutex)->m_refcount != 0)) { ret = EBUSY; /* Unlock the mutex structure: */ THR_LOCK_RELEASE(curthread, &(*mutex)->m_lock); } else { /* * Save a pointer to the mutex so it can be free'd * and set the caller's pointer to NULL: */ m = *mutex; *mutex = NULL; /* Unlock the mutex structure: */ THR_LOCK_RELEASE(curthread, &m->m_lock); /* * Free the memory allocated for the mutex * structure: */ MUTEX_ASSERT_NOT_OWNED(m); MUTEX_DESTROY(m); } } /* Return the completion status: */ return (ret); } static int init_static(struct pthread *thread, pthread_mutex_t *mutex) { int ret; THR_LOCK_ACQUIRE(thread, &_mutex_static_lock); if (*mutex == NULL) ret = pthread_mutex_init(mutex, NULL); else ret = 0; THR_LOCK_RELEASE(thread, &_mutex_static_lock); return (ret); } static int init_static_private(struct pthread *thread, pthread_mutex_t *mutex) { int ret; THR_LOCK_ACQUIRE(thread, &_mutex_static_lock); if (*mutex == NULL) ret = pthread_mutex_init(mutex, &static_mattr); else ret = 0; THR_LOCK_RELEASE(thread, &_mutex_static_lock); return (ret); } static int mutex_trylock_common(struct pthread *curthread, pthread_mutex_t *mutex) { int ret = 0; THR_ASSERT((mutex != NULL) && (*mutex != NULL), "Uninitialized mutex in pthread_mutex_trylock_basic"); /* Lock the mutex structure: */ THR_LOCK_ACQUIRE(curthread, &(*mutex)->m_lock); /* * If the mutex was statically allocated, properly * initialize the tail queue. */ if (((*mutex)->m_flags & MUTEX_FLAGS_INITED) == 0) { TAILQ_INIT(&(*mutex)->m_queue); MUTEX_INIT_LINK(*mutex); (*mutex)->m_flags |= MUTEX_FLAGS_INITED; } /* Process according to mutex type: */ switch ((*mutex)->m_protocol) { /* Default POSIX mutex: */ case PTHREAD_PRIO_NONE: /* Check if this mutex is not locked: */ if ((*mutex)->m_owner == NULL) { /* Lock the mutex for the running thread: */ (*mutex)->m_owner = curthread; /* Add to the list of owned mutexes: */ MUTEX_ASSERT_NOT_OWNED(*mutex); TAILQ_INSERT_TAIL(&curthread->mutexq, (*mutex), m_qe); } else if ((*mutex)->m_owner == curthread) ret = mutex_self_trylock(curthread, *mutex); else /* Return a busy error: */ ret = EBUSY; break; /* POSIX priority inheritence mutex: */ case PTHREAD_PRIO_INHERIT: /* Check if this mutex is not locked: */ if ((*mutex)->m_owner == NULL) { /* Lock the mutex for the running thread: */ (*mutex)->m_owner = curthread; THR_SCHED_LOCK(curthread, curthread); /* Track number of priority mutexes owned: */ curthread->priority_mutex_count++; /* * The mutex takes on the attributes of the * running thread when there are no waiters. */ (*mutex)->m_prio = curthread->active_priority; (*mutex)->m_saved_prio = curthread->inherited_priority; curthread->inherited_priority = (*mutex)->m_prio; THR_SCHED_UNLOCK(curthread, curthread); /* Add to the list of owned mutexes: */ MUTEX_ASSERT_NOT_OWNED(*mutex); TAILQ_INSERT_TAIL(&curthread->mutexq, (*mutex), m_qe); } else if ((*mutex)->m_owner == curthread) ret = mutex_self_trylock(curthread, *mutex); else /* Return a busy error: */ ret = EBUSY; break; /* POSIX priority protection mutex: */ case PTHREAD_PRIO_PROTECT: /* Check for a priority ceiling violation: */ if (curthread->active_priority > (*mutex)->m_prio) ret = EINVAL; /* Check if this mutex is not locked: */ else if ((*mutex)->m_owner == NULL) { /* Lock the mutex for the running thread: */ (*mutex)->m_owner = curthread; THR_SCHED_LOCK(curthread, curthread); /* Track number of priority mutexes owned: */ curthread->priority_mutex_count++; /* * The running thread inherits the ceiling * priority of the mutex and executes at that * priority. */ curthread->active_priority = (*mutex)->m_prio; (*mutex)->m_saved_prio = curthread->inherited_priority; curthread->inherited_priority = (*mutex)->m_prio; THR_SCHED_UNLOCK(curthread, curthread); /* Add to the list of owned mutexes: */ MUTEX_ASSERT_NOT_OWNED(*mutex); TAILQ_INSERT_TAIL(&curthread->mutexq, (*mutex), m_qe); } else if ((*mutex)->m_owner == curthread) ret = mutex_self_trylock(curthread, *mutex); else /* Return a busy error: */ ret = EBUSY; break; /* Trap invalid mutex types: */ default: /* Return an invalid argument error: */ ret = EINVAL; break; } /* Unlock the mutex structure: */ THR_LOCK_RELEASE(curthread, &(*mutex)->m_lock); /* Return the completion status: */ return (ret); } int __pthread_mutex_trylock(pthread_mutex_t *mutex) { struct pthread *curthread = _get_curthread(); int ret = 0; if (mutex == NULL) ret = EINVAL; /* * If the mutex is statically initialized, perform the dynamic * initialization: */ else if ((*mutex != NULL) || ((ret = init_static(curthread, mutex)) == 0)) ret = mutex_trylock_common(curthread, mutex); return (ret); } int _pthread_mutex_trylock(pthread_mutex_t *mutex) { struct pthread *curthread = _get_curthread(); int ret = 0; if (mutex == NULL) ret = EINVAL; /* * If the mutex is statically initialized, perform the dynamic * initialization marking the mutex private (delete safe): */ else if ((*mutex != NULL) || ((ret = init_static_private(curthread, mutex)) == 0)) ret = mutex_trylock_common(curthread, mutex); return (ret); } static int mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m, const struct timespec * abstime) { int ret = 0; THR_ASSERT((m != NULL) && (*m != NULL), "Uninitialized mutex in pthread_mutex_trylock_basic"); if (abstime != NULL && (abstime->tv_sec < 0 || abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)) return (EINVAL); /* Reset the interrupted flag: */ curthread->interrupted = 0; curthread->timeout = 0; curthread->wakeup_time.tv_sec = -1; /* * Enter a loop waiting to become the mutex owner. We need a * loop in case the waiting thread is interrupted by a signal * to execute a signal handler. It is not (currently) possible * to remain in the waiting queue while running a handler. * Instead, the thread is interrupted and backed out of the * waiting queue prior to executing the signal handler. */ do { /* Lock the mutex structure: */ THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock); /* * If the mutex was statically allocated, properly * initialize the tail queue. */ if (((*m)->m_flags & MUTEX_FLAGS_INITED) == 0) { TAILQ_INIT(&(*m)->m_queue); (*m)->m_flags |= MUTEX_FLAGS_INITED; MUTEX_INIT_LINK(*m); } /* Process according to mutex type: */ switch ((*m)->m_protocol) { /* Default POSIX mutex: */ case PTHREAD_PRIO_NONE: if ((*m)->m_owner == NULL) { /* Lock the mutex for this thread: */ (*m)->m_owner = curthread; /* Add to the list of owned mutexes: */ MUTEX_ASSERT_NOT_OWNED(*m); TAILQ_INSERT_TAIL(&curthread->mutexq, (*m), m_qe); /* Unlock the mutex structure: */ THR_LOCK_RELEASE(curthread, &(*m)->m_lock); } else if ((*m)->m_owner == curthread) { ret = mutex_self_lock(curthread, *m); /* Unlock the mutex structure: */ THR_LOCK_RELEASE(curthread, &(*m)->m_lock); } else { /* Set the wakeup time: */ if (abstime) { curthread->wakeup_time.tv_sec = abstime->tv_sec; curthread->wakeup_time.tv_nsec = abstime->tv_nsec; } /* * Join the queue of threads waiting to lock * the mutex and save a pointer to the mutex. */ mutex_queue_enq(*m, curthread); curthread->data.mutex = *m; /* * This thread is active and is in a critical * region (holding the mutex lock); we should * be able to safely set the state. */ THR_SCHED_LOCK(curthread, curthread); THR_SET_STATE(curthread, PS_MUTEX_WAIT); THR_SCHED_UNLOCK(curthread, curthread); /* Unlock the mutex structure: */ THR_LOCK_RELEASE(curthread, &(*m)->m_lock); /* Schedule the next thread: */ _thr_sched_switch(curthread); curthread->data.mutex = NULL; if (THR_IN_MUTEXQ(curthread)) { THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock); mutex_queue_remove(*m, curthread); THR_LOCK_RELEASE(curthread, &(*m)->m_lock); } } break; /* POSIX priority inheritence mutex: */ case PTHREAD_PRIO_INHERIT: /* Check if this mutex is not locked: */ if ((*m)->m_owner == NULL) { /* Lock the mutex for this thread: */ (*m)->m_owner = curthread; THR_SCHED_LOCK(curthread, curthread); /* Track number of priority mutexes owned: */ curthread->priority_mutex_count++; /* * The mutex takes on attributes of the * running thread when there are no waiters. * Make sure the thread's scheduling lock is * held while priorities are adjusted. */ (*m)->m_prio = curthread->active_priority; (*m)->m_saved_prio = curthread->inherited_priority; curthread->inherited_priority = (*m)->m_prio; THR_SCHED_UNLOCK(curthread, curthread); /* Add to the list of owned mutexes: */ MUTEX_ASSERT_NOT_OWNED(*m); TAILQ_INSERT_TAIL(&curthread->mutexq, (*m), m_qe); /* Unlock the mutex structure: */ THR_LOCK_RELEASE(curthread, &(*m)->m_lock); } else if ((*m)->m_owner == curthread) { ret = mutex_self_lock(curthread, *m); /* Unlock the mutex structure: */ THR_LOCK_RELEASE(curthread, &(*m)->m_lock); } else { /* Set the wakeup time: */ if (abstime) { curthread->wakeup_time.tv_sec = abstime->tv_sec; curthread->wakeup_time.tv_nsec = abstime->tv_nsec; } /* * Join the queue of threads waiting to lock * the mutex and save a pointer to the mutex. */ mutex_queue_enq(*m, curthread); curthread->data.mutex = *m; /* * This thread is active and is in a critical * region (holding the mutex lock); we should * be able to safely set the state. */ if (curthread->active_priority > (*m)->m_prio) /* Adjust priorities: */ mutex_priority_adjust(curthread, *m); THR_SCHED_LOCK(curthread, curthread); THR_SET_STATE(curthread, PS_MUTEX_WAIT); THR_SCHED_UNLOCK(curthread, curthread); /* Unlock the mutex structure: */ THR_LOCK_RELEASE(curthread, &(*m)->m_lock); /* Schedule the next thread: */ _thr_sched_switch(curthread); curthread->data.mutex = NULL; if (THR_IN_MUTEXQ(curthread)) { THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock); mutex_queue_remove(*m, curthread); THR_LOCK_RELEASE(curthread, &(*m)->m_lock); } } break; /* POSIX priority protection mutex: */ case PTHREAD_PRIO_PROTECT: /* Check for a priority ceiling violation: */ if (curthread->active_priority > (*m)->m_prio) { /* Unlock the mutex structure: */ THR_LOCK_RELEASE(curthread, &(*m)->m_lock); ret = EINVAL; } /* Check if this mutex is not locked: */ else if ((*m)->m_owner == NULL) { /* * Lock the mutex for the running * thread: */ (*m)->m_owner = curthread; THR_SCHED_LOCK(curthread, curthread); /* Track number of priority mutexes owned: */ curthread->priority_mutex_count++; /* * The running thread inherits the ceiling * priority of the mutex and executes at that * priority. Make sure the thread's * scheduling lock is held while priorities * are adjusted. */ curthread->active_priority = (*m)->m_prio; (*m)->m_saved_prio = curthread->inherited_priority; curthread->inherited_priority = (*m)->m_prio; THR_SCHED_UNLOCK(curthread, curthread); /* Add to the list of owned mutexes: */ MUTEX_ASSERT_NOT_OWNED(*m); TAILQ_INSERT_TAIL(&curthread->mutexq, (*m), m_qe); /* Unlock the mutex structure: */ THR_LOCK_RELEASE(curthread, &(*m)->m_lock); } else if ((*m)->m_owner == curthread) { ret = mutex_self_lock(curthread, *m); /* Unlock the mutex structure: */ THR_LOCK_RELEASE(curthread, &(*m)->m_lock); } else { /* Set the wakeup time: */ if (abstime) { curthread->wakeup_time.tv_sec = abstime->tv_sec; curthread->wakeup_time.tv_nsec = abstime->tv_nsec; } /* * Join the queue of threads waiting to lock * the mutex and save a pointer to the mutex. */ mutex_queue_enq(*m, curthread); curthread->data.mutex = *m; /* Clear any previous error: */ curthread->error = 0; /* * This thread is active and is in a critical * region (holding the mutex lock); we should * be able to safely set the state. */ THR_SCHED_LOCK(curthread, curthread); THR_SET_STATE(curthread, PS_MUTEX_WAIT); THR_SCHED_UNLOCK(curthread, curthread); /* Unlock the mutex structure: */ THR_LOCK_RELEASE(curthread, &(*m)->m_lock); /* Schedule the next thread: */ _thr_sched_switch(curthread); curthread->data.mutex = NULL; if (THR_IN_MUTEXQ(curthread)) { THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock); mutex_queue_remove(*m, curthread); THR_LOCK_RELEASE(curthread, &(*m)->m_lock); } /* * The threads priority may have changed while * waiting for the mutex causing a ceiling * violation. */ ret = curthread->error; curthread->error = 0; } break; /* Trap invalid mutex types: */ default: /* Unlock the mutex structure: */ THR_LOCK_RELEASE(curthread, &(*m)->m_lock); /* Return an invalid argument error: */ ret = EINVAL; break; } } while (((*m)->m_owner != curthread) && (ret == 0) && (curthread->interrupted == 0) && (curthread->timeout == 0)); if (ret == 0 && curthread->timeout) ret = ETIMEDOUT; /* * Check to see if this thread was interrupted and * is still in the mutex queue of waiting threads: */ if (curthread->interrupted != 0) { /* Remove this thread from the mutex queue. */ THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock); if (THR_IN_SYNCQ(curthread)) mutex_queue_remove(*m, curthread); THR_LOCK_RELEASE(curthread, &(*m)->m_lock); /* Check for asynchronous cancellation. */ if (curthread->continuation != NULL) curthread->continuation((void *) curthread); } /* Return the completion status: */ return (ret); } int __pthread_mutex_lock(pthread_mutex_t *m) { struct pthread *curthread; int ret = 0; if (_thr_initial == NULL) _libpthread_init(NULL); curthread = _get_curthread(); if (m == NULL) ret = EINVAL; /* * If the mutex is statically initialized, perform the dynamic * initialization: */ else if ((*m != NULL) || ((ret = init_static(curthread, m)) == 0)) ret = mutex_lock_common(curthread, m, NULL); return (ret); } __strong_reference(__pthread_mutex_lock, _thr_mutex_lock); int _pthread_mutex_lock(pthread_mutex_t *m) { struct pthread *curthread; int ret = 0; if (_thr_initial == NULL) _libpthread_init(NULL); curthread = _get_curthread(); if (m == NULL) ret = EINVAL; /* * If the mutex is statically initialized, perform the dynamic * initialization marking it private (delete safe): */ else if ((*m != NULL) || ((ret = init_static_private(curthread, m)) == 0)) ret = mutex_lock_common(curthread, m, NULL); return (ret); } int __pthread_mutex_timedlock(pthread_mutex_t *m, const struct timespec *abs_timeout) { struct pthread *curthread; int ret = 0; if (_thr_initial == NULL) _libpthread_init(NULL); curthread = _get_curthread(); if (m == NULL) ret = EINVAL; /* * If the mutex is statically initialized, perform the dynamic * initialization: */ else if ((*m != NULL) || ((ret = init_static(curthread, m)) == 0)) ret = mutex_lock_common(curthread, m, abs_timeout); return (ret); } int _pthread_mutex_timedlock(pthread_mutex_t *m, const struct timespec *abs_timeout) { struct pthread *curthread; int ret = 0; if (_thr_initial == NULL) _libpthread_init(NULL); curthread = _get_curthread(); if (m == NULL) ret = EINVAL; /* * If the mutex is statically initialized, perform the dynamic * initialization marking it private (delete safe): */ else if ((*m != NULL) || ((ret = init_static_private(curthread, m)) == 0)) ret = mutex_lock_common(curthread, m, abs_timeout); return (ret); } int _pthread_mutex_unlock(pthread_mutex_t *m) { return (mutex_unlock_common(m, /* add reference */ 0)); } __strong_reference(_pthread_mutex_unlock, _thr_mutex_unlock); int _mutex_cv_unlock(pthread_mutex_t *m) { return (mutex_unlock_common(m, /* add reference */ 1)); } int _mutex_cv_lock(pthread_mutex_t *m) { struct pthread *curthread; int ret; curthread = _get_curthread(); if ((ret = _pthread_mutex_lock(m)) == 0) { THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock); (*m)->m_refcount--; THR_LOCK_RELEASE(curthread, &(*m)->m_lock); } return (ret); } static inline int mutex_self_trylock(struct pthread *curthread, pthread_mutex_t m) { int ret = 0; switch (m->m_type) { /* case PTHREAD_MUTEX_DEFAULT: */ case PTHREAD_MUTEX_ERRORCHECK: case PTHREAD_MUTEX_NORMAL: /* * POSIX specifies that mutexes should return EDEADLK if a * recursive lock is detected. */ if (m->m_owner == curthread) ret = EDEADLK; else ret = EBUSY; break; case PTHREAD_MUTEX_RECURSIVE: /* Increment the lock count: */ m->m_count++; break; default: /* Trap invalid mutex types; */ ret = EINVAL; } return (ret); } static inline int mutex_self_lock(struct pthread *curthread, pthread_mutex_t m) { int ret = 0; switch (m->m_type) { /* case PTHREAD_MUTEX_DEFAULT: */ case PTHREAD_MUTEX_ERRORCHECK: /* * POSIX specifies that mutexes should return EDEADLK if a * recursive lock is detected. */ ret = EDEADLK; break; case PTHREAD_MUTEX_NORMAL: /* * What SS2 define as a 'normal' mutex. Intentionally * deadlock on attempts to get a lock you already own. */ THR_SCHED_LOCK(curthread, curthread); THR_SET_STATE(curthread, PS_DEADLOCK); THR_SCHED_UNLOCK(curthread, curthread); /* Unlock the mutex structure: */ THR_LOCK_RELEASE(curthread, &m->m_lock); /* Schedule the next thread: */ _thr_sched_switch(curthread); break; case PTHREAD_MUTEX_RECURSIVE: /* Increment the lock count: */ m->m_count++; break; default: /* Trap invalid mutex types; */ ret = EINVAL; } return (ret); } static int mutex_unlock_common(pthread_mutex_t *m, int add_reference) { struct pthread *curthread = _get_curthread(); struct kse_mailbox *kmbx = NULL; int ret = 0; if (m == NULL || *m == NULL) ret = EINVAL; else { /* Lock the mutex structure: */ THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock); /* Process according to mutex type: */ switch ((*m)->m_protocol) { /* Default POSIX mutex: */ case PTHREAD_PRIO_NONE: /* * Check if the running thread is not the owner of the * mutex: */ if ((*m)->m_owner != curthread) /* * Return an invalid argument error for no * owner and a permission error otherwise: */ ret = (*m)->m_owner == NULL ? EINVAL : EPERM; else if (((*m)->m_type == PTHREAD_MUTEX_RECURSIVE) && ((*m)->m_count > 0)) /* Decrement the count: */ (*m)->m_count--; else { /* * Clear the count in case this is a recursive * mutex. */ (*m)->m_count = 0; /* Remove the mutex from the threads queue. */ MUTEX_ASSERT_IS_OWNED(*m); TAILQ_REMOVE(&(*m)->m_owner->mutexq, (*m), m_qe); MUTEX_INIT_LINK(*m); /* * Hand off the mutex to the next waiting * thread: */ kmbx = mutex_handoff(curthread, *m); } break; /* POSIX priority inheritence mutex: */ case PTHREAD_PRIO_INHERIT: /* * Check if the running thread is not the owner of the * mutex: */ if ((*m)->m_owner != curthread) /* * Return an invalid argument error for no * owner and a permission error otherwise: */ ret = (*m)->m_owner == NULL ? EINVAL : EPERM; else if (((*m)->m_type == PTHREAD_MUTEX_RECURSIVE) && ((*m)->m_count > 0)) /* Decrement the count: */ (*m)->m_count--; else { /* * Clear the count in case this is recursive * mutex. */ (*m)->m_count = 0; /* * Restore the threads inherited priority and * recompute the active priority (being careful * not to override changes in the threads base * priority subsequent to locking the mutex). */ THR_SCHED_LOCK(curthread, curthread); curthread->inherited_priority = (*m)->m_saved_prio; curthread->active_priority = MAX(curthread->inherited_priority, curthread->base_priority); /* * This thread now owns one less priority mutex. */ curthread->priority_mutex_count--; THR_SCHED_UNLOCK(curthread, curthread); /* Remove the mutex from the threads queue. */ MUTEX_ASSERT_IS_OWNED(*m); TAILQ_REMOVE(&(*m)->m_owner->mutexq, (*m), m_qe); MUTEX_INIT_LINK(*m); /* * Hand off the mutex to the next waiting * thread: */ kmbx = mutex_handoff(curthread, *m); } break; /* POSIX priority ceiling mutex: */ case PTHREAD_PRIO_PROTECT: /* * Check if the running thread is not the owner of the * mutex: */ if ((*m)->m_owner != curthread) /* * Return an invalid argument error for no * owner and a permission error otherwise: */ ret = (*m)->m_owner == NULL ? EINVAL : EPERM; else if (((*m)->m_type == PTHREAD_MUTEX_RECURSIVE) && ((*m)->m_count > 0)) /* Decrement the count: */ (*m)->m_count--; else { /* * Clear the count in case this is a recursive * mutex. */ (*m)->m_count = 0; /* * Restore the threads inherited priority and * recompute the active priority (being careful * not to override changes in the threads base * priority subsequent to locking the mutex). */ THR_SCHED_LOCK(curthread, curthread); curthread->inherited_priority = (*m)->m_saved_prio; curthread->active_priority = MAX(curthread->inherited_priority, curthread->base_priority); /* * This thread now owns one less priority mutex. */ curthread->priority_mutex_count--; THR_SCHED_UNLOCK(curthread, curthread); /* Remove the mutex from the threads queue. */ MUTEX_ASSERT_IS_OWNED(*m); TAILQ_REMOVE(&(*m)->m_owner->mutexq, (*m), m_qe); MUTEX_INIT_LINK(*m); /* * Hand off the mutex to the next waiting * thread: */ kmbx = mutex_handoff(curthread, *m); } break; /* Trap invalid mutex types: */ default: /* Return an invalid argument error: */ ret = EINVAL; break; } if ((ret == 0) && (add_reference != 0)) /* Increment the reference count: */ (*m)->m_refcount++; /* Unlock the mutex structure: */ THR_LOCK_RELEASE(curthread, &(*m)->m_lock); if (kmbx != NULL) kse_wakeup(kmbx); } /* Return the completion status: */ return (ret); } /* * This function is called when a change in base priority occurs for * a thread that is holding or waiting for a priority protection or * inheritence mutex. A change in a threads base priority can effect * changes to active priorities of other threads and to the ordering * of mutex locking by waiting threads. * * This must be called without the target thread's scheduling lock held. */ void _mutex_notify_priochange(struct pthread *curthread, struct pthread *pthread, int propagate_prio) { struct pthread_mutex *m; /* Adjust the priorites of any owned priority mutexes: */ if (pthread->priority_mutex_count > 0) { /* * Rescan the mutexes owned by this thread and correct * their priorities to account for this threads change * in priority. This has the side effect of changing * the threads active priority. * * Be sure to lock the first mutex in the list of owned * mutexes. This acts as a barrier against another * simultaneous call to change the threads priority * and from the owning thread releasing the mutex. */ m = TAILQ_FIRST(&pthread->mutexq); if (m != NULL) { THR_LOCK_ACQUIRE(curthread, &m->m_lock); /* * Make sure the thread still owns the lock. */ if (m == TAILQ_FIRST(&pthread->mutexq)) mutex_rescan_owned(curthread, pthread, /* rescan all owned */ NULL); THR_LOCK_RELEASE(curthread, &m->m_lock); } } /* * If this thread is waiting on a priority inheritence mutex, * check for priority adjustments. A change in priority can * also cause a ceiling violation(*) for a thread waiting on * a priority protection mutex; we don't perform the check here * as it is done in pthread_mutex_unlock. * * (*) It should be noted that a priority change to a thread * _after_ taking and owning a priority ceiling mutex * does not affect ownership of that mutex; the ceiling * priority is only checked before mutex ownership occurs. */ if (propagate_prio != 0) { /* * Lock the thread's scheduling queue. This is a bit * convoluted; the "in synchronization queue flag" can * only be cleared with both the thread's scheduling and * mutex locks held. The thread's pointer to the wanted * mutex is guaranteed to be valid during this time. */ THR_SCHED_LOCK(curthread, pthread); if (((pthread->sflags & THR_FLAGS_IN_SYNCQ) == 0) || ((m = pthread->data.mutex) == NULL)) THR_SCHED_UNLOCK(curthread, pthread); else { /* * This thread is currently waiting on a mutex; unlock * the scheduling queue lock and lock the mutex. We * can't hold both at the same time because the locking * order could cause a deadlock. */ THR_SCHED_UNLOCK(curthread, pthread); THR_LOCK_ACQUIRE(curthread, &m->m_lock); /* * Check to make sure this thread is still in the * same state (the lock above can yield the CPU to * another thread or the thread may be running on * another CPU). */ if (((pthread->sflags & THR_FLAGS_IN_SYNCQ) != 0) && (pthread->data.mutex == m)) { /* * Remove and reinsert this thread into * the list of waiting threads to preserve * decreasing priority order. */ mutex_queue_remove(m, pthread); mutex_queue_enq(m, pthread); if (m->m_protocol == PTHREAD_PRIO_INHERIT) /* Adjust priorities: */ mutex_priority_adjust(curthread, m); } /* Unlock the mutex structure: */ THR_LOCK_RELEASE(curthread, &m->m_lock); } } } /* * Called when a new thread is added to the mutex waiting queue or * when a threads priority changes that is already in the mutex * waiting queue. * * This must be called with the mutex locked by the current thread. */ static void mutex_priority_adjust(struct pthread *curthread, pthread_mutex_t mutex) { pthread_mutex_t m = mutex; struct pthread *pthread_next, *pthread = mutex->m_owner; int done, temp_prio; /* * Calculate the mutex priority as the maximum of the highest * active priority of any waiting threads and the owning threads * active priority(*). * * (*) Because the owning threads current active priority may * reflect priority inherited from this mutex (and the mutex * priority may have changed) we must recalculate the active * priority based on the threads saved inherited priority * and its base priority. */ pthread_next = TAILQ_FIRST(&m->m_queue); /* should never be NULL */ temp_prio = MAX(pthread_next->active_priority, MAX(m->m_saved_prio, pthread->base_priority)); /* See if this mutex really needs adjusting: */ if (temp_prio == m->m_prio) /* No need to propagate the priority: */ return; /* Set new priority of the mutex: */ m->m_prio = temp_prio; /* * Don't unlock the mutex passed in as an argument. It is * expected to be locked and unlocked by the caller. */ done = 1; do { /* * Save the threads priority before rescanning the * owned mutexes: */ temp_prio = pthread->active_priority; /* * Fix the priorities for all mutexes held by the owning * thread since taking this mutex. This also has a * potential side-effect of changing the threads priority. * * At this point the mutex is locked by the current thread. * The owning thread can't release the mutex until it is * unlocked, so we should be able to safely walk its list * of owned mutexes. */ mutex_rescan_owned(curthread, pthread, m); /* * If this isn't the first time through the loop, * the current mutex needs to be unlocked. */ if (done == 0) THR_LOCK_RELEASE(curthread, &m->m_lock); /* Assume we're done unless told otherwise: */ done = 1; /* * If the thread is currently waiting on a mutex, check * to see if the threads new priority has affected the * priority of the mutex. */ if ((temp_prio != pthread->active_priority) && ((pthread->sflags & THR_FLAGS_IN_SYNCQ) != 0) && ((m = pthread->data.mutex) != NULL) && (m->m_protocol == PTHREAD_PRIO_INHERIT)) { /* Lock the mutex structure: */ THR_LOCK_ACQUIRE(curthread, &m->m_lock); /* * Make sure the thread is still waiting on the * mutex: */ if (((pthread->sflags & THR_FLAGS_IN_SYNCQ) != 0) && (m == pthread->data.mutex)) { /* * The priority for this thread has changed. * Remove and reinsert this thread into the * list of waiting threads to preserve * decreasing priority order. */ mutex_queue_remove(m, pthread); mutex_queue_enq(m, pthread); /* * Grab the waiting thread with highest * priority: */ pthread_next = TAILQ_FIRST(&m->m_queue); /* * Calculate the mutex priority as the maximum * of the highest active priority of any * waiting threads and the owning threads * active priority. */ temp_prio = MAX(pthread_next->active_priority, MAX(m->m_saved_prio, m->m_owner->base_priority)); if (temp_prio != m->m_prio) { /* * The priority needs to be propagated * to the mutex this thread is waiting * on and up to the owner of that mutex. */ m->m_prio = temp_prio; pthread = m->m_owner; /* We're not done yet: */ done = 0; } } /* Only release the mutex if we're done: */ if (done != 0) THR_LOCK_RELEASE(curthread, &m->m_lock); } } while (done == 0); } static void mutex_rescan_owned(struct pthread *curthread, struct pthread *pthread, struct pthread_mutex *mutex) { struct pthread_mutex *m; struct pthread *pthread_next; int active_prio, inherited_prio; /* * Start walking the mutexes the thread has taken since * taking this mutex. */ if (mutex == NULL) { /* * A null mutex means start at the beginning of the owned * mutex list. */ m = TAILQ_FIRST(&pthread->mutexq); /* There is no inherited priority yet. */ inherited_prio = 0; } else { /* * The caller wants to start after a specific mutex. It * is assumed that this mutex is a priority inheritence * mutex and that its priority has been correctly * calculated. */ m = TAILQ_NEXT(mutex, m_qe); /* Start inheriting priority from the specified mutex. */ inherited_prio = mutex->m_prio; } active_prio = MAX(inherited_prio, pthread->base_priority); for (; m != NULL; m = TAILQ_NEXT(m, m_qe)) { /* * We only want to deal with priority inheritence * mutexes. This might be optimized by only placing * priority inheritence mutexes into the owned mutex * list, but it may prove to be useful having all * owned mutexes in this list. Consider a thread * exiting while holding mutexes... */ if (m->m_protocol == PTHREAD_PRIO_INHERIT) { /* * Fix the owners saved (inherited) priority to * reflect the priority of the previous mutex. */ m->m_saved_prio = inherited_prio; if ((pthread_next = TAILQ_FIRST(&m->m_queue)) != NULL) /* Recalculate the priority of the mutex: */ m->m_prio = MAX(active_prio, pthread_next->active_priority); else m->m_prio = active_prio; /* Recalculate new inherited and active priorities: */ inherited_prio = m->m_prio; active_prio = MAX(m->m_prio, pthread->base_priority); } } /* * Fix the threads inherited priority and recalculate its * active priority. */ pthread->inherited_priority = inherited_prio; active_prio = MAX(inherited_prio, pthread->base_priority); if (active_prio != pthread->active_priority) { /* Lock the thread's scheduling queue: */ THR_SCHED_LOCK(curthread, pthread); if ((pthread->flags & THR_FLAGS_IN_RUNQ) == 0) { /* * This thread is not in a run queue. Just set * its active priority. */ pthread->active_priority = active_prio; } else { /* * This thread is in a run queue. Remove it from * the queue before changing its priority: */ THR_RUNQ_REMOVE(pthread); /* * POSIX states that if the priority is being * lowered, the thread must be inserted at the * head of the queue for its priority if it owns * any priority protection or inheritence mutexes. */ if ((active_prio < pthread->active_priority) && (pthread->priority_mutex_count > 0)) { /* Set the new active priority. */ pthread->active_priority = active_prio; THR_RUNQ_INSERT_HEAD(pthread); } else { /* Set the new active priority. */ pthread->active_priority = active_prio; THR_RUNQ_INSERT_TAIL(pthread); } } THR_SCHED_UNLOCK(curthread, pthread); } } void _mutex_unlock_private(pthread_t pthread) { struct pthread_mutex *m, *m_next; for (m = TAILQ_FIRST(&pthread->mutexq); m != NULL; m = m_next) { m_next = TAILQ_NEXT(m, m_qe); if ((m->m_flags & MUTEX_FLAGS_PRIVATE) != 0) pthread_mutex_unlock(&m); } } /* * This is called by the current thread when it wants to back out of a * mutex_lock in order to run a signal handler. */ void _mutex_lock_backout(struct pthread *curthread) { struct pthread_mutex *m; if ((curthread->sflags & THR_FLAGS_IN_SYNCQ) != 0) { /* * Any other thread may clear the "in sync queue flag", * but only the current thread can clear the pointer * to the mutex. So if the flag is set, we can * guarantee that the pointer to the mutex is valid. * The only problem may be if the mutex is destroyed * out from under us, but that should be considered * an application bug. */ m = curthread->data.mutex; /* Lock the mutex structure: */ THR_LOCK_ACQUIRE(curthread, &m->m_lock); /* * Check to make sure this thread doesn't already own * the mutex. Since mutexes are unlocked with direct * handoffs, it is possible the previous owner gave it * to us after we checked the sync queue flag and before * we locked the mutex structure. */ if (m->m_owner == curthread) { THR_LOCK_RELEASE(curthread, &m->m_lock); mutex_unlock_common(&m, /* add_reference */ 0); } else { /* * Remove ourselves from the mutex queue and * clear the pointer to the mutex. We may no * longer be in the mutex queue, but the removal * function will DTRT. */ mutex_queue_remove(m, curthread); curthread->data.mutex = NULL; THR_LOCK_RELEASE(curthread, &m->m_lock); } } } /* * Dequeue a waiting thread from the head of a mutex queue in descending * priority order. * * In order to properly dequeue a thread from the mutex queue and * make it runnable without the possibility of errant wakeups, it * is necessary to lock the thread's scheduling queue while also * holding the mutex lock. */ static struct kse_mailbox * mutex_handoff(struct pthread *curthread, struct pthread_mutex *mutex) { struct kse_mailbox *kmbx = NULL; struct pthread *pthread; /* Keep dequeueing until we find a valid thread: */ mutex->m_owner = NULL; pthread = TAILQ_FIRST(&mutex->m_queue); while (pthread != NULL) { /* Take the thread's scheduling lock: */ THR_SCHED_LOCK(curthread, pthread); /* Remove the thread from the mutex queue: */ TAILQ_REMOVE(&mutex->m_queue, pthread, sqe); pthread->sflags &= ~THR_FLAGS_IN_SYNCQ; /* * Only exit the loop if the thread hasn't been * cancelled. */ switch (mutex->m_protocol) { case PTHREAD_PRIO_NONE: /* * Assign the new owner and add the mutex to the * thread's list of owned mutexes. */ mutex->m_owner = pthread; TAILQ_INSERT_TAIL(&pthread->mutexq, mutex, m_qe); break; case PTHREAD_PRIO_INHERIT: /* * Assign the new owner and add the mutex to the * thread's list of owned mutexes. */ mutex->m_owner = pthread; TAILQ_INSERT_TAIL(&pthread->mutexq, mutex, m_qe); /* Track number of priority mutexes owned: */ pthread->priority_mutex_count++; /* * Set the priority of the mutex. Since our waiting * threads are in descending priority order, the * priority of the mutex becomes the active priority * of the thread we just dequeued. */ mutex->m_prio = pthread->active_priority; /* Save the owning threads inherited priority: */ mutex->m_saved_prio = pthread->inherited_priority; /* * The owning threads inherited priority now becomes * his active priority (the priority of the mutex). */ pthread->inherited_priority = mutex->m_prio; break; case PTHREAD_PRIO_PROTECT: if (pthread->active_priority > mutex->m_prio) { /* * Either the mutex ceiling priority has * been lowered and/or this threads priority * has been raised subsequent to the thread * being queued on the waiting list. */ pthread->error = EINVAL; } else { /* * Assign the new owner and add the mutex * to the thread's list of owned mutexes. */ mutex->m_owner = pthread; TAILQ_INSERT_TAIL(&pthread->mutexq, mutex, m_qe); /* Track number of priority mutexes owned: */ pthread->priority_mutex_count++; /* * Save the owning threads inherited * priority: */ mutex->m_saved_prio = pthread->inherited_priority; /* * The owning thread inherits the ceiling * priority of the mutex and executes at * that priority: */ pthread->inherited_priority = mutex->m_prio; pthread->active_priority = mutex->m_prio; } break; } /* Make the thread runnable and unlock the scheduling queue: */ kmbx = _thr_setrunnable_unlocked(pthread); /* Add a preemption point. */ if ((curthread->kseg == pthread->kseg) && (pthread->active_priority > curthread->active_priority)) curthread->critical_yield = 1; THR_SCHED_UNLOCK(curthread, pthread); if (mutex->m_owner == pthread) /* We're done; a valid owner was found. */ break; else /* Get the next thread from the waiting queue: */ pthread = TAILQ_NEXT(pthread, sqe); } if ((pthread == NULL) && (mutex->m_protocol == PTHREAD_PRIO_INHERIT)) /* This mutex has no priority: */ mutex->m_prio = 0; return (kmbx); } /* * Dequeue a waiting thread from the head of a mutex queue in descending * priority order. */ static inline pthread_t mutex_queue_deq(struct pthread_mutex *mutex) { pthread_t pthread; while ((pthread = TAILQ_FIRST(&mutex->m_queue)) != NULL) { TAILQ_REMOVE(&mutex->m_queue, pthread, sqe); pthread->sflags &= ~THR_FLAGS_IN_SYNCQ; /* * Only exit the loop if the thread hasn't been * cancelled. */ if (pthread->interrupted == 0) break; } return (pthread); } /* * Remove a waiting thread from a mutex queue in descending priority order. */ static inline void mutex_queue_remove(pthread_mutex_t mutex, pthread_t pthread) { if ((pthread->sflags & THR_FLAGS_IN_SYNCQ) != 0) { TAILQ_REMOVE(&mutex->m_queue, pthread, sqe); pthread->sflags &= ~THR_FLAGS_IN_SYNCQ; } } /* * Enqueue a waiting thread to a queue in descending priority order. */ static inline void mutex_queue_enq(pthread_mutex_t mutex, pthread_t pthread) { pthread_t tid = TAILQ_LAST(&mutex->m_queue, mutex_head); THR_ASSERT_NOT_IN_SYNCQ(pthread); /* * For the common case of all threads having equal priority, * we perform a quick check against the priority of the thread * at the tail of the queue. */ if ((tid == NULL) || (pthread->active_priority <= tid->active_priority)) TAILQ_INSERT_TAIL(&mutex->m_queue, pthread, sqe); else { tid = TAILQ_FIRST(&mutex->m_queue); while (pthread->active_priority <= tid->active_priority) tid = TAILQ_NEXT(tid, sqe); TAILQ_INSERT_BEFORE(tid, pthread, sqe); } pthread->sflags |= THR_FLAGS_IN_SYNCQ; }