2003-04-01 01:10:42 +00:00
|
|
|
/*
|
|
|
|
* Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
|
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
* notice unmodified, this list of conditions, and the following
|
|
|
|
* disclaimer.
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
|
|
|
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
|
|
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
|
|
|
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
|
|
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
|
|
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
|
|
|
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
*/
|
|
|
|
|
2003-06-11 00:56:59 +00:00
|
|
|
#include <sys/cdefs.h>
|
|
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
|
2003-04-01 01:10:42 +00:00
|
|
|
#include <sys/param.h>
|
|
|
|
#include <sys/kernel.h>
|
2004-07-02 00:40:07 +00:00
|
|
|
#include <sys/limits.h>
|
2003-04-01 01:10:42 +00:00
|
|
|
#include <sys/lock.h>
|
2003-06-03 05:24:46 +00:00
|
|
|
#include <sys/malloc.h>
|
2003-04-01 01:10:42 +00:00
|
|
|
#include <sys/mutex.h>
|
|
|
|
#include <sys/proc.h>
|
|
|
|
#include <sys/sysent.h>
|
|
|
|
#include <sys/systm.h>
|
|
|
|
#include <sys/sysproto.h>
|
|
|
|
#include <sys/thr.h>
|
|
|
|
#include <sys/umtx.h>
|
|
|
|
|
2003-06-03 05:24:46 +00:00
|
|
|
struct umtx_q {
|
|
|
|
LIST_ENTRY(umtx_q) uq_next; /* Linked list for the hash. */
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
TAILQ_HEAD(, thread) uq_tdq; /* List of threads blocked here. */
|
|
|
|
struct umtx *uq_umtx; /* Pointer key component. */
|
|
|
|
pid_t uq_pid; /* Pid key component. */
|
|
|
|
int uq_count; /* How many threads blocked. */
|
2003-06-03 05:24:46 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
LIST_HEAD(umtx_head, umtx_q);
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
struct umtxq_chain {
|
|
|
|
struct mtx uc_lock; /* lock for this chain. */
|
|
|
|
struct umtx_head uc_queues; /* List of sleep queues. */
|
|
|
|
};
|
2003-06-03 05:24:46 +00:00
|
|
|
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
#define GOLDEN_RATIO_PRIME 2654404609U
|
|
|
|
#define UMTX_CHAINS 128
|
|
|
|
#define UMTX_SHIFTS (__WORD_BIT - 7)
|
2003-06-03 05:24:46 +00:00
|
|
|
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
static struct umtxq_chain umtxq_chains[UMTX_CHAINS];
|
|
|
|
static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory");
|
2003-05-25 18:18:32 +00:00
|
|
|
|
2004-07-02 00:40:07 +00:00
|
|
|
#define UMTX_CONTESTED LONG_MIN
|
2003-05-25 18:18:32 +00:00
|
|
|
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
static void umtx_init_chains(void *);
|
|
|
|
static int umtxq_hash(struct thread *, struct umtx *);
|
|
|
|
static void umtxq_lock(struct thread *td, struct umtx *key);
|
|
|
|
static void umtxq_unlock(struct thread *td, struct umtx *key);
|
|
|
|
static struct umtx_q *umtxq_lookup(struct thread *, struct umtx *);
|
|
|
|
static struct umtx_q *umtxq_insert(struct thread *, struct umtx *);
|
|
|
|
static int umtxq_count(struct thread *td, struct umtx *umtx);
|
|
|
|
static int umtx_sleep(struct thread *td, struct umtx *umtx, int priority,
|
|
|
|
const char *wmesg, int timo);
|
|
|
|
static void umtx_signal(struct thread *td, struct umtx *umtx);
|
|
|
|
|
|
|
|
SYSINIT(umtx, SI_SUB_LOCK, SI_ORDER_MIDDLE, umtx_init_chains, NULL);
|
|
|
|
|
|
|
|
static void
|
|
|
|
umtx_init_chains(void *arg __unused)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < UMTX_CHAINS; ++i) {
|
|
|
|
mtx_init(&umtxq_chains[i].uc_lock, "umtxq_lock", NULL,
|
|
|
|
MTX_DEF | MTX_DUPOK);
|
|
|
|
LIST_INIT(&umtxq_chains[i].uc_queues);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int
|
|
|
|
umtxq_hash(struct thread *td, struct umtx *umtx)
|
|
|
|
{
|
|
|
|
unsigned n = (uintptr_t)umtx + td->td_proc->p_pid;
|
|
|
|
return (((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
umtxq_lock(struct thread *td, struct umtx *key)
|
|
|
|
{
|
|
|
|
int chain = umtxq_hash(td, key);
|
|
|
|
mtx_lock(&umtxq_chains[chain].uc_lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
umtxq_unlock(struct thread *td, struct umtx *key)
|
|
|
|
{
|
|
|
|
int chain = umtxq_hash(td, key);
|
|
|
|
mtx_unlock(&umtxq_chains[chain].uc_lock);
|
|
|
|
}
|
2003-06-03 05:24:46 +00:00
|
|
|
|
|
|
|
static struct umtx_q *
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
umtxq_lookup(struct thread *td, struct umtx *umtx)
|
2003-06-03 05:24:46 +00:00
|
|
|
{
|
|
|
|
struct umtx_head *head;
|
|
|
|
struct umtx_q *uq;
|
|
|
|
pid_t pid;
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
int chain;
|
2003-06-03 05:24:46 +00:00
|
|
|
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
chain = umtxq_hash(td, umtx);
|
|
|
|
mtx_assert(&umtxq_chains[chain].uc_lock, MA_OWNED);
|
2003-06-03 05:24:46 +00:00
|
|
|
pid = td->td_proc->p_pid;
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
head = &umtxq_chains[chain].uc_queues;
|
2003-06-03 05:24:46 +00:00
|
|
|
LIST_FOREACH(uq, head, uq_next) {
|
|
|
|
if (uq->uq_pid == pid && uq->uq_umtx == umtx)
|
|
|
|
return (uq);
|
|
|
|
}
|
|
|
|
return (NULL);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Insert a thread onto the umtx queue.
|
|
|
|
*/
|
|
|
|
static struct umtx_q *
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
umtxq_insert(struct thread *td, struct umtx *umtx)
|
2003-06-03 05:24:46 +00:00
|
|
|
{
|
|
|
|
struct umtx_head *head;
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
struct umtx_q *uq, *ins = NULL;
|
2003-06-03 05:24:46 +00:00
|
|
|
pid_t pid;
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
int chain;
|
2003-06-03 05:24:46 +00:00
|
|
|
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
chain = umtxq_hash(td, umtx);
|
2003-06-03 05:24:46 +00:00
|
|
|
pid = td->td_proc->p_pid;
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
if ((uq = umtxq_lookup(td, umtx)) == NULL) {
|
|
|
|
umtxq_unlock(td, umtx);
|
2003-06-03 05:24:46 +00:00
|
|
|
ins = malloc(sizeof(*uq), M_UMTX, M_ZERO | M_WAITOK);
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
umtxq_lock(td, umtx);
|
2003-06-03 05:24:46 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Some one else could have succeeded while we were blocked
|
|
|
|
* waiting on memory.
|
|
|
|
*/
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
if ((uq = umtxq_lookup(td, umtx)) == NULL) {
|
|
|
|
head = &umtxq_chains[chain].uc_queues;
|
2003-06-03 05:24:46 +00:00
|
|
|
uq = ins;
|
|
|
|
uq->uq_pid = pid;
|
|
|
|
uq->uq_umtx = umtx;
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
uq->uq_count = 0;
|
2003-06-03 05:24:46 +00:00
|
|
|
LIST_INSERT_HEAD(head, uq, uq_next);
|
|
|
|
TAILQ_INIT(&uq->uq_tdq);
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
ins = NULL;
|
|
|
|
}
|
2003-06-03 05:24:46 +00:00
|
|
|
}
|
|
|
|
TAILQ_INSERT_TAIL(&uq->uq_tdq, td, td_umtx);
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
uq->uq_count++;
|
|
|
|
if (ins) {
|
|
|
|
umtxq_unlock(td, umtx);
|
|
|
|
free(ins, M_UMTX);
|
|
|
|
umtxq_lock(td, umtx);
|
|
|
|
}
|
2003-06-03 05:24:46 +00:00
|
|
|
return (uq);
|
|
|
|
}
|
|
|
|
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
/*
|
|
|
|
* Remove thread from umtx queue, umtx chain lock is also
|
|
|
|
* released.
|
|
|
|
*/
|
2003-06-03 05:24:46 +00:00
|
|
|
static void
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
umtx_remove(struct umtx_q *uq, struct thread *td, struct umtx *umtx)
|
2003-06-03 05:24:46 +00:00
|
|
|
{
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
int chain;
|
2003-06-03 05:24:46 +00:00
|
|
|
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
chain = umtxq_hash(td, umtx);
|
|
|
|
mtx_assert(&umtxq_chains[chain].uc_lock, MA_OWNED);
|
|
|
|
TAILQ_REMOVE(&uq->uq_tdq, td, td_umtx);
|
|
|
|
uq->uq_count--;
|
2003-06-03 05:24:46 +00:00
|
|
|
if (TAILQ_EMPTY(&uq->uq_tdq)) {
|
|
|
|
LIST_REMOVE(uq, uq_next);
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
umtxq_unlock(td, umtx);
|
2003-06-03 05:24:46 +00:00
|
|
|
free(uq, M_UMTX);
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
} else
|
|
|
|
umtxq_unlock(td, umtx);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int
|
|
|
|
umtxq_count(struct thread *td, struct umtx *umtx)
|
|
|
|
{
|
|
|
|
struct umtx_q *uq;
|
|
|
|
int count = 0;
|
|
|
|
|
|
|
|
umtxq_lock(td, umtx);
|
|
|
|
if ((uq = umtxq_lookup(td, umtx)) != NULL)
|
|
|
|
count = uq->uq_count;
|
|
|
|
umtxq_unlock(td, umtx);
|
|
|
|
return (count);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int
|
|
|
|
umtx_sleep(struct thread *td, struct umtx *umtx, int priority,
|
|
|
|
const char *wmesg, int timo)
|
|
|
|
{
|
|
|
|
int chain;
|
|
|
|
|
|
|
|
chain = umtxq_hash(td, umtx);
|
|
|
|
mtx_assert(&umtxq_chains[chain].uc_lock, MA_OWNED);
|
|
|
|
return (msleep(td, &umtxq_chains[chain].uc_lock, priority,
|
|
|
|
wmesg, timo));
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
umtx_signal(struct thread *td, struct umtx *umtx)
|
|
|
|
{
|
|
|
|
struct umtx_q *uq;
|
|
|
|
struct thread *blocked = NULL;
|
|
|
|
|
|
|
|
umtxq_lock(td, umtx);
|
|
|
|
if ((uq = umtxq_lookup(td, umtx)) != NULL) {
|
|
|
|
if ((blocked = TAILQ_FIRST(&uq->uq_tdq)) != NULL) {
|
|
|
|
mtx_lock_spin(&sched_lock);
|
|
|
|
blocked->td_flags |= TDF_UMTXWAKEUP;
|
|
|
|
mtx_unlock_spin(&sched_lock);
|
|
|
|
}
|
2003-06-03 05:24:46 +00:00
|
|
|
}
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
umtxq_unlock(td, umtx);
|
|
|
|
if (blocked != NULL)
|
|
|
|
wakeup(blocked);
|
2003-06-03 05:24:46 +00:00
|
|
|
}
|
2003-05-25 18:18:32 +00:00
|
|
|
|
2003-04-01 01:10:42 +00:00
|
|
|
int
|
|
|
|
_umtx_lock(struct thread *td, struct _umtx_lock_args *uap)
|
|
|
|
/* struct umtx *umtx */
|
|
|
|
{
|
2003-06-03 05:24:46 +00:00
|
|
|
struct umtx_q *uq;
|
2003-04-01 01:10:42 +00:00
|
|
|
struct umtx *umtx;
|
|
|
|
intptr_t owner;
|
2003-04-02 08:02:27 +00:00
|
|
|
intptr_t old;
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
int error = 0;
|
2003-04-01 01:10:42 +00:00
|
|
|
|
2003-06-03 05:24:46 +00:00
|
|
|
uq = NULL;
|
2003-04-01 01:10:42 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Care must be exercised when dealing with this structure. It
|
|
|
|
* can fault on any access.
|
|
|
|
*/
|
|
|
|
umtx = uap->umtx;
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
/*
|
|
|
|
* Try the uncontested case. This should be done in userland.
|
|
|
|
*/
|
|
|
|
owner = casuptr((intptr_t *)&umtx->u_owner,
|
2004-07-02 00:40:07 +00:00
|
|
|
UMTX_UNOWNED, td->td_tid);
|
2003-04-01 01:10:42 +00:00
|
|
|
|
|
|
|
/* The acquire succeeded. */
|
2003-06-03 05:24:46 +00:00
|
|
|
if (owner == UMTX_UNOWNED)
|
|
|
|
return (0);
|
2003-04-01 01:10:42 +00:00
|
|
|
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
/* The address was invalid. */
|
|
|
|
if (owner == -1)
|
|
|
|
return (EFAULT);
|
|
|
|
|
2003-06-03 05:24:46 +00:00
|
|
|
/* If no one owns it but it is contested try to acquire it. */
|
|
|
|
if (owner == UMTX_CONTESTED) {
|
|
|
|
owner = casuptr((intptr_t *)&umtx->u_owner,
|
2004-07-02 00:40:07 +00:00
|
|
|
UMTX_CONTESTED, td->td_tid | UMTX_CONTESTED);
|
2003-06-03 05:24:46 +00:00
|
|
|
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
if (owner == UMTX_CONTESTED)
|
|
|
|
return (0);
|
|
|
|
|
2003-06-03 05:24:46 +00:00
|
|
|
/* The address was invalid. */
|
|
|
|
if (owner == -1)
|
|
|
|
return (EFAULT);
|
|
|
|
|
|
|
|
/* If this failed the lock has changed, restart. */
|
|
|
|
continue;
|
2003-04-01 01:10:42 +00:00
|
|
|
}
|
|
|
|
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
/*
|
|
|
|
* If we caught a signal, we have retried and now
|
|
|
|
* exit immediately.
|
|
|
|
*/
|
|
|
|
if (error)
|
|
|
|
return (error);
|
2003-06-03 05:24:46 +00:00
|
|
|
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
umtxq_lock(td, umtx);
|
|
|
|
uq = umtxq_insert(td, umtx);
|
|
|
|
umtxq_unlock(td, umtx);
|
2003-04-01 01:10:42 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Set the contested bit so that a release in user space
|
|
|
|
* knows to use the system call for unlock. If this fails
|
|
|
|
* either some one else has acquired the lock or it has been
|
|
|
|
* released.
|
|
|
|
*/
|
2003-04-02 08:02:27 +00:00
|
|
|
old = casuptr((intptr_t *)&umtx->u_owner, owner,
|
|
|
|
owner | UMTX_CONTESTED);
|
2003-04-01 01:10:42 +00:00
|
|
|
|
|
|
|
/* The address was invalid. */
|
2003-04-02 08:02:27 +00:00
|
|
|
if (old == -1) {
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
umtxq_lock(td, umtx);
|
|
|
|
umtx_remove(uq, td, umtx);
|
|
|
|
/* unlocked by umtx_remove */
|
2003-06-03 05:24:46 +00:00
|
|
|
return (EFAULT);
|
2003-04-01 01:10:42 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2003-06-03 05:24:46 +00:00
|
|
|
* We set the contested bit, sleep. Otherwise the lock changed
|
Fix umtx locking, for libthr, in the kernel.
1. There was a race condition between a thread unlocking
a umtx and the thread contesting it. If the unlocking
thread won the race it may try to wakeup a thread that
was not yet in msleep(). The contesting thread would then
go to sleep to await a wakeup that would never come. It's
not possible to close the race by using a lock because
calls to casuptr() may have to fault a page in from swap.
Instead, the race was closed by introducing a flag that
the unlocking thread will set when waking up a thread.
The contesting thread will check for this flag before
going to sleep. For now the flag is kept in td_flags,
but it may be better to use some other member or create
a new one because of the possible performance/contention
issues of having to own sched_lock. Thanks to jhb for
pointing me in the right direction on this one.
2. Once a umtx was contested all future locks and unlocks
were happening in the kernel, regardless of whether it
was contested or not. To prevent this from happening,
when a thread locks a umtx it checks the queue for that
umtx and unsets the contested bit if there are no other
threads waiting on it. Again, this is slightly more
complicated than it needs to be because we can't hold
a lock across casuptr(). So, the thread has to check
the queue again after unseting the bit, and reset the
contested bit if it finds that another thread has put
itself on the queue in the mean time.
3. Remove the if... block for unlocking an uncontested
umtx, and replace it with a KASSERT. The _only_ time
a thread should be unlocking a umtx in the kernel is
if it is contested.
2003-07-17 11:06:40 +00:00
|
|
|
* and we need to retry or we lost a race to the thread
|
|
|
|
* unlocking the umtx.
|
2003-04-01 01:10:42 +00:00
|
|
|
*/
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
umtxq_lock(td, umtx);
|
2004-07-12 15:28:31 +00:00
|
|
|
if (old == owner && (td->td_flags & TDF_UMTXWAKEUP) == 0)
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
error = umtx_sleep(td, umtx, td->td_priority | PCATCH,
|
|
|
|
"umtx", 0);
|
2004-07-12 15:28:31 +00:00
|
|
|
else
|
2003-04-01 01:10:42 +00:00
|
|
|
error = 0;
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
umtx_remove(uq, td, umtx);
|
|
|
|
/* unlocked by umtx_remove */
|
|
|
|
|
|
|
|
if (td->td_flags & TDF_UMTXWAKEUP) {
|
|
|
|
/*
|
|
|
|
* If we were resumed by umtxq_unlock, we should retry
|
|
|
|
* to avoid a race.
|
|
|
|
*/
|
|
|
|
mtx_lock_spin(&sched_lock);
|
|
|
|
td->td_flags &= ~TDF_UMTXWAKEUP;
|
|
|
|
mtx_unlock_spin(&sched_lock);
|
|
|
|
continue;
|
|
|
|
}
|
2003-04-01 01:10:42 +00:00
|
|
|
|
|
|
|
/*
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
* If we caught a signal, exit immediately.
|
2003-04-01 01:10:42 +00:00
|
|
|
*/
|
|
|
|
if (error)
|
2003-06-03 05:24:46 +00:00
|
|
|
return (error);
|
2003-04-01 01:10:42 +00:00
|
|
|
}
|
2003-07-18 17:58:37 +00:00
|
|
|
|
|
|
|
return (0);
|
2003-04-01 01:10:42 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
_umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap)
|
|
|
|
/* struct umtx *umtx */
|
|
|
|
{
|
|
|
|
struct umtx *umtx;
|
|
|
|
intptr_t owner;
|
2003-04-02 08:02:27 +00:00
|
|
|
intptr_t old;
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
int count;
|
2003-04-01 01:10:42 +00:00
|
|
|
|
|
|
|
umtx = uap->umtx;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Make sure we own this mtx.
|
|
|
|
*
|
|
|
|
* XXX Need a {fu,su}ptr this is not correct on arch where
|
|
|
|
* sizeof(intptr_t) != sizeof(long).
|
|
|
|
*/
|
2003-06-03 05:24:46 +00:00
|
|
|
if ((owner = fuword(&umtx->u_owner)) == -1)
|
|
|
|
return (EFAULT);
|
|
|
|
|
2004-07-02 00:40:07 +00:00
|
|
|
if ((owner & ~UMTX_CONTESTED) != td->td_tid)
|
2003-06-03 05:24:46 +00:00
|
|
|
return (EPERM);
|
2003-04-01 01:10:42 +00:00
|
|
|
|
Fix umtx locking, for libthr, in the kernel.
1. There was a race condition between a thread unlocking
a umtx and the thread contesting it. If the unlocking
thread won the race it may try to wakeup a thread that
was not yet in msleep(). The contesting thread would then
go to sleep to await a wakeup that would never come. It's
not possible to close the race by using a lock because
calls to casuptr() may have to fault a page in from swap.
Instead, the race was closed by introducing a flag that
the unlocking thread will set when waking up a thread.
The contesting thread will check for this flag before
going to sleep. For now the flag is kept in td_flags,
but it may be better to use some other member or create
a new one because of the possible performance/contention
issues of having to own sched_lock. Thanks to jhb for
pointing me in the right direction on this one.
2. Once a umtx was contested all future locks and unlocks
were happening in the kernel, regardless of whether it
was contested or not. To prevent this from happening,
when a thread locks a umtx it checks the queue for that
umtx and unsets the contested bit if there are no other
threads waiting on it. Again, this is slightly more
complicated than it needs to be because we can't hold
a lock across casuptr(). So, the thread has to check
the queue again after unseting the bit, and reset the
contested bit if it finds that another thread has put
itself on the queue in the mean time.
3. Remove the if... block for unlocking an uncontested
umtx, and replace it with a KASSERT. The _only_ time
a thread should be unlocking a umtx in the kernel is
if it is contested.
2003-07-17 11:06:40 +00:00
|
|
|
/* We should only ever be in here for contested locks */
|
2003-09-07 11:14:52 +00:00
|
|
|
if ((owner & UMTX_CONTESTED) == 0)
|
|
|
|
return (EINVAL);
|
2003-04-01 01:10:42 +00:00
|
|
|
|
2003-07-18 17:58:37 +00:00
|
|
|
/*
|
|
|
|
* When unlocking the umtx, it must be marked as unowned if
|
|
|
|
* there is zero or one thread only waiting for it.
|
|
|
|
* Otherwise, it must be marked as contested.
|
|
|
|
*/
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
old = casuptr((intptr_t *)&umtx->u_owner, owner, UMTX_UNOWNED);
|
2003-06-03 05:24:46 +00:00
|
|
|
if (old == -1)
|
|
|
|
return (EFAULT);
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
if (old != owner)
|
|
|
|
return (EINVAL);
|
2003-04-01 01:10:42 +00:00
|
|
|
|
|
|
|
/*
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
* At the point, a new thread can lock the umtx before we
|
|
|
|
* reach here, so contested bit will not be set, if there
|
|
|
|
* are two or more threads on wait queue, we should set
|
|
|
|
* contensted bit for them.
|
2003-04-01 01:10:42 +00:00
|
|
|
*/
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
count = umtxq_count(td, umtx);
|
|
|
|
if (count <= 0)
|
|
|
|
return (0);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If there is second thread waiting on umtx, set contested bit,
|
|
|
|
* if they are resumed before we reach here, it is harmless,
|
|
|
|
* just a bit unefficient.
|
|
|
|
*/
|
|
|
|
if (count > 1) {
|
|
|
|
owner = UMTX_UNOWNED;
|
|
|
|
for (;;) {
|
|
|
|
old = casuptr((intptr_t *)&umtx->u_owner, owner,
|
|
|
|
owner | UMTX_CONTESTED);
|
|
|
|
if (old == owner)
|
|
|
|
break;
|
|
|
|
if (old == -1)
|
|
|
|
return (EFAULT);
|
|
|
|
owner = old;
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* Another thread locked the umtx before us, so don't bother
|
|
|
|
* to wake more threads, that thread will do it when it unlocks
|
|
|
|
* the umtx.
|
|
|
|
*/
|
|
|
|
if ((owner & ~UMTX_CONTESTED) != 0)
|
|
|
|
return (0);
|
2003-04-01 01:10:42 +00:00
|
|
|
}
|
|
|
|
|
1. use per-chain mutex instead of global mutex to reduce
lock collision.
2. Fix two race conditions. One is between _umtx_unlock and signal,
also a thread was marked TDF_UMTXWAKEUP by _umtx_unlock, it is
possible a signal delivered to the thread will cause msleep
returns EINTR, and the thread breaks out of loop, this causes
umtx ownership is not transfered to the thread. Another is in
_umtx_unlock itself, when the function sets the umtx to
UMTX_UNOWNED state, a new thread can come in and lock the umtx,
also the function tries to set contested bit flag, but it will
fail. Although the function will wake a blocked thread, if that
thread breaks out of loop by signal, no contested bit will be set.
2004-11-30 12:02:53 +00:00
|
|
|
/* Wake blocked thread. */
|
|
|
|
umtx_signal(td, umtx);
|
|
|
|
|
2003-06-03 05:24:46 +00:00
|
|
|
return (0);
|
2003-04-01 01:10:42 +00:00
|
|
|
}
|