This is initial version of POSIX priority mutex support, a new userland

mutex structure is added as following:
struct umutex {
        __lwpid_t       m_owner;
        uint32_t        m_flags;
        uint32_t        m_ceilings[2];
        uint32_t        m_spare[4];
};
The m_owner represents owner thread, it is a thread id, in non-contested
case, userland can simply use atomic_cmpset_int to lock the mutex, if the
mutex is contested, high order bit will be set, and userland should do locking
and unlocking via kernel syscall. Flag UMUTEX_PRIO_INHERIT represents
pthread's PTHREAD_PRIO_INHERIT mutex, which when contention happens, kernel
should do priority propagating. Flag UMUTEX_PRIO_PROTECT indicates it is
pthread's PTHREAD_PRIO_PROTECT mutex, userland should initialize m_owner
to contested state UMUTEX_CONTESTED, then atomic_cmpset_int will be failure
and kernel syscall should be invoked to do locking, this becauses
for such a mutex, kernel should always boost the thread's priority before
it can lock the mutex, m_ceilings is used by PTHREAD_PRIO_PROTECT mutex,
the first element is used to boost thread's priority when it locked the mutex,
second element is used when the mutex is unlocked, the PTHREAD_PRIO_PROTECT
mutex's link list is kept in userland, the m_ceiling[1] is managed by thread
library so kernel needn't allocate memory to keep the link list, when such
a mutex is unlocked, kernel reset m_owner to UMUTEX_CONTESTED.
Flag USYNC_PROCESS_SHARED indicate if the synchronization object is process
shared, if the flag is not set, it saves a vm_map_lookup() call.

The umtx chain is still used as a sleep queue, when a thread is blocked on
PTHREAD_PRIO_INHERIT mutex, a umtx_pi is allocated to support priority
propagating, it is dynamically allocated and reference count is used,
it is not optimized but works well in my tests, while the umtx chain has
its own locking protocol, the priority propagating protocol are all protected
by sched_lock because priority propagating function is called with sched_lock
held from scheduler.

No visible performance degradation is found which these changes. Some parameter
names in _umtx_op syscall are renamed.

sys/kern/kern_thread.c

@@ -140,6 +140,7 @@ thread_ctor(void *mem, int size, void *arg, int flags)
 #ifdef AUDIT
 	audit_thread_alloc(td);
 #endif
+	umtx_thread_alloc(td);
 	return (0);
 }
 
@@ -194,9 +195,9 @@ thread_init(void *mem, int size, int flags)
 	cpu_thread_setup(td);
 	td->td_sleepqueue = sleepq_alloc();
 	td->td_turnstile = turnstile_alloc();
-	td->td_umtxq = umtxq_alloc();
 	td->td_sched = (struct td_sched *)&td[1];
 	sched_newthread(td);
+	umtx_thread_init(td);
 	return (0);
 }
 
@@ -211,7 +212,7 @@ thread_fini(void *mem, int size)
 	td = (struct thread *)mem;
 	turnstile_free(td->td_turnstile);
 	sleepq_free(td->td_sleepqueue);
-	umtxq_free(td->td_umtxq);
+	umtx_thread_fini(td);
 	vm_thread_dispose(td);
 }
 
@@ -486,6 +487,8 @@ thread_exit(void)
 		td->td_standin = NULL;
 	}
 
+	umtx_thread_exit(td);
+
 	/*
 	 * drop FPU & debug register state storage, or any other
 	 * architecture specific resources that

sys/kern/syscalls.master

@@ -794,8 +794,8 @@
 				    struct auditinfo_addr *auditinfo_addr, \
 				    u_int length); }
 453	AUE_AUDITCTL	STD	{ int auditctl(char *path); }
-454	AUE_NULL	STD	{ int _umtx_op(struct umtx *umtx, int op, \
+454	AUE_NULL	STD	{ int _umtx_op(void *obj, int op, \
-				    long id, void *uaddr, void *uaddr2); }
+				    uintptr_t val, void *uaddr1, void *uaddr2); }
 455	AUE_NULL	STD	{ int thr_new(struct thr_param *param, \
 				    int param_size); }
 456	AUE_NULL	STD	{ int sigqueue(pid_t pid, int signum, void *value); }

sys/sys/umtx.h

@@ -30,6 +30,7 @@
 #ifndef _SYS_UMTX_H_
 #define	_SYS_UMTX_H_
 
+#include <sys/_types.h>
 #include <sys/limits.h>
 
 /* 
@@ -43,11 +44,30 @@ struct umtx {
 	uintptr_t	u_owner;	/* Owner of the mutex. */
 };
 
+#define USYNC_PROCESS_SHARED	0x0001	/* Process shared sync objs */
+
+#define	UMUTEX_UNOWNED		0x0
+#define	UMUTEX_CONTESTED	0x80000000
+
+#define UMUTEX_ERROR_CHECK	0x0002	/* Error-checking mutex */
+#define UMUTEX_PRIO_INHERIT	0x0004	/* Priority inherited mutex */
+#define UMUTEX_PRIO_PROTECT	0x0008	/* Priority protect mutex */
+struct umutex {
+	__lwpid_t	m_owner;	/* Owner of the mutex */
+	uint32_t	m_flags;	/* Flags of the mutex */
+	uint32_t	m_ceilings[2];	/* Priority protect ceiling */
+	uint32_t	m_spare[4];	/* Spare space */
+};
+
 /* op code for _umtx_op */
 #define UMTX_OP_LOCK		0
 #define UMTX_OP_UNLOCK		1
 #define UMTX_OP_WAIT		2
 #define UMTX_OP_WAKE		3
+#define UMTX_OP_MUTEX_TRYLOCK	4
+#define UMTX_OP_MUTEX_LOCK	5
+#define UMTX_OP_MUTEX_UNLOCK	6
+#define UMTX_OP_SET_CEILING	7
 
 #ifndef _KERNEL
 
@@ -55,11 +75,7 @@ struct umtx {
  * System call for userland mutex operations.
  * Bug: assume sizeof(uintptr_t) == sizeof(long)
  */
-int _umtx_wait(struct umtx *umtx, uintptr_t expect,
+int _umtx_op(void *obj, int op, uintptr_t val, void *uaddr, void *uaddr2);
-	const struct timespec *timeout);
-int _umtx_wake(struct umtx *umtx, int nr_wakeup);
-int _umtx_op(struct umtx *umtx, int op, uintptr_t val,
-	void *uaddr, void *uaddr2);
 
 /*
  * Old (deprecated) userland mutex system calls.
@@ -120,9 +136,9 @@ umtx_unlock(struct umtx *umtx, uintptr_t id)
 }
 
 static __inline int
-umtx_wait(struct umtx *umtx, long id, const struct timespec *timeout)
+umtx_wait(long *p, long val, const struct timespec *timeout)
 {
-	if (_umtx_op(umtx, UMTX_OP_WAIT, id, 0,
+	if (_umtx_op(p, UMTX_OP_WAIT, val, 0,
 	    __DECONST(void *, timeout)) == -1)
 		return (errno);
 	return (0);
@@ -130,18 +146,24 @@ umtx_wait(struct umtx *umtx, long id, const struct timespec *timeout)
 
 /* Wake threads waiting on a user address. */
 static __inline int
-umtx_wake(struct umtx *umtx, int nr_wakeup)
+umtx_wake(long *p, int nr_wakeup)
 {
-	if (_umtx_op(umtx, UMTX_OP_WAKE, nr_wakeup, 0, 0) == -1)
+	if (_umtx_op(p, UMTX_OP_WAKE, nr_wakeup, 0, 0) == -1)
 		return (errno);
 	return (0);
 }
+
 #else
 
+struct thread;
+
 struct umtx_q *umtxq_alloc(void);
 void umtxq_free(struct umtx_q *);
-struct thread;
 int kern_umtx_wake(struct thread *td, void *uaddr, int n_wake);
 void umtx_pi_adjust(struct thread *td, u_char oldpri);
+void umtx_thread_init(struct thread *td);
+void umtx_thread_fini(struct thread *td);
+void umtx_thread_alloc(struct thread *td);
+void umtx_thread_exit(struct thread *td);
 #endif /* !_KERNEL */
 #endif /* !_SYS_UMTX_H_ */