Add a new "non-portable" mutex type, PTHREAD_MUTEX_ADAPTIVE_NP. This

is also implemented in glibc and is used by a number of existing
applications (mysql, firefox, etc).

This mutex type is a default mutex with the additional property that
it spins briefly when attempting to acquire a contested lock, doing
trylock operations in userland before entering the kernel to block if
eventually unsuccessful.

The expectation is that applications requesting this mutex type know
that the mutex is likely to be only held for very brief periods, so it
is faster to spin in userland and probably succeed in acquiring the
mutex, than to enter the kernel and sleep, only to be woken up almost
immediately.  This can help significantly in certain cases when
pthread mutexes are heavily contended and held for brief durations
(such as mysql).

Spin up to 200 times before entering the kernel, which represents only
a few us on modern CPUs.  No performance degradation was observed with
this value and it is sufficient to avoid a large performance drop in
mysql performance in the heavily contended pthread mutex case.

The libkse implementation is a NOP.

Reviewed by:      jeff
MFC after:        3 days

include/pthread.h

@@ -98,6 +98,7 @@
  * Static initialization values. 
  */
 #define PTHREAD_MUTEX_INITIALIZER	NULL
+#define PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP	NULL
 #define PTHREAD_COND_INITIALIZER	NULL
 #define PTHREAD_RWLOCK_INITIALIZER	NULL
 
@@ -128,6 +129,7 @@ enum pthread_mutextype {
 	PTHREAD_MUTEX_ERRORCHECK	= 1,	/* Default POSIX mutex */
 	PTHREAD_MUTEX_RECURSIVE		= 2,	/* Recursive mutex */
 	PTHREAD_MUTEX_NORMAL		= 3,	/* No error checking */
+	PTHREAD_MUTEX_ADAPTIVE_NP	= 4,	/* Adaptive mutex, spins briefly before blocking on lock */
 	PTHREAD_MUTEX_TYPE_MAX
 };
 

lib/libkse/thread/thr_mutex.c

@@ -179,6 +179,7 @@ __pthread_mutex_init(pthread_mutex_t *mutex,
 			/* case PTHREAD_MUTEX_DEFAULT: */
 			case PTHREAD_MUTEX_ERRORCHECK:
 			case PTHREAD_MUTEX_NORMAL:
+			case PTHREAD_MUTEX_ADAPTIVE_NP:
 				/* Nothing to do here. */
 				break;
 
@@ -971,6 +972,7 @@ mutex_self_trylock(struct pthread *curthread, pthread_mutex_t m)
 	/* case PTHREAD_MUTEX_DEFAULT: */
 	case PTHREAD_MUTEX_ERRORCHECK:
 	case PTHREAD_MUTEX_NORMAL:
+	case PTHREAD_MUTEX_ADAPTIVE_NP:
 		ret = EBUSY; 
 		break;
 
@@ -1002,6 +1004,7 @@ mutex_self_lock(struct pthread *curthread, pthread_mutex_t m)
 	switch (m->m_type) {
 	/* case PTHREAD_MUTEX_DEFAULT: */
 	case PTHREAD_MUTEX_ERRORCHECK:
+	case PTHREAD_MUTEX_ADAPTIVE_NP:
 		/*
 		 * POSIX specifies that mutexes should return EDEADLK if a
 		 * recursive lock is detected.

lib/libthr/thread/thr_mutex.c

@@ -39,6 +39,8 @@
 #include <string.h>
 #include <sys/param.h>
 #include <sys/queue.h>
+#include <machine/cpu.h>
+#include <machine/cpufunc.h>
 #include <pthread.h>
 #include "un-namespace.h"
 
@@ -64,6 +66,12 @@
 #define MUTEX_ASSERT_NOT_OWNED(m)
 #endif
 
+/*
+ * For adaptive mutexes, how many times to spin doing trylock2
+ * before entering the kernel to block
+ */
+#define MUTEX_ADAPTIVE_SPINS	200
+
 /*
  * Prototypes
  */
@@ -355,6 +363,25 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *mutex,
 	} else if (m->m_owner == curthread) {
 		ret = mutex_self_lock(m, abstime);
 	} else {
+		/*
+		 * For adaptive mutexes, spin for a bit in the expectation
+		 * that if the application requests this mutex type then
+		 * the lock is likely to be released quickly and it is
+		 * faster than entering the kernel
+		 */
+		if (m->m_type == PTHREAD_MUTEX_ADAPTIVE_NP) {
+			int count = MUTEX_ADAPTIVE_SPINS;
+
+			while (count--) {
+				ret = _thr_umutex_trylock2(&m->m_lock, id);
+				if (ret == 0)
+					break;
+				cpu_spinwait();
+			}
+		}
+		if (ret == 0)
+			goto done;
+
 		if (abstime == NULL) {
 			ret = __thr_umutex_lock(&m->m_lock);
 		} else if (__predict_false(
@@ -372,6 +399,7 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *mutex,
 			if (ret == EINTR)
 				ret = ETIMEDOUT;
 		}
+done:
 		if (ret == 0) {
 			m->m_owner = curthread;
 			/* Add to the list of owned mutexes: */
@@ -501,6 +529,7 @@ mutex_self_trylock(pthread_mutex_t m)
 	switch (m->m_type) {
 	case PTHREAD_MUTEX_ERRORCHECK:
 	case PTHREAD_MUTEX_NORMAL:
+	case PTHREAD_MUTEX_ADAPTIVE_NP:
 		ret = EBUSY; 
 		break;