Change atfork lock from mutex to rwlock, also make mutexes used by malloc()

module private type, when private type mutex is locked/unlocked, thread
critical region is entered or leaved. These changes makes fork()
async-signal safe which required by POSIX. Note that user's atfork handler
still needs to be async-signal safe, but it is not problem of libthr, it
is user's responsiblity.

lib/libthr/thread/thr_fork.c

@@ -89,9 +89,9 @@ _pthread_atfork(void (*prepare)(void), void (*parent)(void),
 	af->prepare = prepare;
 	af->parent = parent;
 	af->child = child;
-	THR_UMUTEX_LOCK(curthread, &_thr_atfork_lock);
+	_thr_rwl_rdlock(&_thr_atfork_lock);
 	TAILQ_INSERT_TAIL(&_thr_atfork_list, af, qe);
-	THR_UMUTEX_UNLOCK(curthread, &_thr_atfork_lock);
+	_thr_rwl_unlock(&_thr_atfork_lock);
 	return (0);
 }
 
@@ -104,7 +104,7 @@ __pthread_cxa_finalize(struct dl_phdr_info *phdr_info)
 	_thr_check_init();
 
 	curthread = _get_curthread();
-	THR_UMUTEX_LOCK(curthread, &_thr_atfork_lock);
+	_thr_rwl_wrlock(&_thr_atfork_lock);
 	TAILQ_FOREACH_SAFE(af, &_thr_atfork_list, qe, af1) {
 		if (__elf_phdr_match_addr(phdr_info, af->prepare) ||
 		    __elf_phdr_match_addr(phdr_info, af->parent) ||
@@ -113,7 +113,7 @@ __pthread_cxa_finalize(struct dl_phdr_info *phdr_info)
 			free(af);
 		}
 	}
-	THR_UMUTEX_UNLOCK(curthread, &_thr_atfork_lock);
+	_thr_rwl_unlock(&_thr_atfork_lock);
 	_thr_tsd_unload(phdr_info);
 	_thr_sigact_unload(phdr_info);
 }
@@ -137,7 +137,7 @@ _fork(void)
 
 	curthread = _get_curthread();
 
-	THR_UMUTEX_LOCK(curthread, &_thr_atfork_lock);
+	_thr_rwl_rdlock(&_thr_atfork_lock);
 
 	/* Run down atfork prepare handlers. */
 	TAILQ_FOREACH_REVERSE(af, &_thr_atfork_list, atfork_head, qe) {
@@ -145,6 +145,12 @@ _fork(void)
 			af->prepare();
 	}
 
+	/*
+	 * Block all signals until we reach a safe point.
+	 */
+	_thr_signal_block(curthread);
+	_thr_signal_prefork();
+
 	/*
 	 * All bets are off as to what should happen soon if the parent
 	 * process was not so kindly as to set up pthread fork hooks to
@@ -158,12 +164,6 @@ _fork(void)
 		was_threaded = 0;
 	}
 
-	/*
-	 * Block all signals until we reach a safe point.
-	 */
-	_thr_signal_block(curthread);
-	_thr_signal_prefork();
-
 	/* Fork a new process: */
 	if ((ret = __sys_fork()) == 0) {
 		/* Child process */
@@ -182,7 +182,7 @@ _fork(void)
 
 		/* clear other threads locked us. */
 		_thr_umutex_init(&curthread->lock);
-		_thr_umutex_init(&_thr_atfork_lock);
+		_mutex_fork(curthread);
 
 		_thr_signal_postfork_child();
 
@@ -192,13 +192,12 @@ _fork(void)
 
 		/* reinitialize libc spinlocks. */
 		_thr_spinlock_init();
-		_mutex_fork(curthread);
 
 		/* reinitalize library. */
 		_libpthread_init(curthread);
 
-		/* Ready to continue, unblock signals. */ 
-		_thr_signal_unblock(curthread);
+		/* atfork is reinitializeded by _libpthread_init()! */
+		_thr_rwl_rdlock(&_thr_atfork_lock);
 
 		if (was_threaded) {
 			__isthreaded = 1;
@@ -206,32 +205,36 @@ _fork(void)
 			__isthreaded = 0;
 		}
 
+		/* Ready to continue, unblock signals. */ 
+		_thr_signal_unblock(curthread);
+
 		/* Run down atfork child handlers. */
 		TAILQ_FOREACH(af, &_thr_atfork_list, qe) {
 			if (af->child != NULL)
 				af->child();
 		}
+		_thr_rwlock_unlock(&_thr_atfork_lock);
 	} else {
 		/* Parent process */
 		errsave = errno;
 
 		_thr_signal_postfork();
 
-		/* Ready to continue, unblock signals. */ 
-		_thr_signal_unblock(curthread);
-
 		if (was_threaded) {
 			_rtld_atfork_post(rtld_locks);
 			_malloc_postfork();
 		}
 
+		/* Ready to continue, unblock signals. */ 
+		_thr_signal_unblock(curthread);
+
 		/* Run down atfork parent handlers. */
 		TAILQ_FOREACH(af, &_thr_atfork_list, qe) {
 			if (af->parent != NULL)
 				af->parent();
 		}
 
-		THR_UMUTEX_UNLOCK(curthread, &_thr_atfork_lock);
+		_thr_rwlock_unlock(&_thr_atfork_lock);
 	}
 	errno = errsave;
 

lib/libthr/thread/thr_init.c

@@ -65,7 +65,7 @@ pthreadlist	_thread_list = TAILQ_HEAD_INITIALIZER(_thread_list);
 pthreadlist 	_thread_gc_list = TAILQ_HEAD_INITIALIZER(_thread_gc_list);
 int		_thread_active_threads = 1;
 atfork_head	_thr_atfork_list = TAILQ_HEAD_INITIALIZER(_thr_atfork_list);
-struct umutex	_thr_atfork_lock = DEFAULT_UMUTEX;
+struct urwlock	_thr_atfork_lock = DEFAULT_URWLOCK;
 
 struct pthread_prio	_thr_priorities[3] = {
 	{RTP_PRIO_MIN,  RTP_PRIO_MAX, 0}, /* FIFO */
@@ -427,7 +427,7 @@ init_private(void)
 	_thr_umutex_init(&_cond_static_lock);
 	_thr_umutex_init(&_rwlock_static_lock);
 	_thr_umutex_init(&_keytable_lock);
-	_thr_umutex_init(&_thr_atfork_lock);
+	_thr_urwlock_init(&_thr_atfork_lock);
 	_thr_umutex_init(&_thr_event_lock);
 	_thr_once_init();
 	_thr_spinlock_init();

lib/libthr/thread/thr_mutex.c

@@ -224,8 +224,12 @@ _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
 		.m_ceiling = 0
 	};
 	static const struct pthread_mutex_attr *pattr = &attr;
+	int ret;
 
-	return mutex_init(mutex, (pthread_mutexattr_t *)&pattr, calloc_cb);
+	ret = mutex_init(mutex, (pthread_mutexattr_t *)&pattr, calloc_cb);
+	if (ret == 0)
+		(*mutex)->m_private = 1;
+	return (ret);
 }
 
 void
@@ -319,13 +323,16 @@ mutex_trylock_common(struct pthread *curthread, pthread_mutex_t *mutex)
 
 	id = TID(curthread);
 	m = *mutex;
+	if (m->m_private)
+		THR_CRITICAL_ENTER(curthread);
 	ret = _thr_umutex_trylock(&m->m_lock, id);
 	if (ret == 0) {
 		ENQUEUE_MUTEX(curthread, m);
 	} else if (m->m_owner == curthread) {
 		ret = mutex_self_trylock(m);
 	} /* else {} */
-
+	if (ret && m->m_private)
+		THR_CRITICAL_LEAVE(curthread);
 	return (ret);
 }
 
@@ -417,13 +424,19 @@ static inline int
 mutex_lock_common(struct pthread *curthread, struct pthread_mutex *m,
 	const struct timespec *abstime)
 {
+	int ret;
 
+	if (m->m_private)
+		THR_CRITICAL_ENTER(curthread);
 	if (_thr_umutex_trylock2(&m->m_lock, TID(curthread)) == 0) {
 		ENQUEUE_MUTEX(curthread, m);
-		return (0);
+		ret = 0;
+	} else {
+		ret = mutex_lock_sleep(curthread, m, abstime);
 	}
-	
-	return (mutex_lock_sleep(curthread, m, abstime));
+	if (ret && m->m_private)
+		THR_CRITICAL_LEAVE(curthread);
+	return (ret);
 }
 
 int
@@ -625,6 +638,8 @@ mutex_unlock_common(pthread_mutex_t *mutex)
 		MUTEX_INIT_LINK(m);
 		_thr_umutex_unlock(&m->m_lock, id);
 	}
+	if (m->m_private)
+		THR_CRITICAL_LEAVE(curthread);
 	return (0);
 }
 
@@ -660,6 +675,9 @@ _mutex_cv_unlock(pthread_mutex_t *mutex, int *count)
 	}
 	MUTEX_INIT_LINK(m);
 	_thr_umutex_unlock(&m->m_lock, TID(curthread));
+
+	if (m->m_private)
+		THR_CRITICAL_LEAVE(curthread);
 	return (0);
 }
 

lib/libthr/thread/thr_private.h

@@ -132,6 +132,7 @@ struct pthread_mutex {
 	int				m_refcount;
 	int				m_spinloops;
 	int				m_yieldloops;
+	int				m_private;
 	/*
 	 * Link for all mutexes a thread currently owns.
 	 */
@@ -587,7 +588,7 @@ extern pthreadlist	_thread_gc_list __hidden;
 
 extern int		_thread_active_threads;
 extern atfork_head	_thr_atfork_list __hidden;
-extern struct umutex	_thr_atfork_lock __hidden;
+extern struct urwlock	_thr_atfork_lock __hidden;
 
 /* Default thread attributes: */
 extern struct pthread_attr _pthread_attr_default __hidden;

lib/libthr/thread/thr_umtx.c

@@ -47,6 +47,13 @@ _thr_umutex_init(struct umutex *mtx)
 	*mtx = default_mtx;
 }
 
+void
+_thr_urwlock_init(struct urwlock *rwl)
+{
+	static struct urwlock default_rwl = DEFAULT_URWLOCK;
+	*rwl = default_rwl;
+}
+
 int
 __thr_umutex_lock(struct umutex *mtx, uint32_t id)
 {

lib/libthr/thread/thr_umtx.h

@@ -32,7 +32,8 @@
 #include <strings.h>
 #include <sys/umtx.h>
 
-#define DEFAULT_UMUTEX	{0,0, {0,0},{0,0,0,0}}
+#define DEFAULT_UMUTEX	{0,0,{0,0},{0,0,0,0}}
+#define DEFAULT_URWLOCK {0,0,0,0,{0,0,0,0}}
 
 int __thr_umutex_lock(struct umutex *mtx, uint32_t id) __hidden;
 int __thr_umutex_timedlock(struct umutex *mtx, uint32_t id,
@@ -43,6 +44,8 @@ int __thr_umutex_set_ceiling(struct umutex *mtx, uint32_t ceiling,
 	uint32_t *oldceiling) __hidden;
 
 void _thr_umutex_init(struct umutex *mtx) __hidden;
+void _thr_urwlock_init(struct urwlock *rwl) __hidden;
+
 int _thr_umtx_wait(volatile long *mtx, long exp,
 	const struct timespec *timeout) __hidden;
 int _thr_umtx_wait_uint(volatile u_int *mtx, u_int exp,