The libthr code makes use of higher-level primitives (pthread_mutex_t and

pthread_cond_t) internaly in addition to the low-level spinlock_t. The
garbage collector mutex and condition variable are two such examples. This
might lead to critical sections nested within critical sections. Implement
a reference counting mechanism so that signals are masked only on the first
entry and unmasked on the last exit.

I'm not sure I like the idea of nested critical sections, but if
the library is going to use the pthread primitives it might be necessary.

Approved by:	re/blanket libthr

lib/libthr/thread/thr_kern.c

@@ -68,6 +68,13 @@ _thread_critical_enter(pthread_t pthread)
 	 * acquired the giant lock.
 	 */
 	_SPINLOCK(&pthread->lock);
+
+	/* If we are already in a critical section, just up the refcount */
+	if (++curthread->crit_ref > 1)
+		return;
+	PTHREAD_ASSERT(curthread->crit_ref == 1,
+	    ("Critical section reference count must be 1!"));
+
 	if (__sys_sigprocmask(SIG_SETMASK, &set, &sav)) {
 		_thread_printf(STDERR_FILENO, "Critical Enter: sig err %d\n",
 		    errno);
@@ -81,6 +88,12 @@ _thread_critical_exit(pthread_t pthread)
 {
 	sigset_t set;
 
+	/* We might be in a nested critical section */
+	if (--curthread->crit_ref > 0)
+		return;
+	PTHREAD_ASSERT(curthread->crit_ref == 0,
+	    ("Non-Zero critical section reference count."));
+
 	/*
 	 * Restore signals.
 	 */

lib/libthr/thread/thr_private.h

@@ -426,6 +426,7 @@ struct pthread {
 	u_int64_t		uniqueid; /* for gdb */
 	thr_id_t		thr_id;
 	sigset_t		savedsig;
+	int			crit_ref; /* crit. section netsting level */
 
 	/*
 	 * Lock for accesses to this thread structure.