Implement MTX_RECURSE flag for mtx_init().

All calls to mtx_init() for mutexes that recurse must now include
the MTX_RECURSE bit in the flag argument variable. This change is in
preparation for an upcoming (further) mutex API cleanup.
The witness code will call panic() if a lock is found to recurse but
the MTX_RECURSE bit was not set during the lock's initialization.

The old MTX_RECURSE "state" bit (in mtx_lock) has been renamed to
MTX_RECURSED, which is more appropriate given its meaning.

The following locks have been made "recursive," thus far:
eventhandler, Giant, callout, sched_lock, possibly some others declared
in the architecture-specific code, all of the network card driver locks
in pci/, as well as some other locks in dev/ stuff that I've found to
be recursive.

Reviewed by: jhb

sys/alpha/alpha/machdep.c

@@ -390,7 +390,7 @@ cpu_startup(dummy)
 		TAILQ_INIT(&callwheel[i]);
 	}
 
-	mtx_init(&callout_lock, "callout", MTX_SPIN);
+	mtx_init(&callout_lock, "callout", MTX_SPIN | MTX_RECURSE);
 
 #if defined(USERCONFIG)
 #if defined(USERCONFIG_BOOT)
@@ -1003,8 +1003,8 @@ alpha_init(pfn, ptb, bim, bip, biv)
 	/*
 	 * Initialise mutexes.
 	 */
-	mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD);
-	mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD);
+	mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE);
+	mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE);
 
 	/*
 	 * Look at arguments passed to us and compute boothowto.

sys/amd64/amd64/machdep.c

@@ -422,7 +422,7 @@ cpu_startup(dummy)
 		TAILQ_INIT(&callwheel[i]);
 	}
 
-	mtx_init(&callout_lock, "callout", MTX_SPIN);
+	mtx_init(&callout_lock, "callout", MTX_SPIN | MTX_RECURSE);
 
 #if defined(USERCONFIG)
 	userconfig();
@@ -441,7 +441,7 @@ cpu_startup(dummy)
 	SLIST_INIT(&cpuhead);
 	SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu);
 
-	mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD);
+	mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE);
 
 #ifdef SMP
 	/*
@@ -1939,7 +1939,7 @@ init386(first)
 	/*
 	 * We need this mutex before the console probe.
 	 */
-	mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD);
+	mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD | MTX_RECURSE);
 
 	/*
 	 * Initialize the console before we print anything out.
@@ -1954,7 +1954,7 @@ init386(first)
 	/*
 	 * Giant is used early for at least debugger traps and unexpected traps.
 	 */
-	mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD);
+	mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE);
 
 #ifdef DDB
 	kdb_init();

sys/dev/an/if_an.c

@@ -288,7 +288,8 @@ int an_attach(sc, unit, flags)
 {
 	struct ifnet		*ifp = &sc->arpcom.ac_if;
 
-	mtx_init(&sc->an_mtx, device_get_nameunit(sc->an_dev), MTX_DEF);
+	mtx_init(&sc->an_mtx, device_get_nameunit(sc->an_dev), MTX_DEF |
+	    MTX_RECURSE);
 	AN_LOCK(sc);
 
 	sc->an_gone = 0;

sys/dev/dc/if_dc.c

@@ -1762,7 +1762,7 @@ static int dc_attach(dev)
 	unit = device_get_unit(dev);
 	bzero(sc, sizeof(struct dc_softc));
 
-	mtx_init(&sc->dc_mtx, device_get_nameunit(dev), MTX_DEF);
+	mtx_init(&sc->dc_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 	DC_LOCK(sc);
 
 	/*

sys/dev/fxp/if_fxp.c

@@ -316,7 +316,7 @@ fxp_attach(device_t dev)
 	u_long val;
 	int rid;
 
-	mtx_init(&sc->sc_mtx, device_get_nameunit(dev), MTX_DEF);
+	mtx_init(&sc->sc_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 	callout_handle_init(&sc->stat_ch);
 
 	FXP_LOCK(sc);

sys/dev/sf/if_sf.c

@@ -679,7 +679,7 @@ static int sf_attach(dev)
 	unit = device_get_unit(dev);
 	bzero(sc, sizeof(struct sf_softc));
 
-	mtx_init(&sc->sf_mtx, device_get_nameunit(dev), MTX_DEF);
+	mtx_init(&sc->sf_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 	SF_LOCK(sc);
 	/*
 	 * Handle power management nonsense.

sys/dev/sk/if_sk.c

@@ -1194,7 +1194,7 @@ static int sk_attach(dev)
 	unit = device_get_unit(dev);
 	bzero(sc, sizeof(struct sk_softc));
 
-	mtx_init(&sc->sk_mtx, device_get_nameunit(dev), MTX_DEF);
+	mtx_init(&sc->sk_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 	SK_LOCK(sc);
 
 	/*

sys/dev/ti/if_ti.c

@@ -1476,7 +1476,7 @@ static int ti_attach(dev)
 	unit = device_get_unit(dev);
 	bzero(sc, sizeof(struct ti_softc));
 
-	mtx_init(&sc->ti_mtx, device_get_nameunit(dev), MTX_DEF);
+	mtx_init(&sc->ti_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 	TI_LOCK(sc);
 
 	/*

sys/dev/usb/if_aue.c

@@ -681,7 +681,8 @@ USB_ATTACH(aue)
 		}
 	}
 
-	mtx_init(&sc->aue_mtx, device_get_nameunit(self), MTX_DEF);
+	mtx_init(&sc->aue_mtx, device_get_nameunit(self), MTX_DEF |
+	    MTX_RECURSE);
 	AUE_LOCK(sc);
 
 	/* Reset the adapter. */

sys/dev/usb/if_cue.c

@@ -504,7 +504,8 @@ USB_ATTACH(cue)
 		}
 	}
 
-	mtx_init(&sc->cue_mtx, device_get_nameunit(self), MTX_DEF);
+	mtx_init(&sc->cue_mtx, device_get_nameunit(self), MTX_DEF |
+	    MTX_RECURSE);
 	CUE_LOCK(sc);
 
 #ifdef notdef

sys/dev/usb/if_kue.c

@@ -450,7 +450,8 @@ USB_ATTACH(kue)
 		}
 	}
 
-	mtx_init(&sc->kue_mtx, device_get_nameunit(self), MTX_DEF);
+	mtx_init(&sc->kue_mtx, device_get_nameunit(self), MTX_DEF |
+	    MTX_RECURSE);
 	KUE_LOCK(sc);
 
 	/* Load the firmware into the NIC. */

sys/dev/vr/if_vr.c

@@ -643,7 +643,7 @@ static int vr_attach(dev)
 	unit = device_get_unit(dev);
 	bzero(sc, sizeof(struct vr_softc *));
 
-	mtx_init(&sc->vr_mtx, device_get_nameunit(dev), MTX_DEF);
+	mtx_init(&sc->vr_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 	VR_LOCK(sc);
 
 	/*

sys/dev/wi/if_wi.c

@@ -304,7 +304,7 @@ static int wi_pccard_attach(device_t dev)
 		return (error);
 	}
 
-	mtx_init(&sc->wi_mtx, device_get_nameunit(dev), MTX_DEF);
+	mtx_init(&sc->wi_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 	WI_LOCK(sc);
 
 	/* Reset the NIC. */

sys/i386/i386/machdep.c

@@ -422,7 +422,7 @@ cpu_startup(dummy)
 		TAILQ_INIT(&callwheel[i]);
 	}
 
-	mtx_init(&callout_lock, "callout", MTX_SPIN);
+	mtx_init(&callout_lock, "callout", MTX_SPIN | MTX_RECURSE);
 
 #if defined(USERCONFIG)
 	userconfig();
@@ -441,7 +441,7 @@ cpu_startup(dummy)
 	SLIST_INIT(&cpuhead);
 	SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu);
 
-	mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD);
+	mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE);
 
 #ifdef SMP
 	/*
@@ -1939,7 +1939,7 @@ init386(first)
 	/*
 	 * We need this mutex before the console probe.
 	 */
-	mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD);
+	mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD | MTX_RECURSE);
 
 	/*
 	 * Initialize the console before we print anything out.
@@ -1954,7 +1954,7 @@ init386(first)
 	/*
 	 * Giant is used early for at least debugger traps and unexpected traps.
 	 */
-	mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD);
+	mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE);
 
 #ifdef DDB
 	kdb_init();

sys/i386/isa/if_el.c

@@ -149,7 +149,7 @@ el_probe(device_t dev)
 	sc->el_btag = rman_get_bustag(sc->el_res);
 	sc->el_bhandle = rman_get_bushandle(sc->el_res);
 	mtx_init(&sc->el_mtx,
-	    device_get_nameunit(dev), MTX_DEF);
+	    device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 	EL_LOCK(sc);
 
 	/* Now attempt to grab the station address from the PROM

sys/ia64/ia64/machdep.c

@@ -319,7 +319,7 @@ cpu_startup(dummy)
 		TAILQ_INIT(&callwheel[i]);
 	}
 
-	mtx_init(&callout_lock, "callout", MTX_SPIN);
+	mtx_init(&callout_lock, "callout", MTX_SPIN | MTX_RECURSE);
 
 #if defined(USERCONFIG)
 #if defined(USERCONFIG_BOOT)
@@ -590,8 +590,8 @@ ia64_init()
 	/*
 	 * Initialise mutexes.
 	 */
-	mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD);
-	mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD);
+	mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE);
+	mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE);
 
 #if 0
 	/*

sys/kern/kern_mutex.c

@@ -259,7 +259,7 @@ mtx_enter_hard(struct mtx *m, int type, int saveintr)
 	case MTX_DEF:
 		if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)p) {
 			m->mtx_recurse++;
-			atomic_set_ptr(&m->mtx_lock, MTX_RECURSE);
+			atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
 			if ((type & MTX_QUIET) == 0)
 				CTR1(KTR_LOCK, "mtx_enter: 0x%p recurse", m);
 			return;
@@ -434,9 +434,9 @@ mtx_exit_hard(struct mtx *m, int type)
 	switch (type) {
 	case MTX_DEF:
 	case MTX_DEF | MTX_NOSWITCH:
-		if (m->mtx_recurse != 0) {
+		if (mtx_recursed(m)) {
 			if (--(m->mtx_recurse) == 0)
-				atomic_clear_ptr(&m->mtx_lock, MTX_RECURSE);
+				atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED);
 			if ((type & MTX_QUIET) == 0)
 				CTR1(KTR_LOCK, "mtx_exit: 0x%p unrecurse", m);
 			return;
@@ -501,7 +501,7 @@ mtx_exit_hard(struct mtx *m, int type)
 		break;
 	case MTX_SPIN:
 	case MTX_SPIN | MTX_FIRST:
-		if (m->mtx_recurse != 0) {
+		if (mtx_recursed(m)) {
 			m->mtx_recurse--;
 			return;
 		}
@@ -515,7 +515,7 @@ mtx_exit_hard(struct mtx *m, int type)
 		}
 		break;
 	case MTX_SPIN | MTX_TOPHALF:
-		if (m->mtx_recurse != 0) {
+		if (mtx_recursed(m)) {
 			m->mtx_recurse--;
 			return;
 		}
@@ -655,7 +655,7 @@ mtx_destroy(struct mtx *m)
 	if (!mtx_owned(m)) {
 		MPASS(m->mtx_lock == MTX_UNOWNED);
 	} else {
-		MPASS((m->mtx_lock & (MTX_RECURSE|MTX_CONTESTED)) == 0);
+		MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0);
 	}
 	mtx_validate(m, MV_DESTROY);		/* diagnostic */
 #endif
@@ -701,8 +701,9 @@ struct witness {
 	u_char		 w_Giant_squawked:1;
 	u_char		 w_other_squawked:1;
 	u_char		 w_same_squawked:1;
-	u_char		 w_sleep:1;
-	u_char		 w_spin:1;	/* this is a spin mutex */
+	u_char		 w_sleep:1;	/* MTX_DEF type mutex. */
+	u_char		 w_spin:1;	/* MTX_SPIN type mutex. */
+	u_char		 w_recurse:1;	/* MTX_RECURSE mutex option. */
 	u_int		 w_level;
 	struct witness	*w_children[WITNESS_NCHILDREN];
 };
@@ -838,11 +839,16 @@ witness_enter(struct mtx *m, int flags, const char *file, int line)
 	p = CURPROC;
 
 	if (flags & MTX_SPIN) {
-		if (!w->w_spin)
+		if (!(w->w_spin))
 			panic("mutex_enter: MTX_SPIN on MTX_DEF mutex %s @"
 			    " %s:%d", m->mtx_description, file, line);
-		if (m->mtx_recurse != 0)
+		if (mtx_recursed(m)) {
+			if (!(w->w_recurse))
+				panic("mutex_enter: recursion on non-recursive"
+				    " mutex %s @ %s:%d", m->mtx_description,
+				    file, line);
 			return;
+		}
 		mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
 		i = PCPU_GET(witness_spin_check);
 		if (i != 0 && w->w_level < i) {
@@ -864,8 +870,13 @@ witness_enter(struct mtx *m, int flags, const char *file, int line)
 		panic("mutex_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
 		    m->mtx_description, file, line);
 
-	if (m->mtx_recurse != 0)
+	if (mtx_recursed(m)) {
+		if (!(w->w_recurse))
+			panic("mutex_enter: recursion on non-recursive"
+			    " mutex %s @ %s:%d", m->mtx_description,
+			    file, line);
 		return;
+	}
 	if (witness_dead)
 		goto out;
 	if (cold)
@@ -971,11 +982,16 @@ witness_exit(struct mtx *m, int flags, const char *file, int line)
 	w = m->mtx_witness;
 
 	if (flags & MTX_SPIN) {
-		if (!w->w_spin)
+		if (!(w->w_spin))
 			panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @"
 			    " %s:%d", m->mtx_description, file, line);
-		if (m->mtx_recurse != 0)
+		if (mtx_recursed(m)) {
+			if (!(w->w_recurse))
+				panic("mutex_exit: recursion on non-recursive"
+				    " mutex %s @ %s:%d", m->mtx_description,
+				    file, line); 
 			return;
+		}
 		mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
 		PCPU_SET(witness_spin_check,
 		    PCPU_GET(witness_spin_check) & ~w->w_level);
@@ -986,8 +1002,13 @@ witness_exit(struct mtx *m, int flags, const char *file, int line)
 		panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
 		    m->mtx_description, file, line);
 
-	if (m->mtx_recurse != 0)
+	if (mtx_recursed(m)) {
+		if (!(w->w_recurse))
+			panic("mutex_exit: recursion on non-recursive"
+			    " mutex %s @ %s:%d", m->mtx_description,
+			    file, line); 
 		return;
+	}
 
 	if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold)
 		panic("switchable mtx_exit() of %s when not legal @ %s:%d",
@@ -1005,12 +1026,17 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line)
 	if (panicstr)
 		return;
 	if (flags & MTX_SPIN) {
-		if (!w->w_spin)
+		if (!(w->w_spin))
 			panic("mutex_try_enter: "
 			    "MTX_SPIN on MTX_DEF mutex %s @ %s:%d",
 			    m->mtx_description, file, line);
-		if (m->mtx_recurse != 0)
+		if (mtx_recursed(m)) {
+			if (!(w->w_recurse))
+				panic("mutex_try_enter: recursion on"
+				    " non-recursive mutex %s @ %s:%d",
+				    m->mtx_description, file, line);
 			return;
+		}
 		mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
 		PCPU_SET(witness_spin_check,
 		    PCPU_GET(witness_spin_check) | w->w_level);
@@ -1026,9 +1052,13 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line)
 		panic("mutex_try_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
 		    m->mtx_description, file, line);
 
-	if (m->mtx_recurse != 0)
+	if (mtx_recursed(m)) {
+		if (!(w->w_recurse))
+			panic("mutex_try_enter: recursion on non-recursive"
+			    " mutex %s @ %s:%d", m->mtx_description, file,
+			    line);
 		return;
-
+	}
 	w->w_file = file;
 	w->w_line = line;
 	m->mtx_line = line;
@@ -1156,7 +1186,12 @@ enroll(const char *description, int flag)
 		if (*order == NULL)
 			panic("spin lock %s not in order list", description);
 		w->w_level = i; 
-	}
+	} else
+		w->w_sleep = 1;
+
+	if (flag & MTX_RECURSE)
+		w->w_recurse = 1;
+
 	return (w);
 }
 
@@ -1270,7 +1305,7 @@ witness_levelall (void)
 	struct witness *w, *w1;
 
 	for (w = w_all; w; w = w->w_next)
-		if (!w->w_spin)
+		if (!(w->w_spin))
 			w->w_level = 0;
 	for (w = w_all; w; w = w->w_next) {
 		if (w->w_spin)

sys/kern/subr_eventhandler.c

@@ -53,7 +53,7 @@ static void
 eventhandler_init(void *dummy __unused)
 {
     TAILQ_INIT(&eventhandler_lists);
-    mtx_init(&eventhandler_mutex, "eventhandler", MTX_DEF);
+    mtx_init(&eventhandler_mutex, "eventhandler", MTX_DEF | MTX_RECURSE);
     eventhandler_lists_initted = 1;
 }
 SYSINIT(eventhandlers, SI_SUB_EVENTHANDLER, SI_ORDER_FIRST, eventhandler_init,

sys/kern/subr_turnstile.c

@@ -259,7 +259,7 @@ mtx_enter_hard(struct mtx *m, int type, int saveintr)
 	case MTX_DEF:
 		if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)p) {
 			m->mtx_recurse++;
-			atomic_set_ptr(&m->mtx_lock, MTX_RECURSE);
+			atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
 			if ((type & MTX_QUIET) == 0)
 				CTR1(KTR_LOCK, "mtx_enter: 0x%p recurse", m);
 			return;
@@ -434,9 +434,9 @@ mtx_exit_hard(struct mtx *m, int type)
 	switch (type) {
 	case MTX_DEF:
 	case MTX_DEF | MTX_NOSWITCH:
-		if (m->mtx_recurse != 0) {
+		if (mtx_recursed(m)) {
 			if (--(m->mtx_recurse) == 0)
-				atomic_clear_ptr(&m->mtx_lock, MTX_RECURSE);
+				atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED);
 			if ((type & MTX_QUIET) == 0)
 				CTR1(KTR_LOCK, "mtx_exit: 0x%p unrecurse", m);
 			return;
@@ -501,7 +501,7 @@ mtx_exit_hard(struct mtx *m, int type)
 		break;
 	case MTX_SPIN:
 	case MTX_SPIN | MTX_FIRST:
-		if (m->mtx_recurse != 0) {
+		if (mtx_recursed(m)) {
 			m->mtx_recurse--;
 			return;
 		}
@@ -515,7 +515,7 @@ mtx_exit_hard(struct mtx *m, int type)
 		}
 		break;
 	case MTX_SPIN | MTX_TOPHALF:
-		if (m->mtx_recurse != 0) {
+		if (mtx_recursed(m)) {
 			m->mtx_recurse--;
 			return;
 		}
@@ -655,7 +655,7 @@ mtx_destroy(struct mtx *m)
 	if (!mtx_owned(m)) {
 		MPASS(m->mtx_lock == MTX_UNOWNED);
 	} else {
-		MPASS((m->mtx_lock & (MTX_RECURSE|MTX_CONTESTED)) == 0);
+		MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0);
 	}
 	mtx_validate(m, MV_DESTROY);		/* diagnostic */
 #endif
@@ -701,8 +701,9 @@ struct witness {
 	u_char		 w_Giant_squawked:1;
 	u_char		 w_other_squawked:1;
 	u_char		 w_same_squawked:1;
-	u_char		 w_sleep:1;
-	u_char		 w_spin:1;	/* this is a spin mutex */
+	u_char		 w_sleep:1;	/* MTX_DEF type mutex. */
+	u_char		 w_spin:1;	/* MTX_SPIN type mutex. */
+	u_char		 w_recurse:1;	/* MTX_RECURSE mutex option. */
 	u_int		 w_level;
 	struct witness	*w_children[WITNESS_NCHILDREN];
 };
@@ -838,11 +839,16 @@ witness_enter(struct mtx *m, int flags, const char *file, int line)
 	p = CURPROC;
 
 	if (flags & MTX_SPIN) {
-		if (!w->w_spin)
+		if (!(w->w_spin))
 			panic("mutex_enter: MTX_SPIN on MTX_DEF mutex %s @"
 			    " %s:%d", m->mtx_description, file, line);
-		if (m->mtx_recurse != 0)
+		if (mtx_recursed(m)) {
+			if (!(w->w_recurse))
+				panic("mutex_enter: recursion on non-recursive"
+				    " mutex %s @ %s:%d", m->mtx_description,
+				    file, line);
 			return;
+		}
 		mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
 		i = PCPU_GET(witness_spin_check);
 		if (i != 0 && w->w_level < i) {
@@ -864,8 +870,13 @@ witness_enter(struct mtx *m, int flags, const char *file, int line)
 		panic("mutex_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
 		    m->mtx_description, file, line);
 
-	if (m->mtx_recurse != 0)
+	if (mtx_recursed(m)) {
+		if (!(w->w_recurse))
+			panic("mutex_enter: recursion on non-recursive"
+			    " mutex %s @ %s:%d", m->mtx_description,
+			    file, line);
 		return;
+	}
 	if (witness_dead)
 		goto out;
 	if (cold)
@@ -971,11 +982,16 @@ witness_exit(struct mtx *m, int flags, const char *file, int line)
 	w = m->mtx_witness;
 
 	if (flags & MTX_SPIN) {
-		if (!w->w_spin)
+		if (!(w->w_spin))
 			panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @"
 			    " %s:%d", m->mtx_description, file, line);
-		if (m->mtx_recurse != 0)
+		if (mtx_recursed(m)) {
+			if (!(w->w_recurse))
+				panic("mutex_exit: recursion on non-recursive"
+				    " mutex %s @ %s:%d", m->mtx_description,
+				    file, line); 
 			return;
+		}
 		mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
 		PCPU_SET(witness_spin_check,
 		    PCPU_GET(witness_spin_check) & ~w->w_level);
@@ -986,8 +1002,13 @@ witness_exit(struct mtx *m, int flags, const char *file, int line)
 		panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
 		    m->mtx_description, file, line);
 
-	if (m->mtx_recurse != 0)
+	if (mtx_recursed(m)) {
+		if (!(w->w_recurse))
+			panic("mutex_exit: recursion on non-recursive"
+			    " mutex %s @ %s:%d", m->mtx_description,
+			    file, line); 
 		return;
+	}
 
 	if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold)
 		panic("switchable mtx_exit() of %s when not legal @ %s:%d",
@@ -1005,12 +1026,17 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line)
 	if (panicstr)
 		return;
 	if (flags & MTX_SPIN) {
-		if (!w->w_spin)
+		if (!(w->w_spin))
 			panic("mutex_try_enter: "
 			    "MTX_SPIN on MTX_DEF mutex %s @ %s:%d",
 			    m->mtx_description, file, line);
-		if (m->mtx_recurse != 0)
+		if (mtx_recursed(m)) {
+			if (!(w->w_recurse))
+				panic("mutex_try_enter: recursion on"
+				    " non-recursive mutex %s @ %s:%d",
+				    m->mtx_description, file, line);
 			return;
+		}
 		mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
 		PCPU_SET(witness_spin_check,
 		    PCPU_GET(witness_spin_check) | w->w_level);
@@ -1026,9 +1052,13 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line)
 		panic("mutex_try_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
 		    m->mtx_description, file, line);
 
-	if (m->mtx_recurse != 0)
+	if (mtx_recursed(m)) {
+		if (!(w->w_recurse))
+			panic("mutex_try_enter: recursion on non-recursive"
+			    " mutex %s @ %s:%d", m->mtx_description, file,
+			    line);
 		return;
-
+	}
 	w->w_file = file;
 	w->w_line = line;
 	m->mtx_line = line;
@@ -1156,7 +1186,12 @@ enroll(const char *description, int flag)
 		if (*order == NULL)
 			panic("spin lock %s not in order list", description);
 		w->w_level = i; 
-	}
+	} else
+		w->w_sleep = 1;
+
+	if (flag & MTX_RECURSE)
+		w->w_recurse = 1;
+
 	return (w);
 }
 
@@ -1270,7 +1305,7 @@ witness_levelall (void)
 	struct witness *w, *w1;
 
 	for (w = w_all; w; w = w->w_next)
-		if (!w->w_spin)
+		if (!(w->w_spin))
 			w->w_level = 0;
 	for (w = w_all; w; w = w->w_next) {
 		if (w->w_spin)

sys/kern/subr_witness.c

@@ -259,7 +259,7 @@ mtx_enter_hard(struct mtx *m, int type, int saveintr)
 	case MTX_DEF:
 		if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)p) {
 			m->mtx_recurse++;
-			atomic_set_ptr(&m->mtx_lock, MTX_RECURSE);
+			atomic_set_ptr(&m->mtx_lock, MTX_RECURSED);
 			if ((type & MTX_QUIET) == 0)
 				CTR1(KTR_LOCK, "mtx_enter: 0x%p recurse", m);
 			return;
@@ -434,9 +434,9 @@ mtx_exit_hard(struct mtx *m, int type)
 	switch (type) {
 	case MTX_DEF:
 	case MTX_DEF | MTX_NOSWITCH:
-		if (m->mtx_recurse != 0) {
+		if (mtx_recursed(m)) {
 			if (--(m->mtx_recurse) == 0)
-				atomic_clear_ptr(&m->mtx_lock, MTX_RECURSE);
+				atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED);
 			if ((type & MTX_QUIET) == 0)
 				CTR1(KTR_LOCK, "mtx_exit: 0x%p unrecurse", m);
 			return;
@@ -501,7 +501,7 @@ mtx_exit_hard(struct mtx *m, int type)
 		break;
 	case MTX_SPIN:
 	case MTX_SPIN | MTX_FIRST:
-		if (m->mtx_recurse != 0) {
+		if (mtx_recursed(m)) {
 			m->mtx_recurse--;
 			return;
 		}
@@ -515,7 +515,7 @@ mtx_exit_hard(struct mtx *m, int type)
 		}
 		break;
 	case MTX_SPIN | MTX_TOPHALF:
-		if (m->mtx_recurse != 0) {
+		if (mtx_recursed(m)) {
 			m->mtx_recurse--;
 			return;
 		}
@@ -655,7 +655,7 @@ mtx_destroy(struct mtx *m)
 	if (!mtx_owned(m)) {
 		MPASS(m->mtx_lock == MTX_UNOWNED);
 	} else {
-		MPASS((m->mtx_lock & (MTX_RECURSE|MTX_CONTESTED)) == 0);
+		MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0);
 	}
 	mtx_validate(m, MV_DESTROY);		/* diagnostic */
 #endif
@@ -701,8 +701,9 @@ struct witness {
 	u_char		 w_Giant_squawked:1;
 	u_char		 w_other_squawked:1;
 	u_char		 w_same_squawked:1;
-	u_char		 w_sleep:1;
-	u_char		 w_spin:1;	/* this is a spin mutex */
+	u_char		 w_sleep:1;	/* MTX_DEF type mutex. */
+	u_char		 w_spin:1;	/* MTX_SPIN type mutex. */
+	u_char		 w_recurse:1;	/* MTX_RECURSE mutex option. */
 	u_int		 w_level;
 	struct witness	*w_children[WITNESS_NCHILDREN];
 };
@@ -838,11 +839,16 @@ witness_enter(struct mtx *m, int flags, const char *file, int line)
 	p = CURPROC;
 
 	if (flags & MTX_SPIN) {
-		if (!w->w_spin)
+		if (!(w->w_spin))
 			panic("mutex_enter: MTX_SPIN on MTX_DEF mutex %s @"
 			    " %s:%d", m->mtx_description, file, line);
-		if (m->mtx_recurse != 0)
+		if (mtx_recursed(m)) {
+			if (!(w->w_recurse))
+				panic("mutex_enter: recursion on non-recursive"
+				    " mutex %s @ %s:%d", m->mtx_description,
+				    file, line);
 			return;
+		}
 		mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
 		i = PCPU_GET(witness_spin_check);
 		if (i != 0 && w->w_level < i) {
@@ -864,8 +870,13 @@ witness_enter(struct mtx *m, int flags, const char *file, int line)
 		panic("mutex_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
 		    m->mtx_description, file, line);
 
-	if (m->mtx_recurse != 0)
+	if (mtx_recursed(m)) {
+		if (!(w->w_recurse))
+			panic("mutex_enter: recursion on non-recursive"
+			    " mutex %s @ %s:%d", m->mtx_description,
+			    file, line);
 		return;
+	}
 	if (witness_dead)
 		goto out;
 	if (cold)
@@ -971,11 +982,16 @@ witness_exit(struct mtx *m, int flags, const char *file, int line)
 	w = m->mtx_witness;
 
 	if (flags & MTX_SPIN) {
-		if (!w->w_spin)
+		if (!(w->w_spin))
 			panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @"
 			    " %s:%d", m->mtx_description, file, line);
-		if (m->mtx_recurse != 0)
+		if (mtx_recursed(m)) {
+			if (!(w->w_recurse))
+				panic("mutex_exit: recursion on non-recursive"
+				    " mutex %s @ %s:%d", m->mtx_description,
+				    file, line); 
 			return;
+		}
 		mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
 		PCPU_SET(witness_spin_check,
 		    PCPU_GET(witness_spin_check) & ~w->w_level);
@@ -986,8 +1002,13 @@ witness_exit(struct mtx *m, int flags, const char *file, int line)
 		panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
 		    m->mtx_description, file, line);
 
-	if (m->mtx_recurse != 0)
+	if (mtx_recursed(m)) {
+		if (!(w->w_recurse))
+			panic("mutex_exit: recursion on non-recursive"
+			    " mutex %s @ %s:%d", m->mtx_description,
+			    file, line); 
 		return;
+	}
 
 	if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold)
 		panic("switchable mtx_exit() of %s when not legal @ %s:%d",
@@ -1005,12 +1026,17 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line)
 	if (panicstr)
 		return;
 	if (flags & MTX_SPIN) {
-		if (!w->w_spin)
+		if (!(w->w_spin))
 			panic("mutex_try_enter: "
 			    "MTX_SPIN on MTX_DEF mutex %s @ %s:%d",
 			    m->mtx_description, file, line);
-		if (m->mtx_recurse != 0)
+		if (mtx_recursed(m)) {
+			if (!(w->w_recurse))
+				panic("mutex_try_enter: recursion on"
+				    " non-recursive mutex %s @ %s:%d",
+				    m->mtx_description, file, line);
 			return;
+		}
 		mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
 		PCPU_SET(witness_spin_check,
 		    PCPU_GET(witness_spin_check) | w->w_level);
@@ -1026,9 +1052,13 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line)
 		panic("mutex_try_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
 		    m->mtx_description, file, line);
 
-	if (m->mtx_recurse != 0)
+	if (mtx_recursed(m)) {
+		if (!(w->w_recurse))
+			panic("mutex_try_enter: recursion on non-recursive"
+			    " mutex %s @ %s:%d", m->mtx_description, file,
+			    line);
 		return;
-
+	}
 	w->w_file = file;
 	w->w_line = line;
 	m->mtx_line = line;
@@ -1156,7 +1186,12 @@ enroll(const char *description, int flag)
 		if (*order == NULL)
 			panic("spin lock %s not in order list", description);
 		w->w_level = i; 
-	}
+	} else
+		w->w_sleep = 1;
+
+	if (flag & MTX_RECURSE)
+		w->w_recurse = 1;
+
 	return (w);
 }
 
@@ -1270,7 +1305,7 @@ witness_levelall (void)
 	struct witness *w, *w1;
 
 	for (w = w_all; w; w = w->w_next)
-		if (!w->w_spin)
+		if (!(w->w_spin))
 			w->w_level = 0;
 	for (w = w_all; w; w = w->w_next) {
 		if (w->w_spin)

sys/pc98/i386/machdep.c

@@ -436,7 +436,7 @@ cpu_startup(dummy)
 		TAILQ_INIT(&callwheel[i]);
 	}
 
-	mtx_init(&callout_lock, "callout", MTX_SPIN);
+	mtx_init(&callout_lock, "callout", MTX_SPIN | MTX_RECURSE);
 
 #if defined(USERCONFIG)
 	userconfig();
@@ -455,7 +455,7 @@ cpu_startup(dummy)
 	SLIST_INIT(&cpuhead);
 	SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu);
 
-	mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD);
+	mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE);
 
 #ifdef SMP
 	/*
@@ -2249,7 +2249,7 @@ init386(first)
 	/*
 	 * We need this mutex before the console probe.
 	 */
-	mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD);
+	mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD | MTX_RECURSE);
 
 	/*
 	 * Initialize the console before we print anything out.
@@ -2264,7 +2264,7 @@ init386(first)
 	/*
 	 * Giant is used early for at least debugger traps and unexpected traps.
 	 */
-	mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD);
+	mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE);
 
 #ifdef DDB
 	kdb_init();

sys/pc98/pc98/machdep.c

@@ -436,7 +436,7 @@ cpu_startup(dummy)
 		TAILQ_INIT(&callwheel[i]);
 	}
 
-	mtx_init(&callout_lock, "callout", MTX_SPIN);
+	mtx_init(&callout_lock, "callout", MTX_SPIN | MTX_RECURSE);
 
 #if defined(USERCONFIG)
 	userconfig();
@@ -455,7 +455,7 @@ cpu_startup(dummy)
 	SLIST_INIT(&cpuhead);
 	SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu);
 
-	mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD);
+	mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE);
 
 #ifdef SMP
 	/*
@@ -2249,7 +2249,7 @@ init386(first)
 	/*
 	 * We need this mutex before the console probe.
 	 */
-	mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD);
+	mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD | MTX_RECURSE);
 
 	/*
 	 * Initialize the console before we print anything out.
@@ -2264,7 +2264,7 @@ init386(first)
 	/*
 	 * Giant is used early for at least debugger traps and unexpected traps.
 	 */
-	mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD);
+	mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE);
 
 #ifdef DDB
 	kdb_init();

sys/pci/if_dc.c

@@ -1762,7 +1762,7 @@ static int dc_attach(dev)
 	unit = device_get_unit(dev);
 	bzero(sc, sizeof(struct dc_softc));
 
-	mtx_init(&sc->dc_mtx, device_get_nameunit(dev), MTX_DEF);
+	mtx_init(&sc->dc_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 	DC_LOCK(sc);
 
 	/*

sys/pci/if_fxp.c

@@ -316,7 +316,7 @@ fxp_attach(device_t dev)
 	u_long val;
 	int rid;
 
-	mtx_init(&sc->sc_mtx, device_get_nameunit(dev), MTX_DEF);
+	mtx_init(&sc->sc_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 	callout_handle_init(&sc->stat_ch);
 
 	FXP_LOCK(sc);

sys/pci/if_pcn.c

@@ -497,7 +497,7 @@ static int pcn_attach(dev)
 	unit = device_get_unit(dev);
 
 	/* Initialize our mutex. */
-	mtx_init(&sc->pcn_mtx, device_get_nameunit(dev), MTX_DEF);
+	mtx_init(&sc->pcn_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 	PCN_LOCK(sc);
 
 	/*

sys/pci/if_rl.c

@@ -802,7 +802,7 @@ static int rl_attach(dev)
 	unit = device_get_unit(dev);
 	bzero(sc, sizeof(struct rl_softc));
 
-	mtx_init(&sc->rl_mtx, device_get_nameunit(dev), MTX_DEF);
+	mtx_init(&sc->rl_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 	RL_LOCK(sc);
 
 	/*

sys/pci/if_sf.c

@@ -679,7 +679,7 @@ static int sf_attach(dev)
 	unit = device_get_unit(dev);
 	bzero(sc, sizeof(struct sf_softc));
 
-	mtx_init(&sc->sf_mtx, device_get_nameunit(dev), MTX_DEF);
+	mtx_init(&sc->sf_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 	SF_LOCK(sc);
 	/*
 	 * Handle power management nonsense.

sys/pci/if_sis.c

@@ -638,7 +638,7 @@ static int sis_attach(dev)
 	unit = device_get_unit(dev);
 	bzero(sc, sizeof(struct sis_softc));
 
-	mtx_init(&sc->sis_mtx, device_get_nameunit(dev), MTX_DEF);
+	mtx_init(&sc->sis_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 	SIS_LOCK(sc);
 
 	if (pci_get_device(dev) == SIS_DEVICEID_900)

sys/pci/if_sk.c

@@ -1194,7 +1194,7 @@ static int sk_attach(dev)
 	unit = device_get_unit(dev);
 	bzero(sc, sizeof(struct sk_softc));
 
-	mtx_init(&sc->sk_mtx, device_get_nameunit(dev), MTX_DEF);
+	mtx_init(&sc->sk_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 	SK_LOCK(sc);
 
 	/*

sys/pci/if_ste.c

@@ -917,7 +917,7 @@ static int ste_attach(dev)
 	unit = device_get_unit(dev);
 	bzero(sc, sizeof(struct ste_softc));
 
-	mtx_init(&sc->ste_mtx, device_get_nameunit(dev), MTX_DEF);
+	mtx_init(&sc->ste_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 	STE_LOCK(sc);
 
 	/*

sys/pci/if_ti.c

@@ -1476,7 +1476,7 @@ static int ti_attach(dev)
 	unit = device_get_unit(dev);
 	bzero(sc, sizeof(struct ti_softc));
 
-	mtx_init(&sc->ti_mtx, device_get_nameunit(dev), MTX_DEF);
+	mtx_init(&sc->ti_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 	TI_LOCK(sc);
 
 	/*

sys/pci/if_tl.c

@@ -1132,7 +1132,7 @@ static int tl_attach(dev)
 		goto fail;
 	}
 
-	mtx_init(&sc->tl_mtx, device_get_nameunit(dev), MTX_DEF);
+	mtx_init(&sc->tl_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 	TL_LOCK(sc);
 
 	/*

sys/pci/if_vr.c

@@ -643,7 +643,7 @@ static int vr_attach(dev)
 	unit = device_get_unit(dev);
 	bzero(sc, sizeof(struct vr_softc *));
 
-	mtx_init(&sc->vr_mtx, device_get_nameunit(dev), MTX_DEF);
+	mtx_init(&sc->vr_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 	VR_LOCK(sc);
 
 	/*

sys/pci/if_wb.c

@@ -817,7 +817,7 @@ static int wb_attach(dev)
 	sc = device_get_softc(dev);
 	unit = device_get_unit(dev);
 
-	mtx_init(&sc->wb_mtx, device_get_nameunit(dev), MTX_DEF);
+	mtx_init(&sc->wb_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 	WB_LOCK(sc);
 
 	/*

sys/pci/if_wx.c

@@ -597,7 +597,7 @@ wx_attach(device_t dev)
 	}
 
 #ifdef	SMPNG
-	mtx_init(&sc->wx_mtx, device_get_nameunit(dev), MTX_DEF);
+	mtx_init(&sc->wx_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 #endif
 
 	WX_LOCK(sc);

sys/pci/if_xl.c

@@ -1239,7 +1239,7 @@ static int xl_attach(dev)
 	sc = device_get_softc(dev);
 	unit = device_get_unit(dev);
 
-	mtx_init(&sc->xl_mtx, device_get_nameunit(dev), MTX_DEF);
+	mtx_init(&sc->xl_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE);
 	XL_LOCK(sc);
 
 	sc->xl_flags = 0;

sys/sys/mutex.h

@@ -67,7 +67,8 @@
 #define MTX_SPIN	0x1		/* Spin only lock */
 
 /* Options */
-#define	MTX_RLIKELY	0x4		/* (opt) Recursion likely */
+#define MTX_RECURSE	0x2		/* Recursive lock (for mtx_init) */
+#define	MTX_RLIKELY	0x4		/* Recursion likely */
 #define	MTX_NORECURSE	0x8		/* No recursion possible */
 #define	MTX_NOSPIN	0x10		/* Don't spin before sleeping */
 #define	MTX_NOSWITCH	0x20		/* Do not switch on release */
@@ -80,9 +81,9 @@
 #define	MTX_HARDOPTS	(MTX_SPIN | MTX_FIRST | MTX_TOPHALF | MTX_NOSWITCH)
 
 /* Flags/value used in mtx_lock */
-#define	MTX_RECURSE	0x01		/* (non-spin) lock held recursively */
+#define	MTX_RECURSED	0x01		/* (non-spin) lock held recursively */
 #define	MTX_CONTESTED	0x02		/* (non-spin) lock contested */
-#define	MTX_FLAGMASK	~(MTX_RECURSE | MTX_CONTESTED)
+#define	MTX_FLAGMASK	~(MTX_RECURSED | MTX_CONTESTED)
 #define MTX_UNOWNED	0x8		/* Cookie for free mutex */
 
 #endif	/* _KERNEL */
@@ -360,7 +361,7 @@ void	witness_restore(struct mtx *, const char *, int);
 		if (((mp)->mtx_lock & MTX_FLAGMASK) != ((uintptr_t)(tid)))\
 			mtx_enter_hard(mp, (type) & MTX_HARDOPTS, 0);	\
 		else {							\
-			atomic_set_ptr(&(mp)->mtx_lock, MTX_RECURSE);	\
+			atomic_set_ptr(&(mp)->mtx_lock, MTX_RECURSED);	\
 			(mp)->mtx_recurse++;				\
 		}							\
 	}								\
@@ -408,10 +409,10 @@ void	witness_restore(struct mtx *, const char *, int);
  */
 #define	_exitlock(mp, tid, type) do {					\
 	if (!_release_lock(mp, tid)) {					\
-		if ((mp)->mtx_lock & MTX_RECURSE) {			\
+		if ((mp)->mtx_lock & MTX_RECURSED) {			\
 			if (--((mp)->mtx_recurse) == 0)			\
 				atomic_clear_ptr(&(mp)->mtx_lock,	\
-				    MTX_RECURSE);			\
+				    MTX_RECURSED);			\
 		} else {						\
 			mtx_exit_hard((mp), (type) & MTX_HARDOPTS);	\
 		}							\
@@ -422,7 +423,7 @@ void	witness_restore(struct mtx *, const char *, int);
 #ifndef _exitlock_spin
 /* Release a spin lock (with possible recursion). */
 #define	_exitlock_spin(mp) do {						\
-	if ((mp)->mtx_recurse == 0) {					\
+	if (!mtx_recursed((mp))) {					\
 		int _mtx_intr = (mp)->mtx_saveintr;			\
 									\
 		_release_lock_quick(mp);				\