Add a new 'show sleepchain' ddb command similar to 'show lockchain' except

that it operates on lockmgr and sx locks.  This can be useful for tracking
down vnode deadlocks in VFS for example.  Note that this command is a bit
more fragile than 'show lockchain' as we have to poke around at the
wait channel of a thread to see if it points to either a struct lock or
a condition variable inside of a struct sx.  If td_wchan points to
something unmapped, then this command will terminate early due to a fault,
but no harm will be done.

sys/kern/kern_lock.c

@@ -589,6 +589,34 @@ lockmgr_printinfo(lkp)
 }
 
 #ifdef DDB
+/*
+ * Check to see if a thread that is blocked on a sleep queue is actually
+ * blocked on a 'struct lock'.  If so, output some details and return true.
+ * If the lock has an exclusive owner, return that in *ownerp.
+ */
+int
+lockmgr_chain(struct thread *td, struct thread **ownerp)
+{
+	struct lock *lkp;
+
+	lkp = td->td_wchan;
+
+	/* Simple test to see if wchan points to a lockmgr lock. */
+	if (lkp->lk_wmesg != td->td_wmesg)
+		return (0);
+
+	/* Ok, we think we have a lockmgr lock, so output some details. */
+	db_printf("blocked on lk \"%s\" ", lkp->lk_wmesg);
+	if (lkp->lk_sharecount) {
+		db_printf("SHARED (count %d)\n", lkp->lk_sharecount);
+		*ownerp = NULL;
+	} else {
+		db_printf("EXCL (count %d)\n", lkp->lk_exclusivecount);
+		*ownerp = lkp->lk_lockholder;
+	}
+	return (1);
+}
+
 DB_SHOW_COMMAND(lockmgr, db_show_lockmgr)
 {
 	struct thread *td;

sys/kern/kern_sx.c

@@ -48,9 +48,9 @@ __FBSDID("$FreeBSD$");
 #include <sys/proc.h>
 #include <sys/sx.h>
 
+#ifdef DDB
 #include <ddb/ddb.h>
 
-#ifdef DDB
 static void	db_show_sx(struct lock_object *lock);
 #endif
 
@@ -395,4 +395,57 @@ db_show_sx(struct lock_object *lock)
 	db_printf(" waiters: %d shared, %d exclusive\n", sx->sx_shrd_wcnt,
 	    sx->sx_excl_wcnt);
 }
+
+/*
+ * Check to see if a thread that is blocked on a sleep queue is actually
+ * blocked on an sx lock.  If so, output some details and return true.
+ * If the lock has an exclusive owner, return that in *ownerp.
+ */
+int
+sx_chain(struct thread *td, struct thread **ownerp)
+{
+	struct sx *sx;
+	struct cv *cv;
+
+	/*
+	 * First, see if it looks like td is blocked on a condition
+	 * variable.
+	 */
+	cv = td->td_wchan;
+	if (cv->cv_description != td->td_wmesg)
+		return (0);
+
+	/*
+	 * Ok, see if it looks like td is blocked on the exclusive
+	 * condition variable.
+	 */
+	sx = (struct sx *)((char *)cv - offsetof(struct sx, sx_excl_cv));
+	if (LOCK_CLASS(&sx->sx_object) == &lock_class_sx &&
+	    sx->sx_excl_wcnt > 0)
+		goto ok;
+
+	/*
+	 * Second, see if it looks like td is blocked on the shared
+	 * condition variable.
+	 */
+	sx = (struct sx *)((char *)cv - offsetof(struct sx, sx_shrd_cv));
+	if (LOCK_CLASS(&sx->sx_object) == &lock_class_sx &&
+	    sx->sx_shrd_wcnt > 0)
+		goto ok;
+
+	/* Doesn't seem to be an sx lock. */
+	return (0);
+
+ok:
+	/* We think we have an sx lock, so output some details. */
+	db_printf("blocked on sx \"%s\" ", td->td_wmesg);
+	if (sx->sx_cnt >= 0) {
+		db_printf("SLOCK (count %d)\n", sx->sx_cnt);
+		*ownerp = NULL;
+	} else {
+		db_printf("XLOCK\n");
+		*ownerp = sx->sx_xholder;
+	}
+	return (1);
+}
 #endif

sys/kern/subr_turnstile.c

@@ -78,6 +78,8 @@ __FBSDID("$FreeBSD$");
 #ifdef DDB
 #include <sys/kdb.h>
 #include <ddb/ddb.h>
+#include <sys/lockmgr.h>
+#include <sys/sx.h>
 #endif
 
 /*
@@ -1120,6 +1122,71 @@ DB_SHOW_COMMAND(allchains, db_show_allchains)
 	}
 }
 
+/*
+ * Show all the threads a particular thread is waiting on based on
+ * sleepable locks.
+ */
+static void
+print_sleepchain(struct thread *td, const char *prefix)
+{
+	struct thread *owner;
+
+	/*
+	 * Follow the chain.  We keep walking as long as the thread is
+	 * blocked on a sleep lock that has an owner.
+	 */
+	while (!db_pager_quit) {
+		db_printf("%sthread %d (pid %d, %s) ", prefix, td->td_tid,
+		    td->td_proc->p_pid, td->td_name[0] != '\0' ? td->td_name :
+		    td->td_proc->p_comm);
+		switch (td->td_state) {
+		case TDS_INACTIVE:
+			db_printf("is inactive\n");
+			return;
+		case TDS_CAN_RUN:
+			db_printf("can run\n");
+			return;
+		case TDS_RUNQ:
+			db_printf("is on a run queue\n");
+			return;
+		case TDS_RUNNING:
+			db_printf("running on CPU %d\n", td->td_oncpu);
+			return;
+		case TDS_INHIBITED:
+			if (TD_ON_SLEEPQ(td)) {
+				if (lockmgr_chain(td, &owner) ||
+				    sx_chain(td, &owner)) {
+					if (owner == NULL)
+						return;
+					td = owner;
+					break;
+				}
+				db_printf("sleeping on %p \"%s\"\n",
+				    td->td_wchan, td->td_wmesg);
+				return;
+			}
+			db_printf("inhibited\n");
+			return;
+		default:
+			db_printf("??? (%#x)\n", td->td_state);
+			return;
+		}
+	}
+}
+
+DB_SHOW_COMMAND(sleepchain, db_show_sleepchain)
+{
+	struct thread *td;
+
+	/* Figure out which thread to start with. */
+	if (have_addr)
+		td = db_lookup_thread(addr, TRUE);
+	else
+		td = kdb_thread;
+
+	print_sleepchain(td, "");
+}
+
 static void	print_waiters(struct turnstile *ts, int indent);
 	
 static void

sys/sys/lockmgr.h

@@ -203,5 +203,8 @@ void	transferlockers(struct lock *, struct lock *);
 void	lockmgr_printinfo(struct lock *);
 int	lockstatus(struct lock *, struct thread *);
 int	lockcount(struct lock *);
+#ifdef DDB
+int	lockmgr_chain(struct thread *td, struct thread **ownerp);
+#endif
 
 #endif /* !_SYS_LOCKMGR_H_ */

sys/sys/sx.h

@@ -60,6 +60,9 @@ void	_sx_downgrade(struct sx *sx, const char *file, int line);
 #ifdef INVARIANT_SUPPORT
 void	_sx_assert(struct sx *sx, int what, const char *file, int line);
 #endif
+#ifdef DDB
+int	sx_chain(struct thread *td, struct thread **ownerp);
+#endif
 
 struct sx_args {
 	struct sx 	*sa_sx;