From 09828ba947eb87c5ae5e8527e274bcb26eef1c89 Mon Sep 17 00:00:00 2001
From: Konstantin Belousov <kib@FreeBSD.org>
Date: Wed, 4 Jul 2007 06:56:58 +0000
Subject: [PATCH] Since cdev mutex is after system map mutex in global lock
 order, free() shall not be called while holding cdev mutex. devfs_inos unrhdr
 has cdev as mutex, thus creating this LOR situation.

Postpone calling free() in kern/subr_unit.c:alloc_unr() and nested functions
until the unrhdr mutex is dropped. Save the freed items on the ppfree list
instead, and provide the clean_unrhdrl() and clean_unrhdr() functions to
clean the list.
Call clean_unrhdrl() after devfs_create() calls immediately before
dropping cdev mutex. devfs_create() is the only user of the alloc_unrl()
in the tree.

Reviewed by:	phk
Tested by:	Peter Holm
LOR:	80
Approved by:	re (kensmith)
---
 sys/kern/kern_conf.c |  2 ++
 sys/kern/subr_unit.c | 35 ++++++++++++++++++++++++++++++++++-
 sys/sys/systm.h      |  2 ++
 3 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/sys/kern/kern_conf.c b/sys/kern/kern_conf.c
index 20d3bc379218..751f524636a7 100644
--- a/sys/kern/kern_conf.c
+++ b/sys/kern/kern_conf.c
@@ -617,6 +617,7 @@ make_dev_credv(int flags, struct cdevsw *devsw, int minornr,
 	dev->si_mode = mode;
 
 	devfs_create(dev);
+	clean_unrhdrl(devfs_inos);
 	dev_unlock();
 	return (dev);
 }
@@ -703,6 +704,7 @@ make_dev_alias(struct cdev *pdev, const char *fmt, ...)
 	va_end(ap);
 
 	devfs_create(dev);
+	clean_unrhdrl(devfs_inos);
 	dev_unlock();
 	dev_depends(pdev, dev);
 	return (dev);
diff --git a/sys/kern/subr_unit.c b/sys/kern/subr_unit.c
index 764f26f6784e..4cd77b052ca5 100644
--- a/sys/kern/subr_unit.c
+++ b/sys/kern/subr_unit.c
@@ -197,6 +197,8 @@ struct unrhdr {
 	u_int			first;	/* items in allocated from start */
 	u_int			last;	/* items free at end */
 	struct mtx		*mtx;
+	TAILQ_HEAD(unrfr,unr)	ppfree;	/* Items to be freed after mtx
+					   lock dropped */
 };
 
 
@@ -281,9 +283,35 @@ new_unr(struct unrhdr *uh, void **p1, void **p2)
 static __inline void
 delete_unr(struct unrhdr *uh, void *ptr)
 {
+	struct unr *up;
 
 	uh->alloc--;
-	Free(ptr);
+	up = ptr;
+	TAILQ_INSERT_TAIL(&uh->ppfree, up, list);
+}
+
+void
+clean_unrhdrl(struct unrhdr *uh)
+{
+	struct unr *up;
+
+	mtx_assert(uh->mtx, MA_OWNED);
+	while ((up = TAILQ_FIRST(&uh->ppfree)) != NULL) {
+		TAILQ_REMOVE(&uh->ppfree, up, list);
+		mtx_unlock(uh->mtx);
+		Free(up);
+		mtx_lock(uh->mtx);
+	}
+
+}
+
+void
+clean_unrhdr(struct unrhdr *uh)
+{
+
+	mtx_lock(uh->mtx);
+	clean_unrhdrl(uh);
+	mtx_unlock(uh->mtx);
 }
 
 /*
@@ -305,6 +333,7 @@ new_unrhdr(int low, int high, struct mtx *mutex)
 	else
 		uh->mtx = &unitmtx;
 	TAILQ_INIT(&uh->head);
+	TAILQ_INIT(&uh->ppfree);
 	uh->low = low;
 	uh->high = high;
 	uh->first = 0;
@@ -320,6 +349,8 @@ delete_unrhdr(struct unrhdr *uh)
 	check_unrhdr(uh, __LINE__);
 	KASSERT(uh->busy == 0, ("unrhdr has %u allocations", uh->busy));
 	KASSERT(uh->alloc == 0, ("UNR memory leak in delete_unrhdr"));
+	KASSERT(TAILQ_FIRST(&uh->ppfree) == NULL,
+	    ("unrhdr has postponed item for free"));
 	Free(uh);
 }
 
@@ -591,6 +622,7 @@ alloc_unr(struct unrhdr *uh)
 
 	mtx_lock(uh->mtx);
 	i = alloc_unrl(uh);
+	clean_unrhdrl(uh);
 	mtx_unlock(uh->mtx);
 	return (i);
 }
@@ -719,6 +751,7 @@ free_unr(struct unrhdr *uh, u_int item)
 	p2 = Malloc(sizeof(struct unr));
 	mtx_lock(uh->mtx);
 	free_unrl(uh, item, &p1, &p2);
+	clean_unrhdrl(uh);
 	mtx_unlock(uh->mtx);
 	if (p1 != NULL)
 		Free(p1);
diff --git a/sys/sys/systm.h b/sys/sys/systm.h
index 13b7ef7f5495..e254d5a6219f 100644
--- a/sys/sys/systm.h
+++ b/sys/sys/systm.h
@@ -342,6 +342,8 @@ int root_mounted(void);
 struct unrhdr;
 struct unrhdr *new_unrhdr(int low, int high, struct mtx *mutex);
 void delete_unrhdr(struct unrhdr *uh);
+void clean_unrhdr(struct unrhdr *uh);
+void clean_unrhdrl(struct unrhdr *uh);
 int alloc_unr(struct unrhdr *uh);
 int alloc_unrl(struct unrhdr *uh);
 void free_unr(struct unrhdr *uh, u_int item);