Add a named reference-count KPI to hold off mounting of the root filesystem.

While we wait for holds to be released, print a list of who holds us
back once per second.

Use the new KPI from GEOM instead of vfs_mount.c calling g_waitidle().

Use the new KPI also from ata.

With ATAmkIII's newbusification, ata could narrowly miss the window
and ad0 would not exist when we tried to mount root.

sys/dev/ata/ata-all.c

@@ -81,6 +81,7 @@ int ata_wc = 1;
 
 /* local vars */
 static struct intr_config_hook *ata_delayed_attach = NULL;
+static struct root_hold_token *ata_root_hold_token;
 static int ata_dma = 1;
 static int atapi_dma = 1;
 
@@ -559,6 +560,7 @@ ata_boot_attach(void)
 	    ata_identify(ch->dev);
 	}
     }
+    root_mount_rel(ata_root_hold_token);
 }
 
 /*
@@ -814,6 +816,7 @@ ata_module_event_handler(module_t mod, int what, void *arg)
 		return EIO;
 	    }
 	    ata_delayed_attach->ich_func = (void*)ata_boot_attach;
+	    ata_root_hold_token = root_mount_hold("ATA");
 	    if (config_intrhook_establish(ata_delayed_attach) != 0) {
 		printf("ata: config_intrhook_establish failed\n");
 		free(ata_delayed_attach, M_TEMP);

sys/geom/geom_kern.c

@@ -132,13 +132,19 @@ g_event_procbody(void)
 {
 	struct proc *p = g_event_proc;
 	struct thread *tp = FIRST_THREAD_IN_PROC(p);
+	struct root_hold_token *t;
 
 	mtx_assert(&Giant, MA_NOTOWNED);
 	mtx_lock_spin(&sched_lock);
 	sched_prio(tp, PRIBIO);
 	mtx_unlock_spin(&sched_lock);
+	t = root_mount_hold("GEOM");
 	for(;;) {
 		g_run_events();
+		if (t != 0) {
+			root_mount_rel(t);
+			t = NULL;
+		}
 		tsleep(&g_wait_event, PRIBIO, "-", hz/10);
 	}
 }

sys/kern/vfs_mount.c

@@ -1004,6 +1004,54 @@ dounmount(mp, flags, td)
  *
  */
 
+struct root_hold_token {
+	const char 			*who;
+	LIST_ENTRY(root_hold_token)	list;
+};
+
+static LIST_HEAD(, root_hold_token)	root_holds =
+    LIST_HEAD_INITIALIZER(&root_holds);
+
+struct root_hold_token *
+root_mount_hold(const char *identifier)
+{
+	struct root_hold_token *h;
+
+	h = malloc(sizeof *h, M_DEVBUF, M_ZERO | M_WAITOK);
+	h->who = identifier;
+	mtx_lock(&mountlist_mtx);
+	LIST_INSERT_HEAD(&root_holds, h, list);
+	mtx_unlock(&mountlist_mtx);
+	return (h);
+}
+
+void
+root_mount_rel(struct root_hold_token *h)
+{
+
+	mtx_lock(&mountlist_mtx);
+	LIST_REMOVE(h, list);
+	wakeup(&root_holds);
+	mtx_unlock(&mountlist_mtx);
+	free(h, M_DEVBUF);
+}
+
+static void
+root_mount_wait(void)
+{
+	struct root_hold_token *h;
+
+	mtx_lock(&mountlist_mtx);
+	while (!LIST_EMPTY(&root_holds)) {
+		printf("Root mount waiting for:");
+		LIST_FOREACH(h, &root_holds, list)
+			printf(" %s", h->who);
+		printf("\n");
+		msleep(&root_holds, &mountlist_mtx, PZERO, "roothold", hz);
+	}
+	mtx_unlock(&mountlist_mtx);
+}
+
 static void
 set_rootvnode(struct thread *td)
 {
@@ -1140,12 +1188,7 @@ vfs_mountroot(void)
 	int error, i, asked = 0;
 	struct mount *mp;
 
-	/*
-	 * Wait for GEOM to settle down
-	 */
-	DROP_GIANT();
-	g_waitidle();
-	PICKUP_GIANT();
+	root_mount_wait();
 
 	mp = devfs_first();
 

sys/sys/systm.h

@@ -315,6 +315,13 @@ const char *devtoname(struct cdev *cdev);
 /* XXX: Should be void nanodelay(u_int nsec); */
 void	DELAY(int usec);
 
+/* Root mount holdback API */
+struct root_hold_token;
+
+struct root_hold_token *root_mount_hold(const char *identifier);
+void root_mount_rel(struct root_hold_token *h);
+
+
 /*
  * Unit number allocation API. (kern/subr_unit.c)
  */