Enable use of ofwcons for early debug

This change enables the use of OpenFirmware Console (ofwcons), even when VGA is
available, allowing early kernel messages to be seen, that is important in case
of crashes before VGA console initialization.

This is specially useful in virtualized environments, where the user/developer
doesn't have full control of the virtualization engine (e.g. OpenStack).

The old behavior is preserved by default and, in order to use ofwcons, a few
tunables that have been introduced need to be set:
- hw.ofwfb.disable=1     - disable OFW FrameBuffer device
- machdep.ofw.mtx_spin=1 - change PPC OFW mutex to SPIN type, to match kernel
                           console's mutex type
- debug.quiesce_ofw=0    - don't call OFW quiesce, needed to keep ofwcons I/O
                           working

More details can be found at differential revision D20640.

Reviewed by:	jhibbits
Differential Revision:	https://reviews.freebsd.org/D20640

sys/dev/vt/hw/ofwfb/ofwfb.c

@@ -90,10 +90,16 @@ VT_DRIVER_DECLARE(vt_ofwfb, vt_ofwfb_driver);
 static int
 ofwfb_probe(struct vt_device *vd)
 {
+	int disabled;
 	phandle_t chosen, node;
 	ihandle_t stdout;
 	char buf[64];
 
+	disabled = 0;
+	TUNABLE_INT_FETCH("hw.ofwfb.disable", &disabled);
+	if (disabled)
+		return (CN_DEAD);
+
 	chosen = OF_finddevice("/chosen");
 	if (chosen == -1)
 		return (CN_DEAD);

sys/powerpc/ofw/ofw_real.c

@@ -168,7 +168,45 @@ static vm_offset_t	of_bounce_phys;
 static caddr_t		of_bounce_virt;
 static off_t		of_bounce_offset;
 static size_t		of_bounce_size;
+
+/*
+ * To be able to use OFW console on PPC, that requires real mode OFW,
+ * the mutex that guards the mapping/unmapping of virtual to physical
+ * buffers (of_real_mtx) must be of SPIN type. This is needed because
+ * kernel console first locks a SPIN mutex before calling OFW real.
+ * By default, of_real_mtx is a sleepable mutex. To make it of SPIN
+ * type, use the following tunnable:
+ * machdep.ofw.mtx_spin=1
+ *
+ * Besides that, a few more tunables are needed to select and use the
+ * OFW console with real mode OFW.
+ *
+ * In order to disable the use of OFW FrameBuffer and fallback to the
+ * OFW console, use:
+ * hw.ofwfb.disable=1
+ *
+ * To disable the use of FDT (that doesn't support OFW read/write methods)
+ * and use real OFW instead, unset the following loader variable:
+ * unset usefdt
+ *
+ * OFW is put in quiesce state in early kernel boot, which usually disables
+ * OFW read/write capabilities (in QEMU write continue to work, but
+ * read doesn't). To avoid OFW quiesce, use:
+ * debug.quiesce_ofw=0
+ *
+ * Note that disabling OFW quiesce can cause conflicts between kernel and
+ * OFW trying to control the same hardware. Thus, it must be used with care.
+ * Some conflicts can be avoided by disabling kernel drivers with hints.
+ * For instance, to disable a xhci controller and an USB keyboard connected
+ * to it, that may be already being used for input by OFW, use:
+ * hint.xhci.0.disabled=1
+ */
+
 static struct mtx	of_bounce_mtx;
+static struct mtx	of_spin_mtx;
+static struct mtx	*of_real_mtx;
+static void		(*of_mtx_lock)(void);
+static void		(*of_mtx_unlock)(void);
 
 extern int		ofw_real_mode;
 
@@ -181,17 +219,41 @@ static void ofw_real_bounce_alloc(void *);
 SYSINIT(ofw_real_bounce_alloc, SI_SUB_KMEM, SI_ORDER_ANY, 
     ofw_real_bounce_alloc, NULL);
 
+static void
+ofw_real_mtx_lock_spin(void)
+{
+	mtx_lock_spin(of_real_mtx);
+}
+
+static void
+ofw_real_mtx_lock(void)
+{
+	mtx_lock(of_real_mtx);
+}
+
+static void
+ofw_real_mtx_unlock_spin(void)
+{
+	mtx_unlock_spin(of_real_mtx);
+}
+
+static void
+ofw_real_mtx_unlock(void)
+{
+	mtx_unlock(of_real_mtx);
+}
+
 static void
 ofw_real_start(void)
 {
-	mtx_lock(&of_bounce_mtx);
+	(*of_mtx_lock)();
 	of_bounce_offset = 0;
 }
-	
+
 static void
 ofw_real_stop(void)
 {
-	mtx_unlock(&of_bounce_mtx);
+	(*of_mtx_unlock)();
 }
 
 static void
@@ -228,7 +290,7 @@ ofw_real_bounce_alloc(void *junk)
 	 * we have a 32-bit virtual address to give OF.
 	 */
 
-	if (!ofw_real_mode && (!hw_direct_map || DMAP_BASE_ADDRESS != 0)) 
+	if (!ofw_real_mode && (!hw_direct_map || DMAP_BASE_ADDRESS != 0))
 		pmap_kenter(of_bounce_phys, of_bounce_phys);
 
 	mtx_unlock(&of_bounce_mtx);
@@ -240,7 +302,7 @@ ofw_real_map(const void *buf, size_t len)
 	static char emergency_buffer[255];
 	cell_t phys;
 
-	mtx_assert(&of_bounce_mtx, MA_OWNED);
+	mtx_assert(of_real_mtx, MA_OWNED);
 
 	if (of_bounce_virt == NULL) {
 		/*
@@ -290,7 +352,7 @@ ofw_real_map(const void *buf, size_t len)
 static void
 ofw_real_unmap(cell_t physaddr, void *buf, size_t len)
 {
-	mtx_assert(&of_bounce_mtx, MA_OWNED);
+	mtx_assert(of_real_mtx, MA_OWNED);
 
 	if (of_bounce_virt == NULL)
 		return;
@@ -306,9 +368,24 @@ ofw_real_unmap(cell_t physaddr, void *buf, size_t len)
 static int
 ofw_real_init(ofw_t ofw, void *openfirm)
 {
-	openfirmware = (int (*)(void *))openfirm;
+	int mtx_spin;
 
+	openfirmware = (int (*)(void *))openfirm;
 	mtx_init(&of_bounce_mtx, "OF Bounce Page", NULL, MTX_DEF);
+
+	mtx_spin = 0;
+	TUNABLE_INT_FETCH("machdep.ofw.mtx_spin", &mtx_spin);
+	if (mtx_spin) {
+		mtx_init(&of_spin_mtx, "OF Real", NULL, MTX_SPIN);
+		of_real_mtx = &of_spin_mtx;
+		of_mtx_lock = ofw_real_mtx_lock_spin;
+		of_mtx_unlock = ofw_real_mtx_unlock_spin;
+	} else {
+		of_real_mtx = &of_bounce_mtx;
+		of_mtx_lock = ofw_real_mtx_lock;
+		of_mtx_unlock = ofw_real_mtx_unlock;
+	}
+
 	of_bounce_virt = NULL;
 	return (0);
 }

sys/powerpc/pseries/platform_chrp.c

@@ -133,6 +133,7 @@ chrp_probe(platform_t plat)
 static int
 chrp_attach(platform_t plat)
 {
+	int quiesce;
 #ifdef __powerpc64__
 	int i;
 
@@ -175,7 +176,10 @@ chrp_attach(platform_t plat)
 	chrp_cpuref_init();
 
 	/* Some systems (e.g. QEMU) need Open Firmware to stand down */
-	ofw_quiesce();
+	quiesce = 1;
+	TUNABLE_INT_FETCH("debug.quiesce_ofw", &quiesce);
+	if (quiesce)
+		ofw_quiesce();
 
 	return (0);
 }