Turn PREEMPTION into a kernel option. Make sure that it's defined if

FULL_PREEMPTION is defined.  Add a runtime warning to ULE if PREEMPTION is
enabled (code inspired by the PREEMPTION warning in kern_switch.c).  This
is a possible MT5 candidate.

sys/alpha/include/param.h

@@ -113,11 +113,6 @@
 #define	SSIZE		1		/* initial stack size/NBPG */
 #define	SINCR		1		/* increment of stack/NBPG */
 
-/* PREEMPTION exposes scheduler bugs that need to be fixed. */
-#if 0
-#define	PREEMPTION
-#endif
-
 #ifndef	KSTACK_PAGES
 #define	KSTACK_PAGES	2		/* pages of kstack (with pcb) */
 #endif

sys/amd64/include/param.h

@@ -119,11 +119,6 @@
 #define	NBPML4		(1ul<<PML4SHIFT)/* bytes/page map lev4 table */
 #define	PML4MASK	(NBPML4-1)
 
-/* PREEMPTION exposes scheduler bugs that need to be fixed. */
-#if 0
-#define	PREEMPTION
-#endif
-
 #define IOPAGES	2		/* pages of i/o permission bitmap */
 
 #ifndef	KSTACK_PAGES

sys/conf/NOTES

@@ -196,12 +196,16 @@ options 	MUTEX_WAKE_ALL
 
 # SMP Debugging Options:
 #
+# PREEMPTION allows the threads that are in the kernel to be preempted
+#	  by higher priority threads.  It helps with interactivity and
+#	  allows interrupt threads to run sooner rather than waiting.
+#	  WARNING! Only tested on alpha, amd64, and i386.
 # FULL_PREEMPTION instructs the kernel to preempt non-realtime kernel
 #	  threads.  It sole use is to expose race conditions and other
 #	  bugs during development.  Enabling this option will reduce
 #	  performance and increase the frequency of kernel panics by
 #	  design.  If you aren't sure that you need it then you don't.
-#         DON'T TURN THIS ON.
+#	  Relies on the PREEMPTION option.  DON'T TURN THIS ON.
 # MUTEX_DEBUG enables various extra assertions in the mutex code.
 # SLEEPQUEUE_PROFILING enables rudimentary profiling of the hash table
 #	  used to hold active sleep queues.
@@ -213,6 +217,7 @@ options 	MUTEX_WAKE_ALL
 #	  a lock hierarchy violation occurs or if locks are held when going to
 #	  sleep.
 # WITNESS_SKIPSPIN disables the witness checks on spin mutexes.
+options		PREEMPTION
 options 	FULL_PREEMPTION
 options 	MUTEX_DEBUG
 options 	WITNESS

sys/conf/options

@@ -66,7 +66,8 @@ COMPILING_LINT	opt_global.h
 CONSPEED	opt_comconsole.h
 CY_PCI_FASTINTR
 DIRECTIO	opt_directio.h
-FULL_PREEMPTION
+FULL_PREEMPTION	opt_sched.h
+PREEMPTION	opt_sched.h
 GEOM_AES	opt_geom.h
 GEOM_APPLE	opt_geom.h
 GEOM_BDE	opt_geom.h

sys/i386/include/param.h

@@ -97,11 +97,6 @@
 #define NBPDR		(1<<PDRSHIFT)	/* bytes/page dir */
 #define PDRMASK		(NBPDR-1)
 
-/* PREEMPTION exposes scheduler bugs that need to be fixed. */
-#if 0
-#define	PREEMPTION
-#endif
-
 #define IOPAGES	2		/* pages of i/o permission bitmap */
 
 #ifndef KSTACK_PAGES

sys/kern/kern_mutex.c

@@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
 #include "opt_ddb.h"
 #include "opt_mprof.h"
 #include "opt_mutex_wake_all.h"
+#include "opt_sched.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>

sys/kern/kern_shutdown.c

@@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$");
 #include "opt_mac.h"
 #include "opt_panic.h"
 #include "opt_show_busybufs.h"
+#include "opt_sched.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>

sys/kern/kern_switch.c

@@ -88,7 +88,6 @@ reassigned to keep this true.
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include "opt_full_preemption.h"
 #include "opt_sched.h"
 
 #include <sys/param.h>
@@ -109,6 +108,11 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysctl.h>
 #endif
 
+#ifdef FULL_PREEMPTION
+#ifndef PREEMPTION
+#error "The FULL_PREEMPTION option requires the PREEMPTION option"
+#endif
+#endif
 
 CTASSERT((RQB_BPW * RQB_LEN) == RQ_NQS);
 

sys/kern/sched_ule.c

@@ -27,6 +27,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <opt_sched.h>
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kdb.h>
@@ -75,6 +77,18 @@ SYSCTL_INT(_kern_sched, OID_AUTO, slice_max, CTLFLAG_RW, &slice_max, 0, "");
 int realstathz;
 int tickincr = 1;
 
+#ifdef PREEMPTION
+static void
+printf_caddr_t(void *data)
+{
+	printf("%s", (char *)data);
+}
+static char preempt_warning[] =
+    "WARNING: Kernel PREEMPTION is unstable under SCHED_ULE.\n"; 
+SYSINIT(preempt_warning, SI_SUB_COPYRIGHT, SI_ORDER_ANY, printf_caddr_t,
+    preempt_warning)
+#endif
+
 /*
  * These datastructures are allocated within their parent datastructure but
  * are scheduler specific.

sys/vm/vm_zeroidle.c

@@ -35,6 +35,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <opt_sched.h>
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>