Implement preemptive scheduling of hardware interrupt threads.

- If possible, context switch to the thread directly in sched_ithd(),
  rather than triggering a delayed ast reschedule.

- Disable interrupts while restoring fpu state in the trap handler,
  in order to ensure that we are not preempted in the middle, which
  could cause migration to another cpu.

Reviewed by:	peter
Tested by:	peter (alpha)

sys/alpha/alpha/interrupt.c

@@ -553,6 +553,12 @@ alpha_dispatch_intr(void *frame, unsigned long vector)
 	 * is higher priority than their current thread, it gets run now.
 	 */
 	ithd->it_need = 1;
+	if (i->disable) {
+		CTR1(KTR_INTR,
+		    "alpha_dispatch_intr: disabling vector 0x%x", i->vector);
+		i->disable(i->vector);
+	}
+	enable_intr();
 	mtx_enter(&sched_lock, MTX_SPIN);
 	if (ithd->it_proc->p_stat == SWAIT) {
 		/* not on the run queue and not running */
@@ -562,19 +568,17 @@ alpha_dispatch_intr(void *frame, unsigned long vector)
 		alpha_mb();	/* XXX - ??? */
 		ithd->it_proc->p_stat = SRUN;
 		setrunqueue(ithd->it_proc);
-		aston();
+		if (!cold) {
+			if (curproc != PCPU_GET(idleproc))
+				setrunqueue(curproc);
+			mi_switch();
+		}
 	} else {
 		CTR3(KTR_INTR, "alpha_dispatch_intr: %d: it_need %d, state %d",
 		    ithd->it_proc->p_pid, ithd->it_need, ithd->it_proc->p_stat);
-	}
-	if (i->disable) {
-		CTR1(KTR_INTR,
-		    "alpha_dispatch_intr: disabling vector 0x%x", i->vector);
-		i->disable(i->vector);
+		need_resched();
 	}
 	mtx_exit(&sched_lock, MTX_SPIN);
-
-	need_resched();
 }
  
 void

sys/alpha/alpha/machdep.c

@@ -2120,6 +2120,10 @@ alpha_fpstate_check(struct proc *p)
 void
 alpha_fpstate_save(struct proc *p, int write)
 {
+	int s;
+
+	s = save_intr();
+	disable_intr();
 	if (p == PCPU_GET(fpcurproc)) {
 		/*
 		 * If curproc != fpcurproc, then we need to enable FEN 
@@ -2154,6 +2158,7 @@ alpha_fpstate_save(struct proc *p, int write)
 				alpha_pal_wrfen(0);
 		}
 	}
+	restore_intr(s);
 }
 
 /*
@@ -2164,6 +2169,10 @@ alpha_fpstate_save(struct proc *p, int write)
 void
 alpha_fpstate_drop(struct proc *p)
 {
+	int s;
+
+	s = save_intr();
+	disable_intr();
 	if (p == PCPU_GET(fpcurproc)) {
 		if (p == curproc) {
 			/*
@@ -2179,6 +2188,7 @@ alpha_fpstate_drop(struct proc *p)
 		}
 		PCPU_SET(fpcurproc, NULL);
 	}
+	restore_intr(s);
 }
 
 /*
@@ -2188,9 +2198,13 @@ alpha_fpstate_drop(struct proc *p)
 void
 alpha_fpstate_switch(struct proc *p)
 {
+	int s;
+
 	/*
 	 * Enable FEN so that we can access the fp registers.
 	 */
+	s = save_intr();
+	disable_intr();
 	alpha_pal_wrfen(1);
 	if (PCPU_GET(fpcurproc)) {
 		/*
@@ -2217,6 +2231,7 @@ alpha_fpstate_switch(struct proc *p)
 	}
 
 	p->p_md.md_flags |= MDP_FPUSED;
+	restore_intr(s);
 }
 
 /*

sys/amd64/amd64/fpu.c

@@ -796,6 +796,8 @@ npx_intr(dummy)
 int
 npxdna()
 {
+	int s;
+
 	if (!npx_exists)
 		return (0);
 	if (PCPU_GET(npxproc) != NULL) {
@@ -803,6 +805,8 @@ npxdna()
 		       PCPU_GET(npxproc), curproc);
 		panic("npxdna");
 	}
+	s = save_intr();
+	disable_intr();
 	stop_emulating();
 	/*
 	 * Record new context early in case frstor causes an IRQ13.
@@ -822,6 +826,7 @@ npxdna()
 	 * first FPU instruction after a context switch.
 	 */
 	frstor(&PCPU_GET(curpcb)->pcb_savefpu);
+	restore_intr(s);
 
 	return (1);
 }

sys/amd64/isa/ithread.c

@@ -121,16 +121,20 @@ sched_ithd(void *cookie)
 /*		membar_lock(); */
 		ir->it_proc->p_stat = SRUN;
 		setrunqueue(ir->it_proc);
-		aston();
+		if (!cold) {
+			if (curproc != PCPU_GET(idleproc))
+				setrunqueue(curproc);
+			mi_switch();
+		}
 	}
 	else {
 		CTR3(KTR_INTR, "sched_ithd %d: it_need %d, state %d",
 			ir->it_proc->p_pid,
 		        ir->it_need,
 		        ir->it_proc->p_stat );
+		need_resched();
 	}
 	mtx_exit(&sched_lock, MTX_SPIN);
-	need_resched();
 }
 
 /*

sys/amd64/isa/npx.c

@@ -796,6 +796,8 @@ npx_intr(dummy)
 int
 npxdna()
 {
+	int s;
+
 	if (!npx_exists)
 		return (0);
 	if (PCPU_GET(npxproc) != NULL) {
@@ -803,6 +805,8 @@ npxdna()
 		       PCPU_GET(npxproc), curproc);
 		panic("npxdna");
 	}
+	s = save_intr();
+	disable_intr();
 	stop_emulating();
 	/*
 	 * Record new context early in case frstor causes an IRQ13.
@@ -822,6 +826,7 @@ npxdna()
 	 * first FPU instruction after a context switch.
 	 */
 	frstor(&PCPU_GET(curpcb)->pcb_savefpu);
+	restore_intr(s);
 
 	return (1);
 }

sys/i386/isa/ithread.c

@@ -121,16 +121,20 @@ sched_ithd(void *cookie)
 /*		membar_lock(); */
 		ir->it_proc->p_stat = SRUN;
 		setrunqueue(ir->it_proc);
-		aston();
+		if (!cold) {
+			if (curproc != PCPU_GET(idleproc))
+				setrunqueue(curproc);
+			mi_switch();
+		}
 	}
 	else {
 		CTR3(KTR_INTR, "sched_ithd %d: it_need %d, state %d",
 			ir->it_proc->p_pid,
 		        ir->it_need,
 		        ir->it_proc->p_stat );
+		need_resched();
 	}
 	mtx_exit(&sched_lock, MTX_SPIN);
-	need_resched();
 }
 
 /*

sys/i386/isa/npx.c

@@ -796,6 +796,8 @@ npx_intr(dummy)
 int
 npxdna()
 {
+	int s;
+
 	if (!npx_exists)
 		return (0);
 	if (PCPU_GET(npxproc) != NULL) {
@@ -803,6 +805,8 @@ npxdna()
 		       PCPU_GET(npxproc), curproc);
 		panic("npxdna");
 	}
+	s = save_intr();
+	disable_intr();
 	stop_emulating();
 	/*
 	 * Record new context early in case frstor causes an IRQ13.
@@ -822,6 +826,7 @@ npxdna()
 	 * first FPU instruction after a context switch.
 	 */
 	frstor(&PCPU_GET(curpcb)->pcb_savefpu);
+	restore_intr(s);
 
 	return (1);
 }