diff --git a/sys/amd64/amd64/autoconf.c b/sys/amd64/amd64/autoconf.c
index 268b0f92e0e4..94a8b549c102 100644
--- a/sys/amd64/amd64/autoconf.c
+++ b/sys/amd64/amd64/autoconf.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)autoconf.c	7.1 (Berkeley) 5/9/91
- *	$Id: autoconf.c,v 1.72 1997/07/20 18:05:16 fsmp Exp $
+ *	$Id: autoconf.c,v 1.5 1997/07/22 18:37:49 smp Exp smp $
  */
 
 /*
diff --git a/sys/amd64/amd64/tsc.c b/sys/amd64/amd64/tsc.c
index e9faacc1ecf0..a9d2110be245 100644
--- a/sys/amd64/amd64/tsc.c
+++ b/sys/amd64/amd64/tsc.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)clock.c	7.2 (Berkeley) 5/12/91
- *	$Id: clock.c,v 1.3 1997/07/20 18:11:45 smp Exp smp $
+ *	$Id: clock.c,v 1.6 1997/07/22 18:36:06 smp Exp smp $
  */
 
 /*
@@ -829,33 +829,6 @@ resettodr()
 	writertc(RTC_STATUSB, rtc_statusb);
 }
 
-#ifdef APIC_IO
-
-/* XXX FIXME: from icu.s: */
-#ifdef NEW_STRATEGY
-
-extern u_int	ivectors[];
-extern u_int	vec[];
-extern void	vec8254	__P((void));
-extern u_int	Xintr8254;
-extern u_int	mask8254;
-
-#else /** NEW_STRATEGY */
-
-#if !defined(APIC_PIN0_TIMER)
-extern u_int	ivectors[];
-extern u_int	vec[];
-#endif
-
-#ifndef APIC_PIN0_TIMER
-extern void	vec8254	__P((void));
-extern u_int	Xintr8254;
-extern u_int	mask8254;
-#endif /* APIC_PIN0_TIMER */
-
-#endif /** NEW_STRATEGY */
-
-#endif /* APIC_IO */
 
 /*
  * Start both clocks running.
@@ -951,7 +924,7 @@ cpu_initclocks()
 	register_intr(/* irq */ 0, /* XXX id */ 0, /* flags */ 0,
 		      /* XXX */ (inthand2_t *)clkintr, &clk_imask,
 		      /* unit */ 0);
-	INTREN(IRQ0);
+	INTREN(APIC_IRQ0);
 #else /* APIC_PIN0_TIMER */
 	/* 8254 is traditionally on ISA IRQ0 */
 	if ((x = isa_apic_pin(0)) < 0) {
@@ -1027,7 +1000,11 @@ cpu_initclocks()
 	register_intr(/* irq */ 8, /* XXX id */ 1, /* flags */ 0,
 		      /* XXX */ (inthand2_t *)rtcintr, &stat_imask,
 		      /* unit */ 0);
+#ifdef APIC_IO
+	INTREN(APIC_IRQ8);
+#else
 	INTREN(IRQ8);
+#endif /* APIC_IO */
 
 	writertc(RTC_STATUSB, rtc_statusb);
 }
diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h
index 51c7f66269b4..ab62210c214c 100644
--- a/sys/amd64/include/smp.h
+++ b/sys/amd64/include/smp.h
@@ -6,7 +6,7 @@
  * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
  * ----------------------------------------------------------------------------
  *
- * $Id: smp.h,v 1.8 1997/07/20 17:48:00 smp Exp smp $
+ * $Id: smp.h,v 1.13 1997/07/22 18:31:51 smp Exp smp $
  *
  */
 
@@ -57,9 +57,24 @@ void	rel_mplock		__P((void));
 void	try_mplock		__P((void));
 
 /* global data in apic_vector.s */
+extern u_int			ivectors[];
 extern volatile u_int		stopped_cpus;
 extern volatile u_int		started_cpus;
 
+/* global data in apic_ipl.s */
+extern u_int			vec[];
+extern u_int			Xintr8254;
+extern u_int			mask8254;
+
+/* functions in apic_ipl.s */
+void	vec8254			__P((void));
+void	INTREN			__P((u_int));
+void	INTRDIS			__P((u_int));
+void	apic_eoi		__P((void));
+u_int	io_apic_read		__P((int, int));
+void	io_apic_write		__P((int, int, u_int));
+void	write_io_apic_mask24	__P((int, u_int));
+
 /* global data in mp_machdep.c */
 extern int			mp_ncpus;
 extern int			mp_naps;
@@ -112,11 +127,11 @@ extern volatile ioapic_t	*ioapic[];
 /* functions in mpapic.c */
 void	apic_dump		__P((char*));
 void	apic_initialize		__P((void));
+void	imen_dump		__P((void));
 int	apic_ipi		__P((int, int, int));
 int	selected_apic_ipi	__P((u_int, int, int));
 int	io_apic_setup		__P((int));
 int	ext_int_setup		__P((int, int));
-void	write_io_apic_mask24	__P((int, u_int32_t));
 
 #if defined(READY)
 void	clr_io_apic_mask24	__P((int, u_int32_t));
diff --git a/sys/amd64/isa/clock.c b/sys/amd64/isa/clock.c
index e9faacc1ecf0..a9d2110be245 100644
--- a/sys/amd64/isa/clock.c
+++ b/sys/amd64/isa/clock.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)clock.c	7.2 (Berkeley) 5/12/91
- *	$Id: clock.c,v 1.3 1997/07/20 18:11:45 smp Exp smp $
+ *	$Id: clock.c,v 1.6 1997/07/22 18:36:06 smp Exp smp $
  */
 
 /*
@@ -829,33 +829,6 @@ resettodr()
 	writertc(RTC_STATUSB, rtc_statusb);
 }
 
-#ifdef APIC_IO
-
-/* XXX FIXME: from icu.s: */
-#ifdef NEW_STRATEGY
-
-extern u_int	ivectors[];
-extern u_int	vec[];
-extern void	vec8254	__P((void));
-extern u_int	Xintr8254;
-extern u_int	mask8254;
-
-#else /** NEW_STRATEGY */
-
-#if !defined(APIC_PIN0_TIMER)
-extern u_int	ivectors[];
-extern u_int	vec[];
-#endif
-
-#ifndef APIC_PIN0_TIMER
-extern void	vec8254	__P((void));
-extern u_int	Xintr8254;
-extern u_int	mask8254;
-#endif /* APIC_PIN0_TIMER */
-
-#endif /** NEW_STRATEGY */
-
-#endif /* APIC_IO */
 
 /*
  * Start both clocks running.
@@ -951,7 +924,7 @@ cpu_initclocks()
 	register_intr(/* irq */ 0, /* XXX id */ 0, /* flags */ 0,
 		      /* XXX */ (inthand2_t *)clkintr, &clk_imask,
 		      /* unit */ 0);
-	INTREN(IRQ0);
+	INTREN(APIC_IRQ0);
 #else /* APIC_PIN0_TIMER */
 	/* 8254 is traditionally on ISA IRQ0 */
 	if ((x = isa_apic_pin(0)) < 0) {
@@ -1027,7 +1000,11 @@ cpu_initclocks()
 	register_intr(/* irq */ 8, /* XXX id */ 1, /* flags */ 0,
 		      /* XXX */ (inthand2_t *)rtcintr, &stat_imask,
 		      /* unit */ 0);
+#ifdef APIC_IO
+	INTREN(APIC_IRQ8);
+#else
 	INTREN(IRQ8);
+#endif /* APIC_IO */
 
 	writertc(RTC_STATUSB, rtc_statusb);
 }
diff --git a/sys/amd64/isa/icu.h b/sys/amd64/isa/icu.h
index 28e3d8d861e9..4197130a5bfd 100644
--- a/sys/amd64/isa/icu.h
+++ b/sys/amd64/isa/icu.h
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)icu.h	5.6 (Berkeley) 5/9/91
- *	$Id: icu.h,v 1.13 1997/05/29 04:58:04 peter Exp $
+ *	$Id: icu.h,v 1.4 1997/07/22 18:36:06 smp Exp smp $
  */
 
 /*
@@ -47,58 +47,27 @@
 
 #ifndef	LOCORE
 
+#ifdef APIC_IO
+
 /*
- * Interrupt "level" mechanism variables, masks, and macros
+#define MP_SAFE
+ * Note:
+ *	Most of the SMP equivilants of the icu macros are coded
+ *	elsewhere in an MP-safe fashion.
+ *	In particular note that the 'imen' variable is opaque.
+ *	DO NOT access imen directly, use INTREN()/INTRDIS().
  */
-extern	unsigned imen;		/* interrupt mask enable */
-
-#if defined(APIC_IO)
-
-# if !defined(_MACHINE_SMP_H_)
-/** XXX what a hack, its this or include <machine/smp.h>! */
-void write_io_apic_mask24 __P((int, u_int32_t)); /* i386/i386/mpapic.c */
-# endif /* _MACHINE_SMP_H_ */
-
-#if defined(MULTIPLE_IOAPICS)
-#error MULTIPLE_IOAPICSXXX: cannot assume apic #0 in the following functions.
-#endif /* MULTIPLE_IOAPICS */
-
-#ifdef nevermore
-/* see if we can get by without these, they're NOT MP_SAFE... */
-static __inline u_int32_t
-INTRGET(void)
-{
-	return (imen & 0x00ffffff);		/* return our global copy */
-}
-
-static __inline void
-INTRSET(unsigned s)
-{
-	write_io_apic_mask24(0, s);
-	imen = s;
-}
-#endif /** nevermore */
-
-static __inline void
-INTREN(unsigned s)
-{
-	write_io_apic_mask24(0, imen & ~s);
-	imen &= ~s;
-}
-
-static __inline void
-INTRDIS(unsigned s)
-{
-	write_io_apic_mask24(0, imen | s);
-	imen |= s;
-}
-
 
 #define	INTRMASK(msk,s)		(msk |= (s))
 #define INTRUNMASK(msk,s)	(msk &= ~(s))
 
 #else /* APIC_IO */
 
+/*
+ * Interrupt "level" mechanism variables, masks, and macros
+ */
+extern	unsigned imen;		/* interrupt mask enable */
+
 #define	INTREN(s)		(imen &= ~(s), SET_ICUS())
 #define	INTRDIS(s)		(imen |= (s), SET_ICUS())
 #define	INTRMASK(msk,s)		(msk |= (s))
@@ -131,6 +100,17 @@ INTRDIS(unsigned s)
 
 #endif /* LOCORE */
 
+
+#ifdef APIC_IO
+/*
+ * Note: The APIC uses different values for IRQxxx.
+ *	 Unfortunately many drivers use the 8259 values as indexes
+ *	 into tables, etc.  The APIC equivilants are kept as APIC_IRQxxx.
+ *	 The 8259 versions have to be used in SMP for legacy operation
+ *	 of the drivers.
+ */
+#endif /* APIC_IO */
+
 /*
  * Interrupt enable bits - in normal order of priority (which we change)
  */
@@ -159,12 +139,13 @@ INTRDIS(unsigned s)
 #define	IRQ_SLAVE	0x0080
 #endif
 
+
 /*
  * Interrupt Control offset into Interrupt descriptor table (IDT)
  */
 #define	ICU_OFFSET	32		/* 0-31 are processor exceptions */
 
-#if defined(APIC_IO)
+#ifdef APIC_IO
 
 /* 32-47: ISA IRQ0-IRQ15, 48-55: IO APIC IRQ16-IRQ23 */
 #define	ICU_LEN		24
diff --git a/sys/i386/i386/autoconf.c b/sys/i386/i386/autoconf.c
index 268b0f92e0e4..94a8b549c102 100644
--- a/sys/i386/i386/autoconf.c
+++ b/sys/i386/i386/autoconf.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)autoconf.c	7.1 (Berkeley) 5/9/91
- *	$Id: autoconf.c,v 1.72 1997/07/20 18:05:16 fsmp Exp $
+ *	$Id: autoconf.c,v 1.5 1997/07/22 18:37:49 smp Exp smp $
  */
 
 /*
diff --git a/sys/i386/i386/mpapic.c b/sys/i386/i386/mpapic.c
index a03b7963c4ba..2f60ffed8e59 100644
--- a/sys/i386/i386/mpapic.c
+++ b/sys/i386/i386/mpapic.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mpapic.c,v 1.18 1997/07/22 18:48:07 fsmp Exp $
+ *	$Id: mpapic.c,v 1.18 1997/07/22 18:37:49 smp Exp smp $
  */
 
 #include "opt_smp.h"
@@ -113,9 +113,12 @@ apic_initialize(void)
 void
 apic_dump(char* str)
 {
-	printf("SMP: CPU%d %s:\n", cpuid, str);
-	printf("    lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n",
-		lapic.lvt_lint0, lapic.lvt_lint1, lapic.tpr, lapic.svr);
+	printf("SMP: CPU%02d %s lint0: 0x%08x\n",
+	       cpuid, str, lapic.lvt_lint0);
+	printf("                                lint1: 0x%08x\n",
+	       lapic.lvt_lint1);
+	printf("                                TPR:   0x%08x\n", lapic.tpr);
+	printf("                                SVR:   0x%08x\n", lapic.svr);
 }
 
 
@@ -432,137 +435,20 @@ bad:
 
 
 /*
- * Set INT mask bit for each bit set in 'mask'.
- * Clear INT mask bit for all others.
- * Only consider lower 24 bits in mask.
- */
-#if defined(MULTIPLE_IOAPICS)
-#error MULTIPLE_IOAPICSXXX
-#else
-#define IO_MASK		(imen & 0x00ffffff)
-#define IO_FIELD	0x00ffffff
-
-void
-write_io_apic_mask24(int apic, u_int32_t mask)
-{
-	int     x, y;
-	u_char  select;			/* the select register is 8 bits */
-	u_int32_t low_reg;		/* the window register is 32 bits */
-	u_int32_t diffs;
-
-	mask &= IO_FIELD;		/* safety valve, only use 24 bits */
-	if (mask == IO_MASK)		/* check for same value as current */
-		return;
-
-	diffs = mask ^ IO_MASK;		/* record differences */
-
-	for (x = 0, y = REDIRCNT_IOAPIC(apic); x < y; ++x) {
-		if (!(diffs & (1 << x)))
-			continue;			/* no change, skip */
-
-		select = IOAPIC_REDTBL + (x * 2);	/* calculate addr */
-		low_reg = io_apic_read(apic, select);	/* read contents */
-
-		if (mask & (1 << x))
-			low_reg |= IOART_INTMASK;	/* set mask */
-		else
-			low_reg &= ~IOART_INTMASK;	/* clear mask */
-
-		io_apic_write(apic, select, low_reg);	/* new value */
-	}
-}
-#endif /* MULTIPLE_IOAPICS */
-
-
-#if defined(READY)
-/*
- * Read current IRQ0 -IRQ23 masks.
- */
-#if defined(MULTIPLE_IOAPICS)
-#error MULTIPLE_IOAPICSXXX
-#else
-static __inline u_int32_t
-read_io_apic_mask24(int apic)
-{
-	
-}
-#endif /* MULTIPLE_IOAPICS */
-#endif /* READY */
-
-
-#if defined(READY)
-/*
- * Set INT mask bit for each bit set in 'mask'.
- * Ignore INT mask bit for all others.
- * Only consider lower 24 bits in mask.
+ * Print contents of imen, keeps imen 'opaque'.
  */
 void
-set_io_apic_mask24(apic, u_int32_t bits)
+imen_dump(void)
 {
-	int     x, y;
-	u_char  select;			/* the select register is 8 bits */
-	u_int32_t low_reg;		/* the window register is 32 bits */
-	u_int32_t diffs;
+	extern	unsigned imen;		/* interrupt mask enable */
+	int x;
 
-	bits &= IO_FIELD;		/* safety valve, only use 24 bits */
-	diffs = bits & ~IO_MASK;	/* clear AND needing 'set'ing */
-	if (!diffs)
-		return;
-
-#error imen/io_apic_mask NOT merged in set_io_apic_mask24()
-
-	for (x = 0, y = REDIRCNT_IOAPIC(apic); x < y; ++x) {
-		if (!(diffs & (1 << x)))
-			continue;			/* no change, skip */
-
-		select = IOAPIC_REDTBL + (x * 2);	/* calculate addr */
-		low_reg = io_apic_read(apic, select);	/* read contents */
-
-		lowReg |= IOART_INTMASK;		/* set mask */
-
-		io_apic_write(apic, select, low_reg);	/* new value */
-	}
+	printf("SMP: enabled INTs: ");
+	for (x = 0; x < 24; ++x)
+		if ((imen & (1 << x)) == 0)
+        		printf("%d, ", x);
+	printf("imen: 0x%08x\n", imen);
 }
-#endif	/* READY */
-
-
-#if defined(READY)
-/*
- * Clear INT mask bit for each bit set in 'mask'.
- * Ignore INT mask bit for all others.
- * Only consider lower 24 bits in mask.
- */
-void
-clr_io_apic_mask24(int apic, u_int32_t bits)
-{
-	int     x, y;
-	u_char  select;			/* the select register is 8 bits */
-	u_int32_t low_reg;		/* the window register is 32 bits */
-	u_int32_t diffs;
-
-	bits &= IO_FIELD;		/* safety valve, only use 24 bits */
-	diffs = bits & IO_MASK;		/* set AND needing 'clr'ing */
-	if (!diffs)
-		return;
-
-#error imen/io_apic_mask NOT merged in clr_io_apic_mask24()
-
-	for (x = 0, y = REDIRCNT_IOAPIC(apic); x < y; ++x) {
-		if (!(diffs & (1 << x)))
-			continue;			/* no change, skip */
-
-		select = IOAPIC_REDTBL + (x * 2);	/* calculate addr */
-		low_reg = io_apic_read(apic, select);	/* read contents */
-
-		low_reg &= ~IOART_INTMASK;		/* clear mask */
-
-		io_apic_write(apic, select, low_reg);	/* new value */
-	}
-}
-#endif	/* READY */
-
-#undef IO_FIELD
-#undef IO_MASK
 
 
 /*
@@ -686,7 +572,6 @@ selected_proc_ipi(int target, int vector)
 }
 #endif /* READY */
 
-
 #endif	/* APIC_IO */
 
 
diff --git a/sys/i386/i386/tsc.c b/sys/i386/i386/tsc.c
index e9faacc1ecf0..a9d2110be245 100644
--- a/sys/i386/i386/tsc.c
+++ b/sys/i386/i386/tsc.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)clock.c	7.2 (Berkeley) 5/12/91
- *	$Id: clock.c,v 1.3 1997/07/20 18:11:45 smp Exp smp $
+ *	$Id: clock.c,v 1.6 1997/07/22 18:36:06 smp Exp smp $
  */
 
 /*
@@ -829,33 +829,6 @@ resettodr()
 	writertc(RTC_STATUSB, rtc_statusb);
 }
 
-#ifdef APIC_IO
-
-/* XXX FIXME: from icu.s: */
-#ifdef NEW_STRATEGY
-
-extern u_int	ivectors[];
-extern u_int	vec[];
-extern void	vec8254	__P((void));
-extern u_int	Xintr8254;
-extern u_int	mask8254;
-
-#else /** NEW_STRATEGY */
-
-#if !defined(APIC_PIN0_TIMER)
-extern u_int	ivectors[];
-extern u_int	vec[];
-#endif
-
-#ifndef APIC_PIN0_TIMER
-extern void	vec8254	__P((void));
-extern u_int	Xintr8254;
-extern u_int	mask8254;
-#endif /* APIC_PIN0_TIMER */
-
-#endif /** NEW_STRATEGY */
-
-#endif /* APIC_IO */
 
 /*
  * Start both clocks running.
@@ -951,7 +924,7 @@ cpu_initclocks()
 	register_intr(/* irq */ 0, /* XXX id */ 0, /* flags */ 0,
 		      /* XXX */ (inthand2_t *)clkintr, &clk_imask,
 		      /* unit */ 0);
-	INTREN(IRQ0);
+	INTREN(APIC_IRQ0);
 #else /* APIC_PIN0_TIMER */
 	/* 8254 is traditionally on ISA IRQ0 */
 	if ((x = isa_apic_pin(0)) < 0) {
@@ -1027,7 +1000,11 @@ cpu_initclocks()
 	register_intr(/* irq */ 8, /* XXX id */ 1, /* flags */ 0,
 		      /* XXX */ (inthand2_t *)rtcintr, &stat_imask,
 		      /* unit */ 0);
+#ifdef APIC_IO
+	INTREN(APIC_IRQ8);
+#else
 	INTREN(IRQ8);
+#endif /* APIC_IO */
 
 	writertc(RTC_STATUSB, rtc_statusb);
 }
diff --git a/sys/i386/include/mpapic.h b/sys/i386/include/mpapic.h
index 3ea2f6d8d9d5..d984a3b19189 100644
--- a/sys/i386/include/mpapic.h
+++ b/sys/i386/include/mpapic.h
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mpapic.h,v 1.7 1997/05/31 03:29:06 fsmp Exp $
+ *	$Id: mpapic.h,v 1.3 1997/07/22 18:31:51 smp Exp smp $
  */
 
 #ifndef _MACHINE_MPAPIC_H_
@@ -75,46 +75,6 @@ enum busTypes {
 #define ID_TO_IO(ID)	(apic_id_to_logical[ID])
 
 
-/*
- * inline functions to read/write the IO APIC
- * NOTES:
- *  unlike the local APIC, the IO APIC is accessed indirectly thru 2 registers.
- *  the select register is loaded with an index to the desired 'window' reg.
- *  the 'window' is accessed as a 32 bit unsigned.
- */
-
-/*
- * read 'reg' from 'apic'
- */
-static __inline u_int32_t
-io_apic_read(int apic, int reg)
-{
-	ioapic[apic]->ioregsel = reg;
-	return ioapic[apic]->iowin;
-}
-
-
-/*
- * write 'value' to 'reg' of 'apic'
- */
-static __inline void
-io_apic_write(int apic, int reg, u_int32_t value)
-{
-	ioapic[apic]->ioregsel = reg;
-	ioapic[apic]->iowin = value;
-}
-
-
-/*
- * send an EndOfInterrupt to the local APIC
- */
-static __inline void
-apic_eoi(void)
-{
-	lapic.eoi = 0;
-}
-
-
 /*
  * send an IPI INTerrupt containing 'vector' to CPUs in 'targetMap'
  * 'targetMap' is a bitfiled of length 14,
diff --git a/sys/i386/include/smp.h b/sys/i386/include/smp.h
index 51c7f66269b4..ab62210c214c 100644
--- a/sys/i386/include/smp.h
+++ b/sys/i386/include/smp.h
@@ -6,7 +6,7 @@
  * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
  * ----------------------------------------------------------------------------
  *
- * $Id: smp.h,v 1.8 1997/07/20 17:48:00 smp Exp smp $
+ * $Id: smp.h,v 1.13 1997/07/22 18:31:51 smp Exp smp $
  *
  */
 
@@ -57,9 +57,24 @@ void	rel_mplock		__P((void));
 void	try_mplock		__P((void));
 
 /* global data in apic_vector.s */
+extern u_int			ivectors[];
 extern volatile u_int		stopped_cpus;
 extern volatile u_int		started_cpus;
 
+/* global data in apic_ipl.s */
+extern u_int			vec[];
+extern u_int			Xintr8254;
+extern u_int			mask8254;
+
+/* functions in apic_ipl.s */
+void	vec8254			__P((void));
+void	INTREN			__P((u_int));
+void	INTRDIS			__P((u_int));
+void	apic_eoi		__P((void));
+u_int	io_apic_read		__P((int, int));
+void	io_apic_write		__P((int, int, u_int));
+void	write_io_apic_mask24	__P((int, u_int));
+
 /* global data in mp_machdep.c */
 extern int			mp_ncpus;
 extern int			mp_naps;
@@ -112,11 +127,11 @@ extern volatile ioapic_t	*ioapic[];
 /* functions in mpapic.c */
 void	apic_dump		__P((char*));
 void	apic_initialize		__P((void));
+void	imen_dump		__P((void));
 int	apic_ipi		__P((int, int, int));
 int	selected_apic_ipi	__P((u_int, int, int));
 int	io_apic_setup		__P((int));
 int	ext_int_setup		__P((int, int));
-void	write_io_apic_mask24	__P((int, u_int32_t));
 
 #if defined(READY)
 void	clr_io_apic_mask24	__P((int, u_int32_t));
diff --git a/sys/i386/isa/apic_ipl.s b/sys/i386/isa/apic_ipl.s
index e5f56032a1bd..34536dac081b 100644
--- a/sys/i386/isa/apic_ipl.s
+++ b/sys/i386/isa/apic_ipl.s
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: apic_ipl.s,v 1.2 1997/07/20 18:11:45 smp Exp smp $
+ *	$Id: apic_ipl.s,v 1.13 1997/07/22 18:36:06 smp Exp smp $
  */
 
 
@@ -197,3 +197,197 @@ __CONCAT(vec,irq_num): ;						\
 	BUILD_VEC(22)
 	BUILD_VEC(23)
 
+
+/******************************************************************************
+ * XXX FIXME: figure out where these belong.
+ */
+
+/*
+ * MULTIPLE_IOAPICSXXX: cannot assume apic #0 in the following function.
+ * (soon to be) MP-safe function to set ONE INT mask bit.
+ * The passed arg is a 32bit u_int MASK.
+ * It sets the associated bit in imen.
+ * It sets the mask bit of the associated IO APIC register.
+ */
+	ALIGN_TEXT
+	.globl _INTREN
+_INTREN:
+	movl 4(%esp), %eax		/* mask into %eax */
+	notl %eax			/* mask = ~mask */
+	andl _imen, %eax		/* %eax = imen & ~mask */
+
+	pushl %eax			/* new (future) imen value */
+	pushl $0			/* APIC# arg */
+	call write_io_apic_mask		/* modify the APIC registers */
+
+	addl $4, %esp			/* remove APIC# arg from stack */
+	popl _imen			/* imen &= ~mask */
+	ret
+
+/*
+ * MULTIPLE_IOAPICSXXX: cannot assume apic #0 in the following function.
+ * (soon to be) MP-safe function to clear ONE INT mask bit.
+ * The passed arg is a 32bit u_int MASK.
+ * It clears the associated bit in imen.
+ * It clears the mask bit of the associated IO APIC register.
+ */
+	ALIGN_TEXT
+	.globl _INTRDIS
+_INTRDIS:
+	movl _imen, %eax
+	orl 4(%esp), %eax		/* %eax = imen | mask */
+
+	pushl %eax			/* new (future) imen value */
+	pushl $0			/* APIC# arg */
+	call write_io_apic_mask		/* modify the APIC registers */
+
+	addl $4, %esp			/* remove APIC# arg from stack */
+	popl _imen			/* imen |= mask */
+	ret
+
+
+/******************************************************************************
+ *
+ */
+
+/*
+ * void write_io_apic_mask(int apic, u_int mask); 
+ */
+
+#define _INT_MASK	0x00010000
+#define _PIN_MASK	0x00ffffff
+
+#define _OLD_ESI	  0(%esp)
+#define _OLD_EBX	  4(%esp)
+#define _RETADDR	  8(%esp)
+#define _APIC		 12(%esp)
+#define _MASK		 16(%esp)
+
+	.align 2
+write_io_apic_mask:
+	pushl %ebx			/* scratch */
+	pushl %esi			/* scratch */
+
+	movl _imen, %ebx
+	xorl _MASK, %ebx		/* %ebx = imen ^ mask */
+	andl $_PIN_MASK, %ebx		/* %ebx = imen & 0x00ffffff */
+	jz all_done			/* no change, return */
+
+	movl _APIC, %esi		/* APIC # */
+	movl _ioapic(,%esi,4), %esi	/* %esi holds APIC base address */
+
+next_loop:				/* %ebx = diffs, %esi = APIC base */
+	bsfl %ebx, %ecx			/* %ecx = index if 1st/next set bit */
+	jz all_done
+
+	btrl %ecx, %ebx			/* clear this bit in diffs */
+	leal 16(,%ecx,2), %edx		/* calculate register index */
+
+	movl %edx, (%esi)		/* write the target register index */
+	movl 16(%esi), %eax		/* read the target register data */
+
+	btl %ecx, _MASK			/* test for mask or unmask */
+	jnc clear			/* bit is clear */
+	orl $_INT_MASK, %eax		/* set mask bit */
+	jmp write
+clear:	andl $~_INT_MASK, %eax		/* clear mask bit */
+
+write:	movl %edx, (%esi)		/* write the target register index */
+	movl %eax, 16(%esi)		/* write the APIC register data */
+
+	jmp next_loop			/* try another pass */
+
+all_done:
+	popl %esi
+	popl %ebx
+	ret
+
+#undef _OLD_ESI
+#undef _OLD_EBX
+#undef _RETADDR
+#undef _APIC
+#undef _MASK
+
+#undef _PIN_MASK
+#undef _INT_MASK
+
+
+#ifdef ready
+
+/*
+ * u_int read_io_apic_mask(int apic); 
+ */
+	.align 2
+read_io_apic_mask:
+	ret
+
+/*
+ * Set INT mask bit for each bit set in 'mask'.
+ * Ignore INT mask bit for all others.
+ * Only consider lower 24 bits in mask.
+ *
+ * void set_io_apic_mask24(apic, u_int32_t bits); 
+ */
+	.align 2
+set_io_apic_mask:
+	ret
+
+/*
+ * Clear INT mask bit for each bit set in 'mask'.
+ * Ignore INT mask bit for all others.
+ * Only consider lower 24 bits in mask.
+ *
+ * void clr_io_apic_mask24(int apic, u_int32_t bits); 
+ */
+	.align 2
+clr_io_apic_mask24:
+	ret
+
+/*
+ * 
+ */
+	.align 2
+
+	ret
+
+#endif /** ready */
+
+/******************************************************************************
+ * 
+ */
+
+/*
+ * u_int io_apic_write(int apic, int select);
+ */
+	.align 2
+	.globl _io_apic_read
+_io_apic_read:
+	movl 4(%esp), %ecx		/* APIC # */
+	movl _ioapic(,%ecx,4), %edx	/* APIC base register address */
+	movl 8(%esp), %eax		/* target register index */
+	movl %eax, (%edx)		/* write the target register index */
+	movl 16(%edx), %eax		/* read the APIC register data */
+	ret				/* %eax = register value */
+
+/*
+ * void io_apic_write(int apic, int select, int value);
+ */
+	.align 2
+	.globl _io_apic_write
+_io_apic_write:
+	movl 4(%esp), %ecx		/* APIC # */
+	movl _ioapic(,%ecx,4), %edx	/* APIC base register address */
+	movl 8(%esp), %eax		/* target register index */
+	movl %eax, (%edx)		/* write the target register index */
+	movl 12(%esp), %eax		/* target register value */
+	movl %eax, 16(%edx)		/* write the APIC register data */
+	ret				/* %eax = void */
+
+/*
+ * Send an EOI to the local APIC.
+ */
+	.align 2
+	.globl _apic_eoi
+_apic_eoi:
+	movl $0, _lapic+0xb0
+	ret
diff --git a/sys/i386/isa/clock.c b/sys/i386/isa/clock.c
index e9faacc1ecf0..a9d2110be245 100644
--- a/sys/i386/isa/clock.c
+++ b/sys/i386/isa/clock.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)clock.c	7.2 (Berkeley) 5/12/91
- *	$Id: clock.c,v 1.3 1997/07/20 18:11:45 smp Exp smp $
+ *	$Id: clock.c,v 1.6 1997/07/22 18:36:06 smp Exp smp $
  */
 
 /*
@@ -829,33 +829,6 @@ resettodr()
 	writertc(RTC_STATUSB, rtc_statusb);
 }
 
-#ifdef APIC_IO
-
-/* XXX FIXME: from icu.s: */
-#ifdef NEW_STRATEGY
-
-extern u_int	ivectors[];
-extern u_int	vec[];
-extern void	vec8254	__P((void));
-extern u_int	Xintr8254;
-extern u_int	mask8254;
-
-#else /** NEW_STRATEGY */
-
-#if !defined(APIC_PIN0_TIMER)
-extern u_int	ivectors[];
-extern u_int	vec[];
-#endif
-
-#ifndef APIC_PIN0_TIMER
-extern void	vec8254	__P((void));
-extern u_int	Xintr8254;
-extern u_int	mask8254;
-#endif /* APIC_PIN0_TIMER */
-
-#endif /** NEW_STRATEGY */
-
-#endif /* APIC_IO */
 
 /*
  * Start both clocks running.
@@ -951,7 +924,7 @@ cpu_initclocks()
 	register_intr(/* irq */ 0, /* XXX id */ 0, /* flags */ 0,
 		      /* XXX */ (inthand2_t *)clkintr, &clk_imask,
 		      /* unit */ 0);
-	INTREN(IRQ0);
+	INTREN(APIC_IRQ0);
 #else /* APIC_PIN0_TIMER */
 	/* 8254 is traditionally on ISA IRQ0 */
 	if ((x = isa_apic_pin(0)) < 0) {
@@ -1027,7 +1000,11 @@ cpu_initclocks()
 	register_intr(/* irq */ 8, /* XXX id */ 1, /* flags */ 0,
 		      /* XXX */ (inthand2_t *)rtcintr, &stat_imask,
 		      /* unit */ 0);
+#ifdef APIC_IO
+	INTREN(APIC_IRQ8);
+#else
 	INTREN(IRQ8);
+#endif /* APIC_IO */
 
 	writertc(RTC_STATUSB, rtc_statusb);
 }
diff --git a/sys/i386/isa/icu.h b/sys/i386/isa/icu.h
index 28e3d8d861e9..4197130a5bfd 100644
--- a/sys/i386/isa/icu.h
+++ b/sys/i386/isa/icu.h
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)icu.h	5.6 (Berkeley) 5/9/91
- *	$Id: icu.h,v 1.13 1997/05/29 04:58:04 peter Exp $
+ *	$Id: icu.h,v 1.4 1997/07/22 18:36:06 smp Exp smp $
  */
 
 /*
@@ -47,58 +47,27 @@
 
 #ifndef	LOCORE
 
+#ifdef APIC_IO
+
 /*
- * Interrupt "level" mechanism variables, masks, and macros
+#define MP_SAFE
+ * Note:
+ *	Most of the SMP equivilants of the icu macros are coded
+ *	elsewhere in an MP-safe fashion.
+ *	In particular note that the 'imen' variable is opaque.
+ *	DO NOT access imen directly, use INTREN()/INTRDIS().
  */
-extern	unsigned imen;		/* interrupt mask enable */
-
-#if defined(APIC_IO)
-
-# if !defined(_MACHINE_SMP_H_)
-/** XXX what a hack, its this or include <machine/smp.h>! */
-void write_io_apic_mask24 __P((int, u_int32_t)); /* i386/i386/mpapic.c */
-# endif /* _MACHINE_SMP_H_ */
-
-#if defined(MULTIPLE_IOAPICS)
-#error MULTIPLE_IOAPICSXXX: cannot assume apic #0 in the following functions.
-#endif /* MULTIPLE_IOAPICS */
-
-#ifdef nevermore
-/* see if we can get by without these, they're NOT MP_SAFE... */
-static __inline u_int32_t
-INTRGET(void)
-{
-	return (imen & 0x00ffffff);		/* return our global copy */
-}
-
-static __inline void
-INTRSET(unsigned s)
-{
-	write_io_apic_mask24(0, s);
-	imen = s;
-}
-#endif /** nevermore */
-
-static __inline void
-INTREN(unsigned s)
-{
-	write_io_apic_mask24(0, imen & ~s);
-	imen &= ~s;
-}
-
-static __inline void
-INTRDIS(unsigned s)
-{
-	write_io_apic_mask24(0, imen | s);
-	imen |= s;
-}
-
 
 #define	INTRMASK(msk,s)		(msk |= (s))
 #define INTRUNMASK(msk,s)	(msk &= ~(s))
 
 #else /* APIC_IO */
 
+/*
+ * Interrupt "level" mechanism variables, masks, and macros
+ */
+extern	unsigned imen;		/* interrupt mask enable */
+
 #define	INTREN(s)		(imen &= ~(s), SET_ICUS())
 #define	INTRDIS(s)		(imen |= (s), SET_ICUS())
 #define	INTRMASK(msk,s)		(msk |= (s))
@@ -131,6 +100,17 @@ INTRDIS(unsigned s)
 
 #endif /* LOCORE */
 
+
+#ifdef APIC_IO
+/*
+ * Note: The APIC uses different values for IRQxxx.
+ *	 Unfortunately many drivers use the 8259 values as indexes
+ *	 into tables, etc.  The APIC equivilants are kept as APIC_IRQxxx.
+ *	 The 8259 versions have to be used in SMP for legacy operation
+ *	 of the drivers.
+ */
+#endif /* APIC_IO */
+
 /*
  * Interrupt enable bits - in normal order of priority (which we change)
  */
@@ -159,12 +139,13 @@ INTRDIS(unsigned s)
 #define	IRQ_SLAVE	0x0080
 #endif
 
+
 /*
  * Interrupt Control offset into Interrupt descriptor table (IDT)
  */
 #define	ICU_OFFSET	32		/* 0-31 are processor exceptions */
 
-#if defined(APIC_IO)
+#ifdef APIC_IO
 
 /* 32-47: ISA IRQ0-IRQ15, 48-55: IO APIC IRQ16-IRQ23 */
 #define	ICU_LEN		24
diff --git a/sys/isa/atrtc.c b/sys/isa/atrtc.c
index e9faacc1ecf0..a9d2110be245 100644
--- a/sys/isa/atrtc.c
+++ b/sys/isa/atrtc.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)clock.c	7.2 (Berkeley) 5/12/91
- *	$Id: clock.c,v 1.3 1997/07/20 18:11:45 smp Exp smp $
+ *	$Id: clock.c,v 1.6 1997/07/22 18:36:06 smp Exp smp $
  */
 
 /*
@@ -829,33 +829,6 @@ resettodr()
 	writertc(RTC_STATUSB, rtc_statusb);
 }
 
-#ifdef APIC_IO
-
-/* XXX FIXME: from icu.s: */
-#ifdef NEW_STRATEGY
-
-extern u_int	ivectors[];
-extern u_int	vec[];
-extern void	vec8254	__P((void));
-extern u_int	Xintr8254;
-extern u_int	mask8254;
-
-#else /** NEW_STRATEGY */
-
-#if !defined(APIC_PIN0_TIMER)
-extern u_int	ivectors[];
-extern u_int	vec[];
-#endif
-
-#ifndef APIC_PIN0_TIMER
-extern void	vec8254	__P((void));
-extern u_int	Xintr8254;
-extern u_int	mask8254;
-#endif /* APIC_PIN0_TIMER */
-
-#endif /** NEW_STRATEGY */
-
-#endif /* APIC_IO */
 
 /*
  * Start both clocks running.
@@ -951,7 +924,7 @@ cpu_initclocks()
 	register_intr(/* irq */ 0, /* XXX id */ 0, /* flags */ 0,
 		      /* XXX */ (inthand2_t *)clkintr, &clk_imask,
 		      /* unit */ 0);
-	INTREN(IRQ0);
+	INTREN(APIC_IRQ0);
 #else /* APIC_PIN0_TIMER */
 	/* 8254 is traditionally on ISA IRQ0 */
 	if ((x = isa_apic_pin(0)) < 0) {
@@ -1027,7 +1000,11 @@ cpu_initclocks()
 	register_intr(/* irq */ 8, /* XXX id */ 1, /* flags */ 0,
 		      /* XXX */ (inthand2_t *)rtcintr, &stat_imask,
 		      /* unit */ 0);
+#ifdef APIC_IO
+	INTREN(APIC_IRQ8);
+#else
 	INTREN(IRQ8);
+#endif /* APIC_IO */
 
 	writertc(RTC_STATUSB, rtc_statusb);
 }
diff --git a/sys/sys/smp.h b/sys/sys/smp.h
index 51c7f66269b4..ab62210c214c 100644
--- a/sys/sys/smp.h
+++ b/sys/sys/smp.h
@@ -6,7 +6,7 @@
  * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
  * ----------------------------------------------------------------------------
  *
- * $Id: smp.h,v 1.8 1997/07/20 17:48:00 smp Exp smp $
+ * $Id: smp.h,v 1.13 1997/07/22 18:31:51 smp Exp smp $
  *
  */
 
@@ -57,9 +57,24 @@ void	rel_mplock		__P((void));
 void	try_mplock		__P((void));
 
 /* global data in apic_vector.s */
+extern u_int			ivectors[];
 extern volatile u_int		stopped_cpus;
 extern volatile u_int		started_cpus;
 
+/* global data in apic_ipl.s */
+extern u_int			vec[];
+extern u_int			Xintr8254;
+extern u_int			mask8254;
+
+/* functions in apic_ipl.s */
+void	vec8254			__P((void));
+void	INTREN			__P((u_int));
+void	INTRDIS			__P((u_int));
+void	apic_eoi		__P((void));
+u_int	io_apic_read		__P((int, int));
+void	io_apic_write		__P((int, int, u_int));
+void	write_io_apic_mask24	__P((int, u_int));
+
 /* global data in mp_machdep.c */
 extern int			mp_ncpus;
 extern int			mp_naps;
@@ -112,11 +127,11 @@ extern volatile ioapic_t	*ioapic[];
 /* functions in mpapic.c */
 void	apic_dump		__P((char*));
 void	apic_initialize		__P((void));
+void	imen_dump		__P((void));
 int	apic_ipi		__P((int, int, int));
 int	selected_apic_ipi	__P((u_int, int, int));
 int	io_apic_setup		__P((int));
 int	ext_int_setup		__P((int, int));
-void	write_io_apic_mask24	__P((int, u_int32_t));
 
 #if defined(READY)
 void	clr_io_apic_mask24	__P((int, u_int32_t));