New APIC support code:

- The apic interrupt entry points have been rewritten so that each entry
  point can serve 32 different vectors.  When the entry is executed, it
  uses one of the 32-bit ISR registers to determine which vector in its
  assigned range was triggered.  Thus, the apic code can support 159
  different interrupt vectors with only 5 entry points.
- We now always to disable the local APIC to work around an errata in
  certain PPros and then re-enable it again if we decide to use the APICs
  to route interrupts.
- We no longer map IO APICs or local APICs using special page table
  entries.  Instead, we just use pmap_mapdev().  We also no longer
  export the virtual address of the local APIC as a global symbol to
  the rest of the system, but only in local_apic.c.  To aid this, the
  APIC ID of each CPU is exported as a per-CPU variable.
- Interrupt sources are provided for each intpin on each IO APIC.
  Currently, each source is given a unique interrupt vector meaning that
  PCI interrupts are not shared on most machines with an I/O APIC.
  That mapping for interrupt sources to interrupt vectors is up to the
  APIC enumerator driver however.
- We no longer probe to see if we need to use mixed mode to route IRQ 0,
  instead we always use mixed mode to route IRQ 0 for now.  This can be
  disabled via the 'NO_MIXED_MODE' kernel option.
- The npx(4) driver now always probes to see if a built-in FPU is present
  since this test can now be performed with the new APIC code.  However,
  an SMP kernel will panic if there is more than one CPU and a built-in
  FPU is not found.
- PCI interrupts are now properly routed when using APICs to route
  interrupts, so remove the hack to psuedo-route interrupts when the
  intpin register was read.
- The apic.h header was moved to apicreg.h and a new apicvar.h header
  that declares the APIs used by the new APIC code was added.

sys/amd64/amd64/apic_vector.S

@@ -1,19 +1,53 @@
-/*
+/*-
+ * Copyright (c) 1989, 1990 William F. Jolitz.
+ * Copyright (c) 1990 The Regents of the University of California.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by the University of
+ *	California, Berkeley and its contributors.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
  *	from: vector.s, 386BSD 0.1 unknown origin
  * $FreeBSD$
  */
 
-#include <machine/apic.h>
-#include <machine/smp.h>
+/*
+ * Interrupt entry points for external interrupts triggered by I/O APICs
+ * as well as IPI handlers.
+ */
 
-/* convert an absolute IRQ# into a bitmask */
-#define IRQ_BIT(irq_num)	(1 << (irq_num))
+#include <machine/asmacros.h>
+#include <machine/apicreg.h>
+#include <machine/smptests.h>
 
-/* make an index into the IO APIC from the IRQ# */
-#define REDTBL_IDX(irq_num)	(0x10 + ((irq_num) * 2))
+#include "assym.s"
 
 /*
- * 
+ * Macros to create and destroy a trap frame.
  */
 #define PUSH_FRAME							\
 	pushl	$0 ;		/* dummy error code */			\
@@ -23,14 +57,6 @@
 	pushl	%es ;							\
 	pushl	%fs
 
-#define PUSH_DUMMY							\
-	pushfl ;		/* eflags */				\
-	pushl	%cs ;		/* cs */				\
-	pushl	12(%esp) ;	/* original caller eip */		\
-	pushl	$0 ;		/* dummy error code */			\
-	pushl	$0 ;		/* dummy trap type */			\
-	subl	$11*4,%esp ;
-
 #define POP_FRAME							\
 	popl	%fs ;							\
 	popl	%es ;							\
@@ -38,209 +64,40 @@
 	popal ;								\
 	addl	$4+4,%esp
 
-#define POP_DUMMY							\
-	addl	$16*4,%esp
-
-#define IOAPICADDR(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 8
-#define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 12
-	
-#define MASK_IRQ(irq_num)						\
-	ICU_LOCK ;				/* into critical reg */	\
-	testl	$IRQ_BIT(irq_num), apic_imen ;				\
-	jne	7f ;			/* masked, don't mask */	\
-	orl	$IRQ_BIT(irq_num), apic_imen ;	/* set the mask bit */	\
-	movl	IOAPICADDR(irq_num), %ecx ;	/* ioapic addr */	\
-	movl	REDIRIDX(irq_num), %eax ;	/* get the index */	\
-	movl	%eax, (%ecx) ;			/* write the index */	\
-	movl	IOAPIC_WINDOW(%ecx), %eax ;	/* current value */	\
-	orl	$IOART_INTMASK, %eax ;		/* set the mask */	\
-	movl	%eax, IOAPIC_WINDOW(%ecx) ;	/* new value */		\
-7: ;						/* already masked */	\
-	ICU_UNLOCK
 /*
- * Test to see whether we are handling an edge or level triggered INT.
- *  Level-triggered INTs must still be masked as we don't clear the source,
- *  and the EOI cycle would cause redundant INTs to occur.
+ * I/O Interrupt Entry Point.  Rather than having one entry point for
+ * each interrupt source, we use one entry point for each 32-bit word
+ * in the ISR.  The handler determines the highest bit set in the ISR,
+ * translates that into a vector, and passes the vector to the
+ * lapic_handle_intr() function.
  */
-#define MASK_LEVEL_IRQ(irq_num)						\
-	testl	$IRQ_BIT(irq_num), apic_pin_trigger ;			\
-	jz	9f ;				/* edge, don't mask */	\
-	MASK_IRQ(irq_num) ;						\
-9:
-
-
-#ifdef APIC_INTR_REORDER
-#define EOI_IRQ(irq_num)						\
-	movl	apic_isrbit_location + 8 * (irq_num), %eax ;		\
-	movl	(%eax), %eax ;						\
-	testl	apic_isrbit_location + 4 + 8 * (irq_num), %eax ;	\
-	jz	9f ;				/* not active */	\
-	movl	$0, lapic+LA_EOI ;					\
-9:
-
-#else
-#define EOI_IRQ(irq_num)						\
-	testl	$IRQ_BIT(irq_num), lapic+LA_ISR1;			\
-	jz	9f	;			/* not active */	\
-	movl	$0, lapic+LA_EOI;					\
-9:
-#endif
-	
-	
-/*
- * Test to see if the source is currently masked, clear if so.
- */
-#define UNMASK_IRQ(irq_num)					\
-	ICU_LOCK ;				/* into critical reg */	\
-	testl	$IRQ_BIT(irq_num), apic_imen ;				\
-	je	7f ;			/* bit clear, not masked */	\
-	andl	$~IRQ_BIT(irq_num), apic_imen ;/* clear mask bit */	\
-	movl	IOAPICADDR(irq_num), %ecx ;	/* ioapic addr */	\
-	movl	REDIRIDX(irq_num), %eax ;	/* get the index */	\
-	movl	%eax, (%ecx) ;			/* write the index */	\
-	movl	IOAPIC_WINDOW(%ecx), %eax ;	/* current value */	\
-	andl	$~IOART_INTMASK, %eax ;		/* clear the mask */	\
-	movl	%eax, IOAPIC_WINDOW(%ecx) ;	/* new value */		\
-7: ;						/* already unmasked */	\
-	ICU_UNLOCK
-
-/*
- * Test to see whether we are handling an edge or level triggered INT.
- *  Level-triggered INTs have to be unmasked.
- */
-#define UNMASK_LEVEL_IRQ(irq_num)					\
-	testl	$IRQ_BIT(irq_num), apic_pin_trigger ;			\
-	jz	9f ;			/* edge, don't unmask */	\
-	UNMASK_IRQ(irq_num) ;						\
-9:
-
-/*
- * Macros for interrupt entry, call to handler, and exit.
- */
-
-#define	FAST_INTR(irq_num, vec_name)					\
+#define	ISR_VEC(index, vec_name)					\
 	.text ;								\
 	SUPERALIGN_TEXT ;						\
-IDTVEC(vec_name) ;							\
-	PUSH_FRAME ;							\
-	movl	$KDSEL,%eax ;						\
-	mov	%ax,%ds ;						\
-	mov	%ax,%es ;						\
-	movl	$KPSEL,%eax ;						\
-	mov	%ax,%fs ;						\
-	FAKE_MCOUNT(13*4(%esp)) ;					\
-	movl	PCPU(CURTHREAD),%ebx ;					\
-	cmpl	$0,TD_CRITNEST(%ebx) ;					\
-	je	1f ;							\
-;									\
-	movl	$1,PCPU(INT_PENDING) ;					\
-	orl	$IRQ_BIT(irq_num),PCPU(FPENDING) ;			\
-	MASK_LEVEL_IRQ(irq_num) ;					\
-	movl	$0, lapic+LA_EOI ;					\
-	jmp	10f ;							\
-1: ;									\
-	incl	TD_CRITNEST(%ebx) ;					\
-	incl	TD_INTR_NESTING_LEVEL(%ebx) ;				\
-	pushl	intr_unit + (irq_num) * 4 ;				\
-	call	*intr_handler + (irq_num) * 4 ;	/* do the work ASAP */	\
-	addl	$4, %esp ;						\
-	movl	$0, lapic+LA_EOI ;					\
-	lock ; 								\
-	incl	cnt+V_INTR ;	/* book-keeping can wait */		\
-	movl	intr_countp + (irq_num) * 4, %eax ;			\
-	lock ; 								\
-	incl	(%eax) ;						\
-	decl	TD_CRITNEST(%ebx) ;					\
-	cmpl	$0,PCPU(INT_PENDING) ;					\
-	je	2f ;							\
-;									\
-	call	i386_unpend ;						\
-2: ;									\
-	decl	TD_INTR_NESTING_LEVEL(%ebx) ;				\
-10: ;									\
-	MEXITCOUNT ;							\
-	jmp	doreti
-
-/*
- * Restart a fast interrupt that was held up by a critical section.
- * This routine is called from unpend().  unpend() ensures we are
- * in a critical section and deals with the interrupt nesting level
- * for us.  If we previously masked the irq, we have to unmask it.
- *
- * We have a choice.  We can regenerate the irq using the 'int'
- * instruction or we can create a dummy frame and call the interrupt
- * handler directly.  I've chosen to use the dummy-frame method.
- */
-#define	FAST_UNPEND(irq_num, vec_name)					\
-	.text ;								\
-	SUPERALIGN_TEXT ;						\
-IDTVEC(vec_name) ;							\
-;									\
-	pushl	%ebp ;							\
-	movl	%esp, %ebp ;						\
-	PUSH_DUMMY ;							\
-	pushl	intr_unit + (irq_num) * 4 ;				\
-	call	*intr_handler + (irq_num) * 4 ;	/* do the work ASAP */	\
-	addl	$4, %esp ;						\
-	lock ; 								\
-	incl	cnt+V_INTR ;	/* book-keeping can wait */		\
-	movl	intr_countp + (irq_num) * 4, %eax ;			\
-	lock ; 								\
-	incl	(%eax) ;						\
-	UNMASK_LEVEL_IRQ(irq_num) ;					\
-	POP_DUMMY ;							\
-	popl %ebp ;							\
-	ret ;								\
-
-
-/* 
- * Slow, threaded interrupts.
- *
- * XXX Most of the parameters here are obsolete.  Fix this when we're
- * done.
- * XXX we really shouldn't return via doreti if we just schedule the
- * interrupt handler and don't run anything.  We could just do an
- * iret.  FIXME.
- */
-#define	INTR(irq_num, vec_name, maybe_extra_ipending)			\
-	.text ;								\
-	SUPERALIGN_TEXT ;						\
-/* _XintrNN: entry point used by IDT/HWIs via _vec[]. */		\
 IDTVEC(vec_name) ;							\
 	PUSH_FRAME ;							\
 	movl	$KDSEL, %eax ;	/* reload with kernel's data segment */	\
 	mov	%ax, %ds ;						\
 	mov	%ax, %es ;						\
-	movl	$KPSEL, %eax ;						\
+	movl	$KPSEL, %eax ;	/* reload with per-CPU data segment */	\
 	mov	%ax, %fs ;						\
-;									\
-	maybe_extra_ipending ;						\
-;									\
-	MASK_LEVEL_IRQ(irq_num) ;					\
-	EOI_IRQ(irq_num) ;						\
-;									\
-	movl	PCPU(CURTHREAD),%ebx ;					\
-	cmpl	$0,TD_CRITNEST(%ebx) ;					\
-	je	1f ;							\
-	movl	$1,PCPU(INT_PENDING) ;					\
-	orl	$IRQ_BIT(irq_num),PCPU(IPENDING) ;			\
-	jmp	10f ;							\
-1: ;									\
+	movl	lapic, %edx ;	/* pointer to local APIC */		\
+	movl	PCPU(CURTHREAD), %ebx ;					\
+	movl	LA_ISR + 16 * (index)(%edx), %eax ;	/* load ISR */	\
 	incl	TD_INTR_NESTING_LEVEL(%ebx) ;				\
-;	 								\
-	FAKE_MCOUNT(13*4(%esp)) ;		/* XXX avoid dbl cnt */ \
-	cmpl	$0,PCPU(INT_PENDING) ;					\
-	je	9f ;							\
-	call	i386_unpend ;						\
-9: ;									\
-	pushl	$irq_num;			/* pass the IRQ */	\
-	call	sched_ithd ;						\
-	addl	$4, %esp ;		/* discard the parameter */	\
-;									\
+	bsrl	%eax, %eax ;	/* index of highset set bit in ISR */	\
+	jz	2f ;							\
+	addl	$(32 * index),%eax ;					\
+1: ;									\
+	FAKE_MCOUNT(13*4(%esp)) ;	/* XXX avoid double count */	\
+	pushl	%eax ;		/* pass the IRQ */			\
+	call	lapic_handle_intr ;					\
+	addl	$4, %esp ;	/* discard parameter */			\
 	decl	TD_INTR_NESTING_LEVEL(%ebx) ;				\
-10: ;									\
 	MEXITCOUNT ;							\
-	jmp	doreti
+	jmp	doreti ;						\
+2:	movl	$-1, %eax ;	/* send a vector of -1 */		\
+	jmp	1b
 
 /*
  * Handle "spurious INTerrupts".
@@ -257,6 +114,14 @@ IDTVEC(spuriousint)
 
 	iret
 
+MCOUNT_LABEL(bintr2)
+	ISR_VEC(1,apic_isr1)
+	ISR_VEC(2,apic_isr2)
+	ISR_VEC(3,apic_isr3)
+	ISR_VEC(4,apic_isr4)
+	ISR_VEC(5,apic_isr5)
+MCOUNT_LABEL(eintr2)
+
 #ifdef SMP
 /*
  * Global address space TLB shootdown.
@@ -281,7 +146,8 @@ IDTVEC(invltlb)
 	movl	%cr3, %eax		/* invalidate the TLB */
 	movl	%eax, %cr3
 
-	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
+	movl	lapic, %eax
+	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
 
 	lock
 	incl	smp_tlb_wait
@@ -313,7 +179,8 @@ IDTVEC(invlpg)
 	movl	smp_tlb_addr1, %eax
 	invlpg	(%eax)			/* invalidate single page */
 
-	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
+	movl	lapic, %eax
+	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
 
 	lock
 	incl	smp_tlb_wait
@@ -350,7 +217,8 @@ IDTVEC(invlrng)
 	cmpl	%eax, %edx
 	jb	1b
 
-	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
+	movl	lapic, %eax
+	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
 
 	lock
 	incl	smp_tlb_wait
@@ -374,21 +242,15 @@ IDTVEC(hardclock)
 	movl	$KPSEL, %eax
 	mov	%ax, %fs
 
-	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
+	movl	lapic, %edx
+	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
 
 	movl	PCPU(CURTHREAD),%ebx
-	cmpl	$0,TD_CRITNEST(%ebx)
-	je	1f
-	movl	$1,PCPU(INT_PENDING)
-	orl	$1,PCPU(SPENDING);
-	jmp	10f
-1:
 	incl	TD_INTR_NESTING_LEVEL(%ebx)
 	pushl	$0		/* XXX convert trapframe to clockframe */
 	call	forwarded_hardclock
 	addl	$4, %esp	/* XXX convert clockframe to trapframe */
 	decl	TD_INTR_NESTING_LEVEL(%ebx)
-10:
 	MEXITCOUNT
 	jmp	doreti
 
@@ -406,23 +268,17 @@ IDTVEC(statclock)
 	movl	$KPSEL, %eax
 	mov	%ax, %fs
 
-	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
+	movl	lapic, %edx
+	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
 
 	FAKE_MCOUNT(13*4(%esp))
 
 	movl	PCPU(CURTHREAD),%ebx
-	cmpl	$0,TD_CRITNEST(%ebx)
-	je	1f
-	movl	$1,PCPU(INT_PENDING)
-	orl	$2,PCPU(SPENDING);
-	jmp	10f
-1:
 	incl	TD_INTR_NESTING_LEVEL(%ebx)
 	pushl	$0		/* XXX convert trapframe to clockframe */
 	call	forwarded_statclock
 	addl	$4, %esp	/* XXX convert clockframe to trapframe */
 	decl	TD_INTR_NESTING_LEVEL(%ebx)
-10:
 	MEXITCOUNT
 	jmp	doreti
 
@@ -444,7 +300,8 @@ IDTVEC(cpuast)
 	movl	$KPSEL, %eax
 	mov	%ax, %fs
 
-	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
+	movl	lapic, %edx
+	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
 
 	FAKE_MCOUNT(13*4(%esp))
 
@@ -476,7 +333,8 @@ IDTVEC(cpustop)
 	movl	$KPSEL, %eax
 	mov	%ax, %fs
 
-	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
+	movl	lapic, %eax
+	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
 
 	movl	PCPU(CPUID), %eax
 	imull	$PCB_SIZE, %eax
@@ -518,111 +376,6 @@ IDTVEC(cpustop)
 	popl	%ebp
 	iret
 
-#endif /* SMP */
-
-MCOUNT_LABEL(bintr)
-	FAST_INTR(0,fastintr0)
-	FAST_INTR(1,fastintr1)
-	FAST_INTR(2,fastintr2)
-	FAST_INTR(3,fastintr3)
-	FAST_INTR(4,fastintr4)
-	FAST_INTR(5,fastintr5)
-	FAST_INTR(6,fastintr6)
-	FAST_INTR(7,fastintr7)
-	FAST_INTR(8,fastintr8)
-	FAST_INTR(9,fastintr9)
-	FAST_INTR(10,fastintr10)
-	FAST_INTR(11,fastintr11)
-	FAST_INTR(12,fastintr12)
-	FAST_INTR(13,fastintr13)
-	FAST_INTR(14,fastintr14)
-	FAST_INTR(15,fastintr15)
-	FAST_INTR(16,fastintr16)
-	FAST_INTR(17,fastintr17)
-	FAST_INTR(18,fastintr18)
-	FAST_INTR(19,fastintr19)
-	FAST_INTR(20,fastintr20)
-	FAST_INTR(21,fastintr21)
-	FAST_INTR(22,fastintr22)
-	FAST_INTR(23,fastintr23)
-	FAST_INTR(24,fastintr24)
-	FAST_INTR(25,fastintr25)
-	FAST_INTR(26,fastintr26)
-	FAST_INTR(27,fastintr27)
-	FAST_INTR(28,fastintr28)
-	FAST_INTR(29,fastintr29)
-	FAST_INTR(30,fastintr30)
-	FAST_INTR(31,fastintr31)
-#define	CLKINTR_PENDING	movl $1,CNAME(clkintr_pending)
-/* Threaded interrupts */
-	INTR(0,intr0, CLKINTR_PENDING)
-	INTR(1,intr1,)
-	INTR(2,intr2,)
-	INTR(3,intr3,)
-	INTR(4,intr4,)
-	INTR(5,intr5,)
-	INTR(6,intr6,)
-	INTR(7,intr7,)
-	INTR(8,intr8,)
-	INTR(9,intr9,)
-	INTR(10,intr10,)
-	INTR(11,intr11,)
-	INTR(12,intr12,)
-	INTR(13,intr13,)
-	INTR(14,intr14,)
-	INTR(15,intr15,)
-	INTR(16,intr16,)
-	INTR(17,intr17,)
-	INTR(18,intr18,)
-	INTR(19,intr19,)
-	INTR(20,intr20,)
-	INTR(21,intr21,)
-	INTR(22,intr22,)
-	INTR(23,intr23,)
-	INTR(24,intr24,)
-	INTR(25,intr25,)
-	INTR(26,intr26,)
-	INTR(27,intr27,)
-	INTR(28,intr28,)
-	INTR(29,intr29,)
-	INTR(30,intr30,)
-	INTR(31,intr31,)
-
-	FAST_UNPEND(0,fastunpend0)
-	FAST_UNPEND(1,fastunpend1)
-	FAST_UNPEND(2,fastunpend2)
-	FAST_UNPEND(3,fastunpend3)
-	FAST_UNPEND(4,fastunpend4)
-	FAST_UNPEND(5,fastunpend5)
-	FAST_UNPEND(6,fastunpend6)
-	FAST_UNPEND(7,fastunpend7)
-	FAST_UNPEND(8,fastunpend8)
-	FAST_UNPEND(9,fastunpend9)
-	FAST_UNPEND(10,fastunpend10)
-	FAST_UNPEND(11,fastunpend11)
-	FAST_UNPEND(12,fastunpend12)
-	FAST_UNPEND(13,fastunpend13)
-	FAST_UNPEND(14,fastunpend14)
-	FAST_UNPEND(15,fastunpend15)
-	FAST_UNPEND(16,fastunpend16)
-	FAST_UNPEND(17,fastunpend17)
-	FAST_UNPEND(18,fastunpend18)
-	FAST_UNPEND(19,fastunpend19)
-	FAST_UNPEND(20,fastunpend20)
-	FAST_UNPEND(21,fastunpend21)
-	FAST_UNPEND(22,fastunpend22)
-	FAST_UNPEND(23,fastunpend23)
-	FAST_UNPEND(24,fastunpend24)
-	FAST_UNPEND(25,fastunpend25)
-	FAST_UNPEND(26,fastunpend26)
-	FAST_UNPEND(27,fastunpend27)
-	FAST_UNPEND(28,fastunpend28)
-	FAST_UNPEND(29,fastunpend29)
-	FAST_UNPEND(30,fastunpend30)
-	FAST_UNPEND(31,fastunpend31)
-MCOUNT_LABEL(eintr)
-
-#ifdef SMP
 /*
  * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
  *
@@ -640,7 +393,8 @@ IDTVEC(rendezvous)
 
 	call	smp_rendezvous_action
 
-	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
+	movl	lapic, %eax
+	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
 	POP_FRAME
 	iret
 	
@@ -658,16 +412,9 @@ IDTVEC(lazypmap)
 	mov	%ax, %fs
 
 	call	pmap_lazyfix_action
-	
-	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
+
+	movl	lapic, %eax	
+	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
 	POP_FRAME
 	iret
 #endif /* SMP */
-
-	.data
-
-	.globl	apic_pin_trigger
-apic_pin_trigger:
-	.long	0
-
-	.text

sys/amd64/amd64/io_apic.c

@@ -0,0 +1,691 @@
+/*-
+ * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the author nor the names of any co-contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_isa.h"
+#include "opt_no_mixed_mode.h"
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/apicreg.h>
+#include <machine/frame.h>
+#include <machine/intr_machdep.h>
+#include <machine/apicvar.h>
+#include <machine/segments.h>
+
+#if defined(DEV_ISA) && !defined(NO_MIXED_MODE)
+#define MIXED_MODE
+#endif
+
+#define IOAPIC_ISA_INTS		16
+#define	IOAPIC_MEM_REGION	32
+#define	IOAPIC_REDTBL_LO(i)	(IOAPIC_REDTBL + (i) * 2)
+#define	IOAPIC_REDTBL_HI(i)	(IOAPIC_REDTBL_LO(i) + 1)
+
+#define	VECTOR_EXTINT		-1
+#define	VECTOR_NMI		-2
+#define	VECTOR_SMI		-3
+#define	VECTOR_DISABLED		-4
+
+#define	DEST_NONE		-1
+#define	DEST_EXTINT		-2
+
+#define	TODO		printf("%s: not implemented!\n", __func__)
+
+MALLOC_DEFINE(M_IOAPIC, "I/O APIC", "I/O APIC structures");
+
+/*
+ * New interrupt support code..
+ *
+ * XXX: we really should have the interrupt cookie passed up from new-bus
+ * just be a int pin, and not map 1:1 to interrupt vector number but should
+ * use INTR_TYPE_FOO to set priority bands for device classes and do all the
+ * magic remapping of intpin to vector in here.  For now we just cheat as on
+ * ia64 and map intpin X to vector NRSVIDT + X.  Note that we assume that the
+ * first IO APIC has ISA interrupts on pins 1-15.  Not sure how you are
+ * really supposed to figure out which IO APIC in a system with multiple IO
+ * APIC's actually has the ISA interrupts routed to it.  As far as interrupt
+ * pin numbers, we use the ACPI System Interrupt number model where each
+ * IO APIC has a contiguous chunk of the System Interrupt address space.
+ */
+
+/*
+ * Direct the ExtINT pin on the first I/O APIC to a logical cluster of
+ * CPUs rather than a physical destination of just the BSP.
+ *
+ * Note: This is disabled by default as test systems seem to croak with it
+ * enabled.
+#define ENABLE_EXTINT_LOGICAL_DESTINATION
+ */
+
+struct ioapic_intsrc {
+	struct intsrc io_intsrc;
+	int io_intpin:8;
+	int io_vector:8;
+	int io_activehi:1;
+	int io_edgetrigger:1;
+	int io_masked:1;
+	int io_dest:5;
+};
+
+struct ioapic {
+	struct pic io_pic;
+	u_int io_id:8;			/* logical ID */
+	u_int io_apic_id:4;
+	u_int io_intbase:8;		/* System Interrupt base */
+	u_int io_numintr:8;
+	volatile ioapic_t *io_addr;	/* XXX: should use bus_space */
+	STAILQ_ENTRY(ioapic) io_next;
+	struct ioapic_intsrc io_pins[0];
+};
+
+static STAILQ_HEAD(,ioapic) ioapic_list = STAILQ_HEAD_INITIALIZER(ioapic_list);
+static u_int next_id, program_logical_dest;
+
+static u_int	ioapic_read(volatile ioapic_t *apic, int reg);
+static void	ioapic_write(volatile ioapic_t *apic, int reg, u_int val);
+static void	ioapic_enable_source(struct intsrc *isrc);
+static void	ioapic_disable_source(struct intsrc *isrc);
+static void	ioapic_eoi_source(struct intsrc *isrc);
+static void	ioapic_enable_intr(struct intsrc *isrc);
+static int	ioapic_vector(struct intsrc *isrc);
+static int	ioapic_source_pending(struct intsrc *isrc);
+static void	ioapic_suspend(struct intsrc *isrc);
+static void	ioapic_resume(struct intsrc *isrc);
+static void	ioapic_program_destination(struct ioapic_intsrc *intpin);
+#ifdef MIXED_MODE
+static void	ioapic_setup_mixed_mode(struct ioapic_intsrc *intpin);
+#endif
+
+struct pic ioapic_template = { ioapic_enable_source, ioapic_disable_source,
+			       ioapic_eoi_source, ioapic_enable_intr,
+			       ioapic_vector, ioapic_source_pending,
+			       ioapic_suspend, ioapic_resume };
+	
+static int next_ioapic_base, logical_clusters, current_cluster;
+
+static u_int
+ioapic_read(volatile ioapic_t *apic, int reg)
+{
+
+	mtx_assert(&icu_lock, MA_OWNED);
+	apic->ioregsel = reg;
+	return (apic->iowin);
+}
+
+static void
+ioapic_write(volatile ioapic_t *apic, int reg, u_int val)
+{
+
+	mtx_assert(&icu_lock, MA_OWNED);
+	apic->ioregsel = reg;
+	apic->iowin = val;
+}
+
+static void
+ioapic_enable_source(struct intsrc *isrc)
+{
+	struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc;
+	struct ioapic *io = (struct ioapic *)isrc->is_pic;
+	uint32_t flags;
+
+	mtx_lock_spin(&icu_lock);
+	if (intpin->io_masked) {
+		flags = ioapic_read(io->io_addr,
+		    IOAPIC_REDTBL_LO(intpin->io_intpin));
+		flags &= ~(IOART_INTMASK);
+		ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin),
+		    flags);
+		intpin->io_masked = 0;
+	}
+	mtx_unlock_spin(&icu_lock);
+}
+
+static void
+ioapic_disable_source(struct intsrc *isrc)
+{
+	struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc;
+	struct ioapic *io = (struct ioapic *)isrc->is_pic;
+	uint32_t flags;
+
+	mtx_lock_spin(&icu_lock);
+	if (!intpin->io_masked && !intpin->io_edgetrigger) {
+		flags = ioapic_read(io->io_addr,
+		    IOAPIC_REDTBL_LO(intpin->io_intpin));
+		flags |= IOART_INTMSET;
+		ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin),
+		    flags);
+		intpin->io_masked = 1;
+	}
+	mtx_unlock_spin(&icu_lock);
+}
+
+static void
+ioapic_eoi_source(struct intsrc *isrc)
+{
+	TODO;
+	/* lapic_eoi(); */
+}
+
+/*
+ * Program an individual intpin's logical destination.
+ */
+static void
+ioapic_program_destination(struct ioapic_intsrc *intpin)
+{
+	struct ioapic *io = (struct ioapic *)intpin->io_intsrc.is_pic;
+	uint32_t value;
+
+	KASSERT(intpin->io_dest != DEST_NONE,
+	    ("intpin not assigned to a cluster"));
+	KASSERT(intpin->io_dest != DEST_EXTINT,
+	    ("intpin routed via ExtINT"));
+	if (bootverbose) {
+		printf("ioapic%u: routing intpin %u (", io->io_id,
+		    intpin->io_intpin);
+		if (intpin->io_vector == VECTOR_EXTINT)
+			printf("ExtINT");
+		else
+			printf("IRQ %u", intpin->io_vector);
+		printf(") to cluster %u\n", intpin->io_dest);
+	}
+	mtx_lock_spin(&icu_lock);
+	value = ioapic_read(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin));
+	value &= ~IOART_DESTMOD;
+	value |= IOART_DESTLOG;
+	ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin), value);
+	value = ioapic_read(io->io_addr, IOAPIC_REDTBL_HI(intpin->io_intpin));
+	value &= ~IOART_DEST;
+	value |= (intpin->io_dest << APIC_ID_CLUSTER_SHIFT |
+	    APIC_ID_CLUSTER_ID) << APIC_ID_SHIFT;
+	ioapic_write(io->io_addr, IOAPIC_REDTBL_HI(intpin->io_intpin), value);
+	mtx_unlock_spin(&icu_lock);
+}
+
+static void
+ioapic_assign_cluster(struct ioapic_intsrc *intpin)
+{
+	/*
+	 * Assign this intpin to a logical APIC cluster in a
+	 * round-robin fashion.  We don't actually use the logical
+	 * destination for this intpin until after all the CPU's
+	 * have been started so that we don't end up with interrupts
+	 * that don't go anywhere.  Another alternative might be to
+	 * start up the CPU's earlier so that they can handle interrupts
+	 * sooner.
+	 */
+	intpin->io_dest = current_cluster;
+	current_cluster++;
+	if (current_cluster >= logical_clusters)
+		current_cluster = 0;
+	if (program_logical_dest)
+		ioapic_program_destination(intpin);
+}
+
+static void
+ioapic_enable_intr(struct intsrc *isrc)
+{
+	struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc;
+
+	KASSERT(intpin->io_dest != DEST_EXTINT,
+	    ("ExtINT pin trying to use ioapic enable_intr method"));
+	if (intpin->io_dest == DEST_NONE) {
+		ioapic_assign_cluster(intpin);
+		lapic_enable_intr(intpin->io_vector);
+	}
+}
+
+static int
+ioapic_vector(struct intsrc *isrc)
+{
+	struct ioapic_intsrc *pin;
+
+	pin = (struct ioapic_intsrc *)isrc;
+	return (pin->io_vector);
+}
+
+static int
+ioapic_source_pending(struct intsrc *isrc)
+{
+	struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc;
+
+	return (lapic_intr_pending(intpin->io_vector));
+}
+
+static void
+ioapic_suspend(struct intsrc *isrc)
+{
+
+	TODO;
+}
+
+static void
+ioapic_resume(struct intsrc *isrc)
+{
+
+	TODO;
+}
+
+/*
+ * Allocate and return a logical cluster ID.  Note that the first time
+ * this is called, it returns cluster 0.  ioapic_enable_intr() treats
+ * the two cases of logical_clusters == 0 and logical_clusters == 1 the
+ * same: one cluster of ID 0 exists.  The logical_clusters == 0 case is
+ * for UP kernels, which should never call this function.
+ */
+int
+ioapic_next_logical_cluster(void)
+{
+
+	if (logical_clusters >= APIC_MAX_CLUSTER)
+		panic("WARNING: Local APIC cluster IDs exhausted!");
+	return (logical_clusters++);
+}
+
+/*
+ * Create a plain I/O APIC object.
+ */
+void *
+ioapic_create(uintptr_t addr, int32_t apic_id, int intbase)
+{
+	struct ioapic *io;
+	struct ioapic_intsrc *intpin;
+	volatile ioapic_t *apic;
+	u_int numintr, i;
+	uint32_t value;
+
+	apic = (ioapic_t *)pmap_mapdev(addr, IOAPIC_MEM_REGION);
+	mtx_lock_spin(&icu_lock);
+	numintr = ((ioapic_read(apic, IOAPIC_VER) & IOART_VER_MAXREDIR) >>
+	    MAXREDIRSHIFT) + 1;
+	mtx_unlock_spin(&icu_lock);
+	io = malloc(sizeof(struct ioapic) +
+	    numintr * sizeof(struct ioapic_intsrc), M_IOAPIC, M_WAITOK);
+	io->io_pic = ioapic_template;
+	mtx_lock_spin(&icu_lock);
+	io->io_id = next_id++;
+	io->io_apic_id = ioapic_read(apic, IOAPIC_ID) >> APIC_ID_SHIFT;	
+	if (apic_id != -1 && io->io_apic_id != apic_id) {
+		ioapic_write(apic, IOAPIC_ID, apic_id << APIC_ID_SHIFT);
+		mtx_unlock_spin(&icu_lock);
+		io->io_apic_id = apic_id;
+		printf("ioapic%u: Changing APIC ID to %d\n", io->io_id,
+		    apic_id);
+	} else
+		mtx_unlock_spin(&icu_lock);
+	if (intbase == -1) {
+		intbase = next_ioapic_base;
+		printf("ioapic%u: Assuming intbase of %d\n", io->io_id,
+		    intbase);
+	} else if (intbase != next_ioapic_base)
+		printf("ioapic%u: WARNING: intbase %d != expected base %d\n",
+		    io->io_id, intbase, next_ioapic_base);
+	io->io_intbase = intbase;
+	next_ioapic_base += numintr;
+	io->io_numintr = numintr;
+	io->io_addr = apic;
+
+	/*
+	 * Initialize pins.  Start off with interrupts disabled.  Default
+	 * to active-hi and edge-triggered for ISA interrupts and active-lo
+	 * and level-triggered for all others.
+	 */
+	bzero(io->io_pins, sizeof(struct ioapic_intsrc) * numintr);
+	mtx_lock_spin(&icu_lock);
+	for (i = 0, intpin = io->io_pins; i < numintr; i++, intpin++) {
+		intpin->io_intsrc.is_pic = (struct pic *)io;
+		intpin->io_intpin = i;
+		intpin->io_vector = intbase + i;
+
+		/*
+		 * Assume that pin 0 on the first IO APIC is an ExtINT pin by
+		 * default.  Assume that intpins 1-15 are ISA interrupts and
+		 * use suitable defaults for those.  Assume that all other
+		 * intpins are PCI interrupts.  Enable the ExtINT pin by
+		 * default but mask all other pins.
+		 */
+		if (intpin->io_vector == 0) {
+			intpin->io_activehi = 1;
+			intpin->io_edgetrigger = 1;
+			intpin->io_vector = VECTOR_EXTINT;
+			intpin->io_masked = 0;
+		} else if (intpin->io_vector < IOAPIC_ISA_INTS) {
+			intpin->io_activehi = 1;
+			intpin->io_edgetrigger = 1;
+			intpin->io_masked = 1;
+		} else {
+			intpin->io_activehi = 0;
+			intpin->io_edgetrigger = 0;
+			intpin->io_masked = 1;
+		}
+
+		/*
+		 * Start off without a logical cluster destination until
+		 * the pin is enabled.
+		 */
+		intpin->io_dest = DEST_NONE;
+		if (bootverbose) {
+			printf("ioapic%u: intpin %d -> ",  io->io_id, i);
+			if (intpin->io_vector == VECTOR_EXTINT)
+				printf("ExtINT\n");
+			else
+				printf("irq %d\n", intpin->io_vector);
+		}
+		value = ioapic_read(apic, IOAPIC_REDTBL_LO(i));
+		ioapic_write(apic, IOAPIC_REDTBL_LO(i), value | IOART_INTMSET);
+	}
+	mtx_unlock_spin(&icu_lock);
+
+	return (io);
+}
+
+int
+ioapic_get_vector(void *cookie, u_int pin)
+{
+	struct ioapic *io;
+
+	io = (struct ioapic *)cookie;
+	if (pin >= io->io_numintr)
+		return (-1);
+	return (io->io_pins[pin].io_vector);
+}
+
+int
+ioapic_disable_pin(void *cookie, u_int pin)
+{
+	struct ioapic *io;
+
+	io = (struct ioapic *)cookie;
+	if (pin >= io->io_numintr)
+		return (EINVAL);
+	if (io->io_pins[pin].io_vector == VECTOR_DISABLED)
+		return (EINVAL);
+	io->io_pins[pin].io_vector = VECTOR_DISABLED;
+	if (bootverbose)
+		printf("ioapic%u: intpin %d disabled\n", io->io_id, pin);
+	return (0);
+}
+
+int
+ioapic_remap_vector(void *cookie, u_int pin, int vector)
+{
+	struct ioapic *io;
+
+	io = (struct ioapic *)cookie;
+	if (pin >= io->io_numintr || vector < 0)
+		return (EINVAL);
+	if (io->io_pins[pin].io_vector < 0)
+		return (EINVAL);
+	io->io_pins[pin].io_vector = vector;
+	if (bootverbose)
+		printf("ioapic%u: Routing IRQ %d -> intpin %d\n", io->io_id,
+		    vector, pin);
+	return (0);
+}
+
+int
+ioapic_set_nmi(void *cookie, u_int pin)
+{
+	struct ioapic *io;
+
+	io = (struct ioapic *)cookie;
+	if (pin >= io->io_numintr)
+		return (EINVAL);
+	if (io->io_pins[pin].io_vector < 0)
+		return (EINVAL);
+	io->io_pins[pin].io_vector = VECTOR_NMI;
+	io->io_pins[pin].io_masked = 0;
+	io->io_pins[pin].io_edgetrigger = 1;
+	io->io_pins[pin].io_activehi = 1;
+	if (bootverbose)
+		printf("ioapic%u: Routing NMI -> intpin %d\n",
+		    io->io_id, pin);
+	return (0);
+}
+
+int
+ioapic_set_smi(void *cookie, u_int pin)
+{
+	struct ioapic *io;
+
+	io = (struct ioapic *)cookie;
+	if (pin >= io->io_numintr)
+		return (EINVAL);
+	if (io->io_pins[pin].io_vector < 0)
+		return (EINVAL);
+	io->io_pins[pin].io_vector = VECTOR_SMI;
+	io->io_pins[pin].io_masked = 0;
+	io->io_pins[pin].io_edgetrigger = 1;
+	io->io_pins[pin].io_activehi = 1;
+	if (bootverbose)
+		printf("ioapic%u: Routing SMI -> intpin %d\n",
+		    io->io_id, pin);
+	return (0);
+}
+
+int
+ioapic_set_extint(void *cookie, u_int pin)
+{
+	struct ioapic *io;
+
+	io = (struct ioapic *)cookie;
+	if (pin >= io->io_numintr)
+		return (EINVAL);
+	if (io->io_pins[pin].io_vector < 0)
+		return (EINVAL);
+	io->io_pins[pin].io_vector = VECTOR_EXTINT;
+	io->io_pins[pin].io_masked = 0;
+	io->io_pins[pin].io_edgetrigger = 1;
+	io->io_pins[pin].io_activehi = 1;
+	if (bootverbose)
+		printf("ioapic%u: Routing external 8259A's -> intpin %d\n",
+		    io->io_id, pin);
+	return (0);
+}
+
+int
+ioapic_set_polarity(void *cookie, u_int pin, char activehi)
+{
+	struct ioapic *io;
+
+	io = (struct ioapic *)cookie;
+	if (pin >= io->io_numintr)
+		return (EINVAL);
+	if (io->io_pins[pin].io_vector < 0)
+		return (EINVAL);
+	io->io_pins[pin].io_activehi = activehi;
+	if (bootverbose)
+		printf("ioapic%u: intpin %d polarity: %s\n", io->io_id, pin,
+		    activehi ? "active-hi" : "active-lo");
+	return (0);
+}
+
+int
+ioapic_set_triggermode(void *cookie, u_int pin, char edgetrigger)
+{
+	struct ioapic *io;
+
+	io = (struct ioapic *)cookie;
+	if (pin >= io->io_numintr)
+		return (EINVAL);
+	if (io->io_pins[pin].io_vector < 0)
+		return (EINVAL);
+	io->io_pins[pin].io_edgetrigger = edgetrigger;
+	if (bootverbose)
+		printf("ioapic%u: intpin %d trigger: %s\n", io->io_id, pin,
+		    edgetrigger ? "edge" : "level");
+	return (0);
+}
+
+/*
+ * Register a complete I/O APIC object with the interrupt subsystem.
+ */
+void
+ioapic_register(void *cookie)
+{
+	struct ioapic_intsrc *pin;
+	struct ioapic *io;
+	volatile ioapic_t *apic;
+	uint32_t flags;
+	int i;
+
+	io = (struct ioapic *)cookie;
+	apic = io->io_addr;
+	mtx_lock_spin(&icu_lock);
+	flags = ioapic_read(apic, IOAPIC_VER) & IOART_VER_VERSION;
+	STAILQ_INSERT_TAIL(&ioapic_list, io, io_next);
+	mtx_unlock_spin(&icu_lock);
+	printf("ioapic%u <Version %u> irqs %u-%u on motherboard\n", io->io_id,
+	    flags, io->io_intbase, io->io_intbase + io->io_numintr - 1);
+	for (i = 0, pin = io->io_pins; i < io->io_numintr; i++, pin++) {
+		/*
+		 * Finish initializing the pins by programming the vectors
+		 * and delivery mode.
+		 */
+		if (pin->io_vector == VECTOR_DISABLED)
+			continue;
+		flags = IOART_DESTPHY;
+		if (pin->io_edgetrigger)
+			flags |= IOART_TRGREDG;
+		else
+			flags |= IOART_TRGRLVL;
+		if (pin->io_activehi)
+			flags |= IOART_INTAHI;
+		else
+			flags |= IOART_INTALO;
+		if (pin->io_masked)
+			flags |= IOART_INTMSET;
+		switch (pin->io_vector) {
+		case VECTOR_EXTINT:
+			KASSERT(pin->io_edgetrigger,
+			    ("EXTINT not edge triggered"));
+			flags |= IOART_DELEXINT;
+			break;
+		case VECTOR_NMI:
+			KASSERT(pin->io_edgetrigger,
+			    ("NMI not edge triggered"));
+			flags |= IOART_DELNMI;
+			break;
+		case VECTOR_SMI:
+			KASSERT(pin->io_edgetrigger,
+			    ("SMI not edge triggered"));
+			flags |= IOART_DELSMI;
+			break;
+		default:
+			flags |= IOART_DELLOPRI |
+			    apic_irq_to_idt(pin->io_vector);
+		}
+		mtx_lock_spin(&icu_lock);
+		ioapic_write(apic, IOAPIC_REDTBL_LO(i), flags);
+
+		/*
+		 * Route interrupts to the BSP by default using physical
+		 * addressing.  Vectored interrupts get readdressed using
+		 * logical IDs to CPU clusters when they are enabled.
+		 */
+		flags = ioapic_read(apic, IOAPIC_REDTBL_HI(i));
+		flags &= ~IOART_DEST;
+		flags |= PCPU_GET(apic_id) << APIC_ID_SHIFT;
+		ioapic_write(apic, IOAPIC_REDTBL_HI(i), flags);
+		mtx_unlock_spin(&icu_lock);
+		if (pin->io_vector >= 0) {
+#ifdef MIXED_MODE
+			/* Route IRQ0 via the 8259A using mixed mode. */
+			if (pin->io_vector == 0)
+				ioapic_setup_mixed_mode(pin);
+			else
+#endif
+				intr_register_source(&pin->io_intsrc);
+		}
+			
+	}
+}
+
+/*
+ * Program all the intpins to use logical destinations once the AP's
+ * have been launched.
+ */
+static void
+ioapic_set_logical_destinations(void *arg __unused)
+{
+	struct ioapic *io;
+	int i;
+
+	program_logical_dest = 1;
+	STAILQ_FOREACH(io, &ioapic_list, io_next)
+	    for (i = 0; i < io->io_numintr; i++)
+		    if (io->io_pins[i].io_dest != DEST_NONE &&
+			io->io_pins[i].io_dest != DEST_EXTINT)
+			    ioapic_program_destination(&io->io_pins[i]);
+}
+SYSINIT(ioapic_destinations, SI_SUB_SMP, SI_ORDER_SECOND,
+    ioapic_set_logical_destinations, NULL)
+
+#ifdef MIXED_MODE
+/*
+ * Support for mixed-mode interrupt sources.  These sources route an ISA
+ * IRQ through the 8259A's via the ExtINT on pin 0 of the I/O APIC that
+ * routes the ISA interrupts.  We just ignore the intpins that use this
+ * mode and allow the atpic driver to register its interrupt source for
+ * that IRQ instead.
+ */
+
+void
+ioapic_setup_mixed_mode(struct ioapic_intsrc *intpin)
+{
+	struct ioapic_intsrc *extint;
+	struct ioapic *io;
+
+	/*
+	 * Mark the associated I/O APIC intpin as being delivered via
+	 * ExtINT and enable the ExtINT pin on the I/O APIC if needed.
+	 */
+	intpin->io_dest = DEST_EXTINT;
+	io = (struct ioapic *)intpin->io_intsrc.is_pic;
+	extint = &io->io_pins[0];
+	if (extint->io_vector != VECTOR_EXTINT)
+		panic("Can't find ExtINT pin to route through!");
+#ifdef ENABLE_EXTINT_LOGICAL_DESTINATION
+	if (extint->io_dest == DEST_NONE)
+		ioapic_assign_cluster(extint);
+#endif
+}
+
+#endif /* MIXED_MODE */

sys/amd64/amd64/local_apic.c

@@ -0,0 +1,758 @@
+/*-
+ * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org>
+ * Copyright (c) 1996, by Steve Passe
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. The name of the developer may NOT be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ * 3. Neither the name of the author nor the names of any co-contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Local APIC support on Pentium and later processors.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/pcpu.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/apicreg.h>
+#include <machine/cputypes.h>
+#include <machine/frame.h>
+#include <machine/intr_machdep.h>
+#include <machine/apicvar.h>
+#include <machine/md_var.h>
+#include <machine/smp.h>
+#include <machine/specialreg.h>
+
+/*
+ * We can handle up to 60 APICs via our logical cluster IDs, but currently
+ * the physical IDs on Intel processors up to the Pentium 4 are limited to
+ * 16.
+ */
+#define	MAX_APICID	16
+
+/*
+ * Support for local APICs.  Local APICs manage interrupts on each
+ * individual processor as opposed to I/O APICs which receive interrupts
+ * from I/O devices and then forward them on to the local APICs.
+ *
+ * Local APICs can also send interrupts to each other thus providing the
+ * mechanism for IPIs.
+ */
+
+struct lvt {
+	u_int lvt_edgetrigger:1;
+	u_int lvt_activehi:1;
+	u_int lvt_masked:1;
+	u_int lvt_active:1;
+	u_int lvt_mode:16;
+	u_int lvt_vector:8;
+};
+
+struct lapic {
+	struct lvt la_lvts[LVT_MAX + 1];
+	u_int la_id:8;
+	u_int la_cluster:4;
+	u_int la_cluster_id:2;
+	u_int la_present:1;
+} static lapics[MAX_APICID];
+
+/* XXX: should thermal be an NMI? */
+
+/* Global defaults for local APIC LVT entries. */
+static struct lvt lvts[LVT_MAX + 1] = {
+	{ 1, 1, 1, 1, APIC_LVT_DM_EXTINT, 0 },	/* LINT0: masked ExtINT */
+	{ 1, 1, 0, 1, APIC_LVT_DM_NMI, 0 },	/* LINT1: NMI */
+	{ 1, 1, 1, 1, APIC_LVT_DM_FIXED, 0 },	/* Timer: needs a vector */
+	{ 1, 1, 1, 1, APIC_LVT_DM_FIXED, 0 },	/* Error: needs a vector */
+	{ 1, 1, 1, 1, APIC_LVT_DM_FIXED, 0 },	/* PMC */
+	{ 1, 1, 1, 1, APIC_LVT_DM_FIXED, 0 },	/* Thermal: needs a vector */
+};
+
+static inthand_t *ioint_handlers[] = {
+	NULL,			/* 0 - 31 */
+	IDTVEC(apic_isr1),	/* 32 - 63 */
+	IDTVEC(apic_isr2),	/* 64 - 95 */
+	IDTVEC(apic_isr3),	/* 96 - 127 */
+	IDTVEC(apic_isr4),	/* 128 - 159 */
+	IDTVEC(apic_isr5),	/* 160 - 191 */
+	NULL,			/* 192 - 223 */
+	NULL			/* 224 - 255 */
+};
+
+volatile lapic_t *lapic;
+
+static uint32_t
+lvt_mode(struct lapic *la, u_int pin, uint32_t value)
+{
+	struct lvt *lvt;
+
+	KASSERT(pin <= LVT_MAX, ("%s: pin %u out of range", __func__, pin));
+	if (la->la_lvts[pin].lvt_active)
+		lvt = &la->la_lvts[pin];
+	else
+		lvt = &lvts[pin];
+
+	value &= ~(APIC_LVT_M | APIC_LVT_TM | APIC_LVT_IIPP | APIC_LVT_DM |
+	    APIC_LVT_VECTOR);
+	if (lvt->lvt_edgetrigger == 0)
+		value |= APIC_LVT_TM;
+	if (lvt->lvt_activehi == 0)
+		value |= APIC_LVT_IIPP_INTALO;
+	if (lvt->lvt_masked)
+		value |= APIC_LVT_M;
+	value |= lvt->lvt_mode;
+	switch (lvt->lvt_mode) {
+	case APIC_LVT_DM_NMI:
+	case APIC_LVT_DM_SMI:
+	case APIC_LVT_DM_INIT:
+	case APIC_LVT_DM_EXTINT:
+		if (!lvt->lvt_edgetrigger) {
+			printf("lapic%u: Forcing LINT%u to edge trigger\n",
+			    la->la_id, pin);
+			value |= APIC_LVT_TM;
+		}
+		/* Use a vector of 0. */
+		break;
+	case APIC_LVT_DM_FIXED:
+#if 0
+		value |= lvt->lvt_vector;
+#else
+		panic("Fixed LINT pins not supported");
+#endif
+		break;
+	default:
+		panic("bad APIC LVT delivery mode: %#x\n", value);
+	}
+	return (value);
+}
+
+/*
+ * Map the local APIC and setup necessary interrupt vectors.
+ */
+void
+lapic_init(uintptr_t addr)
+{
+	u_int32_t value;
+
+	/* Map the local APIC and setup the spurious interrupt handler. */
+	KASSERT(trunc_page(addr) == addr,
+	    ("local APIC not aligned on a page boundary"));
+	lapic = (lapic_t *)pmap_mapdev(addr, sizeof(lapic_t));
+	setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_SYS386IGT, SEL_KPL,
+	    GSEL(GCODE_SEL, SEL_KPL));
+
+	/* Perform basic initialization of the BSP's local APIC. */
+	value = lapic->svr;
+	value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS);
+	value |= (APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT);
+	lapic->svr = value;
+
+	/* Set BSP's per-CPU local APIC ID. */
+	PCPU_SET(apic_id, lapic_id());
+
+	/* XXX: timer/error/thermal interrupts */
+}
+
+/*
+ * Create a local APIC instance.
+ */
+void
+lapic_create(u_int apic_id, int boot_cpu)
+{
+	int i;
+
+	if (apic_id > MAX_APICID) {
+		printf("APIC: Ignoring local APIC with ID %d\n", apic_id);
+		if (boot_cpu)
+			panic("Can't ignore BSP");
+		return;
+	}
+	KASSERT(!lapics[apic_id].la_present, ("duplicate local APIC %u",
+	    apic_id));
+
+	/*
+	 * Assume no local LVT overrides and a cluster of 0 and
+	 * intra-cluster ID of 0.
+	 */
+	lapics[apic_id].la_present = 1;
+	lapics[apic_id].la_id = apic_id;
+	for (i = 0; i < LVT_MAX; i++) {
+		lapics[apic_id].la_lvts[i] = lvts[i];
+		lapics[apic_id].la_lvts[i].lvt_active = 0;
+	}
+
+#ifdef SMP
+	cpu_add(apic_id, boot_cpu);
+#endif
+}
+
+/*
+ * Dump contents of local APIC registers
+ */
+void
+lapic_dump(const char* str)
+{
+
+	printf("cpu%d %s:\n", PCPU_GET(cpuid), str);
+	printf("     ID: 0x%08x   VER: 0x%08x LDR: 0x%08x DFR: 0x%08x\n",
+	    lapic->id, lapic->version, lapic->ldr, lapic->dfr);
+	printf("  lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n",
+	    lapic->lvt_lint0, lapic->lvt_lint1, lapic->tpr, lapic->svr);
+}
+
+void
+lapic_enable_intr(u_int irq)
+{
+	u_int vector;
+
+	vector = apic_irq_to_idt(irq);
+	KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
+	KASSERT(ioint_handlers[vector / 32] != NULL,
+	    ("No ISR handler for IRQ %u", irq));
+	setidt(vector, ioint_handlers[vector / 32], SDT_SYS386IGT, SEL_KPL,
+	    GSEL(GCODE_SEL, SEL_KPL));
+}
+
+void
+lapic_setup(void)
+{
+	struct lapic *la;
+	u_int32_t value, maxlvt;
+	register_t eflags;
+
+	la = &lapics[lapic_id()];
+	KASSERT(la->la_present, ("missing APIC structure"));
+	eflags = intr_disable();
+	maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
+
+	/* Program LINT[01] LVT entries. */
+	lapic->lvt_lint0 = lvt_mode(la, LVT_LINT0, lapic->lvt_lint0);
+	lapic->lvt_lint1 = lvt_mode(la, LVT_LINT1, lapic->lvt_lint1);
+
+	/* XXX: more LVT entries */
+
+	/* Clear the TPR. */
+	value = lapic->tpr;
+	value &= ~APIC_TPR_PRIO;
+	lapic->tpr = value;
+
+	/* Use the cluster model for logical IDs. */
+	value = lapic->dfr;
+	value &= ~APIC_DFR_MODEL_MASK;
+	value |= APIC_DFR_MODEL_CLUSTER;
+	lapic->dfr = value;
+
+	/* Set this APIC's logical ID. */
+	value = lapic->ldr;
+	value &= ~APIC_ID_MASK;
+	value |= (la->la_cluster << APIC_ID_CLUSTER_SHIFT |
+	    1 << la->la_cluster_id) << APIC_ID_SHIFT;
+	lapic->ldr = value;
+
+	/* Setup spurious vector and enable the local APIC. */
+	value = lapic->svr;
+	value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS);
+	value |= (APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT);
+	lapic->svr = value;
+	intr_restore(eflags);
+}
+
+void
+lapic_disable(void)
+{
+	uint32_t value;
+
+	/* Software disable the local APIC. */
+	value = lapic->svr;
+	value &= ~APIC_SVR_SWEN;
+	lapic->svr = value;
+}
+
+int
+lapic_id(void)
+{
+
+	KASSERT(lapic != NULL, ("local APIC is not mapped"));
+	return (lapic->id >> APIC_ID_SHIFT);
+}
+
+int
+lapic_intr_pending(u_int vector)
+{
+	volatile u_int32_t *irr;
+
+	/*
+	 * The IRR registers are an array of 128-bit registers each of
+	 * which only describes 32 interrupts in the low 32 bits..  Thus,
+	 * we divide the vector by 32 to get the 128-bit index.  We then
+	 * multiply that index by 4 to get the equivalent index from
+	 * treating the IRR as an array of 32-bit registers.  Finally, we
+	 * modulus the vector by 32 to determine the individual bit to
+	 * test.
+	 */
+	irr = &lapic->irr0;
+	return (irr[(vector / 32) * 4] & 1 << (vector % 32));
+}
+
+void
+lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id)
+{
+	struct lapic *la;
+
+	KASSERT(lapics[apic_id].la_present, ("%s: APIC %u doesn't exist",
+	    __func__, apic_id));
+	KASSERT(cluster <= APIC_MAX_CLUSTER, ("%s: cluster %u too big",
+	    __func__, cluster));
+	KASSERT(cluster_id <= APIC_MAX_INTRACLUSTER_ID,
+	    ("%s: intra cluster id %u too big", __func__, cluster_id));
+	la = &lapics[apic_id];
+	la->la_cluster = cluster;
+	la->la_cluster_id = cluster_id;
+}
+
+int
+lapic_set_lvt_mask(u_int apic_id, u_int pin, u_char masked)
+{
+
+	if (pin > LVT_MAX)
+		return (EINVAL);
+	if (apic_id == APIC_ID_ALL) {
+		lvts[pin].lvt_masked = masked;
+		if (bootverbose)
+			printf("lapic:");
+	} else {
+		KASSERT(lapics[apic_id].la_present,
+		    ("%s: missing APIC %u", __func__, apic_id));
+		lapics[apic_id].la_lvts[pin].lvt_masked = masked;
+		lapics[apic_id].la_lvts[pin].lvt_active = 1;
+		if (bootverbose)
+			printf("lapic%u:", apic_id);
+	}
+	if (bootverbose)
+		printf(" LINT%u %s\n", pin, masked ? "masked" : "unmasked");
+	return (0);
+}
+
+int
+lapic_set_lvt_mode(u_int apic_id, u_int pin, u_int32_t mode)
+{
+	struct lvt *lvt;
+
+	if (pin > LVT_MAX)
+		return (EINVAL);
+	if (apic_id == APIC_ID_ALL) {
+		lvt = &lvts[pin];
+		if (bootverbose)
+			printf("lapic:");
+	} else {
+		KASSERT(lapics[apic_id].la_present,
+		    ("%s: missing APIC %u", __func__, apic_id));
+		lvt = &lapics[apic_id].la_lvts[pin];
+		lvt->lvt_active = 1;
+		if (bootverbose)
+			printf("lapic%u:", apic_id);
+	}
+	lvt->lvt_mode = mode;
+	switch (mode) {
+	case APIC_LVT_DM_NMI:
+	case APIC_LVT_DM_SMI:
+	case APIC_LVT_DM_INIT:
+	case APIC_LVT_DM_EXTINT:
+		lvt->lvt_edgetrigger = 1;
+		lvt->lvt_activehi = 1;
+		if (mode == APIC_LVT_DM_EXTINT)
+			lvt->lvt_masked = 1;
+		else
+			lvt->lvt_masked = 0;
+		break;
+	default:
+		panic("Unsupported delivery mode: 0x%x\n", mode);
+	}
+	if (bootverbose) {
+		printf(" Routing ");
+		switch (mode) {
+		case APIC_LVT_DM_NMI:
+			printf("NMI");
+			break;
+		case APIC_LVT_DM_SMI:
+			printf("SMI");
+			break;
+		case APIC_LVT_DM_INIT:
+			printf("INIT");
+			break;
+		case APIC_LVT_DM_EXTINT:
+			printf("ExtINT");
+			break;
+		}
+		printf(" -> LINT%u\n", pin);
+	}
+	return (0);
+}
+
+int
+lapic_set_lvt_polarity(u_int apic_id, u_int pin, u_char activehi)
+{
+
+	if (pin > LVT_MAX)
+		return (EINVAL);
+	if (apic_id == APIC_ID_ALL) {
+		lvts[pin].lvt_activehi = activehi;
+		if (bootverbose)
+			printf("lapic:");
+	} else {
+		KASSERT(lapics[apic_id].la_present,
+		    ("%s: missing APIC %u", __func__, apic_id));
+		lapics[apic_id].la_lvts[pin].lvt_active = 1;
+		lapics[apic_id].la_lvts[pin].lvt_activehi = activehi;
+		if (bootverbose)
+			printf("lapic%u:", apic_id);
+	}
+	if (bootverbose)
+		printf(" LINT%u polarity: active-%s\n", pin,
+		    activehi ? "hi" : "lo");
+	return (0);
+}
+
+int
+lapic_set_lvt_triggermode(u_int apic_id, u_int pin, u_char edgetrigger)
+{
+
+	if (pin > LVT_MAX)
+		return (EINVAL);
+	if (apic_id == APIC_ID_ALL) {
+		lvts[pin].lvt_edgetrigger = edgetrigger;
+		if (bootverbose)
+			printf("lapic:");
+	} else {
+		KASSERT(lapics[apic_id].la_present,
+		    ("%s: missing APIC %u", __func__, apic_id));
+		lapics[apic_id].la_lvts[pin].lvt_edgetrigger = edgetrigger;
+		lapics[apic_id].la_lvts[pin].lvt_active = 1;
+		if (bootverbose)
+			printf("lapic%u:", apic_id);
+	}
+	if (bootverbose)
+		printf(" LINT%u trigger: %s\n", pin,
+		    edgetrigger ? "edge" : "level");
+	return (0);
+}
+
+void
+lapic_handle_intr(struct intrframe frame)
+{
+	struct intsrc *isrc;
+
+	if (frame.if_vec == -1)
+		panic("Couldn't get vector from ISR!");
+	isrc = intr_lookup_source(apic_idt_to_irq(frame.if_vec));
+	isrc->is_pic->pic_disable_source(isrc);
+	lapic->eoi = 0;
+	intr_execute_handlers(isrc, &frame);
+}
+
+/* Translate between IDT vectors and IRQ vectors. */
+u_int
+apic_irq_to_idt(u_int irq)
+{
+	u_int vector;
+
+	KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq));
+	vector = irq + IDT_IO_INTS;
+	if (vector >= IDT_SYSCALL)
+		vector++;
+	return (vector);
+}
+
+u_int
+apic_idt_to_irq(u_int vector)
+{
+
+	KASSERT(vector >= IDT_IO_INTS && vector != IDT_SYSCALL &&
+	    vector <= IDT_IO_INTS + NUM_IO_INTS,
+	    ("Vector %u does not map to an IRQ line", vector));
+	if (vector > IDT_SYSCALL)
+		vector--;
+	return (vector - IDT_IO_INTS);
+}
+
+/*
+ * APIC probing support code.  This includes code to manage enumerators.
+ */
+
+static SLIST_HEAD(, apic_enumerator) enumerators =
+	SLIST_HEAD_INITIALIZER(enumerators);
+static struct apic_enumerator *best_enum;
+	
+void
+apic_register_enumerator(struct apic_enumerator *enumerator)
+{
+#ifdef INVARIANTS
+	struct apic_enumerator *apic_enum;
+
+	SLIST_FOREACH(apic_enum, &enumerators, apic_next) {
+		if (apic_enum == enumerator)
+			panic("%s: Duplicate register of %s", __func__,
+			    enumerator->apic_name);
+	}
+#endif
+	SLIST_INSERT_HEAD(&enumerators, enumerator, apic_next);
+}
+
+/*
+ * We have to look for CPU's very, very early because certain subsystems
+ * want to know how many CPU's we have extremely early on in the boot
+ * process.
+ */
+static void
+apic_init(void *dummy __unused)
+{
+	struct apic_enumerator *enumerator;
+	int retval, best;
+
+	/* We only support built in local APICs. */
+	if (!(cpu_feature & CPUID_APIC))
+		return;
+
+	/* First, probe all the enumerators to find the best match. */
+	best_enum = NULL;
+	best = 0;
+	SLIST_FOREACH(enumerator, &enumerators, apic_next) {
+		retval = enumerator->apic_probe();
+		if (retval > 0)
+			continue;
+		if (best_enum == NULL || best < retval) {
+			best_enum = enumerator;
+			best = retval;
+		}
+	}
+	if (best_enum == NULL) {
+		if (bootverbose)
+			printf("APIC: Could not find any APICs.\n");
+		return;
+	}
+
+	if (bootverbose)
+		printf("APIC: Using the %s enumerator.\n",
+		    best_enum->apic_name);
+
+	/* Second, probe the CPU's in the system. */
+	retval = best_enum->apic_probe_cpus();
+	if (retval != 0)
+		printf("%s: Failed to probe CPUs: returned %d\n",
+		    best_enum->apic_name, retval);
+}
+SYSINIT(apic_init, SI_SUB_TUNABLES - 1, SI_ORDER_SECOND, apic_init, NULL)
+
+/*
+ * Setup the local APIC.  We have to do this prior to starting up the APs
+ * in the SMP case.
+ */
+static void
+apic_setup_local(void *dummy __unused)
+{
+	int retval;
+	uint64_t apic_base;
+
+	if (best_enum == NULL)
+		return;
+	/*
+	 * To work around an errata, we disable the local APIC on some
+	 * CPUs during early startup.  We need to turn the local APIC back
+	 * on on such CPUs now.
+	 */
+	if (cpu == CPU_686 && strcmp(cpu_vendor, "GenuineIntel") == 0 &&
+	    (cpu_id & 0xff0) == 0x610) {
+		apic_base = rdmsr(MSR_APICBASE);
+		apic_base |= APICBASE_ENABLED;
+		wrmsr(MSR_APICBASE, apic_base);
+	}
+	retval = best_enum->apic_setup_local();
+	if (retval != 0)
+		printf("%s: Failed to setup the local APIC: returned %d\n",
+		    best_enum->apic_name, retval);
+}
+SYSINIT(apic_setup_local, SI_SUB_CPU, SI_ORDER_FIRST, apic_setup_local, NULL)
+
+/*
+ * Setup the I/O APICs.
+ */
+static void
+apic_setup_io(void *dummy __unused)
+{
+	int retval;
+
+	if (best_enum == NULL)
+		return;
+	retval = best_enum->apic_setup_io();
+	if (retval != 0)
+		printf("%s: Failed to setup I/O APICs: returned %d\n",
+		    best_enum->apic_name, retval);
+
+	/*
+	 * Finish setting up the local APIC on the BSP once we know how to
+	 * properly program the LINT pins.
+	 */
+	lapic_setup();
+	if (bootverbose)
+		lapic_dump("BSP");
+}
+SYSINIT(apic_setup_io, SI_SUB_INTR, SI_ORDER_SECOND, apic_setup_io, NULL)
+
+#ifdef SMP
+/*
+ * Inter Processor Interrupt functions.  The lapic_ipi_*() functions are
+ * private the sys/i386 code.  The public interface for the rest of the
+ * kernel is defined in mp_machdep.c.
+ */
+#define DETECT_DEADLOCK
+
+int
+lapic_ipi_wait(int delay)
+{
+	int x, incr;
+
+	/*
+	 * Wait delay loops for IPI to be sent.  This is highly bogus
+	 * since this is sensitive to CPU clock speed.  If delay is
+	 * -1, we wait forever.
+	 */
+	if (delay == -1) {
+		incr = 0;
+		delay = 1;
+	} else
+		incr = 1;
+	for (x = 0; x < delay; x += incr) {
+		if ((lapic->icr_lo & APIC_DELSTAT_MASK) == APIC_DELSTAT_IDLE)
+			return (1);
+		ia32_pause();
+	}
+	return (0);
+}
+
+void
+lapic_ipi_raw(register_t icrlo, u_int dest)
+{
+	register_t value, eflags;
+
+	/* XXX: Need more sanity checking of icrlo? */
+	KASSERT(lapic != NULL, ("%s called too early", __func__));
+	KASSERT((dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
+	    ("%s: invalid dest field", __func__));
+	KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0,
+	    ("%s: reserved bits set in ICR LO register", __func__));
+
+	/* Set destination in ICR HI register if it is being used. */
+	eflags = intr_disable();
+	if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) {
+		value = lapic->icr_hi;
+		value &= ~APIC_ID_MASK;
+		value |= dest << APIC_ID_SHIFT;
+		lapic->icr_hi = value;
+	}
+
+	/* Program the contents of the IPI and dispatch it. */
+	value = lapic->icr_lo;
+	value &= APIC_ICRLO_RESV_MASK;
+	value |= icrlo;
+	lapic->icr_lo = value;
+	intr_restore(eflags);
+}
+
+#ifdef DETECT_DEADLOCK
+#define	BEFORE_SPIN	1000000
+#define	AFTER_SPIN	1000
+#endif
+
+void
+lapic_ipi_vectored(u_int vector, int dest)
+{
+	register_t icrlo, destfield;
+
+	KASSERT((vector & ~APIC_VECTOR_MASK) == 0,
+	    ("%s: invalid vector %d", __func__, vector));
+
+	icrlo = vector | APIC_DELMODE_FIXED | APIC_DESTMODE_PHY |
+	    APIC_LEVEL_DEASSERT | APIC_TRIGMOD_EDGE;
+	destfield = 0;
+	switch (dest) {
+	case APIC_IPI_DEST_SELF:
+		icrlo |= APIC_DEST_SELF;
+		break;
+	case APIC_IPI_DEST_ALL:
+		icrlo |= APIC_DEST_ALLISELF;
+		break;
+	case APIC_IPI_DEST_OTHERS:
+		icrlo |= APIC_DEST_ALLESELF;
+		break;
+	default:
+		KASSERT((dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
+		    ("%s: invalid destination 0x%x", __func__, dest));
+		destfield = dest;
+	}
+
+#ifdef DETECT_DEADLOCK
+	/* Check for an earlier stuck IPI. */
+	if (!lapic_ipi_wait(BEFORE_SPIN))
+		panic("APIC: Previous IPI is stuck");
+#endif
+
+	lapic_ipi_raw(icrlo, destfield);
+
+#ifdef DETECT_DEADLOCK
+	/* Wait for IPI to be delivered. */
+	if (!lapic_ipi_wait(AFTER_SPIN)) {
+#ifdef needsattention
+		/*
+		 * XXX FIXME:
+		 *
+		 * The above function waits for the message to actually be
+		 * delivered.  It breaks out after an arbitrary timeout
+		 * since the message should eventually be delivered (at
+		 * least in theory) and that if it wasn't we would catch
+		 * the failure with the check above when the next IPI is
+		 * sent.
+		 *
+		 * We could skiip this wait entirely, EXCEPT it probably
+		 * protects us from other routines that assume that the
+		 * message was delivered and acted upon when this function
+		 * returns.
+		 */
+		printf("APIC: IPI might be stuck\n");
+#else /* !needsattention */
+		/* Wait until mesage is sent without a timeout. */
+		while (lapic->icr_lo & APIC_DELSTAT_PEND)
+			ia32_pause();
+#endif /* needsattention */
+	}
+#endif /* DETECT_DEADLOCK */
+}
+#endif /* SMP */

sys/amd64/include/apicreg.h

@@ -25,8 +25,8 @@
  * $FreeBSD$
  */
 
-#ifndef _MACHINE_APIC_H_
-#define _MACHINE_APIC_H_
+#ifndef _MACHINE_APICREG_H_
+#define _MACHINE_APICREG_H_
 
 /*
  * Local && I/O APIC definitions.
@@ -221,11 +221,29 @@ typedef struct IOAPIC ioapic_t;
 /* default physical locations of LOCAL (CPU) APICs */
 #define DEFAULT_APIC_BASE	0xfee00000
 
+/* constants relating to APIC ID registers */
+#define APIC_ID_MASK		0xff000000
+#define	APIC_ID_SHIFT		24
+#define	APIC_ID_CLUSTER		0xf0
+#define	APIC_ID_CLUSTER_ID	0x0f
+#define	APIC_MAX_CLUSTER	0xe
+#define	APIC_MAX_INTRACLUSTER_ID 3
+#define	APIC_ID_CLUSTER_SHIFT	4
+
 /* fields in VER */
 #define APIC_VER_VERSION	0x000000ff
 #define APIC_VER_MAXLVT		0x00ff0000
 #define MAXLVTSHIFT		16
 
+/* fields in LDR */
+#define	APIC_LDR_RESERVED	0x00ffffff
+
+/* fields in DFR */
+#define	APIC_DFR_RESERVED	0x0fffffff
+#define	APIC_DFR_MODEL_MASK	0xf0000000
+#define	APIC_DFR_MODEL_FLAT	0xf0000000
+#define	APIC_DFR_MODEL_CLUSTER	0x00000000
+
 /* fields in SVR */
 #define APIC_SVR_VECTOR		0x000000ff
 #define APIC_SVR_VEC_PROG	0x000000f0
@@ -290,10 +308,6 @@ typedef struct IOAPIC ioapic_t;
 
 #define	APIC_ICRLO_RESV_MASK	(APIC_RESV1_MASK | APIC_RESV2_MASK)
 
-/* fields in ICR_HIGH */
-#define APIC_ID_MASK		0xff000000
-#define	APIC_ID_SHIFT		24
-
 /* fields in LVT1/2 */
 #define APIC_LVT_VECTOR		0x000000ff
 #define APIC_LVT_DM		0x00000700
@@ -444,4 +458,4 @@ typedef struct IOAPIC ioapic_t;
 
 #define IOART_INTVEC	0x000000ff	/* R/W: INTerrupt vector field */
 
-#endif /* _MACHINE_APIC_H_ */
+#endif /* _MACHINE_APICREG_H_ */

sys/amd64/include/apicvar.h

@@ -0,0 +1,165 @@
+/*-
+ * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the author nor the names of any co-contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_APICVAR_H_
+#define _MACHINE_APICVAR_H_
+
+/*
+ * Local && I/O APIC variable definitions.
+ */
+
+/*
+ * Layout of local APIC interrupt vectors:
+ *
+ *	0xff (255)  +-------------+
+ *                  |             | 15 (Spurios Vector)
+ *	0xf0 (240)  +-------------+
+ *                  |             | 14 (Interprocessor Interrupts)
+ *	0xe0 (224)  +-------------+
+ *                  |             | 13 (Local Interrupt (LINT[01]))
+ *	0xd0 (208)  +-------------+
+ *                  |             | 12 (Local Timer and Error Interrupts)
+ *	0xc0 (192)  +-------------+
+ *                  |             | 11 (I/O Interrupts)
+ *	0xb0 (176)  +-------------+
+ *                  |             | 10 (I/O Interrupts)
+ *	0xa0 (160)  +-------------+
+ *                  |             | 9 (I/O Interrupts)
+ *	0x90 (144)  +-------------+
+ *                  |             | 8 (I/O Interrupts / System Calls)
+ *	0x80 (128)  +-------------+
+ *                  |             | 7 (I/O Interrupts)
+ *	0x70 (112)  +-------------+
+ *                  |             | 6 (I/O Interrupts)
+ *	0x60 (96)   +-------------+
+ *                  |             | 5 (I/O Interrupts)
+ *	0x50 (80)   +-------------+
+ *                  |             | 4 (I/O Interrupts)
+ *	0x40 (64)   +-------------+
+ *                  |             | 3 (I/O Interrupts)
+ *	0x30 (48)   +-------------+
+ *                  |             | 2 (I/O Interrupts)
+ *	0x20 (32)   +-------------+
+ *                  |             | 1 (Exceptions, traps, faults, etc.)
+ *	0x10 (16)   +-------------+
+ *                  |             | 0 (Exceptions, traps, faults, etc.)
+ *	0x00 (0)    +-------------+
+ *
+ * Note: 0x80 needs to be handled specially and not allocated to an
+ * I/O device!
+ */
+
+#define	APIC_ID_ALL	0xff
+#define	APIC_NUM_IOINTS	160
+
+#define	APIC_LOCAL_INTS	(IDT_IO_INTS + APIC_NUM_IOINTS)
+#define	APIC_TIMER_INT	APIC_LOCAL_INTS
+#define	APIC_ERROR_INT	(APIC_LOCAL_INTS + 1)
+#define	APIC_THERMAL_INT (APIC_LOCAL_INTS + 2)
+
+#define	APIC_IPI_INTS	(APIC_LOCAL_INTS + 32)
+#define	IPI_AST		APIC_IPI_INTS		/* Generate software trap. */
+#define	IPI_INVLTLB	(APIC_IPI_INTS + 1)	/* TLB Shootdown IPIs */
+#define	IPI_INVLPG	(APIC_IPI_INTS + 2)
+#define	IPI_INVLRNG	(APIC_IPI_INTS + 3)
+#define	IPI_HARDCLOCK	(APIC_IPI_INTS + 8)	/* Inter-CPU clock handling. */
+#define	IPI_STATCLOCK	(APIC_IPI_INTS + 9)
+#define	IPI_RENDEZVOUS	(APIC_IPI_INTS + 10)	/* Inter-CPU rendezvous. */
+#define	IPI_LAZYPMAP	(APIC_IPI_INTS + 11)	/* Lazy pmap release. */
+#define	IPI_STOP	(APIC_IPI_INTS + 12)	/* Stop CPU until restarted. */
+
+#define	APIC_SPURIOUS_INT 255
+
+#define	LVT_LINT0	0
+#define	LVT_LINT1	1
+#define	LVT_TIMER	2
+#define	LVT_ERROR	3
+#define	LVT_PMC		4
+#define	LVT_THERMAL	5
+#define	LVT_MAX		LVT_THERMAL
+
+#ifndef LOCORE
+
+#define	APIC_IPI_DEST_SELF	-1
+#define	APIC_IPI_DEST_ALL	-2
+#define	APIC_IPI_DEST_OTHERS	-3
+
+/*
+ * An APIC enumerator is a psuedo bus driver that enumerates APIC's including
+ * CPU's and I/O APIC's.
+ */
+struct apic_enumerator {
+	const char *apic_name;
+	int (*apic_probe)(void);
+	int (*apic_probe_cpus)(void);
+	int (*apic_setup_local)(void);
+	int (*apic_setup_io)(void);
+	SLIST_ENTRY(apic_enumerator) apic_next;
+};
+
+inthand_t
+	IDTVEC(apic_isr1), IDTVEC(apic_isr2), IDTVEC(apic_isr3),
+	IDTVEC(apic_isr4), IDTVEC(apic_isr5), IDTVEC(spuriousint);
+
+u_int	apic_irq_to_idt(u_int irq);
+u_int	apic_idt_to_irq(u_int vector);
+void	apic_register_enumerator(struct apic_enumerator *enumerator);
+void	*ioapic_create(uintptr_t addr, int32_t id, int intbase);
+int	ioapic_disable_pin(void *cookie, u_int pin);
+int	ioapic_get_vector(void *cookie, u_int pin);
+int	ioapic_next_logical_cluster(void);
+void	ioapic_register(void *cookie);
+int	ioapic_remap_vector(void *cookie, u_int pin, int vector);
+int	ioapic_set_extint(void *cookie, u_int pin);
+int	ioapic_set_nmi(void *cookie, u_int pin);
+int	ioapic_set_polarity(void *cookie, u_int pin, char activehi);
+int	ioapic_set_triggermode(void *cookie, u_int pin, char edgetrigger);
+int	ioapic_set_smi(void *cookie, u_int pin);
+void	lapic_create(u_int apic_id, int boot_cpu);
+void	lapic_disable(void);
+void	lapic_dump(const char *str);
+void	lapic_enable_intr(u_int vector);
+int	lapic_id(void);
+void	lapic_init(uintptr_t addr);
+int	lapic_intr_pending(u_int vector);
+void	lapic_ipi_raw(register_t icrlo, u_int dest);
+void	lapic_ipi_vectored(u_int vector, int dest);
+int	lapic_ipi_wait(int delay);
+void	lapic_handle_intr(struct intrframe frame);
+void	lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id);
+int	lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked);
+int	lapic_set_lvt_mode(u_int apic_id, u_int lvt, u_int32_t mode);
+int	lapic_set_lvt_polarity(u_int apic_id, u_int lvt, u_char activehi);
+int	lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, u_char edgetrigger);
+void	lapic_setup(void);
+
+#endif /* !LOCORE */
+#endif /* _MACHINE_APICVAR_H_ */

sys/i386/i386/apic_vector.s

@@ -1,19 +1,53 @@
-/*
+/*-
+ * Copyright (c) 1989, 1990 William F. Jolitz.
+ * Copyright (c) 1990 The Regents of the University of California.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by the University of
+ *	California, Berkeley and its contributors.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
  *	from: vector.s, 386BSD 0.1 unknown origin
  * $FreeBSD$
  */
 
-#include <machine/apic.h>
-#include <machine/smp.h>
+/*
+ * Interrupt entry points for external interrupts triggered by I/O APICs
+ * as well as IPI handlers.
+ */
 
-/* convert an absolute IRQ# into a bitmask */
-#define IRQ_BIT(irq_num)	(1 << (irq_num))
+#include <machine/asmacros.h>
+#include <machine/apicreg.h>
+#include <machine/smptests.h>
 
-/* make an index into the IO APIC from the IRQ# */
-#define REDTBL_IDX(irq_num)	(0x10 + ((irq_num) * 2))
+#include "assym.s"
 
 /*
- * 
+ * Macros to create and destroy a trap frame.
  */
 #define PUSH_FRAME							\
 	pushl	$0 ;		/* dummy error code */			\
@@ -23,14 +57,6 @@
 	pushl	%es ;							\
 	pushl	%fs
 
-#define PUSH_DUMMY							\
-	pushfl ;		/* eflags */				\
-	pushl	%cs ;		/* cs */				\
-	pushl	12(%esp) ;	/* original caller eip */		\
-	pushl	$0 ;		/* dummy error code */			\
-	pushl	$0 ;		/* dummy trap type */			\
-	subl	$11*4,%esp ;
-
 #define POP_FRAME							\
 	popl	%fs ;							\
 	popl	%es ;							\
@@ -38,209 +64,40 @@
 	popal ;								\
 	addl	$4+4,%esp
 
-#define POP_DUMMY							\
-	addl	$16*4,%esp
-
-#define IOAPICADDR(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 8
-#define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 12
-	
-#define MASK_IRQ(irq_num)						\
-	ICU_LOCK ;				/* into critical reg */	\
-	testl	$IRQ_BIT(irq_num), apic_imen ;				\
-	jne	7f ;			/* masked, don't mask */	\
-	orl	$IRQ_BIT(irq_num), apic_imen ;	/* set the mask bit */	\
-	movl	IOAPICADDR(irq_num), %ecx ;	/* ioapic addr */	\
-	movl	REDIRIDX(irq_num), %eax ;	/* get the index */	\
-	movl	%eax, (%ecx) ;			/* write the index */	\
-	movl	IOAPIC_WINDOW(%ecx), %eax ;	/* current value */	\
-	orl	$IOART_INTMASK, %eax ;		/* set the mask */	\
-	movl	%eax, IOAPIC_WINDOW(%ecx) ;	/* new value */		\
-7: ;						/* already masked */	\
-	ICU_UNLOCK
 /*
- * Test to see whether we are handling an edge or level triggered INT.
- *  Level-triggered INTs must still be masked as we don't clear the source,
- *  and the EOI cycle would cause redundant INTs to occur.
+ * I/O Interrupt Entry Point.  Rather than having one entry point for
+ * each interrupt source, we use one entry point for each 32-bit word
+ * in the ISR.  The handler determines the highest bit set in the ISR,
+ * translates that into a vector, and passes the vector to the
+ * lapic_handle_intr() function.
  */
-#define MASK_LEVEL_IRQ(irq_num)						\
-	testl	$IRQ_BIT(irq_num), apic_pin_trigger ;			\
-	jz	9f ;				/* edge, don't mask */	\
-	MASK_IRQ(irq_num) ;						\
-9:
-
-
-#ifdef APIC_INTR_REORDER
-#define EOI_IRQ(irq_num)						\
-	movl	apic_isrbit_location + 8 * (irq_num), %eax ;		\
-	movl	(%eax), %eax ;						\
-	testl	apic_isrbit_location + 4 + 8 * (irq_num), %eax ;	\
-	jz	9f ;				/* not active */	\
-	movl	$0, lapic+LA_EOI ;					\
-9:
-
-#else
-#define EOI_IRQ(irq_num)						\
-	testl	$IRQ_BIT(irq_num), lapic+LA_ISR1;			\
-	jz	9f	;			/* not active */	\
-	movl	$0, lapic+LA_EOI;					\
-9:
-#endif
-	
-	
-/*
- * Test to see if the source is currently masked, clear if so.
- */
-#define UNMASK_IRQ(irq_num)					\
-	ICU_LOCK ;				/* into critical reg */	\
-	testl	$IRQ_BIT(irq_num), apic_imen ;				\
-	je	7f ;			/* bit clear, not masked */	\
-	andl	$~IRQ_BIT(irq_num), apic_imen ;/* clear mask bit */	\
-	movl	IOAPICADDR(irq_num), %ecx ;	/* ioapic addr */	\
-	movl	REDIRIDX(irq_num), %eax ;	/* get the index */	\
-	movl	%eax, (%ecx) ;			/* write the index */	\
-	movl	IOAPIC_WINDOW(%ecx), %eax ;	/* current value */	\
-	andl	$~IOART_INTMASK, %eax ;		/* clear the mask */	\
-	movl	%eax, IOAPIC_WINDOW(%ecx) ;	/* new value */		\
-7: ;						/* already unmasked */	\
-	ICU_UNLOCK
-
-/*
- * Test to see whether we are handling an edge or level triggered INT.
- *  Level-triggered INTs have to be unmasked.
- */
-#define UNMASK_LEVEL_IRQ(irq_num)					\
-	testl	$IRQ_BIT(irq_num), apic_pin_trigger ;			\
-	jz	9f ;			/* edge, don't unmask */	\
-	UNMASK_IRQ(irq_num) ;						\
-9:
-
-/*
- * Macros for interrupt entry, call to handler, and exit.
- */
-
-#define	FAST_INTR(irq_num, vec_name)					\
+#define	ISR_VEC(index, vec_name)					\
 	.text ;								\
 	SUPERALIGN_TEXT ;						\
-IDTVEC(vec_name) ;							\
-	PUSH_FRAME ;							\
-	movl	$KDSEL,%eax ;						\
-	mov	%ax,%ds ;						\
-	mov	%ax,%es ;						\
-	movl	$KPSEL,%eax ;						\
-	mov	%ax,%fs ;						\
-	FAKE_MCOUNT(13*4(%esp)) ;					\
-	movl	PCPU(CURTHREAD),%ebx ;					\
-	cmpl	$0,TD_CRITNEST(%ebx) ;					\
-	je	1f ;							\
-;									\
-	movl	$1,PCPU(INT_PENDING) ;					\
-	orl	$IRQ_BIT(irq_num),PCPU(FPENDING) ;			\
-	MASK_LEVEL_IRQ(irq_num) ;					\
-	movl	$0, lapic+LA_EOI ;					\
-	jmp	10f ;							\
-1: ;									\
-	incl	TD_CRITNEST(%ebx) ;					\
-	incl	TD_INTR_NESTING_LEVEL(%ebx) ;				\
-	pushl	intr_unit + (irq_num) * 4 ;				\
-	call	*intr_handler + (irq_num) * 4 ;	/* do the work ASAP */	\
-	addl	$4, %esp ;						\
-	movl	$0, lapic+LA_EOI ;					\
-	lock ; 								\
-	incl	cnt+V_INTR ;	/* book-keeping can wait */		\
-	movl	intr_countp + (irq_num) * 4, %eax ;			\
-	lock ; 								\
-	incl	(%eax) ;						\
-	decl	TD_CRITNEST(%ebx) ;					\
-	cmpl	$0,PCPU(INT_PENDING) ;					\
-	je	2f ;							\
-;									\
-	call	i386_unpend ;						\
-2: ;									\
-	decl	TD_INTR_NESTING_LEVEL(%ebx) ;				\
-10: ;									\
-	MEXITCOUNT ;							\
-	jmp	doreti
-
-/*
- * Restart a fast interrupt that was held up by a critical section.
- * This routine is called from unpend().  unpend() ensures we are
- * in a critical section and deals with the interrupt nesting level
- * for us.  If we previously masked the irq, we have to unmask it.
- *
- * We have a choice.  We can regenerate the irq using the 'int'
- * instruction or we can create a dummy frame and call the interrupt
- * handler directly.  I've chosen to use the dummy-frame method.
- */
-#define	FAST_UNPEND(irq_num, vec_name)					\
-	.text ;								\
-	SUPERALIGN_TEXT ;						\
-IDTVEC(vec_name) ;							\
-;									\
-	pushl	%ebp ;							\
-	movl	%esp, %ebp ;						\
-	PUSH_DUMMY ;							\
-	pushl	intr_unit + (irq_num) * 4 ;				\
-	call	*intr_handler + (irq_num) * 4 ;	/* do the work ASAP */	\
-	addl	$4, %esp ;						\
-	lock ; 								\
-	incl	cnt+V_INTR ;	/* book-keeping can wait */		\
-	movl	intr_countp + (irq_num) * 4, %eax ;			\
-	lock ; 								\
-	incl	(%eax) ;						\
-	UNMASK_LEVEL_IRQ(irq_num) ;					\
-	POP_DUMMY ;							\
-	popl %ebp ;							\
-	ret ;								\
-
-
-/* 
- * Slow, threaded interrupts.
- *
- * XXX Most of the parameters here are obsolete.  Fix this when we're
- * done.
- * XXX we really shouldn't return via doreti if we just schedule the
- * interrupt handler and don't run anything.  We could just do an
- * iret.  FIXME.
- */
-#define	INTR(irq_num, vec_name, maybe_extra_ipending)			\
-	.text ;								\
-	SUPERALIGN_TEXT ;						\
-/* _XintrNN: entry point used by IDT/HWIs via _vec[]. */		\
 IDTVEC(vec_name) ;							\
 	PUSH_FRAME ;							\
 	movl	$KDSEL, %eax ;	/* reload with kernel's data segment */	\
 	mov	%ax, %ds ;						\
 	mov	%ax, %es ;						\
-	movl	$KPSEL, %eax ;						\
+	movl	$KPSEL, %eax ;	/* reload with per-CPU data segment */	\
 	mov	%ax, %fs ;						\
-;									\
-	maybe_extra_ipending ;						\
-;									\
-	MASK_LEVEL_IRQ(irq_num) ;					\
-	EOI_IRQ(irq_num) ;						\
-;									\
-	movl	PCPU(CURTHREAD),%ebx ;					\
-	cmpl	$0,TD_CRITNEST(%ebx) ;					\
-	je	1f ;							\
-	movl	$1,PCPU(INT_PENDING) ;					\
-	orl	$IRQ_BIT(irq_num),PCPU(IPENDING) ;			\
-	jmp	10f ;							\
-1: ;									\
+	movl	lapic, %edx ;	/* pointer to local APIC */		\
+	movl	PCPU(CURTHREAD), %ebx ;					\
+	movl	LA_ISR + 16 * (index)(%edx), %eax ;	/* load ISR */	\
 	incl	TD_INTR_NESTING_LEVEL(%ebx) ;				\
-;	 								\
-	FAKE_MCOUNT(13*4(%esp)) ;		/* XXX avoid dbl cnt */ \
-	cmpl	$0,PCPU(INT_PENDING) ;					\
-	je	9f ;							\
-	call	i386_unpend ;						\
-9: ;									\
-	pushl	$irq_num;			/* pass the IRQ */	\
-	call	sched_ithd ;						\
-	addl	$4, %esp ;		/* discard the parameter */	\
-;									\
+	bsrl	%eax, %eax ;	/* index of highset set bit in ISR */	\
+	jz	2f ;							\
+	addl	$(32 * index),%eax ;					\
+1: ;									\
+	FAKE_MCOUNT(13*4(%esp)) ;	/* XXX avoid double count */	\
+	pushl	%eax ;		/* pass the IRQ */			\
+	call	lapic_handle_intr ;					\
+	addl	$4, %esp ;	/* discard parameter */			\
 	decl	TD_INTR_NESTING_LEVEL(%ebx) ;				\
-10: ;									\
 	MEXITCOUNT ;							\
-	jmp	doreti
+	jmp	doreti ;						\
+2:	movl	$-1, %eax ;	/* send a vector of -1 */		\
+	jmp	1b
 
 /*
  * Handle "spurious INTerrupts".
@@ -257,6 +114,14 @@ IDTVEC(spuriousint)
 
 	iret
 
+MCOUNT_LABEL(bintr2)
+	ISR_VEC(1,apic_isr1)
+	ISR_VEC(2,apic_isr2)
+	ISR_VEC(3,apic_isr3)
+	ISR_VEC(4,apic_isr4)
+	ISR_VEC(5,apic_isr5)
+MCOUNT_LABEL(eintr2)
+
 #ifdef SMP
 /*
  * Global address space TLB shootdown.
@@ -281,7 +146,8 @@ IDTVEC(invltlb)
 	movl	%cr3, %eax		/* invalidate the TLB */
 	movl	%eax, %cr3
 
-	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
+	movl	lapic, %eax
+	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
 
 	lock
 	incl	smp_tlb_wait
@@ -313,7 +179,8 @@ IDTVEC(invlpg)
 	movl	smp_tlb_addr1, %eax
 	invlpg	(%eax)			/* invalidate single page */
 
-	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
+	movl	lapic, %eax
+	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
 
 	lock
 	incl	smp_tlb_wait
@@ -350,7 +217,8 @@ IDTVEC(invlrng)
 	cmpl	%eax, %edx
 	jb	1b
 
-	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
+	movl	lapic, %eax
+	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
 
 	lock
 	incl	smp_tlb_wait
@@ -374,21 +242,15 @@ IDTVEC(hardclock)
 	movl	$KPSEL, %eax
 	mov	%ax, %fs
 
-	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
+	movl	lapic, %edx
+	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
 
 	movl	PCPU(CURTHREAD),%ebx
-	cmpl	$0,TD_CRITNEST(%ebx)
-	je	1f
-	movl	$1,PCPU(INT_PENDING)
-	orl	$1,PCPU(SPENDING);
-	jmp	10f
-1:
 	incl	TD_INTR_NESTING_LEVEL(%ebx)
 	pushl	$0		/* XXX convert trapframe to clockframe */
 	call	forwarded_hardclock
 	addl	$4, %esp	/* XXX convert clockframe to trapframe */
 	decl	TD_INTR_NESTING_LEVEL(%ebx)
-10:
 	MEXITCOUNT
 	jmp	doreti
 
@@ -406,23 +268,17 @@ IDTVEC(statclock)
 	movl	$KPSEL, %eax
 	mov	%ax, %fs
 
-	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
+	movl	lapic, %edx
+	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
 
 	FAKE_MCOUNT(13*4(%esp))
 
 	movl	PCPU(CURTHREAD),%ebx
-	cmpl	$0,TD_CRITNEST(%ebx)
-	je	1f
-	movl	$1,PCPU(INT_PENDING)
-	orl	$2,PCPU(SPENDING);
-	jmp	10f
-1:
 	incl	TD_INTR_NESTING_LEVEL(%ebx)
 	pushl	$0		/* XXX convert trapframe to clockframe */
 	call	forwarded_statclock
 	addl	$4, %esp	/* XXX convert clockframe to trapframe */
 	decl	TD_INTR_NESTING_LEVEL(%ebx)
-10:
 	MEXITCOUNT
 	jmp	doreti
 
@@ -444,7 +300,8 @@ IDTVEC(cpuast)
 	movl	$KPSEL, %eax
 	mov	%ax, %fs
 
-	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
+	movl	lapic, %edx
+	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
 
 	FAKE_MCOUNT(13*4(%esp))
 
@@ -476,7 +333,8 @@ IDTVEC(cpustop)
 	movl	$KPSEL, %eax
 	mov	%ax, %fs
 
-	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
+	movl	lapic, %eax
+	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
 
 	movl	PCPU(CPUID), %eax
 	imull	$PCB_SIZE, %eax
@@ -518,111 +376,6 @@ IDTVEC(cpustop)
 	popl	%ebp
 	iret
 
-#endif /* SMP */
-
-MCOUNT_LABEL(bintr)
-	FAST_INTR(0,fastintr0)
-	FAST_INTR(1,fastintr1)
-	FAST_INTR(2,fastintr2)
-	FAST_INTR(3,fastintr3)
-	FAST_INTR(4,fastintr4)
-	FAST_INTR(5,fastintr5)
-	FAST_INTR(6,fastintr6)
-	FAST_INTR(7,fastintr7)
-	FAST_INTR(8,fastintr8)
-	FAST_INTR(9,fastintr9)
-	FAST_INTR(10,fastintr10)
-	FAST_INTR(11,fastintr11)
-	FAST_INTR(12,fastintr12)
-	FAST_INTR(13,fastintr13)
-	FAST_INTR(14,fastintr14)
-	FAST_INTR(15,fastintr15)
-	FAST_INTR(16,fastintr16)
-	FAST_INTR(17,fastintr17)
-	FAST_INTR(18,fastintr18)
-	FAST_INTR(19,fastintr19)
-	FAST_INTR(20,fastintr20)
-	FAST_INTR(21,fastintr21)
-	FAST_INTR(22,fastintr22)
-	FAST_INTR(23,fastintr23)
-	FAST_INTR(24,fastintr24)
-	FAST_INTR(25,fastintr25)
-	FAST_INTR(26,fastintr26)
-	FAST_INTR(27,fastintr27)
-	FAST_INTR(28,fastintr28)
-	FAST_INTR(29,fastintr29)
-	FAST_INTR(30,fastintr30)
-	FAST_INTR(31,fastintr31)
-#define	CLKINTR_PENDING	movl $1,CNAME(clkintr_pending)
-/* Threaded interrupts */
-	INTR(0,intr0, CLKINTR_PENDING)
-	INTR(1,intr1,)
-	INTR(2,intr2,)
-	INTR(3,intr3,)
-	INTR(4,intr4,)
-	INTR(5,intr5,)
-	INTR(6,intr6,)
-	INTR(7,intr7,)
-	INTR(8,intr8,)
-	INTR(9,intr9,)
-	INTR(10,intr10,)
-	INTR(11,intr11,)
-	INTR(12,intr12,)
-	INTR(13,intr13,)
-	INTR(14,intr14,)
-	INTR(15,intr15,)
-	INTR(16,intr16,)
-	INTR(17,intr17,)
-	INTR(18,intr18,)
-	INTR(19,intr19,)
-	INTR(20,intr20,)
-	INTR(21,intr21,)
-	INTR(22,intr22,)
-	INTR(23,intr23,)
-	INTR(24,intr24,)
-	INTR(25,intr25,)
-	INTR(26,intr26,)
-	INTR(27,intr27,)
-	INTR(28,intr28,)
-	INTR(29,intr29,)
-	INTR(30,intr30,)
-	INTR(31,intr31,)
-
-	FAST_UNPEND(0,fastunpend0)
-	FAST_UNPEND(1,fastunpend1)
-	FAST_UNPEND(2,fastunpend2)
-	FAST_UNPEND(3,fastunpend3)
-	FAST_UNPEND(4,fastunpend4)
-	FAST_UNPEND(5,fastunpend5)
-	FAST_UNPEND(6,fastunpend6)
-	FAST_UNPEND(7,fastunpend7)
-	FAST_UNPEND(8,fastunpend8)
-	FAST_UNPEND(9,fastunpend9)
-	FAST_UNPEND(10,fastunpend10)
-	FAST_UNPEND(11,fastunpend11)
-	FAST_UNPEND(12,fastunpend12)
-	FAST_UNPEND(13,fastunpend13)
-	FAST_UNPEND(14,fastunpend14)
-	FAST_UNPEND(15,fastunpend15)
-	FAST_UNPEND(16,fastunpend16)
-	FAST_UNPEND(17,fastunpend17)
-	FAST_UNPEND(18,fastunpend18)
-	FAST_UNPEND(19,fastunpend19)
-	FAST_UNPEND(20,fastunpend20)
-	FAST_UNPEND(21,fastunpend21)
-	FAST_UNPEND(22,fastunpend22)
-	FAST_UNPEND(23,fastunpend23)
-	FAST_UNPEND(24,fastunpend24)
-	FAST_UNPEND(25,fastunpend25)
-	FAST_UNPEND(26,fastunpend26)
-	FAST_UNPEND(27,fastunpend27)
-	FAST_UNPEND(28,fastunpend28)
-	FAST_UNPEND(29,fastunpend29)
-	FAST_UNPEND(30,fastunpend30)
-	FAST_UNPEND(31,fastunpend31)
-MCOUNT_LABEL(eintr)
-
-#ifdef SMP
 /*
  * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
  *
@@ -640,7 +393,8 @@ IDTVEC(rendezvous)
 
 	call	smp_rendezvous_action
 
-	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
+	movl	lapic, %eax
+	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
 	POP_FRAME
 	iret
 	
@@ -658,16 +412,9 @@ IDTVEC(lazypmap)
 	mov	%ax, %fs
 
 	call	pmap_lazyfix_action
-	
-	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
+
+	movl	lapic, %eax	
+	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
 	POP_FRAME
 	iret
 #endif /* SMP */
-
-	.data
-
-	.globl	apic_pin_trigger
-apic_pin_trigger:
-	.long	0
-
-	.text

sys/i386/i386/initcpu.c

@@ -474,7 +474,6 @@ init_6x86MX(void)
 static void
 init_ppro(void)
 {
-#ifndef SMP
 	u_int64_t	apicbase;
 
 	/*
@@ -483,7 +482,6 @@ init_ppro(void)
 	apicbase = rdmsr(MSR_APICBASE);
 	apicbase &= ~APICBASE_ENABLED;
 	wrmsr(MSR_APICBASE, apicbase);
-#endif
 }
 
 /*

sys/i386/i386/io_apic.c

@@ -0,0 +1,691 @@
+/*-
+ * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the author nor the names of any co-contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_isa.h"
+#include "opt_no_mixed_mode.h"
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/apicreg.h>
+#include <machine/frame.h>
+#include <machine/intr_machdep.h>
+#include <machine/apicvar.h>
+#include <machine/segments.h>
+
+#if defined(DEV_ISA) && !defined(NO_MIXED_MODE)
+#define MIXED_MODE
+#endif
+
+#define IOAPIC_ISA_INTS		16
+#define	IOAPIC_MEM_REGION	32
+#define	IOAPIC_REDTBL_LO(i)	(IOAPIC_REDTBL + (i) * 2)
+#define	IOAPIC_REDTBL_HI(i)	(IOAPIC_REDTBL_LO(i) + 1)
+
+#define	VECTOR_EXTINT		-1
+#define	VECTOR_NMI		-2
+#define	VECTOR_SMI		-3
+#define	VECTOR_DISABLED		-4
+
+#define	DEST_NONE		-1
+#define	DEST_EXTINT		-2
+
+#define	TODO		printf("%s: not implemented!\n", __func__)
+
+MALLOC_DEFINE(M_IOAPIC, "I/O APIC", "I/O APIC structures");
+
+/*
+ * New interrupt support code..
+ *
+ * XXX: we really should have the interrupt cookie passed up from new-bus
+ * just be a int pin, and not map 1:1 to interrupt vector number but should
+ * use INTR_TYPE_FOO to set priority bands for device classes and do all the
+ * magic remapping of intpin to vector in here.  For now we just cheat as on
+ * ia64 and map intpin X to vector NRSVIDT + X.  Note that we assume that the
+ * first IO APIC has ISA interrupts on pins 1-15.  Not sure how you are
+ * really supposed to figure out which IO APIC in a system with multiple IO
+ * APIC's actually has the ISA interrupts routed to it.  As far as interrupt
+ * pin numbers, we use the ACPI System Interrupt number model where each
+ * IO APIC has a contiguous chunk of the System Interrupt address space.
+ */
+
+/*
+ * Direct the ExtINT pin on the first I/O APIC to a logical cluster of
+ * CPUs rather than a physical destination of just the BSP.
+ *
+ * Note: This is disabled by default as test systems seem to croak with it
+ * enabled.
+#define ENABLE_EXTINT_LOGICAL_DESTINATION
+ */
+
+struct ioapic_intsrc {
+	struct intsrc io_intsrc;
+	int io_intpin:8;
+	int io_vector:8;
+	int io_activehi:1;
+	int io_edgetrigger:1;
+	int io_masked:1;
+	int io_dest:5;
+};
+
+struct ioapic {
+	struct pic io_pic;
+	u_int io_id:8;			/* logical ID */
+	u_int io_apic_id:4;
+	u_int io_intbase:8;		/* System Interrupt base */
+	u_int io_numintr:8;
+	volatile ioapic_t *io_addr;	/* XXX: should use bus_space */
+	STAILQ_ENTRY(ioapic) io_next;
+	struct ioapic_intsrc io_pins[0];
+};
+
+static STAILQ_HEAD(,ioapic) ioapic_list = STAILQ_HEAD_INITIALIZER(ioapic_list);
+static u_int next_id, program_logical_dest;
+
+static u_int	ioapic_read(volatile ioapic_t *apic, int reg);
+static void	ioapic_write(volatile ioapic_t *apic, int reg, u_int val);
+static void	ioapic_enable_source(struct intsrc *isrc);
+static void	ioapic_disable_source(struct intsrc *isrc);
+static void	ioapic_eoi_source(struct intsrc *isrc);
+static void	ioapic_enable_intr(struct intsrc *isrc);
+static int	ioapic_vector(struct intsrc *isrc);
+static int	ioapic_source_pending(struct intsrc *isrc);
+static void	ioapic_suspend(struct intsrc *isrc);
+static void	ioapic_resume(struct intsrc *isrc);
+static void	ioapic_program_destination(struct ioapic_intsrc *intpin);
+#ifdef MIXED_MODE
+static void	ioapic_setup_mixed_mode(struct ioapic_intsrc *intpin);
+#endif
+
+struct pic ioapic_template = { ioapic_enable_source, ioapic_disable_source,
+			       ioapic_eoi_source, ioapic_enable_intr,
+			       ioapic_vector, ioapic_source_pending,
+			       ioapic_suspend, ioapic_resume };
+	
+static int next_ioapic_base, logical_clusters, current_cluster;
+
+static u_int
+ioapic_read(volatile ioapic_t *apic, int reg)
+{
+
+	mtx_assert(&icu_lock, MA_OWNED);
+	apic->ioregsel = reg;
+	return (apic->iowin);
+}
+
+static void
+ioapic_write(volatile ioapic_t *apic, int reg, u_int val)
+{
+
+	mtx_assert(&icu_lock, MA_OWNED);
+	apic->ioregsel = reg;
+	apic->iowin = val;
+}
+
+static void
+ioapic_enable_source(struct intsrc *isrc)
+{
+	struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc;
+	struct ioapic *io = (struct ioapic *)isrc->is_pic;
+	uint32_t flags;
+
+	mtx_lock_spin(&icu_lock);
+	if (intpin->io_masked) {
+		flags = ioapic_read(io->io_addr,
+		    IOAPIC_REDTBL_LO(intpin->io_intpin));
+		flags &= ~(IOART_INTMASK);
+		ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin),
+		    flags);
+		intpin->io_masked = 0;
+	}
+	mtx_unlock_spin(&icu_lock);
+}
+
+static void
+ioapic_disable_source(struct intsrc *isrc)
+{
+	struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc;
+	struct ioapic *io = (struct ioapic *)isrc->is_pic;
+	uint32_t flags;
+
+	mtx_lock_spin(&icu_lock);
+	if (!intpin->io_masked && !intpin->io_edgetrigger) {
+		flags = ioapic_read(io->io_addr,
+		    IOAPIC_REDTBL_LO(intpin->io_intpin));
+		flags |= IOART_INTMSET;
+		ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin),
+		    flags);
+		intpin->io_masked = 1;
+	}
+	mtx_unlock_spin(&icu_lock);
+}
+
+static void
+ioapic_eoi_source(struct intsrc *isrc)
+{
+	TODO;
+	/* lapic_eoi(); */
+}
+
+/*
+ * Program an individual intpin's logical destination.
+ */
+static void
+ioapic_program_destination(struct ioapic_intsrc *intpin)
+{
+	struct ioapic *io = (struct ioapic *)intpin->io_intsrc.is_pic;
+	uint32_t value;
+
+	KASSERT(intpin->io_dest != DEST_NONE,
+	    ("intpin not assigned to a cluster"));
+	KASSERT(intpin->io_dest != DEST_EXTINT,
+	    ("intpin routed via ExtINT"));
+	if (bootverbose) {
+		printf("ioapic%u: routing intpin %u (", io->io_id,
+		    intpin->io_intpin);
+		if (intpin->io_vector == VECTOR_EXTINT)
+			printf("ExtINT");
+		else
+			printf("IRQ %u", intpin->io_vector);
+		printf(") to cluster %u\n", intpin->io_dest);
+	}
+	mtx_lock_spin(&icu_lock);
+	value = ioapic_read(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin));
+	value &= ~IOART_DESTMOD;
+	value |= IOART_DESTLOG;
+	ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin), value);
+	value = ioapic_read(io->io_addr, IOAPIC_REDTBL_HI(intpin->io_intpin));
+	value &= ~IOART_DEST;
+	value |= (intpin->io_dest << APIC_ID_CLUSTER_SHIFT |
+	    APIC_ID_CLUSTER_ID) << APIC_ID_SHIFT;
+	ioapic_write(io->io_addr, IOAPIC_REDTBL_HI(intpin->io_intpin), value);
+	mtx_unlock_spin(&icu_lock);
+}
+
+static void
+ioapic_assign_cluster(struct ioapic_intsrc *intpin)
+{
+	/*
+	 * Assign this intpin to a logical APIC cluster in a
+	 * round-robin fashion.  We don't actually use the logical
+	 * destination for this intpin until after all the CPU's
+	 * have been started so that we don't end up with interrupts
+	 * that don't go anywhere.  Another alternative might be to
+	 * start up the CPU's earlier so that they can handle interrupts
+	 * sooner.
+	 */
+	intpin->io_dest = current_cluster;
+	current_cluster++;
+	if (current_cluster >= logical_clusters)
+		current_cluster = 0;
+	if (program_logical_dest)
+		ioapic_program_destination(intpin);
+}
+
+static void
+ioapic_enable_intr(struct intsrc *isrc)
+{
+	struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc;
+
+	KASSERT(intpin->io_dest != DEST_EXTINT,
+	    ("ExtINT pin trying to use ioapic enable_intr method"));
+	if (intpin->io_dest == DEST_NONE) {
+		ioapic_assign_cluster(intpin);
+		lapic_enable_intr(intpin->io_vector);
+	}
+}
+
+static int
+ioapic_vector(struct intsrc *isrc)
+{
+	struct ioapic_intsrc *pin;
+
+	pin = (struct ioapic_intsrc *)isrc;
+	return (pin->io_vector);
+}
+
+static int
+ioapic_source_pending(struct intsrc *isrc)
+{
+	struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc;
+
+	return (lapic_intr_pending(intpin->io_vector));
+}
+
+static void
+ioapic_suspend(struct intsrc *isrc)
+{
+
+	TODO;
+}
+
+static void
+ioapic_resume(struct intsrc *isrc)
+{
+
+	TODO;
+}
+
+/*
+ * Allocate and return a logical cluster ID.  Note that the first time
+ * this is called, it returns cluster 0.  ioapic_enable_intr() treats
+ * the two cases of logical_clusters == 0 and logical_clusters == 1 the
+ * same: one cluster of ID 0 exists.  The logical_clusters == 0 case is
+ * for UP kernels, which should never call this function.
+ */
+int
+ioapic_next_logical_cluster(void)
+{
+
+	if (logical_clusters >= APIC_MAX_CLUSTER)
+		panic("WARNING: Local APIC cluster IDs exhausted!");
+	return (logical_clusters++);
+}
+
+/*
+ * Create a plain I/O APIC object.
+ */
+void *
+ioapic_create(uintptr_t addr, int32_t apic_id, int intbase)
+{
+	struct ioapic *io;
+	struct ioapic_intsrc *intpin;
+	volatile ioapic_t *apic;
+	u_int numintr, i;
+	uint32_t value;
+
+	apic = (ioapic_t *)pmap_mapdev(addr, IOAPIC_MEM_REGION);
+	mtx_lock_spin(&icu_lock);
+	numintr = ((ioapic_read(apic, IOAPIC_VER) & IOART_VER_MAXREDIR) >>
+	    MAXREDIRSHIFT) + 1;
+	mtx_unlock_spin(&icu_lock);
+	io = malloc(sizeof(struct ioapic) +
+	    numintr * sizeof(struct ioapic_intsrc), M_IOAPIC, M_WAITOK);
+	io->io_pic = ioapic_template;
+	mtx_lock_spin(&icu_lock);
+	io->io_id = next_id++;
+	io->io_apic_id = ioapic_read(apic, IOAPIC_ID) >> APIC_ID_SHIFT;	
+	if (apic_id != -1 && io->io_apic_id != apic_id) {
+		ioapic_write(apic, IOAPIC_ID, apic_id << APIC_ID_SHIFT);
+		mtx_unlock_spin(&icu_lock);
+		io->io_apic_id = apic_id;
+		printf("ioapic%u: Changing APIC ID to %d\n", io->io_id,
+		    apic_id);
+	} else
+		mtx_unlock_spin(&icu_lock);
+	if (intbase == -1) {
+		intbase = next_ioapic_base;
+		printf("ioapic%u: Assuming intbase of %d\n", io->io_id,
+		    intbase);
+	} else if (intbase != next_ioapic_base)
+		printf("ioapic%u: WARNING: intbase %d != expected base %d\n",
+		    io->io_id, intbase, next_ioapic_base);
+	io->io_intbase = intbase;
+	next_ioapic_base += numintr;
+	io->io_numintr = numintr;
+	io->io_addr = apic;
+
+	/*
+	 * Initialize pins.  Start off with interrupts disabled.  Default
+	 * to active-hi and edge-triggered for ISA interrupts and active-lo
+	 * and level-triggered for all others.
+	 */
+	bzero(io->io_pins, sizeof(struct ioapic_intsrc) * numintr);
+	mtx_lock_spin(&icu_lock);
+	for (i = 0, intpin = io->io_pins; i < numintr; i++, intpin++) {
+		intpin->io_intsrc.is_pic = (struct pic *)io;
+		intpin->io_intpin = i;
+		intpin->io_vector = intbase + i;
+
+		/*
+		 * Assume that pin 0 on the first IO APIC is an ExtINT pin by
+		 * default.  Assume that intpins 1-15 are ISA interrupts and
+		 * use suitable defaults for those.  Assume that all other
+		 * intpins are PCI interrupts.  Enable the ExtINT pin by
+		 * default but mask all other pins.
+		 */
+		if (intpin->io_vector == 0) {
+			intpin->io_activehi = 1;
+			intpin->io_edgetrigger = 1;
+			intpin->io_vector = VECTOR_EXTINT;
+			intpin->io_masked = 0;
+		} else if (intpin->io_vector < IOAPIC_ISA_INTS) {
+			intpin->io_activehi = 1;
+			intpin->io_edgetrigger = 1;
+			intpin->io_masked = 1;
+		} else {
+			intpin->io_activehi = 0;
+			intpin->io_edgetrigger = 0;
+			intpin->io_masked = 1;
+		}
+
+		/*
+		 * Start off without a logical cluster destination until
+		 * the pin is enabled.
+		 */
+		intpin->io_dest = DEST_NONE;
+		if (bootverbose) {
+			printf("ioapic%u: intpin %d -> ",  io->io_id, i);
+			if (intpin->io_vector == VECTOR_EXTINT)
+				printf("ExtINT\n");
+			else
+				printf("irq %d\n", intpin->io_vector);
+		}
+		value = ioapic_read(apic, IOAPIC_REDTBL_LO(i));
+		ioapic_write(apic, IOAPIC_REDTBL_LO(i), value | IOART_INTMSET);
+	}
+	mtx_unlock_spin(&icu_lock);
+
+	return (io);
+}
+
+int
+ioapic_get_vector(void *cookie, u_int pin)
+{
+	struct ioapic *io;
+
+	io = (struct ioapic *)cookie;
+	if (pin >= io->io_numintr)
+		return (-1);
+	return (io->io_pins[pin].io_vector);
+}
+
+int
+ioapic_disable_pin(void *cookie, u_int pin)
+{
+	struct ioapic *io;
+
+	io = (struct ioapic *)cookie;
+	if (pin >= io->io_numintr)
+		return (EINVAL);
+	if (io->io_pins[pin].io_vector == VECTOR_DISABLED)
+		return (EINVAL);
+	io->io_pins[pin].io_vector = VECTOR_DISABLED;
+	if (bootverbose)
+		printf("ioapic%u: intpin %d disabled\n", io->io_id, pin);
+	return (0);
+}
+
+int
+ioapic_remap_vector(void *cookie, u_int pin, int vector)
+{
+	struct ioapic *io;
+
+	io = (struct ioapic *)cookie;
+	if (pin >= io->io_numintr || vector < 0)
+		return (EINVAL);
+	if (io->io_pins[pin].io_vector < 0)
+		return (EINVAL);
+	io->io_pins[pin].io_vector = vector;
+	if (bootverbose)
+		printf("ioapic%u: Routing IRQ %d -> intpin %d\n", io->io_id,
+		    vector, pin);
+	return (0);
+}
+
+int
+ioapic_set_nmi(void *cookie, u_int pin)
+{
+	struct ioapic *io;
+
+	io = (struct ioapic *)cookie;
+	if (pin >= io->io_numintr)
+		return (EINVAL);
+	if (io->io_pins[pin].io_vector < 0)
+		return (EINVAL);
+	io->io_pins[pin].io_vector = VECTOR_NMI;
+	io->io_pins[pin].io_masked = 0;
+	io->io_pins[pin].io_edgetrigger = 1;
+	io->io_pins[pin].io_activehi = 1;
+	if (bootverbose)
+		printf("ioapic%u: Routing NMI -> intpin %d\n",
+		    io->io_id, pin);
+	return (0);
+}
+
+int
+ioapic_set_smi(void *cookie, u_int pin)
+{
+	struct ioapic *io;
+
+	io = (struct ioapic *)cookie;
+	if (pin >= io->io_numintr)
+		return (EINVAL);
+	if (io->io_pins[pin].io_vector < 0)
+		return (EINVAL);
+	io->io_pins[pin].io_vector = VECTOR_SMI;
+	io->io_pins[pin].io_masked = 0;
+	io->io_pins[pin].io_edgetrigger = 1;
+	io->io_pins[pin].io_activehi = 1;
+	if (bootverbose)
+		printf("ioapic%u: Routing SMI -> intpin %d\n",
+		    io->io_id, pin);
+	return (0);
+}
+
+int
+ioapic_set_extint(void *cookie, u_int pin)
+{
+	struct ioapic *io;
+
+	io = (struct ioapic *)cookie;
+	if (pin >= io->io_numintr)
+		return (EINVAL);
+	if (io->io_pins[pin].io_vector < 0)
+		return (EINVAL);
+	io->io_pins[pin].io_vector = VECTOR_EXTINT;
+	io->io_pins[pin].io_masked = 0;
+	io->io_pins[pin].io_edgetrigger = 1;
+	io->io_pins[pin].io_activehi = 1;
+	if (bootverbose)
+		printf("ioapic%u: Routing external 8259A's -> intpin %d\n",
+		    io->io_id, pin);
+	return (0);
+}
+
+int
+ioapic_set_polarity(void *cookie, u_int pin, char activehi)
+{
+	struct ioapic *io;
+
+	io = (struct ioapic *)cookie;
+	if (pin >= io->io_numintr)
+		return (EINVAL);
+	if (io->io_pins[pin].io_vector < 0)
+		return (EINVAL);
+	io->io_pins[pin].io_activehi = activehi;
+	if (bootverbose)
+		printf("ioapic%u: intpin %d polarity: %s\n", io->io_id, pin,
+		    activehi ? "active-hi" : "active-lo");
+	return (0);
+}
+
+int
+ioapic_set_triggermode(void *cookie, u_int pin, char edgetrigger)
+{
+	struct ioapic *io;
+
+	io = (struct ioapic *)cookie;
+	if (pin >= io->io_numintr)
+		return (EINVAL);
+	if (io->io_pins[pin].io_vector < 0)
+		return (EINVAL);
+	io->io_pins[pin].io_edgetrigger = edgetrigger;
+	if (bootverbose)
+		printf("ioapic%u: intpin %d trigger: %s\n", io->io_id, pin,
+		    edgetrigger ? "edge" : "level");
+	return (0);
+}
+
+/*
+ * Register a complete I/O APIC object with the interrupt subsystem.
+ */
+void
+ioapic_register(void *cookie)
+{
+	struct ioapic_intsrc *pin;
+	struct ioapic *io;
+	volatile ioapic_t *apic;
+	uint32_t flags;
+	int i;
+
+	io = (struct ioapic *)cookie;
+	apic = io->io_addr;
+	mtx_lock_spin(&icu_lock);
+	flags = ioapic_read(apic, IOAPIC_VER) & IOART_VER_VERSION;
+	STAILQ_INSERT_TAIL(&ioapic_list, io, io_next);
+	mtx_unlock_spin(&icu_lock);
+	printf("ioapic%u <Version %u> irqs %u-%u on motherboard\n", io->io_id,
+	    flags, io->io_intbase, io->io_intbase + io->io_numintr - 1);
+	for (i = 0, pin = io->io_pins; i < io->io_numintr; i++, pin++) {
+		/*
+		 * Finish initializing the pins by programming the vectors
+		 * and delivery mode.
+		 */
+		if (pin->io_vector == VECTOR_DISABLED)
+			continue;
+		flags = IOART_DESTPHY;
+		if (pin->io_edgetrigger)
+			flags |= IOART_TRGREDG;
+		else
+			flags |= IOART_TRGRLVL;
+		if (pin->io_activehi)
+			flags |= IOART_INTAHI;
+		else
+			flags |= IOART_INTALO;
+		if (pin->io_masked)
+			flags |= IOART_INTMSET;
+		switch (pin->io_vector) {
+		case VECTOR_EXTINT:
+			KASSERT(pin->io_edgetrigger,
+			    ("EXTINT not edge triggered"));
+			flags |= IOART_DELEXINT;
+			break;
+		case VECTOR_NMI:
+			KASSERT(pin->io_edgetrigger,
+			    ("NMI not edge triggered"));
+			flags |= IOART_DELNMI;
+			break;
+		case VECTOR_SMI:
+			KASSERT(pin->io_edgetrigger,
+			    ("SMI not edge triggered"));
+			flags |= IOART_DELSMI;
+			break;
+		default:
+			flags |= IOART_DELLOPRI |
+			    apic_irq_to_idt(pin->io_vector);
+		}
+		mtx_lock_spin(&icu_lock);
+		ioapic_write(apic, IOAPIC_REDTBL_LO(i), flags);
+
+		/*
+		 * Route interrupts to the BSP by default using physical
+		 * addressing.  Vectored interrupts get readdressed using
+		 * logical IDs to CPU clusters when they are enabled.
+		 */
+		flags = ioapic_read(apic, IOAPIC_REDTBL_HI(i));
+		flags &= ~IOART_DEST;
+		flags |= PCPU_GET(apic_id) << APIC_ID_SHIFT;
+		ioapic_write(apic, IOAPIC_REDTBL_HI(i), flags);
+		mtx_unlock_spin(&icu_lock);
+		if (pin->io_vector >= 0) {
+#ifdef MIXED_MODE
+			/* Route IRQ0 via the 8259A using mixed mode. */
+			if (pin->io_vector == 0)
+				ioapic_setup_mixed_mode(pin);
+			else
+#endif
+				intr_register_source(&pin->io_intsrc);
+		}
+			
+	}
+}
+
+/*
+ * Program all the intpins to use logical destinations once the AP's
+ * have been launched.
+ */
+static void
+ioapic_set_logical_destinations(void *arg __unused)
+{
+	struct ioapic *io;
+	int i;
+
+	program_logical_dest = 1;
+	STAILQ_FOREACH(io, &ioapic_list, io_next)
+	    for (i = 0; i < io->io_numintr; i++)
+		    if (io->io_pins[i].io_dest != DEST_NONE &&
+			io->io_pins[i].io_dest != DEST_EXTINT)
+			    ioapic_program_destination(&io->io_pins[i]);
+}
+SYSINIT(ioapic_destinations, SI_SUB_SMP, SI_ORDER_SECOND,
+    ioapic_set_logical_destinations, NULL)
+
+#ifdef MIXED_MODE
+/*
+ * Support for mixed-mode interrupt sources.  These sources route an ISA
+ * IRQ through the 8259A's via the ExtINT on pin 0 of the I/O APIC that
+ * routes the ISA interrupts.  We just ignore the intpins that use this
+ * mode and allow the atpic driver to register its interrupt source for
+ * that IRQ instead.
+ */
+
+void
+ioapic_setup_mixed_mode(struct ioapic_intsrc *intpin)
+{
+	struct ioapic_intsrc *extint;
+	struct ioapic *io;
+
+	/*
+	 * Mark the associated I/O APIC intpin as being delivered via
+	 * ExtINT and enable the ExtINT pin on the I/O APIC if needed.
+	 */
+	intpin->io_dest = DEST_EXTINT;
+	io = (struct ioapic *)intpin->io_intsrc.is_pic;
+	extint = &io->io_pins[0];
+	if (extint->io_vector != VECTOR_EXTINT)
+		panic("Can't find ExtINT pin to route through!");
+#ifdef ENABLE_EXTINT_LOGICAL_DESTINATION
+	if (extint->io_dest == DEST_NONE)
+		ioapic_assign_cluster(extint);
+#endif
+}
+
+#endif /* MIXED_MODE */

sys/i386/i386/local_apic.c

@@ -0,0 +1,758 @@
+/*-
+ * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org>
+ * Copyright (c) 1996, by Steve Passe
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. The name of the developer may NOT be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ * 3. Neither the name of the author nor the names of any co-contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Local APIC support on Pentium and later processors.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/pcpu.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/apicreg.h>
+#include <machine/cputypes.h>
+#include <machine/frame.h>
+#include <machine/intr_machdep.h>
+#include <machine/apicvar.h>
+#include <machine/md_var.h>
+#include <machine/smp.h>
+#include <machine/specialreg.h>
+
+/*
+ * We can handle up to 60 APICs via our logical cluster IDs, but currently
+ * the physical IDs on Intel processors up to the Pentium 4 are limited to
+ * 16.
+ */
+#define	MAX_APICID	16
+
+/*
+ * Support for local APICs.  Local APICs manage interrupts on each
+ * individual processor as opposed to I/O APICs which receive interrupts
+ * from I/O devices and then forward them on to the local APICs.
+ *
+ * Local APICs can also send interrupts to each other thus providing the
+ * mechanism for IPIs.
+ */
+
+struct lvt {
+	u_int lvt_edgetrigger:1;
+	u_int lvt_activehi:1;
+	u_int lvt_masked:1;
+	u_int lvt_active:1;
+	u_int lvt_mode:16;
+	u_int lvt_vector:8;
+};
+
+struct lapic {
+	struct lvt la_lvts[LVT_MAX + 1];
+	u_int la_id:8;
+	u_int la_cluster:4;
+	u_int la_cluster_id:2;
+	u_int la_present:1;
+} static lapics[MAX_APICID];
+
+/* XXX: should thermal be an NMI? */
+
+/* Global defaults for local APIC LVT entries. */
+static struct lvt lvts[LVT_MAX + 1] = {
+	{ 1, 1, 1, 1, APIC_LVT_DM_EXTINT, 0 },	/* LINT0: masked ExtINT */
+	{ 1, 1, 0, 1, APIC_LVT_DM_NMI, 0 },	/* LINT1: NMI */
+	{ 1, 1, 1, 1, APIC_LVT_DM_FIXED, 0 },	/* Timer: needs a vector */
+	{ 1, 1, 1, 1, APIC_LVT_DM_FIXED, 0 },	/* Error: needs a vector */
+	{ 1, 1, 1, 1, APIC_LVT_DM_FIXED, 0 },	/* PMC */
+	{ 1, 1, 1, 1, APIC_LVT_DM_FIXED, 0 },	/* Thermal: needs a vector */
+};
+
+static inthand_t *ioint_handlers[] = {
+	NULL,			/* 0 - 31 */
+	IDTVEC(apic_isr1),	/* 32 - 63 */
+	IDTVEC(apic_isr2),	/* 64 - 95 */
+	IDTVEC(apic_isr3),	/* 96 - 127 */
+	IDTVEC(apic_isr4),	/* 128 - 159 */
+	IDTVEC(apic_isr5),	/* 160 - 191 */
+	NULL,			/* 192 - 223 */
+	NULL			/* 224 - 255 */
+};
+
+volatile lapic_t *lapic;
+
+static uint32_t
+lvt_mode(struct lapic *la, u_int pin, uint32_t value)
+{
+	struct lvt *lvt;
+
+	KASSERT(pin <= LVT_MAX, ("%s: pin %u out of range", __func__, pin));
+	if (la->la_lvts[pin].lvt_active)
+		lvt = &la->la_lvts[pin];
+	else
+		lvt = &lvts[pin];
+
+	value &= ~(APIC_LVT_M | APIC_LVT_TM | APIC_LVT_IIPP | APIC_LVT_DM |
+	    APIC_LVT_VECTOR);
+	if (lvt->lvt_edgetrigger == 0)
+		value |= APIC_LVT_TM;
+	if (lvt->lvt_activehi == 0)
+		value |= APIC_LVT_IIPP_INTALO;
+	if (lvt->lvt_masked)
+		value |= APIC_LVT_M;
+	value |= lvt->lvt_mode;
+	switch (lvt->lvt_mode) {
+	case APIC_LVT_DM_NMI:
+	case APIC_LVT_DM_SMI:
+	case APIC_LVT_DM_INIT:
+	case APIC_LVT_DM_EXTINT:
+		if (!lvt->lvt_edgetrigger) {
+			printf("lapic%u: Forcing LINT%u to edge trigger\n",
+			    la->la_id, pin);
+			value |= APIC_LVT_TM;
+		}
+		/* Use a vector of 0. */
+		break;
+	case APIC_LVT_DM_FIXED:
+#if 0
+		value |= lvt->lvt_vector;
+#else
+		panic("Fixed LINT pins not supported");
+#endif
+		break;
+	default:
+		panic("bad APIC LVT delivery mode: %#x\n", value);
+	}
+	return (value);
+}
+
+/*
+ * Map the local APIC and setup necessary interrupt vectors.
+ */
+void
+lapic_init(uintptr_t addr)
+{
+	u_int32_t value;
+
+	/* Map the local APIC and setup the spurious interrupt handler. */
+	KASSERT(trunc_page(addr) == addr,
+	    ("local APIC not aligned on a page boundary"));
+	lapic = (lapic_t *)pmap_mapdev(addr, sizeof(lapic_t));
+	setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_SYS386IGT, SEL_KPL,
+	    GSEL(GCODE_SEL, SEL_KPL));
+
+	/* Perform basic initialization of the BSP's local APIC. */
+	value = lapic->svr;
+	value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS);
+	value |= (APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT);
+	lapic->svr = value;
+
+	/* Set BSP's per-CPU local APIC ID. */
+	PCPU_SET(apic_id, lapic_id());
+
+	/* XXX: timer/error/thermal interrupts */
+}
+
+/*
+ * Create a local APIC instance.
+ */
+void
+lapic_create(u_int apic_id, int boot_cpu)
+{
+	int i;
+
+	if (apic_id > MAX_APICID) {
+		printf("APIC: Ignoring local APIC with ID %d\n", apic_id);
+		if (boot_cpu)
+			panic("Can't ignore BSP");
+		return;
+	}
+	KASSERT(!lapics[apic_id].la_present, ("duplicate local APIC %u",
+	    apic_id));
+
+	/*
+	 * Assume no local LVT overrides and a cluster of 0 and
+	 * intra-cluster ID of 0.
+	 */
+	lapics[apic_id].la_present = 1;
+	lapics[apic_id].la_id = apic_id;
+	for (i = 0; i < LVT_MAX; i++) {
+		lapics[apic_id].la_lvts[i] = lvts[i];
+		lapics[apic_id].la_lvts[i].lvt_active = 0;
+	}
+
+#ifdef SMP
+	cpu_add(apic_id, boot_cpu);
+#endif
+}
+
+/*
+ * Dump contents of local APIC registers
+ */
+void
+lapic_dump(const char* str)
+{
+
+	printf("cpu%d %s:\n", PCPU_GET(cpuid), str);
+	printf("     ID: 0x%08x   VER: 0x%08x LDR: 0x%08x DFR: 0x%08x\n",
+	    lapic->id, lapic->version, lapic->ldr, lapic->dfr);
+	printf("  lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n",
+	    lapic->lvt_lint0, lapic->lvt_lint1, lapic->tpr, lapic->svr);
+}
+
+void
+lapic_enable_intr(u_int irq)
+{
+	u_int vector;
+
+	vector = apic_irq_to_idt(irq);
+	KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
+	KASSERT(ioint_handlers[vector / 32] != NULL,
+	    ("No ISR handler for IRQ %u", irq));
+	setidt(vector, ioint_handlers[vector / 32], SDT_SYS386IGT, SEL_KPL,
+	    GSEL(GCODE_SEL, SEL_KPL));
+}
+
+void
+lapic_setup(void)
+{
+	struct lapic *la;
+	u_int32_t value, maxlvt;
+	register_t eflags;
+
+	la = &lapics[lapic_id()];
+	KASSERT(la->la_present, ("missing APIC structure"));
+	eflags = intr_disable();
+	maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
+
+	/* Program LINT[01] LVT entries. */
+	lapic->lvt_lint0 = lvt_mode(la, LVT_LINT0, lapic->lvt_lint0);
+	lapic->lvt_lint1 = lvt_mode(la, LVT_LINT1, lapic->lvt_lint1);
+
+	/* XXX: more LVT entries */
+
+	/* Clear the TPR. */
+	value = lapic->tpr;
+	value &= ~APIC_TPR_PRIO;
+	lapic->tpr = value;
+
+	/* Use the cluster model for logical IDs. */
+	value = lapic->dfr;
+	value &= ~APIC_DFR_MODEL_MASK;
+	value |= APIC_DFR_MODEL_CLUSTER;
+	lapic->dfr = value;
+
+	/* Set this APIC's logical ID. */
+	value = lapic->ldr;
+	value &= ~APIC_ID_MASK;
+	value |= (la->la_cluster << APIC_ID_CLUSTER_SHIFT |
+	    1 << la->la_cluster_id) << APIC_ID_SHIFT;
+	lapic->ldr = value;
+
+	/* Setup spurious vector and enable the local APIC. */
+	value = lapic->svr;
+	value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS);
+	value |= (APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT);
+	lapic->svr = value;
+	intr_restore(eflags);
+}
+
+void
+lapic_disable(void)
+{
+	uint32_t value;
+
+	/* Software disable the local APIC. */
+	value = lapic->svr;
+	value &= ~APIC_SVR_SWEN;
+	lapic->svr = value;
+}
+
+int
+lapic_id(void)
+{
+
+	KASSERT(lapic != NULL, ("local APIC is not mapped"));
+	return (lapic->id >> APIC_ID_SHIFT);
+}
+
+int
+lapic_intr_pending(u_int vector)
+{
+	volatile u_int32_t *irr;
+
+	/*
+	 * The IRR registers are an array of 128-bit registers each of
+	 * which only describes 32 interrupts in the low 32 bits..  Thus,
+	 * we divide the vector by 32 to get the 128-bit index.  We then
+	 * multiply that index by 4 to get the equivalent index from
+	 * treating the IRR as an array of 32-bit registers.  Finally, we
+	 * modulus the vector by 32 to determine the individual bit to
+	 * test.
+	 */
+	irr = &lapic->irr0;
+	return (irr[(vector / 32) * 4] & 1 << (vector % 32));
+}
+
+void
+lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id)
+{
+	struct lapic *la;
+
+	KASSERT(lapics[apic_id].la_present, ("%s: APIC %u doesn't exist",
+	    __func__, apic_id));
+	KASSERT(cluster <= APIC_MAX_CLUSTER, ("%s: cluster %u too big",
+	    __func__, cluster));
+	KASSERT(cluster_id <= APIC_MAX_INTRACLUSTER_ID,
+	    ("%s: intra cluster id %u too big", __func__, cluster_id));
+	la = &lapics[apic_id];
+	la->la_cluster = cluster;
+	la->la_cluster_id = cluster_id;
+}
+
+int
+lapic_set_lvt_mask(u_int apic_id, u_int pin, u_char masked)
+{
+
+	if (pin > LVT_MAX)
+		return (EINVAL);
+	if (apic_id == APIC_ID_ALL) {
+		lvts[pin].lvt_masked = masked;
+		if (bootverbose)
+			printf("lapic:");
+	} else {
+		KASSERT(lapics[apic_id].la_present,
+		    ("%s: missing APIC %u", __func__, apic_id));
+		lapics[apic_id].la_lvts[pin].lvt_masked = masked;
+		lapics[apic_id].la_lvts[pin].lvt_active = 1;
+		if (bootverbose)
+			printf("lapic%u:", apic_id);
+	}
+	if (bootverbose)
+		printf(" LINT%u %s\n", pin, masked ? "masked" : "unmasked");
+	return (0);
+}
+
+int
+lapic_set_lvt_mode(u_int apic_id, u_int pin, u_int32_t mode)
+{
+	struct lvt *lvt;
+
+	if (pin > LVT_MAX)
+		return (EINVAL);
+	if (apic_id == APIC_ID_ALL) {
+		lvt = &lvts[pin];
+		if (bootverbose)
+			printf("lapic:");
+	} else {
+		KASSERT(lapics[apic_id].la_present,
+		    ("%s: missing APIC %u", __func__, apic_id));
+		lvt = &lapics[apic_id].la_lvts[pin];
+		lvt->lvt_active = 1;
+		if (bootverbose)
+			printf("lapic%u:", apic_id);
+	}
+	lvt->lvt_mode = mode;
+	switch (mode) {
+	case APIC_LVT_DM_NMI:
+	case APIC_LVT_DM_SMI:
+	case APIC_LVT_DM_INIT:
+	case APIC_LVT_DM_EXTINT:
+		lvt->lvt_edgetrigger = 1;
+		lvt->lvt_activehi = 1;
+		if (mode == APIC_LVT_DM_EXTINT)
+			lvt->lvt_masked = 1;
+		else
+			lvt->lvt_masked = 0;
+		break;
+	default:
+		panic("Unsupported delivery mode: 0x%x\n", mode);
+	}
+	if (bootverbose) {
+		printf(" Routing ");
+		switch (mode) {
+		case APIC_LVT_DM_NMI:
+			printf("NMI");
+			break;
+		case APIC_LVT_DM_SMI:
+			printf("SMI");
+			break;
+		case APIC_LVT_DM_INIT:
+			printf("INIT");
+			break;
+		case APIC_LVT_DM_EXTINT:
+			printf("ExtINT");
+			break;
+		}
+		printf(" -> LINT%u\n", pin);
+	}
+	return (0);
+}
+
+int
+lapic_set_lvt_polarity(u_int apic_id, u_int pin, u_char activehi)
+{
+
+	if (pin > LVT_MAX)
+		return (EINVAL);
+	if (apic_id == APIC_ID_ALL) {
+		lvts[pin].lvt_activehi = activehi;
+		if (bootverbose)
+			printf("lapic:");
+	} else {
+		KASSERT(lapics[apic_id].la_present,
+		    ("%s: missing APIC %u", __func__, apic_id));
+		lapics[apic_id].la_lvts[pin].lvt_active = 1;
+		lapics[apic_id].la_lvts[pin].lvt_activehi = activehi;
+		if (bootverbose)
+			printf("lapic%u:", apic_id);
+	}
+	if (bootverbose)
+		printf(" LINT%u polarity: active-%s\n", pin,
+		    activehi ? "hi" : "lo");
+	return (0);
+}
+
+int
+lapic_set_lvt_triggermode(u_int apic_id, u_int pin, u_char edgetrigger)
+{
+
+	if (pin > LVT_MAX)
+		return (EINVAL);
+	if (apic_id == APIC_ID_ALL) {
+		lvts[pin].lvt_edgetrigger = edgetrigger;
+		if (bootverbose)
+			printf("lapic:");
+	} else {
+		KASSERT(lapics[apic_id].la_present,
+		    ("%s: missing APIC %u", __func__, apic_id));
+		lapics[apic_id].la_lvts[pin].lvt_edgetrigger = edgetrigger;
+		lapics[apic_id].la_lvts[pin].lvt_active = 1;
+		if (bootverbose)
+			printf("lapic%u:", apic_id);
+	}
+	if (bootverbose)
+		printf(" LINT%u trigger: %s\n", pin,
+		    edgetrigger ? "edge" : "level");
+	return (0);
+}
+
+void
+lapic_handle_intr(struct intrframe frame)
+{
+	struct intsrc *isrc;
+
+	if (frame.if_vec == -1)
+		panic("Couldn't get vector from ISR!");
+	isrc = intr_lookup_source(apic_idt_to_irq(frame.if_vec));
+	isrc->is_pic->pic_disable_source(isrc);
+	lapic->eoi = 0;
+	intr_execute_handlers(isrc, &frame);
+}
+
+/* Translate between IDT vectors and IRQ vectors. */
+u_int
+apic_irq_to_idt(u_int irq)
+{
+	u_int vector;
+
+	KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq));
+	vector = irq + IDT_IO_INTS;
+	if (vector >= IDT_SYSCALL)
+		vector++;
+	return (vector);
+}
+
+u_int
+apic_idt_to_irq(u_int vector)
+{
+
+	KASSERT(vector >= IDT_IO_INTS && vector != IDT_SYSCALL &&
+	    vector <= IDT_IO_INTS + NUM_IO_INTS,
+	    ("Vector %u does not map to an IRQ line", vector));
+	if (vector > IDT_SYSCALL)
+		vector--;
+	return (vector - IDT_IO_INTS);
+}
+
+/*
+ * APIC probing support code.  This includes code to manage enumerators.
+ */
+
+static SLIST_HEAD(, apic_enumerator) enumerators =
+	SLIST_HEAD_INITIALIZER(enumerators);
+static struct apic_enumerator *best_enum;
+	
+void
+apic_register_enumerator(struct apic_enumerator *enumerator)
+{
+#ifdef INVARIANTS
+	struct apic_enumerator *apic_enum;
+
+	SLIST_FOREACH(apic_enum, &enumerators, apic_next) {
+		if (apic_enum == enumerator)
+			panic("%s: Duplicate register of %s", __func__,
+			    enumerator->apic_name);
+	}
+#endif
+	SLIST_INSERT_HEAD(&enumerators, enumerator, apic_next);
+}
+
+/*
+ * We have to look for CPU's very, very early because certain subsystems
+ * want to know how many CPU's we have extremely early on in the boot
+ * process.
+ */
+static void
+apic_init(void *dummy __unused)
+{
+	struct apic_enumerator *enumerator;
+	int retval, best;
+
+	/* We only support built in local APICs. */
+	if (!(cpu_feature & CPUID_APIC))
+		return;
+
+	/* First, probe all the enumerators to find the best match. */
+	best_enum = NULL;
+	best = 0;
+	SLIST_FOREACH(enumerator, &enumerators, apic_next) {
+		retval = enumerator->apic_probe();
+		if (retval > 0)
+			continue;
+		if (best_enum == NULL || best < retval) {
+			best_enum = enumerator;
+			best = retval;
+		}
+	}
+	if (best_enum == NULL) {
+		if (bootverbose)
+			printf("APIC: Could not find any APICs.\n");
+		return;
+	}
+
+	if (bootverbose)
+		printf("APIC: Using the %s enumerator.\n",
+		    best_enum->apic_name);
+
+	/* Second, probe the CPU's in the system. */
+	retval = best_enum->apic_probe_cpus();
+	if (retval != 0)
+		printf("%s: Failed to probe CPUs: returned %d\n",
+		    best_enum->apic_name, retval);
+}
+SYSINIT(apic_init, SI_SUB_TUNABLES - 1, SI_ORDER_SECOND, apic_init, NULL)
+
+/*
+ * Setup the local APIC.  We have to do this prior to starting up the APs
+ * in the SMP case.
+ */
+static void
+apic_setup_local(void *dummy __unused)
+{
+	int retval;
+	uint64_t apic_base;
+
+	if (best_enum == NULL)
+		return;
+	/*
+	 * To work around an errata, we disable the local APIC on some
+	 * CPUs during early startup.  We need to turn the local APIC back
+	 * on on such CPUs now.
+	 */
+	if (cpu == CPU_686 && strcmp(cpu_vendor, "GenuineIntel") == 0 &&
+	    (cpu_id & 0xff0) == 0x610) {
+		apic_base = rdmsr(MSR_APICBASE);
+		apic_base |= APICBASE_ENABLED;
+		wrmsr(MSR_APICBASE, apic_base);
+	}
+	retval = best_enum->apic_setup_local();
+	if (retval != 0)
+		printf("%s: Failed to setup the local APIC: returned %d\n",
+		    best_enum->apic_name, retval);
+}
+SYSINIT(apic_setup_local, SI_SUB_CPU, SI_ORDER_FIRST, apic_setup_local, NULL)
+
+/*
+ * Setup the I/O APICs.
+ */
+static void
+apic_setup_io(void *dummy __unused)
+{
+	int retval;
+
+	if (best_enum == NULL)
+		return;
+	retval = best_enum->apic_setup_io();
+	if (retval != 0)
+		printf("%s: Failed to setup I/O APICs: returned %d\n",
+		    best_enum->apic_name, retval);
+
+	/*
+	 * Finish setting up the local APIC on the BSP once we know how to
+	 * properly program the LINT pins.
+	 */
+	lapic_setup();
+	if (bootverbose)
+		lapic_dump("BSP");
+}
+SYSINIT(apic_setup_io, SI_SUB_INTR, SI_ORDER_SECOND, apic_setup_io, NULL)
+
+#ifdef SMP
+/*
+ * Inter Processor Interrupt functions.  The lapic_ipi_*() functions are
+ * private the sys/i386 code.  The public interface for the rest of the
+ * kernel is defined in mp_machdep.c.
+ */
+#define DETECT_DEADLOCK
+
+int
+lapic_ipi_wait(int delay)
+{
+	int x, incr;
+
+	/*
+	 * Wait delay loops for IPI to be sent.  This is highly bogus
+	 * since this is sensitive to CPU clock speed.  If delay is
+	 * -1, we wait forever.
+	 */
+	if (delay == -1) {
+		incr = 0;
+		delay = 1;
+	} else
+		incr = 1;
+	for (x = 0; x < delay; x += incr) {
+		if ((lapic->icr_lo & APIC_DELSTAT_MASK) == APIC_DELSTAT_IDLE)
+			return (1);
+		ia32_pause();
+	}
+	return (0);
+}
+
+void
+lapic_ipi_raw(register_t icrlo, u_int dest)
+{
+	register_t value, eflags;
+
+	/* XXX: Need more sanity checking of icrlo? */
+	KASSERT(lapic != NULL, ("%s called too early", __func__));
+	KASSERT((dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
+	    ("%s: invalid dest field", __func__));
+	KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0,
+	    ("%s: reserved bits set in ICR LO register", __func__));
+
+	/* Set destination in ICR HI register if it is being used. */
+	eflags = intr_disable();
+	if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) {
+		value = lapic->icr_hi;
+		value &= ~APIC_ID_MASK;
+		value |= dest << APIC_ID_SHIFT;
+		lapic->icr_hi = value;
+	}
+
+	/* Program the contents of the IPI and dispatch it. */
+	value = lapic->icr_lo;
+	value &= APIC_ICRLO_RESV_MASK;
+	value |= icrlo;
+	lapic->icr_lo = value;
+	intr_restore(eflags);
+}
+
+#ifdef DETECT_DEADLOCK
+#define	BEFORE_SPIN	1000000
+#define	AFTER_SPIN	1000
+#endif
+
+void
+lapic_ipi_vectored(u_int vector, int dest)
+{
+	register_t icrlo, destfield;
+
+	KASSERT((vector & ~APIC_VECTOR_MASK) == 0,
+	    ("%s: invalid vector %d", __func__, vector));
+
+	icrlo = vector | APIC_DELMODE_FIXED | APIC_DESTMODE_PHY |
+	    APIC_LEVEL_DEASSERT | APIC_TRIGMOD_EDGE;
+	destfield = 0;
+	switch (dest) {
+	case APIC_IPI_DEST_SELF:
+		icrlo |= APIC_DEST_SELF;
+		break;
+	case APIC_IPI_DEST_ALL:
+		icrlo |= APIC_DEST_ALLISELF;
+		break;
+	case APIC_IPI_DEST_OTHERS:
+		icrlo |= APIC_DEST_ALLESELF;
+		break;
+	default:
+		KASSERT((dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
+		    ("%s: invalid destination 0x%x", __func__, dest));
+		destfield = dest;
+	}
+
+#ifdef DETECT_DEADLOCK
+	/* Check for an earlier stuck IPI. */
+	if (!lapic_ipi_wait(BEFORE_SPIN))
+		panic("APIC: Previous IPI is stuck");
+#endif
+
+	lapic_ipi_raw(icrlo, destfield);
+
+#ifdef DETECT_DEADLOCK
+	/* Wait for IPI to be delivered. */
+	if (!lapic_ipi_wait(AFTER_SPIN)) {
+#ifdef needsattention
+		/*
+		 * XXX FIXME:
+		 *
+		 * The above function waits for the message to actually be
+		 * delivered.  It breaks out after an arbitrary timeout
+		 * since the message should eventually be delivered (at
+		 * least in theory) and that if it wasn't we would catch
+		 * the failure with the check above when the next IPI is
+		 * sent.
+		 *
+		 * We could skiip this wait entirely, EXCEPT it probably
+		 * protects us from other routines that assume that the
+		 * message was delivered and acted upon when this function
+		 * returns.
+		 */
+		printf("APIC: IPI might be stuck\n");
+#else /* !needsattention */
+		/* Wait until mesage is sent without a timeout. */
+		while (lapic->icr_lo & APIC_DELSTAT_PEND)
+			ia32_pause();
+#endif /* needsattention */
+	}
+#endif /* DETECT_DEADLOCK */
+}
+#endif /* SMP */

sys/i386/i386/locore.s

@@ -82,9 +82,8 @@
  * This is "constructed" in locore.s on the BSP and in mp_machdep.c
  * for each AP.  DO NOT REORDER THESE WITHOUT UPDATING THE REST!
  */
-	.globl	SMP_prvspace, lapic
+	.globl	SMP_prvspace
 	.set	SMP_prvspace,(MPPTDI << PDRSHIFT)
-	.set	lapic,SMP_prvspace + (NPTEPG-1) * PAGE_SIZE
 #endif /* SMP */
 
 /*

sys/i386/i386/trap.c

@@ -82,6 +82,7 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_extern.h>
 
 #include <machine/cpu.h>
+#include <machine/intr_machdep.h>
 #include <machine/md_var.h>
 #include <machine/pcb.h>
 #ifdef SMP
@@ -90,9 +91,6 @@ __FBSDID("$FreeBSD$");
 #include <machine/tss.h>
 #include <machine/vm86.h>
 
-#include <i386/isa/icu.h>
-#include <i386/isa/intr_machdep.h>
-
 #ifdef POWERFAIL_NMI
 #include <sys/syslog.h>
 #include <machine/clock.h>
@@ -764,7 +762,7 @@ trap_fatal(frame, eva)
 #ifdef SMP
 	/* two separate prints in case of a trap on an unmapped page */
 	printf("cpuid = %d; ", PCPU_GET(cpuid));
-	printf("lapic.id = %08x\n", lapic.id);
+	printf("apic id = %02x\n", PCPU_GET(apic_id));
 #endif
 	if (type == T_PAGEFLT) {
 		printf("fault virtual address	= 0x%x\n", eva);
@@ -847,7 +845,7 @@ dblfault_handler()
 #ifdef SMP
 	/* two separate prints in case of a trap on an unmapped page */
 	printf("cpuid = %d; ", PCPU_GET(cpuid));
-	printf("lapic.id = %08x\n", lapic.id);
+	printf("apic id = %02x\n", PCPU_GET(apic_id));
 #endif
 	panic("double fault");
 }

sys/i386/include/apicreg.h

@@ -25,8 +25,8 @@
  * $FreeBSD$
  */
 
-#ifndef _MACHINE_APIC_H_
-#define _MACHINE_APIC_H_
+#ifndef _MACHINE_APICREG_H_
+#define _MACHINE_APICREG_H_
 
 /*
  * Local && I/O APIC definitions.
@@ -221,11 +221,29 @@ typedef struct IOAPIC ioapic_t;
 /* default physical locations of LOCAL (CPU) APICs */
 #define DEFAULT_APIC_BASE	0xfee00000
 
+/* constants relating to APIC ID registers */
+#define APIC_ID_MASK		0xff000000
+#define	APIC_ID_SHIFT		24
+#define	APIC_ID_CLUSTER		0xf0
+#define	APIC_ID_CLUSTER_ID	0x0f
+#define	APIC_MAX_CLUSTER	0xe
+#define	APIC_MAX_INTRACLUSTER_ID 3
+#define	APIC_ID_CLUSTER_SHIFT	4
+
 /* fields in VER */
 #define APIC_VER_VERSION	0x000000ff
 #define APIC_VER_MAXLVT		0x00ff0000
 #define MAXLVTSHIFT		16
 
+/* fields in LDR */
+#define	APIC_LDR_RESERVED	0x00ffffff
+
+/* fields in DFR */
+#define	APIC_DFR_RESERVED	0x0fffffff
+#define	APIC_DFR_MODEL_MASK	0xf0000000
+#define	APIC_DFR_MODEL_FLAT	0xf0000000
+#define	APIC_DFR_MODEL_CLUSTER	0x00000000
+
 /* fields in SVR */
 #define APIC_SVR_VECTOR		0x000000ff
 #define APIC_SVR_VEC_PROG	0x000000f0
@@ -290,10 +308,6 @@ typedef struct IOAPIC ioapic_t;
 
 #define	APIC_ICRLO_RESV_MASK	(APIC_RESV1_MASK | APIC_RESV2_MASK)
 
-/* fields in ICR_HIGH */
-#define APIC_ID_MASK		0xff000000
-#define	APIC_ID_SHIFT		24
-
 /* fields in LVT1/2 */
 #define APIC_LVT_VECTOR		0x000000ff
 #define APIC_LVT_DM		0x00000700
@@ -444,4 +458,4 @@ typedef struct IOAPIC ioapic_t;
 
 #define IOART_INTVEC	0x000000ff	/* R/W: INTerrupt vector field */
 
-#endif /* _MACHINE_APIC_H_ */
+#endif /* _MACHINE_APICREG_H_ */

sys/i386/include/apicvar.h

@@ -0,0 +1,165 @@
+/*-
+ * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the author nor the names of any co-contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_APICVAR_H_
+#define _MACHINE_APICVAR_H_
+
+/*
+ * Local && I/O APIC variable definitions.
+ */
+
+/*
+ * Layout of local APIC interrupt vectors:
+ *
+ *	0xff (255)  +-------------+
+ *                  |             | 15 (Spurios Vector)
+ *	0xf0 (240)  +-------------+
+ *                  |             | 14 (Interprocessor Interrupts)
+ *	0xe0 (224)  +-------------+
+ *                  |             | 13 (Local Interrupt (LINT[01]))
+ *	0xd0 (208)  +-------------+
+ *                  |             | 12 (Local Timer and Error Interrupts)
+ *	0xc0 (192)  +-------------+
+ *                  |             | 11 (I/O Interrupts)
+ *	0xb0 (176)  +-------------+
+ *                  |             | 10 (I/O Interrupts)
+ *	0xa0 (160)  +-------------+
+ *                  |             | 9 (I/O Interrupts)
+ *	0x90 (144)  +-------------+
+ *                  |             | 8 (I/O Interrupts / System Calls)
+ *	0x80 (128)  +-------------+
+ *                  |             | 7 (I/O Interrupts)
+ *	0x70 (112)  +-------------+
+ *                  |             | 6 (I/O Interrupts)
+ *	0x60 (96)   +-------------+
+ *                  |             | 5 (I/O Interrupts)
+ *	0x50 (80)   +-------------+
+ *                  |             | 4 (I/O Interrupts)
+ *	0x40 (64)   +-------------+
+ *                  |             | 3 (I/O Interrupts)
+ *	0x30 (48)   +-------------+
+ *                  |             | 2 (I/O Interrupts)
+ *	0x20 (32)   +-------------+
+ *                  |             | 1 (Exceptions, traps, faults, etc.)
+ *	0x10 (16)   +-------------+
+ *                  |             | 0 (Exceptions, traps, faults, etc.)
+ *	0x00 (0)    +-------------+
+ *
+ * Note: 0x80 needs to be handled specially and not allocated to an
+ * I/O device!
+ */
+
+#define	APIC_ID_ALL	0xff
+#define	APIC_NUM_IOINTS	160
+
+#define	APIC_LOCAL_INTS	(IDT_IO_INTS + APIC_NUM_IOINTS)
+#define	APIC_TIMER_INT	APIC_LOCAL_INTS
+#define	APIC_ERROR_INT	(APIC_LOCAL_INTS + 1)
+#define	APIC_THERMAL_INT (APIC_LOCAL_INTS + 2)
+
+#define	APIC_IPI_INTS	(APIC_LOCAL_INTS + 32)
+#define	IPI_AST		APIC_IPI_INTS		/* Generate software trap. */
+#define	IPI_INVLTLB	(APIC_IPI_INTS + 1)	/* TLB Shootdown IPIs */
+#define	IPI_INVLPG	(APIC_IPI_INTS + 2)
+#define	IPI_INVLRNG	(APIC_IPI_INTS + 3)
+#define	IPI_HARDCLOCK	(APIC_IPI_INTS + 8)	/* Inter-CPU clock handling. */
+#define	IPI_STATCLOCK	(APIC_IPI_INTS + 9)
+#define	IPI_RENDEZVOUS	(APIC_IPI_INTS + 10)	/* Inter-CPU rendezvous. */
+#define	IPI_LAZYPMAP	(APIC_IPI_INTS + 11)	/* Lazy pmap release. */
+#define	IPI_STOP	(APIC_IPI_INTS + 12)	/* Stop CPU until restarted. */
+
+#define	APIC_SPURIOUS_INT 255
+
+#define	LVT_LINT0	0
+#define	LVT_LINT1	1
+#define	LVT_TIMER	2
+#define	LVT_ERROR	3
+#define	LVT_PMC		4
+#define	LVT_THERMAL	5
+#define	LVT_MAX		LVT_THERMAL
+
+#ifndef LOCORE
+
+#define	APIC_IPI_DEST_SELF	-1
+#define	APIC_IPI_DEST_ALL	-2
+#define	APIC_IPI_DEST_OTHERS	-3
+
+/*
+ * An APIC enumerator is a psuedo bus driver that enumerates APIC's including
+ * CPU's and I/O APIC's.
+ */
+struct apic_enumerator {
+	const char *apic_name;
+	int (*apic_probe)(void);
+	int (*apic_probe_cpus)(void);
+	int (*apic_setup_local)(void);
+	int (*apic_setup_io)(void);
+	SLIST_ENTRY(apic_enumerator) apic_next;
+};
+
+inthand_t
+	IDTVEC(apic_isr1), IDTVEC(apic_isr2), IDTVEC(apic_isr3),
+	IDTVEC(apic_isr4), IDTVEC(apic_isr5), IDTVEC(spuriousint);
+
+u_int	apic_irq_to_idt(u_int irq);
+u_int	apic_idt_to_irq(u_int vector);
+void	apic_register_enumerator(struct apic_enumerator *enumerator);
+void	*ioapic_create(uintptr_t addr, int32_t id, int intbase);
+int	ioapic_disable_pin(void *cookie, u_int pin);
+int	ioapic_get_vector(void *cookie, u_int pin);
+int	ioapic_next_logical_cluster(void);
+void	ioapic_register(void *cookie);
+int	ioapic_remap_vector(void *cookie, u_int pin, int vector);
+int	ioapic_set_extint(void *cookie, u_int pin);
+int	ioapic_set_nmi(void *cookie, u_int pin);
+int	ioapic_set_polarity(void *cookie, u_int pin, char activehi);
+int	ioapic_set_triggermode(void *cookie, u_int pin, char edgetrigger);
+int	ioapic_set_smi(void *cookie, u_int pin);
+void	lapic_create(u_int apic_id, int boot_cpu);
+void	lapic_disable(void);
+void	lapic_dump(const char *str);
+void	lapic_enable_intr(u_int vector);
+int	lapic_id(void);
+void	lapic_init(uintptr_t addr);
+int	lapic_intr_pending(u_int vector);
+void	lapic_ipi_raw(register_t icrlo, u_int dest);
+void	lapic_ipi_vectored(u_int vector, int dest);
+int	lapic_ipi_wait(int delay);
+void	lapic_handle_intr(struct intrframe frame);
+void	lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id);
+int	lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked);
+int	lapic_set_lvt_mode(u_int apic_id, u_int lvt, u_int32_t mode);
+int	lapic_set_lvt_polarity(u_int apic_id, u_int lvt, u_char activehi);
+int	lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, u_char edgetrigger);
+void	lapic_setup(void);
+
+#endif /* !LOCORE */
+#endif /* _MACHINE_APICVAR_H_ */

sys/i386/include/clock.h

@@ -24,9 +24,6 @@ extern int	timer0_max_count;
 extern uint64_t	tsc_freq;
 extern int	tsc_is_broken;
 extern int	wall_cmos_clock;
-#ifdef APIC_IO
-extern int	apic_8254_intr;
-#endif
 
 /*
  * Driver to clock driver interface.

sys/i386/include/pcpu.h

@@ -48,10 +48,8 @@
 	struct	segment_descriptor pc_common_tssd;			\
 	struct	segment_descriptor *pc_tss_gdt;				\
 	int	pc_currentldt;						\
-	u_int32_t pc_int_pending;	/* master int pending flag */   \
-	u_int32_t pc_ipending;	/* pending slow interrupts */		\
-	u_int32_t pc_fpending;	/* pending fast interrupts */		\
-	u_int32_t pc_spending 	/* pending soft interrupts */
+	u_int	pc_acpi_id;						\
+	u_int	pc_apic_id;
 
 #if defined(lint)
  

sys/i386/isa/clock.c

@@ -71,14 +71,12 @@ __FBSDID("$FreeBSD$");
 #include <machine/clock.h>
 #include <machine/cputypes.h>
 #include <machine/frame.h>
+#include <machine/intr_machdep.h>
 #include <machine/md_var.h>
 #include <machine/psl.h>
-#ifdef APIC_IO
-#include <machine/segments.h>
-#endif
-#if defined(SMP) || defined(APIC_IO)
+#if defined(SMP)
 #include <machine/smp.h>
-#endif /* SMP || APIC_IO */
+#endif
 #include <machine/specialreg.h>
 
 #include <i386/isa/icu.h>
@@ -89,20 +87,10 @@ __FBSDID("$FreeBSD$");
 #endif
 #include <i386/isa/timerreg.h>
 
-#include <i386/isa/intr_machdep.h>
-
 #ifdef DEV_MCA
 #include <i386/bios/mca_machdep.h>
 #endif
 
-#ifdef APIC_IO
-#include <i386/isa/intr_machdep.h>
-/* The interrupt triggered by the 8254 (timer) chip */
-int apic_8254_intr;
-static u_long read_intr_count(int vec);
-static void setup_8254_mixed_mode(void);
-#endif
-
 /*
  * 32-bit time_t's can't reach leap years before 1904 or after 2036, so we
  * can use a simple formula for leap years.
@@ -150,6 +138,7 @@ static	u_int	hardclock_max_count;
 static	u_int32_t i8254_lastcount;
 static	u_int32_t i8254_offset;
 static	int	i8254_ticked;
+static	struct intsrc *i8254_intsrc;
 #ifndef BURN_BRIDGES
 /*
  * XXX new_function and timer_func should not handle clockframes, but
@@ -187,7 +176,7 @@ static struct timecounter i8254_timecounter = {
 };
 
 static void
-clkintr(struct clockframe frame)
+clkintr(struct clockframe *frame)
 {
 
 	if (timecounter->tc_get_timecount == i8254_get_timecount) {
@@ -201,7 +190,7 @@ clkintr(struct clockframe frame)
 		clkintr_pending = 0;
 		mtx_unlock_spin(&clock_lock);
 	}
-	timer_func(&frame);
+	timer_func(frame);
 #ifdef SMP
 	if (timer_func == hardclock)
 		forward_hardclock();
@@ -216,7 +205,7 @@ clkintr(struct clockframe frame)
 		if ((timer0_prescaler_count += timer0_max_count)
 		    >= hardclock_max_count) {
 			timer0_prescaler_count -= hardclock_max_count;
-			hardclock(&frame);
+			hardclock(frame);
 #ifdef SMP
 			forward_hardclock();
 #endif
@@ -251,7 +240,7 @@ clkintr(struct clockframe frame)
 			timer0_prescaler_count = 0;
 			timer_func = hardclock;
 			timer0_state = RELEASED;
-			hardclock(&frame);
+			hardclock(frame);
 #ifdef SMP
 			forward_hardclock();
 #endif
@@ -379,16 +368,16 @@ release_timer2()
  * in the statistics, but the stat clock will no longer stop.
  */
 static void
-rtcintr(struct clockframe frame)
+rtcintr(struct clockframe *frame)
 {
 	while (rtcin(RTC_INTR) & RTCIR_PERIOD) {
 		if (profprocs != 0) {
 			if (--pscnt == 0)
 				pscnt = psdiv;
-			profclock(&frame);
+			profclock(frame);
 		}
 		if (pscnt == psdiv)
-			statclock(&frame);
+			statclock(frame);
 #ifdef SMP
 		forward_statclock();
 #endif
@@ -931,11 +920,6 @@ void
 cpu_initclocks()
 {
 	int diag;
-#ifdef APIC_IO
-	int apic_8254_trial;
-	void *clkdesc;
-#endif /* APIC_IO */
-	register_t crit;
 
 	if (statclock_disable) {
 		/*
@@ -951,47 +935,9 @@ cpu_initclocks()
 		profhz = RTC_PROFRATE;
         }
 
-	/* Finish initializing 8253 timer 0. */
-#ifdef APIC_IO
-
-	apic_8254_intr = isa_apic_irq(0);
-	apic_8254_trial = 0;
-	if (apic_8254_intr >= 0 ) {
-		if (apic_int_type(0, 0) == 3)
-			apic_8254_trial = 1;
-	} else {
-		/* look for ExtInt on pin 0 */
-		if (apic_int_type(0, 0) == 3) {
-			apic_8254_intr = apic_irq(0, 0);
-			setup_8254_mixed_mode();
-		} else 
-			panic("APIC_IO: Cannot route 8254 interrupt to CPU");
-	}
-
-	inthand_add("clk", apic_8254_intr, (driver_intr_t *)clkintr, NULL,
-	    INTR_TYPE_CLK | INTR_FAST, &clkdesc);
-	crit = intr_disable();
-	mtx_lock_spin(&icu_lock);
-	INTREN(1 << apic_8254_intr);
-	mtx_unlock_spin(&icu_lock);
-	intr_restore(crit);
-
-#else /* APIC_IO */
-
-	/*
-	 * XXX Check the priority of this interrupt handler.  I
-	 * couldn't find anything suitable in the BSD/OS code (grog,
-	 * 19 July 2000).
-	 */
-	inthand_add("clk", 0, (driver_intr_t *)clkintr, NULL,
+	/* Finish initializing 8254 timer 0. */
+	intr_add_handler("clk", 0, (driver_intr_t *)clkintr, NULL,
 	    INTR_TYPE_CLK | INTR_FAST, NULL);
-	crit = intr_disable();
-	mtx_lock_spin(&icu_lock);
-	INTREN(IRQ0);
-	mtx_unlock_spin(&icu_lock);
-	intr_restore(crit);
-
-#endif /* APIC_IO */
 
 	/* Initialize RTC. */
 	writertc(RTC_STATUSA, rtc_statusa);
@@ -1004,120 +950,15 @@ cpu_initclocks()
 	if (diag != 0)
 		printf("RTC BIOS diagnostic error %b\n", diag, RTCDG_BITS);
 
-#ifdef APIC_IO
-	if (isa_apic_irq(8) != 8)
-		panic("APIC RTC != 8");
-#endif /* APIC_IO */
-
-	inthand_add("rtc", 8, (driver_intr_t *)rtcintr, NULL,
+	intr_add_handler("rtc", 8, (driver_intr_t *)rtcintr, NULL,
 	    INTR_TYPE_CLK | INTR_FAST, NULL);
-
-	crit = intr_disable();
-	mtx_lock_spin(&icu_lock);
-#ifdef APIC_IO
-	INTREN(APIC_IRQ8);
-#else
-	INTREN(IRQ8);
-#endif /* APIC_IO */
-	mtx_unlock_spin(&icu_lock);
-	intr_restore(crit);
+	i8254_intsrc = intr_lookup_source(8);
 
 	writertc(RTC_STATUSB, rtc_statusb);
 
-#ifdef APIC_IO
-	if (apic_8254_trial) {
-
-		printf("APIC_IO: Testing 8254 interrupt delivery\n");
-		while (read_intr_count(8) < 6)
-			;	/* nothing */
-		if (read_intr_count(apic_8254_intr) < 3) {
-			/* 
-			 * The MP table is broken.
-			 * The 8254 was not connected to the specified pin
-			 * on the IO APIC.
-			 * Workaround: Limited variant of mixed mode.
-			 */
-
-			crit = intr_disable();
-			mtx_lock_spin(&icu_lock);
-			INTRDIS(1 << apic_8254_intr);
-			mtx_unlock_spin(&icu_lock);
-			intr_restore(crit);
-			inthand_remove(clkdesc);
-			printf("APIC_IO: Broken MP table detected: "
-			       "8254 is not connected to "
-			       "IOAPIC #%d intpin %d\n",
-			       int_to_apicintpin[apic_8254_intr].ioapic,
-			       int_to_apicintpin[apic_8254_intr].int_pin);
-			/* 
-			 * Revoke current ISA IRQ 0 assignment and 
-			 * configure a fallback interrupt routing from
-			 * the 8254 Timer via the 8259 PIC to the
-			 * an ExtInt interrupt line on IOAPIC #0 intpin 0.
-			 * We reuse the low level interrupt handler number.
-			 */
-			if (apic_irq(0, 0) < 0) {
-				revoke_apic_irq(apic_8254_intr);
-				assign_apic_irq(0, 0, apic_8254_intr);
-			}
-			apic_8254_intr = apic_irq(0, 0);
-			setup_8254_mixed_mode();
-			inthand_add("clk", apic_8254_intr,
-				    (driver_intr_t *)clkintr, NULL,
-				    INTR_TYPE_CLK | INTR_FAST, NULL);
-			crit = intr_disable();
-			mtx_lock_spin(&icu_lock);
-			INTREN(1 << apic_8254_intr);
-			mtx_unlock_spin(&icu_lock);
-			intr_restore(crit);
-		}
-		
-	}
-	if (apic_int_type(0, 0) != 3 ||
-	    int_to_apicintpin[apic_8254_intr].ioapic != 0 ||
-	    int_to_apicintpin[apic_8254_intr].int_pin != 0)
-		printf("APIC_IO: routing 8254 via IOAPIC #%d intpin %d\n",
-		       int_to_apicintpin[apic_8254_intr].ioapic,
-		       int_to_apicintpin[apic_8254_intr].int_pin);
-	else
-		printf("APIC_IO: "
-		       "routing 8254 via 8259 and IOAPIC #0 intpin 0\n");
-#endif
-	
 	init_TSC_tc();
 }
 
-#ifdef APIC_IO
-static u_long
-read_intr_count(int vec)
-{
-	u_long *up;
-	up = intr_countp[vec];
-	if (up)
-		return *up;
-	return 0UL;
-}
-
-static void 
-setup_8254_mixed_mode()
-{
-	/*
-	 * Allow 8254 timer to INTerrupt 8259:
-	 *  re-initialize master 8259:
-	 *   reset; prog 4 bytes, single ICU, edge triggered
-	 */
-	outb(IO_ICU1, 0x13);
-	outb(IO_ICU1 + 1, NRSVIDT);	/* start vector (unused) */
-	outb(IO_ICU1 + 1, 0x00);	/* ignore slave */
-	outb(IO_ICU1 + 1, 0x03);	/* auto EOI, 8086 */
-	outb(IO_ICU1 + 1, 0xfe);	/* unmask INT0 */
-	
-	/* program IO APIC for type 3 INT on INT0 */
-	if (ext_int_setup(0, 0) < 0)
-		panic("8254 redirect via APIC pin0 impossible!");
-}
-#endif
-
 void
 cpu_startprofclock(void)
 {
@@ -1181,14 +1022,8 @@ i8254_get_timecount(struct timecounter *tc)
 	if (count < i8254_lastcount ||
 	    (!i8254_ticked && (clkintr_pending ||
 	    ((count < 20 || (!(eflags & PSL_I) && count < timer0_max_count / 2u)) &&
-#ifdef APIC_IO
-#define	lapic_irr1	((volatile u_int *)&lapic)[0x210 / 4]	/* XXX XXX */
-	    /* XXX this assumes that apic_8254_intr is < 24. */
-	    (lapic_irr1 & (1 << apic_8254_intr))))
-#else
-	    (inb(IO_ICU1) & 1)))
-#endif
-	    )) {
+	    i8254_intsrc != NULL &&
+	    i8254_intsrc->is_pic->pic_source_pending(i8254_intsrc))))) {
 		i8254_ticked = 1;
 		i8254_offset += timer0_max_count;
 	}

sys/i386/isa/npx.c

@@ -52,6 +52,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/mutex.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
+#include <sys/smp.h>
 #include <sys/sysctl.h>
 #include <machine/bus.h>
 #include <sys/rman.h>
@@ -61,31 +62,25 @@ __FBSDID("$FreeBSD$");
 #include <sys/signalvar.h>
 #include <sys/user.h>
 
-#ifndef SMP
 #include <machine/asmacros.h>
-#endif
 #include <machine/cputypes.h>
 #include <machine/frame.h>
 #include <machine/md_var.h>
 #include <machine/pcb.h>
 #include <machine/psl.h>
-#ifndef SMP
 #include <machine/clock.h>
-#endif
 #include <machine/resource.h>
 #include <machine/specialreg.h>
 #include <machine/segments.h>
 #include <machine/ucontext.h>
 
-#ifndef SMP
 #include <i386/isa/icu.h>
 #ifdef PC98
 #include <pc98/pc98/pc98.h>
 #else
 #include <i386/isa/isa.h>
 #endif
-#endif
-#include <i386/isa/intr_machdep.h>
+#include <machine/intr_machdep.h>
 #ifdef DEV_ISA
 #include <isa/isavar.h>
 #endif
@@ -165,9 +160,7 @@ static	void	fpusave(union savefpu *);
 static	void	fpurstor(union savefpu *);
 static	int	npx_attach(device_t dev);
 static	void	npx_identify(driver_t *driver, device_t parent);
-#ifndef SMP
 static	void	npx_intr(void *);
-#endif
 static	int	npx_probe(device_t dev);
 #ifdef I586_CPU_XXX
 static	long	timezero(const char *funcname,
@@ -180,10 +173,8 @@ SYSCTL_INT(_hw,HW_FLOATINGPT, floatingpoint,
 	CTLFLAG_RD, &hw_float, 0, 
 	"Floatingpoint instructions executed in hardware");
 
-#ifndef SMP
 static	volatile u_int		npx_intrs_while_probing;
 static	volatile u_int		npx_traps_while_probing;
-#endif
 
 static	union savefpu		npx_cleanstate;
 static	bool_t			npx_cleanstate_ready;
@@ -191,7 +182,6 @@ static	bool_t			npx_ex16;
 static	bool_t			npx_exists;
 static	bool_t			npx_irq13;
 
-#ifndef SMP
 alias_for_inthand_t probetrap;
 __asm("								\n\
 	.text							\n\
@@ -203,7 +193,6 @@ __asm("								\n\
 	fnclex							\n\
 	iret							\n\
 ");
-#endif /* SMP */
 
 /*
  * Identify routine.  Create a connection point on our parent for probing.
@@ -220,7 +209,6 @@ npx_identify(driver, parent)
 		panic("npx_identify");
 }
 
-#ifndef SMP
 /*
  * Do minimal handling of npx interrupts to convert them to traps.
  */
@@ -230,9 +218,7 @@ npx_intr(dummy)
 {
 	struct thread *td;
 
-#ifndef SMP
 	npx_intrs_while_probing++;
-#endif
 
 	/*
 	 * The BUSY# latch must be cleared in all cases so that the next
@@ -264,7 +250,6 @@ npx_intr(dummy)
 		mtx_unlock_spin(&sched_lock);
 	}
 }
-#endif /* !SMP */
 
 /*
  * Probe routine.  Initialize cr0 to give correct behaviour for [f]wait
@@ -276,7 +261,6 @@ static int
 npx_probe(dev)
 	device_t dev;
 {
-#ifndef SMP
 	struct gate_descriptor save_idt_npxtrap;
 	struct resource *ioport_res, *irq_res;
 	void *irq_cookie;
@@ -307,7 +291,6 @@ npx_probe(dev)
 	if (bus_setup_intr(dev, irq_res, INTR_TYPE_MISC | INTR_FAST, npx_intr,
 	    NULL, &irq_cookie) != 0)
 		panic("npx: can't create intr");
-#endif /* !SMP */
 
 	/*
 	 * Partially reset the coprocessor, if any.  Some BIOS's don't reset
@@ -348,16 +331,6 @@ npx_probe(dev)
 
 	device_set_desc(dev, "math processor");
 
-#ifdef SMP
-
-	/*
-	 * Exception 16 MUST work for SMP.
-	 */
-	npx_ex16 = hw_float = npx_exists = 1;
-	return (0);
-
-#else /* !SMP */
-
 	/*
 	 * Don't use fwait here because it might hang.
 	 * Don't use fnop here because it usually hangs if there is no FPU.
@@ -413,6 +386,10 @@ npx_probe(dev)
 				 */
 				npx_irq13 = 1;
 				idt[IDT_MF] = save_idt_npxtrap;
+#ifdef SMP
+				if (mp_ncpus > 1)
+					panic("npx0 cannot use IRQ 13 on an SMP system");
+#endif
 				return (0);
 			}
 			/*
@@ -425,6 +402,10 @@ npx_probe(dev)
 	 * emulator and say that it has been installed.  XXX handle devices
 	 * that aren't really devices better.
 	 */
+#ifdef SMP
+	if (mp_ncpus > 1)
+		panic("npx0 cannot be emulated on an SMP system");
+#endif
 	/* FALLTHROUGH */
 no_irq13:
 	idt[IDT_MF] = save_idt_npxtrap;
@@ -435,20 +416,15 @@ npx_probe(dev)
 	 * irq active then we would get it instead of exception 16.
 	 */
 	{
-		register_t crit;
+		struct intsrc *isrc;
 
-		crit = intr_disable();
-		mtx_lock_spin(&icu_lock);
-		INTRDIS(1 << irq_num);
-		mtx_unlock_spin(&icu_lock);
-		intr_restore(crit);
+		isrc = intr_lookup_source(irq_num);
+		isrc->is_pic->pic_disable_source(isrc);
 	}
 
 	bus_release_resource(dev, SYS_RES_IRQ, irq_rid, irq_res);
 	bus_release_resource(dev, SYS_RES_IOPORT, ioport_rid, ioport_res);
 	return (0);
-
-#endif /* SMP */
 }
 
 /*

sys/i386/pci/pci_cfgreg.c

@@ -48,10 +48,6 @@ __FBSDID("$FreeBSD$");
 #include <machine/segments.h>
 #include <machine/pc/bios.h>
 
-#ifdef APIC_IO
-#include <machine/smp.h>
-#endif /* APIC_IO */
-
 #include "pcib_if.h"
 
 #define PRVERB(a) do {							\
@@ -201,49 +197,7 @@ u_int32_t
 pci_cfgregread(int bus, int slot, int func, int reg, int bytes)
 {
 	uint32_t line;
-#ifdef APIC_IO
-	uint32_t pin;
 
-	/*
-	 * If we are using the APIC, the contents of the intline
-	 * register will probably be wrong (since they are set up for
-	 * use with the PIC.  Rather than rewrite these registers
-	 * (maybe that would be smarter) we trap attempts to read them
-	 * and translate to our private vector numbers.
-	 */
-	if ((reg == PCIR_INTLINE) && (bytes == 1)) {
-
-		pin = pcireg_cfgread(bus, slot, func, PCIR_INTPIN, 1);
-		line = pcireg_cfgread(bus, slot, func, PCIR_INTLINE, 1);
-
-		if (pin != 0) {
-			int airq;
-
-			airq = pci_apic_irq(bus, slot, pin);
-			if (airq >= 0) {
-				/* PCI specific entry found in MP table */
-				if (airq != line)
-					undirect_pci_irq(line);
-				return(airq);
-			} else {
-				/* 
-				 * PCI interrupts might be redirected
-				 * to the ISA bus according to some MP
-				 * tables. Use the same methods as
-				 * used by the ISA devices devices to
-				 * find the proper IOAPIC int pin.
-				 */
-				airq = isa_apic_irq(line);
-				if ((airq >= 0) && (airq != line)) {
-					/* XXX: undirect_pci_irq() ? */
-					undirect_isa_irq(line);
-					return(airq);
-				}
-			}
-		}
-		return(line);
-	}
-#else
 	/*
 	 * Some BIOS writers seem to want to ignore the spec and put
 	 * 0 in the intline rather than 255 to indicate none.  The rest of
@@ -251,10 +205,9 @@ pci_cfgregread(int bus, int slot, int func, int reg, int bytes)
 	 */
 	if (reg == PCIR_INTLINE && bytes == 1) {
 		line = pcireg_cfgread(bus, slot, func, PCIR_INTLINE, 1);
-		return pci_i386_map_intline(line);
+		return (pci_i386_map_intline(line));
 	}
-#endif /* APIC_IO */
-	return(pcireg_cfgread(bus, slot, func, reg, bytes));
+	return (pcireg_cfgread(bus, slot, func, reg, bytes));
 }
 
 /* 

sys/i386/pci/pci_pir.c

@@ -48,10 +48,6 @@ __FBSDID("$FreeBSD$");
 #include <machine/segments.h>
 #include <machine/pc/bios.h>
 
-#ifdef APIC_IO
-#include <machine/smp.h>
-#endif /* APIC_IO */
-
 #include "pcib_if.h"
 
 #define PRVERB(a) do {							\
@@ -201,49 +197,7 @@ u_int32_t
 pci_cfgregread(int bus, int slot, int func, int reg, int bytes)
 {
 	uint32_t line;
-#ifdef APIC_IO
-	uint32_t pin;
 
-	/*
-	 * If we are using the APIC, the contents of the intline
-	 * register will probably be wrong (since they are set up for
-	 * use with the PIC.  Rather than rewrite these registers
-	 * (maybe that would be smarter) we trap attempts to read them
-	 * and translate to our private vector numbers.
-	 */
-	if ((reg == PCIR_INTLINE) && (bytes == 1)) {
-
-		pin = pcireg_cfgread(bus, slot, func, PCIR_INTPIN, 1);
-		line = pcireg_cfgread(bus, slot, func, PCIR_INTLINE, 1);
-
-		if (pin != 0) {
-			int airq;
-
-			airq = pci_apic_irq(bus, slot, pin);
-			if (airq >= 0) {
-				/* PCI specific entry found in MP table */
-				if (airq != line)
-					undirect_pci_irq(line);
-				return(airq);
-			} else {
-				/* 
-				 * PCI interrupts might be redirected
-				 * to the ISA bus according to some MP
-				 * tables. Use the same methods as
-				 * used by the ISA devices devices to
-				 * find the proper IOAPIC int pin.
-				 */
-				airq = isa_apic_irq(line);
-				if ((airq >= 0) && (airq != line)) {
-					/* XXX: undirect_pci_irq() ? */
-					undirect_isa_irq(line);
-					return(airq);
-				}
-			}
-		}
-		return(line);
-	}
-#else
 	/*
 	 * Some BIOS writers seem to want to ignore the spec and put
 	 * 0 in the intline rather than 255 to indicate none.  The rest of
@@ -251,10 +205,9 @@ pci_cfgregread(int bus, int slot, int func, int reg, int bytes)
 	 */
 	if (reg == PCIR_INTLINE && bytes == 1) {
 		line = pcireg_cfgread(bus, slot, func, PCIR_INTLINE, 1);
-		return pci_i386_map_intline(line);
+		return (pci_i386_map_intline(line));
 	}
-#endif /* APIC_IO */
-	return(pcireg_cfgread(bus, slot, func, reg, bytes));
+	return (pcireg_cfgread(bus, slot, func, reg, bytes));
 }
 
 /* 

sys/isa/atrtc.c

@@ -71,14 +71,12 @@ __FBSDID("$FreeBSD$");
 #include <machine/clock.h>
 #include <machine/cputypes.h>
 #include <machine/frame.h>
+#include <machine/intr_machdep.h>
 #include <machine/md_var.h>
 #include <machine/psl.h>
-#ifdef APIC_IO
-#include <machine/segments.h>
-#endif
-#if defined(SMP) || defined(APIC_IO)
+#if defined(SMP)
 #include <machine/smp.h>
-#endif /* SMP || APIC_IO */
+#endif
 #include <machine/specialreg.h>
 
 #include <i386/isa/icu.h>
@@ -89,20 +87,10 @@ __FBSDID("$FreeBSD$");
 #endif
 #include <i386/isa/timerreg.h>
 
-#include <i386/isa/intr_machdep.h>
-
 #ifdef DEV_MCA
 #include <i386/bios/mca_machdep.h>
 #endif
 
-#ifdef APIC_IO
-#include <i386/isa/intr_machdep.h>
-/* The interrupt triggered by the 8254 (timer) chip */
-int apic_8254_intr;
-static u_long read_intr_count(int vec);
-static void setup_8254_mixed_mode(void);
-#endif
-
 /*
  * 32-bit time_t's can't reach leap years before 1904 or after 2036, so we
  * can use a simple formula for leap years.
@@ -150,6 +138,7 @@ static	u_int	hardclock_max_count;
 static	u_int32_t i8254_lastcount;
 static	u_int32_t i8254_offset;
 static	int	i8254_ticked;
+static	struct intsrc *i8254_intsrc;
 #ifndef BURN_BRIDGES
 /*
  * XXX new_function and timer_func should not handle clockframes, but
@@ -187,7 +176,7 @@ static struct timecounter i8254_timecounter = {
 };
 
 static void
-clkintr(struct clockframe frame)
+clkintr(struct clockframe *frame)
 {
 
 	if (timecounter->tc_get_timecount == i8254_get_timecount) {
@@ -201,7 +190,7 @@ clkintr(struct clockframe frame)
 		clkintr_pending = 0;
 		mtx_unlock_spin(&clock_lock);
 	}
-	timer_func(&frame);
+	timer_func(frame);
 #ifdef SMP
 	if (timer_func == hardclock)
 		forward_hardclock();
@@ -216,7 +205,7 @@ clkintr(struct clockframe frame)
 		if ((timer0_prescaler_count += timer0_max_count)
 		    >= hardclock_max_count) {
 			timer0_prescaler_count -= hardclock_max_count;
-			hardclock(&frame);
+			hardclock(frame);
 #ifdef SMP
 			forward_hardclock();
 #endif
@@ -251,7 +240,7 @@ clkintr(struct clockframe frame)
 			timer0_prescaler_count = 0;
 			timer_func = hardclock;
 			timer0_state = RELEASED;
-			hardclock(&frame);
+			hardclock(frame);
 #ifdef SMP
 			forward_hardclock();
 #endif
@@ -379,16 +368,16 @@ release_timer2()
  * in the statistics, but the stat clock will no longer stop.
  */
 static void
-rtcintr(struct clockframe frame)
+rtcintr(struct clockframe *frame)
 {
 	while (rtcin(RTC_INTR) & RTCIR_PERIOD) {
 		if (profprocs != 0) {
 			if (--pscnt == 0)
 				pscnt = psdiv;
-			profclock(&frame);
+			profclock(frame);
 		}
 		if (pscnt == psdiv)
-			statclock(&frame);
+			statclock(frame);
 #ifdef SMP
 		forward_statclock();
 #endif
@@ -931,11 +920,6 @@ void
 cpu_initclocks()
 {
 	int diag;
-#ifdef APIC_IO
-	int apic_8254_trial;
-	void *clkdesc;
-#endif /* APIC_IO */
-	register_t crit;
 
 	if (statclock_disable) {
 		/*
@@ -951,47 +935,9 @@ cpu_initclocks()
 		profhz = RTC_PROFRATE;
         }
 
-	/* Finish initializing 8253 timer 0. */
-#ifdef APIC_IO
-
-	apic_8254_intr = isa_apic_irq(0);
-	apic_8254_trial = 0;
-	if (apic_8254_intr >= 0 ) {
-		if (apic_int_type(0, 0) == 3)
-			apic_8254_trial = 1;
-	} else {
-		/* look for ExtInt on pin 0 */
-		if (apic_int_type(0, 0) == 3) {
-			apic_8254_intr = apic_irq(0, 0);
-			setup_8254_mixed_mode();
-		} else 
-			panic("APIC_IO: Cannot route 8254 interrupt to CPU");
-	}
-
-	inthand_add("clk", apic_8254_intr, (driver_intr_t *)clkintr, NULL,
-	    INTR_TYPE_CLK | INTR_FAST, &clkdesc);
-	crit = intr_disable();
-	mtx_lock_spin(&icu_lock);
-	INTREN(1 << apic_8254_intr);
-	mtx_unlock_spin(&icu_lock);
-	intr_restore(crit);
-
-#else /* APIC_IO */
-
-	/*
-	 * XXX Check the priority of this interrupt handler.  I
-	 * couldn't find anything suitable in the BSD/OS code (grog,
-	 * 19 July 2000).
-	 */
-	inthand_add("clk", 0, (driver_intr_t *)clkintr, NULL,
+	/* Finish initializing 8254 timer 0. */
+	intr_add_handler("clk", 0, (driver_intr_t *)clkintr, NULL,
 	    INTR_TYPE_CLK | INTR_FAST, NULL);
-	crit = intr_disable();
-	mtx_lock_spin(&icu_lock);
-	INTREN(IRQ0);
-	mtx_unlock_spin(&icu_lock);
-	intr_restore(crit);
-
-#endif /* APIC_IO */
 
 	/* Initialize RTC. */
 	writertc(RTC_STATUSA, rtc_statusa);
@@ -1004,120 +950,15 @@ cpu_initclocks()
 	if (diag != 0)
 		printf("RTC BIOS diagnostic error %b\n", diag, RTCDG_BITS);
 
-#ifdef APIC_IO
-	if (isa_apic_irq(8) != 8)
-		panic("APIC RTC != 8");
-#endif /* APIC_IO */
-
-	inthand_add("rtc", 8, (driver_intr_t *)rtcintr, NULL,
+	intr_add_handler("rtc", 8, (driver_intr_t *)rtcintr, NULL,
 	    INTR_TYPE_CLK | INTR_FAST, NULL);
-
-	crit = intr_disable();
-	mtx_lock_spin(&icu_lock);
-#ifdef APIC_IO
-	INTREN(APIC_IRQ8);
-#else
-	INTREN(IRQ8);
-#endif /* APIC_IO */
-	mtx_unlock_spin(&icu_lock);
-	intr_restore(crit);
+	i8254_intsrc = intr_lookup_source(8);
 
 	writertc(RTC_STATUSB, rtc_statusb);
 
-#ifdef APIC_IO
-	if (apic_8254_trial) {
-
-		printf("APIC_IO: Testing 8254 interrupt delivery\n");
-		while (read_intr_count(8) < 6)
-			;	/* nothing */
-		if (read_intr_count(apic_8254_intr) < 3) {
-			/* 
-			 * The MP table is broken.
-			 * The 8254 was not connected to the specified pin
-			 * on the IO APIC.
-			 * Workaround: Limited variant of mixed mode.
-			 */
-
-			crit = intr_disable();
-			mtx_lock_spin(&icu_lock);
-			INTRDIS(1 << apic_8254_intr);
-			mtx_unlock_spin(&icu_lock);
-			intr_restore(crit);
-			inthand_remove(clkdesc);
-			printf("APIC_IO: Broken MP table detected: "
-			       "8254 is not connected to "
-			       "IOAPIC #%d intpin %d\n",
-			       int_to_apicintpin[apic_8254_intr].ioapic,
-			       int_to_apicintpin[apic_8254_intr].int_pin);
-			/* 
-			 * Revoke current ISA IRQ 0 assignment and 
-			 * configure a fallback interrupt routing from
-			 * the 8254 Timer via the 8259 PIC to the
-			 * an ExtInt interrupt line on IOAPIC #0 intpin 0.
-			 * We reuse the low level interrupt handler number.
-			 */
-			if (apic_irq(0, 0) < 0) {
-				revoke_apic_irq(apic_8254_intr);
-				assign_apic_irq(0, 0, apic_8254_intr);
-			}
-			apic_8254_intr = apic_irq(0, 0);
-			setup_8254_mixed_mode();
-			inthand_add("clk", apic_8254_intr,
-				    (driver_intr_t *)clkintr, NULL,
-				    INTR_TYPE_CLK | INTR_FAST, NULL);
-			crit = intr_disable();
-			mtx_lock_spin(&icu_lock);
-			INTREN(1 << apic_8254_intr);
-			mtx_unlock_spin(&icu_lock);
-			intr_restore(crit);
-		}
-		
-	}
-	if (apic_int_type(0, 0) != 3 ||
-	    int_to_apicintpin[apic_8254_intr].ioapic != 0 ||
-	    int_to_apicintpin[apic_8254_intr].int_pin != 0)
-		printf("APIC_IO: routing 8254 via IOAPIC #%d intpin %d\n",
-		       int_to_apicintpin[apic_8254_intr].ioapic,
-		       int_to_apicintpin[apic_8254_intr].int_pin);
-	else
-		printf("APIC_IO: "
-		       "routing 8254 via 8259 and IOAPIC #0 intpin 0\n");
-#endif
-	
 	init_TSC_tc();
 }
 
-#ifdef APIC_IO
-static u_long
-read_intr_count(int vec)
-{
-	u_long *up;
-	up = intr_countp[vec];
-	if (up)
-		return *up;
-	return 0UL;
-}
-
-static void 
-setup_8254_mixed_mode()
-{
-	/*
-	 * Allow 8254 timer to INTerrupt 8259:
-	 *  re-initialize master 8259:
-	 *   reset; prog 4 bytes, single ICU, edge triggered
-	 */
-	outb(IO_ICU1, 0x13);
-	outb(IO_ICU1 + 1, NRSVIDT);	/* start vector (unused) */
-	outb(IO_ICU1 + 1, 0x00);	/* ignore slave */
-	outb(IO_ICU1 + 1, 0x03);	/* auto EOI, 8086 */
-	outb(IO_ICU1 + 1, 0xfe);	/* unmask INT0 */
-	
-	/* program IO APIC for type 3 INT on INT0 */
-	if (ext_int_setup(0, 0) < 0)
-		panic("8254 redirect via APIC pin0 impossible!");
-}
-#endif
-
 void
 cpu_startprofclock(void)
 {
@@ -1181,14 +1022,8 @@ i8254_get_timecount(struct timecounter *tc)
 	if (count < i8254_lastcount ||
 	    (!i8254_ticked && (clkintr_pending ||
 	    ((count < 20 || (!(eflags & PSL_I) && count < timer0_max_count / 2u)) &&
-#ifdef APIC_IO
-#define	lapic_irr1	((volatile u_int *)&lapic)[0x210 / 4]	/* XXX XXX */
-	    /* XXX this assumes that apic_8254_intr is < 24. */
-	    (lapic_irr1 & (1 << apic_8254_intr))))
-#else
-	    (inb(IO_ICU1) & 1)))
-#endif
-	    )) {
+	    i8254_intsrc != NULL &&
+	    i8254_intsrc->is_pic->pic_source_pending(i8254_intsrc))))) {
 		i8254_ticked = 1;
 		i8254_offset += timer0_max_count;
 	}