/* * from: vector.s, 386BSD 0.1 unknown origin * $Id: apic_vector.s,v 1.38 1999/05/28 14:08:57 bde Exp $ */ #include #include #include "i386/isa/intr_machdep.h" #ifdef FAST_SIMPLELOCK #define GET_FAST_INTR_LOCK \ pushl $_fast_intr_lock ; /* address of lock */ \ call _s_lock ; /* MP-safe */ \ addl $4,%esp #define REL_FAST_INTR_LOCK \ pushl $_fast_intr_lock ; /* address of lock */ \ call _s_unlock ; /* MP-safe */ \ addl $4,%esp #else /* FAST_SIMPLELOCK */ #define GET_FAST_INTR_LOCK \ call _get_isrlock #define REL_FAST_INTR_LOCK \ pushl $_mp_lock ; /* GIANT_LOCK */ \ call _MPrellock ; \ add $4, %esp #endif /* FAST_SIMPLELOCK */ /* convert an absolute IRQ# into a bitmask */ #define IRQ_BIT(irq_num) (1 << (irq_num)) /* make an index into the IO APIC from the IRQ# */ #define REDTBL_IDX(irq_num) (0x10 + ((irq_num) * 2)) /* * Macros for interrupt interrupt entry, call to handler, and exit. */ #ifdef FAST_WITHOUTCPL /* */ #define FAST_INTR(irq_num, vec_name) \ .text ; \ SUPERALIGN_TEXT ; \ IDTVEC(vec_name) ; \ pushl %eax ; /* save only call-used registers */ \ pushl %ecx ; \ pushl %edx ; \ pushl %ds ; \ MAYBE_PUSHL_ES ; \ pushl %fs ; \ movl $KDSEL,%eax ; \ movl %ax,%ds ; \ MAYBE_MOVW_AX_ES ; \ movl $KPSEL,%eax ; \ movl %ax,%fs ; \ FAKE_MCOUNT((5+ACTUALLY_PUSHED)*4(%esp)) ; \ pushl _intr_unit + (irq_num) * 4 ; \ GET_FAST_INTR_LOCK ; \ call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ REL_FAST_INTR_LOCK ; \ addl $4, %esp ; \ movl $0, lapic_eoi ; \ lock ; \ incl _cnt+V_INTR ; /* book-keeping can wait */ \ movl _intr_countp + (irq_num) * 4, %eax ; \ lock ; \ incl (%eax) ; \ MEXITCOUNT ; \ popl %fs ; \ MAYBE_POPL_ES ; \ popl %ds ; \ popl %edx ; \ popl %ecx ; \ popl %eax ; \ iret #else /* FAST_WITHOUTCPL */ #define FAST_INTR(irq_num, vec_name) \ .text ; \ SUPERALIGN_TEXT ; \ IDTVEC(vec_name) ; \ pushl %eax ; /* save only call-used registers */ \ pushl %ecx ; \ pushl %edx ; \ pushl %ds ; \ MAYBE_PUSHL_ES ; \ pushl %fs ; \ movl $KDSEL, %eax ; \ movl %ax, %ds ; \ MAYBE_MOVW_AX_ES ; \ movl $KPSEL, %eax ; \ movl %ax, %fs ; \ FAKE_MCOUNT((5+ACTUALLY_PUSHED)*4(%esp)) ; \ GET_FAST_INTR_LOCK ; \ pushl _intr_unit + (irq_num) * 4 ; \ call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ addl $4, %esp ; \ movl $0, lapic_eoi ; \ lock ; \ incl _cnt+V_INTR ; /* book-keeping can wait */ \ movl _intr_countp + (irq_num) * 4,%eax ; \ lock ; \ incl (%eax) ; \ movl _cpl, %eax ; /* unmasking pending HWIs or SWIs? */ \ notl %eax ; \ andl _ipending, %eax ; \ jne 2f ; /* yes, maybe handle them */ \ 1: ; \ MEXITCOUNT ; \ REL_FAST_INTR_LOCK ; \ popl %fs ; \ MAYBE_POPL_ES ; \ popl %ds ; \ popl %edx ; \ popl %ecx ; \ popl %eax ; \ iret ; \ ; \ ALIGN_TEXT ; \ 2: ; \ cmpb $3, _intr_nesting_level ; /* enough stack? */ \ jae 1b ; /* no, return */ \ movl _cpl, %eax ; \ /* XXX next line is probably unnecessary now. */ \ movl $HWI_MASK|SWI_MASK, _cpl ; /* limit nesting ... */ \ lock ; \ incb _intr_nesting_level ; /* ... really limit it ... */ \ sti ; /* to do this as early as possible */ \ popl %fs ; /* discard most of thin frame ... */ \ MAYBE_POPL_ES ; /* discard most of thin frame ... */ \ popl %ecx ; /* ... original %ds ... */ \ popl %edx ; \ xchgl %eax, 4(%esp) ; /* orig %eax; save cpl */ \ pushal ; /* build fat frame (grrr) ... */ \ pushl %ecx ; /* ... actually %ds ... */ \ pushl %es ; \ pushl %fs ; movl $KDSEL, %eax ; \ movl %ax, %es ; \ movl $KPSEL, %eax ; movl %ax, %fs ; movl (3+8+0)*4(%esp), %ecx ; /* %ecx from thin frame ... */ \ movl %ecx, (3+6)*4(%esp) ; /* ... to fat frame ... */ \ movl (3+8+1)*4(%esp), %eax ; /* ... cpl from thin frame */ \ pushl %eax ; \ subl $4, %esp ; /* junk for unit number */ \ MEXITCOUNT ; \ jmp _doreti #endif /** FAST_WITHOUTCPL */ /* * */ #define PUSH_FRAME \ pushl $0 ; /* dummy error code */ \ pushl $0 ; /* dummy trap type */ \ pushal ; \ pushl %ds ; /* save data and extra segments ... */ \ pushl %es ; \ pushl %fs #define POP_FRAME \ popl %fs ; \ popl %es ; \ popl %ds ; \ popal ; \ addl $4+4,%esp #define IOAPICADDR(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 8 #define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 12 #define MASK_IRQ(irq_num) \ IMASK_LOCK ; /* into critical reg */ \ testl $IRQ_BIT(irq_num), _apic_imen ; \ jne 7f ; /* masked, don't mask */ \ orl $IRQ_BIT(irq_num), _apic_imen ; /* set the mask bit */ \ movl IOAPICADDR(irq_num), %ecx ; /* ioapic addr */ \ movl REDIRIDX(irq_num), %eax ; /* get the index */ \ movl %eax, (%ecx) ; /* write the index */ \ movl IOAPIC_WINDOW(%ecx), %eax ; /* current value */ \ orl $IOART_INTMASK, %eax ; /* set the mask */ \ movl %eax, IOAPIC_WINDOW(%ecx) ; /* new value */ \ 7: ; /* already masked */ \ IMASK_UNLOCK /* * Test to see whether we are handling an edge or level triggered INT. * Level-triggered INTs must still be masked as we don't clear the source, * and the EOI cycle would cause redundant INTs to occur. */ #define MASK_LEVEL_IRQ(irq_num) \ testl $IRQ_BIT(irq_num), _apic_pin_trigger ; \ jz 9f ; /* edge, don't mask */ \ MASK_IRQ(irq_num) ; \ 9: #ifdef APIC_INTR_REORDER #define EOI_IRQ(irq_num) \ movl _apic_isrbit_location + 8 * (irq_num), %eax ; \ movl (%eax), %eax ; \ testl _apic_isrbit_location + 4 + 8 * (irq_num), %eax ; \ jz 9f ; /* not active */ \ movl $0, lapic_eoi ; \ APIC_ITRACE(apic_itrace_eoi, irq_num, APIC_ITRACE_EOI) ; \ 9: #else #define EOI_IRQ(irq_num) \ testl $IRQ_BIT(irq_num), lapic_isr1; \ jz 9f ; /* not active */ \ movl $0, lapic_eoi; \ APIC_ITRACE(apic_itrace_eoi, irq_num, APIC_ITRACE_EOI) ; \ 9: #endif /* * Test to see if the source is currntly masked, clear if so. */ #define UNMASK_IRQ(irq_num) \ IMASK_LOCK ; /* into critical reg */ \ testl $IRQ_BIT(irq_num), _apic_imen ; \ je 7f ; /* bit clear, not masked */ \ andl $~IRQ_BIT(irq_num), _apic_imen ;/* clear mask bit */ \ movl IOAPICADDR(irq_num),%ecx ; /* ioapic addr */ \ movl REDIRIDX(irq_num), %eax ; /* get the index */ \ movl %eax,(%ecx) ; /* write the index */ \ movl IOAPIC_WINDOW(%ecx),%eax ; /* current value */ \ andl $~IOART_INTMASK,%eax ; /* clear the mask */ \ movl %eax,IOAPIC_WINDOW(%ecx) ; /* new value */ \ 7: ; \ IMASK_UNLOCK #ifdef INTR_SIMPLELOCK #define ENLOCK #define DELOCK #define LATELOCK call _get_isrlock #else #define ENLOCK \ ISR_TRYLOCK ; /* XXX this is going away... */ \ testl %eax, %eax ; /* did we get it? */ \ jz 3f #define DELOCK ISR_RELLOCK #define LATELOCK #endif #ifdef APIC_INTR_DIAGNOSTIC #ifdef APIC_INTR_DIAGNOSTIC_IRQ log_intr_event: pushf cli pushl $CNAME(apic_itrace_debuglock) call _s_lock_np addl $4, %esp movl CNAME(apic_itrace_debugbuffer_idx), %ecx andl $32767, %ecx movl _cpuid, %eax shll $8, %eax orl 8(%esp), %eax movw %ax, CNAME(apic_itrace_debugbuffer)(,%ecx,2) incl %ecx andl $32767, %ecx movl %ecx, CNAME(apic_itrace_debugbuffer_idx) pushl $CNAME(apic_itrace_debuglock) call _s_unlock_np addl $4, %esp popf ret #define APIC_ITRACE(name, irq_num, id) \ lock ; /* MP-safe */ \ incl CNAME(name) + (irq_num) * 4 ; \ pushl %eax ; \ pushl %ecx ; \ pushl %edx ; \ movl $(irq_num), %eax ; \ cmpl $APIC_INTR_DIAGNOSTIC_IRQ, %eax ; \ jne 7f ; \ pushl $id ; \ call log_intr_event ; \ addl $4, %esp ; \ 7: ; \ popl %edx ; \ popl %ecx ; \ popl %eax #else #define APIC_ITRACE(name, irq_num, id) \ lock ; /* MP-safe */ \ incl CNAME(name) + (irq_num) * 4 #endif #define APIC_ITRACE_ENTER 1 #define APIC_ITRACE_EOI 2 #define APIC_ITRACE_TRYISRLOCK 3 #define APIC_ITRACE_GOTISRLOCK 4 #define APIC_ITRACE_ENTER2 5 #define APIC_ITRACE_LEAVE 6 #define APIC_ITRACE_UNMASK 7 #define APIC_ITRACE_ACTIVE 8 #define APIC_ITRACE_MASKED 9 #define APIC_ITRACE_NOISRLOCK 10 #define APIC_ITRACE_MASKED2 11 #define APIC_ITRACE_SPLZ 12 #define APIC_ITRACE_DORETI 13 #else #define APIC_ITRACE(name, irq_num, id) #endif #ifdef CPL_AND_CML #define INTR(irq_num, vec_name, maybe_extra_ipending) \ .text ; \ SUPERALIGN_TEXT ; \ /* _XintrNN: entry point used by IDT/HWIs & splz_unpend via _vec[]. */ \ IDTVEC(vec_name) ; \ PUSH_FRAME ; \ movl $KDSEL, %eax ; /* reload with kernel's data segment */ \ movl %ax, %ds ; \ movl %ax, %es ; \ movl $KPSEL, %eax ; \ movl %ax, %fs ; \ ; \ maybe_extra_ipending ; \ ; \ APIC_ITRACE(apic_itrace_enter, irq_num, APIC_ITRACE_ENTER) ; \ lock ; /* MP-safe */ \ btsl $(irq_num), iactive ; /* lazy masking */ \ jc 1f ; /* already active */ \ ; \ MASK_LEVEL_IRQ(irq_num) ; \ EOI_IRQ(irq_num) ; \ 0: ; \ APIC_ITRACE(apic_itrace_tryisrlock, irq_num, APIC_ITRACE_TRYISRLOCK) ;\ ENLOCK ; \ ; \ APIC_ITRACE(apic_itrace_gotisrlock, irq_num, APIC_ITRACE_GOTISRLOCK) ;\ AVCPL_LOCK ; /* MP-safe */ \ testl $IRQ_BIT(irq_num), _cpl ; \ jne 2f ; /* this INT masked */ \ testl $IRQ_BIT(irq_num), _cml ; \ jne 2f ; /* this INT masked */ \ orl $IRQ_BIT(irq_num), _cil ; \ AVCPL_UNLOCK ; \ ; \ incb _intr_nesting_level ; \ ; \ /* entry point used by doreti_unpend for HWIs. */ \ __CONCAT(Xresume,irq_num): ; \ FAKE_MCOUNT(13*4(%esp)) ; /* XXX avoid dbl cnt */ \ lock ; incl _cnt+V_INTR ; /* tally interrupts */ \ movl _intr_countp + (irq_num) * 4, %eax ; \ lock ; incl (%eax) ; \ ; \ AVCPL_LOCK ; /* MP-safe */ \ movl _cml, %eax ; \ pushl %eax ; \ orl _intr_mask + (irq_num) * 4, %eax ; \ movl %eax, _cml ; \ AVCPL_UNLOCK ; \ ; \ pushl _intr_unit + (irq_num) * 4 ; \ incl _inside_intr ; \ APIC_ITRACE(apic_itrace_enter2, irq_num, APIC_ITRACE_ENTER2) ; \ sti ; \ call *_intr_handler + (irq_num) * 4 ; \ cli ; \ APIC_ITRACE(apic_itrace_leave, irq_num, APIC_ITRACE_LEAVE) ; \ decl _inside_intr ; \ ; \ lock ; andl $~IRQ_BIT(irq_num), iactive ; \ lock ; andl $~IRQ_BIT(irq_num), _cil ; \ UNMASK_IRQ(irq_num) ; \ APIC_ITRACE(apic_itrace_unmask, irq_num, APIC_ITRACE_UNMASK) ; \ sti ; /* doreti repeats cli/sti */ \ MEXITCOUNT ; \ LATELOCK ; \ jmp _doreti ; \ ; \ ALIGN_TEXT ; \ 1: ; /* active */ \ APIC_ITRACE(apic_itrace_active, irq_num, APIC_ITRACE_ACTIVE) ; \ MASK_IRQ(irq_num) ; \ EOI_IRQ(irq_num) ; \ AVCPL_LOCK ; /* MP-safe */ \ orl $IRQ_BIT(irq_num), _ipending ; \ AVCPL_UNLOCK ; \ lock ; \ btsl $(irq_num), iactive ; /* still active */ \ jnc 0b ; /* retry */ \ POP_FRAME ; \ iret ; \ ; \ ALIGN_TEXT ; \ 2: ; /* masked by cpl|cml */ \ APIC_ITRACE(apic_itrace_masked, irq_num, APIC_ITRACE_MASKED) ; \ orl $IRQ_BIT(irq_num), _ipending ; \ AVCPL_UNLOCK ; \ DELOCK ; /* XXX this is going away... */ \ POP_FRAME ; \ iret ; \ ALIGN_TEXT ; \ 3: ; /* other cpu has isr lock */ \ APIC_ITRACE(apic_itrace_noisrlock, irq_num, APIC_ITRACE_NOISRLOCK) ;\ AVCPL_LOCK ; /* MP-safe */ \ orl $IRQ_BIT(irq_num), _ipending ; \ testl $IRQ_BIT(irq_num), _cpl ; \ jne 4f ; /* this INT masked */ \ testl $IRQ_BIT(irq_num), _cml ; \ jne 4f ; /* this INT masked */ \ orl $IRQ_BIT(irq_num), _cil ; \ AVCPL_UNLOCK ; \ call forward_irq ; /* forward irq to lock holder */ \ POP_FRAME ; /* and return */ \ iret ; \ ALIGN_TEXT ; \ 4: ; /* blocked */ \ APIC_ITRACE(apic_itrace_masked2, irq_num, APIC_ITRACE_MASKED2) ;\ AVCPL_UNLOCK ; \ POP_FRAME ; /* and return */ \ iret #else /* CPL_AND_CML */ #define INTR(irq_num, vec_name, maybe_extra_ipending) \ .text ; \ SUPERALIGN_TEXT ; \ /* _XintrNN: entry point used by IDT/HWIs & splz_unpend via _vec[]. */ \ IDTVEC(vec_name) ; \ PUSH_FRAME ; \ movl $KDSEL, %eax ; /* reload with kernel's data segment */ \ movl %ax, %ds ; \ movl %ax, %es ; \ movl $KPSEL, %eax ; \ movl %ax, %fs ; \ ; \ maybe_extra_ipending ; \ ; \ APIC_ITRACE(apic_itrace_enter, irq_num, APIC_ITRACE_ENTER) ; \ lock ; /* MP-safe */ \ btsl $(irq_num), iactive ; /* lazy masking */ \ jc 1f ; /* already active */ \ ; \ MASK_LEVEL_IRQ(irq_num) ; \ EOI_IRQ(irq_num) ; \ 0: ; \ APIC_ITRACE(apic_itrace_tryisrlock, irq_num, APIC_ITRACE_TRYISRLOCK) ;\ ISR_TRYLOCK ; /* XXX this is going away... */ \ testl %eax, %eax ; /* did we get it? */ \ jz 3f ; /* no */ \ ; \ APIC_ITRACE(apic_itrace_gotisrlock, irq_num, APIC_ITRACE_GOTISRLOCK) ;\ AVCPL_LOCK ; /* MP-safe */ \ testl $IRQ_BIT(irq_num), _cpl ; \ jne 2f ; /* this INT masked */ \ AVCPL_UNLOCK ; \ ; \ incb _intr_nesting_level ; \ ; \ /* entry point used by doreti_unpend for HWIs. */ \ __CONCAT(Xresume,irq_num): ; \ FAKE_MCOUNT(13*4(%esp)) ; /* XXX avoid dbl cnt */ \ lock ; incl _cnt+V_INTR ; /* tally interrupts */ \ movl _intr_countp + (irq_num) * 4, %eax ; \ lock ; incl (%eax) ; \ ; \ AVCPL_LOCK ; /* MP-safe */ \ movl _cpl, %eax ; \ pushl %eax ; \ orl _intr_mask + (irq_num) * 4, %eax ; \ movl %eax, _cpl ; \ andl $~IRQ_BIT(irq_num), _ipending ; \ AVCPL_UNLOCK ; \ ; \ pushl _intr_unit + (irq_num) * 4 ; \ APIC_ITRACE(apic_itrace_enter2, irq_num, APIC_ITRACE_ENTER2) ; \ sti ; \ call *_intr_handler + (irq_num) * 4 ; \ cli ; \ APIC_ITRACE(apic_itrace_leave, irq_num, APIC_ITRACE_LEAVE) ; \ ; \ lock ; andl $~IRQ_BIT(irq_num), iactive ; \ UNMASK_IRQ(irq_num) ; \ APIC_ITRACE(apic_itrace_unmask, irq_num, APIC_ITRACE_UNMASK) ; \ sti ; /* doreti repeats cli/sti */ \ MEXITCOUNT ; \ jmp _doreti ; \ ; \ ALIGN_TEXT ; \ 1: ; /* active */ \ APIC_ITRACE(apic_itrace_active, irq_num, APIC_ITRACE_ACTIVE) ; \ MASK_IRQ(irq_num) ; \ EOI_IRQ(irq_num) ; \ AVCPL_LOCK ; /* MP-safe */ \ orl $IRQ_BIT(irq_num), _ipending ; \ AVCPL_UNLOCK ; \ lock ; \ btsl $(irq_num), iactive ; /* still active */ \ jnc 0b ; /* retry */ \ POP_FRAME ; \ iret ; /* XXX: iactive bit might be 0 now */ \ ALIGN_TEXT ; \ 2: ; /* masked by cpl, leave iactive set */ \ APIC_ITRACE(apic_itrace_masked, irq_num, APIC_ITRACE_MASKED) ; \ orl $IRQ_BIT(irq_num), _ipending ; \ AVCPL_UNLOCK ; \ ISR_RELLOCK ; /* XXX this is going away... */ \ POP_FRAME ; \ iret ; \ ALIGN_TEXT ; \ 3: ; /* other cpu has isr lock */ \ APIC_ITRACE(apic_itrace_noisrlock, irq_num, APIC_ITRACE_NOISRLOCK) ;\ AVCPL_LOCK ; /* MP-safe */ \ orl $IRQ_BIT(irq_num), _ipending ; \ testl $IRQ_BIT(irq_num), _cpl ; \ jne 4f ; /* this INT masked */ \ AVCPL_UNLOCK ; \ call forward_irq ; /* forward irq to lock holder */ \ POP_FRAME ; /* and return */ \ iret ; \ ALIGN_TEXT ; \ 4: ; /* blocked */ \ APIC_ITRACE(apic_itrace_masked2, irq_num, APIC_ITRACE_MASKED2) ;\ AVCPL_UNLOCK ; \ POP_FRAME ; /* and return */ \ iret #endif /* CPL_AND_CML */ /* * Handle "spurious INTerrupts". * Notes: * This is different than the "spurious INTerrupt" generated by an * 8259 PIC for missing INTs. See the APIC documentation for details. * This routine should NOT do an 'EOI' cycle. */ .text SUPERALIGN_TEXT .globl _Xspuriousint _Xspuriousint: /* No EOI cycle used here */ iret /* * Handle TLB shootdowns. */ .text SUPERALIGN_TEXT .globl _Xinvltlb _Xinvltlb: pushl %eax #ifdef COUNT_XINVLTLB_HITS pushl %fs movl $KPSEL, %eax movl %ax, %fs movl _cpuid, %eax popl %fs ss incl _xhits(,%eax,4) #endif /* COUNT_XINVLTLB_HITS */ movl %cr3, %eax /* invalidate the TLB */ movl %eax, %cr3 ss /* stack segment, avoid %ds load */ movl $0, lapic_eoi /* End Of Interrupt to APIC */ popl %eax iret #ifdef BETTER_CLOCK /* * Executed by a CPU when it receives an Xcpucheckstate IPI from another CPU, * * - Stores current cpu state in checkstate_cpustate[cpuid] * 0 == user, 1 == sys, 2 == intr * - Stores current process in checkstate_curproc[cpuid] * * - Signals its receipt by setting bit cpuid in checkstate_probed_cpus. * * stack: 0->ds, 4->fs, 8->ebx, 12->eax, 16->eip, 20->cs, 24->eflags */ .text SUPERALIGN_TEXT .globl _Xcpucheckstate .globl _checkstate_cpustate .globl _checkstate_curproc .globl _checkstate_pc _Xcpucheckstate: pushl %eax pushl %ebx pushl %ds /* save current data segment */ pushl %fs movl $KDSEL, %eax movl %ax, %ds /* use KERNEL data segment */ movl $KPSEL, %eax movl %ax, %fs movl $0, lapic_eoi /* End Of Interrupt to APIC */ movl $0, %ebx movl 20(%esp), %eax andl $3, %eax cmpl $3, %eax je 1f testl $PSL_VM, 24(%esp) jne 1f incl %ebx /* system or interrupt */ #ifdef CPL_AND_CML cmpl $0, _inside_intr je 1f incl %ebx /* interrupt */ #endif 1: movl _cpuid, %eax movl %ebx, _checkstate_cpustate(,%eax,4) movl _curproc, %ebx movl %ebx, _checkstate_curproc(,%eax,4) movl 16(%esp), %ebx movl %ebx, _checkstate_pc(,%eax,4) lock /* checkstate_probed_cpus |= (1<