d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
e4fd6edfd8
freebsd-dev/sys/i386/isa/apic_vector.s
Mike Smith 91fe3dc1e1 Implement an all-CPU shootdown-style rendezvous facility. This allows 
the caller to specify a function to be guarded between an entry and exit
barrier, as well as pre- and post-barrier functions.

The primary use for this function is synchronised update of per-cpu private
data.  The implementation is almost (but not quite) MI; with a better
mechanism for masking per-CPU interrupts it could probably be hoisted.

Reviewed by:	peter (partially)
1999-07-20 06:52:35 +00:00

1065 lines
25 KiB
ArmAsm
Raw Blame History

/*
* from: vector.s, 386BSD 0.1 unknown origin
* $Id: apic_vector.s,v 1.42 1999/07/10 15:27:59 bde Exp $
*/
#include <machine/apic.h>
#include <machine/smp.h>
#include "i386/isa/intr_machdep.h"
#ifdef FAST_SIMPLELOCK
#define GET_FAST_INTR_LOCK \
pushl $_fast_intr_lock ; /* address of lock */ \
call _s_lock ; /* MP-safe */ \
addl $4,%esp
#define REL_FAST_INTR_LOCK \
pushl $_fast_intr_lock ; /* address of lock */ \
call _s_unlock ; /* MP-safe */ \
addl $4,%esp
#else /* FAST_SIMPLELOCK */
#define GET_FAST_INTR_LOCK \
call _get_isrlock
#define REL_FAST_INTR_LOCK \
pushl $_mp_lock ; /* GIANT_LOCK */ \
call _MPrellock ; \
add $4, %esp
#endif /* FAST_SIMPLELOCK */
/* convert an absolute IRQ# into a bitmask */
#define IRQ_BIT(irq_num) (1 << (irq_num))
/* make an index into the IO APIC from the IRQ# */
#define REDTBL_IDX(irq_num) (0x10 + ((irq_num) * 2))
/*
* Macros for interrupt interrupt entry, call to handler, and exit.
*/
#ifdef FAST_WITHOUTCPL
/*
*/
#define FAST_INTR(irq_num, vec_name) \
.text ; \
SUPERALIGN_TEXT ; \
IDTVEC(vec_name) ; \
pushl %eax ; /* save only call-used registers */ \
pushl %ecx ; \
pushl %edx ; \
pushl %ds ; \
MAYBE_PUSHL_ES ; \
pushl %fs ; \
movl $KDSEL,%eax ; \
movl %ax,%ds ; \
MAYBE_MOVW_AX_ES ; \
movl $KPSEL,%eax ; \
movl %ax,%fs ; \
FAKE_MCOUNT((5+ACTUALLY_PUSHED)*4(%esp)) ; \
pushl _intr_unit + (irq_num) * 4 ; \
GET_FAST_INTR_LOCK ; \
call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
REL_FAST_INTR_LOCK ; \
addl $4, %esp ; \
movl $0, lapic_eoi ; \
lock ; \
incl _cnt+V_INTR ; /* book-keeping can wait */ \
movl _intr_countp + (irq_num) * 4, %eax ; \
lock ; \
incl (%eax) ; \
MEXITCOUNT ; \
popl %fs ; \
MAYBE_POPL_ES ; \
popl %ds ; \
popl %edx ; \
popl %ecx ; \
popl %eax ; \
iret
#else /* FAST_WITHOUTCPL */
#define FAST_INTR(irq_num, vec_name) \
.text ; \
SUPERALIGN_TEXT ; \
IDTVEC(vec_name) ; \
pushl %eax ; /* save only call-used registers */ \
pushl %ecx ; \
pushl %edx ; \
pushl %ds ; \
MAYBE_PUSHL_ES ; \
pushl %fs ; \
movl $KDSEL, %eax ; \
movl %ax, %ds ; \
MAYBE_MOVW_AX_ES ; \
movl $KPSEL, %eax ; \
movl %ax, %fs ; \
FAKE_MCOUNT((5+ACTUALLY_PUSHED)*4(%esp)) ; \
GET_FAST_INTR_LOCK ; \
pushl _intr_unit + (irq_num) * 4 ; \
call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
addl $4, %esp ; \
movl $0, lapic_eoi ; \
lock ; \
incl _cnt+V_INTR ; /* book-keeping can wait */ \
movl _intr_countp + (irq_num) * 4,%eax ; \
lock ; \
incl (%eax) ; \
movl _cpl, %eax ; /* unmasking pending HWIs or SWIs? */ \
notl %eax ; \
andl _ipending, %eax ; \
jne 2f ; /* yes, maybe handle them */ \
1: ; \
MEXITCOUNT ; \
REL_FAST_INTR_LOCK ; \
popl %fs ; \
MAYBE_POPL_ES ; \
popl %ds ; \
popl %edx ; \
popl %ecx ; \
popl %eax ; \
iret ; \
; \
ALIGN_TEXT ; \
2: ; \
cmpb $3, _intr_nesting_level ; /* enough stack? */ \
jae 1b ; /* no, return */ \
movl _cpl, %eax ; \
/* XXX next line is probably unnecessary now. */ \
movl $HWI_MASK|SWI_MASK, _cpl ; /* limit nesting ... */ \
lock ; \
incb _intr_nesting_level ; /* ... really limit it ... */ \
sti ; /* to do this as early as possible */ \
popl %fs ; /* discard most of thin frame ... */ \
MAYBE_POPL_ES ; /* discard most of thin frame ... */ \
popl %ecx ; /* ... original %ds ... */ \
popl %edx ; \
xchgl %eax, 4(%esp) ; /* orig %eax; save cpl */ \
pushal ; /* build fat frame (grrr) ... */ \
pushl %ecx ; /* ... actually %ds ... */ \
pushl %es ; \
pushl %fs ;
movl $KDSEL, %eax ; \
movl %ax, %es ; \
movl $KPSEL, %eax ;
movl %ax, %fs ;
movl (3+8+0)*4(%esp), %ecx ; /* %ecx from thin frame ... */ \
movl %ecx, (3+6)*4(%esp) ; /* ... to fat frame ... */ \
movl (3+8+1)*4(%esp), %eax ; /* ... cpl from thin frame */ \
pushl %eax ; \
subl $4, %esp ; /* junk for unit number */ \
MEXITCOUNT ; \
jmp _doreti
#endif /** FAST_WITHOUTCPL */
/*
*
*/
#define PUSH_FRAME \
pushl $0 ; /* dummy error code */ \
pushl $0 ; /* dummy trap type */ \
pushal ; \
pushl %ds ; /* save data and extra segments ... */ \
pushl %es ; \
pushl %fs
#define POP_FRAME \
popl %fs ; \
popl %es ; \
popl %ds ; \
popal ; \
addl $4+4,%esp
#define IOAPICADDR(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 8
#define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 12
#define MASK_IRQ(irq_num) \
IMASK_LOCK ; /* into critical reg */ \
testl $IRQ_BIT(irq_num), _apic_imen ; \
jne 7f ; /* masked, don't mask */ \
orl $IRQ_BIT(irq_num), _apic_imen ; /* set the mask bit */ \
movl IOAPICADDR(irq_num), %ecx ; /* ioapic addr */ \
movl REDIRIDX(irq_num), %eax ; /* get the index */ \
movl %eax, (%ecx) ; /* write the index */ \
movl IOAPIC_WINDOW(%ecx), %eax ; /* current value */ \
orl $IOART_INTMASK, %eax ; /* set the mask */ \
movl %eax, IOAPIC_WINDOW(%ecx) ; /* new value */ \
7: ; /* already masked */ \
IMASK_UNLOCK
/*
* Test to see whether we are handling an edge or level triggered INT.
* Level-triggered INTs must still be masked as we don't clear the source,
* and the EOI cycle would cause redundant INTs to occur.
*/
#define MASK_LEVEL_IRQ(irq_num) \
testl $IRQ_BIT(irq_num), _apic_pin_trigger ; \
jz 9f ; /* edge, don't mask */ \
MASK_IRQ(irq_num) ; \
9:
#ifdef APIC_INTR_REORDER
#define EOI_IRQ(irq_num) \
movl _apic_isrbit_location + 8 * (irq_num), %eax ; \
movl (%eax), %eax ; \
testl _apic_isrbit_location + 4 + 8 * (irq_num), %eax ; \
jz 9f ; /* not active */ \
movl $0, lapic_eoi ; \
APIC_ITRACE(apic_itrace_eoi, irq_num, APIC_ITRACE_EOI) ; \
9:
#else
#define EOI_IRQ(irq_num) \
testl $IRQ_BIT(irq_num), lapic_isr1; \
jz 9f ; /* not active */ \
movl $0, lapic_eoi; \
APIC_ITRACE(apic_itrace_eoi, irq_num, APIC_ITRACE_EOI) ; \
9:
#endif
/*
* Test to see if the source is currntly masked, clear if so.
*/
#define UNMASK_IRQ(irq_num) \
IMASK_LOCK ; /* into critical reg */ \
testl $IRQ_BIT(irq_num), _apic_imen ; \
je 7f ; /* bit clear, not masked */ \
andl $~IRQ_BIT(irq_num), _apic_imen ;/* clear mask bit */ \
movl IOAPICADDR(irq_num),%ecx ; /* ioapic addr */ \
movl REDIRIDX(irq_num), %eax ; /* get the index */ \
movl %eax,(%ecx) ; /* write the index */ \
movl IOAPIC_WINDOW(%ecx),%eax ; /* current value */ \
andl $~IOART_INTMASK,%eax ; /* clear the mask */ \
movl %eax,IOAPIC_WINDOW(%ecx) ; /* new value */ \
7: ; \
IMASK_UNLOCK
#ifdef INTR_SIMPLELOCK
#define ENLOCK
#define DELOCK
#define LATELOCK call _get_isrlock
#else
#define ENLOCK \
ISR_TRYLOCK ; /* XXX this is going away... */ \
testl %eax, %eax ; /* did we get it? */ \
jz 3f
#define DELOCK ISR_RELLOCK
#define LATELOCK
#endif
#ifdef APIC_INTR_DIAGNOSTIC
#ifdef APIC_INTR_DIAGNOSTIC_IRQ
log_intr_event:
pushf
cli
pushl $CNAME(apic_itrace_debuglock)
call CNAME(s_lock_np)
addl $4, %esp
movl CNAME(apic_itrace_debugbuffer_idx), %ecx
andl $32767, %ecx
movl _cpuid, %eax
shll $8, %eax
orl 8(%esp), %eax
movw %ax, CNAME(apic_itrace_debugbuffer)(,%ecx,2)
incl %ecx
andl $32767, %ecx
movl %ecx, CNAME(apic_itrace_debugbuffer_idx)
pushl $CNAME(apic_itrace_debuglock)
call CNAME(s_unlock_np)
addl $4, %esp
popf
ret
#define APIC_ITRACE(name, irq_num, id) \
lock ; /* MP-safe */ \
incl CNAME(name) + (irq_num) * 4 ; \
pushl %eax ; \
pushl %ecx ; \
pushl %edx ; \
movl $(irq_num), %eax ; \
cmpl $APIC_INTR_DIAGNOSTIC_IRQ, %eax ; \
jne 7f ; \
pushl $id ; \
call log_intr_event ; \
addl $4, %esp ; \
7: ; \
popl %edx ; \
popl %ecx ; \
popl %eax
#else
#define APIC_ITRACE(name, irq_num, id) \
lock ; /* MP-safe */ \
incl CNAME(name) + (irq_num) * 4
#endif
#define APIC_ITRACE_ENTER 1
#define APIC_ITRACE_EOI 2
#define APIC_ITRACE_TRYISRLOCK 3
#define APIC_ITRACE_GOTISRLOCK 4
#define APIC_ITRACE_ENTER2 5
#define APIC_ITRACE_LEAVE 6
#define APIC_ITRACE_UNMASK 7
#define APIC_ITRACE_ACTIVE 8
#define APIC_ITRACE_MASKED 9
#define APIC_ITRACE_NOISRLOCK 10
#define APIC_ITRACE_MASKED2 11
#define APIC_ITRACE_SPLZ 12
#define APIC_ITRACE_DORETI 13
#else
#define APIC_ITRACE(name, irq_num, id)
#endif
#ifdef CPL_AND_CML
#define INTR(irq_num, vec_name, maybe_extra_ipending) \
.text ; \
SUPERALIGN_TEXT ; \
/* _XintrNN: entry point used by IDT/HWIs & splz_unpend via _vec[]. */ \
IDTVEC(vec_name) ; \
PUSH_FRAME ; \
movl $KDSEL, %eax ; /* reload with kernel's data segment */ \
movl %ax, %ds ; \
movl %ax, %es ; \
movl $KPSEL, %eax ; \
movl %ax, %fs ; \
; \
maybe_extra_ipending ; \
; \
APIC_ITRACE(apic_itrace_enter, irq_num, APIC_ITRACE_ENTER) ; \
lock ; /* MP-safe */ \
btsl $(irq_num), iactive ; /* lazy masking */ \
jc 1f ; /* already active */ \
; \
MASK_LEVEL_IRQ(irq_num) ; \
EOI_IRQ(irq_num) ; \
0: ; \
APIC_ITRACE(apic_itrace_tryisrlock, irq_num, APIC_ITRACE_TRYISRLOCK) ;\
ENLOCK ; \
; \
APIC_ITRACE(apic_itrace_gotisrlock, irq_num, APIC_ITRACE_GOTISRLOCK) ;\
AVCPL_LOCK ; /* MP-safe */ \
testl $IRQ_BIT(irq_num), _cpl ; \
jne 2f ; /* this INT masked */ \
testl $IRQ_BIT(irq_num), _cml ; \
jne 2f ; /* this INT masked */ \
orl $IRQ_BIT(irq_num), _cil ; \
AVCPL_UNLOCK ; \
; \
incb _intr_nesting_level ; \
; \
/* entry point used by doreti_unpend for HWIs. */ \
__CONCAT(Xresume,irq_num): ; \
FAKE_MCOUNT(13*4(%esp)) ; /* XXX avoid dbl cnt */ \
lock ; incl _cnt+V_INTR ; /* tally interrupts */ \
movl _intr_countp + (irq_num) * 4, %eax ; \
lock ; incl (%eax) ; \
; \
AVCPL_LOCK ; /* MP-safe */ \
movl _cml, %eax ; \
pushl %eax ; \
orl _intr_mask + (irq_num) * 4, %eax ; \
movl %eax, _cml ; \
AVCPL_UNLOCK ; \
; \
pushl _intr_unit + (irq_num) * 4 ; \
incl _inside_intr ; \
APIC_ITRACE(apic_itrace_enter2, irq_num, APIC_ITRACE_ENTER2) ; \
sti ; \
call *_intr_handler + (irq_num) * 4 ; \
cli ; \
APIC_ITRACE(apic_itrace_leave, irq_num, APIC_ITRACE_LEAVE) ; \
decl _inside_intr ; \
; \
lock ; andl $~IRQ_BIT(irq_num), iactive ; \
lock ; andl $~IRQ_BIT(irq_num), _cil ; \
UNMASK_IRQ(irq_num) ; \
APIC_ITRACE(apic_itrace_unmask, irq_num, APIC_ITRACE_UNMASK) ; \
sti ; /* doreti repeats cli/sti */ \
MEXITCOUNT ; \
LATELOCK ; \
jmp _doreti ; \
; \
ALIGN_TEXT ; \
1: ; /* active */ \
APIC_ITRACE(apic_itrace_active, irq_num, APIC_ITRACE_ACTIVE) ; \
MASK_IRQ(irq_num) ; \
EOI_IRQ(irq_num) ; \
AVCPL_LOCK ; /* MP-safe */ \
lock ; \
orl $IRQ_BIT(irq_num), _ipending ; \
AVCPL_UNLOCK ; \
lock ; \
btsl $(irq_num), iactive ; /* still active */ \
jnc 0b ; /* retry */ \
POP_FRAME ; \
iret ; \
; \
ALIGN_TEXT ; \
2: ; /* masked by cpl|cml */ \
APIC_ITRACE(apic_itrace_masked, irq_num, APIC_ITRACE_MASKED) ; \
lock ; \
orl $IRQ_BIT(irq_num), _ipending ; \
AVCPL_UNLOCK ; \
DELOCK ; /* XXX this is going away... */ \
POP_FRAME ; \
iret ; \
ALIGN_TEXT ; \
3: ; /* other cpu has isr lock */ \
APIC_ITRACE(apic_itrace_noisrlock, irq_num, APIC_ITRACE_NOISRLOCK) ;\
AVCPL_LOCK ; /* MP-safe */ \
lock ; \
orl $IRQ_BIT(irq_num), _ipending ; \
testl $IRQ_BIT(irq_num), _cpl ; \
jne 4f ; /* this INT masked */ \
testl $IRQ_BIT(irq_num), _cml ; \
jne 4f ; /* this INT masked */ \
orl $IRQ_BIT(irq_num), _cil ; \
AVCPL_UNLOCK ; \
call forward_irq ; /* forward irq to lock holder */ \
POP_FRAME ; /* and return */ \
iret ; \
ALIGN_TEXT ; \
4: ; /* blocked */ \
APIC_ITRACE(apic_itrace_masked2, irq_num, APIC_ITRACE_MASKED2) ;\
AVCPL_UNLOCK ; \
POP_FRAME ; /* and return */ \
iret
#else /* CPL_AND_CML */
#define INTR(irq_num, vec_name, maybe_extra_ipending) \
.text ; \
SUPERALIGN_TEXT ; \
/* _XintrNN: entry point used by IDT/HWIs & splz_unpend via _vec[]. */ \
IDTVEC(vec_name) ; \
PUSH_FRAME ; \
movl $KDSEL, %eax ; /* reload with kernel's data segment */ \
movl %ax, %ds ; \
movl %ax, %es ; \
movl $KPSEL, %eax ; \
movl %ax, %fs ; \
; \
maybe_extra_ipending ; \
; \
APIC_ITRACE(apic_itrace_enter, irq_num, APIC_ITRACE_ENTER) ; \
lock ; /* MP-safe */ \
btsl $(irq_num), iactive ; /* lazy masking */ \
jc 1f ; /* already active */ \
; \
MASK_LEVEL_IRQ(irq_num) ; \
EOI_IRQ(irq_num) ; \
0: ; \
APIC_ITRACE(apic_itrace_tryisrlock, irq_num, APIC_ITRACE_TRYISRLOCK) ;\
ISR_TRYLOCK ; /* XXX this is going away... */ \
testl %eax, %eax ; /* did we get it? */ \
jz 3f ; /* no */ \
; \
APIC_ITRACE(apic_itrace_gotisrlock, irq_num, APIC_ITRACE_GOTISRLOCK) ;\
AVCPL_LOCK ; /* MP-safe */ \
testl $IRQ_BIT(irq_num), _cpl ; \
jne 2f ; /* this INT masked */ \
AVCPL_UNLOCK ; \
; \
incb _intr_nesting_level ; \
; \
/* entry point used by doreti_unpend for HWIs. */ \
__CONCAT(Xresume,irq_num): ; \
FAKE_MCOUNT(13*4(%esp)) ; /* XXX avoid dbl cnt */ \
lock ; incl _cnt+V_INTR ; /* tally interrupts */ \
movl _intr_countp + (irq_num) * 4, %eax ; \
lock ; incl (%eax) ; \
; \
AVCPL_LOCK ; /* MP-safe */ \
movl _cpl, %eax ; \
pushl %eax ; \
orl _intr_mask + (irq_num) * 4, %eax ; \
movl %eax, _cpl ; \
lock ; \
andl $~IRQ_BIT(irq_num), _ipending ; \
AVCPL_UNLOCK ; \
; \
pushl _intr_unit + (irq_num) * 4 ; \
APIC_ITRACE(apic_itrace_enter2, irq_num, APIC_ITRACE_ENTER2) ; \
sti ; \
call *_intr_handler + (irq_num) * 4 ; \
cli ; \
APIC_ITRACE(apic_itrace_leave, irq_num, APIC_ITRACE_LEAVE) ; \
; \
lock ; andl $~IRQ_BIT(irq_num), iactive ; \
UNMASK_IRQ(irq_num) ; \
APIC_ITRACE(apic_itrace_unmask, irq_num, APIC_ITRACE_UNMASK) ; \
sti ; /* doreti repeats cli/sti */ \
MEXITCOUNT ; \
jmp _doreti ; \
; \
ALIGN_TEXT ; \
1: ; /* active */ \
APIC_ITRACE(apic_itrace_active, irq_num, APIC_ITRACE_ACTIVE) ; \
MASK_IRQ(irq_num) ; \
EOI_IRQ(irq_num) ; \
AVCPL_LOCK ; /* MP-safe */ \
lock ; \
orl $IRQ_BIT(irq_num), _ipending ; \
AVCPL_UNLOCK ; \
lock ; \
btsl $(irq_num), iactive ; /* still active */ \
jnc 0b ; /* retry */ \
POP_FRAME ; \
iret ; /* XXX: iactive bit might be 0 now */ \
ALIGN_TEXT ; \
2: ; /* masked by cpl, leave iactive set */ \
APIC_ITRACE(apic_itrace_masked, irq_num, APIC_ITRACE_MASKED) ; \
lock ; \
orl $IRQ_BIT(irq_num), _ipending ; \
AVCPL_UNLOCK ; \
ISR_RELLOCK ; /* XXX this is going away... */ \
POP_FRAME ; \
iret ; \
ALIGN_TEXT ; \
3: ; /* other cpu has isr lock */ \
APIC_ITRACE(apic_itrace_noisrlock, irq_num, APIC_ITRACE_NOISRLOCK) ;\
AVCPL_LOCK ; /* MP-safe */ \
lock ; \
orl $IRQ_BIT(irq_num), _ipending ; \
testl $IRQ_BIT(irq_num), _cpl ; \
jne 4f ; /* this INT masked */ \
AVCPL_UNLOCK ; \
call forward_irq ; /* forward irq to lock holder */ \
POP_FRAME ; /* and return */ \
iret ; \
ALIGN_TEXT ; \
4: ; /* blocked */ \
APIC_ITRACE(apic_itrace_masked2, irq_num, APIC_ITRACE_MASKED2) ;\
AVCPL_UNLOCK ; \
POP_FRAME ; /* and return */ \
iret
#endif /* CPL_AND_CML */
/*
* Handle "spurious INTerrupts".
* Notes:
* This is different than the "spurious INTerrupt" generated by an
* 8259 PIC for missing INTs. See the APIC documentation for details.
* This routine should NOT do an 'EOI' cycle.
*/
.text
SUPERALIGN_TEXT
.globl _Xspuriousint
_Xspuriousint:
/* No EOI cycle used here */
iret
/*
* Handle TLB shootdowns.
*/
.text
SUPERALIGN_TEXT
.globl _Xinvltlb
_Xinvltlb:
pushl %eax
#ifdef COUNT_XINVLTLB_HITS
pushl %fs
movl $KPSEL, %eax
movl %ax, %fs
movl _cpuid, %eax
popl %fs
ss
incl _xhits(,%eax,4)
#endif /* COUNT_XINVLTLB_HITS */
movl %cr3, %eax /* invalidate the TLB */
movl %eax, %cr3
ss /* stack segment, avoid %ds load */
movl $0, lapic_eoi /* End Of Interrupt to APIC */
popl %eax
iret
#ifdef BETTER_CLOCK
/*
* Executed by a CPU when it receives an Xcpucheckstate IPI from another CPU,
*
* - Stores current cpu state in checkstate_cpustate[cpuid]
* 0 == user, 1 == sys, 2 == intr
* - Stores current process in checkstate_curproc[cpuid]
*
* - Signals its receipt by setting bit cpuid in checkstate_probed_cpus.
*
* stack: 0->ds, 4->fs, 8->ebx, 12->eax, 16->eip, 20->cs, 24->eflags
*/
.text
SUPERALIGN_TEXT
.globl _Xcpucheckstate
.globl _checkstate_cpustate
.globl _checkstate_curproc
.globl _checkstate_pc
_Xcpucheckstate:
pushl %eax
pushl %ebx
pushl %ds /* save current data segment */
pushl %fs
movl $KDSEL, %eax
movl %ax, %ds /* use KERNEL data segment */
movl $KPSEL, %eax
movl %ax, %fs
movl $0, lapic_eoi /* End Of Interrupt to APIC */
movl $0, %ebx
movl 20(%esp), %eax
andl $3, %eax
cmpl $3, %eax
je 1f
testl $PSL_VM, 24(%esp)
jne 1f
incl %ebx /* system or interrupt */
#ifdef CPL_AND_CML
cmpl $0, _inside_intr
je 1f
incl %ebx /* interrupt */
#endif
1:
movl _cpuid, %eax
movl %ebx, _checkstate_cpustate(,%eax,4)
movl _curproc, %ebx
movl %ebx, _checkstate_curproc(,%eax,4)
movl 16(%esp), %ebx
movl %ebx, _checkstate_pc(,%eax,4)
lock /* checkstate_probed_cpus |= (1<<id) */
btsl %eax, _checkstate_probed_cpus
popl %fs
popl %ds /* restore previous data segment */
popl %ebx
popl %eax
iret
#endif /* BETTER_CLOCK */
/*
* Executed by a CPU when it receives an Xcpuast IPI from another CPU,
*
* - Signals its receipt by clearing bit cpuid in checkstate_need_ast.
*
* - We need a better method of triggering asts on other cpus.
*/
.text
SUPERALIGN_TEXT
.globl _Xcpuast
_Xcpuast:
PUSH_FRAME
movl $KDSEL, %eax
movl %ax, %ds /* use KERNEL data segment */
movl %ax, %es
movl $KPSEL, %eax
movl %ax, %fs
movl _cpuid, %eax
lock /* checkstate_need_ast &= ~(1<<id) */
btrl %eax, _checkstate_need_ast
movl $0, lapic_eoi /* End Of Interrupt to APIC */
lock
btsl %eax, _checkstate_pending_ast
jc 1f
FAKE_MCOUNT(13*4(%esp))
/*
* Giant locks do not come cheap.
* A lot of cycles are going to be wasted here.
*/
call _get_isrlock
AVCPL_LOCK
#ifdef CPL_AND_CML
movl _cml, %eax
#else
movl _cpl, %eax
#endif
pushl %eax
movl $1, _astpending /* XXX */
AVCPL_UNLOCK
lock
incb _intr_nesting_level
sti
pushl $0
movl _cpuid, %eax
lock
btrl %eax, _checkstate_pending_ast
lock
btrl %eax, CNAME(resched_cpus)
jnc 2f
movl $1, CNAME(want_resched)
lock
incl CNAME(want_resched_cnt)
2:
lock
incl CNAME(cpuast_cnt)
MEXITCOUNT
jmp _doreti
1:
/* We are already in the process of delivering an ast for this CPU */
POP_FRAME
iret
/*
* Executed by a CPU when it receives an XFORWARD_IRQ IPI.
*/
.text
SUPERALIGN_TEXT
.globl _Xforward_irq
_Xforward_irq:
PUSH_FRAME
movl $KDSEL, %eax
movl %ax, %ds /* use KERNEL data segment */
movl %ax, %es
movl $KPSEL, %eax
movl %ax, %fs
movl $0, lapic_eoi /* End Of Interrupt to APIC */
FAKE_MCOUNT(13*4(%esp))
ISR_TRYLOCK
testl %eax,%eax /* Did we get the lock ? */
jz 1f /* No */
lock
incl CNAME(forward_irq_hitcnt)
cmpb $4, _intr_nesting_level
jae 2f
AVCPL_LOCK
#ifdef CPL_AND_CML
movl _cml, %eax
#else
movl _cpl, %eax
#endif
pushl %eax
AVCPL_UNLOCK
lock
incb _intr_nesting_level
sti
pushl $0
MEXITCOUNT
jmp _doreti /* Handle forwarded interrupt */
1:
lock
incl CNAME(forward_irq_misscnt)
call forward_irq /* Oops, we've lost the isr lock */
MEXITCOUNT
POP_FRAME
iret
2:
lock
incl CNAME(forward_irq_toodeepcnt)
3:
ISR_RELLOCK
MEXITCOUNT
POP_FRAME
iret
/*
*
*/
forward_irq:
MCOUNT
cmpl $0,_invltlb_ok
jz 4f
cmpl $0, CNAME(forward_irq_enabled)
jz 4f
movl _mp_lock,%eax
cmpl $FREE_LOCK,%eax
jne 1f
movl $0, %eax /* Pick CPU #0 if noone has lock */
1:
shrl $24,%eax
movl _cpu_num_to_apic_id(,%eax,4),%ecx
shll $24,%ecx
movl lapic_icr_hi, %eax
andl $~APIC_ID_MASK, %eax
orl %ecx, %eax
movl %eax, lapic_icr_hi
2:
movl lapic_icr_lo, %eax
andl $APIC_DELSTAT_MASK,%eax
jnz 2b
movl lapic_icr_lo, %eax
andl $APIC_RESV2_MASK, %eax
orl $(APIC_DEST_DESTFLD|APIC_DELMODE_FIXED|XFORWARD_IRQ_OFFSET), %eax
movl %eax, lapic_icr_lo
3:
movl lapic_icr_lo, %eax
andl $APIC_DELSTAT_MASK,%eax
jnz 3b
4:
ret
/*
* Executed by a CPU when it receives an Xcpustop IPI from another CPU,
*
* - Signals its receipt.
* - Waits for permission to restart.
* - Signals its restart.
*/
.text
SUPERALIGN_TEXT
.globl _Xcpustop
_Xcpustop:
pushl %ebp
movl %esp, %ebp
pushl %eax
pushl %ecx
pushl %edx
pushl %ds /* save current data segment */
pushl %fs
movl $KDSEL, %eax
movl %ax, %ds /* use KERNEL data segment */
movl $KPSEL, %eax
movl %ax, %fs
movl $0, lapic_eoi /* End Of Interrupt to APIC */
movl _cpuid, %eax
imull $PCB_SIZE, %eax
leal CNAME(stoppcbs)(%eax), %eax
pushl %eax
call CNAME(savectx) /* Save process context */
addl $4, %esp
movl _cpuid, %eax
lock
btsl %eax, _stopped_cpus /* stopped_cpus |= (1<<id) */
1:
btl %eax, _started_cpus /* while (!(started_cpus & (1<<id))) */
jnc 1b
lock
btrl %eax, _started_cpus /* started_cpus &= ~(1<<id) */
lock
btrl %eax, _stopped_cpus /* stopped_cpus &= ~(1<<id) */
test %eax, %eax
jnz 2f
movl CNAME(cpustop_restartfunc), %eax
test %eax, %eax
jz 2f
movl $0, CNAME(cpustop_restartfunc) /* One-shot */
call %eax
2:
popl %fs
popl %ds /* restore previous data segment */
popl %edx
popl %ecx
popl %eax
movl %ebp, %esp
popl %ebp
iret
MCOUNT_LABEL(bintr)
FAST_INTR(0,fastintr0)
FAST_INTR(1,fastintr1)
FAST_INTR(2,fastintr2)
FAST_INTR(3,fastintr3)
FAST_INTR(4,fastintr4)
FAST_INTR(5,fastintr5)
FAST_INTR(6,fastintr6)
FAST_INTR(7,fastintr7)
FAST_INTR(8,fastintr8)
FAST_INTR(9,fastintr9)
FAST_INTR(10,fastintr10)
FAST_INTR(11,fastintr11)
FAST_INTR(12,fastintr12)
FAST_INTR(13,fastintr13)
FAST_INTR(14,fastintr14)
FAST_INTR(15,fastintr15)
FAST_INTR(16,fastintr16)
FAST_INTR(17,fastintr17)
FAST_INTR(18,fastintr18)
FAST_INTR(19,fastintr19)
FAST_INTR(20,fastintr20)
FAST_INTR(21,fastintr21)
FAST_INTR(22,fastintr22)
FAST_INTR(23,fastintr23)
#define CLKINTR_PENDING movl $1,CNAME(clkintr_pending)
INTR(0,intr0, CLKINTR_PENDING)
INTR(1,intr1,)
INTR(2,intr2,)
INTR(3,intr3,)
INTR(4,intr4,)
INTR(5,intr5,)
INTR(6,intr6,)
INTR(7,intr7,)
INTR(8,intr8,)
INTR(9,intr9,)
INTR(10,intr10,)
INTR(11,intr11,)
INTR(12,intr12,)
INTR(13,intr13,)
INTR(14,intr14,)
INTR(15,intr15,)
INTR(16,intr16,)
INTR(17,intr17,)
INTR(18,intr18,)
INTR(19,intr19,)
INTR(20,intr20,)
INTR(21,intr21,)
INTR(22,intr22,)
INTR(23,intr23,)
MCOUNT_LABEL(eintr)
/*
* Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
*
* - Calls the generic rendezvous action function.
*/
.text
SUPERALIGN_TEXT
.globl _Xrendezvous
_Xrendezvous:
PUSH_FRAME
movl $KDSEL, %eax
movl %ax, %ds /* use KERNEL data segment */
movl %ax, %es
movl $KPSEL, %eax
movl %ax, %fs
call smp_rendezvous_action
movl $0, lapic_eoi /* End Of Interrupt to APIC */
POP_FRAME
iret
.data
/*
* Addresses of interrupt handlers.
* XresumeNN: Resumption addresses for HWIs.
*/
.globl _ihandlers
_ihandlers:
/*
* used by:
* ipl.s: doreti_unpend
*/
.long Xresume0, Xresume1, Xresume2, Xresume3
.long Xresume4, Xresume5, Xresume6, Xresume7
.long Xresume8, Xresume9, Xresume10, Xresume11
.long Xresume12, Xresume13, Xresume14, Xresume15
.long Xresume16, Xresume17, Xresume18, Xresume19
.long Xresume20, Xresume21, Xresume22, Xresume23
/*
* used by:
* ipl.s: doreti_unpend
* apic_ipl.s: splz_unpend
*/
.long _swi_null, swi_net, _swi_null, _swi_null
.long _swi_vm, _swi_null, _softclock, _swi_null
imasks: /* masks for interrupt handlers */
.space NHWI*4 /* padding; HWI masks are elsewhere */
.long SWI_TTY_MASK, SWI_NET_MASK, SWI_CAMNET_MASK, SWI_CAMBIO_MASK
.long SWI_VM_MASK, 0, SWI_CLOCK_MASK, 0
/* active flag for lazy masking */
iactive:
.long 0
#ifdef COUNT_XINVLTLB_HITS
.globl _xhits
_xhits:
.space (NCPU * 4), 0
#endif /* COUNT_XINVLTLB_HITS */
/* variables used by stop_cpus()/restart_cpus()/Xcpustop */
.globl _stopped_cpus, _started_cpus
_stopped_cpus:
.long 0
_started_cpus:
.long 0
#ifdef BETTER_CLOCK
.globl _checkstate_probed_cpus
_checkstate_probed_cpus:
.long 0
#endif /* BETTER_CLOCK */
.globl _checkstate_need_ast
_checkstate_need_ast:
.long 0
_checkstate_pending_ast:
.long 0
.globl CNAME(forward_irq_misscnt)
.globl CNAME(forward_irq_toodeepcnt)
.globl CNAME(forward_irq_hitcnt)
.globl CNAME(resched_cpus)
.globl CNAME(want_resched_cnt)
.globl CNAME(cpuast_cnt)
.globl CNAME(cpustop_restartfunc)
CNAME(forward_irq_misscnt):
.long 0
CNAME(forward_irq_hitcnt):
.long 0
CNAME(forward_irq_toodeepcnt):
.long 0
CNAME(resched_cpus):
.long 0
CNAME(want_resched_cnt):
.long 0
CNAME(cpuast_cnt):
.long 0
CNAME(cpustop_restartfunc):
.long 0
.globl _apic_pin_trigger
_apic_pin_trigger:
.long 0
.text
Reference in New Issue View Git Blame Copy Permalink