Commit major SMP cleanups and move the BGL (big giant lock) in the
syscall path inward. A system call may select whether it needs the MP lock or not (the default being that it does need it). A great deal of conditional SMP code for various deadended experiments has been removed. 'cil' and 'cml' have been removed entirely, and the locking around the cpl has been removed. The conditional separately-locked fast-interrupt code has been removed, meaning that interrupts must hold the CPL now (but they pretty much had to anyway). Another reason for doing this is that the original separate-lock for interrupts just doesn't apply to the interrupt thread mechanism being contemplated. Modifications to the cpl may now ONLY occur while holding the MP lock. For example, if an otherwise MP safe syscall needs to mess with the cpl, it must hold the MP lock for the duration and must (as usual) save/restore the cpl in a nested fashion. This is precursor work for the real meat coming later: avoiding having to hold the MP lock for common syscalls and I/O's and interrupt threads. It is expected that the spl mechanisms and new interrupt threading mechanisms will be able to run in tandem, allowing a slow piecemeal transition to occur. This patch should result in a moderate performance improvement due to the considerable amount of code that has been removed from the critical path, especially the simplification of the spl*() calls. The real performance gains will come later. Approved by: jkh Reviewed by: current, bde (exception.s) Some work taken from: luoqi's patch
This commit is contained in:
parent
6d23c3828e
commit
36e9f877df
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=58717
@ -615,7 +615,7 @@ syscall(code, framep)
|
||||
else
|
||||
callp = &p->p_sysent->sv_table[code];
|
||||
|
||||
nargs = callp->sy_narg + hidden;
|
||||
nargs = (callp->sy_narg & SYF_ARGMASK) + hidden;
|
||||
switch (nargs) {
|
||||
default:
|
||||
if (nargs > 10) /* XXX */
|
||||
@ -639,13 +639,13 @@ syscall(code, framep)
|
||||
}
|
||||
#ifdef KTRACE
|
||||
if (KTRPOINT(p, KTR_SYSCALL))
|
||||
ktrsyscall(p->p_tracep, code, callp->sy_narg, args + hidden);
|
||||
ktrsyscall(p->p_tracep, code, (callp->sy_narg & SYF_ARGMASK), args + hidden);
|
||||
#endif
|
||||
if (error == 0) {
|
||||
p->p_retval[0] = 0;
|
||||
p->p_retval[1] = 0;
|
||||
|
||||
STOPEVENT(p, S_SCE, callp->sy_narg);
|
||||
STOPEVENT(p, S_SCE, (callp->sy_narg & SYF_ARGMASK));
|
||||
|
||||
error = (*callp->sy_call)(p, args + hidden);
|
||||
}
|
||||
|
@ -9,28 +9,17 @@
|
||||
|
||||
#include "i386/isa/intr_machdep.h"
|
||||
|
||||
|
||||
#ifdef FAST_SIMPLELOCK
|
||||
/*
|
||||
* Interrupts must be enabled while waiting for the MP lock.
|
||||
*/
|
||||
|
||||
#define GET_FAST_INTR_LOCK \
|
||||
pushl $_fast_intr_lock ; /* address of lock */ \
|
||||
call _s_lock ; /* MP-safe */ \
|
||||
addl $4,%esp
|
||||
|
||||
#define REL_FAST_INTR_LOCK \
|
||||
movl $0, _fast_intr_lock
|
||||
|
||||
#else /* FAST_SIMPLELOCK */
|
||||
|
||||
#define GET_FAST_INTR_LOCK \
|
||||
call _get_isrlock
|
||||
sti; call _get_mplock; cli
|
||||
|
||||
#define REL_FAST_INTR_LOCK \
|
||||
movl $_mp_lock, %edx ; /* GIANT_LOCK */ \
|
||||
call _MPrellock_edx
|
||||
|
||||
#endif /* FAST_SIMPLELOCK */
|
||||
|
||||
/* convert an absolute IRQ# into a bitmask */
|
||||
#define IRQ_BIT(irq_num) (1 << (irq_num))
|
||||
|
||||
@ -42,10 +31,6 @@
|
||||
* Macros for interrupt interrupt entry, call to handler, and exit.
|
||||
*/
|
||||
|
||||
#ifdef FAST_WITHOUTCPL
|
||||
|
||||
/*
|
||||
*/
|
||||
#define FAST_INTR(irq_num, vec_name) \
|
||||
.text ; \
|
||||
SUPERALIGN_TEXT ; \
|
||||
@ -82,83 +67,6 @@ IDTVEC(vec_name) ; \
|
||||
popl %eax ; \
|
||||
iret
|
||||
|
||||
#else /* FAST_WITHOUTCPL */
|
||||
|
||||
#define FAST_INTR(irq_num, vec_name) \
|
||||
.text ; \
|
||||
SUPERALIGN_TEXT ; \
|
||||
IDTVEC(vec_name) ; \
|
||||
pushl %eax ; /* save only call-used registers */ \
|
||||
pushl %ecx ; \
|
||||
pushl %edx ; \
|
||||
pushl %ds ; \
|
||||
MAYBE_PUSHL_ES ; \
|
||||
pushl %fs ; \
|
||||
movl $KDSEL, %eax ; \
|
||||
movl %ax, %ds ; \
|
||||
MAYBE_MOVW_AX_ES ; \
|
||||
movl $KPSEL, %eax ; \
|
||||
movl %ax, %fs ; \
|
||||
FAKE_MCOUNT((5+ACTUALLY_PUSHED)*4(%esp)) ; \
|
||||
GET_FAST_INTR_LOCK ; \
|
||||
pushl _intr_unit + (irq_num) * 4 ; \
|
||||
call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
|
||||
addl $4, %esp ; \
|
||||
movl $0, lapic_eoi ; \
|
||||
lock ; \
|
||||
incl _cnt+V_INTR ; /* book-keeping can wait */ \
|
||||
movl _intr_countp + (irq_num) * 4,%eax ; \
|
||||
lock ; \
|
||||
incl (%eax) ; \
|
||||
movl _cpl, %eax ; /* unmasking pending HWIs or SWIs? */ \
|
||||
notl %eax ; \
|
||||
andl _ipending, %eax ; \
|
||||
jne 2f ; /* yes, maybe handle them */ \
|
||||
1: ; \
|
||||
MEXITCOUNT ; \
|
||||
REL_FAST_INTR_LOCK ; \
|
||||
popl %fs ; \
|
||||
MAYBE_POPL_ES ; \
|
||||
popl %ds ; \
|
||||
popl %edx ; \
|
||||
popl %ecx ; \
|
||||
popl %eax ; \
|
||||
iret ; \
|
||||
; \
|
||||
ALIGN_TEXT ; \
|
||||
2: ; \
|
||||
cmpb $3, _intr_nesting_level ; /* enough stack? */ \
|
||||
jae 1b ; /* no, return */ \
|
||||
movl _cpl, %eax ; \
|
||||
/* XXX next line is probably unnecessary now. */ \
|
||||
movl $HWI_MASK|SWI_MASK, _cpl ; /* limit nesting ... */ \
|
||||
lock ; \
|
||||
incb _intr_nesting_level ; /* ... really limit it ... */ \
|
||||
sti ; /* to do this as early as possible */ \
|
||||
popl %fs ; /* discard most of thin frame ... */ \
|
||||
MAYBE_POPL_ES ; /* discard most of thin frame ... */ \
|
||||
popl %ecx ; /* ... original %ds ... */ \
|
||||
popl %edx ; \
|
||||
xchgl %eax, 4(%esp) ; /* orig %eax; save cpl */ \
|
||||
pushal ; /* build fat frame (grrr) ... */ \
|
||||
pushl %ecx ; /* ... actually %ds ... */ \
|
||||
pushl %es ; \
|
||||
pushl %fs ;
|
||||
movl $KDSEL, %eax ; \
|
||||
movl %ax, %es ; \
|
||||
movl $KPSEL, %eax ;
|
||||
movl %ax, %fs ;
|
||||
movl (3+8+0)*4(%esp), %ecx ; /* %ecx from thin frame ... */ \
|
||||
movl %ecx, (3+6)*4(%esp) ; /* ... to fat frame ... */ \
|
||||
movl (3+8+1)*4(%esp), %eax ; /* ... cpl from thin frame */ \
|
||||
pushl %eax ; \
|
||||
subl $4, %esp ; /* junk for unit number */ \
|
||||
MEXITCOUNT ; \
|
||||
jmp _doreti
|
||||
|
||||
#endif /** FAST_WITHOUTCPL */
|
||||
|
||||
|
||||
/*
|
||||
*
|
||||
*/
|
||||
@ -242,19 +150,6 @@ IDTVEC(vec_name) ; \
|
||||
7: ; \
|
||||
IMASK_UNLOCK
|
||||
|
||||
#ifdef INTR_SIMPLELOCK
|
||||
#define ENLOCK
|
||||
#define DELOCK
|
||||
#define LATELOCK call _get_isrlock
|
||||
#else
|
||||
#define ENLOCK \
|
||||
ISR_TRYLOCK ; /* XXX this is going away... */ \
|
||||
testl %eax, %eax ; /* did we get it? */ \
|
||||
jz 3f
|
||||
#define DELOCK ISR_RELLOCK
|
||||
#define LATELOCK
|
||||
#endif
|
||||
|
||||
#ifdef APIC_INTR_DIAGNOSTIC
|
||||
#ifdef APIC_INTR_DIAGNOSTIC_IRQ
|
||||
log_intr_event:
|
||||
@ -319,8 +214,6 @@ log_intr_event:
|
||||
#define APIC_ITRACE(name, irq_num, id)
|
||||
#endif
|
||||
|
||||
#ifdef CPL_AND_CML
|
||||
|
||||
#define INTR(irq_num, vec_name, maybe_extra_ipending) \
|
||||
.text ; \
|
||||
SUPERALIGN_TEXT ; \
|
||||
@ -344,132 +237,13 @@ IDTVEC(vec_name) ; \
|
||||
EOI_IRQ(irq_num) ; \
|
||||
0: ; \
|
||||
APIC_ITRACE(apic_itrace_tryisrlock, irq_num, APIC_ITRACE_TRYISRLOCK) ;\
|
||||
ENLOCK ; \
|
||||
; \
|
||||
APIC_ITRACE(apic_itrace_gotisrlock, irq_num, APIC_ITRACE_GOTISRLOCK) ;\
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
testl $IRQ_BIT(irq_num), _cpl ; \
|
||||
jne 2f ; /* this INT masked */ \
|
||||
testl $IRQ_BIT(irq_num), _cml ; \
|
||||
jne 2f ; /* this INT masked */ \
|
||||
orl $IRQ_BIT(irq_num), _cil ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
; \
|
||||
incb _intr_nesting_level ; \
|
||||
; \
|
||||
/* entry point used by doreti_unpend for HWIs. */ \
|
||||
__CONCAT(Xresume,irq_num): ; \
|
||||
FAKE_MCOUNT(13*4(%esp)) ; /* XXX avoid dbl cnt */ \
|
||||
lock ; incl _cnt+V_INTR ; /* tally interrupts */ \
|
||||
movl _intr_countp + (irq_num) * 4, %eax ; \
|
||||
lock ; incl (%eax) ; \
|
||||
; \
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
movl _cml, %eax ; \
|
||||
pushl %eax ; \
|
||||
orl _intr_mask + (irq_num) * 4, %eax ; \
|
||||
movl %eax, _cml ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
; \
|
||||
pushl _intr_unit + (irq_num) * 4 ; \
|
||||
incl _inside_intr ; \
|
||||
APIC_ITRACE(apic_itrace_enter2, irq_num, APIC_ITRACE_ENTER2) ; \
|
||||
sti ; \
|
||||
call *_intr_handler + (irq_num) * 4 ; \
|
||||
cli ; \
|
||||
APIC_ITRACE(apic_itrace_leave, irq_num, APIC_ITRACE_LEAVE) ; \
|
||||
decl _inside_intr ; \
|
||||
; \
|
||||
lock ; andl $~IRQ_BIT(irq_num), iactive ; \
|
||||
lock ; andl $~IRQ_BIT(irq_num), _cil ; \
|
||||
UNMASK_IRQ(irq_num) ; \
|
||||
APIC_ITRACE(apic_itrace_unmask, irq_num, APIC_ITRACE_UNMASK) ; \
|
||||
sti ; /* doreti repeats cli/sti */ \
|
||||
MEXITCOUNT ; \
|
||||
LATELOCK ; \
|
||||
jmp _doreti ; \
|
||||
; \
|
||||
ALIGN_TEXT ; \
|
||||
1: ; /* active */ \
|
||||
APIC_ITRACE(apic_itrace_active, irq_num, APIC_ITRACE_ACTIVE) ; \
|
||||
MASK_IRQ(irq_num) ; \
|
||||
EOI_IRQ(irq_num) ; \
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
lock ; \
|
||||
orl $IRQ_BIT(irq_num), _ipending ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
lock ; \
|
||||
btsl $(irq_num), iactive ; /* still active */ \
|
||||
jnc 0b ; /* retry */ \
|
||||
POP_FRAME ; \
|
||||
iret ; \
|
||||
; \
|
||||
ALIGN_TEXT ; \
|
||||
2: ; /* masked by cpl|cml */ \
|
||||
APIC_ITRACE(apic_itrace_masked, irq_num, APIC_ITRACE_MASKED) ; \
|
||||
lock ; \
|
||||
orl $IRQ_BIT(irq_num), _ipending ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
DELOCK ; /* XXX this is going away... */ \
|
||||
POP_FRAME ; \
|
||||
iret ; \
|
||||
ALIGN_TEXT ; \
|
||||
3: ; /* other cpu has isr lock */ \
|
||||
APIC_ITRACE(apic_itrace_noisrlock, irq_num, APIC_ITRACE_NOISRLOCK) ;\
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
lock ; \
|
||||
orl $IRQ_BIT(irq_num), _ipending ; \
|
||||
testl $IRQ_BIT(irq_num), _cpl ; \
|
||||
jne 4f ; /* this INT masked */ \
|
||||
testl $IRQ_BIT(irq_num), _cml ; \
|
||||
jne 4f ; /* this INT masked */ \
|
||||
orl $IRQ_BIT(irq_num), _cil ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
call forward_irq ; /* forward irq to lock holder */ \
|
||||
POP_FRAME ; /* and return */ \
|
||||
iret ; \
|
||||
ALIGN_TEXT ; \
|
||||
4: ; /* blocked */ \
|
||||
APIC_ITRACE(apic_itrace_masked2, irq_num, APIC_ITRACE_MASKED2) ;\
|
||||
AVCPL_UNLOCK ; \
|
||||
POP_FRAME ; /* and return */ \
|
||||
iret
|
||||
|
||||
#else /* CPL_AND_CML */
|
||||
|
||||
|
||||
#define INTR(irq_num, vec_name, maybe_extra_ipending) \
|
||||
.text ; \
|
||||
SUPERALIGN_TEXT ; \
|
||||
/* _XintrNN: entry point used by IDT/HWIs & splz_unpend via _vec[]. */ \
|
||||
IDTVEC(vec_name) ; \
|
||||
PUSH_FRAME ; \
|
||||
movl $KDSEL, %eax ; /* reload with kernel's data segment */ \
|
||||
movl %ax, %ds ; \
|
||||
movl %ax, %es ; \
|
||||
movl $KPSEL, %eax ; \
|
||||
movl %ax, %fs ; \
|
||||
; \
|
||||
maybe_extra_ipending ; \
|
||||
; \
|
||||
APIC_ITRACE(apic_itrace_enter, irq_num, APIC_ITRACE_ENTER) ; \
|
||||
lock ; /* MP-safe */ \
|
||||
btsl $(irq_num), iactive ; /* lazy masking */ \
|
||||
jc 1f ; /* already active */ \
|
||||
; \
|
||||
MASK_LEVEL_IRQ(irq_num) ; \
|
||||
EOI_IRQ(irq_num) ; \
|
||||
0: ; \
|
||||
APIC_ITRACE(apic_itrace_tryisrlock, irq_num, APIC_ITRACE_TRYISRLOCK) ;\
|
||||
ISR_TRYLOCK ; /* XXX this is going away... */ \
|
||||
MP_TRYLOCK ; /* XXX this is going away... */ \
|
||||
testl %eax, %eax ; /* did we get it? */ \
|
||||
jz 3f ; /* no */ \
|
||||
; \
|
||||
APIC_ITRACE(apic_itrace_gotisrlock, irq_num, APIC_ITRACE_GOTISRLOCK) ;\
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
testl $IRQ_BIT(irq_num), _cpl ; \
|
||||
jne 2f ; /* this INT masked */ \
|
||||
AVCPL_UNLOCK ; \
|
||||
; \
|
||||
incb _intr_nesting_level ; \
|
||||
; \
|
||||
@ -480,14 +254,12 @@ __CONCAT(Xresume,irq_num): ; \
|
||||
movl _intr_countp + (irq_num) * 4, %eax ; \
|
||||
lock ; incl (%eax) ; \
|
||||
; \
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
movl _cpl, %eax ; \
|
||||
pushl %eax ; \
|
||||
orl _intr_mask + (irq_num) * 4, %eax ; \
|
||||
movl %eax, _cpl ; \
|
||||
lock ; \
|
||||
andl $~IRQ_BIT(irq_num), _ipending ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
; \
|
||||
pushl _intr_unit + (irq_num) * 4 ; \
|
||||
APIC_ITRACE(apic_itrace_enter2, irq_num, APIC_ITRACE_ENTER2) ; \
|
||||
@ -508,10 +280,8 @@ __CONCAT(Xresume,irq_num): ; \
|
||||
APIC_ITRACE(apic_itrace_active, irq_num, APIC_ITRACE_ACTIVE) ; \
|
||||
MASK_IRQ(irq_num) ; \
|
||||
EOI_IRQ(irq_num) ; \
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
lock ; \
|
||||
orl $IRQ_BIT(irq_num), _ipending ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
lock ; \
|
||||
btsl $(irq_num), iactive ; /* still active */ \
|
||||
jnc 0b ; /* retry */ \
|
||||
@ -522,32 +292,25 @@ __CONCAT(Xresume,irq_num): ; \
|
||||
APIC_ITRACE(apic_itrace_masked, irq_num, APIC_ITRACE_MASKED) ; \
|
||||
lock ; \
|
||||
orl $IRQ_BIT(irq_num), _ipending ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
ISR_RELLOCK ; /* XXX this is going away... */ \
|
||||
MP_RELLOCK ; \
|
||||
POP_FRAME ; \
|
||||
iret ; \
|
||||
ALIGN_TEXT ; \
|
||||
3: ; /* other cpu has isr lock */ \
|
||||
APIC_ITRACE(apic_itrace_noisrlock, irq_num, APIC_ITRACE_NOISRLOCK) ;\
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
lock ; \
|
||||
orl $IRQ_BIT(irq_num), _ipending ; \
|
||||
testl $IRQ_BIT(irq_num), _cpl ; \
|
||||
jne 4f ; /* this INT masked */ \
|
||||
AVCPL_UNLOCK ; \
|
||||
call forward_irq ; /* forward irq to lock holder */ \
|
||||
POP_FRAME ; /* and return */ \
|
||||
iret ; \
|
||||
ALIGN_TEXT ; \
|
||||
4: ; /* blocked */ \
|
||||
APIC_ITRACE(apic_itrace_masked2, irq_num, APIC_ITRACE_MASKED2) ;\
|
||||
AVCPL_UNLOCK ; \
|
||||
POP_FRAME ; /* and return */ \
|
||||
iret
|
||||
|
||||
#endif /* CPL_AND_CML */
|
||||
|
||||
|
||||
/*
|
||||
* Handle "spurious INTerrupts".
|
||||
* Notes:
|
||||
@ -635,11 +398,6 @@ _Xcpucheckstate:
|
||||
testl $PSL_VM, 24(%esp)
|
||||
jne 1f
|
||||
incl %ebx /* system or interrupt */
|
||||
#ifdef CPL_AND_CML
|
||||
cmpl $0, _inside_intr
|
||||
je 1f
|
||||
incl %ebx /* interrupt */
|
||||
#endif
|
||||
1:
|
||||
movl _cpuid, %eax
|
||||
movl %ebx, _checkstate_cpustate(,%eax,4)
|
||||
@ -693,17 +451,11 @@ _Xcpuast:
|
||||
* Giant locks do not come cheap.
|
||||
* A lot of cycles are going to be wasted here.
|
||||
*/
|
||||
call _get_isrlock
|
||||
call _get_mplock
|
||||
|
||||
AVCPL_LOCK
|
||||
#ifdef CPL_AND_CML
|
||||
movl _cml, %eax
|
||||
#else
|
||||
movl _cpl, %eax
|
||||
#endif
|
||||
pushl %eax
|
||||
movl $1, _astpending /* XXX */
|
||||
AVCPL_UNLOCK
|
||||
lock
|
||||
incb _intr_nesting_level
|
||||
sti
|
||||
@ -716,7 +468,7 @@ _Xcpuast:
|
||||
lock
|
||||
btrl %eax, CNAME(resched_cpus)
|
||||
jnc 2f
|
||||
movl $1, CNAME(want_resched)
|
||||
orl $AST_RESCHED,_astpending
|
||||
lock
|
||||
incl CNAME(want_resched_cnt)
|
||||
2:
|
||||
@ -749,7 +501,7 @@ _Xforward_irq:
|
||||
|
||||
FAKE_MCOUNT(13*4(%esp))
|
||||
|
||||
ISR_TRYLOCK
|
||||
MP_TRYLOCK
|
||||
testl %eax,%eax /* Did we get the lock ? */
|
||||
jz 1f /* No */
|
||||
|
||||
@ -758,14 +510,8 @@ _Xforward_irq:
|
||||
cmpb $4, _intr_nesting_level
|
||||
jae 2f
|
||||
|
||||
AVCPL_LOCK
|
||||
#ifdef CPL_AND_CML
|
||||
movl _cml, %eax
|
||||
#else
|
||||
movl _cpl, %eax
|
||||
#endif
|
||||
pushl %eax
|
||||
AVCPL_UNLOCK
|
||||
lock
|
||||
incb _intr_nesting_level
|
||||
sti
|
||||
@ -785,7 +531,7 @@ _Xforward_irq:
|
||||
lock
|
||||
incl CNAME(forward_irq_toodeepcnt)
|
||||
3:
|
||||
ISR_RELLOCK
|
||||
MP_RELLOCK
|
||||
MEXITCOUNT
|
||||
POP_FRAME
|
||||
iret
|
||||
|
@ -65,8 +65,6 @@ _hlt_vector: .long _default_halt /* pointer to halt routine */
|
||||
|
||||
.globl _panic
|
||||
|
||||
.globl _want_resched
|
||||
_want_resched: .long 0 /* we need to re-run the scheduler */
|
||||
#if defined(SWTCH_OPTIM_STATS)
|
||||
.globl _swtch_optim_stats, _tlb_flush_count
|
||||
_swtch_optim_stats: .long 0 /* number of _swtch_optims */
|
||||
@ -129,6 +127,9 @@ _idle:
|
||||
/*
|
||||
* XXX callers of cpu_switch() do a bogus splclock(). Locking should
|
||||
* be left to cpu_switch().
|
||||
*
|
||||
* NOTE: spl*() may only be called while we hold the MP lock (which
|
||||
* we do).
|
||||
*/
|
||||
call _spl0
|
||||
|
||||
@ -159,14 +160,14 @@ idle_loop:
|
||||
testl %eax,%eax
|
||||
jnz 3f
|
||||
|
||||
/*
|
||||
* Handle page-zeroing in the idle loop. Called with interrupts
|
||||
* disabled and the MP lock released. Inside vm_page_zero_idle
|
||||
* we enable interrupts and grab the mplock as required.
|
||||
*/
|
||||
cmpl $0,_do_page_zero_idle
|
||||
je 2f
|
||||
|
||||
/* XXX appears to cause panics */
|
||||
/*
|
||||
* Inside zero_idle we enable interrupts and grab the mplock
|
||||
* as needed. It needs to be careful about entry/exit mutexes.
|
||||
*/
|
||||
call _vm_page_zero_idle /* internal locking */
|
||||
testl %eax, %eax
|
||||
jnz idle_loop
|
||||
@ -178,9 +179,15 @@ idle_loop:
|
||||
cli
|
||||
jmp idle_loop
|
||||
|
||||
/*
|
||||
* Note that interrupts must be enabled while obtaining the MP lock
|
||||
* in order to be able to take IPI's while blocked.
|
||||
*/
|
||||
3:
|
||||
movl $LOPRIO_LEVEL, lapic_tpr /* arbitrate for INTs */
|
||||
sti
|
||||
call _get_mplock
|
||||
cli
|
||||
call _procrunnable
|
||||
testl %eax,%eax
|
||||
CROSSJUMP(jnz, sw1a, jz)
|
||||
@ -355,8 +362,8 @@ sw1a:
|
||||
CROSSJUMP(je, _idle, jne) /* if no proc, idle */
|
||||
movl %eax,%ecx
|
||||
|
||||
movl $0,%eax
|
||||
movl %eax,_want_resched
|
||||
xorl %eax,%eax
|
||||
andl $~WANT_RESCHED,_astpending
|
||||
|
||||
#ifdef DIAGNOSTIC
|
||||
cmpl %eax,P_WCHAN(%ecx)
|
||||
|
@ -41,23 +41,11 @@
|
||||
#include <machine/psl.h>
|
||||
#include <machine/trap.h>
|
||||
#ifdef SMP
|
||||
#include <machine/smptests.h> /** CPL_AND_CML, REAL_ */
|
||||
#include <machine/smptests.h> /** various SMP options */
|
||||
#endif
|
||||
|
||||
#include "assym.s"
|
||||
|
||||
#ifndef SMP
|
||||
#define ECPL_LOCK /* make these nops */
|
||||
#define ECPL_UNLOCK
|
||||
#define ICPL_LOCK
|
||||
#define ICPL_UNLOCK
|
||||
#define FAST_ICPL_UNLOCK
|
||||
#define AICPL_LOCK
|
||||
#define AICPL_UNLOCK
|
||||
#define AVCPL_LOCK
|
||||
#define AVCPL_UNLOCK
|
||||
#endif /* SMP */
|
||||
|
||||
#ifdef SMP
|
||||
#define MOVL_KPSEL_EAX movl $KPSEL,%eax
|
||||
#else
|
||||
@ -71,16 +59,45 @@
|
||||
/* Trap handling */
|
||||
/*****************************************************************************/
|
||||
/*
|
||||
* Trap and fault vector routines
|
||||
* Trap and fault vector routines.
|
||||
*
|
||||
* Most traps are 'trap gates', SDT_SYS386TGT. A trap gate pushes state on
|
||||
* the stack that mostly looks like an interrupt, but does not disable
|
||||
* interrupts. A few of the traps we are use are interrupt gates,
|
||||
* SDT_SYS386IGT, which are nearly the same thing except interrupts are
|
||||
* disabled on entry.
|
||||
*
|
||||
* The cpu will push a certain amount of state onto the kernel stack for
|
||||
* the current process. The amount of state depends on the type of trap
|
||||
* and whether the trap crossed rings or not. See i386/include/frame.h.
|
||||
* At the very least the current EFLAGS (status register, which includes
|
||||
* the interrupt disable state prior to the trap), the code segment register,
|
||||
* and the return instruction pointer are pushed by the cpu. The cpu
|
||||
* will also push an 'error' code for certain traps. We push a dummy
|
||||
* error code for those traps where the cpu doesn't in order to maintain
|
||||
* a consistent frame. We also push a contrived 'trap number'.
|
||||
*
|
||||
* The cpu does not push the general registers, we must do that, and we
|
||||
* must restore them prior to calling 'iret'. The cpu adjusts the %cs and
|
||||
* %ss segment registers, but does not mess with %ds, %es, or %fs. Thus we
|
||||
* must load them with appropriate values for supervisor mode operation.
|
||||
*
|
||||
* On entry to a trap or interrupt WE DO NOT OWN THE MP LOCK. This means
|
||||
* that we must be careful in regards to accessing global variables. We
|
||||
* save (push) the current cpl (our software interrupt disable mask), call
|
||||
* the trap function, then call _doreti to restore the cpl and deal with
|
||||
* ASTs (software interrupts). _doreti will determine if the restoration
|
||||
* of the cpl unmasked any pending interrupts and will issue those interrupts
|
||||
* synchronously prior to doing the iret.
|
||||
*
|
||||
* At the moment we must own the MP lock to do any cpl manipulation, which
|
||||
* means we must own it prior to calling _doreti. The syscall case attempts
|
||||
* to avoid this by handling a reduced set of cases itself and iret'ing.
|
||||
*/
|
||||
#define IDTVEC(name) ALIGN_TEXT; .globl __CONCAT(_X,name); \
|
||||
.type __CONCAT(_X,name),@function; __CONCAT(_X,name):
|
||||
#define TRAP(a) pushl $(a) ; jmp _alltraps
|
||||
|
||||
/*
|
||||
* XXX - debugger traps are now interrupt gates so at least bdb doesn't lose
|
||||
* control. The sti's give the standard losing behaviour for ddb and kgdb.
|
||||
*/
|
||||
#ifdef BDE_DEBUGGER
|
||||
#define BDBTRAP(name) \
|
||||
ss ; \
|
||||
@ -160,16 +177,9 @@ IDTVEC(fpu)
|
||||
|
||||
#ifdef SMP
|
||||
MPLOCKED incl _cnt+V_TRAP
|
||||
FPU_LOCK
|
||||
ECPL_LOCK
|
||||
#ifdef CPL_AND_CML
|
||||
movl _cml,%eax
|
||||
pushl %eax /* save original cml */
|
||||
#else
|
||||
MP_LOCK
|
||||
movl _cpl,%eax
|
||||
pushl %eax /* save original cpl */
|
||||
#endif /* CPL_AND_CML */
|
||||
ECPL_UNLOCK
|
||||
pushl $0 /* dummy unit to finish intr frame */
|
||||
#else /* SMP */
|
||||
movl _cpl,%eax
|
||||
@ -190,6 +200,16 @@ IDTVEC(fpu)
|
||||
IDTVEC(align)
|
||||
TRAP(T_ALIGNFLT)
|
||||
|
||||
/*
|
||||
* _alltraps entry point. Interrupts are enabled if this was a trap
|
||||
* gate (TGT), else disabled if this was an interrupt gate (IGT).
|
||||
* Note that int0x80_syscall is a trap gate. Only page faults
|
||||
* use an interrupt gate.
|
||||
*
|
||||
* Note that all calls to MP_LOCK must occur with interrupts enabled
|
||||
* in order to be able to take IPI's while waiting for the lock.
|
||||
*/
|
||||
|
||||
SUPERALIGN_TEXT
|
||||
.globl _alltraps
|
||||
.type _alltraps,@function
|
||||
@ -208,14 +228,8 @@ alltraps_with_regs_pushed:
|
||||
calltrap:
|
||||
FAKE_MCOUNT(_btrap) /* init "from" _btrap -> calltrap */
|
||||
MPLOCKED incl _cnt+V_TRAP
|
||||
ALIGN_LOCK
|
||||
ECPL_LOCK
|
||||
#ifdef CPL_AND_CML
|
||||
movl _cml,%ebx /* keep orig. cml here during trap() */
|
||||
#else
|
||||
MP_LOCK
|
||||
movl _cpl,%ebx /* keep orig. cpl here during trap() */
|
||||
#endif
|
||||
ECPL_UNLOCK
|
||||
call _trap
|
||||
|
||||
/*
|
||||
@ -229,17 +243,19 @@ calltrap:
|
||||
jmp _doreti
|
||||
|
||||
/*
|
||||
* Call gate entry for syscall.
|
||||
* SYSCALL CALL GATE (old entry point for a.out binaries)
|
||||
*
|
||||
* The intersegment call has been set up to specify one dummy parameter.
|
||||
*
|
||||
* This leaves a place to put eflags so that the call frame can be
|
||||
* converted to a trap frame. Note that the eflags is (semi-)bogusly
|
||||
* pushed into (what will be) tf_err and then copied later into the
|
||||
* final spot. It has to be done this way because esp can't be just
|
||||
* temporarily altered for the pushfl - an interrupt might come in
|
||||
* and clobber the saved cs/eip.
|
||||
*/
|
||||
/*
|
||||
* THis first callgate is used for the old a.out binaries
|
||||
*
|
||||
* We do not obtain the MP lock, but the call to syscall2 might. If it
|
||||
* does it will release the lock prior to returning.
|
||||
*/
|
||||
SUPERALIGN_TEXT
|
||||
IDTVEC(syscall)
|
||||
@ -259,20 +275,28 @@ IDTVEC(syscall)
|
||||
movl $7,TF_ERR(%esp) /* sizeof "lcall 7,0" */
|
||||
FAKE_MCOUNT(13*4(%esp))
|
||||
MPLOCKED incl _cnt+V_SYSCALL
|
||||
SYSCALL_LOCK
|
||||
call _syscall
|
||||
|
||||
/*
|
||||
* Return via _doreti to handle ASTs.
|
||||
*/
|
||||
pushl $0 /* cpl to restore */
|
||||
subl $4,%esp /* dummy unit to finish intr frame */
|
||||
movb $1,_intr_nesting_level
|
||||
call _syscall2
|
||||
MEXITCOUNT
|
||||
cli /* atomic astpending access */
|
||||
cmpl $0,_astpending
|
||||
je doreti_syscall_ret
|
||||
#ifdef SMP
|
||||
MP_LOCK
|
||||
#endif
|
||||
pushl $0 /* cpl to restore */
|
||||
subl $4,%esp /* dummy unit for interrupt frame */
|
||||
movb $1,_intr_nesting_level
|
||||
jmp _doreti
|
||||
|
||||
/*
|
||||
* Call gate entry for FreeBSD ELF and Linux/NetBSD syscall (int 0x80)
|
||||
*
|
||||
* Even though the name says 'int0x80', this is actually a TGT (trap gate)
|
||||
* rather then an IGT (interrupt gate). Thus interrupts are enabled on
|
||||
* entry just as they are for a normal syscall.
|
||||
*
|
||||
* We do not obtain the MP lock, but the call to syscall2 might. If it
|
||||
* does it will release the lock prior to returning.
|
||||
*/
|
||||
SUPERALIGN_TEXT
|
||||
IDTVEC(int0x80_syscall)
|
||||
@ -289,16 +313,17 @@ IDTVEC(int0x80_syscall)
|
||||
movl $2,TF_ERR(%esp) /* sizeof "int 0x80" */
|
||||
FAKE_MCOUNT(13*4(%esp))
|
||||
MPLOCKED incl _cnt+V_SYSCALL
|
||||
ALTSYSCALL_LOCK
|
||||
call _syscall
|
||||
|
||||
/*
|
||||
* Return via _doreti to handle ASTs.
|
||||
*/
|
||||
pushl $0 /* cpl to restore */
|
||||
subl $4,%esp /* dummy unit to finish intr frame */
|
||||
movb $1,_intr_nesting_level
|
||||
call _syscall2
|
||||
MEXITCOUNT
|
||||
cli /* atomic astpending access */
|
||||
cmpl $0,_astpending
|
||||
je doreti_syscall_ret
|
||||
#ifdef SMP
|
||||
MP_LOCK
|
||||
#endif
|
||||
pushl $0 /* cpl to restore */
|
||||
subl $4,%esp /* dummy unit for interrupt frame */
|
||||
movb $1,_intr_nesting_level
|
||||
jmp _doreti
|
||||
|
||||
ENTRY(fork_trampoline)
|
||||
|
@ -41,23 +41,11 @@
|
||||
#include <machine/psl.h>
|
||||
#include <machine/trap.h>
|
||||
#ifdef SMP
|
||||
#include <machine/smptests.h> /** CPL_AND_CML, REAL_ */
|
||||
#include <machine/smptests.h> /** various SMP options */
|
||||
#endif
|
||||
|
||||
#include "assym.s"
|
||||
|
||||
#ifndef SMP
|
||||
#define ECPL_LOCK /* make these nops */
|
||||
#define ECPL_UNLOCK
|
||||
#define ICPL_LOCK
|
||||
#define ICPL_UNLOCK
|
||||
#define FAST_ICPL_UNLOCK
|
||||
#define AICPL_LOCK
|
||||
#define AICPL_UNLOCK
|
||||
#define AVCPL_LOCK
|
||||
#define AVCPL_UNLOCK
|
||||
#endif /* SMP */
|
||||
|
||||
#ifdef SMP
|
||||
#define MOVL_KPSEL_EAX movl $KPSEL,%eax
|
||||
#else
|
||||
@ -71,16 +59,45 @@
|
||||
/* Trap handling */
|
||||
/*****************************************************************************/
|
||||
/*
|
||||
* Trap and fault vector routines
|
||||
* Trap and fault vector routines.
|
||||
*
|
||||
* Most traps are 'trap gates', SDT_SYS386TGT. A trap gate pushes state on
|
||||
* the stack that mostly looks like an interrupt, but does not disable
|
||||
* interrupts. A few of the traps we are use are interrupt gates,
|
||||
* SDT_SYS386IGT, which are nearly the same thing except interrupts are
|
||||
* disabled on entry.
|
||||
*
|
||||
* The cpu will push a certain amount of state onto the kernel stack for
|
||||
* the current process. The amount of state depends on the type of trap
|
||||
* and whether the trap crossed rings or not. See i386/include/frame.h.
|
||||
* At the very least the current EFLAGS (status register, which includes
|
||||
* the interrupt disable state prior to the trap), the code segment register,
|
||||
* and the return instruction pointer are pushed by the cpu. The cpu
|
||||
* will also push an 'error' code for certain traps. We push a dummy
|
||||
* error code for those traps where the cpu doesn't in order to maintain
|
||||
* a consistent frame. We also push a contrived 'trap number'.
|
||||
*
|
||||
* The cpu does not push the general registers, we must do that, and we
|
||||
* must restore them prior to calling 'iret'. The cpu adjusts the %cs and
|
||||
* %ss segment registers, but does not mess with %ds, %es, or %fs. Thus we
|
||||
* must load them with appropriate values for supervisor mode operation.
|
||||
*
|
||||
* On entry to a trap or interrupt WE DO NOT OWN THE MP LOCK. This means
|
||||
* that we must be careful in regards to accessing global variables. We
|
||||
* save (push) the current cpl (our software interrupt disable mask), call
|
||||
* the trap function, then call _doreti to restore the cpl and deal with
|
||||
* ASTs (software interrupts). _doreti will determine if the restoration
|
||||
* of the cpl unmasked any pending interrupts and will issue those interrupts
|
||||
* synchronously prior to doing the iret.
|
||||
*
|
||||
* At the moment we must own the MP lock to do any cpl manipulation, which
|
||||
* means we must own it prior to calling _doreti. The syscall case attempts
|
||||
* to avoid this by handling a reduced set of cases itself and iret'ing.
|
||||
*/
|
||||
#define IDTVEC(name) ALIGN_TEXT; .globl __CONCAT(_X,name); \
|
||||
.type __CONCAT(_X,name),@function; __CONCAT(_X,name):
|
||||
#define TRAP(a) pushl $(a) ; jmp _alltraps
|
||||
|
||||
/*
|
||||
* XXX - debugger traps are now interrupt gates so at least bdb doesn't lose
|
||||
* control. The sti's give the standard losing behaviour for ddb and kgdb.
|
||||
*/
|
||||
#ifdef BDE_DEBUGGER
|
||||
#define BDBTRAP(name) \
|
||||
ss ; \
|
||||
@ -160,16 +177,9 @@ IDTVEC(fpu)
|
||||
|
||||
#ifdef SMP
|
||||
MPLOCKED incl _cnt+V_TRAP
|
||||
FPU_LOCK
|
||||
ECPL_LOCK
|
||||
#ifdef CPL_AND_CML
|
||||
movl _cml,%eax
|
||||
pushl %eax /* save original cml */
|
||||
#else
|
||||
MP_LOCK
|
||||
movl _cpl,%eax
|
||||
pushl %eax /* save original cpl */
|
||||
#endif /* CPL_AND_CML */
|
||||
ECPL_UNLOCK
|
||||
pushl $0 /* dummy unit to finish intr frame */
|
||||
#else /* SMP */
|
||||
movl _cpl,%eax
|
||||
@ -190,6 +200,16 @@ IDTVEC(fpu)
|
||||
IDTVEC(align)
|
||||
TRAP(T_ALIGNFLT)
|
||||
|
||||
/*
|
||||
* _alltraps entry point. Interrupts are enabled if this was a trap
|
||||
* gate (TGT), else disabled if this was an interrupt gate (IGT).
|
||||
* Note that int0x80_syscall is a trap gate. Only page faults
|
||||
* use an interrupt gate.
|
||||
*
|
||||
* Note that all calls to MP_LOCK must occur with interrupts enabled
|
||||
* in order to be able to take IPI's while waiting for the lock.
|
||||
*/
|
||||
|
||||
SUPERALIGN_TEXT
|
||||
.globl _alltraps
|
||||
.type _alltraps,@function
|
||||
@ -208,14 +228,8 @@ alltraps_with_regs_pushed:
|
||||
calltrap:
|
||||
FAKE_MCOUNT(_btrap) /* init "from" _btrap -> calltrap */
|
||||
MPLOCKED incl _cnt+V_TRAP
|
||||
ALIGN_LOCK
|
||||
ECPL_LOCK
|
||||
#ifdef CPL_AND_CML
|
||||
movl _cml,%ebx /* keep orig. cml here during trap() */
|
||||
#else
|
||||
MP_LOCK
|
||||
movl _cpl,%ebx /* keep orig. cpl here during trap() */
|
||||
#endif
|
||||
ECPL_UNLOCK
|
||||
call _trap
|
||||
|
||||
/*
|
||||
@ -229,17 +243,19 @@ calltrap:
|
||||
jmp _doreti
|
||||
|
||||
/*
|
||||
* Call gate entry for syscall.
|
||||
* SYSCALL CALL GATE (old entry point for a.out binaries)
|
||||
*
|
||||
* The intersegment call has been set up to specify one dummy parameter.
|
||||
*
|
||||
* This leaves a place to put eflags so that the call frame can be
|
||||
* converted to a trap frame. Note that the eflags is (semi-)bogusly
|
||||
* pushed into (what will be) tf_err and then copied later into the
|
||||
* final spot. It has to be done this way because esp can't be just
|
||||
* temporarily altered for the pushfl - an interrupt might come in
|
||||
* and clobber the saved cs/eip.
|
||||
*/
|
||||
/*
|
||||
* THis first callgate is used for the old a.out binaries
|
||||
*
|
||||
* We do not obtain the MP lock, but the call to syscall2 might. If it
|
||||
* does it will release the lock prior to returning.
|
||||
*/
|
||||
SUPERALIGN_TEXT
|
||||
IDTVEC(syscall)
|
||||
@ -259,20 +275,28 @@ IDTVEC(syscall)
|
||||
movl $7,TF_ERR(%esp) /* sizeof "lcall 7,0" */
|
||||
FAKE_MCOUNT(13*4(%esp))
|
||||
MPLOCKED incl _cnt+V_SYSCALL
|
||||
SYSCALL_LOCK
|
||||
call _syscall
|
||||
|
||||
/*
|
||||
* Return via _doreti to handle ASTs.
|
||||
*/
|
||||
pushl $0 /* cpl to restore */
|
||||
subl $4,%esp /* dummy unit to finish intr frame */
|
||||
movb $1,_intr_nesting_level
|
||||
call _syscall2
|
||||
MEXITCOUNT
|
||||
cli /* atomic astpending access */
|
||||
cmpl $0,_astpending
|
||||
je doreti_syscall_ret
|
||||
#ifdef SMP
|
||||
MP_LOCK
|
||||
#endif
|
||||
pushl $0 /* cpl to restore */
|
||||
subl $4,%esp /* dummy unit for interrupt frame */
|
||||
movb $1,_intr_nesting_level
|
||||
jmp _doreti
|
||||
|
||||
/*
|
||||
* Call gate entry for FreeBSD ELF and Linux/NetBSD syscall (int 0x80)
|
||||
*
|
||||
* Even though the name says 'int0x80', this is actually a TGT (trap gate)
|
||||
* rather then an IGT (interrupt gate). Thus interrupts are enabled on
|
||||
* entry just as they are for a normal syscall.
|
||||
*
|
||||
* We do not obtain the MP lock, but the call to syscall2 might. If it
|
||||
* does it will release the lock prior to returning.
|
||||
*/
|
||||
SUPERALIGN_TEXT
|
||||
IDTVEC(int0x80_syscall)
|
||||
@ -289,16 +313,17 @@ IDTVEC(int0x80_syscall)
|
||||
movl $2,TF_ERR(%esp) /* sizeof "int 0x80" */
|
||||
FAKE_MCOUNT(13*4(%esp))
|
||||
MPLOCKED incl _cnt+V_SYSCALL
|
||||
ALTSYSCALL_LOCK
|
||||
call _syscall
|
||||
|
||||
/*
|
||||
* Return via _doreti to handle ASTs.
|
||||
*/
|
||||
pushl $0 /* cpl to restore */
|
||||
subl $4,%esp /* dummy unit to finish intr frame */
|
||||
movb $1,_intr_nesting_level
|
||||
call _syscall2
|
||||
MEXITCOUNT
|
||||
cli /* atomic astpending access */
|
||||
cmpl $0,_astpending
|
||||
je doreti_syscall_ret
|
||||
#ifdef SMP
|
||||
MP_LOCK
|
||||
#endif
|
||||
pushl $0 /* cpl to restore */
|
||||
subl $4,%esp /* dummy unit for interrupt frame */
|
||||
movb $1,_intr_nesting_level
|
||||
jmp _doreti
|
||||
|
||||
ENTRY(fork_trampoline)
|
||||
|
@ -176,6 +176,7 @@ ASSYM(GD_SWITCHTIME, offsetof(struct globaldata, gd_switchtime));
|
||||
ASSYM(GD_SWITCHTICKS, offsetof(struct globaldata, gd_switchticks));
|
||||
ASSYM(GD_COMMON_TSSD, offsetof(struct globaldata, gd_common_tssd));
|
||||
ASSYM(GD_TSS_GDT, offsetof(struct globaldata, gd_tss_gdt));
|
||||
ASSYM(GD_ASTPENDING, offsetof(struct globaldata, gd_astpending));
|
||||
|
||||
#ifdef USER_LDT
|
||||
ASSYM(GD_CURRENTLDT, offsetof(struct globaldata, gd_currentldt));
|
||||
|
@ -1770,8 +1770,10 @@ init_locks(void)
|
||||
*/
|
||||
mp_lock = 0x00000001;
|
||||
|
||||
#if 0
|
||||
/* ISR uses its own "giant lock" */
|
||||
isr_lock = FREE_LOCK;
|
||||
#endif
|
||||
|
||||
#if defined(APIC_INTR_DIAGNOSTIC) && defined(APIC_INTR_DIAGNOSTIC_IRQ)
|
||||
s_lock_init((struct simplelock*)&apic_itrace_debuglock);
|
||||
|
@ -1770,8 +1770,10 @@ init_locks(void)
|
||||
*/
|
||||
mp_lock = 0x00000001;
|
||||
|
||||
#if 0
|
||||
/* ISR uses its own "giant lock" */
|
||||
isr_lock = FREE_LOCK;
|
||||
#endif
|
||||
|
||||
#if defined(APIC_INTR_DIAGNOSTIC) && defined(APIC_INTR_DIAGNOSTIC_IRQ)
|
||||
s_lock_init((struct simplelock*)&apic_itrace_debuglock);
|
||||
|
@ -848,7 +848,11 @@ ENTRY(i586_copyout)
|
||||
jmp done_copyout
|
||||
#endif /* I586_CPU && NNPX > 0 */
|
||||
|
||||
/* copyin(from_user, to_kernel, len) */
|
||||
/*
|
||||
* copyin(from_user, to_kernel, len)
|
||||
*
|
||||
* MPSAFE
|
||||
*/
|
||||
ENTRY(copyin)
|
||||
MEXITCOUNT
|
||||
jmp *_copyin_vector
|
||||
@ -1130,6 +1134,8 @@ fastmove_tail_fault:
|
||||
|
||||
/*
|
||||
* fu{byte,sword,word} : fetch a byte (sword, word) from user memory
|
||||
*
|
||||
* MP SAFE
|
||||
*/
|
||||
ENTRY(fuword)
|
||||
movl _curpcb,%ecx
|
||||
@ -1154,6 +1160,9 @@ ENTRY(fuswintr)
|
||||
movl $-1,%eax
|
||||
ret
|
||||
|
||||
/*
|
||||
* MP SAFE
|
||||
*/
|
||||
ENTRY(fusword)
|
||||
movl _curpcb,%ecx
|
||||
movl $fusufault,PCB_ONFAULT(%ecx)
|
||||
@ -1166,6 +1175,9 @@ ENTRY(fusword)
|
||||
movl $0,PCB_ONFAULT(%ecx)
|
||||
ret
|
||||
|
||||
/*
|
||||
* MP SAFE
|
||||
*/
|
||||
ENTRY(fubyte)
|
||||
movl _curpcb,%ecx
|
||||
movl $fusufault,PCB_ONFAULT(%ecx)
|
||||
|
@ -848,7 +848,11 @@ ENTRY(i586_copyout)
|
||||
jmp done_copyout
|
||||
#endif /* I586_CPU && NNPX > 0 */
|
||||
|
||||
/* copyin(from_user, to_kernel, len) */
|
||||
/*
|
||||
* copyin(from_user, to_kernel, len)
|
||||
*
|
||||
* MPSAFE
|
||||
*/
|
||||
ENTRY(copyin)
|
||||
MEXITCOUNT
|
||||
jmp *_copyin_vector
|
||||
@ -1130,6 +1134,8 @@ fastmove_tail_fault:
|
||||
|
||||
/*
|
||||
* fu{byte,sword,word} : fetch a byte (sword, word) from user memory
|
||||
*
|
||||
* MP SAFE
|
||||
*/
|
||||
ENTRY(fuword)
|
||||
movl _curpcb,%ecx
|
||||
@ -1154,6 +1160,9 @@ ENTRY(fuswintr)
|
||||
movl $-1,%eax
|
||||
ret
|
||||
|
||||
/*
|
||||
* MP SAFE
|
||||
*/
|
||||
ENTRY(fusword)
|
||||
movl _curpcb,%ecx
|
||||
movl $fusufault,PCB_ONFAULT(%ecx)
|
||||
@ -1166,6 +1175,9 @@ ENTRY(fusword)
|
||||
movl $0,PCB_ONFAULT(%ecx)
|
||||
ret
|
||||
|
||||
/*
|
||||
* MP SAFE
|
||||
*/
|
||||
ENTRY(fubyte)
|
||||
movl _curpcb,%ecx
|
||||
movl $fusufault,PCB_ONFAULT(%ecx)
|
||||
|
@ -65,8 +65,6 @@ _hlt_vector: .long _default_halt /* pointer to halt routine */
|
||||
|
||||
.globl _panic
|
||||
|
||||
.globl _want_resched
|
||||
_want_resched: .long 0 /* we need to re-run the scheduler */
|
||||
#if defined(SWTCH_OPTIM_STATS)
|
||||
.globl _swtch_optim_stats, _tlb_flush_count
|
||||
_swtch_optim_stats: .long 0 /* number of _swtch_optims */
|
||||
@ -129,6 +127,9 @@ _idle:
|
||||
/*
|
||||
* XXX callers of cpu_switch() do a bogus splclock(). Locking should
|
||||
* be left to cpu_switch().
|
||||
*
|
||||
* NOTE: spl*() may only be called while we hold the MP lock (which
|
||||
* we do).
|
||||
*/
|
||||
call _spl0
|
||||
|
||||
@ -159,14 +160,14 @@ idle_loop:
|
||||
testl %eax,%eax
|
||||
jnz 3f
|
||||
|
||||
/*
|
||||
* Handle page-zeroing in the idle loop. Called with interrupts
|
||||
* disabled and the MP lock released. Inside vm_page_zero_idle
|
||||
* we enable interrupts and grab the mplock as required.
|
||||
*/
|
||||
cmpl $0,_do_page_zero_idle
|
||||
je 2f
|
||||
|
||||
/* XXX appears to cause panics */
|
||||
/*
|
||||
* Inside zero_idle we enable interrupts and grab the mplock
|
||||
* as needed. It needs to be careful about entry/exit mutexes.
|
||||
*/
|
||||
call _vm_page_zero_idle /* internal locking */
|
||||
testl %eax, %eax
|
||||
jnz idle_loop
|
||||
@ -178,9 +179,15 @@ idle_loop:
|
||||
cli
|
||||
jmp idle_loop
|
||||
|
||||
/*
|
||||
* Note that interrupts must be enabled while obtaining the MP lock
|
||||
* in order to be able to take IPI's while blocked.
|
||||
*/
|
||||
3:
|
||||
movl $LOPRIO_LEVEL, lapic_tpr /* arbitrate for INTs */
|
||||
sti
|
||||
call _get_mplock
|
||||
cli
|
||||
call _procrunnable
|
||||
testl %eax,%eax
|
||||
CROSSJUMP(jnz, sw1a, jz)
|
||||
@ -355,8 +362,8 @@ sw1a:
|
||||
CROSSJUMP(je, _idle, jne) /* if no proc, idle */
|
||||
movl %eax,%ecx
|
||||
|
||||
movl $0,%eax
|
||||
movl %eax,_want_resched
|
||||
xorl %eax,%eax
|
||||
andl $~WANT_RESCHED,_astpending
|
||||
|
||||
#ifdef DIAGNOSTIC
|
||||
cmpl %eax,P_WCHAN(%ecx)
|
||||
|
@ -53,6 +53,7 @@
|
||||
#include <sys/user.h>
|
||||
|
||||
#include <machine/cpu.h>
|
||||
#include <machine/ipl.h>
|
||||
#include <machine/pcb_ext.h> /* pcb.h included by sys/user.h */
|
||||
#include <machine/sysarch.h>
|
||||
#ifdef SMP
|
||||
|
@ -99,7 +99,7 @@ int (*pmath_emulate) __P((struct trapframe *));
|
||||
|
||||
extern void trap __P((struct trapframe frame));
|
||||
extern int trapwrite __P((unsigned addr));
|
||||
extern void syscall __P((struct trapframe frame));
|
||||
extern void syscall2 __P((struct trapframe frame));
|
||||
|
||||
static int trap_pfault __P((struct trapframe *, int, vm_offset_t));
|
||||
static void trap_fatal __P((struct trapframe *, vm_offset_t));
|
||||
@ -140,38 +140,32 @@ static char *trap_msg[] = {
|
||||
"machine check trap", /* 28 T_MCHK */
|
||||
};
|
||||
|
||||
static __inline void userret __P((struct proc *p, struct trapframe *frame,
|
||||
u_quad_t oticks));
|
||||
static __inline int userret __P((struct proc *p, struct trapframe *frame,
|
||||
u_quad_t oticks, int have_mplock));
|
||||
|
||||
#if defined(I586_CPU) && !defined(NO_F00F_HACK)
|
||||
extern int has_f00f_bug;
|
||||
#endif
|
||||
|
||||
static __inline void
|
||||
userret(p, frame, oticks)
|
||||
static __inline int
|
||||
userret(p, frame, oticks, have_mplock)
|
||||
struct proc *p;
|
||||
struct trapframe *frame;
|
||||
u_quad_t oticks;
|
||||
int have_mplock;
|
||||
{
|
||||
int sig, s;
|
||||
|
||||
while ((sig = CURSIG(p)) != 0)
|
||||
while ((sig = CURSIG(p)) != 0) {
|
||||
if (have_mplock == 0) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
}
|
||||
postsig(sig);
|
||||
|
||||
#if 0
|
||||
if (!want_resched &&
|
||||
(p->p_priority <= p->p_usrpri) &&
|
||||
(p->p_rtprio.type == RTP_PRIO_NORMAL)) {
|
||||
int newpriority;
|
||||
p->p_estcpu += 1;
|
||||
newpriority = PUSER + p->p_estcpu / 4 + 2 * p->p_nice;
|
||||
newpriority = min(newpriority, MAXPRI);
|
||||
p->p_usrpri = newpriority;
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
p->p_priority = p->p_usrpri;
|
||||
if (want_resched) {
|
||||
if (resched_wanted()) {
|
||||
/*
|
||||
* Since we are curproc, clock will normally just change
|
||||
* our priority without moving us from one queue to another
|
||||
@ -180,6 +174,10 @@ userret(p, frame, oticks)
|
||||
* mi_switch()'ed, we might not be on the queue indicated by
|
||||
* our priority.
|
||||
*/
|
||||
if (have_mplock == 0) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
}
|
||||
s = splhigh();
|
||||
setrunqueue(p);
|
||||
p->p_stats->p_ru.ru_nivcsw++;
|
||||
@ -191,11 +189,16 @@ userret(p, frame, oticks)
|
||||
/*
|
||||
* Charge system time if profiling.
|
||||
*/
|
||||
if (p->p_flag & P_PROFIL)
|
||||
if (p->p_flag & P_PROFIL) {
|
||||
if (have_mplock == 0) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
}
|
||||
addupc_task(p, frame->tf_eip,
|
||||
(u_int)(p->p_sticks - oticks) * psratio);
|
||||
|
||||
}
|
||||
curpriority = p->p_priority;
|
||||
return(have_mplock);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -604,7 +607,7 @@ trap(frame)
|
||||
#endif
|
||||
|
||||
out:
|
||||
userret(p, &frame, sticks);
|
||||
userret(p, &frame, sticks, 1);
|
||||
}
|
||||
|
||||
#ifdef notyet
|
||||
@ -999,11 +1002,18 @@ int trapwrite(addr)
|
||||
}
|
||||
|
||||
/*
|
||||
* System call request from POSIX system call gate interface to kernel.
|
||||
* Like trap(), argument is call by reference.
|
||||
* syscall2 - MP aware system call request C handler
|
||||
*
|
||||
* A system call is essentially treated as a trap except that the
|
||||
* MP lock is not held on entry or return. We are responsible for
|
||||
* obtaining the MP lock if necessary and for handling ASTs
|
||||
* (e.g. a task switch) prior to return.
|
||||
*
|
||||
* In general, only simple access and manipulation of curproc and
|
||||
* the current stack is allowed without having to hold MP lock.
|
||||
*/
|
||||
void
|
||||
syscall(frame)
|
||||
syscall2(frame)
|
||||
struct trapframe frame;
|
||||
{
|
||||
caddr_t params;
|
||||
@ -1012,22 +1022,42 @@ syscall(frame)
|
||||
struct proc *p = curproc;
|
||||
u_quad_t sticks;
|
||||
int error;
|
||||
int narg;
|
||||
int args[8];
|
||||
int have_mplock = 0;
|
||||
u_int code;
|
||||
|
||||
#ifdef DIAGNOSTIC
|
||||
if (ISPL(frame.tf_cs) != SEL_UPL)
|
||||
if (ISPL(frame.tf_cs) != SEL_UPL) {
|
||||
get_mplock();
|
||||
panic("syscall");
|
||||
/* NOT REACHED */
|
||||
}
|
||||
#endif
|
||||
sticks = p->p_sticks;
|
||||
|
||||
/*
|
||||
* handle atomicy by looping since interrupts are enabled and the
|
||||
* MP lock is not held.
|
||||
*/
|
||||
sticks = ((volatile struct proc *)p)->p_sticks;
|
||||
while (sticks != ((volatile struct proc *)p)->p_sticks)
|
||||
sticks = ((volatile struct proc *)p)->p_sticks;
|
||||
|
||||
p->p_md.md_regs = &frame;
|
||||
params = (caddr_t)frame.tf_esp + sizeof(int);
|
||||
code = frame.tf_eax;
|
||||
|
||||
if (p->p_sysent->sv_prepsyscall) {
|
||||
/*
|
||||
* The prep code is not MP aware.
|
||||
*/
|
||||
get_mplock();
|
||||
(*p->p_sysent->sv_prepsyscall)(&frame, args, &code, ¶ms);
|
||||
rel_mplock();
|
||||
} else {
|
||||
/*
|
||||
* Need to check if this is a 32 bit or 64 bit syscall.
|
||||
* fuword is MP aware.
|
||||
*/
|
||||
if (code == SYS_syscall) {
|
||||
/*
|
||||
@ -1053,27 +1083,52 @@ syscall(frame)
|
||||
else
|
||||
callp = &p->p_sysent->sv_table[code];
|
||||
|
||||
if (params && (i = callp->sy_narg * sizeof(int)) &&
|
||||
narg = callp->sy_narg & SYF_ARGMASK;
|
||||
|
||||
/*
|
||||
* copyin is MP aware, but the tracing code is not
|
||||
*/
|
||||
if (params && (i = narg * sizeof(int)) &&
|
||||
(error = copyin(params, (caddr_t)args, (u_int)i))) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
#ifdef KTRACE
|
||||
if (KTRPOINT(p, KTR_SYSCALL))
|
||||
ktrsyscall(p->p_tracep, code, callp->sy_narg, args);
|
||||
ktrsyscall(p->p_tracep, code, narg, args);
|
||||
#endif
|
||||
goto bad;
|
||||
}
|
||||
|
||||
/*
|
||||
* Try to run the syscall without the MP lock if the syscall
|
||||
* is MP safe. We have to obtain the MP lock no matter what if
|
||||
* we are ktracing
|
||||
*/
|
||||
if ((callp->sy_narg & SYF_MPSAFE) == 0) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
}
|
||||
|
||||
#ifdef KTRACE
|
||||
if (KTRPOINT(p, KTR_SYSCALL))
|
||||
ktrsyscall(p->p_tracep, code, callp->sy_narg, args);
|
||||
if (KTRPOINT(p, KTR_SYSCALL)) {
|
||||
if (have_mplock == 0) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
}
|
||||
ktrsyscall(p->p_tracep, code, narg, args);
|
||||
}
|
||||
#endif
|
||||
p->p_retval[0] = 0;
|
||||
p->p_retval[1] = frame.tf_edx;
|
||||
|
||||
STOPEVENT(p, S_SCE, callp->sy_narg);
|
||||
STOPEVENT(p, S_SCE, narg); /* MP aware */
|
||||
|
||||
error = (*callp->sy_call)(p, args);
|
||||
|
||||
/*
|
||||
* MP SAFE (we may or may not have the MP lock at this point)
|
||||
*/
|
||||
switch (error) {
|
||||
|
||||
case 0:
|
||||
/*
|
||||
* Reinitialize proc pointer `p' as it may be different
|
||||
@ -1109,17 +1164,31 @@ syscall(frame)
|
||||
break;
|
||||
}
|
||||
|
||||
/*
|
||||
* Traced syscall. trapsignal() is not MP aware.
|
||||
*/
|
||||
if ((frame.tf_eflags & PSL_T) && !(frame.tf_eflags & PSL_VM)) {
|
||||
/* Traced syscall. */
|
||||
if (have_mplock == 0) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
}
|
||||
frame.tf_eflags &= ~PSL_T;
|
||||
trapsignal(p, SIGTRAP, 0);
|
||||
}
|
||||
|
||||
userret(p, &frame, sticks);
|
||||
/*
|
||||
* Handle reschedule and other end-of-syscall issues
|
||||
*/
|
||||
have_mplock = userret(p, &frame, sticks, have_mplock);
|
||||
|
||||
#ifdef KTRACE
|
||||
if (KTRPOINT(p, KTR_SYSRET))
|
||||
if (KTRPOINT(p, KTR_SYSRET)) {
|
||||
if (have_mplock == 0) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
}
|
||||
ktrsysret(p->p_tracep, code, error, p->p_retval[0]);
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
@ -1129,11 +1198,17 @@ syscall(frame)
|
||||
*/
|
||||
STOPEVENT(p, S_SCX, code);
|
||||
|
||||
/*
|
||||
* Release the MP lock if we had to get it
|
||||
*/
|
||||
if (have_mplock)
|
||||
rel_mplock();
|
||||
}
|
||||
|
||||
/*
|
||||
* Simplified back end of syscall(), used when returning from fork()
|
||||
* directly into user mode.
|
||||
* directly into user mode. MP lock is held on entry and should be
|
||||
* held on return.
|
||||
*/
|
||||
void
|
||||
fork_return(p, frame)
|
||||
@ -1144,7 +1219,7 @@ fork_return(p, frame)
|
||||
frame.tf_eflags &= ~PSL_C; /* success */
|
||||
frame.tf_edx = 1;
|
||||
|
||||
userret(p, &frame, 0);
|
||||
userret(p, &frame, 0, 1);
|
||||
#ifdef KTRACE
|
||||
if (KTRPOINT(p, KTR_SYSRET))
|
||||
ktrsysret(p->p_tracep, SYS_fork, 0, 0);
|
||||
|
@ -578,13 +578,7 @@ vm_page_zero_idle()
|
||||
TAILQ_REMOVE(&vm_page_queues[m->queue].pl, m, pageq);
|
||||
m->queue = PQ_NONE;
|
||||
splx(s);
|
||||
#if 0
|
||||
rel_mplock();
|
||||
#endif
|
||||
pmap_zero_page(VM_PAGE_TO_PHYS(m));
|
||||
#if 0
|
||||
get_mplock();
|
||||
#endif
|
||||
(void)splvm();
|
||||
vm_page_flag_set(m, PG_ZERO);
|
||||
m->queue = PQ_FREE + m->pc;
|
||||
@ -606,6 +600,12 @@ vm_page_zero_idle()
|
||||
#ifdef SMP
|
||||
}
|
||||
#endif
|
||||
/*
|
||||
* We have to enable interrupts for a moment if the try_mplock fails
|
||||
* in order to potentially take an IPI. XXX this should be in
|
||||
* swtch.s
|
||||
*/
|
||||
__asm __volatile("sti; nop; cli" : : : "memory");
|
||||
return (0);
|
||||
}
|
||||
|
||||
|
@ -82,10 +82,13 @@
|
||||
/*
|
||||
* Preempt the current process if in interrupt from user mode,
|
||||
* or after the current trap/syscall if in system mode.
|
||||
*
|
||||
* XXX: if astpending is later changed to an |= here due to more flags being
|
||||
* added, we will have an atomicy problem. The type of atomicy we need is
|
||||
* a non-locked orl.
|
||||
*/
|
||||
#define need_resched() do { want_resched = 1; aston(); } while (0)
|
||||
|
||||
#define resched_wanted() want_resched
|
||||
#define need_resched() do { astpending = AST_RESCHED|AST_PENDING; } while (0)
|
||||
#define resched_wanted() (astpending & AST_RESCHED)
|
||||
|
||||
/*
|
||||
* Arrange to handle pending profiling ticks before returning to user mode.
|
||||
@ -100,10 +103,15 @@
|
||||
/*
|
||||
* Notify the current process (p) that it has a signal pending,
|
||||
* process as soon as possible.
|
||||
*
|
||||
* XXX: aston() really needs to be an atomic (not locked, but an orl),
|
||||
* in case need_resched() is set by an interrupt. But with astpending a
|
||||
* per-cpu variable this is not trivial to do efficiently. For now we blow
|
||||
* it off (asynchronous need_resched() conflicts are not critical).
|
||||
*/
|
||||
#define signotify(p) aston()
|
||||
|
||||
#define aston() do { astpending = 1; } while (0)
|
||||
#define aston() do { astpending |= AST_PENDING; } while (0)
|
||||
#define astoff()
|
||||
|
||||
/*
|
||||
@ -126,11 +134,9 @@
|
||||
}
|
||||
|
||||
#ifdef _KERNEL
|
||||
extern int astpending;
|
||||
extern char btext[];
|
||||
extern char etext[];
|
||||
extern u_char intr_nesting_level;
|
||||
extern int want_resched; /* resched was called */
|
||||
|
||||
void fork_trampoline __P((void));
|
||||
void fork_return __P((struct proc *, struct trapframe));
|
||||
|
@ -1770,8 +1770,10 @@ init_locks(void)
|
||||
*/
|
||||
mp_lock = 0x00000001;
|
||||
|
||||
#if 0
|
||||
/* ISR uses its own "giant lock" */
|
||||
isr_lock = FREE_LOCK;
|
||||
#endif
|
||||
|
||||
#if defined(APIC_INTR_DIAGNOSTIC) && defined(APIC_INTR_DIAGNOSTIC_IRQ)
|
||||
s_lock_init((struct simplelock*)&apic_itrace_debuglock);
|
||||
|
@ -66,6 +66,7 @@ struct globaldata {
|
||||
caddr_t gd_prv_CADDR3;
|
||||
unsigned *gd_prv_PADDR1;
|
||||
#endif
|
||||
u_int gd_astpending;
|
||||
};
|
||||
|
||||
#ifdef SMP
|
||||
|
@ -9,28 +9,17 @@
|
||||
|
||||
#include "i386/isa/intr_machdep.h"
|
||||
|
||||
|
||||
#ifdef FAST_SIMPLELOCK
|
||||
/*
|
||||
* Interrupts must be enabled while waiting for the MP lock.
|
||||
*/
|
||||
|
||||
#define GET_FAST_INTR_LOCK \
|
||||
pushl $_fast_intr_lock ; /* address of lock */ \
|
||||
call _s_lock ; /* MP-safe */ \
|
||||
addl $4,%esp
|
||||
|
||||
#define REL_FAST_INTR_LOCK \
|
||||
movl $0, _fast_intr_lock
|
||||
|
||||
#else /* FAST_SIMPLELOCK */
|
||||
|
||||
#define GET_FAST_INTR_LOCK \
|
||||
call _get_isrlock
|
||||
sti; call _get_mplock; cli
|
||||
|
||||
#define REL_FAST_INTR_LOCK \
|
||||
movl $_mp_lock, %edx ; /* GIANT_LOCK */ \
|
||||
call _MPrellock_edx
|
||||
|
||||
#endif /* FAST_SIMPLELOCK */
|
||||
|
||||
/* convert an absolute IRQ# into a bitmask */
|
||||
#define IRQ_BIT(irq_num) (1 << (irq_num))
|
||||
|
||||
@ -42,10 +31,6 @@
|
||||
* Macros for interrupt interrupt entry, call to handler, and exit.
|
||||
*/
|
||||
|
||||
#ifdef FAST_WITHOUTCPL
|
||||
|
||||
/*
|
||||
*/
|
||||
#define FAST_INTR(irq_num, vec_name) \
|
||||
.text ; \
|
||||
SUPERALIGN_TEXT ; \
|
||||
@ -82,83 +67,6 @@ IDTVEC(vec_name) ; \
|
||||
popl %eax ; \
|
||||
iret
|
||||
|
||||
#else /* FAST_WITHOUTCPL */
|
||||
|
||||
#define FAST_INTR(irq_num, vec_name) \
|
||||
.text ; \
|
||||
SUPERALIGN_TEXT ; \
|
||||
IDTVEC(vec_name) ; \
|
||||
pushl %eax ; /* save only call-used registers */ \
|
||||
pushl %ecx ; \
|
||||
pushl %edx ; \
|
||||
pushl %ds ; \
|
||||
MAYBE_PUSHL_ES ; \
|
||||
pushl %fs ; \
|
||||
movl $KDSEL, %eax ; \
|
||||
movl %ax, %ds ; \
|
||||
MAYBE_MOVW_AX_ES ; \
|
||||
movl $KPSEL, %eax ; \
|
||||
movl %ax, %fs ; \
|
||||
FAKE_MCOUNT((5+ACTUALLY_PUSHED)*4(%esp)) ; \
|
||||
GET_FAST_INTR_LOCK ; \
|
||||
pushl _intr_unit + (irq_num) * 4 ; \
|
||||
call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
|
||||
addl $4, %esp ; \
|
||||
movl $0, lapic_eoi ; \
|
||||
lock ; \
|
||||
incl _cnt+V_INTR ; /* book-keeping can wait */ \
|
||||
movl _intr_countp + (irq_num) * 4,%eax ; \
|
||||
lock ; \
|
||||
incl (%eax) ; \
|
||||
movl _cpl, %eax ; /* unmasking pending HWIs or SWIs? */ \
|
||||
notl %eax ; \
|
||||
andl _ipending, %eax ; \
|
||||
jne 2f ; /* yes, maybe handle them */ \
|
||||
1: ; \
|
||||
MEXITCOUNT ; \
|
||||
REL_FAST_INTR_LOCK ; \
|
||||
popl %fs ; \
|
||||
MAYBE_POPL_ES ; \
|
||||
popl %ds ; \
|
||||
popl %edx ; \
|
||||
popl %ecx ; \
|
||||
popl %eax ; \
|
||||
iret ; \
|
||||
; \
|
||||
ALIGN_TEXT ; \
|
||||
2: ; \
|
||||
cmpb $3, _intr_nesting_level ; /* enough stack? */ \
|
||||
jae 1b ; /* no, return */ \
|
||||
movl _cpl, %eax ; \
|
||||
/* XXX next line is probably unnecessary now. */ \
|
||||
movl $HWI_MASK|SWI_MASK, _cpl ; /* limit nesting ... */ \
|
||||
lock ; \
|
||||
incb _intr_nesting_level ; /* ... really limit it ... */ \
|
||||
sti ; /* to do this as early as possible */ \
|
||||
popl %fs ; /* discard most of thin frame ... */ \
|
||||
MAYBE_POPL_ES ; /* discard most of thin frame ... */ \
|
||||
popl %ecx ; /* ... original %ds ... */ \
|
||||
popl %edx ; \
|
||||
xchgl %eax, 4(%esp) ; /* orig %eax; save cpl */ \
|
||||
pushal ; /* build fat frame (grrr) ... */ \
|
||||
pushl %ecx ; /* ... actually %ds ... */ \
|
||||
pushl %es ; \
|
||||
pushl %fs ;
|
||||
movl $KDSEL, %eax ; \
|
||||
movl %ax, %es ; \
|
||||
movl $KPSEL, %eax ;
|
||||
movl %ax, %fs ;
|
||||
movl (3+8+0)*4(%esp), %ecx ; /* %ecx from thin frame ... */ \
|
||||
movl %ecx, (3+6)*4(%esp) ; /* ... to fat frame ... */ \
|
||||
movl (3+8+1)*4(%esp), %eax ; /* ... cpl from thin frame */ \
|
||||
pushl %eax ; \
|
||||
subl $4, %esp ; /* junk for unit number */ \
|
||||
MEXITCOUNT ; \
|
||||
jmp _doreti
|
||||
|
||||
#endif /** FAST_WITHOUTCPL */
|
||||
|
||||
|
||||
/*
|
||||
*
|
||||
*/
|
||||
@ -242,19 +150,6 @@ IDTVEC(vec_name) ; \
|
||||
7: ; \
|
||||
IMASK_UNLOCK
|
||||
|
||||
#ifdef INTR_SIMPLELOCK
|
||||
#define ENLOCK
|
||||
#define DELOCK
|
||||
#define LATELOCK call _get_isrlock
|
||||
#else
|
||||
#define ENLOCK \
|
||||
ISR_TRYLOCK ; /* XXX this is going away... */ \
|
||||
testl %eax, %eax ; /* did we get it? */ \
|
||||
jz 3f
|
||||
#define DELOCK ISR_RELLOCK
|
||||
#define LATELOCK
|
||||
#endif
|
||||
|
||||
#ifdef APIC_INTR_DIAGNOSTIC
|
||||
#ifdef APIC_INTR_DIAGNOSTIC_IRQ
|
||||
log_intr_event:
|
||||
@ -319,8 +214,6 @@ log_intr_event:
|
||||
#define APIC_ITRACE(name, irq_num, id)
|
||||
#endif
|
||||
|
||||
#ifdef CPL_AND_CML
|
||||
|
||||
#define INTR(irq_num, vec_name, maybe_extra_ipending) \
|
||||
.text ; \
|
||||
SUPERALIGN_TEXT ; \
|
||||
@ -344,132 +237,13 @@ IDTVEC(vec_name) ; \
|
||||
EOI_IRQ(irq_num) ; \
|
||||
0: ; \
|
||||
APIC_ITRACE(apic_itrace_tryisrlock, irq_num, APIC_ITRACE_TRYISRLOCK) ;\
|
||||
ENLOCK ; \
|
||||
; \
|
||||
APIC_ITRACE(apic_itrace_gotisrlock, irq_num, APIC_ITRACE_GOTISRLOCK) ;\
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
testl $IRQ_BIT(irq_num), _cpl ; \
|
||||
jne 2f ; /* this INT masked */ \
|
||||
testl $IRQ_BIT(irq_num), _cml ; \
|
||||
jne 2f ; /* this INT masked */ \
|
||||
orl $IRQ_BIT(irq_num), _cil ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
; \
|
||||
incb _intr_nesting_level ; \
|
||||
; \
|
||||
/* entry point used by doreti_unpend for HWIs. */ \
|
||||
__CONCAT(Xresume,irq_num): ; \
|
||||
FAKE_MCOUNT(13*4(%esp)) ; /* XXX avoid dbl cnt */ \
|
||||
lock ; incl _cnt+V_INTR ; /* tally interrupts */ \
|
||||
movl _intr_countp + (irq_num) * 4, %eax ; \
|
||||
lock ; incl (%eax) ; \
|
||||
; \
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
movl _cml, %eax ; \
|
||||
pushl %eax ; \
|
||||
orl _intr_mask + (irq_num) * 4, %eax ; \
|
||||
movl %eax, _cml ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
; \
|
||||
pushl _intr_unit + (irq_num) * 4 ; \
|
||||
incl _inside_intr ; \
|
||||
APIC_ITRACE(apic_itrace_enter2, irq_num, APIC_ITRACE_ENTER2) ; \
|
||||
sti ; \
|
||||
call *_intr_handler + (irq_num) * 4 ; \
|
||||
cli ; \
|
||||
APIC_ITRACE(apic_itrace_leave, irq_num, APIC_ITRACE_LEAVE) ; \
|
||||
decl _inside_intr ; \
|
||||
; \
|
||||
lock ; andl $~IRQ_BIT(irq_num), iactive ; \
|
||||
lock ; andl $~IRQ_BIT(irq_num), _cil ; \
|
||||
UNMASK_IRQ(irq_num) ; \
|
||||
APIC_ITRACE(apic_itrace_unmask, irq_num, APIC_ITRACE_UNMASK) ; \
|
||||
sti ; /* doreti repeats cli/sti */ \
|
||||
MEXITCOUNT ; \
|
||||
LATELOCK ; \
|
||||
jmp _doreti ; \
|
||||
; \
|
||||
ALIGN_TEXT ; \
|
||||
1: ; /* active */ \
|
||||
APIC_ITRACE(apic_itrace_active, irq_num, APIC_ITRACE_ACTIVE) ; \
|
||||
MASK_IRQ(irq_num) ; \
|
||||
EOI_IRQ(irq_num) ; \
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
lock ; \
|
||||
orl $IRQ_BIT(irq_num), _ipending ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
lock ; \
|
||||
btsl $(irq_num), iactive ; /* still active */ \
|
||||
jnc 0b ; /* retry */ \
|
||||
POP_FRAME ; \
|
||||
iret ; \
|
||||
; \
|
||||
ALIGN_TEXT ; \
|
||||
2: ; /* masked by cpl|cml */ \
|
||||
APIC_ITRACE(apic_itrace_masked, irq_num, APIC_ITRACE_MASKED) ; \
|
||||
lock ; \
|
||||
orl $IRQ_BIT(irq_num), _ipending ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
DELOCK ; /* XXX this is going away... */ \
|
||||
POP_FRAME ; \
|
||||
iret ; \
|
||||
ALIGN_TEXT ; \
|
||||
3: ; /* other cpu has isr lock */ \
|
||||
APIC_ITRACE(apic_itrace_noisrlock, irq_num, APIC_ITRACE_NOISRLOCK) ;\
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
lock ; \
|
||||
orl $IRQ_BIT(irq_num), _ipending ; \
|
||||
testl $IRQ_BIT(irq_num), _cpl ; \
|
||||
jne 4f ; /* this INT masked */ \
|
||||
testl $IRQ_BIT(irq_num), _cml ; \
|
||||
jne 4f ; /* this INT masked */ \
|
||||
orl $IRQ_BIT(irq_num), _cil ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
call forward_irq ; /* forward irq to lock holder */ \
|
||||
POP_FRAME ; /* and return */ \
|
||||
iret ; \
|
||||
ALIGN_TEXT ; \
|
||||
4: ; /* blocked */ \
|
||||
APIC_ITRACE(apic_itrace_masked2, irq_num, APIC_ITRACE_MASKED2) ;\
|
||||
AVCPL_UNLOCK ; \
|
||||
POP_FRAME ; /* and return */ \
|
||||
iret
|
||||
|
||||
#else /* CPL_AND_CML */
|
||||
|
||||
|
||||
#define INTR(irq_num, vec_name, maybe_extra_ipending) \
|
||||
.text ; \
|
||||
SUPERALIGN_TEXT ; \
|
||||
/* _XintrNN: entry point used by IDT/HWIs & splz_unpend via _vec[]. */ \
|
||||
IDTVEC(vec_name) ; \
|
||||
PUSH_FRAME ; \
|
||||
movl $KDSEL, %eax ; /* reload with kernel's data segment */ \
|
||||
movl %ax, %ds ; \
|
||||
movl %ax, %es ; \
|
||||
movl $KPSEL, %eax ; \
|
||||
movl %ax, %fs ; \
|
||||
; \
|
||||
maybe_extra_ipending ; \
|
||||
; \
|
||||
APIC_ITRACE(apic_itrace_enter, irq_num, APIC_ITRACE_ENTER) ; \
|
||||
lock ; /* MP-safe */ \
|
||||
btsl $(irq_num), iactive ; /* lazy masking */ \
|
||||
jc 1f ; /* already active */ \
|
||||
; \
|
||||
MASK_LEVEL_IRQ(irq_num) ; \
|
||||
EOI_IRQ(irq_num) ; \
|
||||
0: ; \
|
||||
APIC_ITRACE(apic_itrace_tryisrlock, irq_num, APIC_ITRACE_TRYISRLOCK) ;\
|
||||
ISR_TRYLOCK ; /* XXX this is going away... */ \
|
||||
MP_TRYLOCK ; /* XXX this is going away... */ \
|
||||
testl %eax, %eax ; /* did we get it? */ \
|
||||
jz 3f ; /* no */ \
|
||||
; \
|
||||
APIC_ITRACE(apic_itrace_gotisrlock, irq_num, APIC_ITRACE_GOTISRLOCK) ;\
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
testl $IRQ_BIT(irq_num), _cpl ; \
|
||||
jne 2f ; /* this INT masked */ \
|
||||
AVCPL_UNLOCK ; \
|
||||
; \
|
||||
incb _intr_nesting_level ; \
|
||||
; \
|
||||
@ -480,14 +254,12 @@ __CONCAT(Xresume,irq_num): ; \
|
||||
movl _intr_countp + (irq_num) * 4, %eax ; \
|
||||
lock ; incl (%eax) ; \
|
||||
; \
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
movl _cpl, %eax ; \
|
||||
pushl %eax ; \
|
||||
orl _intr_mask + (irq_num) * 4, %eax ; \
|
||||
movl %eax, _cpl ; \
|
||||
lock ; \
|
||||
andl $~IRQ_BIT(irq_num), _ipending ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
; \
|
||||
pushl _intr_unit + (irq_num) * 4 ; \
|
||||
APIC_ITRACE(apic_itrace_enter2, irq_num, APIC_ITRACE_ENTER2) ; \
|
||||
@ -508,10 +280,8 @@ __CONCAT(Xresume,irq_num): ; \
|
||||
APIC_ITRACE(apic_itrace_active, irq_num, APIC_ITRACE_ACTIVE) ; \
|
||||
MASK_IRQ(irq_num) ; \
|
||||
EOI_IRQ(irq_num) ; \
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
lock ; \
|
||||
orl $IRQ_BIT(irq_num), _ipending ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
lock ; \
|
||||
btsl $(irq_num), iactive ; /* still active */ \
|
||||
jnc 0b ; /* retry */ \
|
||||
@ -522,32 +292,25 @@ __CONCAT(Xresume,irq_num): ; \
|
||||
APIC_ITRACE(apic_itrace_masked, irq_num, APIC_ITRACE_MASKED) ; \
|
||||
lock ; \
|
||||
orl $IRQ_BIT(irq_num), _ipending ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
ISR_RELLOCK ; /* XXX this is going away... */ \
|
||||
MP_RELLOCK ; \
|
||||
POP_FRAME ; \
|
||||
iret ; \
|
||||
ALIGN_TEXT ; \
|
||||
3: ; /* other cpu has isr lock */ \
|
||||
APIC_ITRACE(apic_itrace_noisrlock, irq_num, APIC_ITRACE_NOISRLOCK) ;\
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
lock ; \
|
||||
orl $IRQ_BIT(irq_num), _ipending ; \
|
||||
testl $IRQ_BIT(irq_num), _cpl ; \
|
||||
jne 4f ; /* this INT masked */ \
|
||||
AVCPL_UNLOCK ; \
|
||||
call forward_irq ; /* forward irq to lock holder */ \
|
||||
POP_FRAME ; /* and return */ \
|
||||
iret ; \
|
||||
ALIGN_TEXT ; \
|
||||
4: ; /* blocked */ \
|
||||
APIC_ITRACE(apic_itrace_masked2, irq_num, APIC_ITRACE_MASKED2) ;\
|
||||
AVCPL_UNLOCK ; \
|
||||
POP_FRAME ; /* and return */ \
|
||||
iret
|
||||
|
||||
#endif /* CPL_AND_CML */
|
||||
|
||||
|
||||
/*
|
||||
* Handle "spurious INTerrupts".
|
||||
* Notes:
|
||||
@ -635,11 +398,6 @@ _Xcpucheckstate:
|
||||
testl $PSL_VM, 24(%esp)
|
||||
jne 1f
|
||||
incl %ebx /* system or interrupt */
|
||||
#ifdef CPL_AND_CML
|
||||
cmpl $0, _inside_intr
|
||||
je 1f
|
||||
incl %ebx /* interrupt */
|
||||
#endif
|
||||
1:
|
||||
movl _cpuid, %eax
|
||||
movl %ebx, _checkstate_cpustate(,%eax,4)
|
||||
@ -693,17 +451,11 @@ _Xcpuast:
|
||||
* Giant locks do not come cheap.
|
||||
* A lot of cycles are going to be wasted here.
|
||||
*/
|
||||
call _get_isrlock
|
||||
call _get_mplock
|
||||
|
||||
AVCPL_LOCK
|
||||
#ifdef CPL_AND_CML
|
||||
movl _cml, %eax
|
||||
#else
|
||||
movl _cpl, %eax
|
||||
#endif
|
||||
pushl %eax
|
||||
movl $1, _astpending /* XXX */
|
||||
AVCPL_UNLOCK
|
||||
lock
|
||||
incb _intr_nesting_level
|
||||
sti
|
||||
@ -716,7 +468,7 @@ _Xcpuast:
|
||||
lock
|
||||
btrl %eax, CNAME(resched_cpus)
|
||||
jnc 2f
|
||||
movl $1, CNAME(want_resched)
|
||||
orl $AST_RESCHED,_astpending
|
||||
lock
|
||||
incl CNAME(want_resched_cnt)
|
||||
2:
|
||||
@ -749,7 +501,7 @@ _Xforward_irq:
|
||||
|
||||
FAKE_MCOUNT(13*4(%esp))
|
||||
|
||||
ISR_TRYLOCK
|
||||
MP_TRYLOCK
|
||||
testl %eax,%eax /* Did we get the lock ? */
|
||||
jz 1f /* No */
|
||||
|
||||
@ -758,14 +510,8 @@ _Xforward_irq:
|
||||
cmpb $4, _intr_nesting_level
|
||||
jae 2f
|
||||
|
||||
AVCPL_LOCK
|
||||
#ifdef CPL_AND_CML
|
||||
movl _cml, %eax
|
||||
#else
|
||||
movl _cpl, %eax
|
||||
#endif
|
||||
pushl %eax
|
||||
AVCPL_UNLOCK
|
||||
lock
|
||||
incb _intr_nesting_level
|
||||
sti
|
||||
@ -785,7 +531,7 @@ _Xforward_irq:
|
||||
lock
|
||||
incl CNAME(forward_irq_toodeepcnt)
|
||||
3:
|
||||
ISR_RELLOCK
|
||||
MP_RELLOCK
|
||||
MEXITCOUNT
|
||||
POP_FRAME
|
||||
iret
|
||||
|
@ -41,23 +41,11 @@
|
||||
#include <machine/psl.h>
|
||||
#include <machine/trap.h>
|
||||
#ifdef SMP
|
||||
#include <machine/smptests.h> /** CPL_AND_CML, REAL_ */
|
||||
#include <machine/smptests.h> /** various SMP options */
|
||||
#endif
|
||||
|
||||
#include "assym.s"
|
||||
|
||||
#ifndef SMP
|
||||
#define ECPL_LOCK /* make these nops */
|
||||
#define ECPL_UNLOCK
|
||||
#define ICPL_LOCK
|
||||
#define ICPL_UNLOCK
|
||||
#define FAST_ICPL_UNLOCK
|
||||
#define AICPL_LOCK
|
||||
#define AICPL_UNLOCK
|
||||
#define AVCPL_LOCK
|
||||
#define AVCPL_UNLOCK
|
||||
#endif /* SMP */
|
||||
|
||||
#ifdef SMP
|
||||
#define MOVL_KPSEL_EAX movl $KPSEL,%eax
|
||||
#else
|
||||
@ -71,16 +59,45 @@
|
||||
/* Trap handling */
|
||||
/*****************************************************************************/
|
||||
/*
|
||||
* Trap and fault vector routines
|
||||
* Trap and fault vector routines.
|
||||
*
|
||||
* Most traps are 'trap gates', SDT_SYS386TGT. A trap gate pushes state on
|
||||
* the stack that mostly looks like an interrupt, but does not disable
|
||||
* interrupts. A few of the traps we are use are interrupt gates,
|
||||
* SDT_SYS386IGT, which are nearly the same thing except interrupts are
|
||||
* disabled on entry.
|
||||
*
|
||||
* The cpu will push a certain amount of state onto the kernel stack for
|
||||
* the current process. The amount of state depends on the type of trap
|
||||
* and whether the trap crossed rings or not. See i386/include/frame.h.
|
||||
* At the very least the current EFLAGS (status register, which includes
|
||||
* the interrupt disable state prior to the trap), the code segment register,
|
||||
* and the return instruction pointer are pushed by the cpu. The cpu
|
||||
* will also push an 'error' code for certain traps. We push a dummy
|
||||
* error code for those traps where the cpu doesn't in order to maintain
|
||||
* a consistent frame. We also push a contrived 'trap number'.
|
||||
*
|
||||
* The cpu does not push the general registers, we must do that, and we
|
||||
* must restore them prior to calling 'iret'. The cpu adjusts the %cs and
|
||||
* %ss segment registers, but does not mess with %ds, %es, or %fs. Thus we
|
||||
* must load them with appropriate values for supervisor mode operation.
|
||||
*
|
||||
* On entry to a trap or interrupt WE DO NOT OWN THE MP LOCK. This means
|
||||
* that we must be careful in regards to accessing global variables. We
|
||||
* save (push) the current cpl (our software interrupt disable mask), call
|
||||
* the trap function, then call _doreti to restore the cpl and deal with
|
||||
* ASTs (software interrupts). _doreti will determine if the restoration
|
||||
* of the cpl unmasked any pending interrupts and will issue those interrupts
|
||||
* synchronously prior to doing the iret.
|
||||
*
|
||||
* At the moment we must own the MP lock to do any cpl manipulation, which
|
||||
* means we must own it prior to calling _doreti. The syscall case attempts
|
||||
* to avoid this by handling a reduced set of cases itself and iret'ing.
|
||||
*/
|
||||
#define IDTVEC(name) ALIGN_TEXT; .globl __CONCAT(_X,name); \
|
||||
.type __CONCAT(_X,name),@function; __CONCAT(_X,name):
|
||||
#define TRAP(a) pushl $(a) ; jmp _alltraps
|
||||
|
||||
/*
|
||||
* XXX - debugger traps are now interrupt gates so at least bdb doesn't lose
|
||||
* control. The sti's give the standard losing behaviour for ddb and kgdb.
|
||||
*/
|
||||
#ifdef BDE_DEBUGGER
|
||||
#define BDBTRAP(name) \
|
||||
ss ; \
|
||||
@ -160,16 +177,9 @@ IDTVEC(fpu)
|
||||
|
||||
#ifdef SMP
|
||||
MPLOCKED incl _cnt+V_TRAP
|
||||
FPU_LOCK
|
||||
ECPL_LOCK
|
||||
#ifdef CPL_AND_CML
|
||||
movl _cml,%eax
|
||||
pushl %eax /* save original cml */
|
||||
#else
|
||||
MP_LOCK
|
||||
movl _cpl,%eax
|
||||
pushl %eax /* save original cpl */
|
||||
#endif /* CPL_AND_CML */
|
||||
ECPL_UNLOCK
|
||||
pushl $0 /* dummy unit to finish intr frame */
|
||||
#else /* SMP */
|
||||
movl _cpl,%eax
|
||||
@ -190,6 +200,16 @@ IDTVEC(fpu)
|
||||
IDTVEC(align)
|
||||
TRAP(T_ALIGNFLT)
|
||||
|
||||
/*
|
||||
* _alltraps entry point. Interrupts are enabled if this was a trap
|
||||
* gate (TGT), else disabled if this was an interrupt gate (IGT).
|
||||
* Note that int0x80_syscall is a trap gate. Only page faults
|
||||
* use an interrupt gate.
|
||||
*
|
||||
* Note that all calls to MP_LOCK must occur with interrupts enabled
|
||||
* in order to be able to take IPI's while waiting for the lock.
|
||||
*/
|
||||
|
||||
SUPERALIGN_TEXT
|
||||
.globl _alltraps
|
||||
.type _alltraps,@function
|
||||
@ -208,14 +228,8 @@ alltraps_with_regs_pushed:
|
||||
calltrap:
|
||||
FAKE_MCOUNT(_btrap) /* init "from" _btrap -> calltrap */
|
||||
MPLOCKED incl _cnt+V_TRAP
|
||||
ALIGN_LOCK
|
||||
ECPL_LOCK
|
||||
#ifdef CPL_AND_CML
|
||||
movl _cml,%ebx /* keep orig. cml here during trap() */
|
||||
#else
|
||||
MP_LOCK
|
||||
movl _cpl,%ebx /* keep orig. cpl here during trap() */
|
||||
#endif
|
||||
ECPL_UNLOCK
|
||||
call _trap
|
||||
|
||||
/*
|
||||
@ -229,17 +243,19 @@ calltrap:
|
||||
jmp _doreti
|
||||
|
||||
/*
|
||||
* Call gate entry for syscall.
|
||||
* SYSCALL CALL GATE (old entry point for a.out binaries)
|
||||
*
|
||||
* The intersegment call has been set up to specify one dummy parameter.
|
||||
*
|
||||
* This leaves a place to put eflags so that the call frame can be
|
||||
* converted to a trap frame. Note that the eflags is (semi-)bogusly
|
||||
* pushed into (what will be) tf_err and then copied later into the
|
||||
* final spot. It has to be done this way because esp can't be just
|
||||
* temporarily altered for the pushfl - an interrupt might come in
|
||||
* and clobber the saved cs/eip.
|
||||
*/
|
||||
/*
|
||||
* THis first callgate is used for the old a.out binaries
|
||||
*
|
||||
* We do not obtain the MP lock, but the call to syscall2 might. If it
|
||||
* does it will release the lock prior to returning.
|
||||
*/
|
||||
SUPERALIGN_TEXT
|
||||
IDTVEC(syscall)
|
||||
@ -259,20 +275,28 @@ IDTVEC(syscall)
|
||||
movl $7,TF_ERR(%esp) /* sizeof "lcall 7,0" */
|
||||
FAKE_MCOUNT(13*4(%esp))
|
||||
MPLOCKED incl _cnt+V_SYSCALL
|
||||
SYSCALL_LOCK
|
||||
call _syscall
|
||||
|
||||
/*
|
||||
* Return via _doreti to handle ASTs.
|
||||
*/
|
||||
pushl $0 /* cpl to restore */
|
||||
subl $4,%esp /* dummy unit to finish intr frame */
|
||||
movb $1,_intr_nesting_level
|
||||
call _syscall2
|
||||
MEXITCOUNT
|
||||
cli /* atomic astpending access */
|
||||
cmpl $0,_astpending
|
||||
je doreti_syscall_ret
|
||||
#ifdef SMP
|
||||
MP_LOCK
|
||||
#endif
|
||||
pushl $0 /* cpl to restore */
|
||||
subl $4,%esp /* dummy unit for interrupt frame */
|
||||
movb $1,_intr_nesting_level
|
||||
jmp _doreti
|
||||
|
||||
/*
|
||||
* Call gate entry for FreeBSD ELF and Linux/NetBSD syscall (int 0x80)
|
||||
*
|
||||
* Even though the name says 'int0x80', this is actually a TGT (trap gate)
|
||||
* rather then an IGT (interrupt gate). Thus interrupts are enabled on
|
||||
* entry just as they are for a normal syscall.
|
||||
*
|
||||
* We do not obtain the MP lock, but the call to syscall2 might. If it
|
||||
* does it will release the lock prior to returning.
|
||||
*/
|
||||
SUPERALIGN_TEXT
|
||||
IDTVEC(int0x80_syscall)
|
||||
@ -289,16 +313,17 @@ IDTVEC(int0x80_syscall)
|
||||
movl $2,TF_ERR(%esp) /* sizeof "int 0x80" */
|
||||
FAKE_MCOUNT(13*4(%esp))
|
||||
MPLOCKED incl _cnt+V_SYSCALL
|
||||
ALTSYSCALL_LOCK
|
||||
call _syscall
|
||||
|
||||
/*
|
||||
* Return via _doreti to handle ASTs.
|
||||
*/
|
||||
pushl $0 /* cpl to restore */
|
||||
subl $4,%esp /* dummy unit to finish intr frame */
|
||||
movb $1,_intr_nesting_level
|
||||
call _syscall2
|
||||
MEXITCOUNT
|
||||
cli /* atomic astpending access */
|
||||
cmpl $0,_astpending
|
||||
je doreti_syscall_ret
|
||||
#ifdef SMP
|
||||
MP_LOCK
|
||||
#endif
|
||||
pushl $0 /* cpl to restore */
|
||||
subl $4,%esp /* dummy unit for interrupt frame */
|
||||
movb $1,_intr_nesting_level
|
||||
jmp _doreti
|
||||
|
||||
ENTRY(fork_trampoline)
|
||||
|
@ -176,6 +176,7 @@ ASSYM(GD_SWITCHTIME, offsetof(struct globaldata, gd_switchtime));
|
||||
ASSYM(GD_SWITCHTICKS, offsetof(struct globaldata, gd_switchticks));
|
||||
ASSYM(GD_COMMON_TSSD, offsetof(struct globaldata, gd_common_tssd));
|
||||
ASSYM(GD_TSS_GDT, offsetof(struct globaldata, gd_tss_gdt));
|
||||
ASSYM(GD_ASTPENDING, offsetof(struct globaldata, gd_astpending));
|
||||
|
||||
#ifdef USER_LDT
|
||||
ASSYM(GD_CURRENTLDT, offsetof(struct globaldata, gd_currentldt));
|
||||
|
@ -61,9 +61,10 @@ globaldata:
|
||||
#else
|
||||
.set globaldata,0
|
||||
#endif
|
||||
.globl gd_curproc, gd_curpcb, gd_npxproc
|
||||
.globl gd_curproc, gd_curpcb, gd_npxproc, gd_astpending
|
||||
.globl gd_common_tss, gd_switchtime, gd_switchticks
|
||||
.set gd_curproc,globaldata + GD_CURPROC
|
||||
.set gd_astpending,globaldata + GD_ASTPENDING
|
||||
.set gd_curpcb,globaldata + GD_CURPCB
|
||||
.set gd_npxproc,globaldata + GD_NPXPROC
|
||||
.set gd_common_tss,globaldata + GD_COMMON_TSS
|
||||
@ -80,9 +81,10 @@ globaldata:
|
||||
#endif
|
||||
|
||||
#ifndef SMP
|
||||
.globl _curproc, _curpcb, _npxproc
|
||||
.globl _curproc, _curpcb, _npxproc, _astpending
|
||||
.globl _common_tss, _switchtime, _switchticks
|
||||
.set _curproc,globaldata + GD_CURPROC
|
||||
.set _astpending,globaldata + GD_ASTPENDING
|
||||
.set _curpcb,globaldata + GD_CURPCB
|
||||
.set _npxproc,globaldata + GD_NPXPROC
|
||||
.set _common_tss,globaldata + GD_COMMON_TSS
|
||||
|
@ -1770,8 +1770,10 @@ init_locks(void)
|
||||
*/
|
||||
mp_lock = 0x00000001;
|
||||
|
||||
#if 0
|
||||
/* ISR uses its own "giant lock" */
|
||||
isr_lock = FREE_LOCK;
|
||||
#endif
|
||||
|
||||
#if defined(APIC_INTR_DIAGNOSTIC) && defined(APIC_INTR_DIAGNOSTIC_IRQ)
|
||||
s_lock_init((struct simplelock*)&apic_itrace_debuglock);
|
||||
|
@ -18,7 +18,6 @@
|
||||
* The attempt to seize/release the semaphore and the increment/decrement
|
||||
* is done in one atomic operation. This way we are safe from all kinds
|
||||
* of weird reentrancy situations.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <machine/asmacros.h>
|
||||
@ -51,12 +50,8 @@
|
||||
* Claim LOWest PRIOrity, ie. attempt to grab ALL INTerrupts.
|
||||
*/
|
||||
|
||||
/* location of saved TPR on stack */
|
||||
#define TPR_TARGET 8(%esp)
|
||||
|
||||
/* after 1st acquire of lock we attempt to grab all hardware INTs */
|
||||
#define GRAB_HWI movl $ALLHWI_LEVEL, TPR_TARGET
|
||||
#define GRAB_HWI_2 movl $ALLHWI_LEVEL, lapic_tpr /* CHEAP_TPR */
|
||||
/* after 1st acquire of lock we grab all hardware INTs */
|
||||
#define GRAB_HWI movl $ALLHWI_LEVEL, lapic_tpr
|
||||
|
||||
/* after last release of lock give up LOW PRIO (ie, arbitrate INTerrupts) */
|
||||
#define ARB_HWI movl $LOPRIO_LEVEL, lapic_tpr /* CHEAP_TPR */
|
||||
@ -64,7 +59,6 @@
|
||||
#else /* GRAB_LOPRIO */
|
||||
|
||||
#define GRAB_HWI /* nop */
|
||||
#define GRAB_HWI_2 /* nop */
|
||||
#define ARB_HWI /* nop */
|
||||
|
||||
#endif /* GRAB_LOPRIO */
|
||||
@ -75,7 +69,8 @@
|
||||
* void MPgetlock_edx(unsigned int *lock : %edx)
|
||||
* ----------------------------------
|
||||
* Destroys %eax, %ecx. %edx must hold lock argument.
|
||||
* Note: TPR_TARGET (relative to the stack) is destroyed in GRAB_HWI
|
||||
*
|
||||
* Grabs hardware interrupts on first aquire.
|
||||
*
|
||||
* NOTE: Serialization is not required if we already hold the lock, since
|
||||
* we already hold the lock, nor do we need a locked instruction if we
|
||||
@ -131,7 +126,7 @@ NON_GPROF_ENTRY(MPtrylock)
|
||||
#ifdef GLPROFILE
|
||||
incl _tryhits2
|
||||
#endif /* GLPROFILE */
|
||||
GRAB_HWI_2 /* 1st acquire, grab hw INTs */
|
||||
GRAB_HWI /* 1st acquire, grab hw INTs */
|
||||
movl $1, %eax
|
||||
ret
|
||||
1:
|
||||
@ -197,38 +192,51 @@ NON_GPROF_ENTRY(MPrellock_edx)
|
||||
*
|
||||
* Stack (after call to _MPgetlock):
|
||||
*
|
||||
* EFLAGS 4(%esp)
|
||||
* local APIC TPR 8(%esp) <-- note, TPR_TARGET
|
||||
* edx 12(%esp)
|
||||
* ecx 16(%esp)
|
||||
* eax 20(%esp)
|
||||
* edx 4(%esp)
|
||||
* ecx 8(%esp)
|
||||
* eax 12(%esp)
|
||||
*
|
||||
* Requirements: Interrupts should be enabled on call so we can take
|
||||
* IPI's and FAST INTs while we are waiting for the lock
|
||||
* (else the system may not be able to halt).
|
||||
*
|
||||
* XXX there are still places where get_mplock() is called
|
||||
* with interrupts disabled, so we have to temporarily reenable
|
||||
* interrupts.
|
||||
*
|
||||
* Side effects: The current cpu will be given ownership of the
|
||||
* hardware interrupts when it first aquires the lock.
|
||||
*
|
||||
* Costs: Initial aquisition requires the use of a costly locked
|
||||
* instruction, but recursive aquisition is cheap. Release
|
||||
* is very cheap.
|
||||
*/
|
||||
|
||||
NON_GPROF_ENTRY(get_mplock)
|
||||
pushl %eax
|
||||
pushl %ecx
|
||||
pushl %edx
|
||||
|
||||
/* block all HW INTs via Task Priority Register */
|
||||
pushl lapic_tpr /* save current TPR */
|
||||
pushfl /* save current EFLAGS */
|
||||
testl $(1<<9), (%esp) /* test EI bit */
|
||||
jnz 1f /* INTs currently enabled */
|
||||
sti /* allow IPI and FAST INTs */
|
||||
1:
|
||||
movl $_mp_lock, %edx
|
||||
pushfl
|
||||
testl $(1<<9), (%esp)
|
||||
jz 2f
|
||||
call _MPgetlock_edx
|
||||
|
||||
popfl /* restore original EFLAGS */
|
||||
popl lapic_tpr /* restore TPR */
|
||||
addl $4,%esp
|
||||
1:
|
||||
popl %edx
|
||||
popl %ecx
|
||||
popl %eax
|
||||
ret
|
||||
2:
|
||||
sti
|
||||
call _MPgetlock_edx
|
||||
popfl
|
||||
jmp 1b
|
||||
|
||||
/*
|
||||
* Special version of get_mplock that is used during bootstrap when we can't
|
||||
* yet enable interrupts of any sort since the APIC isn't online yet.
|
||||
* yet enable interrupts of any sort since the APIC isn't online yet. We
|
||||
* do an endrun around MPgetlock_edx to avoid enabling interrupts.
|
||||
*
|
||||
* XXX FIXME.. - APIC should be online from the start to simplify IPI's.
|
||||
*/
|
||||
@ -236,20 +244,19 @@ NON_GPROF_ENTRY(boot_get_mplock)
|
||||
pushl %eax
|
||||
pushl %ecx
|
||||
pushl %edx
|
||||
|
||||
#ifdef GRAB_LOPRIO
|
||||
pushl $0 /* dummy TPR (TPR_TARGET) */
|
||||
pushfl
|
||||
pushl lapic_tpr
|
||||
cli
|
||||
#endif
|
||||
|
||||
movl $_mp_lock, %edx
|
||||
call _MPgetlock_edx
|
||||
|
||||
#ifdef GRAB_LOPRIO
|
||||
popl lapic_tpr
|
||||
popfl
|
||||
addl $4, %esp
|
||||
#endif
|
||||
|
||||
popl %edx
|
||||
popl %ecx
|
||||
popl %eax
|
||||
@ -286,214 +293,6 @@ NON_GPROF_ENTRY(rel_mplock)
|
||||
popl %ecx
|
||||
ret
|
||||
|
||||
/***********************************************************************
|
||||
* void get_isrlock()
|
||||
* -----------------
|
||||
* no registers preserved, assummed the calling ISR does!
|
||||
*
|
||||
* Stack (after call to _MPgetlock):
|
||||
*
|
||||
* EFLAGS 4(%esp)
|
||||
* local APIC TPR 8(%esp)
|
||||
*/
|
||||
|
||||
NON_GPROF_ENTRY(get_isrlock)
|
||||
|
||||
/* block all HW INTs via Task Priority Register */
|
||||
pushl lapic_tpr /* save current TPR (TPR_TARGET) */
|
||||
pushfl /* save current EFLAGS */
|
||||
sti /* allow IPI and FAST INTs */
|
||||
|
||||
movl $_mp_lock, %edx
|
||||
call _MPgetlock_edx
|
||||
|
||||
popfl /* restore original EFLAGS */
|
||||
popl lapic_tpr /* restore TPR */
|
||||
ret
|
||||
|
||||
|
||||
/***********************************************************************
|
||||
* void try_isrlock()
|
||||
* -----------------
|
||||
* no registers preserved, assummed the calling ISR does!
|
||||
* reg %eax == 1 if success
|
||||
*/
|
||||
|
||||
NON_GPROF_ENTRY(try_isrlock)
|
||||
pushl $_mp_lock
|
||||
call _MPtrylock
|
||||
add $4, %esp
|
||||
ret
|
||||
|
||||
|
||||
/***********************************************************************
|
||||
* void rel_isrlock()
|
||||
* -----------------
|
||||
* no registers preserved, assummed the calling ISR does!
|
||||
*/
|
||||
|
||||
NON_GPROF_ENTRY(rel_isrlock)
|
||||
movl $_mp_lock,%edx
|
||||
jmp _MPrellock_edx
|
||||
|
||||
|
||||
/***********************************************************************
|
||||
* FPU locks
|
||||
*/
|
||||
|
||||
NON_GPROF_ENTRY(get_fpu_lock)
|
||||
pushl lapic_tpr /* save current TPR (TPR_TARGET) */
|
||||
pushfl
|
||||
sti
|
||||
movl $_mp_lock, %edx
|
||||
call _MPgetlock_edx
|
||||
popfl
|
||||
popl lapic_tpr
|
||||
ret
|
||||
|
||||
#ifdef notneeded
|
||||
NON_GPROF_ENTRY(try_fpu_lock)
|
||||
pushl $_mp_lock
|
||||
call _MPtrylock
|
||||
add $4, %esp
|
||||
ret
|
||||
|
||||
NON_GPROF_ENTRY(rel_fpu_lock)
|
||||
movl $_mp_lock,%edx
|
||||
jmp _MPrellock_edx
|
||||
#endif /* notneeded */
|
||||
|
||||
|
||||
/***********************************************************************
|
||||
* align locks
|
||||
*/
|
||||
|
||||
NON_GPROF_ENTRY(get_align_lock)
|
||||
pushl lapic_tpr /* save current TPR (TPR_TARGET) */
|
||||
pushfl
|
||||
sti
|
||||
movl $_mp_lock, %edx
|
||||
call _MPgetlock_edx
|
||||
popfl
|
||||
popl lapic_tpr
|
||||
ret
|
||||
|
||||
#ifdef notneeded
|
||||
NON_GPROF_ENTRY(try_align_lock)
|
||||
pushl $_mp_lock
|
||||
call _MPtrylock
|
||||
add $4, %esp
|
||||
ret
|
||||
|
||||
NON_GPROF_ENTRY(rel_align_lock)
|
||||
movl $_mp_lock,%edx
|
||||
jmp _MPrellock_edx
|
||||
#endif /* notneeded */
|
||||
|
||||
|
||||
/***********************************************************************
|
||||
* syscall locks
|
||||
*/
|
||||
|
||||
NON_GPROF_ENTRY(get_syscall_lock)
|
||||
pushl lapic_tpr /* save current TPR (TPR_TARGET) */
|
||||
pushfl
|
||||
sti
|
||||
movl $_mp_lock, %edx
|
||||
call _MPgetlock_edx
|
||||
popfl
|
||||
popl lapic_tpr
|
||||
ret
|
||||
|
||||
#ifdef notneeded
|
||||
NON_GPROF_ENTRY(try_syscall_lock)
|
||||
pushl $_mp_lock
|
||||
call _MPtrylock
|
||||
add $4, %esp
|
||||
ret
|
||||
#endif /* notneeded */
|
||||
|
||||
NON_GPROF_ENTRY(rel_syscall_lock)
|
||||
movl $_mp_lock,%edx
|
||||
jmp _MPrellock_edx
|
||||
|
||||
|
||||
/***********************************************************************
|
||||
* altsyscall locks
|
||||
*/
|
||||
|
||||
NON_GPROF_ENTRY(get_altsyscall_lock)
|
||||
pushl lapic_tpr /* save current TPR (TPR_TARGET) */
|
||||
pushfl
|
||||
sti
|
||||
movl $_mp_lock, %edx
|
||||
call _MPgetlock_edx
|
||||
popfl
|
||||
popl lapic_tpr
|
||||
ret
|
||||
|
||||
#ifdef notneeded
|
||||
NON_GPROF_ENTRY(try_altsyscall_lock)
|
||||
pushl $_mp_lock
|
||||
call _MPtrylock
|
||||
add $4, %esp
|
||||
ret
|
||||
|
||||
NON_GPROF_ENTRY(rel_altsyscall_lock)
|
||||
movl $_mp_lock,%edx
|
||||
jmp _MPrellock_edx
|
||||
#endif /* notneeded */
|
||||
|
||||
|
||||
#ifdef RECURSIVE_MPINTRLOCK
|
||||
/***********************************************************************
|
||||
* void get_mpintrlock()
|
||||
* -----------------
|
||||
* All registers preserved
|
||||
*/
|
||||
|
||||
NON_GPROF_ENTRY(get_mpintrlock)
|
||||
pushl %eax
|
||||
pushl %ecx
|
||||
pushl %edx
|
||||
|
||||
#ifdef GRAB_LOPRIO
|
||||
pushl lapic_tpr /* save current TPR (TPR_TARGET) */
|
||||
pushfl
|
||||
#endif
|
||||
|
||||
movl $_mpintr_lock, %edx
|
||||
call _MPgetlock_edx
|
||||
|
||||
#ifdef GRAB_LOPRIO
|
||||
popfl
|
||||
popl lapic_tpr
|
||||
#endif
|
||||
|
||||
popl %edx
|
||||
popl %ecx
|
||||
popl %eax
|
||||
ret
|
||||
|
||||
/***********************************************************************
|
||||
* void rel_mpintrlock()
|
||||
* -----------------
|
||||
* All registers preserved
|
||||
*/
|
||||
|
||||
NON_GPROF_ENTRY(rel_mpintrlock)
|
||||
pushl %ecx
|
||||
pushl %edx
|
||||
|
||||
movl $_mpintr_lock,%edx
|
||||
call _MPrellock_edx
|
||||
|
||||
popl %edx
|
||||
popl %ecx
|
||||
ret
|
||||
#endif /* RECURSIVE_MPINTRLOCK */
|
||||
|
||||
|
||||
/***********************************************************************
|
||||
*
|
||||
*/
|
||||
@ -503,15 +302,6 @@ NON_GPROF_ENTRY(rel_mpintrlock)
|
||||
.globl _mp_lock
|
||||
_mp_lock: .long 0
|
||||
|
||||
.globl _isr_lock
|
||||
_isr_lock: .long 0
|
||||
|
||||
#ifdef RECURSIVE_MPINTRLOCK
|
||||
.globl _mpintr_lock
|
||||
_mpintr_lock: .long 0xffffffff
|
||||
#endif /* RECURSIVE_MPINTRLOCK */
|
||||
|
||||
|
||||
#ifdef GLPROFILE
|
||||
.globl _gethits
|
||||
_gethits:
|
||||
|
@ -1770,8 +1770,10 @@ init_locks(void)
|
||||
*/
|
||||
mp_lock = 0x00000001;
|
||||
|
||||
#if 0
|
||||
/* ISR uses its own "giant lock" */
|
||||
isr_lock = FREE_LOCK;
|
||||
#endif
|
||||
|
||||
#if defined(APIC_INTR_DIAGNOSTIC) && defined(APIC_INTR_DIAGNOSTIC_IRQ)
|
||||
s_lock_init((struct simplelock*)&apic_itrace_debuglock);
|
||||
|
@ -186,8 +186,11 @@ ENTRY(s_unlock)
|
||||
movl $0, (%eax)
|
||||
ret
|
||||
|
||||
#if 0
|
||||
|
||||
/*
|
||||
* XXX CRUFTY SS_LOCK IMPLEMENTATION REMOVED XXX
|
||||
*
|
||||
* These versions of simple_lock block interrupts,
|
||||
* making it suitable for regions accessed by both top and bottom levels.
|
||||
* This is done by saving the current value of the cpu flags in a per-cpu
|
||||
@ -279,6 +282,8 @@ ENTRY(ss_unlock)
|
||||
ss_unlock2:
|
||||
ret
|
||||
|
||||
#endif
|
||||
|
||||
/*
|
||||
* These versions of simple_lock does not contain calls to profiling code.
|
||||
* Thus they can be called from the profiling code.
|
||||
|
@ -848,7 +848,11 @@ ENTRY(i586_copyout)
|
||||
jmp done_copyout
|
||||
#endif /* I586_CPU && NNPX > 0 */
|
||||
|
||||
/* copyin(from_user, to_kernel, len) */
|
||||
/*
|
||||
* copyin(from_user, to_kernel, len)
|
||||
*
|
||||
* MPSAFE
|
||||
*/
|
||||
ENTRY(copyin)
|
||||
MEXITCOUNT
|
||||
jmp *_copyin_vector
|
||||
@ -1130,6 +1134,8 @@ fastmove_tail_fault:
|
||||
|
||||
/*
|
||||
* fu{byte,sword,word} : fetch a byte (sword, word) from user memory
|
||||
*
|
||||
* MP SAFE
|
||||
*/
|
||||
ENTRY(fuword)
|
||||
movl _curpcb,%ecx
|
||||
@ -1154,6 +1160,9 @@ ENTRY(fuswintr)
|
||||
movl $-1,%eax
|
||||
ret
|
||||
|
||||
/*
|
||||
* MP SAFE
|
||||
*/
|
||||
ENTRY(fusword)
|
||||
movl _curpcb,%ecx
|
||||
movl $fusufault,PCB_ONFAULT(%ecx)
|
||||
@ -1166,6 +1175,9 @@ ENTRY(fusword)
|
||||
movl $0,PCB_ONFAULT(%ecx)
|
||||
ret
|
||||
|
||||
/*
|
||||
* MP SAFE
|
||||
*/
|
||||
ENTRY(fubyte)
|
||||
movl _curpcb,%ecx
|
||||
movl $fusufault,PCB_ONFAULT(%ecx)
|
||||
|
@ -65,8 +65,6 @@ _hlt_vector: .long _default_halt /* pointer to halt routine */
|
||||
|
||||
.globl _panic
|
||||
|
||||
.globl _want_resched
|
||||
_want_resched: .long 0 /* we need to re-run the scheduler */
|
||||
#if defined(SWTCH_OPTIM_STATS)
|
||||
.globl _swtch_optim_stats, _tlb_flush_count
|
||||
_swtch_optim_stats: .long 0 /* number of _swtch_optims */
|
||||
@ -129,6 +127,9 @@ _idle:
|
||||
/*
|
||||
* XXX callers of cpu_switch() do a bogus splclock(). Locking should
|
||||
* be left to cpu_switch().
|
||||
*
|
||||
* NOTE: spl*() may only be called while we hold the MP lock (which
|
||||
* we do).
|
||||
*/
|
||||
call _spl0
|
||||
|
||||
@ -159,14 +160,14 @@ idle_loop:
|
||||
testl %eax,%eax
|
||||
jnz 3f
|
||||
|
||||
/*
|
||||
* Handle page-zeroing in the idle loop. Called with interrupts
|
||||
* disabled and the MP lock released. Inside vm_page_zero_idle
|
||||
* we enable interrupts and grab the mplock as required.
|
||||
*/
|
||||
cmpl $0,_do_page_zero_idle
|
||||
je 2f
|
||||
|
||||
/* XXX appears to cause panics */
|
||||
/*
|
||||
* Inside zero_idle we enable interrupts and grab the mplock
|
||||
* as needed. It needs to be careful about entry/exit mutexes.
|
||||
*/
|
||||
call _vm_page_zero_idle /* internal locking */
|
||||
testl %eax, %eax
|
||||
jnz idle_loop
|
||||
@ -178,9 +179,15 @@ idle_loop:
|
||||
cli
|
||||
jmp idle_loop
|
||||
|
||||
/*
|
||||
* Note that interrupts must be enabled while obtaining the MP lock
|
||||
* in order to be able to take IPI's while blocked.
|
||||
*/
|
||||
3:
|
||||
movl $LOPRIO_LEVEL, lapic_tpr /* arbitrate for INTs */
|
||||
sti
|
||||
call _get_mplock
|
||||
cli
|
||||
call _procrunnable
|
||||
testl %eax,%eax
|
||||
CROSSJUMP(jnz, sw1a, jz)
|
||||
@ -355,8 +362,8 @@ sw1a:
|
||||
CROSSJUMP(je, _idle, jne) /* if no proc, idle */
|
||||
movl %eax,%ecx
|
||||
|
||||
movl $0,%eax
|
||||
movl %eax,_want_resched
|
||||
xorl %eax,%eax
|
||||
andl $~WANT_RESCHED,_astpending
|
||||
|
||||
#ifdef DIAGNOSTIC
|
||||
cmpl %eax,P_WCHAN(%ecx)
|
||||
|
@ -53,6 +53,7 @@
|
||||
#include <sys/user.h>
|
||||
|
||||
#include <machine/cpu.h>
|
||||
#include <machine/ipl.h>
|
||||
#include <machine/pcb_ext.h> /* pcb.h included by sys/user.h */
|
||||
#include <machine/sysarch.h>
|
||||
#ifdef SMP
|
||||
|
@ -99,7 +99,7 @@ int (*pmath_emulate) __P((struct trapframe *));
|
||||
|
||||
extern void trap __P((struct trapframe frame));
|
||||
extern int trapwrite __P((unsigned addr));
|
||||
extern void syscall __P((struct trapframe frame));
|
||||
extern void syscall2 __P((struct trapframe frame));
|
||||
|
||||
static int trap_pfault __P((struct trapframe *, int, vm_offset_t));
|
||||
static void trap_fatal __P((struct trapframe *, vm_offset_t));
|
||||
@ -140,38 +140,32 @@ static char *trap_msg[] = {
|
||||
"machine check trap", /* 28 T_MCHK */
|
||||
};
|
||||
|
||||
static __inline void userret __P((struct proc *p, struct trapframe *frame,
|
||||
u_quad_t oticks));
|
||||
static __inline int userret __P((struct proc *p, struct trapframe *frame,
|
||||
u_quad_t oticks, int have_mplock));
|
||||
|
||||
#if defined(I586_CPU) && !defined(NO_F00F_HACK)
|
||||
extern int has_f00f_bug;
|
||||
#endif
|
||||
|
||||
static __inline void
|
||||
userret(p, frame, oticks)
|
||||
static __inline int
|
||||
userret(p, frame, oticks, have_mplock)
|
||||
struct proc *p;
|
||||
struct trapframe *frame;
|
||||
u_quad_t oticks;
|
||||
int have_mplock;
|
||||
{
|
||||
int sig, s;
|
||||
|
||||
while ((sig = CURSIG(p)) != 0)
|
||||
while ((sig = CURSIG(p)) != 0) {
|
||||
if (have_mplock == 0) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
}
|
||||
postsig(sig);
|
||||
|
||||
#if 0
|
||||
if (!want_resched &&
|
||||
(p->p_priority <= p->p_usrpri) &&
|
||||
(p->p_rtprio.type == RTP_PRIO_NORMAL)) {
|
||||
int newpriority;
|
||||
p->p_estcpu += 1;
|
||||
newpriority = PUSER + p->p_estcpu / 4 + 2 * p->p_nice;
|
||||
newpriority = min(newpriority, MAXPRI);
|
||||
p->p_usrpri = newpriority;
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
p->p_priority = p->p_usrpri;
|
||||
if (want_resched) {
|
||||
if (resched_wanted()) {
|
||||
/*
|
||||
* Since we are curproc, clock will normally just change
|
||||
* our priority without moving us from one queue to another
|
||||
@ -180,6 +174,10 @@ userret(p, frame, oticks)
|
||||
* mi_switch()'ed, we might not be on the queue indicated by
|
||||
* our priority.
|
||||
*/
|
||||
if (have_mplock == 0) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
}
|
||||
s = splhigh();
|
||||
setrunqueue(p);
|
||||
p->p_stats->p_ru.ru_nivcsw++;
|
||||
@ -191,11 +189,16 @@ userret(p, frame, oticks)
|
||||
/*
|
||||
* Charge system time if profiling.
|
||||
*/
|
||||
if (p->p_flag & P_PROFIL)
|
||||
if (p->p_flag & P_PROFIL) {
|
||||
if (have_mplock == 0) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
}
|
||||
addupc_task(p, frame->tf_eip,
|
||||
(u_int)(p->p_sticks - oticks) * psratio);
|
||||
|
||||
}
|
||||
curpriority = p->p_priority;
|
||||
return(have_mplock);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -604,7 +607,7 @@ trap(frame)
|
||||
#endif
|
||||
|
||||
out:
|
||||
userret(p, &frame, sticks);
|
||||
userret(p, &frame, sticks, 1);
|
||||
}
|
||||
|
||||
#ifdef notyet
|
||||
@ -999,11 +1002,18 @@ int trapwrite(addr)
|
||||
}
|
||||
|
||||
/*
|
||||
* System call request from POSIX system call gate interface to kernel.
|
||||
* Like trap(), argument is call by reference.
|
||||
* syscall2 - MP aware system call request C handler
|
||||
*
|
||||
* A system call is essentially treated as a trap except that the
|
||||
* MP lock is not held on entry or return. We are responsible for
|
||||
* obtaining the MP lock if necessary and for handling ASTs
|
||||
* (e.g. a task switch) prior to return.
|
||||
*
|
||||
* In general, only simple access and manipulation of curproc and
|
||||
* the current stack is allowed without having to hold MP lock.
|
||||
*/
|
||||
void
|
||||
syscall(frame)
|
||||
syscall2(frame)
|
||||
struct trapframe frame;
|
||||
{
|
||||
caddr_t params;
|
||||
@ -1012,22 +1022,42 @@ syscall(frame)
|
||||
struct proc *p = curproc;
|
||||
u_quad_t sticks;
|
||||
int error;
|
||||
int narg;
|
||||
int args[8];
|
||||
int have_mplock = 0;
|
||||
u_int code;
|
||||
|
||||
#ifdef DIAGNOSTIC
|
||||
if (ISPL(frame.tf_cs) != SEL_UPL)
|
||||
if (ISPL(frame.tf_cs) != SEL_UPL) {
|
||||
get_mplock();
|
||||
panic("syscall");
|
||||
/* NOT REACHED */
|
||||
}
|
||||
#endif
|
||||
sticks = p->p_sticks;
|
||||
|
||||
/*
|
||||
* handle atomicy by looping since interrupts are enabled and the
|
||||
* MP lock is not held.
|
||||
*/
|
||||
sticks = ((volatile struct proc *)p)->p_sticks;
|
||||
while (sticks != ((volatile struct proc *)p)->p_sticks)
|
||||
sticks = ((volatile struct proc *)p)->p_sticks;
|
||||
|
||||
p->p_md.md_regs = &frame;
|
||||
params = (caddr_t)frame.tf_esp + sizeof(int);
|
||||
code = frame.tf_eax;
|
||||
|
||||
if (p->p_sysent->sv_prepsyscall) {
|
||||
/*
|
||||
* The prep code is not MP aware.
|
||||
*/
|
||||
get_mplock();
|
||||
(*p->p_sysent->sv_prepsyscall)(&frame, args, &code, ¶ms);
|
||||
rel_mplock();
|
||||
} else {
|
||||
/*
|
||||
* Need to check if this is a 32 bit or 64 bit syscall.
|
||||
* fuword is MP aware.
|
||||
*/
|
||||
if (code == SYS_syscall) {
|
||||
/*
|
||||
@ -1053,27 +1083,52 @@ syscall(frame)
|
||||
else
|
||||
callp = &p->p_sysent->sv_table[code];
|
||||
|
||||
if (params && (i = callp->sy_narg * sizeof(int)) &&
|
||||
narg = callp->sy_narg & SYF_ARGMASK;
|
||||
|
||||
/*
|
||||
* copyin is MP aware, but the tracing code is not
|
||||
*/
|
||||
if (params && (i = narg * sizeof(int)) &&
|
||||
(error = copyin(params, (caddr_t)args, (u_int)i))) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
#ifdef KTRACE
|
||||
if (KTRPOINT(p, KTR_SYSCALL))
|
||||
ktrsyscall(p->p_tracep, code, callp->sy_narg, args);
|
||||
ktrsyscall(p->p_tracep, code, narg, args);
|
||||
#endif
|
||||
goto bad;
|
||||
}
|
||||
|
||||
/*
|
||||
* Try to run the syscall without the MP lock if the syscall
|
||||
* is MP safe. We have to obtain the MP lock no matter what if
|
||||
* we are ktracing
|
||||
*/
|
||||
if ((callp->sy_narg & SYF_MPSAFE) == 0) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
}
|
||||
|
||||
#ifdef KTRACE
|
||||
if (KTRPOINT(p, KTR_SYSCALL))
|
||||
ktrsyscall(p->p_tracep, code, callp->sy_narg, args);
|
||||
if (KTRPOINT(p, KTR_SYSCALL)) {
|
||||
if (have_mplock == 0) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
}
|
||||
ktrsyscall(p->p_tracep, code, narg, args);
|
||||
}
|
||||
#endif
|
||||
p->p_retval[0] = 0;
|
||||
p->p_retval[1] = frame.tf_edx;
|
||||
|
||||
STOPEVENT(p, S_SCE, callp->sy_narg);
|
||||
STOPEVENT(p, S_SCE, narg); /* MP aware */
|
||||
|
||||
error = (*callp->sy_call)(p, args);
|
||||
|
||||
/*
|
||||
* MP SAFE (we may or may not have the MP lock at this point)
|
||||
*/
|
||||
switch (error) {
|
||||
|
||||
case 0:
|
||||
/*
|
||||
* Reinitialize proc pointer `p' as it may be different
|
||||
@ -1109,17 +1164,31 @@ syscall(frame)
|
||||
break;
|
||||
}
|
||||
|
||||
/*
|
||||
* Traced syscall. trapsignal() is not MP aware.
|
||||
*/
|
||||
if ((frame.tf_eflags & PSL_T) && !(frame.tf_eflags & PSL_VM)) {
|
||||
/* Traced syscall. */
|
||||
if (have_mplock == 0) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
}
|
||||
frame.tf_eflags &= ~PSL_T;
|
||||
trapsignal(p, SIGTRAP, 0);
|
||||
}
|
||||
|
||||
userret(p, &frame, sticks);
|
||||
/*
|
||||
* Handle reschedule and other end-of-syscall issues
|
||||
*/
|
||||
have_mplock = userret(p, &frame, sticks, have_mplock);
|
||||
|
||||
#ifdef KTRACE
|
||||
if (KTRPOINT(p, KTR_SYSRET))
|
||||
if (KTRPOINT(p, KTR_SYSRET)) {
|
||||
if (have_mplock == 0) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
}
|
||||
ktrsysret(p->p_tracep, code, error, p->p_retval[0]);
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
@ -1129,11 +1198,17 @@ syscall(frame)
|
||||
*/
|
||||
STOPEVENT(p, S_SCX, code);
|
||||
|
||||
/*
|
||||
* Release the MP lock if we had to get it
|
||||
*/
|
||||
if (have_mplock)
|
||||
rel_mplock();
|
||||
}
|
||||
|
||||
/*
|
||||
* Simplified back end of syscall(), used when returning from fork()
|
||||
* directly into user mode.
|
||||
* directly into user mode. MP lock is held on entry and should be
|
||||
* held on return.
|
||||
*/
|
||||
void
|
||||
fork_return(p, frame)
|
||||
@ -1144,7 +1219,7 @@ fork_return(p, frame)
|
||||
frame.tf_eflags &= ~PSL_C; /* success */
|
||||
frame.tf_edx = 1;
|
||||
|
||||
userret(p, &frame, 0);
|
||||
userret(p, &frame, 0, 1);
|
||||
#ifdef KTRACE
|
||||
if (KTRPOINT(p, KTR_SYSRET))
|
||||
ktrsysret(p->p_tracep, SYS_fork, 0, 0);
|
||||
|
@ -64,7 +64,7 @@ ENTRY(vm86_bioscall)
|
||||
|
||||
#ifdef SMP
|
||||
pushl %edx
|
||||
ALIGN_LOCK /* Get global lock */
|
||||
MP_LOCK /* Get global lock */
|
||||
popl %edx
|
||||
#endif
|
||||
|
||||
@ -136,12 +136,7 @@ ENTRY(vm86_bioscall)
|
||||
* Return via _doreti
|
||||
*/
|
||||
#ifdef SMP
|
||||
ECPL_LOCK
|
||||
#ifdef CPL_AND_CML
|
||||
#error Not ready for CPL_AND_CML
|
||||
#endif
|
||||
pushl _cpl /* cpl to restore */
|
||||
ECPL_UNLOCK
|
||||
#else
|
||||
pushl _cpl /* cpl to restore */
|
||||
#endif
|
||||
|
@ -578,13 +578,7 @@ vm_page_zero_idle()
|
||||
TAILQ_REMOVE(&vm_page_queues[m->queue].pl, m, pageq);
|
||||
m->queue = PQ_NONE;
|
||||
splx(s);
|
||||
#if 0
|
||||
rel_mplock();
|
||||
#endif
|
||||
pmap_zero_page(VM_PAGE_TO_PHYS(m));
|
||||
#if 0
|
||||
get_mplock();
|
||||
#endif
|
||||
(void)splvm();
|
||||
vm_page_flag_set(m, PG_ZERO);
|
||||
m->queue = PQ_FREE + m->pc;
|
||||
@ -606,6 +600,12 @@ vm_page_zero_idle()
|
||||
#ifdef SMP
|
||||
}
|
||||
#endif
|
||||
/*
|
||||
* We have to enable interrupts for a moment if the try_mplock fails
|
||||
* in order to potentially take an IPI. XXX this should be in
|
||||
* swtch.s
|
||||
*/
|
||||
__asm __volatile("sti; nop; cli" : : : "memory");
|
||||
return (0);
|
||||
}
|
||||
|
||||
|
@ -155,7 +155,6 @@
|
||||
#define _arith_invalid arith_invalid
|
||||
#define _arith_overflow arith_overflow
|
||||
#define _arith_underflow arith_underflow
|
||||
#define _astpending astpending
|
||||
#define _bcopy bcopy
|
||||
#define _bcopy_vector bcopy_vector
|
||||
#define _bigJump bigJump
|
||||
@ -307,7 +306,7 @@
|
||||
#define _swi_generic swi_generic
|
||||
#define _swi_null swi_null
|
||||
#define _swi_vm swi_vm
|
||||
#define _syscall syscall
|
||||
#define _syscall2 syscall2
|
||||
#define _szsigcode szsigcode
|
||||
#define _ticks ticks
|
||||
#define _time time
|
||||
@ -321,7 +320,6 @@
|
||||
#define _vm86paddr vm86paddr
|
||||
#define _vm86pcb vm86pcb
|
||||
#define _vm_page_zero_idle vm_page_zero_idle
|
||||
#define _want_resched want_resched
|
||||
#define _wm_sqrt wm_sqrt
|
||||
|
||||
#endif /* __ELF__ */
|
||||
@ -339,6 +337,7 @@
|
||||
#define _cpu_lockid FS(cpu_lockid)
|
||||
#define _curpcb FS(curpcb)
|
||||
#define _curproc FS(curproc)
|
||||
#define _astpending FS(astpending)
|
||||
#define _currentldt FS(currentldt)
|
||||
#define _inside_intr FS(inside_intr)
|
||||
#define _npxproc FS(npxproc)
|
||||
|
@ -82,10 +82,13 @@
|
||||
/*
|
||||
* Preempt the current process if in interrupt from user mode,
|
||||
* or after the current trap/syscall if in system mode.
|
||||
*
|
||||
* XXX: if astpending is later changed to an |= here due to more flags being
|
||||
* added, we will have an atomicy problem. The type of atomicy we need is
|
||||
* a non-locked orl.
|
||||
*/
|
||||
#define need_resched() do { want_resched = 1; aston(); } while (0)
|
||||
|
||||
#define resched_wanted() want_resched
|
||||
#define need_resched() do { astpending = AST_RESCHED|AST_PENDING; } while (0)
|
||||
#define resched_wanted() (astpending & AST_RESCHED)
|
||||
|
||||
/*
|
||||
* Arrange to handle pending profiling ticks before returning to user mode.
|
||||
@ -100,10 +103,15 @@
|
||||
/*
|
||||
* Notify the current process (p) that it has a signal pending,
|
||||
* process as soon as possible.
|
||||
*
|
||||
* XXX: aston() really needs to be an atomic (not locked, but an orl),
|
||||
* in case need_resched() is set by an interrupt. But with astpending a
|
||||
* per-cpu variable this is not trivial to do efficiently. For now we blow
|
||||
* it off (asynchronous need_resched() conflicts are not critical).
|
||||
*/
|
||||
#define signotify(p) aston()
|
||||
|
||||
#define aston() do { astpending = 1; } while (0)
|
||||
#define aston() do { astpending |= AST_PENDING; } while (0)
|
||||
#define astoff()
|
||||
|
||||
/*
|
||||
@ -126,11 +134,9 @@
|
||||
}
|
||||
|
||||
#ifdef _KERNEL
|
||||
extern int astpending;
|
||||
extern char btext[];
|
||||
extern char etext[];
|
||||
extern u_char intr_nesting_level;
|
||||
extern int want_resched; /* resched was called */
|
||||
|
||||
void fork_trampoline __P((void));
|
||||
void fork_return __P((struct proc *, struct trapframe));
|
||||
|
@ -66,6 +66,7 @@ struct globaldata {
|
||||
caddr_t gd_prv_CADDR3;
|
||||
unsigned *gd_prv_PADDR1;
|
||||
#endif
|
||||
u_int gd_astpending;
|
||||
};
|
||||
|
||||
#ifdef SMP
|
||||
|
@ -90,6 +90,7 @@
|
||||
|
||||
#define common_tssd GLOBAL_LVALUE(common_tssd, struct segment_descriptor)
|
||||
#define tss_gdt GLOBAL_LVALUE(tss_gdt, struct segment_descriptor *)
|
||||
#define astpending GLOBAL_LVALUE(astpending, u_int)
|
||||
|
||||
#ifdef USER_LDT
|
||||
#define currentldt GLOBAL_LVALUE(currentldt, int)
|
||||
@ -111,6 +112,7 @@
|
||||
#endif /*UP kernel*/
|
||||
|
||||
GLOBAL_FUNC(curproc)
|
||||
GLOBAL_FUNC(astpending)
|
||||
GLOBAL_FUNC(curpcb)
|
||||
GLOBAL_FUNC(npxproc)
|
||||
GLOBAL_FUNC(common_tss)
|
||||
|
@ -85,6 +85,12 @@
|
||||
#define SWI_CLOCK_MASK SWI_CLOCK_PENDING
|
||||
#define SWI_MASK (~HWI_MASK)
|
||||
|
||||
/*
|
||||
* astpending bits
|
||||
*/
|
||||
#define AST_PENDING 0x00000001
|
||||
#define AST_RESCHED 0x00000002
|
||||
|
||||
#ifndef LOCORE
|
||||
|
||||
/*
|
||||
|
@ -37,24 +37,17 @@
|
||||
#define MPLOCKED lock ;
|
||||
|
||||
/*
|
||||
* Some handy macros to allow logical organization and
|
||||
* convenient reassignment of various locks.
|
||||
* Some handy macros to allow logical organization.
|
||||
*/
|
||||
|
||||
#define FPU_LOCK call _get_fpu_lock
|
||||
#define ALIGN_LOCK call _get_align_lock
|
||||
#define SYSCALL_LOCK call _get_syscall_lock
|
||||
#define ALTSYSCALL_LOCK call _get_altsyscall_lock
|
||||
#define MP_LOCK call _get_mplock
|
||||
|
||||
/*
|
||||
* Protects INTR() ISRs.
|
||||
*/
|
||||
#define ISR_TRYLOCK \
|
||||
#define MP_TRYLOCK \
|
||||
pushl $_mp_lock ; /* GIANT_LOCK */ \
|
||||
call _MPtrylock ; /* try to get lock */ \
|
||||
add $4, %esp
|
||||
|
||||
#define ISR_RELLOCK \
|
||||
#define MP_RELLOCK \
|
||||
movl $_mp_lock,%edx ; /* GIANT_LOCK */ \
|
||||
call _MPrellock_edx
|
||||
|
||||
@ -69,55 +62,11 @@
|
||||
#define IMASK_UNLOCK \
|
||||
movl $0, _imen_lock
|
||||
|
||||
/*
|
||||
* Variations of CPL_LOCK protect spl updates as a critical region.
|
||||
* Items within this 'region' include:
|
||||
* cpl
|
||||
* cml
|
||||
* cil
|
||||
* ipending
|
||||
*/
|
||||
|
||||
/*
|
||||
* Bottom half routines, ie. those already protected from INTs.
|
||||
*
|
||||
* Used in:
|
||||
* sys/i386/isa/ipl.s: _doreti
|
||||
* sys/i386/isa/apic_vector.s: _Xintr0, ..., _Xintr23
|
||||
*/
|
||||
#define CPL_LOCK \
|
||||
pushl $_cpl_lock ; /* address of lock */ \
|
||||
call _s_lock ; /* MP-safe */ \
|
||||
addl $4, %esp
|
||||
|
||||
#define CPL_UNLOCK \
|
||||
movl $0, _cpl_lock
|
||||
|
||||
/*
|
||||
* INT safe version for top half of kernel.
|
||||
*
|
||||
* Used in:
|
||||
* sys/i386/i386/exception.s: _Xfpu, _Xalign, _Xsyscall, _Xint0x80_syscall
|
||||
* sys/i386/isa/apic_ipl.s: splz()
|
||||
*/
|
||||
#define SCPL_LOCK \
|
||||
pushl $_cpl_lock ; \
|
||||
call _ss_lock ; \
|
||||
addl $4, %esp
|
||||
|
||||
#define SCPL_UNLOCK \
|
||||
pushl $_cpl_lock ; \
|
||||
call _ss_unlock ; \
|
||||
addl $4, %esp
|
||||
|
||||
#else /* SMP */
|
||||
|
||||
#define MPLOCKED /* NOP */
|
||||
|
||||
#define FPU_LOCK /* NOP */
|
||||
#define ALIGN_LOCK /* NOP */
|
||||
#define SYSCALL_LOCK /* NOP */
|
||||
#define ALTSYSCALL_LOCK /* NOP */
|
||||
#define MP_LOCK /* NOP */
|
||||
|
||||
#endif /* SMP */
|
||||
|
||||
@ -138,18 +87,6 @@
|
||||
#define MPINTR_UNLOCK()
|
||||
#endif /* USE_MPINTRLOCK */
|
||||
|
||||
/*
|
||||
* Protects cpl/cml/cil/ipending data as a critical region.
|
||||
*
|
||||
* Used in:
|
||||
* sys/i386/isa/ipl_funcs.c: DO_SETBITS, softclockpending(), GENSPL,
|
||||
* spl0(), splx(), splq()
|
||||
*/
|
||||
#define CPL_LOCK() s_lock(&cpl_lock) /* Bottom end */
|
||||
#define CPL_UNLOCK() s_unlock(&cpl_lock)
|
||||
#define SCPL_LOCK() ss_lock(&cpl_lock) /* INT safe: top end */
|
||||
#define SCPL_UNLOCK() ss_unlock(&cpl_lock)
|
||||
|
||||
/*
|
||||
* sio/cy lock.
|
||||
* XXX should rc (RISCom/8) use this?
|
||||
@ -191,11 +128,6 @@
|
||||
#define MPINTR_LOCK()
|
||||
#define MPINTR_UNLOCK()
|
||||
|
||||
#define CPL_LOCK()
|
||||
#define CPL_UNLOCK()
|
||||
#define SCPL_LOCK()
|
||||
#define SCPL_UNLOCK()
|
||||
|
||||
#define COM_LOCK()
|
||||
#define COM_UNLOCK()
|
||||
#define CLOCK_LOCK()
|
||||
|
@ -1770,8 +1770,10 @@ init_locks(void)
|
||||
*/
|
||||
mp_lock = 0x00000001;
|
||||
|
||||
#if 0
|
||||
/* ISR uses its own "giant lock" */
|
||||
isr_lock = FREE_LOCK;
|
||||
#endif
|
||||
|
||||
#if defined(APIC_INTR_DIAGNOSTIC) && defined(APIC_INTR_DIAGNOSTIC_IRQ)
|
||||
s_lock_init((struct simplelock*)&apic_itrace_debuglock);
|
||||
|
@ -66,6 +66,7 @@ struct globaldata {
|
||||
caddr_t gd_prv_CADDR3;
|
||||
unsigned *gd_prv_PADDR1;
|
||||
#endif
|
||||
u_int gd_astpending;
|
||||
};
|
||||
|
||||
#ifdef SMP
|
||||
|
@ -63,65 +63,6 @@
|
||||
#define PUSHDOWN_LEVEL_3_NOT
|
||||
#define PUSHDOWN_LEVEL_4_NOT
|
||||
|
||||
|
||||
/*
|
||||
* XXX some temp debug control of cpl locks
|
||||
*/
|
||||
#ifdef PUSHDOWN_LEVEL_2
|
||||
#define REAL_ECPL /* exception.s: SCPL_LOCK/SCPL_UNLOCK */
|
||||
#define REAL_ICPL /* ipl.s: CPL_LOCK/CPL_UNLOCK/FAST */
|
||||
#define REAL_AICPL /* apic_ipl.s: SCPL_LOCK/SCPL_UNLOCK */
|
||||
#define REAL_AVCPL /* apic_vector.s: CPL_LOCK/CPL_UNLOCK */
|
||||
#define REAL_IFCPL /* ipl_funcs.c: SCPL_LOCK/SCPL_UNLOCK */
|
||||
#endif /* PUSHDOWN_LEVEL_2 */
|
||||
|
||||
/*
|
||||
* The xCPL_LOCK/xCPL_UNLOCK defines control the spinlocks
|
||||
* that protect cpl/cml/cil and the spl functions.
|
||||
*/
|
||||
#ifdef REAL_ECPL
|
||||
#define ECPL_LOCK SCPL_LOCK
|
||||
#define ECPL_UNLOCK SCPL_UNLOCK
|
||||
#else
|
||||
#define ECPL_LOCK
|
||||
#define ECPL_UNLOCK
|
||||
#endif /* REAL_ECPL */
|
||||
|
||||
#ifdef REAL_ICPL
|
||||
#define ICPL_LOCK CPL_LOCK
|
||||
#define ICPL_UNLOCK CPL_UNLOCK
|
||||
#define FAST_ICPL_UNLOCK movl $0, _cpl_lock
|
||||
#else
|
||||
#define ICPL_LOCK
|
||||
#define ICPL_UNLOCK
|
||||
#define FAST_ICPL_UNLOCK
|
||||
#endif /* REAL_ICPL */
|
||||
|
||||
#ifdef REAL_AICPL
|
||||
#define AICPL_LOCK SCPL_LOCK
|
||||
#define AICPL_UNLOCK SCPL_UNLOCK
|
||||
#else
|
||||
#define AICPL_LOCK
|
||||
#define AICPL_UNLOCK
|
||||
#endif /* REAL_AICPL */
|
||||
|
||||
#ifdef REAL_AVCPL
|
||||
#define AVCPL_LOCK CPL_LOCK
|
||||
#define AVCPL_UNLOCK CPL_UNLOCK
|
||||
#else
|
||||
#define AVCPL_LOCK
|
||||
#define AVCPL_UNLOCK
|
||||
#endif /* REAL_AVCPL */
|
||||
|
||||
#ifdef REAL_IFCPL
|
||||
#define IFCPL_LOCK() SCPL_LOCK()
|
||||
#define IFCPL_UNLOCK() SCPL_UNLOCK()
|
||||
#else
|
||||
#define IFCPL_LOCK()
|
||||
#define IFCPL_UNLOCK()
|
||||
#endif /* REAL_IFCPL */
|
||||
|
||||
|
||||
/*
|
||||
* Debug version of simple_lock. This will store the CPU id of the
|
||||
* holding CPU along with the lock. When a CPU fails to get the lock
|
||||
@ -160,36 +101,40 @@
|
||||
|
||||
|
||||
/*
|
||||
* Regular INTerrupts without the giant lock, NOT READY YET!!!
|
||||
* INTR_SIMPLELOCK has been removed, as the interrupt mechanism will likely
|
||||
* not use this sort of optimization if we move to interrupt threads.
|
||||
*/
|
||||
#ifdef PUSHDOWN_LEVEL_4
|
||||
#define INTR_SIMPLELOCK
|
||||
#endif
|
||||
|
||||
|
||||
/*
|
||||
* Separate the INTR() portion of cpl into another variable: cml.
|
||||
* CPL_AND_CML has been removed. Interrupt threads will eventually not
|
||||
* use either mechanism so there is no point trying to optimize it.
|
||||
*/
|
||||
#ifdef PUSHDOWN_LEVEL_3
|
||||
#define CPL_AND_CML
|
||||
#endif
|
||||
|
||||
|
||||
/*
|
||||
* Forces spl functions to spin while waiting for safe time to change cpl.
|
||||
* SPL_DEBUG_POSTCODE/INTR_SPL/SPL_DEBUG - removed
|
||||
*
|
||||
#define SPL_DEBUG_POSTCODE (slows the system down noticably)
|
||||
* These functions were too expensive for the standard case but, more
|
||||
* importantly, we should be able to come up with a much cleaner way
|
||||
* to handle the cpl. Having to do any locking at all is a mistake
|
||||
* for something that is modified as often as cpl is.
|
||||
*/
|
||||
#ifdef PUSHDOWN_LEVEL_3
|
||||
#define INTR_SPL
|
||||
#define SPL_DEBUG
|
||||
#endif
|
||||
|
||||
|
||||
/*
|
||||
* FAST_WITHOUTCPL - now made the default (define removed). Text below
|
||||
* contains the current discussion. I am confident we can find a solution
|
||||
* that does not require us to process softints from a hard int, which can
|
||||
* kill serial performance due to the lack of true hardware ipl's.
|
||||
*
|
||||
****
|
||||
*
|
||||
* Ignore the ipending bits when exiting FAST_INTR() routines.
|
||||
*
|
||||
***
|
||||
* according to Bruce:
|
||||
*
|
||||
* setsoft*() may set ipending. setsofttty() is actually used in the
|
||||
@ -209,21 +154,17 @@
|
||||
* I finish making spl/cpl MP-safe.
|
||||
*/
|
||||
#ifdef PUSHDOWN_LEVEL_1
|
||||
#define FAST_WITHOUTCPL
|
||||
#endif
|
||||
|
||||
|
||||
/*
|
||||
* Use a simplelock to serialize FAST_INTR()s.
|
||||
* sio.c, and probably other FAST_INTR() drivers, never expected several CPUs
|
||||
* to be inside them at once. Things such as global vars prevent more
|
||||
* than 1 thread of execution from existing at once, so we serialize
|
||||
* the access of FAST_INTR()s via a simple lock.
|
||||
* One optimization on this would be a simple lock per DRIVER, but I'm
|
||||
* not sure how to organize that yet...
|
||||
* FAST_SIMPLELOCK no longer exists, because it doesn't help us. The cpu
|
||||
* is likely to already hold the MP lock and recursive MP locks are now
|
||||
* very cheap, so we do not need this optimization. Eventually *ALL*
|
||||
* interrupts will run in their own thread, so there is no sense complicating
|
||||
* matters now.
|
||||
*/
|
||||
#ifdef PUSHDOWN_LEVEL_1
|
||||
#define FAST_SIMPLELOCK
|
||||
#endif
|
||||
|
||||
|
||||
|
@ -29,15 +29,6 @@
|
||||
.data
|
||||
ALIGN_DATA
|
||||
|
||||
/* current INTerrupt level */
|
||||
.globl _cil
|
||||
_cil: .long 0
|
||||
|
||||
/* current INTerrupt level mask */
|
||||
.globl _cml
|
||||
_cml: .long 0
|
||||
|
||||
|
||||
/*
|
||||
* Routines used by splz_unpend to build an interrupt frame from a
|
||||
* trap frame. The _vec[] routines build the proper frame on the stack,
|
||||
@ -78,6 +69,8 @@ _apic_imen:
|
||||
SUPERALIGN_TEXT
|
||||
|
||||
/*
|
||||
* splz() - dispatch pending interrupts after cpl reduced
|
||||
*
|
||||
* Interrupt priority mechanism
|
||||
* -- soft splXX masks with group mechanism (cpl)
|
||||
* -- h/w masks for currently active or unused interrupts (imen)
|
||||
@ -87,20 +80,25 @@ _apic_imen:
|
||||
ENTRY(splz)
|
||||
/*
|
||||
* The caller has restored cpl and checked that (ipending & ~cpl)
|
||||
* is nonzero. We have to repeat the check since if there is an
|
||||
* interrupt while we're looking, _doreti processing for the
|
||||
* interrupt will handle all the unmasked pending interrupts
|
||||
* because we restored early. We're repeating the calculation
|
||||
* of (ipending & ~cpl) anyway so that the caller doesn't have
|
||||
* to pass it, so this only costs one "jne". "bsfl %ecx,%ecx"
|
||||
* is undefined when %ecx is 0 so we can't rely on the secondary
|
||||
* btrl tests.
|
||||
* is nonzero. However, since ipending can change at any time
|
||||
* (by an interrupt or, with SMP, by another cpu), we have to
|
||||
* repeat the check. At the moment we must own the MP lock in
|
||||
* the SMP case because the interruput handlers require it. We
|
||||
* loop until no unmasked pending interrupts remain.
|
||||
*
|
||||
* No new unmaksed pending interrupts will be added during the
|
||||
* loop because, being unmasked, the interrupt code will be able
|
||||
* to execute the interrupts.
|
||||
*
|
||||
* Interrupts come in two flavors: Hardware interrupts and software
|
||||
* interrupts. We have to detect the type of interrupt (based on the
|
||||
* position of the interrupt bit) and call the appropriate dispatch
|
||||
* routine.
|
||||
*
|
||||
* NOTE: "bsfl %ecx,%ecx" is undefined when %ecx is 0 so we can't
|
||||
* rely on the secondary btrl tests.
|
||||
*/
|
||||
AICPL_LOCK
|
||||
movl _cpl,%eax
|
||||
#ifdef CPL_AND_CML
|
||||
orl _cml, %eax /* add cml to cpl */
|
||||
#endif
|
||||
splz_next:
|
||||
/*
|
||||
* We don't need any locking here. (ipending & ~cpl) cannot grow
|
||||
@ -110,7 +108,6 @@ splz_next:
|
||||
notl %ecx /* set bit = unmasked level */
|
||||
andl _ipending,%ecx /* set bit = unmasked pending INT */
|
||||
jne splz_unpend
|
||||
AICPL_UNLOCK
|
||||
ret
|
||||
|
||||
ALIGN_TEXT
|
||||
@ -131,9 +128,6 @@ splz_unpend:
|
||||
* The vec[] routines build the proper frame on the stack,
|
||||
* then call one of _Xintr0 thru _XintrNN.
|
||||
*/
|
||||
pushl %ecx
|
||||
AICPL_UNLOCK
|
||||
popl %ecx
|
||||
jmp *_vec(,%ecx,4)
|
||||
|
||||
ALIGN_TEXT
|
||||
@ -141,11 +135,7 @@ splz_swi:
|
||||
pushl %eax
|
||||
orl imasks(,%ecx,4),%eax
|
||||
movl %eax,_cpl
|
||||
pushl %ecx
|
||||
AICPL_UNLOCK
|
||||
popl %ecx
|
||||
call *_ihandlers(,%ecx,4)
|
||||
AICPL_LOCK
|
||||
popl %eax
|
||||
movl %eax,_cpl
|
||||
jmp splz_next
|
||||
|
@ -9,28 +9,17 @@
|
||||
|
||||
#include "i386/isa/intr_machdep.h"
|
||||
|
||||
|
||||
#ifdef FAST_SIMPLELOCK
|
||||
/*
|
||||
* Interrupts must be enabled while waiting for the MP lock.
|
||||
*/
|
||||
|
||||
#define GET_FAST_INTR_LOCK \
|
||||
pushl $_fast_intr_lock ; /* address of lock */ \
|
||||
call _s_lock ; /* MP-safe */ \
|
||||
addl $4,%esp
|
||||
|
||||
#define REL_FAST_INTR_LOCK \
|
||||
movl $0, _fast_intr_lock
|
||||
|
||||
#else /* FAST_SIMPLELOCK */
|
||||
|
||||
#define GET_FAST_INTR_LOCK \
|
||||
call _get_isrlock
|
||||
sti; call _get_mplock; cli
|
||||
|
||||
#define REL_FAST_INTR_LOCK \
|
||||
movl $_mp_lock, %edx ; /* GIANT_LOCK */ \
|
||||
call _MPrellock_edx
|
||||
|
||||
#endif /* FAST_SIMPLELOCK */
|
||||
|
||||
/* convert an absolute IRQ# into a bitmask */
|
||||
#define IRQ_BIT(irq_num) (1 << (irq_num))
|
||||
|
||||
@ -42,10 +31,6 @@
|
||||
* Macros for interrupt interrupt entry, call to handler, and exit.
|
||||
*/
|
||||
|
||||
#ifdef FAST_WITHOUTCPL
|
||||
|
||||
/*
|
||||
*/
|
||||
#define FAST_INTR(irq_num, vec_name) \
|
||||
.text ; \
|
||||
SUPERALIGN_TEXT ; \
|
||||
@ -82,83 +67,6 @@ IDTVEC(vec_name) ; \
|
||||
popl %eax ; \
|
||||
iret
|
||||
|
||||
#else /* FAST_WITHOUTCPL */
|
||||
|
||||
#define FAST_INTR(irq_num, vec_name) \
|
||||
.text ; \
|
||||
SUPERALIGN_TEXT ; \
|
||||
IDTVEC(vec_name) ; \
|
||||
pushl %eax ; /* save only call-used registers */ \
|
||||
pushl %ecx ; \
|
||||
pushl %edx ; \
|
||||
pushl %ds ; \
|
||||
MAYBE_PUSHL_ES ; \
|
||||
pushl %fs ; \
|
||||
movl $KDSEL, %eax ; \
|
||||
movl %ax, %ds ; \
|
||||
MAYBE_MOVW_AX_ES ; \
|
||||
movl $KPSEL, %eax ; \
|
||||
movl %ax, %fs ; \
|
||||
FAKE_MCOUNT((5+ACTUALLY_PUSHED)*4(%esp)) ; \
|
||||
GET_FAST_INTR_LOCK ; \
|
||||
pushl _intr_unit + (irq_num) * 4 ; \
|
||||
call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
|
||||
addl $4, %esp ; \
|
||||
movl $0, lapic_eoi ; \
|
||||
lock ; \
|
||||
incl _cnt+V_INTR ; /* book-keeping can wait */ \
|
||||
movl _intr_countp + (irq_num) * 4,%eax ; \
|
||||
lock ; \
|
||||
incl (%eax) ; \
|
||||
movl _cpl, %eax ; /* unmasking pending HWIs or SWIs? */ \
|
||||
notl %eax ; \
|
||||
andl _ipending, %eax ; \
|
||||
jne 2f ; /* yes, maybe handle them */ \
|
||||
1: ; \
|
||||
MEXITCOUNT ; \
|
||||
REL_FAST_INTR_LOCK ; \
|
||||
popl %fs ; \
|
||||
MAYBE_POPL_ES ; \
|
||||
popl %ds ; \
|
||||
popl %edx ; \
|
||||
popl %ecx ; \
|
||||
popl %eax ; \
|
||||
iret ; \
|
||||
; \
|
||||
ALIGN_TEXT ; \
|
||||
2: ; \
|
||||
cmpb $3, _intr_nesting_level ; /* enough stack? */ \
|
||||
jae 1b ; /* no, return */ \
|
||||
movl _cpl, %eax ; \
|
||||
/* XXX next line is probably unnecessary now. */ \
|
||||
movl $HWI_MASK|SWI_MASK, _cpl ; /* limit nesting ... */ \
|
||||
lock ; \
|
||||
incb _intr_nesting_level ; /* ... really limit it ... */ \
|
||||
sti ; /* to do this as early as possible */ \
|
||||
popl %fs ; /* discard most of thin frame ... */ \
|
||||
MAYBE_POPL_ES ; /* discard most of thin frame ... */ \
|
||||
popl %ecx ; /* ... original %ds ... */ \
|
||||
popl %edx ; \
|
||||
xchgl %eax, 4(%esp) ; /* orig %eax; save cpl */ \
|
||||
pushal ; /* build fat frame (grrr) ... */ \
|
||||
pushl %ecx ; /* ... actually %ds ... */ \
|
||||
pushl %es ; \
|
||||
pushl %fs ;
|
||||
movl $KDSEL, %eax ; \
|
||||
movl %ax, %es ; \
|
||||
movl $KPSEL, %eax ;
|
||||
movl %ax, %fs ;
|
||||
movl (3+8+0)*4(%esp), %ecx ; /* %ecx from thin frame ... */ \
|
||||
movl %ecx, (3+6)*4(%esp) ; /* ... to fat frame ... */ \
|
||||
movl (3+8+1)*4(%esp), %eax ; /* ... cpl from thin frame */ \
|
||||
pushl %eax ; \
|
||||
subl $4, %esp ; /* junk for unit number */ \
|
||||
MEXITCOUNT ; \
|
||||
jmp _doreti
|
||||
|
||||
#endif /** FAST_WITHOUTCPL */
|
||||
|
||||
|
||||
/*
|
||||
*
|
||||
*/
|
||||
@ -242,19 +150,6 @@ IDTVEC(vec_name) ; \
|
||||
7: ; \
|
||||
IMASK_UNLOCK
|
||||
|
||||
#ifdef INTR_SIMPLELOCK
|
||||
#define ENLOCK
|
||||
#define DELOCK
|
||||
#define LATELOCK call _get_isrlock
|
||||
#else
|
||||
#define ENLOCK \
|
||||
ISR_TRYLOCK ; /* XXX this is going away... */ \
|
||||
testl %eax, %eax ; /* did we get it? */ \
|
||||
jz 3f
|
||||
#define DELOCK ISR_RELLOCK
|
||||
#define LATELOCK
|
||||
#endif
|
||||
|
||||
#ifdef APIC_INTR_DIAGNOSTIC
|
||||
#ifdef APIC_INTR_DIAGNOSTIC_IRQ
|
||||
log_intr_event:
|
||||
@ -319,8 +214,6 @@ log_intr_event:
|
||||
#define APIC_ITRACE(name, irq_num, id)
|
||||
#endif
|
||||
|
||||
#ifdef CPL_AND_CML
|
||||
|
||||
#define INTR(irq_num, vec_name, maybe_extra_ipending) \
|
||||
.text ; \
|
||||
SUPERALIGN_TEXT ; \
|
||||
@ -344,132 +237,13 @@ IDTVEC(vec_name) ; \
|
||||
EOI_IRQ(irq_num) ; \
|
||||
0: ; \
|
||||
APIC_ITRACE(apic_itrace_tryisrlock, irq_num, APIC_ITRACE_TRYISRLOCK) ;\
|
||||
ENLOCK ; \
|
||||
; \
|
||||
APIC_ITRACE(apic_itrace_gotisrlock, irq_num, APIC_ITRACE_GOTISRLOCK) ;\
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
testl $IRQ_BIT(irq_num), _cpl ; \
|
||||
jne 2f ; /* this INT masked */ \
|
||||
testl $IRQ_BIT(irq_num), _cml ; \
|
||||
jne 2f ; /* this INT masked */ \
|
||||
orl $IRQ_BIT(irq_num), _cil ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
; \
|
||||
incb _intr_nesting_level ; \
|
||||
; \
|
||||
/* entry point used by doreti_unpend for HWIs. */ \
|
||||
__CONCAT(Xresume,irq_num): ; \
|
||||
FAKE_MCOUNT(13*4(%esp)) ; /* XXX avoid dbl cnt */ \
|
||||
lock ; incl _cnt+V_INTR ; /* tally interrupts */ \
|
||||
movl _intr_countp + (irq_num) * 4, %eax ; \
|
||||
lock ; incl (%eax) ; \
|
||||
; \
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
movl _cml, %eax ; \
|
||||
pushl %eax ; \
|
||||
orl _intr_mask + (irq_num) * 4, %eax ; \
|
||||
movl %eax, _cml ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
; \
|
||||
pushl _intr_unit + (irq_num) * 4 ; \
|
||||
incl _inside_intr ; \
|
||||
APIC_ITRACE(apic_itrace_enter2, irq_num, APIC_ITRACE_ENTER2) ; \
|
||||
sti ; \
|
||||
call *_intr_handler + (irq_num) * 4 ; \
|
||||
cli ; \
|
||||
APIC_ITRACE(apic_itrace_leave, irq_num, APIC_ITRACE_LEAVE) ; \
|
||||
decl _inside_intr ; \
|
||||
; \
|
||||
lock ; andl $~IRQ_BIT(irq_num), iactive ; \
|
||||
lock ; andl $~IRQ_BIT(irq_num), _cil ; \
|
||||
UNMASK_IRQ(irq_num) ; \
|
||||
APIC_ITRACE(apic_itrace_unmask, irq_num, APIC_ITRACE_UNMASK) ; \
|
||||
sti ; /* doreti repeats cli/sti */ \
|
||||
MEXITCOUNT ; \
|
||||
LATELOCK ; \
|
||||
jmp _doreti ; \
|
||||
; \
|
||||
ALIGN_TEXT ; \
|
||||
1: ; /* active */ \
|
||||
APIC_ITRACE(apic_itrace_active, irq_num, APIC_ITRACE_ACTIVE) ; \
|
||||
MASK_IRQ(irq_num) ; \
|
||||
EOI_IRQ(irq_num) ; \
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
lock ; \
|
||||
orl $IRQ_BIT(irq_num), _ipending ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
lock ; \
|
||||
btsl $(irq_num), iactive ; /* still active */ \
|
||||
jnc 0b ; /* retry */ \
|
||||
POP_FRAME ; \
|
||||
iret ; \
|
||||
; \
|
||||
ALIGN_TEXT ; \
|
||||
2: ; /* masked by cpl|cml */ \
|
||||
APIC_ITRACE(apic_itrace_masked, irq_num, APIC_ITRACE_MASKED) ; \
|
||||
lock ; \
|
||||
orl $IRQ_BIT(irq_num), _ipending ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
DELOCK ; /* XXX this is going away... */ \
|
||||
POP_FRAME ; \
|
||||
iret ; \
|
||||
ALIGN_TEXT ; \
|
||||
3: ; /* other cpu has isr lock */ \
|
||||
APIC_ITRACE(apic_itrace_noisrlock, irq_num, APIC_ITRACE_NOISRLOCK) ;\
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
lock ; \
|
||||
orl $IRQ_BIT(irq_num), _ipending ; \
|
||||
testl $IRQ_BIT(irq_num), _cpl ; \
|
||||
jne 4f ; /* this INT masked */ \
|
||||
testl $IRQ_BIT(irq_num), _cml ; \
|
||||
jne 4f ; /* this INT masked */ \
|
||||
orl $IRQ_BIT(irq_num), _cil ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
call forward_irq ; /* forward irq to lock holder */ \
|
||||
POP_FRAME ; /* and return */ \
|
||||
iret ; \
|
||||
ALIGN_TEXT ; \
|
||||
4: ; /* blocked */ \
|
||||
APIC_ITRACE(apic_itrace_masked2, irq_num, APIC_ITRACE_MASKED2) ;\
|
||||
AVCPL_UNLOCK ; \
|
||||
POP_FRAME ; /* and return */ \
|
||||
iret
|
||||
|
||||
#else /* CPL_AND_CML */
|
||||
|
||||
|
||||
#define INTR(irq_num, vec_name, maybe_extra_ipending) \
|
||||
.text ; \
|
||||
SUPERALIGN_TEXT ; \
|
||||
/* _XintrNN: entry point used by IDT/HWIs & splz_unpend via _vec[]. */ \
|
||||
IDTVEC(vec_name) ; \
|
||||
PUSH_FRAME ; \
|
||||
movl $KDSEL, %eax ; /* reload with kernel's data segment */ \
|
||||
movl %ax, %ds ; \
|
||||
movl %ax, %es ; \
|
||||
movl $KPSEL, %eax ; \
|
||||
movl %ax, %fs ; \
|
||||
; \
|
||||
maybe_extra_ipending ; \
|
||||
; \
|
||||
APIC_ITRACE(apic_itrace_enter, irq_num, APIC_ITRACE_ENTER) ; \
|
||||
lock ; /* MP-safe */ \
|
||||
btsl $(irq_num), iactive ; /* lazy masking */ \
|
||||
jc 1f ; /* already active */ \
|
||||
; \
|
||||
MASK_LEVEL_IRQ(irq_num) ; \
|
||||
EOI_IRQ(irq_num) ; \
|
||||
0: ; \
|
||||
APIC_ITRACE(apic_itrace_tryisrlock, irq_num, APIC_ITRACE_TRYISRLOCK) ;\
|
||||
ISR_TRYLOCK ; /* XXX this is going away... */ \
|
||||
MP_TRYLOCK ; /* XXX this is going away... */ \
|
||||
testl %eax, %eax ; /* did we get it? */ \
|
||||
jz 3f ; /* no */ \
|
||||
; \
|
||||
APIC_ITRACE(apic_itrace_gotisrlock, irq_num, APIC_ITRACE_GOTISRLOCK) ;\
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
testl $IRQ_BIT(irq_num), _cpl ; \
|
||||
jne 2f ; /* this INT masked */ \
|
||||
AVCPL_UNLOCK ; \
|
||||
; \
|
||||
incb _intr_nesting_level ; \
|
||||
; \
|
||||
@ -480,14 +254,12 @@ __CONCAT(Xresume,irq_num): ; \
|
||||
movl _intr_countp + (irq_num) * 4, %eax ; \
|
||||
lock ; incl (%eax) ; \
|
||||
; \
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
movl _cpl, %eax ; \
|
||||
pushl %eax ; \
|
||||
orl _intr_mask + (irq_num) * 4, %eax ; \
|
||||
movl %eax, _cpl ; \
|
||||
lock ; \
|
||||
andl $~IRQ_BIT(irq_num), _ipending ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
; \
|
||||
pushl _intr_unit + (irq_num) * 4 ; \
|
||||
APIC_ITRACE(apic_itrace_enter2, irq_num, APIC_ITRACE_ENTER2) ; \
|
||||
@ -508,10 +280,8 @@ __CONCAT(Xresume,irq_num): ; \
|
||||
APIC_ITRACE(apic_itrace_active, irq_num, APIC_ITRACE_ACTIVE) ; \
|
||||
MASK_IRQ(irq_num) ; \
|
||||
EOI_IRQ(irq_num) ; \
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
lock ; \
|
||||
orl $IRQ_BIT(irq_num), _ipending ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
lock ; \
|
||||
btsl $(irq_num), iactive ; /* still active */ \
|
||||
jnc 0b ; /* retry */ \
|
||||
@ -522,32 +292,25 @@ __CONCAT(Xresume,irq_num): ; \
|
||||
APIC_ITRACE(apic_itrace_masked, irq_num, APIC_ITRACE_MASKED) ; \
|
||||
lock ; \
|
||||
orl $IRQ_BIT(irq_num), _ipending ; \
|
||||
AVCPL_UNLOCK ; \
|
||||
ISR_RELLOCK ; /* XXX this is going away... */ \
|
||||
MP_RELLOCK ; \
|
||||
POP_FRAME ; \
|
||||
iret ; \
|
||||
ALIGN_TEXT ; \
|
||||
3: ; /* other cpu has isr lock */ \
|
||||
APIC_ITRACE(apic_itrace_noisrlock, irq_num, APIC_ITRACE_NOISRLOCK) ;\
|
||||
AVCPL_LOCK ; /* MP-safe */ \
|
||||
lock ; \
|
||||
orl $IRQ_BIT(irq_num), _ipending ; \
|
||||
testl $IRQ_BIT(irq_num), _cpl ; \
|
||||
jne 4f ; /* this INT masked */ \
|
||||
AVCPL_UNLOCK ; \
|
||||
call forward_irq ; /* forward irq to lock holder */ \
|
||||
POP_FRAME ; /* and return */ \
|
||||
iret ; \
|
||||
ALIGN_TEXT ; \
|
||||
4: ; /* blocked */ \
|
||||
APIC_ITRACE(apic_itrace_masked2, irq_num, APIC_ITRACE_MASKED2) ;\
|
||||
AVCPL_UNLOCK ; \
|
||||
POP_FRAME ; /* and return */ \
|
||||
iret
|
||||
|
||||
#endif /* CPL_AND_CML */
|
||||
|
||||
|
||||
/*
|
||||
* Handle "spurious INTerrupts".
|
||||
* Notes:
|
||||
@ -635,11 +398,6 @@ _Xcpucheckstate:
|
||||
testl $PSL_VM, 24(%esp)
|
||||
jne 1f
|
||||
incl %ebx /* system or interrupt */
|
||||
#ifdef CPL_AND_CML
|
||||
cmpl $0, _inside_intr
|
||||
je 1f
|
||||
incl %ebx /* interrupt */
|
||||
#endif
|
||||
1:
|
||||
movl _cpuid, %eax
|
||||
movl %ebx, _checkstate_cpustate(,%eax,4)
|
||||
@ -693,17 +451,11 @@ _Xcpuast:
|
||||
* Giant locks do not come cheap.
|
||||
* A lot of cycles are going to be wasted here.
|
||||
*/
|
||||
call _get_isrlock
|
||||
call _get_mplock
|
||||
|
||||
AVCPL_LOCK
|
||||
#ifdef CPL_AND_CML
|
||||
movl _cml, %eax
|
||||
#else
|
||||
movl _cpl, %eax
|
||||
#endif
|
||||
pushl %eax
|
||||
movl $1, _astpending /* XXX */
|
||||
AVCPL_UNLOCK
|
||||
lock
|
||||
incb _intr_nesting_level
|
||||
sti
|
||||
@ -716,7 +468,7 @@ _Xcpuast:
|
||||
lock
|
||||
btrl %eax, CNAME(resched_cpus)
|
||||
jnc 2f
|
||||
movl $1, CNAME(want_resched)
|
||||
orl $AST_RESCHED,_astpending
|
||||
lock
|
||||
incl CNAME(want_resched_cnt)
|
||||
2:
|
||||
@ -749,7 +501,7 @@ _Xforward_irq:
|
||||
|
||||
FAKE_MCOUNT(13*4(%esp))
|
||||
|
||||
ISR_TRYLOCK
|
||||
MP_TRYLOCK
|
||||
testl %eax,%eax /* Did we get the lock ? */
|
||||
jz 1f /* No */
|
||||
|
||||
@ -758,14 +510,8 @@ _Xforward_irq:
|
||||
cmpb $4, _intr_nesting_level
|
||||
jae 2f
|
||||
|
||||
AVCPL_LOCK
|
||||
#ifdef CPL_AND_CML
|
||||
movl _cml, %eax
|
||||
#else
|
||||
movl _cpl, %eax
|
||||
#endif
|
||||
pushl %eax
|
||||
AVCPL_UNLOCK
|
||||
lock
|
||||
incb _intr_nesting_level
|
||||
sti
|
||||
@ -785,7 +531,7 @@ _Xforward_irq:
|
||||
lock
|
||||
incl CNAME(forward_irq_toodeepcnt)
|
||||
3:
|
||||
ISR_RELLOCK
|
||||
MP_RELLOCK
|
||||
MEXITCOUNT
|
||||
POP_FRAME
|
||||
iret
|
||||
|
@ -43,6 +43,10 @@
|
||||
/*
|
||||
* AT/386
|
||||
* Vector interrupt control section
|
||||
*
|
||||
* cpl - Current interrupt disable mask
|
||||
* *_imask - Interrupt masks for various spl*() functions
|
||||
* ipending - Pending interrupts (set when a masked interrupt occurs)
|
||||
*/
|
||||
|
||||
.data
|
||||
@ -67,9 +71,6 @@ _softnet_imask: .long SWI_NET_MASK
|
||||
.globl _softtty_imask
|
||||
_softtty_imask: .long SWI_TTY_MASK
|
||||
|
||||
.globl _astpending
|
||||
_astpending: .long 0
|
||||
|
||||
/* pending interrupts blocked by splxxx() */
|
||||
.globl _ipending
|
||||
_ipending: .long 0
|
||||
@ -91,29 +92,12 @@ _netisrs:
|
||||
|
||||
.text
|
||||
|
||||
#ifdef SMP
|
||||
#ifdef notnow
|
||||
#define TEST_CIL \
|
||||
cmpl $0x0100, _cil ; \
|
||||
jne 1f ; \
|
||||
cmpl $0, _inside_intr ; \
|
||||
jne 1f ; \
|
||||
int $3 ; \
|
||||
1:
|
||||
#else
|
||||
#define TEST_CIL
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Handle return from interrupts, traps and syscalls.
|
||||
*/
|
||||
SUPERALIGN_TEXT
|
||||
.type _doreti,@function
|
||||
_doreti:
|
||||
#ifdef SMP
|
||||
TEST_CIL
|
||||
#endif
|
||||
FAKE_MCOUNT(_bintr) /* init "from" _bintr -> _doreti */
|
||||
addl $4,%esp /* discard unit number */
|
||||
popl %eax /* cpl or cml to restore */
|
||||
@ -128,32 +112,17 @@ doreti_next:
|
||||
* handlers is limited by the number of bits in cpl).
|
||||
*/
|
||||
#ifdef SMP
|
||||
TEST_CIL
|
||||
cli /* early to prevent INT deadlock */
|
||||
pushl %eax /* preserve cpl while getting lock */
|
||||
ICPL_LOCK
|
||||
popl %eax
|
||||
doreti_next2:
|
||||
#endif
|
||||
movl %eax,%ecx
|
||||
#ifdef CPL_AND_CML
|
||||
orl _cpl, %ecx /* add cpl to cml */
|
||||
#endif
|
||||
notl %ecx /* set bit = unmasked level */
|
||||
#ifndef SMP
|
||||
cli
|
||||
#endif
|
||||
andl _ipending,%ecx /* set bit = unmasked pending INT */
|
||||
jne doreti_unpend
|
||||
#ifdef SMP
|
||||
TEST_CIL
|
||||
#endif
|
||||
#ifdef CPL_AND_CML
|
||||
movl %eax, _cml
|
||||
#else
|
||||
movl %eax,_cpl
|
||||
#endif
|
||||
FAST_ICPL_UNLOCK /* preserves %eax */
|
||||
MPLOCKED decb _intr_nesting_level
|
||||
|
||||
/* Check for ASTs that can be handled now. */
|
||||
@ -166,19 +135,27 @@ doreti_next2:
|
||||
cmpl $1,_in_vm86call
|
||||
jne doreti_ast
|
||||
|
||||
/*
|
||||
* doreti_exit - release MP lock, pop registers, iret.
|
||||
*
|
||||
* Note that the syscall trap shotcuts to doreti_syscall_ret.
|
||||
* The segment register pop is a special case, since it may
|
||||
* fault if (for example) a sigreturn specifies bad segment
|
||||
* registers. The fault is handled in trap.c
|
||||
*/
|
||||
|
||||
doreti_exit:
|
||||
MEXITCOUNT
|
||||
|
||||
#ifdef SMP
|
||||
#ifdef INTR_SIMPLELOCK
|
||||
#error code needed here to decide which lock to release, INTR or giant
|
||||
#endif
|
||||
/* release the kernel lock */
|
||||
movl $_mp_lock, %edx /* GIANT_LOCK */
|
||||
call _MPrellock_edx
|
||||
#endif /* SMP */
|
||||
|
||||
.globl doreti_popl_fs
|
||||
.globl doreti_syscall_ret
|
||||
doreti_syscall_ret:
|
||||
doreti_popl_fs:
|
||||
popl %fs
|
||||
.globl doreti_popl_es
|
||||
@ -215,22 +192,13 @@ doreti_popl_fs_fault:
|
||||
doreti_unpend:
|
||||
/*
|
||||
* Enabling interrupts is safe because we haven't restored cpl yet.
|
||||
* The locking from the "btrl" test is probably no longer necessary.
|
||||
* We won't miss any new pending interrupts because we will check
|
||||
* for them again.
|
||||
* %ecx contains the next probable ready interrupt (~cpl & ipending)
|
||||
*/
|
||||
#ifdef SMP
|
||||
TEST_CIL
|
||||
/* we enter with cpl locked */
|
||||
bsfl %ecx, %ecx /* slow, but not worth optimizing */
|
||||
bsfl %ecx, %ecx /* locate the next dispatchable int */
|
||||
lock
|
||||
btrl %ecx, _ipending
|
||||
btrl %ecx, _ipending /* is it really still pending? */
|
||||
jnc doreti_next2 /* some intr cleared memory copy */
|
||||
cmpl $NHWI, %ecx
|
||||
jae 1f
|
||||
btsl %ecx, _cil
|
||||
1:
|
||||
FAST_ICPL_UNLOCK /* preserves %eax */
|
||||
sti /* late to prevent INT deadlock */
|
||||
#else
|
||||
sti
|
||||
@ -238,8 +206,9 @@ doreti_unpend:
|
||||
btrl %ecx,_ipending
|
||||
jnc doreti_next /* some intr cleared memory copy */
|
||||
#endif /* SMP */
|
||||
|
||||
/*
|
||||
* Execute handleable interrupt
|
||||
*
|
||||
* Set up JUMP to _ihandlers[%ecx] for HWIs.
|
||||
* Set up CALL of _ihandlers[%ecx] for SWIs.
|
||||
* This is a bit early for the SMP case - we have to push %ecx and
|
||||
@ -247,25 +216,10 @@ doreti_unpend:
|
||||
*/
|
||||
movl _ihandlers(,%ecx,4),%edx
|
||||
cmpl $NHWI,%ecx
|
||||
jae doreti_swi
|
||||
cli
|
||||
jae doreti_swi /* software interrupt handling */
|
||||
cli /* else hardware int handling */
|
||||
#ifdef SMP
|
||||
pushl %edx /* preserve %edx */
|
||||
#ifdef APIC_INTR_DIAGNOSTIC
|
||||
pushl %ecx
|
||||
#endif
|
||||
pushl %eax /* preserve %eax */
|
||||
ICPL_LOCK
|
||||
#ifdef CPL_AND_CML
|
||||
popl _cml
|
||||
#else
|
||||
popl _cpl
|
||||
#endif
|
||||
FAST_ICPL_UNLOCK
|
||||
#ifdef APIC_INTR_DIAGNOSTIC
|
||||
popl %ecx
|
||||
#endif
|
||||
popl %edx
|
||||
movl %eax,_cpl /* same as non-smp case right now */
|
||||
#else
|
||||
movl %eax,_cpl
|
||||
#endif
|
||||
@ -292,9 +246,6 @@ doreti_unpend:
|
||||
|
||||
ALIGN_TEXT
|
||||
doreti_swi:
|
||||
#ifdef SMP
|
||||
TEST_CIL
|
||||
#endif
|
||||
pushl %eax
|
||||
/*
|
||||
* At least the SWI_CLOCK handler has to run at a possibly strictly
|
||||
@ -304,29 +255,18 @@ doreti_swi:
|
||||
* in dying interrupt frames and about 12 HWIs nested in active
|
||||
* interrupt frames. There are only 4 different SWIs and the HWI
|
||||
* and SWI masks limit the nesting further.
|
||||
*
|
||||
* The SMP case is currently the same as the non-SMP case.
|
||||
*/
|
||||
#ifdef SMP
|
||||
orl imasks(,%ecx,4), %eax
|
||||
pushl %ecx /* preserve for use by _swi_generic */
|
||||
pushl %edx /* save handler entry point */
|
||||
cli /* prevent INT deadlock */
|
||||
pushl %eax /* save cpl|cml */
|
||||
ICPL_LOCK
|
||||
#ifdef CPL_AND_CML
|
||||
popl _cml /* restore cml */
|
||||
#else
|
||||
popl _cpl /* restore cpl */
|
||||
#endif
|
||||
FAST_ICPL_UNLOCK
|
||||
sti
|
||||
popl %edx /* restore handler entry point */
|
||||
popl %ecx
|
||||
orl imasks(,%ecx,4), %eax /* or in imasks */
|
||||
movl %eax,_cpl /* set cpl for call */
|
||||
#else
|
||||
orl imasks(,%ecx,4),%eax
|
||||
movl %eax,_cpl
|
||||
#endif
|
||||
call %edx
|
||||
popl %eax
|
||||
popl %eax /* cpl to restore */
|
||||
jmp doreti_next
|
||||
|
||||
ALIGN_TEXT
|
||||
@ -336,9 +276,6 @@ doreti_ast:
|
||||
movl $T_ASTFLT,TF_TRAPNO(%esp)
|
||||
call _trap
|
||||
subl %eax,%eax /* recover cpl|cml */
|
||||
#ifdef CPL_AND_CML
|
||||
movl %eax, _cpl
|
||||
#endif
|
||||
movb $1,_intr_nesting_level /* for doreti_next to decrement */
|
||||
jmp doreti_next
|
||||
|
||||
|
@ -35,8 +35,9 @@
|
||||
#include <i386/isa/intr_machdep.h>
|
||||
|
||||
/*
|
||||
* The volatile bitmap variables must be set atomically. This normally
|
||||
* involves using a machine-dependent bit-set or `or' instruction.
|
||||
* Bits in the ipending bitmap variable must be set atomically because
|
||||
* ipending may be manipulated by interrupts or other cpu's without holding
|
||||
* any locks.
|
||||
*
|
||||
* Note: setbits uses a locked or, making simple cases MP safe.
|
||||
*/
|
||||
@ -67,6 +68,10 @@ softclockpending(void)
|
||||
return (ipending & SWI_CLOCK_PENDING);
|
||||
}
|
||||
|
||||
/*
|
||||
* Support for SPL assertions.
|
||||
*/
|
||||
|
||||
#ifdef INVARIANT_SUPPORT
|
||||
|
||||
#define SPLASSERT_IGNORE 0
|
||||
@ -112,6 +117,40 @@ NAME##assert(const char *msg) \
|
||||
#define GENSPLASSERT(NAME, MODIFIER)
|
||||
#endif
|
||||
|
||||
/************************************************************************
|
||||
* GENERAL SPL CODE *
|
||||
************************************************************************
|
||||
*
|
||||
* Implement splXXX(), spl0(), splx(), and splq(). splXXX() disables a
|
||||
* set of interrupts (e.g. splbio() disables interrupts relating to
|
||||
* device I/O) and returns the previous interrupt mask. splx() restores
|
||||
* the previous interrupt mask, spl0() is a special case which enables
|
||||
* all interrupts and is typically used inside i386/i386 swtch.s and
|
||||
* fork_trampoline. splq() is a generic version of splXXX().
|
||||
*
|
||||
* The SPL routines mess around with the 'cpl' global, which masks
|
||||
* interrupts. Interrupts are not *actually* masked. What happens is
|
||||
* that if an interrupt masked by the cpl occurs, the appropriate bit
|
||||
* in 'ipending' is set and the interrupt is defered. When we clear
|
||||
* bits in the cpl we must check to see if any ipending interrupts have
|
||||
* been unmasked and issue the synchronously, which is what the splz()
|
||||
* call does.
|
||||
*
|
||||
* Because the cpl is often saved and restored in a nested fashion, cpl
|
||||
* modifications are only allowed in the SMP case when the MP lock is held
|
||||
* to prevent multiple processes from tripping over each other's masks.
|
||||
* The cpl is saved when you do a context switch (mi_switch()) and restored
|
||||
* when your process gets cpu again.
|
||||
*
|
||||
* An interrupt routine is allowed to modify the cpl as long as it restores
|
||||
* it prior to returning (thus the interrupted mainline code doesn't notice
|
||||
* anything amiss). For the SMP case, the interrupt routine must hold
|
||||
* the MP lock for any cpl manipulation.
|
||||
*
|
||||
* Likewise, due to the deterministic nature of cpl modifications, we do
|
||||
* NOT need to use locked instructions to modify it.
|
||||
*/
|
||||
|
||||
#ifndef SMP
|
||||
|
||||
#define GENSPL(NAME, OP, MODIFIER, PC) \
|
||||
@ -154,219 +193,65 @@ splq(intrmask_t mask)
|
||||
#include <machine/smp.h>
|
||||
#include <machine/smptests.h>
|
||||
|
||||
#ifndef SPL_DEBUG_POSTCODE
|
||||
#undef POSTCODE
|
||||
#undef POSTCODE_LO
|
||||
#undef POSTCODE_HI
|
||||
#define POSTCODE(X)
|
||||
#define POSTCODE_LO(X)
|
||||
#define POSTCODE_HI(X)
|
||||
#endif /* SPL_DEBUG_POSTCODE */
|
||||
|
||||
|
||||
/*
|
||||
* This version has to check for bsp_apic_ready,
|
||||
* as calling simple_lock() (ie ss_lock) before then deadlocks the system.
|
||||
* A sample count of GENSPL calls before bsp_apic_ready was set: 2193
|
||||
* SMP CASE
|
||||
*
|
||||
* Mostly the same as the non-SMP case now, but it didn't used to be
|
||||
* this clean.
|
||||
*/
|
||||
|
||||
#ifdef INTR_SPL
|
||||
|
||||
#ifdef SPL_DEBUG
|
||||
#define MAXZ 100000000
|
||||
#define SPIN_VAR unsigned z;
|
||||
#define SPIN_RESET z = 0;
|
||||
#if 0
|
||||
#define SPIN_SPL \
|
||||
if (++z >= MAXZ) { \
|
||||
/* XXX allow lock-free panic */ \
|
||||
bsp_apic_ready = 0; \
|
||||
panic("\ncil: 0x%08x", cil); \
|
||||
}
|
||||
#else
|
||||
#define SPIN_SPL \
|
||||
if (++z >= MAXZ) { \
|
||||
/* XXX allow lock-free panic */ \
|
||||
bsp_apic_ready = 0; \
|
||||
printf("\ncil: 0x%08x", cil); \
|
||||
breakpoint(); \
|
||||
}
|
||||
#endif /* 0/1 */
|
||||
#else /* SPL_DEBUG */
|
||||
#define SPIN_VAR
|
||||
#define SPIN_RESET
|
||||
#define SPIN_SPL
|
||||
#endif /* SPL_DEBUG */
|
||||
|
||||
#endif
|
||||
|
||||
#ifdef INTR_SPL
|
||||
|
||||
#define GENSPL(NAME, OP, MODIFIER, PC) \
|
||||
GENSPLASSERT(NAME, MODIFIER) \
|
||||
unsigned NAME(void) \
|
||||
{ \
|
||||
unsigned x, y; \
|
||||
SPIN_VAR; \
|
||||
\
|
||||
if (!bsp_apic_ready) { \
|
||||
x = cpl; \
|
||||
cpl OP MODIFIER; \
|
||||
return (x); \
|
||||
} \
|
||||
\
|
||||
for (;;) { \
|
||||
IFCPL_LOCK(); /* MP-safe */ \
|
||||
x = y = cpl; /* current value */ \
|
||||
POSTCODE(0x20 | PC); \
|
||||
if (inside_intr) \
|
||||
break; /* XXX only 1 INT allowed */ \
|
||||
y OP MODIFIER; /* desired value */ \
|
||||
if (cil & y) { /* not now */ \
|
||||
IFCPL_UNLOCK(); /* allow cil to change */ \
|
||||
SPIN_RESET; \
|
||||
while (cil & y) \
|
||||
SPIN_SPL \
|
||||
continue; /* try again */ \
|
||||
} \
|
||||
break; \
|
||||
} \
|
||||
cpl OP MODIFIER; /* make the change */ \
|
||||
IFCPL_UNLOCK(); \
|
||||
\
|
||||
return (x); \
|
||||
}
|
||||
|
||||
#else /* INTR_SPL */
|
||||
|
||||
#define GENSPL(NAME, OP, MODIFIER, PC) \
|
||||
GENSPLASSERT(NAME, MODIFIER) \
|
||||
unsigned NAME(void) \
|
||||
{ \
|
||||
unsigned x; \
|
||||
\
|
||||
IFCPL_LOCK(); \
|
||||
x = cpl; \
|
||||
cpl OP MODIFIER; \
|
||||
IFCPL_UNLOCK(); \
|
||||
\
|
||||
return (x); \
|
||||
}
|
||||
|
||||
#endif /* INTR_SPL */
|
||||
|
||||
|
||||
/*
|
||||
* spl0() - unmask all interrupts
|
||||
*
|
||||
* The MP lock must be held on entry
|
||||
* This routine may only be called from mainline code.
|
||||
*/
|
||||
void
|
||||
spl0(void)
|
||||
{
|
||||
int unpend;
|
||||
#ifdef INTR_SPL
|
||||
SPIN_VAR;
|
||||
|
||||
for (;;) {
|
||||
IFCPL_LOCK();
|
||||
POSTCODE_HI(0xc);
|
||||
/*
|
||||
* XXX SWI_AST_MASK in ipending has moved to 1 in astpending,
|
||||
* so the following code is dead, but just removing it may
|
||||
* not be right.
|
||||
*/
|
||||
#if 0
|
||||
if (cil & SWI_AST_MASK) { /* not now */
|
||||
IFCPL_UNLOCK(); /* allow cil to change */
|
||||
SPIN_RESET;
|
||||
while (cil & SWI_AST_MASK)
|
||||
SPIN_SPL
|
||||
continue; /* try again */
|
||||
}
|
||||
#endif
|
||||
break;
|
||||
}
|
||||
#else /* INTR_SPL */
|
||||
IFCPL_LOCK();
|
||||
#endif /* INTR_SPL */
|
||||
|
||||
KASSERT(inside_intr == 0, ("spl0: called from interrupt"));
|
||||
cpl = 0;
|
||||
unpend = ipending;
|
||||
IFCPL_UNLOCK();
|
||||
|
||||
if (unpend && !inside_intr)
|
||||
if (ipending)
|
||||
splz();
|
||||
}
|
||||
|
||||
/*
|
||||
* splx() - restore previous interrupt mask
|
||||
*
|
||||
* The MP lock must be held on entry
|
||||
*/
|
||||
|
||||
void
|
||||
splx(unsigned ipl)
|
||||
{
|
||||
int unpend;
|
||||
#ifdef INTR_SPL
|
||||
SPIN_VAR;
|
||||
|
||||
for (;;) {
|
||||
IFCPL_LOCK();
|
||||
POSTCODE_HI(0xe);
|
||||
if (inside_intr)
|
||||
break; /* XXX only 1 INT allowed */
|
||||
POSTCODE_HI(0xf);
|
||||
if (cil & ipl) { /* not now */
|
||||
IFCPL_UNLOCK(); /* allow cil to change */
|
||||
SPIN_RESET;
|
||||
while (cil & ipl)
|
||||
SPIN_SPL
|
||||
continue; /* try again */
|
||||
}
|
||||
break;
|
||||
}
|
||||
#else /* INTR_SPL */
|
||||
IFCPL_LOCK();
|
||||
#endif /* INTR_SPL */
|
||||
|
||||
cpl = ipl;
|
||||
unpend = ipending & ~ipl;
|
||||
IFCPL_UNLOCK();
|
||||
|
||||
if (unpend && !inside_intr)
|
||||
if (inside_intr == 0 && (ipending & ~cpl) != 0)
|
||||
splz();
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* Replaces UP specific inline found in (?) pci/pci_support.c.
|
||||
* splq() - blocks specified interrupts
|
||||
*
|
||||
* Stefan said:
|
||||
* You know, that splq() is used in the shared interrupt multiplexer, and that
|
||||
* the SMP version should not have too much overhead. If it is significantly
|
||||
* slower, then moving the splq() out of the loop in intr_mux() and passing in
|
||||
* the logical OR of all mask values might be a better solution than the
|
||||
* current code. (This logical OR could of course be pre-calculated whenever
|
||||
* another shared interrupt is registered ...)
|
||||
* The MP lock must be held on entry
|
||||
*/
|
||||
intrmask_t
|
||||
splq(intrmask_t mask)
|
||||
{
|
||||
intrmask_t tmp;
|
||||
#ifdef INTR_SPL
|
||||
intrmask_t tmp2;
|
||||
|
||||
for (;;) {
|
||||
IFCPL_LOCK();
|
||||
tmp = tmp2 = cpl;
|
||||
tmp2 |= mask;
|
||||
if (cil & tmp2) { /* not now */
|
||||
IFCPL_UNLOCK(); /* allow cil to change */
|
||||
while (cil & tmp2)
|
||||
/* spin */ ;
|
||||
continue; /* try again */
|
||||
}
|
||||
break;
|
||||
}
|
||||
cpl = tmp2;
|
||||
#else /* INTR_SPL */
|
||||
IFCPL_LOCK();
|
||||
tmp = cpl;
|
||||
intrmask_t tmp = cpl;
|
||||
cpl |= mask;
|
||||
#endif /* INTR_SPL */
|
||||
|
||||
IFCPL_UNLOCK();
|
||||
return (tmp);
|
||||
}
|
||||
|
||||
|
@ -44,7 +44,7 @@ struct sysent sysent[] = {
|
||||
{ 4, (sy_call_t *)mount }, /* 21 = mount */
|
||||
{ 2, (sy_call_t *)unmount }, /* 22 = unmount */
|
||||
{ 1, (sy_call_t *)setuid }, /* 23 = setuid */
|
||||
{ 0, (sy_call_t *)getuid }, /* 24 = getuid */
|
||||
{ SYF_MPSAFE | 0, (sy_call_t *)getuid }, /* 24 = getuid */
|
||||
{ 0, (sy_call_t *)geteuid }, /* 25 = geteuid */
|
||||
{ 4, (sy_call_t *)ptrace }, /* 26 = ptrace */
|
||||
{ 3, (sy_call_t *)recvmsg }, /* 27 = recvmsg */
|
||||
@ -67,7 +67,7 @@ struct sysent sysent[] = {
|
||||
{ 4, (sy_call_t *)profil }, /* 44 = profil */
|
||||
{ 4, (sy_call_t *)ktrace }, /* 45 = ktrace */
|
||||
{ compat(3,sigaction) }, /* 46 = old sigaction */
|
||||
{ 0, (sy_call_t *)getgid }, /* 47 = getgid */
|
||||
{ SYF_MPSAFE | 0, (sy_call_t *)getgid }, /* 47 = getgid */
|
||||
{ compat(2,sigprocmask) }, /* 48 = old sigprocmask */
|
||||
{ 2, (sy_call_t *)getlogin }, /* 49 = getlogin */
|
||||
{ 1, (sy_call_t *)setlogin }, /* 50 = setlogin */
|
||||
@ -80,7 +80,7 @@ struct sysent sysent[] = {
|
||||
{ 2, (sy_call_t *)symlink }, /* 57 = symlink */
|
||||
{ 3, (sy_call_t *)readlink }, /* 58 = readlink */
|
||||
{ 3, (sy_call_t *)execve }, /* 59 = execve */
|
||||
{ 1, (sy_call_t *)umask }, /* 60 = umask */
|
||||
{ SYF_MPSAFE | 1, (sy_call_t *)umask }, /* 60 = umask */
|
||||
{ 1, (sy_call_t *)chroot }, /* 61 = chroot */
|
||||
{ compat(2,fstat) }, /* 62 = old fstat */
|
||||
{ compat(4,getkerninfo) }, /* 63 = old getkerninfo */
|
||||
@ -101,7 +101,7 @@ struct sysent sysent[] = {
|
||||
{ 3, (sy_call_t *)mincore }, /* 78 = mincore */
|
||||
{ 2, (sy_call_t *)getgroups }, /* 79 = getgroups */
|
||||
{ 2, (sy_call_t *)setgroups }, /* 80 = setgroups */
|
||||
{ 0, (sy_call_t *)getpgrp }, /* 81 = getpgrp */
|
||||
{ SYF_MPSAFE | 0, (sy_call_t *)getpgrp }, /* 81 = getpgrp */
|
||||
{ 2, (sy_call_t *)setpgid }, /* 82 = setpgid */
|
||||
{ 3, (sy_call_t *)setitimer }, /* 83 = setitimer */
|
||||
{ compat(0,wait) }, /* 84 = old wait */
|
||||
|
@ -62,6 +62,9 @@ struct getpid_args {
|
||||
};
|
||||
#endif
|
||||
|
||||
/*
|
||||
* NOT MP SAFE due to p_pptr access
|
||||
*/
|
||||
/* ARGSUSED */
|
||||
int
|
||||
getpid(p, uap)
|
||||
@ -92,7 +95,11 @@ getppid(p, uap)
|
||||
return (0);
|
||||
}
|
||||
|
||||
/* Get process group ID; note that POSIX getpgrp takes no parameter */
|
||||
/*
|
||||
* Get process group ID; note that POSIX getpgrp takes no parameter
|
||||
*
|
||||
* MP SAFE
|
||||
*/
|
||||
#ifndef _SYS_SYSPROTO_H_
|
||||
struct getpgrp_args {
|
||||
int dummy;
|
||||
@ -168,6 +175,9 @@ struct getuid_args {
|
||||
};
|
||||
#endif
|
||||
|
||||
/*
|
||||
* MP SAFE
|
||||
*/
|
||||
/* ARGSUSED */
|
||||
int
|
||||
getuid(p, uap)
|
||||
@ -205,6 +215,9 @@ struct getgid_args {
|
||||
};
|
||||
#endif
|
||||
|
||||
/*
|
||||
* MP SAFE
|
||||
*/
|
||||
/* ARGSUSED */
|
||||
int
|
||||
getgid(p, uap)
|
||||
|
@ -64,6 +64,7 @@
|
||||
|
||||
#include <vm/vm_zone.h>
|
||||
|
||||
#include <machine/ipl.h>
|
||||
#include <machine/cpu.h>
|
||||
#ifdef SMP
|
||||
#include <machine/smp.h>
|
||||
|
@ -140,6 +140,8 @@ remrunqueue(struct proc *p)
|
||||
* procrunnable() returns a boolean true (non-zero) value if there are
|
||||
* any runnable processes. This is intended to be called from the idle
|
||||
* loop to avoid the more expensive (and destructive) chooseproc().
|
||||
*
|
||||
* MP SAFE. CALLED WITHOUT THE MP LOCK
|
||||
*/
|
||||
u_int32_t
|
||||
procrunnable(void)
|
||||
|
@ -57,6 +57,7 @@
|
||||
#endif
|
||||
|
||||
#include <machine/cpu.h>
|
||||
#include <machine/ipl.h>
|
||||
#ifdef SMP
|
||||
#include <machine/smp.h>
|
||||
#endif
|
||||
|
@ -41,6 +41,7 @@
|
||||
#include <sys/kernel.h>
|
||||
#include <sys/resource.h>
|
||||
#include <machine/cpu.h> /* For need_resched */
|
||||
#include <machine/ipl.h> /* For need_resched */
|
||||
|
||||
#include <posix4/posix4.h>
|
||||
|
||||
|
@ -42,6 +42,7 @@
|
||||
#include <sys/resourcevar.h>
|
||||
#include <sys/sysctl.h>
|
||||
|
||||
#include <machine/ipl.h>
|
||||
#include <machine/cpu.h>
|
||||
|
||||
#ifdef GPROF
|
||||
|
@ -1770,8 +1770,10 @@ init_locks(void)
|
||||
*/
|
||||
mp_lock = 0x00000001;
|
||||
|
||||
#if 0
|
||||
/* ISR uses its own "giant lock" */
|
||||
isr_lock = FREE_LOCK;
|
||||
#endif
|
||||
|
||||
#if defined(APIC_INTR_DIAGNOSTIC) && defined(APIC_INTR_DIAGNOSTIC_IRQ)
|
||||
s_lock_init((struct simplelock*)&apic_itrace_debuglock);
|
||||
|
@ -99,7 +99,7 @@ int (*pmath_emulate) __P((struct trapframe *));
|
||||
|
||||
extern void trap __P((struct trapframe frame));
|
||||
extern int trapwrite __P((unsigned addr));
|
||||
extern void syscall __P((struct trapframe frame));
|
||||
extern void syscall2 __P((struct trapframe frame));
|
||||
|
||||
static int trap_pfault __P((struct trapframe *, int, vm_offset_t));
|
||||
static void trap_fatal __P((struct trapframe *, vm_offset_t));
|
||||
@ -140,38 +140,32 @@ static char *trap_msg[] = {
|
||||
"machine check trap", /* 28 T_MCHK */
|
||||
};
|
||||
|
||||
static __inline void userret __P((struct proc *p, struct trapframe *frame,
|
||||
u_quad_t oticks));
|
||||
static __inline int userret __P((struct proc *p, struct trapframe *frame,
|
||||
u_quad_t oticks, int have_mplock));
|
||||
|
||||
#if defined(I586_CPU) && !defined(NO_F00F_HACK)
|
||||
extern int has_f00f_bug;
|
||||
#endif
|
||||
|
||||
static __inline void
|
||||
userret(p, frame, oticks)
|
||||
static __inline int
|
||||
userret(p, frame, oticks, have_mplock)
|
||||
struct proc *p;
|
||||
struct trapframe *frame;
|
||||
u_quad_t oticks;
|
||||
int have_mplock;
|
||||
{
|
||||
int sig, s;
|
||||
|
||||
while ((sig = CURSIG(p)) != 0)
|
||||
while ((sig = CURSIG(p)) != 0) {
|
||||
if (have_mplock == 0) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
}
|
||||
postsig(sig);
|
||||
|
||||
#if 0
|
||||
if (!want_resched &&
|
||||
(p->p_priority <= p->p_usrpri) &&
|
||||
(p->p_rtprio.type == RTP_PRIO_NORMAL)) {
|
||||
int newpriority;
|
||||
p->p_estcpu += 1;
|
||||
newpriority = PUSER + p->p_estcpu / 4 + 2 * p->p_nice;
|
||||
newpriority = min(newpriority, MAXPRI);
|
||||
p->p_usrpri = newpriority;
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
p->p_priority = p->p_usrpri;
|
||||
if (want_resched) {
|
||||
if (resched_wanted()) {
|
||||
/*
|
||||
* Since we are curproc, clock will normally just change
|
||||
* our priority without moving us from one queue to another
|
||||
@ -180,6 +174,10 @@ userret(p, frame, oticks)
|
||||
* mi_switch()'ed, we might not be on the queue indicated by
|
||||
* our priority.
|
||||
*/
|
||||
if (have_mplock == 0) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
}
|
||||
s = splhigh();
|
||||
setrunqueue(p);
|
||||
p->p_stats->p_ru.ru_nivcsw++;
|
||||
@ -191,11 +189,16 @@ userret(p, frame, oticks)
|
||||
/*
|
||||
* Charge system time if profiling.
|
||||
*/
|
||||
if (p->p_flag & P_PROFIL)
|
||||
if (p->p_flag & P_PROFIL) {
|
||||
if (have_mplock == 0) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
}
|
||||
addupc_task(p, frame->tf_eip,
|
||||
(u_int)(p->p_sticks - oticks) * psratio);
|
||||
|
||||
}
|
||||
curpriority = p->p_priority;
|
||||
return(have_mplock);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -604,7 +607,7 @@ trap(frame)
|
||||
#endif
|
||||
|
||||
out:
|
||||
userret(p, &frame, sticks);
|
||||
userret(p, &frame, sticks, 1);
|
||||
}
|
||||
|
||||
#ifdef notyet
|
||||
@ -999,11 +1002,18 @@ int trapwrite(addr)
|
||||
}
|
||||
|
||||
/*
|
||||
* System call request from POSIX system call gate interface to kernel.
|
||||
* Like trap(), argument is call by reference.
|
||||
* syscall2 - MP aware system call request C handler
|
||||
*
|
||||
* A system call is essentially treated as a trap except that the
|
||||
* MP lock is not held on entry or return. We are responsible for
|
||||
* obtaining the MP lock if necessary and for handling ASTs
|
||||
* (e.g. a task switch) prior to return.
|
||||
*
|
||||
* In general, only simple access and manipulation of curproc and
|
||||
* the current stack is allowed without having to hold MP lock.
|
||||
*/
|
||||
void
|
||||
syscall(frame)
|
||||
syscall2(frame)
|
||||
struct trapframe frame;
|
||||
{
|
||||
caddr_t params;
|
||||
@ -1012,22 +1022,42 @@ syscall(frame)
|
||||
struct proc *p = curproc;
|
||||
u_quad_t sticks;
|
||||
int error;
|
||||
int narg;
|
||||
int args[8];
|
||||
int have_mplock = 0;
|
||||
u_int code;
|
||||
|
||||
#ifdef DIAGNOSTIC
|
||||
if (ISPL(frame.tf_cs) != SEL_UPL)
|
||||
if (ISPL(frame.tf_cs) != SEL_UPL) {
|
||||
get_mplock();
|
||||
panic("syscall");
|
||||
/* NOT REACHED */
|
||||
}
|
||||
#endif
|
||||
sticks = p->p_sticks;
|
||||
|
||||
/*
|
||||
* handle atomicy by looping since interrupts are enabled and the
|
||||
* MP lock is not held.
|
||||
*/
|
||||
sticks = ((volatile struct proc *)p)->p_sticks;
|
||||
while (sticks != ((volatile struct proc *)p)->p_sticks)
|
||||
sticks = ((volatile struct proc *)p)->p_sticks;
|
||||
|
||||
p->p_md.md_regs = &frame;
|
||||
params = (caddr_t)frame.tf_esp + sizeof(int);
|
||||
code = frame.tf_eax;
|
||||
|
||||
if (p->p_sysent->sv_prepsyscall) {
|
||||
/*
|
||||
* The prep code is not MP aware.
|
||||
*/
|
||||
get_mplock();
|
||||
(*p->p_sysent->sv_prepsyscall)(&frame, args, &code, ¶ms);
|
||||
rel_mplock();
|
||||
} else {
|
||||
/*
|
||||
* Need to check if this is a 32 bit or 64 bit syscall.
|
||||
* fuword is MP aware.
|
||||
*/
|
||||
if (code == SYS_syscall) {
|
||||
/*
|
||||
@ -1053,27 +1083,52 @@ syscall(frame)
|
||||
else
|
||||
callp = &p->p_sysent->sv_table[code];
|
||||
|
||||
if (params && (i = callp->sy_narg * sizeof(int)) &&
|
||||
narg = callp->sy_narg & SYF_ARGMASK;
|
||||
|
||||
/*
|
||||
* copyin is MP aware, but the tracing code is not
|
||||
*/
|
||||
if (params && (i = narg * sizeof(int)) &&
|
||||
(error = copyin(params, (caddr_t)args, (u_int)i))) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
#ifdef KTRACE
|
||||
if (KTRPOINT(p, KTR_SYSCALL))
|
||||
ktrsyscall(p->p_tracep, code, callp->sy_narg, args);
|
||||
ktrsyscall(p->p_tracep, code, narg, args);
|
||||
#endif
|
||||
goto bad;
|
||||
}
|
||||
|
||||
/*
|
||||
* Try to run the syscall without the MP lock if the syscall
|
||||
* is MP safe. We have to obtain the MP lock no matter what if
|
||||
* we are ktracing
|
||||
*/
|
||||
if ((callp->sy_narg & SYF_MPSAFE) == 0) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
}
|
||||
|
||||
#ifdef KTRACE
|
||||
if (KTRPOINT(p, KTR_SYSCALL))
|
||||
ktrsyscall(p->p_tracep, code, callp->sy_narg, args);
|
||||
if (KTRPOINT(p, KTR_SYSCALL)) {
|
||||
if (have_mplock == 0) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
}
|
||||
ktrsyscall(p->p_tracep, code, narg, args);
|
||||
}
|
||||
#endif
|
||||
p->p_retval[0] = 0;
|
||||
p->p_retval[1] = frame.tf_edx;
|
||||
|
||||
STOPEVENT(p, S_SCE, callp->sy_narg);
|
||||
STOPEVENT(p, S_SCE, narg); /* MP aware */
|
||||
|
||||
error = (*callp->sy_call)(p, args);
|
||||
|
||||
/*
|
||||
* MP SAFE (we may or may not have the MP lock at this point)
|
||||
*/
|
||||
switch (error) {
|
||||
|
||||
case 0:
|
||||
/*
|
||||
* Reinitialize proc pointer `p' as it may be different
|
||||
@ -1109,17 +1164,31 @@ syscall(frame)
|
||||
break;
|
||||
}
|
||||
|
||||
/*
|
||||
* Traced syscall. trapsignal() is not MP aware.
|
||||
*/
|
||||
if ((frame.tf_eflags & PSL_T) && !(frame.tf_eflags & PSL_VM)) {
|
||||
/* Traced syscall. */
|
||||
if (have_mplock == 0) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
}
|
||||
frame.tf_eflags &= ~PSL_T;
|
||||
trapsignal(p, SIGTRAP, 0);
|
||||
}
|
||||
|
||||
userret(p, &frame, sticks);
|
||||
/*
|
||||
* Handle reschedule and other end-of-syscall issues
|
||||
*/
|
||||
have_mplock = userret(p, &frame, sticks, have_mplock);
|
||||
|
||||
#ifdef KTRACE
|
||||
if (KTRPOINT(p, KTR_SYSRET))
|
||||
if (KTRPOINT(p, KTR_SYSRET)) {
|
||||
if (have_mplock == 0) {
|
||||
get_mplock();
|
||||
have_mplock = 1;
|
||||
}
|
||||
ktrsysret(p->p_tracep, code, error, p->p_retval[0]);
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
@ -1129,11 +1198,17 @@ syscall(frame)
|
||||
*/
|
||||
STOPEVENT(p, S_SCX, code);
|
||||
|
||||
/*
|
||||
* Release the MP lock if we had to get it
|
||||
*/
|
||||
if (have_mplock)
|
||||
rel_mplock();
|
||||
}
|
||||
|
||||
/*
|
||||
* Simplified back end of syscall(), used when returning from fork()
|
||||
* directly into user mode.
|
||||
* directly into user mode. MP lock is held on entry and should be
|
||||
* held on return.
|
||||
*/
|
||||
void
|
||||
fork_return(p, frame)
|
||||
@ -1144,7 +1219,7 @@ fork_return(p, frame)
|
||||
frame.tf_eflags &= ~PSL_C; /* success */
|
||||
frame.tf_edx = 1;
|
||||
|
||||
userret(p, &frame, 0);
|
||||
userret(p, &frame, 0, 1);
|
||||
#ifdef KTRACE
|
||||
if (KTRPOINT(p, KTR_SYSRET))
|
||||
ktrsysret(p->p_tracep, SYS_fork, 0, 0);
|
||||
|
@ -3000,6 +3000,8 @@ getdents(p, uap)
|
||||
|
||||
/*
|
||||
* Set the mode mask for creation of filesystem nodes.
|
||||
*
|
||||
* MP SAFE
|
||||
*/
|
||||
#ifndef _SYS_SYSPROTO_H_
|
||||
struct umask_args {
|
||||
|
@ -3000,6 +3000,8 @@ getdents(p, uap)
|
||||
|
||||
/*
|
||||
* Set the mode mask for creation of filesystem nodes.
|
||||
*
|
||||
* MP SAFE
|
||||
*/
|
||||
#ifndef _SYS_SYSPROTO_H_
|
||||
struct umask_args {
|
||||
|
@ -41,6 +41,7 @@
|
||||
#include <sys/kernel.h>
|
||||
#include <sys/resource.h>
|
||||
#include <machine/cpu.h> /* For need_resched */
|
||||
#include <machine/ipl.h> /* For need_resched */
|
||||
|
||||
#include <posix4/posix4.h>
|
||||
|
||||
|
@ -62,7 +62,7 @@ struct ktr_header {
|
||||
};
|
||||
|
||||
/*
|
||||
* Test for kernel trace point
|
||||
* Test for kernel trace point (MP SAFE)
|
||||
*/
|
||||
#define KTRPOINT(p, type) \
|
||||
(((p)->p_traceflag & ((1<<(type))|KTRFAC_ACTIVE)) == (1<<(type)))
|
||||
|
@ -338,9 +338,18 @@ MALLOC_DECLARE(M_PARGS);
|
||||
FREE(s, M_SESSION); \
|
||||
}
|
||||
|
||||
/*
|
||||
* STOPEVENT is MP SAFE.
|
||||
*/
|
||||
extern void stopevent(struct proc*, unsigned int, unsigned int);
|
||||
#define STOPEVENT(p,e,v) do { \
|
||||
if ((p)->p_stops & (e)) stopevent(p,e,v); } while (0)
|
||||
#define STOPEVENT(p,e,v) \
|
||||
do { \
|
||||
if ((p)->p_stops & (e)) { \
|
||||
get_mplock(); \
|
||||
stopevent(p,e,v); \
|
||||
rel_mplock(); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
/* hold process U-area in memory, normally for ptrace/procfs work */
|
||||
#define PHOLD(p) { \
|
||||
|
@ -39,6 +39,9 @@
|
||||
|
||||
#include <sys/signal.h>
|
||||
#include <sys/proc.h>
|
||||
#ifdef SMP
|
||||
#include <machine/smp.h>
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Kernel signal definitions and data structures,
|
||||
@ -225,16 +228,24 @@ void sendsig __P((sig_t action, int sig, sigset_t *retmask, u_long code));
|
||||
* Determine signal that should be delivered to process p, the current
|
||||
* process, 0 if none. If there is a pending stop signal with default
|
||||
* action, the process stops in issignal().
|
||||
*
|
||||
* MP SAFE
|
||||
*/
|
||||
extern __inline int __cursig(struct proc *p)
|
||||
{
|
||||
sigset_t tmpset;
|
||||
int r;
|
||||
|
||||
tmpset = p->p_siglist;
|
||||
SIGSETNAND(tmpset, p->p_sigmask);
|
||||
return ((SIGISEMPTY(p->p_siglist) ||
|
||||
(!(p->p_flag & P_TRACED) && SIGISEMPTY(tmpset)))
|
||||
? 0 : issignal(p));
|
||||
if (SIGISEMPTY(p->p_siglist) ||
|
||||
(!(p->p_flag & P_TRACED) && SIGISEMPTY(tmpset))) {
|
||||
return(0);
|
||||
}
|
||||
get_mplock();
|
||||
r = issignal(p);
|
||||
rel_mplock();
|
||||
return(r);
|
||||
}
|
||||
|
||||
#endif /* _KERNEL */
|
||||
|
@ -44,6 +44,10 @@ struct sysent { /* system call table */
|
||||
int sy_narg; /* number of arguments */
|
||||
sy_call_t *sy_call; /* implementing function */
|
||||
};
|
||||
|
||||
#define SYF_ARGMASK 0x0000FFFF
|
||||
#define SYF_MPSAFE 0x00010000
|
||||
|
||||
#define SCARG(p,k) ((p)->k) /* get arg from args pointer */
|
||||
/* placeholder till we integrate rest of lite2 syscallargs changes XXX */
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user