From 2f1e70693d5139b207a083e6a7255871ef15cdeb Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Sun, 17 May 1998 22:12:14 +0000 Subject: [PATCH] Add forwarding of roundrobin to other cpus. This gives a more regular update of cpu usage as shown by top when one process is cpu bound (no system calls) while the system is otherwise idle (except for top). Don't attempt to switch to the BSP in boot(). If the system was idle when an interrupt caused a panic, this won't work. Instead, switch to the BSP in cpu_reset. Remove some spurious forward_statclock/forward_hardclock warnings. --- sys/amd64/amd64/apic_vector.S | 57 ++++++++++++++++++++-- sys/amd64/amd64/genassym.c | 5 +- sys/amd64/amd64/mp_machdep.c | 58 +++++++++++++++++++---- sys/amd64/amd64/mptable.c | 58 +++++++++++++++++++---- sys/amd64/amd64/vm_machdep.c | 89 ++++++++++++++++++++++++++++++++++- sys/amd64/include/mptable.h | 58 +++++++++++++++++++---- sys/amd64/include/smp.h | 6 ++- sys/i386/i386/apic_vector.s | 57 ++++++++++++++++++++-- sys/i386/i386/genassym.c | 5 +- sys/i386/i386/mp_machdep.c | 58 +++++++++++++++++++---- sys/i386/i386/mptable.c | 58 +++++++++++++++++++---- sys/i386/i386/vm_machdep.c | 89 ++++++++++++++++++++++++++++++++++- sys/i386/include/mptable.h | 58 +++++++++++++++++++---- sys/i386/include/smp.h | 6 ++- sys/i386/isa/apic_vector.s | 57 ++++++++++++++++++++-- sys/kern/kern_shutdown.c | 17 +------ sys/kern/kern_synch.c | 11 ++++- sys/kern/subr_smp.c | 58 +++++++++++++++++++---- sys/sys/smp.h | 6 ++- 19 files changed, 708 insertions(+), 103 deletions(-) diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S index b3dd56af0d35..edd82250ca3a 100644 --- a/sys/amd64/amd64/apic_vector.S +++ b/sys/amd64/amd64/apic_vector.S @@ -1,6 +1,6 @@ /* * from: vector.s, 386BSD 0.1 unknown origin - * $Id: apic_vector.s,v 1.28 1998/03/05 21:45:53 tegge Exp $ + * $Id: apic_vector.s,v 1.29 1998/04/22 22:49:27 tegge Exp $ */ @@ -675,7 +675,15 @@ _Xcpuast: movl _cpuid, %eax lock btrl %eax, _checkstate_pending_ast - + lock + btrl %eax, CNAME(resched_cpus) + jz 2f + movl $1, CNAME(want_resched) + lock + incl CNAME(want_resched_cnt) +2: + lock + incl CNAME(cpuast_cnt) MEXITCOUNT jmp _doreti 1: @@ -803,12 +811,27 @@ forward_irq: SUPERALIGN_TEXT .globl _Xcpustop _Xcpustop: + pushl %ebp + movl %esp, %ebp pushl %eax + pushl %ecx + pushl %edx pushl %ds /* save current data segment */ + pushl %es movl $KDSEL, %eax movl %ax, %ds /* use KERNEL data segment */ + movl $0, lapic_eoi /* End Of Interrupt to APIC */ + + movl _cpuid, %eax + imull $PCB_SIZE, %eax + leal CNAME(stoppcbs)(%eax), %eax + pushl %eax + call CNAME(savectx) /* Save process context */ + addl $4, %esp + + movl _cpuid, %eax lock @@ -819,11 +842,26 @@ _Xcpustop: lock btrl %eax, _started_cpus /* started_cpus &= ~(1<pcb_savefpu); printf("#define\tPCB_SAVEFPU_SIZE %d\n", sizeof pcb->pcb_savefpu); printf("#define\tPCB_ONFAULT %p\n", &pcb->pcb_onfault); +#ifdef SMP + printf("#define\tPCB_SIZE %d\n", sizeof(struct pcb)); +#endif printf("#define\tTF_ES %p\n", &tf->tf_es); printf("#define\tTF_DS %p\n", &tf->tf_ds); diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c index 7b58b87d385b..c5e392a67522 100644 --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.74 1998/05/11 01:06:06 dyson Exp $ + * $Id: mp_machdep.c,v 1.75 1998/05/17 18:53:17 tegge Exp $ */ #include "opt_smp.h" @@ -312,6 +312,8 @@ extern pt_entry_t SMP_prvpt[]; /* Private page pointer to curcpu's PTD, used during BSP init */ extern pd_entry_t *my_idlePTD; +struct pcb stoppcbs[NCPU]; + static int smp_started; /* has the system started? */ /* @@ -2009,13 +2011,10 @@ stop_cpus(u_int map) if (!smp_started) return 0; - /* send IPI to all CPUs in map */ - stopped_cpus = 0; - /* send the Xcpustop IPI to all CPUs in map */ selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED); - - while (stopped_cpus != map) + + while ((stopped_cpus & map) != map) /* spin */ ; return 1; @@ -2043,9 +2042,8 @@ restart_cpus(u_int map) started_cpus = map; /* signal other cpus to restart */ - while (started_cpus) /* wait for each to clear its bit */ + while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */ /* spin */ ; - stopped_cpus = 0; return 1; } @@ -2075,6 +2073,11 @@ int forward_signal_enabled = 1; SYSCTL_INT(_machdep, OID_AUTO, forward_signal_enabled, CTLFLAG_RW, &forward_signal_enabled, 0, ""); +/* Enable forwarding of roundrobin to all other cpus */ +int forward_roundrobin_enabled = 1; +SYSCTL_INT(_machdep, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW, + &forward_roundrobin_enabled, 0, ""); + /* * This is called once the rest of the system is up and running and we're * ready to let the AP's out of the pen. @@ -2296,9 +2299,11 @@ forward_statclock(int pscnt) while (checkstate_probed_cpus != map) { /* spin */ i++; - if (i == 1000000) { + if (i == 100000) { +#ifdef BETTER_CLOCK_DIAGNOSTIC printf("forward_statclock: checkstate %x\n", checkstate_probed_cpus); +#endif break; } } @@ -2369,9 +2374,11 @@ forward_hardclock(int pscnt) while (checkstate_probed_cpus != map) { /* spin */ i++; - if (i == 1000000) { + if (i == 100000) { +#ifdef BETTER_CLOCK_DIAGNOSTIC printf("forward_hardclock: checkstate %x\n", checkstate_probed_cpus); +#endif break; } } @@ -2475,6 +2482,37 @@ forward_signal(struct proc *p) } } +void +forward_roundrobin(void) +{ + u_int map; + int i; + + if (!smp_started || !invltlb_ok || cold || panicstr) + return; + if (!forward_roundrobin_enabled) + return; + resched_cpus |= other_cpus; + map = other_cpus & ~stopped_cpus ; +#if 1 + selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED); +#else + (void) all_but_self_ipi(XCPUAST_OFFSET); +#endif + i = 0; + while ((checkstate_need_ast & map) != 0) { + /* spin */ + i++; + if (i > 100000) { +#if 0 + printf("forward_roundrobin: dropped ast 0x%x\n", + checkstate_need_ast & map); +#endif + break; + } + } +} + #ifdef APIC_INTR_REORDER /* diff --git a/sys/amd64/amd64/mptable.c b/sys/amd64/amd64/mptable.c index 7b58b87d385b..c5e392a67522 100644 --- a/sys/amd64/amd64/mptable.c +++ b/sys/amd64/amd64/mptable.c @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.74 1998/05/11 01:06:06 dyson Exp $ + * $Id: mp_machdep.c,v 1.75 1998/05/17 18:53:17 tegge Exp $ */ #include "opt_smp.h" @@ -312,6 +312,8 @@ extern pt_entry_t SMP_prvpt[]; /* Private page pointer to curcpu's PTD, used during BSP init */ extern pd_entry_t *my_idlePTD; +struct pcb stoppcbs[NCPU]; + static int smp_started; /* has the system started? */ /* @@ -2009,13 +2011,10 @@ stop_cpus(u_int map) if (!smp_started) return 0; - /* send IPI to all CPUs in map */ - stopped_cpus = 0; - /* send the Xcpustop IPI to all CPUs in map */ selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED); - - while (stopped_cpus != map) + + while ((stopped_cpus & map) != map) /* spin */ ; return 1; @@ -2043,9 +2042,8 @@ restart_cpus(u_int map) started_cpus = map; /* signal other cpus to restart */ - while (started_cpus) /* wait for each to clear its bit */ + while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */ /* spin */ ; - stopped_cpus = 0; return 1; } @@ -2075,6 +2073,11 @@ int forward_signal_enabled = 1; SYSCTL_INT(_machdep, OID_AUTO, forward_signal_enabled, CTLFLAG_RW, &forward_signal_enabled, 0, ""); +/* Enable forwarding of roundrobin to all other cpus */ +int forward_roundrobin_enabled = 1; +SYSCTL_INT(_machdep, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW, + &forward_roundrobin_enabled, 0, ""); + /* * This is called once the rest of the system is up and running and we're * ready to let the AP's out of the pen. @@ -2296,9 +2299,11 @@ forward_statclock(int pscnt) while (checkstate_probed_cpus != map) { /* spin */ i++; - if (i == 1000000) { + if (i == 100000) { +#ifdef BETTER_CLOCK_DIAGNOSTIC printf("forward_statclock: checkstate %x\n", checkstate_probed_cpus); +#endif break; } } @@ -2369,9 +2374,11 @@ forward_hardclock(int pscnt) while (checkstate_probed_cpus != map) { /* spin */ i++; - if (i == 1000000) { + if (i == 100000) { +#ifdef BETTER_CLOCK_DIAGNOSTIC printf("forward_hardclock: checkstate %x\n", checkstate_probed_cpus); +#endif break; } } @@ -2475,6 +2482,37 @@ forward_signal(struct proc *p) } } +void +forward_roundrobin(void) +{ + u_int map; + int i; + + if (!smp_started || !invltlb_ok || cold || panicstr) + return; + if (!forward_roundrobin_enabled) + return; + resched_cpus |= other_cpus; + map = other_cpus & ~stopped_cpus ; +#if 1 + selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED); +#else + (void) all_but_self_ipi(XCPUAST_OFFSET); +#endif + i = 0; + while ((checkstate_need_ast & map) != 0) { + /* spin */ + i++; + if (i > 100000) { +#if 0 + printf("forward_roundrobin: dropped ast 0x%x\n", + checkstate_need_ast & map); +#endif + break; + } + } +} + #ifdef APIC_INTR_REORDER /* diff --git a/sys/amd64/amd64/vm_machdep.c b/sys/amd64/amd64/vm_machdep.c index 78f16aed537f..7e3b32a9fb6f 100644 --- a/sys/amd64/amd64/vm_machdep.c +++ b/sys/amd64/amd64/vm_machdep.c @@ -38,7 +38,7 @@ * * from: @(#)vm_machdep.c 7.3 (Berkeley) 5/13/91 * Utah $Hdr: vm_machdep.c 1.16.1.1 89/06/23$ - * $Id: vm_machdep.c,v 1.105 1998/03/23 19:52:42 jlemon Exp $ + * $Id: vm_machdep.c,v 1.106 1998/05/16 14:44:11 kato Exp $ */ #include "npx.h" @@ -87,6 +87,13 @@ #include #endif +static void cpu_reset_real __P((void)); +#ifdef SMP +static void cpu_reset_proxy __P((void)); +static u_int cpu_reset_proxyid; +static volatile u_int cpu_reset_proxy_active; +#endif + #ifdef BOUNCE_BUFFERS static vm_offset_t vm_bounce_kva __P((int size, int waitok)); @@ -838,9 +845,89 @@ vunmapbuf(bp) /* * Force reset the processor by invalidating the entire address space! */ + +#ifdef SMP +static void +cpu_reset_proxy() +{ + u_int saved_mp_lock; + + cpu_reset_proxy_active = 1; + while (cpu_reset_proxy_active == 1) + ; /* Wait for other cpu to disable interupts */ + saved_mp_lock = mp_lock; + mp_lock = 1; + printf("cpu_reset_proxy: Grabbed mp lock for BSP\n"); + cpu_reset_proxy_active = 3; + while (cpu_reset_proxy_active == 3) + ; /* Wait for other cpu to enable interrupts */ + stop_cpus((1< 100000) { +#if 0 + printf("forward_roundrobin: dropped ast 0x%x\n", + checkstate_need_ast & map); +#endif + break; + } + } +} + #ifdef APIC_INTR_REORDER /* diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h index 283c925f2955..f2586304524c 100644 --- a/sys/amd64/include/smp.h +++ b/sys/amd64/include/smp.h @@ -6,7 +6,7 @@ * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp * ---------------------------------------------------------------------------- * - * $Id: smp.h,v 1.41 1998/04/01 20:38:28 tegge Exp $ + * $Id: smp.h,v 1.42 1998/04/01 21:07:36 tegge Exp $ * */ @@ -80,6 +80,8 @@ extern volatile u_int started_cpus; extern volatile u_int checkstate_probed_cpus; extern volatile u_int checkstate_need_ast; +extern volatile u_int resched_cpus; +extern void (*cpustop_restartfunc) __P((void)); /* functions in apic_ipl.s */ void apic_eoi __P((void)); @@ -103,6 +105,7 @@ extern int io_num_to_apic_id[]; extern int apic_id_to_logical[]; extern u_int all_cpus; extern u_char SMP_ioapic[]; +extern struct pcb stoppcbs[]; /* functions in mp_machdep.c */ u_int mp_bootaddress __P((u_int)); @@ -131,6 +134,7 @@ void forward_statclock __P((int pscnt)); void forward_hardclock __P((int pscnt)); #endif /* BETTER_CLOCK */ void forward_signal __P((struct proc *)); +void forward_roundrobin __P((void)); #ifdef APIC_INTR_REORDER void set_lapic_isrloc __P((int, int)); #endif /* APIC_INTR_REORDER */ diff --git a/sys/i386/i386/apic_vector.s b/sys/i386/i386/apic_vector.s index b3dd56af0d35..edd82250ca3a 100644 --- a/sys/i386/i386/apic_vector.s +++ b/sys/i386/i386/apic_vector.s @@ -1,6 +1,6 @@ /* * from: vector.s, 386BSD 0.1 unknown origin - * $Id: apic_vector.s,v 1.28 1998/03/05 21:45:53 tegge Exp $ + * $Id: apic_vector.s,v 1.29 1998/04/22 22:49:27 tegge Exp $ */ @@ -675,7 +675,15 @@ _Xcpuast: movl _cpuid, %eax lock btrl %eax, _checkstate_pending_ast - + lock + btrl %eax, CNAME(resched_cpus) + jz 2f + movl $1, CNAME(want_resched) + lock + incl CNAME(want_resched_cnt) +2: + lock + incl CNAME(cpuast_cnt) MEXITCOUNT jmp _doreti 1: @@ -803,12 +811,27 @@ forward_irq: SUPERALIGN_TEXT .globl _Xcpustop _Xcpustop: + pushl %ebp + movl %esp, %ebp pushl %eax + pushl %ecx + pushl %edx pushl %ds /* save current data segment */ + pushl %es movl $KDSEL, %eax movl %ax, %ds /* use KERNEL data segment */ + movl $0, lapic_eoi /* End Of Interrupt to APIC */ + + movl _cpuid, %eax + imull $PCB_SIZE, %eax + leal CNAME(stoppcbs)(%eax), %eax + pushl %eax + call CNAME(savectx) /* Save process context */ + addl $4, %esp + + movl _cpuid, %eax lock @@ -819,11 +842,26 @@ _Xcpustop: lock btrl %eax, _started_cpus /* started_cpus &= ~(1<pcb_savefpu); printf("#define\tPCB_SAVEFPU_SIZE %d\n", sizeof pcb->pcb_savefpu); printf("#define\tPCB_ONFAULT %p\n", &pcb->pcb_onfault); +#ifdef SMP + printf("#define\tPCB_SIZE %d\n", sizeof(struct pcb)); +#endif printf("#define\tTF_ES %p\n", &tf->tf_es); printf("#define\tTF_DS %p\n", &tf->tf_ds); diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c index 7b58b87d385b..c5e392a67522 100644 --- a/sys/i386/i386/mp_machdep.c +++ b/sys/i386/i386/mp_machdep.c @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.74 1998/05/11 01:06:06 dyson Exp $ + * $Id: mp_machdep.c,v 1.75 1998/05/17 18:53:17 tegge Exp $ */ #include "opt_smp.h" @@ -312,6 +312,8 @@ extern pt_entry_t SMP_prvpt[]; /* Private page pointer to curcpu's PTD, used during BSP init */ extern pd_entry_t *my_idlePTD; +struct pcb stoppcbs[NCPU]; + static int smp_started; /* has the system started? */ /* @@ -2009,13 +2011,10 @@ stop_cpus(u_int map) if (!smp_started) return 0; - /* send IPI to all CPUs in map */ - stopped_cpus = 0; - /* send the Xcpustop IPI to all CPUs in map */ selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED); - - while (stopped_cpus != map) + + while ((stopped_cpus & map) != map) /* spin */ ; return 1; @@ -2043,9 +2042,8 @@ restart_cpus(u_int map) started_cpus = map; /* signal other cpus to restart */ - while (started_cpus) /* wait for each to clear its bit */ + while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */ /* spin */ ; - stopped_cpus = 0; return 1; } @@ -2075,6 +2073,11 @@ int forward_signal_enabled = 1; SYSCTL_INT(_machdep, OID_AUTO, forward_signal_enabled, CTLFLAG_RW, &forward_signal_enabled, 0, ""); +/* Enable forwarding of roundrobin to all other cpus */ +int forward_roundrobin_enabled = 1; +SYSCTL_INT(_machdep, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW, + &forward_roundrobin_enabled, 0, ""); + /* * This is called once the rest of the system is up and running and we're * ready to let the AP's out of the pen. @@ -2296,9 +2299,11 @@ forward_statclock(int pscnt) while (checkstate_probed_cpus != map) { /* spin */ i++; - if (i == 1000000) { + if (i == 100000) { +#ifdef BETTER_CLOCK_DIAGNOSTIC printf("forward_statclock: checkstate %x\n", checkstate_probed_cpus); +#endif break; } } @@ -2369,9 +2374,11 @@ forward_hardclock(int pscnt) while (checkstate_probed_cpus != map) { /* spin */ i++; - if (i == 1000000) { + if (i == 100000) { +#ifdef BETTER_CLOCK_DIAGNOSTIC printf("forward_hardclock: checkstate %x\n", checkstate_probed_cpus); +#endif break; } } @@ -2475,6 +2482,37 @@ forward_signal(struct proc *p) } } +void +forward_roundrobin(void) +{ + u_int map; + int i; + + if (!smp_started || !invltlb_ok || cold || panicstr) + return; + if (!forward_roundrobin_enabled) + return; + resched_cpus |= other_cpus; + map = other_cpus & ~stopped_cpus ; +#if 1 + selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED); +#else + (void) all_but_self_ipi(XCPUAST_OFFSET); +#endif + i = 0; + while ((checkstate_need_ast & map) != 0) { + /* spin */ + i++; + if (i > 100000) { +#if 0 + printf("forward_roundrobin: dropped ast 0x%x\n", + checkstate_need_ast & map); +#endif + break; + } + } +} + #ifdef APIC_INTR_REORDER /* diff --git a/sys/i386/i386/mptable.c b/sys/i386/i386/mptable.c index 7b58b87d385b..c5e392a67522 100644 --- a/sys/i386/i386/mptable.c +++ b/sys/i386/i386/mptable.c @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.74 1998/05/11 01:06:06 dyson Exp $ + * $Id: mp_machdep.c,v 1.75 1998/05/17 18:53:17 tegge Exp $ */ #include "opt_smp.h" @@ -312,6 +312,8 @@ extern pt_entry_t SMP_prvpt[]; /* Private page pointer to curcpu's PTD, used during BSP init */ extern pd_entry_t *my_idlePTD; +struct pcb stoppcbs[NCPU]; + static int smp_started; /* has the system started? */ /* @@ -2009,13 +2011,10 @@ stop_cpus(u_int map) if (!smp_started) return 0; - /* send IPI to all CPUs in map */ - stopped_cpus = 0; - /* send the Xcpustop IPI to all CPUs in map */ selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED); - - while (stopped_cpus != map) + + while ((stopped_cpus & map) != map) /* spin */ ; return 1; @@ -2043,9 +2042,8 @@ restart_cpus(u_int map) started_cpus = map; /* signal other cpus to restart */ - while (started_cpus) /* wait for each to clear its bit */ + while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */ /* spin */ ; - stopped_cpus = 0; return 1; } @@ -2075,6 +2073,11 @@ int forward_signal_enabled = 1; SYSCTL_INT(_machdep, OID_AUTO, forward_signal_enabled, CTLFLAG_RW, &forward_signal_enabled, 0, ""); +/* Enable forwarding of roundrobin to all other cpus */ +int forward_roundrobin_enabled = 1; +SYSCTL_INT(_machdep, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW, + &forward_roundrobin_enabled, 0, ""); + /* * This is called once the rest of the system is up and running and we're * ready to let the AP's out of the pen. @@ -2296,9 +2299,11 @@ forward_statclock(int pscnt) while (checkstate_probed_cpus != map) { /* spin */ i++; - if (i == 1000000) { + if (i == 100000) { +#ifdef BETTER_CLOCK_DIAGNOSTIC printf("forward_statclock: checkstate %x\n", checkstate_probed_cpus); +#endif break; } } @@ -2369,9 +2374,11 @@ forward_hardclock(int pscnt) while (checkstate_probed_cpus != map) { /* spin */ i++; - if (i == 1000000) { + if (i == 100000) { +#ifdef BETTER_CLOCK_DIAGNOSTIC printf("forward_hardclock: checkstate %x\n", checkstate_probed_cpus); +#endif break; } } @@ -2475,6 +2482,37 @@ forward_signal(struct proc *p) } } +void +forward_roundrobin(void) +{ + u_int map; + int i; + + if (!smp_started || !invltlb_ok || cold || panicstr) + return; + if (!forward_roundrobin_enabled) + return; + resched_cpus |= other_cpus; + map = other_cpus & ~stopped_cpus ; +#if 1 + selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED); +#else + (void) all_but_self_ipi(XCPUAST_OFFSET); +#endif + i = 0; + while ((checkstate_need_ast & map) != 0) { + /* spin */ + i++; + if (i > 100000) { +#if 0 + printf("forward_roundrobin: dropped ast 0x%x\n", + checkstate_need_ast & map); +#endif + break; + } + } +} + #ifdef APIC_INTR_REORDER /* diff --git a/sys/i386/i386/vm_machdep.c b/sys/i386/i386/vm_machdep.c index 78f16aed537f..7e3b32a9fb6f 100644 --- a/sys/i386/i386/vm_machdep.c +++ b/sys/i386/i386/vm_machdep.c @@ -38,7 +38,7 @@ * * from: @(#)vm_machdep.c 7.3 (Berkeley) 5/13/91 * Utah $Hdr: vm_machdep.c 1.16.1.1 89/06/23$ - * $Id: vm_machdep.c,v 1.105 1998/03/23 19:52:42 jlemon Exp $ + * $Id: vm_machdep.c,v 1.106 1998/05/16 14:44:11 kato Exp $ */ #include "npx.h" @@ -87,6 +87,13 @@ #include #endif +static void cpu_reset_real __P((void)); +#ifdef SMP +static void cpu_reset_proxy __P((void)); +static u_int cpu_reset_proxyid; +static volatile u_int cpu_reset_proxy_active; +#endif + #ifdef BOUNCE_BUFFERS static vm_offset_t vm_bounce_kva __P((int size, int waitok)); @@ -838,9 +845,89 @@ vunmapbuf(bp) /* * Force reset the processor by invalidating the entire address space! */ + +#ifdef SMP +static void +cpu_reset_proxy() +{ + u_int saved_mp_lock; + + cpu_reset_proxy_active = 1; + while (cpu_reset_proxy_active == 1) + ; /* Wait for other cpu to disable interupts */ + saved_mp_lock = mp_lock; + mp_lock = 1; + printf("cpu_reset_proxy: Grabbed mp lock for BSP\n"); + cpu_reset_proxy_active = 3; + while (cpu_reset_proxy_active == 3) + ; /* Wait for other cpu to enable interrupts */ + stop_cpus((1< 100000) { +#if 0 + printf("forward_roundrobin: dropped ast 0x%x\n", + checkstate_need_ast & map); +#endif + break; + } + } +} + #ifdef APIC_INTR_REORDER /* diff --git a/sys/i386/include/smp.h b/sys/i386/include/smp.h index 283c925f2955..f2586304524c 100644 --- a/sys/i386/include/smp.h +++ b/sys/i386/include/smp.h @@ -6,7 +6,7 @@ * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp * ---------------------------------------------------------------------------- * - * $Id: smp.h,v 1.41 1998/04/01 20:38:28 tegge Exp $ + * $Id: smp.h,v 1.42 1998/04/01 21:07:36 tegge Exp $ * */ @@ -80,6 +80,8 @@ extern volatile u_int started_cpus; extern volatile u_int checkstate_probed_cpus; extern volatile u_int checkstate_need_ast; +extern volatile u_int resched_cpus; +extern void (*cpustop_restartfunc) __P((void)); /* functions in apic_ipl.s */ void apic_eoi __P((void)); @@ -103,6 +105,7 @@ extern int io_num_to_apic_id[]; extern int apic_id_to_logical[]; extern u_int all_cpus; extern u_char SMP_ioapic[]; +extern struct pcb stoppcbs[]; /* functions in mp_machdep.c */ u_int mp_bootaddress __P((u_int)); @@ -131,6 +134,7 @@ void forward_statclock __P((int pscnt)); void forward_hardclock __P((int pscnt)); #endif /* BETTER_CLOCK */ void forward_signal __P((struct proc *)); +void forward_roundrobin __P((void)); #ifdef APIC_INTR_REORDER void set_lapic_isrloc __P((int, int)); #endif /* APIC_INTR_REORDER */ diff --git a/sys/i386/isa/apic_vector.s b/sys/i386/isa/apic_vector.s index b3dd56af0d35..edd82250ca3a 100644 --- a/sys/i386/isa/apic_vector.s +++ b/sys/i386/isa/apic_vector.s @@ -1,6 +1,6 @@ /* * from: vector.s, 386BSD 0.1 unknown origin - * $Id: apic_vector.s,v 1.28 1998/03/05 21:45:53 tegge Exp $ + * $Id: apic_vector.s,v 1.29 1998/04/22 22:49:27 tegge Exp $ */ @@ -675,7 +675,15 @@ _Xcpuast: movl _cpuid, %eax lock btrl %eax, _checkstate_pending_ast - + lock + btrl %eax, CNAME(resched_cpus) + jz 2f + movl $1, CNAME(want_resched) + lock + incl CNAME(want_resched_cnt) +2: + lock + incl CNAME(cpuast_cnt) MEXITCOUNT jmp _doreti 1: @@ -803,12 +811,27 @@ forward_irq: SUPERALIGN_TEXT .globl _Xcpustop _Xcpustop: + pushl %ebp + movl %esp, %ebp pushl %eax + pushl %ecx + pushl %edx pushl %ds /* save current data segment */ + pushl %es movl $KDSEL, %eax movl %ax, %ds /* use KERNEL data segment */ + movl $0, lapic_eoi /* End Of Interrupt to APIC */ + + movl _cpuid, %eax + imull $PCB_SIZE, %eax + leal CNAME(stoppcbs)(%eax), %eax + pushl %eax + call CNAME(savectx) /* Save process context */ + addl $4, %esp + + movl _cpuid, %eax lock @@ -819,11 +842,26 @@ _Xcpustop: lock btrl %eax, _started_cpus /* started_cpus &= ~(1< +#ifdef SMP +#include +#endif +#include #include /* for UCHAR_MAX = typeof(p_priority)_MAX */ static void rqinit __P((void *)); @@ -130,8 +134,13 @@ roundrobin(arg) { struct proc *p = curproc; /* XXX */ +#ifdef SMP + need_resched(); + forward_roundrobin(); +#else if (p == 0 || RTP_PRIO_NEED_RR(p->p_rtprio.type)) need_resched(); +#endif timeout(roundrobin, NULL, ROUNDROBIN_INTERVAL); } diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c index 7b58b87d385b..c5e392a67522 100644 --- a/sys/kern/subr_smp.c +++ b/sys/kern/subr_smp.c @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.74 1998/05/11 01:06:06 dyson Exp $ + * $Id: mp_machdep.c,v 1.75 1998/05/17 18:53:17 tegge Exp $ */ #include "opt_smp.h" @@ -312,6 +312,8 @@ extern pt_entry_t SMP_prvpt[]; /* Private page pointer to curcpu's PTD, used during BSP init */ extern pd_entry_t *my_idlePTD; +struct pcb stoppcbs[NCPU]; + static int smp_started; /* has the system started? */ /* @@ -2009,13 +2011,10 @@ stop_cpus(u_int map) if (!smp_started) return 0; - /* send IPI to all CPUs in map */ - stopped_cpus = 0; - /* send the Xcpustop IPI to all CPUs in map */ selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED); - - while (stopped_cpus != map) + + while ((stopped_cpus & map) != map) /* spin */ ; return 1; @@ -2043,9 +2042,8 @@ restart_cpus(u_int map) started_cpus = map; /* signal other cpus to restart */ - while (started_cpus) /* wait for each to clear its bit */ + while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */ /* spin */ ; - stopped_cpus = 0; return 1; } @@ -2075,6 +2073,11 @@ int forward_signal_enabled = 1; SYSCTL_INT(_machdep, OID_AUTO, forward_signal_enabled, CTLFLAG_RW, &forward_signal_enabled, 0, ""); +/* Enable forwarding of roundrobin to all other cpus */ +int forward_roundrobin_enabled = 1; +SYSCTL_INT(_machdep, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW, + &forward_roundrobin_enabled, 0, ""); + /* * This is called once the rest of the system is up and running and we're * ready to let the AP's out of the pen. @@ -2296,9 +2299,11 @@ forward_statclock(int pscnt) while (checkstate_probed_cpus != map) { /* spin */ i++; - if (i == 1000000) { + if (i == 100000) { +#ifdef BETTER_CLOCK_DIAGNOSTIC printf("forward_statclock: checkstate %x\n", checkstate_probed_cpus); +#endif break; } } @@ -2369,9 +2374,11 @@ forward_hardclock(int pscnt) while (checkstate_probed_cpus != map) { /* spin */ i++; - if (i == 1000000) { + if (i == 100000) { +#ifdef BETTER_CLOCK_DIAGNOSTIC printf("forward_hardclock: checkstate %x\n", checkstate_probed_cpus); +#endif break; } } @@ -2475,6 +2482,37 @@ forward_signal(struct proc *p) } } +void +forward_roundrobin(void) +{ + u_int map; + int i; + + if (!smp_started || !invltlb_ok || cold || panicstr) + return; + if (!forward_roundrobin_enabled) + return; + resched_cpus |= other_cpus; + map = other_cpus & ~stopped_cpus ; +#if 1 + selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED); +#else + (void) all_but_self_ipi(XCPUAST_OFFSET); +#endif + i = 0; + while ((checkstate_need_ast & map) != 0) { + /* spin */ + i++; + if (i > 100000) { +#if 0 + printf("forward_roundrobin: dropped ast 0x%x\n", + checkstate_need_ast & map); +#endif + break; + } + } +} + #ifdef APIC_INTR_REORDER /* diff --git a/sys/sys/smp.h b/sys/sys/smp.h index 283c925f2955..f2586304524c 100644 --- a/sys/sys/smp.h +++ b/sys/sys/smp.h @@ -6,7 +6,7 @@ * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp * ---------------------------------------------------------------------------- * - * $Id: smp.h,v 1.41 1998/04/01 20:38:28 tegge Exp $ + * $Id: smp.h,v 1.42 1998/04/01 21:07:36 tegge Exp $ * */ @@ -80,6 +80,8 @@ extern volatile u_int started_cpus; extern volatile u_int checkstate_probed_cpus; extern volatile u_int checkstate_need_ast; +extern volatile u_int resched_cpus; +extern void (*cpustop_restartfunc) __P((void)); /* functions in apic_ipl.s */ void apic_eoi __P((void)); @@ -103,6 +105,7 @@ extern int io_num_to_apic_id[]; extern int apic_id_to_logical[]; extern u_int all_cpus; extern u_char SMP_ioapic[]; +extern struct pcb stoppcbs[]; /* functions in mp_machdep.c */ u_int mp_bootaddress __P((u_int)); @@ -131,6 +134,7 @@ void forward_statclock __P((int pscnt)); void forward_hardclock __P((int pscnt)); #endif /* BETTER_CLOCK */ void forward_signal __P((struct proc *)); +void forward_roundrobin __P((void)); #ifdef APIC_INTR_REORDER void set_lapic_isrloc __P((int, int)); #endif /* APIC_INTR_REORDER */