d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Overhaul of the SMP code. Several portions of the SMP kernel support have

Browse Source 
been made machine independent and various other adjustments have been made
to support Alpha SMP.

- It splits the per-process portions of hardclock() and statclock() off
  into hardclock_process() and statclock_process() respectively.  hardclock()
  and statclock() call the *_process() functions for the current process so
  that UP systems will run as before.  For SMP systems, it is simply necessary
  to ensure that all other processors execute the *_process() functions when the
  main clock functions are triggered on one CPU by an interrupt.  For the alpha
  4100, clock interrupts are delievered in a staggered broadcast fashion, so
  we simply call hardclock/statclock on the boot CPU and call the *_process()
  functions on the secondaries.  For x86, we call statclock and hardclock as
  usual and then call forward_hardclock/statclock in the MD code to send an IPI
  to cause the AP's to execute forwared_hardclock/statclock which then call the
  *_process() functions.
- forward_signal() and forward_roundrobin() have been reworked to be MI and to
  involve less hackery.  Now the cpu doing the forward sets any flags, etc. and
  sends a very simple IPI_AST to the other cpu(s).  AST IPIs now just basically
  return so that they can execute ast() and don't bother with setting the
  astpending or needresched flags themselves.  This also removes the loop in
  forward_signal() as sched_lock closes the race condition that the loop worked
  around.
- need_resched(), resched_wanted() and clear_resched() have been changed to take
  a process to act on rather than assuming curproc so that they can be used to
  implement forward_roundrobin() as described above.
- Various other SMP variables have been moved to a MI subr_smp.c and a new
  header sys/smp.h declares MI SMP variables and API's.   The IPI API's from
  machine/ipl.h have moved to machine/smp.h which is included by sys/smp.h.
- The globaldata_register() and globaldata_find() functions as well as the
  SLIST of globaldata structures has become MI and moved into subr_smp.c.
  Also, the globaldata list is only available if SMP support is compiled in.

Reviewed by:	jake, peter
Looked over by:	eivind
This commit is contained in:
John Baldwin 2001-04-27 19:28:25 +00:00
parent e5878540ed
commit 6caa8a1501
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=76078
77 changed files with 1208 additions and 8010 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
sys/alpha/alpha/db_interface.c
View File

@ -60,13 +60,13 @@
#include <sys/cons.h>
#include <sys/ktr.h>
#include <sys/mutex.h>
#include <sys/smp.h>
#include <vm/vm.h>
#include <machine/db_machdep.h>
#include <machine/pal.h>
#include <machine/prom.h>
#include <machine/smp.h>
#include <alpha/alpha/db_instruction.h>
@ -573,45 +573,46 @@ db_branch_taken(ins, pc, regs)
DB_SHOW_COMMAND(pcpu, db_show_pcpu)
{
struct globaldata *gd;
#ifdef SMP
int id;
if (have_addr)
id = ((addr >> 4) % 16) * 10 + (addr % 16);
else
id = PCPU_GET(cpuid);
SLIST_FOREACH(gd, &cpuhead, gd_allcpu) {
if (gd->gd_cpuid == id)
break;
}
if (gd == NULL)
gd = globaldata_find(id);
if (gd == NULL) {
db_printf("CPU %d not found\n", id);
else {
db_printf("cpuid = %d\n", gd->gd_cpuid);
db_printf("ipis = %lx\n", gd->gd_pending_ipis);
db_printf("next ASN = %d\n", gd->gd_next_asn);
db_printf("curproc = ");
if (gd->gd_curproc != NULL)
db_printf("%p: pid %d \"%s\"\n", gd->gd_curproc,
gd->gd_curproc->p_pid, gd->gd_curproc->p_comm);
else
db_printf("none\n");
db_printf("curpcb = %p\n", gd->gd_curpcb);
db_printf("fpcurproc = ");
if (gd->gd_fpcurproc != NULL)
db_printf("%p: pid %d \"%s\"\n", gd->gd_fpcurproc,
gd->gd_fpcurproc->p_pid, gd->gd_fpcurproc->p_comm);
else
db_printf("none\n");
db_printf("idleproc = ");
if (gd->gd_idleproc != NULL)
db_printf("%p: pid %d \"%s\"\n", gd->gd_idleproc,
gd->gd_idleproc->p_pid, gd->gd_idleproc->p_comm);
else
db_printf("none\n");
return;
}
#else
gd = GLOBALP;
#endif
db_printf("cpuid = %d\n", gd->gd_cpuid);
db_printf("ipis = %lx\n", gd->gd_pending_ipis);
db_printf("next ASN = %d\n", gd->gd_next_asn);
db_printf("curproc = ");
if (gd->gd_curproc != NULL)
db_printf("%p: pid %d \"%s\"\n", gd->gd_curproc,
gd->gd_curproc->p_pid, gd->gd_curproc->p_comm);
else
db_printf("none\n");
db_printf("curpcb = %p\n", gd->gd_curpcb);
db_printf("fpcurproc = ");
if (gd->gd_fpcurproc != NULL)
db_printf("%p: pid %d \"%s\"\n", gd->gd_fpcurproc,
gd->gd_fpcurproc->p_pid, gd->gd_fpcurproc->p_comm);
else
db_printf("none\n");
db_printf("idleproc = ");
if (gd->gd_idleproc != NULL)
db_printf("%p: pid %d \"%s\"\n", gd->gd_idleproc,
gd->gd_idleproc->p_pid, gd->gd_idleproc->p_comm);
else
db_printf("none\n");
#ifdef WITNESS
db_printf("spin locks held:\n");
witness_list_locks(&gd->gd_spinlocks);
db_printf("spin locks held:\n");
witness_list_locks(&gd->gd_spinlocks);
#endif
}
}

31
sys/alpha/alpha/interrupt.c
View File

@ -52,6 +52,7 @@
#include <sys/kthread.h>
#include <sys/ktr.h>
#include <sys/mutex.h>
#include <sys/smp.h>
#include <sys/unistd.h>
#include <machine/reg.h>
@ -83,7 +84,7 @@ void (*perf_irq)(unsigned long, struct trapframe *) = dummy_perf;
static u_int schedclk2;
static driver_intr_t alpha_clock_interrupt;
static void alpha_clock_interrupt(struct trapframe *framep);
void
interrupt(a0, a1, a2, framep)
@ -453,7 +454,7 @@ alpha_dispatch_intr(void *frame, unsigned long vector)
}
static void
alpha_clock_interrupt(void *framep)
alpha_clock_interrupt(struct trapframe *framep)
{
cnt.v_intr++;
@ -462,10 +463,26 @@ alpha_clock_interrupt(void *framep)
#else
intrcnt[INTRCNT_CLOCK]++;
#endif
if (platform.clockintr){
(*platform.clockintr)((struct trapframe *)framep);
/* divide hz (1024) by 8 to get stathz (128) */
if((++schedclk2 & 0x7) == 0)
statclock((struct clockframe *)framep);
if (platform.clockintr) {
#ifdef SMP
/*
* Only one processor drives the actual timer.
*/
if (PCPU_GET(cpuid) == boot_cpu_id) {
#endif
(*platform.clockintr)(framep);
/* divide hz (1024) by 8 to get stathz (128) */
if ((++schedclk2 & 0x7) == 0)
statclock((struct clockframe *)framep);
#ifdef SMP
} else {
mtx_lock_spin(&sched_lock);
hardclock_process(curproc, TRAPF_USERMODE(framep));
if ((schedclk2 & 0x7) == 0)
statclock_process(curproc, TRAPF_PC(framep),
TRAPF_USERMODE(framep));
mtx_unlock_spin(&sched_lock);
}
#endif
}
}

18
sys/alpha/alpha/machdep.c
View File

@ -112,6 +112,7 @@
#include <sys/vmmeter.h>
#include <sys/msgbuf.h>
#include <sys/exec.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/uio.h>
#include <sys/linker.h>
@ -130,7 +131,6 @@
#include <machine/reg.h>
#include <machine/fpu.h>
#include <machine/pal.h>
#include <machine/smp.h>
#include <machine/globaldata.h>
#include <machine/cpuconf.h>
#include <machine/bootinfo.h>
@ -152,8 +152,6 @@ struct platform platform;
alpha_chipset_t chipset;
struct bootinfo_kernel bootinfo;
struct cpuhead cpuhead;
struct mtx sched_lock;
struct mtx Giant;
@ -415,14 +413,6 @@ cpu_startup(dummy)
vm_pager_bufferinit();
EVENTHANDLER_REGISTER(shutdown_final, alpha_srm_shutdown, 0,
SHUTDOWN_PRI_LAST);
#ifdef SMP
/*
* OK, enough kmem_alloc/malloc state should be up, lets get on with it!
*/
mp_start(); /* fire up the secondaries */
mp_announce();
#endif /* SMP */
}
/*
@ -1002,12 +992,6 @@ alpha_init(pfn, ptb, bim, bip, biv)
*/
globalp->gd_idlepcb.apcb_ptbr = proc0.p_addr->u_pcb.pcb_hw.apcb_ptbr;
/*
* Record all cpus in a list.
*/
SLIST_INIT(&cpuhead);
SLIST_INSERT_HEAD(&cpuhead, GLOBALP, gd_allcpu);
/* Setup curproc so that mutexes work */
PCPU_SET(curproc, &proc0);
PCPU_SET(spinlocks, NULL);

685
sys/alpha/alpha/mp_machdep.c
View File

@ -35,6 +35,7 @@
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/kernel.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <vm/vm.h>
@ -43,55 +44,18 @@
#include <sys/user.h>
#include <sys/dkstat.h>
#include <machine/smp.h>
#include <machine/atomic.h>
#include <machine/globaldata.h>
#include <machine/pmap.h>
#include <machine/rpb.h>
#include <machine/clock.h>
#define CHECKSTATE_USER 0
#define CHECKSTATE_SYS 1
#define CHECKSTATE_INTR 2
volatile u_int stopped_cpus;
volatile u_int started_cpus;
volatile u_int checkstate_probed_cpus;
volatile u_int checkstate_need_ast;
/* Set to 1 once we're ready to let the APs out of the pen. */
static volatile int aps_ready = 0;
static struct mtx ap_boot_mtx;
struct proc* checkstate_curproc[MAXCPU];
int checkstate_cpustate[MAXCPU];
u_long checkstate_pc[MAXCPU];
volatile u_int resched_cpus;
void (*cpustop_restartfunc) __P((void));
int mp_ncpus;
volatile int smp_started;
int boot_cpu_id;
u_int32_t all_cpus;
static struct globaldata *cpuid_to_globaldata[MAXCPU];
int smp_active = 0; /* are the APs allowed to run? */
SYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RW, &smp_active, 0, "");
static int smp_cpus = 1; /* how many cpu's running */
SYSCTL_INT(_machdep, OID_AUTO, smp_cpus, CTLFLAG_RD, &smp_cpus, 0, "");
/* Enable forwarding of a signal to a process running on a different CPU */
static int forward_signal_enabled = 1;
SYSCTL_INT(_machdep, OID_AUTO, forward_signal_enabled, CTLFLAG_RW,
&forward_signal_enabled, 0, "");
/* Enable forwarding of roundrobin to all other cpus */
static int forward_roundrobin_enabled = 1;
SYSCTL_INT(_machdep, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW,
&forward_roundrobin_enabled, 0, "");
u_int boot_cpu_id;
/*
* Communicate with a console running on a secondary processor.
@ -225,7 +189,6 @@ smp_init_secondary(void)
if (smp_cpus == mp_ncpus) {
smp_started = 1;
smp_active = 1;
ipi_all(0);
}
mtx_unlock_spin(&ap_boot_mtx);
@ -250,7 +213,7 @@ static int
smp_start_secondary(int cpuid)
{
struct pcs *cpu = LOCATE_PCS(hwrpb, cpuid);
struct pcs *bootcpu = LOCATE_PCS(hwrpb, hwrpb->rpb_primary_cpu_id);
struct pcs *bootcpu = LOCATE_PCS(hwrpb, boot_cpu_id);
struct alpha_pcb *pcb = (struct alpha_pcb *) cpu->pcs_hwpcb;
struct globaldata *globaldata;
int i;
@ -272,7 +235,6 @@ smp_start_secondary(int cpuid)
}
globaldata_init(globaldata, cpuid, sz);
SLIST_INSERT_HEAD(&cpuhead, globaldata, gd_allcpu);
/*
* Copy the idle pcb and setup the address to start executing.
@ -329,48 +291,53 @@ smp_start_secondary(int cpuid)
return 1;
}
/*
* Register a struct globaldata.
*/
void
globaldata_register(struct globaldata *globaldata)
{
cpuid_to_globaldata[globaldata->gd_cpuid] = globaldata;
}
struct globaldata *
globaldata_find(int cpuid)
{
return cpuid_to_globaldata[cpuid];
}
/* Other stuff */
/* lock around the MP rendezvous */
static struct mtx smp_rv_mtx;
static void
init_locks(void)
int
cpu_mp_probe(void)
{
mtx_init(&smp_rv_mtx, "smp rendezvous", MTX_SPIN);
mtx_init(&ap_boot_mtx, "ap boot", MTX_SPIN);
struct pcs *pcsp;
int i, cpus;
/* XXX: Need to check for valid platforms here. */
/* Make sure we have at least one secondary CPU. */
cpus = 0;
for (i = 0; i < hwrpb->rpb_pcs_cnt; i++) {
if (i == PCPU_GET(cpuid))
continue;
pcsp = (struct pcs *)((char *)hwrpb + hwrpb->rpb_pcs_off +
(i * hwrpb->rpb_pcs_size));
if ((pcsp->pcs_flags & PCS_PP) == 0)
continue;
if ((pcsp->pcs_flags & PCS_PA) == 0)
continue;
if ((pcsp->pcs_flags & PCS_PV) == 0)
continue;
if (i > MAXCPU)
continue;
cpus++;
}
return (cpus);
}
void
mp_start()
cpu_mp_start()
{
int i;
int cpuid = PCPU_GET(cpuid);
init_locks();
mtx_init(&ap_boot_mtx, "ap boot", MTX_SPIN);
mp_ncpus = 1;
all_cpus = 1 << cpuid;
boot_cpu_id = PCPU_GET(cpuid);
KASSERT(boot_cpu_id == hwrpb->rpb_primary_cpu_id,
("mp_start() called on non-primary CPU"));
all_cpus = 1 << boot_cpu_id;
for (i = 0; i < hwrpb->rpb_pcs_cnt; i++) {
struct pcs *pcsp;
if (i == cpuid)
if (i == boot_cpu_id)
continue;
pcsp = (struct pcs *)((char *)hwrpb + hwrpb->rpb_pcs_off +
(i * hwrpb->rpb_pcs_size));
@ -397,10 +364,10 @@ mp_start()
all_cpus |= 1 << i;
mp_ncpus++;
}
PCPU_SET(other_cpus, all_cpus & ~(1<<cpuid));
PCPU_SET(other_cpus, all_cpus & ~(1 << boot_cpu_id));
for (i = 0; i < hwrpb->rpb_pcs_cnt; i++) {
if (i == cpuid)
if (i == boot_cpu_id)
continue;
if (all_cpus & 1 << i)
smp_start_secondary(i);
@ -408,561 +375,8 @@ mp_start()
}
void
mp_announce()
cpu_mp_announce()
{
printf("SMP System Detected: %d usable CPUs\n", mp_ncpus);
}
void
smp_invltlb()
{
}
#if 0
#define GD_TO_INDEX(pc, prof) \
((int)(((u_quad_t)((pc) - (prof)->pr_off) * \
(u_quad_t)((prof)->pr_scale)) >> 16) & ~1)
static void
addupc_intr_forwarded(struct proc *p, int id, int *astmap)
{
int i;
struct uprof *prof;
u_long pc;
pc = checkstate_pc[id];
prof = &p->p_stats->p_prof;
if (pc >= prof->pr_off &&
(i = GD_TO_INDEX(pc, prof)) < prof->pr_size) {
if ((p->p_sflag & PS_OWEUPC) == 0) {
prof->pr_addr = pc;
prof->pr_ticks = 1;
p->p_sflag |= PS_OWEUPC;
}
*astmap |= (1 << id);
}
}
static void
forwarded_statclock(int id, int pscnt, int *astmap)
{
struct pstats *pstats;
long rss;
struct rusage *ru;
struct vmspace *vm;
int cpustate;
struct proc *p;
#ifdef GPROF
register struct gmonparam *g;
int i;
#endif
p = checkstate_curproc[id];
cpustate = checkstate_cpustate[id];
/* XXX */
if (p->p_ithd)
cpustate = CHECKSTATE_INTR;
else if (p == cpuid_to_globaldata[id]->gd_idleproc)
cpustate = CHECKSTATE_SYS;
switch (cpustate) {
case CHECKSTATE_USER:
if (p->p_sflag & PS_PROFIL)
addupc_intr_forwarded(p, id, astmap);
if (pscnt > 1)
return;
p->p_uticks++;
if (p->p_nice > NZERO)
cp_time[CP_NICE]++;
else
cp_time[CP_USER]++;
break;
case CHECKSTATE_SYS:
#ifdef GPROF
/*
* Kernel statistics are just like addupc_intr, only easier.
*/
g = &_gmonparam;
if (g->state == GMON_PROF_ON) {
i = checkstate_pc[id] - g->lowpc;
if (i < g->textsize) {
i /= HISTFRACTION * sizeof(*g->kcount);
g->kcount[i]++;
}
}
#endif
if (pscnt > 1)
return;
p->p_sticks++;
if (p == cpuid_to_globaldata[id]->gd_idleproc)
cp_time[CP_IDLE]++;
else
cp_time[CP_SYS]++;
break;
case CHECKSTATE_INTR:
default:
#ifdef GPROF
/*
* Kernel statistics are just like addupc_intr, only easier.
*/
g = &_gmonparam;
if (g->state == GMON_PROF_ON) {
i = checkstate_pc[id] - g->lowpc;
if (i < g->textsize) {
i /= HISTFRACTION * sizeof(*g->kcount);
g->kcount[i]++;
}
}
#endif
if (pscnt > 1)
return;
KASSERT(p != NULL, ("NULL process in interrupt state"));
p->p_iticks++;
cp_time[CP_INTR]++;
}
schedclock(p);
/* Update resource usage integrals and maximums. */
if ((pstats = p->p_stats) != NULL &&
(ru = &pstats->p_ru) != NULL &&
(vm = p->p_vmspace) != NULL) {
ru->ru_ixrss += pgtok(vm->vm_tsize);
ru->ru_idrss += pgtok(vm->vm_dsize);
ru->ru_isrss += pgtok(vm->vm_ssize);
rss = pgtok(vmspace_resident_count(vm));
if (ru->ru_maxrss < rss)
ru->ru_maxrss = rss;
}
}
void
forward_statclock(int pscnt)
{
int map;
int id;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_statclock(%d)", pscnt);
if (!smp_started || cold || panicstr)
return;
/* Step 1: Probe state (user, cpu, interrupt, spinlock, idle ) */
mtx_lock_spin(&smp_rv_mtx);
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
checkstate_probed_cpus = 0;
if (map != 0)
ipi_selected(map, IPI_CHECKSTATE);
i = 0;
while (checkstate_probed_cpus != map) {
alpha_mb();
/* spin */
i++;
if (i == 100000) {
#ifdef DIAGNOSTIC
printf("forward_statclock: checkstate %x\n",
checkstate_probed_cpus);
#endif
break;
}
}
mtx_unlock_spin(&smp_rv_mtx);
/*
* Step 2: walk through other processors processes, update ticks and
* profiling info.
*/
map = 0;
/* XXX: wrong, should walk bits in all_cpus */
for (id = 0; id < mp_ncpus; id++) {
if (id == PCPU_GET(cpuid))
continue;
if (((1 << id) & checkstate_probed_cpus) == 0)
continue;
forwarded_statclock(id, pscnt, &map);
}
if (map != 0) {
mtx_lock_spin(&smp_rv_mtx);
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
alpha_mb();
/* spin */
i++;
if (i > 100000) {
#ifdef DIAGNOSTIC
printf("forward_statclock: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
mtx_unlock_spin(&smp_rv_mtx);
}
}
void
forward_hardclock(int pscnt)
{
int map;
int id;
struct proc *p;
struct pstats *pstats;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_hardclock(%d)", pscnt);
if (!smp_started || cold || panicstr)
return;
/* Step 1: Probe state (user, cpu, interrupt, spinlock, idle) */
mtx_lock_spin(&smp_rv_mtx);
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
checkstate_probed_cpus = 0;
if (map != 0)
ipi_selected(map, IPI_CHECKSTATE);
i = 0;
while (checkstate_probed_cpus != map) {
alpha_mb();
/* spin */
i++;
if (i == 100000) {
#ifdef DIAGNOSTIC
printf("forward_hardclock: checkstate %x\n",
checkstate_probed_cpus);
#endif
breakpoint();
break;
}
}
mtx_unlock_spin(&smp_rv_mtx);
/*
* Step 2: walk through other processors processes, update virtual
* timer and profiling timer. If stathz == 0, also update ticks and
* profiling info.
*/
map = 0;
/* XXX: wrong, should walk bits in all_cpus */
for (id = 0; id < mp_ncpus; id++) {
if (id == PCPU_GET(cpuid))
continue;
if (((1 << id) & checkstate_probed_cpus) == 0)
continue;
p = checkstate_curproc[id];
if (p) {
pstats = p->p_stats;
if (checkstate_cpustate[id] == CHECKSTATE_USER &&
timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) &&
itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) {
p->p_sflag |= PS_ALRMPEND;
map |= (1 << id);
}
if (timevalisset(&pstats->p_timer[ITIMER_PROF].it_value) &&
itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) {
p->p_sflag |= PS_PROFPEND;
map |= (1 << id);
}
}
if (stathz == 0) {
forwarded_statclock( id, pscnt, &map);
}
}
if (map != 0) {
mtx_lock_spin(&smp_rv_mtx);
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
alpha_mb();
/* spin */
i++;
if (i > 100000) {
#ifdef DIAGNOSTIC
printf("forward_hardclock: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
mtx_unlock_spin(&smp_rv_mtx);
}
}
#endif /* 0 */
void
forward_signal(struct proc *p)
{
int map;
int id;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_signal(%p)", p);
if (!smp_started || cold || panicstr)
return;
if (!forward_signal_enabled)
return;
mtx_lock_spin(&sched_lock);
while (1) {
if (p->p_stat != SRUN) {
mtx_unlock_spin(&sched_lock);
return;
}
id = p->p_oncpu;
mtx_unlock_spin(&sched_lock);
if (id == 0xff)
return;
map = (1<<id);
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
alpha_mb();
/* spin */
i++;
if (i > 100000) {
#if DIAGNOSTIC
printf("forward_signal: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
mtx_lock_spin(&sched_lock);
if (id == p->p_oncpu) {
mtx_unlock_spin(&sched_lock);
return;
}
}
}
void
forward_roundrobin(void)
{
u_int map;
int i;
CTR0(KTR_SMP, "forward_roundrobin()");
if (!smp_started || cold || panicstr)
return;
if (!forward_roundrobin_enabled)
return;
resched_cpus |= PCPU_GET(other_cpus);
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
alpha_mb();
/* spin */
i++;
if (i > 100000) {
#if DIAGNOSTIC
printf("forward_roundrobin: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
}
/*
* When called the executing CPU will send an IPI to all other CPUs
* requesting that they halt execution.
*
* Usually (but not necessarily) called with 'other_cpus' as its arg.
*
* - Signals all CPUs in map to stop.
* - Waits for each to stop.
*
* Returns:
* -1: error
* 0: NA
* 1: ok
*
* XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
* from executing at same time.
*/
int
stop_cpus(u_int map)
{
int i;
if (!smp_started)
return 0;
CTR1(KTR_SMP, "stop_cpus(%x)", map);
/* send the stop IPI to all CPUs in map */
ipi_selected(map, IPI_STOP);
i = 0;
while ((stopped_cpus & map) != map) {
/* spin */
i++;
#ifdef DIAGNOSTIC
if (i == 100000) {
printf("timeout stopping cpus\n");
break;
}
#endif
alpha_mb();
}
return 1;
}
/*
* Called by a CPU to restart stopped CPUs.
*
* Usually (but not necessarily) called with 'stopped_cpus' as its arg.
*
* - Signals all CPUs in map to restart.
* - Waits for each to restart.
*
* Returns:
* -1: error
* 0: NA
* 1: ok
*/
int
restart_cpus(u_int map)
{
if (!smp_started)
return 0;
CTR1(KTR_SMP, "restart_cpus(%x)", map);
started_cpus = map; /* signal other cpus to restart */
alpha_mb();
while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */
alpha_mb();
return 1;
}
/*
* All-CPU rendezvous. CPUs are signalled, all execute the setup function
* (if specified), rendezvous, execute the action function (if specified),
* rendezvous again, execute the teardown function (if specified), and then
* resume.
*
* Note that the supplied external functions _must_ be reentrant and aware
* that they are running in parallel and in an unknown lock context.
*/
static void (*smp_rv_setup_func)(void *arg);
static void (*smp_rv_action_func)(void *arg);
static void (*smp_rv_teardown_func)(void *arg);
static void *smp_rv_func_arg;
static volatile int smp_rv_waiters[2];
void
smp_rendezvous_action(void)
{
/* setup function */
if (smp_rv_setup_func != NULL)
smp_rv_setup_func(smp_rv_func_arg);
/* spin on entry rendezvous */
atomic_add_int(&smp_rv_waiters[0], 1);
while (smp_rv_waiters[0] < mp_ncpus)
alpha_mb();
/* action function */
if (smp_rv_action_func != NULL)
smp_rv_action_func(smp_rv_func_arg);
/* spin on exit rendezvous */
atomic_add_int(&smp_rv_waiters[1], 1);
while (smp_rv_waiters[1] < mp_ncpus)
alpha_mb();
/* teardown function */
if (smp_rv_teardown_func != NULL)
smp_rv_teardown_func(smp_rv_func_arg);
}
void
smp_rendezvous(void (* setup_func)(void *),
void (* action_func)(void *),
void (* teardown_func)(void *),
void *arg)
{
if (!smp_started) {
if (setup_func != NULL)
setup_func(arg);
if (action_func != NULL)
action_func(arg);
if (teardown_func != NULL)
teardown_func(arg);
return;
}
/* obtain rendezvous lock */
mtx_lock_spin(&smp_rv_mtx);
/* set static function pointers */
smp_rv_setup_func = setup_func;
smp_rv_action_func = action_func;
smp_rv_teardown_func = teardown_func;
smp_rv_func_arg = arg;
smp_rv_waiters[0] = 0;
smp_rv_waiters[1] = 0;
/* signal other processors, which will enter the IPI with interrupts off */
ipi_all_but_self(IPI_RENDEZVOUS);
/* call executor function */
CTR2(KTR_SMP, "smp_rv: calling action = %p, arg = %p\n", action_func,
arg);
smp_rendezvous_action();
/* release lock */
mtx_unlock_spin(&smp_rv_mtx);
}
/*
@ -979,7 +393,7 @@ ipi_selected(u_int32_t cpus, u_int64_t ipi)
int cpuid = ffs(cpus) - 1;
cpus &= ~(1 << cpuid);
globaldata = cpuid_to_globaldata[cpuid];
globaldata = globaldata_find(cpuid);
if (globaldata) {
atomic_set_64(&globaldata->gd_pending_ipis, ipi);
alpha_mb();
@ -1048,25 +462,6 @@ smp_handle_ipi(struct trapframe *frame)
case IPI_AST:
CTR0(KTR_SMP, "IPI_AST");
atomic_clear_int(&checkstate_need_ast, cpumask);
mtx_lock_spin(&sched_lock);
curproc->p_sflag |= PS_ASTPENDING;
mtx_unlock_spin(&sched_lock);
break;
case IPI_CHECKSTATE:
CTR0(KTR_SMP, "IPI_CHECKSTATE");
if (frame->tf_regs[FRAME_PS] & ALPHA_PSL_USERMODE)
checkstate_cpustate[PCPU_GET(cpuid)] =
CHECKSTATE_USER;
else if (curproc->p_intr_nesting_level == 1)
checkstate_cpustate[PCPU_GET(cpuid)] =
CHECKSTATE_SYS;
else
checkstate_cpustate[PCPU_GET(cpuid)] =
CHECKSTATE_INTR;
checkstate_curproc[PCPU_GET(cpuid)] = curproc;
atomic_set_int(&checkstate_probed_cpus, cpumask);
break;
case IPI_STOP:
@ -1085,7 +480,7 @@ smp_handle_ipi(struct trapframe *frame)
* requests to provide PALcode to secondaries and to start up new
* secondaries that are added to the system on the fly.
*/
if (PCPU_GET(cpuid) == hwrpb->rpb_primary_cpu_id) {
if (PCPU_GET(cpuid) == boot_cpu_id) {
u_int cpuid;
u_int64_t txrdy;
#ifdef DIAGNOSTIC

2
sys/alpha/alpha/pmap.c
View File

@ -154,6 +154,7 @@
#include <sys/msgbuf.h>
#include <sys/vmmeter.h>
#include <sys/mman.h>
#include <sys/smp.h>
#include <sys/sx.h>
#include <vm/vm.h>
@ -172,7 +173,6 @@
#include <machine/md_var.h>
#include <machine/rpb.h>
#include <machine/smp.h>
#ifndef PMAP_SHPGPERPROC
#define PMAP_SHPGPERPROC 200

6
sys/alpha/alpha/trap.c
View File

@ -42,6 +42,7 @@
#include <sys/exec.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/smp.h>
#include <sys/vmmeter.h>
#include <sys/sysent.h>
#include <sys/syscall.h>
@ -59,7 +60,6 @@
#include <machine/reg.h>
#include <machine/pal.h>
#include <machine/fpu.h>
#include <machine/smp.h>
#ifdef KTRACE
#include <sys/uio.h>
@ -150,7 +150,7 @@ userret(p, frame, oticks)
mtx_lock_spin(&sched_lock);
p->p_pri.pri_level = p->p_pri.pri_user;
if (resched_wanted()) {
if (resched_wanted(p)) {
/*
* Since we are curproc, a clock interrupt could
* change our priority without changing run queues
@ -877,7 +877,7 @@ ast(framep)
* acquiring and releasing mutexes in assembly is not fun.
*/
mtx_lock_spin(&sched_lock);
if (!(astpending(p) || resched_wanted())) {
if (!(astpending(p) || resched_wanted(p))) {
mtx_unlock_spin(&sched_lock);
return;
}

1
sys/alpha/include/cpu.h
View File

@ -68,7 +68,6 @@ struct clockframe {
#define CLKF_USERMODE(framep) TRAPF_USERMODE(&(framep)->cf_tf)
#define CLKF_PC(framep) TRAPF_PC(&(framep)->cf_tf)
#define CLKF_INTR(framep) (curproc->p_intr_nesting_level >= 2)
/*
* Arrange to handle pending profiling ticks before returning to user mode.

7
sys/alpha/include/globaldata.h
View File

@ -66,14 +66,7 @@ struct globaldata {
#endif
};
SLIST_HEAD(cpuhead, globaldata);
extern struct cpuhead cpuhead;
void globaldata_init(struct globaldata *pcpu, int cpuid, size_t sz);
struct globaldata *globaldata_find(int cpuid);
#ifdef SMP
void globaldata_register(struct globaldata *pcpu);
#endif
#endif /* _KERNEL */

19
sys/alpha/include/ipl.h
View File

@ -30,23 +30,4 @@
#define _MACHINE_IPL_H_
#include <machine/cpu.h> /* for pal inlines */
#ifdef SMP
/*
* Interprocessor interrupts for SMP.
*/
#define IPI_INVLTLB 0x0001
#define IPI_RENDEZVOUS 0x0002
#define IPI_AST 0x0004
#define IPI_CHECKSTATE 0x0008
#define IPI_STOP 0x0010
void ipi_selected(u_int32_t cpus, u_int64_t ipi);
void ipi_all(u_int64_t ipi);
void ipi_all_but_self(u_int64_t ipi);
void ipi_self(u_int64_t ipi);
void smp_handle_ipi(struct trapframe *frame);
#endif
#endif /* !_MACHINE_IPL_H_ */

7
sys/alpha/include/pcpu.h
View File

@ -66,14 +66,7 @@ struct globaldata {
#endif
};
SLIST_HEAD(cpuhead, globaldata);
extern struct cpuhead cpuhead;
void globaldata_init(struct globaldata *pcpu, int cpuid, size_t sz);
struct globaldata *globaldata_find(int cpuid);
#ifdef SMP
void globaldata_register(struct globaldata *pcpu);
#endif
#endif /* _KERNEL */

45
sys/alpha/include/smp.h
View File

@ -15,41 +15,24 @@
#ifdef _KERNEL
#include <sys/ipl.h>
#include <sys/ktr.h>
#include <sys/mutex.h>
/*
* Interprocessor interrupts for SMP.
*/
#define IPI_INVLTLB 0x0001
#define IPI_RENDEZVOUS 0x0002
#define IPI_AST 0x0004
#define IPI_CHECKSTATE 0x0008
#define IPI_STOP 0x0010
#ifndef LOCORE
#define BETTER_CLOCK /* unconditional on alpha */
extern u_int boot_cpu_id;
/* global data in mp_machdep.c */
extern volatile u_int checkstate_probed_cpus;
extern volatile u_int checkstate_need_ast;
extern volatile u_int resched_cpus;
extern void (*cpustop_restartfunc) __P((void));
extern int smp_active;
extern int mp_ncpus;
extern u_int all_cpus;
extern u_int started_cpus;
extern u_int stopped_cpus;
/* functions in mp_machdep.c */
void mp_start(void);
void mp_announce(void);
void smp_invltlb(void);
void forward_statclock(int pscnt);
void forward_hardclock(int pscnt);
void forward_signal(struct proc *);
void forward_roundrobin(void);
int stop_cpus(u_int);
int restart_cpus(u_int);
void smp_rendezvous_action(void);
void smp_rendezvous(void (*)(void *),
void (*)(void *),
void (*)(void *),
void *arg);
void ipi_selected(u_int cpus, u_int64_t ipi);
void ipi_all(u_int64_t ipi);
void ipi_all_but_self(u_int64_t ipi);
void ipi_self(u_int64_t ipi);
void smp_handle_ipi(struct trapframe *frame);
void smp_init_secondary(void);
#endif /* !LOCORE */

5
sys/amd64/amd64/amd64_mem.c
View File

@ -31,14 +31,11 @@
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/memrange.h>
#include <sys/smp.h>
#include <machine/md_var.h>
#include <machine/specialreg.h>
#ifdef SMP
#include <machine/smp.h>
#endif
/*
* i686 memory range operations
*

155
sys/amd64/amd64/apic_vector.S
View File

@ -182,7 +182,6 @@ Xspuriousint:
iret
/*
* Handle TLB shootdowns.
*/
@ -211,71 +210,61 @@ Xinvltlb:
popl %eax
iret
/*
* Executed by a CPU when it receives an Xcpucheckstate IPI from another CPU,
*
* - Stores current cpu state in checkstate_cpustate[cpuid]
* 0 == user, 1 == sys, 2 == intr
* - Stores current process in checkstate_curproc[cpuid]
*
* - Signals its receipt by setting bit cpuid in checkstate_probed_cpus.
*
* stack: 0->ds, 4->fs, 8->ebx, 12->eax, 16->eip, 20->cs, 24->eflags
* Forward hardclock to another CPU. Pushes a trapframe and calls
* forwarded_hardclock().
*/
.text
SUPERALIGN_TEXT
.globl Xcpucheckstate
.globl checkstate_cpustate
.globl checkstate_curproc
.globl checkstate_pc
Xcpucheckstate:
pushl %eax
pushl %ebx
pushl %ds /* save current data segment */
pushl %fs
movl $KDSEL, %eax
mov %ax, %ds /* use KERNEL data segment */
.globl Xhardclock
Xhardclock:
PUSH_FRAME
movl $KDSEL, %eax /* reload with kernel's data segment */
mov %ax, %ds
mov %ax, %es
movl $KPSEL, %eax
mov %ax, %fs
movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */
movl $0, %ebx
movl 20(%esp), %eax
andl $3, %eax
cmpl $3, %eax
je 1f
testl $PSL_VM, 24(%esp)
jne 1f
incl %ebx /* system or interrupt */
1:
movl PCPU(CPUID), %eax
movl %ebx, checkstate_cpustate(,%eax,4)
movl PCPU(CURPROC), %ebx
movl %ebx, checkstate_curproc(,%eax,4)
movl PCPU(CURPROC),%ebx
incl P_INTR_NESTING_LEVEL(%ebx)
call forwarded_hardclock
decl P_INTR_NESTING_LEVEL(%ebx)
MEXITCOUNT
jmp doreti
movl 16(%esp), %ebx
movl %ebx, checkstate_pc(,%eax,4)
/*
* Forward statclock to another CPU. Pushes a trapframe and calls
* forwarded_statclock().
*/
.text
SUPERALIGN_TEXT
.globl Xstatclock
Xstatclock:
PUSH_FRAME
movl $KDSEL, %eax /* reload with kernel's data segment */
mov %ax, %ds
mov %ax, %es
movl $KPSEL, %eax
mov %ax, %fs
lock /* checkstate_probed_cpus |= (1<<id) */
btsl %eax, checkstate_probed_cpus
popl %fs
popl %ds /* restore previous data segment */
popl %ebx
popl %eax
iret
movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */
FAKE_MCOUNT(13*4(%esp))
movl PCPU(CURPROC),%ebx
incl P_INTR_NESTING_LEVEL(%ebx)
call forwarded_statclock
decl P_INTR_NESTING_LEVEL(%ebx)
MEXITCOUNT
jmp doreti
/*
* Executed by a CPU when it receives an Xcpuast IPI from another CPU,
*
* - Signals its receipt by clearing bit cpuid in checkstate_need_ast.
*
* - We need a better method of triggering asts on other cpus.
* The other CPU has already executed aston() or need_resched() on our
* current process, so we simply need to ack the interrupt and return
* via doreti to run ast().
*/
.text
@ -289,40 +278,12 @@ Xcpuast:
movl $KPSEL, %eax
mov %ax, %fs
movl PCPU(CPUID), %eax
lock /* checkstate_need_ast &= ~(1<<id) */
btrl %eax, checkstate_need_ast
movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */
lock
btsl %eax, checkstate_pending_ast
jc 1f
FAKE_MCOUNT(13*4(%esp))
MTX_LOCK_SPIN(sched_lock, 0)
movl PCPU(CURPROC),%ebx
orl $PS_ASTPENDING, P_SFLAG(%ebx)
movl PCPU(CPUID), %eax
lock
btrl %eax, checkstate_pending_ast
lock
btrl %eax, CNAME(resched_cpus)
jnc 2f
orl $PS_NEEDRESCHED, P_SFLAG(%ebx)
lock
incl CNAME(want_resched_cnt)
2:
MTX_UNLOCK_SPIN(sched_lock)
lock
incl CNAME(cpuast_cnt)
MEXITCOUNT
jmp doreti
1:
/* We are already in the process of delivering an ast for this CPU */
POP_FRAME
iret
/*
* Executed by a CPU when it receives an Xcpustop IPI from another CPU,
@ -331,7 +292,6 @@ Xcpuast:
* - Waits for permission to restart.
* - Signals its restart.
*/
.text
SUPERALIGN_TEXT
.globl Xcpustop
@ -357,20 +317,19 @@ Xcpustop:
pushl %eax
call CNAME(savectx) /* Save process context */
addl $4, %esp
movl PCPU(CPUID), %eax
lock
btsl %eax, stopped_cpus /* stopped_cpus |= (1<<id) */
btsl %eax, CNAME(stopped_cpus) /* stopped_cpus |= (1<<id) */
1:
btl %eax, started_cpus /* while (!(started_cpus & (1<<id))) */
btl %eax, CNAME(started_cpus) /* while (!(started_cpus & (1<<id))) */
jnc 1b
lock
btrl %eax, started_cpus /* started_cpus &= ~(1<<id) */
btrl %eax, CNAME(started_cpus) /* started_cpus &= ~(1<<id) */
lock
btrl %eax, stopped_cpus /* stopped_cpus &= ~(1<<id) */
btrl %eax, CNAME(stopped_cpus) /* stopped_cpus &= ~(1<<id) */
test %eax, %eax
jnz 2f
@ -492,34 +451,6 @@ _xhits:
.space (NCPU * 4), 0
#endif /* COUNT_XINVLTLB_HITS */
/* variables used by stop_cpus()/restart_cpus()/Xcpustop */
.globl stopped_cpus, started_cpus
stopped_cpus:
.long 0
started_cpus:
.long 0
.globl checkstate_probed_cpus
checkstate_probed_cpus:
.long 0
.globl checkstate_need_ast
checkstate_need_ast:
.long 0
checkstate_pending_ast:
.long 0
.globl CNAME(resched_cpus)
.globl CNAME(want_resched_cnt)
.globl CNAME(cpuast_cnt)
.globl CNAME(cpustop_restartfunc)
CNAME(resched_cpus):
.long 0
CNAME(want_resched_cnt):
.long 0
CNAME(cpuast_cnt):
.long 0
CNAME(cpustop_restartfunc):
.long 0
.globl apic_pin_trigger
apic_pin_trigger:
.long 0

61
sys/amd64/amd64/db_interface.c
View File

@ -37,10 +37,10 @@
#include <sys/linker_set.h>
#include <sys/lock.h>
#include <sys/proc.h>
#include <sys/smp.h>
#include <machine/cpu.h>
#ifdef SMP
#include <machine/smp.h>
#include <machine/smptests.h> /** CPUSTOP_ON_DDBBREAK */
#endif
@ -334,43 +334,44 @@ Debugger(msg)
DB_SHOW_COMMAND(pcpu, db_show_pcpu)
{
struct globaldata *gd;
#ifdef SMP
int id;
if (have_addr)
id = ((addr >> 4) % 16) * 10 + (addr % 16);
else
id = PCPU_GET(cpuid);
SLIST_FOREACH(gd, &cpuhead, gd_allcpu) {
if (gd->gd_cpuid == id)
break;
}
if (gd == NULL)
gd = globaldata_find(id);
if (gd == NULL) {
db_printf("CPU %d not found\n", id);
else {
db_printf("cpuid = %d\n", gd->gd_cpuid);
db_printf("curproc = ");
if (gd->gd_curproc != NULL)
db_printf("%p: pid %d \"%s\"\n", gd->gd_curproc,
gd->gd_curproc->p_pid, gd->gd_curproc->p_comm);
else
db_printf("none\n");
db_printf("curpcb = %p\n", gd->gd_curpcb);
db_printf("npxproc = ");
if (gd->gd_npxproc != NULL)
db_printf("%p: pid %d \"%s\"\n", gd->gd_npxproc,
gd->gd_npxproc->p_pid, gd->gd_npxproc->p_comm);
else
db_printf("none\n");
db_printf("idleproc = ");
if (gd->gd_idleproc != NULL)
db_printf("%p: pid %d \"%s\"\n", gd->gd_idleproc,
gd->gd_idleproc->p_pid, gd->gd_idleproc->p_comm);
else
db_printf("none\n");
return;
}
#else
gd = GLOBALDATA;
#endif
db_printf("cpuid = %d\n", gd->gd_cpuid);
db_printf("curproc = ");
if (gd->gd_curproc != NULL)
db_printf("%p: pid %d \"%s\"\n", gd->gd_curproc,
gd->gd_curproc->p_pid, gd->gd_curproc->p_comm);
else
db_printf("none\n");
db_printf("curpcb = %p\n", gd->gd_curpcb);
db_printf("npxproc = ");
if (gd->gd_npxproc != NULL)
db_printf("%p: pid %d \"%s\"\n", gd->gd_npxproc,
gd->gd_npxproc->p_pid, gd->gd_npxproc->p_comm);
else
db_printf("none\n");
db_printf("idleproc = ");
if (gd->gd_idleproc != NULL)
db_printf("%p: pid %d \"%s\"\n", gd->gd_idleproc,
gd->gd_idleproc->p_pid, gd->gd_idleproc->p_comm);
else
db_printf("none\n");
#ifdef WITNESS
db_printf("spin locks held:\n");
witness_list_locks(&gd->gd_spinlocks);
db_printf("spin locks held:\n");
witness_list_locks(&gd->gd_spinlocks);
#endif
}
}

21
sys/amd64/amd64/machdep.c
View File

@ -66,6 +66,7 @@
#include <sys/bio.h>
#include <sys/buf.h>
#include <sys/reboot.h>
#include <sys/smp.h>
#include <sys/callout.h>
#include <sys/msgbuf.h>
#include <sys/sysent.h>
@ -104,9 +105,6 @@
#include <machine/globaldata.h>
#include <machine/globals.h>
#include <machine/intrcnt.h>
#ifdef SMP
#include <machine/smp.h>
#endif
#ifdef PERFMON
#include <machine/perfmon.h>
#endif
@ -249,8 +247,6 @@ static struct trapframe proc0_tf;
static struct globaldata __globaldata;
#endif
struct cpuhead cpuhead;
struct mtx sched_lock;
struct mtx Giant;
@ -441,17 +437,12 @@ cpu_startup(dummy)
bufinit();
vm_pager_bufferinit();
SLIST_INIT(&cpuhead);
SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu);
#ifdef SMP
/*
* OK, enough kmem_alloc/malloc state should be up, lets get on with it!
*/
mp_start(); /* fire up the APs and APICs */
mp_announce();
#endif /* SMP */
globaldata_register(GLOBALDATA);
#else
/* For SMP, we delay the cpu_setregs() until after SMP startup. */
cpu_setregs();
#endif
}
/*
@ -1631,7 +1622,7 @@ getmemsize(int first)
physmap[1] = mp_bootaddress(physmap[1] / 1024);
/* look for the MP hardware - needed for apic addresses */
mp_probe();
i386_mp_probe();
#endif
/*

624
sys/amd64/amd64/mp_machdep.c
View File

@ -42,6 +42,7 @@
#include <sys/malloc.h>
#include <sys/memrange.h>
#include <sys/mutex.h>
#include <sys/smp.h>
#include <sys/dkstat.h>
#include <sys/cons.h> /* cngetc() */
@ -57,9 +58,9 @@
#include <sys/gmon.h>
#endif
#include <machine/smp.h>
#include <machine/apic.h>
#include <machine/atomic.h>
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/ipl.h>
#include <machine/mpapic.h>
@ -243,7 +244,6 @@ int current_postcode;
extern struct region_descriptor r_gdt, r_idt;
int bsp_apic_ready = 0; /* flags useability of BSP apic */
int mp_ncpus; /* # of CPUs, including BSP */
int mp_naps; /* # of Applications processors */
int mp_nbusses; /* # of busses */
int mp_napics; /* # of IO APICs */
@ -273,9 +273,6 @@ int io_num_to_apic_id[NAPICID];
int apic_id_to_logical[NAPICID];
/* Bitmap of all available CPUs */
u_int all_cpus;
/* AP uses this during bootstrap. Do not staticize. */
char *bootSTK;
static int bootAP;
@ -288,28 +285,9 @@ extern pt_entry_t *SMPpt;
struct pcb stoppcbs[MAXCPU];
int smp_started; /* has the system started? */
int smp_active = 0; /* are the APs allowed to run? */
SYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RW, &smp_active, 0, "");
/* XXX maybe should be hw.ncpu */
static int smp_cpus = 1; /* how many cpu's running */
SYSCTL_INT(_machdep, OID_AUTO, smp_cpus, CTLFLAG_RD, &smp_cpus, 0, "");
int invltlb_ok = 0; /* throttle smp_invltlb() till safe */
SYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, "");
/* Enable forwarding of a signal to a process running on a different CPU */
static int forward_signal_enabled = 1;
SYSCTL_INT(_machdep, OID_AUTO, forward_signal_enabled, CTLFLAG_RW,
&forward_signal_enabled, 0, "");
/* Enable forwarding of roundrobin to all other cpus */
static int forward_roundrobin_enabled = 1;
SYSCTL_INT(_machdep, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW,
&forward_roundrobin_enabled, 0, "");
/*
* Local data and functions.
*/
@ -354,9 +332,6 @@ struct mtx mcount_mtx;
struct mtx com_mtx;
#endif /* USE_COMLOCK */
/* lock around the MP rendezvous */
static struct mtx smp_rv_mtx;
static void
init_locks(void)
{
@ -367,13 +342,9 @@ init_locks(void)
*/
mtx_init(&mcount_mtx, "mcount", MTX_DEF);
mtx_init(&smp_rv_mtx, "smp rendezvous", MTX_SPIN);
#ifdef USE_COMLOCK
mtx_init(&com_mtx, "com", MTX_SPIN);
#endif /* USE_COMLOCK */
mtx_init(&ap_boot_mtx, "ap boot", MTX_SPIN);
}
/*
@ -397,8 +368,8 @@ mp_bootaddress(u_int basemem)
/*
* Look for an Intel MP spec table (ie, SMP capable hardware).
*/
int
mp_probe(void)
void
i386_mp_probe(void)
{
int x;
u_long segment;
@ -427,7 +398,7 @@ mp_probe(void)
/* nothing found */
mpfps = (mpfps_t)0;
mp_capable = 0;
return 0;
return;
found:
/* calculate needed resources */
@ -436,15 +407,19 @@ mp_probe(void)
/* flag fact that we are running multiple processors */
mp_capable = 1;
return 1;
}
int
cpu_mp_probe(void)
{
return (mp_capable);
}
/*
* Initialize the SMP hardware and the APIC and start up the AP's.
*/
void
mp_start(void)
cpu_mp_start(void)
{
POSTCODE(MP_START_POST);
@ -453,6 +428,8 @@ mp_start(void)
mp_enable(boot_address);
else
panic("MP hardware not found!");
cpu_setregs();
}
@ -460,13 +437,12 @@ mp_start(void)
* Print various information about the SMP system hardware and setup.
*/
void
mp_announce(void)
cpu_mp_announce(void)
{
int x;
POSTCODE(MP_ANNOUNCE_POST);
printf("FreeBSD/SMP: Multiprocessor motherboard\n");
printf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0));
printf(", version: 0x%08x", cpu_apic_versions[0]);
printf(", at 0x%08x\n", cpu_apic_address);
@ -623,8 +599,12 @@ mp_enable(u_int boot_addr)
setidt(XINVLTLB_OFFSET, Xinvltlb,
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
/* install an inter-CPU IPI for reading processor state */
setidt(XCPUCHECKSTATE_OFFSET, Xcpucheckstate,
/* install an inter-CPU IPI for forwarding hardclock() */
setidt(XHARDCLOCK_OFFSET, Xhardclock,
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
/* install an inter-CPU IPI for forwarding statclock() */
setidt(XSTATCLOCK_OFFSET, Xstatclock,
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
/* install an inter-CPU IPI for all-CPU rendezvous */
@ -1938,6 +1918,8 @@ start_all_aps(u_int boot_addr)
POSTCODE(START_ALL_APS_POST);
mtx_init(&ap_boot_mtx, "ap boot", MTX_SPIN);
/* initialize BSP's local APIC */
apic_initialize();
bsp_apic_ready = 1;
@ -1985,8 +1967,8 @@ start_all_aps(u_int boot_addr)
(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
/* prime data page for it to use */
SLIST_INSERT_HEAD(&cpuhead, gd, gd_allcpu);
gd->gd_cpuid = x;
globaldata_register(gd);
/* setup a vector to our boot code */
*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
@ -2328,472 +2310,65 @@ ap_init(void)
panic("scheduler returned us to ap_init");
}
#define CHECKSTATE_USER 0
#define CHECKSTATE_SYS 1
#define CHECKSTATE_INTR 2
/* Do not staticize. Used from apic_vector.s */
struct proc* checkstate_curproc[MAXCPU];
int checkstate_cpustate[MAXCPU];
u_long checkstate_pc[MAXCPU];
#define PC_TO_INDEX(pc, prof) \
((int)(((u_quad_t)((pc) - (prof)->pr_off) * \
(u_quad_t)((prof)->pr_scale)) >> 16) & ~1)
static void
addupc_intr_forwarded(struct proc *p, int id, int *astmap)
{
int i;
struct uprof *prof;
u_long pc;
pc = checkstate_pc[id];
prof = &p->p_stats->p_prof;
if (pc >= prof->pr_off &&
(i = PC_TO_INDEX(pc, prof)) < prof->pr_size) {
mtx_assert(&sched_lock, MA_OWNED);
if ((p->p_sflag & PS_OWEUPC) == 0) {
prof->pr_addr = pc;
prof->pr_ticks = 1;
p->p_sflag |= PS_OWEUPC;
}
*astmap |= (1 << id);
}
}
static void
forwarded_statclock(int id, int pscnt, int *astmap)
{
struct pstats *pstats;
long rss;
struct rusage *ru;
struct vmspace *vm;
int cpustate;
struct proc *p;
#ifdef GPROF
register struct gmonparam *g;
int i;
#endif
mtx_assert(&sched_lock, MA_OWNED);
p = checkstate_curproc[id];
cpustate = checkstate_cpustate[id];
/* XXX */
if (p->p_ithd)
cpustate = CHECKSTATE_INTR;
else if (p == SMP_prvspace[id].globaldata.gd_idleproc)
cpustate = CHECKSTATE_SYS;
switch (cpustate) {
case CHECKSTATE_USER:
if (p->p_sflag & PS_PROFIL)
addupc_intr_forwarded(p, id, astmap);
if (pscnt > 1)
return;
p->p_uticks++;
if (p->p_nice > NZERO)
cp_time[CP_NICE]++;
else
cp_time[CP_USER]++;
break;
case CHECKSTATE_SYS:
#ifdef GPROF
/*
* Kernel statistics are just like addupc_intr, only easier.
*/
g = &_gmonparam;
if (g->state == GMON_PROF_ON) {
i = checkstate_pc[id] - g->lowpc;
if (i < g->textsize) {
i /= HISTFRACTION * sizeof(*g->kcount);
g->kcount[i]++;
}
}
#endif
if (pscnt > 1)
return;
p->p_sticks++;
if (p == SMP_prvspace[id].globaldata.gd_idleproc)
cp_time[CP_IDLE]++;
else
cp_time[CP_SYS]++;
break;
case CHECKSTATE_INTR:
default:
#ifdef GPROF
/*
* Kernel statistics are just like addupc_intr, only easier.
*/
g = &_gmonparam;
if (g->state == GMON_PROF_ON) {
i = checkstate_pc[id] - g->lowpc;
if (i < g->textsize) {
i /= HISTFRACTION * sizeof(*g->kcount);
g->kcount[i]++;
}
}
#endif
if (pscnt > 1)
return;
KASSERT(p != NULL, ("NULL process in interrupt state"));
p->p_iticks++;
cp_time[CP_INTR]++;
}
schedclock(p);
/* Update resource usage integrals and maximums. */
if ((pstats = p->p_stats) != NULL &&
(ru = &pstats->p_ru) != NULL &&
(vm = p->p_vmspace) != NULL) {
ru->ru_ixrss += pgtok(vm->vm_tsize);
ru->ru_idrss += pgtok(vm->vm_dsize);
ru->ru_isrss += pgtok(vm->vm_ssize);
rss = pgtok(vmspace_resident_count(vm));
if (ru->ru_maxrss < rss)
ru->ru_maxrss = rss;
}
}
/*
* For statclock, we send an IPI to all CPU's to have them call this
* function.
*/
void
forward_statclock(int pscnt)
forwarded_statclock(struct trapframe frame)
{
int map;
int id;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_statclock(%d)", pscnt);
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
/* Step 1: Probe state (user, cpu, interrupt, spinlock, idle ) */
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
checkstate_probed_cpus = 0;
if (map != 0)
ipi_selected(map, IPI_CHECKSTATE);
i = 0;
while (checkstate_probed_cpus != map) {
/* spin */
i++;
if (i == 100000) {
#ifdef DIAGNOSTIC
printf("forward_statclock: checkstate %x\n",
checkstate_probed_cpus);
#endif
break;
}
}
/*
* Step 2: walk through other processors processes, update ticks and
* profiling info.
*/
map = 0;
for (id = 0; id < mp_ncpus; id++) {
if (id == PCPU_GET(cpuid))
continue;
if (((1 << id) & checkstate_probed_cpus) == 0)
continue;
forwarded_statclock(id, pscnt, &map);
}
if (map != 0) {
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#ifdef DIAGNOSTIC
printf("forward_statclock: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
}
}
void
forward_hardclock(int pscnt)
{
int map;
int id;
struct proc *p;
struct pstats *pstats;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_hardclock(%d)", pscnt);
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
/* Step 1: Probe state (user, cpu, interrupt, spinlock, idle) */
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
checkstate_probed_cpus = 0;
if (map != 0)
ipi_selected(map, IPI_CHECKSTATE);
i = 0;
while (checkstate_probed_cpus != map) {
/* spin */
i++;
if (i == 100000) {
#ifdef DIAGNOSTIC
printf("forward_hardclock: checkstate %x\n",
checkstate_probed_cpus);
#endif
break;
}
}
/*
* Step 2: walk through other processors processes, update virtual
* timer and profiling timer. If stathz == 0, also update ticks and
* profiling info.
*/
map = 0;
for (id = 0; id < mp_ncpus; id++) {
if (id == PCPU_GET(cpuid))
continue;
if (((1 << id) & checkstate_probed_cpus) == 0)
continue;
p = checkstate_curproc[id];
if (p) {
pstats = p->p_stats;
if (checkstate_cpustate[id] == CHECKSTATE_USER &&
timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) &&
itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) {
p->p_sflag |= PS_ALRMPEND;
map |= (1 << id);
}
if (timevalisset(&pstats->p_timer[ITIMER_PROF].it_value) &&
itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) {
p->p_sflag |= PS_PROFPEND;
map |= (1 << id);
}
}
if (stathz == 0) {
forwarded_statclock( id, pscnt, &map);
}
}
if (map != 0) {
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#ifdef DIAGNOSTIC
printf("forward_hardclock: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
}
}
void
forward_signal(struct proc *p)
{
int map;
int id;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_signal(%p)", p);
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
if (!forward_signal_enabled)
return;
mtx_lock_spin(&sched_lock);
while (1) {
if (p->p_stat != SRUN) {
mtx_unlock_spin(&sched_lock);
return;
}
id = p->p_oncpu;
mtx_unlock_spin(&sched_lock);
if (id == 0xff)
return;
map = (1<<id);
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#if 0
printf("forward_signal: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
mtx_lock_spin(&sched_lock);
if (id == p->p_oncpu) {
mtx_unlock_spin(&sched_lock);
return;
}
}
statclock_process(curproc, TRAPF_PC(&frame), TRAPF_USERMODE(&frame));
mtx_unlock_spin(&sched_lock);
}
void
forward_roundrobin(void)
forward_statclock(void)
{
int map;
CTR0(KTR_SMP, "forward_statclock");
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
if (map != 0)
ipi_selected(map, IPI_STATCLOCK);
}
/*
* For each hardclock(), we send an IPI to all other CPU's to have them
* execute this function. It would be nice to reduce contention on
* sched_lock if we could simply peek at the CPU to determine the user/kernel
* state and call hardclock_process() on the CPU receiving the clock interrupt
* and then just use a simple IPI to handle any ast's if needed.
*/
void
forwarded_hardclock(struct trapframe frame)
{
mtx_lock_spin(&sched_lock);
hardclock_process(curproc, TRAPF_USERMODE(&frame));
mtx_unlock_spin(&sched_lock);
}
void
forward_hardclock(void)
{
u_int map;
int i;
CTR0(KTR_SMP, "forward_roundrobin()");
CTR0(KTR_SMP, "forward_hardclock");
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
if (!forward_roundrobin_enabled)
return;
resched_cpus |= PCPU_GET(other_cpus);
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
#if 1
ipi_selected(map, IPI_AST);
#else
ipi_all_but_self(IPI_AST);
#endif
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#if 0
printf("forward_roundrobin: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
if (map != 0)
ipi_selected(map, IPI_HARDCLOCK);
}
/*
* When called the executing CPU will send an IPI to all other CPUs
* requesting that they halt execution.
*
* Usually (but not necessarily) called with 'other_cpus' as its arg.
*
* - Signals all CPUs in map to stop.
* - Waits for each to stop.
*
* Returns:
* -1: error
* 0: NA
* 1: ok
*
* XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
* from executing at same time.
*/
int
stop_cpus(u_int map)
{
int count = 0;
if (!smp_started)
return 0;
/* send the Xcpustop IPI to all CPUs in map */
ipi_selected(map, IPI_STOP);
while (count++ < 100000 && (stopped_cpus & map) != map)
/* spin */ ;
#ifdef DIAGNOSTIC
if ((stopped_cpus & map) != map)
printf("Warning: CPUs 0x%x did not stop!\n",
(~(stopped_cpus & map)) & map);
#endif
return 1;
}
/*
* Called by a CPU to restart stopped CPUs.
*
* Usually (but not necessarily) called with 'stopped_cpus' as its arg.
*
* - Signals all CPUs in map to restart.
* - Waits for each to restart.
*
* Returns:
* -1: error
* 0: NA
* 1: ok
*/
int
restart_cpus(u_int map)
{
int count = 0;
if (!smp_started)
return 0;
started_cpus = map; /* signal other cpus to restart */
/* wait for each to clear its bit */
while (count++ < 100000 && (stopped_cpus & map) != 0)
/* spin */ ;
#ifdef DIAGNOSTIC
if ((stopped_cpus & map) != 0)
printf("Warning: CPUs 0x%x did not restart!\n",
(~(stopped_cpus & map)) & map);
#endif
return 1;
}
#ifdef APIC_INTR_REORDER
/*
* Maintain mapping from softintr vector to isr bit in local apic.
@ -2810,73 +2385,6 @@ set_lapic_isrloc(int intr, int vector)
}
#endif
/*
* All-CPU rendezvous. CPUs are signalled, all execute the setup function
* (if specified), rendezvous, execute the action function (if specified),
* rendezvous again, execute the teardown function (if specified), and then
* resume.
*
* Note that the supplied external functions _must_ be reentrant and aware
* that they are running in parallel and in an unknown lock context.
*/
static void (*smp_rv_setup_func)(void *arg);
static void (*smp_rv_action_func)(void *arg);
static void (*smp_rv_teardown_func)(void *arg);
static void *smp_rv_func_arg;
static volatile int smp_rv_waiters[2];
void
smp_rendezvous_action(void)
{
/* setup function */
if (smp_rv_setup_func != NULL)
smp_rv_setup_func(smp_rv_func_arg);
/* spin on entry rendezvous */
atomic_add_int(&smp_rv_waiters[0], 1);
while (smp_rv_waiters[0] < mp_ncpus)
;
/* action function */
if (smp_rv_action_func != NULL)
smp_rv_action_func(smp_rv_func_arg);
/* spin on exit rendezvous */
atomic_add_int(&smp_rv_waiters[1], 1);
while (smp_rv_waiters[1] < mp_ncpus)
;
/* teardown function */
if (smp_rv_teardown_func != NULL)
smp_rv_teardown_func(smp_rv_func_arg);
}
void
smp_rendezvous(void (* setup_func)(void *),
void (* action_func)(void *),
void (* teardown_func)(void *),
void *arg)
{
/* obtain rendezvous lock */
mtx_lock_spin(&smp_rv_mtx);
/* set static function pointers */
smp_rv_setup_func = setup_func;
smp_rv_action_func = action_func;
smp_rv_teardown_func = teardown_func;
smp_rv_func_arg = arg;
smp_rv_waiters[0] = 0;
smp_rv_waiters[1] = 0;
/*
* signal other processors, which will enter the IPI with interrupts off
*/
ipi_all_but_self(IPI_RENDEZVOUS);
/* call executor function */
smp_rendezvous_action();
/* release lock */
mtx_unlock_spin(&smp_rv_mtx);
}
/*
* send an IPI to a set of cpus.
*/

624
sys/amd64/amd64/mptable.c
View File

@ -42,6 +42,7 @@
#include <sys/malloc.h>
#include <sys/memrange.h>
#include <sys/mutex.h>
#include <sys/smp.h>
#include <sys/dkstat.h>
#include <sys/cons.h> /* cngetc() */
@ -57,9 +58,9 @@
#include <sys/gmon.h>
#endif
#include <machine/smp.h>
#include <machine/apic.h>
#include <machine/atomic.h>
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/ipl.h>
#include <machine/mpapic.h>
@ -243,7 +244,6 @@ int current_postcode;
extern struct region_descriptor r_gdt, r_idt;
int bsp_apic_ready = 0; /* flags useability of BSP apic */
int mp_ncpus; /* # of CPUs, including BSP */
int mp_naps; /* # of Applications processors */
int mp_nbusses; /* # of busses */
int mp_napics; /* # of IO APICs */
@ -273,9 +273,6 @@ int io_num_to_apic_id[NAPICID];
int apic_id_to_logical[NAPICID];
/* Bitmap of all available CPUs */
u_int all_cpus;
/* AP uses this during bootstrap. Do not staticize. */
char *bootSTK;
static int bootAP;
@ -288,28 +285,9 @@ extern pt_entry_t *SMPpt;
struct pcb stoppcbs[MAXCPU];
int smp_started; /* has the system started? */
int smp_active = 0; /* are the APs allowed to run? */
SYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RW, &smp_active, 0, "");
/* XXX maybe should be hw.ncpu */
static int smp_cpus = 1; /* how many cpu's running */
SYSCTL_INT(_machdep, OID_AUTO, smp_cpus, CTLFLAG_RD, &smp_cpus, 0, "");
int invltlb_ok = 0; /* throttle smp_invltlb() till safe */
SYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, "");
/* Enable forwarding of a signal to a process running on a different CPU */
static int forward_signal_enabled = 1;
SYSCTL_INT(_machdep, OID_AUTO, forward_signal_enabled, CTLFLAG_RW,
&forward_signal_enabled, 0, "");
/* Enable forwarding of roundrobin to all other cpus */
static int forward_roundrobin_enabled = 1;
SYSCTL_INT(_machdep, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW,
&forward_roundrobin_enabled, 0, "");
/*
* Local data and functions.
*/
@ -354,9 +332,6 @@ struct mtx mcount_mtx;
struct mtx com_mtx;
#endif /* USE_COMLOCK */
/* lock around the MP rendezvous */
static struct mtx smp_rv_mtx;
static void
init_locks(void)
{
@ -367,13 +342,9 @@ init_locks(void)
*/
mtx_init(&mcount_mtx, "mcount", MTX_DEF);
mtx_init(&smp_rv_mtx, "smp rendezvous", MTX_SPIN);
#ifdef USE_COMLOCK
mtx_init(&com_mtx, "com", MTX_SPIN);
#endif /* USE_COMLOCK */
mtx_init(&ap_boot_mtx, "ap boot", MTX_SPIN);
}
/*
@ -397,8 +368,8 @@ mp_bootaddress(u_int basemem)
/*
* Look for an Intel MP spec table (ie, SMP capable hardware).
*/
int
mp_probe(void)
void
i386_mp_probe(void)
{
int x;
u_long segment;
@ -427,7 +398,7 @@ mp_probe(void)
/* nothing found */
mpfps = (mpfps_t)0;
mp_capable = 0;
return 0;
return;
found:
/* calculate needed resources */
@ -436,15 +407,19 @@ mp_probe(void)
/* flag fact that we are running multiple processors */
mp_capable = 1;
return 1;
}
int
cpu_mp_probe(void)
{
return (mp_capable);
}
/*
* Initialize the SMP hardware and the APIC and start up the AP's.
*/
void
mp_start(void)
cpu_mp_start(void)
{
POSTCODE(MP_START_POST);
@ -453,6 +428,8 @@ mp_start(void)
mp_enable(boot_address);
else
panic("MP hardware not found!");
cpu_setregs();
}
@ -460,13 +437,12 @@ mp_start(void)
* Print various information about the SMP system hardware and setup.
*/
void
mp_announce(void)
cpu_mp_announce(void)
{
int x;
POSTCODE(MP_ANNOUNCE_POST);
printf("FreeBSD/SMP: Multiprocessor motherboard\n");
printf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0));
printf(", version: 0x%08x", cpu_apic_versions[0]);
printf(", at 0x%08x\n", cpu_apic_address);
@ -623,8 +599,12 @@ mp_enable(u_int boot_addr)
setidt(XINVLTLB_OFFSET, Xinvltlb,
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
/* install an inter-CPU IPI for reading processor state */
setidt(XCPUCHECKSTATE_OFFSET, Xcpucheckstate,
/* install an inter-CPU IPI for forwarding hardclock() */
setidt(XHARDCLOCK_OFFSET, Xhardclock,
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
/* install an inter-CPU IPI for forwarding statclock() */
setidt(XSTATCLOCK_OFFSET, Xstatclock,
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
/* install an inter-CPU IPI for all-CPU rendezvous */
@ -1938,6 +1918,8 @@ start_all_aps(u_int boot_addr)
POSTCODE(START_ALL_APS_POST);
mtx_init(&ap_boot_mtx, "ap boot", MTX_SPIN);
/* initialize BSP's local APIC */
apic_initialize();
bsp_apic_ready = 1;
@ -1985,8 +1967,8 @@ start_all_aps(u_int boot_addr)
(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
/* prime data page for it to use */
SLIST_INSERT_HEAD(&cpuhead, gd, gd_allcpu);
gd->gd_cpuid = x;
globaldata_register(gd);
/* setup a vector to our boot code */
*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
@ -2328,472 +2310,65 @@ ap_init(void)
panic("scheduler returned us to ap_init");
}
#define CHECKSTATE_USER 0
#define CHECKSTATE_SYS 1
#define CHECKSTATE_INTR 2
/* Do not staticize. Used from apic_vector.s */
struct proc* checkstate_curproc[MAXCPU];
int checkstate_cpustate[MAXCPU];
u_long checkstate_pc[MAXCPU];
#define PC_TO_INDEX(pc, prof) \
((int)(((u_quad_t)((pc) - (prof)->pr_off) * \
(u_quad_t)((prof)->pr_scale)) >> 16) & ~1)
static void
addupc_intr_forwarded(struct proc *p, int id, int *astmap)
{
int i;
struct uprof *prof;
u_long pc;
pc = checkstate_pc[id];
prof = &p->p_stats->p_prof;
if (pc >= prof->pr_off &&
(i = PC_TO_INDEX(pc, prof)) < prof->pr_size) {
mtx_assert(&sched_lock, MA_OWNED);
if ((p->p_sflag & PS_OWEUPC) == 0) {
prof->pr_addr = pc;
prof->pr_ticks = 1;
p->p_sflag |= PS_OWEUPC;
}
*astmap |= (1 << id);
}
}
static void
forwarded_statclock(int id, int pscnt, int *astmap)
{
struct pstats *pstats;
long rss;
struct rusage *ru;
struct vmspace *vm;
int cpustate;
struct proc *p;
#ifdef GPROF
register struct gmonparam *g;
int i;
#endif
mtx_assert(&sched_lock, MA_OWNED);
p = checkstate_curproc[id];
cpustate = checkstate_cpustate[id];
/* XXX */
if (p->p_ithd)
cpustate = CHECKSTATE_INTR;
else if (p == SMP_prvspace[id].globaldata.gd_idleproc)
cpustate = CHECKSTATE_SYS;
switch (cpustate) {
case CHECKSTATE_USER:
if (p->p_sflag & PS_PROFIL)
addupc_intr_forwarded(p, id, astmap);
if (pscnt > 1)
return;
p->p_uticks++;
if (p->p_nice > NZERO)
cp_time[CP_NICE]++;
else
cp_time[CP_USER]++;
break;
case CHECKSTATE_SYS:
#ifdef GPROF
/*
* Kernel statistics are just like addupc_intr, only easier.
*/
g = &_gmonparam;
if (g->state == GMON_PROF_ON) {
i = checkstate_pc[id] - g->lowpc;
if (i < g->textsize) {
i /= HISTFRACTION * sizeof(*g->kcount);
g->kcount[i]++;
}
}
#endif
if (pscnt > 1)
return;
p->p_sticks++;
if (p == SMP_prvspace[id].globaldata.gd_idleproc)
cp_time[CP_IDLE]++;
else
cp_time[CP_SYS]++;
break;
case CHECKSTATE_INTR:
default:
#ifdef GPROF
/*
* Kernel statistics are just like addupc_intr, only easier.
*/
g = &_gmonparam;
if (g->state == GMON_PROF_ON) {
i = checkstate_pc[id] - g->lowpc;
if (i < g->textsize) {
i /= HISTFRACTION * sizeof(*g->kcount);
g->kcount[i]++;
}
}
#endif
if (pscnt > 1)
return;
KASSERT(p != NULL, ("NULL process in interrupt state"));
p->p_iticks++;
cp_time[CP_INTR]++;
}
schedclock(p);
/* Update resource usage integrals and maximums. */
if ((pstats = p->p_stats) != NULL &&
(ru = &pstats->p_ru) != NULL &&
(vm = p->p_vmspace) != NULL) {
ru->ru_ixrss += pgtok(vm->vm_tsize);
ru->ru_idrss += pgtok(vm->vm_dsize);
ru->ru_isrss += pgtok(vm->vm_ssize);
rss = pgtok(vmspace_resident_count(vm));
if (ru->ru_maxrss < rss)
ru->ru_maxrss = rss;
}
}
/*
* For statclock, we send an IPI to all CPU's to have them call this
* function.
*/
void
forward_statclock(int pscnt)
forwarded_statclock(struct trapframe frame)
{
int map;
int id;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_statclock(%d)", pscnt);
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
/* Step 1: Probe state (user, cpu, interrupt, spinlock, idle ) */
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
checkstate_probed_cpus = 0;
if (map != 0)
ipi_selected(map, IPI_CHECKSTATE);
i = 0;
while (checkstate_probed_cpus != map) {
/* spin */
i++;
if (i == 100000) {
#ifdef DIAGNOSTIC
printf("forward_statclock: checkstate %x\n",
checkstate_probed_cpus);
#endif
break;
}
}
/*
* Step 2: walk through other processors processes, update ticks and
* profiling info.
*/
map = 0;
for (id = 0; id < mp_ncpus; id++) {
if (id == PCPU_GET(cpuid))
continue;
if (((1 << id) & checkstate_probed_cpus) == 0)
continue;
forwarded_statclock(id, pscnt, &map);
}
if (map != 0) {
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#ifdef DIAGNOSTIC
printf("forward_statclock: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
}
}
void
forward_hardclock(int pscnt)
{
int map;
int id;
struct proc *p;
struct pstats *pstats;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_hardclock(%d)", pscnt);
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
/* Step 1: Probe state (user, cpu, interrupt, spinlock, idle) */
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
checkstate_probed_cpus = 0;
if (map != 0)
ipi_selected(map, IPI_CHECKSTATE);
i = 0;
while (checkstate_probed_cpus != map) {
/* spin */
i++;
if (i == 100000) {
#ifdef DIAGNOSTIC
printf("forward_hardclock: checkstate %x\n",
checkstate_probed_cpus);
#endif
break;
}
}
/*
* Step 2: walk through other processors processes, update virtual
* timer and profiling timer. If stathz == 0, also update ticks and
* profiling info.
*/
map = 0;
for (id = 0; id < mp_ncpus; id++) {
if (id == PCPU_GET(cpuid))
continue;
if (((1 << id) & checkstate_probed_cpus) == 0)
continue;
p = checkstate_curproc[id];
if (p) {
pstats = p->p_stats;
if (checkstate_cpustate[id] == CHECKSTATE_USER &&
timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) &&
itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) {
p->p_sflag |= PS_ALRMPEND;
map |= (1 << id);
}
if (timevalisset(&pstats->p_timer[ITIMER_PROF].it_value) &&
itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) {
p->p_sflag |= PS_PROFPEND;
map |= (1 << id);
}
}
if (stathz == 0) {
forwarded_statclock( id, pscnt, &map);
}
}
if (map != 0) {
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#ifdef DIAGNOSTIC
printf("forward_hardclock: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
}
}
void
forward_signal(struct proc *p)
{
int map;
int id;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_signal(%p)", p);
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
if (!forward_signal_enabled)
return;
mtx_lock_spin(&sched_lock);
while (1) {
if (p->p_stat != SRUN) {
mtx_unlock_spin(&sched_lock);
return;
}
id = p->p_oncpu;
mtx_unlock_spin(&sched_lock);
if (id == 0xff)
return;
map = (1<<id);
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#if 0
printf("forward_signal: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
mtx_lock_spin(&sched_lock);
if (id == p->p_oncpu) {
mtx_unlock_spin(&sched_lock);
return;
}
}
statclock_process(curproc, TRAPF_PC(&frame), TRAPF_USERMODE(&frame));
mtx_unlock_spin(&sched_lock);
}
void
forward_roundrobin(void)
forward_statclock(void)
{
int map;
CTR0(KTR_SMP, "forward_statclock");
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
if (map != 0)
ipi_selected(map, IPI_STATCLOCK);
}
/*
* For each hardclock(), we send an IPI to all other CPU's to have them
* execute this function. It would be nice to reduce contention on
* sched_lock if we could simply peek at the CPU to determine the user/kernel
* state and call hardclock_process() on the CPU receiving the clock interrupt
* and then just use a simple IPI to handle any ast's if needed.
*/
void
forwarded_hardclock(struct trapframe frame)
{
mtx_lock_spin(&sched_lock);
hardclock_process(curproc, TRAPF_USERMODE(&frame));
mtx_unlock_spin(&sched_lock);
}
void
forward_hardclock(void)
{
u_int map;
int i;
CTR0(KTR_SMP, "forward_roundrobin()");
CTR0(KTR_SMP, "forward_hardclock");
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
if (!forward_roundrobin_enabled)
return;
resched_cpus |= PCPU_GET(other_cpus);
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
#if 1
ipi_selected(map, IPI_AST);
#else
ipi_all_but_self(IPI_AST);
#endif
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#if 0
printf("forward_roundrobin: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
if (map != 0)
ipi_selected(map, IPI_HARDCLOCK);
}
/*
* When called the executing CPU will send an IPI to all other CPUs
* requesting that they halt execution.
*
* Usually (but not necessarily) called with 'other_cpus' as its arg.
*
* - Signals all CPUs in map to stop.
* - Waits for each to stop.
*
* Returns:
* -1: error
* 0: NA
* 1: ok
*
* XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
* from executing at same time.
*/
int
stop_cpus(u_int map)
{
int count = 0;
if (!smp_started)
return 0;
/* send the Xcpustop IPI to all CPUs in map */
ipi_selected(map, IPI_STOP);
while (count++ < 100000 && (stopped_cpus & map) != map)
/* spin */ ;
#ifdef DIAGNOSTIC
if ((stopped_cpus & map) != map)
printf("Warning: CPUs 0x%x did not stop!\n",
(~(stopped_cpus & map)) & map);
#endif
return 1;
}
/*
* Called by a CPU to restart stopped CPUs.
*
* Usually (but not necessarily) called with 'stopped_cpus' as its arg.
*
* - Signals all CPUs in map to restart.
* - Waits for each to restart.
*
* Returns:
* -1: error
* 0: NA
* 1: ok
*/
int
restart_cpus(u_int map)
{
int count = 0;
if (!smp_started)
return 0;
started_cpus = map; /* signal other cpus to restart */
/* wait for each to clear its bit */
while (count++ < 100000 && (stopped_cpus & map) != 0)
/* spin */ ;
#ifdef DIAGNOSTIC
if ((stopped_cpus & map) != 0)
printf("Warning: CPUs 0x%x did not restart!\n",
(~(stopped_cpus & map)) & map);
#endif
return 1;
}
#ifdef APIC_INTR_REORDER
/*
* Maintain mapping from softintr vector to isr bit in local apic.
@ -2810,73 +2385,6 @@ set_lapic_isrloc(int intr, int vector)
}
#endif
/*
* All-CPU rendezvous. CPUs are signalled, all execute the setup function
* (if specified), rendezvous, execute the action function (if specified),
* rendezvous again, execute the teardown function (if specified), and then
* resume.
*
* Note that the supplied external functions _must_ be reentrant and aware
* that they are running in parallel and in an unknown lock context.
*/
static void (*smp_rv_setup_func)(void *arg);
static void (*smp_rv_action_func)(void *arg);
static void (*smp_rv_teardown_func)(void *arg);
static void *smp_rv_func_arg;
static volatile int smp_rv_waiters[2];
void
smp_rendezvous_action(void)
{
/* setup function */
if (smp_rv_setup_func != NULL)
smp_rv_setup_func(smp_rv_func_arg);
/* spin on entry rendezvous */
atomic_add_int(&smp_rv_waiters[0], 1);
while (smp_rv_waiters[0] < mp_ncpus)
;
/* action function */
if (smp_rv_action_func != NULL)
smp_rv_action_func(smp_rv_func_arg);
/* spin on exit rendezvous */
atomic_add_int(&smp_rv_waiters[1], 1);
while (smp_rv_waiters[1] < mp_ncpus)
;
/* teardown function */
if (smp_rv_teardown_func != NULL)
smp_rv_teardown_func(smp_rv_func_arg);
}
void
smp_rendezvous(void (* setup_func)(void *),
void (* action_func)(void *),
void (* teardown_func)(void *),
void *arg)
{
/* obtain rendezvous lock */
mtx_lock_spin(&smp_rv_mtx);
/* set static function pointers */
smp_rv_setup_func = setup_func;
smp_rv_action_func = action_func;
smp_rv_teardown_func = teardown_func;
smp_rv_func_arg = arg;
smp_rv_waiters[0] = 0;
smp_rv_waiters[1] = 0;
/*
* signal other processors, which will enter the IPI with interrupts off
*/
ipi_all_but_self(IPI_RENDEZVOUS);
/* call executor function */
smp_rendezvous_action();
/* release lock */
mtx_unlock_spin(&smp_rv_mtx);
}
/*
* send an IPI to a set of cpus.
*/

14
sys/amd64/amd64/sys_machdep.c
View File

@ -41,6 +41,7 @@
#include <sys/ipl.h>
#include <sys/malloc.h>
#include <sys/proc.h>
#include <sys/smp.h>
#include <vm/vm.h>
#include <sys/lock.h>
@ -53,9 +54,6 @@
#include <machine/cpu.h>
#include <machine/pcb_ext.h> /* pcb.h included by sys/user.h */
#include <machine/sysarch.h>
#ifdef SMP
#include <machine/smp.h>
#endif
#include <vm/vm_kern.h> /* for kernel_map */
@ -154,7 +152,15 @@ i386_extend_pcb(struct proc *p)
ssdtosd(&ssd, &ext->ext_tssd);
/* switch to the new TSS after syscall completes */
need_resched();
/*
* XXX: The sched_lock here needs to be over a slightly larger area.
* I have patches to more properly lock accesses to process ldt's
* and tss's that still need to be reviewed, but this keeps us from
* panic'ing on the mtx_assert() in need_resched() for the time being.
*/
mtx_lock_spin(&sched_lock);
need_resched(p);
mtx_unlock_spin(&sched_lock);
return 0;
}

4
sys/amd64/amd64/trap.c
View File

@ -177,7 +177,7 @@ userret(p, frame, oticks)
mtx_lock_spin(&sched_lock);
p->p_pri.pri_level = p->p_pri.pri_user;
if (resched_wanted()) {
if (resched_wanted(p)) {
/*
* Since we are curproc, clock will normally just change
* our priority without moving us from one queue to another
@ -1277,7 +1277,7 @@ ast(framep)
* acquiring and releasing mutexes in assembly is not fun.
*/
mtx_lock_spin(&sched_lock);
if (!(astpending(p) || resched_wanted())) {
if (!(astpending(p) || resched_wanted(p))) {
mtx_unlock_spin(&sched_lock);
return;
}

13
sys/amd64/amd64/tsc.c
View File

@ -211,6 +211,10 @@ clkintr(struct clockframe frame)
mtx_unlock_spin(&clock_lock);
}
timer_func(&frame);
#ifdef SMP
if (timer_func == hardclock)
forward_hardclock();
#endif
switch (timer0_state) {
case RELEASED:
@ -253,6 +257,9 @@ clkintr(struct clockframe frame)
timer_func = hardclock;
timer0_state = RELEASED;
hardclock(&frame);
#ifdef SMP
forward_hardclock();
#endif
}
break;
}
@ -374,8 +381,12 @@ release_timer2()
static void
rtcintr(struct clockframe frame)
{
while (rtcin(RTC_INTR) & RTCIR_PERIOD)
while (rtcin(RTC_INTR) & RTCIR_PERIOD) {
statclock(&frame);
#ifdef SMP
forward_statclock();
#endif
}
}
#include "opt_ddb.h"

4
sys/amd64/amd64/vm_machdep.c
View File

@ -59,14 +59,12 @@
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/mutex.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/unistd.h>
#include <machine/cpu.h>
#include <machine/md_var.h>
#ifdef SMP
#include <machine/smp.h>
#endif
#include <machine/pcb.h>
#include <machine/pcb_ext.h>
#include <machine/vm86.h>

2
sys/amd64/include/cpu.h
View File

@ -65,8 +65,6 @@
#define CLKF_USERMODE(framep) \
((ISPL((framep)->cf_cs) == SEL_UPL) || ((framep)->cf_eflags & PSL_VM))
#define CLKF_INTR(framep) (curproc->p_intr_nesting_level >= 2)
#define CLKF_PC(framep) ((framep)->cf_eip)
/*

624
sys/amd64/include/mptable.h
View File

@ -42,6 +42,7 @@
#include <sys/malloc.h>
#include <sys/memrange.h>
#include <sys/mutex.h>
#include <sys/smp.h>
#include <sys/dkstat.h>
#include <sys/cons.h> /* cngetc() */
@ -57,9 +58,9 @@
#include <sys/gmon.h>
#endif
#include <machine/smp.h>
#include <machine/apic.h>
#include <machine/atomic.h>
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/ipl.h>
#include <machine/mpapic.h>
@ -243,7 +244,6 @@ int current_postcode;
extern struct region_descriptor r_gdt, r_idt;
int bsp_apic_ready = 0; /* flags useability of BSP apic */
int mp_ncpus; /* # of CPUs, including BSP */
int mp_naps; /* # of Applications processors */
int mp_nbusses; /* # of busses */
int mp_napics; /* # of IO APICs */
@ -273,9 +273,6 @@ int io_num_to_apic_id[NAPICID];
int apic_id_to_logical[NAPICID];
/* Bitmap of all available CPUs */
u_int all_cpus;
/* AP uses this during bootstrap. Do not staticize. */
char *bootSTK;
static int bootAP;
@ -288,28 +285,9 @@ extern pt_entry_t *SMPpt;
struct pcb stoppcbs[MAXCPU];
int smp_started; /* has the system started? */
int smp_active = 0; /* are the APs allowed to run? */
SYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RW, &smp_active, 0, "");
/* XXX maybe should be hw.ncpu */
static int smp_cpus = 1; /* how many cpu's running */
SYSCTL_INT(_machdep, OID_AUTO, smp_cpus, CTLFLAG_RD, &smp_cpus, 0, "");
int invltlb_ok = 0; /* throttle smp_invltlb() till safe */
SYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, "");
/* Enable forwarding of a signal to a process running on a different CPU */
static int forward_signal_enabled = 1;
SYSCTL_INT(_machdep, OID_AUTO, forward_signal_enabled, CTLFLAG_RW,
&forward_signal_enabled, 0, "");
/* Enable forwarding of roundrobin to all other cpus */
static int forward_roundrobin_enabled = 1;
SYSCTL_INT(_machdep, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW,
&forward_roundrobin_enabled, 0, "");
/*
* Local data and functions.
*/
@ -354,9 +332,6 @@ struct mtx mcount_mtx;
struct mtx com_mtx;
#endif /* USE_COMLOCK */
/* lock around the MP rendezvous */
static struct mtx smp_rv_mtx;
static void
init_locks(void)
{
@ -367,13 +342,9 @@ init_locks(void)
*/
mtx_init(&mcount_mtx, "mcount", MTX_DEF);
mtx_init(&smp_rv_mtx, "smp rendezvous", MTX_SPIN);
#ifdef USE_COMLOCK
mtx_init(&com_mtx, "com", MTX_SPIN);
#endif /* USE_COMLOCK */
mtx_init(&ap_boot_mtx, "ap boot", MTX_SPIN);
}
/*
@ -397,8 +368,8 @@ mp_bootaddress(u_int basemem)
/*
* Look for an Intel MP spec table (ie, SMP capable hardware).
*/
int
mp_probe(void)
void
i386_mp_probe(void)
{
int x;
u_long segment;
@ -427,7 +398,7 @@ mp_probe(void)
/* nothing found */
mpfps = (mpfps_t)0;
mp_capable = 0;
return 0;
return;
found:
/* calculate needed resources */
@ -436,15 +407,19 @@ mp_probe(void)
/* flag fact that we are running multiple processors */
mp_capable = 1;
return 1;
}
int
cpu_mp_probe(void)
{
return (mp_capable);
}
/*
* Initialize the SMP hardware and the APIC and start up the AP's.
*/
void
mp_start(void)
cpu_mp_start(void)
{
POSTCODE(MP_START_POST);
@ -453,6 +428,8 @@ mp_start(void)
mp_enable(boot_address);
else
panic("MP hardware not found!");
cpu_setregs();
}
@ -460,13 +437,12 @@ mp_start(void)
* Print various information about the SMP system hardware and setup.
*/
void
mp_announce(void)
cpu_mp_announce(void)
{
int x;
POSTCODE(MP_ANNOUNCE_POST);
printf("FreeBSD/SMP: Multiprocessor motherboard\n");
printf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0));
printf(", version: 0x%08x", cpu_apic_versions[0]);
printf(", at 0x%08x\n", cpu_apic_address);
@ -623,8 +599,12 @@ mp_enable(u_int boot_addr)
setidt(XINVLTLB_OFFSET, Xinvltlb,
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
/* install an inter-CPU IPI for reading processor state */
setidt(XCPUCHECKSTATE_OFFSET, Xcpucheckstate,
/* install an inter-CPU IPI for forwarding hardclock() */
setidt(XHARDCLOCK_OFFSET, Xhardclock,
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
/* install an inter-CPU IPI for forwarding statclock() */
setidt(XSTATCLOCK_OFFSET, Xstatclock,
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
/* install an inter-CPU IPI for all-CPU rendezvous */
@ -1938,6 +1918,8 @@ start_all_aps(u_int boot_addr)
POSTCODE(START_ALL_APS_POST);
mtx_init(&ap_boot_mtx, "ap boot", MTX_SPIN);
/* initialize BSP's local APIC */
apic_initialize();
bsp_apic_ready = 1;
@ -1985,8 +1967,8 @@ start_all_aps(u_int boot_addr)
(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
/* prime data page for it to use */
SLIST_INSERT_HEAD(&cpuhead, gd, gd_allcpu);
gd->gd_cpuid = x;
globaldata_register(gd);
/* setup a vector to our boot code */
*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
@ -2328,472 +2310,65 @@ ap_init(void)
panic("scheduler returned us to ap_init");
}
#define CHECKSTATE_USER 0
#define CHECKSTATE_SYS 1
#define CHECKSTATE_INTR 2
/* Do not staticize. Used from apic_vector.s */
struct proc* checkstate_curproc[MAXCPU];
int checkstate_cpustate[MAXCPU];
u_long checkstate_pc[MAXCPU];
#define PC_TO_INDEX(pc, prof) \
((int)(((u_quad_t)((pc) - (prof)->pr_off) * \
(u_quad_t)((prof)->pr_scale)) >> 16) & ~1)
static void
addupc_intr_forwarded(struct proc *p, int id, int *astmap)
{
int i;
struct uprof *prof;
u_long pc;
pc = checkstate_pc[id];
prof = &p->p_stats->p_prof;
if (pc >= prof->pr_off &&
(i = PC_TO_INDEX(pc, prof)) < prof->pr_size) {
mtx_assert(&sched_lock, MA_OWNED);
if ((p->p_sflag & PS_OWEUPC) == 0) {
prof->pr_addr = pc;
prof->pr_ticks = 1;
p->p_sflag |= PS_OWEUPC;
}
*astmap |= (1 << id);
}
}
static void
forwarded_statclock(int id, int pscnt, int *astmap)
{
struct pstats *pstats;
long rss;
struct rusage *ru;
struct vmspace *vm;
int cpustate;
struct proc *p;
#ifdef GPROF
register struct gmonparam *g;
int i;
#endif
mtx_assert(&sched_lock, MA_OWNED);
p = checkstate_curproc[id];
cpustate = checkstate_cpustate[id];
/* XXX */
if (p->p_ithd)
cpustate = CHECKSTATE_INTR;
else if (p == SMP_prvspace[id].globaldata.gd_idleproc)
cpustate = CHECKSTATE_SYS;
switch (cpustate) {
case CHECKSTATE_USER:
if (p->p_sflag & PS_PROFIL)
addupc_intr_forwarded(p, id, astmap);
if (pscnt > 1)
return;
p->p_uticks++;
if (p->p_nice > NZERO)
cp_time[CP_NICE]++;
else
cp_time[CP_USER]++;
break;
case CHECKSTATE_SYS:
#ifdef GPROF
/*
* Kernel statistics are just like addupc_intr, only easier.
*/
g = &_gmonparam;
if (g->state == GMON_PROF_ON) {
i = checkstate_pc[id] - g->lowpc;
if (i < g->textsize) {
i /= HISTFRACTION * sizeof(*g->kcount);
g->kcount[i]++;
}
}
#endif
if (pscnt > 1)
return;
p->p_sticks++;
if (p == SMP_prvspace[id].globaldata.gd_idleproc)
cp_time[CP_IDLE]++;
else
cp_time[CP_SYS]++;
break;
case CHECKSTATE_INTR:
default:
#ifdef GPROF
/*
* Kernel statistics are just like addupc_intr, only easier.
*/
g = &_gmonparam;
if (g->state == GMON_PROF_ON) {
i = checkstate_pc[id] - g->lowpc;
if (i < g->textsize) {
i /= HISTFRACTION * sizeof(*g->kcount);
g->kcount[i]++;
}
}
#endif
if (pscnt > 1)
return;
KASSERT(p != NULL, ("NULL process in interrupt state"));
p->p_iticks++;
cp_time[CP_INTR]++;
}
schedclock(p);
/* Update resource usage integrals and maximums. */
if ((pstats = p->p_stats) != NULL &&
(ru = &pstats->p_ru) != NULL &&
(vm = p->p_vmspace) != NULL) {
ru->ru_ixrss += pgtok(vm->vm_tsize);
ru->ru_idrss += pgtok(vm->vm_dsize);
ru->ru_isrss += pgtok(vm->vm_ssize);
rss = pgtok(vmspace_resident_count(vm));
if (ru->ru_maxrss < rss)
ru->ru_maxrss = rss;
}
}
/*
* For statclock, we send an IPI to all CPU's to have them call this
* function.
*/
void
forward_statclock(int pscnt)
forwarded_statclock(struct trapframe frame)
{
int map;
int id;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_statclock(%d)", pscnt);
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
/* Step 1: Probe state (user, cpu, interrupt, spinlock, idle ) */
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
checkstate_probed_cpus = 0;
if (map != 0)
ipi_selected(map, IPI_CHECKSTATE);
i = 0;
while (checkstate_probed_cpus != map) {
/* spin */
i++;
if (i == 100000) {
#ifdef DIAGNOSTIC
printf("forward_statclock: checkstate %x\n",
checkstate_probed_cpus);
#endif
break;
}
}
/*
* Step 2: walk through other processors processes, update ticks and
* profiling info.
*/
map = 0;
for (id = 0; id < mp_ncpus; id++) {
if (id == PCPU_GET(cpuid))
continue;
if (((1 << id) & checkstate_probed_cpus) == 0)
continue;
forwarded_statclock(id, pscnt, &map);
}
if (map != 0) {
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#ifdef DIAGNOSTIC
printf("forward_statclock: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
}
}
void
forward_hardclock(int pscnt)
{
int map;
int id;
struct proc *p;
struct pstats *pstats;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_hardclock(%d)", pscnt);
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
/* Step 1: Probe state (user, cpu, interrupt, spinlock, idle) */
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
checkstate_probed_cpus = 0;
if (map != 0)
ipi_selected(map, IPI_CHECKSTATE);
i = 0;
while (checkstate_probed_cpus != map) {
/* spin */
i++;
if (i == 100000) {
#ifdef DIAGNOSTIC
printf("forward_hardclock: checkstate %x\n",
checkstate_probed_cpus);
#endif
break;
}
}
/*
* Step 2: walk through other processors processes, update virtual
* timer and profiling timer. If stathz == 0, also update ticks and
* profiling info.
*/
map = 0;
for (id = 0; id < mp_ncpus; id++) {
if (id == PCPU_GET(cpuid))
continue;
if (((1 << id) & checkstate_probed_cpus) == 0)
continue;
p = checkstate_curproc[id];
if (p) {
pstats = p->p_stats;
if (checkstate_cpustate[id] == CHECKSTATE_USER &&
timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) &&
itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) {
p->p_sflag |= PS_ALRMPEND;
map |= (1 << id);
}
if (timevalisset(&pstats->p_timer[ITIMER_PROF].it_value) &&
itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) {
p->p_sflag |= PS_PROFPEND;
map |= (1 << id);
}
}
if (stathz == 0) {
forwarded_statclock( id, pscnt, &map);
}
}
if (map != 0) {
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#ifdef DIAGNOSTIC
printf("forward_hardclock: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
}
}
void
forward_signal(struct proc *p)
{
int map;
int id;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_signal(%p)", p);
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
if (!forward_signal_enabled)
return;
mtx_lock_spin(&sched_lock);
while (1) {
if (p->p_stat != SRUN) {
mtx_unlock_spin(&sched_lock);
return;
}
id = p->p_oncpu;
mtx_unlock_spin(&sched_lock);
if (id == 0xff)
return;
map = (1<<id);
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#if 0
printf("forward_signal: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
mtx_lock_spin(&sched_lock);
if (id == p->p_oncpu) {
mtx_unlock_spin(&sched_lock);
return;
}
}
statclock_process(curproc, TRAPF_PC(&frame), TRAPF_USERMODE(&frame));
mtx_unlock_spin(&sched_lock);
}
void
forward_roundrobin(void)
forward_statclock(void)
{
int map;
CTR0(KTR_SMP, "forward_statclock");
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
if (map != 0)
ipi_selected(map, IPI_STATCLOCK);
}
/*
* For each hardclock(), we send an IPI to all other CPU's to have them
* execute this function. It would be nice to reduce contention on
* sched_lock if we could simply peek at the CPU to determine the user/kernel
* state and call hardclock_process() on the CPU receiving the clock interrupt
* and then just use a simple IPI to handle any ast's if needed.
*/
void
forwarded_hardclock(struct trapframe frame)
{
mtx_lock_spin(&sched_lock);
hardclock_process(curproc, TRAPF_USERMODE(&frame));
mtx_unlock_spin(&sched_lock);
}
void
forward_hardclock(void)
{
u_int map;
int i;
CTR0(KTR_SMP, "forward_roundrobin()");
CTR0(KTR_SMP, "forward_hardclock");
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
if (!forward_roundrobin_enabled)
return;
resched_cpus |= PCPU_GET(other_cpus);
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
#if 1
ipi_selected(map, IPI_AST);
#else
ipi_all_but_self(IPI_AST);
#endif
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#if 0
printf("forward_roundrobin: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
if (map != 0)
ipi_selected(map, IPI_HARDCLOCK);
}
/*
* When called the executing CPU will send an IPI to all other CPUs
* requesting that they halt execution.
*
* Usually (but not necessarily) called with 'other_cpus' as its arg.
*
* - Signals all CPUs in map to stop.
* - Waits for each to stop.
*
* Returns:
* -1: error
* 0: NA
* 1: ok
*
* XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
* from executing at same time.
*/
int
stop_cpus(u_int map)
{
int count = 0;
if (!smp_started)
return 0;
/* send the Xcpustop IPI to all CPUs in map */
ipi_selected(map, IPI_STOP);
while (count++ < 100000 && (stopped_cpus & map) != map)
/* spin */ ;
#ifdef DIAGNOSTIC
if ((stopped_cpus & map) != map)
printf("Warning: CPUs 0x%x did not stop!\n",
(~(stopped_cpus & map)) & map);
#endif
return 1;
}
/*
* Called by a CPU to restart stopped CPUs.
*
* Usually (but not necessarily) called with 'stopped_cpus' as its arg.
*
* - Signals all CPUs in map to restart.
* - Waits for each to restart.
*
* Returns:
* -1: error
* 0: NA
* 1: ok
*/
int
restart_cpus(u_int map)
{
int count = 0;
if (!smp_started)
return 0;
started_cpus = map; /* signal other cpus to restart */
/* wait for each to clear its bit */
while (count++ < 100000 && (stopped_cpus & map) != 0)
/* spin */ ;
#ifdef DIAGNOSTIC
if ((stopped_cpus & map) != 0)
printf("Warning: CPUs 0x%x did not restart!\n",
(~(stopped_cpus & map)) & map);
#endif
return 1;
}
#ifdef APIC_INTR_REORDER
/*
* Maintain mapping from softintr vector to isr bit in local apic.
@ -2810,73 +2385,6 @@ set_lapic_isrloc(int intr, int vector)
}
#endif
/*
* All-CPU rendezvous. CPUs are signalled, all execute the setup function
* (if specified), rendezvous, execute the action function (if specified),
* rendezvous again, execute the teardown function (if specified), and then
* resume.
*
* Note that the supplied external functions _must_ be reentrant and aware
* that they are running in parallel and in an unknown lock context.
*/
static void (*smp_rv_setup_func)(void *arg);
static void (*smp_rv_action_func)(void *arg);
static void (*smp_rv_teardown_func)(void *arg);
static void *smp_rv_func_arg;
static volatile int smp_rv_waiters[2];
void
smp_rendezvous_action(void)
{
/* setup function */
if (smp_rv_setup_func != NULL)
smp_rv_setup_func(smp_rv_func_arg);
/* spin on entry rendezvous */
atomic_add_int(&smp_rv_waiters[0], 1);
while (smp_rv_waiters[0] < mp_ncpus)
;
/* action function */
if (smp_rv_action_func != NULL)
smp_rv_action_func(smp_rv_func_arg);
/* spin on exit rendezvous */
atomic_add_int(&smp_rv_waiters[1], 1);
while (smp_rv_waiters[1] < mp_ncpus)
;
/* teardown function */
if (smp_rv_teardown_func != NULL)
smp_rv_teardown_func(smp_rv_func_arg);
}
void
smp_rendezvous(void (* setup_func)(void *),
void (* action_func)(void *),
void (* teardown_func)(void *),
void *arg)
{
/* obtain rendezvous lock */
mtx_lock_spin(&smp_rv_mtx);
/* set static function pointers */
smp_rv_setup_func = setup_func;
smp_rv_action_func = action_func;
smp_rv_teardown_func = teardown_func;
smp_rv_func_arg = arg;
smp_rv_waiters[0] = 0;
smp_rv_waiters[1] = 0;
/*
* signal other processors, which will enter the IPI with interrupts off
*/
ipi_all_but_self(IPI_RENDEZVOUS);
/* call executor function */
smp_rendezvous_action();
/* release lock */
mtx_unlock_spin(&smp_rv_mtx);
}
/*
* send an IPI to a set of cpus.
*/

3
sys/amd64/include/pcpu.h
View File

@ -74,9 +74,6 @@ struct globaldata {
#endif
};
SLIST_HEAD(cpuhead, globaldata);
extern struct cpuhead cpuhead;
#ifdef SMP
/*
* This is the upper (0xff800000) address space layout that is per-cpu.

50
sys/amd64/include/smp.h
View File

@ -41,7 +41,21 @@ extern int current_postcode; /** XXX currently in mp_machdep.c */
outb(0x80, current_postcode)
#include <sys/bus.h> /* XXX */
#include <machine/apic.h>
#include <machine/frame.h>
#include <i386/isa/icu.h>
#include <i386/isa/intr_machdep.h>
/*
* Interprocessor interrupts for SMP.
*/
#define IPI_INVLTLB XINVLTLB_OFFSET
#define IPI_RENDEZVOUS XRENDEZVOUS_OFFSET
#define IPI_AST XCPUAST_OFFSET
#define IPI_STOP XCPUSTOP_OFFSET
#define IPI_HARDCLOCK XHARDCLOCK_OFFSET
#define IPI_STATCLOCK XSTATCLOCK_OFFSET
/* global data in mpboot.s */
extern int bootMP_size;
@ -49,18 +63,8 @@ extern int bootMP_size;
/* functions in mpboot.s */
void bootMP __P((void));
/* global data in apic_vector.s */
extern volatile u_int stopped_cpus;
extern volatile u_int started_cpus;
extern volatile u_int checkstate_probed_cpus;
extern volatile u_int checkstate_need_ast;
extern volatile u_int resched_cpus;
extern void (*cpustop_restartfunc) __P((void));
/* global data in mp_machdep.c */
extern int bsp_apic_ready;
extern int mp_ncpus;
extern int mp_naps;
extern int mp_nbusses;
extern int mp_napics;
@ -81,14 +85,11 @@ struct apic_intmapinfo {
int redirindex;
};
extern struct apic_intmapinfo int_to_apicintpin[];
extern u_int all_cpus;
extern struct pcb stoppcbs[];
/* functions in mp_machdep.c */
void i386_mp_probe __P((void));
u_int mp_bootaddress __P((u_int));
int mp_probe __P((void));
void mp_start __P((void));
void mp_announce __P((void));
u_int isa_apic_mask __P((u_int));
int isa_apic_irq __P((int));
int pci_apic_irq __P((int, int, int));
@ -107,20 +108,17 @@ void revoke_apic_irq __P((int irq));
void bsp_apic_configure __P((void));
void init_secondary __P((void));
void smp_invltlb __P((void));
int stop_cpus __P((u_int));
int restart_cpus __P((u_int));
void forward_statclock __P((int pscnt));
void forward_hardclock __P((int pscnt));
void forward_signal __P((struct proc *));
void forward_roundrobin __P((void));
void forward_statclock __P((void));
void forwarded_statclock __P((struct trapframe frame));
void forward_hardclock __P((void));
void forwarded_hardclock __P((struct trapframe frame));
void ipi_selected __P((u_int cpus, u_int ipi));
void ipi_all __P((u_int ipi));
void ipi_all_but_self __P((u_int ipi));
void ipi_self __P((u_int ipi));
#ifdef APIC_INTR_REORDER
void set_lapic_isrloc __P((int, int));
#endif /* APIC_INTR_REORDER */
void smp_rendezvous_action __P((void));
void smp_rendezvous __P((void (*)(void *),
void (*)(void *),
void (*)(void *),
void *arg));
/* global data in mpapic.c */
extern volatile lapic_t lapic;
@ -147,8 +145,6 @@ void io_apic_write __P((int, int, u_int));
/* global data in init_smp.c */
extern int invltlb_ok;
extern int smp_active;
extern int smp_started;
extern volatile int smp_idle_loops;
#endif /* !LOCORE */

13
sys/amd64/isa/clock.c
View File

@ -211,6 +211,10 @@ clkintr(struct clockframe frame)
mtx_unlock_spin(&clock_lock);
}
timer_func(&frame);
#ifdef SMP
if (timer_func == hardclock)
forward_hardclock();
#endif
switch (timer0_state) {
case RELEASED:
@ -253,6 +257,9 @@ clkintr(struct clockframe frame)
timer_func = hardclock;
timer0_state = RELEASED;
hardclock(&frame);
#ifdef SMP
forward_hardclock();
#endif
}
break;
}
@ -374,8 +381,12 @@ release_timer2()
static void
rtcintr(struct clockframe frame)
{
while (rtcin(RTC_INTR) & RTCIR_PERIOD)
while (rtcin(RTC_INTR) & RTCIR_PERIOD) {
statclock(&frame);
#ifdef SMP
forward_statclock();
#endif
}
}
#include "opt_ddb.h"

8
sys/amd64/isa/intr_machdep.h
View File

@ -107,10 +107,11 @@
#define XINVLTLB_OFFSET (ICU_OFFSET + 112)
/* inter-cpu clock handling */
#define XCPUCHECKSTATE_OFFSET (ICU_OFFSET + 113)
#define XHARDCLOCK_OFFSET (ICU_OFFSET + 113)
#define XSTATCLOCK_OFFSET (ICU_OFFSET + 114)
/* inter-CPU rendezvous */
#define XRENDEZVOUS_OFFSET (ICU_OFFSET + 114)
#define XRENDEZVOUS_OFFSET (ICU_OFFSET + 115)
/* IPI to generate an additional software trap at the target CPU */
#define XCPUAST_OFFSET (ICU_OFFSET + 48)
@ -173,7 +174,8 @@ inthand_t
inthand_t
Xinvltlb, /* TLB shootdowns */
Xcpucheckstate, /* Check cpu state */
Xhardclock, /* Forward hardclock() */
Xstatclock, /* Forward statclock() */
Xcpuast, /* Additional software trap on other cpu */
Xcpustop, /* CPU stops & waits for another CPU to restart it */
Xspuriousint, /* handle APIC "spurious INTs" */

1
sys/conf/files
View File

@ -716,6 +716,7 @@ kern/subr_prof.c standard
kern/subr_rman.c standard
kern/subr_sbuf.c standard
kern/subr_scanf.c standard
kern/subr_smp.c optional smp
kern/subr_taskqueue.c standard
kern/subr_witness.c optional witness
kern/subr_xxx.c standard

2
sys/conf/files.ia64
View File

@ -36,7 +36,7 @@ ia64/ia64/interrupt.c standard
ia64/ia64/machdep.c standard
ia64/ia64/mem.c standard
ia64/ia64/mountroot.c optional slice
ia64/ia64/mp_machdep.c standard
ia64/ia64/mp_machdep.c optional smp
ia64/ia64/pal.s standard
ia64/ia64/perfmon.c optional perfmon profiling-routine
ia64/ia64/perfmon.c optional perfmon

155
sys/i386/i386/apic_vector.s
View File

@ -182,7 +182,6 @@ Xspuriousint:
iret
/*
* Handle TLB shootdowns.
*/
@ -211,71 +210,61 @@ Xinvltlb:
popl %eax
iret
/*
* Executed by a CPU when it receives an Xcpucheckstate IPI from another CPU,
*
* - Stores current cpu state in checkstate_cpustate[cpuid]
* 0 == user, 1 == sys, 2 == intr
* - Stores current process in checkstate_curproc[cpuid]
*
* - Signals its receipt by setting bit cpuid in checkstate_probed_cpus.
*
* stack: 0->ds, 4->fs, 8->ebx, 12->eax, 16->eip, 20->cs, 24->eflags
* Forward hardclock to another CPU. Pushes a trapframe and calls
* forwarded_hardclock().
*/
.text
SUPERALIGN_TEXT
.globl Xcpucheckstate
.globl checkstate_cpustate
.globl checkstate_curproc
.globl checkstate_pc
Xcpucheckstate:
pushl %eax
pushl %ebx
pushl %ds /* save current data segment */
pushl %fs
movl $KDSEL, %eax
mov %ax, %ds /* use KERNEL data segment */
.globl Xhardclock
Xhardclock:
PUSH_FRAME
movl $KDSEL, %eax /* reload with kernel's data segment */
mov %ax, %ds
mov %ax, %es
movl $KPSEL, %eax
mov %ax, %fs
movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */
movl $0, %ebx
movl 20(%esp), %eax
andl $3, %eax
cmpl $3, %eax
je 1f
testl $PSL_VM, 24(%esp)
jne 1f
incl %ebx /* system or interrupt */
1:
movl PCPU(CPUID), %eax
movl %ebx, checkstate_cpustate(,%eax,4)
movl PCPU(CURPROC), %ebx
movl %ebx, checkstate_curproc(,%eax,4)
movl PCPU(CURPROC),%ebx
incl P_INTR_NESTING_LEVEL(%ebx)
call forwarded_hardclock
decl P_INTR_NESTING_LEVEL(%ebx)
MEXITCOUNT
jmp doreti
movl 16(%esp), %ebx
movl %ebx, checkstate_pc(,%eax,4)
/*
* Forward statclock to another CPU. Pushes a trapframe and calls
* forwarded_statclock().
*/
.text
SUPERALIGN_TEXT
.globl Xstatclock
Xstatclock:
PUSH_FRAME
movl $KDSEL, %eax /* reload with kernel's data segment */
mov %ax, %ds
mov %ax, %es
movl $KPSEL, %eax
mov %ax, %fs
lock /* checkstate_probed_cpus |= (1<<id) */
btsl %eax, checkstate_probed_cpus
popl %fs
popl %ds /* restore previous data segment */
popl %ebx
popl %eax
iret
movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */
FAKE_MCOUNT(13*4(%esp))
movl PCPU(CURPROC),%ebx
incl P_INTR_NESTING_LEVEL(%ebx)
call forwarded_statclock
decl P_INTR_NESTING_LEVEL(%ebx)
MEXITCOUNT
jmp doreti
/*
* Executed by a CPU when it receives an Xcpuast IPI from another CPU,
*
* - Signals its receipt by clearing bit cpuid in checkstate_need_ast.
*
* - We need a better method of triggering asts on other cpus.
* The other CPU has already executed aston() or need_resched() on our
* current process, so we simply need to ack the interrupt and return
* via doreti to run ast().
*/
.text
@ -289,40 +278,12 @@ Xcpuast:
movl $KPSEL, %eax
mov %ax, %fs
movl PCPU(CPUID), %eax
lock /* checkstate_need_ast &= ~(1<<id) */
btrl %eax, checkstate_need_ast
movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */
lock
btsl %eax, checkstate_pending_ast
jc 1f
FAKE_MCOUNT(13*4(%esp))
MTX_LOCK_SPIN(sched_lock, 0)
movl PCPU(CURPROC),%ebx
orl $PS_ASTPENDING, P_SFLAG(%ebx)
movl PCPU(CPUID), %eax
lock
btrl %eax, checkstate_pending_ast
lock
btrl %eax, CNAME(resched_cpus)
jnc 2f
orl $PS_NEEDRESCHED, P_SFLAG(%ebx)
lock
incl CNAME(want_resched_cnt)
2:
MTX_UNLOCK_SPIN(sched_lock)
lock
incl CNAME(cpuast_cnt)
MEXITCOUNT
jmp doreti
1:
/* We are already in the process of delivering an ast for this CPU */
POP_FRAME
iret
/*
* Executed by a CPU when it receives an Xcpustop IPI from another CPU,
@ -331,7 +292,6 @@ Xcpuast:
* - Waits for permission to restart.
* - Signals its restart.
*/
.text
SUPERALIGN_TEXT
.globl Xcpustop
@ -357,20 +317,19 @@ Xcpustop:
pushl %eax
call CNAME(savectx) /* Save process context */
addl $4, %esp
movl PCPU(CPUID), %eax
lock
btsl %eax, stopped_cpus /* stopped_cpus |= (1<<id) */
btsl %eax, CNAME(stopped_cpus) /* stopped_cpus |= (1<<id) */
1:
btl %eax, started_cpus /* while (!(started_cpus & (1<<id))) */
btl %eax, CNAME(started_cpus) /* while (!(started_cpus & (1<<id))) */
jnc 1b
lock
btrl %eax, started_cpus /* started_cpus &= ~(1<<id) */
btrl %eax, CNAME(started_cpus) /* started_cpus &= ~(1<<id) */
lock
btrl %eax, stopped_cpus /* stopped_cpus &= ~(1<<id) */
btrl %eax, CNAME(stopped_cpus) /* stopped_cpus &= ~(1<<id) */
test %eax, %eax
jnz 2f
@ -492,34 +451,6 @@ _xhits:
.space (NCPU * 4), 0
#endif /* COUNT_XINVLTLB_HITS */
/* variables used by stop_cpus()/restart_cpus()/Xcpustop */
.globl stopped_cpus, started_cpus
stopped_cpus:
.long 0
started_cpus:
.long 0
.globl checkstate_probed_cpus
checkstate_probed_cpus:
.long 0
.globl checkstate_need_ast
checkstate_need_ast:
.long 0
checkstate_pending_ast:
.long 0
.globl CNAME(resched_cpus)
.globl CNAME(want_resched_cnt)
.globl CNAME(cpuast_cnt)
.globl CNAME(cpustop_restartfunc)
CNAME(resched_cpus):
.long 0
CNAME(want_resched_cnt):
.long 0
CNAME(cpuast_cnt):
.long 0
CNAME(cpustop_restartfunc):
.long 0
.globl apic_pin_trigger
apic_pin_trigger:
.long 0

61
sys/i386/i386/db_interface.c
View File

@ -37,10 +37,10 @@
#include <sys/linker_set.h>
#include <sys/lock.h>
#include <sys/proc.h>
#include <sys/smp.h>
#include <machine/cpu.h>
#ifdef SMP
#include <machine/smp.h>
#include <machine/smptests.h> /** CPUSTOP_ON_DDBBREAK */
#endif
@ -334,43 +334,44 @@ Debugger(msg)
DB_SHOW_COMMAND(pcpu, db_show_pcpu)
{
struct globaldata *gd;
#ifdef SMP
int id;
if (have_addr)
id = ((addr >> 4) % 16) * 10 + (addr % 16);
else
id = PCPU_GET(cpuid);
SLIST_FOREACH(gd, &cpuhead, gd_allcpu) {
if (gd->gd_cpuid == id)
break;
}
if (gd == NULL)
gd = globaldata_find(id);
if (gd == NULL) {
db_printf("CPU %d not found\n", id);
else {
db_printf("cpuid = %d\n", gd->gd_cpuid);
db_printf("curproc = ");
if (gd->gd_curproc != NULL)
db_printf("%p: pid %d \"%s\"\n", gd->gd_curproc,
gd->gd_curproc->p_pid, gd->gd_curproc->p_comm);
else
db_printf("none\n");
db_printf("curpcb = %p\n", gd->gd_curpcb);
db_printf("npxproc = ");
if (gd->gd_npxproc != NULL)
db_printf("%p: pid %d \"%s\"\n", gd->gd_npxproc,
gd->gd_npxproc->p_pid, gd->gd_npxproc->p_comm);
else
db_printf("none\n");
db_printf("idleproc = ");
if (gd->gd_idleproc != NULL)
db_printf("%p: pid %d \"%s\"\n", gd->gd_idleproc,
gd->gd_idleproc->p_pid, gd->gd_idleproc->p_comm);
else
db_printf("none\n");
return;
}
#else
gd = GLOBALDATA;
#endif
db_printf("cpuid = %d\n", gd->gd_cpuid);
db_printf("curproc = ");
if (gd->gd_curproc != NULL)
db_printf("%p: pid %d \"%s\"\n", gd->gd_curproc,
gd->gd_curproc->p_pid, gd->gd_curproc->p_comm);
else
db_printf("none\n");
db_printf("curpcb = %p\n", gd->gd_curpcb);
db_printf("npxproc = ");
if (gd->gd_npxproc != NULL)
db_printf("%p: pid %d \"%s\"\n", gd->gd_npxproc,
gd->gd_npxproc->p_pid, gd->gd_npxproc->p_comm);
else
db_printf("none\n");
db_printf("idleproc = ");
if (gd->gd_idleproc != NULL)
db_printf("%p: pid %d \"%s\"\n", gd->gd_idleproc,
gd->gd_idleproc->p_pid, gd->gd_idleproc->p_comm);
else
db_printf("none\n");
#ifdef WITNESS
db_printf("spin locks held:\n");
witness_list_locks(&gd->gd_spinlocks);
db_printf("spin locks held:\n");
witness_list_locks(&gd->gd_spinlocks);
#endif
}
}

5
sys/i386/i386/i686_mem.c
View File

@ -31,14 +31,11 @@
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/memrange.h>
#include <sys/smp.h>
#include <machine/md_var.h>
#include <machine/specialreg.h>
#ifdef SMP
#include <machine/smp.h>
#endif
/*
* i686 memory range operations
*

21
sys/i386/i386/machdep.c
View File

@ -66,6 +66,7 @@
#include <sys/bio.h>
#include <sys/buf.h>
#include <sys/reboot.h>
#include <sys/smp.h>
#include <sys/callout.h>
#include <sys/msgbuf.h>
#include <sys/sysent.h>
@ -104,9 +105,6 @@
#include <machine/globaldata.h>
#include <machine/globals.h>
#include <machine/intrcnt.h>
#ifdef SMP
#include <machine/smp.h>
#endif
#ifdef PERFMON
#include <machine/perfmon.h>
#endif
@ -249,8 +247,6 @@ static struct trapframe proc0_tf;
static struct globaldata __globaldata;
#endif
struct cpuhead cpuhead;
struct mtx sched_lock;
struct mtx Giant;
@ -441,17 +437,12 @@ cpu_startup(dummy)
bufinit();
vm_pager_bufferinit();
SLIST_INIT(&cpuhead);
SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu);
#ifdef SMP
/*
* OK, enough kmem_alloc/malloc state should be up, lets get on with it!
*/
mp_start(); /* fire up the APs and APICs */
mp_announce();
#endif /* SMP */
globaldata_register(GLOBALDATA);
#else
/* For SMP, we delay the cpu_setregs() until after SMP startup. */
cpu_setregs();
#endif
}
/*
@ -1631,7 +1622,7 @@ getmemsize(int first)
physmap[1] = mp_bootaddress(physmap[1] / 1024);
/* look for the MP hardware - needed for apic addresses */
mp_probe();
i386_mp_probe();
#endif
/*

624
sys/i386/i386/mp_machdep.c
View File

@ -42,6 +42,7 @@
#include <sys/malloc.h>
#include <sys/memrange.h>
#include <sys/mutex.h>
#include <sys/smp.h>
#include <sys/dkstat.h>
#include <sys/cons.h> /* cngetc() */
@ -57,9 +58,9 @@
#include <sys/gmon.h>
#endif
#include <machine/smp.h>
#include <machine/apic.h>
#include <machine/atomic.h>
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/ipl.h>
#include <machine/mpapic.h>
@ -243,7 +244,6 @@ int current_postcode;
extern struct region_descriptor r_gdt, r_idt;
int bsp_apic_ready = 0; /* flags useability of BSP apic */
int mp_ncpus; /* # of CPUs, including BSP */
int mp_naps; /* # of Applications processors */
int mp_nbusses; /* # of busses */
int mp_napics; /* # of IO APICs */
@ -273,9 +273,6 @@ int io_num_to_apic_id[NAPICID];
int apic_id_to_logical[NAPICID];
/* Bitmap of all available CPUs */
u_int all_cpus;
/* AP uses this during bootstrap. Do not staticize. */
char *bootSTK;
static int bootAP;
@ -288,28 +285,9 @@ extern pt_entry_t *SMPpt;
struct pcb stoppcbs[MAXCPU];
int smp_started; /* has the system started? */
int smp_active = 0; /* are the APs allowed to run? */
SYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RW, &smp_active, 0, "");
/* XXX maybe should be hw.ncpu */
static int smp_cpus = 1; /* how many cpu's running */
SYSCTL_INT(_machdep, OID_AUTO, smp_cpus, CTLFLAG_RD, &smp_cpus, 0, "");
int invltlb_ok = 0; /* throttle smp_invltlb() till safe */
SYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, "");
/* Enable forwarding of a signal to a process running on a different CPU */
static int forward_signal_enabled = 1;
SYSCTL_INT(_machdep, OID_AUTO, forward_signal_enabled, CTLFLAG_RW,
&forward_signal_enabled, 0, "");
/* Enable forwarding of roundrobin to all other cpus */
static int forward_roundrobin_enabled = 1;
SYSCTL_INT(_machdep, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW,
&forward_roundrobin_enabled, 0, "");
/*
* Local data and functions.
*/
@ -354,9 +332,6 @@ struct mtx mcount_mtx;
struct mtx com_mtx;
#endif /* USE_COMLOCK */
/* lock around the MP rendezvous */
static struct mtx smp_rv_mtx;
static void
init_locks(void)
{
@ -367,13 +342,9 @@ init_locks(void)
*/
mtx_init(&mcount_mtx, "mcount", MTX_DEF);
mtx_init(&smp_rv_mtx, "smp rendezvous", MTX_SPIN);
#ifdef USE_COMLOCK
mtx_init(&com_mtx, "com", MTX_SPIN);
#endif /* USE_COMLOCK */
mtx_init(&ap_boot_mtx, "ap boot", MTX_SPIN);
}
/*
@ -397,8 +368,8 @@ mp_bootaddress(u_int basemem)
/*
* Look for an Intel MP spec table (ie, SMP capable hardware).
*/
int
mp_probe(void)
void
i386_mp_probe(void)
{
int x;
u_long segment;
@ -427,7 +398,7 @@ mp_probe(void)
/* nothing found */
mpfps = (mpfps_t)0;
mp_capable = 0;
return 0;
return;
found:
/* calculate needed resources */
@ -436,15 +407,19 @@ mp_probe(void)
/* flag fact that we are running multiple processors */
mp_capable = 1;
return 1;
}
int
cpu_mp_probe(void)
{
return (mp_capable);
}
/*
* Initialize the SMP hardware and the APIC and start up the AP's.
*/
void
mp_start(void)
cpu_mp_start(void)
{
POSTCODE(MP_START_POST);
@ -453,6 +428,8 @@ mp_start(void)
mp_enable(boot_address);
else
panic("MP hardware not found!");
cpu_setregs();
}
@ -460,13 +437,12 @@ mp_start(void)
* Print various information about the SMP system hardware and setup.
*/
void
mp_announce(void)
cpu_mp_announce(void)
{
int x;
POSTCODE(MP_ANNOUNCE_POST);
printf("FreeBSD/SMP: Multiprocessor motherboard\n");
printf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0));
printf(", version: 0x%08x", cpu_apic_versions[0]);
printf(", at 0x%08x\n", cpu_apic_address);
@ -623,8 +599,12 @@ mp_enable(u_int boot_addr)
setidt(XINVLTLB_OFFSET, Xinvltlb,
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
/* install an inter-CPU IPI for reading processor state */
setidt(XCPUCHECKSTATE_OFFSET, Xcpucheckstate,
/* install an inter-CPU IPI for forwarding hardclock() */
setidt(XHARDCLOCK_OFFSET, Xhardclock,
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
/* install an inter-CPU IPI for forwarding statclock() */
setidt(XSTATCLOCK_OFFSET, Xstatclock,
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
/* install an inter-CPU IPI for all-CPU rendezvous */
@ -1938,6 +1918,8 @@ start_all_aps(u_int boot_addr)
POSTCODE(START_ALL_APS_POST);
mtx_init(&ap_boot_mtx, "ap boot", MTX_SPIN);
/* initialize BSP's local APIC */
apic_initialize();
bsp_apic_ready = 1;
@ -1985,8 +1967,8 @@ start_all_aps(u_int boot_addr)
(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
/* prime data page for it to use */
SLIST_INSERT_HEAD(&cpuhead, gd, gd_allcpu);
gd->gd_cpuid = x;
globaldata_register(gd);
/* setup a vector to our boot code */
*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
@ -2328,472 +2310,65 @@ ap_init(void)
panic("scheduler returned us to ap_init");
}
#define CHECKSTATE_USER 0
#define CHECKSTATE_SYS 1
#define CHECKSTATE_INTR 2
/* Do not staticize. Used from apic_vector.s */
struct proc* checkstate_curproc[MAXCPU];
int checkstate_cpustate[MAXCPU];
u_long checkstate_pc[MAXCPU];
#define PC_TO_INDEX(pc, prof) \
((int)(((u_quad_t)((pc) - (prof)->pr_off) * \
(u_quad_t)((prof)->pr_scale)) >> 16) & ~1)
static void
addupc_intr_forwarded(struct proc *p, int id, int *astmap)
{
int i;
struct uprof *prof;
u_long pc;
pc = checkstate_pc[id];
prof = &p->p_stats->p_prof;
if (pc >= prof->pr_off &&
(i = PC_TO_INDEX(pc, prof)) < prof->pr_size) {
mtx_assert(&sched_lock, MA_OWNED);
if ((p->p_sflag & PS_OWEUPC) == 0) {
prof->pr_addr = pc;
prof->pr_ticks = 1;
p->p_sflag |= PS_OWEUPC;
}
*astmap |= (1 << id);
}
}
static void
forwarded_statclock(int id, int pscnt, int *astmap)
{
struct pstats *pstats;
long rss;
struct rusage *ru;
struct vmspace *vm;
int cpustate;
struct proc *p;
#ifdef GPROF
register struct gmonparam *g;
int i;
#endif
mtx_assert(&sched_lock, MA_OWNED);
p = checkstate_curproc[id];
cpustate = checkstate_cpustate[id];
/* XXX */
if (p->p_ithd)
cpustate = CHECKSTATE_INTR;
else if (p == SMP_prvspace[id].globaldata.gd_idleproc)
cpustate = CHECKSTATE_SYS;
switch (cpustate) {
case CHECKSTATE_USER:
if (p->p_sflag & PS_PROFIL)
addupc_intr_forwarded(p, id, astmap);
if (pscnt > 1)
return;
p->p_uticks++;
if (p->p_nice > NZERO)
cp_time[CP_NICE]++;
else
cp_time[CP_USER]++;
break;
case CHECKSTATE_SYS:
#ifdef GPROF
/*
* Kernel statistics are just like addupc_intr, only easier.
*/
g = &_gmonparam;
if (g->state == GMON_PROF_ON) {
i = checkstate_pc[id] - g->lowpc;
if (i < g->textsize) {
i /= HISTFRACTION * sizeof(*g->kcount);
g->kcount[i]++;
}
}
#endif
if (pscnt > 1)
return;
p->p_sticks++;
if (p == SMP_prvspace[id].globaldata.gd_idleproc)
cp_time[CP_IDLE]++;
else
cp_time[CP_SYS]++;
break;
case CHECKSTATE_INTR:
default:
#ifdef GPROF
/*
* Kernel statistics are just like addupc_intr, only easier.
*/
g = &_gmonparam;
if (g->state == GMON_PROF_ON) {
i = checkstate_pc[id] - g->lowpc;
if (i < g->textsize) {
i /= HISTFRACTION * sizeof(*g->kcount);
g->kcount[i]++;
}
}
#endif
if (pscnt > 1)
return;
KASSERT(p != NULL, ("NULL process in interrupt state"));
p->p_iticks++;
cp_time[CP_INTR]++;
}
schedclock(p);
/* Update resource usage integrals and maximums. */
if ((pstats = p->p_stats) != NULL &&
(ru = &pstats->p_ru) != NULL &&
(vm = p->p_vmspace) != NULL) {
ru->ru_ixrss += pgtok(vm->vm_tsize);
ru->ru_idrss += pgtok(vm->vm_dsize);
ru->ru_isrss += pgtok(vm->vm_ssize);
rss = pgtok(vmspace_resident_count(vm));
if (ru->ru_maxrss < rss)
ru->ru_maxrss = rss;
}
}
/*
* For statclock, we send an IPI to all CPU's to have them call this
* function.
*/
void
forward_statclock(int pscnt)
forwarded_statclock(struct trapframe frame)
{
int map;
int id;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_statclock(%d)", pscnt);
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
/* Step 1: Probe state (user, cpu, interrupt, spinlock, idle ) */
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
checkstate_probed_cpus = 0;
if (map != 0)
ipi_selected(map, IPI_CHECKSTATE);
i = 0;
while (checkstate_probed_cpus != map) {
/* spin */
i++;
if (i == 100000) {
#ifdef DIAGNOSTIC
printf("forward_statclock: checkstate %x\n",
checkstate_probed_cpus);
#endif
break;
}
}
/*
* Step 2: walk through other processors processes, update ticks and
* profiling info.
*/
map = 0;
for (id = 0; id < mp_ncpus; id++) {
if (id == PCPU_GET(cpuid))
continue;
if (((1 << id) & checkstate_probed_cpus) == 0)
continue;
forwarded_statclock(id, pscnt, &map);
}
if (map != 0) {
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#ifdef DIAGNOSTIC
printf("forward_statclock: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
}
}
void
forward_hardclock(int pscnt)
{
int map;
int id;
struct proc *p;
struct pstats *pstats;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_hardclock(%d)", pscnt);
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
/* Step 1: Probe state (user, cpu, interrupt, spinlock, idle) */
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
checkstate_probed_cpus = 0;
if (map != 0)
ipi_selected(map, IPI_CHECKSTATE);
i = 0;
while (checkstate_probed_cpus != map) {
/* spin */
i++;
if (i == 100000) {
#ifdef DIAGNOSTIC
printf("forward_hardclock: checkstate %x\n",
checkstate_probed_cpus);
#endif
break;
}
}
/*
* Step 2: walk through other processors processes, update virtual
* timer and profiling timer. If stathz == 0, also update ticks and
* profiling info.
*/
map = 0;
for (id = 0; id < mp_ncpus; id++) {
if (id == PCPU_GET(cpuid))
continue;
if (((1 << id) & checkstate_probed_cpus) == 0)
continue;
p = checkstate_curproc[id];
if (p) {
pstats = p->p_stats;
if (checkstate_cpustate[id] == CHECKSTATE_USER &&
timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) &&
itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) {
p->p_sflag |= PS_ALRMPEND;
map |= (1 << id);
}
if (timevalisset(&pstats->p_timer[ITIMER_PROF].it_value) &&
itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) {
p->p_sflag |= PS_PROFPEND;
map |= (1 << id);
}
}
if (stathz == 0) {
forwarded_statclock( id, pscnt, &map);
}
}
if (map != 0) {
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#ifdef DIAGNOSTIC
printf("forward_hardclock: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
}
}
void
forward_signal(struct proc *p)
{
int map;
int id;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_signal(%p)", p);
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
if (!forward_signal_enabled)
return;
mtx_lock_spin(&sched_lock);
while (1) {
if (p->p_stat != SRUN) {
mtx_unlock_spin(&sched_lock);
return;
}
id = p->p_oncpu;
mtx_unlock_spin(&sched_lock);
if (id == 0xff)
return;
map = (1<<id);
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#if 0
printf("forward_signal: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
mtx_lock_spin(&sched_lock);
if (id == p->p_oncpu) {
mtx_unlock_spin(&sched_lock);
return;
}
}
statclock_process(curproc, TRAPF_PC(&frame), TRAPF_USERMODE(&frame));
mtx_unlock_spin(&sched_lock);
}
void
forward_roundrobin(void)
forward_statclock(void)
{
int map;
CTR0(KTR_SMP, "forward_statclock");
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
if (map != 0)
ipi_selected(map, IPI_STATCLOCK);
}
/*
* For each hardclock(), we send an IPI to all other CPU's to have them
* execute this function. It would be nice to reduce contention on
* sched_lock if we could simply peek at the CPU to determine the user/kernel
* state and call hardclock_process() on the CPU receiving the clock interrupt
* and then just use a simple IPI to handle any ast's if needed.
*/
void
forwarded_hardclock(struct trapframe frame)
{
mtx_lock_spin(&sched_lock);
hardclock_process(curproc, TRAPF_USERMODE(&frame));
mtx_unlock_spin(&sched_lock);
}
void
forward_hardclock(void)
{
u_int map;
int i;
CTR0(KTR_SMP, "forward_roundrobin()");
CTR0(KTR_SMP, "forward_hardclock");
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
if (!forward_roundrobin_enabled)
return;
resched_cpus |= PCPU_GET(other_cpus);
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
#if 1
ipi_selected(map, IPI_AST);
#else
ipi_all_but_self(IPI_AST);
#endif
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#if 0
printf("forward_roundrobin: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
if (map != 0)
ipi_selected(map, IPI_HARDCLOCK);
}
/*
* When called the executing CPU will send an IPI to all other CPUs
* requesting that they halt execution.
*
* Usually (but not necessarily) called with 'other_cpus' as its arg.
*
* - Signals all CPUs in map to stop.
* - Waits for each to stop.
*
* Returns:
* -1: error
* 0: NA
* 1: ok
*
* XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
* from executing at same time.
*/
int
stop_cpus(u_int map)
{
int count = 0;
if (!smp_started)
return 0;
/* send the Xcpustop IPI to all CPUs in map */
ipi_selected(map, IPI_STOP);
while (count++ < 100000 && (stopped_cpus & map) != map)
/* spin */ ;
#ifdef DIAGNOSTIC
if ((stopped_cpus & map) != map)
printf("Warning: CPUs 0x%x did not stop!\n",
(~(stopped_cpus & map)) & map);
#endif
return 1;
}
/*
* Called by a CPU to restart stopped CPUs.
*
* Usually (but not necessarily) called with 'stopped_cpus' as its arg.
*
* - Signals all CPUs in map to restart.
* - Waits for each to restart.
*
* Returns:
* -1: error
* 0: NA
* 1: ok
*/
int
restart_cpus(u_int map)
{
int count = 0;
if (!smp_started)
return 0;
started_cpus = map; /* signal other cpus to restart */
/* wait for each to clear its bit */
while (count++ < 100000 && (stopped_cpus & map) != 0)
/* spin */ ;
#ifdef DIAGNOSTIC
if ((stopped_cpus & map) != 0)
printf("Warning: CPUs 0x%x did not restart!\n",
(~(stopped_cpus & map)) & map);
#endif
return 1;
}
#ifdef APIC_INTR_REORDER
/*
* Maintain mapping from softintr vector to isr bit in local apic.
@ -2810,73 +2385,6 @@ set_lapic_isrloc(int intr, int vector)
}
#endif
/*
* All-CPU rendezvous. CPUs are signalled, all execute the setup function
* (if specified), rendezvous, execute the action function (if specified),
* rendezvous again, execute the teardown function (if specified), and then
* resume.
*
* Note that the supplied external functions _must_ be reentrant and aware
* that they are running in parallel and in an unknown lock context.
*/
static void (*smp_rv_setup_func)(void *arg);
static void (*smp_rv_action_func)(void *arg);
static void (*smp_rv_teardown_func)(void *arg);
static void *smp_rv_func_arg;
static volatile int smp_rv_waiters[2];
void
smp_rendezvous_action(void)
{
/* setup function */
if (smp_rv_setup_func != NULL)
smp_rv_setup_func(smp_rv_func_arg);
/* spin on entry rendezvous */
atomic_add_int(&smp_rv_waiters[0], 1);
while (smp_rv_waiters[0] < mp_ncpus)
;
/* action function */
if (smp_rv_action_func != NULL)
smp_rv_action_func(smp_rv_func_arg);
/* spin on exit rendezvous */
atomic_add_int(&smp_rv_waiters[1], 1);
while (smp_rv_waiters[1] < mp_ncpus)
;
/* teardown function */
if (smp_rv_teardown_func != NULL)
smp_rv_teardown_func(smp_rv_func_arg);
}
void
smp_rendezvous(void (* setup_func)(void *),
void (* action_func)(void *),
void (* teardown_func)(void *),
void *arg)
{
/* obtain rendezvous lock */
mtx_lock_spin(&smp_rv_mtx);
/* set static function pointers */
smp_rv_setup_func = setup_func;
smp_rv_action_func = action_func;
smp_rv_teardown_func = teardown_func;
smp_rv_func_arg = arg;
smp_rv_waiters[0] = 0;
smp_rv_waiters[1] = 0;
/*
* signal other processors, which will enter the IPI with interrupts off
*/
ipi_all_but_self(IPI_RENDEZVOUS);
/* call executor function */
smp_rendezvous_action();
/* release lock */
mtx_unlock_spin(&smp_rv_mtx);
}
/*
* send an IPI to a set of cpus.
*/

624
sys/i386/i386/mptable.c
View File

@ -42,6 +42,7 @@
#include <sys/malloc.h>
#include <sys/memrange.h>
#include <sys/mutex.h>
#include <sys/smp.h>
#include <sys/dkstat.h>
#include <sys/cons.h> /* cngetc() */
@ -57,9 +58,9 @@
#include <sys/gmon.h>
#endif
#include <machine/smp.h>
#include <machine/apic.h>
#include <machine/atomic.h>
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/ipl.h>
#include <machine/mpapic.h>
@ -243,7 +244,6 @@ int current_postcode;
extern struct region_descriptor r_gdt, r_idt;
int bsp_apic_ready = 0; /* flags useability of BSP apic */
int mp_ncpus; /* # of CPUs, including BSP */
int mp_naps; /* # of Applications processors */
int mp_nbusses; /* # of busses */
int mp_napics; /* # of IO APICs */
@ -273,9 +273,6 @@ int io_num_to_apic_id[NAPICID];
int apic_id_to_logical[NAPICID];
/* Bitmap of all available CPUs */
u_int all_cpus;
/* AP uses this during bootstrap. Do not staticize. */
char *bootSTK;
static int bootAP;
@ -288,28 +285,9 @@ extern pt_entry_t *SMPpt;
struct pcb stoppcbs[MAXCPU];
int smp_started; /* has the system started? */
int smp_active = 0; /* are the APs allowed to run? */
SYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RW, &smp_active, 0, "");
/* XXX maybe should be hw.ncpu */
static int smp_cpus = 1; /* how many cpu's running */
SYSCTL_INT(_machdep, OID_AUTO, smp_cpus, CTLFLAG_RD, &smp_cpus, 0, "");
int invltlb_ok = 0; /* throttle smp_invltlb() till safe */
SYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, "");
/* Enable forwarding of a signal to a process running on a different CPU */
static int forward_signal_enabled = 1;
SYSCTL_INT(_machdep, OID_AUTO, forward_signal_enabled, CTLFLAG_RW,
&forward_signal_enabled, 0, "");
/* Enable forwarding of roundrobin to all other cpus */
static int forward_roundrobin_enabled = 1;
SYSCTL_INT(_machdep, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW,
&forward_roundrobin_enabled, 0, "");
/*
* Local data and functions.
*/
@ -354,9 +332,6 @@ struct mtx mcount_mtx;
struct mtx com_mtx;
#endif /* USE_COMLOCK */
/* lock around the MP rendezvous */
static struct mtx smp_rv_mtx;
static void
init_locks(void)
{
@ -367,13 +342,9 @@ init_locks(void)
*/
mtx_init(&mcount_mtx, "mcount", MTX_DEF);
mtx_init(&smp_rv_mtx, "smp rendezvous", MTX_SPIN);
#ifdef USE_COMLOCK
mtx_init(&com_mtx, "com", MTX_SPIN);
#endif /* USE_COMLOCK */
mtx_init(&ap_boot_mtx, "ap boot", MTX_SPIN);
}
/*
@ -397,8 +368,8 @@ mp_bootaddress(u_int basemem)
/*
* Look for an Intel MP spec table (ie, SMP capable hardware).
*/
int
mp_probe(void)
void
i386_mp_probe(void)
{
int x;
u_long segment;
@ -427,7 +398,7 @@ mp_probe(void)
/* nothing found */
mpfps = (mpfps_t)0;
mp_capable = 0;
return 0;
return;
found:
/* calculate needed resources */
@ -436,15 +407,19 @@ mp_probe(void)
/* flag fact that we are running multiple processors */
mp_capable = 1;
return 1;
}
int
cpu_mp_probe(void)
{
return (mp_capable);
}
/*
* Initialize the SMP hardware and the APIC and start up the AP's.
*/
void
mp_start(void)
cpu_mp_start(void)
{
POSTCODE(MP_START_POST);
@ -453,6 +428,8 @@ mp_start(void)
mp_enable(boot_address);
else
panic("MP hardware not found!");
cpu_setregs();
}
@ -460,13 +437,12 @@ mp_start(void)
* Print various information about the SMP system hardware and setup.
*/
void
mp_announce(void)
cpu_mp_announce(void)
{
int x;
POSTCODE(MP_ANNOUNCE_POST);
printf("FreeBSD/SMP: Multiprocessor motherboard\n");
printf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0));
printf(", version: 0x%08x", cpu_apic_versions[0]);
printf(", at 0x%08x\n", cpu_apic_address);
@ -623,8 +599,12 @@ mp_enable(u_int boot_addr)
setidt(XINVLTLB_OFFSET, Xinvltlb,
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
/* install an inter-CPU IPI for reading processor state */
setidt(XCPUCHECKSTATE_OFFSET, Xcpucheckstate,
/* install an inter-CPU IPI for forwarding hardclock() */
setidt(XHARDCLOCK_OFFSET, Xhardclock,
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
/* install an inter-CPU IPI for forwarding statclock() */
setidt(XSTATCLOCK_OFFSET, Xstatclock,
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
/* install an inter-CPU IPI for all-CPU rendezvous */
@ -1938,6 +1918,8 @@ start_all_aps(u_int boot_addr)
POSTCODE(START_ALL_APS_POST);
mtx_init(&ap_boot_mtx, "ap boot", MTX_SPIN);
/* initialize BSP's local APIC */
apic_initialize();
bsp_apic_ready = 1;
@ -1985,8 +1967,8 @@ start_all_aps(u_int boot_addr)
(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
/* prime data page for it to use */
SLIST_INSERT_HEAD(&cpuhead, gd, gd_allcpu);
gd->gd_cpuid = x;
globaldata_register(gd);
/* setup a vector to our boot code */
*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
@ -2328,472 +2310,65 @@ ap_init(void)
panic("scheduler returned us to ap_init");
}
#define CHECKSTATE_USER 0
#define CHECKSTATE_SYS 1
#define CHECKSTATE_INTR 2
/* Do not staticize. Used from apic_vector.s */
struct proc* checkstate_curproc[MAXCPU];
int checkstate_cpustate[MAXCPU];
u_long checkstate_pc[MAXCPU];
#define PC_TO_INDEX(pc, prof) \
((int)(((u_quad_t)((pc) - (prof)->pr_off) * \
(u_quad_t)((prof)->pr_scale)) >> 16) & ~1)
static void
addupc_intr_forwarded(struct proc *p, int id, int *astmap)
{
int i;
struct uprof *prof;
u_long pc;
pc = checkstate_pc[id];
prof = &p->p_stats->p_prof;
if (pc >= prof->pr_off &&
(i = PC_TO_INDEX(pc, prof)) < prof->pr_size) {
mtx_assert(&sched_lock, MA_OWNED);
if ((p->p_sflag & PS_OWEUPC) == 0) {
prof->pr_addr = pc;
prof->pr_ticks = 1;
p->p_sflag |= PS_OWEUPC;
}
*astmap |= (1 << id);
}
}
static void
forwarded_statclock(int id, int pscnt, int *astmap)
{
struct pstats *pstats;
long rss;
struct rusage *ru;
struct vmspace *vm;
int cpustate;
struct proc *p;
#ifdef GPROF
register struct gmonparam *g;
int i;
#endif
mtx_assert(&sched_lock, MA_OWNED);
p = checkstate_curproc[id];
cpustate = checkstate_cpustate[id];
/* XXX */
if (p->p_ithd)
cpustate = CHECKSTATE_INTR;
else if (p == SMP_prvspace[id].globaldata.gd_idleproc)
cpustate = CHECKSTATE_SYS;
switch (cpustate) {
case CHECKSTATE_USER:
if (p->p_sflag & PS_PROFIL)
addupc_intr_forwarded(p, id, astmap);
if (pscnt > 1)
return;
p->p_uticks++;
if (p->p_nice > NZERO)
cp_time[CP_NICE]++;
else
cp_time[CP_USER]++;
break;
case CHECKSTATE_SYS:
#ifdef GPROF
/*
* Kernel statistics are just like addupc_intr, only easier.
*/
g = &_gmonparam;
if (g->state == GMON_PROF_ON) {
i = checkstate_pc[id] - g->lowpc;
if (i < g->textsize) {
i /= HISTFRACTION * sizeof(*g->kcount);
g->kcount[i]++;
}
}
#endif
if (pscnt > 1)
return;
p->p_sticks++;
if (p == SMP_prvspace[id].globaldata.gd_idleproc)
cp_time[CP_IDLE]++;
else
cp_time[CP_SYS]++;
break;
case CHECKSTATE_INTR:
default:
#ifdef GPROF
/*
* Kernel statistics are just like addupc_intr, only easier.
*/
g = &_gmonparam;
if (g->state == GMON_PROF_ON) {
i = checkstate_pc[id] - g->lowpc;
if (i < g->textsize) {
i /= HISTFRACTION * sizeof(*g->kcount);
g->kcount[i]++;
}
}
#endif
if (pscnt > 1)
return;
KASSERT(p != NULL, ("NULL process in interrupt state"));
p->p_iticks++;
cp_time[CP_INTR]++;
}
schedclock(p);
/* Update resource usage integrals and maximums. */
if ((pstats = p->p_stats) != NULL &&
(ru = &pstats->p_ru) != NULL &&
(vm = p->p_vmspace) != NULL) {
ru->ru_ixrss += pgtok(vm->vm_tsize);
ru->ru_idrss += pgtok(vm->vm_dsize);
ru->ru_isrss += pgtok(vm->vm_ssize);
rss = pgtok(vmspace_resident_count(vm));
if (ru->ru_maxrss < rss)
ru->ru_maxrss = rss;
}
}
/*
* For statclock, we send an IPI to all CPU's to have them call this
* function.
*/
void
forward_statclock(int pscnt)
forwarded_statclock(struct trapframe frame)
{
int map;
int id;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_statclock(%d)", pscnt);
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
/* Step 1: Probe state (user, cpu, interrupt, spinlock, idle ) */
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
checkstate_probed_cpus = 0;
if (map != 0)
ipi_selected(map, IPI_CHECKSTATE);
i = 0;
while (checkstate_probed_cpus != map) {
/* spin */
i++;
if (i == 100000) {
#ifdef DIAGNOSTIC
printf("forward_statclock: checkstate %x\n",
checkstate_probed_cpus);
#endif
break;
}
}
/*
* Step 2: walk through other processors processes, update ticks and
* profiling info.
*/
map = 0;
for (id = 0; id < mp_ncpus; id++) {
if (id == PCPU_GET(cpuid))
continue;
if (((1 << id) & checkstate_probed_cpus) == 0)
continue;
forwarded_statclock(id, pscnt, &map);
}
if (map != 0) {
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#ifdef DIAGNOSTIC
printf("forward_statclock: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
}
}
void
forward_hardclock(int pscnt)
{
int map;
int id;
struct proc *p;
struct pstats *pstats;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_hardclock(%d)", pscnt);
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
/* Step 1: Probe state (user, cpu, interrupt, spinlock, idle) */
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
checkstate_probed_cpus = 0;
if (map != 0)
ipi_selected(map, IPI_CHECKSTATE);
i = 0;
while (checkstate_probed_cpus != map) {
/* spin */
i++;
if (i == 100000) {
#ifdef DIAGNOSTIC
printf("forward_hardclock: checkstate %x\n",
checkstate_probed_cpus);
#endif
break;
}
}
/*
* Step 2: walk through other processors processes, update virtual
* timer and profiling timer. If stathz == 0, also update ticks and
* profiling info.
*/
map = 0;
for (id = 0; id < mp_ncpus; id++) {
if (id == PCPU_GET(cpuid))
continue;
if (((1 << id) & checkstate_probed_cpus) == 0)
continue;
p = checkstate_curproc[id];
if (p) {
pstats = p->p_stats;
if (checkstate_cpustate[id] == CHECKSTATE_USER &&
timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) &&
itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) {
p->p_sflag |= PS_ALRMPEND;
map |= (1 << id);
}
if (timevalisset(&pstats->p_timer[ITIMER_PROF].it_value) &&
itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) {
p->p_sflag |= PS_PROFPEND;
map |= (1 << id);
}
}
if (stathz == 0) {
forwarded_statclock( id, pscnt, &map);
}
}
if (map != 0) {
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#ifdef DIAGNOSTIC
printf("forward_hardclock: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
}
}
void
forward_signal(struct proc *p)
{
int map;
int id;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_signal(%p)", p);
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
if (!forward_signal_enabled)
return;
mtx_lock_spin(&sched_lock);
while (1) {
if (p->p_stat != SRUN) {
mtx_unlock_spin(&sched_lock);
return;
}
id = p->p_oncpu;
mtx_unlock_spin(&sched_lock);
if (id == 0xff)
return;
map = (1<<id);
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#if 0
printf("forward_signal: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
mtx_lock_spin(&sched_lock);
if (id == p->p_oncpu) {
mtx_unlock_spin(&sched_lock);
return;
}
}
statclock_process(curproc, TRAPF_PC(&frame), TRAPF_USERMODE(&frame));
mtx_unlock_spin(&sched_lock);
}
void
forward_roundrobin(void)
forward_statclock(void)
{
int map;
CTR0(KTR_SMP, "forward_statclock");
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
if (map != 0)
ipi_selected(map, IPI_STATCLOCK);
}
/*
* For each hardclock(), we send an IPI to all other CPU's to have them
* execute this function. It would be nice to reduce contention on
* sched_lock if we could simply peek at the CPU to determine the user/kernel
* state and call hardclock_process() on the CPU receiving the clock interrupt
* and then just use a simple IPI to handle any ast's if needed.
*/
void
forwarded_hardclock(struct trapframe frame)
{
mtx_lock_spin(&sched_lock);
hardclock_process(curproc, TRAPF_USERMODE(&frame));
mtx_unlock_spin(&sched_lock);
}
void
forward_hardclock(void)
{
u_int map;
int i;
CTR0(KTR_SMP, "forward_roundrobin()");
CTR0(KTR_SMP, "forward_hardclock");
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
if (!forward_roundrobin_enabled)
return;
resched_cpus |= PCPU_GET(other_cpus);
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
#if 1
ipi_selected(map, IPI_AST);
#else
ipi_all_but_self(IPI_AST);
#endif
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#if 0
printf("forward_roundrobin: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
if (map != 0)
ipi_selected(map, IPI_HARDCLOCK);
}
/*
* When called the executing CPU will send an IPI to all other CPUs
* requesting that they halt execution.
*
* Usually (but not necessarily) called with 'other_cpus' as its arg.
*
* - Signals all CPUs in map to stop.
* - Waits for each to stop.
*
* Returns:
* -1: error
* 0: NA
* 1: ok
*
* XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
* from executing at same time.
*/
int
stop_cpus(u_int map)
{
int count = 0;
if (!smp_started)
return 0;
/* send the Xcpustop IPI to all CPUs in map */
ipi_selected(map, IPI_STOP);
while (count++ < 100000 && (stopped_cpus & map) != map)
/* spin */ ;
#ifdef DIAGNOSTIC
if ((stopped_cpus & map) != map)
printf("Warning: CPUs 0x%x did not stop!\n",
(~(stopped_cpus & map)) & map);
#endif
return 1;
}
/*
* Called by a CPU to restart stopped CPUs.
*
* Usually (but not necessarily) called with 'stopped_cpus' as its arg.
*
* - Signals all CPUs in map to restart.
* - Waits for each to restart.
*
* Returns:
* -1: error
* 0: NA
* 1: ok
*/
int
restart_cpus(u_int map)
{
int count = 0;
if (!smp_started)
return 0;
started_cpus = map; /* signal other cpus to restart */
/* wait for each to clear its bit */
while (count++ < 100000 && (stopped_cpus & map) != 0)
/* spin */ ;
#ifdef DIAGNOSTIC
if ((stopped_cpus & map) != 0)
printf("Warning: CPUs 0x%x did not restart!\n",
(~(stopped_cpus & map)) & map);
#endif
return 1;
}
#ifdef APIC_INTR_REORDER
/*
* Maintain mapping from softintr vector to isr bit in local apic.
@ -2810,73 +2385,6 @@ set_lapic_isrloc(int intr, int vector)
}
#endif
/*
* All-CPU rendezvous. CPUs are signalled, all execute the setup function
* (if specified), rendezvous, execute the action function (if specified),
* rendezvous again, execute the teardown function (if specified), and then
* resume.
*
* Note that the supplied external functions _must_ be reentrant and aware
* that they are running in parallel and in an unknown lock context.
*/
static void (*smp_rv_setup_func)(void *arg);
static void (*smp_rv_action_func)(void *arg);
static void (*smp_rv_teardown_func)(void *arg);
static void *smp_rv_func_arg;
static volatile int smp_rv_waiters[2];
void
smp_rendezvous_action(void)
{
/* setup function */
if (smp_rv_setup_func != NULL)
smp_rv_setup_func(smp_rv_func_arg);
/* spin on entry rendezvous */
atomic_add_int(&smp_rv_waiters[0], 1);
while (smp_rv_waiters[0] < mp_ncpus)
;
/* action function */
if (smp_rv_action_func != NULL)
smp_rv_action_func(smp_rv_func_arg);
/* spin on exit rendezvous */
atomic_add_int(&smp_rv_waiters[1], 1);
while (smp_rv_waiters[1] < mp_ncpus)
;
/* teardown function */
if (smp_rv_teardown_func != NULL)
smp_rv_teardown_func(smp_rv_func_arg);
}
void
smp_rendezvous(void (* setup_func)(void *),
void (* action_func)(void *),
void (* teardown_func)(void *),
void *arg)
{
/* obtain rendezvous lock */
mtx_lock_spin(&smp_rv_mtx);
/* set static function pointers */
smp_rv_setup_func = setup_func;
smp_rv_action_func = action_func;
smp_rv_teardown_func = teardown_func;
smp_rv_func_arg = arg;
smp_rv_waiters[0] = 0;
smp_rv_waiters[1] = 0;
/*
* signal other processors, which will enter the IPI with interrupts off
*/
ipi_all_but_self(IPI_RENDEZVOUS);
/* call executor function */
smp_rendezvous_action();
/* release lock */
mtx_unlock_spin(&smp_rv_mtx);
}
/*
* send an IPI to a set of cpus.
*/

14
sys/i386/i386/sys_machdep.c
View File

@ -41,6 +41,7 @@
#include <sys/ipl.h>
#include <sys/malloc.h>
#include <sys/proc.h>
#include <sys/smp.h>
#include <vm/vm.h>
#include <sys/lock.h>
@ -53,9 +54,6 @@
#include <machine/cpu.h>
#include <machine/pcb_ext.h> /* pcb.h included by sys/user.h */
#include <machine/sysarch.h>
#ifdef SMP
#include <machine/smp.h>
#endif
#include <vm/vm_kern.h> /* for kernel_map */
@ -154,7 +152,15 @@ i386_extend_pcb(struct proc *p)
ssdtosd(&ssd, &ext->ext_tssd);
/* switch to the new TSS after syscall completes */
need_resched();
/*
* XXX: The sched_lock here needs to be over a slightly larger area.
* I have patches to more properly lock accesses to process ldt's
* and tss's that still need to be reviewed, but this keeps us from
* panic'ing on the mtx_assert() in need_resched() for the time being.
*/
mtx_lock_spin(&sched_lock);
need_resched(p);
mtx_unlock_spin(&sched_lock);
return 0;
}

4
sys/i386/i386/trap.c
View File

@ -177,7 +177,7 @@ userret(p, frame, oticks)
mtx_lock_spin(&sched_lock);
p->p_pri.pri_level = p->p_pri.pri_user;
if (resched_wanted()) {
if (resched_wanted(p)) {
/*
* Since we are curproc, clock will normally just change
* our priority without moving us from one queue to another
@ -1277,7 +1277,7 @@ ast(framep)
* acquiring and releasing mutexes in assembly is not fun.
*/
mtx_lock_spin(&sched_lock);
if (!(astpending(p) || resched_wanted())) {
if (!(astpending(p) || resched_wanted(p))) {
mtx_unlock_spin(&sched_lock);
return;
}

13
sys/i386/i386/tsc.c
View File

@ -211,6 +211,10 @@ clkintr(struct clockframe frame)
mtx_unlock_spin(&clock_lock);
}
timer_func(&frame);
#ifdef SMP
if (timer_func == hardclock)
forward_hardclock();
#endif
switch (timer0_state) {
case RELEASED:
@ -253,6 +257,9 @@ clkintr(struct clockframe frame)
timer_func = hardclock;
timer0_state = RELEASED;
hardclock(&frame);
#ifdef SMP
forward_hardclock();
#endif
}
break;
}
@ -374,8 +381,12 @@ release_timer2()
static void
rtcintr(struct clockframe frame)
{
while (rtcin(RTC_INTR) & RTCIR_PERIOD)
while (rtcin(RTC_INTR) & RTCIR_PERIOD) {
statclock(&frame);
#ifdef SMP
forward_statclock();
#endif
}
}
#include "opt_ddb.h"

4
sys/i386/i386/vm_machdep.c
View File

@ -59,14 +59,12 @@
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/mutex.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/unistd.h>
#include <machine/cpu.h>
#include <machine/md_var.h>
#ifdef SMP
#include <machine/smp.h>
#endif
#include <machine/pcb.h>
#include <machine/pcb_ext.h>
#include <machine/vm86.h>

2
sys/i386/include/cpu.h
View File

@ -65,8 +65,6 @@
#define CLKF_USERMODE(framep) \
((ISPL((framep)->cf_cs) == SEL_UPL) || ((framep)->cf_eflags & PSL_VM))
#define CLKF_INTR(framep) (curproc->p_intr_nesting_level >= 2)
#define CLKF_PC(framep) ((framep)->cf_eip)
/*

3
sys/i386/include/globaldata.h
View File

@ -74,9 +74,6 @@ struct globaldata {
#endif
};
SLIST_HEAD(cpuhead, globaldata);
extern struct cpuhead cpuhead;
#ifdef SMP
/*
* This is the upper (0xff800000) address space layout that is per-cpu.

19
sys/i386/include/ipl.h
View File

@ -42,23 +42,4 @@
#include <i386/isa/icu_ipl.h>
#endif
#ifndef LOCORE
#ifdef SMP
/*
* Interprocessor interrupts for SMP.
*/
#define IPI_INVLTLB XINVLTLB_OFFSET
#define IPI_RENDEZVOUS XRENDEZVOUS_OFFSET
#define IPI_AST XCPUAST_OFFSET
#define IPI_CHECKSTATE XCPUCHECKSTATE_OFFSET
#define IPI_STOP XCPUSTOP_OFFSET
void ipi_selected(u_int32_t cpus, u_int ipi);
void ipi_all(u_int ipi);
void ipi_all_but_self(u_int ipi);
void ipi_self(u_int ipi);
#endif /* SMP */
#endif /* !LOCORE */
#endif /* !_MACHINE_IPL_H_ */

624
sys/i386/include/mptable.h
View File

@ -42,6 +42,7 @@
#include <sys/malloc.h>
#include <sys/memrange.h>
#include <sys/mutex.h>
#include <sys/smp.h>
#include <sys/dkstat.h>
#include <sys/cons.h> /* cngetc() */
@ -57,9 +58,9 @@
#include <sys/gmon.h>
#endif
#include <machine/smp.h>
#include <machine/apic.h>
#include <machine/atomic.h>
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/ipl.h>
#include <machine/mpapic.h>
@ -243,7 +244,6 @@ int current_postcode;
extern struct region_descriptor r_gdt, r_idt;
int bsp_apic_ready = 0; /* flags useability of BSP apic */
int mp_ncpus; /* # of CPUs, including BSP */
int mp_naps; /* # of Applications processors */
int mp_nbusses; /* # of busses */
int mp_napics; /* # of IO APICs */
@ -273,9 +273,6 @@ int io_num_to_apic_id[NAPICID];
int apic_id_to_logical[NAPICID];
/* Bitmap of all available CPUs */
u_int all_cpus;
/* AP uses this during bootstrap. Do not staticize. */
char *bootSTK;
static int bootAP;
@ -288,28 +285,9 @@ extern pt_entry_t *SMPpt;
struct pcb stoppcbs[MAXCPU];
int smp_started; /* has the system started? */
int smp_active = 0; /* are the APs allowed to run? */
SYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RW, &smp_active, 0, "");
/* XXX maybe should be hw.ncpu */
static int smp_cpus = 1; /* how many cpu's running */
SYSCTL_INT(_machdep, OID_AUTO, smp_cpus, CTLFLAG_RD, &smp_cpus, 0, "");
int invltlb_ok = 0; /* throttle smp_invltlb() till safe */
SYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, "");
/* Enable forwarding of a signal to a process running on a different CPU */
static int forward_signal_enabled = 1;
SYSCTL_INT(_machdep, OID_AUTO, forward_signal_enabled, CTLFLAG_RW,
&forward_signal_enabled, 0, "");
/* Enable forwarding of roundrobin to all other cpus */
static int forward_roundrobin_enabled = 1;
SYSCTL_INT(_machdep, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW,
&forward_roundrobin_enabled, 0, "");
/*
* Local data and functions.
*/
@ -354,9 +332,6 @@ struct mtx mcount_mtx;
struct mtx com_mtx;
#endif /* USE_COMLOCK */
/* lock around the MP rendezvous */
static struct mtx smp_rv_mtx;
static void
init_locks(void)
{
@ -367,13 +342,9 @@ init_locks(void)
*/
mtx_init(&mcount_mtx, "mcount", MTX_DEF);
mtx_init(&smp_rv_mtx, "smp rendezvous", MTX_SPIN);
#ifdef USE_COMLOCK
mtx_init(&com_mtx, "com", MTX_SPIN);
#endif /* USE_COMLOCK */
mtx_init(&ap_boot_mtx, "ap boot", MTX_SPIN);
}
/*
@ -397,8 +368,8 @@ mp_bootaddress(u_int basemem)
/*
* Look for an Intel MP spec table (ie, SMP capable hardware).
*/
int
mp_probe(void)
void
i386_mp_probe(void)
{
int x;
u_long segment;
@ -427,7 +398,7 @@ mp_probe(void)
/* nothing found */
mpfps = (mpfps_t)0;
mp_capable = 0;
return 0;
return;
found:
/* calculate needed resources */
@ -436,15 +407,19 @@ mp_probe(void)
/* flag fact that we are running multiple processors */
mp_capable = 1;
return 1;
}
int
cpu_mp_probe(void)
{
return (mp_capable);
}
/*
* Initialize the SMP hardware and the APIC and start up the AP's.
*/
void
mp_start(void)
cpu_mp_start(void)
{
POSTCODE(MP_START_POST);
@ -453,6 +428,8 @@ mp_start(void)
mp_enable(boot_address);
else
panic("MP hardware not found!");
cpu_setregs();
}
@ -460,13 +437,12 @@ mp_start(void)
* Print various information about the SMP system hardware and setup.
*/
void
mp_announce(void)
cpu_mp_announce(void)
{
int x;
POSTCODE(MP_ANNOUNCE_POST);
printf("FreeBSD/SMP: Multiprocessor motherboard\n");
printf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0));
printf(", version: 0x%08x", cpu_apic_versions[0]);
printf(", at 0x%08x\n", cpu_apic_address);
@ -623,8 +599,12 @@ mp_enable(u_int boot_addr)
setidt(XINVLTLB_OFFSET, Xinvltlb,
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
/* install an inter-CPU IPI for reading processor state */
setidt(XCPUCHECKSTATE_OFFSET, Xcpucheckstate,
/* install an inter-CPU IPI for forwarding hardclock() */
setidt(XHARDCLOCK_OFFSET, Xhardclock,
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
/* install an inter-CPU IPI for forwarding statclock() */
setidt(XSTATCLOCK_OFFSET, Xstatclock,
SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
/* install an inter-CPU IPI for all-CPU rendezvous */
@ -1938,6 +1918,8 @@ start_all_aps(u_int boot_addr)
POSTCODE(START_ALL_APS_POST);
mtx_init(&ap_boot_mtx, "ap boot", MTX_SPIN);
/* initialize BSP's local APIC */
apic_initialize();
bsp_apic_ready = 1;
@ -1985,8 +1967,8 @@ start_all_aps(u_int boot_addr)
(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
/* prime data page for it to use */
SLIST_INSERT_HEAD(&cpuhead, gd, gd_allcpu);
gd->gd_cpuid = x;
globaldata_register(gd);
/* setup a vector to our boot code */
*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
@ -2328,472 +2310,65 @@ ap_init(void)
panic("scheduler returned us to ap_init");
}
#define CHECKSTATE_USER 0
#define CHECKSTATE_SYS 1
#define CHECKSTATE_INTR 2
/* Do not staticize. Used from apic_vector.s */
struct proc* checkstate_curproc[MAXCPU];
int checkstate_cpustate[MAXCPU];
u_long checkstate_pc[MAXCPU];
#define PC_TO_INDEX(pc, prof) \
((int)(((u_quad_t)((pc) - (prof)->pr_off) * \
(u_quad_t)((prof)->pr_scale)) >> 16) & ~1)
static void
addupc_intr_forwarded(struct proc *p, int id, int *astmap)
{
int i;
struct uprof *prof;
u_long pc;
pc = checkstate_pc[id];
prof = &p->p_stats->p_prof;
if (pc >= prof->pr_off &&
(i = PC_TO_INDEX(pc, prof)) < prof->pr_size) {
mtx_assert(&sched_lock, MA_OWNED);
if ((p->p_sflag & PS_OWEUPC) == 0) {
prof->pr_addr = pc;
prof->pr_ticks = 1;
p->p_sflag |= PS_OWEUPC;
}
*astmap |= (1 << id);
}
}
static void
forwarded_statclock(int id, int pscnt, int *astmap)
{
struct pstats *pstats;
long rss;
struct rusage *ru;
struct vmspace *vm;
int cpustate;
struct proc *p;
#ifdef GPROF
register struct gmonparam *g;
int i;
#endif
mtx_assert(&sched_lock, MA_OWNED);
p = checkstate_curproc[id];
cpustate = checkstate_cpustate[id];
/* XXX */
if (p->p_ithd)
cpustate = CHECKSTATE_INTR;
else if (p == SMP_prvspace[id].globaldata.gd_idleproc)
cpustate = CHECKSTATE_SYS;
switch (cpustate) {
case CHECKSTATE_USER:
if (p->p_sflag & PS_PROFIL)
addupc_intr_forwarded(p, id, astmap);
if (pscnt > 1)
return;
p->p_uticks++;
if (p->p_nice > NZERO)
cp_time[CP_NICE]++;
else
cp_time[CP_USER]++;
break;
case CHECKSTATE_SYS:
#ifdef GPROF
/*
* Kernel statistics are just like addupc_intr, only easier.
*/
g = &_gmonparam;
if (g->state == GMON_PROF_ON) {
i = checkstate_pc[id] - g->lowpc;
if (i < g->textsize) {
i /= HISTFRACTION * sizeof(*g->kcount);
g->kcount[i]++;
}
}
#endif
if (pscnt > 1)
return;
p->p_sticks++;
if (p == SMP_prvspace[id].globaldata.gd_idleproc)
cp_time[CP_IDLE]++;
else
cp_time[CP_SYS]++;
break;
case CHECKSTATE_INTR:
default:
#ifdef GPROF
/*
* Kernel statistics are just like addupc_intr, only easier.
*/
g = &_gmonparam;
if (g->state == GMON_PROF_ON) {
i = checkstate_pc[id] - g->lowpc;
if (i < g->textsize) {
i /= HISTFRACTION * sizeof(*g->kcount);
g->kcount[i]++;
}
}
#endif
if (pscnt > 1)
return;
KASSERT(p != NULL, ("NULL process in interrupt state"));
p->p_iticks++;
cp_time[CP_INTR]++;
}
schedclock(p);
/* Update resource usage integrals and maximums. */
if ((pstats = p->p_stats) != NULL &&
(ru = &pstats->p_ru) != NULL &&
(vm = p->p_vmspace) != NULL) {
ru->ru_ixrss += pgtok(vm->vm_tsize);
ru->ru_idrss += pgtok(vm->vm_dsize);
ru->ru_isrss += pgtok(vm->vm_ssize);
rss = pgtok(vmspace_resident_count(vm));
if (ru->ru_maxrss < rss)
ru->ru_maxrss = rss;
}
}
/*
* For statclock, we send an IPI to all CPU's to have them call this
* function.
*/
void
forward_statclock(int pscnt)
forwarded_statclock(struct trapframe frame)
{
int map;
int id;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_statclock(%d)", pscnt);
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
/* Step 1: Probe state (user, cpu, interrupt, spinlock, idle ) */
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
checkstate_probed_cpus = 0;
if (map != 0)
ipi_selected(map, IPI_CHECKSTATE);
i = 0;
while (checkstate_probed_cpus != map) {
/* spin */
i++;
if (i == 100000) {
#ifdef DIAGNOSTIC
printf("forward_statclock: checkstate %x\n",
checkstate_probed_cpus);
#endif
break;
}
}
/*
* Step 2: walk through other processors processes, update ticks and
* profiling info.
*/
map = 0;
for (id = 0; id < mp_ncpus; id++) {
if (id == PCPU_GET(cpuid))
continue;
if (((1 << id) & checkstate_probed_cpus) == 0)
continue;
forwarded_statclock(id, pscnt, &map);
}
if (map != 0) {
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#ifdef DIAGNOSTIC
printf("forward_statclock: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
}
}
void
forward_hardclock(int pscnt)
{
int map;
int id;
struct proc *p;
struct pstats *pstats;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_hardclock(%d)", pscnt);
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
/* Step 1: Probe state (user, cpu, interrupt, spinlock, idle) */
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
checkstate_probed_cpus = 0;
if (map != 0)
ipi_selected(map, IPI_CHECKSTATE);
i = 0;
while (checkstate_probed_cpus != map) {
/* spin */
i++;
if (i == 100000) {
#ifdef DIAGNOSTIC
printf("forward_hardclock: checkstate %x\n",
checkstate_probed_cpus);
#endif
break;
}
}
/*
* Step 2: walk through other processors processes, update virtual
* timer and profiling timer. If stathz == 0, also update ticks and
* profiling info.
*/
map = 0;
for (id = 0; id < mp_ncpus; id++) {
if (id == PCPU_GET(cpuid))
continue;
if (((1 << id) & checkstate_probed_cpus) == 0)
continue;
p = checkstate_curproc[id];
if (p) {
pstats = p->p_stats;
if (checkstate_cpustate[id] == CHECKSTATE_USER &&
timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) &&
itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) {
p->p_sflag |= PS_ALRMPEND;
map |= (1 << id);
}
if (timevalisset(&pstats->p_timer[ITIMER_PROF].it_value) &&
itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) {
p->p_sflag |= PS_PROFPEND;
map |= (1 << id);
}
}
if (stathz == 0) {
forwarded_statclock( id, pscnt, &map);
}
}
if (map != 0) {
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#ifdef DIAGNOSTIC
printf("forward_hardclock: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
}
}
void
forward_signal(struct proc *p)
{
int map;
int id;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_signal(%p)", p);
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
if (!forward_signal_enabled)
return;
mtx_lock_spin(&sched_lock);
while (1) {
if (p->p_stat != SRUN) {
mtx_unlock_spin(&sched_lock);
return;
}
id = p->p_oncpu;
mtx_unlock_spin(&sched_lock);
if (id == 0xff)
return;
map = (1<<id);
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#if 0
printf("forward_signal: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
mtx_lock_spin(&sched_lock);
if (id == p->p_oncpu) {
mtx_unlock_spin(&sched_lock);
return;
}
}
statclock_process(curproc, TRAPF_PC(&frame), TRAPF_USERMODE(&frame));
mtx_unlock_spin(&sched_lock);
}
void
forward_roundrobin(void)
forward_statclock(void)
{
int map;
CTR0(KTR_SMP, "forward_statclock");
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
if (map != 0)
ipi_selected(map, IPI_STATCLOCK);
}
/*
* For each hardclock(), we send an IPI to all other CPU's to have them
* execute this function. It would be nice to reduce contention on
* sched_lock if we could simply peek at the CPU to determine the user/kernel
* state and call hardclock_process() on the CPU receiving the clock interrupt
* and then just use a simple IPI to handle any ast's if needed.
*/
void
forwarded_hardclock(struct trapframe frame)
{
mtx_lock_spin(&sched_lock);
hardclock_process(curproc, TRAPF_USERMODE(&frame));
mtx_unlock_spin(&sched_lock);
}
void
forward_hardclock(void)
{
u_int map;
int i;
CTR0(KTR_SMP, "forward_roundrobin()");
CTR0(KTR_SMP, "forward_hardclock");
if (!smp_started || !invltlb_ok || cold || panicstr)
return;
if (!forward_roundrobin_enabled)
return;
resched_cpus |= PCPU_GET(other_cpus);
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
#if 1
ipi_selected(map, IPI_AST);
#else
ipi_all_but_self(IPI_AST);
#endif
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#if 0
printf("forward_roundrobin: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
if (map != 0)
ipi_selected(map, IPI_HARDCLOCK);
}
/*
* When called the executing CPU will send an IPI to all other CPUs
* requesting that they halt execution.
*
* Usually (but not necessarily) called with 'other_cpus' as its arg.
*
* - Signals all CPUs in map to stop.
* - Waits for each to stop.
*
* Returns:
* -1: error
* 0: NA
* 1: ok
*
* XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
* from executing at same time.
*/
int
stop_cpus(u_int map)
{
int count = 0;
if (!smp_started)
return 0;
/* send the Xcpustop IPI to all CPUs in map */
ipi_selected(map, IPI_STOP);
while (count++ < 100000 && (stopped_cpus & map) != map)
/* spin */ ;
#ifdef DIAGNOSTIC
if ((stopped_cpus & map) != map)
printf("Warning: CPUs 0x%x did not stop!\n",
(~(stopped_cpus & map)) & map);
#endif
return 1;
}
/*
* Called by a CPU to restart stopped CPUs.
*
* Usually (but not necessarily) called with 'stopped_cpus' as its arg.
*
* - Signals all CPUs in map to restart.
* - Waits for each to restart.
*
* Returns:
* -1: error
* 0: NA
* 1: ok
*/
int
restart_cpus(u_int map)
{
int count = 0;
if (!smp_started)
return 0;
started_cpus = map; /* signal other cpus to restart */
/* wait for each to clear its bit */
while (count++ < 100000 && (stopped_cpus & map) != 0)
/* spin */ ;
#ifdef DIAGNOSTIC
if ((stopped_cpus & map) != 0)
printf("Warning: CPUs 0x%x did not restart!\n",
(~(stopped_cpus & map)) & map);
#endif
return 1;
}
#ifdef APIC_INTR_REORDER
/*
* Maintain mapping from softintr vector to isr bit in local apic.
@ -2810,73 +2385,6 @@ set_lapic_isrloc(int intr, int vector)
}
#endif
/*
* All-CPU rendezvous. CPUs are signalled, all execute the setup function
* (if specified), rendezvous, execute the action function (if specified),
* rendezvous again, execute the teardown function (if specified), and then
* resume.
*
* Note that the supplied external functions _must_ be reentrant and aware
* that they are running in parallel and in an unknown lock context.
*/
static void (*smp_rv_setup_func)(void *arg);
static void (*smp_rv_action_func)(void *arg);
static void (*smp_rv_teardown_func)(void *arg);
static void *smp_rv_func_arg;
static volatile int smp_rv_waiters[2];
void
smp_rendezvous_action(void)
{
/* setup function */
if (smp_rv_setup_func != NULL)
smp_rv_setup_func(smp_rv_func_arg);
/* spin on entry rendezvous */
atomic_add_int(&smp_rv_waiters[0], 1);
while (smp_rv_waiters[0] < mp_ncpus)
;
/* action function */
if (smp_rv_action_func != NULL)
smp_rv_action_func(smp_rv_func_arg);
/* spin on exit rendezvous */
atomic_add_int(&smp_rv_waiters[1], 1);
while (smp_rv_waiters[1] < mp_ncpus)
;
/* teardown function */
if (smp_rv_teardown_func != NULL)
smp_rv_teardown_func(smp_rv_func_arg);
}
void
smp_rendezvous(void (* setup_func)(void *),
void (* action_func)(void *),
void (* teardown_func)(void *),
void *arg)
{
/* obtain rendezvous lock */
mtx_lock_spin(&smp_rv_mtx);
/* set static function pointers */
smp_rv_setup_func = setup_func;
smp_rv_action_func = action_func;
smp_rv_teardown_func = teardown_func;
smp_rv_func_arg = arg;
smp_rv_waiters[0] = 0;
smp_rv_waiters[1] = 0;
/*
* signal other processors, which will enter the IPI with interrupts off
*/
ipi_all_but_self(IPI_RENDEZVOUS);
/* call executor function */
smp_rendezvous_action();
/* release lock */
mtx_unlock_spin(&smp_rv_mtx);
}
/*
* send an IPI to a set of cpus.
*/

3
sys/i386/include/pcpu.h
View File

@ -74,9 +74,6 @@ struct globaldata {
#endif
};
SLIST_HEAD(cpuhead, globaldata);
extern struct cpuhead cpuhead;
#ifdef SMP
/*
* This is the upper (0xff800000) address space layout that is per-cpu.

50
sys/i386/include/smp.h
View File

@ -41,7 +41,21 @@ extern int current_postcode; /** XXX currently in mp_machdep.c */
outb(0x80, current_postcode)
#include <sys/bus.h> /* XXX */
#include <machine/apic.h>
#include <machine/frame.h>
#include <i386/isa/icu.h>
#include <i386/isa/intr_machdep.h>
/*
* Interprocessor interrupts for SMP.
*/
#define IPI_INVLTLB XINVLTLB_OFFSET
#define IPI_RENDEZVOUS XRENDEZVOUS_OFFSET
#define IPI_AST XCPUAST_OFFSET
#define IPI_STOP XCPUSTOP_OFFSET
#define IPI_HARDCLOCK XHARDCLOCK_OFFSET
#define IPI_STATCLOCK XSTATCLOCK_OFFSET
/* global data in mpboot.s */
extern int bootMP_size;
@ -49,18 +63,8 @@ extern int bootMP_size;
/* functions in mpboot.s */
void bootMP __P((void));
/* global data in apic_vector.s */
extern volatile u_int stopped_cpus;
extern volatile u_int started_cpus;
extern volatile u_int checkstate_probed_cpus;
extern volatile u_int checkstate_need_ast;
extern volatile u_int resched_cpus;
extern void (*cpustop_restartfunc) __P((void));
/* global data in mp_machdep.c */
extern int bsp_apic_ready;
extern int mp_ncpus;
extern int mp_naps;
extern int mp_nbusses;
extern int mp_napics;
@ -81,14 +85,11 @@ struct apic_intmapinfo {
int redirindex;
};
extern struct apic_intmapinfo int_to_apicintpin[];
extern u_int all_cpus;
extern struct pcb stoppcbs[];
/* functions in mp_machdep.c */
void i386_mp_probe __P((void));
u_int mp_bootaddress __P((u_int));
int mp_probe __P((void));
void mp_start __P((void));
void mp_announce __P((void));
u_int isa_apic_mask __P((u_int));
int isa_apic_irq __P((int));
int pci_apic_irq __P((int, int, int));
@ -107,20 +108,17 @@ void revoke_apic_irq __P((int irq));
void bsp_apic_configure __P((void));
void init_secondary __P((void));
void smp_invltlb __P((void));
int stop_cpus __P((u_int));
int restart_cpus __P((u_int));
void forward_statclock __P((int pscnt));
void forward_hardclock __P((int pscnt));
void forward_signal __P((struct proc *));
void forward_roundrobin __P((void));
void forward_statclock __P((void));
void forwarded_statclock __P((struct trapframe frame));
void forward_hardclock __P((void));
void forwarded_hardclock __P((struct trapframe frame));
void ipi_selected __P((u_int cpus, u_int ipi));
void ipi_all __P((u_int ipi));
void ipi_all_but_self __P((u_int ipi));
void ipi_self __P((u_int ipi));
#ifdef APIC_INTR_REORDER
void set_lapic_isrloc __P((int, int));
#endif /* APIC_INTR_REORDER */
void smp_rendezvous_action __P((void));
void smp_rendezvous __P((void (*)(void *),
void (*)(void *),
void (*)(void *),
void *arg));
/* global data in mpapic.c */
extern volatile lapic_t lapic;
@ -147,8 +145,6 @@ void io_apic_write __P((int, int, u_int));
/* global data in init_smp.c */
extern int invltlb_ok;
extern int smp_active;
extern int smp_started;
extern volatile int smp_idle_loops;
#endif /* !LOCORE */

155
sys/i386/isa/apic_vector.s
View File

@ -182,7 +182,6 @@ Xspuriousint:
iret
/*
* Handle TLB shootdowns.
*/
@ -211,71 +210,61 @@ Xinvltlb:
popl %eax
iret
/*
* Executed by a CPU when it receives an Xcpucheckstate IPI from another CPU,
*
* - Stores current cpu state in checkstate_cpustate[cpuid]
* 0 == user, 1 == sys, 2 == intr
* - Stores current process in checkstate_curproc[cpuid]
*
* - Signals its receipt by setting bit cpuid in checkstate_probed_cpus.
*
* stack: 0->ds, 4->fs, 8->ebx, 12->eax, 16->eip, 20->cs, 24->eflags
* Forward hardclock to another CPU. Pushes a trapframe and calls
* forwarded_hardclock().
*/
.text
SUPERALIGN_TEXT
.globl Xcpucheckstate
.globl checkstate_cpustate
.globl checkstate_curproc
.globl checkstate_pc
Xcpucheckstate:
pushl %eax
pushl %ebx
pushl %ds /* save current data segment */
pushl %fs
movl $KDSEL, %eax
mov %ax, %ds /* use KERNEL data segment */
.globl Xhardclock
Xhardclock:
PUSH_FRAME
movl $KDSEL, %eax /* reload with kernel's data segment */
mov %ax, %ds
mov %ax, %es
movl $KPSEL, %eax
mov %ax, %fs
movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */
movl $0, %ebx
movl 20(%esp), %eax
andl $3, %eax
cmpl $3, %eax
je 1f
testl $PSL_VM, 24(%esp)
jne 1f
incl %ebx /* system or interrupt */
1:
movl PCPU(CPUID), %eax
movl %ebx, checkstate_cpustate(,%eax,4)
movl PCPU(CURPROC), %ebx
movl %ebx, checkstate_curproc(,%eax,4)
movl PCPU(CURPROC),%ebx
incl P_INTR_NESTING_LEVEL(%ebx)
call forwarded_hardclock
decl P_INTR_NESTING_LEVEL(%ebx)
MEXITCOUNT
jmp doreti
movl 16(%esp), %ebx
movl %ebx, checkstate_pc(,%eax,4)
/*
* Forward statclock to another CPU. Pushes a trapframe and calls
* forwarded_statclock().
*/
.text
SUPERALIGN_TEXT
.globl Xstatclock
Xstatclock:
PUSH_FRAME
movl $KDSEL, %eax /* reload with kernel's data segment */
mov %ax, %ds
mov %ax, %es
movl $KPSEL, %eax
mov %ax, %fs
lock /* checkstate_probed_cpus |= (1<<id) */
btsl %eax, checkstate_probed_cpus
popl %fs
popl %ds /* restore previous data segment */
popl %ebx
popl %eax
iret
movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */
FAKE_MCOUNT(13*4(%esp))
movl PCPU(CURPROC),%ebx
incl P_INTR_NESTING_LEVEL(%ebx)
call forwarded_statclock
decl P_INTR_NESTING_LEVEL(%ebx)
MEXITCOUNT
jmp doreti
/*
* Executed by a CPU when it receives an Xcpuast IPI from another CPU,
*
* - Signals its receipt by clearing bit cpuid in checkstate_need_ast.
*
* - We need a better method of triggering asts on other cpus.
* The other CPU has already executed aston() or need_resched() on our
* current process, so we simply need to ack the interrupt and return
* via doreti to run ast().
*/
.text
@ -289,40 +278,12 @@ Xcpuast:
movl $KPSEL, %eax
mov %ax, %fs
movl PCPU(CPUID), %eax
lock /* checkstate_need_ast &= ~(1<<id) */
btrl %eax, checkstate_need_ast
movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */
lock
btsl %eax, checkstate_pending_ast
jc 1f
FAKE_MCOUNT(13*4(%esp))
MTX_LOCK_SPIN(sched_lock, 0)
movl PCPU(CURPROC),%ebx
orl $PS_ASTPENDING, P_SFLAG(%ebx)
movl PCPU(CPUID), %eax
lock
btrl %eax, checkstate_pending_ast
lock
btrl %eax, CNAME(resched_cpus)
jnc 2f
orl $PS_NEEDRESCHED, P_SFLAG(%ebx)
lock
incl CNAME(want_resched_cnt)
2:
MTX_UNLOCK_SPIN(sched_lock)
lock
incl CNAME(cpuast_cnt)
MEXITCOUNT
jmp doreti
1:
/* We are already in the process of delivering an ast for this CPU */
POP_FRAME
iret
/*
* Executed by a CPU when it receives an Xcpustop IPI from another CPU,
@ -331,7 +292,6 @@ Xcpuast:
* - Waits for permission to restart.
* - Signals its restart.
*/
.text
SUPERALIGN_TEXT
.globl Xcpustop
@ -357,20 +317,19 @@ Xcpustop:
pushl %eax
call CNAME(savectx) /* Save process context */
addl $4, %esp
movl PCPU(CPUID), %eax
lock
btsl %eax, stopped_cpus /* stopped_cpus |= (1<<id) */
btsl %eax, CNAME(stopped_cpus) /* stopped_cpus |= (1<<id) */
1:
btl %eax, started_cpus /* while (!(started_cpus & (1<<id))) */
btl %eax, CNAME(started_cpus) /* while (!(started_cpus & (1<<id))) */
jnc 1b
lock
btrl %eax, started_cpus /* started_cpus &= ~(1<<id) */
btrl %eax, CNAME(started_cpus) /* started_cpus &= ~(1<<id) */
lock
btrl %eax, stopped_cpus /* stopped_cpus &= ~(1<<id) */
btrl %eax, CNAME(stopped_cpus) /* stopped_cpus &= ~(1<<id) */
test %eax, %eax
jnz 2f
@ -492,34 +451,6 @@ _xhits:
.space (NCPU * 4), 0
#endif /* COUNT_XINVLTLB_HITS */
/* variables used by stop_cpus()/restart_cpus()/Xcpustop */
.globl stopped_cpus, started_cpus
stopped_cpus:
.long 0
started_cpus:
.long 0
.globl checkstate_probed_cpus
checkstate_probed_cpus:
.long 0
.globl checkstate_need_ast
checkstate_need_ast:
.long 0
checkstate_pending_ast:
.long 0
.globl CNAME(resched_cpus)
.globl CNAME(want_resched_cnt)
.globl CNAME(cpuast_cnt)
.globl CNAME(cpustop_restartfunc)
CNAME(resched_cpus):
.long 0
CNAME(want_resched_cnt):
.long 0
CNAME(cpuast_cnt):
.long 0
CNAME(cpustop_restartfunc):
.long 0
.globl apic_pin_trigger
apic_pin_trigger:
.long 0

13
sys/i386/isa/clock.c
View File

@ -211,6 +211,10 @@ clkintr(struct clockframe frame)
mtx_unlock_spin(&clock_lock);
}
timer_func(&frame);
#ifdef SMP
if (timer_func == hardclock)
forward_hardclock();
#endif
switch (timer0_state) {
case RELEASED:
@ -253,6 +257,9 @@ clkintr(struct clockframe frame)
timer_func = hardclock;
timer0_state = RELEASED;
hardclock(&frame);
#ifdef SMP
forward_hardclock();
#endif
}
break;
}
@ -374,8 +381,12 @@ release_timer2()
static void
rtcintr(struct clockframe frame)
{
while (rtcin(RTC_INTR) & RTCIR_PERIOD)
while (rtcin(RTC_INTR) & RTCIR_PERIOD) {
statclock(&frame);
#ifdef SMP
forward_statclock();
#endif
}
}
#include "opt_ddb.h"

8
sys/i386/isa/intr_machdep.h
View File

@ -107,10 +107,11 @@
#define XINVLTLB_OFFSET (ICU_OFFSET + 112)
/* inter-cpu clock handling */
#define XCPUCHECKSTATE_OFFSET (ICU_OFFSET + 113)
#define XHARDCLOCK_OFFSET (ICU_OFFSET + 113)
#define XSTATCLOCK_OFFSET (ICU_OFFSET + 114)
/* inter-CPU rendezvous */
#define XRENDEZVOUS_OFFSET (ICU_OFFSET + 114)
#define XRENDEZVOUS_OFFSET (ICU_OFFSET + 115)
/* IPI to generate an additional software trap at the target CPU */
#define XCPUAST_OFFSET (ICU_OFFSET + 48)
@ -173,7 +174,8 @@ inthand_t
inthand_t
Xinvltlb, /* TLB shootdowns */
Xcpucheckstate, /* Check cpu state */
Xhardclock, /* Forward hardclock() */
Xstatclock, /* Forward statclock() */
Xcpuast, /* Additional software trap on other cpu */
Xcpustop, /* CPU stops & waits for another CPU to restart it */
Xspuriousint, /* handle APIC "spurious INTs" */

2
sys/ia64/ia64/db_interface.c
View File

@ -44,6 +44,7 @@
#include <sys/reboot.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/smp.h>
#include <sys/cons.h>
#include <sys/ktr.h>
@ -51,7 +52,6 @@
#include <machine/db_machdep.h>
#include <machine/mutex.h>
#include <machine/smp.h>
#include <machine/inst.h>

21
sys/ia64/ia64/machdep.c
View File

@ -82,8 +82,6 @@ u_int32_t cycles_per_sec;
int cold = 1;
struct bootinfo_kernel bootinfo;
struct cpuhead cpuhead;
struct mtx sched_lock;
struct mtx Giant;
@ -579,12 +577,6 @@ ia64_init()
proc0.p_md.md_tf =
(struct trapframe *)(proc0.p_addr->u_pcb.pcb_sp + 16);
/*
* Record all cpus in a list.
*/
SLIST_INIT(&cpuhead);
SLIST_INSERT_HEAD(&cpuhead, GLOBALP, gd_allcpu);
/* Setup curproc so that mutexes work */
PCPU_SET(curproc, &proc0);
PCPU_SET(spinlocks, NULL);
@ -1367,3 +1359,16 @@ ia64_fpstate_switch(struct proc *p)
p->p_md.md_flags |= MDP_FPUSED;
}
/*
* Initialise a struct globaldata.
*/
void
globaldata_init(struct globaldata *globaldata, int cpuid, size_t sz)
{
bzero(globaldata, sz);
globaldata->gd_cpuid = cpuid;
#ifdef SMP
globaldata_register(globaldata);
#endif
}

289
sys/ia64/ia64/mp_machdep.c
View File

@ -34,6 +34,7 @@
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/kernel.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <vm/vm.h>
@ -42,7 +43,6 @@
#include <sys/user.h>
#include <sys/dkstat.h>
#include <machine/smp.h>
#include <machine/atomic.h>
#include <machine/ipl.h>
#include <machine/globaldata.h>
@ -53,8 +53,6 @@
#define CHECKSTATE_SYS 1
#define CHECKSTATE_INTR 2
volatile u_int stopped_cpus;
volatile u_int started_cpus;
volatile u_int checkstate_probed_cpus;
volatile u_int checkstate_need_ast;
volatile u_int checkstate_pending_ast;
@ -62,76 +60,27 @@ struct proc* checkstate_curproc[MAXCPU];
int checkstate_cpustate[MAXCPU];
u_long checkstate_pc[MAXCPU];
volatile u_int resched_cpus;
void (*cpustop_restartfunc) __P((void));
int mp_ncpus;
int smp_started;
int boot_cpu_id;
u_int32_t all_cpus;
static struct globaldata *cpuid_to_globaldata[MAXCPU];
int smp_active = 0; /* are the APs allowed to run? */
SYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RW, &smp_active, 0, "");
/* Is forwarding of a interrupt to the CPU holding the ISR lock enabled ? */
int forward_irq_enabled = 1;
SYSCTL_INT(_machdep, OID_AUTO, forward_irq_enabled, CTLFLAG_RW,
&forward_irq_enabled, 0, "");
/* Enable forwarding of a signal to a process running on a different CPU */
static int forward_signal_enabled = 1;
SYSCTL_INT(_machdep, OID_AUTO, forward_signal_enabled, CTLFLAG_RW,
&forward_signal_enabled, 0, "");
/* Enable forwarding of roundrobin to all other cpus */
static int forward_roundrobin_enabled = 1;
SYSCTL_INT(_machdep, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW,
&forward_roundrobin_enabled, 0, "");
/*
* Initialise a struct globaldata.
*/
void
globaldata_init(struct globaldata *globaldata, int cpuid, size_t sz)
int
cpu_mp_probe()
{
bzero(globaldata, sz);
globaldata->gd_cpuid = cpuid;
globaldata->gd_other_cpus = all_cpus & ~(1 << cpuid);
cpuid_to_globaldata[cpuid] = globaldata;
}
struct globaldata *
globaldata_find(int cpuid)
{
return cpuid_to_globaldata[cpuid];
}
/* Other stuff */
/* lock around the MP rendezvous */
static struct mtx smp_rv_mtx;
static void
init_locks(void)
{
mtx_init(&smp_rv_mtx, "smp_rendezvous", MTX_SPIN);
return (0);
}
void
mp_start()
{
init_locks();
}
void
mp_announce()
cpu_mp_start()
{
}
void
smp_invltlb()
cpu_mp_announce()
{
}
@ -426,227 +375,6 @@ forward_hardclock(int pscnt)
}
}
void
forward_signal(struct proc *p)
{
int map;
int id;
int i;
/* Kludge. We don't yet have separate locks for the interrupts
* and the kernel. This means that we cannot let the other processors
* handle complex interrupts while inhibiting them from entering
* the kernel in a non-interrupt context.
*
* What we can do, without changing the locking mechanisms yet,
* is letting the other processors handle a very simple interrupt
* (wich determines the processor states), and do the main
* work ourself.
*/
CTR1(KTR_SMP, "forward_signal(%p)", p);
if (!smp_started || cold || panicstr)
return;
if (!forward_signal_enabled)
return;
while (1) {
if (p->p_stat != SRUN)
return;
id = p->p_oncpu;
if (id == 0xff)
return;
map = (1<<id);
checkstate_need_ast |= map;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#if 0
printf("forward_signal: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
if (id == p->p_oncpu)
return;
}
}
void
forward_roundrobin(void)
{
u_int map;
int i;
CTR0(KTR_SMP, "forward_roundrobin()");
if (!smp_started || cold || panicstr)
return;
if (!forward_roundrobin_enabled)
return;
resched_cpus |= PCPU_GET(other_cpus);
map = PCPU_GET(other_cpus) & ~stopped_cpus ;
ipi_selected(map, IPI_AST);
i = 0;
while ((checkstate_need_ast & map) != 0) {
/* spin */
i++;
if (i > 100000) {
#if 0
printf("forward_roundrobin: dropped ast 0x%x\n",
checkstate_need_ast & map);
#endif
break;
}
}
}
/*
* When called the executing CPU will send an IPI to all other CPUs
* requesting that they halt execution.
*
* Usually (but not necessarily) called with 'other_cpus' as its arg.
*
* - Signals all CPUs in map to stop.
* - Waits for each to stop.
*
* Returns:
* -1: error
* 0: NA
* 1: ok
*
* XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
* from executing at same time.
*/
int
stop_cpus(u_int map)
{
int i;
if (!smp_started)
return 0;
CTR1(KTR_SMP, "stop_cpus(%x)", map);
/* send the stop IPI to all CPUs in map */
ipi_selected(map, IPI_STOP);
i = 0;
while ((stopped_cpus & map) != map) {
/* spin */
i++;
if (i == 100000) {
printf("timeout stopping cpus\n");
break;
}
ia64_mf();
}
printf("stopped_cpus=%x\n", stopped_cpus);
return 1;
}
/*
* Called by a CPU to restart stopped CPUs.
*
* Usually (but not necessarily) called with 'stopped_cpus' as its arg.
*
* - Signals all CPUs in map to restart.
* - Waits for each to restart.
*
* Returns:
* -1: error
* 0: NA
* 1: ok
*/
int
restart_cpus(u_int map)
{
if (!smp_started)
return 0;
CTR1(KTR_SMP, "restart_cpus(%x)", map);
started_cpus = map; /* signal other cpus to restart */
ia64_mf();
while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */
ia64_mf();
return 1;
}
/*
* All-CPU rendezvous. CPUs are signalled, all execute the setup function
* (if specified), rendezvous, execute the action function (if specified),
* rendezvous again, execute the teardown function (if specified), and then
* resume.
*
* Note that the supplied external functions _must_ be reentrant and aware
* that they are running in parallel and in an unknown lock context.
*/
static void (*smp_rv_setup_func)(void *arg);
static void (*smp_rv_action_func)(void *arg);
static void (*smp_rv_teardown_func)(void *arg);
static void *smp_rv_func_arg;
static volatile int smp_rv_waiters[2];
void
smp_rendezvous_action(void)
{
/* setup function */
if (smp_rv_setup_func != NULL)
smp_rv_setup_func(smp_rv_func_arg);
/* spin on entry rendezvous */
atomic_add_int(&smp_rv_waiters[0], 1);
while (smp_rv_waiters[0] < mp_ncpus)
;
/* action function */
if (smp_rv_action_func != NULL)
smp_rv_action_func(smp_rv_func_arg);
/* spin on exit rendezvous */
atomic_add_int(&smp_rv_waiters[1], 1);
while (smp_rv_waiters[1] < mp_ncpus)
;
/* teardown function */
if (smp_rv_teardown_func != NULL)
smp_rv_teardown_func(smp_rv_func_arg);
}
void
smp_rendezvous(void (* setup_func)(void *),
void (* action_func)(void *),
void (* teardown_func)(void *),
void *arg)
{
/* obtain rendezvous lock */
mtx_lock_spin(&smp_rv_mtx);
/* set static function pointers */
smp_rv_setup_func = setup_func;
smp_rv_action_func = action_func;
smp_rv_teardown_func = teardown_func;
smp_rv_func_arg = arg;
smp_rv_waiters[0] = 0;
smp_rv_waiters[1] = 0;
/* signal other processors, which will enter the IPI with interrupts off */
ipi_all_but_self(IPI_RENDEZVOUS);
/* call executor function */
smp_rendezvous_action();
/* release lock */
mtx_unlock_spin(&smp_rv_mtx);
}
/*
* send an IPI to a set of cpus.
*/
@ -661,7 +389,7 @@ ipi_selected(u_int32_t cpus, u_int64_t ipi)
int cpuid = ffs(cpus) - 1;
cpus &= ~(1 << cpuid);
globaldata = cpuid_to_globaldata[cpuid];
globaldata = globaldata_find(cpuid);
if (globaldata) {
atomic_set_64(&globaldata->gd_pending_ipis, ipi);
ia64_mf();
@ -733,9 +461,6 @@ smp_handle_ipi(struct trapframe *frame)
CTR0(KTR_SMP, "IPI_AST");
atomic_clear_int(&checkstate_need_ast, 1<<cpuid);
atomic_set_int(&checkstate_pending_ast, 1<<cpuid);
if ((frame->tf_cr_ipsr & IA64_PSR_CPL)
== IA64_PSR_CPL_USER)
ast(frame); /* XXX */
break;
case IPI_CHECKSTATE:

6
sys/ia64/ia64/trap.c
View File

@ -41,6 +41,7 @@
#include <sys/exec.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/smp.h>
#include <sys/vmmeter.h>
#include <sys/sysent.h>
#include <sys/syscall.h>
@ -58,7 +59,6 @@
#include <machine/reg.h>
#include <machine/pal.h>
#include <machine/fpu.h>
#include <machine/smp.h>
#ifdef KTRACE
#include <sys/uio.h>
@ -90,7 +90,7 @@ userret(register struct proc *p, struct trapframe *frame, u_quad_t oticks)
mtx_lock_spin(&sched_lock);
p->p_pri.pri_level = p->p_pri.pri_user;
if (resched_wanted()) {
if (resched_wanted(p)) {
/*
* Since we are curproc, a clock interrupt could
* change our priority without changing run queues
@ -638,7 +638,7 @@ ast(framep)
* acquiring and release mutexes in assembly is not fun.
*/
mtx_lock_spin(&sched_lock);
if (!(astpending(p) || resched_wanted())) {
if (!(astpending(p) || resched_wanted(p))) {
mtx_unlock_spin(&sched_lock);
return;
}

3
sys/ia64/include/cpu.h
View File

@ -68,9 +68,6 @@ struct clockframe {
#define CLKF_USERMODE(framep) TRAPF_USERMODE(&(framep)->cf_tf)
#define CLKF_PC(framep) TRAPF_PC(&(framep)->cf_tf)
#define CLKF_BASEPRI(framep) \
(((framep)->cf_tf.tf_cr_ipsr & IA64_PSR_I) == 0)
#define CLKF_INTR(framep) (curproc->p_intr_nesting_level >= 2)
/*
* Give a profiling tick to the current process when the user profiling

4
sys/ia64/include/globaldata.h
View File

@ -66,11 +66,7 @@ struct globaldata {
#endif
};
SLIST_HEAD(cpuhead, globaldata);
extern struct cpuhead cpuhead;
void globaldata_init(struct globaldata *pcpu, int cpuid, size_t sz);
struct globaldata *globaldata_find(int cpuid);
#endif /* _KERNEL */

15
sys/ia64/include/ipl.h
View File

@ -29,19 +29,4 @@
#ifndef _MACHINE_IPL_H_
#define _MACHINE_IPL_H_
/*
* Interprocessor interrupts for SMP.
*/
#define IPI_INVLTLB 0x0001
#define IPI_RENDEZVOUS 0x0002
#define IPI_AST 0x0004
#define IPI_CHECKSTATE 0x0008
#define IPI_STOP 0x0010
void ipi_selected(u_int32_t cpus, u_int64_t ipi);
void ipi_all(u_int64_t ipi);
void ipi_all_but_self(u_int64_t ipi);
void ipi_self(u_int64_t ipi);
void smp_handle_ipi(struct trapframe *frame);
#endif /* !_MACHINE_IPL_H_ */

4
sys/ia64/include/pcpu.h
View File

@ -66,11 +66,7 @@ struct globaldata {
#endif
};
SLIST_HEAD(cpuhead, globaldata);
extern struct cpuhead cpuhead;
void globaldata_init(struct globaldata *pcpu, int cpuid, size_t sz);
struct globaldata *globaldata_find(int cpuid);
#endif /* _KERNEL */

41
sys/ia64/include/smp.h
View File

@ -6,43 +6,28 @@
#ifdef _KERNEL
#include <machine/mutex.h>
#include <machine/ipl.h>
#include <sys/ktr.h>
/*
* Interprocessor interrupts for SMP.
*/
#define IPI_INVLTLB 0x0001
#define IPI_RENDEZVOUS 0x0002
#define IPI_AST 0x0004
#define IPI_CHECKSTATE 0x0008
#define IPI_STOP 0x0010
#ifndef LOCORE
#define BETTER_CLOCK /* unconditional on ia64 */
/* global data in mp_machdep.c */
extern volatile u_int checkstate_probed_cpus;
extern volatile u_int checkstate_need_ast;
extern volatile u_int resched_cpus;
extern void (*cpustop_restartfunc) __P((void));
extern int smp_active;
extern int mp_ncpus;
extern u_int all_cpus;
extern u_int started_cpus;
extern u_int stopped_cpus;
/* functions in mp_machdep.c */
void mp_start(void);
void mp_announce(void);
void smp_invltlb(void);
void forward_statclock(int pscnt);
void forward_hardclock(int pscnt);
void forward_signal(struct proc *);
void forward_roundrobin(void);
int stop_cpus(u_int);
int restart_cpus(u_int);
void smp_rendezvous_action(void);
void smp_rendezvous(void (*)(void *),
void (*)(void *),
void (*)(void *),
void *arg);
void ipi_selected(u_int cpus, u_int ipi);
void ipi_all(u_int ipi);
void ipi_all_but_self(u_int ipi);
void ipi_self(u_int ipi);
void smp_init_secondary(void);
#endif /* !LOCORE */
#endif /* _KERNEL */
#endif
#endif /* !_MACHINE_SMP_H */

13
sys/isa/atrtc.c
View File

@ -211,6 +211,10 @@ clkintr(struct clockframe frame)
mtx_unlock_spin(&clock_lock);
}
timer_func(&frame);
#ifdef SMP
if (timer_func == hardclock)
forward_hardclock();
#endif
switch (timer0_state) {
case RELEASED:
@ -253,6 +257,9 @@ clkintr(struct clockframe frame)
timer_func = hardclock;
timer0_state = RELEASED;
hardclock(&frame);
#ifdef SMP
forward_hardclock();
#endif
}
break;
}
@ -374,8 +381,12 @@ release_timer2()
static void
rtcintr(struct clockframe frame)
{
while (rtcin(RTC_INTR) & RTCIR_PERIOD)
while (rtcin(RTC_INTR) & RTCIR_PERIOD) {
statclock(&frame);
#ifdef SMP
forward_statclock();
#endif
}
}
#include "opt_ddb.h"

135
sys/kern/kern_clock.c
View File

@ -52,6 +52,7 @@
#include <sys/proc.h>
#include <sys/resourcevar.h>
#include <sys/signalvar.h>
#include <sys/smp.h>
#include <sys/timetc.h>
#include <sys/timepps.h>
#include <vm/vm.h>
@ -63,7 +64,6 @@
#include <machine/cpu.h>
#include <machine/limits.h>
#include <machine/smp.h>
#ifdef GPROF
#include <sys/gmon.h>
@ -149,6 +149,41 @@ initclocks(dummy)
psratio = profhz / i;
}
/*
* Each time the real-time timer fires, this function is called on all CPUs
* with each CPU passing in its curproc as the first argument. If possible
* a nice optimization in the future would be to allow the CPU receiving the
* actual real-time timer interrupt to call this function on behalf of the
* other CPUs rather than sending an IPI to all other CPUs so that they
* can call this function. Note that hardclock() calls hardclock_process()
* for the CPU receiving the timer interrupt, so only the other CPUs in the
* system need to call this function (or have it called on their behalf.
*/
void
hardclock_process(p, user)
struct proc *p;
int user;
{
struct pstats *pstats;
/*
* Run current process's virtual and profile time, as needed.
*/
mtx_assert(&sched_lock, MA_OWNED);
pstats = p->p_stats;
if (user &&
timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) &&
itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) {
p->p_sflag |= PS_ALRMPEND;
aston(p);
}
if (timevalisset(&pstats->p_timer[ITIMER_PROF].it_value) &&
itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) {
p->p_sflag |= PS_PROFPEND;
aston(p);
}
}
/*
* The real-time timer, interrupting hz times per second.
*/
@ -156,40 +191,16 @@ void
hardclock(frame)
register struct clockframe *frame;
{
register struct proc *p;
int need_softclock = 0;
p = curproc;
if (p != PCPU_GET(idleproc)) {
register struct pstats *pstats;
/*
* Run current process's virtual and profile time, as needed.
*/
pstats = p->p_stats;
if (CLKF_USERMODE(frame) &&
timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) &&
itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) {
mtx_lock_spin(&sched_lock);
p->p_sflag |= PS_ALRMPEND;
aston(p);
mtx_unlock_spin(&sched_lock);
}
if (timevalisset(&pstats->p_timer[ITIMER_PROF].it_value) &&
itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) {
mtx_lock_spin(&sched_lock);
p->p_sflag |= PS_PROFPEND;
aston(p);
mtx_unlock_spin(&sched_lock);
}
}
#if defined(SMP) && defined(__i386__)
forward_hardclock(pscnt);
#endif
mtx_lock_spin(&sched_lock);
hardclock_process(curproc, CLKF_USERMODE(frame));
mtx_unlock_spin(&sched_lock);
/*
* If no separate statistics clock is available, run it from here.
*
* XXX: this only works for UP
*/
if (stathz == 0)
statclock(frame);
@ -328,43 +339,39 @@ stopprofclock(p)
}
/*
* Statistics clock. Grab profile sample, and if divider reaches 0,
* do process and kernel statistics. Most of the statistics are only
* Do process and kernel statistics. Most of the statistics are only
* used by user-level statistics programs. The main exceptions are
* p->p_uticks, p->p_sticks, p->p_iticks, and p->p_estcpu.
* p->p_uticks, p->p_sticks, p->p_iticks, and p->p_estcpu. This function
* should be called by all CPUs in the system for each statistics clock
* interrupt. See the description of hardclock_process for more detail on
* this function's relationship to statclock.
*/
void
statclock(frame)
register struct clockframe *frame;
statclock_process(p, pc, user)
struct proc *p;
register_t pc;
int user;
{
#ifdef GPROF
register struct gmonparam *g;
struct gmonparam *g;
int i;
#endif
register struct proc *p;
struct pstats *pstats;
long rss;
struct rusage *ru;
struct vmspace *vm;
mtx_lock_spin(&sched_lock);
if (CLKF_USERMODE(frame)) {
KASSERT(p == curproc, ("statclock_process: p != curproc"));
mtx_assert(&sched_lock, MA_OWNED);
if (user) {
/*
* Came from user mode; CPU was in user state.
* If this process is being profiled, record the tick.
*/
p = curproc;
if (p->p_sflag & PS_PROFIL)
addupc_intr(p, CLKF_PC(frame), 1);
#if defined(SMP) && defined(__i386__)
if (stathz != 0)
forward_statclock(pscnt);
#endif
if (--pscnt > 0) {
mtx_unlock_spin(&sched_lock);
addupc_intr(p, pc, 1);
if (pscnt < psdiv)
return;
}
/*
* Charge the time as appropriate.
*/
@ -380,21 +387,15 @@ statclock(frame)
*/
g = &_gmonparam;
if (g->state == GMON_PROF_ON) {
i = CLKF_PC(frame) - g->lowpc;
i = pc - g->lowpc;
if (i < g->textsize) {
i /= HISTFRACTION * sizeof(*g->kcount);
g->kcount[i]++;
}
}
#endif
#if defined(SMP) && defined(__i386__)
if (stathz != 0)
forward_statclock(pscnt);
#endif
if (--pscnt > 0) {
mtx_unlock_spin(&sched_lock);
if (pscnt < psdiv)
return;
}
/*
* Came from kernel mode, so we were:
* - handling an interrupt,
@ -407,8 +408,7 @@ statclock(frame)
* so that we know how much of its real time was spent
* in ``non-process'' (i.e., interrupt) work.
*/
p = curproc;
if ((p->p_ithd != NULL) || CLKF_INTR(frame)) {
if ((p->p_ithd != NULL) || p->p_intr_nesting_level >= 2) {
p->p_iticks++;
cp_time[CP_INTR]++;
} else {
@ -419,7 +419,6 @@ statclock(frame)
cp_time[CP_IDLE]++;
}
}
pscnt = psdiv;
schedclock(p);
@ -434,7 +433,23 @@ statclock(frame)
if (ru->ru_maxrss < rss)
ru->ru_maxrss = rss;
}
}
/*
* Statistics clock. Grab profile sample, and if divider reaches 0,
* do process and kernel statistics. Most of the statistics are only
* used by user-level statistics programs. The main exceptions are
* p->p_uticks, p->p_sticks, p->p_iticks, and p->p_estcpu.
*/
void
statclock(frame)
register struct clockframe *frame;
{
mtx_lock_spin(&sched_lock);
if (--pscnt == 0)
pscnt = psdiv;
statclock_process(curproc, CLKF_PC(frame), CLKF_USERMODE(frame));
mtx_unlock_spin(&sched_lock);
}

28
sys/kern/kern_idle.c
View File

@ -12,6 +12,7 @@
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/signalvar.h>
#include <sys/smp.h>
#include <sys/resourcevar.h>
#include <sys/vmmeter.h>
#include <sys/sysctl.h>
@ -30,7 +31,6 @@
#include <machine/cpu.h>
#include <machine/md_var.h>
#include <machine/smp.h>
#include <machine/globaldata.h>
#include <machine/globals.h>
@ -48,26 +48,32 @@ static void idle_proc(void *dummy);
static void
idle_setup(void *dummy)
{
#ifdef SMP
struct globaldata *gd;
#endif
struct proc *p;
int error;
SLIST_FOREACH(gd, &cpuhead, gd_allcpu) {
#ifdef SMP
error = kthread_create(idle_proc, NULL, &gd->gd_idleproc,
RFSTOPPED|RFHIGHPID, "idle: cpu%d",
gd->gd_cpuid);
SLIST_FOREACH(gd, &cpuhead, gd_allcpu) {
error = kthread_create(idle_proc, NULL, &p,
RFSTOPPED | RFHIGHPID, "idle: cpu%d", gd->gd_cpuid);
gd->gd_idleproc = p;
if (gd->gd_curproc == NULL)
gd->gd_curproc = p;
#else
error = kthread_create(idle_proc, NULL, &gd->gd_idleproc,
RFSTOPPED|RFHIGHPID, "idle");
error = kthread_create(idle_proc, NULL, &p,
RFSTOPPED | RFHIGHPID, "idle");
PCPU_SET(idleproc, p);
#endif
if (error)
panic("idle_setup: kthread_create error %d\n", error);
gd->gd_idleproc->p_flag |= P_NOLOAD;
gd->gd_idleproc->p_stat = SRUN;
if (gd->gd_curproc == NULL)
gd->gd_curproc = gd->gd_idleproc;
p->p_flag |= P_NOLOAD;
p->p_stat = SRUN;
#ifdef SMP
}
#endif
}
/*

2
sys/kern/kern_intr.c
View File

@ -381,7 +381,7 @@ ithread_schedule(struct ithd *ithread, int do_switch)
mi_switch();
sched_lock.mtx_savecrit = savecrit;
} else
need_resched();
need_resched(curproc);
} else {
CTR3(KTR_INTR, __func__ ": pid %d: it_need %d, state %d",
p->p_pid, ithread->it_need, p->p_stat);

2
sys/kern/kern_mib.c
View File

@ -46,7 +46,7 @@
#include <sys/sysctl.h>
#include <sys/proc.h>
#include <sys/jail.h>
#include <machine/smp.h>
#include <sys/smp.h>
SYSCTL_NODE(, 0, sysctl, CTLFLAG_RW, 0,
"Sysctl internal magic");

2
sys/kern/kern_shutdown.c
View File

@ -59,6 +59,7 @@
#include <sys/mount.h>
#include <sys/mutex.h>
#include <sys/queue.h>
#include <sys/smp.h> /* smp_active, cpuid */
#include <sys/sysctl.h>
#include <sys/conf.h>
#include <sys/sysproto.h>
@ -66,7 +67,6 @@
#include <machine/pcb.h>
#include <machine/md_var.h>
#include <machine/smp.h> /* smp_active, cpuid */
#include <sys/signalvar.h>

7
sys/kern/kern_sig.c
View File

@ -62,6 +62,7 @@
#include <sys/ktr.h>
#include <sys/ktrace.h>
#include <sys/resourcevar.h>
#include <sys/smp.h>
#include <sys/stat.h>
#include <sys/sx.h>
#include <sys/syslog.h>
@ -70,7 +71,6 @@
#include <sys/malloc.h>
#include <machine/cpu.h>
#include <machine/smp.h>
#define ONSIG 32 /* NSIG for osig* syscalls. XXX. */
@ -1301,12 +1301,11 @@ psignal(p, sig)
*/
if (p->p_stat == SRUN) {
signotify(p);
mtx_unlock_spin(&sched_lock);
#ifdef SMP
forward_signal(p);
#endif
} else
mtx_unlock_spin(&sched_lock);
}
mtx_unlock_spin(&sched_lock);
goto out;
}
/*NOTREACHED*/

10
sys/kern/kern_synch.c
View File

@ -52,6 +52,7 @@
#include <sys/proc.h>
#include <sys/resourcevar.h>
#include <sys/signalvar.h>
#include <sys/smp.h>
#include <sys/sx.h>
#include <sys/sysctl.h>
#include <sys/sysproto.h>
@ -64,7 +65,6 @@
#endif
#include <machine/cpu.h>
#include <machine/smp.h>
static void sched_setup __P((void *dummy));
SYSINIT(sched_setup, SI_SUB_KICK_SCHEDULER, SI_ORDER_FIRST, sched_setup, NULL)
@ -110,7 +110,7 @@ maybe_resched(p)
mtx_assert(&sched_lock, MA_OWNED);
if (p->p_pri.pri_level < curproc->p_pri.pri_level)
need_resched();
need_resched(curproc);
}
int
@ -129,11 +129,11 @@ roundrobin(arg)
{
mtx_lock_spin(&sched_lock);
need_resched();
mtx_unlock_spin(&sched_lock);
need_resched(curproc);
#ifdef SMP
forward_roundrobin();
#endif
mtx_unlock_spin(&sched_lock);
callout_reset(&roundrobin_callout, sched_quantum, roundrobin, NULL);
}
@ -911,7 +911,7 @@ mi_switch()
sched_nest = sched_lock.mtx_recurse;
curproc->p_lastcpu = curproc->p_oncpu;
curproc->p_oncpu = NOCPU;
clear_resched();
clear_resched(curproc);
cpu_switch();
curproc->p_oncpu = PCPU_GET(cpuid);
sched_lock.mtx_recurse = sched_nest;

6
sys/kern/ksched.c
View File

@ -175,7 +175,7 @@ int ksched_setscheduler(register_t *ret, struct ksched *ksched,
mtx_lock_spin(&sched_lock);
rtp_to_pri(&rtp, &p->p_pri);
need_resched();
need_resched(p);
mtx_unlock_spin(&sched_lock);
}
else
@ -197,7 +197,7 @@ int ksched_setscheduler(register_t *ret, struct ksched *ksched,
* on the scheduling code: You must leave the
* scheduling info alone.
*/
need_resched();
need_resched(p);
mtx_unlock_spin(&sched_lock);
}
break;
@ -216,7 +216,7 @@ int ksched_getscheduler(register_t *ret, struct ksched *ksched, struct proc *p)
int ksched_yield(register_t *ret, struct ksched *ksched)
{
mtx_lock_spin(&sched_lock);
need_resched();
need_resched(curproc);
mtx_unlock_spin(&sched_lock);
return 0;
}

2865
sys/kern/subr_smp.c
View File

File diff suppressed because it is too large Load Diff

4
sys/kern/subr_trap.c
View File

@ -177,7 +177,7 @@ userret(p, frame, oticks)
mtx_lock_spin(&sched_lock);
p->p_pri.pri_level = p->p_pri.pri_user;
if (resched_wanted()) {
if (resched_wanted(p)) {
/*
* Since we are curproc, clock will normally just change
* our priority without moving us from one queue to another
@ -1277,7 +1277,7 @@ ast(framep)
* acquiring and releasing mutexes in assembly is not fun.
*/
mtx_lock_spin(&sched_lock);
if (!(astpending(p) || resched_wanted())) {
if (!(astpending(p) || resched_wanted(p))) {
mtx_unlock_spin(&sched_lock);
return;
}

14
sys/pc98/i386/machdep.c
View File

@ -262,8 +262,6 @@ static struct trapframe proc0_tf;
static struct globaldata __globaldata;
#endif
struct cpuhead cpuhead;
struct mtx sched_lock;
struct mtx Giant;
@ -454,16 +452,10 @@ cpu_startup(dummy)
bufinit();
vm_pager_bufferinit();
SLIST_INIT(&cpuhead);
SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu);
#ifdef SMP
/*
* OK, enough kmem_alloc/malloc state should be up, lets get on with it!
*/
mp_start(); /* fire up the APs and APICs */
mp_announce();
#endif /* SMP */
globaldata_register(GLOBALDATA);
#endif
cpu_setregs();
}

14
sys/pc98/pc98/machdep.c
View File

@ -262,8 +262,6 @@ static struct trapframe proc0_tf;
static struct globaldata __globaldata;
#endif
struct cpuhead cpuhead;
struct mtx sched_lock;
struct mtx Giant;
@ -454,16 +452,10 @@ cpu_startup(dummy)
bufinit();
vm_pager_bufferinit();
SLIST_INIT(&cpuhead);
SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu);
#ifdef SMP
/*
* OK, enough kmem_alloc/malloc state should be up, lets get on with it!
*/
mp_start(); /* fire up the APs and APICs */
mp_announce();
#endif /* SMP */
globaldata_register(GLOBALDATA);
#endif
cpu_setregs();
}

6
sys/posix4/ksched.c
View File

@ -175,7 +175,7 @@ int ksched_setscheduler(register_t *ret, struct ksched *ksched,
mtx_lock_spin(&sched_lock);
rtp_to_pri(&rtp, &p->p_pri);
need_resched();
need_resched(p);
mtx_unlock_spin(&sched_lock);
}
else
@ -197,7 +197,7 @@ int ksched_setscheduler(register_t *ret, struct ksched *ksched,
* on the scheduling code: You must leave the
* scheduling info alone.
*/
need_resched();
need_resched(p);
mtx_unlock_spin(&sched_lock);
}
break;
@ -216,7 +216,7 @@ int ksched_getscheduler(register_t *ret, struct ksched *ksched, struct proc *p)
int ksched_yield(register_t *ret, struct ksched *ksched)
{
mtx_lock_spin(&sched_lock);
need_resched();
need_resched(curproc);
mtx_unlock_spin(&sched_lock);
return 0;
}

7
sys/powerpc/include/globaldata.h
View File

@ -66,14 +66,7 @@ struct globaldata {
#endif
};
SLIST_HEAD(cpuhead, globaldata);
extern struct cpuhead cpuhead;
void globaldata_init(struct globaldata *pcpu, int cpuid, size_t sz);
struct globaldata *globaldata_find(int cpuid);
#ifdef SMP
void globaldata_register(struct globaldata *pcpu);
#endif
#endif /* _KERNEL */

7
sys/powerpc/include/pcpu.h
View File

@ -66,14 +66,7 @@ struct globaldata {
#endif
};
SLIST_HEAD(cpuhead, globaldata);
extern struct cpuhead cpuhead;
void globaldata_init(struct globaldata *pcpu, int cpuid, size_t sz);
struct globaldata *globaldata_find(int cpuid);
#ifdef SMP
void globaldata_register(struct globaldata *pcpu);
#endif
#endif /* _KERNEL */

2
sys/powerpc/powerpc/mp_machdep.c
View File

@ -34,6 +34,7 @@
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/kernel.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <vm/vm.h>
@ -42,7 +43,6 @@
#include <sys/user.h>
#include <sys/dkstat.h>
#include <machine/smp.h>
#include <machine/atomic.h>
#include <machine/globaldata.h>
#include <machine/pmap.h>

10
sys/sys/proc.h
View File

@ -382,16 +382,16 @@ sigonstack(size_t sp)
* Preempt the current process if in interrupt from user mode,
* or after the current trap/syscall if in system mode.
*/
#define need_resched() do { \
#define need_resched(p) do { \
mtx_assert(&sched_lock, MA_OWNED); \
curproc->p_sflag |= PS_NEEDRESCHED; \
(p)->p_sflag |= PS_NEEDRESCHED; \
} while (0)
#define resched_wanted() (curproc->p_sflag & PS_NEEDRESCHED)
#define resched_wanted(p) ((p)->p_sflag & PS_NEEDRESCHED)
#define clear_resched() do { \
#define clear_resched(p) do { \
mtx_assert(&sched_lock, MA_OWNED); \
curproc->p_sflag &= ~PS_NEEDRESCHED; \
(p)->p_sflag &= ~PS_NEEDRESCHED; \
} while (0)
/*

181
sys/sys/smp.h
View File

@ -7,152 +7,61 @@
* ----------------------------------------------------------------------------
*
* $FreeBSD$
*
*/
#ifndef _MACHINE_SMP_H_
#define _MACHINE_SMP_H_
#ifndef _SYS_SMP_H_
#define _SYS_SMP_H_
#ifdef _KERNEL
#if defined(SMP) && defined(I386_CPU) && !defined(COMPILING_LINT)
#error SMP not supported with I386_CPU
#endif
#if defined(SMP) && !defined(APIC_IO)
# error APIC_IO required for SMP, add "options APIC_IO" to your config file.
#endif /* SMP && !APIC_IO */
#if defined(SMP) || defined(APIC_IO)
#include <machine/globaldata.h>
#include <machine/smp.h>
#ifndef LOCORE
#ifdef SMP
SLIST_HEAD(cpuhead, globaldata);
extern struct cpuhead cpuhead;
extern void (*cpustop_restartfunc)(void);
extern int mp_ncpus;
extern int smp_active;
extern int smp_started;
extern int smp_cpus;
extern u_int all_cpus;
extern volatile u_int started_cpus;
extern volatile u_int stopped_cpus;
/*
* For sending values to POST displays.
* XXX FIXME: where does this really belong, isa.h/isa.c perhaps?
* Machine dependent functions used to initialize MP support.
*
* The cpu_mp_probe() should check to see if MP support is present and return
* zero if it is not or non-zero if it is. If MP support is present, then
* cpu_mp_start() will be called so that MP can be enabled. This function
* should do things such as startup secondary processors. It should also
* setup mp_ncpus, all_cpus, and smp_cpus. It should also ensure that
* smp_active and smp_started are initialized at the appropriate time.
* Once cpu_mp_start() returns, machine independent MP startup code will be
* executed and a simple message will be output to the console. Finally,
* cpu_mp_announce() will be called so that machine dependent messages about
* the MP support may be output to the console if desired.
*/
extern int current_postcode; /** XXX currently in mp_machdep.c */
#define POSTCODE(X) current_postcode = (X), \
outb(0x80, current_postcode)
#define POSTCODE_LO(X) current_postcode &= 0xf0, \
current_postcode |= ((X) & 0x0f), \
outb(0x80, current_postcode)
#define POSTCODE_HI(X) current_postcode &= 0x0f, \
current_postcode |= (((X) << 4) & 0xf0), \
outb(0x80, current_postcode)
void cpu_mp_announce(void);
int cpu_mp_probe(void);
void cpu_mp_start(void);
void forward_signal(struct proc *);
void forward_roundrobin(void);
void globaldata_register(struct globaldata *gd);
struct globaldata *globaldata_find(u_int cpuid);
int restart_cpus(u_int);
int stop_cpus(u_int);
void smp_rendezvous_action(void);
void smp_rendezvous(void (*)(void *),
void (*)(void *),
void (*)(void *),
void *arg);
#include <machine/apic.h>
/* global data in mpboot.s */
extern int bootMP_size;
/* functions in mpboot.s */
void bootMP __P((void));
/* global data in apic_vector.s */
extern volatile u_int stopped_cpus;
extern volatile u_int started_cpus;
extern volatile u_int checkstate_probed_cpus;
extern volatile u_int checkstate_need_ast;
extern volatile u_int resched_cpus;
extern void (*cpustop_restartfunc) __P((void));
/* global data in mp_machdep.c */
extern int bsp_apic_ready;
extern int mp_ncpus;
extern int mp_naps;
extern int mp_nbusses;
extern int mp_napics;
extern int mp_picmode;
extern int boot_cpu_id;
extern vm_offset_t cpu_apic_address;
extern vm_offset_t io_apic_address[];
extern u_int32_t cpu_apic_versions[];
extern u_int32_t *io_apic_versions;
extern int cpu_num_to_apic_id[];
extern int io_num_to_apic_id[];
extern int apic_id_to_logical[];
#define APIC_INTMAPSIZE 32
struct apic_intmapinfo {
int ioapic;
int int_pin;
volatile void *apic_address;
int redirindex;
};
extern struct apic_intmapinfo int_to_apicintpin[];
extern u_int all_cpus;
extern struct pcb stoppcbs[];
/* functions in mp_machdep.c */
u_int mp_bootaddress __P((u_int));
int mp_probe __P((void));
void mp_start __P((void));
void mp_announce __P((void));
u_int isa_apic_mask __P((u_int));
int isa_apic_irq __P((int));
int pci_apic_irq __P((int, int, int));
int apic_irq __P((int, int));
int next_apic_irq __P((int));
int undirect_isa_irq __P((int));
int undirect_pci_irq __P((int));
int apic_bus_type __P((int));
int apic_src_bus_id __P((int, int));
int apic_src_bus_irq __P((int, int));
int apic_int_type __P((int, int));
int apic_trigger __P((int, int));
int apic_polarity __P((int, int));
void assign_apic_irq __P((int apic, int intpin, int irq));
void revoke_apic_irq __P((int irq));
void bsp_apic_configure __P((void));
void init_secondary __P((void));
void smp_invltlb __P((void));
int stop_cpus __P((u_int));
int restart_cpus __P((u_int));
void forward_statclock __P((int pscnt));
void forward_hardclock __P((int pscnt));
void forward_signal __P((struct proc *));
void forward_roundrobin __P((void));
#ifdef APIC_INTR_REORDER
void set_lapic_isrloc __P((int, int));
#endif /* APIC_INTR_REORDER */
void smp_rendezvous_action __P((void));
void smp_rendezvous __P((void (*)(void *),
void (*)(void *),
void (*)(void *),
void *arg));
/* global data in mpapic.c */
extern volatile lapic_t lapic;
extern volatile ioapic_t **ioapic;
/* functions in mpapic.c */
void apic_dump __P((char*));
void apic_initialize __P((void));
void imen_dump __P((void));
int apic_ipi __P((int, int, int));
int selected_apic_ipi __P((u_int, int, int));
int io_apic_setup __P((int));
void io_apic_setup_intpin __P((int, int));
void io_apic_set_id __P((int, int));
int io_apic_get_id __P((int));
int ext_int_setup __P((int, int));
void set_apic_timer __P((int));
int read_apic_timer __P((void));
void u_sleep __P((int));
u_int io_apic_read __P((int, int));
void io_apic_write __P((int, int, u_int));
/* global data in init_smp.c */
extern int invltlb_ok;
extern int smp_active;
extern int smp_started;
extern volatile int smp_idle_loops;
#endif /* SMP */
#endif /* !LOCORE */
#endif /* SMP && !APIC_IO */
#endif /* _KERNEL */
#endif /* _MACHINE_SMP_H_ */
#endif /* _SYS_SMP_H_ */

2
sys/sys/systm.h
View File

@ -152,8 +152,10 @@ int susword __P((void *base, int word));
void realitexpire __P((void *));
void hardclock __P((struct clockframe *frame));
void hardclock_process __P((struct proc *p, int user));
void softclock __P((void *));
void statclock __P((struct clockframe *frame));
void statclock_process __P((struct proc *p, register_t pc, int user));
void startprofclock __P((struct proc *));
void stopprofclock __P((struct proc *));