d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
4cdb86f203
freebsd-skq/sys/i386/xen/mp_machdep.c
adrian 30e73eac6d Fix the MP IPI code to differentiate between bitmapped IPIs and function IPIs. 
This attempts to fix the IPI handling code to correctly differentiate
between bitmapped IPIs and function IPIs. The Xen IPIs were on low numbers
which clashed with the bitmapped IPIs.

This commit bumps those IPI numbers up to 240 and above (just like in the i386
code) and fiddles with the ipi_vectors[] logic to call the correct function.

This still isn't "right". Specifically, the IPI code may work fine for TLB
shootdown events but the rendezvous/lazypmap IPIs are thrown by calling ipi_*()
routines which don't set the call_func stuff (function id, addr1, addr2) that
the TLB shootdown events are. So the Xen SMP support is still broken.

PR:		135069
2009-05-31 08:11:39 +00:00

1193 lines
27 KiB
C
Raw Blame History

/*-
* Copyright (c) 1996, by Steve Passe
* Copyright (c) 2008, by Kip Macy
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. The name of the developer may NOT be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_apic.h"
#include "opt_cpu.h"
#include "opt_kstack_pages.h"
#include "opt_mp_watchdog.h"
#include "opt_sched.h"
#include "opt_smp.h"
#if !defined(lint)
#if !defined(SMP)
#error How did you get here?
#endif
#ifndef DEV_APIC
#error The apic device is required for SMP, add "device apic" to your config file.
#endif
#if defined(CPU_DISABLE_CMPXCHG) && !defined(COMPILING_LINT)
#error SMP not supported with CPU_DISABLE_CMPXCHG
#endif
#endif /* not lint */
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/cons.h> /* cngetc() */
#ifdef GPROF
#include <sys/gmon.h>
#endif
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/memrange.h>
#include <sys/mutex.h>
#include <sys/pcpu.h>
#include <sys/proc.h>
#include <sys/sched.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/pmap.h>
#include <vm/vm_kern.h>
#include <vm/vm_extern.h>
#include <vm/vm_page.h>
#include <machine/apicreg.h>
#include <machine/md_var.h>
#include <machine/mp_watchdog.h>
#include <machine/pcb.h>
#include <machine/psl.h>
#include <machine/smp.h>
#include <machine/specialreg.h>
#include <machine/pcpu.h>
#include <machine/xen/xen-os.h>
#include <xen/evtchn.h>
#include <xen/xen_intr.h>
#include <xen/hypervisor.h>
#include <xen/interface/vcpu.h>
#define stop_cpus_with_nmi 0
int mp_naps; /* # of Applications processors */
int boot_cpu_id = -1; /* designated BSP */
extern struct pcpu __pcpu[];
static int bootAP;
static union descriptor *bootAPgdt;
static char resched_name[NR_CPUS][15];
static char callfunc_name[NR_CPUS][15];
/* Free these after use */
void *bootstacks[MAXCPU];
/* Hotwire a 0->4MB V==P mapping */
extern pt_entry_t *KPTphys;
struct pcb stoppcbs[MAXCPU];
/* Variables needed for SMP tlb shootdown. */
vm_offset_t smp_tlb_addr1;
vm_offset_t smp_tlb_addr2;
volatile int smp_tlb_wait;
typedef void call_data_func_t(uintptr_t , uintptr_t);
static u_int logical_cpus;
/* used to hold the AP's until we are ready to release them */
static struct mtx ap_boot_mtx;
/* Set to 1 once we're ready to let the APs out of the pen. */
static volatile int aps_ready = 0;
/*
* Store data from cpu_add() until later in the boot when we actually setup
* the APs.
*/
struct cpu_info {
int cpu_present:1;
int cpu_bsp:1;
int cpu_disabled:1;
} static cpu_info[MAX_APIC_ID + 1];
int cpu_apic_ids[MAXCPU];
int apic_cpuids[MAX_APIC_ID + 1];
/* Holds pending bitmap based IPIs per CPU */
static volatile u_int cpu_ipi_pending[MAXCPU];
static int cpu_logical;
static int cpu_cores;
static void assign_cpu_ids(void);
static void set_interrupt_apic_ids(void);
int start_all_aps(void);
static int start_ap(int apic_id);
static void release_aps(void *dummy);
static u_int hyperthreading_cpus;
static cpumask_t hyperthreading_cpus_mask;
extern void Xhypervisor_callback(void);
extern void failsafe_callback(void);
extern void pmap_lazyfix_action(void);
struct cpu_group *
cpu_topo(void)
{
if (cpu_cores == 0)
cpu_cores = 1;
if (cpu_logical == 0)
cpu_logical = 1;
if (mp_ncpus % (cpu_cores * cpu_logical) != 0) {
printf("WARNING: Non-uniform processors.\n");
printf("WARNING: Using suboptimal topology.\n");
return (smp_topo_none());
}
/*
* No multi-core or hyper-threaded.
*/
if (cpu_logical * cpu_cores == 1)
return (smp_topo_none());
/*
* Only HTT no multi-core.
*/
if (cpu_logical > 1 && cpu_cores == 1)
return (smp_topo_1level(CG_SHARE_L1, cpu_logical, CG_FLAG_HTT));
/*
* Only multi-core no HTT.
*/
if (cpu_cores > 1 && cpu_logical == 1)
return (smp_topo_1level(CG_SHARE_NONE, cpu_cores, 0));
/*
* Both HTT and multi-core.
*/
return (smp_topo_2level(CG_SHARE_NONE, cpu_cores,
CG_SHARE_L1, cpu_logical, CG_FLAG_HTT));
}
/*
* Calculate usable address in base memory for AP trampoline code.
*/
u_int
mp_bootaddress(u_int basemem)
{
return (basemem);
}
void
cpu_add(u_int apic_id, char boot_cpu)
{
if (apic_id > MAX_APIC_ID) {
panic("SMP: APIC ID %d too high", apic_id);
return;
}
KASSERT(cpu_info[apic_id].cpu_present == 0, ("CPU %d added twice",
apic_id));
cpu_info[apic_id].cpu_present = 1;
if (boot_cpu) {
KASSERT(boot_cpu_id == -1,
("CPU %d claims to be BSP, but CPU %d already is", apic_id,
boot_cpu_id));
boot_cpu_id = apic_id;
cpu_info[apic_id].cpu_bsp = 1;
}
if (mp_ncpus < MAXCPU)
mp_ncpus++;
if (bootverbose)
printf("SMP: Added CPU %d (%s)\n", apic_id, boot_cpu ? "BSP" :
"AP");
}
void
cpu_mp_setmaxid(void)
{
mp_maxid = MAXCPU - 1;
}
int
cpu_mp_probe(void)
{
/*
* Always record BSP in CPU map so that the mbuf init code works
* correctly.
*/
all_cpus = 1;
if (mp_ncpus == 0) {
/*
* No CPUs were found, so this must be a UP system. Setup
* the variables to represent a system with a single CPU
* with an id of 0.
*/
mp_ncpus = 1;
return (0);
}
/* At least one CPU was found. */
if (mp_ncpus == 1) {
/*
* One CPU was found, so this must be a UP system with
* an I/O APIC.
*/
return (0);
}
/* At least two CPUs were found. */
return (1);
}
/*
* Initialize the IPI handlers and start up the AP's.
*/
void
cpu_mp_start(void)
{
int i;
/* Initialize the logical ID to APIC ID table. */
for (i = 0; i < MAXCPU; i++) {
cpu_apic_ids[i] = -1;
cpu_ipi_pending[i] = 0;
}
/* Set boot_cpu_id if needed. */
if (boot_cpu_id == -1) {
boot_cpu_id = PCPU_GET(apic_id);
cpu_info[boot_cpu_id].cpu_bsp = 1;
} else
KASSERT(boot_cpu_id == PCPU_GET(apic_id),
("BSP's APIC ID doesn't match boot_cpu_id"));
cpu_apic_ids[0] = boot_cpu_id;
apic_cpuids[boot_cpu_id] = 0;
assign_cpu_ids();
/* Start each Application Processor */
start_all_aps();
/* Setup the initial logical CPUs info. */
logical_cpus = logical_cpus_mask = 0;
if (cpu_feature & CPUID_HTT)
logical_cpus = (cpu_procinfo & CPUID_HTT_CORES) >> 16;
set_interrupt_apic_ids();
}
static void
iv_rendezvous(uintptr_t a, uintptr_t b)
{
smp_rendezvous_action();
}
static void
iv_invltlb(uintptr_t a, uintptr_t b)
{
xen_tlb_flush();
}
static void
iv_invlpg(uintptr_t a, uintptr_t b)
{
xen_invlpg(a);
}
static void
iv_invlrng(uintptr_t a, uintptr_t b)
{
vm_offset_t start = (vm_offset_t)a;
vm_offset_t end = (vm_offset_t)b;
while (start < end) {
xen_invlpg(start);
start += PAGE_SIZE;
}
}
static void
iv_invlcache(uintptr_t a, uintptr_t b)
{
wbinvd();
atomic_add_int(&smp_tlb_wait, 1);
}
static void
iv_lazypmap(uintptr_t a, uintptr_t b)
{
pmap_lazyfix_action();
atomic_add_int(&smp_tlb_wait, 1);
}
/*
* These start from "IPI offset" APIC_IPI_INTS
*/
static call_data_func_t *ipi_vectors[6] =
{
iv_rendezvous,
iv_invltlb,
iv_invlpg,
iv_invlrng,
iv_invlcache,
iv_lazypmap,
};
/*
* Reschedule call back. Nothing to do,
* all the work is done automatically when
* we return from the interrupt.
*/
static int
smp_reschedule_interrupt(void *unused)
{
int cpu = PCPU_GET(cpuid);
u_int ipi_bitmap;
ipi_bitmap = atomic_readandclear_int(&cpu_ipi_pending[cpu]);
if (ipi_bitmap & (1 << IPI_PREEMPT)) {
#ifdef COUNT_IPIS
(*ipi_preempt_counts[cpu])++;
#endif
sched_preempt(curthread);
}
if (ipi_bitmap & (1 << IPI_AST)) {
#ifdef COUNT_IPIS
(*ipi_ast_counts[cpu])++;
#endif
/* Nothing to do for AST */
}
return (FILTER_HANDLED);
}
struct _call_data {
uint16_t func_id;
uint16_t wait;
uintptr_t arg1;
uintptr_t arg2;
atomic_t started;
atomic_t finished;
};
static struct _call_data *call_data;
static int
smp_call_function_interrupt(void *unused)
{
call_data_func_t *func;
uintptr_t arg1 = call_data->arg1;
uintptr_t arg2 = call_data->arg2;
int wait = call_data->wait;
atomic_t *started = &call_data->started;
atomic_t *finished = &call_data->finished;
/* We only handle function IPIs, not bitmap IPIs */
if (call_data->func_id < APIC_IPI_INTS || call_data->func_id > IPI_BITMAP_VECTOR)
panic("invalid function id %u", call_data->func_id);
func = ipi_vectors[call_data->func_id - APIC_IPI_INTS];
/*
* Notify initiating CPU that I've grabbed the data and am
* about to execute the function
*/
mb();
atomic_inc(started);
/*
* At this point the info structure may be out of scope unless wait==1
*/
(*func)(arg1, arg2);
if (wait) {
mb();
atomic_inc(finished);
}
atomic_add_int(&smp_tlb_wait, 1);
return (FILTER_HANDLED);
}
/*
* Print various information about the SMP system hardware and setup.
*/
void
cpu_mp_announce(void)
{
int i, x;
/* List CPUs */
printf(" cpu0 (BSP): APIC ID: %2d\n", boot_cpu_id);
for (i = 1, x = 0; x <= MAX_APIC_ID; x++) {
if (!cpu_info[x].cpu_present || cpu_info[x].cpu_bsp)
continue;
if (cpu_info[x].cpu_disabled)
printf(" cpu (AP): APIC ID: %2d (disabled)\n", x);
else {
KASSERT(i < mp_ncpus,
("mp_ncpus and actual cpus are out of whack"));
printf(" cpu%d (AP): APIC ID: %2d\n", i++, x);
}
}
}
static int
xen_smp_intr_init(unsigned int cpu)
{
int rc;
unsigned int irq;
per_cpu(resched_irq, cpu) = per_cpu(callfunc_irq, cpu) = -1;
sprintf(resched_name[cpu], "resched%u", cpu);
rc = bind_ipi_to_irqhandler(RESCHEDULE_VECTOR,
cpu,
resched_name[cpu],
smp_reschedule_interrupt,
INTR_FAST|INTR_TYPE_TTY|INTR_MPSAFE, &irq);
printf("[XEN] IPI cpu=%d irq=%d vector=RESCHEDULE_VECTOR (%d)\n",
cpu, irq, RESCHEDULE_VECTOR);
per_cpu(resched_irq, cpu) = irq;
sprintf(callfunc_name[cpu], "callfunc%u", cpu);
rc = bind_ipi_to_irqhandler(CALL_FUNCTION_VECTOR,
cpu,
callfunc_name[cpu],
smp_call_function_interrupt,
INTR_FAST|INTR_TYPE_TTY|INTR_MPSAFE, &irq);
if (rc < 0)
goto fail;
per_cpu(callfunc_irq, cpu) = irq;
printf("[XEN] IPI cpu=%d irq=%d vector=CALL_FUNCTION_VECTOR (%d)\n",
cpu, irq, CALL_FUNCTION_VECTOR);
if ((cpu != 0) && ((rc = ap_cpu_initclocks(cpu)) != 0))
goto fail;
return 0;
fail:
if (per_cpu(resched_irq, cpu) >= 0)
unbind_from_irqhandler(per_cpu(resched_irq, cpu));
if (per_cpu(callfunc_irq, cpu) >= 0)
unbind_from_irqhandler(per_cpu(callfunc_irq, cpu));
return rc;
}
static void
xen_smp_intr_init_cpus(void *unused)
{
int i;
for (i = 0; i < mp_ncpus; i++)
xen_smp_intr_init(i);
}
#define MTOPSIZE (1<<(14 + PAGE_SHIFT))
/*
* AP CPU's call this to initialize themselves.
*/
void
init_secondary(void)
{
vm_offset_t addr;
int gsel_tss;
/* bootAP is set in start_ap() to our ID. */
PCPU_SET(currentldt, _default_ldt);
gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
#if 0
gdt[bootAP * NGDT + GPROC0_SEL].sd.sd_type = SDT_SYS386TSS;
#endif
PCPU_SET(common_tss.tss_esp0, 0); /* not used until after switch */
PCPU_SET(common_tss.tss_ss0, GSEL(GDATA_SEL, SEL_KPL));
PCPU_SET(common_tss.tss_ioopt, (sizeof (struct i386tss)) << 16);
#if 0
PCPU_SET(tss_gdt, &gdt[bootAP * NGDT + GPROC0_SEL].sd);
PCPU_SET(common_tssd, *PCPU_GET(tss_gdt));
#endif
PCPU_SET(fsgs_gdt, &gdt[GUFS_SEL].sd);
/*
* Set to a known state:
* Set by mpboot.s: CR0_PG, CR0_PE
* Set by cpu_setregs: CR0_NE, CR0_MP, CR0_TS, CR0_WP, CR0_AM
*/
/*
* signal our startup to the BSP.
*/
mp_naps++;
/* Spin until the BSP releases the AP's. */
while (!aps_ready)
ia32_pause();
/* BSP may have changed PTD while we were waiting */
invltlb();
for (addr = 0; addr < NKPT * NBPDR - 1; addr += PAGE_SIZE)
invlpg(addr);
/* set up FPU state on the AP */
npxinit();
#if 0
/* set up SSE registers */
enable_sse();
#endif
#if 0 && defined(PAE)
/* Enable the PTE no-execute bit. */
if ((amd_feature & AMDID_NX) != 0) {
uint64_t msr;
msr = rdmsr(MSR_EFER) | EFER_NXE;
wrmsr(MSR_EFER, msr);
}
#endif
#if 0
/* A quick check from sanity claus */
if (PCPU_GET(apic_id) != lapic_id()) {
printf("SMP: cpuid = %d\n", PCPU_GET(cpuid));
printf("SMP: actual apic_id = %d\n", lapic_id());
printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id));
panic("cpuid mismatch! boom!!");
}
#endif
/* Initialize curthread. */
KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
PCPU_SET(curthread, PCPU_GET(idlethread));
mtx_lock_spin(&ap_boot_mtx);
#if 0
/* Init local apic for irq's */
lapic_setup(1);
#endif
smp_cpus++;
CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid));
printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid));
/* Determine if we are a logical CPU. */
if (logical_cpus > 1 && PCPU_GET(apic_id) % logical_cpus != 0)
logical_cpus_mask |= PCPU_GET(cpumask);
/* Determine if we are a hyperthread. */
if (hyperthreading_cpus > 1 &&
PCPU_GET(apic_id) % hyperthreading_cpus != 0)
hyperthreading_cpus_mask |= PCPU_GET(cpumask);
/* Build our map of 'other' CPUs. */
PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask));
#if 0
if (bootverbose)
lapic_dump("AP");
#endif
if (smp_cpus == mp_ncpus) {
/* enable IPI's, tlb shootdown, freezes etc */
atomic_store_rel_int(&smp_started, 1);
smp_active = 1; /* historic */
}
mtx_unlock_spin(&ap_boot_mtx);
/* wait until all the AP's are up */
while (smp_started == 0)
ia32_pause();
PCPU_SET(curthread, PCPU_GET(idlethread));
/* enter the scheduler */
sched_throw(NULL);
panic("scheduler returned us to %s", __func__);
/* NOTREACHED */
}
/*******************************************************************
* local functions and data
*/
/*
* We tell the I/O APIC code about all the CPUs we want to receive
* interrupts. If we don't want certain CPUs to receive IRQs we
* can simply not tell the I/O APIC code about them in this function.
* We also do not tell it about the BSP since it tells itself about
* the BSP internally to work with UP kernels and on UP machines.
*/
static void
set_interrupt_apic_ids(void)
{
u_int i, apic_id;
for (i = 0; i < MAXCPU; i++) {
apic_id = cpu_apic_ids[i];
if (apic_id == -1)
continue;
if (cpu_info[apic_id].cpu_bsp)
continue;
if (cpu_info[apic_id].cpu_disabled)
continue;
/* Don't let hyperthreads service interrupts. */
if (hyperthreading_cpus > 1 &&
apic_id % hyperthreading_cpus != 0)
continue;
intr_add_cpu(i);
}
}
/*
* Assign logical CPU IDs to local APICs.
*/
static void
assign_cpu_ids(void)
{
u_int i;
/* Check for explicitly disabled CPUs. */
for (i = 0; i <= MAX_APIC_ID; i++) {
if (!cpu_info[i].cpu_present || cpu_info[i].cpu_bsp)
continue;
/* Don't use this CPU if it has been disabled by a tunable. */
if (resource_disabled("lapic", i)) {
cpu_info[i].cpu_disabled = 1;
continue;
}
}
/*
* Assign CPU IDs to local APIC IDs and disable any CPUs
* beyond MAXCPU. CPU 0 has already been assigned to the BSP,
* so we only have to assign IDs for APs.
*/
mp_ncpus = 1;
for (i = 0; i <= MAX_APIC_ID; i++) {
if (!cpu_info[i].cpu_present || cpu_info[i].cpu_bsp ||
cpu_info[i].cpu_disabled)
continue;
if (mp_ncpus < MAXCPU) {
cpu_apic_ids[mp_ncpus] = i;
apic_cpuids[i] = mp_ncpus;
mp_ncpus++;
} else
cpu_info[i].cpu_disabled = 1;
}
KASSERT(mp_maxid >= mp_ncpus - 1,
("%s: counters out of sync: max %d, count %d", __func__, mp_maxid,
mp_ncpus));
}
/*
* start each AP in our list
*/
/* Lowest 1MB is already mapped: don't touch*/
#define TMPMAP_START 1
int
start_all_aps(void)
{
int x,apic_id, cpu;
struct pcpu *pc;
mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
/* set up temporary P==V mapping for AP boot */
/* XXX this is a hack, we should boot the AP on its own stack/PTD */
/* start each AP */
for (cpu = 1; cpu < mp_ncpus; cpu++) {
apic_id = cpu_apic_ids[cpu];
bootAP = cpu;
bootAPgdt = gdt + (512*cpu);
/* Get per-cpu data */
pc = &__pcpu[bootAP];
pcpu_init(pc, bootAP, sizeof(struct pcpu));
pc->pc_apic_id = cpu_apic_ids[bootAP];
pc->pc_prvspace = pc;
pc->pc_curthread = 0;
gdt_segs[GPRIV_SEL].ssd_base = (int) pc;
gdt_segs[GPROC0_SEL].ssd_base = (int) &pc->pc_common_tss;
PT_SET_MA(bootAPgdt, xpmap_ptom(VTOP(bootAPgdt)) | PG_V | PG_RW);
bzero(bootAPgdt, PAGE_SIZE);
for (x = 0; x < NGDT; x++)
ssdtosd(&gdt_segs[x], &bootAPgdt[x].sd);
PT_SET_MA(bootAPgdt, vtomach(bootAPgdt) | PG_V);
#ifdef notyet
if (HYPERVISOR_vcpu_op(VCPUOP_get_physid, cpu, &cpu_id) == 0) {
apicid = xen_vcpu_physid_to_x86_apicid(cpu_id.phys_id);
acpiid = xen_vcpu_physid_to_x86_acpiid(cpu_id.phys_id);
#ifdef CONFIG_ACPI
if (acpiid != 0xff)
x86_acpiid_to_apicid[acpiid] = apicid;
#endif
}
#endif
/* attempt to start the Application Processor */
if (!start_ap(cpu)) {
printf("AP #%d (PHY# %d) failed!\n", cpu, apic_id);
/* better panic as the AP may be running loose */
printf("panic y/n? [y] ");
if (cngetc() != 'n')
panic("bye-bye");
}
all_cpus |= (1 << cpu); /* record AP in CPU map */
}
/* build our map of 'other' CPUs */
PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask));
pmap_invalidate_range(kernel_pmap, 0, NKPT * NBPDR - 1);
/* number of APs actually started */
return mp_naps;
}
extern uint8_t *pcpu_boot_stack;
extern trap_info_t trap_table[];
static void
smp_trap_init(trap_info_t *trap_ctxt)
{
const trap_info_t *t = trap_table;
for (t = trap_table; t->address; t++) {
trap_ctxt[t->vector].flags = t->flags;
trap_ctxt[t->vector].cs = t->cs;
trap_ctxt[t->vector].address = t->address;
}
}
extern int nkpt;
static void
cpu_initialize_context(unsigned int cpu)
{
/* vcpu_guest_context_t is too large to allocate on the stack.
* Hence we allocate statically and protect it with a lock */
vm_page_t m[4];
static vcpu_guest_context_t ctxt;
vm_offset_t boot_stack;
vm_offset_t newPTD;
vm_paddr_t ma[NPGPTD];
static int color;
int i;
/*
* Page 0,[0-3] PTD
* Page 1, [4] boot stack
* Page [5] PDPT
*
*/
for (i = 0; i < NPGPTD + 2; i++) {
m[i] = vm_page_alloc(NULL, color++,
VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
VM_ALLOC_ZERO);
pmap_zero_page(m[i]);
}
boot_stack = kmem_alloc_nofault(kernel_map, 1);
newPTD = kmem_alloc_nofault(kernel_map, NPGPTD);
ma[0] = xpmap_ptom(VM_PAGE_TO_PHYS(m[0]))|PG_V;
#ifdef PAE
pmap_kenter(boot_stack, VM_PAGE_TO_PHYS(m[NPGPTD + 1]));
for (i = 0; i < NPGPTD; i++) {
((vm_paddr_t *)boot_stack)[i] =
ma[i] =
xpmap_ptom(VM_PAGE_TO_PHYS(m[i]))|PG_V;
}
#endif
/*
* Copy cpu0 IdlePTD to new IdlePTD - copying only
* kernel mappings
*/
pmap_qenter(newPTD, m, 4);
memcpy((uint8_t *)newPTD + KPTDI*sizeof(vm_paddr_t),
(uint8_t *)PTOV(IdlePTD) + KPTDI*sizeof(vm_paddr_t),
nkpt*sizeof(vm_paddr_t));
pmap_qremove(newPTD, 4);
kmem_free(kernel_map, newPTD, 4);
/*
* map actual idle stack to boot_stack
*/
pmap_kenter(boot_stack, VM_PAGE_TO_PHYS(m[NPGPTD]));
xen_pgdpt_pin(xpmap_ptom(VM_PAGE_TO_PHYS(m[NPGPTD + 1])));
vm_page_lock_queues();
for (i = 0; i < 4; i++) {
int pdir = (PTDPTDI + i) / NPDEPG;
int curoffset = (PTDPTDI + i) % NPDEPG;
xen_queue_pt_update((vm_paddr_t)
((ma[pdir] & ~PG_V) + (curoffset*sizeof(vm_paddr_t))),
ma[i]);
}
PT_UPDATES_FLUSH();
vm_page_unlock_queues();
memset(&ctxt, 0, sizeof(ctxt));
ctxt.flags = VGCF_IN_KERNEL;
ctxt.user_regs.ds = GSEL(GDATA_SEL, SEL_KPL);
ctxt.user_regs.es = GSEL(GDATA_SEL, SEL_KPL);
ctxt.user_regs.fs = GSEL(GPRIV_SEL, SEL_KPL);
ctxt.user_regs.gs = GSEL(GDATA_SEL, SEL_KPL);
ctxt.user_regs.cs = GSEL(GCODE_SEL, SEL_KPL);
ctxt.user_regs.ss = GSEL(GDATA_SEL, SEL_KPL);
ctxt.user_regs.eip = (unsigned long)init_secondary;
ctxt.user_regs.eflags = PSL_KERNEL | 0x1000; /* IOPL_RING1 */
memset(&ctxt.fpu_ctxt, 0, sizeof(ctxt.fpu_ctxt));
smp_trap_init(ctxt.trap_ctxt);
ctxt.ldt_ents = 0;
ctxt.gdt_frames[0] = (uint32_t)((uint64_t)vtomach(bootAPgdt) >> PAGE_SHIFT);
ctxt.gdt_ents = 512;
#ifdef __i386__
ctxt.user_regs.esp = boot_stack + PAGE_SIZE;
ctxt.kernel_ss = GSEL(GDATA_SEL, SEL_KPL);
ctxt.kernel_sp = boot_stack + PAGE_SIZE;
ctxt.event_callback_cs = GSEL(GCODE_SEL, SEL_KPL);
ctxt.event_callback_eip = (unsigned long)Xhypervisor_callback;
ctxt.failsafe_callback_cs = GSEL(GCODE_SEL, SEL_KPL);
ctxt.failsafe_callback_eip = (unsigned long)failsafe_callback;
ctxt.ctrlreg[3] = xpmap_ptom(VM_PAGE_TO_PHYS(m[NPGPTD + 1]));
#else /* __x86_64__ */
ctxt.user_regs.esp = idle->thread.rsp0 - sizeof(struct pt_regs);
ctxt.kernel_ss = GSEL(GDATA_SEL, SEL_KPL);
ctxt.kernel_sp = idle->thread.rsp0;
ctxt.event_callback_eip = (unsigned long)hypervisor_callback;
ctxt.failsafe_callback_eip = (unsigned long)failsafe_callback;
ctxt.syscall_callback_eip = (unsigned long)system_call;
ctxt.ctrlreg[3] = xen_pfn_to_cr3(virt_to_mfn(init_level4_pgt));
ctxt.gs_base_kernel = (unsigned long)(cpu_pda(cpu));
#endif
printf("gdtpfn=%lx pdptpfn=%lx\n",
ctxt.gdt_frames[0],
ctxt.ctrlreg[3] >> PAGE_SHIFT);
PANIC_IF(HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, &ctxt));
DELAY(3000);
PANIC_IF(HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL));
}
/*
* This function starts the AP (application processor) identified
* by the APIC ID 'physicalCpu'. It does quite a "song and dance"
* to accomplish this. This is necessary because of the nuances
* of the different hardware we might encounter. It isn't pretty,
* but it seems to work.
*/
int cpus;
static int
start_ap(int apic_id)
{
int ms;
/* used as a watchpoint to signal AP startup */
cpus = mp_naps;
cpu_initialize_context(apic_id);
/* Wait up to 5 seconds for it to start. */
for (ms = 0; ms < 5000; ms++) {
if (mp_naps > cpus)
return 1; /* return SUCCESS */
DELAY(1000);
}
return 0; /* return FAILURE */
}
/*
* Flush the TLB on all other CPU's
*/
static void
smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2)
{
u_int ncpu;
struct _call_data data;
ncpu = mp_ncpus - 1; /* does not shootdown self */
if (ncpu < 1)
return; /* no other cpus */
if (!(read_eflags() & PSL_I))
panic("%s: interrupts disabled", __func__);
mtx_lock_spin(&smp_ipi_mtx);
KASSERT(call_data == NULL, ("call_data isn't null?!"));
call_data = &data;
call_data->func_id = vector;
call_data->arg1 = addr1;
call_data->arg2 = addr2;
atomic_store_rel_int(&smp_tlb_wait, 0);
ipi_all_but_self(vector);
while (smp_tlb_wait < ncpu)
ia32_pause();
call_data = NULL;
mtx_unlock_spin(&smp_ipi_mtx);
}
static void
smp_targeted_tlb_shootdown(cpumask_t mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2)
{
int ncpu, othercpus;
struct _call_data data;
othercpus = mp_ncpus - 1;
if (mask == (u_int)-1) {
ncpu = othercpus;
if (ncpu < 1)
return;
} else {
mask &= ~PCPU_GET(cpumask);
if (mask == 0)
return;
ncpu = bitcount32(mask);
if (ncpu > othercpus) {
/* XXX this should be a panic offence */
printf("SMP: tlb shootdown to %d other cpus (only have %d)\n",
ncpu, othercpus);
ncpu = othercpus;
}
/* XXX should be a panic, implied by mask == 0 above */
if (ncpu < 1)
return;
}
if (!(read_eflags() & PSL_I))
panic("%s: interrupts disabled", __func__);
mtx_lock_spin(&smp_ipi_mtx);
KASSERT(call_data == NULL, ("call_data isn't null?!"));
call_data = &data;
call_data->func_id = vector;
call_data->arg1 = addr1;
call_data->arg2 = addr2;
atomic_store_rel_int(&smp_tlb_wait, 0);
if (mask == (u_int)-1)
ipi_all_but_self(vector);
else
ipi_selected(mask, vector);
while (smp_tlb_wait < ncpu)
ia32_pause();
call_data = NULL;
mtx_unlock_spin(&smp_ipi_mtx);
}
void
smp_cache_flush(void)
{
if (smp_started)
smp_tlb_shootdown(IPI_INVLCACHE, 0, 0);
}
void
smp_invltlb(void)
{
if (smp_started) {
smp_tlb_shootdown(IPI_INVLTLB, 0, 0);
}
}
void
smp_invlpg(vm_offset_t addr)
{
if (smp_started) {
smp_tlb_shootdown(IPI_INVLPG, addr, 0);
}
}
void
smp_invlpg_range(vm_offset_t addr1, vm_offset_t addr2)
{
if (smp_started) {
smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2);
}
}
void
smp_masked_invltlb(cpumask_t mask)
{
if (smp_started) {
smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0);
}
}
void
smp_masked_invlpg(cpumask_t mask, vm_offset_t addr)
{
if (smp_started) {
smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0);
}
}
void
smp_masked_invlpg_range(cpumask_t mask, vm_offset_t addr1, vm_offset_t addr2)
{
if (smp_started) {
smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2);
}
}
/*
* send an IPI to a set of cpus.
*/
void
ipi_selected(cpumask_t cpus, u_int ipi)
{
int cpu;
u_int bitmap = 0;
u_int old_pending;
u_int new_pending;
if (IPI_IS_BITMAPED(ipi)) {
bitmap = 1 << ipi;
ipi = IPI_BITMAP_VECTOR;
}
CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi);
while ((cpu = ffs(cpus)) != 0) {
cpu--;
cpus &= ~(1 << cpu);
KASSERT(cpu_apic_ids[cpu] != -1,
("IPI to non-existent CPU %d", cpu));
if (bitmap) {
do {
old_pending = cpu_ipi_pending[cpu];
new_pending = old_pending | bitmap;
} while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],old_pending, new_pending));
if (!old_pending)
ipi_pcpu(cpu, RESCHEDULE_VECTOR);
continue;
} else {
KASSERT(call_data != NULL, ("call_data not set"));
ipi_pcpu(cpu, CALL_FUNCTION_VECTOR);
}
}
}
/*
* send an IPI to all CPUs EXCEPT myself
*/
void
ipi_all_but_self(u_int ipi)
{
CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
ipi_selected(PCPU_GET(other_cpus), ipi);
}
/*
* Handle an IPI_STOP by saving our current context and spinning until we
* are resumed.
*/
void
cpustop_handler(void)
{
int cpu = PCPU_GET(cpuid);
int cpumask = PCPU_GET(cpumask);
savectx(&stoppcbs[cpu]);
/* Indicate that we are stopped */
atomic_set_int(&stopped_cpus, cpumask);
/* Wait for restart */
while (!(started_cpus & cpumask))
ia32_pause();
atomic_clear_int(&started_cpus, cpumask);
atomic_clear_int(&stopped_cpus, cpumask);
if (cpu == 0 && cpustop_restartfunc != NULL) {
cpustop_restartfunc();
cpustop_restartfunc = NULL;
}
}
/*
* This is called once the rest of the system is up and running and we're
* ready to let the AP's out of the pen.
*/
static void
release_aps(void *dummy __unused)
{
if (mp_ncpus == 1)
return;
atomic_store_rel_int(&aps_ready, 1);
while (smp_started == 0)
ia32_pause();
}
SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL);
SYSINIT(start_ipis, SI_SUB_INTR, SI_ORDER_ANY, xen_smp_intr_init_cpus, NULL);
Reference in New Issue View Git Blame Copy Permalink