d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove the secondary_stacks array in arm64 and riscv kernels.

Browse Source 
Instead, dynamically allocate a page for the boot stack of each AP when
starting them up, like we do on x86.  This shrinks the bss by
MAXCPU*KSTACK_PAGES pages, which corresponds to 4MB on arm64 and 256KB
on riscv.

Duplicate the logic used on x86 to free the bootstacks, by using a
sysinit to wait for each AP to switch to a thread before freeing its
stack.

While here, mark some static MD variables as such.

Reviewed by:	kib
MFC after:	1 month
Sponsored by:	Juniper Networks, Klara Inc.
Differential Revision:	https://reviews.freebsd.org/D24158
This commit is contained in:
Mark Johnston 2020-03-24 18:43:23 +00:00
parent d681bc9e64
commit 8db2e8fd16
4 changed files with 107 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
sys/arm64/arm64/locore.S
View File

@ -214,11 +214,10 @@ ENTRY(mpentry)
br x15
mp_virtdone:
ldr x4, =secondary_stacks
mov x5, #(PAGE_SIZE * KSTACK_PAGES)
mul x5, x0, x5
add sp, x4, x5
/* Start using the AP boot stack */
ldr x4, =bootstack
ldr x4, [x4]
mov sp, x4
b init_secondary
END(mpentry)
#endif

68
sys/arm64/arm64/mp_machdep.c
View File

@ -123,7 +123,6 @@ static void ipi_preempt(void *);
static void ipi_rendezvous(void *);
static void ipi_stop(void *);
struct mtx ap_boot_mtx;
struct pcb stoppcbs[MAXCPU];
/*
@ -136,10 +135,18 @@ static int cpu0 = -1;
void mpentry(unsigned long cpuid);
void init_secondary(uint64_t);
uint8_t secondary_stacks[MAXCPU - 1][PAGE_SIZE * KSTACK_PAGES] __aligned(16);
/* Synchronize AP startup. */
static struct mtx ap_boot_mtx;
/* Stacks for AP initialization, discarded once idle threads are started. */
void *bootstack;
static void *bootstacks[MAXCPU];
/* Count of started APs, used to synchronize access to bootstack. */
static volatile int aps_started;
/* Set to 1 once we're ready to let the APs out of the pen. */
volatile int aps_ready = 0;
static volatile int aps_ready;
/* Temporary variables for init_secondary() */
void *dpcpu[MAXCPU - 1];
@ -205,14 +212,14 @@ init_secondary(uint64_t cpu)
"mov x18, %0 \n"
"msr tpidr_el1, %0" :: "r"(pcpup));
/* Spin until the BSP releases the APs */
while (!aps_ready)
/* Signal the BSP and spin until it has released all APs. */
atomic_add_int(&aps_started, 1);
while (!atomic_load_int(&aps_ready))
__asm __volatile("wfe");
/* Initialize curthread */
KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
pcpup->pc_curthread = pcpup->pc_idlethread;
pcpup->pc_curpcb = pcpup->pc_idlethread->td_pcb;
/* Initialize curpmap to match TTBR0's current setting. */
pmap0 = vmspace_pmap(&vmspace0);
@ -250,6 +257,11 @@ init_secondary(uint64_t cpu)
kcsan_cpu_init(cpu);
/*
* Assert that smp_after_idle_runnable condition is reasonable.
*/
MPASS(PCPU_GET(curpcb) == NULL);
/* Enter the scheduler */
sched_throw(NULL);
@ -257,6 +269,24 @@ init_secondary(uint64_t cpu)
/* NOTREACHED */
}
static void
smp_after_idle_runnable(void *arg __unused)
{
struct pcpu *pc;
int cpu;
for (cpu = 1; cpu < mp_ncpus; cpu++) {
if (bootstacks[cpu] != NULL) {
pc = pcpu_find(cpu);
while (atomic_load_ptr(&pc->pc_curpcb) == NULL)
cpu_spinwait();
kmem_free((vm_offset_t)bootstacks[cpu], PAGE_SIZE);
}
}
}
SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY,
smp_after_idle_runnable, NULL);
/*
* Send IPI thru interrupt controller.
*/
@ -391,7 +421,7 @@ start_cpu(u_int id, uint64_t target_cpu)
struct pcpu *pcpup;
vm_paddr_t pa;
u_int cpuid;
int err;
int err, naps;
/* Check we are able to start this cpu */
if (id > mp_maxid)
@ -405,7 +435,7 @@ start_cpu(u_int id, uint64_t target_cpu)
/*
* Rotate the CPU IDs to put the boot CPU as CPU 0. We keep the other
* CPUs ordered as the are likely grouped into clusters so it can be
* CPUs ordered as they are likely grouped into clusters so it can be
* useful to keep that property, e.g. for the GICv3 driver to send
* an IPI to all CPUs in the cluster.
*/
@ -420,29 +450,41 @@ start_cpu(u_int id, uint64_t target_cpu)
dpcpu[cpuid - 1] = (void *)kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO);
dpcpu_init(dpcpu[cpuid - 1], cpuid);
bootstacks[cpuid] = (void *)kmem_malloc(PAGE_SIZE, M_WAITOK | M_ZERO);
naps = atomic_load_int(&aps_started);
bootstack = (char *)bootstacks[cpuid] + PAGE_SIZE;
printf("Starting CPU %u (%lx)\n", cpuid, target_cpu);
pa = pmap_extract(kernel_pmap, (vm_offset_t)mpentry);
err = psci_cpu_on(target_cpu, pa, cpuid);
if (err != PSCI_RETVAL_SUCCESS) {
/*
* Panic here if INVARIANTS are enabled and PSCI failed to
* start the requested CPU. If psci_cpu_on returns PSCI_MISSING
* start the requested CPU. psci_cpu_on() returns PSCI_MISSING
* to indicate we are unable to use it to start the given CPU.
*/
KASSERT(err == PSCI_MISSING ||
(mp_quirks & MP_QUIRK_CPULIST) == MP_QUIRK_CPULIST,
("Failed to start CPU %u (%lx)\n", id, target_cpu));
("Failed to start CPU %u (%lx), error %d\n",
id, target_cpu, err));
pcpu_destroy(pcpup);
kmem_free((vm_offset_t)dpcpu[cpuid - 1], DPCPU_SIZE);
dpcpu[cpuid - 1] = NULL;
kmem_free((vm_offset_t)bootstacks[cpuid], PAGE_SIZE);
bootstacks[cpuid] = NULL;
mp_ncpus--;
/* Notify the user that the CPU failed to start */
printf("Failed to start CPU %u (%lx)\n", id, target_cpu);
} else
printf("Failed to start CPU %u (%lx), error %d\n",
id, target_cpu, err);
} else {
/* Wait for the AP to switch to its boot stack. */
while (atomic_load_int(&aps_started) < naps + 1)
cpu_spinwait();
CPU_SET(cpuid, &all_cpus);
}
return (true);
}

10
sys/riscv/riscv/locore.S
View File

@ -301,14 +301,8 @@ ENTRY(mpentry)
beqz t1, 1b
/* Setup stack pointer */
lla t0, secondary_stacks
li t1, (PAGE_SIZE * KSTACK_PAGES)
mulw t2, t1, a0
add t0, t0, t2
add t0, t0, t1
sub t0, t0, s9
li t1, KERNBASE
add sp, t0, t1
lla t0, bootstack
ld sp, 0(t0)
/* Setup supervisor trap vector */
lla t0, mpva

54
sys/riscv/riscv/mp_machdep.c
View File

@ -87,7 +87,6 @@ static device_attach_t riscv64_cpu_attach;
static int ipi_handler(void *);
struct mtx ap_boot_mtx;
struct pcb stoppcbs[MAXCPU];
extern uint32_t boot_hart;
@ -98,13 +97,19 @@ static uint32_t cpu_reg[MAXCPU][2];
#endif
static device_t cpu_list[MAXCPU];
void mpentry(unsigned long cpuid);
void init_secondary(uint64_t);
uint8_t secondary_stacks[MAXCPU][PAGE_SIZE * KSTACK_PAGES] __aligned(16);
static struct mtx ap_boot_mtx;
/* Stacks for AP initialization, discarded once idle threads are started. */
void *bootstack;
static void *bootstacks[MAXCPU];
/* Count of started APs, used to synchronize access to bootstack. */
static volatile int aps_started;
/* Set to 1 once we're ready to let the APs out of the pen. */
volatile int aps_ready = 0;
static volatile int aps_ready;
/* Temporary variables for init_secondary() */
void *dpcpu[MAXCPU - 1];
@ -233,14 +238,14 @@ init_secondary(uint64_t hart)
csr_set(sie, SIE_SSIE);
csr_set(sip, SIE_SSIE);
/* Spin until the BSP releases the APs */
while (!aps_ready)
/* Signal the BSP and spin until it has released all APs. */
atomic_add_int(&aps_started, 1);
while (!atomic_load_int(&aps_ready))
__asm __volatile("wfi");
/* Initialize curthread */
KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
pcpup->pc_curthread = pcpup->pc_idlethread;
pcpup->pc_curpcb = pcpup->pc_idlethread->td_pcb;
/*
* Identify current CPU. This is necessary to setup
@ -274,6 +279,11 @@ init_secondary(uint64_t hart)
mtx_unlock_spin(&ap_boot_mtx);
/*
* Assert that smp_after_idle_runnable condition is reasonable.
*/
MPASS(PCPU_GET(curpcb) == NULL);
/* Enter the scheduler */
sched_throw(NULL);
@ -281,6 +291,24 @@ init_secondary(uint64_t hart)
/* NOTREACHED */
}
static void
smp_after_idle_runnable(void *arg __unused)
{
struct pcpu *pc;
int cpu;
for (cpu = 1; cpu < mp_ncpus; cpu++) {
if (bootstacks[cpu] != NULL) {
pc = pcpu_find(cpu);
while (atomic_load_ptr(&pc->pc_curpcb) == NULL)
cpu_spinwait();
kmem_free((vm_offset_t)bootstacks[cpu], PAGE_SIZE);
}
}
}
SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY,
smp_after_idle_runnable, NULL);
static int
ipi_handler(void *arg)
{
@ -373,6 +401,7 @@ cpu_init_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg)
struct pcpu *pcpup;
uint64_t hart;
u_int cpuid;
int naps;
/* Check if this hart supports MMU. */
if (OF_getproplen(node, "mmu-type") < 0)
@ -419,8 +448,17 @@ cpu_init_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg)
dpcpu[cpuid - 1] = (void *)kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO);
dpcpu_init(dpcpu[cpuid - 1], cpuid);
bootstacks[cpuid] = (void *)kmem_malloc(PAGE_SIZE, M_WAITOK | M_ZERO);
naps = atomic_load_int(&aps_started);
bootstack = (char *)bootstacks[cpuid] + PAGE_SIZE;
printf("Starting CPU %u (hart %lx)\n", cpuid, hart);
__riscv_boot_ap[hart] = 1;
atomic_store_32(&__riscv_boot_ap[hart], 1);
/* Wait for the AP to switch to its boot stack. */
while (atomic_load_int(&aps_started) < naps + 1)
cpu_spinwait();
CPU_SET(cpuid, &all_cpus);
CPU_SET(hart, &all_harts);