Implement a facility for dynamic per-cpu variables.
- Modules and kernel code alike may use DPCPU_DEFINE(), DPCPU_GET(), DPCPU_SET(), etc. akin to the statically defined PCPU_*. Requires only one extra instruction more than PCPU_* and is virtually the same as __thread for builtin and much faster for shared objects. DPCPU variables can be initialized when defined. - Modules are supported by relocating the module's per-cpu linker set over space reserved in the kernel. Modules may fail to load if there is insufficient space available. - Track space available for modules with a one-off extent allocator. Free may block for memory to allocate space for an extent. Reviewed by: jhb, rwatson, kan, sam, grehan, marius, marcel, stas
This commit is contained in:
parent
9cf330555a
commit
50c202c592
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=194784
@ -1501,6 +1501,8 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
|
|||||||
wrmsr(MSR_KGSBASE, 0); /* User value while in the kernel */
|
wrmsr(MSR_KGSBASE, 0); /* User value while in the kernel */
|
||||||
|
|
||||||
pcpu_init(pc, 0, sizeof(struct pcpu));
|
pcpu_init(pc, 0, sizeof(struct pcpu));
|
||||||
|
dpcpu_init((void *)(physfree + KERNBASE), 0);
|
||||||
|
physfree += DPCPU_SIZE;
|
||||||
PCPU_SET(prvspace, pc);
|
PCPU_SET(prvspace, pc);
|
||||||
PCPU_SET(curthread, &thread0);
|
PCPU_SET(curthread, &thread0);
|
||||||
PCPU_SET(curpcb, thread0.td_pcb);
|
PCPU_SET(curpcb, thread0.td_pcb);
|
||||||
|
@ -93,9 +93,10 @@ static int bootAP;
|
|||||||
/* Free these after use */
|
/* Free these after use */
|
||||||
void *bootstacks[MAXCPU];
|
void *bootstacks[MAXCPU];
|
||||||
|
|
||||||
/* Temporary holder for double fault stack */
|
/* Temporary variables for init_secondary() */
|
||||||
char *doublefault_stack;
|
char *doublefault_stack;
|
||||||
char *nmi_stack;
|
char *nmi_stack;
|
||||||
|
void *dpcpu;
|
||||||
|
|
||||||
/* Hotwire a 0->4MB V==P mapping */
|
/* Hotwire a 0->4MB V==P mapping */
|
||||||
extern pt_entry_t *KPTphys;
|
extern pt_entry_t *KPTphys;
|
||||||
@ -590,6 +591,7 @@ init_secondary(void)
|
|||||||
|
|
||||||
/* prime data page for it to use */
|
/* prime data page for it to use */
|
||||||
pcpu_init(pc, cpu, sizeof(struct pcpu));
|
pcpu_init(pc, cpu, sizeof(struct pcpu));
|
||||||
|
dpcpu_init(dpcpu, cpu);
|
||||||
pc->pc_apic_id = cpu_apic_ids[cpu];
|
pc->pc_apic_id = cpu_apic_ids[cpu];
|
||||||
pc->pc_prvspace = pc;
|
pc->pc_prvspace = pc;
|
||||||
pc->pc_curthread = 0;
|
pc->pc_curthread = 0;
|
||||||
@ -885,6 +887,7 @@ start_all_aps(void)
|
|||||||
bootstacks[cpu] = (void *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE);
|
bootstacks[cpu] = (void *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE);
|
||||||
doublefault_stack = (char *)kmem_alloc(kernel_map, PAGE_SIZE);
|
doublefault_stack = (char *)kmem_alloc(kernel_map, PAGE_SIZE);
|
||||||
nmi_stack = (char *)kmem_alloc(kernel_map, PAGE_SIZE);
|
nmi_stack = (char *)kmem_alloc(kernel_map, PAGE_SIZE);
|
||||||
|
dpcpu = (void *)kmem_alloc(kernel_map, DPCPU_SIZE);
|
||||||
|
|
||||||
bootSTK = (char *)bootstacks[cpu] + KSTACK_PAGES * PAGE_SIZE - 8;
|
bootSTK = (char *)bootstacks[cpu] + KSTACK_PAGES * PAGE_SIZE - 8;
|
||||||
bootAP = cpu;
|
bootAP = cpu;
|
||||||
|
@ -149,7 +149,7 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
|
|||||||
|
|
||||||
if (local) {
|
if (local) {
|
||||||
if (rtype == R_ARM_RELATIVE) { /* A + B */
|
if (rtype == R_ARM_RELATIVE) { /* A + B */
|
||||||
addr = relocbase + addend;
|
addr = elf_relocaddr(lf, relocbase + addend);
|
||||||
if (*where != addr)
|
if (*where != addr)
|
||||||
*where = addr;
|
*where = addr;
|
||||||
}
|
}
|
||||||
|
@ -215,6 +215,7 @@ void *
|
|||||||
initarm(void *arg, void *arg2)
|
initarm(void *arg, void *arg2)
|
||||||
{
|
{
|
||||||
struct pv_addr kernel_l1pt;
|
struct pv_addr kernel_l1pt;
|
||||||
|
struct pv_addr dpcpu;
|
||||||
int loop, i;
|
int loop, i;
|
||||||
u_int l1pagetable;
|
u_int l1pagetable;
|
||||||
vm_offset_t freemempos;
|
vm_offset_t freemempos;
|
||||||
@ -264,6 +265,10 @@ initarm(void *arg, void *arg2)
|
|||||||
*/
|
*/
|
||||||
valloc_pages(systempage, 1);
|
valloc_pages(systempage, 1);
|
||||||
|
|
||||||
|
/* Allocate dynamic per-cpu area. */
|
||||||
|
valloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE);
|
||||||
|
dpcpu_init((void *)dpcpu.pv_va, 0);
|
||||||
|
|
||||||
/* Allocate stacks for all modes */
|
/* Allocate stacks for all modes */
|
||||||
valloc_pages(irqstack, IRQ_STACK_SIZE);
|
valloc_pages(irqstack, IRQ_STACK_SIZE);
|
||||||
valloc_pages(abtstack, ABT_STACK_SIZE);
|
valloc_pages(abtstack, ABT_STACK_SIZE);
|
||||||
|
@ -358,6 +358,7 @@ void *
|
|||||||
initarm(void *mdp, void *unused __unused)
|
initarm(void *mdp, void *unused __unused)
|
||||||
{
|
{
|
||||||
struct pv_addr kernel_l1pt;
|
struct pv_addr kernel_l1pt;
|
||||||
|
struct pv_addr dpcpu;
|
||||||
vm_offset_t freemempos, l2_start, lastaddr;
|
vm_offset_t freemempos, l2_start, lastaddr;
|
||||||
uint32_t memsize, l2size;
|
uint32_t memsize, l2size;
|
||||||
struct bi_mem_region *mr;
|
struct bi_mem_region *mr;
|
||||||
@ -479,6 +480,10 @@ initarm(void *mdp, void *unused __unused)
|
|||||||
*/
|
*/
|
||||||
valloc_pages(systempage, 1);
|
valloc_pages(systempage, 1);
|
||||||
|
|
||||||
|
/* Allocate dynamic per-cpu area. */
|
||||||
|
valloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE);
|
||||||
|
dpcpu_init((void *)dpcpu.pv_va, 0);
|
||||||
|
|
||||||
/* Allocate stacks for all modes */
|
/* Allocate stacks for all modes */
|
||||||
valloc_pages(irqstack, IRQ_STACK_SIZE);
|
valloc_pages(irqstack, IRQ_STACK_SIZE);
|
||||||
valloc_pages(abtstack, ABT_STACK_SIZE);
|
valloc_pages(abtstack, ABT_STACK_SIZE);
|
||||||
|
@ -209,6 +209,7 @@ initarm(void *arg, void *arg2)
|
|||||||
struct pv_addr kernel_l1pt;
|
struct pv_addr kernel_l1pt;
|
||||||
struct pv_addr md_addr;
|
struct pv_addr md_addr;
|
||||||
struct pv_addr md_bla;
|
struct pv_addr md_bla;
|
||||||
|
struct pv_addr dpcpu;
|
||||||
int loop;
|
int loop;
|
||||||
u_int l1pagetable;
|
u_int l1pagetable;
|
||||||
vm_offset_t freemempos;
|
vm_offset_t freemempos;
|
||||||
@ -268,6 +269,10 @@ initarm(void *arg, void *arg2)
|
|||||||
*/
|
*/
|
||||||
valloc_pages(systempage, 1);
|
valloc_pages(systempage, 1);
|
||||||
|
|
||||||
|
/* Allocate dynamic per-cpu area. */
|
||||||
|
valloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE);
|
||||||
|
dpcpu_init((void *)dpcpu.pv_va, 0);
|
||||||
|
|
||||||
/* Allocate stacks for all modes */
|
/* Allocate stacks for all modes */
|
||||||
valloc_pages(irqstack, IRQ_STACK_SIZE);
|
valloc_pages(irqstack, IRQ_STACK_SIZE);
|
||||||
valloc_pages(abtstack, ABT_STACK_SIZE);
|
valloc_pages(abtstack, ABT_STACK_SIZE);
|
||||||
|
@ -186,6 +186,7 @@ void *
|
|||||||
initarm(void *arg, void *arg2)
|
initarm(void *arg, void *arg2)
|
||||||
{
|
{
|
||||||
struct pv_addr kernel_l1pt;
|
struct pv_addr kernel_l1pt;
|
||||||
|
struct pv_addr dpcpu;
|
||||||
int loop, i;
|
int loop, i;
|
||||||
u_int l1pagetable;
|
u_int l1pagetable;
|
||||||
vm_offset_t freemempos;
|
vm_offset_t freemempos;
|
||||||
@ -236,6 +237,10 @@ initarm(void *arg, void *arg2)
|
|||||||
*/
|
*/
|
||||||
valloc_pages(systempage, 1);
|
valloc_pages(systempage, 1);
|
||||||
|
|
||||||
|
/* Allocate dynamic per-cpu area. */
|
||||||
|
valloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE);
|
||||||
|
dpcpu_init((void *)dpcpu.pv_va, 0);
|
||||||
|
|
||||||
/* Allocate stacks for all modes */
|
/* Allocate stacks for all modes */
|
||||||
valloc_pages(irqstack, IRQ_STACK_SIZE);
|
valloc_pages(irqstack, IRQ_STACK_SIZE);
|
||||||
valloc_pages(abtstack, ABT_STACK_SIZE);
|
valloc_pages(abtstack, ABT_STACK_SIZE);
|
||||||
|
@ -187,6 +187,7 @@ void *
|
|||||||
initarm(void *arg, void *arg2)
|
initarm(void *arg, void *arg2)
|
||||||
{
|
{
|
||||||
struct pv_addr kernel_l1pt;
|
struct pv_addr kernel_l1pt;
|
||||||
|
struct pv_addr dpcpu;
|
||||||
int loop, i;
|
int loop, i;
|
||||||
u_int l1pagetable;
|
u_int l1pagetable;
|
||||||
vm_offset_t freemempos;
|
vm_offset_t freemempos;
|
||||||
@ -236,6 +237,10 @@ initarm(void *arg, void *arg2)
|
|||||||
*/
|
*/
|
||||||
valloc_pages(systempage, 1);
|
valloc_pages(systempage, 1);
|
||||||
|
|
||||||
|
/* Allocate dynamic per-cpu area. */
|
||||||
|
valloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE);
|
||||||
|
dpcpu_init((void *)dpcpu.pv_va, 0);
|
||||||
|
|
||||||
/* Allocate stacks for all modes */
|
/* Allocate stacks for all modes */
|
||||||
valloc_pages(irqstack, IRQ_STACK_SIZE);
|
valloc_pages(irqstack, IRQ_STACK_SIZE);
|
||||||
valloc_pages(abtstack, ABT_STACK_SIZE);
|
valloc_pages(abtstack, ABT_STACK_SIZE);
|
||||||
|
@ -183,6 +183,7 @@ void *
|
|||||||
initarm(void *arg, void *arg2)
|
initarm(void *arg, void *arg2)
|
||||||
{
|
{
|
||||||
struct pv_addr kernel_l1pt;
|
struct pv_addr kernel_l1pt;
|
||||||
|
struct pv_addr dpcpu;
|
||||||
int loop, i;
|
int loop, i;
|
||||||
u_int l1pagetable;
|
u_int l1pagetable;
|
||||||
vm_offset_t freemempos;
|
vm_offset_t freemempos;
|
||||||
@ -232,6 +233,10 @@ initarm(void *arg, void *arg2)
|
|||||||
*/
|
*/
|
||||||
valloc_pages(systempage, 1);
|
valloc_pages(systempage, 1);
|
||||||
|
|
||||||
|
/* Allocate dynamic per-cpu area. */
|
||||||
|
valloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE);
|
||||||
|
dpcpu_init((void *)dpcpu.pv_va, 0);
|
||||||
|
|
||||||
/* Allocate stacks for all modes */
|
/* Allocate stacks for all modes */
|
||||||
valloc_pages(irqstack, IRQ_STACK_SIZE);
|
valloc_pages(irqstack, IRQ_STACK_SIZE);
|
||||||
valloc_pages(abtstack, ABT_STACK_SIZE);
|
valloc_pages(abtstack, ABT_STACK_SIZE);
|
||||||
|
@ -232,6 +232,7 @@ initarm(void *arg, void *arg2)
|
|||||||
#define next_chunk2(a,b) (((a) + (b)) &~ ((b)-1))
|
#define next_chunk2(a,b) (((a) + (b)) &~ ((b)-1))
|
||||||
#define next_page(a) next_chunk2(a,PAGE_SIZE)
|
#define next_page(a) next_chunk2(a,PAGE_SIZE)
|
||||||
struct pv_addr kernel_l1pt;
|
struct pv_addr kernel_l1pt;
|
||||||
|
struct pv_addr dpcpu;
|
||||||
int loop, i;
|
int loop, i;
|
||||||
u_int l1pagetable;
|
u_int l1pagetable;
|
||||||
vm_offset_t freemempos;
|
vm_offset_t freemempos;
|
||||||
@ -303,6 +304,10 @@ initarm(void *arg, void *arg2)
|
|||||||
*/
|
*/
|
||||||
valloc_pages(systempage, 1);
|
valloc_pages(systempage, 1);
|
||||||
|
|
||||||
|
/* Allocate dynamic per-cpu area. */
|
||||||
|
valloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE);
|
||||||
|
dpcpu_init((void *)dpcpu.pv_va, 0);
|
||||||
|
|
||||||
/* Allocate stacks for all modes */
|
/* Allocate stacks for all modes */
|
||||||
valloc_pages(irqstack, IRQ_STACK_SIZE);
|
valloc_pages(irqstack, IRQ_STACK_SIZE);
|
||||||
valloc_pages(abtstack, ABT_STACK_SIZE);
|
valloc_pages(abtstack, ABT_STACK_SIZE);
|
||||||
|
@ -166,6 +166,7 @@ void *
|
|||||||
initarm(void *arg, void *arg2)
|
initarm(void *arg, void *arg2)
|
||||||
{
|
{
|
||||||
struct pv_addr kernel_l1pt;
|
struct pv_addr kernel_l1pt;
|
||||||
|
struct pv_addr dpcpu;
|
||||||
int loop;
|
int loop;
|
||||||
u_int l1pagetable;
|
u_int l1pagetable;
|
||||||
vm_offset_t freemempos;
|
vm_offset_t freemempos;
|
||||||
@ -218,6 +219,10 @@ initarm(void *arg, void *arg2)
|
|||||||
*/
|
*/
|
||||||
valloc_pages(systempage, 1);
|
valloc_pages(systempage, 1);
|
||||||
|
|
||||||
|
/* Allocate dynamic per-cpu area. */
|
||||||
|
valloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE);
|
||||||
|
dpcpu_init((void *)dpcpu.pv_va, 0);
|
||||||
|
|
||||||
/* Allocate stacks for all modes */
|
/* Allocate stacks for all modes */
|
||||||
valloc_pages(irqstack, IRQ_STACK_SIZE);
|
valloc_pages(irqstack, IRQ_STACK_SIZE);
|
||||||
valloc_pages(abtstack, ABT_STACK_SIZE);
|
valloc_pages(abtstack, ABT_STACK_SIZE);
|
||||||
|
@ -149,7 +149,7 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
|
|||||||
|
|
||||||
if (local) {
|
if (local) {
|
||||||
if (rtype == R_386_RELATIVE) { /* A + B */
|
if (rtype == R_386_RELATIVE) { /* A + B */
|
||||||
addr = relocbase + addend;
|
addr = elf_relocaddr(lf, relocbase + addend);
|
||||||
if (*where != addr)
|
if (*where != addr)
|
||||||
*where = addr;
|
*where = addr;
|
||||||
}
|
}
|
||||||
|
@ -2448,7 +2448,7 @@ init386(first)
|
|||||||
int first;
|
int first;
|
||||||
{
|
{
|
||||||
unsigned long gdtmachpfn;
|
unsigned long gdtmachpfn;
|
||||||
int error, gsel_tss, metadata_missing, x;
|
int error, gsel_tss, metadata_missing, x, pa;
|
||||||
struct pcpu *pc;
|
struct pcpu *pc;
|
||||||
struct callback_register event = {
|
struct callback_register event = {
|
||||||
.type = CALLBACKTYPE_event,
|
.type = CALLBACKTYPE_event,
|
||||||
@ -2532,6 +2532,11 @@ init386(first)
|
|||||||
GSEL(GCODE_SEL, SEL_KPL), (unsigned long)failsafe_callback);
|
GSEL(GCODE_SEL, SEL_KPL), (unsigned long)failsafe_callback);
|
||||||
#endif
|
#endif
|
||||||
pcpu_init(pc, 0, sizeof(struct pcpu));
|
pcpu_init(pc, 0, sizeof(struct pcpu));
|
||||||
|
for (pa = first; pa < first + DPCPU_SIZE; pa += PAGE_SIZE)
|
||||||
|
pmap_kenter(pa + KERNBASE, pa);
|
||||||
|
dpcpu_init((void *)(first + KERNBASE), 0);
|
||||||
|
first += DPCPU_SIZE;
|
||||||
|
|
||||||
PCPU_SET(prvspace, pc);
|
PCPU_SET(prvspace, pc);
|
||||||
PCPU_SET(curthread, &thread0);
|
PCPU_SET(curthread, &thread0);
|
||||||
PCPU_SET(curpcb, thread0.td_pcb);
|
PCPU_SET(curpcb, thread0.td_pcb);
|
||||||
@ -2665,7 +2670,7 @@ init386(first)
|
|||||||
int first;
|
int first;
|
||||||
{
|
{
|
||||||
struct gate_descriptor *gdp;
|
struct gate_descriptor *gdp;
|
||||||
int gsel_tss, metadata_missing, x;
|
int gsel_tss, metadata_missing, x, pa;
|
||||||
struct pcpu *pc;
|
struct pcpu *pc;
|
||||||
|
|
||||||
thread0.td_kstack = proc0kstack;
|
thread0.td_kstack = proc0kstack;
|
||||||
@ -2718,6 +2723,10 @@ init386(first)
|
|||||||
lgdt(&r_gdt);
|
lgdt(&r_gdt);
|
||||||
|
|
||||||
pcpu_init(pc, 0, sizeof(struct pcpu));
|
pcpu_init(pc, 0, sizeof(struct pcpu));
|
||||||
|
for (pa = first; pa < first + DPCPU_SIZE; pa += PAGE_SIZE)
|
||||||
|
pmap_kenter(pa + KERNBASE, pa);
|
||||||
|
dpcpu_init((void *)(first + KERNBASE), 0);
|
||||||
|
first += DPCPU_SIZE;
|
||||||
PCPU_SET(prvspace, pc);
|
PCPU_SET(prvspace, pc);
|
||||||
PCPU_SET(curthread, &thread0);
|
PCPU_SET(curthread, &thread0);
|
||||||
PCPU_SET(curpcb, thread0.td_pcb);
|
PCPU_SET(curpcb, thread0.td_pcb);
|
||||||
|
@ -143,6 +143,7 @@ static int bootAP;
|
|||||||
|
|
||||||
/* Free these after use */
|
/* Free these after use */
|
||||||
void *bootstacks[MAXCPU];
|
void *bootstacks[MAXCPU];
|
||||||
|
static void *dpcpu;
|
||||||
|
|
||||||
/* Hotwire a 0->4MB V==P mapping */
|
/* Hotwire a 0->4MB V==P mapping */
|
||||||
extern pt_entry_t *KPTphys;
|
extern pt_entry_t *KPTphys;
|
||||||
@ -610,6 +611,7 @@ init_secondary(void)
|
|||||||
|
|
||||||
/* prime data page for it to use */
|
/* prime data page for it to use */
|
||||||
pcpu_init(pc, myid, sizeof(struct pcpu));
|
pcpu_init(pc, myid, sizeof(struct pcpu));
|
||||||
|
dpcpu_init(dpcpu, myid);
|
||||||
pc->pc_apic_id = cpu_apic_ids[myid];
|
pc->pc_apic_id = cpu_apic_ids[myid];
|
||||||
pc->pc_prvspace = pc;
|
pc->pc_prvspace = pc;
|
||||||
pc->pc_curthread = 0;
|
pc->pc_curthread = 0;
|
||||||
@ -897,8 +899,9 @@ start_all_aps(void)
|
|||||||
apic_id = cpu_apic_ids[cpu];
|
apic_id = cpu_apic_ids[cpu];
|
||||||
|
|
||||||
/* allocate and set up a boot stack data page */
|
/* allocate and set up a boot stack data page */
|
||||||
bootstacks[cpu] = (char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE);
|
bootstacks[cpu] =
|
||||||
|
(char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE);
|
||||||
|
dpcpu = (void *)kmem_alloc(kernel_map, DPCPU_SIZE);
|
||||||
/* setup a vector to our boot code */
|
/* setup a vector to our boot code */
|
||||||
*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
|
*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
|
||||||
*((volatile u_short *) WARMBOOT_SEG) = (boot_address >> 4);
|
*((volatile u_short *) WARMBOOT_SEG) = (boot_address >> 4);
|
||||||
|
@ -744,6 +744,7 @@ start_all_aps(void)
|
|||||||
/* Get per-cpu data */
|
/* Get per-cpu data */
|
||||||
pc = &__pcpu[bootAP];
|
pc = &__pcpu[bootAP];
|
||||||
pcpu_init(pc, bootAP, sizeof(struct pcpu));
|
pcpu_init(pc, bootAP, sizeof(struct pcpu));
|
||||||
|
dpcpu_init((void *)kmem_alloc(kernel_map, DPCPU_SIZE), bootAP);
|
||||||
pc->pc_apic_id = cpu_apic_ids[bootAP];
|
pc->pc_apic_id = cpu_apic_ids[bootAP];
|
||||||
pc->pc_prvspace = pc;
|
pc->pc_prvspace = pc;
|
||||||
pc->pc_curthread = 0;
|
pc->pc_curthread = 0;
|
||||||
|
@ -211,7 +211,7 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
|
|||||||
|
|
||||||
if (local) {
|
if (local) {
|
||||||
if (rtype == R_IA_64_REL64LSB)
|
if (rtype == R_IA_64_REL64LSB)
|
||||||
*where = relocbase + addend;
|
*where = elf_relocaddr(lf, relocbase + addend);
|
||||||
return (0);
|
return (0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -646,6 +646,21 @@ ia64_init(void)
|
|||||||
if (boothowto & RB_VERBOSE)
|
if (boothowto & RB_VERBOSE)
|
||||||
bootverbose = 1;
|
bootverbose = 1;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Find the beginning and end of the kernel.
|
||||||
|
*/
|
||||||
|
kernstart = trunc_page(kernel_text);
|
||||||
|
#ifdef DDB
|
||||||
|
ksym_start = bootinfo.bi_symtab;
|
||||||
|
ksym_end = bootinfo.bi_esymtab;
|
||||||
|
kernend = (vm_offset_t)round_page(ksym_end);
|
||||||
|
#else
|
||||||
|
kernend = (vm_offset_t)round_page(_end);
|
||||||
|
#endif
|
||||||
|
/* But if the bootstrap tells us otherwise, believe it! */
|
||||||
|
if (bootinfo.bi_kernend)
|
||||||
|
kernend = round_page(bootinfo.bi_kernend);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Setup the PCPU data for the bootstrap processor. It is needed
|
* Setup the PCPU data for the bootstrap processor. It is needed
|
||||||
* by printf(). Also, since printf() has critical sections, we
|
* by printf(). Also, since printf() has critical sections, we
|
||||||
@ -654,6 +669,8 @@ ia64_init(void)
|
|||||||
pcpup = &pcpu0;
|
pcpup = &pcpu0;
|
||||||
ia64_set_k4((u_int64_t)pcpup);
|
ia64_set_k4((u_int64_t)pcpup);
|
||||||
pcpu_init(pcpup, 0, sizeof(pcpu0));
|
pcpu_init(pcpup, 0, sizeof(pcpu0));
|
||||||
|
dpcpu_init((void *)kernend, 0);
|
||||||
|
kernend += DPCPU_SIZE;
|
||||||
PCPU_SET(curthread, &thread0);
|
PCPU_SET(curthread, &thread0);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@ -682,21 +699,6 @@ ia64_init(void)
|
|||||||
ia64_sal_init();
|
ia64_sal_init();
|
||||||
calculate_frequencies();
|
calculate_frequencies();
|
||||||
|
|
||||||
/*
|
|
||||||
* Find the beginning and end of the kernel.
|
|
||||||
*/
|
|
||||||
kernstart = trunc_page(kernel_text);
|
|
||||||
#ifdef DDB
|
|
||||||
ksym_start = bootinfo.bi_symtab;
|
|
||||||
ksym_end = bootinfo.bi_esymtab;
|
|
||||||
kernend = (vm_offset_t)round_page(ksym_end);
|
|
||||||
#else
|
|
||||||
kernend = (vm_offset_t)round_page(_end);
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/* But if the bootstrap tells us otherwise, believe it! */
|
|
||||||
if (bootinfo.bi_kernend)
|
|
||||||
kernend = round_page(bootinfo.bi_kernend);
|
|
||||||
if (metadata_missing)
|
if (metadata_missing)
|
||||||
printf("WARNING: loader(8) metadata is missing!\n");
|
printf("WARNING: loader(8) metadata is missing!\n");
|
||||||
|
|
||||||
|
@ -207,6 +207,7 @@ cpu_mp_add(u_int acpiid, u_int apicid, u_int apiceid)
|
|||||||
{
|
{
|
||||||
struct pcpu *pc;
|
struct pcpu *pc;
|
||||||
u_int64_t lid;
|
u_int64_t lid;
|
||||||
|
void *dpcpu;
|
||||||
|
|
||||||
/* Ignore any processor numbers outside our range */
|
/* Ignore any processor numbers outside our range */
|
||||||
if (acpiid > mp_maxid)
|
if (acpiid > mp_maxid)
|
||||||
@ -224,7 +225,9 @@ cpu_mp_add(u_int acpiid, u_int apicid, u_int apiceid)
|
|||||||
|
|
||||||
if (acpiid != 0) {
|
if (acpiid != 0) {
|
||||||
pc = (struct pcpu *)malloc(sizeof(*pc), M_SMP, M_WAITOK);
|
pc = (struct pcpu *)malloc(sizeof(*pc), M_SMP, M_WAITOK);
|
||||||
|
dpcpu = (void *)kmem_alloc(kernel_map, DPCPU_SIZE);
|
||||||
pcpu_init(pc, acpiid, sizeof(*pc));
|
pcpu_init(pc, acpiid, sizeof(*pc));
|
||||||
|
dpcpu_init(dpcpu, acpiid);
|
||||||
} else
|
} else
|
||||||
pc = pcpup;
|
pc = pcpup;
|
||||||
|
|
||||||
|
@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
|
|||||||
#include <sys/malloc.h>
|
#include <sys/malloc.h>
|
||||||
#include <sys/mutex.h>
|
#include <sys/mutex.h>
|
||||||
#include <sys/mount.h>
|
#include <sys/mount.h>
|
||||||
|
#include <sys/pcpu.h>
|
||||||
#include <sys/proc.h>
|
#include <sys/proc.h>
|
||||||
#include <sys/namei.h>
|
#include <sys/namei.h>
|
||||||
#include <sys/fcntl.h>
|
#include <sys/fcntl.h>
|
||||||
@ -107,6 +108,9 @@ typedef struct elf_file {
|
|||||||
caddr_t ctfoff; /* CTF offset table */
|
caddr_t ctfoff; /* CTF offset table */
|
||||||
caddr_t typoff; /* Type offset table */
|
caddr_t typoff; /* Type offset table */
|
||||||
long typlen; /* Number of type entries. */
|
long typlen; /* Number of type entries. */
|
||||||
|
Elf_Addr pcpu_start; /* Pre-relocation pcpu set start. */
|
||||||
|
Elf_Addr pcpu_stop; /* Pre-relocation pcpu set stop. */
|
||||||
|
Elf_Addr pcpu_base; /* Relocated pcpu set address. */
|
||||||
#ifdef GDB
|
#ifdef GDB
|
||||||
struct link_map gdb; /* hooks for gdb */
|
struct link_map gdb; /* hooks for gdb */
|
||||||
#endif
|
#endif
|
||||||
@ -474,6 +478,34 @@ parse_dynamic(elf_file_t ef)
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int
|
||||||
|
parse_dpcpu(elf_file_t ef)
|
||||||
|
{
|
||||||
|
int count;
|
||||||
|
int error;
|
||||||
|
|
||||||
|
ef->pcpu_start = 0;
|
||||||
|
ef->pcpu_stop = 0;
|
||||||
|
error = link_elf_lookup_set(&ef->lf, "pcpu", (void ***)&ef->pcpu_start,
|
||||||
|
(void ***)&ef->pcpu_stop, &count);
|
||||||
|
/* Error just means there is no pcpu set to relocate. */
|
||||||
|
if (error)
|
||||||
|
return (0);
|
||||||
|
count *= sizeof(void *);
|
||||||
|
/*
|
||||||
|
* Allocate space in the primary pcpu area. Copy in our initialization
|
||||||
|
* from the data section and then initialize all per-cpu storage from
|
||||||
|
* that.
|
||||||
|
*/
|
||||||
|
ef->pcpu_base = (Elf_Addr)(uintptr_t)dpcpu_alloc(count);
|
||||||
|
if (ef->pcpu_base == (Elf_Addr)NULL)
|
||||||
|
return (ENOSPC);
|
||||||
|
memcpy((void *)ef->pcpu_base, (void *)ef->pcpu_start, count);
|
||||||
|
dpcpu_copy((void *)ef->pcpu_base, count);
|
||||||
|
|
||||||
|
return (0);
|
||||||
|
}
|
||||||
|
|
||||||
static int
|
static int
|
||||||
link_elf_link_preload(linker_class_t cls,
|
link_elf_link_preload(linker_class_t cls,
|
||||||
const char* filename, linker_file_t *result)
|
const char* filename, linker_file_t *result)
|
||||||
@ -519,6 +551,8 @@ link_elf_link_preload(linker_class_t cls,
|
|||||||
lf->size = *(size_t *)sizeptr;
|
lf->size = *(size_t *)sizeptr;
|
||||||
|
|
||||||
error = parse_dynamic(ef);
|
error = parse_dynamic(ef);
|
||||||
|
if (error == 0)
|
||||||
|
error = parse_dpcpu(ef);
|
||||||
if (error) {
|
if (error) {
|
||||||
linker_file_unload(lf, LINKER_UNLOAD_FORCE);
|
linker_file_unload(lf, LINKER_UNLOAD_FORCE);
|
||||||
return error;
|
return error;
|
||||||
@ -801,6 +835,9 @@ link_elf_load_file(linker_class_t cls, const char* filename,
|
|||||||
error = parse_dynamic(ef);
|
error = parse_dynamic(ef);
|
||||||
if (error)
|
if (error)
|
||||||
goto out;
|
goto out;
|
||||||
|
error = parse_dpcpu(ef);
|
||||||
|
if (error)
|
||||||
|
goto out;
|
||||||
link_elf_reloc_local(lf);
|
link_elf_reloc_local(lf);
|
||||||
|
|
||||||
VOP_UNLOCK(nd.ni_vp, 0);
|
VOP_UNLOCK(nd.ni_vp, 0);
|
||||||
@ -897,11 +934,26 @@ link_elf_load_file(linker_class_t cls, const char* filename,
|
|||||||
return error;
|
return error;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Elf_Addr
|
||||||
|
elf_relocaddr(linker_file_t lf, Elf_Addr x)
|
||||||
|
{
|
||||||
|
elf_file_t ef;
|
||||||
|
|
||||||
|
ef = (elf_file_t)lf;
|
||||||
|
if (x >= ef->pcpu_start && x < ef->pcpu_stop)
|
||||||
|
return ((x - ef->pcpu_start) + ef->pcpu_base);
|
||||||
|
return (x);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
static void
|
static void
|
||||||
link_elf_unload_file(linker_file_t file)
|
link_elf_unload_file(linker_file_t file)
|
||||||
{
|
{
|
||||||
elf_file_t ef = (elf_file_t) file;
|
elf_file_t ef = (elf_file_t) file;
|
||||||
|
|
||||||
|
if (ef->pcpu_base) {
|
||||||
|
dpcpu_free((void *)ef->pcpu_base, ef->pcpu_stop - ef->pcpu_start);
|
||||||
|
}
|
||||||
#ifdef GDB
|
#ifdef GDB
|
||||||
if (ef->gdb.l_ld) {
|
if (ef->gdb.l_ld) {
|
||||||
GDB_STATE(RT_DELETE);
|
GDB_STATE(RT_DELETE);
|
||||||
|
@ -333,6 +333,20 @@ link_elf_link_preload(linker_class_t cls, const char *filename,
|
|||||||
if (ef->shstrtab && shdr[i].sh_name != 0)
|
if (ef->shstrtab && shdr[i].sh_name != 0)
|
||||||
ef->progtab[pb].name =
|
ef->progtab[pb].name =
|
||||||
ef->shstrtab + shdr[i].sh_name;
|
ef->shstrtab + shdr[i].sh_name;
|
||||||
|
if (ef->progtab[pb].name != NULL &&
|
||||||
|
!strcmp(ef->progtab[pb].name, "set_pcpu")) {
|
||||||
|
void *dpcpu;
|
||||||
|
|
||||||
|
dpcpu = dpcpu_alloc(shdr[i].sh_size);
|
||||||
|
if (dpcpu == NULL) {
|
||||||
|
error = ENOSPC;
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
memcpy(dpcpu, ef->progtab[pb].addr,
|
||||||
|
ef->progtab[pb].size);
|
||||||
|
dpcpu_copy(dpcpu, shdr[i].sh_size);
|
||||||
|
ef->progtab[pb].addr = dpcpu;
|
||||||
|
}
|
||||||
|
|
||||||
/* Update all symbol values with the offset. */
|
/* Update all symbol values with the offset. */
|
||||||
for (j = 0; j < ef->ddbsymcnt; j++) {
|
for (j = 0; j < ef->ddbsymcnt; j++) {
|
||||||
@ -712,9 +726,27 @@ link_elf_load_file(linker_class_t cls, const char *filename,
|
|||||||
alignmask = shdr[i].sh_addralign - 1;
|
alignmask = shdr[i].sh_addralign - 1;
|
||||||
mapbase += alignmask;
|
mapbase += alignmask;
|
||||||
mapbase &= ~alignmask;
|
mapbase &= ~alignmask;
|
||||||
ef->progtab[pb].addr = (void *)(uintptr_t)mapbase;
|
if (ef->shstrtab && shdr[i].sh_name != 0)
|
||||||
if (shdr[i].sh_type == SHT_PROGBITS) {
|
ef->progtab[pb].name =
|
||||||
|
ef->shstrtab + shdr[i].sh_name;
|
||||||
|
else if (shdr[i].sh_type == SHT_PROGBITS)
|
||||||
ef->progtab[pb].name = "<<PROGBITS>>";
|
ef->progtab[pb].name = "<<PROGBITS>>";
|
||||||
|
else
|
||||||
|
ef->progtab[pb].name = "<<NOBITS>>";
|
||||||
|
if (ef->progtab[pb].name != NULL &&
|
||||||
|
!strcmp(ef->progtab[pb].name, "set_pcpu"))
|
||||||
|
ef->progtab[pb].addr =
|
||||||
|
dpcpu_alloc(shdr[i].sh_size);
|
||||||
|
else
|
||||||
|
ef->progtab[pb].addr =
|
||||||
|
(void *)(uintptr_t)mapbase;
|
||||||
|
if (ef->progtab[pb].addr == NULL) {
|
||||||
|
error = ENOSPC;
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
ef->progtab[pb].size = shdr[i].sh_size;
|
||||||
|
ef->progtab[pb].sec = i;
|
||||||
|
if (shdr[i].sh_type == SHT_PROGBITS) {
|
||||||
error = vn_rdwr(UIO_READ, nd.ni_vp,
|
error = vn_rdwr(UIO_READ, nd.ni_vp,
|
||||||
ef->progtab[pb].addr,
|
ef->progtab[pb].addr,
|
||||||
shdr[i].sh_size, shdr[i].sh_offset,
|
shdr[i].sh_size, shdr[i].sh_offset,
|
||||||
@ -726,15 +758,12 @@ link_elf_load_file(linker_class_t cls, const char *filename,
|
|||||||
error = EINVAL;
|
error = EINVAL;
|
||||||
goto out;
|
goto out;
|
||||||
}
|
}
|
||||||
} else {
|
/* Initialize the per-cpu area. */
|
||||||
ef->progtab[pb].name = "<<NOBITS>>";
|
if (ef->progtab[pb].addr != (void *)mapbase)
|
||||||
|
dpcpu_copy(ef->progtab[pb].addr,
|
||||||
|
shdr[i].sh_size);
|
||||||
|
} else
|
||||||
bzero(ef->progtab[pb].addr, shdr[i].sh_size);
|
bzero(ef->progtab[pb].addr, shdr[i].sh_size);
|
||||||
}
|
|
||||||
ef->progtab[pb].size = shdr[i].sh_size;
|
|
||||||
ef->progtab[pb].sec = i;
|
|
||||||
if (ef->shstrtab && shdr[i].sh_name != 0)
|
|
||||||
ef->progtab[pb].name =
|
|
||||||
ef->shstrtab + shdr[i].sh_name;
|
|
||||||
|
|
||||||
/* Update all symbol values with the offset. */
|
/* Update all symbol values with the offset. */
|
||||||
for (j = 0; j < ef->ddbsymcnt; j++) {
|
for (j = 0; j < ef->ddbsymcnt; j++) {
|
||||||
@ -839,6 +868,17 @@ link_elf_unload_file(linker_file_t file)
|
|||||||
/* Notify MD code that a module is being unloaded. */
|
/* Notify MD code that a module is being unloaded. */
|
||||||
elf_cpu_unload_file(file);
|
elf_cpu_unload_file(file);
|
||||||
|
|
||||||
|
if (ef->progtab) {
|
||||||
|
for (i = 0; i < ef->nprogtab; i++) {
|
||||||
|
if (ef->progtab[i].size == 0)
|
||||||
|
continue;
|
||||||
|
if (ef->progtab[i].name == NULL)
|
||||||
|
continue;
|
||||||
|
if (!strcmp(ef->progtab[i].name, "set_pcpu"))
|
||||||
|
dpcpu_free(ef->progtab[i].addr,
|
||||||
|
ef->progtab[i].size);
|
||||||
|
}
|
||||||
|
}
|
||||||
if (ef->preloaded) {
|
if (ef->preloaded) {
|
||||||
if (ef->reltab)
|
if (ef->reltab)
|
||||||
free(ef->reltab, M_LINKER);
|
free(ef->reltab, M_LINKER);
|
||||||
|
@ -3,6 +3,9 @@
|
|||||||
* All rights reserved.
|
* All rights reserved.
|
||||||
* Written by: John Baldwin <jhb@FreeBSD.org>
|
* Written by: John Baldwin <jhb@FreeBSD.org>
|
||||||
*
|
*
|
||||||
|
* Copyright (c) 2009 Jeffrey Roberson <jeff@freebsd.org>
|
||||||
|
* All rights reserved.
|
||||||
|
*
|
||||||
* Redistribution and use in source and binary forms, with or without
|
* Redistribution and use in source and binary forms, with or without
|
||||||
* modification, are permitted provided that the following conditions
|
* modification, are permitted provided that the following conditions
|
||||||
* are met:
|
* are met:
|
||||||
@ -49,13 +52,28 @@ __FBSDID("$FreeBSD$");
|
|||||||
|
|
||||||
#include <sys/param.h>
|
#include <sys/param.h>
|
||||||
#include <sys/systm.h>
|
#include <sys/systm.h>
|
||||||
|
#include <sys/sysctl.h>
|
||||||
#include <sys/linker_set.h>
|
#include <sys/linker_set.h>
|
||||||
#include <sys/lock.h>
|
#include <sys/lock.h>
|
||||||
|
#include <sys/malloc.h>
|
||||||
#include <sys/pcpu.h>
|
#include <sys/pcpu.h>
|
||||||
#include <sys/proc.h>
|
#include <sys/proc.h>
|
||||||
#include <sys/smp.h>
|
#include <sys/smp.h>
|
||||||
|
#include <sys/sx.h>
|
||||||
#include <ddb/ddb.h>
|
#include <ddb/ddb.h>
|
||||||
|
|
||||||
|
MALLOC_DEFINE(M_PCPU, "Per-cpu", "Per-cpu resource accouting.");
|
||||||
|
|
||||||
|
struct dpcpu_free {
|
||||||
|
uintptr_t df_start;
|
||||||
|
int df_len;
|
||||||
|
TAILQ_ENTRY(dpcpu_free) df_link;
|
||||||
|
};
|
||||||
|
|
||||||
|
static DPCPU_DEFINE(char, modspace[DPCPU_MODMIN]);
|
||||||
|
static TAILQ_HEAD(, dpcpu_free) dpcpu_head = TAILQ_HEAD_INITIALIZER(dpcpu_head);
|
||||||
|
static struct sx dpcpu_lock;
|
||||||
|
uintptr_t dpcpu_off[MAXCPU];
|
||||||
struct pcpu *cpuid_to_pcpu[MAXCPU];
|
struct pcpu *cpuid_to_pcpu[MAXCPU];
|
||||||
struct cpuhead cpuhead = SLIST_HEAD_INITIALIZER(cpuhead);
|
struct cpuhead cpuhead = SLIST_HEAD_INITIALIZER(cpuhead);
|
||||||
|
|
||||||
@ -79,7 +97,146 @@ pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
|
|||||||
#ifdef KTR
|
#ifdef KTR
|
||||||
snprintf(pcpu->pc_name, sizeof(pcpu->pc_name), "CPU %d", cpuid);
|
snprintf(pcpu->pc_name, sizeof(pcpu->pc_name), "CPU %d", cpuid);
|
||||||
#endif
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
dpcpu_init(void *dpcpu, int cpuid)
|
||||||
|
{
|
||||||
|
struct pcpu *pcpu;
|
||||||
|
|
||||||
|
pcpu = pcpu_find(cpuid);
|
||||||
|
pcpu->pc_dynamic = (uintptr_t)dpcpu - DPCPU_START;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Initialize defaults from our linker section.
|
||||||
|
*/
|
||||||
|
memcpy(dpcpu, (void *)DPCPU_START, DPCPU_BYTES);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Place it in the global pcpu offset array.
|
||||||
|
*/
|
||||||
|
dpcpu_off[cpuid] = pcpu->pc_dynamic;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
dpcpu_startup(void *dummy __unused)
|
||||||
|
{
|
||||||
|
struct dpcpu_free *df;
|
||||||
|
|
||||||
|
df = malloc(sizeof(*df), M_PCPU, M_WAITOK | M_ZERO);
|
||||||
|
df->df_start = (uintptr_t)&DPCPU_NAME(modspace);
|
||||||
|
df->df_len = DPCPU_MODSIZE;
|
||||||
|
TAILQ_INSERT_HEAD(&dpcpu_head, df, df_link);
|
||||||
|
sx_init(&dpcpu_lock, "dpcpu alloc lock");
|
||||||
|
}
|
||||||
|
SYSINIT(dpcpu, SI_SUB_KLD, SI_ORDER_FIRST, dpcpu_startup, 0);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* First-fit extent based allocator for allocating space in the per-cpu
|
||||||
|
* region reserved for modules. This is only intended for use by the
|
||||||
|
* kernel linkers to place module linker sets.
|
||||||
|
*/
|
||||||
|
void *
|
||||||
|
dpcpu_alloc(int size)
|
||||||
|
{
|
||||||
|
struct dpcpu_free *df;
|
||||||
|
void *s;
|
||||||
|
|
||||||
|
s = NULL;
|
||||||
|
size = roundup2(size, sizeof(void *));
|
||||||
|
sx_xlock(&dpcpu_lock);
|
||||||
|
TAILQ_FOREACH(df, &dpcpu_head, df_link) {
|
||||||
|
if (df->df_len < size)
|
||||||
|
continue;
|
||||||
|
if (df->df_len == size) {
|
||||||
|
s = (void *)df->df_start;
|
||||||
|
TAILQ_REMOVE(&dpcpu_head, df, df_link);
|
||||||
|
free(df, M_PCPU);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
s = (void *)df->df_start;
|
||||||
|
df->df_len -= size;
|
||||||
|
df->df_start = df->df_start + size;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
sx_xunlock(&dpcpu_lock);
|
||||||
|
|
||||||
|
return (s);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Free dynamic per-cpu space at module unload time.
|
||||||
|
*/
|
||||||
|
void
|
||||||
|
dpcpu_free(void *s, int size)
|
||||||
|
{
|
||||||
|
struct dpcpu_free *df;
|
||||||
|
struct dpcpu_free *dn;
|
||||||
|
uintptr_t start;
|
||||||
|
uintptr_t end;
|
||||||
|
|
||||||
|
size = roundup2(size, sizeof(void *));
|
||||||
|
start = (uintptr_t)s;
|
||||||
|
end = start + size;
|
||||||
|
/*
|
||||||
|
* Free a region of space and merge it with as many neighbors as
|
||||||
|
* possible. Keeping the list sorted simplifies this operation.
|
||||||
|
*/
|
||||||
|
sx_xlock(&dpcpu_lock);
|
||||||
|
TAILQ_FOREACH(df, &dpcpu_head, df_link) {
|
||||||
|
if (df->df_start > end)
|
||||||
|
break;
|
||||||
|
/*
|
||||||
|
* If we expand at the end of an entry we may have to
|
||||||
|
* merge it with the one following it as well.
|
||||||
|
*/
|
||||||
|
if (df->df_start + df->df_len == start) {
|
||||||
|
df->df_len += size;
|
||||||
|
dn = TAILQ_NEXT(df, df_link);
|
||||||
|
if (df->df_start + df->df_len == dn->df_start) {
|
||||||
|
df->df_len += dn->df_len;
|
||||||
|
TAILQ_REMOVE(&dpcpu_head, dn, df_link);
|
||||||
|
free(dn, M_PCPU);
|
||||||
|
}
|
||||||
|
sx_xunlock(&dpcpu_lock);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (df->df_start == end) {
|
||||||
|
df->df_start = start;
|
||||||
|
df->df_len += size;
|
||||||
|
sx_xunlock(&dpcpu_lock);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dn = malloc(sizeof(*df), M_PCPU, M_WAITOK | M_ZERO);
|
||||||
|
dn->df_start = start;
|
||||||
|
dn->df_len = size;
|
||||||
|
if (df)
|
||||||
|
TAILQ_INSERT_BEFORE(df, dn, df_link);
|
||||||
|
else
|
||||||
|
TAILQ_INSERT_TAIL(&dpcpu_head, dn, df_link);
|
||||||
|
sx_xunlock(&dpcpu_lock);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Initialize the per-cpu storage from an updated linker-set region.
|
||||||
|
*/
|
||||||
|
void
|
||||||
|
dpcpu_copy(void *s, int size)
|
||||||
|
{
|
||||||
|
#ifdef SMP
|
||||||
|
uintptr_t dpcpu;
|
||||||
|
int i;
|
||||||
|
|
||||||
|
for (i = 0; i < mp_ncpus; ++i) {
|
||||||
|
dpcpu = dpcpu_off[i];
|
||||||
|
if (dpcpu == 0)
|
||||||
|
continue;
|
||||||
|
memcpy((void *)(dpcpu + (uintptr_t)s), s, size);
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
memcpy((void *)(dpcpu_off[0] + (uintptr_t)s), s, size);
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@ -91,6 +248,7 @@ pcpu_destroy(struct pcpu *pcpu)
|
|||||||
|
|
||||||
SLIST_REMOVE(&cpuhead, pcpu, pcpu, pc_allcpu);
|
SLIST_REMOVE(&cpuhead, pcpu, pcpu, pc_allcpu);
|
||||||
cpuid_to_pcpu[pcpu->pc_cpuid] = NULL;
|
cpuid_to_pcpu[pcpu->pc_cpuid] = NULL;
|
||||||
|
dpcpu_off[pcpu->pc_cpuid] = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@ -103,6 +261,48 @@ pcpu_find(u_int cpuid)
|
|||||||
return (cpuid_to_pcpu[cpuid]);
|
return (cpuid_to_pcpu[cpuid]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int
|
||||||
|
sysctl_dpcpu_quad(SYSCTL_HANDLER_ARGS)
|
||||||
|
{
|
||||||
|
int64_t count;
|
||||||
|
#ifdef SMP
|
||||||
|
uintptr_t dpcpu;
|
||||||
|
int i;
|
||||||
|
|
||||||
|
count = 0;
|
||||||
|
for (i = 0; i < mp_ncpus; ++i) {
|
||||||
|
dpcpu = dpcpu_off[i];
|
||||||
|
if (dpcpu == 0)
|
||||||
|
continue;
|
||||||
|
count += *(int64_t *)(dpcpu + (uintptr_t)arg1);
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
count = *(int64_t *)(dpcpu_off[0] + (uintptr_t)arg1);
|
||||||
|
#endif
|
||||||
|
return (SYSCTL_OUT(req, &count, sizeof(count)));
|
||||||
|
}
|
||||||
|
|
||||||
|
int
|
||||||
|
sysctl_dpcpu_int(SYSCTL_HANDLER_ARGS)
|
||||||
|
{
|
||||||
|
int count;
|
||||||
|
#ifdef SMP
|
||||||
|
uintptr_t dpcpu;
|
||||||
|
int i;
|
||||||
|
|
||||||
|
count = 0;
|
||||||
|
for (i = 0; i < mp_ncpus; ++i) {
|
||||||
|
dpcpu = dpcpu_off[i];
|
||||||
|
if (dpcpu == 0)
|
||||||
|
continue;
|
||||||
|
count += *(int *)(dpcpu + (uintptr_t)arg1);
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
count = *(int *)(dpcpu_off[0] + (uintptr_t)arg1);
|
||||||
|
#endif
|
||||||
|
return (SYSCTL_OUT(req, &count, sizeof(count)));
|
||||||
|
}
|
||||||
|
|
||||||
#ifdef DDB
|
#ifdef DDB
|
||||||
|
|
||||||
static void
|
static void
|
||||||
@ -111,6 +311,7 @@ show_pcpu(struct pcpu *pc)
|
|||||||
struct thread *td;
|
struct thread *td;
|
||||||
|
|
||||||
db_printf("cpuid = %d\n", pc->pc_cpuid);
|
db_printf("cpuid = %d\n", pc->pc_cpuid);
|
||||||
|
db_printf("dynamic pcpu = %p\n", (void *)pc->pc_dynamic);
|
||||||
db_printf("curthread = ");
|
db_printf("curthread = ");
|
||||||
td = pc->pc_curthread;
|
td = pc->pc_curthread;
|
||||||
if (td != NULL)
|
if (td != NULL)
|
||||||
|
@ -134,7 +134,7 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
|
|||||||
if (local) {
|
if (local) {
|
||||||
#if 0 /* TBD */
|
#if 0 /* TBD */
|
||||||
if (rtype == R_386_RELATIVE) { /* A + B */
|
if (rtype == R_386_RELATIVE) { /* A + B */
|
||||||
addr = relocbase + addend;
|
addr = elf_relocaddr(lf, relocbase + addend);
|
||||||
if (*where != addr)
|
if (*where != addr)
|
||||||
*where = addr;
|
*where = addr;
|
||||||
}
|
}
|
||||||
|
@ -224,12 +224,15 @@ static int
|
|||||||
smp_start_secondary(int cpuid)
|
smp_start_secondary(int cpuid)
|
||||||
{
|
{
|
||||||
struct pcpu *pcpu;
|
struct pcpu *pcpu;
|
||||||
|
void *dpcpu;
|
||||||
int i;
|
int i;
|
||||||
|
|
||||||
if (bootverbose)
|
if (bootverbose)
|
||||||
printf("smp_start_secondary: starting cpu %d\n", cpuid);
|
printf("smp_start_secondary: starting cpu %d\n", cpuid);
|
||||||
|
|
||||||
|
dpcpu = (void *)kmem_alloc(kernel_map, DPCPU_SIZE);
|
||||||
pcpu_init(&__pcpu[cpuid], cpuid, sizeof(struct pcpu));
|
pcpu_init(&__pcpu[cpuid], cpuid, sizeof(struct pcpu));
|
||||||
|
dpcpu_init(dpcpu, cpuid);
|
||||||
|
|
||||||
if (bootverbose)
|
if (bootverbose)
|
||||||
printf("smp_start_secondary: cpu %d started\n", cpuid);
|
printf("smp_start_secondary: cpu %d started\n", cpuid);
|
||||||
|
@ -331,6 +331,9 @@ pmap_bootstrap(void)
|
|||||||
msgbufp = (struct msgbuf *)pmap_steal_memory(MSGBUF_SIZE);
|
msgbufp = (struct msgbuf *)pmap_steal_memory(MSGBUF_SIZE);
|
||||||
msgbufinit(msgbufp, MSGBUF_SIZE);
|
msgbufinit(msgbufp, MSGBUF_SIZE);
|
||||||
|
|
||||||
|
/* Steal memory for the dynamic per-cpu area. */
|
||||||
|
dpcpu_init((void *)pmap_steal_memory(DPCPU_SIZE), 0);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Steal thread0 kstack.
|
* Steal thread0 kstack.
|
||||||
*/
|
*/
|
||||||
|
@ -1954,6 +1954,7 @@ init386(first)
|
|||||||
struct gate_descriptor *gdp;
|
struct gate_descriptor *gdp;
|
||||||
int gsel_tss, metadata_missing, x;
|
int gsel_tss, metadata_missing, x;
|
||||||
struct pcpu *pc;
|
struct pcpu *pc;
|
||||||
|
int pa;
|
||||||
|
|
||||||
thread0.td_kstack = proc0kstack;
|
thread0.td_kstack = proc0kstack;
|
||||||
thread0.td_pcb = (struct pcb *)
|
thread0.td_pcb = (struct pcb *)
|
||||||
@ -2010,6 +2011,11 @@ init386(first)
|
|||||||
lgdt(&r_gdt);
|
lgdt(&r_gdt);
|
||||||
|
|
||||||
pcpu_init(pc, 0, sizeof(struct pcpu));
|
pcpu_init(pc, 0, sizeof(struct pcpu));
|
||||||
|
for (pa = first; pa < first + DPCPU_SIZE; pa += PAGE_SIZE)
|
||||||
|
pmap_kenter(pa + KERNBASE, pa);
|
||||||
|
dpcpu_init((void *)(first + KERNBASE), 0);
|
||||||
|
first += DPCPU_SIZE;
|
||||||
|
|
||||||
PCPU_SET(prvspace, pc);
|
PCPU_SET(prvspace, pc);
|
||||||
PCPU_SET(curthread, &thread0);
|
PCPU_SET(curthread, &thread0);
|
||||||
PCPU_SET(curpcb, thread0.td_pcb);
|
PCPU_SET(curpcb, thread0.td_pcb);
|
||||||
|
@ -669,6 +669,7 @@ moea_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
|
|||||||
int ofw_mappings;
|
int ofw_mappings;
|
||||||
vm_size_t size, physsz, hwphyssz;
|
vm_size_t size, physsz, hwphyssz;
|
||||||
vm_offset_t pa, va, off;
|
vm_offset_t pa, va, off;
|
||||||
|
void *dpcpu;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Set up BAT0 to map the lowest 256 MB area
|
* Set up BAT0 to map the lowest 256 MB area
|
||||||
@ -938,6 +939,20 @@ moea_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
|
|||||||
pa += PAGE_SIZE;
|
pa += PAGE_SIZE;
|
||||||
va += PAGE_SIZE;
|
va += PAGE_SIZE;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Allocate virtual address space for the dynamic percpu area.
|
||||||
|
*/
|
||||||
|
pa = moea_bootstrap_alloc(DPCPU_SIZE, PAGE_SIZE);
|
||||||
|
dpcpu = (void *)virtual_avail;
|
||||||
|
va = virtual_avail;
|
||||||
|
virtual_avail += DPCPU_SIZE;
|
||||||
|
while (va < virtual_avail) {
|
||||||
|
moea_kenter(mmup, va, pa);;
|
||||||
|
pa += PAGE_SIZE;
|
||||||
|
va += PAGE_SIZE;
|
||||||
|
}
|
||||||
|
dpcpu_init(dpcpu, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -726,6 +726,7 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
|
|||||||
vm_size_t size, physsz, hwphyssz;
|
vm_size_t size, physsz, hwphyssz;
|
||||||
vm_offset_t pa, va, off;
|
vm_offset_t pa, va, off;
|
||||||
uint32_t msr;
|
uint32_t msr;
|
||||||
|
void *dpcpu;
|
||||||
|
|
||||||
/* We don't have a direct map since there is no BAT */
|
/* We don't have a direct map since there is no BAT */
|
||||||
hw_direct_map = 0;
|
hw_direct_map = 0;
|
||||||
@ -1027,6 +1028,20 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
|
|||||||
pa += PAGE_SIZE;
|
pa += PAGE_SIZE;
|
||||||
va += PAGE_SIZE;
|
va += PAGE_SIZE;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Allocate virtual address space for the dynamic percpu area.
|
||||||
|
*/
|
||||||
|
pa = moea64_bootstrap_alloc(DPCPU_SIZE, PAGE_SIZE);
|
||||||
|
dpcpu = (void *)virtual_avail;
|
||||||
|
va = virtual_avail;
|
||||||
|
virtual_avail += DPCPU_SIZE;
|
||||||
|
while (va < virtual_avail) {
|
||||||
|
moea64_kenter(mmup, va, pa);;
|
||||||
|
pa += PAGE_SIZE;
|
||||||
|
va += PAGE_SIZE;
|
||||||
|
}
|
||||||
|
dpcpu_init(dpcpu, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -963,6 +963,7 @@ mmu_booke_bootstrap(mmu_t mmu, vm_offset_t start, vm_offset_t kernelend)
|
|||||||
vm_size_t physsz, hwphyssz, kstack0_sz;
|
vm_size_t physsz, hwphyssz, kstack0_sz;
|
||||||
vm_offset_t kernel_pdir, kstack0, va;
|
vm_offset_t kernel_pdir, kstack0, va;
|
||||||
vm_paddr_t kstack0_phys;
|
vm_paddr_t kstack0_phys;
|
||||||
|
void *dpcpu;
|
||||||
pte_t *pte;
|
pte_t *pte;
|
||||||
|
|
||||||
debugf("mmu_booke_bootstrap: entered\n");
|
debugf("mmu_booke_bootstrap: entered\n");
|
||||||
@ -988,6 +989,11 @@ mmu_booke_bootstrap(mmu_t mmu, vm_offset_t start, vm_offset_t kernelend)
|
|||||||
|
|
||||||
data_end = round_page(data_end);
|
data_end = round_page(data_end);
|
||||||
|
|
||||||
|
/* Allocate the dynamic per-cpu area. */
|
||||||
|
dpcpu = (void *)data_end;
|
||||||
|
data_end += DPCPU_SIZE;
|
||||||
|
dpcpu_init(dpcpu, 0);
|
||||||
|
|
||||||
/* Allocate space for ptbl_bufs. */
|
/* Allocate space for ptbl_bufs. */
|
||||||
ptbl_bufs = (struct ptbl_buf *)data_end;
|
ptbl_bufs = (struct ptbl_buf *)data_end;
|
||||||
data_end += sizeof(struct ptbl_buf) * PTBL_BUFS;
|
data_end += sizeof(struct ptbl_buf) * PTBL_BUFS;
|
||||||
|
@ -194,7 +194,7 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
|
|||||||
break;
|
break;
|
||||||
|
|
||||||
case R_PPC_RELATIVE: /* word32 B + A */
|
case R_PPC_RELATIVE: /* word32 B + A */
|
||||||
*where = relocbase + addend;
|
*where = elf_relocaddr(lf, relocbase + addend);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
default:
|
default:
|
||||||
|
@ -146,8 +146,12 @@ cpu_mp_start(void)
|
|||||||
goto next;
|
goto next;
|
||||||
}
|
}
|
||||||
if (cpu.cr_cpuid != bsp.cr_cpuid) {
|
if (cpu.cr_cpuid != bsp.cr_cpuid) {
|
||||||
|
void *dpcpu;
|
||||||
|
|
||||||
pc = &__pcpu[cpu.cr_cpuid];
|
pc = &__pcpu[cpu.cr_cpuid];
|
||||||
|
dpcpu = (void *)kmem_alloc(kernel_map, DPCPU_SIZE);
|
||||||
pcpu_init(pc, cpu.cr_cpuid, sizeof(*pc));
|
pcpu_init(pc, cpu.cr_cpuid, sizeof(*pc));
|
||||||
|
dpcpu_init(dpcpu, cpu.cr_cpuid);
|
||||||
} else {
|
} else {
|
||||||
pc = pcpup;
|
pc = pcpup;
|
||||||
pc->pc_cpuid = bsp.cr_cpuid;
|
pc->pc_cpuid = bsp.cr_cpuid;
|
||||||
|
@ -62,6 +62,8 @@ struct pmap;
|
|||||||
|
|
||||||
#ifdef _KERNEL
|
#ifdef _KERNEL
|
||||||
|
|
||||||
|
extern void *dpcpu0;
|
||||||
|
|
||||||
struct pcb;
|
struct pcb;
|
||||||
struct pcpu;
|
struct pcpu;
|
||||||
|
|
||||||
|
@ -285,7 +285,7 @@ elf_reloc_local(linker_file_t lf, Elf_Addr relocbase, const void *data,
|
|||||||
value = rela->r_addend + (Elf_Addr)lf->address;
|
value = rela->r_addend + (Elf_Addr)lf->address;
|
||||||
where = (Elf_Addr *)((Elf_Addr)lf->address + rela->r_offset);
|
where = (Elf_Addr *)((Elf_Addr)lf->address + rela->r_offset);
|
||||||
|
|
||||||
*where = value;
|
*where = elf_relocaddr(lf, value);
|
||||||
|
|
||||||
return (0);
|
return (0);
|
||||||
}
|
}
|
||||||
@ -338,8 +338,9 @@ elf_reloc(linker_file_t lf, Elf_Addr relocbase, const void *data, int type,
|
|||||||
if (RELOC_PC_RELATIVE(rtype))
|
if (RELOC_PC_RELATIVE(rtype))
|
||||||
value -= (Elf_Addr)where;
|
value -= (Elf_Addr)where;
|
||||||
|
|
||||||
if (RELOC_BASE_RELATIVE(rtype))
|
if (RELOC_BASE_RELATIVE(rtype)) {
|
||||||
value += relocbase;
|
value = elf_relocaddr(lf, value + relocbase);
|
||||||
|
}
|
||||||
|
|
||||||
mask = RELOC_VALUE_BITMASK(rtype);
|
mask = RELOC_VALUE_BITMASK(rtype);
|
||||||
value >>= RELOC_VALUE_RIGHTSHIFT(rtype);
|
value >>= RELOC_VALUE_RIGHTSHIFT(rtype);
|
||||||
|
@ -124,6 +124,7 @@ int cold = 1;
|
|||||||
long Maxmem;
|
long Maxmem;
|
||||||
long realmem;
|
long realmem;
|
||||||
|
|
||||||
|
void *dpcpu0;
|
||||||
char pcpu0[PCPU_PAGES * PAGE_SIZE];
|
char pcpu0[PCPU_PAGES * PAGE_SIZE];
|
||||||
struct trapframe frame0;
|
struct trapframe frame0;
|
||||||
|
|
||||||
@ -480,8 +481,10 @@ sparc64_init(caddr_t mdp, u_long o1, u_long o2, u_long o3, ofw_vec_t *vec)
|
|||||||
delay_func = delay_tick;
|
delay_func = delay_tick;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Initialize the message buffer (after setting trap table).
|
* Initialize the dynamic per-CPU area for the BSP and the message
|
||||||
|
* buffer (after setting the trap table).
|
||||||
*/
|
*/
|
||||||
|
dpcpu_init(dpcpu0, 0);
|
||||||
msgbufinit(msgbufp, MSGBUF_SIZE);
|
msgbufinit(msgbufp, MSGBUF_SIZE);
|
||||||
|
|
||||||
mutex_init();
|
mutex_init();
|
||||||
|
@ -290,6 +290,8 @@ cpu_mp_start(void)
|
|||||||
va = kmem_alloc(kernel_map, PCPU_PAGES * PAGE_SIZE);
|
va = kmem_alloc(kernel_map, PCPU_PAGES * PAGE_SIZE);
|
||||||
pc = (struct pcpu *)(va + (PCPU_PAGES * PAGE_SIZE)) - 1;
|
pc = (struct pcpu *)(va + (PCPU_PAGES * PAGE_SIZE)) - 1;
|
||||||
pcpu_init(pc, cpuid, sizeof(*pc));
|
pcpu_init(pc, cpuid, sizeof(*pc));
|
||||||
|
dpcpu_init((void *)kmem_alloc(kernel_map, DPCPU_SIZE),
|
||||||
|
cpuid);
|
||||||
pc->pc_addr = va;
|
pc->pc_addr = va;
|
||||||
pc->pc_clock = clock;
|
pc->pc_clock = clock;
|
||||||
pc->pc_mid = mid;
|
pc->pc_mid = mid;
|
||||||
|
@ -362,6 +362,11 @@ pmap_bootstrap(vm_offset_t ekva)
|
|||||||
pmap_map_tsb();
|
pmap_map_tsb();
|
||||||
bzero(tsb_kernel, tsb_kernel_size);
|
bzero(tsb_kernel, tsb_kernel_size);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Allocate and map the dynamic per-CPU area for the BSP.
|
||||||
|
*/
|
||||||
|
dpcpu0 = (void *)TLB_PHYS_TO_DIRECT(pmap_bootstrap_alloc(DPCPU_SIZE));
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Allocate and map the message buffer.
|
* Allocate and map the message buffer.
|
||||||
*/
|
*/
|
||||||
|
@ -89,6 +89,8 @@ struct pmap;
|
|||||||
|
|
||||||
#ifdef _KERNEL
|
#ifdef _KERNEL
|
||||||
|
|
||||||
|
extern void *dpcpu0;
|
||||||
|
|
||||||
struct pcpu;
|
struct pcpu;
|
||||||
|
|
||||||
register struct pcpu *pcpup __asm__(__XSTRING(PCPU_REG));
|
register struct pcpu *pcpup __asm__(__XSTRING(PCPU_REG));
|
||||||
|
@ -129,6 +129,7 @@ int cold = 1;
|
|||||||
long Maxmem;
|
long Maxmem;
|
||||||
long realmem;
|
long realmem;
|
||||||
|
|
||||||
|
void *dpcpu0;
|
||||||
char pcpu0[PCPU_PAGES * PAGE_SIZE];
|
char pcpu0[PCPU_PAGES * PAGE_SIZE];
|
||||||
struct trapframe frame0;
|
struct trapframe frame0;
|
||||||
int trap_conversion[256];
|
int trap_conversion[256];
|
||||||
@ -500,6 +501,7 @@ sparc64_init(caddr_t mdp, u_long o1, u_long o2, u_long o3, ofw_vec_t *vec)
|
|||||||
* Initialize the message buffer (after setting trap table).
|
* Initialize the message buffer (after setting trap table).
|
||||||
*/
|
*/
|
||||||
BVPRINTF("initialize msgbuf\n");
|
BVPRINTF("initialize msgbuf\n");
|
||||||
|
dpcpu_init(dpcpu0, 0);
|
||||||
msgbufinit(msgbufp, MSGBUF_SIZE);
|
msgbufinit(msgbufp, MSGBUF_SIZE);
|
||||||
|
|
||||||
BVPRINTF("initialize mutexes\n");
|
BVPRINTF("initialize mutexes\n");
|
||||||
|
@ -324,6 +324,8 @@ cpu_mp_start(void)
|
|||||||
va = kmem_alloc(kernel_map, PCPU_PAGES * PAGE_SIZE);
|
va = kmem_alloc(kernel_map, PCPU_PAGES * PAGE_SIZE);
|
||||||
pc = (struct pcpu *)(va + (PCPU_PAGES * PAGE_SIZE)) - 1;
|
pc = (struct pcpu *)(va + (PCPU_PAGES * PAGE_SIZE)) - 1;
|
||||||
pcpu_init(pc, cpuid, sizeof(*pc));
|
pcpu_init(pc, cpuid, sizeof(*pc));
|
||||||
|
dpcpu_init((void *)kmem_alloc(kernel_map, DPCPU_SIZE),
|
||||||
|
cpuid);
|
||||||
pc->pc_addr = va;
|
pc->pc_addr = va;
|
||||||
|
|
||||||
all_cpus |= 1 << cpuid;
|
all_cpus |= 1 << cpuid;
|
||||||
|
@ -766,6 +766,11 @@ pmap_bootstrap(vm_offset_t ekva)
|
|||||||
*/
|
*/
|
||||||
mmu_fault_status_area = pmap_bootstrap_alloc(MMFSA_SIZE*MAXCPU);
|
mmu_fault_status_area = pmap_bootstrap_alloc(MMFSA_SIZE*MAXCPU);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Allocate and map the dynamic per-CPU area for the BSP.
|
||||||
|
*/
|
||||||
|
dpcpu0 = (void *)TLB_PHYS_TO_DIRECT(pmap_bootstrap_alloc(DPCPU_SIZE));
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Allocate and map the message buffer.
|
* Allocate and map the message buffer.
|
||||||
*/
|
*/
|
||||||
|
@ -264,6 +264,7 @@ typedef Elf_Addr elf_lookup_fn(linker_file_t, Elf_Size, int);
|
|||||||
/* Support functions */
|
/* Support functions */
|
||||||
int elf_reloc(linker_file_t _lf, Elf_Addr base, const void *_rel, int _type, elf_lookup_fn _lu);
|
int elf_reloc(linker_file_t _lf, Elf_Addr base, const void *_rel, int _type, elf_lookup_fn _lu);
|
||||||
int elf_reloc_local(linker_file_t _lf, Elf_Addr base, const void *_rel, int _type, elf_lookup_fn _lu);
|
int elf_reloc_local(linker_file_t _lf, Elf_Addr base, const void *_rel, int _type, elf_lookup_fn _lu);
|
||||||
|
Elf_Addr elf_relocaddr(linker_file_t _lf, Elf_Addr addr);
|
||||||
const Elf_Sym *elf_get_sym(linker_file_t _lf, Elf_Size _symidx);
|
const Elf_Sym *elf_get_sym(linker_file_t _lf, Elf_Size _symidx);
|
||||||
const char *elf_get_symname(linker_file_t _lf, Elf_Size _symidx);
|
const char *elf_get_symname(linker_file_t _lf, Elf_Size _symidx);
|
||||||
|
|
||||||
|
@ -45,6 +45,68 @@
|
|||||||
struct pcb;
|
struct pcb;
|
||||||
struct thread;
|
struct thread;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Define a set for pcpu data.
|
||||||
|
*
|
||||||
|
* We don't use SET_DECLARE because it defines the set as 'a' when we
|
||||||
|
* want 'aw'. GCC considers uninitialized data in a seperate section
|
||||||
|
* writable and there is no generic zero initializer that works for
|
||||||
|
* structs and scalars.
|
||||||
|
*/
|
||||||
|
extern uintptr_t *__start_set_pcpu;
|
||||||
|
extern uintptr_t *__stop_set_pcpu;
|
||||||
|
|
||||||
|
#if defined(__arm__)
|
||||||
|
__asm__(".section set_pcpu, \"aw\", %progbits");
|
||||||
|
#else
|
||||||
|
__asm__(".section set_pcpu, \"aw\", @progbits");
|
||||||
|
#endif
|
||||||
|
__asm__(".previous");
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Array of dynamic pcpu base offsets. Indexed by id.
|
||||||
|
*/
|
||||||
|
extern uintptr_t dpcpu_off[];
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Convenience defines.
|
||||||
|
*/
|
||||||
|
#define DPCPU_START (uintptr_t)&__start_set_pcpu
|
||||||
|
#define DPCPU_STOP (uintptr_t)&__stop_set_pcpu
|
||||||
|
#define DPCPU_BYTES (DPCPU_STOP - DPCPU_START)
|
||||||
|
#define DPCPU_MODMIN 2048
|
||||||
|
#define DPCPU_SIZE roundup2(DPCPU_BYTES, PAGE_SIZE)
|
||||||
|
#define DPCPU_MODSIZE (DPCPU_SIZE - (DPCPU_BYTES - DPCPU_MODMIN))
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Declaration and definition.
|
||||||
|
*/
|
||||||
|
#define DPCPU_NAME(n) pcpu_entry_##n
|
||||||
|
#define DPCPU_DECLARE(t, n) extern t DPCPU_NAME(n)
|
||||||
|
#define DPCPU_DEFINE(t, n) t DPCPU_NAME(n) __section("set_pcpu") __used
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Accessors with a given base.
|
||||||
|
*/
|
||||||
|
#define _DPCPU_PTR(b, n) \
|
||||||
|
(__typeof(DPCPU_NAME(n))*)((b) + (uintptr_t)&DPCPU_NAME(n))
|
||||||
|
#define _DPCPU_GET(b, n) (*_DPCPU_PTR(b, n))
|
||||||
|
#define _DPCPU_SET(b, n, v) (*_DPCPU_PTR(b, n) = v)
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Accessors for the current cpu.
|
||||||
|
*/
|
||||||
|
#define DPCPU_PTR(n) _DPCPU_PTR(PCPU_GET(dynamic), n)
|
||||||
|
#define DPCPU_GET(n) (*DPCPU_PTR(n))
|
||||||
|
#define DPCPU_SET(n, v) (*DPCPU_PTR(n) = v)
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Accessors for remote cpus.
|
||||||
|
*/
|
||||||
|
#define DPCPU_ID_PTR(i, n) _DPCPU_PTR(dpcpu_off[(i)], n)
|
||||||
|
#define DPCPU_ID_GET(i, n) (*DPCPU_ID_PTR(i, n))
|
||||||
|
#define DPCPU_ID_SET(i, n, v) (*DPCPU_ID_PTR(i, n) = v)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* XXXUPS remove as soon as we have per cpu variable
|
* XXXUPS remove as soon as we have per cpu variable
|
||||||
* linker sets and can define rm_queue in _rm_lock.h
|
* linker sets and can define rm_queue in _rm_lock.h
|
||||||
@ -92,6 +154,11 @@ struct pcpu {
|
|||||||
*/
|
*/
|
||||||
struct rm_queue pc_rm_queue;
|
struct rm_queue pc_rm_queue;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Dynamic per-cpu data area.
|
||||||
|
*/
|
||||||
|
uintptr_t pc_dynamic;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Keep MD fields last, so that CPU-specific variations on a
|
* Keep MD fields last, so that CPU-specific variations on a
|
||||||
* single architecture don't result in offset variations of
|
* single architecture don't result in offset variations of
|
||||||
@ -103,7 +170,7 @@ struct pcpu {
|
|||||||
* If only to make kernel debugging easier...
|
* If only to make kernel debugging easier...
|
||||||
*/
|
*/
|
||||||
PCPU_MD_FIELDS;
|
PCPU_MD_FIELDS;
|
||||||
};
|
} __aligned(128);
|
||||||
|
|
||||||
#ifdef _KERNEL
|
#ifdef _KERNEL
|
||||||
|
|
||||||
@ -133,6 +200,10 @@ void db_show_mdpcpu(struct pcpu *pcpu);
|
|||||||
void pcpu_destroy(struct pcpu *pcpu);
|
void pcpu_destroy(struct pcpu *pcpu);
|
||||||
struct pcpu *pcpu_find(u_int cpuid);
|
struct pcpu *pcpu_find(u_int cpuid);
|
||||||
void pcpu_init(struct pcpu *pcpu, int cpuid, size_t size);
|
void pcpu_init(struct pcpu *pcpu, int cpuid, size_t size);
|
||||||
|
void *dpcpu_alloc(int size);
|
||||||
|
void dpcpu_copy(void *s, int size);
|
||||||
|
void dpcpu_free(void *s, int size);
|
||||||
|
void dpcpu_init(void *dpcpu, int cpuid);
|
||||||
|
|
||||||
#endif /* _KERNEL */
|
#endif /* _KERNEL */
|
||||||
|
|
||||||
|
@ -178,6 +178,9 @@ int sysctl_handle_intptr(SYSCTL_HANDLER_ARGS);
|
|||||||
int sysctl_handle_string(SYSCTL_HANDLER_ARGS);
|
int sysctl_handle_string(SYSCTL_HANDLER_ARGS);
|
||||||
int sysctl_handle_opaque(SYSCTL_HANDLER_ARGS);
|
int sysctl_handle_opaque(SYSCTL_HANDLER_ARGS);
|
||||||
|
|
||||||
|
int sysctl_dpcpu_int(SYSCTL_HANDLER_ARGS);
|
||||||
|
int sysctl_dpcpu_quad(SYSCTL_HANDLER_ARGS);
|
||||||
|
|
||||||
#ifdef VIMAGE
|
#ifdef VIMAGE
|
||||||
int sysctl_handle_v_int(SYSCTL_HANDLER_ARGS);
|
int sysctl_handle_v_int(SYSCTL_HANDLER_ARGS);
|
||||||
int sysctl_handle_v_string(SYSCTL_HANDLER_ARGS);
|
int sysctl_handle_v_string(SYSCTL_HANDLER_ARGS);
|
||||||
|
Loading…
Reference in New Issue
Block a user