d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

amd64/i386: switch IPI handlers to C code.

Browse Source 
Move asm IPIs handlers to C code, so both Xen and native IPI handlers
share the same code.

Reviewed by: jhb
Approved by: gibbs
Sponsored by: Citrix Systems R&D

amd64/amd64/apic_vector.S:
i386/i386/apic_vector.s:
 - Remove asm coded IPI handlers and instead call the newly introduced
   C variants.

amd64/amd64/mp_machdep.c:
i386/i386/mp_machdep.c:
 - Add C coded clones to the asm IPI handlers (moved from
   x86/xen/hvm.c).

i386/include/smp.h:
amd64/include/smp.h:
 - Add prototypes for the C IPI handlers.

x86/xen/hvm.c:
 - Move the C IPI handlers to mp_machdep and call those in the Xen IPI
   handlers.

i386/xen/mp_machdep.c:
 - Add dummy IPI handlers to the i386 Xen PV port (this port doesn't
   support SMP).
This commit is contained in:
Roger Pau Monné 2014-03-11 10:03:29 +00:00
parent 0096a6594b
commit e8da1c4877
8 changed files with 322 additions and 539 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

250
sys/amd64/amd64/apic_vector.S
View File

@ -159,101 +159,26 @@ IDTVEC(xen_intr_upcall)
#define NAKE_INTR_CS 24
SUPERALIGN_TEXT
global_invltlb:
movq %cr4,%rax
andq $~0x80,%rax /* PGE */
movq %rax,%cr4
orq $0x80,%rax
movq %rax,%cr4
invltlb_ret_clear_pm_save:
movq smp_tlb_pmap,%rdx
testq %rdx,%rdx
jz invltlb_ret_rdx
testb $SEL_RPL_MASK,NAKE_INTR_CS(%rsp)
jz 1f
swapgs
1:
movl PCPU(CPUID),%eax
jz 2f
swapgs
2:
LK btcl %eax,PM_SAVE(%rdx)
SUPERALIGN_TEXT
invltlb_ret_rdx:
popq %rdx
invltlb_ret_rax:
invltlb_ret:
movq lapic, %rax
movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
LK incl smp_tlb_wait
popq %rax
POP_FRAME
jmp doreti_iret
SUPERALIGN_TEXT
IDTVEC(invltlb_pcid)
#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
PUSH_FRAME
movl PCPU(CPUID), %eax
#ifdef COUNT_XINVLTLB_HITS
incl xhits_gbl(,%rax,4)
#endif
#ifdef COUNT_IPIS
movq ipi_invltlb_counts(,%rax,8),%rax
incq (%rax)
#endif
POP_FRAME
#endif
pushq %rax
pushq %rdx
call invltlb_pcid_handler
jmp invltlb_ret
movq %cr3,%rax
movq $smp_tlb_invpcid,%rdx
cmpl $0,(%rdx)
je global_invltlb
cmpl $-1,(%rdx)
je global_invltlb
/*
* Only invalidate TLB for entries with current PCID.
*/
cmpl $0,invpcid_works
je 1f
/* Use invpcid if available. */
movl $1,%eax /* INVPCID_CTX */
/* invpcid (%rdx),%rax */
.byte 0x66,0x0f,0x38,0x82,0x02
jmp invltlb_ret_clear_pm_save
1:
/* Otherwise reload %cr3 twice. */
movq pcid_cr3,%rdx
cmpq %rax,%rdx
je 2f
movq %rdx,%cr3 /* Invalidate, bit 63 is zero. */
btsq $63,%rax
2:
movq %rax,%cr3
jmp invltlb_ret_clear_pm_save
SUPERALIGN_TEXT
IDTVEC(invltlb)
#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
PUSH_FRAME
movl PCPU(CPUID), %eax
#ifdef COUNT_XINVLTLB_HITS
incl xhits_gbl(,%rax,4)
#endif
#ifdef COUNT_IPIS
movq ipi_invltlb_counts(,%rax,8),%rax
incq (%rax)
#endif
POP_FRAME
#endif
pushq %rax
movq %cr3, %rax /* invalidate the TLB */
movq %rax, %cr3
jmp invltlb_ret_rax
call invltlb_handler
jmp invltlb_ret
/*
* Single page TLB shootdown
@ -261,86 +186,17 @@ IDTVEC(invltlb)
.text
SUPERALIGN_TEXT
IDTVEC(invlpg_pcid)
#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
PUSH_FRAME
movl PCPU(CPUID), %eax
#ifdef COUNT_XINVLTLB_HITS
incl xhits_pg(,%rax,4)
#endif
#ifdef COUNT_IPIS
movq ipi_invlpg_counts(,%rax,8),%rax
incq (%rax)
#endif
POP_FRAME
#endif
pushq %rax
pushq %rdx
movq $smp_tlb_invpcid,%rdx
cmpl $0,invpcid_works
jne 2f
/* kernel pmap - use invlpg to invalidate global mapping */
cmpl $0,(%rdx)
je 3f
cmpl $-1,(%rdx)
je global_invltlb
/*
* PCID supported, but INVPCID is not.
* Temporarily switch to the target address space and do INVLPG.
*/
pushq %rcx
movq %cr3,%rcx
movq pcid_cr3,%rax
cmp %rcx,%rax
je 1f
btsq $63,%rax
movq %rax,%cr3
1: movq 8(%rdx),%rax
invlpg (%rax)
btsq $63,%rcx
movq %rcx,%cr3
popq %rcx
jmp invltlb_ret_rdx
/*
* Invalidate the TLB entry using INVPCID_ADDR.
*/
2:
xorl %eax,%eax
/* invpcid (%rdx),%rax */
.byte 0x66,0x0f,0x38,0x82,0x02
jmp invltlb_ret_rdx
/*
* PCID is not supported or kernel pmap.
* Invalidate single page using INVLPG.
*/
3:
movq 8(%rdx),%rax
invlpg (%rax)
jmp invltlb_ret_rdx
call invlpg_pcid_handler
jmp invltlb_ret
SUPERALIGN_TEXT
IDTVEC(invlpg)
#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
PUSH_FRAME
movl PCPU(CPUID), %eax
#ifdef COUNT_XINVLTLB_HITS
incl xhits_pg(,%rax,4)
#endif
#ifdef COUNT_IPIS
movq ipi_invlpg_counts(,%rax,8),%rax
incq (%rax)
#endif
POP_FRAME
#endif
pushq %rax
movq smp_tlb_invpcid+8,%rax
invlpg (%rax) /* invalidate single page */
jmp invltlb_ret_rax
call invlpg_handler
jmp invltlb_ret
/*
* Page range TLB shootdown.
@ -348,81 +204,10 @@ IDTVEC(invlpg)
.text
SUPERALIGN_TEXT
IDTVEC(invlrng)
#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
PUSH_FRAME
movl PCPU(CPUID), %eax
#ifdef COUNT_XINVLTLB_HITS
incl xhits_rng(,%rax,4)
#endif
#ifdef COUNT_IPIS
movq ipi_invlrng_counts(,%rax,8),%rax
incq (%rax)
#endif
POP_FRAME
#endif
pushq %rax
pushq %rdx
movq $smp_tlb_invpcid,%rdx
cmpl $0,pmap_pcid_enabled
je invlrng_single_page
/* kernel pmap - use invlpg to invalidate global mapping */
cmpl $0,(%rdx)
je invlrng_single_page
cmpl $-1,(%rdx)
je global_invltlb
cmpl $0,invpcid_works
jne invlrng_invpcid
pushq %rcx
movq %cr3,%rcx
movq pcid_cr3,%rax
cmpq %rcx,%rax
je 1f
btsq $63,%rax
movq %rax,%cr3
1:
movq 8(%rdx),%rdx
movq smp_tlb_addr2,%rax
2:
invlpg (%rdx)
addq $PAGE_SIZE,%rdx
cmpq %rax,%rdx
jb 2b
btsq $63,%rcx
movq %rcx,%cr3
popq %rcx
jmp invltlb_ret_rdx
invlrng_invpcid:
pushq %rcx
subq $16,%rsp
movq (%rdx),%rcx
movq %rcx,(%rsp)
movq 8(%rdx),%rax
movq %rax,8(%rsp)
movq smp_tlb_addr2,%rcx
subq %rax,%rcx
shrq $PAGE_SHIFT,%rcx
1:
// invpcid (%rdx),%rax
.byte 0x66,0x0f,0x38,0x82,0x02
addq $PAGE_SIZE,8(%rsp)
dec %rcx
jne 1b
addq $16,%rsp
popq %rcx
jmp invltlb_ret_rdx
invlrng_single_page:
movq 8(%rdx),%rdx
movq smp_tlb_addr2,%rax
1: invlpg (%rdx) /* invalidate single page */
addq $PAGE_SIZE,%rdx
cmpq %rax,%rdx
jb 1b
jmp invltlb_ret_rdx
call invlrng_handler
jmp invltlb_ret
/*
* Invalidate cache.
@ -430,17 +215,10 @@ invlrng_single_page:
.text
SUPERALIGN_TEXT
IDTVEC(invlcache)
#ifdef COUNT_IPIS
PUSH_FRAME
movl PCPU(CPUID), %eax
movq ipi_invlcache_counts(,%rax,8),%rax
incq (%rax)
POP_FRAME
#endif
pushq %rax
wbinvd
jmp invltlb_ret_rax
call invlcache_handler
jmp invltlb_ret
/*
* Handler for IPIs sent via the per-cpu IPI bitmap.

170
sys/amd64/amd64/mp_machdep.c
View File

@ -109,6 +109,7 @@ struct invpcid_descr smp_tlb_invpcid;
volatile int smp_tlb_wait;
uint64_t pcid_cr3;
pmap_t smp_tlb_pmap;
extern int invpcid_works;
#ifdef COUNT_IPIS
/* Interrupt counts. */
@ -1495,6 +1496,175 @@ cpususpend_handler(void)
CPU_CLR_ATOMIC(cpu, &suspended_cpus);
}
/*
* Handlers for TLB related IPIs
*/
void
invltlb_handler(void)
{
#ifdef COUNT_XINVLTLB_HITS
xhits_gbl[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
invltlb();
atomic_add_int(&smp_tlb_wait, 1);
}
void
invltlb_pcid_handler(void)
{
uint64_t cr3;
#ifdef COUNT_XINVLTLB_HITS
xhits_gbl[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
cr3 = rcr3();
if (smp_tlb_invpcid.pcid != (uint64_t)-1 &&
smp_tlb_invpcid.pcid != 0) {
if (invpcid_works) {
invpcid(&smp_tlb_invpcid, INVPCID_CTX);
} else {
/* Otherwise reload %cr3 twice. */
if (cr3 != pcid_cr3) {
load_cr3(pcid_cr3);
cr3 |= CR3_PCID_SAVE;
}
load_cr3(cr3);
}
} else {
invltlb_globpcid();
}
if (smp_tlb_pmap != NULL)
CPU_CLR_ATOMIC(PCPU_GET(cpuid), &smp_tlb_pmap->pm_save);
atomic_add_int(&smp_tlb_wait, 1);
}
void
invlpg_handler(void)
{
#ifdef COUNT_XINVLTLB_HITS
xhits_pg[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invlpg_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
invlpg(smp_tlb_invpcid.addr);
atomic_add_int(&smp_tlb_wait, 1);
}
void
invlpg_pcid_handler(void)
{
#ifdef COUNT_XINVLTLB_HITS
xhits_pg[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invlpg_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
if (invpcid_works) {
invpcid(&smp_tlb_invpcid, INVPCID_ADDR);
} else if (smp_tlb_invpcid.pcid == 0) {
invlpg(smp_tlb_invpcid.addr);
} else if (smp_tlb_invpcid.pcid == (uint64_t)-1) {
invltlb_globpcid();
} else {
uint64_t cr3;
/*
* PCID supported, but INVPCID is not.
* Temporarily switch to the target address
* space and do INVLPG.
*/
cr3 = rcr3();
if (cr3 != pcid_cr3)
load_cr3(pcid_cr3 | CR3_PCID_SAVE);
invlpg(smp_tlb_invpcid.addr);
load_cr3(cr3 | CR3_PCID_SAVE);
}
atomic_add_int(&smp_tlb_wait, 1);
}
static inline void
invlpg_range(vm_offset_t start, vm_offset_t end)
{
do {
invlpg(start);
start += PAGE_SIZE;
} while (start < end);
}
void
invlrng_handler(void)
{
vm_offset_t addr;
#ifdef COUNT_XINVLTLB_HITS
xhits_rng[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invlrng_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
addr = smp_tlb_invpcid.addr;
if (pmap_pcid_enabled) {
if (invpcid_works) {
struct invpcid_descr d;
d = smp_tlb_invpcid;
do {
invpcid(&d, INVPCID_ADDR);
d.addr += PAGE_SIZE;
} while (d.addr < smp_tlb_addr2);
} else if (smp_tlb_invpcid.pcid == 0) {
/*
* kernel pmap - use invlpg to invalidate
* global mapping.
*/
invlpg_range(addr, smp_tlb_addr2);
} else if (smp_tlb_invpcid.pcid == (uint64_t)-1) {
invltlb_globpcid();
if (smp_tlb_pmap != NULL) {
CPU_CLR_ATOMIC(PCPU_GET(cpuid),
&smp_tlb_pmap->pm_save);
}
} else {
uint64_t cr3;
cr3 = rcr3();
if (cr3 != pcid_cr3)
load_cr3(pcid_cr3 | CR3_PCID_SAVE);
invlpg_range(addr, smp_tlb_addr2);
load_cr3(cr3 | CR3_PCID_SAVE);
}
} else {
invlpg_range(addr, smp_tlb_addr2);
}
atomic_add_int(&smp_tlb_wait, 1);
}
void
invlcache_handler(void)
{
#ifdef COUNT_IPIS
(*ipi_invlcache_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
wbinvd();
atomic_add_int(&smp_tlb_wait, 1);
}
/*
* This is called once the rest of the system is up and running and we're
* ready to let the AP's out of the pen.

6
sys/amd64/include/smp.h
View File

@ -62,6 +62,12 @@ struct pmap;
void cpu_add(u_int apic_id, char boot_cpu);
void cpustop_handler(void);
void cpususpend_handler(void);
void invltlb_handler(void);
void invltlb_pcid_handler(void);
void invlpg_handler(void);
void invlpg_pcid_handler(void);
void invlrng_handler(void);
void invlcache_handler(void);
void init_secondary(void);
void ipi_startup(int apic_id, int vector);
void ipi_all_but_self(u_int ipi);

147
sys/i386/i386/apic_vector.s
View File

@ -163,78 +163,35 @@ IDTVEC(xen_intr_upcall)
*/
.text
SUPERALIGN_TEXT
IDTVEC(invltlb)
pushl %eax
pushl %ds
movl $KDSEL, %eax /* Kernel data selector */
movl %eax, %ds
#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
pushl %fs
movl $KPSEL, %eax /* Private space selector */
movl %eax, %fs
movl PCPU(CPUID), %eax
popl %fs
#ifdef COUNT_XINVLTLB_HITS
incl xhits_gbl(,%eax,4)
#endif
#ifdef COUNT_IPIS
movl ipi_invltlb_counts(,%eax,4),%eax
incl (%eax)
#endif
#endif
movl %cr3, %eax /* invalidate the TLB */
movl %eax, %cr3
invltlb_ret:
movl lapic, %eax
movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
lock
incl smp_tlb_wait
popl %ds
popl %eax
POP_FRAME
iret
SUPERALIGN_TEXT
IDTVEC(invltlb)
PUSH_FRAME
SET_KERNEL_SREGS
cld
call invltlb_handler
jmp invltlb_ret
/*
* Single page TLB shootdown
*/
.text
SUPERALIGN_TEXT
IDTVEC(invlpg)
pushl %eax
pushl %ds
movl $KDSEL, %eax /* Kernel data selector */
movl %eax, %ds
PUSH_FRAME
SET_KERNEL_SREGS
cld
#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
pushl %fs
movl $KPSEL, %eax /* Private space selector */
movl %eax, %fs
movl PCPU(CPUID), %eax
popl %fs
#ifdef COUNT_XINVLTLB_HITS
incl xhits_pg(,%eax,4)
#endif
#ifdef COUNT_IPIS
movl ipi_invlpg_counts(,%eax,4),%eax
incl (%eax)
#endif
#endif
call invlpg_handler
movl smp_tlb_addr1, %eax
invlpg (%eax) /* invalidate single page */
movl lapic, %eax
movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
lock
incl smp_tlb_wait
popl %ds
popl %eax
iret
jmp invltlb_ret
/*
* Page range TLB shootdown.
@ -242,44 +199,13 @@ IDTVEC(invlpg)
.text
SUPERALIGN_TEXT
IDTVEC(invlrng)
pushl %eax
pushl %edx
pushl %ds
movl $KDSEL, %eax /* Kernel data selector */
movl %eax, %ds
PUSH_FRAME
SET_KERNEL_SREGS
cld
#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
pushl %fs
movl $KPSEL, %eax /* Private space selector */
movl %eax, %fs
movl PCPU(CPUID), %eax
popl %fs
#ifdef COUNT_XINVLTLB_HITS
incl xhits_rng(,%eax,4)
#endif
#ifdef COUNT_IPIS
movl ipi_invlrng_counts(,%eax,4),%eax
incl (%eax)
#endif
#endif
call invlrng_handler
movl smp_tlb_addr1, %edx
movl smp_tlb_addr2, %eax
1: invlpg (%edx) /* invalidate single page */
addl $PAGE_SIZE, %edx
cmpl %eax, %edx
jb 1b
movl lapic, %eax
movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
lock
incl smp_tlb_wait
popl %ds
popl %edx
popl %eax
iret
jmp invltlb_ret
/*
* Invalidate cache.
@ -287,32 +213,13 @@ IDTVEC(invlrng)
.text
SUPERALIGN_TEXT
IDTVEC(invlcache)
pushl %eax
pushl %ds
movl $KDSEL, %eax /* Kernel data selector */
movl %eax, %ds
PUSH_FRAME
SET_KERNEL_SREGS
cld
#ifdef COUNT_IPIS
pushl %fs
movl $KPSEL, %eax /* Private space selector */
movl %eax, %fs
movl PCPU(CPUID), %eax
popl %fs
movl ipi_invlcache_counts(,%eax,4),%eax
incl (%eax)
#endif
call invlcache_handler
wbinvd
movl lapic, %eax
movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */
lock
incl smp_tlb_wait
popl %ds
popl %eax
iret
jmp invltlb_ret
/*
* Handler for IPIs sent via the per-cpu IPI bitmap.

66
sys/i386/i386/mp_machdep.c
View File

@ -1551,6 +1551,72 @@ cpususpend_handler(void)
CPU_CLR_ATOMIC(cpu, &suspended_cpus);
CPU_CLR_ATOMIC(cpu, &started_cpus);
}
/*
* Handlers for TLB related IPIs
*/
void
invltlb_handler(void)
{
uint64_t cr3;
#ifdef COUNT_XINVLTLB_HITS
xhits_gbl[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
cr3 = rcr3();
load_cr3(cr3);
atomic_add_int(&smp_tlb_wait, 1);
}
void
invlpg_handler(void)
{
#ifdef COUNT_XINVLTLB_HITS
xhits_pg[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invlpg_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
invlpg(smp_tlb_addr1);
atomic_add_int(&smp_tlb_wait, 1);
}
void
invlrng_handler(void)
{
vm_offset_t addr;
#ifdef COUNT_XINVLTLB_HITS
xhits_rng[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invlrng_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
addr = smp_tlb_addr1;
do {
invlpg(addr);
addr += PAGE_SIZE;
} while (addr < smp_tlb_addr2);
atomic_add_int(&smp_tlb_wait, 1);
}
void
invlcache_handler(void)
{
#ifdef COUNT_IPIS
(*ipi_invlcache_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
wbinvd();
atomic_add_int(&smp_tlb_wait, 1);
}
/*
* This is called once the rest of the system is up and running and we're
* ready to let the AP's out of the pen.

4
sys/i386/include/smp.h
View File

@ -63,6 +63,10 @@ void cpustop_handler(void);
#ifndef XEN
void cpususpend_handler(void);
#endif
void invltlb_handler(void);
void invlpg_handler(void);
void invlrng_handler(void);
void invlcache_handler(void);
void init_secondary(void);
void ipi_startup(int apic_id, int vector);
void ipi_all_but_self(u_int ipi);

25
sys/i386/xen/mp_machdep.c
View File

@ -1275,6 +1275,31 @@ cpustop_handler(void)
}
}
/*
* Handlers for TLB related IPIs
*
* On i386 Xen PV this are no-ops since this port doesn't support SMP.
*/
void
invltlb_handler(void)
{
}
void
invlpg_handler(void)
{
}
void
invlrng_handler(void)
{
}
void
invlcache_handler(void)
{
}
/*
* This is called once the rest of the system is up and running and we're
* ready to let the AP's out of the pen.

193
sys/x86/xen/hvm.c
View File

@ -77,22 +77,12 @@ static void xen_hvm_cpu_resume(void);
static void xen_hvm_cpu_init(void);
/*---------------------------- Extern Declarations ---------------------------*/
/* Variables used by mp_machdep to perform the MMU related IPIs */
extern volatile int smp_tlb_wait;
extern vm_offset_t smp_tlb_addr2;
#ifdef __i386__
extern vm_offset_t smp_tlb_addr1;
#else
extern struct invpcid_descr smp_tlb_invpcid;
extern uint64_t pcid_cr3;
extern int invpcid_works;
extern int pmap_pcid_enabled;
extern pmap_t smp_tlb_pmap;
#endif
#ifdef __i386__
extern void pmap_lazyfix_action(void);
#endif
#ifdef __amd64__
extern int pmap_pcid_enabled;
#endif
/* Variables used by mp_machdep to perform the bitmap IPI */
extern volatile u_int cpu_ipi_pending[MAXCPU];
@ -179,10 +169,7 @@ static int
xen_smp_rendezvous_action(void *arg)
{
#ifdef COUNT_IPIS
int cpu;
cpu = PCPU_GET(cpuid);
(*ipi_rendezvous_counts[cpu])++;
(*ipi_rendezvous_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
smp_rendezvous_action();
@ -192,20 +179,8 @@ xen_smp_rendezvous_action(void *arg)
static int
xen_invltlb(void *arg)
{
#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
int cpu;
cpu = PCPU_GET(cpuid);
#ifdef COUNT_XINVLTLB_HITS
xhits_gbl[cpu]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invltlb_counts[cpu])++;
#endif /* COUNT_IPIS */
#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */
invltlb();
atomic_add_int(&smp_tlb_wait, 1);
invltlb_handler();
return (FILTER_HANDLED);
}
@ -213,40 +188,8 @@ xen_invltlb(void *arg)
static int
xen_invltlb_pcid(void *arg)
{
uint64_t cr3;
#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
int cpu;
cpu = PCPU_GET(cpuid);
#ifdef COUNT_XINVLTLB_HITS
xhits_gbl[cpu]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invltlb_counts[cpu])++;
#endif /* COUNT_IPIS */
#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */
cr3 = rcr3();
if (smp_tlb_invpcid.pcid != (uint64_t)-1 &&
smp_tlb_invpcid.pcid != 0) {
if (invpcid_works) {
invpcid(&smp_tlb_invpcid, INVPCID_CTX);
} else {
/* Otherwise reload %cr3 twice. */
if (cr3 != pcid_cr3) {
load_cr3(pcid_cr3);
cr3 |= CR3_PCID_SAVE;
}
load_cr3(cr3);
}
} else {
invltlb_globpcid();
}
if (smp_tlb_pmap != NULL)
CPU_CLR_ATOMIC(PCPU_GET(cpuid), &smp_tlb_pmap->pm_save);
atomic_add_int(&smp_tlb_wait, 1);
invltlb_pcid_handler();
return (FILTER_HANDLED);
}
#endif
@ -254,24 +197,8 @@ xen_invltlb_pcid(void *arg)
static int
xen_invlpg(void *arg)
{
#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
int cpu;
cpu = PCPU_GET(cpuid);
#ifdef COUNT_XINVLTLB_HITS
xhits_pg[cpu]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invlpg_counts[cpu])++;
#endif /* COUNT_IPIS */
#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */
#ifdef __i386__
invlpg(smp_tlb_addr1);
#else
invlpg(smp_tlb_invpcid.addr);
#endif
atomic_add_int(&smp_tlb_wait, 1);
invlpg_handler();
return (FILTER_HANDLED);
}
@ -279,125 +206,25 @@ xen_invlpg(void *arg)
static int
xen_invlpg_pcid(void *arg)
{
#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
int cpu;
cpu = PCPU_GET(cpuid);
#ifdef COUNT_XINVLTLB_HITS
xhits_pg[cpu]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invlpg_counts[cpu])++;
#endif /* COUNT_IPIS */
#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */
if (invpcid_works) {
invpcid(&smp_tlb_invpcid, INVPCID_ADDR);
} else if (smp_tlb_invpcid.pcid == 0) {
invlpg(smp_tlb_invpcid.addr);
} else if (smp_tlb_invpcid.pcid == (uint64_t)-1) {
invltlb_globpcid();
} else {
uint64_t cr3;
/*
* PCID supported, but INVPCID is not.
* Temporarily switch to the target address
* space and do INVLPG.
*/
cr3 = rcr3();
if (cr3 != pcid_cr3)
load_cr3(pcid_cr3 | CR3_PCID_SAVE);
invlpg(smp_tlb_invpcid.addr);
load_cr3(cr3 | CR3_PCID_SAVE);
}
atomic_add_int(&smp_tlb_wait, 1);
invlpg_pcid_handler();
return (FILTER_HANDLED);
}
#endif
static inline void
invlpg_range(vm_offset_t start, vm_offset_t end)
{
do {
invlpg(start);
start += PAGE_SIZE;
} while (start < end);
}
static int
xen_invlrng(void *arg)
{
vm_offset_t addr;
#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
int cpu;
cpu = PCPU_GET(cpuid);
#ifdef COUNT_XINVLTLB_HITS
xhits_rng[cpu]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invlrng_counts[cpu])++;
#endif /* COUNT_IPIS */
#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */
#ifdef __i386__
addr = smp_tlb_addr1;
invlpg_range(addr, smp_tlb_addr2);
#else
addr = smp_tlb_invpcid.addr;
if (pmap_pcid_enabled) {
if (invpcid_works) {
struct invpcid_descr d;
d = smp_tlb_invpcid;
do {
invpcid(&d, INVPCID_ADDR);
d.addr += PAGE_SIZE;
} while (d.addr < smp_tlb_addr2);
} else if (smp_tlb_invpcid.pcid == 0) {
/*
* kernel pmap - use invlpg to invalidate
* global mapping.
*/
invlpg_range(addr, smp_tlb_addr2);
} else if (smp_tlb_invpcid.pcid != (uint64_t)-1) {
invltlb_globpcid();
if (smp_tlb_pmap != NULL) {
CPU_CLR_ATOMIC(PCPU_GET(cpuid),
&smp_tlb_pmap->pm_save);
}
} else {
uint64_t cr3;
cr3 = rcr3();
if (cr3 != pcid_cr3)
load_cr3(pcid_cr3 | CR3_PCID_SAVE);
invlpg_range(addr, smp_tlb_addr2);
load_cr3(cr3 | CR3_PCID_SAVE);
}
} else {
invlpg_range(addr, smp_tlb_addr2);
}
#endif
atomic_add_int(&smp_tlb_wait, 1);
invlrng_handler();
return (FILTER_HANDLED);
}
static int
xen_invlcache(void *arg)
{
#ifdef COUNT_IPIS
int cpu = PCPU_GET(cpuid);
cpu = PCPU_GET(cpuid);
(*ipi_invlcache_counts[cpu])++;
#endif /* COUNT_IPIS */
wbinvd();
atomic_add_int(&smp_tlb_wait, 1);
invlcache_handler();
return (FILTER_HANDLED);
}