d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

For amd64 non-PCID machines, and for i386 machines with support for

Browse Source 
the PG_G global pte flag, pmap_invalidate_all() fails to flush global
TLB entries [*].  This is because TLB shootdown handler for such
configs reloads CR3, and on i386 pmap_invalidate_all() does the same
for the initiating CPU.  Note that current code does not issue total
invalidation requests for the kernel_pmap.

Rename amd64 function invltlb_globpcid() to invltlb_glob(), it is not
specific for PCID for quite some time, and implement the same
functionality for i386.  Use the function instead of invltlb() in
shootdown handlers and in i386 pmap_invalidate_all(), but only for the
kernel pmap (which maps pages with the PG_G attribute set), which
takes care of PG_G TLB entries on flush.

To detect the affected pmap in i386 TLB shootdown handler, pmap should
be passed to the smp_masked_invltlb() function, which makes amd64 and
i386 TLB shootdown code almost identical.  Merge the code under x86/.

Noted by:	jhb [*]
Reviewed by:	cem, jhb, pho
Tested by:	pho
Sponsored by:	The FreeBSD Foundation
Differential revision:	https://reviews.freebsd.org/D4346
This commit is contained in:
kib 2015-12-03 11:14:14 +00:00
parent 559961c499
commit f741f698b7
9 changed files with 210 additions and 402 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

157
sys/amd64/amd64/mp_machdep.c
View File

@ -87,11 +87,6 @@ extern struct pcpu __pcpu[];
char *doublefault_stack;
char *nmi_stack;
/* Variables needed for SMP tlb shootdown. */
static vm_offset_t smp_tlb_addr1, smp_tlb_addr2;
static pmap_t smp_tlb_pmap;
volatile int smp_tlb_wait;
extern inthand_t IDTVEC(fast_syscall), IDTVEC(fast_syscall32);
/*
@ -410,121 +405,6 @@ start_ap(int apic_id)
return 0; /* return FAILURE */
}
/*
* Flush the TLB on other CPU's
*/
static void
smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, pmap_t pmap,
vm_offset_t addr1, vm_offset_t addr2)
{
int cpu, ncpu, othercpus;
othercpus = mp_ncpus - 1; /* does not shootdown self */
/*
* Check for other cpus. Return if none.
*/
if (CPU_ISFULLSET(&mask)) {
if (othercpus < 1)
return;
} else {
CPU_CLR(PCPU_GET(cpuid), &mask);
if (CPU_EMPTY(&mask))
return;
}
if (!(read_rflags() & PSL_I))
panic("%s: interrupts disabled", __func__);
mtx_lock_spin(&smp_ipi_mtx);
smp_tlb_addr1 = addr1;
smp_tlb_addr2 = addr2;
smp_tlb_pmap = pmap;
smp_tlb_wait = 0;
if (CPU_ISFULLSET(&mask)) {
ncpu = othercpus;
ipi_all_but_self(vector);
} else {
ncpu = 0;
while ((cpu = CPU_FFS(&mask)) != 0) {
cpu--;
CPU_CLR(cpu, &mask);
CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__,
cpu, vector);
ipi_send_cpu(cpu, vector);
ncpu++;
}
}
while (smp_tlb_wait < ncpu)
ia32_pause();
mtx_unlock_spin(&smp_ipi_mtx);
}
void
smp_masked_invltlb(cpuset_t mask, pmap_t pmap)
{
if (smp_started) {
smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, pmap, 0, 0);
#ifdef COUNT_XINVLTLB_HITS
ipi_global++;
#endif
}
}
void
smp_masked_invlpg(cpuset_t mask, vm_offset_t addr)
{
if (smp_started) {
smp_targeted_tlb_shootdown(mask, IPI_INVLPG, NULL, addr, 0);
#ifdef COUNT_XINVLTLB_HITS
ipi_page++;
#endif
}
}
void
smp_masked_invlpg_range(cpuset_t mask, vm_offset_t addr1, vm_offset_t addr2)
{
if (smp_started) {
smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, NULL,
addr1, addr2);
#ifdef COUNT_XINVLTLB_HITS
ipi_range++;
ipi_range_size += (addr2 - addr1) / PAGE_SIZE;
#endif
}
}
void
smp_cache_flush(void)
{
if (smp_started) {
smp_targeted_tlb_shootdown(all_cpus, IPI_INVLCACHE, NULL,
0, 0);
}
}
/*
* Handlers for TLB related IPIs
*/
void
invltlb_handler(void)
{
#ifdef COUNT_XINVLTLB_HITS
xhits_gbl[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
invltlb();
atomic_add_int(&smp_tlb_wait, 1);
}
void
invltlb_invpcid_handler(void)
{
@ -556,7 +436,7 @@ invltlb_pcid_handler(void)
#endif /* COUNT_IPIS */
if (smp_tlb_pmap == kernel_pmap) {
invltlb_globpcid();
invltlb_glob();
} else {
/*
* The current pmap might not be equal to
@ -572,38 +452,3 @@ invltlb_pcid_handler(void)
}
atomic_add_int(&smp_tlb_wait, 1);
}
void
invlpg_handler(void)
{
#ifdef COUNT_XINVLTLB_HITS
xhits_pg[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invlpg_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
invlpg(smp_tlb_addr1);
atomic_add_int(&smp_tlb_wait, 1);
}
void
invlrng_handler(void)
{
vm_offset_t addr;
#ifdef COUNT_XINVLTLB_HITS
xhits_rng[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invlrng_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
addr = smp_tlb_addr1;
do {
invlpg(addr);
addr += PAGE_SIZE;
} while (addr < smp_tlb_addr2);
atomic_add_int(&smp_tlb_wait, 1);
}

6
sys/amd64/amd64/pmap.c
View File

@ -1321,7 +1321,7 @@ pmap_update_pde_invalidate(pmap_t pmap, vm_offset_t va, pd_entry_t newpde)
* Promotion: flush every 4KB page mapping from the TLB,
* including any global (PG_G) mappings.
*/
invltlb_globpcid();
invltlb_glob();
}
}
#ifdef SMP
@ -1482,7 +1482,7 @@ pmap_invalidate_all(pmap_t pmap)
bzero(&d, sizeof(d));
invpcid(&d, INVPCID_CTXGLOB);
} else {
invltlb_globpcid();
invltlb_glob();
}
mask = &all_cpus;
} else {
@ -1653,7 +1653,7 @@ pmap_invalidate_all(pmap_t pmap)
bzero(&d, sizeof(d));
invpcid(&d, INVPCID_CTXGLOB);
} else {
invltlb_globpcid();
invltlb_glob();
}
} else if (pmap == PCPU_GET(curpmap)) {
if (pmap_pcid_enabled) {

2
sys/amd64/include/cpufunc.h
View File

@ -505,7 +505,7 @@ invltlb(void)
* Operations that Invalidate TLBs and Paging-Structure Caches.
*/
static __inline void
invltlb_globpcid(void)
invltlb_glob(void)
{
uint64_t cr4;

5
sys/amd64/include/smp.h
View File

@ -25,6 +25,8 @@
#include <x86/apicvar.h>
#include <machine/pcb.h>
struct pmap;
/* global symbols in mpboot.S */
extern char mptramp_start[];
extern char mptramp_end[];
@ -53,6 +55,7 @@ extern u_int ipi_global;
extern u_int ipi_page;
extern u_int ipi_range;
extern u_int ipi_range_size;
extern struct pmap *smp_tlb_pmap;
extern volatile int smp_tlb_wait;
@ -86,8 +89,6 @@ inthand_t
IDTVEC(justreturn), /* interrupt CPU with minimum overhead */
IDTVEC(rendezvous); /* handle CPU rendezvous */
struct pmap;
/* functions in mp_machdep.c */
void assign_cpu_ids(void);
void cpu_add(u_int apic_id, char boot_cpu);

203
sys/i386/i386/mp_machdep.c
View File

@ -132,11 +132,6 @@ __FBSDID("$FreeBSD$");
extern struct pcpu __pcpu[];
/* Variables needed for SMP tlb shootdown. */
vm_offset_t smp_tlb_addr1;
vm_offset_t smp_tlb_addr2;
volatile int smp_tlb_wait;
/*
* Local data and functions.
*/
@ -487,201 +482,3 @@ start_ap(int apic_id)
}
return 0; /* return FAILURE */
}
/*
* Flush the TLB on all other CPU's
*/
static void
smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2)
{
u_int ncpu;
ncpu = mp_ncpus - 1; /* does not shootdown self */
if (ncpu < 1)
return; /* no other cpus */
if (!(read_eflags() & PSL_I))
panic("%s: interrupts disabled", __func__);
mtx_lock_spin(&smp_ipi_mtx);
smp_tlb_addr1 = addr1;
smp_tlb_addr2 = addr2;
smp_tlb_wait = 0;
ipi_all_but_self(vector);
while (smp_tlb_wait < ncpu)
ia32_pause();
mtx_unlock_spin(&smp_ipi_mtx);
}
static void
smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2)
{
int cpu, ncpu, othercpus;
othercpus = mp_ncpus - 1;
if (CPU_ISFULLSET(&mask)) {
if (othercpus < 1)
return;
} else {
CPU_CLR(PCPU_GET(cpuid), &mask);
if (CPU_EMPTY(&mask))
return;
}
if (!(read_eflags() & PSL_I))
panic("%s: interrupts disabled", __func__);
mtx_lock_spin(&smp_ipi_mtx);
smp_tlb_addr1 = addr1;
smp_tlb_addr2 = addr2;
atomic_store_rel_int(&smp_tlb_wait, 0);
if (CPU_ISFULLSET(&mask)) {
ncpu = othercpus;
ipi_all_but_self(vector);
} else {
ncpu = 0;
while ((cpu = CPU_FFS(&mask)) != 0) {
cpu--;
CPU_CLR(cpu, &mask);
CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu,
vector);
ipi_send_cpu(cpu, vector);
ncpu++;
}
}
while (smp_tlb_wait < ncpu)
ia32_pause();
mtx_unlock_spin(&smp_ipi_mtx);
}
void
smp_invltlb(void)
{
if (smp_started) {
smp_tlb_shootdown(IPI_INVLTLB, 0, 0);
#ifdef COUNT_XINVLTLB_HITS
ipi_global++;
#endif
}
}
void
smp_invlpg(vm_offset_t addr)
{
if (smp_started) {
smp_tlb_shootdown(IPI_INVLPG, addr, 0);
#ifdef COUNT_XINVLTLB_HITS
ipi_page++;
#endif
}
}
void
smp_invlpg_range(vm_offset_t addr1, vm_offset_t addr2)
{
if (smp_started) {
smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2);
#ifdef COUNT_XINVLTLB_HITS
ipi_range++;
ipi_range_size += (addr2 - addr1) / PAGE_SIZE;
#endif
}
}
void
smp_masked_invltlb(cpuset_t mask)
{
if (smp_started) {
smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0);
#ifdef COUNT_XINVLTLB_HITS
ipi_masked_global++;
#endif
}
}
void
smp_masked_invlpg(cpuset_t mask, vm_offset_t addr)
{
if (smp_started) {
smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0);
#ifdef COUNT_XINVLTLB_HITS
ipi_masked_page++;
#endif
}
}
void
smp_masked_invlpg_range(cpuset_t mask, vm_offset_t addr1, vm_offset_t addr2)
{
if (smp_started) {
smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2);
#ifdef COUNT_XINVLTLB_HITS
ipi_masked_range++;
ipi_masked_range_size += (addr2 - addr1) / PAGE_SIZE;
#endif
}
}
void
smp_cache_flush(void)
{
if (smp_started)
smp_tlb_shootdown(IPI_INVLCACHE, 0, 0);
}
/*
* Handlers for TLB related IPIs
*/
void
invltlb_handler(void)
{
uint64_t cr3;
#ifdef COUNT_XINVLTLB_HITS
xhits_gbl[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
cr3 = rcr3();
load_cr3(cr3);
atomic_add_int(&smp_tlb_wait, 1);
}
void
invlpg_handler(void)
{
#ifdef COUNT_XINVLTLB_HITS
xhits_pg[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invlpg_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
invlpg(smp_tlb_addr1);
atomic_add_int(&smp_tlb_wait, 1);
}
void
invlrng_handler(void)
{
vm_offset_t addr;
#ifdef COUNT_XINVLTLB_HITS
xhits_rng[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invlrng_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
addr = smp_tlb_addr1;
do {
invlpg(addr);
addr += PAGE_SIZE;
} while (addr < smp_tlb_addr2);
atomic_add_int(&smp_tlb_wait, 1);
}

53
sys/i386/i386/pmap.c
View File

@ -655,7 +655,7 @@ pmap_set_pg(void)
va = KERNBASE + KERNLOAD;
while (va < endva) {
pdir_pde(PTD, va) |= pgeflag;
invltlb(); /* Play it safe, invltlb() every time */
invltlb_glob(); /* Play it safe, invltlb() every time */
va += NBPDR;
}
} else {
@ -664,7 +664,7 @@ pmap_set_pg(void)
pte = vtopte(va);
if (*pte)
*pte |= pgeflag;
invltlb(); /* Play it safe, invltlb() every time */
invltlb_glob(); /* Play it safe, invltlb() every time */
va += PAGE_SIZE;
}
}
@ -973,6 +973,22 @@ pmap_update_pde_invalidate(vm_offset_t va, pd_entry_t newpde)
load_cr4(cr4 | CR4_PGE);
}
}
void
invltlb_glob(void)
{
uint64_t cr4;
if (pgeflag == 0) {
invltlb();
} else {
cr4 = rcr4();
load_cr4(cr4 & ~CR4_PGE);
load_cr4(cr4 | CR4_PGE);
}
}
#ifdef SMP
/*
* For SMP, these functions have to use the IPI mechanism for coherence.
@ -996,13 +1012,13 @@ pmap_update_pde_invalidate(vm_offset_t va, pd_entry_t newpde)
void
pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
{
cpuset_t other_cpus;
cpuset_t *mask, other_cpus;
u_int cpuid;
sched_pin();
if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
invlpg(va);
smp_invlpg(va);
mask = &all_cpus;
} else {
cpuid = PCPU_GET(cpuid);
other_cpus = all_cpus;
@ -1010,16 +1026,16 @@ pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
if (CPU_ISSET(cpuid, &pmap->pm_active))
invlpg(va);
CPU_AND(&other_cpus, &pmap->pm_active);
if (!CPU_EMPTY(&other_cpus))
smp_masked_invlpg(other_cpus, va);
mask = &other_cpus;
}
smp_masked_invlpg(*mask, va);
sched_unpin();
}
void
pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
{
cpuset_t other_cpus;
cpuset_t *mask, other_cpus;
vm_offset_t addr;
u_int cpuid;
@ -1027,7 +1043,7 @@ pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
for (addr = sva; addr < eva; addr += PAGE_SIZE)
invlpg(addr);
smp_invlpg_range(sva, eva);
mask = &all_cpus;
} else {
cpuid = PCPU_GET(cpuid);
other_cpus = all_cpus;
@ -1036,22 +1052,25 @@ pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
for (addr = sva; addr < eva; addr += PAGE_SIZE)
invlpg(addr);
CPU_AND(&other_cpus, &pmap->pm_active);
if (!CPU_EMPTY(&other_cpus))
smp_masked_invlpg_range(other_cpus, sva, eva);
mask = &other_cpus;
}
smp_masked_invlpg_range(*mask, sva, eva);
sched_unpin();
}
void
pmap_invalidate_all(pmap_t pmap)
{
cpuset_t other_cpus;
cpuset_t *mask, other_cpus;
u_int cpuid;
sched_pin();
if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
if (pmap == kernel_pmap) {
invltlb_glob();
mask = &all_cpus;
} else if (!CPU_CMP(&pmap->pm_active, &all_cpus)) {
invltlb();
smp_invltlb();
mask = &all_cpus;
} else {
cpuid = PCPU_GET(cpuid);
other_cpus = all_cpus;
@ -1059,9 +1078,9 @@ pmap_invalidate_all(pmap_t pmap)
if (CPU_ISSET(cpuid, &pmap->pm_active))
invltlb();
CPU_AND(&other_cpus, &pmap->pm_active);
if (!CPU_EMPTY(&other_cpus))
smp_masked_invltlb(other_cpus);
mask = &other_cpus;
}
smp_masked_invltlb(*mask, pmap);
sched_unpin();
}
@ -1193,7 +1212,9 @@ PMAP_INLINE void
pmap_invalidate_all(pmap_t pmap)
{
if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))
if (pmap == kernel_pmap)
invltlb_glob();
else if (!CPU_EMPTY(&pmap->pm_active))
invltlb();
}

2
sys/i386/include/pmap.h
View File

@ -394,6 +394,8 @@ void pmap_invalidate_cache_pages(vm_page_t *pages, int count);
void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva,
boolean_t force);
void invltlb_glob(void);
#endif /* _KERNEL */
#endif /* !LOCORE */

9
sys/i386/include/smp.h
View File

@ -53,10 +53,6 @@ extern u_int ipi_global;
extern u_int ipi_page;
extern u_int ipi_range;
extern u_int ipi_range_size;
extern u_int ipi_masked_global;
extern u_int ipi_masked_page;
extern u_int ipi_masked_range;
extern u_int ipi_masked_range_size;
struct cpu_info {
int cpu_present:1;
@ -105,13 +101,10 @@ void ipi_selected(cpuset_t cpus, u_int ipi);
u_int mp_bootaddress(u_int);
void set_interrupt_apic_ids(void);
void smp_cache_flush(void);
void smp_invlpg(vm_offset_t addr);
void smp_masked_invlpg(cpuset_t mask, vm_offset_t addr);
void smp_invlpg_range(vm_offset_t startva, vm_offset_t endva);
void smp_masked_invlpg_range(cpuset_t mask, vm_offset_t startva,
vm_offset_t endva);
void smp_invltlb(void);
void smp_masked_invltlb(cpuset_t mask);
void smp_masked_invltlb(cpuset_t mask, struct pmap *pmap);
void mem_range_AP_init(void);
void topo_probe(void);
void ipi_send_cpu(int cpu, u_int ipi);

175
sys/x86/x86/mp_x86.c
View File

@ -713,19 +713,6 @@ SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_page, CTLFLAG_RW, &ipi_page, 0, "");
SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range, CTLFLAG_RW, &ipi_range, 0, "");
SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range_size, CTLFLAG_RW, &ipi_range_size,
0, "");
u_int ipi_masked_global;
u_int ipi_masked_page;
u_int ipi_masked_range;
u_int ipi_masked_range_size;
SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_global, CTLFLAG_RW,
&ipi_masked_global, 0, "");
SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_page, CTLFLAG_RW,
&ipi_masked_page, 0, "");
SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range, CTLFLAG_RW,
&ipi_masked_range, 0, "");
SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range_size, CTLFLAG_RW,
&ipi_masked_range_size, 0, "");
#endif /* COUNT_XINVLTLB_HITS */
/*
@ -1090,3 +1077,165 @@ mp_ipi_intrcnt(void *dummy)
}
SYSINIT(mp_ipi_intrcnt, SI_SUB_INTR, SI_ORDER_MIDDLE, mp_ipi_intrcnt, NULL);
#endif
/*
* Flush the TLB on other CPU's
*/
/* Variables needed for SMP tlb shootdown. */
static vm_offset_t smp_tlb_addr1, smp_tlb_addr2;
pmap_t smp_tlb_pmap;
volatile int smp_tlb_wait;
#ifdef __amd64__
#define read_eflags() read_rflags()
#endif
static void
smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, pmap_t pmap,
vm_offset_t addr1, vm_offset_t addr2)
{
int cpu, ncpu, othercpus;
othercpus = mp_ncpus - 1; /* does not shootdown self */
/*
* Check for other cpus. Return if none.
*/
if (CPU_ISFULLSET(&mask)) {
if (othercpus < 1)
return;
} else {
CPU_CLR(PCPU_GET(cpuid), &mask);
if (CPU_EMPTY(&mask))
return;
}
if (!(read_eflags() & PSL_I))
panic("%s: interrupts disabled", __func__);
mtx_lock_spin(&smp_ipi_mtx);
smp_tlb_addr1 = addr1;
smp_tlb_addr2 = addr2;
smp_tlb_pmap = pmap;
smp_tlb_wait = 0;
if (CPU_ISFULLSET(&mask)) {
ncpu = othercpus;
ipi_all_but_self(vector);
} else {
ncpu = 0;
while ((cpu = CPU_FFS(&mask)) != 0) {
cpu--;
CPU_CLR(cpu, &mask);
CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__,
cpu, vector);
ipi_send_cpu(cpu, vector);
ncpu++;
}
}
while (smp_tlb_wait < ncpu)
ia32_pause();
mtx_unlock_spin(&smp_ipi_mtx);
}
void
smp_masked_invltlb(cpuset_t mask, pmap_t pmap)
{
if (smp_started) {
smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, pmap, 0, 0);
#ifdef COUNT_XINVLTLB_HITS
ipi_global++;
#endif
}
}
void
smp_masked_invlpg(cpuset_t mask, vm_offset_t addr)
{
if (smp_started) {
smp_targeted_tlb_shootdown(mask, IPI_INVLPG, NULL, addr, 0);
#ifdef COUNT_XINVLTLB_HITS
ipi_page++;
#endif
}
}
void
smp_masked_invlpg_range(cpuset_t mask, vm_offset_t addr1, vm_offset_t addr2)
{
if (smp_started) {
smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, NULL,
addr1, addr2);
#ifdef COUNT_XINVLTLB_HITS
ipi_range++;
ipi_range_size += (addr2 - addr1) / PAGE_SIZE;
#endif
}
}
void
smp_cache_flush(void)
{
if (smp_started) {
smp_targeted_tlb_shootdown(all_cpus, IPI_INVLCACHE, NULL,
0, 0);
}
}
/*
* Handlers for TLB related IPIs
*/
void
invltlb_handler(void)
{
#ifdef COUNT_XINVLTLB_HITS
xhits_gbl[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
if (smp_tlb_pmap == kernel_pmap)
invltlb_glob();
else
invltlb();
atomic_add_int(&smp_tlb_wait, 1);
}
void
invlpg_handler(void)
{
#ifdef COUNT_XINVLTLB_HITS
xhits_pg[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invlpg_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
invlpg(smp_tlb_addr1);
atomic_add_int(&smp_tlb_wait, 1);
}
void
invlrng_handler(void)
{
vm_offset_t addr;
#ifdef COUNT_XINVLTLB_HITS
xhits_rng[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invlrng_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
addr = smp_tlb_addr1;
do {
invlpg(addr);
addr += PAGE_SIZE;
} while (addr < smp_tlb_addr2);
atomic_add_int(&smp_tlb_wait, 1);
}