d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Coalesce TLB shootdowns of global PTEs in pmap_advise() on x86.

Browse Source 
We would previously invalidate such entries individually, resulting in more
IPIs than necessary.

Reviewed by:	alc, kib
MFC after:	3 weeks
Differential Revision:	https://reviews.freebsd.org/D9094
This commit is contained in:
Mark Johnston 2017-01-10 21:52:48 +00:00
parent 90e17792c8
commit bd7abab0c9
2 changed files with 35 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
sys/amd64/amd64/pmap.c
View File

@ -6010,7 +6010,7 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
pdp_entry_t *pdpe;
pd_entry_t oldpde, *pde;
pt_entry_t *pte, PG_A, PG_G, PG_M, PG_RW, PG_V;
vm_offset_t va_next;
vm_offset_t va, va_next;
vm_page_t m;
boolean_t anychanged;
@ -6090,11 +6090,11 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
}
if (va_next > eva)
va_next = eva;
va = va_next;
for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
sva += PAGE_SIZE) {
if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED |
PG_V))
continue;
if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V))
goto maybe_invlrng;
else if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
if (advice == MADV_DONTNEED) {
/*
@ -6109,12 +6109,22 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
} else if ((*pte & PG_A) != 0)
atomic_clear_long(pte, PG_A);
else
continue;
if ((*pte & PG_G) != 0)
pmap_invalidate_page(pmap, sva);
else
goto maybe_invlrng;
if ((*pte & PG_G) != 0) {
if (va == va_next)
va = sva;
} else
anychanged = TRUE;
continue;
maybe_invlrng:
if (va != va_next) {
pmap_invalidate_range(pmap, va, sva);
va = va_next;
}
}
if (va != va_next)
pmap_invalidate_range(pmap, va, sva);
}
if (anychanged)
pmap_invalidate_all(pmap);

25
sys/i386/i386/pmap.c
View File

@ -4905,7 +4905,7 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
{
pd_entry_t oldpde, *pde;
pt_entry_t *pte;
vm_offset_t pdnxt;
vm_offset_t va, pdnxt;
vm_page_t m;
boolean_t anychanged, pv_lists_locked;
@ -4966,11 +4966,11 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
}
if (pdnxt > eva)
pdnxt = eva;
va = pdnxt;
for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,
sva += PAGE_SIZE) {
if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED |
PG_V))
continue;
if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V))
goto maybe_invlrng;
else if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
if (advice == MADV_DONTNEED) {
/*
@ -4985,12 +4985,21 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
} else if ((*pte & PG_A) != 0)
atomic_clear_int((u_int *)pte, PG_A);
else
continue;
if ((*pte & PG_G) != 0)
pmap_invalidate_page(pmap, sva);
else
goto maybe_invlrng;
if ((*pte & PG_G) != 0) {
if (va == pdnxt)
va = sva;
} else
anychanged = TRUE;
continue;
maybe_invlrng:
if (va != pdnxt) {
pmap_invalidate_range(pmap, va, sva);
va = pdnxt;
}
}
if (va != pdnxt)
pmap_invalidate_range(pmap, va, sva);
}
if (anychanged)
pmap_invalidate_all(pmap);