d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Reduce the frequency that the PowerPC/AIM pmaps invalidate instruction

Browse Source 
caches, by invalidating kernel icaches only when needed and not flushing
user caches for shared pages.

Suggested by:	kib
MFC after:	2 weeks
This commit is contained in:
Nathan Whitehorn 2012-04-06 16:03:38 +00:00
parent 629e40e45e
commit 57bd5cce62
4 changed files with 33 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
sys/powerpc/aim/machdep.c
View File

@ -405,6 +405,9 @@ powerpc_init(vm_offset_t startkernel, vm_offset_t endkernel,
cacheline_size = 32;
}
/* Make sure the kernel icache is valid before we go too much further */
__syncicache((caddr_t)startkernel, endkernel - startkernel);
#ifndef __powerpc64__
/*
* Figure out whether we need to use the 64 bit PMAP. This works by

40
sys/powerpc/aim/mmu_oea.c
View File

@ -1087,7 +1087,7 @@ moea_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
struct pvo_head *pvo_head;
uma_zone_t zone;
vm_page_t pg;
u_int pte_lo, pvo_flags, was_exec;
u_int pte_lo, pvo_flags;
int error;
if (!moea_initialized) {
@ -1095,13 +1095,11 @@ moea_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
zone = moea_upvo_zone;
pvo_flags = 0;
pg = NULL;
was_exec = PTE_EXEC;
} else {
pvo_head = vm_page_to_pvoh(m);
pg = m;
zone = moea_mpvo_zone;
pvo_flags = PVO_MANAGED;
was_exec = 0;
}
if (pmap_bootstrapped)
mtx_assert(&vm_page_queue_mtx, MA_OWNED);
@ -1117,18 +1115,6 @@ moea_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
zone = moea_upvo_zone;
}
/*
* If this is a managed page, and it's the first reference to the page,
* clear the execness of the page. Otherwise fetch the execness.
*/
if ((pg != NULL) && ((m->oflags & VPO_UNMANAGED) == 0)) {
if (LIST_EMPTY(pvo_head)) {
moea_attr_clear(pg, PTE_EXEC);
} else {
was_exec = moea_attr_fetch(pg) & PTE_EXEC;
}
}
pte_lo = moea_calc_wimg(VM_PAGE_TO_PHYS(m), pmap_page_get_memattr(m));
if (prot & VM_PROT_WRITE) {
@ -1149,22 +1135,12 @@ moea_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
pte_lo, pvo_flags);
/*
* Flush the real page from the instruction cache if this page is
* mapped executable and cacheable and was not previously mapped (or
* was not mapped executable).
* Flush the real page from the instruction cache. This has be done
* for all user mappings to prevent information leakage via the
* instruction cache.
*/
if (error == 0 && (pvo_flags & PVO_EXECUTABLE) &&
(pte_lo & PTE_I) == 0 && was_exec == 0) {
/*
* Flush the real memory from the cache.
*/
if (pmap != kernel_pmap && LIST_EMPTY(vm_page_to_pvoh(m)))
moea_syncicache(VM_PAGE_TO_PHYS(m), PAGE_SIZE);
if (pg != NULL)
moea_attr_save(pg, PTE_EXEC);
}
/* XXX syncicache always until problems are sorted */
moea_syncicache(VM_PAGE_TO_PHYS(m), PAGE_SIZE);
}
/*
@ -1486,12 +1462,6 @@ moea_kenter_attr(mmu_t mmu, vm_offset_t va, vm_offset_t pa, vm_memattr_t ma)
panic("moea_kenter: failed to enter va %#x pa %#x: %d", va,
pa, error);
/*
* Flush the real memory from the instruction cache.
*/
if ((pte_lo & (PTE_I | PTE_G)) == 0) {
moea_syncicache(pa, PAGE_SIZE);
}
PMAP_UNLOCK(kernel_pmap);
}

43
sys/powerpc/aim/mmu_oea64.c
View File

@ -1246,8 +1246,11 @@ moea64_enter_locked(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m,
* Flush the page from the instruction cache if this page is
* mapped executable and cacheable.
*/
if ((pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0)
if (pmap != kernel_pmap && !(m->aflags & PGA_EXECUTABLE) &&
(pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
vm_page_aflag_set(m, PGA_EXECUTABLE);
moea64_syncicache(mmu, pmap, va, VM_PAGE_TO_PHYS(m), PAGE_SIZE);
}
}
static void
@ -1670,12 +1673,6 @@ moea64_kenter_attr(mmu_t mmu, vm_offset_t va, vm_offset_t pa, vm_memattr_t ma)
if (error != 0 && error != ENOENT)
panic("moea64_kenter: failed to enter va %#zx pa %#zx: %d", va,
pa, error);
/*
* Flush the memory from the instruction cache.
*/
if ((pte_lo & (LPTE_I | LPTE_G)) == 0)
__syncicache((void *)va, PAGE_SIZE);
}
void
@ -1906,6 +1903,7 @@ static void
moea64_pvo_protect(mmu_t mmu, pmap_t pm, struct pvo_entry *pvo, vm_prot_t prot)
{
uintptr_t pt;
struct vm_page *pg;
uint64_t oldlo;
PMAP_LOCK_ASSERT(pm, MA_OWNED);
@ -1929,17 +1927,21 @@ moea64_pvo_protect(mmu_t mmu, pmap_t pm, struct pvo_entry *pvo, vm_prot_t prot)
else
pvo->pvo_pte.lpte.pte_lo |= LPTE_BR;
pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
/*
* If the PVO is in the page table, update that pte as well.
*/
if (pt != -1) {
MOEA64_PTE_CHANGE(mmu, pt, &pvo->pvo_pte.lpte,
pvo->pvo_vpn);
if ((pvo->pvo_pte.lpte.pte_lo &
(LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
if (pm != kernel_pmap && pg != NULL &&
!(pg->aflags & PGA_EXECUTABLE) &&
(pvo->pvo_pte.lpte.pte_lo &
(LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
vm_page_aflag_set(pg, PGA_EXECUTABLE);
moea64_syncicache(mmu, pm, PVO_VADDR(pvo),
pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN,
PAGE_SIZE);
pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN, PAGE_SIZE);
}
}
@ -1949,9 +1951,6 @@ moea64_pvo_protect(mmu_t mmu, pmap_t pm, struct pvo_entry *pvo, vm_prot_t prot)
*/
if ((pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED &&
(oldlo & LPTE_PP) != LPTE_BR && !(prot && VM_PROT_WRITE)) {
struct vm_page *pg;
pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
if (pg != NULL) {
if (pvo->pvo_pte.lpte.pte_lo & LPTE_CHG)
vm_page_dirty(pg);
@ -2134,15 +2133,11 @@ moea64_remove(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva)
void
moea64_remove_all(mmu_t mmu, vm_page_t m)
{
struct pvo_head *pvo_head;
struct pvo_entry *pvo, *next_pvo;
pmap_t pmap;
pvo_head = vm_page_to_pvoh(m);
LOCK_TABLE_WR();
for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) {
next_pvo = LIST_NEXT(pvo, pvo_vlink);
LIST_FOREACH_SAFE(pvo, vm_page_to_pvoh(m), pvo_vlink, next_pvo) {
pmap = pvo->pvo_pmap;
PMAP_LOCK(pmap);
moea64_pvo_remove(mmu, pvo);
@ -2152,6 +2147,7 @@ moea64_remove_all(mmu_t mmu, vm_page_t m)
if ((m->aflags & PGA_WRITEABLE) && moea64_is_modified(mmu, m))
vm_page_dirty(m);
vm_page_aflag_clear(m, PGA_WRITEABLE);
vm_page_aflag_clear(m, PGA_EXECUTABLE);
}
/*
@ -2356,6 +2352,7 @@ moea64_pvo_enter(mmu_t mmu, pmap_t pm, uma_zone_t zone,
static void
moea64_pvo_remove(mmu_t mmu, struct pvo_entry *pvo)
{
struct vm_page *pg;
uintptr_t pt;
PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
@ -2395,11 +2392,10 @@ moea64_pvo_remove(mmu_t mmu, struct pvo_entry *pvo)
/*
* Update vm about the REF/CHG bits if the page is managed.
*/
pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
if ((pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED &&
(pvo->pvo_pte.lpte.pte_lo & LPTE_PP) != LPTE_BR) {
struct vm_page *pg;
pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
if (pg != NULL) {
if (pvo->pvo_pte.lpte.pte_lo & LPTE_CHG)
vm_page_dirty(pg);
@ -2410,6 +2406,9 @@ moea64_pvo_remove(mmu_t mmu, struct pvo_entry *pvo)
}
}
if (pg != NULL && LIST_EMPTY(vm_page_to_pvoh(pg)))
vm_page_aflag_clear(pg, PGA_EXECUTABLE);
moea64_pvo_entries--;
moea64_pvo_remove_calls++;

4
sys/vm/vm_page.h
View File

@ -248,9 +248,13 @@ extern struct vpglocks pa_lock[];
*
* PGA_WRITEABLE is set exclusively on managed pages by pmap_enter(). When it
* does so, the page must be VPO_BUSY.
*
* PGA_EXECUTABLE may be set by pmap routines, and indicates that a page has
* at least one executable mapping. It is not consumed by the VM layer.
*/
#define PGA_WRITEABLE 0x01 /* page may be mapped writeable */
#define PGA_REFERENCED 0x02 /* page has been referenced */
#define PGA_EXECUTABLE 0x04 /* page may be mapped executable */
/*
* Page flags. If changed at any other time than page allocation or