d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add more tlb invalidations. We currently invalidate when we may not need

Browse Source 
to, but with this I can boot on a simulator that models the tlb.

Obtained from:	ABT Systems Ltd
Sponsored by:	The FreeBSD Foundation
This commit is contained in:
Andrew Turner 2015-07-06 18:27:18 +00:00
parent 1d7a730974
commit b67d1aad6f
1 changed files with 38 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
sys/arm64/arm64/pmap.c
View File

@ -844,17 +844,19 @@ pmap_kextract(vm_offset_t va)
***************************************************/
void
pmap_kenter_device(vm_offset_t va, vm_size_t size, vm_paddr_t pa)
pmap_kenter_device(vm_offset_t sva, vm_size_t size, vm_paddr_t pa)
{
pt_entry_t *l3;
vm_offset_t va;
KASSERT((pa & L3_OFFSET) == 0,
("pmap_kenter_device: Invalid physical address"));
KASSERT((va & L3_OFFSET) == 0,
KASSERT((sva & L3_OFFSET) == 0,
("pmap_kenter_device: Invalid virtual address"));
KASSERT((size & PAGE_MASK) == 0,
("pmap_kenter_device: Mapping is not page-sized"));
va = sva;
while (size != 0) {
l3 = pmap_l3(kernel_pmap, va);
KASSERT(l3 != NULL, ("Invalid page table, va: 0x%lx", va));
@ -866,6 +868,7 @@ pmap_kenter_device(vm_offset_t va, vm_size_t size, vm_paddr_t pa)
pa += PAGE_SIZE;
size -= PAGE_SIZE;
}
pmap_invalidate_range(kernel_pmap, sva, va);
}
/*
@ -884,18 +887,21 @@ pmap_kremove(vm_offset_t va)
cpu_dcache_wb_range(va, L3_SIZE);
pmap_load_clear(l3);
PTE_SYNC(l3);
pmap_invalidate_page(kernel_pmap, va);
}
void
pmap_kremove_device(vm_offset_t va, vm_size_t size)
pmap_kremove_device(vm_offset_t sva, vm_size_t size)
{
pt_entry_t *l3;
vm_offset_t va;
KASSERT((va & L3_OFFSET) == 0,
KASSERT((sva & L3_OFFSET) == 0,
("pmap_kremove_device: Invalid virtual address"));
KASSERT((size & PAGE_MASK) == 0,
("pmap_kremove_device: Mapping is not page-sized"));
va = sva;
while (size != 0) {
l3 = pmap_l3(kernel_pmap, va);
KASSERT(l3 != NULL, ("Invalid page table, va: 0x%lx", va));
@ -905,6 +911,7 @@ pmap_kremove_device(vm_offset_t va, vm_size_t size)
va += PAGE_SIZE;
size -= PAGE_SIZE;
}
pmap_invalidate_range(kernel_pmap, sva, va);
}
/*
@ -954,6 +961,7 @@ pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count)
va += L3_SIZE;
}
pmap_invalidate_range(kernel_pmap, sva, va);
}
/*
@ -964,12 +972,21 @@ pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count)
void
pmap_qremove(vm_offset_t sva, int count)
{
pt_entry_t *l3;
vm_offset_t va;
KASSERT(sva >= VM_MIN_KERNEL_ADDRESS, ("usermode va %lx", sva));
va = sva;
while (count-- > 0) {
KASSERT(va >= VM_MIN_KERNEL_ADDRESS, ("usermode va %lx", va));
pmap_kremove(va);
l3 = pmap_l3(kernel_pmap, va);
KASSERT(l3 != NULL, ("pmap_kremove: Invalid address"));
if (pmap_l3_valid_cacheable(pmap_load(l3)))
cpu_dcache_wb_range(va, L3_SIZE);
pmap_load_clear(l3);
PTE_SYNC(l3);
va += PAGE_SIZE;
}
pmap_invalidate_range(kernel_pmap, sva, va);
@ -1054,6 +1071,7 @@ _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)
pdpg = PHYS_TO_VM_PAGE(*pmap_l1(pmap, va) & ~ATTR_MASK);
pmap_unwire_l3(pmap, va, pdpg, free);
}
pmap_invalidate_page(pmap, va);
/*
* This is a release store so that the ordinary store unmapping
@ -1239,11 +1257,6 @@ retry:
if (m == NULL && lockp != NULL)
goto retry;
}
/*
* XXXARM64: I'm not sure why we need this but it fixes a crash
* when running things from a shell script.
*/
pmap_invalidate_all(pmap);
return (m);
}
@ -1346,6 +1359,7 @@ pmap_growkernel(vm_offset_t addr)
paddr = VM_PAGE_TO_PHYS(nkpg);
pmap_load_store(l2, paddr | L2_TABLE);
PTE_SYNC(l2);
pmap_invalidate_page(kernel_pmap, kernel_vm_end);
kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET;
if (kernel_vm_end - 1 >= kernel_map->max_offset) {
@ -1626,6 +1640,7 @@ pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t va,
cpu_dcache_wb_range(va, L3_SIZE);
old_l3 = pmap_load_clear(l3);
PTE_SYNC(l3);
pmap_invalidate_page(pmap, va);
if (old_l3 & ATTR_SW_WIRED)
pmap->pm_stats.wired_count -= 1;
pmap_resident_count_dec(pmap, 1);
@ -1782,6 +1797,7 @@ pmap_remove_all(vm_page_t m)
cpu_dcache_wb_range(pv->pv_va, L3_SIZE);
tl3 = pmap_load_clear(l3);
PTE_SYNC(l3);
pmap_invalidate_page(pmap, pv->pv_va);
if (tl3 & ATTR_SW_WIRED)
pmap->pm_stats.wired_count--;
if ((tl3 & ATTR_AF) != 0)
@ -1793,7 +1809,6 @@ pmap_remove_all(vm_page_t m)
if (pmap_page_dirty(tl3))
vm_page_dirty(m);
pmap_unuse_l3(pmap, pv->pv_va, *l2, &free);
pmap_invalidate_page(pmap, pv->pv_va);
TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
m->md.pv_gen++;
free_pv_entry(pmap, pv);
@ -1852,6 +1867,8 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
if (pmap_l3_valid(l3)) {
pmap_set(l3p, ATTR_AP(ATTR_AP_RO));
PTE_SYNC(l3p);
/* XXX: Use pmap_invalidate_range */
pmap_invalidate_page(pmap, va);
}
}
}
@ -1899,6 +1916,8 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
new_l3 |= ATTR_AP(ATTR_AP_USER);
new_l3 |= ATTR_IDX(m->md.pv_memattr);
CTR2(KTR_PMAP, "pmap_enter: %.16lx -> %.16lx", va, pa);
mpte = NULL;
lock = NULL;
@ -1909,6 +1928,7 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0;
mpte = pmap_alloc_l3(pmap, va, nosleep ? NULL : &lock);
if (mpte == NULL && nosleep) {
CTR0(KTR_PMAP, "pmap_enter: mpte == NULL");
if (lock != NULL)
rw_wunlock(lock);
rw_runlock(&pvh_global_lock);
@ -1953,6 +1973,7 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
PTE_SYNC(l2);
l3 = pmap_l2_to_l3(l2, va);
}
pmap_invalidate_page(pmap, va);
}
om = NULL;
@ -2051,12 +2072,11 @@ validate:
if ((orig_l3 & ATTR_SW_MANAGED) != 0)
vm_page_dirty(m);
}
if ((orig_l3 & ATTR_AF) != 0)
pmap_invalidate_page(pmap, va);
} else {
pmap_load_store(l3, new_l3);
PTE_SYNC(l3);
}
pmap_invalidate_page(pmap, va);
if ((pmap != pmap_kernel()) && (pmap == &curproc->p_vmspace->vm_pmap))
cpu_icache_sync_range(va, PAGE_SIZE);
@ -2146,6 +2166,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
rw_assert(&pvh_global_lock, RA_LOCKED);
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
CTR2(KTR_PMAP, "pmap_enter_quick_locked: %p %lx", pmap, va);
/*
* In the case that a page table page is not
* resident, we are creating it here.
@ -2193,7 +2214,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
}
if (l3 == NULL)
panic("pmap_enter_quick_locked: No l3");
if (*l3) {
if (pmap_load(l3) != 0) {
if (mpte != NULL) {
mpte->wire_count--;
mpte = NULL;
@ -2579,6 +2600,7 @@ pmap_remove_pages(pmap_t pmap)
cpu_dcache_wb_range(pv->pv_va, L3_SIZE);
pmap_load_clear(l3);
PTE_SYNC(l3);
pmap_invalidate_page(pmap, pv->pv_va);
/*
* Update the vm_page_t clean/reference bits.
@ -2972,6 +2994,7 @@ pmap_activate(struct thread *td)
pmap = vmspace_pmap(td->td_proc->p_vmspace);
td->td_pcb->pcb_l1addr = vtophys(pmap->pm_l1);
__asm __volatile("msr ttbr0_el1, %0" : : "r"(td->td_pcb->pcb_l1addr));
pmap_invalidate_all(pmap);
critical_exit();
}