diff --git a/sys/arm/arm/pmap-v6-new.c b/sys/arm/arm/pmap-v6-new.c index 6865888127ce..450093f4a113 100644 --- a/sys/arm/arm/pmap-v6-new.c +++ b/sys/arm/arm/pmap-v6-new.c @@ -2748,7 +2748,6 @@ pmap_pv_reclaim(pmap_t locked_pmap) TAILQ_REMOVE(&pv_chunks, pc, pc_lru); if (pmap != pc->pc_pmap) { if (pmap != NULL) { - pmap_tlb_flush_ng(pmap); if (pmap != locked_pmap) PMAP_UNLOCK(pmap); } @@ -2786,8 +2785,7 @@ pmap_pv_reclaim(pmap_t locked_pmap) KASSERT(tpte2 != 0, ("pmap_pv_reclaim: pmap %p va %#x zero pte", pmap, va)); - if (pte2_is_global(tpte2)) - tlb_flush(va); + pmap_tlb_flush(pmap, va); m = PHYS_TO_VM_PAGE(pte2_pa(tpte2)); if (pte2_is_dirty(tpte2)) vm_page_dirty(m); @@ -2845,7 +2843,6 @@ pmap_pv_reclaim(pmap_t locked_pmap) out: TAILQ_CONCAT(&pv_chunks, &newtail, pc_lru); if (pmap != NULL) { - pmap_tlb_flush_ng(pmap); if (pmap != locked_pmap) PMAP_UNLOCK(pmap); } @@ -3375,18 +3372,16 @@ pmap_remove_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t sva, KASSERT((sva & PTE1_OFFSET) == 0, ("%s: sva is not 1mpage aligned", __func__)); + /* + * Clear and invalidate the mapping. It should occupy one and only TLB + * entry. So, pmap_tlb_flush() called with aligned address should be + * sufficient. + */ opte1 = pte1_load_clear(pte1p); + pmap_tlb_flush(pmap, sva); + if (pte1_is_wired(opte1)) pmap->pm_stats.wired_count -= PTE1_SIZE / PAGE_SIZE; - - /* - * If the mapping was global, invalidate it even if given pmap - * is not active (kernel_pmap is active always). The mapping should - * occupy one and only TLB entry. So, pmap_tlb_flush() called - * with aligned address should be sufficient. - */ - if (pte1_is_global(opte1)) - tlb_flush(sva); pmap->pm_stats.resident_count -= PTE1_SIZE / PAGE_SIZE; if (pte1_is_managed(opte1)) { pvh = pa_to_pvh(pte1_pa(opte1)); @@ -3470,7 +3465,6 @@ pmap_demote_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va) VM_ALLOC_NORMAL | VM_ALLOC_WIRED)) == NULL) { SLIST_INIT(&free); pmap_remove_pte1(pmap, pte1p, pte1_trunc(va), &free); - pmap_tlb_flush(pmap, pte1_trunc(va)); pmap_free_zero_pages(&free); CTR3(KTR_PMAP, "%s: failure for va %#x in pmap %p", __func__, va, pmap); @@ -3856,17 +3850,15 @@ pmap_remove_pte2(pmap_t pmap, pt2_entry_t *pte2p, vm_offset_t va, rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); + /* Clear and invalidate the mapping. */ opte2 = pte2_load_clear(pte2p); + pmap_tlb_flush(pmap, va); + KASSERT(pte2_is_valid(opte2), ("%s: pmap %p va %#x not link pte2 %#x", __func__, pmap, va, opte2)); + if (opte2 & PTE2_W) pmap->pm_stats.wired_count -= 1; - /* - * If the mapping was global, invalidate it even if given pmap - * is not active (kernel_pmap is active always). - */ - if (pte2_is_global(opte2)) - tlb_flush(va); pmap->pm_stats.resident_count -= 1; if (pte2_is_managed(opte2)) { m = PHYS_TO_VM_PAGE(pte2_pa(opte2)); @@ -3895,7 +3887,6 @@ pmap_remove_page(pmap_t pmap, vm_offset_t va, struct spglist *free) !pte2_is_valid(pte2_load(pte2p))) return; pmap_remove_pte2(pmap, pte2p, va, free); - pmap_tlb_flush(pmap, va); } /* @@ -3911,7 +3902,6 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) pt1_entry_t *pte1p, pte1; pt2_entry_t *pte2p, pte2; struct spglist free; - int anyvalid; /* * Perform an unsynchronized read. This is, however, safe. @@ -3919,7 +3909,6 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) if (pmap->pm_stats.resident_count == 0) return; - anyvalid = 0; SLIST_INIT(&free); rw_wlock(&pvh_global_lock); @@ -3964,12 +3953,6 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) * demote the mapping and fall through. */ if (sva + PTE1_SIZE == nextva && eva >= nextva) { - /* - * The TLB entry for global mapping is - * invalidated by pmap_remove_pte1(). - */ - if (!pte1_is_global(pte1)) - anyvalid = 1; pmap_remove_pte1(pmap, pte1p, sva, &free); continue; } else if (!pmap_demote_pte1(pmap, pte1p, sva)) { @@ -4000,21 +3983,12 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) pte2 = pte2_load(pte2p); if (!pte2_is_valid(pte2)) continue; - - /* - * The TLB entry for global mapping is invalidated - * by pmap_remove_pte2(). - */ - if (!pte2_is_global(pte2)) - anyvalid = 1; if (pmap_remove_pte2(pmap, pte2p, sva, &free)) break; } } out: sched_unpin(); - if (anyvalid) - pmap_tlb_flush_ng(pmap); rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); pmap_free_zero_pages(&free); @@ -4070,6 +4044,7 @@ small_mappings: "a 1mpage in page %p's pv list", __func__, m)); pte2p = pmap_pte2_quick(pmap, pv->pv_va); opte2 = pte2_load_clear(pte2p); + pmap_tlb_flush(pmap, pv->pv_va); KASSERT(pte2_is_valid(opte2), ("%s: pmap %p va %x zero pte2", __func__, pmap, pv->pv_va)); if (pte2_is_wired(opte2)) @@ -4083,7 +4058,6 @@ small_mappings: if (pte2_is_dirty(opte2)) vm_page_dirty(m); pmap_unuse_pt2(pmap, pv->pv_va, &free); - pmap_tlb_flush(pmap, pv->pv_va); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); free_pv_entry(pmap, pv); PMAP_UNLOCK(pmap); @@ -4602,19 +4576,17 @@ pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, /* * Do the things to protect a 1mpage in a process. */ -static boolean_t +static void pmap_protect_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t sva, vm_prot_t prot) { pt1_entry_t npte1, opte1; vm_offset_t eva, va; vm_page_t m; - boolean_t anychanged; PMAP_LOCK_ASSERT(pmap, MA_OWNED); KASSERT((sva & PTE1_OFFSET) == 0, ("%s: sva is not 1mpage aligned", __func__)); - anychanged = FALSE; retry: opte1 = npte1 = pte1_load(pte1p); if (pte1_is_managed(opte1)) { @@ -4638,12 +4610,8 @@ retry: if (npte1 != opte1) { if (!pte1_cmpset(pte1p, opte1, npte1)) goto retry; - if (pte1_is_global(opte1)) - tlb_flush(sva); - else - anychanged = TRUE; + pmap_tlb_flush(pmap, sva); } - return (anychanged); } /* @@ -4653,7 +4621,7 @@ retry: void pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) { - boolean_t anychanged, pv_lists_locked; + boolean_t pv_lists_locked; vm_offset_t nextva; pt1_entry_t *pte1p, pte1; pt2_entry_t *pte2p, opte2, npte2; @@ -4676,7 +4644,6 @@ resume: rw_wlock(&pvh_global_lock); sched_pin(); } - anychanged = FALSE; PMAP_LOCK(pmap); for (; sva < eva; sva = nextva) { @@ -4703,19 +4670,12 @@ resume: * demote the mapping and fall through. */ if (sva + PTE1_SIZE == nextva && eva >= nextva) { - /* - * The TLB entry for global mapping is - * invalidated by pmap_protect_pte1(). - */ - if (pmap_protect_pte1(pmap, pte1p, sva, prot)) - anychanged = TRUE; + pmap_protect_pte1(pmap, pte1p, sva, prot); continue; } else { if (!pv_lists_locked) { pv_lists_locked = TRUE; if (!rw_try_wlock(&pvh_global_lock)) { - if (anychanged) - pmap_tlb_flush_ng(pmap); PMAP_UNLOCK(pmap); goto resume; } @@ -4778,16 +4738,10 @@ retry: if (!pte2_cmpset(pte2p, opte2, npte2)) goto retry; - - if (pte2_is_global(opte2)) - tlb_flush(sva); - else - anychanged = TRUE; + pmap_tlb_flush(pmap, sva); } } } - if (anychanged) - pmap_tlb_flush_ng(pmap); if (pv_lists_locked) { sched_unpin(); rw_wunlock(&pvh_global_lock); @@ -5294,7 +5248,7 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) pt2_entry_t *pte2p, pte2; vm_offset_t pdnxt; vm_page_t m; - boolean_t anychanged, pv_lists_locked; + boolean_t pv_lists_locked; if (advice != MADV_DONTNEED && advice != MADV_FREE) return; @@ -5306,7 +5260,6 @@ resume: rw_wlock(&pvh_global_lock); sched_pin(); } - anychanged = FALSE; PMAP_LOCK(pmap); for (; sva < eva; sva = pdnxt) { pdnxt = pte1_trunc(sva + PTE1_SIZE); @@ -5322,8 +5275,6 @@ resume: if (!pv_lists_locked) { pv_lists_locked = TRUE; if (!rw_try_wlock(&pvh_global_lock)) { - if (anychanged) - pmap_tlb_flush_ng(pmap); PMAP_UNLOCK(pmap); goto resume; } @@ -5348,7 +5299,6 @@ resume: KASSERT(pte2_is_valid(pte2_load(pte2p)), ("%s: invalid PTE2", __func__)); pmap_remove_pte2(pmap, pte2p, sva, NULL); - anychanged = TRUE; } } if (pdnxt > eva) @@ -5374,14 +5324,9 @@ resume: pte2_clear_bit(pte2p, PTE2_A); else continue; - if (pte2_is_global(pte2)) - tlb_flush(sva); - else - anychanged = TRUE; + pmap_tlb_flush(pmap, sva); } } - if (anychanged) - pmap_tlb_flush_ng(pmap); if (pv_lists_locked) { sched_unpin(); rw_wunlock(&pvh_global_lock);