d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Replace all postponed TLB flushes by immediate ones except the one

Browse Source 
in pmap_remove_pages().

Some points were considered:
(1) There is no range TLB flush cp15 function.
(2) There is no target selection for hardware TLB flush broadcasting.
(3) Some memory ranges could be mapped sparsely.
(4) Some memory ranges could be quite large.

Tested by buildworld on RPi2 and Jetson TK1, i.e. 4 core platforms.
It turned out that the buildworld time is faster. On the other hand,
when the postponed TLB flush was also removed from pmap_remove_pages(),
the result was worse. But pmap_remove_pages() is called for removing
all user mapping from a process, thus it's quite expected.

Note that the postponed TLB flushes came here from i386 pmap where
hardware TLB flush broadcasting is not available.

Approved by:	kib (mentor)
This commit is contained in:
Svatopluk Kraus 2015-12-15 15:22:33 +00:00
parent d0ec8fd065
commit 28dff94844
1 changed files with 20 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
sys/arm/arm/pmap-v6-new.c
View File

@ -2748,7 +2748,6 @@ pmap_pv_reclaim(pmap_t locked_pmap)
TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
if (pmap != pc->pc_pmap) {
if (pmap != NULL) {
pmap_tlb_flush_ng(pmap);
if (pmap != locked_pmap)
PMAP_UNLOCK(pmap);
}
@ -2786,8 +2785,7 @@ pmap_pv_reclaim(pmap_t locked_pmap)
KASSERT(tpte2 != 0,
("pmap_pv_reclaim: pmap %p va %#x zero pte",
pmap, va));
if (pte2_is_global(tpte2))
tlb_flush(va);
pmap_tlb_flush(pmap, va);
m = PHYS_TO_VM_PAGE(pte2_pa(tpte2));
if (pte2_is_dirty(tpte2))
vm_page_dirty(m);
@ -2845,7 +2843,6 @@ pmap_pv_reclaim(pmap_t locked_pmap)
out:
TAILQ_CONCAT(&pv_chunks, &newtail, pc_lru);
if (pmap != NULL) {
pmap_tlb_flush_ng(pmap);
if (pmap != locked_pmap)
PMAP_UNLOCK(pmap);
}
@ -3375,18 +3372,16 @@ pmap_remove_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t sva,
KASSERT((sva & PTE1_OFFSET) == 0,
("%s: sva is not 1mpage aligned", __func__));
/*
* Clear and invalidate the mapping. It should occupy one and only TLB
* entry. So, pmap_tlb_flush() called with aligned address should be
* sufficient.
*/
opte1 = pte1_load_clear(pte1p);
pmap_tlb_flush(pmap, sva);
if (pte1_is_wired(opte1))
pmap->pm_stats.wired_count -= PTE1_SIZE / PAGE_SIZE;
/*
* If the mapping was global, invalidate it even if given pmap
* is not active (kernel_pmap is active always). The mapping should
* occupy one and only TLB entry. So, pmap_tlb_flush() called
* with aligned address should be sufficient.
*/
if (pte1_is_global(opte1))
tlb_flush(sva);
pmap->pm_stats.resident_count -= PTE1_SIZE / PAGE_SIZE;
if (pte1_is_managed(opte1)) {
pvh = pa_to_pvh(pte1_pa(opte1));
@ -3470,7 +3465,6 @@ pmap_demote_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va)
VM_ALLOC_NORMAL | VM_ALLOC_WIRED)) == NULL) {
SLIST_INIT(&free);
pmap_remove_pte1(pmap, pte1p, pte1_trunc(va), &free);
pmap_tlb_flush(pmap, pte1_trunc(va));
pmap_free_zero_pages(&free);
CTR3(KTR_PMAP, "%s: failure for va %#x in pmap %p",
__func__, va, pmap);
@ -3856,17 +3850,15 @@ pmap_remove_pte2(pmap_t pmap, pt2_entry_t *pte2p, vm_offset_t va,
rw_assert(&pvh_global_lock, RA_WLOCKED);
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
/* Clear and invalidate the mapping. */
opte2 = pte2_load_clear(pte2p);
pmap_tlb_flush(pmap, va);
KASSERT(pte2_is_valid(opte2), ("%s: pmap %p va %#x not link pte2 %#x",
__func__, pmap, va, opte2));
if (opte2 & PTE2_W)
pmap->pm_stats.wired_count -= 1;
/*
* If the mapping was global, invalidate it even if given pmap
* is not active (kernel_pmap is active always).
*/
if (pte2_is_global(opte2))
tlb_flush(va);
pmap->pm_stats.resident_count -= 1;
if (pte2_is_managed(opte2)) {
m = PHYS_TO_VM_PAGE(pte2_pa(opte2));
@ -3895,7 +3887,6 @@ pmap_remove_page(pmap_t pmap, vm_offset_t va, struct spglist *free)
!pte2_is_valid(pte2_load(pte2p)))
return;
pmap_remove_pte2(pmap, pte2p, va, free);
pmap_tlb_flush(pmap, va);
}
/*
@ -3911,7 +3902,6 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
pt1_entry_t *pte1p, pte1;
pt2_entry_t *pte2p, pte2;
struct spglist free;
int anyvalid;
/*
* Perform an unsynchronized read. This is, however, safe.
@ -3919,7 +3909,6 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
if (pmap->pm_stats.resident_count == 0)
return;
anyvalid = 0;
SLIST_INIT(&free);
rw_wlock(&pvh_global_lock);
@ -3964,12 +3953,6 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
* demote the mapping and fall through.
*/
if (sva + PTE1_SIZE == nextva && eva >= nextva) {
/*
* The TLB entry for global mapping is
* invalidated by pmap_remove_pte1().
*/
if (!pte1_is_global(pte1))
anyvalid = 1;
pmap_remove_pte1(pmap, pte1p, sva, &free);
continue;
} else if (!pmap_demote_pte1(pmap, pte1p, sva)) {
@ -4000,21 +3983,12 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
pte2 = pte2_load(pte2p);
if (!pte2_is_valid(pte2))
continue;
/*
* The TLB entry for global mapping is invalidated
* by pmap_remove_pte2().
*/
if (!pte2_is_global(pte2))
anyvalid = 1;
if (pmap_remove_pte2(pmap, pte2p, sva, &free))
break;
}
}
out:
sched_unpin();
if (anyvalid)
pmap_tlb_flush_ng(pmap);
rw_wunlock(&pvh_global_lock);
PMAP_UNLOCK(pmap);
pmap_free_zero_pages(&free);
@ -4070,6 +4044,7 @@ pmap_remove_all(vm_page_t m)
"a 1mpage in page %p's pv list", __func__, m));
pte2p = pmap_pte2_quick(pmap, pv->pv_va);
opte2 = pte2_load_clear(pte2p);
pmap_tlb_flush(pmap, pv->pv_va);
KASSERT(pte2_is_valid(opte2), ("%s: pmap %p va %x zero pte2",
__func__, pmap, pv->pv_va));
if (pte2_is_wired(opte2))
@ -4083,7 +4058,6 @@ pmap_remove_all(vm_page_t m)
if (pte2_is_dirty(opte2))
vm_page_dirty(m);
pmap_unuse_pt2(pmap, pv->pv_va, &free);
pmap_tlb_flush(pmap, pv->pv_va);
TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
free_pv_entry(pmap, pv);
PMAP_UNLOCK(pmap);
@ -4602,19 +4576,17 @@ pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,
/*
* Do the things to protect a 1mpage in a process.
*/
static boolean_t
static void
pmap_protect_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t sva,
vm_prot_t prot)
{
pt1_entry_t npte1, opte1;
vm_offset_t eva, va;
vm_page_t m;
boolean_t anychanged;
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
KASSERT((sva & PTE1_OFFSET) == 0,
("%s: sva is not 1mpage aligned", __func__));
anychanged = FALSE;
retry:
opte1 = npte1 = pte1_load(pte1p);
if (pte1_is_managed(opte1)) {
@ -4638,12 +4610,8 @@ pmap_protect_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t sva,
if (npte1 != opte1) {
if (!pte1_cmpset(pte1p, opte1, npte1))
goto retry;
if (pte1_is_global(opte1))
tlb_flush(sva);
else
anychanged = TRUE;
pmap_tlb_flush(pmap, sva);
}
return (anychanged);
}
/*
@ -4653,7 +4621,7 @@ pmap_protect_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t sva,
void
pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
{
boolean_t anychanged, pv_lists_locked;
boolean_t pv_lists_locked;
vm_offset_t nextva;
pt1_entry_t *pte1p, pte1;
pt2_entry_t *pte2p, opte2, npte2;
@ -4676,7 +4644,6 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
rw_wlock(&pvh_global_lock);
sched_pin();
}
anychanged = FALSE;
PMAP_LOCK(pmap);
for (; sva < eva; sva = nextva) {
@ -4703,19 +4670,12 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
* demote the mapping and fall through.
*/
if (sva + PTE1_SIZE == nextva && eva >= nextva) {
/*
* The TLB entry for global mapping is
* invalidated by pmap_protect_pte1().
*/
if (pmap_protect_pte1(pmap, pte1p, sva, prot))
anychanged = TRUE;
pmap_protect_pte1(pmap, pte1p, sva, prot);
continue;
} else {
if (!pv_lists_locked) {
pv_lists_locked = TRUE;
if (!rw_try_wlock(&pvh_global_lock)) {
if (anychanged)
pmap_tlb_flush_ng(pmap);
PMAP_UNLOCK(pmap);
goto resume;
}
@ -4778,16 +4738,10 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
if (!pte2_cmpset(pte2p, opte2, npte2))
goto retry;
if (pte2_is_global(opte2))
tlb_flush(sva);
else
anychanged = TRUE;
pmap_tlb_flush(pmap, sva);
}
}
}
if (anychanged)
pmap_tlb_flush_ng(pmap);
if (pv_lists_locked) {
sched_unpin();
rw_wunlock(&pvh_global_lock);
@ -5294,7 +5248,7 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
pt2_entry_t *pte2p, pte2;
vm_offset_t pdnxt;
vm_page_t m;
boolean_t anychanged, pv_lists_locked;
boolean_t pv_lists_locked;
if (advice != MADV_DONTNEED && advice != MADV_FREE)
return;
@ -5306,7 +5260,6 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
rw_wlock(&pvh_global_lock);
sched_pin();
}
anychanged = FALSE;
PMAP_LOCK(pmap);
for (; sva < eva; sva = pdnxt) {
pdnxt = pte1_trunc(sva + PTE1_SIZE);
@ -5322,8 +5275,6 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
if (!pv_lists_locked) {
pv_lists_locked = TRUE;
if (!rw_try_wlock(&pvh_global_lock)) {
if (anychanged)
pmap_tlb_flush_ng(pmap);
PMAP_UNLOCK(pmap);
goto resume;
}
@ -5348,7 +5299,6 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
KASSERT(pte2_is_valid(pte2_load(pte2p)),
("%s: invalid PTE2", __func__));
pmap_remove_pte2(pmap, pte2p, sva, NULL);
anychanged = TRUE;
}
}
if (pdnxt > eva)
@ -5374,14 +5324,9 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
pte2_clear_bit(pte2p, PTE2_A);
else
continue;
if (pte2_is_global(pte2))
tlb_flush(sva);
else
anychanged = TRUE;
pmap_tlb_flush(pmap, sva);
}
}
if (anychanged)
pmap_tlb_flush_ng(pmap);
if (pv_lists_locked) {
sched_unpin();
rw_wunlock(&pvh_global_lock);