d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add PV list locking to pmap_enter(). Its execution is no longer serialized

Browse Source 
by the pvh global lock.

Add a needed atomic operation to pmap_object_init_pt().
This commit is contained in:
Alan Cox 2012-06-26 06:02:43 +00:00
parent af5b2c7853
commit 0d646df757
1 changed files with 70 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
sys/amd64/amd64/pmap.c
View File

@ -264,7 +264,8 @@ static void pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
struct rwlock **lockp);
static boolean_t pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
struct rwlock **lockp);
static void pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
static void pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
struct rwlock **lockp);
static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,
vm_offset_t va);
@ -287,7 +288,8 @@ static boolean_t pmap_is_referenced_pvh(struct md_page *pvh);
static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
static vm_page_t pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va);
static void pmap_pde_attr(pd_entry_t *pde, int cache_bits);
static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,
struct rwlock **lockp);
static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva,
vm_prot_t prot);
static void pmap_pte_attr(pt_entry_t *pte, int cache_bits);
@ -305,10 +307,13 @@ static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,
pd_entry_t newpde);
static void pmap_update_pde_invalidate(vm_offset_t va, pd_entry_t newpde);
static vm_page_t pmap_allocpde(pmap_t pmap, vm_offset_t va, int flags);
static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags);
static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex,
struct rwlock **lockp);
static vm_page_t pmap_allocpde(pmap_t pmap, vm_offset_t va,
struct rwlock **lockp);
static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va,
struct rwlock **lockp);
static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags);
static int _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m,
vm_page_t* free);
static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, vm_page_t *);
@ -1686,8 +1691,10 @@ pmap_pinit(pmap_t pmap)
}
/*
* this routine is called if the page table page is not
* mapped correctly.
* This routine is called if the desired page table page does not exist.
*
* If page table page allocation fails, this routine may sleep before
* returning NULL. It sleeps only if a lock pointer was given.
*
* Note: If a page allocation fails at page table level two or three,
* one or two pages may be held during the wait, only to be released
@ -1695,25 +1702,26 @@ pmap_pinit(pmap_t pmap)
* race conditions.
*/
static vm_page_t
_pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags)
_pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp)
{
vm_page_t m, pdppg, pdpg;
KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
(flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
("_pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK"));
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
/*
* Allocate a page table page.
*/
if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ |
VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
if (flags & M_WAITOK) {
if (lockp != NULL) {
if (*lockp != NULL) {
rw_wunlock(*lockp);
*lockp = NULL;
}
PMAP_UNLOCK(pmap);
rw_wunlock(&pvh_global_lock);
rw_runlock(&pvh_global_lock);
VM_WAIT;
rw_wlock(&pvh_global_lock);
rw_rlock(&pvh_global_lock);
PMAP_LOCK(pmap);
}
@ -1754,7 +1762,7 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags)
if ((*pml4 & PG_V) == 0) {
/* Have to allocate a new pdp, recurse */
if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml4index,
flags) == NULL) {
lockp) == NULL) {
--m->wire_count;
atomic_subtract_int(&cnt.v_wire_count, 1);
vm_page_free_zero(m);
@ -1787,7 +1795,7 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags)
if ((*pml4 & PG_V) == 0) {
/* Have to allocate a new pd, recurse */
if (_pmap_allocpte(pmap, NUPDE + pdpindex,
flags) == NULL) {
lockp) == NULL) {
--m->wire_count;
atomic_subtract_int(&cnt.v_wire_count, 1);
vm_page_free_zero(m);
@ -1801,7 +1809,7 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags)
if ((*pdp & PG_V) == 0) {
/* Have to allocate a new pd, recurse */
if (_pmap_allocpte(pmap, NUPDE + pdpindex,
flags) == NULL) {
lockp) == NULL) {
--m->wire_count;
atomic_subtract_int(&cnt.v_wire_count,
1);
@ -1827,15 +1835,12 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags)
}
static vm_page_t
pmap_allocpde(pmap_t pmap, vm_offset_t va, int flags)
pmap_allocpde(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
{
vm_pindex_t pdpindex, ptepindex;
pdp_entry_t *pdpe;
vm_page_t pdpg;
KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
(flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
("pmap_allocpde: flags is neither M_NOWAIT nor M_WAITOK"));
retry:
pdpe = pmap_pdpe(pmap, va);
if (pdpe != NULL && (*pdpe & PG_V) != 0) {
@ -1846,24 +1851,20 @@ retry:
/* Allocate a pd page. */
ptepindex = pmap_pde_pindex(va);
pdpindex = ptepindex >> NPDPEPGSHIFT;
pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, flags);
if (pdpg == NULL && (flags & M_WAITOK))
pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp);
if (pdpg == NULL && lockp != NULL)
goto retry;
}
return (pdpg);
}
static vm_page_t
pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags)
pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
{
vm_pindex_t ptepindex;
pd_entry_t *pd;
vm_page_t m;
KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
(flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
("pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK"));
/*
* Calculate pagetable page index
*/
@ -1879,7 +1880,7 @@ retry:
* normal 4K page.
*/
if (pd != NULL && (*pd & (PG_PS | PG_V)) == (PG_PS | PG_V)) {
if (!pmap_demote_pde(pmap, pd, va)) {
if (!pmap_demote_pde_locked(pmap, pd, va, lockp)) {
/*
* Invalidation of the 2MB page mapping may have caused
* the deallocation of the underlying PD page.
@ -1900,8 +1901,8 @@ retry:
* Here if the pte page isn't mapped, or if it has been
* deallocated.
*/
m = _pmap_allocpte(pmap, ptepindex, flags);
if (m == NULL && (flags & M_WAITOK))
m = _pmap_allocpte(pmap, ptepindex, lockp);
if (m == NULL && lockp != NULL)
goto retry;
}
return (m);
@ -2504,16 +2505,18 @@ pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
* for the 2MB page mapping.
*/
static void
pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
struct rwlock **lockp)
{
struct md_page *pvh;
pv_entry_t pv;
vm_offset_t va_last;
vm_page_t m;
rw_assert(&pvh_global_lock, RA_WLOCKED);
rw_assert(&pvh_global_lock, RA_LOCKED);
KASSERT((pa & PDRMASK) == 0,
("pmap_pv_promote_pde: pa is not 2mpage aligned"));
CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
/*
* Transfer the first page's pv entry for this mapping to the 2mpage's
@ -3255,7 +3258,8 @@ retry:
* identical characteristics.
*/
static void
pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,
struct rwlock **lockp)
{
pd_entry_t newpde;
pt_entry_t *firstpte, oldpte, pa, *pte;
@ -3273,7 +3277,7 @@ pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
setpde:
newpde = *firstpte;
if ((newpde & ((PG_FRAME & PDRMASK) | PG_A | PG_V)) != (PG_A | PG_V)) {
pmap_pde_p_failures++;
atomic_add_long(&pmap_pde_p_failures, 1);
CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
" in pmap %p", va, pmap);
return;
@ -3298,7 +3302,7 @@ setpde:
setpte:
oldpte = *pte;
if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) {
pmap_pde_p_failures++;
atomic_add_long(&pmap_pde_p_failures, 1);
CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
" in pmap %p", va, pmap);
return;
@ -3317,7 +3321,7 @@ setpte:
" in pmap %p", oldpteva, pmap);
}
if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) {
pmap_pde_p_failures++;
atomic_add_long(&pmap_pde_p_failures, 1);
CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
" in pmap %p", va, pmap);
return;
@ -3342,7 +3346,7 @@ setpte:
* Promote the pv entries.
*/
if ((newpde & PG_MANAGED) != 0)
pmap_pv_promote_pde(pmap, va, newpde & PG_PS_FRAME);
pmap_pv_promote_pde(pmap, va, newpde & PG_PS_FRAME, lockp);
/*
* Propagate the PAT index to its proper position.
@ -3358,7 +3362,7 @@ setpte:
else
pde_store(pde, PG_PS | newpde);
pmap_pde_promotions++;
atomic_add_long(&pmap_pde_promotions, 1);
CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#lx"
" in pmap %p", va, pmap);
}
@ -3379,6 +3383,7 @@ void
pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m,
vm_prot_t prot, boolean_t wired)
{
struct rwlock *lock;
pd_entry_t *pde;
pt_entry_t *pte;
pt_entry_t newpte, origpte;
@ -3398,7 +3403,8 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m,
mpte = NULL;
rw_wlock(&pvh_global_lock);
lock = NULL;
rw_rlock(&pvh_global_lock);
PMAP_LOCK(pmap);
/*
@ -3406,7 +3412,7 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m,
* resident, we are creating it here.
*/
if (va < VM_MAXUSER_ADDRESS)
mpte = pmap_allocpte(pmap, va, M_WAITOK);
mpte = pmap_allocpte(pmap, va, &lock);
pde = pmap_pde(pmap, va);
if (pde != NULL && (*pde & PG_V) != 0) {
@ -3460,6 +3466,7 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m,
pmap->pm_stats.wired_count--;
if (origpte & PG_MANAGED) {
om = PHYS_TO_VM_PAGE(opa);
CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, om);
pv = pmap_pvh_remove(&om->md, pmap, va);
}
if (mpte != NULL) {
@ -3479,6 +3486,7 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m,
("pmap_enter: managed mapping within the clean submap"));
if (pv == NULL)
pv = get_pv_entry(pmap, FALSE);
CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m);
pv->pv_va = va;
TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
pa |= PG_MANAGED;
@ -3534,11 +3542,13 @@ validate:
if ((newpte & PG_RW) == 0)
invlva = TRUE;
}
if ((origpte & PG_MANAGED) != 0 &&
TAILQ_EMPTY(&om->md.pv_list) &&
((om->flags & PG_FICTITIOUS) != 0 ||
TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
vm_page_aflag_clear(om, PGA_WRITEABLE);
if ((origpte & PG_MANAGED) != 0) {
CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, om);
if (TAILQ_EMPTY(&om->md.pv_list) &&
((om->flags & PG_FICTITIOUS) != 0 ||
TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
vm_page_aflag_clear(om, PGA_WRITEABLE);
}
if (invlva)
pmap_invalidate_page(pmap, va);
} else
@ -3552,9 +3562,11 @@ validate:
if ((mpte == NULL || mpte->wire_count == NPTEPG) &&
pg_ps_enabled && (m->flags & PG_FICTITIOUS) == 0 &&
vm_reserv_level_iffullpop(m) == 0)
pmap_promote_pde(pmap, pde, va);
pmap_promote_pde(pmap, pde, va, &lock);
rw_wunlock(&pvh_global_lock);
if (lock != NULL)
rw_wunlock(lock);
rw_runlock(&pvh_global_lock);
PMAP_UNLOCK(pmap);
}
@ -3573,7 +3585,7 @@ pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
rw_assert(&pvh_global_lock, RA_LOCKED);
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
if ((mpde = pmap_allocpde(pmap, va, M_NOWAIT)) == NULL) {
if ((mpde = pmap_allocpde(pmap, va, NULL)) == NULL) {
CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
" in pmap %p", va, pmap);
return (FALSE);
@ -3735,7 +3747,9 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
/*
* If the page table page is mapped, we just increment
* the hold count, and activate it.
* the hold count, and activate it. Otherwise, we
* attempt to allocate a page table page. If this
* attempt fails, we don't retry. Instead, we give up.
*/
if (ptepa && (*ptepa & PG_V) != 0) {
if (*ptepa & PG_PS)
@ -3743,8 +3757,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
mpte = PHYS_TO_VM_PAGE(*ptepa & PG_FRAME);
mpte->wire_count++;
} else {
mpte = _pmap_allocpte(pmap, ptepindex,
M_NOWAIT);
mpte = _pmap_allocpte(pmap, ptepindex, NULL);
if (mpte == NULL)
return (mpte);
}
@ -3870,7 +3883,7 @@ pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,
PMAP_LOCK(pmap);
for (pa = ptepa | pmap_cache_bits(pat_mode, 1); pa < ptepa +
size; pa += NBPDR) {
pdpg = pmap_allocpde(pmap, addr, M_NOWAIT);
pdpg = pmap_allocpde(pmap, addr, NULL);
if (pdpg == NULL) {
/*
* The creation of mappings below is only an
@ -3888,7 +3901,7 @@ pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,
pde_store(pde, pa | PG_PS | PG_M | PG_A |
PG_U | PG_RW | PG_V);
pmap_resident_count_inc(pmap, NBPDR / PAGE_SIZE);
pmap_pde_mappings++;
atomic_add_long(&pmap_pde_mappings, 1);
} else {
/* Continue on if the PDE is already valid. */
pdpg->wire_count--;
@ -4020,7 +4033,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
continue;
if (srcptepaddr & PG_PS) {
dstmpde = pmap_allocpde(dst_pmap, addr, M_NOWAIT);
dstmpde = pmap_allocpde(dst_pmap, addr, NULL);
if (dstmpde == NULL)
break;
pde = (pd_entry_t *)
@ -4058,7 +4071,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
dstmpte->pindex == pmap_pde_pindex(addr))
dstmpte->wire_count++;
else if ((dstmpte = pmap_allocpte(dst_pmap,
addr, M_NOWAIT)) == NULL)
addr, NULL)) == NULL)
goto out;
dst_pte = (pt_entry_t *)
PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte));