Add pmap_enter(9) PMAP_ENTER_LARGEPAGE flag and implement it on amd64.
The flag requests entry of non-managed superpage mapping of size pagesizes[psind] into the page table. Pmap supports fake wiring of the largepage mappings. Only attributes of the largepage mapping can be changed by calling pmap_enter(9) over existing mapping, physical address of the page must be unchanged. Reviewed by: markj Tested by: pho Sponsored by: The FreeBSD Foundation MFC after: 1 week Differential revision: https://reviews.freebsd.org/D24652
This commit is contained in:
parent
cd6298d5c5
commit
6cadbcd203
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=365518
@ -6475,6 +6475,119 @@ pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,
|
|||||||
}
|
}
|
||||||
#endif /* VM_NRESERVLEVEL > 0 */
|
#endif /* VM_NRESERVLEVEL > 0 */
|
||||||
|
|
||||||
|
static int
|
||||||
|
pmap_enter_largepage(pmap_t pmap, vm_offset_t va, pt_entry_t newpte, int flags,
|
||||||
|
int psind)
|
||||||
|
{
|
||||||
|
vm_page_t mp;
|
||||||
|
pt_entry_t origpte, *pml4e, *pdpe, *pde, pten, PG_V;
|
||||||
|
|
||||||
|
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
|
||||||
|
KASSERT(psind > 0 && psind < MAXPAGESIZES,
|
||||||
|
("psind %d unexpected", psind));
|
||||||
|
KASSERT(((newpte & PG_FRAME) & (pagesizes[psind] - 1)) == 0,
|
||||||
|
("unaligned phys address %#lx newpte %#lx psind %d",
|
||||||
|
newpte & PG_FRAME, newpte, psind));
|
||||||
|
KASSERT((va & (pagesizes[psind] - 1)) == 0,
|
||||||
|
("unaligned va %#lx psind %d", va, psind));
|
||||||
|
KASSERT(va < VM_MAXUSER_ADDRESS,
|
||||||
|
("kernel mode non-transparent superpage")); /* XXXKIB */
|
||||||
|
KASSERT(va + pagesizes[psind] < VM_MAXUSER_ADDRESS,
|
||||||
|
("overflowing user map va %#lx psind %d", va, psind)); /* XXXKIB */
|
||||||
|
|
||||||
|
PG_V = pmap_valid_bit(pmap);
|
||||||
|
|
||||||
|
restart:
|
||||||
|
pten = newpte;
|
||||||
|
if (va < VM_MAXUSER_ADDRESS && pmap->pm_type == PT_X86)
|
||||||
|
pten |= pmap_pkru_get(pmap, va);
|
||||||
|
|
||||||
|
if (psind == 2) { /* 1G */
|
||||||
|
if (!pmap_pkru_same(pmap, va, va + NBPDP))
|
||||||
|
return (KERN_PROTECTION_FAILURE);
|
||||||
|
pml4e = pmap_pml4e(pmap, va);
|
||||||
|
if ((*pml4e & PG_V) == 0) {
|
||||||
|
mp = _pmap_allocpte(pmap, pmap_pml4e_pindex(va),
|
||||||
|
NULL, va);
|
||||||
|
if (mp == NULL) {
|
||||||
|
if ((flags & PMAP_ENTER_NOSLEEP) != 0)
|
||||||
|
return (KERN_RESOURCE_SHORTAGE);
|
||||||
|
PMAP_UNLOCK(pmap);
|
||||||
|
vm_wait(NULL);
|
||||||
|
PMAP_LOCK(pmap);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Restart at least to recalcuate the pkru
|
||||||
|
* key. Our caller must keep the map locked
|
||||||
|
* so no paging structure can be validated
|
||||||
|
* under us.
|
||||||
|
*/
|
||||||
|
goto restart;
|
||||||
|
}
|
||||||
|
pdpe = pmap_pdpe(pmap, va);
|
||||||
|
KASSERT(pdpe != NULL, ("va %#lx lost pdpe", va));
|
||||||
|
origpte = *pdpe;
|
||||||
|
MPASS(origpte == 0);
|
||||||
|
} else {
|
||||||
|
mp = PHYS_TO_VM_PAGE(*pml4e & PG_FRAME);
|
||||||
|
pdpe = pmap_pdpe(pmap, va);
|
||||||
|
KASSERT(pdpe != NULL, ("va %#lx lost pdpe", va));
|
||||||
|
origpte = *pdpe;
|
||||||
|
if ((origpte & PG_V) == 0)
|
||||||
|
mp->ref_count++;
|
||||||
|
}
|
||||||
|
KASSERT((origpte & PG_V) == 0 || ((origpte & PG_PS) != 0 &&
|
||||||
|
(origpte & PG_FRAME) == (newpte & PG_FRAME)),
|
||||||
|
("va %#lx changing 1G phys page pdpe %#lx newpte %#lx",
|
||||||
|
va, origpte, newpte));
|
||||||
|
if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0)
|
||||||
|
pmap->pm_stats.wired_count += NBPDP / PAGE_SIZE;
|
||||||
|
else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0)
|
||||||
|
pmap->pm_stats.wired_count -= NBPDP / PAGE_SIZE;
|
||||||
|
*pdpe = newpte;
|
||||||
|
} else /* (psind == 1) */ { /* 2M */
|
||||||
|
if (!pmap_pkru_same(pmap, va, va + NBPDR))
|
||||||
|
return (KERN_PROTECTION_FAILURE);
|
||||||
|
pde = pmap_pde(pmap, va);
|
||||||
|
if (pde == NULL) {
|
||||||
|
mp = _pmap_allocpte(pmap, pmap_pdpe_pindex(va),
|
||||||
|
NULL, va);
|
||||||
|
if (mp == NULL) {
|
||||||
|
if ((flags & PMAP_ENTER_NOSLEEP) != 0)
|
||||||
|
return (KERN_RESOURCE_SHORTAGE);
|
||||||
|
PMAP_UNLOCK(pmap);
|
||||||
|
vm_wait(NULL);
|
||||||
|
PMAP_LOCK(pmap);
|
||||||
|
goto restart;
|
||||||
|
}
|
||||||
|
pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mp));
|
||||||
|
pde = &pde[pmap_pde_index(va)];
|
||||||
|
origpte = *pde;
|
||||||
|
MPASS(origpte == 0);
|
||||||
|
} else {
|
||||||
|
pdpe = pmap_pdpe(pmap, va);
|
||||||
|
MPASS(pdpe != NULL && (*pdpe & PG_V) != 0);
|
||||||
|
mp = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME);
|
||||||
|
origpte = *pde;
|
||||||
|
if ((origpte & PG_V) == 0)
|
||||||
|
mp->ref_count++;
|
||||||
|
}
|
||||||
|
KASSERT((origpte & PG_V) == 0 || ((origpte & PG_PS) != 0 &&
|
||||||
|
(origpte & PG_FRAME) == (newpte & PG_FRAME)),
|
||||||
|
("va %#lx changing 2M phys page pde %#lx newpte %#lx",
|
||||||
|
va, origpte, newpte));
|
||||||
|
if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0)
|
||||||
|
pmap->pm_stats.wired_count += NBPDR / PAGE_SIZE;
|
||||||
|
else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0)
|
||||||
|
pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE;
|
||||||
|
*pde = newpte;
|
||||||
|
}
|
||||||
|
if ((origpte & PG_V) == 0)
|
||||||
|
pmap_resident_count_inc(pmap, pagesizes[psind] / PAGE_SIZE);
|
||||||
|
|
||||||
|
return (KERN_SUCCESS);
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Insert the given physical page (p) at
|
* Insert the given physical page (p) at
|
||||||
* the specified virtual address (v) in the
|
* the specified virtual address (v) in the
|
||||||
@ -6554,6 +6667,13 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
|
|||||||
|
|
||||||
lock = NULL;
|
lock = NULL;
|
||||||
PMAP_LOCK(pmap);
|
PMAP_LOCK(pmap);
|
||||||
|
if ((flags & PMAP_ENTER_LARGEPAGE) != 0) {
|
||||||
|
KASSERT((m->oflags & VPO_UNMANAGED) != 0,
|
||||||
|
("managed largepage va %#lx flags %#x", va, flags));
|
||||||
|
rv = pmap_enter_largepage(pmap, va, newpte | PG_PS, flags,
|
||||||
|
psind);
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
if (psind == 1) {
|
if (psind == 1) {
|
||||||
/* Assert the required virtual and physical alignment. */
|
/* Assert the required virtual and physical alignment. */
|
||||||
KASSERT((va & PDRMASK) == 0, ("pmap_enter: va unaligned"));
|
KASSERT((va & PDRMASK) == 0, ("pmap_enter: va unaligned"));
|
||||||
|
@ -106,6 +106,7 @@ extern vm_offset_t kernel_vm_end;
|
|||||||
*/
|
*/
|
||||||
#define PMAP_ENTER_NOSLEEP 0x00000100
|
#define PMAP_ENTER_NOSLEEP 0x00000100
|
||||||
#define PMAP_ENTER_WIRED 0x00000200
|
#define PMAP_ENTER_WIRED 0x00000200
|
||||||
|
#define PMAP_ENTER_LARGEPAGE 0x00000400
|
||||||
#define PMAP_ENTER_RESERVED 0xFF000000
|
#define PMAP_ENTER_RESERVED 0xFF000000
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
Loading…
Reference in New Issue
Block a user