diff --git a/sys/amd64/amd64/amd64_mem.c b/sys/amd64/amd64/amd64_mem.c index e50d3e72828b..0f7bc25ec720 100644 --- a/sys/amd64/amd64/amd64_mem.c +++ b/sys/amd64/amd64/amd64_mem.c @@ -35,6 +35,10 @@ __FBSDID("$FreeBSD$"); #include #include +#include +#include +#include + #include #include #include @@ -527,9 +531,9 @@ static int amd64_mrset(struct mem_range_softc *sc, struct mem_range_desc *mrd, int *arg) { struct mem_range_desc *targ; - int error = 0; + int error, i; - switch(*arg) { + switch (*arg) { case MEMRANGE_SET_UPDATE: /* * Make sure that what's being asked for is even @@ -568,6 +572,21 @@ amd64_mrset(struct mem_range_softc *sc, struct mem_range_desc *mrd, int *arg) return (EOPNOTSUPP); } + /* + * Ensure that the direct map region does not contain any mappings + * that span MTRRs of different types. However, the fixed MTRRs can + * be ignored, because a large page mapping the first 1 MB of physical + * memory is a special case that the processor handles. The entire + * TLB will be invalidated by amd64_mrstore(), so pmap_demote_DMAP() + * needn't do it. + */ + i = (sc->mr_cap & MR686_FIXMTRR) ? MTRR_N64K + MTRR_N16K + MTRR_N4K : 0; + mrd = sc->mr_desc + i; + for (; i < sc->mr_ndesc; i++, mrd++) { + if (mrd->mr_flags & MDF_ACTIVE) + pmap_demote_DMAP(mrd->mr_base, mrd->mr_len, FALSE); + } + /* Update the hardware. */ amd64_mrstore(sc); @@ -657,6 +676,21 @@ amd64_mrinit(struct mem_range_softc *sc) if (mrd->mr_flags & MDF_ACTIVE) mrd->mr_flags |= MDF_FIRMWARE; } + + /* + * Ensure that the direct map region does not contain any mappings + * that span MTRRs of different types. However, the fixed MTRRs can + * be ignored, because a large page mapping the first 1 MB of physical + * memory is a special case that the processor handles. Invalidate + * any old TLB entries that might hold inconsistent memory type + * information. + */ + i = (sc->mr_cap & MR686_FIXMTRR) ? MTRR_N64K + MTRR_N16K + MTRR_N4K : 0; + mrd = sc->mr_desc + i; + for (; i < sc->mr_ndesc; i++, mrd++) { + if (mrd->mr_flags & MDF_ACTIVE) + pmap_demote_DMAP(mrd->mr_base, mrd->mr_len, TRUE); + } } /* diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index 020391756d87..e436cd7a47fe 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -462,7 +462,7 @@ create_pagetables(vm_paddr_t *firstaddr) if (ndmpdp < 4) /* Minimum 4GB of dirmap */ ndmpdp = 4; DMPDPphys = allocpages(firstaddr, NDMPML4E); - if (TRUE || (amd_feature & AMDID_PAGE1GB) == 0) + if ((amd_feature & AMDID_PAGE1GB) == 0) DMPDphys = allocpages(firstaddr, ndmpdp); dmaplimit = (vm_paddr_t)ndmpdp << PDPSHIFT; @@ -494,11 +494,16 @@ create_pagetables(vm_paddr_t *firstaddr) ((pdp_entry_t *)KPDPphys)[i + KPDPI] |= PG_RW | PG_V | PG_U; } - /* Now set up the direct map space using either 2MB or 1GB pages */ - /* Preset PG_M and PG_A because demotion expects it */ - if (TRUE || (amd_feature & AMDID_PAGE1GB) == 0) { + /* + * Now, set up the direct map region using either 2MB or 1GB pages. + * Later, if pmap_mapdev{_attr}() uses the direct map for non-write- + * back memory, pmap_change_attr() will demote any 2MB or 1GB page + * mappings that are partially used. + */ + if ((amd_feature & AMDID_PAGE1GB) == 0) { for (i = 0; i < NPDEPG * ndmpdp; i++) { ((pd_entry_t *)DMPDphys)[i] = (vm_paddr_t)i << PDRSHIFT; + /* Preset PG_M and PG_A because demotion expects it. */ ((pd_entry_t *)DMPDphys)[i] |= PG_RW | PG_V | PG_PS | PG_G | PG_M | PG_A; } @@ -512,6 +517,7 @@ create_pagetables(vm_paddr_t *firstaddr) for (i = 0; i < ndmpdp; i++) { ((pdp_entry_t *)DMPDPphys)[i] = (vm_paddr_t)i << PDPSHIFT; + /* Preset PG_M and PG_A because demotion expects it. */ ((pdp_entry_t *)DMPDPphys)[i] |= PG_RW | PG_V | PG_PS | PG_G | PG_M | PG_A; } @@ -4947,6 +4953,49 @@ pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode) return (error); } +/* + * Demotes any mapping within the direct map region that covers more than the + * specified range of physical addresses. This range's size must be a power + * of two and its starting address must be a multiple of its size. Since the + * demotion does not change any attributes of the mapping, a TLB invalidation + * is not mandatory. The caller may, however, request a TLB invalidation. + */ +void +pmap_demote_DMAP(vm_paddr_t base, vm_size_t len, boolean_t invalidate) +{ + pdp_entry_t *pdpe; + pd_entry_t *pde; + vm_offset_t va; + boolean_t changed; + + if (len < NBPDP) { + va = PHYS_TO_DMAP(base); + changed = FALSE; + PMAP_LOCK(kernel_pmap); + pdpe = pmap_pdpe(kernel_pmap, va); + if ((*pdpe & PG_V) == 0) + panic("pmap_demote_DMAP: invalid PDPE"); + if ((*pdpe & PG_PS) != 0) { + if (!pmap_demote_pdpe(kernel_pmap, pdpe, va)) + panic("pmap_demote_DMAP: PDPE failed"); + changed = TRUE; + } + if (len < NBPDR) { + pde = pmap_pdpe_to_pde(pdpe, va); + if ((*pde & PG_V) == 0) + panic("pmap_demote_DMAP: invalid PDE"); + if ((*pde & PG_PS) != 0) { + if (!pmap_demote_pde(kernel_pmap, pde, va)) + panic("pmap_demote_DMAP: PDE failed"); + changed = TRUE; + } + } + if (changed && invalidate) + pmap_invalidate_page(kernel_pmap, va); + PMAP_UNLOCK(kernel_pmap); + } +} + /* * perform the pmap work for mincore */ diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h index 53a4d6393b3f..45ccc6454715 100644 --- a/sys/amd64/include/pmap.h +++ b/sys/amd64/include/pmap.h @@ -307,6 +307,7 @@ extern vm_offset_t virtual_end; void pmap_bootstrap(vm_paddr_t *); int pmap_change_attr(vm_offset_t, vm_size_t, int); +void pmap_demote_DMAP(vm_paddr_t base, vm_size_t len, boolean_t invalidate); void pmap_init_pat(void); void pmap_kenter(vm_offset_t va, vm_paddr_t pa); void *pmap_kenter_temporary(vm_paddr_t pa, int i);