2005-01-05 20:17:21 +00:00
|
|
|
/*-
|
2017-11-20 19:43:44 +00:00
|
|
|
* SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
*
|
2003-11-08 04:39:22 +00:00
|
|
|
* Copyright (c) 2003 Peter Wemm.
|
1993-06-12 14:58:17 +00:00
|
|
|
* Copyright (c) 1991 Regents of the University of California.
|
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* This code is derived from software contributed to Berkeley by
|
|
|
|
* the Systems Programming Group of the University of Utah Computer
|
|
|
|
* Science Department and William Jolitz of UUNET Technologies Inc.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
2017-02-28 23:42:47 +00:00
|
|
|
* 3. Neither the name of the University nor the names of its contributors
|
1993-06-12 14:58:17 +00:00
|
|
|
* may be used to endorse or promote products derived from this software
|
|
|
|
* without specific prior written permission.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
|
|
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
|
|
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
|
|
* SUCH DAMAGE.
|
|
|
|
*
|
|
|
|
* Derived from hp300 version by Mike Hibler, this version by William
|
|
|
|
* Jolitz uses a recursive map [a pde points to the page directory] to
|
|
|
|
* map the page tables using the pagetables themselves. This is done to
|
|
|
|
* reduce the impact on kernel virtual memory for lots of sparse address
|
|
|
|
* space, and to reduce the cost of memory to each process.
|
|
|
|
*
|
1993-10-15 10:07:45 +00:00
|
|
|
* from: hp300: @(#)pmap.h 7.2 (Berkeley) 12/16/90
|
|
|
|
* from: @(#)pmap.h 7.4 (Berkeley) 5/12/91
|
1999-08-28 01:08:13 +00:00
|
|
|
* $FreeBSD$
|
1993-06-12 14:58:17 +00:00
|
|
|
*/
|
|
|
|
|
1994-11-14 14:12:24 +00:00
|
|
|
#ifndef _MACHINE_PMAP_H_
|
|
|
|
#define _MACHINE_PMAP_H_
|
1993-06-12 14:58:17 +00:00
|
|
|
|
1996-05-02 22:25:18 +00:00
|
|
|
/*
|
2005-12-06 21:09:01 +00:00
|
|
|
* Page-directory and page-table entries follow this format, with a few
|
1996-05-02 22:25:18 +00:00
|
|
|
* of the fields not present here and there, depending on a lot of things.
|
|
|
|
*/
|
|
|
|
/* ---- Intel Nomenclature ---- */
|
2013-10-05 21:22:35 +00:00
|
|
|
#define X86_PG_V 0x001 /* P Valid */
|
|
|
|
#define X86_PG_RW 0x002 /* R/W Read/Write */
|
|
|
|
#define X86_PG_U 0x004 /* U/S User/Supervisor */
|
|
|
|
#define X86_PG_NC_PWT 0x008 /* PWT Write through */
|
|
|
|
#define X86_PG_NC_PCD 0x010 /* PCD Cache disable */
|
|
|
|
#define X86_PG_A 0x020 /* A Accessed */
|
|
|
|
#define X86_PG_M 0x040 /* D Dirty */
|
|
|
|
#define X86_PG_PS 0x080 /* PS Page size (0=4k,1=2M) */
|
|
|
|
#define X86_PG_PTE_PAT 0x080 /* PAT PAT index */
|
|
|
|
#define X86_PG_G 0x100 /* G Global */
|
|
|
|
#define X86_PG_AVAIL1 0x200 /* / Available for system */
|
|
|
|
#define X86_PG_AVAIL2 0x400 /* < programmers use */
|
|
|
|
#define X86_PG_AVAIL3 0x800 /* \ */
|
|
|
|
#define X86_PG_PDE_PAT 0x1000 /* PAT PAT index */
|
2019-02-20 09:51:13 +00:00
|
|
|
#define X86_PG_PKU(idx) ((pt_entry_t)idx << 59)
|
2013-10-05 21:22:35 +00:00
|
|
|
#define X86_PG_NX (1ul<<63) /* No-execute */
|
|
|
|
#define X86_PG_AVAIL(x) (1ul << (x))
|
1996-05-02 22:25:18 +00:00
|
|
|
|
2013-10-05 21:22:35 +00:00
|
|
|
/* Page level cache control fields used to determine the PAT type */
|
|
|
|
#define X86_PG_PDE_CACHE (X86_PG_PDE_PAT | X86_PG_NC_PWT | X86_PG_NC_PCD)
|
|
|
|
#define X86_PG_PTE_CACHE (X86_PG_PTE_PAT | X86_PG_NC_PWT | X86_PG_NC_PCD)
|
|
|
|
|
2019-02-20 09:51:13 +00:00
|
|
|
/* Protection keys indexes */
|
|
|
|
#define PMAP_MAX_PKRU_IDX 0xf
|
|
|
|
#define X86_PG_PKU_MASK X86_PG_PKU(PMAP_MAX_PKRU_IDX)
|
|
|
|
|
2013-10-05 21:22:35 +00:00
|
|
|
/*
|
|
|
|
* Intel extended page table (EPT) bit definitions.
|
|
|
|
*/
|
|
|
|
#define EPT_PG_READ 0x001 /* R Read */
|
|
|
|
#define EPT_PG_WRITE 0x002 /* W Write */
|
|
|
|
#define EPT_PG_EXECUTE 0x004 /* X Execute */
|
|
|
|
#define EPT_PG_IGNORE_PAT 0x040 /* IPAT Ignore PAT */
|
|
|
|
#define EPT_PG_PS 0x080 /* PS Page size */
|
|
|
|
#define EPT_PG_A 0x100 /* A Accessed */
|
|
|
|
#define EPT_PG_M 0x200 /* D Dirty */
|
|
|
|
#define EPT_PG_MEMORY_TYPE(x) ((x) << 3) /* MT Memory Type */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Define the PG_xx macros in terms of the bits on x86 PTEs.
|
|
|
|
*/
|
|
|
|
#define PG_V X86_PG_V
|
|
|
|
#define PG_RW X86_PG_RW
|
|
|
|
#define PG_U X86_PG_U
|
|
|
|
#define PG_NC_PWT X86_PG_NC_PWT
|
|
|
|
#define PG_NC_PCD X86_PG_NC_PCD
|
|
|
|
#define PG_A X86_PG_A
|
|
|
|
#define PG_M X86_PG_M
|
|
|
|
#define PG_PS X86_PG_PS
|
|
|
|
#define PG_PTE_PAT X86_PG_PTE_PAT
|
|
|
|
#define PG_G X86_PG_G
|
|
|
|
#define PG_AVAIL1 X86_PG_AVAIL1
|
|
|
|
#define PG_AVAIL2 X86_PG_AVAIL2
|
|
|
|
#define PG_AVAIL3 X86_PG_AVAIL3
|
|
|
|
#define PG_PDE_PAT X86_PG_PDE_PAT
|
|
|
|
#define PG_NX X86_PG_NX
|
|
|
|
#define PG_PDE_CACHE X86_PG_PDE_CACHE
|
|
|
|
#define PG_PTE_CACHE X86_PG_PTE_CACHE
|
1996-05-02 22:25:18 +00:00
|
|
|
|
|
|
|
/* Our various interpretations of the above */
|
2013-10-05 21:22:35 +00:00
|
|
|
#define PG_W X86_PG_AVAIL3 /* "Wired" pseudoflag */
|
|
|
|
#define PG_MANAGED X86_PG_AVAIL2
|
|
|
|
#define EPT_PG_EMUL_V X86_PG_AVAIL(52)
|
|
|
|
#define EPT_PG_EMUL_RW X86_PG_AVAIL(53)
|
2017-02-26 19:54:02 +00:00
|
|
|
#define PG_PROMOTED X86_PG_AVAIL(54) /* PDE only */
|
2004-06-08 00:29:42 +00:00
|
|
|
#define PG_FRAME (0x000ffffffffff000ul)
|
2006-12-05 11:31:33 +00:00
|
|
|
#define PG_PS_FRAME (0x000fffffffe00000ul)
|
2019-05-24 23:26:17 +00:00
|
|
|
#define PG_PS_PDP_FRAME (0x000fffffc0000000ul)
|
2008-07-31 22:45:28 +00:00
|
|
|
|
2008-03-04 18:50:15 +00:00
|
|
|
/*
|
|
|
|
* Promotion to a 2MB (PDE) page mapping requires that the corresponding 4KB
|
|
|
|
* (PTE) page mappings have identical settings for the following fields:
|
|
|
|
*/
|
2013-10-05 21:22:35 +00:00
|
|
|
#define PG_PTE_PROMOTE (PG_NX | PG_MANAGED | PG_W | PG_G | PG_PTE_CACHE | \
|
2019-02-20 09:51:13 +00:00
|
|
|
PG_M | PG_A | PG_U | PG_RW | PG_V | PG_PKU_MASK)
|
2008-03-04 18:50:15 +00:00
|
|
|
|
1996-05-02 14:21:14 +00:00
|
|
|
/*
|
|
|
|
* Page Protection Exception bits
|
|
|
|
*/
|
|
|
|
|
|
|
|
#define PGEX_P 0x01 /* Protection violation vs. not present */
|
|
|
|
#define PGEX_W 0x02 /* during a Write cycle */
|
|
|
|
#define PGEX_U 0x04 /* access from User mode (UPL) */
|
2006-08-02 16:24:23 +00:00
|
|
|
#define PGEX_RSV 0x08 /* reserved PTE field is non-zero */
|
|
|
|
#define PGEX_I 0x10 /* during an instruction fetch */
|
2019-02-20 09:46:44 +00:00
|
|
|
#define PGEX_PK 0x20 /* protection key violation */
|
2019-06-08 20:26:04 +00:00
|
|
|
#define PGEX_SGX 0x8000 /* SGX-related */
|
1996-05-02 14:21:14 +00:00
|
|
|
|
2013-10-05 21:22:35 +00:00
|
|
|
/*
|
|
|
|
* undef the PG_xx macros that define bits in the regular x86 PTEs that
|
|
|
|
* have a different position in nested PTEs. This is done when compiling
|
|
|
|
* code that needs to be aware of the differences between regular x86 and
|
|
|
|
* nested PTEs.
|
|
|
|
*
|
|
|
|
* The appropriate bitmask will be calculated at runtime based on the pmap
|
|
|
|
* type.
|
|
|
|
*/
|
|
|
|
#ifdef AMD64_NPT_AWARE
|
|
|
|
#undef PG_AVAIL1 /* X86_PG_AVAIL1 aliases with EPT_PG_M */
|
|
|
|
#undef PG_G
|
|
|
|
#undef PG_A
|
|
|
|
#undef PG_M
|
|
|
|
#undef PG_PDE_PAT
|
|
|
|
#undef PG_PDE_CACHE
|
|
|
|
#undef PG_PTE_PAT
|
|
|
|
#undef PG_PTE_CACHE
|
|
|
|
#undef PG_RW
|
|
|
|
#undef PG_V
|
|
|
|
#endif
|
|
|
|
|
1996-05-02 14:21:14 +00:00
|
|
|
/*
|
2003-05-01 01:05:25 +00:00
|
|
|
* Pte related macros. This is complicated by having to deal with
|
|
|
|
* the sign extension of the 48th bit.
|
1996-05-02 14:21:14 +00:00
|
|
|
*/
|
amd64 pmap: LA57 AKA 5-level paging
Since LA57 was moved to the main SDM document with revision 072, it
seems that we should have a support for it, and silicons are coming.
This patch makes pmap support both LA48 and LA57 hardware. The
selection of page table level is done at startup, kernel always
receives control from loader with 4-level paging. It is not clear how
UEFI spec would adapt LA57, for instance it could hand out control in
LA57 mode sometimes.
To switch from LA48 to LA57 requires turning off long mode, requesting
LA57 in CR4, then re-entering long mode. This is somewhat delicate
and done in pmap_bootstrap_la57(). AP startup in LA57 mode is much
easier, we only need to toggle a bit in CR4 and load right value in CR3.
I decided to not change kernel map for now. Single PML5 entry is
created that points to the existing kernel_pml4 (KML4Phys) page, and a
pml5 entry to create our recursive mapping for vtopte()/vtopde().
This decision is motivated by the fact that we cannot overcommit for
KVA, so large space there is unusable until machines start providing
wider physical memory addressing. Another reason is that I do not
want to break our fragile autotuning, so the KVA expansion is not
included into this first step. Nice side effect is that minidumps are
compatible.
On the other hand, (very) large address space is definitely
immediately useful for some userspace applications.
For userspace, numbering of pte entries (or page table pages) is
always done for 5-level structures even if we operate in 4-level mode.
The pmap_is_la57() function is added to report the mode of the
specified pmap, this is done not to allow simultaneous 4-/5-levels
(which is not allowed by hw), but to accomodate for EPT which has
separate level control and in principle might not allow 5-leve EPT
despite x86 paging supports it. Anyway, it does not seems critical to
have 5-level EPT support now.
Tested by: pho (LA48 hardware)
Reviewed by: alc
Sponsored by: The FreeBSD Foundation
Differential revision: https://reviews.freebsd.org/D25273
2020-08-23 20:19:04 +00:00
|
|
|
#define KV4ADDR(l4, l3, l2, l1) ( \
|
2003-07-09 23:04:23 +00:00
|
|
|
((unsigned long)-1 << 47) | \
|
|
|
|
((unsigned long)(l4) << PML4SHIFT) | \
|
2003-05-01 01:05:25 +00:00
|
|
|
((unsigned long)(l3) << PDPSHIFT) | \
|
|
|
|
((unsigned long)(l2) << PDRSHIFT) | \
|
|
|
|
((unsigned long)(l1) << PAGE_SHIFT))
|
amd64 pmap: LA57 AKA 5-level paging
Since LA57 was moved to the main SDM document with revision 072, it
seems that we should have a support for it, and silicons are coming.
This patch makes pmap support both LA48 and LA57 hardware. The
selection of page table level is done at startup, kernel always
receives control from loader with 4-level paging. It is not clear how
UEFI spec would adapt LA57, for instance it could hand out control in
LA57 mode sometimes.
To switch from LA48 to LA57 requires turning off long mode, requesting
LA57 in CR4, then re-entering long mode. This is somewhat delicate
and done in pmap_bootstrap_la57(). AP startup in LA57 mode is much
easier, we only need to toggle a bit in CR4 and load right value in CR3.
I decided to not change kernel map for now. Single PML5 entry is
created that points to the existing kernel_pml4 (KML4Phys) page, and a
pml5 entry to create our recursive mapping for vtopte()/vtopde().
This decision is motivated by the fact that we cannot overcommit for
KVA, so large space there is unusable until machines start providing
wider physical memory addressing. Another reason is that I do not
want to break our fragile autotuning, so the KVA expansion is not
included into this first step. Nice side effect is that minidumps are
compatible.
On the other hand, (very) large address space is definitely
immediately useful for some userspace applications.
For userspace, numbering of pte entries (or page table pages) is
always done for 5-level structures even if we operate in 4-level mode.
The pmap_is_la57() function is added to report the mode of the
specified pmap, this is done not to allow simultaneous 4-/5-levels
(which is not allowed by hw), but to accomodate for EPT which has
separate level control and in principle might not allow 5-leve EPT
despite x86 paging supports it. Anyway, it does not seems critical to
have 5-level EPT support now.
Tested by: pho (LA48 hardware)
Reviewed by: alc
Sponsored by: The FreeBSD Foundation
Differential revision: https://reviews.freebsd.org/D25273
2020-08-23 20:19:04 +00:00
|
|
|
#define KV5ADDR(l5, l4, l3, l2, l1) ( \
|
|
|
|
((unsigned long)-1 << 56) | \
|
|
|
|
((unsigned long)(l5) << PML5SHIFT) | \
|
|
|
|
((unsigned long)(l4) << PML4SHIFT) | \
|
|
|
|
((unsigned long)(l3) << PDPSHIFT) | \
|
|
|
|
((unsigned long)(l2) << PDRSHIFT) | \
|
|
|
|
((unsigned long)(l1) << PAGE_SHIFT))
|
2003-05-01 01:05:25 +00:00
|
|
|
|
amd64 pmap: LA57 AKA 5-level paging
Since LA57 was moved to the main SDM document with revision 072, it
seems that we should have a support for it, and silicons are coming.
This patch makes pmap support both LA48 and LA57 hardware. The
selection of page table level is done at startup, kernel always
receives control from loader with 4-level paging. It is not clear how
UEFI spec would adapt LA57, for instance it could hand out control in
LA57 mode sometimes.
To switch from LA48 to LA57 requires turning off long mode, requesting
LA57 in CR4, then re-entering long mode. This is somewhat delicate
and done in pmap_bootstrap_la57(). AP startup in LA57 mode is much
easier, we only need to toggle a bit in CR4 and load right value in CR3.
I decided to not change kernel map for now. Single PML5 entry is
created that points to the existing kernel_pml4 (KML4Phys) page, and a
pml5 entry to create our recursive mapping for vtopte()/vtopde().
This decision is motivated by the fact that we cannot overcommit for
KVA, so large space there is unusable until machines start providing
wider physical memory addressing. Another reason is that I do not
want to break our fragile autotuning, so the KVA expansion is not
included into this first step. Nice side effect is that minidumps are
compatible.
On the other hand, (very) large address space is definitely
immediately useful for some userspace applications.
For userspace, numbering of pte entries (or page table pages) is
always done for 5-level structures even if we operate in 4-level mode.
The pmap_is_la57() function is added to report the mode of the
specified pmap, this is done not to allow simultaneous 4-/5-levels
(which is not allowed by hw), but to accomodate for EPT which has
separate level control and in principle might not allow 5-leve EPT
despite x86 paging supports it. Anyway, it does not seems critical to
have 5-level EPT support now.
Tested by: pho (LA48 hardware)
Reviewed by: alc
Sponsored by: The FreeBSD Foundation
Differential revision: https://reviews.freebsd.org/D25273
2020-08-23 20:19:04 +00:00
|
|
|
#define UVADDR(l5, l4, l3, l2, l1) ( \
|
|
|
|
((unsigned long)(l5) << PML5SHIFT) | \
|
2003-07-09 23:04:23 +00:00
|
|
|
((unsigned long)(l4) << PML4SHIFT) | \
|
|
|
|
((unsigned long)(l3) << PDPSHIFT) | \
|
|
|
|
((unsigned long)(l2) << PDRSHIFT) | \
|
|
|
|
((unsigned long)(l1) << PAGE_SHIFT))
|
1993-06-12 14:58:17 +00:00
|
|
|
|
2013-08-17 19:49:08 +00:00
|
|
|
/*
|
|
|
|
* Number of kernel PML4 slots. Can be anywhere from 1 to 64 or so,
|
|
|
|
* but setting it larger than NDMPML4E makes no sense.
|
|
|
|
*
|
|
|
|
* Each slot provides .5 TB of kernel virtual space.
|
|
|
|
*/
|
|
|
|
#define NKPML4E 4
|
2003-05-23 05:04:54 +00:00
|
|
|
|
amd64 pmap: LA57 AKA 5-level paging
Since LA57 was moved to the main SDM document with revision 072, it
seems that we should have a support for it, and silicons are coming.
This patch makes pmap support both LA48 and LA57 hardware. The
selection of page table level is done at startup, kernel always
receives control from loader with 4-level paging. It is not clear how
UEFI spec would adapt LA57, for instance it could hand out control in
LA57 mode sometimes.
To switch from LA48 to LA57 requires turning off long mode, requesting
LA57 in CR4, then re-entering long mode. This is somewhat delicate
and done in pmap_bootstrap_la57(). AP startup in LA57 mode is much
easier, we only need to toggle a bit in CR4 and load right value in CR3.
I decided to not change kernel map for now. Single PML5 entry is
created that points to the existing kernel_pml4 (KML4Phys) page, and a
pml5 entry to create our recursive mapping for vtopte()/vtopde().
This decision is motivated by the fact that we cannot overcommit for
KVA, so large space there is unusable until machines start providing
wider physical memory addressing. Another reason is that I do not
want to break our fragile autotuning, so the KVA expansion is not
included into this first step. Nice side effect is that minidumps are
compatible.
On the other hand, (very) large address space is definitely
immediately useful for some userspace applications.
For userspace, numbering of pte entries (or page table pages) is
always done for 5-level structures even if we operate in 4-level mode.
The pmap_is_la57() function is added to report the mode of the
specified pmap, this is done not to allow simultaneous 4-/5-levels
(which is not allowed by hw), but to accomodate for EPT which has
separate level control and in principle might not allow 5-leve EPT
despite x86 paging supports it. Anyway, it does not seems critical to
have 5-level EPT support now.
Tested by: pho (LA48 hardware)
Reviewed by: alc
Sponsored by: The FreeBSD Foundation
Differential revision: https://reviews.freebsd.org/D25273
2020-08-23 20:19:04 +00:00
|
|
|
/*
|
|
|
|
* We use the same numbering of the page table pages for 5-level and
|
|
|
|
* 4-level paging structures.
|
|
|
|
*/
|
|
|
|
#define NUPML5E (NPML5EPG / 2) /* number of userland PML5
|
|
|
|
pages */
|
|
|
|
#define NUPML4E (NUPML5E * NPML4EPG) /* number of userland PML4
|
|
|
|
pages */
|
|
|
|
#define NUPDPE (NUPML4E * NPDPEPG) /* number of userland PDP
|
|
|
|
pages */
|
|
|
|
#define NUPDE (NUPDPE * NPDEPG) /* number of userland PD
|
|
|
|
entries */
|
|
|
|
#define NUP4ML4E (NPML4EPG / 2)
|
2003-05-23 05:04:54 +00:00
|
|
|
|
2010-11-26 19:36:26 +00:00
|
|
|
/*
|
2013-08-17 19:49:08 +00:00
|
|
|
* NDMPML4E is the maximum number of PML4 entries that will be
|
|
|
|
* used to implement the direct map. It must be a power of two,
|
|
|
|
* and should generally exceed NKPML4E. The maximum possible
|
|
|
|
* value is 64; using 128 will make the direct map intrude into
|
|
|
|
* the recursive page table map.
|
2010-11-26 19:36:26 +00:00
|
|
|
*/
|
2013-08-17 19:49:08 +00:00
|
|
|
#define NDMPML4E 8
|
1994-01-14 16:25:31 +00:00
|
|
|
|
1993-10-12 13:58:01 +00:00
|
|
|
/*
|
2013-08-17 19:49:08 +00:00
|
|
|
* These values control the layout of virtual memory. The starting address
|
2010-11-26 19:36:26 +00:00
|
|
|
* of the direct map, which is controlled by DMPML4I, must be a multiple of
|
|
|
|
* its size. (See the PHYS_TO_DMAP() and DMAP_TO_PHYS() macros.)
|
2013-08-17 19:49:08 +00:00
|
|
|
*
|
|
|
|
* Note: KPML4I is the index of the (single) level 4 page that maps
|
|
|
|
* the KVA that holds KERNBASE, while KPML4BASE is the index of the
|
|
|
|
* first level 4 page that maps VM_MIN_KERNEL_ADDRESS. If NKPML4E
|
|
|
|
* is 1, these are the same, otherwise KPML4BASE < KPML4I and extra
|
|
|
|
* level 4 PDEs are needed to map from VM_MIN_KERNEL_ADDRESS up to
|
|
|
|
* KERNBASE.
|
|
|
|
*
|
|
|
|
* (KPML4I combines with KPDPI to choose where KERNBASE starts.
|
|
|
|
* Or, in other words, KPML4I provides bits 39..47 of KERNBASE,
|
|
|
|
* and KPDPI provides bits 30..38.)
|
1993-10-12 13:58:01 +00:00
|
|
|
*/
|
amd64 pmap: LA57 AKA 5-level paging
Since LA57 was moved to the main SDM document with revision 072, it
seems that we should have a support for it, and silicons are coming.
This patch makes pmap support both LA48 and LA57 hardware. The
selection of page table level is done at startup, kernel always
receives control from loader with 4-level paging. It is not clear how
UEFI spec would adapt LA57, for instance it could hand out control in
LA57 mode sometimes.
To switch from LA48 to LA57 requires turning off long mode, requesting
LA57 in CR4, then re-entering long mode. This is somewhat delicate
and done in pmap_bootstrap_la57(). AP startup in LA57 mode is much
easier, we only need to toggle a bit in CR4 and load right value in CR3.
I decided to not change kernel map for now. Single PML5 entry is
created that points to the existing kernel_pml4 (KML4Phys) page, and a
pml5 entry to create our recursive mapping for vtopte()/vtopde().
This decision is motivated by the fact that we cannot overcommit for
KVA, so large space there is unusable until machines start providing
wider physical memory addressing. Another reason is that I do not
want to break our fragile autotuning, so the KVA expansion is not
included into this first step. Nice side effect is that minidumps are
compatible.
On the other hand, (very) large address space is definitely
immediately useful for some userspace applications.
For userspace, numbering of pte entries (or page table pages) is
always done for 5-level structures even if we operate in 4-level mode.
The pmap_is_la57() function is added to report the mode of the
specified pmap, this is done not to allow simultaneous 4-/5-levels
(which is not allowed by hw), but to accomodate for EPT which has
separate level control and in principle might not allow 5-leve EPT
despite x86 paging supports it. Anyway, it does not seems critical to
have 5-level EPT support now.
Tested by: pho (LA48 hardware)
Reviewed by: alc
Sponsored by: The FreeBSD Foundation
Differential revision: https://reviews.freebsd.org/D25273
2020-08-23 20:19:04 +00:00
|
|
|
#define PML4PML4I (NPML4EPG / 2) /* Index of recursive pml4 mapping */
|
|
|
|
#define PML5PML5I (NPML5EPG / 2) /* Index of recursive pml5 mapping */
|
2003-05-23 05:04:54 +00:00
|
|
|
|
2013-08-17 19:49:08 +00:00
|
|
|
#define KPML4BASE (NPML4EPG-NKPML4E) /* KVM at highest addresses */
|
2019-09-30 20:39:25 +00:00
|
|
|
#define DMPML4I rounddown(KPML4BASE-NDMPML4E, NDMPML4E) /* Below KVM */
|
2003-05-23 05:04:54 +00:00
|
|
|
|
2013-08-17 19:49:08 +00:00
|
|
|
#define KPML4I (NPML4EPG-1)
|
2008-07-08 22:59:17 +00:00
|
|
|
#define KPDPI (NPDPEPG-2) /* kernbase at -2GB */
|
First steps in rewriting locore.s, and making info useful
when the machine panics.
i386/i386/locore.s:
1) got rid of most .set directives that were being used like
#define's, and replaced them with appropriate #define's in
the appropriate header files (accessed via genassym).
2) added comments to header inclusions and global definitions,
and global variables
3) replaced some hardcoded constants with cpp defines (such as
PDESIZE and others)
4) aligned all comments to the same column to make them easier to
read
5) moved macro definitions for ENTRY, ALIGN, NOP, etc. to
/sys/i386/include/asmacros.h
6) added #ifdef BDE_DEBUGGER around all of Bruce's debugger code
7) added new global '_KERNend' to store last location+1 of kernel
8) cleaned up zeroing of bss so that only bss is zeroed
9) fix zeroing of page tables so that it really does zero them all
- not just if they follow the bss.
10) rewrote page table initialization code so that 1) works correctly
and 2) write protects the kernel text by default
11) properly initialize the kernel page directory, upages, p0stack PT,
and page tables. The previous scheme was more than a bit
screwy.
12) change allocation of virtual area of IO hole so that it is
fixed at KERNBASE + 0xa0000. The previous scheme put it
right after the kernel page tables and then later expected
it to be at KERNBASE +0xa0000
13) change multiple bogus settings of user read/write of various
areas of kernel VM - including the IO hole; we should never
be accessing the IO hole in user mode through the kernel
page tables
14) split kernel support routines such as bcopy, bzero, copyin,
copyout, etc. into a seperate file 'support.s'
15) split swtch and related routines into a seperate 'swtch.s'
16) split routines related to traps, syscalls, and interrupts
into a seperate file 'exception.s'
17) remove some unused global variables from locore that got
inserted by Garrett when he pulled them out of some .h
files.
i386/isa/icu.s:
1) clean up global variable declarations
2) move in declaration of astpending and netisr
i386/i386/pmap.c:
1) fix calculation of virtual_avail. It previously was calculated
to be right in the middle of the kernel page tables - not
a good place to start allocating kernel VM.
2) properly allocate kernel page dir/tables etc out of kernel map
- previously only took out 2 pages.
i386/i386/machdep.c:
1) modify boot() to print a warning that the system will reboot in
PANIC_REBOOT_WAIT_TIME amount of seconds, and let the user
abort with a key on the console. The machine will wait for
ever if a key is typed before the reboot. The default is
15 seconds, but can be set to 0 to mean don't wait at all,
-1 to mean wait forever, or any positive value to wait for
that many seconds.
2) print "Rebooting..." just before doing it.
kern/subr_prf.c:
1) remove PANICWAIT as it is deprecated by the change to machdep.c
i386/i386/trap.c:
1) add table of trap type strings and use it to print a real trap/
panic message rather than just a number. Lot's of work to
be done here, but this is the first step. Symbolic traceback
is in the TODO.
i386/i386/Makefile.i386:
1) add support in to build support.s, exception.s and swtch.s
...and various changes to various header files to make all of the
above happen.
1993-11-13 02:25:21 +00:00
|
|
|
|
2018-10-16 17:28:10 +00:00
|
|
|
/* Large map: index of the first and max last pml4 entry */
|
|
|
|
#define LMSPML4I (PML4PML4I + 1)
|
|
|
|
#define LMEPML4I (DMPML4I - 1)
|
|
|
|
|
1996-05-02 22:25:18 +00:00
|
|
|
/*
|
|
|
|
* XXX doesn't really belong here I guess...
|
|
|
|
*/
|
|
|
|
#define ISA_HOLE_START 0xa0000
|
|
|
|
#define ISA_HOLE_LENGTH (0x100000-ISA_HOLE_START)
|
|
|
|
|
Rewrite amd64 PCID implementation to follow an algorithm described in
the Vahalia' "Unix Internals" section 15.12 "Other TLB Consistency
Algorithms". The same algorithm is already utilized by the MIPS pmap
to handle ASIDs.
The PCID for the address space is now allocated per-cpu during context
switch to the thread using pmap, when no PCID on the cpu was ever
allocated, or the current PCID is invalidated. If the PCID is reused,
bit 63 of %cr3 can be set to avoid TLB flush.
Each cpu has PCID' algorithm generation count, which is saved in the
pmap pcpu block when pcpu PCID is allocated. On invalidation, the
pmap generation count is zeroed, which signals the context switch code
that already allocated PCID is no longer valid. The implication is
the TLB shootdown for the given cpu/address space, due to the
allocation of new PCID.
The pm_save mask is no longer has to be tracked, which (significantly)
reduces the targets of the TLB shootdown IPIs. Previously, pm_save
was reset only on pmap_invalidate_all(), which made it accumulate the
cpuids of all processors on which the thread was scheduled between
full TLB shootdowns.
Besides reducing the amount of TLB shootdowns and removing atomics to
update pm_saves in the context switch code, the algorithm is much
simpler than the maintanence of pm_save and selection of the right
address space in the shootdown IPI handler.
Reviewed by: alc
Tested by: pho
Sponsored by: The FreeBSD Foundation
MFC after: 3 weeks
2015-05-09 19:11:01 +00:00
|
|
|
#define PMAP_PCID_NONE 0xffffffff
|
|
|
|
#define PMAP_PCID_KERN 0
|
|
|
|
#define PMAP_PCID_OVERMAX 0x1000
|
Use PCID to optimize PTI.
Use PCID to avoid complete TLB shootdown when switching between user
and kernel mode with PTI enabled.
I use the model close to what I read about KAISER, user-mode PCID has
1:1 correspondence to the kernel-mode PCID, by setting bit 11 in PCID.
Full kernel-mode TLB shootdown is performed on context switches, since
KVA TLB invalidation only works in the current pmap. User-mode part of
TLB is flushed on the pmap activations as well.
Similarly, IPI TLB shootdowns must handle both kernel and user address
spaces for each address. Note that machines which implement PCID but
do not have INVPCID instructions, cause the usual complications in the
IPI handlers, due to the need to switch to the target PCID temporary.
This is racy, but because for PCID/no-INVPCID we disable the
interrupts in pmap_activate_sw(), IPI handler cannot see inconsistent
state of CPU PCID vs PCPU pmap/kcr3/ucr3 pointers.
On the other hand, on kernel/user switches, CR3_PCID_SAVE bit is set
and we do not clear TLB.
I can imagine alternative use of PCID, where there is only one PCID
allocated for the kernel pmap. Then, there is no need to shootdown
kernel TLB entries on context switch. But copyout(3) would need to
either use method similar to proc_rwmem() to access the userspace
data, or (in reverse) provide a temporal mapping for the kernel buffer
into user mode PCID and use trampoline for copy.
Reviewed by: markj (previous version)
Tested by: pho
Discussed with: alc (some aspects)
Sponsored by: The FreeBSD Foundation
MFC after: 3 weeks
Differential revision: https://reviews.freebsd.org/D13985
2018-01-27 11:49:37 +00:00
|
|
|
#define PMAP_PCID_OVERMAX_KERN 0x800
|
|
|
|
#define PMAP_PCID_USER_PT 0x800
|
|
|
|
|
2020-10-16 16:22:32 +00:00
|
|
|
#define PMAP_NO_CR3 0xffffffffffffffff
|
|
|
|
#define PMAP_UCR3_NOMASK 0xffffffffffffffff
|
Rewrite amd64 PCID implementation to follow an algorithm described in
the Vahalia' "Unix Internals" section 15.12 "Other TLB Consistency
Algorithms". The same algorithm is already utilized by the MIPS pmap
to handle ASIDs.
The PCID for the address space is now allocated per-cpu during context
switch to the thread using pmap, when no PCID on the cpu was ever
allocated, or the current PCID is invalidated. If the PCID is reused,
bit 63 of %cr3 can be set to avoid TLB flush.
Each cpu has PCID' algorithm generation count, which is saved in the
pmap pcpu block when pcpu PCID is allocated. On invalidation, the
pmap generation count is zeroed, which signals the context switch code
that already allocated PCID is no longer valid. The implication is
the TLB shootdown for the given cpu/address space, due to the
allocation of new PCID.
The pm_save mask is no longer has to be tracked, which (significantly)
reduces the targets of the TLB shootdown IPIs. Previously, pm_save
was reset only on pmap_invalidate_all(), which made it accumulate the
cpuids of all processors on which the thread was scheduled between
full TLB shootdowns.
Besides reducing the amount of TLB shootdowns and removing atomics to
update pm_saves in the context switch code, the algorithm is much
simpler than the maintanence of pm_save and selection of the right
address space in the shootdown IPI handler.
Reviewed by: alc
Tested by: pho
Sponsored by: The FreeBSD Foundation
MFC after: 3 weeks
2015-05-09 19:11:01 +00:00
|
|
|
|
1996-05-02 14:21:14 +00:00
|
|
|
#ifndef LOCORE
|
1996-09-08 16:57:53 +00:00
|
|
|
|
2004-06-19 14:58:35 +00:00
|
|
|
#include <sys/queue.h>
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
#include <sys/_cpuset.h>
|
2004-06-14 01:17:50 +00:00
|
|
|
#include <sys/_lock.h>
|
|
|
|
#include <sys/_mutex.h>
|
2019-02-20 09:51:13 +00:00
|
|
|
#include <sys/_pctrie.h>
|
|
|
|
#include <sys/_rangeset.h>
|
1996-09-08 16:57:53 +00:00
|
|
|
|
Sync back vmcontention branch into HEAD:
Replace the per-object resident and cached pages splay tree with a
path-compressed multi-digit radix trie.
Along with this, switch also the x86-specific handling of idle page
tables to using the radix trie.
This change is supposed to do the following:
- Allowing the acquisition of read locking for lookup operations of the
resident/cached pages collections as the per-vm_page_t splay iterators
are now removed.
- Increase the scalability of the operations on the page collections.
The radix trie does rely on the consumers locking to ensure atomicity of
its operations. In order to avoid deadlocks the bisection nodes are
pre-allocated in the UMA zone. This can be done safely because the
algorithm needs at maximum one new node per insert which means the
maximum number of the desired nodes is the number of available physical
frames themselves. However, not all the times a new bisection node is
really needed.
The radix trie implements path-compression because UFS indirect blocks
can lead to several objects with a very sparse trie, increasing the number
of levels to usually scan. It also helps in the nodes pre-fetching by
introducing the single node per-insert property.
This code is not generalized (yet) because of the possible loss of
performance by having much of the sizes in play configurable.
However, efforts to make this code more general and then reusable in
further different consumers might be really done.
The only KPI change is the removal of the function vm_page_splay() which
is now reaped.
The only KBI change, instead, is the removal of the left/right iterators
from struct vm_page, which are now reaped.
Further technical notes broken into mealpieces can be retrieved from the
svn branch:
http://svn.freebsd.org/base/user/attilio/vmcontention/
Sponsored by: EMC / Isilon storage division
In collaboration with: alc, jeff
Tested by: flo, pho, jhb, davide
Tested by: ian (arm)
Tested by: andreast (powerpc)
2013-03-18 00:25:02 +00:00
|
|
|
#include <vm/_vm_radix.h>
|
|
|
|
|
2003-05-01 01:05:25 +00:00
|
|
|
typedef u_int64_t pd_entry_t;
|
|
|
|
typedef u_int64_t pt_entry_t;
|
|
|
|
typedef u_int64_t pdp_entry_t;
|
|
|
|
typedef u_int64_t pml4_entry_t;
|
amd64 pmap: LA57 AKA 5-level paging
Since LA57 was moved to the main SDM document with revision 072, it
seems that we should have a support for it, and silicons are coming.
This patch makes pmap support both LA48 and LA57 hardware. The
selection of page table level is done at startup, kernel always
receives control from loader with 4-level paging. It is not clear how
UEFI spec would adapt LA57, for instance it could hand out control in
LA57 mode sometimes.
To switch from LA48 to LA57 requires turning off long mode, requesting
LA57 in CR4, then re-entering long mode. This is somewhat delicate
and done in pmap_bootstrap_la57(). AP startup in LA57 mode is much
easier, we only need to toggle a bit in CR4 and load right value in CR3.
I decided to not change kernel map for now. Single PML5 entry is
created that points to the existing kernel_pml4 (KML4Phys) page, and a
pml5 entry to create our recursive mapping for vtopte()/vtopde().
This decision is motivated by the fact that we cannot overcommit for
KVA, so large space there is unusable until machines start providing
wider physical memory addressing. Another reason is that I do not
want to break our fragile autotuning, so the KVA expansion is not
included into this first step. Nice side effect is that minidumps are
compatible.
On the other hand, (very) large address space is definitely
immediately useful for some userspace applications.
For userspace, numbering of pte entries (or page table pages) is
always done for 5-level structures even if we operate in 4-level mode.
The pmap_is_la57() function is added to report the mode of the
specified pmap, this is done not to allow simultaneous 4-/5-levels
(which is not allowed by hw), but to accomodate for EPT which has
separate level control and in principle might not allow 5-leve EPT
despite x86 paging supports it. Anyway, it does not seems critical to
have 5-level EPT support now.
Tested by: pho (LA48 hardware)
Reviewed by: alc
Sponsored by: The FreeBSD Foundation
Differential revision: https://reviews.freebsd.org/D25273
2020-08-23 20:19:04 +00:00
|
|
|
typedef u_int64_t pml5_entry_t;
|
2003-03-30 05:24:52 +00:00
|
|
|
|
1993-06-12 14:58:17 +00:00
|
|
|
/*
|
2009-03-22 18:56:26 +00:00
|
|
|
* Address of current address space page table maps and directories.
|
1993-06-12 14:58:17 +00:00
|
|
|
*/
|
1999-12-29 04:46:21 +00:00
|
|
|
#ifdef _KERNEL
|
amd64 pmap: LA57 AKA 5-level paging
Since LA57 was moved to the main SDM document with revision 072, it
seems that we should have a support for it, and silicons are coming.
This patch makes pmap support both LA48 and LA57 hardware. The
selection of page table level is done at startup, kernel always
receives control from loader with 4-level paging. It is not clear how
UEFI spec would adapt LA57, for instance it could hand out control in
LA57 mode sometimes.
To switch from LA48 to LA57 requires turning off long mode, requesting
LA57 in CR4, then re-entering long mode. This is somewhat delicate
and done in pmap_bootstrap_la57(). AP startup in LA57 mode is much
easier, we only need to toggle a bit in CR4 and load right value in CR3.
I decided to not change kernel map for now. Single PML5 entry is
created that points to the existing kernel_pml4 (KML4Phys) page, and a
pml5 entry to create our recursive mapping for vtopte()/vtopde().
This decision is motivated by the fact that we cannot overcommit for
KVA, so large space there is unusable until machines start providing
wider physical memory addressing. Another reason is that I do not
want to break our fragile autotuning, so the KVA expansion is not
included into this first step. Nice side effect is that minidumps are
compatible.
On the other hand, (very) large address space is definitely
immediately useful for some userspace applications.
For userspace, numbering of pte entries (or page table pages) is
always done for 5-level structures even if we operate in 4-level mode.
The pmap_is_la57() function is added to report the mode of the
specified pmap, this is done not to allow simultaneous 4-/5-levels
(which is not allowed by hw), but to accomodate for EPT which has
separate level control and in principle might not allow 5-leve EPT
despite x86 paging supports it. Anyway, it does not seems critical to
have 5-level EPT support now.
Tested by: pho (LA48 hardware)
Reviewed by: alc
Sponsored by: The FreeBSD Foundation
Differential revision: https://reviews.freebsd.org/D25273
2020-08-23 20:19:04 +00:00
|
|
|
#define addr_P4Tmap (KV4ADDR(PML4PML4I, 0, 0, 0))
|
|
|
|
#define addr_P4Dmap (KV4ADDR(PML4PML4I, PML4PML4I, 0, 0))
|
|
|
|
#define addr_P4DPmap (KV4ADDR(PML4PML4I, PML4PML4I, PML4PML4I, 0))
|
|
|
|
#define addr_P4ML4map (KV4ADDR(PML4PML4I, PML4PML4I, PML4PML4I, PML4PML4I))
|
|
|
|
#define addr_P4ML4pml4e (addr_PML4map + (PML4PML4I * sizeof(pml4_entry_t)))
|
|
|
|
#define P4Tmap ((pt_entry_t *)(addr_P4Tmap))
|
|
|
|
#define P4Dmap ((pd_entry_t *)(addr_P4Dmap))
|
|
|
|
|
|
|
|
#define addr_P5Tmap (KV5ADDR(PML5PML5I, 0, 0, 0, 0))
|
|
|
|
#define addr_P5Dmap (KV5ADDR(PML5PML5I, PML5PML5I, 0, 0, 0))
|
|
|
|
#define addr_P5DPmap (KV5ADDR(PML5PML5I, PML5PML5I, PML5PML5I, 0, 0))
|
|
|
|
#define addr_P5ML4map (KV5ADDR(PML5PML5I, PML5PML5I, PML5PML5I, PML5PML5I, 0))
|
|
|
|
#define addr_P5ML5map \
|
|
|
|
(KVADDR(PML5PML5I, PML5PML5I, PML5PML5I, PML5PML5I, PML5PML5I))
|
|
|
|
#define addr_P5ML5pml5e (addr_P5ML5map + (PML5PML5I * sizeof(pml5_entry_t)))
|
|
|
|
#define P5Tmap ((pt_entry_t *)(addr_P5Tmap))
|
|
|
|
#define P5Dmap ((pd_entry_t *)(addr_P5Dmap))
|
2003-05-23 05:04:54 +00:00
|
|
|
|
2013-02-06 04:53:00 +00:00
|
|
|
extern int nkpt; /* Initial number of kernel page tables */
|
2011-03-26 06:21:05 +00:00
|
|
|
extern u_int64_t KPDPphys; /* physical address of kernel level 3 */
|
2003-05-23 05:04:54 +00:00
|
|
|
extern u_int64_t KPML4phys; /* physical address of kernel level 4 */
|
amd64 pmap: LA57 AKA 5-level paging
Since LA57 was moved to the main SDM document with revision 072, it
seems that we should have a support for it, and silicons are coming.
This patch makes pmap support both LA48 and LA57 hardware. The
selection of page table level is done at startup, kernel always
receives control from loader with 4-level paging. It is not clear how
UEFI spec would adapt LA57, for instance it could hand out control in
LA57 mode sometimes.
To switch from LA48 to LA57 requires turning off long mode, requesting
LA57 in CR4, then re-entering long mode. This is somewhat delicate
and done in pmap_bootstrap_la57(). AP startup in LA57 mode is much
easier, we only need to toggle a bit in CR4 and load right value in CR3.
I decided to not change kernel map for now. Single PML5 entry is
created that points to the existing kernel_pml4 (KML4Phys) page, and a
pml5 entry to create our recursive mapping for vtopte()/vtopde().
This decision is motivated by the fact that we cannot overcommit for
KVA, so large space there is unusable until machines start providing
wider physical memory addressing. Another reason is that I do not
want to break our fragile autotuning, so the KVA expansion is not
included into this first step. Nice side effect is that minidumps are
compatible.
On the other hand, (very) large address space is definitely
immediately useful for some userspace applications.
For userspace, numbering of pte entries (or page table pages) is
always done for 5-level structures even if we operate in 4-level mode.
The pmap_is_la57() function is added to report the mode of the
specified pmap, this is done not to allow simultaneous 4-/5-levels
(which is not allowed by hw), but to accomodate for EPT which has
separate level control and in principle might not allow 5-leve EPT
despite x86 paging supports it. Anyway, it does not seems critical to
have 5-level EPT support now.
Tested by: pho (LA48 hardware)
Reviewed by: alc
Sponsored by: The FreeBSD Foundation
Differential revision: https://reviews.freebsd.org/D25273
2020-08-23 20:19:04 +00:00
|
|
|
extern u_int64_t KPML5phys; /* physical address of kernel level 5 */
|
1993-06-12 14:58:17 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* virtual address to page table entry and
|
2005-12-06 21:09:01 +00:00
|
|
|
* to physical address.
|
1993-06-12 14:58:17 +00:00
|
|
|
* Note: these work recursively, thus vtopte of a pte will give
|
|
|
|
* the corresponding pde that in turn maps it.
|
|
|
|
*/
|
2003-05-23 05:04:54 +00:00
|
|
|
pt_entry_t *vtopte(vm_offset_t);
|
2006-03-14 00:01:56 +00:00
|
|
|
#define vtophys(va) pmap_kextract(((vm_offset_t) (va)))
|
2003-03-30 05:24:52 +00:00
|
|
|
|
2013-08-21 22:40:29 +00:00
|
|
|
#define pte_load_store(ptep, pte) atomic_swap_long(ptep, pte)
|
|
|
|
#define pte_load_clear(ptep) atomic_swap_long(ptep, 0)
|
|
|
|
#define pte_store(ptep, pte) do { \
|
|
|
|
*(u_long *)(ptep) = (u_long)(pte); \
|
|
|
|
} while (0)
|
|
|
|
#define pte_clear(ptep) pte_store(ptep, 0)
|
|
|
|
|
|
|
|
#define pde_store(pdep, pde) pte_store(pdep, pde)
|
2003-04-28 20:35:36 +00:00
|
|
|
|
2004-06-08 01:02:52 +00:00
|
|
|
extern pt_entry_t pg_nx;
|
|
|
|
|
2003-04-28 20:35:36 +00:00
|
|
|
#endif /* _KERNEL */
|
1997-07-17 04:34:03 +00:00
|
|
|
|
1993-06-12 14:58:17 +00:00
|
|
|
/*
|
|
|
|
* Pmap stuff
|
|
|
|
*/
|
1996-09-08 16:57:53 +00:00
|
|
|
struct pv_entry;
|
Shrink the amd64 pv entry from 48 bytes to about 24 bytes. On a machine
with large mmap files mapped into many processes, this saves hundreds of
megabytes of ram.
pv entries were individually allocated and had two tailq entries and two
pointers (or addresses). Each pv entry was linked to a vm_page_t and
a process's address space (pmap). It had the virtual address and a
pointer to the pmap.
This change replaces the individual allocation with a per-process
allocation system. A page ("pv chunk") is allocated and this provides
168 pv entries for that process. We can now eliminate one of the 16 byte
tailq entries because we can simply iterate through the pv chunks to find
all the pv entries for a process. We can eliminate one of the 8 byte
pointers because the location of the pv entry implies the containing
pv chunk, which has the pointer. After overheads from the pv chunk
bitmap and tailq linkage, this works out that each pv entry has an
effective size of 24.38 bytes.
Future work still required, and other problems:
* when running low on pv entries or system ram, we may need to defrag
the chunk pages and free any spares. The stats (vm.pmap.*) show that
this doesn't seem to be that much of a problem, but it can be done if
needed.
* running low on pv entries is now a much bigger problem. The old
get_pv_entry() routine just needed to reclaim one other pv entry.
Now, since they are per-process, we can only use pv entries that are
assigned to our current process, or by stealing an entire page worth
from another process. Under normal circumstances, the pmap_collect()
code should be able to dislodge some pv entries from the current
process. But if needed, it can still reclaim entire pv chunk pages
from other processes.
* This should port to i386 really easily, except there it would reduce
pv entries from 24 bytes to about 12 bytes.
(I have integrated Alan's recent changes.)
2006-04-03 21:36:01 +00:00
|
|
|
struct pv_chunk;
|
2000-05-21 12:50:18 +00:00
|
|
|
|
2016-05-10 09:58:51 +00:00
|
|
|
/*
|
|
|
|
* Locks
|
|
|
|
* (p) PV list lock
|
|
|
|
*/
|
2000-05-21 12:50:18 +00:00
|
|
|
struct md_page {
|
2016-05-10 09:58:51 +00:00
|
|
|
TAILQ_HEAD(, pv_entry) pv_list; /* (p) */
|
|
|
|
int pv_gen; /* (p) */
|
2009-07-12 23:31:20 +00:00
|
|
|
int pat_mode;
|
2000-05-21 12:50:18 +00:00
|
|
|
};
|
1993-06-12 14:58:17 +00:00
|
|
|
|
2013-09-20 17:06:49 +00:00
|
|
|
enum pmap_type {
|
|
|
|
PT_X86, /* regular x86 page tables */
|
|
|
|
PT_EPT, /* Intel's nested page tables */
|
|
|
|
PT_RVI, /* AMD's nested page tables */
|
|
|
|
};
|
|
|
|
|
Rewrite amd64 PCID implementation to follow an algorithm described in
the Vahalia' "Unix Internals" section 15.12 "Other TLB Consistency
Algorithms". The same algorithm is already utilized by the MIPS pmap
to handle ASIDs.
The PCID for the address space is now allocated per-cpu during context
switch to the thread using pmap, when no PCID on the cpu was ever
allocated, or the current PCID is invalidated. If the PCID is reused,
bit 63 of %cr3 can be set to avoid TLB flush.
Each cpu has PCID' algorithm generation count, which is saved in the
pmap pcpu block when pcpu PCID is allocated. On invalidation, the
pmap generation count is zeroed, which signals the context switch code
that already allocated PCID is no longer valid. The implication is
the TLB shootdown for the given cpu/address space, due to the
allocation of new PCID.
The pm_save mask is no longer has to be tracked, which (significantly)
reduces the targets of the TLB shootdown IPIs. Previously, pm_save
was reset only on pmap_invalidate_all(), which made it accumulate the
cpuids of all processors on which the thread was scheduled between
full TLB shootdowns.
Besides reducing the amount of TLB shootdowns and removing atomics to
update pm_saves in the context switch code, the algorithm is much
simpler than the maintanence of pm_save and selection of the right
address space in the shootdown IPI handler.
Reviewed by: alc
Tested by: pho
Sponsored by: The FreeBSD Foundation
MFC after: 3 weeks
2015-05-09 19:11:01 +00:00
|
|
|
struct pmap_pcids {
|
|
|
|
uint32_t pm_pcid;
|
|
|
|
uint32_t pm_gen;
|
|
|
|
};
|
|
|
|
|
2009-03-22 04:32:05 +00:00
|
|
|
/*
|
|
|
|
* The kernel virtual address (KVA) of the level 4 page table page is always
|
|
|
|
* within the direct map (DMAP) region.
|
|
|
|
*/
|
1993-06-12 14:58:17 +00:00
|
|
|
struct pmap {
|
2004-06-14 01:17:50 +00:00
|
|
|
struct mtx pm_mtx;
|
amd64 pmap: LA57 AKA 5-level paging
Since LA57 was moved to the main SDM document with revision 072, it
seems that we should have a support for it, and silicons are coming.
This patch makes pmap support both LA48 and LA57 hardware. The
selection of page table level is done at startup, kernel always
receives control from loader with 4-level paging. It is not clear how
UEFI spec would adapt LA57, for instance it could hand out control in
LA57 mode sometimes.
To switch from LA48 to LA57 requires turning off long mode, requesting
LA57 in CR4, then re-entering long mode. This is somewhat delicate
and done in pmap_bootstrap_la57(). AP startup in LA57 mode is much
easier, we only need to toggle a bit in CR4 and load right value in CR3.
I decided to not change kernel map for now. Single PML5 entry is
created that points to the existing kernel_pml4 (KML4Phys) page, and a
pml5 entry to create our recursive mapping for vtopte()/vtopde().
This decision is motivated by the fact that we cannot overcommit for
KVA, so large space there is unusable until machines start providing
wider physical memory addressing. Another reason is that I do not
want to break our fragile autotuning, so the KVA expansion is not
included into this first step. Nice side effect is that minidumps are
compatible.
On the other hand, (very) large address space is definitely
immediately useful for some userspace applications.
For userspace, numbering of pte entries (or page table pages) is
always done for 5-level structures even if we operate in 4-level mode.
The pmap_is_la57() function is added to report the mode of the
specified pmap, this is done not to allow simultaneous 4-/5-levels
(which is not allowed by hw), but to accomodate for EPT which has
separate level control and in principle might not allow 5-leve EPT
despite x86 paging supports it. Anyway, it does not seems critical to
have 5-level EPT support now.
Tested by: pho (LA48 hardware)
Reviewed by: alc
Sponsored by: The FreeBSD Foundation
Differential revision: https://reviews.freebsd.org/D25273
2020-08-23 20:19:04 +00:00
|
|
|
pml4_entry_t *pm_pmltop; /* KVA of top level page table */
|
|
|
|
pml4_entry_t *pm_pmltopu; /* KVA of user top page table */
|
2013-09-04 23:31:29 +00:00
|
|
|
uint64_t pm_cr3;
|
PTI for amd64.
The implementation of the Kernel Page Table Isolation (KPTI) for
amd64, first version. It provides a workaround for the 'meltdown'
vulnerability. PTI is turned off by default for now, enable with the
loader tunable vm.pmap.pti=1.
The pmap page table is split into kernel-mode table and user-mode
table. Kernel-mode table is identical to the non-PTI table, while
usermode table is obtained from kernel table by leaving userspace
mappings intact, but only leaving the following parts of the kernel
mapped:
kernel text (but not modules text)
PCPU
GDT/IDT/user LDT/task structures
IST stacks for NMI and doublefault handlers.
Kernel switches to user page table before returning to usermode, and
restores full kernel page table on the entry. Initial kernel-mode
stack for PTI trampoline is allocated in PCPU, it is only 16
qwords. Kernel entry trampoline switches page tables. then the
hardware trap frame is copied to the normal kstack, and execution
continues.
IST stacks are kept mapped and no trampoline is needed for
NMI/doublefault, but of course page table switch is performed.
On return to usermode, the trampoline is used again, iret frame is
copied to the trampoline stack, page tables are switched and iretq is
executed. The case of iretq faulting due to the invalid usermode
context is tricky, since the frame for fault is appended to the
trampoline frame. Besides copying the fault frame and original
(corrupted) frame to kstack, the fault frame must be patched to make
it look as if the fault occured on the kstack, see the comment in
doret_iret detection code in trap().
Currently kernel pages which are mapped during trampoline operation
are identical for all pmaps. They are registered using
pmap_pti_add_kva(). Besides initial registrations done during boot,
LDT and non-common TSS segments are registered if user requested their
use. In principle, they can be installed into kernel page table per
pmap with some work. Similarly, PCPU can be hidden from userspace
mapping using trampoline PCPU page, but again I do not see much
benefits besides complexity.
PDPE pages for the kernel half of the user page tables are
pre-allocated during boot because we need to know pml4 entries which
are copied to the top-level paging structure page, in advance on a new
pmap creation. I enforce this to avoid iterating over the all
existing pmaps if a new PDPE page is needed for PTI kernel mappings.
The iteration is a known problematic operation on i386.
The need to flush hidden kernel translations on the switch to user
mode make global tables (PG_G) meaningless and even harming, so PG_G
use is disabled for PTI case. Our existing use of PCID is
incompatible with PTI and is automatically disabled if PTI is
enabled. PCID can be forced on only for developer's benefit.
MCE is known to be broken, it requires IST stack to operate completely
correctly even for non-PTI case, and absolutely needs dedicated IST
stack because MCE delivery while trampoline did not switched from PTI
stack is fatal. The fix is pending.
Reviewed by: markj (partially)
Tested by: pho (previous version)
Discussed with: jeff, jhb
Sponsored by: The FreeBSD Foundation
MFC after: 2 weeks
2018-01-17 11:44:21 +00:00
|
|
|
uint64_t pm_ucr3;
|
Shrink the amd64 pv entry from 48 bytes to about 24 bytes. On a machine
with large mmap files mapped into many processes, this saves hundreds of
megabytes of ram.
pv entries were individually allocated and had two tailq entries and two
pointers (or addresses). Each pv entry was linked to a vm_page_t and
a process's address space (pmap). It had the virtual address and a
pointer to the pmap.
This change replaces the individual allocation with a per-process
allocation system. A page ("pv chunk") is allocated and this provides
168 pv entries for that process. We can now eliminate one of the 16 byte
tailq entries because we can simply iterate through the pv chunks to find
all the pv entries for a process. We can eliminate one of the 8 byte
pointers because the location of the pv entry implies the containing
pv chunk, which has the pointer. After overheads from the pv chunk
bitmap and tailq linkage, this works out that each pv entry has an
effective size of 24.38 bytes.
Future work still required, and other problems:
* when running low on pv entries or system ram, we may need to defrag
the chunk pages and free any spares. The stats (vm.pmap.*) show that
this doesn't seem to be that much of a problem, but it can be done if
needed.
* running low on pv entries is now a much bigger problem. The old
get_pv_entry() routine just needed to reclaim one other pv entry.
Now, since they are per-process, we can only use pv entries that are
assigned to our current process, or by stealing an entire page worth
from another process. Under normal circumstances, the pmap_collect()
code should be able to dislodge some pv entries from the current
process. But if needed, it can still reclaim entire pv chunk pages
from other processes.
* This should port to i386 really easily, except there it would reduce
pv entries from 24 bytes to about 12 bytes.
(I have integrated Alan's recent changes.)
2006-04-03 21:36:01 +00:00
|
|
|
TAILQ_HEAD(,pv_chunk) pm_pvchunk; /* list of mappings in pmap */
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
cpuset_t pm_active; /* active on cpus */
|
2013-09-20 17:06:49 +00:00
|
|
|
enum pmap_type pm_type; /* regular or nested tables */
|
1993-06-12 14:58:17 +00:00
|
|
|
struct pmap_statistics pm_stats; /* pmap statistics */
|
Sync back vmcontention branch into HEAD:
Replace the per-object resident and cached pages splay tree with a
path-compressed multi-digit radix trie.
Along with this, switch also the x86-specific handling of idle page
tables to using the radix trie.
This change is supposed to do the following:
- Allowing the acquisition of read locking for lookup operations of the
resident/cached pages collections as the per-vm_page_t splay iterators
are now removed.
- Increase the scalability of the operations on the page collections.
The radix trie does rely on the consumers locking to ensure atomicity of
its operations. In order to avoid deadlocks the bisection nodes are
pre-allocated in the UMA zone. This can be done safely because the
algorithm needs at maximum one new node per insert which means the
maximum number of the desired nodes is the number of available physical
frames themselves. However, not all the times a new bisection node is
really needed.
The radix trie implements path-compression because UFS indirect blocks
can lead to several objects with a very sparse trie, increasing the number
of levels to usually scan. It also helps in the nodes pre-fetching by
introducing the single node per-insert property.
This code is not generalized (yet) because of the possible loss of
performance by having much of the sizes in play configurable.
However, efforts to make this code more general and then reusable in
further different consumers might be really done.
The only KPI change is the removal of the function vm_page_splay() which
is now reaped.
The only KBI change, instead, is the removal of the left/right iterators
from struct vm_page, which are now reaped.
Further technical notes broken into mealpieces can be retrieved from the
svn branch:
http://svn.freebsd.org/base/user/attilio/vmcontention/
Sponsored by: EMC / Isilon storage division
In collaboration with: alc, jeff
Tested by: flo, pho, jhb, davide
Tested by: ian (arm)
Tested by: andreast (powerpc)
2013-03-18 00:25:02 +00:00
|
|
|
struct vm_radix pm_root; /* spare page table pages */
|
2013-09-20 17:06:49 +00:00
|
|
|
long pm_eptgen; /* EPT pmap generation id */
|
|
|
|
int pm_flags;
|
Rewrite amd64 PCID implementation to follow an algorithm described in
the Vahalia' "Unix Internals" section 15.12 "Other TLB Consistency
Algorithms". The same algorithm is already utilized by the MIPS pmap
to handle ASIDs.
The PCID for the address space is now allocated per-cpu during context
switch to the thread using pmap, when no PCID on the cpu was ever
allocated, or the current PCID is invalidated. If the PCID is reused,
bit 63 of %cr3 can be set to avoid TLB flush.
Each cpu has PCID' algorithm generation count, which is saved in the
pmap pcpu block when pcpu PCID is allocated. On invalidation, the
pmap generation count is zeroed, which signals the context switch code
that already allocated PCID is no longer valid. The implication is
the TLB shootdown for the given cpu/address space, due to the
allocation of new PCID.
The pm_save mask is no longer has to be tracked, which (significantly)
reduces the targets of the TLB shootdown IPIs. Previously, pm_save
was reset only on pmap_invalidate_all(), which made it accumulate the
cpuids of all processors on which the thread was scheduled between
full TLB shootdowns.
Besides reducing the amount of TLB shootdowns and removing atomics to
update pm_saves in the context switch code, the algorithm is much
simpler than the maintanence of pm_save and selection of the right
address space in the shootdown IPI handler.
Reviewed by: alc
Tested by: pho
Sponsored by: The FreeBSD Foundation
MFC after: 3 weeks
2015-05-09 19:11:01 +00:00
|
|
|
struct pmap_pcids pm_pcids[MAXCPU];
|
2019-02-20 09:51:13 +00:00
|
|
|
struct rangeset pm_pkru;
|
1993-06-12 14:58:17 +00:00
|
|
|
};
|
|
|
|
|
2013-10-05 21:22:35 +00:00
|
|
|
/* flags */
|
2013-12-20 05:50:22 +00:00
|
|
|
#define PMAP_NESTED_IPIMASK 0xff
|
|
|
|
#define PMAP_PDE_SUPERPAGE (1 << 8) /* supports 2MB superpages */
|
|
|
|
#define PMAP_EMULATE_AD_BITS (1 << 9) /* needs A/D bits emulation */
|
|
|
|
#define PMAP_SUPPORTS_EXEC_ONLY (1 << 10) /* execute only mappings ok */
|
2013-10-05 21:22:35 +00:00
|
|
|
|
1993-06-12 14:58:17 +00:00
|
|
|
typedef struct pmap *pmap_t;
|
|
|
|
|
1999-12-29 04:46:21 +00:00
|
|
|
#ifdef _KERNEL
|
2002-04-29 07:43:16 +00:00
|
|
|
extern struct pmap kernel_pmap_store;
|
|
|
|
#define kernel_pmap (&kernel_pmap_store)
|
2004-06-14 01:17:50 +00:00
|
|
|
|
|
|
|
#define PMAP_LOCK(pmap) mtx_lock(&(pmap)->pm_mtx)
|
|
|
|
#define PMAP_LOCK_ASSERT(pmap, type) \
|
|
|
|
mtx_assert(&(pmap)->pm_mtx, (type))
|
|
|
|
#define PMAP_LOCK_DESTROY(pmap) mtx_destroy(&(pmap)->pm_mtx)
|
|
|
|
#define PMAP_LOCK_INIT(pmap) mtx_init(&(pmap)->pm_mtx, "pmap", \
|
2004-09-29 19:20:40 +00:00
|
|
|
NULL, MTX_DEF | MTX_DUPOK)
|
2004-06-14 01:17:50 +00:00
|
|
|
#define PMAP_LOCKED(pmap) mtx_owned(&(pmap)->pm_mtx)
|
|
|
|
#define PMAP_MTX(pmap) (&(pmap)->pm_mtx)
|
|
|
|
#define PMAP_TRYLOCK(pmap) mtx_trylock(&(pmap)->pm_mtx)
|
|
|
|
#define PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->pm_mtx)
|
2013-10-05 21:22:35 +00:00
|
|
|
|
|
|
|
int pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags);
|
|
|
|
int pmap_emulate_accessed_dirty(pmap_t pmap, vm_offset_t va, int ftype);
|
1993-06-12 14:58:17 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
/*
|
|
|
|
* For each vm_page_t, there is a list of all currently valid virtual
|
2006-11-13 06:26:57 +00:00
|
|
|
* mappings of that page. An entry is a pv_entry_t, the list is pv_list.
|
1993-06-12 14:58:17 +00:00
|
|
|
*/
|
|
|
|
typedef struct pv_entry {
|
|
|
|
vm_offset_t pv_va; /* virtual address for mapping */
|
2013-03-02 14:19:08 +00:00
|
|
|
TAILQ_ENTRY(pv_entry) pv_next;
|
1993-06-12 14:58:17 +00:00
|
|
|
} *pv_entry_t;
|
|
|
|
|
Shrink the amd64 pv entry from 48 bytes to about 24 bytes. On a machine
with large mmap files mapped into many processes, this saves hundreds of
megabytes of ram.
pv entries were individually allocated and had two tailq entries and two
pointers (or addresses). Each pv entry was linked to a vm_page_t and
a process's address space (pmap). It had the virtual address and a
pointer to the pmap.
This change replaces the individual allocation with a per-process
allocation system. A page ("pv chunk") is allocated and this provides
168 pv entries for that process. We can now eliminate one of the 16 byte
tailq entries because we can simply iterate through the pv chunks to find
all the pv entries for a process. We can eliminate one of the 8 byte
pointers because the location of the pv entry implies the containing
pv chunk, which has the pointer. After overheads from the pv chunk
bitmap and tailq linkage, this works out that each pv entry has an
effective size of 24.38 bytes.
Future work still required, and other problems:
* when running low on pv entries or system ram, we may need to defrag
the chunk pages and free any spares. The stats (vm.pmap.*) show that
this doesn't seem to be that much of a problem, but it can be done if
needed.
* running low on pv entries is now a much bigger problem. The old
get_pv_entry() routine just needed to reclaim one other pv entry.
Now, since they are per-process, we can only use pv entries that are
assigned to our current process, or by stealing an entire page worth
from another process. Under normal circumstances, the pmap_collect()
code should be able to dislodge some pv entries from the current
process. But if needed, it can still reclaim entire pv chunk pages
from other processes.
* This should port to i386 really easily, except there it would reduce
pv entries from 24 bytes to about 12 bytes.
(I have integrated Alan's recent changes.)
2006-04-03 21:36:01 +00:00
|
|
|
/*
|
|
|
|
* pv_entries are allocated in chunks per-process. This avoids the
|
|
|
|
* need to track per-pmap assignments.
|
|
|
|
*/
|
|
|
|
#define _NPCM 3
|
|
|
|
#define _NPCPV 168
|
Fix the pv_chunks pc_lru tailq handling in reclaim_pv_chunk().
For processing, reclaim_pv_chunk() removes the pv_chunk from the lru
list, which makes pc_lru linkage invalid. Then the pmap lock is
released, which allows for other thread to free the last pv entry
allocated from the chunk and call free_pv_chunk(), which tries to
modify the invalid linkage.
Similarly, the chunk is inserted into the private tailq new_tail
temporary. Again, free_pv_chunk() might be run and corrupt the
linkage for the new_tail after the pmap lock is dropped.
This is a consequence of r299788 elimination of pvh_global_lock, which
allowed for reclaim to run in parallel with other pmap calls which
free pv chunks.
As a fix, do not remove the chunk from pc_lru queue, use a marker to
remember the position in the queue iteration. We can safely operate
on the chunks after the chunk's pmap is locked, we fetched the chunk
after the marker, and we checked that chunk pmap is same as we have
locked, because chunk removal from pc_lru requires both pv_chunk_mutex
and the pmap mutex owned.
Note that the fix lost an optimization which was present in the
previous algorithm. Namely, new_tail requeueing rotated the pv chunks
list so that reclaim didn't scan the same pv chunks that couldn't be
freed (because they contained a wired and/or superpage mapping) on
every invocation. An additional change is planned which would improve
this.
Reported and tested by: pho
Reviewed by: alc
Sponsored by: The FreeBSD Foundation
MFC after: 1 week
2017-10-16 15:16:24 +00:00
|
|
|
#define PV_CHUNK_HEADER \
|
|
|
|
pmap_t pc_pmap; \
|
|
|
|
TAILQ_ENTRY(pv_chunk) pc_list; \
|
|
|
|
uint64_t pc_map[_NPCM]; /* bitmap; 1 = free */ \
|
2012-05-18 05:36:04 +00:00
|
|
|
TAILQ_ENTRY(pv_chunk) pc_lru;
|
Fix the pv_chunks pc_lru tailq handling in reclaim_pv_chunk().
For processing, reclaim_pv_chunk() removes the pv_chunk from the lru
list, which makes pc_lru linkage invalid. Then the pmap lock is
released, which allows for other thread to free the last pv entry
allocated from the chunk and call free_pv_chunk(), which tries to
modify the invalid linkage.
Similarly, the chunk is inserted into the private tailq new_tail
temporary. Again, free_pv_chunk() might be run and corrupt the
linkage for the new_tail after the pmap lock is dropped.
This is a consequence of r299788 elimination of pvh_global_lock, which
allowed for reclaim to run in parallel with other pmap calls which
free pv chunks.
As a fix, do not remove the chunk from pc_lru queue, use a marker to
remember the position in the queue iteration. We can safely operate
on the chunks after the chunk's pmap is locked, we fetched the chunk
after the marker, and we checked that chunk pmap is same as we have
locked, because chunk removal from pc_lru requires both pv_chunk_mutex
and the pmap mutex owned.
Note that the fix lost an optimization which was present in the
previous algorithm. Namely, new_tail requeueing rotated the pv chunks
list so that reclaim didn't scan the same pv chunks that couldn't be
freed (because they contained a wired and/or superpage mapping) on
every invocation. An additional change is planned which would improve
this.
Reported and tested by: pho
Reviewed by: alc
Sponsored by: The FreeBSD Foundation
MFC after: 1 week
2017-10-16 15:16:24 +00:00
|
|
|
|
|
|
|
struct pv_chunk_header {
|
|
|
|
PV_CHUNK_HEADER
|
|
|
|
};
|
|
|
|
|
|
|
|
struct pv_chunk {
|
|
|
|
PV_CHUNK_HEADER
|
Shrink the amd64 pv entry from 48 bytes to about 24 bytes. On a machine
with large mmap files mapped into many processes, this saves hundreds of
megabytes of ram.
pv entries were individually allocated and had two tailq entries and two
pointers (or addresses). Each pv entry was linked to a vm_page_t and
a process's address space (pmap). It had the virtual address and a
pointer to the pmap.
This change replaces the individual allocation with a per-process
allocation system. A page ("pv chunk") is allocated and this provides
168 pv entries for that process. We can now eliminate one of the 16 byte
tailq entries because we can simply iterate through the pv chunks to find
all the pv entries for a process. We can eliminate one of the 8 byte
pointers because the location of the pv entry implies the containing
pv chunk, which has the pointer. After overheads from the pv chunk
bitmap and tailq linkage, this works out that each pv entry has an
effective size of 24.38 bytes.
Future work still required, and other problems:
* when running low on pv entries or system ram, we may need to defrag
the chunk pages and free any spares. The stats (vm.pmap.*) show that
this doesn't seem to be that much of a problem, but it can be done if
needed.
* running low on pv entries is now a much bigger problem. The old
get_pv_entry() routine just needed to reclaim one other pv entry.
Now, since they are per-process, we can only use pv entries that are
assigned to our current process, or by stealing an entire page worth
from another process. Under normal circumstances, the pmap_collect()
code should be able to dislodge some pv entries from the current
process. But if needed, it can still reclaim entire pv chunk pages
from other processes.
* This should port to i386 really easily, except there it would reduce
pv entries from 24 bytes to about 12 bytes.
(I have integrated Alan's recent changes.)
2006-04-03 21:36:01 +00:00
|
|
|
struct pv_entry pc_pventry[_NPCPV];
|
|
|
|
};
|
|
|
|
|
1999-12-29 04:46:21 +00:00
|
|
|
#ifdef _KERNEL
|
1993-06-12 14:58:17 +00:00
|
|
|
|
1995-03-16 18:17:34 +00:00
|
|
|
extern caddr_t CADDR1;
|
|
|
|
extern pt_entry_t *CMAP1;
|
|
|
|
extern vm_offset_t virtual_avail;
|
|
|
|
extern vm_offset_t virtual_end;
|
2014-03-21 17:17:19 +00:00
|
|
|
extern vm_paddr_t dmaplimit;
|
2016-10-31 18:37:05 +00:00
|
|
|
extern int pmap_pcid_enabled;
|
|
|
|
extern int invpcid_works;
|
1993-06-12 14:58:17 +00:00
|
|
|
|
2009-07-12 23:31:20 +00:00
|
|
|
#define pmap_page_get_memattr(m) ((vm_memattr_t)(m)->md.pat_mode)
|
2019-12-10 18:14:50 +00:00
|
|
|
#define pmap_page_is_write_mapped(m) (((m)->a.flags & PGA_WRITEABLE) != 0)
|
|
|
|
#define pmap_unmapbios(va, sz) pmap_unmapdev((va), (sz))
|
2004-06-13 03:44:11 +00:00
|
|
|
|
Rewrite amd64 PCID implementation to follow an algorithm described in
the Vahalia' "Unix Internals" section 15.12 "Other TLB Consistency
Algorithms". The same algorithm is already utilized by the MIPS pmap
to handle ASIDs.
The PCID for the address space is now allocated per-cpu during context
switch to the thread using pmap, when no PCID on the cpu was ever
allocated, or the current PCID is invalidated. If the PCID is reused,
bit 63 of %cr3 can be set to avoid TLB flush.
Each cpu has PCID' algorithm generation count, which is saved in the
pmap pcpu block when pcpu PCID is allocated. On invalidation, the
pmap generation count is zeroed, which signals the context switch code
that already allocated PCID is no longer valid. The implication is
the TLB shootdown for the given cpu/address space, due to the
allocation of new PCID.
The pm_save mask is no longer has to be tracked, which (significantly)
reduces the targets of the TLB shootdown IPIs. Previously, pm_save
was reset only on pmap_invalidate_all(), which made it accumulate the
cpuids of all processors on which the thread was scheduled between
full TLB shootdowns.
Besides reducing the amount of TLB shootdowns and removing atomics to
update pm_saves in the context switch code, the algorithm is much
simpler than the maintanence of pm_save and selection of the right
address space in the shootdown IPI handler.
Reviewed by: alc
Tested by: pho
Sponsored by: The FreeBSD Foundation
MFC after: 3 weeks
2015-05-09 19:11:01 +00:00
|
|
|
struct thread;
|
|
|
|
|
2018-08-14 16:37:14 +00:00
|
|
|
void pmap_activate_boot(pmap_t pmap);
|
Rewrite amd64 PCID implementation to follow an algorithm described in
the Vahalia' "Unix Internals" section 15.12 "Other TLB Consistency
Algorithms". The same algorithm is already utilized by the MIPS pmap
to handle ASIDs.
The PCID for the address space is now allocated per-cpu during context
switch to the thread using pmap, when no PCID on the cpu was ever
allocated, or the current PCID is invalidated. If the PCID is reused,
bit 63 of %cr3 can be set to avoid TLB flush.
Each cpu has PCID' algorithm generation count, which is saved in the
pmap pcpu block when pcpu PCID is allocated. On invalidation, the
pmap generation count is zeroed, which signals the context switch code
that already allocated PCID is no longer valid. The implication is
the TLB shootdown for the given cpu/address space, due to the
allocation of new PCID.
The pm_save mask is no longer has to be tracked, which (significantly)
reduces the targets of the TLB shootdown IPIs. Previously, pm_save
was reset only on pmap_invalidate_all(), which made it accumulate the
cpuids of all processors on which the thread was scheduled between
full TLB shootdowns.
Besides reducing the amount of TLB shootdowns and removing atomics to
update pm_saves in the context switch code, the algorithm is much
simpler than the maintanence of pm_save and selection of the right
address space in the shootdown IPI handler.
Reviewed by: alc
Tested by: pho
Sponsored by: The FreeBSD Foundation
MFC after: 3 weeks
2015-05-09 19:11:01 +00:00
|
|
|
void pmap_activate_sw(struct thread *);
|
2019-11-12 18:01:33 +00:00
|
|
|
void pmap_allow_2m_x_ept_recalculate(void);
|
2003-05-23 05:04:54 +00:00
|
|
|
void pmap_bootstrap(vm_paddr_t *);
|
2016-09-21 10:05:51 +00:00
|
|
|
int pmap_cache_bits(pmap_t pmap, int mode, boolean_t is_pde);
|
2006-08-11 19:22:57 +00:00
|
|
|
int pmap_change_attr(vm_offset_t, vm_size_t, int);
|
2019-10-16 22:12:34 +00:00
|
|
|
int pmap_change_prot(vm_offset_t, vm_size_t, vm_prot_t);
|
2010-10-27 16:46:37 +00:00
|
|
|
void pmap_demote_DMAP(vm_paddr_t base, vm_size_t len, boolean_t invalidate);
|
2018-10-16 17:28:10 +00:00
|
|
|
void pmap_flush_cache_range(vm_offset_t, vm_offset_t);
|
|
|
|
void pmap_flush_cache_phys_range(vm_paddr_t, vm_paddr_t, vm_memattr_t);
|
2006-05-01 22:07:00 +00:00
|
|
|
void pmap_init_pat(void);
|
2003-03-25 00:07:06 +00:00
|
|
|
void pmap_kenter(vm_offset_t va, vm_paddr_t pa);
|
2004-05-16 20:44:41 +00:00
|
|
|
void *pmap_kenter_temporary(vm_paddr_t pa, int i);
|
2005-12-06 21:09:01 +00:00
|
|
|
vm_paddr_t pmap_kextract(vm_offset_t);
|
2003-03-16 04:16:03 +00:00
|
|
|
void pmap_kremove(vm_offset_t);
|
2018-10-16 17:28:10 +00:00
|
|
|
int pmap_large_map(vm_paddr_t, vm_size_t, void **, vm_memattr_t);
|
|
|
|
void pmap_large_map_wb(void *sva, vm_size_t len);
|
|
|
|
void pmap_large_unmap(void *sva, vm_size_t len);
|
2006-08-11 19:22:57 +00:00
|
|
|
void *pmap_mapbios(vm_paddr_t, vm_size_t);
|
2003-03-25 00:07:06 +00:00
|
|
|
void *pmap_mapdev(vm_paddr_t, vm_size_t);
|
2006-08-11 19:22:57 +00:00
|
|
|
void *pmap_mapdev_attr(vm_paddr_t, vm_size_t, int);
|
2018-10-18 20:49:16 +00:00
|
|
|
void *pmap_mapdev_pciecfg(vm_paddr_t pa, vm_size_t size);
|
2019-05-16 13:28:48 +00:00
|
|
|
bool pmap_not_in_di(void);
|
2008-03-04 18:50:15 +00:00
|
|
|
boolean_t pmap_page_is_mapped(vm_page_t m);
|
2009-07-12 23:31:20 +00:00
|
|
|
void pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma);
|
2016-09-21 10:05:51 +00:00
|
|
|
void pmap_pinit_pml4(vm_page_t);
|
amd64 pmap: LA57 AKA 5-level paging
Since LA57 was moved to the main SDM document with revision 072, it
seems that we should have a support for it, and silicons are coming.
This patch makes pmap support both LA48 and LA57 hardware. The
selection of page table level is done at startup, kernel always
receives control from loader with 4-level paging. It is not clear how
UEFI spec would adapt LA57, for instance it could hand out control in
LA57 mode sometimes.
To switch from LA48 to LA57 requires turning off long mode, requesting
LA57 in CR4, then re-entering long mode. This is somewhat delicate
and done in pmap_bootstrap_la57(). AP startup in LA57 mode is much
easier, we only need to toggle a bit in CR4 and load right value in CR3.
I decided to not change kernel map for now. Single PML5 entry is
created that points to the existing kernel_pml4 (KML4Phys) page, and a
pml5 entry to create our recursive mapping for vtopte()/vtopde().
This decision is motivated by the fact that we cannot overcommit for
KVA, so large space there is unusable until machines start providing
wider physical memory addressing. Another reason is that I do not
want to break our fragile autotuning, so the KVA expansion is not
included into this first step. Nice side effect is that minidumps are
compatible.
On the other hand, (very) large address space is definitely
immediately useful for some userspace applications.
For userspace, numbering of pte entries (or page table pages) is
always done for 5-level structures even if we operate in 4-level mode.
The pmap_is_la57() function is added to report the mode of the
specified pmap, this is done not to allow simultaneous 4-/5-levels
(which is not allowed by hw), but to accomodate for EPT which has
separate level control and in principle might not allow 5-leve EPT
despite x86 paging supports it. Anyway, it does not seems critical to
have 5-level EPT support now.
Tested by: pho (LA48 hardware)
Reviewed by: alc
Sponsored by: The FreeBSD Foundation
Differential revision: https://reviews.freebsd.org/D25273
2020-08-23 20:19:04 +00:00
|
|
|
void pmap_pinit_pml5(vm_page_t);
|
Add support for pmap_enter(..., psind=1) to the amd64 pmap. In other words,
add support for explicitly requesting that pmap_enter() create a 2MB page
mapping. (Essentially, this feature allows the machine-independent layer to
create superpage mappings preemptively, and not wait for automatic promotion
to occur.)
Export pmap_ps_enabled() to the machine-independent layer.
Add a flag to pmap_pv_insert_pde() that specifies whether it should fail or
reclaim a PV entry when one is not available.
Refactor pmap_enter_pde() into two functions, one by the same name, that is
a general-purpose function for creating PDE PG_PS mappings, and another,
pmap_enter_2mpage(), that is used to prefault 2MB read- and/or execute-only
mappings for execve(2), mmap(2), and shmat(2).
Submitted by: Yufeng Zhou <yz70@rice.edu> (an earlier version)
Reviewed by: kib, markj
Tested by: pho
MFC after: 10 days
Differential Revision: https://reviews.freebsd.org/D11556
2017-07-23 06:33:58 +00:00
|
|
|
bool pmap_ps_enabled(pmap_t pmap);
|
2002-03-20 05:48:58 +00:00
|
|
|
void pmap_unmapdev(vm_offset_t, vm_size_t);
|
2002-07-12 07:56:11 +00:00
|
|
|
void pmap_invalidate_page(pmap_t, vm_offset_t);
|
|
|
|
void pmap_invalidate_range(pmap_t, vm_offset_t, vm_offset_t);
|
|
|
|
void pmap_invalidate_all(pmap_t);
|
2006-05-01 21:36:47 +00:00
|
|
|
void pmap_invalidate_cache(void);
|
2011-04-18 21:24:42 +00:00
|
|
|
void pmap_invalidate_cache_pages(vm_page_t *pages, int count);
|
2018-09-19 19:35:02 +00:00
|
|
|
void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva);
|
|
|
|
void pmap_force_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva);
|
2013-10-05 21:22:35 +00:00
|
|
|
void pmap_get_mapping(pmap_t pmap, vm_offset_t va, uint64_t *ptr, int *num);
|
2014-10-24 09:48:58 +00:00
|
|
|
boolean_t pmap_map_io_transient(vm_page_t *, vm_offset_t *, int, boolean_t);
|
|
|
|
void pmap_unmap_io_transient(vm_page_t *, vm_offset_t *, int, boolean_t);
|
PTI for amd64.
The implementation of the Kernel Page Table Isolation (KPTI) for
amd64, first version. It provides a workaround for the 'meltdown'
vulnerability. PTI is turned off by default for now, enable with the
loader tunable vm.pmap.pti=1.
The pmap page table is split into kernel-mode table and user-mode
table. Kernel-mode table is identical to the non-PTI table, while
usermode table is obtained from kernel table by leaving userspace
mappings intact, but only leaving the following parts of the kernel
mapped:
kernel text (but not modules text)
PCPU
GDT/IDT/user LDT/task structures
IST stacks for NMI and doublefault handlers.
Kernel switches to user page table before returning to usermode, and
restores full kernel page table on the entry. Initial kernel-mode
stack for PTI trampoline is allocated in PCPU, it is only 16
qwords. Kernel entry trampoline switches page tables. then the
hardware trap frame is copied to the normal kstack, and execution
continues.
IST stacks are kept mapped and no trampoline is needed for
NMI/doublefault, but of course page table switch is performed.
On return to usermode, the trampoline is used again, iret frame is
copied to the trampoline stack, page tables are switched and iretq is
executed. The case of iretq faulting due to the invalid usermode
context is tricky, since the frame for fault is appended to the
trampoline frame. Besides copying the fault frame and original
(corrupted) frame to kstack, the fault frame must be patched to make
it look as if the fault occured on the kstack, see the comment in
doret_iret detection code in trap().
Currently kernel pages which are mapped during trampoline operation
are identical for all pmaps. They are registered using
pmap_pti_add_kva(). Besides initial registrations done during boot,
LDT and non-common TSS segments are registered if user requested their
use. In principle, they can be installed into kernel page table per
pmap with some work. Similarly, PCPU can be hidden from userspace
mapping using trampoline PCPU page, but again I do not see much
benefits besides complexity.
PDPE pages for the kernel half of the user page tables are
pre-allocated during boot because we need to know pml4 entries which
are copied to the top-level paging structure page, in advance on a new
pmap creation. I enforce this to avoid iterating over the all
existing pmaps if a new PDPE page is needed for PTI kernel mappings.
The iteration is a known problematic operation on i386.
The need to flush hidden kernel translations on the switch to user
mode make global tables (PG_G) meaningless and even harming, so PG_G
use is disabled for PTI case. Our existing use of PCID is
incompatible with PTI and is automatically disabled if PTI is
enabled. PCID can be forced on only for developer's benefit.
MCE is known to be broken, it requires IST stack to operate completely
correctly even for non-PTI case, and absolutely needs dedicated IST
stack because MCE delivery while trampoline did not switched from PTI
stack is fatal. The fix is pending.
Reviewed by: markj (partially)
Tested by: pho (previous version)
Discussed with: jeff, jhb
Sponsored by: The FreeBSD Foundation
MFC after: 2 weeks
2018-01-17 11:44:21 +00:00
|
|
|
void pmap_pti_add_kva(vm_offset_t sva, vm_offset_t eva, bool exec);
|
|
|
|
void pmap_pti_remove_kva(vm_offset_t sva, vm_offset_t eva);
|
Use PCID to optimize PTI.
Use PCID to avoid complete TLB shootdown when switching between user
and kernel mode with PTI enabled.
I use the model close to what I read about KAISER, user-mode PCID has
1:1 correspondence to the kernel-mode PCID, by setting bit 11 in PCID.
Full kernel-mode TLB shootdown is performed on context switches, since
KVA TLB invalidation only works in the current pmap. User-mode part of
TLB is flushed on the pmap activations as well.
Similarly, IPI TLB shootdowns must handle both kernel and user address
spaces for each address. Note that machines which implement PCID but
do not have INVPCID instructions, cause the usual complications in the
IPI handlers, due to the need to switch to the target PCID temporary.
This is racy, but because for PCID/no-INVPCID we disable the
interrupts in pmap_activate_sw(), IPI handler cannot see inconsistent
state of CPU PCID vs PCPU pmap/kcr3/ucr3 pointers.
On the other hand, on kernel/user switches, CR3_PCID_SAVE bit is set
and we do not clear TLB.
I can imagine alternative use of PCID, where there is only one PCID
allocated for the kernel pmap. Then, there is no need to shootdown
kernel TLB entries on context switch. But copyout(3) would need to
either use method similar to proc_rwmem() to access the userspace
data, or (in reverse) provide a temporal mapping for the kernel buffer
into user mode PCID and use trampoline for copy.
Reviewed by: markj (previous version)
Tested by: pho
Discussed with: alc (some aspects)
Sponsored by: The FreeBSD Foundation
MFC after: 3 weeks
Differential revision: https://reviews.freebsd.org/D13985
2018-01-27 11:49:37 +00:00
|
|
|
void pmap_pti_pcid_invalidate(uint64_t ucr3, uint64_t kcr3);
|
|
|
|
void pmap_pti_pcid_invlpg(uint64_t ucr3, uint64_t kcr3, vm_offset_t va);
|
|
|
|
void pmap_pti_pcid_invlrng(uint64_t ucr3, uint64_t kcr3, vm_offset_t sva,
|
|
|
|
vm_offset_t eva);
|
2019-02-20 09:51:13 +00:00
|
|
|
int pmap_pkru_clear(pmap_t pmap, vm_offset_t sva, vm_offset_t eva);
|
|
|
|
int pmap_pkru_set(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
|
|
|
|
u_int keyidx, int flags);
|
2019-05-16 13:28:48 +00:00
|
|
|
void pmap_thread_init_invl_gen(struct thread *td);
|
2019-02-20 09:51:13 +00:00
|
|
|
int pmap_vmspace_copy(pmap_t dst_pmap, pmap_t src_pmap);
|
2019-08-18 23:07:56 +00:00
|
|
|
void pmap_page_array_startup(long count);
|
1999-12-29 04:46:21 +00:00
|
|
|
#endif /* _KERNEL */
|
1996-10-12 20:36:15 +00:00
|
|
|
|
2016-09-20 09:38:07 +00:00
|
|
|
/* Return various clipped indexes for a given VA */
|
|
|
|
static __inline vm_pindex_t
|
|
|
|
pmap_pte_index(vm_offset_t va)
|
|
|
|
{
|
|
|
|
|
|
|
|
return ((va >> PAGE_SHIFT) & ((1ul << NPTEPGSHIFT) - 1));
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline vm_pindex_t
|
|
|
|
pmap_pde_index(vm_offset_t va)
|
|
|
|
{
|
|
|
|
|
|
|
|
return ((va >> PDRSHIFT) & ((1ul << NPDEPGSHIFT) - 1));
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline vm_pindex_t
|
|
|
|
pmap_pdpe_index(vm_offset_t va)
|
|
|
|
{
|
|
|
|
|
|
|
|
return ((va >> PDPSHIFT) & ((1ul << NPDPEPGSHIFT) - 1));
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline vm_pindex_t
|
|
|
|
pmap_pml4e_index(vm_offset_t va)
|
|
|
|
{
|
|
|
|
|
|
|
|
return ((va >> PML4SHIFT) & ((1ul << NPML4EPGSHIFT) - 1));
|
|
|
|
}
|
|
|
|
|
amd64 pmap: LA57 AKA 5-level paging
Since LA57 was moved to the main SDM document with revision 072, it
seems that we should have a support for it, and silicons are coming.
This patch makes pmap support both LA48 and LA57 hardware. The
selection of page table level is done at startup, kernel always
receives control from loader with 4-level paging. It is not clear how
UEFI spec would adapt LA57, for instance it could hand out control in
LA57 mode sometimes.
To switch from LA48 to LA57 requires turning off long mode, requesting
LA57 in CR4, then re-entering long mode. This is somewhat delicate
and done in pmap_bootstrap_la57(). AP startup in LA57 mode is much
easier, we only need to toggle a bit in CR4 and load right value in CR3.
I decided to not change kernel map for now. Single PML5 entry is
created that points to the existing kernel_pml4 (KML4Phys) page, and a
pml5 entry to create our recursive mapping for vtopte()/vtopde().
This decision is motivated by the fact that we cannot overcommit for
KVA, so large space there is unusable until machines start providing
wider physical memory addressing. Another reason is that I do not
want to break our fragile autotuning, so the KVA expansion is not
included into this first step. Nice side effect is that minidumps are
compatible.
On the other hand, (very) large address space is definitely
immediately useful for some userspace applications.
For userspace, numbering of pte entries (or page table pages) is
always done for 5-level structures even if we operate in 4-level mode.
The pmap_is_la57() function is added to report the mode of the
specified pmap, this is done not to allow simultaneous 4-/5-levels
(which is not allowed by hw), but to accomodate for EPT which has
separate level control and in principle might not allow 5-leve EPT
despite x86 paging supports it. Anyway, it does not seems critical to
have 5-level EPT support now.
Tested by: pho (LA48 hardware)
Reviewed by: alc
Sponsored by: The FreeBSD Foundation
Differential revision: https://reviews.freebsd.org/D25273
2020-08-23 20:19:04 +00:00
|
|
|
static __inline vm_pindex_t
|
|
|
|
pmap_pml5e_index(vm_offset_t va)
|
|
|
|
{
|
|
|
|
|
|
|
|
return ((va >> PML5SHIFT) & ((1ul << NPML5EPGSHIFT) - 1));
|
|
|
|
}
|
|
|
|
|
1996-05-02 14:21:14 +00:00
|
|
|
#endif /* !LOCORE */
|
1993-06-12 14:58:17 +00:00
|
|
|
|
1994-11-14 14:12:24 +00:00
|
|
|
#endif /* !_MACHINE_PMAP_H_ */
|