d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move the arm64 DMAP creation to C

Browse Source 
To simplify the creation of the direct map (DMAP) region on arm64 move
it from the pre-C code into pmap. This simplifies the DMAP creation
as we can use the notmal index macros, and should reduce the number
of pages needed to hold the level 1 tables to just those needed.

Reviewed by:	alc, dch
Tested by:	dch, kevans
Sponsored by:	The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D34568
This commit is contained in:
Andrew Turner 2022-03-15 17:05:48 +00:00
parent 40b97c41bc
commit bcd763b642
2 changed files with 203 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
sys/arm64/arm64/locore.S
View File

@ -38,7 +38,6 @@
#include <machine/vmparam.h>
#define VIRT_BITS 48
#define DMAP_TABLES ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT)
.globl kernbase
.set kernbase, KERNBASE
@ -378,7 +377,6 @@ LEND(get_virt_delta)
* - The Kernel L0 table (TTBR1)
* - The identity (PA = VA) L1 table
* - The identity (PA = VA) L0 table (TTBR0)
* - The DMAP L1 tables
*/
LENTRY(create_pagetables)
/* Save the Link register */
@ -476,13 +474,6 @@ common:
mov x10, #1
bl link_l0_pagetable
/* Link the DMAP tables */
ldr x8, =DMAP_MIN_ADDRESS
adrp x9, pagetable_dmap
add x9, x9, :lo12:pagetable_dmap
mov x10, #DMAP_TABLES
bl link_l0_pagetable
/*
* Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG.
* They are only needed early on, so the VA = PA map is uncached.
@ -771,6 +762,7 @@ END(abort)
* L0 bootstrap for user
* L0 for user
*/
.globl pagetable_l0_ttbr1
pagetable:
.space PAGE_SIZE
pagetable_l1_ttbr1:
@ -786,9 +778,6 @@ pagetable_l0_ttbr0_boostrap:
pagetable_l0_ttbr0:
.space PAGE_SIZE
.globl pagetable_dmap
pagetable_dmap:
.space PAGE_SIZE * DMAP_TABLES
pagetable_end:
el2_pagetable:

261
sys/arm64/arm64/pmap.c
View File

@ -290,8 +290,7 @@ vm_offset_t dmap_max_addr; /* The virtual address limit of the dmap */
CTASSERT((DMAP_MIN_ADDRESS & ~L0_OFFSET) == DMAP_MIN_ADDRESS);
CTASSERT((DMAP_MAX_ADDRESS & ~L0_OFFSET) == DMAP_MAX_ADDRESS);
#define DMAP_TABLES ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT)
extern pt_entry_t pagetable_dmap[];
extern pt_entry_t pagetable_l0_ttbr1[];
#define PHYSMAP_SIZE (2 * (VM_PHYSSEG_MAX - 1))
static vm_paddr_t physmap[PHYSMAP_SIZE];
@ -808,104 +807,248 @@ pmap_early_vtophys(vm_offset_t va)
return (pa_page | (va & PAR_LOW_MASK));
}
static vm_offset_t
pmap_bootstrap_dmap_l2(vm_offset_t *va, vm_paddr_t *pa, u_int *prev_l1_slot,
pt_entry_t **l2p, int i, vm_offset_t freemempos)
/* State of the bootstrapped DMAP page tables */
struct dmap_bootstrap_state {
vm_offset_t va;
vm_paddr_t pa;
pt_entry_t *l1;
pt_entry_t *l2;
pt_entry_t *l3;
u_int l0_slot;
u_int l1_slot;
u_int l2_slot;
vm_offset_t freemempos;
};
static void
pmap_bootstrap_dmap_l0_table(struct dmap_bootstrap_state *state)
{
vm_paddr_t l1_pa;
u_int l0_slot;
/* Link the level 0 table to a level 1 table */
l0_slot = pmap_l0_index(state->va);
if (l0_slot != state->l0_slot) {
MPASS(state->l0_slot < l0_slot ||
state->l0_slot == L0_ENTRIES);
/* Create a new L0 table entry */
state->l0_slot = l0_slot;
state->l1 = (pt_entry_t *)state->freemempos;
memset(state->l1, 0, PAGE_SIZE);
state->freemempos += PAGE_SIZE;
/* Reset lower levels */
state->l2 = NULL;
state->l3 = NULL;
state->l1_slot = Ln_ENTRIES;
state->l2_slot = Ln_ENTRIES;
l1_pa = pmap_early_vtophys((vm_offset_t)state->l1);
MPASS((l1_pa & Ln_TABLE_MASK) == 0);
MPASS(pagetable_l0_ttbr1[l0_slot] == 0);
pmap_store(&pagetable_l0_ttbr1[l0_slot], l1_pa |
TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0 | L0_TABLE);
}
KASSERT(state->l1 != NULL, ("%s: NULL l1", __func__));
}
static void
pmap_bootstrap_dmap_l1_table(struct dmap_bootstrap_state *state)
{
pt_entry_t *l2;
vm_paddr_t l2_pa;
u_int l1_slot, l2_slot;
u_int l1_slot;
/* Make sure there is a valid L0 -> L1 table */
pmap_bootstrap_dmap_l0_table(state);
/* Link the level 1 table to a level 2 table */
l1_slot = pmap_l1_index(state->va);
if (l1_slot != state->l1_slot) {
MPASS(state->l1_slot < l1_slot ||
state->l1_slot == Ln_ENTRIES);
/* Create a new L1 table entry */
state->l1_slot = l1_slot;
state->l2 = (pt_entry_t *)state->freemempos;
memset(state->l2, 0, PAGE_SIZE);
state->freemempos += PAGE_SIZE;
/* Reset lower levels */
state->l3 = NULL;
state->l2_slot = Ln_ENTRIES;
l2_pa = pmap_early_vtophys((vm_offset_t)state->l2);
MPASS((l2_pa & Ln_TABLE_MASK) == 0);
MPASS(state->l1[l1_slot] == 0);
pmap_store(&state->l1[l1_slot], l2_pa | TATTR_PXN_TABLE |
L1_TABLE);
}
KASSERT(state->l2 != NULL, ("%s: NULL l2", __func__));
}
static void
pmap_bootstrap_dmap_l2_table(struct dmap_bootstrap_state *state)
{
vm_paddr_t l3_pa;
u_int l2_slot;
/* Make sure there is a valid L1 -> L2 table */
pmap_bootstrap_dmap_l1_table(state);
/* Link the level 2 table to a level 3 table */
l2_slot = pmap_l2_index(state->va);
if (l2_slot != state->l2_slot) {
MPASS(state->l2_slot < l2_slot ||
state->l2_slot == Ln_ENTRIES);
/* Create a new L2 table entry */
state->l2_slot = l2_slot;
state->l3 = (pt_entry_t *)state->freemempos;
memset(state->l3, 0, PAGE_SIZE);
state->freemempos += PAGE_SIZE;
l3_pa = pmap_early_vtophys((vm_offset_t)state->l3);
MPASS((l3_pa & Ln_TABLE_MASK) == 0);
MPASS(state->l2[l2_slot] == 0);
pmap_store(&state->l2[l2_slot], l3_pa | TATTR_PXN_TABLE |
L2_TABLE);
}
KASSERT(state->l3 != NULL, ("%s: NULL l3", __func__));
}
static void
pmap_bootstrap_dmap_l2_block(struct dmap_bootstrap_state *state, int i)
{
u_int l2_slot;
bool first;
l2 = *l2p;
l1_slot = ((*va - DMAP_MIN_ADDRESS) >> L1_SHIFT);
if (l1_slot != *prev_l1_slot) {
*prev_l1_slot = l1_slot;
l2 = (pt_entry_t *)freemempos;
l2_pa = pmap_early_vtophys((vm_offset_t)l2);
freemempos += PAGE_SIZE;
if ((physmap[i + 1] - state->pa) < L2_SIZE)
return;
pmap_store(&pagetable_dmap[l1_slot],
(l2_pa & ~Ln_TABLE_MASK) |
TATTR_PXN_TABLE | L1_TABLE);
/* Make sure there is a valid L1 table */
pmap_bootstrap_dmap_l1_table(state);
memset(l2, 0, PAGE_SIZE);
}
KASSERT(l2 != NULL,
("pmap_bootstrap_dmap_l2: NULL l2 map"));
for (first = true; *va < DMAP_MAX_ADDRESS && *pa < physmap[i + 1];
*pa += L2_SIZE, *va += L2_SIZE) {
MPASS((state->va & L2_OFFSET) == 0);
for (first = true;
state->va < DMAP_MAX_ADDRESS &&
(physmap[i + 1] - state->pa) >= L2_SIZE;
state->va += L2_SIZE, state->pa += L2_SIZE) {
/*
* Stop if we are about to walk off the end of what the
* current L1 slot can address.
*/
if (!first && (*pa & L1_OFFSET) == 0)
if (!first && (state->pa & L1_OFFSET) == 0)
break;
first = false;
l2_slot = pmap_l2_index(*va);
pmap_store(&l2[l2_slot],
(*pa & ~L2_OFFSET) | ATTR_DEFAULT |
ATTR_S1_XN |
ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) |
l2_slot = pmap_l2_index(state->va);
MPASS((state->pa & L2_OFFSET) == 0);
MPASS(state->l2[l2_slot] == 0);
pmap_store(&state->l2[l2_slot], state->pa | ATTR_DEFAULT |
ATTR_S1_XN | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) |
L2_BLOCK);
}
MPASS(*va == (*pa - dmap_phys_base + DMAP_MIN_ADDRESS));
*l2p = l2;
MPASS(state->va == (state->pa - dmap_phys_base + DMAP_MIN_ADDRESS));
}
return (freemempos);
static void
pmap_bootstrap_dmap_l3_page(struct dmap_bootstrap_state *state, int i)
{
u_int l3_slot;
bool first;
if ((physmap[i + 1] - state->pa) < L3_SIZE)
return;
/* Make sure there is a valid L2 table */
pmap_bootstrap_dmap_l2_table(state);
MPASS((state->va & L3_OFFSET) == 0);
for (first = true;
state->va < DMAP_MAX_ADDRESS &&
(physmap[i + 1] - state->pa) >= L3_SIZE;
state->va += L3_SIZE, state->pa += L3_SIZE) {
/*
* Stop if we are about to walk off the end of what the
* current L2 slot can address.
*/
if (!first && (state->pa & L2_OFFSET) == 0)
break;
first = false;
l3_slot = pmap_l3_index(state->va);
MPASS((state->pa & L3_OFFSET) == 0);
MPASS(state->l3[l3_slot] == 0);
pmap_store(&state->l3[l3_slot], state->pa | ATTR_DEFAULT |
ATTR_S1_XN | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) |
L3_PAGE);
}
MPASS(state->va == (state->pa - dmap_phys_base + DMAP_MIN_ADDRESS));
}
static vm_offset_t
pmap_bootstrap_dmap(vm_offset_t kern_l1, vm_paddr_t min_pa,
vm_offset_t freemempos)
{
pt_entry_t *l2;
vm_offset_t va;
vm_paddr_t pa;
u_int l1_slot, prev_l1_slot;
struct dmap_bootstrap_state state;
int i;
dmap_phys_base = min_pa & ~L1_OFFSET;
dmap_phys_max = 0;
dmap_max_addr = 0;
l2 = NULL;
prev_l1_slot = -1;
state.l1 = state.l2 = state.l3 = NULL;
state.l0_slot = L0_ENTRIES;
state.l1_slot = Ln_ENTRIES;
state.l2_slot = Ln_ENTRIES;
state.freemempos = freemempos;
for (i = 0; i < (physmap_idx * 2); i += 2) {
pa = physmap[i] & ~L2_OFFSET;
va = pa - dmap_phys_base + DMAP_MIN_ADDRESS;
state.pa = physmap[i] & ~L3_OFFSET;
state.va = state.pa - dmap_phys_base + DMAP_MIN_ADDRESS;
/* Create L3 mappings at the start of the region */
if ((state.pa & L2_OFFSET) != 0)
pmap_bootstrap_dmap_l3_page(&state, i);
MPASS(state.pa <= physmap[i + 1]);
/* Create L2 mappings at the start of the region */
if ((pa & L1_OFFSET) != 0) {
freemempos = pmap_bootstrap_dmap_l2(&va, &pa,
&prev_l1_slot, &l2, i, freemempos);
}
if ((state.pa & L1_OFFSET) != 0)
pmap_bootstrap_dmap_l2_block(&state, i);
MPASS(state.pa <= physmap[i + 1]);
for (; va < DMAP_MAX_ADDRESS && pa < physmap[i + 1] &&
(physmap[i + 1] - pa) >= L1_SIZE;
pa += L1_SIZE, va += L1_SIZE) {
l1_slot = ((va - DMAP_MIN_ADDRESS) >> L1_SHIFT);
pmap_store(&pagetable_dmap[l1_slot],
(pa & ~L1_OFFSET) | ATTR_DEFAULT | ATTR_S1_XN |
ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | L1_BLOCK);
/* Create the main L1 block mappings */
for (; state.va < DMAP_MAX_ADDRESS &&
(physmap[i + 1] - state.pa) >= L1_SIZE;
state.va += L1_SIZE, state.pa += L1_SIZE) {
/* Make sure there is a valid L1 table */
pmap_bootstrap_dmap_l0_table(&state);
MPASS((state.pa & L1_OFFSET) == 0);
pmap_store(&state.l1[pmap_l1_index(state.va)],
state.pa | ATTR_DEFAULT | ATTR_S1_XN |
ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) |
L1_BLOCK);
}
MPASS(state.pa <= physmap[i + 1]);
/* Create L2 mappings at the end of the region */
if (pa < physmap[i + 1]) {
freemempos = pmap_bootstrap_dmap_l2(&va, &pa,
&prev_l1_slot, &l2, i, freemempos);
}
pmap_bootstrap_dmap_l2_block(&state, i);
MPASS(state.pa <= physmap[i + 1]);
if (pa > dmap_phys_max) {
dmap_phys_max = pa;
dmap_max_addr = va;
/* Create L3 mappings at the end of the region */
pmap_bootstrap_dmap_l3_page(&state, i);
MPASS(state.pa == physmap[i + 1]);
if (state.pa > dmap_phys_max) {
dmap_phys_max = state.pa;
dmap_max_addr = state.va;
}
}
cpu_tlb_flushID();
return (freemempos);
return (state.freemempos);
}
static vm_offset_t