d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix arm64 TLB invalidation with non-4k pages

Browse Source 
When using 16k or 64k pages atop will shift the address by more than
the needed amount for a tlbi instruction. Replace this with a new macro
to shift the address by 12 and use PAGE_SIZE in the for loop to let the
code work with any page size.

Reviewed by:	alc, markj
Sponsored by:	The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D34516
This commit is contained in:
Andrew Turner 2022-03-10 14:39:03 +00:00
parent 51f5cafcdc
commit 813738faba
1 changed files with 12 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
sys/arm64/arm64/pmap.c
View File

@ -360,6 +360,10 @@ void (*pmap_invalidate_vpipt_icache)(void);
#define COOKIE_TO_ASID(cookie) ((int)(cookie))
#define COOKIE_TO_EPOCH(cookie) ((int)((u_long)(cookie) >> 32))
#define TLBI_VA_SHIFT 12
#define TLBI_VA(addr) ((addr) >> TLBI_VA_SHIFT)
#define TLBI_VA_L3_INCR (L3_SIZE >> TLBI_VA_SHIFT)
static int superpages_enabled = 1;
SYSCTL_INT(_vm_pmap, OID_AUTO, superpages_enabled,
CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &superpages_enabled, 0,
@ -1248,11 +1252,11 @@ pmap_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only)
PMAP_ASSERT_STAGE1(pmap);
dsb(ishst);
r = TLBI_VA(va);
if (pmap == kernel_pmap) {
r = atop(va);
pmap_invalidate_kernel(r, final_only);
} else {
r = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)) | atop(va);
r |= ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie));
pmap_invalidate_user(r, final_only);
}
dsb(ish);
@ -1273,15 +1277,15 @@ pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
dsb(ishst);
if (pmap == kernel_pmap) {
start = atop(sva);
end = atop(eva);
for (r = start; r < end; r++)
start = TLBI_VA(sva);
end = TLBI_VA(eva);
for (r = start; r < end; r += TLBI_VA_L3_INCR)
pmap_invalidate_kernel(r, final_only);
} else {
start = end = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie));
start |= atop(sva);
end |= atop(eva);
for (r = start; r < end; r++)
start |= TLBI_VA(sva);
end |= TLBI_VA(eva);
for (r = start; r < end; r += TLBI_VA_L3_INCR)
pmap_invalidate_user(r, final_only);
}
dsb(ish);