d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

amd64: add pmap_get_pcid() helper

Browse Source 
Reviewed by:	markj
Tested by:	pho
Sponsored by:	The FreeBSD Foundation
Differential revision:	https://reviews.freebsd.org/D39890
This commit is contained in:
Konstantin Belousov 2023-05-01 14:02:59 +03:00
parent 86b61ccb34
commit 9e0143694a
4 changed files with 33 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
sys/amd64/amd64/efirt_machdep.c
View File

@ -298,6 +298,7 @@ int
efi_arch_enter(void)
{
pmap_t curpmap;
uint64_t cr3;
curpmap = PCPU_GET(curpmap);
PMAP_LOCK_ASSERT(curpmap, MA_OWNED);
@ -313,8 +314,10 @@ efi_arch_enter(void)
if (pmap_pcid_enabled && !invpcid_works)
PCPU_SET(curpmap, NULL);
load_cr3(VM_PAGE_TO_PHYS(efi_pmltop_page) | (pmap_pcid_enabled ?
curpmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid : 0));
cr3 = VM_PAGE_TO_PHYS(efi_pmltop_page);
if (pmap_pcid_enabled)
cr3 |= pmap_get_pcid(curpmap);
load_cr3(cr3);
/*
* If PCID is enabled, the clear CR3_PCID_SAVE bit in the loaded %cr3
* causes TLB invalidation.
@ -328,12 +331,16 @@ void
efi_arch_leave(void)
{
pmap_t curpmap;
uint64_t cr3;
curpmap = &curproc->p_vmspace->vm_pmap;
if (pmap_pcid_enabled && !invpcid_works)
PCPU_SET(curpmap, curpmap);
load_cr3(curpmap->pm_cr3 | (pmap_pcid_enabled ?
curpmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid : 0));
cr3 = curpmap->pm_cr3;
if (pmap_pcid_enabled) {
cr3 |= pmap_get_pcid(curpmap);
if (!invpcid_works)
PCPU_SET(curpmap, curpmap);
}
load_cr3(cr3);
if (!pmap_pcid_enabled)
invltlb();
vm_fault_enable_pagefaults(curthread->td_md.md_efirt_dis_pf);

20
sys/amd64/amd64/mp_machdep.c
View File

@ -767,7 +767,7 @@ invltlb_invpcid_handler(pmap_t smp_tlb_pmap)
(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
d.pcid = pmap_get_pcid(smp_tlb_pmap);
d.pad = 0;
d.addr = 0;
invpcid(&d, smp_tlb_pmap == kernel_pmap ? INVPCID_CTXGLOB :
@ -786,7 +786,7 @@ invltlb_invpcid_pti_handler(pmap_t smp_tlb_pmap)
(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
d.pcid = pmap_get_pcid(smp_tlb_pmap);
d.pad = 0;
d.addr = 0;
if (smp_tlb_pmap == kernel_pmap) {
@ -808,8 +808,6 @@ invltlb_invpcid_pti_handler(pmap_t smp_tlb_pmap)
static void
invltlb_pcid_handler(pmap_t smp_tlb_pmap)
{
uint32_t pcid;
#ifdef COUNT_XINVLTLB_HITS
xhits_gbl[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
@ -828,8 +826,8 @@ invltlb_pcid_handler(pmap_t smp_tlb_pmap)
* CPU.
*/
if (smp_tlb_pmap == PCPU_GET(curpmap)) {
pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
load_cr3(smp_tlb_pmap->pm_cr3 | pcid);
load_cr3(smp_tlb_pmap->pm_cr3 |
pmap_get_pcid(smp_tlb_pmap));
if (smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3)
PCPU_SET(ucr3_load_mask, ~CR3_PCID_SAVE);
}
@ -865,8 +863,7 @@ invlpg_invpcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1)
if (smp_tlb_pmap == PCPU_GET(curpmap) &&
smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3 &&
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid |
PMAP_PCID_USER_PT;
d.pcid = pmap_get_pcid(smp_tlb_pmap) | PMAP_PCID_USER_PT;
d.pad = 0;
d.addr = smp_tlb_addr1;
invpcid(&d, INVPCID_ADDR);
@ -890,7 +887,7 @@ invlpg_pcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1)
if (smp_tlb_pmap == PCPU_GET(curpmap) &&
(ucr3 = smp_tlb_pmap->pm_ucr3) != PMAP_NO_CR3 &&
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
pcid = pmap_get_pcid(smp_tlb_pmap);
kcr3 = smp_tlb_pmap->pm_cr3 | pcid | CR3_PCID_SAVE;
ucr3 |= pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE;
pmap_pti_pcid_invlpg(ucr3, kcr3, smp_tlb_addr1);
@ -944,8 +941,7 @@ invlrng_invpcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1,
if (smp_tlb_pmap == PCPU_GET(curpmap) &&
smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3 &&
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid |
PMAP_PCID_USER_PT;
d.pcid = pmap_get_pcid(smp_tlb_pmap) | PMAP_PCID_USER_PT;
d.pad = 0;
d.addr = smp_tlb_addr1;
do {
@ -978,7 +974,7 @@ invlrng_pcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1,
if (smp_tlb_pmap == PCPU_GET(curpmap) &&
(ucr3 = smp_tlb_pmap->pm_ucr3) != PMAP_NO_CR3 &&
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
pcid = pmap_get_pcid(smp_tlb_pmap);
kcr3 = smp_tlb_pmap->pm_cr3 | pcid | CR3_PCID_SAVE;
ucr3 |= pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE;
pmap_pti_pcid_invlrng(ucr3, kcr3, smp_tlb_addr1, smp_tlb_addr2);

16
sys/amd64/amd64/pmap.c
View File

@ -3076,7 +3076,6 @@ pmap_invalidate_page_pcid_cb(pmap_t pmap, vm_offset_t va,
struct invpcid_descr d;
uint64_t kcr3, ucr3;
uint32_t pcid;
u_int cpuid;
/*
* Because pm_pcid is recalculated on a context switch, we
@ -3095,9 +3094,7 @@ pmap_invalidate_page_pcid_cb(pmap_t pmap, vm_offset_t va,
PCPU_GET(ucr3_load_mask) != PMAP_UCR3_NOMASK)
return;
cpuid = PCPU_GET(cpuid);
pcid = pmap->pm_pcids[cpuid].pm_pcid;
pcid = pmap_get_pcid(pmap);
if (invpcid_works1) {
d.pcid = pcid | PMAP_PCID_USER_PT;
d.pad = 0;
@ -3172,7 +3169,6 @@ pmap_invalidate_range_pcid_cb(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
struct invpcid_descr d;
uint64_t kcr3, ucr3;
uint32_t pcid;
u_int cpuid;
CRITICAL_ASSERT(curthread);
@ -3181,9 +3177,7 @@ pmap_invalidate_range_pcid_cb(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
PCPU_GET(ucr3_load_mask) != PMAP_UCR3_NOMASK)
return;
cpuid = PCPU_GET(cpuid);
pcid = pmap->pm_pcids[cpuid].pm_pcid;
pcid = pmap_get_pcid(pmap);
if (invpcid_works1) {
d.pcid = pcid | PMAP_PCID_USER_PT;
d.pad = 0;
@ -3273,7 +3267,6 @@ pmap_invalidate_all_pcid_cb(pmap_t pmap, bool invpcid_works1)
struct invpcid_descr d;
uint64_t kcr3;
uint32_t pcid;
u_int cpuid;
if (pmap == kernel_pmap) {
if (invpcid_works1) {
@ -3284,9 +3277,8 @@ pmap_invalidate_all_pcid_cb(pmap_t pmap, bool invpcid_works1)
}
} else if (pmap == PCPU_GET(curpmap)) {
CRITICAL_ASSERT(curthread);
cpuid = PCPU_GET(cpuid);
pcid = pmap->pm_pcids[cpuid].pm_pcid;
pcid = pmap_get_pcid(pmap);
if (invpcid_works1) {
d.pcid = pcid;
d.pad = 0;
@ -10152,7 +10144,7 @@ pmap_activate_boot(pmap_t pmap)
if (pti) {
kcr3 = pmap->pm_cr3;
if (pmap_pcid_enabled)
kcr3 |= pmap->pm_pcids[cpuid].pm_pcid | CR3_PCID_SAVE;
kcr3 |= pmap_get_pcid(pmap) | CR3_PCID_SAVE;
} else {
kcr3 = PMAP_NO_CR3;
}

8
sys/amd64/include/pmap.h
View File

@ -533,6 +533,14 @@ pmap_invlpg(pmap_t pmap, vm_offset_t va)
}
#endif /* sys/pcpu.h && machine/cpufunc.h */
/* Return pcid for the pmap pmap on current cpu */
static __inline uint32_t
pmap_get_pcid(pmap_t pmap)
{
MPASS(pmap_pcid_enabled);
return (pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid);
}
#endif /* _KERNEL */
/* Return various clipped indexes for a given VA */