d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Ordinarily, during a superpage promotion or demotion within a pmap, the

Browse Source 
pmap's lock ensures that other operations on the pmap don't observe the
old mapping being broken before the new mapping is established.  However,
pmap_kextract() doesn't acquire the kernel pmap's lock, so it may observe
the broken mapping.  And, if it does, it returns an incorrect result.

This revision implements a lock-free solution to this problem in
pmap_update_entry() and pmap_kextract() because pmap_kextract() can't
acquire the kernel pmap's lock.

Reported by:	andrew, greg_unrelenting.technology
Reviewed by:	andrew, markj
Tested by:	greg_unrelenting.technology
X-MFC with:	r350579
Differential Revision:	https://reviews.freebsd.org/D21169
This commit is contained in:
Alan Cox 2019-08-08 06:26:34 +00:00
parent 47fec6f3b5
commit 56e66ce802
2 changed files with 40 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
sys/arm64/arm64/pmap.c
View File

@ -1128,40 +1128,35 @@ vm_paddr_t
pmap_kextract(vm_offset_t va)
{
pt_entry_t *pte, tpte;
vm_paddr_t pa;
int lvl;
if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) {
pa = DMAP_TO_PHYS(va);
} else {
pa = 0;
pte = pmap_pte(kernel_pmap, va, &lvl);
if (pte != NULL) {
tpte = pmap_load(pte);
pa = tpte & ~ATTR_MASK;
switch(lvl) {
case 1:
KASSERT((tpte & ATTR_DESCR_MASK) == L1_BLOCK,
("pmap_kextract: Invalid L1 pte found: %lx",
tpte & ATTR_DESCR_MASK));
pa |= (va & L1_OFFSET);
break;
case 2:
KASSERT((tpte & ATTR_DESCR_MASK) == L2_BLOCK,
("pmap_kextract: Invalid L2 pte found: %lx",
tpte & ATTR_DESCR_MASK));
pa |= (va & L2_OFFSET);
break;
case 3:
KASSERT((tpte & ATTR_DESCR_MASK) == L3_PAGE,
("pmap_kextract: Invalid L3 pte found: %lx",
tpte & ATTR_DESCR_MASK));
pa |= (va & L3_OFFSET);
break;
}
}
}
return (pa);
if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS)
return (DMAP_TO_PHYS(va));
pte = pmap_l1(kernel_pmap, va);
if (pte == NULL)
return (0);
/*
* A concurrent pmap_update_entry() will clear the entry's valid bit
* but leave the rest of the entry unchanged. Therefore, we treat a
* non-zero entry as being valid, and we ignore the valid bit when
* determining whether the entry maps a block, page, or table.
*/
tpte = pmap_load(pte);
if (tpte == 0)
return (0);
if ((tpte & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_BLOCK)
return ((tpte & ~ATTR_MASK) | (va & L1_OFFSET));
pte = pmap_l1_to_l2(&tpte, va);
tpte = pmap_load(pte);
if (tpte == 0)
return (0);
if ((tpte & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_BLOCK)
return ((tpte & ~ATTR_MASK) | (va & L2_OFFSET));
pte = pmap_l2_to_l3(&tpte, va);
tpte = pmap_load(pte);
if (tpte == 0)
return (0);
return ((tpte & ~ATTR_MASK) | (va & L3_OFFSET));
}
/***************************************************
@ -3021,8 +3016,12 @@ pmap_update_entry(pmap_t pmap, pd_entry_t *pte, pd_entry_t newpte,
intr = intr_disable();
critical_enter();
/* Clear the old mapping */
pmap_clear(pte);
/*
* Clear the old mapping's valid bit, but leave the rest of the entry
* unchanged, so that a lockless, concurrent pmap_kextract() can still
* lookup the physical address.
*/
pmap_clear_bits(pte, ATTR_DESCR_VALID);
pmap_invalidate_range_nopin(pmap, va, va + size);
/* Create the new mapping */

7
sys/arm64/include/pte.h
View File

@ -71,7 +71,12 @@ typedef uint64_t pt_entry_t; /* page table entry */
#define ATTR_DEFAULT (ATTR_AF | ATTR_SH(ATTR_SH_IS))
#define ATTR_DESCR_MASK 3
#define ATTR_DESCR_MASK 3
#define ATTR_DESCR_VALID 1
#define ATTR_DESCR_TYPE_MASK 2
#define ATTR_DESCR_TYPE_TABLE 2
#define ATTR_DESCR_TYPE_PAGE 2
#define ATTR_DESCR_TYPE_BLOCK 0
/* Level 0 table, 512GiB per entry */
#define L0_SHIFT 39