d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Split the SLB mirror cache into two kinds of object, one for kernel maps

Browse Source 
which are similar to the previous ones, and one for user maps, which
are arrays of pointers into the SLB tree. This changes makes user SLB
updates atomic, closing a window for memory corruption. While here,
rearrange the allocation functions to make context switches faster.
This commit is contained in:
Nathan Whitehorn 2010-09-16 03:46:17 +00:00
parent c59528330a
commit 6416b9a85d
7 changed files with 110 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
sys/powerpc/aim/mmu_oea64.c
View File

@ -838,7 +838,7 @@ moea64_bootstrap_slb_prefault(vm_offset_t va, int large)
if (large)
entry.slbv |= SLBV_L;
slb_insert(kernel_pmap, cache, &entry);
slb_insert_kernel(entry.slbe, entry.slbv);
}
#endif
@ -2099,6 +2099,7 @@ moea64_pinit(mmu_t mmu, pmap_t pmap)
pmap->pm_slb_tree_root = slb_alloc_tree();
pmap->pm_slb = slb_alloc_user_cache();
pmap->pm_slb_len = 0;
}
#else
void

84
sys/powerpc/aim/slb.c
View File

@ -263,13 +263,14 @@ va_to_vsid(pmap_t pm, vm_offset_t va)
entry = user_va_to_slb_entry(pm, va);
if (entry == NULL)
return (allocate_vsid(pm, (uintptr_t)va >> ADDR_SR_SHFT, 0));
return (allocate_user_vsid(pm,
(uintptr_t)va >> ADDR_SR_SHFT, 0));
return ((entry->slbv & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT);
}
uint64_t
allocate_vsid(pmap_t pm, uint64_t esid, int large)
allocate_user_vsid(pmap_t pm, uint64_t esid, int large)
{
uint64_t vsid, slbv;
struct slbtnode *ua, *next, *inter;
@ -327,7 +328,7 @@ allocate_vsid(pmap_t pm, uint64_t esid, int large)
* SLB mapping, so pre-spill this entry.
*/
eieio();
slb_insert(pm, pm->pm_slb, slb);
slb_insert_user(pm, slb);
return (vsid);
}
@ -410,57 +411,68 @@ slb_alloc_tree(void)
(slbe & SLBE_ESID_MASK) > 16*SEGMENT_LENGTH) || \
(slbe & SLBE_ESID_MASK) > VM_MAX_KERNEL_ADDRESS)
void
slb_insert(pmap_t pm, struct slb *slbcache, struct slb *slb_entry)
slb_insert_kernel(uint64_t slbe, uint64_t slbv)
{
uint64_t slbe, slbv;
int i, j, to_spill;
struct slb *slbcache;
int i, j;
/* We don't want to be preempted while modifying the kernel map */
critical_enter();
to_spill = -1;
slbv = slb_entry->slbv;
slbe = slb_entry->slbe;
slbcache = PCPU_GET(slb);
/* Hunt for a likely candidate */
for (i = mftb() % 64, j = 0; j < 64; j++, i = (i+1) % 64) {
if (pm == kernel_pmap && i == USER_SR)
continue;
if (!(slbcache[i].slbe & SLBE_VALID)) {
to_spill = i;
break;
/* Check for an unused slot, abusing the USER_SR slot as a full flag */
if (slbcache[USER_SR].slbe == 0) {
for (i = 0; i < USER_SR; i++) {
if (!(slbcache[i].slbe & SLBE_VALID))
goto fillkernslb;
}
if (to_spill < 0 && (pm != kernel_pmap ||
SLB_SPILLABLE(slbcache[i].slbe)))
to_spill = i;
if (i == USER_SR)
slbcache[USER_SR].slbe = 1;
}
if (to_spill < 0)
panic("SLB spill on ESID %#lx, but no available candidates!\n",
(slbe & SLBE_ESID_MASK) >> SLBE_ESID_SHIFT);
for (i = mftb() % 64, j = 0; j < 64; j++, i = (i+1) % 64) {
if (i == USER_SR)
continue;
if (slbcache[to_spill].slbe & SLBE_VALID) {
/* Invalidate this first to avoid races */
slbcache[to_spill].slbe = 0;
mb();
if (SLB_SPILLABLE(slbcache[i].slbe))
break;
}
slbcache[to_spill].slbv = slbv;
slbcache[to_spill].slbe = slbe | (uint64_t)to_spill;
KASSERT(j < 64, ("All kernel SLB slots locked!"));
fillkernslb:
slbcache[i].slbv = slbv;
slbcache[i].slbe = slbe | (uint64_t)i;
/* If it is for this CPU, put it in the SLB right away */
if (pm == kernel_pmap && pmap_bootstrapped) {
if (pmap_bootstrapped) {
/* slbie not required */
__asm __volatile ("slbmte %0, %1" ::
"r"(slbcache[to_spill].slbv),
"r"(slbcache[to_spill].slbe));
"r"(slbcache[i].slbv), "r"(slbcache[i].slbe));
}
critical_exit();
}
void
slb_insert_user(pmap_t pm, struct slb *slb)
{
int i;
PMAP_LOCK_ASSERT(pm, MA_OWNED);
if (pm->pm_slb_len < 64) {
i = pm->pm_slb_len;
pm->pm_slb_len++;
} else {
i = mftb() % 64;
}
/* Note that this replacement is atomic with respect to trap_subr */
pm->pm_slb[i] = slb;
}
static void
slb_zone_init(void *dummy)
@ -468,18 +480,18 @@ slb_zone_init(void *dummy)
slbt_zone = uma_zcreate("SLB tree node", sizeof(struct slbtnode),
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM);
slb_cache_zone = uma_zcreate("SLB cache", 64*sizeof(struct slb),
slb_cache_zone = uma_zcreate("SLB cache", 64*sizeof(struct slb *),
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM);
}
struct slb *
struct slb **
slb_alloc_user_cache(void)
{
return (uma_zalloc(slb_cache_zone, M_ZERO));
}
void
slb_free_user_cache(struct slb *slb)
slb_free_user_cache(struct slb **slb)
{
uma_zfree(slb_cache_zone, slb);
}

18
sys/powerpc/aim/trap.c
View File

@ -445,17 +445,15 @@ syscall(struct trapframe *frame)
static int
handle_slb_spill(pmap_t pm, vm_offset_t addr)
{
struct slb kern_entry, *user_entry;
struct slb *user_entry;
uint64_t esid;
int i;
esid = (uintptr_t)addr >> ADDR_SR_SHFT;
if (pm == kernel_pmap) {
kern_entry.slbv = kernel_va_to_slbv(addr);
kern_entry.slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID;
slb_insert(pm, PCPU_GET(slb), &kern_entry);
slb_insert_kernel((esid << SLBE_ESID_SHIFT) | SLBE_VALID,
kernel_va_to_slbv(addr));
return (0);
}
@ -464,18 +462,18 @@ handle_slb_spill(pmap_t pm, vm_offset_t addr)
if (user_entry == NULL) {
/* allocate_vsid auto-spills it */
(void)allocate_vsid(pm, esid, 0);
(void)allocate_user_vsid(pm, esid, 0);
} else {
/*
* Check that another CPU has not already mapped this.
* XXX: Per-thread SLB caches would be better.
*/
for (i = 0; i < 64; i++)
if (pm->pm_slb[i].slbe == (user_entry->slbe | i))
for (i = 0; i < pm->pm_slb_len; i++)
if (pm->pm_slb[i] == user_entry)
break;
if (i == 64)
slb_insert(pm, pm->pm_slb, user_entry);
if (i == pm->pm_slb_len)
slb_insert_user(pm, user_entry);
}
PMAP_UNLOCK(pm);

66
sys/powerpc/aim/trap_subr64.S
View File

@ -49,45 +49,59 @@
* Requires that r28-r31 be scratch, with r28 initialized to the SLB cache
*/
restoresrs:
/*
* User SRs are loaded through a pointer to the current pmap.
*/
restore_usersrs:
GET_CPUINFO(%r28);
ld %r28,PC_USERSLB(%r28);
li %r29, 0 /* Set the counter to zero */
slbia
slbmfee %r31,%r29
clrrdi %r31,%r31,28
slbie %r31
instslb:
ld %r31, 8(%r28); /* Load SLBE */
instuserslb:
ld %r31, 0(%r28); /* Load SLB entry pointer */
cmpli 0, %r31, 0; /* If NULL, stop */
beqlr;
cmpli 0, %r31, 0; /* If SLBE is not valid, get the next */
beq nslb;
ld %r30, 0(%r28) /* Load SLBV */
ld %r30, 0(%r31) /* Load SLBV */
ld %r31, 8(%r31) /* Load SLBE */
or %r31, %r31, %r29 /* Set SLBE slot */
slbmte %r30, %r31; /* Install SLB entry */
nslb:
addi %r28, %r28, 16; /* Advance */
addi %r28, %r28, 8; /* Advance pointer */
addi %r29, %r29, 1;
cmpli 0, %r29, 64; /* Repeat if we are not at the end */
blt instslb;
blt instuserslb;
blr;
/*
* User SRs are loaded through a pointer to the current pmap.
* Kernel SRs are loaded directly from the PCPU fields
*/
#define RESTORE_USER_SRS() \
GET_CPUINFO(%r28); \
ld %r28,PC_USERSLB(%r28); \
bl restoresrs;
restore_kernsrs:
GET_CPUINFO(%r28);
addi %r28,%r28,PC_KERNSLB;
li %r29, 0 /* Set the counter to zero */
/*
* Kernel SRs are loaded directly from kernel_pmap_
*/
#define RESTORE_KERN_SRS() \
GET_CPUINFO(%r28); \
addi %r28,%r28,PC_KERNSLB; \
bl restoresrs;
slbia
slbmfee %r31,%r29
clrrdi %r31,%r31,28
slbie %r31
instkernslb:
ld %r31, 8(%r28); /* Load SLBE */
cmpli 0, %r31, 0; /* If SLBE is not valid, stop */
beqlr;
ld %r30, 0(%r28) /* Load SLBV */
slbmte %r30, %r31; /* Install SLB entry */
addi %r28, %r28, 16; /* Advance pointer */
addi %r29, %r29, 1;
cmpli 0, %r29, USER_SR; /* Repeat if we are not at the end */
blt instkernslb;
blr;
/*
* FRAME_SETUP assumes:
@ -237,7 +251,7 @@ nslb:
std %r30,(savearea+CPUSAVE_R30)(%r3); \
std %r31,(savearea+CPUSAVE_R31)(%r3); \
mflr %r27; /* preserve LR */ \
RESTORE_USER_SRS(); /* uses r28-r31 */ \
bl restore_usersrs; /* uses r28-r31 */ \
mtlr %r27; \
ld %r31,(savearea+CPUSAVE_R31)(%r3); \
ld %r30,(savearea+CPUSAVE_R30)(%r3); \
@ -432,7 +446,7 @@ realtrap:
ld %r1,PC_CURPCB(%r1)
mr %r27,%r28 /* Save LR, r29 */
mtsprg2 %r29
RESTORE_KERN_SRS() /* enable kernel mapping */
bl restore_kernsrs /* enable kernel mapping */
mfsprg2 %r29
mr %r28,%r27
ba s_trap
@ -482,7 +496,7 @@ u_trap:
ld %r1,PC_CURPCB(%r1)
mr %r27,%r28 /* Save LR, r29 */
mtsprg2 %r29
RESTORE_KERN_SRS() /* enable kernel mapping */
bl restore_kernsrs /* enable kernel mapping */
mfsprg2 %r29
mr %r28,%r27

2
sys/powerpc/include/pcpu.h
View File

@ -55,7 +55,7 @@ struct pmap;
#define PCPU_MD_AIM64_FIELDS \
struct slb pc_slb[64]; \
struct slb *pc_userslb;
struct slb **pc_userslb;
#ifdef __powerpc64__
#define PCPU_MD_AIM_FIELDS PCPU_MD_AIM64_FIELDS

12
sys/powerpc/include/pmap.h
View File

@ -93,7 +93,8 @@ struct pmap {
#ifdef __powerpc64__
struct slbtnode *pm_slb_tree_root;
struct slb *pm_slb;
struct slb **pm_slb;
int pm_slb_len;
#else
register_t pm_sr[16];
#endif
@ -142,14 +143,15 @@ uint64_t va_to_vsid(pmap_t pm, vm_offset_t va);
uint64_t kernel_va_to_slbv(vm_offset_t va);
struct slb *user_va_to_slb_entry(pmap_t pm, vm_offset_t va);
uint64_t allocate_vsid(pmap_t pm, uint64_t esid, int large);
uint64_t allocate_user_vsid(pmap_t pm, uint64_t esid, int large);
void free_vsid(pmap_t pm, uint64_t esid, int large);
void slb_insert(pmap_t pm, struct slb *dst, struct slb *);
void slb_insert_user(pmap_t pm, struct slb *slb);
void slb_insert_kernel(uint64_t slbe, uint64_t slbv);
struct slbtnode *slb_alloc_tree(void);
void slb_free_tree(pmap_t pm);
struct slb *slb_alloc_user_cache(void);
void slb_free_user_cache(struct slb *);
struct slb **slb_alloc_user_cache(void);
void slb_free_user_cache(struct slb **);
#else

4
sys/powerpc/include/sr.h
View File

@ -42,7 +42,11 @@
#define SR_VSID_MASK 0x00ffffff /* Virtual Segment ID mask */
/* Kernel segment register usage */
#ifdef __powerpc64__
#define USER_SR 63
#else
#define USER_SR 12
#endif
#define KERNEL_SR 13
#define KERNEL2_SR 14
#define KERNEL3_SR 15