d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Restructure the way the copyin/copyout segment is stored to prevent a

Browse Source 
concurrency bug. Since all SLB/SR entries were invalidated during an
exception, a decrementer exception could cause the user segment to be
invalidated during a copyin()/copyout() without a thread switch that
would cause it to be restored from the PCB, potentially causing the
operation to continue on invalid memory. This is now handled by explicit
restoration of segment 12 from the PCB on 32-bit systems and a check in
the Data Segment Exception handler on 64-bit.

While here, cause copyin()/copyout() to check whether the requested
user segment is already installed, saving some pipeline flushes, and
fix the synchronization primitives around the mtsr and slbmte
instructions to prevent accessing stale segments.

MFC after:	2 weeks
This commit is contained in:
Nathan Whitehorn 2010-10-30 23:07:30 +00:00
parent f5a1822131
commit 54c562081f
12 changed files with 60 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
sys/powerpc/aim/copyinout.c
View File

@ -81,9 +81,7 @@ static __inline void
set_user_sr(pmap_t pm, const void *addr)
{
struct slb *slb;
register_t esid, vsid, slb1, slb2;
esid = USER_ADDR >> ADDR_SR_SHFT;
register_t slbv;
/* Try lockless look-up first */
slb = user_va_to_slb_entry(pm, (vm_offset_t)addr);
@ -91,20 +89,21 @@ set_user_sr(pmap_t pm, const void *addr)
if (slb == NULL) {
/* If it isn't there, we need to pre-fault the VSID */
PMAP_LOCK(pm);
vsid = va_to_vsid(pm, (vm_offset_t)addr);
slbv = va_to_vsid(pm, (vm_offset_t)addr) << SLBV_VSID_SHIFT;
PMAP_UNLOCK(pm);
} else {
vsid = slb->slbv >> SLBV_VSID_SHIFT;
slbv = slb->slbv;
}
slb1 = vsid << SLBV_VSID_SHIFT;
slb2 = (esid << SLBE_ESID_SHIFT) | SLBE_VALID | USER_SR;
/* If we have already set this VSID, we can just return */
if (curthread->td_pcb->pcb_cpu.aim.usr_vsid == slbv)
return;
__asm __volatile ("isync; slbie %0; slbmte %1, %2; isync" ::
"r"(USER_ADDR), "r"(slbv), "r"(USER_SLB_SLBE));
curthread->td_pcb->pcb_cpu.aim.usr_segm =
(uintptr_t)addr >> ADDR_SR_SHFT;
__asm __volatile ("slbie %0; slbmte %1, %2" :: "r"(esid << 28),
"r"(slb1), "r"(slb2));
isync();
curthread->td_pcb->pcb_cpu.aim.usr_vsid = slbv;
}
#else
static __inline void
@ -114,9 +113,13 @@ set_user_sr(pmap_t pm, const void *addr)
vsid = va_to_vsid(pm, (vm_offset_t)addr);
isync();
__asm __volatile ("mtsr %0,%1" :: "n"(USER_SR), "r"(vsid));
isync();
/* If we have already set this VSID, we can just return */
if (curthread->td_pcb->pcb_cpu.aim.usr_vsid == vsid)
return;
__asm __volatile ("sync; mtsr %0,%1; sync; isync" :: "n"(USER_SR),
"r"(vsid));
curthread->td_pcb->pcb_cpu.aim.usr_vsid = vsid;
}
#endif

14
sys/powerpc/aim/slb.c
View File

@ -200,7 +200,7 @@ kernel_va_to_slbv(vm_offset_t va)
esid = (uintptr_t)va >> ADDR_SR_SHFT;
/* Set kernel VSID to deterministic value */
slbv = va_to_vsid(kernel_pmap, va) << SLBV_VSID_SHIFT;
slbv = (KERNEL_VSID((uintptr_t)va >> ADDR_SR_SHFT)) << SLBV_VSID_SHIFT;
/* Figure out if this is a large-page mapping */
if (hw_direct_map && va < VM_MIN_KERNEL_ADDRESS) {
@ -421,19 +421,19 @@ slb_insert_kernel(uint64_t slbe, uint64_t slbv)
slbcache = PCPU_GET(slb);
/* Check for an unused slot, abusing the USER_SR slot as a full flag */
if (slbcache[USER_SR].slbe == 0) {
for (i = 0; i < USER_SR; i++) {
/* Check for an unused slot, abusing the user slot as a full flag */
if (slbcache[USER_SLB_SLOT].slbe == 0) {
for (i = 0; i < USER_SLB_SLOT; i++) {
if (!(slbcache[i].slbe & SLBE_VALID))
goto fillkernslb;
}
if (i == USER_SR)
slbcache[USER_SR].slbe = 1;
if (i == USER_SLB_SLOT)
slbcache[USER_SLB_SLOT].slbe = 1;
}
for (i = mftb() % 64, j = 0; j < 64; j++, i = (i+1) % 64) {
if (i == USER_SR)
if (i == USER_SLB_SLOT)
continue;
if (SLB_SPILLABLE(slbcache[i].slbe))

16
sys/powerpc/aim/swtch64.S
View File

@ -110,13 +110,10 @@ ENTRY(cpu_switch)
std %r1,PCB_SP(%r6) /* Save the stack pointer */
std %r2,PCB_TOC(%r6) /* Save the TOC pointer */
li %r14,0 /* Save USER_SR for copyin/out */
li %r15,0
li %r16,USER_SR
slbmfee %r14, %r16
li %r15,0 /* Save user segment for copyin/out */
li %r16,USER_SLB_SLOT
slbmfev %r15, %r16
isync
std %r14,PCB_AIM_USR_ESID(%r6)
std %r15,PCB_AIM_USR_VSID(%r6)
mr %r14,%r3 /* Copy the old thread ptr... */
@ -221,14 +218,17 @@ blocked_loop:
ld %r1,PCB_SP(%r3) /* Load the stack pointer */
ld %r2,PCB_TOC(%r3) /* Load the TOC pointer */
lis %r5,USER_ADDR@highesta /* Load the USER_SR segment reg */
lis %r5,USER_ADDR@highesta /* Load the copyin/out segment reg */
ori %r5,%r5,USER_ADDR@highera
sldi %r5,%r5,32
oris %r5,%r5,USER_ADDR@ha
slbie %r5
lis %r6,USER_SLB_SLBE@highesta
ori %r6,%r6,USER_SLB_SLBE@highera
sldi %r6,%r6,32
oris %r6,%r6,USER_SLB_SLBE@ha
ori %r6,%r6,USER_SLB_SLBE@l
ld %r5,PCB_AIM_USR_VSID(%r3)
ld %r6,PCB_AIM_USR_ESID(%r3)
ori %r6,%r6,USER_SR
slbmte %r5,%r6
isync

10
sys/powerpc/aim/trap.c
View File

@ -249,8 +249,16 @@ trap(struct trapframe *frame)
return;
break;
#ifdef __powerpc64__
case EXC_ISE:
case EXC_DSE:
if ((frame->cpu.aim.dar & SEGMENT_MASK) == USER_ADDR) {
__asm __volatile ("slbmte %0, %1" ::
"r"(td->td_pcb->pcb_cpu.aim.usr_vsid),
"r"(USER_SLB_SLBE));
return;
}
/* FALLTHROUGH */
case EXC_ISE:
if (handle_slb_spill(kernel_pmap,
(type == EXC_ISE) ? frame->srr0 :
frame->cpu.aim.dar) != 0)

11
sys/powerpc/aim/trap_subr32.S
View File

@ -54,7 +54,7 @@
lwz sr,9*4(pmap); mtsr 9,sr; \
lwz sr,10*4(pmap); mtsr 10,sr; \
lwz sr,11*4(pmap); mtsr 11,sr; \
lwz sr,12*4(pmap); mtsr 12,sr; \
/* Skip segment 12 (USER_SR), which is restored differently */ \
lwz sr,13*4(pmap); mtsr 13,sr; \
lwz sr,14*4(pmap); mtsr 14,sr; \
lwz sr,15*4(pmap); mtsr 15,sr; isync;
@ -66,7 +66,9 @@
GET_CPUINFO(pmap); \
lwz pmap,PC_CURPMAP(pmap); \
lwzu sr,PM_SR(pmap); \
RESTORE_SRS(pmap,sr)
RESTORE_SRS(pmap,sr) \
/* Restore SR 12 */ \
lwz sr,12*4(pmap); mtsr 12,sr
/*
* Kernel SRs are loaded directly from kernel_pmap_
@ -537,6 +539,11 @@ u_trap:
*/
k_trap:
FRAME_SETUP(PC_TEMPSAVE)
/* Restore USER_SR */
GET_CPUINFO(%r30)
lwz %r30,PC_CURPCB(%r30)
lwz %r30,PCB_AIM_USR_VSID(%r30)
mtsr USER_SR,%r30; sync; isync
/* Call C interrupt dispatcher: */
trapagain:
addi %r3,%r1,8

2
sys/powerpc/aim/trap_subr64.S
View File

@ -99,7 +99,7 @@ instkernslb:
addi %r28, %r28, 16; /* Advance pointer */
addi %r29, %r29, 1;
cmpli 0, %r29, USER_SR; /* Repeat if we are not at the end */
cmpli 0, %r29, USER_SLB_SLOT; /* Repeat if we are not at the end */
blt instkernslb;
blr;

1
sys/powerpc/aim/vm_machdep.c
View File

@ -197,7 +197,6 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
pcb->pcb_lr = (register_t)fork_trampoline;
#endif
pcb->pcb_cpu.aim.usr_vsid = 0;
pcb->pcb_cpu.aim.usr_esid = 0;
/* Setup to release spin count in fork_exit(). */
td2->td_md.md_spinlock_count = 1;

1
sys/powerpc/include/pcb.h
View File

@ -67,7 +67,6 @@ struct pcb {
union {
struct {
vm_offset_t usr_segm; /* Base address */
register_t usr_esid; /* USER_SR segment */
register_t usr_vsid; /* USER_SR segment */
} aim;
struct {

7
sys/powerpc/include/slb.h
View File

@ -62,6 +62,13 @@
#define SLBE_ESID_MASK 0xfffffffff0000000UL /* Effective segment ID mask */
#define SLBE_ESID_SHIFT 28
/*
* User segment for copyin/out
*/
#define USER_SLB_SLOT 63
#define USER_SLB_SLBE (((USER_ADDR >> ADDR_SR_SHFT) << SLBE_ESID_SHIFT) | \
SLBE_VALID | USER_SLB_SLOT)
struct slb {
uint64_t slbv;
uint64_t slbe;

4
sys/powerpc/include/sr.h
View File

@ -42,11 +42,7 @@
#define SR_VSID_MASK 0x00ffffff /* Virtual Segment ID mask */
/* Kernel segment register usage */
#ifdef __powerpc64__
#define USER_SR 63
#else
#define USER_SR 12
#endif
#define KERNEL_SR 13
#define KERNEL2_SR 14
#define KERNEL3_SR 15

1
sys/powerpc/powerpc/exec_machdep.c
View File

@ -986,7 +986,6 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
pcb2->pcb_lr = (register_t)fork_trampoline;
#endif
pcb2->pcb_cpu.aim.usr_vsid = 0;
pcb2->pcb_cpu.aim.usr_esid = 0;
/* Setup to release spin count in fork_exit(). */
td->td_md.md_spinlock_count = 1;

5
sys/powerpc/powerpc/genassym.c
View File

@ -103,13 +103,15 @@ ASSYM(TLBSAVE_BOOKE_R31, TLBSAVE_BOOKE_R31*sizeof(register_t));
ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock));
#if defined(AIM)
ASSYM(USER_SR, USER_SR);
ASSYM(USER_ADDR, USER_ADDR);
#ifdef __powerpc64__
ASSYM(PC_KERNSLB, offsetof(struct pcpu, pc_slb));
ASSYM(PC_USERSLB, offsetof(struct pcpu, pc_userslb));
ASSYM(USER_SLB_SLOT, USER_SLB_SLOT);
ASSYM(USER_SLB_SLBE, USER_SLB_SLBE);
#else
ASSYM(PM_SR, offsetof(struct pmap, pm_sr));
ASSYM(USER_SR, USER_SR);
#endif
#elif defined(E500)
ASSYM(PM_PDIR, offsetof(struct pmap, pm_pdir));
@ -187,7 +189,6 @@ ASSYM(PCB_FLAGS, offsetof(struct pcb, pcb_flags));
ASSYM(PCB_FPU, PCB_FPU);
ASSYM(PCB_VEC, PCB_VEC);
ASSYM(PCB_AIM_USR_ESID, offsetof(struct pcb, pcb_cpu.aim.usr_esid));
ASSYM(PCB_AIM_USR_VSID, offsetof(struct pcb, pcb_cpu.aim.usr_vsid));
ASSYM(PCB_BOOKE_CTR, offsetof(struct pcb, pcb_cpu.booke.ctr));
ASSYM(PCB_BOOKE_XER, offsetof(struct pcb, pcb_cpu.booke.xer));