d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use SMAP on amd64.

Browse Source 
Ifuncs selectors dispatch copyin(9) family to the suitable variant, to
set rflags.AC around userspace access.  Rflags.AC bit is cleared in
all kernel entry points unconditionally even on machines not
supporting SMAP.

Reviewed by:	jhb
Sponsored by:	The FreeBSD Foundation
Differential revision:	https://reviews.freebsd.org/D13838
This commit is contained in:
Konstantin Belousov 2018-07-29 20:47:00 +00:00
parent 70eaeb3622
commit b3a7db3b06
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=336876
11 changed files with 614 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

178
sys/amd64/amd64/copyout.c Normal file
View File

@ -0,0 +1,178 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2018 The FreeBSD Foundation
* All rights reserved.
*
* This software was developed by Konstantin Belousov <kib@FreeBSD.org>
* under sponsorship from the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <machine/md_var.h>
#include <machine/specialreg.h>
#include <x86/ifunc.h>
int fubyte_nosmap(volatile const void *base);
int fubyte_smap(volatile const void *base);
DEFINE_IFUNC(, int, fubyte, (volatile const void *), static)
{
return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
fubyte_smap : fubyte_nosmap);
}
int fuword16_nosmap(volatile const void *base);
int fuword16_smap(volatile const void *base);
DEFINE_IFUNC(, int, fuword16, (volatile const void *), static)
{
return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
fuword16_smap : fuword16_nosmap);
}
int fueword_nosmap(volatile const void *base, long *val);
int fueword_smap(volatile const void *base, long *val);
DEFINE_IFUNC(, int, fueword, (volatile const void *, long *), static)
{
return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
fueword_smap : fueword_nosmap);
}
DEFINE_IFUNC(, int, fueword64, (volatile const void *, int64_t *), static)
{
return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
fueword_smap : fueword_nosmap);
}
int fueword32_nosmap(volatile const void *base, int32_t *val);
int fueword32_smap(volatile const void *base, int32_t *val);
DEFINE_IFUNC(, int, fueword32, (volatile const void *, int32_t *), static)
{
return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
fueword32_smap : fueword32_nosmap);
}
int subyte_nosmap(volatile void *base, int byte);
int subyte_smap(volatile void *base, int byte);
DEFINE_IFUNC(, int, subyte, (volatile void *, int), static)
{
return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
subyte_smap : subyte_nosmap);
}
int suword16_nosmap(volatile void *base, int word);
int suword16_smap(volatile void *base, int word);
DEFINE_IFUNC(, int, suword16, (volatile void *, int), static)
{
return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
suword16_smap : suword16_nosmap);
}
int suword32_nosmap(volatile void *base, int32_t word);
int suword32_smap(volatile void *base, int32_t word);
DEFINE_IFUNC(, int, suword32, (volatile void *, int32_t), static)
{
return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
suword32_smap : suword32_nosmap);
}
int suword_nosmap(volatile void *base, long word);
int suword_smap(volatile void *base, long word);
DEFINE_IFUNC(, int, suword, (volatile void *, long), static)
{
return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
suword_smap : suword_nosmap);
}
DEFINE_IFUNC(, int, suword64, (volatile void *, int64_t), static)
{
return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
suword_smap : suword_nosmap);
}
int casueword32_nosmap(volatile uint32_t *base, uint32_t oldval,
uint32_t *oldvalp, uint32_t newval);
int casueword32_smap(volatile uint32_t *base, uint32_t oldval,
uint32_t *oldvalp, uint32_t newval);
DEFINE_IFUNC(, int, casueword32, (volatile uint32_t *, uint32_t, uint32_t *,
uint32_t), static)
{
return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
casueword32_smap : casueword32_nosmap);
}
int casueword_nosmap(volatile u_long *p, u_long oldval, u_long *oldvalp,
u_long newval);
int casueword_smap(volatile u_long *p, u_long oldval, u_long *oldvalp,
u_long newval);
DEFINE_IFUNC(, int, casueword, (volatile u_long *, u_long, u_long *, u_long),
static)
{
return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
casueword_smap : casueword_nosmap);
}
int copyinstr_nosmap(const void *udaddr, void *kaddr, size_t len,
size_t *lencopied);
int copyinstr_smap(const void *udaddr, void *kaddr, size_t len,
size_t *lencopied);
DEFINE_IFUNC(, int, copyinstr, (const void *, void *, size_t, size_t *),
static)
{
return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
copyinstr_smap : copyinstr_nosmap);
}
int copyin_nosmap(const void *udaddr, void *kaddr, size_t len);
int copyin_smap(const void *udaddr, void *kaddr, size_t len);
DEFINE_IFUNC(, int, copyin, (const void *, void *, size_t), static)
{
return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
copyin_smap : copyin_nosmap);
}
int copyout_nosmap(const void *kaddr, void *udaddr, size_t len);
int copyout_smap(const void *kaddr, void *udaddr, size_t len);
DEFINE_IFUNC(, int, copyout, (const void *, void *, size_t), static)
{
return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ?
copyout_smap : copyout_nosmap);
}

26
sys/amd64/amd64/exception.S
View File

@ -43,8 +43,8 @@
#include "assym.inc"
#include <machine/asmacros.h>
#include <machine/psl.h>
#include <machine/asmacros.h>
#include <machine/trap.h>
#include <machine/specialreg.h>
@ -196,7 +196,9 @@ alltraps_pushregs_no_rax:
movq %r14,TF_R14(%rsp)
movq %r15,TF_R15(%rsp)
movl $TF_HASSEGS,TF_FLAGS(%rsp)
cld
pushfq
andq $~(PSL_D | PSL_AC),(%rsp)
popfq
FAKE_MCOUNT(TF_RIP(%rsp))
#ifdef KDTRACE_HOOKS
/*
@ -277,7 +279,9 @@ IDTVEC(dblfault)
movq %r15,TF_R15(%rsp)
SAVE_SEGS
movl $TF_HASSEGS,TF_FLAGS(%rsp)
cld
pushfq
andq $~(PSL_D | PSL_AC),(%rsp)
popfq
testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
jz 1f /* already running with kernel GS.base */
swapgs
@ -571,7 +575,9 @@ IDTVEC(dbg)
movq %r15,TF_R15(%rsp)
SAVE_SEGS
movl $TF_HASSEGS,TF_FLAGS(%rsp)
cld
pushfq
andq $~(PSL_D | PSL_AC),(%rsp)
popfq
testb $SEL_RPL_MASK,TF_CS(%rsp)
jnz dbg_fromuserspace
/*
@ -704,7 +710,9 @@ IDTVEC(nmi)
movq %r15,TF_R15(%rsp)
SAVE_SEGS
movl $TF_HASSEGS,TF_FLAGS(%rsp)
cld
pushfq
andq $~(PSL_D | PSL_AC),(%rsp)
popfq
xorl %ebx,%ebx
testb $SEL_RPL_MASK,TF_CS(%rsp)
jnz nmi_fromuserspace
@ -793,7 +801,9 @@ nmi_calltrap:
subq %rcx,%rdx
movq %rdx,%rdi /* destination stack pointer */
shrq $3,%rcx /* trap frame size in long words */
cld
pushfq
andq $~(PSL_D | PSL_AC),(%rsp)
popfq
rep
movsq /* copy trapframe */
movq %rdx,%rsp /* we are on the regular kstack */
@ -902,7 +912,9 @@ IDTVEC(mchk)
movq %r15,TF_R15(%rsp)
SAVE_SEGS
movl $TF_HASSEGS,TF_FLAGS(%rsp)
cld
pushfq
andq $~(PSL_D | PSL_AC),(%rsp)
popfq
xorl %ebx,%ebx
testb $SEL_RPL_MASK,TF_CS(%rsp)
jnz mchk_fromuserspace

8
sys/amd64/amd64/initcpu.c
View File

@ -239,8 +239,12 @@ initializecpu(void)
* to the kernel tables. The boot loader enables the U bit in
* its tables.
*/
if (!IS_BSP() && (cpu_stdext_feature & CPUID_STDEXT_SMEP))
cr4 |= CR4_SMEP;
if (!IS_BSP()) {
if (cpu_stdext_feature & CPUID_STDEXT_SMEP)
cr4 |= CR4_SMEP;
if (cpu_stdext_feature & CPUID_STDEXT_SMAP)
cr4 |= CR4_SMAP;
}
load_cr4(cr4);
if (IS_BSP() && (amd_feature & AMDID_NX) != 0) {
msr = rdmsr(MSR_EFER) | EFER_NXE;

2
sys/amd64/amd64/machdep.c
View File

@ -1548,7 +1548,7 @@ amd64_conf_fast_syscall(void)
msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48);
wrmsr(MSR_STAR, msr);
wrmsr(MSR_SF_MASK, PSL_NT | PSL_T | PSL_I | PSL_C | PSL_D);
wrmsr(MSR_SF_MASK, PSL_NT | PSL_T | PSL_I | PSL_C | PSL_D | PSL_AC);
}
u_int64_t

17
sys/amd64/amd64/pmap.c
View File

@ -1092,6 +1092,7 @@ pmap_bootstrap(vm_paddr_t *firstaddr)
{
vm_offset_t va;
pt_entry_t *pte;
uint64_t cr4;
int i;
KERNend = *firstaddr;
@ -1118,11 +1119,21 @@ pmap_bootstrap(vm_paddr_t *firstaddr)
virtual_end = VM_MAX_KERNEL_ADDRESS;
/* XXX do %cr0 as well */
load_cr4(rcr4() | CR4_PGE);
/*
* Enable PG_G global pages, then switch to the kernel page
* table from the bootstrap page table. After the switch, it
* is possible to enable SMEP and SMAP since PG_U bits are
* correct now.
*/
cr4 = rcr4();
cr4 |= CR4_PGE;
load_cr4(cr4);
load_cr3(KPML4phys);
if (cpu_stdext_feature & CPUID_STDEXT_SMEP)
load_cr4(rcr4() | CR4_SMEP);
cr4 |= CR4_SMEP;
if (cpu_stdext_feature & CPUID_STDEXT_SMAP)
cr4 |= CR4_SMAP;
load_cr4(cr4);
/*
* Initialize the kernel pmap (which is statically allocated).

397
sys/amd64/amd64/support.S
View File

@ -226,7 +226,7 @@ END(fillw)
* copyout(from_kernel, to_user, len)
* %rdi, %rsi, %rdx
*/
ENTRY(copyout)
ENTRY(copyout_nosmap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rax
movq $copyout_fault,PCB_ONFAULT(%rax)
@ -268,6 +268,55 @@ ENTRY(copyout)
rep
movsb
jmp done_copyout
END(copyout_nosmap)
ENTRY(copyout_smap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rax
/* Trap entry clears PSL.AC */
movq $copyout_fault,PCB_ONFAULT(%rax)
testq %rdx,%rdx /* anything to do? */
jz done_copyout
/*
* Check explicitly for non-user addresses. If 486 write protection
* is being used, this check is essential because we are in kernel
* mode so the h/w does not provide any protection against writing
* kernel addresses.
*/
/*
* First, prevent address wrapping.
*/
movq %rsi,%rax
addq %rdx,%rax
jc copyout_fault
/*
* XXX STOP USING VM_MAXUSER_ADDRESS.
* It is an end address, not a max, so every time it is used correctly it
* looks like there is an off by one error, and of course it caused an off
* by one error in several places.
*/
movq $VM_MAXUSER_ADDRESS,%rcx
cmpq %rcx,%rax
ja copyout_fault
xchgq %rdi,%rsi
/* bcopy(%rsi, %rdi, %rdx) */
movq %rdx,%rcx
shrq $3,%rcx
cld
stac
rep
movsq
movb %dl,%cl
andb $7,%cl
rep
movsb
clac
done_copyout:
xorl %eax,%eax
movq PCPU(CURPCB),%rdx
@ -288,7 +337,7 @@ END(copyout)
* copyin(from_user, to_kernel, len)
* %rdi, %rsi, %rdx
*/
ENTRY(copyin)
ENTRY(copyin_nosmap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rax
movq $copyin_fault,PCB_ONFAULT(%rax)
@ -309,6 +358,39 @@ ENTRY(copyin)
movq %rdx,%rcx
movb %cl,%al
shrq $3,%rcx /* copy longword-wise */
cld
rep
movsq
movb %al,%cl
andb $7,%cl /* copy remaining bytes */
rep
movsb
jmp done_copyin
END(copyin_nosmap)
ENTRY(copyin_smap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rax
movq $copyin_fault,PCB_ONFAULT(%rax)
testq %rdx,%rdx /* anything to do? */
jz done_copyin
/*
* make sure address is valid
*/
movq %rdi,%rax
addq %rdx,%rax
jc copyin_fault
movq $VM_MAXUSER_ADDRESS,%rcx
cmpq %rcx,%rax
ja copyin_fault
xchgq %rdi,%rsi
movq %rdx,%rcx
movb %cl,%al
shrq $3,%rcx /* copy longword-wise */
stac
rep
movsq
movb %al,%cl
@ -316,6 +398,7 @@ ENTRY(copyin)
je done_copyin
rep
movsb
clac
done_copyin:
xorl %eax,%eax
@ -323,6 +406,7 @@ done_copyin:
movq %rax,PCB_ONFAULT(%rdx)
POP_FRAME_POINTER
ret
END(copyin_smap)
ALIGN_TEXT
copyin_fault:
@ -331,14 +415,13 @@ copyin_fault:
movq $EFAULT,%rax
POP_FRAME_POINTER
ret
END(copyin)
/*
* casueword32. Compare and set user integer. Returns -1 on fault,
* 0 if access was successful. Old value is written to *oldp.
* dst = %rdi, old = %esi, oldp = %rdx, new = %ecx
*/
ENTRY(casueword32)
ENTRY(casueword32_nosmap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%r8
movq $fusufault,PCB_ONFAULT(%r8)
@ -370,14 +453,50 @@ ENTRY(casueword32)
movl %esi,(%rdx) /* oldp = %rdx */
POP_FRAME_POINTER
ret
END(casueword32)
END(casueword32_nosmap)
ENTRY(casueword32_smap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%r8
movq $fusufault,PCB_ONFAULT(%r8)
movq $VM_MAXUSER_ADDRESS-4,%rax
cmpq %rax,%rdi /* verify address is valid */
ja fusufault
movl %esi,%eax /* old */
stac
#ifdef SMP
lock
#endif
cmpxchgl %ecx,(%rdi) /* new = %ecx */
clac
/*
* The old value is in %eax. If the store succeeded it will be the
* value we expected (old) from before the store, otherwise it will
* be the current value. Save %eax into %esi to prepare the return
* value.
*/
movl %eax,%esi
xorl %eax,%eax
movq %rax,PCB_ONFAULT(%r8)
/*
* Access the oldp after the pcb_onfault is cleared, to correctly
* catch corrupted pointer.
*/
movl %esi,(%rdx) /* oldp = %rdx */
POP_FRAME_POINTER
ret
END(casueword32_smap)
/*
* casueword. Compare and set user long. Returns -1 on fault,
* 0 if access was successful. Old value is written to *oldp.
* dst = %rdi, old = %rsi, oldp = %rdx, new = %rcx
*/
ENTRY(casueword)
ENTRY(casueword_nosmap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%r8
movq $fusufault,PCB_ONFAULT(%r8)
@ -403,7 +522,37 @@ ENTRY(casueword)
movq %rsi,(%rdx)
POP_FRAME_POINTER
ret
END(casueword)
END(casueword_nosmap)
ENTRY(casueword_smap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%r8
movq $fusufault,PCB_ONFAULT(%r8)
movq $VM_MAXUSER_ADDRESS-4,%rax
cmpq %rax,%rdi /* verify address is valid */
ja fusufault
movq %rsi,%rax /* old */
stac
#ifdef SMP
lock
#endif
cmpxchgq %rcx,(%rdi) /* new = %rcx */
clac
/*
* The old value is in %rax. If the store succeeded it will be the
* value we expected (old) from before the store, otherwise it will
* be the current value.
*/
movq %rax,%rsi
xorl %eax,%eax
movq %rax,PCB_ONFAULT(%r8)
movq %rsi,(%rdx)
POP_FRAME_POINTER
ret
END(casueword_smap)
/*
* Fetch (load) a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit
@ -411,8 +560,7 @@ END(casueword)
* addr = %rdi, valp = %rsi
*/
ALTENTRY(fueword64)
ENTRY(fueword)
ENTRY(fueword_nosmap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)
@ -427,10 +575,28 @@ ENTRY(fueword)
movq %r11,(%rsi)
POP_FRAME_POINTER
ret
END(fueword64)
END(fueword)
END(fueword64_nosmap)
ENTRY(fueword32)
ENTRY(fueword_smap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)
movq $VM_MAXUSER_ADDRESS-8,%rax
cmpq %rax,%rdi /* verify address is valid */
ja fusufault
xorl %eax,%eax
stac
movq (%rdi),%r11
clac
movq %rax,PCB_ONFAULT(%rcx)
movq %r11,(%rsi)
POP_FRAME_POINTER
ret
END(fueword64_smap)
ENTRY(fueword32_nosmap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)
@ -445,9 +611,28 @@ ENTRY(fueword32)
movl %r11d,(%rsi)
POP_FRAME_POINTER
ret
END(fueword32)
END(fueword32_nosmap)
ENTRY(fuword16)
ENTRY(fueword32_smap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)
movq $VM_MAXUSER_ADDRESS-4,%rax
cmpq %rax,%rdi /* verify address is valid */
ja fusufault
xorl %eax,%eax
stac
movl (%rdi),%r11d
clac
movq %rax,PCB_ONFAULT(%rcx)
movl %r11d,(%rsi)
POP_FRAME_POINTER
ret
END(fueword32_smap)
ENTRY(fuword16_nosmap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)
@ -460,9 +645,26 @@ ENTRY(fuword16)
movq $0,PCB_ONFAULT(%rcx)
POP_FRAME_POINTER
ret
END(fuword16)
END(fuword16_nosmap)
ENTRY(fubyte)
ENTRY(fuword16_smap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)
movq $VM_MAXUSER_ADDRESS-2,%rax
cmpq %rax,%rdi
ja fusufault
stac
movzwl (%rdi),%eax
clac
movq $0,PCB_ONFAULT(%rcx)
POP_FRAME_POINTER
ret
END(fuword16_smap)
ENTRY(fubyte_nosmap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)
@ -475,9 +677,27 @@ ENTRY(fubyte)
movq $0,PCB_ONFAULT(%rcx)
POP_FRAME_POINTER
ret
END(fubyte)
END(fubyte_nosmap)
ENTRY(fubyte_smap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)
movq $VM_MAXUSER_ADDRESS-1,%rax
cmpq %rax,%rdi
ja fusufault
stac
movzbl (%rdi),%eax
clac
movq $0,PCB_ONFAULT(%rcx)
POP_FRAME_POINTER
ret
END(fubyte_smap)
ALIGN_TEXT
/* Fault entry clears PSL.AC */
fusufault:
movq PCPU(CURPCB),%rcx
xorl %eax,%eax
@ -491,8 +711,7 @@ fusufault:
* user memory.
* addr = %rdi, value = %rsi
*/
ALTENTRY(suword64)
ENTRY(suword)
ENTRY(suword_nosmap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)
@ -507,10 +726,28 @@ ENTRY(suword)
movq %rax,PCB_ONFAULT(%rcx)
POP_FRAME_POINTER
ret
END(suword64)
END(suword)
END(suword_nosmap)
ENTRY(suword32)
ENTRY(suword_smap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)
movq $VM_MAXUSER_ADDRESS-8,%rax
cmpq %rax,%rdi /* verify address validity */
ja fusufault
stac
movq %rsi,(%rdi)
clac
xorl %eax,%eax
movq PCPU(CURPCB),%rcx
movq %rax,PCB_ONFAULT(%rcx)
POP_FRAME_POINTER
ret
END(suword_smap)
ENTRY(suword32_nosmap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)
@ -525,9 +762,28 @@ ENTRY(suword32)
movq %rax,PCB_ONFAULT(%rcx)
POP_FRAME_POINTER
ret
END(suword32)
END(suword32_nosmap)
ENTRY(suword16)
ENTRY(suword32_smap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)
movq $VM_MAXUSER_ADDRESS-4,%rax
cmpq %rax,%rdi /* verify address validity */
ja fusufault
stac
movl %esi,(%rdi)
clac
xorl %eax,%eax
movq PCPU(CURPCB),%rcx
movq %rax,PCB_ONFAULT(%rcx)
POP_FRAME_POINTER
ret
END(suword32_smap)
ENTRY(suword16_nosmap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)
@ -542,9 +798,28 @@ ENTRY(suword16)
movq %rax,PCB_ONFAULT(%rcx)
POP_FRAME_POINTER
ret
END(suword16)
END(suword16_nosmap)
ENTRY(subyte)
ENTRY(suword16_smap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)
movq $VM_MAXUSER_ADDRESS-2,%rax
cmpq %rax,%rdi /* verify address validity */
ja fusufault
stac
movw %si,(%rdi)
clac
xorl %eax,%eax
movq PCPU(CURPCB),%rcx /* restore trashed register */
movq %rax,PCB_ONFAULT(%rcx)
POP_FRAME_POINTER
ret
END(suword16_smap)
ENTRY(subyte_nosmap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)
@ -560,7 +835,27 @@ ENTRY(subyte)
movq %rax,PCB_ONFAULT(%rcx)
POP_FRAME_POINTER
ret
END(subyte)
END(subyte_nosmap)
ENTRY(subyte_smap)
PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)
movq $VM_MAXUSER_ADDRESS-1,%rax
cmpq %rax,%rdi /* verify address validity */
ja fusufault
movl %esi,%eax
stac
movb %al,(%rdi)
clac
xorl %eax,%eax
movq PCPU(CURPCB),%rcx /* restore trashed register */
movq %rax,PCB_ONFAULT(%rcx)
POP_FRAME_POINTER
ret
END(subyte_smap)
/*
* copyinstr(from, to, maxlen, int *lencopied)
@ -571,7 +866,42 @@ END(subyte)
* EFAULT on protection violations. If lencopied is non-zero,
* return the actual length in *lencopied.
*/
ENTRY(copyinstr)
ENTRY(copyinstr_nosmap)
PUSH_FRAME_POINTER
movq %rdx,%r8 /* %r8 = maxlen */
movq %rcx,%r9 /* %r9 = *len */
xchgq %rdi,%rsi /* %rdi = from, %rsi = to */
movq PCPU(CURPCB),%rcx
movq $cpystrflt,PCB_ONFAULT(%rcx)
movq $VM_MAXUSER_ADDRESS,%rax
/* make sure 'from' is within bounds */
subq %rsi,%rax
jbe cpystrflt
/* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */
cmpq %rdx,%rax
jae 1f
movq %rax,%rdx
movq %rax,%r8
1:
incq %rdx
cld
2:
decq %rdx
jz copyinstr_toolong
lodsb
stosb
orb %al,%al
jnz 2b
jmp copyinstr_succ
END(copyinstr_nosmap)
ENTRY(copyinstr_smap)
PUSH_FRAME_POINTER
movq %rdx,%r8 /* %r8 = maxlen */
movq %rcx,%r9 /* %r9 = *len */
@ -595,26 +925,29 @@ ENTRY(copyinstr)
2:
decq %rdx
jz 3f
jz copyinstr_succ
stac
lodsb
stosb
clac
orb %al,%al
jnz 2b
copyinstr_succ:
/* Success -- 0 byte reached */
decq %rdx
xorl %eax,%eax
jmp cpystrflt_x
3:
copyinstr_toolong:
/* rdx is zero - return ENAMETOOLONG or EFAULT */
movq $VM_MAXUSER_ADDRESS,%rax
cmpq %rax,%rsi
jae cpystrflt
4:
movq $ENAMETOOLONG,%rax
jmp cpystrflt_x
/* Fault entry clears PSL.AC */
cpystrflt:
movq $EFAULT,%rax
@ -630,7 +963,7 @@ cpystrflt_x:
1:
POP_FRAME_POINTER
ret
END(copyinstr)
END(copyinstr_smap)
/*
* copystr(from, to, maxlen, int *lencopied)

24
sys/amd64/amd64/trap.c
View File

@ -673,6 +673,24 @@ trap_check(struct trapframe *frame)
trap(frame);
}
static bool
trap_is_smap(struct trapframe *frame)
{
/*
* A page fault on a userspace address is classified as
* SMAP-induced if:
* - SMAP is supported;
* - kernel mode accessed present data page;
* - rflags.AC was cleared.
* Kernel must never access user space with rflags.AC cleared
* if SMAP is enabled.
*/
return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 &&
(frame->tf_err & (PGEX_P | PGEX_U | PGEX_I | PGEX_RSV)) ==
PGEX_P && (frame->tf_rflags & PSL_AC) == 0);
}
static int
trap_pfault(struct trapframe *frame, int usermode)
{
@ -750,9 +768,13 @@ trap_pfault(struct trapframe *frame, int usermode)
* handling routine. Since accessing the address
* without the handler is a bug, do not try to handle
* it normally, and panic immediately.
*
* If SMAP is enabled, filter SMAP faults also,
* because illegal access might occur to the mapped
* user address, causing infinite loop.
*/
if (!usermode && (td->td_intr_nesting_level != 0 ||
curpcb->pcb_onfault == NULL)) {
trap_is_smap(frame) || curpcb->pcb_onfault == NULL)) {
trap_fatal(frame, eva);
return (-1);
}

4
sys/amd64/ia32/ia32_exception.S
View File

@ -70,7 +70,9 @@ int0x80_syscall_common:
movq %r14,TF_R14(%rsp)
movq %r15,TF_R15(%rsp)
movl $TF_HASSEGS,TF_FLAGS(%rsp)
cld
pushfq
andq $~(PSL_D | PSL_AC),(%rsp)
popfq
FAKE_MCOUNT(TF_RIP(%rsp))
movq %rsp, %rdi
call ia32_syscall

4
sys/amd64/include/asmacros.h
View File

@ -255,7 +255,9 @@ X\vec_name:
movq %r15,TF_R15(%rsp)
SAVE_SEGS
movl $TF_HASSEGS,TF_FLAGS(%rsp)
cld
pushfq
andq $~(PSL_D|PSL_AC),(%rsp)
popfq
testb $SEL_RPL_MASK,TF_CS(%rsp) /* come from kernel ? */
jz 1f /* yes, leave PCB_FULL_IRET alone */
movq PCPU(CURPCB),%r8

1
sys/conf/files.amd64
View File

@ -128,6 +128,7 @@ acpi_wakedata.h optional acpi \
#amd64/amd64/apic_vector.S standard
amd64/amd64/bios.c standard
amd64/amd64/bpf_jit_machdep.c optional bpf_jitter
amd64/amd64/copyout.c standard
amd64/amd64/cpu_switch.S standard
amd64/amd64/db_disasm.c optional ddb
amd64/amd64/db_interface.c optional ddb

1
sys/dev/hyperv/vmbus/amd64/vmbus_vector.S
View File

@ -28,6 +28,7 @@
#include "assym.inc"
#include <machine/psl.h>
#include <machine/asmacros.h>
#include <machine/specialreg.h>