diff --git a/sys/amd64/linux/linux_machdep.c b/sys/amd64/linux/linux_machdep.c index 6c5b014853fc..434ea0eac07c 100644 --- a/sys/amd64/linux/linux_machdep.c +++ b/sys/amd64/linux/linux_machdep.c @@ -78,6 +78,9 @@ __FBSDID("$FreeBSD$"); #include #include +#include +#include + #include #include #include @@ -88,8 +91,6 @@ __FBSDID("$FreeBSD$"); #include #include -#include - int linux_execve(struct thread *td, struct linux_execve_args *args) { @@ -276,3 +277,48 @@ linux_set_cloned_tls(struct thread *td, void *desc) return (0); } + +int futex_xchgl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_xchgl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_xchgl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_xchgl_smap : futex_xchgl_nosmap); +} + +int futex_addl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_addl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_addl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_addl_smap : futex_addl_nosmap); +} + +int futex_orl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_orl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_orl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_orl_smap : futex_orl_nosmap); +} + +int futex_andl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_andl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_andl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_andl_smap : futex_andl_nosmap); +} + +int futex_xorl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_xorl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_xorl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_xorl_smap : futex_xorl_nosmap); +} diff --git a/sys/amd64/linux/linux_support.s b/sys/amd64/linux/linux_support.s index a9f02160be26..391f76414f22 100644 --- a/sys/amd64/linux/linux_support.s +++ b/sys/amd64/linux/linux_support.s @@ -38,7 +38,7 @@ futex_fault: movl $-EFAULT,%eax ret -ENTRY(futex_xchgl) +ENTRY(futex_xchgl_nosmap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax @@ -49,9 +49,24 @@ ENTRY(futex_xchgl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_xchgl) +END(futex_xchgl_nosmap) -ENTRY(futex_addl) +ENTRY(futex_xchgl_smap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + stac + xchgl %edi,(%rsi) + clac + movl %edi,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_xchgl_smap) + +ENTRY(futex_addl_nosmap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax @@ -65,9 +80,27 @@ ENTRY(futex_addl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_addl) +END(futex_addl_nosmap) -ENTRY(futex_orl) +ENTRY(futex_addl_smap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + stac +#ifdef SMP + lock +#endif + xaddl %edi,(%rsi) + clac + movl %edi,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_addl_smap) + +ENTRY(futex_orl_nosmap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax @@ -85,9 +118,31 @@ ENTRY(futex_orl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_orl) +END(futex_orl_nosmap) -ENTRY(futex_andl) +ENTRY(futex_orl_smap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + movl (%rsi),%eax +1: movl %eax,%ecx + orl %edi,%ecx + stac +#ifdef SMP + lock +#endif + cmpxchgl %ecx,(%rsi) + clac + jnz 1b + movl %eax,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_orl_smap) + +ENTRY(futex_andl_nosmap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax @@ -105,9 +160,31 @@ ENTRY(futex_andl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_andl) +END(futex_andl_nosmap) -ENTRY(futex_xorl) +ENTRY(futex_andl_smap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + movl (%rsi),%eax +1: movl %eax,%ecx + andl %edi,%ecx + stac +#ifdef SMP + lock +#endif + cmpxchgl %ecx,(%rsi) + clac + jnz 1b + movl %eax,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_andl_smap) + +ENTRY(futex_xorl_nosmap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax @@ -125,4 +202,26 @@ ENTRY(futex_xorl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_xorl) +END(futex_xorl_nosmap) + +ENTRY(futex_xorl_smap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + movl (%rsi),%eax +1: movl %eax,%ecx + xorl %edi,%ecx + stac +#ifdef SMP + lock +#endif + cmpxchgl %ecx,(%rsi) + clac + jnz 1b + movl %eax,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_xorl_smap) diff --git a/sys/amd64/linux32/linux32_machdep.c b/sys/amd64/linux32/linux32_machdep.c index ce06be57e9fa..61ecc87dc77c 100644 --- a/sys/amd64/linux32/linux32_machdep.c +++ b/sys/amd64/linux32/linux32_machdep.c @@ -58,10 +58,12 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include #include #include +#include #include #include @@ -822,3 +824,48 @@ linux_set_thread_area(struct thread *td, return (0); } + +int futex_xchgl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_xchgl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_xchgl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_xchgl_smap : futex_xchgl_nosmap); +} + +int futex_addl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_addl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_addl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_addl_smap : futex_addl_nosmap); +} + +int futex_orl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_orl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_orl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_orl_smap : futex_orl_nosmap); +} + +int futex_andl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_andl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_andl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_andl_smap : futex_andl_nosmap); +} + +int futex_xorl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_xorl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_xorl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_xorl_smap : futex_xorl_nosmap); +} diff --git a/sys/amd64/linux32/linux32_support.s b/sys/amd64/linux32/linux32_support.s index bba0d8d5e715..981bba9f5821 100644 --- a/sys/amd64/linux32/linux32_support.s +++ b/sys/amd64/linux32/linux32_support.s @@ -38,7 +38,7 @@ futex_fault: movl $-EFAULT,%eax ret -ENTRY(futex_xchgl) +ENTRY(futex_xchgl_nosmap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax @@ -49,9 +49,24 @@ ENTRY(futex_xchgl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_xchgl) +END(futex_xchgl_nosmap) -ENTRY(futex_addl) +ENTRY(futex_xchgl_smap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + stac + xchgl %edi,(%rsi) + clac + movl %edi,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_xchgl_smap) + +ENTRY(futex_addl_nosmap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax @@ -65,9 +80,27 @@ ENTRY(futex_addl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_addl) +END(futex_addl_nosmap) -ENTRY(futex_orl) +ENTRY(futex_addl_smap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + stac +#ifdef SMP + lock +#endif + xaddl %edi,(%rsi) + clac + movl %edi,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_addl_smap) + +ENTRY(futex_orl_nosmap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax @@ -85,9 +118,31 @@ ENTRY(futex_orl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_orl) +END(futex_orl_nosmap) -ENTRY(futex_andl) +ENTRY(futex_orl_smap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + movl (%rsi),%eax +1: movl %eax,%ecx + orl %edi,%ecx + stac +#ifdef SMP + lock +#endif + cmpxchgl %ecx,(%rsi) + clac + jnz 1b + movl %eax,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_orl_smap) + +ENTRY(futex_andl_nosmap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax @@ -105,9 +160,31 @@ ENTRY(futex_andl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_andl) +END(futex_andl_nosmap) -ENTRY(futex_xorl) +ENTRY(futex_andl_smap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + movl (%rsi),%eax +1: movl %eax,%ecx + andl %edi,%ecx + stac +#ifdef SMP + lock +#endif + cmpxchgl %ecx,(%rsi) + clac + jnz 1b + movl %eax,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_andl_smap) + +ENTRY(futex_xorl_nosmap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax @@ -125,4 +202,26 @@ ENTRY(futex_xorl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_xorl) +END(futex_xorl_nosmap) + +ENTRY(futex_xorl_smap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + movl (%rsi),%eax +1: movl %eax,%ecx + xorl %edi,%ecx + stac +#ifdef SMP + lock +#endif + cmpxchgl %ecx,(%rsi) + clac + jnz 1b + movl %eax,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_xorl_smap)