From 4d22d07a075d6d899dd496eaf2decdce9b8327ae Mon Sep 17 00:00:00 2001 From: Konstantin Belousov Date: Mon, 7 Dec 2015 12:20:26 +0000 Subject: [PATCH] Add support for usermode (vdso-like) gettimeofday(2) and clock_gettime(2) on ARMv7 and ARMv8 systems which have architectural generic timer hardware. It is similar how the RDTSC timer is used in userspace on x86. Fix a permission problem where generic timer access from EL0 (or userspace on v7) was not properly initialized on APs. For ARMv7, mark the stack non-executable. The shared page is added for all arms (including ARMv8 64bit), and the signal trampoline code is moved to the page. Reviewed by: andrew Discussed with: emaste, mmel Sponsored by: The FreeBSD Foundation Differential revision: https://reviews.freebsd.org/D4209 --- lib/libc/aarch64/sys/Makefile.inc | 2 +- lib/libc/aarch64/sys/__vdso_gettc.c | 74 +++++++++++++++++++++++++++ lib/libc/arm/sys/Makefile.inc | 2 +- lib/libc/arm/sys/__vdso_gettc.c | 79 +++++++++++++++++++++++++++++ sys/arm/arm/elf_machdep.c | 10 +++- sys/arm/arm/generic_timer.c | 37 ++++++++++++-- sys/arm/arm/machdep.c | 20 +++++++- sys/arm/include/md_var.h | 5 ++ sys/arm/include/vdso.h | 3 +- sys/arm/include/vmparam.h | 3 +- sys/arm64/arm64/elf_machdep.c | 5 +- sys/arm64/arm64/machdep.c | 21 +++++++- sys/arm64/include/md_var.h | 5 ++ sys/arm64/include/vdso.h | 3 +- sys/arm64/include/vmparam.h | 3 +- sys/conf/files.arm | 1 - sys/conf/files.arm64 | 1 - sys/kern/imgact_elf.c | 6 ++- 18 files changed, 264 insertions(+), 16 deletions(-) create mode 100644 lib/libc/aarch64/sys/__vdso_gettc.c create mode 100644 lib/libc/arm/sys/__vdso_gettc.c diff --git a/lib/libc/aarch64/sys/Makefile.inc b/lib/libc/aarch64/sys/Makefile.inc index cb56f738b8cb..b33b568809d2 100644 --- a/lib/libc/aarch64/sys/Makefile.inc +++ b/lib/libc/aarch64/sys/Makefile.inc @@ -1,6 +1,6 @@ # $FreeBSD$ -SRCS+= trivial-vdso_tc.c +SRCS+= __vdso_gettc.c #MDASM= ptrace.S MDASM= brk.S \ diff --git a/lib/libc/aarch64/sys/__vdso_gettc.c b/lib/libc/aarch64/sys/__vdso_gettc.c new file mode 100644 index 000000000000..f9015e26ca76 --- /dev/null +++ b/lib/libc/aarch64/sys/__vdso_gettc.c @@ -0,0 +1,74 @@ +/*- + * Copyright (c) 2015 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include "libc_private.h" + +static inline uint64_t +cp15_cntvct_get(void) +{ + uint64_t reg; + + __asm __volatile("mrs %0, cntvct_el0" : "=r" (reg)); + return (reg); +} + +static inline uint64_t +cp15_cntpct_get(void) +{ + uint64_t reg; + + __asm __volatile("mrs %0, cntpct_el0" : "=r" (reg)); + return (reg); +} + +#pragma weak __vdso_gettc +u_int +__vdso_gettc(const struct vdso_timehands *th) +{ + uint64_t val; + + __asm __volatile("isb" : : : "memory"); + val = th->th_physical == 0 ? cp15_cntvct_get() : cp15_cntpct_get(); + return (val); +} + +#pragma weak __vdso_gettimekeep +int +__vdso_gettimekeep(struct vdso_timekeep **tk) +{ + + return (_elf_aux_info(AT_TIMEKEEP, tk, sizeof(*tk))); +} diff --git a/lib/libc/arm/sys/Makefile.inc b/lib/libc/arm/sys/Makefile.inc index 60c2dc3c6f43..5e89109828f0 100644 --- a/lib/libc/arm/sys/Makefile.inc +++ b/lib/libc/arm/sys/Makefile.inc @@ -1,6 +1,6 @@ # $FreeBSD$ -SRCS+= trivial-vdso_tc.c +SRCS+= __vdso_gettc.c MDASM= Ovfork.S brk.S cerror.S pipe.S ptrace.S sbrk.S shmat.S sigreturn.S syscall.S diff --git a/lib/libc/arm/sys/__vdso_gettc.c b/lib/libc/arm/sys/__vdso_gettc.c new file mode 100644 index 000000000000..d75d8662927e --- /dev/null +++ b/lib/libc/arm/sys/__vdso_gettc.c @@ -0,0 +1,79 @@ +/*- + * Copyright (c) 2015 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include "libc_private.h" + +static inline uint64_t +cp15_cntvct_get(void) +{ + uint64_t reg; + + __asm __volatile("mrrc\tp15, 1, %Q0, %R0, c14" : "=r" (reg)); + return (reg); +} + +static inline uint64_t +cp15_cntpct_get(void) +{ + uint64_t reg; + + __asm __volatile("mrrc\tp15, 0, %Q0, %R0, c14" : "=r" (reg)); + return (reg); +} + +#pragma weak __vdso_gettc +u_int +__vdso_gettc(const struct vdso_timehands *th) +{ + uint64_t val; + + /* + * Userspace gettimeofday() is only enabled on ARMv7 CPUs, but + * libc is compiled for ARMv6. Due to clang issues, .arch + * armv7-a directive does not work. + */ + __asm __volatile(".word\t0xf57ff06f" : : : "memory"); /* isb */ + val = th->th_physical == 0 ? cp15_cntvct_get() : cp15_cntpct_get(); + return (val); +} + +#pragma weak __vdso_gettimekeep +int +__vdso_gettimekeep(struct vdso_timekeep **tk) +{ + + return (_elf_aux_info(AT_TIMEKEEP, tk, sizeof(*tk))); +} diff --git a/sys/arm/arm/elf_machdep.c b/sys/arm/arm/elf_machdep.c index 1081bdff8ec4..02f7128db497 100644 --- a/sys/arm/arm/elf_machdep.c +++ b/sys/arm/arm/elf_machdep.c @@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include @@ -73,13 +74,20 @@ struct sysentvec elf32_freebsd_sysvec = { .sv_setregs = exec_setregs, .sv_fixlimit = NULL, .sv_maxssiz = NULL, - .sv_flags = SV_ABI_FREEBSD | SV_ILP32, + .sv_flags = +#if __ARM_ARCH >= 6 + SV_SHP | SV_TIMEKEEP | +#endif + SV_ABI_FREEBSD | SV_ILP32, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_syscallnames = syscallnames, + .sv_shared_page_base = SHAREDPAGE, + .sv_shared_page_len = PAGE_SIZE, .sv_schedtail = NULL, .sv_thread_detach = NULL, }; +INIT_SYSENTVEC(elf32_sysvec, &elf32_freebsd_sysvec); static Elf32_Brandinfo freebsd_brand_info = { .brand = ELFOSABI_FREEBSD, diff --git a/sys/arm/arm/generic_timer.c b/sys/arm/arm/generic_timer.c index 636ba7502e68..fd1f029ae193 100644 --- a/sys/arm/arm/generic_timer.c +++ b/sys/arm/arm/generic_timer.c @@ -49,10 +49,13 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include +#include #include #include #include #include +#include #ifdef FDT #include @@ -121,6 +124,9 @@ static struct timecounter arm_tmr_timecount = { #define set_el1(x, val) WRITE_SPECIALREG(x ##_el1, val) #endif +static uint32_t arm_tmr_fill_vdso_timehands(struct vdso_timehands *vdso_th, + struct timecounter *tc); + static int get_freq(void) { @@ -181,17 +187,32 @@ get_ctrl(bool physical) } static void -disable_user_access(void) +setup_user_access(void *arg __unused) { uint32_t cntkctl; cntkctl = get_el1(cntkctl); cntkctl &= ~(GT_CNTKCTL_PL0PTEN | GT_CNTKCTL_PL0VTEN | - GT_CNTKCTL_EVNTEN | GT_CNTKCTL_PL0VCTEN | GT_CNTKCTL_PL0PCTEN); + GT_CNTKCTL_EVNTEN); + if (arm_tmr_sc->physical) { + cntkctl |= GT_CNTKCTL_PL0PCTEN; + cntkctl &= ~GT_CNTKCTL_PL0VCTEN; + } else { + cntkctl |= GT_CNTKCTL_PL0VCTEN; + cntkctl &= ~GT_CNTKCTL_PL0PCTEN; + } set_el1(cntkctl, cntkctl); isb(); } +static void +tmr_setup_user_access(void *arg __unused) +{ + + smp_rendezvous(NULL, setup_user_access, NULL, NULL); +} +SYSINIT(tmr_ua, SI_SUB_SMP, SI_ORDER_SECOND, tmr_setup_user_access, NULL); + static unsigned arm_tmr_get_timecount(struct timecounter *tc) { @@ -381,7 +402,7 @@ arm_tmr_attach(device_t dev) } } - disable_user_access(); + arm_cpu_fill_vdso_timehands = arm_tmr_fill_vdso_timehands; arm_tmr_timecount.tc_frequency = sc->clkfreq; tc_init(&arm_tmr_timecount); @@ -485,3 +506,13 @@ DELAY(int usec) first = last; } } + +static uint32_t +arm_tmr_fill_vdso_timehands(struct vdso_timehands *vdso_th, + struct timecounter *tc) +{ + + vdso_th->th_physical = arm_tmr_sc->physical; + bzero(vdso_th->th_res, sizeof(vdso_th->th_res)); + return (tc == &arm_tmr_timecount); +} diff --git a/sys/arm/arm/machdep.c b/sys/arm/arm/machdep.c index bd01383ed946..c8ca2cc91fca 100644 --- a/sys/arm/arm/machdep.c +++ b/sys/arm/arm/machdep.c @@ -82,6 +82,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include @@ -271,6 +272,7 @@ sendsig(catcher, ksi, mask) struct trapframe *tf; struct sigframe *fp, frame; struct sigacts *psp; + struct sysentvec *sysent; int onstack; int sig; int code; @@ -337,7 +339,12 @@ sendsig(catcher, ksi, mask) tf->tf_r5 = (register_t)&fp->sf_uc; tf->tf_pc = (register_t)catcher; tf->tf_usr_sp = (register_t)fp; - tf->tf_usr_lr = (register_t)(PS_STRINGS - *(p->p_sysent->sv_szsigcode)); + sysent = p->p_sysent; + if (sysent->sv_sigcode_base != 0) + tf->tf_usr_lr = (register_t)sysent->sv_sigcode_base; + else + tf->tf_usr_lr = (register_t)(sysent->sv_psstrings - + *(sysent->sv_szsigcode)); /* Set the mode to enter in the signal handler */ #if __ARM_ARCH >= 7 if ((register_t)catcher & 1) @@ -1910,3 +1917,14 @@ initarm(struct arm_boot_params *abp) #endif /* !ARM_NEW_PMAP */ #endif /* FDT */ + +uint32_t (*arm_cpu_fill_vdso_timehands)(struct vdso_timehands *, + struct timecounter *); + +uint32_t +cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th, struct timecounter *tc) +{ + + return (arm_cpu_fill_vdso_timehands != NULL ? + arm_cpu_fill_vdso_timehands(vdso_th, tc) : 0); +} diff --git a/sys/arm/include/md_var.h b/sys/arm/include/md_var.h index 642124d9d214..981c0b379b70 100644 --- a/sys/arm/include/md_var.h +++ b/sys/arm/include/md_var.h @@ -45,6 +45,11 @@ extern int (*_arm_bzero)(void *, int, int); extern int _min_memcpy_size; extern int _min_bzero_size; +struct vdso_timehands; +struct timecounter; +extern uint32_t (*arm_cpu_fill_vdso_timehands)(struct vdso_timehands *, + struct timecounter *); + #define DST_IS_USER 0x1 #define SRC_IS_USER 0x2 #define IS_PHYSICAL 0x4 diff --git a/sys/arm/include/vdso.h b/sys/arm/include/vdso.h index d6aa162a322d..6a0ec7b57595 100644 --- a/sys/arm/include/vdso.h +++ b/sys/arm/include/vdso.h @@ -29,6 +29,7 @@ #define _ARM_VDSO_H #define VDSO_TIMEHANDS_MD \ - uint32_t th_res[8]; + uint32_t th_physical; \ + uint32_t th_res[7]; #endif diff --git a/sys/arm/include/vmparam.h b/sys/arm/include/vmparam.h index b6e76bf35758..36f19a39c1f8 100644 --- a/sys/arm/include/vmparam.h +++ b/sys/arm/include/vmparam.h @@ -124,7 +124,8 @@ #endif #define VM_MAX_ADDRESS VM_MAXUSER_ADDRESS -#define USRSTACK VM_MAXUSER_ADDRESS +#define SHAREDPAGE (VM_MAXUSER_ADDRESS - PAGE_SIZE) +#define USRSTACK SHAREDPAGE /* initial pagein size of beginning of executable file */ #ifndef VM_INITIAL_PAGEIN diff --git a/sys/arm64/arm64/elf_machdep.c b/sys/arm64/arm64/elf_machdep.c index 2506de3e4f69..9ba2541432fb 100644 --- a/sys/arm64/arm64/elf_machdep.c +++ b/sys/arm64/arm64/elf_machdep.c @@ -80,12 +80,15 @@ static struct sysentvec elf64_freebsd_sysvec = { .sv_setregs = exec_setregs, .sv_fixlimit = NULL, .sv_maxssiz = NULL, - .sv_flags = SV_ABI_FREEBSD | SV_LP64, + .sv_flags = SV_SHP | SV_TIMEKEEP | SV_ABI_FREEBSD | SV_LP64, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_syscallnames = syscallnames, + .sv_shared_page_base = SHAREDPAGE, + .sv_shared_page_len = PAGE_SIZE, .sv_schedtail = NULL, }; +INIT_SYSENTVEC(elf64_sysvec, &elf64_freebsd_sysvec); static Elf64_Brandinfo freebsd_brand_info = { .brand = ELFOSABI_FREEBSD, diff --git a/sys/arm64/arm64/machdep.c b/sys/arm64/arm64/machdep.c index e92a99ff3057..f7e1456ceb8d 100644 --- a/sys/arm64/arm64/machdep.c +++ b/sys/arm64/arm64/machdep.c @@ -56,6 +56,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include @@ -72,6 +73,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -505,6 +507,7 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) struct trapframe *tf; struct sigframe *fp, frame; struct sigacts *psp; + struct sysentvec *sysent; int code, onstack, sig; td = curthread; @@ -563,7 +566,12 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) tf->tf_elr = (register_t)catcher; tf->tf_sp = (register_t)fp; - tf->tf_lr = (register_t)(PS_STRINGS - *(p->p_sysent->sv_szsigcode)); + sysent = p->p_sysent; + if (sysent->sv_sigcode_base != 0) + tf->tf_lr = (register_t)sysent->sv_sigcode_base; + else + tf->tf_lr = (register_t)(sysent->sv_psstrings - + *(sysent->sv_szsigcode)); CTR3(KTR_SIG, "sendsig: return td=%p pc=%#x sp=%#x", td, tf->tf_elr, tf->tf_sp); @@ -875,6 +883,17 @@ initarm(struct arm64_bootparams *abp) early_boot = 0; } +uint32_t (*arm_cpu_fill_vdso_timehands)(struct vdso_timehands *, + struct timecounter *); + +uint32_t +cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th, struct timecounter *tc) +{ + + return (arm_cpu_fill_vdso_timehands != NULL ? + arm_cpu_fill_vdso_timehands(vdso_th, tc) : 0); +} + #ifdef DDB #include diff --git a/sys/arm64/include/md_var.h b/sys/arm64/include/md_var.h index 85455910ee75..d6cecfcd7941 100644 --- a/sys/arm64/include/md_var.h +++ b/sys/arm64/include/md_var.h @@ -46,4 +46,9 @@ void dump_add_page(vm_paddr_t); void dump_drop_page(vm_paddr_t); int minidumpsys(struct dumperinfo *); +struct vdso_timehands; +struct timecounter; +extern uint32_t (*arm_cpu_fill_vdso_timehands)(struct vdso_timehands *, + struct timecounter *); + #endif /* !_MACHINE_MD_VAR_H_ */ diff --git a/sys/arm64/include/vdso.h b/sys/arm64/include/vdso.h index 5a8f7f555b50..285e986d7a1f 100644 --- a/sys/arm64/include/vdso.h +++ b/sys/arm64/include/vdso.h @@ -29,6 +29,7 @@ #define _MACHINE_VDSO_H_ #define VDSO_TIMEHANDS_MD \ - uint32_t th_res[8]; + uint32_t th_physical; \ + uint32_t th_res[7]; #endif /* !_MACHINE_VDSO_H_ */ diff --git a/sys/arm64/include/vmparam.h b/sys/arm64/include/vmparam.h index e3c1d74bb438..2752ee118be9 100644 --- a/sys/arm64/include/vmparam.h +++ b/sys/arm64/include/vmparam.h @@ -194,7 +194,8 @@ extern vm_paddr_t dmap_phys_base; #define VM_MAXUSER_ADDRESS (VM_MAX_USER_ADDRESS) #define KERNBASE (VM_MIN_KERNEL_ADDRESS) -#define USRSTACK (VM_MAX_USER_ADDRESS) +#define SHAREDPAGE (VM_MAXUSER_ADDRESS - PAGE_SIZE) +#define USRSTACK SHAREDPAGE /* * How many physical pages per kmem arena virtual page. diff --git a/sys/conf/files.arm b/sys/conf/files.arm index cd27a3918726..1c57106ef84b 100644 --- a/sys/conf/files.arm +++ b/sys/conf/files.arm @@ -110,7 +110,6 @@ font.h optional sc \ no-obj no-implicit-rule before-depend \ clean "font.h ${SC_DFLT_FONT}-8x14 ${SC_DFLT_FONT}-8x16 ${SC_DFLT_FONT}-8x8" kern/subr_busdma_bufalloc.c standard -kern/subr_dummy_vdso_tc.c standard kern/subr_sfbuf.c standard libkern/arm/aeabi_unwind.c standard libkern/arm/divsi3.S standard diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 index 7c8c4f783d2a..6ecf36abb58e 100644 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -79,7 +79,6 @@ dev/vnic/thunder_mdio_fdt.c optional vnic fdt dev/vnic/thunder_mdio.c optional vnic dev/vnic/lmac_if.m optional vnic kern/kern_clocksource.c standard -kern/subr_dummy_vdso_tc.c standard libkern/bcmp.c standard libkern/ffs.c standard libkern/ffsl.c standard diff --git a/sys/kern/imgact_elf.c b/sys/kern/imgact_elf.c index 7fe36cfc7a1a..ea802e07c7ea 100644 --- a/sys/kern/imgact_elf.c +++ b/sys/kern/imgact_elf.c @@ -80,6 +80,9 @@ __FBSDID("$FreeBSD$"); #include #include +#ifdef __arm__ +#include +#endif #define ELF_NOTE_ROUNDSIZE 4 #define OLD_EI_BRAND 8 @@ -116,7 +119,8 @@ SYSCTL_INT(_debug, OID_AUTO, __elfN(legacy_coredump), CTLFLAG_RW, &elf_legacy_coredump, 0, ""); int __elfN(nxstack) = -#if defined(__amd64__) || defined(__powerpc64__) /* both 64 and 32 bit */ +#if defined(__amd64__) || defined(__powerpc64__) /* both 64 and 32 bit */ || \ + (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__) 1; #else 0;