From 849aef496d2ae19961310f4e92f3a0b928732d26 Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Thu, 21 Nov 2019 11:22:08 +0000 Subject: [PATCH] Port the NetBSD KCSAN runtime to FreeBSD. Update the NetBSD Kernel Concurrency Sanitizer (KCSAN) runtime to work in the FreeBSD kernel. It is a useful tool for finding data races between threads executing on different CPUs. This can be enabled by enabling KCSAN in the kernel config, or by using the GENERIC-KCSAN amd64 kernel. It works on amd64 and arm64, however the later needs a compiler change to allow -fsanitize=thread that KCSAN uses. Sponsored by: DARPA, AFRL Differential Revision: https://reviews.freebsd.org/D22315 --- sys/amd64/amd64/copyout.c | 4 + sys/amd64/amd64/machdep.c | 46 +- sys/amd64/conf/GENERIC | 1 + sys/amd64/conf/GENERIC-KCSAN | 33 ++ sys/amd64/include/atomic.h | 25 +- sys/amd64/include/csan.h | 67 +++ sys/arm64/arm64/bus_machdep.c | 2 + sys/arm64/arm64/copystr.c | 2 +- sys/arm64/arm64/machdep.c | 3 + sys/arm64/arm64/mp_machdep.c | 3 + sys/arm64/conf/GENERIC | 1 + sys/arm64/include/atomic.h | 10 +- sys/arm64/include/bus.h | 5 + sys/arm64/include/csan.h | 104 ++++ sys/conf/files | 2 + sys/conf/files.arm64 | 6 +- sys/conf/kern.post.mk | 4 + sys/conf/kern.pre.mk | 5 + sys/conf/options | 1 + sys/kern/subr_csan.c | 881 +++++++++++++++++++--------------- sys/kern/vfs_aio.c | 12 +- sys/libkern/strcmp.c | 2 +- sys/libkern/strcpy.c | 2 +- sys/libkern/strlen.c | 2 +- sys/modules/Makefile | 4 +- sys/sys/_cscan_atomic.h | 307 ++++++++++++ sys/sys/_cscan_bus.h | 190 ++++++++ sys/sys/csan.h | 12 +- sys/sys/libkern.h | 9 + sys/sys/systm.h | 38 +- sys/x86/include/bus.h | 15 +- sys/x86/x86/bus_machdep.c | 2 + sys/x86/x86/mp_x86.c | 3 + 33 files changed, 1359 insertions(+), 444 deletions(-) create mode 100644 sys/amd64/conf/GENERIC-KCSAN create mode 100644 sys/amd64/include/csan.h create mode 100644 sys/arm64/include/csan.h create mode 100644 sys/sys/_cscan_atomic.h create mode 100644 sys/sys/_cscan_bus.h diff --git a/sys/amd64/amd64/copyout.c b/sys/amd64/amd64/copyout.c index d4464e2c39f9..43d7ee6f629d 100644 --- a/sys/amd64/amd64/copyout.c +++ b/sys/amd64/amd64/copyout.c @@ -146,6 +146,10 @@ DEFINE_IFUNC(, int, casueword, (volatile u_long *, u_long, u_long *, u_long)) casueword_smap : casueword_nosmap); } +#undef copyinstr +#undef copyin +#undef copyout + int copyinstr_nosmap(const void *udaddr, void *kaddr, size_t len, size_t *lencopied); int copyinstr_smap(const void *udaddr, void *kaddr, size_t len, diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c index 0575fe5e9611..e956118cc38a 100644 --- a/sys/amd64/amd64/machdep.c +++ b/sys/amd64/amd64/machdep.c @@ -64,6 +64,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -1899,6 +1900,8 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) cpu_probe_amdc1e(); + kcsan_cpu_init(0); + #ifdef FDT x86_init_fdt(); #endif @@ -2722,6 +2725,40 @@ outb_(u_short port, u_char data) void *memset_std(void *buf, int c, size_t len); void *memset_erms(void *buf, int c, size_t len); +void *memmove_std(void * _Nonnull dst, const void * _Nonnull src, + size_t len); +void *memmove_erms(void * _Nonnull dst, const void * _Nonnull src, + size_t len); +void *memcpy_std(void * _Nonnull dst, const void * _Nonnull src, + size_t len); +void *memcpy_erms(void * _Nonnull dst, const void * _Nonnull src, + size_t len); + +#ifdef KCSAN +/* + * These fail to build as ifuncs when used with KCSAN. + */ +void * +memset(void *buf, int c, size_t len) +{ + + return memset_std(buf, c, len); +} + +void * +memmove(void * _Nonnull dst, const void * _Nonnull src, size_t len) +{ + + return memmove_std(dst, src, len); +} + +void * +memcpy(void * _Nonnull dst, const void * _Nonnull src, size_t len) +{ + + return memcpy_std(dst, src, len); +} +#else DEFINE_IFUNC(, void *, memset, (void *, int, size_t)) { @@ -2729,10 +2766,6 @@ DEFINE_IFUNC(, void *, memset, (void *, int, size_t)) memset_erms : memset_std); } -void *memmove_std(void * _Nonnull dst, const void * _Nonnull src, - size_t len); -void *memmove_erms(void * _Nonnull dst, const void * _Nonnull src, - size_t len); DEFINE_IFUNC(, void *, memmove, (void * _Nonnull, const void * _Nonnull, size_t)) { @@ -2741,16 +2774,13 @@ DEFINE_IFUNC(, void *, memmove, (void * _Nonnull, const void * _Nonnull, memmove_erms : memmove_std); } -void *memcpy_std(void * _Nonnull dst, const void * _Nonnull src, - size_t len); -void *memcpy_erms(void * _Nonnull dst, const void * _Nonnull src, - size_t len); DEFINE_IFUNC(, void *, memcpy, (void * _Nonnull, const void * _Nonnull,size_t)) { return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ? memcpy_erms : memcpy_std); } +#endif void pagezero_std(void *addr); void pagezero_erms(void *addr); diff --git a/sys/amd64/conf/GENERIC b/sys/amd64/conf/GENERIC index 8f667a805068..95b95b681761 100644 --- a/sys/amd64/conf/GENERIC +++ b/sys/amd64/conf/GENERIC @@ -106,6 +106,7 @@ options VERBOSE_SYSINIT=0 # Support debug.verbose_sysinit, off by default #options KCOV # Kernel Coverage Sanitizer # Warning: KUBSAN can result in a kernel too large for loader to load #options KUBSAN # Kernel Undefined Behavior Sanitizer +#options KCSAN # Kernel Concurrency Sanitizer # Kernel dump features. options EKCD # Support for encrypted kernel dumps diff --git a/sys/amd64/conf/GENERIC-KCSAN b/sys/amd64/conf/GENERIC-KCSAN new file mode 100644 index 000000000000..2864596860ab --- /dev/null +++ b/sys/amd64/conf/GENERIC-KCSAN @@ -0,0 +1,33 @@ +# +# GENERIC-KCSAN -- Kernel Concurrency Sanitizer kernel configuration file +# for FreeBSD/amd64 +# +# This configuration file removes several debugging options, including +# WITNESS and INVARIANTS checking, which are known to have significant +# performance impact on running systems. When benchmarking new features +# this kernel should be used instead of the standard GENERIC. +# This kernel configuration should never appear outside of the HEAD +# of the FreeBSD tree. +# +# For more information on this file, please read the config(5) manual page, +# and/or the handbook section on Kernel Configuration Files: +# +# https://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html +# +# The handbook is also available locally in /usr/share/doc/handbook +# if you've installed the doc distribution, otherwise always see the +# FreeBSD World Wide Web server (https://www.FreeBSD.org/) for the +# latest information. +# +# An exhaustive list of options and more detailed explanations of the +# device lines is also present in the ../../conf/NOTES and NOTES files. +# If you are in doubt as to the purpose or necessity of a line, check first +# in NOTES. +# +# $FreeBSD$ + +include GENERIC + +ident GENERIC-KCSAN + +options KCSAN diff --git a/sys/amd64/include/atomic.h b/sys/amd64/include/atomic.h index 000d13aa135a..293aa6870056 100644 --- a/sys/amd64/include/atomic.h +++ b/sys/amd64/include/atomic.h @@ -57,6 +57,20 @@ #define wmb() __asm __volatile("sfence;" : : : "memory") #define rmb() __asm __volatile("lfence;" : : : "memory") +#ifdef _KERNEL +/* + * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf). + * + * The open-coded number is used instead of the symbolic expression to + * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. + * An assertion in amd64/vm_machdep.c ensures that the value is correct. + */ +#define OFFSETOF_MONITORBUF 0x100 +#endif + +#if defined(KCSAN) && !defined(KCSAN_RUNTIME) +#include +#else #include /* @@ -345,15 +359,6 @@ atomic_testandclear_long(volatile u_long *p, u_int v) #if defined(_KERNEL) -/* - * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf). - * - * The open-coded number is used instead of the symbolic expression to - * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. - * An assertion in amd64/vm_machdep.c ensures that the value is correct. - */ -#define OFFSETOF_MONITORBUF 0x100 - #if defined(SMP) || defined(KLD_MODULE) static __inline void __storeload_barrier(void) @@ -679,4 +684,6 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); #endif /* !WANT_FUNCTIONS */ +#endif /* KCSAN && !KCSAN_RUNTIME */ + #endif /* !_MACHINE_ATOMIC_H_ */ diff --git a/sys/amd64/include/csan.h b/sys/amd64/include/csan.h new file mode 100644 index 000000000000..43c37dd1a541 --- /dev/null +++ b/sys/amd64/include/csan.h @@ -0,0 +1,67 @@ +/* $NetBSD: csan.h,v 1.2 2019/11/06 06:57:22 maxv Exp $ */ + +/* + * Copyright (c) 2019 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Maxime Villard. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include + +static inline bool +kcsan_md_is_avail(void) +{ + return true; +} + +static inline void +kcsan_md_disable_intrs(uint64_t *state) +{ + + *state = intr_disable(); +} + +static inline void +kcsan_md_enable_intrs(uint64_t *state) +{ + + intr_restore(*state); +} + +static inline void +kcsan_md_delay(uint64_t us) +{ + DELAY(us); +} + +static void +kcsan_md_unwind(void) +{ +} diff --git a/sys/arm64/arm64/bus_machdep.c b/sys/arm64/arm64/bus_machdep.c index f6df4a1eb52a..1fabb91c575f 100644 --- a/sys/arm64/arm64/bus_machdep.c +++ b/sys/arm64/arm64/bus_machdep.c @@ -25,6 +25,8 @@ * */ +#define KCSAN_RUNTIME + #include "opt_platform.h" #include diff --git a/sys/arm64/arm64/copystr.c b/sys/arm64/arm64/copystr.c index ebb4acddfbaf..fd4e9af14223 100644 --- a/sys/arm64/arm64/copystr.c +++ b/sys/arm64/arm64/copystr.c @@ -32,7 +32,7 @@ __FBSDID("$FreeBSD$"); #include int -copystr(const void * __restrict kfaddr, void * __restrict kdaddr, size_t len, +(copystr)(const void * __restrict kfaddr, void * __restrict kdaddr, size_t len, size_t * __restrict lencopied) { const char *src; diff --git a/sys/arm64/arm64/machdep.c b/sys/arm64/arm64/machdep.c index d41bae02c0b5..77d01d399ae7 100644 --- a/sys/arm64/arm64/machdep.c +++ b/sys/arm64/arm64/machdep.c @@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -1209,6 +1210,8 @@ initarm(struct arm64_bootparams *abp) kdb_init(); pan_enable(); + kcsan_cpu_init(0); + env = kern_getenv("kernelname"); if (env != NULL) strlcpy(kernelname, env, sizeof(kernelname)); diff --git a/sys/arm64/arm64/mp_machdep.c b/sys/arm64/arm64/mp_machdep.c index 2efdce76c891..bbda274bb226 100644 --- a/sys/arm64/arm64/mp_machdep.c +++ b/sys/arm64/arm64/mp_machdep.c @@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -253,6 +254,8 @@ init_secondary(uint64_t cpu) mtx_unlock_spin(&ap_boot_mtx); + kcsan_cpu_init(cpu); + /* Enter the scheduler */ sched_throw(NULL); diff --git a/sys/arm64/conf/GENERIC b/sys/arm64/conf/GENERIC index d48c84e2f102..4306c45ba532 100644 --- a/sys/arm64/conf/GENERIC +++ b/sys/arm64/conf/GENERIC @@ -100,6 +100,7 @@ options VERBOSE_SYSINIT=0 # Support debug.verbose_sysinit, off by default #options KCOV # Kernel Coverage Sanitizer # Warning: KUBSAN can result in a kernel too large for loader to load #options KUBSAN # Kernel Undefined Behavior Sanitizer +#options KCSAN # Kernel Concurrency Sanitizer # Kernel dump features. options EKCD # Support for encrypted kernel dumps diff --git a/sys/arm64/include/atomic.h b/sys/arm64/include/atomic.h index 27f03359da34..8897ad77716c 100644 --- a/sys/arm64/include/atomic.h +++ b/sys/arm64/include/atomic.h @@ -29,8 +29,6 @@ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ -#include - #define isb() __asm __volatile("isb" : : : "memory") /* @@ -55,6 +53,12 @@ #define wmb() dmb(st) /* Full system memory barrier store */ #define rmb() dmb(ld) /* Full system memory barrier load */ +#if defined(KCSAN) && !defined(KCSAN_RUNTIME) +#include +#else + +#include + #define ATOMIC_OP(op, asm_op, bar, a, l) \ static __inline void \ atomic_##op##_##bar##8(volatile uint8_t *p, uint8_t val) \ @@ -601,5 +605,7 @@ atomic_thread_fence_seq_cst(void) dmb(sy); } +#endif /* KCSAN && !KCSAN_RUNTIME */ + #endif /* _MACHINE_ATOMIC_H_ */ diff --git a/sys/arm64/include/bus.h b/sys/arm64/include/bus.h index 90e00bbd635c..f2f4bd1f457d 100644 --- a/sys/arm64/include/bus.h +++ b/sys/arm64/include/bus.h @@ -89,6 +89,9 @@ #define BUS_SPACE_BARRIER_READ 0x01 #define BUS_SPACE_BARRIER_WRITE 0x02 +#if defined(KCSAN) && !defined(KCSAN_RUNTIME) +#include +#else struct bus_space { /* cookie */ @@ -464,6 +467,8 @@ struct bus_space { #define bus_space_copy_region_8(t, h1, o1, h2, o2, c) \ __bs_copy(8, t, h1, o1, h2, o2, c) +#endif + #include #endif /* _MACHINE_BUS_H_ */ diff --git a/sys/arm64/include/csan.h b/sys/arm64/include/csan.h new file mode 100644 index 000000000000..ddcd5972d06f --- /dev/null +++ b/sys/arm64/include/csan.h @@ -0,0 +1,104 @@ +/* $NetBSD: csan.h,v 1.2 2019/11/06 06:57:22 maxv Exp $ */ + +/* + * Copyright (c) 2019 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Maxime Villard. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include + +static inline bool +kcsan_md_is_avail(void) +{ + return true; +} + +static inline void +kcsan_md_disable_intrs(uint64_t *state) +{ + + *state = intr_disable(); +} + +static inline void +kcsan_md_enable_intrs(uint64_t *state) +{ + + intr_restore(*state); +} + +static inline void +kcsan_md_delay(uint64_t us) +{ + DELAY(us); +} + +static void +kcsan_md_unwind(void) +{ +#ifdef DDB + c_db_sym_t sym; + db_expr_t offset; + const char *symname; +#endif + struct unwind_state frame; + uint64_t sp; + int nsym; + + __asm __volatile("mov %0, sp" : "=&r" (sp)); + + frame.sp = sp; + frame.fp = (uint64_t)__builtin_frame_address(0); + frame.pc = (uint64_t)kcsan_md_unwind; + nsym = 0; + + while (1) { + unwind_frame(&frame); + if (!INKERNEL((vm_offset_t)frame.fp) || + !INKERNEL((vm_offset_t)frame.pc)) + break; + +#ifdef DDB + sym = db_search_symbol((vm_offset_t)frame.pc, DB_STGY_PROC, + &offset); + db_symbol_values(sym, &symname, NULL); + printf("#%d %p in %s+%#lx\n", nsym, (void *)frame.pc, + symname, offset); +#else + printf("#%d %p\n", nsym, (void *)frame.pc); +#endif + nsym++; + + if (nsym >= 15) { + break; + } + } +} diff --git a/sys/conf/files b/sys/conf/files index dee97b6d7a08..9a9b454d2af4 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -3805,6 +3805,8 @@ kern/subr_compressor.c standard \ kern/subr_coverage.c optional coverage \ compile-with "${NORMAL_C:N-fsanitize*}" kern/subr_counter.c standard +kern/subr_csan.c optional kcsan \ + compile-with "${NORMAL_C:N-fsanitize*}" kern/subr_devstat.c standard kern/subr_disk.c standard kern/subr_early.c standard diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 index ba5a95d4eec6..89d5594fb6bc 100644 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -286,8 +286,10 @@ kern/pic_if.m optional intrng kern/subr_devmap.c standard kern/subr_intr.c optional intrng libkern/bcmp.c standard -libkern/memcmp.c standard -libkern/memset.c standard +libkern/memcmp.c standard \ + compile-with "${NORMAL_C:N-fsanitize*}" +libkern/memset.c standard \ + compile-with "${NORMAL_C:N-fsanitize*}" libkern/arm64/crc32c_armv8.S standard cddl/dev/dtrace/aarch64/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/aarch64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" diff --git a/sys/conf/kern.post.mk b/sys/conf/kern.post.mk index ff10daf1a0a5..2c58ef33cd98 100644 --- a/sys/conf/kern.post.mk +++ b/sys/conf/kern.post.mk @@ -38,6 +38,10 @@ MKMODULESENV+= WITH_CTF="${WITH_CTF}" MKMODULESENV+= WITH_EXTRA_TCP_STACKS="${WITH_EXTRA_TCP_STACKS}" .endif +.if defined(KCSAN_ENABLED) +MKMODULESENV+= KCSAN_ENABLED="yes" +.endif + .if defined(SAN_CFLAGS) MKMODULESENV+= SAN_CFLAGS="${SAN_CFLAGS}" .endif diff --git a/sys/conf/kern.pre.mk b/sys/conf/kern.pre.mk index 9319fa037586..b4986d6f33f4 100644 --- a/sys/conf/kern.pre.mk +++ b/sys/conf/kern.pre.mk @@ -117,6 +117,11 @@ PROF= -pg .endif DEFINED_PROF= ${PROF} +KCSAN_ENABLED!= grep KCSAN opt_global.h || true ; echo +.if !empty(KCSAN_ENABLED) +SAN_CFLAGS+= -fsanitize=thread +.endif + KUBSAN_ENABLED!= grep KUBSAN opt_global.h || true ; echo .if !empty(KUBSAN_ENABLED) SAN_CFLAGS+= -fsanitize=undefined diff --git a/sys/conf/options b/sys/conf/options index f4ceede4a965..b73b3dfb11ac 100644 --- a/sys/conf/options +++ b/sys/conf/options @@ -230,6 +230,7 @@ ZSTDIO opt_zstdio.h # Sanitizers COVERAGE opt_global.h KCOV +KCSAN opt_global.h KUBSAN opt_global.h # POSIX kernel options diff --git a/sys/kern/subr_csan.c b/sys/kern/subr_csan.c index d8c52f4fdd3a..6159a46835b4 100644 --- a/sys/kern/subr_csan.c +++ b/sys/kern/subr_csan.c @@ -3,6 +3,7 @@ /* * Copyright (c) 2019 The NetBSD Foundation, Inc. * All rights reserved. + * Copyright (c) 2019 Andrew Turner * * This code is derived from software contributed to The NetBSD Foundation * by Maxime Villard. @@ -29,18 +30,25 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#define KCSAN_RUNTIME + +#include "opt_ddb.h" + #include -__KERNEL_RCSID(0, "$NetBSD: subr_csan.c,v 1.5 2019/11/15 08:11:37 maxv Exp $"); +__FBSDID("$FreeBSD$"); #include -#include #include -#include +#include #include -#include -#include -#include #include +#include +#include +#include +#include + +#include +#include #ifdef KCSAN_PANIC #define REPORT panic @@ -62,7 +70,7 @@ typedef struct { csan_cell_t cell; } csan_cpu_t; -static csan_cpu_t kcsan_cpus[MAXCPUS]; +static csan_cpu_t kcsan_cpus[MAXCPU]; static bool kcsan_enabled __read_mostly; #define __RET_ADDR (uintptr_t)__builtin_return_address(0) @@ -77,34 +85,43 @@ static bool kcsan_enabled __read_mostly; /* -------------------------------------------------------------------------- */ -void -kcsan_init(void) +static void +kcsan_enable(void *dummy __unused) { + + printf("Enabling KCSCAN, expect reduced performance.\n"); kcsan_enabled = true; } +SYSINIT(kcsan_enable, SI_SUB_SMP, SI_ORDER_SECOND, kcsan_enable, NULL); void -kcsan_cpu_init(struct cpu_info *ci) +kcsan_cpu_init(u_int cpu) { - kcsan_cpus[cpu_index(ci)].inited = true; + kcsan_cpus[cpu].inited = true; } /* -------------------------------------------------------------------------- */ static inline void -kcsan_report(csan_cell_t *new, cpuid_t newcpu, csan_cell_t *old, cpuid_t oldcpu) +kcsan_report(csan_cell_t *new, u_int newcpu, csan_cell_t *old, u_int oldcpu) { const char *newsym, *oldsym; +#ifdef DDB + c_db_sym_t sym; + db_expr_t offset; - if (ksyms_getname(NULL, &newsym, (vaddr_t)new->pc, KSYMS_PROC) != 0) { - newsym = "Unknown"; - } - if (ksyms_getname(NULL, &oldsym, (vaddr_t)old->pc, KSYMS_PROC) != 0) { - oldsym = "Unknown"; - } + sym = db_search_symbol((vm_offset_t)new->pc, DB_STGY_PROC, &offset); + db_symbol_values(sym, &newsym, NULL); + + sym = db_search_symbol((vm_offset_t)old->pc, DB_STGY_PROC, &offset); + db_symbol_values(sym, &oldsym, NULL); +#else + newsym = ""; + oldsym = ""; +#endif REPORT("CSan: Racy Access " - "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>] " - "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>]\n", + "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>] " + "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>]\n", newcpu, (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"), (void *)new->addr, new->size, (void *)new->pc, newsym, @@ -134,8 +151,6 @@ kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc) if (__predict_false(!kcsan_enabled)) return; - if (__predict_false(kcsan_md_unsupported((vaddr_t)addr))) - return; new.addr = addr; new.size = size; @@ -143,7 +158,7 @@ kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc) new.atomic = atomic; new.pc = pc; - for (i = 0; i < ncpu; i++) { + CPU_FOREACH(i) { __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old)); if (old.addr + old.size <= new.addr) @@ -155,7 +170,7 @@ kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc) if (__predict_true(kcsan_access_is_atomic(&new, &old))) continue; - kcsan_report(&new, cpu_number(), &old, i); + kcsan_report(&new, PCPU_GET(cpuid), &old, i); break; } @@ -164,7 +179,7 @@ kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc) kcsan_md_disable_intrs(&intr); - cpu = &kcsan_cpus[cpu_number()]; + cpu = &kcsan_cpus[PCPU_GET(cpuid)]; if (__predict_false(!cpu->inited)) goto out; cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES; @@ -182,6 +197,11 @@ kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc) #define CSAN_READ(size) \ void __tsan_read##size(uintptr_t); \ void __tsan_read##size(uintptr_t addr) \ + { \ + kcsan_access(addr, size, false, false, __RET_ADDR); \ + } \ + void __tsan_unaligned_read##size(uintptr_t); \ + void __tsan_unaligned_read##size(uintptr_t addr) \ { \ kcsan_access(addr, size, false, false, __RET_ADDR); \ } @@ -195,6 +215,11 @@ CSAN_READ(16) #define CSAN_WRITE(size) \ void __tsan_write##size(uintptr_t); \ void __tsan_write##size(uintptr_t addr) \ + { \ + kcsan_access(addr, size, true, false, __RET_ADDR); \ + } \ + void __tsan_unaligned_write##size(uintptr_t); \ + void __tsan_unaligned_write##size(uintptr_t addr) \ { \ kcsan_access(addr, size, true, false, __RET_ADDR); \ } @@ -321,33 +346,12 @@ kcsan_strlen(const char *str) return (s - str); } -#undef kcopy #undef copystr -#undef copyinstr -#undef copyoutstr #undef copyin +#undef copyin_nofault +#undef copyinstr #undef copyout - -int kcsan_kcopy(const void *, void *, size_t); -int kcsan_copystr(const void *, void *, size_t, size_t *); -int kcsan_copyinstr(const void *, void *, size_t, size_t *); -int kcsan_copyoutstr(const void *, void *, size_t, size_t *); -int kcsan_copyin(const void *, void *, size_t); -int kcsan_copyout(const void *, void *, size_t); -int kcopy(const void *, void *, size_t); -int copystr(const void *, void *, size_t, size_t *); -int copyinstr(const void *, void *, size_t, size_t *); -int copyoutstr(const void *, void *, size_t, size_t *); -int copyin(const void *, void *, size_t); -int copyout(const void *, void *, size_t); - -int -kcsan_kcopy(const void *src, void *dst, size_t len) -{ - kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); - kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); - return kcopy(src, dst, len); -} +#undef copyout_nofault int kcsan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done) @@ -363,13 +367,6 @@ kcsan_copyin(const void *uaddr, void *kaddr, size_t len) return copyin(uaddr, kaddr, len); } -int -kcsan_copyout(const void *kaddr, void *uaddr, size_t len) -{ - kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); - return copyout(kaddr, uaddr, len); -} - int kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) { @@ -378,377 +375,477 @@ kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) } int -kcsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) +kcsan_copyout(const void *kaddr, void *uaddr, size_t len) { kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); - return copyoutstr(kaddr, uaddr, len, done); + return copyout(kaddr, uaddr, len); } /* -------------------------------------------------------------------------- */ -#undef atomic_add_32 -#undef atomic_add_int -#undef atomic_add_long -#undef atomic_add_ptr -#undef atomic_add_64 -#undef atomic_add_32_nv -#undef atomic_add_int_nv -#undef atomic_add_long_nv -#undef atomic_add_ptr_nv -#undef atomic_add_64_nv -#undef atomic_and_32 -#undef atomic_and_uint -#undef atomic_and_ulong -#undef atomic_and_64 -#undef atomic_and_32_nv -#undef atomic_and_uint_nv -#undef atomic_and_ulong_nv -#undef atomic_and_64_nv -#undef atomic_or_32 -#undef atomic_or_uint -#undef atomic_or_ulong -#undef atomic_or_64 -#undef atomic_or_32_nv -#undef atomic_or_uint_nv -#undef atomic_or_ulong_nv -#undef atomic_or_64_nv -#undef atomic_cas_32 -#undef atomic_cas_uint -#undef atomic_cas_ulong -#undef atomic_cas_ptr -#undef atomic_cas_64 -#undef atomic_cas_32_ni -#undef atomic_cas_uint_ni -#undef atomic_cas_ulong_ni -#undef atomic_cas_ptr_ni -#undef atomic_cas_64_ni -#undef atomic_swap_32 -#undef atomic_swap_uint -#undef atomic_swap_ulong -#undef atomic_swap_ptr -#undef atomic_swap_64 -#undef atomic_dec_32 -#undef atomic_dec_uint -#undef atomic_dec_ulong -#undef atomic_dec_ptr -#undef atomic_dec_64 -#undef atomic_dec_32_nv -#undef atomic_dec_uint_nv -#undef atomic_dec_ulong_nv -#undef atomic_dec_ptr_nv -#undef atomic_dec_64_nv -#undef atomic_inc_32 -#undef atomic_inc_uint -#undef atomic_inc_ulong -#undef atomic_inc_ptr -#undef atomic_inc_64 -#undef atomic_inc_32_nv -#undef atomic_inc_uint_nv -#undef atomic_inc_ulong_nv -#undef atomic_inc_ptr_nv -#undef atomic_inc_64_nv +#include +#include -#define CSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \ - void atomic_add_##name(volatile targ1 *, targ2); \ - void kcsan_atomic_add_##name(volatile targ1 *, targ2); \ - void kcsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - atomic_add_##name(ptr, val); \ - } \ - tret atomic_add_##name##_nv(volatile targ1 *, targ2); \ - tret kcsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \ - tret kcsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - return atomic_add_##name##_nv(ptr, val); \ +#define _CSAN_ATOMIC_FUNC_ADD(name, type) \ + void kcsan_atomic_add_##name(volatile type *ptr, type val) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + atomic_add_##name(ptr, val); \ } -#define CSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \ - void atomic_and_##name(volatile targ1 *, targ2); \ - void kcsan_atomic_and_##name(volatile targ1 *, targ2); \ - void kcsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - atomic_and_##name(ptr, val); \ - } \ - tret atomic_and_##name##_nv(volatile targ1 *, targ2); \ - tret kcsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \ - tret kcsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - return atomic_and_##name##_nv(ptr, val); \ +#define CSAN_ATOMIC_FUNC_ADD(name, type) \ + _CSAN_ATOMIC_FUNC_ADD(name, type) \ + _CSAN_ATOMIC_FUNC_ADD(acq_##name, type) \ + _CSAN_ATOMIC_FUNC_ADD(rel_##name, type) + +#define _CSAN_ATOMIC_FUNC_CLEAR(name, type) \ + void kcsan_atomic_clear_##name(volatile type *ptr, type val) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + atomic_clear_##name(ptr, val); \ } -#define CSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \ - void atomic_or_##name(volatile targ1 *, targ2); \ - void kcsan_atomic_or_##name(volatile targ1 *, targ2); \ - void kcsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - atomic_or_##name(ptr, val); \ - } \ - tret atomic_or_##name##_nv(volatile targ1 *, targ2); \ - tret kcsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \ - tret kcsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - return atomic_or_##name##_nv(ptr, val); \ +#define CSAN_ATOMIC_FUNC_CLEAR(name, type) \ + _CSAN_ATOMIC_FUNC_CLEAR(name, type) \ + _CSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ + _CSAN_ATOMIC_FUNC_CLEAR(rel_##name, type) + +#define _CSAN_ATOMIC_FUNC_CMPSET(name, type) \ + int kcsan_atomic_cmpset_##name(volatile type *ptr, type val1, \ + type val2) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + return (atomic_cmpset_##name(ptr, val1, val2)); \ } -#define CSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \ - tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ - tret kcsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ - tret kcsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - return atomic_cas_##name(ptr, exp, new); \ - } \ - tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ - tret kcsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ - tret kcsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - return atomic_cas_##name##_ni(ptr, exp, new); \ +#define CSAN_ATOMIC_FUNC_CMPSET(name, type) \ + _CSAN_ATOMIC_FUNC_CMPSET(name, type) \ + _CSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ + _CSAN_ATOMIC_FUNC_CMPSET(rel_##name, type) + +#define _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ + int kcsan_atomic_fcmpset_##name(volatile type *ptr, type *val1, \ + type val2) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + return (atomic_fcmpset_##name(ptr, val1, val2)); \ } -#define CSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \ - tret atomic_swap_##name(volatile targ1 *, targ2); \ - tret kcsan_atomic_swap_##name(volatile targ1 *, targ2); \ - tret kcsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - return atomic_swap_##name(ptr, val); \ +#define CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ + _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ + _CSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ + _CSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) + +#define CSAN_ATOMIC_FUNC_FETCHADD(name, type) \ + type kcsan_atomic_fetchadd_##name(volatile type *ptr, type val) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + return (atomic_fetchadd_##name(ptr, val)); \ } -#define CSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \ - void atomic_dec_##name(volatile targ1 *); \ - void kcsan_atomic_dec_##name(volatile targ1 *); \ - void kcsan_atomic_dec_##name(volatile targ1 *ptr) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - atomic_dec_##name(ptr); \ - } \ - tret atomic_dec_##name##_nv(volatile targ1 *); \ - tret kcsan_atomic_dec_##name##_nv(volatile targ1 *); \ - tret kcsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - return atomic_dec_##name##_nv(ptr); \ +#define _CSAN_ATOMIC_FUNC_LOAD(name, type) \ + type kcsan_atomic_load_##name(volatile type *ptr) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), false, true, \ + __RET_ADDR); \ + return (atomic_load_##name(ptr)); \ } -#define CSAN_ATOMIC_FUNC_INC(name, tret, targ1) \ - void atomic_inc_##name(volatile targ1 *); \ - void kcsan_atomic_inc_##name(volatile targ1 *); \ - void kcsan_atomic_inc_##name(volatile targ1 *ptr) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - atomic_inc_##name(ptr); \ - } \ - tret atomic_inc_##name##_nv(volatile targ1 *); \ - tret kcsan_atomic_inc_##name##_nv(volatile targ1 *); \ - tret kcsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \ - { \ - kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ - __RET_ADDR); \ - return atomic_inc_##name##_nv(ptr); \ +#define CSAN_ATOMIC_FUNC_LOAD(name, type) \ + _CSAN_ATOMIC_FUNC_LOAD(name, type) \ + _CSAN_ATOMIC_FUNC_LOAD(acq_##name, type) \ + +#define CSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ + type kcsan_atomic_readandclear_##name(volatile type *ptr) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + return (atomic_readandclear_##name(ptr)); \ } -CSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t); -CSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t); -CSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int); -CSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long); -CSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t); +#define _CSAN_ATOMIC_FUNC_SET(name, type) \ + void kcsan_atomic_set_##name(volatile type *ptr, type val) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + atomic_set_##name(ptr, val); \ + } -CSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t); -CSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t); -CSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int); -CSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long); +#define CSAN_ATOMIC_FUNC_SET(name, type) \ + _CSAN_ATOMIC_FUNC_SET(name, type) \ + _CSAN_ATOMIC_FUNC_SET(acq_##name, type) \ + _CSAN_ATOMIC_FUNC_SET(rel_##name, type) -CSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t); -CSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t); -CSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int); -CSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long); +#define _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ + void kcsan_atomic_subtract_##name(volatile type *ptr, type val) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + atomic_subtract_##name(ptr, val); \ + } -CSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t); -CSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t); -CSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int); -CSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long); -CSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *); -CSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t); -CSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t); -CSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int); -CSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long); -CSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *); +#define CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ + _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ + _CSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ + _CSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) -CSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t) -CSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t) -CSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int); -CSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long); -CSAN_ATOMIC_FUNC_DEC(ptr, void *, void); +#define _CSAN_ATOMIC_FUNC_STORE(name, type) \ + void kcsan_atomic_store_##name(volatile type *ptr, type val) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + atomic_store_##name(ptr, val); \ + } -CSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t) -CSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t) -CSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int); -CSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long); -CSAN_ATOMIC_FUNC_INC(ptr, void *, void); +#define CSAN_ATOMIC_FUNC_STORE(name, type) \ + _CSAN_ATOMIC_FUNC_STORE(name, type) \ + _CSAN_ATOMIC_FUNC_STORE(rel_##name, type) + +#define CSAN_ATOMIC_FUNC_SWAP(name, type) \ + type kcsan_atomic_swap_##name(volatile type *ptr, type val) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + return(atomic_swap_##name(ptr, val)); \ + } + +#define CSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ + int kcsan_atomic_testandclear_##name(volatile type *ptr, u_int val) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + return(atomic_testandclear_##name(ptr, val)); \ + } + +#define CSAN_ATOMIC_FUNC_TESTANDSET(name, type) \ + int kcsan_atomic_testandset_##name(volatile type *ptr, u_int val) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + return (atomic_testandset_##name(ptr, val)); \ + } + + +CSAN_ATOMIC_FUNC_ADD(8, uint8_t) +CSAN_ATOMIC_FUNC_CLEAR(8, uint8_t) +CSAN_ATOMIC_FUNC_CMPSET(8, uint8_t) +CSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t) +_CSAN_ATOMIC_FUNC_LOAD(8, uint8_t) +CSAN_ATOMIC_FUNC_SET(8, uint8_t) +CSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t) +_CSAN_ATOMIC_FUNC_STORE(8, uint8_t) +#if 0 +CSAN_ATOMIC_FUNC_FETCHADD(8, uint8_t) +CSAN_ATOMIC_FUNC_READANDCLEAR(8, uint8_t) +CSAN_ATOMIC_FUNC_SWAP(8, uint8_t) +CSAN_ATOMIC_FUNC_TESTANDCLEAR(8, uint8_t) +CSAN_ATOMIC_FUNC_TESTANDSET(8, uint8_t) +#endif + +CSAN_ATOMIC_FUNC_ADD(16, uint16_t) +CSAN_ATOMIC_FUNC_CLEAR(16, uint16_t) +CSAN_ATOMIC_FUNC_CMPSET(16, uint16_t) +CSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t) +#if defined(__aarch64__) +_CSAN_ATOMIC_FUNC_LOAD(16, uint16_t) +#else +CSAN_ATOMIC_FUNC_LOAD(16, uint16_t) +#endif +CSAN_ATOMIC_FUNC_SET(16, uint16_t) +CSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t) +_CSAN_ATOMIC_FUNC_STORE(16, uint16_t) +#if 0 +CSAN_ATOMIC_FUNC_FETCHADD(16, uint16_t) +CSAN_ATOMIC_FUNC_READANDCLEAR(16, uint16_t) +CSAN_ATOMIC_FUNC_SWAP(16, uint16_t) +CSAN_ATOMIC_FUNC_TESTANDCLEAR(16, uint16_t) +CSAN_ATOMIC_FUNC_TESTANDSET(16, uint16_t) +#endif + +CSAN_ATOMIC_FUNC_ADD(32, uint32_t) +CSAN_ATOMIC_FUNC_CLEAR(32, uint32_t) +CSAN_ATOMIC_FUNC_CMPSET(32, uint32_t) +CSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t) +CSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t) +CSAN_ATOMIC_FUNC_LOAD(32, uint32_t) +CSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t) +CSAN_ATOMIC_FUNC_SET(32, uint32_t) +CSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t) +CSAN_ATOMIC_FUNC_STORE(32, uint32_t) +CSAN_ATOMIC_FUNC_SWAP(32, uint32_t) +#if !defined(__aarch64__) +CSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t) +CSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t) +#endif + +CSAN_ATOMIC_FUNC_ADD(64, uint64_t) +CSAN_ATOMIC_FUNC_CLEAR(64, uint64_t) +CSAN_ATOMIC_FUNC_CMPSET(64, uint64_t) +CSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t) +CSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t) +CSAN_ATOMIC_FUNC_LOAD(64, uint64_t) +CSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t) +CSAN_ATOMIC_FUNC_SET(64, uint64_t) +CSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t) +CSAN_ATOMIC_FUNC_STORE(64, uint64_t) +CSAN_ATOMIC_FUNC_SWAP(64, uint64_t) +#if !defined(__aarch64__) +CSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t) +CSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t) +#endif + +CSAN_ATOMIC_FUNC_ADD(int, u_int) +CSAN_ATOMIC_FUNC_CLEAR(int, u_int) +CSAN_ATOMIC_FUNC_CMPSET(int, u_int) +CSAN_ATOMIC_FUNC_FCMPSET(int, u_int) +CSAN_ATOMIC_FUNC_FETCHADD(int, u_int) +CSAN_ATOMIC_FUNC_LOAD(int, u_int) +CSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int) +CSAN_ATOMIC_FUNC_SET(int, u_int) +CSAN_ATOMIC_FUNC_SUBTRACT(int, u_int) +CSAN_ATOMIC_FUNC_STORE(int, u_int) +CSAN_ATOMIC_FUNC_SWAP(int, u_int) +#if !defined(__aarch64__) +CSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int) +CSAN_ATOMIC_FUNC_TESTANDSET(int, u_int) +#endif + +CSAN_ATOMIC_FUNC_ADD(long, u_long) +CSAN_ATOMIC_FUNC_CLEAR(long, u_long) +CSAN_ATOMIC_FUNC_CMPSET(long, u_long) +CSAN_ATOMIC_FUNC_FCMPSET(long, u_long) +CSAN_ATOMIC_FUNC_FETCHADD(long, u_long) +CSAN_ATOMIC_FUNC_LOAD(long, u_long) +CSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long) +CSAN_ATOMIC_FUNC_SET(long, u_long) +CSAN_ATOMIC_FUNC_SUBTRACT(long, u_long) +CSAN_ATOMIC_FUNC_STORE(long, u_long) +CSAN_ATOMIC_FUNC_SWAP(long, u_long) +#if !defined(__aarch64__) +CSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long) +CSAN_ATOMIC_FUNC_TESTANDSET(long, u_long) +#endif + +CSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t) +CSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t) +CSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t) +CSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t) +#if !defined(__amd64__) +CSAN_ATOMIC_FUNC_FETCHADD(ptr, uintptr_t) +#endif +CSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t) +CSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t) +CSAN_ATOMIC_FUNC_SET(ptr, uintptr_t) +CSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t) +CSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t) +CSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t) +#if 0 +CSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t) +CSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t) +#endif + +#define CSAN_ATOMIC_FUNC_THREAD_FENCE(name) \ + void kcsan_atomic_thread_fence_##name(void) \ + { \ + atomic_thread_fence_##name(); \ + } + + +CSAN_ATOMIC_FUNC_THREAD_FENCE(acq) +CSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel) +CSAN_ATOMIC_FUNC_THREAD_FENCE(rel) +CSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst) /* -------------------------------------------------------------------------- */ #include +#include +#include -#undef bus_space_read_multi_1 -#undef bus_space_read_multi_2 -#undef bus_space_read_multi_4 -#undef bus_space_read_multi_8 -#undef bus_space_read_multi_stream_1 -#undef bus_space_read_multi_stream_2 -#undef bus_space_read_multi_stream_4 -#undef bus_space_read_multi_stream_8 -#undef bus_space_read_region_1 -#undef bus_space_read_region_2 -#undef bus_space_read_region_4 -#undef bus_space_read_region_8 -#undef bus_space_read_region_stream_1 -#undef bus_space_read_region_stream_2 -#undef bus_space_read_region_stream_4 -#undef bus_space_read_region_stream_8 -#undef bus_space_write_multi_1 -#undef bus_space_write_multi_2 -#undef bus_space_write_multi_4 -#undef bus_space_write_multi_8 -#undef bus_space_write_multi_stream_1 -#undef bus_space_write_multi_stream_2 -#undef bus_space_write_multi_stream_4 -#undef bus_space_write_multi_stream_8 -#undef bus_space_write_region_1 -#undef bus_space_write_region_2 -#undef bus_space_write_region_4 -#undef bus_space_write_region_8 -#undef bus_space_write_region_stream_1 -#undef bus_space_write_region_stream_2 -#undef bus_space_write_region_stream_4 -#undef bus_space_write_region_stream_8 +int +kcsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, + int flags, bus_space_handle_t *handlep) +{ -#define CSAN_BUS_READ_FUNC(bytes, bits) \ - void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ - bus_size_t, uint##bits##_t *, bus_size_t); \ - void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t, \ - bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ - void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \ - bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ - bus_size_t count) \ - { \ - kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ - false, false, __RET_ADDR); \ - bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \ - } \ - void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ - bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ - void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ - bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ - void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \ - bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ - bus_size_t count) \ - { \ - kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ - false, false, __RET_ADDR); \ - bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\ - } \ - void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \ - bus_size_t, uint##bits##_t *, bus_size_t); \ - void kcsan_bus_space_read_region_##bytes(bus_space_tag_t, \ - bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ - void kcsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \ - bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ - bus_size_t count) \ - { \ - kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ - false, false, __RET_ADDR); \ - bus_space_read_region_##bytes(tag, hnd, size, buf, count); \ - } \ - void bus_space_read_region_stream_##bytes(bus_space_tag_t, \ - bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ - void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \ - bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ - void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \ - bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ - bus_size_t count) \ - { \ - kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ - false, false, __RET_ADDR); \ - bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\ + return (bus_space_map(tag, hnd, size, flags, handlep)); +} + +void +kcsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, + bus_size_t size) +{ + + bus_space_unmap(tag, hnd, size); +} + +int +kcsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, + bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) +{ + + return (bus_space_subregion(tag, hnd, offset, size, handlep)); +} + +#if !defined(__amd64__) +int +kcsan_bus_space_alloc(bus_space_tag_t tag, bus_addr_t reg_start, + bus_addr_t reg_end, bus_size_t size, bus_size_t alignment, + bus_size_t boundary, int flags, bus_addr_t *addrp, + bus_space_handle_t *handlep) +{ + + return (bus_space_alloc(tag, reg_start, reg_end, size, alignment, + boundary, flags, addrp, handlep)); +} +#endif + +void +kcsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, + bus_size_t size) +{ + + bus_space_free(tag, hnd, size); +} + +void +kcsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, + bus_size_t offset, bus_size_t size, int flags) +{ + + bus_space_barrier(tag, hnd, offset, size, flags); +} + +#define CSAN_BUS_READ_FUNC(func, width, type) \ + type kcsan_bus_space_read##func##_##width(bus_space_tag_t tag, \ + bus_space_handle_t hnd, bus_size_t offset) \ + { \ + return (bus_space_read##func##_##width(tag, hnd, \ + offset)); \ + } \ + +#define CSAN_BUS_READ_PTR_FUNC(func, width, type) \ + void kcsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ + bus_space_handle_t hnd, bus_size_t size, type *buf, \ + bus_size_t count) \ + { \ + kcsan_access((uintptr_t)buf, sizeof(type) * count, \ + false, false, __RET_ADDR); \ + bus_space_read_##func##_##width(tag, hnd, size, buf, \ + count); \ } -#define CSAN_BUS_WRITE_FUNC(bytes, bits) \ - void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ - bus_size_t, const uint##bits##_t *, bus_size_t); \ - void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t, \ - bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ - void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \ - bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ - bus_size_t count) \ - { \ - kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ - true, false, __RET_ADDR); \ - bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \ - } \ - void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ - bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ - void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ - bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ - void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \ - bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ - bus_size_t count) \ - { \ - kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ - true, false, __RET_ADDR); \ - bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\ - } \ - void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\ - bus_size_t, const uint##bits##_t *, bus_size_t); \ - void kcsan_bus_space_write_region_##bytes(bus_space_tag_t, \ - bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ - void kcsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \ - bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ - bus_size_t count) \ - { \ - kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ - true, false, __RET_ADDR); \ - bus_space_write_region_##bytes(tag, hnd, size, buf, count); \ - } \ - void bus_space_write_region_stream_##bytes(bus_space_tag_t, \ - bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ - void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \ - bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ - void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \ - bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ - bus_size_t count) \ - { \ - kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ - true, false, __RET_ADDR); \ - bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\ +CSAN_BUS_READ_FUNC(, 1, uint8_t) +CSAN_BUS_READ_FUNC(_stream, 1, uint8_t) +CSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) +CSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) +CSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) +CSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) + +CSAN_BUS_READ_FUNC(, 2, uint16_t) +CSAN_BUS_READ_FUNC(_stream, 2, uint16_t) +CSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) +CSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) +CSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) +CSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) + +CSAN_BUS_READ_FUNC(, 4, uint32_t) +CSAN_BUS_READ_FUNC(_stream, 4, uint32_t) +CSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) +CSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) +CSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) +CSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) + +CSAN_BUS_READ_FUNC(, 8, uint64_t) +#if defined(__aarch64__) +CSAN_BUS_READ_FUNC(_stream, 8, uint64_t) +CSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t) +CSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t) +CSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t) +CSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t) +#endif + +#define CSAN_BUS_WRITE_FUNC(func, width, type) \ + void kcsan_bus_space_write##func##_##width(bus_space_tag_t tag, \ + bus_space_handle_t hnd, bus_size_t offset, type value) \ + { \ + bus_space_write##func##_##width(tag, hnd, offset, value); \ + } \ + +#define CSAN_BUS_WRITE_PTR_FUNC(func, width, type) \ + void kcsan_bus_space_write_##func##_##width(bus_space_tag_t tag, \ + bus_space_handle_t hnd, bus_size_t size, const type *buf, \ + bus_size_t count) \ + { \ + kcsan_access((uintptr_t)buf, sizeof(type) * count, \ + true, false, __RET_ADDR); \ + bus_space_write_##func##_##width(tag, hnd, size, buf, \ + count); \ } -CSAN_BUS_READ_FUNC(1, 8) -CSAN_BUS_READ_FUNC(2, 16) -CSAN_BUS_READ_FUNC(4, 32) -CSAN_BUS_READ_FUNC(8, 64) +CSAN_BUS_WRITE_FUNC(, 1, uint8_t) +CSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) +CSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) +CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) +CSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) +CSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) + +CSAN_BUS_WRITE_FUNC(, 2, uint16_t) +CSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) +CSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) +CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) +CSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) +CSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) + +CSAN_BUS_WRITE_FUNC(, 4, uint32_t) +CSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) +CSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) +CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) +CSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) +CSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) + +CSAN_BUS_WRITE_FUNC(, 8, uint64_t) +#if defined(__aarch64__) +CSAN_BUS_WRITE_FUNC(_stream, 8, uint64_t) +CSAN_BUS_WRITE_PTR_FUNC(multi, 8, uint64_t) +CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 8, uint64_t) +CSAN_BUS_WRITE_PTR_FUNC(region, 8, uint64_t) +CSAN_BUS_WRITE_PTR_FUNC(region_stream, 8, uint64_t) +#endif + +#define CSAN_BUS_SET_FUNC(func, width, type) \ + void kcsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ + bus_space_handle_t hnd, bus_size_t offset, type value, \ + bus_size_t count) \ + { \ + bus_space_set_##func##_##width(tag, hnd, offset, value, \ + count); \ + } + +CSAN_BUS_SET_FUNC(multi, 1, uint8_t) +CSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) +CSAN_BUS_SET_FUNC(region, 1, uint8_t) +CSAN_BUS_SET_FUNC(region_stream, 1, uint8_t) + +CSAN_BUS_SET_FUNC(multi, 2, uint16_t) +CSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) +CSAN_BUS_SET_FUNC(region, 2, uint16_t) +CSAN_BUS_SET_FUNC(region_stream, 2, uint16_t) + +CSAN_BUS_SET_FUNC(multi, 4, uint32_t) +CSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) +CSAN_BUS_SET_FUNC(region, 4, uint32_t) +CSAN_BUS_SET_FUNC(region_stream, 4, uint32_t) + +#if !defined(__amd64__) +CSAN_BUS_SET_FUNC(multi, 8, uint64_t) +CSAN_BUS_SET_FUNC(multi_stream, 8, uint64_t) +CSAN_BUS_SET_FUNC(region, 8, uint64_t) +CSAN_BUS_SET_FUNC(region_stream, 8, uint64_t) +#endif -CSAN_BUS_WRITE_FUNC(1, 8) -CSAN_BUS_WRITE_FUNC(2, 16) -CSAN_BUS_WRITE_FUNC(4, 32) -CSAN_BUS_WRITE_FUNC(8, 64) diff --git a/sys/kern/vfs_aio.c b/sys/kern/vfs_aio.c index 995cfef8344f..3c84d96274db 100644 --- a/sys/kern/vfs_aio.c +++ b/sys/kern/vfs_aio.c @@ -292,7 +292,7 @@ struct kaioinfo { * Different ABIs provide their own operations. */ struct aiocb_ops { - int (*copyin)(struct aiocb *ujob, struct aiocb *kjob); + int (*aio_copyin)(struct aiocb *ujob, struct aiocb *kjob); long (*fetch_status)(struct aiocb *ujob); long (*fetch_error)(struct aiocb *ujob); int (*store_status)(struct aiocb *ujob, long status); @@ -1420,7 +1420,7 @@ aiocb_store_aiocb(struct aiocb **ujobp, struct aiocb *ujob) } static struct aiocb_ops aiocb_ops = { - .copyin = aiocb_copyin, + .aio_copyin = aiocb_copyin, .fetch_status = aiocb_fetch_status, .fetch_error = aiocb_fetch_error, .store_status = aiocb_store_status, @@ -1431,7 +1431,7 @@ static struct aiocb_ops aiocb_ops = { #ifdef COMPAT_FREEBSD6 static struct aiocb_ops aiocb_ops_osigevent = { - .copyin = aiocb_copyin_old_sigevent, + .aio_copyin = aiocb_copyin_old_sigevent, .fetch_status = aiocb_fetch_status, .fetch_error = aiocb_fetch_error, .store_status = aiocb_store_status, @@ -1478,7 +1478,7 @@ aio_aqueue(struct thread *td, struct aiocb *ujob, struct aioliojob *lj, job = uma_zalloc(aiocb_zone, M_WAITOK | M_ZERO); knlist_init_mtx(&job->klist, AIO_MTX(ki)); - error = ops->copyin(ujob, &job->uaiocb); + error = ops->aio_copyin(ujob, &job->uaiocb); if (error) { ops->store_error(ujob, error); uma_zfree(aiocb_zone, job); @@ -2743,7 +2743,7 @@ aiocb32_store_aiocb(struct aiocb **ujobp, struct aiocb *ujob) } static struct aiocb_ops aiocb32_ops = { - .copyin = aiocb32_copyin, + .aio_copyin = aiocb32_copyin, .fetch_status = aiocb32_fetch_status, .fetch_error = aiocb32_fetch_error, .store_status = aiocb32_store_status, @@ -2754,7 +2754,7 @@ static struct aiocb_ops aiocb32_ops = { #ifdef COMPAT_FREEBSD6 static struct aiocb_ops aiocb32_ops_osigevent = { - .copyin = aiocb32_copyin_old_sigevent, + .aio_copyin = aiocb32_copyin_old_sigevent, .fetch_status = aiocb32_fetch_status, .fetch_error = aiocb32_fetch_error, .store_status = aiocb32_store_status, diff --git a/sys/libkern/strcmp.c b/sys/libkern/strcmp.c index fadbd30779ad..00220fa6d584 100644 --- a/sys/libkern/strcmp.c +++ b/sys/libkern/strcmp.c @@ -41,7 +41,7 @@ __FBSDID("$FreeBSD$"); * Compare strings. */ int -strcmp(const char *s1, const char *s2) +(strcmp)(const char *s1, const char *s2) { while (*s1 == *s2++) if (*s1++ == '\0') diff --git a/sys/libkern/strcpy.c b/sys/libkern/strcpy.c index 7c61542d9636..8528ed72ac3e 100644 --- a/sys/libkern/strcpy.c +++ b/sys/libkern/strcpy.c @@ -35,7 +35,7 @@ __FBSDID("$FreeBSD$"); #include char * -strcpy(char * __restrict to, const char * __restrict from) +(strcpy)(char * __restrict to, const char * __restrict from) { char *save = to; diff --git a/sys/libkern/strlen.c b/sys/libkern/strlen.c index 5f2f5867771e..a8c7964f69a3 100644 --- a/sys/libkern/strlen.c +++ b/sys/libkern/strlen.c @@ -80,7 +80,7 @@ static const unsigned long mask80 = 0x8080808080808080; } while (0) size_t -strlen(const char *str) +(strlen)(const char *str) { const char *p; const unsigned long *lp; diff --git a/sys/modules/Makefile b/sys/modules/Makefile index 80857a22613a..48f3ec60e79f 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -644,11 +644,11 @@ _ciss= ciss _chromebook_platform= chromebook_platform _cmx= cmx _coretemp= coretemp -.if ${MK_SOURCELESS_HOST} != "no" +.if ${MK_SOURCELESS_HOST} != "no" && empty(KCSAN_ENABLED) _hpt27xx= hpt27xx .endif _hptiop= hptiop -.if ${MK_SOURCELESS_HOST} != "no" +.if ${MK_SOURCELESS_HOST} != "no" && empty(KCSAN_ENABLED) _hptmv= hptmv _hptnr= hptnr _hptrr= hptrr diff --git a/sys/sys/_cscan_atomic.h b/sys/sys/_cscan_atomic.h new file mode 100644 index 000000000000..93c557bcee58 --- /dev/null +++ b/sys/sys/_cscan_atomic.h @@ -0,0 +1,307 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2019 Andrew Turner + * + * This software was developed by SRI International and the University of + * Cambridge Computer Laboratory (Department of Computer Science and + * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the + * DARPA SSITH research programme. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SYS__CSAN_ATOMIC_H_ +#define _SYS__CSAN_ATOMIC_H_ + +#ifndef _MACHINE_ATOMIC_H_ +#error do not include this header, use machine/atomic.h +#endif + +#define KCSAN_ATOMIC_FUNC_1(op, name, type) \ + void kcsan_atomic_##op##_##name(volatile type *, type); \ + void kcsan_atomic_##op##_acq_##name(volatile type *, type); \ + void kcsan_atomic_##op##_rel_##name(volatile type *, type) + +#define KCSAN_ATOMIC_CMPSET(name, type) \ + int kcsan_atomic_cmpset_##name(volatile type *, type, type); \ + int kcsan_atomic_cmpset_acq_##name(volatile type *, type, type); \ + int kcsan_atomic_cmpset_rel_##name(volatile type *, type, type) + +#define KCSAN_ATOMIC_FCMPSET(name, type) \ + int kcsan_atomic_fcmpset_##name(volatile type *, type *, type); \ + int kcsan_atomic_fcmpset_acq_##name(volatile type *, type *, type); \ + int kcsan_atomic_fcmpset_rel_##name(volatile type *, type *, type) + +#define KCSAN_ATOMIC_READ(op, name, type) \ + type kcsan_atomic_##op##_##name(volatile type *, type) + +#define KCSAN_ATOMIC_READANDCLEAR(name, type) \ + type kcsan_atomic_readandclear_##name(volatile type *) + +#define KCSAN_ATOMIC_LOAD(name, type) \ + type kcsan_atomic_load_##name(volatile type *); \ + type kcsan_atomic_load_acq_##name(volatile type *) + +#define KCSAN_ATOMIC_STORE(name, type) \ + void kcsan_atomic_store_##name(volatile type *, type); \ + void kcsan_atomic_store_rel_##name(volatile type *, type) + +#define KCSAN_ATOMIC_TEST(op, name, type) \ + int kcsan_atomic_##op##_##name(volatile type *, u_int) + +#define KCSAN_ATOMIC_FUNCS(name, type) \ + KCSAN_ATOMIC_FUNC_1(add, name, type); \ + KCSAN_ATOMIC_FUNC_1(clear, name, type); \ + KCSAN_ATOMIC_CMPSET(name, type); \ + KCSAN_ATOMIC_FCMPSET(name, type); \ + KCSAN_ATOMIC_READ(fetchadd, name, type); \ + KCSAN_ATOMIC_LOAD(name, type); \ + KCSAN_ATOMIC_READANDCLEAR(name, type); \ + KCSAN_ATOMIC_FUNC_1(set, name, type); \ + KCSAN_ATOMIC_FUNC_1(subtract, name, type); \ + KCSAN_ATOMIC_STORE(name, type); \ + KCSAN_ATOMIC_READ(swap, name, type); \ + KCSAN_ATOMIC_TEST(testandclear, name, type); \ + KCSAN_ATOMIC_TEST(testandset, name, type) + +KCSAN_ATOMIC_FUNCS(int, u_int); +KCSAN_ATOMIC_FUNCS(long, u_long); +KCSAN_ATOMIC_FUNCS(ptr, uintptr_t); +KCSAN_ATOMIC_FUNCS(8, uint8_t); +KCSAN_ATOMIC_FUNCS(16, uint16_t); +KCSAN_ATOMIC_FUNCS(32, uint32_t); +KCSAN_ATOMIC_FUNCS(64, uint64_t); + +void kcsan_atomic_thread_fence_acq(void); +void kcsan_atomic_thread_fence_acq_rel(void); +void kcsan_atomic_thread_fence_rel(void); +void kcsan_atomic_thread_fence_seq_cst(void); + +#ifndef KCSAN_RUNTIME + +#define atomic_add_int kcsan_atomic_add_int +#define atomic_add_acq_int kcsan_atomic_add_acq_int +#define atomic_add_rel_int kcsan_atomic_add_rel_int +#define atomic_clear_int kcsan_atomic_clear_int +#define atomic_clear_acq_int kcsan_atomic_clear_acq_int +#define atomic_clear_rel_int kcsan_atomic_clear_rel_int +#define atomic_cmpset_int kcsan_atomic_cmpset_int +#define atomic_cmpset_acq_int kcsan_atomic_cmpset_acq_int +#define atomic_cmpset_rel_int kcsan_atomic_cmpset_rel_int +#define atomic_fcmpset_int kcsan_atomic_fcmpset_int +#define atomic_fcmpset_acq_int kcsan_atomic_fcmpset_acq_int +#define atomic_fcmpset_rel_int kcsan_atomic_fcmpset_rel_int +#define atomic_fetchadd_int kcsan_atomic_fetchadd_int +#define atomic_load_int kcsan_atomic_load_int +#define atomic_load_acq_int kcsan_atomic_load_acq_int +#define atomic_readandclear_int kcsan_atomic_readandclear_int +#define atomic_set_int kcsan_atomic_set_int +#define atomic_set_acq_int kcsan_atomic_set_acq_int +#define atomic_set_rel_int kcsan_atomic_set_rel_int +#define atomic_subtract_int kcsan_atomic_subtract_int +#define atomic_subtract_acq_int kcsan_atomic_subtract_acq_int +#define atomic_subtract_rel_int kcsan_atomic_subtract_rel_int +#define atomic_store_int kcsan_atomic_store_int +#define atomic_store_rel_int kcsan_atomic_store_rel_int +#define atomic_swap_int kcsan_atomic_swap_int +#define atomic_testandclear_int kcsan_atomic_testandclear_int +#define atomic_testandset_int kcsan_atomic_testandset_int + +#define atomic_add_long kcsan_atomic_add_long +#define atomic_add_acq_long kcsan_atomic_add_acq_long +#define atomic_add_rel_long kcsan_atomic_add_rel_long +#define atomic_clear_long kcsan_atomic_clear_long +#define atomic_clear_acq_long kcsan_atomic_clear_acq_long +#define atomic_clear_rel_long kcsan_atomic_clear_rel_long +#define atomic_cmpset_long kcsan_atomic_cmpset_long +#define atomic_cmpset_acq_long kcsan_atomic_cmpset_acq_long +#define atomic_cmpset_rel_long kcsan_atomic_cmpset_rel_long +#define atomic_fcmpset_long kcsan_atomic_fcmpset_long +#define atomic_fcmpset_acq_long kcsan_atomic_fcmpset_acq_long +#define atomic_fcmpset_rel_long kcsan_atomic_fcmpset_rel_long +#define atomic_fetchadd_long kcsan_atomic_fetchadd_long +#define atomic_load_long kcsan_atomic_load_long +#define atomic_load_acq_long kcsan_atomic_load_acq_long +#define atomic_readandclear_long kcsan_atomic_readandclear_long +#define atomic_set_long kcsan_atomic_set_long +#define atomic_set_acq_long kcsan_atomic_set_acq_long +#define atomic_set_rel_long kcsan_atomic_set_rel_long +#define atomic_subtract_long kcsan_atomic_subtract_long +#define atomic_subtract_acq_long kcsan_atomic_subtract_acq_long +#define atomic_subtract_rel_long kcsan_atomic_subtract_rel_long +#define atomic_store_long kcsan_atomic_store_long +#define atomic_store_rel_long kcsan_atomic_store_rel_long +#define atomic_swap_long kcsan_atomic_swap_long +#define atomic_testandclear_long kcsan_atomic_testandclear_long +#define atomic_testandset_long kcsan_atomic_testandset_long + +#define atomic_add_ptr kcsan_atomic_add_ptr +#define atomic_add_acq_ptr kcsan_atomic_add_acq_ptr +#define atomic_add_rel_ptr kcsan_atomic_add_rel_ptr +#define atomic_clear_ptr kcsan_atomic_clear_ptr +#define atomic_clear_acq_ptr kcsan_atomic_clear_acq_ptr +#define atomic_clear_rel_ptr kcsan_atomic_clear_rel_ptr +#define atomic_cmpset_ptr kcsan_atomic_cmpset_ptr +#define atomic_cmpset_acq_ptr kcsan_atomic_cmpset_acq_ptr +#define atomic_cmpset_rel_ptr kcsan_atomic_cmpset_rel_ptr +#define atomic_fcmpset_ptr kcsan_atomic_fcmpset_ptr +#define atomic_fcmpset_acq_ptr kcsan_atomic_fcmpset_acq_ptr +#define atomic_fcmpset_rel_ptr kcsan_atomic_fcmpset_rel_ptr +#define atomic_fetchadd_ptr kcsan_atomic_fetchadd_ptr +#define atomic_load_ptr(x) kcsan_atomic_load_ptr((volatile uintptr_t *)(x)) +#define atomic_load_acq_ptr kcsan_atomic_load_acq_ptr +#define atomic_readandclear_ptr kcsan_atomic_readandclear_ptr +#define atomic_set_ptr kcsan_atomic_set_ptr +#define atomic_set_acq_ptr kcsan_atomic_set_acq_ptr +#define atomic_set_rel_ptr kcsan_atomic_set_rel_ptr +#define atomic_subtract_ptr kcsan_atomic_subtract_ptr +#define atomic_subtract_acq_ptr kcsan_atomic_subtract_acq_ptr +#define atomic_subtract_rel_ptr kcsan_atomic_subtract_rel_ptr +#define atomic_store_ptr kcsan_atomic_store_ptr +#define atomic_store_rel_ptr kcsan_atomic_store_rel_ptr +#define atomic_swap_ptr kcsan_atomic_swap_ptr +#define atomic_testandclear_ptr kcsan_atomic_testandclear_ptr +#define atomic_testandset_ptr kcsan_atomic_testandset_ptr + +#define atomic_add_8 kcsan_atomic_add_8 +#define atomic_add_acq_8 kcsan_atomic_add_acq_8 +#define atomic_add_rel_8 kcsan_atomic_add_rel_8 +#define atomic_clear_8 kcsan_atomic_clear_8 +#define atomic_clear_acq_8 kcsan_atomic_clear_acq_8 +#define atomic_clear_rel_8 kcsan_atomic_clear_rel_8 +#define atomic_cmpset_8 kcsan_atomic_cmpset_8 +#define atomic_cmpset_acq_8 kcsan_atomic_cmpset_acq_8 +#define atomic_cmpset_rel_8 kcsan_atomic_cmpset_rel_8 +#define atomic_fcmpset_8 kcsan_atomic_fcmpset_8 +#define atomic_fcmpset_acq_8 kcsan_atomic_fcmpset_acq_8 +#define atomic_fcmpset_rel_8 kcsan_atomic_fcmpset_rel_8 +#define atomic_fetchadd_8 kcsan_atomic_fetchadd_8 +#define atomic_load_8 kcsan_atomic_load_8 +#define atomic_load_acq_8 kcsan_atomic_load_acq_8 +#define atomic_readandclear_8 kcsan_atomic_readandclear_8 +#define atomic_set_8 kcsan_atomic_set_8 +#define atomic_set_acq_8 kcsan_atomic_set_acq_8 +#define atomic_set_rel_8 kcsan_atomic_set_rel_8 +#define atomic_subtract_8 kcsan_atomic_subtract_8 +#define atomic_subtract_acq_8 kcsan_atomic_subtract_acq_8 +#define atomic_subtract_rel_8 kcsan_atomic_subtract_rel_8 +#define atomic_store_8 kcsan_atomic_store_8 +#define atomic_store_rel_8 kcsan_atomic_store_rel_8 +#define atomic_swap_8 kcsan_atomic_swap_8 +#define atomic_testandclear_8 kcsan_atomic_testandclear_8 +#define atomic_testandset_8 kcsan_atomic_testandset_8 + +#define atomic_add_16 kcsan_atomic_add_16 +#define atomic_add_acq_16 kcsan_atomic_add_acq_16 +#define atomic_add_rel_16 kcsan_atomic_add_rel_16 +#define atomic_clear_16 kcsan_atomic_clear_16 +#define atomic_clear_acq_16 kcsan_atomic_clear_acq_16 +#define atomic_clear_rel_16 kcsan_atomic_clear_rel_16 +#define atomic_cmpset_16 kcsan_atomic_cmpset_16 +#define atomic_cmpset_acq_16 kcsan_atomic_cmpset_acq_16 +#define atomic_cmpset_rel_16 kcsan_atomic_cmpset_rel_16 +#define atomic_fcmpset_16 kcsan_atomic_fcmpset_16 +#define atomic_fcmpset_acq_16 kcsan_atomic_fcmpset_acq_16 +#define atomic_fcmpset_rel_16 kcsan_atomic_fcmpset_rel_16 +#define atomic_fetchadd_16 kcsan_atomic_fetchadd_16 +#define atomic_load_16 kcsan_atomic_load_16 +#define atomic_load_acq_16 kcsan_atomic_load_acq_16 +#define atomic_readandclear_16 kcsan_atomic_readandclear_16 +#define atomic_set_16 kcsan_atomic_set_16 +#define atomic_set_acq_16 kcsan_atomic_set_acq_16 +#define atomic_set_rel_16 kcsan_atomic_set_rel_16 +#define atomic_subtract_16 kcsan_atomic_subtract_16 +#define atomic_subtract_acq_16 kcsan_atomic_subtract_acq_16 +#define atomic_subtract_rel_16 kcsan_atomic_subtract_rel_16 +#define atomic_store_16 kcsan_atomic_store_16 +#define atomic_store_rel_16 kcsan_atomic_store_rel_16 +#define atomic_swap_16 kcsan_atomic_swap_16 +#define atomic_testandclear_16 kcsan_atomic_testandclear_16 +#define atomic_testandset_16 kcsan_atomic_testandset_16 + +#define atomic_add_32 kcsan_atomic_add_32 +#define atomic_add_acq_32 kcsan_atomic_add_acq_32 +#define atomic_add_rel_32 kcsan_atomic_add_rel_32 +#define atomic_clear_32 kcsan_atomic_clear_32 +#define atomic_clear_acq_32 kcsan_atomic_clear_acq_32 +#define atomic_clear_rel_32 kcsan_atomic_clear_rel_32 +#define atomic_cmpset_32 kcsan_atomic_cmpset_32 +#define atomic_cmpset_acq_32 kcsan_atomic_cmpset_acq_32 +#define atomic_cmpset_rel_32 kcsan_atomic_cmpset_rel_32 +#define atomic_fcmpset_32 kcsan_atomic_fcmpset_32 +#define atomic_fcmpset_acq_32 kcsan_atomic_fcmpset_acq_32 +#define atomic_fcmpset_rel_32 kcsan_atomic_fcmpset_rel_32 +#define atomic_fetchadd_32 kcsan_atomic_fetchadd_32 +#define atomic_load_32 kcsan_atomic_load_32 +#define atomic_load_acq_32 kcsan_atomic_load_acq_32 +#define atomic_readandclear_32 kcsan_atomic_readandclear_32 +#define atomic_set_32 kcsan_atomic_set_32 +#define atomic_set_acq_32 kcsan_atomic_set_acq_32 +#define atomic_set_rel_32 kcsan_atomic_set_rel_32 +#define atomic_subtract_32 kcsan_atomic_subtract_32 +#define atomic_subtract_acq_32 kcsan_atomic_subtract_acq_32 +#define atomic_subtract_rel_32 kcsan_atomic_subtract_rel_32 +#define atomic_store_32 kcsan_atomic_store_32 +#define atomic_store_rel_32 kcsan_atomic_store_rel_32 +#define atomic_swap_32 kcsan_atomic_swap_32 +#define atomic_testandclear_32 kcsan_atomic_testandclear_32 +#define atomic_testandset_32 kcsan_atomic_testandset_32 + +#define atomic_add_64 kcsan_atomic_add_64 +#define atomic_add_acq_64 kcsan_atomic_add_acq_64 +#define atomic_add_rel_64 kcsan_atomic_add_rel_64 +#define atomic_clear_64 kcsan_atomic_clear_64 +#define atomic_clear_acq_64 kcsan_atomic_clear_acq_64 +#define atomic_clear_rel_64 kcsan_atomic_clear_rel_64 +#define atomic_cmpset_64 kcsan_atomic_cmpset_64 +#define atomic_cmpset_acq_64 kcsan_atomic_cmpset_acq_64 +#define atomic_cmpset_rel_64 kcsan_atomic_cmpset_rel_64 +#define atomic_fcmpset_64 kcsan_atomic_fcmpset_64 +#define atomic_fcmpset_acq_64 kcsan_atomic_fcmpset_acq_64 +#define atomic_fcmpset_rel_64 kcsan_atomic_fcmpset_rel_64 +#define atomic_fetchadd_64 kcsan_atomic_fetchadd_64 +#define atomic_load_64 kcsan_atomic_load_64 +#define atomic_load_acq_64 kcsan_atomic_load_acq_64 +#define atomic_readandclear_64 kcsan_atomic_readandclear_64 +#define atomic_set_64 kcsan_atomic_set_64 +#define atomic_set_acq_64 kcsan_atomic_set_acq_64 +#define atomic_set_rel_64 kcsan_atomic_set_rel_64 +#define atomic_subtract_64 kcsan_atomic_subtract_64 +#define atomic_subtract_acq_64 kcsan_atomic_subtract_acq_64 +#define atomic_subtract_rel_64 kcsan_atomic_subtract_rel_64 +#define atomic_store_64 kcsan_atomic_store_64 +#define atomic_store_rel_64 kcsan_atomic_store_rel_64 +#define atomic_swap_64 kcsan_atomic_swap_64 +#define atomic_testandclear_64 kcsan_atomic_testandclear_64 +#define atomic_testandset_64 kcsan_atomic_testandset_64 + +#define atomic_thread_fence_acq kcsan_atomic_thread_fence_acq +#define atomic_thread_fence_acq_rel kcsan_atomic_thread_fence_acq_rel +#define atomic_thread_fence_rel kcsan_atomic_thread_fence_rel +#define atomic_thread_fence_seq_cst kcsan_atomic_thread_fence_seq_cst + +#endif /* !KCSAN_RUNTIME */ + +#endif /* !_SYS__CSAN_ATOMIC_H_ */ diff --git a/sys/sys/_cscan_bus.h b/sys/sys/_cscan_bus.h new file mode 100644 index 000000000000..edb5ae93d4f9 --- /dev/null +++ b/sys/sys/_cscan_bus.h @@ -0,0 +1,190 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2019 Andrew Turner + * + * This software was developed by SRI International and the University of + * Cambridge Computer Laboratory (Department of Computer Science and + * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the + * DARPA SSITH research programme. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SYS__CSAN_BUS_H_ +#define _SYS__CSAN_BUS_H_ + +#define KCSAN_BS_MULTI(rw, width, type) \ + void kcsan_bus_space_##rw##_multi_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, type *, bus_size_t); \ + void kcsan_bus_space_##rw##_multi_stream_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, type *, bus_size_t); \ + void kcsan_bus_space_##rw##_region_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, type *, bus_size_t); \ + void kcsan_bus_space_##rw##_region_stream_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, type *, bus_size_t) + +#define KCSAN_BS_READ(width, type) \ + type kcsan_bus_space_read_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t); \ + type kcsan_bus_space_read_stream_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t); \ + KCSAN_BS_MULTI(read, width, type) + +#define KCSAN_BS_WRITE(width, type) \ + void kcsan_bus_space_write_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, type); \ + void kcsan_bus_space_write_stream_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, type); \ + KCSAN_BS_MULTI(write, width, const type) + +#define KCSAN_BS_SET(width, type) \ + void kcsan_bus_space_set_multi_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, type, bus_size_t); \ + void kcsan_bus_space_set_multi_stream_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, type, bus_size_t); \ + void kcsan_bus_space_set_region_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, type, bus_size_t); \ + void kcsan_bus_space_set_region_stream_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, type, bus_size_t) + +#define KCSAN_BS_COPY(width, type) \ + void kcsan_bus_space_copy_region_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, bus_space_handle_t, \ + bus_size_t, bus_size_t); \ + void kcsan_bus_space_copy_region_stream_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, bus_space_handle_t, \ + bus_size_t, bus_size_t); + +#define KCSAN_BS(width, type) \ + KCSAN_BS_READ(width, type); \ + KCSAN_BS_WRITE(width, type); \ + KCSAN_BS_SET(width, type); \ + KCSAN_BS_COPY(width, type) + +KCSAN_BS(1, uint8_t); +KCSAN_BS(2, uint16_t); +KCSAN_BS(4, uint32_t); +KCSAN_BS(8, uint64_t); + +int kcsan_bus_space_map(bus_space_tag_t, bus_addr_t, bus_size_t, int, + bus_space_handle_t *); +void kcsan_bus_space_unmap(bus_space_tag_t, bus_space_handle_t, bus_size_t); +int kcsan_bus_space_subregion(bus_space_tag_t, bus_space_handle_t, bus_size_t, + bus_size_t, bus_space_handle_t *); +int kcsan_bus_space_alloc(bus_space_tag_t, bus_addr_t, bus_addr_t, + bus_size_t, bus_size_t, bus_size_t, int, bus_addr_t *, + bus_space_handle_t *); +void kcsan_bus_space_free(bus_space_tag_t, bus_space_handle_t, bus_size_t); +void kcsan_bus_space_barrier(bus_space_tag_t, bus_space_handle_t, bus_size_t, + bus_size_t, int); + +#ifndef KCSAN_RUNTIME + +#define bus_space_map kcsan_bus_space_map +#define bus_space_unmap kcsan_bus_space_unmap +#define bus_space_subregion kcsan_bus_space_subregion +#define bus_space_alloc kcsan_bus_space_alloc +#define bus_space_free kcsan_bus_space_free +#define bus_space_barrier kcsan_bus_space_barrier + +#define bus_space_read_1 kcsan_bus_space_read_1 +#define bus_space_read_stream_1 kcsan_bus_space_read_stream_1 +#define bus_space_read_multi_1 kcsan_bus_space_read_multi_1 +#define bus_space_read_multi_stream_1 kcsan_bus_space_read_multi_stream_1 +#define bus_space_read_region_1 kcsan_bus_space_read_region_1 +#define bus_space_read_region_stream_1 kcsan_bus_space_read_region_stream_1 +#define bus_space_write_1 kcsan_bus_space_write_1 +#define bus_space_write_stream_1 kcsan_bus_space_write_stream_1 +#define bus_space_write_multi_1 kcsan_bus_space_write_multi_1 +#define bus_space_write_multi_stream_1 kcsan_bus_space_write_multi_stream_1 +#define bus_space_write_region_1 kcsan_bus_space_write_region_1 +#define bus_space_write_region_stream_1 kcsan_bus_space_write_region_stream_1 +#define bus_space_set_multi_1 kcsan_bus_space_set_multi_1 +#define bus_space_set_multi_stream_1 kcsan_bus_space_set_multi_stream_1 +#define bus_space_set_region_1 kcsan_bus_space_set_region_1 +#define bus_space_set_region_stream_1 kcsan_bus_space_set_region_stream_1 +#define bus_space_copy_multi_1 kcsan_bus_space_copy_multi_1 +#define bus_space_copy_multi_stream_1 kcsan_bus_space_copy_multi_stream_1 + +#define bus_space_read_2 kcsan_bus_space_read_2 +#define bus_space_read_stream_2 kcsan_bus_space_read_stream_2 +#define bus_space_read_multi_2 kcsan_bus_space_read_multi_2 +#define bus_space_read_multi_stream_2 kcsan_bus_space_read_multi_stream_2 +#define bus_space_read_region_2 kcsan_bus_space_read_region_2 +#define bus_space_read_region_stream_2 kcsan_bus_space_read_region_stream_2 +#define bus_space_write_2 kcsan_bus_space_write_2 +#define bus_space_write_stream_2 kcsan_bus_space_write_stream_2 +#define bus_space_write_multi_2 kcsan_bus_space_write_multi_2 +#define bus_space_write_multi_stream_2 kcsan_bus_space_write_multi_stream_2 +#define bus_space_write_region_2 kcsan_bus_space_write_region_2 +#define bus_space_write_region_stream_2 kcsan_bus_space_write_region_stream_2 +#define bus_space_set_multi_2 kcsan_bus_space_set_multi_2 +#define bus_space_set_multi_stream_2 kcsan_bus_space_set_multi_stream_2 +#define bus_space_set_region_2 kcsan_bus_space_set_region_2 +#define bus_space_set_region_stream_2 kcsan_bus_space_set_region_stream_2 +#define bus_space_copy_multi_2 kcsan_bus_space_copy_multi_2 +#define bus_space_copy_multi_stream_2 kcsan_bus_space_copy_multi_stream_2 + +#define bus_space_read_4 kcsan_bus_space_read_4 +#define bus_space_read_stream_4 kcsan_bus_space_read_stream_4 +#define bus_space_read_multi_4 kcsan_bus_space_read_multi_4 +#define bus_space_read_multi_stream_4 kcsan_bus_space_read_multi_stream_4 +#define bus_space_read_region_4 kcsan_bus_space_read_region_4 +#define bus_space_read_region_stream_4 kcsan_bus_space_read_region_stream_4 +#define bus_space_write_4 kcsan_bus_space_write_4 +#define bus_space_write_stream_4 kcsan_bus_space_write_stream_4 +#define bus_space_write_multi_4 kcsan_bus_space_write_multi_4 +#define bus_space_write_multi_stream_4 kcsan_bus_space_write_multi_stream_4 +#define bus_space_write_region_4 kcsan_bus_space_write_region_4 +#define bus_space_write_region_stream_4 kcsan_bus_space_write_region_stream_4 +#define bus_space_set_multi_4 kcsan_bus_space_set_multi_4 +#define bus_space_set_multi_stream_4 kcsan_bus_space_set_multi_stream_4 +#define bus_space_set_region_4 kcsan_bus_space_set_region_4 +#define bus_space_set_region_stream_4 kcsan_bus_space_set_region_stream_4 +#define bus_space_copy_multi_4 kcsan_bus_space_copy_multi_4 +#define bus_space_copy_multi_stream_4 kcsan_bus_space_copy_multi_stream_4 + +#define bus_space_read_8 kcsan_bus_space_read_8 +#define bus_space_read_stream_8 kcsan_bus_space_read_stream_8 +#define bus_space_read_multi_8 kcsan_bus_space_read_multi_8 +#define bus_space_read_multi_stream_8 kcsan_bus_space_read_multi_stream_8 +#define bus_space_read_region_8 kcsan_bus_space_read_region_8 +#define bus_space_read_region_stream_8 kcsan_bus_space_read_region_stream_8 +#define bus_space_write_8 kcsan_bus_space_write_8 +#define bus_space_write_stream_8 kcsan_bus_space_write_stream_8 +#define bus_space_write_multi_8 kcsan_bus_space_write_multi_8 +#define bus_space_write_multi_stream_8 kcsan_bus_space_write_multi_stream_8 +#define bus_space_write_region_8 kcsan_bus_space_write_region_8 +#define bus_space_write_region_stream_8 kcsan_bus_space_write_region_stream_8 +#define bus_space_set_multi_8 kcsan_bus_space_set_multi_8 +#define bus_space_set_multi_stream_8 kcsan_bus_space_set_multi_stream_8 +#define bus_space_set_region_8 kcsan_bus_space_set_region_8 +#define bus_space_set_region_stream_8 kcsan_bus_space_set_region_stream_8 +#define bus_space_copy_multi_8 kcsan_bus_space_copy_multi_8 +#define bus_space_copy_multi_stream_8 kcsan_bus_space_copy_multi_stream_8 + +#endif /* !KCSAN_RUNTIME */ + +#endif /* !_SYS__CSAN_BUS_H_ */ diff --git a/sys/sys/csan.h b/sys/sys/csan.h index d38b656b6a20..3b5f9ee74304 100644 --- a/sys/sys/csan.h +++ b/sys/sys/csan.h @@ -27,23 +27,19 @@ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ */ #ifndef _SYS_CSAN_H_ #define _SYS_CSAN_H_ -#ifdef _KERNEL_OPT -#include "opt_kcsan.h" -#endif - #include #ifdef KCSAN -void kcsan_init(void); -void kcsan_cpu_init(struct cpu_info *); +void kcsan_cpu_init(u_int); #else -#define kcsan_init() __nothing -#define kcsan_cpu_init(ci) __nothing +#define kcsan_cpu_init(ci) ((void)0) #endif #endif /* !_SYS_CSAN_H_ */ diff --git a/sys/sys/libkern.h b/sys/sys/libkern.h index ceb7ee2ab5b0..ba182739cfe7 100644 --- a/sys/sys/libkern.h +++ b/sys/sys/libkern.h @@ -191,6 +191,15 @@ size_t strspn(const char *, const char *); char *strstr(const char *, const char *); int strvalid(const char *, size_t); +#ifdef KCSAN +char *kcsan_strcpy(char *, const char *); +int kcsan_strcmp(const char *, const char *); +size_t kcsan_strlen(const char *); +#define strcpy(d, s) kcsan_strcpy((d), (s)) +#define strcmp(s1, s2) kcsan_strcmp((s1), (s2)) +#define strlen(s) kcsan_strlen((s)) +#endif + static __inline char * index(const char *p, int ch) { diff --git a/sys/sys/systm.h b/sys/sys/systm.h index 733348c4fae9..6619cf974dad 100644 --- a/sys/sys/systm.h +++ b/sys/sys/systm.h @@ -315,21 +315,36 @@ void hexdump(const void *ptr, int length, const char *hdr, int flags); #define ovbcopy(f, t, l) bcopy((f), (t), (l)) void bcopy(const void * _Nonnull from, void * _Nonnull to, size_t len); -#define bcopy(from, to, len) __builtin_memmove((to), (from), (len)) void bzero(void * _Nonnull buf, size_t len); -#define bzero(buf, len) __builtin_memset((buf), 0, (len)) void explicit_bzero(void * _Nonnull, size_t); int bcmp(const void *b1, const void *b2, size_t len); -#define bcmp(b1, b2, len) __builtin_memcmp((b1), (b2), (len)) void *memset(void * _Nonnull buf, int c, size_t len); -#define memset(buf, c, len) __builtin_memset((buf), (c), (len)) void *memcpy(void * _Nonnull to, const void * _Nonnull from, size_t len); -#define memcpy(to, from, len) __builtin_memcpy((to), (from), (len)) void *memmove(void * _Nonnull dest, const void * _Nonnull src, size_t n); -#define memmove(dest, src, n) __builtin_memmove((dest), (src), (n)) int memcmp(const void *b1, const void *b2, size_t len); + +#ifdef KCSAN +void *kcsan_memset(void *, int, size_t); +void *kcsan_memcpy(void *, const void *, size_t); +void *kcsan_memmove(void *, const void *, size_t); +int kcsan_memcmp(const void *, const void *, size_t); +#define bcopy(from, to, len) kcsan_memmove((to), (from), (len)) +#define bzero(buf, len) kcsan_memset((buf), 0, (len)) +#define bcmp(b1, b2, len) kcsan_memcmp((b1), (b2), (len)) +#define memset(buf, c, len) kcsan_memset((buf), (c), (len)) +#define memcpy(to, from, len) kcsan_memcpy((to), (from), (len)) +#define memmove(dest, src, n) kcsan_memmove((dest), (src), (n)) +#define memcmp(b1, b2, len) kcsan_memcmp((b1), (b2), (len)) +#else +#define bcopy(from, to, len) __builtin_memmove((to), (from), (len)) +#define bzero(buf, len) __builtin_memset((buf), 0, (len)) +#define bcmp(b1, b2, len) __builtin_memcmp((b1), (b2), (len)) +#define memset(buf, c, len) __builtin_memset((buf), (c), (len)) +#define memcpy(to, from, len) __builtin_memcpy((to), (from), (len)) +#define memmove(dest, src, n) __builtin_memmove((dest), (src), (n)) #define memcmp(b1, b2, len) __builtin_memcmp((b1), (b2), (len)) +#endif void *memset_early(void * _Nonnull buf, int c, size_t len); #define bzero_early(buf, len) memset_early((buf), 0, (len)) @@ -352,6 +367,17 @@ int copyout(const void * _Nonnull __restrict kaddr, int copyout_nofault(const void * _Nonnull __restrict kaddr, void * __restrict udaddr, size_t len); +#ifdef KCSAN +int kcsan_copystr(const void *, void *, size_t, size_t *); +int kcsan_copyin(const void *, void *, size_t); +int kcsan_copyinstr(const void *, void *, size_t, size_t *); +int kcsan_copyout(const void *, void *, size_t); +#define copystr(kf, k, l, lc) kcsan_copystr((kf), (k), (l), (lc)) +#define copyin(u, k, l) kcsan_copyin((u), (k), (l)) +#define copyinstr(u, k, l, lc) kcsan_copyinstr((u), (k), (l), (lc)) +#define copyout(k, u, l) kcsan_copyout((k), (u), (l)) +#endif + int fubyte(volatile const void *base); long fuword(volatile const void *base); int fuword16(volatile const void *base); diff --git a/sys/x86/include/bus.h b/sys/x86/include/bus.h index d8e0d6ac20db..b1675ee26cf4 100644 --- a/sys/x86/include/bus.h +++ b/sys/x86/include/bus.h @@ -101,6 +101,7 @@ #include #include +#include #ifndef __GNUCLIKE_ASM #error "no assembler code for your compiler" @@ -131,6 +132,13 @@ #define BUS_SPACE_INVALID_DATA (~0) #define BUS_SPACE_UNRESTRICTED (~0) +#define BUS_SPACE_BARRIER_READ 0x01 /* force read barrier */ +#define BUS_SPACE_BARRIER_WRITE 0x02 /* force write barrier */ + +#if defined(KCSAN) && !defined(KCSAN_RUNTIME) +#include +#else + /* * Map a region of device bus space into CPU virtual address space. */ @@ -992,9 +1000,6 @@ bus_space_copy_region_4(bus_space_tag_t tag, bus_space_handle_t bsh1, * prevent reordering by the compiler; all Intel x86 processors currently * retire operations outside the CPU in program order. */ -#define BUS_SPACE_BARRIER_READ 0x01 /* force read barrier */ -#define BUS_SPACE_BARRIER_WRITE 0x02 /* force write barrier */ - static __inline void bus_space_barrier(bus_space_tag_t tag __unused, bus_space_handle_t bsh __unused, bus_size_t offset __unused, bus_size_t len __unused, int flags) @@ -1022,8 +1027,6 @@ bus_space_barrier(bus_space_tag_t tag __unused, bus_space_handle_t bsh __unused, #define outl(a, b) compiler_error #endif -#include - /* * Stream accesses are the same as normal accesses on x86; there are no * supported bus systems with an endianess different from the host one. @@ -1088,4 +1091,6 @@ bus_space_barrier(bus_space_tag_t tag __unused, bus_space_handle_t bsh __unused, #define bus_space_copy_region_stream_4(t, h1, o1, h2, o2, c) \ bus_space_copy_region_4((t), (h1), (o1), (h2), (o2), (c)) +#endif /* KCSAN && !KCSAN_RUNTIME */ + #endif /* _X86_BUS_H_ */ diff --git a/sys/x86/x86/bus_machdep.c b/sys/x86/x86/bus_machdep.c index a629a811563d..67edbb976415 100644 --- a/sys/x86/x86/bus_machdep.c +++ b/sys/x86/x86/bus_machdep.c @@ -24,6 +24,8 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#define KCSAN_RUNTIME + #include __FBSDID("$FreeBSD$"); diff --git a/sys/x86/x86/mp_x86.c b/sys/x86/x86/mp_x86.c index 064234b3ca9e..3fa7f2c8e3ce 100644 --- a/sys/x86/x86/mp_x86.c +++ b/sys/x86/x86/mp_x86.c @@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$"); #include #include /* cngetc() */ #include +#include #ifdef GPROF #include #endif @@ -1080,6 +1081,8 @@ init_secondary_tail(void) cpu_initclocks_ap(); #endif + kcsan_cpu_init(cpuid); + sched_throw(NULL); panic("scheduler returned us to %s", __func__);