d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Generalize bus_space(9) and atomic(9) sanitizer interceptors

Browse Source 
Make it easy to define interceptors for new sanitizer runtimes, rather
than assuming KCSAN.  Lay a bit of groundwork for KASAN and KMSAN.

When a sanitizer is compiled in, atomic(9) and bus_space(9) definitions
in atomic_san.h are used by default instead of the inline
implementations in the platform's atomic.h.  These definitions are
implemented in the sanitizer runtime, which includes
machine/{atomic,bus}.h with SAN_RUNTIME defined to pull in the actual
implementations.

No functional change intended.

Sponsored by:	The FreeBSD Foundation

(cherry picked from commit 3ead60236f)
This commit is contained in:
Mark Johnston 2021-03-22 21:44:55 -04:00
parent 252b6ae3e6
commit bf0986b742
10 changed files with 533 additions and 451 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
sys/amd64/include/atomic.h
View File

@ -68,7 +68,15 @@
#define OFFSETOF_MONITORBUF 0x100
#endif
#if defined(KCSAN) && !defined(KCSAN_RUNTIME)
#ifndef SAN_RUNTIME
#if defined(KASAN)
#define ATOMIC_SAN_PREFIX kasan
#elif defined(KCSAN)
#define ATOMIC_SAN_PREFIX kcsan
#endif
#endif
#ifdef ATOMIC_SAN_PREFIX
#include <sys/atomic_san.h>
#else
#include <sys/atomic_common.h>
@ -679,6 +687,6 @@ u_long atomic_swap_long(volatile u_long *p, u_long v);
#endif /* !WANT_FUNCTIONS */
#endif /* KCSAN && !KCSAN_RUNTIME */
#endif /* !ATOMIC_SAN_PREFIX */
#endif /* !_MACHINE_ATOMIC_H_ */

4
sys/arm64/arm64/bus_machdep.c
View File

@ -25,7 +25,9 @@
*
*/
#define KCSAN_RUNTIME
#if defined(KASAN) || defined(KCSAN)
#define SAN_RUNTIME
#endif
#include "opt_platform.h"

10
sys/arm64/include/bus.h
View File

@ -91,7 +91,15 @@
#define BUS_SPACE_BARRIER_READ 0x01
#define BUS_SPACE_BARRIER_WRITE 0x02
#if defined(KCSAN) && !defined(KCSAN_RUNTIME)
#ifndef SAN_RUNTIME
#if defined(KASAN)
#define BUS_SAN_PREFIX kasan
#elif defined(KCSAN)
#define BUS_SAN_PREFIX kcsan
#endif
#endif
#ifdef BUS_SAN_PREFIX
#include <sys/bus_san.h>
#else

4
sys/kern/kern_kcov.c
View File

@ -35,7 +35,9 @@
* $FreeBSD$
*/
#define KCSAN_RUNTIME
#ifdef KCSAN
#define SAN_RUNTIME
#endif
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");

4
sys/kern/subr_coverage.c
View File

@ -35,7 +35,9 @@
* $FreeBSD$
*/
#define KCSAN_RUNTIME
#ifdef KCSAN
#define SAN_RUNTIME
#endif
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");

10
sys/kern/subr_csan.c
View File

@ -30,7 +30,7 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
#define KCSAN_RUNTIME
#define SAN_RUNTIME
#include "opt_ddb.h"
@ -380,6 +380,7 @@ kcsan_copyout(const void *kaddr, void *uaddr, size_t len)
/* -------------------------------------------------------------------------- */
#include <machine/atomic.h>
#define ATOMIC_SAN_PREFIX kcsan
#include <sys/atomic_san.h>
#define _CSAN_ATOMIC_FUNC_ADD(name, type) \
@ -684,10 +685,17 @@ CSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel)
CSAN_ATOMIC_FUNC_THREAD_FENCE(rel)
CSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst)
void
kcsan_atomic_interrupt_fence(void)
{
atomic_interrupt_fence();
}
/* -------------------------------------------------------------------------- */
#include <sys/bus.h>
#include <machine/bus.h>
#define BUS_SAN_PREFIX kcsan
#include <sys/bus_san.h>
int

630
sys/sys/atomic_san.h
View File

@ -2,12 +2,16 @@
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2019 Andrew Turner
* Copyright (c) 2021 The FreeBSD Foundation
*
* This software was developed by SRI International and the University of
* Cambridge Computer Laboratory (Department of Computer Science and
* Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
* DARPA SSITH research programme.
*
* Portions of this software were written by Mark Johnston under sponsorship
* by the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
@ -39,342 +43,360 @@
#error do not include this header, use machine/atomic.h
#endif
#define KCSAN_ATOMIC_FUNC_1(op, name, type) \
void kcsan_atomic_##op##_##name(volatile type *, type); \
void kcsan_atomic_##op##_acq_##name(volatile type *, type); \
void kcsan_atomic_##op##_rel_##name(volatile type *, type)
#ifndef ATOMIC_SAN_PREFIX
#error No sanitizer prefix available
#endif
#define KCSAN_ATOMIC_CMPSET(name, type) \
int kcsan_atomic_cmpset_##name(volatile type *, type, type); \
int kcsan_atomic_cmpset_acq_##name(volatile type *, type, type); \
int kcsan_atomic_cmpset_rel_##name(volatile type *, type, type)
#define ATOMIC_SAN_FUNC_1(sp, op, name, type) \
void sp##_atomic_##op##_##name(volatile type *, type); \
void sp##_atomic_##op##_acq_##name(volatile type *, type); \
void sp##_atomic_##op##_rel_##name(volatile type *, type)
#define KCSAN_ATOMIC_FCMPSET(name, type) \
int kcsan_atomic_fcmpset_##name(volatile type *, type *, type); \
int kcsan_atomic_fcmpset_acq_##name(volatile type *, type *, type); \
int kcsan_atomic_fcmpset_rel_##name(volatile type *, type *, type)
#define ATOMIC_SAN_CMPSET(sp, name, type) \
int sp##_atomic_cmpset_##name(volatile type *, type, type); \
int sp##_atomic_cmpset_acq_##name(volatile type *, type, type); \
int sp##_atomic_cmpset_rel_##name(volatile type *, type, type)
#define KCSAN_ATOMIC_READ(op, name, type) \
type kcsan_atomic_##op##_##name(volatile type *, type)
#define ATOMIC_SAN_FCMPSET(sp, name, type) \
int sp##_atomic_fcmpset_##name(volatile type *, type *, type); \
int sp##_atomic_fcmpset_acq_##name(volatile type *, type *, type); \
int sp##_atomic_fcmpset_rel_##name(volatile type *, type *, type)
#define KCSAN_ATOMIC_READANDCLEAR(name, type) \
type kcsan_atomic_readandclear_##name(volatile type *)
#define ATOMIC_SAN_READ(sp, op, name, type) \
type sp##_atomic_##op##_##name(volatile type *, type)
#define KCSAN_ATOMIC_LOAD(name, type) \
type kcsan_atomic_load_##name(volatile type *); \
type kcsan_atomic_load_acq_##name(volatile type *)
#define ATOMIC_SAN_READANDCLEAR(sp, name, type) \
type sp##_atomic_readandclear_##name(volatile type *)
#define KCSAN_ATOMIC_STORE(name, type) \
void kcsan_atomic_store_##name(volatile type *, type); \
void kcsan_atomic_store_rel_##name(volatile type *, type)
#define ATOMIC_SAN_LOAD(sp, name, type) \
type sp##_atomic_load_##name(volatile type *); \
type sp##_atomic_load_acq_##name(volatile type *)
#define KCSAN_ATOMIC_TEST(op, name, type) \
int kcsan_atomic_##op##_##name(volatile type *, u_int); \
int kcsan_atomic_##op##_acq_##name(volatile type *, u_int)
#define ATOMIC_SAN_STORE(sp, name, type) \
void sp##_atomic_store_##name(volatile type *, type); \
void sp##_atomic_store_rel_##name(volatile type *, type)
#define KCSAN_ATOMIC_FUNCS(name, type) \
KCSAN_ATOMIC_FUNC_1(add, name, type); \
KCSAN_ATOMIC_FUNC_1(clear, name, type); \
KCSAN_ATOMIC_CMPSET(name, type); \
KCSAN_ATOMIC_FCMPSET(name, type); \
KCSAN_ATOMIC_READ(fetchadd, name, type); \
KCSAN_ATOMIC_LOAD(name, type); \
KCSAN_ATOMIC_READANDCLEAR(name, type); \
KCSAN_ATOMIC_FUNC_1(set, name, type); \
KCSAN_ATOMIC_FUNC_1(subtract, name, type); \
KCSAN_ATOMIC_STORE(name, type); \
KCSAN_ATOMIC_READ(swap, name, type); \
KCSAN_ATOMIC_TEST(testandclear, name, type); \
KCSAN_ATOMIC_TEST(testandset, name, type)
#define ATOMIC_SAN_TEST(sp, op, name, type) \
int sp##_atomic_##op##_##name(volatile type *, u_int); \
int sp##_atomic_##op##_acq_##name(volatile type *, u_int)
KCSAN_ATOMIC_FUNCS(char, uint8_t);
KCSAN_ATOMIC_FUNCS(short, uint16_t);
KCSAN_ATOMIC_FUNCS(int, u_int);
KCSAN_ATOMIC_FUNCS(long, u_long);
KCSAN_ATOMIC_FUNCS(ptr, uintptr_t);
KCSAN_ATOMIC_FUNCS(8, uint8_t);
KCSAN_ATOMIC_FUNCS(16, uint16_t);
KCSAN_ATOMIC_FUNCS(32, uint32_t);
KCSAN_ATOMIC_FUNCS(64, uint64_t);
#define ATOMIC_SAN_THREAD_FENCE(sp) \
void sp##_atomic_thread_fence_acq(void); \
void sp##_atomic_thread_fence_rel(void); \
void sp##_atomic_thread_fence_acq_rel(void); \
void sp##_atomic_thread_fence_seq_cst(void); \
void sp##_atomic_interrupt_fence(void)
void kcsan_atomic_thread_fence_acq(void);
void kcsan_atomic_thread_fence_acq_rel(void);
void kcsan_atomic_thread_fence_rel(void);
void kcsan_atomic_thread_fence_seq_cst(void);
#define _ATOMIC_SAN_FUNCS(sp, name, type) \
ATOMIC_SAN_FUNC_1(sp, add, name, type); \
ATOMIC_SAN_FUNC_1(sp, clear, name, type); \
ATOMIC_SAN_CMPSET(sp, name, type); \
ATOMIC_SAN_FCMPSET(sp, name, type); \
ATOMIC_SAN_READ(sp, fetchadd, name, type); \
ATOMIC_SAN_LOAD(sp, name, type); \
ATOMIC_SAN_READANDCLEAR(sp, name, type); \
ATOMIC_SAN_FUNC_1(sp, set, name, type); \
ATOMIC_SAN_FUNC_1(sp, subtract, name, type); \
ATOMIC_SAN_STORE(sp, name, type); \
ATOMIC_SAN_READ(sp, swap, name, type); \
ATOMIC_SAN_TEST(sp, testandclear, name, type); \
ATOMIC_SAN_TEST(sp, testandset, name, type); \
ATOMIC_SAN_THREAD_FENCE(sp);
#ifndef KCSAN_RUNTIME
#define ATOMIC_SAN_FUNCS(name, type) \
_ATOMIC_SAN_FUNCS(ATOMIC_SAN_PREFIX, name, type)
#define atomic_add_char kcsan_atomic_add_char
#define atomic_add_acq_char kcsan_atomic_add_acq_char
#define atomic_add_rel_char kcsan_atomic_add_rel_char
#define atomic_clear_char kcsan_atomic_clear_char
#define atomic_clear_acq_char kcsan_atomic_clear_acq_char
#define atomic_clear_rel_char kcsan_atomic_clear_rel_char
#define atomic_cmpset_char kcsan_atomic_cmpset_char
#define atomic_cmpset_acq_char kcsan_atomic_cmpset_acq_char
#define atomic_cmpset_rel_char kcsan_atomic_cmpset_rel_char
#define atomic_fcmpset_char kcsan_atomic_fcmpset_char
#define atomic_fcmpset_acq_char kcsan_atomic_fcmpset_acq_char
#define atomic_fcmpset_rel_char kcsan_atomic_fcmpset_rel_char
#define atomic_fetchadd_char kcsan_atomic_fetchadd_char
#define atomic_load_char kcsan_atomic_load_char
#define atomic_load_acq_char kcsan_atomic_load_acq_char
#define atomic_readandclear_char kcsan_atomic_readandclear_char
#define atomic_set_char kcsan_atomic_set_char
#define atomic_set_acq_char kcsan_atomic_set_acq_char
#define atomic_set_rel_char kcsan_atomic_set_rel_char
#define atomic_subtract_char kcsan_atomic_subtract_char
#define atomic_subtract_acq_char kcsan_atomic_subtract_acq_char
#define atomic_subtract_rel_char kcsan_atomic_subtract_rel_char
#define atomic_store_char kcsan_atomic_store_char
#define atomic_store_rel_char kcsan_atomic_store_rel_char
#define atomic_swap_char kcsan_atomic_swap_char
#define atomic_testandclear_char kcsan_atomic_testandclear_char
#define atomic_testandset_char kcsan_atomic_testandset_char
ATOMIC_SAN_FUNCS(char, uint8_t);
ATOMIC_SAN_FUNCS(short, uint16_t);
ATOMIC_SAN_FUNCS(int, u_int);
ATOMIC_SAN_FUNCS(long, u_long);
ATOMIC_SAN_FUNCS(ptr, uintptr_t);
ATOMIC_SAN_FUNCS(8, uint8_t);
ATOMIC_SAN_FUNCS(16, uint16_t);
ATOMIC_SAN_FUNCS(32, uint32_t);
ATOMIC_SAN_FUNCS(64, uint64_t);
#define atomic_add_short kcsan_atomic_add_short
#define atomic_add_acq_short kcsan_atomic_add_acq_short
#define atomic_add_rel_short kcsan_atomic_add_rel_short
#define atomic_clear_short kcsan_atomic_clear_short
#define atomic_clear_acq_short kcsan_atomic_clear_acq_short
#define atomic_clear_rel_short kcsan_atomic_clear_rel_short
#define atomic_cmpset_short kcsan_atomic_cmpset_short
#define atomic_cmpset_acq_short kcsan_atomic_cmpset_acq_short
#define atomic_cmpset_rel_short kcsan_atomic_cmpset_rel_short
#define atomic_fcmpset_short kcsan_atomic_fcmpset_short
#define atomic_fcmpset_acq_short kcsan_atomic_fcmpset_acq_short
#define atomic_fcmpset_rel_short kcsan_atomic_fcmpset_rel_short
#define atomic_fetchadd_short kcsan_atomic_fetchadd_short
#define atomic_load_short kcsan_atomic_load_short
#define atomic_load_acq_short kcsan_atomic_load_acq_short
#define atomic_readandclear_short kcsan_atomic_readandclear_short
#define atomic_set_short kcsan_atomic_set_short
#define atomic_set_acq_short kcsan_atomic_set_acq_short
#define atomic_set_rel_short kcsan_atomic_set_rel_short
#define atomic_subtract_short kcsan_atomic_subtract_short
#define atomic_subtract_acq_short kcsan_atomic_subtract_acq_short
#define atomic_subtract_rel_short kcsan_atomic_subtract_rel_short
#define atomic_store_short kcsan_atomic_store_short
#define atomic_store_rel_short kcsan_atomic_store_rel_short
#define atomic_swap_short kcsan_atomic_swap_short
#define atomic_testandclear_short kcsan_atomic_testandclear_short
#define atomic_testandset_short kcsan_atomic_testandset_short
#ifndef SAN_RUNTIME
#define atomic_add_int kcsan_atomic_add_int
#define atomic_add_acq_int kcsan_atomic_add_acq_int
#define atomic_add_rel_int kcsan_atomic_add_rel_int
#define atomic_clear_int kcsan_atomic_clear_int
#define atomic_clear_acq_int kcsan_atomic_clear_acq_int
#define atomic_clear_rel_int kcsan_atomic_clear_rel_int
#define atomic_cmpset_int kcsan_atomic_cmpset_int
#define atomic_cmpset_acq_int kcsan_atomic_cmpset_acq_int
#define atomic_cmpset_rel_int kcsan_atomic_cmpset_rel_int
#define atomic_fcmpset_int kcsan_atomic_fcmpset_int
#define atomic_fcmpset_acq_int kcsan_atomic_fcmpset_acq_int
#define atomic_fcmpset_rel_int kcsan_atomic_fcmpset_rel_int
#define atomic_fetchadd_int kcsan_atomic_fetchadd_int
#define atomic_load_int kcsan_atomic_load_int
#define atomic_load_acq_int kcsan_atomic_load_acq_int
#define atomic_readandclear_int kcsan_atomic_readandclear_int
#define atomic_set_int kcsan_atomic_set_int
#define atomic_set_acq_int kcsan_atomic_set_acq_int
#define atomic_set_rel_int kcsan_atomic_set_rel_int
#define atomic_subtract_int kcsan_atomic_subtract_int
#define atomic_subtract_acq_int kcsan_atomic_subtract_acq_int
#define atomic_subtract_rel_int kcsan_atomic_subtract_rel_int
#define atomic_store_int kcsan_atomic_store_int
#define atomic_store_rel_int kcsan_atomic_store_rel_int
#define atomic_swap_int kcsan_atomic_swap_int
#define atomic_testandclear_int kcsan_atomic_testandclear_int
#define atomic_testandset_int kcsan_atomic_testandset_int
/*
* Redirect uses of an atomic(9) function to the sanitizer's interceptor.
* For instance, KASAN callers of atomic_add_char() will be redirected to
* kasan_atomic_add_char().
*/
#define ATOMIC_SAN(func) \
__CONCAT(ATOMIC_SAN_PREFIX, __CONCAT(_atomic_, func))
#define atomic_add_long kcsan_atomic_add_long
#define atomic_add_acq_long kcsan_atomic_add_acq_long
#define atomic_add_rel_long kcsan_atomic_add_rel_long
#define atomic_clear_long kcsan_atomic_clear_long
#define atomic_clear_acq_long kcsan_atomic_clear_acq_long
#define atomic_clear_rel_long kcsan_atomic_clear_rel_long
#define atomic_cmpset_long kcsan_atomic_cmpset_long
#define atomic_cmpset_acq_long kcsan_atomic_cmpset_acq_long
#define atomic_cmpset_rel_long kcsan_atomic_cmpset_rel_long
#define atomic_fcmpset_long kcsan_atomic_fcmpset_long
#define atomic_fcmpset_acq_long kcsan_atomic_fcmpset_acq_long
#define atomic_fcmpset_rel_long kcsan_atomic_fcmpset_rel_long
#define atomic_fetchadd_long kcsan_atomic_fetchadd_long
#define atomic_load_long kcsan_atomic_load_long
#define atomic_load_acq_long kcsan_atomic_load_acq_long
#define atomic_readandclear_long kcsan_atomic_readandclear_long
#define atomic_set_long kcsan_atomic_set_long
#define atomic_set_acq_long kcsan_atomic_set_acq_long
#define atomic_set_rel_long kcsan_atomic_set_rel_long
#define atomic_subtract_long kcsan_atomic_subtract_long
#define atomic_subtract_acq_long kcsan_atomic_subtract_acq_long
#define atomic_subtract_rel_long kcsan_atomic_subtract_rel_long
#define atomic_store_long kcsan_atomic_store_long
#define atomic_store_rel_long kcsan_atomic_store_rel_long
#define atomic_swap_long kcsan_atomic_swap_long
#define atomic_testandclear_long kcsan_atomic_testandclear_long
#define atomic_testandset_long kcsan_atomic_testandset_long
#define atomic_testandset_acq_long kcsan_atomic_testandset_acq_long
#define atomic_add_char ATOMIC_SAN(add_char)
#define atomic_add_acq_char ATOMIC_SAN(add_acq_char)
#define atomic_add_rel_char ATOMIC_SAN(add_rel_char)
#define atomic_clear_char ATOMIC_SAN(clear_char)
#define atomic_clear_acq_char ATOMIC_SAN(clear_acq_char)
#define atomic_clear_rel_char ATOMIC_SAN(clear_rel_char)
#define atomic_cmpset_char ATOMIC_SAN(cmpset_char)
#define atomic_cmpset_acq_char ATOMIC_SAN(cmpset_acq_char)
#define atomic_cmpset_rel_char ATOMIC_SAN(cmpset_rel_char)
#define atomic_fcmpset_char ATOMIC_SAN(fcmpset_char)
#define atomic_fcmpset_acq_char ATOMIC_SAN(fcmpset_acq_char)
#define atomic_fcmpset_rel_char ATOMIC_SAN(fcmpset_rel_char)
#define atomic_fetchadd_char ATOMIC_SAN(fetchadd_char)
#define atomic_load_char ATOMIC_SAN(load_char)
#define atomic_load_acq_char ATOMIC_SAN(load_acq_char)
#define atomic_readandclear_char ATOMIC_SAN(readandclear_char)
#define atomic_set_char ATOMIC_SAN(set_char)
#define atomic_set_acq_char ATOMIC_SAN(set_acq_char)
#define atomic_set_rel_char ATOMIC_SAN(set_rel_char)
#define atomic_subtract_char ATOMIC_SAN(subtract_char)
#define atomic_subtract_acq_char ATOMIC_SAN(subtract_acq_char)
#define atomic_subtract_rel_char ATOMIC_SAN(subtract_rel_char)
#define atomic_store_char ATOMIC_SAN(store_char)
#define atomic_store_rel_char ATOMIC_SAN(store_rel_char)
#define atomic_swap_char ATOMIC_SAN(swap_char)
#define atomic_testandclear_char ATOMIC_SAN(testandclear_char)
#define atomic_testandset_char ATOMIC_SAN(testandset_char)
#define atomic_add_ptr kcsan_atomic_add_ptr
#define atomic_add_acq_ptr kcsan_atomic_add_acq_ptr
#define atomic_add_rel_ptr kcsan_atomic_add_rel_ptr
#define atomic_clear_ptr kcsan_atomic_clear_ptr
#define atomic_clear_acq_ptr kcsan_atomic_clear_acq_ptr
#define atomic_clear_rel_ptr kcsan_atomic_clear_rel_ptr
#define atomic_cmpset_ptr kcsan_atomic_cmpset_ptr
#define atomic_cmpset_acq_ptr kcsan_atomic_cmpset_acq_ptr
#define atomic_cmpset_rel_ptr kcsan_atomic_cmpset_rel_ptr
#define atomic_fcmpset_ptr kcsan_atomic_fcmpset_ptr
#define atomic_fcmpset_acq_ptr kcsan_atomic_fcmpset_acq_ptr
#define atomic_fcmpset_rel_ptr kcsan_atomic_fcmpset_rel_ptr
#define atomic_fetchadd_ptr kcsan_atomic_fetchadd_ptr
#define atomic_add_short ATOMIC_SAN(add_short)
#define atomic_add_acq_short ATOMIC_SAN(add_acq_short)
#define atomic_add_rel_short ATOMIC_SAN(add_rel_short)
#define atomic_clear_short ATOMIC_SAN(clear_short)
#define atomic_clear_acq_short ATOMIC_SAN(clear_acq_short)
#define atomic_clear_rel_short ATOMIC_SAN(clear_rel_short)
#define atomic_cmpset_short ATOMIC_SAN(cmpset_short)
#define atomic_cmpset_acq_short ATOMIC_SAN(cmpset_acq_short)
#define atomic_cmpset_rel_short ATOMIC_SAN(cmpset_rel_short)
#define atomic_fcmpset_short ATOMIC_SAN(fcmpset_short)
#define atomic_fcmpset_acq_short ATOMIC_SAN(fcmpset_acq_short)
#define atomic_fcmpset_rel_short ATOMIC_SAN(fcmpset_rel_short)
#define atomic_fetchadd_short ATOMIC_SAN(fetchadd_short)
#define atomic_load_short ATOMIC_SAN(load_short)
#define atomic_load_acq_short ATOMIC_SAN(load_acq_short)
#define atomic_readandclear_short ATOMIC_SAN(readandclear_short)
#define atomic_set_short ATOMIC_SAN(set_short)
#define atomic_set_acq_short ATOMIC_SAN(set_acq_short)
#define atomic_set_rel_short ATOMIC_SAN(set_rel_short)
#define atomic_subtract_short ATOMIC_SAN(subtract_short)
#define atomic_subtract_acq_short ATOMIC_SAN(subtract_acq_short)
#define atomic_subtract_rel_short ATOMIC_SAN(subtract_rel_short)
#define atomic_store_short ATOMIC_SAN(store_short)
#define atomic_store_rel_short ATOMIC_SAN(store_rel_short)
#define atomic_swap_short ATOMIC_SAN(swap_short)
#define atomic_testandclear_short ATOMIC_SAN(testandclear_short)
#define atomic_testandset_short ATOMIC_SAN(testandset_short)
#define atomic_add_int ATOMIC_SAN(add_int)
#define atomic_add_acq_int ATOMIC_SAN(add_acq_int)
#define atomic_add_rel_int ATOMIC_SAN(add_rel_int)
#define atomic_clear_int ATOMIC_SAN(clear_int)
#define atomic_clear_acq_int ATOMIC_SAN(clear_acq_int)
#define atomic_clear_rel_int ATOMIC_SAN(clear_rel_int)
#define atomic_cmpset_int ATOMIC_SAN(cmpset_int)
#define atomic_cmpset_acq_int ATOMIC_SAN(cmpset_acq_int)
#define atomic_cmpset_rel_int ATOMIC_SAN(cmpset_rel_int)
#define atomic_fcmpset_int ATOMIC_SAN(fcmpset_int)
#define atomic_fcmpset_acq_int ATOMIC_SAN(fcmpset_acq_int)
#define atomic_fcmpset_rel_int ATOMIC_SAN(fcmpset_rel_int)
#define atomic_fetchadd_int ATOMIC_SAN(fetchadd_int)
#define atomic_load_int ATOMIC_SAN(load_int)
#define atomic_load_acq_int ATOMIC_SAN(load_acq_int)
#define atomic_readandclear_int ATOMIC_SAN(readandclear_int)
#define atomic_set_int ATOMIC_SAN(set_int)
#define atomic_set_acq_int ATOMIC_SAN(set_acq_int)
#define atomic_set_rel_int ATOMIC_SAN(set_rel_int)
#define atomic_subtract_int ATOMIC_SAN(subtract_int)
#define atomic_subtract_acq_int ATOMIC_SAN(subtract_acq_int)
#define atomic_subtract_rel_int ATOMIC_SAN(subtract_rel_int)
#define atomic_store_int ATOMIC_SAN(store_int)
#define atomic_store_rel_int ATOMIC_SAN(store_rel_int)
#define atomic_swap_int ATOMIC_SAN(swap_int)
#define atomic_testandclear_int ATOMIC_SAN(testandclear_int)
#define atomic_testandset_int ATOMIC_SAN(testandset_int)
#define atomic_add_long ATOMIC_SAN(add_long)
#define atomic_add_acq_long ATOMIC_SAN(add_acq_long)
#define atomic_add_rel_long ATOMIC_SAN(add_rel_long)
#define atomic_clear_long ATOMIC_SAN(clear_long)
#define atomic_clear_acq_long ATOMIC_SAN(clear_acq_long)
#define atomic_clear_rel_long ATOMIC_SAN(clear_rel_long)
#define atomic_cmpset_long ATOMIC_SAN(cmpset_long)
#define atomic_cmpset_acq_long ATOMIC_SAN(cmpset_acq_long)
#define atomic_cmpset_rel_long ATOMIC_SAN(cmpset_rel_long)
#define atomic_fcmpset_long ATOMIC_SAN(fcmpset_long)
#define atomic_fcmpset_acq_long ATOMIC_SAN(fcmpset_acq_long)
#define atomic_fcmpset_rel_long ATOMIC_SAN(fcmpset_rel_long)
#define atomic_fetchadd_long ATOMIC_SAN(fetchadd_long)
#define atomic_load_long ATOMIC_SAN(load_long)
#define atomic_load_acq_long ATOMIC_SAN(load_acq_long)
#define atomic_readandclear_long ATOMIC_SAN(readandclear_long)
#define atomic_set_long ATOMIC_SAN(set_long)
#define atomic_set_acq_long ATOMIC_SAN(set_acq_long)
#define atomic_set_rel_long ATOMIC_SAN(set_rel_long)
#define atomic_subtract_long ATOMIC_SAN(subtract_long)
#define atomic_subtract_acq_long ATOMIC_SAN(subtract_acq_long)
#define atomic_subtract_rel_long ATOMIC_SAN(subtract_rel_long)
#define atomic_store_long ATOMIC_SAN(store_long)
#define atomic_store_rel_long ATOMIC_SAN(store_rel_long)
#define atomic_swap_long ATOMIC_SAN(swap_long)
#define atomic_testandclear_long ATOMIC_SAN(testandclear_long)
#define atomic_testandset_long ATOMIC_SAN(testandset_long)
#define atomic_testandset_acq_long ATOMIC_SAN(testandset_acq_long)
#define atomic_add_ptr ATOMIC_SAN(add_ptr)
#define atomic_add_acq_ptr ATOMIC_SAN(add_acq_ptr)
#define atomic_add_rel_ptr ATOMIC_SAN(add_rel_ptr)
#define atomic_clear_ptr ATOMIC_SAN(clear_ptr)
#define atomic_clear_acq_ptr ATOMIC_SAN(clear_acq_ptr)
#define atomic_clear_rel_ptr ATOMIC_SAN(clear_rel_ptr)
#define atomic_cmpset_ptr ATOMIC_SAN(cmpset_ptr)
#define atomic_cmpset_acq_ptr ATOMIC_SAN(cmpset_acq_ptr)
#define atomic_cmpset_rel_ptr ATOMIC_SAN(cmpset_rel_ptr)
#define atomic_fcmpset_ptr ATOMIC_SAN(fcmpset_ptr)
#define atomic_fcmpset_acq_ptr ATOMIC_SAN(fcmpset_acq_ptr)
#define atomic_fcmpset_rel_ptr ATOMIC_SAN(fcmpset_rel_ptr)
#define atomic_fetchadd_ptr ATOMIC_SAN(fetchadd_ptr)
#define atomic_load_ptr(x) ({ \
__typeof(*x) __retptr; \
__retptr = (void *)kcsan_atomic_load_ptr((volatile uintptr_t *)(x)); \
__retptr = (void *)ATOMIC_SAN(load_ptr)((volatile uintptr_t *)(x)); \
__retptr; \
})
#define atomic_load_acq_ptr kcsan_atomic_load_acq_ptr
#define atomic_load_acq_ptr ATOMIC_SAN(load_acq_ptr)
#define atomic_load_consume_ptr(x) ({ \
__typeof(*x) __retptr; \
__retptr = (void *)kcsan_atomic_load_acq_ptr((volatile uintptr_t *)(x));\
__retptr = (void *)atomic_load_acq_ptr((volatile uintptr_t *)(x));\
__retptr; \
})
#define atomic_readandclear_ptr kcsan_atomic_readandclear_ptr
#define atomic_set_ptr kcsan_atomic_set_ptr
#define atomic_set_acq_ptr kcsan_atomic_set_acq_ptr
#define atomic_set_rel_ptr kcsan_atomic_set_rel_ptr
#define atomic_subtract_ptr kcsan_atomic_subtract_ptr
#define atomic_subtract_acq_ptr kcsan_atomic_subtract_acq_ptr
#define atomic_subtract_rel_ptr kcsan_atomic_subtract_rel_ptr
#define atomic_readandclear_ptr ATOMIC_SAN(readandclear_ptr)
#define atomic_set_ptr ATOMIC_SAN(set_ptr)
#define atomic_set_acq_ptr ATOMIC_SAN(set_acq_ptr)
#define atomic_set_rel_ptr ATOMIC_SAN(set_rel_ptr)
#define atomic_subtract_ptr ATOMIC_SAN(subtract_ptr)
#define atomic_subtract_acq_ptr ATOMIC_SAN(subtract_acq_ptr)
#define atomic_subtract_rel_ptr ATOMIC_SAN(subtract_rel_ptr)
#define atomic_store_ptr(x, v) ({ \
__typeof(*x) __value = (v); \
kcsan_atomic_store_ptr((volatile uintptr_t *)(x), (uintptr_t)(__value));\
ATOMIC_SAN(store_ptr)((volatile uintptr_t *)(x), (uintptr_t)(__value));\
})
#define atomic_store_rel_ptr kcsan_atomic_store_rel_ptr
#define atomic_swap_ptr kcsan_atomic_swap_ptr
#define atomic_testandclear_ptr kcsan_atomic_testandclear_ptr
#define atomic_testandset_ptr kcsan_atomic_testandset_ptr
#define atomic_store_rel_ptr ATOMIC_SAN(store_rel_ptr)
#define atomic_swap_ptr ATOMIC_SAN(swap_ptr)
#define atomic_testandclear_ptr ATOMIC_SAN(testandclear_ptr)
#define atomic_testandset_ptr ATOMIC_SAN(testandset_ptr)
#define atomic_add_8 kcsan_atomic_add_8
#define atomic_add_acq_8 kcsan_atomic_add_acq_8
#define atomic_add_rel_8 kcsan_atomic_add_rel_8
#define atomic_clear_8 kcsan_atomic_clear_8
#define atomic_clear_acq_8 kcsan_atomic_clear_acq_8
#define atomic_clear_rel_8 kcsan_atomic_clear_rel_8
#define atomic_cmpset_8 kcsan_atomic_cmpset_8
#define atomic_cmpset_acq_8 kcsan_atomic_cmpset_acq_8
#define atomic_cmpset_rel_8 kcsan_atomic_cmpset_rel_8
#define atomic_fcmpset_8 kcsan_atomic_fcmpset_8
#define atomic_fcmpset_acq_8 kcsan_atomic_fcmpset_acq_8
#define atomic_fcmpset_rel_8 kcsan_atomic_fcmpset_rel_8
#define atomic_fetchadd_8 kcsan_atomic_fetchadd_8
#define atomic_load_8 kcsan_atomic_load_8
#define atomic_load_acq_8 kcsan_atomic_load_acq_8
#define atomic_readandclear_8 kcsan_atomic_readandclear_8
#define atomic_set_8 kcsan_atomic_set_8
#define atomic_set_acq_8 kcsan_atomic_set_acq_8
#define atomic_set_rel_8 kcsan_atomic_set_rel_8
#define atomic_subtract_8 kcsan_atomic_subtract_8
#define atomic_subtract_acq_8 kcsan_atomic_subtract_acq_8
#define atomic_subtract_rel_8 kcsan_atomic_subtract_rel_8
#define atomic_store_8 kcsan_atomic_store_8
#define atomic_store_rel_8 kcsan_atomic_store_rel_8
#define atomic_swap_8 kcsan_atomic_swap_8
#define atomic_testandclear_8 kcsan_atomic_testandclear_8
#define atomic_testandset_8 kcsan_atomic_testandset_8
#define atomic_add_8 ATOMIC_SAN(add_8)
#define atomic_add_acq_8 ATOMIC_SAN(add_acq_8)
#define atomic_add_rel_8 ATOMIC_SAN(add_rel_8)
#define atomic_clear_8 ATOMIC_SAN(clear_8)
#define atomic_clear_acq_8 ATOMIC_SAN(clear_acq_8)
#define atomic_clear_rel_8 ATOMIC_SAN(clear_rel_8)
#define atomic_cmpset_8 ATOMIC_SAN(cmpset_8)
#define atomic_cmpset_acq_8 ATOMIC_SAN(cmpset_acq_8)
#define atomic_cmpset_rel_8 ATOMIC_SAN(cmpset_rel_8)
#define atomic_fcmpset_8 ATOMIC_SAN(fcmpset_8)
#define atomic_fcmpset_acq_8 ATOMIC_SAN(fcmpset_acq_8)
#define atomic_fcmpset_rel_8 ATOMIC_SAN(fcmpset_rel_8)
#define atomic_fetchadd_8 ATOMIC_SAN(fetchadd_8)
#define atomic_load_8 ATOMIC_SAN(load_8)
#define atomic_load_acq_8 ATOMIC_SAN(load_acq_8)
#define atomic_readandclear_8 ATOMIC_SAN(readandclear_8)
#define atomic_set_8 ATOMIC_SAN(set_8)
#define atomic_set_acq_8 ATOMIC_SAN(set_acq_8)
#define atomic_set_rel_8 ATOMIC_SAN(set_rel_8)
#define atomic_subtract_8 ATOMIC_SAN(subtract_8)
#define atomic_subtract_acq_8 ATOMIC_SAN(subtract_acq_8)
#define atomic_subtract_rel_8 ATOMIC_SAN(subtract_rel_8)
#define atomic_store_8 ATOMIC_SAN(store_8)
#define atomic_store_rel_8 ATOMIC_SAN(store_rel_8)
#define atomic_swap_8 ATOMIC_SAN(swap_8)
#define atomic_testandclear_8 ATOMIC_SAN(testandclear_8)
#define atomic_testandset_8 ATOMIC_SAN(testandset_8)
#define atomic_add_16 kcsan_atomic_add_16
#define atomic_add_acq_16 kcsan_atomic_add_acq_16
#define atomic_add_rel_16 kcsan_atomic_add_rel_16
#define atomic_clear_16 kcsan_atomic_clear_16
#define atomic_clear_acq_16 kcsan_atomic_clear_acq_16
#define atomic_clear_rel_16 kcsan_atomic_clear_rel_16
#define atomic_cmpset_16 kcsan_atomic_cmpset_16
#define atomic_cmpset_acq_16 kcsan_atomic_cmpset_acq_16
#define atomic_cmpset_rel_16 kcsan_atomic_cmpset_rel_16
#define atomic_fcmpset_16 kcsan_atomic_fcmpset_16
#define atomic_fcmpset_acq_16 kcsan_atomic_fcmpset_acq_16
#define atomic_fcmpset_rel_16 kcsan_atomic_fcmpset_rel_16
#define atomic_fetchadd_16 kcsan_atomic_fetchadd_16
#define atomic_load_16 kcsan_atomic_load_16
#define atomic_load_acq_16 kcsan_atomic_load_acq_16
#define atomic_readandclear_16 kcsan_atomic_readandclear_16
#define atomic_set_16 kcsan_atomic_set_16
#define atomic_set_acq_16 kcsan_atomic_set_acq_16
#define atomic_set_rel_16 kcsan_atomic_set_rel_16
#define atomic_subtract_16 kcsan_atomic_subtract_16
#define atomic_subtract_acq_16 kcsan_atomic_subtract_acq_16
#define atomic_subtract_rel_16 kcsan_atomic_subtract_rel_16
#define atomic_store_16 kcsan_atomic_store_16
#define atomic_store_rel_16 kcsan_atomic_store_rel_16
#define atomic_swap_16 kcsan_atomic_swap_16
#define atomic_testandclear_16 kcsan_atomic_testandclear_16
#define atomic_testandset_16 kcsan_atomic_testandset_16
#define atomic_add_16 ATOMIC_SAN(add_16)
#define atomic_add_acq_16 ATOMIC_SAN(add_acq_16)
#define atomic_add_rel_16 ATOMIC_SAN(add_rel_16)
#define atomic_clear_16 ATOMIC_SAN(clear_16)
#define atomic_clear_acq_16 ATOMIC_SAN(clear_acq_16)
#define atomic_clear_rel_16 ATOMIC_SAN(clear_rel_16)
#define atomic_cmpset_16 ATOMIC_SAN(cmpset_16)
#define atomic_cmpset_acq_16 ATOMIC_SAN(cmpset_acq_16)
#define atomic_cmpset_rel_16 ATOMIC_SAN(cmpset_rel_16)
#define atomic_fcmpset_16 ATOMIC_SAN(fcmpset_16)
#define atomic_fcmpset_acq_16 ATOMIC_SAN(fcmpset_acq_16)
#define atomic_fcmpset_rel_16 ATOMIC_SAN(fcmpset_rel_16)
#define atomic_fetchadd_16 ATOMIC_SAN(fetchadd_16)
#define atomic_load_16 ATOMIC_SAN(load_16)
#define atomic_load_acq_16 ATOMIC_SAN(load_acq_16)
#define atomic_readandclear_16 ATOMIC_SAN(readandclear_16)
#define atomic_set_16 ATOMIC_SAN(set_16)
#define atomic_set_acq_16 ATOMIC_SAN(set_acq_16)
#define atomic_set_rel_16 ATOMIC_SAN(set_rel_16)
#define atomic_subtract_16 ATOMIC_SAN(subtract_16)
#define atomic_subtract_acq_16 ATOMIC_SAN(subtract_acq_16)
#define atomic_subtract_rel_16 ATOMIC_SAN(subtract_rel_16)
#define atomic_store_16 ATOMIC_SAN(store_16)
#define atomic_store_rel_16 ATOMIC_SAN(store_rel_16)
#define atomic_swap_16 ATOMIC_SAN(swap_16)
#define atomic_testandclear_16 ATOMIC_SAN(testandclear_16)
#define atomic_testandset_16 ATOMIC_SAN(testandset_16)
#define atomic_add_32 kcsan_atomic_add_32
#define atomic_add_acq_32 kcsan_atomic_add_acq_32
#define atomic_add_rel_32 kcsan_atomic_add_rel_32
#define atomic_clear_32 kcsan_atomic_clear_32
#define atomic_clear_acq_32 kcsan_atomic_clear_acq_32
#define atomic_clear_rel_32 kcsan_atomic_clear_rel_32
#define atomic_cmpset_32 kcsan_atomic_cmpset_32
#define atomic_cmpset_acq_32 kcsan_atomic_cmpset_acq_32
#define atomic_cmpset_rel_32 kcsan_atomic_cmpset_rel_32
#define atomic_fcmpset_32 kcsan_atomic_fcmpset_32
#define atomic_fcmpset_acq_32 kcsan_atomic_fcmpset_acq_32
#define atomic_fcmpset_rel_32 kcsan_atomic_fcmpset_rel_32
#define atomic_fetchadd_32 kcsan_atomic_fetchadd_32
#define atomic_load_32 kcsan_atomic_load_32
#define atomic_load_acq_32 kcsan_atomic_load_acq_32
#define atomic_readandclear_32 kcsan_atomic_readandclear_32
#define atomic_set_32 kcsan_atomic_set_32
#define atomic_set_acq_32 kcsan_atomic_set_acq_32
#define atomic_set_rel_32 kcsan_atomic_set_rel_32
#define atomic_subtract_32 kcsan_atomic_subtract_32
#define atomic_subtract_acq_32 kcsan_atomic_subtract_acq_32
#define atomic_subtract_rel_32 kcsan_atomic_subtract_rel_32
#define atomic_store_32 kcsan_atomic_store_32
#define atomic_store_rel_32 kcsan_atomic_store_rel_32
#define atomic_swap_32 kcsan_atomic_swap_32
#define atomic_testandclear_32 kcsan_atomic_testandclear_32
#define atomic_testandset_32 kcsan_atomic_testandset_32
#define atomic_add_32 ATOMIC_SAN(add_32)
#define atomic_add_acq_32 ATOMIC_SAN(add_acq_32)
#define atomic_add_rel_32 ATOMIC_SAN(add_rel_32)
#define atomic_clear_32 ATOMIC_SAN(clear_32)
#define atomic_clear_acq_32 ATOMIC_SAN(clear_acq_32)
#define atomic_clear_rel_32 ATOMIC_SAN(clear_rel_32)
#define atomic_cmpset_32 ATOMIC_SAN(cmpset_32)
#define atomic_cmpset_acq_32 ATOMIC_SAN(cmpset_acq_32)
#define atomic_cmpset_rel_32 ATOMIC_SAN(cmpset_rel_32)
#define atomic_fcmpset_32 ATOMIC_SAN(fcmpset_32)
#define atomic_fcmpset_acq_32 ATOMIC_SAN(fcmpset_acq_32)
#define atomic_fcmpset_rel_32 ATOMIC_SAN(fcmpset_rel_32)
#define atomic_fetchadd_32 ATOMIC_SAN(fetchadd_32)
#define atomic_load_32 ATOMIC_SAN(load_32)
#define atomic_load_acq_32 ATOMIC_SAN(load_acq_32)
#define atomic_readandclear_32 ATOMIC_SAN(readandclear_32)
#define atomic_set_32 ATOMIC_SAN(set_32)
#define atomic_set_acq_32 ATOMIC_SAN(set_acq_32)
#define atomic_set_rel_32 ATOMIC_SAN(set_rel_32)
#define atomic_subtract_32 ATOMIC_SAN(subtract_32)
#define atomic_subtract_acq_32 ATOMIC_SAN(subtract_acq_32)
#define atomic_subtract_rel_32 ATOMIC_SAN(subtract_rel_32)
#define atomic_store_32 ATOMIC_SAN(store_32)
#define atomic_store_rel_32 ATOMIC_SAN(store_rel_32)
#define atomic_swap_32 ATOMIC_SAN(swap_32)
#define atomic_testandclear_32 ATOMIC_SAN(testandclear_32)
#define atomic_testandset_32 ATOMIC_SAN(testandset_32)
#define atomic_add_64 kcsan_atomic_add_64
#define atomic_add_acq_64 kcsan_atomic_add_acq_64
#define atomic_add_rel_64 kcsan_atomic_add_rel_64
#define atomic_clear_64 kcsan_atomic_clear_64
#define atomic_clear_acq_64 kcsan_atomic_clear_acq_64
#define atomic_clear_rel_64 kcsan_atomic_clear_rel_64
#define atomic_cmpset_64 kcsan_atomic_cmpset_64
#define atomic_cmpset_acq_64 kcsan_atomic_cmpset_acq_64
#define atomic_cmpset_rel_64 kcsan_atomic_cmpset_rel_64
#define atomic_fcmpset_64 kcsan_atomic_fcmpset_64
#define atomic_fcmpset_acq_64 kcsan_atomic_fcmpset_acq_64
#define atomic_fcmpset_rel_64 kcsan_atomic_fcmpset_rel_64
#define atomic_fetchadd_64 kcsan_atomic_fetchadd_64
#define atomic_load_64 kcsan_atomic_load_64
#define atomic_load_acq_64 kcsan_atomic_load_acq_64
#define atomic_readandclear_64 kcsan_atomic_readandclear_64
#define atomic_set_64 kcsan_atomic_set_64
#define atomic_set_acq_64 kcsan_atomic_set_acq_64
#define atomic_set_rel_64 kcsan_atomic_set_rel_64
#define atomic_subtract_64 kcsan_atomic_subtract_64
#define atomic_subtract_acq_64 kcsan_atomic_subtract_acq_64
#define atomic_subtract_rel_64 kcsan_atomic_subtract_rel_64
#define atomic_store_64 kcsan_atomic_store_64
#define atomic_store_rel_64 kcsan_atomic_store_rel_64
#define atomic_swap_64 kcsan_atomic_swap_64
#define atomic_testandclear_64 kcsan_atomic_testandclear_64
#define atomic_testandset_64 kcsan_atomic_testandset_64
#define atomic_add_64 ATOMIC_SAN(add_64)
#define atomic_add_acq_64 ATOMIC_SAN(add_acq_64)
#define atomic_add_rel_64 ATOMIC_SAN(add_rel_64)
#define atomic_clear_64 ATOMIC_SAN(clear_64)
#define atomic_clear_acq_64 ATOMIC_SAN(clear_acq_64)
#define atomic_clear_rel_64 ATOMIC_SAN(clear_rel_64)
#define atomic_cmpset_64 ATOMIC_SAN(cmpset_64)
#define atomic_cmpset_acq_64 ATOMIC_SAN(cmpset_acq_64)
#define atomic_cmpset_rel_64 ATOMIC_SAN(cmpset_rel_64)
#define atomic_fcmpset_64 ATOMIC_SAN(fcmpset_64)
#define atomic_fcmpset_acq_64 ATOMIC_SAN(fcmpset_acq_64)
#define atomic_fcmpset_rel_64 ATOMIC_SAN(fcmpset_rel_64)
#define atomic_fetchadd_64 ATOMIC_SAN(fetchadd_64)
#define atomic_load_64 ATOMIC_SAN(load_64)
#define atomic_load_acq_64 ATOMIC_SAN(load_acq_64)
#define atomic_readandclear_64 ATOMIC_SAN(readandclear_64)
#define atomic_set_64 ATOMIC_SAN(set_64)
#define atomic_set_acq_64 ATOMIC_SAN(set_acq_64)
#define atomic_set_rel_64 ATOMIC_SAN(set_rel_64)
#define atomic_subtract_64 ATOMIC_SAN(subtract_64)
#define atomic_subtract_acq_64 ATOMIC_SAN(subtract_acq_64)
#define atomic_subtract_rel_64 ATOMIC_SAN(subtract_rel_64)
#define atomic_store_64 ATOMIC_SAN(store_64)
#define atomic_store_rel_64 ATOMIC_SAN(store_rel_64)
#define atomic_swap_64 ATOMIC_SAN(swap_64)
#define atomic_testandclear_64 ATOMIC_SAN(testandclear_64)
#define atomic_testandset_64 ATOMIC_SAN(testandset_64)
#define atomic_thread_fence_acq kcsan_atomic_thread_fence_acq
#define atomic_thread_fence_acq_rel kcsan_atomic_thread_fence_acq_rel
#define atomic_thread_fence_rel kcsan_atomic_thread_fence_rel
#define atomic_thread_fence_seq_cst kcsan_atomic_thread_fence_seq_cst
#define atomic_interrupt_fence __compiler_membar
#define atomic_thread_fence_acq ATOMIC_SAN(thread_fence_acq)
#define atomic_thread_fence_acq_rel ATOMIC_SAN(thread_fence_acq_rel)
#define atomic_thread_fence_rel ATOMIC_SAN(thread_fence_rel)
#define atomic_thread_fence_seq_cst ATOMIC_SAN(thread_fence_seq_cst)
#define atomic_interrupt_fence ATOMIC_SAN(interrupt_fence)
#endif /* !KCSAN_RUNTIME */
#endif /* !SAN_RUNTIME */
#endif /* !_SYS_ATOMIC_SAN_H_ */

288
sys/sys/bus_san.h
View File

@ -2,12 +2,16 @@
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2019 Andrew Turner
* Copyright (c) 2021 The FreeBSD Foundation
*
* This software was developed by SRI International and the University of
* Cambridge Computer Laboratory (Department of Computer Science and
* Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
* DARPA SSITH research programme.
*
* Portions of this software were written by Mark Johnston under sponsorship by
* the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
@ -35,174 +39,190 @@
#ifndef _SYS_BUS_SAN_H_
#define _SYS_BUS_SAN_H_
#define KCSAN_BS_MULTI(rw, width, type) \
void kcsan_bus_space_##rw##_multi_##width(bus_space_tag_t, \
#ifndef _MACHINE_BUS_H_
#error do not include this header, use machine/bus.h
#endif
#ifndef BUS_SAN_PREFIX
#error No sanitizer prefix defined
#endif
#define BUS_SAN_MULTI(sp, rw, width, type) \
void sp##_bus_space_##rw##_multi_##width(bus_space_tag_t, \
bus_space_handle_t, bus_size_t, type *, bus_size_t); \
void kcsan_bus_space_##rw##_multi_stream_##width(bus_space_tag_t, \
void sp##_bus_space_##rw##_multi_stream_##width(bus_space_tag_t, \
bus_space_handle_t, bus_size_t, type *, bus_size_t); \
void kcsan_bus_space_##rw##_region_##width(bus_space_tag_t, \
void sp##_bus_space_##rw##_region_##width(bus_space_tag_t, \
bus_space_handle_t, bus_size_t, type *, bus_size_t); \
void kcsan_bus_space_##rw##_region_stream_##width(bus_space_tag_t, \
void sp##_bus_space_##rw##_region_stream_##width(bus_space_tag_t, \
bus_space_handle_t, bus_size_t, type *, bus_size_t)
#define KCSAN_BS_READ(width, type) \
type kcsan_bus_space_read_##width(bus_space_tag_t, \
#define BUS_SAN_READ(sp, width, type) \
type sp##_bus_space_read_##width(bus_space_tag_t, \
bus_space_handle_t, bus_size_t); \
type kcsan_bus_space_read_stream_##width(bus_space_tag_t, \
type sp##_bus_space_read_stream_##width(bus_space_tag_t, \
bus_space_handle_t, bus_size_t); \
KCSAN_BS_MULTI(read, width, type)
BUS_SAN_MULTI(sp, read, width, type)
#define KCSAN_BS_WRITE(width, type) \
void kcsan_bus_space_write_##width(bus_space_tag_t, \
#define BUS_SAN_WRITE(sp, width, type) \
void sp##_bus_space_write_##width(bus_space_tag_t, \
bus_space_handle_t, bus_size_t, type); \
void kcsan_bus_space_write_stream_##width(bus_space_tag_t, \
void sp##_bus_space_write_stream_##width(bus_space_tag_t, \
bus_space_handle_t, bus_size_t, type); \
KCSAN_BS_MULTI(write, width, const type)
BUS_SAN_MULTI(sp, write, width, const type)
#define KCSAN_BS_SET(width, type) \
void kcsan_bus_space_set_multi_##width(bus_space_tag_t, \
#define BUS_SAN_SET(sp, width, type) \
void sp##_bus_space_set_multi_##width(bus_space_tag_t, \
bus_space_handle_t, bus_size_t, type, bus_size_t); \
void kcsan_bus_space_set_multi_stream_##width(bus_space_tag_t, \
void sp##_bus_space_set_multi_stream_##width(bus_space_tag_t, \
bus_space_handle_t, bus_size_t, type, bus_size_t); \
void kcsan_bus_space_set_region_##width(bus_space_tag_t, \
void sp##_bus_space_set_region_##width(bus_space_tag_t, \
bus_space_handle_t, bus_size_t, type, bus_size_t); \
void kcsan_bus_space_set_region_stream_##width(bus_space_tag_t, \
void sp##_bus_space_set_region_stream_##width(bus_space_tag_t, \
bus_space_handle_t, bus_size_t, type, bus_size_t)
#define KCSAN_BS_COPY(width, type) \
void kcsan_bus_space_copy_region_##width(bus_space_tag_t, \
#define BUS_SAN_COPY(sp, width, type) \
void sp##_bus_space_copy_region_##width(bus_space_tag_t, \
bus_space_handle_t, bus_size_t, bus_space_handle_t, \
bus_size_t, bus_size_t); \
void kcsan_bus_space_copy_region_stream_##width(bus_space_tag_t, \
void sp##_bus_space_copy_region_stream_##width(bus_space_tag_t, \
bus_space_handle_t, bus_size_t, bus_space_handle_t, \
bus_size_t, bus_size_t);
#define KCSAN_BS_PEEK(width, type) \
int kcsan_bus_space_peek_##width(bus_space_tag_t, \
#define BUS_SAN_PEEK(sp, width, type) \
int sp##_bus_space_peek_##width(bus_space_tag_t, \
bus_space_handle_t, bus_size_t, type *);
#define KCSAN_BS_POKE(width, type) \
int kcsan_bus_space_poke_##width(bus_space_tag_t, \
#define BUS_SAN_POKE(sp, width, type) \
int sp##_bus_space_poke_##width(bus_space_tag_t, \
bus_space_handle_t, bus_size_t, type);
#define KCSAN_BS(width, type) \
KCSAN_BS_READ(width, type); \
KCSAN_BS_WRITE(width, type); \
KCSAN_BS_SET(width, type); \
KCSAN_BS_COPY(width, type) \
KCSAN_BS_PEEK(width, type); \
KCSAN_BS_POKE(width, type);
#define BUS_SAN_MISC(sp) \
int sp##_bus_space_map(bus_space_tag_t, bus_addr_t, bus_size_t, \
int, bus_space_handle_t *); \
void sp##_bus_space_unmap(bus_space_tag_t, bus_space_handle_t, \
bus_size_t); \
int sp##_bus_space_subregion(bus_space_tag_t, bus_space_handle_t,\
bus_size_t, bus_size_t, bus_space_handle_t *); \
int sp##_bus_space_alloc(bus_space_tag_t, bus_addr_t, bus_addr_t,\
bus_size_t, bus_size_t, bus_size_t, int, bus_addr_t *, \
bus_space_handle_t *); \
void sp##_bus_space_free(bus_space_tag_t, bus_space_handle_t, \
bus_size_t); \
void sp##_bus_space_barrier(bus_space_tag_t, bus_space_handle_t,\
bus_size_t, bus_size_t, int);
KCSAN_BS(1, uint8_t);
KCSAN_BS(2, uint16_t);
KCSAN_BS(4, uint32_t);
KCSAN_BS(8, uint64_t);
#define _BUS_SAN_FUNCS(sp, width, type) \
BUS_SAN_READ(sp, width, type); \
BUS_SAN_WRITE(sp, width, type); \
BUS_SAN_SET(sp, width, type); \
BUS_SAN_COPY(sp, width, type) \
BUS_SAN_PEEK(sp, width, type); \
BUS_SAN_POKE(sp, width, type); \
BUS_SAN_MISC(sp);
int kcsan_bus_space_map(bus_space_tag_t, bus_addr_t, bus_size_t, int,
bus_space_handle_t *);
void kcsan_bus_space_unmap(bus_space_tag_t, bus_space_handle_t, bus_size_t);
int kcsan_bus_space_subregion(bus_space_tag_t, bus_space_handle_t, bus_size_t,
bus_size_t, bus_space_handle_t *);
int kcsan_bus_space_alloc(bus_space_tag_t, bus_addr_t, bus_addr_t,
bus_size_t, bus_size_t, bus_size_t, int, bus_addr_t *,
bus_space_handle_t *);
void kcsan_bus_space_free(bus_space_tag_t, bus_space_handle_t, bus_size_t);
void kcsan_bus_space_barrier(bus_space_tag_t, bus_space_handle_t, bus_size_t,
bus_size_t, int);
#define BUS_SAN_FUNCS(width, type) \
_BUS_SAN_FUNCS(BUS_SAN_PREFIX, width, type)
#ifndef KCSAN_RUNTIME
BUS_SAN_FUNCS(1, uint8_t);
BUS_SAN_FUNCS(2, uint16_t);
BUS_SAN_FUNCS(4, uint32_t);
BUS_SAN_FUNCS(8, uint64_t);
#define bus_space_map kcsan_bus_space_map
#define bus_space_unmap kcsan_bus_space_unmap
#define bus_space_subregion kcsan_bus_space_subregion
#define bus_space_alloc kcsan_bus_space_alloc
#define bus_space_free kcsan_bus_space_free
#define bus_space_barrier kcsan_bus_space_barrier
#ifndef SAN_RUNTIME
#define bus_space_read_1 kcsan_bus_space_read_1
#define bus_space_read_stream_1 kcsan_bus_space_read_stream_1
#define bus_space_read_multi_1 kcsan_bus_space_read_multi_1
#define bus_space_read_multi_stream_1 kcsan_bus_space_read_multi_stream_1
#define bus_space_read_region_1 kcsan_bus_space_read_region_1
#define bus_space_read_region_stream_1 kcsan_bus_space_read_region_stream_1
#define bus_space_write_1 kcsan_bus_space_write_1
#define bus_space_write_stream_1 kcsan_bus_space_write_stream_1
#define bus_space_write_multi_1 kcsan_bus_space_write_multi_1
#define bus_space_write_multi_stream_1 kcsan_bus_space_write_multi_stream_1
#define bus_space_write_region_1 kcsan_bus_space_write_region_1
#define bus_space_write_region_stream_1 kcsan_bus_space_write_region_stream_1
#define bus_space_set_multi_1 kcsan_bus_space_set_multi_1
#define bus_space_set_multi_stream_1 kcsan_bus_space_set_multi_stream_1
#define bus_space_set_region_1 kcsan_bus_space_set_region_1
#define bus_space_set_region_stream_1 kcsan_bus_space_set_region_stream_1
#define bus_space_copy_multi_1 kcsan_bus_space_copy_multi_1
#define bus_space_copy_multi_stream_1 kcsan_bus_space_copy_multi_stream_1
#define bus_space_poke_1 kcsan_bus_space_poke_1
#define bus_space_peek_1 kcsan_bus_space_peek_1
#define BUS_SAN(func) __CONCAT(BUS_SAN_PREFIX, __CONCAT(_bus_space_, func))
#define bus_space_read_2 kcsan_bus_space_read_2
#define bus_space_read_stream_2 kcsan_bus_space_read_stream_2
#define bus_space_read_multi_2 kcsan_bus_space_read_multi_2
#define bus_space_read_multi_stream_2 kcsan_bus_space_read_multi_stream_2
#define bus_space_read_region_2 kcsan_bus_space_read_region_2
#define bus_space_read_region_stream_2 kcsan_bus_space_read_region_stream_2
#define bus_space_write_2 kcsan_bus_space_write_2
#define bus_space_write_stream_2 kcsan_bus_space_write_stream_2
#define bus_space_write_multi_2 kcsan_bus_space_write_multi_2
#define bus_space_write_multi_stream_2 kcsan_bus_space_write_multi_stream_2
#define bus_space_write_region_2 kcsan_bus_space_write_region_2
#define bus_space_write_region_stream_2 kcsan_bus_space_write_region_stream_2
#define bus_space_set_multi_2 kcsan_bus_space_set_multi_2
#define bus_space_set_multi_stream_2 kcsan_bus_space_set_multi_stream_2
#define bus_space_set_region_2 kcsan_bus_space_set_region_2
#define bus_space_set_region_stream_2 kcsan_bus_space_set_region_stream_2
#define bus_space_copy_multi_2 kcsan_bus_space_copy_multi_2
#define bus_space_copy_multi_stream_2 kcsan_bus_space_copy_multi_stream_2
#define bus_space_poke_2 kcsan_bus_space_poke_2
#define bus_space_peek_2 kcsan_bus_space_peek_2
#define bus_space_map BUS_SAN(map)
#define bus_space_unmap BUS_SAN(unmap)
#define bus_space_subregion BUS_SAN(subregion)
#define bus_space_alloc BUS_SAN(alloc)
#define bus_space_free BUS_SAN(free)
#define bus_space_barrier BUS_SAN(barrier)
#define bus_space_read_4 kcsan_bus_space_read_4
#define bus_space_read_stream_4 kcsan_bus_space_read_stream_4
#define bus_space_read_multi_4 kcsan_bus_space_read_multi_4
#define bus_space_read_multi_stream_4 kcsan_bus_space_read_multi_stream_4
#define bus_space_read_region_4 kcsan_bus_space_read_region_4
#define bus_space_read_region_stream_4 kcsan_bus_space_read_region_stream_4
#define bus_space_write_4 kcsan_bus_space_write_4
#define bus_space_write_stream_4 kcsan_bus_space_write_stream_4
#define bus_space_write_multi_4 kcsan_bus_space_write_multi_4
#define bus_space_write_multi_stream_4 kcsan_bus_space_write_multi_stream_4
#define bus_space_write_region_4 kcsan_bus_space_write_region_4
#define bus_space_write_region_stream_4 kcsan_bus_space_write_region_stream_4
#define bus_space_set_multi_4 kcsan_bus_space_set_multi_4
#define bus_space_set_multi_stream_4 kcsan_bus_space_set_multi_stream_4
#define bus_space_set_region_4 kcsan_bus_space_set_region_4
#define bus_space_set_region_stream_4 kcsan_bus_space_set_region_stream_4
#define bus_space_copy_multi_4 kcsan_bus_space_copy_multi_4
#define bus_space_copy_multi_stream_4 kcsan_bus_space_copy_multi_stream_4
#define bus_space_poke_4 kcsan_bus_space_poke_4
#define bus_space_peek_4 kcsan_bus_space_peek_4
#define bus_space_read_1 BUS_SAN(read_1)
#define bus_space_read_stream_1 BUS_SAN(read_stream_1)
#define bus_space_read_multi_1 BUS_SAN(read_multi_1)
#define bus_space_read_multi_stream_1 BUS_SAN(read_multi_stream_1)
#define bus_space_read_region_1 BUS_SAN(read_region_1)
#define bus_space_read_region_stream_1 BUS_SAN(read_region_stream_1)
#define bus_space_write_1 BUS_SAN(write_1)
#define bus_space_write_stream_1 BUS_SAN(write_stream_1)
#define bus_space_write_multi_1 BUS_SAN(write_multi_1)
#define bus_space_write_multi_stream_1 BUS_SAN(write_multi_stream_1)
#define bus_space_write_region_1 BUS_SAN(write_region_1)
#define bus_space_write_region_stream_1 BUS_SAN(write_region_stream_1)
#define bus_space_set_multi_1 BUS_SAN(set_multi_1)
#define bus_space_set_multi_stream_1 BUS_SAN(set_multi_stream_1)
#define bus_space_set_region_1 BUS_SAN(set_region_1)
#define bus_space_set_region_stream_1 BUS_SAN(set_region_stream_1)
#define bus_space_copy_multi_1 BUS_SAN(copy_multi_1)
#define bus_space_copy_multi_stream_1 BUS_SAN(copy_multi_stream_1)
#define bus_space_poke_1 BUS_SAN(poke_1)
#define bus_space_peek_1 BUS_SAN(peek_1)
#define bus_space_read_8 kcsan_bus_space_read_8
#define bus_space_read_stream_8 kcsan_bus_space_read_stream_8
#define bus_space_read_multi_8 kcsan_bus_space_read_multi_8
#define bus_space_read_multi_stream_8 kcsan_bus_space_read_multi_stream_8
#define bus_space_read_region_8 kcsan_bus_space_read_region_8
#define bus_space_read_region_stream_8 kcsan_bus_space_read_region_stream_8
#define bus_space_write_8 kcsan_bus_space_write_8
#define bus_space_write_stream_8 kcsan_bus_space_write_stream_8
#define bus_space_write_multi_8 kcsan_bus_space_write_multi_8
#define bus_space_write_multi_stream_8 kcsan_bus_space_write_multi_stream_8
#define bus_space_write_region_8 kcsan_bus_space_write_region_8
#define bus_space_write_region_stream_8 kcsan_bus_space_write_region_stream_8
#define bus_space_set_multi_8 kcsan_bus_space_set_multi_8
#define bus_space_set_multi_stream_8 kcsan_bus_space_set_multi_stream_8
#define bus_space_set_region_8 kcsan_bus_space_set_region_8
#define bus_space_set_region_stream_8 kcsan_bus_space_set_region_stream_8
#define bus_space_copy_multi_8 kcsan_bus_space_copy_multi_8
#define bus_space_copy_multi_stream_8 kcsan_bus_space_copy_multi_stream_8
#define bus_space_poke_8 kcsan_bus_space_poke_8
#define bus_space_peek_8 kcsan_bus_space_peek_8
#define bus_space_read_2 BUS_SAN(read_2)
#define bus_space_read_stream_2 BUS_SAN(read_stream_2)
#define bus_space_read_multi_2 BUS_SAN(read_multi_2)
#define bus_space_read_multi_stream_2 BUS_SAN(read_multi_stream_2)
#define bus_space_read_region_2 BUS_SAN(read_region_2)
#define bus_space_read_region_stream_2 BUS_SAN(read_region_stream_2)
#define bus_space_write_2 BUS_SAN(write_2)
#define bus_space_write_stream_2 BUS_SAN(write_stream_2)
#define bus_space_write_multi_2 BUS_SAN(write_multi_2)
#define bus_space_write_multi_stream_2 BUS_SAN(write_multi_stream_2)
#define bus_space_write_region_2 BUS_SAN(write_region_2)
#define bus_space_write_region_stream_2 BUS_SAN(write_region_stream_2)
#define bus_space_set_multi_2 BUS_SAN(set_multi_2)
#define bus_space_set_multi_stream_2 BUS_SAN(set_multi_stream_2)
#define bus_space_set_region_2 BUS_SAN(set_region_2)
#define bus_space_set_region_stream_2 BUS_SAN(set_region_stream_2)
#define bus_space_copy_multi_2 BUS_SAN(copy_multi_2)
#define bus_space_copy_multi_stream_2 BUS_SAN(copy_multi_stream_2)
#define bus_space_poke_2 BUS_SAN(poke_2)
#define bus_space_peek_2 BUS_SAN(peek_2)
#define bus_space_read_4 BUS_SAN(read_4)
#define bus_space_read_stream_4 BUS_SAN(read_stream_4)
#define bus_space_read_multi_4 BUS_SAN(read_multi_4)
#define bus_space_read_multi_stream_4 BUS_SAN(read_multi_stream_4)
#define bus_space_read_region_4 BUS_SAN(read_region_4)
#define bus_space_read_region_stream_4 BUS_SAN(read_region_stream_4)
#define bus_space_write_4 BUS_SAN(write_4)
#define bus_space_write_stream_4 BUS_SAN(write_stream_4)
#define bus_space_write_multi_4 BUS_SAN(write_multi_4)
#define bus_space_write_multi_stream_4 BUS_SAN(write_multi_stream_4)
#define bus_space_write_region_4 BUS_SAN(write_region_4)
#define bus_space_write_region_stream_4 BUS_SAN(write_region_stream_4)
#define bus_space_set_multi_4 BUS_SAN(set_multi_4)
#define bus_space_set_multi_stream_4 BUS_SAN(set_multi_stream_4)
#define bus_space_set_region_4 BUS_SAN(set_region_4)
#define bus_space_set_region_stream_4 BUS_SAN(set_region_stream_4)
#define bus_space_copy_multi_4 BUS_SAN(copy_multi_4)
#define bus_space_copy_multi_stream_4 BUS_SAN(copy_multi_stream_4)
#define bus_space_poke_4 BUS_SAN(poke_4)
#define bus_space_peek_4 BUS_SAN(peek_4)
#define bus_space_read_8 BUS_SAN(read_8)
#define bus_space_read_stream_8 BUS_SAN(read_stream_8)
#define bus_space_read_multi_8 BUS_SAN(read_multi_8)
#define bus_space_read_multi_stream_8 BUS_SAN(read_multi_stream_8)
#define bus_space_read_region_8 BUS_SAN(read_region_8)
#define bus_space_read_region_stream_8 BUS_SAN(read_region_stream_8)
#define bus_space_write_8 BUS_SAN(write_8)
#define bus_space_write_stream_8 BUS_SAN(write_stream_8)
#define bus_space_write_multi_8 BUS_SAN(write_multi_8)
#define bus_space_write_multi_stream_8 BUS_SAN(write_multi_stream_8)
#define bus_space_write_region_8 BUS_SAN(write_region_8)
#define bus_space_write_region_stream_8 BUS_SAN(write_region_stream_8)
#define bus_space_set_multi_8 BUS_SAN(set_multi_8)
#define bus_space_set_multi_stream_8 BUS_SAN(set_multi_stream_8)
#define bus_space_set_region_8 BUS_SAN(set_region_8)
#define bus_space_set_region_stream_8 BUS_SAN(set_region_stream_8)
#define bus_space_copy_multi_8 BUS_SAN(copy_multi_8)
#define bus_space_copy_multi_stream_8 BUS_SAN(copy_multi_stream_8)
#define bus_space_poke_8 BUS_SAN(poke_8)
#define bus_space_peek_8 BUS_SAN(peek_8)
#endif /* !KCSAN_RUNTIME */

18
sys/x86/include/bus.h
View File

@ -96,8 +96,8 @@
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef _X86_BUS_H_
#define _X86_BUS_H_
#ifndef _MACHINE_BUS_H_
#define _MACHINE_BUS_H_
#include <machine/_bus.h>
#include <machine/cpufunc.h>
@ -135,7 +135,15 @@
#define BUS_SPACE_BARRIER_READ 0x01 /* force read barrier */
#define BUS_SPACE_BARRIER_WRITE 0x02 /* force write barrier */
#if defined(KCSAN) && !defined(KCSAN_RUNTIME)
#ifndef SAN_RUNTIME
#if defined(KASAN)
#define BUS_SAN_PREFIX kasan
#elif defined(KCSAN)
#define BUS_SAN_PREFIX kcsan
#endif
#endif
#ifdef BUS_SAN_PREFIX
#include <sys/bus_san.h>
#else
@ -1121,6 +1129,6 @@ BUS_POKE_FUNC(4, uint32_t)
BUS_POKE_FUNC(8, uint64_t)
#endif
#endif /* KCSAN && !KCSAN_RUNTIME */
#endif /* !BUS_SAN_PREFIX */
#endif /* _X86_BUS_H_ */
#endif /* !_MACHINE_BUS_H_ */

4
sys/x86/x86/bus_machdep.c
View File

@ -24,7 +24,9 @@
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#define KCSAN_RUNTIME
#if defined(KASAN) || defined(KCSAN)
#define SAN_RUNTIME
#endif
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");