d8da59ddf8
We need to not include the MI _san.h files when builing some parts of the kernel. Fix these checks in the arm64 header files. Sponsored by: The FreeBSD Foundation
670 lines
21 KiB
C
670 lines
21 KiB
C
/*-
|
|
* Copyright (c) 2013 Andrew Turner <andrew@freebsd.org>
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
* SUCH DAMAGE.
|
|
*
|
|
* $FreeBSD$
|
|
*/
|
|
|
|
#ifndef _MACHINE_ATOMIC_H_
|
|
#define _MACHINE_ATOMIC_H_
|
|
|
|
#define isb() __asm __volatile("isb" : : : "memory")
|
|
|
|
/*
|
|
* Options for DMB and DSB:
|
|
* oshld Outer Shareable, load
|
|
* oshst Outer Shareable, store
|
|
* osh Outer Shareable, all
|
|
* nshld Non-shareable, load
|
|
* nshst Non-shareable, store
|
|
* nsh Non-shareable, all
|
|
* ishld Inner Shareable, load
|
|
* ishst Inner Shareable, store
|
|
* ish Inner Shareable, all
|
|
* ld Full system, load
|
|
* st Full system, store
|
|
* sy Full system, all
|
|
*/
|
|
#define dsb(opt) __asm __volatile("dsb " __STRING(opt) : : : "memory")
|
|
#define dmb(opt) __asm __volatile("dmb " __STRING(opt) : : : "memory")
|
|
|
|
#define mb() dmb(sy) /* Full system memory barrier all */
|
|
#define wmb() dmb(st) /* Full system memory barrier store */
|
|
#define rmb() dmb(ld) /* Full system memory barrier load */
|
|
|
|
#if defined(SAN_NEEDS_INTERCEPTORS) && !defined(SAN_RUNTIME)
|
|
#include <sys/atomic_san.h>
|
|
#else
|
|
|
|
#include <sys/atomic_common.h>
|
|
|
|
#ifdef _KERNEL
|
|
extern bool lse_supported;
|
|
|
|
#ifdef LSE_ATOMICS
|
|
#define _ATOMIC_LSE_SUPPORTED 1
|
|
#else
|
|
#define _ATOMIC_LSE_SUPPORTED lse_supported
|
|
#endif
|
|
#else
|
|
#define _ATOMIC_LSE_SUPPORTED 0
|
|
#endif
|
|
|
|
#define _ATOMIC_OP_PROTO(t, op, bar, flav) \
|
|
static __inline void \
|
|
atomic_##op##_##bar##t##flav(volatile uint##t##_t *p, uint##t##_t val)
|
|
|
|
#define _ATOMIC_OP_IMPL(t, w, s, op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \
|
|
_ATOMIC_OP_PROTO(t, op, bar, _llsc) \
|
|
{ \
|
|
uint##t##_t tmp; \
|
|
int res; \
|
|
\
|
|
pre; \
|
|
__asm __volatile( \
|
|
"1: ld"#a"xr"#s" %"#w"0, [%2]\n" \
|
|
" "#llsc_asm_op" %"#w"0, %"#w"0, %"#w"3\n" \
|
|
" st"#l"xr"#s" %w1, %"#w"0, [%2]\n" \
|
|
" cbnz %w1, 1b\n" \
|
|
: "=&r"(tmp), "=&r"(res) \
|
|
: "r" (p), "r" (val) \
|
|
: "memory" \
|
|
); \
|
|
} \
|
|
\
|
|
_ATOMIC_OP_PROTO(t, op, bar, _lse) \
|
|
{ \
|
|
uint##t##_t tmp; \
|
|
\
|
|
pre; \
|
|
__asm __volatile( \
|
|
".arch_extension lse\n" \
|
|
"ld"#lse_asm_op#a#l#s" %"#w"2, %"#w"0, [%1]\n" \
|
|
".arch_extension nolse\n" \
|
|
: "=r" (tmp) \
|
|
: "r" (p), "r" (val) \
|
|
: "memory" \
|
|
); \
|
|
} \
|
|
\
|
|
_ATOMIC_OP_PROTO(t, op, bar, ) \
|
|
{ \
|
|
if (_ATOMIC_LSE_SUPPORTED) \
|
|
atomic_##op##_##bar##t##_lse(p, val); \
|
|
else \
|
|
atomic_##op##_##bar##t##_llsc(p, val); \
|
|
}
|
|
|
|
#define __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \
|
|
_ATOMIC_OP_IMPL(8, w, b, op, llsc_asm_op, lse_asm_op, pre, \
|
|
bar, a, l) \
|
|
_ATOMIC_OP_IMPL(16, w, h, op, llsc_asm_op, lse_asm_op, pre, \
|
|
bar, a, l) \
|
|
_ATOMIC_OP_IMPL(32, w, , op, llsc_asm_op, lse_asm_op, pre, \
|
|
bar, a, l) \
|
|
_ATOMIC_OP_IMPL(64, , , op, llsc_asm_op, lse_asm_op, pre, \
|
|
bar, a, l)
|
|
|
|
#define _ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre) \
|
|
__ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, , , ) \
|
|
__ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, acq_, a, ) \
|
|
__ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, rel_, , l)
|
|
|
|
_ATOMIC_OP(add, add, add, )
|
|
_ATOMIC_OP(clear, bic, clr, )
|
|
_ATOMIC_OP(set, orr, set, )
|
|
_ATOMIC_OP(subtract, add, add, val = -val)
|
|
|
|
#define _ATOMIC_CMPSET_PROTO(t, bar, flav) \
|
|
static __inline int \
|
|
atomic_cmpset_##bar##t##flav(volatile uint##t##_t *p, \
|
|
uint##t##_t cmpval, uint##t##_t newval)
|
|
|
|
#define _ATOMIC_FCMPSET_PROTO(t, bar, flav) \
|
|
static __inline int \
|
|
atomic_fcmpset_##bar##t##flav(volatile uint##t##_t *p, \
|
|
uint##t##_t *cmpval, uint##t##_t newval)
|
|
|
|
#define _ATOMIC_CMPSET_IMPL(t, w, s, bar, a, l) \
|
|
_ATOMIC_CMPSET_PROTO(t, bar, _llsc) \
|
|
{ \
|
|
uint##t##_t tmp; \
|
|
int res; \
|
|
\
|
|
__asm __volatile( \
|
|
"1: mov %w1, #1\n" \
|
|
" ld"#a"xr"#s" %"#w"0, [%2]\n" \
|
|
" cmp %"#w"0, %"#w"3\n" \
|
|
" b.ne 2f\n" \
|
|
" st"#l"xr"#s" %w1, %"#w"4, [%2]\n" \
|
|
" cbnz %w1, 1b\n" \
|
|
"2:" \
|
|
: "=&r"(tmp), "=&r"(res) \
|
|
: "r" (p), "r" (cmpval), "r" (newval) \
|
|
: "cc", "memory" \
|
|
); \
|
|
\
|
|
return (!res); \
|
|
} \
|
|
\
|
|
_ATOMIC_CMPSET_PROTO(t, bar, _lse) \
|
|
{ \
|
|
uint##t##_t oldval; \
|
|
int res; \
|
|
\
|
|
oldval = cmpval; \
|
|
__asm __volatile( \
|
|
".arch_extension lse\n" \
|
|
"cas"#a#l#s" %"#w"1, %"#w"4, [%3]\n" \
|
|
"cmp %"#w"1, %"#w"2\n" \
|
|
"cset %w0, eq\n" \
|
|
".arch_extension nolse\n" \
|
|
: "=r" (res), "+&r" (cmpval) \
|
|
: "r" (oldval), "r" (p), "r" (newval) \
|
|
: "cc", "memory" \
|
|
); \
|
|
\
|
|
return (res); \
|
|
} \
|
|
\
|
|
_ATOMIC_CMPSET_PROTO(t, bar, ) \
|
|
{ \
|
|
if (_ATOMIC_LSE_SUPPORTED) \
|
|
return (atomic_cmpset_##bar##t##_lse(p, cmpval, \
|
|
newval)); \
|
|
else \
|
|
return (atomic_cmpset_##bar##t##_llsc(p, cmpval, \
|
|
newval)); \
|
|
} \
|
|
\
|
|
_ATOMIC_FCMPSET_PROTO(t, bar, _llsc) \
|
|
{ \
|
|
uint##t##_t _cmpval, tmp; \
|
|
int res; \
|
|
\
|
|
_cmpval = *cmpval; \
|
|
__asm __volatile( \
|
|
" mov %w1, #1\n" \
|
|
" ld"#a"xr"#s" %"#w"0, [%2]\n" \
|
|
" cmp %"#w"0, %"#w"3\n" \
|
|
" b.ne 1f\n" \
|
|
" st"#l"xr"#s" %w1, %"#w"4, [%2]\n" \
|
|
"1:" \
|
|
: "=&r"(tmp), "=&r"(res) \
|
|
: "r" (p), "r" (_cmpval), "r" (newval) \
|
|
: "cc", "memory" \
|
|
); \
|
|
*cmpval = tmp; \
|
|
\
|
|
return (!res); \
|
|
} \
|
|
\
|
|
_ATOMIC_FCMPSET_PROTO(t, bar, _lse) \
|
|
{ \
|
|
uint##t##_t _cmpval, tmp; \
|
|
int res; \
|
|
\
|
|
_cmpval = tmp = *cmpval; \
|
|
__asm __volatile( \
|
|
".arch_extension lse\n" \
|
|
"cas"#a#l#s" %"#w"1, %"#w"4, [%3]\n" \
|
|
"cmp %"#w"1, %"#w"2\n" \
|
|
"cset %w0, eq\n" \
|
|
".arch_extension nolse\n" \
|
|
: "=r" (res), "+&r" (tmp) \
|
|
: "r" (_cmpval), "r" (p), "r" (newval) \
|
|
: "cc", "memory" \
|
|
); \
|
|
*cmpval = tmp; \
|
|
\
|
|
return (res); \
|
|
} \
|
|
\
|
|
_ATOMIC_FCMPSET_PROTO(t, bar, ) \
|
|
{ \
|
|
if (_ATOMIC_LSE_SUPPORTED) \
|
|
return (atomic_fcmpset_##bar##t##_lse(p, cmpval, \
|
|
newval)); \
|
|
else \
|
|
return (atomic_fcmpset_##bar##t##_llsc(p, cmpval, \
|
|
newval)); \
|
|
}
|
|
|
|
#define _ATOMIC_CMPSET(bar, a, l) \
|
|
_ATOMIC_CMPSET_IMPL(8, w, b, bar, a, l) \
|
|
_ATOMIC_CMPSET_IMPL(16, w, h, bar, a, l) \
|
|
_ATOMIC_CMPSET_IMPL(32, w, , bar, a, l) \
|
|
_ATOMIC_CMPSET_IMPL(64, , , bar, a, l)
|
|
|
|
#define atomic_cmpset_8 atomic_cmpset_8
|
|
#define atomic_fcmpset_8 atomic_fcmpset_8
|
|
#define atomic_cmpset_16 atomic_cmpset_16
|
|
#define atomic_fcmpset_16 atomic_fcmpset_16
|
|
|
|
_ATOMIC_CMPSET( , , )
|
|
_ATOMIC_CMPSET(acq_, a, )
|
|
_ATOMIC_CMPSET(rel_, ,l)
|
|
|
|
#define _ATOMIC_FETCHADD_PROTO(t, flav) \
|
|
static __inline uint##t##_t \
|
|
atomic_fetchadd_##t##flav(volatile uint##t##_t *p, uint##t##_t val)
|
|
|
|
#define _ATOMIC_FETCHADD_IMPL(t, w) \
|
|
_ATOMIC_FETCHADD_PROTO(t, _llsc) \
|
|
{ \
|
|
uint##t##_t ret, tmp; \
|
|
int res; \
|
|
\
|
|
__asm __volatile( \
|
|
"1: ldxr %"#w"2, [%3]\n" \
|
|
" add %"#w"0, %"#w"2, %"#w"4\n" \
|
|
" stxr %w1, %"#w"0, [%3]\n" \
|
|
" cbnz %w1, 1b\n" \
|
|
: "=&r" (tmp), "=&r" (res), "=&r" (ret) \
|
|
: "r" (p), "r" (val) \
|
|
: "memory" \
|
|
); \
|
|
\
|
|
return (ret); \
|
|
} \
|
|
\
|
|
_ATOMIC_FETCHADD_PROTO(t, _lse) \
|
|
{ \
|
|
uint##t##_t ret; \
|
|
\
|
|
__asm __volatile( \
|
|
".arch_extension lse\n" \
|
|
"ldadd %"#w"2, %"#w"0, [%1]\n" \
|
|
".arch_extension nolse\n" \
|
|
: "=r" (ret) \
|
|
: "r" (p), "r" (val) \
|
|
: "memory" \
|
|
); \
|
|
\
|
|
return (ret); \
|
|
} \
|
|
\
|
|
_ATOMIC_FETCHADD_PROTO(t, ) \
|
|
{ \
|
|
if (_ATOMIC_LSE_SUPPORTED) \
|
|
return (atomic_fetchadd_##t##_lse(p, val)); \
|
|
else \
|
|
return (atomic_fetchadd_##t##_llsc(p, val)); \
|
|
}
|
|
|
|
_ATOMIC_FETCHADD_IMPL(32, w)
|
|
_ATOMIC_FETCHADD_IMPL(64, )
|
|
|
|
#define _ATOMIC_SWAP_PROTO(t, flav) \
|
|
static __inline uint##t##_t \
|
|
atomic_swap_##t##flav(volatile uint##t##_t *p, uint##t##_t val)
|
|
|
|
#define _ATOMIC_READANDCLEAR_PROTO(t, flav) \
|
|
static __inline uint##t##_t \
|
|
atomic_readandclear_##t##flav(volatile uint##t##_t *p)
|
|
|
|
#define _ATOMIC_SWAP_IMPL(t, w, zreg) \
|
|
_ATOMIC_SWAP_PROTO(t, _llsc) \
|
|
{ \
|
|
uint##t##_t ret; \
|
|
int res; \
|
|
\
|
|
__asm __volatile( \
|
|
"1: ldxr %"#w"1, [%2]\n" \
|
|
" stxr %w0, %"#w"3, [%2]\n" \
|
|
" cbnz %w0, 1b\n" \
|
|
: "=&r" (res), "=&r" (ret) \
|
|
: "r" (p), "r" (val) \
|
|
: "memory" \
|
|
); \
|
|
\
|
|
return (ret); \
|
|
} \
|
|
\
|
|
_ATOMIC_SWAP_PROTO(t, _lse) \
|
|
{ \
|
|
uint##t##_t ret; \
|
|
\
|
|
__asm __volatile( \
|
|
".arch_extension lse\n" \
|
|
"swp %"#w"2, %"#w"0, [%1]\n" \
|
|
".arch_extension nolse\n" \
|
|
: "=r" (ret) \
|
|
: "r" (p), "r" (val) \
|
|
: "memory" \
|
|
); \
|
|
\
|
|
return (ret); \
|
|
} \
|
|
\
|
|
_ATOMIC_SWAP_PROTO(t, ) \
|
|
{ \
|
|
if (_ATOMIC_LSE_SUPPORTED) \
|
|
return (atomic_swap_##t##_lse(p, val)); \
|
|
else \
|
|
return (atomic_swap_##t##_llsc(p, val)); \
|
|
} \
|
|
\
|
|
_ATOMIC_READANDCLEAR_PROTO(t, _llsc) \
|
|
{ \
|
|
uint##t##_t ret; \
|
|
int res; \
|
|
\
|
|
__asm __volatile( \
|
|
"1: ldxr %"#w"1, [%2]\n" \
|
|
" stxr %w0, "#zreg", [%2]\n" \
|
|
" cbnz %w0, 1b\n" \
|
|
: "=&r" (res), "=&r" (ret) \
|
|
: "r" (p) \
|
|
: "memory" \
|
|
); \
|
|
\
|
|
return (ret); \
|
|
} \
|
|
\
|
|
_ATOMIC_READANDCLEAR_PROTO(t, _lse) \
|
|
{ \
|
|
return (atomic_swap_##t##_lse(p, 0)); \
|
|
} \
|
|
\
|
|
_ATOMIC_READANDCLEAR_PROTO(t, ) \
|
|
{ \
|
|
if (_ATOMIC_LSE_SUPPORTED) \
|
|
return (atomic_readandclear_##t##_lse(p)); \
|
|
else \
|
|
return (atomic_readandclear_##t##_llsc(p)); \
|
|
}
|
|
|
|
_ATOMIC_SWAP_IMPL(32, w, wzr)
|
|
_ATOMIC_SWAP_IMPL(64, , xzr)
|
|
|
|
#define _ATOMIC_TEST_OP_PROTO(t, op, bar, flav) \
|
|
static __inline int \
|
|
atomic_testand##op##_##bar##t##flav(volatile uint##t##_t *p, u_int val)
|
|
|
|
#define _ATOMIC_TEST_OP_IMPL(t, w, op, llsc_asm_op, lse_asm_op, bar, a) \
|
|
_ATOMIC_TEST_OP_PROTO(t, op, bar, _llsc) \
|
|
{ \
|
|
uint##t##_t mask, old, tmp; \
|
|
int res; \
|
|
\
|
|
mask = ((uint##t##_t)1) << (val & (t - 1)); \
|
|
__asm __volatile( \
|
|
"1: ld"#a"xr %"#w"2, [%3]\n" \
|
|
" "#llsc_asm_op" %"#w"0, %"#w"2, %"#w"4\n" \
|
|
" stxr %w1, %"#w"0, [%3]\n" \
|
|
" cbnz %w1, 1b\n" \
|
|
: "=&r" (tmp), "=&r" (res), "=&r" (old) \
|
|
: "r" (p), "r" (mask) \
|
|
: "memory" \
|
|
); \
|
|
\
|
|
return ((old & mask) != 0); \
|
|
} \
|
|
\
|
|
_ATOMIC_TEST_OP_PROTO(t, op, bar, _lse) \
|
|
{ \
|
|
uint##t##_t mask, old; \
|
|
\
|
|
mask = ((uint##t##_t)1) << (val & (t - 1)); \
|
|
__asm __volatile( \
|
|
".arch_extension lse\n" \
|
|
"ld"#lse_asm_op#a" %"#w"2, %"#w"0, [%1]\n" \
|
|
".arch_extension nolse\n" \
|
|
: "=r" (old) \
|
|
: "r" (p), "r" (mask) \
|
|
: "memory" \
|
|
); \
|
|
\
|
|
return ((old & mask) != 0); \
|
|
} \
|
|
\
|
|
_ATOMIC_TEST_OP_PROTO(t, op, bar, ) \
|
|
{ \
|
|
if (_ATOMIC_LSE_SUPPORTED) \
|
|
return (atomic_testand##op##_##bar##t##_lse(p, val)); \
|
|
else \
|
|
return (atomic_testand##op##_##bar##t##_llsc(p, val)); \
|
|
}
|
|
|
|
#define _ATOMIC_TEST_OP(op, llsc_asm_op, lse_asm_op) \
|
|
_ATOMIC_TEST_OP_IMPL(32, w, op, llsc_asm_op, lse_asm_op, , ) \
|
|
_ATOMIC_TEST_OP_IMPL(32, w, op, llsc_asm_op, lse_asm_op, acq_, a) \
|
|
_ATOMIC_TEST_OP_IMPL(64, , op, llsc_asm_op, lse_asm_op, , ) \
|
|
_ATOMIC_TEST_OP_IMPL(64, , op, llsc_asm_op, lse_asm_op, acq_, a)
|
|
|
|
_ATOMIC_TEST_OP(clear, bic, clr)
|
|
_ATOMIC_TEST_OP(set, orr, set)
|
|
|
|
#define _ATOMIC_LOAD_ACQ_IMPL(t, w, s) \
|
|
static __inline uint##t##_t \
|
|
atomic_load_acq_##t(volatile uint##t##_t *p) \
|
|
{ \
|
|
uint##t##_t ret; \
|
|
\
|
|
__asm __volatile( \
|
|
"ldar"#s" %"#w"0, [%1]\n" \
|
|
: "=&r" (ret) \
|
|
: "r" (p) \
|
|
: "memory"); \
|
|
\
|
|
return (ret); \
|
|
}
|
|
|
|
#define atomic_load_acq_8 atomic_load_acq_8
|
|
#define atomic_load_acq_16 atomic_load_acq_16
|
|
_ATOMIC_LOAD_ACQ_IMPL(8, w, b)
|
|
_ATOMIC_LOAD_ACQ_IMPL(16, w, h)
|
|
_ATOMIC_LOAD_ACQ_IMPL(32, w, )
|
|
_ATOMIC_LOAD_ACQ_IMPL(64, , )
|
|
|
|
#define _ATOMIC_STORE_REL_IMPL(t, w, s) \
|
|
static __inline void \
|
|
atomic_store_rel_##t(volatile uint##t##_t *p, uint##t##_t val) \
|
|
{ \
|
|
__asm __volatile( \
|
|
"stlr"#s" %"#w"0, [%1]\n" \
|
|
: \
|
|
: "r" (val), "r" (p) \
|
|
: "memory"); \
|
|
}
|
|
|
|
_ATOMIC_STORE_REL_IMPL(8, w, b)
|
|
_ATOMIC_STORE_REL_IMPL(16, w, h)
|
|
_ATOMIC_STORE_REL_IMPL(32, w, )
|
|
_ATOMIC_STORE_REL_IMPL(64, , )
|
|
|
|
#define atomic_add_char atomic_add_8
|
|
#define atomic_fcmpset_char atomic_fcmpset_8
|
|
#define atomic_clear_char atomic_clear_8
|
|
#define atomic_cmpset_char atomic_cmpset_8
|
|
#define atomic_fetchadd_char atomic_fetchadd_8
|
|
#define atomic_readandclear_char atomic_readandclear_8
|
|
#define atomic_set_char atomic_set_8
|
|
#define atomic_swap_char atomic_swap_8
|
|
#define atomic_subtract_char atomic_subtract_8
|
|
#define atomic_testandclear_char atomic_testandclear_8
|
|
#define atomic_testandset_char atomic_testandset_8
|
|
|
|
#define atomic_add_acq_char atomic_add_acq_8
|
|
#define atomic_fcmpset_acq_char atomic_fcmpset_acq_8
|
|
#define atomic_clear_acq_char atomic_clear_acq_8
|
|
#define atomic_cmpset_acq_char atomic_cmpset_acq_8
|
|
#define atomic_load_acq_char atomic_load_acq_8
|
|
#define atomic_set_acq_char atomic_set_acq_8
|
|
#define atomic_subtract_acq_char atomic_subtract_acq_8
|
|
#define atomic_testandset_acq_char atomic_testandset_acq_8
|
|
|
|
#define atomic_add_rel_char atomic_add_rel_8
|
|
#define atomic_fcmpset_rel_char atomic_fcmpset_rel_8
|
|
#define atomic_clear_rel_char atomic_clear_rel_8
|
|
#define atomic_cmpset_rel_char atomic_cmpset_rel_8
|
|
#define atomic_set_rel_char atomic_set_rel_8
|
|
#define atomic_subtract_rel_char atomic_subtract_rel_8
|
|
#define atomic_store_rel_char atomic_store_rel_8
|
|
|
|
#define atomic_add_short atomic_add_16
|
|
#define atomic_fcmpset_short atomic_fcmpset_16
|
|
#define atomic_clear_short atomic_clear_16
|
|
#define atomic_cmpset_short atomic_cmpset_16
|
|
#define atomic_fetchadd_short atomic_fetchadd_16
|
|
#define atomic_readandclear_short atomic_readandclear_16
|
|
#define atomic_set_short atomic_set_16
|
|
#define atomic_swap_short atomic_swap_16
|
|
#define atomic_subtract_short atomic_subtract_16
|
|
#define atomic_testandclear_short atomic_testandclear_16
|
|
#define atomic_testandset_short atomic_testandset_16
|
|
|
|
#define atomic_add_acq_short atomic_add_acq_16
|
|
#define atomic_fcmpset_acq_short atomic_fcmpset_acq_16
|
|
#define atomic_clear_acq_short atomic_clear_acq_16
|
|
#define atomic_cmpset_acq_short atomic_cmpset_acq_16
|
|
#define atomic_load_acq_short atomic_load_acq_16
|
|
#define atomic_set_acq_short atomic_set_acq_16
|
|
#define atomic_subtract_acq_short atomic_subtract_acq_16
|
|
#define atomic_testandset_acq_short atomic_testandset_acq_16
|
|
|
|
#define atomic_add_rel_short atomic_add_rel_16
|
|
#define atomic_fcmpset_rel_short atomic_fcmpset_rel_16
|
|
#define atomic_clear_rel_short atomic_clear_rel_16
|
|
#define atomic_cmpset_rel_short atomic_cmpset_rel_16
|
|
#define atomic_set_rel_short atomic_set_rel_16
|
|
#define atomic_subtract_rel_short atomic_subtract_rel_16
|
|
#define atomic_store_rel_short atomic_store_rel_16
|
|
|
|
#define atomic_add_int atomic_add_32
|
|
#define atomic_fcmpset_int atomic_fcmpset_32
|
|
#define atomic_clear_int atomic_clear_32
|
|
#define atomic_cmpset_int atomic_cmpset_32
|
|
#define atomic_fetchadd_int atomic_fetchadd_32
|
|
#define atomic_readandclear_int atomic_readandclear_32
|
|
#define atomic_set_int atomic_set_32
|
|
#define atomic_swap_int atomic_swap_32
|
|
#define atomic_subtract_int atomic_subtract_32
|
|
#define atomic_testandclear_int atomic_testandclear_32
|
|
#define atomic_testandset_int atomic_testandset_32
|
|
|
|
#define atomic_add_acq_int atomic_add_acq_32
|
|
#define atomic_fcmpset_acq_int atomic_fcmpset_acq_32
|
|
#define atomic_clear_acq_int atomic_clear_acq_32
|
|
#define atomic_cmpset_acq_int atomic_cmpset_acq_32
|
|
#define atomic_load_acq_int atomic_load_acq_32
|
|
#define atomic_set_acq_int atomic_set_acq_32
|
|
#define atomic_subtract_acq_int atomic_subtract_acq_32
|
|
#define atomic_testandset_acq_int atomic_testandset_acq_32
|
|
|
|
#define atomic_add_rel_int atomic_add_rel_32
|
|
#define atomic_fcmpset_rel_int atomic_fcmpset_rel_32
|
|
#define atomic_clear_rel_int atomic_clear_rel_32
|
|
#define atomic_cmpset_rel_int atomic_cmpset_rel_32
|
|
#define atomic_set_rel_int atomic_set_rel_32
|
|
#define atomic_subtract_rel_int atomic_subtract_rel_32
|
|
#define atomic_store_rel_int atomic_store_rel_32
|
|
|
|
#define atomic_add_long atomic_add_64
|
|
#define atomic_fcmpset_long atomic_fcmpset_64
|
|
#define atomic_clear_long atomic_clear_64
|
|
#define atomic_cmpset_long atomic_cmpset_64
|
|
#define atomic_fetchadd_long atomic_fetchadd_64
|
|
#define atomic_readandclear_long atomic_readandclear_64
|
|
#define atomic_set_long atomic_set_64
|
|
#define atomic_swap_long atomic_swap_64
|
|
#define atomic_subtract_long atomic_subtract_64
|
|
#define atomic_testandclear_long atomic_testandclear_64
|
|
#define atomic_testandset_long atomic_testandset_64
|
|
|
|
#define atomic_add_ptr atomic_add_64
|
|
#define atomic_fcmpset_ptr atomic_fcmpset_64
|
|
#define atomic_clear_ptr atomic_clear_64
|
|
#define atomic_cmpset_ptr atomic_cmpset_64
|
|
#define atomic_fetchadd_ptr atomic_fetchadd_64
|
|
#define atomic_readandclear_ptr atomic_readandclear_64
|
|
#define atomic_set_ptr atomic_set_64
|
|
#define atomic_swap_ptr atomic_swap_64
|
|
#define atomic_subtract_ptr atomic_subtract_64
|
|
|
|
#define atomic_add_acq_long atomic_add_acq_64
|
|
#define atomic_fcmpset_acq_long atomic_fcmpset_acq_64
|
|
#define atomic_clear_acq_long atomic_clear_acq_64
|
|
#define atomic_cmpset_acq_long atomic_cmpset_acq_64
|
|
#define atomic_load_acq_long atomic_load_acq_64
|
|
#define atomic_set_acq_long atomic_set_acq_64
|
|
#define atomic_subtract_acq_long atomic_subtract_acq_64
|
|
#define atomic_testandset_acq_long atomic_testandset_acq_64
|
|
|
|
#define atomic_add_acq_ptr atomic_add_acq_64
|
|
#define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64
|
|
#define atomic_clear_acq_ptr atomic_clear_acq_64
|
|
#define atomic_cmpset_acq_ptr atomic_cmpset_acq_64
|
|
#define atomic_load_acq_ptr atomic_load_acq_64
|
|
#define atomic_set_acq_ptr atomic_set_acq_64
|
|
#define atomic_subtract_acq_ptr atomic_subtract_acq_64
|
|
|
|
#define atomic_add_rel_long atomic_add_rel_64
|
|
#define atomic_fcmpset_rel_long atomic_fcmpset_rel_64
|
|
#define atomic_clear_rel_long atomic_clear_rel_64
|
|
#define atomic_cmpset_rel_long atomic_cmpset_rel_64
|
|
#define atomic_set_rel_long atomic_set_rel_64
|
|
#define atomic_subtract_rel_long atomic_subtract_rel_64
|
|
#define atomic_store_rel_long atomic_store_rel_64
|
|
|
|
#define atomic_add_rel_ptr atomic_add_rel_64
|
|
#define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64
|
|
#define atomic_clear_rel_ptr atomic_clear_rel_64
|
|
#define atomic_cmpset_rel_ptr atomic_cmpset_rel_64
|
|
#define atomic_set_rel_ptr atomic_set_rel_64
|
|
#define atomic_subtract_rel_ptr atomic_subtract_rel_64
|
|
#define atomic_store_rel_ptr atomic_store_rel_64
|
|
|
|
static __inline void
|
|
atomic_thread_fence_acq(void)
|
|
{
|
|
|
|
dmb(ld);
|
|
}
|
|
|
|
static __inline void
|
|
atomic_thread_fence_rel(void)
|
|
{
|
|
|
|
dmb(sy);
|
|
}
|
|
|
|
static __inline void
|
|
atomic_thread_fence_acq_rel(void)
|
|
{
|
|
|
|
dmb(sy);
|
|
}
|
|
|
|
static __inline void
|
|
atomic_thread_fence_seq_cst(void)
|
|
{
|
|
|
|
dmb(sy);
|
|
}
|
|
|
|
#endif /* KCSAN && !KCSAN_RUNTIME */
|
|
#endif /* _MACHINE_ATOMIC_H_ */
|