Expand generic subword atomic primitives
The goal of this change is to make the atomic_load_acq_{8,16},
atomic_testandset{,_acq}_long, and atomic_testandclear_long primitives
available in MI-namespace.
The second goal is to get this draft out of my local tree, as anything that
requires a full tinderbox is a big burden out of tree. MD specifics can be
refined individually afterwards.
The generic implementations may not be ideal for your architecture; feel
free to implement better versions. If no subword_atomic definitions are
needed, the include can be removed from your arch's machine/atomic.h.
Generic definitions are guarded by defined macros of the same name. To
avoid picking up conflicting generic definitions, some macro defines are
added to various MD machine/atomic.h to register an existing implementation.
Include _atomic_subword.h in arm and arm64 machine/atomic.h.
For some odd reason, KCSAN only generates some versions of primitives.
Generate the _acq variants of atomic_load.*_8, atomic_load.*_16, and
atomic_testandset.*_long. There are other questionably disabled primitives,
but I didn't run into them, so I left them alone. KCSAN is only built for
amd64 in tinderbox for now.
Add atomic_subword implementations of atomic_load_acq_{8,16} implemented
using masking and atomic_load_acq_32.
Add generic atomic_subword implementations of atomic_testandset_long(),
atomic_testandclear_long(), and atomic_testandset_acq_long(), using
atomic_fcmpset_long() and atomic_fcmpset_acq_long().
On x86, add atomic_testandset_acq_long as an alias for
atomic_testandset_long.
Reviewed by: kevans, rlibby (previous versions both)
Differential Revision: https://reviews.freebsd.org/D22963
This commit is contained in:
Conrad Meyer
parent
9b2877353b
commit
ca0ec73c11
@ -553,6 +553,7 @@ u_long atomic_swap_long(volatile u_long *p, u_long v);
|
||||
|
||||
#define atomic_readandclear_int(p) atomic_swap_int(p, 0)
|
||||
#define atomic_readandclear_long(p) atomic_swap_long(p, 0)
|
||||
#define atomic_testandset_acq_long atomic_testandset_long
|
||||
|
||||
/* Operations on 8-bit bytes. */
|
||||
#define atomic_set_8 atomic_set_char
|
||||
|
||||
@ -521,8 +521,10 @@ atomic_swap_32(volatile u_int32_t *p, u_int32_t v)
|
||||
#define atomic_fcmpset_rel_32 atomic_fcmpset_32
|
||||
#define atomic_fcmpset_acq_32 atomic_fcmpset_32
|
||||
#ifdef _KERNEL
|
||||
#define atomic_fcmpset_8 atomic_fcmpset_8
|
||||
#define atomic_fcmpset_rel_8 atomic_fcmpset_8
|
||||
#define atomic_fcmpset_acq_8 atomic_fcmpset_8
|
||||
#define atomic_fcmpset_16 atomic_fcmpset_16
|
||||
#define atomic_fcmpset_rel_16 atomic_fcmpset_16
|
||||
#define atomic_fcmpset_acq_16 atomic_fcmpset_16
|
||||
#define atomic_fcmpset_rel_64 atomic_fcmpset_64
|
||||
@ -533,8 +535,10 @@ atomic_swap_32(volatile u_int32_t *p, u_int32_t v)
|
||||
#define atomic_cmpset_rel_32 atomic_cmpset_32
|
||||
#define atomic_cmpset_acq_32 atomic_cmpset_32
|
||||
#ifdef _KERNEL
|
||||
#define atomic_cmpset_8 atomic_cmpset_8
|
||||
#define atomic_cmpset_rel_8 atomic_cmpset_8
|
||||
#define atomic_cmpset_acq_8 atomic_cmpset_8
|
||||
#define atomic_cmpset_16 atomic_cmpset_16
|
||||
#define atomic_cmpset_rel_16 atomic_cmpset_16
|
||||
#define atomic_cmpset_acq_16 atomic_cmpset_16
|
||||
#define atomic_cmpset_rel_64 atomic_cmpset_64
|
||||
|
||||
@ -245,6 +245,7 @@ atomic_fcmpset_8(volatile uint8_t *_ptr, uint8_t *_old, uint8_t _new)
|
||||
ATOMIC_FCMPSET_CODE(ret, uint8_t, "b");
|
||||
return (ret);
|
||||
}
|
||||
#define atomic_fcmpset_8 atomic_fcmpset_8
|
||||
|
||||
static __inline int
|
||||
atomic_fcmpset_acq_8(volatile uint8_t *_ptr, uint8_t *_old, uint8_t _new)
|
||||
@ -274,6 +275,7 @@ atomic_fcmpset_16(volatile uint16_t *_ptr, uint16_t *_old, uint16_t _new)
|
||||
ATOMIC_FCMPSET_CODE(ret, uint16_t, "h");
|
||||
return (ret);
|
||||
}
|
||||
#define atomic_fcmpset_16 atomic_fcmpset_16
|
||||
|
||||
static __inline int
|
||||
atomic_fcmpset_acq_16(volatile uint16_t *_ptr, uint16_t *_old, uint16_t _new)
|
||||
@ -429,6 +431,7 @@ atomic_cmpset_8(volatile uint8_t *_ptr, uint8_t _old, uint8_t _new)
|
||||
ATOMIC_CMPSET_CODE(ret, "b");
|
||||
return (ret);
|
||||
}
|
||||
#define atomic_cmpset_8 atomic_cmpset_8
|
||||
|
||||
static __inline int
|
||||
atomic_cmpset_acq_8(volatile uint8_t *_ptr, uint8_t _old, uint8_t _new)
|
||||
@ -458,6 +461,7 @@ atomic_cmpset_16(volatile uint16_t *_ptr, uint16_t _old, uint16_t _new)
|
||||
ATOMIC_CMPSET_CODE(ret, "h");
|
||||
return (ret);
|
||||
}
|
||||
#define atomic_cmpset_16 atomic_cmpset_16
|
||||
|
||||
static __inline int
|
||||
atomic_cmpset_acq_16(volatile uint16_t *_ptr, uint16_t _old, uint16_t _new)
|
||||
@ -899,6 +903,7 @@ atomic_testandclear_long(volatile u_long *p, u_int v)
|
||||
|
||||
return (atomic_testandclear_32((volatile uint32_t *)p, v));
|
||||
}
|
||||
#define atomic_testandclear_long atomic_testandclear_long
|
||||
|
||||
static __inline int
|
||||
atomic_testandset_32(volatile uint32_t *ptr, u_int bit)
|
||||
@ -942,6 +947,7 @@ atomic_testandset_long(volatile u_long *p, u_int v)
|
||||
|
||||
return (atomic_testandset_32((volatile uint32_t *)p, v));
|
||||
}
|
||||
#define atomic_testandset_long atomic_testandset_long
|
||||
|
||||
static __inline int
|
||||
atomic_testandset_64(volatile uint64_t *p, u_int v)
|
||||
|
||||
@ -103,4 +103,6 @@ atomic_swap_long(volatile u_long *p, u_long v)
|
||||
#define atomic_store_rel_int atomic_store_rel_32
|
||||
#define atomic_swap_int atomic_swap_32
|
||||
|
||||
#include <sys/_atomic_subword.h>
|
||||
|
||||
#endif /* _MACHINE_ATOMIC_H_ */
|
||||
|
||||
@ -257,6 +257,11 @@ _ATOMIC_FCMPSET_PROTO(t, bar, ) \
|
||||
_ATOMIC_CMPSET_IMPL(32, w, , bar, a, l) \
|
||||
_ATOMIC_CMPSET_IMPL(64, , , bar, a, l)
|
||||
|
||||
#define atomic_cmpset_8 atomic_cmpset_8
|
||||
#define atomic_fcmpset_8 atomic_fcmpset_8
|
||||
#define atomic_cmpset_16 atomic_cmpset_16
|
||||
#define atomic_fcmpset_16 atomic_fcmpset_16
|
||||
|
||||
_ATOMIC_CMPSET( , , )
|
||||
_ATOMIC_CMPSET(acq_, a, )
|
||||
_ATOMIC_CMPSET(rel_, ,l)
|
||||
@ -465,6 +470,8 @@ atomic_load_acq_##t(volatile uint##t##_t *p) \
|
||||
return (ret); \
|
||||
}
|
||||
|
||||
#define atomic_load_acq_8 atomic_load_acq_8
|
||||
#define atomic_load_acq_16 atomic_load_acq_16
|
||||
_ATOMIC_LOAD_ACQ_IMPL(8, w, b)
|
||||
_ATOMIC_LOAD_ACQ_IMPL(16, w, h)
|
||||
_ATOMIC_LOAD_ACQ_IMPL(32, w, )
|
||||
@ -596,5 +603,7 @@ atomic_thread_fence_seq_cst(void)
|
||||
dmb(sy);
|
||||
}
|
||||
|
||||
#include <sys/_atomic_subword.h>
|
||||
|
||||
#endif /* KCSAN && !KCSAN_RUNTIME */
|
||||
#endif /* _MACHINE_ATOMIC_H_ */
|
||||
|
||||
@ -808,6 +808,7 @@ u_long atomic_swap_long(volatile u_long *p, u_long v);
|
||||
|
||||
#define atomic_readandclear_int(p) atomic_swap_int(p, 0)
|
||||
#define atomic_readandclear_long(p) atomic_swap_long(p, 0)
|
||||
#define atomic_testandset_acq_long atomic_testandset_long
|
||||
|
||||
/* Operations on 8-bit bytes. */
|
||||
#define atomic_set_8 atomic_set_char
|
||||
|
||||
@ -538,7 +538,7 @@ CSAN_ATOMIC_FUNC_ADD(8, uint8_t)
|
||||
CSAN_ATOMIC_FUNC_CLEAR(8, uint8_t)
|
||||
CSAN_ATOMIC_FUNC_CMPSET(8, uint8_t)
|
||||
CSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t)
|
||||
_CSAN_ATOMIC_FUNC_LOAD(8, uint8_t)
|
||||
CSAN_ATOMIC_FUNC_LOAD(8, uint8_t)
|
||||
CSAN_ATOMIC_FUNC_SET(8, uint8_t)
|
||||
CSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t)
|
||||
_CSAN_ATOMIC_FUNC_STORE(8, uint8_t)
|
||||
@ -554,11 +554,7 @@ CSAN_ATOMIC_FUNC_ADD(16, uint16_t)
|
||||
CSAN_ATOMIC_FUNC_CLEAR(16, uint16_t)
|
||||
CSAN_ATOMIC_FUNC_CMPSET(16, uint16_t)
|
||||
CSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t)
|
||||
#if defined(__aarch64__)
|
||||
_CSAN_ATOMIC_FUNC_LOAD(16, uint16_t)
|
||||
#else
|
||||
CSAN_ATOMIC_FUNC_LOAD(16, uint16_t)
|
||||
#endif
|
||||
CSAN_ATOMIC_FUNC_SET(16, uint16_t)
|
||||
CSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t)
|
||||
_CSAN_ATOMIC_FUNC_STORE(16, uint16_t)
|
||||
@ -632,6 +628,7 @@ CSAN_ATOMIC_FUNC_SWAP(long, u_long)
|
||||
#if !defined(__aarch64__)
|
||||
CSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long)
|
||||
CSAN_ATOMIC_FUNC_TESTANDSET(long, u_long)
|
||||
CSAN_ATOMIC_FUNC_TESTANDSET(acq_long, u_long)
|
||||
#endif
|
||||
|
||||
CSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t)
|
||||
|
||||
@ -727,11 +727,15 @@ ATOMIC_CMPSET_ACQ_REL(int);
|
||||
ATOMIC_CMPSET_ACQ_REL(long);
|
||||
|
||||
|
||||
#ifdef ISA_206_ATOMICS
|
||||
#define atomic_cmpset_8 atomic_cmpset_char
|
||||
#endif
|
||||
#define atomic_cmpset_acq_8 atomic_cmpset_acq_char
|
||||
#define atomic_cmpset_rel_8 atomic_cmpset_rel_char
|
||||
|
||||
#ifdef ISA_206_ATOMICS
|
||||
#define atomic_cmpset_16 atomic_cmpset_short
|
||||
#endif
|
||||
#define atomic_cmpset_acq_16 atomic_cmpset_acq_short
|
||||
#define atomic_cmpset_rel_16 atomic_cmpset_rel_short
|
||||
|
||||
@ -894,11 +898,15 @@ atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval)
|
||||
ATOMIC_FCMPSET_ACQ_REL(int);
|
||||
ATOMIC_FCMPSET_ACQ_REL(long);
|
||||
|
||||
#ifdef ISA_206_ATOMICS
|
||||
#define atomic_fcmpset_8 atomic_fcmpset_char
|
||||
#endif
|
||||
#define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char
|
||||
#define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char
|
||||
|
||||
#ifdef ISA_206_ATOMICS
|
||||
#define atomic_fcmpset_16 atomic_fcmpset_short
|
||||
#endif
|
||||
#define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short
|
||||
#define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short
|
||||
|
||||
@ -1018,6 +1026,10 @@ atomic_thread_fence_seq_cst(void)
|
||||
|
||||
#ifndef ISA_206_ATOMICS
|
||||
#include <sys/_atomic_subword.h>
|
||||
#define atomic_cmpset_char atomic_cmpset_8
|
||||
#define atomic_cmpset_short atomic_cmpset_16
|
||||
#define atomic_fcmpset_char atomic_fcmpset_8
|
||||
#define atomic_fcmpset_short atomic_fcmpset_16
|
||||
#endif
|
||||
|
||||
/* These need sys/_atomic_subword.h on non-ISA-2.06-atomic platforms. */
|
||||
|
||||
@ -41,6 +41,9 @@
|
||||
#endif
|
||||
|
||||
#include <machine/endian.h>
|
||||
#ifndef _KERNEL
|
||||
#include <stdbool.h>
|
||||
#endif
|
||||
|
||||
#ifndef NBBY
|
||||
#define NBBY 8
|
||||
@ -113,6 +116,7 @@ _atomic_fcmpset_masked_word(uint32_t *addr, uint32_t *old, uint32_t val,
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef atomic_cmpset_8
|
||||
static __inline int
|
||||
atomic_cmpset_8(__volatile uint8_t *addr, uint8_t old, uint8_t val)
|
||||
{
|
||||
@ -123,7 +127,9 @@ atomic_cmpset_8(__volatile uint8_t *addr, uint8_t old, uint8_t val)
|
||||
return (_atomic_cmpset_masked_word(_ATOMIC_WORD_ALIGNED(addr),
|
||||
old << shift, val << shift, 0xff << shift));
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef atomic_fcmpset_8
|
||||
static __inline int
|
||||
atomic_fcmpset_8(__volatile uint8_t *addr, uint8_t *old, uint8_t val)
|
||||
{
|
||||
@ -138,7 +144,9 @@ atomic_fcmpset_8(__volatile uint8_t *addr, uint8_t *old, uint8_t val)
|
||||
*old = (wold >> shift) & 0xff;
|
||||
return (ret);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef atomic_cmpset_16
|
||||
static __inline int
|
||||
atomic_cmpset_16(__volatile uint16_t *addr, uint16_t old, uint16_t val)
|
||||
{
|
||||
@ -149,7 +157,9 @@ atomic_cmpset_16(__volatile uint16_t *addr, uint16_t old, uint16_t val)
|
||||
return (_atomic_cmpset_masked_word(_ATOMIC_WORD_ALIGNED(addr),
|
||||
old << shift, val << shift, 0xffff << shift));
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef atomic_fcmpset_16
|
||||
static __inline int
|
||||
atomic_fcmpset_16(__volatile uint16_t *addr, uint16_t *old, uint16_t val)
|
||||
{
|
||||
@ -164,9 +174,101 @@ atomic_fcmpset_16(__volatile uint16_t *addr, uint16_t *old, uint16_t val)
|
||||
*old = (wold >> shift) & 0xffff;
|
||||
return (ret);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef atomic_load_acq_8
|
||||
static __inline uint8_t
|
||||
atomic_load_acq_8(volatile uint8_t *p)
|
||||
{
|
||||
int shift;
|
||||
uint8_t ret;
|
||||
|
||||
shift = _ATOMIC_BYTE_SHIFT(p);
|
||||
ret = (atomic_load_acq_32(_ATOMIC_WORD_ALIGNED(p)) >> shift) & 0xff;
|
||||
return (ret);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef atomic_load_acq_16
|
||||
static __inline uint16_t
|
||||
atomic_load_acq_16(volatile uint16_t *p)
|
||||
{
|
||||
int shift;
|
||||
uint16_t ret;
|
||||
|
||||
shift = _ATOMIC_HWORD_SHIFT(p);
|
||||
ret = (atomic_load_acq_32(_ATOMIC_WORD_ALIGNED(p)) >> shift) &
|
||||
0xffff;
|
||||
return (ret);
|
||||
}
|
||||
#endif
|
||||
|
||||
#undef _ATOMIC_WORD_ALIGNED
|
||||
#undef _ATOMIC_BYTE_SHIFT
|
||||
#undef _ATOMIC_HWORD_SHIFT
|
||||
|
||||
/*
|
||||
* Provide generic testandset_long implementation based on fcmpset long
|
||||
* primitive. It may not be ideal for any given arch, so machine/atomic.h
|
||||
* should define the macro atomic_testandset_long to override with an
|
||||
* MD-specific version.
|
||||
*
|
||||
* (Organizationally, this isn't really subword atomics. But atomic_common is
|
||||
* included too early in machine/atomic.h, so it isn't a good place for derived
|
||||
* primitives like this.)
|
||||
*/
|
||||
#ifndef atomic_testandset_acq_long
|
||||
static __inline int
|
||||
atomic_testandset_acq_long(volatile u_long *p, u_int v)
|
||||
{
|
||||
u_long bit, old;
|
||||
bool ret;
|
||||
|
||||
bit = (1ul << (v % (sizeof(*p) * NBBY)));
|
||||
|
||||
old = atomic_load_acq_long(p);
|
||||
ret = false;
|
||||
while (!ret && (old & bit) == 0)
|
||||
ret = atomic_fcmpset_acq_long(p, &old, old | bit);
|
||||
|
||||
return (!ret);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef atomic_testandset_long
|
||||
static __inline int
|
||||
atomic_testandset_long(volatile u_long *p, u_int v)
|
||||
{
|
||||
u_long bit, old;
|
||||
bool ret;
|
||||
|
||||
bit = (1ul << (v % (sizeof(*p) * NBBY)));
|
||||
|
||||
old = atomic_load_long(p);
|
||||
ret = false;
|
||||
while (!ret && (old & bit) == 0)
|
||||
ret = atomic_fcmpset_long(p, &old, old | bit);
|
||||
|
||||
return (!ret);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef atomic_testandclear_long
|
||||
static __inline int
|
||||
atomic_testandclear_long(volatile u_long *p, u_int v)
|
||||
{
|
||||
u_long bit, old;
|
||||
bool ret;
|
||||
|
||||
bit = (1ul << (v % (sizeof(*p) * NBBY)));
|
||||
|
||||
old = atomic_load_long(p);
|
||||
ret = false;
|
||||
while (!ret && (old & bit) != 0)
|
||||
ret = atomic_fcmpset_long(p, &old, old & ~bit);
|
||||
|
||||
return (ret);
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* _SYS__ATOMIC_SUBWORD_H_ */
|
||||
|
||||
@ -69,7 +69,8 @@
|
||||
void kcsan_atomic_store_rel_##name(volatile type *, type)
|
||||
|
||||
#define KCSAN_ATOMIC_TEST(op, name, type) \
|
||||
int kcsan_atomic_##op##_##name(volatile type *, u_int)
|
||||
int kcsan_atomic_##op##_##name(volatile type *, u_int); \
|
||||
int kcsan_atomic_##op##_acq_##name(volatile type *, u_int)
|
||||
|
||||
#define KCSAN_ATOMIC_FUNCS(name, type) \
|
||||
KCSAN_ATOMIC_FUNC_1(add, name, type); \
|
||||
@ -156,6 +157,7 @@ void kcsan_atomic_thread_fence_seq_cst(void);
|
||||
#define atomic_swap_long kcsan_atomic_swap_long
|
||||
#define atomic_testandclear_long kcsan_atomic_testandclear_long
|
||||
#define atomic_testandset_long kcsan_atomic_testandset_long
|
||||
#define atomic_testandset_acq_long kcsan_atomic_testandset_acq_long
|
||||
|
||||
#define atomic_add_ptr kcsan_atomic_add_ptr
|
||||
#define atomic_add_acq_ptr kcsan_atomic_add_acq_ptr
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user