Add the atomic_thread_fence() family of functions with intent to
provide a semantic defined by the C11 fences with corresponding memory_order. atomic_thread_fence_acq() gives r | r, w, where r and w are read and write accesses, and | denotes the fence itself. atomic_thread_fence_rel() is r, w | w. atomic_thread_fence_acq_rel() is the combination of the acquire and release in single operation. Note that reads after the acq+rel fence could be made visible before writes preceeding the fence. atomic_thread_fence_seq_cst() orders all accesses before/after the fence, and the fence itself is globally ordered against other sequentially consistent atomic operations. Reviewed by: alc Discussed with: bde Sponsored by: The FreeBSD Foundation MFC after: 3 weeks
This commit is contained in:
parent
22cf98d1f3
commit
8954a9a4e6
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=285283
@ -84,6 +84,10 @@ u_int atomic_fetchadd_int(volatile u_int *p, u_int v);
|
||||
u_long atomic_fetchadd_long(volatile u_long *p, u_long v);
|
||||
int atomic_testandset_int(volatile u_int *p, u_int v);
|
||||
int atomic_testandset_long(volatile u_long *p, u_int v);
|
||||
void atomic_thread_fence_acq(void);
|
||||
void atomic_thread_fence_acq_rel(void);
|
||||
void atomic_thread_fence_rel(void);
|
||||
void atomic_thread_fence_seq_cst(void);
|
||||
|
||||
#define ATOMIC_LOAD(TYPE) \
|
||||
u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p)
|
||||
@ -328,6 +332,34 @@ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \
|
||||
} \
|
||||
struct __hack
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_acq(void)
|
||||
{
|
||||
|
||||
__compiler_membar();
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_rel(void)
|
||||
{
|
||||
|
||||
__compiler_membar();
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_acq_rel(void)
|
||||
{
|
||||
|
||||
__compiler_membar();
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_seq_cst(void)
|
||||
{
|
||||
|
||||
__storeload_barrier();
|
||||
}
|
||||
|
||||
#endif /* KLD_MODULE || !__GNUCLIKE_ASM */
|
||||
|
||||
ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v);
|
||||
|
@ -1103,6 +1103,34 @@ atomic_store_long(volatile u_long *dst, u_long src)
|
||||
*dst = src;
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_acq(void)
|
||||
{
|
||||
|
||||
dmb();
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_rel(void)
|
||||
{
|
||||
|
||||
dmb();
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_acq_rel(void)
|
||||
{
|
||||
|
||||
dmb();
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_seq_cst(void)
|
||||
{
|
||||
|
||||
dmb();
|
||||
}
|
||||
|
||||
#define atomic_clear_ptr atomic_clear_32
|
||||
#define atomic_set_ptr atomic_set_32
|
||||
#define atomic_cmpset_ptr atomic_cmpset_32
|
||||
|
@ -728,6 +728,34 @@ atomic_subtract_rel_64(volatile uint64_t *p, uint64_t val)
|
||||
);
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_acq(void)
|
||||
{
|
||||
|
||||
dmb(ld);
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_rel(void)
|
||||
{
|
||||
|
||||
dmb(sy);
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_acq_rel(void)
|
||||
{
|
||||
|
||||
dmb(sy);
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_seq_cst(void)
|
||||
{
|
||||
|
||||
dmb(sy);
|
||||
}
|
||||
|
||||
#define atomic_add_rel_long atomic_add_rel_64
|
||||
#define atomic_clear_rel_long atomic_clear_rel_64
|
||||
#define atomic_cmpset_rel_long atomic_cmpset_rel_64
|
||||
|
@ -86,6 +86,10 @@ void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
|
||||
int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src);
|
||||
u_int atomic_fetchadd_int(volatile u_int *p, u_int v);
|
||||
int atomic_testandset_int(volatile u_int *p, u_int v);
|
||||
void atomic_thread_fence_acq(void);
|
||||
void atomic_thread_fence_acq_rel(void);
|
||||
void atomic_thread_fence_rel(void);
|
||||
void atomic_thread_fence_seq_cst(void);
|
||||
|
||||
#define ATOMIC_LOAD(TYPE) \
|
||||
u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p)
|
||||
@ -310,6 +314,34 @@ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \
|
||||
} \
|
||||
struct __hack
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_acq(void)
|
||||
{
|
||||
|
||||
__compiler_membar();
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_rel(void)
|
||||
{
|
||||
|
||||
__compiler_membar();
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_acq_rel(void)
|
||||
{
|
||||
|
||||
__compiler_membar();
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_seq_cst(void)
|
||||
{
|
||||
|
||||
__storeload_barrier();
|
||||
}
|
||||
|
||||
#ifdef _KERNEL
|
||||
|
||||
#ifdef WANT_FUNCTIONS
|
||||
|
@ -496,6 +496,34 @@ atomic_fetchadd_64(__volatile uint64_t *p, uint64_t v)
|
||||
}
|
||||
#endif
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_acq(void)
|
||||
{
|
||||
|
||||
mips_sync();
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_rel(void)
|
||||
{
|
||||
|
||||
mips_sync();
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_acq_rel(void)
|
||||
{
|
||||
|
||||
mips_sync();
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_seq_cst(void)
|
||||
{
|
||||
|
||||
mips_sync();
|
||||
}
|
||||
|
||||
/* Operations on chars. */
|
||||
#define atomic_set_char atomic_set_8
|
||||
#define atomic_set_acq_char atomic_set_acq_8
|
||||
|
@ -730,4 +730,45 @@ atomic_swap_64(volatile u_long *p, u_long v)
|
||||
#undef __ATOMIC_REL
|
||||
#undef __ATOMIC_ACQ
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_acq(void)
|
||||
{
|
||||
|
||||
/* See above comment about lwsync being broken on Book-E. */
|
||||
#ifdef __powerpc64__
|
||||
__asm __volatile("lwsync" : : : "memory");
|
||||
#else
|
||||
__asm __volatile("sync" : : : "memory");
|
||||
#endif
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_rel(void)
|
||||
{
|
||||
|
||||
#ifdef __powerpc64__
|
||||
__asm __volatile("lwsync" : : : "memory");
|
||||
#else
|
||||
__asm __volatile("sync" : : : "memory");
|
||||
#endif
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_acq_rel(void)
|
||||
{
|
||||
|
||||
#ifdef __powerpc64__
|
||||
__asm __volatile("lwsync" : : : "memory");
|
||||
#else
|
||||
__asm __volatile("sync" : : : "memory");
|
||||
#endif
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_seq_cst(void)
|
||||
{
|
||||
|
||||
__asm __volatile("sync" : : : "memory");
|
||||
}
|
||||
|
||||
#endif /* ! _MACHINE_ATOMIC_H_ */
|
||||
|
@ -279,6 +279,35 @@ atomic_store_rel_ ## name(volatile ptype p, vtype v) \
|
||||
atomic_st_rel((p), (v), sz); \
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_acq(void)
|
||||
{
|
||||
|
||||
__compiler_membar();
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_rel(void)
|
||||
{
|
||||
|
||||
__compiler_membar();
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_acq_rel(void)
|
||||
{
|
||||
|
||||
__compiler_membar();
|
||||
}
|
||||
|
||||
static __inline void
|
||||
atomic_thread_fence_seq_cst(void)
|
||||
{
|
||||
|
||||
membar(LoadLoad | LoadStore | StoreStore | StoreLoad);
|
||||
}
|
||||
|
||||
|
||||
ATOMIC_GEN(int, u_int *, u_int, u_int, 32);
|
||||
ATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32);
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user