8954a9a4e6
provide a semantic defined by the C11 fences with corresponding memory_order. atomic_thread_fence_acq() gives r | r, w, where r and w are read and write accesses, and | denotes the fence itself. atomic_thread_fence_rel() is r, w | w. atomic_thread_fence_acq_rel() is the combination of the acquire and release in single operation. Note that reads after the acq+rel fence could be made visible before writes preceeding the fence. atomic_thread_fence_seq_cst() orders all accesses before/after the fence, and the fence itself is globally ordered against other sequentially consistent atomic operations. Reviewed by: alc Discussed with: bde Sponsored by: The FreeBSD Foundation MFC after: 3 weeks
337 lines
9.5 KiB
C
337 lines
9.5 KiB
C
/*-
|
|
* Copyright (c) 1998 Doug Rabson.
|
|
* Copyright (c) 2001 Jake Burkholder.
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
* SUCH DAMAGE.
|
|
*
|
|
* from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11
|
|
* $FreeBSD$
|
|
*/
|
|
|
|
#ifndef _MACHINE_ATOMIC_H_
|
|
#define _MACHINE_ATOMIC_H_
|
|
|
|
#include <machine/cpufunc.h>
|
|
|
|
#define mb() __asm__ __volatile__ ("membar #MemIssue": : :"memory")
|
|
#define wmb() mb()
|
|
#define rmb() mb()
|
|
|
|
/* Userland needs different ASI's. */
|
|
#ifdef _KERNEL
|
|
#define __ASI_ATOMIC ASI_N
|
|
#else
|
|
#define __ASI_ATOMIC ASI_P
|
|
#endif
|
|
|
|
/*
|
|
* Various simple arithmetic on memory which is atomic in the presence
|
|
* of interrupts and multiple processors. See atomic(9) for details.
|
|
* Note that efficient hardware support exists only for the 32 and 64
|
|
* bit variants; the 8 and 16 bit versions are not provided and should
|
|
* not be used in MI code.
|
|
*
|
|
* This implementation takes advantage of the fact that the sparc64
|
|
* cas instruction is both a load and a store. The loop is often coded
|
|
* as follows:
|
|
*
|
|
* do {
|
|
* expect = *p;
|
|
* new = expect + 1;
|
|
* } while (cas(p, expect, new) != expect);
|
|
*
|
|
* which performs an unnnecessary load on each iteration that the cas
|
|
* operation fails. Modified as follows:
|
|
*
|
|
* expect = *p;
|
|
* for (;;) {
|
|
* new = expect + 1;
|
|
* result = cas(p, expect, new);
|
|
* if (result == expect)
|
|
* break;
|
|
* expect = result;
|
|
* }
|
|
*
|
|
* the return value of cas is used to avoid the extra reload.
|
|
*
|
|
* We only include a memory barrier in the rel variants as in total store
|
|
* order which we use for running the kernel and all of the userland atomic
|
|
* loads and stores behave as if the were followed by a membar with a mask
|
|
* of #LoadLoad | #LoadStore | #StoreStore. In order to be also sufficient
|
|
* for use of relaxed memory ordering, the atomic_cas() in the acq variants
|
|
* additionally would have to be followed by a membar #LoadLoad | #LoadStore.
|
|
* Due to the suggested assembly syntax of the membar operands containing a
|
|
* # character, they cannot be used in macros. The cmask and mmask bits thus
|
|
* are hard coded in machine/cpufunc.h and used here through macros.
|
|
* Hopefully the bit numbers won't change in the future.
|
|
*/
|
|
|
|
#define itype(sz) uint ## sz ## _t
|
|
|
|
#define atomic_cas_32(p, e, s) casa((p), (e), (s), __ASI_ATOMIC)
|
|
#define atomic_cas_64(p, e, s) casxa((p), (e), (s), __ASI_ATOMIC)
|
|
|
|
#define atomic_cas(p, e, s, sz) \
|
|
atomic_cas_ ## sz((p), (e), (s))
|
|
|
|
#define atomic_cas_acq(p, e, s, sz) ({ \
|
|
itype(sz) v; \
|
|
v = atomic_cas((p), (e), (s), sz); \
|
|
__compiler_membar(); \
|
|
v; \
|
|
})
|
|
|
|
#define atomic_cas_rel(p, e, s, sz) ({ \
|
|
itype(sz) v; \
|
|
membar(LoadStore | StoreStore); \
|
|
v = atomic_cas((p), (e), (s), sz); \
|
|
v; \
|
|
})
|
|
|
|
#define atomic_op(p, op, v, sz) ({ \
|
|
itype(sz) e, r, s; \
|
|
for (e = *(volatile itype(sz) *)(p);; e = r) { \
|
|
s = e op (v); \
|
|
r = atomic_cas_ ## sz((p), e, s); \
|
|
if (r == e) \
|
|
break; \
|
|
} \
|
|
e; \
|
|
})
|
|
|
|
#define atomic_op_acq(p, op, v, sz) ({ \
|
|
itype(sz) t; \
|
|
t = atomic_op((p), op, (v), sz); \
|
|
__compiler_membar(); \
|
|
t; \
|
|
})
|
|
|
|
#define atomic_op_rel(p, op, v, sz) ({ \
|
|
itype(sz) t; \
|
|
membar(LoadStore | StoreStore); \
|
|
t = atomic_op((p), op, (v), sz); \
|
|
t; \
|
|
})
|
|
|
|
#define atomic_ld_acq(p, sz) ({ \
|
|
itype(sz) v; \
|
|
v = atomic_cas((p), 0, 0, sz); \
|
|
__compiler_membar(); \
|
|
v; \
|
|
})
|
|
|
|
#define atomic_ld_clear(p, sz) ({ \
|
|
itype(sz) e, r; \
|
|
for (e = *(volatile itype(sz) *)(p);; e = r) { \
|
|
r = atomic_cas((p), e, 0, sz); \
|
|
if (r == e) \
|
|
break; \
|
|
} \
|
|
e; \
|
|
})
|
|
|
|
#define atomic_st(p, v, sz) do { \
|
|
itype(sz) e, r; \
|
|
for (e = *(volatile itype(sz) *)(p);; e = r) { \
|
|
r = atomic_cas((p), e, (v), sz); \
|
|
if (r == e) \
|
|
break; \
|
|
} \
|
|
} while (0)
|
|
|
|
#define atomic_st_acq(p, v, sz) do { \
|
|
atomic_st((p), (v), sz); \
|
|
__compiler_membar(); \
|
|
} while (0)
|
|
|
|
#define atomic_st_rel(p, v, sz) do { \
|
|
membar(LoadStore | StoreStore); \
|
|
atomic_st((p), (v), sz); \
|
|
} while (0)
|
|
|
|
#define ATOMIC_GEN(name, ptype, vtype, atype, sz) \
|
|
\
|
|
static __inline vtype \
|
|
atomic_add_ ## name(volatile ptype p, atype v) \
|
|
{ \
|
|
return ((vtype)atomic_op((p), +, (v), sz)); \
|
|
} \
|
|
static __inline vtype \
|
|
atomic_add_acq_ ## name(volatile ptype p, atype v) \
|
|
{ \
|
|
return ((vtype)atomic_op_acq((p), +, (v), sz)); \
|
|
} \
|
|
static __inline vtype \
|
|
atomic_add_rel_ ## name(volatile ptype p, atype v) \
|
|
{ \
|
|
return ((vtype)atomic_op_rel((p), +, (v), sz)); \
|
|
} \
|
|
\
|
|
static __inline vtype \
|
|
atomic_clear_ ## name(volatile ptype p, atype v) \
|
|
{ \
|
|
return ((vtype)atomic_op((p), &, ~(v), sz)); \
|
|
} \
|
|
static __inline vtype \
|
|
atomic_clear_acq_ ## name(volatile ptype p, atype v) \
|
|
{ \
|
|
return ((vtype)atomic_op_acq((p), &, ~(v), sz)); \
|
|
} \
|
|
static __inline vtype \
|
|
atomic_clear_rel_ ## name(volatile ptype p, atype v) \
|
|
{ \
|
|
return ((vtype)atomic_op_rel((p), &, ~(v), sz)); \
|
|
} \
|
|
\
|
|
static __inline int \
|
|
atomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s) \
|
|
{ \
|
|
return (((vtype)atomic_cas((p), (e), (s), sz)) == (e)); \
|
|
} \
|
|
static __inline int \
|
|
atomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s) \
|
|
{ \
|
|
return (((vtype)atomic_cas_acq((p), (e), (s), sz)) == (e)); \
|
|
} \
|
|
static __inline int \
|
|
atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s) \
|
|
{ \
|
|
return (((vtype)atomic_cas_rel((p), (e), (s), sz)) == (e)); \
|
|
} \
|
|
\
|
|
static __inline vtype \
|
|
atomic_load_ ## name(volatile ptype p) \
|
|
{ \
|
|
return ((vtype)atomic_cas((p), 0, 0, sz)); \
|
|
} \
|
|
static __inline vtype \
|
|
atomic_load_acq_ ## name(volatile ptype p) \
|
|
{ \
|
|
return ((vtype)atomic_cas_acq((p), 0, 0, sz)); \
|
|
} \
|
|
\
|
|
static __inline vtype \
|
|
atomic_readandclear_ ## name(volatile ptype p) \
|
|
{ \
|
|
return ((vtype)atomic_ld_clear((p), sz)); \
|
|
} \
|
|
\
|
|
static __inline vtype \
|
|
atomic_set_ ## name(volatile ptype p, atype v) \
|
|
{ \
|
|
return ((vtype)atomic_op((p), |, (v), sz)); \
|
|
} \
|
|
static __inline vtype \
|
|
atomic_set_acq_ ## name(volatile ptype p, atype v) \
|
|
{ \
|
|
return ((vtype)atomic_op_acq((p), |, (v), sz)); \
|
|
} \
|
|
static __inline vtype \
|
|
atomic_set_rel_ ## name(volatile ptype p, atype v) \
|
|
{ \
|
|
return ((vtype)atomic_op_rel((p), |, (v), sz)); \
|
|
} \
|
|
\
|
|
static __inline vtype \
|
|
atomic_subtract_ ## name(volatile ptype p, atype v) \
|
|
{ \
|
|
return ((vtype)atomic_op((p), -, (v), sz)); \
|
|
} \
|
|
static __inline vtype \
|
|
atomic_subtract_acq_ ## name(volatile ptype p, atype v) \
|
|
{ \
|
|
return ((vtype)atomic_op_acq((p), -, (v), sz)); \
|
|
} \
|
|
static __inline vtype \
|
|
atomic_subtract_rel_ ## name(volatile ptype p, atype v) \
|
|
{ \
|
|
return ((vtype)atomic_op_rel((p), -, (v), sz)); \
|
|
} \
|
|
\
|
|
static __inline void \
|
|
atomic_store_acq_ ## name(volatile ptype p, vtype v) \
|
|
{ \
|
|
atomic_st_acq((p), (v), sz); \
|
|
} \
|
|
static __inline void \
|
|
atomic_store_rel_ ## name(volatile ptype p, vtype v) \
|
|
{ \
|
|
atomic_st_rel((p), (v), sz); \
|
|
}
|
|
|
|
static __inline void
|
|
atomic_thread_fence_acq(void)
|
|
{
|
|
|
|
__compiler_membar();
|
|
}
|
|
|
|
static __inline void
|
|
atomic_thread_fence_rel(void)
|
|
{
|
|
|
|
__compiler_membar();
|
|
}
|
|
|
|
static __inline void
|
|
atomic_thread_fence_acq_rel(void)
|
|
{
|
|
|
|
__compiler_membar();
|
|
}
|
|
|
|
static __inline void
|
|
atomic_thread_fence_seq_cst(void)
|
|
{
|
|
|
|
membar(LoadLoad | LoadStore | StoreStore | StoreLoad);
|
|
}
|
|
|
|
|
|
ATOMIC_GEN(int, u_int *, u_int, u_int, 32);
|
|
ATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32);
|
|
|
|
ATOMIC_GEN(long, u_long *, u_long, u_long, 64);
|
|
ATOMIC_GEN(64, uint64_t *, uint64_t, uint64_t, 64);
|
|
|
|
ATOMIC_GEN(ptr, uintptr_t *, uintptr_t, uintptr_t, 64);
|
|
|
|
#define atomic_fetchadd_int atomic_add_int
|
|
#define atomic_fetchadd_32 atomic_add_32
|
|
#define atomic_fetchadd_long atomic_add_long
|
|
|
|
#undef ATOMIC_GEN
|
|
#undef atomic_cas
|
|
#undef atomic_cas_acq
|
|
#undef atomic_cas_rel
|
|
#undef atomic_op
|
|
#undef atomic_op_acq
|
|
#undef atomic_op_rel
|
|
#undef atomic_ld_acq
|
|
#undef atomic_ld_clear
|
|
#undef atomic_st
|
|
#undef atomic_st_acq
|
|
#undef atomic_st_rel
|
|
|
|
#endif /* !_MACHINE_ATOMIC_H_ */
|