d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Dynamically select LSE-based atomic(9)s on arm64.

Browse Source 
Once all CPUs are online, determine if they all support LSE atomics and
set lse_supported to indicate this.  For now the atomic(9)
implementations are still always inlined, though it would be preferable
to create out-of-line functions to avoid text bloat.  This was not done
here since big.little systems exist in which some CPUs implement LSE
while others do not, and ifunc resolution must occur well before this
scenario can be detected.  It does seem unlikely that FreeBSD will
ever run on such platforms, however, so converting atomic(9) to use
ifuncs is probably a good next step.

Add a LSE_ATOMICS arm64 kernel configuration option to unconditionally
select LSE-based atomic(9) implementations when the target system is
known.

Reviewed by:	andrew, kib
MFC after:	1 month
Sponsored by:	The FreeBSD Foundation, Amazon (hardware)
Differential Revision:	https://reviews.freebsd.org/D23325
This commit is contained in:
markj 2020-02-03 18:23:50 +00:00
parent 1f66354182
commit 73a1608d91
3 changed files with 58 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
sys/arm64/arm64/identcpu.c
View File

@ -114,8 +114,8 @@ struct cpu_desc {
uint64_t id_aa64pfr1;
};
struct cpu_desc cpu_desc[MAXCPU];
struct cpu_desc user_cpu_desc;
static struct cpu_desc cpu_desc[MAXCPU];
static struct cpu_desc user_cpu_desc;
static u_int cpu_print_regs;
#define PRINT_ID_AA64_AFR0 0x00000001
#define PRINT_ID_AA64_AFR1 0x00000002
@ -961,6 +961,7 @@ update_user_regs(u_int cpu)
/* HWCAP */
extern u_long elf_hwcap;
bool __read_frequently lse_supported = false;
static void
identify_cpu_sysinit(void *dummy __unused)
@ -975,7 +976,6 @@ identify_cpu_sysinit(void *dummy __unused)
ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 | ID_AA64PFR0_EL0_64;
user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8;
CPU_FOREACH(cpu) {
print_cpu_features(cpu);
hwcap = parse_cpu_features_hwcap(cpu);
@ -986,6 +986,16 @@ identify_cpu_sysinit(void *dummy __unused)
update_user_regs(cpu);
}
if ((elf_hwcap & HWCAP_ATOMICS) != 0) {
lse_supported = true;
if (bootverbose)
printf("Enabling LSE atomics in the kernel\n");
}
#ifdef LSE_ATOMICS
if (!lse_supported)
panic("CPU does not support LSE atomic instructions");
#endif
install_undef_handler(true, user_mrs_handler);
}
SYSINIT(idenrity_cpu, SI_SUB_SMP, SI_ORDER_ANY, identify_cpu_sysinit, NULL);

51
sys/arm64/include/atomic.h
View File

@ -59,6 +59,18 @@
#include <sys/atomic_common.h>
#ifdef _KERNEL
extern bool lse_supported;
#ifdef LSE_ATOMICS
#define _ATOMIC_LSE_SUPPORTED 1
#else
#define _ATOMIC_LSE_SUPPORTED lse_supported
#endif
#else
#define _ATOMIC_LSE_SUPPORTED 0
#endif
#define _ATOMIC_OP_PROTO(t, op, bar, flav) \
static __inline void \
atomic_##op##_##bar##t##flav(volatile uint##t##_t *p, uint##t##_t val)
@ -98,7 +110,10 @@ _ATOMIC_OP_PROTO(t, op, bar, _lse) \
\
_ATOMIC_OP_PROTO(t, op, bar, ) \
{ \
atomic_##op##_##bar##t##_llsc(p, val); \
if (_ATOMIC_LSE_SUPPORTED) \
atomic_##op##_##bar##t##_lse(p, val); \
else \
atomic_##op##_##bar##t##_llsc(p, val); \
}
#define __ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \
@ -175,7 +190,12 @@ _ATOMIC_CMPSET_PROTO(t, bar, _lse) \
\
_ATOMIC_CMPSET_PROTO(t, bar, ) \
{ \
return (atomic_cmpset_##bar##t##_llsc(p, cmpval, newval)); \
if (_ATOMIC_LSE_SUPPORTED) \
return (atomic_cmpset_##bar##t##_lse(p, cmpval, \
newval)); \
else \
return (atomic_cmpset_##bar##t##_llsc(p, cmpval, \
newval)); \
} \
\
_ATOMIC_FCMPSET_PROTO(t, bar, _llsc) \
@ -223,7 +243,12 @@ _ATOMIC_FCMPSET_PROTO(t, bar, _lse) \
\
_ATOMIC_FCMPSET_PROTO(t, bar, ) \
{ \
return (atomic_fcmpset_##bar##t##_llsc(p, cmpval, newval)); \
if (_ATOMIC_LSE_SUPPORTED) \
return (atomic_fcmpset_##bar##t##_lse(p, cmpval, \
newval)); \
else \
return (atomic_fcmpset_##bar##t##_llsc(p, cmpval, \
newval)); \
}
#define _ATOMIC_CMPSET(bar, a, l) \
@ -277,7 +302,10 @@ _ATOMIC_FETCHADD_PROTO(t, _lse) \
\
_ATOMIC_FETCHADD_PROTO(t, ) \
{ \
return (atomic_fetchadd_##t##_llsc(p, val)); \
if (_ATOMIC_LSE_SUPPORTED) \
return (atomic_fetchadd_##t##_lse(p, val)); \
else \
return (atomic_fetchadd_##t##_llsc(p, val)); \
}
_ATOMIC_FETCHADD_IMPL(32, w)
@ -327,7 +355,10 @@ _ATOMIC_SWAP_PROTO(t, _lse) \
\
_ATOMIC_SWAP_PROTO(t, ) \
{ \
return (atomic_swap_##t##_llsc(p, val)); \
if (_ATOMIC_LSE_SUPPORTED) \
return (atomic_swap_##t##_lse(p, val)); \
else \
return (atomic_swap_##t##_llsc(p, val)); \
} \
\
_ATOMIC_READANDCLEAR_PROTO(t, _llsc) \
@ -354,7 +385,10 @@ _ATOMIC_READANDCLEAR_PROTO(t, _lse) \
\
_ATOMIC_READANDCLEAR_PROTO(t, ) \
{ \
return (atomic_readandclear_##t##_llsc(p)); \
if (_ATOMIC_LSE_SUPPORTED) \
return (atomic_readandclear_##t##_lse(p)); \
else \
return (atomic_readandclear_##t##_llsc(p)); \
}
_ATOMIC_SWAP_IMPL(32, w, wzr)
@ -403,7 +437,10 @@ _ATOMIC_TEST_OP_PROTO(t, op, _lse) \
\
_ATOMIC_TEST_OP_PROTO(t, op, ) \
{ \
return (atomic_testand##op##_##t##_llsc(p, val)); \
if (_ATOMIC_LSE_SUPPORTED) \
return (atomic_testand##op##_##t##_lse(p, val)); \
else \
return (atomic_testand##op##_##t##_llsc(p, val)); \
}
#define _ATOMIC_TEST_OP(op, llsc_asm_op, lse_asm_op) \

1
sys/conf/options.arm64
View File

@ -7,6 +7,7 @@ SOCDEV_VA opt_global.h
THUNDERX_PASS_1_1_ERRATA opt_global.h
VFP opt_global.h
LINUX_BOOT_ABI opt_global.h
LSE_ATOMICS opt_global.h
# Binary compatibility
COMPAT_FREEBSD32 opt_global.h