d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Stop performing a full icache sync when the DIC and IDC flags are set

Browse Source 
The DIC and IDC bits in the CTR_EL0 register signal to the kernel when it
can relax the instruction cache synchronisation operations. The IDC bit
means we can relax cleaning the data cache to the point of unification
while the DIC bit means we don't need to invalidate the instruction cache
for data coherence. In both cases an appropriate barrier is still needed.

For now only implement the case where both bits are set, as is the case
on the Neoverse-N1 as used in the Amazon AWS Graviton 2 CPU. Note that
this behaviour is a optional on the N1 so we may later need to implement
only one or the other bit being set.

There is a tunable to disable each flag on boot.

Testing on a 4 core Graviton 2 instance found a significant improvement
in sys and real time when running "make buildkernel -j4", with no
significant difference in user time.

Reviewed by:	markj
Sponsored by:	Innovate UK
Differential Revision:	https://reviews.freebsd.org/D24853
This commit is contained in:
andrew 2020-05-19 16:04:27 +00:00
parent 0388e6d768
commit 9daa38339c
3 changed files with 50 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
sys/arm64/arm64/cpufunc_asm.S
View File

@ -133,9 +133,20 @@ ENTRY(arm64_dcache_inv_range)
END(arm64_dcache_inv_range)
/*
* void arm64_icache_sync_range(vm_offset_t, vm_size_t)
* void arm64_dic_idc_icache_sync_range(vm_offset_t, vm_size_t)
* When the CTR_EL0.IDC bit is set cleaning to PoU becomes a dsb.
* When the CTR_EL0.DIC bit is set icache invalidation becomes an isb.
*/
ENTRY(arm64_icache_sync_range)
ENTRY(arm64_dic_idc_icache_sync_range)
dsb ishst
isb
ret
END(arm64_dic_idc_icache_sync_range)
/*
* void arm64_aliasing_icache_sync_range(vm_offset_t, vm_size_t)
*/
ENTRY(arm64_aliasing_icache_sync_range)
/*
* XXX Temporary solution - I-cache flush should be range based for
* PIPT cache or IALLUIS for VIVT or VIPT caches
@ -146,7 +157,7 @@ ENTRY(arm64_icache_sync_range)
dsb ish
isb
ret
END(arm64_icache_sync_range)
END(arm64_aliasing_icache_sync_range)
/*
* int arm64_icache_sync_range_checked(vm_offset_t, vm_size_t)

32
sys/arm64/arm64/identcpu.c
View File

@ -56,6 +56,24 @@ char machine[] = "arm64";
extern int adaptive_machine_arch;
#endif
static SYSCTL_NODE(_machdep, OID_AUTO, cache, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
"Cache management tuning");
static int allow_dic = 1;
SYSCTL_INT(_machdep_cache, OID_AUTO, allow_dic, CTLFLAG_RDTUN, &allow_dic, 0,
"Allow optimizations based on the DIC cache bit");
static int allow_idc = 1;
SYSCTL_INT(_machdep_cache, OID_AUTO, allow_idc, CTLFLAG_RDTUN, &allow_idc, 0,
"Allow optimizations based on the IDC cache bit");
/*
* The default implementation of I-cache sync assumes we have an
* aliasing cache until we know otherwise.
*/
void (*arm64_icache_sync_range)(vm_offset_t, vm_size_t) =
&arm64_aliasing_icache_sync_range;
static int
sysctl_hw_machine(SYSCTL_HANDLER_ARGS)
{
@ -977,6 +995,7 @@ identify_cpu_sysinit(void *dummy __unused)
{
int cpu;
u_long hwcap;
bool dic, idc;
/* Create a user visible cpu description with safe values */
memset(&user_cpu_desc, 0, sizeof(user_cpu_desc));
@ -985,6 +1004,8 @@ identify_cpu_sysinit(void *dummy __unused)
ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 | ID_AA64PFR0_EL0_64;
user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8;
dic = (allow_dic != 0);
idc = (allow_idc != 0);
CPU_FOREACH(cpu) {
print_cpu_features(cpu);
hwcap = parse_cpu_features_hwcap(cpu);
@ -993,6 +1014,17 @@ identify_cpu_sysinit(void *dummy __unused)
else
elf_hwcap &= hwcap;
update_user_regs(cpu);
if (CTR_DIC_VAL(cpu_desc[cpu].ctr) == 0)
dic = false;
if (CTR_IDC_VAL(cpu_desc[cpu].ctr) == 0)
idc = false;
}
if (dic && idc) {
arm64_icache_sync_range = &arm64_dic_idc_icache_sync_range;
if (bootverbose)
printf("Enabling DIC & IDC ICache sync\n");
}
if ((elf_hwcap & HWCAP_ATOMICS) != 0) {

5
sys/arm64/include/cpufunc.h
View File

@ -216,12 +216,15 @@ extern int64_t dczva_line_size;
#define cpu_dcache_inv_range(a, s) arm64_dcache_inv_range((a), (s))
#define cpu_dcache_wb_range(a, s) arm64_dcache_wb_range((a), (s))
extern void (*arm64_icache_sync_range)(vm_offset_t, vm_size_t);
#define cpu_icache_sync_range(a, s) arm64_icache_sync_range((a), (s))
#define cpu_icache_sync_range_checked(a, s) arm64_icache_sync_range_checked((a), (s))
void arm64_nullop(void);
void arm64_tlb_flushID(void);
void arm64_icache_sync_range(vm_offset_t, vm_size_t);
void arm64_dic_idc_icache_sync_range(vm_offset_t, vm_size_t);
void arm64_aliasing_icache_sync_range(vm_offset_t, vm_size_t);
int arm64_icache_sync_range_checked(vm_offset_t, vm_size_t);
void arm64_dcache_wbinv_range(vm_offset_t, vm_size_t);
void arm64_dcache_inv_range(vm_offset_t, vm_size_t);