From 2611aba1801dd557c465c1d81a201f96c4e0b6b0 Mon Sep 17 00:00:00 2001 From: Marius Strobl Date: Mon, 8 Sep 2008 20:38:48 +0000 Subject: [PATCH] USIII and beyond CPUs have stricter requirements when it comes to synchronization needed after stores to internal ASIs in order to make side-effects visible. This mainly requires the MEMBAR #Sync after such stores to be replaced with a FLUSH. We use KERNBASE as the address to FLUSH as it is guaranteed to not trap. Actually, the USII synchronization rules also already require a FLUSH in pretty much all of the cases changed. We're also hitting an additional USIII synchronization rule which requires stores to AA_IMMU_SFSR to be immediately followed by a DONE, FLUSH or RETRY. Doing so triggers a RED state exception though so leave the MEMBAR #Sync. Linux apparently also has gotten away with doing the same for quite some time now, apart from the fact that it's not clear to me why we need to clear the valid bit from the SFSR in the first place. Reviewed by: nwhitehorn --- sys/boot/sparc64/loader/main.c | 3 ++- sys/sparc64/sparc64/exception.S | 13 ++++++++++++- sys/sparc64/sparc64/mp_exception.S | 12 +++++++----- sys/sparc64/sparc64/pmap.c | 4 ++-- sys/sparc64/sparc64/support.S | 3 ++- sys/sparc64/sparc64/swtch.S | 3 ++- sys/sparc64/sparc64/tlb.c | 7 ++++--- 7 files changed, 31 insertions(+), 14 deletions(-) diff --git a/sys/boot/sparc64/loader/main.c b/sys/boot/sparc64/loader/main.c index 34fe4cec1d7f..5091caa52237 100644 --- a/sys/boot/sparc64/loader/main.c +++ b/sys/boot/sparc64/loader/main.c @@ -37,6 +37,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include "bootstrap.h" #include "libofw.h" @@ -461,7 +462,7 @@ itlb_enter_sun4u(u_long vpn, u_long data) stxa(AA_IMMU_TAR, ASI_IMMU, TLB_TAR_VA(vpn) | TLB_TAR_CTX(TLB_CTX_KERNEL)); stxa(0, ASI_ITLB_DATA_IN_REG, data); - membar(Sync); + flush(KERNBASE); wrpr(pstate, reg, 0); } diff --git a/sys/sparc64/sparc64/exception.S b/sys/sparc64/sparc64/exception.S index 1e097e92bd18..5f81c0402af2 100644 --- a/sys/sparc64/sparc64/exception.S +++ b/sys/sparc64/sparc64/exception.S @@ -498,6 +498,11 @@ END(rsf_fatal) wr %g0, ASI_IMMU, %asi rdpr %tpc, %g3 ldxa [%g0 + AA_IMMU_SFSR] %asi, %g4 + /* + * XXX in theory, a store to AA_IMMU_SFSR must be immediately + * followed by a DONE, FLUSH or RETRY for USIII. In practice, + * this triggers a RED state exception though. + */ stxa %g0, [%g0 + AA_IMMU_SFSR] %asi membar #Sync ba %xcc, tl0_sfsr_trap @@ -716,8 +721,9 @@ ENTRY(tl0_immu_miss_trap) * Put back the contents of the tag access register, in case we * faulted. */ + sethi %hi(KERNBASE), %g2 stxa %g1, [%g0 + AA_IMMU_TAR] %asi - membar #Sync + flush %g2 /* * Switch to alternate globals. @@ -1213,6 +1219,11 @@ END(tl0_fp_restore) wr %g0, ASI_IMMU, %asi rdpr %tpc, %g3 ldxa [%g0 + AA_IMMU_SFSR] %asi, %g4 + /* + * XXX in theory, a store to AA_IMMU_SFSR must be immediately + * followed by a DONE, FLUSH or RETRY for USIII. In practice, + * this triggers a RED state exception though. + */ stxa %g0, [%g0 + AA_IMMU_SFSR] %asi membar #Sync ba %xcc, tl1_insn_exceptn_trap diff --git a/sys/sparc64/sparc64/mp_exception.S b/sys/sparc64/sparc64/mp_exception.S index fbb1c252e147..3e53377e7b73 100644 --- a/sys/sparc64/sparc64/mp_exception.S +++ b/sys/sparc64/sparc64/mp_exception.S @@ -199,9 +199,10 @@ ENTRY(tl_ipi_tlb_page_demap) ldx [%g5 + ITA_VA], %g2 or %g2, %g3, %g2 + sethi %hi(KERNBASE), %g3 stxa %g0, [%g2] ASI_DMMU_DEMAP stxa %g0, [%g2] ASI_IMMU_DEMAP - membar #Sync + flush %g3 IPI_DONE(%g5, %g1, %g2, %g3) retry @@ -234,13 +235,13 @@ ENTRY(tl_ipi_tlb_range_demap) ldx [%g5 + ITA_START], %g1 ldx [%g5 + ITA_END], %g2 - set PAGE_SIZE, %g6 - 1: or %g1, %g3, %g4 + sethi %hi(KERNBASE), %g6 stxa %g0, [%g4] ASI_DMMU_DEMAP stxa %g0, [%g4] ASI_IMMU_DEMAP - membar #Sync + flush %g6 + set PAGE_SIZE, %g6 add %g1, %g6, %g1 cmp %g1, %g2 blt,a,pt %xcc, 1b @@ -265,9 +266,10 @@ ENTRY(tl_ipi_tlb_context_demap) #endif mov TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, %g1 + sethi %hi(KERNBASE), %g3 stxa %g0, [%g1] ASI_DMMU_DEMAP stxa %g0, [%g1] ASI_IMMU_DEMAP - membar #Sync + flush %g3 IPI_DONE(%g5, %g1, %g2, %g3) retry diff --git a/sys/sparc64/sparc64/pmap.c b/sys/sparc64/sparc64/pmap.c index d3126fe66e75..be3226a15544 100644 --- a/sys/sparc64/sparc64/pmap.c +++ b/sys/sparc64/sparc64/pmap.c @@ -556,7 +556,7 @@ pmap_map_tsb(void) * FP block operations in the kernel). */ stxa(AA_DMMU_SCXR, ASI_DMMU, TLB_CTX_KERNEL); - membar(Sync); + flush(KERNBASE); intr_restore(s); } @@ -1980,7 +1980,7 @@ pmap_activate(struct thread *td) stxa(AA_DMMU_TSB, ASI_DMMU, pm->pm_tsb); stxa(AA_IMMU_TSB, ASI_IMMU, pm->pm_tsb); stxa(AA_DMMU_PCXR, ASI_DMMU, context); - membar(Sync); + flush(KERNBASE); mtx_unlock_spin(&sched_lock); } diff --git a/sys/sparc64/sparc64/support.S b/sys/sparc64/sparc64/support.S index 6bec2b80bd32..5013dc64f8f1 100644 --- a/sys/sparc64/sparc64/support.S +++ b/sys/sparc64/sparc64/support.S @@ -780,8 +780,9 @@ ENTRY(openfirmware_exit) sub %l0, SPOFF, %fp ! setup a stack in a locked page sub %l0, SPOFF + CCFSZ, %sp mov AA_DMMU_PCXR, %l3 ! force primary DMMU context 0 + sethi %hi(KERNBASE), %l5 stxa %g0, [%l3] ASI_DMMU - membar #Sync + flush %l5 wrpr %g0, 0, %tl ! force trap level 0 call %l6 mov %i0, %o0 diff --git a/sys/sparc64/sparc64/swtch.S b/sys/sparc64/sparc64/swtch.S index f44d4f3f84a4..552233f03b71 100644 --- a/sys/sparc64/sparc64/swtch.S +++ b/sys/sparc64/sparc64/swtch.S @@ -237,8 +237,9 @@ ENTRY(cpu_switch) mov AA_IMMU_TSB, %i5 stxa %i4, [%i5] ASI_IMMU mov AA_DMMU_PCXR, %i5 + sethi %hi(KERNBASE), %i4 stxa %i3, [%i5] ASI_DMMU - membar #Sync + flush %i4 /* * Done, return and load the new process's window from the stack. diff --git a/sys/sparc64/sparc64/tlb.c b/sys/sparc64/sparc64/tlb.c index d76eaec3a0b7..9bd90a1dd475 100644 --- a/sys/sparc64/sparc64/tlb.c +++ b/sys/sparc64/sparc64/tlb.c @@ -45,6 +45,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include PMAP_STATS_VAR(tlb_ncontext_demap); PMAP_STATS_VAR(tlb_npage_demap); @@ -85,7 +86,7 @@ tlb_context_demap(struct pmap *pm) s = intr_disable(); stxa(TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, ASI_DMMU_DEMAP, 0); stxa(TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, ASI_IMMU_DEMAP, 0); - membar(Sync); + flush(KERNBASE); intr_restore(s); } ipi_wait(cookie); @@ -111,7 +112,7 @@ tlb_page_demap(struct pmap *pm, vm_offset_t va) s = intr_disable(); stxa(TLB_DEMAP_VA(va) | flags, ASI_DMMU_DEMAP, 0); stxa(TLB_DEMAP_VA(va) | flags, ASI_IMMU_DEMAP, 0); - membar(Sync); + flush(KERNBASE); intr_restore(s); } ipi_wait(cookie); @@ -139,7 +140,7 @@ tlb_range_demap(struct pmap *pm, vm_offset_t start, vm_offset_t end) for (va = start; va < end; va += PAGE_SIZE) { stxa(TLB_DEMAP_VA(va) | flags, ASI_DMMU_DEMAP, 0); stxa(TLB_DEMAP_VA(va) | flags, ASI_IMMU_DEMAP, 0); - membar(Sync); + flush(KERNBASE); } intr_restore(s); }