USIII and beyond CPUs have stricter requirements when it comes

to synchronization needed after stores to internal ASIs in order
to make side-effects visible. This mainly requires the MEMBAR #Sync
after such stores to be replaced with a FLUSH. We use KERNBASE as
the address to FLUSH as it is guaranteed to not trap. Actually,
the USII synchronization rules also already require a FLUSH in
pretty much all of the cases changed.
We're also hitting an additional USIII synchronization rule which
requires stores to AA_IMMU_SFSR to be immediately followed by a DONE,
FLUSH or RETRY. Doing so triggers a RED state exception though so
leave the MEMBAR #Sync. Linux apparently also has gotten away with
doing the same for quite some time now, apart from the fact that
it's not clear to me why we need to clear the valid bit from the
SFSR in the first place.

Reviewed by:	nwhitehorn

sys/boot/sparc64/loader/main.c

@@ -37,6 +37,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/tlb.h>
 #include <machine/upa.h>
 #include <machine/ver.h>
+#include <machine/vmparam.h>
 
 #include "bootstrap.h"
 #include "libofw.h"
@@ -461,7 +462,7 @@ itlb_enter_sun4u(u_long vpn, u_long data)
 	stxa(AA_IMMU_TAR, ASI_IMMU,
 	     TLB_TAR_VA(vpn) | TLB_TAR_CTX(TLB_CTX_KERNEL));
 	stxa(0, ASI_ITLB_DATA_IN_REG, data);
-	membar(Sync);
+	flush(KERNBASE);
 	wrpr(pstate, reg, 0);
 }
 

sys/sparc64/sparc64/exception.S

@@ -498,6 +498,11 @@ END(rsf_fatal)
 	wr	%g0, ASI_IMMU, %asi
 	rdpr	%tpc, %g3
 	ldxa	[%g0 + AA_IMMU_SFSR] %asi, %g4
+	/*
+	 * XXX in theory, a store to AA_IMMU_SFSR must be immediately
+	 * followed by a DONE, FLUSH or RETRY for USIII.  In practice,
+	 * this triggers a RED state exception though.
+	 */
 	stxa	%g0, [%g0 + AA_IMMU_SFSR] %asi
 	membar	#Sync
 	ba	%xcc, tl0_sfsr_trap
@@ -716,8 +721,9 @@ ENTRY(tl0_immu_miss_trap)
 	 * Put back the contents of the tag access register, in case we
 	 * faulted.
 	 */
+	sethi	%hi(KERNBASE), %g2
 	stxa	%g1, [%g0 + AA_IMMU_TAR] %asi
-	membar	#Sync
+	flush	%g2
 
 	/*
 	 * Switch to alternate globals.
@@ -1213,6 +1219,11 @@ END(tl0_fp_restore)
 	wr	%g0, ASI_IMMU, %asi
 	rdpr	%tpc, %g3
 	ldxa	[%g0 + AA_IMMU_SFSR] %asi, %g4
+	/*
+	 * XXX in theory, a store to AA_IMMU_SFSR must be immediately
+	 * followed by a DONE, FLUSH or RETRY for USIII.  In practice,
+	 * this triggers a RED state exception though.
+	 */
 	stxa	%g0, [%g0 + AA_IMMU_SFSR] %asi
 	membar	#Sync
 	ba	%xcc, tl1_insn_exceptn_trap

sys/sparc64/sparc64/mp_exception.S

@@ -199,9 +199,10 @@ ENTRY(tl_ipi_tlb_page_demap)
 	ldx	[%g5 + ITA_VA], %g2
 	or	%g2, %g3, %g2
 
+	sethi	%hi(KERNBASE), %g3
 	stxa	%g0, [%g2] ASI_DMMU_DEMAP
 	stxa	%g0, [%g2] ASI_IMMU_DEMAP
-	membar	#Sync
+	flush	%g3
 
 	IPI_DONE(%g5, %g1, %g2, %g3)
 	retry
@@ -234,13 +235,13 @@ ENTRY(tl_ipi_tlb_range_demap)
 	ldx	[%g5 + ITA_START], %g1
 	ldx	[%g5 + ITA_END], %g2
 
-	set	PAGE_SIZE, %g6
-
 1:	or	%g1, %g3, %g4
+	sethi	%hi(KERNBASE), %g6
 	stxa	%g0, [%g4] ASI_DMMU_DEMAP
 	stxa	%g0, [%g4] ASI_IMMU_DEMAP
-	membar	#Sync
+	flush	%g6
 
+	set	PAGE_SIZE, %g6
 	add	%g1, %g6, %g1
 	cmp	%g1, %g2
 	blt,a,pt %xcc, 1b
@@ -265,9 +266,10 @@ ENTRY(tl_ipi_tlb_context_demap)
 #endif
 
 	mov	TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, %g1
+	sethi	%hi(KERNBASE), %g3
 	stxa	%g0, [%g1] ASI_DMMU_DEMAP
 	stxa	%g0, [%g1] ASI_IMMU_DEMAP
-	membar	#Sync
+	flush	%g3
 
 	IPI_DONE(%g5, %g1, %g2, %g3)
 	retry

sys/sparc64/sparc64/pmap.c

@@ -556,7 +556,7 @@ pmap_map_tsb(void)
 	 * FP block operations in the kernel).
 	 */
 	stxa(AA_DMMU_SCXR, ASI_DMMU, TLB_CTX_KERNEL);
-	membar(Sync);
+	flush(KERNBASE);
 
 	intr_restore(s);
 }
@@ -1980,7 +1980,7 @@ pmap_activate(struct thread *td)
 	stxa(AA_DMMU_TSB, ASI_DMMU, pm->pm_tsb);
 	stxa(AA_IMMU_TSB, ASI_IMMU, pm->pm_tsb);
 	stxa(AA_DMMU_PCXR, ASI_DMMU, context);
-	membar(Sync);
+	flush(KERNBASE);
 
 	mtx_unlock_spin(&sched_lock);
 }

sys/sparc64/sparc64/support.S

@@ -780,8 +780,9 @@ ENTRY(openfirmware_exit)
 	sub	%l0, SPOFF, %fp			! setup a stack in a locked page
 	sub	%l0, SPOFF + CCFSZ, %sp
 	mov	AA_DMMU_PCXR, %l3		! force primary DMMU context 0
+	sethi	%hi(KERNBASE), %l5
 	stxa	%g0, [%l3] ASI_DMMU
-	membar	#Sync
+	flush	%l5
 	wrpr	%g0, 0, %tl			! force trap level 0
 	call	%l6
 	 mov	%i0, %o0

sys/sparc64/sparc64/swtch.S

@@ -237,8 +237,9 @@ ENTRY(cpu_switch)
 	mov	AA_IMMU_TSB, %i5
 	stxa	%i4, [%i5] ASI_IMMU
 	mov	AA_DMMU_PCXR, %i5
+	sethi	%hi(KERNBASE), %i4
 	stxa	%i3, [%i5] ASI_DMMU
-	membar	#Sync
+	flush	%i4
 
 	/*
 	 * Done, return and load the new process's window from the stack.

sys/sparc64/sparc64/tlb.c

@@ -45,6 +45,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/pmap.h>
 #include <machine/smp.h>
 #include <machine/tlb.h>
+#include <machine/vmparam.h>
 
 PMAP_STATS_VAR(tlb_ncontext_demap);
 PMAP_STATS_VAR(tlb_npage_demap);
@@ -85,7 +86,7 @@ tlb_context_demap(struct pmap *pm)
 		s = intr_disable();
 		stxa(TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, ASI_DMMU_DEMAP, 0);
 		stxa(TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, ASI_IMMU_DEMAP, 0);
-		membar(Sync);
+		flush(KERNBASE);
 		intr_restore(s);
 	}
 	ipi_wait(cookie);
@@ -111,7 +112,7 @@ tlb_page_demap(struct pmap *pm, vm_offset_t va)
 		s = intr_disable();
 		stxa(TLB_DEMAP_VA(va) | flags, ASI_DMMU_DEMAP, 0);
 		stxa(TLB_DEMAP_VA(va) | flags, ASI_IMMU_DEMAP, 0);
-		membar(Sync);
+		flush(KERNBASE);
 		intr_restore(s);
 	}
 	ipi_wait(cookie);
@@ -139,7 +140,7 @@ tlb_range_demap(struct pmap *pm, vm_offset_t start, vm_offset_t end)
 		for (va = start; va < end; va += PAGE_SIZE) {
 			stxa(TLB_DEMAP_VA(va) | flags, ASI_DMMU_DEMAP, 0);
 			stxa(TLB_DEMAP_VA(va) | flags, ASI_IMMU_DEMAP, 0);
-			membar(Sync);
+			flush(KERNBASE);
 		}
 		intr_restore(s);
 	}