Stop speculation past an eret instruction

On arm64 the eret instruction is used to return from an exception handler.
Some implementations may speculate past this instruction into the next
function. As the user may control many registers in these functions add
a synchronisation barrier sequence after the eret instruction to stop these
CPUs from speculating out of the exception handler.

PR:		242676
Submitted by:	Anthony Steinhauser <asteinhauser@google.com> (previous version)
MFC after:	1 week

sys/arm64/arm64/exception.S

@@ -175,7 +175,7 @@ ENTRY(handle_el1h_sync)
 	mov	x1, sp
 	bl	do_el1h_sync
 	restore_registers 1
-	eret
+	ERET
 END(handle_el1h_sync)
 
 ENTRY(handle_el1h_irq)
@@ -183,7 +183,7 @@ ENTRY(handle_el1h_irq)
 	mov	x0, sp
 	bl	intr_irq_handler
 	restore_registers 1
-	eret
+	ERET
 END(handle_el1h_irq)
 
 ENTRY(handle_el0_sync)
@@ -194,7 +194,7 @@ ENTRY(handle_el0_sync)
 	bl	do_el0_sync
 	do_ast
 	restore_registers 0
-	eret
+	ERET
 END(handle_el0_sync)
 
 ENTRY(handle_el0_irq)
@@ -203,7 +203,7 @@ ENTRY(handle_el0_irq)
 	bl	intr_irq_handler
 	do_ast
 	restore_registers 0
-	eret
+	ERET
 END(handle_el0_irq)
 
 ENTRY(handle_serror)

sys/arm64/arm64/swtch.S

@@ -253,7 +253,7 @@ ENTRY(fork_trampoline)
 	 * No need for interrupts reenabling since PSR
 	 * will be set to the desired value anyway.
 	 */
-	eret
+	ERET
 	
 END(fork_trampoline)
 

sys/arm64/include/asm.h

@@ -90,4 +90,16 @@
 	.inst	0xd500409f | (1 << 8);		/* Set PAN */		\
 	999:
 
+/*
+ * Some AArch64 CPUs speculate past an eret instruction. As the user may
+ * control the registers at this point add a speculation barrier usable on
+ * all AArch64 CPUs after the eret instruction.
+ * TODO: ARMv8.5 adds a specific instruction for this, we could use that
+ * if we know we are running on something that supports it.
+ */
+#define	ERET								\
+	eret;								\
+	dsb	sy;							\
+	isb
+
 #endif /* _MACHINE_ASM_H_ */