Workaround Spectre Variant 2 on arm64.

We need to handle two cases:

1. One process attacking another process.
2. A process attacking the kernel.

For the first case we clear the branch predictor state on context switch
between different processes. For the second we do this when taking an
instruction abort on a non-userspace address.

To clear the branch predictor state a per-CPU function pointer has been
added. This is set by the new cpu errata code based on if the CPU is
known to be affected.

On Cortex-A57, A72, A73, and A75 we call into the PSCI firmware as newer
versions of this will clear the branch predictor state for us.

It has been reported the ThunderX is unaffected, however the ThunderX2 is
vulnerable. The Qualcomm Falkor core is also affected. As FreeBSD doesn't
yet run on the ThunderX2 or Falkor no workaround is included for these CPUs.

MFC after:	3 days
Sponsored by:	DARPA, AFRL
Differential Revision:	https://reviews.freebsd.org/D13812

sys/arm64/arm64/cpu_errata.c

@@ -30,6 +30,8 @@
  * SUCH DAMAGE.
  */
 
+#include "opt_platform.h"
+
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
@@ -39,6 +41,10 @@ __FBSDID("$FreeBSD$");
 
 #include <machine/cpu.h>
 
+#ifdef DEV_PSCI
+#include <dev/psci/psci.h>
+#endif
+
 typedef void (cpu_quirk_install)(void);
 struct cpu_quirks {
 	cpu_quirk_install *quirk_install;
@@ -49,8 +55,37 @@ struct cpu_quirks {
 static cpu_quirk_install install_psci_bp_hardening;
 
 static struct cpu_quirks cpu_quirks[] = {
+	{
+		.midr_mask = CPU_IMPL_MASK | CPU_PART_MASK,
+		.midr_value = CPU_ID_RAW(CPU_IMPL_ARM, CPU_PART_CORTEX_A57,0,0),
+		.quirk_install = install_psci_bp_hardening,
+	},
+	{
+		.midr_mask = CPU_IMPL_MASK | CPU_PART_MASK,
+		.midr_value = CPU_ID_RAW(CPU_IMPL_ARM, CPU_PART_CORTEX_A72,0,0),
+		.quirk_install = install_psci_bp_hardening,
+	},
+	{
+		.midr_mask = CPU_IMPL_MASK | CPU_PART_MASK,
+		.midr_value = CPU_ID_RAW(CPU_IMPL_ARM, CPU_PART_CORTEX_A73,0,0),
+		.quirk_install = install_psci_bp_hardening,
+	},
+	{
+		.midr_mask = CPU_IMPL_MASK | CPU_PART_MASK,
+		.midr_value = CPU_ID_RAW(CPU_IMPL_ARM, CPU_PART_CORTEX_A75,0,0),
+		.quirk_install = install_psci_bp_hardening,
+	},
 };
 
+static void
+install_psci_bp_hardening(void)
+{
+
+#ifdef DEV_PSCI
+	PCPU_SET(bp_harden, psci_get_version);
+#endif
+}
+
 void
 install_cpu_errata(void)
 {

sys/arm64/arm64/pmap.c

@@ -4663,6 +4663,7 @@ pmap_activate(struct thread *td)
 struct pcb *
 pmap_switch(struct thread *old, struct thread *new)
 {
+	pcpu_bp_harden bp_harden;
 	struct pcb *pcb;
 
 	/* Store the new curthread */
@@ -4690,6 +4691,15 @@ pmap_switch(struct thread *old, struct thread *new)
 		    "dsb	ish		\n"
 		    "isb			\n"
 		    : : "r"(new->td_proc->p_md.md_l0addr));
+
+		/*
+		 * Stop userspace from training the branch predictor against
+		 * other processes. This will call into a CPU specific
+		 * function that clears the branch predictor state.
+		 */
+		bp_harden = PCPU_GET(bp_harden);
+		if (bp_harden != NULL)
+			bp_harden();
 	}
 
 	return (pcb);

sys/arm64/arm64/trap.c

@@ -352,6 +352,7 @@ do_el1h_sync(struct thread *td, struct trapframe *frame)
 void
 do_el0_sync(struct thread *td, struct trapframe *frame)
 {
+	pcpu_bp_harden bp_harden;
 	uint32_t exception;
 	uint64_t esr, far;
 
@@ -363,11 +364,25 @@ do_el0_sync(struct thread *td, struct trapframe *frame)
 	esr = frame->tf_esr;
 	exception = ESR_ELx_EXCEPTION(esr);
 	switch (exception) {
-	case EXCP_UNKNOWN:
 	case EXCP_INSN_ABORT_L:
+		far = READ_SPECIALREG(far_el1);
+
+		/*
+		 * Userspace may be trying to train the branch predictor to
+		 * attack the kernel. If we are on a CPU affected by this
+		 * call the handler to clear the branch predictor state.
+		 */
+		if (far > VM_MAXUSER_ADDRESS) {
+			bp_harden = PCPU_GET(bp_harden);
+			if (bp_harden != NULL)
+				bp_harden();
+		}
+		break;
+	case EXCP_UNKNOWN:
 	case EXCP_DATA_ABORT_L:
 	case EXCP_DATA_ABORT:
 		far = READ_SPECIALREG(far_el1);
+		break;
 	}
 	intr_enable();
 

sys/arm64/include/pcpu.h

@@ -35,11 +35,14 @@
 
 #define	ALT_STACK_SIZE	128
 
+typedef int (*pcpu_bp_harden)(void);
+
 #define	PCPU_MD_FIELDS							\
 	u_int	pc_acpi_id;	/* ACPI CPU id */		\
 	u_int	pc_midr;	/* stored MIDR value */	\
 	uint64_t pc_clock;						\
-	char __pad[241]
+	pcpu_bp_harden pc_bp_harden;					\
+	char __pad[233]
 
 #ifdef _KERNEL