Support IBRS for i386.

Tested by:	pho
Sponsored by:	The FreeBSD Foundation
MFC after:	1 week
Differential revision:	https://reviews.freebsd.org/D15522

sys/i386/i386/exception.s

@@ -310,6 +310,8 @@ IDTVEC(dbg)
 	jz	calltrap
 dbg_user:
 	NMOVE_STACKS
+	movl	$handle_ibrs_entry,%eax
+	call	*%eax
 	pushl	%esp
 	movl	$trap,%eax
 	call	*%eax
@@ -337,6 +339,8 @@ nmi_mchk_common:
 	 * Do not switch to the thread kernel stack, otherwise we might
 	 * obliterate the previous context partially copied from the
 	 * trampoline stack.
+	 * Do not re-enable IBRS, there is no good place to store
+	 * previous state if we come from the kernel.
 	 */
 	movl	%cr3, %eax
 	movl	%eax, TF_ERR(%esp)
@@ -364,6 +368,8 @@ IDTVEC(int0x80_syscall)
 	SET_KERNEL_SREGS
 	cld
 	MOVE_STACKS
+	movl	$handle_ibrs_entry,%eax
+	call	*%eax
 	sti
 	FAKE_MCOUNT(TF_EIP(%esp))
 	pushl	%esp
@@ -509,7 +515,9 @@ doreti_exit:
 	jmp	2f
 1:	testl	$SEL_RPL_MASK, TF_CS(%esp)
 	jz	doreti_popl_fs
-2:	movl	%esp, %esi
+2:	movl	$handle_ibrs_exit,%eax
+	call	*%eax
+	movl	%esp, %esi
 	movl	PCPU(TRAMPSTK), %edx
 	subl	%ecx, %edx
 	movl	%edx, %edi

sys/i386/i386/genassym.c

@@ -221,6 +221,7 @@ ASSYM(PC_PRIVATE_TSS, offsetof(struct pcpu, pc_private_tss));
 ASSYM(PC_KESP0, offsetof(struct pcpu, pc_kesp0));
 ASSYM(PC_TRAMPSTK, offsetof(struct pcpu, pc_trampstk));
 ASSYM(PC_COPYOUT_BUF, offsetof(struct pcpu, pc_copyout_buf));
+ASSYM(PC_IBPB_SET, offsetof(struct pcpu, pc_ibpb_set));
 
 #ifdef DEV_APIC
 ASSYM(LA_EOI, LAPIC_EOI * LAPIC_MEM_MUL);

sys/i386/i386/support.s

@@ -433,9 +433,31 @@ msr_onfault:
 	ret
 
 ENTRY(handle_ibrs_entry)
-	ret
+	cmpb	$0,hw_ibrs_active
+	je	1f
+	movl	$MSR_IA32_SPEC_CTRL,%ecx
+	rdmsr
+	orl	$(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
+	orl	$(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32,%edx
+	wrmsr
+	movb	$1,PCPU(IBPB_SET)
+	/*
+	 * i386 does not implement SMEP, but the 4/4 split makes this not
+	 * that important.
+	 */
+1:	ret
 END(handle_ibrs_entry)
 
 ENTRY(handle_ibrs_exit)
-	ret
+	cmpb	$0,PCPU(IBPB_SET)
+	je	1f
+	pushl	%ecx
+	movl	$MSR_IA32_SPEC_CTRL,%ecx
+	rdmsr
+	andl	$~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
+	andl	$~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx
+	wrmsr
+	popl	%ecx
+	movb	$0,PCPU(IBPB_SET)
+1:	ret
 END(handle_ibrs_exit)

sys/i386/include/asmacros.h

@@ -218,10 +218,14 @@
 	testl	$PCB_VM86CALL, PCB_FLAGS(%eax)
 	jnz	.L\@.3
 	NMOVE_STACKS
+	movl	$handle_ibrs_entry,%edx
+	call	*%edx
 	jmp	.L\@.3
 .L\@.1:	testb	$SEL_RPL_MASK, TF_CS(%esp)
 	jz	.L\@.3
 .L\@.2:	MOVE_STACKS
+	movl	$handle_ibrs_entry,%edx
+	call	*%edx
 .L\@.3:
 	.endm