amd64: Add MD bits for KMSAN
Interrupt and exception handlers must call kmsan_intr_enter() prior to calling any C code. This is because the KMSAN runtime maintains some TLS in order to track initialization state of function parameters and return values across function calls. Then, to ensure that this state is kept consistent in the face of asynchronous kernel-mode excpeptions, the runtime uses a stack of TLS blocks, and kmsan_intr_enter() and kmsan_intr_leave() push and pop that stack, respectively. Use these functions in amd64 interrupt and exception handlers. Note that handlers for user->kernel transitions need not be annotated. Also ensure that trap frames pushed by the CPU and by handlers are marked as initialized before they are used. Reviewed by: kib Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D31467
This commit is contained in:
parent
8978608832
commit
b0f71f1bc5
@ -81,6 +81,7 @@ as_lapic_eoi:
|
||||
*/
|
||||
.macro ISR_VEC index, vec_name
|
||||
INTR_HANDLER \vec_name
|
||||
KMSAN_ENTER
|
||||
cmpl $0,x2apic_mode
|
||||
je 1f
|
||||
movl $(MSR_APIC_ISR0 + \index),%ecx
|
||||
@ -97,6 +98,7 @@ as_lapic_eoi:
|
||||
movl %eax, %edi /* pass the IRQ */
|
||||
call lapic_handle_intr
|
||||
3:
|
||||
KMSAN_LEAVE
|
||||
jmp doreti
|
||||
.endm
|
||||
|
||||
@ -125,22 +127,28 @@ IDTVEC(spuriousint)
|
||||
* Local APIC periodic timer handler.
|
||||
*/
|
||||
INTR_HANDLER timerint
|
||||
KMSAN_ENTER
|
||||
movq %rsp, %rdi
|
||||
call lapic_handle_timer
|
||||
KMSAN_LEAVE
|
||||
jmp doreti
|
||||
|
||||
/*
|
||||
* Local APIC CMCI handler.
|
||||
*/
|
||||
INTR_HANDLER cmcint
|
||||
KMSAN_ENTER
|
||||
call lapic_handle_cmc
|
||||
KMSAN_LEAVE
|
||||
jmp doreti
|
||||
|
||||
/*
|
||||
* Local APIC error interrupt handler.
|
||||
*/
|
||||
INTR_HANDLER errorint
|
||||
KMSAN_ENTER
|
||||
call lapic_handle_error
|
||||
KMSAN_LEAVE
|
||||
jmp doreti
|
||||
|
||||
#ifdef XENHVM
|
||||
@ -149,8 +157,10 @@ IDTVEC(spuriousint)
|
||||
* Only used when the hypervisor supports direct vector callbacks.
|
||||
*/
|
||||
INTR_HANDLER xen_intr_upcall
|
||||
KMSAN_ENTER
|
||||
movq %rsp, %rdi
|
||||
call xen_intr_handle_upcall
|
||||
KMSAN_LEAVE
|
||||
jmp doreti
|
||||
#endif
|
||||
|
||||
@ -165,8 +175,10 @@ IDTVEC(spuriousint)
|
||||
* IPI handler for cache and TLB shootdown
|
||||
*/
|
||||
INTR_HANDLER invlop
|
||||
KMSAN_ENTER
|
||||
call invlop_handler
|
||||
call as_lapic_eoi
|
||||
KMSAN_LEAVE
|
||||
jmp ld_regs
|
||||
|
||||
/*
|
||||
@ -174,7 +186,9 @@ IDTVEC(spuriousint)
|
||||
*/
|
||||
INTR_HANDLER ipi_intr_bitmap_handler
|
||||
call as_lapic_eoi
|
||||
KMSAN_ENTER
|
||||
call ipi_bitmap_handler
|
||||
KMSAN_LEAVE
|
||||
jmp doreti
|
||||
|
||||
/*
|
||||
@ -182,15 +196,19 @@ IDTVEC(spuriousint)
|
||||
*/
|
||||
INTR_HANDLER cpustop
|
||||
call as_lapic_eoi
|
||||
KMSAN_ENTER
|
||||
call cpustop_handler
|
||||
KMSAN_LEAVE
|
||||
jmp doreti
|
||||
|
||||
/*
|
||||
* Executed by a CPU when it receives an IPI_SUSPEND from another CPU.
|
||||
*/
|
||||
INTR_HANDLER cpususpend
|
||||
KMSAN_ENTER
|
||||
call cpususpend_handler
|
||||
call as_lapic_eoi
|
||||
KMSAN_LEAVE
|
||||
jmp doreti
|
||||
|
||||
/*
|
||||
@ -198,7 +216,9 @@ IDTVEC(spuriousint)
|
||||
*/
|
||||
INTR_HANDLER ipi_swi
|
||||
call as_lapic_eoi
|
||||
KMSAN_ENTER
|
||||
call ipi_swi_handler
|
||||
KMSAN_LEAVE
|
||||
jmp doreti
|
||||
|
||||
/*
|
||||
@ -212,8 +232,10 @@ IDTVEC(spuriousint)
|
||||
movq ipi_rendezvous_counts(,%rax,8), %rax
|
||||
incq (%rax)
|
||||
#endif
|
||||
KMSAN_ENTER
|
||||
call smp_rendezvous_action
|
||||
call as_lapic_eoi
|
||||
KMSAN_LEAVE
|
||||
jmp doreti
|
||||
|
||||
/*
|
||||
|
@ -44,9 +44,11 @@
|
||||
*/
|
||||
.macro INTR irq_num, vec_name
|
||||
INTR_HANDLER \vec_name
|
||||
KMSAN_ENTER
|
||||
movq %rsp, %rsi
|
||||
movl $\irq_num, %edi /* pass the IRQ */
|
||||
call atpic_handle_intr
|
||||
KMSAN_LEAVE
|
||||
jmp doreti
|
||||
.endm
|
||||
|
||||
|
@ -282,8 +282,10 @@ alltraps_pushregs_no_rax:
|
||||
.globl calltrap
|
||||
.type calltrap,@function
|
||||
calltrap:
|
||||
movq %rsp,%rdi
|
||||
KMSAN_ENTER
|
||||
movq %rsp, %rdi
|
||||
call trap_check
|
||||
KMSAN_LEAVE
|
||||
jmp doreti /* Handle any pending ASTs */
|
||||
|
||||
/*
|
||||
@ -352,8 +354,10 @@ IDTVEC(dblfault)
|
||||
cmpq $~0,%rax
|
||||
je 2f
|
||||
movq %rax,%cr3
|
||||
2: movq %rsp,%rdi
|
||||
2: KMSAN_ENTER
|
||||
movq %rsp,%rdi
|
||||
call dblfault_handler
|
||||
KMSAN_LEAVE
|
||||
3: hlt
|
||||
jmp 3b
|
||||
|
||||
@ -856,8 +860,10 @@ nmi_fromuserspace:
|
||||
3:
|
||||
/* Note: this label is also used by ddb and gdb: */
|
||||
nmi_calltrap:
|
||||
KMSAN_ENTER
|
||||
movq %rsp,%rdi
|
||||
call trap
|
||||
KMSAN_LEAVE
|
||||
#ifdef HWPMC_HOOKS
|
||||
/*
|
||||
* Capture a userspace callchain if needed.
|
||||
@ -1043,8 +1049,10 @@ mchk_fromuserspace:
|
||||
1: call handle_ibrs_entry
|
||||
/* Note: this label is also used by ddb and gdb: */
|
||||
mchk_calltrap:
|
||||
KMSAN_ENTER
|
||||
movq %rsp,%rdi
|
||||
call mca_intr
|
||||
KMSAN_LEAVE
|
||||
testl %ebx,%ebx /* %ebx != 0 => return to userland */
|
||||
jnz doreti_exit
|
||||
/*
|
||||
|
@ -77,6 +77,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include <sys/lock.h>
|
||||
#include <sys/malloc.h>
|
||||
#include <sys/memrange.h>
|
||||
#include <sys/msan.h>
|
||||
#include <sys/msgbuf.h>
|
||||
#include <sys/mutex.h>
|
||||
#include <sys/pcpu.h>
|
||||
@ -1943,6 +1944,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
|
||||
thread0.td_critnest = 0;
|
||||
|
||||
kasan_init();
|
||||
kmsan_init();
|
||||
|
||||
TSEXIT();
|
||||
|
||||
|
@ -63,6 +63,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include <sys/kernel.h>
|
||||
#include <sys/ktr.h>
|
||||
#include <sys/lock.h>
|
||||
#include <sys/msan.h>
|
||||
#include <sys/mutex.h>
|
||||
#include <sys/resourcevar.h>
|
||||
#include <sys/signalvar.h>
|
||||
@ -229,6 +230,7 @@ trap(struct trapframe *frame)
|
||||
dr6 = 0;
|
||||
|
||||
kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
|
||||
kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
|
||||
|
||||
VM_CNT_INC(v_trap);
|
||||
type = frame->tf_trapno;
|
||||
@ -973,6 +975,7 @@ trap_user_dtrace(struct trapframe *frame, int (**hookp)(struct trapframe *))
|
||||
void
|
||||
dblfault_handler(struct trapframe *frame)
|
||||
{
|
||||
kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
|
||||
#ifdef KDTRACE_HOOKS
|
||||
if (dtrace_doubletrap_func != NULL)
|
||||
(*dtrace_doubletrap_func)();
|
||||
@ -1177,6 +1180,8 @@ amd64_syscall(struct thread *td, int traced)
|
||||
{
|
||||
ksiginfo_t ksi;
|
||||
|
||||
kmsan_mark(td->td_frame, sizeof(*td->td_frame), KMSAN_STATE_INITED);
|
||||
|
||||
#ifdef DIAGNOSTIC
|
||||
if (!TRAPF_USERMODE(td->td_frame)) {
|
||||
panic("syscall");
|
||||
|
@ -55,6 +55,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include <sys/kernel.h>
|
||||
#include <sys/ktr.h>
|
||||
#include <sys/lock.h>
|
||||
#include <sys/msan.h>
|
||||
#include <sys/mutex.h>
|
||||
#include <sys/proc.h>
|
||||
#include <sys/ptrace.h>
|
||||
@ -208,6 +209,8 @@ ia32_syscall(struct trapframe *frame)
|
||||
register_t orig_tf_rflags;
|
||||
ksiginfo_t ksi;
|
||||
|
||||
kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
|
||||
|
||||
orig_tf_rflags = frame->tf_rflags;
|
||||
td = curthread;
|
||||
td->td_frame = frame;
|
||||
|
@ -213,6 +213,28 @@ X\vec_name:
|
||||
movq TF_R15(%rsp),%r15
|
||||
.endm
|
||||
|
||||
#ifdef KMSAN
|
||||
/*
|
||||
* The KMSAN runtime relies on a TLS block to track initialization and origin
|
||||
* state for function parameters and return values. To keep this state
|
||||
* consistent in the face of asynchronous kernel-mode traps, the runtime
|
||||
* maintains a stack of blocks: when handling an exception or interrupt,
|
||||
* kmsan_intr_enter() pushes the new block to be used until the handler is
|
||||
* complete, at which point kmsan_intr_leave() restores the previous block.
|
||||
*
|
||||
* Thus, KMSAN_ENTER/LEAVE hooks are required only in handlers for events that
|
||||
* may have happened while in kernel-mode. In particular, they are not required
|
||||
* around amd64_syscall() or ast() calls. Otherwise, kmsan_intr_enter() can be
|
||||
* called unconditionally, without distinguishing between entry from user-mode
|
||||
* or kernel-mode.
|
||||
*/
|
||||
#define KMSAN_ENTER callq kmsan_intr_enter
|
||||
#define KMSAN_LEAVE callq kmsan_intr_leave
|
||||
#else
|
||||
#define KMSAN_ENTER
|
||||
#define KMSAN_LEAVE
|
||||
#endif
|
||||
|
||||
#endif /* LOCORE */
|
||||
|
||||
#ifdef __STDC__
|
||||
|
@ -1057,6 +1057,8 @@ fork_exit(void (*callout)(void *, struct trapframe *), void *arg,
|
||||
struct thread *td;
|
||||
struct thread *dtd;
|
||||
|
||||
kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
|
||||
|
||||
td = curthread;
|
||||
p = td->td_proc;
|
||||
KASSERT(p->p_state == PRS_NORMAL, ("executing process is still new"));
|
||||
|
@ -55,6 +55,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include <sys/capsicum.h>
|
||||
#include <sys/kernel.h>
|
||||
#include <sys/lock.h>
|
||||
#include <sys/msan.h>
|
||||
#include <sys/mutex.h>
|
||||
#include <sys/pmckern.h>
|
||||
#include <sys/proc.h>
|
||||
@ -216,6 +217,8 @@ ast(struct trapframe *framep)
|
||||
int flags, sig;
|
||||
bool resched_sigs;
|
||||
|
||||
kmsan_mark(framep, sizeof(*framep), KMSAN_STATE_INITED);
|
||||
|
||||
td = curthread;
|
||||
p = td->td_proc;
|
||||
|
||||
|
@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include <sys/kernel.h>
|
||||
#include <sys/lock.h>
|
||||
#include <sys/module.h>
|
||||
#include <sys/msan.h>
|
||||
|
||||
#include <machine/cpufunc.h>
|
||||
#include <machine/frame.h>
|
||||
@ -523,8 +524,8 @@ atpic_handle_intr(u_int vector, struct trapframe *frame)
|
||||
{
|
||||
struct intsrc *isrc;
|
||||
|
||||
/* The frame may have been written into a poisoned region. */
|
||||
kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
|
||||
kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
|
||||
|
||||
KASSERT(vector < NUM_ISA_IRQS, ("unknown int %u\n", vector));
|
||||
isrc = &atintrs[vector].at_intsrc;
|
||||
|
@ -48,6 +48,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include <sys/kernel.h>
|
||||
#include <sys/lock.h>
|
||||
#include <sys/malloc.h>
|
||||
#include <sys/msan.h>
|
||||
#include <sys/mutex.h>
|
||||
#include <sys/pcpu.h>
|
||||
#include <sys/proc.h>
|
||||
@ -1306,8 +1307,9 @@ lapic_handle_intr(int vector, struct trapframe *frame)
|
||||
{
|
||||
struct intsrc *isrc;
|
||||
|
||||
/* The frame may have been written into a poisoned region. */
|
||||
kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
|
||||
kmsan_mark(&vector, sizeof(vector), KMSAN_STATE_INITED);
|
||||
kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
|
||||
|
||||
isrc = intr_lookup_source(apic_idt_to_irq(PCPU_GET(apic_id),
|
||||
vector));
|
||||
@ -1324,8 +1326,8 @@ lapic_handle_timer(struct trapframe *frame)
|
||||
/* Send EOI first thing. */
|
||||
lapic_eoi();
|
||||
|
||||
/* The frame may have been written into a poisoned region. */
|
||||
kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
|
||||
kmsan_mark(frame, sizeof(*frame), KMSAN_STATE_INITED);
|
||||
|
||||
#if defined(SMP) && !defined(SCHED_ULE)
|
||||
/*
|
||||
|
Loading…
Reference in New Issue
Block a user