diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S index 68a87d36d540..a005c3b70591 100644 --- a/sys/amd64/amd64/exception.S +++ b/sys/amd64/amd64/exception.S @@ -345,10 +345,11 @@ IDTVEC(dblfault) pushfq andq $~(PSL_D | PSL_AC),(%rsp) popfq - testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ - jz 1f /* already running with kernel GS.base */ - swapgs -1: lfence + movq TF_SIZE(%rsp),%rdx + movl %edx,%eax + shrq $32,%rdx + movl $MSR_GSBASE,%ecx + wrmsr movq %cr3,%rax movq %rax,PCPU(SAVED_UCR3) movq PCPU(KCR3),%rax diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c index fbc5342b4c47..0575fe5e9611 100644 --- a/sys/amd64/amd64/machdep.c +++ b/sys/amd64/amd64/machdep.c @@ -1575,7 +1575,9 @@ amd64_bsp_ist_init(struct pcpu *pc) tssp = &pc->pc_common_tss; /* doublefault stack space, runs on ist1 */ - tssp->tss_ist1 = (long)&dblfault_stack[sizeof(dblfault_stack)]; + np = ((struct nmi_pcpu *)&dblfault_stack[sizeof(dblfault_stack)]) - 1; + np->np_pcpu = (register_t)pc; + tssp->tss_ist1 = (long)np; /* * NMI stack, runs on ist2. The pcpu pointer is stored just diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c index 7ac40f507caa..9511a9c018c8 100644 --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -314,18 +314,24 @@ init_secondary(void) IOPERM_BITMAP_SIZE; pc->pc_common_tss.tss_rsp0 = 0; - pc->pc_common_tss.tss_ist1 = (long)&doublefault_stack[PAGE_SIZE]; + /* The doublefault stack runs on IST1. */ + np = ((struct nmi_pcpu *)&doublefault_stack[PAGE_SIZE]) - 1; + np->np_pcpu = (register_t)pc; + pc->pc_common_tss.tss_ist1 = (long)np; /* The NMI stack runs on IST2. */ np = ((struct nmi_pcpu *) &nmi_stack[PAGE_SIZE]) - 1; + np->np_pcpu = (register_t)pc; pc->pc_common_tss.tss_ist2 = (long)np; /* The MC# stack runs on IST3. */ np = ((struct nmi_pcpu *) &mce_stack[PAGE_SIZE]) - 1; + np->np_pcpu = (register_t)pc; pc->pc_common_tss.tss_ist3 = (long)np; /* The DB# stack runs on IST4. */ np = ((struct nmi_pcpu *) &dbg_stack[PAGE_SIZE]) - 1; + np->np_pcpu = (register_t)pc; pc->pc_common_tss.tss_ist4 = (long)np; /* Prepare private GDT */ @@ -341,18 +347,6 @@ init_secondary(void) ap_gdt.rd_base = (u_long)gdt; lgdt(&ap_gdt); /* does magic intra-segment return */ - /* Save the per-cpu pointer for use by the NMI handler. */ - np = ((struct nmi_pcpu *) &nmi_stack[PAGE_SIZE]) - 1; - np->np_pcpu = (register_t)pc; - - /* Save the per-cpu pointer for use by the MC# handler. */ - np = ((struct nmi_pcpu *) &mce_stack[PAGE_SIZE]) - 1; - np->np_pcpu = (register_t)pc; - - /* Save the per-cpu pointer for use by the DB# handler. */ - np = ((struct nmi_pcpu *) &dbg_stack[PAGE_SIZE]) - 1; - np->np_pcpu = (register_t)pc; - wrmsr(MSR_FSBASE, 0); /* User value */ wrmsr(MSR_GSBASE, (u_int64_t)pc); wrmsr(MSR_KGSBASE, (u_int64_t)pc); /* XXX User value while we're in the kernel */