diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c index 9d3bf8f4ff67..d1e4cb28752a 100644 --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -192,6 +192,10 @@ mp_topology(void) } +#ifdef KDB_STOP_NMI +volatile cpumask_t ipi_nmi_pending; +#endif + /* * Calculate usable address in base memory for AP trampoline code. */ @@ -972,6 +976,76 @@ ipi_self(u_int ipi) lapic_ipi_vectored(ipi, APIC_IPI_DEST_SELF); } +#ifdef KDB_STOP_NMI +/* + * send NMI IPI to selected CPUs + */ + +#define BEFORE_SPIN 1000000 + +void +ipi_nmi_selected(u_int32_t cpus) +{ + + int cpu; + register_t icrlo; + + icrlo = APIC_DELMODE_NMI | APIC_DESTMODE_PHY | APIC_LEVEL_ASSERT + | APIC_TRIGMOD_EDGE; + + CTR2(KTR_SMP, "%s: cpus: %x nmi", __func__, cpus); + + + atomic_set_int(&ipi_nmi_pending, cpus); + + + while ((cpu = ffs(cpus)) != 0) { + cpu--; + cpus &= ~(1 << cpu); + + KASSERT(cpu_apic_ids[cpu] != -1, + ("IPI NMI to non-existent CPU %d", cpu)); + + /* Wait for an earlier IPI to finish. */ + if (!lapic_ipi_wait(BEFORE_SPIN)) + panic("ipi_nmi_selected: previous IPI has not cleared"); + + lapic_ipi_raw(icrlo,cpu_apic_ids[cpu]); + } +} + + +int +ipi_nmi_handler() +{ + int cpu = PCPU_GET(cpuid); + + if(!(atomic_load_acq_int(&ipi_nmi_pending) & (1 << cpu))) + return 1; + + atomic_clear_int(&ipi_nmi_pending,1 << cpu); + + savectx(&stoppcbs[cpu]); + + /* Indicate that we are stopped */ + atomic_set_int(&stopped_cpus,1 << cpu); + + + /* Wait for restart */ + while(!(atomic_load_acq_int(&started_cpus) & (1 << cpu))) + ia32_pause(); + + atomic_clear_int(&started_cpus,1 << cpu); + atomic_clear_int(&stopped_cpus,1 << cpu); + + if(cpu == 0 && cpustop_restartfunc != NULL) + cpustop_restartfunc(); + + return 0; +} + +#endif /* KDB_STOP_NMI */ + /* * This is called once the rest of the system is up and running and we're * ready to let the AP's out of the pen. diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c index 7baaeee08b93..9820f871cbe0 100644 --- a/sys/amd64/amd64/trap.c +++ b/sys/amd64/amd64/trap.c @@ -165,6 +165,14 @@ trap(frame) PCPU_LAZY_INC(cnt.v_trap); type = frame.tf_trapno; +#ifdef KDB_STOP_NMI + /* Handler for NMI IPIs used for debugging */ + if (type == T_NMI) { + if (ipi_nmi_handler() == 0) + goto out; + } +#endif /* KDB_STOP_NMI */ + #ifdef KDB if (kdb_active) { kdb_reenter(); diff --git a/sys/amd64/conf/NOTES b/sys/amd64/conf/NOTES index 6677fbe4698b..8978c5508780 100644 --- a/sys/amd64/conf/NOTES +++ b/sys/amd64/conf/NOTES @@ -33,6 +33,12 @@ device mptable # Optional MPSPEC mptable support # options MP_WATCHDOG +# +# Debugging options. +# +options KDB_STOP_NMI # Stop CPUS using NMI instead of IPI + + ##################################################################### # CPU OPTIONS diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h index 8aea3b9998d5..2c63dc9ce1ca 100644 --- a/sys/amd64/include/smp.h +++ b/sys/amd64/include/smp.h @@ -63,6 +63,11 @@ void smp_masked_invlpg_range(u_int mask, vm_offset_t startva, void smp_invltlb(void); void smp_masked_invltlb(u_int mask); +#ifdef KDB_STOP_NMI +int ipi_nmi_handler(void); +void ipi_nmi_selected(u_int32_t cpus); +#endif + #endif /* !LOCORE */ #endif /* SMP */ diff --git a/sys/conf/options.amd64 b/sys/conf/options.amd64 index 82f864db04d8..0227bfd93a16 100644 --- a/sys/conf/options.amd64 +++ b/sys/conf/options.amd64 @@ -55,3 +55,6 @@ PSM_HOOKRESUME opt_psm.h PSM_RESETAFTERSUSPEND opt_psm.h PSM_DEBUG opt_psm.h DEV_ATPIC opt_atpic.h + +# Debugging +KDB_STOP_NMI opt_global.h diff --git a/sys/conf/options.i386 b/sys/conf/options.i386 index a1dfee63797d..405614fb1e1e 100644 --- a/sys/conf/options.i386 +++ b/sys/conf/options.i386 @@ -160,3 +160,6 @@ IPR_LOG opt_i4b.h DEV_APIC opt_apic.h DEV_NPX opt_npx.h ASR_COMPAT opt_asr.h + +# Debugging +KDB_STOP_NMI opt_global.h diff --git a/sys/i386/conf/NOTES b/sys/i386/conf/NOTES index 3c3489410dbb..82b2c6a69650 100644 --- a/sys/i386/conf/NOTES +++ b/sys/i386/conf/NOTES @@ -51,6 +51,11 @@ options MPTABLE_FORCE_HTT # Enable HTT CPUs with the MP Table # options MP_WATCHDOG +# Debugging options. +# +options KDB_STOP_NMI # Stop CPUS using NMI instead of IPI + + ##################################################################### # CPU OPTIONS diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c index d601f6dd4a45..2fc958c16eb9 100644 --- a/sys/i386/i386/mp_machdep.c +++ b/sys/i386/i386/mp_machdep.c @@ -175,6 +175,10 @@ vm_offset_t smp_tlb_addr1; vm_offset_t smp_tlb_addr2; volatile int smp_tlb_wait; +#ifdef KDB_STOP_NMI +volatile cpumask_t ipi_nmi_pending; +#endif + /* * Local data and functions. */ @@ -1191,6 +1195,76 @@ ipi_self(u_int ipi) lapic_ipi_vectored(ipi, APIC_IPI_DEST_SELF); } +#ifdef KDB_STOP_NMI +/* + * send NMI IPI to selected CPUs + */ + +#define BEFORE_SPIN 1000000 + +void +ipi_nmi_selected(u_int32_t cpus) +{ + + int cpu; + register_t icrlo; + + icrlo = APIC_DELMODE_NMI | APIC_DESTMODE_PHY | APIC_LEVEL_ASSERT + | APIC_TRIGMOD_EDGE; + + CTR2(KTR_SMP, "%s: cpus: %x nmi", __func__, cpus); + + + atomic_set_int(&ipi_nmi_pending, cpus); + + + while ((cpu = ffs(cpus)) != 0) { + cpu--; + cpus &= ~(1 << cpu); + + KASSERT(cpu_apic_ids[cpu] != -1, + ("IPI NMI to non-existent CPU %d", cpu)); + + /* Wait for an earlier IPI to finish. */ + if (!lapic_ipi_wait(BEFORE_SPIN)) + panic("ipi_nmi_selected: previous IPI has not cleared"); + + lapic_ipi_raw(icrlo,cpu_apic_ids[cpu]); + } +} + + +int +ipi_nmi_handler() +{ + int cpu = PCPU_GET(cpuid); + + if(!(atomic_load_acq_int(&ipi_nmi_pending) & (1 << cpu))) + return 1; + + atomic_clear_int(&ipi_nmi_pending,1 << cpu); + + savectx(&stoppcbs[cpu]); + + /* Indicate that we are stopped */ + atomic_set_int(&stopped_cpus,1 << cpu); + + + /* Wait for restart */ + while(!(atomic_load_acq_int(&started_cpus) & (1 << cpu))) + ia32_pause(); + + atomic_clear_int(&started_cpus,1 << cpu); + atomic_clear_int(&stopped_cpus,1 << cpu); + + if(cpu == 0 && cpustop_restartfunc != NULL) + cpustop_restartfunc(); + + return 0; +} + +#endif /* KDB_STOP_NMI */ + /* * This is called once the rest of the system is up and running and we're * ready to let the AP's out of the pen. diff --git a/sys/i386/i386/trap.c b/sys/i386/i386/trap.c index ae2b4ed21394..2bca06889a93 100644 --- a/sys/i386/i386/trap.c +++ b/sys/i386/i386/trap.c @@ -183,6 +183,14 @@ trap(frame) PCPU_LAZY_INC(cnt.v_trap); type = frame.tf_trapno; +#ifdef KDB_STOP_NMI + /* Handler for NMI IPIs used for debugging */ + if (type == T_NMI) { + if (ipi_nmi_handler() == 0) + goto out; + } +#endif /* KDB_STOP_NMI */ + #ifdef KDB if (kdb_active) { kdb_reenter(); diff --git a/sys/i386/include/smp.h b/sys/i386/include/smp.h index 7837120a7f2a..4c9ae76117a2 100644 --- a/sys/i386/include/smp.h +++ b/sys/i386/include/smp.h @@ -79,6 +79,11 @@ void smp_masked_invlpg_range(u_int mask, vm_offset_t startva, void smp_invltlb(void); void smp_masked_invltlb(u_int mask); +#ifdef KDB_STOP_NMI +int ipi_nmi_handler(void); +void ipi_nmi_selected(u_int32_t cpus); +#endif + #endif /* !LOCORE */ #endif /* SMP */ diff --git a/sys/kern/subr_kdb.c b/sys/kern/subr_kdb.c index 54274db67c42..5726a845f4d6 100644 --- a/sys/kern/subr_kdb.c +++ b/sys/kern/subr_kdb.c @@ -40,6 +40,18 @@ __FBSDID("$FreeBSD$"); #include #include +#ifdef KDB_STOP_NMI +#include +#endif + +/* + * KDB_STOP_NMI requires SMP to pick up the right dependencies + * (And isn't useful on UP anyway) + */ +#if defined(KDB_STOP_NMI) && !defined(SMP) +#error "options KDB_STOP_NMI" requires "options SMP" +#endif + int kdb_active = 0; void *kdb_jmpbufp = NULL; struct kdb_dbbe *kdb_dbbe = NULL; @@ -77,6 +89,19 @@ static int kdb_stop_cpus = 1; SYSCTL_INT(_debug_kdb, OID_AUTO, stop_cpus, CTLTYPE_INT | CTLFLAG_RW, &kdb_stop_cpus, 0, ""); TUNABLE_INT("debug.kdb.stop_cpus", &kdb_stop_cpus); + +#ifdef KDB_STOP_NMI +/* + * Provide an alternate method of stopping other CPUs. If another CPU has + * disabled interrupts the conventional STOP IPI will be blocked. This + * NMI-based stop should get through in that case. + */ +static int kdb_stop_cpus_with_nmi = 0; +SYSCTL_INT(_debug_kdb, OID_AUTO, stop_cpus_with_nmi, CTLTYPE_INT | CTLFLAG_RW, + &kdb_stop_cpus_with_nmi, 0, ""); +TUNABLE_INT("debug.kdb.stop_cpus_with_nmi", &kdb_stop_cpus_with_nmi); +#endif /* KDB_STOP_NMI */ + #endif static int @@ -308,9 +333,27 @@ kdb_reenter(void) struct pcb * kdb_thr_ctx(struct thread *thr) +#ifdef KDB_STOP_NMI +{ + u_int cpuid; + struct pcpu *pc; + + if (thr == curthread) + return &kdb_pcb; + + SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { + cpuid = pc->pc_cpuid; + if (pc->pc_curthread == thr && (atomic_load_acq_int(&stopped_cpus) & (1 << cpuid))) + return &stoppcbs[cpuid]; + } + + return thr->td_pcb; +} +#else { return ((thr == curthread) ? &kdb_pcb : thr->td_pcb); } +#endif /* KDB_STOP_NMI */ struct thread * kdb_thr_first(void) @@ -407,7 +450,14 @@ kdb_trap(int type, int code, struct trapframe *tf) #ifdef SMP if ((did_stop_cpus = kdb_stop_cpus) != 0) + { +#ifdef KDB_STOP_NMI + if(kdb_stop_cpus_with_nmi) + stop_cpus_nmi(PCPU_GET(other_cpus)); + else +#endif /* KDB_STOP_NMI */ stop_cpus(PCPU_GET(other_cpus)); + } #endif kdb_frame = tf; diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c index aed31182a0c0..3d08ad880f02 100644 --- a/sys/kern/subr_smp.c +++ b/sys/kern/subr_smp.c @@ -254,6 +254,35 @@ stop_cpus(cpumask_t map) return 1; } +#ifdef KDB_STOP_NMI +int +stop_cpus_nmi(cpumask_t map) +{ + int i; + + if (!smp_started) + return 0; + + CTR1(KTR_SMP, "stop_cpus(%x)", map); + + /* send the stop IPI to all CPUs in map */ + ipi_nmi_selected(map); + + i = 0; + while ((atomic_load_acq_int(&stopped_cpus) & map) != map) { + /* spin */ + i++; +#ifdef DIAGNOSTIC + if (i == 100000) { + printf("timeout stopping cpus\n"); + break; + } +#endif + } + + return 1; +} +#endif /* KDB_STOP_NMI */ /* * Called by a CPU to restart stopped CPUs. diff --git a/sys/sys/smp.h b/sys/sys/smp.h index 88353fcfee1b..88c6ae27d0b0 100644 --- a/sys/sys/smp.h +++ b/sys/sys/smp.h @@ -101,6 +101,10 @@ int restart_cpus(cpumask_t); int stop_cpus(cpumask_t); void smp_rendezvous_action(void); extern struct mtx smp_ipi_mtx; + +#ifdef KDB_STOP_NMI +int stop_cpus_nmi(cpumask_t); +#endif #endif /* SMP */ void smp_rendezvous(void (*)(void *), void (*)(void *),