Use NMIs to halt all cores when entering the debugger and resume all cores on exit.

sys/amd64/debug.c

@@ -9,20 +9,73 @@
 
 #include <machine/amd64.h>
 #include <machine/amd64op.h>
+#include <machine/atomic.h>
 #include <machine/trap.h>
+#include <machine/lapic.h>
+#include <machine/mp.h>
 
 TrapFrame *frames[MAX_CPUS];
 
+volatile static uint64_t debugLock = 0;
+volatile static uint64_t debugCmd = 0;
+volatile static uint64_t debugHalted = 0;
+
+void
+Debug_HaltCPUs()
+{
+    debugCmd = 0;
+    LAPIC_BroadcastNMI(T_DEBUGIPI);
+
+    // Wait for processors to enter
+    while (debugHalted < (MP_GetCPUs() - 1)) {
+	pause();
+    }
+}
+
+void
+Debug_ResumeCPUs()
+{
+    debugCmd = 1;
+
+    // Wait for processors to resume
+    while (debugHalted > 0) {
+	pause();
+    }
+}
+
+void
+Debug_HaltIPI()
+{
+    MP_SetState(CPUSTATE_HALTED);
+    __sync_fetch_and_add(&debugHalted, 1);
+
+    while (debugCmd == 0) {
+	pause();
+    }
+
+    __sync_fetch_and_sub(&debugHalted, 1);
+    MP_SetState(CPUSTATE_BOOTED);
+}
+
 void
 Debug_Breakpoint(TrapFrame *tf)
 {
     frames[CPU()] = tf;
 
-    // Stop all processors
+    // Should probably force all cores into the debugger
+    while(atomic_swap_uint64(&debugLock, 1) == 1) {
+	// Wait to acquire debugger lock
+    }
 
+    // Stop all processors
+    Debug_HaltCPUs();
+
+    // Enter prompt
     Debug_Prompt();
 
     // Resume all processors
+    Debug_ResumeCPUs();
+    atomic_set_uint64(&debugLock, 0);
 }
 
 static void

sys/amd64/include/lapic.h

@@ -10,6 +10,7 @@ uint32_t LAPIC_CPU();
 void LAPIC_SendEOI();
 void LAPIC_StartAP(uint8_t apicid, uint32_t addr);
 int LAPIC_Broadcast(int vector);
+int LAPIC_BroadcastNMI(int vector);
 void LAPIC_Periodic(uint64_t rate);
 
 #endif /* __LAPIC_H__ */

sys/amd64/include/mp.h

@@ -2,8 +2,15 @@
 #ifndef __MACHINE_MP_H__
 #define __MACHINE_MP_H__
 
+#define CPUSTATE_NOT_PRESENT	0
+#define CPUSTATE_BOOTED		1
+#define CPUSTATE_HALTED		2
+#define CPUSTATE_MAX		2
+
 void MP_Init();
 void MP_InitAP();
+void MP_SetState(int state);
+int MP_GetCPUs();
 
 /* Cross Calls */
 typedef int (*CrossCallCB)(void *);

sys/amd64/include/trap.h

@@ -41,6 +41,7 @@
 
 #define T_SYSCALL	60	/* System Call */
 #define T_CROSSCALL	61	/* Cross Call (IPI) */
+#define T_DEBUGIPI	62	/* Kernel Debugger Halt (IPI */
 
 #define T_UNKNOWN	63	/* Unknown Trap */
 

sys/amd64/lapic.c

@@ -34,6 +34,7 @@
 #define LAPIC_ICR_LO        0x0300 /* Interrupt Command Register */
 #define LAPIC_ICR_HI        0x0310 /* Interrupt Command Register */
 #define LAPIC_ICR_FIXED			0x0000 /* Delivery Mode */
+#define LAPIC_ICR_NMI			0x0400
 #define LAPIC_ICR_INIT			0x0500
 #define LAPIC_ICR_STARTUP		0x0600
 #define LAPIC_ICR_ASSERT		0x4000
@@ -156,6 +157,24 @@ LAPIC_Broadcast(int vector)
     return 0;
 }
 
+int
+LAPIC_BroadcastNMI(int vector)
+{
+    int i = 0;
+    LAPIC_Write(LAPIC_ICR_LO, LAPIC_ICR_EXCSELF | LAPIC_ICR_NMI | vector);
+
+    while ((LAPIC_Read(LAPIC_ICR_LO) & LAPIC_ICR_DELIVERY_PENDING) != 0) {
+	pause();
+
+	if (i > 1000000) {
+	    kprintf("IPI not delivered?\n");
+	    return -1;
+	}
+    }
+
+    return 0;
+}
+
 void
 LAPIC_Init()
 {

sys/amd64/mp.c

@@ -32,8 +32,11 @@ typedef struct CrossCallFrame {
     volatile int	status[MAX_CPUS];
 } CrossCallFrame;
 
-#define CPUSTATE_NOT_PRESENT	0
+const char *CPUStateToString[] = {
-#define CPUSTATE_BOOTED		1
+    "NOT PRESENT",
+    "BOOTED",
+    "HALTED",
+};
 
 typedef struct CPUState {
     int			state;
@@ -83,6 +86,7 @@ MPBootAP(int procNo)
 
     return 0;
 }
+
 void
 MP_Init()
 {
@@ -118,6 +122,19 @@ MP_InitAP()
     booted = 1;
 }
 
+void
+MP_SetState(int state)
+{
+    ASSERT(state > 0 && state <= CPUSTATE_MAX);
+    cpus[CPU()].state = state;
+}
+
+int
+MP_GetCPUs()
+{
+    return lastCPU;
+}
+
 void
 MP_CrossCallTrap()
 {
@@ -217,7 +234,7 @@ Debug_CPUS(int argc, const char *argv[])
 
     for (c = 0; c < MAX_CPUS; c++) {
 	if (cpus[c].state != CPUSTATE_NOT_PRESENT) {
-	    kprintf("CPU %d: BOOTED\n", c);
+	    kprintf("CPU %d: %s\n", c, CPUStateToString[cpus[c].state]);
 	}
     }
 }

sys/amd64/trap.c

@@ -22,6 +22,7 @@
 extern uint64_t trap_table[T_MAX];
 extern void trap_pop(TrapFrame *tf);
 extern void Debug_Breakpoint(TrapFrame *tf);
+extern void Debug_HaltIPI(TrapFrame *tf);
 extern void KTimer_Process();
 
 static InteruptGate64 idt[256];
@@ -147,12 +148,18 @@ trap_entry(TrapFrame *tf)
     // XXX: USE ATOMIC!
     intStats[tf->vector]++;
 
+    // Debug NMI (should be T_DEBUGIPI)
+    if (tf->vector == T_NMI) {
+	Debug_HaltIPI(tf);
+	LAPIC_SendEOI();
+	return;
+    }
+
     // Kernel
     if (tf->cs == SEL_KCS)
     {
 	// Kernel Debugger
-	if (tf->vector == T_BP || tf->vector == T_DE)
+	if (tf->vector == T_BP || tf->vector == T_DE) {
-	{
 	    Debug_Breakpoint(tf);
 	    return;
 	}

sys/kern/debug.c

@@ -251,6 +251,7 @@ Debug_Prompt()
     char *argv[DEBUG_MAX_ARGS];
     char *nextArg;
     char buf[DEBUG_MAX_LINE];
+
     kprintf("Entered Debugger!\n");
 
     /*