From 8d9888d37a63f1f8c8ffcc9dad4b47fa6bbff39a Mon Sep 17 00:00:00 2001
From: John Baldwin <jhb@FreeBSD.org>
Date: Tue, 28 Nov 2000 23:52:36 +0000
Subject: [PATCH] Don't wait forever for CPUs to stop or restart.  Instead,
 give up after a timeout.  If DIAGNOSTIC is turned on, then display a message
 to the console with a map of which CPUs failed to stop or restart.  This
 gives an SMP box at least a fighting chance of getting into DDB if one of the
 other CPUs has interrupts disabled.

---
 sys/amd64/amd64/mp_machdep.c | 21 +++++++++++++++++++--
 sys/amd64/amd64/mptable.c    | 21 +++++++++++++++++++--
 sys/amd64/include/mptable.h  | 21 +++++++++++++++++++--
 sys/i386/i386/mp_machdep.c   | 21 +++++++++++++++++++--
 sys/i386/i386/mptable.c      | 21 +++++++++++++++++++--
 sys/i386/include/mptable.h   | 21 +++++++++++++++++++--
 sys/kern/subr_smp.c          | 21 +++++++++++++++++++--
 7 files changed, 133 insertions(+), 14 deletions(-)

diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index 4788ee47e24f..0c1fc03c3f2d 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -2275,15 +2275,23 @@ invltlb(void)
 int
 stop_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	/* send the Xcpustop IPI to all CPUs in map */
 	selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
 	
-	while ((stopped_cpus & map) != map)
+	while (count++ < 100000 && (stopped_cpus & map) != map)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != map)
+		printf("Warning: CPUs 0x%x did not stop!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
@@ -2304,14 +2312,23 @@ stop_cpus(u_int map)
 int
 restart_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	started_cpus = map;		/* signal other cpus to restart */
 
-	while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */
+	/* wait for each to clear its bit */
+	while (count++ < 100000 && (stopped_cpus & map) != 0)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != 0)
+		printf("Warning: CPUs 0x%x did not restart!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
diff --git a/sys/amd64/amd64/mptable.c b/sys/amd64/amd64/mptable.c
index 4788ee47e24f..0c1fc03c3f2d 100644
--- a/sys/amd64/amd64/mptable.c
+++ b/sys/amd64/amd64/mptable.c
@@ -2275,15 +2275,23 @@ invltlb(void)
 int
 stop_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	/* send the Xcpustop IPI to all CPUs in map */
 	selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
 	
-	while ((stopped_cpus & map) != map)
+	while (count++ < 100000 && (stopped_cpus & map) != map)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != map)
+		printf("Warning: CPUs 0x%x did not stop!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
@@ -2304,14 +2312,23 @@ stop_cpus(u_int map)
 int
 restart_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	started_cpus = map;		/* signal other cpus to restart */
 
-	while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */
+	/* wait for each to clear its bit */
+	while (count++ < 100000 && (stopped_cpus & map) != 0)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != 0)
+		printf("Warning: CPUs 0x%x did not restart!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
diff --git a/sys/amd64/include/mptable.h b/sys/amd64/include/mptable.h
index 4788ee47e24f..0c1fc03c3f2d 100644
--- a/sys/amd64/include/mptable.h
+++ b/sys/amd64/include/mptable.h
@@ -2275,15 +2275,23 @@ invltlb(void)
 int
 stop_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	/* send the Xcpustop IPI to all CPUs in map */
 	selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
 	
-	while ((stopped_cpus & map) != map)
+	while (count++ < 100000 && (stopped_cpus & map) != map)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != map)
+		printf("Warning: CPUs 0x%x did not stop!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
@@ -2304,14 +2312,23 @@ stop_cpus(u_int map)
 int
 restart_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	started_cpus = map;		/* signal other cpus to restart */
 
-	while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */
+	/* wait for each to clear its bit */
+	while (count++ < 100000 && (stopped_cpus & map) != 0)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != 0)
+		printf("Warning: CPUs 0x%x did not restart!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index 4788ee47e24f..0c1fc03c3f2d 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -2275,15 +2275,23 @@ invltlb(void)
 int
 stop_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	/* send the Xcpustop IPI to all CPUs in map */
 	selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
 	
-	while ((stopped_cpus & map) != map)
+	while (count++ < 100000 && (stopped_cpus & map) != map)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != map)
+		printf("Warning: CPUs 0x%x did not stop!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
@@ -2304,14 +2312,23 @@ stop_cpus(u_int map)
 int
 restart_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	started_cpus = map;		/* signal other cpus to restart */
 
-	while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */
+	/* wait for each to clear its bit */
+	while (count++ < 100000 && (stopped_cpus & map) != 0)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != 0)
+		printf("Warning: CPUs 0x%x did not restart!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
diff --git a/sys/i386/i386/mptable.c b/sys/i386/i386/mptable.c
index 4788ee47e24f..0c1fc03c3f2d 100644
--- a/sys/i386/i386/mptable.c
+++ b/sys/i386/i386/mptable.c
@@ -2275,15 +2275,23 @@ invltlb(void)
 int
 stop_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	/* send the Xcpustop IPI to all CPUs in map */
 	selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
 	
-	while ((stopped_cpus & map) != map)
+	while (count++ < 100000 && (stopped_cpus & map) != map)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != map)
+		printf("Warning: CPUs 0x%x did not stop!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
@@ -2304,14 +2312,23 @@ stop_cpus(u_int map)
 int
 restart_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	started_cpus = map;		/* signal other cpus to restart */
 
-	while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */
+	/* wait for each to clear its bit */
+	while (count++ < 100000 && (stopped_cpus & map) != 0)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != 0)
+		printf("Warning: CPUs 0x%x did not restart!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
diff --git a/sys/i386/include/mptable.h b/sys/i386/include/mptable.h
index 4788ee47e24f..0c1fc03c3f2d 100644
--- a/sys/i386/include/mptable.h
+++ b/sys/i386/include/mptable.h
@@ -2275,15 +2275,23 @@ invltlb(void)
 int
 stop_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	/* send the Xcpustop IPI to all CPUs in map */
 	selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
 	
-	while ((stopped_cpus & map) != map)
+	while (count++ < 100000 && (stopped_cpus & map) != map)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != map)
+		printf("Warning: CPUs 0x%x did not stop!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
@@ -2304,14 +2312,23 @@ stop_cpus(u_int map)
 int
 restart_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	started_cpus = map;		/* signal other cpus to restart */
 
-	while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */
+	/* wait for each to clear its bit */
+	while (count++ < 100000 && (stopped_cpus & map) != 0)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != 0)
+		printf("Warning: CPUs 0x%x did not restart!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c
index 4788ee47e24f..0c1fc03c3f2d 100644
--- a/sys/kern/subr_smp.c
+++ b/sys/kern/subr_smp.c
@@ -2275,15 +2275,23 @@ invltlb(void)
 int
 stop_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	/* send the Xcpustop IPI to all CPUs in map */
 	selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
 	
-	while ((stopped_cpus & map) != map)
+	while (count++ < 100000 && (stopped_cpus & map) != map)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != map)
+		printf("Warning: CPUs 0x%x did not stop!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
@@ -2304,14 +2312,23 @@ stop_cpus(u_int map)
 int
 restart_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	started_cpus = map;		/* signal other cpus to restart */
 
-	while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */
+	/* wait for each to clear its bit */
+	while (count++ < 100000 && (stopped_cpus & map) != 0)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != 0)
+		printf("Warning: CPUs 0x%x did not restart!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }