x86: Defer LAPIC calibration until after timecounters are available

This ensures that we have a good reference timecounter for performing
calibration.

Change lapic_setup to avoid configuring the timer when booting, and move
calibration and initial configuration to a new lapic routine,
lapic_calibrate_timer.  This calibration will be initiated from
cpu_initclocks(), before an eventtimer is selected.

Reviewed by:	kib, jhb
Sponsored by:	The FreeBSD Foundation

(cherry picked from commit 62d09b46ad)

sys/x86/include/apicvar.h

@@ -229,6 +229,9 @@ struct apic_ops {
 	void	(*disable_vector)(u_int, u_int);
 	void	(*free_vector)(u_int, u_int, u_int);
 
+	/* Timer */
+	void	(*calibrate_timer)(void);
+
 	/* PMC */
 	int	(*enable_pmc)(void);
 	void	(*disable_pmc)(void);
@@ -376,6 +379,13 @@ apic_free_vector(u_int apic_id, u_int vector, u_int irq)
 	apic_ops.free_vector(apic_id, vector, irq);
 }
 
+static inline void
+lapic_calibrate_timer(void)
+{
+
+	apic_ops.calibrate_timer();
+}
+
 static inline int
 lapic_enable_pmc(void)
 {

sys/x86/include/clock.h

@@ -27,6 +27,7 @@ extern int	smp_tsc;
 void	i8254_init(void);
 void	i8254_delay(int);
 void	clock_init(void);
+void	lapic_calibrate(void);
 
 /*
  * Driver to clock driver interface.

sys/x86/isa/clock.c

@@ -66,6 +66,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/intr_machdep.h>
 #include <machine/ppireg.h>
 #include <machine/timerreg.h>
+#include <x86/apicvar.h>
 #include <x86/init.h>
 
 #include <isa/rtc.h>
@@ -411,6 +412,8 @@ cpu_initclocks(void)
 	int i;
 
 	td = curthread;
+
+	lapic_calibrate_timer();
 	cpu_initclocks_bsp();
 	CPU_FOREACH(i) {
 		if (i == 0)
@@ -425,6 +428,7 @@ cpu_initclocks(void)
 		sched_unbind(td);
 	thread_unlock(td);
 #else
+	lapic_calibrate_timer();
 	cpu_initclocks_bsp();
 #endif
 }

sys/x86/x86/local_apic.c

@@ -214,7 +214,6 @@ SYSCTL_INT(_hw_apic, OID_AUTO, timer_tsc_deadline, CTLFLAG_RD,
     &lapic_timer_tsc_deadline, 0, "");
 
 static void lapic_calibrate_initcount(struct lapic *la);
-static void lapic_calibrate_deadline(struct lapic *la);
 
 /*
  * Use __nosanitizethread to exempt the LAPIC I/O accessors from KCSan
@@ -366,6 +365,7 @@ static void 	native_apic_enable_vector(u_int apic_id, u_int vector);
 static void 	native_apic_free_vector(u_int apic_id, u_int vector, u_int irq);
 static void 	native_lapic_set_logical_id(u_int apic_id, u_int cluster,
 		    u_int cluster_id);
+static void	native_lapic_calibrate_timer(void);
 static int 	native_lapic_enable_pmc(void);
 static void 	native_lapic_disable_pmc(void);
 static void 	native_lapic_reenable_pmc(void);
@@ -405,6 +405,7 @@ struct apic_ops apic_ops = {
 	.enable_vector		= native_apic_enable_vector,
 	.disable_vector		= native_apic_disable_vector,
 	.free_vector		= native_apic_free_vector,
+	.calibrate_timer	= native_lapic_calibrate_timer,
 	.enable_pmc		= native_lapic_enable_pmc,
 	.disable_pmc		= native_lapic_disable_pmc,
 	.reenable_pmc		= native_lapic_reenable_pmc,
@@ -797,21 +798,18 @@ native_lapic_setup(int boot)
 		    LAPIC_LVT_PCINT));
 	}
 
-	/* Program timer LVT. */
+	/*
+	 * Program the timer LVT.  Calibration is deferred until it is certain
+	 * that we have a reliable timecounter.
+	 */
 	la->lvt_timer_base = lvt_mode(la, APIC_LVT_TIMER,
 	    lapic_read32(LAPIC_LVT_TIMER));
 	la->lvt_timer_last = la->lvt_timer_base;
 	lapic_write32(LAPIC_LVT_TIMER, la->lvt_timer_base);
 
-	/* Calibrate the timer parameters using BSP. */
-	if (boot && IS_BSP()) {
-		lapic_calibrate_initcount(la);
-		if (lapic_timer_tsc_deadline)
-			lapic_calibrate_deadline(la);
-	}
-
-	/* Setup the timer if configured. */
-	if (la->la_timer_mode != LAT_MODE_UNDEF) {
+	if (boot)
+		la->la_timer_mode = LAT_MODE_UNDEF;
+	else if (la->la_timer_mode != LAT_MODE_UNDEF) {
 		KASSERT(la->la_timer_period != 0, ("lapic%u: zero divisor",
 		    lapic_id()));
 		switch (la->la_timer_mode) {
@@ -902,6 +900,25 @@ lapic_update_pmc(void *dummy)
 }
 #endif
 
+static void
+native_lapic_calibrate_timer(void)
+{
+	struct lapic *la;
+	register_t intr;
+
+	intr = intr_disable();
+	la = &lapics[lapic_id()];
+
+	lapic_calibrate_initcount(la);
+
+	intr_restore(intr);
+
+	if (lapic_timer_tsc_deadline && bootverbose) {
+		printf("lapic: deadline tsc mode, Frequency %ju Hz\n",
+		    (uintmax_t)tsc_freq);
+	}
+}
+
 static int
 native_lapic_enable_pmc(void)
 {
@@ -992,27 +1009,11 @@ lapic_calibrate_initcount(struct lapic *la)
 	count_freq = value;
 }
 
-static void
-lapic_calibrate_deadline(struct lapic *la __unused)
-{
-
-	if (bootverbose) {
-		printf("lapic: deadline tsc mode, Frequency %ju Hz\n",
-		    (uintmax_t)tsc_freq);
-	}
-}
-
 static void
 lapic_change_mode(struct eventtimer *et, struct lapic *la,
     enum lat_timer_mode newmode)
 {
-
-	/*
-	 * The TSC frequency may change during late calibration against other
-	 * timecounters (HPET or ACPI PMTimer).
-	 */
-	if (la->la_timer_mode == newmode &&
-	    (newmode != LAT_MODE_DEADLINE || et->et_frequency == tsc_freq))
+	if (la->la_timer_mode == newmode)
 		return;
 	switch (newmode) {
 	case LAT_MODE_PERIODIC:

sys/x86/xen/xen_apic.c

@@ -223,6 +223,12 @@ xen_pv_apic_free_vector(u_int apic_id, u_int vector, u_int irq)
 	XEN_APIC_UNSUPPORTED;
 }
 
+static void
+xen_pv_lapic_calibrate_timer(void)
+{
+
+}
+
 static void
 xen_pv_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id)
 {
@@ -420,6 +426,7 @@ struct apic_ops xen_apic_ops = {
 	.enable_vector		= xen_pv_apic_enable_vector,
 	.disable_vector		= xen_pv_apic_disable_vector,
 	.free_vector		= xen_pv_apic_free_vector,
+	.calibrate_timer	= xen_pv_lapic_calibrate_timer,
 	.enable_pmc		= xen_pv_lapic_enable_pmc,
 	.disable_pmc		= xen_pv_lapic_disable_pmc,
 	.reenable_pmc		= xen_pv_lapic_reenable_pmc,