- Implement run-time expansion of the KTR buffer via sysctl.

- Implement a function to ensure that all preempted threads have switched
   back out at least once.  Use this to make sure there are no stale
   references to the old ktr_buf or the lock profiling buffers before
   updating them.

Reviewed by:	marius (sparc64 parts), attilio (earlier patch)
Sponsored by:	EMC / Isilon Storage Division

sys/kern/kern_ktr.c

@@ -47,7 +47,11 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/ktr.h>
 #include <sys/libkern.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/mutex.h>
 #include <sys/proc.h>
+#include <sys/smp.h>
 #include <sys/sysctl.h>
 #include <sys/systm.h>
 #include <sys/time.h>
@@ -66,6 +70,9 @@ __FBSDID("$FreeBSD$");
 #define	KTR_ENTRIES	1024
 #endif
 
+/* Limit the allocations to something manageable. */
+#define	KTR_ENTRIES_MAX	(8 * 1024 * 1024)
+
 #ifndef KTR_MASK
 #define	KTR_MASK	(0)
 #endif
@@ -82,30 +89,31 @@ __FBSDID("$FreeBSD$");
 #define	KTR_CPU		PCPU_GET(cpuid)
 #endif
 
+static MALLOC_DEFINE(M_KTR, "KTR", "KTR");
+
 FEATURE(ktr, "Kernel support for KTR kernel tracing facility");
 
+volatile int	ktr_idx = 0;
+int	ktr_mask = KTR_MASK;
+int	ktr_compile = KTR_COMPILE;
+int	ktr_entries = KTR_ENTRIES;
+int	ktr_version = KTR_VERSION;
+struct	ktr_entry ktr_buf_init[KTR_ENTRIES];
+struct	ktr_entry *ktr_buf = ktr_buf_init;
+cpuset_t ktr_cpumask = CPUSET_T_INITIALIZER(KTR_CPUMASK);
+static char ktr_cpumask_str[CPUSETBUFSIZ];
+
+TUNABLE_INT("debug.ktr.mask", &ktr_mask);
+
+TUNABLE_STR("debug.ktr.cpumask", ktr_cpumask_str, sizeof(ktr_cpumask_str));
+
 static SYSCTL_NODE(_debug, OID_AUTO, ktr, CTLFLAG_RD, 0, "KTR options");
 
-int	ktr_mask = KTR_MASK;
-TUNABLE_INT("debug.ktr.mask", &ktr_mask);
-SYSCTL_INT(_debug_ktr, OID_AUTO, mask, CTLFLAG_RW,
-    &ktr_mask, 0, "Bitmask of KTR event classes for which logging is enabled");
-
-int	ktr_compile = KTR_COMPILE;
-SYSCTL_INT(_debug_ktr, OID_AUTO, compile, CTLFLAG_RD,
-    &ktr_compile, 0, "Bitmask of KTR event classes compiled into the kernel");
-
-int	ktr_entries = KTR_ENTRIES;
-SYSCTL_INT(_debug_ktr, OID_AUTO, entries, CTLFLAG_RD,
-    &ktr_entries, 0, "Number of entries in the KTR buffer");
-
-int	ktr_version = KTR_VERSION;
 SYSCTL_INT(_debug_ktr, OID_AUTO, version, CTLFLAG_RD,
     &ktr_version, 0, "Version of the KTR interface");
 
-cpuset_t ktr_cpumask = CPUSET_T_INITIALIZER(KTR_CPUMASK);
-static char ktr_cpumask_str[CPUSETBUFSIZ];
-TUNABLE_STR("debug.ktr.cpumask", ktr_cpumask_str, sizeof(ktr_cpumask_str));
+SYSCTL_INT(_debug_ktr, OID_AUTO, compile, CTLFLAG_RD,
+    &ktr_compile, 0, "Bitmask of KTR event classes compiled into the kernel");
 
 static void
 ktr_cpumask_initializer(void *dummy __unused)
@@ -145,9 +153,6 @@ SYSCTL_PROC(_debug_ktr, OID_AUTO, cpumask,
     sysctl_debug_ktr_cpumask, "S",
     "Bitmask of CPUs on which KTR logging is enabled");
 
-volatile int	ktr_idx = 0;
-struct	ktr_entry ktr_buf[KTR_ENTRIES];
-
 static int
 sysctl_debug_ktr_clear(SYSCTL_HANDLER_ARGS)
 {
@@ -159,7 +164,7 @@ sysctl_debug_ktr_clear(SYSCTL_HANDLER_ARGS)
 		return (error);
 
 	if (clear) {
-		bzero(ktr_buf, sizeof(ktr_buf));
+		bzero(ktr_buf, sizeof(*ktr_buf) * ktr_entries);
 		ktr_idx = 0;
 	}
 
@@ -168,6 +173,67 @@ sysctl_debug_ktr_clear(SYSCTL_HANDLER_ARGS)
 SYSCTL_PROC(_debug_ktr, OID_AUTO, clear, CTLTYPE_INT|CTLFLAG_RW, 0, 0,
     sysctl_debug_ktr_clear, "I", "Clear KTR Buffer");
 
+/*
+ * This is a sysctl proc so that it is serialized as !MPSAFE along with
+ * the other ktr sysctl procs.
+ */
+static int
+sysctl_debug_ktr_mask(SYSCTL_HANDLER_ARGS)
+{
+	int mask, error;
+
+	mask = ktr_mask;
+	error = sysctl_handle_int(oidp, &mask, 0, req);
+	if (error || !req->newptr)
+		return (error);
+	ktr_mask = mask;
+	return (error);
+}
+
+SYSCTL_PROC(_debug_ktr, OID_AUTO, mask, CTLTYPE_INT|CTLFLAG_RW, 0, 0,
+    sysctl_debug_ktr_mask, "I",
+    "Bitmask of KTR event classes for which logging is enabled");
+
+static int
+sysctl_debug_ktr_entries(SYSCTL_HANDLER_ARGS)
+{
+	int entries, error, mask;
+	struct ktr_entry *buf, *oldbuf;
+
+	entries = ktr_entries;
+	error = sysctl_handle_int(oidp, &entries, 0, req);
+	if (error || !req->newptr)
+		return (error);
+	if (entries > KTR_ENTRIES_MAX)
+		return (ERANGE);
+	/* Disable ktr temporarily. */
+	mask = ktr_mask;
+	atomic_store_rel_int(&ktr_mask, 0);
+	/* Wait for threads to go idle. */
+	if ((error = quiesce_all_cpus("ktrent", PCATCH)) != 0) {
+		ktr_mask = mask;
+		return (error);
+	}
+	if (ktr_buf != ktr_buf_init)
+		oldbuf = ktr_buf;
+	else
+		oldbuf = NULL;
+	/* Allocate a new buffer. */
+	buf = malloc(sizeof(*buf) * entries, M_KTR, M_WAITOK | M_ZERO);
+	/* Install the new buffer and restart ktr. */
+	ktr_buf = buf;
+	ktr_entries = entries;
+	ktr_idx = 0;
+	atomic_store_rel_int(&ktr_mask, mask);
+	if (oldbuf != NULL)
+		free(oldbuf, M_KTR);
+
+	return (error);
+}
+
+SYSCTL_PROC(_debug_ktr, OID_AUTO, entries, CTLTYPE_INT|CTLFLAG_RW, 0, 0,
+    sysctl_debug_ktr_entries, "I", "Number of entries in the KTR buffer");
+
 #ifdef KTR_VERBOSE
 int	ktr_verbose = KTR_VERBOSE;
 TUNABLE_INT("debug.ktr.verbose", &ktr_verbose);
@@ -249,7 +315,7 @@ ktr_tracepoint(u_int mask, const char *file, int line, const char *format,
 
 	if (panicstr)
 		return;
-	if ((ktr_mask & mask) == 0)
+	if ((ktr_mask & mask) == 0 || ktr_buf == NULL)
 		return;
 	cpu = KTR_CPU;
 	if (!CPU_ISSET(cpu, &ktr_cpumask))
@@ -281,7 +347,7 @@ ktr_tracepoint(u_int mask, const char *file, int line, const char *format,
 	{
 		do {
 			saveindex = ktr_idx;
-			newindex = (saveindex + 1) % KTR_ENTRIES;
+			newindex = (saveindex + 1) % ktr_entries;
 		} while (atomic_cmpset_rel_int(&ktr_idx, saveindex, newindex) == 0);
 		entry = &ktr_buf[saveindex];
 	}
@@ -336,7 +402,7 @@ static	int db_mach_vtrace(void);
 DB_SHOW_COMMAND(ktr, db_ktr_all)
 {
 	
-	tstate.cur = (ktr_idx - 1) % KTR_ENTRIES;
+	tstate.cur = (ktr_idx - 1) % ktr_entries;
 	tstate.first = -1;
 	db_ktr_verbose = 0;
 	db_ktr_verbose |= (strchr(modif, 'v') != NULL) ? 2 : 0;
@@ -358,7 +424,7 @@ db_mach_vtrace(void)
 {
 	struct ktr_entry	*kp;
 
-	if (tstate.cur == tstate.first) {
+	if (tstate.cur == tstate.first || ktr_buf == NULL) {
 		db_printf("--- End of trace buffer ---\n");
 		return (0);
 	}
@@ -390,7 +456,7 @@ db_mach_vtrace(void)
 		tstate.first = tstate.cur;
 
 	if (--tstate.cur < 0)
-		tstate.cur = KTR_ENTRIES - 1;
+		tstate.cur = ktr_entries - 1;
 
 	return (1);
 }

sys/kern/subr_lock.c

@@ -240,34 +240,13 @@ lock_prof_init(void *arg)
 }
 SYSINIT(lockprof, SI_SUB_SMP, SI_ORDER_ANY, lock_prof_init, NULL);
 
-/*
- * To be certain that lock profiling has idled on all cpus before we
- * reset, we schedule the resetting thread on all active cpus.  Since
- * all operations happen within critical sections we can be sure that
- * it is safe to zero the profiling structures.
- */
-static void
-lock_prof_idle(void)
-{
-	struct thread *td;
-	int cpu;
-
-	td = curthread;
-	thread_lock(td);
-	CPU_FOREACH(cpu) {
-		sched_bind(td, cpu);
-	}
-	sched_unbind(td);
-	thread_unlock(td);
-}
-
 static void
 lock_prof_reset_wait(void)
 {
 
 	/*
-	 * Spin relinquishing our cpu so that lock_prof_idle may
-	 * run on it.
+	 * Spin relinquishing our cpu so that quiesce_all_cpus may
+	 * complete.
 	 */
 	while (lock_prof_resetting)
 		sched_relinquish(curthread);
@@ -289,7 +268,7 @@ lock_prof_reset(void)
 	atomic_store_rel_int(&lock_prof_resetting, 1);
 	enabled = lock_prof_enable;
 	lock_prof_enable = 0;
-	lock_prof_idle();
+	quiesce_all_cpus("profreset", 0);
 	/*
 	 * Some objects may have migrated between CPUs.  Clear all links
 	 * before we zero the structures.  Some items may still be linked
@@ -401,7 +380,7 @@ dump_lock_prof_stats(SYSCTL_HANDLER_ARGS)
 	    "max", "wait_max", "total", "wait_total", "count", "avg", "wait_avg", "cnt_hold", "cnt_lock", "name");
 	enabled = lock_prof_enable;
 	lock_prof_enable = 0;
-	lock_prof_idle();
+	quiesce_all_cpus("profstat", 0);
 	t = ticks;
 	for (cpu = 0; cpu <= mp_maxid; cpu++) {
 		if (lp_cpu[cpu] == NULL)

sys/kern/subr_smp.c

@@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/pcpu.h>
+#include <sys/sched.h>
 #include <sys/smp.h>
 #include <sys/sysctl.h>
 
@@ -734,3 +735,52 @@ smp_no_rendevous_barrier(void *dummy)
 	KASSERT((!smp_started),("smp_no_rendevous called and smp is started"));
 #endif
 }
+
+/*
+ * Wait specified idle threads to switch once.  This ensures that even
+ * preempted threads have cycled through the switch function once,
+ * exiting their codepaths.  This allows us to change global pointers
+ * with no other synchronization.
+ */
+int
+quiesce_cpus(cpuset_t map, const char *wmesg, int prio)
+{
+	struct pcpu *pcpu;
+	u_int gen[MAXCPU];
+	int error;
+	int cpu;
+
+	error = 0;
+	for (cpu = 0; cpu <= mp_maxid; cpu++) {
+		if (!CPU_ISSET(cpu, &map) || CPU_ABSENT(cpu))
+			continue;
+		pcpu = pcpu_find(cpu);
+		gen[cpu] = pcpu->pc_idlethread->td_generation;
+	}
+	for (cpu = 0; cpu <= mp_maxid; cpu++) {
+		if (!CPU_ISSET(cpu, &map) || CPU_ABSENT(cpu))
+			continue;
+		pcpu = pcpu_find(cpu);
+		thread_lock(curthread);
+		sched_bind(curthread, cpu);
+		thread_unlock(curthread);
+		while (gen[cpu] == pcpu->pc_idlethread->td_generation) {
+			error = tsleep(quiesce_cpus, prio, wmesg, 1);
+			if (error)
+				goto out;
+		}
+	}
+out:
+	thread_lock(curthread);
+	sched_unbind(curthread);
+	thread_unlock(curthread);
+
+	return (error);
+}
+
+int
+quiesce_all_cpus(const char *wmesg, int prio)
+{
+
+	return quiesce_cpus(all_cpus, wmesg, prio);
+}

sys/sparc64/include/ktr.h

@@ -58,6 +58,7 @@ l2:	add	r2, 1, r3 ; \
 	bne	%icc, l2 ## b ; \
 	 mov	r3, r2 ; \
 	SET(ktr_buf, r3, r1) ; \
+	ldx	[r1], r1 ; \
 	mulx	r2, KTR_SIZEOF, r2 ; \
 	add	r1, r2, r1 ; \
 	rd	%tick, r2 ; \

sys/sys/ktr.h

@@ -116,7 +116,7 @@ extern int ktr_entries;
 extern int ktr_verbose;
 
 extern volatile int ktr_idx;
-extern struct ktr_entry ktr_buf[];
+extern struct ktr_entry *ktr_buf;
 
 #ifdef KTR
 

sys/sys/smp.h

@@ -167,10 +167,14 @@ int	stop_cpus_hard(cpuset_t);
 #if defined(__amd64__) || defined(__i386__)
 int	suspend_cpus(cpuset_t);
 #endif
+
 void	smp_rendezvous_action(void);
 extern	struct mtx smp_ipi_mtx;
 
 #endif /* SMP */
+
+int	quiesce_all_cpus(const char *, int);
+int	quiesce_cpus(cpuset_t, const char *, int);
 void	smp_no_rendevous_barrier(void *);
 void	smp_rendezvous(void (*)(void *), 
 		       void (*)(void *),

usr.bin/ktrdump/ktrdump.c

@@ -86,6 +86,7 @@ main(int ac, char **av)
 	u_long parms[KTR_PARMS];
 	struct ktr_entry *buf;
 	uintmax_t tlast, tnow;
+	unsigned long bufptr;
 	struct stat sb;
 	kvm_t *kd;
 	FILE *out;
@@ -179,8 +180,9 @@ main(int ac, char **av)
 		if ((buf = malloc(sizeof(*buf) * entries)) == NULL)
 			err(1, NULL);
 		if (kvm_read(kd, nl[2].n_value, &index, sizeof(index)) == -1 ||
-		    kvm_read(kd, nl[3].n_value, buf, sizeof(*buf) * entries)
-		    == -1)
+		    kvm_read(kd, nl[3].n_value, &bufptr,
+		    sizeof(bufptr)) == -1 ||
+		    kvm_read(kd, bufptr, buf, sizeof(*buf) * entries) == -1)
 			errx(1, "%s", kvm_geterr(kd));
 	}