softclock: Use dedicated ithreads for running callouts.

Rather than using the swi infrastructure, rewrite softclock() as a
thread loop (softclock_thread()) and use it as the main routine of the
softclock threads.  The threads use the CC_LOCK as the thread lock
when idle.

Reviewed by:	mav, imp, kib
Sponsored by:	Netflix
Differential Revision:	https://reviews.freebsd.org/D33683

sys/kern/kern_timeout.c

@@ -52,14 +52,17 @@ __FBSDID("$FreeBSD$");
 #include <sys/interrupt.h>
 #include <sys/kernel.h>
 #include <sys/ktr.h>
+#include <sys/kthread.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
+#include <sys/sched.h>
 #include <sys/sdt.h>
 #include <sys/sleepqueue.h>
 #include <sys/sysctl.h>
 #include <sys/smp.h>
+#include <sys/unistd.h>
 
 #ifdef DDB
 #include <ddb/ddb.h>
@@ -77,6 +80,8 @@ SDT_PROVIDER_DEFINE(callout_execute);
 SDT_PROBE_DEFINE1(callout_execute, , , callout__start, "struct callout *");
 SDT_PROBE_DEFINE1(callout_execute, , , callout__end, "struct callout *");
 
+static void	softclock_thread(void *arg);
+
 #ifdef CALLOUT_PROFILING
 static int avg_depth;
 SYSCTL_INT(_debug, OID_AUTO, to_avg_depth, CTLFLAG_RD, &avg_depth, 0,
@@ -166,7 +171,7 @@ struct callout_cpu {
 	struct callout_tailq	cc_expireq;
 	sbintime_t		cc_firstevent;
 	sbintime_t		cc_lastscan;
-	void			*cc_cookie;
+	struct thread		*cc_thread;
 	u_int			cc_bucket;
 	u_int			cc_inited;
 #ifdef KTR
@@ -222,7 +227,7 @@ static MALLOC_DEFINE(M_CALLOUT, "callout", "Callout datastructures");
  *                     relevant callout completes.
  *   cc_cancel       - Changing to 1 with both callout_lock and cc_lock held
  *                     guarantees that the current callout will not run.
- *                     The softclock() function sets this to 0 before it
+ *                     The softclock_call_cc() function sets this to 0 before it
  *                     drops callout_lock to acquire c_lock, and it calls
  *                     the handler only if curr_cancelled is still 0 after
  *                     cc_lock is successfully acquired.
@@ -316,7 +321,7 @@ callout_cpu_init(struct callout_cpu *cc, int cpu)
 {
 	int i;
 
-	mtx_init(&cc->cc_lock, "callout", NULL, MTX_SPIN | MTX_RECURSE);
+	mtx_init(&cc->cc_lock, "callout", NULL, MTX_SPIN);
 	cc->cc_inited = 1;
 	cc->cc_callwheel = malloc_domainset(sizeof(struct callout_list) *
 	    callwheelsize, M_CALLOUT,
@@ -369,28 +374,38 @@ callout_cpu_switch(struct callout *c, struct callout_cpu *cc, int new_cpu)
 static void
 start_softclock(void *dummy)
 {
+	struct proc *p;
+	struct thread *td;
 	struct callout_cpu *cc;
-	char name[MAXCOMLEN];
+	int cpu, error;
-	int cpu;
 	bool pin_swi;
-	struct intr_event *ie;
 
+	p = NULL;
 	CPU_FOREACH(cpu) {
 		cc = CC_CPU(cpu);
-		snprintf(name, sizeof(name), "clock (%d)", cpu);
+		error = kproc_kthread_add(softclock_thread, cc, &p, &td,
-		ie = NULL;
+		    RFSTOPPED, 0, "clock", "clock (%d)", cpu);
-		if (swi_add(&ie, name, softclock, cc, SWI_CLOCK,
+		if (error != 0)
-		    INTR_MPSAFE, &cc->cc_cookie))
+			panic("failed to create softclock thread for cpu %d: %d",
-			panic("died while creating standard software ithreads");
+			    cpu, error);
+		CC_LOCK(cc);
+		cc->cc_thread = td;
+		thread_lock(td);
+		sched_class(td, PRI_ITHD);
+		sched_prio(td, PI_SWI(SWI_CLOCK));
+		TD_SET_IWAIT(td);
+		thread_lock_set(td, (struct mtx *)&cc->cc_lock);
+		thread_unlock(td);
 		if (cpu == cc_default_cpu)
 			pin_swi = pin_default_swi;
 		else
 			pin_swi = pin_pcpu_swi;
-		if (pin_swi && (intr_event_bind(ie, cpu) != 0)) {
+		if (pin_swi) {
-			printf("%s: %s clock couldn't be pinned to cpu %d\n",
+			error = cpuset_setithread(td->td_tid, cpu);
-			    __func__,
+			if (error != 0)
-			    cpu == cc_default_cpu ? "default" : "per-cpu",
+				printf("%s: %s clock couldn't be pinned to cpu %d: %d\n",
-			    cpu);
+				    __func__, cpu == cc_default_cpu ?
+				    "default" : "per-cpu", cpu, error);
 		}
 	}
 }
@@ -418,6 +433,7 @@ callout_process(sbintime_t now)
 	struct callout *tmp, *tmpn;
 	struct callout_cpu *cc;
 	struct callout_list *sc;
+	struct thread *td;
 	sbintime_t first, last, max, tmp_max;
 	uint32_t lookahead;
 	u_int firstb, lastb, nowb;
@@ -529,13 +545,15 @@ callout_process(sbintime_t now)
 	avg_mpcalls_dir += (mpcalls_dir * 1000 - avg_mpcalls_dir) >> 8;
 	avg_lockcalls_dir += (lockcalls_dir * 1000 - avg_lockcalls_dir) >> 8;
 #endif
-	mtx_unlock_spin_flags(&cc->cc_lock, MTX_QUIET);
+	if (!TAILQ_EMPTY(&cc->cc_expireq)) {
-	/*
+		td = cc->cc_thread;
-	 * swi_sched acquires the thread lock, so we don't want to call it
+		if (TD_AWAITING_INTR(td)) {
-	 * with cc_lock held; incorrect locking order.
+			TD_CLR_IWAIT(td);
-	 */
+			sched_add(td, SRQ_INTR);
-	if (!TAILQ_EMPTY(&cc->cc_expireq))
+		} else
-		swi_sched(cc->cc_cookie, 0);
+			mtx_unlock_spin_flags(&cc->cc_lock, MTX_QUIET);
+	} else
+		mtx_unlock_spin_flags(&cc->cc_lock, MTX_QUIET);
 }
 
 static struct callout_cpu *
@@ -797,38 +815,57 @@ softclock_call_cc(struct callout *c, struct callout_cpu *cc,
  */
 
 /*
- * Software (low priority) clock interrupt.
+ * Software (low priority) clock interrupt thread handler.
  * Run periodic events from timeout queue.
  */
-void
+static void
-softclock(void *arg)
+softclock_thread(void *arg)
 {
+	struct thread *td = curthread;
 	struct callout_cpu *cc;
 	struct callout *c;
 #ifdef CALLOUT_PROFILING
-	int depth = 0, gcalls = 0, lockcalls = 0, mpcalls = 0;
+	int depth, gcalls, lockcalls, mpcalls;
 #endif
 
 	cc = (struct callout_cpu *)arg;
 	CC_LOCK(cc);
-	while ((c = TAILQ_FIRST(&cc->cc_expireq)) != NULL) {
+	for (;;) {
-		TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);
+		while (TAILQ_EMPTY(&cc->cc_expireq)) {
-		softclock_call_cc(c, cc,
+			/*
+			 * Use CC_LOCK(cc) as the thread_lock while
+			 * idle.
+			 */
+			thread_lock(td);
+			thread_lock_set(td, (struct mtx *)&cc->cc_lock);
+			TD_SET_IWAIT(td);
+			mi_switch(SW_VOL | SWT_IWAIT);
+
+			/* mi_switch() drops thread_lock(). */
+			CC_LOCK(cc);
+		}
+
 #ifdef CALLOUT_PROFILING
-		    &mpcalls, &lockcalls, &gcalls,
+		depth = gcalls = lockcalls = mpcalls = 0;
 #endif
-		    0);
+		while ((c = TAILQ_FIRST(&cc->cc_expireq)) != NULL) {
+			TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);
+			softclock_call_cc(c, cc,
 #ifdef CALLOUT_PROFILING
-		++depth;
+			    &mpcalls, &lockcalls, &gcalls,
+#endif
+			    0);
+#ifdef CALLOUT_PROFILING
+			++depth;
+#endif
+		}
+#ifdef CALLOUT_PROFILING
+		avg_depth += (depth * 1000 - avg_depth) >> 8;
+		avg_mpcalls += (mpcalls * 1000 - avg_mpcalls) >> 8;
+		avg_lockcalls += (lockcalls * 1000 - avg_lockcalls) >> 8;
+		avg_gcalls += (gcalls * 1000 - avg_gcalls) >> 8;
 #endif
 	}
-#ifdef CALLOUT_PROFILING
-	avg_depth += (depth * 1000 - avg_depth) >> 8;
-	avg_mpcalls += (mpcalls * 1000 - avg_mpcalls) >> 8;
-	avg_lockcalls += (lockcalls * 1000 - avg_lockcalls) >> 8;
-	avg_gcalls += (gcalls * 1000 - avg_gcalls) >> 8;
-#endif
-	CC_UNLOCK(cc);
 }
 
 void

sys/sys/systm.h

@@ -471,7 +471,6 @@ int	sysbeep(int hertz, sbintime_t duration);
 
 void	hardclock(int cnt, int usermode);
 void	hardclock_sync(int cpu);
-void	softclock(void *);
 void	statclock(int cnt, int usermode);
 void	profclock(int cnt, int usermode, uintfptr_t pc);