zfs: enable SPA_PROCESS on the kernel side

The purpose of this change is to group kernelthreads specific to a
particular ZFS pool under a kernel process.  There can be many dozens of
threads per pool.  This change improves observability of those threads.

This change consists of several subchanges:
1. illumos taskq_create_proc can now pass its process parameter to
taskqueue.  Also, use zfsproc instead of NULL for taskq_create.  Caveat:
zfsproc might not be initialized yet.  But in that case it is still NULL,
so not worse than before.

2. illumos sys/proc.h: kthread id is stored in t_did field, not t_tid.

3. zfs: enable SPA_PROCESS on the kernel side.  The change is a bit hairy
as newproc() is implemented privately to spa.c.  I couldn't think of a
better way to populate process name than to poke inside the argument for
the process routine.

4. illumos thread_create: allow assigning thread to process other than
zfsproc.

5. zfs: expose spa_proc to other users, assign sync and quiesce threads
to it.

Pool-specific threads created using (relatively new) zthr mechanism are
still assigned to the zfskern process rather than to a respective
zpool-xxx process.  I am going to address this a bit later.

Reviewed by:	no one
MFC after:	5 weeks
Relnotes:	perhaps
Differential Revision: https://reviews.freebsd.org/D9720

sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c

@@ -62,9 +62,9 @@ system_taskq_fini(void *arg)
 }
 SYSUNINIT(system_taskq_fini, SI_SUB_CONFIGURE, SI_ORDER_ANY, system_taskq_fini, NULL);
 
-taskq_t *
-taskq_create(const char *name, int nthreads, pri_t pri, int minalloc __unused,
-    int maxalloc __unused, uint_t flags)
+static taskq_t *
+taskq_create_impl(const char *name, int nthreads, pri_t pri, proc_t *proc,
+    uint_t flags)
 {
 	taskq_t *tq;
 
@@ -74,17 +74,24 @@ taskq_create(const char *name, int nthreads, pri_t pri, int minalloc __unused,
 	tq = kmem_alloc(sizeof(*tq), KM_SLEEP);
 	tq->tq_queue = taskqueue_create(name, M_WAITOK, taskqueue_thread_enqueue,
 	    &tq->tq_queue);
-	(void) taskqueue_start_threads(&tq->tq_queue, nthreads, pri, "%s", name);
+	(void) taskqueue_start_threads_in_proc(&tq->tq_queue, nthreads, pri,
+	    proc, "%s", name);
 
 	return ((taskq_t *)tq);
 }
 
 taskq_t *
-taskq_create_proc(const char *name, int nthreads, pri_t pri, int minalloc,
-    int maxalloc, proc_t *proc __unused, uint_t flags)
+taskq_create(const char *name, int nthreads, pri_t pri, int minalloc __unused,
+    int maxalloc __unused, uint_t flags)
 {
+	return (taskq_create_impl(name, nthreads, pri, zfsproc, flags));
+}
 
-	return (taskq_create(name, nthreads, pri, minalloc, maxalloc, flags));
+taskq_t *
+taskq_create_proc(const char *name, int nthreads, pri_t pri, int minalloc,
+    int maxalloc, proc_t *proc, uint_t flags)
+{
+	return (taskq_create_impl(name, nthreads, pri, proc, flags));
 }
 
 void

sys/cddl/compat/opensolaris/sys/proc.h

@@ -47,12 +47,13 @@
 #define	maxclsyspri	PVM
 #define	max_ncpus	(mp_maxid + 1)
 #define	boot_max_ncpus	(mp_maxid + 1)
+#define	syscid		1
 
 #define	TS_RUN	0
 
 #define	p0	proc0
 
-#define	t_tid	td_tid
+#define	t_did	td_tid
 
 typedef	short		pri_t;
 typedef	struct thread	_kthread;
@@ -67,6 +68,7 @@ do_thread_create(caddr_t stk, size_t stksize, void (*proc)(void *), void *arg,
     size_t len, proc_t *pp, int state, pri_t pri)
 {
 	kthread_t *td = NULL;
+	proc_t **ppp;
 	int error;
 
 	/*
@@ -75,9 +77,13 @@ do_thread_create(caddr_t stk, size_t stksize, void (*proc)(void *), void *arg,
 	ASSERT(stk == NULL);
 	ASSERT(len == 0);
 	ASSERT(state == TS_RUN);
-	ASSERT(pp == &p0);
+	ASSERT(pp != NULL);
 
-	error = kproc_kthread_add(proc, arg, &zfsproc, &td, RFSTOPPED,
+	if (pp == &p0)
+		ppp = &zfsproc;
+	else
+		ppp = &pp;
+	error = kproc_kthread_add(proc, arg, ppp, &td, RFSTOPPED,
 	    stksize / PAGE_SIZE, "zfskern", "solthread %p", proc);
 	if (error == 0) {
 		thread_lock(td);

sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c

@@ -176,7 +176,11 @@ boolean_t	zio_taskq_sysdc = B_TRUE;	/* use SDC scheduling class */
 uint_t		zio_taskq_basedc = 80;		/* base duty cycle */
 #endif
 
+#ifdef _KERNEL
+#define SPA_PROCESS
+#endif
 boolean_t	spa_create_process = B_TRUE;	/* no process ==> no sysdc */
+
 extern int	zfs_sync_pass_deferred_free;
 
 /*
@@ -1090,23 +1094,49 @@ spa_create_zio_taskqs(spa_t *spa)
 	}
 }
 
-#ifdef _KERNEL
 #ifdef SPA_PROCESS
+static int
+newproc(void (*pc)(void *), void *arg, id_t cid, int pri,
+    void **ct, pid_t pid)
+{
+	va_list ap;
+	spa_t *spa = (spa_t *)arg;	/* XXX */
+	struct proc *newp;
+	struct thread *td;
+	int error;
+
+	ASSERT(ct == NULL);
+	ASSERT(pid == 0);
+	ASSERT(cid == syscid);
+
+	error = kproc_create(pc, arg, &newp, 0, 0, "zpool-%s", spa->spa_name);
+	if (error != 0)
+		return (error);
+	td = FIRST_THREAD_IN_PROC(newp);
+	thread_lock(td);
+	sched_prio(td, pri);
+	thread_unlock(td);
+	return (0);
+}
+
 static void
 spa_thread(void *arg)
 {
 	callb_cpr_t cprinfo;
 
 	spa_t *spa = arg;
+#ifdef illumos
 	user_t *pu = PTOU(curproc);
-
+#endif
 	CALLB_CPR_INIT(&cprinfo, &spa->spa_proc_lock, callb_generic_cpr,
 	    spa->spa_name);
 
 	ASSERT(curproc != &p0);
+#ifdef illumos
 	(void) snprintf(pu->u_psargs, sizeof (pu->u_psargs),
 	    "zpool-%s", spa->spa_name);
 	(void) strlcpy(pu->u_comm, pu->u_psargs, sizeof (pu->u_comm));
+#endif
 
 #ifdef PSRSET_BIND
 	/* bind this thread to the requested psrset */
@@ -1160,11 +1190,14 @@ spa_thread(void *arg)
 	cv_broadcast(&spa->spa_proc_cv);
 	CALLB_CPR_EXIT(&cprinfo);	/* drops spa_proc_lock */
 
+#ifdef illumos
 	mutex_enter(&curproc->p_lock);
 	lwp_exit();
+#else
+	kthread_exit();
+#endif
 }
 #endif	/* SPA_PROCESS */
-#endif
 
 /*
  * Activate an uninitialized pool.
@@ -1211,7 +1244,9 @@ spa_activate(spa_t *spa, int mode)
 	mutex_exit(&spa->spa_proc_lock);
 
 	/* If we didn't create a process, we need to create our taskqs. */
+#ifndef SPA_PROCESS
 	ASSERT(spa->spa_proc == &p0);
+#endif	/* SPA_PROCESS */
 	if (spa->spa_proc == &p0) {
 		spa_create_zio_taskqs(spa);
 	}
@@ -1315,6 +1350,7 @@ spa_deactivate(spa_t *spa)
 	mutex_exit(&spa->spa_proc_lock);
 
 #ifdef SPA_PROCESS
+#ifdef illumos
 	/*
 	 * We want to make sure spa_thread() has actually exited the ZFS
 	 * module, so that the module can't be unloaded out from underneath
@@ -1324,6 +1360,7 @@ spa_deactivate(spa_t *spa)
 		thread_join(spa->spa_did);
 		spa->spa_did = 0;
 	}
+#endif
 #endif	/* SPA_PROCESS */
 }
 

sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c

@@ -1935,6 +1935,12 @@ spa_deadman_synctime(spa_t *spa)
 	return (spa->spa_deadman_synctime);
 }
 
+struct proc *
+spa_proc(spa_t *spa)
+{
+	return (spa->spa_proc);
+}
+
 uint64_t
 dva_get_dsize_sync(spa_t *spa, const dva_t *dva)
 {

sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/spa.h

@@ -833,6 +833,7 @@ extern uint64_t spa_bootfs(spa_t *spa);
 extern uint64_t spa_delegation(spa_t *spa);
 extern objset_t *spa_meta_objset(spa_t *spa);
 extern uint64_t spa_deadman_synctime(spa_t *spa);
+extern struct proc *spa_proc(spa_t *spa);
 extern uint64_t spa_dirty_data(spa_t *spa);
 
 /* Miscellaneous support routines */

sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c

@@ -209,7 +209,7 @@ txg_sync_start(dsl_pool_t *dp)
 	tx->tx_threads = 2;
 
 	tx->tx_quiesce_thread = thread_create(NULL, 0, txg_quiesce_thread,
-	    dp, 0, &p0, TS_RUN, minclsyspri);
+	    dp, 0, spa_proc(dp->dp_spa), TS_RUN, minclsyspri);
 
 	/*
 	 * The sync thread can need a larger-than-default stack size on
@@ -217,7 +217,7 @@ txg_sync_start(dsl_pool_t *dp)
 	 * scrub_visitbp() recursion.
 	 */
 	tx->tx_sync_thread = thread_create(NULL, 32<<10, txg_sync_thread,
-	    dp, 0, &p0, TS_RUN, minclsyspri);
+	    dp, 0, spa_proc(dp->dp_spa), TS_RUN, minclsyspri);
 
 	mutex_exit(&tx->tx_sync_lock);
 }