Add a taskqueue_quiesce(9) KPI.

This is similar to taskqueue_drain_all(9) but will wait for the queue
to become idle before returning instead of only waiting for
already-enqueued tasks to finish.  This will be used in the opensolaris
compat layer.

PR:		227784
Reviewed by:	cem
MFC after:	1 week
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D17975

share/man/man9/Makefile

@@ -2056,6 +2056,7 @@ MLINKS+=taskqueue.9 TASK_INIT.9 \
 	taskqueue.9 TASKQUEUE_FAST_DEFINE_THREAD.9 \
 	taskqueue.9 taskqueue_free.9 \
 	taskqueue.9 taskqueue_member.9 \
+	taskqueue.9 taskqueue_quiesce.9 \
 	taskqueue.9 taskqueue_run.9 \
 	taskqueue.9 taskqueue_set_callback.9 \
 	taskqueue.9 taskqueue_start_threads.9 \

share/man/man9/taskqueue.9

@@ -28,7 +28,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd July 30, 2017
+.Dd November 21, 2018
 .Dt TASKQUEUE 9
 .Os
 .Sh NAME
@@ -94,6 +94,8 @@ struct timeout_task;
 .Ft void
 .Fn taskqueue_drain_all "struct taskqueue *queue"
 .Ft void
+.Fn taskqueue_quiesce "struct taskqueue *queue"
+.Ft void
 .Fn taskqueue_block "struct taskqueue *queue"
 .Ft void
 .Fn taskqueue_unblock "struct taskqueue *queue"
@@ -298,6 +300,12 @@ do not extend the wait time of
 and may complete after
 .Fn taskqueue_drain_all
 returns.
+The
+.Fn taskqueue_quiesce
+function is used to wait for the queue to become empty and for all
+running tasks to finish.
+To avoid blocking indefinitely, the caller must ensure by some mechanism
+that tasks will eventually stop being posted to the queue.
 .Pp
 The
 .Fn taskqueue_block

sys/kern/subr_taskqueue.c

@@ -346,13 +346,13 @@ taskqueue_task_nop_fn(void *context, int pending)
  * have begun execution.  Tasks queued during execution of
  * this function are ignored.
  */
-static void
+static int
 taskqueue_drain_tq_queue(struct taskqueue *queue)
 {
 	struct task t_barrier;
 
 	if (STAILQ_EMPTY(&queue->tq_queue))
-		return;
+		return (0);
 
 	/*
 	 * Enqueue our barrier after all current tasks, but with
@@ -372,6 +372,7 @@ taskqueue_drain_tq_queue(struct taskqueue *queue)
 	 */
 	while (t_barrier.ta_pending != 0)
 		TQ_SLEEP(queue, &t_barrier, &queue->tq_mutex, PWAIT, "-", 0);
+	return (1);
 }
 
 /*
@@ -379,13 +380,13 @@ taskqueue_drain_tq_queue(struct taskqueue *queue)
  * complete.  Tasks that begin execution during the execution
  * of this function are ignored.
  */
-static void
+static int
 taskqueue_drain_tq_active(struct taskqueue *queue)
 {
 	struct taskqueue_busy tb_marker, *tb_first;
 
 	if (TAILQ_EMPTY(&queue->tq_active))
-		return;
+		return (0);
 
 	/* Block taskq_terminate().*/
 	queue->tq_callouts++;
@@ -412,6 +413,7 @@ taskqueue_drain_tq_active(struct taskqueue *queue)
 	queue->tq_callouts--;
 	if ((queue->tq_flags & TQ_FLAGS_ACTIVE) == 0)
 		wakeup_one(queue->tq_threads);
+	return (1);
 }
 
 void
@@ -582,8 +584,8 @@ taskqueue_drain_all(struct taskqueue *queue)
 		WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, __func__);
 
 	TQ_LOCK(queue);
-	taskqueue_drain_tq_queue(queue);
-	taskqueue_drain_tq_active(queue);
+	(void)taskqueue_drain_tq_queue(queue);
+	(void)taskqueue_drain_tq_active(queue);
 	TQ_UNLOCK(queue);
 }
 
@@ -612,6 +614,20 @@ taskqueue_drain_timeout(struct taskqueue *queue,
 	TQ_UNLOCK(queue);
 }
 
+void
+taskqueue_quiesce(struct taskqueue *queue)
+{
+	int ret;
+
+	TQ_LOCK(queue);
+	do {
+		ret = taskqueue_drain_tq_queue(queue);
+		if (ret == 0)
+			ret = taskqueue_drain_tq_active(queue);
+	} while (ret != 0);
+	TQ_UNLOCK(queue);
+}
+
 static void
 taskqueue_swi_enqueue(void *context)
 {

sys/sys/taskqueue.h

@@ -93,6 +93,7 @@ void	taskqueue_drain(struct taskqueue *queue, struct task *task);
 void	taskqueue_drain_timeout(struct taskqueue *queue,
 	    struct timeout_task *timeout_task);
 void	taskqueue_drain_all(struct taskqueue *queue);
+void	taskqueue_quiesce(struct taskqueue *queue);
 void	taskqueue_free(struct taskqueue *queue);
 void	taskqueue_run(struct taskqueue *queue);
 void	taskqueue_block(struct taskqueue *queue);