Change the way shutdown is handled for log file.

pmc_flush_logfile is now non-blocking and just ask the kernel
to shutdown the file. From that point, no more data is
accepted by the log thread and when the last buffer is flushed
the file is closed.

This will remove a deadlock between pmcstat asking for
flush while it cannot flush the pipe itself.

MFC after: 3 days

sys/dev/hwpmc/hwpmc_logging.c

@@ -237,7 +237,7 @@ pmclog_get_buffer(struct pmc_owner *po)
 static void
 pmclog_loop(void *arg)
 {
-	int error;
+	int error, last_buffer;
 	struct pmc_owner *po;
 	struct pmclog_buffer *lb;
 	struct proc *p;
@@ -252,6 +252,7 @@ pmclog_loop(void *arg)
 	p = po->po_owner;
 	td = curthread;
 	mycred = td->td_ucred;
+	last_buffer = 0;
 
 	PROC_LOCK(p);
 	ownercred = crhold(p->p_ucred);
@@ -284,27 +285,20 @@ pmclog_loop(void *arg)
 			if ((lb = TAILQ_FIRST(&po->po_logbuffers)) == NULL) {
 				mtx_unlock_spin(&po->po_mtx);
 
-				/*
-				 * Wakeup the thread waiting for the
-				 * PMC_OP_FLUSHLOG request to
-				 * complete.
-				 */
-				if (po->po_flags & PMC_PO_IN_FLUSH) {
-					po->po_flags &= ~PMC_PO_IN_FLUSH;
-					wakeup_one(po->po_kthread);
-				}
-
 				(void) msleep(po, &pmc_kthread_mtx, PWAIT,
 				    "pmcloop", 0);
 				continue;
 			}
 
 			TAILQ_REMOVE(&po->po_logbuffers, lb, plb_next);
+			if (po->po_flags & PMC_PO_SHUTDOWN)
+				last_buffer = TAILQ_EMPTY(&po->po_logbuffers);
 			mtx_unlock_spin(&po->po_mtx);
 		}
 
 		mtx_unlock(&pmc_kthread_mtx);
 
+sigpipe_retry:
 		/* process the request */
 		PMCDBG(LOG,WRI,2, "po=%p base=%p ptr=%p", po,
 		    lb->plb_base, lb->plb_ptr);
@@ -328,7 +322,8 @@ pmclog_loop(void *arg)
 
 		if (error) {
 			/* XXX some errors are recoverable */
-			/* XXX also check for SIGPIPE if a socket */
+			if (error == EPIPE)
+				goto sigpipe_retry;
 
 			/* send a SIGIO to the owner and exit */
 			PROC_LOCK(p);
@@ -344,6 +339,14 @@ pmclog_loop(void *arg)
 			break;
 		}
 
+		if (last_buffer) {
+			/*
+			 * Close the file to get PMCLOG_EOF error
+			 * in pmclog(3).
+			 */
+			fo_close(po->po_file, curthread);
+		}
+
 		mtx_lock(&pmc_kthread_mtx);
 
 		/* put the used buffer back into the global pool */
@@ -425,6 +428,12 @@ pmclog_reserve(struct pmc_owner *po, int length)
 
 	mtx_lock_spin(&po->po_mtx);
 
+	/* No more data when shutdown in progress. */
+	if (po->po_flags & PMC_PO_SHUTDOWN) {
+		mtx_unlock_spin(&po->po_mtx);
+		return (NULL);
+	}
+
 	if (po->po_curbuf == NULL)
 		if (pmclog_get_buffer(po) != 0) {
 			mtx_unlock_spin(&po->po_mtx);
@@ -686,7 +695,7 @@ pmclog_deconfigure_log(struct pmc_owner *po)
 int
 pmclog_flush(struct pmc_owner *po)
 {
-	int error, has_pending_buffers;
+	int error;
 
 	PMCDBG(LOG,FLS,1, "po=%p", po);
 
@@ -714,16 +723,13 @@ pmclog_flush(struct pmc_owner *po)
 	mtx_lock_spin(&po->po_mtx);
 	if (po->po_curbuf)
 		pmclog_schedule_io(po);
-	has_pending_buffers = !TAILQ_EMPTY(&po->po_logbuffers);
 	mtx_unlock_spin(&po->po_mtx);
 
-	if (has_pending_buffers) {
-		po->po_flags |= PMC_PO_IN_FLUSH; /* ask for a wakeup */
-		error = msleep(po->po_kthread, &pmc_kthread_mtx, PWAIT,
-		    "pmcflush", 0);
-		if (error == 0)
-			error = po->po_error;
-	}
+	/*
+	 * Initiate shutdown: no new data queued,
+	 * thread will close file on last block.
+	 */
+	po->po_flags |= PMC_PO_SHUTDOWN;
 
  error:
 	mtx_unlock(&pmc_kthread_mtx);

sys/sys/pmc.h

@@ -759,7 +759,7 @@ struct pmc_owner  {
 };
 
 #define	PMC_PO_OWNS_LOGFILE		0x00000001 /* has a log file */
-#define	PMC_PO_IN_FLUSH			0x00000010 /* in the middle of a flush */
+#define	PMC_PO_SHUTDOWN			0x00000010 /* in the process of shutdown */
 #define	PMC_PO_INITIAL_MAPPINGS_DONE	0x00000020
 
 /*

usr.sbin/pmcstat/pmcstat_log.c

@@ -1670,10 +1670,8 @@ pmcstat_print_log(void)
 int
 pmcstat_close_log(void)
 {
-	if (pmc_flush_logfile() < 0 ||
-	    pmc_configure_logfile(-1) < 0)
+	if (pmc_flush_logfile() < 0)
 		err(EX_OSERR, "ERROR: logging failed");
-	args.pa_flags &= ~(FLAG_HAS_OUTPUT_LOGFILE | FLAG_HAS_PIPE);
 	return (args.pa_flags & FLAG_HAS_PIPE ? PMCSTAT_EXITING :
 	    PMCSTAT_FINISHED);
 }