libproc: Add support for some proc_attach() flags.

This change adds some handling for the equivalent of Solaris' PGRAB_*
flags. In particular, support for PGRAB_RDONLY is needed to avoid a
nasty deadlock: dtrace(1) may otherwise stop the master process for its
pseudo-terminal and end up blocking while writing to standard output.

cddl/compat/opensolaris/include/libproc.h

@@ -38,9 +38,6 @@
 #define PR_RLC		0x0001
 #define PR_KLC		0x0002
 
-#define	PGRAB_RDONLY	O_RDONLY
-#define	PGRAB_FORCE	0
-
 #include_next <libproc.h>
 
 #endif

cddl/lib/libdtrace/libproc_compat.h

@@ -34,6 +34,9 @@
  * Functions sorted alphabetically.
  */
 #define	PR_LMID_EVERY 0
+#define	PGRAB_RDONLY	PATTACH_RDONLY
+#define	PGRAB_FORCE	PATTACH_FORCE
+
 #define	Psetrun(p, a1, a2) proc_continue((p))
 #define	Pxlookup_by_addr(p, a, n, s, sym, i) \
     proc_addr2sym(p, a, n, s, sym)

lib/libproc/libproc.h

@@ -50,6 +50,11 @@ typedef void (*proc_child_func)(void *);
 #define PS_DEAD		5
 #define PS_LOST		6
 
+/* Flags for proc_attach(). */
+#define	PATTACH_FORCE	0x01
+#define	PATTACH_RDONLY	0x02
+#define	PATTACH_NOSTOP	0x04
+
 /* Reason values for proc_detach(). */
 #define PRELEASE_HANG	1
 #define PRELEASE_KILL	2

lib/libproc/proc_create.c

@@ -127,7 +127,7 @@ proc_attach(pid_t pid, int flags, struct proc_handle **pphdl)
 	struct proc_handle *phdl;
 	int error, status;
 
-	if (pid == 0 || pid == getpid())
+	if (pid == 0 || (pid == getpid() && (flags & PATTACH_RDONLY) == 0))
 		return (EINVAL);
 	if (elf_version(EV_CURRENT) == EV_NONE)
 		return (ENOENT);
@@ -140,27 +140,32 @@ proc_attach(pid_t pid, int flags, struct proc_handle **pphdl)
 	if (error != 0)
 		goto out;
 
-	if (ptrace(PT_ATTACH, proc_getpid(phdl), 0, 0) != 0) {
+	if ((flags & PATTACH_RDONLY) == 0) {
-		error = errno;
+		if (ptrace(PT_ATTACH, proc_getpid(phdl), 0, 0) != 0) {
-		DPRINTF("ERROR: cannot ptrace child process %d", pid);
+			error = errno;
-		goto out;
+			DPRINTF("ERROR: cannot ptrace child process %d", pid);
-	}
+			goto out;
+		}
 
-	/* Wait for the child process to stop. */
+		/* Wait for the child process to stop. */
-	if (waitpid(pid, &status, WUNTRACED) == -1) {
+		if (waitpid(pid, &status, WUNTRACED) == -1) {
-		error = errno;
+			error = errno;
-		DPRINTF("ERROR: child process %d didn't stop as expected", pid);
+			DPRINTF("ERROR: child process %d didn't stop as expected", pid);
-		goto out;
+			goto out;
-	}
+		}
 
-	/* Check for an unexpected status. */
+		/* Check for an unexpected status. */
-	if (!WIFSTOPPED(status))
+		if (!WIFSTOPPED(status))
-		DPRINTFX("ERROR: child process %d status 0x%x", pid, status);
+			DPRINTFX("ERROR: child process %d status 0x%x", pid, status);
-	else
+		else
-		phdl->status = PS_STOP;
+			phdl->status = PS_STOP;
+
+		if ((flags & PATTACH_NOSTOP) != 0)
+			proc_continue(phdl);
+	}
 
 out:
-	if (error && phdl != NULL) {
+	if (error != 0 && phdl != NULL) {
 		proc_free(phdl);
 		phdl = NULL;
 	}

lib/libproc/proc_util.c

@@ -87,21 +87,25 @@ proc_detach(struct proc_handle *phdl, int reason)
 
 	if (phdl == NULL)
 		return (EINVAL);
+	if (reason == PRELEASE_HANG)
+		return (EINVAL);
 	if (reason == PRELEASE_KILL) {
 		kill(proc_getpid(phdl), SIGKILL);
-		return (0);
+		goto free;
 	}
+	if ((phdl->flags & PATTACH_RDONLY) != 0)
+		goto free;
 	pid = proc_getpid(phdl);
 	if (ptrace(PT_DETACH, pid, 0, 0) != 0 && errno == ESRCH)
-		return (0);
+		goto free;
 	if (errno == EBUSY) {
 		kill(pid, SIGSTOP);
 		waitpid(pid, &status, WUNTRACED);
 		ptrace(PT_DETACH, pid, 0, 0);
 		kill(pid, SIGCONT);
-		return (0);
 	}
-
+free:
+	proc_free(phdl);
 	return (0);
 }