tty: implement TIOCNOTTY

Generally, it's preferred that an application fork/setsid if it doesn't want
to keep its controlling TTY, but it could be that a debugger is trying to
steal it instead -- so it would hook in, drop the controlling TTY, then do
some magic to set things up again. In this case, TIOCNOTTY is quite handy
and still respected by at least OpenBSD, NetBSD, and Linux as far as I can
tell.

I've dropped the note about obsoletion, as I intend to support TIOCNOTTY as
long as it doesn't impose a major burden.

Reviewed by:	bcr (manpages), kib
Differential Revision:	https://reviews.freebsd.org/D22572

share/man/man4/tty.4

@@ -28,7 +28,7 @@
 .\"     @(#)tty.4	8.3 (Berkeley) 4/19/94
 .\" $FreeBSD$
 .\"
-.Dd January 11, 2017
+.Dd November 27, 2019
 .Dt TTY 4
 .Os
 .Sh NAME
@@ -194,7 +194,6 @@ Simulate typed input.
 Pretend as if the terminal received the character pointed to by
 .Fa cp .
 .It Dv TIOCNOTTY Fa void
-This call is obsolete but left for compatibility.
 In the past, when a process that did not have a controlling terminal (see
 .Em The Controlling Terminal
 in
@@ -203,7 +202,7 @@ first opened a terminal device, it acquired that terminal as its
 controlling terminal.
 For some programs this was a hazard as they
 did not want a controlling terminal in the first place, and this
-provided a mechanism to disassociate the controlling terminal from
+provides a mechanism to disassociate the controlling terminal from
 the calling process.
 It
 .Em must
@@ -228,6 +227,14 @@ system call which will place the process into its own session - which
 has the effect of disassociating it from the controlling terminal.
 This is the new and preferred method for programs to lose their controlling
 terminal.
+.Pp
+However, environmental restrictions may prohibit the process from being able to
+.Fn fork
+and call the
+.Fn setsid
+system call to disassociate it from the controlling terminal.
+In this case, it must use
+.Dv TIOCNOTTY .
 .It Dv TIOCSTOP Fa void
 Stop output on the terminal (like typing ^S at the keyboard).
 .It Dv TIOCSTART Fa void

sys/fs/devfs/devfs_vnops.c

@@ -829,9 +829,16 @@ devfs_ioctl(struct vop_ioctl_args *ap)
 		error = ENOTTY;
 
 	if (error == 0 && com == TIOCSCTTY) {
-		/* Do nothing if reassigning same control tty */
+		/*
+		 * Do nothing if reassigning same control tty, or if the
+		 * control tty has already disappeared.  If it disappeared,
+		 * it's because we were racing with TIOCNOTTY.  TIOCNOTTY
+		 * already took care of releasing the old vnode and we have
+		 * nothing left to do.
+		 */
 		sx_slock(&proctree_lock);
-		if (td->td_proc->p_session->s_ttyvp == vp) {
+		if (td->td_proc->p_session->s_ttyvp == vp ||
+		    td->td_proc->p_session->s_ttyp == NULL) {
 			sx_sunlock(&proctree_lock);
 			return (0);
 		}

sys/kern/tty.c

@@ -1195,6 +1195,71 @@ tty_rel_gone(struct tty *tp)
 	tty_rel_free(tp);
 }
 
+static int
+tty_drop_ctty(struct tty *tp, struct proc *p)
+{
+	struct session *session;
+	struct vnode *vp;
+
+	/*
+	 * This looks terrible, but it's generally safe as long as the tty
+	 * hasn't gone away while we had the lock dropped.  All of our sanity
+	 * checking that this operation is OK happens after we've picked it back
+	 * up, so other state changes are generally not fatal and the potential
+	 * for this particular operation to happen out-of-order in a
+	 * multithreaded scenario is likely a non-issue.
+	 */
+	tty_unlock(tp);
+	sx_xlock(&proctree_lock);
+	tty_lock(tp);
+	if (tty_gone(tp)) {
+		sx_xunlock(&proctree_lock);
+		return (ENODEV);
+	}
+
+	/*
+	 * If the session doesn't have a controlling TTY, or if we weren't
+	 * invoked on the controlling TTY, we'll return ENOIOCTL as we've
+	 * historically done.
+	 */
+	session = p->p_session;
+	if (session->s_ttyp == NULL || session->s_ttyp != tp) {
+		sx_xunlock(&proctree_lock);
+		return (ENOTTY);
+	}
+
+	if (!SESS_LEADER(p)) {
+		sx_xunlock(&proctree_lock);
+		return (EPERM);
+	}
+
+	PROC_LOCK(p);
+	SESS_LOCK(session);
+	vp = session->s_ttyvp;
+	session->s_ttyp = NULL;
+	session->s_ttyvp = NULL;
+	session->s_ttydp = NULL;
+	SESS_UNLOCK(session);
+
+	tp->t_sessioncnt--;
+	p->p_flag &= ~P_CONTROLT;
+	PROC_UNLOCK(p);
+	sx_xunlock(&proctree_lock);
+
+	/*
+	 * If we did have a vnode, release our reference.  Ordinarily we manage
+	 * these at the devfs layer, but we can't necessarily know that we were
+	 * invoked on the vnode referenced in the session (i.e. the vnode we
+	 * hold a reference to).  We explicitly don't check VBAD/VI_DOOMED here
+	 * to avoid a vnode leak -- in circumstances elsewhere where we'd hit a
+	 * VI_DOOMED vnode, release has been deferred until the controlling TTY
+	 * is either changed or released.
+	 */
+	if (vp != NULL)
+		vrele(vp);
+	return (0);
+}
+
 /*
  * Exposing information about current TTY's through sysctl
  */
@@ -1709,6 +1774,8 @@ tty_generic_ioctl(struct tty *tp, u_long cmd, void *data, int fflag,
 		MPASS(tp->t_session);
 		*(int *)data = tp->t_session->s_sid;
 		return (0);
+	case TIOCNOTTY:
+		return (tty_drop_ctty(tp, td->td_proc));
 	case TIOCSCTTY: {
 		struct proc *p = td->td_proc;