Fix line discipline switching issues: If opening a new ldisc fails,

we have to revert to TTYDISC which we know will successfully open
rather than try the previous ldisc which might also fail to open.

Do not let ldisc implementations muck about with ->t_line, and remove
code which checks for reopens, it should never happen.

Move ldisc->l_hotchar to tty->t_hotchar and have ldisc implementation
initialize it in their open routines.  Reset to zero when we enter
TTYDISC.  ("no" should really be -1 since zero could be a valid
hotchar for certain old european mainframe protocols.)

sys/kern/tty.c

@@ -236,6 +236,7 @@ ttyopen(struct cdev *device, struct tty *tp)
 
 	s = spltty();
 	tp->t_dev = device;
+	tp->t_hotchar = 0;
 	if (!ISSET(tp->t_state, TS_ISOPEN)) {
 		ttyref(tp);
 		SET(tp->t_state, TS_ISOPEN);
@@ -276,6 +277,7 @@ ttyclose(struct tty *tp)
 
 	tp->t_gen++;
 	tp->t_line = TTYDISC;
+	tp->t_hotchar = 0;
 	tp->t_pgrp = NULL;
 	tp->t_session = NULL;
 	tp->t_state = 0;
@@ -1072,18 +1074,25 @@ ttioctl(struct tty *tp, u_long cmd, void *data, int flag)
 
 		if ((u_int)t >= nlinesw)
 			return (ENXIO);
-		if (t != tp->t_line) {
-			s = spltty();
-			ttyld_close(tp, flag);
-			error = (*linesw[t]->l_open)(device, tp);
-			if (error) {
-				(void)ttyld_open(tp, device);
-				splx(s);
-				return (error);
-			}
-			tp->t_line = t;
-			splx(s);
+		if (t == tp->t_line)
+			return (0);
+		s = spltty();
+		ttyld_close(tp, flag);
+		tp->t_line = t;
+		error = ttyld_open(tp, device);
+		if (error) {
+			/*
+			 * If we fail to switch line discipline we cannot
+			 * fall back to the previous, because we can not
+			 * trust that ldisc to open successfully either.
+			 * Fall back to the default ldisc which we know 
+			 * will allways succeed.
+			 */
+			tp->t_line = TTYDISC;
+			(void)ttyld_open(tp, device);
 		}
+		splx(s);
+		return (error);
 		break;
 	}
 	case TIOCSTART:			/* start output, like ^Q */
@@ -2905,7 +2914,7 @@ ttyldoptim(struct tty *tp)
 		tp->t_state |= TS_CAN_BYPASS_L_RINT;
 	else
 		tp->t_state &= ~TS_CAN_BYPASS_L_RINT;
-	return (linesw[tp->t_line]->l_hotchar);
+	return (tp->t_hotchar);
 }
 
 

sys/net/if_sl.c

@@ -186,8 +186,7 @@ static int	slstart(struct tty *);
 
 static struct linesw slipdisc = {
 	slopen,		slclose,	l_noread,	l_nowrite,
-	sltioctl,	slinput,	slstart,	ttymodem,
-	FRAME_END
+	sltioctl,	slinput,	slstart,	ttymodem
 };
 
 /*
@@ -350,17 +349,14 @@ slopen(dev, tp)
 	if (error)
 		return (error);
 
-	if (tp->t_line == SLIPDISC)
-		return (0);
-
 	if ((sc = slcreate()) == NULL)
 		return (ENOBUFS);
 
+	tp->t_hotchar = FRAME_END;
 	tp->t_sc = (caddr_t)sc;
 	sc->sc_ttyp = tp;
 	sc->sc_if.if_baudrate = tp->t_ospeed;
 	ttyflush(tp, FREAD | FWRITE);
-	tp->t_line = SLIPDISC;
 
 	/*
 	 * We don't use t_canq or t_rawq, so reduce their
@@ -417,7 +413,6 @@ slclose(tp,flag)
 	 */
 	s = splimp();		/* actually, max(spltty, splnet) */
 	clist_free_cblocks(&tp->t_outq);
-	tp->t_line = 0;
 	sc = (struct sl_softc *)tp->t_sc;
 	if (sc != NULL) {
 		if (sc->sc_outfill) {

sys/net/ppp_tty.c

@@ -148,8 +148,7 @@ void		pppasyncdetach(void);
 
 static struct linesw pppdisc = {
 	pppopen,	pppclose,	pppread,	pppwrite,
-	ppptioctl,	pppinput,	pppstart,	ttymodem,
-	PPP_FLAG
+	ppptioctl,	pppinput,	pppstart,	ttymodem
 };
 
 void
@@ -185,13 +184,7 @@ pppopen(dev, tp)
 
     s = spltty();
 
-    if (tp->t_line == PPPDISC) {
-	sc = (struct ppp_softc *) tp->t_sc;
-	if (sc != NULL && sc->sc_devp == (void *) tp) {
-	    splx(s);
-	    return (0);
-	}
-    }
+    tp->t_hotchar = PPP_FLAG;
 
     if ((sc = pppalloc(td->td_proc->p_pid)) == NULL) {
 	splx(s);
@@ -254,7 +247,6 @@ pppclose(tp, flag)
     ttyflush(tp, FREAD | FWRITE);
     clist_free_cblocks(&tp->t_canq);
     clist_free_cblocks(&tp->t_outq);
-    tp->t_line = 0;
     sc = (struct ppp_softc *) tp->t_sc;
     if (sc != NULL) {
 	tp->t_sc = NULL;

sys/netgraph/bluetooth/drivers/h4/ng_h4.c

@@ -97,8 +97,7 @@ static struct linesw		ng_h4_disc = {
 	ng_h4_ioctl,		/* ioctl */
 	ng_h4_input,		/* input */
 	ng_h4_start,		/* start */
-	ttymodem,		/* modem */
-	0 			/* hotchar (don't really care which one) */
+	ttymodem		/* modem */
 };
 
 /* Netgraph methods */
@@ -161,13 +160,6 @@ ng_h4_open(struct cdev *dev, struct tty *tp)
 	s = splnet(); /* XXX */
 	spltty(); /* XXX */
 
-	/* Already installed? */
-	if (tp->t_line == H4DISC) {
-		sc = (ng_h4_info_p) tp->t_sc;
-		if (sc != NULL && sc->tp == tp)
-			goto out;
-	}
-
 	/* Initialize private struct */
 	MALLOC(sc, ng_h4_info_p, sizeof(*sc), M_NETGRAPH_H4, M_NOWAIT|M_ZERO);
 	if (sc == NULL) {
@@ -243,7 +235,6 @@ ng_h4_close(struct tty *tp, int flag)
 
 	ttyflush(tp, FREAD | FWRITE);
 	clist_free_cblocks(&tp->t_outq);
-	tp->t_line = 0;
 	if (sc != NULL) {
 		tp->t_sc = NULL;
 

sys/netgraph/ng_tty.c

@@ -148,8 +148,7 @@ static struct linesw ngt_disc = {
 	ngt_tioctl,
 	ngt_input,
 	ngt_start,
-	ttymodem,
-	NG_TTY_DFL_HOTCHAR	/* XXX can't change this in serial driver */
+	ttymodem
 };
 
 /* Netgraph node type descriptor */
@@ -193,12 +192,7 @@ ngt_open(struct cdev *dev, struct tty *tp)
 	s = splnet();
 	(void) spltty();	/* XXX is this necessary? */
 
-	/* Already installed? */
-	if (tp->t_line == NETGRAPHDISC) {
-		sc = (sc_p) tp->t_sc;
-		if (sc != NULL && sc->tp == tp)
-			goto done;
-	}
+	tp->t_hotchar = NG_TTY_DFL_HOTCHAR;
 
 	/* Initialize private struct */
 	MALLOC(sc, sc_p, sizeof(*sc), M_NETGRAPH, M_WAITOK | M_ZERO);
@@ -265,7 +259,6 @@ ngt_close(struct tty *tp, int flag)
 	s = spltty();
 	ttyflush(tp, FREAD | FWRITE);
 	clist_free_cblocks(&tp->t_outq);
-	tp->t_line = 0;
 	if (sc != NULL) {
 		if (sc->flags & FLG_TIMEOUT) {
 			untimeout(ngt_timeout, sc, sc->chand);

sys/sys/linedisc.h

@@ -66,7 +66,6 @@ struct linesw {
 	l_rint_t	*l_rint;
 	l_start_t	*l_start;
 	l_modem_t	*l_modem;
-	u_char		l_hotchar;
 };
 
 extern struct linesw *linesw[];

sys/sys/tty.h

@@ -116,6 +116,7 @@ struct tty {
 
 	struct mtx t_mtx;
 	int	t_refcnt;
+	int	t_hotchar;		/* linedisc preferred hot char */
 };
 
 #define	t_cc		t_termios.c_cc