Fix bright colors for syscons, and make them work for the first time

for vt.  Restore syscons' rendering of background (bg) brightness as
foreground (fg) blinking and vice versa, and add rendering of blinking
as background brightness to vt.

Bright/saturated is conflated with light/white in the implementation
and in this description.

Bright colors were broken in all cases, but appeared to work in the
only case shown by "vidcontrol show".  A boldness hack was applied
only in 1 layering-violation place (for some syscons sequences) where
it made some cases seem to work but was undone by clearing bold using
ANSI sequences, and more seriously was not undone when setting
ANSI/xterm dark colors so left them bright.  Move this hack to drivers.

The boldness hack is only for fg brightness.  Restore/add a similar hack
for bg brightness rendered as fg blinking and vice versa.  This works
even better for vt, since vt changes the default text mode to give the
more useful bg brightness instead of fg blinking.

The brightness bit in colors was unnecessarily removed by the boldness
hack.  In other cases, it was lost later by teken_256to8().  Use
teken_256to16() to not lose it.  teken_256to8() was intended to be
used for bg colors to allow finer or bg-specific control for the more
difficult reduction to 8; however, since 16 bg colors actually work
on VGA except in syscons text mode and the conversion isn't subtle
enough to significantly in that mode, teken_256to8() is not used now.

There are still bugs, especially in vidcontrol, if bright/blinking
background colors are set.

Restore XOR logic for bold/bright fg in syscons (don't change OR
logic for vt).  Remove broken ifdef on FG_UNDERLINE and its wrong
or missing bit and restore the correct hard-coded bit.  FG_UNDERLINE
is only for mono mode which is not really supported.

Restore XOR logic for blinking/bright bg in syscons (in vt, add
OR logic and render as bright bg).  Remove related broken ifdef
on BG_BLINKING and its missing bit and restore the correct
hard-coded bit.  The same bit means blinking or bright bg depending
on the mode, and we want to ignore the difference everywhere.

Simplify conversions of attributes in syscons.  Don't pretend to
support bold fonts.  Don't support unusual encodings of brightness.
It is as good as possible to map 16 VGA colors to 16 xterm-16
colors.  E.g., VGA brown -> xterm-16 Olive will be converted back
to VGA brown, so we don't need to convert to xterm-256 Brown.  Teken
cons25 compatibility code already does the same, and duplicates some
small tables.  This is mostly for the sc -> te direction.  The other
direction uses teken_256to16() which is too generic.

sys/dev/syscons/scterm-teken.c

@@ -315,88 +315,57 @@ scteken_nop(void)
  * libteken routines.
  */
 
-static const unsigned char fgcolors_normal[TC_NCOLORS] = {
-	FG_BLACK,     FG_RED,          FG_GREEN,      FG_BROWN,
-	FG_BLUE,      FG_MAGENTA,      FG_CYAN,       FG_LIGHTGREY,
+static const teken_color_t sc_to_te_color[] = {
+	TC_BLACK,     TC_BLUE,         TC_GREEN,     TC_CYAN,
+	TC_RED,       TC_MAGENTA,      TC_BROWN,     TC_WHITE,
 };
 
-static const unsigned char fgcolors_bold[TC_NCOLORS] = {
-	FG_DARKGREY,  FG_LIGHTRED,     FG_LIGHTGREEN, FG_YELLOW,
-	FG_LIGHTBLUE, FG_LIGHTMAGENTA, FG_LIGHTCYAN,  FG_WHITE,
-};
-
-static const unsigned char bgcolors[TC_NCOLORS] = {
-	BG_BLACK,     BG_RED,          BG_GREEN,      BG_BROWN,
-	BG_BLUE,      BG_MAGENTA,      BG_CYAN,       BG_LIGHTGREY,
+static const unsigned char te_to_sc_color[] = {
+	FG_BLACK,     FG_RED,          FG_GREEN,     FG_BROWN,
+	FG_BLUE,      FG_MAGENTA,      FG_CYAN,      FG_LIGHTGREY,
 };
 
 static void
 scteken_sc_to_te_attr(unsigned char color, teken_attr_t *a)
 {
-	teken_color_t fg, bg;
 
 	/*
-	 * XXX: Reverse conversion of syscons to teken attributes. Not
-	 * realiable. Maybe we should turn it into a 1:1 mapping one of
-	 * these days?
+	 * Conversions of attrs are not reversible.  Since sc attrs are
+	 * pure colors in the simplest mode (16-color graphics) and the
+	 * API is too deficient to tell us the mode, always convert to
+	 * pure colors.  The conversion is essentially the identity except
+	 * for reordering the non-brightness bits in the 2 color numbers.
 	 */
-
 	a->ta_format = 0;
-	a->ta_fgcolor = TC_WHITE;
-	a->ta_bgcolor = TC_BLACK;
-
-#ifdef FG_BLINK
-	if (color & FG_BLINK) {
-		a->ta_format |= TF_BLINK;
-		color &= ~FG_BLINK;
-	}
-#endif /* FG_BLINK */
-
-	for (fg = 0; fg < TC_NCOLORS; fg++) {
-		for (bg = 0; bg < TC_NCOLORS; bg++) {
-			if ((fgcolors_normal[fg] | bgcolors[bg]) == color) {
-				a->ta_fgcolor = fg;
-				a->ta_bgcolor = bg;
-				return;
-			}
-
-			if ((fgcolors_bold[fg] | bgcolors[bg]) == color) {
-				a->ta_fgcolor = fg;
-				a->ta_bgcolor = bg;
-				a->ta_format |= TF_BOLD;
-				return;
-			}
-		}
-	}
+	a->ta_fgcolor = sc_to_te_color[color & 7] | (color & 8);
+	a->ta_bgcolor = sc_to_te_color[(color >> 4) & 7] | ((color >> 4) & 8);
 }
 
 static int
 scteken_te_to_sc_attr(const teken_attr_t *a)
 {
-	int attr = 0;
+	int attr;
 	teken_color_t fg, bg;
 
 	if (a->ta_format & TF_REVERSE) {
-		fg = teken_256to8(a->ta_bgcolor);
-		bg = teken_256to8(a->ta_fgcolor);
+		fg = a->ta_bgcolor;
+		bg = a->ta_fgcolor;
 	} else {
-		fg = teken_256to8(a->ta_fgcolor);
-		bg = teken_256to8(a->ta_bgcolor);
+		fg = a->ta_fgcolor;
+		bg = a->ta_bgcolor;
 	}
-	if (a->ta_format & TF_BOLD)
-		attr |= fgcolors_bold[fg];
-	else
-		attr |= fgcolors_normal[fg];
-	attr |= bgcolors[bg];
+	if (fg >= 16)
+		fg = teken_256to16(fg);
+	if (bg >= 16)
+		bg = teken_256to16(bg);
+	attr = te_to_sc_color[fg & 7] | (fg & 8) |
+	    ((te_to_sc_color[bg & 7] | (bg & 8)) << 4);
 
-#ifdef FG_UNDERLINE
-	if (a->ta_format & TF_UNDERLINE)
-		attr |= FG_UNDERLINE;
-#endif /* FG_UNDERLINE */
-#ifdef FG_BLINK
+	/* XXX: underline mapping for Hercules adapter can be better. */
+	if (a->ta_format & (TF_BOLD | TF_UNDERLINE))
+		attr ^= 8;
 	if (a->ta_format & TF_BLINK)
-		attr |= FG_BLINK;
-#endif /* FG_BLINK */
+		attr ^= 0x80;
 
 	return (attr);
 }

sys/dev/vt/vt_core.c

@@ -1075,6 +1075,8 @@ vt_determine_colors(term_char_t c, int cursor,
 	if (TCHAR_FORMAT(c) & TF_BOLD)
 		*fg = TCOLOR_LIGHT(*fg);
 	*bg = TCHAR_BGCOLOR(c);
+	if (TCHAR_FORMAT(c) & TF_BLINK)
+		*bg = TCOLOR_LIGHT(*bg);
 
 	if (TCHAR_FORMAT(c) & TF_REVERSE)
 		invert ^= 1;

sys/kern/subr_terminal.c

@@ -130,9 +130,25 @@ static const teken_attr_t default_message = {
 	.ta_format	= TCHAR_FORMAT(TERMINAL_NORM_ATTR)
 };
 
+/* Fudge fg brightness as TF_BOLD (shifted). */
+#define	TCOLOR_FG_FUDGED(color) __extension__ ({			\
+	teken_color_t _c;						\
+									\
+	_c = (color);							\
+	TCOLOR_FG(_c & 7) | ((_c & 8) << 18);				\
+})
+
+/* Fudge bg brightness as TF_BLINK (shifted). */
+#define	TCOLOR_BG_FUDGED(color) __extension__ ({			\
+	teken_color_t _c;						\
+									\
+	_c = (color);							\
+	TCOLOR_BG(_c & 7) | ((_c & 8) << 20);				\
+})
+
 #define	TCHAR_CREATE(c, a)	((c) | TFORMAT((a)->ta_format) |	\
-	TCOLOR_FG(teken_256to8((a)->ta_fgcolor)) |			\
-	TCOLOR_BG(teken_256to8((a)->ta_bgcolor)))
+	TCOLOR_FG_FUDGED(teken_256to16((a)->ta_fgcolor)) |		\
+	TCOLOR_BG_FUDGED(teken_256to16((a)->ta_bgcolor)))
 
 static void
 terminal_init(struct terminal *tm)

sys/teken/teken_subr_compat.h

@@ -40,23 +40,16 @@ static void
 teken_subr_cons25_set_adapter_background(teken_t *t, unsigned int c)
 {
 
-	t->t_defattr.ta_bgcolor = cons25_colors[c % 8];
-	t->t_curattr.ta_bgcolor = cons25_colors[c % 8];
+	t->t_defattr.ta_bgcolor = cons25_colors[c % 8] | (c & 8);
+	t->t_curattr.ta_bgcolor = cons25_colors[c % 8] | (c & 8);
 }
 
 static void
 teken_subr_cons25_set_adapter_foreground(teken_t *t, unsigned int c)
 {
 
-	t->t_defattr.ta_fgcolor = cons25_colors[c % 8];
-	t->t_curattr.ta_fgcolor = cons25_colors[c % 8];
-	if (c >= 8) {
-		t->t_defattr.ta_format |= TF_BOLD;
-		t->t_curattr.ta_format |= TF_BOLD;
-	} else {
-		t->t_defattr.ta_format &= ~TF_BOLD;
-		t->t_curattr.ta_format &= ~TF_BOLD;
-	}
+	t->t_defattr.ta_fgcolor = cons25_colors[c % 8] | (c & 8);
+	t->t_curattr.ta_fgcolor = cons25_colors[c % 8] | (c & 8);
 }
 
 static const teken_color_t cons25_revcolors[8] = { 0, 4, 2, 6, 1, 5, 3, 7 };