From 17c10964b404c2906fa1a1edfe3c327ca4ba5d58 Mon Sep 17 00:00:00 2001
From: yuripv <yuripv@FreeBSD.org>
Date: Fri, 9 Nov 2018 03:32:53 +0000
Subject: [PATCH] Reset persistent mbstates when rune locale encoding changes.

This was shown to be a problem by side effect of now-enabled test case,
which was going through C, en_US.UTF-8, ja_JP.SJIS, and ja_JP.eucJP,
and failing eventually as data in mbrtowc's mbstate, that was
perfectly correct for en_US.UTF-8 was treated as incorrect for
ja_JP.SJIS, failing the entire test case.

This makes the persistent mbstates to be per ctype-component,
and not per-locale so we could easily reset the mbstates when
only LC_CTYPE is changed.

Reviewed by:	bapt, pfg
Approved by:	kib (mentor, implicit)
Differential Revision:	https://reviews.freebsd.org/D17796
---
 .../netbsd-tests/lib/libc/locale/t_mbrtowc.c  |  3 --
 lib/libc/locale/cXXrtomb_iconv.h              |  4 +--
 lib/libc/locale/mblen.c                       |  5 ++--
 lib/libc/locale/mblocal.h                     | 28 +++++++++++++++++++
 lib/libc/locale/mbrlen.c                      |  2 +-
 lib/libc/locale/mbrtocXX_iconv.h              |  4 +--
 lib/libc/locale/mbrtowc.c                     |  2 +-
 lib/libc/locale/mbsnrtowcs.c                  |  2 +-
 lib/libc/locale/mbsrtowcs.c                   |  2 +-
 lib/libc/locale/mbtowc.c                      |  5 ++--
 lib/libc/locale/setrunelocale.c               | 15 ++++++++++
 lib/libc/locale/wcrtomb.c                     |  2 +-
 lib/libc/locale/wcsnrtombs.c                  |  2 +-
 lib/libc/locale/wcsrtombs.c                   |  2 +-
 lib/libc/locale/wctomb.c                      |  5 ++--
 lib/libc/locale/xlocale_private.h             | 28 -------------------
 16 files changed, 63 insertions(+), 48 deletions(-)

diff --git a/contrib/netbsd-tests/lib/libc/locale/t_mbrtowc.c b/contrib/netbsd-tests/lib/libc/locale/t_mbrtowc.c
index e52111002f7e..c3e193cfc6a0 100644
--- a/contrib/netbsd-tests/lib/libc/locale/t_mbrtowc.c
+++ b/contrib/netbsd-tests/lib/libc/locale/t_mbrtowc.c
@@ -243,9 +243,6 @@ ATF_TC_BODY(mbrtowc_internal, tc)
 {
 	struct test *t;
 
-#ifdef __FreeBSD__
-	atf_tc_expect_fail("ja_* locale fails");
-#endif
 	for (t = &tests[0]; t->data != NULL; ++t)
 		h_ctype2(t, false);
 }
diff --git a/lib/libc/locale/cXXrtomb_iconv.h b/lib/libc/locale/cXXrtomb_iconv.h
index d0dadac6c312..1f87e353e052 100644
--- a/lib/libc/locale/cXXrtomb_iconv.h
+++ b/lib/libc/locale/cXXrtomb_iconv.h
@@ -39,7 +39,7 @@ __FBSDID("$FreeBSD$");
 #include "../iconv/citrus_hash.h"
 #include "../iconv/citrus_module.h"
 #include "../iconv/citrus_iconv.h"
-#include "xlocale_private.h"
+#include "mblocal.h"
 
 typedef struct {
 	bool			initialized;
@@ -65,7 +65,7 @@ cXXrtomb_l(char * __restrict s, charXX_t c, mbstate_t * __restrict ps,
 
 	FIX_LOCALE(locale);
 	if (ps == NULL)
-		ps = &locale->cXXrtomb;
+		ps = &(XLOCALE_CTYPE(locale)->cXXrtomb);
 	cs = (_ConversionState *)ps;
 	handle = &cs->iconv;
 
diff --git a/lib/libc/locale/mblen.c b/lib/libc/locale/mblen.c
index e972298f5b40..77d9745da198 100644
--- a/lib/libc/locale/mblen.c
+++ b/lib/libc/locale/mblen.c
@@ -47,10 +47,11 @@ mblen_l(const char *s, size_t n, locale_t locale)
 
 	if (s == NULL) {
 		/* No support for state dependent encodings. */
-		locale->mblen = initial;
+		XLOCALE_CTYPE(locale)->mblen = initial;
 		return (0);
 	}
-	rval = XLOCALE_CTYPE(locale)->__mbrtowc(NULL, s, n, &locale->mblen);
+	rval = XLOCALE_CTYPE(locale)->__mbrtowc(NULL, s, n,
+	    &(XLOCALE_CTYPE(locale)->mblen));
 	if (rval == (size_t)-1 || rval == (size_t)-2)
 		return (-1);
 	return ((int)rval);
diff --git a/lib/libc/locale/mblocal.h b/lib/libc/locale/mblocal.h
index cffe3ba92bc8..1fb5902a8d11 100644
--- a/lib/libc/locale/mblocal.h
+++ b/lib/libc/locale/mblocal.h
@@ -60,6 +60,34 @@ struct xlocale_ctype {
 		size_t, size_t, mbstate_t * __restrict);
 	int __mb_cur_max;
 	int __mb_sb_limit;
+	/** Persistent state used by mblen() calls. */
+	__mbstate_t mblen;
+	/** Persistent state used by mbrlen() calls. */
+	__mbstate_t mbrlen;
+	/** Persistent state used by mbrtoc16() calls. */
+	__mbstate_t mbrtoc16;
+	/** Persistent state used by mbrtoc32() calls. */
+	__mbstate_t mbrtoc32;
+	/** Persistent state used by mbrtowc() calls. */
+	__mbstate_t mbrtowc;
+	/** Persistent state used by mbsnrtowcs() calls. */
+	__mbstate_t mbsnrtowcs;
+	/** Persistent state used by mbsrtowcs() calls. */
+	__mbstate_t mbsrtowcs;
+	/** Persistent state used by mbtowc() calls. */
+	__mbstate_t mbtowc;
+	/** Persistent state used by c16rtomb() calls. */
+	__mbstate_t c16rtomb;
+	/** Persistent state used by c32rtomb() calls. */
+	__mbstate_t c32rtomb;
+	/** Persistent state used by wcrtomb() calls. */
+	__mbstate_t wcrtomb;
+	/** Persistent state used by wcsnrtombs() calls. */
+	__mbstate_t wcsnrtombs;
+	/** Persistent state used by wcsrtombs() calls. */
+	__mbstate_t wcsrtombs;
+	/** Persistent state used by wctomb() calls. */
+	__mbstate_t wctomb;
 };
 #define XLOCALE_CTYPE(x) ((struct xlocale_ctype*)(x)->components[XLC_CTYPE])
 extern struct xlocale_ctype __xlocale_global_ctype;
diff --git a/lib/libc/locale/mbrlen.c b/lib/libc/locale/mbrlen.c
index f84fce7b61b0..4f687e98628f 100644
--- a/lib/libc/locale/mbrlen.c
+++ b/lib/libc/locale/mbrlen.c
@@ -42,7 +42,7 @@ mbrlen_l(const char * __restrict s, size_t n, mbstate_t * __restrict ps, locale_
 {
 	FIX_LOCALE(locale);
 	if (ps == NULL)
-		ps = &locale->mbrlen;
+		ps = &(XLOCALE_CTYPE(locale)->mbrlen);
 	return (XLOCALE_CTYPE(locale)->__mbrtowc(NULL, s, n, ps));
 }
 
diff --git a/lib/libc/locale/mbrtocXX_iconv.h b/lib/libc/locale/mbrtocXX_iconv.h
index 262818ee79d5..d753b3523244 100644
--- a/lib/libc/locale/mbrtocXX_iconv.h
+++ b/lib/libc/locale/mbrtocXX_iconv.h
@@ -41,7 +41,7 @@ __FBSDID("$FreeBSD$");
 #include "../iconv/citrus_hash.h"
 #include "../iconv/citrus_module.h"
 #include "../iconv/citrus_iconv.h"
-#include "xlocale_private.h"
+#include "mblocal.h"
 
 typedef struct {
 	bool			initialized;
@@ -68,7 +68,7 @@ mbrtocXX_l(charXX_t * __restrict pc, const char * __restrict s, size_t n,
 
 	FIX_LOCALE(locale);
 	if (ps == NULL)
-		ps = &locale->mbrtocXX;
+		ps = &(XLOCALE_CTYPE(locale)->mbrtocXX);
 	cs = (_ConversionState *)ps;
 	handle = &cs->iconv;
 
diff --git a/lib/libc/locale/mbrtowc.c b/lib/libc/locale/mbrtowc.c
index 4171886c8efa..7585ef16bca2 100644
--- a/lib/libc/locale/mbrtowc.c
+++ b/lib/libc/locale/mbrtowc.c
@@ -43,7 +43,7 @@ mbrtowc_l(wchar_t * __restrict pwc, const char * __restrict s,
 {
 	FIX_LOCALE(locale);
 	if (ps == NULL)
-		ps = &locale->mbrtowc;
+		ps = &(XLOCALE_CTYPE(locale)->mbrtowc);
 	return (XLOCALE_CTYPE(locale)->__mbrtowc(pwc, s, n, ps));
 }
 
diff --git a/lib/libc/locale/mbsnrtowcs.c b/lib/libc/locale/mbsnrtowcs.c
index 59574386b0af..69e48eafdc57 100644
--- a/lib/libc/locale/mbsnrtowcs.c
+++ b/lib/libc/locale/mbsnrtowcs.c
@@ -48,7 +48,7 @@ mbsnrtowcs_l(wchar_t * __restrict dst, const char ** __restrict src,
 {
 	FIX_LOCALE(locale);
 	if (ps == NULL)
-		ps = &locale->mbsnrtowcs;
+		ps = &(XLOCALE_CTYPE(locale)->mbsnrtowcs);
 	return (XLOCALE_CTYPE(locale)->__mbsnrtowcs(dst, src, nms, len, ps));
 }
 size_t
diff --git a/lib/libc/locale/mbsrtowcs.c b/lib/libc/locale/mbsrtowcs.c
index aefbee1c2d09..e85b22b492bf 100644
--- a/lib/libc/locale/mbsrtowcs.c
+++ b/lib/libc/locale/mbsrtowcs.c
@@ -46,7 +46,7 @@ mbsrtowcs_l(wchar_t * __restrict dst, const char ** __restrict src, size_t len,
 {
 	FIX_LOCALE(locale);
 	if (ps == NULL)
-		ps = &locale->mbsrtowcs;
+		ps = &(XLOCALE_CTYPE(locale)->mbsrtowcs);
 	return (XLOCALE_CTYPE(locale)->__mbsnrtowcs(dst, src, SIZE_T_MAX, len, ps));
 }
 size_t
diff --git a/lib/libc/locale/mbtowc.c b/lib/libc/locale/mbtowc.c
index df1b204187e9..ec634807560d 100644
--- a/lib/libc/locale/mbtowc.c
+++ b/lib/libc/locale/mbtowc.c
@@ -48,10 +48,11 @@ mbtowc_l(wchar_t * __restrict pwc, const char * __restrict s, size_t n, locale_t
 
 	if (s == NULL) {
 		/* No support for state dependent encodings. */
-		locale->mbtowc = initial;
+		XLOCALE_CTYPE(locale)->mbtowc = initial;
 		return (0);
 	}
-	rval = XLOCALE_CTYPE(locale)->__mbrtowc(pwc, s, n, &locale->mbtowc);
+	rval = XLOCALE_CTYPE(locale)->__mbrtowc(pwc, s, n,
+	    &(XLOCALE_CTYPE(locale)->mbtowc));
 	switch (rval) {
 	case (size_t)-2:
 		errno = EILSEQ;
diff --git a/lib/libc/locale/setrunelocale.c b/lib/libc/locale/setrunelocale.c
index 97af903f2724..2e6fed9466e4 100644
--- a/lib/libc/locale/setrunelocale.c
+++ b/lib/libc/locale/setrunelocale.c
@@ -160,6 +160,21 @@ __setrunelocale(struct xlocale_ctype *l, const char *encoding)
 	if (ret == 0) {
 		/* Free the old runes if it exists. */
 		free_runes(saved.runes);
+		/* Reset the mbstates */
+		memset(&l->c16rtomb, 0, sizeof(l->c16rtomb));
+		memset(&l->c32rtomb, 0, sizeof(l->c32rtomb));
+		memset(&l->mblen, 0, sizeof(l->mblen));
+		memset(&l->mbrlen, 0, sizeof(l->mbrlen));
+		memset(&l->mbrtoc16, 0, sizeof(l->mbrtoc16));
+		memset(&l->mbrtoc32, 0, sizeof(l->mbrtoc32));
+		memset(&l->mbrtowc, 0, sizeof(l->mbrtowc));
+		memset(&l->mbsnrtowcs, 0, sizeof(l->mbsnrtowcs));
+		memset(&l->mbsrtowcs, 0, sizeof(l->mbsrtowcs));
+		memset(&l->mbtowc, 0, sizeof(l->mbtowc));
+		memset(&l->wcrtomb, 0, sizeof(l->wcrtomb));
+		memset(&l->wcsnrtombs, 0, sizeof(l->wcsnrtombs));
+		memset(&l->wcsrtombs, 0, sizeof(l->wcsrtombs));
+		memset(&l->wctomb, 0, sizeof(l->wctomb));
 	} else {
 		/* Restore the saved version if this failed. */
 		memcpy(l, &saved, sizeof(struct xlocale_ctype));
diff --git a/lib/libc/locale/wcrtomb.c b/lib/libc/locale/wcrtomb.c
index 1afa8f77acc9..fa5c08474475 100644
--- a/lib/libc/locale/wcrtomb.c
+++ b/lib/libc/locale/wcrtomb.c
@@ -43,7 +43,7 @@ wcrtomb_l(char * __restrict s, wchar_t wc, mbstate_t * __restrict ps,
 {
 	FIX_LOCALE(locale);
 	if (ps == NULL)
-		ps = &locale->wcrtomb;
+		ps = &(XLOCALE_CTYPE(locale)->wcrtomb);
 	return (XLOCALE_CTYPE(locale)->__wcrtomb(s, wc, ps));
 }
 
diff --git a/lib/libc/locale/wcsnrtombs.c b/lib/libc/locale/wcsnrtombs.c
index 8d90445aacf1..7fdbfc75156d 100644
--- a/lib/libc/locale/wcsnrtombs.c
+++ b/lib/libc/locale/wcsnrtombs.c
@@ -48,7 +48,7 @@ wcsnrtombs_l(char * __restrict dst, const wchar_t ** __restrict src, size_t nwc,
 {
 	FIX_LOCALE(locale);
 	if (ps == NULL)
-		ps = &locale->wcsnrtombs;
+		ps = &(XLOCALE_CTYPE(locale)->wcsnrtombs);
 	return (XLOCALE_CTYPE(locale)->__wcsnrtombs(dst, src, nwc, len, ps));
 }
 size_t
diff --git a/lib/libc/locale/wcsrtombs.c b/lib/libc/locale/wcsrtombs.c
index ca9875799a3d..1466eab7046d 100644
--- a/lib/libc/locale/wcsrtombs.c
+++ b/lib/libc/locale/wcsrtombs.c
@@ -46,7 +46,7 @@ wcsrtombs_l(char * __restrict dst, const wchar_t ** __restrict src, size_t len,
 {
 	FIX_LOCALE(locale);
 	if (ps == NULL)
-		ps = &locale->wcsrtombs;
+		ps = &(XLOCALE_CTYPE(locale)->wcsrtombs);
 	return (XLOCALE_CTYPE(locale)->__wcsnrtombs(dst, src, SIZE_T_MAX, len, ps));
 }
 
diff --git a/lib/libc/locale/wctomb.c b/lib/libc/locale/wctomb.c
index 151d67997548..820021a3e0ab 100644
--- a/lib/libc/locale/wctomb.c
+++ b/lib/libc/locale/wctomb.c
@@ -47,10 +47,11 @@ wctomb_l(char *s, wchar_t wchar, locale_t locale)
 
 	if (s == NULL) {
 		/* No support for state dependent encodings. */
-		locale->wctomb = initial;
+		XLOCALE_CTYPE(locale)->wctomb = initial;
 		return (0);
 	}
-	if ((rval = XLOCALE_CTYPE(locale)->__wcrtomb(s, wchar, &locale->wctomb)) == (size_t)-1)
+	if ((rval = XLOCALE_CTYPE(locale)->__wcrtomb(s, wchar,
+	    &(XLOCALE_CTYPE(locale)->wctomb))) == (size_t)-1)
 		return (-1);
 	return ((int)rval);
 }
diff --git a/lib/libc/locale/xlocale_private.h b/lib/libc/locale/xlocale_private.h
index 9aa4d86c87ca..fc04c9dd43a3 100644
--- a/lib/libc/locale/xlocale_private.h
+++ b/lib/libc/locale/xlocale_private.h
@@ -128,34 +128,6 @@ struct _xlocale {
 	int using_messages_locale;
 	/** The structure to be returned from localeconv_l() for this locale. */
 	struct lconv lconv;
-	/** Persistent state used by mblen() calls. */
-	__mbstate_t mblen;
-	/** Persistent state used by mbrlen() calls. */
-	__mbstate_t mbrlen;
-	/** Persistent state used by mbrtoc16() calls. */
-	__mbstate_t mbrtoc16;
-	/** Persistent state used by mbrtoc32() calls. */
-	__mbstate_t mbrtoc32;
-	/** Persistent state used by mbrtowc() calls. */
-	__mbstate_t mbrtowc;
-	/** Persistent state used by mbsnrtowcs() calls. */
-	__mbstate_t mbsnrtowcs;
-	/** Persistent state used by mbsrtowcs() calls. */
-	__mbstate_t mbsrtowcs;
-	/** Persistent state used by mbtowc() calls. */
-	__mbstate_t mbtowc;
-	/** Persistent state used by c16rtomb() calls. */
-	__mbstate_t c16rtomb;
-	/** Persistent state used by c32rtomb() calls. */
-	__mbstate_t c32rtomb;
-	/** Persistent state used by wcrtomb() calls. */
-	__mbstate_t wcrtomb;
-	/** Persistent state used by wcsnrtombs() calls. */
-	__mbstate_t wcsnrtombs;
-	/** Persistent state used by wcsrtombs() calls. */
-	__mbstate_t wcsrtombs;
-	/** Persistent state used by wctomb() calls. */
-	__mbstate_t wctomb;
 	/** Buffer used by nl_langinfo_l() */
 	char *csym;
 };