d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Reset persistent mbstates when rune locale encoding changes.

Browse Source 
This was shown to be a problem by side effect of now-enabled test case,
which was going through C, en_US.UTF-8, ja_JP.SJIS, and ja_JP.eucJP,
and failing eventually as data in mbrtowc's mbstate, that was
perfectly correct for en_US.UTF-8 was treated as incorrect for
ja_JP.SJIS, failing the entire test case.

This makes the persistent mbstates to be per ctype-component,
and not per-locale so we could easily reset the mbstates when
only LC_CTYPE is changed.

Reviewed by:	bapt, pfg
Approved by:	kib (mentor, implicit)
Differential Revision:	https://reviews.freebsd.org/D17796
This commit is contained in:
Yuri Pankov 2018-11-09 03:32:53 +00:00
parent e1e300b671
commit 9eb7d595e1
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=340276
16 changed files with 63 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
contrib/netbsd-tests/lib/libc/locale/t_mbrtowc.c
View File

@ -243,9 +243,6 @@ ATF_TC_BODY(mbrtowc_internal, tc)
{
struct test *t;
#ifdef __FreeBSD__
atf_tc_expect_fail("ja_* locale fails");
#endif
for (t = &tests[0]; t->data != NULL; ++t)
h_ctype2(t, false);
}

4
lib/libc/locale/cXXrtomb_iconv.h
View File

@ -39,7 +39,7 @@ __FBSDID("$FreeBSD$");
#include "../iconv/citrus_hash.h"
#include "../iconv/citrus_module.h"
#include "../iconv/citrus_iconv.h"
#include "xlocale_private.h"
#include "mblocal.h"
typedef struct {
bool initialized;
@ -65,7 +65,7 @@ cXXrtomb_l(char * __restrict s, charXX_t c, mbstate_t * __restrict ps,
FIX_LOCALE(locale);
if (ps == NULL)
ps = &locale->cXXrtomb;
ps = &(XLOCALE_CTYPE(locale)->cXXrtomb);
cs = (_ConversionState *)ps;
handle = &cs->iconv;

5
lib/libc/locale/mblen.c
View File

@ -47,10 +47,11 @@ mblen_l(const char *s, size_t n, locale_t locale)
if (s == NULL) {
/* No support for state dependent encodings. */
locale->mblen = initial;
XLOCALE_CTYPE(locale)->mblen = initial;
return (0);
}
rval = XLOCALE_CTYPE(locale)->__mbrtowc(NULL, s, n, &locale->mblen);
rval = XLOCALE_CTYPE(locale)->__mbrtowc(NULL, s, n,
&(XLOCALE_CTYPE(locale)->mblen));
if (rval == (size_t)-1 || rval == (size_t)-2)
return (-1);
return ((int)rval);

28
lib/libc/locale/mblocal.h
View File

@ -60,6 +60,34 @@ struct xlocale_ctype {
size_t, size_t, mbstate_t * __restrict);
int __mb_cur_max;
int __mb_sb_limit;
/** Persistent state used by mblen() calls. */
__mbstate_t mblen;
/** Persistent state used by mbrlen() calls. */
__mbstate_t mbrlen;
/** Persistent state used by mbrtoc16() calls. */
__mbstate_t mbrtoc16;
/** Persistent state used by mbrtoc32() calls. */
__mbstate_t mbrtoc32;
/** Persistent state used by mbrtowc() calls. */
__mbstate_t mbrtowc;
/** Persistent state used by mbsnrtowcs() calls. */
__mbstate_t mbsnrtowcs;
/** Persistent state used by mbsrtowcs() calls. */
__mbstate_t mbsrtowcs;
/** Persistent state used by mbtowc() calls. */
__mbstate_t mbtowc;
/** Persistent state used by c16rtomb() calls. */
__mbstate_t c16rtomb;
/** Persistent state used by c32rtomb() calls. */
__mbstate_t c32rtomb;
/** Persistent state used by wcrtomb() calls. */
__mbstate_t wcrtomb;
/** Persistent state used by wcsnrtombs() calls. */
__mbstate_t wcsnrtombs;
/** Persistent state used by wcsrtombs() calls. */
__mbstate_t wcsrtombs;
/** Persistent state used by wctomb() calls. */
__mbstate_t wctomb;
};
#define XLOCALE_CTYPE(x) ((struct xlocale_ctype*)(x)->components[XLC_CTYPE])
extern struct xlocale_ctype __xlocale_global_ctype;

2
lib/libc/locale/mbrlen.c
View File

@ -42,7 +42,7 @@ mbrlen_l(const char * __restrict s, size_t n, mbstate_t * __restrict ps, locale_
{
FIX_LOCALE(locale);
if (ps == NULL)
ps = &locale->mbrlen;
ps = &(XLOCALE_CTYPE(locale)->mbrlen);
return (XLOCALE_CTYPE(locale)->__mbrtowc(NULL, s, n, ps));
}

4
lib/libc/locale/mbrtocXX_iconv.h
View File

@ -41,7 +41,7 @@ __FBSDID("$FreeBSD$");
#include "../iconv/citrus_hash.h"
#include "../iconv/citrus_module.h"
#include "../iconv/citrus_iconv.h"
#include "xlocale_private.h"
#include "mblocal.h"
typedef struct {
bool initialized;
@ -68,7 +68,7 @@ mbrtocXX_l(charXX_t * __restrict pc, const char * __restrict s, size_t n,
FIX_LOCALE(locale);
if (ps == NULL)
ps = &locale->mbrtocXX;
ps = &(XLOCALE_CTYPE(locale)->mbrtocXX);
cs = (_ConversionState *)ps;
handle = &cs->iconv;

2
lib/libc/locale/mbrtowc.c
View File

@ -43,7 +43,7 @@ mbrtowc_l(wchar_t * __restrict pwc, const char * __restrict s,
{
FIX_LOCALE(locale);
if (ps == NULL)
ps = &locale->mbrtowc;
ps = &(XLOCALE_CTYPE(locale)->mbrtowc);
return (XLOCALE_CTYPE(locale)->__mbrtowc(pwc, s, n, ps));
}

2
lib/libc/locale/mbsnrtowcs.c
View File

@ -48,7 +48,7 @@ mbsnrtowcs_l(wchar_t * __restrict dst, const char ** __restrict src,
{
FIX_LOCALE(locale);
if (ps == NULL)
ps = &locale->mbsnrtowcs;
ps = &(XLOCALE_CTYPE(locale)->mbsnrtowcs);
return (XLOCALE_CTYPE(locale)->__mbsnrtowcs(dst, src, nms, len, ps));
}
size_t

2
lib/libc/locale/mbsrtowcs.c
View File

@ -46,7 +46,7 @@ mbsrtowcs_l(wchar_t * __restrict dst, const char ** __restrict src, size_t len,
{
FIX_LOCALE(locale);
if (ps == NULL)
ps = &locale->mbsrtowcs;
ps = &(XLOCALE_CTYPE(locale)->mbsrtowcs);
return (XLOCALE_CTYPE(locale)->__mbsnrtowcs(dst, src, SIZE_T_MAX, len, ps));
}
size_t

5
lib/libc/locale/mbtowc.c
View File

@ -48,10 +48,11 @@ mbtowc_l(wchar_t * __restrict pwc, const char * __restrict s, size_t n, locale_t
if (s == NULL) {
/* No support for state dependent encodings. */
locale->mbtowc = initial;
XLOCALE_CTYPE(locale)->mbtowc = initial;
return (0);
}
rval = XLOCALE_CTYPE(locale)->__mbrtowc(pwc, s, n, &locale->mbtowc);
rval = XLOCALE_CTYPE(locale)->__mbrtowc(pwc, s, n,
&(XLOCALE_CTYPE(locale)->mbtowc));
switch (rval) {
case (size_t)-2:
errno = EILSEQ;

15
lib/libc/locale/setrunelocale.c
View File

@ -160,6 +160,21 @@ __setrunelocale(struct xlocale_ctype *l, const char *encoding)
if (ret == 0) {
/* Free the old runes if it exists. */
free_runes(saved.runes);
/* Reset the mbstates */
memset(&l->c16rtomb, 0, sizeof(l->c16rtomb));
memset(&l->c32rtomb, 0, sizeof(l->c32rtomb));
memset(&l->mblen, 0, sizeof(l->mblen));
memset(&l->mbrlen, 0, sizeof(l->mbrlen));
memset(&l->mbrtoc16, 0, sizeof(l->mbrtoc16));
memset(&l->mbrtoc32, 0, sizeof(l->mbrtoc32));
memset(&l->mbrtowc, 0, sizeof(l->mbrtowc));
memset(&l->mbsnrtowcs, 0, sizeof(l->mbsnrtowcs));
memset(&l->mbsrtowcs, 0, sizeof(l->mbsrtowcs));
memset(&l->mbtowc, 0, sizeof(l->mbtowc));
memset(&l->wcrtomb, 0, sizeof(l->wcrtomb));
memset(&l->wcsnrtombs, 0, sizeof(l->wcsnrtombs));
memset(&l->wcsrtombs, 0, sizeof(l->wcsrtombs));
memset(&l->wctomb, 0, sizeof(l->wctomb));
} else {
/* Restore the saved version if this failed. */
memcpy(l, &saved, sizeof(struct xlocale_ctype));

2
lib/libc/locale/wcrtomb.c
View File

@ -43,7 +43,7 @@ wcrtomb_l(char * __restrict s, wchar_t wc, mbstate_t * __restrict ps,
{
FIX_LOCALE(locale);
if (ps == NULL)
ps = &locale->wcrtomb;
ps = &(XLOCALE_CTYPE(locale)->wcrtomb);
return (XLOCALE_CTYPE(locale)->__wcrtomb(s, wc, ps));
}

2
lib/libc/locale/wcsnrtombs.c
View File

@ -48,7 +48,7 @@ wcsnrtombs_l(char * __restrict dst, const wchar_t ** __restrict src, size_t nwc,
{
FIX_LOCALE(locale);
if (ps == NULL)
ps = &locale->wcsnrtombs;
ps = &(XLOCALE_CTYPE(locale)->wcsnrtombs);
return (XLOCALE_CTYPE(locale)->__wcsnrtombs(dst, src, nwc, len, ps));
}
size_t

2
lib/libc/locale/wcsrtombs.c
View File

@ -46,7 +46,7 @@ wcsrtombs_l(char * __restrict dst, const wchar_t ** __restrict src, size_t len,
{
FIX_LOCALE(locale);
if (ps == NULL)
ps = &locale->wcsrtombs;
ps = &(XLOCALE_CTYPE(locale)->wcsrtombs);
return (XLOCALE_CTYPE(locale)->__wcsnrtombs(dst, src, SIZE_T_MAX, len, ps));
}

5
lib/libc/locale/wctomb.c
View File

@ -47,10 +47,11 @@ wctomb_l(char *s, wchar_t wchar, locale_t locale)
if (s == NULL) {
/* No support for state dependent encodings. */
locale->wctomb = initial;
XLOCALE_CTYPE(locale)->wctomb = initial;
return (0);
}
if ((rval = XLOCALE_CTYPE(locale)->__wcrtomb(s, wchar, &locale->wctomb)) == (size_t)-1)
if ((rval = XLOCALE_CTYPE(locale)->__wcrtomb(s, wchar,
&(XLOCALE_CTYPE(locale)->wctomb))) == (size_t)-1)
return (-1);
return ((int)rval);
}

28
lib/libc/locale/xlocale_private.h
View File

@ -128,34 +128,6 @@ struct _xlocale {
int using_messages_locale;
/** The structure to be returned from localeconv_l() for this locale. */
struct lconv lconv;
/** Persistent state used by mblen() calls. */
__mbstate_t mblen;
/** Persistent state used by mbrlen() calls. */
__mbstate_t mbrlen;
/** Persistent state used by mbrtoc16() calls. */
__mbstate_t mbrtoc16;
/** Persistent state used by mbrtoc32() calls. */
__mbstate_t mbrtoc32;
/** Persistent state used by mbrtowc() calls. */
__mbstate_t mbrtowc;
/** Persistent state used by mbsnrtowcs() calls. */
__mbstate_t mbsnrtowcs;
/** Persistent state used by mbsrtowcs() calls. */
__mbstate_t mbsrtowcs;
/** Persistent state used by mbtowc() calls. */
__mbstate_t mbtowc;
/** Persistent state used by c16rtomb() calls. */
__mbstate_t c16rtomb;
/** Persistent state used by c32rtomb() calls. */
__mbstate_t c32rtomb;
/** Persistent state used by wcrtomb() calls. */
__mbstate_t wcrtomb;
/** Persistent state used by wcsnrtombs() calls. */
__mbstate_t wcsnrtombs;
/** Persistent state used by wcsrtombs() calls. */
__mbstate_t wcsrtombs;
/** Persistent state used by wctomb() calls. */
__mbstate_t wctomb;
/** Buffer used by nl_langinfo_l() */
char *csym;
};