d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Locale fix for endian big (EB) machines.

Browse Source 
We have locale files generated on EL machines (e.g. during cross-build
on amd64 host), but then we are using them on EB machines (e.g. MIPS64EB),
so proceed byte-swap if necessary.

All the libc tests passed successfully, including Russian collation.

Tested by:	br@, Hongyan Xia <hx242@cam.ac.uk>
Sponsored by:	DARPA, AFRL
Sponsored by:	HEIF5
Differential Revision:	https://reviews.freebsd.org/D8281
This commit is contained in:
Ruslan Bukin 2016-11-01 13:54:44 +00:00
parent 2d1d8f4c8f
commit 77bc2a1cd6
3 changed files with 103 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
lib/libc/locale/collate.c
View File

@ -53,6 +53,7 @@ __FBSDID("$FreeBSD$");
#include <fcntl.h>
#include "un-namespace.h"
#include "endian.h"
#include "collate.h"
#include "setlocale.h"
#include "ldpart.h"
@ -160,7 +161,7 @@ __collate_load_tables_l(const char *encoding, struct xlocale_collate *table)
if ((info->directive_count < 1) ||
(info->directive_count >= COLL_WEIGHTS_MAX) ||
((chains = info->chain_count) < 0)) {
((chains = BSWAP(info->chain_count)) < 0)) {
(void) munmap(map, sbuf.st_size);
errno = EINVAL;
return (_LDP_ERROR);
@ -168,9 +169,9 @@ __collate_load_tables_l(const char *encoding, struct xlocale_collate *table)
i = (sizeof (collate_char_t) * (UCHAR_MAX + 1)) +
(sizeof (collate_chain_t) * chains) +
(sizeof (collate_large_t) * info->large_count);
(sizeof (collate_large_t) * BSWAP(info->large_count));
for (z = 0; z < info->directive_count; z++) {
i += sizeof (collate_subst_t) * info->subst_count[z];
i += sizeof (collate_subst_t) * BSWAP(info->subst_count[z]);
}
if (i != (sbuf.st_size - (TMP - map))) {
(void) munmap(map, sbuf.st_size);
@ -183,9 +184,9 @@ __collate_load_tables_l(const char *encoding, struct xlocale_collate *table)
TMP += sizeof (collate_char_t) * (UCHAR_MAX + 1);
for (z = 0; z < info->directive_count; z++) {
if (info->subst_count[z] > 0) {
if (BSWAP(info->subst_count[z]) > 0) {
table->subst_table[z] = (void *)TMP;
TMP += info->subst_count[z] * sizeof (collate_subst_t);
TMP += BSWAP(info->subst_count[z]) * sizeof (collate_subst_t);
} else {
table->subst_table[z] = NULL;
}
@ -196,7 +197,7 @@ __collate_load_tables_l(const char *encoding, struct xlocale_collate *table)
TMP += chains * sizeof (collate_chain_t);
} else
table->chain_pri_table = NULL;
if (info->large_count > 0)
if (BSWAP(info->large_count) > 0)
table->large_pri_table = (void *)TMP;
else
table->large_pri_table = NULL;
@ -209,7 +210,7 @@ static const int32_t *
substsearch(struct xlocale_collate *table, const wchar_t key, int pass)
{
const collate_subst_t *p;
int n = table->info->subst_count[pass];
int n = BSWAP(table->info->subst_count[pass]);
if (n == 0)
return (NULL);
@ -221,7 +222,8 @@ substsearch(struct xlocale_collate *table, const wchar_t key, int pass)
return (NULL);
p = table->subst_table[pass] + (key & ~COLLATE_SUBST_PRIORITY);
assert(p->key == key);
assert(BSWAP(p->key) == key);
return (p->pri);
}
@ -229,7 +231,7 @@ static collate_chain_t *
chainsearch(struct xlocale_collate *table, const wchar_t *key, int *len)
{
int low = 0;
int high = table->info->chain_count - 1;;
int high = BSWAP(table->info->chain_count) - 1;
int next, compar, l;
collate_chain_t *p;
collate_chain_t *tab = table->chain_pri_table;
@ -240,7 +242,7 @@ chainsearch(struct xlocale_collate *table, const wchar_t *key, int *len)
while (low <= high) {
next = (low + high) / 2;
p = tab + next;
compar = *key - *p->str;
compar = *key - le16toh(*p->str);
if (compar == 0) {
l = wcsnlen(p->str, COLLATE_STR_LEN);
compar = wcsncmp(key, p->str, l);
@ -261,7 +263,7 @@ static collate_large_t *
largesearch(struct xlocale_collate *table, const wchar_t key)
{
int low = 0;
int high = table->info->large_count - 1;
int high = BSWAP(table->info->large_count) - 1;
int next, compar;
collate_large_t *p;
collate_large_t *tab = table->large_pri_table;
@ -272,7 +274,7 @@ largesearch(struct xlocale_collate *table, const wchar_t key)
while (low <= high) {
next = (low + high) / 2;
p = tab + next;
compar = key - p->val;
compar = key - BSWAP(p->val);
if (compar == 0)
return (p);
if (compar > 0)
@ -337,15 +339,15 @@ _collate_lookup(struct xlocale_collate *table, const wchar_t *t, int *len,
* Character is a small (8-bit) character.
* We just look these up directly for speed.
*/
*pri = table->char_pri_table[*t].pri[which];
*pri = BSWAP(table->char_pri_table[*t].pri[which]);
} else if ((table->info->large_count > 0) &&
} else if ((BSWAP(table->info->large_count) > 0) &&
((match = largesearch(table, *t)) != NULL)) {
/*
* Character was found in the extended table.
*/
*pri = match->pri.pri[which];
*pri = BSWAP(match->pri.pri[which]);
} else {
/*
@ -355,7 +357,7 @@ _collate_lookup(struct xlocale_collate *table, const wchar_t *t, int *len,
/* Mask off sign bit to prevent ordering confusion. */
*pri = (*t & COLLATE_MAX_PRIORITY);
} else {
*pri = table->info->undef_pri[which];
*pri = BSWAP(table->info->undef_pri[which]);
}
/* No substitutions for undefined characters! */
return;
@ -374,9 +376,9 @@ _collate_lookup(struct xlocale_collate *table, const wchar_t *t, int *len,
* code ensures this for us.
*/
if ((sptr = substsearch(table, *pri, which)) != NULL) {
if ((*pri = *sptr) > 0) {
if ((*pri = BSWAP(*sptr)) > 0) {
sptr++;
*state = *sptr ? sptr : NULL;
*state = BSWAP(*sptr) ? sptr : NULL;
}
}
@ -518,7 +520,7 @@ static int
xfrm(struct xlocale_collate *table, unsigned char *p, int pri, int pass)
{
/* we use unsigned to ensure zero fill on right shift */
uint32_t val = (uint32_t)table->info->pri_count[pass];
uint32_t val = BSWAP((uint32_t)table->info->pri_count[pass]);
int nc = 0;
while (val) {
@ -678,7 +680,7 @@ __collate_equiv_value(locale_t locale, const wchar_t *str, size_t len)
e = -1;
if (*str <= UCHAR_MAX)
e = table->char_pri_table[*str].pri[0];
else if (table->info->large_count > 0) {
else if (BSWAP(table->info->large_count) > 0) {
collate_large_t *match_large;
match_large = largesearch(table, *str);
if (match_large)
@ -688,7 +690,7 @@ __collate_equiv_value(locale_t locale, const wchar_t *str, size_t len)
return (1);
return (e > 0 ? e : 0);
}
if (table->info->chain_count > 0) {
if (BSWAP(table->info->chain_count) > 0) {
wchar_t name[COLLATE_STR_LEN];
collate_chain_t *match_chain;
int clen;

52
lib/libc/locale/endian.h Normal file
View File

@ -0,0 +1,52 @@
/*-
* Copyright (c) 2016 Ruslan Bukin <br@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
* University of Cambridge Computer Laboratory under DARPA/AFRL contract
* FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
*
* Portions of this software were developed by the University of Cambridge
* Computer Laboratory as part of the CTSRD Project, with support from the
* UK Higher Education Innovation Fund (HEIF).
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#include <sys/endian.h>
/*
* We assume locale files were generated on EL machine
* (e.g. during cross build on amd64 host), but used on EB
* machine (e.g. MIPS64EB), so convert it to host endianness.
*
* TODO: detect host endianness on the build machine and use
* correct macros here.
*/
#if BYTE_ORDER == BIG_ENDIAN && defined(__mips__)
#define BSWAP(x) le32toh(x)
#else
#define BSWAP(x) x
#endif

55
lib/libc/locale/rune.c
View File

@ -52,6 +52,7 @@ __FBSDID("$FreeBSD$");
#include <unistd.h>
#include "un-namespace.h"
#include "endian.h"
#include "runefile.h"
_RuneLocale *
@ -107,29 +108,29 @@ _Read_RuneMagi(const char *fname)
}
runetype_ext_ranges = (_FileRuneEntry *)variable;
variable = runetype_ext_ranges + frl->runetype_ext_nranges;
variable = runetype_ext_ranges + BSWAP(frl->runetype_ext_nranges);
if (variable > lastp) {
goto invalid;
}
maplower_ext_ranges = (_FileRuneEntry *)variable;
variable = maplower_ext_ranges + frl->maplower_ext_nranges;
variable = maplower_ext_ranges + BSWAP(frl->maplower_ext_nranges);
if (variable > lastp) {
goto invalid;
}
mapupper_ext_ranges = (_FileRuneEntry *)variable;
variable = mapupper_ext_ranges + frl->mapupper_ext_nranges;
variable = mapupper_ext_ranges + BSWAP(frl->mapupper_ext_nranges);
if (variable > lastp) {
goto invalid;
}
frr = runetype_ext_ranges;
for (x = 0; x < frl->runetype_ext_nranges; ++x) {
for (x = 0; x < BSWAP(frl->runetype_ext_nranges); ++x) {
uint32_t *types;
if (frr[x].map == 0) {
int len = frr[x].max - frr[x].min + 1;
if (BSWAP(frr[x].map) == 0) {
int len = BSWAP(frr[x].max) - BSWAP(frr[x].min) + 1;
types = variable;
variable = types + len;
runetype_ext_len += len;
@ -139,7 +140,7 @@ _Read_RuneMagi(const char *fname)
}
}
if ((char *)variable + frl->variable_len > (char *)lastp) {
if ((char *)variable + BSWAP(frl->variable_len) > (char *)lastp) {
goto invalid;
}
@ -147,10 +148,10 @@ _Read_RuneMagi(const char *fname)
* Convert from disk format to host format.
*/
data = malloc(sizeof(_RuneLocale) +
(frl->runetype_ext_nranges + frl->maplower_ext_nranges +
frl->mapupper_ext_nranges) * sizeof(_RuneEntry) +
(BSWAP(frl->runetype_ext_nranges) + BSWAP(frl->maplower_ext_nranges) +
BSWAP(frl->mapupper_ext_nranges)) * sizeof(_RuneEntry) +
runetype_ext_len * sizeof(*rr->__types) +
frl->variable_len);
BSWAP(frl->variable_len));
if (data == NULL) {
saverr = errno;
munmap(fdata, sb.st_size);
@ -164,15 +165,15 @@ _Read_RuneMagi(const char *fname)
memcpy(rl->__magic, _RUNE_MAGIC_1, sizeof(rl->__magic));
memcpy(rl->__encoding, frl->encoding, sizeof(rl->__encoding));
rl->__variable_len = frl->variable_len;
rl->__runetype_ext.__nranges = frl->runetype_ext_nranges;
rl->__maplower_ext.__nranges = frl->maplower_ext_nranges;
rl->__mapupper_ext.__nranges = frl->mapupper_ext_nranges;
rl->__variable_len = BSWAP(frl->variable_len);
rl->__runetype_ext.__nranges = BSWAP(frl->runetype_ext_nranges);
rl->__maplower_ext.__nranges = BSWAP(frl->maplower_ext_nranges);
rl->__mapupper_ext.__nranges = BSWAP(frl->mapupper_ext_nranges);
for (x = 0; x < _CACHED_RUNES; ++x) {
rl->__runetype[x] = frl->runetype[x];
rl->__maplower[x] = frl->maplower[x];
rl->__mapupper[x] = frl->mapupper[x];
rl->__runetype[x] = BSWAP(frl->runetype[x]);
rl->__maplower[x] = BSWAP(frl->maplower[x]);
rl->__mapupper[x] = BSWAP(frl->mapupper[x]);
}
rl->__runetype_ext.__ranges = (_RuneEntry *)rl->__variable;
@ -187,15 +188,15 @@ _Read_RuneMagi(const char *fname)
rl->__variable = rl->__mapupper_ext.__ranges +
rl->__mapupper_ext.__nranges;
variable = mapupper_ext_ranges + frl->mapupper_ext_nranges;
variable = mapupper_ext_ranges + BSWAP(frl->mapupper_ext_nranges);
frr = runetype_ext_ranges;
rr = rl->__runetype_ext.__ranges;
for (x = 0; x < rl->__runetype_ext.__nranges; ++x) {
uint32_t *types;
rr[x].__min = frr[x].min;
rr[x].__max = frr[x].max;
rr[x].__map = frr[x].map;
rr[x].__min = BSWAP(frr[x].min);
rr[x].__max = BSWAP(frr[x].max);
rr[x].__map = BSWAP(frr[x].map);
if (rr[x].__map == 0) {
int len = rr[x].__max - rr[x].__min + 1;
types = variable;
@ -211,17 +212,17 @@ _Read_RuneMagi(const char *fname)
frr = maplower_ext_ranges;
rr = rl->__maplower_ext.__ranges;
for (x = 0; x < rl->__maplower_ext.__nranges; ++x) {
rr[x].__min = frr[x].min;
rr[x].__max = frr[x].max;
rr[x].__map = frr[x].map;
rr[x].__min = BSWAP(frr[x].min);
rr[x].__max = BSWAP(frr[x].max);
rr[x].__map = BSWAP(frr[x].map);
}
frr = mapupper_ext_ranges;
rr = rl->__mapupper_ext.__ranges;
for (x = 0; x < rl->__mapupper_ext.__nranges; ++x) {
rr[x].__min = frr[x].min;
rr[x].__max = frr[x].max;
rr[x].__map = frr[x].map;
rr[x].__min = BSWAP(frr[x].min);
rr[x].__max = BSWAP(frr[x].max);
rr[x].__map = BSWAP(frr[x].map);
}
memcpy(rl->__variable, variable, rl->__variable_len);