MFV r262756:

Import xz-embedded from git.

This is from commit hash '6a8a2364434763a033781f6b2a605ace9a021013'.

This makes it possible to use CRC64 but for now it's intentionally
not added to build.
This commit is contained in:
Xin LI 2014-03-05 01:33:15 +00:00
commit f0bd5302dd
12 changed files with 343 additions and 46 deletions

@ -7,7 +7,7 @@ XZ Embedded
XZ Embedded was written for use in the Linux kernel, but the code can
be easily used in other environments too, including regular userspace
applications.
applications. See userspace/xzminidec.c for an example program.
This README contains information that is useful only when the copy
of XZ Embedded isn't part of the Linux kernel tree. You should also
@ -84,6 +84,42 @@ Embedding into userspace applications
environment. Probably you should at least skim through it even if the
default file works as is.
Integrity check support
XZ Embedded always supports the integrity check types None and
CRC32. Support for CRC64 is optional. SHA-256 is currently not
supported in XZ Embedded although the .xz format does support it.
The xz tool from XZ Utils uses CRC64 by default, but CRC32 is usually
enough in embedded systems to keep the code size smaller.
If you want support for CRC64, you need to copy linux/lib/xz/xz_crc64.c
into your application, and #define XZ_USE_CRC64 in xz_config.h or in
compiler flags.
When using the internal CRC32 or CRC64, their lookup tables need to be
initialized with xz_crc32_init() and xz_crc64_init(), respectively.
See xz.h for details.
To use external CRC32 or CRC64 code instead of the code from
xz_crc32.c or xz_crc64.c, the following #defines may be used
in xz_config.h or in compiler flags:
#define XZ_INTERNAL_CRC32 0
#define XZ_INTERNAL_CRC64 0
Then it is up to you to provide compatible xz_crc32() or xz_crc64()
functions.
If the .xz file being decompressed uses an integrity check type that
isn't supported by XZ Embedded, it is treated as an error and the
file cannot be decompressed. For multi-call mode, this can be modified
by #defining XZ_DEC_ANY_CHECK. Then xz_dec_run() will return
XZ_UNSUPPORTED_CHECK when unsupported check type is detected. After
that decompression can be continued normally except that the
integrity check won't be verified. In single-call mode there's
no way to continue decoding, so XZ_DEC_ANY_CHECK is almost useless
in single-call mode.
BCJ filter support
If you want support for one or more BCJ filters, you need to copy also

@ -256,6 +256,22 @@ XZ_EXTERN void xz_dec_end(struct xz_dec *s);
# endif
#endif
/*
* If CRC64 support has been enabled with XZ_USE_CRC64, a CRC64
* implementation is needed too.
*/
#ifndef XZ_USE_CRC64
# undef XZ_INTERNAL_CRC64
# define XZ_INTERNAL_CRC64 0
#endif
#ifndef XZ_INTERNAL_CRC64
# ifdef __KERNEL__
# error Using CRC64 in the kernel has not been implemented.
# else
# define XZ_INTERNAL_CRC64 1
# endif
#endif
#if XZ_INTERNAL_CRC32
/*
* This must be called before any other xz_* function to initialize
@ -271,6 +287,21 @@ XZ_EXTERN void xz_crc32_init(void);
XZ_EXTERN uint32_t xz_crc32(const uint8_t *buf, size_t size, uint32_t crc);
#endif
#if XZ_INTERNAL_CRC64
/*
* This must be called before any other xz_* function (except xz_crc32_init())
* to initialize the CRC64 lookup table.
*/
XZ_EXTERN void xz_crc64_init(void);
/*
* Update CRC64 value using the polynomial from ECMA-182. To start a new
* calculation, the third argument must be zero. To continue the calculation,
* the previously returned value is passed as the third argument.
*/
XZ_EXTERN uint64_t xz_crc64(const uint8_t *buf, size_t size, uint64_t crc);
#endif
#ifdef __cplusplus
}
#endif

@ -83,7 +83,7 @@
* safety_margin = 128 + uncompressed_size * 8 / 32768 + 65536
* = 128 + (uncompressed_size >> 12) + 65536
*
* For comparision, according to arch/x86/boot/compressed/misc.c, the
* For comparison, according to arch/x86/boot/compressed/misc.c, the
* equivalent formula for Deflate is this:
*
* safety_margin = 18 + (uncompressed_size >> 12) + 32768

@ -6,42 +6,40 @@ config XZ_DEC
the .xz file format as the container. For integrity checking,
CRC32 is supported. See Documentation/xz.txt for more information.
if XZ_DEC
config XZ_DEC_X86
bool "x86 BCJ filter decoder" if EXPERT
default y
depends on XZ_DEC
bool "x86 BCJ filter decoder"
default y if X86
select XZ_DEC_BCJ
config XZ_DEC_POWERPC
bool "PowerPC BCJ filter decoder" if EXPERT
default y
depends on XZ_DEC
bool "PowerPC BCJ filter decoder"
default y if PPC
select XZ_DEC_BCJ
config XZ_DEC_IA64
bool "IA-64 BCJ filter decoder" if EXPERT
default y
depends on XZ_DEC
bool "IA-64 BCJ filter decoder"
default y if IA64
select XZ_DEC_BCJ
config XZ_DEC_ARM
bool "ARM BCJ filter decoder" if EXPERT
default y
depends on XZ_DEC
bool "ARM BCJ filter decoder"
default y if ARM
select XZ_DEC_BCJ
config XZ_DEC_ARMTHUMB
bool "ARM-Thumb BCJ filter decoder" if EXPERT
default y
depends on XZ_DEC
bool "ARM-Thumb BCJ filter decoder"
default y if (ARM && ARM_THUMB)
select XZ_DEC_BCJ
config XZ_DEC_SPARC
bool "SPARC BCJ filter decoder" if EXPERT
default y
depends on XZ_DEC
bool "SPARC BCJ filter decoder"
default y if SPARC
select XZ_DEC_BCJ
endif
config XZ_DEC_BCJ
bool
default n

@ -0,0 +1,50 @@
/*
* CRC64 using the polynomial from ECMA-182
*
* This file is similar to xz_crc32.c. See the comments there.
*
* Authors: Lasse Collin <lasse.collin@tukaani.org>
* Igor Pavlov <http://7-zip.org/>
*
* This file has been put into the public domain.
* You can do whatever you want with this file.
*/
#include "xz_private.h"
#ifndef STATIC_RW_DATA
# define STATIC_RW_DATA static
#endif
STATIC_RW_DATA uint64_t xz_crc64_table[256];
XZ_EXTERN void xz_crc64_init(void)
{
const uint64_t poly = 0xC96C5795D7870F42;
uint32_t i;
uint32_t j;
uint64_t r;
for (i = 0; i < 256; ++i) {
r = i;
for (j = 0; j < 8; ++j)
r = (r >> 1) ^ (poly & ~((r & 1) - 1));
xz_crc64_table[i] = r;
}
return;
}
XZ_EXTERN uint64_t xz_crc64(const uint8_t *buf, size_t size, uint64_t crc)
{
crc = ~crc;
while (size != 0) {
crc = xz_crc64_table[*buf++ ^ (crc & 0xFF)] ^ (crc >> 8);
--size;
}
return ~crc;
}

@ -10,6 +10,12 @@
#include "xz_private.h"
#include "xz_stream.h"
#ifdef XZ_USE_CRC64
# define IS_CRC64(check_type) ((check_type) == XZ_CHECK_CRC64)
#else
# define IS_CRC64(check_type) false
#endif
/* Hash used to validate the Index field */
struct xz_dec_hash {
vli_type unpadded;
@ -42,8 +48,13 @@ struct xz_dec {
size_t in_start;
size_t out_start;
#ifdef XZ_USE_CRC64
/* CRC32 or CRC64 value in Block or CRC32 value in Index */
uint64_t crc;
#else
/* CRC32 value in Block or Index */
uint32_t crc32;
uint32_t crc;
#endif
/* Type of the integrity check calculated from uncompressed data */
enum xz_check check_type;
@ -208,8 +219,8 @@ static enum xz_ret dec_vli(struct xz_dec *s, const uint8_t *in,
* the observed compressed and uncompressed sizes of the Block so that
* they don't exceed the values possibly stored in the Block Header
* (validation assumes that no integer overflow occurs, since vli_type
* is normally uint64_t). Update the CRC32 if presence of the CRC32
* field was indicated in Stream Header.
* is normally uint64_t). Update the CRC32 or CRC64 value if presence of
* the CRC32 or CRC64 field was indicated in Stream Header.
*
* Once the decoding is finished, validate that the observed sizes match
* the sizes possibly stored in the Block Header. Update the hash and
@ -242,8 +253,13 @@ static enum xz_ret dec_block(struct xz_dec *s, struct xz_buf *b)
return XZ_DATA_ERROR;
if (s->check_type == XZ_CHECK_CRC32)
s->crc32 = xz_crc32(b->out + s->out_start,
b->out_pos - s->out_start, s->crc32);
s->crc = xz_crc32(b->out + s->out_start,
b->out_pos - s->out_start, s->crc);
#ifdef XZ_USE_CRC64
else if (s->check_type == XZ_CHECK_CRC64)
s->crc = xz_crc64(b->out + s->out_start,
b->out_pos - s->out_start, s->crc);
#endif
if (ret == XZ_STREAM_END) {
if (s->block_header.compressed != VLI_UNKNOWN
@ -264,6 +280,8 @@ static enum xz_ret dec_block(struct xz_dec *s, struct xz_buf *b)
#else
if (s->check_type == XZ_CHECK_CRC32)
s->block.hash.unpadded += 4;
else if (IS_CRC64(s->check_type))
s->block.hash.unpadded += 8;
#endif
s->block.hash.uncompressed += s->block.uncompressed;
@ -282,7 +300,7 @@ static void index_update(struct xz_dec *s, const struct xz_buf *b)
{
size_t in_used = b->in_pos - s->in_start;
s->index.size += in_used;
s->crc32 = xz_crc32(b->in + s->in_start, in_used, s->crc32);
s->crc = xz_crc32(b->in + s->in_start, in_used, s->crc);
}
/*
@ -340,23 +358,25 @@ static enum xz_ret dec_index(struct xz_dec *s, struct xz_buf *b)
}
/*
* Validate that the next four input bytes match the value of s->crc32.
* s->pos must be zero when starting to validate the first byte.
* Validate that the next four or eight input bytes match the value
* of s->crc. s->pos must be zero when starting to validate the first byte.
* The "bits" argument allows using the same code for both CRC32 and CRC64.
*/
static enum xz_ret crc32_validate(struct xz_dec *s, struct xz_buf *b)
static enum xz_ret crc_validate(struct xz_dec *s, struct xz_buf *b,
uint32_t bits)
{
do {
if (b->in_pos == b->in_size)
return XZ_OK;
if (((s->crc32 >> s->pos) & 0xFF) != b->in[b->in_pos++])
if (((s->crc >> s->pos) & 0xFF) != b->in[b->in_pos++])
return XZ_DATA_ERROR;
s->pos += 8;
} while (s->pos < 32);
} while (s->pos < bits);
s->crc32 = 0;
s->crc = 0;
s->pos = 0;
return XZ_STREAM_END;
@ -397,10 +417,11 @@ static enum xz_ret dec_stream_header(struct xz_dec *s)
return XZ_OPTIONS_ERROR;
/*
* Of integrity checks, we support only none (Check ID = 0) and
* CRC32 (Check ID = 1). However, if XZ_DEC_ANY_CHECK is defined,
* we will accept other check types too, but then the check won't
* be verified and a warning (XZ_UNSUPPORTED_CHECK) will be given.
* Of integrity checks, we support none (Check ID = 0),
* CRC32 (Check ID = 1), and optionally CRC64 (Check ID = 4).
* However, if XZ_DEC_ANY_CHECK is defined, we will accept other
* check types too, but then the check won't be verified and
* a warning (XZ_UNSUPPORTED_CHECK) will be given.
*/
s->check_type = s->temp.buf[HEADER_MAGIC_SIZE + 1];
@ -408,10 +429,10 @@ static enum xz_ret dec_stream_header(struct xz_dec *s)
if (s->check_type > XZ_CHECK_MAX)
return XZ_OPTIONS_ERROR;
if (s->check_type > XZ_CHECK_CRC32)
if (s->check_type > XZ_CHECK_CRC32 && !IS_CRC64(s->check_type))
return XZ_UNSUPPORTED_CHECK;
#else
if (s->check_type > XZ_CHECK_CRC32)
if (s->check_type > XZ_CHECK_CRC32 && !IS_CRC64(s->check_type))
return XZ_OPTIONS_ERROR;
#endif
@ -645,7 +666,12 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b)
case SEQ_BLOCK_CHECK:
if (s->check_type == XZ_CHECK_CRC32) {
ret = crc32_validate(s, b);
ret = crc_validate(s, b, 32);
if (ret != XZ_STREAM_END)
return ret;
}
else if (IS_CRC64(s->check_type)) {
ret = crc_validate(s, b, 64);
if (ret != XZ_STREAM_END)
return ret;
}
@ -688,7 +714,7 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b)
s->sequence = SEQ_INDEX_CRC32;
case SEQ_INDEX_CRC32:
ret = crc32_validate(s, b);
ret = crc_validate(s, b, 32);
if (ret != XZ_STREAM_END)
return ret;
@ -802,7 +828,7 @@ XZ_EXTERN void xz_dec_reset(struct xz_dec *s)
s->sequence = SEQ_STREAM_HEADER;
s->allow_buf_error = false;
s->pos = 0;
s->crc32 = 0;
s->crc = 0;
memzero(&s->block, sizeof(s->block));
memzero(&s->index, sizeof(s->index));
s->temp.pos = 0;

@ -12,8 +12,8 @@
BCJ=
LZMA2OPTS=
case $ARCH in
x86|x86_64) BCJ=--x86 ;;
case $SRCARCH in
x86) BCJ=--x86 ;;
powerpc) BCJ=--powerpc ;;
ia64) BCJ=--ia64; LZMA2OPTS=pb=4 ;;
arm) BCJ=--arm ;;

@ -10,11 +10,11 @@
CC = gcc -std=gnu89
BCJ_CPPFLAGS = -DXZ_DEC_X86 -DXZ_DEC_POWERPC -DXZ_DEC_IA64 \
-DXZ_DEC_ARM -DXZ_DEC_ARMTHUMB -DXZ_DEC_SPARC
CPPFLAGS = -DXZ_DEC_ANY_CHECK
CPPFLAGS = -DXZ_USE_CRC64 -DXZ_DEC_ANY_CHECK
CFLAGS = -ggdb3 -O2 -pedantic -Wall -Wextra
RM = rm -f
VPATH = ../linux/include/linux ../linux/lib/xz
COMMON_SRCS = xz_crc32.c xz_dec_stream.c xz_dec_lzma2.c xz_dec_bcj.c
COMMON_SRCS = xz_crc32.c xz_crc64.c xz_dec_stream.c xz_dec_lzma2.c xz_dec_bcj.c
COMMON_OBJS = $(COMMON_SRCS:.c=.o)
XZMINIDEC_OBJS = xzminidec.o
BYTETEST_OBJS = bytetest.o

@ -19,6 +19,9 @@ static void error(/*const*/ char *msg)
fprintf(stderr, "%s\n", msg);
}
/* Disable the CRC64 support even if it was enabled in the Makefile. */
#undef XZ_USE_CRC64
#include "../linux/lib/decompress_unxz.c"
static uint8_t in[1024 * 1024];

@ -0,0 +1,135 @@
/*
* Lazy test for the case when the output size is known
*
* Author: Lasse Collin <lasse.collin@tukaani.org>
*
* This file has been put into the public domain.
* You can do whatever you want with this file.
*/
#include <stdbool.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include "xz.h"
static uint8_t in[1];
static uint8_t out[BUFSIZ];
int main(int argc, char **argv)
{
struct xz_buf b;
struct xz_dec *s;
enum xz_ret ret;
const char *msg;
size_t uncomp_size;
if (argc != 2) {
fputs("Give uncompressed size as the argument", stderr);
return 1;
}
uncomp_size = atoi(argv[1]);
xz_crc32_init();
/*
* Support up to 64 MiB dictionary. The actually needed memory
* is allocated once the headers have been parsed.
*/
s = xz_dec_init(XZ_DYNALLOC, 1 << 26);
if (s == NULL) {
msg = "Memory allocation failed\n";
goto error;
}
b.in = in;
b.in_pos = 0;
b.in_size = 0;
b.out = out;
b.out_pos = 0;
b.out_size = uncomp_size < BUFSIZ ? uncomp_size : BUFSIZ;
while (true) {
if (b.in_pos == b.in_size) {
b.in_size = fread(in, 1, sizeof(in), stdin);
b.in_pos = 0;
}
ret = xz_dec_run(s, &b);
if (b.out_pos == sizeof(out)) {
if (fwrite(out, 1, b.out_pos, stdout) != b.out_pos) {
msg = "Write error\n";
goto error;
}
uncomp_size -= b.out_pos;
b.out_pos = 0;
b.out_size = uncomp_size < BUFSIZ
? uncomp_size : BUFSIZ;
}
if (ret == XZ_OK)
continue;
#ifdef XZ_DEC_ANY_CHECK
if (ret == XZ_UNSUPPORTED_CHECK) {
fputs(argv[0], stderr);
fputs(": ", stderr);
fputs("Unsupported check; not verifying "
"file integrity\n", stderr);
continue;
}
#endif
if (uncomp_size != b.out_pos) {
msg = "Uncompressed size doesn't match\n";
goto error;
}
if (fwrite(out, 1, b.out_pos, stdout) != b.out_pos
|| fclose(stdout)) {
msg = "Write error\n";
goto error;
}
switch (ret) {
case XZ_STREAM_END:
xz_dec_end(s);
return 0;
case XZ_MEM_ERROR:
msg = "Memory allocation failed\n";
goto error;
case XZ_MEMLIMIT_ERROR:
msg = "Memory usage limit reached\n";
goto error;
case XZ_FORMAT_ERROR:
msg = "Not a .xz file\n";
goto error;
case XZ_OPTIONS_ERROR:
msg = "Unsupported options in the .xz headers\n";
goto error;
case XZ_DATA_ERROR:
case XZ_BUF_ERROR:
msg = "File is corrupt\n";
goto error;
default:
msg = "Bug!\n";
goto error;
}
}
error:
xz_dec_end(s);
fputs(argv[0], stderr);
fputs(": ", stderr);
fputs(msg, stderr);
return 1;
}

@ -10,6 +10,9 @@
#ifndef XZ_CONFIG_H
#define XZ_CONFIG_H
/* Uncomment to enable CRC64 support. */
/* #define XZ_USE_CRC64 */
/* Uncomment as needed to enable BCJ filter decoders. */
/* #define XZ_DEC_X86 */
/* #define XZ_DEC_POWERPC */
@ -18,7 +21,19 @@
/* #define XZ_DEC_ARMTHUMB */
/* #define XZ_DEC_SPARC */
#include <stdbool.h>
/*
* MSVC doesn't support modern C but XZ Embedded is mostly C89
* so these are enough.
*/
#ifdef _MSC_VER
typedef unsigned char bool;
# define true 1
# define false 0
# define inline __inline
#else
# include <stdbool.h>
#endif
#include <stdlib.h>
#include <string.h>

@ -37,6 +37,9 @@ int main(int argc, char **argv)
}
xz_crc32_init();
#ifdef XZ_USE_CRC64
xz_crc64_init();
#endif
/*
* Support up to 64 MiB dictionary. The actually needed memory