Update to Zstandard 1.3.3
Includes patch to conditionalize use of __builtin_clz(ll) on __has_builtin(). The issue is tracked upstream at https://github.com/facebook/zstd/pull/884 . Otherwise, these are vanilla Zstandard 1.3.3 files. Note that the 1.3.4 release should be due out soon. Sponsored by: Dell EMC Isilon
This commit is contained in:
parent
0cb59ea75a
commit
052d3c1290
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=330894
@ -72,9 +72,12 @@ zstdmt:
|
||||
zlibwrapper:
|
||||
$(MAKE) -C $(ZWRAPDIR) test
|
||||
|
||||
.PHONY: check
|
||||
check: shortest
|
||||
|
||||
.PHONY: test shortest
|
||||
test shortest:
|
||||
$(MAKE) -C $(PRGDIR) allVariants
|
||||
$(MAKE) -C $(PRGDIR) allVariants MOREFLAGS="-g -DZSTD_DEBUG=1"
|
||||
$(MAKE) -C $(TESTDIR) $@
|
||||
|
||||
.PHONY: examples
|
||||
@ -127,11 +130,6 @@ uninstall:
|
||||
travis-install:
|
||||
$(MAKE) install PREFIX=~/install_test_dir
|
||||
|
||||
.PHONY: gppbuild
|
||||
gppbuild: clean
|
||||
g++ -v
|
||||
CC=g++ $(MAKE) -C programs all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
|
||||
|
||||
.PHONY: gcc5build
|
||||
gcc5build: clean
|
||||
gcc-5 -v
|
||||
@ -163,7 +161,7 @@ aarch64build: clean
|
||||
CC=aarch64-linux-gnu-gcc CFLAGS="-Werror" $(MAKE) allzstd
|
||||
|
||||
ppcbuild: clean
|
||||
CC=powerpc-linux-gnu-gcc CLAGS="-m32 -Wno-attributes -Werror" $(MAKE) allzstd
|
||||
CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) allzstd
|
||||
|
||||
ppc64build: clean
|
||||
CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allzstd
|
||||
|
@ -1,3 +1,15 @@
|
||||
v1.3.3
|
||||
perf: faster zstd_opt strategy (levels 17-19)
|
||||
fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen
|
||||
cli : fix : content size written in header by default
|
||||
cli : fix : improved LZ4 format support, by @felixhandte
|
||||
cli : new : hidden command `-S`, to benchmark multiple files while generating one result per file
|
||||
api : fix : support large skippable frames, by @terrelln
|
||||
api : fix : streaming interface was adding a useless 3-bytes null block to small frames
|
||||
api : change : when setting `pledgedSrcSize`, use `ZSTD_CONTENTSIZE_UNKNOWN` macro value to mean "unknown"
|
||||
build: fix : compilation under rhel6 and centos6, reported by @pixelb
|
||||
build: added `check` target
|
||||
|
||||
v1.3.2
|
||||
new : long range mode, using --long command, by Stella Lau (@stellamplau)
|
||||
new : ability to generate and decode magicless frames (#591)
|
||||
|
@ -1,14 +1,15 @@
|
||||
__Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm,
|
||||
targeting real-time compression scenarios at zlib-level and better compression ratios.
|
||||
<p align="center"><img src="https://raw.githubusercontent.com/facebook/zstd/readme/doc/images/zstd_logo86.png" alt="Zstandard"></p>
|
||||
|
||||
It is provided as an open-source BSD-licensed **C** library,
|
||||
and a command line utility producing and decoding `.zst` and `.gz` files.
|
||||
For other programming languages,
|
||||
you can consult a list of known ports on [Zstandard homepage](http://www.zstd.net/#other-languages).
|
||||
__Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm,
|
||||
targeting real-time compression scenarios at zlib-level and better compression ratios.
|
||||
It's backed by a very fast entropy stage, provided by [Huff0 and FSE library](https://github.com/Cyan4973/FiniteStateEntropy).
|
||||
|
||||
| dev branch status |
|
||||
|-------------------|
|
||||
| [![Build Status][travisDevBadge]][travisLink] [![Build status][AppveyorDevBadge]][AppveyorLink] [![Build status][CircleDevBadge]][CircleLink]
|
||||
The project is provided as an open-source BSD-licensed **C** library,
|
||||
and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4` files.
|
||||
Should your project require another programming language,
|
||||
a list of known ports and bindings is provided on [Zstandard homepage](http://www.zstd.net/#other-languages).
|
||||
|
||||
Development branch status : [![Build Status][travisDevBadge]][travisLink] [![Build status][AppveyorDevBadge]][AppveyorLink] [![Build status][CircleDevBadge]][CircleLink]
|
||||
|
||||
[travisDevBadge]: https://travis-ci.org/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
|
||||
[travisLink]: https://travis-ci.org/facebook/zstd
|
||||
@ -17,8 +18,9 @@ you can consult a list of known ports on [Zstandard homepage](http://www.zstd.ne
|
||||
[CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite"
|
||||
[CircleLink]: https://circleci.com/gh/facebook/zstd
|
||||
|
||||
### Benchmarks
|
||||
|
||||
As a reference, several fast compression algorithms were tested and compared
|
||||
For reference, several fast compression algorithms were tested and compared
|
||||
on a server running Linux Debian (`Linux version 4.8.0-1-amd64`),
|
||||
with a Core i7-6700K CPU @ 4.0GHz,
|
||||
using [lzbench], an open-source in-memory benchmark by @inikep
|
||||
@ -43,7 +45,9 @@ on the [Silesia compression corpus].
|
||||
[LZ4]: http://www.lz4.org/
|
||||
|
||||
Zstd can also offer stronger compression ratios at the cost of compression speed.
|
||||
Speed vs Compression trade-off is configurable by small increments. Decompression speed is preserved and remains roughly the same at all settings, a property shared by most LZ compression algorithms, such as [zlib] or lzma.
|
||||
Speed vs Compression trade-off is configurable by small increments.
|
||||
Decompression speed is preserved and remains roughly the same at all settings,
|
||||
a property shared by most LZ compression algorithms, such as [zlib] or lzma.
|
||||
|
||||
The following tests were run
|
||||
on a server running Linux Debian (`Linux version 4.8.0-1-amd64`)
|
||||
@ -56,8 +60,8 @@ Compression Speed vs Ratio | Decompression Speed
|
||||
---------------------------|--------------------
|
||||
![Compression Speed vs Ratio](doc/images/Cspeed4.png "Compression Speed vs Ratio") | ![Decompression Speed](doc/images/Dspeed4.png "Decompression Speed")
|
||||
|
||||
Several algorithms can produce higher compression ratios, but at slower speeds, falling outside of the graph.
|
||||
For a larger picture including very slow modes, [click on this link](doc/images/DCspeed5.png) .
|
||||
A few other algorithms can produce higher compression ratios at slower speeds, falling outside of the graph.
|
||||
For a larger picture including slow modes, [click on this link](doc/images/DCspeed5.png).
|
||||
|
||||
|
||||
### The case for Small Data compression
|
||||
@ -84,7 +88,7 @@ Training works if there is some correlation in a family of small data samples. T
|
||||
Hence, deploying one dictionary per type of data will provide the greatest benefits.
|
||||
Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file.
|
||||
|
||||
#### Dictionary compression How To :
|
||||
#### Dictionary compression How To:
|
||||
|
||||
1) Create the dictionary
|
||||
|
||||
@ -99,19 +103,16 @@ Dictionary gains are mostly effective in the first few KB. Then, the compression
|
||||
`zstd -D dictionaryName --decompress FILE.zst`
|
||||
|
||||
|
||||
### Build
|
||||
|
||||
Once you have the repository cloned, there are multiple ways provided to build Zstandard.
|
||||
### Build instructions
|
||||
|
||||
#### Makefile
|
||||
|
||||
If your system is compatible with a standard `make` (or `gmake`) binary generator,
|
||||
you can simply run it at the root directory.
|
||||
It will generate `zstd` within root directory.
|
||||
If your system is compatible with standard `make` (or `gmake`),
|
||||
invoking `make` in root directory will generate `zstd` cli in root directory.
|
||||
|
||||
Other available options include :
|
||||
- `make install` : create and install zstd binary, library and man page
|
||||
- `make test` : create and run `zstd` and test tools on local platform
|
||||
Other available options include:
|
||||
- `make install` : create and install zstd cli, library and man pages
|
||||
- `make check` : create and run `zstd`, tests its behavior on local platform
|
||||
|
||||
#### cmake
|
||||
|
||||
@ -125,9 +126,9 @@ A Meson project is provided within `contrib/meson`.
|
||||
|
||||
#### Visual Studio (Windows)
|
||||
|
||||
Going into `build` directory, you will find additional possibilities :
|
||||
- Projects for Visual Studio 2005, 2008 and 2010
|
||||
+ VS2010 project is compatible with VS2012, VS2013 and VS2015
|
||||
Going into `build` directory, you will find additional possibilities:
|
||||
- Projects for Visual Studio 2005, 2008 and 2010.
|
||||
+ VS2010 project is compatible with VS2012, VS2013 and VS2015.
|
||||
- Automated build scripts for Visual compiler by @KrzysFR , in `build/VS_scripts`,
|
||||
which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution.
|
||||
|
||||
@ -143,11 +144,7 @@ Zstandard is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING).
|
||||
|
||||
### Contributing
|
||||
|
||||
The "dev" branch is the one where all contributions will be merged before reaching "master".
|
||||
If you plan to propose a patch, please commit into the "dev" branch or its own feature branch.
|
||||
The "dev" branch is the one where all contributions are merged before reaching "master".
|
||||
If you plan to propose a patch, please commit into the "dev" branch, or its own feature branch.
|
||||
Direct commit to "master" are not permitted.
|
||||
For more information, please read [CONTRIBUTING](CONTRIBUTING.md).
|
||||
|
||||
### Miscellaneous
|
||||
|
||||
Zstd entropy stage is provided by [Huff0 and FSE, from Finite State Entropy library](https://github.com/Cyan4973/FiniteStateEntropy).
|
||||
|
@ -3,13 +3,11 @@ dependencies:
|
||||
- sudo dpkg --add-architecture i386
|
||||
- sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
|
||||
- sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
|
||||
- sudo apt-get -y install libstdc++-7-dev clang gcc g++ gcc-5 gcc-6 gcc-7 zlib1g-dev liblzma-dev
|
||||
- sudo apt-get -y install linux-libc-dev:i386 libc6-dev-i386
|
||||
|
||||
test:
|
||||
override:
|
||||
- ? |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then cc -v; make all && make clean && make -C lib libzstd-nomt && make clean; fi &&
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then cc -v; CFLAGS="-O0 -Werror" make all && make clean; fi &&
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu90build && make clean; fi
|
||||
:
|
||||
parallel: true
|
||||
@ -20,22 +18,7 @@ test:
|
||||
parallel: true
|
||||
- ? |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make c11build && make clean; fi &&
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make cmakebuild && make clean; fi
|
||||
:
|
||||
parallel: true
|
||||
- ? |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make gppbuild && make clean; fi &&
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc5build && make clean; fi
|
||||
:
|
||||
parallel: true
|
||||
- ? |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make gcc6build && make clean; fi &&
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make clangbuild && make clean; fi
|
||||
:
|
||||
parallel: true
|
||||
- ? |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make m32build && make clean; fi &&
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make armbuild && make clean; fi
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppc64build && make clean; fi
|
||||
:
|
||||
parallel: true
|
||||
- ? |
|
||||
@ -44,8 +27,8 @@ test:
|
||||
:
|
||||
parallel: true
|
||||
- ? |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make ppc64build && make clean; fi &&
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc7build && make clean; fi
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make -j regressiontest && make clean; fi &&
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make armbuild && make clean; fi
|
||||
:
|
||||
parallel: true
|
||||
- ? |
|
||||
@ -54,8 +37,8 @@ test:
|
||||
:
|
||||
parallel: true
|
||||
- ? |
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make -j regressiontest && make clean; fi &&
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then true; fi # Could add another test here
|
||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make cxxtest && make clean; fi &&
|
||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C lib libzstd-nomt && make clean; fi
|
||||
:
|
||||
parallel: true
|
||||
|
||||
|
@ -2,14 +2,39 @@ project('zstd', 'c', license: 'BSD')
|
||||
|
||||
libm = meson.get_compiler('c').find_library('m', required: true)
|
||||
|
||||
lib_dir = join_paths(meson.source_root(), '..', '..', 'lib')
|
||||
lib_dir = join_paths('..', '..', 'lib')
|
||||
common_dir = join_paths(lib_dir, 'common')
|
||||
compress_dir = join_paths(lib_dir, 'compress')
|
||||
decompress_dir = join_paths(lib_dir, 'decompress')
|
||||
dictbuilder_dir = join_paths(lib_dir, 'dictBuilder')
|
||||
deprecated_dir = join_paths(lib_dir, 'deprecated')
|
||||
|
||||
libzstd_srcs = [join_paths(common_dir, 'entropy_common.c'), join_paths(common_dir, 'fse_decompress.c'), join_paths(common_dir, 'threading.c'), join_paths(common_dir, 'pool.c'), join_paths(common_dir, 'zstd_common.c'), join_paths(common_dir, 'error_private.c'), join_paths(common_dir, 'xxhash.c'), join_paths(compress_dir, 'fse_compress.c'), join_paths(compress_dir, 'huf_compress.c'), join_paths(compress_dir, 'zstd_compress.c'), join_paths(compress_dir, 'zstdmt_compress.c'), join_paths(decompress_dir, 'huf_decompress.c'), join_paths(decompress_dir, 'zstd_decompress.c'), join_paths(dictbuilder_dir, 'cover.c'), join_paths(dictbuilder_dir, 'divsufsort.c'), join_paths(dictbuilder_dir, 'zdict.c'), join_paths(deprecated_dir, 'zbuff_common.c'), join_paths(deprecated_dir, 'zbuff_compress.c'), join_paths(deprecated_dir, 'zbuff_decompress.c')]
|
||||
libzstd_srcs = [
|
||||
join_paths(common_dir, 'entropy_common.c'),
|
||||
join_paths(common_dir, 'fse_decompress.c'),
|
||||
join_paths(common_dir, 'threading.c'),
|
||||
join_paths(common_dir, 'pool.c'),
|
||||
join_paths(common_dir, 'zstd_common.c'),
|
||||
join_paths(common_dir, 'error_private.c'),
|
||||
join_paths(common_dir, 'xxhash.c'),
|
||||
join_paths(compress_dir, 'fse_compress.c'),
|
||||
join_paths(compress_dir, 'huf_compress.c'),
|
||||
join_paths(compress_dir, 'zstd_compress.c'),
|
||||
join_paths(compress_dir, 'zstd_fast.c'),
|
||||
join_paths(compress_dir, 'zstd_double_fast.c'),
|
||||
join_paths(compress_dir, 'zstd_lazy.c'),
|
||||
join_paths(compress_dir, 'zstd_opt.c'),
|
||||
join_paths(compress_dir, 'zstd_ldm.c'),
|
||||
join_paths(compress_dir, 'zstdmt_compress.c'),
|
||||
join_paths(decompress_dir, 'huf_decompress.c'),
|
||||
join_paths(decompress_dir, 'zstd_decompress.c'),
|
||||
join_paths(dictbuilder_dir, 'cover.c'),
|
||||
join_paths(dictbuilder_dir, 'divsufsort.c'),
|
||||
join_paths(dictbuilder_dir, 'zdict.c'),
|
||||
join_paths(deprecated_dir, 'zbuff_common.c'),
|
||||
join_paths(deprecated_dir, 'zbuff_compress.c'),
|
||||
join_paths(deprecated_dir, 'zbuff_decompress.c')
|
||||
]
|
||||
|
||||
libzstd_includes = [include_directories(common_dir, dictbuilder_dir, compress_dir, lib_dir)]
|
||||
|
||||
@ -19,7 +44,15 @@ if get_option('legacy_support')
|
||||
|
||||
legacy_dir = join_paths(lib_dir, 'legacy')
|
||||
libzstd_includes += [include_directories(legacy_dir)]
|
||||
libzstd_srcs += [join_paths(legacy_dir, 'zstd_v01.c'), join_paths(legacy_dir, 'zstd_v02.c'), join_paths(legacy_dir, 'zstd_v03.c'), join_paths(legacy_dir, 'zstd_v04.c'), join_paths(legacy_dir, 'zstd_v05.c'), join_paths(legacy_dir, 'zstd_v06.c'), join_paths(legacy_dir, 'zstd_v07.c')]
|
||||
libzstd_srcs += [
|
||||
join_paths(legacy_dir, 'zstd_v01.c'),
|
||||
join_paths(legacy_dir, 'zstd_v02.c'),
|
||||
join_paths(legacy_dir, 'zstd_v03.c'),
|
||||
join_paths(legacy_dir, 'zstd_v04.c'),
|
||||
join_paths(legacy_dir, 'zstd_v05.c'),
|
||||
join_paths(legacy_dir, 'zstd_v06.c'),
|
||||
join_paths(legacy_dir, 'zstd_v07.c')
|
||||
]
|
||||
else
|
||||
libzstd_cflags = []
|
||||
endif
|
||||
@ -39,16 +72,20 @@ libzstd = library('zstd',
|
||||
dependencies: libzstd_deps,
|
||||
install: true)
|
||||
|
||||
programs_dir = join_paths(meson.source_root(), '..', '..', 'programs')
|
||||
programs_dir = join_paths('..', '..', 'programs')
|
||||
|
||||
zstd = executable('zstd',
|
||||
join_paths(programs_dir, 'bench.c'), join_paths(programs_dir, 'datagen.c'), join_paths(programs_dir, 'dibio.c'), join_paths(programs_dir, 'fileio.c'), join_paths(programs_dir, 'zstdcli.c'),
|
||||
join_paths(programs_dir, 'bench.c'),
|
||||
join_paths(programs_dir, 'datagen.c'),
|
||||
join_paths(programs_dir, 'dibio.c'),
|
||||
join_paths(programs_dir, 'fileio.c'),
|
||||
join_paths(programs_dir, 'zstdcli.c'),
|
||||
include_directories: libzstd_includes,
|
||||
c_args: ['-DZSTD_NODICT', '-DZSTD_NOBENCH'],
|
||||
link_with: libzstd,
|
||||
install: true)
|
||||
|
||||
tests_dir = join_paths(meson.source_root(), '..', '..', 'tests')
|
||||
tests_dir = join_paths('..', '..', 'tests')
|
||||
datagen_c = join_paths(programs_dir, 'datagen.c')
|
||||
test_includes = libzstd_includes + [include_directories(programs_dir)]
|
||||
|
||||
|
BIN
sys/contrib/zstd/doc/images/zstd_logo86.png
Normal file
BIN
sys/contrib/zstd/doc/images/zstd_logo86.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 5.8 KiB |
@ -91,10 +91,10 @@ Overview
|
||||
|
||||
Frames
|
||||
------
|
||||
Zstandard compressed data is made of up one or more __frames__.
|
||||
Zstandard compressed data is made of one or more __frames__.
|
||||
Each frame is independent and can be decompressed indepedently of other frames.
|
||||
The decompressed content of multiple concatenated frames is the concatenation of
|
||||
each frames decompressed content.
|
||||
each frame decompressed content.
|
||||
|
||||
There are two frame formats defined by Zstandard:
|
||||
Zstandard frames and Skippable frames.
|
||||
@ -182,7 +182,7 @@ data must be regenerated within a single continuous memory segment.
|
||||
In this case, `Window_Descriptor` byte is skipped,
|
||||
but `Frame_Content_Size` is necessarily present.
|
||||
As a consequence, the decoder must allocate a memory segment
|
||||
of size equal or bigger than `Frame_Content_Size`.
|
||||
of size equal or larger than `Frame_Content_Size`.
|
||||
|
||||
In order to preserve the decoder from unreasonable memory requirements,
|
||||
a decoder is allowed to reject a compressed frame
|
||||
|
@ -1,10 +1,10 @@
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
|
||||
<title>zstd 1.3.2 Manual</title>
|
||||
<title>zstd 1.3.3 Manual</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>zstd 1.3.2 Manual</h1>
|
||||
<h1>zstd 1.3.3 Manual</h1>
|
||||
<hr>
|
||||
<a name="Contents"></a><h2>Contents</h2>
|
||||
<ol>
|
||||
@ -19,16 +19,17 @@
|
||||
<li><a href="#Chapter9">Streaming decompression - HowTo</a></li>
|
||||
<li><a href="#Chapter10">START OF ADVANCED AND EXPERIMENTAL FUNCTIONS</a></li>
|
||||
<li><a href="#Chapter11">Advanced types</a></li>
|
||||
<li><a href="#Chapter12">Frame size functions</a></li>
|
||||
<li><a href="#Chapter13">Context memory usage</a></li>
|
||||
<li><a href="#Chapter14">Advanced compression functions</a></li>
|
||||
<li><a href="#Chapter15">Advanced decompression functions</a></li>
|
||||
<li><a href="#Chapter16">Advanced streaming functions</a></li>
|
||||
<li><a href="#Chapter17">Buffer-less and synchronous inner streaming functions</a></li>
|
||||
<li><a href="#Chapter18">Buffer-less streaming compression (synchronous mode)</a></li>
|
||||
<li><a href="#Chapter19">Buffer-less streaming decompression (synchronous mode)</a></li>
|
||||
<li><a href="#Chapter20">New advanced API (experimental)</a></li>
|
||||
<li><a href="#Chapter21">Block level API</a></li>
|
||||
<li><a href="#Chapter12">Custom memory allocation functions</a></li>
|
||||
<li><a href="#Chapter13">Frame size functions</a></li>
|
||||
<li><a href="#Chapter14">Context memory usage</a></li>
|
||||
<li><a href="#Chapter15">Advanced compression functions</a></li>
|
||||
<li><a href="#Chapter16">Advanced decompression functions</a></li>
|
||||
<li><a href="#Chapter17">Advanced streaming functions</a></li>
|
||||
<li><a href="#Chapter18">Buffer-less and synchronous inner streaming functions</a></li>
|
||||
<li><a href="#Chapter19">Buffer-less streaming compression (synchronous mode)</a></li>
|
||||
<li><a href="#Chapter20">Buffer-less streaming decompression (synchronous mode)</a></li>
|
||||
<li><a href="#Chapter21">New advanced API (experimental)</a></li>
|
||||
<li><a href="#Chapter22">Block level API</a></li>
|
||||
</ol>
|
||||
<hr>
|
||||
<a name="Chapter1"></a><h2>Introduction</h2><pre>
|
||||
@ -111,7 +112,7 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
|
||||
@return : content size to be decompressed, as a 64-bits value _if known and not empty_, 0 otherwise.
|
||||
</p></pre><BR>
|
||||
|
||||
<h3>Helper functions</h3><pre></pre><b><pre>#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < 128 KB) ? ((128 KB - (srcSize)) >> 11) </b>/* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */<b>
|
||||
<h3>Helper functions</h3><pre></pre><b><pre>#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) </b>/* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */<b>
|
||||
size_t ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case scenario */<b>
|
||||
unsigned ZSTD_isError(size_t code); </b>/*!< tells if a `size_t` function result is an error code */<b>
|
||||
const char* ZSTD_getErrorName(size_t code); </b>/*!< provides readable string from an error code */<b>
|
||||
@ -346,17 +347,15 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
||||
ZSTD_frameParameters fParams;
|
||||
} ZSTD_parameters;
|
||||
</b></pre><BR>
|
||||
<h3>Custom memory allocation functions</h3><pre></pre><b><pre>typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
|
||||
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
|
||||
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
|
||||
</b>/* use this constant to defer to stdlib's functions */<b>
|
||||
static const ZSTD_customMem ZSTD_defaultCMem = { NULL, NULL, NULL };
|
||||
</pre></b><BR>
|
||||
<a name="Chapter12"></a><h2>Frame size functions</h2><pre></pre>
|
||||
<a name="Chapter12"></a><h2>Custom memory allocation functions</h2><pre></pre>
|
||||
|
||||
<pre><b>typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
|
||||
</b></pre><BR>
|
||||
<a name="Chapter13"></a><h2>Frame size functions</h2><pre></pre>
|
||||
|
||||
<pre><b>size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
|
||||
</b><p> `src` should point to the start of a ZSTD encoded frame or skippable frame
|
||||
`srcSize` must be at least as large as the frame
|
||||
`srcSize` must be >= first frame size
|
||||
@return : the compressed size of the first frame starting at `src`,
|
||||
suitable to pass to `ZSTD_decompress` or similar,
|
||||
or an error code if input is invalid
|
||||
@ -391,7 +390,7 @@ static const ZSTD_customMem ZSTD_defaultCMem = { NULL, NULL, NULL };
|
||||
@return : size of the Frame Header
|
||||
</p></pre><BR>
|
||||
|
||||
<a name="Chapter13"></a><h2>Context memory usage</h2><pre></pre>
|
||||
<a name="Chapter14"></a><h2>Context memory usage</h2><pre></pre>
|
||||
|
||||
<pre><b>size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
|
||||
size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
|
||||
@ -450,7 +449,7 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMet
|
||||
|
||||
</p></pre><BR>
|
||||
|
||||
<a name="Chapter14"></a><h2>Advanced compression functions</h2><pre></pre>
|
||||
<a name="Chapter15"></a><h2>Advanced compression functions</h2><pre></pre>
|
||||
|
||||
<pre><b>ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
|
||||
</b><p> Create a ZSTD compression context using external alloc and free functions
|
||||
@ -462,7 +461,8 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMet
|
||||
It must outlive context usage.
|
||||
workspaceSize: Use ZSTD_estimateCCtxSize() or ZSTD_estimateCStreamSize()
|
||||
to determine how large workspace must be to support scenario.
|
||||
@return : pointer to ZSTD_CCtx*, or NULL if error (size too small)
|
||||
@return : pointer to ZSTD_CCtx* (same address as workspace, but different type),
|
||||
or NULL if error (typically size too small)
|
||||
Note : zstd will never resize nor malloc() when using a static cctx.
|
||||
If it needs more memory than available, it will simply error out.
|
||||
Note 2 : there is no corresponding "free" function.
|
||||
@ -505,7 +505,8 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMet
|
||||
to determine how large workspace must be.
|
||||
cParams : use ZSTD_getCParams() to transform a compression level
|
||||
into its relevants cParams.
|
||||
@return : pointer to ZSTD_CDict*, or NULL if error (size too small)
|
||||
@return : pointer to ZSTD_CDict* (same address as workspace, but different type),
|
||||
or NULL if error (typically, size too small).
|
||||
Note : there is no corresponding "free" function.
|
||||
Since workspace was allocated externally, it must be freed externally.
|
||||
|
||||
@ -518,7 +519,7 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMet
|
||||
|
||||
<pre><b>ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
|
||||
</b><p> same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
|
||||
All fields of `ZSTD_frameParameters` are set to default (0)
|
||||
All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
|
||||
@ -545,7 +546,7 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMet
|
||||
</b><p> Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters
|
||||
</p></pre><BR>
|
||||
|
||||
<a name="Chapter15"></a><h2>Advanced decompression functions</h2><pre></pre>
|
||||
<a name="Chapter16"></a><h2>Advanced decompression functions</h2><pre></pre>
|
||||
|
||||
<pre><b>unsigned ZSTD_isFrame(const void* buffer, size_t size);
|
||||
</b><p> Tells if the content of `buffer` starts with a valid Frame Identifier.
|
||||
@ -564,7 +565,8 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMet
|
||||
It must outlive context usage.
|
||||
workspaceSize: Use ZSTD_estimateDCtxSize() or ZSTD_estimateDStreamSize()
|
||||
to determine how large workspace must be to support scenario.
|
||||
@return : pointer to ZSTD_DCtx*, or NULL if error (size too small)
|
||||
@return : pointer to ZSTD_DCtx* (same address as workspace, but different type),
|
||||
or NULL if error (typically size too small)
|
||||
Note : zstd will never resize nor malloc() when using a static dctx.
|
||||
If it needs more memory than available, it will simply error out.
|
||||
Note 2 : static dctx is incompatible with legacy support
|
||||
@ -627,24 +629,26 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMet
|
||||
When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code.
|
||||
</p></pre><BR>
|
||||
|
||||
<a name="Chapter16"></a><h2>Advanced streaming functions</h2><pre></pre>
|
||||
<a name="Chapter17"></a><h2>Advanced streaming functions</h2><pre></pre>
|
||||
|
||||
<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
|
||||
ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); </b>/**< same as ZSTD_initStaticCCtx() */<b>
|
||||
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */<b>
|
||||
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */<b>
|
||||
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/<b>
|
||||
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */<b>
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */<b>
|
||||
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); </b>/**< note : cdict will just be referenced, and must outlive compression session */<b>
|
||||
size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); </b>/**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */<b>
|
||||
size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); </b>/**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters. pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. */<b>
|
||||
</pre></b><BR>
|
||||
<pre><b>size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
|
||||
</b><p> start a new compression job, using same parameters from previous job.
|
||||
This is typically useful to skip dictionary loading stage, since it will re-use it in-place..
|
||||
Note that zcs must be init at least once before using ZSTD_resetCStream().
|
||||
pledgedSrcSize==0 means "srcSize unknown".
|
||||
If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
|
||||
If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
|
||||
@return : 0, or an error code (which can be tested using ZSTD_isError())
|
||||
For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
|
||||
but it may change to mean "empty" in some future version, so prefer using macro ZSTD_CONTENTSIZE_UNKNOWN.
|
||||
@return : 0, or an error code (which can be tested using ZSTD_isError())
|
||||
</p></pre><BR>
|
||||
|
||||
<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
|
||||
@ -655,14 +659,14 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
|
||||
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); </b>/**< note : ddict is referenced, it must outlive decompression session */<b>
|
||||
size_t ZSTD_resetDStream(ZSTD_DStream* zds); </b>/**< re-use decompression parameters from previous init; saves dictionary loading */<b>
|
||||
</pre></b><BR>
|
||||
<a name="Chapter17"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
|
||||
<a name="Chapter18"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
|
||||
This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
|
||||
But it's also a complex one, with several restrictions, documented below.
|
||||
Prefer normal streaming API for an easier experience.
|
||||
|
||||
<BR></pre>
|
||||
|
||||
<a name="Chapter18"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
|
||||
<a name="Chapter19"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
|
||||
A ZSTD_CCtx object is required to track streaming operations.
|
||||
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
|
||||
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
|
||||
@ -693,12 +697,12 @@ size_t ZSTD_resetDStream(ZSTD_DStream* zds); </b>/**< re-use decompression para
|
||||
|
||||
<h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
|
||||
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
||||
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */<b>
|
||||
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
|
||||
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); </b>/**< note: fails if cdict==NULL */<b>
|
||||
size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); </b>/* compression parameters are already set within cdict. pledgedSrcSize=0 means null-size */<b>
|
||||
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */<b>
|
||||
size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); </b>/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */<b>
|
||||
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
|
||||
</pre></b><BR>
|
||||
<a name="Chapter19"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
|
||||
<a name="Chapter20"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
|
||||
A ZSTD_DCtx object is required to track streaming operations.
|
||||
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
|
||||
A ZSTD_DCtx object can be re-used multiple times.
|
||||
@ -784,7 +788,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
|
||||
</pre></b><BR>
|
||||
<pre><b>typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
|
||||
</b></pre><BR>
|
||||
<a name="Chapter20"></a><h2>New advanced API (experimental)</h2><pre></pre>
|
||||
<a name="Chapter21"></a><h2>New advanced API (experimental)</h2><pre></pre>
|
||||
|
||||
<pre><b>typedef enum {
|
||||
</b>/* Question : should we have a format ZSTD_f_auto ?<b>
|
||||
@ -848,18 +852,19 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
|
||||
* Special: value 0 means "do not change strategy". */
|
||||
|
||||
</b>/* frame parameters */<b>
|
||||
ZSTD_p_contentSizeFlag=200, </b>/* Content size is written into frame header _whenever known_ (default:1)<b>
|
||||
* note that content size must be known at the beginning,
|
||||
* it is sent using ZSTD_CCtx_setPledgedSrcSize() */
|
||||
ZSTD_p_contentSizeFlag=200, </b>/* Content size will be written into frame header _whenever known_ (default:1)<b>
|
||||
* Content size must be known at the beginning of compression,
|
||||
* it is provided using ZSTD_CCtx_setPledgedSrcSize() */
|
||||
ZSTD_p_checksumFlag, </b>/* A 32-bits checksum of content is written at end of frame (default:0) */<b>
|
||||
ZSTD_p_dictIDFlag, </b>/* When applicable, dictID of dictionary is provided in frame header (default:1) */<b>
|
||||
ZSTD_p_dictIDFlag, </b>/* When applicable, dictionary's ID is written into frame header (default:1) */<b>
|
||||
|
||||
</b>/* multi-threading parameters */<b>
|
||||
ZSTD_p_nbThreads=400, </b>/* Select how many threads a compression job can spawn (default:1)<b>
|
||||
* More threads improve speed, but also increase memory usage.
|
||||
* Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
|
||||
* Special: value 0 means "do not change nbThreads" */
|
||||
ZSTD_p_jobSize, </b>/* Size of a compression job. Each compression job is completed in parallel.<b>
|
||||
ZSTD_p_jobSize, </b>/* Size of a compression job. This value is only enforced in streaming (non-blocking) mode.<b>
|
||||
* Each compression job is completed in parallel, so indirectly controls the nb of active threads.
|
||||
* 0 means default, which is dynamically determined based on compression parameters.
|
||||
* Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
|
||||
* The minimum size is automatically and transparently enforced */
|
||||
@ -904,7 +909,8 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
|
||||
<pre><b>size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
|
||||
</b><p> Set one compression parameter, selected by enum ZSTD_cParameter.
|
||||
Note : when `value` is an enum, cast it to unsigned for proper type checking.
|
||||
@result : 0, or an error code (which can be tested with ZSTD_isError()).
|
||||
@result : informational value (typically, the one being set, possibly corrected),
|
||||
or an error code (which can be tested with ZSTD_isError()).
|
||||
</p></pre><BR>
|
||||
|
||||
<pre><b>size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
|
||||
@ -913,7 +919,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
|
||||
@result : 0, or an error code (which can be tested with ZSTD_isError()).
|
||||
Note 1 : 0 means zero, empty.
|
||||
In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
|
||||
Note that ZSTD_CONTENTSIZE_UNKNOWN is default value for new compression jobs.
|
||||
ZSTD_CONTENTSIZE_UNKNOWN is default value for any new compression job.
|
||||
Note 2 : If all data is provided and consumed in a single round,
|
||||
this value is overriden by srcSize instead.
|
||||
</p></pre><BR>
|
||||
@ -985,13 +991,19 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t
|
||||
- Compression parameters cannot be changed once compression is started.
|
||||
- outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
|
||||
- outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
|
||||
- @return provides the minimum amount of data still to flush from internal buffers
|
||||
- In single-thread mode (default), function is blocking : it completed its job before returning to caller.
|
||||
- In multi-thread mode, function is non-blocking : it just acquires a copy of input, and distribute job to internal worker threads,
|
||||
and then immediately returns, just indicating that there is some data remaining to be flushed.
|
||||
The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
|
||||
- Exception : in multi-threading mode, if the first call requests a ZSTD_e_end directive, it is blocking : it will complete compression before giving back control to caller.
|
||||
- @return provides the minimum amount of data remaining to be flushed from internal buffers
|
||||
or an error code, which can be tested using ZSTD_isError().
|
||||
if @return != 0, flush is not fully completed, there is some data left within internal buffers.
|
||||
- after a ZSTD_e_end directive, if internal buffer is not fully flushed,
|
||||
if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
|
||||
This is useful to determine if a ZSTD_e_flush or ZSTD_e_end directive is completed.
|
||||
- after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
|
||||
only ZSTD_e_end or ZSTD_e_flush operations are allowed.
|
||||
It is necessary to fully flush internal buffers
|
||||
before starting a new compression job, or changing compression parameters.
|
||||
Before starting a new compression job, or changing compression parameters,
|
||||
it is required to fully flush internal buffers.
|
||||
|
||||
</p></pre><BR>
|
||||
|
||||
@ -1166,7 +1178,7 @@ size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t
|
||||
|
||||
</p></pre><BR>
|
||||
|
||||
<a name="Chapter21"></a><h2>Block level API</h2><pre></pre>
|
||||
<a name="Chapter22"></a><h2>Block level API</h2><pre></pre>
|
||||
|
||||
<pre><b></b><p> Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
|
||||
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
|
||||
|
@ -15,15 +15,9 @@ cxx_library(
|
||||
header_namespace='',
|
||||
visibility=['PUBLIC'],
|
||||
exported_headers=subdir_glob([
|
||||
('compress', 'zstdmt_compress.h'),
|
||||
('compress', 'zstd*.h'),
|
||||
]),
|
||||
headers=subdir_glob([
|
||||
('compress', 'zstd_opt.h'),
|
||||
]),
|
||||
srcs=[
|
||||
'compress/zstd_compress.c',
|
||||
'compress/zstdmt_compress.c',
|
||||
],
|
||||
srcs=glob(['compress/zstd*.c']),
|
||||
deps=[':common'],
|
||||
)
|
||||
|
||||
@ -31,7 +25,7 @@ cxx_library(
|
||||
name='decompress',
|
||||
header_namespace='',
|
||||
visibility=['PUBLIC'],
|
||||
srcs=['decompress/zstd_decompress.c'],
|
||||
srcs=glob(['decompress/zstd*.c']),
|
||||
deps=[
|
||||
':common',
|
||||
':legacy',
|
||||
@ -58,6 +52,9 @@ cxx_library(
|
||||
]),
|
||||
srcs=glob(['legacy/*.c']),
|
||||
deps=[':common'],
|
||||
exported_preprocessor_flags=[
|
||||
'-DZSTD_LEGACY_SUPPORT=4',
|
||||
],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
@ -74,6 +71,15 @@ cxx_library(
|
||||
deps=[':common'],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='compiler',
|
||||
header_namespace='',
|
||||
visibility=['PUBLIC'],
|
||||
exported_headers=subdir_glob([
|
||||
('common', 'compiler.h'),
|
||||
]),
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='bitstream',
|
||||
header_namespace='',
|
||||
@ -100,6 +106,7 @@ cxx_library(
|
||||
],
|
||||
deps=[
|
||||
':bitstream',
|
||||
':compiler',
|
||||
':errors',
|
||||
':mem',
|
||||
],
|
||||
@ -133,7 +140,10 @@ cxx_library(
|
||||
('common', 'pool.h'),
|
||||
]),
|
||||
srcs=['common/pool.c'],
|
||||
deps=[':threading'],
|
||||
deps=[
|
||||
':threading',
|
||||
':zstd_common',
|
||||
],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
@ -144,6 +154,12 @@ cxx_library(
|
||||
('common', 'threading.h'),
|
||||
]),
|
||||
srcs=['common/threading.c'],
|
||||
exported_preprocessor_flags=[
|
||||
'-DZSTD_MULTITHREAD',
|
||||
],
|
||||
exported_linker_flags=[
|
||||
'-pthread',
|
||||
],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
@ -154,6 +170,9 @@ cxx_library(
|
||||
('common', 'xxhash.h'),
|
||||
]),
|
||||
srcs=['common/xxhash.c'],
|
||||
exported_preprocessor_flags=[
|
||||
'-DXXH_NAMESPACE=ZSTD_',
|
||||
],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
@ -166,6 +185,7 @@ cxx_library(
|
||||
]),
|
||||
srcs=['common/zstd_common.c'],
|
||||
deps=[
|
||||
':compiler',
|
||||
':errors',
|
||||
':mem',
|
||||
],
|
||||
@ -175,6 +195,7 @@ cxx_library(
|
||||
name='common',
|
||||
deps=[
|
||||
':bitstream',
|
||||
':compiler',
|
||||
':entropy',
|
||||
':errors',
|
||||
':mem',
|
||||
|
@ -167,7 +167,7 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
|
||||
/*-**************************************************************
|
||||
* Internal functions
|
||||
****************************************************************/
|
||||
MEM_STATIC unsigned BIT_highbit32 (register U32 val)
|
||||
MEM_STATIC unsigned BIT_highbit32 (U32 val)
|
||||
{
|
||||
assert(val != 0);
|
||||
{
|
||||
|
@ -56,8 +56,6 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
typedef int64_t S64;
|
||||
typedef intptr_t iPtrDiff;
|
||||
typedef uintptr_t uPtrDiff;
|
||||
#else
|
||||
typedef unsigned char BYTE;
|
||||
typedef unsigned short U16;
|
||||
@ -66,8 +64,6 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size
|
||||
typedef signed int S32;
|
||||
typedef unsigned long long U64;
|
||||
typedef signed long long S64;
|
||||
typedef ptrdiff_t iPtrDiff;
|
||||
typedef size_t uPtrDiff;
|
||||
#endif
|
||||
|
||||
|
||||
@ -123,20 +119,26 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
|
||||
/* currently only defined for gcc and icc */
|
||||
#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32))
|
||||
__pragma( pack(push, 1) )
|
||||
typedef union { U16 u16; U32 u32; U64 u64; size_t st; } unalign;
|
||||
typedef struct { U16 v; } unalign16;
|
||||
typedef struct { U32 v; } unalign32;
|
||||
typedef struct { U64 v; } unalign64;
|
||||
typedef struct { size_t v; } unalignArch;
|
||||
__pragma( pack(pop) )
|
||||
#else
|
||||
typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign;
|
||||
typedef struct { U16 v; } __attribute__((packed)) unalign16;
|
||||
typedef struct { U32 v; } __attribute__((packed)) unalign32;
|
||||
typedef struct { U64 v; } __attribute__((packed)) unalign64;
|
||||
typedef struct { size_t v; } __attribute__((packed)) unalignArch;
|
||||
#endif
|
||||
|
||||
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
|
||||
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
|
||||
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
|
||||
MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; }
|
||||
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; }
|
||||
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; }
|
||||
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; }
|
||||
MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; }
|
||||
|
||||
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
|
||||
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
|
||||
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
|
||||
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; }
|
||||
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; }
|
||||
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; }
|
||||
|
||||
#else
|
||||
|
||||
|
@ -11,7 +11,6 @@
|
||||
|
||||
/* ====== Dependencies ======= */
|
||||
#include <stddef.h> /* size_t */
|
||||
#include <stdlib.h> /* malloc, calloc, free */
|
||||
#include "pool.h"
|
||||
|
||||
/* ====== Compiler specifics ====== */
|
||||
@ -115,7 +114,7 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customM
|
||||
* and full queues.
|
||||
*/
|
||||
ctx->queueSize = queueSize + 1;
|
||||
ctx->queue = (POOL_job*) malloc(ctx->queueSize * sizeof(POOL_job));
|
||||
ctx->queue = (POOL_job*)ZSTD_malloc(ctx->queueSize * sizeof(POOL_job), customMem);
|
||||
ctx->queueHead = 0;
|
||||
ctx->queueTail = 0;
|
||||
ctx->numThreadsBusy = 0;
|
||||
|
@ -212,7 +212,7 @@ static U64 XXH_read64(const void* memPtr)
|
||||
#if defined(_MSC_VER) /* Visual Studio */
|
||||
# define XXH_swap32 _byteswap_ulong
|
||||
# define XXH_swap64 _byteswap_uint64
|
||||
#elif (GCC_VERSION >= 403 && !defined(__riscv))
|
||||
#elif GCC_VERSION >= 403
|
||||
# define XXH_swap32 __builtin_bswap32
|
||||
# define XXH_swap64 __builtin_bswap64
|
||||
#else
|
||||
|
@ -31,21 +31,27 @@ const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; }
|
||||
* ZSTD Error Management
|
||||
******************************************/
|
||||
/*! ZSTD_isError() :
|
||||
* tells if a return value is an error code */
|
||||
* tells if a return value is an error code */
|
||||
unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
|
||||
|
||||
/*! ZSTD_getErrorName() :
|
||||
* provides error code string from function result (useful for debugging) */
|
||||
* provides error code string from function result (useful for debugging) */
|
||||
const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
|
||||
/*! ZSTD_getError() :
|
||||
* convert a `size_t` function result into a proper ZSTD_errorCode enum */
|
||||
* convert a `size_t` function result into a proper ZSTD_errorCode enum */
|
||||
ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
|
||||
|
||||
/*! ZSTD_getErrorString() :
|
||||
* provides error code string from enum */
|
||||
* provides error code string from enum */
|
||||
const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); }
|
||||
|
||||
/*! g_debuglog_enable :
|
||||
* turn on/off debug traces (global switch) */
|
||||
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 2)
|
||||
int g_debuglog_enable = 1;
|
||||
#endif
|
||||
|
||||
|
||||
/*=**************************************************************
|
||||
* Custom allocator
|
||||
|
@ -11,6 +11,10 @@
|
||||
#ifndef ZSTD_CCOMMON_H_MODULE
|
||||
#define ZSTD_CCOMMON_H_MODULE
|
||||
|
||||
/* this module contains definitions which must be identical
|
||||
* across compression, decompression and dictBuilder.
|
||||
* It also contains a few functions useful to at least 2 of them
|
||||
* and which benefit from being inlined */
|
||||
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
@ -50,21 +54,26 @@ extern "C" {
|
||||
|
||||
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
|
||||
# include <stdio.h>
|
||||
extern int g_debuglog_enable;
|
||||
/* recommended values for ZSTD_DEBUG display levels :
|
||||
* 1 : no display, enables assert() only
|
||||
* 2 : reserved for currently active debugging path
|
||||
* 3 : events once per object lifetime (CCtx, CDict)
|
||||
* 2 : reserved for currently active debug path
|
||||
* 3 : events once per object lifetime (CCtx, CDict, etc.)
|
||||
* 4 : events once per frame
|
||||
* 5 : events once per block
|
||||
* 6 : events once per sequence (*very* verbose) */
|
||||
# define DEBUGLOG(l, ...) { \
|
||||
if (l<=ZSTD_DEBUG) { \
|
||||
fprintf(stderr, __FILE__ ": "); \
|
||||
fprintf(stderr, __VA_ARGS__); \
|
||||
fprintf(stderr, " \n"); \
|
||||
# define RAWLOG(l, ...) { \
|
||||
if ((g_debuglog_enable) & (l<=ZSTD_DEBUG)) { \
|
||||
fprintf(stderr, __VA_ARGS__); \
|
||||
} }
|
||||
# define DEBUGLOG(l, ...) { \
|
||||
if ((g_debuglog_enable) & (l<=ZSTD_DEBUG)) { \
|
||||
fprintf(stderr, __FILE__ ": " __VA_ARGS__); \
|
||||
fprintf(stderr, " \n"); \
|
||||
} }
|
||||
#else
|
||||
# define DEBUGLOG(l, ...) {} /* disabled */
|
||||
# define RAWLOG(l, ...) {} /* disabled */
|
||||
# define DEBUGLOG(l, ...) {} /* disabled */
|
||||
#endif
|
||||
|
||||
|
||||
@ -85,9 +94,7 @@ extern "C" {
|
||||
#define ZSTD_OPT_NUM (1<<12)
|
||||
|
||||
#define ZSTD_REP_NUM 3 /* number of repcodes */
|
||||
#define ZSTD_REP_CHECK (ZSTD_REP_NUM) /* number of repcodes to check by the optimal parser */
|
||||
#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1)
|
||||
#define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM)
|
||||
static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
|
||||
|
||||
#define KB *(1 <<10)
|
||||
@ -134,28 +141,40 @@ typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingTy
|
||||
#define LLFSELog 9
|
||||
#define OffFSELog 8
|
||||
|
||||
static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
|
||||
static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
1, 1, 1, 1, 2, 2, 3, 3,
|
||||
4, 6, 7, 8, 9,10,11,12,
|
||||
13,14,15,16 };
|
||||
static const S16 LL_defaultNorm[MaxLL+1] = { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
|
||||
static const S16 LL_defaultNorm[MaxLL+1] = { 4, 3, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 1, 1, 1,
|
||||
2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 3, 2, 1, 1, 1, 1, 1,
|
||||
-1,-1,-1,-1 };
|
||||
#define LL_DEFAULTNORMLOG 6 /* for static allocation */
|
||||
static const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG;
|
||||
|
||||
static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9,10,11,
|
||||
static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
1, 1, 1, 1, 2, 2, 3, 3,
|
||||
4, 4, 5, 7, 8, 9,10,11,
|
||||
12,13,14,15,16 };
|
||||
static const S16 ML_defaultNorm[MaxML+1] = { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,
|
||||
static const S16 ML_defaultNorm[MaxML+1] = { 1, 4, 3, 2, 2, 2, 2, 2,
|
||||
2, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1,-1,-1,
|
||||
-1,-1,-1,-1,-1 };
|
||||
#define ML_DEFAULTNORMLOG 6 /* for static allocation */
|
||||
static const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG;
|
||||
|
||||
static const S16 OF_defaultNorm[DefaultMaxOff+1] = { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 };
|
||||
static const S16 OF_defaultNorm[DefaultMaxOff+1] = { 1, 1, 1, 1, 1, 1, 2, 2,
|
||||
2, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
-1,-1,-1,-1,-1 };
|
||||
#define OF_DEFAULTNORMLOG 5 /* for static allocation */
|
||||
static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
|
||||
|
||||
@ -167,7 +186,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
||||
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
|
||||
|
||||
/*! ZSTD_wildcopy() :
|
||||
* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
|
||||
* custom version of memcpy(), can overwrite up to WILDCOPY_OVERLENGTH bytes (if length==0) */
|
||||
#define WILDCOPY_OVERLENGTH 8
|
||||
MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
|
||||
{
|
||||
@ -191,17 +210,14 @@ MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd) /* s
|
||||
|
||||
|
||||
/*-*******************************************
|
||||
* Private interfaces
|
||||
* Private declarations
|
||||
*********************************************/
|
||||
typedef struct ZSTD_stats_s ZSTD_stats_t;
|
||||
|
||||
typedef struct seqDef_s {
|
||||
U32 offset;
|
||||
U16 litLength;
|
||||
U16 matchLength;
|
||||
} seqDef;
|
||||
|
||||
|
||||
typedef struct {
|
||||
seqDef* sequencesStart;
|
||||
seqDef* sequences;
|
||||
@ -216,100 +232,8 @@ typedef struct {
|
||||
U32 repToConfirm[ZSTD_REP_NUM];
|
||||
} seqStore_t;
|
||||
|
||||
typedef struct {
|
||||
U32 off;
|
||||
U32 len;
|
||||
} ZSTD_match_t;
|
||||
|
||||
typedef struct {
|
||||
U32 price;
|
||||
U32 off;
|
||||
U32 mlen;
|
||||
U32 litlen;
|
||||
U32 rep[ZSTD_REP_NUM];
|
||||
} ZSTD_optimal_t;
|
||||
|
||||
typedef struct {
|
||||
U32* litFreq;
|
||||
U32* litLengthFreq;
|
||||
U32* matchLengthFreq;
|
||||
U32* offCodeFreq;
|
||||
ZSTD_match_t* matchTable;
|
||||
ZSTD_optimal_t* priceTable;
|
||||
|
||||
U32 matchLengthSum;
|
||||
U32 matchSum;
|
||||
U32 litLengthSum;
|
||||
U32 litSum;
|
||||
U32 offCodeSum;
|
||||
U32 log2matchLengthSum;
|
||||
U32 log2matchSum;
|
||||
U32 log2litLengthSum;
|
||||
U32 log2litSum;
|
||||
U32 log2offCodeSum;
|
||||
U32 factor;
|
||||
U32 staticPrices;
|
||||
U32 cachedPrice;
|
||||
U32 cachedLitLength;
|
||||
const BYTE* cachedLiterals;
|
||||
} optState_t;
|
||||
|
||||
typedef struct {
|
||||
U32 offset;
|
||||
U32 checksum;
|
||||
} ldmEntry_t;
|
||||
|
||||
typedef struct {
|
||||
ldmEntry_t* hashTable;
|
||||
BYTE* bucketOffsets; /* Next position in bucket to insert entry */
|
||||
U64 hashPower; /* Used to compute the rolling hash.
|
||||
* Depends on ldmParams.minMatchLength */
|
||||
} ldmState_t;
|
||||
|
||||
typedef struct {
|
||||
U32 enableLdm; /* 1 if enable long distance matching */
|
||||
U32 hashLog; /* Log size of hashTable */
|
||||
U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */
|
||||
U32 minMatchLength; /* Minimum match length */
|
||||
U32 hashEveryLog; /* Log number of entries to skip */
|
||||
} ldmParams_t;
|
||||
|
||||
typedef struct {
|
||||
U32 hufCTable[HUF_CTABLE_SIZE_U32(255)];
|
||||
FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
|
||||
FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
|
||||
FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
|
||||
U32 workspace[HUF_WORKSPACE_SIZE_U32];
|
||||
HUF_repeat hufCTable_repeatMode;
|
||||
FSE_repeat offcode_repeatMode;
|
||||
FSE_repeat matchlength_repeatMode;
|
||||
FSE_repeat litlength_repeatMode;
|
||||
} ZSTD_entropyCTables_t;
|
||||
|
||||
struct ZSTD_CCtx_params_s {
|
||||
ZSTD_format_e format;
|
||||
ZSTD_compressionParameters cParams;
|
||||
ZSTD_frameParameters fParams;
|
||||
|
||||
int compressionLevel;
|
||||
U32 forceWindow; /* force back-references to respect limit of
|
||||
* 1<<wLog, even for dictionary */
|
||||
|
||||
/* Multithreading: used to pass parameters to mtctx */
|
||||
U32 nbThreads;
|
||||
unsigned jobSize;
|
||||
unsigned overlapSizeLog;
|
||||
|
||||
/* Long distance matching parameters */
|
||||
ldmParams_t ldmParams;
|
||||
|
||||
/* For use with createCCtxParams() and freeCCtxParams() only */
|
||||
ZSTD_customMem customMem;
|
||||
|
||||
}; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
|
||||
|
||||
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
|
||||
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr);
|
||||
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */
|
||||
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
|
||||
|
||||
/* custom memory allocation functions */
|
||||
void* ZSTD_malloc(size_t size, ZSTD_customMem customMem);
|
||||
@ -317,9 +241,7 @@ void* ZSTD_calloc(size_t size, ZSTD_customMem customMem);
|
||||
void ZSTD_free(void* ptr, ZSTD_customMem customMem);
|
||||
|
||||
|
||||
/*====== common function ======*/
|
||||
|
||||
MEM_STATIC U32 ZSTD_highbit32(U32 val)
|
||||
MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */
|
||||
{
|
||||
assert(val != 0);
|
||||
{
|
||||
@ -330,67 +252,26 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val)
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3) && __has_builtin(__builtin_clz) /* GCC Intrinsic */
|
||||
return 31 - __builtin_clz(val);
|
||||
# else /* Software version */
|
||||
static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
||||
static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
||||
U32 v = val;
|
||||
int r;
|
||||
v |= v >> 1;
|
||||
v |= v >> 2;
|
||||
v |= v >> 4;
|
||||
v |= v >> 8;
|
||||
v |= v >> 16;
|
||||
r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
|
||||
return r;
|
||||
return DeBruijnClz[(v * 0x07C4ACDDU) >> 27];
|
||||
# endif
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* hidden functions */
|
||||
|
||||
/* ZSTD_invalidateRepCodes() :
|
||||
* ensures next compression will not use repcodes from previous block.
|
||||
* Note : only works with regular variant;
|
||||
* do not use with extDict variant ! */
|
||||
void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx);
|
||||
void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */
|
||||
|
||||
|
||||
/*! ZSTD_initCStream_internal() :
|
||||
* Private use only. Init streaming operation.
|
||||
* expects params to be valid.
|
||||
* must receive dict, or cdict, or none, but not both.
|
||||
* @return : 0, or an error code */
|
||||
size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
|
||||
const void* dict, size_t dictSize,
|
||||
const ZSTD_CDict* cdict,
|
||||
ZSTD_CCtx_params params, unsigned long long pledgedSrcSize);
|
||||
|
||||
/*! ZSTD_compressStream_generic() :
|
||||
* Private use only. To be called from zstdmt_compress.c in single-thread mode. */
|
||||
size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
|
||||
ZSTD_outBuffer* output,
|
||||
ZSTD_inBuffer* input,
|
||||
ZSTD_EndDirective const flushMode);
|
||||
|
||||
/*! ZSTD_getCParamsFromCDict() :
|
||||
* as the name implies */
|
||||
ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict);
|
||||
|
||||
/* ZSTD_compressBegin_advanced_internal() :
|
||||
* Private use only. To be called from zstdmt_compress.c. */
|
||||
size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_dictMode_e dictMode,
|
||||
ZSTD_CCtx_params params,
|
||||
unsigned long long pledgedSrcSize);
|
||||
|
||||
/* ZSTD_compress_advanced_internal() :
|
||||
* Private use only. To be called from zstdmt_compress.c. */
|
||||
size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict,size_t dictSize,
|
||||
ZSTD_CCtx_params params);
|
||||
|
||||
typedef struct {
|
||||
blockType_e blockType;
|
||||
U32 lastBlock;
|
||||
@ -398,7 +279,8 @@ typedef struct {
|
||||
} blockProperties_t;
|
||||
|
||||
/*! ZSTD_getcBlockSize() :
|
||||
* Provides the size of compressed block from block header `src` */
|
||||
* Provides the size of compressed block from block header `src` */
|
||||
/* Used by: decompress, fullbench (does not get its definition from here) */
|
||||
size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
|
||||
blockProperties_t* bpPtr);
|
||||
|
||||
|
File diff suppressed because it is too large
Load Diff
@ -8,6 +8,9 @@
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
/* This header contains definitions
|
||||
* that shall **only** be used by modules within lib/compress.
|
||||
*/
|
||||
|
||||
#ifndef ZSTD_COMPRESS_H
|
||||
#define ZSTD_COMPRESS_H
|
||||
@ -43,6 +46,95 @@ typedef struct ZSTD_prefixDict_s {
|
||||
ZSTD_dictMode_e dictMode;
|
||||
} ZSTD_prefixDict;
|
||||
|
||||
typedef struct {
|
||||
U32 hufCTable[HUF_CTABLE_SIZE_U32(255)];
|
||||
FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
|
||||
FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
|
||||
FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
|
||||
U32 workspace[HUF_WORKSPACE_SIZE_U32];
|
||||
HUF_repeat hufCTable_repeatMode;
|
||||
FSE_repeat offcode_repeatMode;
|
||||
FSE_repeat matchlength_repeatMode;
|
||||
FSE_repeat litlength_repeatMode;
|
||||
} ZSTD_entropyCTables_t;
|
||||
|
||||
typedef struct {
|
||||
U32 off;
|
||||
U32 len;
|
||||
} ZSTD_match_t;
|
||||
|
||||
typedef struct {
|
||||
int price;
|
||||
U32 off;
|
||||
U32 mlen;
|
||||
U32 litlen;
|
||||
U32 rep[ZSTD_REP_NUM];
|
||||
} ZSTD_optimal_t;
|
||||
|
||||
typedef struct {
|
||||
/* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */
|
||||
U32* litFreq; /* table of literals statistics, of size 256 */
|
||||
U32* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */
|
||||
U32* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */
|
||||
U32* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */
|
||||
ZSTD_match_t* matchTable; /* list of found matches, of size ZSTD_OPT_NUM+1 */
|
||||
ZSTD_optimal_t* priceTable; /* All positions tracked by optimal parser, of size ZSTD_OPT_NUM+1 */
|
||||
|
||||
U32 litSum; /* nb of literals */
|
||||
U32 litLengthSum; /* nb of litLength codes */
|
||||
U32 matchLengthSum; /* nb of matchLength codes */
|
||||
U32 offCodeSum; /* nb of offset codes */
|
||||
/* begin updated by ZSTD_setLog2Prices */
|
||||
U32 log2litSum; /* pow2 to compare log2(litfreq) to */
|
||||
U32 log2litLengthSum; /* pow2 to compare log2(llfreq) to */
|
||||
U32 log2matchLengthSum; /* pow2 to compare log2(mlfreq) to */
|
||||
U32 log2offCodeSum; /* pow2 to compare log2(offreq) to */
|
||||
/* end : updated by ZSTD_setLog2Prices */
|
||||
U32 staticPrices; /* prices follow a pre-defined cost structure, statistics are irrelevant */
|
||||
} optState_t;
|
||||
|
||||
typedef struct {
|
||||
U32 offset;
|
||||
U32 checksum;
|
||||
} ldmEntry_t;
|
||||
|
||||
typedef struct {
|
||||
ldmEntry_t* hashTable;
|
||||
BYTE* bucketOffsets; /* Next position in bucket to insert entry */
|
||||
U64 hashPower; /* Used to compute the rolling hash.
|
||||
* Depends on ldmParams.minMatchLength */
|
||||
} ldmState_t;
|
||||
|
||||
typedef struct {
|
||||
U32 enableLdm; /* 1 if enable long distance matching */
|
||||
U32 hashLog; /* Log size of hashTable */
|
||||
U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */
|
||||
U32 minMatchLength; /* Minimum match length */
|
||||
U32 hashEveryLog; /* Log number of entries to skip */
|
||||
} ldmParams_t;
|
||||
|
||||
struct ZSTD_CCtx_params_s {
|
||||
ZSTD_format_e format;
|
||||
ZSTD_compressionParameters cParams;
|
||||
ZSTD_frameParameters fParams;
|
||||
|
||||
int compressionLevel;
|
||||
U32 forceWindow; /* force back-references to respect limit of
|
||||
* 1<<wLog, even for dictionary */
|
||||
|
||||
/* Multithreading: used to pass parameters to mtctx */
|
||||
U32 nbThreads;
|
||||
unsigned jobSize;
|
||||
unsigned overlapSizeLog;
|
||||
|
||||
/* Long distance matching parameters */
|
||||
ldmParams_t ldmParams;
|
||||
|
||||
/* For use with createCCtxParams() and freeCCtxParams() only */
|
||||
ZSTD_customMem customMem;
|
||||
|
||||
}; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
|
||||
|
||||
struct ZSTD_CCtx_s {
|
||||
const BYTE* nextSrc; /* next block here to continue on current prefix */
|
||||
const BYTE* base; /* All regular indexes relative to this position */
|
||||
@ -99,38 +191,51 @@ struct ZSTD_CCtx_s {
|
||||
};
|
||||
|
||||
|
||||
static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
|
||||
8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 16, 17, 17, 18, 18, 19, 19,
|
||||
20, 20, 20, 20, 21, 21, 21, 21,
|
||||
22, 22, 22, 22, 22, 22, 22, 22,
|
||||
23, 23, 23, 23, 23, 23, 23, 23,
|
||||
24, 24, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24 };
|
||||
MEM_STATIC U32 ZSTD_LLcode(U32 litLength)
|
||||
{
|
||||
static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
|
||||
8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 16, 17, 17, 18, 18, 19, 19,
|
||||
20, 20, 20, 20, 21, 21, 21, 21,
|
||||
22, 22, 22, 22, 22, 22, 22, 22,
|
||||
23, 23, 23, 23, 23, 23, 23, 23,
|
||||
24, 24, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24 };
|
||||
static const U32 LL_deltaCode = 19;
|
||||
return (litLength > 63) ? ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
|
||||
}
|
||||
|
||||
static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
|
||||
32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
|
||||
38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
|
||||
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
|
||||
41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
|
||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
|
||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
|
||||
/* ZSTD_MLcode() :
|
||||
* note : mlBase = matchLength - MINMATCH;
|
||||
* because it's the format it's stored in seqStore->sequences */
|
||||
MEM_STATIC U32 ZSTD_MLcode(U32 mlBase)
|
||||
{
|
||||
static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
|
||||
32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
|
||||
38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
|
||||
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
|
||||
41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
|
||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
|
||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
|
||||
static const U32 ML_deltaCode = 36;
|
||||
return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase];
|
||||
}
|
||||
|
||||
/*! ZSTD_storeSeq() :
|
||||
Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
|
||||
`offsetCode` : distance to match, or 0 == repCode.
|
||||
`matchCode` : matchLength - MINMATCH
|
||||
* Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
|
||||
* `offsetCode` : distance to match + 3 (values 1-3 are repCodes).
|
||||
* `mlBase` : matchLength - MINMATCH
|
||||
*/
|
||||
MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode)
|
||||
MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t mlBase)
|
||||
{
|
||||
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 6)
|
||||
static const BYTE* g_start = NULL;
|
||||
U32 const pos = (U32)((const BYTE*)literals - g_start);
|
||||
if (g_start==NULL) g_start = (const BYTE*)literals;
|
||||
if ((pos > 0) && (pos < 1000000000))
|
||||
DEBUGLOG(6, "Cpos %6u :%5u literals & match %3u bytes at distance %6u",
|
||||
pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
|
||||
if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */
|
||||
{ U32 const pos = (U32)((const BYTE*)literals - g_start);
|
||||
DEBUGLOG(6, "Cpos%7u :%3u literals, match%3u bytes at dist.code%7u",
|
||||
pos, (U32)litLength, (U32)mlBase+MINMATCH, (U32)offsetCode);
|
||||
}
|
||||
#endif
|
||||
/* copy Literals */
|
||||
assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + 128 KB);
|
||||
@ -139,6 +244,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v
|
||||
|
||||
/* literal Length */
|
||||
if (litLength>0xFFFF) {
|
||||
assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */
|
||||
seqStorePtr->longLengthID = 1;
|
||||
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
|
||||
}
|
||||
@ -148,11 +254,12 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v
|
||||
seqStorePtr->sequences[0].offset = offsetCode + 1;
|
||||
|
||||
/* match Length */
|
||||
if (matchCode>0xFFFF) {
|
||||
if (mlBase>0xFFFF) {
|
||||
assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */
|
||||
seqStorePtr->longLengthID = 2;
|
||||
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
|
||||
}
|
||||
seqStorePtr->sequences[0].matchLength = (U16)matchCode;
|
||||
seqStorePtr->sequences[0].matchLength = (U16)mlBase;
|
||||
|
||||
seqStorePtr->sequences++;
|
||||
}
|
||||
@ -161,7 +268,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v
|
||||
/*-*************************************
|
||||
* Match length counter
|
||||
***************************************/
|
||||
static unsigned ZSTD_NbCommonBytes (register size_t val)
|
||||
static unsigned ZSTD_NbCommonBytes (size_t val)
|
||||
{
|
||||
if (MEM_isLittleEndian()) {
|
||||
if (MEM_64bits()) {
|
||||
@ -235,13 +342,17 @@ MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* co
|
||||
const BYTE* const pStart = pIn;
|
||||
const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1);
|
||||
|
||||
while (pIn < pInLoopLimit) {
|
||||
size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
|
||||
if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
|
||||
pIn += ZSTD_NbCommonBytes(diff);
|
||||
return (size_t)(pIn - pStart);
|
||||
}
|
||||
if (MEM_64bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
|
||||
if (pIn < pInLoopLimit) {
|
||||
{ size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
|
||||
if (diff) return ZSTD_NbCommonBytes(diff); }
|
||||
pIn+=sizeof(size_t); pMatch+=sizeof(size_t);
|
||||
while (pIn < pInLoopLimit) {
|
||||
size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
|
||||
if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
|
||||
pIn += ZSTD_NbCommonBytes(diff);
|
||||
return (size_t)(pIn - pStart);
|
||||
} }
|
||||
if (MEM_64bits() && (pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
|
||||
if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
|
||||
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
|
||||
return (size_t)(pIn - pStart);
|
||||
@ -304,4 +415,48 @@ MEM_STATIC size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
/* ==============================================================
|
||||
* Private declarations
|
||||
* These prototypes shall only be called from within lib/compress
|
||||
* ============================================================== */
|
||||
|
||||
/*! ZSTD_initCStream_internal() :
|
||||
* Private use only. Init streaming operation.
|
||||
* expects params to be valid.
|
||||
* must receive dict, or cdict, or none, but not both.
|
||||
* @return : 0, or an error code */
|
||||
size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
|
||||
const void* dict, size_t dictSize,
|
||||
const ZSTD_CDict* cdict,
|
||||
ZSTD_CCtx_params params, unsigned long long pledgedSrcSize);
|
||||
|
||||
/*! ZSTD_compressStream_generic() :
|
||||
* Private use only. To be called from zstdmt_compress.c in single-thread mode. */
|
||||
size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
|
||||
ZSTD_outBuffer* output,
|
||||
ZSTD_inBuffer* input,
|
||||
ZSTD_EndDirective const flushMode);
|
||||
|
||||
/*! ZSTD_getCParamsFromCDict() :
|
||||
* as the name implies */
|
||||
ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict);
|
||||
|
||||
/* ZSTD_compressBegin_advanced_internal() :
|
||||
* Private use only. To be called from zstdmt_compress.c. */
|
||||
size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_dictMode_e dictMode,
|
||||
const ZSTD_CDict* cdict,
|
||||
ZSTD_CCtx_params params,
|
||||
unsigned long long pledgedSrcSize);
|
||||
|
||||
/* ZSTD_compress_advanced_internal() :
|
||||
* Private use only. To be called from zstdmt_compress.c. */
|
||||
size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict,size_t dictSize,
|
||||
ZSTD_CCtx_params params);
|
||||
|
||||
#endif /* ZSTD_COMPRESS_H */
|
@ -8,6 +8,7 @@
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#include "zstd_compress_internal.h"
|
||||
#include "zstd_double_fast.h"
|
||||
|
||||
|
||||
|
@ -11,12 +11,13 @@
|
||||
#ifndef ZSTD_DOUBLE_FAST_H
|
||||
#define ZSTD_DOUBLE_FAST_H
|
||||
|
||||
#include "zstd_compress.h"
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "mem.h" /* U32 */
|
||||
#include "zstd.h" /* ZSTD_CCtx, size_t */
|
||||
|
||||
void ZSTD_fillDoubleHashTable(ZSTD_CCtx* cctx, const void* end, const U32 mls);
|
||||
size_t ZSTD_compressBlock_doubleFast(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
|
||||
|
@ -8,6 +8,7 @@
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#include "zstd_compress_internal.h"
|
||||
#include "zstd_fast.h"
|
||||
|
||||
|
||||
|
@ -11,12 +11,13 @@
|
||||
#ifndef ZSTD_FAST_H
|
||||
#define ZSTD_FAST_H
|
||||
|
||||
#include "zstd_compress.h"
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "mem.h" /* U32 */
|
||||
#include "zstd.h" /* ZSTD_CCtx, size_t */
|
||||
|
||||
void ZSTD_fillHashTable(ZSTD_CCtx* zc, const void* end, const U32 mls);
|
||||
size_t ZSTD_compressBlock_fast(ZSTD_CCtx* ctx,
|
||||
const void* src, size_t srcSize);
|
||||
|
@ -8,6 +8,7 @@
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#include "zstd_compress_internal.h"
|
||||
#include "zstd_lazy.h"
|
||||
|
||||
|
||||
@ -15,10 +16,11 @@
|
||||
* Binary Tree search
|
||||
***************************************/
|
||||
/** ZSTD_insertBt1() : add one or multiple positions to tree.
|
||||
* ip : assumed <= iend-8 .
|
||||
* @return : nb of positions added */
|
||||
static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares,
|
||||
U32 extDict)
|
||||
* ip : assumed <= iend-8 .
|
||||
* @return : nb of positions added */
|
||||
static U32 ZSTD_insertBt1(ZSTD_CCtx* zc,
|
||||
const BYTE* const ip, const BYTE* const iend,
|
||||
U32 nbCompares, U32 const mls, U32 const extDict)
|
||||
{
|
||||
U32* const hashTable = zc->hashTable;
|
||||
U32 const hashLog = zc->appliedParams.cParams.hashLog;
|
||||
@ -40,7 +42,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
|
||||
U32* largerPtr = smallerPtr + 1;
|
||||
U32 dummy32; /* to be nullified at the end */
|
||||
U32 const windowLow = zc->lowLimit;
|
||||
U32 matchEndIdx = current+8;
|
||||
U32 matchEndIdx = current+8+1;
|
||||
size_t bestLength = 8;
|
||||
#ifdef ZSTD_C_PREDICT
|
||||
U32 predictedSmall = *(bt + 2*((current-1)&btMask) + 0);
|
||||
@ -49,12 +51,15 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
|
||||
predictedLarge += (predictedLarge>0);
|
||||
#endif /* ZSTD_C_PREDICT */
|
||||
|
||||
DEBUGLOG(8, "ZSTD_insertBt1 (%u)", current);
|
||||
|
||||
assert(ip <= iend-8); /* required for h calculation */
|
||||
hashTable[h] = current; /* Update Hash Table */
|
||||
|
||||
while (nbCompares-- && (matchIndex > windowLow)) {
|
||||
U32* const nextPtr = bt + 2*(matchIndex & btMask);
|
||||
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
|
||||
assert(matchIndex < current);
|
||||
|
||||
#ifdef ZSTD_C_PREDICT /* note : can create issues when hlog small <= 11 */
|
||||
const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */
|
||||
@ -76,10 +81,11 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
|
||||
continue;
|
||||
}
|
||||
#endif
|
||||
|
||||
if ((!extDict) || (matchIndex+matchLength >= dictLimit)) {
|
||||
assert(matchIndex+matchLength >= dictLimit); /* might be wrong if extDict is incorrectly set to 0 */
|
||||
match = base + matchIndex;
|
||||
if (match[matchLength] == ip[matchLength])
|
||||
matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1;
|
||||
matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend);
|
||||
} else {
|
||||
match = dictBase + matchIndex;
|
||||
matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart);
|
||||
@ -93,16 +99,17 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
|
||||
matchEndIdx = matchIndex + (U32)matchLength;
|
||||
}
|
||||
|
||||
if (ip+matchLength == iend) /* equal : no way to know if inf or sup */
|
||||
if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */
|
||||
break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */
|
||||
}
|
||||
|
||||
if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */
|
||||
/* match+1 is smaller than current */
|
||||
/* match is smaller than current */
|
||||
*smallerPtr = matchIndex; /* update smaller idx */
|
||||
commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
|
||||
if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */
|
||||
smallerPtr = nextPtr+1; /* new "smaller" => larger of match */
|
||||
matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */
|
||||
smallerPtr = nextPtr+1; /* new "candidate" => larger than match, which was smaller than target */
|
||||
matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */
|
||||
} else {
|
||||
/* match is larger than current */
|
||||
*largerPtr = matchIndex;
|
||||
@ -114,8 +121,38 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
|
||||
|
||||
*smallerPtr = *largerPtr = 0;
|
||||
if (bestLength > 384) return MIN(192, (U32)(bestLength - 384)); /* speed optimization */
|
||||
if (matchEndIdx > current + 8) return matchEndIdx - (current + 8);
|
||||
return 1;
|
||||
assert(matchEndIdx > current + 8);
|
||||
return matchEndIdx - (current + 8);
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE
|
||||
void ZSTD_updateTree_internal(ZSTD_CCtx* zc,
|
||||
const BYTE* const ip, const BYTE* const iend,
|
||||
const U32 nbCompares, const U32 mls, const U32 extDict)
|
||||
{
|
||||
const BYTE* const base = zc->base;
|
||||
U32 const target = (U32)(ip - base);
|
||||
U32 idx = zc->nextToUpdate;
|
||||
DEBUGLOG(7, "ZSTD_updateTree_internal, from %u to %u (extDict:%u)",
|
||||
idx, target, extDict);
|
||||
|
||||
while(idx < target)
|
||||
idx += ZSTD_insertBt1(zc, base+idx, iend, nbCompares, mls, extDict);
|
||||
zc->nextToUpdate = target;
|
||||
}
|
||||
|
||||
void ZSTD_updateTree(ZSTD_CCtx* zc,
|
||||
const BYTE* const ip, const BYTE* const iend,
|
||||
const U32 nbCompares, const U32 mls)
|
||||
{
|
||||
ZSTD_updateTree_internal(zc, ip, iend, nbCompares, mls, 0 /*extDict*/);
|
||||
}
|
||||
|
||||
void ZSTD_updateTree_extDict(ZSTD_CCtx* zc,
|
||||
const BYTE* const ip, const BYTE* const iend,
|
||||
const U32 nbCompares, const U32 mls)
|
||||
{
|
||||
ZSTD_updateTree_internal(zc, ip, iend, nbCompares, mls, 1 /*extDict*/);
|
||||
}
|
||||
|
||||
|
||||
@ -144,7 +181,7 @@ static size_t ZSTD_insertBtAndFindBestMatch (
|
||||
const U32 windowLow = zc->lowLimit;
|
||||
U32* smallerPtr = bt + 2*(current&btMask);
|
||||
U32* largerPtr = bt + 2*(current&btMask) + 1;
|
||||
U32 matchEndIdx = current+8;
|
||||
U32 matchEndIdx = current+8+1;
|
||||
U32 dummy32; /* to be nullified at the end */
|
||||
size_t bestLength = 0;
|
||||
|
||||
@ -158,8 +195,7 @@ static size_t ZSTD_insertBtAndFindBestMatch (
|
||||
|
||||
if ((!extDict) || (matchIndex+matchLength >= dictLimit)) {
|
||||
match = base + matchIndex;
|
||||
if (match[matchLength] == ip[matchLength])
|
||||
matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1;
|
||||
matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend);
|
||||
} else {
|
||||
match = dictBase + matchIndex;
|
||||
matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart);
|
||||
@ -172,8 +208,9 @@ static size_t ZSTD_insertBtAndFindBestMatch (
|
||||
matchEndIdx = matchIndex + (U32)matchLength;
|
||||
if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(current-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) )
|
||||
bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex;
|
||||
if (ip+matchLength == iend) /* equal : no way to know if inf or sup */
|
||||
if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */
|
||||
break; /* drop, to guarantee consistency (miss a little bit of compression) */
|
||||
}
|
||||
}
|
||||
|
||||
if (match[matchLength] < ip[matchLength]) {
|
||||
@ -194,21 +231,12 @@ static size_t ZSTD_insertBtAndFindBestMatch (
|
||||
|
||||
*smallerPtr = *largerPtr = 0;
|
||||
|
||||
zc->nextToUpdate = (matchEndIdx > current + 8) ? matchEndIdx - 8 : current+1;
|
||||
assert(matchEndIdx > current+8);
|
||||
zc->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */
|
||||
return bestLength;
|
||||
}
|
||||
|
||||
|
||||
void ZSTD_updateTree(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls)
|
||||
{
|
||||
const BYTE* const base = zc->base;
|
||||
const U32 target = (U32)(ip - base);
|
||||
U32 idx = zc->nextToUpdate;
|
||||
|
||||
while(idx < target)
|
||||
idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 0);
|
||||
}
|
||||
|
||||
/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */
|
||||
static size_t ZSTD_BtFindBestMatch (
|
||||
ZSTD_CCtx* zc,
|
||||
@ -239,16 +267,6 @@ static size_t ZSTD_BtFindBestMatch_selectMLS (
|
||||
}
|
||||
|
||||
|
||||
void ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls)
|
||||
{
|
||||
const BYTE* const base = zc->base;
|
||||
const U32 target = (U32)(ip - base);
|
||||
U32 idx = zc->nextToUpdate;
|
||||
|
||||
while (idx < target) idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 1);
|
||||
}
|
||||
|
||||
|
||||
/** Tree updater, providing best match */
|
||||
static size_t ZSTD_BtFindBestMatch_extDict (
|
||||
ZSTD_CCtx* zc,
|
||||
@ -335,14 +353,14 @@ size_t ZSTD_HcFindBestMatch_generic (
|
||||
U32 matchIndex = ZSTD_insertAndFindFirstIndex (zc, ip, mls);
|
||||
|
||||
for ( ; (matchIndex>lowLimit) & (nbAttempts>0) ; nbAttempts--) {
|
||||
const BYTE* match;
|
||||
size_t currentMl=0;
|
||||
if ((!extDict) || matchIndex >= dictLimit) {
|
||||
match = base + matchIndex;
|
||||
const BYTE* const match = base + matchIndex;
|
||||
if (match[ml] == ip[ml]) /* potentially better */
|
||||
currentMl = ZSTD_count(ip, match, iLimit);
|
||||
} else {
|
||||
match = dictBase + matchIndex;
|
||||
const BYTE* const match = dictBase + matchIndex;
|
||||
assert(match+4 <= dictEnd);
|
||||
if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */
|
||||
currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4;
|
||||
}
|
||||
@ -380,10 +398,10 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS (
|
||||
|
||||
|
||||
FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS (
|
||||
ZSTD_CCtx* zc,
|
||||
ZSTD_CCtx* const zc,
|
||||
const BYTE* ip, const BYTE* const iLimit,
|
||||
size_t* offsetPtr,
|
||||
const U32 maxNbAttempts, const U32 matchLengthSearch)
|
||||
size_t* const offsetPtr,
|
||||
U32 const maxNbAttempts, U32 const matchLengthSearch)
|
||||
{
|
||||
switch(matchLengthSearch)
|
||||
{
|
||||
@ -502,9 +520,8 @@ size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
|
||||
*/
|
||||
/* catch up */
|
||||
if (offset) {
|
||||
while ( (start > anchor)
|
||||
&& (start > base+offset-ZSTD_REP_MOVE)
|
||||
&& (start[-1] == (start-offset+ZSTD_REP_MOVE)[-1]) ) /* only search for offset within prefix */
|
||||
while ( ((start > anchor) & (start - (offset-ZSTD_REP_MOVE) > base))
|
||||
&& (start[-1] == (start-(offset-ZSTD_REP_MOVE))[-1]) ) /* only search for offset within prefix */
|
||||
{ start--; matchLength++; }
|
||||
offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE);
|
||||
}
|
||||
@ -516,9 +533,8 @@ size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
|
||||
}
|
||||
|
||||
/* check immediate repcode */
|
||||
while ( (ip <= ilimit)
|
||||
&& ((offset_2>0)
|
||||
& (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) {
|
||||
while ( ((ip <= ilimit) & (offset_2>0))
|
||||
&& (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) {
|
||||
/* store sequence */
|
||||
matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
|
||||
offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */
|
||||
|
@ -11,12 +11,13 @@
|
||||
#ifndef ZSTD_LAZY_H
|
||||
#define ZSTD_LAZY_H
|
||||
|
||||
#include "zstd_compress.h"
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "mem.h" /* U32 */
|
||||
#include "zstd.h" /* ZSTD_CCtx, size_t */
|
||||
|
||||
U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls);
|
||||
void ZSTD_updateTree(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls);
|
||||
void ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls);
|
||||
|
@ -10,12 +10,13 @@
|
||||
#ifndef ZSTD_LDM_H
|
||||
#define ZSTD_LDM_H
|
||||
|
||||
#include "zstd_compress.h"
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "zstd_compress_internal.h" /* ldmParams_t, U32 */
|
||||
#include "zstd.h" /* ZSTD_CCtx, size_t */
|
||||
|
||||
/*-*************************************
|
||||
* Long distance matching
|
||||
***************************************/
|
||||
|
File diff suppressed because it is too large
Load Diff
@ -11,12 +11,12 @@
|
||||
#ifndef ZSTD_OPT_H
|
||||
#define ZSTD_OPT_H
|
||||
|
||||
#include "zstd_compress.h"
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "zstd.h" /* ZSTD_CCtx, size_t */
|
||||
|
||||
size_t ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_btultra(ZSTD_CCtx* ctx, const void* src, size_t srcSize);
|
||||
|
||||
|
@ -24,7 +24,7 @@
|
||||
#include <string.h> /* memcpy, memset */
|
||||
#include "pool.h" /* threadpool */
|
||||
#include "threading.h" /* mutex */
|
||||
#include "zstd_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
|
||||
#include "zstd_compress_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
|
||||
#include "zstdmt_compress.h"
|
||||
|
||||
|
||||
@ -140,9 +140,12 @@ static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool)
|
||||
return poolSize + totalBufferSize;
|
||||
}
|
||||
|
||||
static void ZSTDMT_setBufferSize(ZSTDMT_bufferPool* bufPool, size_t bSize)
|
||||
static void ZSTDMT_setBufferSize(ZSTDMT_bufferPool* const bufPool, size_t const bSize)
|
||||
{
|
||||
ZSTD_pthread_mutex_lock(&bufPool->poolMutex);
|
||||
DEBUGLOG(4, "ZSTDMT_setBufferSize: bSize = %u", (U32)bSize);
|
||||
bufPool->bufferSize = bSize;
|
||||
ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
|
||||
}
|
||||
|
||||
/** ZSTDMT_getBuffer() :
|
||||
@ -150,28 +153,31 @@ static void ZSTDMT_setBufferSize(ZSTDMT_bufferPool* bufPool, size_t bSize)
|
||||
static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool)
|
||||
{
|
||||
size_t const bSize = bufPool->bufferSize;
|
||||
DEBUGLOG(5, "ZSTDMT_getBuffer");
|
||||
DEBUGLOG(5, "ZSTDMT_getBuffer: bSize = %u", (U32)bufPool->bufferSize);
|
||||
ZSTD_pthread_mutex_lock(&bufPool->poolMutex);
|
||||
if (bufPool->nbBuffers) { /* try to use an existing buffer */
|
||||
buffer_t const buf = bufPool->bTable[--(bufPool->nbBuffers)];
|
||||
size_t const availBufferSize = buf.size;
|
||||
bufPool->bTable[bufPool->nbBuffers] = g_nullBuffer;
|
||||
if ((availBufferSize >= bSize) & (availBufferSize <= 10*bSize)) {
|
||||
if ((availBufferSize >= bSize) & ((availBufferSize>>3) <= bSize)) {
|
||||
/* large enough, but not too much */
|
||||
DEBUGLOG(5, "ZSTDMT_getBuffer: provide buffer %u of size %u",
|
||||
bufPool->nbBuffers, (U32)buf.size);
|
||||
ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
|
||||
return buf;
|
||||
}
|
||||
/* size conditions not respected : scratch this buffer, create new one */
|
||||
DEBUGLOG(5, "existing buffer does not meet size conditions => freeing");
|
||||
DEBUGLOG(5, "ZSTDMT_getBuffer: existing buffer does not meet size conditions => freeing");
|
||||
ZSTD_free(buf.start, bufPool->cMem);
|
||||
}
|
||||
ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
|
||||
/* create new buffer */
|
||||
DEBUGLOG(5, "create a new buffer");
|
||||
DEBUGLOG(5, "ZSTDMT_getBuffer: create a new buffer");
|
||||
{ buffer_t buffer;
|
||||
void* const start = ZSTD_malloc(bSize, bufPool->cMem);
|
||||
buffer.start = start; /* note : start can be NULL if malloc fails ! */
|
||||
buffer.size = (start==NULL) ? 0 : bSize;
|
||||
DEBUGLOG(5, "ZSTDMT_getBuffer: created buffer of size %u", (U32)bSize);
|
||||
return buffer;
|
||||
}
|
||||
}
|
||||
@ -184,12 +190,14 @@ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf)
|
||||
ZSTD_pthread_mutex_lock(&bufPool->poolMutex);
|
||||
if (bufPool->nbBuffers < bufPool->totalBuffers) {
|
||||
bufPool->bTable[bufPool->nbBuffers++] = buf; /* stored for later use */
|
||||
DEBUGLOG(5, "ZSTDMT_releaseBuffer: stored buffer of size %u in slot %u",
|
||||
(U32)buf.size, (U32)(bufPool->nbBuffers-1));
|
||||
ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
|
||||
return;
|
||||
}
|
||||
ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
|
||||
/* Reached bufferPool capacity (should not happen) */
|
||||
DEBUGLOG(5, "buffer pool capacity reached => freeing ");
|
||||
DEBUGLOG(5, "ZSTDMT_releaseBuffer: pool capacity reached => freeing ");
|
||||
ZSTD_free(buf.start, bufPool->cMem);
|
||||
}
|
||||
|
||||
@ -302,7 +310,7 @@ static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx)
|
||||
typedef struct {
|
||||
buffer_t src;
|
||||
const void* srcStart;
|
||||
size_t dictSize;
|
||||
size_t prefixSize;
|
||||
size_t srcSize;
|
||||
buffer_t dstBuff;
|
||||
size_t cSize;
|
||||
@ -324,11 +332,11 @@ typedef struct {
|
||||
void ZSTDMT_compressChunk(void* jobDescription)
|
||||
{
|
||||
ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription;
|
||||
ZSTD_CCtx* cctx = ZSTDMT_getCCtx(job->cctxPool);
|
||||
const void* const src = (const char*)job->srcStart + job->dictSize;
|
||||
ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(job->cctxPool);
|
||||
const void* const src = (const char*)job->srcStart + job->prefixSize;
|
||||
buffer_t dstBuff = job->dstBuff;
|
||||
DEBUGLOG(5, "job (first:%u) (last:%u) : dictSize %u, srcSize %u",
|
||||
job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
|
||||
DEBUGLOG(5, "ZSTDMT_compressChunk: job (first:%u) (last:%u) : prefixSize %u, srcSize %u ",
|
||||
job->firstChunk, job->lastChunk, (U32)job->prefixSize, (U32)job->srcSize);
|
||||
|
||||
if (cctx==NULL) {
|
||||
job->cSize = ERROR(memory_allocation);
|
||||
@ -342,38 +350,48 @@ void ZSTDMT_compressChunk(void* jobDescription)
|
||||
goto _endJob;
|
||||
}
|
||||
job->dstBuff = dstBuff;
|
||||
DEBUGLOG(5, "ZSTDMT_compressChunk: received dstBuff of size %u", (U32)dstBuff.size);
|
||||
}
|
||||
|
||||
if (job->cdict) { /* should only happen for first segment */
|
||||
size_t const initError = ZSTD_compressBegin_usingCDict_advanced(cctx, job->cdict, job->params.fParams, job->fullFrameSize);
|
||||
DEBUGLOG(5, "using CDict");
|
||||
if (job->cdict) {
|
||||
size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dm_auto, job->cdict, job->params, job->fullFrameSize);
|
||||
DEBUGLOG(4, "ZSTDMT_compressChunk: init using CDict (windowLog=%u)", job->params.cParams.windowLog);
|
||||
assert(job->firstChunk); /* only allowed for first job */
|
||||
if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
|
||||
} else { /* srcStart points at reloaded section */
|
||||
if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0; /* ensure no srcSize control */
|
||||
{ ZSTD_CCtx_params jobParams = job->params;
|
||||
size_t const forceWindowError =
|
||||
ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk);
|
||||
/* Force loading dictionary in "content-only" mode (no header analysis) */
|
||||
size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->srcStart, job->dictSize, ZSTD_dm_rawContent, jobParams, job->fullFrameSize);
|
||||
if (ZSTD_isError(initError) || ZSTD_isError(forceWindowError)) {
|
||||
U64 const pledgedSrcSize = job->firstChunk ? job->fullFrameSize : ZSTD_CONTENTSIZE_UNKNOWN;
|
||||
ZSTD_CCtx_params jobParams = job->params; /* do not modify job->params ! copy it, modify the copy */
|
||||
size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk);
|
||||
if (ZSTD_isError(forceWindowError)) {
|
||||
DEBUGLOG(5, "ZSTD_CCtxParam_setParameter error : %s ", ZSTD_getErrorName(forceWindowError));
|
||||
job->cSize = forceWindowError;
|
||||
goto _endJob;
|
||||
}
|
||||
DEBUGLOG(5, "ZSTDMT_compressChunk: invoking ZSTD_compressBegin_advanced_internal with windowLog = %u ", jobParams.cParams.windowLog);
|
||||
{ size_t const initError = ZSTD_compressBegin_advanced_internal(cctx,
|
||||
job->srcStart, job->prefixSize, ZSTD_dm_rawContent, /* load dictionary in "content-only" mode (no header analysis) */
|
||||
NULL,
|
||||
jobParams, pledgedSrcSize);
|
||||
if (ZSTD_isError(initError)) {
|
||||
DEBUGLOG(5, "ZSTD_compressBegin_advanced_internal error : %s ", ZSTD_getErrorName(initError));
|
||||
job->cSize = initError;
|
||||
goto _endJob;
|
||||
}
|
||||
} }
|
||||
if (!job->firstChunk) { /* flush and overwrite frame header when it's not first segment */
|
||||
} }
|
||||
}
|
||||
if (!job->firstChunk) { /* flush and overwrite frame header when it's not first job */
|
||||
size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, 0);
|
||||
if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; }
|
||||
if (ZSTD_isError(hSize)) { job->cSize = hSize; /* save error code */ goto _endJob; }
|
||||
ZSTD_invalidateRepCodes(cctx);
|
||||
}
|
||||
|
||||
DEBUGLOG(5, "Compressing : ");
|
||||
DEBUG_PRINTHEX(4, job->srcStart, 12);
|
||||
DEBUGLOG(5, "Compressing into dstBuff of size %u", (U32)dstBuff.size);
|
||||
DEBUG_PRINTHEX(6, job->srcStart, 12);
|
||||
job->cSize = (job->lastChunk) ?
|
||||
ZSTD_compressEnd (cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
|
||||
ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
|
||||
DEBUGLOG(5, "compressed %u bytes into %u bytes (first:%u) (last:%u)",
|
||||
DEBUGLOG(5, "compressed %u bytes into %u bytes (first:%u) (last:%u) ",
|
||||
(unsigned)job->srcSize, (unsigned)job->cSize, job->firstChunk, job->lastChunk);
|
||||
DEBUGLOG(5, "dstBuff.size : %u ; => %s", (U32)dstBuff.size, ZSTD_getErrorName(job->cSize));
|
||||
DEBUGLOG(5, "dstBuff.size : %u ; => %s ", (U32)dstBuff.size, ZSTD_getErrorName(job->cSize));
|
||||
|
||||
_endJob:
|
||||
ZSTDMT_releaseCCtx(job->cctxPool, cctx);
|
||||
@ -403,13 +421,14 @@ struct ZSTDMT_CCtx_s {
|
||||
ZSTDMT_CCtxPool* cctxPool;
|
||||
ZSTD_pthread_mutex_t jobCompleted_mutex;
|
||||
ZSTD_pthread_cond_t jobCompleted_cond;
|
||||
ZSTD_CCtx_params params;
|
||||
size_t targetSectionSize;
|
||||
size_t inBuffSize;
|
||||
size_t dictSize;
|
||||
size_t targetDictSize;
|
||||
inBuff_t inBuff;
|
||||
ZSTD_CCtx_params params;
|
||||
XXH64_state_t xxhState;
|
||||
unsigned singleThreaded;
|
||||
unsigned jobIDMask;
|
||||
unsigned doneJobID;
|
||||
unsigned nextJobID;
|
||||
@ -430,20 +449,32 @@ static ZSTDMT_jobDescription* ZSTDMT_allocJobsTable(U32* nbJobsPtr, ZSTD_customM
|
||||
nbJobs * sizeof(ZSTDMT_jobDescription), cMem);
|
||||
}
|
||||
|
||||
/* Internal only */
|
||||
size_t ZSTDMT_initializeCCtxParameters(ZSTD_CCtx_params* params, unsigned nbThreads)
|
||||
/* ZSTDMT_CCtxParam_setNbThreads():
|
||||
* Internal use only */
|
||||
size_t ZSTDMT_CCtxParam_setNbThreads(ZSTD_CCtx_params* params, unsigned nbThreads)
|
||||
{
|
||||
if (nbThreads > ZSTDMT_NBTHREADS_MAX) nbThreads = ZSTDMT_NBTHREADS_MAX;
|
||||
if (nbThreads < 1) nbThreads = 1;
|
||||
params->nbThreads = nbThreads;
|
||||
params->overlapSizeLog = ZSTDMT_OVERLAPLOG_DEFAULT;
|
||||
params->jobSize = 0;
|
||||
return 0;
|
||||
return nbThreads;
|
||||
}
|
||||
|
||||
/* ZSTDMT_getNbThreads():
|
||||
* @return nb threads currently active in mtctx.
|
||||
* mtctx must be valid */
|
||||
size_t ZSTDMT_getNbThreads(const ZSTDMT_CCtx* mtctx)
|
||||
{
|
||||
assert(mtctx != NULL);
|
||||
return mtctx->params.nbThreads;
|
||||
}
|
||||
|
||||
ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem)
|
||||
{
|
||||
ZSTDMT_CCtx* mtctx;
|
||||
U32 nbJobs = nbThreads + 2;
|
||||
DEBUGLOG(3, "ZSTDMT_createCCtx_advanced");
|
||||
DEBUGLOG(3, "ZSTDMT_createCCtx_advanced (nbThreads = %u)", nbThreads);
|
||||
|
||||
if (nbThreads < 1) return NULL;
|
||||
nbThreads = MIN(nbThreads , ZSTDMT_NBTHREADS_MAX);
|
||||
@ -453,7 +484,7 @@ ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem)
|
||||
|
||||
mtctx = (ZSTDMT_CCtx*) ZSTD_calloc(sizeof(ZSTDMT_CCtx), cMem);
|
||||
if (!mtctx) return NULL;
|
||||
ZSTDMT_initializeCCtxParameters(&mtctx->params, nbThreads);
|
||||
ZSTDMT_CCtxParam_setNbThreads(&mtctx->params, nbThreads);
|
||||
mtctx->cMem = cMem;
|
||||
mtctx->allJobsCompleted = 1;
|
||||
mtctx->factory = POOL_create_advanced(nbThreads, 0, cMem);
|
||||
@ -545,17 +576,23 @@ size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx)
|
||||
}
|
||||
|
||||
/* Internal only */
|
||||
size_t ZSTDMT_CCtxParam_setMTCtxParameter(
|
||||
ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, unsigned value) {
|
||||
size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params,
|
||||
ZSTDMT_parameter parameter, unsigned value) {
|
||||
DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter");
|
||||
switch(parameter)
|
||||
{
|
||||
case ZSTDMT_p_sectionSize :
|
||||
case ZSTDMT_p_jobSize :
|
||||
DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %u", value);
|
||||
if ( (value > 0) /* value==0 => automatic job size */
|
||||
& (value < ZSTDMT_JOBSIZE_MIN) )
|
||||
value = ZSTDMT_JOBSIZE_MIN;
|
||||
params->jobSize = value;
|
||||
return 0;
|
||||
return value;
|
||||
case ZSTDMT_p_overlapSectionLog :
|
||||
if (value > 9) value = 9;
|
||||
DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %u", value);
|
||||
params->overlapSizeLog = (value >= 9) ? 9 : value;
|
||||
return 0;
|
||||
return value;
|
||||
default :
|
||||
return ERROR(parameter_unsupported);
|
||||
}
|
||||
@ -563,9 +600,10 @@ size_t ZSTDMT_CCtxParam_setMTCtxParameter(
|
||||
|
||||
size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned value)
|
||||
{
|
||||
DEBUGLOG(4, "ZSTDMT_setMTCtxParameter");
|
||||
switch(parameter)
|
||||
{
|
||||
case ZSTDMT_p_sectionSize :
|
||||
case ZSTDMT_p_jobSize :
|
||||
return ZSTDMT_CCtxParam_setMTCtxParameter(&mtctx->params, parameter, value);
|
||||
case ZSTDMT_p_overlapSectionLog :
|
||||
return ZSTDMT_CCtxParam_setMTCtxParameter(&mtctx->params, parameter, value);
|
||||
@ -601,7 +639,7 @@ static size_t ZSTDMT_compress_advanced_internal(
|
||||
size_t const overlapSize = (overlapRLog>=9) ? 0 : (size_t)1 << (params.cParams.windowLog - overlapRLog);
|
||||
unsigned nbChunks = computeNbChunks(srcSize, params.cParams.windowLog, params.nbThreads);
|
||||
size_t const proposedChunkSize = (srcSize + (nbChunks-1)) / nbChunks;
|
||||
size_t const avgChunkSize = ((proposedChunkSize & 0x1FFFF) < 0x7FFF) ? proposedChunkSize + 0xFFFF : proposedChunkSize; /* avoid too small last block */
|
||||
size_t const avgChunkSize = (((proposedChunkSize-1) & 0x1FFFF) < 0x7FFF) ? proposedChunkSize + 0xFFFF : proposedChunkSize; /* avoid too small last block */
|
||||
const char* const srcStart = (const char*)src;
|
||||
size_t remainingSrcSize = srcSize;
|
||||
unsigned const compressWithinDst = (dstCapacity >= ZSTD_compressBound(srcSize)) ? nbChunks : (unsigned)(dstCapacity / ZSTD_compressBound(avgChunkSize)); /* presumes avgChunkSize >= 256 KB, which should be the case */
|
||||
@ -610,7 +648,8 @@ static size_t ZSTDMT_compress_advanced_internal(
|
||||
assert(jobParams.nbThreads == 0);
|
||||
assert(mtctx->cctxPool->totalCCtx == params.nbThreads);
|
||||
|
||||
DEBUGLOG(4, "nbChunks : %2u (chunkSize : %u bytes) ", nbChunks, (U32)avgChunkSize);
|
||||
DEBUGLOG(4, "ZSTDMT_compress_advanced_internal: nbChunks=%2u (rawSize=%u bytes; fixedSize=%u) ",
|
||||
nbChunks, (U32)proposedChunkSize, (U32)avgChunkSize);
|
||||
if (nbChunks==1) { /* fallback to single-thread mode */
|
||||
ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0];
|
||||
if (cdict) return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, jobParams.fParams);
|
||||
@ -639,9 +678,9 @@ static size_t ZSTDMT_compress_advanced_internal(
|
||||
|
||||
mtctx->jobs[u].src = g_nullBuffer;
|
||||
mtctx->jobs[u].srcStart = srcStart + frameStartPos - dictSize;
|
||||
mtctx->jobs[u].dictSize = dictSize;
|
||||
mtctx->jobs[u].prefixSize = dictSize;
|
||||
mtctx->jobs[u].srcSize = chunkSize;
|
||||
mtctx->jobs[u].cdict = mtctx->nextJobID==0 ? cdict : NULL;
|
||||
mtctx->jobs[u].cdict = (u==0) ? cdict : NULL;
|
||||
mtctx->jobs[u].fullFrameSize = srcSize;
|
||||
mtctx->jobs[u].params = jobParams;
|
||||
/* do not calculate checksum within sections, but write it in header for first section */
|
||||
@ -659,7 +698,7 @@ static size_t ZSTDMT_compress_advanced_internal(
|
||||
XXH64_update(&xxh64, srcStart + frameStartPos, chunkSize);
|
||||
}
|
||||
|
||||
DEBUGLOG(5, "posting job %u (%u bytes)", u, (U32)chunkSize);
|
||||
DEBUGLOG(5, "ZSTDMT_compress_advanced_internal: posting job %u (%u bytes)", u, (U32)chunkSize);
|
||||
DEBUG_PRINTHEX(6, mtctx->jobs[u].srcStart, 12);
|
||||
POOL_add(mtctx->factory, ZSTDMT_compressChunk, &mtctx->jobs[u]);
|
||||
|
||||
@ -753,13 +792,14 @@ size_t ZSTDMT_initCStream_internal(
|
||||
const ZSTD_CDict* cdict, ZSTD_CCtx_params params,
|
||||
unsigned long long pledgedSrcSize)
|
||||
{
|
||||
DEBUGLOG(4, "ZSTDMT_initCStream_internal");
|
||||
DEBUGLOG(4, "ZSTDMT_initCStream_internal (pledgedSrcSize=%u)", (U32)pledgedSrcSize);
|
||||
/* params are supposed to be fully validated at this point */
|
||||
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
|
||||
assert(!((dict) && (cdict))); /* either dict or cdict, not both */
|
||||
assert(zcs->cctxPool->totalCCtx == params.nbThreads);
|
||||
zcs->singleThreaded = (params.nbThreads==1) | (pledgedSrcSize <= ZSTDMT_JOBSIZE_MIN); /* do not trigger multi-threading when srcSize is too small */
|
||||
|
||||
if (params.nbThreads==1) {
|
||||
if (zcs->singleThreaded) {
|
||||
ZSTD_CCtx_params const singleThreadParams = ZSTDMT_makeJobCCtxParams(params);
|
||||
DEBUGLOG(4, "single thread mode");
|
||||
assert(singleThreadParams.nbThreads == 0);
|
||||
@ -767,6 +807,7 @@ size_t ZSTDMT_initCStream_internal(
|
||||
dict, dictSize, cdict,
|
||||
singleThreadParams, pledgedSrcSize);
|
||||
}
|
||||
DEBUGLOG(4, "multi-threading mode (%u threads)", params.nbThreads);
|
||||
|
||||
if (zcs->allJobsCompleted == 0) { /* previous compression not correctly finished */
|
||||
ZSTDMT_waitForAllJobsCompleted(zcs);
|
||||
@ -777,7 +818,6 @@ size_t ZSTDMT_initCStream_internal(
|
||||
zcs->params = params;
|
||||
zcs->frameContentSize = pledgedSrcSize;
|
||||
if (dict) {
|
||||
DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal);
|
||||
ZSTD_freeCDict(zcs->cdictLocal);
|
||||
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
|
||||
ZSTD_dlm_byCopy, dictMode, /* note : a loadPrefix becomes an internal CDict */
|
||||
@ -785,20 +825,20 @@ size_t ZSTDMT_initCStream_internal(
|
||||
zcs->cdict = zcs->cdictLocal;
|
||||
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
|
||||
} else {
|
||||
DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal);
|
||||
ZSTD_freeCDict(zcs->cdictLocal);
|
||||
zcs->cdictLocal = NULL;
|
||||
zcs->cdict = cdict;
|
||||
}
|
||||
|
||||
assert(params.overlapSizeLog <= 9);
|
||||
zcs->targetDictSize = (params.overlapSizeLog==0) ? 0 : (size_t)1 << (params.cParams.windowLog - (9 - params.overlapSizeLog));
|
||||
DEBUGLOG(4, "overlapLog : %u ", params.overlapSizeLog);
|
||||
DEBUGLOG(4, "overlap Size : %u KB", (U32)(zcs->targetDictSize>>10));
|
||||
DEBUGLOG(4, "overlapLog=%u => %u KB", params.overlapSizeLog, (U32)(zcs->targetDictSize>>10));
|
||||
zcs->targetSectionSize = params.jobSize ? params.jobSize : (size_t)1 << (params.cParams.windowLog + 2);
|
||||
zcs->targetSectionSize = MAX(ZSTDMT_SECTION_SIZE_MIN, zcs->targetSectionSize);
|
||||
zcs->targetSectionSize = MAX(zcs->targetDictSize, zcs->targetSectionSize);
|
||||
DEBUGLOG(4, "Section Size : %u KB", (U32)(zcs->targetSectionSize>>10));
|
||||
if (zcs->targetSectionSize < ZSTDMT_JOBSIZE_MIN) zcs->targetSectionSize = ZSTDMT_JOBSIZE_MIN;
|
||||
if (zcs->targetSectionSize < zcs->targetDictSize) zcs->targetSectionSize = zcs->targetDictSize; /* job size must be >= overlap size */
|
||||
DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(zcs->targetSectionSize>>10), params.jobSize);
|
||||
zcs->inBuffSize = zcs->targetDictSize + zcs->targetSectionSize;
|
||||
DEBUGLOG(4, "inBuff Size : %u KB", (U32)(zcs->inBuffSize>>10));
|
||||
ZSTDMT_setBufferSize(zcs->bufPool, MAX(zcs->inBuffSize, ZSTD_compressBound(zcs->targetSectionSize)) );
|
||||
zcs->inBuff.buffer = g_nullBuffer;
|
||||
zcs->dictSize = 0;
|
||||
@ -816,7 +856,7 @@ size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
|
||||
unsigned long long pledgedSrcSize)
|
||||
{
|
||||
ZSTD_CCtx_params cctxParams = mtctx->params;
|
||||
DEBUGLOG(5, "ZSTDMT_initCStream_advanced");
|
||||
DEBUGLOG(5, "ZSTDMT_initCStream_advanced (pledgedSrcSize=%u)", (U32)pledgedSrcSize);
|
||||
cctxParams.cParams = params.cParams;
|
||||
cctxParams.fParams = params.fParams;
|
||||
return ZSTDMT_initCStream_internal(mtctx, dict, dictSize, ZSTD_dm_auto, NULL,
|
||||
@ -838,9 +878,12 @@ size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
|
||||
|
||||
|
||||
/* ZSTDMT_resetCStream() :
|
||||
* pledgedSrcSize is optional and can be zero == unknown */
|
||||
* pledgedSrcSize can be zero == unknown (for the time being)
|
||||
* prefer using ZSTD_CONTENTSIZE_UNKNOWN,
|
||||
* as `0` might mean "empty" in the future */
|
||||
size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* zcs, unsigned long long pledgedSrcSize)
|
||||
{
|
||||
if (!pledgedSrcSize) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
|
||||
if (zcs->params.nbThreads==1)
|
||||
return ZSTD_resetCStream(zcs->cctxPool->cctx[0], pledgedSrcSize);
|
||||
return ZSTDMT_initCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, 0, zcs->params,
|
||||
@ -852,7 +895,7 @@ size_t ZSTDMT_initCStream(ZSTDMT_CCtx* zcs, int compressionLevel) {
|
||||
ZSTD_CCtx_params cctxParams = zcs->params;
|
||||
cctxParams.cParams = params.cParams;
|
||||
cctxParams.fParams = params.fParams;
|
||||
return ZSTDMT_initCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, NULL, cctxParams, 0);
|
||||
return ZSTDMT_initCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN);
|
||||
}
|
||||
|
||||
|
||||
@ -860,12 +903,12 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
|
||||
{
|
||||
unsigned const jobID = zcs->nextJobID & zcs->jobIDMask;
|
||||
|
||||
DEBUGLOG(4, "preparing job %u to compress %u bytes with %u preload ",
|
||||
DEBUGLOG(5, "ZSTDMT_createCompressionJob: preparing job %u to compress %u bytes with %u preload ",
|
||||
zcs->nextJobID, (U32)srcSize, (U32)zcs->dictSize);
|
||||
zcs->jobs[jobID].src = zcs->inBuff.buffer;
|
||||
zcs->jobs[jobID].srcStart = zcs->inBuff.buffer.start;
|
||||
zcs->jobs[jobID].srcSize = srcSize;
|
||||
zcs->jobs[jobID].dictSize = zcs->dictSize;
|
||||
zcs->jobs[jobID].prefixSize = zcs->dictSize;
|
||||
assert(zcs->inBuff.filled >= srcSize + zcs->dictSize);
|
||||
zcs->jobs[jobID].params = zcs->params;
|
||||
/* do not calculate checksum within sections, but write it in header for first section */
|
||||
@ -911,7 +954,7 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
|
||||
zcs->params.fParams.checksumFlag = 0;
|
||||
} }
|
||||
|
||||
DEBUGLOG(4, "posting job %u : %u bytes (end:%u) (note : doneJob = %u=>%u)",
|
||||
DEBUGLOG(5, "ZSTDMT_createCompressionJob: posting job %u : %u bytes (end:%u) (note : doneJob = %u=>%u)",
|
||||
zcs->nextJobID,
|
||||
(U32)zcs->jobs[jobID].srcSize,
|
||||
zcs->jobs[jobID].lastChunk,
|
||||
@ -930,6 +973,7 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
|
||||
static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsigned blockToFlush)
|
||||
{
|
||||
unsigned const wJobID = zcs->doneJobID & zcs->jobIDMask;
|
||||
DEBUGLOG(5, "ZSTDMT_flushNextJob");
|
||||
if (zcs->doneJobID == zcs->nextJobID) return 0; /* all flushed ! */
|
||||
ZSTD_PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
|
||||
while (zcs->jobs[wJobID].jobCompleted==0) {
|
||||
@ -942,7 +986,8 @@ static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsi
|
||||
{ ZSTDMT_jobDescription job = zcs->jobs[wJobID];
|
||||
if (!job.jobScanned) {
|
||||
if (ZSTD_isError(job.cSize)) {
|
||||
DEBUGLOG(5, "compression error detected ");
|
||||
DEBUGLOG(5, "job %u : compression error detected : %s",
|
||||
zcs->doneJobID, ZSTD_getErrorName(job.cSize));
|
||||
ZSTDMT_waitForAllJobsCompleted(zcs);
|
||||
ZSTDMT_releaseAllJobResources(zcs);
|
||||
return job.cSize;
|
||||
@ -991,15 +1036,18 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
|
||||
{
|
||||
size_t const newJobThreshold = mtctx->dictSize + mtctx->targetSectionSize;
|
||||
unsigned forwardInputProgress = 0;
|
||||
DEBUGLOG(5, "ZSTDMT_compressStream_generic ");
|
||||
assert(output->pos <= output->size);
|
||||
assert(input->pos <= input->size);
|
||||
|
||||
if (mtctx->singleThreaded) { /* delegate to single-thread (synchronous) */
|
||||
return ZSTD_compressStream_generic(mtctx->cctxPool->cctx[0], output, input, endOp);
|
||||
}
|
||||
|
||||
if ((mtctx->frameEnded) && (endOp==ZSTD_e_continue)) {
|
||||
/* current frame being ended. Only flush/end are allowed */
|
||||
return ERROR(stage_wrong);
|
||||
}
|
||||
if (mtctx->params.nbThreads==1) { /* delegate to single-thread (synchronous) */
|
||||
return ZSTD_compressStream_generic(mtctx->cctxPool->cctx[0], output, input, endOp);
|
||||
}
|
||||
|
||||
/* single-pass shortcut (note : synchronous-mode) */
|
||||
if ( (mtctx->nextJobID == 0) /* just started */
|
||||
@ -1068,32 +1116,34 @@ size_t ZSTDMT_compressStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, ZSTD_inBu
|
||||
}
|
||||
|
||||
|
||||
static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsigned endFrame)
|
||||
static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, unsigned endFrame)
|
||||
{
|
||||
size_t const srcSize = zcs->inBuff.filled - zcs->dictSize;
|
||||
size_t const srcSize = mtctx->inBuff.filled - mtctx->dictSize;
|
||||
DEBUGLOG(5, "ZSTDMT_flushStream_internal");
|
||||
|
||||
if ( ((srcSize > 0) || (endFrame && !zcs->frameEnded))
|
||||
&& (zcs->nextJobID <= zcs->doneJobID + zcs->jobIDMask) ) {
|
||||
CHECK_F( ZSTDMT_createCompressionJob(zcs, srcSize, endFrame) );
|
||||
if ( ((srcSize > 0) || (endFrame && !mtctx->frameEnded))
|
||||
&& (mtctx->nextJobID <= mtctx->doneJobID + mtctx->jobIDMask) ) {
|
||||
DEBUGLOG(5, "ZSTDMT_flushStream_internal : create a new job");
|
||||
CHECK_F( ZSTDMT_createCompressionJob(mtctx, srcSize, endFrame) );
|
||||
}
|
||||
|
||||
/* check if there is any data available to flush */
|
||||
return ZSTDMT_flushNextJob(zcs, output, 1 /* blockToFlush */);
|
||||
return ZSTDMT_flushNextJob(mtctx, output, 1 /* blockToFlush */);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTDMT_flushStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output)
|
||||
size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output)
|
||||
{
|
||||
DEBUGLOG(5, "ZSTDMT_flushStream");
|
||||
if (zcs->params.nbThreads==1)
|
||||
return ZSTD_flushStream(zcs->cctxPool->cctx[0], output);
|
||||
return ZSTDMT_flushStream_internal(zcs, output, 0 /* endFrame */);
|
||||
if (mtctx->singleThreaded)
|
||||
return ZSTD_flushStream(mtctx->cctxPool->cctx[0], output);
|
||||
return ZSTDMT_flushStream_internal(mtctx, output, 0 /* endFrame */);
|
||||
}
|
||||
|
||||
size_t ZSTDMT_endStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output)
|
||||
size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output)
|
||||
{
|
||||
DEBUGLOG(4, "ZSTDMT_endStream");
|
||||
if (zcs->params.nbThreads==1)
|
||||
return ZSTD_endStream(zcs->cctxPool->cctx[0], output);
|
||||
return ZSTDMT_flushStream_internal(zcs, output, 1 /* endFrame */);
|
||||
if (mtctx->singleThreaded)
|
||||
return ZSTD_endStream(mtctx->cctxPool->cctx[0], output);
|
||||
return ZSTDMT_flushStream_internal(mtctx, output, 1 /* endFrame */);
|
||||
}
|
||||
|
@ -50,7 +50,7 @@ ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
|
||||
/* === Streaming functions === */
|
||||
|
||||
ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
|
||||
ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it may change in the future, to mean "empty" */
|
||||
|
||||
ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
|
||||
|
||||
@ -60,8 +60,8 @@ ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);
|
||||
|
||||
/* === Advanced functions and parameters === */
|
||||
|
||||
#ifndef ZSTDMT_SECTION_SIZE_MIN
|
||||
# define ZSTDMT_SECTION_SIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */
|
||||
#ifndef ZSTDMT_JOBSIZE_MIN
|
||||
# define ZSTDMT_JOBSIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */
|
||||
#endif
|
||||
|
||||
ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
|
||||
@ -84,13 +84,13 @@ ZSTDLIB_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
|
||||
/* ZSTDMT_parameter :
|
||||
* List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
|
||||
typedef enum {
|
||||
ZSTDMT_p_sectionSize, /* size of input "section". Each section is compressed in parallel. 0 means default, which is dynamically determined within compression functions */
|
||||
ZSTDMT_p_overlapSectionLog /* Log of overlapped section; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window */
|
||||
ZSTDMT_p_jobSize, /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */
|
||||
ZSTDMT_p_overlapSectionLog /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window */
|
||||
} ZSTDMT_parameter;
|
||||
|
||||
/* ZSTDMT_setMTCtxParameter() :
|
||||
* allow setting individual parameters, one at a time, among a list of enums defined in ZSTDMT_parameter.
|
||||
* The function must be called typically after ZSTD_createCCtx().
|
||||
* The function must be called typically after ZSTD_createCCtx() but __before ZSTDMT_init*() !__
|
||||
* Parameters not explicitly reset by ZSTDMT_init*() remain the same in consecutive compression sessions.
|
||||
* @return : 0, or an error code (which can be tested using ZSTD_isError()) */
|
||||
ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned value);
|
||||
@ -112,7 +112,15 @@ ZSTDLIB_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
|
||||
|
||||
size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, unsigned value);
|
||||
|
||||
size_t ZSTDMT_initializeCCtxParameters(ZSTD_CCtx_params* params, unsigned nbThreads);
|
||||
/* ZSTDMT_CCtxParam_setNbThreads()
|
||||
* Set nbThreads, and clamp it correctly,
|
||||
* also reset jobSize and overlapLog */
|
||||
size_t ZSTDMT_CCtxParam_setNbThreads(ZSTD_CCtx_params* params, unsigned nbThreads);
|
||||
|
||||
/* ZSTDMT_getNbThreads():
|
||||
* @return nb threads currently active in mtctx.
|
||||
* mtctx must be valid */
|
||||
size_t ZSTDMT_getNbThreads(const ZSTDMT_CCtx* mtctx);
|
||||
|
||||
/*! ZSTDMT_initCStream_internal() :
|
||||
* Private use only. Init streaming operation.
|
||||
|
@ -827,9 +827,9 @@ typedef struct {
|
||||
FSE_DState_t stateOffb;
|
||||
FSE_DState_t stateML;
|
||||
size_t prevOffset[ZSTD_REP_NUM];
|
||||
const BYTE* base;
|
||||
const BYTE* prefixStart;
|
||||
const BYTE* dictEnd;
|
||||
size_t pos;
|
||||
uPtrDiff gotoDict;
|
||||
} seqState_t;
|
||||
|
||||
|
||||
@ -1224,8 +1224,9 @@ seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, ZSTD_longOffset_e const long
|
||||
BIT_reloadDStream(&seqState->DStream);
|
||||
|
||||
{ size_t const pos = seqState->pos + seq.litLength;
|
||||
seq.match = seqState->base + pos - seq.offset; /* single memory segment */
|
||||
if (seq.offset > pos) seq.match += seqState->gotoDict; /* separate memory segment */
|
||||
const BYTE* const matchBase = (seq.offset > pos) ? seqState->dictEnd : seqState->prefixStart;
|
||||
seq.match = matchBase + pos - seq.offset; /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted.
|
||||
* No consequence though : no memory access will occur, overly large offset will be detected in ZSTD_execSequenceLong() */
|
||||
seqState->pos = pos + seq.matchLength;
|
||||
}
|
||||
|
||||
@ -1243,7 +1244,7 @@ HINT_INLINE
|
||||
size_t ZSTD_execSequenceLong(BYTE* op,
|
||||
BYTE* const oend, seq_t sequence,
|
||||
const BYTE** litPtr, const BYTE* const litLimit,
|
||||
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
|
||||
const BYTE* const prefixStart, const BYTE* const dictStart, const BYTE* const dictEnd)
|
||||
{
|
||||
BYTE* const oLitEnd = op + sequence.litLength;
|
||||
size_t const sequenceLength = sequence.litLength + sequence.matchLength;
|
||||
@ -1253,21 +1254,21 @@ size_t ZSTD_execSequenceLong(BYTE* op,
|
||||
const BYTE* match = sequence.match;
|
||||
|
||||
/* check */
|
||||
if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
|
||||
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
|
||||
if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */
|
||||
if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd);
|
||||
if (oLitEnd > oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, dictStart, dictEnd);
|
||||
|
||||
/* copy Literals */
|
||||
ZSTD_copy8(op, *litPtr);
|
||||
ZSTD_copy8(op, *litPtr); /* note : op <= oLitEnd <= oend_w == oend - 8 */
|
||||
if (sequence.litLength > 8)
|
||||
ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
|
||||
op = oLitEnd;
|
||||
*litPtr = iLitEnd; /* update for next sequence */
|
||||
|
||||
/* copy Match */
|
||||
if (sequence.offset > (size_t)(oLitEnd - base)) {
|
||||
if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
|
||||
/* offset beyond prefix */
|
||||
if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected);
|
||||
if (sequence.offset > (size_t)(oLitEnd - dictStart)) return ERROR(corruption_detected);
|
||||
if (match + sequence.matchLength <= dictEnd) {
|
||||
memmove(oLitEnd, match, sequence.matchLength);
|
||||
return sequenceLength;
|
||||
@ -1277,7 +1278,7 @@ size_t ZSTD_execSequenceLong(BYTE* op,
|
||||
memmove(oLitEnd, match, length1);
|
||||
op = oLitEnd + length1;
|
||||
sequence.matchLength -= length1;
|
||||
match = base;
|
||||
match = prefixStart;
|
||||
if (op > oend_w || sequence.matchLength < MINMATCH) {
|
||||
U32 i;
|
||||
for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i];
|
||||
@ -1331,8 +1332,8 @@ static size_t ZSTD_decompressSequencesLong(
|
||||
BYTE* op = ostart;
|
||||
const BYTE* litPtr = dctx->litPtr;
|
||||
const BYTE* const litEnd = litPtr + dctx->litSize;
|
||||
const BYTE* const base = (const BYTE*) (dctx->base);
|
||||
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
|
||||
const BYTE* const prefixStart = (const BYTE*) (dctx->base);
|
||||
const BYTE* const dictStart = (const BYTE*) (dctx->vBase);
|
||||
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
|
||||
int nbSeq;
|
||||
|
||||
@ -1353,9 +1354,9 @@ static size_t ZSTD_decompressSequencesLong(
|
||||
int seqNb;
|
||||
dctx->fseEntropy = 1;
|
||||
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
|
||||
seqState.base = base;
|
||||
seqState.pos = (size_t)(op-base);
|
||||
seqState.gotoDict = (uPtrDiff)dictEnd - (uPtrDiff)base; /* cast to avoid undefined behaviour */
|
||||
seqState.prefixStart = prefixStart;
|
||||
seqState.pos = (size_t)(op-prefixStart);
|
||||
seqState.dictEnd = dictEnd;
|
||||
CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected);
|
||||
FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
|
||||
FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
|
||||
@ -1370,9 +1371,9 @@ static size_t ZSTD_decompressSequencesLong(
|
||||
/* decode and decompress */
|
||||
for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && seqNb<nbSeq ; seqNb++) {
|
||||
seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, isLongOffset);
|
||||
size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
|
||||
size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd);
|
||||
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
||||
PREFETCH(sequence.match);
|
||||
PREFETCH(sequence.match); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */
|
||||
sequences[seqNb&STOSEQ_MASK] = sequence;
|
||||
op += oneSeqSize;
|
||||
}
|
||||
@ -1381,7 +1382,7 @@ static size_t ZSTD_decompressSequencesLong(
|
||||
/* finish queue */
|
||||
seqNb -= seqAdvance;
|
||||
for ( ; seqNb<nbSeq ; seqNb++) {
|
||||
size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[seqNb&STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
|
||||
size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[seqNb&STOSEQ_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd);
|
||||
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
||||
op += oneSeqSize;
|
||||
}
|
||||
@ -2450,14 +2451,16 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
||||
return ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input);
|
||||
}
|
||||
#endif
|
||||
return hSize; /* error */
|
||||
return hSize; /* error */
|
||||
}
|
||||
if (hSize != 0) { /* need more input */
|
||||
size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */
|
||||
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
|
||||
if (iend-ip > 0) {
|
||||
memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip);
|
||||
zds->lhSize += iend-ip;
|
||||
size_t const remainingInput = (size_t)(iend-ip);
|
||||
assert(iend >= ip);
|
||||
if (toLoad > remainingInput) { /* not enough input to load full header */
|
||||
if (remainingInput > 0) {
|
||||
memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput);
|
||||
zds->lhSize += remainingInput;
|
||||
}
|
||||
input->pos = input->size;
|
||||
return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
|
||||
@ -2472,8 +2475,10 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
||||
&& (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
|
||||
size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart);
|
||||
if (cSize <= (size_t)(iend-istart)) {
|
||||
/* shortcut : using single-pass mode */
|
||||
size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, oend-op, istart, cSize, zds->ddict);
|
||||
if (ZSTD_isError(decompressedSize)) return decompressedSize;
|
||||
DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()")
|
||||
ip = istart + cSize;
|
||||
op += decompressedSize;
|
||||
zds->expected = 0;
|
||||
@ -2496,8 +2501,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
||||
}
|
||||
|
||||
/* control buffer memory usage */
|
||||
DEBUGLOG(4, "Control max buffer memory usage (max %u KB)",
|
||||
(U32)(zds->maxWindowSize >> 10));
|
||||
DEBUGLOG(4, "Control max memory usage (%u KB <= max %u KB)",
|
||||
(U32)(zds->fParams.windowSize >>10),
|
||||
(U32)(zds->maxWindowSize >> 10) );
|
||||
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
|
||||
if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
|
||||
|
||||
@ -2555,17 +2561,21 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
||||
/* fall-through */
|
||||
case zdss_load:
|
||||
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
|
||||
size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */
|
||||
size_t const toLoad = neededInSize - zds->inPos;
|
||||
int const isSkipFrame = ZSTD_isSkipFrame(zds);
|
||||
size_t loadedSize;
|
||||
if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */
|
||||
loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip);
|
||||
if (isSkipFrame) {
|
||||
loadedSize = MIN(toLoad, (size_t)(iend-ip));
|
||||
} else {
|
||||
if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */
|
||||
loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip);
|
||||
}
|
||||
ip += loadedSize;
|
||||
zds->inPos += loadedSize;
|
||||
if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */
|
||||
|
||||
/* decode loaded input */
|
||||
{ const int isSkipFrame = ZSTD_isSkipFrame(zds);
|
||||
size_t const decodedSize = ZSTD_decompressContinue(zds,
|
||||
{ size_t const decodedSize = ZSTD_decompressContinue(zds,
|
||||
zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart,
|
||||
zds->inBuff, neededInSize);
|
||||
if (ZSTD_isError(decodedSize)) return decodedSize;
|
||||
|
@ -72,6 +72,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize)
|
||||
{
|
||||
if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; /* preserve "0 == unknown" behavior */
|
||||
return ZSTD_initCStream_advanced(zbc, dict, dictSize, params, pledgedSrcSize);
|
||||
}
|
||||
|
||||
|
@ -103,7 +103,7 @@ unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize)
|
||||
/*-********************************************************
|
||||
* Dictionary training functions
|
||||
**********************************************************/
|
||||
static unsigned ZDICT_NbCommonBytes (register size_t val)
|
||||
static unsigned ZDICT_NbCommonBytes (size_t val)
|
||||
{
|
||||
if (MEM_isLittleEndian()) {
|
||||
if (MEM_64bits()) {
|
||||
|
@ -339,7 +339,7 @@ typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
|
||||
/****************************************************************
|
||||
* Internal functions
|
||||
****************************************************************/
|
||||
FORCE_INLINE unsigned FSE_highbit32 (register U32 val)
|
||||
FORCE_INLINE unsigned FSE_highbit32 (U32 val)
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
unsigned long r;
|
||||
|
@ -330,18 +330,6 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
|
||||
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
|
||||
|
||||
|
||||
/*
|
||||
* Start by invoking BIT_initDStream().
|
||||
* A chunk of the bitStream is then stored into a local register.
|
||||
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
||||
* You can then retrieve bitFields stored into the local register, **in reverse order**.
|
||||
* Local register is manually filled from memory by the BIT_reloadDStream() method.
|
||||
* A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished.
|
||||
* Otherwise, it can be less than that, so proceed accordingly.
|
||||
* Checking if DStream has reached its end can be performed with BIT_endOfDStream()
|
||||
*/
|
||||
|
||||
|
||||
/******************************************
|
||||
* unsafe API
|
||||
******************************************/
|
||||
@ -353,7 +341,7 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
|
||||
/****************************************************************
|
||||
* Helper functions
|
||||
****************************************************************/
|
||||
MEM_STATIC unsigned BIT_highbit32 (register U32 val)
|
||||
MEM_STATIC unsigned BIT_highbit32 (U32 val)
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
unsigned long r=0;
|
||||
@ -427,13 +415,6 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
||||
return srcSize;
|
||||
}
|
||||
|
||||
/*!BIT_lookBits
|
||||
* Provides next n bits from local register
|
||||
* local register is not modified (bits are still present for next read/look)
|
||||
* On 32-bits, maxNbBits==25
|
||||
* On 64-bits, maxNbBits==57
|
||||
* @return : value extracted
|
||||
*/
|
||||
MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
||||
@ -453,11 +434,6 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
bitD->bitsConsumed += nbBits;
|
||||
}
|
||||
|
||||
/*!BIT_readBits
|
||||
* Read next n bits from local register.
|
||||
* pay attention to not read more than nbBits contained into local register.
|
||||
* @return : extracted value.
|
||||
*/
|
||||
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
size_t value = BIT_lookBits(bitD, nbBits);
|
||||
@ -695,55 +671,6 @@ static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bi
|
||||
|
||||
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
|
||||
|
||||
/*
|
||||
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
|
||||
You will decode FSE-encoded symbols from the bitStream,
|
||||
and also any other bitFields you put in, **in reverse order**.
|
||||
|
||||
You will need a few variables to track your bitStream. They are :
|
||||
|
||||
BIT_DStream_t DStream; // Stream context
|
||||
FSE_DState_t DState; // State context. Multiple ones are possible
|
||||
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
|
||||
|
||||
The first thing to do is to init the bitStream.
|
||||
errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
|
||||
|
||||
You should then retrieve your initial state(s)
|
||||
(in reverse flushing order if you have several ones) :
|
||||
errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
|
||||
|
||||
You can then decode your data, symbol after symbol.
|
||||
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
|
||||
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
||||
unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
|
||||
|
||||
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
||||
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
||||
size_t bitField = BIT_readBits(&DStream, nbBits);
|
||||
|
||||
All above operations only read from local register (which size depends on size_t).
|
||||
Refueling the register from memory is manually performed by the reload method.
|
||||
endSignal = FSE_reloadDStream(&DStream);
|
||||
|
||||
BIT_reloadDStream() result tells if there is still some more data to read from DStream.
|
||||
BIT_DStream_unfinished : there is still some data left into the DStream.
|
||||
BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
||||
BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
||||
BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
||||
|
||||
When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
||||
to properly detect the exact end of stream.
|
||||
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
||||
BIT_reloadDStream(&DStream) >= BIT_DStream_completed
|
||||
|
||||
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
||||
Checking if DStream has reached its end is performed by :
|
||||
BIT_endOfDStream(&DStream);
|
||||
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
||||
FSE_endOfDState(&DState);
|
||||
*/
|
||||
|
||||
|
||||
/******************************************
|
||||
* FSE unsafe API
|
||||
|
@ -332,17 +332,6 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
|
||||
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
|
||||
|
||||
|
||||
/*
|
||||
* Start by invoking BIT_initDStream().
|
||||
* A chunk of the bitStream is then stored into a local register.
|
||||
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
||||
* You can then retrieve bitFields stored into the local register, **in reverse order**.
|
||||
* Local register is manually filled from memory by the BIT_reloadDStream() method.
|
||||
* A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished.
|
||||
* Otherwise, it can be less than that, so proceed accordingly.
|
||||
* Checking if DStream has reached its end can be performed with BIT_endOfDStream()
|
||||
*/
|
||||
|
||||
|
||||
/******************************************
|
||||
* unsafe API
|
||||
@ -355,7 +344,7 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
|
||||
/****************************************************************
|
||||
* Helper functions
|
||||
****************************************************************/
|
||||
MEM_STATIC unsigned BIT_highbit32 (register U32 val)
|
||||
MEM_STATIC unsigned BIT_highbit32 (U32 val)
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
unsigned long r=0;
|
||||
@ -428,14 +417,6 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
||||
|
||||
return srcSize;
|
||||
}
|
||||
|
||||
/*!BIT_lookBits
|
||||
* Provides next n bits from local register
|
||||
* local register is not modified (bits are still present for next read/look)
|
||||
* On 32-bits, maxNbBits==25
|
||||
* On 64-bits, maxNbBits==57
|
||||
* @return : value extracted
|
||||
*/
|
||||
MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
||||
@ -455,11 +436,6 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
bitD->bitsConsumed += nbBits;
|
||||
}
|
||||
|
||||
/*!BIT_readBits
|
||||
* Read next n bits from local register.
|
||||
* pay attention to not read more than nbBits contained into local register.
|
||||
* @return : extracted value.
|
||||
*/
|
||||
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
size_t value = BIT_lookBits(bitD, nbBits);
|
||||
@ -697,55 +673,6 @@ static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bi
|
||||
|
||||
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
|
||||
|
||||
/*
|
||||
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
|
||||
You will decode FSE-encoded symbols from the bitStream,
|
||||
and also any other bitFields you put in, **in reverse order**.
|
||||
|
||||
You will need a few variables to track your bitStream. They are :
|
||||
|
||||
BIT_DStream_t DStream; // Stream context
|
||||
FSE_DState_t DState; // State context. Multiple ones are possible
|
||||
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
|
||||
|
||||
The first thing to do is to init the bitStream.
|
||||
errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
|
||||
|
||||
You should then retrieve your initial state(s)
|
||||
(in reverse flushing order if you have several ones) :
|
||||
errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
|
||||
|
||||
You can then decode your data, symbol after symbol.
|
||||
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
|
||||
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
||||
unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
|
||||
|
||||
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
||||
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
||||
size_t bitField = BIT_readBits(&DStream, nbBits);
|
||||
|
||||
All above operations only read from local register (which size depends on size_t).
|
||||
Refueling the register from memory is manually performed by the reload method.
|
||||
endSignal = FSE_reloadDStream(&DStream);
|
||||
|
||||
BIT_reloadDStream() result tells if there is still some more data to read from DStream.
|
||||
BIT_DStream_unfinished : there is still some data left into the DStream.
|
||||
BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
||||
BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
||||
BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
||||
|
||||
When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
||||
to properly detect the exact end of stream.
|
||||
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
||||
BIT_reloadDStream(&DStream) >= BIT_DStream_completed
|
||||
|
||||
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
||||
Checking if DStream has reached its end is performed by :
|
||||
BIT_endOfDStream(&DStream);
|
||||
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
||||
FSE_endOfDState(&DState);
|
||||
*/
|
||||
|
||||
|
||||
/******************************************
|
||||
* FSE unsafe API
|
||||
|
@ -738,16 +738,6 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
|
||||
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
|
||||
|
||||
|
||||
/*
|
||||
* Start by invoking BIT_initDStream().
|
||||
* A chunk of the bitStream is then stored into a local register.
|
||||
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
||||
* You can then retrieve bitFields stored into the local register, **in reverse order**.
|
||||
* Local register is manually filled from memory by the BIT_reloadDStream() method.
|
||||
* A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished.
|
||||
* Otherwise, it can be less than that, so proceed accordingly.
|
||||
* Checking if DStream has reached its end can be performed with BIT_endOfDStream()
|
||||
*/
|
||||
|
||||
|
||||
/******************************************
|
||||
@ -761,7 +751,7 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
|
||||
/****************************************************************
|
||||
* Helper functions
|
||||
****************************************************************/
|
||||
MEM_STATIC unsigned BIT_highbit32 (register U32 val)
|
||||
MEM_STATIC unsigned BIT_highbit32 (U32 val)
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
unsigned long r=0;
|
||||
@ -834,13 +824,6 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
||||
return srcSize;
|
||||
}
|
||||
|
||||
/*!BIT_lookBits
|
||||
* Provides next n bits from local register
|
||||
* local register is not modified (bits are still present for next read/look)
|
||||
* On 32-bits, maxNbBits==25
|
||||
* On 64-bits, maxNbBits==57
|
||||
* @return : value extracted
|
||||
*/
|
||||
MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
||||
@ -860,11 +843,6 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
bitD->bitsConsumed += nbBits;
|
||||
}
|
||||
|
||||
/*!BIT_readBits
|
||||
* Read next n bits from local register.
|
||||
* pay attention to not read more than nbBits contained into local register.
|
||||
* @return : extracted value.
|
||||
*/
|
||||
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
size_t value = BIT_lookBits(bitD, nbBits);
|
||||
@ -1011,55 +989,6 @@ static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bi
|
||||
|
||||
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
|
||||
|
||||
/*!
|
||||
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
|
||||
You will decode FSE-encoded symbols from the bitStream,
|
||||
and also any other bitFields you put in, **in reverse order**.
|
||||
|
||||
You will need a few variables to track your bitStream. They are :
|
||||
|
||||
BIT_DStream_t DStream; // Stream context
|
||||
FSE_DState_t DState; // State context. Multiple ones are possible
|
||||
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
|
||||
|
||||
The first thing to do is to init the bitStream.
|
||||
errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
|
||||
|
||||
You should then retrieve your initial state(s)
|
||||
(in reverse flushing order if you have several ones) :
|
||||
errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
|
||||
|
||||
You can then decode your data, symbol after symbol.
|
||||
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
|
||||
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
||||
unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
|
||||
|
||||
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
||||
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
||||
size_t bitField = BIT_readBits(&DStream, nbBits);
|
||||
|
||||
All above operations only read from local register (which size depends on size_t).
|
||||
Refueling the register from memory is manually performed by the reload method.
|
||||
endSignal = FSE_reloadDStream(&DStream);
|
||||
|
||||
BIT_reloadDStream() result tells if there is still some more data to read from DStream.
|
||||
BIT_DStream_unfinished : there is still some data left into the DStream.
|
||||
BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
||||
BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
||||
BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
||||
|
||||
When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
||||
to properly detect the exact end of stream.
|
||||
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
||||
BIT_reloadDStream(&DStream) >= BIT_DStream_completed
|
||||
|
||||
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
||||
Checking if DStream has reached its end is performed by :
|
||||
BIT_endOfDStream(&DStream);
|
||||
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
||||
FSE_endOfDState(&DState);
|
||||
*/
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* FSE unsafe API
|
||||
|
@ -736,18 +736,6 @@ MEM_STATIC BITv05_DStream_status BITv05_reloadDStream(BITv05_DStream_t* bitD);
|
||||
MEM_STATIC unsigned BITv05_endOfDStream(const BITv05_DStream_t* bitD);
|
||||
|
||||
|
||||
/*!
|
||||
* Start by invoking BITv05_initDStream().
|
||||
* A chunk of the bitStream is then stored into a local register.
|
||||
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
||||
* You can then retrieve bitFields stored into the local register, **in reverse order**.
|
||||
* Local register is explicitly reloaded from memory by the BITv05_reloadDStream() method.
|
||||
* A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BITv05_DStream_unfinished.
|
||||
* Otherwise, it can be less than that, so proceed accordingly.
|
||||
* Checking if DStream has reached its end can be performed with BITv05_endOfDStream()
|
||||
*/
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* unsafe API
|
||||
******************************************/
|
||||
@ -759,7 +747,7 @@ MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, unsigned nbBits);
|
||||
/*-**************************************************************
|
||||
* Helper functions
|
||||
****************************************************************/
|
||||
MEM_STATIC unsigned BITv05_highbit32 (register U32 val)
|
||||
MEM_STATIC unsigned BITv05_highbit32 (U32 val)
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
unsigned long r=0;
|
||||
@ -829,13 +817,6 @@ MEM_STATIC size_t BITv05_initDStream(BITv05_DStream_t* bitD, const void* srcBuff
|
||||
return srcSize;
|
||||
}
|
||||
|
||||
/*!BITv05_lookBits
|
||||
* Provides next n bits from local register
|
||||
* local register is not modified (bits are still present for next read/look)
|
||||
* On 32-bits, maxNbBits==25
|
||||
* On 64-bits, maxNbBits==57
|
||||
* @return : value extracted
|
||||
*/
|
||||
MEM_STATIC size_t BITv05_lookBits(BITv05_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
||||
@ -855,11 +836,6 @@ MEM_STATIC void BITv05_skipBits(BITv05_DStream_t* bitD, U32 nbBits)
|
||||
bitD->bitsConsumed += nbBits;
|
||||
}
|
||||
|
||||
/*!BITv05_readBits
|
||||
* Read next n bits from local register.
|
||||
* pay attention to not read more than nbBits contained into local register.
|
||||
* @return : extracted value.
|
||||
*/
|
||||
MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
size_t value = BITv05_lookBits(bitD, nbBits);
|
||||
@ -995,54 +971,6 @@ static unsigned char FSEv05_decodeSymbol(FSEv05_DState_t* DStatePtr, BITv05_DStr
|
||||
|
||||
static unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr);
|
||||
|
||||
/*!
|
||||
Let's now decompose FSEv05_decompress_usingDTable() into its unitary components.
|
||||
You will decode FSEv05-encoded symbols from the bitStream,
|
||||
and also any other bitFields you put in, **in reverse order**.
|
||||
|
||||
You will need a few variables to track your bitStream. They are :
|
||||
|
||||
BITv05_DStream_t DStream; // Stream context
|
||||
FSEv05_DState_t DState; // State context. Multiple ones are possible
|
||||
FSEv05_DTable* DTablePtr; // Decoding table, provided by FSEv05_buildDTable()
|
||||
|
||||
The first thing to do is to init the bitStream.
|
||||
errorCode = BITv05_initDStream(&DStream, srcBuffer, srcSize);
|
||||
|
||||
You should then retrieve your initial state(s)
|
||||
(in reverse flushing order if you have several ones) :
|
||||
errorCode = FSEv05_initDState(&DState, &DStream, DTablePtr);
|
||||
|
||||
You can then decode your data, symbol after symbol.
|
||||
For information the maximum number of bits read by FSEv05_decodeSymbol() is 'tableLog'.
|
||||
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
||||
unsigned char symbol = FSEv05_decodeSymbol(&DState, &DStream);
|
||||
|
||||
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
||||
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
||||
size_t bitField = BITv05_readBits(&DStream, nbBits);
|
||||
|
||||
All above operations only read from local register (which size depends on size_t).
|
||||
Refueling the register from memory is manually performed by the reload method.
|
||||
endSignal = FSEv05_reloadDStream(&DStream);
|
||||
|
||||
BITv05_reloadDStream() result tells if there is still some more data to read from DStream.
|
||||
BITv05_DStream_unfinished : there is still some data left into the DStream.
|
||||
BITv05_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
||||
BITv05_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
||||
BITv05_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
||||
|
||||
When reaching end of buffer (BITv05_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
||||
to properly detect the exact end of stream.
|
||||
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
||||
BITv05_reloadDStream(&DStream) >= BITv05_DStream_completed
|
||||
|
||||
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
||||
Checking if DStream has reached its end is performed by :
|
||||
BITv05_endOfDStream(&DStream);
|
||||
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
||||
FSEv05_endOfDState(&DState);
|
||||
*/
|
||||
|
||||
|
||||
/* *****************************************
|
||||
|
@ -839,16 +839,6 @@ MEM_STATIC BITv06_DStream_status BITv06_reloadDStream(BITv06_DStream_t* bitD);
|
||||
MEM_STATIC unsigned BITv06_endOfDStream(const BITv06_DStream_t* bitD);
|
||||
|
||||
|
||||
/* Start by invoking BITv06_initDStream().
|
||||
* A chunk of the bitStream is then stored into a local register.
|
||||
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
||||
* You can then retrieve bitFields stored into the local register, **in reverse order**.
|
||||
* Local register is explicitly reloaded from memory by the BITv06_reloadDStream() method.
|
||||
* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BITv06_DStream_unfinished.
|
||||
* Otherwise, it can be less than that, so proceed accordingly.
|
||||
* Checking if DStream has reached its end can be performed with BITv06_endOfDStream().
|
||||
*/
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* unsafe API
|
||||
@ -861,7 +851,7 @@ MEM_STATIC size_t BITv06_readBitsFast(BITv06_DStream_t* bitD, unsigned nbBits);
|
||||
/*-**************************************************************
|
||||
* Internal functions
|
||||
****************************************************************/
|
||||
MEM_STATIC unsigned BITv06_highbit32 (register U32 val)
|
||||
MEM_STATIC unsigned BITv06_highbit32 ( U32 val)
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
unsigned long r=0;
|
||||
@ -929,13 +919,6 @@ MEM_STATIC size_t BITv06_initDStream(BITv06_DStream_t* bitD, const void* srcBuff
|
||||
}
|
||||
|
||||
|
||||
/*! BITv06_lookBits() :
|
||||
* Provides next n bits from local register.
|
||||
* local register is not modified.
|
||||
* On 32-bits, maxNbBits==24.
|
||||
* On 64-bits, maxNbBits==56.
|
||||
* @return : value extracted
|
||||
*/
|
||||
MEM_STATIC size_t BITv06_lookBits(const BITv06_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
||||
@ -955,11 +938,6 @@ MEM_STATIC void BITv06_skipBits(BITv06_DStream_t* bitD, U32 nbBits)
|
||||
bitD->bitsConsumed += nbBits;
|
||||
}
|
||||
|
||||
/*! BITv06_readBits() :
|
||||
* Read (consume) next n bits from local register and update.
|
||||
* Pay attention to not read more than nbBits contained into local register.
|
||||
* @return : extracted value.
|
||||
*/
|
||||
MEM_STATIC size_t BITv06_readBits(BITv06_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
size_t const value = BITv06_lookBits(bitD, nbBits);
|
||||
@ -976,11 +954,6 @@ MEM_STATIC size_t BITv06_readBitsFast(BITv06_DStream_t* bitD, U32 nbBits)
|
||||
return value;
|
||||
}
|
||||
|
||||
/*! BITv06_reloadDStream() :
|
||||
* Refill `BITv06_DStream_t` from src buffer previously defined (see BITv06_initDStream() ).
|
||||
* This function is safe, it guarantees it will not read beyond src buffer.
|
||||
* @return : status of `BITv06_DStream_t` internal register.
|
||||
if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
|
||||
MEM_STATIC BITv06_DStream_status BITv06_reloadDStream(BITv06_DStream_t* bitD)
|
||||
{
|
||||
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
|
||||
@ -1103,55 +1076,6 @@ static void FSEv06_initDState(FSEv06_DState_t* DStatePtr, BITv06_DStream_t*
|
||||
|
||||
static unsigned char FSEv06_decodeSymbol(FSEv06_DState_t* DStatePtr, BITv06_DStream_t* bitD);
|
||||
|
||||
/*!
|
||||
Let's now decompose FSEv06_decompress_usingDTable() into its unitary components.
|
||||
You will decode FSE-encoded symbols from the bitStream,
|
||||
and also any other bitFields you put in, **in reverse order**.
|
||||
|
||||
You will need a few variables to track your bitStream. They are :
|
||||
|
||||
BITv06_DStream_t DStream; // Stream context
|
||||
FSEv06_DState_t DState; // State context. Multiple ones are possible
|
||||
FSEv06_DTable* DTablePtr; // Decoding table, provided by FSEv06_buildDTable()
|
||||
|
||||
The first thing to do is to init the bitStream.
|
||||
errorCode = BITv06_initDStream(&DStream, srcBuffer, srcSize);
|
||||
|
||||
You should then retrieve your initial state(s)
|
||||
(in reverse flushing order if you have several ones) :
|
||||
errorCode = FSEv06_initDState(&DState, &DStream, DTablePtr);
|
||||
|
||||
You can then decode your data, symbol after symbol.
|
||||
For information the maximum number of bits read by FSEv06_decodeSymbol() is 'tableLog'.
|
||||
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
||||
unsigned char symbol = FSEv06_decodeSymbol(&DState, &DStream);
|
||||
|
||||
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
||||
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
||||
size_t bitField = BITv06_readBits(&DStream, nbBits);
|
||||
|
||||
All above operations only read from local register (which size depends on size_t).
|
||||
Refueling the register from memory is manually performed by the reload method.
|
||||
endSignal = FSEv06_reloadDStream(&DStream);
|
||||
|
||||
BITv06_reloadDStream() result tells if there is still some more data to read from DStream.
|
||||
BITv06_DStream_unfinished : there is still some data left into the DStream.
|
||||
BITv06_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
||||
BITv06_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
||||
BITv06_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
||||
|
||||
When reaching end of buffer (BITv06_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
||||
to properly detect the exact end of stream.
|
||||
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
||||
BITv06_reloadDStream(&DStream) >= BITv06_DStream_completed
|
||||
|
||||
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
||||
Checking if DStream has reached its end is performed by :
|
||||
BITv06_endOfDStream(&DStream);
|
||||
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
||||
FSEv06_endOfDState(&DState);
|
||||
*/
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* FSE unsafe API
|
||||
|
@ -511,16 +511,6 @@ MEM_STATIC BITv07_DStream_status BITv07_reloadDStream(BITv07_DStream_t* bitD);
|
||||
MEM_STATIC unsigned BITv07_endOfDStream(const BITv07_DStream_t* bitD);
|
||||
|
||||
|
||||
/* Start by invoking BITv07_initDStream().
|
||||
* A chunk of the bitStream is then stored into a local register.
|
||||
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
||||
* You can then retrieve bitFields stored into the local register, **in reverse order**.
|
||||
* Local register is explicitly reloaded from memory by the BITv07_reloadDStream() method.
|
||||
* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BITv07_DStream_unfinished.
|
||||
* Otherwise, it can be less than that, so proceed accordingly.
|
||||
* Checking if DStream has reached its end can be performed with BITv07_endOfDStream().
|
||||
*/
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* unsafe API
|
||||
@ -533,7 +523,7 @@ MEM_STATIC size_t BITv07_readBitsFast(BITv07_DStream_t* bitD, unsigned nbBits);
|
||||
/*-**************************************************************
|
||||
* Internal functions
|
||||
****************************************************************/
|
||||
MEM_STATIC unsigned BITv07_highbit32 (register U32 val)
|
||||
MEM_STATIC unsigned BITv07_highbit32 (U32 val)
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
unsigned long r=0;
|
||||
@ -599,13 +589,6 @@ MEM_STATIC size_t BITv07_initDStream(BITv07_DStream_t* bitD, const void* srcBuff
|
||||
}
|
||||
|
||||
|
||||
/*! BITv07_lookBits() :
|
||||
* Provides next n bits from local register.
|
||||
* local register is not modified.
|
||||
* On 32-bits, maxNbBits==24.
|
||||
* On 64-bits, maxNbBits==56.
|
||||
* @return : value extracted
|
||||
*/
|
||||
MEM_STATIC size_t BITv07_lookBits(const BITv07_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
||||
@ -625,11 +608,6 @@ MEM_STATIC void BITv07_skipBits(BITv07_DStream_t* bitD, U32 nbBits)
|
||||
bitD->bitsConsumed += nbBits;
|
||||
}
|
||||
|
||||
/*! BITv07_readBits() :
|
||||
* Read (consume) next n bits from local register and update.
|
||||
* Pay attention to not read more than nbBits contained into local register.
|
||||
* @return : extracted value.
|
||||
*/
|
||||
MEM_STATIC size_t BITv07_readBits(BITv07_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
size_t const value = BITv07_lookBits(bitD, nbBits);
|
||||
@ -646,11 +624,6 @@ MEM_STATIC size_t BITv07_readBitsFast(BITv07_DStream_t* bitD, U32 nbBits)
|
||||
return value;
|
||||
}
|
||||
|
||||
/*! BITv07_reloadDStream() :
|
||||
* Refill `BITv07_DStream_t` from src buffer previously defined (see BITv07_initDStream() ).
|
||||
* This function is safe, it guarantees it will not read beyond src buffer.
|
||||
* @return : status of `BITv07_DStream_t` internal register.
|
||||
if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
|
||||
MEM_STATIC BITv07_DStream_status BITv07_reloadDStream(BITv07_DStream_t* bitD)
|
||||
{
|
||||
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */
|
||||
@ -874,55 +847,6 @@ static void FSEv07_initDState(FSEv07_DState_t* DStatePtr, BITv07_DStream_t*
|
||||
static unsigned char FSEv07_decodeSymbol(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD);
|
||||
|
||||
|
||||
/**<
|
||||
Let's now decompose FSEv07_decompress_usingDTable() into its unitary components.
|
||||
You will decode FSE-encoded symbols from the bitStream,
|
||||
and also any other bitFields you put in, **in reverse order**.
|
||||
|
||||
You will need a few variables to track your bitStream. They are :
|
||||
|
||||
BITv07_DStream_t DStream; // Stream context
|
||||
FSEv07_DState_t DState; // State context. Multiple ones are possible
|
||||
FSEv07_DTable* DTablePtr; // Decoding table, provided by FSEv07_buildDTable()
|
||||
|
||||
The first thing to do is to init the bitStream.
|
||||
errorCode = BITv07_initDStream(&DStream, srcBuffer, srcSize);
|
||||
|
||||
You should then retrieve your initial state(s)
|
||||
(in reverse flushing order if you have several ones) :
|
||||
errorCode = FSEv07_initDState(&DState, &DStream, DTablePtr);
|
||||
|
||||
You can then decode your data, symbol after symbol.
|
||||
For information the maximum number of bits read by FSEv07_decodeSymbol() is 'tableLog'.
|
||||
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
||||
unsigned char symbol = FSEv07_decodeSymbol(&DState, &DStream);
|
||||
|
||||
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
||||
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
||||
size_t bitField = BITv07_readBits(&DStream, nbBits);
|
||||
|
||||
All above operations only read from local register (which size depends on size_t).
|
||||
Refueling the register from memory is manually performed by the reload method.
|
||||
endSignal = FSEv07_reloadDStream(&DStream);
|
||||
|
||||
BITv07_reloadDStream() result tells if there is still some more data to read from DStream.
|
||||
BITv07_DStream_unfinished : there is still some data left into the DStream.
|
||||
BITv07_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
||||
BITv07_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
||||
BITv07_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
||||
|
||||
When reaching end of buffer (BITv07_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
||||
to properly detect the exact end of stream.
|
||||
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
||||
BITv07_reloadDStream(&DStream) >= BITv07_DStream_completed
|
||||
|
||||
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
||||
Checking if DStream has reached its end is performed by :
|
||||
BITv07_endOfDStream(&DStream);
|
||||
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
||||
FSEv07_endOfDState(&DState);
|
||||
*/
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* FSE unsafe API
|
||||
|
@ -59,7 +59,7 @@ extern "C" {
|
||||
/*------ Version ------*/
|
||||
#define ZSTD_VERSION_MAJOR 1
|
||||
#define ZSTD_VERSION_MINOR 3
|
||||
#define ZSTD_VERSION_RELEASE 2
|
||||
#define ZSTD_VERSION_RELEASE 3
|
||||
|
||||
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
|
||||
ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< useful to check dll version */
|
||||
@ -131,7 +131,7 @@ ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t
|
||||
|
||||
|
||||
/*====== Helper functions ======*/
|
||||
#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < 128 KB) ? ((128 KB - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
|
||||
#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
|
||||
ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case scenario */
|
||||
ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
|
||||
ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
|
||||
@ -432,12 +432,12 @@ typedef struct {
|
||||
|
||||
typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params;
|
||||
|
||||
/*= Custom memory allocation functions */
|
||||
/*--- Custom memory allocation functions ---*/
|
||||
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
|
||||
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
|
||||
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
|
||||
/* use this constant to defer to stdlib's functions */
|
||||
static const ZSTD_customMem ZSTD_defaultCMem = { NULL, NULL, NULL };
|
||||
static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };
|
||||
|
||||
|
||||
/***************************************
|
||||
@ -446,7 +446,7 @@ static const ZSTD_customMem ZSTD_defaultCMem = { NULL, NULL, NULL };
|
||||
|
||||
/*! ZSTD_findFrameCompressedSize() :
|
||||
* `src` should point to the start of a ZSTD encoded frame or skippable frame
|
||||
* `srcSize` must be at least as large as the frame
|
||||
* `srcSize` must be >= first frame size
|
||||
* @return : the compressed size of the first frame starting at `src`,
|
||||
* suitable to pass to `ZSTD_decompress` or similar,
|
||||
* or an error code if input is invalid */
|
||||
@ -557,7 +557,8 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
|
||||
* It must outlive context usage.
|
||||
* workspaceSize: Use ZSTD_estimateCCtxSize() or ZSTD_estimateCStreamSize()
|
||||
* to determine how large workspace must be to support scenario.
|
||||
* @return : pointer to ZSTD_CCtx*, or NULL if error (size too small)
|
||||
* @return : pointer to ZSTD_CCtx* (same address as workspace, but different type),
|
||||
* or NULL if error (typically size too small)
|
||||
* Note : zstd will never resize nor malloc() when using a static cctx.
|
||||
* If it needs more memory than available, it will simply error out.
|
||||
* Note 2 : there is no corresponding "free" function.
|
||||
@ -587,7 +588,7 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS
|
||||
ZSTD_compressionParameters cParams,
|
||||
ZSTD_customMem customMem);
|
||||
|
||||
/*! ZSTD_initStaticCDict_advanced() :
|
||||
/*! ZSTD_initStaticCDict() :
|
||||
* Generate a digested dictionary in provided memory area.
|
||||
* workspace: The memory area to emplace the dictionary into.
|
||||
* Provided pointer must 8-bytes aligned.
|
||||
@ -596,7 +597,8 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS
|
||||
* to determine how large workspace must be.
|
||||
* cParams : use ZSTD_getCParams() to transform a compression level
|
||||
* into its relevants cParams.
|
||||
* @return : pointer to ZSTD_CDict*, or NULL if error (size too small)
|
||||
* @return : pointer to ZSTD_CDict* (same address as workspace, but different type),
|
||||
* or NULL if error (typically, size too small).
|
||||
* Note : there is no corresponding "free" function.
|
||||
* Since workspace was allocated externally, it must be freed externally.
|
||||
*/
|
||||
@ -613,7 +615,7 @@ ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, uns
|
||||
|
||||
/*! ZSTD_getParams() :
|
||||
* same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
|
||||
* All fields of `ZSTD_frameParameters` are set to default (0) */
|
||||
* All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */
|
||||
ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
|
||||
|
||||
/*! ZSTD_checkCParams() :
|
||||
@ -660,7 +662,8 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
|
||||
* It must outlive context usage.
|
||||
* workspaceSize: Use ZSTD_estimateDCtxSize() or ZSTD_estimateDStreamSize()
|
||||
* to determine how large workspace must be to support scenario.
|
||||
* @return : pointer to ZSTD_DCtx*, or NULL if error (size too small)
|
||||
* @return : pointer to ZSTD_DCtx* (same address as workspace, but different type),
|
||||
* or NULL if error (typically size too small)
|
||||
* Note : zstd will never resize nor malloc() when using a static dctx.
|
||||
* If it needs more memory than available, it will simply error out.
|
||||
* Note 2 : static dctx is incompatible with legacy support
|
||||
@ -731,20 +734,22 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
|
||||
/*===== Advanced Streaming compression functions =====*/
|
||||
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
|
||||
ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
|
||||
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters. pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. */
|
||||
|
||||
/*! ZSTD_resetCStream() :
|
||||
* start a new compression job, using same parameters from previous job.
|
||||
* This is typically useful to skip dictionary loading stage, since it will re-use it in-place..
|
||||
* Note that zcs must be init at least once before using ZSTD_resetCStream().
|
||||
* pledgedSrcSize==0 means "srcSize unknown".
|
||||
* If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
|
||||
* If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
|
||||
* @return : 0, or an error code (which can be tested using ZSTD_isError()) */
|
||||
* For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
|
||||
* but it may change to mean "empty" in some future version, so prefer using macro ZSTD_CONTENTSIZE_UNKNOWN.
|
||||
* @return : 0, or an error code (which can be tested using ZSTD_isError()) */
|
||||
ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
|
||||
|
||||
|
||||
@ -800,10 +805,10 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompress
|
||||
/*===== Buffer-less streaming compression functions =====*/
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize=0 means null-size */
|
||||
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
|
||||
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
|
||||
|
||||
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
@ -1000,18 +1005,19 @@ typedef enum {
|
||||
* Special: value 0 means "do not change strategy". */
|
||||
|
||||
/* frame parameters */
|
||||
ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1)
|
||||
* note that content size must be known at the beginning,
|
||||
* it is sent using ZSTD_CCtx_setPledgedSrcSize() */
|
||||
ZSTD_p_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
|
||||
* Content size must be known at the beginning of compression,
|
||||
* it is provided using ZSTD_CCtx_setPledgedSrcSize() */
|
||||
ZSTD_p_checksumFlag, /* A 32-bits checksum of content is written at end of frame (default:0) */
|
||||
ZSTD_p_dictIDFlag, /* When applicable, dictID of dictionary is provided in frame header (default:1) */
|
||||
ZSTD_p_dictIDFlag, /* When applicable, dictionary's ID is written into frame header (default:1) */
|
||||
|
||||
/* multi-threading parameters */
|
||||
ZSTD_p_nbThreads=400, /* Select how many threads a compression job can spawn (default:1)
|
||||
* More threads improve speed, but also increase memory usage.
|
||||
* Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
|
||||
* Special: value 0 means "do not change nbThreads" */
|
||||
ZSTD_p_jobSize, /* Size of a compression job. Each compression job is completed in parallel.
|
||||
ZSTD_p_jobSize, /* Size of a compression job. This value is only enforced in streaming (non-blocking) mode.
|
||||
* Each compression job is completed in parallel, so indirectly controls the nb of active threads.
|
||||
* 0 means default, which is dynamically determined based on compression parameters.
|
||||
* Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
|
||||
* The minimum size is automatically and transparently enforced */
|
||||
@ -1057,7 +1063,8 @@ typedef enum {
|
||||
/*! ZSTD_CCtx_setParameter() :
|
||||
* Set one compression parameter, selected by enum ZSTD_cParameter.
|
||||
* Note : when `value` is an enum, cast it to unsigned for proper type checking.
|
||||
* @result : 0, or an error code (which can be tested with ZSTD_isError()). */
|
||||
* @result : informational value (typically, the one being set, possibly corrected),
|
||||
* or an error code (which can be tested with ZSTD_isError()). */
|
||||
ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
|
||||
|
||||
/*! ZSTD_CCtx_setPledgedSrcSize() :
|
||||
@ -1066,7 +1073,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param
|
||||
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
|
||||
* Note 1 : 0 means zero, empty.
|
||||
* In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
|
||||
* Note that ZSTD_CONTENTSIZE_UNKNOWN is default value for new compression jobs.
|
||||
* ZSTD_CONTENTSIZE_UNKNOWN is default value for any new compression job.
|
||||
* Note 2 : If all data is provided and consumed in a single round,
|
||||
* this value is overriden by srcSize instead. */
|
||||
ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
|
||||
@ -1138,13 +1145,19 @@ typedef enum {
|
||||
* - Compression parameters cannot be changed once compression is started.
|
||||
* - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
|
||||
* - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
|
||||
* - @return provides the minimum amount of data still to flush from internal buffers
|
||||
* - In single-thread mode (default), function is blocking : it completed its job before returning to caller.
|
||||
* - In multi-thread mode, function is non-blocking : it just acquires a copy of input, and distribute job to internal worker threads,
|
||||
* and then immediately returns, just indicating that there is some data remaining to be flushed.
|
||||
* The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
|
||||
* - Exception : in multi-threading mode, if the first call requests a ZSTD_e_end directive, it is blocking : it will complete compression before giving back control to caller.
|
||||
* - @return provides the minimum amount of data remaining to be flushed from internal buffers
|
||||
* or an error code, which can be tested using ZSTD_isError().
|
||||
* if @return != 0, flush is not fully completed, there is some data left within internal buffers.
|
||||
* - after a ZSTD_e_end directive, if internal buffer is not fully flushed,
|
||||
* if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
|
||||
* This is useful to determine if a ZSTD_e_flush or ZSTD_e_end directive is completed.
|
||||
* - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
|
||||
* only ZSTD_e_end or ZSTD_e_flush operations are allowed.
|
||||
* It is necessary to fully flush internal buffers
|
||||
* before starting a new compression job, or changing compression parameters.
|
||||
* Before starting a new compression job, or changing compression parameters,
|
||||
* it is required to fully flush internal buffers.
|
||||
*/
|
||||
ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
|
||||
ZSTD_outBuffer* output,
|
||||
|
@ -10,38 +10,19 @@ cxx_binary(
|
||||
'//lib:mem',
|
||||
'//lib:xxhash',
|
||||
],
|
||||
)
|
||||
|
||||
cxx_binary(
|
||||
name='zstdmt',
|
||||
headers=glob(['*.h'], excludes=['datagen.h', 'platform.h', 'util.h']),
|
||||
srcs=glob(['*.c'], excludes=['datagen.c']),
|
||||
deps=[
|
||||
':datagen',
|
||||
':util',
|
||||
'//lib:zstd',
|
||||
'//lib:zdict',
|
||||
'//lib:mem',
|
||||
'//lib:xxhash',
|
||||
preprocessor_flags=[
|
||||
'-DZSTD_GZCOMPRESS',
|
||||
'-DZSTD_GZDECOMPRESS',
|
||||
'-DZSTD_LZMACOMPRESS',
|
||||
'-DZSTD_LZMADECOMPRES',
|
||||
'-DZSTD_LZ4COMPRESS',
|
||||
'-DZSTD_LZ4DECOMPRES',
|
||||
],
|
||||
preprocessor_flags=['-DZSTD_MULTITHREAD'],
|
||||
linker_flags=['-lpthread'],
|
||||
)
|
||||
|
||||
cxx_binary(
|
||||
name='gzstd',
|
||||
headers=glob(['*.h'], excludes=['datagen.h', 'platform.h', 'util.h']),
|
||||
srcs=glob(['*.c'], excludes=['datagen.c']),
|
||||
deps=[
|
||||
':datagen',
|
||||
':util',
|
||||
'//lib:zstd',
|
||||
'//lib:zdict',
|
||||
'//lib:mem',
|
||||
'//lib:xxhash',
|
||||
linker_flags=[
|
||||
'-lz',
|
||||
'-llzma',
|
||||
'-llz4',
|
||||
],
|
||||
preprocessor_flags=['-DZSTD_GZDECOMPRESS'],
|
||||
linker_flags=['-lz'],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
|
@ -37,8 +37,7 @@ endif
|
||||
|
||||
CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
|
||||
-I$(ZSTDDIR)/dictBuilder \
|
||||
-DZSTD_NEWAPI \
|
||||
-DXXH_NAMESPACE=ZSTD_ # because xxhash.o already compiled with this macro from library
|
||||
-DXXH_NAMESPACE=ZSTD_
|
||||
CFLAGS ?= -O3
|
||||
DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
||||
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
|
||||
@ -66,6 +65,7 @@ endif
|
||||
else
|
||||
endif
|
||||
|
||||
# Sort files in alphabetical order for reproducible builds
|
||||
ZSTDLIB_FILES := $(sort $(wildcard $(ZSTD_FILES)) $(wildcard $(ZSTDLEGACY_FILES)) $(wildcard $(ZDICT_FILES)))
|
||||
|
||||
# Define *.exe as extension for Windows systems
|
||||
@ -226,8 +226,8 @@ clean:
|
||||
MD2ROFF = ronn
|
||||
MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="zstd $(ZSTD_VERSION)"
|
||||
|
||||
zstd.1: zstd.1.md
|
||||
cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
|
||||
zstd.1: zstd.1.md ../lib/zstd.h
|
||||
cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
|
||||
|
||||
.PHONY: man
|
||||
man: zstd.1
|
||||
@ -263,9 +263,10 @@ mandir ?= $(datarootdir)/man
|
||||
man1dir ?= $(mandir)/man1
|
||||
|
||||
ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS))
|
||||
MANDIR ?= $(PREFIX)/man/man1
|
||||
MANDIR ?= $(PREFIX)/man
|
||||
MAN1DIR ?= $(MANDIR)/man1
|
||||
else
|
||||
MANDIR ?= $(man1dir)
|
||||
MAN1DIR ?= $(man1dir)
|
||||
endif
|
||||
|
||||
ifneq (,$(filter $(shell uname),SunOS))
|
||||
@ -282,7 +283,7 @@ INSTALL_MAN ?= $(INSTALL_DATA)
|
||||
.PHONY: install
|
||||
install: zstd
|
||||
@echo Installing binaries
|
||||
@$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
|
||||
@$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MAN1DIR)/
|
||||
@$(INSTALL_PROGRAM) zstd $(DESTDIR)$(BINDIR)/zstd
|
||||
@ln -sf zstd $(DESTDIR)$(BINDIR)/zstdcat
|
||||
@ln -sf zstd $(DESTDIR)$(BINDIR)/unzstd
|
||||
@ -290,9 +291,9 @@ install: zstd
|
||||
@$(INSTALL_SCRIPT) zstdless $(DESTDIR)$(BINDIR)/zstdless
|
||||
@$(INSTALL_SCRIPT) zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep
|
||||
@echo Installing man pages
|
||||
@$(INSTALL_MAN) zstd.1 $(DESTDIR)$(MANDIR)/zstd.1
|
||||
@ln -sf zstd.1 $(DESTDIR)$(MANDIR)/zstdcat.1
|
||||
@ln -sf zstd.1 $(DESTDIR)$(MANDIR)/unzstd.1
|
||||
@$(INSTALL_MAN) zstd.1 $(DESTDIR)$(MAN1DIR)/zstd.1
|
||||
@ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/zstdcat.1
|
||||
@ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/unzstd.1
|
||||
@echo zstd installation completed
|
||||
|
||||
.PHONY: uninstall
|
||||
@ -302,9 +303,9 @@ uninstall:
|
||||
@$(RM) $(DESTDIR)$(BINDIR)/zstdcat
|
||||
@$(RM) $(DESTDIR)$(BINDIR)/unzstd
|
||||
@$(RM) $(DESTDIR)$(BINDIR)/zstd
|
||||
@$(RM) $(DESTDIR)$(MANDIR)/zstdcat.1
|
||||
@$(RM) $(DESTDIR)$(MANDIR)/unzstd.1
|
||||
@$(RM) $(DESTDIR)$(MANDIR)/zstd.1
|
||||
@$(RM) $(DESTDIR)$(MAN1DIR)/zstdcat.1
|
||||
@$(RM) $(DESTDIR)$(MAN1DIR)/unzstd.1
|
||||
@$(RM) $(DESTDIR)$(MAN1DIR)/zstd.1
|
||||
@echo zstd programs successfully uninstalled
|
||||
|
||||
endif
|
||||
|
@ -34,14 +34,12 @@
|
||||
#include <stdlib.h> /* malloc, free */
|
||||
#include <string.h> /* memset */
|
||||
#include <stdio.h> /* fprintf, fopen */
|
||||
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
|
||||
|
||||
#include "mem.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "zstd.h"
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#include "xxhash.h"
|
||||
#include "zstdmt_compress.h"
|
||||
|
||||
|
||||
/* *************************************
|
||||
@ -73,12 +71,13 @@ static U32 g_compressibilityDefault = 50;
|
||||
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
|
||||
|
||||
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
|
||||
if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
|
||||
{ g_time = clock(); DISPLAY(__VA_ARGS__); \
|
||||
if (g_displayLevel>=4) fflush(stderr); } }
|
||||
static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
||||
static clock_t g_time = 0;
|
||||
static const U64 g_refreshRate = SEC_TO_MICRO / 6;
|
||||
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
|
||||
|
||||
#define DISPLAYUPDATE(l, ...) { if (g_displayLevel>=l) { \
|
||||
if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
|
||||
{ g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
|
||||
if (g_displayLevel>=4) fflush(stderr); } } }
|
||||
|
||||
|
||||
/* *************************************
|
||||
@ -130,6 +129,17 @@ void BMK_setNbThreads(unsigned nbThreads) {
|
||||
#endif
|
||||
g_nbThreads = nbThreads;
|
||||
}
|
||||
|
||||
static U32 g_realTime = 0;
|
||||
void BMK_setRealTime(unsigned priority) {
|
||||
g_realTime = (priority>0);
|
||||
}
|
||||
|
||||
static U32 g_separateFiles = 0;
|
||||
void BMK_setSeparateFiles(unsigned separate) {
|
||||
g_separateFiles = (separate>0);
|
||||
}
|
||||
|
||||
static U32 g_ldmFlag = 0;
|
||||
void BMK_setLdmFlag(unsigned ldmFlag) {
|
||||
g_ldmFlag = ldmFlag;
|
||||
@ -181,7 +191,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
const char* displayName, int cLevel,
|
||||
const size_t* fileSizes, U32 nbFiles,
|
||||
const void* dictBuffer, size_t dictBufferSize,
|
||||
const ZSTD_compressionParameters* comprParams)
|
||||
const ZSTD_compressionParameters* const comprParams)
|
||||
{
|
||||
size_t const blockSize = ((g_blockSize>=32 && !g_decodeOnly) ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
|
||||
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
|
||||
@ -189,7 +199,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
|
||||
void* const compressedBuffer = malloc(maxCompressedSize);
|
||||
void* resultBuffer = malloc(srcSize);
|
||||
ZSTDMT_CCtx* const mtctx = ZSTDMT_createCCtx(g_nbThreads);
|
||||
ZSTD_CCtx* const ctx = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
size_t const loadedCompressedSize = srcSize;
|
||||
@ -286,8 +295,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
if (!cCompleted) { /* still some time to do compression tests */
|
||||
U64 const clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
|
||||
U32 nbLoops = 0;
|
||||
ZSTD_CDict* cdict = NULL;
|
||||
#ifdef ZSTD_NEWAPI
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbThreads, g_nbThreads);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag);
|
||||
@ -306,84 +313,64 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy);
|
||||
ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
|
||||
#else
|
||||
size_t const avgSize = MIN(blockSize, (srcSize / nbFiles));
|
||||
ZSTD_parameters zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
|
||||
ZSTD_customMem const cmem = { NULL, NULL, NULL };
|
||||
if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog;
|
||||
if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog;
|
||||
if (comprParams->hashLog) zparams.cParams.hashLog = comprParams->hashLog;
|
||||
if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog;
|
||||
if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength;
|
||||
if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength;
|
||||
if (comprParams->strategy) zparams.cParams.strategy = comprParams->strategy;
|
||||
cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dm_auto, zparams.cParams, cmem);
|
||||
if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
|
||||
#endif
|
||||
do {
|
||||
U32 blockNb;
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
|
||||
size_t rSize;
|
||||
#ifdef ZSTD_NEWAPI
|
||||
ZSTD_outBuffer out = { blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, 0 };
|
||||
ZSTD_inBuffer in = { blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize, 0 };
|
||||
size_t cError = 1;
|
||||
while (cError) {
|
||||
cError = ZSTD_compress_generic(ctx,
|
||||
#if 0 /* direct compression function, for occasional comparison */
|
||||
ZSTD_parameters const params = ZSTD_getParams(cLevel, blockTable[blockNb].srcSize, dictBufferSize);
|
||||
blockTable[blockNb].cSize = ZSTD_compress_advanced(ctx,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
||||
blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize,
|
||||
dictBuffer, dictBufferSize,
|
||||
params);
|
||||
#else
|
||||
size_t moreToFlush = 1;
|
||||
ZSTD_outBuffer out;
|
||||
ZSTD_inBuffer in;
|
||||
in.src = blockTable[blockNb].srcPtr;
|
||||
in.size = blockTable[blockNb].srcSize;
|
||||
in.pos = 0;
|
||||
out.dst = blockTable[blockNb].cPtr;
|
||||
out.size = blockTable[blockNb].cRoom;
|
||||
out.pos = 0;
|
||||
while (moreToFlush) {
|
||||
moreToFlush = ZSTD_compress_generic(ctx,
|
||||
&out, &in, ZSTD_e_end);
|
||||
if (ZSTD_isError(cError))
|
||||
if (ZSTD_isError(moreToFlush))
|
||||
EXM_THROW(1, "ZSTD_compress_generic() error : %s",
|
||||
ZSTD_getErrorName(cError));
|
||||
ZSTD_getErrorName(moreToFlush));
|
||||
}
|
||||
rSize = out.pos;
|
||||
#else /* ! ZSTD_NEWAPI */
|
||||
if (dictBufferSize) {
|
||||
rSize = ZSTD_compress_usingCDict(ctx,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
||||
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
|
||||
cdict);
|
||||
} else {
|
||||
# ifdef ZSTD_MULTITHREAD /* note : limitation : MT single-pass does not support compression with dictionary */
|
||||
rSize = ZSTDMT_compressCCtx(mtctx,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
||||
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
|
||||
cLevel);
|
||||
# else
|
||||
rSize = ZSTD_compress_advanced (ctx,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
||||
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
|
||||
NULL, 0, zparams);
|
||||
# endif
|
||||
}
|
||||
if (ZSTD_isError(rSize))
|
||||
EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s",
|
||||
ZSTD_getErrorName(rSize));
|
||||
#endif /* ZSTD_NEWAPI */
|
||||
blockTable[blockNb].cSize = rSize;
|
||||
blockTable[blockNb].cSize = out.pos;
|
||||
#endif
|
||||
}
|
||||
nbLoops++;
|
||||
} while (UTIL_clockSpanMicro(clockStart) < clockLoop);
|
||||
ZSTD_freeCDict(cdict);
|
||||
{ U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart);
|
||||
if (clockSpanMicro < fastestC*nbLoops) fastestC = clockSpanMicro / nbLoops;
|
||||
totalCTime += clockSpanMicro;
|
||||
cCompleted = (totalCTime >= maxTime);
|
||||
{ U64 const loopDuration = UTIL_clockSpanMicro(clockStart);
|
||||
if (loopDuration < fastestC*nbLoops)
|
||||
fastestC = loopDuration / nbLoops;
|
||||
totalCTime += loopDuration;
|
||||
cCompleted = (totalCTime >= maxTime); /* end compression tests */
|
||||
} }
|
||||
|
||||
cSize = 0;
|
||||
{ U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
|
||||
ratio = (double)srcSize / (double)cSize;
|
||||
markNb = (markNb+1) % NB_MARKS;
|
||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
|
||||
marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
|
||||
(double)srcSize / fastestC );
|
||||
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
|
||||
double const compressionSpeed = (double)srcSize / fastestC;
|
||||
int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
|
||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
|
||||
marks[markNb], displayName, (U32)srcSize, (U32)cSize,
|
||||
ratioAccuracy, ratio,
|
||||
cSpeedAccuracy, compressionSpeed );
|
||||
}
|
||||
} else { /* g_decodeOnly */
|
||||
memcpy(compressedBuffer, srcBuffer, loadedCompressedSize);
|
||||
}
|
||||
|
||||
#if 0 /* disable decompression test */
|
||||
dCompleted=1;
|
||||
(void)totalDTime; (void)fastestD; (void)crcOrig; /* unused when decompression disabled */
|
||||
(void)totalDTime; (void)fastestD; (void)crcOrig; /* unused when decompression disabled */
|
||||
#else
|
||||
/* Decompression */
|
||||
if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */
|
||||
@ -413,17 +400,24 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
nbLoops++;
|
||||
} while (UTIL_clockSpanMicro(clockStart) < clockLoop);
|
||||
ZSTD_freeDDict(ddict);
|
||||
{ U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart);
|
||||
if (clockSpanMicro < fastestD*nbLoops) fastestD = clockSpanMicro / nbLoops;
|
||||
totalDTime += clockSpanMicro;
|
||||
{ U64 const loopDuration = UTIL_clockSpanMicro(clockStart);
|
||||
if (loopDuration < fastestD*nbLoops)
|
||||
fastestD = loopDuration / nbLoops;
|
||||
totalDTime += loopDuration;
|
||||
dCompleted = (totalDTime >= maxTime);
|
||||
} }
|
||||
|
||||
markNb = (markNb+1) % NB_MARKS;
|
||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
|
||||
marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
|
||||
(double)srcSize / fastestC,
|
||||
(double)srcSize / fastestD );
|
||||
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
|
||||
double const compressionSpeed = (double)srcSize / fastestC;
|
||||
int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
|
||||
double const decompressionSpeed = (double)srcSize / fastestD;
|
||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
|
||||
marks[markNb], displayName, (U32)srcSize, (U32)cSize,
|
||||
ratioAccuracy, ratio,
|
||||
cSpeedAccuracy, compressionSpeed,
|
||||
decompressionSpeed);
|
||||
}
|
||||
|
||||
/* CRC Checking */
|
||||
{ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
|
||||
@ -444,10 +438,12 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
DISPLAY("(sample %u, block %u, pos %u) \n", segNb, bNb, pos);
|
||||
if (u>5) {
|
||||
int n;
|
||||
DISPLAY("origin: ");
|
||||
for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
|
||||
DISPLAY(" :%02X: ", ((const BYTE*)srcBuffer)[u]);
|
||||
for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
|
||||
DISPLAY(" \n");
|
||||
DISPLAY("decode: ");
|
||||
for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)resultBuffer)[u+n]);
|
||||
DISPLAY(" :%02X: ", ((const BYTE*)resultBuffer)[u]);
|
||||
for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)resultBuffer)[u+n]);
|
||||
@ -463,7 +459,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
#endif
|
||||
} /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */
|
||||
|
||||
if (g_displayLevel == 1) {
|
||||
if (g_displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */
|
||||
double cSpeed = (double)srcSize / fastestC;
|
||||
double dSpeed = (double)srcSize / fastestD;
|
||||
if (g_additionalParam)
|
||||
@ -478,7 +474,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
free(blockTable);
|
||||
free(compressedBuffer);
|
||||
free(resultBuffer);
|
||||
ZSTDMT_freeCCtx(mtctx);
|
||||
ZSTD_freeCCtx(ctx);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
return 0;
|
||||
@ -503,11 +498,11 @@ static size_t BMK_findMaxMem(U64 requiredMem)
|
||||
return (size_t)(requiredMem);
|
||||
}
|
||||
|
||||
static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
|
||||
static void BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
|
||||
const char* displayName, int cLevel, int cLevelLast,
|
||||
const size_t* fileSizes, unsigned nbFiles,
|
||||
const void* dictBuffer, size_t dictBufferSize,
|
||||
ZSTD_compressionParameters *compressionParams, int setRealTimePrio)
|
||||
const ZSTD_compressionParameters* const compressionParams)
|
||||
{
|
||||
int l;
|
||||
|
||||
@ -515,8 +510,8 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
|
||||
if (!pch) pch = strrchr(displayName, '/'); /* Linux */
|
||||
if (pch) displayName = pch+1;
|
||||
|
||||
if (setRealTimePrio) {
|
||||
DISPLAYLEVEL(2, "Note : switching to a real-time priority \n");
|
||||
if (g_realTime) {
|
||||
DISPLAYLEVEL(2, "Note : switching to real-time priority \n");
|
||||
SET_REALTIME_PRIORITY;
|
||||
}
|
||||
|
||||
@ -539,7 +534,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
|
||||
At most, fills `buffer` entirely */
|
||||
static void BMK_loadFiles(void* buffer, size_t bufferSize,
|
||||
size_t* fileSizes,
|
||||
const char** fileNamesTable, unsigned nbFiles)
|
||||
const char* const * const fileNamesTable, unsigned nbFiles)
|
||||
{
|
||||
size_t pos = 0, totalSize = 0;
|
||||
unsigned n;
|
||||
@ -551,6 +546,11 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
|
||||
fileSizes[n] = 0;
|
||||
continue;
|
||||
}
|
||||
if (fileSize == UTIL_FILESIZE_UNKNOWN) {
|
||||
DISPLAYLEVEL(2, "Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]);
|
||||
fileSizes[n] = 0;
|
||||
continue;
|
||||
}
|
||||
f = fopen(fileNamesTable[n], "rb");
|
||||
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
|
||||
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
|
||||
@ -566,26 +566,30 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
|
||||
if (totalSize == 0) EXM_THROW(12, "no data to bench");
|
||||
}
|
||||
|
||||
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel,
|
||||
int cLevelLast, ZSTD_compressionParameters *compressionParams, int setRealTimePrio)
|
||||
static void BMK_benchFileTable(const char* const * const fileNamesTable, unsigned const nbFiles,
|
||||
const char* const dictFileName,
|
||||
int const cLevel, int const cLevelLast,
|
||||
const ZSTD_compressionParameters* const compressionParams)
|
||||
{
|
||||
void* srcBuffer;
|
||||
size_t benchedSize;
|
||||
void* dictBuffer = NULL;
|
||||
size_t dictBufferSize = 0;
|
||||
size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
|
||||
size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
|
||||
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
|
||||
char mfName[20] = {0};
|
||||
|
||||
if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
|
||||
|
||||
/* Load dictionary */
|
||||
if (dictFileName != NULL) {
|
||||
U64 dictFileSize = UTIL_getFileSize(dictFileName);
|
||||
if (dictFileSize > 64 MB) EXM_THROW(10, "dictionary file %s too large", dictFileName);
|
||||
U64 const dictFileSize = UTIL_getFileSize(dictFileName);
|
||||
if (dictFileSize > 64 MB)
|
||||
EXM_THROW(10, "dictionary file %s too large", dictFileName);
|
||||
dictBufferSize = (size_t)dictFileSize;
|
||||
dictBuffer = malloc(dictBufferSize);
|
||||
if (dictBuffer==NULL) EXM_THROW(11, "not enough memory for dictionary (%u bytes)", (U32)dictBufferSize);
|
||||
if (dictBuffer==NULL)
|
||||
EXM_THROW(11, "not enough memory for dictionary (%u bytes)",
|
||||
(U32)dictBufferSize);
|
||||
BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1);
|
||||
}
|
||||
|
||||
@ -601,13 +605,26 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co
|
||||
BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
|
||||
|
||||
/* Bench */
|
||||
snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
|
||||
{ const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
|
||||
BMK_benchCLevel(srcBuffer, benchedSize,
|
||||
displayName, cLevel, cLevelLast,
|
||||
fileSizes, nbFiles,
|
||||
dictBuffer, dictBufferSize, compressionParams, setRealTimePrio);
|
||||
}
|
||||
if (g_separateFiles) {
|
||||
const BYTE* srcPtr = (const BYTE*)srcBuffer;
|
||||
U32 fileNb;
|
||||
for (fileNb=0; fileNb<nbFiles; fileNb++) {
|
||||
size_t const fileSize = fileSizes[fileNb];
|
||||
BMK_benchCLevel(srcPtr, fileSize,
|
||||
fileNamesTable[fileNb], cLevel, cLevelLast,
|
||||
fileSizes+fileNb, 1,
|
||||
dictBuffer, dictBufferSize, compressionParams);
|
||||
srcPtr += fileSize;
|
||||
}
|
||||
} else {
|
||||
char mfName[20] = {0};
|
||||
snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
|
||||
{ const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
|
||||
BMK_benchCLevel(srcBuffer, benchedSize,
|
||||
displayName, cLevel, cLevelLast,
|
||||
fileSizes, nbFiles,
|
||||
dictBuffer, dictBufferSize, compressionParams);
|
||||
} }
|
||||
|
||||
/* clean up */
|
||||
free(srcBuffer);
|
||||
@ -616,7 +633,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co
|
||||
}
|
||||
|
||||
|
||||
static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams, int setRealTimePrio)
|
||||
static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, const ZSTD_compressionParameters* compressionParams)
|
||||
{
|
||||
char name[20] = {0};
|
||||
size_t benchedSize = 10000000;
|
||||
@ -630,15 +647,17 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
|
||||
|
||||
/* Bench */
|
||||
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
|
||||
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams, setRealTimePrio);
|
||||
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams);
|
||||
|
||||
/* clean up */
|
||||
free(srcBuffer);
|
||||
}
|
||||
|
||||
|
||||
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
|
||||
int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio)
|
||||
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
|
||||
const char* dictFileName,
|
||||
int cLevel, int cLevelLast,
|
||||
const ZSTD_compressionParameters* compressionParams)
|
||||
{
|
||||
double const compressibility = (double)g_compressibilityDefault / 100;
|
||||
|
||||
@ -646,11 +665,12 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* di
|
||||
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
|
||||
if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
|
||||
if (cLevelLast < cLevel) cLevelLast = cLevel;
|
||||
if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
|
||||
if (cLevelLast > cLevel)
|
||||
DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
|
||||
|
||||
if (nbFiles == 0)
|
||||
BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams, setRealTimePrio);
|
||||
BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams);
|
||||
else
|
||||
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, setRealTimePrio);
|
||||
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams);
|
||||
return 0;
|
||||
}
|
||||
|
@ -16,14 +16,16 @@
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
|
||||
#include "zstd.h" /* ZSTD_compressionParameters */
|
||||
|
||||
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,const char* dictFileName,
|
||||
int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio);
|
||||
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
|
||||
int cLevel, int cLevelLast, const ZSTD_compressionParameters* compressionParams);
|
||||
|
||||
/* Set Parameters */
|
||||
void BMK_setNbSeconds(unsigned nbLoops);
|
||||
void BMK_setBlockSize(size_t blockSize);
|
||||
void BMK_setNbThreads(unsigned nbThreads);
|
||||
void BMK_setRealTime(unsigned priority);
|
||||
void BMK_setNotificationLevel(unsigned level);
|
||||
void BMK_setSeparateFiles(unsigned separate);
|
||||
void BMK_setAdditionalParam(int additionalParam);
|
||||
void BMK_setDecodeOnlyMode(unsigned decodeFlag);
|
||||
void BMK_setLdmFlag(unsigned ldmFlag);
|
||||
|
@ -26,7 +26,6 @@
|
||||
#include <stdlib.h> /* malloc, free */
|
||||
#include <string.h> /* memset */
|
||||
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
||||
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
|
||||
#include <errno.h> /* errno */
|
||||
|
||||
#include "mem.h" /* read */
|
||||
@ -55,15 +54,13 @@ static const size_t g_maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((siz
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
|
||||
#define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \
|
||||
if ((DIB_clockSpan(g_time) > refreshRate) || (displayLevel>=4)) \
|
||||
{ g_time = clock(); DISPLAY(__VA_ARGS__); \
|
||||
if (displayLevel>=4) fflush(stderr); } }
|
||||
static const clock_t refreshRate = CLOCKS_PER_SEC * 2 / 10;
|
||||
static clock_t g_time = 0;
|
||||
|
||||
static clock_t DIB_clockSpan(clock_t nPrevious) { return clock() - nPrevious; }
|
||||
static const U64 g_refreshRate = SEC_TO_MICRO / 6;
|
||||
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
|
||||
|
||||
#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \
|
||||
if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \
|
||||
{ g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
|
||||
if (displayLevel>=4) fflush(stderr); } } }
|
||||
|
||||
/*-*************************************
|
||||
* Exceptions
|
||||
@ -117,7 +114,7 @@ static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr,
|
||||
for (fileIndex=0; fileIndex<nbFiles; fileIndex++) {
|
||||
const char* const fileName = fileNamesTable[fileIndex];
|
||||
unsigned long long const fs64 = UTIL_getFileSize(fileName);
|
||||
unsigned long long remainingToLoad = fs64;
|
||||
unsigned long long remainingToLoad = (fs64 == UTIL_FILESIZE_UNKNOWN) ? 0 : fs64;
|
||||
U32 const nbChunks = targetChunkSize ? (U32)((fs64 + (targetChunkSize-1)) / targetChunkSize) : 1;
|
||||
U64 const chunkSize = targetChunkSize ? MIN(targetChunkSize, fs64) : fs64;
|
||||
size_t const maxChunkSize = (size_t)MIN(chunkSize, SAMPLESIZE_MAX);
|
||||
@ -245,8 +242,9 @@ static fileStats DiB_fileStats(const char** fileNamesTable, unsigned nbFiles, si
|
||||
memset(&fs, 0, sizeof(fs));
|
||||
for (n=0; n<nbFiles; n++) {
|
||||
U64 const fileSize = UTIL_getFileSize(fileNamesTable[n]);
|
||||
U32 const nbSamples = (U32)(chunkSize ? (fileSize + (chunkSize-1)) / chunkSize : 1);
|
||||
U64 const chunkToLoad = chunkSize ? MIN(chunkSize, fileSize) : fileSize;
|
||||
U64 const srcSize = (fileSize == UTIL_FILESIZE_UNKNOWN) ? 0 : fileSize;
|
||||
U32 const nbSamples = (U32)(chunkSize ? (srcSize + (chunkSize-1)) / chunkSize : 1);
|
||||
U64 const chunkToLoad = chunkSize ? MIN(chunkSize, srcSize) : srcSize;
|
||||
size_t const cappedChunkSize = (size_t)MIN(chunkToLoad, SAMPLESIZE_MAX);
|
||||
fs.totalSizeToLoad += cappedChunkSize * nbSamples;
|
||||
fs.oneSampleTooLarge |= (chunkSize > 2*SAMPLESIZE_MAX);
|
||||
|
@ -25,11 +25,10 @@
|
||||
* Includes
|
||||
***************************************/
|
||||
#include "platform.h" /* Large Files support, SET_BINARY_MODE */
|
||||
#include "util.h" /* UTIL_getFileSize */
|
||||
#include "util.h" /* UTIL_getFileSize, UTIL_isRegularFile */
|
||||
#include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */
|
||||
#include <stdlib.h> /* malloc, free */
|
||||
#include <string.h> /* strcmp, strlen */
|
||||
#include <time.h> /* clock */
|
||||
#include <errno.h> /* errno */
|
||||
|
||||
#if defined (_MSC_VER)
|
||||
@ -40,11 +39,9 @@
|
||||
#include "bitstream.h"
|
||||
#include "mem.h"
|
||||
#include "fileio.h"
|
||||
#include "util.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
|
||||
#include "zstd.h"
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
# include "zstdmt_compress.h"
|
||||
#endif
|
||||
#if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS)
|
||||
# include <zlib.h>
|
||||
# if !defined(z_const)
|
||||
@ -70,18 +67,6 @@
|
||||
#define MB *(1<<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
#define _1BIT 0x01
|
||||
#define _2BITS 0x03
|
||||
#define _3BITS 0x07
|
||||
#define _4BITS 0x0F
|
||||
#define _6BITS 0x3F
|
||||
#define _8BITS 0xFF
|
||||
|
||||
#define BLOCKSIZE (128 KB)
|
||||
#define ROLLBUFFERSIZE (BLOCKSIZE*8*64)
|
||||
|
||||
#define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */
|
||||
|
||||
#define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */
|
||||
|
||||
#define FNSPACE 30
|
||||
@ -96,12 +81,13 @@
|
||||
static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2: + result + interaction + warnings; 3: + progression; 4: + information */
|
||||
void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
|
||||
|
||||
static const U64 g_refreshRate = SEC_TO_MICRO / 6;
|
||||
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
|
||||
|
||||
#define DISPLAYUPDATE(l, ...) { if (g_displayLevel>=l) { \
|
||||
if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
|
||||
{ g_time = clock(); DISPLAY(__VA_ARGS__); \
|
||||
if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
|
||||
{ g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
|
||||
if (g_displayLevel>=4) fflush(stderr); } } }
|
||||
static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
||||
static clock_t g_time = 0;
|
||||
|
||||
#undef MIN /* in case it would be already defined */
|
||||
#define MIN(a,b) ((a) < (b) ? (a) : (b))
|
||||
@ -122,22 +108,23 @@ static clock_t g_time = 0;
|
||||
# define ZSTD_DEBUG 0
|
||||
#endif
|
||||
#define DEBUGLOG(l,...) if (l<=ZSTD_DEBUG) DISPLAY(__VA_ARGS__);
|
||||
#define EXM_THROW(error, ...) \
|
||||
{ \
|
||||
DISPLAYLEVEL(1, "zstd: "); \
|
||||
DEBUGLOG(1, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \
|
||||
DISPLAYLEVEL(1, "error %i : ", error); \
|
||||
DISPLAYLEVEL(1, __VA_ARGS__); \
|
||||
DISPLAYLEVEL(1, " \n"); \
|
||||
exit(error); \
|
||||
#define EXM_THROW(error, ...) \
|
||||
{ \
|
||||
DISPLAYLEVEL(1, "zstd: "); \
|
||||
DEBUGLOG(1, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \
|
||||
DISPLAYLEVEL(1, "error %i : ", error); \
|
||||
DISPLAYLEVEL(1, __VA_ARGS__); \
|
||||
DISPLAYLEVEL(1, " \n"); \
|
||||
exit(error); \
|
||||
}
|
||||
|
||||
#define CHECK(f) { \
|
||||
size_t const err = f; \
|
||||
if (ZSTD_isError(err)) { \
|
||||
#define CHECK_V(v, f) \
|
||||
v = f; \
|
||||
if (ZSTD_isError(v)) { \
|
||||
DEBUGLOG(1, "%s \n", #f); \
|
||||
EXM_THROW(11, "%s", ZSTD_getErrorName(err)); \
|
||||
} }
|
||||
EXM_THROW(11, "%s", ZSTD_getErrorName(v)); \
|
||||
}
|
||||
#define CHECK(f) { size_t err; CHECK_V(err, f); }
|
||||
|
||||
|
||||
/*-************************************
|
||||
@ -156,6 +143,21 @@ static void INThandler(int sig)
|
||||
DISPLAY("\n");
|
||||
exit(2);
|
||||
}
|
||||
static void addHandler(char const* dstFileName)
|
||||
{
|
||||
if (UTIL_isRegularFile(dstFileName)) {
|
||||
g_artefact = dstFileName;
|
||||
signal(SIGINT, INThandler);
|
||||
} else {
|
||||
g_artefact = NULL;
|
||||
}
|
||||
}
|
||||
/* Idempotent */
|
||||
static void clearHandler(void)
|
||||
{
|
||||
if (g_artefact) signal(SIGINT, SIG_DFL);
|
||||
g_artefact = NULL;
|
||||
}
|
||||
|
||||
|
||||
/* ************************************************************
|
||||
@ -218,10 +220,6 @@ static U32 g_blockSize = 0;
|
||||
void FIO_setBlockSize(unsigned blockSize) {
|
||||
if (blockSize && g_nbThreads==1)
|
||||
DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n");
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
if (blockSize-1 < ZSTDMT_SECTION_SIZE_MIN-1) /* intentional underflow */
|
||||
DISPLAYLEVEL(2, "Note : minimum block size is %u KB \n", (ZSTDMT_SECTION_SIZE_MIN>>10));
|
||||
#endif
|
||||
g_blockSize = blockSize;
|
||||
}
|
||||
#define FIO_OVERLAP_LOG_NOTSET 9999
|
||||
@ -264,6 +262,10 @@ void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog) {
|
||||
* @result : Unlink `fileName`, even if it's read-only */
|
||||
static int FIO_remove(const char* path)
|
||||
{
|
||||
if (!UTIL_isRegularFile(path)) {
|
||||
DISPLAYLEVEL(2, "zstd: Refusing to remove non-regular file %s\n", path);
|
||||
return 0;
|
||||
}
|
||||
#if defined(_WIN32) || defined(WIN32)
|
||||
/* windows doesn't allow remove read-only files,
|
||||
* so try to make it writable first */
|
||||
@ -273,86 +275,92 @@ static int FIO_remove(const char* path)
|
||||
}
|
||||
|
||||
/** FIO_openSrcFile() :
|
||||
* condition : `dstFileName` must be non-NULL.
|
||||
* @result : FILE* to `dstFileName`, or NULL if it fails */
|
||||
* condition : `srcFileName` must be non-NULL.
|
||||
* @result : FILE* to `srcFileName`, or NULL if it fails */
|
||||
static FILE* FIO_openSrcFile(const char* srcFileName)
|
||||
{
|
||||
FILE* f;
|
||||
|
||||
assert(srcFileName != NULL);
|
||||
if (!strcmp (srcFileName, stdinmark)) {
|
||||
DISPLAYLEVEL(4,"Using stdin for input\n");
|
||||
f = stdin;
|
||||
SET_BINARY_MODE(stdin);
|
||||
} else {
|
||||
if (!UTIL_isRegularFile(srcFileName)) {
|
||||
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
|
||||
srcFileName);
|
||||
return NULL;
|
||||
}
|
||||
f = fopen(srcFileName, "rb");
|
||||
if ( f==NULL )
|
||||
DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
|
||||
return stdin;
|
||||
}
|
||||
|
||||
return f;
|
||||
if (!UTIL_isRegularFile(srcFileName)) {
|
||||
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
|
||||
srcFileName);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
{ FILE* const f = fopen(srcFileName, "rb");
|
||||
if (f == NULL)
|
||||
DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
|
||||
return f;
|
||||
}
|
||||
}
|
||||
|
||||
/** FIO_openDstFile() :
|
||||
* condition : `dstFileName` must be non-NULL.
|
||||
* condition : `dstFileName` must be non-NULL.
|
||||
* @result : FILE* to `dstFileName`, or NULL if it fails */
|
||||
static FILE* FIO_openDstFile(const char* dstFileName)
|
||||
{
|
||||
FILE* f;
|
||||
|
||||
assert(dstFileName != NULL);
|
||||
if (!strcmp (dstFileName, stdoutmark)) {
|
||||
DISPLAYLEVEL(4,"Using stdout for output\n");
|
||||
f = stdout;
|
||||
SET_BINARY_MODE(stdout);
|
||||
if (g_sparseFileSupport==1) {
|
||||
g_sparseFileSupport = 0;
|
||||
DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
|
||||
}
|
||||
} else {
|
||||
if (g_sparseFileSupport == 1) {
|
||||
g_sparseFileSupport = ZSTD_SPARSE_DEFAULT;
|
||||
}
|
||||
if (strcmp (dstFileName, nulmark)) {
|
||||
/* Check if destination file already exists */
|
||||
f = fopen( dstFileName, "rb" );
|
||||
if (f != 0) { /* dst file exists, prompt for overwrite authorization */
|
||||
fclose(f);
|
||||
if (!g_overwrite) {
|
||||
if (g_displayLevel <= 1) {
|
||||
/* No interaction possible */
|
||||
DISPLAY("zstd: %s already exists; not overwritten \n",
|
||||
dstFileName);
|
||||
return NULL;
|
||||
}
|
||||
DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ",
|
||||
dstFileName);
|
||||
{ int ch = getchar();
|
||||
if ((ch!='Y') && (ch!='y')) {
|
||||
DISPLAY(" not overwritten \n");
|
||||
return NULL;
|
||||
}
|
||||
/* flush rest of input line */
|
||||
while ((ch!=EOF) && (ch!='\n')) ch = getchar();
|
||||
} }
|
||||
/* need to unlink */
|
||||
FIO_remove(dstFileName);
|
||||
} }
|
||||
f = fopen( dstFileName, "wb" );
|
||||
if (f==NULL) DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
|
||||
return stdout;
|
||||
}
|
||||
|
||||
return f;
|
||||
if (g_sparseFileSupport == 1) {
|
||||
g_sparseFileSupport = ZSTD_SPARSE_DEFAULT;
|
||||
}
|
||||
|
||||
if (UTIL_isRegularFile(dstFileName)) {
|
||||
FILE* fCheck;
|
||||
if (!strcmp(dstFileName, nulmark)) {
|
||||
EXM_THROW(40, "%s is unexpectedly a regular file", dstFileName);
|
||||
}
|
||||
/* Check if destination file already exists */
|
||||
fCheck = fopen( dstFileName, "rb" );
|
||||
if (fCheck != NULL) { /* dst file exists, authorization prompt */
|
||||
fclose(fCheck);
|
||||
if (!g_overwrite) {
|
||||
if (g_displayLevel <= 1) {
|
||||
/* No interaction possible */
|
||||
DISPLAY("zstd: %s already exists; not overwritten \n",
|
||||
dstFileName);
|
||||
return NULL;
|
||||
}
|
||||
DISPLAY("zstd: %s already exists; overwrite (y/N) ? ",
|
||||
dstFileName);
|
||||
{ int ch = getchar();
|
||||
if ((ch!='Y') && (ch!='y')) {
|
||||
DISPLAY(" not overwritten \n");
|
||||
return NULL;
|
||||
}
|
||||
/* flush rest of input line */
|
||||
while ((ch!=EOF) && (ch!='\n')) ch = getchar();
|
||||
} }
|
||||
/* need to unlink */
|
||||
FIO_remove(dstFileName);
|
||||
} }
|
||||
|
||||
{ FILE* const f = fopen( dstFileName, "wb" );
|
||||
if (f == NULL)
|
||||
DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
|
||||
return f;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*! FIO_createDictBuffer() :
|
||||
* creates a buffer, pointed by `*bufferPtr`,
|
||||
* loads `filename` content into it, up to DICTSIZE_MAX bytes.
|
||||
* @return : loaded size
|
||||
* @return : loaded size
|
||||
* if fileName==NULL, returns 0 and a NULL pointer
|
||||
*/
|
||||
static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
|
||||
@ -360,12 +368,13 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
|
||||
FILE* fileHandle;
|
||||
U64 fileSize;
|
||||
|
||||
assert(bufferPtr != NULL);
|
||||
*bufferPtr = NULL;
|
||||
if (fileName == NULL) return 0;
|
||||
|
||||
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
|
||||
fileHandle = fopen(fileName, "rb");
|
||||
if (fileHandle==0) EXM_THROW(31, "%s: %s", fileName, strerror(errno));
|
||||
if (fileHandle==NULL) EXM_THROW(31, "%s: %s", fileName, strerror(errno));
|
||||
fileSize = UTIL_getFileSize(fileName);
|
||||
if (fileSize > DICTSIZE_MAX) {
|
||||
EXM_THROW(32, "Dictionary file %s is too large (> %u MB)",
|
||||
@ -393,11 +402,7 @@ typedef struct {
|
||||
size_t srcBufferSize;
|
||||
void* dstBuffer;
|
||||
size_t dstBufferSize;
|
||||
#if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD)
|
||||
ZSTDMT_CCtx* cctx;
|
||||
#else
|
||||
ZSTD_CStream* cctx;
|
||||
#endif
|
||||
} cRess_t;
|
||||
|
||||
static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
|
||||
@ -406,24 +411,9 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
|
||||
cRess_t ress;
|
||||
memset(&ress, 0, sizeof(ress));
|
||||
|
||||
#ifdef ZSTD_NEWAPI
|
||||
ress.cctx = ZSTD_createCCtx();
|
||||
if (ress.cctx == NULL)
|
||||
EXM_THROW(30, "allocation error : can't create ZSTD_CCtx");
|
||||
#elif defined(ZSTD_MULTITHREAD)
|
||||
ress.cctx = ZSTDMT_createCCtx(g_nbThreads);
|
||||
if (ress.cctx == NULL)
|
||||
EXM_THROW(30, "allocation error : can't create ZSTDMT_CCtx");
|
||||
if ((cLevel==ZSTD_maxCLevel()) && (g_overlapLog==FIO_OVERLAP_LOG_NOTSET))
|
||||
/* use complete window for overlap */
|
||||
ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, 9);
|
||||
if (g_overlapLog != FIO_OVERLAP_LOG_NOTSET)
|
||||
ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, g_overlapLog);
|
||||
#else
|
||||
ress.cctx = ZSTD_createCStream();
|
||||
if (ress.cctx == NULL)
|
||||
EXM_THROW(30, "allocation error : can't create ZSTD_CStream");
|
||||
#endif
|
||||
ress.srcBufferSize = ZSTD_CStreamInSize();
|
||||
ress.srcBuffer = malloc(ress.srcBufferSize);
|
||||
ress.dstBufferSize = ZSTD_CStreamOutSize();
|
||||
@ -431,72 +421,44 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
|
||||
if (!ress.srcBuffer || !ress.dstBuffer)
|
||||
EXM_THROW(31, "allocation error : not enough memory");
|
||||
|
||||
/* dictionary */
|
||||
/* Advances parameters, including dictionary */
|
||||
{ void* dictBuffer;
|
||||
size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName); /* works with dictFileName==NULL */
|
||||
if (dictFileName && (dictBuffer==NULL))
|
||||
EXM_THROW(32, "allocation error : can't create dictBuffer");
|
||||
|
||||
#ifdef ZSTD_NEWAPI
|
||||
{ /* frame parameters */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
|
||||
(void)srcSize;
|
||||
/* compression level */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) );
|
||||
/* long distance matching */
|
||||
CHECK( ZSTD_CCtx_setParameter(
|
||||
ress.cctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashLog, g_ldmHashLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch) );
|
||||
if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) {
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmBucketSizeLog, g_ldmBucketSizeLog) );
|
||||
}
|
||||
if (g_ldmHashEveryLog != FIO_LDM_PARAM_NOTSET) {
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog) );
|
||||
}
|
||||
/* compression parameters */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams->windowLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams->chainLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams->hashLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams->searchLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams->searchLength) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams->targetLength) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams->strategy) );
|
||||
/* multi-threading */
|
||||
DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads);
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
|
||||
/* dictionary */
|
||||
CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) ); /* always enable content size when available (note: supposed to be default) */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
|
||||
/* compression level */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) );
|
||||
/* long distance matching */
|
||||
CHECK( ZSTD_CCtx_setParameter(
|
||||
ress.cctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashLog, g_ldmHashLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch) );
|
||||
if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) {
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmBucketSizeLog, g_ldmBucketSizeLog) );
|
||||
}
|
||||
#elif defined(ZSTD_MULTITHREAD)
|
||||
{ ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
|
||||
params.fParams.checksumFlag = g_checksumFlag;
|
||||
params.fParams.noDictIDFlag = !g_dictIDFlag;
|
||||
if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
|
||||
if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog;
|
||||
if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog;
|
||||
if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog;
|
||||
if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
|
||||
if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
|
||||
if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy;
|
||||
CHECK( ZSTDMT_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) );
|
||||
ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_sectionSize, g_blockSize);
|
||||
if (g_ldmHashEveryLog != FIO_LDM_PARAM_NOTSET) {
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog) );
|
||||
}
|
||||
#else
|
||||
{ ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
|
||||
params.fParams.checksumFlag = g_checksumFlag;
|
||||
params.fParams.noDictIDFlag = !g_dictIDFlag;
|
||||
if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
|
||||
if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog;
|
||||
if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog;
|
||||
if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog;
|
||||
if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
|
||||
if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
|
||||
if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy;
|
||||
CHECK( ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) );
|
||||
}
|
||||
#endif
|
||||
/* compression parameters */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams->windowLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams->chainLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams->hashLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams->searchLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams->searchLength) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams->targetLength) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams->strategy) );
|
||||
/* multi-threading */
|
||||
DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads);
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
|
||||
/* dictionary */
|
||||
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just for dictionary loading, for compression parameters adaptation */
|
||||
CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
|
||||
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */
|
||||
|
||||
free(dictBuffer);
|
||||
}
|
||||
|
||||
@ -507,11 +469,7 @@ static void FIO_freeCResources(cRess_t ress)
|
||||
{
|
||||
free(ress.srcBuffer);
|
||||
free(ress.dstBuffer);
|
||||
#if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD)
|
||||
ZSTDMT_freeCCtx(ress.cctx);
|
||||
#else
|
||||
ZSTD_freeCStream(ress.cctx); /* never fails */
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@ -562,7 +520,7 @@ static unsigned long long FIO_compressGzFrame(cRess_t* ress,
|
||||
strm.avail_out = (uInt)ress->dstBufferSize;
|
||||
}
|
||||
}
|
||||
if (!srcFileSize)
|
||||
if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
|
||||
(U32)(inFileSize>>20),
|
||||
(double)outFileSize/inFileSize*100)
|
||||
@ -649,7 +607,7 @@ static unsigned long long FIO_compressLzmaFrame(cRess_t* ress,
|
||||
strm.next_out = (BYTE*)ress->dstBuffer;
|
||||
strm.avail_out = ress->dstBufferSize;
|
||||
} }
|
||||
if (!srcFileSize)
|
||||
if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
|
||||
(U32)(inFileSize>>20),
|
||||
(double)outFileSize/inFileSize*100)
|
||||
@ -668,11 +626,16 @@ static unsigned long long FIO_compressLzmaFrame(cRess_t* ress,
|
||||
#endif
|
||||
|
||||
#ifdef ZSTD_LZ4COMPRESS
|
||||
#if LZ4_VERSION_NUMBER <= 10600
|
||||
#define LZ4F_blockLinked blockLinked
|
||||
#define LZ4F_max64KB max64KB
|
||||
#endif
|
||||
static int FIO_LZ4_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }
|
||||
static unsigned long long FIO_compressLz4Frame(cRess_t* ress,
|
||||
const char* srcFileName, U64 const srcFileSize,
|
||||
int compressionLevel, U64* readsize)
|
||||
{
|
||||
const size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max64KB);
|
||||
unsigned long long inFileSize = 0, outFileSize = 0;
|
||||
|
||||
LZ4F_preferences_t prefs;
|
||||
@ -684,29 +647,26 @@ static unsigned long long FIO_compressLz4Frame(cRess_t* ress,
|
||||
|
||||
memset(&prefs, 0, sizeof(prefs));
|
||||
|
||||
#if LZ4_VERSION_NUMBER <= 10600
|
||||
#define LZ4F_blockIndependent blockIndependent
|
||||
#define LZ4F_max4MB max4MB
|
||||
#endif
|
||||
assert(blockSize <= ress->srcBufferSize);
|
||||
|
||||
prefs.autoFlush = 1;
|
||||
prefs.compressionLevel = compressionLevel;
|
||||
prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* stick to defaults for lz4 cli */
|
||||
prefs.frameInfo.blockSizeID = LZ4F_max4MB;
|
||||
prefs.frameInfo.blockMode = LZ4F_blockLinked;
|
||||
prefs.frameInfo.blockSizeID = LZ4F_max64KB;
|
||||
prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_checksumFlag;
|
||||
#if LZ4_VERSION_NUMBER >= 10600
|
||||
prefs.frameInfo.contentSize = srcFileSize;
|
||||
prefs.frameInfo.contentSize = (srcFileSize==UTIL_FILESIZE_UNKNOWN) ? 0 : srcFileSize;
|
||||
#endif
|
||||
assert(LZ4F_compressBound(blockSize, &prefs) <= ress->dstBufferSize);
|
||||
|
||||
{
|
||||
size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max4MB);
|
||||
size_t readSize;
|
||||
size_t headerSize = LZ4F_compressBegin(ctx, ress->dstBuffer, ress->dstBufferSize, &prefs);
|
||||
if (LZ4F_isError(headerSize))
|
||||
EXM_THROW(33, "File header generation failed : %s",
|
||||
LZ4F_getErrorName(headerSize));
|
||||
{ size_t const sizeCheck = fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile);
|
||||
if (sizeCheck!=headerSize) EXM_THROW(34, "Write error : cannot write header"); }
|
||||
if (fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile) != headerSize)
|
||||
EXM_THROW(34, "Write error : cannot write header");
|
||||
outFileSize += headerSize;
|
||||
|
||||
/* Read first block */
|
||||
@ -723,7 +683,7 @@ static unsigned long long FIO_compressLz4Frame(cRess_t* ress,
|
||||
EXM_THROW(35, "zstd: %s: lz4 compression failed : %s",
|
||||
srcFileName, LZ4F_getErrorName(outSize));
|
||||
outFileSize += outSize;
|
||||
if (!srcFileSize)
|
||||
if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
|
||||
(U32)(inFileSize>>20),
|
||||
(double)outFileSize/inFileSize*100)
|
||||
@ -774,6 +734,7 @@ static int FIO_compressFilename_internal(cRess_t ress,
|
||||
U64 readsize = 0;
|
||||
U64 compressedfilesize = 0;
|
||||
U64 const fileSize = UTIL_getFileSize(srcFileName);
|
||||
ZSTD_EndDirective directive = ZSTD_e_continue;
|
||||
DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (U32)fileSize);
|
||||
|
||||
switch (g_compressionType) {
|
||||
@ -813,83 +774,52 @@ static int FIO_compressFilename_internal(cRess_t ress,
|
||||
}
|
||||
|
||||
/* init */
|
||||
#ifdef ZSTD_NEWAPI
|
||||
if (fileSize!=0) /* when src is stdin, fileSize==0, but is effectively unknown */
|
||||
ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize); /* note : fileSize==0 means "empty" */
|
||||
#elif defined(ZSTD_MULTITHREAD)
|
||||
CHECK( ZSTDMT_resetCStream(ress.cctx, fileSize) ); /* note : fileSize==0 means "unknown" */
|
||||
#else
|
||||
CHECK( ZSTD_resetCStream(ress.cctx, fileSize) ); /* note : fileSize==0 means "unknown" */
|
||||
#endif
|
||||
if (fileSize != UTIL_FILESIZE_UNKNOWN)
|
||||
ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize);
|
||||
|
||||
/* Main compression loop */
|
||||
while (1) {
|
||||
do {
|
||||
size_t result;
|
||||
/* Fill input Buffer */
|
||||
size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile);
|
||||
ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 };
|
||||
if (inSize==0) break;
|
||||
readsize += inSize;
|
||||
|
||||
while (inBuff.pos != inBuff.size) {
|
||||
if (inSize == 0 || (fileSize != UTIL_FILESIZE_UNKNOWN && readsize == fileSize))
|
||||
directive = ZSTD_e_end;
|
||||
|
||||
result = 1;
|
||||
while (inBuff.pos != inBuff.size || (directive == ZSTD_e_end && result != 0)) {
|
||||
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
|
||||
#ifdef ZSTD_NEWAPI
|
||||
CHECK( ZSTD_compress_generic(ress.cctx,
|
||||
&outBuff, &inBuff, ZSTD_e_continue) );
|
||||
#elif defined(ZSTD_MULTITHREAD)
|
||||
CHECK( ZSTDMT_compressStream(ress.cctx, &outBuff, &inBuff) );
|
||||
#else
|
||||
CHECK( ZSTD_compressStream(ress.cctx, &outBuff, &inBuff) );
|
||||
#endif
|
||||
CHECK_V(result, ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, directive));
|
||||
|
||||
/* Write compressed stream */
|
||||
DISPLAYLEVEL(6, "ZSTD_compress_generic,ZSTD_e_continue: generated %u bytes \n",
|
||||
(U32)outBuff.pos);
|
||||
if (outBuff.pos) {
|
||||
size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
|
||||
if (sizeCheck!=outBuff.pos)
|
||||
EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName);
|
||||
compressedfilesize += outBuff.pos;
|
||||
} }
|
||||
}
|
||||
}
|
||||
if (g_nbThreads > 1) {
|
||||
if (!fileSize)
|
||||
if (fileSize == UTIL_FILESIZE_UNKNOWN)
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB", (U32)(readsize>>20))
|
||||
else
|
||||
DISPLAYUPDATE(2, "\rRead : %u / %u MB",
|
||||
(U32)(readsize>>20), (U32)(fileSize>>20));
|
||||
} else {
|
||||
if (!fileSize)
|
||||
if (fileSize == UTIL_FILESIZE_UNKNOWN)
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
|
||||
(U32)(readsize>>20),
|
||||
(double)compressedfilesize/readsize*100)
|
||||
else
|
||||
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
|
||||
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
|
||||
(U32)(readsize>>20), (U32)(fileSize>>20),
|
||||
(double)compressedfilesize/readsize*100);
|
||||
}
|
||||
}
|
||||
|
||||
/* End of Frame */
|
||||
{ size_t result = 1;
|
||||
while (result != 0) {
|
||||
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
|
||||
#ifdef ZSTD_NEWAPI
|
||||
ZSTD_inBuffer inBuff = { NULL, 0, 0 };
|
||||
result = ZSTD_compress_generic(ress.cctx,
|
||||
&outBuff, &inBuff, ZSTD_e_end);
|
||||
#elif defined(ZSTD_MULTITHREAD)
|
||||
result = ZSTDMT_endStream(ress.cctx, &outBuff);
|
||||
#else
|
||||
result = ZSTD_endStream(ress.cctx, &outBuff);
|
||||
#endif
|
||||
if (ZSTD_isError(result)) {
|
||||
EXM_THROW(26, "Compression error during frame end : %s",
|
||||
ZSTD_getErrorName(result));
|
||||
}
|
||||
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
|
||||
if (sizeCheck!=outBuff.pos)
|
||||
EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName);
|
||||
}
|
||||
compressedfilesize += outBuff.pos;
|
||||
}
|
||||
}
|
||||
} while (directive != ZSTD_e_end);
|
||||
|
||||
finish:
|
||||
/* Status */
|
||||
@ -927,6 +857,10 @@ static int FIO_compressFilename_srcFile(cRess_t ress,
|
||||
|
||||
fclose(ress.srcFile);
|
||||
if (g_removeSrcFile /* --rm */ && !result && strcmp(srcFileName, stdinmark)) {
|
||||
/* We must clear the handler, since after this point calling it would
|
||||
* delete both the source and destination files.
|
||||
*/
|
||||
clearHandler();
|
||||
if (remove(srcFileName))
|
||||
EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno));
|
||||
}
|
||||
@ -949,18 +883,16 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
|
||||
|
||||
ress.dstFile = FIO_openDstFile(dstFileName);
|
||||
if (ress.dstFile==NULL) return 1; /* could not open dstFileName */
|
||||
|
||||
if (UTIL_isRegularFile(dstFileName)) {
|
||||
g_artefact = dstFileName;
|
||||
signal(SIGINT, INThandler);
|
||||
} else {
|
||||
g_artefact = NULL;
|
||||
}
|
||||
|
||||
/* Must ony be added after FIO_openDstFile() succeeds.
|
||||
* Otherwise we may delete the destination file if at already exists, and
|
||||
* the user presses Ctrl-C when asked if they wish to overwrite.
|
||||
*/
|
||||
addHandler(dstFileName);
|
||||
|
||||
if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf))
|
||||
stat_result = 1;
|
||||
result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, compressionLevel);
|
||||
clearHandler();
|
||||
|
||||
if (fclose(ress.dstFile)) { /* error closing dstFile */
|
||||
DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
|
||||
@ -973,8 +905,6 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
|
||||
else if (strcmp (dstFileName, stdoutmark) && stat_result)
|
||||
UTIL_setFileStat(dstFileName, &statbuf);
|
||||
|
||||
signal(SIGINT, SIG_DFL);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -983,7 +913,8 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
|
||||
const char* dictFileName, int compressionLevel, ZSTD_compressionParameters* comprParams)
|
||||
{
|
||||
clock_t const start = clock();
|
||||
U64 const srcSize = UTIL_getFileSize(srcFileName);
|
||||
U64 const fileSize = UTIL_getFileSize(srcFileName);
|
||||
U64 const srcSize = (fileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize;
|
||||
|
||||
cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams);
|
||||
int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel);
|
||||
@ -997,7 +928,7 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
|
||||
|
||||
|
||||
int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles,
|
||||
const char* suffix,
|
||||
const char* outFileName, const char* suffix,
|
||||
const char* dictFileName, int compressionLevel,
|
||||
ZSTD_compressionParameters* comprParams)
|
||||
{
|
||||
@ -1005,22 +936,23 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
|
||||
size_t dfnSize = FNSPACE;
|
||||
char* dstFileName = (char*)malloc(FNSPACE);
|
||||
size_t const suffixSize = suffix ? strlen(suffix) : 0;
|
||||
U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ;
|
||||
U64 const firstFileSize = UTIL_getFileSize(inFileNamesTable[0]);
|
||||
U64 const firstSrcSize = (firstFileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : firstFileSize;
|
||||
U64 const srcSize = (nbFiles != 1) ? ZSTD_CONTENTSIZE_UNKNOWN : firstSrcSize ;
|
||||
cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams);
|
||||
|
||||
/* init */
|
||||
if (dstFileName==NULL)
|
||||
EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName");
|
||||
if (suffix == NULL)
|
||||
if (outFileName == NULL && suffix == NULL)
|
||||
EXM_THROW(28, "FIO_compressMultipleFilenames : dst unknown"); /* should never happen */
|
||||
|
||||
/* loop on each file */
|
||||
if (!strcmp(suffix, stdoutmark)) {
|
||||
if (outFileName != NULL) {
|
||||
unsigned u;
|
||||
ress.dstFile = stdout;
|
||||
SET_BINARY_MODE(stdout);
|
||||
ress.dstFile = FIO_openDstFile(outFileName);
|
||||
for (u=0; u<nbFiles; u++)
|
||||
missed_files += FIO_compressFilename_srcFile(ress, stdoutmark, inFileNamesTable[u], compressionLevel);
|
||||
missed_files += FIO_compressFilename_srcFile(ress, outFileName, inFileNamesTable[u], compressionLevel);
|
||||
if (fclose(ress.dstFile))
|
||||
EXM_THROW(29, "Write error : cannot properly close stdout");
|
||||
} else {
|
||||
@ -1031,9 +963,9 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
|
||||
free(dstFileName);
|
||||
dfnSize = ifnSize + 20;
|
||||
dstFileName = (char*)malloc(dfnSize);
|
||||
if (!dstFileName)
|
||||
if (!dstFileName) {
|
||||
EXM_THROW(30, "zstd: %s", strerror(errno));
|
||||
}
|
||||
} }
|
||||
strcpy(dstFileName, inFileNamesTable[u]);
|
||||
strcat(dstFileName, suffix);
|
||||
missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u], compressionLevel);
|
||||
@ -1213,7 +1145,7 @@ static void FIO_zstdErrorHelp(dRess_t* ress, size_t ret, char const* srcFileName
|
||||
if (ret == 0) {
|
||||
U32 const windowSize = (U32)header.windowSize;
|
||||
U32 const windowLog = BIT_highbit32(windowSize) + ((windowSize & (windowSize - 1)) != 0);
|
||||
U32 const windowMB = (windowSize >> 20) + (windowSize & ((1 MB) - 1));
|
||||
U32 const windowMB = (windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0);
|
||||
assert(header.windowSize <= (U64)((U32)-1));
|
||||
assert(g_memLimit > 0);
|
||||
DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u\n",
|
||||
@ -1635,6 +1567,10 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
|
||||
if ( g_removeSrcFile /* --rm */
|
||||
&& (result==0) /* decompression successful */
|
||||
&& strcmp(srcFileName, stdinmark) ) /* not stdin */ {
|
||||
/* We must clear the handler, since after this point calling it would
|
||||
* delete both the source and destination files.
|
||||
*/
|
||||
clearHandler();
|
||||
if (remove(srcFileName)) {
|
||||
/* failed to remove src file */
|
||||
DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
|
||||
@ -1658,18 +1594,17 @@ static int FIO_decompressDstFile(dRess_t ress,
|
||||
|
||||
ress.dstFile = FIO_openDstFile(dstFileName);
|
||||
if (ress.dstFile==0) return 1;
|
||||
|
||||
if (UTIL_isRegularFile(dstFileName)) {
|
||||
g_artefact = dstFileName;
|
||||
signal(SIGINT, INThandler);
|
||||
} else {
|
||||
g_artefact = NULL;
|
||||
}
|
||||
/* Must ony be added after FIO_openDstFile() succeeds.
|
||||
* Otherwise we may delete the destination file if at already exists, and
|
||||
* the user presses Ctrl-C when asked if they wish to overwrite.
|
||||
*/
|
||||
addHandler(dstFileName);
|
||||
|
||||
if ( strcmp(srcFileName, stdinmark)
|
||||
&& UTIL_getFileStat(srcFileName, &statbuf) )
|
||||
stat_result = 1;
|
||||
result = FIO_decompressSrcFile(ress, dstFileName, srcFileName);
|
||||
clearHandler();
|
||||
|
||||
if (fclose(ress.dstFile)) {
|
||||
DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
|
||||
@ -1707,24 +1642,21 @@ int FIO_decompressFilename(const char* dstFileName, const char* srcFileName,
|
||||
|
||||
#define MAXSUFFIXSIZE 8
|
||||
int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
|
||||
const char* suffix,
|
||||
const char* outFileName,
|
||||
const char* dictFileName)
|
||||
{
|
||||
int skippedFiles = 0;
|
||||
int missingFiles = 0;
|
||||
dRess_t ress = FIO_createDResources(dictFileName);
|
||||
|
||||
if (suffix==NULL)
|
||||
EXM_THROW(70, "zstd: decompression: unknown dst"); /* should never happen */
|
||||
|
||||
if (!strcmp(suffix, stdoutmark) || !strcmp(suffix, nulmark)) { /* special cases : -c or -t */
|
||||
if (outFileName) {
|
||||
unsigned u;
|
||||
ress.dstFile = FIO_openDstFile(suffix);
|
||||
if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", suffix);
|
||||
ress.dstFile = FIO_openDstFile(outFileName);
|
||||
if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName);
|
||||
for (u=0; u<nbFiles; u++)
|
||||
missingFiles += FIO_decompressSrcFile(ress, suffix, srcNamesTable[u]);
|
||||
missingFiles += FIO_decompressSrcFile(ress, outFileName, srcNamesTable[u]);
|
||||
if (fclose(ress.dstFile))
|
||||
EXM_THROW(72, "Write error : cannot properly close stdout");
|
||||
EXM_THROW(72, "Write error : cannot properly close output file");
|
||||
} else {
|
||||
size_t suffixSize;
|
||||
size_t dfnSize = FNSPACE;
|
||||
@ -1797,7 +1729,7 @@ typedef struct {
|
||||
* 2 for file not compressed with zstd
|
||||
* 3 for cases in which file could not be opened.
|
||||
*/
|
||||
static int getFileInfo(fileInfo_t* info, const char* inFileName){
|
||||
static int getFileInfo_fileConfirmed(fileInfo_t* info, const char* inFileName){
|
||||
int detectError = 0;
|
||||
FILE* const srcFile = FIO_openSrcFile(inFileName);
|
||||
if (srcFile == NULL) {
|
||||
@ -1813,7 +1745,8 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
|
||||
if (numBytesRead < ZSTD_frameHeaderSize_min) {
|
||||
if ( feof(srcFile)
|
||||
&& (numBytesRead == 0)
|
||||
&& (info->compressedSize > 0) ) {
|
||||
&& (info->compressedSize > 0)
|
||||
&& (info->compressedSize != UTIL_FILESIZE_UNKNOWN) ) {
|
||||
break;
|
||||
}
|
||||
else if (feof(srcFile)) {
|
||||
@ -1926,7 +1859,18 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
|
||||
return detectError;
|
||||
}
|
||||
|
||||
static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){
|
||||
static int getFileInfo(fileInfo_t* info, const char* srcFileName)
|
||||
{
|
||||
int const isAFile = UTIL_isRegularFile(srcFileName);
|
||||
if (!isAFile) {
|
||||
DISPLAY("Error : %s is not a file", srcFileName);
|
||||
return 3;
|
||||
}
|
||||
return getFileInfo_fileConfirmed(info, srcFileName);
|
||||
}
|
||||
|
||||
|
||||
static void displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel){
|
||||
unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB);
|
||||
const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB";
|
||||
double const windowSizeUnit = (double)info->windowSize / unit;
|
||||
@ -1949,8 +1893,10 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev
|
||||
checkString, inFileName);
|
||||
}
|
||||
} else {
|
||||
DISPLAYOUT("%s \n", inFileName);
|
||||
DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
|
||||
DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
|
||||
if (info->numSkippableFrames)
|
||||
DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
|
||||
DISPLAYOUT("Window Size: %.2f %2s (%llu B)\n",
|
||||
windowSizeUnit, unitStr,
|
||||
(unsigned long long)info->windowSize);
|
||||
@ -1971,7 +1917,6 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev
|
||||
static fileInfo_t FIO_addFInfo(fileInfo_t fi1, fileInfo_t fi2)
|
||||
{
|
||||
fileInfo_t total;
|
||||
memset(&total, 0, sizeof(total));
|
||||
total.numActualFrames = fi1.numActualFrames + fi2.numActualFrames;
|
||||
total.numSkippableFrames = fi1.numSkippableFrames + fi2.numSkippableFrames;
|
||||
total.compressedSize = fi1.compressedSize + fi2.compressedSize;
|
||||
@ -1983,7 +1928,6 @@ static fileInfo_t FIO_addFInfo(fileInfo_t fi1, fileInfo_t fi2)
|
||||
}
|
||||
|
||||
static int FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel){
|
||||
/* initialize info to avoid warnings */
|
||||
fileInfo_t info;
|
||||
memset(&info, 0, sizeof(info));
|
||||
{ int const error = getFileInfo(&info, inFileName);
|
||||
@ -2023,7 +1967,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
|
||||
for (u=0; u<numFiles;u++) {
|
||||
error |= FIO_listFile(&total, filenameTable[u], displayLevel);
|
||||
}
|
||||
if (numFiles > 1 && displayLevel <= 2) {
|
||||
if (numFiles > 1 && displayLevel <= 2) { /* display total */
|
||||
unsigned const unit = total.compressedSize < (1 MB) ? (1 KB) : (1 MB);
|
||||
const char* const unitStr = total.compressedSize < (1 MB) ? "KB" : "MB";
|
||||
double const compressedSizeUnit = (double)total.compressedSize / unit;
|
||||
@ -2043,8 +1987,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
|
||||
total.numSkippableFrames,
|
||||
compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
|
||||
ratio, checkString, total.nbFiles);
|
||||
}
|
||||
}
|
||||
} }
|
||||
return error;
|
||||
}
|
||||
}
|
||||
|
@ -84,14 +84,14 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
|
||||
/** FIO_compressMultipleFilenames() :
|
||||
@return : nb of missing files */
|
||||
int FIO_compressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
|
||||
const char* suffix,
|
||||
const char* outFileName, const char* suffix,
|
||||
const char* dictFileName, int compressionLevel,
|
||||
ZSTD_compressionParameters* comprParams);
|
||||
|
||||
/** FIO_decompressMultipleFilenames() :
|
||||
@return : nb of missing or skipped files */
|
||||
int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
|
||||
const char* suffix,
|
||||
const char* outFileName,
|
||||
const char* dictFileName);
|
||||
|
||||
|
||||
|
@ -21,10 +21,10 @@ extern "C" {
|
||||
* Compiler Options
|
||||
****************************************/
|
||||
#if defined(_MSC_VER)
|
||||
# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */
|
||||
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before <io.h> and <windows.h> */
|
||||
# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */
|
||||
# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */
|
||||
# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */
|
||||
# if (_MSC_VER <= 1800) /* 1800 == Visual Studio 2013 */
|
||||
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before <io.h> and <windows.h> */
|
||||
# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */
|
||||
# endif
|
||||
#endif
|
||||
|
||||
@ -117,7 +117,7 @@ static __inline int IS_CONSOLE(FILE* stdStream)
|
||||
|
||||
|
||||
/******************************
|
||||
* OS-specific Includes
|
||||
* OS-specific IO behaviors
|
||||
******************************/
|
||||
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32)
|
||||
# include <fcntl.h> /* _O_BINARY */
|
||||
@ -125,7 +125,7 @@ static __inline int IS_CONSOLE(FILE* stdStream)
|
||||
# if !defined(__DJGPP__)
|
||||
# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
|
||||
# include <winioctl.h> /* FSCTL_SET_SPARSE */
|
||||
# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; }
|
||||
# define SET_BINARY_MODE(file) { int const unused=_setmode(_fileno(file), _O_BINARY); (void)unused; }
|
||||
# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); }
|
||||
# else
|
||||
# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY)
|
||||
|
@ -34,7 +34,7 @@ extern "C" {
|
||||
# include <unistd.h> /* chown, stat */
|
||||
# include <utime.h> /* utime */
|
||||
#endif
|
||||
#include <time.h> /* time */
|
||||
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */
|
||||
#include <errno.h>
|
||||
#include "mem.h" /* U32, U64 */
|
||||
|
||||
@ -64,7 +64,6 @@ extern "C" {
|
||||
#elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */
|
||||
# include <unistd.h>
|
||||
# include <sys/resource.h> /* setpriority */
|
||||
# include <time.h> /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */
|
||||
# if defined(PRIO_PROCESS)
|
||||
# define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
|
||||
# else
|
||||
@ -118,6 +117,7 @@ static int g_utilDisplayLevel;
|
||||
* Time functions
|
||||
******************************************/
|
||||
#if defined(_WIN32) /* Windows */
|
||||
#define UTIL_TIME_INITIALIZER { { 0, 0 } }
|
||||
typedef LARGE_INTEGER UTIL_time_t;
|
||||
UTIL_STATIC UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
|
||||
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
|
||||
@ -144,6 +144,7 @@ static int g_utilDisplayLevel;
|
||||
}
|
||||
#elif defined(__APPLE__) && defined(__MACH__)
|
||||
#include <mach/mach_time.h>
|
||||
#define UTIL_TIME_INITIALIZER 0
|
||||
typedef U64 UTIL_time_t;
|
||||
UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
|
||||
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
|
||||
@ -166,8 +167,8 @@ static int g_utilDisplayLevel;
|
||||
}
|
||||
return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom);
|
||||
}
|
||||
#elif (PLATFORM_POSIX_VERSION >= 200112L)
|
||||
#include <time.h>
|
||||
#elif (PLATFORM_POSIX_VERSION >= 200112L) && (defined __UCLIBC__ || ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) || __GLIBC__ > 2))
|
||||
#define UTIL_TIME_INITIALIZER { 0, 0 }
|
||||
typedef struct timespec UTIL_freq_t;
|
||||
typedef struct timespec UTIL_time_t;
|
||||
UTIL_STATIC UTIL_time_t UTIL_getTime(void)
|
||||
@ -207,11 +208,13 @@ static int g_utilDisplayLevel;
|
||||
}
|
||||
#else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
|
||||
typedef clock_t UTIL_time_t;
|
||||
#define UTIL_TIME_INITIALIZER 0
|
||||
UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return clock(); }
|
||||
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
|
||||
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
|
||||
#endif
|
||||
|
||||
#define SEC_TO_MICRO 1000000
|
||||
|
||||
/* returns time span in microseconds */
|
||||
UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart )
|
||||
@ -313,33 +316,40 @@ UTIL_STATIC U32 UTIL_isLink(const char* infilename)
|
||||
}
|
||||
|
||||
|
||||
#define UTIL_FILESIZE_UNKNOWN ((U64)(-1))
|
||||
UTIL_STATIC U64 UTIL_getFileSize(const char* infilename)
|
||||
{
|
||||
int r;
|
||||
if (!UTIL_isRegularFile(infilename)) return UTIL_FILESIZE_UNKNOWN;
|
||||
{ int r;
|
||||
#if defined(_MSC_VER)
|
||||
struct __stat64 statbuf;
|
||||
r = _stat64(infilename, &statbuf);
|
||||
if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
|
||||
struct __stat64 statbuf;
|
||||
r = _stat64(infilename, &statbuf);
|
||||
if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
|
||||
#elif defined(__MINGW32__) && defined (__MSVCRT__)
|
||||
struct _stati64 statbuf;
|
||||
r = _stati64(infilename, &statbuf);
|
||||
if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
|
||||
struct _stati64 statbuf;
|
||||
r = _stati64(infilename, &statbuf);
|
||||
if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
|
||||
#else
|
||||
struct stat statbuf;
|
||||
r = stat(infilename, &statbuf);
|
||||
if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
|
||||
struct stat statbuf;
|
||||
r = stat(infilename, &statbuf);
|
||||
if (r || !S_ISREG(statbuf.st_mode)) return UTIL_FILESIZE_UNKNOWN;
|
||||
#endif
|
||||
return (U64)statbuf.st_size;
|
||||
return (U64)statbuf.st_size;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles)
|
||||
UTIL_STATIC U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles)
|
||||
{
|
||||
U64 total = 0;
|
||||
int error = 0;
|
||||
unsigned n;
|
||||
for (n=0; n<nbFiles; n++)
|
||||
total += UTIL_getFileSize(fileNamesTable[n]);
|
||||
return total;
|
||||
for (n=0; n<nbFiles; n++) {
|
||||
U64 const size = UTIL_getFileSize(fileNamesTable[n]);
|
||||
error |= (size == UTIL_FILESIZE_UNKNOWN);
|
||||
total += size;
|
||||
}
|
||||
return error ? UTIL_FILESIZE_UNKNOWN : total;
|
||||
}
|
||||
|
||||
|
||||
|
@ -1,5 +1,5 @@
|
||||
.
|
||||
.TH "ZSTD" "1" "September 2017" "zstd 1.3.1" "User Commands"
|
||||
.TH "ZSTD" "1" "December 2017" "zstd 1.3.3" "User Commands"
|
||||
.
|
||||
.SH "NAME"
|
||||
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
|
||||
@ -60,11 +60,11 @@ In most places where an integer argument is expected, an optional suffix is supp
|
||||
.
|
||||
.TP
|
||||
\fBKiB\fR
|
||||
Multiply the integer by 1,024 (2\e \fBKi\fR, \fBK\fR, and \fBKB\fR are accepted as synonyms for \fBKiB\fR\.
|
||||
Multiply the integer by 1,024 (2^10)\. \fBKi\fR, \fBK\fR, and \fBKB\fR are accepted as synonyms for \fBKiB\fR\.
|
||||
.
|
||||
.TP
|
||||
\fBMiB\fR
|
||||
Multiply the integer by 1,048,576 (2\e \fBMi\fR, \fBM\fR, and \fBMB\fR are accepted as synonyms for \fBMiB\fR\.
|
||||
Multiply the integer by 1,048,576 (2^20)\. \fBMi\fR, \fBM\fR, and \fBMB\fR are accepted as synonyms for \fBMiB\fR\.
|
||||
.
|
||||
.SS "Operation mode"
|
||||
If multiple operation mode options are given, the last one takes effect\.
|
||||
@ -261,6 +261,12 @@ cut file(s) into independent blocks of size # (default: no block)
|
||||
\fB\-\-priority=rt\fR
|
||||
set process priority to real\-time
|
||||
.
|
||||
.P
|
||||
\fBOutput Format:\fR CompressionLevel#Filename : IntputSize \-> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
|
||||
.
|
||||
.P
|
||||
\fBMethodology:\fR For both compression and decompression speed, the entire input is compressed/decompressed in\-memory to measure speed\. A run lasts at least 1 sec, so when files are small, they are compressed/decompressed several times per run, in order to improve measurement accuracy\.
|
||||
.
|
||||
.SH "ADVANCED COMPRESSION OPTIONS"
|
||||
.
|
||||
.SS "\-\-zstd[=options]:"
|
||||
|
@ -258,6 +258,9 @@ BENCHMARK
|
||||
* `--priority=rt`:
|
||||
set process priority to real-time
|
||||
|
||||
**Output Format:** CompressionLevel#Filename : IntputSize -> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
|
||||
|
||||
**Methodology:** For both compression and decompression speed, the entire input is compressed/decompressed in-memory to measure speed. A run lasts at least 1 sec, so when files are small, they are compressed/decompressed several times per run, in order to improve measurement accuracy.
|
||||
|
||||
ADVANCED COMPRESSION OPTIONS
|
||||
----------------------------
|
||||
|
@ -377,6 +377,7 @@ int main(int argCount, const char* argv[])
|
||||
lastCommand = 0,
|
||||
nbThreads = 1,
|
||||
setRealTimePrio = 0,
|
||||
separateFiles = 0,
|
||||
ldmFlag = 0;
|
||||
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
|
||||
size_t blockSize = 0;
|
||||
@ -633,6 +634,12 @@ int main(int argCount, const char* argv[])
|
||||
blockSize = readU32FromChar(&argument);
|
||||
break;
|
||||
|
||||
/* benchmark files separately (hidden option) */
|
||||
case 'S':
|
||||
argument++;
|
||||
separateFiles = 1;
|
||||
break;
|
||||
|
||||
#endif /* ZSTD_NOBENCH */
|
||||
|
||||
/* nb of threads (hidden option) */
|
||||
@ -751,8 +758,10 @@ int main(int argCount, const char* argv[])
|
||||
if (operation==zom_bench) {
|
||||
#ifndef ZSTD_NOBENCH
|
||||
BMK_setNotificationLevel(g_displayLevel);
|
||||
BMK_setSeparateFiles(separateFiles);
|
||||
BMK_setBlockSize(blockSize);
|
||||
BMK_setNbThreads(nbThreads);
|
||||
BMK_setRealTime(setRealTimePrio);
|
||||
BMK_setNbSeconds(bench_nbSeconds);
|
||||
BMK_setLdmFlag(ldmFlag);
|
||||
BMK_setLdmMinMatch(g_ldmMinMatch);
|
||||
@ -763,9 +772,10 @@ int main(int argCount, const char* argv[])
|
||||
if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
|
||||
BMK_setLdmHashEveryLog(g_ldmHashEveryLog);
|
||||
}
|
||||
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, setRealTimePrio);
|
||||
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams);
|
||||
#else
|
||||
(void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles;
|
||||
#endif
|
||||
(void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio;
|
||||
goto _end;
|
||||
}
|
||||
|
||||
@ -805,12 +815,6 @@ int main(int argCount, const char* argv[])
|
||||
if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !strcmp(filenameTable[0], stdinmark) && !forceStdout && operation!=zom_decompress)
|
||||
CLEAN_RETURN(badusage(programName));
|
||||
|
||||
/* user-selected output filename, only possible with a single file */
|
||||
if (outFileName && strcmp(outFileName,stdoutmark) && strcmp(outFileName,nulmark) && (filenameIdx>1)) {
|
||||
DISPLAY("Too many files (%u) on the command line. \n", filenameIdx);
|
||||
CLEAN_RETURN(filenameIdx);
|
||||
}
|
||||
|
||||
#ifndef ZSTD_NOCOMPRESS
|
||||
/* check compression level limits */
|
||||
{ int const maxCLevel = ultra ? ZSTD_maxCLevel() : ZSTDCLI_CLEVEL_MAX;
|
||||
@ -844,7 +848,7 @@ int main(int argCount, const char* argv[])
|
||||
if ((filenameIdx==1) && outFileName)
|
||||
operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, &compressionParams);
|
||||
else
|
||||
operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : suffix, dictFileName, cLevel, &compressionParams);
|
||||
operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, &compressionParams);
|
||||
#else
|
||||
(void)suffix;
|
||||
DISPLAY("Compression not supported\n");
|
||||
@ -862,7 +866,7 @@ int main(int argCount, const char* argv[])
|
||||
if (filenameIdx==1 && outFileName)
|
||||
operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
|
||||
else
|
||||
operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName);
|
||||
operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName, dictFileName);
|
||||
#else
|
||||
DISPLAY("Decompression not supported\n");
|
||||
#endif
|
||||
|
@ -76,16 +76,16 @@ allnothread: fullbench fuzzer paramgrill datagen decodecorpus
|
||||
dll: fuzzer-dll zstreamtest-dll
|
||||
|
||||
zstd:
|
||||
$(MAKE) -C $(PRGDIR) $@
|
||||
$(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"
|
||||
|
||||
zstd32:
|
||||
$(MAKE) -C $(PRGDIR) $@
|
||||
$(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"
|
||||
|
||||
zstd-nolegacy:
|
||||
$(MAKE) -C $(PRGDIR) $@
|
||||
$(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"
|
||||
|
||||
gzstd:
|
||||
$(MAKE) -C $(PRGDIR) zstd HAVE_ZLIB=1
|
||||
$(MAKE) -C $(PRGDIR) zstd HAVE_ZLIB=1 MOREFLAGS="$(DEBUGFLAGS)"
|
||||
|
||||
fullbench32: CPPFLAGS += -m32
|
||||
fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP)
|
||||
@ -130,15 +130,12 @@ zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zbufftest.c
|
||||
$(MAKE) -C $(ZSTDDIR) libzstd
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
|
||||
|
||||
ZSTREAMFILES := $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c zstreamtest.c
|
||||
zstreamtest : CPPFLAGS += $(MULTITHREAD_CPP)
|
||||
zstreamtest : LDFLAGS += $(MULTITHREAD_LD)
|
||||
zstreamtest : $(ZSTREAMFILES)
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
ZSTREAMFILES := $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c seqgen.c zstreamtest.c
|
||||
zstreamtest32 : CFLAGS += -m32
|
||||
zstreamtest32 : $(ZSTREAMFILES)
|
||||
$(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
|
||||
zstreamtest zstreamtest32 : CPPFLAGS += $(MULTITHREAD_CPP)
|
||||
zstreamtest zstreamtest32 : LDFLAGS += $(MULTITHREAD_LD)
|
||||
zstreamtest zstreamtest32 : $(ZSTREAMFILES)
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zstreamtest_asan : CFLAGS += -fsanitize=address
|
||||
zstreamtest_asan : $(ZSTREAMFILES)
|
||||
@ -303,10 +300,10 @@ test-fullbench32: fullbench32 datagen
|
||||
$(QEMU_SYS) ./fullbench32 -i1 -P0
|
||||
|
||||
test-fuzzer: fuzzer
|
||||
$(QEMU_SYS) ./fuzzer $(FUZZERTEST) $(FUZZER_FLAGS)
|
||||
$(QEMU_SYS) ./fuzzer -v $(FUZZERTEST) $(FUZZER_FLAGS)
|
||||
|
||||
test-fuzzer32: fuzzer32
|
||||
$(QEMU_SYS) ./fuzzer32 $(FUZZERTEST) $(FUZZER_FLAGS)
|
||||
$(QEMU_SYS) ./fuzzer32 -v $(FUZZERTEST) $(FUZZER_FLAGS)
|
||||
|
||||
test-zbuff: zbufftest
|
||||
$(QEMU_SYS) ./zbufftest $(ZSTREAM_TESTTIME)
|
||||
@ -315,7 +312,7 @@ test-zbuff32: zbufftest32
|
||||
$(QEMU_SYS) ./zbufftest32 $(ZSTREAM_TESTTIME)
|
||||
|
||||
test-zstream: zstreamtest
|
||||
$(QEMU_SYS) ./zstreamtest $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
|
||||
$(QEMU_SYS) ./zstreamtest -v $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
|
||||
$(QEMU_SYS) ./zstreamtest --mt -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
|
||||
$(QEMU_SYS) ./zstreamtest --newapi -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
|
||||
$(QEMU_SYS) ./zstreamtest --opaqueapi -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
|
||||
@ -377,9 +374,9 @@ test-pool: poolTests
|
||||
test-lz4: ZSTD = LD_LIBRARY_PATH=/usr/local/lib $(PRGDIR)/zstd
|
||||
test-lz4: ZSTD_LZ4 = LD_LIBRARY_PATH=/usr/local/lib ./lz4
|
||||
test-lz4: ZSTD_UNLZ4 = LD_LIBRARY_PATH=/usr/local/lib ./unlz4
|
||||
test-lz4: zstd decodecorpus
|
||||
ln -s $(PRGDIR)/zstd lz4
|
||||
ln -s $(PRGDIR)/zstd unlz4
|
||||
test-lz4: zstd decodecorpus datagen
|
||||
[ -f lz4 ] || ln -s $(PRGDIR)/zstd lz4
|
||||
[ -f unlz4 ] || ln -s $(PRGDIR)/zstd unlz4
|
||||
|
||||
./decodecorpus -ptmp
|
||||
# lz4 -> zstd
|
||||
@ -405,6 +402,8 @@ test-lz4: zstd decodecorpus
|
||||
$(ZSTD) -d | \
|
||||
cmp - tmp
|
||||
|
||||
./datagen -g384KB | $(ZSTD) --format=lz4 | $(ZSTD) -d > /dev/null
|
||||
|
||||
rm tmp lz4 unlz4
|
||||
|
||||
endif
|
||||
|
@ -14,8 +14,8 @@
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
|
||||
#include "util.h"
|
||||
#include "zstd.h"
|
||||
#include "zstd_internal.h"
|
||||
#include "mem.h"
|
||||
@ -49,20 +49,16 @@ static U32 g_displayLevel = 2;
|
||||
|
||||
#define DISPLAYUPDATE(...) \
|
||||
do { \
|
||||
if ((clockSpan(g_displayClock) > g_refreshRate) || \
|
||||
if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || \
|
||||
(g_displayLevel >= 4)) { \
|
||||
g_displayClock = clock(); \
|
||||
g_displayClock = UTIL_getTime(); \
|
||||
DISPLAY(__VA_ARGS__); \
|
||||
if (g_displayLevel >= 4) fflush(stderr); \
|
||||
} \
|
||||
} while (0)
|
||||
static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
|
||||
static clock_t g_displayClock = 0;
|
||||
|
||||
static clock_t clockSpan(clock_t cStart)
|
||||
{
|
||||
return clock() - cStart; /* works even when overflow; max span ~ 30mn */
|
||||
}
|
||||
static const U64 g_refreshRate = SEC_TO_MICRO / 6;
|
||||
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
|
||||
|
||||
#define CHECKERR(code) \
|
||||
do { \
|
||||
@ -1531,14 +1527,14 @@ static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS
|
||||
{
|
||||
unsigned fnum;
|
||||
|
||||
clock_t const startClock = clock();
|
||||
clock_t const maxClockSpan = testDurationS * CLOCKS_PER_SEC;
|
||||
UTIL_time_t const startClock = UTIL_getTime();
|
||||
U64 const maxClockSpan = testDurationS * SEC_TO_MICRO;
|
||||
|
||||
if (numFiles == 0 && !testDurationS) numFiles = 1;
|
||||
|
||||
DISPLAY("seed: %u\n", seed);
|
||||
|
||||
for (fnum = 0; fnum < numFiles || clockSpan(startClock) < maxClockSpan; fnum++) {
|
||||
for (fnum = 0; fnum < numFiles || UTIL_clockSpanMicro(startClock) < maxClockSpan; fnum++) {
|
||||
if (fnum < numFiles)
|
||||
DISPLAYUPDATE("\r%u/%u ", fnum, numFiles);
|
||||
else
|
||||
|
@ -233,7 +233,7 @@ static ZSTD_CCtx* g_zcc = NULL;
|
||||
size_t local_ZSTD_compressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
|
||||
{
|
||||
(void)buff2;
|
||||
ZSTD_compressBegin(g_zcc, 1);
|
||||
ZSTD_compressBegin(g_zcc, 1 /* compressionLevel */);
|
||||
return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize);
|
||||
}
|
||||
|
||||
@ -484,8 +484,8 @@ static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchN
|
||||
/* Loop for each file */
|
||||
int fileIdx;
|
||||
for (fileIdx=0; fileIdx<nbFiles; fileIdx++) {
|
||||
const char* inFileName = fileNamesTable[fileIdx];
|
||||
FILE* inFile = fopen( inFileName, "rb" );
|
||||
const char* const inFileName = fileNamesTable[fileIdx];
|
||||
FILE* const inFile = fopen( inFileName, "rb" );
|
||||
U64 inFileSize;
|
||||
size_t benchedSize;
|
||||
void* origBuff;
|
||||
@ -495,6 +495,11 @@ static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchN
|
||||
|
||||
/* Memory allocation & restrictions */
|
||||
inFileSize = UTIL_getFileSize(inFileName);
|
||||
if (inFileSize == UTIL_FILESIZE_UNKNOWN) {
|
||||
DISPLAY( "Cannot measure size of %s\n", inFileName);
|
||||
fclose(inFile);
|
||||
return 11;
|
||||
}
|
||||
benchedSize = BMK_findMaxMem(inFileSize*3) / 3;
|
||||
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
|
||||
if (benchedSize < inFileSize)
|
||||
|
@ -25,7 +25,7 @@
|
||||
#include <stdlib.h> /* free */
|
||||
#include <stdio.h> /* fgets, sscanf */
|
||||
#include <string.h> /* strcmp */
|
||||
#include <time.h> /* clock_t */
|
||||
#include <assert.h>
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
|
||||
#include "zstd.h" /* ZSTD_VERSION_STRING */
|
||||
#include "zstd_errors.h" /* ZSTD_getErrorCode */
|
||||
@ -36,6 +36,7 @@
|
||||
#include "mem.h"
|
||||
#define XXH_STATIC_LINKING_ONLY
|
||||
#include "xxhash.h" /* XXH64 */
|
||||
#include "util.h"
|
||||
|
||||
|
||||
/*-************************************
|
||||
@ -56,25 +57,22 @@ static const U32 nbTestsDefault = 30000;
|
||||
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
static U32 g_displayLevel = 2;
|
||||
|
||||
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
|
||||
if ((FUZ_clockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
|
||||
{ g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
|
||||
if (g_displayLevel>=4) fflush(stdout); } }
|
||||
static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
|
||||
static clock_t g_displayClock = 0;
|
||||
static const U64 g_refreshRate = SEC_TO_MICRO / 6;
|
||||
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
|
||||
|
||||
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
|
||||
if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
|
||||
{ g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
|
||||
if (g_displayLevel>=4) fflush(stderr); } }
|
||||
|
||||
/*-*******************************************************
|
||||
* Fuzzer functions
|
||||
*********************************************************/
|
||||
#undef MIN
|
||||
#undef MAX
|
||||
#define MIN(a,b) ((a)<(b)?(a):(b))
|
||||
#define MAX(a,b) ((a)>(b)?(a):(b))
|
||||
|
||||
static clock_t FUZ_clockSpan(clock_t cStart)
|
||||
{
|
||||
return clock() - cStart; /* works even when overflow; max span ~ 30mn */
|
||||
}
|
||||
|
||||
#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
static unsigned FUZ_rand(unsigned* src)
|
||||
{
|
||||
@ -97,6 +95,22 @@ static unsigned FUZ_highbit32(U32 v32)
|
||||
}
|
||||
|
||||
|
||||
/*=============================================
|
||||
* Test macros
|
||||
=============================================*/
|
||||
#define CHECK_Z(f) { \
|
||||
size_t const err = f; \
|
||||
if (ZSTD_isError(err)) { \
|
||||
DISPLAY("Error => %s : %s ", \
|
||||
#f, ZSTD_getErrorName(err)); \
|
||||
exit(1); \
|
||||
} }
|
||||
|
||||
#define CHECK_V(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error
|
||||
#define CHECK(fn) { CHECK_V(err, fn); }
|
||||
#define CHECKPLUS(var, fn, more) { CHECK_V(var, fn); more; }
|
||||
|
||||
|
||||
/*=============================================
|
||||
* Memory Tests
|
||||
=============================================*/
|
||||
@ -146,14 +160,6 @@ static void FUZ_displayMallocStats(mallocCounter_t count)
|
||||
(U32)(count.totalMalloc >> 10));
|
||||
}
|
||||
|
||||
#define CHECK_Z(f) { \
|
||||
size_t const err = f; \
|
||||
if (ZSTD_isError(err)) { \
|
||||
DISPLAY("Error => %s : %s ", \
|
||||
#f, ZSTD_getErrorName(err)); \
|
||||
exit(1); \
|
||||
} }
|
||||
|
||||
static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
|
||||
{
|
||||
size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */
|
||||
@ -259,10 +265,6 @@ static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
|
||||
* Unit tests
|
||||
=============================================*/
|
||||
|
||||
#define CHECK_V(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error
|
||||
#define CHECK(fn) { CHECK_V(err, fn); }
|
||||
#define CHECKPLUS(var, fn, more) { CHECK_V(var, fn); more; }
|
||||
|
||||
static int basicUnitTests(U32 seed, double compressibility)
|
||||
{
|
||||
size_t const CNBuffSize = 5 MB;
|
||||
@ -359,6 +361,31 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%di : check CCtx size after compressing empty input : ", testNb++);
|
||||
{ ZSTD_CCtx* cctx = ZSTD_createCCtx();
|
||||
size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19);
|
||||
if (ZSTD_isError(r)) goto _output_error;
|
||||
if (ZSTD_sizeof_CCtx(cctx) > (1U << 20)) goto _output_error;
|
||||
ZSTD_freeCCtx(cctx);
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%di : re-use CCtx with expanding block size : ", testNb++);
|
||||
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
ZSTD_parameters const params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0);
|
||||
assert(params.fParams.contentSizeFlag == 1); /* block size will be adapted if pledgedSrcSize is enabled */
|
||||
CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, 1 /*pledgedSrcSize*/) );
|
||||
CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); /* creates a block size of 1 */
|
||||
|
||||
CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */
|
||||
{ size_t const inSize = 2* 128 KB;
|
||||
size_t const outSize = ZSTD_compressBound(inSize);
|
||||
CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, outSize, CNBuffer, inSize) );
|
||||
/* will fail if blockSize is not resized */
|
||||
}
|
||||
ZSTD_freeCCtx(cctx);
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* Static CCtx tests */
|
||||
#define STATIC_CCTX_LEVEL 3
|
||||
@ -517,7 +544,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
off += r;
|
||||
if (i == segs/2) {
|
||||
/* insert skippable frame */
|
||||
const U32 skipLen = 128 KB;
|
||||
const U32 skipLen = 129 KB;
|
||||
MEM_writeLE32((BYTE*)compressedBuffer + off, ZSTD_MAGIC_SKIPPABLE_START);
|
||||
MEM_writeLE32((BYTE*)compressedBuffer + off + 4, skipLen);
|
||||
off += skipLen + ZSTD_skippableHeaderSize;
|
||||
@ -830,6 +857,50 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : Dictionary with non-default repcodes : ", testNb++);
|
||||
{ U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
|
||||
dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
|
||||
CNBuffer, samplesSizes, nbSamples);
|
||||
if (ZDICT_isError(dictSize)) goto _output_error;
|
||||
/* Set all the repcodes to non-default */
|
||||
{
|
||||
BYTE* dictPtr = (BYTE*)dictBuffer;
|
||||
BYTE* dictLimit = dictPtr + dictSize - 12;
|
||||
/* Find the repcodes */
|
||||
while (dictPtr < dictLimit &&
|
||||
(MEM_readLE32(dictPtr) != 1 || MEM_readLE32(dictPtr + 4) != 4 ||
|
||||
MEM_readLE32(dictPtr + 8) != 8)) {
|
||||
++dictPtr;
|
||||
}
|
||||
if (dictPtr >= dictLimit) goto _output_error;
|
||||
MEM_writeLE32(dictPtr + 0, 10);
|
||||
MEM_writeLE32(dictPtr + 4, 10);
|
||||
MEM_writeLE32(dictPtr + 8, 10);
|
||||
/* Set the last 8 bytes to 'x' */
|
||||
memset((BYTE*)dictBuffer + dictSize - 8, 'x', 8);
|
||||
}
|
||||
/* The optimal parser checks all the repcodes.
|
||||
* Make sure at least one is a match >= targetLength so that it is
|
||||
* immediately chosen. This will make sure that the compressor and
|
||||
* decompressor agree on at least one of the repcodes.
|
||||
*/
|
||||
{ size_t dSize;
|
||||
BYTE data[1024];
|
||||
ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize);
|
||||
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
|
||||
ZSTD_dlm_byRef, ZSTD_dm_auto,
|
||||
cParams, ZSTD_defaultCMem);
|
||||
memset(data, 'x', sizeof(data));
|
||||
cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
|
||||
data, sizeof(data), cdict);
|
||||
ZSTD_freeCDict(cdict);
|
||||
if (ZSTD_isError(cSize)) { DISPLAYLEVEL(5, "Compression error %s : ", ZSTD_getErrorName(cSize)); goto _output_error; }
|
||||
dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize);
|
||||
if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; }
|
||||
if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; }
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
ZSTD_freeCCtx(cctx);
|
||||
free(dictBuffer);
|
||||
free(samplesSizes);
|
||||
@ -1200,8 +1271,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
||||
U32 result = 0;
|
||||
U32 testNb = 0;
|
||||
U32 coreSeed = seed, lseed = 0;
|
||||
clock_t const startClock = clock();
|
||||
clock_t const maxClockSpan = maxDurationS * CLOCKS_PER_SEC;
|
||||
UTIL_time_t const startClock = UTIL_getTime();
|
||||
U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO;
|
||||
int const cLevelLimiter = bigTests ? 3 : 2;
|
||||
|
||||
/* allocation */
|
||||
@ -1226,7 +1297,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
||||
for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed);
|
||||
|
||||
/* main test loop */
|
||||
for ( ; (testNb <= nbTests) || (FUZ_clockSpan(startClock) < maxClockSpan); testNb++ ) {
|
||||
for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < maxClockSpan); testNb++ ) {
|
||||
size_t sampleSize, maxTestSize, totalTestSize;
|
||||
size_t cSize, totalCSize, totalGenSize;
|
||||
U64 crcOrig;
|
||||
|
@ -17,13 +17,14 @@
|
||||
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
||||
#include <string.h> /* strcmp */
|
||||
#include <math.h> /* log */
|
||||
#include <time.h> /* clock_t */
|
||||
#include <time.h>
|
||||
|
||||
#include "mem.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_estimateCCtxSize */
|
||||
#include "zstd.h"
|
||||
#include "datagen.h"
|
||||
#include "xxhash.h"
|
||||
#include "util.h"
|
||||
|
||||
|
||||
/*-************************************
|
||||
@ -39,7 +40,7 @@
|
||||
#define GB *(1ULL<<30)
|
||||
|
||||
#define NBLOOPS 2
|
||||
#define TIMELOOP (2 * CLOCKS_PER_SEC)
|
||||
#define TIMELOOP (2 * SEC_TO_MICRO)
|
||||
|
||||
#define NB_LEVELS_TRACKED 30
|
||||
|
||||
@ -49,8 +50,8 @@ static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t
|
||||
static const size_t sampleSize = 10000000;
|
||||
|
||||
static const double g_grillDuration_s = 90000; /* about 24 hours */
|
||||
static const clock_t g_maxParamTime = 15 * CLOCKS_PER_SEC;
|
||||
static const clock_t g_maxVariationTime = 60 * CLOCKS_PER_SEC;
|
||||
static const U64 g_maxParamTime = 15 * SEC_TO_MICRO;
|
||||
static const U64 g_maxVariationTime = 60 * SEC_TO_MICRO;
|
||||
static const int g_maxNbVariations = 64;
|
||||
|
||||
|
||||
@ -88,13 +89,9 @@ void BMK_SetNbIterations(int nbLoops)
|
||||
* Private functions
|
||||
*********************************************************/
|
||||
|
||||
/* works even if overflow ; max span ~ 30 mn */
|
||||
static clock_t BMK_clockSpan(clock_t cStart) { return clock() - cStart; }
|
||||
|
||||
/* accuracy in seconds only, span can be multiple years */
|
||||
static double BMK_timeSpan(time_t tStart) { return difftime(time(NULL), tStart); }
|
||||
|
||||
|
||||
static size_t BMK_findMaxMem(U64 requiredMem)
|
||||
{
|
||||
size_t const step = 64 MB;
|
||||
@ -221,7 +218,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
|
||||
size_t cSize = 0;
|
||||
double fastestC = 100000000., fastestD = 100000000.;
|
||||
double ratio = 0.;
|
||||
clock_t const benchStart = clock();
|
||||
UTIL_time_t const benchStart = UTIL_getTime();
|
||||
|
||||
DISPLAY("\r%79s\r", "");
|
||||
memset(¶ms, 0, sizeof(params));
|
||||
@ -229,9 +226,10 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
|
||||
for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
|
||||
int nbLoops;
|
||||
U32 blockNb;
|
||||
clock_t roundStart, roundClock;
|
||||
UTIL_time_t roundStart;
|
||||
U64 roundClock;
|
||||
|
||||
{ clock_t const benchTime = BMK_clockSpan(benchStart);
|
||||
{ U64 const benchTime = UTIL_clockSpanMicro(benchStart);
|
||||
if (benchTime > g_maxParamTime) break; }
|
||||
|
||||
/* Compression */
|
||||
@ -239,10 +237,9 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
|
||||
memset(compressedBuffer, 0xE5, maxCompressedSize);
|
||||
|
||||
nbLoops = 0;
|
||||
roundStart = clock();
|
||||
while (clock() == roundStart);
|
||||
roundStart = clock();
|
||||
while (BMK_clockSpan(roundStart) < TIMELOOP) {
|
||||
UTIL_waitForNextTick();
|
||||
roundStart = UTIL_getTime();
|
||||
while (UTIL_clockSpanMicro(roundStart) < TIMELOOP) {
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++)
|
||||
blockTable[blockNb].cSize = ZSTD_compress_advanced(ctx,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
||||
@ -251,13 +248,13 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
|
||||
params);
|
||||
nbLoops++;
|
||||
}
|
||||
roundClock = BMK_clockSpan(roundStart);
|
||||
roundClock = UTIL_clockSpanMicro(roundStart);
|
||||
|
||||
cSize = 0;
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++)
|
||||
cSize += blockTable[blockNb].cSize;
|
||||
ratio = (double)srcSize / (double)cSize;
|
||||
if ((double)roundClock < fastestC * CLOCKS_PER_SEC * nbLoops) fastestC = ((double)roundClock / CLOCKS_PER_SEC) / nbLoops;
|
||||
if ((double)roundClock < fastestC * SEC_TO_MICRO * nbLoops) fastestC = ((double)roundClock / SEC_TO_MICRO) / nbLoops;
|
||||
DISPLAY("\r");
|
||||
DISPLAY("%1u-%s : %9u ->", loopNb, name, (U32)srcSize);
|
||||
DISPLAY(" %9u (%4.3f),%7.1f MB/s", (U32)cSize, ratio, (double)srcSize / fastestC / 1000000.);
|
||||
@ -269,17 +266,16 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
|
||||
memset(resultBuffer, 0xD6, srcSize);
|
||||
|
||||
nbLoops = 0;
|
||||
roundStart = clock();
|
||||
while (clock() == roundStart);
|
||||
roundStart = clock();
|
||||
for ( ; BMK_clockSpan(roundStart) < TIMELOOP; nbLoops++) {
|
||||
UTIL_waitForNextTick();
|
||||
roundStart = UTIL_getTime();
|
||||
for ( ; UTIL_clockSpanMicro(roundStart) < TIMELOOP; nbLoops++) {
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++)
|
||||
blockTable[blockNb].resSize = ZSTD_decompress(blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cSize);
|
||||
}
|
||||
roundClock = BMK_clockSpan(roundStart);
|
||||
roundClock = UTIL_clockSpanMicro(roundStart);
|
||||
|
||||
if ((double)roundClock < fastestD * CLOCKS_PER_SEC * nbLoops) fastestD = ((double)roundClock / CLOCKS_PER_SEC) / nbLoops;
|
||||
if ((double)roundClock < fastestD * SEC_TO_MICRO * nbLoops) fastestD = ((double)roundClock / SEC_TO_MICRO) / nbLoops;
|
||||
DISPLAY("\r");
|
||||
DISPLAY("%1u-%s : %9u -> ", loopNb, name, (U32)srcSize);
|
||||
DISPLAY("%9u (%4.3f),%7.1f MB/s, ", (U32)cSize, ratio, (double)srcSize / fastestC / 1000000.);
|
||||
@ -310,14 +306,10 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
|
||||
}
|
||||
|
||||
|
||||
const char* g_stratName[] = { "ZSTD_fast ",
|
||||
"ZSTD_dfast ",
|
||||
"ZSTD_greedy ",
|
||||
"ZSTD_lazy ",
|
||||
"ZSTD_lazy2 ",
|
||||
"ZSTD_btlazy2 ",
|
||||
"ZSTD_btopt ",
|
||||
"ZSTD_btultra "};
|
||||
const char* g_stratName[ZSTD_btultra+1] = {
|
||||
"(none) ", "ZSTD_fast ", "ZSTD_dfast ",
|
||||
"ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ",
|
||||
"ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "};
|
||||
|
||||
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compressionParameters params, size_t srcSize)
|
||||
{
|
||||
@ -525,9 +517,9 @@ static void playAround(FILE* f, winnerInfo_t* winners,
|
||||
ZSTD_CCtx* ctx)
|
||||
{
|
||||
int nbVariations = 0;
|
||||
clock_t const clockStart = clock();
|
||||
UTIL_time_t const clockStart = UTIL_getTime();
|
||||
|
||||
while (BMK_clockSpan(clockStart) < g_maxVariationTime) {
|
||||
while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) {
|
||||
ZSTD_compressionParameters p = params;
|
||||
|
||||
if (nbVariations++ > g_maxNbVariations) break;
|
||||
@ -687,6 +679,11 @@ int benchFiles(const char** fileNamesTable, int nbFiles)
|
||||
DISPLAY( "Pb opening %s\n", inFileName);
|
||||
return 11;
|
||||
}
|
||||
if (inFileSize == UTIL_FILESIZE_UNKNOWN) {
|
||||
DISPLAY("Pb evaluatin size of %s \n", inFileName);
|
||||
fclose(inFile);
|
||||
return 11;
|
||||
}
|
||||
|
||||
/* Memory allocation */
|
||||
benchedSize = BMK_findMaxMem(inFileSize*3) / 3;
|
||||
@ -740,6 +737,11 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed)
|
||||
|
||||
/* Init */
|
||||
if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; }
|
||||
if (inFileSize == UTIL_FILESIZE_UNKNOWN) {
|
||||
DISPLAY("Pb evaluatin size of %s \n", inFileName);
|
||||
fclose(inFile);
|
||||
return 11;
|
||||
}
|
||||
|
||||
/* Memory allocation & restrictions */
|
||||
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
|
||||
|
@ -56,10 +56,12 @@ fi
|
||||
|
||||
isWindows=false
|
||||
INTOVOID="/dev/null"
|
||||
DEVDEVICE="/dev/zero"
|
||||
case "$OS" in
|
||||
Windows*)
|
||||
isWindows=true
|
||||
INTOVOID="NUL"
|
||||
DEVDEVICE="NUL"
|
||||
;;
|
||||
esac
|
||||
|
||||
@ -91,7 +93,7 @@ else
|
||||
hasMT="true"
|
||||
fi
|
||||
|
||||
$ECHO "\n**** simple tests **** "
|
||||
$ECHO "\n===> simple tests "
|
||||
|
||||
./datagen > tmp
|
||||
$ECHO "test : basic compression "
|
||||
@ -124,6 +126,10 @@ $ZSTD -d > $INTOVOID && die "should have refused : compressed data from terminal
|
||||
fi
|
||||
$ECHO "test : null-length file roundtrip"
|
||||
$ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout
|
||||
$ECHO "test : ensure small file doesn't add 3-bytes null block"
|
||||
./datagen -g1 > tmp1
|
||||
$ZSTD tmp1 -c | wc -c | grep "14"
|
||||
$ZSTD < tmp1 | wc -c | grep "14"
|
||||
$ECHO "test : decompress file with wrong suffix (must fail)"
|
||||
$ZSTD -d tmpCompressed && die "wrong suffix error not detected!"
|
||||
$ZSTD -df tmp && die "should have refused : wrong extension"
|
||||
@ -159,14 +165,36 @@ $ZSTD -f --rm tmp
|
||||
test ! -f tmp # tmp should no longer be present
|
||||
$ZSTD -f -d --rm tmp.zst
|
||||
test ! -f tmp.zst # tmp.zst should no longer be present
|
||||
$ECHO "test : should quietly not remove non-regular file"
|
||||
$ECHO hello > tmp
|
||||
$ZSTD tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID"
|
||||
grep -v "Refusing to remove non-regular file" tmplog
|
||||
rm -f tmplog
|
||||
$ZSTD tmp -f -o "$INTONULL" 2>&1 | grep -v "Refusing to remove non-regular file"
|
||||
$ECHO "test : --rm on stdin"
|
||||
$ECHO a | $ZSTD --rm > $INTOVOID # --rm should remain silent
|
||||
rm tmp
|
||||
$ZSTD -f tmp && die "tmp not present : should have failed"
|
||||
test ! -f tmp.zst # tmp.zst should not be created
|
||||
|
||||
$ECHO "test : compress multiple files"
|
||||
$ECHO hello > tmp1
|
||||
$ECHO world > tmp2
|
||||
$ZSTD tmp1 tmp2 -o "$INTOVOID"
|
||||
$ZSTD tmp1 tmp2 -c | $ZSTD -t
|
||||
$ZSTD tmp1 tmp2 -o tmp.zst
|
||||
test ! -f tmp1.zst
|
||||
test ! -f tmp2.zst
|
||||
$ZSTD tmp1 tmp2
|
||||
$ZSTD -t tmp1.zst tmp2.zst
|
||||
$ZSTD -dc tmp1.zst tmp2.zst
|
||||
$ZSTD tmp1.zst tmp2.zst -o "$INTOVOID"
|
||||
$ZSTD -d tmp1.zst tmp2.zst -o tmp
|
||||
rm tmp*
|
||||
|
||||
$ECHO "\n**** Advanced compression parameters **** "
|
||||
|
||||
|
||||
$ECHO "\n===> Advanced compression parameters "
|
||||
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!"
|
||||
$ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!"
|
||||
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!"
|
||||
@ -180,14 +208,14 @@ roundTripTest -g512K " --long --zstd=ldmhlog=20,ldmslen=64,ldmblog=1,ldmhevery=7
|
||||
roundTripTest -g512K 19
|
||||
|
||||
|
||||
$ECHO "\n**** Pass-Through mode **** "
|
||||
$ECHO "\n===> Pass-Through mode "
|
||||
$ECHO "Hello world 1!" | $ZSTD -df
|
||||
$ECHO "Hello world 2!" | $ZSTD -dcf
|
||||
$ECHO "Hello world 3!" > tmp1
|
||||
$ZSTD -dcf tmp1
|
||||
|
||||
|
||||
$ECHO "\n**** frame concatenation **** "
|
||||
$ECHO "\n===> frame concatenation "
|
||||
|
||||
$ECHO "hello " > hello.tmp
|
||||
$ECHO "world!" > world.tmp
|
||||
@ -218,7 +246,7 @@ $ECHO "$ECHO foo | $ZSTD | $ZSTD -d > /dev/full"
|
||||
$ECHO foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"
|
||||
|
||||
|
||||
$ECHO "\n**** symbolic link test **** "
|
||||
$ECHO "\n===> symbolic link test "
|
||||
|
||||
rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
|
||||
$ECHO "hello world" > hello.tmp
|
||||
@ -233,7 +261,7 @@ rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
|
||||
fi
|
||||
|
||||
|
||||
$ECHO "\n**** test sparse file support **** "
|
||||
$ECHO "\n===> test sparse file support "
|
||||
|
||||
./datagen -g5M -P100 > tmpSparse
|
||||
$ZSTD tmpSparse -c | $ZSTD -dv -o tmpSparseRegen
|
||||
@ -260,7 +288,7 @@ $DIFF tmpSparse2M tmpSparseRegenerated
|
||||
rm tmpSparse*
|
||||
|
||||
|
||||
$ECHO "\n**** multiple files tests **** "
|
||||
$ECHO "\n===> multiple files tests "
|
||||
|
||||
./datagen -s1 > tmp1 2> $INTOVOID
|
||||
./datagen -s2 -g100K > tmp2 2> $INTOVOID
|
||||
@ -283,7 +311,7 @@ $ECHO "compress multiple files including a missing one (notHere) : "
|
||||
$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
|
||||
|
||||
|
||||
$ECHO "\n**** dictionary tests **** "
|
||||
$ECHO "\n===> dictionary tests "
|
||||
|
||||
$ECHO "- test with raw dict (content only) "
|
||||
./datagen > tmpDict
|
||||
@ -297,6 +325,11 @@ cp $TESTFILE tmp
|
||||
$ZSTD -f tmp -D tmpDict
|
||||
$ZSTD -d tmp.zst -D tmpDict -fo result
|
||||
$DIFF $TESTFILE result
|
||||
if [ -n "$hasMT" ]
|
||||
then
|
||||
$ECHO "- Test dictionary compression with multithreading "
|
||||
./datagen -g5M | $ZSTD -T2 -D tmpDict | $ZSTD -t -D tmpDict # fails with v1.3.2
|
||||
fi
|
||||
$ECHO "- Create second (different) dictionary "
|
||||
$ZSTD --train *.c ../programs/*.c ../programs/*.h -o tmpDictC
|
||||
$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
|
||||
@ -339,7 +372,7 @@ $ZSTD --train-legacy -q tmp && die "Dictionary training should fail : source is
|
||||
rm tmp*
|
||||
|
||||
|
||||
$ECHO "\n**** cover dictionary tests **** "
|
||||
$ECHO "\n===> cover dictionary builder : advanced options "
|
||||
|
||||
TESTFILE=../programs/zstdcli.c
|
||||
./datagen > tmpDict
|
||||
@ -359,7 +392,7 @@ $ECHO "- Create dictionary with size limit"
|
||||
$ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
|
||||
rm tmp*
|
||||
|
||||
$ECHO "\n**** legacy dictionary tests **** "
|
||||
$ECHO "\n===> legacy dictionary builder "
|
||||
|
||||
TESTFILE=../programs/zstdcli.c
|
||||
./datagen > tmpDict
|
||||
@ -380,7 +413,7 @@ $ZSTD --train-legacy -s9 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
|
||||
rm tmp*
|
||||
|
||||
|
||||
$ECHO "\n**** integrity tests **** "
|
||||
$ECHO "\n===> integrity tests "
|
||||
|
||||
$ECHO "test one file (tmp1.zst) "
|
||||
./datagen > tmp1
|
||||
@ -405,13 +438,13 @@ $ZSTD -t tmpSplit.* && die "bad file not detected !"
|
||||
|
||||
|
||||
|
||||
$ECHO "\n**** golden files tests **** "
|
||||
$ECHO "\n===> golden files tests "
|
||||
|
||||
$ZSTD -t -r files
|
||||
$ZSTD -c -r files | $ZSTD -t
|
||||
|
||||
|
||||
$ECHO "\n**** benchmark mode tests **** "
|
||||
$ECHO "\n===> benchmark mode tests "
|
||||
|
||||
$ECHO "bench one file"
|
||||
./datagen > tmp1
|
||||
@ -422,7 +455,7 @@ $ECHO "with recursive and quiet modes"
|
||||
$ZSTD -rqi1b1e2 tmp1
|
||||
|
||||
|
||||
$ECHO "\n**** gzip compatibility tests **** "
|
||||
$ECHO "\n===> gzip compatibility tests "
|
||||
|
||||
GZIPMODE=1
|
||||
$ZSTD --format=gzip -V || GZIPMODE=0
|
||||
@ -445,7 +478,7 @@ else
|
||||
fi
|
||||
|
||||
|
||||
$ECHO "\n**** gzip frame tests **** "
|
||||
$ECHO "\n===> gzip frame tests "
|
||||
|
||||
if [ $GZIPMODE -eq 1 ]; then
|
||||
./datagen > tmp
|
||||
@ -459,7 +492,7 @@ else
|
||||
fi
|
||||
|
||||
|
||||
$ECHO "\n**** xz compatibility tests **** "
|
||||
$ECHO "\n===> xz compatibility tests "
|
||||
|
||||
LZMAMODE=1
|
||||
$ZSTD --format=xz -V || LZMAMODE=0
|
||||
@ -505,7 +538,7 @@ else
|
||||
fi
|
||||
|
||||
|
||||
$ECHO "\n**** xz frame tests **** "
|
||||
$ECHO "\n===> xz frame tests "
|
||||
|
||||
if [ $LZMAMODE -eq 1 ]; then
|
||||
./datagen > tmp
|
||||
@ -520,7 +553,7 @@ else
|
||||
$ECHO "xz mode not supported"
|
||||
fi
|
||||
|
||||
$ECHO "\n**** lz4 compatibility tests **** "
|
||||
$ECHO "\n===> lz4 compatibility tests "
|
||||
|
||||
LZ4MODE=1
|
||||
$ZSTD --format=lz4 -V || LZ4MODE=0
|
||||
@ -543,7 +576,7 @@ else
|
||||
fi
|
||||
|
||||
|
||||
$ECHO "\n**** lz4 frame tests **** "
|
||||
$ECHO "\n===> lz4 frame tests "
|
||||
|
||||
if [ $LZ4MODE -eq 1 ]; then
|
||||
./datagen > tmp
|
||||
@ -556,7 +589,7 @@ else
|
||||
$ECHO "lz4 mode not supported"
|
||||
fi
|
||||
|
||||
$ECHO "\n**** zstd round-trip tests **** "
|
||||
$ECHO "\n===> zstd round-trip tests "
|
||||
|
||||
roundTripTest
|
||||
roundTripTest -g15K # TableID==3
|
||||
@ -569,7 +602,7 @@ roundTripTest -g516K 19 # btopt
|
||||
|
||||
fileRoundTripTest -g500K
|
||||
|
||||
$ECHO "\n**** zstd long distance matching round-trip tests **** "
|
||||
$ECHO "\n===> zstd long distance matching round-trip tests "
|
||||
roundTripTest -g0 "2 --long"
|
||||
roundTripTest -g1000K "1 --long"
|
||||
roundTripTest -g517K "6 --long"
|
||||
@ -580,58 +613,56 @@ fileRoundTripTest -g5M "3 --long"
|
||||
|
||||
if [ -n "$hasMT" ]
|
||||
then
|
||||
$ECHO "\n**** zstdmt round-trip tests **** "
|
||||
$ECHO "\n===> zstdmt round-trip tests "
|
||||
roundTripTest -g4M "1 -T0"
|
||||
roundTripTest -g8M "3 -T2"
|
||||
roundTripTest -g8000K "2 --threads=2"
|
||||
fileRoundTripTest -g4M "19 -T2 -B1M"
|
||||
|
||||
$ECHO "\n**** zstdmt long distance matching round-trip tests **** "
|
||||
$ECHO "\n===> zstdmt long distance matching round-trip tests "
|
||||
roundTripTest -g8M "3 --long -T2"
|
||||
else
|
||||
$ECHO "\n**** no multithreading, skipping zstdmt tests **** "
|
||||
$ECHO "\n===> no multithreading, skipping zstdmt tests "
|
||||
fi
|
||||
|
||||
rm tmp*
|
||||
|
||||
$ECHO "\n**** zstd --list/-l single frame tests ****"
|
||||
$ECHO "\n===> zstd --list/-l single frame tests "
|
||||
./datagen > tmp1
|
||||
./datagen > tmp2
|
||||
./datagen > tmp3
|
||||
$ZSTD tmp*
|
||||
$ZSTD -l *.zst
|
||||
$ZSTD -lv *.zst
|
||||
$ZSTD -lv *.zst | grep "Decompressed Size:" # check that decompressed size is present in header
|
||||
$ZSTD --list *.zst
|
||||
$ZSTD --list -v *.zst
|
||||
|
||||
$ECHO "\n**** zstd --list/-l multiple frame tests ****"
|
||||
$ECHO "\n===> zstd --list/-l multiple frame tests "
|
||||
cat tmp1.zst tmp2.zst > tmp12.zst
|
||||
cat tmp12.zst tmp3.zst > tmp123.zst
|
||||
$ZSTD -l *.zst
|
||||
$ZSTD -lv *.zst
|
||||
$ZSTD --list *.zst
|
||||
$ZSTD --list -v *.zst
|
||||
|
||||
$ECHO "\n**** zstd --list/-l error detection tests ****"
|
||||
$ECHO "\n===> zstd --list/-l error detection tests "
|
||||
! $ZSTD -l tmp1 tmp1.zst
|
||||
! $ZSTD --list tmp*
|
||||
! $ZSTD -lv tmp1*
|
||||
! $ZSTD --list -v tmp2 tmp12.zst
|
||||
|
||||
$ECHO "\n**** zstd --list/-l test with null files ****"
|
||||
$ECHO "\n===> zstd --list/-l test with null files "
|
||||
./datagen -g0 > tmp5
|
||||
$ZSTD tmp5
|
||||
$ZSTD -l tmp5.zst
|
||||
! $ZSTD -l tmp5*
|
||||
$ZSTD -lv tmp5.zst
|
||||
$ZSTD -lv tmp5.zst | grep "Decompressed Size: 0.00 KB (0 B)" # check that 0 size is present in header
|
||||
! $ZSTD -lv tmp5*
|
||||
|
||||
$ECHO "\n**** zstd --list/-l test with no content size field ****"
|
||||
./datagen -g1MB | $ZSTD > tmp6.zst
|
||||
$ECHO "\n===> zstd --list/-l test with no content size field "
|
||||
./datagen -g513K | $ZSTD > tmp6.zst
|
||||
$ZSTD -l tmp6.zst
|
||||
$ZSTD -lv tmp6.zst
|
||||
! $ZSTD -lv tmp6.zst | grep "Decompressed Size:" # must NOT be present in header
|
||||
|
||||
$ECHO "\n**** zstd --list/-l test with no checksum ****"
|
||||
$ECHO "\n===> zstd --list/-l test with no checksum "
|
||||
$ZSTD -f --no-check tmp1
|
||||
$ZSTD -l tmp1.zst
|
||||
$ZSTD -lv tmp1.zst
|
||||
@ -639,7 +670,7 @@ $ZSTD -lv tmp1.zst
|
||||
rm tmp*
|
||||
|
||||
|
||||
$ECHO "\n**** zstd long distance matching tests **** "
|
||||
$ECHO "\n===> zstd long distance matching tests "
|
||||
roundTripTest -g0 " --long"
|
||||
roundTripTest -g9M "2 --long"
|
||||
# Test parameter parsing
|
||||
@ -654,7 +685,7 @@ if [ "$1" != "--test-large-data" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
$ECHO "\n**** large files tests **** "
|
||||
$ECHO "\n===> large files tests "
|
||||
|
||||
roundTripTest -g270000000 1
|
||||
roundTripTest -g250000000 2
|
||||
@ -685,7 +716,7 @@ roundTripTest -g5000000000 -P99 1
|
||||
fileRoundTripTest -g4193M -P99 1
|
||||
|
||||
|
||||
$ECHO "\n**** zstd long, long distance matching round-trip tests **** "
|
||||
$ECHO "\n===> zstd long, long distance matching round-trip tests "
|
||||
roundTripTest -g270000000 "1 --long"
|
||||
roundTripTest -g130000000 -P60 "5 --long"
|
||||
roundTripTest -g35000000 -P70 "8 --long"
|
||||
@ -697,7 +728,7 @@ roundTripTest -g600M -P50 "1 --long --zstd=wlog=29,clog=28"
|
||||
|
||||
if [ -n "$hasMT" ]
|
||||
then
|
||||
$ECHO "\n**** zstdmt long round-trip tests **** "
|
||||
$ECHO "\n===> zstdmt long round-trip tests "
|
||||
roundTripTest -g80000000 -P99 "19 -T2" " "
|
||||
roundTripTest -g5000000000 -P99 "1 -T2" " "
|
||||
roundTripTest -g500000000 -P97 "1 -T999" " "
|
||||
|
260
sys/contrib/zstd/tests/seqgen.c
Normal file
260
sys/contrib/zstd/tests/seqgen.c
Normal file
@ -0,0 +1,260 @@
|
||||
/*
|
||||
* Copyright (c) 2017-present, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#include "seqgen.h"
|
||||
#include "mem.h"
|
||||
#include <string.h>
|
||||
|
||||
#define MIN(a, b) ((a) < (b) ? (a) : (b))
|
||||
|
||||
static const size_t kMatchBytes = 128;
|
||||
|
||||
#define SEQ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
static BYTE SEQ_randByte(U32* src)
|
||||
{
|
||||
static const U32 prime1 = 2654435761U;
|
||||
static const U32 prime2 = 2246822519U;
|
||||
U32 rand32 = *src;
|
||||
rand32 *= prime1;
|
||||
rand32 ^= prime2;
|
||||
rand32 = SEQ_rotl32(rand32, 13);
|
||||
*src = rand32;
|
||||
return (BYTE)(rand32 >> 5);
|
||||
}
|
||||
|
||||
SEQ_stream SEQ_initStream(unsigned seed)
|
||||
{
|
||||
SEQ_stream stream;
|
||||
stream.state = 0;
|
||||
XXH64_reset(&stream.xxh, 0);
|
||||
stream.seed = seed;
|
||||
return stream;
|
||||
}
|
||||
|
||||
/* Generates a single guard byte, then match length + 1 of a different byte,
|
||||
* then another guard byte.
|
||||
*/
|
||||
static size_t SEQ_gen_matchLength(SEQ_stream* stream, unsigned value,
|
||||
SEQ_outBuffer* out)
|
||||
{
|
||||
typedef enum {
|
||||
ml_first_byte = 0,
|
||||
ml_match_bytes,
|
||||
ml_last_byte,
|
||||
} ml_state;
|
||||
BYTE* const ostart = (BYTE*)out->dst;
|
||||
BYTE* const oend = ostart + out->size;
|
||||
BYTE* op = ostart + out->pos;
|
||||
|
||||
switch ((ml_state)stream->state) {
|
||||
case ml_first_byte:
|
||||
/* Generate a single byte and pick a different byte for the match */
|
||||
if (op >= oend) {
|
||||
stream->bytesLeft = 1;
|
||||
break;
|
||||
}
|
||||
*op = SEQ_randByte(&stream->seed) & 0xFF;
|
||||
do {
|
||||
stream->saved = SEQ_randByte(&stream->seed) & 0xFF;
|
||||
} while (*op == stream->saved);
|
||||
++op;
|
||||
/* State transition */
|
||||
stream->state = ml_match_bytes;
|
||||
stream->bytesLeft = value + 1;
|
||||
/* fall-through */
|
||||
case ml_match_bytes: {
|
||||
/* Copy matchLength + 1 bytes to the output buffer */
|
||||
size_t const setLength = MIN(stream->bytesLeft, (size_t)(oend - op));
|
||||
if (setLength > 0) {
|
||||
memset(op, stream->saved, setLength);
|
||||
op += setLength;
|
||||
stream->bytesLeft -= setLength;
|
||||
}
|
||||
if (stream->bytesLeft > 0)
|
||||
break;
|
||||
/* State transition */
|
||||
stream->state = ml_last_byte;
|
||||
}
|
||||
/* fall-through */
|
||||
case ml_last_byte:
|
||||
/* Generate a single byte and pick a different byte for the match */
|
||||
if (op >= oend) {
|
||||
stream->bytesLeft = 1;
|
||||
break;
|
||||
}
|
||||
do {
|
||||
*op = SEQ_randByte(&stream->seed) & 0xFF;
|
||||
} while (*op == stream->saved);
|
||||
++op;
|
||||
/* State transition */
|
||||
/* fall-through */
|
||||
default:
|
||||
stream->state = 0;
|
||||
stream->bytesLeft = 0;
|
||||
break;
|
||||
}
|
||||
XXH64_update(&stream->xxh, ostart + out->pos, (op - ostart) - out->pos);
|
||||
out->pos = op - ostart;
|
||||
return stream->bytesLeft;
|
||||
}
|
||||
|
||||
/* Saves the current seed then generates kMatchBytes random bytes >= 128.
|
||||
* Generates literal length - kMatchBytes random bytes < 128.
|
||||
* Generates another kMatchBytes using the saved seed to generate a match.
|
||||
* This way the match is easy to find for the compressors.
|
||||
*/
|
||||
static size_t SEQ_gen_litLength(SEQ_stream* stream, unsigned value, SEQ_outBuffer* out)
|
||||
{
|
||||
typedef enum {
|
||||
ll_start = 0,
|
||||
ll_run_bytes,
|
||||
ll_literals,
|
||||
ll_run_match,
|
||||
} ll_state;
|
||||
BYTE* const ostart = (BYTE*)out->dst;
|
||||
BYTE* const oend = ostart + out->size;
|
||||
BYTE* op = ostart + out->pos;
|
||||
|
||||
switch ((ll_state)stream->state) {
|
||||
case ll_start:
|
||||
stream->state = ll_run_bytes;
|
||||
stream->saved = stream->seed;
|
||||
stream->bytesLeft = MIN(kMatchBytes, value);
|
||||
/* fall-through */
|
||||
case ll_run_bytes:
|
||||
while (stream->bytesLeft > 0 && op < oend) {
|
||||
*op++ = SEQ_randByte(&stream->seed) | 0x80;
|
||||
--stream->bytesLeft;
|
||||
}
|
||||
if (stream->bytesLeft > 0)
|
||||
break;
|
||||
/* State transition */
|
||||
stream->state = ll_literals;
|
||||
stream->bytesLeft = value - MIN(kMatchBytes, value);
|
||||
/* fall-through */
|
||||
case ll_literals:
|
||||
while (stream->bytesLeft > 0 && op < oend) {
|
||||
*op++ = SEQ_randByte(&stream->seed) & 0x7F;
|
||||
--stream->bytesLeft;
|
||||
}
|
||||
if (stream->bytesLeft > 0)
|
||||
break;
|
||||
/* State transition */
|
||||
stream->state = ll_run_match;
|
||||
stream->bytesLeft = MIN(kMatchBytes, value);
|
||||
/* fall-through */
|
||||
case ll_run_match: {
|
||||
while (stream->bytesLeft > 0 && op < oend) {
|
||||
*op++ = SEQ_randByte(&stream->saved) | 0x80;
|
||||
--stream->bytesLeft;
|
||||
}
|
||||
if (stream->bytesLeft > 0)
|
||||
break;
|
||||
}
|
||||
/* fall-through */
|
||||
default:
|
||||
stream->state = 0;
|
||||
stream->bytesLeft = 0;
|
||||
break;
|
||||
}
|
||||
XXH64_update(&stream->xxh, ostart + out->pos, (op - ostart) - out->pos);
|
||||
out->pos = op - ostart;
|
||||
return stream->bytesLeft;
|
||||
}
|
||||
|
||||
/* Saves the current seed then generates kMatchBytes random bytes >= 128.
|
||||
* Generates offset - kMatchBytes of zeros to get a large offset without
|
||||
* polluting the hash tables.
|
||||
* Generates another kMatchBytes using the saved seed to generate a with the
|
||||
* required offset.
|
||||
*/
|
||||
static size_t SEQ_gen_offset(SEQ_stream* stream, unsigned value, SEQ_outBuffer* out)
|
||||
{
|
||||
typedef enum {
|
||||
of_start = 0,
|
||||
of_run_bytes,
|
||||
of_offset,
|
||||
of_run_match,
|
||||
} of_state;
|
||||
BYTE* const ostart = (BYTE*)out->dst;
|
||||
BYTE* const oend = ostart + out->size;
|
||||
BYTE* op = ostart + out->pos;
|
||||
|
||||
switch ((of_state)stream->state) {
|
||||
case of_start:
|
||||
stream->state = of_run_bytes;
|
||||
stream->saved = stream->seed;
|
||||
stream->bytesLeft = MIN(value, kMatchBytes);
|
||||
/* fall-through */
|
||||
case of_run_bytes: {
|
||||
while (stream->bytesLeft > 0 && op < oend) {
|
||||
*op++ = SEQ_randByte(&stream->seed) | 0x80;
|
||||
--stream->bytesLeft;
|
||||
}
|
||||
if (stream->bytesLeft > 0)
|
||||
break;
|
||||
/* State transition */
|
||||
stream->state = of_offset;
|
||||
stream->bytesLeft = value - MIN(value, kMatchBytes);
|
||||
}
|
||||
/* fall-through */
|
||||
case of_offset: {
|
||||
/* Copy matchLength + 1 bytes to the output buffer */
|
||||
size_t const setLength = MIN(stream->bytesLeft, (size_t)(oend - op));
|
||||
if (setLength > 0) {
|
||||
memset(op, 0, setLength);
|
||||
op += setLength;
|
||||
stream->bytesLeft -= setLength;
|
||||
}
|
||||
if (stream->bytesLeft > 0)
|
||||
break;
|
||||
/* State transition */
|
||||
stream->state = of_run_match;
|
||||
stream->bytesLeft = MIN(value, kMatchBytes);
|
||||
}
|
||||
/* fall-through */
|
||||
case of_run_match: {
|
||||
while (stream->bytesLeft > 0 && op < oend) {
|
||||
*op++ = SEQ_randByte(&stream->saved) | 0x80;
|
||||
--stream->bytesLeft;
|
||||
}
|
||||
if (stream->bytesLeft > 0)
|
||||
break;
|
||||
}
|
||||
/* fall-through */
|
||||
default:
|
||||
stream->state = 0;
|
||||
stream->bytesLeft = 0;
|
||||
break;
|
||||
}
|
||||
XXH64_update(&stream->xxh, ostart + out->pos, (op - ostart) - out->pos);
|
||||
out->pos = op - ostart;
|
||||
return stream->bytesLeft;
|
||||
}
|
||||
|
||||
/* Returns the number of bytes left to generate.
|
||||
* Must pass the same type/value until it returns 0.
|
||||
*/
|
||||
size_t SEQ_gen(SEQ_stream* stream, SEQ_gen_type type, unsigned value, SEQ_outBuffer* out)
|
||||
{
|
||||
switch (type) {
|
||||
case SEQ_gen_ml: return SEQ_gen_matchLength(stream, value, out);
|
||||
case SEQ_gen_ll: return SEQ_gen_litLength(stream, value, out);
|
||||
case SEQ_gen_of: return SEQ_gen_offset(stream, value, out);
|
||||
case SEQ_gen_max: /* fall-through */
|
||||
default: return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* Returns the xxhash of the data produced so far */
|
||||
XXH64_hash_t SEQ_digest(SEQ_stream const* stream)
|
||||
{
|
||||
return XXH64_digest(&stream->xxh);
|
||||
}
|
58
sys/contrib/zstd/tests/seqgen.h
Normal file
58
sys/contrib/zstd/tests/seqgen.h
Normal file
@ -0,0 +1,58 @@
|
||||
/*
|
||||
* Copyright (c) 2017-present, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef SEQGEN_H
|
||||
#define SEQGEN_H
|
||||
|
||||
#define XXH_STATIC_LINKING_ONLY
|
||||
|
||||
#include "xxhash.h"
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
typedef enum {
|
||||
SEQ_gen_ml = 0,
|
||||
SEQ_gen_ll,
|
||||
SEQ_gen_of,
|
||||
SEQ_gen_max /* Must be the last value */
|
||||
} SEQ_gen_type;
|
||||
|
||||
/* Internal state, do not use */
|
||||
typedef struct {
|
||||
XXH64_state_t xxh; /* xxh state for all the data produced so far (seed=0) */
|
||||
unsigned seed;
|
||||
int state; /* enum to control state machine (clean=0) */
|
||||
unsigned saved;
|
||||
size_t bytesLeft;
|
||||
} SEQ_stream;
|
||||
|
||||
SEQ_stream SEQ_initStream(unsigned seed);
|
||||
|
||||
typedef struct {
|
||||
void* dst;
|
||||
size_t size;
|
||||
size_t pos;
|
||||
} SEQ_outBuffer;
|
||||
|
||||
/* Returns non-zero until the current type/value has been generated.
|
||||
* Must pass the same type/value until it returns 0.
|
||||
*
|
||||
* Recommended to pick a value in the middle of the range you want, since there
|
||||
* may be some noise that causes actual results to be slightly different.
|
||||
* We try to be more accurate for smaller values.
|
||||
*
|
||||
* NOTE: Very small values don't work well (< 6).
|
||||
*/
|
||||
size_t SEQ_gen(SEQ_stream* stream, SEQ_gen_type type, unsigned value,
|
||||
SEQ_outBuffer* out);
|
||||
|
||||
/* Returns the xxhash of the data produced so far */
|
||||
XXH64_hash_t SEQ_digest(SEQ_stream const* stream);
|
||||
|
||||
#endif /* SEQGEN_H */
|
@ -24,7 +24,6 @@
|
||||
**************************************/
|
||||
#include <stdlib.h> /* free */
|
||||
#include <stdio.h> /* fgets, sscanf */
|
||||
#include <time.h> /* clock_t, clock() */
|
||||
#include <string.h> /* strcmp */
|
||||
#include "mem.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */
|
||||
@ -34,6 +33,7 @@
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#define XXH_STATIC_LINKING_ONLY
|
||||
#include "xxhash.h" /* XXH64_* */
|
||||
#include "util.h"
|
||||
|
||||
|
||||
/*-************************************
|
||||
@ -58,26 +58,24 @@ static const U32 prime2 = 2246822519U;
|
||||
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
static U32 g_displayLevel = 2;
|
||||
|
||||
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
|
||||
if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
|
||||
{ g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
|
||||
if (g_displayLevel>=4) fflush(stderr); } }
|
||||
static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
||||
static clock_t g_displayClock = 0;
|
||||
static const U64 g_refreshRate = SEC_TO_MICRO / 6;
|
||||
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
|
||||
|
||||
static clock_t g_clockTime = 0;
|
||||
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
|
||||
if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
|
||||
{ g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
|
||||
if (g_displayLevel>=4) fflush(stderr); } }
|
||||
|
||||
static U64 g_clockTime = 0;
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Fuzzer functions
|
||||
*********************************************************/
|
||||
#undef MIN
|
||||
#undef MAX
|
||||
#define MIN(a,b) ((a)<(b)?(a):(b))
|
||||
#define MAX(a,b) ((a)>(b)?(a):(b))
|
||||
|
||||
static clock_t FUZ_GetClockSpan(clock_t clockStart)
|
||||
{
|
||||
return clock() - clockStart; /* works even when overflow. Max span ~ 30 mn */
|
||||
}
|
||||
|
||||
/*! FUZ_rand() :
|
||||
@return : a 27 bits random value, from a 32-bits `seed`.
|
||||
`seed` is also modified */
|
||||
@ -256,8 +254,6 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog)
|
||||
return FUZ_rLogLength(seed, logLength);
|
||||
}
|
||||
|
||||
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
|
||||
|
||||
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
|
||||
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
|
||||
|
||||
@ -278,7 +274,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
|
||||
U32 coreSeed = seed;
|
||||
ZBUFF_CCtx* zc;
|
||||
ZBUFF_DCtx* zd;
|
||||
clock_t startClock = clock();
|
||||
UTIL_time_t startClock = UTIL_getTime();
|
||||
|
||||
/* allocations */
|
||||
zc = ZBUFF_createCCtx();
|
||||
@ -308,7 +304,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
|
||||
FUZ_rand(&coreSeed);
|
||||
|
||||
/* test loop */
|
||||
for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) {
|
||||
for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
|
||||
U32 lseed;
|
||||
const BYTE* srcBuffer;
|
||||
const BYTE* dict;
|
||||
@ -357,7 +353,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
|
||||
{ ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictSize);
|
||||
params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
|
||||
params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
|
||||
{ size_t const initError = ZBUFF_compressInit_advanced(zc, dict, dictSize, params, 0);
|
||||
{ size_t const initError = ZBUFF_compressInit_advanced(zc, dict, dictSize, params, ZSTD_CONTENTSIZE_UNKNOWN);
|
||||
CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError));
|
||||
} } }
|
||||
|
||||
@ -548,7 +544,7 @@ int main(int argc, const char** argv)
|
||||
}
|
||||
if (*argument=='m') g_clockTime *=60, argument++;
|
||||
if (*argument=='n') argument++;
|
||||
g_clockTime *= CLOCKS_PER_SEC;
|
||||
g_clockTime *= SEC_TO_MICRO;
|
||||
break;
|
||||
|
||||
case 's':
|
||||
|
@ -24,7 +24,6 @@
|
||||
**************************************/
|
||||
#include <stdlib.h> /* free */
|
||||
#include <stdio.h> /* fgets, sscanf */
|
||||
#include <time.h> /* clock_t, clock() */
|
||||
#include <string.h> /* strcmp */
|
||||
#include <assert.h> /* assert */
|
||||
#include "mem.h"
|
||||
@ -36,6 +35,8 @@
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
|
||||
#include "xxhash.h" /* XXH64_* */
|
||||
#include "seqgen.h"
|
||||
#include "util.h"
|
||||
|
||||
|
||||
/*-************************************
|
||||
@ -61,26 +62,24 @@ static const U32 prime32 = 2654435761U;
|
||||
if (g_displayLevel>=4) fflush(stderr); }
|
||||
static U32 g_displayLevel = 2;
|
||||
|
||||
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
|
||||
if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
|
||||
{ g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
|
||||
if (g_displayLevel>=4) fflush(stderr); } }
|
||||
static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
|
||||
static clock_t g_displayClock = 0;
|
||||
static const U64 g_refreshRate = SEC_TO_MICRO / 6;
|
||||
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
|
||||
|
||||
static clock_t g_clockTime = 0;
|
||||
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
|
||||
if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
|
||||
{ g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
|
||||
if (g_displayLevel>=4) fflush(stderr); } }
|
||||
|
||||
static U64 g_clockTime = 0;
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Fuzzer functions
|
||||
*********************************************************/
|
||||
#undef MIN
|
||||
#undef MAX
|
||||
#define MIN(a,b) ((a)<(b)?(a):(b))
|
||||
#define MAX(a,b) ((a)>(b)?(a):(b))
|
||||
|
||||
static clock_t FUZ_GetClockSpan(clock_t clockStart)
|
||||
{
|
||||
return clock() - clockStart; /* works even when overflow. Max span ~ 30 mn */
|
||||
}
|
||||
|
||||
/*! FUZ_rand() :
|
||||
@return : a 27 bits random value, from a 32-bits `seed`.
|
||||
`seed` is also modified */
|
||||
@ -96,15 +95,21 @@ unsigned int FUZ_rand(unsigned int* seedPtr)
|
||||
return rand32 >> 5;
|
||||
}
|
||||
|
||||
#define CHECK_Z(f) { \
|
||||
size_t const err = f; \
|
||||
if (ZSTD_isError(err)) { \
|
||||
DISPLAY("Error => %s : %s ", \
|
||||
#f, ZSTD_getErrorName(err)); \
|
||||
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); \
|
||||
#define CHECK(cond, ...) { \
|
||||
if (cond) { \
|
||||
DISPLAY("Error => "); \
|
||||
DISPLAY(__VA_ARGS__); \
|
||||
DISPLAY(" (seed %u, test nb %u, line %u) \n", \
|
||||
seed, testNb, __LINE__); \
|
||||
goto _output_error; \
|
||||
} }
|
||||
|
||||
#define CHECK_Z(f) { \
|
||||
size_t const err = f; \
|
||||
CHECK(ZSTD_isError(err), "%s : %s ", \
|
||||
#f, ZSTD_getErrorName(err)); \
|
||||
}
|
||||
|
||||
|
||||
/*======================================================
|
||||
* Basic Unit tests
|
||||
@ -144,12 +149,69 @@ static void FUZ_freeDictionary(buffer_t dict)
|
||||
free(dict.start);
|
||||
}
|
||||
|
||||
/* Round trips data and updates xxh with the decompressed data produced */
|
||||
static size_t SEQ_roundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
|
||||
XXH64_state_t* xxh, void* data, size_t size,
|
||||
ZSTD_EndDirective endOp)
|
||||
{
|
||||
static BYTE compressed[1024];
|
||||
static BYTE uncompressed[1024];
|
||||
|
||||
static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem)
|
||||
ZSTD_inBuffer cin = {data, size, 0};
|
||||
size_t cret;
|
||||
|
||||
do {
|
||||
ZSTD_outBuffer cout = {compressed, sizeof(compressed), 0};
|
||||
ZSTD_inBuffer din = {compressed, 0, 0};
|
||||
ZSTD_outBuffer dout = {uncompressed, 0, 0};
|
||||
|
||||
cret = ZSTD_compress_generic(cctx, &cout, &cin, endOp);
|
||||
if (ZSTD_isError(cret))
|
||||
return cret;
|
||||
|
||||
din.size = cout.pos;
|
||||
while (din.pos < din.size || (endOp == ZSTD_e_end && cret == 0)) {
|
||||
size_t dret;
|
||||
|
||||
dout.pos = 0;
|
||||
dout.size = sizeof(uncompressed);
|
||||
dret = ZSTD_decompressStream(dctx, &dout, &din);
|
||||
if (ZSTD_isError(dret))
|
||||
return dret;
|
||||
XXH64_update(xxh, dout.dst, dout.pos);
|
||||
if (dret == 0)
|
||||
break;
|
||||
}
|
||||
} while (cin.pos < cin.size || (endOp != ZSTD_e_continue && cret != 0));
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Generates some data and round trips it */
|
||||
static size_t SEQ_generateRoundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
|
||||
XXH64_state_t* xxh, SEQ_stream* seq,
|
||||
SEQ_gen_type type, unsigned value)
|
||||
{
|
||||
static BYTE data[1024];
|
||||
size_t gen;
|
||||
|
||||
do {
|
||||
SEQ_outBuffer sout = {data, sizeof(data), 0};
|
||||
size_t ret;
|
||||
gen = SEQ_gen(seq, type, value, &sout);
|
||||
|
||||
ret = SEQ_roundTrip(cctx, dctx, xxh, sout.dst, sout.pos, ZSTD_e_continue);
|
||||
if (ZSTD_isError(ret))
|
||||
return ret;
|
||||
} while (gen != 0);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int basicUnitTests(U32 seed, double compressibility)
|
||||
{
|
||||
size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
|
||||
void* CNBuffer = malloc(CNBufferSize);
|
||||
size_t const skippableFrameSize = 11;
|
||||
size_t const skippableFrameSize = 200 KB;
|
||||
size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
|
||||
void* compressedBuffer = malloc(compressedBufferSize);
|
||||
size_t const decodedBufferSize = CNBufferSize;
|
||||
@ -157,8 +219,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
size_t cSize;
|
||||
int testResult = 0;
|
||||
U32 testNb = 1;
|
||||
ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem);
|
||||
ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem);
|
||||
ZSTD_CStream* zc = ZSTD_createCStream();
|
||||
ZSTD_DStream* zd = ZSTD_createDStream();
|
||||
ZSTDMT_CCtx* mtctx = ZSTDMT_createCCtx(2);
|
||||
|
||||
ZSTD_inBuffer inBuff, inBuff2;
|
||||
@ -183,16 +245,32 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
}
|
||||
dictID = ZDICT_getDictID(dictionary.start, dictionary.filled);
|
||||
|
||||
/* Basic compression test */
|
||||
DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||
CHECK_Z( ZSTD_initCStream(zc, 1 /* cLevel */) );
|
||||
outBuff.dst = (char*)(compressedBuffer);
|
||||
outBuff.size = compressedBufferSize;
|
||||
outBuff.pos = 0;
|
||||
inBuff.src = CNBuffer;
|
||||
inBuff.size = CNBufferSize;
|
||||
inBuff.pos = 0;
|
||||
CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
|
||||
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
|
||||
{ size_t const r = ZSTD_endStream(zc, &outBuff);
|
||||
if (r != 0) goto _output_error; } /* error, or some data not flushed */
|
||||
DISPLAYLEVEL(3, "OK (%u bytes)\n", (U32)outBuff.pos);
|
||||
|
||||
/* generate skippable frame */
|
||||
MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
|
||||
MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
|
||||
cSize = skippableFrameSize + 8;
|
||||
|
||||
/* Basic compression test */
|
||||
DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||
CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1) );
|
||||
/* Basic compression test using dict */
|
||||
DISPLAYLEVEL(3, "test%3i : skipframe + compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||
CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1 /* cLevel */) );
|
||||
outBuff.dst = (char*)(compressedBuffer)+cSize;
|
||||
outBuff.size = compressedBufferSize;
|
||||
assert(compressedBufferSize > cSize);
|
||||
outBuff.size = compressedBufferSize - cSize;
|
||||
outBuff.pos = 0;
|
||||
inBuff.src = CNBuffer;
|
||||
inBuff.size = CNBufferSize;
|
||||
@ -366,7 +444,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
/* Complex context re-use scenario */
|
||||
DISPLAYLEVEL(3, "test%3i : context re-use : ", testNb++);
|
||||
ZSTD_freeCStream(zc);
|
||||
zc = ZSTD_createCStream_advanced(customMem);
|
||||
zc = ZSTD_createCStream();
|
||||
if (zc==NULL) goto _output_error; /* memory allocation issue */
|
||||
/* use 1 */
|
||||
{ size_t const inSize = 513;
|
||||
@ -408,6 +486,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
DISPLAYLEVEL(3, "test%3i : digested dictionary : ", testNb++);
|
||||
{ ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1 /*byRef*/ );
|
||||
size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
|
||||
DISPLAYLEVEL(5, "ZSTD_initCStream_usingCDict result : %u ", (U32)initError);
|
||||
if (ZSTD_isError(initError)) goto _output_error;
|
||||
cSize = 0;
|
||||
outBuff.dst = compressedBuffer;
|
||||
@ -416,10 +495,13 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
inBuff.src = CNBuffer;
|
||||
inBuff.size = CNBufferSize;
|
||||
inBuff.pos = 0;
|
||||
DISPLAYLEVEL(5, "- starting ZSTD_compressStream ");
|
||||
CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
|
||||
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
|
||||
{ size_t const r = ZSTD_endStream(zc, &outBuff);
|
||||
if (r != 0) goto _output_error; } /* error, or some data not flushed */
|
||||
{ size_t const r = ZSTD_endStream(zc, &outBuff);
|
||||
DISPLAYLEVEL(5, "- ZSTD_endStream result : %u ", (U32)r);
|
||||
if (r != 0) goto _output_error; /* error, or some data not flushed */
|
||||
}
|
||||
cSize = outBuff.pos;
|
||||
ZSTD_freeCDict(cdict);
|
||||
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
|
||||
@ -442,12 +524,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
{ ZSTD_DDict* const ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
|
||||
size_t const initError = ZSTD_initDStream_usingDDict(zd, ddict);
|
||||
if (ZSTD_isError(initError)) goto _output_error;
|
||||
inBuff.src = compressedBuffer;
|
||||
inBuff.size = cSize;
|
||||
inBuff.pos = 0;
|
||||
outBuff.dst = decodedBuffer;
|
||||
outBuff.size = CNBufferSize;
|
||||
outBuff.pos = 0;
|
||||
inBuff.src = compressedBuffer;
|
||||
inBuff.size = cSize;
|
||||
inBuff.pos = 0;
|
||||
{ size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
|
||||
if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
|
||||
if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
|
||||
@ -466,12 +548,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
DISPLAYLEVEL(3, "test%3i : maxWindowSize < frame requirement : ", testNb++);
|
||||
ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
|
||||
CHECK_Z( ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000) ); /* too small limit */
|
||||
inBuff.src = compressedBuffer;
|
||||
inBuff.size = cSize;
|
||||
inBuff.pos = 0;
|
||||
outBuff.dst = decodedBuffer;
|
||||
outBuff.size = CNBufferSize;
|
||||
outBuff.pos = 0;
|
||||
inBuff.src = compressedBuffer;
|
||||
inBuff.size = cSize;
|
||||
inBuff.pos = 0;
|
||||
{ size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
|
||||
if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */
|
||||
DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
|
||||
@ -479,7 +561,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++);
|
||||
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled);
|
||||
ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */};
|
||||
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dm_auto, cParams, customMem);
|
||||
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dm_auto, cParams, ZSTD_defaultCMem);
|
||||
size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize);
|
||||
if (ZSTD_isError(initError)) goto _output_error;
|
||||
cSize = 0;
|
||||
@ -558,14 +640,14 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++);
|
||||
{ ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
|
||||
params.fParams.contentSizeFlag = 1;
|
||||
CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0) );
|
||||
CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0 /* pledgedSrcSize==0 means "empty" when params.fParams.contentSizeFlag is set */) );
|
||||
} /* cstream advanced shall write content size = 0 */
|
||||
inBuff.src = CNBuffer;
|
||||
inBuff.size = 0;
|
||||
inBuff.pos = 0;
|
||||
outBuff.dst = compressedBuffer;
|
||||
outBuff.size = compressedBufferSize;
|
||||
outBuff.pos = 0;
|
||||
inBuff.src = CNBuffer;
|
||||
inBuff.size = 0;
|
||||
inBuff.pos = 0;
|
||||
CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
|
||||
if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
|
||||
cSize = outBuff.pos;
|
||||
@ -589,12 +671,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
|
||||
|
||||
ZSTD_resetCStream(zc, 0); /* resetCStream should treat 0 as unknown */
|
||||
inBuff.src = CNBuffer;
|
||||
inBuff.size = 0;
|
||||
inBuff.pos = 0;
|
||||
outBuff.dst = compressedBuffer;
|
||||
outBuff.size = compressedBufferSize;
|
||||
outBuff.pos = 0;
|
||||
inBuff.src = CNBuffer;
|
||||
inBuff.size = 0;
|
||||
inBuff.pos = 0;
|
||||
CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
|
||||
if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
|
||||
cSize = outBuff.pos;
|
||||
@ -606,7 +688,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
{ ZSTD_parameters const params = ZSTD_getParams(1, 0, 0);
|
||||
CHECK_Z( ZSTDMT_initCStream_advanced(mtctx, CNBuffer, dictSize, params, CNBufferSize) );
|
||||
}
|
||||
outBuff.dst = (char*)(compressedBuffer);
|
||||
outBuff.dst = compressedBuffer;
|
||||
outBuff.size = compressedBufferSize;
|
||||
outBuff.pos = 0;
|
||||
inBuff.src = CNBuffer;
|
||||
@ -618,6 +700,108 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
if (r != 0) goto _output_error; } /* error, or some data not flushed */
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
/* Complex multithreading + dictionary test */
|
||||
{ U32 const nbThreads = 2;
|
||||
size_t const jobSize = 4 * 1 MB;
|
||||
size_t const srcSize = jobSize * nbThreads; /* we want each job to have predictable size */
|
||||
size_t const segLength = 2 KB;
|
||||
size_t const offset = 600 KB; /* must be larger than window defined in cdict */
|
||||
size_t const start = jobSize + (offset-1);
|
||||
const BYTE* const srcToCopy = (const BYTE*)CNBuffer + start;
|
||||
BYTE* const dst = (BYTE*)CNBuffer + start - offset;
|
||||
DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads + dictionary : ", testNb++, (U32)srcSize);
|
||||
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_compressionLevel, 3) );
|
||||
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_nbThreads, 2) );
|
||||
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_jobSize, jobSize) );
|
||||
assert(start > offset);
|
||||
assert(start + segLength < COMPRESSIBLE_NOISE_LENGTH);
|
||||
memcpy(dst, srcToCopy, segLength); /* create a long repetition at long distance for job 2 */
|
||||
outBuff.dst = compressedBuffer;
|
||||
outBuff.size = compressedBufferSize;
|
||||
outBuff.pos = 0;
|
||||
inBuff.src = CNBuffer;
|
||||
inBuff.size = srcSize; assert(srcSize < COMPRESSIBLE_NOISE_LENGTH);
|
||||
inBuff.pos = 0;
|
||||
}
|
||||
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled); /* intentionnally lies on estimatedSrcSize, to push cdict into targeting a small window size */
|
||||
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem);
|
||||
DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog);
|
||||
CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) );
|
||||
CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) );
|
||||
CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) ); /* do not keep a reference to cdict, as its lifetime ends */
|
||||
ZSTD_freeCDict(cdict);
|
||||
}
|
||||
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
|
||||
cSize = outBuff.pos;
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : decompress large frame created from multiple threads + dictionary : ", testNb++);
|
||||
{ ZSTD_DStream* const dstream = ZSTD_createDCtx();
|
||||
ZSTD_frameHeader zfh;
|
||||
ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize);
|
||||
DISPLAYLEVEL(5, "frame windowsize = %u : ", (U32)zfh.windowSize);
|
||||
outBuff.dst = decodedBuffer;
|
||||
outBuff.size = CNBufferSize;
|
||||
outBuff.pos = 0;
|
||||
inBuff.src = compressedBuffer;
|
||||
inBuff.pos = 0;
|
||||
CHECK_Z( ZSTD_initDStream_usingDict(dstream, dictionary.start, dictionary.filled) );
|
||||
inBuff.size = 1; /* avoid shortcut to single-pass mode */
|
||||
CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
|
||||
inBuff.size = cSize;
|
||||
CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
|
||||
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
|
||||
ZSTD_freeDStream(dstream);
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : check dictionary FSE tables can represent every code : ", testNb++);
|
||||
{ unsigned const kMaxWindowLog = 24;
|
||||
unsigned value;
|
||||
ZSTD_compressionParameters cParams = ZSTD_getCParams(3, 1U << kMaxWindowLog, 1024);
|
||||
ZSTD_CDict* cdict;
|
||||
ZSTD_DDict* ddict;
|
||||
SEQ_stream seq = SEQ_initStream(0x87654321);
|
||||
SEQ_gen_type type;
|
||||
XXH64_state_t xxh;
|
||||
|
||||
XXH64_reset(&xxh, 0);
|
||||
cParams.windowLog = kMaxWindowLog;
|
||||
cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem);
|
||||
ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
|
||||
|
||||
if (!cdict || !ddict) goto _output_error;
|
||||
|
||||
ZSTD_CCtx_reset(zc);
|
||||
ZSTD_resetDStream(zd);
|
||||
CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
|
||||
CHECK_Z(ZSTD_initDStream_usingDDict(zd, ddict));
|
||||
CHECK_Z(ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1U << kMaxWindowLog));
|
||||
/* Test all values < 300 */
|
||||
for (value = 0; value < 300; ++value) {
|
||||
for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
|
||||
CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
|
||||
}
|
||||
}
|
||||
/* Test values 2^8 to 2^17 */
|
||||
for (value = (1 << 8); value < (1 << 17); value <<= 1) {
|
||||
for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
|
||||
CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
|
||||
CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value + (value >> 2)));
|
||||
}
|
||||
}
|
||||
/* Test offset values up to the max window log */
|
||||
for (value = 8; value <= kMaxWindowLog; ++value) {
|
||||
CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, SEQ_gen_of, (1U << value) - 1));
|
||||
}
|
||||
|
||||
CHECK_Z(SEQ_roundTrip(zc, zd, &xxh, NULL, 0, ZSTD_e_end));
|
||||
CHECK(SEQ_digest(&seq) != XXH64_digest(&xxh), "SEQ XXH64 does not match");
|
||||
|
||||
ZSTD_freeCDict(cdict);
|
||||
ZSTD_freeDDict(ddict);
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
/* Overlen overwriting window data bug */
|
||||
DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++);
|
||||
@ -699,8 +883,6 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog)
|
||||
return FUZ_rLogLength(seed, logLength);
|
||||
}
|
||||
|
||||
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
|
||||
|
||||
/* Return value in range minVal <= v <= maxVal */
|
||||
static U32 FUZ_randomClampedLength(U32* seed, U32 minVal, U32 maxVal)
|
||||
{
|
||||
@ -708,14 +890,6 @@ static U32 FUZ_randomClampedLength(U32* seed, U32 minVal, U32 maxVal)
|
||||
return (U32)((FUZ_rand(seed) % mod) + minVal);
|
||||
}
|
||||
|
||||
#define CHECK(cond, ...) { \
|
||||
if (cond) { \
|
||||
DISPLAY("Error => "); \
|
||||
DISPLAY(__VA_ARGS__); \
|
||||
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); \
|
||||
goto _output_error; \
|
||||
} }
|
||||
|
||||
static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests)
|
||||
{
|
||||
U32 const maxSrcLog = bigTests ? 24 : 22;
|
||||
@ -734,7 +908,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
|
||||
ZSTD_CStream* zc = ZSTD_createCStream(); /* will be re-created sometimes */
|
||||
ZSTD_DStream* zd = ZSTD_createDStream(); /* will be re-created sometimes */
|
||||
ZSTD_DStream* const zd_noise = ZSTD_createDStream();
|
||||
clock_t const startClock = clock();
|
||||
UTIL_time_t const startClock = UTIL_getTime();
|
||||
const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
|
||||
size_t dictSize = 0;
|
||||
U32 oldTestLog = 0;
|
||||
@ -764,7 +938,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
|
||||
FUZ_rand(&coreSeed);
|
||||
|
||||
/* test loop */
|
||||
for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) {
|
||||
for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
|
||||
U32 lseed;
|
||||
const BYTE* srcBuffer;
|
||||
size_t totalTestSize, totalGenSize, cSize;
|
||||
@ -832,10 +1006,11 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
|
||||
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
|
||||
dict = srcBuffer + dictStart;
|
||||
}
|
||||
{ U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
|
||||
{ U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
|
||||
ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
|
||||
params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
|
||||
params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
|
||||
params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1;
|
||||
CHECK_Z ( ZSTD_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
|
||||
} }
|
||||
|
||||
@ -846,7 +1021,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
|
||||
for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) {
|
||||
/* compress random chunks into randomly sized dst buffers */
|
||||
{ size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const srcSize = MIN (maxTestSize-totalTestSize, randomSrcSize);
|
||||
size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
|
||||
size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
|
||||
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
|
||||
@ -983,7 +1158,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
|
||||
ZSTDMT_CCtx* zc = ZSTDMT_createCCtx(nbThreads); /* will be reset sometimes */
|
||||
ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */
|
||||
ZSTD_DStream* const zd_noise = ZSTD_createDStream();
|
||||
clock_t const startClock = clock();
|
||||
UTIL_time_t const startClock = UTIL_getTime();
|
||||
const BYTE* dict=NULL; /* can keep same dict on 2 consecutive tests */
|
||||
size_t dictSize = 0;
|
||||
U32 oldTestLog = 0;
|
||||
@ -1014,7 +1189,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
|
||||
FUZ_rand(&coreSeed);
|
||||
|
||||
/* test loop */
|
||||
for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) {
|
||||
for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
|
||||
U32 lseed;
|
||||
const BYTE* srcBuffer;
|
||||
size_t totalTestSize, totalGenSize, cSize;
|
||||
@ -1086,17 +1261,17 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
|
||||
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
|
||||
dict = srcBuffer + dictStart;
|
||||
}
|
||||
{ U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
|
||||
{ U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
|
||||
ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
|
||||
DISPLAYLEVEL(5, "Init with windowLog = %u and pledgedSrcSize = %u \n",
|
||||
params.cParams.windowLog, (U32)pledgedSrcSize);
|
||||
DISPLAYLEVEL(5, "Init with windowLog = %u, pledgedSrcSize = %u, dictSize = %u \n",
|
||||
params.cParams.windowLog, (U32)pledgedSrcSize, (U32)dictSize);
|
||||
params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
|
||||
params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
|
||||
params.fParams.contentSizeFlag = pledgedSrcSize>0;
|
||||
params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1;
|
||||
DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag);
|
||||
CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
|
||||
CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12) );
|
||||
CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_sectionSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) );
|
||||
CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_jobSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) ); /* custome job size */
|
||||
CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
|
||||
} }
|
||||
|
||||
/* multi-segments compression test */
|
||||
@ -1113,9 +1288,9 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
|
||||
ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
|
||||
outBuff.size = outBuff.pos + dstBuffSize;
|
||||
|
||||
DISPLAYLEVEL(5, "Sending %u bytes to compress \n", (U32)srcSize);
|
||||
DISPLAYLEVEL(6, "Sending %u bytes to compress \n", (U32)srcSize);
|
||||
CHECK_Z( ZSTDMT_compressStream(zc, &outBuff, &inBuff) );
|
||||
DISPLAYLEVEL(5, "%u bytes read by ZSTDMT_compressStream \n", (U32)inBuff.pos);
|
||||
DISPLAYLEVEL(6, "%u bytes read by ZSTDMT_compressStream \n", (U32)inBuff.pos);
|
||||
|
||||
XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
|
||||
memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
|
||||
@ -1162,10 +1337,10 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
|
||||
size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
|
||||
inBuff.size = inBuff.pos + readCSrcSize;
|
||||
outBuff.size = outBuff.pos + dstBuffSize;
|
||||
DISPLAYLEVEL(5, "ZSTD_decompressStream input %u bytes \n", (U32)readCSrcSize);
|
||||
DISPLAYLEVEL(6, "ZSTD_decompressStream input %u bytes \n", (U32)readCSrcSize);
|
||||
decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
|
||||
CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
|
||||
DISPLAYLEVEL(5, "inBuff.pos = %u \n", (U32)readCSrcSize);
|
||||
DISPLAYLEVEL(6, "inBuff.pos = %u \n", (U32)readCSrcSize);
|
||||
}
|
||||
CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (U32)outBuff.pos, (U32)totalTestSize);
|
||||
CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (U32)inBuff.pos, (U32)cSize);
|
||||
@ -1255,7 +1430,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
|
||||
ZSTD_CCtx* zc = ZSTD_createCCtx(); /* will be reset sometimes */
|
||||
ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */
|
||||
ZSTD_DStream* const zd_noise = ZSTD_createDStream();
|
||||
clock_t const startClock = clock();
|
||||
UTIL_time_t const startClock = UTIL_getTime();
|
||||
const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
|
||||
size_t dictSize = 0;
|
||||
U32 oldTestLog = 0;
|
||||
@ -1288,7 +1463,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
|
||||
FUZ_rand(&coreSeed);
|
||||
|
||||
/* test loop */
|
||||
for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) {
|
||||
for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
|
||||
U32 lseed;
|
||||
const BYTE* srcBuffer;
|
||||
size_t totalTestSize, totalGenSize, cSize;
|
||||
@ -1352,7 +1527,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
|
||||
(MAX(testLog, dictLog) / 2))) +
|
||||
1;
|
||||
U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
|
||||
DISPLAYLEVEL(5, "t%u: cLevel : %u \n", testNb, cLevel);
|
||||
DISPLAYLEVEL(5, "t%u: base cLevel : %u \n", testNb, cLevel);
|
||||
maxTestSize = FUZ_rLogLength(&lseed, testLog);
|
||||
DISPLAYLEVEL(5, "t%u: maxTestSize : %u \n", testNb, (U32)maxTestSize);
|
||||
oldTestLog = testLog;
|
||||
@ -1377,43 +1552,47 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
|
||||
cParams = ZSTD_adjustCParams(cParams, 0, 0);
|
||||
|
||||
if (FUZ_rand(&lseed) & 1) {
|
||||
DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_windowLog, cParams.windowLog, useOpaqueAPI) );
|
||||
assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN); /* guaranteed by ZSTD_adjustCParams() */
|
||||
windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5;
|
||||
DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
|
||||
}
|
||||
if (FUZ_rand(&lseed) & 1) {
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, useOpaqueAPI) );
|
||||
DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog);
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, useOpaqueAPI) );
|
||||
}
|
||||
if (FUZ_rand(&lseed) & 1) {
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, useOpaqueAPI) );
|
||||
DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog);
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, useOpaqueAPI) );
|
||||
}
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_searchLog, cParams.searchLog, useOpaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_minMatch, cParams.searchLength, useOpaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_targetLength, cParams.targetLength, useOpaqueAPI) );
|
||||
|
||||
/* mess with long distance matching parameters */
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, useOpaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), useOpaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), useOpaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_LDM_BUCKETSIZELOG_MAX), useOpaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashEveryLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN), useOpaqueAPI) );
|
||||
if (bigTests) {
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, useOpaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), useOpaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), useOpaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_LDM_BUCKETSIZELOG_MAX), useOpaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashEveryLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN), useOpaqueAPI) );
|
||||
}
|
||||
|
||||
/* mess with frame parameters */
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_checksumFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
|
||||
DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (U32)pledgedSrcSize);
|
||||
if (FUZ_rand(&lseed) & 1) {
|
||||
DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (U32)pledgedSrcSize);
|
||||
CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
|
||||
}
|
||||
|
||||
/* multi-threading parameters */
|
||||
{ U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
|
||||
U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1;
|
||||
U32 const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax);
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_nbThreads, nbThreads, useOpaqueAPI) );
|
||||
DISPLAYLEVEL(5, "t%u: nbThreads : %u \n", testNb, nbThreads);
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_nbThreads, nbThreads, useOpaqueAPI) );
|
||||
if (nbThreads > 1) {
|
||||
U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
|
||||
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10, useOpaqueAPI) );
|
||||
@ -1421,10 +1600,11 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
|
||||
}
|
||||
}
|
||||
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
|
||||
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
|
||||
|
||||
/* Apply parameters */
|
||||
if (useOpaqueAPI) {
|
||||
DISPLAYLEVEL(6," t%u: applying CCtxParams \n", testNb);
|
||||
CHECK_Z (ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams) );
|
||||
}
|
||||
|
||||
@ -1464,8 +1644,8 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
|
||||
outBuff.size = outBuff.pos + dstBuffSize;
|
||||
|
||||
CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, flush) );
|
||||
DISPLAYLEVEL(6, "compress consumed %u bytes (total : %u) \n",
|
||||
(U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos));
|
||||
DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) \n",
|
||||
testNb, (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos));
|
||||
|
||||
XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
|
||||
memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
|
||||
@ -1593,24 +1773,23 @@ typedef enum { simple_api, mt_api, advanced_api } e_api;
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
U32 seed=0;
|
||||
int seedset=0;
|
||||
int argNb;
|
||||
U32 seed = 0;
|
||||
int seedset = 0;
|
||||
int nbTests = nbTestsDefault;
|
||||
int testNb = 0;
|
||||
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
|
||||
int result=0;
|
||||
int result = 0;
|
||||
int mainPause = 0;
|
||||
int bigTests = (sizeof(size_t) == 8);
|
||||
e_api selected_api = simple_api;
|
||||
const char* const programName = argv[0];
|
||||
ZSTD_customMem const customNULL = ZSTD_defaultCMem;
|
||||
U32 useOpaqueAPI = 0;
|
||||
int argNb;
|
||||
|
||||
/* Check command line */
|
||||
for(argNb=1; argNb<argc; argNb++) {
|
||||
const char* argument = argv[argNb];
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
assert(argument != NULL);
|
||||
|
||||
/* Parsing commands. Aggregated commands are allowed */
|
||||
if (argument[0]=='-') {
|
||||
@ -1660,15 +1839,17 @@ int main(int argc, const char** argv)
|
||||
g_clockTime += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (*argument=='m') g_clockTime *=60, argument++;
|
||||
if (*argument=='n') argument++;
|
||||
g_clockTime *= CLOCKS_PER_SEC;
|
||||
if (*argument=='m') { /* -T1m == -T60 */
|
||||
g_clockTime *=60, argument++;
|
||||
if (*argument=='n') argument++; /* -T1mn == -T60 */
|
||||
} else if (*argument=='s') argument++; /* -T10s == -T10 */
|
||||
g_clockTime *= SEC_TO_MICRO;
|
||||
break;
|
||||
|
||||
case 's': /* manually select seed */
|
||||
argument++;
|
||||
seed=0;
|
||||
seedset=1;
|
||||
seed=0;
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
seed *= 10;
|
||||
seed += *argument - '0';
|
||||
@ -1718,7 +1899,7 @@ int main(int argc, const char** argv)
|
||||
if (nbTests<=0) nbTests=1;
|
||||
|
||||
if (testNb==0) {
|
||||
result = basicUnitTests(0, ((double)proba) / 100, customNULL); /* constant seed for predictability */
|
||||
result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */
|
||||
}
|
||||
|
||||
if (!result) {
|
||||
|
@ -12,7 +12,7 @@ cxx_library(
|
||||
deps=[
|
||||
'//lib:zstd',
|
||||
'//lib:zstd_common',
|
||||
]
|
||||
],
|
||||
)
|
||||
|
||||
cxx_binary(
|
||||
|
@ -684,6 +684,11 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
|
||||
fileSizes[n] = 0;
|
||||
continue;
|
||||
}
|
||||
if (fileSize == UTIL_FILESIZE_UNKNOWN) {
|
||||
DISPLAYLEVEL(2, "Cannot determine size of %s ... \n", fileNamesTable[n]);
|
||||
fileSizes[n] = 0;
|
||||
continue;
|
||||
}
|
||||
f = fopen(fileNamesTable[n], "rb");
|
||||
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
|
||||
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
|
||||
@ -714,11 +719,13 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
|
||||
|
||||
/* Load dictionary */
|
||||
if (dictFileName != NULL) {
|
||||
U64 dictFileSize = UTIL_getFileSize(dictFileName);
|
||||
if (dictFileSize > 64 MB) EXM_THROW(10, "dictionary file %s too large", dictFileName);
|
||||
U64 const dictFileSize = UTIL_getFileSize(dictFileName);
|
||||
if (dictFileSize > 64 MB)
|
||||
EXM_THROW(10, "dictionary file %s too large", dictFileName);
|
||||
dictBufferSize = (size_t)dictFileSize;
|
||||
dictBuffer = malloc(dictBufferSize);
|
||||
if (dictBuffer==NULL) EXM_THROW(11, "not enough memory for dictionary (%u bytes)", (U32)dictBufferSize);
|
||||
if (dictBuffer==NULL)
|
||||
EXM_THROW(11, "not enough memory for dictionary (%u bytes)", (U32)dictBufferSize);
|
||||
BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1);
|
||||
}
|
||||
|
||||
|
@ -270,7 +270,7 @@ ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm,
|
||||
zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem);
|
||||
if (zwc->zbc == NULL) return ZWRAPC_finishWithError(zwc, strm, 0);
|
||||
}
|
||||
{ int res = ZWRAP_initializeCStream(zwc, dictionary, dictLength, 0);
|
||||
{ int res = ZWRAP_initializeCStream(zwc, dictionary, dictLength, ZSTD_CONTENTSIZE_UNKNOWN);
|
||||
if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res); }
|
||||
zwc->comprState = ZWRAP_useReset;
|
||||
}
|
||||
@ -295,7 +295,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
|
||||
if (zwc->zbc == NULL) {
|
||||
zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem);
|
||||
if (zwc->zbc == NULL) return ZWRAPC_finishWithError(zwc, strm, 0);
|
||||
{ int const initErr = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : 0);
|
||||
{ int const initErr = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : ZSTD_CONTENTSIZE_UNKNOWN);
|
||||
if (initErr != Z_OK) return ZWRAPC_finishWithError(zwc, strm, initErr); }
|
||||
if (flush != Z_FINISH) zwc->comprState = ZWRAP_useReset;
|
||||
} else {
|
||||
@ -308,7 +308,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
|
||||
return ZWRAPC_finishWithError(zwc, strm, 0);
|
||||
}
|
||||
} else {
|
||||
int const res = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : 0);
|
||||
int const res = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : ZSTD_CONTENTSIZE_UNKNOWN);
|
||||
if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res);
|
||||
if (flush != Z_FINISH) zwc->comprState = ZWRAP_useReset;
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user