d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update to Zstandard 1.3.8

Browse Source 
This merge brings in a couple new files, which needed to be attached to the
build; a new dependency on <limits.h>, which must be stubbed; and a name
change in the Context parameter constants, from ZSTD_p_foo to ZSTD_c_foo.

Significantly, it fixes a kernel build error with GCC where floating-point
functions were included in the kernel build, by hiding them under the same
compile-time #ifdef that already covered their invocation.  That issue was
introduced to FreeBSD in the 1.3.7 update and tracked upstream here:

  https://github.com/facebook/zstd/issues/1386

The full 1.3.8 release notes can be found on Github:

  https://github.com/facebook/zstd/releases/tag/v1.3.8

Relnotes:	yes
This commit is contained in:
cem 2018-12-29 21:18:01 +00:00
commit 432d4753f2
113 changed files with 11301 additions and 6579 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/libzstd/Makefile
View File

@ -13,7 +13,9 @@ SRCS= entropy_common.c \
zstd_compress.c \
zstdmt_compress.c \
huf_decompress.c \
zstd_ddict.c \
zstd_decompress.c \
zstd_decompress_block.c \
zbuff_common.c \
zbuff_compress.c \
zbuff_decompress.c \

2
sys/conf/files
View File

@ -652,7 +652,9 @@ contrib/zstd/lib/compress/zstd_fast.c optional zstdio compile-with ${ZSTD_C}
contrib/zstd/lib/compress/zstd_lazy.c optional zstdio compile-with ${ZSTD_C}
contrib/zstd/lib/compress/zstd_ldm.c optional zstdio compile-with ${ZSTD_C}
contrib/zstd/lib/compress/zstd_opt.c optional zstdio compile-with ${ZSTD_C}
contrib/zstd/lib/decompress/zstd_ddict.c optional zstdio compile-with ${ZSTD_C}
contrib/zstd/lib/decompress/zstd_decompress.c optional zstdio compile-with ${ZSTD_C}
contrib/zstd/lib/decompress/zstd_decompress_block.c optional zstdio compile-with ${ZSTD_C}
contrib/zstd/lib/decompress/huf_decompress.c optional zstdio compile-with ${ZSTD_C}
# Blake 2
contrib/libb2/blake2b-ref.c optional crypto | ipsec | ipsec_support \

10
sys/contrib/zstd/Makefile
View File

@ -64,7 +64,8 @@ zlibwrapper: lib
## test: run long-duration tests
.PHONY: test
test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror
DEBUGLEVEL ?= 1
test: MOREFLAGS += -g -DDEBUGLEVEL=$(DEBUGLEVEL) -Werror
test:
MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
$(MAKE) -C $(TESTDIR) $@
@ -129,7 +130,12 @@ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD Dr
HOST_OS = POSIX
CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
EGREP = egrep --color=never
HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
EGREP_OPTIONS ?=
ifeq ($HAVE_COLORNEVER, 1)
EGREP_OPTIONS += --color=never
endif
EGREP = egrep $(EGREP_OPTIONS)
# Print a two column output of targets and their description. To add a target description, put a
# comment in the Makefile with the format "## <TARGET>: <DESCRIPTION>". For example:

402
sys/contrib/zstd/NEWS
View File

@ -1,402 +0,0 @@
v1.3.7
perf: slightly better decompression speed on clang (depending on hardware target)
fix : performance of dictionary compression for small input < 4 KB at levels 9 and 10
build: no longer build backtrace by default in release mode; restrict further automatic mode
build: control backtrace support through build macro BACKTRACE
misc: added man pages for zstdless and zstdgrep, by @samrussell
v1.3.6
perf: much faster dictionary builder, by @jenniferliu
perf: faster dictionary compression on small data when using multiple contexts, by @felixhandte
perf: faster dictionary decompression when using a very large number of dictionaries simultaneously
cli : fix : does no longer overwrite destination when source does not exist (#1082)
cli : new command --adapt, for automatic compression level adaptation
api : fix : block api can be streamed with > 4 GB, reported by @catid
api : reduced ZSTD_DDict size by 2 KB
api : minimum negative compression level is defined, and can be queried using ZSTD_minCLevel().
build: support Haiku target, by @korli
build: Read Legacy format is limited to v0.5+ by default. Can be changed at compile time with macro ZSTD_LEGACY_SUPPORT.
doc : zstd_compression_format.md updated to match wording in IETF RFC 8478
misc: tests/paramgrill, a parameter optimizer, by @GeorgeLu97
v1.3.5
perf: much faster dictionary compression, by @felixhandte
perf: small quality improvement for dictionary generation, by @terrelln
perf: slightly improved high compression levels (notably level 19)
mem : automatic memory release for long duration contexts
cli : fix : overlapLog can be manually set
cli : fix : decoding invalid lz4 frames
api : fix : performance degradation for dictionary compression when using advanced API, by @terrelln
api : change : clarify ZSTD_CCtx_reset() vs ZSTD_CCtx_resetParameters(), by @terrelln
build: select custom libzstd scope through control macros, by @GeorgeLu97
build: OpenBSD patch, by @bket
build: make and make all are compatible with -j
doc : clarify zstd_compression_format.md, updated for IETF RFC process
misc: pzstd compatible with reproducible compilation, by @lamby
v1.3.4
perf: faster speed (especially decoding speed) on recent cpus (haswell+)
perf: much better performance associating --long with multi-threading, by @terrelln
perf: better compression at levels 13-15
cli : asynchronous compression by default, for faster experience (use --single-thread for former behavior)
cli : smoother status report in multi-threading mode
cli : added command --fast=#, for faster compression modes
cli : fix crash when not overwriting existing files, by Pádraig Brady (@pixelb)
api : `nbThreads` becomes `nbWorkers` : 1 triggers asynchronous mode
api : compression levels can be negative, for even more speed
api : ZSTD_getFrameProgression() : get precise progress status of ZSTDMT anytime
api : ZSTDMT can accept new compression parameters during compression
api : implemented all advanced dictionary decompression prototypes
build: improved meson recipe, by Shawn Landden (@shawnl)
build: VS2017 scripts, by @HaydnTrigg
misc: all /contrib projects fixed
misc: added /contrib/docker script by @gyscos
v1.3.3
perf: faster zstd_opt strategy (levels 16-19)
fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen
cli : fix : content size written in header by default
cli : fix : improved LZ4 format support, by @felixhandte
cli : new : hidden command `-S`, to benchmark multiple files while generating one result per file
api : fix : support large skippable frames, by @terrelln
api : fix : streaming interface was adding a useless 3-bytes null block to small frames
api : change : when setting `pledgedSrcSize`, use `ZSTD_CONTENTSIZE_UNKNOWN` macro value to mean "unknown"
build: fix : compilation under rhel6 and centos6, reported by @pixelb
build: added `check` target
v1.3.2
new : long range mode, using --long command, by Stella Lau (@stellamplau)
new : ability to generate and decode magicless frames (#591)
changed : maximum nb of threads reduced to 200, to avoid address space exhaustion in 32-bits mode
fix : multi-threading compression works with custom allocators
fix : ZSTD_sizeof_CStream() was over-evaluating memory usage
fix : a rare compression bug when compression generates very large distances and bunch of other conditions (only possible at --ultra -22)
fix : 32-bits build can now decode large offsets (levels 21+)
cli : added LZ4 frame support by default, by Felix Handte (@felixhandte)
cli : improved --list output
cli : new : can split input file for dictionary training, using command -B#
cli : new : clean operation artefact on Ctrl-C interruption
cli : fix : do not change /dev/null permissions when using command -t with root access, reported by @mike155 (#851)
cli : fix : write file size in header in multiple-files mode
api : added macro ZSTD_COMPRESSBOUND() for static allocation
api : experimental : new advanced decompression API
api : fix : sizeof_CCtx() used to over-estimate
build: fix : no-multithread variant compiles without pool.c dependency, reported by Mitchell Blank Jr (@mitchblank) (#819)
build: better compatibility with reproducible builds, by Bernhard M. Wiedemann (@bmwiedemann) (#818)
example : added streaming_memory_usage
license : changed /examples license to BSD + GPLv2
license : fix a few header files to reflect new license (#825)
v1.3.1
New license : BSD + GPLv2
perf: substantially decreased memory usage in Multi-threading mode, thanks to reports by Tino Reichardt (@mcmilk)
perf: Multi-threading supports up to 256 threads. Cap at 256 when more are requested (#760)
cli : improved and fixed --list command, by @ib (#772)
cli : command -vV to list supported formats, by @ib (#771)
build : fixed binary variants, reported by @svenha (#788)
build : fix Visual compilation for non x86/x64 targets, reported by Greg Slazinski (@GregSlazinski) (#718)
API exp : breaking change : ZSTD_getframeHeader() provides more information
API exp : breaking change : pinned down values of error codes
doc : fixed huffman example, by Ulrich Kunitz (@ulikunitz)
new : contrib/adaptive-compression, I/O driven compression strength, by Paul Cruz (@paulcruz74)
new : contrib/long_distance_matching, statistics by Stella Lau (@stellamplau)
updated : contrib/linux-kernel, by Nick Terrell (@terrelln)
v1.3.0
cli : new : `--list` command, by Paul Cruz
cli : changed : xz/lzma support enabled by default
cli : changed : `-t *` continue processing list after a decompression error
API : added : ZSTD_versionString()
API : promoted to stable status : ZSTD_getFrameContentSize(), by Sean Purcell
API exp : new advanced API : ZSTD_compress_generic(), ZSTD_CCtx_setParameter()
API exp : new : API for static or external allocation : ZSTD_initStatic?Ctx()
API exp : added : ZSTD_decompressBegin_usingDDict(), requested by Guy Riddle (#700)
API exp : clarified memory estimation / measurement functions.
API exp : changed : strongest strategy renamed ZSTD_btultra, fastest strategy ZSTD_fast set to 1
tools : decodecorpus can generate random dictionary-compressed samples, by Paul Cruz
new : contrib/seekable_format, demo and API, by Sean Purcell
changed : contrib/linux-kernel, updated version and license, by Nick Terrell
v1.2.0
cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable)
cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell
cli : new : zstdmt symlink hardwired to `zstd -T0`
cli : new : command --threads=# (#671)
cli : changed : cover dictionary builder by default, for improved quality, by Nick Terrell
cli : new : commands --train-cover and --train-legacy, to select dictionary algorithm and parameters
cli : experimental targets `zstd4` and `xzstd4`, with support for lz4 format, by Sean Purcell
cli : fix : does not output compressed data on console
cli : fix : ignore symbolic links unless --force specified,
API : breaking change : ZSTD_createCDict_advanced(), only use compressionParameters as argument
API : added : prototypes ZSTD_*_usingCDict_advanced(), for direct control over frameParameters.
API : improved: ZSTDMT_compressCCtx() reduced memory usage
API : fix : ZSTDMT_compressCCtx() now provides srcSize in header (#634)
API : fix : src size stored in frame header is controlled at end of frame
API : fix : enforced consistent rules for pledgedSrcSize==0 (#641)
API : fix : error code "GENERIC" replaced by "dstSizeTooSmall" when appropriate
build: improved cmake script, by @Majlen
build: enabled Multi-threading support for *BSD, by Baptiste Daroussin
tools: updated Paramgrill. Command -O# provides best parameters for sample and speed target.
new : contrib/linux-kernel version, by Nick Terrell
v1.1.4
cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski
cli : new : advanced benchmark command --priority=rt
cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
cli : fix : --rm remains silent when input is stdin
cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski
speed : improved decompression speed in streaming mode for single shot scenarios (+5%)
memory: DDict (decompression dictionary) memory usage down from 150 KB to 20 KB
arch: 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell
API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize()
API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value)
build : new: meson build system in contrib/meson, by Dima Krasner
build : improved cmake script, by @Majlen
build : added -Wformat-security flag, as recommended by Padraig Brady
doc : new : educational decoder, by Sean Purcell
v1.1.3
cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`)
cli : new : experimental target `make zstdmt`, with multi-threading support
cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell, based on prior work by Giuseppe Ottaviano.
cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski
cli : fix zstdless on Mac OS-X, by Andrew Janke
cli : fix #232 "compress non-files"
dictBuilder : improved dictionary generation quality, thanks to Nick Terrell
API : new : lib/compress/ZSTDMT_compress.h multithreading API (experimental)
API : new : ZSTD_create?Dict_byReference(), requested by Bartosz Taudul
API : new : ZDICT_finalizeDictionary()
API : fix : ZSTD_initCStream_usingCDict() properly writes dictID into frame header, by Gregory Szorc (#511)
API : fix : all symbols properly exposed in libzstd, by Nick Terrell
build : support for Solaris target, by Przemyslaw Skibinski
doc : clarified specification, by Sean Purcell
v1.1.2
API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init
API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize()
API : zbuff : changed : prototypes now generate deprecation warnings
lib : improved : faster decompression speed at ultra compression settings and 32-bits mode
lib : changed : only public ZSTD_ symbols are now exposed
lib : changed : reduced usage of stack memory
lib : fixed : several corner case bugs, by Nick Terrell
cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski
cli : new : preserve file attributes
cli : new : added zstdless and zstdgrep tools
cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd)
cli : fixed : zstdcat
zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski
install : better compatibility with FreeBSD, by Dimitry Andric
source tree : changed : zbuff source files moved to lib/deprecated
v1.1.1
New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption
New : doc/zstd_manual.html, by Przemyslaw Skibinski
Improved : slightly better compression ratio at --ultra levels (>= 20)
Improved : better memory usage when using streaming compression API, thanks to @Rogier-5 report
Added : API : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section)
Added : example/multiple_streaming_compression.c
Changed : zstd_errors.h is now installed within /include (and replaces errors_public.h)
Updated man page
Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets
v1.1.0
New : contrib/pzstd, parallel version of zstd, by Nick Terrell
added : NetBSD install target (#338)
Improved : speed for batches of small files
Improved : speed of zlib wrapper, by Przemyslaw Skibinski
Changed : libzstd on Windows supports legacy formats, by Christophe Chevalier
Fixed : CLI -d output to stdout by default when input is stdin (#322)
Fixed : CLI correctly detects console on Mac OS-X
Fixed : CLI supports recursive mode `-r` on Mac OS-X
Fixed : Legacy decoders use unified error codes, reported by benrg (#341), fixed by Przemyslaw Skibinski
Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319)
Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365)
Fixed : zstd-pgo, reported by octoploid (#329)
v1.0.0
Change Licensing, all project is now BSD, Copyright Facebook
Small decompression speed improvement
API : Streaming API supports legacy format
API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParamter()
CLI supports legacy formats v0.4+
Fixed : compression fails on certain huge files, reported by Jesse McGrew
Enhanced documentation, by Przemyslaw Skibinski
v0.8.1
New streaming API
Changed : --ultra now enables levels beyond 19
Changed : -i# now selects benchmark time in second
Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell
Fixed : speed regression on specific patterns (#272)
Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291)
Fixed : ICC compilation, by Przemyslaw Skibinski
v0.8.0
Improved : better speed on clang and gcc -O2, thanks to Eric Biggers
New : Build on FreeBSD and DragonFly, thanks to JrMarino
Changed : modified API : ZSTD_compressEnd()
Fixed : legacy mode with ZSTD_HEAPMODE=0, by Christopher Bergqvist
Fixed : premature end of frame when zero-sized raw block, reported by Eric Biggers
Fixed : large dictionaries (> 384 KB), reported by Ilona Papava
Fixed : checksum correctly checked in single-pass mode
Fixed : combined --test amd --rm, reported by Andreas M. Nilsson
Modified : minor compression level adaptations
Updated : compression format specification to v0.2.0
changed : zstd.h moved to /lib directory
v0.7.5
Transition version, supporting decoding of v0.8.x
v0.7.4
Added : homebrew for Mac, by Daniel Cade
Added : more examples
Fixed : segfault when using small dictionaries, reported by Felix Handte
Modified : default compression level for CLI is now 3
Updated : specification, to v0.1.1
v0.7.3
New : compression format specification
New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner.
New : `ZSTD_getDecompressedSize()`
New : OpenBSD target, by Juan Francisco Cantero Hurtado
New : `examples` directory
fixed : dictBuilder using HC levels, reported by Bartosz Taudul
fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte
fixed : multi-blocks decoding with intermediate uncompressed blocks, reported by Greg Slazinski
modified : removed "mem.h" and "error_public.h" dependencies from "zstd.h" (experimental section)
modified : legacy functions no longer need magic number
v0.7.2
fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski.
fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner.
fixed : CLI displays system error message when destination file cannot be created (#231). Reported by Chip Turner.
v0.7.1
fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier
fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв
fixed : corruption issue, reported by cj
modified : checksum enabled by default in command line mode
v0.7.0
New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski
New : Command `--rm`, to remove source file after successful de/compression
New : Visual build scripts, by Christophe Chevalier
New : Support for Sparse File-systems (do not use space for zero-filled sectors)
New : Frame checksum support
New : Support pass-through mode (when using `-df`)
API : more efficient Dictionary API : `ZSTD_compress_usingCDict()`, `ZSTD_decompress_usingDDict()`
API : create dictionary files from custom content, by Giuseppe Ottaviano
API : support for custom malloc/free functions
New : controllable Dictionary ID
New : Support for skippable frames
v0.6.1
New : zlib wrapper API, thanks to Przemyslaw Skibinski
New : Ability to compile compressor / decompressor separately
Changed : new lib directory structure
Fixed : Legacy codec v0.5 compatible with dictionary decompression
Fixed : Decoder corruption error (#173)
Fixed : null-string roundtrip (#176)
New : benchmark mode can select directory as input
Experimental : midipix support, VMS support
v0.6.0
Stronger high compression modes, thanks to Przemyslaw Skibinski
API : ZSTD_getFrameParams() provides size of decompressed content
New : highest compression modes require `--ultra` command to fully unleash their capacity
Fixed : zstd cli return error code > 0 and removes dst file artifact when decompression fails, thanks to Chip Turner
v0.5.1
New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski
Changed : Dictionary builder integrated into libzstd and zstd cli
Changed (!) : zstd cli now uses "multiple input files" as default mode. See `zstd -h`.
Fix : high compression modes for big-endian platforms
New : zstd cli : `-t` | `--test` command
v0.5.0
New : dictionary builder utility
Changed : streaming & dictionary API
Improved : better compression of small data
v0.4.7
Improved : small compression speed improvement in HC mode
Changed : `zstd_decompress.c` has ZSTD_LEGACY_SUPPORT to 0 by default
fix : bt search bug
v0.4.6
fix : fast compression mode on Windows
New : cmake configuration file, thanks to Artyom Dymchenko
Improved : high compression mode on repetitive data
New : block-level API
New : ZSTD_duplicateCCtx()
v0.4.5
new : -m/--multiple : compress/decompress multiple files
v0.4.4
Fixed : high compression modes for Windows 32 bits
new : external dictionary API extended to buffered mode and accessible through command line
new : windows DLL project, thanks to Christophe Chevalier
v0.4.3 :
new : external dictionary API
new : zstd-frugal
v0.4.2 :
Generic minor improvements for small blocks
Fixed : big-endian compatibility, by Peter Harris (#85)
v0.4.1
Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben)
removed `zstd.c`
v0.4.0
Command line utility compatible with high compression levels
Removed zstdhc => merged into zstd
Added : ZBUFF API (see zstd_buffered.h)
Rolling buffer support
v0.3.6
small blocks params
v0.3.5
minor generic compression improvements
v0.3.4
Faster fast cLevels
v0.3.3
Small compression ratio improvement
v0.3.2
Fixed Visual Studio
v0.3.1 :
Small compression ratio improvement
v0.3
HC mode : compression levels 2-26
v0.2.2
Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier
v0.2.1
Fix : Read errors, advanced fuzzer tests, by Hanno Böck
v0.2.0
**Breaking format change**
Faster decompression speed
Can still decode v0.1 format
v0.1.3
fix uninitialization warning, reported by Evan Nemerson
v0.1.2
frame concatenation support
v0.1.1
fix compression bug
detects write-flush errors
v0.1.0
first release

54
sys/contrib/zstd/README.md
View File

@ -9,7 +9,11 @@ and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4
Should your project require another programming language,
a list of known ports and bindings is provided on [Zstandard homepage](http://www.zstd.net/#other-languages).
Development branch status : [![Build Status][travisDevBadge]][travisLink] [![Build status][AppveyorDevBadge]][AppveyorLink] [![Build status][CircleDevBadge]][CircleLink]
**Development branch status:**
[![Build Status][travisDevBadge]][travisLink]
[![Build status][AppveyorDevBadge]][AppveyorLink]
[![Build status][CircleDevBadge]][CircleLink]
[travisDevBadge]: https://travis-ci.org/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
[travisLink]: https://travis-ci.org/facebook/zstd
@ -18,7 +22,7 @@ Development branch status : [![Build Status][travisDevBadge]][travisLink] [![B
[CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite"
[CircleLink]: https://circleci.com/gh/facebook/zstd
### Benchmarks
## Benchmarks
For reference, several fast compression algorithms were tested and compared
on a server running Linux Debian (`Linux version 4.14.0-3-amd64`),
@ -42,7 +46,7 @@ on the [Silesia compression corpus].
| snappy 1.1.4 | 2.091 | 530 MB/s | 1800 MB/s |
| lzf 3.6 -1 | 2.077 | 400 MB/s | 860 MB/s |
[zlib]:http://www.zlib.net/
[zlib]: http://www.zlib.net/
[LZ4]: http://www.lz4.org/
Zstd can also offer stronger compression ratios at the cost of compression speed.
@ -65,7 +69,7 @@ A few other algorithms can produce higher compression ratios at slower speeds, f
For a larger picture including slow modes, [click on this link](doc/images/DCspeed5.png).
### The case for Small Data compression
## The case for Small Data compression
Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives.
@ -89,24 +93,24 @@ Training works if there is some correlation in a family of small data samples. T
Hence, deploying one dictionary per type of data will provide the greatest benefits.
Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file.
#### Dictionary compression How To:
### Dictionary compression How To:
1) Create the dictionary
1. Create the dictionary
`zstd --train FullPathToTrainingSet/* -o dictionaryName`
`zstd --train FullPathToTrainingSet/* -o dictionaryName`
2) Compress with dictionary
2. Compress with dictionary
`zstd -D dictionaryName FILE`
`zstd -D dictionaryName FILE`
3) Decompress with dictionary
3. Decompress with dictionary
`zstd -D dictionaryName --decompress FILE.zst`
`zstd -D dictionaryName --decompress FILE.zst`
### Build instructions
## Build instructions
#### Makefile
### Makefile
If your system is compatible with standard `make` (or `gmake`),
invoking `make` in root directory will generate `zstd` cli in root directory.
@ -115,7 +119,7 @@ Other available options include:
- `make install` : create and install zstd cli, library and man pages
- `make check` : create and run `zstd`, tests its behavior on local platform
#### cmake
### cmake
A `cmake` project generator is provided within `build/cmake`.
It can generate Makefiles or other build scripts
@ -123,11 +127,17 @@ to create `zstd` binary, and `libzstd` dynamic and static libraries.
By default, `CMAKE_BUILD_TYPE` is set to `Release`.
#### Meson
### Meson
A Meson project is provided within `contrib/meson`.
A Meson project is provided within [`build/meson`](build/meson). Follow
build instructions in that directory.
#### Visual Studio (Windows)
You can also take a look at [`.travis.yml`](.travis.yml) file for an
example about how Meson is used to build this project.
Note that default build type is **release**.
### Visual Studio (Windows)
Going into `build` directory, you will find additional possibilities:
- Projects for Visual Studio 2005, 2008 and 2010.
@ -135,17 +145,21 @@ Going into `build` directory, you will find additional possibilities:
- Automated build scripts for Visual compiler by [@KrzysFR](https://github.com/KrzysFR), in `build/VS_scripts`,
which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution.
### Buck
### Status
You can build the zstd binary via buck by executing: `buck build programs:zstd` from the root of the repo.
The output binary will be in `buck-out/gen/programs/`.
## Status
Zstandard is currently deployed within Facebook. It is used continuously to compress large amounts of data in multiple formats and use cases.
Zstandard is considered safe for production environments.
### License
## License
Zstandard is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING).
### Contributing
## Contributing
The "dev" branch is the one where all contributions are merged before reaching "master".
If you plan to propose a patch, please commit into the "dev" branch, or its own feature branch.

2
sys/contrib/zstd/appveyor.yml
View File

@ -3,6 +3,8 @@
branches:
only:
- master
- appveyorTest
- /visual*/
environment:
matrix:
- COMPILER: "gcc"

3
sys/contrib/zstd/contrib/meson/README
View File

@ -1,3 +0,0 @@
This Meson project is provided with no guarantee and maintained by Dima Krasner <dima@dimakrasner.com>.
It outputs one libzstd, either shared or static, depending on default_library.

144
sys/contrib/zstd/contrib/meson/meson.build
View File

@ -1,144 +0,0 @@
project('zstd', 'c', license: 'BSD')
libm = meson.get_compiler('c').find_library('m', required: true)
lib_dir = join_paths('..', '..', 'lib')
common_dir = join_paths(lib_dir, 'common')
compress_dir = join_paths(lib_dir, 'compress')
decompress_dir = join_paths(lib_dir, 'decompress')
dictbuilder_dir = join_paths(lib_dir, 'dictBuilder')
deprecated_dir = join_paths(lib_dir, 'deprecated')
libzstd_srcs = [
join_paths(common_dir, 'entropy_common.c'),
join_paths(common_dir, 'fse_decompress.c'),
join_paths(common_dir, 'threading.c'),
join_paths(common_dir, 'pool.c'),
join_paths(common_dir, 'zstd_common.c'),
join_paths(common_dir, 'error_private.c'),
join_paths(common_dir, 'xxhash.c'),
join_paths(compress_dir, 'fse_compress.c'),
join_paths(compress_dir, 'hist.c'),
join_paths(compress_dir, 'huf_compress.c'),
join_paths(compress_dir, 'zstd_compress.c'),
join_paths(compress_dir, 'zstd_fast.c'),
join_paths(compress_dir, 'zstd_double_fast.c'),
join_paths(compress_dir, 'zstd_lazy.c'),
join_paths(compress_dir, 'zstd_opt.c'),
join_paths(compress_dir, 'zstd_ldm.c'),
join_paths(compress_dir, 'zstdmt_compress.c'),
join_paths(decompress_dir, 'huf_decompress.c'),
join_paths(decompress_dir, 'zstd_decompress.c'),
join_paths(dictbuilder_dir, 'cover.c'),
join_paths(dictbuilder_dir, 'divsufsort.c'),
join_paths(dictbuilder_dir, 'zdict.c'),
join_paths(deprecated_dir, 'zbuff_common.c'),
join_paths(deprecated_dir, 'zbuff_compress.c'),
join_paths(deprecated_dir, 'zbuff_decompress.c')
]
libzstd_includes = [include_directories(common_dir, dictbuilder_dir, compress_dir, lib_dir)]
legacy = get_option('legacy_support')
if legacy == '0'
legacy = 'false'
endif
if legacy != 'false'
if legacy == 'true'
legacy = '1'
endif
#See ZSTD_LEGACY_SUPPORT of programs/README.md
message('Enabling legacy support back to version 0.' + legacy)
legacy_int = legacy.to_int()
if legacy_int > 7
legacy_int = 7
endif
libzstd_cflags = ['-DZSTD_LEGACY_SUPPORT=' + legacy]
legacy_dir = join_paths(lib_dir, 'legacy')
libzstd_includes += [include_directories(legacy_dir)]
if legacy_int <= 1
libzstd_srcs += join_paths(legacy_dir, 'zstd_v01.c')
endif
if legacy_int <= 2
libzstd_srcs += join_paths(legacy_dir, 'zstd_v02.c')
endif
if legacy_int <= 3
libzstd_srcs += join_paths(legacy_dir, 'zstd_v03.c')
endif
if legacy_int <= 4
libzstd_srcs += join_paths(legacy_dir, 'zstd_v04.c')
endif
if legacy_int <= 5
libzstd_srcs += join_paths(legacy_dir, 'zstd_v05.c')
endif
if legacy_int <= 6
libzstd_srcs += join_paths(legacy_dir, 'zstd_v06.c')
endif
if legacy_int <= 7
libzstd_srcs += join_paths(legacy_dir, 'zstd_v07.c')
endif
else
libzstd_cflags = []
endif
if get_option('multithread')
message('Enabling multi-threading support')
add_global_arguments('-DZSTD_MULTITHREAD', language: 'c')
libzstd_deps = [dependency('threads')]
else
libzstd_deps = []
endif
libzstd = library('zstd',
libzstd_srcs,
include_directories: libzstd_includes,
c_args: libzstd_cflags,
dependencies: libzstd_deps,
install: true,
soversion: '1',
)
programs_dir = join_paths('..', '..', 'programs')
zstd = executable('zstd',
join_paths(programs_dir, 'bench.c'),
join_paths(programs_dir, 'datagen.c'),
join_paths(programs_dir, 'dibio.c'),
join_paths(programs_dir, 'fileio.c'),
join_paths(programs_dir, 'zstdcli.c'),
include_directories: libzstd_includes,
c_args: ['-DZSTD_NODICT', '-DZSTD_NOBENCH'],
link_with: libzstd,
install: true)
tests_dir = join_paths('..', '..', 'tests')
datagen_c = join_paths(programs_dir, 'datagen.c')
test_includes = libzstd_includes + [include_directories(programs_dir)]
fullbench = executable('fullbench',
datagen_c, join_paths(tests_dir, 'fullbench.c'),
include_directories: test_includes,
link_with: libzstd)
test('fullbench', fullbench)
fuzzer = executable('fuzzer',
datagen_c, join_paths(tests_dir, 'fuzzer.c'),
include_directories: test_includes,
link_with: libzstd)
test('fuzzer', fuzzer)
if target_machine.system() != 'windows'
paramgrill = executable('paramgrill',
datagen_c, join_paths(tests_dir, 'paramgrill.c'),
join_paths(programs_dir, 'bench.c'),
include_directories: test_includes,
link_with: libzstd,
dependencies: libm)
test('paramgrill', paramgrill)
datagen = executable('datagen',
datagen_c, join_paths(tests_dir, 'datagencli.c'),
include_directories: test_includes,
link_with: libzstd)
endif

3
sys/contrib/zstd/contrib/meson/meson_options.txt
View File

@ -1,3 +0,0 @@
option('multithread', type: 'boolean', value: false)
option('legacy_support', type: 'string', value: '4',
description: 'True or false, or 7 to 1 for v0.7+ to v0.1+.')

2
sys/contrib/zstd/contrib/pzstd/Makefile
View File

@ -171,7 +171,7 @@ roundtripcheck: roundtrip check
$(TESTPROG) ./test/RoundTripTest$(EXT) $(TESTFLAGS)
# Build the main binary
pzstd$(EXT): main.o Options.o Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a
pzstd$(EXT): main.o $(PROGDIR)/util.o Options.o Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a
$(LD_COMMAND)
# Target that depends on all the tests

16
sys/contrib/zstd/contrib/seekable_format/examples/parallel_processing.c
View File

@ -148,20 +148,20 @@ static void sumFile_orDie(const char* fname, int nbThreads)
size_t const initResult = ZSTD_seekable_initFile(seekable, fin);
if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_seekable_init() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
size_t const numFrames = ZSTD_seekable_getNumFrames(seekable);
unsigned const numFrames = ZSTD_seekable_getNumFrames(seekable);
struct sum_job* jobs = (struct sum_job*)malloc(numFrames * sizeof(struct sum_job));
size_t i;
for (i = 0; i < numFrames; i++) {
jobs[i] = (struct sum_job){ fname, 0, i, 0 };
POOL_add(pool, sumFrame, &jobs[i]);
unsigned fnb;
for (fnb = 0; fnb < numFrames; fnb++) {
jobs[fnb] = (struct sum_job){ fname, 0, fnb, 0 };
POOL_add(pool, sumFrame, &jobs[fnb]);
}
unsigned long long total = 0;
for (i = 0; i < numFrames; i++) {
while (!jobs[i].done) SLEEP(5); /* wake up every 5 milliseconds to check */
total += jobs[i].sum;
for (fnb = 0; fnb < numFrames; fnb++) {
while (!jobs[fnb].done) SLEEP(5); /* wake up every 5 milliseconds to check */
total += jobs[fnb].sum;
}
printf("Sum: %llu\n", total);

35
sys/contrib/zstd/contrib/seekable_format/zstdseek_compress.c
View File

@ -8,6 +8,8 @@
*/
#include <stdlib.h> /* malloc, free */
#include <limits.h> /* UINT_MAX */
#include <assert.h>
#define XXH_STATIC_LINKING_ONLY
#define XXH_NAMESPACE ZSTD_
@ -139,7 +141,7 @@ size_t ZSTD_seekable_freeCStream(ZSTD_seekable_CStream* zcs)
size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs,
int compressionLevel,
int checksumFlag,
U32 maxFrameSize)
unsigned maxFrameSize)
{
zcs->framelog.size = 0;
zcs->frameCSize = 0;
@ -167,9 +169,9 @@ size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs,
}
size_t ZSTD_seekable_logFrame(ZSTD_frameLog* fl,
unsigned compressedSize,
unsigned decompressedSize,
unsigned checksum)
unsigned compressedSize,
unsigned decompressedSize,
unsigned checksum)
{
if (fl->size == ZSTD_SEEKABLE_MAXFRAMES)
return ERROR(frameIndex_tooLarge);
@ -184,7 +186,8 @@ size_t ZSTD_seekable_logFrame(ZSTD_frameLog* fl,
if (newEntries == NULL) return ERROR(memory_allocation);
fl->entries = newEntries;
fl->capacity = newCapacity;
assert(newCapacity <= UINT_MAX);
fl->capacity = (U32)newCapacity;
}
fl->entries[fl->size] = (framelogEntry_t){
@ -268,7 +271,7 @@ size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer*
static inline size_t ZSTD_seekable_seekTableSize(const ZSTD_frameLog* fl)
{
size_t const sizePerFrame = 8 + (fl->checksumFlag?4:0);
size_t const seekTableLen = ZSTD_skippableHeaderSize +
size_t const seekTableLen = ZSTD_SKIPPABLEHEADERSIZE +
sizePerFrame * fl->size +
ZSTD_seekTableFooterSize;
@ -307,32 +310,32 @@ size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output)
size_t const seekTableLen = ZSTD_seekable_seekTableSize(fl);
CHECK_Z(ZSTD_stwrite32(fl, output, ZSTD_MAGIC_SKIPPABLE_START | 0xE, 0));
CHECK_Z(ZSTD_stwrite32(fl, output, seekTableLen - ZSTD_skippableHeaderSize,
4));
assert(seekTableLen <= (size_t)UINT_MAX);
CHECK_Z(ZSTD_stwrite32(fl, output, (U32)seekTableLen - ZSTD_SKIPPABLEHEADERSIZE, 4));
while (fl->seekTableIndex < fl->size) {
unsigned long long const start = ZSTD_SKIPPABLEHEADERSIZE + sizePerFrame * fl->seekTableIndex;
assert(start + 8 <= UINT_MAX);
CHECK_Z(ZSTD_stwrite32(fl, output,
fl->entries[fl->seekTableIndex].cSize,
ZSTD_skippableHeaderSize +
sizePerFrame * fl->seekTableIndex + 0));
(U32)start + 0));
CHECK_Z(ZSTD_stwrite32(fl, output,
fl->entries[fl->seekTableIndex].dSize,
ZSTD_skippableHeaderSize +
sizePerFrame * fl->seekTableIndex + 4));
(U32)start + 4));
if (fl->checksumFlag) {
CHECK_Z(ZSTD_stwrite32(
fl, output, fl->entries[fl->seekTableIndex].checksum,
ZSTD_skippableHeaderSize +
sizePerFrame * fl->seekTableIndex + 8));
(U32)start + 8));
}
fl->seekTableIndex++;
}
assert(seekTableLen <= UINT_MAX);
CHECK_Z(ZSTD_stwrite32(fl, output, fl->size,
seekTableLen - ZSTD_seekTableFooterSize));
(U32)seekTableLen - ZSTD_seekTableFooterSize));
if (output->size - output->pos < 1) return seekTableLen - fl->seekTablePos;
if (fl->seekTablePos < seekTableLen - 4) {
@ -345,7 +348,7 @@ size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output)
}
CHECK_Z(ZSTD_stwrite32(fl, output, ZSTD_SEEKABLE_MAGICNUMBER,
seekTableLen - 4));
(U32)seekTableLen - 4));
if (fl->seekTablePos != seekTableLen) return ERROR(GENERIC);
return 0;

31
sys/contrib/zstd/contrib/seekable_format/zstdseek_decompress.c
View File

@ -54,8 +54,9 @@
# define LONG_SEEK fseek
#endif
#include <stdlib.h> /* malloc, free */
#include <stdio.h> /* FILE* */
#include <stdlib.h> /* malloc, free */
#include <stdio.h> /* FILE* */
#include <limits.h> /* UNIT_MAX */
#include <assert.h>
#define XXH_STATIC_LINKING_ONLY
@ -200,13 +201,14 @@ size_t ZSTD_seekable_free(ZSTD_seekable* zs)
* Performs a binary search to find the last frame with a decompressed offset
* <= pos
* @return : the frame's index */
U32 ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long long pos)
unsigned ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long long pos)
{
U32 lo = 0;
U32 hi = zs->seekTable.tableLen;
U32 hi = (U32)zs->seekTable.tableLen;
assert(zs->seekTable.tableLen <= UINT_MAX);
if (pos >= zs->seekTable.entries[zs->seekTable.tableLen].dOffset) {
return zs->seekTable.tableLen;
return (U32)zs->seekTable.tableLen;
}
while (lo + 1 < hi) {
@ -220,31 +222,32 @@ U32 ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long long
return lo;
}
U32 ZSTD_seekable_getNumFrames(ZSTD_seekable* const zs)
unsigned ZSTD_seekable_getNumFrames(ZSTD_seekable* const zs)
{
return zs->seekTable.tableLen;
assert(zs->seekTable.tableLen <= UINT_MAX);
return (unsigned)zs->seekTable.tableLen;
}
unsigned long long ZSTD_seekable_getFrameCompressedOffset(ZSTD_seekable* const zs, U32 frameIndex)
unsigned long long ZSTD_seekable_getFrameCompressedOffset(ZSTD_seekable* const zs, unsigned frameIndex)
{
if (frameIndex >= zs->seekTable.tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE;
return zs->seekTable.entries[frameIndex].cOffset;
}
unsigned long long ZSTD_seekable_getFrameDecompressedOffset(ZSTD_seekable* const zs, U32 frameIndex)
unsigned long long ZSTD_seekable_getFrameDecompressedOffset(ZSTD_seekable* const zs, unsigned frameIndex)
{
if (frameIndex >= zs->seekTable.tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE;
return zs->seekTable.entries[frameIndex].dOffset;
}
size_t ZSTD_seekable_getFrameCompressedSize(ZSTD_seekable* const zs, U32 frameIndex)
size_t ZSTD_seekable_getFrameCompressedSize(ZSTD_seekable* const zs, unsigned frameIndex)
{
if (frameIndex >= zs->seekTable.tableLen) return ERROR(frameIndex_tooLarge);
return zs->seekTable.entries[frameIndex + 1].cOffset -
zs->seekTable.entries[frameIndex].cOffset;
}
size_t ZSTD_seekable_getFrameDecompressedSize(ZSTD_seekable* const zs, U32 frameIndex)
size_t ZSTD_seekable_getFrameDecompressedSize(ZSTD_seekable* const zs, unsigned frameIndex)
{
if (frameIndex > zs->seekTable.tableLen) return ERROR(frameIndex_tooLarge);
return zs->seekTable.entries[frameIndex + 1].dOffset -
@ -275,7 +278,7 @@ static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs)
{ U32 const numFrames = MEM_readLE32(zs->inBuff);
U32 const sizePerEntry = 8 + (checksumFlag?4:0);
U32 const tableSize = sizePerEntry * numFrames;
U32 const frameSize = tableSize + ZSTD_seekTableFooterSize + ZSTD_skippableHeaderSize;
U32 const frameSize = tableSize + ZSTD_seekTableFooterSize + ZSTD_SKIPPABLEHEADERSIZE;
U32 remaining = frameSize - ZSTD_seekTableFooterSize; /* don't need to re-read footer */
{
@ -290,7 +293,7 @@ static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs)
if (MEM_readLE32(zs->inBuff) != (ZSTD_MAGIC_SKIPPABLE_START | 0xE)) {
return ERROR(prefix_unknown);
}
if (MEM_readLE32(zs->inBuff+4) + ZSTD_skippableHeaderSize != frameSize) {
if (MEM_readLE32(zs->inBuff+4) + ZSTD_SKIPPABLEHEADERSIZE != frameSize) {
return ERROR(prefix_unknown);
}
@ -444,7 +447,7 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign
return len;
}
size_t ZSTD_seekable_decompressFrame(ZSTD_seekable* zs, void* dst, size_t dstSize, U32 frameIndex)
size_t ZSTD_seekable_decompressFrame(ZSTD_seekable* zs, void* dst, size_t dstSize, unsigned frameIndex)
{
if (frameIndex >= zs->seekTable.tableLen) {
return ERROR(frameIndex_tooLarge);

37
sys/contrib/zstd/doc/zstd_compression_format.md
View File

@ -16,7 +16,7 @@ Distribution of this document is unlimited.
### Version
0.3.0 (25/09/18)
0.3.1 (25/10/18)
Introduction
@ -913,11 +913,37 @@ Note that blocks which are not `Compressed_Block` are skipped, they do not contr
###### Offset updates rules
The newest offset takes the lead in offset history,
shifting others back (up to its previous place if it was already present).
shifting others back by one rank,
up to the previous rank of the new offset _if it was present in history_.
__Examples__ :
In the common case, when new offset is not part of history :
`Repeated_Offset3` = `Repeated_Offset2`
`Repeated_Offset2` = `Repeated_Offset1`
`Repeated_Offset1` = `NewOffset`
When the new offset _is_ part of history, there may be specific adjustments.
When `NewOffset` == `Repeated_Offset1`, offset history remains actually unmodified.
When `NewOffset` == `Repeated_Offset2`,
`Repeated_Offset1` and `Repeated_Offset2` ranks are swapped.
`Repeated_Offset3` is unmodified.
When `NewOffset` == `Repeated_Offset3`,
there is actually no difference with the common case :
all offsets are shifted by one rank,
`NewOffset` (== `Repeated_Offset3`) becomes the new `Repeated_Offset1`.
Also worth mentioning, the specific corner case when `offset_value` == 3,
and the literal length of the current sequence is zero.
In which case , `NewOffset` = `Repeated_Offset1` - 1_byte.
Here also, from an offset history update perspective, it's just a common case :
`Repeated_Offset3` = `Repeated_Offset2`
`Repeated_Offset2` = `Repeated_Offset1`
`Repeated_Offset1` = `NewOffset` ( == `Repeated_Offset1` - 1_byte )
This means that when `Repeated_Offset1` (most recent) is used, history is unmodified.
When `Repeated_Offset2` is used, it's swapped with `Repeated_Offset1`.
If any other offset is used, it becomes `Repeated_Offset1` and the rest are shift back by one.
Skippable Frames
@ -1629,6 +1655,7 @@ or at least provide a meaningful error code explaining for which reason it canno
Version changes
---------------
- 0.3.1 : minor clarification regarding offset history update rules
- 0.3.0 : minor edits to match RFC8478
- 0.2.9 : clarifications for huffman weights direct representation, by Ulrich Kunitz
- 0.2.8 : clarifications for IETF RFC discuss

1370
sys/contrib/zstd/doc/zstd_manual.html
View File

File diff suppressed because it is too large Load Diff

17
sys/contrib/zstd/lib/BUCK
View File

@ -1,6 +1,7 @@
cxx_library(
name='zstd',
header_namespace='',
exported_headers=['zstd.h'],
visibility=['PUBLIC'],
deps=[
':common',
@ -17,7 +18,7 @@ cxx_library(
exported_headers=subdir_glob([
('compress', 'zstd*.h'),
]),
srcs=glob(['compress/zstd*.c']),
srcs=glob(['compress/zstd*.c', 'compress/hist.c']),
deps=[':common'],
)
@ -40,7 +41,7 @@ cxx_library(
header_namespace='',
visibility=['PUBLIC'],
exported_headers=subdir_glob([
('decprecated', '*.h'),
('deprecated', '*.h'),
]),
srcs=glob(['deprecated/*.c']),
deps=[':common'],
@ -118,6 +119,7 @@ cxx_library(
'decompress/huf_decompress.c',
],
deps=[
':debug',
':bitstream',
':compiler',
':errors',
@ -204,9 +206,20 @@ cxx_library(
],
)
cxx_library(
name='debug',
header_namespace='',
visibility=['PUBLIC'],
exported_headers=subdir_glob([
('common', 'debug.h'),
]),
srcs=['common/debug.c'],
)
cxx_library(
name='common',
deps=[
':debug',
':bitstream',
':compiler',
':cpu',

39
sys/contrib/zstd/lib/Makefile
View File

@ -27,11 +27,16 @@ DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-Wredundant-decls -Wmissing-prototypes
-Wredundant-decls -Wmissing-prototypes -Wc++-compat
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS)
GREP = grep --color=never
HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
GREP_OPTIONS ?=
ifeq ($HAVE_COLORNEVER, 1)
GREP_OPTIONS += --color=never
endif
GREP = grep $(GREP_OPTIONS)
ZSTDCOMMON_FILES := $(sort $(wildcard common/*.c))
ZSTDCOMP_FILES := $(sort $(wildcard compress/*.c))
@ -45,6 +50,12 @@ ZSTD_LIB_COMPRESSION ?= 1
ZSTD_LIB_DECOMPRESSION ?= 1
ZSTD_LIB_DICTBUILDER ?= 1
ZSTD_LIB_DEPRECATED ?= 1
HUF_FORCE_DECOMPRESS_X1 ?= 0
HUF_FORCE_DECOMPRESS_X2 ?= 0
ZSTD_FORCE_DECOMPRESS_SHORT ?= 0
ZSTD_FORCE_DECOMPRESS_LONG ?= 0
ZSTD_NO_INLINE ?= 0
ZSTD_STRIP_ERROR_STRINGS ?= 0
ifeq ($(ZSTD_LIB_COMPRESSION), 0)
ZSTD_LIB_DICTBUILDER = 0
@ -72,6 +83,30 @@ ifneq ($(ZSTD_LIB_DICTBUILDER), 0)
ZSTD_FILES += $(ZDICT_FILES)
endif
ifneq ($(HUF_FORCE_DECOMPRESS_X1), 0)
CFLAGS += -DHUF_FORCE_DECOMPRESS_X1
endif
ifneq ($(HUF_FORCE_DECOMPRESS_X2), 0)
CFLAGS += -DHUF_FORCE_DECOMPRESS_X2
endif
ifneq ($(ZSTD_FORCE_DECOMPRESS_SHORT), 0)
CFLAGS += -DZSTD_FORCE_DECOMPRESS_SHORT
endif
ifneq ($(ZSTD_FORCE_DECOMPRESS_LONG), 0)
CFLAGS += -DZSTD_FORCE_DECOMPRESS_LONG
endif
ifneq ($(ZSTD_NO_INLINE), 0)
CFLAGS += -DZSTD_NO_INLINE
endif
ifneq ($(ZSTD_STRIP_ERROR_STRINGS), 0)
CFLAGS += -DZSTD_STRIP_ERROR_STRINGS
endif
ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')

108
sys/contrib/zstd/lib/README.md
View File

@ -7,13 +7,32 @@ in order to make it easier to select or exclude features.
#### Building
`Makefile` script is provided, supporting all standard [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions),
`Makefile` script is provided, supporting [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions),
including commands variables, staged install, directory variables and standard targets.
- `make` : generates both static and dynamic libraries
- `make install` : install libraries in default system directories
- `make install` : install libraries and headers in target system directories
`libzstd` default scope includes compression, decompression, dictionary building,
and decoding support for legacy formats >= v0.5.0.
`libzstd` default scope is pretty large, including compression, decompression, dictionary builder,
and support for decoding legacy formats >= v0.5.0.
The scope can be reduced on demand (see paragraph _modular build_).
#### Multithreading support
Multithreading is disabled by default when building with `make`.
Enabling multithreading requires 2 conditions :
- set build macro `ZSTD_MULTITHREAD` (`-DZSTD_MULTITHREAD` for `gcc`)
- for POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`)
Both conditions are automatically applied when invoking `make lib-mt` target.
When linking a POSIX program with a multithreaded version of `libzstd`,
note that it's necessary to request the `-pthread` flag during link stage.
Multithreading capabilities are exposed
via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.3.8/lib/zstd.h#L592).
This API is still labelled experimental,
but is expected to become "stable" in the near future.
#### API
@ -26,63 +45,70 @@ Zstandard's stable API is exposed within [lib/zstd.h](zstd.h).
Optional advanced features are exposed via :
- `lib/common/zstd_errors.h` : translates `size_t` function results
into an `ZSTD_ErrorCode`, for accurate error handling.
into a `ZSTD_ErrorCode`, for accurate error handling.
- `ZSTD_STATIC_LINKING_ONLY` : if this macro is defined _before_ including `zstd.h`,
it unlocks access to advanced experimental API,
exposed in second part of `zstd.h`.
These APIs are not "stable", their definition may change in the future.
As a consequence, it shall ___never be used with dynamic library___ !
it unlocks access to the experimental API,
exposed in the second part of `zstd.h`.
All definitions in the experimental APIs are unstable,
they may still change in the future, or even be removed.
As a consequence, experimental definitions shall ___never be used with dynamic library___ !
Only static linking is allowed.
#### Modular build
It's possible to compile only a limited set of features.
It's possible to compile only a limited set of features within `libzstd`.
The file structure is designed to make this selection manually achievable for any build system :
- Directory `lib/common` is always required, for all variants.
- Compression source code lies in `lib/compress`
- Decompression source code lies in `lib/decompress`
- It's possible to include only `compress` or only `decompress`, they don't depend on each other.
- `lib/dictBuilder` : makes it possible to generate dictionaries from a set of samples.
The API is exposed in `lib/dictBuilder/zdict.h`.
This module depends on both `lib/common` and `lib/compress` .
- `lib/legacy` : source code to decompress legacy zstd formats, starting from `v0.1.0`.
- `lib/legacy` : makes it possible to decompress legacy zstd formats, starting from `v0.1.0`.
This module depends on `lib/common` and `lib/decompress`.
To enable this feature, define `ZSTD_LEGACY_SUPPORT` during compilation.
Specifying a number limits versions supported to that version onward.
For example, `ZSTD_LEGACY_SUPPORT=2` means : "support legacy formats >= v0.2.0".
`ZSTD_LEGACY_SUPPORT=3` means : "support legacy formats >= v0.3.0", and so on.
Currently, the default library setting is `ZST_LEGACY_SUPPORT=5`.
It can be changed at build by any other value.
Note that any number >= 8 translates into "do __not__ support legacy formats",
since all versions of `zstd` >= v0.8 are compatible with v1+ specification.
`ZSTD_LEGACY_SUPPORT=0` also means "do __not__ support legacy formats".
Once enabled, this capability is transparently triggered within decompression functions.
It's also possible to invoke directly legacy API, as exposed in `lib/legacy/zstd_legacy.h`.
Each version also provides an additional dedicated set of advanced API.
Conversely, `ZSTD_LEGACY_SUPPORT=0` means "do __not__ support legacy formats".
By default, this build macro is set as `ZSTD_LEGACY_SUPPORT=5`.
Decoding supported legacy format is a transparent capability triggered within decompression functions.
It's also allowed to invoke legacy API directly, exposed in `lib/legacy/zstd_legacy.h`.
Each version does also provide its own set of advanced API.
For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` .
Note : `lib/legacy` only supports _decoding_ legacy formats.
- Similarly, you can define `ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`,
and `ZSTD_LIB_DEPRECATED` as 0 to forgo compilation of the corresponding features. This will
also disable compilation of all dependencies (eg. `ZSTD_LIB_COMPRESSION=0` will also disable
dictBuilder).
- While invoking `make libzstd`, it's possible to define build macros
`ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`,
and `ZSTD_LIB_DEPRECATED` as `0` to forgo compilation of the corresponding features.
This will also disable compilation of all dependencies
(eg. `ZSTD_LIB_COMPRESSION=0` will also disable dictBuilder).
#### Multithreading support
- There are some additional build macros that can be used to minify the decoder.
Multithreading is disabled by default when building with `make`.
Enabling multithreading requires 2 conditions :
- set macro `ZSTD_MULTITHREAD`
- on POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`)
Zstandard often has more than one implementation of a piece of functionality,
where each implementation optimizes for different scenarios. For example, the
Huffman decoder has complementary implementations that decode the stream one
symbol at a time or two symbols at a time. Zstd normally includes both (and
dispatches between them at runtime), but by defining `HUF_FORCE_DECOMPRESS_X1`
or `HUF_FORCE_DECOMPRESS_X2`, you can force the use of one or the other, avoiding
compilation of the other. Similarly, `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT`
and `ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG` force the compilation and use of
only one or the other of two decompression implementations. The smallest
binary is achieved by using `HUF_FORCE_DECOMPRESS_X1` and
`ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT`.
Both conditions are automatically triggered by invoking `make lib-mt` target.
Note that, when linking a POSIX program with a multithreaded version of `libzstd`,
it's necessary to trigger `-pthread` flag during link stage.
Multithreading capabilities are exposed
via [advanced API `ZSTD_compress_generic()` defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/dev/lib/zstd.h#L919).
This API is still considered experimental,
but is expected to become "stable" at some point in the future.
For squeezing the last ounce of size out, you can also define
`ZSTD_NO_INLINE`, which disables inlining, and `ZSTD_STRIP_ERROR_STRINGS`,
which removes the error messages that are otherwise returned by
`ZSTD_getErrorName`.
#### Windows : using MinGW+MSYS to create DLL
@ -113,8 +139,8 @@ Consider migrating code towards supported streaming API exposed in `zstd.h`.
The other files are not source code. There are :
- `LICENSE` : contains the BSD license text
- `Makefile` : `make` script to build and install zstd library (static and dynamic)
- `BUCK` : support for `buck` build system (https://buckbuild.com/)
- `libzstd.pc.in` : for `pkg-config` (used in `make install`)
- `Makefile` : `make` script to build and install zstd library (static and dynamic)
- `README.md` : this file
- `dll/` : resources directory for Windows compilation
- `libzstd.pc.in` : script for `pkg-config` (used in `make install`)

4
sys/contrib/zstd/lib/common/bitstream.h
View File

@ -389,7 +389,7 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
* Read (consume) next n bits from local register and update.
* Pay attention to not read more than nbBits contained into local register.
* @return : extracted value. */
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
{
size_t const value = BIT_lookBits(bitD, nbBits);
BIT_skipBits(bitD, nbBits);
@ -398,7 +398,7 @@ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
/*! BIT_readBitsFast() :
* unsafe version; only works only if nbBits >= 1 */
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
{
size_t const value = BIT_lookBitsFast(bitD, nbBits);
assert(nbBits >= 1);

31
sys/contrib/zstd/lib/common/compiler.h
View File

@ -15,6 +15,8 @@
* Compiler specifics
*********************************************************/
/* force inlining */
#if !defined(ZSTD_NO_INLINE)
#if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# define INLINE_KEYWORD inline
#else
@ -29,6 +31,13 @@
# define FORCE_INLINE_ATTR
#endif
#else
#define INLINE_KEYWORD
#define FORCE_INLINE_ATTR
#endif
/**
* FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant
* parameters. They must be inlined for the compiler to elimininate the constant
@ -89,23 +98,21 @@
#endif
/* prefetch
* can be disabled, by declaring NO_PREFETCH macro
* All prefetch invocations use a single default locality 2,
* generating instruction prefetcht1,
* which, according to Intel, means "load data into L2 cache".
* This is a good enough "middle ground" for the time being,
* though in theory, it would be better to specialize locality depending on data being prefetched.
* Tests could not determine any sensible difference based on locality value. */
* can be disabled, by declaring NO_PREFETCH build macro */
#if defined(NO_PREFETCH)
# define PREFETCH(ptr) (void)(ptr) /* disabled */
# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */
# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */
#else
# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */
# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
# define PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
# define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
# define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
# define PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
# define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
# define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
# else
# define PREFETCH(ptr) (void)(ptr) /* disabled */
# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */
# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */
# endif
#endif /* NO_PREFETCH */
@ -116,7 +123,7 @@
size_t const _size = (size_t)(s); \
size_t _pos; \
for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \
PREFETCH(_ptr + _pos); \
PREFETCH_L2(_ptr + _pos); \
} \
}

2
sys/contrib/zstd/lib/common/cpu.h
View File

@ -78,7 +78,7 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
__asm__(
"pushl %%ebx\n\t"
"cpuid\n\t"
"movl %%ebx, %%eax\n\r"
"movl %%ebx, %%eax\n\t"
"popl %%ebx"
: "=a"(f7b), "=c"(f7c)
: "a"(7), "c"(0)

33
sys/contrib/zstd/lib/common/debug.h
View File

@ -57,9 +57,9 @@ extern "C" {
#endif
/* static assert is triggered at compile time, leaving no runtime artefact,
* but can only work with compile-time constants.
* This variant can only be used inside a function. */
/* static assert is triggered at compile time, leaving no runtime artefact.
* static assert only works with compile-time constants.
* Also, this variant can only be used inside a function. */
#define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1])
@ -70,9 +70,19 @@ extern "C" {
# define DEBUGLEVEL 0
#endif
/* DEBUGFILE can be defined externally,
* typically through compiler command line.
* note : currently useless.
* Value must be stderr or stdout */
#ifndef DEBUGFILE
# define DEBUGFILE stderr
#endif
/* recommended values for DEBUGLEVEL :
* 0 : no debug, all run-time functions disabled
* 1 : no display, enables assert() only
* 0 : release mode, no debug, all run-time checks disabled
* 1 : enables assert() only, no display
* 2 : reserved, for currently active debug path
* 3 : events once per object lifetime (CCtx, CDict, etc.)
* 4 : events once per frame
@ -81,7 +91,7 @@ extern "C" {
* 7+: events at every position (*very* verbose)
*
* It's generally inconvenient to output traces > 5.
* In which case, it's possible to selectively enable higher verbosity levels
* In which case, it's possible to selectively trigger high verbosity levels
* by modifying g_debug_level.
*/
@ -95,11 +105,12 @@ extern "C" {
#if (DEBUGLEVEL>=2)
# include <stdio.h>
extern int g_debuglevel; /* here, this variable is only declared,
it actually lives in debug.c,
and is shared by the whole process.
It's typically used to enable very verbose levels
on selective conditions (such as position in src) */
extern int g_debuglevel; /* the variable is only declared,
it actually lives in debug.c,
and is shared by the whole process.
It's not thread-safe.
It's useful when enabling very verbose levels
on selective conditions (such as position in src) */
# define RAWLOG(l, ...) { \
if (l<=g_debuglevel) { \

6
sys/contrib/zstd/lib/common/error_private.c
View File

@ -14,6 +14,10 @@
const char* ERR_getErrorString(ERR_enum code)
{
#ifdef ZSTD_STRIP_ERROR_STRINGS
(void)code;
return "Error strings stripped";
#else
static const char* const notErrorCode = "Unspecified error code";
switch( code )
{
@ -39,10 +43,12 @@ const char* ERR_getErrorString(ERR_enum code)
case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
case PREFIX(srcSize_wrong): return "Src size is incorrect";
case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer";
/* following error codes are not stable and may be removed or changed in a future version */
case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
case PREFIX(maxCode):
default: return notErrorCode;
}
#endif
}

4
sys/contrib/zstd/lib/common/fse.h
View File

@ -512,7 +512,7 @@ MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
const U32 tableLog = MEM_read16(ptr);
statePtr->value = (ptrdiff_t)1<<tableLog;
statePtr->stateTable = u16ptr+2;
statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1);
statePtr->stateLog = tableLog;
}
@ -531,7 +531,7 @@ MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U3
}
}
MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol)
MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol)
{
FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
const U16* const stateTable = (const U16*)(statePtr->stateTable);

26
sys/contrib/zstd/lib/common/huf.h
View File

@ -173,15 +173,19 @@ typedef U32 HUF_DTable;
* Advanced decompression functions
******************************************/
size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
#ifndef HUF_FORCE_DECOMPRESS_X1
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
#endif
size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */
size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */
size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
#ifndef HUF_FORCE_DECOMPRESS_X1
size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
#endif
/* ****************************************
@ -228,7 +232,7 @@ size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1)
#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
size_t HUF_buildCTable_wksp (HUF_CElt* tree,
const U32* count, U32 maxSymbolValue, U32 maxNbBits,
const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
void* workSpace, size_t wkspSize);
/*! HUF_readStats() :
@ -277,14 +281,22 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
#define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10)
#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
#ifndef HUF_FORCE_DECOMPRESS_X2
size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize);
size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
#endif
#ifndef HUF_FORCE_DECOMPRESS_X1
size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
#endif
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
#ifndef HUF_FORCE_DECOMPRESS_X2
size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
#endif
#ifndef HUF_FORCE_DECOMPRESS_X1
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
#endif
/* ====================== */
@ -306,24 +318,36 @@ size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
#ifndef HUF_FORCE_DECOMPRESS_X1
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
#endif
size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);
#ifndef HUF_FORCE_DECOMPRESS_X2
size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
#endif
#ifndef HUF_FORCE_DECOMPRESS_X1
size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
#endif
size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */
#ifndef HUF_FORCE_DECOMPRESS_X2
size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
#endif
#ifndef HUF_FORCE_DECOMPRESS_X1
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
#endif
/* BMI2 variants.
* If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
*/
size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
#ifndef HUF_FORCE_DECOMPRESS_X2
size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
#endif
size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);

2
sys/contrib/zstd/lib/common/pool.c
View File

@ -88,8 +88,8 @@ static void* POOL_thread(void* opaque) {
ctx->numThreadsBusy++;
ctx->queueEmpty = ctx->queueHead == ctx->queueTail;
/* Unlock the mutex, signal a pusher, and run the job */
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
job.function(job.opaque);

4
sys/contrib/zstd/lib/common/zstd_common.c
View File

@ -30,8 +30,10 @@ const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; }
/*-****************************************
* ZSTD Error Management
******************************************/
#undef ZSTD_isError /* defined within zstd_internal.h */
/*! ZSTD_isError() :
* tells if a return value is an error code */
* tells if a return value is an error code
* symbol is required for external callers */
unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
/*! ZSTD_getErrorName() :

1
sys/contrib/zstd/lib/common/zstd_errors.h
View File

@ -72,6 +72,7 @@ typedef enum {
ZSTD_error_workSpace_tooSmall= 66,
ZSTD_error_dstSize_tooSmall = 70,
ZSTD_error_srcSize_wrong = 72,
ZSTD_error_dstBuffer_null = 74,
/* following error codes are __NOT STABLE__, they can be removed or changed in future versions */
ZSTD_error_frameIndex_tooLarge = 100,
ZSTD_error_seekableIO = 102,

13
sys/contrib/zstd/lib/common/zstd_internal.h
View File

@ -41,6 +41,9 @@ extern "C" {
/* ---- static assert (debug) --- */
#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c)
#define ZSTD_isError ERR_isError /* for inlining */
#define FSE_isError ERR_isError
#define HUF_isError ERR_isError
/*-*************************************
@ -75,7 +78,6 @@ static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
#define BIT0 1
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
#define ZSTD_WINDOWLOG_DEFAULTMAX 27 /* Default maximum allowed window log */
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
@ -242,7 +244,7 @@ typedef struct {
blockType_e blockType;
U32 lastBlock;
U32 origSize;
} blockProperties_t;
} blockProperties_t; /* declared here for decompress and fullbench */
/*! ZSTD_getcBlockSize() :
* Provides the size of compressed block from block header `src` */
@ -250,6 +252,13 @@ typedef struct {
size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
blockProperties_t* bpPtr);
/*! ZSTD_decodeSeqHeaders() :
* decode sequence header from src */
/* Used by: decompress, fullbench (does not get its definition from here) */
size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
const void* src, size_t srcSize);
#if defined (__cplusplus)
}
#endif

6
sys/contrib/zstd/lib/compress/fse_compress.c
View File

@ -115,7 +115,7 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
/* symbol start positions */
{ U32 u;
cumul[0] = 0;
for (u=1; u<=maxSymbolValue+1; u++) {
for (u=1; u <= maxSymbolValue+1; u++) {
if (normalizedCounter[u-1]==-1) { /* Low proba symbol */
cumul[u] = cumul[u-1] + 1;
tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1);
@ -658,7 +658,7 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src
BYTE* op = ostart;
BYTE* const oend = ostart + dstSize;
U32 count[FSE_MAX_SYMBOL_VALUE+1];
unsigned count[FSE_MAX_SYMBOL_VALUE+1];
S16 norm[FSE_MAX_SYMBOL_VALUE+1];
FSE_CTable* CTable = (FSE_CTable*)workSpace;
size_t const CTableSize = FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue);
@ -672,7 +672,7 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src
if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG;
/* Scan input and build symbol stats */
{ CHECK_V_F(maxCount, HIST_count_wksp(count, &maxSymbolValue, src, srcSize, (unsigned*)scratchBuffer) );
{ CHECK_V_F(maxCount, HIST_count_wksp(count, &maxSymbolValue, src, srcSize, scratchBuffer, scratchBufferSize) );
if (maxCount == srcSize) return 1; /* only a single symbol in src : rle */
if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */
if (maxCount < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */

30
sys/contrib/zstd/lib/compress/hist.c
View File

@ -73,6 +73,7 @@ unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
return largestCount;
}
typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e;
/* HIST_count_parallel_wksp() :
* store histogram into 4 intermediate tables, recombined at the end.
@ -85,8 +86,8 @@ unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
static size_t HIST_count_parallel_wksp(
unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize,
unsigned checkMax,
unsigned* const workSpace)
HIST_checkInput_e check,
U32* const workSpace)
{
const BYTE* ip = (const BYTE*)source;
const BYTE* const iend = ip+sourceSize;
@ -137,7 +138,7 @@ static size_t HIST_count_parallel_wksp(
/* finish last symbols */
while (ip<iend) Counting1[*ip++]++;
if (checkMax) { /* verify stats will fit into destination table */
if (check) { /* verify stats will fit into destination table */
U32 s; for (s=255; s>maxSymbolValue; s--) {
Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s];
if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall);
@ -157,14 +158,18 @@ static size_t HIST_count_parallel_wksp(
/* HIST_countFast_wksp() :
* Same as HIST_countFast(), but using an externally provided scratch buffer.
* `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */
* `workSpace` is a writable buffer which must be 4-bytes aligned,
* `workSpaceSize` must be >= HIST_WKSP_SIZE
*/
size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize,
unsigned* workSpace)
void* workSpace, size_t workSpaceSize)
{
if (sourceSize < 1500) /* heuristic threshold */
return HIST_count_simple(count, maxSymbolValuePtr, source, sourceSize);
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 0, workSpace);
if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall);
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace);
}
/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */
@ -172,24 +177,27 @@ size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize)
{
unsigned tmpCounters[HIST_WKSP_SIZE_U32];
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters);
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters));
}
/* HIST_count_wksp() :
* Same as HIST_count(), but using an externally provided scratch buffer.
* `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */
size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize, unsigned* workSpace)
const void* source, size_t sourceSize,
void* workSpace, size_t workSpaceSize)
{
if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall);
if (*maxSymbolValuePtr < 255)
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 1, workSpace);
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, checkMaxSymbolValue, (U32*)workSpace);
*maxSymbolValuePtr = 255;
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace);
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize);
}
size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr,
const void* src, size_t srcSize)
{
unsigned tmpCounters[HIST_WKSP_SIZE_U32];
return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters);
return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters));
}

19
sys/contrib/zstd/lib/compress/hist.h
View File

@ -41,11 +41,11 @@
/*! HIST_count():
* Provides the precise count of each byte within a table 'count'.
* 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1).
* 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1).
* Updates *maxSymbolValuePtr with actual largest symbol value detected.
* @return : count of the most frequent symbol (which isn't identified).
* or an error code, which can be tested using HIST_isError().
* note : if return == srcSize, there is only one symbol.
* @return : count of the most frequent symbol (which isn't identified).
* or an error code, which can be tested using HIST_isError().
* note : if return == srcSize, there is only one symbol.
*/
size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr,
const void* src, size_t srcSize);
@ -56,14 +56,16 @@ unsigned HIST_isError(size_t code); /**< tells if a return value is an error co
/* --- advanced histogram functions --- */
#define HIST_WKSP_SIZE_U32 1024
#define HIST_WKSP_SIZE (HIST_WKSP_SIZE_U32 * sizeof(unsigned))
/** HIST_count_wksp() :
* Same as HIST_count(), but using an externally provided scratch buffer.
* Benefit is this function will use very little stack space.
* `workSpace` must be a table of unsigned of size >= HIST_WKSP_SIZE_U32
* `workSpace` is a writable buffer which must be 4-bytes aligned,
* `workSpaceSize` must be >= HIST_WKSP_SIZE
*/
size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
const void* src, size_t srcSize,
unsigned* workSpace);
void* workSpace, size_t workSpaceSize);
/** HIST_countFast() :
* same as HIST_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr.
@ -74,11 +76,12 @@ size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
/** HIST_countFast_wksp() :
* Same as HIST_countFast(), but using an externally provided scratch buffer.
* `workSpace` must be a table of unsigned of size >= HIST_WKSP_SIZE_U32
* `workSpace` is a writable buffer which must be 4-bytes aligned,
* `workSpaceSize` must be >= HIST_WKSP_SIZE
*/
size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
const void* src, size_t srcSize,
unsigned* workSpace);
void* workSpace, size_t workSpaceSize);
/*! HIST_count_simple() :
* Same as HIST_countFast(), this function is unsafe,

64
sys/contrib/zstd/lib/compress/huf_compress.c
View File

@ -88,13 +88,13 @@ static size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weight
BYTE* op = ostart;
BYTE* const oend = ostart + dstSize;
U32 maxSymbolValue = HUF_TABLELOG_MAX;
unsigned maxSymbolValue = HUF_TABLELOG_MAX;
U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)];
BYTE scratchBuffer[1<<MAX_FSE_TABLELOG_FOR_HUFF_HEADER];
U32 count[HUF_TABLELOG_MAX+1];
unsigned count[HUF_TABLELOG_MAX+1];
S16 norm[HUF_TABLELOG_MAX+1];
/* init conditions */
@ -134,7 +134,7 @@ struct HUF_CElt_s {
`CTable` : Huffman tree to save, using huf representation.
@return : size of saved CTable */
size_t HUF_writeCTable (void* dst, size_t maxDstSize,
const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog)
{
BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */
BYTE huffWeight[HUF_SYMBOLVALUE_MAX];
@ -169,7 +169,7 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize,
}
size_t HUF_readCTable (HUF_CElt* CTable, U32* maxSymbolValuePtr, const void* src, size_t srcSize)
size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize)
{
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
@ -315,7 +315,7 @@ typedef struct {
U32 current;
} rankPos;
static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
static void HUF_sort(nodeElt* huffNode, const unsigned* count, U32 maxSymbolValue)
{
rankPos rank[32];
U32 n;
@ -347,7 +347,7 @@ static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
*/
#define STARTNODE (HUF_SYMBOLVALUE_MAX+1)
typedef nodeElt huffNodeTable[HUF_CTABLE_WORKSPACE_SIZE_U32];
size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize)
size_t HUF_buildCTable_wksp (HUF_CElt* tree, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize)
{
nodeElt* const huffNode0 = (nodeElt*)workSpace;
nodeElt* const huffNode = huffNode0+1;
@ -421,7 +421,7 @@ size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValu
* @return : maxNbBits
* Note : count is used before tree is written, so they can safely overlap
*/
size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
size_t HUF_buildCTable (HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits)
{
huffNodeTable nodeTable;
return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, nodeTable, sizeof(nodeTable));
@ -610,13 +610,14 @@ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, si
return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0);
}
typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e;
static size_t HUF_compressCTable_internal(
BYTE* const ostart, BYTE* op, BYTE* const oend,
const void* src, size_t srcSize,
unsigned singleStream, const HUF_CElt* CTable, const int bmi2)
HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int bmi2)
{
size_t const cSize = singleStream ?
size_t const cSize = (nbStreams==HUF_singleStream) ?
HUF_compress1X_usingCTable_internal(op, oend - op, src, srcSize, CTable, bmi2) :
HUF_compress4X_usingCTable_internal(op, oend - op, src, srcSize, CTable, bmi2);
if (HUF_isError(cSize)) { return cSize; }
@ -628,21 +629,21 @@ static size_t HUF_compressCTable_internal(
}
typedef struct {
U32 count[HUF_SYMBOLVALUE_MAX + 1];
unsigned count[HUF_SYMBOLVALUE_MAX + 1];
HUF_CElt CTable[HUF_SYMBOLVALUE_MAX + 1];
huffNodeTable nodeTable;
} HUF_compress_tables_t;
/* HUF_compress_internal() :
* `workSpace` must a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
static size_t HUF_compress_internal (
void* dst, size_t dstSize,
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned huffLog,
unsigned singleStream,
void* workSpace, size_t wkspSize,
HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat,
const int bmi2)
static size_t
HUF_compress_internal (void* dst, size_t dstSize,
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned huffLog,
HUF_nbStreams_e nbStreams,
void* workSpace, size_t wkspSize,
HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat,
const int bmi2)
{
HUF_compress_tables_t* const table = (HUF_compress_tables_t*)workSpace;
BYTE* const ostart = (BYTE*)dst;
@ -651,7 +652,7 @@ static size_t HUF_compress_internal (
/* checks & inits */
if (((size_t)workSpace & 3) != 0) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
if (wkspSize < sizeof(*table)) return ERROR(workSpace_tooSmall);
if (wkspSize < HUF_WORKSPACE_SIZE) return ERROR(workSpace_tooSmall);
if (!srcSize) return 0; /* Uncompressed */
if (!dstSize) return 0; /* cannot fit anything within dst budget */
if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */
@ -664,11 +665,11 @@ static size_t HUF_compress_internal (
if (preferRepeat && repeat && *repeat == HUF_repeat_valid) {
return HUF_compressCTable_internal(ostart, op, oend,
src, srcSize,
singleStream, oldHufTable, bmi2);
nbStreams, oldHufTable, bmi2);
}
/* Scan input and build symbol stats */
{ CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, table->count) );
{ CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, workSpace, wkspSize) );
if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */
if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */
}
@ -683,14 +684,15 @@ static size_t HUF_compress_internal (
if (preferRepeat && repeat && *repeat != HUF_repeat_none) {
return HUF_compressCTable_internal(ostart, op, oend,
src, srcSize,
singleStream, oldHufTable, bmi2);
nbStreams, oldHufTable, bmi2);
}
/* Build Huffman Tree */
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
{ CHECK_V_F(maxBits, HUF_buildCTable_wksp(table->CTable, table->count,
maxSymbolValue, huffLog,
table->nodeTable, sizeof(table->nodeTable)) );
{ size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count,
maxSymbolValue, huffLog,
table->nodeTable, sizeof(table->nodeTable));
CHECK_F(maxBits);
huffLog = (U32)maxBits;
/* Zero unused symbols in CTable, so we can check it for validity */
memset(table->CTable + (maxSymbolValue + 1), 0,
@ -706,7 +708,7 @@ static size_t HUF_compress_internal (
if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) {
return HUF_compressCTable_internal(ostart, op, oend,
src, srcSize,
singleStream, oldHufTable, bmi2);
nbStreams, oldHufTable, bmi2);
} }
/* Use the new huffman table */
@ -718,7 +720,7 @@ static size_t HUF_compress_internal (
}
return HUF_compressCTable_internal(ostart, op, oend,
src, srcSize,
singleStream, table->CTable, bmi2);
nbStreams, table->CTable, bmi2);
}
@ -728,7 +730,7 @@ size_t HUF_compress1X_wksp (void* dst, size_t dstSize,
void* workSpace, size_t wkspSize)
{
return HUF_compress_internal(dst, dstSize, src, srcSize,
maxSymbolValue, huffLog, 1 /*single stream*/,
maxSymbolValue, huffLog, HUF_singleStream,
workSpace, wkspSize,
NULL, NULL, 0, 0 /*bmi2*/);
}
@ -740,7 +742,7 @@ size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2)
{
return HUF_compress_internal(dst, dstSize, src, srcSize,
maxSymbolValue, huffLog, 1 /*single stream*/,
maxSymbolValue, huffLog, HUF_singleStream,
workSpace, wkspSize, hufTable,
repeat, preferRepeat, bmi2);
}
@ -762,7 +764,7 @@ size_t HUF_compress4X_wksp (void* dst, size_t dstSize,
void* workSpace, size_t wkspSize)
{
return HUF_compress_internal(dst, dstSize, src, srcSize,
maxSymbolValue, huffLog, 0 /*4 streams*/,
maxSymbolValue, huffLog, HUF_fourStreams,
workSpace, wkspSize,
NULL, NULL, 0, 0 /*bmi2*/);
}
@ -777,7 +779,7 @@ size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2)
{
return HUF_compress_internal(dst, dstSize, src, srcSize,
maxSymbolValue, huffLog, 0 /* 4 streams */,
maxSymbolValue, huffLog, HUF_fourStreams,
workSpace, wkspSize,
hufTable, repeat, preferRepeat, bmi2);
}

990
sys/contrib/zstd/lib/compress/zstd_compress.c
View File

File diff suppressed because it is too large Load Diff

118
sys/contrib/zstd/lib/compress/zstd_compress_internal.h
View File

@ -48,12 +48,6 @@ extern "C" {
typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e;
typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage;
typedef enum {
ZSTD_dictDefaultAttach = 0,
ZSTD_dictForceAttach = 1,
ZSTD_dictForceCopy = -1,
} ZSTD_dictAttachPref_e;
typedef struct ZSTD_prefixDict_s {
const void* dict;
size_t dictSize;
@ -96,10 +90,10 @@ typedef enum { zop_dynamic=0, zop_predef } ZSTD_OptPrice_e;
typedef struct {
/* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */
U32* litFreq; /* table of literals statistics, of size 256 */
U32* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */
U32* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */
U32* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */
unsigned* litFreq; /* table of literals statistics, of size 256 */
unsigned* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */
unsigned* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */
unsigned* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */
ZSTD_match_t* matchTable; /* list of found matches, of size ZSTD_OPT_NUM+1 */
ZSTD_optimal_t* priceTable; /* All positions tracked by optimal parser, of size ZSTD_OPT_NUM+1 */
@ -139,7 +133,7 @@ struct ZSTD_matchState_t {
U32* hashTable3;
U32* chainTable;
optState_t opt; /* optimal parser state */
const ZSTD_matchState_t *dictMatchState;
const ZSTD_matchState_t * dictMatchState;
ZSTD_compressionParameters cParams;
};
@ -167,7 +161,7 @@ typedef struct {
U32 hashLog; /* Log size of hashTable */
U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */
U32 minMatchLength; /* Minimum match length */
U32 hashEveryLog; /* Log number of entries to skip */
U32 hashRateLog; /* Log number of entries to skip */
U32 windowLog; /* Window log for the LDM */
} ldmParams_t;
@ -196,9 +190,10 @@ struct ZSTD_CCtx_params_s {
ZSTD_dictAttachPref_e attachDictPref;
/* Multithreading: used to pass parameters to mtctx */
unsigned nbWorkers;
unsigned jobSize;
unsigned overlapSizeLog;
int nbWorkers;
size_t jobSize;
int overlapLog;
int rsyncable;
/* Long distance matching parameters */
ldmParams_t ldmParams;
@ -498,6 +493,64 @@ MEM_STATIC size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
}
}
/** ZSTD_ipow() :
* Return base^exponent.
*/
static U64 ZSTD_ipow(U64 base, U64 exponent)
{
U64 power = 1;
while (exponent) {
if (exponent & 1) power *= base;
exponent >>= 1;
base *= base;
}
return power;
}
#define ZSTD_ROLL_HASH_CHAR_OFFSET 10
/** ZSTD_rollingHash_append() :
* Add the buffer to the hash value.
*/
static U64 ZSTD_rollingHash_append(U64 hash, void const* buf, size_t size)
{
BYTE const* istart = (BYTE const*)buf;
size_t pos;
for (pos = 0; pos < size; ++pos) {
hash *= prime8bytes;
hash += istart[pos] + ZSTD_ROLL_HASH_CHAR_OFFSET;
}
return hash;
}
/** ZSTD_rollingHash_compute() :
* Compute the rolling hash value of the buffer.
*/
MEM_STATIC U64 ZSTD_rollingHash_compute(void const* buf, size_t size)
{
return ZSTD_rollingHash_append(0, buf, size);
}
/** ZSTD_rollingHash_primePower() :
* Compute the primePower to be passed to ZSTD_rollingHash_rotate() for a hash
* over a window of length bytes.
*/
MEM_STATIC U64 ZSTD_rollingHash_primePower(U32 length)
{
return ZSTD_ipow(prime8bytes, length - 1);
}
/** ZSTD_rollingHash_rotate() :
* Rotate the rolling hash by one byte.
*/
MEM_STATIC U64 ZSTD_rollingHash_rotate(U64 hash, BYTE toRemove, BYTE toAdd, U64 primePower)
{
hash -= (toRemove + ZSTD_ROLL_HASH_CHAR_OFFSET) * primePower;
hash *= prime8bytes;
hash += toAdd + ZSTD_ROLL_HASH_CHAR_OFFSET;
return hash;
}
/*-*************************************
* Round buffer management
***************************************/
@ -626,20 +679,23 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
* dictMatchState mode, lowLimit and dictLimit are the same, and the dictionary
* is below them. forceWindow and dictMatchState are therefore incompatible.
*/
MEM_STATIC void ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
void const* srcEnd, U32 maxDist,
U32* loadedDictEndPtr,
const ZSTD_matchState_t** dictMatchStatePtr)
MEM_STATIC void
ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
void const* srcEnd,
U32 maxDist,
U32* loadedDictEndPtr,
const ZSTD_matchState_t** dictMatchStatePtr)
{
U32 const current = (U32)((BYTE const*)srcEnd - window->base);
U32 loadedDictEnd = loadedDictEndPtr != NULL ? *loadedDictEndPtr : 0;
DEBUGLOG(5, "ZSTD_window_enforceMaxDist: current=%u, maxDist=%u", current, maxDist);
if (current > maxDist + loadedDictEnd) {
U32 const newLowLimit = current - maxDist;
U32 const blockEndIdx = (U32)((BYTE const*)srcEnd - window->base);
U32 loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0;
DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u",
(unsigned)blockEndIdx, (unsigned)maxDist);
if (blockEndIdx > maxDist + loadedDictEnd) {
U32 const newLowLimit = blockEndIdx - maxDist;
if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit;
if (window->dictLimit < window->lowLimit) {
DEBUGLOG(5, "Update dictLimit to match lowLimit, from %u to %u",
window->dictLimit, window->lowLimit);
(unsigned)window->dictLimit, (unsigned)window->lowLimit);
window->dictLimit = window->lowLimit;
}
if (loadedDictEndPtr)
@ -690,20 +746,23 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
/* debug functions */
#if (DEBUGLEVEL>=2)
MEM_STATIC double ZSTD_fWeight(U32 rawStat)
{
U32 const fp_accuracy = 8;
U32 const fp_multiplier = (1 << fp_accuracy);
U32 const stat = rawStat + 1;
U32 const hb = ZSTD_highbit32(stat);
U32 const newStat = rawStat + 1;
U32 const hb = ZSTD_highbit32(newStat);
U32 const BWeight = hb * fp_multiplier;
U32 const FWeight = (stat << fp_accuracy) >> hb;
U32 const FWeight = (newStat << fp_accuracy) >> hb;
U32 const weight = BWeight + FWeight;
assert(hb + fp_accuracy < 31);
return (double)weight / fp_multiplier;
}
/* display a table content,
* listing each element, its frequency, and its predicted bit cost */
MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max)
{
unsigned u, sum;
@ -715,6 +774,9 @@ MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max)
}
}
#endif
#if defined (__cplusplus)
}
#endif

8
sys/contrib/zstd/lib/compress/zstd_double_fast.c
View File

@ -18,7 +18,7 @@ void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const hashLarge = ms->hashTable;
U32 const hBitsL = cParams->hashLog;
U32 const mls = cParams->searchLength;
U32 const mls = cParams->minMatch;
U32* const hashSmall = ms->chainTable;
U32 const hBitsS = cParams->chainLog;
const BYTE* const base = ms->window.base;
@ -309,7 +309,7 @@ size_t ZSTD_compressBlock_doubleFast(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
const U32 mls = ms->cParams.searchLength;
const U32 mls = ms->cParams.minMatch;
switch(mls)
{
default: /* includes case 3 */
@ -329,7 +329,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
const U32 mls = ms->cParams.searchLength;
const U32 mls = ms->cParams.minMatch;
switch(mls)
{
default: /* includes case 3 */
@ -483,7 +483,7 @@ size_t ZSTD_compressBlock_doubleFast_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
U32 const mls = ms->cParams.searchLength;
U32 const mls = ms->cParams.minMatch;
switch(mls)
{
default: /* includes case 3 */

30
sys/contrib/zstd/lib/compress/zstd_fast.c
View File

@ -18,7 +18,7 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const hashTable = ms->hashTable;
U32 const hBits = cParams->hashLog;
U32 const mls = cParams->searchLength;
U32 const mls = cParams->minMatch;
const BYTE* const base = ms->window.base;
const BYTE* ip = base + ms->nextToUpdate;
const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
@ -27,18 +27,18 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
/* Always insert every fastHashFillStep position into the hash table.
* Insert the other positions if their hash entry is empty.
*/
for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) {
for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) {
U32 const current = (U32)(ip - base);
U32 i;
for (i = 0; i < fastHashFillStep; ++i) {
size_t const hash = ZSTD_hashPtr(ip + i, hBits, mls);
if (i == 0 || hashTable[hash] == 0)
hashTable[hash] = current + i;
/* Only load extra positions for ZSTD_dtlm_full */
if (dtlm == ZSTD_dtlm_fast)
break;
}
}
size_t const hash0 = ZSTD_hashPtr(ip, hBits, mls);
hashTable[hash0] = current;
if (dtlm == ZSTD_dtlm_fast) continue;
/* Only load extra positions for ZSTD_dtlm_full */
{ U32 p;
for (p = 1; p < fastHashFillStep; ++p) {
size_t const hash = ZSTD_hashPtr(ip + p, hBits, mls);
if (hashTable[hash] == 0) { /* not yet filled */
hashTable[hash] = current + p;
} } } }
}
FORCE_INLINE_TEMPLATE
@ -235,7 +235,7 @@ size_t ZSTD_compressBlock_fast(
void const* src, size_t srcSize)
{
ZSTD_compressionParameters const* cParams = &ms->cParams;
U32 const mls = cParams->searchLength;
U32 const mls = cParams->minMatch;
assert(ms->dictMatchState == NULL);
switch(mls)
{
@ -256,7 +256,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState(
void const* src, size_t srcSize)
{
ZSTD_compressionParameters const* cParams = &ms->cParams;
U32 const mls = cParams->searchLength;
U32 const mls = cParams->minMatch;
assert(ms->dictMatchState != NULL);
switch(mls)
{
@ -375,7 +375,7 @@ size_t ZSTD_compressBlock_fast_extDict(
void const* src, size_t srcSize)
{
ZSTD_compressionParameters const* cParams = &ms->cParams;
U32 const mls = cParams->searchLength;
U32 const mls = cParams->minMatch;
switch(mls)
{
default: /* includes case 3 */

43
sys/contrib/zstd/lib/compress/zstd_lazy.c
View File

@ -63,12 +63,13 @@ ZSTD_updateDUBT(ZSTD_matchState_t* ms,
static void
ZSTD_insertDUBT1(ZSTD_matchState_t* ms,
U32 current, const BYTE* inputEnd,
U32 nbCompares, U32 btLow, const ZSTD_dictMode_e dictMode)
U32 nbCompares, U32 btLow,
const ZSTD_dictMode_e dictMode)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const bt = ms->chainTable;
U32 const btLog = cParams->chainLog - 1;
U32 const btMask = (1 << btLog) - 1;
U32* const bt = ms->chainTable;
U32 const btLog = cParams->chainLog - 1;
U32 const btMask = (1 << btLog) - 1;
size_t commonLengthSmaller=0, commonLengthLarger=0;
const BYTE* const base = ms->window.base;
const BYTE* const dictBase = ms->window.dictBase;
@ -80,7 +81,7 @@ ZSTD_insertDUBT1(ZSTD_matchState_t* ms,
const BYTE* match;
U32* smallerPtr = bt + 2*(current&btMask);
U32* largerPtr = smallerPtr + 1;
U32 matchIndex = *smallerPtr;
U32 matchIndex = *smallerPtr; /* this candidate is unsorted : next sorted candidate is reached through *smallerPtr, while *largerPtr contains previous unsorted candidate (which is already saved and can be overwritten) */
U32 dummy32; /* to be nullified at the end */
U32 const windowLow = ms->window.lowLimit;
@ -93,6 +94,9 @@ ZSTD_insertDUBT1(ZSTD_matchState_t* ms,
U32* const nextPtr = bt + 2*(matchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
assert(matchIndex < current);
/* note : all candidates are now supposed sorted,
* but it's still possible to have nextPtr[1] == ZSTD_DUBT_UNSORTED_MARK
* when a real index has the same value as ZSTD_DUBT_UNSORTED_MARK */
if ( (dictMode != ZSTD_extDict)
|| (matchIndex+matchLength >= dictLimit) /* both in current segment*/
@ -108,7 +112,7 @@ ZSTD_insertDUBT1(ZSTD_matchState_t* ms,
match = dictBase + matchIndex;
matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart);
if (matchIndex+matchLength >= dictLimit)
match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
match = base + matchIndex; /* preparation for next read of match[matchLength] */
}
DEBUGLOG(8, "ZSTD_insertDUBT1: comparing %u with %u : found %u common bytes ",
@ -258,7 +262,7 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
&& (nbCandidates > 1) ) {
DEBUGLOG(8, "ZSTD_DUBT_findBestMatch: candidate %u is unsorted",
matchIndex);
*unsortedMark = previousCandidate;
*unsortedMark = previousCandidate; /* the unsortedMark becomes a reversed chain, to move up back to original position */
previousCandidate = matchIndex;
matchIndex = *nextCandidate;
nextCandidate = bt + 2*(matchIndex&btMask);
@ -266,11 +270,13 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
nbCandidates --;
}
/* nullify last candidate if it's still unsorted
* simplification, detrimental to compression ratio, beneficial for speed */
if ( (matchIndex > unsortLimit)
&& (*unsortedMark==ZSTD_DUBT_UNSORTED_MARK) ) {
DEBUGLOG(7, "ZSTD_DUBT_findBestMatch: nullify last unsorted candidate %u",
matchIndex);
*nextCandidate = *unsortedMark = 0; /* nullify next candidate if it's still unsorted (note : simplification, detrimental to compression ratio, beneficial for speed) */
*nextCandidate = *unsortedMark = 0;
}
/* batch sort stacked candidates */
@ -285,14 +291,14 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
}
/* find longest match */
{ size_t commonLengthSmaller=0, commonLengthLarger=0;
{ size_t commonLengthSmaller = 0, commonLengthLarger = 0;
const BYTE* const dictBase = ms->window.dictBase;
const U32 dictLimit = ms->window.dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const prefixStart = base + dictLimit;
U32* smallerPtr = bt + 2*(current&btMask);
U32* largerPtr = bt + 2*(current&btMask) + 1;
U32 matchEndIdx = current+8+1;
U32 matchEndIdx = current + 8 + 1;
U32 dummy32; /* to be nullified at the end */
size_t bestLength = 0;
@ -386,7 +392,7 @@ ZSTD_BtFindBestMatch_selectMLS ( ZSTD_matchState_t* ms,
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
switch(ms->cParams.searchLength)
switch(ms->cParams.minMatch)
{
default : /* includes case 3 */
case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict);
@ -402,7 +408,7 @@ static size_t ZSTD_BtFindBestMatch_dictMatchState_selectMLS (
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
switch(ms->cParams.searchLength)
switch(ms->cParams.minMatch)
{
default : /* includes case 3 */
case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState);
@ -418,7 +424,7 @@ static size_t ZSTD_BtFindBestMatch_extDict_selectMLS (
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
switch(ms->cParams.searchLength)
switch(ms->cParams.minMatch)
{
default : /* includes case 3 */
case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict);
@ -433,7 +439,7 @@ static size_t ZSTD_BtFindBestMatch_extDict_selectMLS (
/* *********************************
* Hash Chain
***********************************/
#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & mask]
#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & (mask)]
/* Update chains up to ip (excluded)
Assumption : always within prefix (i.e. not within extDict) */
@ -463,7 +469,7 @@ static U32 ZSTD_insertAndFindFirstIndex_internal(
U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) {
const ZSTD_compressionParameters* const cParams = &ms->cParams;
return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.searchLength);
return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch);
}
@ -497,6 +503,7 @@ size_t ZSTD_HcFindBestMatch_generic (
size_t currentMl=0;
if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) {
const BYTE* const match = base + matchIndex;
assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */
if (match[ml] == ip[ml]) /* potentially better */
currentMl = ZSTD_count(ip, match, iLimit);
} else {
@ -559,7 +566,7 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS (
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
switch(ms->cParams.searchLength)
switch(ms->cParams.minMatch)
{
default : /* includes case 3 */
case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict);
@ -575,7 +582,7 @@ static size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS (
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
switch(ms->cParams.searchLength)
switch(ms->cParams.minMatch)
{
default : /* includes case 3 */
case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState);
@ -591,7 +598,7 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS (
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
switch(ms->cParams.searchLength)
switch(ms->cParams.minMatch)
{
default : /* includes case 3 */
case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict);

85
sys/contrib/zstd/lib/compress/zstd_ldm.c
View File

@ -37,8 +37,8 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params,
params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG);
assert(params->hashLog <= ZSTD_HASHLOG_MAX);
}
if (params->hashEveryLog == 0) {
params->hashEveryLog = params->windowLog < params->hashLog
if (params->hashRateLog == 0) {
params->hashRateLog = params->windowLog < params->hashLog
? 0
: params->windowLog - params->hashLog;
}
@ -119,20 +119,20 @@ static void ZSTD_ldm_insertEntry(ldmState_t* ldmState,
*
* Gets the small hash, checksum, and tag from the rollingHash.
*
* If the tag matches (1 << ldmParams.hashEveryLog)-1, then
* If the tag matches (1 << ldmParams.hashRateLog)-1, then
* creates an ldmEntry from the offset, and inserts it into the hash table.
*
* hBits is the length of the small hash, which is the most significant hBits
* of rollingHash. The checksum is the next 32 most significant bits, followed
* by ldmParams.hashEveryLog bits that make up the tag. */
* by ldmParams.hashRateLog bits that make up the tag. */
static void ZSTD_ldm_makeEntryAndInsertByTag(ldmState_t* ldmState,
U64 const rollingHash,
U32 const hBits,
U32 const offset,
ldmParams_t const ldmParams)
{
U32 const tag = ZSTD_ldm_getTag(rollingHash, hBits, ldmParams.hashEveryLog);
U32 const tagMask = ((U32)1 << ldmParams.hashEveryLog) - 1;
U32 const tag = ZSTD_ldm_getTag(rollingHash, hBits, ldmParams.hashRateLog);
U32 const tagMask = ((U32)1 << ldmParams.hashRateLog) - 1;
if (tag == tagMask) {
U32 const hash = ZSTD_ldm_getSmallHash(rollingHash, hBits);
U32 const checksum = ZSTD_ldm_getChecksum(rollingHash, hBits);
@ -143,56 +143,6 @@ static void ZSTD_ldm_makeEntryAndInsertByTag(ldmState_t* ldmState,
}
}
/** ZSTD_ldm_getRollingHash() :
* Get a 64-bit hash using the first len bytes from buf.
*
* Giving bytes s = s_1, s_2, ... s_k, the hash is defined to be
* H(s) = s_1*(a^(k-1)) + s_2*(a^(k-2)) + ... + s_k*(a^0)
*
* where the constant a is defined to be prime8bytes.
*
* The implementation adds an offset to each byte, so
* H(s) = (s_1 + HASH_CHAR_OFFSET)*(a^(k-1)) + ... */
static U64 ZSTD_ldm_getRollingHash(const BYTE* buf, U32 len)
{
U64 ret = 0;
U32 i;
for (i = 0; i < len; i++) {
ret *= prime8bytes;
ret += buf[i] + LDM_HASH_CHAR_OFFSET;
}
return ret;
}
/** ZSTD_ldm_ipow() :
* Return base^exp. */
static U64 ZSTD_ldm_ipow(U64 base, U64 exp)
{
U64 ret = 1;
while (exp) {
if (exp & 1) { ret *= base; }
exp >>= 1;
base *= base;
}
return ret;
}
U64 ZSTD_ldm_getHashPower(U32 minMatchLength) {
DEBUGLOG(4, "ZSTD_ldm_getHashPower: mml=%u", minMatchLength);
assert(minMatchLength >= ZSTD_LDM_MINMATCH_MIN);
return ZSTD_ldm_ipow(prime8bytes, minMatchLength - 1);
}
/** ZSTD_ldm_updateHash() :
* Updates hash by removing toRemove and adding toAdd. */
static U64 ZSTD_ldm_updateHash(U64 hash, BYTE toRemove, BYTE toAdd, U64 hashPower)
{
hash -= ((toRemove + LDM_HASH_CHAR_OFFSET) * hashPower);
hash *= prime8bytes;
hash += toAdd + LDM_HASH_CHAR_OFFSET;
return hash;
}
/** ZSTD_ldm_countBackwardsMatch() :
* Returns the number of bytes that match backwards before pIn and pMatch.
*
@ -238,6 +188,7 @@ static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms,
case ZSTD_btlazy2:
case ZSTD_btopt:
case ZSTD_btultra:
case ZSTD_btultra2:
break;
default:
assert(0); /* not possible : not a valid strategy id */
@ -261,9 +212,9 @@ static U64 ZSTD_ldm_fillLdmHashTable(ldmState_t* state,
const BYTE* cur = lastHashed + 1;
while (cur < iend) {
rollingHash = ZSTD_ldm_updateHash(rollingHash, cur[-1],
cur[ldmParams.minMatchLength-1],
state->hashPower);
rollingHash = ZSTD_rollingHash_rotate(rollingHash, cur[-1],
cur[ldmParams.minMatchLength-1],
state->hashPower);
ZSTD_ldm_makeEntryAndInsertByTag(state,
rollingHash, hBits,
(U32)(cur - base), ldmParams);
@ -297,8 +248,8 @@ static size_t ZSTD_ldm_generateSequences_internal(
U64 const hashPower = ldmState->hashPower;
U32 const hBits = params->hashLog - params->bucketSizeLog;
U32 const ldmBucketSize = 1U << params->bucketSizeLog;
U32 const hashEveryLog = params->hashEveryLog;
U32 const ldmTagMask = (1U << params->hashEveryLog) - 1;
U32 const hashRateLog = params->hashRateLog;
U32 const ldmTagMask = (1U << params->hashRateLog) - 1;
/* Prefix and extDict parameters */
U32 const dictLimit = ldmState->window.dictLimit;
U32 const lowestIndex = extDict ? ldmState->window.lowLimit : dictLimit;
@ -324,16 +275,16 @@ static size_t ZSTD_ldm_generateSequences_internal(
size_t forwardMatchLength = 0, backwardMatchLength = 0;
ldmEntry_t* bestEntry = NULL;
if (ip != istart) {
rollingHash = ZSTD_ldm_updateHash(rollingHash, lastHashed[0],
lastHashed[minMatchLength],
hashPower);
rollingHash = ZSTD_rollingHash_rotate(rollingHash, lastHashed[0],
lastHashed[minMatchLength],
hashPower);
} else {
rollingHash = ZSTD_ldm_getRollingHash(ip, minMatchLength);
rollingHash = ZSTD_rollingHash_compute(ip, minMatchLength);
}
lastHashed = ip;
/* Do not insert and do not look for a match */
if (ZSTD_ldm_getTag(rollingHash, hBits, hashEveryLog) != ldmTagMask) {
if (ZSTD_ldm_getTag(rollingHash, hBits, hashRateLog) != ldmTagMask) {
ip++;
continue;
}
@ -593,7 +544,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
void const* src, size_t srcSize)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
unsigned const minMatch = cParams->searchLength;
unsigned const minMatch = cParams->minMatch;
ZSTD_blockCompressor const blockCompressor =
ZSTD_selectBlockCompressor(cParams->strategy, ZSTD_matchState_dictMode(ms));
/* Input bounds */

8
sys/contrib/zstd/lib/compress/zstd_ldm.h
View File

@ -21,7 +21,7 @@ extern "C" {
* Long distance matching
***************************************/
#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_DEFAULTMAX
#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_LIMIT_DEFAULT
/**
* ZSTD_ldm_generateSequences():
@ -86,12 +86,8 @@ size_t ZSTD_ldm_getTableSize(ldmParams_t params);
*/
size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize);
/** ZSTD_ldm_getTableSize() :
* Return prime8bytes^(minMatchLength-1) */
U64 ZSTD_ldm_getHashPower(U32 minMatchLength);
/** ZSTD_ldm_adjustParameters() :
* If the params->hashEveryLog is not set, set it to its default value based on
* If the params->hashRateLog is not set, set it to its default value based on
* windowLog and params->hashLog.
*
* Ensures that params->bucketSizeLog is <= params->hashLog (setting it to

181
sys/contrib/zstd/lib/compress/zstd_opt.c
View File

@ -17,6 +17,8 @@
#define ZSTD_FREQ_DIV 4 /* log factor when using previous stats to init next stats */
#define ZSTD_MAX_PRICE (1<<30)
#define ZSTD_PREDEF_THRESHOLD 1024 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */
/*-*************************************
* Price functions for optimal parser
@ -52,11 +54,15 @@ MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat)
return weight;
}
/* debugging function, @return price in bytes */
#if (DEBUGLEVEL>=2)
/* debugging function,
* @return price in bytes as fractional value
* for debug messages only */
MEM_STATIC double ZSTD_fCost(U32 price)
{
return (double)price / (BITCOST_MULTIPLIER*8);
}
#endif
static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel)
{
@ -67,29 +73,44 @@ static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel)
}
static U32 ZSTD_downscaleStat(U32* table, U32 lastEltIndex, int malus)
/* ZSTD_downscaleStat() :
* reduce all elements in table by a factor 2^(ZSTD_FREQ_DIV+malus)
* return the resulting sum of elements */
static U32 ZSTD_downscaleStat(unsigned* table, U32 lastEltIndex, int malus)
{
U32 s, sum=0;
DEBUGLOG(5, "ZSTD_downscaleStat (nbElts=%u)", (unsigned)lastEltIndex+1);
assert(ZSTD_FREQ_DIV+malus > 0 && ZSTD_FREQ_DIV+malus < 31);
for (s=0; s<=lastEltIndex; s++) {
for (s=0; s<lastEltIndex+1; s++) {
table[s] = 1 + (table[s] >> (ZSTD_FREQ_DIV+malus));
sum += table[s];
}
return sum;
}
static void ZSTD_rescaleFreqs(optState_t* const optPtr,
const BYTE* const src, size_t const srcSize,
int optLevel)
/* ZSTD_rescaleFreqs() :
* if first block (detected by optPtr->litLengthSum == 0) : init statistics
* take hints from dictionary if there is one
* or init from zero, using src for literals stats, or flat 1 for match symbols
* otherwise downscale existing stats, to be used as seed for next block.
*/
static void
ZSTD_rescaleFreqs(optState_t* const optPtr,
const BYTE* const src, size_t const srcSize,
int const optLevel)
{
DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize);
optPtr->priceType = zop_dynamic;
if (optPtr->litLengthSum == 0) { /* first block : init */
if (srcSize <= 1024) /* heuristic */
if (srcSize <= ZSTD_PREDEF_THRESHOLD) { /* heuristic */
DEBUGLOG(5, "(srcSize <= ZSTD_PREDEF_THRESHOLD) => zop_predef");
optPtr->priceType = zop_predef;
}
assert(optPtr->symbolCosts != NULL);
if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) { /* huffman table presumed generated by dictionary */
if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) {
/* huffman table presumed generated by dictionary */
optPtr->priceType = zop_dynamic;
assert(optPtr->litFreq != NULL);
@ -208,7 +229,9 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP
/* dynamic statistics */
{ U32 const llCode = ZSTD_LLcode(litLength);
return (LL_bits[llCode] * BITCOST_MULTIPLIER) + (optPtr->litLengthSumBasePrice - WEIGHT(optPtr->litLengthFreq[llCode], optLevel));
return (LL_bits[llCode] * BITCOST_MULTIPLIER)
+ optPtr->litLengthSumBasePrice
- WEIGHT(optPtr->litLengthFreq[llCode], optLevel);
}
}
@ -253,7 +276,7 @@ static int ZSTD_literalsContribution(const BYTE* const literals, U32 const litLe
FORCE_INLINE_TEMPLATE U32
ZSTD_getMatchPrice(U32 const offset,
U32 const matchLength,
const optState_t* const optPtr,
const optState_t* const optPtr,
int const optLevel)
{
U32 price;
@ -385,7 +408,6 @@ static U32 ZSTD_insertBt1(
U32* largerPtr = smallerPtr + 1;
U32 dummy32; /* to be nullified at the end */
U32 const windowLow = ms->window.lowLimit;
U32 const matchLow = windowLow ? windowLow : 1;
U32 matchEndIdx = current+8+1;
size_t bestLength = 8;
U32 nbCompares = 1U << cParams->searchLog;
@ -401,7 +423,8 @@ static U32 ZSTD_insertBt1(
assert(ip <= iend-8); /* required for h calculation */
hashTable[h] = current; /* Update Hash Table */
while (nbCompares-- && (matchIndex >= matchLow)) {
assert(windowLow > 0);
while (nbCompares-- && (matchIndex >= windowLow)) {
U32* const nextPtr = bt + 2*(matchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
assert(matchIndex < current);
@ -479,7 +502,7 @@ void ZSTD_updateTree_internal(
const BYTE* const base = ms->window.base;
U32 const target = (U32)(ip - base);
U32 idx = ms->nextToUpdate;
DEBUGLOG(5, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)",
DEBUGLOG(6, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)",
idx, target, dictMode);
while(idx < target)
@ -488,15 +511,18 @@ void ZSTD_updateTree_internal(
}
void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) {
ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.searchLength, ZSTD_noDict);
ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict);
}
FORCE_INLINE_TEMPLATE
U32 ZSTD_insertBtAndGetAllMatches (
ZSTD_matchState_t* ms,
const BYTE* const ip, const BYTE* const iLimit, const ZSTD_dictMode_e dictMode,
U32 rep[ZSTD_REP_NUM], U32 const ll0,
ZSTD_match_t* matches, const U32 lengthToBeat, U32 const mls /* template */)
U32 rep[ZSTD_REP_NUM],
U32 const ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */
ZSTD_match_t* matches,
const U32 lengthToBeat,
U32 const mls /* template */)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
@ -542,6 +568,7 @@ U32 ZSTD_insertBtAndGetAllMatches (
DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", current);
/* check repCode */
assert(ll0 <= 1); /* necessarily 1 or 0 */
{ U32 const lastR = ZSTD_REP_NUM + ll0;
U32 repCode;
for (repCode = ll0; repCode < lastR; repCode++) {
@ -724,7 +751,7 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
ZSTD_match_t* matches, U32 const lengthToBeat)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32 const matchLengthSearch = cParams->searchLength;
U32 const matchLengthSearch = cParams->minMatch;
DEBUGLOG(8, "ZSTD_BtGetAllMatches");
if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */
ZSTD_updateTree_internal(ms, ip, iHighLimit, matchLengthSearch, dictMode);
@ -774,12 +801,30 @@ static U32 ZSTD_totalLen(ZSTD_optimal_t sol)
return sol.litlen + sol.mlen;
}
#if 0 /* debug */
static void
listStats(const U32* table, int lastEltID)
{
int const nbElts = lastEltID + 1;
int enb;
for (enb=0; enb < nbElts; enb++) {
(void)table;
//RAWLOG(2, "%3i:%3i, ", enb, table[enb]);
RAWLOG(2, "%4i,", table[enb]);
}
RAWLOG(2, " \n");
}
#endif
FORCE_INLINE_TEMPLATE size_t
ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
seqStore_t* seqStore,
U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize,
const int optLevel, const ZSTD_dictMode_e dictMode)
const void* src, size_t srcSize,
const int optLevel,
const ZSTD_dictMode_e dictMode)
{
optState_t* const optStatePtr = &ms->opt;
const BYTE* const istart = (const BYTE*)src;
@ -792,14 +837,15 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
U32 const minMatch = (cParams->searchLength == 3) ? 3 : 4;
U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4;
ZSTD_optimal_t* const opt = optStatePtr->priceTable;
ZSTD_match_t* const matches = optStatePtr->matchTable;
ZSTD_optimal_t lastSequence;
/* init */
DEBUGLOG(5, "ZSTD_compressBlock_opt_generic");
DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u",
(U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate);
assert(optLevel <= 2);
ms->nextToUpdate3 = ms->nextToUpdate;
ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel);
@ -999,7 +1045,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
U32 const offCode = opt[storePos].off;
U32 const advance = llen + mlen;
DEBUGLOG(6, "considering seq starting at %zi, llen=%u, mlen=%u",
anchor - istart, llen, mlen);
anchor - istart, (unsigned)llen, (unsigned)mlen);
if (mlen==0) { /* only literals => must be last "sequence", actually starting a new stream of sequences */
assert(storePos == storeEnd); /* must be last sequence */
@ -1047,11 +1093,11 @@ size_t ZSTD_compressBlock_btopt(
/* used in 2-pass strategy */
static U32 ZSTD_upscaleStat(U32* table, U32 lastEltIndex, int bonus)
static U32 ZSTD_upscaleStat(unsigned* table, U32 lastEltIndex, int bonus)
{
U32 s, sum=0;
assert(ZSTD_FREQ_DIV+bonus > 0);
for (s=0; s<=lastEltIndex; s++) {
assert(ZSTD_FREQ_DIV+bonus >= 0);
for (s=0; s<lastEltIndex+1; s++) {
table[s] <<= ZSTD_FREQ_DIV+bonus;
table[s]--;
sum += table[s];
@ -1063,9 +1109,43 @@ static U32 ZSTD_upscaleStat(U32* table, U32 lastEltIndex, int bonus)
MEM_STATIC void ZSTD_upscaleStats(optState_t* optPtr)
{
optPtr->litSum = ZSTD_upscaleStat(optPtr->litFreq, MaxLit, 0);
optPtr->litLengthSum = ZSTD_upscaleStat(optPtr->litLengthFreq, MaxLL, 1);
optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 1);
optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 1);
optPtr->litLengthSum = ZSTD_upscaleStat(optPtr->litLengthFreq, MaxLL, 0);
optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 0);
optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 0);
}
/* ZSTD_initStats_ultra():
* make a first compression pass, just to seed stats with more accurate starting values.
* only works on first block, with no dictionary and no ldm.
* this function cannot error, hence its constract must be respected.
*/
static void
ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
seqStore_t* seqStore,
U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */
memcpy(tmpRep, rep, sizeof(tmpRep));
DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize);
assert(ms->opt.litLengthSum == 0); /* first block */
assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */
assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */
assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */
ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/
/* invalidate first scan from history */
ZSTD_resetSeqStore(seqStore);
ms->window.base -= srcSize;
ms->window.dictLimit += (U32)srcSize;
ms->window.lowLimit = ms->window.dictLimit;
ms->nextToUpdate = ms->window.dictLimit;
ms->nextToUpdate3 = ms->window.dictLimit;
/* re-inforce weight of collected statistics */
ZSTD_upscaleStats(&ms->opt);
}
size_t ZSTD_compressBlock_btultra(
@ -1073,33 +1153,34 @@ size_t ZSTD_compressBlock_btultra(
const void* src, size_t srcSize)
{
DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize);
#if 0
/* 2-pass strategy (disabled)
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict);
}
size_t ZSTD_compressBlock_btultra2(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
U32 const current = (U32)((const BYTE*)src - ms->window.base);
DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize);
/* 2-pass strategy:
* this strategy makes a first pass over first block to collect statistics
* and seed next round's statistics with it.
* After 1st pass, function forgets everything, and starts a new block.
* Consequently, this can only work if no data has been previously loaded in tables,
* aka, no dictionary, no prefix, no ldm preprocessing.
* The compression ratio gain is generally small (~0.5% on first block),
* the cost is 2x cpu time on first block. */
assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
if ( (ms->opt.litLengthSum==0) /* first block */
&& (seqStore->sequences == seqStore->sequencesStart) /* no ldm */
&& (ms->window.dictLimit == ms->window.lowLimit) ) { /* no dictionary */
U32 tmpRep[ZSTD_REP_NUM];
DEBUGLOG(5, "ZSTD_compressBlock_btultra: first block: collecting statistics");
assert(ms->nextToUpdate >= ms->window.dictLimit
&& ms->nextToUpdate <= ms->window.dictLimit + 1);
memcpy(tmpRep, rep, sizeof(tmpRep));
ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/
ZSTD_resetSeqStore(seqStore);
/* invalidate first scan from history */
ms->window.base -= srcSize;
ms->window.dictLimit += (U32)srcSize;
ms->window.lowLimit = ms->window.dictLimit;
ms->nextToUpdate = ms->window.dictLimit;
ms->nextToUpdate3 = ms->window.dictLimit;
/* re-inforce weight of collected statistics */
ZSTD_upscaleStats(&ms->opt);
&& (seqStore->sequences == seqStore->sequencesStart) /* no ldm */
&& (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */
&& (current == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */
&& (srcSize > ZSTD_PREDEF_THRESHOLD)
) {
ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize);
}
#endif
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict);
}
@ -1130,3 +1211,7 @@ size_t ZSTD_compressBlock_btultra_extDict(
{
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_extDict);
}
/* note : no btultra2 variant for extDict nor dictMatchState,
* because btultra2 is not meant to work with dictionaries
* and is only specific for the first block (no prefix) */

8
sys/contrib/zstd/lib/compress/zstd_opt.h
View File

@ -26,6 +26,10 @@ size_t ZSTD_compressBlock_btopt(
size_t ZSTD_compressBlock_btultra(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btultra2(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btopt_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
@ -41,6 +45,10 @@ size_t ZSTD_compressBlock_btultra_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
/* note : no btultra2 variant for extDict nor dictMatchState,
* because btultra2 is not meant to work with dictionaries
* and is only specific for the first block (no prefix) */
#if defined (__cplusplus)
}
#endif

302
sys/contrib/zstd/lib/compress/zstdmt_compress.c
View File

@ -9,21 +9,19 @@
*/
/* ====== Tuning parameters ====== */
#define ZSTDMT_NBWORKERS_MAX 200
#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (2 GB)) /* note : limited by `jobSize` type, which is `unsigned` */
#define ZSTDMT_OVERLAPLOG_DEFAULT 6
/* ====== Compiler specifics ====== */
#if defined(_MSC_VER)
# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
#endif
/* ====== Constants ====== */
#define ZSTDMT_OVERLAPLOG_DEFAULT 0
/* ====== Dependencies ====== */
#include <string.h> /* memcpy, memset */
#include <limits.h> /* INT_MAX */
#include <limits.h> /* INT_MAX, UINT_MAX */
#include "pool.h" /* threadpool */
#include "threading.h" /* mutex */
#include "zstd_compress_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
@ -57,9 +55,9 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void)
static clock_t _ticksPerSecond = 0;
if (_ticksPerSecond <= 0) _ticksPerSecond = sysconf(_SC_CLK_TCK);
{ struct tms junk; clock_t newTicks = (clock_t) times(&junk);
return ((((unsigned long long)newTicks)*(1000000))/_ticksPerSecond); }
}
{ struct tms junk; clock_t newTicks = (clock_t) times(&junk);
return ((((unsigned long long)newTicks)*(1000000))/_ticksPerSecond);
} }
#define MUTEX_WAIT_TIME_DLEVEL 6
#define ZSTD_PTHREAD_MUTEX_LOCK(mutex) { \
@ -342,8 +340,8 @@ static ZSTDMT_seqPool* ZSTDMT_expandSeqPool(ZSTDMT_seqPool* pool, U32 nbWorkers)
typedef struct {
ZSTD_pthread_mutex_t poolMutex;
unsigned totalCCtx;
unsigned availCCtx;
int totalCCtx;
int availCCtx;
ZSTD_customMem cMem;
ZSTD_CCtx* cctx[1]; /* variable size */
} ZSTDMT_CCtxPool;
@ -351,16 +349,16 @@ typedef struct {
/* note : all CCtx borrowed from the pool should be released back to the pool _before_ freeing the pool */
static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool)
{
unsigned u;
for (u=0; u<pool->totalCCtx; u++)
ZSTD_freeCCtx(pool->cctx[u]); /* note : compatible with free on NULL */
int cid;
for (cid=0; cid<pool->totalCCtx; cid++)
ZSTD_freeCCtx(pool->cctx[cid]); /* note : compatible with free on NULL */
ZSTD_pthread_mutex_destroy(&pool->poolMutex);
ZSTD_free(pool, pool->cMem);
}
/* ZSTDMT_createCCtxPool() :
* implies nbWorkers >= 1 , checked by caller ZSTDMT_createCCtx() */
static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbWorkers,
static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(int nbWorkers,
ZSTD_customMem cMem)
{
ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_calloc(
@ -381,7 +379,7 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbWorkers,
}
static ZSTDMT_CCtxPool* ZSTDMT_expandCCtxPool(ZSTDMT_CCtxPool* srcPool,
unsigned nbWorkers)
int nbWorkers)
{
if (srcPool==NULL) return NULL;
if (nbWorkers <= srcPool->totalCCtx) return srcPool; /* good enough */
@ -469,9 +467,9 @@ static int ZSTDMT_serialState_reset(serialState_t* serialState, ZSTDMT_seqPool*
DEBUGLOG(4, "LDM window size = %u KB", (1U << params.cParams.windowLog) >> 10);
ZSTD_ldm_adjustParameters(&params.ldmParams, &params.cParams);
assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog);
assert(params.ldmParams.hashEveryLog < 32);
assert(params.ldmParams.hashRateLog < 32);
serialState->ldmState.hashPower =
ZSTD_ldm_getHashPower(params.ldmParams.minMatchLength);
ZSTD_rollingHash_primePower(params.ldmParams.minMatchLength);
} else {
memset(&params.ldmParams, 0, sizeof(params.ldmParams));
}
@ -674,7 +672,7 @@ static void ZSTDMT_compressionJob(void* jobDescription)
if (ZSTD_isError(initError)) JOB_ERROR(initError);
} else { /* srcStart points at reloaded section */
U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size;
{ size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstJob);
{ size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob);
if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError);
}
{ size_t const initError = ZSTD_compressBegin_advanced_internal(cctx,
@ -777,6 +775,14 @@ typedef struct {
static const roundBuff_t kNullRoundBuff = {NULL, 0, 0};
#define RSYNC_LENGTH 32
typedef struct {
U64 hash;
U64 hitMask;
U64 primePower;
} rsyncState_t;
struct ZSTDMT_CCtx_s {
POOL_ctx* factory;
ZSTDMT_jobDescription* jobs;
@ -790,6 +796,7 @@ struct ZSTDMT_CCtx_s {
inBuff_t inBuff;
roundBuff_t roundBuff;
serialState_t serial;
rsyncState_t rsync;
unsigned singleBlockingThread;
unsigned jobIDMask;
unsigned doneJobID;
@ -859,7 +866,7 @@ size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorker
{
if (nbWorkers > ZSTDMT_NBWORKERS_MAX) nbWorkers = ZSTDMT_NBWORKERS_MAX;
params->nbWorkers = nbWorkers;
params->overlapSizeLog = ZSTDMT_OVERLAPLOG_DEFAULT;
params->overlapLog = ZSTDMT_OVERLAPLOG_DEFAULT;
params->jobSize = 0;
return nbWorkers;
}
@ -969,52 +976,59 @@ size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx)
}
/* Internal only */
size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params,
ZSTDMT_parameter parameter, unsigned value) {
size_t
ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params,
ZSTDMT_parameter parameter,
int value)
{
DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter");
switch(parameter)
{
case ZSTDMT_p_jobSize :
DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %u", value);
if ( (value > 0) /* value==0 => automatic job size */
& (value < ZSTDMT_JOBSIZE_MIN) )
DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %i", value);
if ( value != 0 /* default */
&& value < ZSTDMT_JOBSIZE_MIN)
value = ZSTDMT_JOBSIZE_MIN;
if (value > ZSTDMT_JOBSIZE_MAX)
value = ZSTDMT_JOBSIZE_MAX;
assert(value >= 0);
if (value > ZSTDMT_JOBSIZE_MAX) value = ZSTDMT_JOBSIZE_MAX;
params->jobSize = value;
return value;
case ZSTDMT_p_overlapSectionLog :
if (value > 9) value = 9;
DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %u", value);
params->overlapSizeLog = (value >= 9) ? 9 : value;
case ZSTDMT_p_overlapLog :
DEBUGLOG(4, "ZSTDMT_p_overlapLog : %i", value);
if (value < ZSTD_OVERLAPLOG_MIN) value = ZSTD_OVERLAPLOG_MIN;
if (value > ZSTD_OVERLAPLOG_MAX) value = ZSTD_OVERLAPLOG_MAX;
params->overlapLog = value;
return value;
case ZSTDMT_p_rsyncable :
value = (value != 0);
params->rsyncable = value;
return value;
default :
return ERROR(parameter_unsupported);
}
}
size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned value)
size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int value)
{
DEBUGLOG(4, "ZSTDMT_setMTCtxParameter");
switch(parameter)
{
case ZSTDMT_p_jobSize :
return ZSTDMT_CCtxParam_setMTCtxParameter(&mtctx->params, parameter, value);
case ZSTDMT_p_overlapSectionLog :
return ZSTDMT_CCtxParam_setMTCtxParameter(&mtctx->params, parameter, value);
default :
return ERROR(parameter_unsupported);
}
return ZSTDMT_CCtxParam_setMTCtxParameter(&mtctx->params, parameter, value);
}
size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned* value)
size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int* value)
{
switch (parameter) {
case ZSTDMT_p_jobSize:
*value = mtctx->params.jobSize;
assert(mtctx->params.jobSize <= INT_MAX);
*value = (int)(mtctx->params.jobSize);
break;
case ZSTDMT_p_overlapSectionLog:
*value = mtctx->params.overlapSizeLog;
case ZSTDMT_p_overlapLog:
*value = mtctx->params.overlapLog;
break;
case ZSTDMT_p_rsyncable:
*value = mtctx->params.rsyncable;
break;
default:
return ERROR(parameter_unsupported);
@ -1140,22 +1154,66 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx)
/* ===== Multi-threaded compression ===== */
/* ------------------------------------------ */
static size_t ZSTDMT_computeTargetJobLog(ZSTD_CCtx_params const params)
static unsigned ZSTDMT_computeTargetJobLog(ZSTD_CCtx_params const params)
{
if (params.ldmParams.enableLdm)
/* In Long Range Mode, the windowLog is typically oversized.
* In which case, it's preferable to determine the jobSize
* based on chainLog instead. */
return MAX(21, params.cParams.chainLog + 4);
return MAX(20, params.cParams.windowLog + 2);
}
static size_t ZSTDMT_computeOverlapLog(ZSTD_CCtx_params const params)
static int ZSTDMT_overlapLog_default(ZSTD_strategy strat)
{
unsigned const overlapRLog = (params.overlapSizeLog>9) ? 0 : 9-params.overlapSizeLog;
if (params.ldmParams.enableLdm)
return (MIN(params.cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) - overlapRLog);
return overlapRLog >= 9 ? 0 : (params.cParams.windowLog - overlapRLog);
switch(strat)
{
case ZSTD_btultra2:
return 9;
case ZSTD_btultra:
case ZSTD_btopt:
return 8;
case ZSTD_btlazy2:
case ZSTD_lazy2:
return 7;
case ZSTD_lazy:
case ZSTD_greedy:
case ZSTD_dfast:
case ZSTD_fast:
default:;
}
return 6;
}
static unsigned ZSTDMT_computeNbJobs(ZSTD_CCtx_params params, size_t srcSize, unsigned nbWorkers) {
static int ZSTDMT_overlapLog(int ovlog, ZSTD_strategy strat)
{
assert(0 <= ovlog && ovlog <= 9);
if (ovlog == 0) return ZSTDMT_overlapLog_default(strat);
return ovlog;
}
static size_t ZSTDMT_computeOverlapSize(ZSTD_CCtx_params const params)
{
int const overlapRLog = 9 - ZSTDMT_overlapLog(params.overlapLog, params.cParams.strategy);
int ovLog = (overlapRLog >= 8) ? 0 : (params.cParams.windowLog - overlapRLog);
assert(0 <= overlapRLog && overlapRLog <= 8);
if (params.ldmParams.enableLdm) {
/* In Long Range Mode, the windowLog is typically oversized.
* In which case, it's preferable to determine the jobSize
* based on chainLog instead.
* Then, ovLog becomes a fraction of the jobSize, rather than windowSize */
ovLog = MIN(params.cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2)
- overlapRLog;
}
assert(0 <= ovLog && ovLog <= 30);
DEBUGLOG(4, "overlapLog : %i", params.overlapLog);
DEBUGLOG(4, "overlap size : %i", 1 << ovLog);
return (ovLog==0) ? 0 : (size_t)1 << ovLog;
}
static unsigned
ZSTDMT_computeNbJobs(ZSTD_CCtx_params params, size_t srcSize, unsigned nbWorkers)
{
assert(nbWorkers>0);
{ size_t const jobSizeTarget = (size_t)1 << ZSTDMT_computeTargetJobLog(params);
size_t const jobMaxSize = jobSizeTarget << 2;
@ -1178,7 +1236,7 @@ static size_t ZSTDMT_compress_advanced_internal(
ZSTD_CCtx_params params)
{
ZSTD_CCtx_params const jobParams = ZSTDMT_initJobCCtxParams(params);
size_t const overlapSize = (size_t)1 << ZSTDMT_computeOverlapLog(params);
size_t const overlapSize = ZSTDMT_computeOverlapSize(params);
unsigned const nbJobs = ZSTDMT_computeNbJobs(params, srcSize, params.nbWorkers);
size_t const proposedJobSize = (srcSize + (nbJobs-1)) / nbJobs;
size_t const avgJobSize = (((proposedJobSize-1) & 0x1FFFF) < 0x7FFF) ? proposedJobSize + 0xFFFF : proposedJobSize; /* avoid too small last block */
@ -1289,16 +1347,17 @@ static size_t ZSTDMT_compress_advanced_internal(
}
size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_CDict* cdict,
ZSTD_parameters params,
unsigned overlapLog)
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_CDict* cdict,
ZSTD_parameters params,
int overlapLog)
{
ZSTD_CCtx_params cctxParams = mtctx->params;
cctxParams.cParams = params.cParams;
cctxParams.fParams = params.fParams;
cctxParams.overlapSizeLog = overlapLog;
assert(ZSTD_OVERLAPLOG_MIN <= overlapLog && overlapLog <= ZSTD_OVERLAPLOG_MAX);
cctxParams.overlapLog = overlapLog;
return ZSTDMT_compress_advanced_internal(mtctx,
dst, dstCapacity,
src, srcSize,
@ -1311,8 +1370,8 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
const void* src, size_t srcSize,
int compressionLevel)
{
U32 const overlapLog = (compressionLevel >= ZSTD_maxCLevel()) ? 9 : ZSTDMT_OVERLAPLOG_DEFAULT;
ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, 0);
int const overlapLog = ZSTDMT_overlapLog_default(params.cParams.strategy);
params.fParams.contentSizeFlag = 1;
return ZSTDMT_compress_advanced(mtctx, dst, dstCapacity, src, srcSize, NULL, params, overlapLog);
}
@ -1339,8 +1398,8 @@ size_t ZSTDMT_initCStream_internal(
if (params.nbWorkers != mtctx->params.nbWorkers)
CHECK_F( ZSTDMT_resize(mtctx, params.nbWorkers) );
if (params.jobSize > 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN;
if (params.jobSize > ZSTDMT_JOBSIZE_MAX) params.jobSize = ZSTDMT_JOBSIZE_MAX;
if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN;
if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = ZSTDMT_JOBSIZE_MAX;
mtctx->singleBlockingThread = (pledgedSrcSize <= ZSTDMT_JOBSIZE_MIN); /* do not trigger multi-threading when srcSize is too small */
if (mtctx->singleBlockingThread) {
@ -1375,14 +1434,24 @@ size_t ZSTDMT_initCStream_internal(
mtctx->cdict = cdict;
}
mtctx->targetPrefixSize = (size_t)1 << ZSTDMT_computeOverlapLog(params);
DEBUGLOG(4, "overlapLog=%u => %u KB", params.overlapSizeLog, (U32)(mtctx->targetPrefixSize>>10));
mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(params);
DEBUGLOG(4, "overlapLog=%i => %u KB", params.overlapLog, (U32)(mtctx->targetPrefixSize>>10));
mtctx->targetSectionSize = params.jobSize;
if (mtctx->targetSectionSize == 0) {
mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(params);
}
if (params.rsyncable) {
/* Aim for the targetsectionSize as the average job size. */
U32 const jobSizeMB = (U32)(mtctx->targetSectionSize >> 20);
U32 const rsyncBits = ZSTD_highbit32(jobSizeMB) + 20;
assert(jobSizeMB >= 1);
DEBUGLOG(4, "rsyncLog = %u", rsyncBits);
mtctx->rsync.hash = 0;
mtctx->rsync.hitMask = (1ULL << rsyncBits) - 1;
mtctx->rsync.primePower = ZSTD_rollingHash_primePower(RSYNC_LENGTH);
}
if (mtctx->targetSectionSize < mtctx->targetPrefixSize) mtctx->targetSectionSize = mtctx->targetPrefixSize; /* job size must be >= overlap size */
DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(mtctx->targetSectionSize>>10), params.jobSize);
DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(mtctx->targetSectionSize>>10), (U32)params.jobSize);
DEBUGLOG(4, "inBuff Size : %u KB", (U32)(mtctx->targetSectionSize>>10));
ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(mtctx->targetSectionSize));
{
@ -1818,6 +1887,89 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx)
return 1;
}
typedef struct {
size_t toLoad; /* The number of bytes to load from the input. */
int flush; /* Boolean declaring if we must flush because we found a synchronization point. */
} syncPoint_t;
/**
* Searches through the input for a synchronization point. If one is found, we
* will instruct the caller to flush, and return the number of bytes to load.
* Otherwise, we will load as many bytes as possible and instruct the caller
* to continue as normal.
*/
static syncPoint_t
findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input)
{
BYTE const* const istart = (BYTE const*)input.src + input.pos;
U64 const primePower = mtctx->rsync.primePower;
U64 const hitMask = mtctx->rsync.hitMask;
syncPoint_t syncPoint;
U64 hash;
BYTE const* prev;
size_t pos;
syncPoint.toLoad = MIN(input.size - input.pos, mtctx->targetSectionSize - mtctx->inBuff.filled);
syncPoint.flush = 0;
if (!mtctx->params.rsyncable)
/* Rsync is disabled. */
return syncPoint;
if (mtctx->inBuff.filled + syncPoint.toLoad < RSYNC_LENGTH)
/* Not enough to compute the hash.
* We will miss any synchronization points in this RSYNC_LENGTH byte
* window. However, since it depends only in the internal buffers, if the
* state is already synchronized, we will remain synchronized.
* Additionally, the probability that we miss a synchronization point is
* low: RSYNC_LENGTH / targetSectionSize.
*/
return syncPoint;
/* Initialize the loop variables. */
if (mtctx->inBuff.filled >= RSYNC_LENGTH) {
/* We have enough bytes buffered to initialize the hash.
* Start scanning at the beginning of the input.
*/
pos = 0;
prev = (BYTE const*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled - RSYNC_LENGTH;
hash = ZSTD_rollingHash_compute(prev, RSYNC_LENGTH);
} else {
/* We don't have enough bytes buffered to initialize the hash, but
* we know we have at least RSYNC_LENGTH bytes total.
* Start scanning after the first RSYNC_LENGTH bytes less the bytes
* already buffered.
*/
pos = RSYNC_LENGTH - mtctx->inBuff.filled;
prev = (BYTE const*)mtctx->inBuff.buffer.start - pos;
hash = ZSTD_rollingHash_compute(mtctx->inBuff.buffer.start, mtctx->inBuff.filled);
hash = ZSTD_rollingHash_append(hash, istart, pos);
}
/* Starting with the hash of the previous RSYNC_LENGTH bytes, roll
* through the input. If we hit a synchronization point, then cut the
* job off, and tell the compressor to flush the job. Otherwise, load
* all the bytes and continue as normal.
* If we go too long without a synchronization point (targetSectionSize)
* then a block will be emitted anyways, but this is okay, since if we
* are already synchronized we will remain synchronized.
*/
for (; pos < syncPoint.toLoad; ++pos) {
BYTE const toRemove = pos < RSYNC_LENGTH ? prev[pos] : istart[pos - RSYNC_LENGTH];
/* if (pos >= RSYNC_LENGTH) assert(ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); */
hash = ZSTD_rollingHash_rotate(hash, toRemove, istart[pos], primePower);
if ((hash & hitMask) == hitMask) {
syncPoint.toLoad = pos + 1;
syncPoint.flush = 1;
break;
}
}
return syncPoint;
}
size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx)
{
size_t hintInSize = mtctx->targetSectionSize - mtctx->inBuff.filled;
if (hintInSize==0) hintInSize = mtctx->targetSectionSize;
return hintInSize;
}
/** ZSTDMT_compressStream_generic() :
* internal use only - exposed to be invoked from zstd_compress.c
@ -1844,7 +1996,8 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
}
/* single-pass shortcut (note : synchronous-mode) */
if ( (mtctx->nextJobID == 0) /* just started */
if ( (!mtctx->params.rsyncable) /* rsyncable mode is disabled */
&& (mtctx->nextJobID == 0) /* just started */
&& (mtctx->inBuff.filled == 0) /* nothing buffered */
&& (!mtctx->jobReady) /* no job already created */
&& (endOp == ZSTD_e_end) /* end order */
@ -1876,14 +2029,17 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
DEBUGLOG(5, "ZSTDMT_tryGetInputRange completed successfully : mtctx->inBuff.buffer.start = %p", mtctx->inBuff.buffer.start);
}
if (mtctx->inBuff.buffer.start != NULL) {
size_t const toLoad = MIN(input->size - input->pos, mtctx->targetSectionSize - mtctx->inBuff.filled);
syncPoint_t const syncPoint = findSynchronizationPoint(mtctx, *input);
if (syncPoint.flush && endOp == ZSTD_e_continue) {
endOp = ZSTD_e_flush;
}
assert(mtctx->inBuff.buffer.capacity >= mtctx->targetSectionSize);
DEBUGLOG(5, "ZSTDMT_compressStream_generic: adding %u bytes on top of %u to buffer of size %u",
(U32)toLoad, (U32)mtctx->inBuff.filled, (U32)mtctx->targetSectionSize);
memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, toLoad);
input->pos += toLoad;
mtctx->inBuff.filled += toLoad;
forwardInputProgress = toLoad>0;
(U32)syncPoint.toLoad, (U32)mtctx->inBuff.filled, (U32)mtctx->targetSectionSize);
memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, syncPoint.toLoad);
input->pos += syncPoint.toLoad;
mtctx->inBuff.filled += syncPoint.toLoad;
forwardInputProgress = syncPoint.toLoad>0;
}
if ((input->pos < input->size) && (endOp == ZSTD_e_end))
endOp = ZSTD_e_flush; /* can't end now : not all input consumed */

28
sys/contrib/zstd/lib/compress/zstdmt_compress.h
View File

@ -28,6 +28,16 @@
#include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
/* === Constants === */
#ifndef ZSTDMT_NBWORKERS_MAX
# define ZSTDMT_NBWORKERS_MAX 200
#endif
#ifndef ZSTDMT_JOBSIZE_MIN
# define ZSTDMT_JOBSIZE_MIN (1 MB)
#endif
#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB))
/* === Memory management === */
typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers);
@ -52,6 +62,7 @@ ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel);
ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it will change in the future to mean "empty" */
ZSTDLIB_API size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx);
ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
ZSTDLIB_API size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */
@ -60,16 +71,12 @@ ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);
/* === Advanced functions and parameters === */
#ifndef ZSTDMT_JOBSIZE_MIN
# define ZSTDMT_JOBSIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */
#endif
ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_CDict* cdict,
ZSTD_parameters params,
unsigned overlapLog);
int overlapLog);
ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
const void* dict, size_t dictSize, /* dict can be released after init, a local copy is preserved within zcs */
@ -84,8 +91,9 @@ ZSTDLIB_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
/* ZSTDMT_parameter :
* List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
typedef enum {
ZSTDMT_p_jobSize, /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */
ZSTDMT_p_overlapSectionLog /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */
ZSTDMT_p_jobSize, /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */
ZSTDMT_p_overlapLog, /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */
ZSTDMT_p_rsyncable /* Enables rsyncable mode. */
} ZSTDMT_parameter;
/* ZSTDMT_setMTCtxParameter() :
@ -93,12 +101,12 @@ typedef enum {
* The function must be called typically after ZSTD_createCCtx() but __before ZSTDMT_init*() !__
* Parameters not explicitly reset by ZSTDMT_init*() remain the same in consecutive compression sessions.
* @return : 0, or an error code (which can be tested using ZSTD_isError()) */
ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned value);
ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int value);
/* ZSTDMT_getMTCtxParameter() :
* Query the ZSTDMT_CCtx for a parameter value.
* @return : 0, or an error code (which can be tested using ZSTD_isError()) */
ZSTDLIB_API size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned* value);
ZSTDLIB_API size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int* value);
/*! ZSTDMT_compressStream_generic() :
@ -129,7 +137,7 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx);
/*! ZSTDMT_CCtxParam_setMTCtxParameter()
* like ZSTDMT_setMTCtxParameter(), but into a ZSTD_CCtx_Params */
size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, unsigned value);
size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, int value);
/*! ZSTDMT_CCtxParam_setNbWorkers()
* Set nbWorkers, and clamp it.

220
sys/contrib/zstd/lib/decompress/huf_decompress.c
View File

@ -43,6 +43,19 @@
#include "huf.h"
#include "error_private.h"
/* **************************************************************
* Macros
****************************************************************/
/* These two optional macros force the use one way or another of the two
* Huffman decompression implementations. You can't force in both directions
* at the same time.
*/
#if defined(HUF_FORCE_DECOMPRESS_X1) && \
defined(HUF_FORCE_DECOMPRESS_X2)
#error "Cannot force the use of the X1 and X2 decoders at the same time!"
#endif
/* **************************************************************
* Error Management
@ -58,6 +71,51 @@
#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask))
/* **************************************************************
* BMI2 Variant Wrappers
****************************************************************/
#if DYNAMIC_BMI2
#define HUF_DGEN(fn) \
\
static size_t fn##_default( \
void* dst, size_t dstSize, \
const void* cSrc, size_t cSrcSize, \
const HUF_DTable* DTable) \
{ \
return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
} \
\
static TARGET_ATTRIBUTE("bmi2") size_t fn##_bmi2( \
void* dst, size_t dstSize, \
const void* cSrc, size_t cSrcSize, \
const HUF_DTable* DTable) \
{ \
return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
} \
\
static size_t fn(void* dst, size_t dstSize, void const* cSrc, \
size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \
{ \
if (bmi2) { \
return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \
} \
return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \
}
#else
#define HUF_DGEN(fn) \
static size_t fn(void* dst, size_t dstSize, void const* cSrc, \
size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \
{ \
(void)bmi2; \
return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
}
#endif
/*-***************************/
/* generic DTableDesc */
/*-***************************/
@ -71,6 +129,8 @@ static DTableDesc HUF_getDTableDesc(const HUF_DTable* table)
}
#ifndef HUF_FORCE_DECOMPRESS_X2
/*-***************************/
/* single-symbol decoding */
/*-***************************/
@ -307,46 +367,6 @@ typedef size_t (*HUF_decompress_usingDTable_t)(void *dst, size_t dstSize,
const void *cSrc,
size_t cSrcSize,
const HUF_DTable *DTable);
#if DYNAMIC_BMI2
#define HUF_DGEN(fn) \
\
static size_t fn##_default( \
void* dst, size_t dstSize, \
const void* cSrc, size_t cSrcSize, \
const HUF_DTable* DTable) \
{ \
return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
} \
\
static TARGET_ATTRIBUTE("bmi2") size_t fn##_bmi2( \
void* dst, size_t dstSize, \
const void* cSrc, size_t cSrcSize, \
const HUF_DTable* DTable) \
{ \
return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
} \
\
static size_t fn(void* dst, size_t dstSize, void const* cSrc, \
size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \
{ \
if (bmi2) { \
return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \
} \
return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \
}
#else
#define HUF_DGEN(fn) \
static size_t fn(void* dst, size_t dstSize, void const* cSrc, \
size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \
{ \
(void)bmi2; \
return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
}
#endif
HUF_DGEN(HUF_decompress1X1_usingDTable_internal)
HUF_DGEN(HUF_decompress4X1_usingDTable_internal)
@ -437,6 +457,10 @@ size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cS
return HUF_decompress4X1_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
}
#endif /* HUF_FORCE_DECOMPRESS_X2 */
#ifndef HUF_FORCE_DECOMPRESS_X1
/* *************************/
/* double-symbols decoding */
@ -911,6 +935,8 @@ size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cS
return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
}
#endif /* HUF_FORCE_DECOMPRESS_X1 */
/* ***********************************/
/* Universal decompression selectors */
@ -921,8 +947,18 @@ size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize,
const HUF_DTable* DTable)
{
DTableDesc const dtd = HUF_getDTableDesc(DTable);
#if defined(HUF_FORCE_DECOMPRESS_X1)
(void)dtd;
assert(dtd.tableType == 0);
return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
#elif defined(HUF_FORCE_DECOMPRESS_X2)
(void)dtd;
assert(dtd.tableType == 1);
return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
#else
return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) :
HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
#endif
}
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize,
@ -930,11 +966,22 @@ size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize,
const HUF_DTable* DTable)
{
DTableDesc const dtd = HUF_getDTableDesc(DTable);
#if defined(HUF_FORCE_DECOMPRESS_X1)
(void)dtd;
assert(dtd.tableType == 0);
return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
#elif defined(HUF_FORCE_DECOMPRESS_X2)
(void)dtd;
assert(dtd.tableType == 1);
return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
#else
return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) :
HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
#endif
}
#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2)
typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t;
static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] =
{
@ -956,6 +1003,7 @@ static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, qu
{{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */
{{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */
};
#endif
/** HUF_selectDecoder() :
* Tells which decoder is likely to decode faster,
@ -966,6 +1014,15 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
{
assert(dstSize > 0);
assert(dstSize <= 128*1024);
#if defined(HUF_FORCE_DECOMPRESS_X1)
(void)dstSize;
(void)cSrcSize;
return 0;
#elif defined(HUF_FORCE_DECOMPRESS_X2)
(void)dstSize;
(void)cSrcSize;
return 1;
#else
/* decoder timing evaluation */
{ U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */
U32 const D256 = (U32)(dstSize >> 8);
@ -973,14 +1030,18 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, to reduce cache eviction */
return DTime1 < DTime0;
} }
}
#endif
}
typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2)
static const decompressionAlgo decompress[2] = { HUF_decompress4X1, HUF_decompress4X2 };
#endif
/* validation checks */
if (dstSize == 0) return ERROR(dstSize_tooSmall);
@ -989,7 +1050,17 @@ size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcS
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
#if defined(HUF_FORCE_DECOMPRESS_X1)
(void)algoNb;
assert(algoNb == 0);
return HUF_decompress4X1(dst, dstSize, cSrc, cSrcSize);
#elif defined(HUF_FORCE_DECOMPRESS_X2)
(void)algoNb;
assert(algoNb == 1);
return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize);
#else
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
#endif
}
}
@ -1002,8 +1073,18 @@ size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
#if defined(HUF_FORCE_DECOMPRESS_X1)
(void)algoNb;
assert(algoNb == 0);
return HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize);
#elif defined(HUF_FORCE_DECOMPRESS_X2)
(void)algoNb;
assert(algoNb == 1);
return HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize);
#else
return algoNb ? HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
#endif
}
}
@ -1025,8 +1106,19 @@ size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst,
if (cSrcSize == 0) return ERROR(corruption_detected);
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize):
#if defined(HUF_FORCE_DECOMPRESS_X1)
(void)algoNb;
assert(algoNb == 0);
return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
#elif defined(HUF_FORCE_DECOMPRESS_X2)
(void)algoNb;
assert(algoNb == 1);
return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
#else
return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
cSrcSize, workSpace, wkspSize):
HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
#endif
}
}
@ -1041,10 +1133,22 @@ size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
#if defined(HUF_FORCE_DECOMPRESS_X1)
(void)algoNb;
assert(algoNb == 0);
return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
cSrcSize, workSpace, wkspSize);
#elif defined(HUF_FORCE_DECOMPRESS_X2)
(void)algoNb;
assert(algoNb == 1);
return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
cSrcSize, workSpace, wkspSize);
#else
return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
cSrcSize, workSpace, wkspSize):
HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
cSrcSize, workSpace, wkspSize);
#endif
}
}
@ -1060,10 +1164,21 @@ size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize,
size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2)
{
DTableDesc const dtd = HUF_getDTableDesc(DTable);
#if defined(HUF_FORCE_DECOMPRESS_X1)
(void)dtd;
assert(dtd.tableType == 0);
return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
#elif defined(HUF_FORCE_DECOMPRESS_X2)
(void)dtd;
assert(dtd.tableType == 1);
return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
#else
return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) :
HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
#endif
}
#ifndef HUF_FORCE_DECOMPRESS_X2
size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2)
{
const BYTE* ip = (const BYTE*) cSrc;
@ -1075,12 +1190,23 @@ size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstS
return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2);
}
#endif
size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2)
{
DTableDesc const dtd = HUF_getDTableDesc(DTable);
#if defined(HUF_FORCE_DECOMPRESS_X1)
(void)dtd;
assert(dtd.tableType == 0);
return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
#elif defined(HUF_FORCE_DECOMPRESS_X2)
(void)dtd;
assert(dtd.tableType == 1);
return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
#else
return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) :
HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
#endif
}
size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2)
@ -1090,7 +1216,17 @@ size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t ds
if (cSrcSize == 0) return ERROR(corruption_detected);
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
#if defined(HUF_FORCE_DECOMPRESS_X1)
(void)algoNb;
assert(algoNb == 0);
return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
#elif defined(HUF_FORCE_DECOMPRESS_X2)
(void)algoNb;
assert(algoNb == 1);
return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
#else
return algoNb ? HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2) :
HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
#endif
}
}

240
sys/contrib/zstd/lib/decompress/zstd_ddict.c Normal file
View File

@ -0,0 +1,240 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/* zstd_ddict.c :
* concentrates all logic that needs to know the internals of ZSTD_DDict object */
/*-*******************************************************
* Dependencies
*********************************************************/
#include <string.h> /* memcpy, memmove, memset */
#include "cpu.h" /* bmi2 */
#include "mem.h" /* low level memory routines */
#define FSE_STATIC_LINKING_ONLY
#include "fse.h"
#define HUF_STATIC_LINKING_ONLY
#include "huf.h"
#include "zstd_decompress_internal.h"
#include "zstd_ddict.h"
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
# include "zstd_legacy.h"
#endif
/*-*******************************************************
* Types
*********************************************************/
struct ZSTD_DDict_s {
void* dictBuffer;
const void* dictContent;
size_t dictSize;
ZSTD_entropyDTables_t entropy;
U32 dictID;
U32 entropyPresent;
ZSTD_customMem cMem;
}; /* typedef'd to ZSTD_DDict within "zstd.h" */
const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict)
{
assert(ddict != NULL);
return ddict->dictContent;
}
size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict)
{
assert(ddict != NULL);
return ddict->dictSize;
}
void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
{
DEBUGLOG(4, "ZSTD_copyDDictParameters");
assert(dctx != NULL);
assert(ddict != NULL);
dctx->dictID = ddict->dictID;
dctx->prefixStart = ddict->dictContent;
dctx->virtualStart = ddict->dictContent;
dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize;
dctx->previousDstEnd = dctx->dictEnd;
if (ddict->entropyPresent) {
dctx->litEntropy = 1;
dctx->fseEntropy = 1;
dctx->LLTptr = ddict->entropy.LLTable;
dctx->MLTptr = ddict->entropy.MLTable;
dctx->OFTptr = ddict->entropy.OFTable;
dctx->HUFptr = ddict->entropy.hufTable;
dctx->entropy.rep[0] = ddict->entropy.rep[0];
dctx->entropy.rep[1] = ddict->entropy.rep[1];
dctx->entropy.rep[2] = ddict->entropy.rep[2];
} else {
dctx->litEntropy = 0;
dctx->fseEntropy = 0;
}
}
static size_t
ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict,
ZSTD_dictContentType_e dictContentType)
{
ddict->dictID = 0;
ddict->entropyPresent = 0;
if (dictContentType == ZSTD_dct_rawContent) return 0;
if (ddict->dictSize < 8) {
if (dictContentType == ZSTD_dct_fullDict)
return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
return 0; /* pure content mode */
}
{ U32 const magic = MEM_readLE32(ddict->dictContent);
if (magic != ZSTD_MAGIC_DICTIONARY) {
if (dictContentType == ZSTD_dct_fullDict)
return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
return 0; /* pure content mode */
}
}
ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE);
/* load entropy tables */
CHECK_E( ZSTD_loadDEntropy(&ddict->entropy,
ddict->dictContent, ddict->dictSize),
dictionary_corrupted );
ddict->entropyPresent = 1;
return 0;
}
static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType)
{
if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
ddict->dictBuffer = NULL;
ddict->dictContent = dict;
if (!dict) dictSize = 0;
} else {
void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem);
ddict->dictBuffer = internalBuffer;
ddict->dictContent = internalBuffer;
if (!internalBuffer) return ERROR(memory_allocation);
memcpy(internalBuffer, dict, dictSize);
}
ddict->dictSize = dictSize;
ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
/* parse dictionary content */
CHECK_F( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) );
return 0;
}
ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType,
ZSTD_customMem customMem)
{
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
{ ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
if (ddict == NULL) return NULL;
ddict->cMem = customMem;
{ size_t const initResult = ZSTD_initDDict_internal(ddict,
dict, dictSize,
dictLoadMethod, dictContentType);
if (ZSTD_isError(initResult)) {
ZSTD_freeDDict(ddict);
return NULL;
} }
return ddict;
}
}
/*! ZSTD_createDDict() :
* Create a digested dictionary, to start decompression without startup delay.
* `dict` content is copied inside DDict.
* Consequently, `dict` can be released after `ZSTD_DDict` creation */
ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
{
ZSTD_customMem const allocator = { NULL, NULL, NULL };
return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator);
}
/*! ZSTD_createDDict_byReference() :
* Create a digested dictionary, to start decompression without startup delay.
* Dictionary content is simply referenced, it will be accessed during decompression.
* Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */
ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
{
ZSTD_customMem const allocator = { NULL, NULL, NULL };
return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator);
}
const ZSTD_DDict* ZSTD_initStaticDDict(
void* sBuffer, size_t sBufferSize,
const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType)
{
size_t const neededSpace = sizeof(ZSTD_DDict)
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer;
assert(sBuffer != NULL);
assert(dict != NULL);
if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */
if (sBufferSize < neededSpace) return NULL;
if (dictLoadMethod == ZSTD_dlm_byCopy) {
memcpy(ddict+1, dict, dictSize); /* local copy */
dict = ddict+1;
}
if (ZSTD_isError( ZSTD_initDDict_internal(ddict,
dict, dictSize,
ZSTD_dlm_byRef, dictContentType) ))
return NULL;
return ddict;
}
size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = ddict->cMem;
ZSTD_free(ddict->dictBuffer, cMem);
ZSTD_free(ddict, cMem);
return 0;
}
}
/*! ZSTD_estimateDDictSize() :
* Estimate amount of memory that will be needed to create a dictionary for decompression.
* Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */
size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod)
{
return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
}
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0; /* support sizeof on NULL */
return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ;
}
/*! ZSTD_getDictID_fromDDict() :
* Provides the dictID of the dictionary loaded into `ddict`.
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0;
return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
}

44
sys/contrib/zstd/lib/decompress/zstd_ddict.h Normal file
View File

@ -0,0 +1,44 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef ZSTD_DDICT_H
#define ZSTD_DDICT_H
/*-*******************************************************
* Dependencies
*********************************************************/
#include <stddef.h> /* size_t */
#include "zstd.h" /* ZSTD_DDict, and several public functions */
/*-*******************************************************
* Interface
*********************************************************/
/* note: several prototypes are already published in `zstd.h` :
* ZSTD_createDDict()
* ZSTD_createDDict_byReference()
* ZSTD_createDDict_advanced()
* ZSTD_freeDDict()
* ZSTD_initStaticDDict()
* ZSTD_sizeof_DDict()
* ZSTD_estimateDDictSize()
* ZSTD_getDictID_fromDict()
*/
const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict);
size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict);
void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
#endif /* ZSTD_DDICT_H */

1924
sys/contrib/zstd/lib/decompress/zstd_decompress.c
View File

File diff suppressed because it is too large Load Diff

1307
sys/contrib/zstd/lib/decompress/zstd_decompress_block.c Normal file
View File

File diff suppressed because it is too large Load Diff

59
sys/contrib/zstd/lib/decompress/zstd_decompress_block.h Normal file
View File

@ -0,0 +1,59 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef ZSTD_DEC_BLOCK_H
#define ZSTD_DEC_BLOCK_H
/*-*******************************************************
* Dependencies
*********************************************************/
#include <stddef.h> /* size_t */
#include "zstd.h" /* DCtx, and some public functions */
#include "zstd_internal.h" /* blockProperties_t, and some public functions */
#include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */
/* === Prototypes === */
/* note: prototypes already published within `zstd.h` :
* ZSTD_decompressBlock()
*/
/* note: prototypes already published within `zstd_internal.h` :
* ZSTD_getcBlockSize()
* ZSTD_decodeSeqHeaders()
*/
/* ZSTD_decompressBlock_internal() :
* decompress block, starting at `src`,
* into destination buffer `dst`.
* @return : decompressed block size,
* or an error code (which can be tested using ZSTD_isError())
*/
size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize, const int frame);
/* ZSTD_buildFSETable() :
* generate FSE decoding table for one symbol (ll, ml or off)
* this function must be called with valid parameters only
* (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.)
* in which case it cannot fail.
* Internal use only.
*/
void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
const short* normalizedCounter, unsigned maxSymbolValue,
const U32* baseValue, const U32* nbAdditionalBits,
unsigned tableLog);
#endif /* ZSTD_DEC_BLOCK_H */

168
sys/contrib/zstd/lib/decompress/zstd_decompress_internal.h Normal file
View File

@ -0,0 +1,168 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/* zstd_decompress_internal:
* objects and definitions shared within lib/decompress modules */
#ifndef ZSTD_DECOMPRESS_INTERNAL_H
#define ZSTD_DECOMPRESS_INTERNAL_H
/*-*******************************************************
* Dependencies
*********************************************************/
#include "mem.h" /* BYTE, U16, U32 */
#include "zstd_internal.h" /* ZSTD_seqSymbol */
/*-*******************************************************
* Constants
*********************************************************/
static const U32 LL_base[MaxLL+1] = {
0, 1, 2, 3, 4, 5, 6, 7,
8, 9, 10, 11, 12, 13, 14, 15,
16, 18, 20, 22, 24, 28, 32, 40,
48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000,
0x2000, 0x4000, 0x8000, 0x10000 };
static const U32 OF_base[MaxOff+1] = {
0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D,
0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD,
0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD };
static const U32 OF_bits[MaxOff+1] = {
0, 1, 2, 3, 4, 5, 6, 7,
8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18, 19, 20, 21, 22, 23,
24, 25, 26, 27, 28, 29, 30, 31 };
static const U32 ML_base[MaxML+1] = {
3, 4, 5, 6, 7, 8, 9, 10,
11, 12, 13, 14, 15, 16, 17, 18,
19, 20, 21, 22, 23, 24, 25, 26,
27, 28, 29, 30, 31, 32, 33, 34,
35, 37, 39, 41, 43, 47, 51, 59,
67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803,
0x1003, 0x2003, 0x4003, 0x8003, 0x10003 };
/*-*******************************************************
* Decompression types
*********************************************************/
typedef struct {
U32 fastMode;
U32 tableLog;
} ZSTD_seqSymbol_header;
typedef struct {
U16 nextState;
BYTE nbAdditionalBits;
BYTE nbBits;
U32 baseValue;
} ZSTD_seqSymbol;
#define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log)))
typedef struct {
ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */
ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */
ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */
HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
U32 rep[ZSTD_REP_NUM];
} ZSTD_entropyDTables_t;
typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock,
ZSTDds_decompressLastBlock, ZSTDds_checkChecksum,
ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
typedef enum { zdss_init=0, zdss_loadHeader,
zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
struct ZSTD_DCtx_s
{
const ZSTD_seqSymbol* LLTptr;
const ZSTD_seqSymbol* MLTptr;
const ZSTD_seqSymbol* OFTptr;
const HUF_DTable* HUFptr;
ZSTD_entropyDTables_t entropy;
U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */
const void* previousDstEnd; /* detect continuity */
const void* prefixStart; /* start of current segment */
const void* virtualStart; /* virtual start of previous segment if it was just before current one */
const void* dictEnd; /* end of previous segment */
size_t expected;
ZSTD_frameHeader fParams;
U64 decodedSize;
blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */
ZSTD_dStage stage;
U32 litEntropy;
U32 fseEntropy;
XXH64_state_t xxhState;
size_t headerSize;
ZSTD_format_e format;
const BYTE* litPtr;
ZSTD_customMem customMem;
size_t litSize;
size_t rleSize;
size_t staticSize;
int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
/* dictionary */
ZSTD_DDict* ddictLocal;
const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */
U32 dictID;
int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */
/* streaming */
ZSTD_dStreamStage streamStage;
char* inBuff;
size_t inBuffSize;
size_t inPos;
size_t maxWindowSize;
char* outBuff;
size_t outBuffSize;
size_t outStart;
size_t outEnd;
size_t lhSize;
void* legacyContext;
U32 previousLegacyVersion;
U32 legacyVersion;
U32 hostageByte;
int noForwardProgress;
/* workspace */
BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
/*-*******************************************************
* Shared internal functions
*********************************************************/
/*! ZSTD_loadDEntropy() :
* dict : must point at beginning of a valid zstd dictionary.
* @return : size of entropy tables read */
size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
const void* const dict, size_t const dictSize);
/*! ZSTD_checkContinuity() :
* check if next `dst` follows previous position, where decompression ended.
* If yes, do nothing (continue on current segment).
* If not, classify previous segment as "external dictionary", and start a new segment.
* This function cannot fail. */
void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst);
#endif /* ZSTD_DECOMPRESS_INTERNAL_H */

24
sys/contrib/zstd/lib/dictBuilder/cover.c
View File

@ -39,7 +39,7 @@
/*-*************************************
* Constants
***************************************/
#define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB))
#define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))
#define DEFAULT_SPLITPOINT 1.0
/*-*************************************
@ -543,7 +543,7 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
if (totalSamplesSize < MAX(d, sizeof(U64)) ||
totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) {
DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n",
(U32)(totalSamplesSize>>20), (COVER_MAX_SAMPLES_SIZE >> 20));
(unsigned)(totalSamplesSize>>20), (COVER_MAX_SAMPLES_SIZE >> 20));
return 0;
}
/* Check if there are at least 5 training samples */
@ -559,9 +559,9 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
/* Zero the context */
memset(ctx, 0, sizeof(*ctx));
DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples,
(U32)trainingSamplesSize);
(unsigned)trainingSamplesSize);
DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples,
(U32)testSamplesSize);
(unsigned)testSamplesSize);
ctx->samples = samples;
ctx->samplesSizes = samplesSizes;
ctx->nbSamples = nbSamples;
@ -639,11 +639,11 @@ static size_t COVER_buildDictionary(const COVER_ctx_t *ctx, U32 *freqs,
/* Divide the data up into epochs of equal size.
* We will select at least one segment from each epoch.
*/
const U32 epochs = MAX(1, (U32)(dictBufferCapacity / parameters.k / 4));
const U32 epochSize = (U32)(ctx->suffixSize / epochs);
const unsigned epochs = MAX(1, (U32)(dictBufferCapacity / parameters.k / 4));
const unsigned epochSize = (U32)(ctx->suffixSize / epochs);
size_t epoch;
DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", epochs,
epochSize);
DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n",
epochs, epochSize);
/* Loop through the epochs until there are no more segments or the dictionary
* is full.
*/
@ -670,7 +670,7 @@ static size_t COVER_buildDictionary(const COVER_ctx_t *ctx, U32 *freqs,
memcpy(dict + tail, ctx->samples + segment.begin, segmentSize);
DISPLAYUPDATE(
2, "\r%u%% ",
(U32)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
(unsigned)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
}
DISPLAYLEVEL(2, "\r%79s\r", "");
return tail;
@ -722,7 +722,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
samplesBuffer, samplesSizes, nbSamples, parameters.zParams);
if (!ZSTD_isError(dictionarySize)) {
DISPLAYLEVEL(2, "Constructed dictionary of size %u\n",
(U32)dictionarySize);
(unsigned)dictionarySize);
}
COVER_ctx_destroy(&ctx);
COVER_map_destroy(&activeDmers);
@ -868,6 +868,8 @@ void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
if (!best->dict) {
best->compressedSize = ERROR(GENERIC);
best->dictSize = 0;
ZSTD_pthread_cond_signal(&best->cond);
ZSTD_pthread_mutex_unlock(&best->mutex);
return;
}
}
@ -1054,7 +1056,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
}
/* Print status */
LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ",
(U32)((iteration * 100) / kIterations));
(unsigned)((iteration * 100) / kIterations));
++iteration;
}
COVER_best_wait(&best);

30
sys/contrib/zstd/lib/dictBuilder/fastcover.c
View File

@ -20,7 +20,7 @@
/*-*************************************
* Constants
***************************************/
#define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB))
#define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))
#define FASTCOVER_MAX_F 31
#define FASTCOVER_MAX_ACCEL 10
#define DEFAULT_SPLITPOINT 0.75
@ -159,15 +159,15 @@ static COVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
*/
while (activeSegment.end < end) {
/* Get hash value of current dmer */
const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, f, d);
const size_t idx = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, f, d);
/* Add frequency of this index to score if this is the first occurence of index in active segment */
if (segmentFreqs[index] == 0) {
activeSegment.score += freqs[index];
if (segmentFreqs[idx] == 0) {
activeSegment.score += freqs[idx];
}
/* Increment end of segment and segmentFreqs*/
activeSegment.end += 1;
segmentFreqs[index] += 1;
segmentFreqs[idx] += 1;
/* If the window is now too large, drop the first position */
if (activeSegment.end - activeSegment.begin == dmersInK + 1) {
/* Get hash value of the dmer to be eliminated from active segment */
@ -309,7 +309,7 @@ FASTCOVER_ctx_init(FASTCOVER_ctx_t* ctx,
if (totalSamplesSize < MAX(d, sizeof(U64)) ||
totalSamplesSize >= (size_t)FASTCOVER_MAX_SAMPLES_SIZE) {
DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n",
(U32)(totalSamplesSize >> 20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20));
(unsigned)(totalSamplesSize >> 20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20));
return 0;
}
@ -328,9 +328,9 @@ FASTCOVER_ctx_init(FASTCOVER_ctx_t* ctx,
/* Zero the context */
memset(ctx, 0, sizeof(*ctx));
DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples,
(U32)trainingSamplesSize);
(unsigned)trainingSamplesSize);
DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples,
(U32)testSamplesSize);
(unsigned)testSamplesSize);
ctx->samples = samples;
ctx->samplesSizes = samplesSizes;
@ -389,11 +389,11 @@ FASTCOVER_buildDictionary(const FASTCOVER_ctx_t* ctx,
/* Divide the data up into epochs of equal size.
* We will select at least one segment from each epoch.
*/
const U32 epochs = MAX(1, (U32)(dictBufferCapacity / parameters.k));
const U32 epochSize = (U32)(ctx->nbDmers / epochs);
const unsigned epochs = MAX(1, (U32)(dictBufferCapacity / parameters.k));
const unsigned epochSize = (U32)(ctx->nbDmers / epochs);
size_t epoch;
DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", epochs,
epochSize);
DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n",
epochs, epochSize);
/* Loop through the epochs until there are no more segments or the dictionary
* is full.
*/
@ -423,7 +423,7 @@ FASTCOVER_buildDictionary(const FASTCOVER_ctx_t* ctx,
memcpy(dict + tail, ctx->samples + segment.begin, segmentSize);
DISPLAYUPDATE(
2, "\r%u%% ",
(U32)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
(unsigned)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
}
DISPLAYLEVEL(2, "\r%79s\r", "");
return tail;
@ -577,7 +577,7 @@ ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity,
samplesBuffer, samplesSizes, nbFinalizeSamples, coverParams.zParams);
if (!ZSTD_isError(dictionarySize)) {
DISPLAYLEVEL(2, "Constructed dictionary of size %u\n",
(U32)dictionarySize);
(unsigned)dictionarySize);
}
FASTCOVER_ctx_destroy(&ctx);
free(segmentFreqs);
@ -702,7 +702,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
}
/* Print status */
LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ",
(U32)((iteration * 100) / kIterations));
(unsigned)((iteration * 100) / kIterations));
++iteration;
}
COVER_best_wait(&best);

56
sys/contrib/zstd/lib/dictBuilder/zdict.c
View File

@ -255,15 +255,15 @@ static dictItem ZDICT_analyzePos(
}
{ int i;
U32 searchLength;
U32 mml;
U32 refinedStart = start;
U32 refinedEnd = end;
DISPLAYLEVEL(4, "\n");
DISPLAYLEVEL(4, "found %3u matches of length >= %i at pos %7u ", (U32)(end-start), MINMATCHLENGTH, (U32)pos);
DISPLAYLEVEL(4, "found %3u matches of length >= %i at pos %7u ", (unsigned)(end-start), MINMATCHLENGTH, (unsigned)pos);
DISPLAYLEVEL(4, "\n");
for (searchLength = MINMATCHLENGTH ; ; searchLength++) {
for (mml = MINMATCHLENGTH ; ; mml++) {
BYTE currentChar = 0;
U32 currentCount = 0;
U32 currentID = refinedStart;
@ -271,13 +271,13 @@ static dictItem ZDICT_analyzePos(
U32 selectedCount = 0;
U32 selectedID = currentID;
for (id =refinedStart; id < refinedEnd; id++) {
if (b[suffix[id] + searchLength] != currentChar) {
if (b[suffix[id] + mml] != currentChar) {
if (currentCount > selectedCount) {
selectedCount = currentCount;
selectedID = currentID;
}
currentID = id;
currentChar = b[ suffix[id] + searchLength];
currentChar = b[ suffix[id] + mml];
currentCount = 0;
}
currentCount ++;
@ -342,7 +342,7 @@ static dictItem ZDICT_analyzePos(
savings[i] = savings[i-1] + (lengthList[i] * (i-3));
DISPLAYLEVEL(4, "Selected dict at position %u, of length %u : saves %u (ratio: %.2f) \n",
(U32)pos, (U32)maxLength, savings[maxLength], (double)savings[maxLength] / maxLength);
(unsigned)pos, (unsigned)maxLength, (unsigned)savings[maxLength], (double)savings[maxLength] / maxLength);
solution.pos = (U32)pos;
solution.length = (U32)maxLength;
@ -497,7 +497,7 @@ static U32 ZDICT_dictSize(const dictItem* dictList)
static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize,
const void* const buffer, size_t bufferSize, /* buffer must end with noisy guard band */
const size_t* fileSizes, unsigned nbFiles,
U32 minRatio, U32 notificationLevel)
unsigned minRatio, U32 notificationLevel)
{
int* const suffix0 = (int*)malloc((bufferSize+2)*sizeof(*suffix0));
int* const suffix = suffix0+1;
@ -523,11 +523,11 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize,
memset(doneMarks, 0, bufferSize+16);
/* limit sample set size (divsufsort limitation)*/
if (bufferSize > ZDICT_MAX_SAMPLES_SIZE) DISPLAYLEVEL(3, "sample set too large : reduced to %u MB ...\n", (U32)(ZDICT_MAX_SAMPLES_SIZE>>20));
if (bufferSize > ZDICT_MAX_SAMPLES_SIZE) DISPLAYLEVEL(3, "sample set too large : reduced to %u MB ...\n", (unsigned)(ZDICT_MAX_SAMPLES_SIZE>>20));
while (bufferSize > ZDICT_MAX_SAMPLES_SIZE) bufferSize -= fileSizes[--nbFiles];
/* sort */
DISPLAYLEVEL(2, "sorting %u files of total size %u MB ...\n", nbFiles, (U32)(bufferSize>>20));
DISPLAYLEVEL(2, "sorting %u files of total size %u MB ...\n", nbFiles, (unsigned)(bufferSize>>20));
{ int const divSuftSortResult = divsufsort((const unsigned char*)buffer, suffix, (int)bufferSize, 0);
if (divSuftSortResult != 0) { result = ERROR(GENERIC); goto _cleanup; }
}
@ -589,7 +589,7 @@ typedef struct
#define MAXREPOFFSET 1024
static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets,
unsigned* countLit, unsigned* offsetcodeCount, unsigned* matchlengthCount, unsigned* litlengthCount, U32* repOffsets,
const void* src, size_t srcSize,
U32 notificationLevel)
{
@ -602,7 +602,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
}
cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (unsigned)srcSize); return; }
if (cSize) { /* if == 0; block is not compressible */
const seqStore_t* const seqStorePtr = ZSTD_getSeqStore(esr.zc);
@ -671,7 +671,7 @@ static void ZDICT_insertSortCount(offsetCount_t table[ZSTD_REP_NUM+1], U32 val,
* rewrite `countLit` to contain a mostly flat but still compressible distribution of literals.
* necessary to avoid generating a non-compressible distribution that HUF_writeCTable() cannot encode.
*/
static void ZDICT_flatLit(U32* countLit)
static void ZDICT_flatLit(unsigned* countLit)
{
int u;
for (u=1; u<256; u++) countLit[u] = 2;
@ -687,14 +687,14 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
const void* dictBuffer, size_t dictBufferSize,
unsigned notificationLevel)
{
U32 countLit[256];
unsigned countLit[256];
HUF_CREATE_STATIC_CTABLE(hufTable, 255);
U32 offcodeCount[OFFCODE_MAX+1];
unsigned offcodeCount[OFFCODE_MAX+1];
short offcodeNCount[OFFCODE_MAX+1];
U32 offcodeMax = ZSTD_highbit32((U32)(dictBufferSize + 128 KB));
U32 matchLengthCount[MaxML+1];
unsigned matchLengthCount[MaxML+1];
short matchLengthNCount[MaxML+1];
U32 litLengthCount[MaxLL+1];
unsigned litLengthCount[MaxLL+1];
short litLengthNCount[MaxLL+1];
U32 repOffset[MAXREPOFFSET];
offsetCount_t bestRepOffset[ZSTD_REP_NUM+1];
@ -983,33 +983,33 @@ size_t ZDICT_trainFromBuffer_unsafe_legacy(
/* display best matches */
if (params.zParams.notificationLevel>= 3) {
U32 const nb = MIN(25, dictList[0].pos);
U32 const dictContentSize = ZDICT_dictSize(dictList);
U32 u;
DISPLAYLEVEL(3, "\n %u segments found, of total size %u \n", dictList[0].pos-1, dictContentSize);
unsigned const nb = MIN(25, dictList[0].pos);
unsigned const dictContentSize = ZDICT_dictSize(dictList);
unsigned u;
DISPLAYLEVEL(3, "\n %u segments found, of total size %u \n", (unsigned)dictList[0].pos-1, dictContentSize);
DISPLAYLEVEL(3, "list %u best segments \n", nb-1);
for (u=1; u<nb; u++) {
U32 const pos = dictList[u].pos;
U32 const length = dictList[u].length;
unsigned const pos = dictList[u].pos;
unsigned const length = dictList[u].length;
U32 const printedLength = MIN(40, length);
if ((pos > samplesBuffSize) || ((pos + length) > samplesBuffSize)) {
free(dictList);
return ERROR(GENERIC); /* should never happen */
}
DISPLAYLEVEL(3, "%3u:%3u bytes at pos %8u, savings %7u bytes |",
u, length, pos, dictList[u].savings);
u, length, pos, (unsigned)dictList[u].savings);
ZDICT_printHex((const char*)samplesBuffer+pos, printedLength);
DISPLAYLEVEL(3, "| \n");
} }
/* create dictionary */
{ U32 dictContentSize = ZDICT_dictSize(dictList);
{ unsigned dictContentSize = ZDICT_dictSize(dictList);
if (dictContentSize < ZDICT_CONTENTSIZE_MIN) { free(dictList); return ERROR(dictionaryCreation_failed); } /* dictionary content too small */
if (dictContentSize < targetDictSize/4) {
DISPLAYLEVEL(2, "! warning : selected content significantly smaller than requested (%u < %u) \n", dictContentSize, (U32)maxDictSize);
DISPLAYLEVEL(2, "! warning : selected content significantly smaller than requested (%u < %u) \n", dictContentSize, (unsigned)maxDictSize);
if (samplesBuffSize < 10 * targetDictSize)
DISPLAYLEVEL(2, "! consider increasing the number of samples (total size : %u MB)\n", (U32)(samplesBuffSize>>20));
DISPLAYLEVEL(2, "! consider increasing the number of samples (total size : %u MB)\n", (unsigned)(samplesBuffSize>>20));
if (minRep > MINRATIO) {
DISPLAYLEVEL(2, "! consider increasing selectivity to produce larger dictionary (-s%u) \n", selectivity+1);
DISPLAYLEVEL(2, "! note : larger dictionaries are not necessarily better, test its efficiency on samples \n");
@ -1017,9 +1017,9 @@ size_t ZDICT_trainFromBuffer_unsafe_legacy(
}
if ((dictContentSize > targetDictSize*3) && (nbSamples > 2*MINRATIO) && (selectivity>1)) {
U32 proposedSelectivity = selectivity-1;
unsigned proposedSelectivity = selectivity-1;
while ((nbSamples >> proposedSelectivity) <= MINRATIO) { proposedSelectivity--; }
DISPLAYLEVEL(2, "! note : calculated dictionary significantly larger than requested (%u > %u) \n", dictContentSize, (U32)maxDictSize);
DISPLAYLEVEL(2, "! note : calculated dictionary significantly larger than requested (%u > %u) \n", dictContentSize, (unsigned)maxDictSize);
DISPLAYLEVEL(2, "! consider increasing dictionary size, or produce denser dictionary (-s%u) \n", proposedSelectivity);
DISPLAYLEVEL(2, "! always test dictionary efficiency on real samples \n");
}

3
sys/contrib/zstd/lib/freebsd/limits.h Normal file
View File

@ -0,0 +1,3 @@
/* This file is in the public domain */
/* $FreeBSD$ */
#include <sys/limits.h>

10
sys/contrib/zstd/lib/legacy/zstd_v04.c
View File

@ -240,17 +240,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
/* *************************************
* Types
***************************************/
#define ZSTD_WINDOWLOG_MAX 26
#define ZSTD_WINDOWLOG_MIN 18
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 11
#define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
#define ZSTD_CONTENTLOG_MIN 4
#define ZSTD_HASHLOG_MAX 28
#define ZSTD_HASHLOG_MIN 4
#define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1)
#define ZSTD_SEARCHLOG_MIN 1
#define ZSTD_SEARCHLENGTH_MAX 7
#define ZSTD_SEARCHLENGTH_MIN 4
/** from faster to stronger */
typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2 } ZSTD_strategy;

30
sys/contrib/zstd/lib/legacy/zstd_v05.c
View File

@ -836,7 +836,7 @@ MEM_STATIC void BITv05_skipBits(BITv05_DStream_t* bitD, U32 nbBits)
bitD->bitsConsumed += nbBits;
}
MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, U32 nbBits)
MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, unsigned nbBits)
{
size_t value = BITv05_lookBits(bitD, nbBits);
BITv05_skipBits(bitD, nbBits);
@ -845,7 +845,7 @@ MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, U32 nbBits)
/*!BITv05_readBitsFast :
* unsafe version; only works only if nbBits >= 1 */
MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, U32 nbBits)
MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, unsigned nbBits)
{
size_t value = BITv05_lookBitsFast(bitD, nbBits);
BITv05_skipBits(bitD, nbBits);
@ -1162,7 +1162,7 @@ MEM_STATIC unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr)
/* **************************************************************
* Complex types
****************************************************************/
typedef U32 DTable_max_t[FSEv05_DTABLE_SIZE_U32(FSEv05_MAX_TABLELOG)];
typedef unsigned DTable_max_t[FSEv05_DTABLE_SIZE_U32(FSEv05_MAX_TABLELOG)];
/* **************************************************************
@ -2191,7 +2191,7 @@ static void HUFv05_fillDTableX4(HUFv05_DEltX4* DTable, const U32 targetLog,
}
}
size_t HUFv05_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
size_t HUFv05_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize)
{
BYTE weightList[HUFv05_MAX_SYMBOL_VALUE + 1];
sortedSymbol_t sortedSymbol[HUFv05_MAX_SYMBOL_VALUE + 1];
@ -2205,7 +2205,7 @@ size_t HUFv05_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
void* dtPtr = DTable;
HUFv05_DEltX4* const dt = ((HUFv05_DEltX4*)dtPtr) + 1;
HUFv05_STATIC_ASSERT(sizeof(HUFv05_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
HUFv05_STATIC_ASSERT(sizeof(HUFv05_DEltX4) == sizeof(unsigned)); /* if compilation fails here, assertion is false */
if (memLog > HUFv05_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
//memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
@ -2332,7 +2332,7 @@ static inline size_t HUFv05_decodeStreamX4(BYTE* p, BITv05_DStream_t* bitDPtr, B
size_t HUFv05_decompress1X4_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const U32* DTable)
const unsigned* DTable)
{
const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
@ -2375,7 +2375,7 @@ size_t HUFv05_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t
size_t HUFv05_decompress4X4_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const U32* DTable)
const unsigned* DTable)
{
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
@ -2999,7 +2999,7 @@ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t
const BYTE* ip = istart;
const BYTE* const iend = istart + srcSize;
U32 LLtype, Offtype, MLtype;
U32 LLlog, Offlog, MLlog;
unsigned LLlog, Offlog, MLlog;
size_t dumpsLength;
/* check */
@ -3057,7 +3057,7 @@ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t
break;
case FSEv05_ENCODING_DYNAMIC :
default : /* impossible */
{ U32 max = MaxLL;
{ unsigned max = MaxLL;
headerSize = FSEv05_readNCount(norm, &max, &LLlog, ip, iend-ip);
if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
if (LLlog > LLFSEv05Log) return ERROR(corruption_detected);
@ -3081,7 +3081,7 @@ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t
break;
case FSEv05_ENCODING_DYNAMIC :
default : /* impossible */
{ U32 max = MaxOff;
{ unsigned max = MaxOff;
headerSize = FSEv05_readNCount(norm, &max, &Offlog, ip, iend-ip);
if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
if (Offlog > OffFSEv05Log) return ERROR(corruption_detected);
@ -3105,7 +3105,7 @@ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t
break;
case FSEv05_ENCODING_DYNAMIC :
default : /* impossible */
{ U32 max = MaxML;
{ unsigned max = MaxML;
headerSize = FSEv05_readNCount(norm, &max, &MLlog, ip, iend-ip);
if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
if (MLlog > MLFSEv05Log) return ERROR(corruption_detected);
@ -3305,9 +3305,9 @@ static size_t ZSTDv05_decompressSequences(
const BYTE* const litEnd = litPtr + dctx->litSize;
int nbSeq=0;
const BYTE* dumps = NULL;
U32* DTableLL = dctx->LLTable;
U32* DTableML = dctx->MLTable;
U32* DTableOffb = dctx->OffTable;
unsigned* DTableLL = dctx->LLTable;
unsigned* DTableML = dctx->MLTable;
unsigned* DTableOffb = dctx->OffTable;
const BYTE* const base = (const BYTE*) (dctx->base);
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
@ -3633,7 +3633,7 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d
{
size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, errorCode, litlengthHeaderSize;
short offcodeNCount[MaxOff+1];
U32 offcodeMaxValue=MaxOff, offcodeLog;
unsigned offcodeMaxValue=MaxOff, offcodeLog;
short matchlengthNCount[MaxML+1];
unsigned matchlengthMaxValue = MaxML, matchlengthLog;
short litlengthNCount[MaxLL+1];

1564
sys/contrib/zstd/lib/zstd.h
View File

File diff suppressed because it is too large Load Diff

45
sys/contrib/zstd/programs/Makefile
View File

@ -29,7 +29,12 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT))
ZSTD_VERSION = $(LIBVER)
GREP = grep --color=never
HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
GREP_OPTIONS ?=
ifeq ($HAVE_COLORNEVER, 1)
GREP_OPTIONS += --color=never
endif
GREP = grep $(GREP_OPTIONS)
ifeq ($(shell $(CC) -v 2>&1 | $(GREP) -c "gcc version "), 1)
ALIGN_LOOP = -falign-loops=32
@ -48,7 +53,7 @@ DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-Wredundant-decls -Wmissing-prototypes
-Wredundant-decls -Wmissing-prototypes -Wc++-compat
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
@ -91,7 +96,7 @@ VOID = /dev/null
# thread detection
NO_THREAD_MSG := ==> no threads, building without multithreading support
HAVE_PTHREAD := $(shell printf '\#include <pthread.h>\nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_pthread$(EXT) -x c - -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0)
HAVE_PTHREAD := $(shell printf '\#include <pthread.h>\nint main(void) { return 0; }' > have_pthread.c && $(CC) $(FLAGS) -o have_pthread$(EXT) have_pthread.c -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0; rm have_pthread.c)
HAVE_THREAD := $(shell [ "$(HAVE_PTHREAD)" -eq "1" -o -n "$(filter Windows%,$(OS))" ] && echo 1 || echo 0)
ifeq ($(HAVE_THREAD), 1)
THREAD_MSG := ==> building with threading support
@ -103,7 +108,7 @@ endif
# zlib detection
NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support
HAVE_ZLIB := $(shell printf '\#include <zlib.h>\nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_zlib$(EXT) -x c - -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0)
HAVE_ZLIB := $(shell printf '\#include <zlib.h>\nint main(void) { return 0; }' > have_zlib.c && $(CC) $(FLAGS) -o have_zlib$(EXT) have_zlib.c -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0; rm have_zlib.c)
ifeq ($(HAVE_ZLIB), 1)
ZLIB_MSG := ==> building zstd with .gz compression support
ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS
@ -114,7 +119,7 @@ endif
# lzma detection
NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support
HAVE_LZMA := $(shell printf '\#include <lzma.h>\nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_lzma$(EXT) -x c - -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0)
HAVE_LZMA := $(shell printf '\#include <lzma.h>\nint main(void) { return 0; }' > have_lzma.c && $(CC) $(FLAGS) -o have_lzma$(EXT) have_lzma.c -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0; rm have_lzma.c)
ifeq ($(HAVE_LZMA), 1)
LZMA_MSG := ==> building zstd with .xz/.lzma compression support
LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS
@ -125,7 +130,7 @@ endif
# lz4 detection
NO_LZ4_MSG := ==> no liblz4, building zstd without .lz4 support
HAVE_LZ4 := $(shell printf '\#include <lz4frame.h>\n\#include <lz4.h>\nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_lz4$(EXT) -x c - -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0)
HAVE_LZ4 := $(shell printf '\#include <lz4frame.h>\n\#include <lz4.h>\nint main(void) { return 0; }' > have_lz4.c && $(CC) $(FLAGS) -o have_lz4$(EXT) have_lz4.c -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0; rm have_lz4.c)
ifeq ($(HAVE_LZ4), 1)
LZ4_MSG := ==> building zstd with .lz4 compression support
LZ4CPP = -DZSTD_LZ4COMPRESS -DZSTD_LZ4DECOMPRESS
@ -160,7 +165,7 @@ $(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP)
zstd : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD)
zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
zstd : $(ZSTDLIB_FILES) zstdcli.o fileio.o bench.o datagen.o dibio.o
zstd : $(ZSTDLIB_FILES) zstdcli.o util.o fileio.o benchfn.o benchzstd.o datagen.o dibio.o
@echo "$(THREAD_MSG)"
@echo "$(ZLIB_MSG)"
@echo "$(LZMA_MSG)"
@ -178,13 +183,13 @@ zstd-release: zstd
zstd32 : CPPFLAGS += $(THREAD_CPP)
zstd32 : LDFLAGS += $(THREAD_LD)
zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
zstd32 : $(ZSTDLIB_FILES) zstdcli.c fileio.c bench.c datagen.c dibio.c
zstd32 : $(ZSTDLIB_FILES) zstdcli.c util.c fileio.c benchfn.c benchzstd.c datagen.c dibio.c
ifneq (,$(filter Windows%,$(OS)))
windres/generate_res.bat
endif
$(CC) -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT)
zstd-nolegacy : $(ZSTD_FILES) $(ZDICT_FILES) zstdcli.o fileio.c bench.o datagen.o dibio.o
zstd-nolegacy : $(ZSTD_FILES) $(ZDICT_FILES) zstdcli.o util.o fileio.c benchfn.o benchzstd.o datagen.o dibio.o
$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
zstd-nomt : THREAD_CPP :=
@ -203,27 +208,27 @@ zstd-noxz : LZMA_MSG := - xz/lzma support is disabled
zstd-noxz : zstd
zstd-pgo : MOREFLAGS = -fprofile-generate
zstd-pgo : clean zstd
zstd-pgo :
$(MAKE) clean
$(MAKE) zstd MOREFLAGS=-fprofile-generate
./zstd -b19i1 $(PROFILE_WITH)
./zstd -b16i1 $(PROFILE_WITH)
./zstd -b9i2 $(PROFILE_WITH)
./zstd -b $(PROFILE_WITH)
./zstd -b7i2 $(PROFILE_WITH)
./zstd -b5 $(PROFILE_WITH)
$(RM) zstd
$(RM) $(ZSTDDECOMP_O)
$(RM) zstd *.o $(ZSTDDECOMP_O) $(ZSTDDIR)/compress/*.o
$(MAKE) zstd MOREFLAGS=-fprofile-use
# minimal target, with only zstd compression and decompression. no bench. no legacy.
zstd-small: CFLAGS = -Os -s
zstd-frugal zstd-small: $(ZSTD_FILES) zstdcli.c fileio.c
zstd-frugal zstd-small: $(ZSTD_FILES) zstdcli.c util.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o $@$(EXT)
zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c fileio.c
zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c util.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o $@$(EXT)
zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c
zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c util.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT)
zstdmt: zstd
@ -275,7 +280,12 @@ preview-man: clean-man man
#-----------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
EGREP = egrep --color=never
HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
EGREP_OPTIONS ?=
ifeq ($HAVE_COLORNEVER, 1)
EGREP_OPTIONS += --color=never
endif
EGREP = egrep $(EGREP_OPTIONS)
# Print a two column output of targets and their description. To add a target description, put a
# comment in the Makefile with the format "## <TARGET>: <DESCRIPTION>". For example:
@ -352,6 +362,7 @@ uninstall:
@$(RM) $(DESTDIR)$(BINDIR)/zstdless
@$(RM) $(DESTDIR)$(BINDIR)/zstdcat
@$(RM) $(DESTDIR)$(BINDIR)/unzstd
@$(RM) $(DESTDIR)$(BINDIR)/zstdmt
@$(RM) $(DESTDIR)$(BINDIR)/zstd
@$(RM) $(DESTDIR)$(MAN1DIR)/zstdless.1
@$(RM) $(DESTDIR)$(MAN1DIR)/zstdgrep.1

5
sys/contrib/zstd/programs/README.md
View File

@ -172,6 +172,11 @@ Benchmark arguments :
--priority=rt : set process priority to real-time
```
#### Restricted usage of Environment Variables
Using environment variables to set compression/decompression parameters has security implications. Therefore,
we intentionally restrict its usage. Currently, only `ZSTD_CLEVEL` is supported for setting compression level.
If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a warning message.
Note that command line options will override corresponding environment variable settings.
#### Long distance matching mode
The long distance matching mode, enabled with `--long`, is designed to improve

263
sys/contrib/zstd/programs/benchfn.c Normal file
View File

@ -0,0 +1,263 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/* *************************************
* Includes
***************************************/
#include "platform.h" /* Large Files support */
#include "util.h" /* UTIL_getFileSize, UTIL_sleep */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* memset */
#include <stdio.h> /* fprintf, fopen */
#undef NDEBUG /* assert must not be disabled */
#include <assert.h> /* assert */
#include "mem.h"
#include "benchfn.h"
/* *************************************
* Constants
***************************************/
#define TIMELOOP_MICROSEC (1*1000000ULL) /* 1 second */
#define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */
#define ACTIVEPERIOD_MICROSEC (70*TIMELOOP_MICROSEC) /* 70 seconds */
#define COOLPERIOD_SEC 10
#define KB *(1 <<10)
#define MB *(1 <<20)
#define GB *(1U<<30)
/* *************************************
* Errors
***************************************/
#ifndef DEBUG
# define DEBUG 0
#endif
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
/* error without displaying */
#define RETURN_QUIET_ERROR(retValue, ...) { \
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
DEBUGOUTPUT("Error : "); \
DEBUGOUTPUT(__VA_ARGS__); \
DEBUGOUTPUT(" \n"); \
return retValue; \
}
/* *************************************
* Benchmarking an arbitrary function
***************************************/
int BMK_isSuccessful_runOutcome(BMK_runOutcome_t outcome)
{
return outcome.error_tag_never_ever_use_directly == 0;
}
/* warning : this function will stop program execution if outcome is invalid !
* check outcome validity first, using BMK_isValid_runResult() */
BMK_runTime_t BMK_extract_runTime(BMK_runOutcome_t outcome)
{
assert(outcome.error_tag_never_ever_use_directly == 0);
return outcome.internal_never_ever_use_directly;
}
size_t BMK_extract_errorResult(BMK_runOutcome_t outcome)
{
assert(outcome.error_tag_never_ever_use_directly != 0);
return outcome.error_result_never_ever_use_directly;
}
static BMK_runOutcome_t BMK_runOutcome_error(size_t errorResult)
{
BMK_runOutcome_t b;
memset(&b, 0, sizeof(b));
b.error_tag_never_ever_use_directly = 1;
b.error_result_never_ever_use_directly = errorResult;
return b;
}
static BMK_runOutcome_t BMK_setValid_runTime(BMK_runTime_t runTime)
{
BMK_runOutcome_t outcome;
outcome.error_tag_never_ever_use_directly = 0;
outcome.internal_never_ever_use_directly = runTime;
return outcome;
}
/* initFn will be measured once, benchFn will be measured `nbLoops` times */
/* initFn is optional, provide NULL if none */
/* benchFn must return a size_t value that errorFn can interpret */
/* takes # of blocks and list of size & stuff for each. */
/* can report result of benchFn for each block into blockResult. */
/* blockResult is optional, provide NULL if this information is not required */
/* note : time per loop can be reported as zero if run time < timer resolution */
BMK_runOutcome_t BMK_benchFunction(BMK_benchParams_t p,
unsigned nbLoops)
{
size_t dstSize = 0;
nbLoops += !nbLoops; /* minimum nbLoops is 1 */
/* init */
{ size_t i;
for(i = 0; i < p.blockCount; i++) {
memset(p.dstBuffers[i], 0xE5, p.dstCapacities[i]); /* warm up and erase result buffer */
}
#if 0
/* based on testing these seem to lower accuracy of multiple calls of 1 nbLoops vs 1 call of multiple nbLoops
* (Makes former slower)
*/
UTIL_sleepMilli(5); /* give processor time to other processes */
UTIL_waitForNextTick();
#endif
}
/* benchmark */
{ UTIL_time_t const clockStart = UTIL_getTime();
unsigned loopNb, blockNb;
if (p.initFn != NULL) p.initFn(p.initPayload);
for (loopNb = 0; loopNb < nbLoops; loopNb++) {
for (blockNb = 0; blockNb < p.blockCount; blockNb++) {
size_t const res = p.benchFn(p.srcBuffers[blockNb], p.srcSizes[blockNb],
p.dstBuffers[blockNb], p.dstCapacities[blockNb],
p.benchPayload);
if (loopNb == 0) {
if (p.blockResults != NULL) p.blockResults[blockNb] = res;
if ((p.errorFn != NULL) && (p.errorFn(res))) {
RETURN_QUIET_ERROR(BMK_runOutcome_error(res),
"Function benchmark failed on block %u (of size %u) with error %i",
blockNb, (unsigned)p.srcSizes[blockNb], (int)res);
}
dstSize += res;
} }
} /* for (loopNb = 0; loopNb < nbLoops; loopNb++) */
{ U64 const totalTime = UTIL_clockSpanNano(clockStart);
BMK_runTime_t rt;
rt.nanoSecPerRun = totalTime / nbLoops;
rt.sumOfReturn = dstSize;
return BMK_setValid_runTime(rt);
} }
}
/* ==== Benchmarking any function, providing intermediate results ==== */
struct BMK_timedFnState_s {
U64 timeSpent_ns;
U64 timeBudget_ns;
U64 runBudget_ns;
BMK_runTime_t fastestRun;
unsigned nbLoops;
UTIL_time_t coolTime;
}; /* typedef'd to BMK_timedFnState_t within bench.h */
BMK_timedFnState_t* BMK_createTimedFnState(unsigned total_ms, unsigned run_ms)
{
BMK_timedFnState_t* const r = (BMK_timedFnState_t*)malloc(sizeof(*r));
if (r == NULL) return NULL; /* malloc() error */
BMK_resetTimedFnState(r, total_ms, run_ms);
return r;
}
void BMK_freeTimedFnState(BMK_timedFnState_t* state) {
free(state);
}
void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms, unsigned run_ms)
{
if (!total_ms) total_ms = 1 ;
if (!run_ms) run_ms = 1;
if (run_ms > total_ms) run_ms = total_ms;
timedFnState->timeSpent_ns = 0;
timedFnState->timeBudget_ns = (U64)total_ms * TIMELOOP_NANOSEC / 1000;
timedFnState->runBudget_ns = (U64)run_ms * TIMELOOP_NANOSEC / 1000;
timedFnState->fastestRun.nanoSecPerRun = (U64)(-1LL);
timedFnState->fastestRun.sumOfReturn = (size_t)(-1LL);
timedFnState->nbLoops = 1;
timedFnState->coolTime = UTIL_getTime();
}
/* Tells if nb of seconds set in timedFnState for all runs is spent.
* note : this function will return 1 if BMK_benchFunctionTimed() has actually errored. */
int BMK_isCompleted_TimedFn(const BMK_timedFnState_t* timedFnState)
{
return (timedFnState->timeSpent_ns >= timedFnState->timeBudget_ns);
}
#undef MIN
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
#define MINUSABLETIME (TIMELOOP_NANOSEC / 2) /* 0.5 seconds */
BMK_runOutcome_t BMK_benchTimedFn(BMK_timedFnState_t* cont,
BMK_benchParams_t p)
{
U64 const runBudget_ns = cont->runBudget_ns;
U64 const runTimeMin_ns = runBudget_ns / 2;
int completed = 0;
BMK_runTime_t bestRunTime = cont->fastestRun;
while (!completed) {
BMK_runOutcome_t runResult;
/* Overheat protection */
if (UTIL_clockSpanMicro(cont->coolTime) > ACTIVEPERIOD_MICROSEC) {
DEBUGOUTPUT("\rcooling down ... \r");
UTIL_sleep(COOLPERIOD_SEC);
cont->coolTime = UTIL_getTime();
}
/* reinitialize capacity */
runResult = BMK_benchFunction(p, cont->nbLoops);
if(!BMK_isSuccessful_runOutcome(runResult)) { /* error : move out */
return runResult;
}
{ BMK_runTime_t const newRunTime = BMK_extract_runTime(runResult);
U64 const loopDuration_ns = newRunTime.nanoSecPerRun * cont->nbLoops;
cont->timeSpent_ns += loopDuration_ns;
/* estimate nbLoops for next run to last approximately 1 second */
if (loopDuration_ns > (runBudget_ns / 50)) {
U64 const fastestRun_ns = MIN(bestRunTime.nanoSecPerRun, newRunTime.nanoSecPerRun);
cont->nbLoops = (U32)(runBudget_ns / fastestRun_ns) + 1;
} else {
/* previous run was too short : blindly increase workload by x multiplier */
const unsigned multiplier = 10;
assert(cont->nbLoops < ((unsigned)-1) / multiplier); /* avoid overflow */
cont->nbLoops *= multiplier;
}
if(loopDuration_ns < runTimeMin_ns) {
/* don't report results for which benchmark run time was too small : increased risks of rounding errors */
assert(completed == 0);
continue;
} else {
if(newRunTime.nanoSecPerRun < bestRunTime.nanoSecPerRun) {
bestRunTime = newRunTime;
}
completed = 1;
}
}
} /* while (!completed) */
return BMK_setValid_runTime(bestRunTime);
}

167
sys/contrib/zstd/programs/benchfn.h Normal file
View File

@ -0,0 +1,167 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/* benchfn :
* benchmark any function on a set of input
* providing result in nanoSecPerRun
* or detecting and returning an error
*/
#if defined (__cplusplus)
extern "C" {
#endif
#ifndef BENCH_FN_H_23876
#define BENCH_FN_H_23876
/* === Dependencies === */
#include <stddef.h> /* size_t */
/* ==== Benchmark any function, iterated on a set of blocks ==== */
/* BMK_runTime_t: valid result return type */
typedef struct {
unsigned long long nanoSecPerRun; /* time per iteration (over all blocks) */
size_t sumOfReturn; /* sum of return values */
} BMK_runTime_t;
/* BMK_runOutcome_t:
* type expressing the outcome of a benchmark run by BMK_benchFunction(),
* which can be either valid or invalid.
* benchmark outcome can be invalid if errorFn is provided.
* BMK_runOutcome_t must be considered "opaque" : never access its members directly.
* Instead, use its assigned methods :
* BMK_isSuccessful_runOutcome, BMK_extract_runTime, BMK_extract_errorResult.
* The structure is only described here to allow its allocation on stack. */
typedef struct {
BMK_runTime_t internal_never_ever_use_directly;
size_t error_result_never_ever_use_directly;
int error_tag_never_ever_use_directly;
} BMK_runOutcome_t;
/* prototypes for benchmarked functions */
typedef size_t (*BMK_benchFn_t)(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* customPayload);
typedef size_t (*BMK_initFn_t)(void* initPayload);
typedef unsigned (*BMK_errorFn_t)(size_t);
/* BMK_benchFunction() parameters are provided through following structure.
* This is preferable for readability,
* as the number of parameters required is pretty large.
* No initializer is provided, because it doesn't make sense to provide some "default" :
* all parameters should be specified by the caller */
typedef struct {
BMK_benchFn_t benchFn; /* the function to benchmark, over the set of blocks */
void* benchPayload; /* pass custom parameters to benchFn :
* (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload) */
BMK_initFn_t initFn; /* (*initFn)(initPayload) is run once per run, at the beginning. */
void* initPayload; /* Both arguments can be NULL, in which case nothing is run. */
BMK_errorFn_t errorFn; /* errorFn will check each return value of benchFn over each block, to determine if it failed or not.
* errorFn can be NULL, in which case no check is performed.
* errorFn must return 0 when benchFn was successful, and >= 1 if it detects an error.
* Execution is stopped as soon as an error is detected.
* the triggering return value can be retrieved using BMK_extract_errorResult(). */
size_t blockCount; /* number of blocks to operate benchFn on.
* It's also the size of all array parameters :
* srcBuffers, srcSizes, dstBuffers, dstCapacities, blockResults */
const void *const * srcBuffers; /* array of buffers to be operated on by benchFn */
const size_t* srcSizes; /* array of the sizes of srcBuffers buffers */
void *const * dstBuffers;/* array of buffers to be written into by benchFn */
const size_t* dstCapacities; /* array of the capacities of dstBuffers buffers */
size_t* blockResults; /* Optional: store the return value of benchFn for each block. Use NULL if this result is not requested. */
} BMK_benchParams_t;
/* BMK_benchFunction() :
* This function benchmarks benchFn and initFn, providing a result.
*
* params : see description of BMK_benchParams_t above.
* nbLoops: defines number of times benchFn is run over the full set of blocks.
* Minimum value is 1. A 0 is interpreted as a 1.
*
* @return: can express either an error or a successful result.
* Use BMK_isSuccessful_runOutcome() to check if benchmark was successful.
* If yes, extract the result with BMK_extract_runTime(),
* it will contain :
* .sumOfReturn : the sum of all return values of benchFn through all of blocks
* .nanoSecPerRun : time per run of benchFn + (time for initFn / nbLoops)
* .sumOfReturn is generally intended for functions which return a # of bytes written into dstBuffer,
* in which case, this value will be the total amount of bytes written into dstBuffer.
*
* blockResults : when provided (!= NULL), and when benchmark is successful,
* params.blockResults contains all return values of `benchFn` over all blocks.
* when provided (!= NULL), and when benchmark failed,
* params.blockResults contains return values of `benchFn` over all blocks preceding and including the failed block.
*/
BMK_runOutcome_t BMK_benchFunction(BMK_benchParams_t params, unsigned nbLoops);
/* check first if the benchmark was successful or not */
int BMK_isSuccessful_runOutcome(BMK_runOutcome_t outcome);
/* If the benchmark was successful, extract the result.
* note : this function will abort() program execution if benchmark failed !
* always check if benchmark was successful first !
*/
BMK_runTime_t BMK_extract_runTime(BMK_runOutcome_t outcome);
/* when benchmark failed, it means one invocation of `benchFn` failed.
* The failure was detected by `errorFn`, operating on return values of `benchFn`.
* Returns the faulty return value.
* note : this function will abort() program execution if benchmark did not failed.
* always check if benchmark failed first !
*/
size_t BMK_extract_errorResult(BMK_runOutcome_t outcome);
/* ==== Benchmark any function, returning intermediate results ==== */
/* state information tracking benchmark session */
typedef struct BMK_timedFnState_s BMK_timedFnState_t;
/* BMK_benchTimedFn() :
* Similar to BMK_benchFunction(), most arguments being identical.
* Automatically determines `nbLoops` so that each result is regularly produced at interval of about run_ms.
* Note : minimum `nbLoops` is 1, therefore a run may last more than run_ms, and possibly even more than total_ms.
* Usage - initialize timedFnState, select benchmark duration (total_ms) and each measurement duration (run_ms)
* call BMK_benchTimedFn() repetitively, each measurement is supposed to last about run_ms
* Check if total time budget is spent or exceeded, using BMK_isCompleted_TimedFn()
*/
BMK_runOutcome_t BMK_benchTimedFn(BMK_timedFnState_t* timedFnState,
BMK_benchParams_t params);
/* Tells if duration of all benchmark runs has exceeded total_ms
*/
int BMK_isCompleted_TimedFn(const BMK_timedFnState_t* timedFnState);
/* BMK_createTimedFnState() and BMK_resetTimedFnState() :
* Create/Set BMK_timedFnState_t for next benchmark session,
* which shall last a minimum of total_ms milliseconds,
* producing intermediate results, paced at interval of (approximately) run_ms.
*/
BMK_timedFnState_t* BMK_createTimedFnState(unsigned total_ms, unsigned run_ms);
void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms, unsigned run_ms);
void BMK_freeTimedFnState(BMK_timedFnState_t* state);
#endif /* BENCH_FN_H_23876 */
#if defined (__cplusplus)
}
#endif

420
sys/contrib/zstd/programs/bench.c → sys/contrib/zstd/programs/benchzstd.c
View File

@ -9,7 +9,6 @@
*/
/* **************************************
* Tuning parameters
****************************************/
@ -18,30 +17,24 @@
#endif
/* **************************************
* Compiler Warnings
****************************************/
#ifdef _MSC_VER
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#endif
/* *************************************
* Includes
***************************************/
#include "platform.h" /* Large Files support */
#include "util.h" /* UTIL_getFileSize, UTIL_sleep */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* memset */
#include <string.h> /* memset, strerror */
#include <stdio.h> /* fprintf, fopen */
#include <errno.h>
#include <assert.h> /* assert */
#include "benchfn.h"
#include "mem.h"
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
#include "datagen.h" /* RDG_genBuffer */
#include "xxhash.h"
#include "bench.h"
#include "benchzstd.h"
#include "zstd_errors.h"
@ -102,6 +95,18 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
return errorNum; \
}
#define CHECK_Z(zf) { \
size_t const zerr = zf; \
if (ZSTD_isError(zerr)) { \
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
DISPLAY("Error : "); \
DISPLAY("%s failed : %s", \
#zf, ZSTD_getErrorName(zerr)); \
DISPLAY(" \n"); \
exit(1); \
} \
}
#define RETURN_ERROR(errorNum, retType, ...) { \
retType r; \
memset(&r, 0, sizeof(retType)); \
@ -113,17 +118,6 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
return r; \
}
/* error without displaying */
#define RETURN_QUIET_ERROR(errorNum, retType, ...) { \
retType r; \
memset(&r, 0, sizeof(retType)); \
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
DEBUGOUTPUT("Error %i : ", errorNum); \
DEBUGOUTPUT(__VA_ARGS__); \
DEBUGOUTPUT(" \n"); \
r.tag = errorNum; \
return r; \
}
/* *************************************
* Benchmark Parameters
@ -141,7 +135,7 @@ BMK_advancedParams_t BMK_initAdvancedParams(void) {
0, /* ldmMinMatch */
0, /* ldmHashLog */
0, /* ldmBuckSizeLog */
0 /* ldmHashEveryLog */
0 /* ldmHashRateLog */
};
return res;
}
@ -168,33 +162,32 @@ typedef struct {
static void BMK_initCCtx(ZSTD_CCtx* ctx,
const void* dictBuffer, size_t dictBufferSize, int cLevel,
const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) {
ZSTD_CCtx_reset(ctx);
ZSTD_CCtx_resetParameters(ctx);
ZSTD_CCtx_reset(ctx, ZSTD_reset_session_and_parameters);
if (adv->nbWorkers==1) {
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0);
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, 0));
} else {
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, adv->nbWorkers);
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, adv->nbWorkers));
}
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, adv->ldmFlag);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, adv->ldmMinMatch);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, adv->ldmHashLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, adv->ldmBucketSizeLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, adv->ldmHashEveryLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_hashLog, comprParams->hashLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy);
ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, cLevel));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_enableLongDistanceMatching, adv->ldmFlag));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmMinMatch, adv->ldmMinMatch));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashLog, adv->ldmHashLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmBucketSizeLog, adv->ldmBucketSizeLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashRateLog, adv->ldmHashRateLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_windowLog, comprParams->windowLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_hashLog, comprParams->hashLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_chainLog, comprParams->chainLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_searchLog, comprParams->searchLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, comprParams->minMatch));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, comprParams->targetLength));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_strategy, comprParams->strategy));
CHECK_Z(ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize));
}
static void BMK_initDCtx(ZSTD_DCtx* dctx,
const void* dictBuffer, size_t dictBufferSize) {
ZSTD_DCtx_reset(dctx);
ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize);
CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters));
CHECK_Z(ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize));
}
@ -232,22 +225,8 @@ static size_t local_defaultCompress(
void* dstBuffer, size_t dstSize,
void* addArgs)
{
size_t moreToFlush = 1;
ZSTD_CCtx* const cctx = (ZSTD_CCtx*)addArgs;
ZSTD_inBuffer in;
ZSTD_outBuffer out;
in.src = srcBuffer; in.size = srcSize; in.pos = 0;
out.dst = dstBuffer; out.size = dstSize; out.pos = 0;
while (moreToFlush) {
if(out.pos == out.size) {
return (size_t)-ZSTD_error_dstSize_tooSmall;
}
moreToFlush = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
if (ZSTD_isError(moreToFlush)) {
return moreToFlush;
}
}
return out.pos;
return ZSTD_compress2(cctx, dstBuffer, dstSize, srcBuffer, srcSize);
}
/* `addArgs` is the context */
@ -266,7 +245,7 @@ static size_t local_defaultDecompress(
if(out.pos == out.size) {
return (size_t)-ZSTD_error_dstSize_tooSmall;
}
moreToFlush = ZSTD_decompress_generic(dctx, &out, &in);
moreToFlush = ZSTD_decompressStream(dctx, &out, &in);
if (ZSTD_isError(moreToFlush)) {
return moreToFlush;
}
@ -276,219 +255,6 @@ static size_t local_defaultDecompress(
}
/*=== Benchmarking an arbitrary function ===*/
int BMK_isSuccessful_runOutcome(BMK_runOutcome_t outcome)
{
return outcome.tag == 0;
}
/* warning : this function will stop program execution if outcome is invalid !
* check outcome validity first, using BMK_isValid_runResult() */
BMK_runTime_t BMK_extract_runTime(BMK_runOutcome_t outcome)
{
assert(outcome.tag == 0);
return outcome.internal_never_use_directly;
}
static BMK_runOutcome_t BMK_runOutcome_error(void)
{
BMK_runOutcome_t b;
memset(&b, 0, sizeof(b));
b.tag = 1;
return b;
}
static BMK_runOutcome_t BMK_setValid_runTime(BMK_runTime_t runTime)
{
BMK_runOutcome_t outcome;
outcome.tag = 0;
outcome.internal_never_use_directly = runTime;
return outcome;
}
/* initFn will be measured once, benchFn will be measured `nbLoops` times */
/* initFn is optional, provide NULL if none */
/* benchFn must return size_t field compliant with ZSTD_isError for error valuee */
/* takes # of blocks and list of size & stuff for each. */
/* can report result of benchFn for each block into blockResult. */
/* blockResult is optional, provide NULL if this information is not required */
/* note : time per loop could be zero if run time < timer resolution */
BMK_runOutcome_t BMK_benchFunction(
BMK_benchFn_t benchFn, void* benchPayload,
BMK_initFn_t initFn, void* initPayload,
size_t blockCount,
const void* const * srcBlockBuffers, const size_t* srcBlockSizes,
void* const * dstBlockBuffers, const size_t* dstBlockCapacities,
size_t* blockResults,
unsigned nbLoops)
{
size_t dstSize = 0;
if(!nbLoops) {
RETURN_QUIET_ERROR(2, BMK_runOutcome_t, "nbLoops must be nonzero ");
}
/* init */
{ size_t i;
for(i = 0; i < blockCount; i++) {
memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */
}
#if 0
/* based on testing these seem to lower accuracy of multiple calls of 1 nbLoops vs 1 call of multiple nbLoops
* (Makes former slower)
*/
UTIL_sleepMilli(5); /* give processor time to other processes */
UTIL_waitForNextTick();
#endif
}
/* benchmark */
{ UTIL_time_t const clockStart = UTIL_getTime();
unsigned loopNb, blockNb;
if (initFn != NULL) initFn(initPayload);
for (loopNb = 0; loopNb < nbLoops; loopNb++) {
for (blockNb = 0; blockNb < blockCount; blockNb++) {
size_t const res = benchFn(srcBlockBuffers[blockNb], srcBlockSizes[blockNb],
dstBlockBuffers[blockNb], dstBlockCapacities[blockNb],
benchPayload);
if(ZSTD_isError(res)) {
RETURN_QUIET_ERROR(2, BMK_runOutcome_t,
"Function benchmark failed on block %u of size %u : %s",
blockNb, (U32)dstBlockCapacities[blockNb], ZSTD_getErrorName(res));
} else if (loopNb == 0) {
dstSize += res;
if (blockResults != NULL) blockResults[blockNb] = res;
} }
} /* for (loopNb = 0; loopNb < nbLoops; loopNb++) */
{ U64 const totalTime = UTIL_clockSpanNano(clockStart);
BMK_runTime_t rt;
rt.nanoSecPerRun = totalTime / nbLoops;
rt.sumOfReturn = dstSize;
return BMK_setValid_runTime(rt);
} }
}
/* ==== Benchmarking any function, providing intermediate results ==== */
struct BMK_timedFnState_s {
U64 timeSpent_ns;
U64 timeBudget_ns;
U64 runBudget_ns;
BMK_runTime_t fastestRun;
unsigned nbLoops;
UTIL_time_t coolTime;
}; /* typedef'd to BMK_timedFnState_t within bench.h */
BMK_timedFnState_t* BMK_createTimedFnState(unsigned total_ms, unsigned run_ms)
{
BMK_timedFnState_t* const r = (BMK_timedFnState_t*)malloc(sizeof(*r));
if (r == NULL) return NULL; /* malloc() error */
BMK_resetTimedFnState(r, total_ms, run_ms);
return r;
}
void BMK_freeTimedFnState(BMK_timedFnState_t* state) {
free(state);
}
void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms, unsigned run_ms)
{
if (!total_ms) total_ms = 1 ;
if (!run_ms) run_ms = 1;
if (run_ms > total_ms) run_ms = total_ms;
timedFnState->timeSpent_ns = 0;
timedFnState->timeBudget_ns = (U64)total_ms * TIMELOOP_NANOSEC / 1000;
timedFnState->runBudget_ns = (U64)run_ms * TIMELOOP_NANOSEC / 1000;
timedFnState->fastestRun.nanoSecPerRun = (U64)(-1LL);
timedFnState->fastestRun.sumOfReturn = (size_t)(-1LL);
timedFnState->nbLoops = 1;
timedFnState->coolTime = UTIL_getTime();
}
/* Tells if nb of seconds set in timedFnState for all runs is spent.
* note : this function will return 1 if BMK_benchFunctionTimed() has actually errored. */
int BMK_isCompleted_TimedFn(const BMK_timedFnState_t* timedFnState)
{
return (timedFnState->timeSpent_ns >= timedFnState->timeBudget_ns);
}
#define MINUSABLETIME (TIMELOOP_NANOSEC / 2) /* 0.5 seconds */
BMK_runOutcome_t BMK_benchTimedFn(
BMK_timedFnState_t* cont,
BMK_benchFn_t benchFn, void* benchPayload,
BMK_initFn_t initFn, void* initPayload,
size_t blockCount,
const void* const* srcBlockBuffers, const size_t* srcBlockSizes,
void * const * dstBlockBuffers, const size_t * dstBlockCapacities,
size_t* blockResults)
{
U64 const runBudget_ns = cont->runBudget_ns;
U64 const runTimeMin_ns = runBudget_ns / 2;
int completed = 0;
BMK_runTime_t bestRunTime = cont->fastestRun;
while (!completed) {
BMK_runOutcome_t runResult;
/* Overheat protection */
if (UTIL_clockSpanMicro(cont->coolTime) > ACTIVEPERIOD_MICROSEC) {
DEBUGOUTPUT("\rcooling down ... \r");
UTIL_sleep(COOLPERIOD_SEC);
cont->coolTime = UTIL_getTime();
}
/* reinitialize capacity */
runResult = BMK_benchFunction(benchFn, benchPayload,
initFn, initPayload,
blockCount,
srcBlockBuffers, srcBlockSizes,
dstBlockBuffers, dstBlockCapacities,
blockResults,
cont->nbLoops);
if(!BMK_isSuccessful_runOutcome(runResult)) { /* error : move out */
return BMK_runOutcome_error();
}
{ BMK_runTime_t const newRunTime = BMK_extract_runTime(runResult);
U64 const loopDuration_ns = newRunTime.nanoSecPerRun * cont->nbLoops;
cont->timeSpent_ns += loopDuration_ns;
/* estimate nbLoops for next run to last approximately 1 second */
if (loopDuration_ns > (runBudget_ns / 50)) {
U64 const fastestRun_ns = MIN(bestRunTime.nanoSecPerRun, newRunTime.nanoSecPerRun);
cont->nbLoops = (U32)(runBudget_ns / fastestRun_ns) + 1;
} else {
/* previous run was too short : blindly increase workload by x multiplier */
const unsigned multiplier = 10;
assert(cont->nbLoops < ((unsigned)-1) / multiplier); /* avoid overflow */
cont->nbLoops *= multiplier;
}
if(loopDuration_ns < runTimeMin_ns) {
/* don't report results for which benchmark run time was too small : increased risks of rounding errors */
assert(completed == 0);
continue;
} else {
if(newRunTime.nanoSecPerRun < bestRunTime.nanoSecPerRun) {
bestRunTime = newRunTime;
}
completed = 1;
}
}
} /* while (!completed) */
return BMK_setValid_runTime(bestRunTime);
}
/* ================================================================= */
/* Benchmark Zstandard, mem-to-mem scenarios */
/* ================================================================= */
@ -522,22 +288,24 @@ static BMK_benchOutcome_t BMK_benchOutcome_setValidResult(BMK_benchResult_t resu
/* benchMem with no allocation */
static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
const void** srcPtrs, size_t* srcSizes,
void** cPtrs, size_t* cCapacities, size_t* cSizes,
void** resPtrs, size_t* resSizes,
void** resultBufferPtr, void* compressedBuffer,
size_t maxCompressedSize,
BMK_timedFnState_t* timeStateCompress,
BMK_timedFnState_t* timeStateDecompress,
static BMK_benchOutcome_t
BMK_benchMemAdvancedNoAlloc(
const void** srcPtrs, size_t* srcSizes,
void** cPtrs, size_t* cCapacities, size_t* cSizes,
void** resPtrs, size_t* resSizes,
void** resultBufferPtr, void* compressedBuffer,
size_t maxCompressedSize,
BMK_timedFnState_t* timeStateCompress,
BMK_timedFnState_t* timeStateDecompress,
const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize,
ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
int displayLevel, const char* displayName,
const BMK_advancedParams_t* adv)
const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, unsigned nbFiles,
const int cLevel,
const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize,
ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
int displayLevel, const char* displayName,
const BMK_advancedParams_t* adv)
{
size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
BMK_benchResult_t benchResult;
@ -599,6 +367,11 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
cPtr += cCapacities[nbBlocks];
resPtr += thisBlockSize;
remaining -= thisBlockSize;
if (adv->mode == BMK_decodeOnly) {
assert(nbBlocks==0);
cSizes[nbBlocks] = thisBlockSize;
benchResult.cSize = thisBlockSize;
}
}
}
}
@ -617,32 +390,51 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
U32 markNb = 0;
int compressionCompleted = (adv->mode == BMK_decodeOnly);
int decompressionCompleted = (adv->mode == BMK_compressOnly);
BMK_benchParams_t cbp, dbp;
BMK_initCCtxArgs cctxprep;
BMK_initDCtxArgs dctxprep;
cbp.benchFn = local_defaultCompress;
cbp.benchPayload = cctx;
cbp.initFn = local_initCCtx;
cbp.initPayload = &cctxprep;
cbp.errorFn = ZSTD_isError;
cbp.blockCount = nbBlocks;
cbp.srcBuffers = srcPtrs;
cbp.srcSizes = srcSizes;
cbp.dstBuffers = cPtrs;
cbp.dstCapacities = cCapacities;
cbp.blockResults = cSizes;
cctxprep.cctx = cctx;
cctxprep.dictBuffer = dictBuffer;
cctxprep.dictBufferSize = dictBufferSize;
cctxprep.cLevel = cLevel;
cctxprep.comprParams = comprParams;
cctxprep.adv = adv;
dbp.benchFn = local_defaultDecompress;
dbp.benchPayload = dctx;
dbp.initFn = local_initDCtx;
dbp.initPayload = &dctxprep;
dbp.errorFn = ZSTD_isError;
dbp.blockCount = nbBlocks;
dbp.srcBuffers = (const void* const *) cPtrs;
dbp.srcSizes = cSizes;
dbp.dstBuffers = resPtrs;
dbp.dstCapacities = resSizes;
dbp.blockResults = NULL;
dctxprep.dctx = dctx;
dctxprep.dictBuffer = dictBuffer;
dctxprep.dictBufferSize = dictBufferSize;
DISPLAYLEVEL(2, "\r%70s\r", ""); /* blank line */
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (unsigned)srcSize);
while (!(compressionCompleted && decompressionCompleted)) {
if (!compressionCompleted) {
BMK_runOutcome_t const cOutcome =
BMK_benchTimedFn( timeStateCompress,
&local_defaultCompress, cctx,
&local_initCCtx, &cctxprep,
nbBlocks,
srcPtrs, srcSizes,
cPtrs, cCapacities,
cSizes);
BMK_runOutcome_t const cOutcome = BMK_benchTimedFn( timeStateCompress, cbp);
if (!BMK_isSuccessful_runOutcome(cOutcome)) {
return BMK_benchOutcome_error();
@ -659,10 +451,9 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
} }
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
marks[markNb], displayName,
(U32)srcSize, (U32)cSize,
(unsigned)srcSize, (unsigned)cSize,
ratioAccuracy, ratio,
benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT);
}
@ -670,14 +461,7 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
}
if(!decompressionCompleted) {
BMK_runOutcome_t const dOutcome =
BMK_benchTimedFn(timeStateDecompress,
&local_defaultDecompress, dctx,
&local_initDCtx, &dctxprep,
nbBlocks,
(const void *const *)cPtrs, cSizes,
resPtrs, resSizes,
NULL);
BMK_runOutcome_t const dOutcome = BMK_benchTimedFn(timeStateDecompress, dbp);
if(!BMK_isSuccessful_runOutcome(dOutcome)) {
return BMK_benchOutcome_error();
@ -690,16 +474,16 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
}
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
marks[markNb], displayName,
(U32)srcSize, (U32)benchResult.cSize,
(unsigned)srcSize, (unsigned)benchResult.cSize,
ratioAccuracy, ratio,
benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT,
(double)benchResult.dSpeed / MB_UNIT);
}
decompressionCompleted = BMK_isCompleted_TimedFn(timeStateDecompress);
}
markNb = (markNb+1) % NB_MARKS;
} /* while (!(compressionCompleted && decompressionCompleted)) */
/* CRC Checking */
@ -707,12 +491,13 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) {
size_t u;
DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n",
displayName, (unsigned)crcOrig, (unsigned)crcCheck);
for (u=0; u<srcSize; u++) {
if (((const BYTE*)srcBuffer)[u] != resultBuffer[u]) {
U32 segNb, bNb, pos;
unsigned segNb, bNb, pos;
size_t bacc = 0;
DISPLAY("Decoding error at pos %u ", (U32)u);
DISPLAY("Decoding error at pos %u ", (unsigned)u);
for (segNb = 0; segNb < nbBlocks; segNb++) {
if (bacc + srcSizes[segNb] > u) break;
bacc += srcSizes[segNb];
@ -883,7 +668,7 @@ static BMK_benchOutcome_t BMK_benchCLevel(const void* srcBuffer, size_t benchedS
if (displayLevel == 1 && !adv->additionalParam) /* --quiet mode */
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n",
ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING,
(U32)benchedSize, adv->nbSeconds, (U32)(adv->blockSize>>10));
(unsigned)benchedSize, adv->nbSeconds, (unsigned)(adv->blockSize>>10));
return BMK_benchMemAdvanced(srcBuffer, benchedSize,
NULL, 0,
@ -1015,6 +800,11 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
/* Load dictionary */
if (dictFileName != NULL) {
U64 const dictFileSize = UTIL_getFileSize(dictFileName);
if (dictFileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAYLEVEL(1, "error loading %s : %s \n", dictFileName, strerror(errno));
free(fileSizes);
RETURN_ERROR(9, BMK_benchOutcome_t, "benchmark aborted");
}
if (dictFileSize > 64 MB) {
free(fileSizes);
RETURN_ERROR(10, BMK_benchOutcome_t, "dictionary file %s too large", dictFileName);
@ -1024,7 +814,7 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
if (dictBuffer==NULL) {
free(fileSizes);
RETURN_ERROR(11, BMK_benchOutcome_t, "not enough memory for dictionary (%u bytes)",
(U32)dictBufferSize);
(unsigned)dictBufferSize);
}
{ int const errorCode = BMK_loadFiles(dictBuffer, dictBufferSize,
@ -1040,7 +830,7 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
if (benchedSize < totalSizeToLoad)
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
DISPLAY("Not enough memory; testing %u MB only...\n", (unsigned)(benchedSize >> 20));
srcBuffer = benchedSize ? malloc(benchedSize) : NULL;
if (!srcBuffer) {

119
sys/contrib/zstd/programs/bench.h → sys/contrib/zstd/programs/benchzstd.h
View File

@ -8,12 +8,18 @@
* You may select, at your option, one of the above-listed licenses.
*/
/* benchzstd :
* benchmark Zstandard compression / decompression
* over a set of files or buffers
* and display progress result and final summary
*/
#if defined (__cplusplus)
extern "C" {
#endif
#ifndef BENCH_H_121279284357
#define BENCH_H_121279284357
#ifndef BENCH_ZSTD_H_3242387
#define BENCH_ZSTD_H_3242387
/* === Dependencies === */
#include <stddef.h> /* size_t */
@ -109,7 +115,7 @@ typedef struct {
unsigned ldmMinMatch; /* below: parameters for long distance matching, see zstd.1.md */
unsigned ldmHashLog;
unsigned ldmBucketSizeLog;
unsigned ldmHashEveryLog;
unsigned ldmHashRateLog;
} BMK_advancedParams_t;
/* returns default parameters used by nonAdvanced functions */
@ -142,9 +148,9 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
* .cMem : memory budget required for the compression context
*/
BMK_benchOutcome_t BMK_syntheticTest(
int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv);
int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv);
@ -181,6 +187,7 @@ BMK_benchOutcome_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
const void* dictBuffer, size_t dictBufferSize,
int displayLevel, const char* displayName);
/* BMK_benchMemAdvanced() : same as BMK_benchMem()
* with following additional options :
* dstBuffer - destination buffer to write compressed output in, NULL if none provided.
@ -197,106 +204,8 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
/* ==== Benchmarking any function, iterated on a set of blocks ==== */
typedef struct {
unsigned long long nanoSecPerRun; /* time per iteration */
size_t sumOfReturn; /* sum of return values */
} BMK_runTime_t;
VARIANT_ERROR_RESULT(BMK_runTime_t, BMK_runOutcome_t);
/* check first if the return structure represents an error or a valid result */
int BMK_isSuccessful_runOutcome(BMK_runOutcome_t outcome);
/* extract result from variant type.
* note : this function will abort() program execution if result is not valid
* check result validity first, by using BMK_isSuccessful_runOutcome()
*/
BMK_runTime_t BMK_extract_runTime(BMK_runOutcome_t outcome);
typedef size_t (*BMK_benchFn_t)(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* customPayload);
typedef size_t (*BMK_initFn_t)(void* initPayload);
/* BMK_benchFunction() :
* This function times the execution of 2 argument functions, benchFn and initFn */
/* benchFn - (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload)
* is run nbLoops times
* initFn - (*initFn)(initPayload) is run once per benchmark, at the beginning.
* This argument can be NULL, in which case nothing is run.
* blockCount - number of blocks. Size of all array parameters : srcBuffers, srcSizes, dstBuffers, dstCapacities, blockResults
* srcBuffers - an array of buffers to be operated on by benchFn
* srcSizes - an array of the sizes of above buffers
* dstBuffers - an array of buffers to be written into by benchFn
* dstCapacities - an array of the capacities of above buffers
* blockResults - Optional: store the return value of benchFn for each block. Use NULL if this result is not requested.
* nbLoops - defines number of times benchFn is run.
* @return: a variant, which express either an error, or can generate a valid BMK_runTime_t result.
* Use BMK_isSuccessful_runOutcome() to check if function was successful.
* If yes, extract the result with BMK_extract_runTime(),
* it will contain :
* .sumOfReturn : the sum of all return values of benchFn through all of blocks
* .nanoSecPerRun : time per run of benchFn + (time for initFn / nbLoops)
* .sumOfReturn is generally intended for functions which return a # of bytes written into dstBuffer,
* in which case, this value will be the total amount of bytes written into dstBuffer.
*/
BMK_runOutcome_t BMK_benchFunction(
BMK_benchFn_t benchFn, void* benchPayload,
BMK_initFn_t initFn, void* initPayload,
size_t blockCount,
const void *const * srcBuffers, const size_t* srcSizes,
void *const * dstBuffers, const size_t* dstCapacities,
size_t* blockResults,
unsigned nbLoops);
/* ==== Benchmark any function, providing intermediate results ==== */
/* state information tracking benchmark session */
typedef struct BMK_timedFnState_s BMK_timedFnState_t;
/* BMK_createTimedFnState() and BMK_resetTimedFnState() :
* Create/Set BMK_timedFnState_t for next benchmark session,
* which shall last a minimum of total_ms milliseconds,
* producing intermediate results, paced at interval of (approximately) run_ms.
*/
BMK_timedFnState_t* BMK_createTimedFnState(unsigned total_ms, unsigned run_ms);
void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms, unsigned run_ms);
void BMK_freeTimedFnState(BMK_timedFnState_t* state);
/* Tells if duration of all benchmark runs has exceeded total_ms
*/
int BMK_isCompleted_TimedFn(const BMK_timedFnState_t* timedFnState);
/* BMK_benchTimedFn() :
* Similar to BMK_benchFunction(), most arguments being identical.
* Automatically determines `nbLoops` so that each result is regularly produced at interval of about run_ms.
* Note : minimum `nbLoops` is 1, therefore a run may last more than run_ms, and possibly even more than total_ms.
* Usage - initialize timedFnState, select benchmark duration (total_ms) and each measurement duration (run_ms)
* call BMK_benchTimedFn() repetitively, each measurement is supposed to last about run_ms
* Check if total time budget is spent or exceeded, using BMK_isCompleted_TimedFn()
*/
BMK_runOutcome_t BMK_benchTimedFn(
BMK_timedFnState_t* timedFnState,
BMK_benchFn_t benchFn, void* benchPayload,
BMK_initFn_t initFn, void* initPayload,
size_t blockCount,
const void *const * srcBlockBuffers, const size_t* srcBlockSizes,
void *const * dstBlockBuffers, const size_t* dstBlockCapacities,
size_t* blockResults);
#endif /* BENCH_H_121279284357 */
#endif /* BENCH_ZSTD_H_3242387 */
#if defined (__cplusplus)
}

14
sys/contrib/zstd/programs/datagen.c
View File

@ -81,18 +81,18 @@ static BYTE RDG_genChar(U32* seed, const BYTE* ldt)
}
static U32 RDG_rand15Bits (unsigned* seedPtr)
static U32 RDG_rand15Bits (U32* seedPtr)
{
return RDG_rand(seedPtr) & 0x7FFF;
}
static U32 RDG_randLength(unsigned* seedPtr)
static U32 RDG_randLength(U32* seedPtr)
{
if (RDG_rand(seedPtr) & 7) return (RDG_rand(seedPtr) & 0xF); /* small length */
return (RDG_rand(seedPtr) & 0x1FF) + 0xF;
}
static void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, const BYTE* ldt, unsigned* seedPtr)
static void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, const BYTE* ldt, U32* seedPtr)
{
BYTE* const buffPtr = (BYTE*)buffer;
U32 const matchProba32 = (U32)(32768 * matchProba);
@ -141,16 +141,18 @@ static void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, doubl
void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed)
{
U32 seed32 = seed;
BYTE ldt[LTSIZE];
memset(ldt, '0', sizeof(ldt)); /* yes, character '0', this is intentional */
if (litProba<=0.0) litProba = matchProba / 4.5;
RDG_fillLiteralDistrib(ldt, litProba);
RDG_genBlock(buffer, size, 0, matchProba, ldt, &seed);
RDG_genBlock(buffer, size, 0, matchProba, ldt, &seed32);
}
void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed)
{
U32 seed32 = seed;
size_t const stdBlockSize = 128 KB;
size_t const stdDictSize = 32 KB;
BYTE* const buff = (BYTE*)malloc(stdDictSize + stdBlockSize);
@ -165,12 +167,12 @@ void RDG_genStdout(unsigned long long size, double matchProba, double litProba,
SET_BINARY_MODE(stdout);
/* Generate initial dict */
RDG_genBlock(buff, stdDictSize, 0, matchProba, ldt, &seed);
RDG_genBlock(buff, stdDictSize, 0, matchProba, ldt, &seed32);
/* Generate compressible data */
while (total < size) {
size_t const genBlockSize = (size_t) (MIN (stdBlockSize, size-total));
RDG_genBlock(buff, stdDictSize+stdBlockSize, stdDictSize, matchProba, ldt, &seed);
RDG_genBlock(buff, stdDictSize+stdBlockSize, stdDictSize, matchProba, ldt, &seed32);
total += genBlockSize;
{ size_t const unused = fwrite(buff, 1, genBlockSize, stdout); (void)unused; }
/* update dict */

6
sys/contrib/zstd/programs/dibio.c
View File

@ -139,7 +139,7 @@ static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr,
}
DISPLAYLEVEL(2, "\r%79s\r", "");
*bufferSizePtr = pos;
DISPLAYLEVEL(4, "loaded : %u KB \n", (U32)(pos >> 10))
DISPLAYLEVEL(4, "loaded : %u KB \n", (unsigned)(pos >> 10))
return nbLoadedChunks;
}
@ -249,7 +249,7 @@ static fileStats DiB_fileStats(const char** fileNamesTable, unsigned nbFiles, si
fs.oneSampleTooLarge |= (chunkSize > 2*SAMPLESIZE_MAX);
fs.nbSamples += nbSamples;
}
DISPLAYLEVEL(4, "Preparing to load : %u KB \n", (U32)(fs.totalSizeToLoad >> 10));
DISPLAYLEVEL(4, "Preparing to load : %u KB \n", (unsigned)(fs.totalSizeToLoad >> 10));
return fs;
}
@ -358,7 +358,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
goto _cleanup;
}
/* save dict */
DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName);
DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (unsigned)dictSize, dictFileName);
DiB_saveDict(dictFileName, dictBuffer, dictSize);
}

262
sys/contrib/zstd/programs/fileio.c
View File

@ -88,10 +88,10 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
static const U64 g_refreshRate = SEC_TO_MICRO / 6;
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
#define READY_FOR_UPDATE() (UTIL_clockSpanMicro(g_displayClock) > g_refreshRate)
#define READY_FOR_UPDATE() (!g_noProgress && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate)
#define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); }
#define DISPLAYUPDATE(l, ...) { \
if (g_displayLevel>=l) { \
if (g_displayLevel>=l && !g_noProgress) { \
if (READY_FOR_UPDATE() || (g_displayLevel>=4)) { \
DELAY_NEXT_UPDATE(); \
DISPLAY(__VA_ARGS__); \
@ -270,18 +270,18 @@ void FIO_addAbortHandler()
static FIO_compressionType_t g_compressionType = FIO_zstdCompression;
void FIO_setCompressionType(FIO_compressionType_t compressionType) { g_compressionType = compressionType; }
static U32 g_overwrite = 0;
void FIO_overwriteMode(void) { g_overwrite=1; }
static U32 g_sparseFileSupport = 1; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
void FIO_overwriteMode(void) { g_overwrite = 1; }
static U32 g_sparseFileSupport = ZSTD_SPARSE_DEFAULT; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport = sparse; }
static U32 g_dictIDFlag = 1;
void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; }
static U32 g_checksumFlag = 1;
void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; }
static U32 g_removeSrcFile = 0;
void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); }
static U32 g_memLimit = 0;
static unsigned g_memLimit = 0;
void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; }
static U32 g_nbWorkers = 1;
static unsigned g_nbWorkers = 1;
void FIO_setNbWorkers(unsigned nbWorkers) {
#ifndef ZSTD_MULTITHREAD
if (nbWorkers > 0) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n");
@ -295,7 +295,7 @@ void FIO_setBlockSize(unsigned blockSize) {
g_blockSize = blockSize;
}
#define FIO_OVERLAP_LOG_NOTSET 9999
static U32 g_overlapLog = FIO_OVERLAP_LOG_NOTSET;
static unsigned g_overlapLog = FIO_OVERLAP_LOG_NOTSET;
void FIO_setOverlapLog(unsigned overlapLog){
if (overlapLog && g_nbWorkers==0)
DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n");
@ -307,6 +307,12 @@ void FIO_setAdaptiveMode(unsigned adapt) {
EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n");
g_adaptiveMode = adapt;
}
static U32 g_rsyncable = 0;
void FIO_setRsyncable(unsigned rsyncable) {
if ((rsyncable>0) && (g_nbWorkers==0))
EXM_THROW(1, "Rsyncable mode is not compatible with single thread mode \n");
g_rsyncable = rsyncable;
}
static int g_minAdaptLevel = -50; /* initializing this value requires a constant, so ZSTD_minCLevel() doesn't work */
void FIO_setAdaptMin(int minCLevel)
{
@ -340,9 +346,13 @@ void FIO_setLdmBucketSizeLog(unsigned ldmBucketSizeLog) {
g_ldmBucketSizeLog = ldmBucketSizeLog;
}
static U32 g_ldmHashEveryLog = FIO_LDM_PARAM_NOTSET;
void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog) {
g_ldmHashEveryLog = ldmHashEveryLog;
static U32 g_ldmHashRateLog = FIO_LDM_PARAM_NOTSET;
void FIO_setLdmHashRateLog(unsigned ldmHashRateLog) {
g_ldmHashRateLog = ldmHashRateLog;
}
static U32 g_noProgress = 0;
void FIO_setNoProgress(unsigned noProgress) {
g_noProgress = noProgress;
}
@ -355,7 +365,7 @@ void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog) {
static int FIO_remove(const char* path)
{
if (!UTIL_isRegularFile(path)) {
DISPLAYLEVEL(2, "zstd: Refusing to remove non-regular file %s\n", path);
DISPLAYLEVEL(2, "zstd: Refusing to remove non-regular file %s \n", path);
return 0;
}
#if defined(_WIN32) || defined(WIN32)
@ -373,11 +383,17 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
{
assert(srcFileName != NULL);
if (!strcmp (srcFileName, stdinmark)) {
DISPLAYLEVEL(4,"Using stdin for input\n");
DISPLAYLEVEL(4,"Using stdin for input \n");
SET_BINARY_MODE(stdin);
return stdin;
}
if (!UTIL_fileExist(srcFileName)) {
DISPLAYLEVEL(1, "zstd: can't stat %s : %s -- ignored \n",
srcFileName, strerror(errno));
return NULL;
}
if (!UTIL_isRegularFile(srcFileName)) {
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
srcFileName);
@ -394,30 +410,54 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
/** FIO_openDstFile() :
* condition : `dstFileName` must be non-NULL.
* @result : FILE* to `dstFileName`, or NULL if it fails */
static FILE* FIO_openDstFile(const char* dstFileName)
static FILE* FIO_openDstFile(const char* srcFileName, const char* dstFileName)
{
assert(dstFileName != NULL);
if (!strcmp (dstFileName, stdoutmark)) {
DISPLAYLEVEL(4,"Using stdout for output\n");
DISPLAYLEVEL(4,"Using stdout for output \n");
SET_BINARY_MODE(stdout);
if (g_sparseFileSupport==1) {
if (g_sparseFileSupport == 1) {
g_sparseFileSupport = 0;
DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
}
return stdout;
}
/* ensure dst is not the same file as src */
if (srcFileName != NULL) {
#ifdef _MSC_VER
/* note : Visual does not support file identification by inode.
* The following work-around is limited to detecting exact name repetition only,
* aka `filename` is considered different from `subdir/../filename` */
if (!strcmp(srcFileName, dstFileName)) {
DISPLAYLEVEL(1, "zstd: Refusing to open a output file which will overwrite the input file \n");
return NULL;
}
#else
stat_t srcStat;
stat_t dstStat;
if (UTIL_getFileStat(srcFileName, &srcStat)
&& UTIL_getFileStat(dstFileName, &dstStat)) {
if (srcStat.st_dev == dstStat.st_dev
&& srcStat.st_ino == dstStat.st_ino) {
DISPLAYLEVEL(1, "zstd: Refusing to open a output file which will overwrite the input file \n");
return NULL;
}
}
#endif
}
if (g_sparseFileSupport == 1) {
g_sparseFileSupport = ZSTD_SPARSE_DEFAULT;
}
if (UTIL_isRegularFile(dstFileName)) {
FILE* fCheck;
if (!strcmp(dstFileName, nulmark)) {
EXM_THROW(40, "%s is unexpectedly a regular file", dstFileName);
}
/* Check if destination file already exists */
fCheck = fopen( dstFileName, "rb" );
FILE* const fCheck = fopen( dstFileName, "rb" );
if (!strcmp(dstFileName, nulmark)) {
EXM_THROW(40, "%s is unexpectedly categorized as a regular file",
dstFileName);
}
if (fCheck != NULL) { /* dst file exists, authorization prompt */
fclose(fCheck);
if (!g_overwrite) {
@ -467,6 +507,7 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
fileHandle = fopen(fileName, "rb");
if (fileHandle==NULL) EXM_THROW(31, "%s: %s", fileName, strerror(errno));
fileSize = UTIL_getFileSize(fileName);
if (fileSize > DICTSIZE_MAX) {
EXM_THROW(32, "Dictionary file %s is too large (> %u MB)",
@ -475,8 +516,9 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
*bufferPtr = malloc((size_t)fileSize);
if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno));
{ size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
if (readSize!=fileSize)
EXM_THROW(35, "Error reading dictionary file %s", fileName);
if (readSize != fileSize)
EXM_THROW(35, "Error reading dictionary file %s : %s",
fileName, strerror(errno));
}
fclose(fileHandle);
return (size_t)fileSize;
@ -506,7 +548,8 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
DISPLAYLEVEL(6, "FIO_createCResources \n");
ress.cctx = ZSTD_createCCtx();
if (ress.cctx == NULL)
EXM_THROW(30, "allocation error : can't create ZSTD_CCtx");
EXM_THROW(30, "allocation error (%s): can't create ZSTD_CCtx",
strerror(errno));
ress.srcBufferSize = ZSTD_CStreamInSize();
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = ZSTD_CStreamOutSize();
@ -523,40 +566,39 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
if (g_adaptiveMode && !g_ldmFlag && !comprParams.windowLog)
comprParams.windowLog = ADAPT_WINDOWLOG_DEFAULT;
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) ); /* always enable content size when available (note: supposed to be default) */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_contentSizeFlag, 1) ); /* always enable content size when available (note: supposed to be default) */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_dictIDFlag, g_dictIDFlag) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, g_checksumFlag) );
/* compression level */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)cLevel) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) );
/* long distance matching */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashLog, g_ldmHashLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableLongDistanceMatching, g_ldmFlag) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashLog, g_ldmHashLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmMinMatch, g_ldmMinMatch) );
if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) {
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmBucketSizeLog, g_ldmBucketSizeLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmBucketSizeLog, g_ldmBucketSizeLog) );
}
if (g_ldmHashEveryLog != FIO_LDM_PARAM_NOTSET) {
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog) );
if (g_ldmHashRateLog != FIO_LDM_PARAM_NOTSET) {
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, g_ldmHashRateLog) );
}
/* compression parameters */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams.windowLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams.chainLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams.hashLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams.searchLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams.searchLength) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams.targetLength) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams.strategy) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, comprParams.windowLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_chainLog, comprParams.chainLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_hashLog, comprParams.hashLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, comprParams.searchLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, comprParams.minMatch) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, comprParams.targetLength) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, comprParams.strategy) );
/* multi-threading */
#ifdef ZSTD_MULTITHREAD
DISPLAYLEVEL(5,"set nb workers = %u \n", g_nbWorkers);
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbWorkers, g_nbWorkers) );
if ( (g_overlapLog == FIO_OVERLAP_LOG_NOTSET)
&& (cLevel == ZSTD_maxCLevel()) )
g_overlapLog = 9; /* full overlap */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, g_nbWorkers) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, g_blockSize) );
if (g_overlapLog != FIO_OVERLAP_LOG_NOTSET) {
DISPLAYLEVEL(3,"set overlapLog = %u \n", g_overlapLog);
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_overlapSizeLog, g_overlapLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, g_overlapLog) );
}
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, g_rsyncable) );
#endif
/* dictionary */
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* set the value temporarily for dictionary loading, to adapt compression parameters */
@ -627,11 +669,11 @@ FIO_compressGzFrame(cRess_t* ress,
}
if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
(U32)(inFileSize>>20),
(unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100)
else
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
(U32)(inFileSize>>20), (U32)(srcFileSize>>20),
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100);
}
@ -640,7 +682,7 @@ FIO_compressGzFrame(cRess_t* ress,
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
if (decompBytes) {
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes)
EXM_THROW(75, "Write error : cannot write to output file");
EXM_THROW(75, "Write error : %s", strerror(errno));
outFileSize += decompBytes;
strm.next_out = (Bytef*)ress->dstBuffer;
strm.avail_out = (uInt)ress->dstBufferSize;
@ -708,18 +750,18 @@ FIO_compressLzmaFrame(cRess_t* ress,
{ size_t const compBytes = ress->dstBufferSize - strm.avail_out;
if (compBytes) {
if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes)
EXM_THROW(73, "Write error : cannot write to output file");
EXM_THROW(73, "Write error : %s", strerror(errno));
outFileSize += compBytes;
strm.next_out = (BYTE*)ress->dstBuffer;
strm.avail_out = ress->dstBufferSize;
} }
if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
(U32)(inFileSize>>20),
(unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100)
else
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
(U32)(inFileSize>>20), (U32)(srcFileSize>>20),
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100);
if (ret == LZMA_STREAM_END) break;
}
@ -773,7 +815,7 @@ FIO_compressLz4Frame(cRess_t* ress,
EXM_THROW(33, "File header generation failed : %s",
LZ4F_getErrorName(headerSize));
if (fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile) != headerSize)
EXM_THROW(34, "Write error : cannot write header");
EXM_THROW(34, "Write error : %s (cannot write header)", strerror(errno));
outFileSize += headerSize;
/* Read first block */
@ -782,26 +824,28 @@ FIO_compressLz4Frame(cRess_t* ress,
/* Main Loop */
while (readSize>0) {
size_t outSize;
/* Compress Block */
outSize = LZ4F_compressUpdate(ctx, ress->dstBuffer, ress->dstBufferSize, ress->srcBuffer, readSize, NULL);
size_t const outSize = LZ4F_compressUpdate(ctx,
ress->dstBuffer, ress->dstBufferSize,
ress->srcBuffer, readSize, NULL);
if (LZ4F_isError(outSize))
EXM_THROW(35, "zstd: %s: lz4 compression failed : %s",
srcFileName, LZ4F_getErrorName(outSize));
outFileSize += outSize;
if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
(U32)(inFileSize>>20),
(unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100)
else
} else {
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
(U32)(inFileSize>>20), (U32)(srcFileSize>>20),
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100);
}
/* Write Block */
{ size_t const sizeCheck = fwrite(ress->dstBuffer, 1, outSize, ress->dstFile);
if (sizeCheck!=outSize) EXM_THROW(36, "Write error : cannot write compressed block"); }
{ size_t const sizeCheck = fwrite(ress->dstBuffer, 1, outSize, ress->dstFile);
if (sizeCheck != outSize)
EXM_THROW(36, "Write error : %s", strerror(errno));
}
/* Read next block */
readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile);
@ -815,8 +859,11 @@ FIO_compressLz4Frame(cRess_t* ress,
EXM_THROW(38, "zstd: %s: lz4 end of file generation failed : %s",
srcFileName, LZ4F_getErrorName(headerSize));
{ size_t const sizeCheck = fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile);
if (sizeCheck!=headerSize) EXM_THROW(39, "Write error : cannot write end of stream"); }
{ size_t const sizeCheck = fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile);
if (sizeCheck != headerSize)
EXM_THROW(39, "Write error : %s (cannot write end of stream)",
strerror(errno));
}
outFileSize += headerSize;
}
@ -863,7 +910,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
/* Fill input Buffer */
size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile);
ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 };
DISPLAYLEVEL(6, "fread %u bytes from source \n", (U32)inSize);
DISPLAYLEVEL(6, "fread %u bytes from source \n", (unsigned)inSize);
*readsize += inSize;
if ((inSize == 0) || (*readsize == fileSize))
@ -876,7 +923,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
size_t const oldIPos = inBuff.pos;
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
size_t const toFlushNow = ZSTD_toFlushNow(ress.cctx);
CHECK_V(stillToFlush, ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, directive));
CHECK_V(stillToFlush, ZSTD_compressStream2(ress.cctx, &outBuff, &inBuff, directive));
/* count stats */
inputPresented++;
@ -885,11 +932,12 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
/* Write compressed stream */
DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => input pos(%u)<=(%u)size ; output generated %u bytes \n",
(U32)directive, (U32)inBuff.pos, (U32)inBuff.size, (U32)outBuff.pos);
(unsigned)directive, (unsigned)inBuff.pos, (unsigned)inBuff.size, (unsigned)outBuff.pos);
if (outBuff.pos) {
size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
if (sizeCheck != outBuff.pos)
EXM_THROW(25, "Write error : cannot write compressed block");
EXM_THROW(25, "Write error : %s (cannot write compressed block)",
strerror(errno));
compressedfilesize += outBuff.pos;
}
@ -902,14 +950,14 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
if (g_displayLevel >= 3) {
DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%% ",
compressionLevel,
(U32)((zfp.ingested - zfp.consumed) >> 20),
(U32)(zfp.consumed >> 20),
(U32)(zfp.produced >> 20),
(unsigned)((zfp.ingested - zfp.consumed) >> 20),
(unsigned)(zfp.consumed >> 20),
(unsigned)(zfp.produced >> 20),
cShare );
} else { /* summarized notifications if == 2; */
DISPLAYLEVEL(2, "\rRead : %u ", (U32)(zfp.consumed >> 20));
DISPLAYLEVEL(2, "\rRead : %u ", (unsigned)(zfp.consumed >> 20));
if (fileSize != UTIL_FILESIZE_UNKNOWN)
DISPLAYLEVEL(2, "/ %u ", (U32)(fileSize >> 20));
DISPLAYLEVEL(2, "/ %u ", (unsigned)(fileSize >> 20));
DISPLAYLEVEL(2, "MB ==> %2.f%% ", cShare);
DELAY_NEXT_UPDATE();
}
@ -965,8 +1013,8 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
assert(inputPresented > 0);
DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n",
inputBlocked, inputPresented, (double)inputBlocked/inputPresented*100,
(U32)newlyIngested, (U32)newlyConsumed,
(U32)newlyFlushed, (U32)newlyProduced);
(unsigned)newlyIngested, (unsigned)newlyConsumed,
(unsigned)newlyFlushed, (unsigned)newlyProduced);
if ( (inputBlocked > inputPresented / 8) /* input is waiting often, because input buffers is full : compression or output too slow */
&& (newlyFlushed * 33 / 32 > newlyProduced) /* flush everything that is produced */
&& (newlyIngested * 33 / 32 > newlyConsumed) /* input speed as fast or faster than compression speed */
@ -986,14 +1034,14 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
if (compressionLevel > g_maxAdaptLevel) compressionLevel = g_maxAdaptLevel;
compressionLevel += (compressionLevel == 0); /* skip 0 */
ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel);
ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
}
if (speedChange == faster) {
DISPLAYLEVEL(6, "faster speed , lighter compression \n")
compressionLevel --;
if (compressionLevel < g_minAdaptLevel) compressionLevel = g_minAdaptLevel;
compressionLevel -= (compressionLevel == 0); /* skip 0 */
ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel);
ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
}
speedChange = noChange;
@ -1028,7 +1076,7 @@ FIO_compressFilename_internal(cRess_t ress,
U64 readsize = 0;
U64 compressedfilesize = 0;
U64 const fileSize = UTIL_getFileSize(srcFileName);
DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (U32)fileSize);
DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (unsigned)fileSize);
/* compression format selection */
switch (g_compressionType) {
@ -1105,7 +1153,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
if (ress.dstFile == NULL) {
closeDstFile = 1;
DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName);
ress.dstFile = FIO_openDstFile(dstFileName);
ress.dstFile = FIO_openDstFile(srcFileName, dstFileName);
if (ress.dstFile==NULL) return 1; /* could not open dstFileName */
/* Must only be added after FIO_openDstFile() succeeds.
* Otherwise we may delete the destination file if it already exists,
@ -1255,7 +1303,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
assert(outFileName != NULL || suffix != NULL);
if (outFileName != NULL) { /* output into a single destination (stdout typically) */
ress.dstFile = FIO_openDstFile(outFileName);
ress.dstFile = FIO_openDstFile(NULL, outFileName);
if (ress.dstFile == NULL) { /* could not open outFileName */
error = 1;
} else {
@ -1263,7 +1311,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
for (u=0; u<nbFiles; u++)
error |= FIO_compressFilename_srcFile(ress, outFileName, inFileNamesTable[u], compressionLevel);
if (fclose(ress.dstFile))
EXM_THROW(29, "Write error : cannot properly close %s", outFileName);
EXM_THROW(29, "Write error (%s) : cannot properly close %s",
strerror(errno), outFileName);
ress.dstFile = NULL;
}
} else {
@ -1304,8 +1353,9 @@ static dRess_t FIO_createDResources(const char* dictFileName)
/* Allocation */
ress.dctx = ZSTD_createDStream();
if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream");
CHECK( ZSTD_setDStreamParameter(ress.dctx, DStream_p_maxWindowSize, g_memLimit) );
if (ress.dctx==NULL)
EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno));
CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, g_memLimit) );
ress.srcBufferSize = ZSTD_DStreamInSize();
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = ZSTD_DStreamOutSize();
@ -1343,14 +1393,17 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi
if (!g_sparseFileSupport) { /* normal write */
size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block");
if (sizeCheck != bufferSize)
EXM_THROW(70, "Write error : %s (cannot write decoded block)",
strerror(errno));
return 0;
}
/* avoid int overflow */
if (storedSkips > 1 GB) {
int const seekResult = LONG_SEEK(file, 1 GB, SEEK_CUR);
if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)");
if (seekResult != 0)
EXM_THROW(71, "1 GB skip error (sparse file support)");
storedSkips -= 1 GB;
}
@ -1401,12 +1454,14 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi
static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
{
if (storedSkips-->0) { /* implies g_sparseFileSupport>0 */
int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)");
if (storedSkips>0) {
assert(g_sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */
if (LONG_SEEK(file, storedSkips-1, SEEK_CUR) != 0)
EXM_THROW(69, "Final skip error (sparse file support)");
/* last zero must be explicitly written,
* so that skipped ones get implicitly translated as zero by FS */
{ const char lastZeroByte[1] = { 0 };
size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file);
if (sizeCheck != 1)
if (fwrite(lastZeroByte, 1, 1, file) != 1)
EXM_THROW(69, "Write error : cannot write last zero");
} }
}
@ -1463,12 +1518,12 @@ static void FIO_zstdErrorHelp(dRess_t* ress, size_t err, char const* srcFileName
err = ZSTD_getFrameHeader(&header, ress->srcBuffer, ress->srcBufferLoaded);
if (err == 0) {
unsigned long long const windowSize = header.windowSize;
U32 const windowLog = FIO_highbit64(windowSize) + ((windowSize & (windowSize - 1)) != 0);
unsigned const windowLog = FIO_highbit64(windowSize) + ((windowSize & (windowSize - 1)) != 0);
assert(g_memLimit > 0);
DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u\n",
srcFileName, windowSize, g_memLimit);
if (windowLog <= ZSTD_WINDOWLOG_MAX) {
U32 const windowMB = (U32)((windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0));
unsigned const windowMB = (unsigned)((windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0));
assert(windowSize < (U64)(1ULL << 52)); /* ensure now overflow for windowMB */
DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB\n",
srcFileName, windowLog, windowMB);
@ -1520,7 +1575,7 @@ static unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips);
frameSize += outBuff.pos;
DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ",
srcFileName, (U32)((alreadyDecoded+frameSize)>>20) );
srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) );
if (inBuff.pos > 0) {
memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos);
@ -1871,7 +1926,7 @@ static int FIO_decompressDstFile(dRess_t ress, FILE* srcFile,
if (ress.dstFile == NULL) {
releaseDstFile = 1;
ress.dstFile = FIO_openDstFile(dstFileName);
ress.dstFile = FIO_openDstFile(srcFileName, dstFileName);
if (ress.dstFile==0) return 1;
/* Must only be added after FIO_openDstFile() succeeds.
@ -2025,7 +2080,7 @@ FIO_determineDstName(const char* srcFileName)
dfnbCapacity = sfnSize + 20;
dstFileNameBuffer = (char*)malloc(dfnbCapacity);
if (dstFileNameBuffer==NULL)
EXM_THROW(74, "not enough memory for dstFileName");
EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno));
}
/* return dst name == src name truncated from suffix */
@ -2048,12 +2103,13 @@ FIO_decompressMultipleFilenames(const char* srcNamesTable[], unsigned nbFiles,
if (outFileName) {
unsigned u;
ress.dstFile = FIO_openDstFile(outFileName);
ress.dstFile = FIO_openDstFile(NULL, outFileName);
if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName);
for (u=0; u<nbFiles; u++)
error |= FIO_decompressSrcFile(ress, outFileName, srcNamesTable[u]);
if (fclose(ress.dstFile))
EXM_THROW(72, "Write error : cannot properly close output file");
EXM_THROW(72, "Write error : %s : cannot properly close output file",
strerror(errno));
} else {
unsigned u;
for (u=0; u<nbFiles; u++) { /* create dstFileName */
@ -2103,7 +2159,7 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
for ( ; ; ) {
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
if (numBytesRead < ZSTD_frameHeaderSize_min) {
if (numBytesRead < ZSTD_FRAMEHEADERSIZE_MIN) {
if ( feof(srcFile)
&& (numBytesRead == 0)
&& (info->compressedSize > 0)
@ -2164,7 +2220,7 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
info->numActualFrames++;
}
/* Skippable frame */
else if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
else if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
U32 const frameSize = MEM_readLE32(headerBuffer + 4);
long const seek = (long)(8 + frameSize - numBytesRead);
ERROR_IF(LONG_SEEK(srcFile, seek, SEEK_CUR) != 0,
@ -2292,7 +2348,7 @@ FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel)
}
displayInfo(inFileName, &info, displayLevel);
*total = FIO_addFInfo(*total, info);
assert(error>=0 || error<=1);
assert(error == info_success || error == info_frame_error);
return error;
}
}
@ -2339,13 +2395,13 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
total.numSkippableFrames + total.numActualFrames,
total.numSkippableFrames,
compressedSizeUnit, unitStr,
checkString, total.nbFiles);
checkString, (unsigned)total.nbFiles);
} else {
DISPLAYOUT("%6d %5d %7.2f %2s %9.2f %2s %5.3f %5s %u files\n",
total.numSkippableFrames + total.numActualFrames,
total.numSkippableFrames,
compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
ratio, checkString, total.nbFiles);
ratio, checkString, (unsigned)total.nbFiles);
} }
return error;
}

4
sys/contrib/zstd/programs/fileio.h
View File

@ -56,7 +56,7 @@ void FIO_setChecksumFlag(unsigned checksumFlag);
void FIO_setDictIDFlag(unsigned dictIDFlag);
void FIO_setLdmBucketSizeLog(unsigned ldmBucketSizeLog);
void FIO_setLdmFlag(unsigned ldmFlag);
void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog);
void FIO_setLdmHashRateLog(unsigned ldmHashRateLog);
void FIO_setLdmHashLog(unsigned ldmHashLog);
void FIO_setLdmMinMatch(unsigned ldmMinMatch);
void FIO_setMemLimit(unsigned memLimit);
@ -65,6 +65,8 @@ void FIO_setNotificationLevel(unsigned level);
void FIO_setOverlapLog(unsigned overlapLog);
void FIO_setRemoveSrcFile(unsigned flag);
void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
void FIO_setRsyncable(unsigned rsyncable);
void FIO_setNoProgress(unsigned noProgress);
/*-*************************************

1
sys/contrib/zstd/programs/platform.h
View File

@ -26,6 +26,7 @@ extern "C" {
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before <io.h> and <windows.h> */
# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */
# endif
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#endif

673
sys/contrib/zstd/programs/util.c Normal file
View File

@ -0,0 +1,673 @@
/*
* Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#if defined (__cplusplus)
extern "C" {
#endif
/*-****************************************
* Dependencies
******************************************/
#include "util.h" /* note : ensure that platform.h is included first ! */
#include <errno.h>
#include <assert.h>
int UTIL_fileExist(const char* filename)
{
stat_t statbuf;
#if defined(_MSC_VER)
int const stat_error = _stat64(filename, &statbuf);
#else
int const stat_error = stat(filename, &statbuf);
#endif
return !stat_error;
}
int UTIL_isRegularFile(const char* infilename)
{
stat_t statbuf;
return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */
}
int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
{
int r;
#if defined(_MSC_VER)
r = _stat64(infilename, statbuf);
if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */
#else
r = stat(infilename, statbuf);
if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */
#endif
return 1;
}
int UTIL_setFileStat(const char *filename, stat_t *statbuf)
{
int res = 0;
struct utimbuf timebuf;
if (!UTIL_isRegularFile(filename))
return -1;
timebuf.actime = time(NULL);
timebuf.modtime = statbuf->st_mtime;
res += utime(filename, &timebuf); /* set access and modification times */
#if !defined(_WIN32)
res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */
#endif
res += chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */
errno = 0;
return -res; /* number of errors is returned */
}
U32 UTIL_isDirectory(const char* infilename)
{
int r;
stat_t statbuf;
#if defined(_MSC_VER)
r = _stat64(infilename, &statbuf);
if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
#else
r = stat(infilename, &statbuf);
if (!r && S_ISDIR(statbuf.st_mode)) return 1;
#endif
return 0;
}
U32 UTIL_isLink(const char* infilename)
{
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
#ifndef __STRICT_ANSI__
#if defined(_BSD_SOURCE) \
|| (defined(_XOPEN_SOURCE) && (_XOPEN_SOURCE >= 500)) \
|| (defined(_XOPEN_SOURCE) && defined(_XOPEN_SOURCE_EXTENDED)) \
|| (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) \
|| (defined(__APPLE__) && defined(__MACH__))
int r;
stat_t statbuf;
r = lstat(infilename, &statbuf);
if (!r && S_ISLNK(statbuf.st_mode)) return 1;
#endif
#endif
(void)infilename;
return 0;
}
U64 UTIL_getFileSize(const char* infilename)
{
if (!UTIL_isRegularFile(infilename)) return UTIL_FILESIZE_UNKNOWN;
{ int r;
#if defined(_MSC_VER)
struct __stat64 statbuf;
r = _stat64(infilename, &statbuf);
if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
#elif defined(__MINGW32__) && defined (__MSVCRT__)
struct _stati64 statbuf;
r = _stati64(infilename, &statbuf);
if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
#else
struct stat statbuf;
r = stat(infilename, &statbuf);
if (r || !S_ISREG(statbuf.st_mode)) return UTIL_FILESIZE_UNKNOWN;
#endif
return (U64)statbuf.st_size;
}
}
U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles)
{
U64 total = 0;
int error = 0;
unsigned n;
for (n=0; n<nbFiles; n++) {
U64 const size = UTIL_getFileSize(fileNamesTable[n]);
error |= (size == UTIL_FILESIZE_UNKNOWN);
total += size;
}
return error ? UTIL_FILESIZE_UNKNOWN : total;
}
#ifdef _WIN32
int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
{
char* path;
int dirLength, fnameLength, pathLength, nbFiles = 0;
WIN32_FIND_DATAA cFile;
HANDLE hFile;
dirLength = (int)strlen(dirName);
path = (char*) malloc(dirLength + 3);
if (!path) return 0;
memcpy(path, dirName, dirLength);
path[dirLength] = '\\';
path[dirLength+1] = '*';
path[dirLength+2] = 0;
hFile=FindFirstFileA(path, &cFile);
if (hFile == INVALID_HANDLE_VALUE) {
UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s'\n", dirName);
return 0;
}
free(path);
do {
fnameLength = (int)strlen(cFile.cFileName);
path = (char*) malloc(dirLength + fnameLength + 2);
if (!path) { FindClose(hFile); return 0; }
memcpy(path, dirName, dirLength);
path[dirLength] = '\\';
memcpy(path+dirLength+1, cFile.cFileName, fnameLength);
pathLength = dirLength+1+fnameLength;
path[pathLength] = 0;
if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
if ( strcmp (cFile.cFileName, "..") == 0
|| strcmp (cFile.cFileName, ".") == 0 )
continue;
/* Recursively call "UTIL_prepareFileList" with the new path. */
nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks);
if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
} else if ( (cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL)
|| (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE)
|| (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED) ) {
if (*bufStart + *pos + pathLength >= *bufEnd) {
ptrdiff_t const newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
*bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
*bufEnd = *bufStart + newListSize;
}
if (*bufStart + *pos + pathLength < *bufEnd) {
memcpy(*bufStart + *pos, path, pathLength+1 /* include final \0 */);
*pos += pathLength + 1;
nbFiles++;
}
}
free(path);
} while (FindNextFileA(hFile, &cFile));
FindClose(hFile);
return nbFiles;
}
#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
{
DIR *dir;
struct dirent *entry;
char* path;
int dirLength, fnameLength, pathLength, nbFiles = 0;
if (!(dir = opendir(dirName))) {
UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
return 0;
}
dirLength = (int)strlen(dirName);
errno = 0;
while ((entry = readdir(dir)) != NULL) {
if (strcmp (entry->d_name, "..") == 0 ||
strcmp (entry->d_name, ".") == 0) continue;
fnameLength = (int)strlen(entry->d_name);
path = (char*) malloc(dirLength + fnameLength + 2);
if (!path) { closedir(dir); return 0; }
memcpy(path, dirName, dirLength);
path[dirLength] = '/';
memcpy(path+dirLength+1, entry->d_name, fnameLength);
pathLength = dirLength+1+fnameLength;
path[pathLength] = 0;
if (!followLinks && UTIL_isLink(path)) {
UTIL_DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", path);
continue;
}
if (UTIL_isDirectory(path)) {
nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */
if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
} else {
if (*bufStart + *pos + pathLength >= *bufEnd) {
ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
*bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
*bufEnd = *bufStart + newListSize;
if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
}
if (*bufStart + *pos + pathLength < *bufEnd) {
memcpy(*bufStart + *pos, path, pathLength + 1); /* with final \0 */
*pos += pathLength + 1;
nbFiles++;
}
}
free(path);
errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */
}
if (errno != 0) {
UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s\n", dirName, strerror(errno));
free(*bufStart);
*bufStart = NULL;
}
closedir(dir);
return nbFiles;
}
#else
int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
{
(void)bufStart; (void)bufEnd; (void)pos; (void)followLinks;
UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
return 0;
}
#endif /* #ifdef _WIN32 */
/*
* UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
* and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb).
* After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer)
* In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called.
*/
const char**
UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
char** allocatedBuffer, unsigned* allocatedNamesNb,
int followLinks)
{
size_t pos;
unsigned i, nbFiles;
char* buf = (char*)malloc(LIST_SIZE_INCREASE);
char* bufend = buf + LIST_SIZE_INCREASE;
const char** fileTable;
if (!buf) return NULL;
for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) {
if (!UTIL_isDirectory(inputNames[i])) {
size_t const len = strlen(inputNames[i]);
if (buf + pos + len >= bufend) {
ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
buf = (char*)UTIL_realloc(buf, newListSize);
bufend = buf + newListSize;
if (!buf) return NULL;
}
if (buf + pos + len < bufend) {
memcpy(buf+pos, inputNames[i], len+1); /* with final \0 */
pos += len + 1;
nbFiles++;
}
} else {
nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks);
if (buf == NULL) return NULL;
} }
if (nbFiles == 0) { free(buf); return NULL; }
fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*));
if (!fileTable) { free(buf); return NULL; }
for (i=0, pos=0; i<nbFiles; i++) {
fileTable[i] = buf + pos;
pos += strlen(fileTable[i]) + 1;
}
if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
*allocatedBuffer = buf;
*allocatedNamesNb = nbFiles;
return fileTable;
}
/*-****************************************
* Console log
******************************************/
int g_utilDisplayLevel;
/*-****************************************
* Time functions
******************************************/
#if defined(_WIN32) /* Windows */
UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static LARGE_INTEGER ticksPerSecond;
static int init = 0;
if (!init) {
if (!QueryPerformanceFrequency(&ticksPerSecond))
UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
init = 1;
}
return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
}
U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static LARGE_INTEGER ticksPerSecond;
static int init = 0;
if (!init) {
if (!QueryPerformanceFrequency(&ticksPerSecond))
UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
init = 1;
}
return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
}
#elif defined(__APPLE__) && defined(__MACH__)
UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static mach_timebase_info_data_t rate;
static int init = 0;
if (!init) {
mach_timebase_info(&rate);
init = 1;
}
return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom))/1000ULL;
}
U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static mach_timebase_info_data_t rate;
static int init = 0;
if (!init) {
mach_timebase_info(&rate);
init = 1;
}
return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom);
}
#elif (PLATFORM_POSIX_VERSION >= 200112L) \
&& (defined(__UCLIBC__) \
|| (defined(__GLIBC__) \
&& ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) \
|| (__GLIBC__ > 2))))
UTIL_time_t UTIL_getTime(void)
{
UTIL_time_t time;
if (clock_gettime(CLOCK_MONOTONIC, &time))
UTIL_DISPLAYLEVEL(1, "ERROR: Failed to get time\n"); /* we could also exit() */
return time;
}
UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
{
UTIL_time_t diff;
if (end.tv_nsec < begin.tv_nsec) {
diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec;
diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec;
} else {
diff.tv_sec = end.tv_sec - begin.tv_sec;
diff.tv_nsec = end.tv_nsec - begin.tv_nsec;
}
return diff;
}
U64 UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
{
UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
U64 micro = 0;
micro += 1000000ULL * diff.tv_sec;
micro += diff.tv_nsec / 1000ULL;
return micro;
}
U64 UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
{
UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
U64 nano = 0;
nano += 1000000000ULL * diff.tv_sec;
nano += diff.tv_nsec;
return nano;
}
#else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
UTIL_time_t UTIL_getTime(void) { return clock(); }
U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
#endif
/* returns time span in microseconds */
U64 UTIL_clockSpanMicro(UTIL_time_t clockStart )
{
UTIL_time_t const clockEnd = UTIL_getTime();
return UTIL_getSpanTimeMicro(clockStart, clockEnd);
}
/* returns time span in microseconds */
U64 UTIL_clockSpanNano(UTIL_time_t clockStart )
{
UTIL_time_t const clockEnd = UTIL_getTime();
return UTIL_getSpanTimeNano(clockStart, clockEnd);
}
void UTIL_waitForNextTick(void)
{
UTIL_time_t const clockStart = UTIL_getTime();
UTIL_time_t clockEnd;
do {
clockEnd = UTIL_getTime();
} while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0);
}
/* count the number of physical cores */
#if defined(_WIN32) || defined(WIN32)
#include <windows.h>
typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD);
int UTIL_countPhysicalCores(void)
{
static int numPhysicalCores = 0;
if (numPhysicalCores != 0) return numPhysicalCores;
{ LPFN_GLPI glpi;
BOOL done = FALSE;
PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = NULL;
PSYSTEM_LOGICAL_PROCESSOR_INFORMATION ptr = NULL;
DWORD returnLength = 0;
size_t byteOffset = 0;
glpi = (LPFN_GLPI)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
"GetLogicalProcessorInformation");
if (glpi == NULL) {
goto failed;
}
while(!done) {
DWORD rc = glpi(buffer, &returnLength);
if (FALSE == rc) {
if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
if (buffer)
free(buffer);
buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(returnLength);
if (buffer == NULL) {
perror("zstd");
exit(1);
}
} else {
/* some other error */
goto failed;
}
} else {
done = TRUE;
}
}
ptr = buffer;
while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) {
if (ptr->Relationship == RelationProcessorCore) {
numPhysicalCores++;
}
ptr++;
byteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
}
free(buffer);
return numPhysicalCores;
}
failed:
/* try to fall back on GetSystemInfo */
{ SYSTEM_INFO sysinfo;
GetSystemInfo(&sysinfo);
numPhysicalCores = sysinfo.dwNumberOfProcessors;
if (numPhysicalCores == 0) numPhysicalCores = 1; /* just in case */
}
return numPhysicalCores;
}
#elif defined(__APPLE__)
#include <sys/sysctl.h>
/* Use apple-provided syscall
* see: man 3 sysctl */
int UTIL_countPhysicalCores(void)
{
static S32 numPhysicalCores = 0; /* apple specifies int32_t */
if (numPhysicalCores != 0) return numPhysicalCores;
{ size_t size = sizeof(S32);
int const ret = sysctlbyname("hw.physicalcpu", &numPhysicalCores, &size, NULL, 0);
if (ret != 0) {
if (errno == ENOENT) {
/* entry not present, fall back on 1 */
numPhysicalCores = 1;
} else {
perror("zstd: can't get number of physical cpus");
exit(1);
}
}
return numPhysicalCores;
}
}
#elif defined(__linux__)
/* parse /proc/cpuinfo
* siblings / cpu cores should give hyperthreading ratio
* otherwise fall back on sysconf */
int UTIL_countPhysicalCores(void)
{
static int numPhysicalCores = 0;
if (numPhysicalCores != 0) return numPhysicalCores;
numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
if (numPhysicalCores == -1) {
/* value not queryable, fall back on 1 */
return numPhysicalCores = 1;
}
/* try to determine if there's hyperthreading */
{ FILE* const cpuinfo = fopen("/proc/cpuinfo", "r");
#define BUF_SIZE 80
char buff[BUF_SIZE];
int siblings = 0;
int cpu_cores = 0;
int ratio = 1;
if (cpuinfo == NULL) {
/* fall back on the sysconf value */
return numPhysicalCores;
}
/* assume the cpu cores/siblings values will be constant across all
* present processors */
while (!feof(cpuinfo)) {
if (fgets(buff, BUF_SIZE, cpuinfo) != NULL) {
if (strncmp(buff, "siblings", 8) == 0) {
const char* const sep = strchr(buff, ':');
if (*sep == '\0') {
/* formatting was broken? */
goto failed;
}
siblings = atoi(sep + 1);
}
if (strncmp(buff, "cpu cores", 9) == 0) {
const char* const sep = strchr(buff, ':');
if (*sep == '\0') {
/* formatting was broken? */
goto failed;
}
cpu_cores = atoi(sep + 1);
}
} else if (ferror(cpuinfo)) {
/* fall back on the sysconf value */
goto failed;
}
}
if (siblings && cpu_cores) {
ratio = siblings / cpu_cores;
}
failed:
fclose(cpuinfo);
return numPhysicalCores = numPhysicalCores / ratio;
}
}
#elif defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)
/* Use apple-provided syscall
* see: man 3 sysctl */
int UTIL_countPhysicalCores(void)
{
static int numPhysicalCores = 0;
if (numPhysicalCores != 0) return numPhysicalCores;
numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
if (numPhysicalCores == -1) {
/* value not queryable, fall back on 1 */
return numPhysicalCores = 1;
}
return numPhysicalCores;
}
#else
int UTIL_countPhysicalCores(void)
{
/* assume 1 */
return 1;
}
#endif
#if defined (__cplusplus)
}
#endif

631
sys/contrib/zstd/programs/util.h
View File

@ -16,15 +16,13 @@ extern "C" {
#endif
/*-****************************************
* Dependencies
******************************************/
#include "platform.h" /* PLATFORM_POSIX_VERSION, ZSTD_NANOSLEEP_SUPPORT, ZSTD_SETPRIORITY_SUPPORT */
#include <stdlib.h> /* malloc */
#include <stdlib.h> /* malloc, realloc, free */
#include <stddef.h> /* size_t, ptrdiff_t */
#include <stdio.h> /* fprintf */
#include <string.h> /* strncmp */
#include <sys/types.h> /* stat, utime */
#include <sys/stat.h> /* stat, chmod */
#if defined(_MSC_VER)
@ -35,11 +33,10 @@ extern "C" {
# include <utime.h> /* utime */
#endif
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */
#include <errno.h>
#include "mem.h" /* U32, U64 */
/* ************************************************************
/*-************************************************************
* Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW
***************************************************************/
#if defined(_MSC_VER) && (_MSC_VER >= 1400)
@ -84,7 +81,7 @@ extern "C" {
#endif
/* *************************************
/*-*************************************
* Constants
***************************************/
#define LIST_SIZE_INCREASE (8*1024)
@ -110,7 +107,7 @@ extern "C" {
/*-****************************************
* Console log
******************************************/
static int g_utilDisplayLevel;
extern int g_utilDisplayLevel;
#define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } }
@ -119,61 +116,16 @@ static int g_utilDisplayLevel;
* Time functions
******************************************/
#if defined(_WIN32) /* Windows */
#define UTIL_TIME_INITIALIZER { { 0, 0 } }
typedef LARGE_INTEGER UTIL_time_t;
UTIL_STATIC UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static LARGE_INTEGER ticksPerSecond;
static int init = 0;
if (!init) {
if (!QueryPerformanceFrequency(&ticksPerSecond))
UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
init = 1;
}
return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
}
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static LARGE_INTEGER ticksPerSecond;
static int init = 0;
if (!init) {
if (!QueryPerformanceFrequency(&ticksPerSecond))
UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
init = 1;
}
return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
}
#elif defined(__APPLE__) && defined(__MACH__)
#include <mach/mach_time.h>
#define UTIL_TIME_INITIALIZER 0
typedef U64 UTIL_time_t;
UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static mach_timebase_info_data_t rate;
static int init = 0;
if (!init) {
mach_timebase_info(&rate);
init = 1;
}
return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom))/1000ULL;
}
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static mach_timebase_info_data_t rate;
static int init = 0;
if (!init) {
mach_timebase_info(&rate);
init = 1;
}
return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom);
}
#elif (PLATFORM_POSIX_VERSION >= 200112L) \
&& (defined(__UCLIBC__) \
|| (defined(__GLIBC__) \
@ -184,79 +136,27 @@ static int g_utilDisplayLevel;
typedef struct timespec UTIL_freq_t;
typedef struct timespec UTIL_time_t;
UTIL_STATIC UTIL_time_t UTIL_getTime(void)
{
UTIL_time_t time;
if (clock_gettime(CLOCK_MONOTONIC, &time))
UTIL_DISPLAYLEVEL(1, "ERROR: Failed to get time\n"); /* we could also exit() */
return time;
}
UTIL_STATIC UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
{
UTIL_time_t diff;
if (end.tv_nsec < begin.tv_nsec) {
diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec;
diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec;
} else {
diff.tv_sec = end.tv_sec - begin.tv_sec;
diff.tv_nsec = end.tv_nsec - begin.tv_nsec;
}
return diff;
}
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
{
UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
U64 micro = 0;
micro += 1000000ULL * diff.tv_sec;
micro += diff.tv_nsec / 1000ULL;
return micro;
}
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
{
UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
U64 nano = 0;
nano += 1000000000ULL * diff.tv_sec;
nano += diff.tv_nsec;
return nano;
}
UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end);
#else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
typedef clock_t UTIL_time_t;
#define UTIL_TIME_INITIALIZER 0
UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return clock(); }
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
#endif
UTIL_time_t UTIL_getTime(void);
U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd);
U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd);
#define SEC_TO_MICRO 1000000
/* returns time span in microseconds */
UTIL_STATIC U64 UTIL_clockSpanMicro(UTIL_time_t clockStart )
{
UTIL_time_t const clockEnd = UTIL_getTime();
return UTIL_getSpanTimeMicro(clockStart, clockEnd);
}
U64 UTIL_clockSpanMicro(UTIL_time_t clockStart);
/* returns time span in microseconds */
UTIL_STATIC U64 UTIL_clockSpanNano(UTIL_time_t clockStart )
{
UTIL_time_t const clockEnd = UTIL_getTime();
return UTIL_getSpanTimeNano(clockStart, clockEnd);
}
UTIL_STATIC void UTIL_waitForNextTick(void)
{
UTIL_time_t const clockStart = UTIL_getTime();
UTIL_time_t clockEnd;
do {
clockEnd = UTIL_getTime();
} while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0);
}
U64 UTIL_clockSpanNano(UTIL_time_t clockStart);
void UTIL_waitForNextTick(void);
/*-****************************************
* File functions
@ -269,129 +169,23 @@ UTIL_STATIC void UTIL_waitForNextTick(void)
#endif
UTIL_STATIC int UTIL_isRegularFile(const char* infilename);
UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf)
{
int res = 0;
struct utimbuf timebuf;
if (!UTIL_isRegularFile(filename))
return -1;
timebuf.actime = time(NULL);
timebuf.modtime = statbuf->st_mtime;
res += utime(filename, &timebuf); /* set access and modification times */
#if !defined(_WIN32)
res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */
#endif
res += chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */
errno = 0;
return -res; /* number of errors is returned */
}
UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
{
int r;
#if defined(_MSC_VER)
r = _stat64(infilename, statbuf);
if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */
#else
r = stat(infilename, statbuf);
if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */
#endif
return 1;
}
UTIL_STATIC int UTIL_isRegularFile(const char* infilename)
{
stat_t statbuf;
return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */
}
UTIL_STATIC U32 UTIL_isDirectory(const char* infilename)
{
int r;
stat_t statbuf;
#if defined(_MSC_VER)
r = _stat64(infilename, &statbuf);
if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
#else
r = stat(infilename, &statbuf);
if (!r && S_ISDIR(statbuf.st_mode)) return 1;
#endif
return 0;
}
UTIL_STATIC U32 UTIL_isLink(const char* infilename)
{
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
#ifndef __STRICT_ANSI__
#if defined(_BSD_SOURCE) \
|| (defined(_XOPEN_SOURCE) && (_XOPEN_SOURCE >= 500)) \
|| (defined(_XOPEN_SOURCE) && defined(_XOPEN_SOURCE_EXTENDED)) \
|| (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) \
|| (defined(__APPLE__) && defined(__MACH__))
int r;
stat_t statbuf;
r = lstat(infilename, &statbuf);
if (!r && S_ISLNK(statbuf.st_mode)) return 1;
#endif
#endif
(void)infilename;
return 0;
}
int UTIL_fileExist(const char* filename);
int UTIL_isRegularFile(const char* infilename);
int UTIL_setFileStat(const char* filename, stat_t* statbuf);
U32 UTIL_isDirectory(const char* infilename);
int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
U32 UTIL_isLink(const char* infilename);
#define UTIL_FILESIZE_UNKNOWN ((U64)(-1))
UTIL_STATIC U64 UTIL_getFileSize(const char* infilename)
{
if (!UTIL_isRegularFile(infilename)) return UTIL_FILESIZE_UNKNOWN;
{ int r;
#if defined(_MSC_VER)
struct __stat64 statbuf;
r = _stat64(infilename, &statbuf);
if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
#elif defined(__MINGW32__) && defined (__MSVCRT__)
struct _stati64 statbuf;
r = _stati64(infilename, &statbuf);
if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
#else
struct stat statbuf;
r = stat(infilename, &statbuf);
if (r || !S_ISREG(statbuf.st_mode)) return UTIL_FILESIZE_UNKNOWN;
#endif
return (U64)statbuf.st_size;
}
}
UTIL_STATIC U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles)
{
U64 total = 0;
int error = 0;
unsigned n;
for (n=0; n<nbFiles; n++) {
U64 const size = UTIL_getFileSize(fileNamesTable[n]);
error |= (size == UTIL_FILESIZE_UNKNOWN);
total += size;
}
return error ? UTIL_FILESIZE_UNKNOWN : total;
}
U64 UTIL_getFileSize(const char* infilename);
U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles);
/*
* A modified version of realloc().
* If UTIL_realloc() fails the original block is freed.
*/
UTIL_STATIC void *UTIL_realloc(void *ptr, size_t size)
UTIL_STATIC void* UTIL_realloc(void *ptr, size_t size)
{
void *newptr = realloc(ptr, size);
if (newptr) return newptr;
@ -399,143 +193,14 @@ UTIL_STATIC void *UTIL_realloc(void *ptr, size_t size)
return NULL;
}
int UTIL_prepareFileList(const char* dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks);
#ifdef _WIN32
# define UTIL_HAS_CREATEFILELIST
UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
{
char* path;
int dirLength, fnameLength, pathLength, nbFiles = 0;
WIN32_FIND_DATAA cFile;
HANDLE hFile;
dirLength = (int)strlen(dirName);
path = (char*) malloc(dirLength + 3);
if (!path) return 0;
memcpy(path, dirName, dirLength);
path[dirLength] = '\\';
path[dirLength+1] = '*';
path[dirLength+2] = 0;
hFile=FindFirstFileA(path, &cFile);
if (hFile == INVALID_HANDLE_VALUE) {
UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s'\n", dirName);
return 0;
}
free(path);
do {
fnameLength = (int)strlen(cFile.cFileName);
path = (char*) malloc(dirLength + fnameLength + 2);
if (!path) { FindClose(hFile); return 0; }
memcpy(path, dirName, dirLength);
path[dirLength] = '\\';
memcpy(path+dirLength+1, cFile.cFileName, fnameLength);
pathLength = dirLength+1+fnameLength;
path[pathLength] = 0;
if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
if (strcmp (cFile.cFileName, "..") == 0 ||
strcmp (cFile.cFileName, ".") == 0) continue;
nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */
if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
}
else if ((cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED)) {
if (*bufStart + *pos + pathLength >= *bufEnd) {
ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
*bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
*bufEnd = *bufStart + newListSize;
if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
}
if (*bufStart + *pos + pathLength < *bufEnd) {
strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos));
*pos += pathLength + 1;
nbFiles++;
}
}
free(path);
} while (FindNextFileA(hFile, &cFile));
FindClose(hFile);
return nbFiles;
}
#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
# define UTIL_HAS_CREATEFILELIST
# include <dirent.h> /* opendir, readdir */
# include <string.h> /* strerror, memcpy */
UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
{
DIR *dir;
struct dirent *entry;
char* path;
int dirLength, fnameLength, pathLength, nbFiles = 0;
if (!(dir = opendir(dirName))) {
UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
return 0;
}
dirLength = (int)strlen(dirName);
errno = 0;
while ((entry = readdir(dir)) != NULL) {
if (strcmp (entry->d_name, "..") == 0 ||
strcmp (entry->d_name, ".") == 0) continue;
fnameLength = (int)strlen(entry->d_name);
path = (char*) malloc(dirLength + fnameLength + 2);
if (!path) { closedir(dir); return 0; }
memcpy(path, dirName, dirLength);
path[dirLength] = '/';
memcpy(path+dirLength+1, entry->d_name, fnameLength);
pathLength = dirLength+1+fnameLength;
path[pathLength] = 0;
if (!followLinks && UTIL_isLink(path)) {
UTIL_DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", path);
continue;
}
if (UTIL_isDirectory(path)) {
nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */
if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
} else {
if (*bufStart + *pos + pathLength >= *bufEnd) {
ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
*bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
*bufEnd = *bufStart + newListSize;
if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
}
if (*bufStart + *pos + pathLength < *bufEnd) {
strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos));
*pos += pathLength + 1;
nbFiles++;
}
}
free(path);
errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */
}
if (errno != 0) {
UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s\n", dirName, strerror(errno));
free(*bufStart);
*bufStart = NULL;
}
closedir(dir);
return nbFiles;
}
#else
UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
{
(void)bufStart; (void)bufEnd; (void)pos; (void)followLinks;
UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
return 0;
}
#endif /* #ifdef _WIN32 */
/*
@ -544,56 +209,10 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
* After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer)
* In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called.
*/
UTIL_STATIC const char**
const char**
UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
char** allocatedBuffer, unsigned* allocatedNamesNb,
int followLinks)
{
size_t pos;
unsigned i, nbFiles;
char* buf = (char*)malloc(LIST_SIZE_INCREASE);
char* bufend = buf + LIST_SIZE_INCREASE;
const char** fileTable;
if (!buf) return NULL;
for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) {
if (!UTIL_isDirectory(inputNames[i])) {
size_t const len = strlen(inputNames[i]);
if (buf + pos + len >= bufend) {
ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
buf = (char*)UTIL_realloc(buf, newListSize);
bufend = buf + newListSize;
if (!buf) return NULL;
}
if (buf + pos + len < bufend) {
strncpy(buf + pos, inputNames[i], bufend - (buf + pos));
pos += len + 1;
nbFiles++;
}
} else {
nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks);
if (buf == NULL) return NULL;
} }
if (nbFiles == 0) { free(buf); return NULL; }
fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*));
if (!fileTable) { free(buf); return NULL; }
for (i=0, pos=0; i<nbFiles; i++) {
fileTable[i] = buf + pos;
pos += strlen(fileTable[i]) + 1;
}
if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
*allocatedBuffer = buf;
*allocatedNamesNb = nbFiles;
return fileTable;
}
int followLinks);
UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer)
{
@ -601,201 +220,7 @@ UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBu
if (filenameTable) free((void*)filenameTable);
}
/* count the number of physical cores */
#if defined(_WIN32) || defined(WIN32)
#include <windows.h>
typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD);
UTIL_STATIC int UTIL_countPhysicalCores(void)
{
static int numPhysicalCores = 0;
if (numPhysicalCores != 0) return numPhysicalCores;
{ LPFN_GLPI glpi;
BOOL done = FALSE;
PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = NULL;
PSYSTEM_LOGICAL_PROCESSOR_INFORMATION ptr = NULL;
DWORD returnLength = 0;
size_t byteOffset = 0;
glpi = (LPFN_GLPI)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
"GetLogicalProcessorInformation");
if (glpi == NULL) {
goto failed;
}
while(!done) {
DWORD rc = glpi(buffer, &returnLength);
if (FALSE == rc) {
if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
if (buffer)
free(buffer);
buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(returnLength);
if (buffer == NULL) {
perror("zstd");
exit(1);
}
} else {
/* some other error */
goto failed;
}
} else {
done = TRUE;
}
}
ptr = buffer;
while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) {
if (ptr->Relationship == RelationProcessorCore) {
numPhysicalCores++;
}
ptr++;
byteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
}
free(buffer);
return numPhysicalCores;
}
failed:
/* try to fall back on GetSystemInfo */
{ SYSTEM_INFO sysinfo;
GetSystemInfo(&sysinfo);
numPhysicalCores = sysinfo.dwNumberOfProcessors;
if (numPhysicalCores == 0) numPhysicalCores = 1; /* just in case */
}
return numPhysicalCores;
}
#elif defined(__APPLE__)
#include <sys/sysctl.h>
/* Use apple-provided syscall
* see: man 3 sysctl */
UTIL_STATIC int UTIL_countPhysicalCores(void)
{
static S32 numPhysicalCores = 0; /* apple specifies int32_t */
if (numPhysicalCores != 0) return numPhysicalCores;
{ size_t size = sizeof(S32);
int const ret = sysctlbyname("hw.physicalcpu", &numPhysicalCores, &size, NULL, 0);
if (ret != 0) {
if (errno == ENOENT) {
/* entry not present, fall back on 1 */
numPhysicalCores = 1;
} else {
perror("zstd: can't get number of physical cpus");
exit(1);
}
}
return numPhysicalCores;
}
}
#elif defined(__linux__)
/* parse /proc/cpuinfo
* siblings / cpu cores should give hyperthreading ratio
* otherwise fall back on sysconf */
UTIL_STATIC int UTIL_countPhysicalCores(void)
{
static int numPhysicalCores = 0;
if (numPhysicalCores != 0) return numPhysicalCores;
numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
if (numPhysicalCores == -1) {
/* value not queryable, fall back on 1 */
return numPhysicalCores = 1;
}
/* try to determine if there's hyperthreading */
{ FILE* const cpuinfo = fopen("/proc/cpuinfo", "r");
#define BUF_SIZE 80
char buff[BUF_SIZE];
int siblings = 0;
int cpu_cores = 0;
int ratio = 1;
if (cpuinfo == NULL) {
/* fall back on the sysconf value */
return numPhysicalCores;
}
/* assume the cpu cores/siblings values will be constant across all
* present processors */
while (!feof(cpuinfo)) {
if (fgets(buff, BUF_SIZE, cpuinfo) != NULL) {
if (strncmp(buff, "siblings", 8) == 0) {
const char* const sep = strchr(buff, ':');
if (*sep == '\0') {
/* formatting was broken? */
goto failed;
}
siblings = atoi(sep + 1);
}
if (strncmp(buff, "cpu cores", 9) == 0) {
const char* const sep = strchr(buff, ':');
if (*sep == '\0') {
/* formatting was broken? */
goto failed;
}
cpu_cores = atoi(sep + 1);
}
} else if (ferror(cpuinfo)) {
/* fall back on the sysconf value */
goto failed;
}
}
if (siblings && cpu_cores) {
ratio = siblings / cpu_cores;
}
failed:
fclose(cpuinfo);
return numPhysicalCores = numPhysicalCores / ratio;
}
}
#elif defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)
/* Use apple-provided syscall
* see: man 3 sysctl */
UTIL_STATIC int UTIL_countPhysicalCores(void)
{
static int numPhysicalCores = 0;
if (numPhysicalCores != 0) return numPhysicalCores;
numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
if (numPhysicalCores == -1) {
/* value not queryable, fall back on 1 */
return numPhysicalCores = 1;
}
return numPhysicalCores;
}
#else
UTIL_STATIC int UTIL_countPhysicalCores(void)
{
/* assume 1 */
return 1;
}
#endif
int UTIL_countPhysicalCores(void);
#if defined (__cplusplus)
}

34
sys/contrib/zstd/programs/zstd.1
View File

@ -1,5 +1,5 @@
.
.TH "ZSTD" "1" "October 2018" "zstd 1.3.7" "User Commands"
.TH "ZSTD" "1" "December 2018" "zstd 1.3.8" "User Commands"
.
.SH "NAME"
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@ -127,6 +127,10 @@ Does not spawn a thread for compression, use a single thread for both I/O and co
\fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
.
.TP
\fB\-\-rsyncable\fR
\fBzstd\fR will periodically synchronize the compression state to make the compressed file more rsync\-friendly\. There is a negligible impact to compression ratio, and the faster compression levels will see a small compression speed hit\. This feature does not work with \fB\-\-single\-thread\fR\. You probably don\'t want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your milage may vary\.
.
.TP
\fB\-D file\fR
use \fBfile\fR as Dictionary to compress or decompress FILE(s)
.
@ -312,7 +316,7 @@ set process priority to real\-time
Specify a strategy used by a match finder\.
.
.IP
There are 8 strategies numbered from 1 to 8, from faster to stronger: 1=ZSTD_fast, 2=ZSTD_dfast, 3=ZSTD_greedy, 4=ZSTD_lazy, 5=ZSTD_lazy2, 6=ZSTD_btlazy2, 7=ZSTD_btopt, 8=ZSTD_btultra\.
There are 9 strategies numbered from 1 to 9, from faster to stronger: 1=ZSTD_fast, 2=ZSTD_dfast, 3=ZSTD_greedy, 4=ZSTD_lazy, 5=ZSTD_lazy2, 6=ZSTD_btlazy2, 7=ZSTD_btopt, 8=ZSTD_btultra, 9=ZSTD_btultra2\.
.
.TP
\fBwindowLog\fR=\fIwlog\fR, \fBwlog\fR=\fIwlog\fR
@ -355,21 +359,21 @@ More searches increases the chance to find a match which usually increases compr
The minimum \fIslog\fR is 1 and the maximum is 26\.
.
.TP
\fBsearchLength\fR=\fIslen\fR, \fBslen\fR=\fIslen\fR
\fBminMatch\fR=\fImml\fR, \fBmml\fR=\fImml\fR
Specify the minimum searched length of a match in a hash table\.
.
.IP
Larger search lengths usually decrease compression ratio but improve decompression speed\.
.
.IP
The minimum \fIslen\fR is 3 and the maximum is 7\.
The minimum \fImml\fR is 3 and the maximum is 7\.
.
.TP
\fBtargetLen\fR=\fItlen\fR, \fBtlen\fR=\fItlen\fR
The impact of this field vary depending on selected strategy\.
.
.IP
For ZSTD_btopt and ZSTD_btultra, it specifies the minimum match length that causes match finder to stop searching for better matches\. A larger \fBtargetLen\fR usually improves compression ratio but decreases compression speed\.
For ZSTD_btopt, ZSTD_btultra and ZSTD_btultra2, it specifies the minimum match length that causes match finder to stop searching\. A larger \fBtargetLen\fR usually improves compression ratio but decreases compression speed\.
.
.IP
For ZSTD_fast, it triggers ultra\-fast mode when > 0\. The value represents the amount of data skipped between match sampling\. Impact is reversed : a larger \fBtargetLen\fR increases compression speed but decreases compression ratio\.
@ -385,10 +389,10 @@ The minimum \fItlen\fR is 0 and the maximum is 999\.
Determine \fBoverlapSize\fR, amount of data reloaded from previous job\. This parameter is only available when multithreading is enabled\. Reloading more data improves compression ratio, but decreases speed\.
.
.IP
The minimum \fIovlog\fR is 0, and the maximum is 9\. 0 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the amount of reload by a factor 2\. Default \fIovlog\fR is 6, which means "reload \fBwindowSize / 8\fR"\. Exception : the maximum compression level (22) has a default \fIovlog\fR of 9\.
The minimum \fIovlog\fR is 0, and the maximum is 9\. 1 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the reloaded amount by a factor 2\. For example, 8 means "windowSize/2", and 6 means "windowSize/8"\. Value 0 is special and means "default" : \fIovlog\fR is automatically determined by \fBzstd\fR\. In which case, \fIovlog\fR will range from 6 to 9, depending on selected \fIstrat\fR\.
.
.TP
\fBldmHashLog\fR=\fIldmhlog\fR, \fBldmhlog\fR=\fIldmhlog\fR
\fBldmHashLog\fR=\fIlhlog\fR, \fBlhlog\fR=\fIlhlog\fR
Specify the maximum size for a hash table used for long distance matching\.
.
.IP
@ -398,10 +402,10 @@ This option is ignored unless long distance matching is enabled\.
Bigger hash tables usually improve compression ratio at the expense of more memory during compression and a decrease in compression speed\.
.
.IP
The minimum \fIldmhlog\fR is 6 and the maximum is 26 (default: 20)\.
The minimum \fIlhlog\fR is 6 and the maximum is 26 (default: 20)\.
.
.TP
\fBldmSearchLength\fR=\fIldmslen\fR, \fBldmslen\fR=\fIldmslen\fR
\fBldmMinMatch\fR=\fIlmml\fR, \fBlmml\fR=\fIlmml\fR
Specify the minimum searched length of a match for long distance matching\.
.
.IP
@ -411,10 +415,10 @@ This option is ignored unless long distance matching is enabled\.
Larger/very small values usually decrease compression ratio\.
.
.IP
The minimum \fIldmslen\fR is 4 and the maximum is 4096 (default: 64)\.
The minimum \fIlmml\fR is 4 and the maximum is 4096 (default: 64)\.
.
.TP
\fBldmBucketSizeLog\fR=\fIldmblog\fR, \fBldmblog\fR=\fIldmblog\fR
\fBldmBucketSizeLog\fR=\fIlblog\fR, \fBlblog\fR=\fIlblog\fR
Specify the size of each bucket for the hash table used for long distance matching\.
.
.IP
@ -424,10 +428,10 @@ This option is ignored unless long distance matching is enabled\.
Larger bucket sizes improve collision resolution but decrease compression speed\.
.
.IP
The minimum \fIldmblog\fR is 0 and the maximum is 8 (default: 3)\.
The minimum \fIlblog\fR is 0 and the maximum is 8 (default: 3)\.
.
.TP
\fBldmHashEveryLog\fR=\fIldmhevery\fR, \fBldmhevery\fR=\fIldmhevery\fR
\fBldmHashRateLog\fR=\fIlhrlog\fR, \fBlhrlog\fR=\fIlhrlog\fR
Specify the frequency of inserting entries into the long distance matching hash table\.
.
.IP
@ -437,13 +441,13 @@ This option is ignored unless long distance matching is enabled\.
Larger values will improve compression speed\. Deviating far from the default value will likely result in a decrease in compression ratio\.
.
.IP
The default value is \fBwlog \- ldmhlog\fR\.
The default value is \fBwlog \- lhlog\fR\.
.
.SS "Example"
The following parameters sets advanced compression options to something similar to predefined level 19 for files bigger than 256 KB:
.
.P
\fB\-\-zstd\fR=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6
\fB\-\-zstd\fR=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
.
.SS "\-B#:"
Select the size of each compression job\. This parameter is available only when multi\-threading is enabled\. Default value is \fB4 * windowSize\fR, which means it varies depending on compression level\. \fB\-B#\fR makes it possible to select a custom value\. Note that job size must respect a minimum value which is enforced transparently\. This minimum is either 1 MB, or \fBoverlapSize\fR, whichever is largest\.

52
sys/contrib/zstd/programs/zstd.1.md
View File

@ -144,6 +144,14 @@ the last one takes effect.
Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible.
_note_ : at the time of this writing, `--adapt` can remain stuck at low speed
when combined with multiple worker threads (>=2).
* `--rsyncable` :
`zstd` will periodically synchronize the compression state to make the
compressed file more rsync-friendly. There is a negligible impact to
compression ratio, and the faster compression levels will see a small
compression speed hit.
This feature does not work with `--single-thread`. You probably don't want
to use it with long range mode, since it will decrease the effectiveness of
the synchronization points, but your milage may vary.
* `-D file`:
use `file` as Dictionary to compress or decompress FILE(s)
* `--no-dictID`:
@ -187,6 +195,8 @@ the last one takes effect.
* `-q`, `--quiet`:
suppress warnings, interactivity, and notifications.
specify twice to suppress errors too.
* `--no-progress`:
do not display the progress bar, but keep all other messages.
* `-C`, `--[no-]check`:
add integrity check computed from uncompressed data (default: enabled)
* `--`:
@ -331,9 +341,10 @@ The list of available _options_:
- `strategy`=_strat_, `strat`=_strat_:
Specify a strategy used by a match finder.
There are 8 strategies numbered from 1 to 8, from faster to stronger:
1=ZSTD\_fast, 2=ZSTD\_dfast, 3=ZSTD\_greedy, 4=ZSTD\_lazy,
5=ZSTD\_lazy2, 6=ZSTD\_btlazy2, 7=ZSTD\_btopt, 8=ZSTD\_btultra.
There are 9 strategies numbered from 1 to 9, from faster to stronger:
1=ZSTD\_fast, 2=ZSTD\_dfast, 3=ZSTD\_greedy,
4=ZSTD\_lazy, 5=ZSTD\_lazy2, 6=ZSTD\_btlazy2,
7=ZSTD\_btopt, 8=ZSTD\_btultra, 9=ZSTD\_btultra2.
- `windowLog`=_wlog_, `wlog`=_wlog_:
Specify the maximum number of bits for a match distance.
@ -375,19 +386,19 @@ The list of available _options_:
The minimum _slog_ is 1 and the maximum is 26.
- `searchLength`=_slen_, `slen`=_slen_:
- `minMatch`=_mml_, `mml`=_mml_:
Specify the minimum searched length of a match in a hash table.
Larger search lengths usually decrease compression ratio but improve
decompression speed.
The minimum _slen_ is 3 and the maximum is 7.
The minimum _mml_ is 3 and the maximum is 7.
- `targetLen`=_tlen_, `tlen`=_tlen_:
The impact of this field vary depending on selected strategy.
For ZSTD\_btopt and ZSTD\_btultra, it specifies the minimum match length
that causes match finder to stop searching for better matches.
For ZSTD\_btopt, ZSTD\_btultra and ZSTD\_btultra2, it specifies
the minimum match length that causes match finder to stop searching.
A larger `targetLen` usually improves compression ratio
but decreases compression speed.
@ -406,13 +417,14 @@ The list of available _options_:
Reloading more data improves compression ratio, but decreases speed.
The minimum _ovlog_ is 0, and the maximum is 9.
0 means "no overlap", hence completely independent jobs.
1 means "no overlap", hence completely independent jobs.
9 means "full overlap", meaning up to `windowSize` is reloaded from previous job.
Reducing _ovlog_ by 1 reduces the amount of reload by a factor 2.
Default _ovlog_ is 6, which means "reload `windowSize / 8`".
Exception : the maximum compression level (22) has a default _ovlog_ of 9.
Reducing _ovlog_ by 1 reduces the reloaded amount by a factor 2.
For example, 8 means "windowSize/2", and 6 means "windowSize/8".
Value 0 is special and means "default" : _ovlog_ is automatically determined by `zstd`.
In which case, _ovlog_ will range from 6 to 9, depending on selected _strat_.
- `ldmHashLog`=_ldmhlog_, `ldmhlog`=_ldmhlog_:
- `ldmHashLog`=_lhlog_, `lhlog`=_lhlog_:
Specify the maximum size for a hash table used for long distance matching.
This option is ignored unless long distance matching is enabled.
@ -420,18 +432,18 @@ The list of available _options_:
Bigger hash tables usually improve compression ratio at the expense of more
memory during compression and a decrease in compression speed.
The minimum _ldmhlog_ is 6 and the maximum is 26 (default: 20).
The minimum _lhlog_ is 6 and the maximum is 26 (default: 20).
- `ldmSearchLength`=_ldmslen_, `ldmslen`=_ldmslen_:
- `ldmMinMatch`=_lmml_, `lmml`=_lmml_:
Specify the minimum searched length of a match for long distance matching.
This option is ignored unless long distance matching is enabled.
Larger/very small values usually decrease compression ratio.
The minimum _ldmslen_ is 4 and the maximum is 4096 (default: 64).
The minimum _lmml_ is 4 and the maximum is 4096 (default: 64).
- `ldmBucketSizeLog`=_ldmblog_, `ldmblog`=_ldmblog_:
- `ldmBucketSizeLog`=_lblog_, `lblog`=_lblog_:
Specify the size of each bucket for the hash table used for long distance
matching.
@ -440,9 +452,9 @@ The list of available _options_:
Larger bucket sizes improve collision resolution but decrease compression
speed.
The minimum _ldmblog_ is 0 and the maximum is 8 (default: 3).
The minimum _lblog_ is 0 and the maximum is 8 (default: 3).
- `ldmHashEveryLog`=_ldmhevery_, `ldmhevery`=_ldmhevery_:
- `ldmHashRateLog`=_lhrlog_, `lhrlog`=_lhrlog_:
Specify the frequency of inserting entries into the long distance matching
hash table.
@ -451,13 +463,13 @@ The list of available _options_:
Larger values will improve compression speed. Deviating far from the
default value will likely result in a decrease in compression ratio.
The default value is `wlog - ldmhlog`.
The default value is `wlog - lhlog`.
### Example
The following parameters sets advanced compression options to something
similar to predefined level 19 for files bigger than 256 KB:
`--zstd`=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6
`--zstd`=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
### -B#:
Select the size of each compression job.

100
sys/contrib/zstd/programs/zstdcli.c
View File

@ -28,11 +28,12 @@
#include "platform.h" /* IS_CONSOLE, PLATFORM_POSIX_VERSION */
#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */
#include <stdio.h> /* fprintf(), stdin, stdout, stderr */
#include <stdlib.h> /* getenv */
#include <string.h> /* strcmp, strlen */
#include <errno.h> /* errno */
#include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */
#ifndef ZSTD_NOBENCH
# include "bench.h" /* BMK_benchFiles */
# include "benchzstd.h" /* BMK_benchFiles */
#endif
#ifndef ZSTD_NODICT
# include "dibio.h" /* ZDICT_cover_params_t, DiB_trainFromFiles() */
@ -81,7 +82,7 @@ static const unsigned g_defaultMaxWindowLog = 27;
static U32 g_overlapLog = OVERLAP_LOG_DEFAULT;
static U32 g_ldmHashLog = 0;
static U32 g_ldmMinMatch = 0;
static U32 g_ldmHashEveryLog = LDM_PARAM_DEFAULT;
static U32 g_ldmHashRateLog = LDM_PARAM_DEFAULT;
static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT;
@ -143,6 +144,7 @@ static int usage_advanced(const char* programName)
#ifdef ZSTD_MULTITHREAD
DISPLAY( " -T# : spawns # compression threads (default: 1, 0==# cores) \n");
DISPLAY( " -B# : select size of each job (default: 0==automatic) \n");
DISPLAY( " --rsyncable : compress using a rsync-friendly method (-B sets block size) \n");
#endif
DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
DISPLAY( "--[no-]check : integrity check (default: enabled) \n");
@ -150,7 +152,7 @@ static int usage_advanced(const char* programName)
#ifdef UTIL_HAS_CREATEFILELIST
DISPLAY( " -r : operate recursively on directories \n");
#endif
DISPLAY( "--format=zstd : compress files to the .zstd format (default) \n");
DISPLAY( "--format=zstd : compress files to the .zst format (default) \n");
#ifdef ZSTD_GZCOMPRESS
DISPLAY( "--format=gzip : compress files to the .gz format \n");
#endif
@ -170,6 +172,7 @@ static int usage_advanced(const char* programName)
#endif
#endif
DISPLAY( " -M# : Set a memory usage limit for decompression \n");
DISPLAY( "--no-progress : do not display the progress bar \n");
DISPLAY( "-- : All arguments after \"--\" are treated as files \n");
#ifndef ZSTD_NODICT
DISPLAY( "\n");
@ -231,32 +234,44 @@ static void errorOut(const char* msg)
DISPLAY("%s \n", msg); exit(1);
}
/*! readU32FromChar() :
* @return : unsigned integer value read from input in `char` format.
/*! readU32FromCharChecked() :
* @return 0 if success, and store the result in *value.
* allows and interprets K, KB, KiB, M, MB and MiB suffix.
* Will also modify `*stringPtr`, advancing it to position where it stopped reading.
* Note : function will exit() program if digit sequence overflows */
static unsigned readU32FromChar(const char** stringPtr)
* @return 1 if an overflow error occurs */
static int readU32FromCharChecked(const char** stringPtr, unsigned* value)
{
const char errorMsg[] = "error: numeric value too large";
static unsigned const max = (((unsigned)(-1)) / 10) - 1;
unsigned result = 0;
while ((**stringPtr >='0') && (**stringPtr <='9')) {
unsigned const max = (((unsigned)(-1)) / 10) - 1;
if (result > max) errorOut(errorMsg);
if (result > max) return 1; // overflow error
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
}
if ((**stringPtr=='K') || (**stringPtr=='M')) {
unsigned const maxK = ((unsigned)(-1)) >> 10;
if (result > maxK) errorOut(errorMsg);
if (result > maxK) return 1; // overflow error
result <<= 10;
if (**stringPtr=='M') {
if (result > maxK) errorOut(errorMsg);
if (result > maxK) return 1; // overflow error
result <<= 10;
}
(*stringPtr)++; /* skip `K` or `M` */
if (**stringPtr=='i') (*stringPtr)++;
if (**stringPtr=='B') (*stringPtr)++;
}
*value = result;
return 0;
}
/*! readU32FromChar() :
* @return : unsigned integer value read from input in `char` format.
* allows and interprets K, KB, KiB, M, MB and MiB suffix.
* Will also modify `*stringPtr`, advancing it to position where it stopped reading.
* Note : function will exit() program if digit sequence overflows */
static unsigned readU32FromChar(const char** stringPtr) {
static const char errorMsg[] = "error: numeric value too large";
unsigned result;
if (readU32FromCharChecked(stringPtr, &result)) { errorOut(errorMsg); }
return result;
}
@ -391,7 +406,7 @@ static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, in
/** parseCompressionParameters() :
* reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params
* reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6") into *params
* @return 1 means that compression parameters were correct
* @return 0 in case of malformed parameters
*/
@ -402,20 +417,20 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi
if (longCommandWArg(&stringPtr, "chainLog=") || longCommandWArg(&stringPtr, "clog=")) { params->chainLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "hashLog=") || longCommandWArg(&stringPtr, "hlog=")) { params->hashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "minMatch=") || longCommandWArg(&stringPtr, "mml=")) { params->minMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "overlapLog=") || longCommandWArg(&stringPtr, "ovlog=")) { g_overlapLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "ldmHashLog=") || longCommandWArg(&stringPtr, "ldmhlog=")) { g_ldmHashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "ldmSearchLength=") || longCommandWArg(&stringPtr, "ldmslen=")) { g_ldmMinMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "ldmBucketSizeLog=") || longCommandWArg(&stringPtr, "ldmblog=")) { g_ldmBucketSizeLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "ldmHashEveryLog=") || longCommandWArg(&stringPtr, "ldmhevery=")) { g_ldmHashEveryLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "ldmHashLog=") || longCommandWArg(&stringPtr, "lhlog=")) { g_ldmHashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "ldmMinMatch=") || longCommandWArg(&stringPtr, "lmml=")) { g_ldmMinMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "ldmBucketSizeLog=") || longCommandWArg(&stringPtr, "lblog=")) { g_ldmBucketSizeLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "ldmHashRateLog=") || longCommandWArg(&stringPtr, "lhrlog=")) { g_ldmHashRateLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
DISPLAYLEVEL(4, "invalid compression parameter \n");
return 0;
}
DISPLAYLEVEL(4, "windowLog=%d, chainLog=%d, hashLog=%d, searchLog=%d \n", params->windowLog, params->chainLog, params->hashLog, params->searchLog);
DISPLAYLEVEL(4, "searchLength=%d, targetLength=%d, strategy=%d \n", params->searchLength, params->targetLength, params->strategy);
DISPLAYLEVEL(4, "minMatch=%d, targetLength=%d, strategy=%d \n", params->minMatch, params->targetLength, params->strategy);
if (stringPtr[0] != 0) return 0; /* check the end of string */
return 1;
}
@ -450,6 +465,38 @@ static void printVersion(void)
#endif
}
/* Environment variables for parameter setting */
#define ENV_CLEVEL "ZSTD_CLEVEL"
/* functions that pick up environment variables */
static int init_cLevel(void) {
const char* const env = getenv(ENV_CLEVEL);
if (env) {
const char *ptr = env;
int sign = 1;
if (*ptr == '-') {
sign = -1;
ptr++;
} else if (*ptr == '+') {
ptr++;
}
if ((*ptr>='0') && (*ptr<='9')) {
unsigned absLevel;
if (readU32FromCharChecked(&ptr, &absLevel)) {
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large\n", ENV_CLEVEL, env);
return ZSTDCLI_CLEVEL_DEFAULT;
} else if (*ptr == 0) {
return sign * absLevel;
}
}
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid integer value\n", ENV_CLEVEL, env);
}
return ZSTDCLI_CLEVEL_DEFAULT;
}
typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode;
#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
@ -475,6 +522,7 @@ int main(int argCount, const char* argv[])
adapt = 0,
adaptMin = MINCLEVEL,
adaptMax = MAXCLEVEL,
rsyncable = 0,
nextArgumentIsOutFileName = 0,
nextArgumentIsMaxDict = 0,
nextArgumentIsDictID = 0,
@ -490,7 +538,7 @@ int main(int argCount, const char* argv[])
size_t blockSize = 0;
zstd_operation_mode operation = zom_compress;
ZSTD_compressionParameters compressionParams;
int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
int cLevel;
int cLevelLast = -1000000000;
unsigned recursive = 0;
unsigned memLimit = 0;
@ -525,6 +573,7 @@ int main(int argCount, const char* argv[])
if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
filenameTable[0] = stdinmark;
g_displayOut = stderr;
cLevel = init_cLevel();
programName = lastNameFromPath(programName);
#ifdef ZSTD_MULTITHREAD
nbWorkers = 1;
@ -607,6 +656,8 @@ int main(int argCount, const char* argv[])
#ifdef ZSTD_LZ4COMPRESS
if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); continue; }
#endif
if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; }
if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; }
/* long commands with arguments */
#ifndef ZSTD_NODICT
@ -937,8 +988,8 @@ int main(int argCount, const char* argv[])
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
benchParams.ldmBucketSizeLog = g_ldmBucketSizeLog;
}
if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
benchParams.ldmHashEveryLog = g_ldmHashEveryLog;
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) {
benchParams.ldmHashRateLog = g_ldmHashRateLog;
}
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
@ -1048,10 +1099,11 @@ int main(int argCount, const char* argv[])
FIO_setLdmHashLog(g_ldmHashLog);
FIO_setLdmMinMatch(g_ldmMinMatch);
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(g_ldmBucketSizeLog);
if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) FIO_setLdmHashEveryLog(g_ldmHashEveryLog);
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(g_ldmHashRateLog);
FIO_setAdaptiveMode(adapt);
FIO_setAdaptMin(adaptMin);
FIO_setAdaptMax(adaptMax);
FIO_setRsyncable(rsyncable);
if (adaptMin > cLevel) cLevel = adaptMin;
if (adaptMax < cLevel) cLevel = adaptMax;
@ -1060,7 +1112,7 @@ int main(int argCount, const char* argv[])
else
operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
#else
(void)suffix; (void)adapt; (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */
(void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */
DISPLAY("Compression not supported \n");
#endif
} else { /* decompression or test */

141
sys/contrib/zstd/programs/zstdgrep
View File

@ -31,94 +31,101 @@ grep_args=""
hyphen=0
silent=0
prg=$(basename $0)
prg=$(basename "$0")
# handle being called 'zegrep' or 'zfgrep'
case ${prg} in
*zegrep)
grep_args="-E";;
*zfgrep)
grep_args="-F";;
case "${prg}" in
*zegrep) grep_args="-E";;
*zfgrep) grep_args="-F";;
esac
# skip all options and pass them on to grep taking care of options
# with arguments, and if -e was supplied
while [ $# -gt 0 -a ${endofopts} -eq 0 ]
do
case $1 in
while [ "$#" -gt 0 ] && [ "${endofopts}" -eq 0 ]; do
case "$1" in
# from GNU grep-2.5.1 -- keep in sync!
-[ABCDXdefm])
if [ $# -lt 2 ]
then
echo "${prg}: missing argument for $1 flag" >&2
exit 1
fi
case $1 in
-e)
pattern="$2"
pattern_found=1
shift 2
break
;;
*)
;;
esac
grep_args="${grep_args} $1 $2"
shift 2
;;
--)
shift
endofopts=1
;;
-)
hyphen=1
shift
;;
-h)
silent=1
shift
;;
-*)
grep_args="${grep_args} $1"
shift
;;
*)
# pattern to grep for
endofopts=1
;;
-[ABCDXdefm])
if [ "$#" -lt 2 ]; then
printf '%s: missing argument for %s flag\n' "${prg}" "$1" >&2
exit 1
fi
case "$1" in
-e)
pattern="$2"
pattern_found=1
shift 2
break
;;
*)
;;
esac
grep_args="${grep_args} $1 $2"
shift 2
;;
--)
shift
endofopts=1
;;
-)
hyphen=1
shift
;;
-h)
silent=1
shift
;;
-*)
grep_args="${grep_args} $1"
shift
;;
*)
# pattern to grep for
endofopts=1
;;
esac
done
# if no -e option was found, take next argument as grep-pattern
if [ ${pattern_found} -lt 1 ]
then
if [ $# -ge 1 ]; then
pattern="$1"
shift
elif [ ${hyphen} -gt 0 ]; then
pattern="-"
if [ "${pattern_found}" -lt 1 ]; then
if [ "$#" -ge 1 ]; then
pattern="$1"
shift
elif [ "${hyphen}" -gt 0 ]; then
pattern="-"
else
echo "${prg}: missing pattern" >&2
exit 1
printf '%s: missing pattern\n' "${prg}" >&2
exit 1
fi
fi
EXIT_CODE=0
# call grep ...
if [ $# -lt 1 ]
then
if [ "$#" -lt 1 ]; then
# ... on stdin
${zcat} -fq - | ${grep} ${grep_args} -- "${pattern}" -
set -f # Disable file name generation (globbing).
# shellcheck disable=SC2086
"${zcat}" -fq - | "${grep}" ${grep_args} -- "${pattern}" -
EXIT_CODE=$?
set +f
else
# ... on all files given on the command line
if [ ${silent} -lt 1 -a $# -gt 1 ]; then
grep_args="-H ${grep_args}"
if [ "${silent}" -lt 1 ] && [ "$#" -gt 1 ]; then
grep_args="-H ${grep_args}"
fi
while [ $# -gt 0 ]
do
${zcat} -fq -- "$1" | ${grep} --label="${1}" ${grep_args} -- "${pattern}" -
shift
CUR_EXIT_CODE=0
EXIT_CODE=1
set -f
while [ "$#" -gt 0 ]; do
# shellcheck disable=SC2086
"${zcat}" -fq -- "$1" | "${grep}" --label="${1}" ${grep_args} -- "${pattern}" -
CUR_EXIT_CODE=$?
if [ "${CUR_EXIT_CODE}" -eq 0 ] && [ "${EXIT_CODE}" -ne 1 ]; then
EXIT_CODE=0
fi
shift
done
set +f
fi
exit 0
exit "${EXIT_CODE}"

2
sys/contrib/zstd/programs/zstdgrep.1
View File

@ -1,5 +1,5 @@
.
.TH "ZSTDGREP" "1" "October 2018" "zstd 1.3.7" "User Commands"
.TH "ZSTDGREP" "1" "December 2018" "zstd 1.3.8" "User Commands"
.
.SH "NAME"
\fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files

2
sys/contrib/zstd/programs/zstdless.1
View File

@ -1,5 +1,5 @@
.
.TH "ZSTDLESS" "1" "October 2018" "zstd 1.3.7" "User Commands"
.TH "ZSTDLESS" "1" "December 2018" "zstd 1.3.8" "User Commands"
.
.SH "NAME"
\fBzstdless\fR \- view zstandard\-compressed files

30
sys/contrib/zstd/tests/Makefile
View File

@ -132,18 +132,18 @@ fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP)
fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD)
fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements
fullbench fullbench32 : $(ZSTD_FILES)
fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/bench.c fullbench.c
fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c fullbench.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fullbench-lib : CPPFLAGS += -DXXH_NAMESPACE=ZSTD_
fullbench-lib : zstd-staticLib
fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/bench.c fullbench.c
fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c fullbench.c
$(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) $(ZSTDDIR)/libzstd.a
# note : broken : requires unavailable symbols
fullbench-dll : zstd-dll
fullbench-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
fullbench-dll: $(PRGDIR)/datagen.c fullbench.c
fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c fullbench.c
# $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
$(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT)
@ -152,32 +152,32 @@ fuzzer : LDFLAGS += $(MULTITHREAD_LD)
fuzzer32: CFLAGS += -m32
fuzzer : $(ZSTDMT_OBJECTS)
fuzzer32: $(ZSTD_FILES)
fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/datagen.c fuzzer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fuzzer-dll : zstd-dll
fuzzer-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c fuzzer.c
fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/datagen.c fuzzer.c
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated
zbufftest : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings
zbufftest : $(ZSTD_OBJECTS) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
zbufftest : $(ZSTD_OBJECTS) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/datagen.c zbufftest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zbufftest32 : CPPFLAGS += -I$(ZSTDDIR)/deprecated
zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32
zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/datagen.c zbufftest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zbufftest-dll : zstd-dll
zbufftest-dll : CPPFLAGS += -I$(ZSTDDIR)/deprecated
zbufftest-dll : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings
zbufftest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zbufftest.c
zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/datagen.c zbufftest.c
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c seqgen.c zstreamtest.c
ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c seqgen.c zstreamtest.c
ZSTREAM_PROPER_FILES := $(ZDICT_FILES) $(ZSTREAM_LOCAL_FILES)
ZSTREAMFILES := $(ZSTD_FILES) $(ZSTREAM_PROPER_FILES)
zstreamtest32 : CFLAGS += -m32
@ -203,7 +203,7 @@ zstreamtest-dll : $(ZSTREAM_LOCAL_FILES)
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
paramgrill : DEBUGFLAGS = # turn off assert() by default for speed measurements
paramgrill : $(ZSTD_FILES) $(PRGDIR)/bench.c $(PRGDIR)/datagen.c paramgrill.c
paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c paramgrill.c
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
datagen : $(PRGDIR)/datagen.c datagencli.c
@ -222,7 +222,7 @@ legacy : CPPFLAGS += -I$(ZSTDDIR)/legacy -DZSTD_LEGACY_SUPPORT=4
legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) decodecorpus.c
decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) $(PRGDIR)/util.c decodecorpus.c
$(CC) $(FLAGS) $^ -o $@$(EXT) -lm
symbols : symbols.c zstd-dll
@ -233,7 +233,7 @@ else
$(CC) $(FLAGS) $< -o $@$(EXT) -Wl,-rpath=$(ZSTDDIR) $(ZSTDDIR)/libzstd.so # broken on Mac
endif
poolTests : poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c
poolTests : $(PRGDIR)/util.c poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c
$(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
.PHONY: versionsTest
@ -320,7 +320,7 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32
test-all: test test32 valgrindTest test-decodecorpus-cli
.PHONY: test-zstd test-zstd32 test-zstd-nolegacy
.PHONY: test-zstd test-zstd32 test-zstd-nolegacy test-zstdgrep
test-zstd: ZSTD = $(PRGDIR)/zstd
test-zstd: zstd
@ -352,6 +352,10 @@ test-gzstd: gzstd
$(PRGDIR)/zstd -dcf - <hello_zst_gz_txt.gz
$(RM) *.gz *.zst README2.md gz_zstd zstd_gz hello.txt
test-zstdgrep: gzstd
@echo a | $(PRGDIR)/zstd | $(PRGDIR)/zstdgrep a
@echo a | $(PRGDIR)/zstd | $(PRGDIR)/zstdgrep b && return 1 || return 0
test-fullbench: fullbench datagen
$(QEMU_SYS) ./fullbench -i1
$(QEMU_SYS) ./fullbench -i1 -P0

18
sys/contrib/zstd/tests/README.md
View File

@ -41,7 +41,7 @@ Additional remarks:
The example usage with two test files, one e-mail address, and with an additional message:
```
./test-zstd-speed.py "silesia.tar calgary.tar" "email@gmail.com" --message "tested on my laptop" --sleepTime 60
```
```
To run the script in background please use:
```
@ -100,19 +100,19 @@ Full list of arguments
h# - hashLog
c# - chainLog
s# - searchLog
l# - searchLength
l# - minMatch
t# - targetLength
S# - strategy
L# - level
--zstd= : Single run, parameter selection syntax same as zstdcli with more parameters
(Added forceAttachDictionary / fadt)
When invoked with --optimize, this represents the sample to exceed.
(Added forceAttachDictionary / fadt)
When invoked with --optimize, this represents the sample to exceed.
--optimize= : find parameters to maximize compression ratio given parameters
Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints
cSpeed= : Minimum compression speed
dSpeed= : Minimum decompression speed
cMem= : Maximum compression memory
lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed,
lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed,
stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 100%)
: In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters
(Lower value will begin with stronger strategies) (Default 90%)
@ -121,13 +121,13 @@ Full list of arguments
when determining overall winner (default 5 (1% ratio = 5% speed)).
tries= : Maximum number of random restarts on a single strategy before switching (Default 5)
Higher values will make optimizer run longer, more chances to find better solution.
memLog : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size.
Setting memLog = 0 turns off memoization
memLog : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size.
Setting memLog = 0 turns off memoization
--display= : specifiy which parameters are included in the output
can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters
can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters
(Default: display all params available)
-P# : generated sample compressibility (when no file is provided)
-t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours ))
-t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours ))
-v : Prints Benchmarking output
-D : Next argument dictionary file
-s : Benchmark all files separately

2
sys/contrib/zstd/tests/datagencli.c
View File

@ -121,7 +121,7 @@ int main(int argc, const char** argv)
DISPLAYLEVEL(4, "Compressible data Generator \n");
if (probaU32!=COMPRESSIBILITY_DEFAULT)
DISPLAYLEVEL(3, "Compressibility : %i%%\n", probaU32);
DISPLAYLEVEL(3, "Seed = %u \n", seed);
DISPLAYLEVEL(3, "Seed = %u \n", (unsigned)seed);
RDG_genStdout(size, (double)probaU32/100, litProba, seed);
DISPLAYLEVEL(1, "\n");

88
sys/contrib/zstd/tests/decodecorpus.c
View File

@ -22,7 +22,7 @@
#define ZDICT_STATIC_LINKING_ONLY
#include "zdict.h"
// Direct access to internal compression functions is required
/* Direct access to internal compression functions is required */
#include "zstd_compress.c"
#define XXH_STATIC_LINKING_ONLY
@ -72,7 +72,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
/*-*******************************************************
* Random function
*********************************************************/
static unsigned RAND(unsigned* src)
static U32 RAND(U32* src)
{
#define RAND_rotl32(x,r) ((x << r) | (x >> (32 - r)))
static const U32 prime1 = 2654435761U;
@ -350,7 +350,7 @@ static void writeFrameHeader(U32* seed, frame_t* frame, dictInfo info)
}
}
DISPLAYLEVEL(3, " frame content size:\t%u\n", (U32)fh.contentSize);
DISPLAYLEVEL(3, " frame content size:\t%u\n", (unsigned)fh.contentSize);
DISPLAYLEVEL(3, " frame window size:\t%u\n", fh.windowSize);
DISPLAYLEVEL(3, " content size flag:\t%d\n", contentSizeFlag);
DISPLAYLEVEL(3, " single segment flag:\t%d\n", singleSegment);
@ -412,7 +412,7 @@ static size_t writeLiteralsBlockSimple(U32* seed, frame_t* frame, size_t content
/* RLE literals */
BYTE const symb = (BYTE) (RAND(seed) % 256);
DISPLAYLEVEL(4, " rle literals: 0x%02x\n", (U32)symb);
DISPLAYLEVEL(4, " rle literals: 0x%02x\n", (unsigned)symb);
memset(LITERAL_BUFFER, symb, litSize);
op[0] = symb;
@ -432,12 +432,12 @@ static size_t writeHufHeader(U32* seed, HUF_CElt* hufTable, void* dst, size_t ds
BYTE* op = ostart;
unsigned huffLog = 11;
U32 maxSymbolValue = 255;
unsigned maxSymbolValue = 255;
U32 count[HUF_SYMBOLVALUE_MAX+1];
unsigned count[HUF_SYMBOLVALUE_MAX+1];
/* Scan input and build symbol stats */
{ size_t const largest = HIST_count_wksp (count, &maxSymbolValue, (const BYTE*)src, srcSize, WKSP);
{ size_t const largest = HIST_count_wksp (count, &maxSymbolValue, (const BYTE*)src, srcSize, WKSP, sizeof(WKSP));
assert(!HIST_isError(largest));
if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 0; } /* single symbol, rle */
if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */
@ -568,8 +568,8 @@ static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t con
op += compressedSize;
compressedSize += hufHeaderSize;
DISPLAYLEVEL(5, " regenerated size: %u\n", (U32)litSize);
DISPLAYLEVEL(5, " compressed size: %u\n", (U32)compressedSize);
DISPLAYLEVEL(5, " regenerated size: %u\n", (unsigned)litSize);
DISPLAYLEVEL(5, " compressed size: %u\n", (unsigned)compressedSize);
if (compressedSize >= litSize) {
DISPLAYLEVEL(5, " trying again\n");
/* if we have to try again, reset the stats so we don't accidentally
@ -656,7 +656,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
excessMatch = remainingMatch - numSequences * MIN_SEQ_LEN;
}
DISPLAYLEVEL(5, " total match lengths: %u\n", (U32)remainingMatch);
DISPLAYLEVEL(5, " total match lengths: %u\n", (unsigned)remainingMatch);
for (i = 0; i < numSequences; i++) {
/* Generate match and literal lengths by exponential distribution to
* ensure nice numbers */
@ -748,12 +748,13 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
frame->stats.rep[0] = offset;
}
DISPLAYLEVEL(6, " LL: %5u OF: %5u ML: %5u", literalLen, offset, matchLen);
DISPLAYLEVEL(6, " LL: %5u OF: %5u ML: %5u",
(unsigned)literalLen, (unsigned)offset, (unsigned)matchLen);
DISPLAYLEVEL(7, " srcPos: %8u seqNb: %3u",
(U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart), i);
(unsigned)((BYTE*)srcPtr - (BYTE*)frame->srcStart), (unsigned)i);
DISPLAYLEVEL(6, "\n");
if (offsetCode < 3) {
DISPLAYLEVEL(7, " repeat offset: %d\n", repIndex);
DISPLAYLEVEL(7, " repeat offset: %d\n", (int)repIndex);
}
/* use libzstd sequence handling */
ZSTD_storeSeq(seqStore, literalLen, literals, offsetCode,
@ -766,8 +767,8 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
memcpy(srcPtr, literals, literalsSize);
srcPtr += literalsSize;
DISPLAYLEVEL(6, " excess literals: %5u", (U32)literalsSize);
DISPLAYLEVEL(7, " srcPos: %8u", (U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart));
DISPLAYLEVEL(6, " excess literals: %5u", (unsigned)literalsSize);
DISPLAYLEVEL(7, " srcPos: %8u", (unsigned)((BYTE*)srcPtr - (BYTE*)frame->srcStart));
DISPLAYLEVEL(6, "\n");
return numSequences;
@ -800,7 +801,7 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr,
size_t nbSeq)
{
/* This code is mostly copied from ZSTD_compressSequences in zstd_compress.c */
U32 count[MaxSeq+1];
unsigned count[MaxSeq+1];
S16 norm[MaxSeq+1];
FSE_CTable* CTable_LitLength = frame->stats.litlengthCTable;
FSE_CTable* CTable_OffsetBits = frame->stats.offcodeCTable;
@ -823,21 +824,20 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr,
else if (nbSeq < LONGNBSEQ) op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2;
else op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3;
/* seqHead : flags for FSE encoding type */
seqHead = op++;
if (nbSeq==0) {
frame->data = op;
return 0;
}
/* seqHead : flags for FSE encoding type */
seqHead = op++;
/* convert length/distances into codes */
ZSTD_seqToCodes(seqStorePtr);
/* CTable for Literal Lengths */
{ U32 max = MaxLL;
size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, WKSP); /* cannot fail */
{ unsigned max = MaxLL;
size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, WKSP, sizeof(WKSP)); /* cannot fail */
assert(!HIST_isError(mostFrequent));
if (mostFrequent == nbSeq) {
/* do RLE if we have the chance */
@ -868,8 +868,8 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr,
/* CTable for Offsets */
/* see Literal Lengths for descriptions of mode choices */
{ U32 max = MaxOff;
size_t const mostFrequent = HIST_countFast_wksp(count, &max, ofCodeTable, nbSeq, WKSP); /* cannot fail */
{ unsigned max = MaxOff;
size_t const mostFrequent = HIST_countFast_wksp(count, &max, ofCodeTable, nbSeq, WKSP, sizeof(WKSP)); /* cannot fail */
assert(!HIST_isError(mostFrequent));
if (mostFrequent == nbSeq) {
*op++ = ofCodeTable[0];
@ -896,8 +896,8 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr,
/* CTable for MatchLengths */
/* see Literal Lengths for descriptions of mode choices */
{ U32 max = MaxML;
size_t const mostFrequent = HIST_countFast_wksp(count, &max, mlCodeTable, nbSeq, WKSP); /* cannot fail */
{ unsigned max = MaxML;
size_t const mostFrequent = HIST_countFast_wksp(count, &max, mlCodeTable, nbSeq, WKSP, sizeof(WKSP)); /* cannot fail */
assert(!HIST_isError(mostFrequent));
if (mostFrequent == nbSeq) {
*op++ = *mlCodeTable;
@ -928,7 +928,7 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr,
initSymbolSet(ofCodeTable, nbSeq, frame->stats.offsetSymbolSet, 28);
initSymbolSet(mlCodeTable, nbSeq, frame->stats.matchlengthSymbolSet, 52);
DISPLAYLEVEL(5, " LL type: %d OF type: %d ML type: %d\n", LLtype, Offtype, MLtype);
DISPLAYLEVEL(5, " LL type: %d OF type: %d ML type: %d\n", (unsigned)LLtype, (unsigned)Offtype, (unsigned)MLtype);
*seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
@ -1015,11 +1015,11 @@ static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize
literalsSize = writeLiteralsBlock(seed, frame, contentSize);
DISPLAYLEVEL(4, " literals size: %u\n", (U32)literalsSize);
DISPLAYLEVEL(4, " literals size: %u\n", (unsigned)literalsSize);
nbSeq = writeSequencesBlock(seed, frame, contentSize, literalsSize, info);
DISPLAYLEVEL(4, " number of sequences: %u\n", (U32)nbSeq);
DISPLAYLEVEL(4, " number of sequences: %u\n", (unsigned)nbSeq);
return (BYTE*)frame->data - blockStart;
}
@ -1035,7 +1035,7 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize,
BYTE *op = header + 3;
DISPLAYLEVEL(4, " block:\n");
DISPLAYLEVEL(4, " block content size: %u\n", (U32)contentSize);
DISPLAYLEVEL(4, " block content size: %u\n", (unsigned)contentSize);
DISPLAYLEVEL(4, " last block: %s\n", lastBlock ? "yes" : "no");
if (blockTypeDesc == 0) {
@ -1083,7 +1083,7 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize,
frame->src = (BYTE*)frame->src + contentSize;
DISPLAYLEVEL(4, " block type: %s\n", BLOCK_TYPES[blockType]);
DISPLAYLEVEL(4, " block size field: %u\n", (U32)blockSize);
DISPLAYLEVEL(4, " block size field: %u\n", (unsigned)blockSize);
header[0] = (BYTE) ((lastBlock | (blockType << 1) | (blockSize << 3)) & 0xff);
MEM_writeLE16(header + 1, (U16) (blockSize >> 5));
@ -1125,7 +1125,7 @@ static void writeChecksum(frame_t* frame)
{
/* write checksum so implementations can verify their output */
U64 digest = XXH64(frame->srcStart, (BYTE*)frame->src-(BYTE*)frame->srcStart, 0);
DISPLAYLEVEL(3, " checksum: %08x\n", (U32)digest);
DISPLAYLEVEL(3, " checksum: %08x\n", (unsigned)digest);
MEM_writeLE32(frame->data, (U32)digest);
frame->data = (BYTE*)frame->data + 4;
}
@ -1186,7 +1186,7 @@ static U32 generateCompressedBlock(U32 seed, frame_t* frame, dictInfo info)
size_t blockContentSize;
int blockWritten = 0;
BYTE* op;
DISPLAYLEVEL(4, "block seed: %u\n", seed);
DISPLAYLEVEL(4, "block seed: %u\n", (unsigned)seed);
initFrame(frame);
op = (BYTE*)frame->data;
@ -1223,7 +1223,7 @@ static U32 generateCompressedBlock(U32 seed, frame_t* frame, dictInfo info)
DISPLAYLEVEL(5, " can't compress block : try again \n");
} else {
blockWritten = 1;
DISPLAYLEVEL(4, " block size: %u \n", (U32)cSize);
DISPLAYLEVEL(4, " block size: %u \n", (unsigned)cSize);
frame->src = (BYTE*)frame->src + blockContentSize;
}
}
@ -1234,7 +1234,7 @@ static U32 generateCompressedBlock(U32 seed, frame_t* frame, dictInfo info)
static U32 generateFrame(U32 seed, frame_t* fr, dictInfo info)
{
/* generate a complete frame */
DISPLAYLEVEL(3, "frame seed: %u\n", seed);
DISPLAYLEVEL(3, "frame seed: %u\n", (unsigned)seed);
initFrame(fr);
writeFrameHeader(&seed, fr, info);
@ -1480,8 +1480,8 @@ static int runBlockTest(U32* seed)
{ size_t const r = testDecodeRawBlock(&fr);
if (ZSTD_isError(r)) {
DISPLAY("Error in block mode on test seed %u: %s\n", seedCopy,
ZSTD_getErrorName(r));
DISPLAY("Error in block mode on test seed %u: %s\n",
(unsigned)seedCopy, ZSTD_getErrorName(r));
return 1;
}
}
@ -1489,7 +1489,7 @@ static int runBlockTest(U32* seed)
{ size_t const r = testDecodeWithDict(*seed, gt_block);
if (ZSTD_isError(r)) {
DISPLAY("Error in block mode with dictionary on test seed %u: %s\n",
seedCopy, ZSTD_getErrorName(r));
(unsigned)seedCopy, ZSTD_getErrorName(r));
return 1;
}
}
@ -1507,21 +1507,21 @@ static int runFrameTest(U32* seed)
{ size_t const r = testDecodeSimple(&fr);
if (ZSTD_isError(r)) {
DISPLAY("Error in simple mode on test seed %u: %s\n",
seedCopy, ZSTD_getErrorName(r));
(unsigned)seedCopy, ZSTD_getErrorName(r));
return 1;
}
}
{ size_t const r = testDecodeStreaming(&fr);
if (ZSTD_isError(r)) {
DISPLAY("Error in streaming mode on test seed %u: %s\n",
seedCopy, ZSTD_getErrorName(r));
(unsigned)seedCopy, ZSTD_getErrorName(r));
return 1;
}
}
{ size_t const r = testDecodeWithDict(*seed, gt_frame); /* avoid big dictionaries */
if (ZSTD_isError(r)) {
DISPLAY("Error in dictionary mode on test seed %u: %s\n",
seedCopy, ZSTD_getErrorName(r));
(unsigned)seedCopy, ZSTD_getErrorName(r));
return 1;
}
}
@ -1538,7 +1538,7 @@ static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS
if (numFiles == 0 && !testDurationS) numFiles = 1;
DISPLAY("seed: %u\n", seed);
DISPLAY("seed: %u\n", (unsigned)seed);
for (fnum = 0; fnum < numFiles || UTIL_clockSpanMicro(startClock) < maxClockSpan; fnum++) {
if (fnum < numFiles)
@ -1568,7 +1568,7 @@ static int generateFile(U32 seed, const char* const path,
{
frame_t fr;
DISPLAY("seed: %u\n", seed);
DISPLAY("seed: %u\n", (unsigned)seed);
{ dictInfo const info = initDictInfo(0, 0, NULL, 0);
if (genType == gt_frame) {
@ -1590,7 +1590,7 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path,
char outPath[MAX_PATH];
unsigned fnum;
DISPLAY("seed: %u\n", seed);
DISPLAY("seed: %u\n", (unsigned)seed);
for (fnum = 0; fnum < numFiles; fnum++) {
frame_t fr;

120
sys/contrib/zstd/tests/fullbench.c
View File

@ -19,7 +19,7 @@
#include "mem.h" /* U32 */
#ifndef ZSTD_DLL_IMPORT
#include "zstd_internal.h" /* ZSTD_blockHeaderSize, blockType_e, KB, MB */
#include "zstd_internal.h" /* ZSTD_decodeSeqHeaders, ZSTD_blockHeaderSize, blockType_e, KB, MB */
#else
#define KB *(1 <<10)
#define MB *(1 <<20)
@ -30,7 +30,8 @@
#include "zstd.h" /* ZSTD_versionString */
#include "util.h" /* time functions */
#include "datagen.h"
#include "bench.h" /* CustomBench*/
#include "benchfn.h" /* CustomBench*/
#include "benchzstd.h" /* MB_UNIT */
/*_************************************
@ -63,10 +64,10 @@ static const size_t g_sampleSize = 10000000;
/*_************************************
* Benchmark Parameters
**************************************/
static U32 g_nbIterations = NBLOOPS;
static unsigned g_nbIterations = NBLOOPS;
static double g_compressibility = COMPRESSIBILITY_DEFAULT;
static void BMK_SetNbIterations(U32 nbLoops)
static void BMK_SetNbIterations(unsigned nbLoops)
{
g_nbIterations = nbLoops;
DISPLAY("- %i iterations -\n", g_nbIterations);
@ -133,7 +134,6 @@ static size_t local_ZSTD_decodeLiteralsBlock(const void* src, size_t srcSize, vo
return ZSTD_decodeLiteralsBlock((ZSTD_DCtx*)g_zdc, buff2, g_cSize);
}
extern size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeq, const void* src, size_t srcSize);
static size_t local_ZSTD_decodeSeqHeaders(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
{
int nbSeq;
@ -171,17 +171,8 @@ local_ZSTD_compress_generic_end(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
{
ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn;
(void)buff2;
buffOut.dst = dst;
buffOut.size = dstCapacity;
buffOut.pos = 0;
buffIn.src = src;
buffIn.size = srcSize;
buffIn.pos = 0;
ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end);
return buffOut.pos;
return ZSTD_compress2(g_cstream, dst, dstCapacity, src, srcSize);
}
static size_t
@ -198,8 +189,8 @@ local_ZSTD_compress_generic_continue(const void* src, size_t srcSize,
buffIn.src = src;
buffIn.size = srcSize;
buffIn.pos = 0;
ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_continue);
ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end);
ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_continue);
ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_end);
return buffOut.pos;
}
@ -208,18 +199,9 @@ local_ZSTD_compress_generic_T2_end(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
{
ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn;
(void)buff2;
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2);
buffOut.dst = dst;
buffOut.size = dstCapacity;
buffOut.pos = 0;
buffIn.src = src;
buffIn.size = srcSize;
buffIn.pos = 0;
while (ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end)) {}
return buffOut.pos;
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_nbWorkers, 2);
return ZSTD_compress2(g_cstream, dst, dstCapacity, src, srcSize);
}
static size_t
@ -230,15 +212,15 @@ local_ZSTD_compress_generic_T2_continue(const void* src, size_t srcSize,
ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn;
(void)buff2;
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_nbWorkers, 2);
buffOut.dst = dst;
buffOut.size = dstCapacity;
buffOut.pos = 0;
buffIn.src = src;
buffIn.size = srcSize;
buffIn.pos = 0;
ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_continue);
while(ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end)) {}
ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_continue);
while(ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_end)) {}
return buffOut.pos;
}
@ -334,7 +316,7 @@ static size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize,
/*_*******************************************************
* Bench functions
*********************************************************/
static size_t benchMem(U32 benchNb,
static size_t benchMem(unsigned benchNb,
const void* src, size_t srcSize,
int cLevel, ZSTD_compressionParameters cparams)
{
@ -408,28 +390,28 @@ static size_t benchMem(U32 benchNb,
if (g_cstream==NULL) g_cstream = ZSTD_createCStream();
if (g_dstream==NULL) g_dstream = ZSTD_createDStream();
/* DISPLAY("params: cLevel %d, wlog %d hlog %d clog %d slog %d slen %d tlen %d strat %d \n",
/* DISPLAY("params: cLevel %d, wlog %d hlog %d clog %d slog %d mml %d tlen %d strat %d \n",
cLevel, cparams->windowLog, cparams->hashLog, cparams->chainLog, cparams->searchLog,
cparams->searchLength, cparams->targetLength, cparams->strategy); */
cparams->minMatch, cparams->targetLength, cparams->strategy); */
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_windowLog, cparams.windowLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_hashLog, cparams.hashLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_chainLog, cparams.chainLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_searchLog, cparams.searchLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_minMatch, cparams.searchLength);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_targetLength, cparams.targetLength);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionStrategy, cparams.strategy);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_windowLog, cparams.windowLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_hashLog, cparams.hashLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_chainLog, cparams.chainLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_searchLog, cparams.searchLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_minMatch, cparams.minMatch);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_targetLength, cparams.targetLength);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_strategy, cparams.strategy);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_windowLog, cparams.windowLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_hashLog, cparams.hashLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_chainLog, cparams.chainLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_searchLog, cparams.searchLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_minMatch, cparams.searchLength);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_targetLength, cparams.targetLength);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionStrategy, cparams.strategy);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_windowLog, cparams.windowLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_hashLog, cparams.hashLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_chainLog, cparams.chainLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_searchLog, cparams.searchLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_minMatch, cparams.minMatch);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_targetLength, cparams.targetLength);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_strategy, cparams.strategy);
/* Preparation */
switch(benchNb)
@ -455,8 +437,8 @@ static size_t benchMem(U32 benchNb,
ZSTD_frameHeader zfp;
size_t frameHeaderSize, skippedSize;
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_FRAMEHEADERSIZE_MIN);
if (frameHeaderSize==0) frameHeaderSize = ZSTD_FRAMEHEADERSIZE_MIN;
ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) {
DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
@ -476,8 +458,8 @@ static size_t benchMem(U32 benchNb,
size_t frameHeaderSize, cBlockSize;
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); /* it would be better to use direct block compression here */
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_FRAMEHEADERSIZE_MIN);
if (frameHeaderSize==0) frameHeaderSize = ZSTD_FRAMEHEADERSIZE_MIN;
ip += frameHeaderSize; /* Skip frame Header */
cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) {
@ -515,20 +497,28 @@ static size_t benchMem(U32 benchNb,
/* benchmark loop */
{ BMK_timedFnState_t* const tfs = BMK_createTimedFnState(g_nbIterations * 1000, 1000);
void* const avoidStrictAliasingPtr = &dstBuff;
BMK_benchParams_t bp;
BMK_runTime_t bestResult;
bestResult.sumOfReturn = 0;
bestResult.nanoSecPerRun = (unsigned long long)(-1LL);
assert(tfs != NULL);
bp.benchFn = benchFunction;
bp.benchPayload = buff2;
bp.initFn = NULL;
bp.initPayload = NULL;
bp.errorFn = ZSTD_isError;
bp.blockCount = 1;
bp.srcBuffers = &src;
bp.srcSizes = &srcSize;
bp.dstBuffers = (void* const*) avoidStrictAliasingPtr; /* circumvent strict aliasing warning on gcc-8,
* because gcc considers that `void* const *` and `void**` are 2 different types */
bp.dstCapacities = &dstBuffSize;
bp.blockResults = NULL;
for (;;) {
void* const dstBuffv = dstBuff;
BMK_runOutcome_t const bOutcome =
BMK_benchTimedFn( tfs,
benchFunction, buff2,
NULL, NULL, /* initFn */
1, /* blockCount */
&src, &srcSize,
&dstBuffv, &dstBuffSize,
NULL);
BMK_runOutcome_t const bOutcome = BMK_benchTimedFn(tfs, bp);
if (!BMK_isSuccessful_runOutcome(bOutcome)) {
DISPLAY("ERROR benchmarking function ! ! \n");
@ -616,7 +606,7 @@ static int benchFiles(U32 benchNb,
benchedSize = (size_t)inFileSize;
if ((U64)benchedSize < inFileSize) {
DISPLAY("Not enough memory for '%s' full size; testing %u MB only... \n",
inFileName, (U32)(benchedSize>>20));
inFileName, (unsigned)(benchedSize>>20));
} }
/* Alloc */
@ -744,7 +734,7 @@ int main(int argc, const char** argv)
if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { cparams.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { cparams.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { cparams.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { cparams.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "minMatch=") || longCommandWArg(&argument, "mml=")) { cparams.minMatch = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { cparams.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { cparams.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = (int)readU32FromChar(&argument); cparams = ZSTD_getCParams(cLevel, 0, 0); if (argument[0]==',') { argument++; continue; } else break; }

6
sys/contrib/zstd/tests/fuzz/Makefile
View File

@ -41,7 +41,7 @@ FUZZ_ARFLAGS := $(ARFLAGS)
FUZZ_TARGET_FLAGS = $(FUZZ_CPPFLAGS) $(FUZZ_CXXFLAGS) $(FUZZ_LDFLAGS)
FUZZ_HEADERS := fuzz_helpers.h fuzz.h zstd_helpers.h
FUZZ_SRC := zstd_helpers.c
FUZZ_SRC := $(PRGDIR)/util.c zstd_helpers.c
ZSTDCOMMON_SRC := $(ZSTDDIR)/common/*.c
ZSTDCOMP_SRC := $(ZSTDDIR)/compress/*.c
@ -90,7 +90,7 @@ stream_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) stream_decompress.o
block_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) block_decompress.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) block_decompress.o $(LIB_FUZZING_ENGINE) -o $@
libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h regression_driver.o
libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h $(PRGDIR)/util.c regression_driver.o
$(AR) $(FUZZ_ARFLAGS) $@ regression_driver.o
# Install libfuzzer (not usable for MSAN testing)
@ -99,7 +99,7 @@ libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h regression_driver.o
.PHONY: libFuzzer
libFuzzer:
@$(RM) -rf Fuzzer
@git clone https://chromium.googlesource.com/chromium/llvm-project/llvm/lib/Fuzzer
@git clone https://chromium.googlesource.com/chromium/llvm-project/compiler-rt/lib/fuzzer Fuzzer
@cd Fuzzer && ./build.sh
corpora/%_seed_corpus.zip:

4
sys/contrib/zstd/tests/fuzz/simple_round_trip.c
View File

@ -40,9 +40,9 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
ZSTD_outBuffer out = {compressed, compressedCapacity, 0};
size_t err;
ZSTD_CCtx_reset(cctx);
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
FUZZ_setRandomParameters(cctx, srcSize, &seed);
err = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
err = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
FUZZ_ZASSERT(err);
FUZZ_ASSERT(err == 0);
cSize = out.pos;

12
sys/contrib/zstd/tests/fuzz/stream_round_trip.c
View File

@ -56,7 +56,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
const uint8_t *src, size_t srcSize)
{
size_t dstSize = 0;
ZSTD_CCtx_reset(cctx);
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
FUZZ_setRandomParameters(cctx, srcSize, &seed);
while (srcSize > 0) {
@ -72,7 +72,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
case 1: /* fall-though */
case 2: {
size_t const ret =
ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_flush);
ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush);
FUZZ_ZASSERT(ret);
if (ret == 0)
mode = -1;
@ -80,11 +80,11 @@ static size_t compress(uint8_t *dst, size_t capacity,
}
case 3: {
size_t ret =
ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
FUZZ_ZASSERT(ret);
/* Reset the compressor when the frame is finished */
if (ret == 0) {
ZSTD_CCtx_reset(cctx);
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
if ((FUZZ_rand(&seed) & 7) == 0) {
size_t const remaining = in.size - in.pos;
FUZZ_setRandomParameters(cctx, remaining, &seed);
@ -95,7 +95,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
}
default: {
size_t const ret =
ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue);
ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
FUZZ_ZASSERT(ret);
mode = -1;
}
@ -108,7 +108,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
for (;;) {
ZSTD_inBuffer in = {NULL, 0, 0};
ZSTD_outBuffer out = makeOutBuffer(dst, capacity);
size_t const ret = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
FUZZ_ZASSERT(ret);
dst += out.pos;

42
sys/contrib/zstd/tests/fuzz/zstd_helpers.c
View File

@ -13,7 +13,7 @@
#include "fuzz_helpers.h"
#include "zstd.h"
static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned value)
static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, int value)
{
FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value));
}
@ -32,10 +32,10 @@ ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state)
cParams.hashLog = FUZZ_rand32(state, ZSTD_HASHLOG_MIN, 15);
cParams.chainLog = FUZZ_rand32(state, ZSTD_CHAINLOG_MIN, 16);
cParams.searchLog = FUZZ_rand32(state, ZSTD_SEARCHLOG_MIN, 9);
cParams.searchLength = FUZZ_rand32(state, ZSTD_SEARCHLENGTH_MIN,
ZSTD_SEARCHLENGTH_MAX);
cParams.minMatch = FUZZ_rand32(state, ZSTD_MINMATCH_MIN,
ZSTD_MINMATCH_MAX);
cParams.targetLength = FUZZ_rand32(state, 0, 512);
cParams.strategy = FUZZ_rand32(state, ZSTD_fast, ZSTD_btultra);
cParams.strategy = FUZZ_rand32(state, ZSTD_STRATEGY_MIN, ZSTD_STRATEGY_MAX);
return ZSTD_adjustCParams(cParams, srcSize, 0);
}
@ -60,25 +60,25 @@ ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state)
void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state)
{
ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, state);
set(cctx, ZSTD_p_windowLog, cParams.windowLog);
set(cctx, ZSTD_p_hashLog, cParams.hashLog);
set(cctx, ZSTD_p_chainLog, cParams.chainLog);
set(cctx, ZSTD_p_searchLog, cParams.searchLog);
set(cctx, ZSTD_p_minMatch, cParams.searchLength);
set(cctx, ZSTD_p_targetLength, cParams.targetLength);
set(cctx, ZSTD_p_compressionStrategy, cParams.strategy);
set(cctx, ZSTD_c_windowLog, cParams.windowLog);
set(cctx, ZSTD_c_hashLog, cParams.hashLog);
set(cctx, ZSTD_c_chainLog, cParams.chainLog);
set(cctx, ZSTD_c_searchLog, cParams.searchLog);
set(cctx, ZSTD_c_minMatch, cParams.minMatch);
set(cctx, ZSTD_c_targetLength, cParams.targetLength);
set(cctx, ZSTD_c_strategy, cParams.strategy);
/* Select frame parameters */
setRand(cctx, ZSTD_p_contentSizeFlag, 0, 1, state);
setRand(cctx, ZSTD_p_checksumFlag, 0, 1, state);
setRand(cctx, ZSTD_p_dictIDFlag, 0, 1, state);
setRand(cctx, ZSTD_p_forceAttachDict, -2, 2, state);
setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, state);
setRand(cctx, ZSTD_c_checksumFlag, 0, 1, state);
setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, state);
setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, state);
/* Select long distance matchig parameters */
setRand(cctx, ZSTD_p_enableLongDistanceMatching, 0, 1, state);
setRand(cctx, ZSTD_p_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state);
setRand(cctx, ZSTD_p_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN,
setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, state);
setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state);
setRand(cctx, ZSTD_c_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN,
ZSTD_LDM_MINMATCH_MAX, state);
setRand(cctx, ZSTD_p_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX,
setRand(cctx, ZSTD_c_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX,
state);
setRand(cctx, ZSTD_p_ldmHashEveryLog, 0,
ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN, state);
setRand(cctx, ZSTD_c_ldmHashRateLog, ZSTD_LDM_HASHRATELOG_MIN,
ZSTD_LDM_HASHRATELOG_MAX, state);
}

576
sys/contrib/zstd/tests/fuzzer.c
View File

File diff suppressed because it is too large Load Diff

89
sys/contrib/zstd/tests/libzstd_partial_builds.sh
View File

@ -1,36 +1,89 @@
#!/bin/sh -e
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
ECHO=echo
RM="rm -f"
GREP="grep"
INTOVOID="/dev/null"
die() {
$ECHO "$@" 1>&2
exit 1
}
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
isPresent() {
$GREP $@ tmplog || die "$@" "should be present"
}
INTOVOID="/dev/null"
case "$OS" in
Windows*)
INTOVOID="NUL"
;;
esac
mustBeAbsent() {
$GREP $@ tmplog && die "$@ should not be there !!"
$ECHO "$@ correctly not present" # for some reason, this $ECHO must exist, otherwise mustBeAbsent() always fails (??)
}
# default compilation : all features enabled
make clean > /dev/null
$ECHO "testing default library compilation"
CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
isPresent "zstd_decompress.o"
isPresent "zdict.o"
isPresent "zstd_v07.o"
isPresent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog
# compression disabled => also disable zdict and zbuff
$ECHO "testing with compression disabled"
ZSTD_LIB_COMPRESSION=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog
! grep -q "zstd_compress" tmplog && grep -q "zstd_decompress" tmplog && ! grep -q "dict" tmplog && grep -q "zstd_v" tmplog && ! grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Compression macro failed"
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
mustBeAbsent "zstd_compress.o"
isPresent "zstd_decompress.o"
mustBeAbsent "zdict.o"
isPresent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog
# decompression disabled => also disable legacy and zbuff
$ECHO "testing with decompression disabled"
ZSTD_LIB_DECOMPRESSION=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog
grep -q "zstd_compress" tmplog && ! grep -q "zstd_decompress" tmplog && grep -q "dict" tmplog && ! grep -q "zstd_v" tmplog && ! grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Decompression macro failed"
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
mustBeAbsent "zstd_decompress.o"
isPresent "zdict.o"
mustBeAbsent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog
# deprecated function disabled => only remove zbuff
$ECHO "testing with deprecated functions disabled"
ZSTD_LIB_DEPRECATED=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog
grep -q "zstd_compress" tmplog && grep -q "zstd_decompress" tmplog && grep -q "dict" tmplog && grep -q "zstd_v" tmplog && ! grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Deprecated macro failed"
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
isPresent "zstd_decompress.o"
isPresent "zdict.o"
isPresent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog
# dictionary builder disabled => only remove zdict
$ECHO "testing with dictionary builder disabled"
ZSTD_LIB_DICTBUILDER=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog
grep -q "zstd_compress" tmplog && grep -q "zstd_decompress" tmplog && ! grep -q "dict" tmplog && grep -q "zstd_v" tmplog && grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Dictbuilder macro failed"
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
isPresent "zstd_decompress.o"
mustBeAbsent "zdict.o"
isPresent "zstd_v07.o"
isPresent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog
# both decompression and dictionary builder disabled => only compression remains
$ECHO "testing with both decompression and dictionary builder disabled (only compression remains)"
ZSTD_LIB_DECOMPRESSION=0 ZSTD_LIB_DICTBUILDER=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog
grep -q "zstd_compress" tmplog && ! grep -q "zstd_decompress" tmplog && ! grep -q "dict" tmplog && ! grep -q "zstd_v" tmplog && ! grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Multi-macro failed"
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
mustBeAbsent "zstd_decompress.o"
mustBeAbsent "zdict.o"
mustBeAbsent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog

2
sys/contrib/zstd/tests/longmatch.c
View File

@ -50,7 +50,7 @@ int main(int argc, const char** argv)
params.cParams.chainLog = 13;
params.cParams.hashLog = 14;
params.cParams.searchLog = 1;
params.cParams.searchLength = 7;
params.cParams.minMatch = 7;
params.cParams.targetLength = 16;
params.cParams.strategy = ZSTD_fast;
windowLog = params.cParams.windowLog;

791
sys/contrib/zstd/tests/paramgrill.c
View File

File diff suppressed because it is too large Load Diff

55
sys/contrib/zstd/tests/playTests.sh
View File

@ -122,6 +122,18 @@ $ZSTD --fast=5000000000 -f tmp && die "too large numeric value : must fail"
$ZSTD -c --fast=0 tmp > $INTOVOID && die "--fast must not accept value 0"
$ECHO "test : too large numeric argument"
$ZSTD --fast=9999999999 -f tmp && die "should have refused numeric value"
$ECHO "test : set compression level with environment variable ZSTD_CLEVEL"
ZSTD_CLEVEL=12 $ZSTD -f tmp # positive compression level
ZSTD_CLEVEL=-12 $ZSTD -f tmp # negative compression level
ZSTD_CLEVEL=+12 $ZSTD -f tmp # valid: verbose '+' sign
ZSTD_CLEVEL= $ZSTD -f tmp # empty env var, warn and revert to default setting
ZSTD_CLEVEL=- $ZSTD -f tmp # malformed env var, warn and revert to default setting
ZSTD_CLEVEL=a $ZSTD -f tmp # malformed env var, warn and revert to default setting
ZSTD_CLEVEL=+a $ZSTD -f tmp # malformed env var, warn and revert to default setting
ZSTD_CLEVEL=3a7 $ZSTD -f tmp # malformed env var, warn and revert to default setting
ZSTD_CLEVEL=50000000000 $ZSTD -f tmp # numeric value too large, warn and revert to default setting
$ECHO "test : override ZSTD_CLEVEL with command line option"
ZSTD_CLEVEL=12 $ZSTD --fast=3 -f tmp # overridden by command line option
$ECHO "test : compress to stdout"
$ZSTD tmp -c > tmpCompressed
$ZSTD tmp --stdout > tmpCompressed # long command format
@ -179,8 +191,15 @@ $ECHO foo > tmpro
chmod 400 tmpro.zst
$ZSTD -q tmpro && die "should have refused to overwrite read-only file"
$ZSTD -q -f tmpro
$ECHO "test: --no-progress flag"
$ZSTD tmpro -c --no-progress | $ZSTD -d -f -o "$INTOVOID" --no-progress
$ZSTD tmpro -cv --no-progress | $ZSTD -dv -f -o "$INTOVOID" --no-progress
rm -f tmpro tmpro.zst
$ECHO "test: overwrite input file (must fail)"
$ZSTD tmp -fo tmp && die "zstd compression overwrote the input file"
$ZSTD tmp.zst -dfo tmp.zst && die "zstd decompression overwrote the input file"
$ECHO "test: detect that input file does not exist"
$ZSTD nothere && die "zstd hasn't detected that input file does not exist"
$ECHO "test : file removal"
$ZSTD -f --rm tmp
@ -213,7 +232,7 @@ rm tmp*
$ECHO "test : compress multiple files"
$ECHO hello > tmp1
$ECHO world > tmp2
$ZSTD tmp1 tmp2 -o "$INTOVOID"
$ZSTD tmp1 tmp2 -o "$INTOVOID" -f
$ZSTD tmp1 tmp2 -c | $ZSTD -t
$ZSTD tmp1 tmp2 -o tmp.zst
test ! -f tmp1.zst
@ -221,7 +240,7 @@ test ! -f tmp2.zst
$ZSTD tmp1 tmp2
$ZSTD -t tmp1.zst tmp2.zst
$ZSTD -dc tmp1.zst tmp2.zst
$ZSTD tmp1.zst tmp2.zst -o "$INTOVOID"
$ZSTD tmp1.zst tmp2.zst -o "$INTOVOID" -f
$ZSTD -d tmp1.zst tmp2.zst -o tmp
touch tmpexists
$ZSTD tmp1 tmp2 -f -o tmpexists
@ -237,14 +256,15 @@ $ECHO "\n===> Advanced compression parameters "
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=strategy=10 - -o tmp.zst && die "parameter out of bound not detected!" # > btultra2 : does not exist
test ! -f tmp.zst # tmp.zst should not be created
roundTripTest -g512K
roundTripTest -g512K " --zstd=slen=3,tlen=48,strat=6"
roundTripTest -g512K " --zstd=mml=3,tlen=48,strat=6"
roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6"
roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6"
roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmSearchLength=64,ldmBucketSizeLog=1,ldmHashEveryLog=7"
roundTripTest -g512K " --single-thread --long --zstd=ldmhlog=20,ldmslen=64,ldmblog=1,ldmhevery=7"
roundTripTest -g512K 19
roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,minMatch=3,targetLength=48,strategy=6"
roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmMinMatch=64,ldmBucketSizeLog=1,ldmHashRateLog=7"
roundTripTest -g512K " --single-thread --long --zstd=lhlog=20,lmml=64,lblog=1,lhrlog=7"
roundTripTest -g64K "19 --zstd=strat=9" # btultra2
$ECHO "\n===> Pass-Through mode "
@ -541,6 +561,9 @@ $ECHO "bench negative level"
$ZSTD -bi0 --fast tmp1
$ECHO "with recursive and quiet modes"
$ZSTD -rqi1b1e2 tmp1
$ECHO "benchmark decompression only"
$ZSTD -f tmp1
$ZSTD -b -d -i1 tmp1.zst
$ECHO "\n===> zstd compatibility tests "
@ -744,17 +767,17 @@ then
./datagen -g2MB > tmp
refSize=$($ZSTD tmp -6 -c --zstd=wlog=18 | wc -c)
ov9Size=$($ZSTD tmp -6 -c --zstd=wlog=18,ovlog=9 | wc -c)
ov0Size=$($ZSTD tmp -6 -c --zstd=wlog=18,ovlog=0 | wc -c)
ov1Size=$($ZSTD tmp -6 -c --zstd=wlog=18,ovlog=1 | wc -c)
if [ $refSize -eq $ov9Size ]; then
echo ov9Size should be different from refSize
exit 1
fi
if [ $refSize -eq $ov0Size ]; then
echo ov0Size should be different from refSize
if [ $refSize -eq $ov1Size ]; then
echo ov1Size should be different from refSize
exit 1
fi
if [ $ov9Size -ge $ov0Size ]; then
echo ov9Size=$ov9Size should be smaller than ov0Size=$ov0Size
if [ $ov9Size -ge $ov1Size ]; then
echo ov9Size=$ov9Size should be smaller than ov1Size=$ov1Size
exit 1
fi
@ -833,6 +856,12 @@ $ECHO "===> test: --adapt must fail on incoherent bounds "
./datagen > tmp
$ZSTD -f -vv --adapt=min=10,max=9 tmp && die "--adapt must fail on incoherent bounds"
$ECHO "\n===> rsyncable mode "
roundTripTest -g10M " --rsyncable"
roundTripTest -g10M " --rsyncable -B100K"
$ECHO "===> test: --rsyncable must fail with --single-thread"
$ZSTD -f -vv --rsyncable --single-thread tmp && die "--rsyncable must fail with --single-thread"
if [ "$1" != "--test-large-data" ]; then
$ECHO "Skipping large data tests"

12
sys/contrib/zstd/tests/poolTests.c
View File

@ -30,7 +30,7 @@ struct data {
size_t i;
};
void fn(void *opaque) {
static void fn(void *opaque) {
struct data *data = (struct data *)opaque;
ZSTD_pthread_mutex_lock(&data->mutex);
data->data[data->i] = data->i;
@ -38,7 +38,7 @@ void fn(void *opaque) {
ZSTD_pthread_mutex_unlock(&data->mutex);
}
int testOrder(size_t numThreads, size_t queueSize) {
static int testOrder(size_t numThreads, size_t queueSize) {
struct data data;
POOL_ctx *ctx = POOL_create(numThreads, queueSize);
ASSERT_TRUE(ctx);
@ -63,13 +63,13 @@ int testOrder(size_t numThreads, size_t queueSize) {
/* --- test deadlocks --- */
void waitFn(void *opaque) {
static void waitFn(void *opaque) {
(void)opaque;
UTIL_sleepMilli(1);
}
/* Tests for deadlock */
int testWait(size_t numThreads, size_t queueSize) {
static int testWait(size_t numThreads, size_t queueSize) {
struct data data;
POOL_ctx *ctx = POOL_create(numThreads, queueSize);
ASSERT_TRUE(ctx);
@ -92,7 +92,7 @@ typedef struct {
ZSTD_pthread_cond_t cond;
} poolTest_t;
void waitLongFn(void *opaque) {
static void waitLongFn(void *opaque) {
poolTest_t* test = (poolTest_t*) opaque;
UTIL_sleepMilli(10);
ZSTD_pthread_mutex_lock(&test->mut);
@ -167,7 +167,7 @@ typedef struct {
int val;
} abruptEndCanary_t;
void waitIncFn(void *opaque) {
static void waitIncFn(void *opaque) {
abruptEndCanary_t* test = (abruptEndCanary_t*) opaque;
UTIL_sleepMilli(10);
ZSTD_pthread_mutex_lock(&test->mut);

58
sys/contrib/zstd/tests/regression/Makefile Normal file
View File

@ -0,0 +1,58 @@
# ################################################################
# Copyright (c) 2015-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# ################################################################
CFLAGS ?= -O3
CURL_CFLAGS := $(shell curl-config --cflags)
CURL_LDFLAGS := $(shell curl-config --libs) -pthread
PROGDIR := ../../programs
LIBDIR := ../../lib
ZSTD_CPPFLAGS := -I$(PROGDIR) -I$(LIBDIR) -I$(LIBDIR)/common
REGRESSION_CFLAGS = $(CFLAGS) $(CURL_CFLAGS)
REGRESSION_CPPFLAGS = $(CPPFLAGS) $(ZSTD_CPPFLAGS)
REGRESSION_LDFLAGS = $(LDFLAGS) $(CURL_LDFLAGS)
all: test
xxhash.o: $(LIBDIR)/common/xxhash.c $(LIBDIR)/common/xxhash.h
$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
util.o: $(PROGDIR)/util.c $(PROGDIR)/util.h
$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
data.o: data.c data.h $(PROGDIR)/util.h $(LIBDIR)/common/xxhash.h
$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
config.o: config.c config.h levels.h
$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
method.h: data.h config.h result.h
method.o: method.c method.h
$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
result.o: result.c result.h
$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
test.o: test.c data.h config.h method.h
$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
libzstd.a:
$(MAKE) -C $(LIBDIR) libzstd.a-mt
cp $(LIBDIR)/libzstd.a .
test: test.o data.o config.o util.o method.o result.o xxhash.o libzstd.a
$(CC) $^ $(REGRESSION_LDFLAGS) -o $@
.PHONY: clean
clean:
$(MAKE) -C $(LIBDIR) clean
$(RM) *.o *.a test

230
sys/contrib/zstd/tests/regression/config.c Normal file
View File

@ -0,0 +1,230 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include "config.h"
/* Define a config for each fast level we want to test with. */
#define FAST_LEVEL(x) \
param_value_t const level_fast##x##_param_values[] = { \
{.param = ZSTD_c_compressionLevel, .value = -x}, \
}; \
config_t const level_fast##x = { \
.name = "level -" #x, \
.cli_args = "--fast=" #x, \
.param_values = PARAM_VALUES(level_fast##x##_param_values), \
}; \
config_t const level_fast##x##_dict = { \
.name = "level -" #x " with dict", \
.cli_args = "--fast=" #x, \
.param_values = PARAM_VALUES(level_fast##x##_param_values), \
.use_dictionary = 1, \
};
/* Define a config for each level we want to test with. */
#define LEVEL(x) \
param_value_t const level_##x##_param_values[] = { \
{.param = ZSTD_c_compressionLevel, .value = x}, \
}; \
config_t const level_##x = { \
.name = "level " #x, \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(level_##x##_param_values), \
}; \
config_t const level_##x##_dict = { \
.name = "level " #x " with dict", \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(level_##x##_param_values), \
.use_dictionary = 1, \
};
#define PARAM_VALUES(pv) \
{ .data = pv, .size = sizeof(pv) / sizeof((pv)[0]) }
#include "levels.h"
#undef LEVEL
#undef FAST_LEVEL
static config_t no_pledged_src_size = {
.name = "no source size",
.cli_args = "",
.param_values = PARAM_VALUES(level_0_param_values),
.no_pledged_src_size = 1,
};
static param_value_t const ldm_param_values[] = {
{.param = ZSTD_c_enableLongDistanceMatching, .value = 1},
};
static config_t ldm = {
.name = "long distance mode",
.cli_args = "--long",
.param_values = PARAM_VALUES(ldm_param_values),
};
static param_value_t const mt_param_values[] = {
{.param = ZSTD_c_nbWorkers, .value = 2},
};
static config_t mt = {
.name = "multithreaded",
.cli_args = "-T2",
.param_values = PARAM_VALUES(mt_param_values),
};
static param_value_t const mt_ldm_param_values[] = {
{.param = ZSTD_c_nbWorkers, .value = 2},
{.param = ZSTD_c_enableLongDistanceMatching, .value = 1},
};
static config_t mt_ldm = {
.name = "multithreaded long distance mode",
.cli_args = "-T2 --long",
.param_values = PARAM_VALUES(mt_ldm_param_values),
};
static param_value_t const small_wlog_param_values[] = {
{.param = ZSTD_c_windowLog, .value = 10},
};
static config_t small_wlog = {
.name = "small window log",
.cli_args = "--zstd=wlog=10",
.param_values = PARAM_VALUES(small_wlog_param_values),
};
static param_value_t const small_hlog_param_values[] = {
{.param = ZSTD_c_hashLog, .value = 6},
{.param = ZSTD_c_strategy, .value = (int)ZSTD_btopt},
};
static config_t small_hlog = {
.name = "small hash log",
.cli_args = "--zstd=hlog=6,strat=7",
.param_values = PARAM_VALUES(small_hlog_param_values),
};
static param_value_t const small_clog_param_values[] = {
{.param = ZSTD_c_chainLog, .value = 6},
{.param = ZSTD_c_strategy, .value = (int)ZSTD_btopt},
};
static config_t small_clog = {
.name = "small chain log",
.cli_args = "--zstd=clog=6,strat=7",
.param_values = PARAM_VALUES(small_clog_param_values),
};
static param_value_t const explicit_params_param_values[] = {
{.param = ZSTD_c_checksumFlag, .value = 1},
{.param = ZSTD_c_contentSizeFlag, .value = 0},
{.param = ZSTD_c_dictIDFlag, .value = 0},
{.param = ZSTD_c_strategy, .value = (int)ZSTD_greedy},
{.param = ZSTD_c_windowLog, .value = 18},
{.param = ZSTD_c_hashLog, .value = 21},
{.param = ZSTD_c_chainLog, .value = 21},
{.param = ZSTD_c_targetLength, .value = 100},
};
static config_t explicit_params = {
.name = "explicit params",
.cli_args = "--no-check --no-dictID --zstd=strategy=3,wlog=18,hlog=21,clog=21,tlen=100",
.param_values = PARAM_VALUES(explicit_params_param_values),
};
static config_t const* g_configs[] = {
#define FAST_LEVEL(x) &level_fast##x, &level_fast##x##_dict,
#define LEVEL(x) &level_##x, &level_##x##_dict,
#include "levels.h"
#undef LEVEL
#undef FAST_LEVEL
&no_pledged_src_size,
&ldm,
&mt,
&mt_ldm,
&small_wlog,
&small_hlog,
&small_clog,
&explicit_params,
NULL,
};
config_t const* const* configs = g_configs;
int config_skip_data(config_t const* config, data_t const* data) {
return config->use_dictionary && !data_has_dict(data);
}
int config_get_level(config_t const* config)
{
param_values_t const params = config->param_values;
size_t i;
for (i = 0; i < params.size; ++i) {
if (params.data[i].param == ZSTD_c_compressionLevel)
return (int)params.data[i].value;
}
return CONFIG_NO_LEVEL;
}
ZSTD_parameters config_get_zstd_params(
config_t const* config,
uint64_t srcSize,
size_t dictSize)
{
ZSTD_parameters zparams = {};
param_values_t const params = config->param_values;
int level = config_get_level(config);
if (level == CONFIG_NO_LEVEL)
level = 3;
zparams = ZSTD_getParams(
level,
config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN : srcSize,
dictSize);
for (size_t i = 0; i < params.size; ++i) {
unsigned const value = params.data[i].value;
switch (params.data[i].param) {
case ZSTD_c_contentSizeFlag:
zparams.fParams.contentSizeFlag = value;
break;
case ZSTD_c_checksumFlag:
zparams.fParams.checksumFlag = value;
break;
case ZSTD_c_dictIDFlag:
zparams.fParams.noDictIDFlag = !value;
break;
case ZSTD_c_windowLog:
zparams.cParams.windowLog = value;
break;
case ZSTD_c_chainLog:
zparams.cParams.chainLog = value;
break;
case ZSTD_c_hashLog:
zparams.cParams.hashLog = value;
break;
case ZSTD_c_searchLog:
zparams.cParams.searchLog = value;
break;
case ZSTD_c_minMatch:
zparams.cParams.minMatch = value;
break;
case ZSTD_c_targetLength:
zparams.cParams.targetLength = value;
break;
case ZSTD_c_strategy:
zparams.cParams.strategy = (ZSTD_strategy)value;
break;
default:
break;
}
}
return zparams;
}

86
sys/contrib/zstd/tests/regression/config.h Normal file
View File

@ -0,0 +1,86 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef CONFIG_H
#define CONFIG_H
#include <stddef.h>
#define ZSTD_STATIC_LINKING_ONLY
#include <zstd.h>
#include "data.h"
typedef struct {
ZSTD_cParameter param;
int value;
} param_value_t;
typedef struct {
size_t size;
param_value_t const* data;
} param_values_t;
/**
* The config tells the compression method what options to use.
*/
typedef struct {
const char* name; /**< Identifies the config in the results table */
/**
* Optional arguments to pass to the CLI. If not set, CLI-based methods
* will skip this config.
*/
char const* cli_args;
/**
* Parameters to pass to the advanced API. If the advanced API isn't used,
* the parameters will be derived from these.
*/
param_values_t param_values;
/**
* Boolean parameter that says if we should use a dictionary. If the data
* doesn't have a dictionary, this config is skipped. Defaults to no.
*/
int use_dictionary;
/**
* Boolean parameter that says if we should pass the pledged source size
* when the method allows it. Defaults to yes.
*/
int no_pledged_src_size;
} config_t;
/**
* Returns true if the config should skip this data.
* For instance, if the config requires a dictionary but the data doesn't have
* one.
*/
int config_skip_data(config_t const* config, data_t const* data);
#define CONFIG_NO_LEVEL (-ZSTD_TARGETLENGTH_MAX - 1)
/**
* Returns the compression level specified by the config, or CONFIG_NO_LEVEL if
* no level is specified. Note that 0 is a valid compression level, meaning
* default.
*/
int config_get_level(config_t const* config);
/**
* Returns the compression parameters specified by the config.
*/
ZSTD_parameters config_get_zstd_params(
config_t const* config,
uint64_t srcSize,
size_t dictSize);
/**
* The NULL-terminated list of configs.
*/
extern config_t const* const* configs;
#endif

617
sys/contrib/zstd/tests/regression/data.c Normal file
View File

@ -0,0 +1,617 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include "data.h"
#include <assert.h>
#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <curl/curl.h>
#include "mem.h"
#include "util.h"
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h"
/**
* Data objects
*/
#define REGRESSION_RELEASE(x) \
"https://github.com/facebook/zstd/releases/download/regression-data/" x
data_t silesia = {
.name = "silesia",
.type = data_type_dir,
.data =
{
.url = REGRESSION_RELEASE("silesia.tar.zst"),
.xxhash64 = 0x48a199f92f93e977LL,
},
};
data_t silesia_tar = {
.name = "silesia.tar",
.type = data_type_file,
.data =
{
.url = REGRESSION_RELEASE("silesia.tar.zst"),
.xxhash64 = 0x48a199f92f93e977LL,
},
};
data_t github = {
.name = "github",
.type = data_type_dir,
.data =
{
.url = REGRESSION_RELEASE("github.tar.zst"),
.xxhash64 = 0xa9b1b44b020df292LL,
},
.dict =
{
.url = REGRESSION_RELEASE("github.dict.zst"),
.xxhash64 = 0x1eddc6f737d3cb53LL,
},
};
static data_t* g_data[] = {
&silesia,
&silesia_tar,
&github,
NULL,
};
data_t const* const* data = (data_t const* const*)g_data;
/**
* data helpers.
*/
int data_has_dict(data_t const* data) {
return data->dict.url != NULL;
}
/**
* data buffer helper functions (documented in header).
*/
data_buffer_t data_buffer_create(size_t const capacity) {
data_buffer_t buffer = {};
buffer.data = (uint8_t*)malloc(capacity);
if (buffer.data == NULL)
return buffer;
buffer.capacity = capacity;
return buffer;
}
data_buffer_t data_buffer_read(char const* filename) {
data_buffer_t buffer = {};
uint64_t const size = UTIL_getFileSize(filename);
if (size == UTIL_FILESIZE_UNKNOWN) {
fprintf(stderr, "unknown size for %s\n", filename);
return buffer;
}
buffer.data = (uint8_t*)malloc(size);
if (buffer.data == NULL) {
fprintf(stderr, "malloc failed\n");
return buffer;
}
buffer.capacity = size;
FILE* file = fopen(filename, "rb");
if (file == NULL) {
fprintf(stderr, "file null\n");
goto err;
}
buffer.size = fread(buffer.data, 1, buffer.capacity, file);
fclose(file);
if (buffer.size != buffer.capacity) {
fprintf(stderr, "read %zu != %zu\n", buffer.size, buffer.capacity);
goto err;
}
return buffer;
err:
free(buffer.data);
memset(&buffer, 0, sizeof(buffer));
return buffer;
}
data_buffer_t data_buffer_get_data(data_t const* data) {
data_buffer_t const kEmptyBuffer = {};
if (data->type != data_type_file)
return kEmptyBuffer;
return data_buffer_read(data->data.path);
}
data_buffer_t data_buffer_get_dict(data_t const* data) {
data_buffer_t const kEmptyBuffer = {};
if (!data_has_dict(data))
return kEmptyBuffer;
return data_buffer_read(data->dict.path);
}
int data_buffer_compare(data_buffer_t buffer1, data_buffer_t buffer2) {
size_t const size =
buffer1.size < buffer2.size ? buffer1.size : buffer2.size;
int const cmp = memcmp(buffer1.data, buffer2.data, size);
if (cmp != 0)
return cmp;
if (buffer1.size < buffer2.size)
return -1;
if (buffer1.size == buffer2.size)
return 0;
assert(buffer1.size > buffer2.size);
return 1;
}
void data_buffer_free(data_buffer_t buffer) {
free(buffer.data);
}
/**
* data filenames helpers.
*/
data_filenames_t data_filenames_get(data_t const* data) {
data_filenames_t filenames = {.buffer = NULL, .size = 0};
char const* path = data->data.path;
filenames.filenames = UTIL_createFileList(
&path,
1,
&filenames.buffer,
&filenames.size,
/* followLinks */ 0);
return filenames;
}
void data_filenames_free(data_filenames_t filenames) {
UTIL_freeFileList(filenames.filenames, filenames.buffer);
}
/**
* data buffers helpers.
*/
data_buffers_t data_buffers_get(data_t const* data) {
data_buffers_t buffers = {.size = 0};
data_filenames_t filenames = data_filenames_get(data);
if (filenames.size == 0)
return buffers;
data_buffer_t* buffersPtr =
(data_buffer_t*)malloc(filenames.size * sizeof(data_buffer_t));
if (buffersPtr == NULL)
return buffers;
buffers.buffers = (data_buffer_t const*)buffersPtr;
buffers.size = filenames.size;
for (size_t i = 0; i < filenames.size; ++i) {
buffersPtr[i] = data_buffer_read(filenames.filenames[i]);
if (buffersPtr[i].data == NULL) {
data_buffers_t const kEmptyBuffer = {};
data_buffers_free(buffers);
return kEmptyBuffer;
}
}
return buffers;
}
/**
* Frees the data buffers.
*/
void data_buffers_free(data_buffers_t buffers) {
free((data_buffer_t*)buffers.buffers);
}
/**
* Initialization and download functions.
*/
static char* g_data_dir = NULL;
/* mkdir -p */
static int ensure_directory_exists(char const* indir) {
char* const dir = strdup(indir);
char* end = dir;
int ret = 0;
if (dir == NULL) {
ret = EINVAL;
goto out;
}
do {
/* Find the next directory level. */
for (++end; *end != '\0' && *end != '/'; ++end)
;
/* End the string there, make the directory, and restore the string. */
char const save = *end;
*end = '\0';
int const isdir = UTIL_isDirectory(dir);
ret = mkdir(dir, S_IRWXU);
*end = save;
/* Its okay if the directory already exists. */
if (ret == 0 || (errno == EEXIST && isdir))
continue;
ret = errno;
fprintf(stderr, "mkdir() failed\n");
goto out;
} while (*end != '\0');
ret = 0;
out:
free(dir);
return ret;
}
/** Concatenate 3 strings into a new buffer. */
static char* cat3(char const* str1, char const* str2, char const* str3) {
size_t const size1 = strlen(str1);
size_t const size2 = strlen(str2);
size_t const size3 = str3 == NULL ? 0 : strlen(str3);
size_t const size = size1 + size2 + size3 + 1;
char* const dst = (char*)malloc(size);
if (dst == NULL)
return NULL;
strcpy(dst, str1);
strcpy(dst + size1, str2);
if (str3 != NULL)
strcpy(dst + size1 + size2, str3);
assert(strlen(dst) == size1 + size2 + size3);
return dst;
}
static char* cat2(char const* str1, char const* str2) {
return cat3(str1, str2, NULL);
}
/**
* State needed by the curl callback.
* It takes data from curl, hashes it, and writes it to the file.
*/
typedef struct {
FILE* file;
XXH64_state_t xxhash64;
int error;
} curl_data_t;
/** Create the curl state. */
static curl_data_t curl_data_create(
data_resource_t const* resource,
data_type_t type) {
curl_data_t cdata = {};
XXH64_reset(&cdata.xxhash64, 0);
assert(UTIL_isDirectory(g_data_dir));
if (type == data_type_file) {
/* Decompress the resource and store to the path. */
char* cmd = cat3("zstd -dqfo '", resource->path, "'");
if (cmd == NULL) {
cdata.error = ENOMEM;
return cdata;
}
cdata.file = popen(cmd, "w");
free(cmd);
} else {
/* Decompress and extract the resource to the cache directory. */
char* cmd = cat3("zstd -dc | tar -x -C '", g_data_dir, "'");
if (cmd == NULL) {
cdata.error = ENOMEM;
return cdata;
}
cdata.file = popen(cmd, "w");
free(cmd);
}
if (cdata.file == NULL) {
cdata.error = errno;
}
return cdata;
}
/** Free the curl state. */
static int curl_data_free(curl_data_t cdata) {
return pclose(cdata.file);
}
/** curl callback. Updates the hash, and writes to the file. */
static size_t curl_write(void* data, size_t size, size_t count, void* ptr) {
curl_data_t* cdata = (curl_data_t*)ptr;
size_t const written = fwrite(data, size, count, cdata->file);
XXH64_update(&cdata->xxhash64, data, written * size);
return written;
}
static int curl_download_resource(
CURL* curl,
data_resource_t const* resource,
data_type_t type) {
curl_data_t cdata;
/* Download the data. */
if (curl_easy_setopt(curl, CURLOPT_URL, resource->url) != 0)
return EINVAL;
if (curl_easy_setopt(curl, CURLOPT_WRITEDATA, &cdata) != 0)
return EINVAL;
cdata = curl_data_create(resource, type);
if (cdata.error != 0)
return cdata.error;
int const curl_err = curl_easy_perform(curl);
int const close_err = curl_data_free(cdata);
if (curl_err) {
fprintf(
stderr,
"downloading '%s' for '%s' failed\n",
resource->url,
resource->path);
return EIO;
}
if (close_err) {
fprintf(stderr, "writing data to '%s' failed\n", resource->path);
return EIO;
}
/* check that the file exists. */
if (type == data_type_file && !UTIL_isRegularFile(resource->path)) {
fprintf(stderr, "output file '%s' does not exist\n", resource->path);
return EIO;
}
if (type == data_type_dir && !UTIL_isDirectory(resource->path)) {
fprintf(
stderr, "output directory '%s' does not exist\n", resource->path);
return EIO;
}
/* Check that the hash matches. */
if (XXH64_digest(&cdata.xxhash64) != resource->xxhash64) {
fprintf(
stderr,
"checksum does not match: 0x%llxLL != 0x%llxLL\n",
(unsigned long long)XXH64_digest(&cdata.xxhash64),
(unsigned long long)resource->xxhash64);
return EINVAL;
}
return 0;
}
/** Download a single data object. */
static int curl_download_datum(CURL* curl, data_t const* data) {
int ret;
ret = curl_download_resource(curl, &data->data, data->type);
if (ret != 0)
return ret;
if (data_has_dict(data)) {
ret = curl_download_resource(curl, &data->dict, data_type_file);
if (ret != 0)
return ret;
}
return ret;
}
/** Download all the data. */
static int curl_download_data(data_t const* const* data) {
if (curl_global_init(CURL_GLOBAL_ALL) != 0)
return EFAULT;
curl_data_t cdata = {};
CURL* curl = curl_easy_init();
int err = EFAULT;
if (curl == NULL)
return EFAULT;
if (curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L) != 0)
goto out;
if (curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L) != 0)
goto out;
if (curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write) != 0)
goto out;
assert(data != NULL);
for (; *data != NULL; ++data) {
if (curl_download_datum(curl, *data) != 0)
goto out;
}
err = 0;
out:
curl_easy_cleanup(curl);
curl_global_cleanup();
return err;
}
/** Fill the path member variable of the data objects. */
static int data_create_paths(data_t* const* data, char const* dir) {
size_t const dirlen = strlen(dir);
assert(data != NULL);
for (; *data != NULL; ++data) {
data_t* const datum = *data;
datum->data.path = cat3(dir, "/", datum->name);
if (datum->data.path == NULL)
return ENOMEM;
if (data_has_dict(datum)) {
datum->dict.path = cat2(datum->data.path, ".dict");
if (datum->dict.path == NULL)
return ENOMEM;
}
}
return 0;
}
/** Free the path member variable of the data objects. */
static void data_free_paths(data_t* const* data) {
assert(data != NULL);
for (; *data != NULL; ++data) {
data_t* datum = *data;
free((void*)datum->data.path);
free((void*)datum->dict.path);
datum->data.path = NULL;
datum->dict.path = NULL;
}
}
static char const kStampName[] = "STAMP";
static void xxh_update_le(XXH64_state_t* state, uint64_t data) {
if (!MEM_isLittleEndian())
data = MEM_swap64(data);
XXH64_update(state, &data, sizeof(data));
}
/** Hash the data to create the stamp. */
static uint64_t stamp_hash(data_t const* const* data) {
XXH64_state_t state;
XXH64_reset(&state, 0);
assert(data != NULL);
for (; *data != NULL; ++data) {
data_t const* datum = *data;
/* We don't care about the URL that we fetch from. */
/* The path is derived from the name. */
XXH64_update(&state, datum->name, strlen(datum->name));
xxh_update_le(&state, datum->data.xxhash64);
xxh_update_le(&state, datum->dict.xxhash64);
xxh_update_le(&state, datum->type);
}
return XXH64_digest(&state);
}
/** Check if the stamp matches the stamp in the cache directory. */
static int stamp_check(char const* dir, data_t const* const* data) {
char* stamp = cat3(dir, "/", kStampName);
uint64_t const expected = stamp_hash(data);
XXH64_canonical_t actual;
FILE* stampfile = NULL;
int matches = 0;
if (stamp == NULL)
goto out;
if (!UTIL_isRegularFile(stamp)) {
fprintf(stderr, "stamp does not exist: recreating the data cache\n");
goto out;
}
stampfile = fopen(stamp, "rb");
if (stampfile == NULL) {
fprintf(stderr, "could not open stamp: recreating the data cache\n");
goto out;
}
size_t b;
if ((b = fread(&actual, sizeof(actual), 1, stampfile)) != 1) {
fprintf(stderr, "invalid stamp: recreating the data cache\n");
goto out;
}
matches = (expected == XXH64_hashFromCanonical(&actual));
if (matches)
fprintf(stderr, "stamp matches: reusing the cached data\n");
else
fprintf(stderr, "stamp does not match: recreating the data cache\n");
out:
free(stamp);
if (stampfile != NULL)
fclose(stampfile);
return matches;
}
/** On success write a new stamp, on failure delete the old stamp. */
static int
stamp_write(char const* dir, data_t const* const* data, int const data_err) {
char* stamp = cat3(dir, "/", kStampName);
FILE* stampfile = NULL;
int err = EIO;
if (stamp == NULL)
return ENOMEM;
if (data_err != 0) {
err = data_err;
goto out;
}
XXH64_canonical_t hash;
XXH64_canonicalFromHash(&hash, stamp_hash(data));
stampfile = fopen(stamp, "wb");
if (stampfile == NULL)
goto out;
if (fwrite(&hash, sizeof(hash), 1, stampfile) != 1)
goto out;
err = 0;
fprintf(stderr, "stamped new data cache\n");
out:
if (err != 0)
/* Ignore errors. */
unlink(stamp);
free(stamp);
if (stampfile != NULL)
fclose(stampfile);
return err;
}
int data_init(char const* dir) {
int err;
if (dir == NULL)
return EINVAL;
/* This must be first to simplify logic. */
err = ensure_directory_exists(dir);
if (err != 0)
return err;
/* Save the cache directory. */
g_data_dir = strdup(dir);
if (g_data_dir == NULL)
return ENOMEM;
err = data_create_paths(g_data, dir);
if (err != 0)
return err;
/* If the stamp matches then we are good to go.
* This must be called before any modifications to the data cache.
* After this point, we MUST call stamp_write() to update the STAMP,
* since we've updated the data cache.
*/
if (stamp_check(dir, data))
return 0;
err = curl_download_data(data);
if (err != 0)
goto out;
out:
/* This must be last, since it must know if data_init() succeeded. */
stamp_write(dir, data, err);
return err;
}
void data_finish(void) {
data_free_paths(g_data);
free(g_data_dir);
g_data_dir = NULL;
}

140
sys/contrib/zstd/tests/regression/data.h Normal file
View File

@ -0,0 +1,140 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef DATA_H
#define DATA_H
#include <stddef.h>
#include <stdint.h>
typedef enum {
data_type_file = 1, /**< This data is a file. *.zst */
data_type_dir = 2, /**< This data is a directory. *.tar.zst */
} data_type_t;
typedef struct {
char const* url; /**< Where to get this resource. */
uint64_t xxhash64; /**< Hash of the url contents. */
char const* path; /**< The path of the unpacked resource (derived). */
} data_resource_t;
typedef struct {
data_resource_t data;
data_resource_t dict;
data_type_t type; /**< The type of the data. */
char const* name; /**< The logical name of the data (no extension). */
} data_t;
/**
* The NULL-terminated list of data objects.
*/
extern data_t const* const* data;
int data_has_dict(data_t const* data);
/**
* Initializes the data module and downloads the data necessary.
* Caches the downloads in dir. We add a stamp file in the directory after
* a successful download. If a stamp file already exists, and matches our
* current data stamp, we will use the cached data without downloading.
*
* @param dir The directory to cache the downloaded data into.
*
* @returns 0 on success.
*/
int data_init(char const* dir);
/**
* Must be called at exit to free resources allocated by data_init().
*/
void data_finish(void);
typedef struct {
uint8_t* data;
size_t size;
size_t capacity;
} data_buffer_t;
/**
* Read the file that data points to into a buffer.
* NOTE: data must be a file, not a directory.
*
* @returns The buffer, which is NULL on failure.
*/
data_buffer_t data_buffer_get_data(data_t const* data);
/**
* Read the dictionary that the data points to into a buffer.
*
* @returns The buffer, which is NULL on failure.
*/
data_buffer_t data_buffer_get_dict(data_t const* data);
/**
* Read the contents of filename into a buffer.
*
* @returns The buffer, which is NULL on failure.
*/
data_buffer_t data_buffer_read(char const* filename);
/**
* Create a buffer with the specified capacity.
*
* @returns The buffer, which is NULL on failure.
*/
data_buffer_t data_buffer_create(size_t capacity);
/**
* Calls memcmp() on the contents [0, size) of both buffers.
*/
int data_buffer_compare(data_buffer_t buffer1, data_buffer_t buffer2);
/**
* Frees an allocated buffer.
*/
void data_buffer_free(data_buffer_t buffer);
typedef struct {
char* buffer;
char const** filenames;
unsigned size;
} data_filenames_t;
/**
* Get a recursive list of filenames in the data object. If it is a file, it
* will only contain one entry. If it is a directory, it will recursively walk
* the directory.
*
* @returns The list of filenames, which has size 0 and NULL pointers on error.
*/
data_filenames_t data_filenames_get(data_t const* data);
/**
* Frees the filenames table.
*/
void data_filenames_free(data_filenames_t filenames);
typedef struct {
data_buffer_t const* buffers;
size_t size;
} data_buffers_t;
/**
* @returns a list of buffers for every file in data. It is zero sized on error.
*/
data_buffers_t data_buffers_get(data_t const* data);
/**
* Frees the data buffers.
*/
void data_buffers_free(data_buffers_t buffers);
#endif

44
sys/contrib/zstd/tests/regression/levels.h Normal file
View File

@ -0,0 +1,44 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef LEVEL
# error LEVEL(x) must be defined
#endif
#ifndef FAST_LEVEL
# error FAST_LEVEL(x) must be defined
#endif
/**
* The levels are chosen to trigger every strategy in every source size,
* as well as some fast levels and the default level.
* If you change the compression levels, you should probably update these.
*/
FAST_LEVEL(5)
FAST_LEVEL(3)
FAST_LEVEL(1)
LEVEL(0)
LEVEL(1)
LEVEL(3)
LEVEL(4)
LEVEL(5)
LEVEL(6)
LEVEL(7)
LEVEL(9)
LEVEL(13)
LEVEL(16)
LEVEL(19)

566
sys/contrib/zstd/tests/regression/method.c Normal file
View File

@ -0,0 +1,566 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include "method.h"
#include <stdio.h>
#include <stdlib.h>
#define ZSTD_STATIC_LINKING_ONLY
#include <zstd.h>
#define MIN(x, y) ((x) < (y) ? (x) : (y))
static char const* g_zstdcli = NULL;
void method_set_zstdcli(char const* zstdcli) {
g_zstdcli = zstdcli;
}
/**
* Macro to get a pointer of type, given ptr, which is a member variable with
* the given name, member.
*
* method_state_t* base = ...;
* buffer_state_t* state = container_of(base, buffer_state_t, base);
*/
#define container_of(ptr, type, member) \
((type*)(ptr == NULL ? NULL : (char*)(ptr)-offsetof(type, member)))
/** State to reuse the same buffers between compression calls. */
typedef struct {
method_state_t base;
data_buffers_t inputs; /**< The input buffer for each file. */
data_buffer_t dictionary; /**< The dictionary. */
data_buffer_t compressed; /**< The compressed data buffer. */
data_buffer_t decompressed; /**< The decompressed data buffer. */
} buffer_state_t;
static size_t buffers_max_size(data_buffers_t buffers) {
size_t max = 0;
for (size_t i = 0; i < buffers.size; ++i) {
if (buffers.buffers[i].size > max)
max = buffers.buffers[i].size;
}
return max;
}
static method_state_t* buffer_state_create(data_t const* data) {
buffer_state_t* state = (buffer_state_t*)calloc(1, sizeof(buffer_state_t));
if (state == NULL)
return NULL;
state->base.data = data;
state->inputs = data_buffers_get(data);
state->dictionary = data_buffer_get_dict(data);
size_t const max_size = buffers_max_size(state->inputs);
state->compressed = data_buffer_create(ZSTD_compressBound(max_size));
state->decompressed = data_buffer_create(max_size);
return &state->base;
}
static void buffer_state_destroy(method_state_t* base) {
if (base == NULL)
return;
buffer_state_t* state = container_of(base, buffer_state_t, base);
free(state);
}
static int buffer_state_bad(
buffer_state_t const* state,
config_t const* config) {
if (state == NULL) {
fprintf(stderr, "buffer_state_t is NULL\n");
return 1;
}
if (state->inputs.size == 0 || state->compressed.data == NULL ||
state->decompressed.data == NULL) {
fprintf(stderr, "buffer state allocation failure\n");
return 1;
}
if (config->use_dictionary && state->dictionary.data == NULL) {
fprintf(stderr, "dictionary loading failed\n");
return 1;
}
return 0;
}
static result_t simple_compress(method_state_t* base, config_t const* config) {
buffer_state_t* state = container_of(base, buffer_state_t, base);
if (buffer_state_bad(state, config))
return result_error(result_error_system_error);
/* Keep the tests short by skipping directories, since behavior shouldn't
* change.
*/
if (base->data->type != data_type_file)
return result_error(result_error_skip);
if (config->use_dictionary || config->no_pledged_src_size)
return result_error(result_error_skip);
/* If the config doesn't specify a level, skip. */
int const level = config_get_level(config);
if (level == CONFIG_NO_LEVEL)
return result_error(result_error_skip);
data_buffer_t const input = state->inputs.buffers[0];
/* Compress, decompress, and check the result. */
state->compressed.size = ZSTD_compress(
state->compressed.data,
state->compressed.capacity,
input.data,
input.size,
level);
if (ZSTD_isError(state->compressed.size))
return result_error(result_error_compression_error);
state->decompressed.size = ZSTD_decompress(
state->decompressed.data,
state->decompressed.capacity,
state->compressed.data,
state->compressed.size);
if (ZSTD_isError(state->decompressed.size))
return result_error(result_error_decompression_error);
if (data_buffer_compare(input, state->decompressed))
return result_error(result_error_round_trip_error);
result_data_t data;
data.total_size = state->compressed.size;
return result_data(data);
}
static result_t compress_cctx_compress(
method_state_t* base,
config_t const* config) {
buffer_state_t* state = container_of(base, buffer_state_t, base);
if (buffer_state_bad(state, config))
return result_error(result_error_system_error);
if (config->no_pledged_src_size)
return result_error(result_error_skip);
if (base->data->type != data_type_dir)
return result_error(result_error_skip);
int const level = config_get_level(config);
ZSTD_CCtx* cctx = ZSTD_createCCtx();
ZSTD_DCtx* dctx = ZSTD_createDCtx();
if (cctx == NULL || dctx == NULL) {
fprintf(stderr, "context creation failed\n");
return result_error(result_error_system_error);
}
result_t result;
result_data_t data = {.total_size = 0};
for (size_t i = 0; i < state->inputs.size; ++i) {
data_buffer_t const input = state->inputs.buffers[i];
ZSTD_parameters const params =
config_get_zstd_params(config, input.size, state->dictionary.size);
if (level == CONFIG_NO_LEVEL)
state->compressed.size = ZSTD_compress_advanced(
cctx,
state->compressed.data,
state->compressed.capacity,
input.data,
input.size,
state->dictionary.data,
state->dictionary.size,
params);
else if (config->use_dictionary)
state->compressed.size = ZSTD_compress_usingDict(
cctx,
state->compressed.data,
state->compressed.capacity,
input.data,
input.size,
state->dictionary.data,
state->dictionary.size,
level);
else
state->compressed.size = ZSTD_compressCCtx(
cctx,
state->compressed.data,
state->compressed.capacity,
input.data,
input.size,
level);
if (ZSTD_isError(state->compressed.size)) {
result = result_error(result_error_compression_error);
goto out;
}
if (config->use_dictionary)
state->decompressed.size = ZSTD_decompress_usingDict(
dctx,
state->decompressed.data,
state->decompressed.capacity,
state->compressed.data,
state->compressed.size,
state->dictionary.data,
state->dictionary.size);
else
state->decompressed.size = ZSTD_decompressDCtx(
dctx,
state->decompressed.data,
state->decompressed.capacity,
state->compressed.data,
state->compressed.size);
if (ZSTD_isError(state->decompressed.size)) {
result = result_error(result_error_decompression_error);
goto out;
}
if (data_buffer_compare(input, state->decompressed)) {
result = result_error(result_error_round_trip_error);
goto out;
}
data.total_size += state->compressed.size;
}
result = result_data(data);
out:
ZSTD_freeCCtx(cctx);
ZSTD_freeDCtx(dctx);
return result;
}
/** Generic state creation function. */
static method_state_t* method_state_create(data_t const* data) {
method_state_t* state = (method_state_t*)malloc(sizeof(method_state_t));
if (state == NULL)
return NULL;
state->data = data;
return state;
}
static void method_state_destroy(method_state_t* state) {
free(state);
}
static result_t cli_compress(method_state_t* state, config_t const* config) {
if (config->cli_args == NULL)
return result_error(result_error_skip);
/* We don't support no pledged source size with directories. Too slow. */
if (state->data->type == data_type_dir && config->no_pledged_src_size)
return result_error(result_error_skip);
if (g_zstdcli == NULL)
return result_error(result_error_system_error);
/* '<zstd>' -cqr <args> [-D '<dict>'] '<file/dir>' */
char cmd[1024];
size_t const cmd_size = snprintf(
cmd,
sizeof(cmd),
"'%s' -cqr %s %s%s%s %s '%s'",
g_zstdcli,
config->cli_args,
config->use_dictionary ? "-D '" : "",
config->use_dictionary ? state->data->dict.path : "",
config->use_dictionary ? "'" : "",
config->no_pledged_src_size ? "<" : "",
state->data->data.path);
if (cmd_size >= sizeof(cmd)) {
fprintf(stderr, "command too large: %s\n", cmd);
return result_error(result_error_system_error);
}
FILE* zstd = popen(cmd, "r");
if (zstd == NULL) {
fprintf(stderr, "failed to popen command: %s\n", cmd);
return result_error(result_error_system_error);
}
char out[4096];
size_t total_size = 0;
while (1) {
size_t const size = fread(out, 1, sizeof(out), zstd);
total_size += size;
if (size != sizeof(out))
break;
}
if (ferror(zstd) || pclose(zstd) != 0) {
fprintf(stderr, "zstd failed with command: %s\n", cmd);
return result_error(result_error_compression_error);
}
result_data_t const data = {.total_size = total_size};
return result_data(data);
}
static int advanced_config(
ZSTD_CCtx* cctx,
buffer_state_t* state,
config_t const* config) {
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
for (size_t p = 0; p < config->param_values.size; ++p) {
param_value_t const pv = config->param_values.data[p];
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, pv.param, pv.value))) {
return 1;
}
}
if (config->use_dictionary) {
if (ZSTD_isError(ZSTD_CCtx_loadDictionary(
cctx, state->dictionary.data, state->dictionary.size))) {
return 1;
}
}
return 0;
}
static result_t advanced_one_pass_compress_output_adjustment(
method_state_t* base,
config_t const* config,
size_t const subtract) {
buffer_state_t* state = container_of(base, buffer_state_t, base);
if (buffer_state_bad(state, config))
return result_error(result_error_system_error);
ZSTD_CCtx* cctx = ZSTD_createCCtx();
result_t result;
if (!cctx || advanced_config(cctx, state, config)) {
result = result_error(result_error_compression_error);
goto out;
}
result_data_t data = {.total_size = 0};
for (size_t i = 0; i < state->inputs.size; ++i) {
data_buffer_t const input = state->inputs.buffers[i];
if (!config->no_pledged_src_size) {
if (ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(cctx, input.size))) {
result = result_error(result_error_compression_error);
goto out;
}
}
size_t const size = ZSTD_compress2(
cctx,
state->compressed.data,
ZSTD_compressBound(input.size) - subtract,
input.data,
input.size);
if (ZSTD_isError(size)) {
result = result_error(result_error_compression_error);
goto out;
}
data.total_size += size;
}
result = result_data(data);
out:
ZSTD_freeCCtx(cctx);
return result;
}
static result_t advanced_one_pass_compress(
method_state_t* base,
config_t const* config) {
return advanced_one_pass_compress_output_adjustment(base, config, 0);
}
static result_t advanced_one_pass_compress_small_output(
method_state_t* base,
config_t const* config) {
return advanced_one_pass_compress_output_adjustment(base, config, 1);
}
static result_t advanced_streaming_compress(
method_state_t* base,
config_t const* config) {
buffer_state_t* state = container_of(base, buffer_state_t, base);
if (buffer_state_bad(state, config))
return result_error(result_error_system_error);
ZSTD_CCtx* cctx = ZSTD_createCCtx();
result_t result;
if (!cctx || advanced_config(cctx, state, config)) {
result = result_error(result_error_compression_error);
goto out;
}
result_data_t data = {.total_size = 0};
for (size_t i = 0; i < state->inputs.size; ++i) {
data_buffer_t input = state->inputs.buffers[i];
if (!config->no_pledged_src_size) {
if (ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(cctx, input.size))) {
result = result_error(result_error_compression_error);
goto out;
}
}
while (input.size > 0) {
ZSTD_inBuffer in = {input.data, MIN(input.size, 4096)};
input.data += in.size;
input.size -= in.size;
ZSTD_EndDirective const op =
input.size > 0 ? ZSTD_e_continue : ZSTD_e_end;
size_t ret = 0;
while (in.pos < in.size || (op == ZSTD_e_end && ret != 0)) {
ZSTD_outBuffer out = {state->compressed.data,
MIN(state->compressed.capacity, 1024)};
ret = ZSTD_compressStream2(cctx, &out, &in, op);
if (ZSTD_isError(ret)) {
result = result_error(result_error_compression_error);
goto out;
}
data.total_size += out.pos;
}
}
}
result = result_data(data);
out:
ZSTD_freeCCtx(cctx);
return result;
}
static result_t old_streaming_compress(
method_state_t* base,
config_t const* config) {
buffer_state_t* state = container_of(base, buffer_state_t, base);
if (buffer_state_bad(state, config))
return result_error(result_error_system_error);
int const level = config_get_level(config);
if (level == CONFIG_NO_LEVEL)
return result_error(result_error_skip);
ZSTD_CStream* zcs = ZSTD_createCStream();
result_t result;
if (zcs == NULL) {
result = result_error(result_error_compression_error);
goto out;
}
size_t zret;
if (config->use_dictionary) {
zret = ZSTD_initCStream_usingDict(
zcs, state->dictionary.data, state->dictionary.size, level);
} else {
zret = ZSTD_initCStream(zcs, level);
}
if (ZSTD_isError(zret)) {
result = result_error(result_error_compression_error);
goto out;
}
result_data_t data = {.total_size = 0};
for (size_t i = 0; i < state->inputs.size; ++i) {
data_buffer_t input = state->inputs.buffers[i];
zret = ZSTD_resetCStream(
zcs,
config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN
: input.size);
if (ZSTD_isError(zret)) {
result = result_error(result_error_compression_error);
goto out;
}
while (input.size > 0) {
ZSTD_inBuffer in = {input.data, MIN(input.size, 4096)};
input.data += in.size;
input.size -= in.size;
ZSTD_EndDirective const op =
input.size > 0 ? ZSTD_e_continue : ZSTD_e_end;
zret = 0;
while (in.pos < in.size || (op == ZSTD_e_end && zret != 0)) {
ZSTD_outBuffer out = {state->compressed.data,
MIN(state->compressed.capacity, 1024)};
if (op == ZSTD_e_continue || in.pos < in.size)
zret = ZSTD_compressStream(zcs, &out, &in);
else
zret = ZSTD_endStream(zcs, &out);
if (ZSTD_isError(zret)) {
result = result_error(result_error_compression_error);
goto out;
}
data.total_size += out.pos;
}
}
}
result = result_data(data);
out:
ZSTD_freeCStream(zcs);
return result;
}
method_t const simple = {
.name = "compress simple",
.create = buffer_state_create,
.compress = simple_compress,
.destroy = buffer_state_destroy,
};
method_t const compress_cctx = {
.name = "compress cctx",
.create = buffer_state_create,
.compress = compress_cctx_compress,
.destroy = buffer_state_destroy,
};
method_t const advanced_one_pass = {
.name = "advanced one pass",
.create = buffer_state_create,
.compress = advanced_one_pass_compress,
.destroy = buffer_state_destroy,
};
method_t const advanced_one_pass_small_out = {
.name = "advanced one pass small out",
.create = buffer_state_create,
.compress = advanced_one_pass_compress,
.destroy = buffer_state_destroy,
};
method_t const advanced_streaming = {
.name = "advanced streaming",
.create = buffer_state_create,
.compress = advanced_streaming_compress,
.destroy = buffer_state_destroy,
};
method_t const old_streaming = {
.name = "old streaming",
.create = buffer_state_create,
.compress = old_streaming_compress,
.destroy = buffer_state_destroy,
};
method_t const cli = {
.name = "zstdcli",
.create = method_state_create,
.compress = cli_compress,
.destroy = method_state_destroy,
};
static method_t const* g_methods[] = {
&simple,
&compress_cctx,
&cli,
&advanced_one_pass,
&advanced_one_pass_small_out,
&advanced_streaming,
&old_streaming,
NULL,
};
method_t const* const* methods = g_methods;

65
sys/contrib/zstd/tests/regression/method.h Normal file
View File

@ -0,0 +1,65 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef METHOD_H
#define METHOD_H
#include <stddef.h>
#include "data.h"
#include "config.h"
#include "result.h"
/**
* The base class for state that methods keep.
* All derived method state classes must have a member of this type.
*/
typedef struct {
data_t const* data;
} method_state_t;
/**
* A method that compresses the data using config.
*/
typedef struct {
char const* name; /**< The identifier for this method in the results. */
/**
* Creates a state that must contain a member variable of method_state_t,
* and returns a pointer to that member variable.
*
* This method can be used to do expensive work that only depends on the
* data, like loading the data file into a buffer.
*/
method_state_t* (*create)(data_t const* data);
/**
* Compresses the data in the state using the given config.
*
* @param state A pointer to the state returned by create().
*
* @returns The total compressed size on success, or an error code.
*/
result_t (*compress)(method_state_t* state, config_t const* config);
/**
* Frees the state.
*/
void (*destroy)(method_state_t* state);
} method_t;
/**
* Set the zstd cli path. Must be called before any methods are used.
*/
void method_set_zstdcli(char const* zstdcli);
/**
* A NULL-terminated list of methods.
*/
extern method_t const* const* methods;
#endif

Some files were not shown because too many files have changed in this diff Show More