import zstd 1.3.8
This commit is contained in:
cem
parent
1955f02623
commit
70d64c5852
10
Makefile
10
Makefile
@ -64,7 +64,8 @@ zlibwrapper: lib
|
||||
|
||||
## test: run long-duration tests
|
||||
.PHONY: test
|
||||
test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror
|
||||
DEBUGLEVEL ?= 1
|
||||
test: MOREFLAGS += -g -DDEBUGLEVEL=$(DEBUGLEVEL) -Werror
|
||||
test:
|
||||
MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
|
||||
$(MAKE) -C $(TESTDIR) $@
|
||||
@ -129,7 +130,12 @@ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD Dr
|
||||
HOST_OS = POSIX
|
||||
CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
|
||||
|
||||
EGREP = egrep --color=never
|
||||
HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
|
||||
EGREP_OPTIONS ?=
|
||||
ifeq ($HAVE_COLORNEVER, 1)
|
||||
EGREP_OPTIONS += --color=never
|
||||
endif
|
||||
EGREP = egrep $(EGREP_OPTIONS)
|
||||
|
||||
# Print a two column output of targets and their description. To add a target description, put a
|
||||
# comment in the Makefile with the format "## <TARGET>: <DESCRIPTION>". For example:
|
||||
|
||||
402
NEWS
402
NEWS
@ -1,402 +0,0 @@
|
||||
v1.3.7
|
||||
perf: slightly better decompression speed on clang (depending on hardware target)
|
||||
fix : performance of dictionary compression for small input < 4 KB at levels 9 and 10
|
||||
build: no longer build backtrace by default in release mode; restrict further automatic mode
|
||||
build: control backtrace support through build macro BACKTRACE
|
||||
misc: added man pages for zstdless and zstdgrep, by @samrussell
|
||||
|
||||
v1.3.6
|
||||
perf: much faster dictionary builder, by @jenniferliu
|
||||
perf: faster dictionary compression on small data when using multiple contexts, by @felixhandte
|
||||
perf: faster dictionary decompression when using a very large number of dictionaries simultaneously
|
||||
cli : fix : does no longer overwrite destination when source does not exist (#1082)
|
||||
cli : new command --adapt, for automatic compression level adaptation
|
||||
api : fix : block api can be streamed with > 4 GB, reported by @catid
|
||||
api : reduced ZSTD_DDict size by 2 KB
|
||||
api : minimum negative compression level is defined, and can be queried using ZSTD_minCLevel().
|
||||
build: support Haiku target, by @korli
|
||||
build: Read Legacy format is limited to v0.5+ by default. Can be changed at compile time with macro ZSTD_LEGACY_SUPPORT.
|
||||
doc : zstd_compression_format.md updated to match wording in IETF RFC 8478
|
||||
misc: tests/paramgrill, a parameter optimizer, by @GeorgeLu97
|
||||
|
||||
v1.3.5
|
||||
perf: much faster dictionary compression, by @felixhandte
|
||||
perf: small quality improvement for dictionary generation, by @terrelln
|
||||
perf: slightly improved high compression levels (notably level 19)
|
||||
mem : automatic memory release for long duration contexts
|
||||
cli : fix : overlapLog can be manually set
|
||||
cli : fix : decoding invalid lz4 frames
|
||||
api : fix : performance degradation for dictionary compression when using advanced API, by @terrelln
|
||||
api : change : clarify ZSTD_CCtx_reset() vs ZSTD_CCtx_resetParameters(), by @terrelln
|
||||
build: select custom libzstd scope through control macros, by @GeorgeLu97
|
||||
build: OpenBSD patch, by @bket
|
||||
build: make and make all are compatible with -j
|
||||
doc : clarify zstd_compression_format.md, updated for IETF RFC process
|
||||
misc: pzstd compatible with reproducible compilation, by @lamby
|
||||
|
||||
v1.3.4
|
||||
perf: faster speed (especially decoding speed) on recent cpus (haswell+)
|
||||
perf: much better performance associating --long with multi-threading, by @terrelln
|
||||
perf: better compression at levels 13-15
|
||||
cli : asynchronous compression by default, for faster experience (use --single-thread for former behavior)
|
||||
cli : smoother status report in multi-threading mode
|
||||
cli : added command --fast=#, for faster compression modes
|
||||
cli : fix crash when not overwriting existing files, by Pádraig Brady (@pixelb)
|
||||
api : `nbThreads` becomes `nbWorkers` : 1 triggers asynchronous mode
|
||||
api : compression levels can be negative, for even more speed
|
||||
api : ZSTD_getFrameProgression() : get precise progress status of ZSTDMT anytime
|
||||
api : ZSTDMT can accept new compression parameters during compression
|
||||
api : implemented all advanced dictionary decompression prototypes
|
||||
build: improved meson recipe, by Shawn Landden (@shawnl)
|
||||
build: VS2017 scripts, by @HaydnTrigg
|
||||
misc: all /contrib projects fixed
|
||||
misc: added /contrib/docker script by @gyscos
|
||||
|
||||
v1.3.3
|
||||
perf: faster zstd_opt strategy (levels 16-19)
|
||||
fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen
|
||||
cli : fix : content size written in header by default
|
||||
cli : fix : improved LZ4 format support, by @felixhandte
|
||||
cli : new : hidden command `-S`, to benchmark multiple files while generating one result per file
|
||||
api : fix : support large skippable frames, by @terrelln
|
||||
api : fix : streaming interface was adding a useless 3-bytes null block to small frames
|
||||
api : change : when setting `pledgedSrcSize`, use `ZSTD_CONTENTSIZE_UNKNOWN` macro value to mean "unknown"
|
||||
build: fix : compilation under rhel6 and centos6, reported by @pixelb
|
||||
build: added `check` target
|
||||
|
||||
v1.3.2
|
||||
new : long range mode, using --long command, by Stella Lau (@stellamplau)
|
||||
new : ability to generate and decode magicless frames (#591)
|
||||
changed : maximum nb of threads reduced to 200, to avoid address space exhaustion in 32-bits mode
|
||||
fix : multi-threading compression works with custom allocators
|
||||
fix : ZSTD_sizeof_CStream() was over-evaluating memory usage
|
||||
fix : a rare compression bug when compression generates very large distances and bunch of other conditions (only possible at --ultra -22)
|
||||
fix : 32-bits build can now decode large offsets (levels 21+)
|
||||
cli : added LZ4 frame support by default, by Felix Handte (@felixhandte)
|
||||
cli : improved --list output
|
||||
cli : new : can split input file for dictionary training, using command -B#
|
||||
cli : new : clean operation artefact on Ctrl-C interruption
|
||||
cli : fix : do not change /dev/null permissions when using command -t with root access, reported by @mike155 (#851)
|
||||
cli : fix : write file size in header in multiple-files mode
|
||||
api : added macro ZSTD_COMPRESSBOUND() for static allocation
|
||||
api : experimental : new advanced decompression API
|
||||
api : fix : sizeof_CCtx() used to over-estimate
|
||||
build: fix : no-multithread variant compiles without pool.c dependency, reported by Mitchell Blank Jr (@mitchblank) (#819)
|
||||
build: better compatibility with reproducible builds, by Bernhard M. Wiedemann (@bmwiedemann) (#818)
|
||||
example : added streaming_memory_usage
|
||||
license : changed /examples license to BSD + GPLv2
|
||||
license : fix a few header files to reflect new license (#825)
|
||||
|
||||
v1.3.1
|
||||
New license : BSD + GPLv2
|
||||
perf: substantially decreased memory usage in Multi-threading mode, thanks to reports by Tino Reichardt (@mcmilk)
|
||||
perf: Multi-threading supports up to 256 threads. Cap at 256 when more are requested (#760)
|
||||
cli : improved and fixed --list command, by @ib (#772)
|
||||
cli : command -vV to list supported formats, by @ib (#771)
|
||||
build : fixed binary variants, reported by @svenha (#788)
|
||||
build : fix Visual compilation for non x86/x64 targets, reported by Greg Slazinski (@GregSlazinski) (#718)
|
||||
API exp : breaking change : ZSTD_getframeHeader() provides more information
|
||||
API exp : breaking change : pinned down values of error codes
|
||||
doc : fixed huffman example, by Ulrich Kunitz (@ulikunitz)
|
||||
new : contrib/adaptive-compression, I/O driven compression strength, by Paul Cruz (@paulcruz74)
|
||||
new : contrib/long_distance_matching, statistics by Stella Lau (@stellamplau)
|
||||
updated : contrib/linux-kernel, by Nick Terrell (@terrelln)
|
||||
|
||||
v1.3.0
|
||||
cli : new : `--list` command, by Paul Cruz
|
||||
cli : changed : xz/lzma support enabled by default
|
||||
cli : changed : `-t *` continue processing list after a decompression error
|
||||
API : added : ZSTD_versionString()
|
||||
API : promoted to stable status : ZSTD_getFrameContentSize(), by Sean Purcell
|
||||
API exp : new advanced API : ZSTD_compress_generic(), ZSTD_CCtx_setParameter()
|
||||
API exp : new : API for static or external allocation : ZSTD_initStatic?Ctx()
|
||||
API exp : added : ZSTD_decompressBegin_usingDDict(), requested by Guy Riddle (#700)
|
||||
API exp : clarified memory estimation / measurement functions.
|
||||
API exp : changed : strongest strategy renamed ZSTD_btultra, fastest strategy ZSTD_fast set to 1
|
||||
tools : decodecorpus can generate random dictionary-compressed samples, by Paul Cruz
|
||||
new : contrib/seekable_format, demo and API, by Sean Purcell
|
||||
changed : contrib/linux-kernel, updated version and license, by Nick Terrell
|
||||
|
||||
v1.2.0
|
||||
cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable)
|
||||
cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell
|
||||
cli : new : zstdmt symlink hardwired to `zstd -T0`
|
||||
cli : new : command --threads=# (#671)
|
||||
cli : changed : cover dictionary builder by default, for improved quality, by Nick Terrell
|
||||
cli : new : commands --train-cover and --train-legacy, to select dictionary algorithm and parameters
|
||||
cli : experimental targets `zstd4` and `xzstd4`, with support for lz4 format, by Sean Purcell
|
||||
cli : fix : does not output compressed data on console
|
||||
cli : fix : ignore symbolic links unless --force specified,
|
||||
API : breaking change : ZSTD_createCDict_advanced(), only use compressionParameters as argument
|
||||
API : added : prototypes ZSTD_*_usingCDict_advanced(), for direct control over frameParameters.
|
||||
API : improved: ZSTDMT_compressCCtx() reduced memory usage
|
||||
API : fix : ZSTDMT_compressCCtx() now provides srcSize in header (#634)
|
||||
API : fix : src size stored in frame header is controlled at end of frame
|
||||
API : fix : enforced consistent rules for pledgedSrcSize==0 (#641)
|
||||
API : fix : error code "GENERIC" replaced by "dstSizeTooSmall" when appropriate
|
||||
build: improved cmake script, by @Majlen
|
||||
build: enabled Multi-threading support for *BSD, by Baptiste Daroussin
|
||||
tools: updated Paramgrill. Command -O# provides best parameters for sample and speed target.
|
||||
new : contrib/linux-kernel version, by Nick Terrell
|
||||
|
||||
v1.1.4
|
||||
cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski
|
||||
cli : new : advanced benchmark command --priority=rt
|
||||
cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
|
||||
cli : fix : --rm remains silent when input is stdin
|
||||
cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski
|
||||
speed : improved decompression speed in streaming mode for single shot scenarios (+5%)
|
||||
memory: DDict (decompression dictionary) memory usage down from 150 KB to 20 KB
|
||||
arch: 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell
|
||||
API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize()
|
||||
API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value)
|
||||
build : new: meson build system in contrib/meson, by Dima Krasner
|
||||
build : improved cmake script, by @Majlen
|
||||
build : added -Wformat-security flag, as recommended by Padraig Brady
|
||||
doc : new : educational decoder, by Sean Purcell
|
||||
|
||||
v1.1.3
|
||||
cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`)
|
||||
cli : new : experimental target `make zstdmt`, with multi-threading support
|
||||
cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell, based on prior work by Giuseppe Ottaviano.
|
||||
cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski
|
||||
cli : fix zstdless on Mac OS-X, by Andrew Janke
|
||||
cli : fix #232 "compress non-files"
|
||||
dictBuilder : improved dictionary generation quality, thanks to Nick Terrell
|
||||
API : new : lib/compress/ZSTDMT_compress.h multithreading API (experimental)
|
||||
API : new : ZSTD_create?Dict_byReference(), requested by Bartosz Taudul
|
||||
API : new : ZDICT_finalizeDictionary()
|
||||
API : fix : ZSTD_initCStream_usingCDict() properly writes dictID into frame header, by Gregory Szorc (#511)
|
||||
API : fix : all symbols properly exposed in libzstd, by Nick Terrell
|
||||
build : support for Solaris target, by Przemyslaw Skibinski
|
||||
doc : clarified specification, by Sean Purcell
|
||||
|
||||
v1.1.2
|
||||
API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init
|
||||
API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize()
|
||||
API : zbuff : changed : prototypes now generate deprecation warnings
|
||||
lib : improved : faster decompression speed at ultra compression settings and 32-bits mode
|
||||
lib : changed : only public ZSTD_ symbols are now exposed
|
||||
lib : changed : reduced usage of stack memory
|
||||
lib : fixed : several corner case bugs, by Nick Terrell
|
||||
cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski
|
||||
cli : new : preserve file attributes
|
||||
cli : new : added zstdless and zstdgrep tools
|
||||
cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd)
|
||||
cli : fixed : zstdcat
|
||||
zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski
|
||||
install : better compatibility with FreeBSD, by Dimitry Andric
|
||||
source tree : changed : zbuff source files moved to lib/deprecated
|
||||
|
||||
v1.1.1
|
||||
New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption
|
||||
New : doc/zstd_manual.html, by Przemyslaw Skibinski
|
||||
Improved : slightly better compression ratio at --ultra levels (>= 20)
|
||||
Improved : better memory usage when using streaming compression API, thanks to @Rogier-5 report
|
||||
Added : API : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section)
|
||||
Added : example/multiple_streaming_compression.c
|
||||
Changed : zstd_errors.h is now installed within /include (and replaces errors_public.h)
|
||||
Updated man page
|
||||
Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets
|
||||
|
||||
v1.1.0
|
||||
New : contrib/pzstd, parallel version of zstd, by Nick Terrell
|
||||
added : NetBSD install target (#338)
|
||||
Improved : speed for batches of small files
|
||||
Improved : speed of zlib wrapper, by Przemyslaw Skibinski
|
||||
Changed : libzstd on Windows supports legacy formats, by Christophe Chevalier
|
||||
Fixed : CLI -d output to stdout by default when input is stdin (#322)
|
||||
Fixed : CLI correctly detects console on Mac OS-X
|
||||
Fixed : CLI supports recursive mode `-r` on Mac OS-X
|
||||
Fixed : Legacy decoders use unified error codes, reported by benrg (#341), fixed by Przemyslaw Skibinski
|
||||
Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319)
|
||||
Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365)
|
||||
Fixed : zstd-pgo, reported by octoploid (#329)
|
||||
|
||||
v1.0.0
|
||||
Change Licensing, all project is now BSD, Copyright Facebook
|
||||
Small decompression speed improvement
|
||||
API : Streaming API supports legacy format
|
||||
API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParamter()
|
||||
CLI supports legacy formats v0.4+
|
||||
Fixed : compression fails on certain huge files, reported by Jesse McGrew
|
||||
Enhanced documentation, by Przemyslaw Skibinski
|
||||
|
||||
v0.8.1
|
||||
New streaming API
|
||||
Changed : --ultra now enables levels beyond 19
|
||||
Changed : -i# now selects benchmark time in second
|
||||
Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell
|
||||
Fixed : speed regression on specific patterns (#272)
|
||||
Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291)
|
||||
Fixed : ICC compilation, by Przemyslaw Skibinski
|
||||
|
||||
v0.8.0
|
||||
Improved : better speed on clang and gcc -O2, thanks to Eric Biggers
|
||||
New : Build on FreeBSD and DragonFly, thanks to JrMarino
|
||||
Changed : modified API : ZSTD_compressEnd()
|
||||
Fixed : legacy mode with ZSTD_HEAPMODE=0, by Christopher Bergqvist
|
||||
Fixed : premature end of frame when zero-sized raw block, reported by Eric Biggers
|
||||
Fixed : large dictionaries (> 384 KB), reported by Ilona Papava
|
||||
Fixed : checksum correctly checked in single-pass mode
|
||||
Fixed : combined --test amd --rm, reported by Andreas M. Nilsson
|
||||
Modified : minor compression level adaptations
|
||||
Updated : compression format specification to v0.2.0
|
||||
changed : zstd.h moved to /lib directory
|
||||
|
||||
v0.7.5
|
||||
Transition version, supporting decoding of v0.8.x
|
||||
|
||||
v0.7.4
|
||||
Added : homebrew for Mac, by Daniel Cade
|
||||
Added : more examples
|
||||
Fixed : segfault when using small dictionaries, reported by Felix Handte
|
||||
Modified : default compression level for CLI is now 3
|
||||
Updated : specification, to v0.1.1
|
||||
|
||||
v0.7.3
|
||||
New : compression format specification
|
||||
New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner.
|
||||
New : `ZSTD_getDecompressedSize()`
|
||||
New : OpenBSD target, by Juan Francisco Cantero Hurtado
|
||||
New : `examples` directory
|
||||
fixed : dictBuilder using HC levels, reported by Bartosz Taudul
|
||||
fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte
|
||||
fixed : multi-blocks decoding with intermediate uncompressed blocks, reported by Greg Slazinski
|
||||
modified : removed "mem.h" and "error_public.h" dependencies from "zstd.h" (experimental section)
|
||||
modified : legacy functions no longer need magic number
|
||||
|
||||
v0.7.2
|
||||
fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski.
|
||||
fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner.
|
||||
fixed : CLI displays system error message when destination file cannot be created (#231). Reported by Chip Turner.
|
||||
|
||||
v0.7.1
|
||||
fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier
|
||||
fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв
|
||||
fixed : corruption issue, reported by cj
|
||||
modified : checksum enabled by default in command line mode
|
||||
|
||||
v0.7.0
|
||||
New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski
|
||||
New : Command `--rm`, to remove source file after successful de/compression
|
||||
New : Visual build scripts, by Christophe Chevalier
|
||||
New : Support for Sparse File-systems (do not use space for zero-filled sectors)
|
||||
New : Frame checksum support
|
||||
New : Support pass-through mode (when using `-df`)
|
||||
API : more efficient Dictionary API : `ZSTD_compress_usingCDict()`, `ZSTD_decompress_usingDDict()`
|
||||
API : create dictionary files from custom content, by Giuseppe Ottaviano
|
||||
API : support for custom malloc/free functions
|
||||
New : controllable Dictionary ID
|
||||
New : Support for skippable frames
|
||||
|
||||
v0.6.1
|
||||
New : zlib wrapper API, thanks to Przemyslaw Skibinski
|
||||
New : Ability to compile compressor / decompressor separately
|
||||
Changed : new lib directory structure
|
||||
Fixed : Legacy codec v0.5 compatible with dictionary decompression
|
||||
Fixed : Decoder corruption error (#173)
|
||||
Fixed : null-string roundtrip (#176)
|
||||
New : benchmark mode can select directory as input
|
||||
Experimental : midipix support, VMS support
|
||||
|
||||
v0.6.0
|
||||
Stronger high compression modes, thanks to Przemyslaw Skibinski
|
||||
API : ZSTD_getFrameParams() provides size of decompressed content
|
||||
New : highest compression modes require `--ultra` command to fully unleash their capacity
|
||||
Fixed : zstd cli return error code > 0 and removes dst file artifact when decompression fails, thanks to Chip Turner
|
||||
|
||||
v0.5.1
|
||||
New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski
|
||||
Changed : Dictionary builder integrated into libzstd and zstd cli
|
||||
Changed (!) : zstd cli now uses "multiple input files" as default mode. See `zstd -h`.
|
||||
Fix : high compression modes for big-endian platforms
|
||||
New : zstd cli : `-t` | `--test` command
|
||||
|
||||
v0.5.0
|
||||
New : dictionary builder utility
|
||||
Changed : streaming & dictionary API
|
||||
Improved : better compression of small data
|
||||
|
||||
v0.4.7
|
||||
Improved : small compression speed improvement in HC mode
|
||||
Changed : `zstd_decompress.c` has ZSTD_LEGACY_SUPPORT to 0 by default
|
||||
fix : bt search bug
|
||||
|
||||
v0.4.6
|
||||
fix : fast compression mode on Windows
|
||||
New : cmake configuration file, thanks to Artyom Dymchenko
|
||||
Improved : high compression mode on repetitive data
|
||||
New : block-level API
|
||||
New : ZSTD_duplicateCCtx()
|
||||
|
||||
v0.4.5
|
||||
new : -m/--multiple : compress/decompress multiple files
|
||||
|
||||
v0.4.4
|
||||
Fixed : high compression modes for Windows 32 bits
|
||||
new : external dictionary API extended to buffered mode and accessible through command line
|
||||
new : windows DLL project, thanks to Christophe Chevalier
|
||||
|
||||
v0.4.3 :
|
||||
new : external dictionary API
|
||||
new : zstd-frugal
|
||||
|
||||
v0.4.2 :
|
||||
Generic minor improvements for small blocks
|
||||
Fixed : big-endian compatibility, by Peter Harris (#85)
|
||||
|
||||
v0.4.1
|
||||
Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben)
|
||||
removed `zstd.c`
|
||||
|
||||
v0.4.0
|
||||
Command line utility compatible with high compression levels
|
||||
Removed zstdhc => merged into zstd
|
||||
Added : ZBUFF API (see zstd_buffered.h)
|
||||
Rolling buffer support
|
||||
|
||||
v0.3.6
|
||||
small blocks params
|
||||
|
||||
v0.3.5
|
||||
minor generic compression improvements
|
||||
|
||||
v0.3.4
|
||||
Faster fast cLevels
|
||||
|
||||
v0.3.3
|
||||
Small compression ratio improvement
|
||||
|
||||
v0.3.2
|
||||
Fixed Visual Studio
|
||||
|
||||
v0.3.1 :
|
||||
Small compression ratio improvement
|
||||
|
||||
v0.3
|
||||
HC mode : compression levels 2-26
|
||||
|
||||
v0.2.2
|
||||
Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier
|
||||
|
||||
v0.2.1
|
||||
Fix : Read errors, advanced fuzzer tests, by Hanno Böck
|
||||
|
||||
v0.2.0
|
||||
**Breaking format change**
|
||||
Faster decompression speed
|
||||
Can still decode v0.1 format
|
||||
|
||||
v0.1.3
|
||||
fix uninitialization warning, reported by Evan Nemerson
|
||||
|
||||
v0.1.2
|
||||
frame concatenation support
|
||||
|
||||
v0.1.1
|
||||
fix compression bug
|
||||
detects write-flush errors
|
||||
|
||||
v0.1.0
|
||||
first release
|
||||
54
README.md
54
README.md
@ -9,7 +9,11 @@ and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4
|
||||
Should your project require another programming language,
|
||||
a list of known ports and bindings is provided on [Zstandard homepage](http://www.zstd.net/#other-languages).
|
||||
|
||||
Development branch status : [![Build Status][travisDevBadge]][travisLink] [![Build status][AppveyorDevBadge]][AppveyorLink] [![Build status][CircleDevBadge]][CircleLink]
|
||||
**Development branch status:**
|
||||
|
||||
[![Build Status][travisDevBadge]][travisLink]
|
||||
[![Build status][AppveyorDevBadge]][AppveyorLink]
|
||||
[![Build status][CircleDevBadge]][CircleLink]
|
||||
|
||||
[travisDevBadge]: https://travis-ci.org/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
|
||||
[travisLink]: https://travis-ci.org/facebook/zstd
|
||||
@ -18,7 +22,7 @@ Development branch status : [![Build Status][travisDevBadge]][travisLink] [![B
|
||||
[CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite"
|
||||
[CircleLink]: https://circleci.com/gh/facebook/zstd
|
||||
|
||||
### Benchmarks
|
||||
## Benchmarks
|
||||
|
||||
For reference, several fast compression algorithms were tested and compared
|
||||
on a server running Linux Debian (`Linux version 4.14.0-3-amd64`),
|
||||
@ -42,7 +46,7 @@ on the [Silesia compression corpus].
|
||||
| snappy 1.1.4 | 2.091 | 530 MB/s | 1800 MB/s |
|
||||
| lzf 3.6 -1 | 2.077 | 400 MB/s | 860 MB/s |
|
||||
|
||||
[zlib]:http://www.zlib.net/
|
||||
[zlib]: http://www.zlib.net/
|
||||
[LZ4]: http://www.lz4.org/
|
||||
|
||||
Zstd can also offer stronger compression ratios at the cost of compression speed.
|
||||
@ -65,7 +69,7 @@ A few other algorithms can produce higher compression ratios at slower speeds, f
|
||||
For a larger picture including slow modes, [click on this link](doc/images/DCspeed5.png).
|
||||
|
||||
|
||||
### The case for Small Data compression
|
||||
## The case for Small Data compression
|
||||
|
||||
Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives.
|
||||
|
||||
@ -89,24 +93,24 @@ Training works if there is some correlation in a family of small data samples. T
|
||||
Hence, deploying one dictionary per type of data will provide the greatest benefits.
|
||||
Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file.
|
||||
|
||||
#### Dictionary compression How To:
|
||||
### Dictionary compression How To:
|
||||
|
||||
1) Create the dictionary
|
||||
1. Create the dictionary
|
||||
|
||||
`zstd --train FullPathToTrainingSet/* -o dictionaryName`
|
||||
`zstd --train FullPathToTrainingSet/* -o dictionaryName`
|
||||
|
||||
2) Compress with dictionary
|
||||
2. Compress with dictionary
|
||||
|
||||
`zstd -D dictionaryName FILE`
|
||||
`zstd -D dictionaryName FILE`
|
||||
|
||||
3) Decompress with dictionary
|
||||
3. Decompress with dictionary
|
||||
|
||||
`zstd -D dictionaryName --decompress FILE.zst`
|
||||
`zstd -D dictionaryName --decompress FILE.zst`
|
||||
|
||||
|
||||
### Build instructions
|
||||
## Build instructions
|
||||
|
||||
#### Makefile
|
||||
### Makefile
|
||||
|
||||
If your system is compatible with standard `make` (or `gmake`),
|
||||
invoking `make` in root directory will generate `zstd` cli in root directory.
|
||||
@ -115,7 +119,7 @@ Other available options include:
|
||||
- `make install` : create and install zstd cli, library and man pages
|
||||
- `make check` : create and run `zstd`, tests its behavior on local platform
|
||||
|
||||
#### cmake
|
||||
### cmake
|
||||
|
||||
A `cmake` project generator is provided within `build/cmake`.
|
||||
It can generate Makefiles or other build scripts
|
||||
@ -123,11 +127,17 @@ to create `zstd` binary, and `libzstd` dynamic and static libraries.
|
||||
|
||||
By default, `CMAKE_BUILD_TYPE` is set to `Release`.
|
||||
|
||||
#### Meson
|
||||
### Meson
|
||||
|
||||
A Meson project is provided within `contrib/meson`.
|
||||
A Meson project is provided within [`build/meson`](build/meson). Follow
|
||||
build instructions in that directory.
|
||||
|
||||
#### Visual Studio (Windows)
|
||||
You can also take a look at [`.travis.yml`](.travis.yml) file for an
|
||||
example about how Meson is used to build this project.
|
||||
|
||||
Note that default build type is **release**.
|
||||
|
||||
### Visual Studio (Windows)
|
||||
|
||||
Going into `build` directory, you will find additional possibilities:
|
||||
- Projects for Visual Studio 2005, 2008 and 2010.
|
||||
@ -135,17 +145,21 @@ Going into `build` directory, you will find additional possibilities:
|
||||
- Automated build scripts for Visual compiler by [@KrzysFR](https://github.com/KrzysFR), in `build/VS_scripts`,
|
||||
which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution.
|
||||
|
||||
### Buck
|
||||
|
||||
### Status
|
||||
You can build the zstd binary via buck by executing: `buck build programs:zstd` from the root of the repo.
|
||||
The output binary will be in `buck-out/gen/programs/`.
|
||||
|
||||
## Status
|
||||
|
||||
Zstandard is currently deployed within Facebook. It is used continuously to compress large amounts of data in multiple formats and use cases.
|
||||
Zstandard is considered safe for production environments.
|
||||
|
||||
### License
|
||||
## License
|
||||
|
||||
Zstandard is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING).
|
||||
|
||||
### Contributing
|
||||
## Contributing
|
||||
|
||||
The "dev" branch is the one where all contributions are merged before reaching "master".
|
||||
If you plan to propose a patch, please commit into the "dev" branch, or its own feature branch.
|
||||
|
||||
@ -3,6 +3,8 @@
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
- appveyorTest
|
||||
- /visual*/
|
||||
environment:
|
||||
matrix:
|
||||
- COMPILER: "gcc"
|
||||
|
||||
@ -22,10 +22,10 @@ FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MULTITHREAD_LDFLAGS)
|
||||
|
||||
all: adapt datagen
|
||||
|
||||
adapt: $(ZSTD_FILES) adapt.c
|
||||
adapt: $(ZSTD_FILES) $(PRGDIR)/util.c adapt.c
|
||||
$(CC) $(FLAGS) $^ -o $@
|
||||
|
||||
adapt-debug: $(ZSTD_FILES) adapt.c
|
||||
adapt-debug: $(ZSTD_FILES) $(PRGDIR)/util.c adapt.c
|
||||
$(CC) $(FLAGS) -DDEBUG_MODE=2 $^ -o adapt
|
||||
|
||||
datagen : $(PRGDIR)/datagen.c datagencli.c
|
||||
|
||||
@ -579,7 +579,7 @@ static void* compressionThread(void* arg)
|
||||
params.cParams.windowLog = 23;
|
||||
{
|
||||
size_t const initError = ZSTD_compressBegin_advanced(ctx->cctx, job->src.start + job->dictSize - useDictSize, useDictSize, params, 0);
|
||||
size_t const windowSizeError = ZSTD_CCtx_setParameter(ctx->cctx, ZSTD_p_forceMaxWindow, 1);
|
||||
size_t const windowSizeError = ZSTD_CCtx_setParameter(ctx->cctx, ZSTD_c_forceMaxWindow, 1);
|
||||
if (ZSTD_isError(initError) || ZSTD_isError(windowSizeError)) {
|
||||
DISPLAY("Error: something went wrong while starting compression\n");
|
||||
signalErrorToThreads(ctx);
|
||||
|
||||
@ -120,7 +120,7 @@ int main(int argc, const char** argv)
|
||||
DISPLAYLEVEL(4, "Compressible data Generator \n");
|
||||
if (probaU32!=COMPRESSIBILITY_DEFAULT)
|
||||
DISPLAYLEVEL(3, "Compressibility : %i%%\n", probaU32);
|
||||
DISPLAYLEVEL(3, "Seed = %u \n", seed);
|
||||
DISPLAYLEVEL(3, "Seed = %u \n", (unsigned)seed);
|
||||
|
||||
RDG_genStdout(size, (double)probaU32/100, litProba, seed);
|
||||
DISPLAYLEVEL(1, "\n");
|
||||
|
||||
@ -1,3 +0,0 @@
|
||||
This Meson project is provided with no guarantee and maintained by Dima Krasner <dima@dimakrasner.com>.
|
||||
|
||||
It outputs one libzstd, either shared or static, depending on default_library.
|
||||
@ -1,144 +0,0 @@
|
||||
project('zstd', 'c', license: 'BSD')
|
||||
|
||||
libm = meson.get_compiler('c').find_library('m', required: true)
|
||||
|
||||
lib_dir = join_paths('..', '..', 'lib')
|
||||
common_dir = join_paths(lib_dir, 'common')
|
||||
compress_dir = join_paths(lib_dir, 'compress')
|
||||
decompress_dir = join_paths(lib_dir, 'decompress')
|
||||
dictbuilder_dir = join_paths(lib_dir, 'dictBuilder')
|
||||
deprecated_dir = join_paths(lib_dir, 'deprecated')
|
||||
|
||||
libzstd_srcs = [
|
||||
join_paths(common_dir, 'entropy_common.c'),
|
||||
join_paths(common_dir, 'fse_decompress.c'),
|
||||
join_paths(common_dir, 'threading.c'),
|
||||
join_paths(common_dir, 'pool.c'),
|
||||
join_paths(common_dir, 'zstd_common.c'),
|
||||
join_paths(common_dir, 'error_private.c'),
|
||||
join_paths(common_dir, 'xxhash.c'),
|
||||
join_paths(compress_dir, 'fse_compress.c'),
|
||||
join_paths(compress_dir, 'hist.c'),
|
||||
join_paths(compress_dir, 'huf_compress.c'),
|
||||
join_paths(compress_dir, 'zstd_compress.c'),
|
||||
join_paths(compress_dir, 'zstd_fast.c'),
|
||||
join_paths(compress_dir, 'zstd_double_fast.c'),
|
||||
join_paths(compress_dir, 'zstd_lazy.c'),
|
||||
join_paths(compress_dir, 'zstd_opt.c'),
|
||||
join_paths(compress_dir, 'zstd_ldm.c'),
|
||||
join_paths(compress_dir, 'zstdmt_compress.c'),
|
||||
join_paths(decompress_dir, 'huf_decompress.c'),
|
||||
join_paths(decompress_dir, 'zstd_decompress.c'),
|
||||
join_paths(dictbuilder_dir, 'cover.c'),
|
||||
join_paths(dictbuilder_dir, 'divsufsort.c'),
|
||||
join_paths(dictbuilder_dir, 'zdict.c'),
|
||||
join_paths(deprecated_dir, 'zbuff_common.c'),
|
||||
join_paths(deprecated_dir, 'zbuff_compress.c'),
|
||||
join_paths(deprecated_dir, 'zbuff_decompress.c')
|
||||
]
|
||||
|
||||
libzstd_includes = [include_directories(common_dir, dictbuilder_dir, compress_dir, lib_dir)]
|
||||
|
||||
legacy = get_option('legacy_support')
|
||||
if legacy == '0'
|
||||
legacy = 'false'
|
||||
endif
|
||||
if legacy != 'false'
|
||||
if legacy == 'true'
|
||||
legacy = '1'
|
||||
endif
|
||||
#See ZSTD_LEGACY_SUPPORT of programs/README.md
|
||||
message('Enabling legacy support back to version 0.' + legacy)
|
||||
legacy_int = legacy.to_int()
|
||||
if legacy_int > 7
|
||||
legacy_int = 7
|
||||
endif
|
||||
libzstd_cflags = ['-DZSTD_LEGACY_SUPPORT=' + legacy]
|
||||
|
||||
legacy_dir = join_paths(lib_dir, 'legacy')
|
||||
libzstd_includes += [include_directories(legacy_dir)]
|
||||
if legacy_int <= 1
|
||||
libzstd_srcs += join_paths(legacy_dir, 'zstd_v01.c')
|
||||
endif
|
||||
if legacy_int <= 2
|
||||
libzstd_srcs += join_paths(legacy_dir, 'zstd_v02.c')
|
||||
endif
|
||||
if legacy_int <= 3
|
||||
libzstd_srcs += join_paths(legacy_dir, 'zstd_v03.c')
|
||||
endif
|
||||
if legacy_int <= 4
|
||||
libzstd_srcs += join_paths(legacy_dir, 'zstd_v04.c')
|
||||
endif
|
||||
if legacy_int <= 5
|
||||
libzstd_srcs += join_paths(legacy_dir, 'zstd_v05.c')
|
||||
endif
|
||||
if legacy_int <= 6
|
||||
libzstd_srcs += join_paths(legacy_dir, 'zstd_v06.c')
|
||||
endif
|
||||
if legacy_int <= 7
|
||||
libzstd_srcs += join_paths(legacy_dir, 'zstd_v07.c')
|
||||
endif
|
||||
else
|
||||
libzstd_cflags = []
|
||||
endif
|
||||
|
||||
if get_option('multithread')
|
||||
message('Enabling multi-threading support')
|
||||
add_global_arguments('-DZSTD_MULTITHREAD', language: 'c')
|
||||
libzstd_deps = [dependency('threads')]
|
||||
else
|
||||
libzstd_deps = []
|
||||
endif
|
||||
|
||||
libzstd = library('zstd',
|
||||
libzstd_srcs,
|
||||
include_directories: libzstd_includes,
|
||||
c_args: libzstd_cflags,
|
||||
dependencies: libzstd_deps,
|
||||
install: true,
|
||||
soversion: '1',
|
||||
)
|
||||
|
||||
programs_dir = join_paths('..', '..', 'programs')
|
||||
|
||||
zstd = executable('zstd',
|
||||
join_paths(programs_dir, 'bench.c'),
|
||||
join_paths(programs_dir, 'datagen.c'),
|
||||
join_paths(programs_dir, 'dibio.c'),
|
||||
join_paths(programs_dir, 'fileio.c'),
|
||||
join_paths(programs_dir, 'zstdcli.c'),
|
||||
include_directories: libzstd_includes,
|
||||
c_args: ['-DZSTD_NODICT', '-DZSTD_NOBENCH'],
|
||||
link_with: libzstd,
|
||||
install: true)
|
||||
|
||||
tests_dir = join_paths('..', '..', 'tests')
|
||||
datagen_c = join_paths(programs_dir, 'datagen.c')
|
||||
test_includes = libzstd_includes + [include_directories(programs_dir)]
|
||||
|
||||
fullbench = executable('fullbench',
|
||||
datagen_c, join_paths(tests_dir, 'fullbench.c'),
|
||||
include_directories: test_includes,
|
||||
link_with: libzstd)
|
||||
test('fullbench', fullbench)
|
||||
|
||||
fuzzer = executable('fuzzer',
|
||||
datagen_c, join_paths(tests_dir, 'fuzzer.c'),
|
||||
include_directories: test_includes,
|
||||
link_with: libzstd)
|
||||
test('fuzzer', fuzzer)
|
||||
|
||||
if target_machine.system() != 'windows'
|
||||
paramgrill = executable('paramgrill',
|
||||
datagen_c, join_paths(tests_dir, 'paramgrill.c'),
|
||||
join_paths(programs_dir, 'bench.c'),
|
||||
include_directories: test_includes,
|
||||
link_with: libzstd,
|
||||
dependencies: libm)
|
||||
test('paramgrill', paramgrill)
|
||||
|
||||
datagen = executable('datagen',
|
||||
datagen_c, join_paths(tests_dir, 'datagencli.c'),
|
||||
include_directories: test_includes,
|
||||
link_with: libzstd)
|
||||
endif
|
||||
@ -1,3 +0,0 @@
|
||||
option('multithread', type: 'boolean', value: false)
|
||||
option('legacy_support', type: 'string', value: '4',
|
||||
description: 'True or false, or 7 to 1 for v0.7+ to v0.1+.')
|
||||
@ -171,7 +171,7 @@ roundtripcheck: roundtrip check
|
||||
$(TESTPROG) ./test/RoundTripTest$(EXT) $(TESTFLAGS)
|
||||
|
||||
# Build the main binary
|
||||
pzstd$(EXT): main.o Options.o Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a
|
||||
pzstd$(EXT): main.o $(PROGDIR)/util.o Options.o Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a
|
||||
$(LD_COMMAND)
|
||||
|
||||
# Target that depends on all the tests
|
||||
|
||||
@ -148,20 +148,20 @@ static void sumFile_orDie(const char* fname, int nbThreads)
|
||||
size_t const initResult = ZSTD_seekable_initFile(seekable, fin);
|
||||
if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_seekable_init() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
|
||||
|
||||
size_t const numFrames = ZSTD_seekable_getNumFrames(seekable);
|
||||
unsigned const numFrames = ZSTD_seekable_getNumFrames(seekable);
|
||||
struct sum_job* jobs = (struct sum_job*)malloc(numFrames * sizeof(struct sum_job));
|
||||
|
||||
size_t i;
|
||||
for (i = 0; i < numFrames; i++) {
|
||||
jobs[i] = (struct sum_job){ fname, 0, i, 0 };
|
||||
POOL_add(pool, sumFrame, &jobs[i]);
|
||||
unsigned fnb;
|
||||
for (fnb = 0; fnb < numFrames; fnb++) {
|
||||
jobs[fnb] = (struct sum_job){ fname, 0, fnb, 0 };
|
||||
POOL_add(pool, sumFrame, &jobs[fnb]);
|
||||
}
|
||||
|
||||
unsigned long long total = 0;
|
||||
|
||||
for (i = 0; i < numFrames; i++) {
|
||||
while (!jobs[i].done) SLEEP(5); /* wake up every 5 milliseconds to check */
|
||||
total += jobs[i].sum;
|
||||
for (fnb = 0; fnb < numFrames; fnb++) {
|
||||
while (!jobs[fnb].done) SLEEP(5); /* wake up every 5 milliseconds to check */
|
||||
total += jobs[fnb].sum;
|
||||
}
|
||||
|
||||
printf("Sum: %llu\n", total);
|
||||
|
||||
@ -8,6 +8,8 @@
|
||||
*/
|
||||
|
||||
#include <stdlib.h> /* malloc, free */
|
||||
#include <limits.h> /* UINT_MAX */
|
||||
#include <assert.h>
|
||||
|
||||
#define XXH_STATIC_LINKING_ONLY
|
||||
#define XXH_NAMESPACE ZSTD_
|
||||
@ -139,7 +141,7 @@ size_t ZSTD_seekable_freeCStream(ZSTD_seekable_CStream* zcs)
|
||||
size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs,
|
||||
int compressionLevel,
|
||||
int checksumFlag,
|
||||
U32 maxFrameSize)
|
||||
unsigned maxFrameSize)
|
||||
{
|
||||
zcs->framelog.size = 0;
|
||||
zcs->frameCSize = 0;
|
||||
@ -167,9 +169,9 @@ size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs,
|
||||
}
|
||||
|
||||
size_t ZSTD_seekable_logFrame(ZSTD_frameLog* fl,
|
||||
unsigned compressedSize,
|
||||
unsigned decompressedSize,
|
||||
unsigned checksum)
|
||||
unsigned compressedSize,
|
||||
unsigned decompressedSize,
|
||||
unsigned checksum)
|
||||
{
|
||||
if (fl->size == ZSTD_SEEKABLE_MAXFRAMES)
|
||||
return ERROR(frameIndex_tooLarge);
|
||||
@ -184,7 +186,8 @@ size_t ZSTD_seekable_logFrame(ZSTD_frameLog* fl,
|
||||
if (newEntries == NULL) return ERROR(memory_allocation);
|
||||
|
||||
fl->entries = newEntries;
|
||||
fl->capacity = newCapacity;
|
||||
assert(newCapacity <= UINT_MAX);
|
||||
fl->capacity = (U32)newCapacity;
|
||||
}
|
||||
|
||||
fl->entries[fl->size] = (framelogEntry_t){
|
||||
@ -268,7 +271,7 @@ size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer*
|
||||
static inline size_t ZSTD_seekable_seekTableSize(const ZSTD_frameLog* fl)
|
||||
{
|
||||
size_t const sizePerFrame = 8 + (fl->checksumFlag?4:0);
|
||||
size_t const seekTableLen = ZSTD_skippableHeaderSize +
|
||||
size_t const seekTableLen = ZSTD_SKIPPABLEHEADERSIZE +
|
||||
sizePerFrame * fl->size +
|
||||
ZSTD_seekTableFooterSize;
|
||||
|
||||
@ -307,32 +310,32 @@ size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output)
|
||||
size_t const seekTableLen = ZSTD_seekable_seekTableSize(fl);
|
||||
|
||||
CHECK_Z(ZSTD_stwrite32(fl, output, ZSTD_MAGIC_SKIPPABLE_START | 0xE, 0));
|
||||
CHECK_Z(ZSTD_stwrite32(fl, output, seekTableLen - ZSTD_skippableHeaderSize,
|
||||
4));
|
||||
assert(seekTableLen <= (size_t)UINT_MAX);
|
||||
CHECK_Z(ZSTD_stwrite32(fl, output, (U32)seekTableLen - ZSTD_SKIPPABLEHEADERSIZE, 4));
|
||||
|
||||
while (fl->seekTableIndex < fl->size) {
|
||||
unsigned long long const start = ZSTD_SKIPPABLEHEADERSIZE + sizePerFrame * fl->seekTableIndex;
|
||||
assert(start + 8 <= UINT_MAX);
|
||||
CHECK_Z(ZSTD_stwrite32(fl, output,
|
||||
fl->entries[fl->seekTableIndex].cSize,
|
||||
ZSTD_skippableHeaderSize +
|
||||
sizePerFrame * fl->seekTableIndex + 0));
|
||||
(U32)start + 0));
|
||||
|
||||
CHECK_Z(ZSTD_stwrite32(fl, output,
|
||||
fl->entries[fl->seekTableIndex].dSize,
|
||||
ZSTD_skippableHeaderSize +
|
||||
sizePerFrame * fl->seekTableIndex + 4));
|
||||
(U32)start + 4));
|
||||
|
||||
if (fl->checksumFlag) {
|
||||
CHECK_Z(ZSTD_stwrite32(
|
||||
fl, output, fl->entries[fl->seekTableIndex].checksum,
|
||||
ZSTD_skippableHeaderSize +
|
||||
sizePerFrame * fl->seekTableIndex + 8));
|
||||
(U32)start + 8));
|
||||
}
|
||||
|
||||
fl->seekTableIndex++;
|
||||
}
|
||||
|
||||
assert(seekTableLen <= UINT_MAX);
|
||||
CHECK_Z(ZSTD_stwrite32(fl, output, fl->size,
|
||||
seekTableLen - ZSTD_seekTableFooterSize));
|
||||
(U32)seekTableLen - ZSTD_seekTableFooterSize));
|
||||
|
||||
if (output->size - output->pos < 1) return seekTableLen - fl->seekTablePos;
|
||||
if (fl->seekTablePos < seekTableLen - 4) {
|
||||
@ -345,7 +348,7 @@ size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output)
|
||||
}
|
||||
|
||||
CHECK_Z(ZSTD_stwrite32(fl, output, ZSTD_SEEKABLE_MAGICNUMBER,
|
||||
seekTableLen - 4));
|
||||
(U32)seekTableLen - 4));
|
||||
|
||||
if (fl->seekTablePos != seekTableLen) return ERROR(GENERIC);
|
||||
return 0;
|
||||
|
||||
@ -54,8 +54,9 @@
|
||||
# define LONG_SEEK fseek
|
||||
#endif
|
||||
|
||||
#include <stdlib.h> /* malloc, free */
|
||||
#include <stdio.h> /* FILE* */
|
||||
#include <stdlib.h> /* malloc, free */
|
||||
#include <stdio.h> /* FILE* */
|
||||
#include <limits.h> /* UNIT_MAX */
|
||||
#include <assert.h>
|
||||
|
||||
#define XXH_STATIC_LINKING_ONLY
|
||||
@ -200,13 +201,14 @@ size_t ZSTD_seekable_free(ZSTD_seekable* zs)
|
||||
* Performs a binary search to find the last frame with a decompressed offset
|
||||
* <= pos
|
||||
* @return : the frame's index */
|
||||
U32 ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long long pos)
|
||||
unsigned ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long long pos)
|
||||
{
|
||||
U32 lo = 0;
|
||||
U32 hi = zs->seekTable.tableLen;
|
||||
U32 hi = (U32)zs->seekTable.tableLen;
|
||||
assert(zs->seekTable.tableLen <= UINT_MAX);
|
||||
|
||||
if (pos >= zs->seekTable.entries[zs->seekTable.tableLen].dOffset) {
|
||||
return zs->seekTable.tableLen;
|
||||
return (U32)zs->seekTable.tableLen;
|
||||
}
|
||||
|
||||
while (lo + 1 < hi) {
|
||||
@ -220,31 +222,32 @@ U32 ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long long
|
||||
return lo;
|
||||
}
|
||||
|
||||
U32 ZSTD_seekable_getNumFrames(ZSTD_seekable* const zs)
|
||||
unsigned ZSTD_seekable_getNumFrames(ZSTD_seekable* const zs)
|
||||
{
|
||||
return zs->seekTable.tableLen;
|
||||
assert(zs->seekTable.tableLen <= UINT_MAX);
|
||||
return (unsigned)zs->seekTable.tableLen;
|
||||
}
|
||||
|
||||
unsigned long long ZSTD_seekable_getFrameCompressedOffset(ZSTD_seekable* const zs, U32 frameIndex)
|
||||
unsigned long long ZSTD_seekable_getFrameCompressedOffset(ZSTD_seekable* const zs, unsigned frameIndex)
|
||||
{
|
||||
if (frameIndex >= zs->seekTable.tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE;
|
||||
return zs->seekTable.entries[frameIndex].cOffset;
|
||||
}
|
||||
|
||||
unsigned long long ZSTD_seekable_getFrameDecompressedOffset(ZSTD_seekable* const zs, U32 frameIndex)
|
||||
unsigned long long ZSTD_seekable_getFrameDecompressedOffset(ZSTD_seekable* const zs, unsigned frameIndex)
|
||||
{
|
||||
if (frameIndex >= zs->seekTable.tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE;
|
||||
return zs->seekTable.entries[frameIndex].dOffset;
|
||||
}
|
||||
|
||||
size_t ZSTD_seekable_getFrameCompressedSize(ZSTD_seekable* const zs, U32 frameIndex)
|
||||
size_t ZSTD_seekable_getFrameCompressedSize(ZSTD_seekable* const zs, unsigned frameIndex)
|
||||
{
|
||||
if (frameIndex >= zs->seekTable.tableLen) return ERROR(frameIndex_tooLarge);
|
||||
return zs->seekTable.entries[frameIndex + 1].cOffset -
|
||||
zs->seekTable.entries[frameIndex].cOffset;
|
||||
}
|
||||
|
||||
size_t ZSTD_seekable_getFrameDecompressedSize(ZSTD_seekable* const zs, U32 frameIndex)
|
||||
size_t ZSTD_seekable_getFrameDecompressedSize(ZSTD_seekable* const zs, unsigned frameIndex)
|
||||
{
|
||||
if (frameIndex > zs->seekTable.tableLen) return ERROR(frameIndex_tooLarge);
|
||||
return zs->seekTable.entries[frameIndex + 1].dOffset -
|
||||
@ -275,7 +278,7 @@ static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs)
|
||||
{ U32 const numFrames = MEM_readLE32(zs->inBuff);
|
||||
U32 const sizePerEntry = 8 + (checksumFlag?4:0);
|
||||
U32 const tableSize = sizePerEntry * numFrames;
|
||||
U32 const frameSize = tableSize + ZSTD_seekTableFooterSize + ZSTD_skippableHeaderSize;
|
||||
U32 const frameSize = tableSize + ZSTD_seekTableFooterSize + ZSTD_SKIPPABLEHEADERSIZE;
|
||||
|
||||
U32 remaining = frameSize - ZSTD_seekTableFooterSize; /* don't need to re-read footer */
|
||||
{
|
||||
@ -290,7 +293,7 @@ static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs)
|
||||
if (MEM_readLE32(zs->inBuff) != (ZSTD_MAGIC_SKIPPABLE_START | 0xE)) {
|
||||
return ERROR(prefix_unknown);
|
||||
}
|
||||
if (MEM_readLE32(zs->inBuff+4) + ZSTD_skippableHeaderSize != frameSize) {
|
||||
if (MEM_readLE32(zs->inBuff+4) + ZSTD_SKIPPABLEHEADERSIZE != frameSize) {
|
||||
return ERROR(prefix_unknown);
|
||||
}
|
||||
|
||||
@ -444,7 +447,7 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign
|
||||
return len;
|
||||
}
|
||||
|
||||
size_t ZSTD_seekable_decompressFrame(ZSTD_seekable* zs, void* dst, size_t dstSize, U32 frameIndex)
|
||||
size_t ZSTD_seekable_decompressFrame(ZSTD_seekable* zs, void* dst, size_t dstSize, unsigned frameIndex)
|
||||
{
|
||||
if (frameIndex >= zs->seekTable.tableLen) {
|
||||
return ERROR(frameIndex_tooLarge);
|
||||
|
||||
@ -16,7 +16,7 @@ Distribution of this document is unlimited.
|
||||
|
||||
### Version
|
||||
|
||||
0.3.0 (25/09/18)
|
||||
0.3.1 (25/10/18)
|
||||
|
||||
|
||||
Introduction
|
||||
@ -913,11 +913,37 @@ Note that blocks which are not `Compressed_Block` are skipped, they do not contr
|
||||
###### Offset updates rules
|
||||
|
||||
The newest offset takes the lead in offset history,
|
||||
shifting others back (up to its previous place if it was already present).
|
||||
shifting others back by one rank,
|
||||
up to the previous rank of the new offset _if it was present in history_.
|
||||
|
||||
__Examples__ :
|
||||
|
||||
In the common case, when new offset is not part of history :
|
||||
`Repeated_Offset3` = `Repeated_Offset2`
|
||||
`Repeated_Offset2` = `Repeated_Offset1`
|
||||
`Repeated_Offset1` = `NewOffset`
|
||||
|
||||
When the new offset _is_ part of history, there may be specific adjustments.
|
||||
|
||||
When `NewOffset` == `Repeated_Offset1`, offset history remains actually unmodified.
|
||||
|
||||
When `NewOffset` == `Repeated_Offset2`,
|
||||
`Repeated_Offset1` and `Repeated_Offset2` ranks are swapped.
|
||||
`Repeated_Offset3` is unmodified.
|
||||
|
||||
When `NewOffset` == `Repeated_Offset3`,
|
||||
there is actually no difference with the common case :
|
||||
all offsets are shifted by one rank,
|
||||
`NewOffset` (== `Repeated_Offset3`) becomes the new `Repeated_Offset1`.
|
||||
|
||||
Also worth mentioning, the specific corner case when `offset_value` == 3,
|
||||
and the literal length of the current sequence is zero.
|
||||
In which case , `NewOffset` = `Repeated_Offset1` - 1_byte.
|
||||
Here also, from an offset history update perspective, it's just a common case :
|
||||
`Repeated_Offset3` = `Repeated_Offset2`
|
||||
`Repeated_Offset2` = `Repeated_Offset1`
|
||||
`Repeated_Offset1` = `NewOffset` ( == `Repeated_Offset1` - 1_byte )
|
||||
|
||||
This means that when `Repeated_Offset1` (most recent) is used, history is unmodified.
|
||||
When `Repeated_Offset2` is used, it's swapped with `Repeated_Offset1`.
|
||||
If any other offset is used, it becomes `Repeated_Offset1` and the rest are shift back by one.
|
||||
|
||||
|
||||
Skippable Frames
|
||||
@ -1629,6 +1655,7 @@ or at least provide a meaningful error code explaining for which reason it canno
|
||||
|
||||
Version changes
|
||||
---------------
|
||||
- 0.3.1 : minor clarification regarding offset history update rules
|
||||
- 0.3.0 : minor edits to match RFC8478
|
||||
- 0.2.9 : clarifications for huffman weights direct representation, by Ulrich Kunitz
|
||||
- 0.2.8 : clarifications for IETF RFC discuss
|
||||
|
||||
1370
doc/zstd_manual.html
1370
doc/zstd_manual.html
File diff suppressed because it is too large
Load Diff
17
lib/BUCK
17
lib/BUCK
@ -1,6 +1,7 @@
|
||||
cxx_library(
|
||||
name='zstd',
|
||||
header_namespace='',
|
||||
exported_headers=['zstd.h'],
|
||||
visibility=['PUBLIC'],
|
||||
deps=[
|
||||
':common',
|
||||
@ -17,7 +18,7 @@ cxx_library(
|
||||
exported_headers=subdir_glob([
|
||||
('compress', 'zstd*.h'),
|
||||
]),
|
||||
srcs=glob(['compress/zstd*.c']),
|
||||
srcs=glob(['compress/zstd*.c', 'compress/hist.c']),
|
||||
deps=[':common'],
|
||||
)
|
||||
|
||||
@ -40,7 +41,7 @@ cxx_library(
|
||||
header_namespace='',
|
||||
visibility=['PUBLIC'],
|
||||
exported_headers=subdir_glob([
|
||||
('decprecated', '*.h'),
|
||||
('deprecated', '*.h'),
|
||||
]),
|
||||
srcs=glob(['deprecated/*.c']),
|
||||
deps=[':common'],
|
||||
@ -118,6 +119,7 @@ cxx_library(
|
||||
'decompress/huf_decompress.c',
|
||||
],
|
||||
deps=[
|
||||
':debug',
|
||||
':bitstream',
|
||||
':compiler',
|
||||
':errors',
|
||||
@ -204,9 +206,20 @@ cxx_library(
|
||||
],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='debug',
|
||||
header_namespace='',
|
||||
visibility=['PUBLIC'],
|
||||
exported_headers=subdir_glob([
|
||||
('common', 'debug.h'),
|
||||
]),
|
||||
srcs=['common/debug.c'],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='common',
|
||||
deps=[
|
||||
':debug',
|
||||
':bitstream',
|
||||
':compiler',
|
||||
':cpu',
|
||||
|
||||
39
lib/Makefile
39
lib/Makefile
@ -27,11 +27,16 @@ DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
||||
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
|
||||
-Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
|
||||
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
|
||||
-Wredundant-decls -Wmissing-prototypes
|
||||
-Wredundant-decls -Wmissing-prototypes -Wc++-compat
|
||||
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS)
|
||||
|
||||
GREP = grep --color=never
|
||||
HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
|
||||
GREP_OPTIONS ?=
|
||||
ifeq ($HAVE_COLORNEVER, 1)
|
||||
GREP_OPTIONS += --color=never
|
||||
endif
|
||||
GREP = grep $(GREP_OPTIONS)
|
||||
|
||||
ZSTDCOMMON_FILES := $(sort $(wildcard common/*.c))
|
||||
ZSTDCOMP_FILES := $(sort $(wildcard compress/*.c))
|
||||
@ -45,6 +50,12 @@ ZSTD_LIB_COMPRESSION ?= 1
|
||||
ZSTD_LIB_DECOMPRESSION ?= 1
|
||||
ZSTD_LIB_DICTBUILDER ?= 1
|
||||
ZSTD_LIB_DEPRECATED ?= 1
|
||||
HUF_FORCE_DECOMPRESS_X1 ?= 0
|
||||
HUF_FORCE_DECOMPRESS_X2 ?= 0
|
||||
ZSTD_FORCE_DECOMPRESS_SHORT ?= 0
|
||||
ZSTD_FORCE_DECOMPRESS_LONG ?= 0
|
||||
ZSTD_NO_INLINE ?= 0
|
||||
ZSTD_STRIP_ERROR_STRINGS ?= 0
|
||||
|
||||
ifeq ($(ZSTD_LIB_COMPRESSION), 0)
|
||||
ZSTD_LIB_DICTBUILDER = 0
|
||||
@ -72,6 +83,30 @@ ifneq ($(ZSTD_LIB_DICTBUILDER), 0)
|
||||
ZSTD_FILES += $(ZDICT_FILES)
|
||||
endif
|
||||
|
||||
ifneq ($(HUF_FORCE_DECOMPRESS_X1), 0)
|
||||
CFLAGS += -DHUF_FORCE_DECOMPRESS_X1
|
||||
endif
|
||||
|
||||
ifneq ($(HUF_FORCE_DECOMPRESS_X2), 0)
|
||||
CFLAGS += -DHUF_FORCE_DECOMPRESS_X2
|
||||
endif
|
||||
|
||||
ifneq ($(ZSTD_FORCE_DECOMPRESS_SHORT), 0)
|
||||
CFLAGS += -DZSTD_FORCE_DECOMPRESS_SHORT
|
||||
endif
|
||||
|
||||
ifneq ($(ZSTD_FORCE_DECOMPRESS_LONG), 0)
|
||||
CFLAGS += -DZSTD_FORCE_DECOMPRESS_LONG
|
||||
endif
|
||||
|
||||
ifneq ($(ZSTD_NO_INLINE), 0)
|
||||
CFLAGS += -DZSTD_NO_INLINE
|
||||
endif
|
||||
|
||||
ifneq ($(ZSTD_STRIP_ERROR_STRINGS), 0)
|
||||
CFLAGS += -DZSTD_STRIP_ERROR_STRINGS
|
||||
endif
|
||||
|
||||
ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
|
||||
ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
|
||||
ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
|
||||
|
||||
108
lib/README.md
108
lib/README.md
@ -7,13 +7,32 @@ in order to make it easier to select or exclude features.
|
||||
|
||||
#### Building
|
||||
|
||||
`Makefile` script is provided, supporting all standard [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions),
|
||||
`Makefile` script is provided, supporting [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions),
|
||||
including commands variables, staged install, directory variables and standard targets.
|
||||
- `make` : generates both static and dynamic libraries
|
||||
- `make install` : install libraries in default system directories
|
||||
- `make install` : install libraries and headers in target system directories
|
||||
|
||||
`libzstd` default scope includes compression, decompression, dictionary building,
|
||||
and decoding support for legacy formats >= v0.5.0.
|
||||
`libzstd` default scope is pretty large, including compression, decompression, dictionary builder,
|
||||
and support for decoding legacy formats >= v0.5.0.
|
||||
The scope can be reduced on demand (see paragraph _modular build_).
|
||||
|
||||
|
||||
#### Multithreading support
|
||||
|
||||
Multithreading is disabled by default when building with `make`.
|
||||
Enabling multithreading requires 2 conditions :
|
||||
- set build macro `ZSTD_MULTITHREAD` (`-DZSTD_MULTITHREAD` for `gcc`)
|
||||
- for POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`)
|
||||
|
||||
Both conditions are automatically applied when invoking `make lib-mt` target.
|
||||
|
||||
When linking a POSIX program with a multithreaded version of `libzstd`,
|
||||
note that it's necessary to request the `-pthread` flag during link stage.
|
||||
|
||||
Multithreading capabilities are exposed
|
||||
via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.3.8/lib/zstd.h#L592).
|
||||
This API is still labelled experimental,
|
||||
but is expected to become "stable" in the near future.
|
||||
|
||||
|
||||
#### API
|
||||
@ -26,63 +45,70 @@ Zstandard's stable API is exposed within [lib/zstd.h](zstd.h).
|
||||
Optional advanced features are exposed via :
|
||||
|
||||
- `lib/common/zstd_errors.h` : translates `size_t` function results
|
||||
into an `ZSTD_ErrorCode`, for accurate error handling.
|
||||
into a `ZSTD_ErrorCode`, for accurate error handling.
|
||||
|
||||
- `ZSTD_STATIC_LINKING_ONLY` : if this macro is defined _before_ including `zstd.h`,
|
||||
it unlocks access to advanced experimental API,
|
||||
exposed in second part of `zstd.h`.
|
||||
These APIs are not "stable", their definition may change in the future.
|
||||
As a consequence, it shall ___never be used with dynamic library___ !
|
||||
it unlocks access to the experimental API,
|
||||
exposed in the second part of `zstd.h`.
|
||||
All definitions in the experimental APIs are unstable,
|
||||
they may still change in the future, or even be removed.
|
||||
As a consequence, experimental definitions shall ___never be used with dynamic library___ !
|
||||
Only static linking is allowed.
|
||||
|
||||
|
||||
#### Modular build
|
||||
|
||||
It's possible to compile only a limited set of features.
|
||||
It's possible to compile only a limited set of features within `libzstd`.
|
||||
The file structure is designed to make this selection manually achievable for any build system :
|
||||
|
||||
- Directory `lib/common` is always required, for all variants.
|
||||
|
||||
- Compression source code lies in `lib/compress`
|
||||
|
||||
- Decompression source code lies in `lib/decompress`
|
||||
|
||||
- It's possible to include only `compress` or only `decompress`, they don't depend on each other.
|
||||
|
||||
- `lib/dictBuilder` : makes it possible to generate dictionaries from a set of samples.
|
||||
The API is exposed in `lib/dictBuilder/zdict.h`.
|
||||
This module depends on both `lib/common` and `lib/compress` .
|
||||
- `lib/legacy` : source code to decompress legacy zstd formats, starting from `v0.1.0`.
|
||||
|
||||
- `lib/legacy` : makes it possible to decompress legacy zstd formats, starting from `v0.1.0`.
|
||||
This module depends on `lib/common` and `lib/decompress`.
|
||||
To enable this feature, define `ZSTD_LEGACY_SUPPORT` during compilation.
|
||||
Specifying a number limits versions supported to that version onward.
|
||||
For example, `ZSTD_LEGACY_SUPPORT=2` means : "support legacy formats >= v0.2.0".
|
||||
`ZSTD_LEGACY_SUPPORT=3` means : "support legacy formats >= v0.3.0", and so on.
|
||||
Currently, the default library setting is `ZST_LEGACY_SUPPORT=5`.
|
||||
It can be changed at build by any other value.
|
||||
Note that any number >= 8 translates into "do __not__ support legacy formats",
|
||||
since all versions of `zstd` >= v0.8 are compatible with v1+ specification.
|
||||
`ZSTD_LEGACY_SUPPORT=0` also means "do __not__ support legacy formats".
|
||||
Once enabled, this capability is transparently triggered within decompression functions.
|
||||
It's also possible to invoke directly legacy API, as exposed in `lib/legacy/zstd_legacy.h`.
|
||||
Each version also provides an additional dedicated set of advanced API.
|
||||
Conversely, `ZSTD_LEGACY_SUPPORT=0` means "do __not__ support legacy formats".
|
||||
By default, this build macro is set as `ZSTD_LEGACY_SUPPORT=5`.
|
||||
Decoding supported legacy format is a transparent capability triggered within decompression functions.
|
||||
It's also allowed to invoke legacy API directly, exposed in `lib/legacy/zstd_legacy.h`.
|
||||
Each version does also provide its own set of advanced API.
|
||||
For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` .
|
||||
Note : `lib/legacy` only supports _decoding_ legacy formats.
|
||||
- Similarly, you can define `ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`,
|
||||
and `ZSTD_LIB_DEPRECATED` as 0 to forgo compilation of the corresponding features. This will
|
||||
also disable compilation of all dependencies (eg. `ZSTD_LIB_COMPRESSION=0` will also disable
|
||||
dictBuilder).
|
||||
|
||||
- While invoking `make libzstd`, it's possible to define build macros
|
||||
`ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`,
|
||||
and `ZSTD_LIB_DEPRECATED` as `0` to forgo compilation of the corresponding features.
|
||||
This will also disable compilation of all dependencies
|
||||
(eg. `ZSTD_LIB_COMPRESSION=0` will also disable dictBuilder).
|
||||
|
||||
#### Multithreading support
|
||||
- There are some additional build macros that can be used to minify the decoder.
|
||||
|
||||
Multithreading is disabled by default when building with `make`.
|
||||
Enabling multithreading requires 2 conditions :
|
||||
- set macro `ZSTD_MULTITHREAD`
|
||||
- on POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`)
|
||||
Zstandard often has more than one implementation of a piece of functionality,
|
||||
where each implementation optimizes for different scenarios. For example, the
|
||||
Huffman decoder has complementary implementations that decode the stream one
|
||||
symbol at a time or two symbols at a time. Zstd normally includes both (and
|
||||
dispatches between them at runtime), but by defining `HUF_FORCE_DECOMPRESS_X1`
|
||||
or `HUF_FORCE_DECOMPRESS_X2`, you can force the use of one or the other, avoiding
|
||||
compilation of the other. Similarly, `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT`
|
||||
and `ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG` force the compilation and use of
|
||||
only one or the other of two decompression implementations. The smallest
|
||||
binary is achieved by using `HUF_FORCE_DECOMPRESS_X1` and
|
||||
`ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT`.
|
||||
|
||||
Both conditions are automatically triggered by invoking `make lib-mt` target.
|
||||
Note that, when linking a POSIX program with a multithreaded version of `libzstd`,
|
||||
it's necessary to trigger `-pthread` flag during link stage.
|
||||
|
||||
Multithreading capabilities are exposed
|
||||
via [advanced API `ZSTD_compress_generic()` defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/dev/lib/zstd.h#L919).
|
||||
This API is still considered experimental,
|
||||
but is expected to become "stable" at some point in the future.
|
||||
For squeezing the last ounce of size out, you can also define
|
||||
`ZSTD_NO_INLINE`, which disables inlining, and `ZSTD_STRIP_ERROR_STRINGS`,
|
||||
which removes the error messages that are otherwise returned by
|
||||
`ZSTD_getErrorName`.
|
||||
|
||||
|
||||
#### Windows : using MinGW+MSYS to create DLL
|
||||
@ -113,8 +139,8 @@ Consider migrating code towards supported streaming API exposed in `zstd.h`.
|
||||
|
||||
The other files are not source code. There are :
|
||||
|
||||
- `LICENSE` : contains the BSD license text
|
||||
- `Makefile` : `make` script to build and install zstd library (static and dynamic)
|
||||
- `BUCK` : support for `buck` build system (https://buckbuild.com/)
|
||||
- `libzstd.pc.in` : for `pkg-config` (used in `make install`)
|
||||
- `Makefile` : `make` script to build and install zstd library (static and dynamic)
|
||||
- `README.md` : this file
|
||||
- `dll/` : resources directory for Windows compilation
|
||||
- `libzstd.pc.in` : script for `pkg-config` (used in `make install`)
|
||||
|
||||
@ -389,7 +389,7 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
* Read (consume) next n bits from local register and update.
|
||||
* Pay attention to not read more than nbBits contained into local register.
|
||||
* @return : extracted value. */
|
||||
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
|
||||
{
|
||||
size_t const value = BIT_lookBits(bitD, nbBits);
|
||||
BIT_skipBits(bitD, nbBits);
|
||||
@ -398,7 +398,7 @@ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
|
||||
/*! BIT_readBitsFast() :
|
||||
* unsafe version; only works only if nbBits >= 1 */
|
||||
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
||||
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
|
||||
{
|
||||
size_t const value = BIT_lookBitsFast(bitD, nbBits);
|
||||
assert(nbBits >= 1);
|
||||
|
||||
@ -15,6 +15,8 @@
|
||||
* Compiler specifics
|
||||
*********************************************************/
|
||||
/* force inlining */
|
||||
|
||||
#if !defined(ZSTD_NO_INLINE)
|
||||
#if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
|
||||
# define INLINE_KEYWORD inline
|
||||
#else
|
||||
@ -29,6 +31,13 @@
|
||||
# define FORCE_INLINE_ATTR
|
||||
#endif
|
||||
|
||||
#else
|
||||
|
||||
#define INLINE_KEYWORD
|
||||
#define FORCE_INLINE_ATTR
|
||||
|
||||
#endif
|
||||
|
||||
/**
|
||||
* FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant
|
||||
* parameters. They must be inlined for the compiler to elimininate the constant
|
||||
@ -89,23 +98,21 @@
|
||||
#endif
|
||||
|
||||
/* prefetch
|
||||
* can be disabled, by declaring NO_PREFETCH macro
|
||||
* All prefetch invocations use a single default locality 2,
|
||||
* generating instruction prefetcht1,
|
||||
* which, according to Intel, means "load data into L2 cache".
|
||||
* This is a good enough "middle ground" for the time being,
|
||||
* though in theory, it would be better to specialize locality depending on data being prefetched.
|
||||
* Tests could not determine any sensible difference based on locality value. */
|
||||
* can be disabled, by declaring NO_PREFETCH build macro */
|
||||
#if defined(NO_PREFETCH)
|
||||
# define PREFETCH(ptr) (void)(ptr) /* disabled */
|
||||
# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */
|
||||
# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */
|
||||
#else
|
||||
# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */
|
||||
# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
|
||||
# define PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
|
||||
# define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
|
||||
# define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
|
||||
# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
|
||||
# define PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
|
||||
# define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
|
||||
# define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
|
||||
# else
|
||||
# define PREFETCH(ptr) (void)(ptr) /* disabled */
|
||||
# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */
|
||||
# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */
|
||||
# endif
|
||||
#endif /* NO_PREFETCH */
|
||||
|
||||
@ -116,7 +123,7 @@
|
||||
size_t const _size = (size_t)(s); \
|
||||
size_t _pos; \
|
||||
for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \
|
||||
PREFETCH(_ptr + _pos); \
|
||||
PREFETCH_L2(_ptr + _pos); \
|
||||
} \
|
||||
}
|
||||
|
||||
|
||||
@ -78,7 +78,7 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
|
||||
__asm__(
|
||||
"pushl %%ebx\n\t"
|
||||
"cpuid\n\t"
|
||||
"movl %%ebx, %%eax\n\r"
|
||||
"movl %%ebx, %%eax\n\t"
|
||||
"popl %%ebx"
|
||||
: "=a"(f7b), "=c"(f7c)
|
||||
: "a"(7), "c"(0)
|
||||
|
||||
@ -57,9 +57,9 @@ extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/* static assert is triggered at compile time, leaving no runtime artefact,
|
||||
* but can only work with compile-time constants.
|
||||
* This variant can only be used inside a function. */
|
||||
/* static assert is triggered at compile time, leaving no runtime artefact.
|
||||
* static assert only works with compile-time constants.
|
||||
* Also, this variant can only be used inside a function. */
|
||||
#define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1])
|
||||
|
||||
|
||||
@ -70,9 +70,19 @@ extern "C" {
|
||||
# define DEBUGLEVEL 0
|
||||
#endif
|
||||
|
||||
|
||||
/* DEBUGFILE can be defined externally,
|
||||
* typically through compiler command line.
|
||||
* note : currently useless.
|
||||
* Value must be stderr or stdout */
|
||||
#ifndef DEBUGFILE
|
||||
# define DEBUGFILE stderr
|
||||
#endif
|
||||
|
||||
|
||||
/* recommended values for DEBUGLEVEL :
|
||||
* 0 : no debug, all run-time functions disabled
|
||||
* 1 : no display, enables assert() only
|
||||
* 0 : release mode, no debug, all run-time checks disabled
|
||||
* 1 : enables assert() only, no display
|
||||
* 2 : reserved, for currently active debug path
|
||||
* 3 : events once per object lifetime (CCtx, CDict, etc.)
|
||||
* 4 : events once per frame
|
||||
@ -81,7 +91,7 @@ extern "C" {
|
||||
* 7+: events at every position (*very* verbose)
|
||||
*
|
||||
* It's generally inconvenient to output traces > 5.
|
||||
* In which case, it's possible to selectively enable higher verbosity levels
|
||||
* In which case, it's possible to selectively trigger high verbosity levels
|
||||
* by modifying g_debug_level.
|
||||
*/
|
||||
|
||||
@ -95,11 +105,12 @@ extern "C" {
|
||||
|
||||
#if (DEBUGLEVEL>=2)
|
||||
# include <stdio.h>
|
||||
extern int g_debuglevel; /* here, this variable is only declared,
|
||||
it actually lives in debug.c,
|
||||
and is shared by the whole process.
|
||||
It's typically used to enable very verbose levels
|
||||
on selective conditions (such as position in src) */
|
||||
extern int g_debuglevel; /* the variable is only declared,
|
||||
it actually lives in debug.c,
|
||||
and is shared by the whole process.
|
||||
It's not thread-safe.
|
||||
It's useful when enabling very verbose levels
|
||||
on selective conditions (such as position in src) */
|
||||
|
||||
# define RAWLOG(l, ...) { \
|
||||
if (l<=g_debuglevel) { \
|
||||
|
||||
@ -14,6 +14,10 @@
|
||||
|
||||
const char* ERR_getErrorString(ERR_enum code)
|
||||
{
|
||||
#ifdef ZSTD_STRIP_ERROR_STRINGS
|
||||
(void)code;
|
||||
return "Error strings stripped";
|
||||
#else
|
||||
static const char* const notErrorCode = "Unspecified error code";
|
||||
switch( code )
|
||||
{
|
||||
@ -39,10 +43,12 @@ const char* ERR_getErrorString(ERR_enum code)
|
||||
case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
|
||||
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
|
||||
case PREFIX(srcSize_wrong): return "Src size is incorrect";
|
||||
case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer";
|
||||
/* following error codes are not stable and may be removed or changed in a future version */
|
||||
case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
|
||||
case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
|
||||
case PREFIX(maxCode):
|
||||
default: return notErrorCode;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
@ -512,7 +512,7 @@ MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
|
||||
const U32 tableLog = MEM_read16(ptr);
|
||||
statePtr->value = (ptrdiff_t)1<<tableLog;
|
||||
statePtr->stateTable = u16ptr+2;
|
||||
statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
|
||||
statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1);
|
||||
statePtr->stateLog = tableLog;
|
||||
}
|
||||
|
||||
@ -531,7 +531,7 @@ MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U3
|
||||
}
|
||||
}
|
||||
|
||||
MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol)
|
||||
MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol)
|
||||
{
|
||||
FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
|
||||
const U16* const stateTable = (const U16*)(statePtr->stateTable);
|
||||
|
||||
@ -173,15 +173,19 @@ typedef U32 HUF_DTable;
|
||||
* Advanced decompression functions
|
||||
******************************************/
|
||||
size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X1
|
||||
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
||||
#endif
|
||||
|
||||
size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */
|
||||
size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
|
||||
size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */
|
||||
size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
||||
size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X1
|
||||
size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
||||
size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
|
||||
#endif
|
||||
|
||||
|
||||
/* ****************************************
|
||||
@ -228,7 +232,7 @@ size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
|
||||
#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1)
|
||||
#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
|
||||
size_t HUF_buildCTable_wksp (HUF_CElt* tree,
|
||||
const U32* count, U32 maxSymbolValue, U32 maxNbBits,
|
||||
const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
|
||||
void* workSpace, size_t wkspSize);
|
||||
|
||||
/*! HUF_readStats() :
|
||||
@ -277,14 +281,22 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
|
||||
#define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10)
|
||||
#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
|
||||
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X2
|
||||
size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize);
|
||||
size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
|
||||
#endif
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X1
|
||||
size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
|
||||
size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
|
||||
#endif
|
||||
|
||||
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X2
|
||||
size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
||||
#endif
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X1
|
||||
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
||||
#endif
|
||||
|
||||
|
||||
/* ====================== */
|
||||
@ -306,24 +318,36 @@ size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
|
||||
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
|
||||
|
||||
size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X1
|
||||
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
|
||||
#endif
|
||||
|
||||
size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
|
||||
size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X2
|
||||
size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
||||
size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
|
||||
#endif
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X1
|
||||
size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
||||
size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
|
||||
#endif
|
||||
|
||||
size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X2
|
||||
size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
||||
#endif
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X1
|
||||
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
||||
#endif
|
||||
|
||||
/* BMI2 variants.
|
||||
* If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
|
||||
*/
|
||||
size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X2
|
||||
size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
|
||||
#endif
|
||||
size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
|
||||
size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
|
||||
|
||||
|
||||
@ -88,8 +88,8 @@ static void* POOL_thread(void* opaque) {
|
||||
ctx->numThreadsBusy++;
|
||||
ctx->queueEmpty = ctx->queueHead == ctx->queueTail;
|
||||
/* Unlock the mutex, signal a pusher, and run the job */
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
|
||||
job.function(job.opaque);
|
||||
|
||||
|
||||
@ -30,8 +30,10 @@ const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; }
|
||||
/*-****************************************
|
||||
* ZSTD Error Management
|
||||
******************************************/
|
||||
#undef ZSTD_isError /* defined within zstd_internal.h */
|
||||
/*! ZSTD_isError() :
|
||||
* tells if a return value is an error code */
|
||||
* tells if a return value is an error code
|
||||
* symbol is required for external callers */
|
||||
unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
|
||||
|
||||
/*! ZSTD_getErrorName() :
|
||||
|
||||
@ -72,6 +72,7 @@ typedef enum {
|
||||
ZSTD_error_workSpace_tooSmall= 66,
|
||||
ZSTD_error_dstSize_tooSmall = 70,
|
||||
ZSTD_error_srcSize_wrong = 72,
|
||||
ZSTD_error_dstBuffer_null = 74,
|
||||
/* following error codes are __NOT STABLE__, they can be removed or changed in future versions */
|
||||
ZSTD_error_frameIndex_tooLarge = 100,
|
||||
ZSTD_error_seekableIO = 102,
|
||||
|
||||
@ -41,6 +41,9 @@ extern "C" {
|
||||
|
||||
/* ---- static assert (debug) --- */
|
||||
#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c)
|
||||
#define ZSTD_isError ERR_isError /* for inlining */
|
||||
#define FSE_isError ERR_isError
|
||||
#define HUF_isError ERR_isError
|
||||
|
||||
|
||||
/*-*************************************
|
||||
@ -75,7 +78,6 @@ static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
|
||||
#define BIT0 1
|
||||
|
||||
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
|
||||
#define ZSTD_WINDOWLOG_DEFAULTMAX 27 /* Default maximum allowed window log */
|
||||
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
|
||||
static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
|
||||
|
||||
@ -242,7 +244,7 @@ typedef struct {
|
||||
blockType_e blockType;
|
||||
U32 lastBlock;
|
||||
U32 origSize;
|
||||
} blockProperties_t;
|
||||
} blockProperties_t; /* declared here for decompress and fullbench */
|
||||
|
||||
/*! ZSTD_getcBlockSize() :
|
||||
* Provides the size of compressed block from block header `src` */
|
||||
@ -250,6 +252,13 @@ typedef struct {
|
||||
size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
|
||||
blockProperties_t* bpPtr);
|
||||
|
||||
/*! ZSTD_decodeSeqHeaders() :
|
||||
* decode sequence header from src */
|
||||
/* Used by: decompress, fullbench (does not get its definition from here) */
|
||||
size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -115,7 +115,7 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
|
||||
/* symbol start positions */
|
||||
{ U32 u;
|
||||
cumul[0] = 0;
|
||||
for (u=1; u<=maxSymbolValue+1; u++) {
|
||||
for (u=1; u <= maxSymbolValue+1; u++) {
|
||||
if (normalizedCounter[u-1]==-1) { /* Low proba symbol */
|
||||
cumul[u] = cumul[u-1] + 1;
|
||||
tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1);
|
||||
@ -658,7 +658,7 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src
|
||||
BYTE* op = ostart;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
|
||||
U32 count[FSE_MAX_SYMBOL_VALUE+1];
|
||||
unsigned count[FSE_MAX_SYMBOL_VALUE+1];
|
||||
S16 norm[FSE_MAX_SYMBOL_VALUE+1];
|
||||
FSE_CTable* CTable = (FSE_CTable*)workSpace;
|
||||
size_t const CTableSize = FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue);
|
||||
@ -672,7 +672,7 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src
|
||||
if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG;
|
||||
|
||||
/* Scan input and build symbol stats */
|
||||
{ CHECK_V_F(maxCount, HIST_count_wksp(count, &maxSymbolValue, src, srcSize, (unsigned*)scratchBuffer) );
|
||||
{ CHECK_V_F(maxCount, HIST_count_wksp(count, &maxSymbolValue, src, srcSize, scratchBuffer, scratchBufferSize) );
|
||||
if (maxCount == srcSize) return 1; /* only a single symbol in src : rle */
|
||||
if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */
|
||||
if (maxCount < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */
|
||||
|
||||
@ -73,6 +73,7 @@ unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
return largestCount;
|
||||
}
|
||||
|
||||
typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e;
|
||||
|
||||
/* HIST_count_parallel_wksp() :
|
||||
* store histogram into 4 intermediate tables, recombined at the end.
|
||||
@ -85,8 +86,8 @@ unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
static size_t HIST_count_parallel_wksp(
|
||||
unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* source, size_t sourceSize,
|
||||
unsigned checkMax,
|
||||
unsigned* const workSpace)
|
||||
HIST_checkInput_e check,
|
||||
U32* const workSpace)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)source;
|
||||
const BYTE* const iend = ip+sourceSize;
|
||||
@ -137,7 +138,7 @@ static size_t HIST_count_parallel_wksp(
|
||||
/* finish last symbols */
|
||||
while (ip<iend) Counting1[*ip++]++;
|
||||
|
||||
if (checkMax) { /* verify stats will fit into destination table */
|
||||
if (check) { /* verify stats will fit into destination table */
|
||||
U32 s; for (s=255; s>maxSymbolValue; s--) {
|
||||
Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s];
|
||||
if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall);
|
||||
@ -157,14 +158,18 @@ static size_t HIST_count_parallel_wksp(
|
||||
|
||||
/* HIST_countFast_wksp() :
|
||||
* Same as HIST_countFast(), but using an externally provided scratch buffer.
|
||||
* `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */
|
||||
* `workSpace` is a writable buffer which must be 4-bytes aligned,
|
||||
* `workSpaceSize` must be >= HIST_WKSP_SIZE
|
||||
*/
|
||||
size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* source, size_t sourceSize,
|
||||
unsigned* workSpace)
|
||||
void* workSpace, size_t workSpaceSize)
|
||||
{
|
||||
if (sourceSize < 1500) /* heuristic threshold */
|
||||
return HIST_count_simple(count, maxSymbolValuePtr, source, sourceSize);
|
||||
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 0, workSpace);
|
||||
if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
|
||||
if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall);
|
||||
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace);
|
||||
}
|
||||
|
||||
/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */
|
||||
@ -172,24 +177,27 @@ size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* source, size_t sourceSize)
|
||||
{
|
||||
unsigned tmpCounters[HIST_WKSP_SIZE_U32];
|
||||
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters);
|
||||
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters));
|
||||
}
|
||||
|
||||
/* HIST_count_wksp() :
|
||||
* Same as HIST_count(), but using an externally provided scratch buffer.
|
||||
* `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */
|
||||
size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* source, size_t sourceSize, unsigned* workSpace)
|
||||
const void* source, size_t sourceSize,
|
||||
void* workSpace, size_t workSpaceSize)
|
||||
{
|
||||
if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
|
||||
if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall);
|
||||
if (*maxSymbolValuePtr < 255)
|
||||
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 1, workSpace);
|
||||
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, checkMaxSymbolValue, (U32*)workSpace);
|
||||
*maxSymbolValuePtr = 255;
|
||||
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace);
|
||||
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize);
|
||||
}
|
||||
|
||||
size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
unsigned tmpCounters[HIST_WKSP_SIZE_U32];
|
||||
return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters);
|
||||
return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters));
|
||||
}
|
||||
|
||||
@ -41,11 +41,11 @@
|
||||
|
||||
/*! HIST_count():
|
||||
* Provides the precise count of each byte within a table 'count'.
|
||||
* 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1).
|
||||
* 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1).
|
||||
* Updates *maxSymbolValuePtr with actual largest symbol value detected.
|
||||
* @return : count of the most frequent symbol (which isn't identified).
|
||||
* or an error code, which can be tested using HIST_isError().
|
||||
* note : if return == srcSize, there is only one symbol.
|
||||
* @return : count of the most frequent symbol (which isn't identified).
|
||||
* or an error code, which can be tested using HIST_isError().
|
||||
* note : if return == srcSize, there is only one symbol.
|
||||
*/
|
||||
size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* src, size_t srcSize);
|
||||
@ -56,14 +56,16 @@ unsigned HIST_isError(size_t code); /**< tells if a return value is an error co
|
||||
/* --- advanced histogram functions --- */
|
||||
|
||||
#define HIST_WKSP_SIZE_U32 1024
|
||||
#define HIST_WKSP_SIZE (HIST_WKSP_SIZE_U32 * sizeof(unsigned))
|
||||
/** HIST_count_wksp() :
|
||||
* Same as HIST_count(), but using an externally provided scratch buffer.
|
||||
* Benefit is this function will use very little stack space.
|
||||
* `workSpace` must be a table of unsigned of size >= HIST_WKSP_SIZE_U32
|
||||
* `workSpace` is a writable buffer which must be 4-bytes aligned,
|
||||
* `workSpaceSize` must be >= HIST_WKSP_SIZE
|
||||
*/
|
||||
size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned* workSpace);
|
||||
void* workSpace, size_t workSpaceSize);
|
||||
|
||||
/** HIST_countFast() :
|
||||
* same as HIST_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr.
|
||||
@ -74,11 +76,12 @@ size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
|
||||
/** HIST_countFast_wksp() :
|
||||
* Same as HIST_countFast(), but using an externally provided scratch buffer.
|
||||
* `workSpace` must be a table of unsigned of size >= HIST_WKSP_SIZE_U32
|
||||
* `workSpace` is a writable buffer which must be 4-bytes aligned,
|
||||
* `workSpaceSize` must be >= HIST_WKSP_SIZE
|
||||
*/
|
||||
size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned* workSpace);
|
||||
void* workSpace, size_t workSpaceSize);
|
||||
|
||||
/*! HIST_count_simple() :
|
||||
* Same as HIST_countFast(), this function is unsafe,
|
||||
|
||||
@ -88,13 +88,13 @@ static size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weight
|
||||
BYTE* op = ostart;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
|
||||
U32 maxSymbolValue = HUF_TABLELOG_MAX;
|
||||
unsigned maxSymbolValue = HUF_TABLELOG_MAX;
|
||||
U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
|
||||
|
||||
FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)];
|
||||
BYTE scratchBuffer[1<<MAX_FSE_TABLELOG_FOR_HUFF_HEADER];
|
||||
|
||||
U32 count[HUF_TABLELOG_MAX+1];
|
||||
unsigned count[HUF_TABLELOG_MAX+1];
|
||||
S16 norm[HUF_TABLELOG_MAX+1];
|
||||
|
||||
/* init conditions */
|
||||
@ -134,7 +134,7 @@ struct HUF_CElt_s {
|
||||
`CTable` : Huffman tree to save, using huf representation.
|
||||
@return : size of saved CTable */
|
||||
size_t HUF_writeCTable (void* dst, size_t maxDstSize,
|
||||
const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
|
||||
const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog)
|
||||
{
|
||||
BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */
|
||||
BYTE huffWeight[HUF_SYMBOLVALUE_MAX];
|
||||
@ -169,7 +169,7 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize,
|
||||
}
|
||||
|
||||
|
||||
size_t HUF_readCTable (HUF_CElt* CTable, U32* maxSymbolValuePtr, const void* src, size_t srcSize)
|
||||
size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize)
|
||||
{
|
||||
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */
|
||||
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
|
||||
@ -315,7 +315,7 @@ typedef struct {
|
||||
U32 current;
|
||||
} rankPos;
|
||||
|
||||
static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
|
||||
static void HUF_sort(nodeElt* huffNode, const unsigned* count, U32 maxSymbolValue)
|
||||
{
|
||||
rankPos rank[32];
|
||||
U32 n;
|
||||
@ -347,7 +347,7 @@ static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
|
||||
*/
|
||||
#define STARTNODE (HUF_SYMBOLVALUE_MAX+1)
|
||||
typedef nodeElt huffNodeTable[HUF_CTABLE_WORKSPACE_SIZE_U32];
|
||||
size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize)
|
||||
size_t HUF_buildCTable_wksp (HUF_CElt* tree, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize)
|
||||
{
|
||||
nodeElt* const huffNode0 = (nodeElt*)workSpace;
|
||||
nodeElt* const huffNode = huffNode0+1;
|
||||
@ -421,7 +421,7 @@ size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValu
|
||||
* @return : maxNbBits
|
||||
* Note : count is used before tree is written, so they can safely overlap
|
||||
*/
|
||||
size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
|
||||
size_t HUF_buildCTable (HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits)
|
||||
{
|
||||
huffNodeTable nodeTable;
|
||||
return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, nodeTable, sizeof(nodeTable));
|
||||
@ -610,13 +610,14 @@ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, si
|
||||
return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0);
|
||||
}
|
||||
|
||||
typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e;
|
||||
|
||||
static size_t HUF_compressCTable_internal(
|
||||
BYTE* const ostart, BYTE* op, BYTE* const oend,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned singleStream, const HUF_CElt* CTable, const int bmi2)
|
||||
HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int bmi2)
|
||||
{
|
||||
size_t const cSize = singleStream ?
|
||||
size_t const cSize = (nbStreams==HUF_singleStream) ?
|
||||
HUF_compress1X_usingCTable_internal(op, oend - op, src, srcSize, CTable, bmi2) :
|
||||
HUF_compress4X_usingCTable_internal(op, oend - op, src, srcSize, CTable, bmi2);
|
||||
if (HUF_isError(cSize)) { return cSize; }
|
||||
@ -628,21 +629,21 @@ static size_t HUF_compressCTable_internal(
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
U32 count[HUF_SYMBOLVALUE_MAX + 1];
|
||||
unsigned count[HUF_SYMBOLVALUE_MAX + 1];
|
||||
HUF_CElt CTable[HUF_SYMBOLVALUE_MAX + 1];
|
||||
huffNodeTable nodeTable;
|
||||
} HUF_compress_tables_t;
|
||||
|
||||
/* HUF_compress_internal() :
|
||||
* `workSpace` must a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
|
||||
static size_t HUF_compress_internal (
|
||||
void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog,
|
||||
unsigned singleStream,
|
||||
void* workSpace, size_t wkspSize,
|
||||
HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat,
|
||||
const int bmi2)
|
||||
static size_t
|
||||
HUF_compress_internal (void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog,
|
||||
HUF_nbStreams_e nbStreams,
|
||||
void* workSpace, size_t wkspSize,
|
||||
HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat,
|
||||
const int bmi2)
|
||||
{
|
||||
HUF_compress_tables_t* const table = (HUF_compress_tables_t*)workSpace;
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
@ -651,7 +652,7 @@ static size_t HUF_compress_internal (
|
||||
|
||||
/* checks & inits */
|
||||
if (((size_t)workSpace & 3) != 0) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
|
||||
if (wkspSize < sizeof(*table)) return ERROR(workSpace_tooSmall);
|
||||
if (wkspSize < HUF_WORKSPACE_SIZE) return ERROR(workSpace_tooSmall);
|
||||
if (!srcSize) return 0; /* Uncompressed */
|
||||
if (!dstSize) return 0; /* cannot fit anything within dst budget */
|
||||
if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */
|
||||
@ -664,11 +665,11 @@ static size_t HUF_compress_internal (
|
||||
if (preferRepeat && repeat && *repeat == HUF_repeat_valid) {
|
||||
return HUF_compressCTable_internal(ostart, op, oend,
|
||||
src, srcSize,
|
||||
singleStream, oldHufTable, bmi2);
|
||||
nbStreams, oldHufTable, bmi2);
|
||||
}
|
||||
|
||||
/* Scan input and build symbol stats */
|
||||
{ CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, table->count) );
|
||||
{ CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, workSpace, wkspSize) );
|
||||
if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */
|
||||
if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */
|
||||
}
|
||||
@ -683,14 +684,15 @@ static size_t HUF_compress_internal (
|
||||
if (preferRepeat && repeat && *repeat != HUF_repeat_none) {
|
||||
return HUF_compressCTable_internal(ostart, op, oend,
|
||||
src, srcSize,
|
||||
singleStream, oldHufTable, bmi2);
|
||||
nbStreams, oldHufTable, bmi2);
|
||||
}
|
||||
|
||||
/* Build Huffman Tree */
|
||||
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
|
||||
{ CHECK_V_F(maxBits, HUF_buildCTable_wksp(table->CTable, table->count,
|
||||
maxSymbolValue, huffLog,
|
||||
table->nodeTable, sizeof(table->nodeTable)) );
|
||||
{ size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count,
|
||||
maxSymbolValue, huffLog,
|
||||
table->nodeTable, sizeof(table->nodeTable));
|
||||
CHECK_F(maxBits);
|
||||
huffLog = (U32)maxBits;
|
||||
/* Zero unused symbols in CTable, so we can check it for validity */
|
||||
memset(table->CTable + (maxSymbolValue + 1), 0,
|
||||
@ -706,7 +708,7 @@ static size_t HUF_compress_internal (
|
||||
if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) {
|
||||
return HUF_compressCTable_internal(ostart, op, oend,
|
||||
src, srcSize,
|
||||
singleStream, oldHufTable, bmi2);
|
||||
nbStreams, oldHufTable, bmi2);
|
||||
} }
|
||||
|
||||
/* Use the new huffman table */
|
||||
@ -718,7 +720,7 @@ static size_t HUF_compress_internal (
|
||||
}
|
||||
return HUF_compressCTable_internal(ostart, op, oend,
|
||||
src, srcSize,
|
||||
singleStream, table->CTable, bmi2);
|
||||
nbStreams, table->CTable, bmi2);
|
||||
}
|
||||
|
||||
|
||||
@ -728,7 +730,7 @@ size_t HUF_compress1X_wksp (void* dst, size_t dstSize,
|
||||
void* workSpace, size_t wkspSize)
|
||||
{
|
||||
return HUF_compress_internal(dst, dstSize, src, srcSize,
|
||||
maxSymbolValue, huffLog, 1 /*single stream*/,
|
||||
maxSymbolValue, huffLog, HUF_singleStream,
|
||||
workSpace, wkspSize,
|
||||
NULL, NULL, 0, 0 /*bmi2*/);
|
||||
}
|
||||
@ -740,7 +742,7 @@ size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
|
||||
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2)
|
||||
{
|
||||
return HUF_compress_internal(dst, dstSize, src, srcSize,
|
||||
maxSymbolValue, huffLog, 1 /*single stream*/,
|
||||
maxSymbolValue, huffLog, HUF_singleStream,
|
||||
workSpace, wkspSize, hufTable,
|
||||
repeat, preferRepeat, bmi2);
|
||||
}
|
||||
@ -762,7 +764,7 @@ size_t HUF_compress4X_wksp (void* dst, size_t dstSize,
|
||||
void* workSpace, size_t wkspSize)
|
||||
{
|
||||
return HUF_compress_internal(dst, dstSize, src, srcSize,
|
||||
maxSymbolValue, huffLog, 0 /*4 streams*/,
|
||||
maxSymbolValue, huffLog, HUF_fourStreams,
|
||||
workSpace, wkspSize,
|
||||
NULL, NULL, 0, 0 /*bmi2*/);
|
||||
}
|
||||
@ -777,7 +779,7 @@ size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
|
||||
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2)
|
||||
{
|
||||
return HUF_compress_internal(dst, dstSize, src, srcSize,
|
||||
maxSymbolValue, huffLog, 0 /* 4 streams */,
|
||||
maxSymbolValue, huffLog, HUF_fourStreams,
|
||||
workSpace, wkspSize,
|
||||
hufTable, repeat, preferRepeat, bmi2);
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -48,12 +48,6 @@ extern "C" {
|
||||
typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e;
|
||||
typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage;
|
||||
|
||||
typedef enum {
|
||||
ZSTD_dictDefaultAttach = 0,
|
||||
ZSTD_dictForceAttach = 1,
|
||||
ZSTD_dictForceCopy = -1,
|
||||
} ZSTD_dictAttachPref_e;
|
||||
|
||||
typedef struct ZSTD_prefixDict_s {
|
||||
const void* dict;
|
||||
size_t dictSize;
|
||||
@ -96,10 +90,10 @@ typedef enum { zop_dynamic=0, zop_predef } ZSTD_OptPrice_e;
|
||||
|
||||
typedef struct {
|
||||
/* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */
|
||||
U32* litFreq; /* table of literals statistics, of size 256 */
|
||||
U32* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */
|
||||
U32* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */
|
||||
U32* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */
|
||||
unsigned* litFreq; /* table of literals statistics, of size 256 */
|
||||
unsigned* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */
|
||||
unsigned* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */
|
||||
unsigned* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */
|
||||
ZSTD_match_t* matchTable; /* list of found matches, of size ZSTD_OPT_NUM+1 */
|
||||
ZSTD_optimal_t* priceTable; /* All positions tracked by optimal parser, of size ZSTD_OPT_NUM+1 */
|
||||
|
||||
@ -139,7 +133,7 @@ struct ZSTD_matchState_t {
|
||||
U32* hashTable3;
|
||||
U32* chainTable;
|
||||
optState_t opt; /* optimal parser state */
|
||||
const ZSTD_matchState_t *dictMatchState;
|
||||
const ZSTD_matchState_t * dictMatchState;
|
||||
ZSTD_compressionParameters cParams;
|
||||
};
|
||||
|
||||
@ -167,7 +161,7 @@ typedef struct {
|
||||
U32 hashLog; /* Log size of hashTable */
|
||||
U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */
|
||||
U32 minMatchLength; /* Minimum match length */
|
||||
U32 hashEveryLog; /* Log number of entries to skip */
|
||||
U32 hashRateLog; /* Log number of entries to skip */
|
||||
U32 windowLog; /* Window log for the LDM */
|
||||
} ldmParams_t;
|
||||
|
||||
@ -196,9 +190,10 @@ struct ZSTD_CCtx_params_s {
|
||||
ZSTD_dictAttachPref_e attachDictPref;
|
||||
|
||||
/* Multithreading: used to pass parameters to mtctx */
|
||||
unsigned nbWorkers;
|
||||
unsigned jobSize;
|
||||
unsigned overlapSizeLog;
|
||||
int nbWorkers;
|
||||
size_t jobSize;
|
||||
int overlapLog;
|
||||
int rsyncable;
|
||||
|
||||
/* Long distance matching parameters */
|
||||
ldmParams_t ldmParams;
|
||||
@ -498,6 +493,64 @@ MEM_STATIC size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
|
||||
}
|
||||
}
|
||||
|
||||
/** ZSTD_ipow() :
|
||||
* Return base^exponent.
|
||||
*/
|
||||
static U64 ZSTD_ipow(U64 base, U64 exponent)
|
||||
{
|
||||
U64 power = 1;
|
||||
while (exponent) {
|
||||
if (exponent & 1) power *= base;
|
||||
exponent >>= 1;
|
||||
base *= base;
|
||||
}
|
||||
return power;
|
||||
}
|
||||
|
||||
#define ZSTD_ROLL_HASH_CHAR_OFFSET 10
|
||||
|
||||
/** ZSTD_rollingHash_append() :
|
||||
* Add the buffer to the hash value.
|
||||
*/
|
||||
static U64 ZSTD_rollingHash_append(U64 hash, void const* buf, size_t size)
|
||||
{
|
||||
BYTE const* istart = (BYTE const*)buf;
|
||||
size_t pos;
|
||||
for (pos = 0; pos < size; ++pos) {
|
||||
hash *= prime8bytes;
|
||||
hash += istart[pos] + ZSTD_ROLL_HASH_CHAR_OFFSET;
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
|
||||
/** ZSTD_rollingHash_compute() :
|
||||
* Compute the rolling hash value of the buffer.
|
||||
*/
|
||||
MEM_STATIC U64 ZSTD_rollingHash_compute(void const* buf, size_t size)
|
||||
{
|
||||
return ZSTD_rollingHash_append(0, buf, size);
|
||||
}
|
||||
|
||||
/** ZSTD_rollingHash_primePower() :
|
||||
* Compute the primePower to be passed to ZSTD_rollingHash_rotate() for a hash
|
||||
* over a window of length bytes.
|
||||
*/
|
||||
MEM_STATIC U64 ZSTD_rollingHash_primePower(U32 length)
|
||||
{
|
||||
return ZSTD_ipow(prime8bytes, length - 1);
|
||||
}
|
||||
|
||||
/** ZSTD_rollingHash_rotate() :
|
||||
* Rotate the rolling hash by one byte.
|
||||
*/
|
||||
MEM_STATIC U64 ZSTD_rollingHash_rotate(U64 hash, BYTE toRemove, BYTE toAdd, U64 primePower)
|
||||
{
|
||||
hash -= (toRemove + ZSTD_ROLL_HASH_CHAR_OFFSET) * primePower;
|
||||
hash *= prime8bytes;
|
||||
hash += toAdd + ZSTD_ROLL_HASH_CHAR_OFFSET;
|
||||
return hash;
|
||||
}
|
||||
|
||||
/*-*************************************
|
||||
* Round buffer management
|
||||
***************************************/
|
||||
@ -626,20 +679,23 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
|
||||
* dictMatchState mode, lowLimit and dictLimit are the same, and the dictionary
|
||||
* is below them. forceWindow and dictMatchState are therefore incompatible.
|
||||
*/
|
||||
MEM_STATIC void ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
|
||||
void const* srcEnd, U32 maxDist,
|
||||
U32* loadedDictEndPtr,
|
||||
const ZSTD_matchState_t** dictMatchStatePtr)
|
||||
MEM_STATIC void
|
||||
ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
|
||||
void const* srcEnd,
|
||||
U32 maxDist,
|
||||
U32* loadedDictEndPtr,
|
||||
const ZSTD_matchState_t** dictMatchStatePtr)
|
||||
{
|
||||
U32 const current = (U32)((BYTE const*)srcEnd - window->base);
|
||||
U32 loadedDictEnd = loadedDictEndPtr != NULL ? *loadedDictEndPtr : 0;
|
||||
DEBUGLOG(5, "ZSTD_window_enforceMaxDist: current=%u, maxDist=%u", current, maxDist);
|
||||
if (current > maxDist + loadedDictEnd) {
|
||||
U32 const newLowLimit = current - maxDist;
|
||||
U32 const blockEndIdx = (U32)((BYTE const*)srcEnd - window->base);
|
||||
U32 loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0;
|
||||
DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u",
|
||||
(unsigned)blockEndIdx, (unsigned)maxDist);
|
||||
if (blockEndIdx > maxDist + loadedDictEnd) {
|
||||
U32 const newLowLimit = blockEndIdx - maxDist;
|
||||
if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit;
|
||||
if (window->dictLimit < window->lowLimit) {
|
||||
DEBUGLOG(5, "Update dictLimit to match lowLimit, from %u to %u",
|
||||
window->dictLimit, window->lowLimit);
|
||||
(unsigned)window->dictLimit, (unsigned)window->lowLimit);
|
||||
window->dictLimit = window->lowLimit;
|
||||
}
|
||||
if (loadedDictEndPtr)
|
||||
@ -690,20 +746,23 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
|
||||
|
||||
|
||||
/* debug functions */
|
||||
#if (DEBUGLEVEL>=2)
|
||||
|
||||
MEM_STATIC double ZSTD_fWeight(U32 rawStat)
|
||||
{
|
||||
U32 const fp_accuracy = 8;
|
||||
U32 const fp_multiplier = (1 << fp_accuracy);
|
||||
U32 const stat = rawStat + 1;
|
||||
U32 const hb = ZSTD_highbit32(stat);
|
||||
U32 const newStat = rawStat + 1;
|
||||
U32 const hb = ZSTD_highbit32(newStat);
|
||||
U32 const BWeight = hb * fp_multiplier;
|
||||
U32 const FWeight = (stat << fp_accuracy) >> hb;
|
||||
U32 const FWeight = (newStat << fp_accuracy) >> hb;
|
||||
U32 const weight = BWeight + FWeight;
|
||||
assert(hb + fp_accuracy < 31);
|
||||
return (double)weight / fp_multiplier;
|
||||
}
|
||||
|
||||
/* display a table content,
|
||||
* listing each element, its frequency, and its predicted bit cost */
|
||||
MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max)
|
||||
{
|
||||
unsigned u, sum;
|
||||
@ -715,6 +774,9 @@ MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max)
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -18,7 +18,7 @@ void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
|
||||
const ZSTD_compressionParameters* const cParams = &ms->cParams;
|
||||
U32* const hashLarge = ms->hashTable;
|
||||
U32 const hBitsL = cParams->hashLog;
|
||||
U32 const mls = cParams->searchLength;
|
||||
U32 const mls = cParams->minMatch;
|
||||
U32* const hashSmall = ms->chainTable;
|
||||
U32 const hBitsS = cParams->chainLog;
|
||||
const BYTE* const base = ms->window.base;
|
||||
@ -309,7 +309,7 @@ size_t ZSTD_compressBlock_doubleFast(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
const U32 mls = ms->cParams.searchLength;
|
||||
const U32 mls = ms->cParams.minMatch;
|
||||
switch(mls)
|
||||
{
|
||||
default: /* includes case 3 */
|
||||
@ -329,7 +329,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
const U32 mls = ms->cParams.searchLength;
|
||||
const U32 mls = ms->cParams.minMatch;
|
||||
switch(mls)
|
||||
{
|
||||
default: /* includes case 3 */
|
||||
@ -483,7 +483,7 @@ size_t ZSTD_compressBlock_doubleFast_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
U32 const mls = ms->cParams.searchLength;
|
||||
U32 const mls = ms->cParams.minMatch;
|
||||
switch(mls)
|
||||
{
|
||||
default: /* includes case 3 */
|
||||
|
||||
@ -18,7 +18,7 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
|
||||
const ZSTD_compressionParameters* const cParams = &ms->cParams;
|
||||
U32* const hashTable = ms->hashTable;
|
||||
U32 const hBits = cParams->hashLog;
|
||||
U32 const mls = cParams->searchLength;
|
||||
U32 const mls = cParams->minMatch;
|
||||
const BYTE* const base = ms->window.base;
|
||||
const BYTE* ip = base + ms->nextToUpdate;
|
||||
const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
|
||||
@ -27,18 +27,18 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
|
||||
/* Always insert every fastHashFillStep position into the hash table.
|
||||
* Insert the other positions if their hash entry is empty.
|
||||
*/
|
||||
for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) {
|
||||
for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) {
|
||||
U32 const current = (U32)(ip - base);
|
||||
U32 i;
|
||||
for (i = 0; i < fastHashFillStep; ++i) {
|
||||
size_t const hash = ZSTD_hashPtr(ip + i, hBits, mls);
|
||||
if (i == 0 || hashTable[hash] == 0)
|
||||
hashTable[hash] = current + i;
|
||||
/* Only load extra positions for ZSTD_dtlm_full */
|
||||
if (dtlm == ZSTD_dtlm_fast)
|
||||
break;
|
||||
}
|
||||
}
|
||||
size_t const hash0 = ZSTD_hashPtr(ip, hBits, mls);
|
||||
hashTable[hash0] = current;
|
||||
if (dtlm == ZSTD_dtlm_fast) continue;
|
||||
/* Only load extra positions for ZSTD_dtlm_full */
|
||||
{ U32 p;
|
||||
for (p = 1; p < fastHashFillStep; ++p) {
|
||||
size_t const hash = ZSTD_hashPtr(ip + p, hBits, mls);
|
||||
if (hashTable[hash] == 0) { /* not yet filled */
|
||||
hashTable[hash] = current + p;
|
||||
} } } }
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE
|
||||
@ -235,7 +235,7 @@ size_t ZSTD_compressBlock_fast(
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
ZSTD_compressionParameters const* cParams = &ms->cParams;
|
||||
U32 const mls = cParams->searchLength;
|
||||
U32 const mls = cParams->minMatch;
|
||||
assert(ms->dictMatchState == NULL);
|
||||
switch(mls)
|
||||
{
|
||||
@ -256,7 +256,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState(
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
ZSTD_compressionParameters const* cParams = &ms->cParams;
|
||||
U32 const mls = cParams->searchLength;
|
||||
U32 const mls = cParams->minMatch;
|
||||
assert(ms->dictMatchState != NULL);
|
||||
switch(mls)
|
||||
{
|
||||
@ -375,7 +375,7 @@ size_t ZSTD_compressBlock_fast_extDict(
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
ZSTD_compressionParameters const* cParams = &ms->cParams;
|
||||
U32 const mls = cParams->searchLength;
|
||||
U32 const mls = cParams->minMatch;
|
||||
switch(mls)
|
||||
{
|
||||
default: /* includes case 3 */
|
||||
|
||||
@ -63,12 +63,13 @@ ZSTD_updateDUBT(ZSTD_matchState_t* ms,
|
||||
static void
|
||||
ZSTD_insertDUBT1(ZSTD_matchState_t* ms,
|
||||
U32 current, const BYTE* inputEnd,
|
||||
U32 nbCompares, U32 btLow, const ZSTD_dictMode_e dictMode)
|
||||
U32 nbCompares, U32 btLow,
|
||||
const ZSTD_dictMode_e dictMode)
|
||||
{
|
||||
const ZSTD_compressionParameters* const cParams = &ms->cParams;
|
||||
U32* const bt = ms->chainTable;
|
||||
U32 const btLog = cParams->chainLog - 1;
|
||||
U32 const btMask = (1 << btLog) - 1;
|
||||
U32* const bt = ms->chainTable;
|
||||
U32 const btLog = cParams->chainLog - 1;
|
||||
U32 const btMask = (1 << btLog) - 1;
|
||||
size_t commonLengthSmaller=0, commonLengthLarger=0;
|
||||
const BYTE* const base = ms->window.base;
|
||||
const BYTE* const dictBase = ms->window.dictBase;
|
||||
@ -80,7 +81,7 @@ ZSTD_insertDUBT1(ZSTD_matchState_t* ms,
|
||||
const BYTE* match;
|
||||
U32* smallerPtr = bt + 2*(current&btMask);
|
||||
U32* largerPtr = smallerPtr + 1;
|
||||
U32 matchIndex = *smallerPtr;
|
||||
U32 matchIndex = *smallerPtr; /* this candidate is unsorted : next sorted candidate is reached through *smallerPtr, while *largerPtr contains previous unsorted candidate (which is already saved and can be overwritten) */
|
||||
U32 dummy32; /* to be nullified at the end */
|
||||
U32 const windowLow = ms->window.lowLimit;
|
||||
|
||||
@ -93,6 +94,9 @@ ZSTD_insertDUBT1(ZSTD_matchState_t* ms,
|
||||
U32* const nextPtr = bt + 2*(matchIndex & btMask);
|
||||
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
|
||||
assert(matchIndex < current);
|
||||
/* note : all candidates are now supposed sorted,
|
||||
* but it's still possible to have nextPtr[1] == ZSTD_DUBT_UNSORTED_MARK
|
||||
* when a real index has the same value as ZSTD_DUBT_UNSORTED_MARK */
|
||||
|
||||
if ( (dictMode != ZSTD_extDict)
|
||||
|| (matchIndex+matchLength >= dictLimit) /* both in current segment*/
|
||||
@ -108,7 +112,7 @@ ZSTD_insertDUBT1(ZSTD_matchState_t* ms,
|
||||
match = dictBase + matchIndex;
|
||||
matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart);
|
||||
if (matchIndex+matchLength >= dictLimit)
|
||||
match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
|
||||
match = base + matchIndex; /* preparation for next read of match[matchLength] */
|
||||
}
|
||||
|
||||
DEBUGLOG(8, "ZSTD_insertDUBT1: comparing %u with %u : found %u common bytes ",
|
||||
@ -258,7 +262,7 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
|
||||
&& (nbCandidates > 1) ) {
|
||||
DEBUGLOG(8, "ZSTD_DUBT_findBestMatch: candidate %u is unsorted",
|
||||
matchIndex);
|
||||
*unsortedMark = previousCandidate;
|
||||
*unsortedMark = previousCandidate; /* the unsortedMark becomes a reversed chain, to move up back to original position */
|
||||
previousCandidate = matchIndex;
|
||||
matchIndex = *nextCandidate;
|
||||
nextCandidate = bt + 2*(matchIndex&btMask);
|
||||
@ -266,11 +270,13 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
|
||||
nbCandidates --;
|
||||
}
|
||||
|
||||
/* nullify last candidate if it's still unsorted
|
||||
* simplification, detrimental to compression ratio, beneficial for speed */
|
||||
if ( (matchIndex > unsortLimit)
|
||||
&& (*unsortedMark==ZSTD_DUBT_UNSORTED_MARK) ) {
|
||||
DEBUGLOG(7, "ZSTD_DUBT_findBestMatch: nullify last unsorted candidate %u",
|
||||
matchIndex);
|
||||
*nextCandidate = *unsortedMark = 0; /* nullify next candidate if it's still unsorted (note : simplification, detrimental to compression ratio, beneficial for speed) */
|
||||
*nextCandidate = *unsortedMark = 0;
|
||||
}
|
||||
|
||||
/* batch sort stacked candidates */
|
||||
@ -285,14 +291,14 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
|
||||
}
|
||||
|
||||
/* find longest match */
|
||||
{ size_t commonLengthSmaller=0, commonLengthLarger=0;
|
||||
{ size_t commonLengthSmaller = 0, commonLengthLarger = 0;
|
||||
const BYTE* const dictBase = ms->window.dictBase;
|
||||
const U32 dictLimit = ms->window.dictLimit;
|
||||
const BYTE* const dictEnd = dictBase + dictLimit;
|
||||
const BYTE* const prefixStart = base + dictLimit;
|
||||
U32* smallerPtr = bt + 2*(current&btMask);
|
||||
U32* largerPtr = bt + 2*(current&btMask) + 1;
|
||||
U32 matchEndIdx = current+8+1;
|
||||
U32 matchEndIdx = current + 8 + 1;
|
||||
U32 dummy32; /* to be nullified at the end */
|
||||
size_t bestLength = 0;
|
||||
|
||||
@ -386,7 +392,7 @@ ZSTD_BtFindBestMatch_selectMLS ( ZSTD_matchState_t* ms,
|
||||
const BYTE* ip, const BYTE* const iLimit,
|
||||
size_t* offsetPtr)
|
||||
{
|
||||
switch(ms->cParams.searchLength)
|
||||
switch(ms->cParams.minMatch)
|
||||
{
|
||||
default : /* includes case 3 */
|
||||
case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict);
|
||||
@ -402,7 +408,7 @@ static size_t ZSTD_BtFindBestMatch_dictMatchState_selectMLS (
|
||||
const BYTE* ip, const BYTE* const iLimit,
|
||||
size_t* offsetPtr)
|
||||
{
|
||||
switch(ms->cParams.searchLength)
|
||||
switch(ms->cParams.minMatch)
|
||||
{
|
||||
default : /* includes case 3 */
|
||||
case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState);
|
||||
@ -418,7 +424,7 @@ static size_t ZSTD_BtFindBestMatch_extDict_selectMLS (
|
||||
const BYTE* ip, const BYTE* const iLimit,
|
||||
size_t* offsetPtr)
|
||||
{
|
||||
switch(ms->cParams.searchLength)
|
||||
switch(ms->cParams.minMatch)
|
||||
{
|
||||
default : /* includes case 3 */
|
||||
case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict);
|
||||
@ -433,7 +439,7 @@ static size_t ZSTD_BtFindBestMatch_extDict_selectMLS (
|
||||
/* *********************************
|
||||
* Hash Chain
|
||||
***********************************/
|
||||
#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & mask]
|
||||
#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & (mask)]
|
||||
|
||||
/* Update chains up to ip (excluded)
|
||||
Assumption : always within prefix (i.e. not within extDict) */
|
||||
@ -463,7 +469,7 @@ static U32 ZSTD_insertAndFindFirstIndex_internal(
|
||||
|
||||
U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) {
|
||||
const ZSTD_compressionParameters* const cParams = &ms->cParams;
|
||||
return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.searchLength);
|
||||
return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch);
|
||||
}
|
||||
|
||||
|
||||
@ -497,6 +503,7 @@ size_t ZSTD_HcFindBestMatch_generic (
|
||||
size_t currentMl=0;
|
||||
if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) {
|
||||
const BYTE* const match = base + matchIndex;
|
||||
assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */
|
||||
if (match[ml] == ip[ml]) /* potentially better */
|
||||
currentMl = ZSTD_count(ip, match, iLimit);
|
||||
} else {
|
||||
@ -559,7 +566,7 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS (
|
||||
const BYTE* ip, const BYTE* const iLimit,
|
||||
size_t* offsetPtr)
|
||||
{
|
||||
switch(ms->cParams.searchLength)
|
||||
switch(ms->cParams.minMatch)
|
||||
{
|
||||
default : /* includes case 3 */
|
||||
case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict);
|
||||
@ -575,7 +582,7 @@ static size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS (
|
||||
const BYTE* ip, const BYTE* const iLimit,
|
||||
size_t* offsetPtr)
|
||||
{
|
||||
switch(ms->cParams.searchLength)
|
||||
switch(ms->cParams.minMatch)
|
||||
{
|
||||
default : /* includes case 3 */
|
||||
case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState);
|
||||
@ -591,7 +598,7 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS (
|
||||
const BYTE* ip, const BYTE* const iLimit,
|
||||
size_t* offsetPtr)
|
||||
{
|
||||
switch(ms->cParams.searchLength)
|
||||
switch(ms->cParams.minMatch)
|
||||
{
|
||||
default : /* includes case 3 */
|
||||
case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict);
|
||||
|
||||
@ -37,8 +37,8 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params,
|
||||
params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG);
|
||||
assert(params->hashLog <= ZSTD_HASHLOG_MAX);
|
||||
}
|
||||
if (params->hashEveryLog == 0) {
|
||||
params->hashEveryLog = params->windowLog < params->hashLog
|
||||
if (params->hashRateLog == 0) {
|
||||
params->hashRateLog = params->windowLog < params->hashLog
|
||||
? 0
|
||||
: params->windowLog - params->hashLog;
|
||||
}
|
||||
@ -119,20 +119,20 @@ static void ZSTD_ldm_insertEntry(ldmState_t* ldmState,
|
||||
*
|
||||
* Gets the small hash, checksum, and tag from the rollingHash.
|
||||
*
|
||||
* If the tag matches (1 << ldmParams.hashEveryLog)-1, then
|
||||
* If the tag matches (1 << ldmParams.hashRateLog)-1, then
|
||||
* creates an ldmEntry from the offset, and inserts it into the hash table.
|
||||
*
|
||||
* hBits is the length of the small hash, which is the most significant hBits
|
||||
* of rollingHash. The checksum is the next 32 most significant bits, followed
|
||||
* by ldmParams.hashEveryLog bits that make up the tag. */
|
||||
* by ldmParams.hashRateLog bits that make up the tag. */
|
||||
static void ZSTD_ldm_makeEntryAndInsertByTag(ldmState_t* ldmState,
|
||||
U64 const rollingHash,
|
||||
U32 const hBits,
|
||||
U32 const offset,
|
||||
ldmParams_t const ldmParams)
|
||||
{
|
||||
U32 const tag = ZSTD_ldm_getTag(rollingHash, hBits, ldmParams.hashEveryLog);
|
||||
U32 const tagMask = ((U32)1 << ldmParams.hashEveryLog) - 1;
|
||||
U32 const tag = ZSTD_ldm_getTag(rollingHash, hBits, ldmParams.hashRateLog);
|
||||
U32 const tagMask = ((U32)1 << ldmParams.hashRateLog) - 1;
|
||||
if (tag == tagMask) {
|
||||
U32 const hash = ZSTD_ldm_getSmallHash(rollingHash, hBits);
|
||||
U32 const checksum = ZSTD_ldm_getChecksum(rollingHash, hBits);
|
||||
@ -143,56 +143,6 @@ static void ZSTD_ldm_makeEntryAndInsertByTag(ldmState_t* ldmState,
|
||||
}
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_getRollingHash() :
|
||||
* Get a 64-bit hash using the first len bytes from buf.
|
||||
*
|
||||
* Giving bytes s = s_1, s_2, ... s_k, the hash is defined to be
|
||||
* H(s) = s_1*(a^(k-1)) + s_2*(a^(k-2)) + ... + s_k*(a^0)
|
||||
*
|
||||
* where the constant a is defined to be prime8bytes.
|
||||
*
|
||||
* The implementation adds an offset to each byte, so
|
||||
* H(s) = (s_1 + HASH_CHAR_OFFSET)*(a^(k-1)) + ... */
|
||||
static U64 ZSTD_ldm_getRollingHash(const BYTE* buf, U32 len)
|
||||
{
|
||||
U64 ret = 0;
|
||||
U32 i;
|
||||
for (i = 0; i < len; i++) {
|
||||
ret *= prime8bytes;
|
||||
ret += buf[i] + LDM_HASH_CHAR_OFFSET;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_ipow() :
|
||||
* Return base^exp. */
|
||||
static U64 ZSTD_ldm_ipow(U64 base, U64 exp)
|
||||
{
|
||||
U64 ret = 1;
|
||||
while (exp) {
|
||||
if (exp & 1) { ret *= base; }
|
||||
exp >>= 1;
|
||||
base *= base;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
U64 ZSTD_ldm_getHashPower(U32 minMatchLength) {
|
||||
DEBUGLOG(4, "ZSTD_ldm_getHashPower: mml=%u", minMatchLength);
|
||||
assert(minMatchLength >= ZSTD_LDM_MINMATCH_MIN);
|
||||
return ZSTD_ldm_ipow(prime8bytes, minMatchLength - 1);
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_updateHash() :
|
||||
* Updates hash by removing toRemove and adding toAdd. */
|
||||
static U64 ZSTD_ldm_updateHash(U64 hash, BYTE toRemove, BYTE toAdd, U64 hashPower)
|
||||
{
|
||||
hash -= ((toRemove + LDM_HASH_CHAR_OFFSET) * hashPower);
|
||||
hash *= prime8bytes;
|
||||
hash += toAdd + LDM_HASH_CHAR_OFFSET;
|
||||
return hash;
|
||||
}
|
||||
|
||||
/** ZSTD_ldm_countBackwardsMatch() :
|
||||
* Returns the number of bytes that match backwards before pIn and pMatch.
|
||||
*
|
||||
@ -238,6 +188,7 @@ static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms,
|
||||
case ZSTD_btlazy2:
|
||||
case ZSTD_btopt:
|
||||
case ZSTD_btultra:
|
||||
case ZSTD_btultra2:
|
||||
break;
|
||||
default:
|
||||
assert(0); /* not possible : not a valid strategy id */
|
||||
@ -261,9 +212,9 @@ static U64 ZSTD_ldm_fillLdmHashTable(ldmState_t* state,
|
||||
const BYTE* cur = lastHashed + 1;
|
||||
|
||||
while (cur < iend) {
|
||||
rollingHash = ZSTD_ldm_updateHash(rollingHash, cur[-1],
|
||||
cur[ldmParams.minMatchLength-1],
|
||||
state->hashPower);
|
||||
rollingHash = ZSTD_rollingHash_rotate(rollingHash, cur[-1],
|
||||
cur[ldmParams.minMatchLength-1],
|
||||
state->hashPower);
|
||||
ZSTD_ldm_makeEntryAndInsertByTag(state,
|
||||
rollingHash, hBits,
|
||||
(U32)(cur - base), ldmParams);
|
||||
@ -297,8 +248,8 @@ static size_t ZSTD_ldm_generateSequences_internal(
|
||||
U64 const hashPower = ldmState->hashPower;
|
||||
U32 const hBits = params->hashLog - params->bucketSizeLog;
|
||||
U32 const ldmBucketSize = 1U << params->bucketSizeLog;
|
||||
U32 const hashEveryLog = params->hashEveryLog;
|
||||
U32 const ldmTagMask = (1U << params->hashEveryLog) - 1;
|
||||
U32 const hashRateLog = params->hashRateLog;
|
||||
U32 const ldmTagMask = (1U << params->hashRateLog) - 1;
|
||||
/* Prefix and extDict parameters */
|
||||
U32 const dictLimit = ldmState->window.dictLimit;
|
||||
U32 const lowestIndex = extDict ? ldmState->window.lowLimit : dictLimit;
|
||||
@ -324,16 +275,16 @@ static size_t ZSTD_ldm_generateSequences_internal(
|
||||
size_t forwardMatchLength = 0, backwardMatchLength = 0;
|
||||
ldmEntry_t* bestEntry = NULL;
|
||||
if (ip != istart) {
|
||||
rollingHash = ZSTD_ldm_updateHash(rollingHash, lastHashed[0],
|
||||
lastHashed[minMatchLength],
|
||||
hashPower);
|
||||
rollingHash = ZSTD_rollingHash_rotate(rollingHash, lastHashed[0],
|
||||
lastHashed[minMatchLength],
|
||||
hashPower);
|
||||
} else {
|
||||
rollingHash = ZSTD_ldm_getRollingHash(ip, minMatchLength);
|
||||
rollingHash = ZSTD_rollingHash_compute(ip, minMatchLength);
|
||||
}
|
||||
lastHashed = ip;
|
||||
|
||||
/* Do not insert and do not look for a match */
|
||||
if (ZSTD_ldm_getTag(rollingHash, hBits, hashEveryLog) != ldmTagMask) {
|
||||
if (ZSTD_ldm_getTag(rollingHash, hBits, hashRateLog) != ldmTagMask) {
|
||||
ip++;
|
||||
continue;
|
||||
}
|
||||
@ -593,7 +544,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
|
||||
void const* src, size_t srcSize)
|
||||
{
|
||||
const ZSTD_compressionParameters* const cParams = &ms->cParams;
|
||||
unsigned const minMatch = cParams->searchLength;
|
||||
unsigned const minMatch = cParams->minMatch;
|
||||
ZSTD_blockCompressor const blockCompressor =
|
||||
ZSTD_selectBlockCompressor(cParams->strategy, ZSTD_matchState_dictMode(ms));
|
||||
/* Input bounds */
|
||||
|
||||
@ -21,7 +21,7 @@ extern "C" {
|
||||
* Long distance matching
|
||||
***************************************/
|
||||
|
||||
#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_DEFAULTMAX
|
||||
#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_LIMIT_DEFAULT
|
||||
|
||||
/**
|
||||
* ZSTD_ldm_generateSequences():
|
||||
@ -86,12 +86,8 @@ size_t ZSTD_ldm_getTableSize(ldmParams_t params);
|
||||
*/
|
||||
size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize);
|
||||
|
||||
/** ZSTD_ldm_getTableSize() :
|
||||
* Return prime8bytes^(minMatchLength-1) */
|
||||
U64 ZSTD_ldm_getHashPower(U32 minMatchLength);
|
||||
|
||||
/** ZSTD_ldm_adjustParameters() :
|
||||
* If the params->hashEveryLog is not set, set it to its default value based on
|
||||
* If the params->hashRateLog is not set, set it to its default value based on
|
||||
* windowLog and params->hashLog.
|
||||
*
|
||||
* Ensures that params->bucketSizeLog is <= params->hashLog (setting it to
|
||||
|
||||
@ -17,6 +17,8 @@
|
||||
#define ZSTD_FREQ_DIV 4 /* log factor when using previous stats to init next stats */
|
||||
#define ZSTD_MAX_PRICE (1<<30)
|
||||
|
||||
#define ZSTD_PREDEF_THRESHOLD 1024 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Price functions for optimal parser
|
||||
@ -52,11 +54,15 @@ MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat)
|
||||
return weight;
|
||||
}
|
||||
|
||||
/* debugging function, @return price in bytes */
|
||||
#if (DEBUGLEVEL>=2)
|
||||
/* debugging function,
|
||||
* @return price in bytes as fractional value
|
||||
* for debug messages only */
|
||||
MEM_STATIC double ZSTD_fCost(U32 price)
|
||||
{
|
||||
return (double)price / (BITCOST_MULTIPLIER*8);
|
||||
}
|
||||
#endif
|
||||
|
||||
static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel)
|
||||
{
|
||||
@ -67,29 +73,44 @@ static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel)
|
||||
}
|
||||
|
||||
|
||||
static U32 ZSTD_downscaleStat(U32* table, U32 lastEltIndex, int malus)
|
||||
/* ZSTD_downscaleStat() :
|
||||
* reduce all elements in table by a factor 2^(ZSTD_FREQ_DIV+malus)
|
||||
* return the resulting sum of elements */
|
||||
static U32 ZSTD_downscaleStat(unsigned* table, U32 lastEltIndex, int malus)
|
||||
{
|
||||
U32 s, sum=0;
|
||||
DEBUGLOG(5, "ZSTD_downscaleStat (nbElts=%u)", (unsigned)lastEltIndex+1);
|
||||
assert(ZSTD_FREQ_DIV+malus > 0 && ZSTD_FREQ_DIV+malus < 31);
|
||||
for (s=0; s<=lastEltIndex; s++) {
|
||||
for (s=0; s<lastEltIndex+1; s++) {
|
||||
table[s] = 1 + (table[s] >> (ZSTD_FREQ_DIV+malus));
|
||||
sum += table[s];
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
static void ZSTD_rescaleFreqs(optState_t* const optPtr,
|
||||
const BYTE* const src, size_t const srcSize,
|
||||
int optLevel)
|
||||
/* ZSTD_rescaleFreqs() :
|
||||
* if first block (detected by optPtr->litLengthSum == 0) : init statistics
|
||||
* take hints from dictionary if there is one
|
||||
* or init from zero, using src for literals stats, or flat 1 for match symbols
|
||||
* otherwise downscale existing stats, to be used as seed for next block.
|
||||
*/
|
||||
static void
|
||||
ZSTD_rescaleFreqs(optState_t* const optPtr,
|
||||
const BYTE* const src, size_t const srcSize,
|
||||
int const optLevel)
|
||||
{
|
||||
DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize);
|
||||
optPtr->priceType = zop_dynamic;
|
||||
|
||||
if (optPtr->litLengthSum == 0) { /* first block : init */
|
||||
if (srcSize <= 1024) /* heuristic */
|
||||
if (srcSize <= ZSTD_PREDEF_THRESHOLD) { /* heuristic */
|
||||
DEBUGLOG(5, "(srcSize <= ZSTD_PREDEF_THRESHOLD) => zop_predef");
|
||||
optPtr->priceType = zop_predef;
|
||||
}
|
||||
|
||||
assert(optPtr->symbolCosts != NULL);
|
||||
if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) { /* huffman table presumed generated by dictionary */
|
||||
if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) {
|
||||
/* huffman table presumed generated by dictionary */
|
||||
optPtr->priceType = zop_dynamic;
|
||||
|
||||
assert(optPtr->litFreq != NULL);
|
||||
@ -208,7 +229,9 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP
|
||||
|
||||
/* dynamic statistics */
|
||||
{ U32 const llCode = ZSTD_LLcode(litLength);
|
||||
return (LL_bits[llCode] * BITCOST_MULTIPLIER) + (optPtr->litLengthSumBasePrice - WEIGHT(optPtr->litLengthFreq[llCode], optLevel));
|
||||
return (LL_bits[llCode] * BITCOST_MULTIPLIER)
|
||||
+ optPtr->litLengthSumBasePrice
|
||||
- WEIGHT(optPtr->litLengthFreq[llCode], optLevel);
|
||||
}
|
||||
}
|
||||
|
||||
@ -253,7 +276,7 @@ static int ZSTD_literalsContribution(const BYTE* const literals, U32 const litLe
|
||||
FORCE_INLINE_TEMPLATE U32
|
||||
ZSTD_getMatchPrice(U32 const offset,
|
||||
U32 const matchLength,
|
||||
const optState_t* const optPtr,
|
||||
const optState_t* const optPtr,
|
||||
int const optLevel)
|
||||
{
|
||||
U32 price;
|
||||
@ -385,7 +408,6 @@ static U32 ZSTD_insertBt1(
|
||||
U32* largerPtr = smallerPtr + 1;
|
||||
U32 dummy32; /* to be nullified at the end */
|
||||
U32 const windowLow = ms->window.lowLimit;
|
||||
U32 const matchLow = windowLow ? windowLow : 1;
|
||||
U32 matchEndIdx = current+8+1;
|
||||
size_t bestLength = 8;
|
||||
U32 nbCompares = 1U << cParams->searchLog;
|
||||
@ -401,7 +423,8 @@ static U32 ZSTD_insertBt1(
|
||||
assert(ip <= iend-8); /* required for h calculation */
|
||||
hashTable[h] = current; /* Update Hash Table */
|
||||
|
||||
while (nbCompares-- && (matchIndex >= matchLow)) {
|
||||
assert(windowLow > 0);
|
||||
while (nbCompares-- && (matchIndex >= windowLow)) {
|
||||
U32* const nextPtr = bt + 2*(matchIndex & btMask);
|
||||
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
|
||||
assert(matchIndex < current);
|
||||
@ -479,7 +502,7 @@ void ZSTD_updateTree_internal(
|
||||
const BYTE* const base = ms->window.base;
|
||||
U32 const target = (U32)(ip - base);
|
||||
U32 idx = ms->nextToUpdate;
|
||||
DEBUGLOG(5, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)",
|
||||
DEBUGLOG(6, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)",
|
||||
idx, target, dictMode);
|
||||
|
||||
while(idx < target)
|
||||
@ -488,15 +511,18 @@ void ZSTD_updateTree_internal(
|
||||
}
|
||||
|
||||
void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) {
|
||||
ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.searchLength, ZSTD_noDict);
|
||||
ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict);
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE
|
||||
U32 ZSTD_insertBtAndGetAllMatches (
|
||||
ZSTD_matchState_t* ms,
|
||||
const BYTE* const ip, const BYTE* const iLimit, const ZSTD_dictMode_e dictMode,
|
||||
U32 rep[ZSTD_REP_NUM], U32 const ll0,
|
||||
ZSTD_match_t* matches, const U32 lengthToBeat, U32 const mls /* template */)
|
||||
U32 rep[ZSTD_REP_NUM],
|
||||
U32 const ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */
|
||||
ZSTD_match_t* matches,
|
||||
const U32 lengthToBeat,
|
||||
U32 const mls /* template */)
|
||||
{
|
||||
const ZSTD_compressionParameters* const cParams = &ms->cParams;
|
||||
U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
|
||||
@ -542,6 +568,7 @@ U32 ZSTD_insertBtAndGetAllMatches (
|
||||
DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", current);
|
||||
|
||||
/* check repCode */
|
||||
assert(ll0 <= 1); /* necessarily 1 or 0 */
|
||||
{ U32 const lastR = ZSTD_REP_NUM + ll0;
|
||||
U32 repCode;
|
||||
for (repCode = ll0; repCode < lastR; repCode++) {
|
||||
@ -724,7 +751,7 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
|
||||
ZSTD_match_t* matches, U32 const lengthToBeat)
|
||||
{
|
||||
const ZSTD_compressionParameters* const cParams = &ms->cParams;
|
||||
U32 const matchLengthSearch = cParams->searchLength;
|
||||
U32 const matchLengthSearch = cParams->minMatch;
|
||||
DEBUGLOG(8, "ZSTD_BtGetAllMatches");
|
||||
if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */
|
||||
ZSTD_updateTree_internal(ms, ip, iHighLimit, matchLengthSearch, dictMode);
|
||||
@ -774,12 +801,30 @@ static U32 ZSTD_totalLen(ZSTD_optimal_t sol)
|
||||
return sol.litlen + sol.mlen;
|
||||
}
|
||||
|
||||
#if 0 /* debug */
|
||||
|
||||
static void
|
||||
listStats(const U32* table, int lastEltID)
|
||||
{
|
||||
int const nbElts = lastEltID + 1;
|
||||
int enb;
|
||||
for (enb=0; enb < nbElts; enb++) {
|
||||
(void)table;
|
||||
//RAWLOG(2, "%3i:%3i, ", enb, table[enb]);
|
||||
RAWLOG(2, "%4i,", table[enb]);
|
||||
}
|
||||
RAWLOG(2, " \n");
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
FORCE_INLINE_TEMPLATE size_t
|
||||
ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
|
||||
seqStore_t* seqStore,
|
||||
U32 rep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize,
|
||||
const int optLevel, const ZSTD_dictMode_e dictMode)
|
||||
const void* src, size_t srcSize,
|
||||
const int optLevel,
|
||||
const ZSTD_dictMode_e dictMode)
|
||||
{
|
||||
optState_t* const optStatePtr = &ms->opt;
|
||||
const BYTE* const istart = (const BYTE*)src;
|
||||
@ -792,14 +837,15 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
|
||||
const ZSTD_compressionParameters* const cParams = &ms->cParams;
|
||||
|
||||
U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
|
||||
U32 const minMatch = (cParams->searchLength == 3) ? 3 : 4;
|
||||
U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4;
|
||||
|
||||
ZSTD_optimal_t* const opt = optStatePtr->priceTable;
|
||||
ZSTD_match_t* const matches = optStatePtr->matchTable;
|
||||
ZSTD_optimal_t lastSequence;
|
||||
|
||||
/* init */
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_opt_generic");
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u",
|
||||
(U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate);
|
||||
assert(optLevel <= 2);
|
||||
ms->nextToUpdate3 = ms->nextToUpdate;
|
||||
ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel);
|
||||
@ -999,7 +1045,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
|
||||
U32 const offCode = opt[storePos].off;
|
||||
U32 const advance = llen + mlen;
|
||||
DEBUGLOG(6, "considering seq starting at %zi, llen=%u, mlen=%u",
|
||||
anchor - istart, llen, mlen);
|
||||
anchor - istart, (unsigned)llen, (unsigned)mlen);
|
||||
|
||||
if (mlen==0) { /* only literals => must be last "sequence", actually starting a new stream of sequences */
|
||||
assert(storePos == storeEnd); /* must be last sequence */
|
||||
@ -1047,11 +1093,11 @@ size_t ZSTD_compressBlock_btopt(
|
||||
|
||||
|
||||
/* used in 2-pass strategy */
|
||||
static U32 ZSTD_upscaleStat(U32* table, U32 lastEltIndex, int bonus)
|
||||
static U32 ZSTD_upscaleStat(unsigned* table, U32 lastEltIndex, int bonus)
|
||||
{
|
||||
U32 s, sum=0;
|
||||
assert(ZSTD_FREQ_DIV+bonus > 0);
|
||||
for (s=0; s<=lastEltIndex; s++) {
|
||||
assert(ZSTD_FREQ_DIV+bonus >= 0);
|
||||
for (s=0; s<lastEltIndex+1; s++) {
|
||||
table[s] <<= ZSTD_FREQ_DIV+bonus;
|
||||
table[s]--;
|
||||
sum += table[s];
|
||||
@ -1063,9 +1109,43 @@ static U32 ZSTD_upscaleStat(U32* table, U32 lastEltIndex, int bonus)
|
||||
MEM_STATIC void ZSTD_upscaleStats(optState_t* optPtr)
|
||||
{
|
||||
optPtr->litSum = ZSTD_upscaleStat(optPtr->litFreq, MaxLit, 0);
|
||||
optPtr->litLengthSum = ZSTD_upscaleStat(optPtr->litLengthFreq, MaxLL, 1);
|
||||
optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 1);
|
||||
optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 1);
|
||||
optPtr->litLengthSum = ZSTD_upscaleStat(optPtr->litLengthFreq, MaxLL, 0);
|
||||
optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 0);
|
||||
optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 0);
|
||||
}
|
||||
|
||||
/* ZSTD_initStats_ultra():
|
||||
* make a first compression pass, just to seed stats with more accurate starting values.
|
||||
* only works on first block, with no dictionary and no ldm.
|
||||
* this function cannot error, hence its constract must be respected.
|
||||
*/
|
||||
static void
|
||||
ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
|
||||
seqStore_t* seqStore,
|
||||
U32 rep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */
|
||||
memcpy(tmpRep, rep, sizeof(tmpRep));
|
||||
|
||||
DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize);
|
||||
assert(ms->opt.litLengthSum == 0); /* first block */
|
||||
assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */
|
||||
assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */
|
||||
assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */
|
||||
|
||||
ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/
|
||||
|
||||
/* invalidate first scan from history */
|
||||
ZSTD_resetSeqStore(seqStore);
|
||||
ms->window.base -= srcSize;
|
||||
ms->window.dictLimit += (U32)srcSize;
|
||||
ms->window.lowLimit = ms->window.dictLimit;
|
||||
ms->nextToUpdate = ms->window.dictLimit;
|
||||
ms->nextToUpdate3 = ms->window.dictLimit;
|
||||
|
||||
/* re-inforce weight of collected statistics */
|
||||
ZSTD_upscaleStats(&ms->opt);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBlock_btultra(
|
||||
@ -1073,33 +1153,34 @@ size_t ZSTD_compressBlock_btultra(
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize);
|
||||
#if 0
|
||||
/* 2-pass strategy (disabled)
|
||||
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBlock_btultra2(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
U32 const current = (U32)((const BYTE*)src - ms->window.base);
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize);
|
||||
|
||||
/* 2-pass strategy:
|
||||
* this strategy makes a first pass over first block to collect statistics
|
||||
* and seed next round's statistics with it.
|
||||
* After 1st pass, function forgets everything, and starts a new block.
|
||||
* Consequently, this can only work if no data has been previously loaded in tables,
|
||||
* aka, no dictionary, no prefix, no ldm preprocessing.
|
||||
* The compression ratio gain is generally small (~0.5% on first block),
|
||||
* the cost is 2x cpu time on first block. */
|
||||
assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
|
||||
if ( (ms->opt.litLengthSum==0) /* first block */
|
||||
&& (seqStore->sequences == seqStore->sequencesStart) /* no ldm */
|
||||
&& (ms->window.dictLimit == ms->window.lowLimit) ) { /* no dictionary */
|
||||
U32 tmpRep[ZSTD_REP_NUM];
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_btultra: first block: collecting statistics");
|
||||
assert(ms->nextToUpdate >= ms->window.dictLimit
|
||||
&& ms->nextToUpdate <= ms->window.dictLimit + 1);
|
||||
memcpy(tmpRep, rep, sizeof(tmpRep));
|
||||
ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/
|
||||
ZSTD_resetSeqStore(seqStore);
|
||||
/* invalidate first scan from history */
|
||||
ms->window.base -= srcSize;
|
||||
ms->window.dictLimit += (U32)srcSize;
|
||||
ms->window.lowLimit = ms->window.dictLimit;
|
||||
ms->nextToUpdate = ms->window.dictLimit;
|
||||
ms->nextToUpdate3 = ms->window.dictLimit;
|
||||
/* re-inforce weight of collected statistics */
|
||||
ZSTD_upscaleStats(&ms->opt);
|
||||
&& (seqStore->sequences == seqStore->sequencesStart) /* no ldm */
|
||||
&& (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */
|
||||
&& (current == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */
|
||||
&& (srcSize > ZSTD_PREDEF_THRESHOLD)
|
||||
) {
|
||||
ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize);
|
||||
}
|
||||
#endif
|
||||
|
||||
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict);
|
||||
}
|
||||
|
||||
@ -1130,3 +1211,7 @@ size_t ZSTD_compressBlock_btultra_extDict(
|
||||
{
|
||||
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_extDict);
|
||||
}
|
||||
|
||||
/* note : no btultra2 variant for extDict nor dictMatchState,
|
||||
* because btultra2 is not meant to work with dictionaries
|
||||
* and is only specific for the first block (no prefix) */
|
||||
|
||||
@ -26,6 +26,10 @@ size_t ZSTD_compressBlock_btopt(
|
||||
size_t ZSTD_compressBlock_btultra(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
size_t ZSTD_compressBlock_btultra2(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
|
||||
size_t ZSTD_compressBlock_btopt_dictMatchState(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
@ -41,6 +45,10 @@ size_t ZSTD_compressBlock_btultra_extDict(
|
||||
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
void const* src, size_t srcSize);
|
||||
|
||||
/* note : no btultra2 variant for extDict nor dictMatchState,
|
||||
* because btultra2 is not meant to work with dictionaries
|
||||
* and is only specific for the first block (no prefix) */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -9,21 +9,19 @@
|
||||
*/
|
||||
|
||||
|
||||
/* ====== Tuning parameters ====== */
|
||||
#define ZSTDMT_NBWORKERS_MAX 200
|
||||
#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (2 GB)) /* note : limited by `jobSize` type, which is `unsigned` */
|
||||
#define ZSTDMT_OVERLAPLOG_DEFAULT 6
|
||||
|
||||
|
||||
/* ====== Compiler specifics ====== */
|
||||
#if defined(_MSC_VER)
|
||||
# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
|
||||
#endif
|
||||
|
||||
|
||||
/* ====== Constants ====== */
|
||||
#define ZSTDMT_OVERLAPLOG_DEFAULT 0
|
||||
|
||||
|
||||
/* ====== Dependencies ====== */
|
||||
#include <string.h> /* memcpy, memset */
|
||||
#include <limits.h> /* INT_MAX */
|
||||
#include <limits.h> /* INT_MAX, UINT_MAX */
|
||||
#include "pool.h" /* threadpool */
|
||||
#include "threading.h" /* mutex */
|
||||
#include "zstd_compress_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
|
||||
@ -57,9 +55,9 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void)
|
||||
static clock_t _ticksPerSecond = 0;
|
||||
if (_ticksPerSecond <= 0) _ticksPerSecond = sysconf(_SC_CLK_TCK);
|
||||
|
||||
{ struct tms junk; clock_t newTicks = (clock_t) times(&junk);
|
||||
return ((((unsigned long long)newTicks)*(1000000))/_ticksPerSecond); }
|
||||
}
|
||||
{ struct tms junk; clock_t newTicks = (clock_t) times(&junk);
|
||||
return ((((unsigned long long)newTicks)*(1000000))/_ticksPerSecond);
|
||||
} }
|
||||
|
||||
#define MUTEX_WAIT_TIME_DLEVEL 6
|
||||
#define ZSTD_PTHREAD_MUTEX_LOCK(mutex) { \
|
||||
@ -342,8 +340,8 @@ static ZSTDMT_seqPool* ZSTDMT_expandSeqPool(ZSTDMT_seqPool* pool, U32 nbWorkers)
|
||||
|
||||
typedef struct {
|
||||
ZSTD_pthread_mutex_t poolMutex;
|
||||
unsigned totalCCtx;
|
||||
unsigned availCCtx;
|
||||
int totalCCtx;
|
||||
int availCCtx;
|
||||
ZSTD_customMem cMem;
|
||||
ZSTD_CCtx* cctx[1]; /* variable size */
|
||||
} ZSTDMT_CCtxPool;
|
||||
@ -351,16 +349,16 @@ typedef struct {
|
||||
/* note : all CCtx borrowed from the pool should be released back to the pool _before_ freeing the pool */
|
||||
static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool)
|
||||
{
|
||||
unsigned u;
|
||||
for (u=0; u<pool->totalCCtx; u++)
|
||||
ZSTD_freeCCtx(pool->cctx[u]); /* note : compatible with free on NULL */
|
||||
int cid;
|
||||
for (cid=0; cid<pool->totalCCtx; cid++)
|
||||
ZSTD_freeCCtx(pool->cctx[cid]); /* note : compatible with free on NULL */
|
||||
ZSTD_pthread_mutex_destroy(&pool->poolMutex);
|
||||
ZSTD_free(pool, pool->cMem);
|
||||
}
|
||||
|
||||
/* ZSTDMT_createCCtxPool() :
|
||||
* implies nbWorkers >= 1 , checked by caller ZSTDMT_createCCtx() */
|
||||
static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbWorkers,
|
||||
static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(int nbWorkers,
|
||||
ZSTD_customMem cMem)
|
||||
{
|
||||
ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_calloc(
|
||||
@ -381,7 +379,7 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbWorkers,
|
||||
}
|
||||
|
||||
static ZSTDMT_CCtxPool* ZSTDMT_expandCCtxPool(ZSTDMT_CCtxPool* srcPool,
|
||||
unsigned nbWorkers)
|
||||
int nbWorkers)
|
||||
{
|
||||
if (srcPool==NULL) return NULL;
|
||||
if (nbWorkers <= srcPool->totalCCtx) return srcPool; /* good enough */
|
||||
@ -469,9 +467,9 @@ static int ZSTDMT_serialState_reset(serialState_t* serialState, ZSTDMT_seqPool*
|
||||
DEBUGLOG(4, "LDM window size = %u KB", (1U << params.cParams.windowLog) >> 10);
|
||||
ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams);
|
||||
assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog);
|
||||
assert(params.ldmParams.hashEveryLog < 32);
|
||||
assert(params.ldmParams.hashRateLog < 32);
|
||||
serialState->ldmState.hashPower =
|
||||
ZSTD_ldm_getHashPower(params.ldmParams.minMatchLength);
|
||||
ZSTD_rollingHash_primePower(params.ldmParams.minMatchLength);
|
||||
} else {
|
||||
memset(¶ms.ldmParams, 0, sizeof(params.ldmParams));
|
||||
}
|
||||
@ -674,7 +672,7 @@ static void ZSTDMT_compressionJob(void* jobDescription)
|
||||
if (ZSTD_isError(initError)) JOB_ERROR(initError);
|
||||
} else { /* srcStart points at reloaded section */
|
||||
U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size;
|
||||
{ size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstJob);
|
||||
{ size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob);
|
||||
if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError);
|
||||
}
|
||||
{ size_t const initError = ZSTD_compressBegin_advanced_internal(cctx,
|
||||
@ -777,6 +775,14 @@ typedef struct {
|
||||
|
||||
static const roundBuff_t kNullRoundBuff = {NULL, 0, 0};
|
||||
|
||||
#define RSYNC_LENGTH 32
|
||||
|
||||
typedef struct {
|
||||
U64 hash;
|
||||
U64 hitMask;
|
||||
U64 primePower;
|
||||
} rsyncState_t;
|
||||
|
||||
struct ZSTDMT_CCtx_s {
|
||||
POOL_ctx* factory;
|
||||
ZSTDMT_jobDescription* jobs;
|
||||
@ -790,6 +796,7 @@ struct ZSTDMT_CCtx_s {
|
||||
inBuff_t inBuff;
|
||||
roundBuff_t roundBuff;
|
||||
serialState_t serial;
|
||||
rsyncState_t rsync;
|
||||
unsigned singleBlockingThread;
|
||||
unsigned jobIDMask;
|
||||
unsigned doneJobID;
|
||||
@ -859,7 +866,7 @@ size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorker
|
||||
{
|
||||
if (nbWorkers > ZSTDMT_NBWORKERS_MAX) nbWorkers = ZSTDMT_NBWORKERS_MAX;
|
||||
params->nbWorkers = nbWorkers;
|
||||
params->overlapSizeLog = ZSTDMT_OVERLAPLOG_DEFAULT;
|
||||
params->overlapLog = ZSTDMT_OVERLAPLOG_DEFAULT;
|
||||
params->jobSize = 0;
|
||||
return nbWorkers;
|
||||
}
|
||||
@ -969,52 +976,59 @@ size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx)
|
||||
}
|
||||
|
||||
/* Internal only */
|
||||
size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params,
|
||||
ZSTDMT_parameter parameter, unsigned value) {
|
||||
size_t
|
||||
ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params,
|
||||
ZSTDMT_parameter parameter,
|
||||
int value)
|
||||
{
|
||||
DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter");
|
||||
switch(parameter)
|
||||
{
|
||||
case ZSTDMT_p_jobSize :
|
||||
DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %u", value);
|
||||
if ( (value > 0) /* value==0 => automatic job size */
|
||||
& (value < ZSTDMT_JOBSIZE_MIN) )
|
||||
DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %i", value);
|
||||
if ( value != 0 /* default */
|
||||
&& value < ZSTDMT_JOBSIZE_MIN)
|
||||
value = ZSTDMT_JOBSIZE_MIN;
|
||||
if (value > ZSTDMT_JOBSIZE_MAX)
|
||||
value = ZSTDMT_JOBSIZE_MAX;
|
||||
assert(value >= 0);
|
||||
if (value > ZSTDMT_JOBSIZE_MAX) value = ZSTDMT_JOBSIZE_MAX;
|
||||
params->jobSize = value;
|
||||
return value;
|
||||
case ZSTDMT_p_overlapSectionLog :
|
||||
if (value > 9) value = 9;
|
||||
DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %u", value);
|
||||
params->overlapSizeLog = (value >= 9) ? 9 : value;
|
||||
|
||||
case ZSTDMT_p_overlapLog :
|
||||
DEBUGLOG(4, "ZSTDMT_p_overlapLog : %i", value);
|
||||
if (value < ZSTD_OVERLAPLOG_MIN) value = ZSTD_OVERLAPLOG_MIN;
|
||||
if (value > ZSTD_OVERLAPLOG_MAX) value = ZSTD_OVERLAPLOG_MAX;
|
||||
params->overlapLog = value;
|
||||
return value;
|
||||
|
||||
case ZSTDMT_p_rsyncable :
|
||||
value = (value != 0);
|
||||
params->rsyncable = value;
|
||||
return value;
|
||||
|
||||
default :
|
||||
return ERROR(parameter_unsupported);
|
||||
}
|
||||
}
|
||||
|
||||
size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned value)
|
||||
size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int value)
|
||||
{
|
||||
DEBUGLOG(4, "ZSTDMT_setMTCtxParameter");
|
||||
switch(parameter)
|
||||
{
|
||||
case ZSTDMT_p_jobSize :
|
||||
return ZSTDMT_CCtxParam_setMTCtxParameter(&mtctx->params, parameter, value);
|
||||
case ZSTDMT_p_overlapSectionLog :
|
||||
return ZSTDMT_CCtxParam_setMTCtxParameter(&mtctx->params, parameter, value);
|
||||
default :
|
||||
return ERROR(parameter_unsupported);
|
||||
}
|
||||
return ZSTDMT_CCtxParam_setMTCtxParameter(&mtctx->params, parameter, value);
|
||||
}
|
||||
|
||||
size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned* value)
|
||||
size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int* value)
|
||||
{
|
||||
switch (parameter) {
|
||||
case ZSTDMT_p_jobSize:
|
||||
*value = mtctx->params.jobSize;
|
||||
assert(mtctx->params.jobSize <= INT_MAX);
|
||||
*value = (int)(mtctx->params.jobSize);
|
||||
break;
|
||||
case ZSTDMT_p_overlapSectionLog:
|
||||
*value = mtctx->params.overlapSizeLog;
|
||||
case ZSTDMT_p_overlapLog:
|
||||
*value = mtctx->params.overlapLog;
|
||||
break;
|
||||
case ZSTDMT_p_rsyncable:
|
||||
*value = mtctx->params.rsyncable;
|
||||
break;
|
||||
default:
|
||||
return ERROR(parameter_unsupported);
|
||||
@ -1140,22 +1154,66 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx)
|
||||
/* ===== Multi-threaded compression ===== */
|
||||
/* ------------------------------------------ */
|
||||
|
||||
static size_t ZSTDMT_computeTargetJobLog(ZSTD_CCtx_params const params)
|
||||
static unsigned ZSTDMT_computeTargetJobLog(ZSTD_CCtx_params const params)
|
||||
{
|
||||
if (params.ldmParams.enableLdm)
|
||||
/* In Long Range Mode, the windowLog is typically oversized.
|
||||
* In which case, it's preferable to determine the jobSize
|
||||
* based on chainLog instead. */
|
||||
return MAX(21, params.cParams.chainLog + 4);
|
||||
return MAX(20, params.cParams.windowLog + 2);
|
||||
}
|
||||
|
||||
static size_t ZSTDMT_computeOverlapLog(ZSTD_CCtx_params const params)
|
||||
static int ZSTDMT_overlapLog_default(ZSTD_strategy strat)
|
||||
{
|
||||
unsigned const overlapRLog = (params.overlapSizeLog>9) ? 0 : 9-params.overlapSizeLog;
|
||||
if (params.ldmParams.enableLdm)
|
||||
return (MIN(params.cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) - overlapRLog);
|
||||
return overlapRLog >= 9 ? 0 : (params.cParams.windowLog - overlapRLog);
|
||||
switch(strat)
|
||||
{
|
||||
case ZSTD_btultra2:
|
||||
return 9;
|
||||
case ZSTD_btultra:
|
||||
case ZSTD_btopt:
|
||||
return 8;
|
||||
case ZSTD_btlazy2:
|
||||
case ZSTD_lazy2:
|
||||
return 7;
|
||||
case ZSTD_lazy:
|
||||
case ZSTD_greedy:
|
||||
case ZSTD_dfast:
|
||||
case ZSTD_fast:
|
||||
default:;
|
||||
}
|
||||
return 6;
|
||||
}
|
||||
|
||||
static unsigned ZSTDMT_computeNbJobs(ZSTD_CCtx_params params, size_t srcSize, unsigned nbWorkers) {
|
||||
static int ZSTDMT_overlapLog(int ovlog, ZSTD_strategy strat)
|
||||
{
|
||||
assert(0 <= ovlog && ovlog <= 9);
|
||||
if (ovlog == 0) return ZSTDMT_overlapLog_default(strat);
|
||||
return ovlog;
|
||||
}
|
||||
|
||||
static size_t ZSTDMT_computeOverlapSize(ZSTD_CCtx_params const params)
|
||||
{
|
||||
int const overlapRLog = 9 - ZSTDMT_overlapLog(params.overlapLog, params.cParams.strategy);
|
||||
int ovLog = (overlapRLog >= 8) ? 0 : (params.cParams.windowLog - overlapRLog);
|
||||
assert(0 <= overlapRLog && overlapRLog <= 8);
|
||||
if (params.ldmParams.enableLdm) {
|
||||
/* In Long Range Mode, the windowLog is typically oversized.
|
||||
* In which case, it's preferable to determine the jobSize
|
||||
* based on chainLog instead.
|
||||
* Then, ovLog becomes a fraction of the jobSize, rather than windowSize */
|
||||
ovLog = MIN(params.cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2)
|
||||
- overlapRLog;
|
||||
}
|
||||
assert(0 <= ovLog && ovLog <= 30);
|
||||
DEBUGLOG(4, "overlapLog : %i", params.overlapLog);
|
||||
DEBUGLOG(4, "overlap size : %i", 1 << ovLog);
|
||||
return (ovLog==0) ? 0 : (size_t)1 << ovLog;
|
||||
}
|
||||
|
||||
static unsigned
|
||||
ZSTDMT_computeNbJobs(ZSTD_CCtx_params params, size_t srcSize, unsigned nbWorkers)
|
||||
{
|
||||
assert(nbWorkers>0);
|
||||
{ size_t const jobSizeTarget = (size_t)1 << ZSTDMT_computeTargetJobLog(params);
|
||||
size_t const jobMaxSize = jobSizeTarget << 2;
|
||||
@ -1178,7 +1236,7 @@ static size_t ZSTDMT_compress_advanced_internal(
|
||||
ZSTD_CCtx_params params)
|
||||
{
|
||||
ZSTD_CCtx_params const jobParams = ZSTDMT_initJobCCtxParams(params);
|
||||
size_t const overlapSize = (size_t)1 << ZSTDMT_computeOverlapLog(params);
|
||||
size_t const overlapSize = ZSTDMT_computeOverlapSize(params);
|
||||
unsigned const nbJobs = ZSTDMT_computeNbJobs(params, srcSize, params.nbWorkers);
|
||||
size_t const proposedJobSize = (srcSize + (nbJobs-1)) / nbJobs;
|
||||
size_t const avgJobSize = (((proposedJobSize-1) & 0x1FFFF) < 0x7FFF) ? proposedJobSize + 0xFFFF : proposedJobSize; /* avoid too small last block */
|
||||
@ -1289,16 +1347,17 @@ static size_t ZSTDMT_compress_advanced_internal(
|
||||
}
|
||||
|
||||
size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const ZSTD_CDict* cdict,
|
||||
ZSTD_parameters params,
|
||||
unsigned overlapLog)
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const ZSTD_CDict* cdict,
|
||||
ZSTD_parameters params,
|
||||
int overlapLog)
|
||||
{
|
||||
ZSTD_CCtx_params cctxParams = mtctx->params;
|
||||
cctxParams.cParams = params.cParams;
|
||||
cctxParams.fParams = params.fParams;
|
||||
cctxParams.overlapSizeLog = overlapLog;
|
||||
assert(ZSTD_OVERLAPLOG_MIN <= overlapLog && overlapLog <= ZSTD_OVERLAPLOG_MAX);
|
||||
cctxParams.overlapLog = overlapLog;
|
||||
return ZSTDMT_compress_advanced_internal(mtctx,
|
||||
dst, dstCapacity,
|
||||
src, srcSize,
|
||||
@ -1311,8 +1370,8 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
|
||||
const void* src, size_t srcSize,
|
||||
int compressionLevel)
|
||||
{
|
||||
U32 const overlapLog = (compressionLevel >= ZSTD_maxCLevel()) ? 9 : ZSTDMT_OVERLAPLOG_DEFAULT;
|
||||
ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, 0);
|
||||
int const overlapLog = ZSTDMT_overlapLog_default(params.cParams.strategy);
|
||||
params.fParams.contentSizeFlag = 1;
|
||||
return ZSTDMT_compress_advanced(mtctx, dst, dstCapacity, src, srcSize, NULL, params, overlapLog);
|
||||
}
|
||||
@ -1339,8 +1398,8 @@ size_t ZSTDMT_initCStream_internal(
|
||||
if (params.nbWorkers != mtctx->params.nbWorkers)
|
||||
CHECK_F( ZSTDMT_resize(mtctx, params.nbWorkers) );
|
||||
|
||||
if (params.jobSize > 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN;
|
||||
if (params.jobSize > ZSTDMT_JOBSIZE_MAX) params.jobSize = ZSTDMT_JOBSIZE_MAX;
|
||||
if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN;
|
||||
if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = ZSTDMT_JOBSIZE_MAX;
|
||||
|
||||
mtctx->singleBlockingThread = (pledgedSrcSize <= ZSTDMT_JOBSIZE_MIN); /* do not trigger multi-threading when srcSize is too small */
|
||||
if (mtctx->singleBlockingThread) {
|
||||
@ -1375,14 +1434,24 @@ size_t ZSTDMT_initCStream_internal(
|
||||
mtctx->cdict = cdict;
|
||||
}
|
||||
|
||||
mtctx->targetPrefixSize = (size_t)1 << ZSTDMT_computeOverlapLog(params);
|
||||
DEBUGLOG(4, "overlapLog=%u => %u KB", params.overlapSizeLog, (U32)(mtctx->targetPrefixSize>>10));
|
||||
mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(params);
|
||||
DEBUGLOG(4, "overlapLog=%i => %u KB", params.overlapLog, (U32)(mtctx->targetPrefixSize>>10));
|
||||
mtctx->targetSectionSize = params.jobSize;
|
||||
if (mtctx->targetSectionSize == 0) {
|
||||
mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(params);
|
||||
}
|
||||
if (params.rsyncable) {
|
||||
/* Aim for the targetsectionSize as the average job size. */
|
||||
U32 const jobSizeMB = (U32)(mtctx->targetSectionSize >> 20);
|
||||
U32 const rsyncBits = ZSTD_highbit32(jobSizeMB) + 20;
|
||||
assert(jobSizeMB >= 1);
|
||||
DEBUGLOG(4, "rsyncLog = %u", rsyncBits);
|
||||
mtctx->rsync.hash = 0;
|
||||
mtctx->rsync.hitMask = (1ULL << rsyncBits) - 1;
|
||||
mtctx->rsync.primePower = ZSTD_rollingHash_primePower(RSYNC_LENGTH);
|
||||
}
|
||||
if (mtctx->targetSectionSize < mtctx->targetPrefixSize) mtctx->targetSectionSize = mtctx->targetPrefixSize; /* job size must be >= overlap size */
|
||||
DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(mtctx->targetSectionSize>>10), params.jobSize);
|
||||
DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(mtctx->targetSectionSize>>10), (U32)params.jobSize);
|
||||
DEBUGLOG(4, "inBuff Size : %u KB", (U32)(mtctx->targetSectionSize>>10));
|
||||
ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(mtctx->targetSectionSize));
|
||||
{
|
||||
@ -1818,6 +1887,89 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx)
|
||||
return 1;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
size_t toLoad; /* The number of bytes to load from the input. */
|
||||
int flush; /* Boolean declaring if we must flush because we found a synchronization point. */
|
||||
} syncPoint_t;
|
||||
|
||||
/**
|
||||
* Searches through the input for a synchronization point. If one is found, we
|
||||
* will instruct the caller to flush, and return the number of bytes to load.
|
||||
* Otherwise, we will load as many bytes as possible and instruct the caller
|
||||
* to continue as normal.
|
||||
*/
|
||||
static syncPoint_t
|
||||
findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input)
|
||||
{
|
||||
BYTE const* const istart = (BYTE const*)input.src + input.pos;
|
||||
U64 const primePower = mtctx->rsync.primePower;
|
||||
U64 const hitMask = mtctx->rsync.hitMask;
|
||||
|
||||
syncPoint_t syncPoint;
|
||||
U64 hash;
|
||||
BYTE const* prev;
|
||||
size_t pos;
|
||||
|
||||
syncPoint.toLoad = MIN(input.size - input.pos, mtctx->targetSectionSize - mtctx->inBuff.filled);
|
||||
syncPoint.flush = 0;
|
||||
if (!mtctx->params.rsyncable)
|
||||
/* Rsync is disabled. */
|
||||
return syncPoint;
|
||||
if (mtctx->inBuff.filled + syncPoint.toLoad < RSYNC_LENGTH)
|
||||
/* Not enough to compute the hash.
|
||||
* We will miss any synchronization points in this RSYNC_LENGTH byte
|
||||
* window. However, since it depends only in the internal buffers, if the
|
||||
* state is already synchronized, we will remain synchronized.
|
||||
* Additionally, the probability that we miss a synchronization point is
|
||||
* low: RSYNC_LENGTH / targetSectionSize.
|
||||
*/
|
||||
return syncPoint;
|
||||
/* Initialize the loop variables. */
|
||||
if (mtctx->inBuff.filled >= RSYNC_LENGTH) {
|
||||
/* We have enough bytes buffered to initialize the hash.
|
||||
* Start scanning at the beginning of the input.
|
||||
*/
|
||||
pos = 0;
|
||||
prev = (BYTE const*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled - RSYNC_LENGTH;
|
||||
hash = ZSTD_rollingHash_compute(prev, RSYNC_LENGTH);
|
||||
} else {
|
||||
/* We don't have enough bytes buffered to initialize the hash, but
|
||||
* we know we have at least RSYNC_LENGTH bytes total.
|
||||
* Start scanning after the first RSYNC_LENGTH bytes less the bytes
|
||||
* already buffered.
|
||||
*/
|
||||
pos = RSYNC_LENGTH - mtctx->inBuff.filled;
|
||||
prev = (BYTE const*)mtctx->inBuff.buffer.start - pos;
|
||||
hash = ZSTD_rollingHash_compute(mtctx->inBuff.buffer.start, mtctx->inBuff.filled);
|
||||
hash = ZSTD_rollingHash_append(hash, istart, pos);
|
||||
}
|
||||
/* Starting with the hash of the previous RSYNC_LENGTH bytes, roll
|
||||
* through the input. If we hit a synchronization point, then cut the
|
||||
* job off, and tell the compressor to flush the job. Otherwise, load
|
||||
* all the bytes and continue as normal.
|
||||
* If we go too long without a synchronization point (targetSectionSize)
|
||||
* then a block will be emitted anyways, but this is okay, since if we
|
||||
* are already synchronized we will remain synchronized.
|
||||
*/
|
||||
for (; pos < syncPoint.toLoad; ++pos) {
|
||||
BYTE const toRemove = pos < RSYNC_LENGTH ? prev[pos] : istart[pos - RSYNC_LENGTH];
|
||||
/* if (pos >= RSYNC_LENGTH) assert(ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); */
|
||||
hash = ZSTD_rollingHash_rotate(hash, toRemove, istart[pos], primePower);
|
||||
if ((hash & hitMask) == hitMask) {
|
||||
syncPoint.toLoad = pos + 1;
|
||||
syncPoint.flush = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return syncPoint;
|
||||
}
|
||||
|
||||
size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx)
|
||||
{
|
||||
size_t hintInSize = mtctx->targetSectionSize - mtctx->inBuff.filled;
|
||||
if (hintInSize==0) hintInSize = mtctx->targetSectionSize;
|
||||
return hintInSize;
|
||||
}
|
||||
|
||||
/** ZSTDMT_compressStream_generic() :
|
||||
* internal use only - exposed to be invoked from zstd_compress.c
|
||||
@ -1844,7 +1996,8 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
|
||||
}
|
||||
|
||||
/* single-pass shortcut (note : synchronous-mode) */
|
||||
if ( (mtctx->nextJobID == 0) /* just started */
|
||||
if ( (!mtctx->params.rsyncable) /* rsyncable mode is disabled */
|
||||
&& (mtctx->nextJobID == 0) /* just started */
|
||||
&& (mtctx->inBuff.filled == 0) /* nothing buffered */
|
||||
&& (!mtctx->jobReady) /* no job already created */
|
||||
&& (endOp == ZSTD_e_end) /* end order */
|
||||
@ -1876,14 +2029,17 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
|
||||
DEBUGLOG(5, "ZSTDMT_tryGetInputRange completed successfully : mtctx->inBuff.buffer.start = %p", mtctx->inBuff.buffer.start);
|
||||
}
|
||||
if (mtctx->inBuff.buffer.start != NULL) {
|
||||
size_t const toLoad = MIN(input->size - input->pos, mtctx->targetSectionSize - mtctx->inBuff.filled);
|
||||
syncPoint_t const syncPoint = findSynchronizationPoint(mtctx, *input);
|
||||
if (syncPoint.flush && endOp == ZSTD_e_continue) {
|
||||
endOp = ZSTD_e_flush;
|
||||
}
|
||||
assert(mtctx->inBuff.buffer.capacity >= mtctx->targetSectionSize);
|
||||
DEBUGLOG(5, "ZSTDMT_compressStream_generic: adding %u bytes on top of %u to buffer of size %u",
|
||||
(U32)toLoad, (U32)mtctx->inBuff.filled, (U32)mtctx->targetSectionSize);
|
||||
memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, toLoad);
|
||||
input->pos += toLoad;
|
||||
mtctx->inBuff.filled += toLoad;
|
||||
forwardInputProgress = toLoad>0;
|
||||
(U32)syncPoint.toLoad, (U32)mtctx->inBuff.filled, (U32)mtctx->targetSectionSize);
|
||||
memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, syncPoint.toLoad);
|
||||
input->pos += syncPoint.toLoad;
|
||||
mtctx->inBuff.filled += syncPoint.toLoad;
|
||||
forwardInputProgress = syncPoint.toLoad>0;
|
||||
}
|
||||
if ((input->pos < input->size) && (endOp == ZSTD_e_end))
|
||||
endOp = ZSTD_e_flush; /* can't end now : not all input consumed */
|
||||
|
||||
@ -28,6 +28,16 @@
|
||||
#include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
|
||||
|
||||
|
||||
/* === Constants === */
|
||||
#ifndef ZSTDMT_NBWORKERS_MAX
|
||||
# define ZSTDMT_NBWORKERS_MAX 200
|
||||
#endif
|
||||
#ifndef ZSTDMT_JOBSIZE_MIN
|
||||
# define ZSTDMT_JOBSIZE_MIN (1 MB)
|
||||
#endif
|
||||
#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB))
|
||||
|
||||
|
||||
/* === Memory management === */
|
||||
typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
|
||||
ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers);
|
||||
@ -52,6 +62,7 @@ ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
|
||||
ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it will change in the future to mean "empty" */
|
||||
|
||||
ZSTDLIB_API size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx);
|
||||
ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
|
||||
|
||||
ZSTDLIB_API size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */
|
||||
@ -60,16 +71,12 @@ ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);
|
||||
|
||||
/* === Advanced functions and parameters === */
|
||||
|
||||
#ifndef ZSTDMT_JOBSIZE_MIN
|
||||
# define ZSTDMT_JOBSIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */
|
||||
#endif
|
||||
|
||||
ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const ZSTD_CDict* cdict,
|
||||
ZSTD_parameters params,
|
||||
unsigned overlapLog);
|
||||
int overlapLog);
|
||||
|
||||
ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
|
||||
const void* dict, size_t dictSize, /* dict can be released after init, a local copy is preserved within zcs */
|
||||
@ -84,8 +91,9 @@ ZSTDLIB_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
|
||||
/* ZSTDMT_parameter :
|
||||
* List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
|
||||
typedef enum {
|
||||
ZSTDMT_p_jobSize, /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */
|
||||
ZSTDMT_p_overlapSectionLog /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */
|
||||
ZSTDMT_p_jobSize, /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */
|
||||
ZSTDMT_p_overlapLog, /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */
|
||||
ZSTDMT_p_rsyncable /* Enables rsyncable mode. */
|
||||
} ZSTDMT_parameter;
|
||||
|
||||
/* ZSTDMT_setMTCtxParameter() :
|
||||
@ -93,12 +101,12 @@ typedef enum {
|
||||
* The function must be called typically after ZSTD_createCCtx() but __before ZSTDMT_init*() !__
|
||||
* Parameters not explicitly reset by ZSTDMT_init*() remain the same in consecutive compression sessions.
|
||||
* @return : 0, or an error code (which can be tested using ZSTD_isError()) */
|
||||
ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned value);
|
||||
ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int value);
|
||||
|
||||
/* ZSTDMT_getMTCtxParameter() :
|
||||
* Query the ZSTDMT_CCtx for a parameter value.
|
||||
* @return : 0, or an error code (which can be tested using ZSTD_isError()) */
|
||||
ZSTDLIB_API size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned* value);
|
||||
ZSTDLIB_API size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int* value);
|
||||
|
||||
|
||||
/*! ZSTDMT_compressStream_generic() :
|
||||
@ -129,7 +137,7 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx);
|
||||
|
||||
/*! ZSTDMT_CCtxParam_setMTCtxParameter()
|
||||
* like ZSTDMT_setMTCtxParameter(), but into a ZSTD_CCtx_Params */
|
||||
size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, unsigned value);
|
||||
size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, int value);
|
||||
|
||||
/*! ZSTDMT_CCtxParam_setNbWorkers()
|
||||
* Set nbWorkers, and clamp it.
|
||||
|
||||
@ -43,6 +43,19 @@
|
||||
#include "huf.h"
|
||||
#include "error_private.h"
|
||||
|
||||
/* **************************************************************
|
||||
* Macros
|
||||
****************************************************************/
|
||||
|
||||
/* These two optional macros force the use one way or another of the two
|
||||
* Huffman decompression implementations. You can't force in both directions
|
||||
* at the same time.
|
||||
*/
|
||||
#if defined(HUF_FORCE_DECOMPRESS_X1) && \
|
||||
defined(HUF_FORCE_DECOMPRESS_X2)
|
||||
#error "Cannot force the use of the X1 and X2 decoders at the same time!"
|
||||
#endif
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Error Management
|
||||
@ -58,6 +71,51 @@
|
||||
#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask))
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* BMI2 Variant Wrappers
|
||||
****************************************************************/
|
||||
#if DYNAMIC_BMI2
|
||||
|
||||
#define HUF_DGEN(fn) \
|
||||
\
|
||||
static size_t fn##_default( \
|
||||
void* dst, size_t dstSize, \
|
||||
const void* cSrc, size_t cSrcSize, \
|
||||
const HUF_DTable* DTable) \
|
||||
{ \
|
||||
return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
|
||||
} \
|
||||
\
|
||||
static TARGET_ATTRIBUTE("bmi2") size_t fn##_bmi2( \
|
||||
void* dst, size_t dstSize, \
|
||||
const void* cSrc, size_t cSrcSize, \
|
||||
const HUF_DTable* DTable) \
|
||||
{ \
|
||||
return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
|
||||
} \
|
||||
\
|
||||
static size_t fn(void* dst, size_t dstSize, void const* cSrc, \
|
||||
size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \
|
||||
{ \
|
||||
if (bmi2) { \
|
||||
return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \
|
||||
} \
|
||||
return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
#define HUF_DGEN(fn) \
|
||||
static size_t fn(void* dst, size_t dstSize, void const* cSrc, \
|
||||
size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \
|
||||
{ \
|
||||
(void)bmi2; \
|
||||
return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
/*-***************************/
|
||||
/* generic DTableDesc */
|
||||
/*-***************************/
|
||||
@ -71,6 +129,8 @@ static DTableDesc HUF_getDTableDesc(const HUF_DTable* table)
|
||||
}
|
||||
|
||||
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X2
|
||||
|
||||
/*-***************************/
|
||||
/* single-symbol decoding */
|
||||
/*-***************************/
|
||||
@ -307,46 +367,6 @@ typedef size_t (*HUF_decompress_usingDTable_t)(void *dst, size_t dstSize,
|
||||
const void *cSrc,
|
||||
size_t cSrcSize,
|
||||
const HUF_DTable *DTable);
|
||||
#if DYNAMIC_BMI2
|
||||
|
||||
#define HUF_DGEN(fn) \
|
||||
\
|
||||
static size_t fn##_default( \
|
||||
void* dst, size_t dstSize, \
|
||||
const void* cSrc, size_t cSrcSize, \
|
||||
const HUF_DTable* DTable) \
|
||||
{ \
|
||||
return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
|
||||
} \
|
||||
\
|
||||
static TARGET_ATTRIBUTE("bmi2") size_t fn##_bmi2( \
|
||||
void* dst, size_t dstSize, \
|
||||
const void* cSrc, size_t cSrcSize, \
|
||||
const HUF_DTable* DTable) \
|
||||
{ \
|
||||
return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
|
||||
} \
|
||||
\
|
||||
static size_t fn(void* dst, size_t dstSize, void const* cSrc, \
|
||||
size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \
|
||||
{ \
|
||||
if (bmi2) { \
|
||||
return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \
|
||||
} \
|
||||
return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
#define HUF_DGEN(fn) \
|
||||
static size_t fn(void* dst, size_t dstSize, void const* cSrc, \
|
||||
size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \
|
||||
{ \
|
||||
(void)bmi2; \
|
||||
return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
HUF_DGEN(HUF_decompress1X1_usingDTable_internal)
|
||||
HUF_DGEN(HUF_decompress4X1_usingDTable_internal)
|
||||
@ -437,6 +457,10 @@ size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cS
|
||||
return HUF_decompress4X1_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
|
||||
}
|
||||
|
||||
#endif /* HUF_FORCE_DECOMPRESS_X2 */
|
||||
|
||||
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X1
|
||||
|
||||
/* *************************/
|
||||
/* double-symbols decoding */
|
||||
@ -911,6 +935,8 @@ size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cS
|
||||
return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
|
||||
}
|
||||
|
||||
#endif /* HUF_FORCE_DECOMPRESS_X1 */
|
||||
|
||||
|
||||
/* ***********************************/
|
||||
/* Universal decompression selectors */
|
||||
@ -921,8 +947,18 @@ size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize,
|
||||
const HUF_DTable* DTable)
|
||||
{
|
||||
DTableDesc const dtd = HUF_getDTableDesc(DTable);
|
||||
#if defined(HUF_FORCE_DECOMPRESS_X1)
|
||||
(void)dtd;
|
||||
assert(dtd.tableType == 0);
|
||||
return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
|
||||
#elif defined(HUF_FORCE_DECOMPRESS_X2)
|
||||
(void)dtd;
|
||||
assert(dtd.tableType == 1);
|
||||
return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
|
||||
#else
|
||||
return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) :
|
||||
HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
|
||||
#endif
|
||||
}
|
||||
|
||||
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize,
|
||||
@ -930,11 +966,22 @@ size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize,
|
||||
const HUF_DTable* DTable)
|
||||
{
|
||||
DTableDesc const dtd = HUF_getDTableDesc(DTable);
|
||||
#if defined(HUF_FORCE_DECOMPRESS_X1)
|
||||
(void)dtd;
|
||||
assert(dtd.tableType == 0);
|
||||
return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
|
||||
#elif defined(HUF_FORCE_DECOMPRESS_X2)
|
||||
(void)dtd;
|
||||
assert(dtd.tableType == 1);
|
||||
return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
|
||||
#else
|
||||
return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) :
|
||||
HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2)
|
||||
typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t;
|
||||
static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] =
|
||||
{
|
||||
@ -956,6 +1003,7 @@ static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, qu
|
||||
{{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */
|
||||
{{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */
|
||||
};
|
||||
#endif
|
||||
|
||||
/** HUF_selectDecoder() :
|
||||
* Tells which decoder is likely to decode faster,
|
||||
@ -966,6 +1014,15 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
|
||||
{
|
||||
assert(dstSize > 0);
|
||||
assert(dstSize <= 128*1024);
|
||||
#if defined(HUF_FORCE_DECOMPRESS_X1)
|
||||
(void)dstSize;
|
||||
(void)cSrcSize;
|
||||
return 0;
|
||||
#elif defined(HUF_FORCE_DECOMPRESS_X2)
|
||||
(void)dstSize;
|
||||
(void)cSrcSize;
|
||||
return 1;
|
||||
#else
|
||||
/* decoder timing evaluation */
|
||||
{ U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */
|
||||
U32 const D256 = (U32)(dstSize >> 8);
|
||||
@ -973,14 +1030,18 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
|
||||
U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
|
||||
DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, to reduce cache eviction */
|
||||
return DTime1 < DTime0;
|
||||
} }
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
|
||||
|
||||
size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2)
|
||||
static const decompressionAlgo decompress[2] = { HUF_decompress4X1, HUF_decompress4X2 };
|
||||
#endif
|
||||
|
||||
/* validation checks */
|
||||
if (dstSize == 0) return ERROR(dstSize_tooSmall);
|
||||
@ -989,7 +1050,17 @@ size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcS
|
||||
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
|
||||
|
||||
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
|
||||
#if defined(HUF_FORCE_DECOMPRESS_X1)
|
||||
(void)algoNb;
|
||||
assert(algoNb == 0);
|
||||
return HUF_decompress4X1(dst, dstSize, cSrc, cSrcSize);
|
||||
#elif defined(HUF_FORCE_DECOMPRESS_X2)
|
||||
(void)algoNb;
|
||||
assert(algoNb == 1);
|
||||
return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize);
|
||||
#else
|
||||
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
@ -1002,8 +1073,18 @@ size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const
|
||||
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
|
||||
|
||||
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
|
||||
#if defined(HUF_FORCE_DECOMPRESS_X1)
|
||||
(void)algoNb;
|
||||
assert(algoNb == 0);
|
||||
return HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize);
|
||||
#elif defined(HUF_FORCE_DECOMPRESS_X2)
|
||||
(void)algoNb;
|
||||
assert(algoNb == 1);
|
||||
return HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize);
|
||||
#else
|
||||
return algoNb ? HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
|
||||
HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
@ -1025,8 +1106,19 @@ size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst,
|
||||
if (cSrcSize == 0) return ERROR(corruption_detected);
|
||||
|
||||
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
|
||||
return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize):
|
||||
#if defined(HUF_FORCE_DECOMPRESS_X1)
|
||||
(void)algoNb;
|
||||
assert(algoNb == 0);
|
||||
return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
|
||||
#elif defined(HUF_FORCE_DECOMPRESS_X2)
|
||||
(void)algoNb;
|
||||
assert(algoNb == 1);
|
||||
return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
|
||||
#else
|
||||
return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
|
||||
cSrcSize, workSpace, wkspSize):
|
||||
HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
@ -1041,10 +1133,22 @@ size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
|
||||
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
|
||||
|
||||
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
|
||||
#if defined(HUF_FORCE_DECOMPRESS_X1)
|
||||
(void)algoNb;
|
||||
assert(algoNb == 0);
|
||||
return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
|
||||
cSrcSize, workSpace, wkspSize);
|
||||
#elif defined(HUF_FORCE_DECOMPRESS_X2)
|
||||
(void)algoNb;
|
||||
assert(algoNb == 1);
|
||||
return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
|
||||
cSrcSize, workSpace, wkspSize);
|
||||
#else
|
||||
return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
|
||||
cSrcSize, workSpace, wkspSize):
|
||||
HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
|
||||
cSrcSize, workSpace, wkspSize);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
@ -1060,10 +1164,21 @@ size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize,
|
||||
size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2)
|
||||
{
|
||||
DTableDesc const dtd = HUF_getDTableDesc(DTable);
|
||||
#if defined(HUF_FORCE_DECOMPRESS_X1)
|
||||
(void)dtd;
|
||||
assert(dtd.tableType == 0);
|
||||
return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
|
||||
#elif defined(HUF_FORCE_DECOMPRESS_X2)
|
||||
(void)dtd;
|
||||
assert(dtd.tableType == 1);
|
||||
return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
|
||||
#else
|
||||
return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) :
|
||||
HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifndef HUF_FORCE_DECOMPRESS_X2
|
||||
size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*) cSrc;
|
||||
@ -1075,12 +1190,23 @@ size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstS
|
||||
|
||||
return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2);
|
||||
}
|
||||
#endif
|
||||
|
||||
size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2)
|
||||
{
|
||||
DTableDesc const dtd = HUF_getDTableDesc(DTable);
|
||||
#if defined(HUF_FORCE_DECOMPRESS_X1)
|
||||
(void)dtd;
|
||||
assert(dtd.tableType == 0);
|
||||
return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
|
||||
#elif defined(HUF_FORCE_DECOMPRESS_X2)
|
||||
(void)dtd;
|
||||
assert(dtd.tableType == 1);
|
||||
return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
|
||||
#else
|
||||
return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) :
|
||||
HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
|
||||
#endif
|
||||
}
|
||||
|
||||
size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2)
|
||||
@ -1090,7 +1216,17 @@ size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t ds
|
||||
if (cSrcSize == 0) return ERROR(corruption_detected);
|
||||
|
||||
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
|
||||
#if defined(HUF_FORCE_DECOMPRESS_X1)
|
||||
(void)algoNb;
|
||||
assert(algoNb == 0);
|
||||
return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
|
||||
#elif defined(HUF_FORCE_DECOMPRESS_X2)
|
||||
(void)algoNb;
|
||||
assert(algoNb == 1);
|
||||
return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
|
||||
#else
|
||||
return algoNb ? HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2) :
|
||||
HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
240
lib/decompress/zstd_ddict.c
Normal file
240
lib/decompress/zstd_ddict.c
Normal file
@ -0,0 +1,240 @@
|
||||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
/* zstd_ddict.c :
|
||||
* concentrates all logic that needs to know the internals of ZSTD_DDict object */
|
||||
|
||||
/*-*******************************************************
|
||||
* Dependencies
|
||||
*********************************************************/
|
||||
#include <string.h> /* memcpy, memmove, memset */
|
||||
#include "cpu.h" /* bmi2 */
|
||||
#include "mem.h" /* low level memory routines */
|
||||
#define FSE_STATIC_LINKING_ONLY
|
||||
#include "fse.h"
|
||||
#define HUF_STATIC_LINKING_ONLY
|
||||
#include "huf.h"
|
||||
#include "zstd_decompress_internal.h"
|
||||
#include "zstd_ddict.h"
|
||||
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
|
||||
# include "zstd_legacy.h"
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Types
|
||||
*********************************************************/
|
||||
struct ZSTD_DDict_s {
|
||||
void* dictBuffer;
|
||||
const void* dictContent;
|
||||
size_t dictSize;
|
||||
ZSTD_entropyDTables_t entropy;
|
||||
U32 dictID;
|
||||
U32 entropyPresent;
|
||||
ZSTD_customMem cMem;
|
||||
}; /* typedef'd to ZSTD_DDict within "zstd.h" */
|
||||
|
||||
const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict)
|
||||
{
|
||||
assert(ddict != NULL);
|
||||
return ddict->dictContent;
|
||||
}
|
||||
|
||||
size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict)
|
||||
{
|
||||
assert(ddict != NULL);
|
||||
return ddict->dictSize;
|
||||
}
|
||||
|
||||
void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
|
||||
{
|
||||
DEBUGLOG(4, "ZSTD_copyDDictParameters");
|
||||
assert(dctx != NULL);
|
||||
assert(ddict != NULL);
|
||||
dctx->dictID = ddict->dictID;
|
||||
dctx->prefixStart = ddict->dictContent;
|
||||
dctx->virtualStart = ddict->dictContent;
|
||||
dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize;
|
||||
dctx->previousDstEnd = dctx->dictEnd;
|
||||
if (ddict->entropyPresent) {
|
||||
dctx->litEntropy = 1;
|
||||
dctx->fseEntropy = 1;
|
||||
dctx->LLTptr = ddict->entropy.LLTable;
|
||||
dctx->MLTptr = ddict->entropy.MLTable;
|
||||
dctx->OFTptr = ddict->entropy.OFTable;
|
||||
dctx->HUFptr = ddict->entropy.hufTable;
|
||||
dctx->entropy.rep[0] = ddict->entropy.rep[0];
|
||||
dctx->entropy.rep[1] = ddict->entropy.rep[1];
|
||||
dctx->entropy.rep[2] = ddict->entropy.rep[2];
|
||||
} else {
|
||||
dctx->litEntropy = 0;
|
||||
dctx->fseEntropy = 0;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static size_t
|
||||
ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict,
|
||||
ZSTD_dictContentType_e dictContentType)
|
||||
{
|
||||
ddict->dictID = 0;
|
||||
ddict->entropyPresent = 0;
|
||||
if (dictContentType == ZSTD_dct_rawContent) return 0;
|
||||
|
||||
if (ddict->dictSize < 8) {
|
||||
if (dictContentType == ZSTD_dct_fullDict)
|
||||
return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
|
||||
return 0; /* pure content mode */
|
||||
}
|
||||
{ U32 const magic = MEM_readLE32(ddict->dictContent);
|
||||
if (magic != ZSTD_MAGIC_DICTIONARY) {
|
||||
if (dictContentType == ZSTD_dct_fullDict)
|
||||
return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
|
||||
return 0; /* pure content mode */
|
||||
}
|
||||
}
|
||||
ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE);
|
||||
|
||||
/* load entropy tables */
|
||||
CHECK_E( ZSTD_loadDEntropy(&ddict->entropy,
|
||||
ddict->dictContent, ddict->dictSize),
|
||||
dictionary_corrupted );
|
||||
ddict->entropyPresent = 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_dictLoadMethod_e dictLoadMethod,
|
||||
ZSTD_dictContentType_e dictContentType)
|
||||
{
|
||||
if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
|
||||
ddict->dictBuffer = NULL;
|
||||
ddict->dictContent = dict;
|
||||
if (!dict) dictSize = 0;
|
||||
} else {
|
||||
void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem);
|
||||
ddict->dictBuffer = internalBuffer;
|
||||
ddict->dictContent = internalBuffer;
|
||||
if (!internalBuffer) return ERROR(memory_allocation);
|
||||
memcpy(internalBuffer, dict, dictSize);
|
||||
}
|
||||
ddict->dictSize = dictSize;
|
||||
ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
|
||||
|
||||
/* parse dictionary content */
|
||||
CHECK_F( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) );
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
|
||||
ZSTD_dictLoadMethod_e dictLoadMethod,
|
||||
ZSTD_dictContentType_e dictContentType,
|
||||
ZSTD_customMem customMem)
|
||||
{
|
||||
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
|
||||
|
||||
{ ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
|
||||
if (ddict == NULL) return NULL;
|
||||
ddict->cMem = customMem;
|
||||
{ size_t const initResult = ZSTD_initDDict_internal(ddict,
|
||||
dict, dictSize,
|
||||
dictLoadMethod, dictContentType);
|
||||
if (ZSTD_isError(initResult)) {
|
||||
ZSTD_freeDDict(ddict);
|
||||
return NULL;
|
||||
} }
|
||||
return ddict;
|
||||
}
|
||||
}
|
||||
|
||||
/*! ZSTD_createDDict() :
|
||||
* Create a digested dictionary, to start decompression without startup delay.
|
||||
* `dict` content is copied inside DDict.
|
||||
* Consequently, `dict` can be released after `ZSTD_DDict` creation */
|
||||
ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
|
||||
{
|
||||
ZSTD_customMem const allocator = { NULL, NULL, NULL };
|
||||
return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator);
|
||||
}
|
||||
|
||||
/*! ZSTD_createDDict_byReference() :
|
||||
* Create a digested dictionary, to start decompression without startup delay.
|
||||
* Dictionary content is simply referenced, it will be accessed during decompression.
|
||||
* Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */
|
||||
ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
|
||||
{
|
||||
ZSTD_customMem const allocator = { NULL, NULL, NULL };
|
||||
return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator);
|
||||
}
|
||||
|
||||
|
||||
const ZSTD_DDict* ZSTD_initStaticDDict(
|
||||
void* sBuffer, size_t sBufferSize,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_dictLoadMethod_e dictLoadMethod,
|
||||
ZSTD_dictContentType_e dictContentType)
|
||||
{
|
||||
size_t const neededSpace = sizeof(ZSTD_DDict)
|
||||
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
|
||||
ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer;
|
||||
assert(sBuffer != NULL);
|
||||
assert(dict != NULL);
|
||||
if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */
|
||||
if (sBufferSize < neededSpace) return NULL;
|
||||
if (dictLoadMethod == ZSTD_dlm_byCopy) {
|
||||
memcpy(ddict+1, dict, dictSize); /* local copy */
|
||||
dict = ddict+1;
|
||||
}
|
||||
if (ZSTD_isError( ZSTD_initDDict_internal(ddict,
|
||||
dict, dictSize,
|
||||
ZSTD_dlm_byRef, dictContentType) ))
|
||||
return NULL;
|
||||
return ddict;
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
|
||||
{
|
||||
if (ddict==NULL) return 0; /* support free on NULL */
|
||||
{ ZSTD_customMem const cMem = ddict->cMem;
|
||||
ZSTD_free(ddict->dictBuffer, cMem);
|
||||
ZSTD_free(ddict, cMem);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/*! ZSTD_estimateDDictSize() :
|
||||
* Estimate amount of memory that will be needed to create a dictionary for decompression.
|
||||
* Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */
|
||||
size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod)
|
||||
{
|
||||
return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
|
||||
}
|
||||
|
||||
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
|
||||
{
|
||||
if (ddict==NULL) return 0; /* support sizeof on NULL */
|
||||
return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ;
|
||||
}
|
||||
|
||||
/*! ZSTD_getDictID_fromDDict() :
|
||||
* Provides the dictID of the dictionary loaded into `ddict`.
|
||||
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
|
||||
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
|
||||
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
|
||||
{
|
||||
if (ddict==NULL) return 0;
|
||||
return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
|
||||
}
|
||||
44
lib/decompress/zstd_ddict.h
Normal file
44
lib/decompress/zstd_ddict.h
Normal file
@ -0,0 +1,44 @@
|
||||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
|
||||
#ifndef ZSTD_DDICT_H
|
||||
#define ZSTD_DDICT_H
|
||||
|
||||
/*-*******************************************************
|
||||
* Dependencies
|
||||
*********************************************************/
|
||||
#include <stddef.h> /* size_t */
|
||||
#include "zstd.h" /* ZSTD_DDict, and several public functions */
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Interface
|
||||
*********************************************************/
|
||||
|
||||
/* note: several prototypes are already published in `zstd.h` :
|
||||
* ZSTD_createDDict()
|
||||
* ZSTD_createDDict_byReference()
|
||||
* ZSTD_createDDict_advanced()
|
||||
* ZSTD_freeDDict()
|
||||
* ZSTD_initStaticDDict()
|
||||
* ZSTD_sizeof_DDict()
|
||||
* ZSTD_estimateDDictSize()
|
||||
* ZSTD_getDictID_fromDict()
|
||||
*/
|
||||
|
||||
const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict);
|
||||
size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict);
|
||||
|
||||
void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
|
||||
|
||||
|
||||
|
||||
#endif /* ZSTD_DDICT_H */
|
||||
File diff suppressed because it is too large
Load Diff
1307
lib/decompress/zstd_decompress_block.c
Normal file
1307
lib/decompress/zstd_decompress_block.c
Normal file
File diff suppressed because it is too large
Load Diff
59
lib/decompress/zstd_decompress_block.h
Normal file
59
lib/decompress/zstd_decompress_block.h
Normal file
@ -0,0 +1,59 @@
|
||||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
|
||||
#ifndef ZSTD_DEC_BLOCK_H
|
||||
#define ZSTD_DEC_BLOCK_H
|
||||
|
||||
/*-*******************************************************
|
||||
* Dependencies
|
||||
*********************************************************/
|
||||
#include <stddef.h> /* size_t */
|
||||
#include "zstd.h" /* DCtx, and some public functions */
|
||||
#include "zstd_internal.h" /* blockProperties_t, and some public functions */
|
||||
#include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */
|
||||
|
||||
|
||||
/* === Prototypes === */
|
||||
|
||||
/* note: prototypes already published within `zstd.h` :
|
||||
* ZSTD_decompressBlock()
|
||||
*/
|
||||
|
||||
/* note: prototypes already published within `zstd_internal.h` :
|
||||
* ZSTD_getcBlockSize()
|
||||
* ZSTD_decodeSeqHeaders()
|
||||
*/
|
||||
|
||||
|
||||
/* ZSTD_decompressBlock_internal() :
|
||||
* decompress block, starting at `src`,
|
||||
* into destination buffer `dst`.
|
||||
* @return : decompressed block size,
|
||||
* or an error code (which can be tested using ZSTD_isError())
|
||||
*/
|
||||
size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize, const int frame);
|
||||
|
||||
/* ZSTD_buildFSETable() :
|
||||
* generate FSE decoding table for one symbol (ll, ml or off)
|
||||
* this function must be called with valid parameters only
|
||||
* (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.)
|
||||
* in which case it cannot fail.
|
||||
* Internal use only.
|
||||
*/
|
||||
void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
|
||||
const short* normalizedCounter, unsigned maxSymbolValue,
|
||||
const U32* baseValue, const U32* nbAdditionalBits,
|
||||
unsigned tableLog);
|
||||
|
||||
|
||||
#endif /* ZSTD_DEC_BLOCK_H */
|
||||
168
lib/decompress/zstd_decompress_internal.h
Normal file
168
lib/decompress/zstd_decompress_internal.h
Normal file
@ -0,0 +1,168 @@
|
||||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
|
||||
/* zstd_decompress_internal:
|
||||
* objects and definitions shared within lib/decompress modules */
|
||||
|
||||
#ifndef ZSTD_DECOMPRESS_INTERNAL_H
|
||||
#define ZSTD_DECOMPRESS_INTERNAL_H
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Dependencies
|
||||
*********************************************************/
|
||||
#include "mem.h" /* BYTE, U16, U32 */
|
||||
#include "zstd_internal.h" /* ZSTD_seqSymbol */
|
||||
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Constants
|
||||
*********************************************************/
|
||||
static const U32 LL_base[MaxLL+1] = {
|
||||
0, 1, 2, 3, 4, 5, 6, 7,
|
||||
8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 18, 20, 22, 24, 28, 32, 40,
|
||||
48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000,
|
||||
0x2000, 0x4000, 0x8000, 0x10000 };
|
||||
|
||||
static const U32 OF_base[MaxOff+1] = {
|
||||
0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D,
|
||||
0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD,
|
||||
0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
|
||||
0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD };
|
||||
|
||||
static const U32 OF_bits[MaxOff+1] = {
|
||||
0, 1, 2, 3, 4, 5, 6, 7,
|
||||
8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 17, 18, 19, 20, 21, 22, 23,
|
||||
24, 25, 26, 27, 28, 29, 30, 31 };
|
||||
|
||||
static const U32 ML_base[MaxML+1] = {
|
||||
3, 4, 5, 6, 7, 8, 9, 10,
|
||||
11, 12, 13, 14, 15, 16, 17, 18,
|
||||
19, 20, 21, 22, 23, 24, 25, 26,
|
||||
27, 28, 29, 30, 31, 32, 33, 34,
|
||||
35, 37, 39, 41, 43, 47, 51, 59,
|
||||
67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803,
|
||||
0x1003, 0x2003, 0x4003, 0x8003, 0x10003 };
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Decompression types
|
||||
*********************************************************/
|
||||
typedef struct {
|
||||
U32 fastMode;
|
||||
U32 tableLog;
|
||||
} ZSTD_seqSymbol_header;
|
||||
|
||||
typedef struct {
|
||||
U16 nextState;
|
||||
BYTE nbAdditionalBits;
|
||||
BYTE nbBits;
|
||||
U32 baseValue;
|
||||
} ZSTD_seqSymbol;
|
||||
|
||||
#define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log)))
|
||||
|
||||
typedef struct {
|
||||
ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */
|
||||
ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */
|
||||
ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */
|
||||
HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
|
||||
U32 rep[ZSTD_REP_NUM];
|
||||
} ZSTD_entropyDTables_t;
|
||||
|
||||
typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
|
||||
ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock,
|
||||
ZSTDds_decompressLastBlock, ZSTDds_checkChecksum,
|
||||
ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
|
||||
|
||||
typedef enum { zdss_init=0, zdss_loadHeader,
|
||||
zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
|
||||
|
||||
struct ZSTD_DCtx_s
|
||||
{
|
||||
const ZSTD_seqSymbol* LLTptr;
|
||||
const ZSTD_seqSymbol* MLTptr;
|
||||
const ZSTD_seqSymbol* OFTptr;
|
||||
const HUF_DTable* HUFptr;
|
||||
ZSTD_entropyDTables_t entropy;
|
||||
U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */
|
||||
const void* previousDstEnd; /* detect continuity */
|
||||
const void* prefixStart; /* start of current segment */
|
||||
const void* virtualStart; /* virtual start of previous segment if it was just before current one */
|
||||
const void* dictEnd; /* end of previous segment */
|
||||
size_t expected;
|
||||
ZSTD_frameHeader fParams;
|
||||
U64 decodedSize;
|
||||
blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */
|
||||
ZSTD_dStage stage;
|
||||
U32 litEntropy;
|
||||
U32 fseEntropy;
|
||||
XXH64_state_t xxhState;
|
||||
size_t headerSize;
|
||||
ZSTD_format_e format;
|
||||
const BYTE* litPtr;
|
||||
ZSTD_customMem customMem;
|
||||
size_t litSize;
|
||||
size_t rleSize;
|
||||
size_t staticSize;
|
||||
int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
|
||||
|
||||
/* dictionary */
|
||||
ZSTD_DDict* ddictLocal;
|
||||
const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */
|
||||
U32 dictID;
|
||||
int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */
|
||||
|
||||
/* streaming */
|
||||
ZSTD_dStreamStage streamStage;
|
||||
char* inBuff;
|
||||
size_t inBuffSize;
|
||||
size_t inPos;
|
||||
size_t maxWindowSize;
|
||||
char* outBuff;
|
||||
size_t outBuffSize;
|
||||
size_t outStart;
|
||||
size_t outEnd;
|
||||
size_t lhSize;
|
||||
void* legacyContext;
|
||||
U32 previousLegacyVersion;
|
||||
U32 legacyVersion;
|
||||
U32 hostageByte;
|
||||
int noForwardProgress;
|
||||
|
||||
/* workspace */
|
||||
BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
|
||||
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
|
||||
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Shared internal functions
|
||||
*********************************************************/
|
||||
|
||||
/*! ZSTD_loadDEntropy() :
|
||||
* dict : must point at beginning of a valid zstd dictionary.
|
||||
* @return : size of entropy tables read */
|
||||
size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
|
||||
const void* const dict, size_t const dictSize);
|
||||
|
||||
/*! ZSTD_checkContinuity() :
|
||||
* check if next `dst` follows previous position, where decompression ended.
|
||||
* If yes, do nothing (continue on current segment).
|
||||
* If not, classify previous segment as "external dictionary", and start a new segment.
|
||||
* This function cannot fail. */
|
||||
void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst);
|
||||
|
||||
|
||||
#endif /* ZSTD_DECOMPRESS_INTERNAL_H */
|
||||
@ -39,7 +39,7 @@
|
||||
/*-*************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
#define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB))
|
||||
#define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))
|
||||
#define DEFAULT_SPLITPOINT 1.0
|
||||
|
||||
/*-*************************************
|
||||
@ -543,7 +543,7 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
|
||||
if (totalSamplesSize < MAX(d, sizeof(U64)) ||
|
||||
totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) {
|
||||
DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n",
|
||||
(U32)(totalSamplesSize>>20), (COVER_MAX_SAMPLES_SIZE >> 20));
|
||||
(unsigned)(totalSamplesSize>>20), (COVER_MAX_SAMPLES_SIZE >> 20));
|
||||
return 0;
|
||||
}
|
||||
/* Check if there are at least 5 training samples */
|
||||
@ -559,9 +559,9 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
|
||||
/* Zero the context */
|
||||
memset(ctx, 0, sizeof(*ctx));
|
||||
DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples,
|
||||
(U32)trainingSamplesSize);
|
||||
(unsigned)trainingSamplesSize);
|
||||
DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples,
|
||||
(U32)testSamplesSize);
|
||||
(unsigned)testSamplesSize);
|
||||
ctx->samples = samples;
|
||||
ctx->samplesSizes = samplesSizes;
|
||||
ctx->nbSamples = nbSamples;
|
||||
@ -639,11 +639,11 @@ static size_t COVER_buildDictionary(const COVER_ctx_t *ctx, U32 *freqs,
|
||||
/* Divide the data up into epochs of equal size.
|
||||
* We will select at least one segment from each epoch.
|
||||
*/
|
||||
const U32 epochs = MAX(1, (U32)(dictBufferCapacity / parameters.k / 4));
|
||||
const U32 epochSize = (U32)(ctx->suffixSize / epochs);
|
||||
const unsigned epochs = MAX(1, (U32)(dictBufferCapacity / parameters.k / 4));
|
||||
const unsigned epochSize = (U32)(ctx->suffixSize / epochs);
|
||||
size_t epoch;
|
||||
DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", epochs,
|
||||
epochSize);
|
||||
DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n",
|
||||
epochs, epochSize);
|
||||
/* Loop through the epochs until there are no more segments or the dictionary
|
||||
* is full.
|
||||
*/
|
||||
@ -670,7 +670,7 @@ static size_t COVER_buildDictionary(const COVER_ctx_t *ctx, U32 *freqs,
|
||||
memcpy(dict + tail, ctx->samples + segment.begin, segmentSize);
|
||||
DISPLAYUPDATE(
|
||||
2, "\r%u%% ",
|
||||
(U32)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
|
||||
(unsigned)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
|
||||
}
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
return tail;
|
||||
@ -722,7 +722,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
|
||||
samplesBuffer, samplesSizes, nbSamples, parameters.zParams);
|
||||
if (!ZSTD_isError(dictionarySize)) {
|
||||
DISPLAYLEVEL(2, "Constructed dictionary of size %u\n",
|
||||
(U32)dictionarySize);
|
||||
(unsigned)dictionarySize);
|
||||
}
|
||||
COVER_ctx_destroy(&ctx);
|
||||
COVER_map_destroy(&activeDmers);
|
||||
@ -868,6 +868,8 @@ void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
|
||||
if (!best->dict) {
|
||||
best->compressedSize = ERROR(GENERIC);
|
||||
best->dictSize = 0;
|
||||
ZSTD_pthread_cond_signal(&best->cond);
|
||||
ZSTD_pthread_mutex_unlock(&best->mutex);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@ -1054,7 +1056,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
|
||||
}
|
||||
/* Print status */
|
||||
LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ",
|
||||
(U32)((iteration * 100) / kIterations));
|
||||
(unsigned)((iteration * 100) / kIterations));
|
||||
++iteration;
|
||||
}
|
||||
COVER_best_wait(&best);
|
||||
|
||||
@ -20,7 +20,7 @@
|
||||
/*-*************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
#define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB))
|
||||
#define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))
|
||||
#define FASTCOVER_MAX_F 31
|
||||
#define FASTCOVER_MAX_ACCEL 10
|
||||
#define DEFAULT_SPLITPOINT 0.75
|
||||
@ -159,15 +159,15 @@ static COVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
|
||||
*/
|
||||
while (activeSegment.end < end) {
|
||||
/* Get hash value of current dmer */
|
||||
const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, f, d);
|
||||
const size_t idx = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, f, d);
|
||||
|
||||
/* Add frequency of this index to score if this is the first occurence of index in active segment */
|
||||
if (segmentFreqs[index] == 0) {
|
||||
activeSegment.score += freqs[index];
|
||||
if (segmentFreqs[idx] == 0) {
|
||||
activeSegment.score += freqs[idx];
|
||||
}
|
||||
/* Increment end of segment and segmentFreqs*/
|
||||
activeSegment.end += 1;
|
||||
segmentFreqs[index] += 1;
|
||||
segmentFreqs[idx] += 1;
|
||||
/* If the window is now too large, drop the first position */
|
||||
if (activeSegment.end - activeSegment.begin == dmersInK + 1) {
|
||||
/* Get hash value of the dmer to be eliminated from active segment */
|
||||
@ -309,7 +309,7 @@ FASTCOVER_ctx_init(FASTCOVER_ctx_t* ctx,
|
||||
if (totalSamplesSize < MAX(d, sizeof(U64)) ||
|
||||
totalSamplesSize >= (size_t)FASTCOVER_MAX_SAMPLES_SIZE) {
|
||||
DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n",
|
||||
(U32)(totalSamplesSize >> 20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20));
|
||||
(unsigned)(totalSamplesSize >> 20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20));
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -328,9 +328,9 @@ FASTCOVER_ctx_init(FASTCOVER_ctx_t* ctx,
|
||||
/* Zero the context */
|
||||
memset(ctx, 0, sizeof(*ctx));
|
||||
DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples,
|
||||
(U32)trainingSamplesSize);
|
||||
(unsigned)trainingSamplesSize);
|
||||
DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples,
|
||||
(U32)testSamplesSize);
|
||||
(unsigned)testSamplesSize);
|
||||
|
||||
ctx->samples = samples;
|
||||
ctx->samplesSizes = samplesSizes;
|
||||
@ -389,11 +389,11 @@ FASTCOVER_buildDictionary(const FASTCOVER_ctx_t* ctx,
|
||||
/* Divide the data up into epochs of equal size.
|
||||
* We will select at least one segment from each epoch.
|
||||
*/
|
||||
const U32 epochs = MAX(1, (U32)(dictBufferCapacity / parameters.k));
|
||||
const U32 epochSize = (U32)(ctx->nbDmers / epochs);
|
||||
const unsigned epochs = MAX(1, (U32)(dictBufferCapacity / parameters.k));
|
||||
const unsigned epochSize = (U32)(ctx->nbDmers / epochs);
|
||||
size_t epoch;
|
||||
DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", epochs,
|
||||
epochSize);
|
||||
DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n",
|
||||
epochs, epochSize);
|
||||
/* Loop through the epochs until there are no more segments or the dictionary
|
||||
* is full.
|
||||
*/
|
||||
@ -423,7 +423,7 @@ FASTCOVER_buildDictionary(const FASTCOVER_ctx_t* ctx,
|
||||
memcpy(dict + tail, ctx->samples + segment.begin, segmentSize);
|
||||
DISPLAYUPDATE(
|
||||
2, "\r%u%% ",
|
||||
(U32)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
|
||||
(unsigned)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
|
||||
}
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
return tail;
|
||||
@ -577,7 +577,7 @@ ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity,
|
||||
samplesBuffer, samplesSizes, nbFinalizeSamples, coverParams.zParams);
|
||||
if (!ZSTD_isError(dictionarySize)) {
|
||||
DISPLAYLEVEL(2, "Constructed dictionary of size %u\n",
|
||||
(U32)dictionarySize);
|
||||
(unsigned)dictionarySize);
|
||||
}
|
||||
FASTCOVER_ctx_destroy(&ctx);
|
||||
free(segmentFreqs);
|
||||
@ -702,7 +702,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
|
||||
}
|
||||
/* Print status */
|
||||
LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ",
|
||||
(U32)((iteration * 100) / kIterations));
|
||||
(unsigned)((iteration * 100) / kIterations));
|
||||
++iteration;
|
||||
}
|
||||
COVER_best_wait(&best);
|
||||
|
||||
@ -255,15 +255,15 @@ static dictItem ZDICT_analyzePos(
|
||||
}
|
||||
|
||||
{ int i;
|
||||
U32 searchLength;
|
||||
U32 mml;
|
||||
U32 refinedStart = start;
|
||||
U32 refinedEnd = end;
|
||||
|
||||
DISPLAYLEVEL(4, "\n");
|
||||
DISPLAYLEVEL(4, "found %3u matches of length >= %i at pos %7u ", (U32)(end-start), MINMATCHLENGTH, (U32)pos);
|
||||
DISPLAYLEVEL(4, "found %3u matches of length >= %i at pos %7u ", (unsigned)(end-start), MINMATCHLENGTH, (unsigned)pos);
|
||||
DISPLAYLEVEL(4, "\n");
|
||||
|
||||
for (searchLength = MINMATCHLENGTH ; ; searchLength++) {
|
||||
for (mml = MINMATCHLENGTH ; ; mml++) {
|
||||
BYTE currentChar = 0;
|
||||
U32 currentCount = 0;
|
||||
U32 currentID = refinedStart;
|
||||
@ -271,13 +271,13 @@ static dictItem ZDICT_analyzePos(
|
||||
U32 selectedCount = 0;
|
||||
U32 selectedID = currentID;
|
||||
for (id =refinedStart; id < refinedEnd; id++) {
|
||||
if (b[suffix[id] + searchLength] != currentChar) {
|
||||
if (b[suffix[id] + mml] != currentChar) {
|
||||
if (currentCount > selectedCount) {
|
||||
selectedCount = currentCount;
|
||||
selectedID = currentID;
|
||||
}
|
||||
currentID = id;
|
||||
currentChar = b[ suffix[id] + searchLength];
|
||||
currentChar = b[ suffix[id] + mml];
|
||||
currentCount = 0;
|
||||
}
|
||||
currentCount ++;
|
||||
@ -342,7 +342,7 @@ static dictItem ZDICT_analyzePos(
|
||||
savings[i] = savings[i-1] + (lengthList[i] * (i-3));
|
||||
|
||||
DISPLAYLEVEL(4, "Selected dict at position %u, of length %u : saves %u (ratio: %.2f) \n",
|
||||
(U32)pos, (U32)maxLength, savings[maxLength], (double)savings[maxLength] / maxLength);
|
||||
(unsigned)pos, (unsigned)maxLength, (unsigned)savings[maxLength], (double)savings[maxLength] / maxLength);
|
||||
|
||||
solution.pos = (U32)pos;
|
||||
solution.length = (U32)maxLength;
|
||||
@ -497,7 +497,7 @@ static U32 ZDICT_dictSize(const dictItem* dictList)
|
||||
static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize,
|
||||
const void* const buffer, size_t bufferSize, /* buffer must end with noisy guard band */
|
||||
const size_t* fileSizes, unsigned nbFiles,
|
||||
U32 minRatio, U32 notificationLevel)
|
||||
unsigned minRatio, U32 notificationLevel)
|
||||
{
|
||||
int* const suffix0 = (int*)malloc((bufferSize+2)*sizeof(*suffix0));
|
||||
int* const suffix = suffix0+1;
|
||||
@ -523,11 +523,11 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize,
|
||||
memset(doneMarks, 0, bufferSize+16);
|
||||
|
||||
/* limit sample set size (divsufsort limitation)*/
|
||||
if (bufferSize > ZDICT_MAX_SAMPLES_SIZE) DISPLAYLEVEL(3, "sample set too large : reduced to %u MB ...\n", (U32)(ZDICT_MAX_SAMPLES_SIZE>>20));
|
||||
if (bufferSize > ZDICT_MAX_SAMPLES_SIZE) DISPLAYLEVEL(3, "sample set too large : reduced to %u MB ...\n", (unsigned)(ZDICT_MAX_SAMPLES_SIZE>>20));
|
||||
while (bufferSize > ZDICT_MAX_SAMPLES_SIZE) bufferSize -= fileSizes[--nbFiles];
|
||||
|
||||
/* sort */
|
||||
DISPLAYLEVEL(2, "sorting %u files of total size %u MB ...\n", nbFiles, (U32)(bufferSize>>20));
|
||||
DISPLAYLEVEL(2, "sorting %u files of total size %u MB ...\n", nbFiles, (unsigned)(bufferSize>>20));
|
||||
{ int const divSuftSortResult = divsufsort((const unsigned char*)buffer, suffix, (int)bufferSize, 0);
|
||||
if (divSuftSortResult != 0) { result = ERROR(GENERIC); goto _cleanup; }
|
||||
}
|
||||
@ -589,7 +589,7 @@ typedef struct
|
||||
#define MAXREPOFFSET 1024
|
||||
|
||||
static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
|
||||
U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets,
|
||||
unsigned* countLit, unsigned* offsetcodeCount, unsigned* matchlengthCount, unsigned* litlengthCount, U32* repOffsets,
|
||||
const void* src, size_t srcSize,
|
||||
U32 notificationLevel)
|
||||
{
|
||||
@ -602,7 +602,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
|
||||
|
||||
}
|
||||
cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
|
||||
if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
|
||||
if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (unsigned)srcSize); return; }
|
||||
|
||||
if (cSize) { /* if == 0; block is not compressible */
|
||||
const seqStore_t* const seqStorePtr = ZSTD_getSeqStore(esr.zc);
|
||||
@ -671,7 +671,7 @@ static void ZDICT_insertSortCount(offsetCount_t table[ZSTD_REP_NUM+1], U32 val,
|
||||
* rewrite `countLit` to contain a mostly flat but still compressible distribution of literals.
|
||||
* necessary to avoid generating a non-compressible distribution that HUF_writeCTable() cannot encode.
|
||||
*/
|
||||
static void ZDICT_flatLit(U32* countLit)
|
||||
static void ZDICT_flatLit(unsigned* countLit)
|
||||
{
|
||||
int u;
|
||||
for (u=1; u<256; u++) countLit[u] = 2;
|
||||
@ -687,14 +687,14 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
const void* dictBuffer, size_t dictBufferSize,
|
||||
unsigned notificationLevel)
|
||||
{
|
||||
U32 countLit[256];
|
||||
unsigned countLit[256];
|
||||
HUF_CREATE_STATIC_CTABLE(hufTable, 255);
|
||||
U32 offcodeCount[OFFCODE_MAX+1];
|
||||
unsigned offcodeCount[OFFCODE_MAX+1];
|
||||
short offcodeNCount[OFFCODE_MAX+1];
|
||||
U32 offcodeMax = ZSTD_highbit32((U32)(dictBufferSize + 128 KB));
|
||||
U32 matchLengthCount[MaxML+1];
|
||||
unsigned matchLengthCount[MaxML+1];
|
||||
short matchLengthNCount[MaxML+1];
|
||||
U32 litLengthCount[MaxLL+1];
|
||||
unsigned litLengthCount[MaxLL+1];
|
||||
short litLengthNCount[MaxLL+1];
|
||||
U32 repOffset[MAXREPOFFSET];
|
||||
offsetCount_t bestRepOffset[ZSTD_REP_NUM+1];
|
||||
@ -983,33 +983,33 @@ size_t ZDICT_trainFromBuffer_unsafe_legacy(
|
||||
|
||||
/* display best matches */
|
||||
if (params.zParams.notificationLevel>= 3) {
|
||||
U32 const nb = MIN(25, dictList[0].pos);
|
||||
U32 const dictContentSize = ZDICT_dictSize(dictList);
|
||||
U32 u;
|
||||
DISPLAYLEVEL(3, "\n %u segments found, of total size %u \n", dictList[0].pos-1, dictContentSize);
|
||||
unsigned const nb = MIN(25, dictList[0].pos);
|
||||
unsigned const dictContentSize = ZDICT_dictSize(dictList);
|
||||
unsigned u;
|
||||
DISPLAYLEVEL(3, "\n %u segments found, of total size %u \n", (unsigned)dictList[0].pos-1, dictContentSize);
|
||||
DISPLAYLEVEL(3, "list %u best segments \n", nb-1);
|
||||
for (u=1; u<nb; u++) {
|
||||
U32 const pos = dictList[u].pos;
|
||||
U32 const length = dictList[u].length;
|
||||
unsigned const pos = dictList[u].pos;
|
||||
unsigned const length = dictList[u].length;
|
||||
U32 const printedLength = MIN(40, length);
|
||||
if ((pos > samplesBuffSize) || ((pos + length) > samplesBuffSize)) {
|
||||
free(dictList);
|
||||
return ERROR(GENERIC); /* should never happen */
|
||||
}
|
||||
DISPLAYLEVEL(3, "%3u:%3u bytes at pos %8u, savings %7u bytes |",
|
||||
u, length, pos, dictList[u].savings);
|
||||
u, length, pos, (unsigned)dictList[u].savings);
|
||||
ZDICT_printHex((const char*)samplesBuffer+pos, printedLength);
|
||||
DISPLAYLEVEL(3, "| \n");
|
||||
} }
|
||||
|
||||
|
||||
/* create dictionary */
|
||||
{ U32 dictContentSize = ZDICT_dictSize(dictList);
|
||||
{ unsigned dictContentSize = ZDICT_dictSize(dictList);
|
||||
if (dictContentSize < ZDICT_CONTENTSIZE_MIN) { free(dictList); return ERROR(dictionaryCreation_failed); } /* dictionary content too small */
|
||||
if (dictContentSize < targetDictSize/4) {
|
||||
DISPLAYLEVEL(2, "! warning : selected content significantly smaller than requested (%u < %u) \n", dictContentSize, (U32)maxDictSize);
|
||||
DISPLAYLEVEL(2, "! warning : selected content significantly smaller than requested (%u < %u) \n", dictContentSize, (unsigned)maxDictSize);
|
||||
if (samplesBuffSize < 10 * targetDictSize)
|
||||
DISPLAYLEVEL(2, "! consider increasing the number of samples (total size : %u MB)\n", (U32)(samplesBuffSize>>20));
|
||||
DISPLAYLEVEL(2, "! consider increasing the number of samples (total size : %u MB)\n", (unsigned)(samplesBuffSize>>20));
|
||||
if (minRep > MINRATIO) {
|
||||
DISPLAYLEVEL(2, "! consider increasing selectivity to produce larger dictionary (-s%u) \n", selectivity+1);
|
||||
DISPLAYLEVEL(2, "! note : larger dictionaries are not necessarily better, test its efficiency on samples \n");
|
||||
@ -1017,9 +1017,9 @@ size_t ZDICT_trainFromBuffer_unsafe_legacy(
|
||||
}
|
||||
|
||||
if ((dictContentSize > targetDictSize*3) && (nbSamples > 2*MINRATIO) && (selectivity>1)) {
|
||||
U32 proposedSelectivity = selectivity-1;
|
||||
unsigned proposedSelectivity = selectivity-1;
|
||||
while ((nbSamples >> proposedSelectivity) <= MINRATIO) { proposedSelectivity--; }
|
||||
DISPLAYLEVEL(2, "! note : calculated dictionary significantly larger than requested (%u > %u) \n", dictContentSize, (U32)maxDictSize);
|
||||
DISPLAYLEVEL(2, "! note : calculated dictionary significantly larger than requested (%u > %u) \n", dictContentSize, (unsigned)maxDictSize);
|
||||
DISPLAYLEVEL(2, "! consider increasing dictionary size, or produce denser dictionary (-s%u) \n", proposedSelectivity);
|
||||
DISPLAYLEVEL(2, "! always test dictionary efficiency on real samples \n");
|
||||
}
|
||||
|
||||
@ -240,17 +240,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
|
||||
/* *************************************
|
||||
* Types
|
||||
***************************************/
|
||||
#define ZSTD_WINDOWLOG_MAX 26
|
||||
#define ZSTD_WINDOWLOG_MIN 18
|
||||
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 11
|
||||
#define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
|
||||
#define ZSTD_CONTENTLOG_MIN 4
|
||||
#define ZSTD_HASHLOG_MAX 28
|
||||
#define ZSTD_HASHLOG_MIN 4
|
||||
#define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1)
|
||||
#define ZSTD_SEARCHLOG_MIN 1
|
||||
#define ZSTD_SEARCHLENGTH_MAX 7
|
||||
#define ZSTD_SEARCHLENGTH_MIN 4
|
||||
|
||||
/** from faster to stronger */
|
||||
typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2 } ZSTD_strategy;
|
||||
|
||||
@ -836,7 +836,7 @@ MEM_STATIC void BITv05_skipBits(BITv05_DStream_t* bitD, U32 nbBits)
|
||||
bitD->bitsConsumed += nbBits;
|
||||
}
|
||||
|
||||
MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, U32 nbBits)
|
||||
MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, unsigned nbBits)
|
||||
{
|
||||
size_t value = BITv05_lookBits(bitD, nbBits);
|
||||
BITv05_skipBits(bitD, nbBits);
|
||||
@ -845,7 +845,7 @@ MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, U32 nbBits)
|
||||
|
||||
/*!BITv05_readBitsFast :
|
||||
* unsafe version; only works only if nbBits >= 1 */
|
||||
MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, U32 nbBits)
|
||||
MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, unsigned nbBits)
|
||||
{
|
||||
size_t value = BITv05_lookBitsFast(bitD, nbBits);
|
||||
BITv05_skipBits(bitD, nbBits);
|
||||
@ -1162,7 +1162,7 @@ MEM_STATIC unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr)
|
||||
/* **************************************************************
|
||||
* Complex types
|
||||
****************************************************************/
|
||||
typedef U32 DTable_max_t[FSEv05_DTABLE_SIZE_U32(FSEv05_MAX_TABLELOG)];
|
||||
typedef unsigned DTable_max_t[FSEv05_DTABLE_SIZE_U32(FSEv05_MAX_TABLELOG)];
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
@ -2191,7 +2191,7 @@ static void HUFv05_fillDTableX4(HUFv05_DEltX4* DTable, const U32 targetLog,
|
||||
}
|
||||
}
|
||||
|
||||
size_t HUFv05_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
|
||||
size_t HUFv05_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize)
|
||||
{
|
||||
BYTE weightList[HUFv05_MAX_SYMBOL_VALUE + 1];
|
||||
sortedSymbol_t sortedSymbol[HUFv05_MAX_SYMBOL_VALUE + 1];
|
||||
@ -2205,7 +2205,7 @@ size_t HUFv05_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
|
||||
void* dtPtr = DTable;
|
||||
HUFv05_DEltX4* const dt = ((HUFv05_DEltX4*)dtPtr) + 1;
|
||||
|
||||
HUFv05_STATIC_ASSERT(sizeof(HUFv05_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
|
||||
HUFv05_STATIC_ASSERT(sizeof(HUFv05_DEltX4) == sizeof(unsigned)); /* if compilation fails here, assertion is false */
|
||||
if (memLog > HUFv05_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||
//memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
|
||||
|
||||
@ -2332,7 +2332,7 @@ static inline size_t HUFv05_decodeStreamX4(BYTE* p, BITv05_DStream_t* bitDPtr, B
|
||||
size_t HUFv05_decompress1X4_usingDTable(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const U32* DTable)
|
||||
const unsigned* DTable)
|
||||
{
|
||||
const BYTE* const istart = (const BYTE*) cSrc;
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
@ -2375,7 +2375,7 @@ size_t HUFv05_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t
|
||||
size_t HUFv05_decompress4X4_usingDTable(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const U32* DTable)
|
||||
const unsigned* DTable)
|
||||
{
|
||||
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
|
||||
|
||||
@ -2999,7 +2999,7 @@ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t
|
||||
const BYTE* ip = istart;
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
U32 LLtype, Offtype, MLtype;
|
||||
U32 LLlog, Offlog, MLlog;
|
||||
unsigned LLlog, Offlog, MLlog;
|
||||
size_t dumpsLength;
|
||||
|
||||
/* check */
|
||||
@ -3057,7 +3057,7 @@ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t
|
||||
break;
|
||||
case FSEv05_ENCODING_DYNAMIC :
|
||||
default : /* impossible */
|
||||
{ U32 max = MaxLL;
|
||||
{ unsigned max = MaxLL;
|
||||
headerSize = FSEv05_readNCount(norm, &max, &LLlog, ip, iend-ip);
|
||||
if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
|
||||
if (LLlog > LLFSEv05Log) return ERROR(corruption_detected);
|
||||
@ -3081,7 +3081,7 @@ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t
|
||||
break;
|
||||
case FSEv05_ENCODING_DYNAMIC :
|
||||
default : /* impossible */
|
||||
{ U32 max = MaxOff;
|
||||
{ unsigned max = MaxOff;
|
||||
headerSize = FSEv05_readNCount(norm, &max, &Offlog, ip, iend-ip);
|
||||
if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
|
||||
if (Offlog > OffFSEv05Log) return ERROR(corruption_detected);
|
||||
@ -3105,7 +3105,7 @@ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t
|
||||
break;
|
||||
case FSEv05_ENCODING_DYNAMIC :
|
||||
default : /* impossible */
|
||||
{ U32 max = MaxML;
|
||||
{ unsigned max = MaxML;
|
||||
headerSize = FSEv05_readNCount(norm, &max, &MLlog, ip, iend-ip);
|
||||
if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
|
||||
if (MLlog > MLFSEv05Log) return ERROR(corruption_detected);
|
||||
@ -3305,9 +3305,9 @@ static size_t ZSTDv05_decompressSequences(
|
||||
const BYTE* const litEnd = litPtr + dctx->litSize;
|
||||
int nbSeq=0;
|
||||
const BYTE* dumps = NULL;
|
||||
U32* DTableLL = dctx->LLTable;
|
||||
U32* DTableML = dctx->MLTable;
|
||||
U32* DTableOffb = dctx->OffTable;
|
||||
unsigned* DTableLL = dctx->LLTable;
|
||||
unsigned* DTableML = dctx->MLTable;
|
||||
unsigned* DTableOffb = dctx->OffTable;
|
||||
const BYTE* const base = (const BYTE*) (dctx->base);
|
||||
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
|
||||
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
|
||||
@ -3633,7 +3633,7 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d
|
||||
{
|
||||
size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, errorCode, litlengthHeaderSize;
|
||||
short offcodeNCount[MaxOff+1];
|
||||
U32 offcodeMaxValue=MaxOff, offcodeLog;
|
||||
unsigned offcodeMaxValue=MaxOff, offcodeLog;
|
||||
short matchlengthNCount[MaxML+1];
|
||||
unsigned matchlengthMaxValue = MaxML, matchlengthLog;
|
||||
short litlengthNCount[MaxLL+1];
|
||||
|
||||
1564
lib/zstd.h
1564
lib/zstd.h
File diff suppressed because it is too large
Load Diff
@ -29,7 +29,12 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT))
|
||||
|
||||
ZSTD_VERSION = $(LIBVER)
|
||||
|
||||
GREP = grep --color=never
|
||||
HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
|
||||
GREP_OPTIONS ?=
|
||||
ifeq ($HAVE_COLORNEVER, 1)
|
||||
GREP_OPTIONS += --color=never
|
||||
endif
|
||||
GREP = grep $(GREP_OPTIONS)
|
||||
|
||||
ifeq ($(shell $(CC) -v 2>&1 | $(GREP) -c "gcc version "), 1)
|
||||
ALIGN_LOOP = -falign-loops=32
|
||||
@ -48,7 +53,7 @@ DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
||||
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
|
||||
-Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
|
||||
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
|
||||
-Wredundant-decls -Wmissing-prototypes
|
||||
-Wredundant-decls -Wmissing-prototypes -Wc++-compat
|
||||
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
|
||||
|
||||
@ -91,7 +96,7 @@ VOID = /dev/null
|
||||
|
||||
# thread detection
|
||||
NO_THREAD_MSG := ==> no threads, building without multithreading support
|
||||
HAVE_PTHREAD := $(shell printf '\#include <pthread.h>\nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_pthread$(EXT) -x c - -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0)
|
||||
HAVE_PTHREAD := $(shell printf '\#include <pthread.h>\nint main(void) { return 0; }' > have_pthread.c && $(CC) $(FLAGS) -o have_pthread$(EXT) have_pthread.c -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0; rm have_pthread.c)
|
||||
HAVE_THREAD := $(shell [ "$(HAVE_PTHREAD)" -eq "1" -o -n "$(filter Windows%,$(OS))" ] && echo 1 || echo 0)
|
||||
ifeq ($(HAVE_THREAD), 1)
|
||||
THREAD_MSG := ==> building with threading support
|
||||
@ -103,7 +108,7 @@ endif
|
||||
|
||||
# zlib detection
|
||||
NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support
|
||||
HAVE_ZLIB := $(shell printf '\#include <zlib.h>\nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_zlib$(EXT) -x c - -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0)
|
||||
HAVE_ZLIB := $(shell printf '\#include <zlib.h>\nint main(void) { return 0; }' > have_zlib.c && $(CC) $(FLAGS) -o have_zlib$(EXT) have_zlib.c -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0; rm have_zlib.c)
|
||||
ifeq ($(HAVE_ZLIB), 1)
|
||||
ZLIB_MSG := ==> building zstd with .gz compression support
|
||||
ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS
|
||||
@ -114,7 +119,7 @@ endif
|
||||
|
||||
# lzma detection
|
||||
NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support
|
||||
HAVE_LZMA := $(shell printf '\#include <lzma.h>\nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_lzma$(EXT) -x c - -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0)
|
||||
HAVE_LZMA := $(shell printf '\#include <lzma.h>\nint main(void) { return 0; }' > have_lzma.c && $(CC) $(FLAGS) -o have_lzma$(EXT) have_lzma.c -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0; rm have_lzma.c)
|
||||
ifeq ($(HAVE_LZMA), 1)
|
||||
LZMA_MSG := ==> building zstd with .xz/.lzma compression support
|
||||
LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS
|
||||
@ -125,7 +130,7 @@ endif
|
||||
|
||||
# lz4 detection
|
||||
NO_LZ4_MSG := ==> no liblz4, building zstd without .lz4 support
|
||||
HAVE_LZ4 := $(shell printf '\#include <lz4frame.h>\n\#include <lz4.h>\nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_lz4$(EXT) -x c - -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0)
|
||||
HAVE_LZ4 := $(shell printf '\#include <lz4frame.h>\n\#include <lz4.h>\nint main(void) { return 0; }' > have_lz4.c && $(CC) $(FLAGS) -o have_lz4$(EXT) have_lz4.c -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0; rm have_lz4.c)
|
||||
ifeq ($(HAVE_LZ4), 1)
|
||||
LZ4_MSG := ==> building zstd with .lz4 compression support
|
||||
LZ4CPP = -DZSTD_LZ4COMPRESS -DZSTD_LZ4DECOMPRESS
|
||||
@ -160,7 +165,7 @@ $(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
|
||||
zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP)
|
||||
zstd : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD)
|
||||
zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
|
||||
zstd : $(ZSTDLIB_FILES) zstdcli.o fileio.o bench.o datagen.o dibio.o
|
||||
zstd : $(ZSTDLIB_FILES) zstdcli.o util.o fileio.o benchfn.o benchzstd.o datagen.o dibio.o
|
||||
@echo "$(THREAD_MSG)"
|
||||
@echo "$(ZLIB_MSG)"
|
||||
@echo "$(LZMA_MSG)"
|
||||
@ -178,13 +183,13 @@ zstd-release: zstd
|
||||
zstd32 : CPPFLAGS += $(THREAD_CPP)
|
||||
zstd32 : LDFLAGS += $(THREAD_LD)
|
||||
zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
|
||||
zstd32 : $(ZSTDLIB_FILES) zstdcli.c fileio.c bench.c datagen.c dibio.c
|
||||
zstd32 : $(ZSTDLIB_FILES) zstdcli.c util.c fileio.c benchfn.c benchzstd.c datagen.c dibio.c
|
||||
ifneq (,$(filter Windows%,$(OS)))
|
||||
windres/generate_res.bat
|
||||
endif
|
||||
$(CC) -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT)
|
||||
|
||||
zstd-nolegacy : $(ZSTD_FILES) $(ZDICT_FILES) zstdcli.o fileio.c bench.o datagen.o dibio.o
|
||||
zstd-nolegacy : $(ZSTD_FILES) $(ZDICT_FILES) zstdcli.o util.o fileio.c benchfn.o benchzstd.o datagen.o dibio.o
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
|
||||
|
||||
zstd-nomt : THREAD_CPP :=
|
||||
@ -203,27 +208,27 @@ zstd-noxz : LZMA_MSG := - xz/lzma support is disabled
|
||||
zstd-noxz : zstd
|
||||
|
||||
|
||||
zstd-pgo : MOREFLAGS = -fprofile-generate
|
||||
zstd-pgo : clean zstd
|
||||
zstd-pgo :
|
||||
$(MAKE) clean
|
||||
$(MAKE) zstd MOREFLAGS=-fprofile-generate
|
||||
./zstd -b19i1 $(PROFILE_WITH)
|
||||
./zstd -b16i1 $(PROFILE_WITH)
|
||||
./zstd -b9i2 $(PROFILE_WITH)
|
||||
./zstd -b $(PROFILE_WITH)
|
||||
./zstd -b7i2 $(PROFILE_WITH)
|
||||
./zstd -b5 $(PROFILE_WITH)
|
||||
$(RM) zstd
|
||||
$(RM) $(ZSTDDECOMP_O)
|
||||
$(RM) zstd *.o $(ZSTDDECOMP_O) $(ZSTDDIR)/compress/*.o
|
||||
$(MAKE) zstd MOREFLAGS=-fprofile-use
|
||||
|
||||
# minimal target, with only zstd compression and decompression. no bench. no legacy.
|
||||
zstd-small: CFLAGS = -Os -s
|
||||
zstd-frugal zstd-small: $(ZSTD_FILES) zstdcli.c fileio.c
|
||||
zstd-frugal zstd-small: $(ZSTD_FILES) zstdcli.c util.c fileio.c
|
||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o $@$(EXT)
|
||||
|
||||
zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c fileio.c
|
||||
zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c util.c fileio.c
|
||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o $@$(EXT)
|
||||
|
||||
zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c
|
||||
zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c util.c fileio.c
|
||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT)
|
||||
|
||||
zstdmt: zstd
|
||||
@ -275,7 +280,12 @@ preview-man: clean-man man
|
||||
#-----------------------------------------------------------------------------
|
||||
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
|
||||
|
||||
EGREP = egrep --color=never
|
||||
HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
|
||||
EGREP_OPTIONS ?=
|
||||
ifeq ($HAVE_COLORNEVER, 1)
|
||||
EGREP_OPTIONS += --color=never
|
||||
endif
|
||||
EGREP = egrep $(EGREP_OPTIONS)
|
||||
|
||||
# Print a two column output of targets and their description. To add a target description, put a
|
||||
# comment in the Makefile with the format "## <TARGET>: <DESCRIPTION>". For example:
|
||||
@ -352,6 +362,7 @@ uninstall:
|
||||
@$(RM) $(DESTDIR)$(BINDIR)/zstdless
|
||||
@$(RM) $(DESTDIR)$(BINDIR)/zstdcat
|
||||
@$(RM) $(DESTDIR)$(BINDIR)/unzstd
|
||||
@$(RM) $(DESTDIR)$(BINDIR)/zstdmt
|
||||
@$(RM) $(DESTDIR)$(BINDIR)/zstd
|
||||
@$(RM) $(DESTDIR)$(MAN1DIR)/zstdless.1
|
||||
@$(RM) $(DESTDIR)$(MAN1DIR)/zstdgrep.1
|
||||
|
||||
@ -172,6 +172,11 @@ Benchmark arguments :
|
||||
--priority=rt : set process priority to real-time
|
||||
```
|
||||
|
||||
#### Restricted usage of Environment Variables
|
||||
Using environment variables to set compression/decompression parameters has security implications. Therefore,
|
||||
we intentionally restrict its usage. Currently, only `ZSTD_CLEVEL` is supported for setting compression level.
|
||||
If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a warning message.
|
||||
Note that command line options will override corresponding environment variable settings.
|
||||
|
||||
#### Long distance matching mode
|
||||
The long distance matching mode, enabled with `--long`, is designed to improve
|
||||
|
||||
263
programs/benchfn.c
Normal file
263
programs/benchfn.c
Normal file
@ -0,0 +1,263 @@
|
||||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Includes
|
||||
***************************************/
|
||||
#include "platform.h" /* Large Files support */
|
||||
#include "util.h" /* UTIL_getFileSize, UTIL_sleep */
|
||||
#include <stdlib.h> /* malloc, free */
|
||||
#include <string.h> /* memset */
|
||||
#include <stdio.h> /* fprintf, fopen */
|
||||
#undef NDEBUG /* assert must not be disabled */
|
||||
#include <assert.h> /* assert */
|
||||
|
||||
#include "mem.h"
|
||||
#include "benchfn.h"
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
#define TIMELOOP_MICROSEC (1*1000000ULL) /* 1 second */
|
||||
#define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */
|
||||
#define ACTIVEPERIOD_MICROSEC (70*TIMELOOP_MICROSEC) /* 70 seconds */
|
||||
#define COOLPERIOD_SEC 10
|
||||
|
||||
#define KB *(1 <<10)
|
||||
#define MB *(1 <<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Errors
|
||||
***************************************/
|
||||
#ifndef DEBUG
|
||||
# define DEBUG 0
|
||||
#endif
|
||||
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
|
||||
|
||||
/* error without displaying */
|
||||
#define RETURN_QUIET_ERROR(retValue, ...) { \
|
||||
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
|
||||
DEBUGOUTPUT("Error : "); \
|
||||
DEBUGOUTPUT(__VA_ARGS__); \
|
||||
DEBUGOUTPUT(" \n"); \
|
||||
return retValue; \
|
||||
}
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Benchmarking an arbitrary function
|
||||
***************************************/
|
||||
|
||||
int BMK_isSuccessful_runOutcome(BMK_runOutcome_t outcome)
|
||||
{
|
||||
return outcome.error_tag_never_ever_use_directly == 0;
|
||||
}
|
||||
|
||||
/* warning : this function will stop program execution if outcome is invalid !
|
||||
* check outcome validity first, using BMK_isValid_runResult() */
|
||||
BMK_runTime_t BMK_extract_runTime(BMK_runOutcome_t outcome)
|
||||
{
|
||||
assert(outcome.error_tag_never_ever_use_directly == 0);
|
||||
return outcome.internal_never_ever_use_directly;
|
||||
}
|
||||
|
||||
size_t BMK_extract_errorResult(BMK_runOutcome_t outcome)
|
||||
{
|
||||
assert(outcome.error_tag_never_ever_use_directly != 0);
|
||||
return outcome.error_result_never_ever_use_directly;
|
||||
}
|
||||
|
||||
static BMK_runOutcome_t BMK_runOutcome_error(size_t errorResult)
|
||||
{
|
||||
BMK_runOutcome_t b;
|
||||
memset(&b, 0, sizeof(b));
|
||||
b.error_tag_never_ever_use_directly = 1;
|
||||
b.error_result_never_ever_use_directly = errorResult;
|
||||
return b;
|
||||
}
|
||||
|
||||
static BMK_runOutcome_t BMK_setValid_runTime(BMK_runTime_t runTime)
|
||||
{
|
||||
BMK_runOutcome_t outcome;
|
||||
outcome.error_tag_never_ever_use_directly = 0;
|
||||
outcome.internal_never_ever_use_directly = runTime;
|
||||
return outcome;
|
||||
}
|
||||
|
||||
|
||||
/* initFn will be measured once, benchFn will be measured `nbLoops` times */
|
||||
/* initFn is optional, provide NULL if none */
|
||||
/* benchFn must return a size_t value that errorFn can interpret */
|
||||
/* takes # of blocks and list of size & stuff for each. */
|
||||
/* can report result of benchFn for each block into blockResult. */
|
||||
/* blockResult is optional, provide NULL if this information is not required */
|
||||
/* note : time per loop can be reported as zero if run time < timer resolution */
|
||||
BMK_runOutcome_t BMK_benchFunction(BMK_benchParams_t p,
|
||||
unsigned nbLoops)
|
||||
{
|
||||
size_t dstSize = 0;
|
||||
nbLoops += !nbLoops; /* minimum nbLoops is 1 */
|
||||
|
||||
/* init */
|
||||
{ size_t i;
|
||||
for(i = 0; i < p.blockCount; i++) {
|
||||
memset(p.dstBuffers[i], 0xE5, p.dstCapacities[i]); /* warm up and erase result buffer */
|
||||
}
|
||||
#if 0
|
||||
/* based on testing these seem to lower accuracy of multiple calls of 1 nbLoops vs 1 call of multiple nbLoops
|
||||
* (Makes former slower)
|
||||
*/
|
||||
UTIL_sleepMilli(5); /* give processor time to other processes */
|
||||
UTIL_waitForNextTick();
|
||||
#endif
|
||||
}
|
||||
|
||||
/* benchmark */
|
||||
{ UTIL_time_t const clockStart = UTIL_getTime();
|
||||
unsigned loopNb, blockNb;
|
||||
if (p.initFn != NULL) p.initFn(p.initPayload);
|
||||
for (loopNb = 0; loopNb < nbLoops; loopNb++) {
|
||||
for (blockNb = 0; blockNb < p.blockCount; blockNb++) {
|
||||
size_t const res = p.benchFn(p.srcBuffers[blockNb], p.srcSizes[blockNb],
|
||||
p.dstBuffers[blockNb], p.dstCapacities[blockNb],
|
||||
p.benchPayload);
|
||||
if (loopNb == 0) {
|
||||
if (p.blockResults != NULL) p.blockResults[blockNb] = res;
|
||||
if ((p.errorFn != NULL) && (p.errorFn(res))) {
|
||||
RETURN_QUIET_ERROR(BMK_runOutcome_error(res),
|
||||
"Function benchmark failed on block %u (of size %u) with error %i",
|
||||
blockNb, (unsigned)p.srcSizes[blockNb], (int)res);
|
||||
}
|
||||
dstSize += res;
|
||||
} }
|
||||
} /* for (loopNb = 0; loopNb < nbLoops; loopNb++) */
|
||||
|
||||
{ U64 const totalTime = UTIL_clockSpanNano(clockStart);
|
||||
BMK_runTime_t rt;
|
||||
rt.nanoSecPerRun = totalTime / nbLoops;
|
||||
rt.sumOfReturn = dstSize;
|
||||
return BMK_setValid_runTime(rt);
|
||||
} }
|
||||
}
|
||||
|
||||
|
||||
/* ==== Benchmarking any function, providing intermediate results ==== */
|
||||
|
||||
struct BMK_timedFnState_s {
|
||||
U64 timeSpent_ns;
|
||||
U64 timeBudget_ns;
|
||||
U64 runBudget_ns;
|
||||
BMK_runTime_t fastestRun;
|
||||
unsigned nbLoops;
|
||||
UTIL_time_t coolTime;
|
||||
}; /* typedef'd to BMK_timedFnState_t within bench.h */
|
||||
|
||||
BMK_timedFnState_t* BMK_createTimedFnState(unsigned total_ms, unsigned run_ms)
|
||||
{
|
||||
BMK_timedFnState_t* const r = (BMK_timedFnState_t*)malloc(sizeof(*r));
|
||||
if (r == NULL) return NULL; /* malloc() error */
|
||||
BMK_resetTimedFnState(r, total_ms, run_ms);
|
||||
return r;
|
||||
}
|
||||
|
||||
void BMK_freeTimedFnState(BMK_timedFnState_t* state) {
|
||||
free(state);
|
||||
}
|
||||
|
||||
void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms, unsigned run_ms)
|
||||
{
|
||||
if (!total_ms) total_ms = 1 ;
|
||||
if (!run_ms) run_ms = 1;
|
||||
if (run_ms > total_ms) run_ms = total_ms;
|
||||
timedFnState->timeSpent_ns = 0;
|
||||
timedFnState->timeBudget_ns = (U64)total_ms * TIMELOOP_NANOSEC / 1000;
|
||||
timedFnState->runBudget_ns = (U64)run_ms * TIMELOOP_NANOSEC / 1000;
|
||||
timedFnState->fastestRun.nanoSecPerRun = (U64)(-1LL);
|
||||
timedFnState->fastestRun.sumOfReturn = (size_t)(-1LL);
|
||||
timedFnState->nbLoops = 1;
|
||||
timedFnState->coolTime = UTIL_getTime();
|
||||
}
|
||||
|
||||
/* Tells if nb of seconds set in timedFnState for all runs is spent.
|
||||
* note : this function will return 1 if BMK_benchFunctionTimed() has actually errored. */
|
||||
int BMK_isCompleted_TimedFn(const BMK_timedFnState_t* timedFnState)
|
||||
{
|
||||
return (timedFnState->timeSpent_ns >= timedFnState->timeBudget_ns);
|
||||
}
|
||||
|
||||
|
||||
#undef MIN
|
||||
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
|
||||
|
||||
#define MINUSABLETIME (TIMELOOP_NANOSEC / 2) /* 0.5 seconds */
|
||||
|
||||
BMK_runOutcome_t BMK_benchTimedFn(BMK_timedFnState_t* cont,
|
||||
BMK_benchParams_t p)
|
||||
{
|
||||
U64 const runBudget_ns = cont->runBudget_ns;
|
||||
U64 const runTimeMin_ns = runBudget_ns / 2;
|
||||
int completed = 0;
|
||||
BMK_runTime_t bestRunTime = cont->fastestRun;
|
||||
|
||||
while (!completed) {
|
||||
BMK_runOutcome_t runResult;
|
||||
|
||||
/* Overheat protection */
|
||||
if (UTIL_clockSpanMicro(cont->coolTime) > ACTIVEPERIOD_MICROSEC) {
|
||||
DEBUGOUTPUT("\rcooling down ... \r");
|
||||
UTIL_sleep(COOLPERIOD_SEC);
|
||||
cont->coolTime = UTIL_getTime();
|
||||
}
|
||||
|
||||
/* reinitialize capacity */
|
||||
runResult = BMK_benchFunction(p, cont->nbLoops);
|
||||
|
||||
if(!BMK_isSuccessful_runOutcome(runResult)) { /* error : move out */
|
||||
return runResult;
|
||||
}
|
||||
|
||||
{ BMK_runTime_t const newRunTime = BMK_extract_runTime(runResult);
|
||||
U64 const loopDuration_ns = newRunTime.nanoSecPerRun * cont->nbLoops;
|
||||
|
||||
cont->timeSpent_ns += loopDuration_ns;
|
||||
|
||||
/* estimate nbLoops for next run to last approximately 1 second */
|
||||
if (loopDuration_ns > (runBudget_ns / 50)) {
|
||||
U64 const fastestRun_ns = MIN(bestRunTime.nanoSecPerRun, newRunTime.nanoSecPerRun);
|
||||
cont->nbLoops = (U32)(runBudget_ns / fastestRun_ns) + 1;
|
||||
} else {
|
||||
/* previous run was too short : blindly increase workload by x multiplier */
|
||||
const unsigned multiplier = 10;
|
||||
assert(cont->nbLoops < ((unsigned)-1) / multiplier); /* avoid overflow */
|
||||
cont->nbLoops *= multiplier;
|
||||
}
|
||||
|
||||
if(loopDuration_ns < runTimeMin_ns) {
|
||||
/* don't report results for which benchmark run time was too small : increased risks of rounding errors */
|
||||
assert(completed == 0);
|
||||
continue;
|
||||
} else {
|
||||
if(newRunTime.nanoSecPerRun < bestRunTime.nanoSecPerRun) {
|
||||
bestRunTime = newRunTime;
|
||||
}
|
||||
completed = 1;
|
||||
}
|
||||
}
|
||||
} /* while (!completed) */
|
||||
|
||||
return BMK_setValid_runTime(bestRunTime);
|
||||
}
|
||||
167
programs/benchfn.h
Normal file
167
programs/benchfn.h
Normal file
@ -0,0 +1,167 @@
|
||||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
|
||||
/* benchfn :
|
||||
* benchmark any function on a set of input
|
||||
* providing result in nanoSecPerRun
|
||||
* or detecting and returning an error
|
||||
*/
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#ifndef BENCH_FN_H_23876
|
||||
#define BENCH_FN_H_23876
|
||||
|
||||
/* === Dependencies === */
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
|
||||
/* ==== Benchmark any function, iterated on a set of blocks ==== */
|
||||
|
||||
/* BMK_runTime_t: valid result return type */
|
||||
|
||||
typedef struct {
|
||||
unsigned long long nanoSecPerRun; /* time per iteration (over all blocks) */
|
||||
size_t sumOfReturn; /* sum of return values */
|
||||
} BMK_runTime_t;
|
||||
|
||||
|
||||
/* BMK_runOutcome_t:
|
||||
* type expressing the outcome of a benchmark run by BMK_benchFunction(),
|
||||
* which can be either valid or invalid.
|
||||
* benchmark outcome can be invalid if errorFn is provided.
|
||||
* BMK_runOutcome_t must be considered "opaque" : never access its members directly.
|
||||
* Instead, use its assigned methods :
|
||||
* BMK_isSuccessful_runOutcome, BMK_extract_runTime, BMK_extract_errorResult.
|
||||
* The structure is only described here to allow its allocation on stack. */
|
||||
|
||||
typedef struct {
|
||||
BMK_runTime_t internal_never_ever_use_directly;
|
||||
size_t error_result_never_ever_use_directly;
|
||||
int error_tag_never_ever_use_directly;
|
||||
} BMK_runOutcome_t;
|
||||
|
||||
|
||||
/* prototypes for benchmarked functions */
|
||||
typedef size_t (*BMK_benchFn_t)(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* customPayload);
|
||||
typedef size_t (*BMK_initFn_t)(void* initPayload);
|
||||
typedef unsigned (*BMK_errorFn_t)(size_t);
|
||||
|
||||
|
||||
/* BMK_benchFunction() parameters are provided through following structure.
|
||||
* This is preferable for readability,
|
||||
* as the number of parameters required is pretty large.
|
||||
* No initializer is provided, because it doesn't make sense to provide some "default" :
|
||||
* all parameters should be specified by the caller */
|
||||
typedef struct {
|
||||
BMK_benchFn_t benchFn; /* the function to benchmark, over the set of blocks */
|
||||
void* benchPayload; /* pass custom parameters to benchFn :
|
||||
* (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload) */
|
||||
BMK_initFn_t initFn; /* (*initFn)(initPayload) is run once per run, at the beginning. */
|
||||
void* initPayload; /* Both arguments can be NULL, in which case nothing is run. */
|
||||
BMK_errorFn_t errorFn; /* errorFn will check each return value of benchFn over each block, to determine if it failed or not.
|
||||
* errorFn can be NULL, in which case no check is performed.
|
||||
* errorFn must return 0 when benchFn was successful, and >= 1 if it detects an error.
|
||||
* Execution is stopped as soon as an error is detected.
|
||||
* the triggering return value can be retrieved using BMK_extract_errorResult(). */
|
||||
size_t blockCount; /* number of blocks to operate benchFn on.
|
||||
* It's also the size of all array parameters :
|
||||
* srcBuffers, srcSizes, dstBuffers, dstCapacities, blockResults */
|
||||
const void *const * srcBuffers; /* array of buffers to be operated on by benchFn */
|
||||
const size_t* srcSizes; /* array of the sizes of srcBuffers buffers */
|
||||
void *const * dstBuffers;/* array of buffers to be written into by benchFn */
|
||||
const size_t* dstCapacities; /* array of the capacities of dstBuffers buffers */
|
||||
size_t* blockResults; /* Optional: store the return value of benchFn for each block. Use NULL if this result is not requested. */
|
||||
} BMK_benchParams_t;
|
||||
|
||||
|
||||
/* BMK_benchFunction() :
|
||||
* This function benchmarks benchFn and initFn, providing a result.
|
||||
*
|
||||
* params : see description of BMK_benchParams_t above.
|
||||
* nbLoops: defines number of times benchFn is run over the full set of blocks.
|
||||
* Minimum value is 1. A 0 is interpreted as a 1.
|
||||
*
|
||||
* @return: can express either an error or a successful result.
|
||||
* Use BMK_isSuccessful_runOutcome() to check if benchmark was successful.
|
||||
* If yes, extract the result with BMK_extract_runTime(),
|
||||
* it will contain :
|
||||
* .sumOfReturn : the sum of all return values of benchFn through all of blocks
|
||||
* .nanoSecPerRun : time per run of benchFn + (time for initFn / nbLoops)
|
||||
* .sumOfReturn is generally intended for functions which return a # of bytes written into dstBuffer,
|
||||
* in which case, this value will be the total amount of bytes written into dstBuffer.
|
||||
*
|
||||
* blockResults : when provided (!= NULL), and when benchmark is successful,
|
||||
* params.blockResults contains all return values of `benchFn` over all blocks.
|
||||
* when provided (!= NULL), and when benchmark failed,
|
||||
* params.blockResults contains return values of `benchFn` over all blocks preceding and including the failed block.
|
||||
*/
|
||||
BMK_runOutcome_t BMK_benchFunction(BMK_benchParams_t params, unsigned nbLoops);
|
||||
|
||||
|
||||
|
||||
/* check first if the benchmark was successful or not */
|
||||
int BMK_isSuccessful_runOutcome(BMK_runOutcome_t outcome);
|
||||
|
||||
/* If the benchmark was successful, extract the result.
|
||||
* note : this function will abort() program execution if benchmark failed !
|
||||
* always check if benchmark was successful first !
|
||||
*/
|
||||
BMK_runTime_t BMK_extract_runTime(BMK_runOutcome_t outcome);
|
||||
|
||||
/* when benchmark failed, it means one invocation of `benchFn` failed.
|
||||
* The failure was detected by `errorFn`, operating on return values of `benchFn`.
|
||||
* Returns the faulty return value.
|
||||
* note : this function will abort() program execution if benchmark did not failed.
|
||||
* always check if benchmark failed first !
|
||||
*/
|
||||
size_t BMK_extract_errorResult(BMK_runOutcome_t outcome);
|
||||
|
||||
|
||||
|
||||
/* ==== Benchmark any function, returning intermediate results ==== */
|
||||
|
||||
/* state information tracking benchmark session */
|
||||
typedef struct BMK_timedFnState_s BMK_timedFnState_t;
|
||||
|
||||
/* BMK_benchTimedFn() :
|
||||
* Similar to BMK_benchFunction(), most arguments being identical.
|
||||
* Automatically determines `nbLoops` so that each result is regularly produced at interval of about run_ms.
|
||||
* Note : minimum `nbLoops` is 1, therefore a run may last more than run_ms, and possibly even more than total_ms.
|
||||
* Usage - initialize timedFnState, select benchmark duration (total_ms) and each measurement duration (run_ms)
|
||||
* call BMK_benchTimedFn() repetitively, each measurement is supposed to last about run_ms
|
||||
* Check if total time budget is spent or exceeded, using BMK_isCompleted_TimedFn()
|
||||
*/
|
||||
BMK_runOutcome_t BMK_benchTimedFn(BMK_timedFnState_t* timedFnState,
|
||||
BMK_benchParams_t params);
|
||||
|
||||
/* Tells if duration of all benchmark runs has exceeded total_ms
|
||||
*/
|
||||
int BMK_isCompleted_TimedFn(const BMK_timedFnState_t* timedFnState);
|
||||
|
||||
/* BMK_createTimedFnState() and BMK_resetTimedFnState() :
|
||||
* Create/Set BMK_timedFnState_t for next benchmark session,
|
||||
* which shall last a minimum of total_ms milliseconds,
|
||||
* producing intermediate results, paced at interval of (approximately) run_ms.
|
||||
*/
|
||||
BMK_timedFnState_t* BMK_createTimedFnState(unsigned total_ms, unsigned run_ms);
|
||||
void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms, unsigned run_ms);
|
||||
void BMK_freeTimedFnState(BMK_timedFnState_t* state);
|
||||
|
||||
|
||||
|
||||
#endif /* BENCH_FN_H_23876 */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
@ -9,7 +9,6 @@
|
||||
*/
|
||||
|
||||
|
||||
|
||||
/* **************************************
|
||||
* Tuning parameters
|
||||
****************************************/
|
||||
@ -18,30 +17,24 @@
|
||||
#endif
|
||||
|
||||
|
||||
/* **************************************
|
||||
* Compiler Warnings
|
||||
****************************************/
|
||||
#ifdef _MSC_VER
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
#endif
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Includes
|
||||
***************************************/
|
||||
#include "platform.h" /* Large Files support */
|
||||
#include "util.h" /* UTIL_getFileSize, UTIL_sleep */
|
||||
#include <stdlib.h> /* malloc, free */
|
||||
#include <string.h> /* memset */
|
||||
#include <string.h> /* memset, strerror */
|
||||
#include <stdio.h> /* fprintf, fopen */
|
||||
#include <errno.h>
|
||||
#include <assert.h> /* assert */
|
||||
|
||||
#include "benchfn.h"
|
||||
#include "mem.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "zstd.h"
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#include "xxhash.h"
|
||||
#include "bench.h"
|
||||
#include "benchzstd.h"
|
||||
#include "zstd_errors.h"
|
||||
|
||||
|
||||
@ -102,6 +95,18 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
|
||||
return errorNum; \
|
||||
}
|
||||
|
||||
#define CHECK_Z(zf) { \
|
||||
size_t const zerr = zf; \
|
||||
if (ZSTD_isError(zerr)) { \
|
||||
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
|
||||
DISPLAY("Error : "); \
|
||||
DISPLAY("%s failed : %s", \
|
||||
#zf, ZSTD_getErrorName(zerr)); \
|
||||
DISPLAY(" \n"); \
|
||||
exit(1); \
|
||||
} \
|
||||
}
|
||||
|
||||
#define RETURN_ERROR(errorNum, retType, ...) { \
|
||||
retType r; \
|
||||
memset(&r, 0, sizeof(retType)); \
|
||||
@ -113,17 +118,6 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
|
||||
return r; \
|
||||
}
|
||||
|
||||
/* error without displaying */
|
||||
#define RETURN_QUIET_ERROR(errorNum, retType, ...) { \
|
||||
retType r; \
|
||||
memset(&r, 0, sizeof(retType)); \
|
||||
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
|
||||
DEBUGOUTPUT("Error %i : ", errorNum); \
|
||||
DEBUGOUTPUT(__VA_ARGS__); \
|
||||
DEBUGOUTPUT(" \n"); \
|
||||
r.tag = errorNum; \
|
||||
return r; \
|
||||
}
|
||||
|
||||
/* *************************************
|
||||
* Benchmark Parameters
|
||||
@ -141,7 +135,7 @@ BMK_advancedParams_t BMK_initAdvancedParams(void) {
|
||||
0, /* ldmMinMatch */
|
||||
0, /* ldmHashLog */
|
||||
0, /* ldmBuckSizeLog */
|
||||
0 /* ldmHashEveryLog */
|
||||
0 /* ldmHashRateLog */
|
||||
};
|
||||
return res;
|
||||
}
|
||||
@ -168,33 +162,32 @@ typedef struct {
|
||||
static void BMK_initCCtx(ZSTD_CCtx* ctx,
|
||||
const void* dictBuffer, size_t dictBufferSize, int cLevel,
|
||||
const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) {
|
||||
ZSTD_CCtx_reset(ctx);
|
||||
ZSTD_CCtx_resetParameters(ctx);
|
||||
ZSTD_CCtx_reset(ctx, ZSTD_reset_session_and_parameters);
|
||||
if (adv->nbWorkers==1) {
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0);
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, 0));
|
||||
} else {
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, adv->nbWorkers);
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, adv->nbWorkers));
|
||||
}
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, adv->ldmFlag);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, adv->ldmMinMatch);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, adv->ldmHashLog);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, adv->ldmBucketSizeLog);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, adv->ldmHashEveryLog);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_hashLog, comprParams->hashLog);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy);
|
||||
ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, cLevel));
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_enableLongDistanceMatching, adv->ldmFlag));
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmMinMatch, adv->ldmMinMatch));
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashLog, adv->ldmHashLog));
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmBucketSizeLog, adv->ldmBucketSizeLog));
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashRateLog, adv->ldmHashRateLog));
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_windowLog, comprParams->windowLog));
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_hashLog, comprParams->hashLog));
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_chainLog, comprParams->chainLog));
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_searchLog, comprParams->searchLog));
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, comprParams->minMatch));
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, comprParams->targetLength));
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_strategy, comprParams->strategy));
|
||||
CHECK_Z(ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize));
|
||||
}
|
||||
|
||||
static void BMK_initDCtx(ZSTD_DCtx* dctx,
|
||||
const void* dictBuffer, size_t dictBufferSize) {
|
||||
ZSTD_DCtx_reset(dctx);
|
||||
ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize);
|
||||
CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters));
|
||||
CHECK_Z(ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize));
|
||||
}
|
||||
|
||||
|
||||
@ -232,22 +225,8 @@ static size_t local_defaultCompress(
|
||||
void* dstBuffer, size_t dstSize,
|
||||
void* addArgs)
|
||||
{
|
||||
size_t moreToFlush = 1;
|
||||
ZSTD_CCtx* const cctx = (ZSTD_CCtx*)addArgs;
|
||||
ZSTD_inBuffer in;
|
||||
ZSTD_outBuffer out;
|
||||
in.src = srcBuffer; in.size = srcSize; in.pos = 0;
|
||||
out.dst = dstBuffer; out.size = dstSize; out.pos = 0;
|
||||
while (moreToFlush) {
|
||||
if(out.pos == out.size) {
|
||||
return (size_t)-ZSTD_error_dstSize_tooSmall;
|
||||
}
|
||||
moreToFlush = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
|
||||
if (ZSTD_isError(moreToFlush)) {
|
||||
return moreToFlush;
|
||||
}
|
||||
}
|
||||
return out.pos;
|
||||
return ZSTD_compress2(cctx, dstBuffer, dstSize, srcBuffer, srcSize);
|
||||
}
|
||||
|
||||
/* `addArgs` is the context */
|
||||
@ -266,7 +245,7 @@ static size_t local_defaultDecompress(
|
||||
if(out.pos == out.size) {
|
||||
return (size_t)-ZSTD_error_dstSize_tooSmall;
|
||||
}
|
||||
moreToFlush = ZSTD_decompress_generic(dctx, &out, &in);
|
||||
moreToFlush = ZSTD_decompressStream(dctx, &out, &in);
|
||||
if (ZSTD_isError(moreToFlush)) {
|
||||
return moreToFlush;
|
||||
}
|
||||
@ -276,219 +255,6 @@ static size_t local_defaultDecompress(
|
||||
}
|
||||
|
||||
|
||||
/*=== Benchmarking an arbitrary function ===*/
|
||||
|
||||
int BMK_isSuccessful_runOutcome(BMK_runOutcome_t outcome)
|
||||
{
|
||||
return outcome.tag == 0;
|
||||
}
|
||||
|
||||
/* warning : this function will stop program execution if outcome is invalid !
|
||||
* check outcome validity first, using BMK_isValid_runResult() */
|
||||
BMK_runTime_t BMK_extract_runTime(BMK_runOutcome_t outcome)
|
||||
{
|
||||
assert(outcome.tag == 0);
|
||||
return outcome.internal_never_use_directly;
|
||||
}
|
||||
|
||||
static BMK_runOutcome_t BMK_runOutcome_error(void)
|
||||
{
|
||||
BMK_runOutcome_t b;
|
||||
memset(&b, 0, sizeof(b));
|
||||
b.tag = 1;
|
||||
return b;
|
||||
}
|
||||
|
||||
static BMK_runOutcome_t BMK_setValid_runTime(BMK_runTime_t runTime)
|
||||
{
|
||||
BMK_runOutcome_t outcome;
|
||||
outcome.tag = 0;
|
||||
outcome.internal_never_use_directly = runTime;
|
||||
return outcome;
|
||||
}
|
||||
|
||||
|
||||
/* initFn will be measured once, benchFn will be measured `nbLoops` times */
|
||||
/* initFn is optional, provide NULL if none */
|
||||
/* benchFn must return size_t field compliant with ZSTD_isError for error valuee */
|
||||
/* takes # of blocks and list of size & stuff for each. */
|
||||
/* can report result of benchFn for each block into blockResult. */
|
||||
/* blockResult is optional, provide NULL if this information is not required */
|
||||
/* note : time per loop could be zero if run time < timer resolution */
|
||||
BMK_runOutcome_t BMK_benchFunction(
|
||||
BMK_benchFn_t benchFn, void* benchPayload,
|
||||
BMK_initFn_t initFn, void* initPayload,
|
||||
size_t blockCount,
|
||||
const void* const * srcBlockBuffers, const size_t* srcBlockSizes,
|
||||
void* const * dstBlockBuffers, const size_t* dstBlockCapacities,
|
||||
size_t* blockResults,
|
||||
unsigned nbLoops)
|
||||
{
|
||||
size_t dstSize = 0;
|
||||
|
||||
if(!nbLoops) {
|
||||
RETURN_QUIET_ERROR(2, BMK_runOutcome_t, "nbLoops must be nonzero ");
|
||||
}
|
||||
|
||||
/* init */
|
||||
{ size_t i;
|
||||
for(i = 0; i < blockCount; i++) {
|
||||
memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */
|
||||
}
|
||||
#if 0
|
||||
/* based on testing these seem to lower accuracy of multiple calls of 1 nbLoops vs 1 call of multiple nbLoops
|
||||
* (Makes former slower)
|
||||
*/
|
||||
UTIL_sleepMilli(5); /* give processor time to other processes */
|
||||
UTIL_waitForNextTick();
|
||||
#endif
|
||||
}
|
||||
|
||||
/* benchmark */
|
||||
{ UTIL_time_t const clockStart = UTIL_getTime();
|
||||
unsigned loopNb, blockNb;
|
||||
if (initFn != NULL) initFn(initPayload);
|
||||
for (loopNb = 0; loopNb < nbLoops; loopNb++) {
|
||||
for (blockNb = 0; blockNb < blockCount; blockNb++) {
|
||||
size_t const res = benchFn(srcBlockBuffers[blockNb], srcBlockSizes[blockNb],
|
||||
dstBlockBuffers[blockNb], dstBlockCapacities[blockNb],
|
||||
benchPayload);
|
||||
if(ZSTD_isError(res)) {
|
||||
RETURN_QUIET_ERROR(2, BMK_runOutcome_t,
|
||||
"Function benchmark failed on block %u of size %u : %s",
|
||||
blockNb, (U32)dstBlockCapacities[blockNb], ZSTD_getErrorName(res));
|
||||
} else if (loopNb == 0) {
|
||||
dstSize += res;
|
||||
if (blockResults != NULL) blockResults[blockNb] = res;
|
||||
} }
|
||||
} /* for (loopNb = 0; loopNb < nbLoops; loopNb++) */
|
||||
|
||||
{ U64 const totalTime = UTIL_clockSpanNano(clockStart);
|
||||
BMK_runTime_t rt;
|
||||
rt.nanoSecPerRun = totalTime / nbLoops;
|
||||
rt.sumOfReturn = dstSize;
|
||||
return BMK_setValid_runTime(rt);
|
||||
} }
|
||||
}
|
||||
|
||||
|
||||
/* ==== Benchmarking any function, providing intermediate results ==== */
|
||||
|
||||
struct BMK_timedFnState_s {
|
||||
U64 timeSpent_ns;
|
||||
U64 timeBudget_ns;
|
||||
U64 runBudget_ns;
|
||||
BMK_runTime_t fastestRun;
|
||||
unsigned nbLoops;
|
||||
UTIL_time_t coolTime;
|
||||
}; /* typedef'd to BMK_timedFnState_t within bench.h */
|
||||
|
||||
BMK_timedFnState_t* BMK_createTimedFnState(unsigned total_ms, unsigned run_ms)
|
||||
{
|
||||
BMK_timedFnState_t* const r = (BMK_timedFnState_t*)malloc(sizeof(*r));
|
||||
if (r == NULL) return NULL; /* malloc() error */
|
||||
BMK_resetTimedFnState(r, total_ms, run_ms);
|
||||
return r;
|
||||
}
|
||||
|
||||
void BMK_freeTimedFnState(BMK_timedFnState_t* state) {
|
||||
free(state);
|
||||
}
|
||||
|
||||
void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms, unsigned run_ms)
|
||||
{
|
||||
if (!total_ms) total_ms = 1 ;
|
||||
if (!run_ms) run_ms = 1;
|
||||
if (run_ms > total_ms) run_ms = total_ms;
|
||||
timedFnState->timeSpent_ns = 0;
|
||||
timedFnState->timeBudget_ns = (U64)total_ms * TIMELOOP_NANOSEC / 1000;
|
||||
timedFnState->runBudget_ns = (U64)run_ms * TIMELOOP_NANOSEC / 1000;
|
||||
timedFnState->fastestRun.nanoSecPerRun = (U64)(-1LL);
|
||||
timedFnState->fastestRun.sumOfReturn = (size_t)(-1LL);
|
||||
timedFnState->nbLoops = 1;
|
||||
timedFnState->coolTime = UTIL_getTime();
|
||||
}
|
||||
|
||||
/* Tells if nb of seconds set in timedFnState for all runs is spent.
|
||||
* note : this function will return 1 if BMK_benchFunctionTimed() has actually errored. */
|
||||
int BMK_isCompleted_TimedFn(const BMK_timedFnState_t* timedFnState)
|
||||
{
|
||||
return (timedFnState->timeSpent_ns >= timedFnState->timeBudget_ns);
|
||||
}
|
||||
|
||||
|
||||
#define MINUSABLETIME (TIMELOOP_NANOSEC / 2) /* 0.5 seconds */
|
||||
|
||||
BMK_runOutcome_t BMK_benchTimedFn(
|
||||
BMK_timedFnState_t* cont,
|
||||
BMK_benchFn_t benchFn, void* benchPayload,
|
||||
BMK_initFn_t initFn, void* initPayload,
|
||||
size_t blockCount,
|
||||
const void* const* srcBlockBuffers, const size_t* srcBlockSizes,
|
||||
void * const * dstBlockBuffers, const size_t * dstBlockCapacities,
|
||||
size_t* blockResults)
|
||||
{
|
||||
U64 const runBudget_ns = cont->runBudget_ns;
|
||||
U64 const runTimeMin_ns = runBudget_ns / 2;
|
||||
int completed = 0;
|
||||
BMK_runTime_t bestRunTime = cont->fastestRun;
|
||||
|
||||
while (!completed) {
|
||||
BMK_runOutcome_t runResult;
|
||||
|
||||
/* Overheat protection */
|
||||
if (UTIL_clockSpanMicro(cont->coolTime) > ACTIVEPERIOD_MICROSEC) {
|
||||
DEBUGOUTPUT("\rcooling down ... \r");
|
||||
UTIL_sleep(COOLPERIOD_SEC);
|
||||
cont->coolTime = UTIL_getTime();
|
||||
}
|
||||
|
||||
/* reinitialize capacity */
|
||||
runResult = BMK_benchFunction(benchFn, benchPayload,
|
||||
initFn, initPayload,
|
||||
blockCount,
|
||||
srcBlockBuffers, srcBlockSizes,
|
||||
dstBlockBuffers, dstBlockCapacities,
|
||||
blockResults,
|
||||
cont->nbLoops);
|
||||
|
||||
if(!BMK_isSuccessful_runOutcome(runResult)) { /* error : move out */
|
||||
return BMK_runOutcome_error();
|
||||
}
|
||||
|
||||
{ BMK_runTime_t const newRunTime = BMK_extract_runTime(runResult);
|
||||
U64 const loopDuration_ns = newRunTime.nanoSecPerRun * cont->nbLoops;
|
||||
|
||||
cont->timeSpent_ns += loopDuration_ns;
|
||||
|
||||
/* estimate nbLoops for next run to last approximately 1 second */
|
||||
if (loopDuration_ns > (runBudget_ns / 50)) {
|
||||
U64 const fastestRun_ns = MIN(bestRunTime.nanoSecPerRun, newRunTime.nanoSecPerRun);
|
||||
cont->nbLoops = (U32)(runBudget_ns / fastestRun_ns) + 1;
|
||||
} else {
|
||||
/* previous run was too short : blindly increase workload by x multiplier */
|
||||
const unsigned multiplier = 10;
|
||||
assert(cont->nbLoops < ((unsigned)-1) / multiplier); /* avoid overflow */
|
||||
cont->nbLoops *= multiplier;
|
||||
}
|
||||
|
||||
if(loopDuration_ns < runTimeMin_ns) {
|
||||
/* don't report results for which benchmark run time was too small : increased risks of rounding errors */
|
||||
assert(completed == 0);
|
||||
continue;
|
||||
} else {
|
||||
if(newRunTime.nanoSecPerRun < bestRunTime.nanoSecPerRun) {
|
||||
bestRunTime = newRunTime;
|
||||
}
|
||||
completed = 1;
|
||||
}
|
||||
}
|
||||
} /* while (!completed) */
|
||||
|
||||
return BMK_setValid_runTime(bestRunTime);
|
||||
}
|
||||
|
||||
|
||||
/* ================================================================= */
|
||||
/* Benchmark Zstandard, mem-to-mem scenarios */
|
||||
/* ================================================================= */
|
||||
@ -522,22 +288,24 @@ static BMK_benchOutcome_t BMK_benchOutcome_setValidResult(BMK_benchResult_t resu
|
||||
|
||||
|
||||
/* benchMem with no allocation */
|
||||
static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
|
||||
const void** srcPtrs, size_t* srcSizes,
|
||||
void** cPtrs, size_t* cCapacities, size_t* cSizes,
|
||||
void** resPtrs, size_t* resSizes,
|
||||
void** resultBufferPtr, void* compressedBuffer,
|
||||
size_t maxCompressedSize,
|
||||
BMK_timedFnState_t* timeStateCompress,
|
||||
BMK_timedFnState_t* timeStateDecompress,
|
||||
static BMK_benchOutcome_t
|
||||
BMK_benchMemAdvancedNoAlloc(
|
||||
const void** srcPtrs, size_t* srcSizes,
|
||||
void** cPtrs, size_t* cCapacities, size_t* cSizes,
|
||||
void** resPtrs, size_t* resSizes,
|
||||
void** resultBufferPtr, void* compressedBuffer,
|
||||
size_t maxCompressedSize,
|
||||
BMK_timedFnState_t* timeStateCompress,
|
||||
BMK_timedFnState_t* timeStateDecompress,
|
||||
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
const size_t* fileSizes, unsigned nbFiles,
|
||||
const int cLevel, const ZSTD_compressionParameters* comprParams,
|
||||
const void* dictBuffer, size_t dictBufferSize,
|
||||
ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
|
||||
int displayLevel, const char* displayName,
|
||||
const BMK_advancedParams_t* adv)
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
const size_t* fileSizes, unsigned nbFiles,
|
||||
const int cLevel,
|
||||
const ZSTD_compressionParameters* comprParams,
|
||||
const void* dictBuffer, size_t dictBufferSize,
|
||||
ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
|
||||
int displayLevel, const char* displayName,
|
||||
const BMK_advancedParams_t* adv)
|
||||
{
|
||||
size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
|
||||
BMK_benchResult_t benchResult;
|
||||
@ -599,6 +367,11 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
|
||||
cPtr += cCapacities[nbBlocks];
|
||||
resPtr += thisBlockSize;
|
||||
remaining -= thisBlockSize;
|
||||
if (adv->mode == BMK_decodeOnly) {
|
||||
assert(nbBlocks==0);
|
||||
cSizes[nbBlocks] = thisBlockSize;
|
||||
benchResult.cSize = thisBlockSize;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -617,32 +390,51 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
|
||||
U32 markNb = 0;
|
||||
int compressionCompleted = (adv->mode == BMK_decodeOnly);
|
||||
int decompressionCompleted = (adv->mode == BMK_compressOnly);
|
||||
BMK_benchParams_t cbp, dbp;
|
||||
BMK_initCCtxArgs cctxprep;
|
||||
BMK_initDCtxArgs dctxprep;
|
||||
|
||||
cbp.benchFn = local_defaultCompress;
|
||||
cbp.benchPayload = cctx;
|
||||
cbp.initFn = local_initCCtx;
|
||||
cbp.initPayload = &cctxprep;
|
||||
cbp.errorFn = ZSTD_isError;
|
||||
cbp.blockCount = nbBlocks;
|
||||
cbp.srcBuffers = srcPtrs;
|
||||
cbp.srcSizes = srcSizes;
|
||||
cbp.dstBuffers = cPtrs;
|
||||
cbp.dstCapacities = cCapacities;
|
||||
cbp.blockResults = cSizes;
|
||||
|
||||
cctxprep.cctx = cctx;
|
||||
cctxprep.dictBuffer = dictBuffer;
|
||||
cctxprep.dictBufferSize = dictBufferSize;
|
||||
cctxprep.cLevel = cLevel;
|
||||
cctxprep.comprParams = comprParams;
|
||||
cctxprep.adv = adv;
|
||||
|
||||
dbp.benchFn = local_defaultDecompress;
|
||||
dbp.benchPayload = dctx;
|
||||
dbp.initFn = local_initDCtx;
|
||||
dbp.initPayload = &dctxprep;
|
||||
dbp.errorFn = ZSTD_isError;
|
||||
dbp.blockCount = nbBlocks;
|
||||
dbp.srcBuffers = (const void* const *) cPtrs;
|
||||
dbp.srcSizes = cSizes;
|
||||
dbp.dstBuffers = resPtrs;
|
||||
dbp.dstCapacities = resSizes;
|
||||
dbp.blockResults = NULL;
|
||||
|
||||
dctxprep.dctx = dctx;
|
||||
dctxprep.dictBuffer = dictBuffer;
|
||||
dctxprep.dictBufferSize = dictBufferSize;
|
||||
|
||||
DISPLAYLEVEL(2, "\r%70s\r", ""); /* blank line */
|
||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
|
||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (unsigned)srcSize);
|
||||
|
||||
while (!(compressionCompleted && decompressionCompleted)) {
|
||||
|
||||
if (!compressionCompleted) {
|
||||
BMK_runOutcome_t const cOutcome =
|
||||
BMK_benchTimedFn( timeStateCompress,
|
||||
&local_defaultCompress, cctx,
|
||||
&local_initCCtx, &cctxprep,
|
||||
nbBlocks,
|
||||
srcPtrs, srcSizes,
|
||||
cPtrs, cCapacities,
|
||||
cSizes);
|
||||
BMK_runOutcome_t const cOutcome = BMK_benchTimedFn( timeStateCompress, cbp);
|
||||
|
||||
if (!BMK_isSuccessful_runOutcome(cOutcome)) {
|
||||
return BMK_benchOutcome_error();
|
||||
@ -659,10 +451,9 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
|
||||
} }
|
||||
|
||||
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
|
||||
markNb = (markNb+1) % NB_MARKS;
|
||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
|
||||
marks[markNb], displayName,
|
||||
(U32)srcSize, (U32)cSize,
|
||||
(unsigned)srcSize, (unsigned)cSize,
|
||||
ratioAccuracy, ratio,
|
||||
benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT);
|
||||
}
|
||||
@ -670,14 +461,7 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
|
||||
}
|
||||
|
||||
if(!decompressionCompleted) {
|
||||
BMK_runOutcome_t const dOutcome =
|
||||
BMK_benchTimedFn(timeStateDecompress,
|
||||
&local_defaultDecompress, dctx,
|
||||
&local_initDCtx, &dctxprep,
|
||||
nbBlocks,
|
||||
(const void *const *)cPtrs, cSizes,
|
||||
resPtrs, resSizes,
|
||||
NULL);
|
||||
BMK_runOutcome_t const dOutcome = BMK_benchTimedFn(timeStateDecompress, dbp);
|
||||
|
||||
if(!BMK_isSuccessful_runOutcome(dOutcome)) {
|
||||
return BMK_benchOutcome_error();
|
||||
@ -690,16 +474,16 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
|
||||
}
|
||||
|
||||
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
|
||||
markNb = (markNb+1) % NB_MARKS;
|
||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
|
||||
marks[markNb], displayName,
|
||||
(U32)srcSize, (U32)benchResult.cSize,
|
||||
(unsigned)srcSize, (unsigned)benchResult.cSize,
|
||||
ratioAccuracy, ratio,
|
||||
benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT,
|
||||
(double)benchResult.dSpeed / MB_UNIT);
|
||||
}
|
||||
decompressionCompleted = BMK_isCompleted_TimedFn(timeStateDecompress);
|
||||
}
|
||||
markNb = (markNb+1) % NB_MARKS;
|
||||
} /* while (!(compressionCompleted && decompressionCompleted)) */
|
||||
|
||||
/* CRC Checking */
|
||||
@ -707,12 +491,13 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
|
||||
U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
|
||||
if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) {
|
||||
size_t u;
|
||||
DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
|
||||
DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n",
|
||||
displayName, (unsigned)crcOrig, (unsigned)crcCheck);
|
||||
for (u=0; u<srcSize; u++) {
|
||||
if (((const BYTE*)srcBuffer)[u] != resultBuffer[u]) {
|
||||
U32 segNb, bNb, pos;
|
||||
unsigned segNb, bNb, pos;
|
||||
size_t bacc = 0;
|
||||
DISPLAY("Decoding error at pos %u ", (U32)u);
|
||||
DISPLAY("Decoding error at pos %u ", (unsigned)u);
|
||||
for (segNb = 0; segNb < nbBlocks; segNb++) {
|
||||
if (bacc + srcSizes[segNb] > u) break;
|
||||
bacc += srcSizes[segNb];
|
||||
@ -883,7 +668,7 @@ static BMK_benchOutcome_t BMK_benchCLevel(const void* srcBuffer, size_t benchedS
|
||||
if (displayLevel == 1 && !adv->additionalParam) /* --quiet mode */
|
||||
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n",
|
||||
ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING,
|
||||
(U32)benchedSize, adv->nbSeconds, (U32)(adv->blockSize>>10));
|
||||
(unsigned)benchedSize, adv->nbSeconds, (unsigned)(adv->blockSize>>10));
|
||||
|
||||
return BMK_benchMemAdvanced(srcBuffer, benchedSize,
|
||||
NULL, 0,
|
||||
@ -1015,6 +800,11 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
|
||||
/* Load dictionary */
|
||||
if (dictFileName != NULL) {
|
||||
U64 const dictFileSize = UTIL_getFileSize(dictFileName);
|
||||
if (dictFileSize == UTIL_FILESIZE_UNKNOWN) {
|
||||
DISPLAYLEVEL(1, "error loading %s : %s \n", dictFileName, strerror(errno));
|
||||
free(fileSizes);
|
||||
RETURN_ERROR(9, BMK_benchOutcome_t, "benchmark aborted");
|
||||
}
|
||||
if (dictFileSize > 64 MB) {
|
||||
free(fileSizes);
|
||||
RETURN_ERROR(10, BMK_benchOutcome_t, "dictionary file %s too large", dictFileName);
|
||||
@ -1024,7 +814,7 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
|
||||
if (dictBuffer==NULL) {
|
||||
free(fileSizes);
|
||||
RETURN_ERROR(11, BMK_benchOutcome_t, "not enough memory for dictionary (%u bytes)",
|
||||
(U32)dictBufferSize);
|
||||
(unsigned)dictBufferSize);
|
||||
}
|
||||
|
||||
{ int const errorCode = BMK_loadFiles(dictBuffer, dictBufferSize,
|
||||
@ -1040,7 +830,7 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
|
||||
benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
|
||||
if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
|
||||
if (benchedSize < totalSizeToLoad)
|
||||
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
|
||||
DISPLAY("Not enough memory; testing %u MB only...\n", (unsigned)(benchedSize >> 20));
|
||||
|
||||
srcBuffer = benchedSize ? malloc(benchedSize) : NULL;
|
||||
if (!srcBuffer) {
|
||||
@ -8,12 +8,18 @@
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
/* benchzstd :
|
||||
* benchmark Zstandard compression / decompression
|
||||
* over a set of files or buffers
|
||||
* and display progress result and final summary
|
||||
*/
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#ifndef BENCH_H_121279284357
|
||||
#define BENCH_H_121279284357
|
||||
#ifndef BENCH_ZSTD_H_3242387
|
||||
#define BENCH_ZSTD_H_3242387
|
||||
|
||||
/* === Dependencies === */
|
||||
#include <stddef.h> /* size_t */
|
||||
@ -109,7 +115,7 @@ typedef struct {
|
||||
unsigned ldmMinMatch; /* below: parameters for long distance matching, see zstd.1.md */
|
||||
unsigned ldmHashLog;
|
||||
unsigned ldmBucketSizeLog;
|
||||
unsigned ldmHashEveryLog;
|
||||
unsigned ldmHashRateLog;
|
||||
} BMK_advancedParams_t;
|
||||
|
||||
/* returns default parameters used by nonAdvanced functions */
|
||||
@ -142,9 +148,9 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
|
||||
* .cMem : memory budget required for the compression context
|
||||
*/
|
||||
BMK_benchOutcome_t BMK_syntheticTest(
|
||||
int cLevel, double compressibility,
|
||||
const ZSTD_compressionParameters* compressionParams,
|
||||
int displayLevel, const BMK_advancedParams_t* adv);
|
||||
int cLevel, double compressibility,
|
||||
const ZSTD_compressionParameters* compressionParams,
|
||||
int displayLevel, const BMK_advancedParams_t* adv);
|
||||
|
||||
|
||||
|
||||
@ -181,6 +187,7 @@ BMK_benchOutcome_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
const void* dictBuffer, size_t dictBufferSize,
|
||||
int displayLevel, const char* displayName);
|
||||
|
||||
|
||||
/* BMK_benchMemAdvanced() : same as BMK_benchMem()
|
||||
* with following additional options :
|
||||
* dstBuffer - destination buffer to write compressed output in, NULL if none provided.
|
||||
@ -197,106 +204,8 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
|
||||
|
||||
|
||||
|
||||
/* ==== Benchmarking any function, iterated on a set of blocks ==== */
|
||||
|
||||
typedef struct {
|
||||
unsigned long long nanoSecPerRun; /* time per iteration */
|
||||
size_t sumOfReturn; /* sum of return values */
|
||||
} BMK_runTime_t;
|
||||
|
||||
VARIANT_ERROR_RESULT(BMK_runTime_t, BMK_runOutcome_t);
|
||||
|
||||
/* check first if the return structure represents an error or a valid result */
|
||||
int BMK_isSuccessful_runOutcome(BMK_runOutcome_t outcome);
|
||||
|
||||
/* extract result from variant type.
|
||||
* note : this function will abort() program execution if result is not valid
|
||||
* check result validity first, by using BMK_isSuccessful_runOutcome()
|
||||
*/
|
||||
BMK_runTime_t BMK_extract_runTime(BMK_runOutcome_t outcome);
|
||||
|
||||
|
||||
|
||||
typedef size_t (*BMK_benchFn_t)(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* customPayload);
|
||||
typedef size_t (*BMK_initFn_t)(void* initPayload);
|
||||
|
||||
|
||||
/* BMK_benchFunction() :
|
||||
* This function times the execution of 2 argument functions, benchFn and initFn */
|
||||
|
||||
/* benchFn - (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload)
|
||||
* is run nbLoops times
|
||||
* initFn - (*initFn)(initPayload) is run once per benchmark, at the beginning.
|
||||
* This argument can be NULL, in which case nothing is run.
|
||||
* blockCount - number of blocks. Size of all array parameters : srcBuffers, srcSizes, dstBuffers, dstCapacities, blockResults
|
||||
* srcBuffers - an array of buffers to be operated on by benchFn
|
||||
* srcSizes - an array of the sizes of above buffers
|
||||
* dstBuffers - an array of buffers to be written into by benchFn
|
||||
* dstCapacities - an array of the capacities of above buffers
|
||||
* blockResults - Optional: store the return value of benchFn for each block. Use NULL if this result is not requested.
|
||||
* nbLoops - defines number of times benchFn is run.
|
||||
* @return: a variant, which express either an error, or can generate a valid BMK_runTime_t result.
|
||||
* Use BMK_isSuccessful_runOutcome() to check if function was successful.
|
||||
* If yes, extract the result with BMK_extract_runTime(),
|
||||
* it will contain :
|
||||
* .sumOfReturn : the sum of all return values of benchFn through all of blocks
|
||||
* .nanoSecPerRun : time per run of benchFn + (time for initFn / nbLoops)
|
||||
* .sumOfReturn is generally intended for functions which return a # of bytes written into dstBuffer,
|
||||
* in which case, this value will be the total amount of bytes written into dstBuffer.
|
||||
*/
|
||||
BMK_runOutcome_t BMK_benchFunction(
|
||||
BMK_benchFn_t benchFn, void* benchPayload,
|
||||
BMK_initFn_t initFn, void* initPayload,
|
||||
size_t blockCount,
|
||||
const void *const * srcBuffers, const size_t* srcSizes,
|
||||
void *const * dstBuffers, const size_t* dstCapacities,
|
||||
size_t* blockResults,
|
||||
unsigned nbLoops);
|
||||
|
||||
|
||||
|
||||
/* ==== Benchmark any function, providing intermediate results ==== */
|
||||
|
||||
/* state information tracking benchmark session */
|
||||
typedef struct BMK_timedFnState_s BMK_timedFnState_t;
|
||||
|
||||
/* BMK_createTimedFnState() and BMK_resetTimedFnState() :
|
||||
* Create/Set BMK_timedFnState_t for next benchmark session,
|
||||
* which shall last a minimum of total_ms milliseconds,
|
||||
* producing intermediate results, paced at interval of (approximately) run_ms.
|
||||
*/
|
||||
BMK_timedFnState_t* BMK_createTimedFnState(unsigned total_ms, unsigned run_ms);
|
||||
void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms, unsigned run_ms);
|
||||
void BMK_freeTimedFnState(BMK_timedFnState_t* state);
|
||||
|
||||
|
||||
/* Tells if duration of all benchmark runs has exceeded total_ms
|
||||
*/
|
||||
int BMK_isCompleted_TimedFn(const BMK_timedFnState_t* timedFnState);
|
||||
|
||||
|
||||
/* BMK_benchTimedFn() :
|
||||
* Similar to BMK_benchFunction(), most arguments being identical.
|
||||
* Automatically determines `nbLoops` so that each result is regularly produced at interval of about run_ms.
|
||||
* Note : minimum `nbLoops` is 1, therefore a run may last more than run_ms, and possibly even more than total_ms.
|
||||
* Usage - initialize timedFnState, select benchmark duration (total_ms) and each measurement duration (run_ms)
|
||||
* call BMK_benchTimedFn() repetitively, each measurement is supposed to last about run_ms
|
||||
* Check if total time budget is spent or exceeded, using BMK_isCompleted_TimedFn()
|
||||
*/
|
||||
BMK_runOutcome_t BMK_benchTimedFn(
|
||||
BMK_timedFnState_t* timedFnState,
|
||||
BMK_benchFn_t benchFn, void* benchPayload,
|
||||
BMK_initFn_t initFn, void* initPayload,
|
||||
size_t blockCount,
|
||||
const void *const * srcBlockBuffers, const size_t* srcBlockSizes,
|
||||
void *const * dstBlockBuffers, const size_t* dstBlockCapacities,
|
||||
size_t* blockResults);
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
#endif /* BENCH_H_121279284357 */
|
||||
#endif /* BENCH_ZSTD_H_3242387 */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
@ -81,18 +81,18 @@ static BYTE RDG_genChar(U32* seed, const BYTE* ldt)
|
||||
}
|
||||
|
||||
|
||||
static U32 RDG_rand15Bits (unsigned* seedPtr)
|
||||
static U32 RDG_rand15Bits (U32* seedPtr)
|
||||
{
|
||||
return RDG_rand(seedPtr) & 0x7FFF;
|
||||
}
|
||||
|
||||
static U32 RDG_randLength(unsigned* seedPtr)
|
||||
static U32 RDG_randLength(U32* seedPtr)
|
||||
{
|
||||
if (RDG_rand(seedPtr) & 7) return (RDG_rand(seedPtr) & 0xF); /* small length */
|
||||
return (RDG_rand(seedPtr) & 0x1FF) + 0xF;
|
||||
}
|
||||
|
||||
static void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, const BYTE* ldt, unsigned* seedPtr)
|
||||
static void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, const BYTE* ldt, U32* seedPtr)
|
||||
{
|
||||
BYTE* const buffPtr = (BYTE*)buffer;
|
||||
U32 const matchProba32 = (U32)(32768 * matchProba);
|
||||
@ -141,16 +141,18 @@ static void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, doubl
|
||||
|
||||
void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed)
|
||||
{
|
||||
U32 seed32 = seed;
|
||||
BYTE ldt[LTSIZE];
|
||||
memset(ldt, '0', sizeof(ldt)); /* yes, character '0', this is intentional */
|
||||
if (litProba<=0.0) litProba = matchProba / 4.5;
|
||||
RDG_fillLiteralDistrib(ldt, litProba);
|
||||
RDG_genBlock(buffer, size, 0, matchProba, ldt, &seed);
|
||||
RDG_genBlock(buffer, size, 0, matchProba, ldt, &seed32);
|
||||
}
|
||||
|
||||
|
||||
void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed)
|
||||
{
|
||||
U32 seed32 = seed;
|
||||
size_t const stdBlockSize = 128 KB;
|
||||
size_t const stdDictSize = 32 KB;
|
||||
BYTE* const buff = (BYTE*)malloc(stdDictSize + stdBlockSize);
|
||||
@ -165,12 +167,12 @@ void RDG_genStdout(unsigned long long size, double matchProba, double litProba,
|
||||
SET_BINARY_MODE(stdout);
|
||||
|
||||
/* Generate initial dict */
|
||||
RDG_genBlock(buff, stdDictSize, 0, matchProba, ldt, &seed);
|
||||
RDG_genBlock(buff, stdDictSize, 0, matchProba, ldt, &seed32);
|
||||
|
||||
/* Generate compressible data */
|
||||
while (total < size) {
|
||||
size_t const genBlockSize = (size_t) (MIN (stdBlockSize, size-total));
|
||||
RDG_genBlock(buff, stdDictSize+stdBlockSize, stdDictSize, matchProba, ldt, &seed);
|
||||
RDG_genBlock(buff, stdDictSize+stdBlockSize, stdDictSize, matchProba, ldt, &seed32);
|
||||
total += genBlockSize;
|
||||
{ size_t const unused = fwrite(buff, 1, genBlockSize, stdout); (void)unused; }
|
||||
/* update dict */
|
||||
|
||||
@ -139,7 +139,7 @@ static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr,
|
||||
}
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
*bufferSizePtr = pos;
|
||||
DISPLAYLEVEL(4, "loaded : %u KB \n", (U32)(pos >> 10))
|
||||
DISPLAYLEVEL(4, "loaded : %u KB \n", (unsigned)(pos >> 10))
|
||||
return nbLoadedChunks;
|
||||
}
|
||||
|
||||
@ -249,7 +249,7 @@ static fileStats DiB_fileStats(const char** fileNamesTable, unsigned nbFiles, si
|
||||
fs.oneSampleTooLarge |= (chunkSize > 2*SAMPLESIZE_MAX);
|
||||
fs.nbSamples += nbSamples;
|
||||
}
|
||||
DISPLAYLEVEL(4, "Preparing to load : %u KB \n", (U32)(fs.totalSizeToLoad >> 10));
|
||||
DISPLAYLEVEL(4, "Preparing to load : %u KB \n", (unsigned)(fs.totalSizeToLoad >> 10));
|
||||
return fs;
|
||||
}
|
||||
|
||||
@ -358,7 +358,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
||||
goto _cleanup;
|
||||
}
|
||||
/* save dict */
|
||||
DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName);
|
||||
DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (unsigned)dictSize, dictFileName);
|
||||
DiB_saveDict(dictFileName, dictBuffer, dictSize);
|
||||
}
|
||||
|
||||
|
||||
@ -88,10 +88,10 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
|
||||
static const U64 g_refreshRate = SEC_TO_MICRO / 6;
|
||||
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
|
||||
|
||||
#define READY_FOR_UPDATE() (UTIL_clockSpanMicro(g_displayClock) > g_refreshRate)
|
||||
#define READY_FOR_UPDATE() (!g_noProgress && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate)
|
||||
#define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); }
|
||||
#define DISPLAYUPDATE(l, ...) { \
|
||||
if (g_displayLevel>=l) { \
|
||||
if (g_displayLevel>=l && !g_noProgress) { \
|
||||
if (READY_FOR_UPDATE() || (g_displayLevel>=4)) { \
|
||||
DELAY_NEXT_UPDATE(); \
|
||||
DISPLAY(__VA_ARGS__); \
|
||||
@ -270,18 +270,18 @@ void FIO_addAbortHandler()
|
||||
static FIO_compressionType_t g_compressionType = FIO_zstdCompression;
|
||||
void FIO_setCompressionType(FIO_compressionType_t compressionType) { g_compressionType = compressionType; }
|
||||
static U32 g_overwrite = 0;
|
||||
void FIO_overwriteMode(void) { g_overwrite=1; }
|
||||
static U32 g_sparseFileSupport = 1; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
|
||||
void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
|
||||
void FIO_overwriteMode(void) { g_overwrite = 1; }
|
||||
static U32 g_sparseFileSupport = ZSTD_SPARSE_DEFAULT; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
|
||||
void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport = sparse; }
|
||||
static U32 g_dictIDFlag = 1;
|
||||
void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; }
|
||||
static U32 g_checksumFlag = 1;
|
||||
void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; }
|
||||
static U32 g_removeSrcFile = 0;
|
||||
void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); }
|
||||
static U32 g_memLimit = 0;
|
||||
static unsigned g_memLimit = 0;
|
||||
void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; }
|
||||
static U32 g_nbWorkers = 1;
|
||||
static unsigned g_nbWorkers = 1;
|
||||
void FIO_setNbWorkers(unsigned nbWorkers) {
|
||||
#ifndef ZSTD_MULTITHREAD
|
||||
if (nbWorkers > 0) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n");
|
||||
@ -295,7 +295,7 @@ void FIO_setBlockSize(unsigned blockSize) {
|
||||
g_blockSize = blockSize;
|
||||
}
|
||||
#define FIO_OVERLAP_LOG_NOTSET 9999
|
||||
static U32 g_overlapLog = FIO_OVERLAP_LOG_NOTSET;
|
||||
static unsigned g_overlapLog = FIO_OVERLAP_LOG_NOTSET;
|
||||
void FIO_setOverlapLog(unsigned overlapLog){
|
||||
if (overlapLog && g_nbWorkers==0)
|
||||
DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n");
|
||||
@ -307,6 +307,12 @@ void FIO_setAdaptiveMode(unsigned adapt) {
|
||||
EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n");
|
||||
g_adaptiveMode = adapt;
|
||||
}
|
||||
static U32 g_rsyncable = 0;
|
||||
void FIO_setRsyncable(unsigned rsyncable) {
|
||||
if ((rsyncable>0) && (g_nbWorkers==0))
|
||||
EXM_THROW(1, "Rsyncable mode is not compatible with single thread mode \n");
|
||||
g_rsyncable = rsyncable;
|
||||
}
|
||||
static int g_minAdaptLevel = -50; /* initializing this value requires a constant, so ZSTD_minCLevel() doesn't work */
|
||||
void FIO_setAdaptMin(int minCLevel)
|
||||
{
|
||||
@ -340,9 +346,13 @@ void FIO_setLdmBucketSizeLog(unsigned ldmBucketSizeLog) {
|
||||
g_ldmBucketSizeLog = ldmBucketSizeLog;
|
||||
}
|
||||
|
||||
static U32 g_ldmHashEveryLog = FIO_LDM_PARAM_NOTSET;
|
||||
void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog) {
|
||||
g_ldmHashEveryLog = ldmHashEveryLog;
|
||||
static U32 g_ldmHashRateLog = FIO_LDM_PARAM_NOTSET;
|
||||
void FIO_setLdmHashRateLog(unsigned ldmHashRateLog) {
|
||||
g_ldmHashRateLog = ldmHashRateLog;
|
||||
}
|
||||
static U32 g_noProgress = 0;
|
||||
void FIO_setNoProgress(unsigned noProgress) {
|
||||
g_noProgress = noProgress;
|
||||
}
|
||||
|
||||
|
||||
@ -355,7 +365,7 @@ void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog) {
|
||||
static int FIO_remove(const char* path)
|
||||
{
|
||||
if (!UTIL_isRegularFile(path)) {
|
||||
DISPLAYLEVEL(2, "zstd: Refusing to remove non-regular file %s\n", path);
|
||||
DISPLAYLEVEL(2, "zstd: Refusing to remove non-regular file %s \n", path);
|
||||
return 0;
|
||||
}
|
||||
#if defined(_WIN32) || defined(WIN32)
|
||||
@ -373,11 +383,17 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
|
||||
{
|
||||
assert(srcFileName != NULL);
|
||||
if (!strcmp (srcFileName, stdinmark)) {
|
||||
DISPLAYLEVEL(4,"Using stdin for input\n");
|
||||
DISPLAYLEVEL(4,"Using stdin for input \n");
|
||||
SET_BINARY_MODE(stdin);
|
||||
return stdin;
|
||||
}
|
||||
|
||||
if (!UTIL_fileExist(srcFileName)) {
|
||||
DISPLAYLEVEL(1, "zstd: can't stat %s : %s -- ignored \n",
|
||||
srcFileName, strerror(errno));
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (!UTIL_isRegularFile(srcFileName)) {
|
||||
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
|
||||
srcFileName);
|
||||
@ -394,30 +410,54 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
|
||||
/** FIO_openDstFile() :
|
||||
* condition : `dstFileName` must be non-NULL.
|
||||
* @result : FILE* to `dstFileName`, or NULL if it fails */
|
||||
static FILE* FIO_openDstFile(const char* dstFileName)
|
||||
static FILE* FIO_openDstFile(const char* srcFileName, const char* dstFileName)
|
||||
{
|
||||
assert(dstFileName != NULL);
|
||||
if (!strcmp (dstFileName, stdoutmark)) {
|
||||
DISPLAYLEVEL(4,"Using stdout for output\n");
|
||||
DISPLAYLEVEL(4,"Using stdout for output \n");
|
||||
SET_BINARY_MODE(stdout);
|
||||
if (g_sparseFileSupport==1) {
|
||||
if (g_sparseFileSupport == 1) {
|
||||
g_sparseFileSupport = 0;
|
||||
DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
|
||||
}
|
||||
return stdout;
|
||||
}
|
||||
|
||||
/* ensure dst is not the same file as src */
|
||||
if (srcFileName != NULL) {
|
||||
#ifdef _MSC_VER
|
||||
/* note : Visual does not support file identification by inode.
|
||||
* The following work-around is limited to detecting exact name repetition only,
|
||||
* aka `filename` is considered different from `subdir/../filename` */
|
||||
if (!strcmp(srcFileName, dstFileName)) {
|
||||
DISPLAYLEVEL(1, "zstd: Refusing to open a output file which will overwrite the input file \n");
|
||||
return NULL;
|
||||
}
|
||||
#else
|
||||
stat_t srcStat;
|
||||
stat_t dstStat;
|
||||
if (UTIL_getFileStat(srcFileName, &srcStat)
|
||||
&& UTIL_getFileStat(dstFileName, &dstStat)) {
|
||||
if (srcStat.st_dev == dstStat.st_dev
|
||||
&& srcStat.st_ino == dstStat.st_ino) {
|
||||
DISPLAYLEVEL(1, "zstd: Refusing to open a output file which will overwrite the input file \n");
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
if (g_sparseFileSupport == 1) {
|
||||
g_sparseFileSupport = ZSTD_SPARSE_DEFAULT;
|
||||
}
|
||||
|
||||
if (UTIL_isRegularFile(dstFileName)) {
|
||||
FILE* fCheck;
|
||||
if (!strcmp(dstFileName, nulmark)) {
|
||||
EXM_THROW(40, "%s is unexpectedly a regular file", dstFileName);
|
||||
}
|
||||
/* Check if destination file already exists */
|
||||
fCheck = fopen( dstFileName, "rb" );
|
||||
FILE* const fCheck = fopen( dstFileName, "rb" );
|
||||
if (!strcmp(dstFileName, nulmark)) {
|
||||
EXM_THROW(40, "%s is unexpectedly categorized as a regular file",
|
||||
dstFileName);
|
||||
}
|
||||
if (fCheck != NULL) { /* dst file exists, authorization prompt */
|
||||
fclose(fCheck);
|
||||
if (!g_overwrite) {
|
||||
@ -467,6 +507,7 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
|
||||
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
|
||||
fileHandle = fopen(fileName, "rb");
|
||||
if (fileHandle==NULL) EXM_THROW(31, "%s: %s", fileName, strerror(errno));
|
||||
|
||||
fileSize = UTIL_getFileSize(fileName);
|
||||
if (fileSize > DICTSIZE_MAX) {
|
||||
EXM_THROW(32, "Dictionary file %s is too large (> %u MB)",
|
||||
@ -475,8 +516,9 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
|
||||
*bufferPtr = malloc((size_t)fileSize);
|
||||
if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno));
|
||||
{ size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
|
||||
if (readSize!=fileSize)
|
||||
EXM_THROW(35, "Error reading dictionary file %s", fileName);
|
||||
if (readSize != fileSize)
|
||||
EXM_THROW(35, "Error reading dictionary file %s : %s",
|
||||
fileName, strerror(errno));
|
||||
}
|
||||
fclose(fileHandle);
|
||||
return (size_t)fileSize;
|
||||
@ -506,7 +548,8 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
|
||||
DISPLAYLEVEL(6, "FIO_createCResources \n");
|
||||
ress.cctx = ZSTD_createCCtx();
|
||||
if (ress.cctx == NULL)
|
||||
EXM_THROW(30, "allocation error : can't create ZSTD_CCtx");
|
||||
EXM_THROW(30, "allocation error (%s): can't create ZSTD_CCtx",
|
||||
strerror(errno));
|
||||
ress.srcBufferSize = ZSTD_CStreamInSize();
|
||||
ress.srcBuffer = malloc(ress.srcBufferSize);
|
||||
ress.dstBufferSize = ZSTD_CStreamOutSize();
|
||||
@ -523,40 +566,39 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
|
||||
if (g_adaptiveMode && !g_ldmFlag && !comprParams.windowLog)
|
||||
comprParams.windowLog = ADAPT_WINDOWLOG_DEFAULT;
|
||||
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) ); /* always enable content size when available (note: supposed to be default) */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_contentSizeFlag, 1) ); /* always enable content size when available (note: supposed to be default) */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_dictIDFlag, g_dictIDFlag) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, g_checksumFlag) );
|
||||
/* compression level */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)cLevel) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) );
|
||||
/* long distance matching */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashLog, g_ldmHashLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableLongDistanceMatching, g_ldmFlag) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashLog, g_ldmHashLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmMinMatch, g_ldmMinMatch) );
|
||||
if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) {
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmBucketSizeLog, g_ldmBucketSizeLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmBucketSizeLog, g_ldmBucketSizeLog) );
|
||||
}
|
||||
if (g_ldmHashEveryLog != FIO_LDM_PARAM_NOTSET) {
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog) );
|
||||
if (g_ldmHashRateLog != FIO_LDM_PARAM_NOTSET) {
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, g_ldmHashRateLog) );
|
||||
}
|
||||
/* compression parameters */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams.windowLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams.chainLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams.hashLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams.searchLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams.searchLength) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams.targetLength) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams.strategy) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, comprParams.windowLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_chainLog, comprParams.chainLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_hashLog, comprParams.hashLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, comprParams.searchLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, comprParams.minMatch) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, comprParams.targetLength) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, comprParams.strategy) );
|
||||
/* multi-threading */
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
DISPLAYLEVEL(5,"set nb workers = %u \n", g_nbWorkers);
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbWorkers, g_nbWorkers) );
|
||||
if ( (g_overlapLog == FIO_OVERLAP_LOG_NOTSET)
|
||||
&& (cLevel == ZSTD_maxCLevel()) )
|
||||
g_overlapLog = 9; /* full overlap */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, g_nbWorkers) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, g_blockSize) );
|
||||
if (g_overlapLog != FIO_OVERLAP_LOG_NOTSET) {
|
||||
DISPLAYLEVEL(3,"set overlapLog = %u \n", g_overlapLog);
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_overlapSizeLog, g_overlapLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, g_overlapLog) );
|
||||
}
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, g_rsyncable) );
|
||||
#endif
|
||||
/* dictionary */
|
||||
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* set the value temporarily for dictionary loading, to adapt compression parameters */
|
||||
@ -627,11 +669,11 @@ FIO_compressGzFrame(cRess_t* ress,
|
||||
}
|
||||
if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
|
||||
(U32)(inFileSize>>20),
|
||||
(unsigned)(inFileSize>>20),
|
||||
(double)outFileSize/inFileSize*100)
|
||||
else
|
||||
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
|
||||
(U32)(inFileSize>>20), (U32)(srcFileSize>>20),
|
||||
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
|
||||
(double)outFileSize/inFileSize*100);
|
||||
}
|
||||
|
||||
@ -640,7 +682,7 @@ FIO_compressGzFrame(cRess_t* ress,
|
||||
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
|
||||
if (decompBytes) {
|
||||
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes)
|
||||
EXM_THROW(75, "Write error : cannot write to output file");
|
||||
EXM_THROW(75, "Write error : %s", strerror(errno));
|
||||
outFileSize += decompBytes;
|
||||
strm.next_out = (Bytef*)ress->dstBuffer;
|
||||
strm.avail_out = (uInt)ress->dstBufferSize;
|
||||
@ -708,18 +750,18 @@ FIO_compressLzmaFrame(cRess_t* ress,
|
||||
{ size_t const compBytes = ress->dstBufferSize - strm.avail_out;
|
||||
if (compBytes) {
|
||||
if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes)
|
||||
EXM_THROW(73, "Write error : cannot write to output file");
|
||||
EXM_THROW(73, "Write error : %s", strerror(errno));
|
||||
outFileSize += compBytes;
|
||||
strm.next_out = (BYTE*)ress->dstBuffer;
|
||||
strm.avail_out = ress->dstBufferSize;
|
||||
} }
|
||||
if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
|
||||
(U32)(inFileSize>>20),
|
||||
(unsigned)(inFileSize>>20),
|
||||
(double)outFileSize/inFileSize*100)
|
||||
else
|
||||
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
|
||||
(U32)(inFileSize>>20), (U32)(srcFileSize>>20),
|
||||
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
|
||||
(double)outFileSize/inFileSize*100);
|
||||
if (ret == LZMA_STREAM_END) break;
|
||||
}
|
||||
@ -773,7 +815,7 @@ FIO_compressLz4Frame(cRess_t* ress,
|
||||
EXM_THROW(33, "File header generation failed : %s",
|
||||
LZ4F_getErrorName(headerSize));
|
||||
if (fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile) != headerSize)
|
||||
EXM_THROW(34, "Write error : cannot write header");
|
||||
EXM_THROW(34, "Write error : %s (cannot write header)", strerror(errno));
|
||||
outFileSize += headerSize;
|
||||
|
||||
/* Read first block */
|
||||
@ -782,26 +824,28 @@ FIO_compressLz4Frame(cRess_t* ress,
|
||||
|
||||
/* Main Loop */
|
||||
while (readSize>0) {
|
||||
size_t outSize;
|
||||
|
||||
/* Compress Block */
|
||||
outSize = LZ4F_compressUpdate(ctx, ress->dstBuffer, ress->dstBufferSize, ress->srcBuffer, readSize, NULL);
|
||||
size_t const outSize = LZ4F_compressUpdate(ctx,
|
||||
ress->dstBuffer, ress->dstBufferSize,
|
||||
ress->srcBuffer, readSize, NULL);
|
||||
if (LZ4F_isError(outSize))
|
||||
EXM_THROW(35, "zstd: %s: lz4 compression failed : %s",
|
||||
srcFileName, LZ4F_getErrorName(outSize));
|
||||
outFileSize += outSize;
|
||||
if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
|
||||
if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
|
||||
(U32)(inFileSize>>20),
|
||||
(unsigned)(inFileSize>>20),
|
||||
(double)outFileSize/inFileSize*100)
|
||||
else
|
||||
} else {
|
||||
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
|
||||
(U32)(inFileSize>>20), (U32)(srcFileSize>>20),
|
||||
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
|
||||
(double)outFileSize/inFileSize*100);
|
||||
}
|
||||
|
||||
/* Write Block */
|
||||
{ size_t const sizeCheck = fwrite(ress->dstBuffer, 1, outSize, ress->dstFile);
|
||||
if (sizeCheck!=outSize) EXM_THROW(36, "Write error : cannot write compressed block"); }
|
||||
{ size_t const sizeCheck = fwrite(ress->dstBuffer, 1, outSize, ress->dstFile);
|
||||
if (sizeCheck != outSize)
|
||||
EXM_THROW(36, "Write error : %s", strerror(errno));
|
||||
}
|
||||
|
||||
/* Read next block */
|
||||
readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile);
|
||||
@ -815,8 +859,11 @@ FIO_compressLz4Frame(cRess_t* ress,
|
||||
EXM_THROW(38, "zstd: %s: lz4 end of file generation failed : %s",
|
||||
srcFileName, LZ4F_getErrorName(headerSize));
|
||||
|
||||
{ size_t const sizeCheck = fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile);
|
||||
if (sizeCheck!=headerSize) EXM_THROW(39, "Write error : cannot write end of stream"); }
|
||||
{ size_t const sizeCheck = fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile);
|
||||
if (sizeCheck != headerSize)
|
||||
EXM_THROW(39, "Write error : %s (cannot write end of stream)",
|
||||
strerror(errno));
|
||||
}
|
||||
outFileSize += headerSize;
|
||||
}
|
||||
|
||||
@ -863,7 +910,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
|
||||
/* Fill input Buffer */
|
||||
size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile);
|
||||
ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 };
|
||||
DISPLAYLEVEL(6, "fread %u bytes from source \n", (U32)inSize);
|
||||
DISPLAYLEVEL(6, "fread %u bytes from source \n", (unsigned)inSize);
|
||||
*readsize += inSize;
|
||||
|
||||
if ((inSize == 0) || (*readsize == fileSize))
|
||||
@ -876,7 +923,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
|
||||
size_t const oldIPos = inBuff.pos;
|
||||
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
|
||||
size_t const toFlushNow = ZSTD_toFlushNow(ress.cctx);
|
||||
CHECK_V(stillToFlush, ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, directive));
|
||||
CHECK_V(stillToFlush, ZSTD_compressStream2(ress.cctx, &outBuff, &inBuff, directive));
|
||||
|
||||
/* count stats */
|
||||
inputPresented++;
|
||||
@ -885,11 +932,12 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
|
||||
|
||||
/* Write compressed stream */
|
||||
DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => input pos(%u)<=(%u)size ; output generated %u bytes \n",
|
||||
(U32)directive, (U32)inBuff.pos, (U32)inBuff.size, (U32)outBuff.pos);
|
||||
(unsigned)directive, (unsigned)inBuff.pos, (unsigned)inBuff.size, (unsigned)outBuff.pos);
|
||||
if (outBuff.pos) {
|
||||
size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
|
||||
if (sizeCheck != outBuff.pos)
|
||||
EXM_THROW(25, "Write error : cannot write compressed block");
|
||||
EXM_THROW(25, "Write error : %s (cannot write compressed block)",
|
||||
strerror(errno));
|
||||
compressedfilesize += outBuff.pos;
|
||||
}
|
||||
|
||||
@ -902,14 +950,14 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
|
||||
if (g_displayLevel >= 3) {
|
||||
DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%% ",
|
||||
compressionLevel,
|
||||
(U32)((zfp.ingested - zfp.consumed) >> 20),
|
||||
(U32)(zfp.consumed >> 20),
|
||||
(U32)(zfp.produced >> 20),
|
||||
(unsigned)((zfp.ingested - zfp.consumed) >> 20),
|
||||
(unsigned)(zfp.consumed >> 20),
|
||||
(unsigned)(zfp.produced >> 20),
|
||||
cShare );
|
||||
} else { /* summarized notifications if == 2; */
|
||||
DISPLAYLEVEL(2, "\rRead : %u ", (U32)(zfp.consumed >> 20));
|
||||
DISPLAYLEVEL(2, "\rRead : %u ", (unsigned)(zfp.consumed >> 20));
|
||||
if (fileSize != UTIL_FILESIZE_UNKNOWN)
|
||||
DISPLAYLEVEL(2, "/ %u ", (U32)(fileSize >> 20));
|
||||
DISPLAYLEVEL(2, "/ %u ", (unsigned)(fileSize >> 20));
|
||||
DISPLAYLEVEL(2, "MB ==> %2.f%% ", cShare);
|
||||
DELAY_NEXT_UPDATE();
|
||||
}
|
||||
@ -965,8 +1013,8 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
|
||||
assert(inputPresented > 0);
|
||||
DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n",
|
||||
inputBlocked, inputPresented, (double)inputBlocked/inputPresented*100,
|
||||
(U32)newlyIngested, (U32)newlyConsumed,
|
||||
(U32)newlyFlushed, (U32)newlyProduced);
|
||||
(unsigned)newlyIngested, (unsigned)newlyConsumed,
|
||||
(unsigned)newlyFlushed, (unsigned)newlyProduced);
|
||||
if ( (inputBlocked > inputPresented / 8) /* input is waiting often, because input buffers is full : compression or output too slow */
|
||||
&& (newlyFlushed * 33 / 32 > newlyProduced) /* flush everything that is produced */
|
||||
&& (newlyIngested * 33 / 32 > newlyConsumed) /* input speed as fast or faster than compression speed */
|
||||
@ -986,14 +1034,14 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
|
||||
if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
|
||||
if (compressionLevel > g_maxAdaptLevel) compressionLevel = g_maxAdaptLevel;
|
||||
compressionLevel += (compressionLevel == 0); /* skip 0 */
|
||||
ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel);
|
||||
ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
|
||||
}
|
||||
if (speedChange == faster) {
|
||||
DISPLAYLEVEL(6, "faster speed , lighter compression \n")
|
||||
compressionLevel --;
|
||||
if (compressionLevel < g_minAdaptLevel) compressionLevel = g_minAdaptLevel;
|
||||
compressionLevel -= (compressionLevel == 0); /* skip 0 */
|
||||
ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel);
|
||||
ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
|
||||
}
|
||||
speedChange = noChange;
|
||||
|
||||
@ -1028,7 +1076,7 @@ FIO_compressFilename_internal(cRess_t ress,
|
||||
U64 readsize = 0;
|
||||
U64 compressedfilesize = 0;
|
||||
U64 const fileSize = UTIL_getFileSize(srcFileName);
|
||||
DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (U32)fileSize);
|
||||
DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (unsigned)fileSize);
|
||||
|
||||
/* compression format selection */
|
||||
switch (g_compressionType) {
|
||||
@ -1105,7 +1153,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
|
||||
if (ress.dstFile == NULL) {
|
||||
closeDstFile = 1;
|
||||
DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName);
|
||||
ress.dstFile = FIO_openDstFile(dstFileName);
|
||||
ress.dstFile = FIO_openDstFile(srcFileName, dstFileName);
|
||||
if (ress.dstFile==NULL) return 1; /* could not open dstFileName */
|
||||
/* Must only be added after FIO_openDstFile() succeeds.
|
||||
* Otherwise we may delete the destination file if it already exists,
|
||||
@ -1255,7 +1303,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
|
||||
assert(outFileName != NULL || suffix != NULL);
|
||||
|
||||
if (outFileName != NULL) { /* output into a single destination (stdout typically) */
|
||||
ress.dstFile = FIO_openDstFile(outFileName);
|
||||
ress.dstFile = FIO_openDstFile(NULL, outFileName);
|
||||
if (ress.dstFile == NULL) { /* could not open outFileName */
|
||||
error = 1;
|
||||
} else {
|
||||
@ -1263,7 +1311,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
|
||||
for (u=0; u<nbFiles; u++)
|
||||
error |= FIO_compressFilename_srcFile(ress, outFileName, inFileNamesTable[u], compressionLevel);
|
||||
if (fclose(ress.dstFile))
|
||||
EXM_THROW(29, "Write error : cannot properly close %s", outFileName);
|
||||
EXM_THROW(29, "Write error (%s) : cannot properly close %s",
|
||||
strerror(errno), outFileName);
|
||||
ress.dstFile = NULL;
|
||||
}
|
||||
} else {
|
||||
@ -1304,8 +1353,9 @@ static dRess_t FIO_createDResources(const char* dictFileName)
|
||||
|
||||
/* Allocation */
|
||||
ress.dctx = ZSTD_createDStream();
|
||||
if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream");
|
||||
CHECK( ZSTD_setDStreamParameter(ress.dctx, DStream_p_maxWindowSize, g_memLimit) );
|
||||
if (ress.dctx==NULL)
|
||||
EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno));
|
||||
CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, g_memLimit) );
|
||||
ress.srcBufferSize = ZSTD_DStreamInSize();
|
||||
ress.srcBuffer = malloc(ress.srcBufferSize);
|
||||
ress.dstBufferSize = ZSTD_DStreamOutSize();
|
||||
@ -1343,14 +1393,17 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi
|
||||
|
||||
if (!g_sparseFileSupport) { /* normal write */
|
||||
size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
|
||||
if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block");
|
||||
if (sizeCheck != bufferSize)
|
||||
EXM_THROW(70, "Write error : %s (cannot write decoded block)",
|
||||
strerror(errno));
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* avoid int overflow */
|
||||
if (storedSkips > 1 GB) {
|
||||
int const seekResult = LONG_SEEK(file, 1 GB, SEEK_CUR);
|
||||
if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)");
|
||||
if (seekResult != 0)
|
||||
EXM_THROW(71, "1 GB skip error (sparse file support)");
|
||||
storedSkips -= 1 GB;
|
||||
}
|
||||
|
||||
@ -1401,12 +1454,14 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi
|
||||
|
||||
static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
|
||||
{
|
||||
if (storedSkips-->0) { /* implies g_sparseFileSupport>0 */
|
||||
int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
|
||||
if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)");
|
||||
if (storedSkips>0) {
|
||||
assert(g_sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */
|
||||
if (LONG_SEEK(file, storedSkips-1, SEEK_CUR) != 0)
|
||||
EXM_THROW(69, "Final skip error (sparse file support)");
|
||||
/* last zero must be explicitly written,
|
||||
* so that skipped ones get implicitly translated as zero by FS */
|
||||
{ const char lastZeroByte[1] = { 0 };
|
||||
size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file);
|
||||
if (sizeCheck != 1)
|
||||
if (fwrite(lastZeroByte, 1, 1, file) != 1)
|
||||
EXM_THROW(69, "Write error : cannot write last zero");
|
||||
} }
|
||||
}
|
||||
@ -1463,12 +1518,12 @@ static void FIO_zstdErrorHelp(dRess_t* ress, size_t err, char const* srcFileName
|
||||
err = ZSTD_getFrameHeader(&header, ress->srcBuffer, ress->srcBufferLoaded);
|
||||
if (err == 0) {
|
||||
unsigned long long const windowSize = header.windowSize;
|
||||
U32 const windowLog = FIO_highbit64(windowSize) + ((windowSize & (windowSize - 1)) != 0);
|
||||
unsigned const windowLog = FIO_highbit64(windowSize) + ((windowSize & (windowSize - 1)) != 0);
|
||||
assert(g_memLimit > 0);
|
||||
DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u\n",
|
||||
srcFileName, windowSize, g_memLimit);
|
||||
if (windowLog <= ZSTD_WINDOWLOG_MAX) {
|
||||
U32 const windowMB = (U32)((windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0));
|
||||
unsigned const windowMB = (unsigned)((windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0));
|
||||
assert(windowSize < (U64)(1ULL << 52)); /* ensure now overflow for windowMB */
|
||||
DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB\n",
|
||||
srcFileName, windowLog, windowMB);
|
||||
@ -1520,7 +1575,7 @@ static unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
|
||||
storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips);
|
||||
frameSize += outBuff.pos;
|
||||
DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ",
|
||||
srcFileName, (U32)((alreadyDecoded+frameSize)>>20) );
|
||||
srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) );
|
||||
|
||||
if (inBuff.pos > 0) {
|
||||
memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos);
|
||||
@ -1871,7 +1926,7 @@ static int FIO_decompressDstFile(dRess_t ress, FILE* srcFile,
|
||||
if (ress.dstFile == NULL) {
|
||||
releaseDstFile = 1;
|
||||
|
||||
ress.dstFile = FIO_openDstFile(dstFileName);
|
||||
ress.dstFile = FIO_openDstFile(srcFileName, dstFileName);
|
||||
if (ress.dstFile==0) return 1;
|
||||
|
||||
/* Must only be added after FIO_openDstFile() succeeds.
|
||||
@ -2025,7 +2080,7 @@ FIO_determineDstName(const char* srcFileName)
|
||||
dfnbCapacity = sfnSize + 20;
|
||||
dstFileNameBuffer = (char*)malloc(dfnbCapacity);
|
||||
if (dstFileNameBuffer==NULL)
|
||||
EXM_THROW(74, "not enough memory for dstFileName");
|
||||
EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno));
|
||||
}
|
||||
|
||||
/* return dst name == src name truncated from suffix */
|
||||
@ -2048,12 +2103,13 @@ FIO_decompressMultipleFilenames(const char* srcNamesTable[], unsigned nbFiles,
|
||||
|
||||
if (outFileName) {
|
||||
unsigned u;
|
||||
ress.dstFile = FIO_openDstFile(outFileName);
|
||||
ress.dstFile = FIO_openDstFile(NULL, outFileName);
|
||||
if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName);
|
||||
for (u=0; u<nbFiles; u++)
|
||||
error |= FIO_decompressSrcFile(ress, outFileName, srcNamesTable[u]);
|
||||
if (fclose(ress.dstFile))
|
||||
EXM_THROW(72, "Write error : cannot properly close output file");
|
||||
EXM_THROW(72, "Write error : %s : cannot properly close output file",
|
||||
strerror(errno));
|
||||
} else {
|
||||
unsigned u;
|
||||
for (u=0; u<nbFiles; u++) { /* create dstFileName */
|
||||
@ -2103,7 +2159,7 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
|
||||
for ( ; ; ) {
|
||||
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
|
||||
size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
|
||||
if (numBytesRead < ZSTD_frameHeaderSize_min) {
|
||||
if (numBytesRead < ZSTD_FRAMEHEADERSIZE_MIN) {
|
||||
if ( feof(srcFile)
|
||||
&& (numBytesRead == 0)
|
||||
&& (info->compressedSize > 0)
|
||||
@ -2164,7 +2220,7 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
|
||||
info->numActualFrames++;
|
||||
}
|
||||
/* Skippable frame */
|
||||
else if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
|
||||
else if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
|
||||
U32 const frameSize = MEM_readLE32(headerBuffer + 4);
|
||||
long const seek = (long)(8 + frameSize - numBytesRead);
|
||||
ERROR_IF(LONG_SEEK(srcFile, seek, SEEK_CUR) != 0,
|
||||
@ -2292,7 +2348,7 @@ FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel)
|
||||
}
|
||||
displayInfo(inFileName, &info, displayLevel);
|
||||
*total = FIO_addFInfo(*total, info);
|
||||
assert(error>=0 || error<=1);
|
||||
assert(error == info_success || error == info_frame_error);
|
||||
return error;
|
||||
}
|
||||
}
|
||||
@ -2339,13 +2395,13 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
|
||||
total.numSkippableFrames + total.numActualFrames,
|
||||
total.numSkippableFrames,
|
||||
compressedSizeUnit, unitStr,
|
||||
checkString, total.nbFiles);
|
||||
checkString, (unsigned)total.nbFiles);
|
||||
} else {
|
||||
DISPLAYOUT("%6d %5d %7.2f %2s %9.2f %2s %5.3f %5s %u files\n",
|
||||
total.numSkippableFrames + total.numActualFrames,
|
||||
total.numSkippableFrames,
|
||||
compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
|
||||
ratio, checkString, total.nbFiles);
|
||||
ratio, checkString, (unsigned)total.nbFiles);
|
||||
} }
|
||||
return error;
|
||||
}
|
||||
|
||||
@ -56,7 +56,7 @@ void FIO_setChecksumFlag(unsigned checksumFlag);
|
||||
void FIO_setDictIDFlag(unsigned dictIDFlag);
|
||||
void FIO_setLdmBucketSizeLog(unsigned ldmBucketSizeLog);
|
||||
void FIO_setLdmFlag(unsigned ldmFlag);
|
||||
void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog);
|
||||
void FIO_setLdmHashRateLog(unsigned ldmHashRateLog);
|
||||
void FIO_setLdmHashLog(unsigned ldmHashLog);
|
||||
void FIO_setLdmMinMatch(unsigned ldmMinMatch);
|
||||
void FIO_setMemLimit(unsigned memLimit);
|
||||
@ -65,6 +65,8 @@ void FIO_setNotificationLevel(unsigned level);
|
||||
void FIO_setOverlapLog(unsigned overlapLog);
|
||||
void FIO_setRemoveSrcFile(unsigned flag);
|
||||
void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
|
||||
void FIO_setRsyncable(unsigned rsyncable);
|
||||
void FIO_setNoProgress(unsigned noProgress);
|
||||
|
||||
|
||||
/*-*************************************
|
||||
|
||||
@ -26,6 +26,7 @@ extern "C" {
|
||||
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before <io.h> and <windows.h> */
|
||||
# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */
|
||||
# endif
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
673
programs/util.c
Normal file
673
programs/util.c
Normal file
@ -0,0 +1,673 @@
|
||||
/*
|
||||
* Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Dependencies
|
||||
******************************************/
|
||||
#include "util.h" /* note : ensure that platform.h is included first ! */
|
||||
#include <errno.h>
|
||||
#include <assert.h>
|
||||
|
||||
|
||||
int UTIL_fileExist(const char* filename)
|
||||
{
|
||||
stat_t statbuf;
|
||||
#if defined(_MSC_VER)
|
||||
int const stat_error = _stat64(filename, &statbuf);
|
||||
#else
|
||||
int const stat_error = stat(filename, &statbuf);
|
||||
#endif
|
||||
return !stat_error;
|
||||
}
|
||||
|
||||
int UTIL_isRegularFile(const char* infilename)
|
||||
{
|
||||
stat_t statbuf;
|
||||
return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */
|
||||
}
|
||||
|
||||
int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
|
||||
{
|
||||
int r;
|
||||
#if defined(_MSC_VER)
|
||||
r = _stat64(infilename, statbuf);
|
||||
if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */
|
||||
#else
|
||||
r = stat(infilename, statbuf);
|
||||
if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */
|
||||
#endif
|
||||
return 1;
|
||||
}
|
||||
|
||||
int UTIL_setFileStat(const char *filename, stat_t *statbuf)
|
||||
{
|
||||
int res = 0;
|
||||
struct utimbuf timebuf;
|
||||
|
||||
if (!UTIL_isRegularFile(filename))
|
||||
return -1;
|
||||
|
||||
timebuf.actime = time(NULL);
|
||||
timebuf.modtime = statbuf->st_mtime;
|
||||
res += utime(filename, &timebuf); /* set access and modification times */
|
||||
|
||||
#if !defined(_WIN32)
|
||||
res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */
|
||||
#endif
|
||||
|
||||
res += chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */
|
||||
|
||||
errno = 0;
|
||||
return -res; /* number of errors is returned */
|
||||
}
|
||||
|
||||
U32 UTIL_isDirectory(const char* infilename)
|
||||
{
|
||||
int r;
|
||||
stat_t statbuf;
|
||||
#if defined(_MSC_VER)
|
||||
r = _stat64(infilename, &statbuf);
|
||||
if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
|
||||
#else
|
||||
r = stat(infilename, &statbuf);
|
||||
if (!r && S_ISDIR(statbuf.st_mode)) return 1;
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
U32 UTIL_isLink(const char* infilename)
|
||||
{
|
||||
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
|
||||
#ifndef __STRICT_ANSI__
|
||||
#if defined(_BSD_SOURCE) \
|
||||
|| (defined(_XOPEN_SOURCE) && (_XOPEN_SOURCE >= 500)) \
|
||||
|| (defined(_XOPEN_SOURCE) && defined(_XOPEN_SOURCE_EXTENDED)) \
|
||||
|| (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) \
|
||||
|| (defined(__APPLE__) && defined(__MACH__))
|
||||
int r;
|
||||
stat_t statbuf;
|
||||
r = lstat(infilename, &statbuf);
|
||||
if (!r && S_ISLNK(statbuf.st_mode)) return 1;
|
||||
#endif
|
||||
#endif
|
||||
(void)infilename;
|
||||
return 0;
|
||||
}
|
||||
|
||||
U64 UTIL_getFileSize(const char* infilename)
|
||||
{
|
||||
if (!UTIL_isRegularFile(infilename)) return UTIL_FILESIZE_UNKNOWN;
|
||||
{ int r;
|
||||
#if defined(_MSC_VER)
|
||||
struct __stat64 statbuf;
|
||||
r = _stat64(infilename, &statbuf);
|
||||
if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
|
||||
#elif defined(__MINGW32__) && defined (__MSVCRT__)
|
||||
struct _stati64 statbuf;
|
||||
r = _stati64(infilename, &statbuf);
|
||||
if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
|
||||
#else
|
||||
struct stat statbuf;
|
||||
r = stat(infilename, &statbuf);
|
||||
if (r || !S_ISREG(statbuf.st_mode)) return UTIL_FILESIZE_UNKNOWN;
|
||||
#endif
|
||||
return (U64)statbuf.st_size;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles)
|
||||
{
|
||||
U64 total = 0;
|
||||
int error = 0;
|
||||
unsigned n;
|
||||
for (n=0; n<nbFiles; n++) {
|
||||
U64 const size = UTIL_getFileSize(fileNamesTable[n]);
|
||||
error |= (size == UTIL_FILESIZE_UNKNOWN);
|
||||
total += size;
|
||||
}
|
||||
return error ? UTIL_FILESIZE_UNKNOWN : total;
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
|
||||
{
|
||||
char* path;
|
||||
int dirLength, fnameLength, pathLength, nbFiles = 0;
|
||||
WIN32_FIND_DATAA cFile;
|
||||
HANDLE hFile;
|
||||
|
||||
dirLength = (int)strlen(dirName);
|
||||
path = (char*) malloc(dirLength + 3);
|
||||
if (!path) return 0;
|
||||
|
||||
memcpy(path, dirName, dirLength);
|
||||
path[dirLength] = '\\';
|
||||
path[dirLength+1] = '*';
|
||||
path[dirLength+2] = 0;
|
||||
|
||||
hFile=FindFirstFileA(path, &cFile);
|
||||
if (hFile == INVALID_HANDLE_VALUE) {
|
||||
UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s'\n", dirName);
|
||||
return 0;
|
||||
}
|
||||
free(path);
|
||||
|
||||
do {
|
||||
fnameLength = (int)strlen(cFile.cFileName);
|
||||
path = (char*) malloc(dirLength + fnameLength + 2);
|
||||
if (!path) { FindClose(hFile); return 0; }
|
||||
memcpy(path, dirName, dirLength);
|
||||
path[dirLength] = '\\';
|
||||
memcpy(path+dirLength+1, cFile.cFileName, fnameLength);
|
||||
pathLength = dirLength+1+fnameLength;
|
||||
path[pathLength] = 0;
|
||||
if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
|
||||
if ( strcmp (cFile.cFileName, "..") == 0
|
||||
|| strcmp (cFile.cFileName, ".") == 0 )
|
||||
continue;
|
||||
/* Recursively call "UTIL_prepareFileList" with the new path. */
|
||||
nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks);
|
||||
if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
|
||||
} else if ( (cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL)
|
||||
|| (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE)
|
||||
|| (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED) ) {
|
||||
if (*bufStart + *pos + pathLength >= *bufEnd) {
|
||||
ptrdiff_t const newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
|
||||
*bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
|
||||
if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
|
||||
*bufEnd = *bufStart + newListSize;
|
||||
}
|
||||
if (*bufStart + *pos + pathLength < *bufEnd) {
|
||||
memcpy(*bufStart + *pos, path, pathLength+1 /* include final \0 */);
|
||||
*pos += pathLength + 1;
|
||||
nbFiles++;
|
||||
}
|
||||
}
|
||||
free(path);
|
||||
} while (FindNextFileA(hFile, &cFile));
|
||||
|
||||
FindClose(hFile);
|
||||
return nbFiles;
|
||||
}
|
||||
|
||||
#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
|
||||
|
||||
int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
|
||||
{
|
||||
DIR *dir;
|
||||
struct dirent *entry;
|
||||
char* path;
|
||||
int dirLength, fnameLength, pathLength, nbFiles = 0;
|
||||
|
||||
if (!(dir = opendir(dirName))) {
|
||||
UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
|
||||
return 0;
|
||||
}
|
||||
|
||||
dirLength = (int)strlen(dirName);
|
||||
errno = 0;
|
||||
while ((entry = readdir(dir)) != NULL) {
|
||||
if (strcmp (entry->d_name, "..") == 0 ||
|
||||
strcmp (entry->d_name, ".") == 0) continue;
|
||||
fnameLength = (int)strlen(entry->d_name);
|
||||
path = (char*) malloc(dirLength + fnameLength + 2);
|
||||
if (!path) { closedir(dir); return 0; }
|
||||
memcpy(path, dirName, dirLength);
|
||||
|
||||
path[dirLength] = '/';
|
||||
memcpy(path+dirLength+1, entry->d_name, fnameLength);
|
||||
pathLength = dirLength+1+fnameLength;
|
||||
path[pathLength] = 0;
|
||||
|
||||
if (!followLinks && UTIL_isLink(path)) {
|
||||
UTIL_DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", path);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (UTIL_isDirectory(path)) {
|
||||
nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */
|
||||
if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
|
||||
} else {
|
||||
if (*bufStart + *pos + pathLength >= *bufEnd) {
|
||||
ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
|
||||
*bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
|
||||
*bufEnd = *bufStart + newListSize;
|
||||
if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
|
||||
}
|
||||
if (*bufStart + *pos + pathLength < *bufEnd) {
|
||||
memcpy(*bufStart + *pos, path, pathLength + 1); /* with final \0 */
|
||||
*pos += pathLength + 1;
|
||||
nbFiles++;
|
||||
}
|
||||
}
|
||||
free(path);
|
||||
errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */
|
||||
}
|
||||
|
||||
if (errno != 0) {
|
||||
UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s\n", dirName, strerror(errno));
|
||||
free(*bufStart);
|
||||
*bufStart = NULL;
|
||||
}
|
||||
closedir(dir);
|
||||
return nbFiles;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
|
||||
{
|
||||
(void)bufStart; (void)bufEnd; (void)pos; (void)followLinks;
|
||||
UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif /* #ifdef _WIN32 */
|
||||
|
||||
/*
|
||||
* UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
|
||||
* and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb).
|
||||
* After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer)
|
||||
* In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called.
|
||||
*/
|
||||
const char**
|
||||
UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
|
||||
char** allocatedBuffer, unsigned* allocatedNamesNb,
|
||||
int followLinks)
|
||||
{
|
||||
size_t pos;
|
||||
unsigned i, nbFiles;
|
||||
char* buf = (char*)malloc(LIST_SIZE_INCREASE);
|
||||
char* bufend = buf + LIST_SIZE_INCREASE;
|
||||
const char** fileTable;
|
||||
|
||||
if (!buf) return NULL;
|
||||
|
||||
for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) {
|
||||
if (!UTIL_isDirectory(inputNames[i])) {
|
||||
size_t const len = strlen(inputNames[i]);
|
||||
if (buf + pos + len >= bufend) {
|
||||
ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
|
||||
buf = (char*)UTIL_realloc(buf, newListSize);
|
||||
bufend = buf + newListSize;
|
||||
if (!buf) return NULL;
|
||||
}
|
||||
if (buf + pos + len < bufend) {
|
||||
memcpy(buf+pos, inputNames[i], len+1); /* with final \0 */
|
||||
pos += len + 1;
|
||||
nbFiles++;
|
||||
}
|
||||
} else {
|
||||
nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks);
|
||||
if (buf == NULL) return NULL;
|
||||
} }
|
||||
|
||||
if (nbFiles == 0) { free(buf); return NULL; }
|
||||
|
||||
fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*));
|
||||
if (!fileTable) { free(buf); return NULL; }
|
||||
|
||||
for (i=0, pos=0; i<nbFiles; i++) {
|
||||
fileTable[i] = buf + pos;
|
||||
pos += strlen(fileTable[i]) + 1;
|
||||
}
|
||||
|
||||
if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
|
||||
|
||||
*allocatedBuffer = buf;
|
||||
*allocatedNamesNb = nbFiles;
|
||||
|
||||
return fileTable;
|
||||
}
|
||||
|
||||
/*-****************************************
|
||||
* Console log
|
||||
******************************************/
|
||||
int g_utilDisplayLevel;
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Time functions
|
||||
******************************************/
|
||||
#if defined(_WIN32) /* Windows */
|
||||
|
||||
UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
|
||||
|
||||
U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
|
||||
{
|
||||
static LARGE_INTEGER ticksPerSecond;
|
||||
static int init = 0;
|
||||
if (!init) {
|
||||
if (!QueryPerformanceFrequency(&ticksPerSecond))
|
||||
UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
|
||||
init = 1;
|
||||
}
|
||||
return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
|
||||
}
|
||||
|
||||
U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
|
||||
{
|
||||
static LARGE_INTEGER ticksPerSecond;
|
||||
static int init = 0;
|
||||
if (!init) {
|
||||
if (!QueryPerformanceFrequency(&ticksPerSecond))
|
||||
UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
|
||||
init = 1;
|
||||
}
|
||||
return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
|
||||
}
|
||||
|
||||
#elif defined(__APPLE__) && defined(__MACH__)
|
||||
|
||||
UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
|
||||
|
||||
U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
|
||||
{
|
||||
static mach_timebase_info_data_t rate;
|
||||
static int init = 0;
|
||||
if (!init) {
|
||||
mach_timebase_info(&rate);
|
||||
init = 1;
|
||||
}
|
||||
return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom))/1000ULL;
|
||||
}
|
||||
|
||||
U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
|
||||
{
|
||||
static mach_timebase_info_data_t rate;
|
||||
static int init = 0;
|
||||
if (!init) {
|
||||
mach_timebase_info(&rate);
|
||||
init = 1;
|
||||
}
|
||||
return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom);
|
||||
}
|
||||
|
||||
#elif (PLATFORM_POSIX_VERSION >= 200112L) \
|
||||
&& (defined(__UCLIBC__) \
|
||||
|| (defined(__GLIBC__) \
|
||||
&& ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) \
|
||||
|| (__GLIBC__ > 2))))
|
||||
|
||||
UTIL_time_t UTIL_getTime(void)
|
||||
{
|
||||
UTIL_time_t time;
|
||||
if (clock_gettime(CLOCK_MONOTONIC, &time))
|
||||
UTIL_DISPLAYLEVEL(1, "ERROR: Failed to get time\n"); /* we could also exit() */
|
||||
return time;
|
||||
}
|
||||
|
||||
UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
|
||||
{
|
||||
UTIL_time_t diff;
|
||||
if (end.tv_nsec < begin.tv_nsec) {
|
||||
diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec;
|
||||
diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec;
|
||||
} else {
|
||||
diff.tv_sec = end.tv_sec - begin.tv_sec;
|
||||
diff.tv_nsec = end.tv_nsec - begin.tv_nsec;
|
||||
}
|
||||
return diff;
|
||||
}
|
||||
|
||||
U64 UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
|
||||
{
|
||||
UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
|
||||
U64 micro = 0;
|
||||
micro += 1000000ULL * diff.tv_sec;
|
||||
micro += diff.tv_nsec / 1000ULL;
|
||||
return micro;
|
||||
}
|
||||
|
||||
U64 UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
|
||||
{
|
||||
UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
|
||||
U64 nano = 0;
|
||||
nano += 1000000000ULL * diff.tv_sec;
|
||||
nano += diff.tv_nsec;
|
||||
return nano;
|
||||
}
|
||||
|
||||
#else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
|
||||
|
||||
UTIL_time_t UTIL_getTime(void) { return clock(); }
|
||||
U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
|
||||
U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
|
||||
|
||||
#endif
|
||||
|
||||
/* returns time span in microseconds */
|
||||
U64 UTIL_clockSpanMicro(UTIL_time_t clockStart )
|
||||
{
|
||||
UTIL_time_t const clockEnd = UTIL_getTime();
|
||||
return UTIL_getSpanTimeMicro(clockStart, clockEnd);
|
||||
}
|
||||
|
||||
/* returns time span in microseconds */
|
||||
U64 UTIL_clockSpanNano(UTIL_time_t clockStart )
|
||||
{
|
||||
UTIL_time_t const clockEnd = UTIL_getTime();
|
||||
return UTIL_getSpanTimeNano(clockStart, clockEnd);
|
||||
}
|
||||
|
||||
void UTIL_waitForNextTick(void)
|
||||
{
|
||||
UTIL_time_t const clockStart = UTIL_getTime();
|
||||
UTIL_time_t clockEnd;
|
||||
do {
|
||||
clockEnd = UTIL_getTime();
|
||||
} while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0);
|
||||
}
|
||||
|
||||
/* count the number of physical cores */
|
||||
#if defined(_WIN32) || defined(WIN32)
|
||||
|
||||
#include <windows.h>
|
||||
|
||||
typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD);
|
||||
|
||||
int UTIL_countPhysicalCores(void)
|
||||
{
|
||||
static int numPhysicalCores = 0;
|
||||
if (numPhysicalCores != 0) return numPhysicalCores;
|
||||
|
||||
{ LPFN_GLPI glpi;
|
||||
BOOL done = FALSE;
|
||||
PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = NULL;
|
||||
PSYSTEM_LOGICAL_PROCESSOR_INFORMATION ptr = NULL;
|
||||
DWORD returnLength = 0;
|
||||
size_t byteOffset = 0;
|
||||
|
||||
glpi = (LPFN_GLPI)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
|
||||
"GetLogicalProcessorInformation");
|
||||
|
||||
if (glpi == NULL) {
|
||||
goto failed;
|
||||
}
|
||||
|
||||
while(!done) {
|
||||
DWORD rc = glpi(buffer, &returnLength);
|
||||
if (FALSE == rc) {
|
||||
if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
|
||||
if (buffer)
|
||||
free(buffer);
|
||||
buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(returnLength);
|
||||
|
||||
if (buffer == NULL) {
|
||||
perror("zstd");
|
||||
exit(1);
|
||||
}
|
||||
} else {
|
||||
/* some other error */
|
||||
goto failed;
|
||||
}
|
||||
} else {
|
||||
done = TRUE;
|
||||
}
|
||||
}
|
||||
|
||||
ptr = buffer;
|
||||
|
||||
while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) {
|
||||
|
||||
if (ptr->Relationship == RelationProcessorCore) {
|
||||
numPhysicalCores++;
|
||||
}
|
||||
|
||||
ptr++;
|
||||
byteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
|
||||
}
|
||||
|
||||
free(buffer);
|
||||
|
||||
return numPhysicalCores;
|
||||
}
|
||||
|
||||
failed:
|
||||
/* try to fall back on GetSystemInfo */
|
||||
{ SYSTEM_INFO sysinfo;
|
||||
GetSystemInfo(&sysinfo);
|
||||
numPhysicalCores = sysinfo.dwNumberOfProcessors;
|
||||
if (numPhysicalCores == 0) numPhysicalCores = 1; /* just in case */
|
||||
}
|
||||
return numPhysicalCores;
|
||||
}
|
||||
|
||||
#elif defined(__APPLE__)
|
||||
|
||||
#include <sys/sysctl.h>
|
||||
|
||||
/* Use apple-provided syscall
|
||||
* see: man 3 sysctl */
|
||||
int UTIL_countPhysicalCores(void)
|
||||
{
|
||||
static S32 numPhysicalCores = 0; /* apple specifies int32_t */
|
||||
if (numPhysicalCores != 0) return numPhysicalCores;
|
||||
|
||||
{ size_t size = sizeof(S32);
|
||||
int const ret = sysctlbyname("hw.physicalcpu", &numPhysicalCores, &size, NULL, 0);
|
||||
if (ret != 0) {
|
||||
if (errno == ENOENT) {
|
||||
/* entry not present, fall back on 1 */
|
||||
numPhysicalCores = 1;
|
||||
} else {
|
||||
perror("zstd: can't get number of physical cpus");
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
return numPhysicalCores;
|
||||
}
|
||||
}
|
||||
|
||||
#elif defined(__linux__)
|
||||
|
||||
/* parse /proc/cpuinfo
|
||||
* siblings / cpu cores should give hyperthreading ratio
|
||||
* otherwise fall back on sysconf */
|
||||
int UTIL_countPhysicalCores(void)
|
||||
{
|
||||
static int numPhysicalCores = 0;
|
||||
|
||||
if (numPhysicalCores != 0) return numPhysicalCores;
|
||||
|
||||
numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
|
||||
if (numPhysicalCores == -1) {
|
||||
/* value not queryable, fall back on 1 */
|
||||
return numPhysicalCores = 1;
|
||||
}
|
||||
|
||||
/* try to determine if there's hyperthreading */
|
||||
{ FILE* const cpuinfo = fopen("/proc/cpuinfo", "r");
|
||||
#define BUF_SIZE 80
|
||||
char buff[BUF_SIZE];
|
||||
|
||||
int siblings = 0;
|
||||
int cpu_cores = 0;
|
||||
int ratio = 1;
|
||||
|
||||
if (cpuinfo == NULL) {
|
||||
/* fall back on the sysconf value */
|
||||
return numPhysicalCores;
|
||||
}
|
||||
|
||||
/* assume the cpu cores/siblings values will be constant across all
|
||||
* present processors */
|
||||
while (!feof(cpuinfo)) {
|
||||
if (fgets(buff, BUF_SIZE, cpuinfo) != NULL) {
|
||||
if (strncmp(buff, "siblings", 8) == 0) {
|
||||
const char* const sep = strchr(buff, ':');
|
||||
if (*sep == '\0') {
|
||||
/* formatting was broken? */
|
||||
goto failed;
|
||||
}
|
||||
|
||||
siblings = atoi(sep + 1);
|
||||
}
|
||||
if (strncmp(buff, "cpu cores", 9) == 0) {
|
||||
const char* const sep = strchr(buff, ':');
|
||||
if (*sep == '\0') {
|
||||
/* formatting was broken? */
|
||||
goto failed;
|
||||
}
|
||||
|
||||
cpu_cores = atoi(sep + 1);
|
||||
}
|
||||
} else if (ferror(cpuinfo)) {
|
||||
/* fall back on the sysconf value */
|
||||
goto failed;
|
||||
}
|
||||
}
|
||||
if (siblings && cpu_cores) {
|
||||
ratio = siblings / cpu_cores;
|
||||
}
|
||||
failed:
|
||||
fclose(cpuinfo);
|
||||
return numPhysicalCores = numPhysicalCores / ratio;
|
||||
}
|
||||
}
|
||||
|
||||
#elif defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)
|
||||
|
||||
/* Use apple-provided syscall
|
||||
* see: man 3 sysctl */
|
||||
int UTIL_countPhysicalCores(void)
|
||||
{
|
||||
static int numPhysicalCores = 0;
|
||||
|
||||
if (numPhysicalCores != 0) return numPhysicalCores;
|
||||
|
||||
numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
|
||||
if (numPhysicalCores == -1) {
|
||||
/* value not queryable, fall back on 1 */
|
||||
return numPhysicalCores = 1;
|
||||
}
|
||||
return numPhysicalCores;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
int UTIL_countPhysicalCores(void)
|
||||
{
|
||||
/* assume 1 */
|
||||
return 1;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
631
programs/util.h
631
programs/util.h
@ -16,15 +16,13 @@ extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Dependencies
|
||||
******************************************/
|
||||
#include "platform.h" /* PLATFORM_POSIX_VERSION, ZSTD_NANOSLEEP_SUPPORT, ZSTD_SETPRIORITY_SUPPORT */
|
||||
#include <stdlib.h> /* malloc */
|
||||
#include <stdlib.h> /* malloc, realloc, free */
|
||||
#include <stddef.h> /* size_t, ptrdiff_t */
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <string.h> /* strncmp */
|
||||
#include <sys/types.h> /* stat, utime */
|
||||
#include <sys/stat.h> /* stat, chmod */
|
||||
#if defined(_MSC_VER)
|
||||
@ -35,11 +33,10 @@ extern "C" {
|
||||
# include <utime.h> /* utime */
|
||||
#endif
|
||||
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */
|
||||
#include <errno.h>
|
||||
#include "mem.h" /* U32, U64 */
|
||||
|
||||
|
||||
/* ************************************************************
|
||||
/*-************************************************************
|
||||
* Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW
|
||||
***************************************************************/
|
||||
#if defined(_MSC_VER) && (_MSC_VER >= 1400)
|
||||
@ -84,7 +81,7 @@ extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/* *************************************
|
||||
/*-*************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
#define LIST_SIZE_INCREASE (8*1024)
|
||||
@ -110,7 +107,7 @@ extern "C" {
|
||||
/*-****************************************
|
||||
* Console log
|
||||
******************************************/
|
||||
static int g_utilDisplayLevel;
|
||||
extern int g_utilDisplayLevel;
|
||||
#define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } }
|
||||
|
||||
@ -119,61 +116,16 @@ static int g_utilDisplayLevel;
|
||||
* Time functions
|
||||
******************************************/
|
||||
#if defined(_WIN32) /* Windows */
|
||||
|
||||
#define UTIL_TIME_INITIALIZER { { 0, 0 } }
|
||||
typedef LARGE_INTEGER UTIL_time_t;
|
||||
|
||||
UTIL_STATIC UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
|
||||
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
|
||||
{
|
||||
static LARGE_INTEGER ticksPerSecond;
|
||||
static int init = 0;
|
||||
if (!init) {
|
||||
if (!QueryPerformanceFrequency(&ticksPerSecond))
|
||||
UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
|
||||
init = 1;
|
||||
}
|
||||
return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
|
||||
}
|
||||
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
|
||||
{
|
||||
static LARGE_INTEGER ticksPerSecond;
|
||||
static int init = 0;
|
||||
if (!init) {
|
||||
if (!QueryPerformanceFrequency(&ticksPerSecond))
|
||||
UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
|
||||
init = 1;
|
||||
}
|
||||
return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
|
||||
}
|
||||
|
||||
#elif defined(__APPLE__) && defined(__MACH__)
|
||||
|
||||
#include <mach/mach_time.h>
|
||||
#define UTIL_TIME_INITIALIZER 0
|
||||
typedef U64 UTIL_time_t;
|
||||
|
||||
UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
|
||||
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
|
||||
{
|
||||
static mach_timebase_info_data_t rate;
|
||||
static int init = 0;
|
||||
if (!init) {
|
||||
mach_timebase_info(&rate);
|
||||
init = 1;
|
||||
}
|
||||
return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom))/1000ULL;
|
||||
}
|
||||
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
|
||||
{
|
||||
static mach_timebase_info_data_t rate;
|
||||
static int init = 0;
|
||||
if (!init) {
|
||||
mach_timebase_info(&rate);
|
||||
init = 1;
|
||||
}
|
||||
return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom);
|
||||
}
|
||||
|
||||
#elif (PLATFORM_POSIX_VERSION >= 200112L) \
|
||||
&& (defined(__UCLIBC__) \
|
||||
|| (defined(__GLIBC__) \
|
||||
@ -184,79 +136,27 @@ static int g_utilDisplayLevel;
|
||||
typedef struct timespec UTIL_freq_t;
|
||||
typedef struct timespec UTIL_time_t;
|
||||
|
||||
UTIL_STATIC UTIL_time_t UTIL_getTime(void)
|
||||
{
|
||||
UTIL_time_t time;
|
||||
if (clock_gettime(CLOCK_MONOTONIC, &time))
|
||||
UTIL_DISPLAYLEVEL(1, "ERROR: Failed to get time\n"); /* we could also exit() */
|
||||
return time;
|
||||
}
|
||||
|
||||
UTIL_STATIC UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
|
||||
{
|
||||
UTIL_time_t diff;
|
||||
if (end.tv_nsec < begin.tv_nsec) {
|
||||
diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec;
|
||||
diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec;
|
||||
} else {
|
||||
diff.tv_sec = end.tv_sec - begin.tv_sec;
|
||||
diff.tv_nsec = end.tv_nsec - begin.tv_nsec;
|
||||
}
|
||||
return diff;
|
||||
}
|
||||
|
||||
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
|
||||
{
|
||||
UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
|
||||
U64 micro = 0;
|
||||
micro += 1000000ULL * diff.tv_sec;
|
||||
micro += diff.tv_nsec / 1000ULL;
|
||||
return micro;
|
||||
}
|
||||
|
||||
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
|
||||
{
|
||||
UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
|
||||
U64 nano = 0;
|
||||
nano += 1000000000ULL * diff.tv_sec;
|
||||
nano += diff.tv_nsec;
|
||||
return nano;
|
||||
}
|
||||
UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end);
|
||||
|
||||
#else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
|
||||
|
||||
typedef clock_t UTIL_time_t;
|
||||
#define UTIL_TIME_INITIALIZER 0
|
||||
UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return clock(); }
|
||||
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
|
||||
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
|
||||
|
||||
#endif
|
||||
|
||||
UTIL_time_t UTIL_getTime(void);
|
||||
U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd);
|
||||
U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd);
|
||||
|
||||
#define SEC_TO_MICRO 1000000
|
||||
|
||||
/* returns time span in microseconds */
|
||||
UTIL_STATIC U64 UTIL_clockSpanMicro(UTIL_time_t clockStart )
|
||||
{
|
||||
UTIL_time_t const clockEnd = UTIL_getTime();
|
||||
return UTIL_getSpanTimeMicro(clockStart, clockEnd);
|
||||
}
|
||||
U64 UTIL_clockSpanMicro(UTIL_time_t clockStart);
|
||||
|
||||
/* returns time span in microseconds */
|
||||
UTIL_STATIC U64 UTIL_clockSpanNano(UTIL_time_t clockStart )
|
||||
{
|
||||
UTIL_time_t const clockEnd = UTIL_getTime();
|
||||
return UTIL_getSpanTimeNano(clockStart, clockEnd);
|
||||
}
|
||||
|
||||
UTIL_STATIC void UTIL_waitForNextTick(void)
|
||||
{
|
||||
UTIL_time_t const clockStart = UTIL_getTime();
|
||||
UTIL_time_t clockEnd;
|
||||
do {
|
||||
clockEnd = UTIL_getTime();
|
||||
} while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0);
|
||||
}
|
||||
|
||||
|
||||
U64 UTIL_clockSpanNano(UTIL_time_t clockStart);
|
||||
void UTIL_waitForNextTick(void);
|
||||
|
||||
/*-****************************************
|
||||
* File functions
|
||||
@ -269,129 +169,23 @@ UTIL_STATIC void UTIL_waitForNextTick(void)
|
||||
#endif
|
||||
|
||||
|
||||
UTIL_STATIC int UTIL_isRegularFile(const char* infilename);
|
||||
|
||||
|
||||
UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf)
|
||||
{
|
||||
int res = 0;
|
||||
struct utimbuf timebuf;
|
||||
|
||||
if (!UTIL_isRegularFile(filename))
|
||||
return -1;
|
||||
|
||||
timebuf.actime = time(NULL);
|
||||
timebuf.modtime = statbuf->st_mtime;
|
||||
res += utime(filename, &timebuf); /* set access and modification times */
|
||||
|
||||
#if !defined(_WIN32)
|
||||
res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */
|
||||
#endif
|
||||
|
||||
res += chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */
|
||||
|
||||
errno = 0;
|
||||
return -res; /* number of errors is returned */
|
||||
}
|
||||
|
||||
|
||||
UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
|
||||
{
|
||||
int r;
|
||||
#if defined(_MSC_VER)
|
||||
r = _stat64(infilename, statbuf);
|
||||
if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */
|
||||
#else
|
||||
r = stat(infilename, statbuf);
|
||||
if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */
|
||||
#endif
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
UTIL_STATIC int UTIL_isRegularFile(const char* infilename)
|
||||
{
|
||||
stat_t statbuf;
|
||||
return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */
|
||||
}
|
||||
|
||||
|
||||
UTIL_STATIC U32 UTIL_isDirectory(const char* infilename)
|
||||
{
|
||||
int r;
|
||||
stat_t statbuf;
|
||||
#if defined(_MSC_VER)
|
||||
r = _stat64(infilename, &statbuf);
|
||||
if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
|
||||
#else
|
||||
r = stat(infilename, &statbuf);
|
||||
if (!r && S_ISDIR(statbuf.st_mode)) return 1;
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
UTIL_STATIC U32 UTIL_isLink(const char* infilename)
|
||||
{
|
||||
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
|
||||
#ifndef __STRICT_ANSI__
|
||||
#if defined(_BSD_SOURCE) \
|
||||
|| (defined(_XOPEN_SOURCE) && (_XOPEN_SOURCE >= 500)) \
|
||||
|| (defined(_XOPEN_SOURCE) && defined(_XOPEN_SOURCE_EXTENDED)) \
|
||||
|| (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) \
|
||||
|| (defined(__APPLE__) && defined(__MACH__))
|
||||
int r;
|
||||
stat_t statbuf;
|
||||
r = lstat(infilename, &statbuf);
|
||||
if (!r && S_ISLNK(statbuf.st_mode)) return 1;
|
||||
#endif
|
||||
#endif
|
||||
(void)infilename;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int UTIL_fileExist(const char* filename);
|
||||
int UTIL_isRegularFile(const char* infilename);
|
||||
int UTIL_setFileStat(const char* filename, stat_t* statbuf);
|
||||
U32 UTIL_isDirectory(const char* infilename);
|
||||
int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
|
||||
|
||||
U32 UTIL_isLink(const char* infilename);
|
||||
#define UTIL_FILESIZE_UNKNOWN ((U64)(-1))
|
||||
UTIL_STATIC U64 UTIL_getFileSize(const char* infilename)
|
||||
{
|
||||
if (!UTIL_isRegularFile(infilename)) return UTIL_FILESIZE_UNKNOWN;
|
||||
{ int r;
|
||||
#if defined(_MSC_VER)
|
||||
struct __stat64 statbuf;
|
||||
r = _stat64(infilename, &statbuf);
|
||||
if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
|
||||
#elif defined(__MINGW32__) && defined (__MSVCRT__)
|
||||
struct _stati64 statbuf;
|
||||
r = _stati64(infilename, &statbuf);
|
||||
if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
|
||||
#else
|
||||
struct stat statbuf;
|
||||
r = stat(infilename, &statbuf);
|
||||
if (r || !S_ISREG(statbuf.st_mode)) return UTIL_FILESIZE_UNKNOWN;
|
||||
#endif
|
||||
return (U64)statbuf.st_size;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
UTIL_STATIC U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles)
|
||||
{
|
||||
U64 total = 0;
|
||||
int error = 0;
|
||||
unsigned n;
|
||||
for (n=0; n<nbFiles; n++) {
|
||||
U64 const size = UTIL_getFileSize(fileNamesTable[n]);
|
||||
error |= (size == UTIL_FILESIZE_UNKNOWN);
|
||||
total += size;
|
||||
}
|
||||
return error ? UTIL_FILESIZE_UNKNOWN : total;
|
||||
}
|
||||
U64 UTIL_getFileSize(const char* infilename);
|
||||
|
||||
U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles);
|
||||
|
||||
/*
|
||||
* A modified version of realloc().
|
||||
* If UTIL_realloc() fails the original block is freed.
|
||||
*/
|
||||
UTIL_STATIC void *UTIL_realloc(void *ptr, size_t size)
|
||||
UTIL_STATIC void* UTIL_realloc(void *ptr, size_t size)
|
||||
{
|
||||
void *newptr = realloc(ptr, size);
|
||||
if (newptr) return newptr;
|
||||
@ -399,143 +193,14 @@ UTIL_STATIC void *UTIL_realloc(void *ptr, size_t size)
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int UTIL_prepareFileList(const char* dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks);
|
||||
#ifdef _WIN32
|
||||
# define UTIL_HAS_CREATEFILELIST
|
||||
|
||||
UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
|
||||
{
|
||||
char* path;
|
||||
int dirLength, fnameLength, pathLength, nbFiles = 0;
|
||||
WIN32_FIND_DATAA cFile;
|
||||
HANDLE hFile;
|
||||
|
||||
dirLength = (int)strlen(dirName);
|
||||
path = (char*) malloc(dirLength + 3);
|
||||
if (!path) return 0;
|
||||
|
||||
memcpy(path, dirName, dirLength);
|
||||
path[dirLength] = '\\';
|
||||
path[dirLength+1] = '*';
|
||||
path[dirLength+2] = 0;
|
||||
|
||||
hFile=FindFirstFileA(path, &cFile);
|
||||
if (hFile == INVALID_HANDLE_VALUE) {
|
||||
UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s'\n", dirName);
|
||||
return 0;
|
||||
}
|
||||
free(path);
|
||||
|
||||
do {
|
||||
fnameLength = (int)strlen(cFile.cFileName);
|
||||
path = (char*) malloc(dirLength + fnameLength + 2);
|
||||
if (!path) { FindClose(hFile); return 0; }
|
||||
memcpy(path, dirName, dirLength);
|
||||
path[dirLength] = '\\';
|
||||
memcpy(path+dirLength+1, cFile.cFileName, fnameLength);
|
||||
pathLength = dirLength+1+fnameLength;
|
||||
path[pathLength] = 0;
|
||||
if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
|
||||
if (strcmp (cFile.cFileName, "..") == 0 ||
|
||||
strcmp (cFile.cFileName, ".") == 0) continue;
|
||||
|
||||
nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */
|
||||
if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
|
||||
}
|
||||
else if ((cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED)) {
|
||||
if (*bufStart + *pos + pathLength >= *bufEnd) {
|
||||
ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
|
||||
*bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
|
||||
*bufEnd = *bufStart + newListSize;
|
||||
if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
|
||||
}
|
||||
if (*bufStart + *pos + pathLength < *bufEnd) {
|
||||
strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos));
|
||||
*pos += pathLength + 1;
|
||||
nbFiles++;
|
||||
}
|
||||
}
|
||||
free(path);
|
||||
} while (FindNextFileA(hFile, &cFile));
|
||||
|
||||
FindClose(hFile);
|
||||
return nbFiles;
|
||||
}
|
||||
|
||||
#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
|
||||
# define UTIL_HAS_CREATEFILELIST
|
||||
# include <dirent.h> /* opendir, readdir */
|
||||
# include <string.h> /* strerror, memcpy */
|
||||
|
||||
UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
|
||||
{
|
||||
DIR *dir;
|
||||
struct dirent *entry;
|
||||
char* path;
|
||||
int dirLength, fnameLength, pathLength, nbFiles = 0;
|
||||
|
||||
if (!(dir = opendir(dirName))) {
|
||||
UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
|
||||
return 0;
|
||||
}
|
||||
|
||||
dirLength = (int)strlen(dirName);
|
||||
errno = 0;
|
||||
while ((entry = readdir(dir)) != NULL) {
|
||||
if (strcmp (entry->d_name, "..") == 0 ||
|
||||
strcmp (entry->d_name, ".") == 0) continue;
|
||||
fnameLength = (int)strlen(entry->d_name);
|
||||
path = (char*) malloc(dirLength + fnameLength + 2);
|
||||
if (!path) { closedir(dir); return 0; }
|
||||
memcpy(path, dirName, dirLength);
|
||||
|
||||
path[dirLength] = '/';
|
||||
memcpy(path+dirLength+1, entry->d_name, fnameLength);
|
||||
pathLength = dirLength+1+fnameLength;
|
||||
path[pathLength] = 0;
|
||||
|
||||
if (!followLinks && UTIL_isLink(path)) {
|
||||
UTIL_DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", path);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (UTIL_isDirectory(path)) {
|
||||
nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */
|
||||
if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
|
||||
} else {
|
||||
if (*bufStart + *pos + pathLength >= *bufEnd) {
|
||||
ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
|
||||
*bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
|
||||
*bufEnd = *bufStart + newListSize;
|
||||
if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
|
||||
}
|
||||
if (*bufStart + *pos + pathLength < *bufEnd) {
|
||||
strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos));
|
||||
*pos += pathLength + 1;
|
||||
nbFiles++;
|
||||
}
|
||||
}
|
||||
free(path);
|
||||
errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */
|
||||
}
|
||||
|
||||
if (errno != 0) {
|
||||
UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s\n", dirName, strerror(errno));
|
||||
free(*bufStart);
|
||||
*bufStart = NULL;
|
||||
}
|
||||
closedir(dir);
|
||||
return nbFiles;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
|
||||
{
|
||||
(void)bufStart; (void)bufEnd; (void)pos; (void)followLinks;
|
||||
UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif /* #ifdef _WIN32 */
|
||||
|
||||
/*
|
||||
@ -544,56 +209,10 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
|
||||
* After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer)
|
||||
* In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called.
|
||||
*/
|
||||
UTIL_STATIC const char**
|
||||
const char**
|
||||
UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
|
||||
char** allocatedBuffer, unsigned* allocatedNamesNb,
|
||||
int followLinks)
|
||||
{
|
||||
size_t pos;
|
||||
unsigned i, nbFiles;
|
||||
char* buf = (char*)malloc(LIST_SIZE_INCREASE);
|
||||
char* bufend = buf + LIST_SIZE_INCREASE;
|
||||
const char** fileTable;
|
||||
|
||||
if (!buf) return NULL;
|
||||
|
||||
for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) {
|
||||
if (!UTIL_isDirectory(inputNames[i])) {
|
||||
size_t const len = strlen(inputNames[i]);
|
||||
if (buf + pos + len >= bufend) {
|
||||
ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
|
||||
buf = (char*)UTIL_realloc(buf, newListSize);
|
||||
bufend = buf + newListSize;
|
||||
if (!buf) return NULL;
|
||||
}
|
||||
if (buf + pos + len < bufend) {
|
||||
strncpy(buf + pos, inputNames[i], bufend - (buf + pos));
|
||||
pos += len + 1;
|
||||
nbFiles++;
|
||||
}
|
||||
} else {
|
||||
nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks);
|
||||
if (buf == NULL) return NULL;
|
||||
} }
|
||||
|
||||
if (nbFiles == 0) { free(buf); return NULL; }
|
||||
|
||||
fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*));
|
||||
if (!fileTable) { free(buf); return NULL; }
|
||||
|
||||
for (i=0, pos=0; i<nbFiles; i++) {
|
||||
fileTable[i] = buf + pos;
|
||||
pos += strlen(fileTable[i]) + 1;
|
||||
}
|
||||
|
||||
if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
|
||||
|
||||
*allocatedBuffer = buf;
|
||||
*allocatedNamesNb = nbFiles;
|
||||
|
||||
return fileTable;
|
||||
}
|
||||
|
||||
int followLinks);
|
||||
|
||||
UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer)
|
||||
{
|
||||
@ -601,201 +220,7 @@ UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBu
|
||||
if (filenameTable) free((void*)filenameTable);
|
||||
}
|
||||
|
||||
/* count the number of physical cores */
|
||||
#if defined(_WIN32) || defined(WIN32)
|
||||
|
||||
#include <windows.h>
|
||||
|
||||
typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD);
|
||||
|
||||
UTIL_STATIC int UTIL_countPhysicalCores(void)
|
||||
{
|
||||
static int numPhysicalCores = 0;
|
||||
if (numPhysicalCores != 0) return numPhysicalCores;
|
||||
|
||||
{ LPFN_GLPI glpi;
|
||||
BOOL done = FALSE;
|
||||
PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = NULL;
|
||||
PSYSTEM_LOGICAL_PROCESSOR_INFORMATION ptr = NULL;
|
||||
DWORD returnLength = 0;
|
||||
size_t byteOffset = 0;
|
||||
|
||||
glpi = (LPFN_GLPI)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
|
||||
"GetLogicalProcessorInformation");
|
||||
|
||||
if (glpi == NULL) {
|
||||
goto failed;
|
||||
}
|
||||
|
||||
while(!done) {
|
||||
DWORD rc = glpi(buffer, &returnLength);
|
||||
if (FALSE == rc) {
|
||||
if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
|
||||
if (buffer)
|
||||
free(buffer);
|
||||
buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(returnLength);
|
||||
|
||||
if (buffer == NULL) {
|
||||
perror("zstd");
|
||||
exit(1);
|
||||
}
|
||||
} else {
|
||||
/* some other error */
|
||||
goto failed;
|
||||
}
|
||||
} else {
|
||||
done = TRUE;
|
||||
}
|
||||
}
|
||||
|
||||
ptr = buffer;
|
||||
|
||||
while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) {
|
||||
|
||||
if (ptr->Relationship == RelationProcessorCore) {
|
||||
numPhysicalCores++;
|
||||
}
|
||||
|
||||
ptr++;
|
||||
byteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
|
||||
}
|
||||
|
||||
free(buffer);
|
||||
|
||||
return numPhysicalCores;
|
||||
}
|
||||
|
||||
failed:
|
||||
/* try to fall back on GetSystemInfo */
|
||||
{ SYSTEM_INFO sysinfo;
|
||||
GetSystemInfo(&sysinfo);
|
||||
numPhysicalCores = sysinfo.dwNumberOfProcessors;
|
||||
if (numPhysicalCores == 0) numPhysicalCores = 1; /* just in case */
|
||||
}
|
||||
return numPhysicalCores;
|
||||
}
|
||||
|
||||
#elif defined(__APPLE__)
|
||||
|
||||
#include <sys/sysctl.h>
|
||||
|
||||
/* Use apple-provided syscall
|
||||
* see: man 3 sysctl */
|
||||
UTIL_STATIC int UTIL_countPhysicalCores(void)
|
||||
{
|
||||
static S32 numPhysicalCores = 0; /* apple specifies int32_t */
|
||||
if (numPhysicalCores != 0) return numPhysicalCores;
|
||||
|
||||
{ size_t size = sizeof(S32);
|
||||
int const ret = sysctlbyname("hw.physicalcpu", &numPhysicalCores, &size, NULL, 0);
|
||||
if (ret != 0) {
|
||||
if (errno == ENOENT) {
|
||||
/* entry not present, fall back on 1 */
|
||||
numPhysicalCores = 1;
|
||||
} else {
|
||||
perror("zstd: can't get number of physical cpus");
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
return numPhysicalCores;
|
||||
}
|
||||
}
|
||||
|
||||
#elif defined(__linux__)
|
||||
|
||||
/* parse /proc/cpuinfo
|
||||
* siblings / cpu cores should give hyperthreading ratio
|
||||
* otherwise fall back on sysconf */
|
||||
UTIL_STATIC int UTIL_countPhysicalCores(void)
|
||||
{
|
||||
static int numPhysicalCores = 0;
|
||||
|
||||
if (numPhysicalCores != 0) return numPhysicalCores;
|
||||
|
||||
numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
|
||||
if (numPhysicalCores == -1) {
|
||||
/* value not queryable, fall back on 1 */
|
||||
return numPhysicalCores = 1;
|
||||
}
|
||||
|
||||
/* try to determine if there's hyperthreading */
|
||||
{ FILE* const cpuinfo = fopen("/proc/cpuinfo", "r");
|
||||
#define BUF_SIZE 80
|
||||
char buff[BUF_SIZE];
|
||||
|
||||
int siblings = 0;
|
||||
int cpu_cores = 0;
|
||||
int ratio = 1;
|
||||
|
||||
if (cpuinfo == NULL) {
|
||||
/* fall back on the sysconf value */
|
||||
return numPhysicalCores;
|
||||
}
|
||||
|
||||
/* assume the cpu cores/siblings values will be constant across all
|
||||
* present processors */
|
||||
while (!feof(cpuinfo)) {
|
||||
if (fgets(buff, BUF_SIZE, cpuinfo) != NULL) {
|
||||
if (strncmp(buff, "siblings", 8) == 0) {
|
||||
const char* const sep = strchr(buff, ':');
|
||||
if (*sep == '\0') {
|
||||
/* formatting was broken? */
|
||||
goto failed;
|
||||
}
|
||||
|
||||
siblings = atoi(sep + 1);
|
||||
}
|
||||
if (strncmp(buff, "cpu cores", 9) == 0) {
|
||||
const char* const sep = strchr(buff, ':');
|
||||
if (*sep == '\0') {
|
||||
/* formatting was broken? */
|
||||
goto failed;
|
||||
}
|
||||
|
||||
cpu_cores = atoi(sep + 1);
|
||||
}
|
||||
} else if (ferror(cpuinfo)) {
|
||||
/* fall back on the sysconf value */
|
||||
goto failed;
|
||||
}
|
||||
}
|
||||
if (siblings && cpu_cores) {
|
||||
ratio = siblings / cpu_cores;
|
||||
}
|
||||
failed:
|
||||
fclose(cpuinfo);
|
||||
return numPhysicalCores = numPhysicalCores / ratio;
|
||||
}
|
||||
}
|
||||
|
||||
#elif defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)
|
||||
|
||||
/* Use apple-provided syscall
|
||||
* see: man 3 sysctl */
|
||||
UTIL_STATIC int UTIL_countPhysicalCores(void)
|
||||
{
|
||||
static int numPhysicalCores = 0;
|
||||
|
||||
if (numPhysicalCores != 0) return numPhysicalCores;
|
||||
|
||||
numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
|
||||
if (numPhysicalCores == -1) {
|
||||
/* value not queryable, fall back on 1 */
|
||||
return numPhysicalCores = 1;
|
||||
}
|
||||
return numPhysicalCores;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
UTIL_STATIC int UTIL_countPhysicalCores(void)
|
||||
{
|
||||
/* assume 1 */
|
||||
return 1;
|
||||
}
|
||||
|
||||
#endif
|
||||
int UTIL_countPhysicalCores(void);
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
.
|
||||
.TH "ZSTD" "1" "October 2018" "zstd 1.3.7" "User Commands"
|
||||
.TH "ZSTD" "1" "December 2018" "zstd 1.3.8" "User Commands"
|
||||
.
|
||||
.SH "NAME"
|
||||
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
|
||||
@ -127,6 +127,10 @@ Does not spawn a thread for compression, use a single thread for both I/O and co
|
||||
\fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
|
||||
.
|
||||
.TP
|
||||
\fB\-\-rsyncable\fR
|
||||
\fBzstd\fR will periodically synchronize the compression state to make the compressed file more rsync\-friendly\. There is a negligible impact to compression ratio, and the faster compression levels will see a small compression speed hit\. This feature does not work with \fB\-\-single\-thread\fR\. You probably don\'t want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your milage may vary\.
|
||||
.
|
||||
.TP
|
||||
\fB\-D file\fR
|
||||
use \fBfile\fR as Dictionary to compress or decompress FILE(s)
|
||||
.
|
||||
@ -312,7 +316,7 @@ set process priority to real\-time
|
||||
Specify a strategy used by a match finder\.
|
||||
.
|
||||
.IP
|
||||
There are 8 strategies numbered from 1 to 8, from faster to stronger: 1=ZSTD_fast, 2=ZSTD_dfast, 3=ZSTD_greedy, 4=ZSTD_lazy, 5=ZSTD_lazy2, 6=ZSTD_btlazy2, 7=ZSTD_btopt, 8=ZSTD_btultra\.
|
||||
There are 9 strategies numbered from 1 to 9, from faster to stronger: 1=ZSTD_fast, 2=ZSTD_dfast, 3=ZSTD_greedy, 4=ZSTD_lazy, 5=ZSTD_lazy2, 6=ZSTD_btlazy2, 7=ZSTD_btopt, 8=ZSTD_btultra, 9=ZSTD_btultra2\.
|
||||
.
|
||||
.TP
|
||||
\fBwindowLog\fR=\fIwlog\fR, \fBwlog\fR=\fIwlog\fR
|
||||
@ -355,21 +359,21 @@ More searches increases the chance to find a match which usually increases compr
|
||||
The minimum \fIslog\fR is 1 and the maximum is 26\.
|
||||
.
|
||||
.TP
|
||||
\fBsearchLength\fR=\fIslen\fR, \fBslen\fR=\fIslen\fR
|
||||
\fBminMatch\fR=\fImml\fR, \fBmml\fR=\fImml\fR
|
||||
Specify the minimum searched length of a match in a hash table\.
|
||||
.
|
||||
.IP
|
||||
Larger search lengths usually decrease compression ratio but improve decompression speed\.
|
||||
.
|
||||
.IP
|
||||
The minimum \fIslen\fR is 3 and the maximum is 7\.
|
||||
The minimum \fImml\fR is 3 and the maximum is 7\.
|
||||
.
|
||||
.TP
|
||||
\fBtargetLen\fR=\fItlen\fR, \fBtlen\fR=\fItlen\fR
|
||||
The impact of this field vary depending on selected strategy\.
|
||||
.
|
||||
.IP
|
||||
For ZSTD_btopt and ZSTD_btultra, it specifies the minimum match length that causes match finder to stop searching for better matches\. A larger \fBtargetLen\fR usually improves compression ratio but decreases compression speed\.
|
||||
For ZSTD_btopt, ZSTD_btultra and ZSTD_btultra2, it specifies the minimum match length that causes match finder to stop searching\. A larger \fBtargetLen\fR usually improves compression ratio but decreases compression speed\.
|
||||
.
|
||||
.IP
|
||||
For ZSTD_fast, it triggers ultra\-fast mode when > 0\. The value represents the amount of data skipped between match sampling\. Impact is reversed : a larger \fBtargetLen\fR increases compression speed but decreases compression ratio\.
|
||||
@ -385,10 +389,10 @@ The minimum \fItlen\fR is 0 and the maximum is 999\.
|
||||
Determine \fBoverlapSize\fR, amount of data reloaded from previous job\. This parameter is only available when multithreading is enabled\. Reloading more data improves compression ratio, but decreases speed\.
|
||||
.
|
||||
.IP
|
||||
The minimum \fIovlog\fR is 0, and the maximum is 9\. 0 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the amount of reload by a factor 2\. Default \fIovlog\fR is 6, which means "reload \fBwindowSize / 8\fR"\. Exception : the maximum compression level (22) has a default \fIovlog\fR of 9\.
|
||||
The minimum \fIovlog\fR is 0, and the maximum is 9\. 1 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the reloaded amount by a factor 2\. For example, 8 means "windowSize/2", and 6 means "windowSize/8"\. Value 0 is special and means "default" : \fIovlog\fR is automatically determined by \fBzstd\fR\. In which case, \fIovlog\fR will range from 6 to 9, depending on selected \fIstrat\fR\.
|
||||
.
|
||||
.TP
|
||||
\fBldmHashLog\fR=\fIldmhlog\fR, \fBldmhlog\fR=\fIldmhlog\fR
|
||||
\fBldmHashLog\fR=\fIlhlog\fR, \fBlhlog\fR=\fIlhlog\fR
|
||||
Specify the maximum size for a hash table used for long distance matching\.
|
||||
.
|
||||
.IP
|
||||
@ -398,10 +402,10 @@ This option is ignored unless long distance matching is enabled\.
|
||||
Bigger hash tables usually improve compression ratio at the expense of more memory during compression and a decrease in compression speed\.
|
||||
.
|
||||
.IP
|
||||
The minimum \fIldmhlog\fR is 6 and the maximum is 26 (default: 20)\.
|
||||
The minimum \fIlhlog\fR is 6 and the maximum is 26 (default: 20)\.
|
||||
.
|
||||
.TP
|
||||
\fBldmSearchLength\fR=\fIldmslen\fR, \fBldmslen\fR=\fIldmslen\fR
|
||||
\fBldmMinMatch\fR=\fIlmml\fR, \fBlmml\fR=\fIlmml\fR
|
||||
Specify the minimum searched length of a match for long distance matching\.
|
||||
.
|
||||
.IP
|
||||
@ -411,10 +415,10 @@ This option is ignored unless long distance matching is enabled\.
|
||||
Larger/very small values usually decrease compression ratio\.
|
||||
.
|
||||
.IP
|
||||
The minimum \fIldmslen\fR is 4 and the maximum is 4096 (default: 64)\.
|
||||
The minimum \fIlmml\fR is 4 and the maximum is 4096 (default: 64)\.
|
||||
.
|
||||
.TP
|
||||
\fBldmBucketSizeLog\fR=\fIldmblog\fR, \fBldmblog\fR=\fIldmblog\fR
|
||||
\fBldmBucketSizeLog\fR=\fIlblog\fR, \fBlblog\fR=\fIlblog\fR
|
||||
Specify the size of each bucket for the hash table used for long distance matching\.
|
||||
.
|
||||
.IP
|
||||
@ -424,10 +428,10 @@ This option is ignored unless long distance matching is enabled\.
|
||||
Larger bucket sizes improve collision resolution but decrease compression speed\.
|
||||
.
|
||||
.IP
|
||||
The minimum \fIldmblog\fR is 0 and the maximum is 8 (default: 3)\.
|
||||
The minimum \fIlblog\fR is 0 and the maximum is 8 (default: 3)\.
|
||||
.
|
||||
.TP
|
||||
\fBldmHashEveryLog\fR=\fIldmhevery\fR, \fBldmhevery\fR=\fIldmhevery\fR
|
||||
\fBldmHashRateLog\fR=\fIlhrlog\fR, \fBlhrlog\fR=\fIlhrlog\fR
|
||||
Specify the frequency of inserting entries into the long distance matching hash table\.
|
||||
.
|
||||
.IP
|
||||
@ -437,13 +441,13 @@ This option is ignored unless long distance matching is enabled\.
|
||||
Larger values will improve compression speed\. Deviating far from the default value will likely result in a decrease in compression ratio\.
|
||||
.
|
||||
.IP
|
||||
The default value is \fBwlog \- ldmhlog\fR\.
|
||||
The default value is \fBwlog \- lhlog\fR\.
|
||||
.
|
||||
.SS "Example"
|
||||
The following parameters sets advanced compression options to something similar to predefined level 19 for files bigger than 256 KB:
|
||||
.
|
||||
.P
|
||||
\fB\-\-zstd\fR=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6
|
||||
\fB\-\-zstd\fR=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
|
||||
.
|
||||
.SS "\-B#:"
|
||||
Select the size of each compression job\. This parameter is available only when multi\-threading is enabled\. Default value is \fB4 * windowSize\fR, which means it varies depending on compression level\. \fB\-B#\fR makes it possible to select a custom value\. Note that job size must respect a minimum value which is enforced transparently\. This minimum is either 1 MB, or \fBoverlapSize\fR, whichever is largest\.
|
||||
|
||||
@ -144,6 +144,14 @@ the last one takes effect.
|
||||
Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible.
|
||||
_note_ : at the time of this writing, `--adapt` can remain stuck at low speed
|
||||
when combined with multiple worker threads (>=2).
|
||||
* `--rsyncable` :
|
||||
`zstd` will periodically synchronize the compression state to make the
|
||||
compressed file more rsync-friendly. There is a negligible impact to
|
||||
compression ratio, and the faster compression levels will see a small
|
||||
compression speed hit.
|
||||
This feature does not work with `--single-thread`. You probably don't want
|
||||
to use it with long range mode, since it will decrease the effectiveness of
|
||||
the synchronization points, but your milage may vary.
|
||||
* `-D file`:
|
||||
use `file` as Dictionary to compress or decompress FILE(s)
|
||||
* `--no-dictID`:
|
||||
@ -187,6 +195,8 @@ the last one takes effect.
|
||||
* `-q`, `--quiet`:
|
||||
suppress warnings, interactivity, and notifications.
|
||||
specify twice to suppress errors too.
|
||||
* `--no-progress`:
|
||||
do not display the progress bar, but keep all other messages.
|
||||
* `-C`, `--[no-]check`:
|
||||
add integrity check computed from uncompressed data (default: enabled)
|
||||
* `--`:
|
||||
@ -331,9 +341,10 @@ The list of available _options_:
|
||||
- `strategy`=_strat_, `strat`=_strat_:
|
||||
Specify a strategy used by a match finder.
|
||||
|
||||
There are 8 strategies numbered from 1 to 8, from faster to stronger:
|
||||
1=ZSTD\_fast, 2=ZSTD\_dfast, 3=ZSTD\_greedy, 4=ZSTD\_lazy,
|
||||
5=ZSTD\_lazy2, 6=ZSTD\_btlazy2, 7=ZSTD\_btopt, 8=ZSTD\_btultra.
|
||||
There are 9 strategies numbered from 1 to 9, from faster to stronger:
|
||||
1=ZSTD\_fast, 2=ZSTD\_dfast, 3=ZSTD\_greedy,
|
||||
4=ZSTD\_lazy, 5=ZSTD\_lazy2, 6=ZSTD\_btlazy2,
|
||||
7=ZSTD\_btopt, 8=ZSTD\_btultra, 9=ZSTD\_btultra2.
|
||||
|
||||
- `windowLog`=_wlog_, `wlog`=_wlog_:
|
||||
Specify the maximum number of bits for a match distance.
|
||||
@ -375,19 +386,19 @@ The list of available _options_:
|
||||
|
||||
The minimum _slog_ is 1 and the maximum is 26.
|
||||
|
||||
- `searchLength`=_slen_, `slen`=_slen_:
|
||||
- `minMatch`=_mml_, `mml`=_mml_:
|
||||
Specify the minimum searched length of a match in a hash table.
|
||||
|
||||
Larger search lengths usually decrease compression ratio but improve
|
||||
decompression speed.
|
||||
|
||||
The minimum _slen_ is 3 and the maximum is 7.
|
||||
The minimum _mml_ is 3 and the maximum is 7.
|
||||
|
||||
- `targetLen`=_tlen_, `tlen`=_tlen_:
|
||||
The impact of this field vary depending on selected strategy.
|
||||
|
||||
For ZSTD\_btopt and ZSTD\_btultra, it specifies the minimum match length
|
||||
that causes match finder to stop searching for better matches.
|
||||
For ZSTD\_btopt, ZSTD\_btultra and ZSTD\_btultra2, it specifies
|
||||
the minimum match length that causes match finder to stop searching.
|
||||
A larger `targetLen` usually improves compression ratio
|
||||
but decreases compression speed.
|
||||
|
||||
@ -406,13 +417,14 @@ The list of available _options_:
|
||||
Reloading more data improves compression ratio, but decreases speed.
|
||||
|
||||
The minimum _ovlog_ is 0, and the maximum is 9.
|
||||
0 means "no overlap", hence completely independent jobs.
|
||||
1 means "no overlap", hence completely independent jobs.
|
||||
9 means "full overlap", meaning up to `windowSize` is reloaded from previous job.
|
||||
Reducing _ovlog_ by 1 reduces the amount of reload by a factor 2.
|
||||
Default _ovlog_ is 6, which means "reload `windowSize / 8`".
|
||||
Exception : the maximum compression level (22) has a default _ovlog_ of 9.
|
||||
Reducing _ovlog_ by 1 reduces the reloaded amount by a factor 2.
|
||||
For example, 8 means "windowSize/2", and 6 means "windowSize/8".
|
||||
Value 0 is special and means "default" : _ovlog_ is automatically determined by `zstd`.
|
||||
In which case, _ovlog_ will range from 6 to 9, depending on selected _strat_.
|
||||
|
||||
- `ldmHashLog`=_ldmhlog_, `ldmhlog`=_ldmhlog_:
|
||||
- `ldmHashLog`=_lhlog_, `lhlog`=_lhlog_:
|
||||
Specify the maximum size for a hash table used for long distance matching.
|
||||
|
||||
This option is ignored unless long distance matching is enabled.
|
||||
@ -420,18 +432,18 @@ The list of available _options_:
|
||||
Bigger hash tables usually improve compression ratio at the expense of more
|
||||
memory during compression and a decrease in compression speed.
|
||||
|
||||
The minimum _ldmhlog_ is 6 and the maximum is 26 (default: 20).
|
||||
The minimum _lhlog_ is 6 and the maximum is 26 (default: 20).
|
||||
|
||||
- `ldmSearchLength`=_ldmslen_, `ldmslen`=_ldmslen_:
|
||||
- `ldmMinMatch`=_lmml_, `lmml`=_lmml_:
|
||||
Specify the minimum searched length of a match for long distance matching.
|
||||
|
||||
This option is ignored unless long distance matching is enabled.
|
||||
|
||||
Larger/very small values usually decrease compression ratio.
|
||||
|
||||
The minimum _ldmslen_ is 4 and the maximum is 4096 (default: 64).
|
||||
The minimum _lmml_ is 4 and the maximum is 4096 (default: 64).
|
||||
|
||||
- `ldmBucketSizeLog`=_ldmblog_, `ldmblog`=_ldmblog_:
|
||||
- `ldmBucketSizeLog`=_lblog_, `lblog`=_lblog_:
|
||||
Specify the size of each bucket for the hash table used for long distance
|
||||
matching.
|
||||
|
||||
@ -440,9 +452,9 @@ The list of available _options_:
|
||||
Larger bucket sizes improve collision resolution but decrease compression
|
||||
speed.
|
||||
|
||||
The minimum _ldmblog_ is 0 and the maximum is 8 (default: 3).
|
||||
The minimum _lblog_ is 0 and the maximum is 8 (default: 3).
|
||||
|
||||
- `ldmHashEveryLog`=_ldmhevery_, `ldmhevery`=_ldmhevery_:
|
||||
- `ldmHashRateLog`=_lhrlog_, `lhrlog`=_lhrlog_:
|
||||
Specify the frequency of inserting entries into the long distance matching
|
||||
hash table.
|
||||
|
||||
@ -451,13 +463,13 @@ The list of available _options_:
|
||||
Larger values will improve compression speed. Deviating far from the
|
||||
default value will likely result in a decrease in compression ratio.
|
||||
|
||||
The default value is `wlog - ldmhlog`.
|
||||
The default value is `wlog - lhlog`.
|
||||
|
||||
### Example
|
||||
The following parameters sets advanced compression options to something
|
||||
similar to predefined level 19 for files bigger than 256 KB:
|
||||
|
||||
`--zstd`=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6
|
||||
`--zstd`=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
|
||||
|
||||
### -B#:
|
||||
Select the size of each compression job.
|
||||
|
||||
@ -28,11 +28,12 @@
|
||||
#include "platform.h" /* IS_CONSOLE, PLATFORM_POSIX_VERSION */
|
||||
#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */
|
||||
#include <stdio.h> /* fprintf(), stdin, stdout, stderr */
|
||||
#include <stdlib.h> /* getenv */
|
||||
#include <string.h> /* strcmp, strlen */
|
||||
#include <errno.h> /* errno */
|
||||
#include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */
|
||||
#ifndef ZSTD_NOBENCH
|
||||
# include "bench.h" /* BMK_benchFiles */
|
||||
# include "benchzstd.h" /* BMK_benchFiles */
|
||||
#endif
|
||||
#ifndef ZSTD_NODICT
|
||||
# include "dibio.h" /* ZDICT_cover_params_t, DiB_trainFromFiles() */
|
||||
@ -81,7 +82,7 @@ static const unsigned g_defaultMaxWindowLog = 27;
|
||||
static U32 g_overlapLog = OVERLAP_LOG_DEFAULT;
|
||||
static U32 g_ldmHashLog = 0;
|
||||
static U32 g_ldmMinMatch = 0;
|
||||
static U32 g_ldmHashEveryLog = LDM_PARAM_DEFAULT;
|
||||
static U32 g_ldmHashRateLog = LDM_PARAM_DEFAULT;
|
||||
static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT;
|
||||
|
||||
|
||||
@ -143,6 +144,7 @@ static int usage_advanced(const char* programName)
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
DISPLAY( " -T# : spawns # compression threads (default: 1, 0==# cores) \n");
|
||||
DISPLAY( " -B# : select size of each job (default: 0==automatic) \n");
|
||||
DISPLAY( " --rsyncable : compress using a rsync-friendly method (-B sets block size) \n");
|
||||
#endif
|
||||
DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
|
||||
DISPLAY( "--[no-]check : integrity check (default: enabled) \n");
|
||||
@ -150,7 +152,7 @@ static int usage_advanced(const char* programName)
|
||||
#ifdef UTIL_HAS_CREATEFILELIST
|
||||
DISPLAY( " -r : operate recursively on directories \n");
|
||||
#endif
|
||||
DISPLAY( "--format=zstd : compress files to the .zstd format (default) \n");
|
||||
DISPLAY( "--format=zstd : compress files to the .zst format (default) \n");
|
||||
#ifdef ZSTD_GZCOMPRESS
|
||||
DISPLAY( "--format=gzip : compress files to the .gz format \n");
|
||||
#endif
|
||||
@ -170,6 +172,7 @@ static int usage_advanced(const char* programName)
|
||||
#endif
|
||||
#endif
|
||||
DISPLAY( " -M# : Set a memory usage limit for decompression \n");
|
||||
DISPLAY( "--no-progress : do not display the progress bar \n");
|
||||
DISPLAY( "-- : All arguments after \"--\" are treated as files \n");
|
||||
#ifndef ZSTD_NODICT
|
||||
DISPLAY( "\n");
|
||||
@ -231,32 +234,44 @@ static void errorOut(const char* msg)
|
||||
DISPLAY("%s \n", msg); exit(1);
|
||||
}
|
||||
|
||||
/*! readU32FromChar() :
|
||||
* @return : unsigned integer value read from input in `char` format.
|
||||
/*! readU32FromCharChecked() :
|
||||
* @return 0 if success, and store the result in *value.
|
||||
* allows and interprets K, KB, KiB, M, MB and MiB suffix.
|
||||
* Will also modify `*stringPtr`, advancing it to position where it stopped reading.
|
||||
* Note : function will exit() program if digit sequence overflows */
|
||||
static unsigned readU32FromChar(const char** stringPtr)
|
||||
* @return 1 if an overflow error occurs */
|
||||
static int readU32FromCharChecked(const char** stringPtr, unsigned* value)
|
||||
{
|
||||
const char errorMsg[] = "error: numeric value too large";
|
||||
static unsigned const max = (((unsigned)(-1)) / 10) - 1;
|
||||
unsigned result = 0;
|
||||
while ((**stringPtr >='0') && (**stringPtr <='9')) {
|
||||
unsigned const max = (((unsigned)(-1)) / 10) - 1;
|
||||
if (result > max) errorOut(errorMsg);
|
||||
if (result > max) return 1; // overflow error
|
||||
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
|
||||
}
|
||||
if ((**stringPtr=='K') || (**stringPtr=='M')) {
|
||||
unsigned const maxK = ((unsigned)(-1)) >> 10;
|
||||
if (result > maxK) errorOut(errorMsg);
|
||||
if (result > maxK) return 1; // overflow error
|
||||
result <<= 10;
|
||||
if (**stringPtr=='M') {
|
||||
if (result > maxK) errorOut(errorMsg);
|
||||
if (result > maxK) return 1; // overflow error
|
||||
result <<= 10;
|
||||
}
|
||||
(*stringPtr)++; /* skip `K` or `M` */
|
||||
if (**stringPtr=='i') (*stringPtr)++;
|
||||
if (**stringPtr=='B') (*stringPtr)++;
|
||||
}
|
||||
*value = result;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*! readU32FromChar() :
|
||||
* @return : unsigned integer value read from input in `char` format.
|
||||
* allows and interprets K, KB, KiB, M, MB and MiB suffix.
|
||||
* Will also modify `*stringPtr`, advancing it to position where it stopped reading.
|
||||
* Note : function will exit() program if digit sequence overflows */
|
||||
static unsigned readU32FromChar(const char** stringPtr) {
|
||||
static const char errorMsg[] = "error: numeric value too large";
|
||||
unsigned result;
|
||||
if (readU32FromCharChecked(stringPtr, &result)) { errorOut(errorMsg); }
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -391,7 +406,7 @@ static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, in
|
||||
|
||||
|
||||
/** parseCompressionParameters() :
|
||||
* reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params
|
||||
* reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6") into *params
|
||||
* @return 1 means that compression parameters were correct
|
||||
* @return 0 in case of malformed parameters
|
||||
*/
|
||||
@ -402,20 +417,20 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi
|
||||
if (longCommandWArg(&stringPtr, "chainLog=") || longCommandWArg(&stringPtr, "clog=")) { params->chainLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "hashLog=") || longCommandWArg(&stringPtr, "hlog=")) { params->hashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "minMatch=") || longCommandWArg(&stringPtr, "mml=")) { params->minMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "overlapLog=") || longCommandWArg(&stringPtr, "ovlog=")) { g_overlapLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "ldmHashLog=") || longCommandWArg(&stringPtr, "ldmhlog=")) { g_ldmHashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "ldmSearchLength=") || longCommandWArg(&stringPtr, "ldmslen=")) { g_ldmMinMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "ldmBucketSizeLog=") || longCommandWArg(&stringPtr, "ldmblog=")) { g_ldmBucketSizeLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "ldmHashEveryLog=") || longCommandWArg(&stringPtr, "ldmhevery=")) { g_ldmHashEveryLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "ldmHashLog=") || longCommandWArg(&stringPtr, "lhlog=")) { g_ldmHashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "ldmMinMatch=") || longCommandWArg(&stringPtr, "lmml=")) { g_ldmMinMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "ldmBucketSizeLog=") || longCommandWArg(&stringPtr, "lblog=")) { g_ldmBucketSizeLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "ldmHashRateLog=") || longCommandWArg(&stringPtr, "lhrlog=")) { g_ldmHashRateLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
DISPLAYLEVEL(4, "invalid compression parameter \n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(4, "windowLog=%d, chainLog=%d, hashLog=%d, searchLog=%d \n", params->windowLog, params->chainLog, params->hashLog, params->searchLog);
|
||||
DISPLAYLEVEL(4, "searchLength=%d, targetLength=%d, strategy=%d \n", params->searchLength, params->targetLength, params->strategy);
|
||||
DISPLAYLEVEL(4, "minMatch=%d, targetLength=%d, strategy=%d \n", params->minMatch, params->targetLength, params->strategy);
|
||||
if (stringPtr[0] != 0) return 0; /* check the end of string */
|
||||
return 1;
|
||||
}
|
||||
@ -450,6 +465,38 @@ static void printVersion(void)
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Environment variables for parameter setting */
|
||||
#define ENV_CLEVEL "ZSTD_CLEVEL"
|
||||
|
||||
/* functions that pick up environment variables */
|
||||
static int init_cLevel(void) {
|
||||
const char* const env = getenv(ENV_CLEVEL);
|
||||
if (env) {
|
||||
const char *ptr = env;
|
||||
int sign = 1;
|
||||
if (*ptr == '-') {
|
||||
sign = -1;
|
||||
ptr++;
|
||||
} else if (*ptr == '+') {
|
||||
ptr++;
|
||||
}
|
||||
|
||||
if ((*ptr>='0') && (*ptr<='9')) {
|
||||
unsigned absLevel;
|
||||
if (readU32FromCharChecked(&ptr, &absLevel)) {
|
||||
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large\n", ENV_CLEVEL, env);
|
||||
return ZSTDCLI_CLEVEL_DEFAULT;
|
||||
} else if (*ptr == 0) {
|
||||
return sign * absLevel;
|
||||
}
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid integer value\n", ENV_CLEVEL, env);
|
||||
}
|
||||
|
||||
return ZSTDCLI_CLEVEL_DEFAULT;
|
||||
}
|
||||
|
||||
typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode;
|
||||
|
||||
#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
|
||||
@ -475,6 +522,7 @@ int main(int argCount, const char* argv[])
|
||||
adapt = 0,
|
||||
adaptMin = MINCLEVEL,
|
||||
adaptMax = MAXCLEVEL,
|
||||
rsyncable = 0,
|
||||
nextArgumentIsOutFileName = 0,
|
||||
nextArgumentIsMaxDict = 0,
|
||||
nextArgumentIsDictID = 0,
|
||||
@ -490,7 +538,7 @@ int main(int argCount, const char* argv[])
|
||||
size_t blockSize = 0;
|
||||
zstd_operation_mode operation = zom_compress;
|
||||
ZSTD_compressionParameters compressionParams;
|
||||
int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
|
||||
int cLevel;
|
||||
int cLevelLast = -1000000000;
|
||||
unsigned recursive = 0;
|
||||
unsigned memLimit = 0;
|
||||
@ -525,6 +573,7 @@ int main(int argCount, const char* argv[])
|
||||
if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
|
||||
filenameTable[0] = stdinmark;
|
||||
g_displayOut = stderr;
|
||||
cLevel = init_cLevel();
|
||||
programName = lastNameFromPath(programName);
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
nbWorkers = 1;
|
||||
@ -607,6 +656,8 @@ int main(int argCount, const char* argv[])
|
||||
#ifdef ZSTD_LZ4COMPRESS
|
||||
if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); continue; }
|
||||
#endif
|
||||
if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; }
|
||||
if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; }
|
||||
|
||||
/* long commands with arguments */
|
||||
#ifndef ZSTD_NODICT
|
||||
@ -937,8 +988,8 @@ int main(int argCount, const char* argv[])
|
||||
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
|
||||
benchParams.ldmBucketSizeLog = g_ldmBucketSizeLog;
|
||||
}
|
||||
if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
|
||||
benchParams.ldmHashEveryLog = g_ldmHashEveryLog;
|
||||
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) {
|
||||
benchParams.ldmHashRateLog = g_ldmHashRateLog;
|
||||
}
|
||||
|
||||
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
|
||||
@ -1048,10 +1099,11 @@ int main(int argCount, const char* argv[])
|
||||
FIO_setLdmHashLog(g_ldmHashLog);
|
||||
FIO_setLdmMinMatch(g_ldmMinMatch);
|
||||
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(g_ldmBucketSizeLog);
|
||||
if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) FIO_setLdmHashEveryLog(g_ldmHashEveryLog);
|
||||
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(g_ldmHashRateLog);
|
||||
FIO_setAdaptiveMode(adapt);
|
||||
FIO_setAdaptMin(adaptMin);
|
||||
FIO_setAdaptMax(adaptMax);
|
||||
FIO_setRsyncable(rsyncable);
|
||||
if (adaptMin > cLevel) cLevel = adaptMin;
|
||||
if (adaptMax < cLevel) cLevel = adaptMax;
|
||||
|
||||
@ -1060,7 +1112,7 @@ int main(int argCount, const char* argv[])
|
||||
else
|
||||
operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
|
||||
#else
|
||||
(void)suffix; (void)adapt; (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */
|
||||
(void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */
|
||||
DISPLAY("Compression not supported \n");
|
||||
#endif
|
||||
} else { /* decompression or test */
|
||||
|
||||
@ -31,94 +31,101 @@ grep_args=""
|
||||
hyphen=0
|
||||
silent=0
|
||||
|
||||
prg=$(basename $0)
|
||||
prg=$(basename "$0")
|
||||
|
||||
# handle being called 'zegrep' or 'zfgrep'
|
||||
case ${prg} in
|
||||
*zegrep)
|
||||
grep_args="-E";;
|
||||
*zfgrep)
|
||||
grep_args="-F";;
|
||||
case "${prg}" in
|
||||
*zegrep) grep_args="-E";;
|
||||
*zfgrep) grep_args="-F";;
|
||||
esac
|
||||
|
||||
# skip all options and pass them on to grep taking care of options
|
||||
# with arguments, and if -e was supplied
|
||||
|
||||
while [ $# -gt 0 -a ${endofopts} -eq 0 ]
|
||||
do
|
||||
case $1 in
|
||||
while [ "$#" -gt 0 ] && [ "${endofopts}" -eq 0 ]; do
|
||||
case "$1" in
|
||||
# from GNU grep-2.5.1 -- keep in sync!
|
||||
-[ABCDXdefm])
|
||||
if [ $# -lt 2 ]
|
||||
then
|
||||
echo "${prg}: missing argument for $1 flag" >&2
|
||||
exit 1
|
||||
fi
|
||||
case $1 in
|
||||
-e)
|
||||
pattern="$2"
|
||||
pattern_found=1
|
||||
shift 2
|
||||
break
|
||||
;;
|
||||
*)
|
||||
;;
|
||||
esac
|
||||
grep_args="${grep_args} $1 $2"
|
||||
shift 2
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
endofopts=1
|
||||
;;
|
||||
-)
|
||||
hyphen=1
|
||||
shift
|
||||
;;
|
||||
-h)
|
||||
silent=1
|
||||
shift
|
||||
;;
|
||||
-*)
|
||||
grep_args="${grep_args} $1"
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
# pattern to grep for
|
||||
endofopts=1
|
||||
;;
|
||||
-[ABCDXdefm])
|
||||
if [ "$#" -lt 2 ]; then
|
||||
printf '%s: missing argument for %s flag\n' "${prg}" "$1" >&2
|
||||
exit 1
|
||||
fi
|
||||
case "$1" in
|
||||
-e)
|
||||
pattern="$2"
|
||||
pattern_found=1
|
||||
shift 2
|
||||
break
|
||||
;;
|
||||
*)
|
||||
;;
|
||||
esac
|
||||
grep_args="${grep_args} $1 $2"
|
||||
shift 2
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
endofopts=1
|
||||
;;
|
||||
-)
|
||||
hyphen=1
|
||||
shift
|
||||
;;
|
||||
-h)
|
||||
silent=1
|
||||
shift
|
||||
;;
|
||||
-*)
|
||||
grep_args="${grep_args} $1"
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
# pattern to grep for
|
||||
endofopts=1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# if no -e option was found, take next argument as grep-pattern
|
||||
if [ ${pattern_found} -lt 1 ]
|
||||
then
|
||||
if [ $# -ge 1 ]; then
|
||||
pattern="$1"
|
||||
shift
|
||||
elif [ ${hyphen} -gt 0 ]; then
|
||||
pattern="-"
|
||||
if [ "${pattern_found}" -lt 1 ]; then
|
||||
if [ "$#" -ge 1 ]; then
|
||||
pattern="$1"
|
||||
shift
|
||||
elif [ "${hyphen}" -gt 0 ]; then
|
||||
pattern="-"
|
||||
else
|
||||
echo "${prg}: missing pattern" >&2
|
||||
exit 1
|
||||
printf '%s: missing pattern\n' "${prg}" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
EXIT_CODE=0
|
||||
# call grep ...
|
||||
if [ $# -lt 1 ]
|
||||
then
|
||||
if [ "$#" -lt 1 ]; then
|
||||
# ... on stdin
|
||||
${zcat} -fq - | ${grep} ${grep_args} -- "${pattern}" -
|
||||
set -f # Disable file name generation (globbing).
|
||||
# shellcheck disable=SC2086
|
||||
"${zcat}" -fq - | "${grep}" ${grep_args} -- "${pattern}" -
|
||||
EXIT_CODE=$?
|
||||
set +f
|
||||
else
|
||||
# ... on all files given on the command line
|
||||
if [ ${silent} -lt 1 -a $# -gt 1 ]; then
|
||||
grep_args="-H ${grep_args}"
|
||||
if [ "${silent}" -lt 1 ] && [ "$#" -gt 1 ]; then
|
||||
grep_args="-H ${grep_args}"
|
||||
fi
|
||||
while [ $# -gt 0 ]
|
||||
do
|
||||
${zcat} -fq -- "$1" | ${grep} --label="${1}" ${grep_args} -- "${pattern}" -
|
||||
shift
|
||||
CUR_EXIT_CODE=0
|
||||
EXIT_CODE=1
|
||||
set -f
|
||||
while [ "$#" -gt 0 ]; do
|
||||
# shellcheck disable=SC2086
|
||||
"${zcat}" -fq -- "$1" | "${grep}" --label="${1}" ${grep_args} -- "${pattern}" -
|
||||
CUR_EXIT_CODE=$?
|
||||
if [ "${CUR_EXIT_CODE}" -eq 0 ] && [ "${EXIT_CODE}" -ne 1 ]; then
|
||||
EXIT_CODE=0
|
||||
fi
|
||||
shift
|
||||
done
|
||||
set +f
|
||||
fi
|
||||
|
||||
exit 0
|
||||
exit "${EXIT_CODE}"
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
.
|
||||
.TH "ZSTDGREP" "1" "October 2018" "zstd 1.3.7" "User Commands"
|
||||
.TH "ZSTDGREP" "1" "December 2018" "zstd 1.3.8" "User Commands"
|
||||
.
|
||||
.SH "NAME"
|
||||
\fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
.
|
||||
.TH "ZSTDLESS" "1" "October 2018" "zstd 1.3.7" "User Commands"
|
||||
.TH "ZSTDLESS" "1" "December 2018" "zstd 1.3.8" "User Commands"
|
||||
.
|
||||
.SH "NAME"
|
||||
\fBzstdless\fR \- view zstandard\-compressed files
|
||||
|
||||
@ -132,18 +132,18 @@ fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP)
|
||||
fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD)
|
||||
fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements
|
||||
fullbench fullbench32 : $(ZSTD_FILES)
|
||||
fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/bench.c fullbench.c
|
||||
fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c fullbench.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fullbench-lib : CPPFLAGS += -DXXH_NAMESPACE=ZSTD_
|
||||
fullbench-lib : zstd-staticLib
|
||||
fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/bench.c fullbench.c
|
||||
fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c fullbench.c
|
||||
$(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) $(ZSTDDIR)/libzstd.a
|
||||
|
||||
# note : broken : requires unavailable symbols
|
||||
fullbench-dll : zstd-dll
|
||||
fullbench-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
|
||||
fullbench-dll: $(PRGDIR)/datagen.c fullbench.c
|
||||
fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c fullbench.c
|
||||
# $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
|
||||
$(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT)
|
||||
|
||||
@ -152,32 +152,32 @@ fuzzer : LDFLAGS += $(MULTITHREAD_LD)
|
||||
fuzzer32: CFLAGS += -m32
|
||||
fuzzer : $(ZSTDMT_OBJECTS)
|
||||
fuzzer32: $(ZSTD_FILES)
|
||||
fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
|
||||
fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/datagen.c fuzzer.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fuzzer-dll : zstd-dll
|
||||
fuzzer-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
|
||||
fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c fuzzer.c
|
||||
fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/datagen.c fuzzer.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
|
||||
|
||||
zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated
|
||||
zbufftest : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings
|
||||
zbufftest : $(ZSTD_OBJECTS) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
|
||||
zbufftest : $(ZSTD_OBJECTS) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/datagen.c zbufftest.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zbufftest32 : CPPFLAGS += -I$(ZSTDDIR)/deprecated
|
||||
zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32
|
||||
zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
|
||||
zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/datagen.c zbufftest.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zbufftest-dll : zstd-dll
|
||||
zbufftest-dll : CPPFLAGS += -I$(ZSTDDIR)/deprecated
|
||||
zbufftest-dll : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings
|
||||
zbufftest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
|
||||
zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zbufftest.c
|
||||
zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/datagen.c zbufftest.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
|
||||
|
||||
ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c seqgen.c zstreamtest.c
|
||||
ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c seqgen.c zstreamtest.c
|
||||
ZSTREAM_PROPER_FILES := $(ZDICT_FILES) $(ZSTREAM_LOCAL_FILES)
|
||||
ZSTREAMFILES := $(ZSTD_FILES) $(ZSTREAM_PROPER_FILES)
|
||||
zstreamtest32 : CFLAGS += -m32
|
||||
@ -203,7 +203,7 @@ zstreamtest-dll : $(ZSTREAM_LOCAL_FILES)
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
|
||||
|
||||
paramgrill : DEBUGFLAGS = # turn off assert() by default for speed measurements
|
||||
paramgrill : $(ZSTD_FILES) $(PRGDIR)/bench.c $(PRGDIR)/datagen.c paramgrill.c
|
||||
paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c paramgrill.c
|
||||
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
|
||||
|
||||
datagen : $(PRGDIR)/datagen.c datagencli.c
|
||||
@ -222,7 +222,7 @@ legacy : CPPFLAGS += -I$(ZSTDDIR)/legacy -DZSTD_LEGACY_SUPPORT=4
|
||||
legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) decodecorpus.c
|
||||
decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) $(PRGDIR)/util.c decodecorpus.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT) -lm
|
||||
|
||||
symbols : symbols.c zstd-dll
|
||||
@ -233,7 +233,7 @@ else
|
||||
$(CC) $(FLAGS) $< -o $@$(EXT) -Wl,-rpath=$(ZSTDDIR) $(ZSTDDIR)/libzstd.so # broken on Mac
|
||||
endif
|
||||
|
||||
poolTests : poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c
|
||||
poolTests : $(PRGDIR)/util.c poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c
|
||||
$(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
|
||||
|
||||
.PHONY: versionsTest
|
||||
@ -320,7 +320,7 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32
|
||||
test-all: test test32 valgrindTest test-decodecorpus-cli
|
||||
|
||||
|
||||
.PHONY: test-zstd test-zstd32 test-zstd-nolegacy
|
||||
.PHONY: test-zstd test-zstd32 test-zstd-nolegacy test-zstdgrep
|
||||
test-zstd: ZSTD = $(PRGDIR)/zstd
|
||||
test-zstd: zstd
|
||||
|
||||
@ -352,6 +352,10 @@ test-gzstd: gzstd
|
||||
$(PRGDIR)/zstd -dcf - <hello_zst_gz_txt.gz
|
||||
$(RM) *.gz *.zst README2.md gz_zstd zstd_gz hello.txt
|
||||
|
||||
test-zstdgrep: gzstd
|
||||
@echo a | $(PRGDIR)/zstd | $(PRGDIR)/zstdgrep a
|
||||
@echo a | $(PRGDIR)/zstd | $(PRGDIR)/zstdgrep b && return 1 || return 0
|
||||
|
||||
test-fullbench: fullbench datagen
|
||||
$(QEMU_SYS) ./fullbench -i1
|
||||
$(QEMU_SYS) ./fullbench -i1 -P0
|
||||
|
||||
@ -41,7 +41,7 @@ Additional remarks:
|
||||
The example usage with two test files, one e-mail address, and with an additional message:
|
||||
```
|
||||
./test-zstd-speed.py "silesia.tar calgary.tar" "email@gmail.com" --message "tested on my laptop" --sleepTime 60
|
||||
```
|
||||
```
|
||||
|
||||
To run the script in background please use:
|
||||
```
|
||||
@ -100,19 +100,19 @@ Full list of arguments
|
||||
h# - hashLog
|
||||
c# - chainLog
|
||||
s# - searchLog
|
||||
l# - searchLength
|
||||
l# - minMatch
|
||||
t# - targetLength
|
||||
S# - strategy
|
||||
L# - level
|
||||
--zstd= : Single run, parameter selection syntax same as zstdcli with more parameters
|
||||
(Added forceAttachDictionary / fadt)
|
||||
When invoked with --optimize, this represents the sample to exceed.
|
||||
(Added forceAttachDictionary / fadt)
|
||||
When invoked with --optimize, this represents the sample to exceed.
|
||||
--optimize= : find parameters to maximize compression ratio given parameters
|
||||
Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints
|
||||
cSpeed= : Minimum compression speed
|
||||
dSpeed= : Minimum decompression speed
|
||||
cMem= : Maximum compression memory
|
||||
lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed,
|
||||
lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed,
|
||||
stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 100%)
|
||||
: In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters
|
||||
(Lower value will begin with stronger strategies) (Default 90%)
|
||||
@ -121,13 +121,13 @@ Full list of arguments
|
||||
when determining overall winner (default 5 (1% ratio = 5% speed)).
|
||||
tries= : Maximum number of random restarts on a single strategy before switching (Default 5)
|
||||
Higher values will make optimizer run longer, more chances to find better solution.
|
||||
memLog : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size.
|
||||
Setting memLog = 0 turns off memoization
|
||||
memLog : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size.
|
||||
Setting memLog = 0 turns off memoization
|
||||
--display= : specifiy which parameters are included in the output
|
||||
can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters
|
||||
can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters
|
||||
(Default: display all params available)
|
||||
-P# : generated sample compressibility (when no file is provided)
|
||||
-t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours ))
|
||||
-t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours ))
|
||||
-v : Prints Benchmarking output
|
||||
-D : Next argument dictionary file
|
||||
-s : Benchmark all files separately
|
||||
|
||||
@ -121,7 +121,7 @@ int main(int argc, const char** argv)
|
||||
DISPLAYLEVEL(4, "Compressible data Generator \n");
|
||||
if (probaU32!=COMPRESSIBILITY_DEFAULT)
|
||||
DISPLAYLEVEL(3, "Compressibility : %i%%\n", probaU32);
|
||||
DISPLAYLEVEL(3, "Seed = %u \n", seed);
|
||||
DISPLAYLEVEL(3, "Seed = %u \n", (unsigned)seed);
|
||||
|
||||
RDG_genStdout(size, (double)probaU32/100, litProba, seed);
|
||||
DISPLAYLEVEL(1, "\n");
|
||||
|
||||
@ -22,7 +22,7 @@
|
||||
#define ZDICT_STATIC_LINKING_ONLY
|
||||
#include "zdict.h"
|
||||
|
||||
// Direct access to internal compression functions is required
|
||||
/* Direct access to internal compression functions is required */
|
||||
#include "zstd_compress.c"
|
||||
|
||||
#define XXH_STATIC_LINKING_ONLY
|
||||
@ -72,7 +72,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
|
||||
/*-*******************************************************
|
||||
* Random function
|
||||
*********************************************************/
|
||||
static unsigned RAND(unsigned* src)
|
||||
static U32 RAND(U32* src)
|
||||
{
|
||||
#define RAND_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
static const U32 prime1 = 2654435761U;
|
||||
@ -350,7 +350,7 @@ static void writeFrameHeader(U32* seed, frame_t* frame, dictInfo info)
|
||||
}
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(3, " frame content size:\t%u\n", (U32)fh.contentSize);
|
||||
DISPLAYLEVEL(3, " frame content size:\t%u\n", (unsigned)fh.contentSize);
|
||||
DISPLAYLEVEL(3, " frame window size:\t%u\n", fh.windowSize);
|
||||
DISPLAYLEVEL(3, " content size flag:\t%d\n", contentSizeFlag);
|
||||
DISPLAYLEVEL(3, " single segment flag:\t%d\n", singleSegment);
|
||||
@ -412,7 +412,7 @@ static size_t writeLiteralsBlockSimple(U32* seed, frame_t* frame, size_t content
|
||||
/* RLE literals */
|
||||
BYTE const symb = (BYTE) (RAND(seed) % 256);
|
||||
|
||||
DISPLAYLEVEL(4, " rle literals: 0x%02x\n", (U32)symb);
|
||||
DISPLAYLEVEL(4, " rle literals: 0x%02x\n", (unsigned)symb);
|
||||
|
||||
memset(LITERAL_BUFFER, symb, litSize);
|
||||
op[0] = symb;
|
||||
@ -432,12 +432,12 @@ static size_t writeHufHeader(U32* seed, HUF_CElt* hufTable, void* dst, size_t ds
|
||||
BYTE* op = ostart;
|
||||
|
||||
unsigned huffLog = 11;
|
||||
U32 maxSymbolValue = 255;
|
||||
unsigned maxSymbolValue = 255;
|
||||
|
||||
U32 count[HUF_SYMBOLVALUE_MAX+1];
|
||||
unsigned count[HUF_SYMBOLVALUE_MAX+1];
|
||||
|
||||
/* Scan input and build symbol stats */
|
||||
{ size_t const largest = HIST_count_wksp (count, &maxSymbolValue, (const BYTE*)src, srcSize, WKSP);
|
||||
{ size_t const largest = HIST_count_wksp (count, &maxSymbolValue, (const BYTE*)src, srcSize, WKSP, sizeof(WKSP));
|
||||
assert(!HIST_isError(largest));
|
||||
if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 0; } /* single symbol, rle */
|
||||
if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */
|
||||
@ -568,8 +568,8 @@ static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t con
|
||||
op += compressedSize;
|
||||
|
||||
compressedSize += hufHeaderSize;
|
||||
DISPLAYLEVEL(5, " regenerated size: %u\n", (U32)litSize);
|
||||
DISPLAYLEVEL(5, " compressed size: %u\n", (U32)compressedSize);
|
||||
DISPLAYLEVEL(5, " regenerated size: %u\n", (unsigned)litSize);
|
||||
DISPLAYLEVEL(5, " compressed size: %u\n", (unsigned)compressedSize);
|
||||
if (compressedSize >= litSize) {
|
||||
DISPLAYLEVEL(5, " trying again\n");
|
||||
/* if we have to try again, reset the stats so we don't accidentally
|
||||
@ -656,7 +656,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
|
||||
excessMatch = remainingMatch - numSequences * MIN_SEQ_LEN;
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(5, " total match lengths: %u\n", (U32)remainingMatch);
|
||||
DISPLAYLEVEL(5, " total match lengths: %u\n", (unsigned)remainingMatch);
|
||||
for (i = 0; i < numSequences; i++) {
|
||||
/* Generate match and literal lengths by exponential distribution to
|
||||
* ensure nice numbers */
|
||||
@ -748,12 +748,13 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
|
||||
frame->stats.rep[0] = offset;
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(6, " LL: %5u OF: %5u ML: %5u", literalLen, offset, matchLen);
|
||||
DISPLAYLEVEL(6, " LL: %5u OF: %5u ML: %5u",
|
||||
(unsigned)literalLen, (unsigned)offset, (unsigned)matchLen);
|
||||
DISPLAYLEVEL(7, " srcPos: %8u seqNb: %3u",
|
||||
(U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart), i);
|
||||
(unsigned)((BYTE*)srcPtr - (BYTE*)frame->srcStart), (unsigned)i);
|
||||
DISPLAYLEVEL(6, "\n");
|
||||
if (offsetCode < 3) {
|
||||
DISPLAYLEVEL(7, " repeat offset: %d\n", repIndex);
|
||||
DISPLAYLEVEL(7, " repeat offset: %d\n", (int)repIndex);
|
||||
}
|
||||
/* use libzstd sequence handling */
|
||||
ZSTD_storeSeq(seqStore, literalLen, literals, offsetCode,
|
||||
@ -766,8 +767,8 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
|
||||
|
||||
memcpy(srcPtr, literals, literalsSize);
|
||||
srcPtr += literalsSize;
|
||||
DISPLAYLEVEL(6, " excess literals: %5u", (U32)literalsSize);
|
||||
DISPLAYLEVEL(7, " srcPos: %8u", (U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart));
|
||||
DISPLAYLEVEL(6, " excess literals: %5u", (unsigned)literalsSize);
|
||||
DISPLAYLEVEL(7, " srcPos: %8u", (unsigned)((BYTE*)srcPtr - (BYTE*)frame->srcStart));
|
||||
DISPLAYLEVEL(6, "\n");
|
||||
|
||||
return numSequences;
|
||||
@ -800,7 +801,7 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr,
|
||||
size_t nbSeq)
|
||||
{
|
||||
/* This code is mostly copied from ZSTD_compressSequences in zstd_compress.c */
|
||||
U32 count[MaxSeq+1];
|
||||
unsigned count[MaxSeq+1];
|
||||
S16 norm[MaxSeq+1];
|
||||
FSE_CTable* CTable_LitLength = frame->stats.litlengthCTable;
|
||||
FSE_CTable* CTable_OffsetBits = frame->stats.offcodeCTable;
|
||||
@ -823,21 +824,20 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr,
|
||||
else if (nbSeq < LONGNBSEQ) op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2;
|
||||
else op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3;
|
||||
|
||||
/* seqHead : flags for FSE encoding type */
|
||||
seqHead = op++;
|
||||
|
||||
if (nbSeq==0) {
|
||||
frame->data = op;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* seqHead : flags for FSE encoding type */
|
||||
seqHead = op++;
|
||||
|
||||
/* convert length/distances into codes */
|
||||
ZSTD_seqToCodes(seqStorePtr);
|
||||
|
||||
/* CTable for Literal Lengths */
|
||||
{ U32 max = MaxLL;
|
||||
size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, WKSP); /* cannot fail */
|
||||
{ unsigned max = MaxLL;
|
||||
size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, WKSP, sizeof(WKSP)); /* cannot fail */
|
||||
assert(!HIST_isError(mostFrequent));
|
||||
if (mostFrequent == nbSeq) {
|
||||
/* do RLE if we have the chance */
|
||||
@ -868,8 +868,8 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr,
|
||||
|
||||
/* CTable for Offsets */
|
||||
/* see Literal Lengths for descriptions of mode choices */
|
||||
{ U32 max = MaxOff;
|
||||
size_t const mostFrequent = HIST_countFast_wksp(count, &max, ofCodeTable, nbSeq, WKSP); /* cannot fail */
|
||||
{ unsigned max = MaxOff;
|
||||
size_t const mostFrequent = HIST_countFast_wksp(count, &max, ofCodeTable, nbSeq, WKSP, sizeof(WKSP)); /* cannot fail */
|
||||
assert(!HIST_isError(mostFrequent));
|
||||
if (mostFrequent == nbSeq) {
|
||||
*op++ = ofCodeTable[0];
|
||||
@ -896,8 +896,8 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr,
|
||||
|
||||
/* CTable for MatchLengths */
|
||||
/* see Literal Lengths for descriptions of mode choices */
|
||||
{ U32 max = MaxML;
|
||||
size_t const mostFrequent = HIST_countFast_wksp(count, &max, mlCodeTable, nbSeq, WKSP); /* cannot fail */
|
||||
{ unsigned max = MaxML;
|
||||
size_t const mostFrequent = HIST_countFast_wksp(count, &max, mlCodeTable, nbSeq, WKSP, sizeof(WKSP)); /* cannot fail */
|
||||
assert(!HIST_isError(mostFrequent));
|
||||
if (mostFrequent == nbSeq) {
|
||||
*op++ = *mlCodeTable;
|
||||
@ -928,7 +928,7 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr,
|
||||
initSymbolSet(ofCodeTable, nbSeq, frame->stats.offsetSymbolSet, 28);
|
||||
initSymbolSet(mlCodeTable, nbSeq, frame->stats.matchlengthSymbolSet, 52);
|
||||
|
||||
DISPLAYLEVEL(5, " LL type: %d OF type: %d ML type: %d\n", LLtype, Offtype, MLtype);
|
||||
DISPLAYLEVEL(5, " LL type: %d OF type: %d ML type: %d\n", (unsigned)LLtype, (unsigned)Offtype, (unsigned)MLtype);
|
||||
|
||||
*seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
|
||||
|
||||
@ -1015,11 +1015,11 @@ static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize
|
||||
|
||||
literalsSize = writeLiteralsBlock(seed, frame, contentSize);
|
||||
|
||||
DISPLAYLEVEL(4, " literals size: %u\n", (U32)literalsSize);
|
||||
DISPLAYLEVEL(4, " literals size: %u\n", (unsigned)literalsSize);
|
||||
|
||||
nbSeq = writeSequencesBlock(seed, frame, contentSize, literalsSize, info);
|
||||
|
||||
DISPLAYLEVEL(4, " number of sequences: %u\n", (U32)nbSeq);
|
||||
DISPLAYLEVEL(4, " number of sequences: %u\n", (unsigned)nbSeq);
|
||||
|
||||
return (BYTE*)frame->data - blockStart;
|
||||
}
|
||||
@ -1035,7 +1035,7 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize,
|
||||
BYTE *op = header + 3;
|
||||
|
||||
DISPLAYLEVEL(4, " block:\n");
|
||||
DISPLAYLEVEL(4, " block content size: %u\n", (U32)contentSize);
|
||||
DISPLAYLEVEL(4, " block content size: %u\n", (unsigned)contentSize);
|
||||
DISPLAYLEVEL(4, " last block: %s\n", lastBlock ? "yes" : "no");
|
||||
|
||||
if (blockTypeDesc == 0) {
|
||||
@ -1083,7 +1083,7 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize,
|
||||
frame->src = (BYTE*)frame->src + contentSize;
|
||||
|
||||
DISPLAYLEVEL(4, " block type: %s\n", BLOCK_TYPES[blockType]);
|
||||
DISPLAYLEVEL(4, " block size field: %u\n", (U32)blockSize);
|
||||
DISPLAYLEVEL(4, " block size field: %u\n", (unsigned)blockSize);
|
||||
|
||||
header[0] = (BYTE) ((lastBlock | (blockType << 1) | (blockSize << 3)) & 0xff);
|
||||
MEM_writeLE16(header + 1, (U16) (blockSize >> 5));
|
||||
@ -1125,7 +1125,7 @@ static void writeChecksum(frame_t* frame)
|
||||
{
|
||||
/* write checksum so implementations can verify their output */
|
||||
U64 digest = XXH64(frame->srcStart, (BYTE*)frame->src-(BYTE*)frame->srcStart, 0);
|
||||
DISPLAYLEVEL(3, " checksum: %08x\n", (U32)digest);
|
||||
DISPLAYLEVEL(3, " checksum: %08x\n", (unsigned)digest);
|
||||
MEM_writeLE32(frame->data, (U32)digest);
|
||||
frame->data = (BYTE*)frame->data + 4;
|
||||
}
|
||||
@ -1186,7 +1186,7 @@ static U32 generateCompressedBlock(U32 seed, frame_t* frame, dictInfo info)
|
||||
size_t blockContentSize;
|
||||
int blockWritten = 0;
|
||||
BYTE* op;
|
||||
DISPLAYLEVEL(4, "block seed: %u\n", seed);
|
||||
DISPLAYLEVEL(4, "block seed: %u\n", (unsigned)seed);
|
||||
initFrame(frame);
|
||||
op = (BYTE*)frame->data;
|
||||
|
||||
@ -1223,7 +1223,7 @@ static U32 generateCompressedBlock(U32 seed, frame_t* frame, dictInfo info)
|
||||
DISPLAYLEVEL(5, " can't compress block : try again \n");
|
||||
} else {
|
||||
blockWritten = 1;
|
||||
DISPLAYLEVEL(4, " block size: %u \n", (U32)cSize);
|
||||
DISPLAYLEVEL(4, " block size: %u \n", (unsigned)cSize);
|
||||
frame->src = (BYTE*)frame->src + blockContentSize;
|
||||
}
|
||||
}
|
||||
@ -1234,7 +1234,7 @@ static U32 generateCompressedBlock(U32 seed, frame_t* frame, dictInfo info)
|
||||
static U32 generateFrame(U32 seed, frame_t* fr, dictInfo info)
|
||||
{
|
||||
/* generate a complete frame */
|
||||
DISPLAYLEVEL(3, "frame seed: %u\n", seed);
|
||||
DISPLAYLEVEL(3, "frame seed: %u\n", (unsigned)seed);
|
||||
initFrame(fr);
|
||||
|
||||
writeFrameHeader(&seed, fr, info);
|
||||
@ -1480,8 +1480,8 @@ static int runBlockTest(U32* seed)
|
||||
|
||||
{ size_t const r = testDecodeRawBlock(&fr);
|
||||
if (ZSTD_isError(r)) {
|
||||
DISPLAY("Error in block mode on test seed %u: %s\n", seedCopy,
|
||||
ZSTD_getErrorName(r));
|
||||
DISPLAY("Error in block mode on test seed %u: %s\n",
|
||||
(unsigned)seedCopy, ZSTD_getErrorName(r));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@ -1489,7 +1489,7 @@ static int runBlockTest(U32* seed)
|
||||
{ size_t const r = testDecodeWithDict(*seed, gt_block);
|
||||
if (ZSTD_isError(r)) {
|
||||
DISPLAY("Error in block mode with dictionary on test seed %u: %s\n",
|
||||
seedCopy, ZSTD_getErrorName(r));
|
||||
(unsigned)seedCopy, ZSTD_getErrorName(r));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@ -1507,21 +1507,21 @@ static int runFrameTest(U32* seed)
|
||||
{ size_t const r = testDecodeSimple(&fr);
|
||||
if (ZSTD_isError(r)) {
|
||||
DISPLAY("Error in simple mode on test seed %u: %s\n",
|
||||
seedCopy, ZSTD_getErrorName(r));
|
||||
(unsigned)seedCopy, ZSTD_getErrorName(r));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
{ size_t const r = testDecodeStreaming(&fr);
|
||||
if (ZSTD_isError(r)) {
|
||||
DISPLAY("Error in streaming mode on test seed %u: %s\n",
|
||||
seedCopy, ZSTD_getErrorName(r));
|
||||
(unsigned)seedCopy, ZSTD_getErrorName(r));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
{ size_t const r = testDecodeWithDict(*seed, gt_frame); /* avoid big dictionaries */
|
||||
if (ZSTD_isError(r)) {
|
||||
DISPLAY("Error in dictionary mode on test seed %u: %s\n",
|
||||
seedCopy, ZSTD_getErrorName(r));
|
||||
(unsigned)seedCopy, ZSTD_getErrorName(r));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@ -1538,7 +1538,7 @@ static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS
|
||||
|
||||
if (numFiles == 0 && !testDurationS) numFiles = 1;
|
||||
|
||||
DISPLAY("seed: %u\n", seed);
|
||||
DISPLAY("seed: %u\n", (unsigned)seed);
|
||||
|
||||
for (fnum = 0; fnum < numFiles || UTIL_clockSpanMicro(startClock) < maxClockSpan; fnum++) {
|
||||
if (fnum < numFiles)
|
||||
@ -1568,7 +1568,7 @@ static int generateFile(U32 seed, const char* const path,
|
||||
{
|
||||
frame_t fr;
|
||||
|
||||
DISPLAY("seed: %u\n", seed);
|
||||
DISPLAY("seed: %u\n", (unsigned)seed);
|
||||
|
||||
{ dictInfo const info = initDictInfo(0, 0, NULL, 0);
|
||||
if (genType == gt_frame) {
|
||||
@ -1590,7 +1590,7 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path,
|
||||
char outPath[MAX_PATH];
|
||||
unsigned fnum;
|
||||
|
||||
DISPLAY("seed: %u\n", seed);
|
||||
DISPLAY("seed: %u\n", (unsigned)seed);
|
||||
|
||||
for (fnum = 0; fnum < numFiles; fnum++) {
|
||||
frame_t fr;
|
||||
|
||||
@ -19,7 +19,7 @@
|
||||
|
||||
#include "mem.h" /* U32 */
|
||||
#ifndef ZSTD_DLL_IMPORT
|
||||
#include "zstd_internal.h" /* ZSTD_blockHeaderSize, blockType_e, KB, MB */
|
||||
#include "zstd_internal.h" /* ZSTD_decodeSeqHeaders, ZSTD_blockHeaderSize, blockType_e, KB, MB */
|
||||
#else
|
||||
#define KB *(1 <<10)
|
||||
#define MB *(1 <<20)
|
||||
@ -30,7 +30,8 @@
|
||||
#include "zstd.h" /* ZSTD_versionString */
|
||||
#include "util.h" /* time functions */
|
||||
#include "datagen.h"
|
||||
#include "bench.h" /* CustomBench*/
|
||||
#include "benchfn.h" /* CustomBench*/
|
||||
#include "benchzstd.h" /* MB_UNIT */
|
||||
|
||||
|
||||
/*_************************************
|
||||
@ -63,10 +64,10 @@ static const size_t g_sampleSize = 10000000;
|
||||
/*_************************************
|
||||
* Benchmark Parameters
|
||||
**************************************/
|
||||
static U32 g_nbIterations = NBLOOPS;
|
||||
static unsigned g_nbIterations = NBLOOPS;
|
||||
static double g_compressibility = COMPRESSIBILITY_DEFAULT;
|
||||
|
||||
static void BMK_SetNbIterations(U32 nbLoops)
|
||||
static void BMK_SetNbIterations(unsigned nbLoops)
|
||||
{
|
||||
g_nbIterations = nbLoops;
|
||||
DISPLAY("- %i iterations -\n", g_nbIterations);
|
||||
@ -133,7 +134,6 @@ static size_t local_ZSTD_decodeLiteralsBlock(const void* src, size_t srcSize, vo
|
||||
return ZSTD_decodeLiteralsBlock((ZSTD_DCtx*)g_zdc, buff2, g_cSize);
|
||||
}
|
||||
|
||||
extern size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeq, const void* src, size_t srcSize);
|
||||
static size_t local_ZSTD_decodeSeqHeaders(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
|
||||
{
|
||||
int nbSeq;
|
||||
@ -171,17 +171,8 @@ local_ZSTD_compress_generic_end(const void* src, size_t srcSize,
|
||||
void* dst, size_t dstCapacity,
|
||||
void* buff2)
|
||||
{
|
||||
ZSTD_outBuffer buffOut;
|
||||
ZSTD_inBuffer buffIn;
|
||||
(void)buff2;
|
||||
buffOut.dst = dst;
|
||||
buffOut.size = dstCapacity;
|
||||
buffOut.pos = 0;
|
||||
buffIn.src = src;
|
||||
buffIn.size = srcSize;
|
||||
buffIn.pos = 0;
|
||||
ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end);
|
||||
return buffOut.pos;
|
||||
return ZSTD_compress2(g_cstream, dst, dstCapacity, src, srcSize);
|
||||
}
|
||||
|
||||
static size_t
|
||||
@ -198,8 +189,8 @@ local_ZSTD_compress_generic_continue(const void* src, size_t srcSize,
|
||||
buffIn.src = src;
|
||||
buffIn.size = srcSize;
|
||||
buffIn.pos = 0;
|
||||
ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_continue);
|
||||
ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end);
|
||||
ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_continue);
|
||||
ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_end);
|
||||
return buffOut.pos;
|
||||
}
|
||||
|
||||
@ -208,18 +199,9 @@ local_ZSTD_compress_generic_T2_end(const void* src, size_t srcSize,
|
||||
void* dst, size_t dstCapacity,
|
||||
void* buff2)
|
||||
{
|
||||
ZSTD_outBuffer buffOut;
|
||||
ZSTD_inBuffer buffIn;
|
||||
(void)buff2;
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2);
|
||||
buffOut.dst = dst;
|
||||
buffOut.size = dstCapacity;
|
||||
buffOut.pos = 0;
|
||||
buffIn.src = src;
|
||||
buffIn.size = srcSize;
|
||||
buffIn.pos = 0;
|
||||
while (ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end)) {}
|
||||
return buffOut.pos;
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_nbWorkers, 2);
|
||||
return ZSTD_compress2(g_cstream, dst, dstCapacity, src, srcSize);
|
||||
}
|
||||
|
||||
static size_t
|
||||
@ -230,15 +212,15 @@ local_ZSTD_compress_generic_T2_continue(const void* src, size_t srcSize,
|
||||
ZSTD_outBuffer buffOut;
|
||||
ZSTD_inBuffer buffIn;
|
||||
(void)buff2;
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_nbWorkers, 2);
|
||||
buffOut.dst = dst;
|
||||
buffOut.size = dstCapacity;
|
||||
buffOut.pos = 0;
|
||||
buffIn.src = src;
|
||||
buffIn.size = srcSize;
|
||||
buffIn.pos = 0;
|
||||
ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_continue);
|
||||
while(ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end)) {}
|
||||
ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_continue);
|
||||
while(ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_end)) {}
|
||||
return buffOut.pos;
|
||||
}
|
||||
|
||||
@ -334,7 +316,7 @@ static size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize,
|
||||
/*_*******************************************************
|
||||
* Bench functions
|
||||
*********************************************************/
|
||||
static size_t benchMem(U32 benchNb,
|
||||
static size_t benchMem(unsigned benchNb,
|
||||
const void* src, size_t srcSize,
|
||||
int cLevel, ZSTD_compressionParameters cparams)
|
||||
{
|
||||
@ -408,28 +390,28 @@ static size_t benchMem(U32 benchNb,
|
||||
if (g_cstream==NULL) g_cstream = ZSTD_createCStream();
|
||||
if (g_dstream==NULL) g_dstream = ZSTD_createDStream();
|
||||
|
||||
/* DISPLAY("params: cLevel %d, wlog %d hlog %d clog %d slog %d slen %d tlen %d strat %d \n",
|
||||
/* DISPLAY("params: cLevel %d, wlog %d hlog %d clog %d slog %d mml %d tlen %d strat %d \n",
|
||||
cLevel, cparams->windowLog, cparams->hashLog, cparams->chainLog, cparams->searchLog,
|
||||
cparams->searchLength, cparams->targetLength, cparams->strategy); */
|
||||
cparams->minMatch, cparams->targetLength, cparams->strategy); */
|
||||
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionLevel, cLevel);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_windowLog, cparams.windowLog);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_hashLog, cparams.hashLog);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_chainLog, cparams.chainLog);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_searchLog, cparams.searchLog);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_minMatch, cparams.searchLength);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_targetLength, cparams.targetLength);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionStrategy, cparams.strategy);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_compressionLevel, cLevel);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_windowLog, cparams.windowLog);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_hashLog, cparams.hashLog);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_chainLog, cparams.chainLog);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_searchLog, cparams.searchLog);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_minMatch, cparams.minMatch);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_targetLength, cparams.targetLength);
|
||||
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_strategy, cparams.strategy);
|
||||
|
||||
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, cLevel);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_windowLog, cparams.windowLog);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_hashLog, cparams.hashLog);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_chainLog, cparams.chainLog);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_searchLog, cparams.searchLog);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_minMatch, cparams.searchLength);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_targetLength, cparams.targetLength);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionStrategy, cparams.strategy);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_compressionLevel, cLevel);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_windowLog, cparams.windowLog);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_hashLog, cparams.hashLog);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_chainLog, cparams.chainLog);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_searchLog, cparams.searchLog);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_minMatch, cparams.minMatch);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_targetLength, cparams.targetLength);
|
||||
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_strategy, cparams.strategy);
|
||||
|
||||
/* Preparation */
|
||||
switch(benchNb)
|
||||
@ -455,8 +437,8 @@ static size_t benchMem(U32 benchNb,
|
||||
ZSTD_frameHeader zfp;
|
||||
size_t frameHeaderSize, skippedSize;
|
||||
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
|
||||
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
|
||||
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
|
||||
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_FRAMEHEADERSIZE_MIN);
|
||||
if (frameHeaderSize==0) frameHeaderSize = ZSTD_FRAMEHEADERSIZE_MIN;
|
||||
ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */
|
||||
if (bp.blockType != bt_compressed) {
|
||||
DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
|
||||
@ -476,8 +458,8 @@ static size_t benchMem(U32 benchNb,
|
||||
size_t frameHeaderSize, cBlockSize;
|
||||
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); /* it would be better to use direct block compression here */
|
||||
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
|
||||
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
|
||||
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
|
||||
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_FRAMEHEADERSIZE_MIN);
|
||||
if (frameHeaderSize==0) frameHeaderSize = ZSTD_FRAMEHEADERSIZE_MIN;
|
||||
ip += frameHeaderSize; /* Skip frame Header */
|
||||
cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */
|
||||
if (bp.blockType != bt_compressed) {
|
||||
@ -515,20 +497,28 @@ static size_t benchMem(U32 benchNb,
|
||||
|
||||
/* benchmark loop */
|
||||
{ BMK_timedFnState_t* const tfs = BMK_createTimedFnState(g_nbIterations * 1000, 1000);
|
||||
void* const avoidStrictAliasingPtr = &dstBuff;
|
||||
BMK_benchParams_t bp;
|
||||
BMK_runTime_t bestResult;
|
||||
bestResult.sumOfReturn = 0;
|
||||
bestResult.nanoSecPerRun = (unsigned long long)(-1LL);
|
||||
assert(tfs != NULL);
|
||||
|
||||
bp.benchFn = benchFunction;
|
||||
bp.benchPayload = buff2;
|
||||
bp.initFn = NULL;
|
||||
bp.initPayload = NULL;
|
||||
bp.errorFn = ZSTD_isError;
|
||||
bp.blockCount = 1;
|
||||
bp.srcBuffers = &src;
|
||||
bp.srcSizes = &srcSize;
|
||||
bp.dstBuffers = (void* const*) avoidStrictAliasingPtr; /* circumvent strict aliasing warning on gcc-8,
|
||||
* because gcc considers that `void* const *` and `void**` are 2 different types */
|
||||
bp.dstCapacities = &dstBuffSize;
|
||||
bp.blockResults = NULL;
|
||||
|
||||
for (;;) {
|
||||
void* const dstBuffv = dstBuff;
|
||||
BMK_runOutcome_t const bOutcome =
|
||||
BMK_benchTimedFn( tfs,
|
||||
benchFunction, buff2,
|
||||
NULL, NULL, /* initFn */
|
||||
1, /* blockCount */
|
||||
&src, &srcSize,
|
||||
&dstBuffv, &dstBuffSize,
|
||||
NULL);
|
||||
BMK_runOutcome_t const bOutcome = BMK_benchTimedFn(tfs, bp);
|
||||
|
||||
if (!BMK_isSuccessful_runOutcome(bOutcome)) {
|
||||
DISPLAY("ERROR benchmarking function ! ! \n");
|
||||
@ -616,7 +606,7 @@ static int benchFiles(U32 benchNb,
|
||||
benchedSize = (size_t)inFileSize;
|
||||
if ((U64)benchedSize < inFileSize) {
|
||||
DISPLAY("Not enough memory for '%s' full size; testing %u MB only... \n",
|
||||
inFileName, (U32)(benchedSize>>20));
|
||||
inFileName, (unsigned)(benchedSize>>20));
|
||||
} }
|
||||
|
||||
/* Alloc */
|
||||
@ -744,7 +734,7 @@ int main(int argc, const char** argv)
|
||||
if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { cparams.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { cparams.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { cparams.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { cparams.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "minMatch=") || longCommandWArg(&argument, "mml=")) { cparams.minMatch = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { cparams.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { cparams.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = (int)readU32FromChar(&argument); cparams = ZSTD_getCParams(cLevel, 0, 0); if (argument[0]==',') { argument++; continue; } else break; }
|
||||
|
||||
@ -41,7 +41,7 @@ FUZZ_ARFLAGS := $(ARFLAGS)
|
||||
FUZZ_TARGET_FLAGS = $(FUZZ_CPPFLAGS) $(FUZZ_CXXFLAGS) $(FUZZ_LDFLAGS)
|
||||
|
||||
FUZZ_HEADERS := fuzz_helpers.h fuzz.h zstd_helpers.h
|
||||
FUZZ_SRC := zstd_helpers.c
|
||||
FUZZ_SRC := $(PRGDIR)/util.c zstd_helpers.c
|
||||
|
||||
ZSTDCOMMON_SRC := $(ZSTDDIR)/common/*.c
|
||||
ZSTDCOMP_SRC := $(ZSTDDIR)/compress/*.c
|
||||
@ -90,7 +90,7 @@ stream_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) stream_decompress.o
|
||||
block_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) block_decompress.o
|
||||
$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) block_decompress.o $(LIB_FUZZING_ENGINE) -o $@
|
||||
|
||||
libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h regression_driver.o
|
||||
libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h $(PRGDIR)/util.c regression_driver.o
|
||||
$(AR) $(FUZZ_ARFLAGS) $@ regression_driver.o
|
||||
|
||||
# Install libfuzzer (not usable for MSAN testing)
|
||||
@ -99,7 +99,7 @@ libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h regression_driver.o
|
||||
.PHONY: libFuzzer
|
||||
libFuzzer:
|
||||
@$(RM) -rf Fuzzer
|
||||
@git clone https://chromium.googlesource.com/chromium/llvm-project/llvm/lib/Fuzzer
|
||||
@git clone https://chromium.googlesource.com/chromium/llvm-project/compiler-rt/lib/fuzzer Fuzzer
|
||||
@cd Fuzzer && ./build.sh
|
||||
|
||||
corpora/%_seed_corpus.zip:
|
||||
|
||||
@ -40,9 +40,9 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
|
||||
ZSTD_outBuffer out = {compressed, compressedCapacity, 0};
|
||||
size_t err;
|
||||
|
||||
ZSTD_CCtx_reset(cctx);
|
||||
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
|
||||
FUZZ_setRandomParameters(cctx, srcSize, &seed);
|
||||
err = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
|
||||
err = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
|
||||
FUZZ_ZASSERT(err);
|
||||
FUZZ_ASSERT(err == 0);
|
||||
cSize = out.pos;
|
||||
|
||||
@ -56,7 +56,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
|
||||
const uint8_t *src, size_t srcSize)
|
||||
{
|
||||
size_t dstSize = 0;
|
||||
ZSTD_CCtx_reset(cctx);
|
||||
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
|
||||
FUZZ_setRandomParameters(cctx, srcSize, &seed);
|
||||
|
||||
while (srcSize > 0) {
|
||||
@ -72,7 +72,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
|
||||
case 1: /* fall-though */
|
||||
case 2: {
|
||||
size_t const ret =
|
||||
ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_flush);
|
||||
ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush);
|
||||
FUZZ_ZASSERT(ret);
|
||||
if (ret == 0)
|
||||
mode = -1;
|
||||
@ -80,11 +80,11 @@ static size_t compress(uint8_t *dst, size_t capacity,
|
||||
}
|
||||
case 3: {
|
||||
size_t ret =
|
||||
ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
|
||||
ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
|
||||
FUZZ_ZASSERT(ret);
|
||||
/* Reset the compressor when the frame is finished */
|
||||
if (ret == 0) {
|
||||
ZSTD_CCtx_reset(cctx);
|
||||
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
|
||||
if ((FUZZ_rand(&seed) & 7) == 0) {
|
||||
size_t const remaining = in.size - in.pos;
|
||||
FUZZ_setRandomParameters(cctx, remaining, &seed);
|
||||
@ -95,7 +95,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
|
||||
}
|
||||
default: {
|
||||
size_t const ret =
|
||||
ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue);
|
||||
ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
|
||||
FUZZ_ZASSERT(ret);
|
||||
mode = -1;
|
||||
}
|
||||
@ -108,7 +108,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
|
||||
for (;;) {
|
||||
ZSTD_inBuffer in = {NULL, 0, 0};
|
||||
ZSTD_outBuffer out = makeOutBuffer(dst, capacity);
|
||||
size_t const ret = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
|
||||
size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
|
||||
FUZZ_ZASSERT(ret);
|
||||
|
||||
dst += out.pos;
|
||||
|
||||
@ -13,7 +13,7 @@
|
||||
#include "fuzz_helpers.h"
|
||||
#include "zstd.h"
|
||||
|
||||
static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned value)
|
||||
static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, int value)
|
||||
{
|
||||
FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value));
|
||||
}
|
||||
@ -32,10 +32,10 @@ ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state)
|
||||
cParams.hashLog = FUZZ_rand32(state, ZSTD_HASHLOG_MIN, 15);
|
||||
cParams.chainLog = FUZZ_rand32(state, ZSTD_CHAINLOG_MIN, 16);
|
||||
cParams.searchLog = FUZZ_rand32(state, ZSTD_SEARCHLOG_MIN, 9);
|
||||
cParams.searchLength = FUZZ_rand32(state, ZSTD_SEARCHLENGTH_MIN,
|
||||
ZSTD_SEARCHLENGTH_MAX);
|
||||
cParams.minMatch = FUZZ_rand32(state, ZSTD_MINMATCH_MIN,
|
||||
ZSTD_MINMATCH_MAX);
|
||||
cParams.targetLength = FUZZ_rand32(state, 0, 512);
|
||||
cParams.strategy = FUZZ_rand32(state, ZSTD_fast, ZSTD_btultra);
|
||||
cParams.strategy = FUZZ_rand32(state, ZSTD_STRATEGY_MIN, ZSTD_STRATEGY_MAX);
|
||||
return ZSTD_adjustCParams(cParams, srcSize, 0);
|
||||
}
|
||||
|
||||
@ -60,25 +60,25 @@ ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state)
|
||||
void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state)
|
||||
{
|
||||
ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, state);
|
||||
set(cctx, ZSTD_p_windowLog, cParams.windowLog);
|
||||
set(cctx, ZSTD_p_hashLog, cParams.hashLog);
|
||||
set(cctx, ZSTD_p_chainLog, cParams.chainLog);
|
||||
set(cctx, ZSTD_p_searchLog, cParams.searchLog);
|
||||
set(cctx, ZSTD_p_minMatch, cParams.searchLength);
|
||||
set(cctx, ZSTD_p_targetLength, cParams.targetLength);
|
||||
set(cctx, ZSTD_p_compressionStrategy, cParams.strategy);
|
||||
set(cctx, ZSTD_c_windowLog, cParams.windowLog);
|
||||
set(cctx, ZSTD_c_hashLog, cParams.hashLog);
|
||||
set(cctx, ZSTD_c_chainLog, cParams.chainLog);
|
||||
set(cctx, ZSTD_c_searchLog, cParams.searchLog);
|
||||
set(cctx, ZSTD_c_minMatch, cParams.minMatch);
|
||||
set(cctx, ZSTD_c_targetLength, cParams.targetLength);
|
||||
set(cctx, ZSTD_c_strategy, cParams.strategy);
|
||||
/* Select frame parameters */
|
||||
setRand(cctx, ZSTD_p_contentSizeFlag, 0, 1, state);
|
||||
setRand(cctx, ZSTD_p_checksumFlag, 0, 1, state);
|
||||
setRand(cctx, ZSTD_p_dictIDFlag, 0, 1, state);
|
||||
setRand(cctx, ZSTD_p_forceAttachDict, -2, 2, state);
|
||||
setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, state);
|
||||
setRand(cctx, ZSTD_c_checksumFlag, 0, 1, state);
|
||||
setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, state);
|
||||
setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, state);
|
||||
/* Select long distance matchig parameters */
|
||||
setRand(cctx, ZSTD_p_enableLongDistanceMatching, 0, 1, state);
|
||||
setRand(cctx, ZSTD_p_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state);
|
||||
setRand(cctx, ZSTD_p_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN,
|
||||
setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, state);
|
||||
setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state);
|
||||
setRand(cctx, ZSTD_c_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN,
|
||||
ZSTD_LDM_MINMATCH_MAX, state);
|
||||
setRand(cctx, ZSTD_p_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX,
|
||||
setRand(cctx, ZSTD_c_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX,
|
||||
state);
|
||||
setRand(cctx, ZSTD_p_ldmHashEveryLog, 0,
|
||||
ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN, state);
|
||||
setRand(cctx, ZSTD_c_ldmHashRateLog, ZSTD_LDM_HASHRATELOG_MIN,
|
||||
ZSTD_LDM_HASHRATELOG_MAX, state);
|
||||
}
|
||||
|
||||
576
tests/fuzzer.c
576
tests/fuzzer.c
File diff suppressed because it is too large
Load Diff
@ -1,36 +1,89 @@
|
||||
#!/bin/sh -e
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
|
||||
ECHO=echo
|
||||
RM="rm -f"
|
||||
GREP="grep"
|
||||
INTOVOID="/dev/null"
|
||||
|
||||
die() {
|
||||
$ECHO "$@" 1>&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
isPresent() {
|
||||
$GREP $@ tmplog || die "$@" "should be present"
|
||||
}
|
||||
|
||||
INTOVOID="/dev/null"
|
||||
case "$OS" in
|
||||
Windows*)
|
||||
INTOVOID="NUL"
|
||||
;;
|
||||
esac
|
||||
mustBeAbsent() {
|
||||
$GREP $@ tmplog && die "$@ should not be there !!"
|
||||
$ECHO "$@ correctly not present" # for some reason, this $ECHO must exist, otherwise mustBeAbsent() always fails (??)
|
||||
}
|
||||
|
||||
# default compilation : all features enabled
|
||||
make clean > /dev/null
|
||||
$ECHO "testing default library compilation"
|
||||
CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
|
||||
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
|
||||
isPresent "zstd_compress.o"
|
||||
isPresent "zstd_decompress.o"
|
||||
isPresent "zdict.o"
|
||||
isPresent "zstd_v07.o"
|
||||
isPresent "zbuff_compress.o"
|
||||
$RM $DIR/../lib/libzstd.a tmplog
|
||||
|
||||
# compression disabled => also disable zdict and zbuff
|
||||
$ECHO "testing with compression disabled"
|
||||
ZSTD_LIB_COMPRESSION=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
|
||||
nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog
|
||||
! grep -q "zstd_compress" tmplog && grep -q "zstd_decompress" tmplog && ! grep -q "dict" tmplog && grep -q "zstd_v" tmplog && ! grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Compression macro failed"
|
||||
|
||||
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
|
||||
mustBeAbsent "zstd_compress.o"
|
||||
isPresent "zstd_decompress.o"
|
||||
mustBeAbsent "zdict.o"
|
||||
isPresent "zstd_v07.o"
|
||||
mustBeAbsent "zbuff_compress.o"
|
||||
$RM $DIR/../lib/libzstd.a tmplog
|
||||
|
||||
# decompression disabled => also disable legacy and zbuff
|
||||
$ECHO "testing with decompression disabled"
|
||||
ZSTD_LIB_DECOMPRESSION=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
|
||||
nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog
|
||||
grep -q "zstd_compress" tmplog && ! grep -q "zstd_decompress" tmplog && grep -q "dict" tmplog && ! grep -q "zstd_v" tmplog && ! grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Decompression macro failed"
|
||||
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
|
||||
isPresent "zstd_compress.o"
|
||||
mustBeAbsent "zstd_decompress.o"
|
||||
isPresent "zdict.o"
|
||||
mustBeAbsent "zstd_v07.o"
|
||||
mustBeAbsent "zbuff_compress.o"
|
||||
$RM $DIR/../lib/libzstd.a tmplog
|
||||
|
||||
# deprecated function disabled => only remove zbuff
|
||||
$ECHO "testing with deprecated functions disabled"
|
||||
ZSTD_LIB_DEPRECATED=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
|
||||
nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog
|
||||
grep -q "zstd_compress" tmplog && grep -q "zstd_decompress" tmplog && grep -q "dict" tmplog && grep -q "zstd_v" tmplog && ! grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Deprecated macro failed"
|
||||
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
|
||||
isPresent "zstd_compress.o"
|
||||
isPresent "zstd_decompress.o"
|
||||
isPresent "zdict.o"
|
||||
isPresent "zstd_v07.o"
|
||||
mustBeAbsent "zbuff_compress.o"
|
||||
$RM $DIR/../lib/libzstd.a tmplog
|
||||
|
||||
# dictionary builder disabled => only remove zdict
|
||||
$ECHO "testing with dictionary builder disabled"
|
||||
ZSTD_LIB_DICTBUILDER=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
|
||||
nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog
|
||||
grep -q "zstd_compress" tmplog && grep -q "zstd_decompress" tmplog && ! grep -q "dict" tmplog && grep -q "zstd_v" tmplog && grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Dictbuilder macro failed"
|
||||
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
|
||||
isPresent "zstd_compress.o"
|
||||
isPresent "zstd_decompress.o"
|
||||
mustBeAbsent "zdict.o"
|
||||
isPresent "zstd_v07.o"
|
||||
isPresent "zbuff_compress.o"
|
||||
$RM $DIR/../lib/libzstd.a tmplog
|
||||
|
||||
# both decompression and dictionary builder disabled => only compression remains
|
||||
$ECHO "testing with both decompression and dictionary builder disabled (only compression remains)"
|
||||
ZSTD_LIB_DECOMPRESSION=0 ZSTD_LIB_DICTBUILDER=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
|
||||
nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog
|
||||
grep -q "zstd_compress" tmplog && ! grep -q "zstd_decompress" tmplog && ! grep -q "dict" tmplog && ! grep -q "zstd_v" tmplog && ! grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Multi-macro failed"
|
||||
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
|
||||
isPresent "zstd_compress.o"
|
||||
mustBeAbsent "zstd_decompress.o"
|
||||
mustBeAbsent "zdict.o"
|
||||
mustBeAbsent "zstd_v07.o"
|
||||
mustBeAbsent "zbuff_compress.o"
|
||||
$RM $DIR/../lib/libzstd.a tmplog
|
||||
|
||||
@ -50,7 +50,7 @@ int main(int argc, const char** argv)
|
||||
params.cParams.chainLog = 13;
|
||||
params.cParams.hashLog = 14;
|
||||
params.cParams.searchLog = 1;
|
||||
params.cParams.searchLength = 7;
|
||||
params.cParams.minMatch = 7;
|
||||
params.cParams.targetLength = 16;
|
||||
params.cParams.strategy = ZSTD_fast;
|
||||
windowLog = params.cParams.windowLog;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -122,6 +122,18 @@ $ZSTD --fast=5000000000 -f tmp && die "too large numeric value : must fail"
|
||||
$ZSTD -c --fast=0 tmp > $INTOVOID && die "--fast must not accept value 0"
|
||||
$ECHO "test : too large numeric argument"
|
||||
$ZSTD --fast=9999999999 -f tmp && die "should have refused numeric value"
|
||||
$ECHO "test : set compression level with environment variable ZSTD_CLEVEL"
|
||||
ZSTD_CLEVEL=12 $ZSTD -f tmp # positive compression level
|
||||
ZSTD_CLEVEL=-12 $ZSTD -f tmp # negative compression level
|
||||
ZSTD_CLEVEL=+12 $ZSTD -f tmp # valid: verbose '+' sign
|
||||
ZSTD_CLEVEL= $ZSTD -f tmp # empty env var, warn and revert to default setting
|
||||
ZSTD_CLEVEL=- $ZSTD -f tmp # malformed env var, warn and revert to default setting
|
||||
ZSTD_CLEVEL=a $ZSTD -f tmp # malformed env var, warn and revert to default setting
|
||||
ZSTD_CLEVEL=+a $ZSTD -f tmp # malformed env var, warn and revert to default setting
|
||||
ZSTD_CLEVEL=3a7 $ZSTD -f tmp # malformed env var, warn and revert to default setting
|
||||
ZSTD_CLEVEL=50000000000 $ZSTD -f tmp # numeric value too large, warn and revert to default setting
|
||||
$ECHO "test : override ZSTD_CLEVEL with command line option"
|
||||
ZSTD_CLEVEL=12 $ZSTD --fast=3 -f tmp # overridden by command line option
|
||||
$ECHO "test : compress to stdout"
|
||||
$ZSTD tmp -c > tmpCompressed
|
||||
$ZSTD tmp --stdout > tmpCompressed # long command format
|
||||
@ -179,8 +191,15 @@ $ECHO foo > tmpro
|
||||
chmod 400 tmpro.zst
|
||||
$ZSTD -q tmpro && die "should have refused to overwrite read-only file"
|
||||
$ZSTD -q -f tmpro
|
||||
$ECHO "test: --no-progress flag"
|
||||
$ZSTD tmpro -c --no-progress | $ZSTD -d -f -o "$INTOVOID" --no-progress
|
||||
$ZSTD tmpro -cv --no-progress | $ZSTD -dv -f -o "$INTOVOID" --no-progress
|
||||
rm -f tmpro tmpro.zst
|
||||
|
||||
$ECHO "test: overwrite input file (must fail)"
|
||||
$ZSTD tmp -fo tmp && die "zstd compression overwrote the input file"
|
||||
$ZSTD tmp.zst -dfo tmp.zst && die "zstd decompression overwrote the input file"
|
||||
$ECHO "test: detect that input file does not exist"
|
||||
$ZSTD nothere && die "zstd hasn't detected that input file does not exist"
|
||||
|
||||
$ECHO "test : file removal"
|
||||
$ZSTD -f --rm tmp
|
||||
@ -213,7 +232,7 @@ rm tmp*
|
||||
$ECHO "test : compress multiple files"
|
||||
$ECHO hello > tmp1
|
||||
$ECHO world > tmp2
|
||||
$ZSTD tmp1 tmp2 -o "$INTOVOID"
|
||||
$ZSTD tmp1 tmp2 -o "$INTOVOID" -f
|
||||
$ZSTD tmp1 tmp2 -c | $ZSTD -t
|
||||
$ZSTD tmp1 tmp2 -o tmp.zst
|
||||
test ! -f tmp1.zst
|
||||
@ -221,7 +240,7 @@ test ! -f tmp2.zst
|
||||
$ZSTD tmp1 tmp2
|
||||
$ZSTD -t tmp1.zst tmp2.zst
|
||||
$ZSTD -dc tmp1.zst tmp2.zst
|
||||
$ZSTD tmp1.zst tmp2.zst -o "$INTOVOID"
|
||||
$ZSTD tmp1.zst tmp2.zst -o "$INTOVOID" -f
|
||||
$ZSTD -d tmp1.zst tmp2.zst -o tmp
|
||||
touch tmpexists
|
||||
$ZSTD tmp1 tmp2 -f -o tmpexists
|
||||
@ -237,14 +256,15 @@ $ECHO "\n===> Advanced compression parameters "
|
||||
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!"
|
||||
$ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!"
|
||||
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!"
|
||||
$ECHO "Hello world!" | $ZSTD --zstd=strategy=10 - -o tmp.zst && die "parameter out of bound not detected!" # > btultra2 : does not exist
|
||||
test ! -f tmp.zst # tmp.zst should not be created
|
||||
roundTripTest -g512K
|
||||
roundTripTest -g512K " --zstd=slen=3,tlen=48,strat=6"
|
||||
roundTripTest -g512K " --zstd=mml=3,tlen=48,strat=6"
|
||||
roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6"
|
||||
roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6"
|
||||
roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmSearchLength=64,ldmBucketSizeLog=1,ldmHashEveryLog=7"
|
||||
roundTripTest -g512K " --single-thread --long --zstd=ldmhlog=20,ldmslen=64,ldmblog=1,ldmhevery=7"
|
||||
roundTripTest -g512K 19
|
||||
roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,minMatch=3,targetLength=48,strategy=6"
|
||||
roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmMinMatch=64,ldmBucketSizeLog=1,ldmHashRateLog=7"
|
||||
roundTripTest -g512K " --single-thread --long --zstd=lhlog=20,lmml=64,lblog=1,lhrlog=7"
|
||||
roundTripTest -g64K "19 --zstd=strat=9" # btultra2
|
||||
|
||||
|
||||
$ECHO "\n===> Pass-Through mode "
|
||||
@ -541,6 +561,9 @@ $ECHO "bench negative level"
|
||||
$ZSTD -bi0 --fast tmp1
|
||||
$ECHO "with recursive and quiet modes"
|
||||
$ZSTD -rqi1b1e2 tmp1
|
||||
$ECHO "benchmark decompression only"
|
||||
$ZSTD -f tmp1
|
||||
$ZSTD -b -d -i1 tmp1.zst
|
||||
|
||||
$ECHO "\n===> zstd compatibility tests "
|
||||
|
||||
@ -744,17 +767,17 @@ then
|
||||
./datagen -g2MB > tmp
|
||||
refSize=$($ZSTD tmp -6 -c --zstd=wlog=18 | wc -c)
|
||||
ov9Size=$($ZSTD tmp -6 -c --zstd=wlog=18,ovlog=9 | wc -c)
|
||||
ov0Size=$($ZSTD tmp -6 -c --zstd=wlog=18,ovlog=0 | wc -c)
|
||||
ov1Size=$($ZSTD tmp -6 -c --zstd=wlog=18,ovlog=1 | wc -c)
|
||||
if [ $refSize -eq $ov9Size ]; then
|
||||
echo ov9Size should be different from refSize
|
||||
exit 1
|
||||
fi
|
||||
if [ $refSize -eq $ov0Size ]; then
|
||||
echo ov0Size should be different from refSize
|
||||
if [ $refSize -eq $ov1Size ]; then
|
||||
echo ov1Size should be different from refSize
|
||||
exit 1
|
||||
fi
|
||||
if [ $ov9Size -ge $ov0Size ]; then
|
||||
echo ov9Size=$ov9Size should be smaller than ov0Size=$ov0Size
|
||||
if [ $ov9Size -ge $ov1Size ]; then
|
||||
echo ov9Size=$ov9Size should be smaller than ov1Size=$ov1Size
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@ -833,6 +856,12 @@ $ECHO "===> test: --adapt must fail on incoherent bounds "
|
||||
./datagen > tmp
|
||||
$ZSTD -f -vv --adapt=min=10,max=9 tmp && die "--adapt must fail on incoherent bounds"
|
||||
|
||||
$ECHO "\n===> rsyncable mode "
|
||||
roundTripTest -g10M " --rsyncable"
|
||||
roundTripTest -g10M " --rsyncable -B100K"
|
||||
$ECHO "===> test: --rsyncable must fail with --single-thread"
|
||||
$ZSTD -f -vv --rsyncable --single-thread tmp && die "--rsyncable must fail with --single-thread"
|
||||
|
||||
|
||||
if [ "$1" != "--test-large-data" ]; then
|
||||
$ECHO "Skipping large data tests"
|
||||
|
||||
@ -30,7 +30,7 @@ struct data {
|
||||
size_t i;
|
||||
};
|
||||
|
||||
void fn(void *opaque) {
|
||||
static void fn(void *opaque) {
|
||||
struct data *data = (struct data *)opaque;
|
||||
ZSTD_pthread_mutex_lock(&data->mutex);
|
||||
data->data[data->i] = data->i;
|
||||
@ -38,7 +38,7 @@ void fn(void *opaque) {
|
||||
ZSTD_pthread_mutex_unlock(&data->mutex);
|
||||
}
|
||||
|
||||
int testOrder(size_t numThreads, size_t queueSize) {
|
||||
static int testOrder(size_t numThreads, size_t queueSize) {
|
||||
struct data data;
|
||||
POOL_ctx *ctx = POOL_create(numThreads, queueSize);
|
||||
ASSERT_TRUE(ctx);
|
||||
@ -63,13 +63,13 @@ int testOrder(size_t numThreads, size_t queueSize) {
|
||||
|
||||
/* --- test deadlocks --- */
|
||||
|
||||
void waitFn(void *opaque) {
|
||||
static void waitFn(void *opaque) {
|
||||
(void)opaque;
|
||||
UTIL_sleepMilli(1);
|
||||
}
|
||||
|
||||
/* Tests for deadlock */
|
||||
int testWait(size_t numThreads, size_t queueSize) {
|
||||
static int testWait(size_t numThreads, size_t queueSize) {
|
||||
struct data data;
|
||||
POOL_ctx *ctx = POOL_create(numThreads, queueSize);
|
||||
ASSERT_TRUE(ctx);
|
||||
@ -92,7 +92,7 @@ typedef struct {
|
||||
ZSTD_pthread_cond_t cond;
|
||||
} poolTest_t;
|
||||
|
||||
void waitLongFn(void *opaque) {
|
||||
static void waitLongFn(void *opaque) {
|
||||
poolTest_t* test = (poolTest_t*) opaque;
|
||||
UTIL_sleepMilli(10);
|
||||
ZSTD_pthread_mutex_lock(&test->mut);
|
||||
@ -167,7 +167,7 @@ typedef struct {
|
||||
int val;
|
||||
} abruptEndCanary_t;
|
||||
|
||||
void waitIncFn(void *opaque) {
|
||||
static void waitIncFn(void *opaque) {
|
||||
abruptEndCanary_t* test = (abruptEndCanary_t*) opaque;
|
||||
UTIL_sleepMilli(10);
|
||||
ZSTD_pthread_mutex_lock(&test->mut);
|
||||
|
||||
58
tests/regression/Makefile
Normal file
58
tests/regression/Makefile
Normal file
@ -0,0 +1,58 @@
|
||||
# ################################################################
|
||||
# Copyright (c) 2015-present, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# ################################################################
|
||||
|
||||
CFLAGS ?= -O3
|
||||
|
||||
CURL_CFLAGS := $(shell curl-config --cflags)
|
||||
CURL_LDFLAGS := $(shell curl-config --libs) -pthread
|
||||
|
||||
PROGDIR := ../../programs
|
||||
LIBDIR := ../../lib
|
||||
ZSTD_CPPFLAGS := -I$(PROGDIR) -I$(LIBDIR) -I$(LIBDIR)/common
|
||||
|
||||
REGRESSION_CFLAGS = $(CFLAGS) $(CURL_CFLAGS)
|
||||
REGRESSION_CPPFLAGS = $(CPPFLAGS) $(ZSTD_CPPFLAGS)
|
||||
REGRESSION_LDFLAGS = $(LDFLAGS) $(CURL_LDFLAGS)
|
||||
|
||||
all: test
|
||||
|
||||
xxhash.o: $(LIBDIR)/common/xxhash.c $(LIBDIR)/common/xxhash.h
|
||||
$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
|
||||
|
||||
util.o: $(PROGDIR)/util.c $(PROGDIR)/util.h
|
||||
$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
|
||||
|
||||
data.o: data.c data.h $(PROGDIR)/util.h $(LIBDIR)/common/xxhash.h
|
||||
$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
|
||||
|
||||
config.o: config.c config.h levels.h
|
||||
$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
|
||||
|
||||
method.h: data.h config.h result.h
|
||||
|
||||
method.o: method.c method.h
|
||||
$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
|
||||
|
||||
result.o: result.c result.h
|
||||
$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
|
||||
|
||||
test.o: test.c data.h config.h method.h
|
||||
$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
|
||||
|
||||
libzstd.a:
|
||||
$(MAKE) -C $(LIBDIR) libzstd.a-mt
|
||||
cp $(LIBDIR)/libzstd.a .
|
||||
|
||||
test: test.o data.o config.o util.o method.o result.o xxhash.o libzstd.a
|
||||
$(CC) $^ $(REGRESSION_LDFLAGS) -o $@
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
$(MAKE) -C $(LIBDIR) clean
|
||||
$(RM) *.o *.a test
|
||||
230
tests/regression/config.c
Normal file
230
tests/regression/config.c
Normal file
@ -0,0 +1,230 @@
|
||||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#include "config.h"
|
||||
|
||||
/* Define a config for each fast level we want to test with. */
|
||||
#define FAST_LEVEL(x) \
|
||||
param_value_t const level_fast##x##_param_values[] = { \
|
||||
{.param = ZSTD_c_compressionLevel, .value = -x}, \
|
||||
}; \
|
||||
config_t const level_fast##x = { \
|
||||
.name = "level -" #x, \
|
||||
.cli_args = "--fast=" #x, \
|
||||
.param_values = PARAM_VALUES(level_fast##x##_param_values), \
|
||||
}; \
|
||||
config_t const level_fast##x##_dict = { \
|
||||
.name = "level -" #x " with dict", \
|
||||
.cli_args = "--fast=" #x, \
|
||||
.param_values = PARAM_VALUES(level_fast##x##_param_values), \
|
||||
.use_dictionary = 1, \
|
||||
};
|
||||
|
||||
/* Define a config for each level we want to test with. */
|
||||
#define LEVEL(x) \
|
||||
param_value_t const level_##x##_param_values[] = { \
|
||||
{.param = ZSTD_c_compressionLevel, .value = x}, \
|
||||
}; \
|
||||
config_t const level_##x = { \
|
||||
.name = "level " #x, \
|
||||
.cli_args = "-" #x, \
|
||||
.param_values = PARAM_VALUES(level_##x##_param_values), \
|
||||
}; \
|
||||
config_t const level_##x##_dict = { \
|
||||
.name = "level " #x " with dict", \
|
||||
.cli_args = "-" #x, \
|
||||
.param_values = PARAM_VALUES(level_##x##_param_values), \
|
||||
.use_dictionary = 1, \
|
||||
};
|
||||
|
||||
#define PARAM_VALUES(pv) \
|
||||
{ .data = pv, .size = sizeof(pv) / sizeof((pv)[0]) }
|
||||
|
||||
#include "levels.h"
|
||||
|
||||
#undef LEVEL
|
||||
#undef FAST_LEVEL
|
||||
|
||||
static config_t no_pledged_src_size = {
|
||||
.name = "no source size",
|
||||
.cli_args = "",
|
||||
.param_values = PARAM_VALUES(level_0_param_values),
|
||||
.no_pledged_src_size = 1,
|
||||
};
|
||||
|
||||
static param_value_t const ldm_param_values[] = {
|
||||
{.param = ZSTD_c_enableLongDistanceMatching, .value = 1},
|
||||
};
|
||||
|
||||
static config_t ldm = {
|
||||
.name = "long distance mode",
|
||||
.cli_args = "--long",
|
||||
.param_values = PARAM_VALUES(ldm_param_values),
|
||||
};
|
||||
|
||||
static param_value_t const mt_param_values[] = {
|
||||
{.param = ZSTD_c_nbWorkers, .value = 2},
|
||||
};
|
||||
|
||||
static config_t mt = {
|
||||
.name = "multithreaded",
|
||||
.cli_args = "-T2",
|
||||
.param_values = PARAM_VALUES(mt_param_values),
|
||||
};
|
||||
|
||||
static param_value_t const mt_ldm_param_values[] = {
|
||||
{.param = ZSTD_c_nbWorkers, .value = 2},
|
||||
{.param = ZSTD_c_enableLongDistanceMatching, .value = 1},
|
||||
};
|
||||
|
||||
static config_t mt_ldm = {
|
||||
.name = "multithreaded long distance mode",
|
||||
.cli_args = "-T2 --long",
|
||||
.param_values = PARAM_VALUES(mt_ldm_param_values),
|
||||
};
|
||||
|
||||
static param_value_t const small_wlog_param_values[] = {
|
||||
{.param = ZSTD_c_windowLog, .value = 10},
|
||||
};
|
||||
|
||||
static config_t small_wlog = {
|
||||
.name = "small window log",
|
||||
.cli_args = "--zstd=wlog=10",
|
||||
.param_values = PARAM_VALUES(small_wlog_param_values),
|
||||
};
|
||||
|
||||
static param_value_t const small_hlog_param_values[] = {
|
||||
{.param = ZSTD_c_hashLog, .value = 6},
|
||||
{.param = ZSTD_c_strategy, .value = (int)ZSTD_btopt},
|
||||
};
|
||||
|
||||
static config_t small_hlog = {
|
||||
.name = "small hash log",
|
||||
.cli_args = "--zstd=hlog=6,strat=7",
|
||||
.param_values = PARAM_VALUES(small_hlog_param_values),
|
||||
};
|
||||
|
||||
static param_value_t const small_clog_param_values[] = {
|
||||
{.param = ZSTD_c_chainLog, .value = 6},
|
||||
{.param = ZSTD_c_strategy, .value = (int)ZSTD_btopt},
|
||||
};
|
||||
|
||||
static config_t small_clog = {
|
||||
.name = "small chain log",
|
||||
.cli_args = "--zstd=clog=6,strat=7",
|
||||
.param_values = PARAM_VALUES(small_clog_param_values),
|
||||
};
|
||||
|
||||
static param_value_t const explicit_params_param_values[] = {
|
||||
{.param = ZSTD_c_checksumFlag, .value = 1},
|
||||
{.param = ZSTD_c_contentSizeFlag, .value = 0},
|
||||
{.param = ZSTD_c_dictIDFlag, .value = 0},
|
||||
{.param = ZSTD_c_strategy, .value = (int)ZSTD_greedy},
|
||||
{.param = ZSTD_c_windowLog, .value = 18},
|
||||
{.param = ZSTD_c_hashLog, .value = 21},
|
||||
{.param = ZSTD_c_chainLog, .value = 21},
|
||||
{.param = ZSTD_c_targetLength, .value = 100},
|
||||
};
|
||||
|
||||
static config_t explicit_params = {
|
||||
.name = "explicit params",
|
||||
.cli_args = "--no-check --no-dictID --zstd=strategy=3,wlog=18,hlog=21,clog=21,tlen=100",
|
||||
.param_values = PARAM_VALUES(explicit_params_param_values),
|
||||
};
|
||||
|
||||
static config_t const* g_configs[] = {
|
||||
|
||||
#define FAST_LEVEL(x) &level_fast##x, &level_fast##x##_dict,
|
||||
#define LEVEL(x) &level_##x, &level_##x##_dict,
|
||||
#include "levels.h"
|
||||
#undef LEVEL
|
||||
#undef FAST_LEVEL
|
||||
|
||||
&no_pledged_src_size,
|
||||
&ldm,
|
||||
&mt,
|
||||
&mt_ldm,
|
||||
&small_wlog,
|
||||
&small_hlog,
|
||||
&small_clog,
|
||||
&explicit_params,
|
||||
NULL,
|
||||
};
|
||||
|
||||
config_t const* const* configs = g_configs;
|
||||
|
||||
int config_skip_data(config_t const* config, data_t const* data) {
|
||||
return config->use_dictionary && !data_has_dict(data);
|
||||
}
|
||||
|
||||
int config_get_level(config_t const* config)
|
||||
{
|
||||
param_values_t const params = config->param_values;
|
||||
size_t i;
|
||||
for (i = 0; i < params.size; ++i) {
|
||||
if (params.data[i].param == ZSTD_c_compressionLevel)
|
||||
return (int)params.data[i].value;
|
||||
}
|
||||
return CONFIG_NO_LEVEL;
|
||||
}
|
||||
|
||||
ZSTD_parameters config_get_zstd_params(
|
||||
config_t const* config,
|
||||
uint64_t srcSize,
|
||||
size_t dictSize)
|
||||
{
|
||||
ZSTD_parameters zparams = {};
|
||||
param_values_t const params = config->param_values;
|
||||
int level = config_get_level(config);
|
||||
if (level == CONFIG_NO_LEVEL)
|
||||
level = 3;
|
||||
zparams = ZSTD_getParams(
|
||||
level,
|
||||
config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN : srcSize,
|
||||
dictSize);
|
||||
for (size_t i = 0; i < params.size; ++i) {
|
||||
unsigned const value = params.data[i].value;
|
||||
switch (params.data[i].param) {
|
||||
case ZSTD_c_contentSizeFlag:
|
||||
zparams.fParams.contentSizeFlag = value;
|
||||
break;
|
||||
case ZSTD_c_checksumFlag:
|
||||
zparams.fParams.checksumFlag = value;
|
||||
break;
|
||||
case ZSTD_c_dictIDFlag:
|
||||
zparams.fParams.noDictIDFlag = !value;
|
||||
break;
|
||||
case ZSTD_c_windowLog:
|
||||
zparams.cParams.windowLog = value;
|
||||
break;
|
||||
case ZSTD_c_chainLog:
|
||||
zparams.cParams.chainLog = value;
|
||||
break;
|
||||
case ZSTD_c_hashLog:
|
||||
zparams.cParams.hashLog = value;
|
||||
break;
|
||||
case ZSTD_c_searchLog:
|
||||
zparams.cParams.searchLog = value;
|
||||
break;
|
||||
case ZSTD_c_minMatch:
|
||||
zparams.cParams.minMatch = value;
|
||||
break;
|
||||
case ZSTD_c_targetLength:
|
||||
zparams.cParams.targetLength = value;
|
||||
break;
|
||||
case ZSTD_c_strategy:
|
||||
zparams.cParams.strategy = (ZSTD_strategy)value;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
return zparams;
|
||||
}
|
||||
86
tests/regression/config.h
Normal file
86
tests/regression/config.h
Normal file
@ -0,0 +1,86 @@
|
||||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef CONFIG_H
|
||||
#define CONFIG_H
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include <zstd.h>
|
||||
|
||||
#include "data.h"
|
||||
|
||||
typedef struct {
|
||||
ZSTD_cParameter param;
|
||||
int value;
|
||||
} param_value_t;
|
||||
|
||||
typedef struct {
|
||||
size_t size;
|
||||
param_value_t const* data;
|
||||
} param_values_t;
|
||||
|
||||
/**
|
||||
* The config tells the compression method what options to use.
|
||||
*/
|
||||
typedef struct {
|
||||
const char* name; /**< Identifies the config in the results table */
|
||||
/**
|
||||
* Optional arguments to pass to the CLI. If not set, CLI-based methods
|
||||
* will skip this config.
|
||||
*/
|
||||
char const* cli_args;
|
||||
/**
|
||||
* Parameters to pass to the advanced API. If the advanced API isn't used,
|
||||
* the parameters will be derived from these.
|
||||
*/
|
||||
param_values_t param_values;
|
||||
/**
|
||||
* Boolean parameter that says if we should use a dictionary. If the data
|
||||
* doesn't have a dictionary, this config is skipped. Defaults to no.
|
||||
*/
|
||||
int use_dictionary;
|
||||
/**
|
||||
* Boolean parameter that says if we should pass the pledged source size
|
||||
* when the method allows it. Defaults to yes.
|
||||
*/
|
||||
int no_pledged_src_size;
|
||||
} config_t;
|
||||
|
||||
/**
|
||||
* Returns true if the config should skip this data.
|
||||
* For instance, if the config requires a dictionary but the data doesn't have
|
||||
* one.
|
||||
*/
|
||||
int config_skip_data(config_t const* config, data_t const* data);
|
||||
|
||||
#define CONFIG_NO_LEVEL (-ZSTD_TARGETLENGTH_MAX - 1)
|
||||
/**
|
||||
* Returns the compression level specified by the config, or CONFIG_NO_LEVEL if
|
||||
* no level is specified. Note that 0 is a valid compression level, meaning
|
||||
* default.
|
||||
*/
|
||||
int config_get_level(config_t const* config);
|
||||
|
||||
/**
|
||||
* Returns the compression parameters specified by the config.
|
||||
*/
|
||||
ZSTD_parameters config_get_zstd_params(
|
||||
config_t const* config,
|
||||
uint64_t srcSize,
|
||||
size_t dictSize);
|
||||
|
||||
/**
|
||||
* The NULL-terminated list of configs.
|
||||
*/
|
||||
extern config_t const* const* configs;
|
||||
|
||||
#endif
|
||||
617
tests/regression/data.c
Normal file
617
tests/regression/data.c
Normal file
@ -0,0 +1,617 @@
|
||||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#include "data.h"
|
||||
|
||||
#include <assert.h>
|
||||
#include <errno.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
#include <sys/stat.h>
|
||||
|
||||
#include <curl/curl.h>
|
||||
|
||||
#include "mem.h"
|
||||
#include "util.h"
|
||||
#define XXH_STATIC_LINKING_ONLY
|
||||
#include "xxhash.h"
|
||||
|
||||
/**
|
||||
* Data objects
|
||||
*/
|
||||
|
||||
#define REGRESSION_RELEASE(x) \
|
||||
"https://github.com/facebook/zstd/releases/download/regression-data/" x
|
||||
|
||||
data_t silesia = {
|
||||
.name = "silesia",
|
||||
.type = data_type_dir,
|
||||
.data =
|
||||
{
|
||||
.url = REGRESSION_RELEASE("silesia.tar.zst"),
|
||||
.xxhash64 = 0x48a199f92f93e977LL,
|
||||
},
|
||||
};
|
||||
|
||||
data_t silesia_tar = {
|
||||
.name = "silesia.tar",
|
||||
.type = data_type_file,
|
||||
.data =
|
||||
{
|
||||
.url = REGRESSION_RELEASE("silesia.tar.zst"),
|
||||
.xxhash64 = 0x48a199f92f93e977LL,
|
||||
},
|
||||
};
|
||||
|
||||
data_t github = {
|
||||
.name = "github",
|
||||
.type = data_type_dir,
|
||||
.data =
|
||||
{
|
||||
.url = REGRESSION_RELEASE("github.tar.zst"),
|
||||
.xxhash64 = 0xa9b1b44b020df292LL,
|
||||
},
|
||||
.dict =
|
||||
{
|
||||
.url = REGRESSION_RELEASE("github.dict.zst"),
|
||||
.xxhash64 = 0x1eddc6f737d3cb53LL,
|
||||
|
||||
},
|
||||
};
|
||||
|
||||
static data_t* g_data[] = {
|
||||
&silesia,
|
||||
&silesia_tar,
|
||||
&github,
|
||||
NULL,
|
||||
};
|
||||
|
||||
data_t const* const* data = (data_t const* const*)g_data;
|
||||
|
||||
/**
|
||||
* data helpers.
|
||||
*/
|
||||
|
||||
int data_has_dict(data_t const* data) {
|
||||
return data->dict.url != NULL;
|
||||
}
|
||||
|
||||
/**
|
||||
* data buffer helper functions (documented in header).
|
||||
*/
|
||||
|
||||
data_buffer_t data_buffer_create(size_t const capacity) {
|
||||
data_buffer_t buffer = {};
|
||||
|
||||
buffer.data = (uint8_t*)malloc(capacity);
|
||||
if (buffer.data == NULL)
|
||||
return buffer;
|
||||
buffer.capacity = capacity;
|
||||
return buffer;
|
||||
}
|
||||
|
||||
data_buffer_t data_buffer_read(char const* filename) {
|
||||
data_buffer_t buffer = {};
|
||||
|
||||
uint64_t const size = UTIL_getFileSize(filename);
|
||||
if (size == UTIL_FILESIZE_UNKNOWN) {
|
||||
fprintf(stderr, "unknown size for %s\n", filename);
|
||||
return buffer;
|
||||
}
|
||||
|
||||
buffer.data = (uint8_t*)malloc(size);
|
||||
if (buffer.data == NULL) {
|
||||
fprintf(stderr, "malloc failed\n");
|
||||
return buffer;
|
||||
}
|
||||
buffer.capacity = size;
|
||||
|
||||
FILE* file = fopen(filename, "rb");
|
||||
if (file == NULL) {
|
||||
fprintf(stderr, "file null\n");
|
||||
goto err;
|
||||
}
|
||||
buffer.size = fread(buffer.data, 1, buffer.capacity, file);
|
||||
fclose(file);
|
||||
if (buffer.size != buffer.capacity) {
|
||||
fprintf(stderr, "read %zu != %zu\n", buffer.size, buffer.capacity);
|
||||
goto err;
|
||||
}
|
||||
|
||||
return buffer;
|
||||
err:
|
||||
free(buffer.data);
|
||||
memset(&buffer, 0, sizeof(buffer));
|
||||
return buffer;
|
||||
}
|
||||
|
||||
data_buffer_t data_buffer_get_data(data_t const* data) {
|
||||
data_buffer_t const kEmptyBuffer = {};
|
||||
|
||||
if (data->type != data_type_file)
|
||||
return kEmptyBuffer;
|
||||
|
||||
return data_buffer_read(data->data.path);
|
||||
}
|
||||
|
||||
data_buffer_t data_buffer_get_dict(data_t const* data) {
|
||||
data_buffer_t const kEmptyBuffer = {};
|
||||
|
||||
if (!data_has_dict(data))
|
||||
return kEmptyBuffer;
|
||||
|
||||
return data_buffer_read(data->dict.path);
|
||||
}
|
||||
|
||||
int data_buffer_compare(data_buffer_t buffer1, data_buffer_t buffer2) {
|
||||
size_t const size =
|
||||
buffer1.size < buffer2.size ? buffer1.size : buffer2.size;
|
||||
int const cmp = memcmp(buffer1.data, buffer2.data, size);
|
||||
if (cmp != 0)
|
||||
return cmp;
|
||||
if (buffer1.size < buffer2.size)
|
||||
return -1;
|
||||
if (buffer1.size == buffer2.size)
|
||||
return 0;
|
||||
assert(buffer1.size > buffer2.size);
|
||||
return 1;
|
||||
}
|
||||
|
||||
void data_buffer_free(data_buffer_t buffer) {
|
||||
free(buffer.data);
|
||||
}
|
||||
|
||||
/**
|
||||
* data filenames helpers.
|
||||
*/
|
||||
|
||||
data_filenames_t data_filenames_get(data_t const* data) {
|
||||
data_filenames_t filenames = {.buffer = NULL, .size = 0};
|
||||
char const* path = data->data.path;
|
||||
|
||||
filenames.filenames = UTIL_createFileList(
|
||||
&path,
|
||||
1,
|
||||
&filenames.buffer,
|
||||
&filenames.size,
|
||||
/* followLinks */ 0);
|
||||
return filenames;
|
||||
}
|
||||
|
||||
void data_filenames_free(data_filenames_t filenames) {
|
||||
UTIL_freeFileList(filenames.filenames, filenames.buffer);
|
||||
}
|
||||
|
||||
/**
|
||||
* data buffers helpers.
|
||||
*/
|
||||
|
||||
data_buffers_t data_buffers_get(data_t const* data) {
|
||||
data_buffers_t buffers = {.size = 0};
|
||||
data_filenames_t filenames = data_filenames_get(data);
|
||||
if (filenames.size == 0)
|
||||
return buffers;
|
||||
|
||||
data_buffer_t* buffersPtr =
|
||||
(data_buffer_t*)malloc(filenames.size * sizeof(data_buffer_t));
|
||||
if (buffersPtr == NULL)
|
||||
return buffers;
|
||||
buffers.buffers = (data_buffer_t const*)buffersPtr;
|
||||
buffers.size = filenames.size;
|
||||
|
||||
for (size_t i = 0; i < filenames.size; ++i) {
|
||||
buffersPtr[i] = data_buffer_read(filenames.filenames[i]);
|
||||
if (buffersPtr[i].data == NULL) {
|
||||
data_buffers_t const kEmptyBuffer = {};
|
||||
data_buffers_free(buffers);
|
||||
return kEmptyBuffer;
|
||||
}
|
||||
}
|
||||
|
||||
return buffers;
|
||||
}
|
||||
|
||||
/**
|
||||
* Frees the data buffers.
|
||||
*/
|
||||
void data_buffers_free(data_buffers_t buffers) {
|
||||
free((data_buffer_t*)buffers.buffers);
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialization and download functions.
|
||||
*/
|
||||
|
||||
static char* g_data_dir = NULL;
|
||||
|
||||
/* mkdir -p */
|
||||
static int ensure_directory_exists(char const* indir) {
|
||||
char* const dir = strdup(indir);
|
||||
char* end = dir;
|
||||
int ret = 0;
|
||||
if (dir == NULL) {
|
||||
ret = EINVAL;
|
||||
goto out;
|
||||
}
|
||||
do {
|
||||
/* Find the next directory level. */
|
||||
for (++end; *end != '\0' && *end != '/'; ++end)
|
||||
;
|
||||
/* End the string there, make the directory, and restore the string. */
|
||||
char const save = *end;
|
||||
*end = '\0';
|
||||
int const isdir = UTIL_isDirectory(dir);
|
||||
ret = mkdir(dir, S_IRWXU);
|
||||
*end = save;
|
||||
/* Its okay if the directory already exists. */
|
||||
if (ret == 0 || (errno == EEXIST && isdir))
|
||||
continue;
|
||||
ret = errno;
|
||||
fprintf(stderr, "mkdir() failed\n");
|
||||
goto out;
|
||||
} while (*end != '\0');
|
||||
|
||||
ret = 0;
|
||||
out:
|
||||
free(dir);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/** Concatenate 3 strings into a new buffer. */
|
||||
static char* cat3(char const* str1, char const* str2, char const* str3) {
|
||||
size_t const size1 = strlen(str1);
|
||||
size_t const size2 = strlen(str2);
|
||||
size_t const size3 = str3 == NULL ? 0 : strlen(str3);
|
||||
size_t const size = size1 + size2 + size3 + 1;
|
||||
char* const dst = (char*)malloc(size);
|
||||
if (dst == NULL)
|
||||
return NULL;
|
||||
strcpy(dst, str1);
|
||||
strcpy(dst + size1, str2);
|
||||
if (str3 != NULL)
|
||||
strcpy(dst + size1 + size2, str3);
|
||||
assert(strlen(dst) == size1 + size2 + size3);
|
||||
return dst;
|
||||
}
|
||||
|
||||
static char* cat2(char const* str1, char const* str2) {
|
||||
return cat3(str1, str2, NULL);
|
||||
}
|
||||
|
||||
/**
|
||||
* State needed by the curl callback.
|
||||
* It takes data from curl, hashes it, and writes it to the file.
|
||||
*/
|
||||
typedef struct {
|
||||
FILE* file;
|
||||
XXH64_state_t xxhash64;
|
||||
int error;
|
||||
} curl_data_t;
|
||||
|
||||
/** Create the curl state. */
|
||||
static curl_data_t curl_data_create(
|
||||
data_resource_t const* resource,
|
||||
data_type_t type) {
|
||||
curl_data_t cdata = {};
|
||||
|
||||
XXH64_reset(&cdata.xxhash64, 0);
|
||||
|
||||
assert(UTIL_isDirectory(g_data_dir));
|
||||
|
||||
if (type == data_type_file) {
|
||||
/* Decompress the resource and store to the path. */
|
||||
char* cmd = cat3("zstd -dqfo '", resource->path, "'");
|
||||
if (cmd == NULL) {
|
||||
cdata.error = ENOMEM;
|
||||
return cdata;
|
||||
}
|
||||
cdata.file = popen(cmd, "w");
|
||||
free(cmd);
|
||||
} else {
|
||||
/* Decompress and extract the resource to the cache directory. */
|
||||
char* cmd = cat3("zstd -dc | tar -x -C '", g_data_dir, "'");
|
||||
if (cmd == NULL) {
|
||||
cdata.error = ENOMEM;
|
||||
return cdata;
|
||||
}
|
||||
cdata.file = popen(cmd, "w");
|
||||
free(cmd);
|
||||
}
|
||||
if (cdata.file == NULL) {
|
||||
cdata.error = errno;
|
||||
}
|
||||
|
||||
return cdata;
|
||||
}
|
||||
|
||||
/** Free the curl state. */
|
||||
static int curl_data_free(curl_data_t cdata) {
|
||||
return pclose(cdata.file);
|
||||
}
|
||||
|
||||
/** curl callback. Updates the hash, and writes to the file. */
|
||||
static size_t curl_write(void* data, size_t size, size_t count, void* ptr) {
|
||||
curl_data_t* cdata = (curl_data_t*)ptr;
|
||||
size_t const written = fwrite(data, size, count, cdata->file);
|
||||
XXH64_update(&cdata->xxhash64, data, written * size);
|
||||
return written;
|
||||
}
|
||||
|
||||
static int curl_download_resource(
|
||||
CURL* curl,
|
||||
data_resource_t const* resource,
|
||||
data_type_t type) {
|
||||
curl_data_t cdata;
|
||||
/* Download the data. */
|
||||
if (curl_easy_setopt(curl, CURLOPT_URL, resource->url) != 0)
|
||||
return EINVAL;
|
||||
if (curl_easy_setopt(curl, CURLOPT_WRITEDATA, &cdata) != 0)
|
||||
return EINVAL;
|
||||
cdata = curl_data_create(resource, type);
|
||||
if (cdata.error != 0)
|
||||
return cdata.error;
|
||||
int const curl_err = curl_easy_perform(curl);
|
||||
int const close_err = curl_data_free(cdata);
|
||||
if (curl_err) {
|
||||
fprintf(
|
||||
stderr,
|
||||
"downloading '%s' for '%s' failed\n",
|
||||
resource->url,
|
||||
resource->path);
|
||||
return EIO;
|
||||
}
|
||||
if (close_err) {
|
||||
fprintf(stderr, "writing data to '%s' failed\n", resource->path);
|
||||
return EIO;
|
||||
}
|
||||
/* check that the file exists. */
|
||||
if (type == data_type_file && !UTIL_isRegularFile(resource->path)) {
|
||||
fprintf(stderr, "output file '%s' does not exist\n", resource->path);
|
||||
return EIO;
|
||||
}
|
||||
if (type == data_type_dir && !UTIL_isDirectory(resource->path)) {
|
||||
fprintf(
|
||||
stderr, "output directory '%s' does not exist\n", resource->path);
|
||||
return EIO;
|
||||
}
|
||||
/* Check that the hash matches. */
|
||||
if (XXH64_digest(&cdata.xxhash64) != resource->xxhash64) {
|
||||
fprintf(
|
||||
stderr,
|
||||
"checksum does not match: 0x%llxLL != 0x%llxLL\n",
|
||||
(unsigned long long)XXH64_digest(&cdata.xxhash64),
|
||||
(unsigned long long)resource->xxhash64);
|
||||
return EINVAL;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** Download a single data object. */
|
||||
static int curl_download_datum(CURL* curl, data_t const* data) {
|
||||
int ret;
|
||||
ret = curl_download_resource(curl, &data->data, data->type);
|
||||
if (ret != 0)
|
||||
return ret;
|
||||
if (data_has_dict(data)) {
|
||||
ret = curl_download_resource(curl, &data->dict, data_type_file);
|
||||
if (ret != 0)
|
||||
return ret;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
/** Download all the data. */
|
||||
static int curl_download_data(data_t const* const* data) {
|
||||
if (curl_global_init(CURL_GLOBAL_ALL) != 0)
|
||||
return EFAULT;
|
||||
|
||||
curl_data_t cdata = {};
|
||||
CURL* curl = curl_easy_init();
|
||||
int err = EFAULT;
|
||||
|
||||
if (curl == NULL)
|
||||
return EFAULT;
|
||||
|
||||
if (curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L) != 0)
|
||||
goto out;
|
||||
if (curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L) != 0)
|
||||
goto out;
|
||||
if (curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write) != 0)
|
||||
goto out;
|
||||
|
||||
assert(data != NULL);
|
||||
for (; *data != NULL; ++data) {
|
||||
if (curl_download_datum(curl, *data) != 0)
|
||||
goto out;
|
||||
}
|
||||
|
||||
err = 0;
|
||||
out:
|
||||
curl_easy_cleanup(curl);
|
||||
curl_global_cleanup();
|
||||
return err;
|
||||
}
|
||||
|
||||
/** Fill the path member variable of the data objects. */
|
||||
static int data_create_paths(data_t* const* data, char const* dir) {
|
||||
size_t const dirlen = strlen(dir);
|
||||
assert(data != NULL);
|
||||
for (; *data != NULL; ++data) {
|
||||
data_t* const datum = *data;
|
||||
datum->data.path = cat3(dir, "/", datum->name);
|
||||
if (datum->data.path == NULL)
|
||||
return ENOMEM;
|
||||
if (data_has_dict(datum)) {
|
||||
datum->dict.path = cat2(datum->data.path, ".dict");
|
||||
if (datum->dict.path == NULL)
|
||||
return ENOMEM;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** Free the path member variable of the data objects. */
|
||||
static void data_free_paths(data_t* const* data) {
|
||||
assert(data != NULL);
|
||||
for (; *data != NULL; ++data) {
|
||||
data_t* datum = *data;
|
||||
free((void*)datum->data.path);
|
||||
free((void*)datum->dict.path);
|
||||
datum->data.path = NULL;
|
||||
datum->dict.path = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static char const kStampName[] = "STAMP";
|
||||
|
||||
static void xxh_update_le(XXH64_state_t* state, uint64_t data) {
|
||||
if (!MEM_isLittleEndian())
|
||||
data = MEM_swap64(data);
|
||||
XXH64_update(state, &data, sizeof(data));
|
||||
}
|
||||
|
||||
/** Hash the data to create the stamp. */
|
||||
static uint64_t stamp_hash(data_t const* const* data) {
|
||||
XXH64_state_t state;
|
||||
|
||||
XXH64_reset(&state, 0);
|
||||
assert(data != NULL);
|
||||
for (; *data != NULL; ++data) {
|
||||
data_t const* datum = *data;
|
||||
/* We don't care about the URL that we fetch from. */
|
||||
/* The path is derived from the name. */
|
||||
XXH64_update(&state, datum->name, strlen(datum->name));
|
||||
xxh_update_le(&state, datum->data.xxhash64);
|
||||
xxh_update_le(&state, datum->dict.xxhash64);
|
||||
xxh_update_le(&state, datum->type);
|
||||
}
|
||||
return XXH64_digest(&state);
|
||||
}
|
||||
|
||||
/** Check if the stamp matches the stamp in the cache directory. */
|
||||
static int stamp_check(char const* dir, data_t const* const* data) {
|
||||
char* stamp = cat3(dir, "/", kStampName);
|
||||
uint64_t const expected = stamp_hash(data);
|
||||
XXH64_canonical_t actual;
|
||||
FILE* stampfile = NULL;
|
||||
int matches = 0;
|
||||
|
||||
if (stamp == NULL)
|
||||
goto out;
|
||||
if (!UTIL_isRegularFile(stamp)) {
|
||||
fprintf(stderr, "stamp does not exist: recreating the data cache\n");
|
||||
goto out;
|
||||
}
|
||||
|
||||
stampfile = fopen(stamp, "rb");
|
||||
if (stampfile == NULL) {
|
||||
fprintf(stderr, "could not open stamp: recreating the data cache\n");
|
||||
goto out;
|
||||
}
|
||||
|
||||
size_t b;
|
||||
if ((b = fread(&actual, sizeof(actual), 1, stampfile)) != 1) {
|
||||
fprintf(stderr, "invalid stamp: recreating the data cache\n");
|
||||
goto out;
|
||||
}
|
||||
|
||||
matches = (expected == XXH64_hashFromCanonical(&actual));
|
||||
if (matches)
|
||||
fprintf(stderr, "stamp matches: reusing the cached data\n");
|
||||
else
|
||||
fprintf(stderr, "stamp does not match: recreating the data cache\n");
|
||||
|
||||
out:
|
||||
free(stamp);
|
||||
if (stampfile != NULL)
|
||||
fclose(stampfile);
|
||||
return matches;
|
||||
}
|
||||
|
||||
/** On success write a new stamp, on failure delete the old stamp. */
|
||||
static int
|
||||
stamp_write(char const* dir, data_t const* const* data, int const data_err) {
|
||||
char* stamp = cat3(dir, "/", kStampName);
|
||||
FILE* stampfile = NULL;
|
||||
int err = EIO;
|
||||
|
||||
if (stamp == NULL)
|
||||
return ENOMEM;
|
||||
|
||||
if (data_err != 0) {
|
||||
err = data_err;
|
||||
goto out;
|
||||
}
|
||||
XXH64_canonical_t hash;
|
||||
|
||||
XXH64_canonicalFromHash(&hash, stamp_hash(data));
|
||||
|
||||
stampfile = fopen(stamp, "wb");
|
||||
if (stampfile == NULL)
|
||||
goto out;
|
||||
if (fwrite(&hash, sizeof(hash), 1, stampfile) != 1)
|
||||
goto out;
|
||||
err = 0;
|
||||
fprintf(stderr, "stamped new data cache\n");
|
||||
out:
|
||||
if (err != 0)
|
||||
/* Ignore errors. */
|
||||
unlink(stamp);
|
||||
free(stamp);
|
||||
if (stampfile != NULL)
|
||||
fclose(stampfile);
|
||||
return err;
|
||||
}
|
||||
|
||||
int data_init(char const* dir) {
|
||||
int err;
|
||||
|
||||
if (dir == NULL)
|
||||
return EINVAL;
|
||||
|
||||
/* This must be first to simplify logic. */
|
||||
err = ensure_directory_exists(dir);
|
||||
if (err != 0)
|
||||
return err;
|
||||
|
||||
/* Save the cache directory. */
|
||||
g_data_dir = strdup(dir);
|
||||
if (g_data_dir == NULL)
|
||||
return ENOMEM;
|
||||
|
||||
err = data_create_paths(g_data, dir);
|
||||
if (err != 0)
|
||||
return err;
|
||||
|
||||
/* If the stamp matches then we are good to go.
|
||||
* This must be called before any modifications to the data cache.
|
||||
* After this point, we MUST call stamp_write() to update the STAMP,
|
||||
* since we've updated the data cache.
|
||||
*/
|
||||
if (stamp_check(dir, data))
|
||||
return 0;
|
||||
|
||||
err = curl_download_data(data);
|
||||
if (err != 0)
|
||||
goto out;
|
||||
|
||||
out:
|
||||
/* This must be last, since it must know if data_init() succeeded. */
|
||||
stamp_write(dir, data, err);
|
||||
return err;
|
||||
}
|
||||
|
||||
void data_finish(void) {
|
||||
data_free_paths(g_data);
|
||||
free(g_data_dir);
|
||||
g_data_dir = NULL;
|
||||
}
|
||||
140
tests/regression/data.h
Normal file
140
tests/regression/data.h
Normal file
@ -0,0 +1,140 @@
|
||||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef DATA_H
|
||||
#define DATA_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
typedef enum {
|
||||
data_type_file = 1, /**< This data is a file. *.zst */
|
||||
data_type_dir = 2, /**< This data is a directory. *.tar.zst */
|
||||
} data_type_t;
|
||||
|
||||
typedef struct {
|
||||
char const* url; /**< Where to get this resource. */
|
||||
uint64_t xxhash64; /**< Hash of the url contents. */
|
||||
char const* path; /**< The path of the unpacked resource (derived). */
|
||||
} data_resource_t;
|
||||
|
||||
typedef struct {
|
||||
data_resource_t data;
|
||||
data_resource_t dict;
|
||||
data_type_t type; /**< The type of the data. */
|
||||
char const* name; /**< The logical name of the data (no extension). */
|
||||
} data_t;
|
||||
|
||||
/**
|
||||
* The NULL-terminated list of data objects.
|
||||
*/
|
||||
extern data_t const* const* data;
|
||||
|
||||
|
||||
int data_has_dict(data_t const* data);
|
||||
|
||||
/**
|
||||
* Initializes the data module and downloads the data necessary.
|
||||
* Caches the downloads in dir. We add a stamp file in the directory after
|
||||
* a successful download. If a stamp file already exists, and matches our
|
||||
* current data stamp, we will use the cached data without downloading.
|
||||
*
|
||||
* @param dir The directory to cache the downloaded data into.
|
||||
*
|
||||
* @returns 0 on success.
|
||||
*/
|
||||
int data_init(char const* dir);
|
||||
|
||||
/**
|
||||
* Must be called at exit to free resources allocated by data_init().
|
||||
*/
|
||||
void data_finish(void);
|
||||
|
||||
typedef struct {
|
||||
uint8_t* data;
|
||||
size_t size;
|
||||
size_t capacity;
|
||||
} data_buffer_t;
|
||||
|
||||
/**
|
||||
* Read the file that data points to into a buffer.
|
||||
* NOTE: data must be a file, not a directory.
|
||||
*
|
||||
* @returns The buffer, which is NULL on failure.
|
||||
*/
|
||||
data_buffer_t data_buffer_get_data(data_t const* data);
|
||||
|
||||
/**
|
||||
* Read the dictionary that the data points to into a buffer.
|
||||
*
|
||||
* @returns The buffer, which is NULL on failure.
|
||||
*/
|
||||
data_buffer_t data_buffer_get_dict(data_t const* data);
|
||||
|
||||
/**
|
||||
* Read the contents of filename into a buffer.
|
||||
*
|
||||
* @returns The buffer, which is NULL on failure.
|
||||
*/
|
||||
data_buffer_t data_buffer_read(char const* filename);
|
||||
|
||||
/**
|
||||
* Create a buffer with the specified capacity.
|
||||
*
|
||||
* @returns The buffer, which is NULL on failure.
|
||||
*/
|
||||
data_buffer_t data_buffer_create(size_t capacity);
|
||||
|
||||
/**
|
||||
* Calls memcmp() on the contents [0, size) of both buffers.
|
||||
*/
|
||||
int data_buffer_compare(data_buffer_t buffer1, data_buffer_t buffer2);
|
||||
|
||||
/**
|
||||
* Frees an allocated buffer.
|
||||
*/
|
||||
void data_buffer_free(data_buffer_t buffer);
|
||||
|
||||
typedef struct {
|
||||
char* buffer;
|
||||
char const** filenames;
|
||||
unsigned size;
|
||||
} data_filenames_t;
|
||||
|
||||
/**
|
||||
* Get a recursive list of filenames in the data object. If it is a file, it
|
||||
* will only contain one entry. If it is a directory, it will recursively walk
|
||||
* the directory.
|
||||
*
|
||||
* @returns The list of filenames, which has size 0 and NULL pointers on error.
|
||||
*/
|
||||
data_filenames_t data_filenames_get(data_t const* data);
|
||||
|
||||
/**
|
||||
* Frees the filenames table.
|
||||
*/
|
||||
void data_filenames_free(data_filenames_t filenames);
|
||||
|
||||
typedef struct {
|
||||
data_buffer_t const* buffers;
|
||||
size_t size;
|
||||
} data_buffers_t;
|
||||
|
||||
/**
|
||||
* @returns a list of buffers for every file in data. It is zero sized on error.
|
||||
*/
|
||||
data_buffers_t data_buffers_get(data_t const* data);
|
||||
|
||||
/**
|
||||
* Frees the data buffers.
|
||||
*/
|
||||
void data_buffers_free(data_buffers_t buffers);
|
||||
|
||||
#endif
|
||||
44
tests/regression/levels.h
Normal file
44
tests/regression/levels.h
Normal file
@ -0,0 +1,44 @@
|
||||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef LEVEL
|
||||
# error LEVEL(x) must be defined
|
||||
#endif
|
||||
#ifndef FAST_LEVEL
|
||||
# error FAST_LEVEL(x) must be defined
|
||||
#endif
|
||||
|
||||
/**
|
||||
* The levels are chosen to trigger every strategy in every source size,
|
||||
* as well as some fast levels and the default level.
|
||||
* If you change the compression levels, you should probably update these.
|
||||
*/
|
||||
|
||||
FAST_LEVEL(5)
|
||||
|
||||
FAST_LEVEL(3)
|
||||
|
||||
FAST_LEVEL(1)
|
||||
LEVEL(0)
|
||||
LEVEL(1)
|
||||
|
||||
LEVEL(3)
|
||||
LEVEL(4)
|
||||
LEVEL(5)
|
||||
LEVEL(6)
|
||||
LEVEL(7)
|
||||
|
||||
LEVEL(9)
|
||||
|
||||
LEVEL(13)
|
||||
|
||||
LEVEL(16)
|
||||
|
||||
LEVEL(19)
|
||||
566
tests/regression/method.c
Normal file
566
tests/regression/method.c
Normal file
@ -0,0 +1,566 @@
|
||||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#include "method.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include <zstd.h>
|
||||
|
||||
#define MIN(x, y) ((x) < (y) ? (x) : (y))
|
||||
|
||||
static char const* g_zstdcli = NULL;
|
||||
|
||||
void method_set_zstdcli(char const* zstdcli) {
|
||||
g_zstdcli = zstdcli;
|
||||
}
|
||||
|
||||
/**
|
||||
* Macro to get a pointer of type, given ptr, which is a member variable with
|
||||
* the given name, member.
|
||||
*
|
||||
* method_state_t* base = ...;
|
||||
* buffer_state_t* state = container_of(base, buffer_state_t, base);
|
||||
*/
|
||||
#define container_of(ptr, type, member) \
|
||||
((type*)(ptr == NULL ? NULL : (char*)(ptr)-offsetof(type, member)))
|
||||
|
||||
/** State to reuse the same buffers between compression calls. */
|
||||
typedef struct {
|
||||
method_state_t base;
|
||||
data_buffers_t inputs; /**< The input buffer for each file. */
|
||||
data_buffer_t dictionary; /**< The dictionary. */
|
||||
data_buffer_t compressed; /**< The compressed data buffer. */
|
||||
data_buffer_t decompressed; /**< The decompressed data buffer. */
|
||||
} buffer_state_t;
|
||||
|
||||
static size_t buffers_max_size(data_buffers_t buffers) {
|
||||
size_t max = 0;
|
||||
for (size_t i = 0; i < buffers.size; ++i) {
|
||||
if (buffers.buffers[i].size > max)
|
||||
max = buffers.buffers[i].size;
|
||||
}
|
||||
return max;
|
||||
}
|
||||
|
||||
static method_state_t* buffer_state_create(data_t const* data) {
|
||||
buffer_state_t* state = (buffer_state_t*)calloc(1, sizeof(buffer_state_t));
|
||||
if (state == NULL)
|
||||
return NULL;
|
||||
state->base.data = data;
|
||||
state->inputs = data_buffers_get(data);
|
||||
state->dictionary = data_buffer_get_dict(data);
|
||||
size_t const max_size = buffers_max_size(state->inputs);
|
||||
state->compressed = data_buffer_create(ZSTD_compressBound(max_size));
|
||||
state->decompressed = data_buffer_create(max_size);
|
||||
return &state->base;
|
||||
}
|
||||
|
||||
static void buffer_state_destroy(method_state_t* base) {
|
||||
if (base == NULL)
|
||||
return;
|
||||
buffer_state_t* state = container_of(base, buffer_state_t, base);
|
||||
free(state);
|
||||
}
|
||||
|
||||
static int buffer_state_bad(
|
||||
buffer_state_t const* state,
|
||||
config_t const* config) {
|
||||
if (state == NULL) {
|
||||
fprintf(stderr, "buffer_state_t is NULL\n");
|
||||
return 1;
|
||||
}
|
||||
if (state->inputs.size == 0 || state->compressed.data == NULL ||
|
||||
state->decompressed.data == NULL) {
|
||||
fprintf(stderr, "buffer state allocation failure\n");
|
||||
return 1;
|
||||
}
|
||||
if (config->use_dictionary && state->dictionary.data == NULL) {
|
||||
fprintf(stderr, "dictionary loading failed\n");
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static result_t simple_compress(method_state_t* base, config_t const* config) {
|
||||
buffer_state_t* state = container_of(base, buffer_state_t, base);
|
||||
|
||||
if (buffer_state_bad(state, config))
|
||||
return result_error(result_error_system_error);
|
||||
|
||||
/* Keep the tests short by skipping directories, since behavior shouldn't
|
||||
* change.
|
||||
*/
|
||||
if (base->data->type != data_type_file)
|
||||
return result_error(result_error_skip);
|
||||
|
||||
if (config->use_dictionary || config->no_pledged_src_size)
|
||||
return result_error(result_error_skip);
|
||||
|
||||
/* If the config doesn't specify a level, skip. */
|
||||
int const level = config_get_level(config);
|
||||
if (level == CONFIG_NO_LEVEL)
|
||||
return result_error(result_error_skip);
|
||||
|
||||
data_buffer_t const input = state->inputs.buffers[0];
|
||||
|
||||
/* Compress, decompress, and check the result. */
|
||||
state->compressed.size = ZSTD_compress(
|
||||
state->compressed.data,
|
||||
state->compressed.capacity,
|
||||
input.data,
|
||||
input.size,
|
||||
level);
|
||||
if (ZSTD_isError(state->compressed.size))
|
||||
return result_error(result_error_compression_error);
|
||||
|
||||
state->decompressed.size = ZSTD_decompress(
|
||||
state->decompressed.data,
|
||||
state->decompressed.capacity,
|
||||
state->compressed.data,
|
||||
state->compressed.size);
|
||||
if (ZSTD_isError(state->decompressed.size))
|
||||
return result_error(result_error_decompression_error);
|
||||
if (data_buffer_compare(input, state->decompressed))
|
||||
return result_error(result_error_round_trip_error);
|
||||
|
||||
result_data_t data;
|
||||
data.total_size = state->compressed.size;
|
||||
return result_data(data);
|
||||
}
|
||||
|
||||
static result_t compress_cctx_compress(
|
||||
method_state_t* base,
|
||||
config_t const* config) {
|
||||
buffer_state_t* state = container_of(base, buffer_state_t, base);
|
||||
|
||||
if (buffer_state_bad(state, config))
|
||||
return result_error(result_error_system_error);
|
||||
|
||||
if (config->no_pledged_src_size)
|
||||
return result_error(result_error_skip);
|
||||
|
||||
if (base->data->type != data_type_dir)
|
||||
return result_error(result_error_skip);
|
||||
|
||||
int const level = config_get_level(config);
|
||||
|
||||
ZSTD_CCtx* cctx = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||
if (cctx == NULL || dctx == NULL) {
|
||||
fprintf(stderr, "context creation failed\n");
|
||||
return result_error(result_error_system_error);
|
||||
}
|
||||
|
||||
result_t result;
|
||||
result_data_t data = {.total_size = 0};
|
||||
for (size_t i = 0; i < state->inputs.size; ++i) {
|
||||
data_buffer_t const input = state->inputs.buffers[i];
|
||||
ZSTD_parameters const params =
|
||||
config_get_zstd_params(config, input.size, state->dictionary.size);
|
||||
|
||||
if (level == CONFIG_NO_LEVEL)
|
||||
state->compressed.size = ZSTD_compress_advanced(
|
||||
cctx,
|
||||
state->compressed.data,
|
||||
state->compressed.capacity,
|
||||
input.data,
|
||||
input.size,
|
||||
state->dictionary.data,
|
||||
state->dictionary.size,
|
||||
params);
|
||||
else if (config->use_dictionary)
|
||||
state->compressed.size = ZSTD_compress_usingDict(
|
||||
cctx,
|
||||
state->compressed.data,
|
||||
state->compressed.capacity,
|
||||
input.data,
|
||||
input.size,
|
||||
state->dictionary.data,
|
||||
state->dictionary.size,
|
||||
level);
|
||||
else
|
||||
state->compressed.size = ZSTD_compressCCtx(
|
||||
cctx,
|
||||
state->compressed.data,
|
||||
state->compressed.capacity,
|
||||
input.data,
|
||||
input.size,
|
||||
level);
|
||||
|
||||
if (ZSTD_isError(state->compressed.size)) {
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (config->use_dictionary)
|
||||
state->decompressed.size = ZSTD_decompress_usingDict(
|
||||
dctx,
|
||||
state->decompressed.data,
|
||||
state->decompressed.capacity,
|
||||
state->compressed.data,
|
||||
state->compressed.size,
|
||||
state->dictionary.data,
|
||||
state->dictionary.size);
|
||||
else
|
||||
state->decompressed.size = ZSTD_decompressDCtx(
|
||||
dctx,
|
||||
state->decompressed.data,
|
||||
state->decompressed.capacity,
|
||||
state->compressed.data,
|
||||
state->compressed.size);
|
||||
if (ZSTD_isError(state->decompressed.size)) {
|
||||
result = result_error(result_error_decompression_error);
|
||||
goto out;
|
||||
}
|
||||
if (data_buffer_compare(input, state->decompressed)) {
|
||||
result = result_error(result_error_round_trip_error);
|
||||
goto out;
|
||||
}
|
||||
|
||||
data.total_size += state->compressed.size;
|
||||
}
|
||||
|
||||
result = result_data(data);
|
||||
out:
|
||||
ZSTD_freeCCtx(cctx);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Generic state creation function. */
|
||||
static method_state_t* method_state_create(data_t const* data) {
|
||||
method_state_t* state = (method_state_t*)malloc(sizeof(method_state_t));
|
||||
if (state == NULL)
|
||||
return NULL;
|
||||
state->data = data;
|
||||
return state;
|
||||
}
|
||||
|
||||
static void method_state_destroy(method_state_t* state) {
|
||||
free(state);
|
||||
}
|
||||
|
||||
static result_t cli_compress(method_state_t* state, config_t const* config) {
|
||||
if (config->cli_args == NULL)
|
||||
return result_error(result_error_skip);
|
||||
|
||||
/* We don't support no pledged source size with directories. Too slow. */
|
||||
if (state->data->type == data_type_dir && config->no_pledged_src_size)
|
||||
return result_error(result_error_skip);
|
||||
|
||||
if (g_zstdcli == NULL)
|
||||
return result_error(result_error_system_error);
|
||||
|
||||
/* '<zstd>' -cqr <args> [-D '<dict>'] '<file/dir>' */
|
||||
char cmd[1024];
|
||||
size_t const cmd_size = snprintf(
|
||||
cmd,
|
||||
sizeof(cmd),
|
||||
"'%s' -cqr %s %s%s%s %s '%s'",
|
||||
g_zstdcli,
|
||||
config->cli_args,
|
||||
config->use_dictionary ? "-D '" : "",
|
||||
config->use_dictionary ? state->data->dict.path : "",
|
||||
config->use_dictionary ? "'" : "",
|
||||
config->no_pledged_src_size ? "<" : "",
|
||||
state->data->data.path);
|
||||
if (cmd_size >= sizeof(cmd)) {
|
||||
fprintf(stderr, "command too large: %s\n", cmd);
|
||||
return result_error(result_error_system_error);
|
||||
}
|
||||
FILE* zstd = popen(cmd, "r");
|
||||
if (zstd == NULL) {
|
||||
fprintf(stderr, "failed to popen command: %s\n", cmd);
|
||||
return result_error(result_error_system_error);
|
||||
}
|
||||
|
||||
char out[4096];
|
||||
size_t total_size = 0;
|
||||
while (1) {
|
||||
size_t const size = fread(out, 1, sizeof(out), zstd);
|
||||
total_size += size;
|
||||
if (size != sizeof(out))
|
||||
break;
|
||||
}
|
||||
if (ferror(zstd) || pclose(zstd) != 0) {
|
||||
fprintf(stderr, "zstd failed with command: %s\n", cmd);
|
||||
return result_error(result_error_compression_error);
|
||||
}
|
||||
|
||||
result_data_t const data = {.total_size = total_size};
|
||||
return result_data(data);
|
||||
}
|
||||
|
||||
static int advanced_config(
|
||||
ZSTD_CCtx* cctx,
|
||||
buffer_state_t* state,
|
||||
config_t const* config) {
|
||||
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
|
||||
for (size_t p = 0; p < config->param_values.size; ++p) {
|
||||
param_value_t const pv = config->param_values.data[p];
|
||||
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, pv.param, pv.value))) {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
if (config->use_dictionary) {
|
||||
if (ZSTD_isError(ZSTD_CCtx_loadDictionary(
|
||||
cctx, state->dictionary.data, state->dictionary.size))) {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static result_t advanced_one_pass_compress_output_adjustment(
|
||||
method_state_t* base,
|
||||
config_t const* config,
|
||||
size_t const subtract) {
|
||||
buffer_state_t* state = container_of(base, buffer_state_t, base);
|
||||
|
||||
if (buffer_state_bad(state, config))
|
||||
return result_error(result_error_system_error);
|
||||
|
||||
ZSTD_CCtx* cctx = ZSTD_createCCtx();
|
||||
result_t result;
|
||||
|
||||
if (!cctx || advanced_config(cctx, state, config)) {
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
}
|
||||
|
||||
result_data_t data = {.total_size = 0};
|
||||
for (size_t i = 0; i < state->inputs.size; ++i) {
|
||||
data_buffer_t const input = state->inputs.buffers[i];
|
||||
|
||||
if (!config->no_pledged_src_size) {
|
||||
if (ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(cctx, input.size))) {
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
size_t const size = ZSTD_compress2(
|
||||
cctx,
|
||||
state->compressed.data,
|
||||
ZSTD_compressBound(input.size) - subtract,
|
||||
input.data,
|
||||
input.size);
|
||||
if (ZSTD_isError(size)) {
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
}
|
||||
data.total_size += size;
|
||||
}
|
||||
|
||||
result = result_data(data);
|
||||
out:
|
||||
ZSTD_freeCCtx(cctx);
|
||||
return result;
|
||||
}
|
||||
|
||||
static result_t advanced_one_pass_compress(
|
||||
method_state_t* base,
|
||||
config_t const* config) {
|
||||
return advanced_one_pass_compress_output_adjustment(base, config, 0);
|
||||
}
|
||||
|
||||
static result_t advanced_one_pass_compress_small_output(
|
||||
method_state_t* base,
|
||||
config_t const* config) {
|
||||
return advanced_one_pass_compress_output_adjustment(base, config, 1);
|
||||
}
|
||||
|
||||
static result_t advanced_streaming_compress(
|
||||
method_state_t* base,
|
||||
config_t const* config) {
|
||||
buffer_state_t* state = container_of(base, buffer_state_t, base);
|
||||
|
||||
if (buffer_state_bad(state, config))
|
||||
return result_error(result_error_system_error);
|
||||
|
||||
ZSTD_CCtx* cctx = ZSTD_createCCtx();
|
||||
result_t result;
|
||||
|
||||
if (!cctx || advanced_config(cctx, state, config)) {
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
}
|
||||
|
||||
result_data_t data = {.total_size = 0};
|
||||
for (size_t i = 0; i < state->inputs.size; ++i) {
|
||||
data_buffer_t input = state->inputs.buffers[i];
|
||||
|
||||
if (!config->no_pledged_src_size) {
|
||||
if (ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(cctx, input.size))) {
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
|
||||
while (input.size > 0) {
|
||||
ZSTD_inBuffer in = {input.data, MIN(input.size, 4096)};
|
||||
input.data += in.size;
|
||||
input.size -= in.size;
|
||||
ZSTD_EndDirective const op =
|
||||
input.size > 0 ? ZSTD_e_continue : ZSTD_e_end;
|
||||
size_t ret = 0;
|
||||
while (in.pos < in.size || (op == ZSTD_e_end && ret != 0)) {
|
||||
ZSTD_outBuffer out = {state->compressed.data,
|
||||
MIN(state->compressed.capacity, 1024)};
|
||||
ret = ZSTD_compressStream2(cctx, &out, &in, op);
|
||||
if (ZSTD_isError(ret)) {
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
}
|
||||
data.total_size += out.pos;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result = result_data(data);
|
||||
out:
|
||||
ZSTD_freeCCtx(cctx);
|
||||
return result;
|
||||
}
|
||||
|
||||
static result_t old_streaming_compress(
|
||||
method_state_t* base,
|
||||
config_t const* config) {
|
||||
buffer_state_t* state = container_of(base, buffer_state_t, base);
|
||||
|
||||
if (buffer_state_bad(state, config))
|
||||
return result_error(result_error_system_error);
|
||||
|
||||
int const level = config_get_level(config);
|
||||
if (level == CONFIG_NO_LEVEL)
|
||||
return result_error(result_error_skip);
|
||||
|
||||
ZSTD_CStream* zcs = ZSTD_createCStream();
|
||||
result_t result;
|
||||
if (zcs == NULL) {
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
}
|
||||
size_t zret;
|
||||
if (config->use_dictionary) {
|
||||
zret = ZSTD_initCStream_usingDict(
|
||||
zcs, state->dictionary.data, state->dictionary.size, level);
|
||||
} else {
|
||||
zret = ZSTD_initCStream(zcs, level);
|
||||
}
|
||||
if (ZSTD_isError(zret)) {
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
}
|
||||
|
||||
result_data_t data = {.total_size = 0};
|
||||
for (size_t i = 0; i < state->inputs.size; ++i) {
|
||||
data_buffer_t input = state->inputs.buffers[i];
|
||||
zret = ZSTD_resetCStream(
|
||||
zcs,
|
||||
config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN
|
||||
: input.size);
|
||||
if (ZSTD_isError(zret)) {
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
}
|
||||
|
||||
while (input.size > 0) {
|
||||
ZSTD_inBuffer in = {input.data, MIN(input.size, 4096)};
|
||||
input.data += in.size;
|
||||
input.size -= in.size;
|
||||
ZSTD_EndDirective const op =
|
||||
input.size > 0 ? ZSTD_e_continue : ZSTD_e_end;
|
||||
zret = 0;
|
||||
while (in.pos < in.size || (op == ZSTD_e_end && zret != 0)) {
|
||||
ZSTD_outBuffer out = {state->compressed.data,
|
||||
MIN(state->compressed.capacity, 1024)};
|
||||
if (op == ZSTD_e_continue || in.pos < in.size)
|
||||
zret = ZSTD_compressStream(zcs, &out, &in);
|
||||
else
|
||||
zret = ZSTD_endStream(zcs, &out);
|
||||
if (ZSTD_isError(zret)) {
|
||||
result = result_error(result_error_compression_error);
|
||||
goto out;
|
||||
}
|
||||
data.total_size += out.pos;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result = result_data(data);
|
||||
out:
|
||||
ZSTD_freeCStream(zcs);
|
||||
return result;
|
||||
}
|
||||
|
||||
method_t const simple = {
|
||||
.name = "compress simple",
|
||||
.create = buffer_state_create,
|
||||
.compress = simple_compress,
|
||||
.destroy = buffer_state_destroy,
|
||||
};
|
||||
|
||||
method_t const compress_cctx = {
|
||||
.name = "compress cctx",
|
||||
.create = buffer_state_create,
|
||||
.compress = compress_cctx_compress,
|
||||
.destroy = buffer_state_destroy,
|
||||
};
|
||||
|
||||
method_t const advanced_one_pass = {
|
||||
.name = "advanced one pass",
|
||||
.create = buffer_state_create,
|
||||
.compress = advanced_one_pass_compress,
|
||||
.destroy = buffer_state_destroy,
|
||||
};
|
||||
|
||||
method_t const advanced_one_pass_small_out = {
|
||||
.name = "advanced one pass small out",
|
||||
.create = buffer_state_create,
|
||||
.compress = advanced_one_pass_compress,
|
||||
.destroy = buffer_state_destroy,
|
||||
};
|
||||
|
||||
method_t const advanced_streaming = {
|
||||
.name = "advanced streaming",
|
||||
.create = buffer_state_create,
|
||||
.compress = advanced_streaming_compress,
|
||||
.destroy = buffer_state_destroy,
|
||||
};
|
||||
|
||||
method_t const old_streaming = {
|
||||
.name = "old streaming",
|
||||
.create = buffer_state_create,
|
||||
.compress = old_streaming_compress,
|
||||
.destroy = buffer_state_destroy,
|
||||
};
|
||||
|
||||
method_t const cli = {
|
||||
.name = "zstdcli",
|
||||
.create = method_state_create,
|
||||
.compress = cli_compress,
|
||||
.destroy = method_state_destroy,
|
||||
};
|
||||
|
||||
static method_t const* g_methods[] = {
|
||||
&simple,
|
||||
&compress_cctx,
|
||||
&cli,
|
||||
&advanced_one_pass,
|
||||
&advanced_one_pass_small_out,
|
||||
&advanced_streaming,
|
||||
&old_streaming,
|
||||
NULL,
|
||||
};
|
||||
|
||||
method_t const* const* methods = g_methods;
|
||||
65
tests/regression/method.h
Normal file
65
tests/regression/method.h
Normal file
@ -0,0 +1,65 @@
|
||||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef METHOD_H
|
||||
#define METHOD_H
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
#include "data.h"
|
||||
#include "config.h"
|
||||
#include "result.h"
|
||||
|
||||
/**
|
||||
* The base class for state that methods keep.
|
||||
* All derived method state classes must have a member of this type.
|
||||
*/
|
||||
typedef struct {
|
||||
data_t const* data;
|
||||
} method_state_t;
|
||||
|
||||
/**
|
||||
* A method that compresses the data using config.
|
||||
*/
|
||||
typedef struct {
|
||||
char const* name; /**< The identifier for this method in the results. */
|
||||
/**
|
||||
* Creates a state that must contain a member variable of method_state_t,
|
||||
* and returns a pointer to that member variable.
|
||||
*
|
||||
* This method can be used to do expensive work that only depends on the
|
||||
* data, like loading the data file into a buffer.
|
||||
*/
|
||||
method_state_t* (*create)(data_t const* data);
|
||||
/**
|
||||
* Compresses the data in the state using the given config.
|
||||
*
|
||||
* @param state A pointer to the state returned by create().
|
||||
*
|
||||
* @returns The total compressed size on success, or an error code.
|
||||
*/
|
||||
result_t (*compress)(method_state_t* state, config_t const* config);
|
||||
/**
|
||||
* Frees the state.
|
||||
*/
|
||||
void (*destroy)(method_state_t* state);
|
||||
} method_t;
|
||||
|
||||
/**
|
||||
* Set the zstd cli path. Must be called before any methods are used.
|
||||
*/
|
||||
void method_set_zstdcli(char const* zstdcli);
|
||||
|
||||
/**
|
||||
* A NULL-terminated list of methods.
|
||||
*/
|
||||
extern method_t const* const* methods;
|
||||
|
||||
#endif
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user