Merge xz 5.2.0.

This brings support for multi-threaded compression.  This brings close
N times faster compression where N is the number of CPU cores.
Because of this, liblzma now depends on libthr.

Soon libarchive will be modified to use the new lzma API.

Thanks to antoine@ for the exp-run.

Differential Revision:	 https://reviews.freebsd.org/D1786
Reviewed by:	bapt

Makefile.inc1

@@ -1671,6 +1671,7 @@ _lib_libcapsicum=lib/libcapsicum
 
 lib/libcapsicum__L: lib/libnv__L
 lib/libpjdlog__L: lib/libutil__L
+lib/liblzma__L: lib/libthr__L
 
 _generic_libs=	${_cddl_lib} gnu/lib ${_kerberos5_lib} lib ${_secure_lib} usr.bin/lex/lib ${_ofed_lib}
 .for _DIR in ${LOCAL_LIB_DIRS}

ObsoleteFiles.inc

@@ -38,6 +38,8 @@
 #   xargs -n1 | sort | uniq -d;
 # done
 
+# 20150209: liblzma header
+OLD_FILES+=usr/include/lzma/lzma.h
 # 20150124: spl.9 and friends
 OLD_FILES+=usr/share/man/man9/spl.9
 OLD_FILES+=usr/share/man/man9/spl0.9

contrib/xz/THANKS

@@ -30,17 +30,26 @@ has been important. :-) In alphabetical order:
   - Denis Excoffier
   - Michael Felt
   - Mike Frysinger
+  - Daniel Richard G.
+  - Bill Glessner
   - Jason Gorski
   - Juan Manuel Guerrero
+  - Diederik de Haas
   - Joachim Henke
+  - Christian Hesse
+  - Vincenzo Innocente
   - Peter Ivanov
   - Jouk Jansen
+  - Jun I Jin
   - Per Øyvind Karlsen
   - Thomas Klausner
   - Richard Koch
   - Ville Koskinen
+  - Jan Kratochvil
+  - Christian Kujau
   - Stephan Kulow
   - Peter Lawler
+  - James M Leddy
   - Hin-Tak Leung
   - Andraž 'ruskie' Levstik
   - Cary Lewis
@@ -49,6 +58,7 @@ has been important. :-) In alphabetical order:
   - Bela Lubkin
   - Gregory Margo
   - Jim Meyering
+  - Arkadiusz Miskiewicz
   - Conley Moorhous
   - Rafał Mużyło
   - Adrien Nader
@@ -61,6 +71,7 @@ has been important. :-) In alphabetical order:
   - Diego Elio Pettenò
   - Elbert Pol
   - Mikko Pouru
+  - Trần Ngọc Quân
   - Pavel Raiskup
   - Robert Readman
   - Bernhard Reutner-Fischer
@@ -75,6 +86,7 @@ has been important. :-) In alphabetical order:
   - Stuart Shelton
   - Jonathan Stott
   - Dan Stromberg
+  - Vincent Torri
   - Paul Townsend
   - Mohammed Adnène Trojette
   - Alexey Tourbin
@@ -82,6 +94,7 @@ has been important. :-) In alphabetical order:
   - Martin Väth
   - Christian Weisgerber
   - Bert Wesarg
+  - Fredrik Wikstrom
   - Ralf Wildenhues
   - Charles Wilson
   - Lars Wirzenius

contrib/xz/TODO

@@ -28,14 +28,36 @@ Known bugs
     time and calculated (de)compression speed won't make sense in the
     progress indicator (xz --verbose).
 
+    If liblzma has created threads and fork() gets called, liblzma
+    code will break in the child process unless it calls exec() and
+    doesn't touch liblzma.
+
 
 Missing features
 ----------------
 
+    Add support for storing metadata in .xz files. A preliminary
+    idea is to create a new Stream type for metadata. When both
+    metadata and data are wanted in the same .xz file, two or more
+    Streams would be concatenated.
+
+    The state stored in lzma_stream should be cloneable, which would
+    be mostly useful when using a preset dictionary in LZMA2, but
+    it may have other uses too. Compare to deflateCopy() in zlib.
+
+    Support LZMA_FINISH in raw decoder to indicate end of LZMA1 and
+    other streams that don't have an end of payload marker.
+
+    Adjust dictionary size when the input file size is known.
+    Maybe do this only if an option is given.
+
     xz doesn't support copying extended attributes, access control
     lists etc. from source to target file.
 
-    Multithreaded compression
+    Multithreaded compression:
+      - Reduce memory usage of the current method.
+      - Implement threaded match finders.
+      - Implement pigz-style threading in LZMA2.
 
     Multithreaded decompression
 
@@ -46,18 +68,44 @@ Missing features
     It will be a separate library that supports uncompressed, .gz,
     .bz2, .lzma, and .xz files.
 
+    Support changing lzma_options_lzma.mode with lzma_filters_update().
+
+    Support LZMA_FULL_FLUSH for lzma_stream_decoder() to stop at
+    Block and Stream boundaries.
+
     lzma_strerror() to convert lzma_ret to human readable form?
     This is tricky, because the same error codes are used with
     slightly different meanings, and this cannot be fixed anymore.
 
+    Make it possible to adjust LZMA2 options in the middle of a Block
+    so that the encoding speed vs. compression ratio can be optimized
+    when the compressed data is streamed over network.
+
+    Improved BCJ filters. The current filters are small but they aren't
+    so great when compressing binary packages that contain various file
+    types. Specifically, they make things worse if there are static
+    libraries or Linux kernel modules. The filtering could also be
+    more effective (without getting overly complex), for example,
+    streamable variant BCJ2 from 7-Zip could be implemented.
+
+    Filter that autodetects specific data types in the input stream
+    and applies appropriate filters for the corrects parts of the input.
+    Perhaps combine this with the BCJ filter improvement point above.
+
+    Long-range LZ77 method as a separate filter or as a new LZMA2
+    match finder.
+
 
 Documentation
 -------------
 
-    Some tutorial is needed for liblzma. I have planned to write some
-    extremely well commented example programs, which would work as
-    a tutorial. I suppose the Doxygen tags are quite OK as a quick
-    reference once one is familiar with the liblzma API.
+    More tutorial programs are needed for liblzma.
 
     Document the LZMA1 and LZMA2 algorithms.
 
+
+Miscellaneous
+------------
+
+    Try to get the media type for .xz registered at IANA.
+

contrib/xz/src/common/mythread.h

@@ -1,7 +1,7 @@
 ///////////////////////////////////////////////////////////////////////////////
 //
 /// \file       mythread.h
-/// \brief      Wrappers for threads
+/// \brief      Some threading related helper macros and functions
 //
 //  Author:     Lasse Collin
 //
@@ -10,33 +10,512 @@
 //
 ///////////////////////////////////////////////////////////////////////////////
 
+#ifndef MYTHREAD_H
+#define MYTHREAD_H
+
 #include "sysdefs.h"
 
+// If any type of threading is enabled, #define MYTHREAD_ENABLED.
+#if defined(MYTHREAD_POSIX) || defined(MYTHREAD_WIN95) \
+		|| defined(MYTHREAD_VISTA)
+#	define MYTHREAD_ENABLED 1
+#endif
 
-#ifdef HAVE_PTHREAD
-#	include <pthread.h>
 
-#	define mythread_once(func) \
+#ifdef MYTHREAD_ENABLED
+
+////////////////////////////////////////
+// Shared between all threading types //
+////////////////////////////////////////
+
+// Locks a mutex for a duration of a block.
+//
+// Perform mythread_mutex_lock(&mutex) in the beginning of a block
+// and mythread_mutex_unlock(&mutex) at the end of the block. "break"
+// may be used to unlock the mutex and jump out of the block.
+// mythread_sync blocks may be nested.
+//
+// Example:
+//
+//     mythread_sync(mutex) {
+//         foo();
+//         if (some_error)
+//             break; // Skips bar()
+//         bar();
+//     }
+//
+// At least GCC optimizes the loops completely away so it doesn't slow
+// things down at all compared to plain mythread_mutex_lock(&mutex)
+// and mythread_mutex_unlock(&mutex) calls.
+//
+#define mythread_sync(mutex) mythread_sync_helper1(mutex, __LINE__)
+#define mythread_sync_helper1(mutex, line) mythread_sync_helper2(mutex, line)
+#define mythread_sync_helper2(mutex, line) \
+	for (unsigned int mythread_i_ ## line = 0; \
+			mythread_i_ ## line \
+				? (mythread_mutex_unlock(&(mutex)), 0) \
+				: (mythread_mutex_lock(&(mutex)), 1); \
+			mythread_i_ ## line = 1) \
+		for (unsigned int mythread_j_ ## line = 0; \
+				!mythread_j_ ## line; \
+				mythread_j_ ## line = 1)
+#endif
+
+
+#if !defined(MYTHREAD_ENABLED)
+
+//////////////////
+// No threading //
+//////////////////
+
+// Calls the given function once. This isn't thread safe.
+#define mythread_once(func) \
+do { \
+	static bool once_ = false; \
+	if (!once_) { \
+		func(); \
+		once_ = true; \
+	} \
+} while (0)
+
+
+#if !(defined(_WIN32) && !defined(__CYGWIN__))
+// Use sigprocmask() to set the signal mask in single-threaded programs.
+#include <signal.h>
+
+static inline void
+mythread_sigmask(int how, const sigset_t *restrict set,
+		sigset_t *restrict oset)
+{
+	int ret = sigprocmask(how, set, oset);
+	assert(ret == 0);
+	(void)ret;
+}
+#endif
+
+
+#elif defined(MYTHREAD_POSIX)
+
+////////////////////
+// Using pthreads //
+////////////////////
+
+#include <sys/time.h>
+#include <pthread.h>
+#include <signal.h>
+#include <time.h>
+#include <errno.h>
+
+#define MYTHREAD_RET_TYPE void *
+#define MYTHREAD_RET_VALUE NULL
+
+typedef pthread_t mythread;
+typedef pthread_mutex_t mythread_mutex;
+
+typedef struct {
+	pthread_cond_t cond;
+#ifdef HAVE_CLOCK_GETTIME
+	// Clock ID (CLOCK_REALTIME or CLOCK_MONOTONIC) associated with
+	// the condition variable.
+	clockid_t clk_id;
+#endif
+} mythread_cond;
+
+typedef struct timespec mythread_condtime;
+
+
+// Calls the given function once in a thread-safe way.
+#define mythread_once(func) \
 	do { \
 		static pthread_once_t once_ = PTHREAD_ONCE_INIT; \
 		pthread_once(&once_, &func); \
 	} while (0)
 
-#	define mythread_sigmask(how, set, oset) \
-		pthread_sigmask(how, set, oset)
 
+// Use pthread_sigmask() to set the signal mask in multi-threaded programs.
+// Do nothing on OpenVMS since it lacks pthread_sigmask().
+static inline void
+mythread_sigmask(int how, const sigset_t *restrict set,
+		sigset_t *restrict oset)
+{
+#ifdef __VMS
+	(void)how;
+	(void)set;
+	(void)oset;
 #else
+	int ret = pthread_sigmask(how, set, oset);
+	assert(ret == 0);
+	(void)ret;
+#endif
+}
 
-#	define mythread_once(func) \
+
+// Creates a new thread with all signals blocked. Returns zero on success
+// and non-zero on error.
+static inline int
+mythread_create(mythread *thread, void *(*func)(void *arg), void *arg)
+{
+	sigset_t old;
+	sigset_t all;
+	sigfillset(&all);
+
+	mythread_sigmask(SIG_SETMASK, &all, &old);
+	const int ret = pthread_create(thread, NULL, func, arg);
+	mythread_sigmask(SIG_SETMASK, &old, NULL);
+
+	return ret;
+}
+
+// Joins a thread. Returns zero on success and non-zero on error.
+static inline int
+mythread_join(mythread thread)
+{
+	return pthread_join(thread, NULL);
+}
+
+
+// Initiatlizes a mutex. Returns zero on success and non-zero on error.
+static inline int
+mythread_mutex_init(mythread_mutex *mutex)
+{
+	return pthread_mutex_init(mutex, NULL);
+}
+
+static inline void
+mythread_mutex_destroy(mythread_mutex *mutex)
+{
+	int ret = pthread_mutex_destroy(mutex);
+	assert(ret == 0);
+	(void)ret;
+}
+
+static inline void
+mythread_mutex_lock(mythread_mutex *mutex)
+{
+	int ret = pthread_mutex_lock(mutex);
+	assert(ret == 0);
+	(void)ret;
+}
+
+static inline void
+mythread_mutex_unlock(mythread_mutex *mutex)
+{
+	int ret = pthread_mutex_unlock(mutex);
+	assert(ret == 0);
+	(void)ret;
+}
+
+
+// Initializes a condition variable.
+//
+// Using CLOCK_MONOTONIC instead of the default CLOCK_REALTIME makes the
+// timeout in pthread_cond_timedwait() work correctly also if system time
+// is suddenly changed. Unfortunately CLOCK_MONOTONIC isn't available
+// everywhere while the default CLOCK_REALTIME is, so the default is
+// used if CLOCK_MONOTONIC isn't available.
+//
+// If clock_gettime() isn't available at all, gettimeofday() will be used.
+static inline int
+mythread_cond_init(mythread_cond *mycond)
+{
+#ifdef HAVE_CLOCK_GETTIME
+	// NOTE: HAVE_DECL_CLOCK_MONOTONIC is always defined to 0 or 1.
+#	if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && HAVE_DECL_CLOCK_MONOTONIC
+	struct timespec ts;
+	pthread_condattr_t condattr;
+
+	// POSIX doesn't seem to *require* that pthread_condattr_setclock()
+	// will fail if given an unsupported clock ID. Test that
+	// CLOCK_MONOTONIC really is supported using clock_gettime().
+	if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0
+			&& pthread_condattr_init(&condattr) == 0) {
+		int ret = pthread_condattr_setclock(
+				&condattr, CLOCK_MONOTONIC);
+		if (ret == 0)
+			ret = pthread_cond_init(&mycond->cond, &condattr);
+
+		pthread_condattr_destroy(&condattr);
+
+		if (ret == 0) {
+			mycond->clk_id = CLOCK_MONOTONIC;
+			return 0;
+		}
+	}
+
+	// If anything above fails, fall back to the default CLOCK_REALTIME.
+	// POSIX requires that all implementations of clock_gettime() must
+	// support at least CLOCK_REALTIME.
+#	endif
+
+	mycond->clk_id = CLOCK_REALTIME;
+#endif
+
+	return pthread_cond_init(&mycond->cond, NULL);
+}
+
+static inline void
+mythread_cond_destroy(mythread_cond *cond)
+{
+	int ret = pthread_cond_destroy(&cond->cond);
+	assert(ret == 0);
+	(void)ret;
+}
+
+static inline void
+mythread_cond_signal(mythread_cond *cond)
+{
+	int ret = pthread_cond_signal(&cond->cond);
+	assert(ret == 0);
+	(void)ret;
+}
+
+static inline void
+mythread_cond_wait(mythread_cond *cond, mythread_mutex *mutex)
+{
+	int ret = pthread_cond_wait(&cond->cond, mutex);
+	assert(ret == 0);
+	(void)ret;
+}
+
+// Waits on a condition or until a timeout expires. If the timeout expires,
+// non-zero is returned, otherwise zero is returned.
+static inline int
+mythread_cond_timedwait(mythread_cond *cond, mythread_mutex *mutex,
+		const mythread_condtime *condtime)
+{
+	int ret = pthread_cond_timedwait(&cond->cond, mutex, condtime);
+	assert(ret == 0 || ret == ETIMEDOUT);
+	return ret;
+}
+
+// Sets condtime to the absolute time that is timeout_ms milliseconds
+// in the future. The type of the clock to use is taken from cond.
+static inline void
+mythread_condtime_set(mythread_condtime *condtime, const mythread_cond *cond,
+		uint32_t timeout_ms)
+{
+	condtime->tv_sec = timeout_ms / 1000;
+	condtime->tv_nsec = (timeout_ms % 1000) * 1000000;
+
+#ifdef HAVE_CLOCK_GETTIME
+	struct timespec now;
+	int ret = clock_gettime(cond->clk_id, &now);
+	assert(ret == 0);
+	(void)ret;
+
+	condtime->tv_sec += now.tv_sec;
+	condtime->tv_nsec += now.tv_nsec;
+#else
+	(void)cond;
+
+	struct timeval now;
+	gettimeofday(&now, NULL);
+
+	condtime->tv_sec += now.tv_sec;
+	condtime->tv_nsec += now.tv_usec * 1000L;
+#endif
+
+	// tv_nsec must stay in the range [0, 999_999_999].
+	if (condtime->tv_nsec >= 1000000000L) {
+		condtime->tv_nsec -= 1000000000L;
+		++condtime->tv_sec;
+	}
+}
+
+
+#elif defined(MYTHREAD_WIN95) || defined(MYTHREAD_VISTA)
+
+/////////////////////
+// Windows threads //
+/////////////////////
+
+#define WIN32_LEAN_AND_MEAN
+#ifdef MYTHREAD_VISTA
+#	undef _WIN32_WINNT
+#	define _WIN32_WINNT 0x0600
+#endif
+#include <windows.h>
+#include <process.h>
+
+#define MYTHREAD_RET_TYPE unsigned int __stdcall
+#define MYTHREAD_RET_VALUE 0
+
+typedef HANDLE mythread;
+typedef CRITICAL_SECTION mythread_mutex;
+
+#ifdef MYTHREAD_WIN95
+typedef HANDLE mythread_cond;
+#else
+typedef CONDITION_VARIABLE mythread_cond;
+#endif
+
+typedef struct {
+	// Tick count (milliseconds) in the beginning of the timeout.
+	// NOTE: This is 32 bits so it wraps around after 49.7 days.
+	// Multi-day timeouts may not work as expected.
+	DWORD start;
+
+	// Length of the timeout in milliseconds. The timeout expires
+	// when the current tick count minus "start" is equal or greater
+	// than "timeout".
+	DWORD timeout;
+} mythread_condtime;
+
+
+// mythread_once() is only available with Vista threads.
+#ifdef MYTHREAD_VISTA
+#define mythread_once(func) \
 	do { \
-		static bool once_ = false; \
-		if (!once_) { \
+		static INIT_ONCE once_ = INIT_ONCE_STATIC_INIT; \
+		BOOL pending_; \
+		if (!InitOnceBeginInitialize(&once_, 0, &pending_, NULL)) \
+			abort(); \
+		if (pending_) \
 			func(); \
-			once_ = true; \
-		} \
+		if (!InitOnceComplete(&once, 0, NULL)) \
+			abort(); \
 	} while (0)
+#endif
 
-#	define mythread_sigmask(how, set, oset) \
-		sigprocmask(how, set, oset)
+
+// mythread_sigmask() isn't available on Windows. Even a dummy version would
+// make no sense because the other POSIX signal functions are missing anyway.
+
+
+static inline int
+mythread_create(mythread *thread,
+		unsigned int (__stdcall *func)(void *arg), void *arg)
+{
+	uintptr_t ret = _beginthreadex(NULL, 0, func, arg, 0, NULL);
+	if (ret == 0)
+		return -1;
+
+	*thread = (HANDLE)ret;
+	return 0;
+}
+
+static inline int
+mythread_join(mythread thread)
+{
+	int ret = 0;
+
+	if (WaitForSingleObject(thread, INFINITE) != WAIT_OBJECT_0)
+		ret = -1;
+
+	if (!CloseHandle(thread))
+		ret = -1;
+
+	return ret;
+}
+
+
+static inline int
+mythread_mutex_init(mythread_mutex *mutex)
+{
+	InitializeCriticalSection(mutex);
+	return 0;
+}
+
+static inline void
+mythread_mutex_destroy(mythread_mutex *mutex)
+{
+	DeleteCriticalSection(mutex);
+}
+
+static inline void
+mythread_mutex_lock(mythread_mutex *mutex)
+{
+	EnterCriticalSection(mutex);
+}
+
+static inline void
+mythread_mutex_unlock(mythread_mutex *mutex)
+{
+	LeaveCriticalSection(mutex);
+}
+
+
+static inline int
+mythread_cond_init(mythread_cond *cond)
+{
+#ifdef MYTHREAD_WIN95
+	*cond = CreateEvent(NULL, FALSE, FALSE, NULL);
+	return *cond == NULL ? -1 : 0;
+#else
+	InitializeConditionVariable(cond);
+	return 0;
+#endif
+}
+
+static inline void
+mythread_cond_destroy(mythread_cond *cond)
+{
+#ifdef MYTHREAD_WIN95
+	CloseHandle(*cond);
+#else
+	(void)cond;
+#endif
+}
+
+static inline void
+mythread_cond_signal(mythread_cond *cond)
+{
+#ifdef MYTHREAD_WIN95
+	SetEvent(*cond);
+#else
+	WakeConditionVariable(cond);
+#endif
+}
+
+static inline void
+mythread_cond_wait(mythread_cond *cond, mythread_mutex *mutex)
+{
+#ifdef MYTHREAD_WIN95
+	LeaveCriticalSection(mutex);
+	WaitForSingleObject(*cond, INFINITE);
+	EnterCriticalSection(mutex);
+#else
+	BOOL ret = SleepConditionVariableCS(cond, mutex, INFINITE);
+	assert(ret);
+	(void)ret;
+#endif
+}
+
+static inline int
+mythread_cond_timedwait(mythread_cond *cond, mythread_mutex *mutex,
+		const mythread_condtime *condtime)
+{
+#ifdef MYTHREAD_WIN95
+	LeaveCriticalSection(mutex);
+#endif
+
+	DWORD elapsed = GetTickCount() - condtime->start;
+	DWORD timeout = elapsed >= condtime->timeout
+			? 0 : condtime->timeout - elapsed;
+
+#ifdef MYTHREAD_WIN95
+	DWORD ret = WaitForSingleObject(*cond, timeout);
+	assert(ret == WAIT_OBJECT_0 || ret == WAIT_TIMEOUT);
+
+	EnterCriticalSection(mutex);
+
+	return ret == WAIT_TIMEOUT;
+#else
+	BOOL ret = SleepConditionVariableCS(cond, mutex, timeout);
+	assert(ret || GetLastError() == ERROR_TIMEOUT);
+	return !ret;
+#endif
+}
+
+static inline void
+mythread_condtime_set(mythread_condtime *condtime, const mythread_cond *cond,
+		uint32_t timeout)
+{
+	(void)cond;
+	condtime->start = GetTickCount();
+	condtime->timeout = timeout;
+}
+
+#endif
 
 #endif

contrib/xz/src/common/sysdefs.h

@@ -165,6 +165,16 @@ typedef unsigned char _Bool;
 #	include <memory.h>
 #endif
 
+// As of MSVC 2013, inline and restrict are supported with
+// non-standard keywords.
+#if defined(_WIN32) && defined(_MSC_VER)
+#	ifndef inline
+#		define inline __inline
+#	endif
+#	ifndef restrict
+#		define restrict __restrict
+#	endif
+#endif
 
 ////////////
 // Macros //

contrib/xz/src/common/tuklib_cpucores.c

@@ -12,7 +12,13 @@
 
 #include "tuklib_cpucores.h"
 
-#if defined(TUKLIB_CPUCORES_SYSCTL)
+#if defined(_WIN32) || defined(__CYGWIN__)
+#	ifndef _WIN32_WINNT
+#		define _WIN32_WINNT 0x0500
+#	endif
+#	include <windows.h>
+
+#elif defined(TUKLIB_CPUCORES_SYSCTL)
 #	ifdef HAVE_SYS_PARAM_H
 #		include <sys/param.h>
 #	endif
@@ -33,7 +39,12 @@ tuklib_cpucores(void)
 {
 	uint32_t ret = 0;
 
-#if defined(TUKLIB_CPUCORES_SYSCTL)
+#if defined(_WIN32) || defined(__CYGWIN__)
+	SYSTEM_INFO sysinfo;
+	GetSystemInfo(&sysinfo);
+	ret = sysinfo.dwNumberOfProcessors;
+
+#elif defined(TUKLIB_CPUCORES_SYSCTL)
 	int name[2] = { CTL_HW, HW_NCPU };
 	int cpus;
 	size_t cpus_size = sizeof(cpus);

contrib/xz/src/common/tuklib_physmem.c

@@ -33,6 +33,10 @@
 #	include <syidef.h>
 #	include <ssdef.h>
 
+#elif defined(AMIGA) || defined(__AROS__)
+#	define __USE_INLINE__
+#	include <proto/exec.h>
+
 // AIX
 #elif defined(TUKLIB_PHYSMEM_AIX)
 #	include <sys/systemcfg.h>
@@ -119,6 +123,9 @@ tuklib_physmem(void)
 	if (LIB$GETSYI(&val, &vms_mem, 0, 0, 0, 0) == SS$_NORMAL)
 		ret = (uint64_t)vms_mem * 8192;
 
+#elif defined(AMIGA) || defined(__AROS__)
+	ret = AvailMem(MEMF_TOTAL);
+
 #elif defined(TUKLIB_PHYSMEM_AIX)
 	ret = _system_configuration.physmem;
 

contrib/xz/src/liblzma/api/lzma.h

@@ -286,7 +286,7 @@ extern "C" {
 #include "lzma/filter.h"
 #include "lzma/bcj.h"
 #include "lzma/delta.h"
-#include "lzma/lzma.h"
+#include "lzma/lzma12.h"
 
 /* Container formats */
 #include "lzma/container.h"

contrib/xz/src/liblzma/api/lzma/base.h

@@ -240,12 +240,12 @@ typedef enum {
 /**
  * \brief       The `action' argument for lzma_code()
  *
- * After the first use of LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, or LZMA_FINISH,
- * the same `action' must is used until lzma_code() returns LZMA_STREAM_END.
- * Also, the amount of input (that is, strm->avail_in) must not be modified
- * by the application until lzma_code() returns LZMA_STREAM_END. Changing the
- * `action' or modifying the amount of input will make lzma_code() return
- * LZMA_PROG_ERROR.
+ * After the first use of LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, LZMA_FULL_BARRIER,
+ * or LZMA_FINISH, the same `action' must is used until lzma_code() returns
+ * LZMA_STREAM_END. Also, the amount of input (that is, strm->avail_in) must
+ * not be modified by the application until lzma_code() returns
+ * LZMA_STREAM_END. Changing the `action' or modifying the amount of input
+ * will make lzma_code() return LZMA_PROG_ERROR.
  */
 typedef enum {
 	LZMA_RUN = 0,
@@ -293,7 +293,7 @@ typedef enum {
 		 *
 		 * All the input data going to the current Block must have
 		 * been given to the encoder (the last bytes can still be
-		 * pending in* next_in). Call lzma_code() with LZMA_FULL_FLUSH
+		 * pending in *next_in). Call lzma_code() with LZMA_FULL_FLUSH
 		 * until it returns LZMA_STREAM_END. Then continue normally
 		 * with LZMA_RUN or finish the Stream with LZMA_FINISH.
 		 *
@@ -302,6 +302,29 @@ typedef enum {
 		 * no unfinished Block, no empty Block is created.
 		 */
 
+	LZMA_FULL_BARRIER = 4,
+		/**<
+		 * \brief       Finish encoding of the current Block
+		 *
+		 * This is like LZMA_FULL_FLUSH except that this doesn't
+		 * necessarily wait until all the input has been made
+		 * available via the output buffer. That is, lzma_code()
+		 * might return LZMA_STREAM_END as soon as all the input
+		 * has been consumed (avail_in == 0).
+		 *
+		 * LZMA_FULL_BARRIER is useful with a threaded encoder if
+		 * one wants to split the .xz Stream into Blocks at specific
+		 * offsets but doesn't care if the output isn't flushed
+		 * immediately. Using LZMA_FULL_BARRIER allows keeping
+		 * the threads busy while LZMA_FULL_FLUSH would make
+		 * lzma_code() wait until all the threads have finished
+		 * until more data could be passed to the encoder.
+		 *
+		 * With a lzma_stream initialized with the single-threaded
+		 * lzma_stream_encoder() or lzma_easy_encoder(),
+		 * LZMA_FULL_BARRIER is an alias for LZMA_FULL_FLUSH.
+		 */
+
 	LZMA_FINISH = 3
 		/**<
 		 * \brief       Finish the coding operation
@@ -332,11 +355,19 @@ typedef enum {
  * malloc() and free(). C++ users should note that the custom memory
  * handling functions must not throw exceptions.
  *
- * liblzma doesn't make an internal copy of lzma_allocator. Thus, it is
- * OK to change these function pointers in the middle of the coding
- * process, but obviously it must be done carefully to make sure that the
- * replacement `free' can deallocate memory allocated by the earlier
- * `alloc' function(s).
+ * Single-threaded mode only: liblzma doesn't make an internal copy of
+ * lzma_allocator. Thus, it is OK to change these function pointers in
+ * the middle of the coding process, but obviously it must be done
+ * carefully to make sure that the replacement `free' can deallocate
+ * memory allocated by the earlier `alloc' function(s).
+ *
+ * Multithreaded mode: liblzma might internally store pointers to the
+ * lzma_allocator given via the lzma_stream structure. The application
+ * must not change the allocator pointer in lzma_stream or the contents
+ * of the pointed lzma_allocator structure until lzma_end() has been used
+ * to free the memory associated with that lzma_stream. The allocation
+ * functions might be called simultaneously from multiple threads, and
+ * thus they must be thread safe.
  */
 typedef struct {
 	/**
@@ -448,7 +479,8 @@ typedef struct lzma_internal_s lzma_internal;
  *
  * Application may modify the values of total_in and total_out as it wants.
  * They are updated by liblzma to match the amount of data read and
- * written, but aren't used for anything else.
+ * written but aren't used for anything else except as a possible return
+ * values from lzma_get_progress().
  */
 typedef struct {
 	const uint8_t *next_in; /**< Pointer to the next input byte. */
@@ -464,8 +496,10 @@ typedef struct {
 	 *
 	 * In most cases this is NULL which makes liblzma use
 	 * the standard malloc() and free().
+	 *
+	 * \note        In 5.0.x this is not a const pointer.
 	 */
-	lzma_allocator *allocator;
+	const lzma_allocator *allocator;
 
 	/** Internal state is not visible to applications. */
 	lzma_internal *internal;
@@ -546,6 +580,25 @@ extern LZMA_API(lzma_ret) lzma_code(lzma_stream *strm, lzma_action action)
 extern LZMA_API(void) lzma_end(lzma_stream *strm) lzma_nothrow;
 
 
+/**
+ * \brief       Get progress information
+ *
+ * In single-threaded mode, applications can get progress information from
+ * strm->total_in and strm->total_out. In multi-threaded mode this is less
+ * useful because a significant amount of both input and output data gets
+ * buffered internally by liblzma. This makes total_in and total_out give
+ * misleading information and also makes the progress indicator updates
+ * non-smooth.
+ *
+ * This function gives realistic progress information also in multi-threaded
+ * mode by taking into account the progress made by each thread. In
+ * single-threaded mode *progress_in and *progress_out are set to
+ * strm->total_in and strm->total_out, respectively.
+ */
+extern LZMA_API(void) lzma_get_progress(lzma_stream *strm,
+		uint64_t *progress_in, uint64_t *progress_out) lzma_nothrow;
+
+
 /**
  * \brief       Get the memory usage of decoder filter chain
  *

contrib/xz/src/liblzma/api/lzma/block.h

@@ -31,11 +31,16 @@ typedef struct {
 	/**
 	 * \brief       Block format version
 	 *
-	 * To prevent API and ABI breakages if new features are needed in
-	 * the Block field, a version number is used to indicate which
-	 * fields in this structure are in use. For now, version must always
-	 * be zero. With non-zero version, most Block related functions will
-	 * return LZMA_OPTIONS_ERROR.
+	 * To prevent API and ABI breakages when new features are needed,
+	 * a version number is used to indicate which fields in this
+	 * structure are in use:
+	 *   - liblzma >= 5.0.0: version = 0 is supported.
+	 *   - liblzma >= 5.1.4beta: Support for version = 1 was added,
+	 *     which adds the ignore_check field.
+	 *
+	 * If version is greater than one, most Block related functions
+	 * will return LZMA_OPTIONS_ERROR (lzma_block_header_decode() works
+	 * with any version value).
 	 *
 	 * Read by:
 	 *  - All functions that take pointer to lzma_block as argument,
@@ -233,7 +238,28 @@ typedef struct {
 	lzma_reserved_enum reserved_enum2;
 	lzma_reserved_enum reserved_enum3;
 	lzma_reserved_enum reserved_enum4;
-	lzma_bool reserved_bool1;
+
+	/**
+	 * \brief       A flag to Block decoder to not verify the Check field
+	 *
+	 * This field is supported by liblzma >= 5.1.4beta if .version >= 1.
+	 *
+	 * If this is set to true, the integrity check won't be calculated
+	 * and verified. Unless you know what you are doing, you should
+	 * leave this to false. (A reason to set this to true is when the
+	 * file integrity is verified externally anyway and you want to
+	 * speed up the decompression, which matters mostly when using
+	 * SHA-256 as the integrity check.)
+	 *
+	 * If .version >= 1, read by:
+	 *   - lzma_block_decoder()
+	 *   - lzma_block_buffer_decode()
+	 *
+	 * Written by (.version is ignored):
+	 *   - lzma_block_header_decode() always sets this to false
+	 */
+	lzma_bool ignore_check;
+
 	lzma_bool reserved_bool2;
 	lzma_bool reserved_bool3;
 	lzma_bool reserved_bool4;
@@ -310,10 +336,14 @@ extern LZMA_API(lzma_ret) lzma_block_header_encode(
 /**
  * \brief       Decode Block Header
  *
- * block->version should be set to the highest value supported by the
- * application; currently the only possible version is zero. This function
- * will set version to the lowest value that still supports all the features
- * required by the Block Header.
+ * block->version should (usually) be set to the highest value supported
+ * by the application. If the application sets block->version to a value
+ * higher than supported by the current liblzma version, this function will
+ * downgrade block->version to the highest value supported by it. Thus one
+ * should check the value of block->version after calling this function if
+ * block->version was set to a non-zero value and the application doesn't
+ * otherwise know that the liblzma version being used is new enough to
+ * support the specified block->version.
  *
  * The size of the Block Header must have already been decoded with
  * lzma_block_header_size_decode() macro and stored to block->header_size.
@@ -344,7 +374,7 @@ extern LZMA_API(lzma_ret) lzma_block_header_encode(
  *                block->header_size is invalid or block->filters is NULL.
  */
 extern LZMA_API(lzma_ret) lzma_block_header_decode(lzma_block *block,
-		lzma_allocator *allocator, const uint8_t *in)
+		const lzma_allocator *allocator, const uint8_t *in)
 		lzma_nothrow lzma_attr_warn_unused_result;
 
 
@@ -493,7 +523,25 @@ extern LZMA_API(size_t) lzma_block_buffer_bound(size_t uncompressed_size)
  *              - LZMA_PROG_ERROR
  */
 extern LZMA_API(lzma_ret) lzma_block_buffer_encode(
-		lzma_block *block, lzma_allocator *allocator,
+		lzma_block *block, const lzma_allocator *allocator,
+		const uint8_t *in, size_t in_size,
+		uint8_t *out, size_t *out_pos, size_t out_size)
+		lzma_nothrow lzma_attr_warn_unused_result;
+
+
+/**
+ * \brief       Single-call uncompressed .xz Block encoder
+ *
+ * This is like lzma_block_buffer_encode() except this doesn't try to
+ * compress the data and instead encodes the data using LZMA2 uncompressed
+ * chunks. The required output buffer size can be determined with
+ * lzma_block_buffer_bound().
+ *
+ * Since the data won't be compressed, this function ignores block->filters.
+ * This function doesn't take lzma_allocator because this function doesn't
+ * allocate any memory from the heap.
+ */
+extern LZMA_API(lzma_ret) lzma_block_uncomp_encode(lzma_block *block,
 		const uint8_t *in, size_t in_size,
 		uint8_t *out, size_t *out_pos, size_t out_size)
 		lzma_nothrow lzma_attr_warn_unused_result;
@@ -527,7 +575,7 @@ extern LZMA_API(lzma_ret) lzma_block_buffer_encode(
  *              - LZMA_PROG_ERROR
  */
 extern LZMA_API(lzma_ret) lzma_block_buffer_decode(
-		lzma_block *block, lzma_allocator *allocator,
+		lzma_block *block, const lzma_allocator *allocator,
 		const uint8_t *in, size_t *in_pos, size_t in_size,
 		uint8_t *out, size_t *out_pos, size_t out_size)
 		lzma_nothrow;

contrib/xz/src/liblzma/api/lzma/container.h

@@ -60,6 +60,131 @@
 #define LZMA_PRESET_EXTREME       (UINT32_C(1) << 31)
 
 
+/**
+ * \brief       Multithreading options
+ */
+typedef struct {
+	/**
+	 * \brief       Flags
+	 *
+	 * Set this to zero if no flags are wanted.
+	 *
+	 * No flags are currently supported.
+	 */
+	uint32_t flags;
+
+	/**
+	 * \brief       Number of worker threads to use
+	 */
+	uint32_t threads;
+
+	/**
+	 * \brief       Maximum uncompressed size of a Block
+	 *
+	 * The encoder will start a new .xz Block every block_size bytes.
+	 * Using LZMA_FULL_FLUSH or LZMA_FULL_BARRIER with lzma_code()
+	 * the caller may tell liblzma to start a new Block earlier.
+	 *
+	 * With LZMA2, a recommended block size is 2-4 times the LZMA2
+	 * dictionary size. With very small dictionaries, it is recommended
+	 * to use at least 1 MiB block size for good compression ratio, even
+	 * if this is more than four times the dictionary size. Note that
+	 * these are only recommendations for typical use cases; feel free
+	 * to use other values. Just keep in mind that using a block size
+	 * less than the LZMA2 dictionary size is waste of RAM.
+	 *
+	 * Set this to 0 to let liblzma choose the block size depending
+	 * on the compression options. For LZMA2 it will be 3*dict_size
+	 * or 1 MiB, whichever is more.
+	 *
+	 * For each thread, about 3 * block_size bytes of memory will be
+	 * allocated. This may change in later liblzma versions. If so,
+	 * the memory usage will probably be reduced, not increased.
+	 */
+	uint64_t block_size;
+
+	/**
+	 * \brief       Timeout to allow lzma_code() to return early
+	 *
+	 * Multithreading can make liblzma to consume input and produce
+	 * output in a very bursty way: it may first read a lot of input
+	 * to fill internal buffers, then no input or output occurs for
+	 * a while.
+	 *
+	 * In single-threaded mode, lzma_code() won't return until it has
+	 * either consumed all the input or filled the output buffer. If
+	 * this is done in multithreaded mode, it may cause a call
+	 * lzma_code() to take even tens of seconds, which isn't acceptable
+	 * in all applications.
+	 *
+	 * To avoid very long blocking times in lzma_code(), a timeout
+	 * (in milliseconds) may be set here. If lzma_code() would block
+	 * longer than this number of milliseconds, it will return with
+	 * LZMA_OK. Reasonable values are 100 ms or more. The xz command
+	 * line tool uses 300 ms.
+	 *
+	 * If long blocking times are fine for you, set timeout to a special
+	 * value of 0, which will disable the timeout mechanism and will make
+	 * lzma_code() block until all the input is consumed or the output
+	 * buffer has been filled.
+	 *
+	 * \note        Even with a timeout, lzma_code() might sometimes take
+	 *              somewhat long time to return. No timing guarantees
+	 *              are made.
+	 */
+	uint32_t timeout;
+
+	/**
+	 * \brief       Compression preset (level and possible flags)
+	 *
+	 * The preset is set just like with lzma_easy_encoder().
+	 * The preset is ignored if filters below is non-NULL.
+	 */
+	uint32_t preset;
+
+	/**
+	 * \brief       Filter chain (alternative to a preset)
+	 *
+	 * If this is NULL, the preset above is used. Otherwise the preset
+	 * is ignored and the filter chain specified here is used.
+	 */
+	const lzma_filter *filters;
+
+	/**
+	 * \brief       Integrity check type
+	 *
+	 * See check.h for available checks. The xz command line tool
+	 * defaults to LZMA_CHECK_CRC64, which is a good choice if you
+	 * are unsure.
+	 */
+	lzma_check check;
+
+	/*
+	 * Reserved space to allow possible future extensions without
+	 * breaking the ABI. You should not touch these, because the names
+	 * of these variables may change. These are and will never be used
+	 * with the currently supported options, so it is safe to leave these
+	 * uninitialized.
+	 */
+	lzma_reserved_enum reserved_enum1;
+	lzma_reserved_enum reserved_enum2;
+	lzma_reserved_enum reserved_enum3;
+	uint32_t reserved_int1;
+	uint32_t reserved_int2;
+	uint32_t reserved_int3;
+	uint32_t reserved_int4;
+	uint64_t reserved_int5;
+	uint64_t reserved_int6;
+	uint64_t reserved_int7;
+	uint64_t reserved_int8;
+	void *reserved_ptr1;
+	void *reserved_ptr2;
+	void *reserved_ptr3;
+	void *reserved_ptr4;
+
+} lzma_mt;
+
+
 /**
  * \brief       Calculate approximate memory usage of easy encoder
  *
@@ -165,7 +290,8 @@ extern LZMA_API(lzma_ret) lzma_easy_encoder(
  */
 extern LZMA_API(lzma_ret) lzma_easy_buffer_encode(
 		uint32_t preset, lzma_check check,
-		lzma_allocator *allocator, const uint8_t *in, size_t in_size,
+		const lzma_allocator *allocator,
+		const uint8_t *in, size_t in_size,
 		uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow;
 
 
@@ -190,6 +316,49 @@ extern LZMA_API(lzma_ret) lzma_stream_encoder(lzma_stream *strm,
 		lzma_nothrow lzma_attr_warn_unused_result;
 
 
+/**
+ * \brief       Calculate approximate memory usage of multithreaded .xz encoder
+ *
+ * Since doing the encoding in threaded mode doesn't affect the memory
+ * requirements of single-threaded decompressor, you can use
+ * lzma_easy_decoder_memusage(options->preset) or
+ * lzma_raw_decoder_memusage(options->filters) to calculate
+ * the decompressor memory requirements.
+ *
+ * \param       options Compression options
+ *
+ * \return      Number of bytes of memory required for encoding with the
+ *              given options. If an error occurs, for example due to
+ *              unsupported preset or filter chain, UINT64_MAX is returned.
+ */
+extern LZMA_API(uint64_t) lzma_stream_encoder_mt_memusage(
+		const lzma_mt *options) lzma_nothrow lzma_attr_pure;
+
+
+/**
+ * \brief       Initialize multithreaded .xz Stream encoder
+ *
+ * This provides the functionality of lzma_easy_encoder() and
+ * lzma_stream_encoder() as a single function for multithreaded use.
+ *
+ * The supported actions for lzma_code() are LZMA_RUN, LZMA_FULL_FLUSH,
+ * LZMA_FULL_BARRIER, and LZMA_FINISH. Support for LZMA_SYNC_FLUSH might be
+ * added in the future.
+ *
+ * \param       strm    Pointer to properly prepared lzma_stream
+ * \param       options Pointer to multithreaded compression options
+ *
+ * \return      - LZMA_OK
+ *              - LZMA_MEM_ERROR
+ *              - LZMA_UNSUPPORTED_CHECK
+ *              - LZMA_OPTIONS_ERROR
+ *              - LZMA_PROG_ERROR
+ */
+extern LZMA_API(lzma_ret) lzma_stream_encoder_mt(
+		lzma_stream *strm, const lzma_mt *options)
+		lzma_nothrow lzma_attr_warn_unused_result;
+
+
 /**
  * \brief       Initialize .lzma encoder (legacy file format)
  *
@@ -269,7 +438,8 @@ extern LZMA_API(size_t) lzma_stream_buffer_bound(size_t uncompressed_size)
  */
 extern LZMA_API(lzma_ret) lzma_stream_buffer_encode(
 		lzma_filter *filters, lzma_check check,
-		lzma_allocator *allocator, const uint8_t *in, size_t in_size,
+		const lzma_allocator *allocator,
+		const uint8_t *in, size_t in_size,
 		uint8_t *out, size_t *out_pos, size_t out_size)
 		lzma_nothrow lzma_attr_warn_unused_result;
 
@@ -304,6 +474,30 @@ extern LZMA_API(lzma_ret) lzma_stream_buffer_encode(
 #define LZMA_TELL_ANY_CHECK             UINT32_C(0x04)
 
 
+/**
+ * This flag makes lzma_code() not calculate and verify the integrity check
+ * of the compressed data in .xz files. This means that invalid integrity
+ * check values won't be detected and LZMA_DATA_ERROR won't be returned in
+ * such cases.
+ *
+ * This flag only affects the checks of the compressed data itself; the CRC32
+ * values in the .xz headers will still be verified normally.
+ *
+ * Don't use this flag unless you know what you are doing. Possible reasons
+ * to use this flag:
+ *
+ *   - Trying to recover data from a corrupt .xz file.
+ *
+ *   - Speeding up decompression, which matters mostly with SHA-256
+ *     or with files that have compressed extremely well. It's recommended
+ *     to not use this flag for this purpose unless the file integrity is
+ *     verified externally in some other way.
+ *
+ * Support for this flag was added in liblzma 5.1.4beta.
+ */
+#define LZMA_IGNORE_CHECK               UINT32_C(0x10)
+
+
 /**
  * This flag enables decoding of concatenated files with file formats that
  * allow concatenating compressed files as is. From the formats currently
@@ -418,7 +612,8 @@ extern LZMA_API(lzma_ret) lzma_alone_decoder(
  *              - LZMA_PROG_ERROR
  */
 extern LZMA_API(lzma_ret) lzma_stream_buffer_decode(
-		uint64_t *memlimit, uint32_t flags, lzma_allocator *allocator,
+		uint64_t *memlimit, uint32_t flags,
+		const lzma_allocator *allocator,
 		const uint8_t *in, size_t *in_pos, size_t in_size,
 		uint8_t *out, size_t *out_pos, size_t out_size)
 		lzma_nothrow lzma_attr_warn_unused_result;

contrib/xz/src/liblzma/api/lzma/filter.h

@@ -116,8 +116,9 @@ extern LZMA_API(lzma_bool) lzma_filter_decoder_is_supported(lzma_vli id)
  *                is not NULL.
  *              - LZMA_PROG_ERROR: src or dest is NULL.
  */
-extern LZMA_API(lzma_ret) lzma_filters_copy(const lzma_filter *src,
-		lzma_filter *dest, lzma_allocator *allocator) lzma_nothrow;
+extern LZMA_API(lzma_ret) lzma_filters_copy(
+		const lzma_filter *src, lzma_filter *dest,
+		const lzma_allocator *allocator) lzma_nothrow;
 
 
 /**
@@ -256,7 +257,7 @@ extern LZMA_API(lzma_ret) lzma_filters_update(
  *              won't necessarily meet that bound.)
  */
 extern LZMA_API(lzma_ret) lzma_raw_buffer_encode(
-		const lzma_filter *filters, lzma_allocator *allocator,
+		const lzma_filter *filters, const lzma_allocator *allocator,
 		const uint8_t *in, size_t in_size, uint8_t *out,
 		size_t *out_pos, size_t out_size) lzma_nothrow;
 
@@ -280,7 +281,7 @@ extern LZMA_API(lzma_ret) lzma_raw_buffer_encode(
  *                          which no data is written to is out[out_size].
  */
 extern LZMA_API(lzma_ret) lzma_raw_buffer_decode(
-		const lzma_filter *filters, lzma_allocator *allocator,
+		const lzma_filter *filters, const lzma_allocator *allocator,
 		const uint8_t *in, size_t *in_pos, size_t in_size,
 		uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow;
 
@@ -356,7 +357,7 @@ extern LZMA_API(lzma_ret) lzma_properties_encode(
  *              - LZMA_MEM_ERROR
  */
 extern LZMA_API(lzma_ret) lzma_properties_decode(
-		lzma_filter *filter, lzma_allocator *allocator,
+		lzma_filter *filter, const lzma_allocator *allocator,
 		const uint8_t *props, size_t props_size) lzma_nothrow;
 
 
@@ -419,6 +420,6 @@ extern LZMA_API(lzma_ret) lzma_filter_flags_encode(const lzma_filter *filter,
  *              - LZMA_PROG_ERROR
  */
 extern LZMA_API(lzma_ret) lzma_filter_flags_decode(
-		lzma_filter *filter, lzma_allocator *allocator,
+		lzma_filter *filter, const lzma_allocator *allocator,
 		const uint8_t *in, size_t *in_pos, size_t in_size)
 		lzma_nothrow lzma_attr_warn_unused_result;

contrib/xz/src/liblzma/api/lzma/hardware.h

@@ -48,3 +48,17 @@
  *              of RAM on the specific operating system.
  */
 extern LZMA_API(uint64_t) lzma_physmem(void) lzma_nothrow;
+
+
+/**
+ * \brief       Get the number of processor cores or threads
+ *
+ * This function may be useful when determining how many threads to use.
+ * If the hardware supports more than one thread per CPU core, the number
+ * of hardware threads is returned if that information is available.
+ *
+ * \brief       On success, the number of available CPU threads or cores is
+ *              returned. If this information isn't available or an error
+ *              occurs, zero is returned.
+ */
+extern LZMA_API(uint32_t) lzma_cputhreads(void) lzma_nothrow;

contrib/xz/src/liblzma/api/lzma/index.h

@@ -303,7 +303,7 @@ extern LZMA_API(uint64_t) lzma_index_memused(const lzma_index *i)
  * \return      On success, a pointer to an empty initialized lzma_index is
  *              returned. If allocation fails, NULL is returned.
  */
-extern LZMA_API(lzma_index *) lzma_index_init(lzma_allocator *allocator)
+extern LZMA_API(lzma_index *) lzma_index_init(const lzma_allocator *allocator)
 		lzma_nothrow;
 
 
@@ -312,8 +312,8 @@ extern LZMA_API(lzma_index *) lzma_index_init(lzma_allocator *allocator)
  *
  * If i is NULL, this does nothing.
  */
-extern LZMA_API(void) lzma_index_end(lzma_index *i, lzma_allocator *allocator)
-		lzma_nothrow;
+extern LZMA_API(void) lzma_index_end(
+		lzma_index *i, const lzma_allocator *allocator) lzma_nothrow;
 
 
 /**
@@ -341,7 +341,7 @@ extern LZMA_API(void) lzma_index_end(lzma_index *i, lzma_allocator *allocator)
  *              - LZMA_PROG_ERROR
  */
 extern LZMA_API(lzma_ret) lzma_index_append(
-		lzma_index *i, lzma_allocator *allocator,
+		lzma_index *i, const lzma_allocator *allocator,
 		lzma_vli unpadded_size, lzma_vli uncompressed_size)
 		lzma_nothrow lzma_attr_warn_unused_result;
 
@@ -564,8 +564,8 @@ extern LZMA_API(lzma_bool) lzma_index_iter_locate(
  *              - LZMA_MEM_ERROR
  *              - LZMA_PROG_ERROR
  */
-extern LZMA_API(lzma_ret) lzma_index_cat(
-		lzma_index *dest, lzma_index *src, lzma_allocator *allocator)
+extern LZMA_API(lzma_ret) lzma_index_cat(lzma_index *dest, lzma_index *src,
+		const lzma_allocator *allocator)
 		lzma_nothrow lzma_attr_warn_unused_result;
 
 
@@ -575,7 +575,7 @@ extern LZMA_API(lzma_ret) lzma_index_cat(
  * \return      A copy of the lzma_index, or NULL if memory allocation failed.
  */
 extern LZMA_API(lzma_index *) lzma_index_dup(
-		const lzma_index *i, lzma_allocator *allocator)
+		const lzma_index *i, const lzma_allocator *allocator)
 		lzma_nothrow lzma_attr_warn_unused_result;
 
 
@@ -677,6 +677,6 @@ extern LZMA_API(lzma_ret) lzma_index_buffer_encode(const lzma_index *i,
  *              - LZMA_PROG_ERROR
  */
 extern LZMA_API(lzma_ret) lzma_index_buffer_decode(lzma_index **i,
-		uint64_t *memlimit, lzma_allocator *allocator,
+		uint64_t *memlimit, const lzma_allocator *allocator,
 		const uint8_t *in, size_t *in_pos, size_t in_size)
 		lzma_nothrow;

contrib/xz/src/liblzma/api/lzma/index_hash.h

@@ -37,7 +37,7 @@ typedef struct lzma_index_hash_s lzma_index_hash;
  * pointer than the index_hash that was given as an argument.
  */
 extern LZMA_API(lzma_index_hash *) lzma_index_hash_init(
-		lzma_index_hash *index_hash, lzma_allocator *allocator)
+		lzma_index_hash *index_hash, const lzma_allocator *allocator)
 		lzma_nothrow lzma_attr_warn_unused_result;
 
 
@@ -45,7 +45,7 @@ extern LZMA_API(lzma_index_hash *) lzma_index_hash_init(
  * \brief       Deallocate lzma_index_hash structure
  */
 extern LZMA_API(void) lzma_index_hash_end(
-		lzma_index_hash *index_hash, lzma_allocator *allocator)
+		lzma_index_hash *index_hash, const lzma_allocator *allocator)
 		lzma_nothrow;
 
 

contrib/xz/src/liblzma/api/lzma/lzma12.h

@@ -1,5 +1,5 @@
 /**
- * \file        lzma/lzma.h
+ * \file        lzma/lzma12.h
  * \brief       LZMA1 and LZMA2 filters
  */
 

contrib/xz/src/liblzma/api/lzma/version.h

@@ -21,8 +21,8 @@
  * Version number split into components
  */
 #define LZMA_VERSION_MAJOR 5
-#define LZMA_VERSION_MINOR 0
-#define LZMA_VERSION_PATCH 7
+#define LZMA_VERSION_MINOR 2
+#define LZMA_VERSION_PATCH 0
 #define LZMA_VERSION_STABILITY LZMA_VERSION_STABILITY_STABLE
 
 #ifndef LZMA_VERSION_COMMIT

contrib/xz/src/liblzma/check/check.h

@@ -15,6 +15,43 @@
 
 #include "common.h"
 
+#if defined(HAVE_COMMONCRYPTO_COMMONDIGEST_H)
+#	include <CommonCrypto/CommonDigest.h>
+#elif defined(HAVE_SHA256_H)
+#	include <sys/types.h>
+#	include <sha256.h>
+#elif defined(HAVE_SHA2_H)
+#	include <sys/types.h>
+#	include <sha2.h>
+#elif defined(HAVE_MINIX_SHA2_H)
+#	include <sys/types.h>
+#	include <minix/sha2.h>
+#endif
+
+#if defined(HAVE_CC_SHA256_CTX)
+typedef CC_SHA256_CTX lzma_sha256_state;
+#elif defined(HAVE_SHA256_CTX)
+typedef SHA256_CTX lzma_sha256_state;
+#elif defined(HAVE_SHA2_CTX)
+typedef SHA2_CTX lzma_sha256_state;
+#else
+/// State for the internal SHA-256 implementation
+typedef struct {
+	/// Internal state
+	uint32_t state[8];
+
+	/// Size of the message excluding padding
+	uint64_t size;
+} lzma_sha256_state;
+#endif
+
+#if defined(HAVE_CC_SHA256_INIT)
+#	define LZMA_SHA256FUNC(x) CC_SHA256_ ## x
+#elif defined(HAVE_SHA256_INIT)
+#	define LZMA_SHA256FUNC(x) SHA256_ ## x
+#elif defined(HAVE_SHA256INIT)
+#	define LZMA_SHA256FUNC(x) SHA256 ## x
+#endif
 
 // Index hashing needs the best possible hash function (preferably
 // a cryptographic hash) for maximum reliability.
@@ -43,14 +80,7 @@ typedef struct {
 	union {
 		uint32_t crc32;
 		uint64_t crc64;
-
-		struct {
-			/// Internal state
-			uint32_t state[8];
-
-			/// Size of the message excluding padding
-			uint64_t size;
-		} sha256;
+		lzma_sha256_state sha256;
 	} state;
 
 } lzma_check_state;
@@ -82,6 +112,8 @@ extern void lzma_check_update(lzma_check_state *check, lzma_check type,
 extern void lzma_check_finish(lzma_check_state *check, lzma_check type);
 
 
+#ifndef LZMA_SHA256FUNC
+
 /// Prepare SHA-256 state for new input.
 extern void lzma_sha256_init(lzma_check_state *check);
 
@@ -92,4 +124,39 @@ extern void lzma_sha256_update(
 /// Finish the SHA-256 calculation and store the result to check->buffer.u8.
 extern void lzma_sha256_finish(lzma_check_state *check);
 
+
+#else
+
+static inline void
+lzma_sha256_init(lzma_check_state *check)
+{
+	LZMA_SHA256FUNC(Init)(&check->state.sha256);
+}
+
+
+static inline void
+lzma_sha256_update(const uint8_t *buf, size_t size, lzma_check_state *check)
+{
+#if defined(HAVE_CC_SHA256_INIT) && SIZE_MAX > UINT32_MAX
+	// Darwin's CC_SHA256_Update takes uint32_t as the buffer size,
+	// so use a loop to support size_t.
+	while (size > UINT32_MAX) {
+		LZMA_SHA256FUNC(Update)(&check->state.sha256, buf, UINT32_MAX);
+		buf += UINT32_MAX;
+		size -= UINT32_MAX;
+	}
+#endif
+
+	LZMA_SHA256FUNC(Update)(&check->state.sha256, buf, size);
+}
+
+
+static inline void
+lzma_sha256_finish(lzma_check_state *check)
+{
+	LZMA_SHA256FUNC(Final)(check->buffer.u8, &check->state.sha256);
+}
+
+#endif
+
 #endif

contrib/xz/src/liblzma/check/sha256.c

@@ -21,22 +21,22 @@
 //
 ///////////////////////////////////////////////////////////////////////////////
 
-// Avoid bogus warnings in transform().
-#if (__GNUC__ == 4 && __GNUC_MINOR__ >= 2) || __GNUC__ > 4
-#	pragma GCC diagnostic ignored "-Wuninitialized"
-#endif
-
 #include "check.h"
 
-// At least on x86, GCC is able to optimize this to a rotate instruction.
-#define rotr_32(num, amount) ((num) >> (amount) | (num) << (32 - (amount)))
+// Rotate a uint32_t. GCC can optimize this to a rotate instruction
+// at least on x86.
+static inline uint32_t
+rotr_32(uint32_t num, unsigned amount)
+{
+        return (num >> amount) | (num << (32 - amount));
+}
 
-#define blk0(i) (W[i] = data[i])
+#define blk0(i) (W[i] = conv32be(data[i]))
 #define blk2(i) (W[i & 15] += s1(W[(i - 2) & 15]) + W[(i - 7) & 15] \
 		+ s0(W[(i - 15) & 15]))
 
 #define Ch(x, y, z) (z ^ (x & (y ^ z)))
-#define Maj(x, y, z) ((x & y) | (z & (x | y)))
+#define Maj(x, y, z) ((x & (y ^ z)) + (y & z))
 
 #define a(i) T[(0 - i) & 7]
 #define b(i) T[(1 - i) & 7]
@@ -47,16 +47,17 @@
 #define g(i) T[(6 - i) & 7]
 #define h(i) T[(7 - i) & 7]
 
-#define R(i) \
-	h(i) += S1(e(i)) + Ch(e(i), f(i), g(i)) + SHA256_K[i + j] \
-		+ (j ? blk2(i) : blk0(i)); \
+#define R(i, j, blk) \
+	h(i) += S1(e(i)) + Ch(e(i), f(i), g(i)) + SHA256_K[i + j] + blk; \
 	d(i) += h(i); \
 	h(i) += S0(a(i)) + Maj(a(i), b(i), c(i))
+#define R0(i) R(i, 0, blk0(i))
+#define R2(i) R(i, j, blk2(i))
 
-#define S0(x) (rotr_32(x, 2) ^ rotr_32(x, 13) ^ rotr_32(x, 22))
-#define S1(x) (rotr_32(x, 6) ^ rotr_32(x, 11) ^ rotr_32(x, 25))
-#define s0(x) (rotr_32(x, 7) ^ rotr_32(x, 18) ^ (x >> 3))
-#define s1(x) (rotr_32(x, 17) ^ rotr_32(x, 19) ^ (x >> 10))
+#define S0(x) rotr_32(x ^ rotr_32(x ^ rotr_32(x, 9), 11), 2)
+#define S1(x) rotr_32(x ^ rotr_32(x ^ rotr_32(x, 14), 5), 6)
+#define s0(x) (rotr_32(x ^ rotr_32(x, 11), 7) ^ (x >> 3))
+#define s1(x) (rotr_32(x ^ rotr_32(x, 2), 17) ^ (x >> 10))
 
 
 static const uint32_t SHA256_K[64] = {
@@ -88,12 +89,18 @@ transform(uint32_t state[8], const uint32_t data[16])
 	// Copy state[] to working vars.
 	memcpy(T, state, sizeof(T));
 
-	// 64 operations, partially loop unrolled
-	for (unsigned int j = 0; j < 64; j += 16) {
-		R( 0); R( 1); R( 2); R( 3);
-		R( 4); R( 5); R( 6); R( 7);
-		R( 8); R( 9); R(10); R(11);
-		R(12); R(13); R(14); R(15);
+	// The first 16 operations unrolled
+	R0( 0); R0( 1); R0( 2); R0( 3);
+	R0( 4); R0( 5); R0( 6); R0( 7);
+	R0( 8); R0( 9); R0(10); R0(11);
+	R0(12); R0(13); R0(14); R0(15);
+
+	// The remaining 48 operations partially unrolled
+	for (unsigned int j = 16; j < 64; j += 16) {
+		R2( 0); R2( 1); R2( 2); R2( 3);
+		R2( 4); R2( 5); R2( 6); R2( 7);
+		R2( 8); R2( 9); R2(10); R2(11);
+		R2(12); R2(13); R2(14); R2(15);
 	}
 
 	// Add the working vars back into state[].
@@ -111,18 +118,7 @@ transform(uint32_t state[8], const uint32_t data[16])
 static void
 process(lzma_check_state *check)
 {
-#ifdef WORDS_BIGENDIAN
 	transform(check->state.sha256.state, check->buffer.u32);
-
-#else
-	uint32_t data[16];
-
-	for (size_t i = 0; i < 16; ++i)
-		data[i] = bswap32(check->buffer.u32[i]);
-
-	transform(check->state.sha256.state, data);
-#endif
-
 	return;
 }
 

contrib/xz/src/liblzma/common/alone_decoder.c

@@ -51,7 +51,7 @@ struct lzma_coder_s {
 
 static lzma_ret
 alone_decode(lzma_coder *coder,
-		lzma_allocator *allocator lzma_attribute((__unused__)),
+		const lzma_allocator *allocator lzma_attribute((__unused__)),
 		const uint8_t *restrict in, size_t *restrict in_pos,
 		size_t in_size, uint8_t *restrict out,
 		size_t *restrict out_pos, size_t out_size,
@@ -166,7 +166,7 @@ alone_decode(lzma_coder *coder,
 
 
 static void
-alone_decoder_end(lzma_coder *coder, lzma_allocator *allocator)
+alone_decoder_end(lzma_coder *coder, const lzma_allocator *allocator)
 {
 	lzma_next_end(&coder->next, allocator);
 	lzma_free(coder, allocator);
@@ -193,7 +193,7 @@ alone_decoder_memconfig(lzma_coder *coder, uint64_t *memusage,
 
 
 extern lzma_ret
-lzma_alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_alone_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		uint64_t memlimit, bool picky)
 {
 	lzma_next_coder_init(&lzma_alone_decoder_init, next, allocator);

contrib/xz/src/liblzma/common/alone_decoder.h

@@ -17,7 +17,7 @@
 
 
 extern lzma_ret lzma_alone_decoder_init(
-		lzma_next_coder *next, lzma_allocator *allocator,
+		lzma_next_coder *next, const lzma_allocator *allocator,
 		uint64_t memlimit, bool picky);
 
 #endif

contrib/xz/src/liblzma/common/alone_encoder.c

@@ -32,7 +32,7 @@ struct lzma_coder_s {
 
 static lzma_ret
 alone_encode(lzma_coder *coder,
-		lzma_allocator *allocator lzma_attribute((__unused__)),
+		const lzma_allocator *allocator lzma_attribute((__unused__)),
 		const uint8_t *restrict in, size_t *restrict in_pos,
 		size_t in_size, uint8_t *restrict out,
 		size_t *restrict out_pos, size_t out_size,
@@ -65,7 +65,7 @@ alone_encode(lzma_coder *coder,
 
 
 static void
-alone_encoder_end(lzma_coder *coder, lzma_allocator *allocator)
+alone_encoder_end(lzma_coder *coder, const lzma_allocator *allocator)
 {
 	lzma_next_end(&coder->next, allocator);
 	lzma_free(coder, allocator);
@@ -75,7 +75,7 @@ alone_encoder_end(lzma_coder *coder, lzma_allocator *allocator)
 
 // At least for now, this is not used by any internal function.
 static lzma_ret
-alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+alone_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_options_lzma *options)
 {
 	lzma_next_coder_init(&alone_encoder_init, next, allocator);
@@ -137,7 +137,7 @@ alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
 
 /*
 extern lzma_ret
-lzma_alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_alone_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_options_alone *options)
 {
 	lzma_next_coder_init(&alone_encoder_init, next, allocator, options);

contrib/xz/src/liblzma/common/auto_decoder.c

@@ -30,7 +30,7 @@ struct lzma_coder_s {
 
 
 static lzma_ret
-auto_decode(lzma_coder *coder, lzma_allocator *allocator,
+auto_decode(lzma_coder *coder, const lzma_allocator *allocator,
 		const uint8_t *restrict in, size_t *restrict in_pos,
 		size_t in_size, uint8_t *restrict out,
 		size_t *restrict out_pos, size_t out_size, lzma_action action)
@@ -100,7 +100,7 @@ auto_decode(lzma_coder *coder, lzma_allocator *allocator,
 
 
 static void
-auto_decoder_end(lzma_coder *coder, lzma_allocator *allocator)
+auto_decoder_end(lzma_coder *coder, const lzma_allocator *allocator)
 {
 	lzma_next_end(&coder->next, allocator);
 	lzma_free(coder, allocator);
@@ -143,7 +143,7 @@ auto_decoder_memconfig(lzma_coder *coder, uint64_t *memusage,
 
 
 static lzma_ret
-auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+auto_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		uint64_t memlimit, uint32_t flags)
 {
 	lzma_next_coder_init(&auto_decoder_init, next, allocator);

contrib/xz/src/liblzma/common/block_buffer_decoder.c

@@ -14,7 +14,7 @@
 
 
 extern LZMA_API(lzma_ret)
-lzma_block_buffer_decode(lzma_block *block, lzma_allocator *allocator,
+lzma_block_buffer_decode(lzma_block *block, const lzma_allocator *allocator,
 		const uint8_t *in, size_t *in_pos, size_t in_size,
 		uint8_t *out, size_t *out_pos, size_t out_size)
 {

contrib/xz/src/liblzma/common/block_buffer_encoder.c

@@ -10,6 +10,7 @@
 //
 ///////////////////////////////////////////////////////////////////////////////
 
+#include "block_buffer_encoder.h"
 #include "block_encoder.h"
 #include "filter_encoder.h"
 #include "lzma2_encoder.h"
@@ -28,8 +29,8 @@
 		+ LZMA_CHECK_SIZE_MAX + 3) & ~3)
 
 
-static lzma_vli
-lzma2_bound(lzma_vli uncompressed_size)
+static uint64_t
+lzma2_bound(uint64_t uncompressed_size)
 {
 	// Prevent integer overflow in overhead calculation.
 	if (uncompressed_size > COMPRESSED_SIZE_MAX)
@@ -39,7 +40,7 @@ lzma2_bound(lzma_vli uncompressed_size)
 	// uncompressed_size up to the next multiple of LZMA2_CHUNK_MAX,
 	// multiply by the size of per-chunk header, and add one byte for
 	// the end marker.
-	const lzma_vli overhead = ((uncompressed_size + LZMA2_CHUNK_MAX - 1)
+	const uint64_t overhead = ((uncompressed_size + LZMA2_CHUNK_MAX - 1)
 				/ LZMA2_CHUNK_MAX)
 			* LZMA2_HEADER_UNCOMPRESSED + 1;
 
@@ -51,30 +52,36 @@ lzma2_bound(lzma_vli uncompressed_size)
 }
 
 
-extern LZMA_API(size_t)
-lzma_block_buffer_bound(size_t uncompressed_size)
+extern uint64_t
+lzma_block_buffer_bound64(uint64_t uncompressed_size)
 {
-	// For now, if the data doesn't compress, we always use uncompressed
-	// chunks of LZMA2. In future we may use Subblock filter too, but
-	// but for simplicity we probably will still use the same bound
-	// calculation even though Subblock filter would have slightly less
-	// overhead.
-	lzma_vli lzma2_size = lzma2_bound(uncompressed_size);
+	// If the data doesn't compress, we always use uncompressed
+	// LZMA2 chunks.
+	uint64_t lzma2_size = lzma2_bound(uncompressed_size);
 	if (lzma2_size == 0)
 		return 0;
 
 	// Take Block Padding into account.
-	lzma2_size = (lzma2_size + 3) & ~LZMA_VLI_C(3);
+	lzma2_size = (lzma2_size + 3) & ~UINT64_C(3);
 
-#if SIZE_MAX < LZMA_VLI_MAX
-	// Catch the possible integer overflow on 32-bit systems. There's no
-	// overflow on 64-bit systems, because lzma2_bound() already takes
+	// No risk of integer overflow because lzma2_bound() already takes
 	// into account the size of the headers in the Block.
-	if (SIZE_MAX - HEADERS_BOUND < lzma2_size)
+	return HEADERS_BOUND + lzma2_size;
+}
+
+
+extern LZMA_API(size_t)
+lzma_block_buffer_bound(size_t uncompressed_size)
+{
+	uint64_t ret = lzma_block_buffer_bound64(uncompressed_size);
+
+#if SIZE_MAX < UINT64_MAX
+	// Catch the possible integer overflow on 32-bit systems.
+	if (ret > SIZE_MAX)
 		return 0;
 #endif
 
-	return HEADERS_BOUND + lzma2_size;
+	return ret;
 }
 
 
@@ -82,9 +89,6 @@ static lzma_ret
 block_encode_uncompressed(lzma_block *block, const uint8_t *in, size_t in_size,
 		uint8_t *out, size_t *out_pos, size_t out_size)
 {
-	// TODO: Figure out if the last filter is LZMA2 or Subblock and use
-	// that filter to encode the uncompressed chunks.
-
 	// Use LZMA2 uncompressed chunks. We wouldn't need a dictionary at
 	// all, but LZMA2 always requires a dictionary, so use the minimum
 	// value to minimize memory usage of the decoder.
@@ -160,16 +164,11 @@ block_encode_uncompressed(lzma_block *block, const uint8_t *in, size_t in_size,
 
 
 static lzma_ret
-block_encode_normal(lzma_block *block, lzma_allocator *allocator,
+block_encode_normal(lzma_block *block, const lzma_allocator *allocator,
 		const uint8_t *in, size_t in_size,
 		uint8_t *out, size_t *out_pos, size_t out_size)
 {
 	// Find out the size of the Block Header.
-	block->compressed_size = lzma2_bound(in_size);
-	if (block->compressed_size == 0)
-		return LZMA_DATA_ERROR;
-
-	block->uncompressed_size = in_size;
 	return_if_error(lzma_block_header_size(block));
 
 	// Reserve space for the Block Header and skip it for now.
@@ -221,10 +220,11 @@ block_encode_normal(lzma_block *block, lzma_allocator *allocator,
 }
 
 
-extern LZMA_API(lzma_ret)
-lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator,
+static lzma_ret
+block_buffer_encode(lzma_block *block, const lzma_allocator *allocator,
 		const uint8_t *in, size_t in_size,
-		uint8_t *out, size_t *out_pos, size_t out_size)
+		uint8_t *out, size_t *out_pos, size_t out_size,
+		bool try_to_compress)
 {
 	// Validate the arguments.
 	if (block == NULL || (in == NULL && in_size != 0) || out == NULL
@@ -233,11 +233,11 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator,
 
 	// The contents of the structure may depend on the version so
 	// check the version before validating the contents of *block.
-	if (block->version != 0)
+	if (block->version > 1)
 		return LZMA_OPTIONS_ERROR;
 
 	if ((unsigned int)(block->check) > LZMA_CHECK_ID_MAX
-			|| block->filters == NULL)
+			|| (try_to_compress && block->filters == NULL))
 		return LZMA_PROG_ERROR;
 
 	if (!lzma_check_is_supported(block->check))
@@ -258,9 +258,19 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator,
 
 	out_size -= check_size;
 
+	// Initialize block->uncompressed_size and calculate the worst-case
+	// value for block->compressed_size.
+	block->uncompressed_size = in_size;
+	block->compressed_size = lzma2_bound(in_size);
+	if (block->compressed_size == 0)
+		return LZMA_DATA_ERROR;
+
 	// Do the actual compression.
-	const lzma_ret ret = block_encode_normal(block, allocator,
-			in, in_size, out, out_pos, out_size);
+	lzma_ret ret = LZMA_BUF_ERROR;
+	if (try_to_compress)
+		ret = block_encode_normal(block, allocator,
+				in, in_size, out, out_pos, out_size);
+
 	if (ret != LZMA_OK) {
 		// If the error was something else than output buffer
 		// becoming full, return the error now.
@@ -303,3 +313,25 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator,
 
 	return LZMA_OK;
 }
+
+
+extern LZMA_API(lzma_ret)
+lzma_block_buffer_encode(lzma_block *block, const lzma_allocator *allocator,
+		const uint8_t *in, size_t in_size,
+		uint8_t *out, size_t *out_pos, size_t out_size)
+{
+	return block_buffer_encode(block, allocator,
+			in, in_size, out, out_pos, out_size, true);
+}
+
+
+extern LZMA_API(lzma_ret)
+lzma_block_uncomp_encode(lzma_block *block,
+		const uint8_t *in, size_t in_size,
+		uint8_t *out, size_t *out_pos, size_t out_size)
+{
+	// It won't allocate any memory from heap so no need
+	// for lzma_allocator.
+	return block_buffer_encode(block, NULL,
+			in, in_size, out, out_pos, out_size, false);
+}

contrib/xz/src/liblzma/common/block_buffer_encoder.h

@@ -0,0 +1,24 @@
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file       block_buffer_encoder.h
+/// \brief      Single-call .xz Block encoder
+//
+//  Author:     Lasse Collin
+//
+//  This file has been put into the public domain.
+//  You can do whatever you want with this file.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef LZMA_BLOCK_BUFFER_ENCODER_H
+#define LZMA_BLOCK_BUFFER_ENCODER_H
+
+#include "common.h"
+
+
+/// uint64_t version of lzma_block_buffer_bound(). It is used by
+/// stream_encoder_mt.c. Probably the original lzma_block_buffer_bound()
+/// should have been 64-bit, but fixing it would break the ABI.
+extern uint64_t lzma_block_buffer_bound64(uint64_t uncompressed_size);
+
+#endif

contrib/xz/src/liblzma/common/block_decoder.c

@@ -45,6 +45,9 @@ struct lzma_coder_s {
 
 	/// Check of the uncompressed data
 	lzma_check_state check;
+
+	/// True if the integrity check won't be calculated and verified.
+	bool ignore_check;
 };
 
 
@@ -71,7 +74,7 @@ is_size_valid(lzma_vli size, lzma_vli reference)
 
 
 static lzma_ret
-block_decode(lzma_coder *coder, lzma_allocator *allocator,
+block_decode(lzma_coder *coder, const lzma_allocator *allocator,
 		const uint8_t *restrict in, size_t *restrict in_pos,
 		size_t in_size, uint8_t *restrict out,
 		size_t *restrict out_pos, size_t out_size, lzma_action action)
@@ -97,8 +100,9 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator,
 					coder->block->uncompressed_size))
 			return LZMA_DATA_ERROR;
 
-		lzma_check_update(&coder->check, coder->block->check,
-				out + out_start, out_used);
+		if (!coder->ignore_check)
+			lzma_check_update(&coder->check, coder->block->check,
+					out + out_start, out_used);
 
 		if (ret != LZMA_STREAM_END)
 			return ret;
@@ -140,7 +144,9 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator,
 		if (coder->block->check == LZMA_CHECK_NONE)
 			return LZMA_STREAM_END;
 
-		lzma_check_finish(&coder->check, coder->block->check);
+		if (!coder->ignore_check)
+			lzma_check_finish(&coder->check, coder->block->check);
+
 		coder->sequence = SEQ_CHECK;
 
 	// Fall through
@@ -155,7 +161,8 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator,
 		// Validate the Check only if we support it.
 		// coder->check.buffer may be uninitialized
 		// when the Check ID is not supported.
-		if (lzma_check_is_supported(coder->block->check)
+		if (!coder->ignore_check
+				&& lzma_check_is_supported(coder->block->check)
 				&& memcmp(coder->block->raw_check,
 					coder->check.buffer.u8,
 					check_size) != 0)
@@ -170,7 +177,7 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator,
 
 
 static void
-block_decoder_end(lzma_coder *coder, lzma_allocator *allocator)
+block_decoder_end(lzma_coder *coder, const lzma_allocator *allocator)
 {
 	lzma_next_end(&coder->next, allocator);
 	lzma_free(coder, allocator);
@@ -179,7 +186,7 @@ block_decoder_end(lzma_coder *coder, lzma_allocator *allocator)
 
 
 extern lzma_ret
-lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_block_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		lzma_block *block)
 {
 	lzma_next_coder_init(&lzma_block_decoder_init, next, allocator);
@@ -224,6 +231,9 @@ lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
 	next->coder->check_pos = 0;
 	lzma_check_init(&next->coder->check, block->check);
 
+	next->coder->ignore_check = block->version >= 1
+			? block->ignore_check : false;
+
 	// Initialize the filter chain.
 	return lzma_raw_decoder_init(&next->coder->next, allocator,
 			block->filters);

contrib/xz/src/liblzma/common/block_decoder.h

@@ -17,6 +17,6 @@
 
 
 extern lzma_ret lzma_block_decoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, lzma_block *block);
+		const lzma_allocator *allocator, lzma_block *block);
 
 #endif

contrib/xz/src/liblzma/common/block_encoder.c

@@ -45,7 +45,7 @@ struct lzma_coder_s {
 
 
 static lzma_ret
-block_encode(lzma_coder *coder, lzma_allocator *allocator,
+block_encode(lzma_coder *coder, const lzma_allocator *allocator,
 		const uint8_t *restrict in, size_t *restrict in_pos,
 		size_t in_size, uint8_t *restrict out,
 		size_t *restrict out_pos, size_t out_size, lzma_action action)
@@ -134,7 +134,7 @@ block_encode(lzma_coder *coder, lzma_allocator *allocator,
 
 
 static void
-block_encoder_end(lzma_coder *coder, lzma_allocator *allocator)
+block_encoder_end(lzma_coder *coder, const lzma_allocator *allocator)
 {
 	lzma_next_end(&coder->next, allocator);
 	lzma_free(coder, allocator);
@@ -143,7 +143,7 @@ block_encoder_end(lzma_coder *coder, lzma_allocator *allocator)
 
 
 static lzma_ret
-block_encoder_update(lzma_coder *coder, lzma_allocator *allocator,
+block_encoder_update(lzma_coder *coder, const lzma_allocator *allocator,
 		const lzma_filter *filters lzma_attribute((__unused__)),
 		const lzma_filter *reversed_filters)
 {
@@ -156,7 +156,7 @@ block_encoder_update(lzma_coder *coder, lzma_allocator *allocator,
 
 
 extern lzma_ret
-lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_block_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		lzma_block *block)
 {
 	lzma_next_coder_init(&lzma_block_encoder_init, next, allocator);
@@ -166,7 +166,7 @@ lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
 
 	// The contents of the structure may depend on the version so
 	// check the version first.
-	if (block->version != 0)
+	if (block->version > 1)
 		return LZMA_OPTIONS_ERROR;
 
 	// If the Check ID is not supported, we cannot calculate the check and

contrib/xz/src/liblzma/common/block_encoder.h

@@ -42,6 +42,6 @@
 
 
 extern lzma_ret lzma_block_encoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, lzma_block *block);
+		const lzma_allocator *allocator, lzma_block *block);
 
 #endif

contrib/xz/src/liblzma/common/block_header_decoder.c

@@ -15,7 +15,7 @@
 
 
 static void
-free_properties(lzma_block *block, lzma_allocator *allocator)
+free_properties(lzma_block *block, const lzma_allocator *allocator)
 {
 	// Free allocated filter options. The last array member is not
 	// touched after the initialization in the beginning of
@@ -32,7 +32,7 @@ free_properties(lzma_block *block, lzma_allocator *allocator)
 
 extern LZMA_API(lzma_ret)
 lzma_block_header_decode(lzma_block *block,
-		lzma_allocator *allocator, const uint8_t *in)
+		const lzma_allocator *allocator, const uint8_t *in)
 {
 	// NOTE: We consider the header to be corrupt not only when the
 	// CRC32 doesn't match, but also when variable-length integers
@@ -46,8 +46,16 @@ lzma_block_header_decode(lzma_block *block,
 		block->filters[i].options = NULL;
 	}
 
-	// Always zero for now.
-	block->version = 0;
+	// Versions 0 and 1 are supported. If a newer version was specified,
+	// we need to downgrade it.
+	if (block->version > 1)
+		block->version = 1;
+
+	// This isn't a Block Header option, but since the decompressor will
+	// read it if version >= 1, it's better to initialize it here than
+	// to expect the caller to do it since in almost all cases this
+	// should be false.
+	block->ignore_check = false;
 
 	// Validate Block Header Size and Check type. The caller must have
 	// already set these, so it is a programming error if this test fails.

contrib/xz/src/liblzma/common/block_header_encoder.c

@@ -17,7 +17,7 @@
 extern LZMA_API(lzma_ret)
 lzma_block_header_size(lzma_block *block)
 {
-	if (block->version != 0)
+	if (block->version > 1)
 		return LZMA_OPTIONS_ERROR;
 
 	// Block Header Size + Block Flags + CRC32.

contrib/xz/src/liblzma/common/block_util.c

@@ -51,7 +51,7 @@ lzma_block_unpadded_size(const lzma_block *block)
 	// NOTE: This function is used for validation too, so it is
 	// essential that these checks are always done even if
 	// Compressed Size is unknown.
-	if (block == NULL || block->version != 0
+	if (block == NULL || block->version > 1
 			|| block->header_size < LZMA_BLOCK_HEADER_SIZE_MIN
 			|| block->header_size > LZMA_BLOCK_HEADER_SIZE_MAX
 			|| (block->header_size & 3)

contrib/xz/src/liblzma/common/common.c

@@ -36,7 +36,7 @@ lzma_version_string(void)
 ///////////////////////
 
 extern void * lzma_attribute((__malloc__)) lzma_attr_alloc_size(1)
-lzma_alloc(size_t size, lzma_allocator *allocator)
+lzma_alloc(size_t size, const lzma_allocator *allocator)
 {
 	// Some malloc() variants return NULL if called with size == 0.
 	if (size == 0)
@@ -53,8 +53,29 @@ lzma_alloc(size_t size, lzma_allocator *allocator)
 }
 
 
+extern void * lzma_attribute((__malloc__)) lzma_attr_alloc_size(1)
+lzma_alloc_zero(size_t size, const lzma_allocator *allocator)
+{
+	// Some calloc() variants return NULL if called with size == 0.
+	if (size == 0)
+		size = 1;
+
+	void *ptr;
+
+	if (allocator != NULL && allocator->alloc != NULL) {
+		ptr = allocator->alloc(allocator->opaque, 1, size);
+		if (ptr != NULL)
+			memzero(ptr, size);
+	} else {
+		ptr = calloc(1, size);
+	}
+
+	return ptr;
+}
+
+
 extern void
-lzma_free(void *ptr, lzma_allocator *allocator)
+lzma_free(void *ptr, const lzma_allocator *allocator)
 {
 	if (allocator != NULL && allocator->free != NULL)
 		allocator->free(allocator->opaque, ptr);
@@ -88,7 +109,7 @@ lzma_bufcpy(const uint8_t *restrict in, size_t *restrict in_pos,
 
 
 extern lzma_ret
-lzma_next_filter_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_next_filter_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter_info *filters)
 {
 	lzma_next_coder_init(filters[0].init, next, allocator);
@@ -99,7 +120,7 @@ lzma_next_filter_init(lzma_next_coder *next, lzma_allocator *allocator,
 
 
 extern lzma_ret
-lzma_next_filter_update(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_next_filter_update(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter *reversed_filters)
 {
 	// Check that the application isn't trying to change the Filter ID.
@@ -117,7 +138,7 @@ lzma_next_filter_update(lzma_next_coder *next, lzma_allocator *allocator,
 
 
 extern void
-lzma_next_end(lzma_next_coder *next, lzma_allocator *allocator)
+lzma_next_end(lzma_next_coder *next, const lzma_allocator *allocator)
 {
 	if (next->init != (uintptr_t)(NULL)) {
 		// To avoid tiny end functions that simply call
@@ -156,10 +177,8 @@ lzma_strm_init(lzma_stream *strm)
 		strm->internal->next = LZMA_NEXT_CODER_INIT;
 	}
 
-	strm->internal->supported_actions[LZMA_RUN] = false;
-	strm->internal->supported_actions[LZMA_SYNC_FLUSH] = false;
-	strm->internal->supported_actions[LZMA_FULL_FLUSH] = false;
-	strm->internal->supported_actions[LZMA_FINISH] = false;
+	memzero(strm->internal->supported_actions,
+			sizeof(strm->internal->supported_actions));
 	strm->internal->sequence = ISEQ_RUN;
 	strm->internal->allow_buf_error = false;
 
@@ -178,7 +197,7 @@ lzma_code(lzma_stream *strm, lzma_action action)
 			|| (strm->next_out == NULL && strm->avail_out != 0)
 			|| strm->internal == NULL
 			|| strm->internal->next.code == NULL
-			|| (unsigned int)(action) > LZMA_FINISH
+			|| (unsigned int)(action) > LZMA_ACTION_MAX
 			|| !strm->internal->supported_actions[action])
 		return LZMA_PROG_ERROR;
 
@@ -213,6 +232,10 @@ lzma_code(lzma_stream *strm, lzma_action action)
 		case LZMA_FINISH:
 			strm->internal->sequence = ISEQ_FINISH;
 			break;
+
+		case LZMA_FULL_BARRIER:
+			strm->internal->sequence = ISEQ_FULL_BARRIER;
+			break;
 		}
 
 		break;
@@ -240,6 +263,13 @@ lzma_code(lzma_stream *strm, lzma_action action)
 
 		break;
 
+	case ISEQ_FULL_BARRIER:
+		if (action != LZMA_FULL_BARRIER
+				|| strm->internal->avail_in != strm->avail_in)
+			return LZMA_PROG_ERROR;
+
+		break;
+
 	case ISEQ_END:
 		return LZMA_STREAM_END;
 
@@ -265,7 +295,9 @@ lzma_code(lzma_stream *strm, lzma_action action)
 
 	strm->internal->avail_in = strm->avail_in;
 
-	switch (ret) {
+	// Cast is needed to silence a warning about LZMA_TIMED_OUT, which
+	// isn't part of lzma_ret enumeration.
+	switch ((unsigned int)(ret)) {
 	case LZMA_OK:
 		// Don't return LZMA_BUF_ERROR when it happens the first time.
 		// This is to avoid returning LZMA_BUF_ERROR when avail_out
@@ -281,9 +313,16 @@ lzma_code(lzma_stream *strm, lzma_action action)
 		}
 		break;
 
+	case LZMA_TIMED_OUT:
+		strm->internal->allow_buf_error = false;
+		ret = LZMA_OK;
+		break;
+
 	case LZMA_STREAM_END:
 		if (strm->internal->sequence == ISEQ_SYNC_FLUSH
-				|| strm->internal->sequence == ISEQ_FULL_FLUSH)
+				|| strm->internal->sequence == ISEQ_FULL_FLUSH
+				|| strm->internal->sequence
+					== ISEQ_FULL_BARRIER)
 			strm->internal->sequence = ISEQ_RUN;
 		else
 			strm->internal->sequence = ISEQ_END;
@@ -323,6 +362,22 @@ lzma_end(lzma_stream *strm)
 }
 
 
+extern LZMA_API(void)
+lzma_get_progress(lzma_stream *strm,
+		uint64_t *progress_in, uint64_t *progress_out)
+{
+	if (strm->internal->next.get_progress != NULL) {
+		strm->internal->next.get_progress(strm->internal->next.coder,
+				progress_in, progress_out);
+	} else {
+		*progress_in = strm->total_in;
+		*progress_out = strm->total_out;
+	}
+
+	return;
+}
+
+
 extern LZMA_API(lzma_check)
 lzma_get_check(const lzma_stream *strm)
 {

contrib/xz/src/liblzma/common/common.h

@@ -49,6 +49,13 @@
 #define LZMA_BUFFER_SIZE 4096
 
 
+/// Maximum number of worker threads within one multithreaded component.
+/// The limit exists solely to make it simpler to prevent integer overflows
+/// when allocating structures etc. This should be big enough for now...
+/// the code won't scale anywhere close to this number anyway.
+#define LZMA_THREADS_MAX 16384
+
+
 /// Starting value for memory usage estimates. Instead of calculating size
 /// of _every_ structure and taking into account malloc() overhead etc., we
 /// add a base size to all memory usage estimates. It's not very accurate
@@ -66,9 +73,21 @@
 	( LZMA_TELL_NO_CHECK \
 	| LZMA_TELL_UNSUPPORTED_CHECK \
 	| LZMA_TELL_ANY_CHECK \
+	| LZMA_IGNORE_CHECK \
 	| LZMA_CONCATENATED )
 
 
+/// Largest valid lzma_action value as unsigned integer.
+#define LZMA_ACTION_MAX ((unsigned int)(LZMA_FULL_BARRIER))
+
+
+/// Special return value (lzma_ret) to indicate that a timeout was reached
+/// and lzma_code() must not return LZMA_BUF_ERROR. This is converted to
+/// LZMA_OK in lzma_code(). This is not in the lzma_ret enumeration because
+/// there's no need to have it in the public API.
+#define LZMA_TIMED_OUT 32
+
+
 /// Type of encoder/decoder specific data; the actual structure is defined
 /// differently in different coders.
 typedef struct lzma_coder_s lzma_coder;
@@ -80,7 +99,7 @@ typedef struct lzma_filter_info_s lzma_filter_info;
 
 /// Type of a function used to initialize a filter encoder or decoder
 typedef lzma_ret (*lzma_init_function)(
-		lzma_next_coder *next, lzma_allocator *allocator,
+		lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter_info *filters);
 
 /// Type of a function to do some kind of coding work (filters, Stream,
@@ -88,7 +107,7 @@ typedef lzma_ret (*lzma_init_function)(
 /// input and output buffers, but for simplicity they still use this same
 /// function prototype.
 typedef lzma_ret (*lzma_code_function)(
-		lzma_coder *coder, lzma_allocator *allocator,
+		lzma_coder *coder, const lzma_allocator *allocator,
 		const uint8_t *restrict in, size_t *restrict in_pos,
 		size_t in_size, uint8_t *restrict out,
 		size_t *restrict out_pos, size_t out_size,
@@ -96,7 +115,7 @@ typedef lzma_ret (*lzma_code_function)(
 
 /// Type of a function to free the memory allocated for the coder
 typedef void (*lzma_end_function)(
-		lzma_coder *coder, lzma_allocator *allocator);
+		lzma_coder *coder, const lzma_allocator *allocator);
 
 
 /// Raw coder validates and converts an array of lzma_filter structures to
@@ -139,6 +158,11 @@ struct lzma_next_coder_s {
 	/// lzma_next_coder.coder.
 	lzma_end_function end;
 
+	/// Pointer to a function to get progress information. If this is NULL,
+	/// lzma_stream.total_in and .total_out are used instead.
+	void (*get_progress)(lzma_coder *coder,
+			uint64_t *progress_in, uint64_t *progress_out);
+
 	/// Pointer to function to return the type of the integrity check.
 	/// Most coders won't support this.
 	lzma_check (*get_check)(const lzma_coder *coder);
@@ -150,7 +174,7 @@ struct lzma_next_coder_s {
 
 	/// Update the filter-specific options or the whole filter chain
 	/// in the encoder.
-	lzma_ret (*update)(lzma_coder *coder, lzma_allocator *allocator,
+	lzma_ret (*update)(lzma_coder *coder, const lzma_allocator *allocator,
 			const lzma_filter *filters,
 			const lzma_filter *reversed_filters);
 };
@@ -164,6 +188,7 @@ struct lzma_next_coder_s {
 		.id = LZMA_VLI_UNKNOWN, \
 		.code = NULL, \
 		.end = NULL, \
+		.get_progress = NULL, \
 		.get_check = NULL, \
 		.memconfig = NULL, \
 		.update = NULL, \
@@ -185,6 +210,7 @@ struct lzma_internal_s {
 		ISEQ_SYNC_FLUSH,
 		ISEQ_FULL_FLUSH,
 		ISEQ_FINISH,
+		ISEQ_FULL_BARRIER,
 		ISEQ_END,
 		ISEQ_ERROR,
 	} sequence;
@@ -195,7 +221,7 @@ struct lzma_internal_s {
 	size_t avail_in;
 
 	/// Indicates which lzma_action values are allowed by next.code.
-	bool supported_actions[4];
+	bool supported_actions[LZMA_ACTION_MAX + 1];
 
 	/// If true, lzma_code will return LZMA_BUF_ERROR if no progress was
 	/// made (no input consumed and no output produced by next.code).
@@ -204,11 +230,17 @@ struct lzma_internal_s {
 
 
 /// Allocates memory
-extern void *lzma_alloc(size_t size, lzma_allocator *allocator)
+extern void *lzma_alloc(size_t size, const lzma_allocator *allocator)
 		lzma_attribute((__malloc__)) lzma_attr_alloc_size(1);
 
+/// Allocates memory and zeroes it (like calloc()). This can be faster
+/// than lzma_alloc() + memzero() while being backward compatible with
+/// custom allocators.
+extern void * lzma_attribute((__malloc__)) lzma_attr_alloc_size(1)
+		lzma_alloc_zero(size_t size, const lzma_allocator *allocator);
+
 /// Frees memory
-extern void lzma_free(void *ptr, lzma_allocator *allocator);
+extern void lzma_free(void *ptr, const lzma_allocator *allocator);
 
 
 /// Allocates strm->internal if it is NULL, and initializes *strm and
@@ -220,17 +252,19 @@ extern lzma_ret lzma_strm_init(lzma_stream *strm);
 /// than the filter being initialized now. This way the actual filter
 /// initialization functions don't need to use lzma_next_coder_init macro.
 extern lzma_ret lzma_next_filter_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters);
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters);
 
 /// Update the next filter in the chain, if any. This checks that
 /// the application is not trying to change the Filter IDs.
 extern lzma_ret lzma_next_filter_update(
-		lzma_next_coder *next, lzma_allocator *allocator,
+		lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter *reversed_filters);
 
 /// Frees the memory allocated for next->coder either using next->end or,
 /// if next->end is NULL, using lzma_free.
-extern void lzma_next_end(lzma_next_coder *next, lzma_allocator *allocator);
+extern void lzma_next_end(lzma_next_coder *next,
+		const lzma_allocator *allocator);
 
 
 /// Copy as much data as possible from in[] to out[] and update *in_pos

contrib/xz/src/liblzma/common/easy_buffer_encoder.c

@@ -15,8 +15,8 @@
 
 extern LZMA_API(lzma_ret)
 lzma_easy_buffer_encode(uint32_t preset, lzma_check check,
-		lzma_allocator *allocator, const uint8_t *in, size_t in_size,
-		uint8_t *out, size_t *out_pos, size_t out_size)
+		const lzma_allocator *allocator, const uint8_t *in,
+		size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size)
 {
 	lzma_options_easy opt_easy;
 	if (lzma_easy_preset(&opt_easy, preset))

contrib/xz/src/liblzma/common/easy_encoder.c

@@ -11,7 +11,6 @@
 ///////////////////////////////////////////////////////////////////////////////
 
 #include "easy_preset.h"
-#include "stream_encoder.h"
 
 
 extern LZMA_API(lzma_ret)

contrib/xz/src/liblzma/common/filter_buffer_decoder.c

@@ -14,7 +14,8 @@
 
 
 extern LZMA_API(lzma_ret)
-lzma_raw_buffer_decode(const lzma_filter *filters, lzma_allocator *allocator,
+lzma_raw_buffer_decode(
+		const lzma_filter *filters, const lzma_allocator *allocator,
 		const uint8_t *in, size_t *in_pos, size_t in_size,
 		uint8_t *out, size_t *out_pos, size_t out_size)
 {

contrib/xz/src/liblzma/common/filter_buffer_encoder.c

@@ -14,9 +14,10 @@
 
 
 extern LZMA_API(lzma_ret)
-lzma_raw_buffer_encode(const lzma_filter *filters, lzma_allocator *allocator,
-		const uint8_t *in, size_t in_size, uint8_t *out,
-		size_t *out_pos, size_t out_size)
+lzma_raw_buffer_encode(
+		const lzma_filter *filters, const lzma_allocator *allocator,
+		const uint8_t *in, size_t in_size,
+		uint8_t *out, size_t *out_pos, size_t out_size)
 {
 	// Validate what isn't validated later in filter_common.c.
 	if ((in == NULL && in_size != 0) || out == NULL

contrib/xz/src/liblzma/common/filter_common.c

@@ -123,7 +123,7 @@ static const struct {
 
 extern LZMA_API(lzma_ret)
 lzma_filters_copy(const lzma_filter *src, lzma_filter *dest,
-		lzma_allocator *allocator)
+		const lzma_allocator *allocator)
 {
 	if (src == NULL || dest == NULL)
 		return LZMA_PROG_ERROR;
@@ -239,7 +239,7 @@ validate_chain(const lzma_filter *filters, size_t *count)
 
 
 extern lzma_ret
-lzma_raw_coder_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_raw_coder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter *options,
 		lzma_filter_find coder_find, bool is_encoder)
 {

contrib/xz/src/liblzma/common/filter_common.h

@@ -36,7 +36,7 @@ typedef const lzma_filter_coder *(*lzma_filter_find)(lzma_vli id);
 
 
 extern lzma_ret lzma_raw_coder_init(
-		lzma_next_coder *next, lzma_allocator *allocator,
+		lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter *filters,
 		lzma_filter_find coder_find, bool is_encoder);
 

contrib/xz/src/liblzma/common/filter_decoder.c

@@ -35,7 +35,8 @@ typedef struct {
 	/// \return     - LZMA_OK: Properties decoded successfully.
 	///             - LZMA_OPTIONS_ERROR: Unsupported properties
 	///             - LZMA_MEM_ERROR: Memory allocation failed.
-	lzma_ret (*props_decode)(void **options, lzma_allocator *allocator,
+	lzma_ret (*props_decode)(
+			void **options, const lzma_allocator *allocator,
 			const uint8_t *props, size_t props_size);
 
 } lzma_filter_decoder;
@@ -136,7 +137,7 @@ lzma_filter_decoder_is_supported(lzma_vli id)
 
 
 extern lzma_ret
-lzma_raw_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_raw_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter *options)
 {
 	return lzma_raw_coder_init(next, allocator,
@@ -165,7 +166,7 @@ lzma_raw_decoder_memusage(const lzma_filter *filters)
 
 
 extern LZMA_API(lzma_ret)
-lzma_properties_decode(lzma_filter *filter, lzma_allocator *allocator,
+lzma_properties_decode(lzma_filter *filter, const lzma_allocator *allocator,
 		const uint8_t *props, size_t props_size)
 {
 	// Make it always NULL so that the caller can always safely free() it.

contrib/xz/src/liblzma/common/filter_decoder.h

@@ -17,7 +17,7 @@
 
 
 extern lzma_ret lzma_raw_decoder_init(
-		lzma_next_coder *next, lzma_allocator *allocator,
+		lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter *options);
 
 #endif

contrib/xz/src/liblzma/common/filter_encoder.c

@@ -30,11 +30,11 @@ typedef struct {
 	/// invalid, UINT64_MAX is returned.
 	uint64_t (*memusage)(const void *options);
 
-	/// Calculates the minimum sane size for Blocks (or other types of
-	/// chunks) to which the input data can be split to make
-	/// multithreaded encoding possible. If this is NULL, it is assumed
-	/// that the encoder is fast enough with single thread.
-	lzma_vli (*chunk_size)(const void *options);
+	/// Calculates the recommended Uncompressed Size for .xz Blocks to
+	/// which the input data can be split to make multithreaded
+	/// encoding possible. If this is NULL, it is assumed that
+	/// the encoder is fast enough with single thread.
+	uint64_t (*block_size)(const void *options);
 
 	/// Tells the size of the Filter Properties field. If options are
 	/// invalid, UINT32_MAX is returned. If this is NULL, props_size_fixed
@@ -59,7 +59,7 @@ static const lzma_filter_encoder encoders[] = {
 		.id = LZMA_FILTER_LZMA1,
 		.init = &lzma_lzma_encoder_init,
 		.memusage = &lzma_lzma_encoder_memusage,
-		.chunk_size = NULL, // FIXME
+		.block_size = NULL, // FIXME
 		.props_size_get = NULL,
 		.props_size_fixed = 5,
 		.props_encode = &lzma_lzma_props_encode,
@@ -70,7 +70,7 @@ static const lzma_filter_encoder encoders[] = {
 		.id = LZMA_FILTER_LZMA2,
 		.init = &lzma_lzma2_encoder_init,
 		.memusage = &lzma_lzma2_encoder_memusage,
-		.chunk_size = NULL, // FIXME
+		.block_size = &lzma_lzma2_block_size, // FIXME
 		.props_size_get = NULL,
 		.props_size_fixed = 1,
 		.props_encode = &lzma_lzma2_props_encode,
@@ -81,7 +81,7 @@ static const lzma_filter_encoder encoders[] = {
 		.id = LZMA_FILTER_X86,
 		.init = &lzma_simple_x86_encoder_init,
 		.memusage = NULL,
-		.chunk_size = NULL,
+		.block_size = NULL,
 		.props_size_get = &lzma_simple_props_size,
 		.props_encode = &lzma_simple_props_encode,
 	},
@@ -91,7 +91,7 @@ static const lzma_filter_encoder encoders[] = {
 		.id = LZMA_FILTER_POWERPC,
 		.init = &lzma_simple_powerpc_encoder_init,
 		.memusage = NULL,
-		.chunk_size = NULL,
+		.block_size = NULL,
 		.props_size_get = &lzma_simple_props_size,
 		.props_encode = &lzma_simple_props_encode,
 	},
@@ -101,7 +101,7 @@ static const lzma_filter_encoder encoders[] = {
 		.id = LZMA_FILTER_IA64,
 		.init = &lzma_simple_ia64_encoder_init,
 		.memusage = NULL,
-		.chunk_size = NULL,
+		.block_size = NULL,
 		.props_size_get = &lzma_simple_props_size,
 		.props_encode = &lzma_simple_props_encode,
 	},
@@ -111,7 +111,7 @@ static const lzma_filter_encoder encoders[] = {
 		.id = LZMA_FILTER_ARM,
 		.init = &lzma_simple_arm_encoder_init,
 		.memusage = NULL,
-		.chunk_size = NULL,
+		.block_size = NULL,
 		.props_size_get = &lzma_simple_props_size,
 		.props_encode = &lzma_simple_props_encode,
 	},
@@ -121,7 +121,7 @@ static const lzma_filter_encoder encoders[] = {
 		.id = LZMA_FILTER_ARMTHUMB,
 		.init = &lzma_simple_armthumb_encoder_init,
 		.memusage = NULL,
-		.chunk_size = NULL,
+		.block_size = NULL,
 		.props_size_get = &lzma_simple_props_size,
 		.props_encode = &lzma_simple_props_encode,
 	},
@@ -131,7 +131,7 @@ static const lzma_filter_encoder encoders[] = {
 		.id = LZMA_FILTER_SPARC,
 		.init = &lzma_simple_sparc_encoder_init,
 		.memusage = NULL,
-		.chunk_size = NULL,
+		.block_size = NULL,
 		.props_size_get = &lzma_simple_props_size,
 		.props_encode = &lzma_simple_props_encode,
 	},
@@ -141,7 +141,7 @@ static const lzma_filter_encoder encoders[] = {
 		.id = LZMA_FILTER_DELTA,
 		.init = &lzma_delta_encoder_init,
 		.memusage = &lzma_delta_coder_memusage,
-		.chunk_size = NULL,
+		.block_size = NULL,
 		.props_size_get = NULL,
 		.props_size_fixed = 1,
 		.props_encode = &lzma_delta_props_encode,
@@ -196,7 +196,7 @@ lzma_filters_update(lzma_stream *strm, const lzma_filter *filters)
 
 
 extern lzma_ret
-lzma_raw_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_raw_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter *options)
 {
 	return lzma_raw_coder_init(next, allocator,
@@ -226,20 +226,19 @@ lzma_raw_encoder_memusage(const lzma_filter *filters)
 }
 
 
-/*
-extern LZMA_API(lzma_vli)
-lzma_chunk_size(const lzma_filter *filters)
+extern uint64_t
+lzma_mt_block_size(const lzma_filter *filters)
 {
-	lzma_vli max = 0;
+	uint64_t max = 0;
 
 	for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) {
 		const lzma_filter_encoder *const fe
 				= encoder_find(filters[i].id);
-		if (fe->chunk_size != NULL) {
-			const lzma_vli size
-					= fe->chunk_size(filters[i].options);
-			if (size == LZMA_VLI_UNKNOWN)
-				return LZMA_VLI_UNKNOWN;
+		if (fe->block_size != NULL) {
+			const uint64_t size
+					= fe->block_size(filters[i].options);
+			if (size == 0)
+				return 0;
 
 			if (size > max)
 				max = size;
@@ -248,7 +247,6 @@ lzma_chunk_size(const lzma_filter *filters)
 
 	return max;
 }
-*/
 
 
 extern LZMA_API(lzma_ret)

contrib/xz/src/liblzma/common/filter_encoder.h

@@ -16,12 +16,12 @@
 #include "common.h"
 
 
-// FIXME: Might become a part of the public API once finished.
-// extern lzma_vli lzma_chunk_size(const lzma_filter *filters);
+// FIXME: Might become a part of the public API.
+extern uint64_t lzma_mt_block_size(const lzma_filter *filters);
 
 
 extern lzma_ret lzma_raw_encoder_init(
-		lzma_next_coder *next, lzma_allocator *allocator,
+		lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter *filters);
 
 #endif

contrib/xz/src/liblzma/common/filter_flags_decoder.c

@@ -15,7 +15,7 @@
 
 extern LZMA_API(lzma_ret)
 lzma_filter_flags_decode(
-		lzma_filter *filter, lzma_allocator *allocator,
+		lzma_filter *filter, const lzma_allocator *allocator,
 		const uint8_t *in, size_t *in_pos, size_t in_size)
 {
 	// Set the pointer to NULL so the caller can always safely free it.

contrib/xz/src/liblzma/common/hardware_cputhreads.c

@@ -1,7 +1,7 @@
 ///////////////////////////////////////////////////////////////////////////////
 //
-/// \file       stream_encoder.h
-/// \brief      Encodes .xz Streams
+/// \file       hardware_cputhreads.c
+/// \brief      Get the number of CPU threads or cores
 //
 //  Author:     Lasse Collin
 //
@@ -10,14 +10,13 @@
 //
 ///////////////////////////////////////////////////////////////////////////////
 
-#ifndef LZMA_STREAM_ENCODER_H
-#define LZMA_STREAM_ENCODER_H
-
 #include "common.h"
 
+#include "tuklib_cpucores.h"
 
-extern lzma_ret lzma_stream_encoder_init(
-		lzma_next_coder *next, lzma_allocator *allocator,
-		const lzma_filter *filters, lzma_check check);
 
-#endif
+extern LZMA_API(uint32_t)
+lzma_cputhreads(void)
+{
+	return tuklib_cpucores();
+}

contrib/xz/src/liblzma/common/index.c

@@ -191,8 +191,8 @@ index_tree_init(index_tree *tree)
 
 /// Helper for index_tree_end()
 static void
-index_tree_node_end(index_tree_node *node, lzma_allocator *allocator,
-		void (*free_func)(void *node, lzma_allocator *allocator))
+index_tree_node_end(index_tree_node *node, const lzma_allocator *allocator,
+		void (*free_func)(void *node, const lzma_allocator *allocator))
 {
 	// The tree won't ever be very huge, so recursion should be fine.
 	// 20 levels in the tree is likely quite a lot already in practice.
@@ -215,8 +215,8 @@ index_tree_node_end(index_tree_node *node, lzma_allocator *allocator,
 /// to free the Record groups from each index_stream before freeing
 /// the index_stream itself.
 static void
-index_tree_end(index_tree *tree, lzma_allocator *allocator,
-		void (*free_func)(void *node, lzma_allocator *allocator))
+index_tree_end(index_tree *tree, const lzma_allocator *allocator,
+		void (*free_func)(void *node, const lzma_allocator *allocator))
 {
 	if (tree->root != NULL)
 		index_tree_node_end(tree->root, allocator, free_func);
@@ -340,7 +340,7 @@ index_tree_locate(const index_tree *tree, lzma_vli target)
 static index_stream *
 index_stream_init(lzma_vli compressed_base, lzma_vli uncompressed_base,
 		lzma_vli stream_number, lzma_vli block_number_base,
-		lzma_allocator *allocator)
+		const lzma_allocator *allocator)
 {
 	index_stream *s = lzma_alloc(sizeof(index_stream), allocator);
 	if (s == NULL)
@@ -368,7 +368,7 @@ index_stream_init(lzma_vli compressed_base, lzma_vli uncompressed_base,
 
 /// Free the memory allocated for a Stream and its Record groups.
 static void
-index_stream_end(void *node, lzma_allocator *allocator)
+index_stream_end(void *node, const lzma_allocator *allocator)
 {
 	index_stream *s = node;
 	index_tree_end(&s->groups, allocator, NULL);
@@ -377,7 +377,7 @@ index_stream_end(void *node, lzma_allocator *allocator)
 
 
 static lzma_index *
-index_init_plain(lzma_allocator *allocator)
+index_init_plain(const lzma_allocator *allocator)
 {
 	lzma_index *i = lzma_alloc(sizeof(lzma_index), allocator);
 	if (i != NULL) {
@@ -395,7 +395,7 @@ index_init_plain(lzma_allocator *allocator)
 
 
 extern LZMA_API(lzma_index *)
-lzma_index_init(lzma_allocator *allocator)
+lzma_index_init(const lzma_allocator *allocator)
 {
 	lzma_index *i = index_init_plain(allocator);
 	if (i == NULL)
@@ -414,7 +414,7 @@ lzma_index_init(lzma_allocator *allocator)
 
 
 extern LZMA_API(void)
-lzma_index_end(lzma_index *i, lzma_allocator *allocator)
+lzma_index_end(lzma_index *i, const lzma_allocator *allocator)
 {
 	// NOTE: If you modify this function, check also the bottom
 	// of lzma_index_cat().
@@ -637,7 +637,7 @@ lzma_index_stream_padding(lzma_index *i, lzma_vli stream_padding)
 
 
 extern LZMA_API(lzma_ret)
-lzma_index_append(lzma_index *i, lzma_allocator *allocator,
+lzma_index_append(lzma_index *i, const lzma_allocator *allocator,
 		lzma_vli unpadded_size, lzma_vli uncompressed_size)
 {
 	// Validate.
@@ -765,7 +765,7 @@ index_cat_helper(const index_cat_info *info, index_stream *this)
 
 extern LZMA_API(lzma_ret)
 lzma_index_cat(lzma_index *restrict dest, lzma_index *restrict src,
-		lzma_allocator *allocator)
+		const lzma_allocator *allocator)
 {
 	const lzma_vli dest_file_size = lzma_index_file_size(dest);
 
@@ -859,7 +859,7 @@ lzma_index_cat(lzma_index *restrict dest, lzma_index *restrict src,
 
 /// Duplicate an index_stream.
 static index_stream *
-index_dup_stream(const index_stream *src, lzma_allocator *allocator)
+index_dup_stream(const index_stream *src, const lzma_allocator *allocator)
 {
 	// Catch a somewhat theoretical integer overflow.
 	if (src->record_count > PREALLOC_MAX)
@@ -919,7 +919,7 @@ index_dup_stream(const index_stream *src, lzma_allocator *allocator)
 
 
 extern LZMA_API(lzma_index *)
-lzma_index_dup(const lzma_index *src, lzma_allocator *allocator)
+lzma_index_dup(const lzma_index *src, const lzma_allocator *allocator)
 {
 	// Allocate the base structure (no initial Stream).
 	lzma_index *dest = index_init_plain(allocator);

contrib/xz/src/liblzma/common/index_decoder.c

@@ -54,7 +54,7 @@ struct lzma_coder_s {
 
 
 static lzma_ret
-index_decode(lzma_coder *coder, lzma_allocator *allocator,
+index_decode(lzma_coder *coder, const lzma_allocator *allocator,
 		const uint8_t *restrict in, size_t *restrict in_pos,
 		size_t in_size,
 		uint8_t *restrict out lzma_attribute((__unused__)),
@@ -207,7 +207,7 @@ index_decode(lzma_coder *coder, lzma_allocator *allocator,
 
 
 static void
-index_decoder_end(lzma_coder *coder, lzma_allocator *allocator)
+index_decoder_end(lzma_coder *coder, const lzma_allocator *allocator)
 {
 	lzma_index_end(coder->index, allocator);
 	lzma_free(coder, allocator);
@@ -234,7 +234,7 @@ index_decoder_memconfig(lzma_coder *coder, uint64_t *memusage,
 
 
 static lzma_ret
-index_decoder_reset(lzma_coder *coder, lzma_allocator *allocator,
+index_decoder_reset(lzma_coder *coder, const lzma_allocator *allocator,
 		lzma_index **i, uint64_t memlimit)
 {
 	// Remember the pointer given by the application. We will set it
@@ -261,7 +261,7 @@ index_decoder_reset(lzma_coder *coder, lzma_allocator *allocator,
 
 
 static lzma_ret
-index_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+index_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		lzma_index **i, uint64_t memlimit)
 {
 	lzma_next_coder_init(&index_decoder_init, next, allocator);
@@ -299,8 +299,8 @@ lzma_index_decoder(lzma_stream *strm, lzma_index **i, uint64_t memlimit)
 
 
 extern LZMA_API(lzma_ret)
-lzma_index_buffer_decode(
-		lzma_index **i, uint64_t *memlimit, lzma_allocator *allocator,
+lzma_index_buffer_decode(lzma_index **i, uint64_t *memlimit,
+		const lzma_allocator *allocator,
 		const uint8_t *in, size_t *in_pos, size_t in_size)
 {
 	// Sanity checks

contrib/xz/src/liblzma/common/index_encoder.c

@@ -42,7 +42,7 @@ struct lzma_coder_s {
 
 static lzma_ret
 index_encode(lzma_coder *coder,
-		lzma_allocator *allocator lzma_attribute((__unused__)),
+		const lzma_allocator *allocator lzma_attribute((__unused__)),
 		const uint8_t *restrict in lzma_attribute((__unused__)),
 		size_t *restrict in_pos lzma_attribute((__unused__)),
 		size_t in_size lzma_attribute((__unused__)),
@@ -159,7 +159,7 @@ index_encode(lzma_coder *coder,
 
 
 static void
-index_encoder_end(lzma_coder *coder, lzma_allocator *allocator)
+index_encoder_end(lzma_coder *coder, const lzma_allocator *allocator)
 {
 	lzma_free(coder, allocator);
 	return;
@@ -181,7 +181,7 @@ index_encoder_reset(lzma_coder *coder, const lzma_index *i)
 
 
 extern lzma_ret
-lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_index_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_index *i)
 {
 	lzma_next_coder_init(&lzma_index_encoder_init, next, allocator);

contrib/xz/src/liblzma/common/index_encoder.h

@@ -17,7 +17,7 @@
 
 
 extern lzma_ret lzma_index_encoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_index *i);
+		const lzma_allocator *allocator, const lzma_index *i);
 
 
 #endif

contrib/xz/src/liblzma/common/index_hash.c

@@ -70,7 +70,8 @@ struct lzma_index_hash_s {
 
 
 extern LZMA_API(lzma_index_hash *)
-lzma_index_hash_init(lzma_index_hash *index_hash, lzma_allocator *allocator)
+lzma_index_hash_init(lzma_index_hash *index_hash,
+		const lzma_allocator *allocator)
 {
 	if (index_hash == NULL) {
 		index_hash = lzma_alloc(sizeof(lzma_index_hash), allocator);
@@ -101,7 +102,8 @@ lzma_index_hash_init(lzma_index_hash *index_hash, lzma_allocator *allocator)
 
 
 extern LZMA_API(void)
-lzma_index_hash_end(lzma_index_hash *index_hash, lzma_allocator *allocator)
+lzma_index_hash_end(lzma_index_hash *index_hash,
+		const lzma_allocator *allocator)
 {
 	lzma_free(index_hash, allocator);
 	return;

contrib/xz/src/liblzma/common/memcmplen.h

@@ -0,0 +1,170 @@
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file       memcmplen.h
+/// \brief      Optimized comparison of two buffers
+//
+//  Author:     Lasse Collin
+//
+//  This file has been put into the public domain.
+//  You can do whatever you want with this file.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef LZMA_MEMCMPLEN_H
+#define LZMA_MEMCMPLEN_H
+
+#include "common.h"
+
+#ifdef HAVE_IMMINTRIN_H
+#	include <immintrin.h>
+#endif
+
+/// How many extra bytes lzma_memcmplen() may read. This depends on
+/// the method but since it is just a few bytes the biggest possible
+/// value is used here.
+#define LZMA_MEMCMPLEN_EXTRA 16
+
+
+/// Find out how many equal bytes the two buffers have.
+///
+/// \param      buf1    First buffer
+/// \param      buf2    Second buffer
+/// \param      len     How many bytes have already been compared and will
+///                     be assumed to match
+/// \param      limit   How many bytes to compare at most, including the
+///                     already-compared bytes. This must be significantly
+///                     smaller than UINT32_MAX to avoid integer overflows.
+///                     Up to LZMA_MEMCMPLEN_EXTRA bytes may be read past
+///                     the specified limit from both buf1 and buf2.
+///
+/// \return     Number of equal bytes in the buffers is returned.
+///             This is always at least len and at most limit.
+static inline uint32_t lzma_attribute((__always_inline__))
+lzma_memcmplen(const uint8_t *buf1, const uint8_t *buf2,
+		uint32_t len, uint32_t limit)
+{
+	assert(len <= limit);
+	assert(limit <= UINT32_MAX / 2);
+
+#if defined(TUKLIB_FAST_UNALIGNED_ACCESS) \
+		&& ((TUKLIB_GNUC_REQ(3, 4) && defined(__x86_64__)) \
+			|| (defined(__INTEL_COMPILER) && defined(__x86_64__)) \
+			|| (defined(__INTEL_COMPILER) && defined(_M_X64)) \
+			|| (defined(_MSC_VER) && defined(_M_X64)))
+	// NOTE: This will use 64-bit unaligned access which
+	// TUKLIB_FAST_UNALIGNED_ACCESS wasn't meant to permit, but
+	// it's convenient here at least as long as it's x86-64 only.
+	//
+	// I keep this x86-64 only for now since that's where I know this
+	// to be a good method. This may be fine on other 64-bit CPUs too.
+	// On big endian one should use xor instead of subtraction and switch
+	// to __builtin_clzll().
+	while (len < limit) {
+		const uint64_t x = *(const uint64_t *)(buf1 + len)
+				- *(const uint64_t *)(buf2 + len);
+		if (x != 0) {
+#	if defined(_M_X64) // MSVC or Intel C compiler on Windows
+			unsigned long tmp;
+			_BitScanForward64(&tmp, x);
+			len += (uint32_t)tmp >> 3;
+#	else // GCC, clang, or Intel C compiler
+			len += (uint32_t)__builtin_ctzll(x) >> 3;
+#	endif
+			return my_min(len, limit);
+		}
+
+		len += 8;
+	}
+
+	return limit;
+
+#elif defined(TUKLIB_FAST_UNALIGNED_ACCESS) \
+		&& defined(HAVE__MM_MOVEMASK_EPI8) \
+		&& ((defined(__GNUC__) && defined(__SSE2_MATH__)) \
+			|| (defined(__INTEL_COMPILER) && defined(__SSE2__)) \
+			|| (defined(_MSC_VER) && defined(_M_IX86_FP) \
+				&& _M_IX86_FP >= 2))
+	// NOTE: Like above, this will use 128-bit unaligned access which
+	// TUKLIB_FAST_UNALIGNED_ACCESS wasn't meant to permit.
+	//
+	// SSE2 version for 32-bit and 64-bit x86. On x86-64 the above
+	// version is sometimes significantly faster and sometimes
+	// slightly slower than this SSE2 version, so this SSE2
+	// version isn't used on x86-64.
+	while (len < limit) {
+		const uint32_t x = 0xFFFF ^ _mm_movemask_epi8(_mm_cmpeq_epi8(
+			_mm_loadu_si128((const __m128i *)(buf1 + len)),
+			_mm_loadu_si128((const __m128i *)(buf2 + len))));
+
+		if (x != 0) {
+#	if defined(__INTEL_COMPILER)
+			len += _bit_scan_forward(x);
+#	elif defined(_MSC_VER)
+			unsigned long tmp;
+			_BitScanForward(&tmp, x);
+			len += tmp;
+#	else
+			len += __builtin_ctz(x);
+#	endif
+			return my_min(len, limit);
+		}
+
+		len += 16;
+	}
+
+	return limit;
+
+#elif defined(TUKLIB_FAST_UNALIGNED_ACCESS) && !defined(WORDS_BIGENDIAN)
+	// Generic 32-bit little endian method
+	while (len < limit) {
+		uint32_t x = *(const uint32_t *)(buf1 + len)
+				- *(const uint32_t *)(buf2 + len);
+		if (x != 0) {
+			if ((x & 0xFFFF) == 0) {
+				len += 2;
+				x >>= 16;
+			}
+
+			if ((x & 0xFF) == 0)
+				++len;
+
+			return my_min(len, limit);
+		}
+
+		len += 4;
+	}
+
+	return limit;
+
+#elif defined(TUKLIB_FAST_UNALIGNED_ACCESS) && defined(WORDS_BIGENDIAN)
+	// Generic 32-bit big endian method
+	while (len < limit) {
+		uint32_t x = *(const uint32_t *)(buf1 + len)
+				^ *(const uint32_t *)(buf2 + len);
+		if (x != 0) {
+			if ((x & 0xFFFF0000) == 0) {
+				len += 2;
+				x <<= 16;
+			}
+
+			if ((x & 0xFF000000) == 0)
+				++len;
+
+			return my_min(len, limit);
+		}
+
+		len += 4;
+	}
+
+	return limit;
+
+#else
+	// Simple portable version that doesn't use unaligned access.
+	while (len < limit && buf1[len] == buf2[len])
+		++len;
+
+	return len;
+#endif
+}
+
+#endif

contrib/xz/src/liblzma/common/outqueue.c

@@ -0,0 +1,184 @@
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file       outqueue.c
+/// \brief      Output queue handling in multithreaded coding
+//
+//  Author:     Lasse Collin
+//
+//  This file has been put into the public domain.
+//  You can do whatever you want with this file.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#include "outqueue.h"
+
+
+/// This is to ease integer overflow checking: We may allocate up to
+/// 2 * LZMA_THREADS_MAX buffers and we need some extra memory for other
+/// data structures (that's the second /2).
+#define BUF_SIZE_MAX (UINT64_MAX / LZMA_THREADS_MAX / 2 / 2)
+
+
+static lzma_ret
+get_options(uint64_t *bufs_alloc_size, uint32_t *bufs_count,
+		uint64_t buf_size_max, uint32_t threads)
+{
+	if (threads > LZMA_THREADS_MAX || buf_size_max > BUF_SIZE_MAX)
+		return LZMA_OPTIONS_ERROR;
+
+	// The number of buffers is twice the number of threads.
+	// This wastes RAM but keeps the threads busy when buffers
+	// finish out of order.
+	//
+	// NOTE: If this is changed, update BUF_SIZE_MAX too.
+	*bufs_count = threads * 2;
+	*bufs_alloc_size = *bufs_count * buf_size_max;
+
+	return LZMA_OK;
+}
+
+
+extern uint64_t
+lzma_outq_memusage(uint64_t buf_size_max, uint32_t threads)
+{
+	uint64_t bufs_alloc_size;
+	uint32_t bufs_count;
+
+	if (get_options(&bufs_alloc_size, &bufs_count, buf_size_max, threads)
+			!= LZMA_OK)
+		return UINT64_MAX;
+
+	return sizeof(lzma_outq) + bufs_count * sizeof(lzma_outbuf)
+			+ bufs_alloc_size;
+}
+
+
+extern lzma_ret
+lzma_outq_init(lzma_outq *outq, const lzma_allocator *allocator,
+		uint64_t buf_size_max, uint32_t threads)
+{
+	uint64_t bufs_alloc_size;
+	uint32_t bufs_count;
+
+	// Set bufs_count and bufs_alloc_size.
+	return_if_error(get_options(&bufs_alloc_size, &bufs_count,
+			buf_size_max, threads));
+
+	// Allocate memory if needed.
+	if (outq->buf_size_max != buf_size_max
+			|| outq->bufs_allocated != bufs_count) {
+		lzma_outq_end(outq, allocator);
+
+#if SIZE_MAX < UINT64_MAX
+		if (bufs_alloc_size > SIZE_MAX)
+			return LZMA_MEM_ERROR;
+#endif
+
+		outq->bufs = lzma_alloc(bufs_count * sizeof(lzma_outbuf),
+				allocator);
+		outq->bufs_mem = lzma_alloc((size_t)(bufs_alloc_size),
+				allocator);
+
+		if (outq->bufs == NULL || outq->bufs_mem == NULL) {
+			lzma_outq_end(outq, allocator);
+			return LZMA_MEM_ERROR;
+		}
+	}
+
+	// Initialize the rest of the main structure. Initialization of
+	// outq->bufs[] is done when they are actually needed.
+	outq->buf_size_max = (size_t)(buf_size_max);
+	outq->bufs_allocated = bufs_count;
+	outq->bufs_pos = 0;
+	outq->bufs_used = 0;
+	outq->read_pos = 0;
+
+	return LZMA_OK;
+}
+
+
+extern void
+lzma_outq_end(lzma_outq *outq, const lzma_allocator *allocator)
+{
+	lzma_free(outq->bufs, allocator);
+	outq->bufs = NULL;
+
+	lzma_free(outq->bufs_mem, allocator);
+	outq->bufs_mem = NULL;
+
+	return;
+}
+
+
+extern lzma_outbuf *
+lzma_outq_get_buf(lzma_outq *outq)
+{
+	// Caller must have checked it with lzma_outq_has_buf().
+	assert(outq->bufs_used < outq->bufs_allocated);
+
+	// Initialize the new buffer.
+	lzma_outbuf *buf = &outq->bufs[outq->bufs_pos];
+	buf->buf = outq->bufs_mem + outq->bufs_pos * outq->buf_size_max;
+	buf->size = 0;
+	buf->finished = false;
+
+	// Update the queue state.
+	if (++outq->bufs_pos == outq->bufs_allocated)
+		outq->bufs_pos = 0;
+
+	++outq->bufs_used;
+
+	return buf;
+}
+
+
+extern bool
+lzma_outq_is_readable(const lzma_outq *outq)
+{
+	uint32_t i = outq->bufs_pos - outq->bufs_used;
+	if (outq->bufs_pos < outq->bufs_used)
+		i += outq->bufs_allocated;
+
+	return outq->bufs[i].finished;
+}
+
+
+extern lzma_ret
+lzma_outq_read(lzma_outq *restrict outq, uint8_t *restrict out,
+		size_t *restrict out_pos, size_t out_size,
+		lzma_vli *restrict unpadded_size,
+		lzma_vli *restrict uncompressed_size)
+{
+	// There must be at least one buffer from which to read.
+	if (outq->bufs_used == 0)
+		return LZMA_OK;
+
+	// Get the buffer.
+	uint32_t i = outq->bufs_pos - outq->bufs_used;
+	if (outq->bufs_pos < outq->bufs_used)
+		i += outq->bufs_allocated;
+
+	lzma_outbuf *buf = &outq->bufs[i];
+
+	// If it isn't finished yet, we cannot read from it.
+	if (!buf->finished)
+		return LZMA_OK;
+
+	// Copy from the buffer to output.
+	lzma_bufcpy(buf->buf, &outq->read_pos, buf->size,
+			out, out_pos, out_size);
+
+	// Return if we didn't get all the data from the buffer.
+	if (outq->read_pos < buf->size)
+		return LZMA_OK;
+
+	// The buffer was finished. Tell the caller its size information.
+	*unpadded_size = buf->unpadded_size;
+	*uncompressed_size = buf->uncompressed_size;
+
+	// Free this buffer for further use.
+	--outq->bufs_used;
+	outq->read_pos = 0;
+
+	return LZMA_STREAM_END;
+}

contrib/xz/src/liblzma/common/outqueue.h

@@ -0,0 +1,156 @@
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file       outqueue.h
+/// \brief      Output queue handling in multithreaded coding
+//
+//  Author:     Lasse Collin
+//
+//  This file has been put into the public domain.
+//  You can do whatever you want with this file.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#include "common.h"
+
+
+/// Output buffer for a single thread
+typedef struct {
+	/// Pointer to the output buffer of lzma_outq.buf_size_max bytes
+	uint8_t *buf;
+
+	/// Amount of data written to buf
+	size_t size;
+
+	/// Additional size information
+	lzma_vli unpadded_size;
+	lzma_vli uncompressed_size;
+
+	/// True when no more data will be written into this buffer.
+	///
+	/// \note       This is read by another thread and thus access
+	///             to this variable needs a mutex.
+	bool finished;
+
+} lzma_outbuf;
+
+
+typedef struct {
+	/// Array of buffers that are used cyclically.
+	lzma_outbuf *bufs;
+
+	/// Memory allocated for all the buffers
+	uint8_t *bufs_mem;
+
+	/// Amount of buffer space available in each buffer
+	size_t buf_size_max;
+
+	/// Number of buffers allocated
+	uint32_t bufs_allocated;
+
+	/// Position in the bufs array. The next buffer to be taken
+	/// into use is bufs[bufs_pos].
+	uint32_t bufs_pos;
+
+	/// Number of buffers in use
+	uint32_t bufs_used;
+
+	/// Position in the buffer in lzma_outq_read()
+	size_t read_pos;
+
+} lzma_outq;
+
+
+/**
+ * \brief       Calculate the memory usage of an output queue
+ *
+ * \return      Approximate memory usage in bytes or UINT64_MAX on error.
+ */
+extern uint64_t lzma_outq_memusage(uint64_t buf_size_max, uint32_t threads);
+
+
+/// \brief      Initialize an output queue
+///
+/// \param      outq            Pointer to an output queue. Before calling
+///                             this function the first time, *outq should
+///                             have been zeroed with memzero() so that this
+///                             function knows that there are no previous
+///                             allocations to free.
+/// \param      allocator       Pointer to allocator or NULL
+/// \param      buf_size_max    Maximum amount of data that a single buffer
+///                             in the queue may need to store.
+/// \param      threads         Number of buffers that may be in use
+///                             concurrently. Note that more than this number
+///                             of buffers will actually get allocated to
+///                             improve performance when buffers finish
+///                             out of order.
+///
+/// \return     - LZMA_OK
+///             - LZMA_MEM_ERROR
+///
+extern lzma_ret lzma_outq_init(
+		lzma_outq *outq, const lzma_allocator *allocator,
+		uint64_t buf_size_max, uint32_t threads);
+
+
+/// \brief      Free the memory associated with the output queue
+extern void lzma_outq_end(lzma_outq *outq, const lzma_allocator *allocator);
+
+
+/// \brief      Get a new buffer
+///
+/// lzma_outq_has_buf() must be used to check that there is a buffer
+/// available before calling lzma_outq_get_buf().
+///
+extern lzma_outbuf *lzma_outq_get_buf(lzma_outq *outq);
+
+
+/// \brief      Test if there is data ready to be read
+///
+/// Call to this function must be protected with the same mutex that
+/// is used to protect lzma_outbuf.finished.
+///
+extern bool lzma_outq_is_readable(const lzma_outq *outq);
+
+
+/// \brief      Read finished data
+///
+/// \param      outq            Pointer to an output queue
+/// \param      out             Beginning of the output buffer
+/// \param      out_pos         The next byte will be written to
+///                             out[*out_pos].
+/// \param      out_size        Size of the out buffer; the first byte into
+///                             which no data is written to is out[out_size].
+/// \param      unpadded_size   Unpadded Size from the Block encoder
+/// \param      uncompressed_size Uncompressed Size from the Block encoder
+///
+/// \return     - LZMA: All OK. Either no data was available or the buffer
+///               being read didn't become empty yet.
+///             - LZMA_STREAM_END: The buffer being read was finished.
+///               *unpadded_size and *uncompressed_size were set.
+///
+/// \note       This reads lzma_outbuf.finished variables and thus call
+///             to this function needs to be protected with a mutex.
+///
+extern lzma_ret lzma_outq_read(lzma_outq *restrict outq,
+		uint8_t *restrict out, size_t *restrict out_pos,
+		size_t out_size, lzma_vli *restrict unpadded_size,
+		lzma_vli *restrict uncompressed_size);
+
+
+/// \brief      Test if there is at least one buffer free
+///
+/// This must be used before getting a new buffer with lzma_outq_get_buf().
+///
+static inline bool
+lzma_outq_has_buf(const lzma_outq *outq)
+{
+	return outq->bufs_used < outq->bufs_allocated;
+}
+
+
+/// \brief      Test if the queue is completely empty
+static inline bool
+lzma_outq_is_empty(const lzma_outq *outq)
+{
+	return outq->bufs_used == 0;
+}

contrib/xz/src/liblzma/common/stream_buffer_decoder.c

@@ -15,7 +15,7 @@
 
 extern LZMA_API(lzma_ret)
 lzma_stream_buffer_decode(uint64_t *memlimit, uint32_t flags,
-		lzma_allocator *allocator,
+		const lzma_allocator *allocator,
 		const uint8_t *in, size_t *in_pos, size_t in_size,
 		uint8_t *out, size_t *out_pos, size_t out_size)
 {

contrib/xz/src/liblzma/common/stream_buffer_encoder.c

@@ -42,7 +42,8 @@ lzma_stream_buffer_bound(size_t uncompressed_size)
 
 extern LZMA_API(lzma_ret)
 lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check,
-		lzma_allocator *allocator, const uint8_t *in, size_t in_size,
+		const lzma_allocator *allocator,
+		const uint8_t *in, size_t in_size,
 		uint8_t *out, size_t *out_pos_ptr, size_t out_size)
 {
 	// Sanity checks

contrib/xz/src/liblzma/common/stream_decoder.c

@@ -57,6 +57,10 @@ struct lzma_coder_s {
 	/// If true, LZMA_GET_CHECK is returned after decoding Stream Header.
 	bool tell_any_check;
 
+	/// If true, we will tell the Block decoder to skip calculating
+	/// and verifying the integrity check.
+	bool ignore_check;
+
 	/// If true, we will decode concatenated Streams that possibly have
 	/// Stream Padding between or after them. LZMA_STREAM_END is returned
 	/// once the application isn't giving us any new input, and we aren't
@@ -80,7 +84,7 @@ struct lzma_coder_s {
 
 
 static lzma_ret
-stream_decoder_reset(lzma_coder *coder, lzma_allocator *allocator)
+stream_decoder_reset(lzma_coder *coder, const lzma_allocator *allocator)
 {
 	// Initialize the Index hash used to verify the Index.
 	coder->index_hash = lzma_index_hash_init(coder->index_hash, allocator);
@@ -96,7 +100,7 @@ stream_decoder_reset(lzma_coder *coder, lzma_allocator *allocator)
 
 
 static lzma_ret
-stream_decode(lzma_coder *coder, lzma_allocator *allocator,
+stream_decode(lzma_coder *coder, const lzma_allocator *allocator,
 		const uint8_t *restrict in, size_t *restrict in_pos,
 		size_t in_size, uint8_t *restrict out,
 		size_t *restrict out_pos, size_t out_size, lzma_action action)
@@ -182,8 +186,8 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator,
 
 		coder->pos = 0;
 
-		// Version 0 is currently the only possible version.
-		coder->block_options.version = 0;
+		// Version 1 is needed to support the .ignore_check option.
+		coder->block_options.version = 1;
 
 		// Set up a buffer to hold the filter chain. Block Header
 		// decoder will initialize all members of this array so
@@ -195,6 +199,11 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator,
 		return_if_error(lzma_block_header_decode(&coder->block_options,
 				allocator, coder->buffer));
 
+		// If LZMA_IGNORE_CHECK was used, this flag needs to be set.
+		// It has to be set after lzma_block_header_decode() because
+		// it always resets this to false.
+		coder->block_options.ignore_check = coder->ignore_check;
+
 		// Check the memory usage limit.
 		const uint64_t memusage = lzma_raw_decoder_memusage(filters);
 		lzma_ret ret;
@@ -366,7 +375,7 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator,
 
 
 static void
-stream_decoder_end(lzma_coder *coder, lzma_allocator *allocator)
+stream_decoder_end(lzma_coder *coder, const lzma_allocator *allocator)
 {
 	lzma_next_end(&coder->block_decoder, allocator);
 	lzma_index_hash_end(coder->index_hash, allocator);
@@ -401,7 +410,8 @@ stream_decoder_memconfig(lzma_coder *coder, uint64_t *memusage,
 
 
 extern lzma_ret
-lzma_stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_stream_decoder_init(
+		lzma_next_coder *next, const lzma_allocator *allocator,
 		uint64_t memlimit, uint32_t flags)
 {
 	lzma_next_coder_init(&lzma_stream_decoder_init, next, allocator);
@@ -432,6 +442,7 @@ lzma_stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
 	next->coder->tell_unsupported_check
 			= (flags & LZMA_TELL_UNSUPPORTED_CHECK) != 0;
 	next->coder->tell_any_check = (flags & LZMA_TELL_ANY_CHECK) != 0;
+	next->coder->ignore_check = (flags & LZMA_IGNORE_CHECK) != 0;
 	next->coder->concatenated = (flags & LZMA_CONCATENATED) != 0;
 	next->coder->first_stream = true;
 

contrib/xz/src/liblzma/common/stream_decoder.h

@@ -15,7 +15,8 @@
 
 #include "common.h"
 
-extern lzma_ret lzma_stream_decoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, uint64_t memlimit, uint32_t flags);
+extern lzma_ret lzma_stream_decoder_init(
+		lzma_next_coder *next, const lzma_allocator *allocator,
+		uint64_t memlimit, uint32_t flags);
 
 #endif

contrib/xz/src/liblzma/common/stream_encoder.c

@@ -10,7 +10,6 @@
 //
 ///////////////////////////////////////////////////////////////////////////////
 
-#include "stream_encoder.h"
 #include "block_encoder.h"
 #include "index_encoder.h"
 
@@ -26,7 +25,7 @@ struct lzma_coder_s {
 	} sequence;
 
 	/// True if Block encoder has been initialized by
-	/// lzma_stream_encoder_init() or stream_encoder_update()
+	/// stream_encoder_init() or stream_encoder_update()
 	/// and thus doesn't need to be initialized in stream_encode().
 	bool block_encoder_is_initialized;
 
@@ -60,7 +59,7 @@ struct lzma_coder_s {
 
 
 static lzma_ret
-block_encoder_init(lzma_coder *coder, lzma_allocator *allocator)
+block_encoder_init(lzma_coder *coder, const lzma_allocator *allocator)
 {
 	// Prepare the Block options. Even though Block encoder doesn't need
 	// compressed_size, uncompressed_size, and header_size to be
@@ -79,7 +78,7 @@ block_encoder_init(lzma_coder *coder, lzma_allocator *allocator)
 
 
 static lzma_ret
-stream_encode(lzma_coder *coder, lzma_allocator *allocator,
+stream_encode(lzma_coder *coder, const lzma_allocator *allocator,
 		const uint8_t *restrict in, size_t *restrict in_pos,
 		size_t in_size, uint8_t *restrict out,
 		size_t *restrict out_pos, size_t out_size, lzma_action action)
@@ -126,7 +125,7 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator,
 		}
 
 		// Initialize the Block encoder unless it was already
-		// initialized by lzma_stream_encoder_init() or
+		// initialized by stream_encoder_init() or
 		// stream_encoder_update().
 		if (!coder->block_encoder_is_initialized)
 			return_if_error(block_encoder_init(coder, allocator));
@@ -147,11 +146,12 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator,
 	}
 
 	case SEQ_BLOCK_ENCODE: {
-		static const lzma_action convert[4] = {
+		static const lzma_action convert[LZMA_ACTION_MAX + 1] = {
 			LZMA_RUN,
 			LZMA_SYNC_FLUSH,
 			LZMA_FINISH,
 			LZMA_FINISH,
+			LZMA_FINISH,
 		};
 
 		const lzma_ret ret = coder->block_encoder.code(
@@ -209,7 +209,7 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator,
 
 
 static void
-stream_encoder_end(lzma_coder *coder, lzma_allocator *allocator)
+stream_encoder_end(lzma_coder *coder, const lzma_allocator *allocator)
 {
 	lzma_next_end(&coder->block_encoder, allocator);
 	lzma_next_end(&coder->index_encoder, allocator);
@@ -224,7 +224,7 @@ stream_encoder_end(lzma_coder *coder, lzma_allocator *allocator)
 
 
 static lzma_ret
-stream_encoder_update(lzma_coder *coder, lzma_allocator *allocator,
+stream_encoder_update(lzma_coder *coder, const lzma_allocator *allocator,
 		const lzma_filter *filters,
 		const lzma_filter *reversed_filters)
 {
@@ -262,11 +262,11 @@ stream_encoder_update(lzma_coder *coder, lzma_allocator *allocator,
 }
 
 
-extern lzma_ret
-lzma_stream_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+static lzma_ret
+stream_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter *filters, lzma_check check)
 {
-	lzma_next_coder_init(&lzma_stream_encoder_init, next, allocator);
+	lzma_next_coder_init(&stream_encoder_init, next, allocator);
 
 	if (filters == NULL)
 		return LZMA_PROG_ERROR;
@@ -320,11 +320,12 @@ extern LZMA_API(lzma_ret)
 lzma_stream_encoder(lzma_stream *strm,
 		const lzma_filter *filters, lzma_check check)
 {
-	lzma_next_strm_init(lzma_stream_encoder_init, strm, filters, check);
+	lzma_next_strm_init(stream_encoder_init, strm, filters, check);
 
 	strm->internal->supported_actions[LZMA_RUN] = true;
 	strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true;
 	strm->internal->supported_actions[LZMA_FULL_FLUSH] = true;
+	strm->internal->supported_actions[LZMA_FULL_BARRIER] = true;
 	strm->internal->supported_actions[LZMA_FINISH] = true;
 
 	return LZMA_OK;

contrib/xz/src/liblzma/delta/delta_common.c

@@ -15,7 +15,7 @@
 
 
 static void
-delta_coder_end(lzma_coder *coder, lzma_allocator *allocator)
+delta_coder_end(lzma_coder *coder, const lzma_allocator *allocator)
 {
 	lzma_next_end(&coder->next, allocator);
 	lzma_free(coder, allocator);
@@ -24,7 +24,7 @@ delta_coder_end(lzma_coder *coder, lzma_allocator *allocator)
 
 
 extern lzma_ret
-lzma_delta_coder_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_delta_coder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter_info *filters)
 {
 	// Allocate memory for the decoder if needed.

contrib/xz/src/liblzma/delta/delta_decoder.c

@@ -27,7 +27,7 @@ decode_buffer(lzma_coder *coder, uint8_t *buffer, size_t size)
 
 
 static lzma_ret
-delta_decode(lzma_coder *coder, lzma_allocator *allocator,
+delta_decode(lzma_coder *coder, const lzma_allocator *allocator,
 		const uint8_t *restrict in, size_t *restrict in_pos,
 		size_t in_size, uint8_t *restrict out,
 		size_t *restrict out_pos, size_t out_size, lzma_action action)
@@ -47,7 +47,7 @@ delta_decode(lzma_coder *coder, lzma_allocator *allocator,
 
 
 extern lzma_ret
-lzma_delta_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_delta_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter_info *filters)
 {
 	next->code = &delta_decode;
@@ -56,7 +56,7 @@ lzma_delta_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
 
 
 extern lzma_ret
-lzma_delta_props_decode(void **options, lzma_allocator *allocator,
+lzma_delta_props_decode(void **options, const lzma_allocator *allocator,
 		const uint8_t *props, size_t props_size)
 {
 	if (props_size != 1)

contrib/xz/src/liblzma/delta/delta_decoder.h

@@ -16,10 +16,11 @@
 #include "delta_common.h"
 
 extern lzma_ret lzma_delta_decoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters);
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters);
 
 extern lzma_ret lzma_delta_props_decode(
-		void **options, lzma_allocator *allocator,
+		void **options, const lzma_allocator *allocator,
 		const uint8_t *props, size_t props_size);
 
 #endif

contrib/xz/src/liblzma/delta/delta_encoder.c

@@ -49,7 +49,7 @@ encode_in_place(lzma_coder *coder, uint8_t *buffer, size_t size)
 
 
 static lzma_ret
-delta_encode(lzma_coder *coder, lzma_allocator *allocator,
+delta_encode(lzma_coder *coder, const lzma_allocator *allocator,
 		const uint8_t *restrict in, size_t *restrict in_pos,
 		size_t in_size, uint8_t *restrict out,
 		size_t *restrict out_pos, size_t out_size, lzma_action action)
@@ -84,7 +84,7 @@ delta_encode(lzma_coder *coder, lzma_allocator *allocator,
 
 
 static lzma_ret
-delta_encoder_update(lzma_coder *coder, lzma_allocator *allocator,
+delta_encoder_update(lzma_coder *coder, const lzma_allocator *allocator,
 		const lzma_filter *filters_null lzma_attribute((__unused__)),
 		const lzma_filter *reversed_filters)
 {
@@ -97,7 +97,7 @@ delta_encoder_update(lzma_coder *coder, lzma_allocator *allocator,
 
 
 extern lzma_ret
-lzma_delta_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_delta_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter_info *filters)
 {
 	next->code = &delta_encode;

contrib/xz/src/liblzma/delta/delta_encoder.h

@@ -16,7 +16,8 @@
 #include "delta_common.h"
 
 extern lzma_ret lzma_delta_encoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters);
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters);
 
 extern lzma_ret lzma_delta_props_encode(const void *options, uint8_t *out);
 

contrib/xz/src/liblzma/delta/delta_private.h

@@ -31,7 +31,7 @@ struct lzma_coder_s {
 
 
 extern lzma_ret lzma_delta_coder_init(
-		lzma_next_coder *next, lzma_allocator *allocator,
+		lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter_info *filters);
 
 #endif

contrib/xz/src/liblzma/liblzma.map

@@ -0,0 +1,108 @@
+XZ_5.0 {
+global:
+	lzma_alone_decoder;
+	lzma_alone_encoder;
+	lzma_auto_decoder;
+	lzma_block_buffer_bound;
+	lzma_block_buffer_decode;
+	lzma_block_buffer_encode;
+	lzma_block_compressed_size;
+	lzma_block_decoder;
+	lzma_block_encoder;
+	lzma_block_header_decode;
+	lzma_block_header_encode;
+	lzma_block_header_size;
+	lzma_block_total_size;
+	lzma_block_unpadded_size;
+	lzma_check_is_supported;
+	lzma_check_size;
+	lzma_code;
+	lzma_crc32;
+	lzma_crc64;
+	lzma_easy_buffer_encode;
+	lzma_easy_decoder_memusage;
+	lzma_easy_encoder;
+	lzma_easy_encoder_memusage;
+	lzma_end;
+	lzma_filter_decoder_is_supported;
+	lzma_filter_encoder_is_supported;
+	lzma_filter_flags_decode;
+	lzma_filter_flags_encode;
+	lzma_filter_flags_size;
+	lzma_filters_copy;
+	lzma_filters_update;
+	lzma_get_check;
+	lzma_index_append;
+	lzma_index_block_count;
+	lzma_index_buffer_decode;
+	lzma_index_buffer_encode;
+	lzma_index_cat;
+	lzma_index_checks;
+	lzma_index_decoder;
+	lzma_index_dup;
+	lzma_index_encoder;
+	lzma_index_end;
+	lzma_index_file_size;
+	lzma_index_hash_append;
+	lzma_index_hash_decode;
+	lzma_index_hash_end;
+	lzma_index_hash_init;
+	lzma_index_hash_size;
+	lzma_index_init;
+	lzma_index_iter_init;
+	lzma_index_iter_locate;
+	lzma_index_iter_next;
+	lzma_index_iter_rewind;
+	lzma_index_memusage;
+	lzma_index_memused;
+	lzma_index_size;
+	lzma_index_stream_count;
+	lzma_index_stream_flags;
+	lzma_index_stream_padding;
+	lzma_index_stream_size;
+	lzma_index_total_size;
+	lzma_index_uncompressed_size;
+	lzma_lzma_preset;
+	lzma_memlimit_get;
+	lzma_memlimit_set;
+	lzma_memusage;
+	lzma_mf_is_supported;
+	lzma_mode_is_supported;
+	lzma_physmem;
+	lzma_properties_decode;
+	lzma_properties_encode;
+	lzma_properties_size;
+	lzma_raw_buffer_decode;
+	lzma_raw_buffer_encode;
+	lzma_raw_decoder;
+	lzma_raw_decoder_memusage;
+	lzma_raw_encoder;
+	lzma_raw_encoder_memusage;
+	lzma_stream_buffer_bound;
+	lzma_stream_buffer_decode;
+	lzma_stream_buffer_encode;
+	lzma_stream_decoder;
+	lzma_stream_encoder;
+	lzma_stream_flags_compare;
+	lzma_stream_footer_decode;
+	lzma_stream_footer_encode;
+	lzma_stream_header_decode;
+	lzma_stream_header_encode;
+	lzma_version_number;
+	lzma_version_string;
+	lzma_vli_decode;
+	lzma_vli_encode;
+	lzma_vli_size;
+};
+
+XZ_5.2 {
+global:
+	lzma_block_uncomp_encode;
+	lzma_cputhreads;
+	lzma_get_progress;
+	lzma_stream_encoder_mt;
+	lzma_stream_encoder_mt_memusage;
+
+local:
+	*;
+} XZ_5.0;

contrib/xz/src/liblzma/liblzma.pc.in

@@ -16,4 +16,4 @@ URL: @PACKAGE_URL@
 Version: @PACKAGE_VERSION@
 Cflags: -I${includedir}
 Libs: -L${libdir} -llzma
-Libs.private: @PTHREAD_CFLAGS@ @PTHREAD_LIBS@
+Libs.private: @PTHREAD_CFLAGS@ @LIBS@

contrib/xz/src/liblzma/lz/lz_decoder.c

@@ -126,7 +126,7 @@ decode_buffer(lzma_coder *coder,
 
 static lzma_ret
 lz_decode(lzma_coder *coder,
-		lzma_allocator *allocator lzma_attribute((__unused__)),
+		const lzma_allocator *allocator lzma_attribute((__unused__)),
 		const uint8_t *restrict in, size_t *restrict in_pos,
 		size_t in_size, uint8_t *restrict out,
 		size_t *restrict out_pos, size_t out_size,
@@ -184,7 +184,7 @@ lz_decode(lzma_coder *coder,
 
 
 static void
-lz_decoder_end(lzma_coder *coder, lzma_allocator *allocator)
+lz_decoder_end(lzma_coder *coder, const lzma_allocator *allocator)
 {
 	lzma_next_end(&coder->next, allocator);
 	lzma_free(coder->dict.buf, allocator);
@@ -200,10 +200,10 @@ lz_decoder_end(lzma_coder *coder, lzma_allocator *allocator)
 
 
 extern lzma_ret
-lzma_lz_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_lz_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter_info *filters,
 		lzma_ret (*lz_init)(lzma_lz_decoder *lz,
-			lzma_allocator *allocator, const void *options,
+			const lzma_allocator *allocator, const void *options,
 			lzma_lz_options *lz_options))
 {
 	// Allocate the base structure if it isn't already allocated.

contrib/xz/src/liblzma/lz/lz_decoder.h

@@ -67,7 +67,7 @@ typedef struct {
 			lzma_vli uncompressed_size);
 
 	/// Free allocated resources
-	void (*end)(lzma_coder *coder, lzma_allocator *allocator);
+	void (*end)(lzma_coder *coder, const lzma_allocator *allocator);
 
 } lzma_lz_decoder;
 
@@ -83,9 +83,10 @@ typedef struct {
 
 
 extern lzma_ret lzma_lz_decoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters,
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters,
 		lzma_ret (*lz_init)(lzma_lz_decoder *lz,
-			lzma_allocator *allocator, const void *options,
+			const lzma_allocator *allocator, const void *options,
 			lzma_lz_options *lz_options));
 
 extern uint64_t lzma_lz_decoder_memusage(size_t dictionary_size);

contrib/xz/src/liblzma/lz/lz_encoder.c

@@ -20,6 +20,8 @@
 #	include "lz_encoder_hash_table.h"
 #endif
 
+#include "memcmplen.h"
+
 
 struct lzma_coder_s {
 	/// LZ-based encoder e.g. LZMA
@@ -76,8 +78,9 @@ move_window(lzma_mf *mf)
 /// This function must not be called once it has returned LZMA_STREAM_END.
 ///
 static lzma_ret
-fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in,
-		size_t *in_pos, size_t in_size, lzma_action action)
+fill_window(lzma_coder *coder, const lzma_allocator *allocator,
+		const uint8_t *in, size_t *in_pos, size_t in_size,
+		lzma_action action)
 {
 	assert(coder->mf.read_pos <= coder->mf.write_pos);
 
@@ -148,7 +151,7 @@ fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in,
 
 
 static lzma_ret
-lz_encode(lzma_coder *coder, lzma_allocator *allocator,
+lz_encode(lzma_coder *coder, const lzma_allocator *allocator,
 		const uint8_t *restrict in, size_t *restrict in_pos,
 		size_t in_size,
 		uint8_t *restrict out, size_t *restrict out_pos,
@@ -179,7 +182,7 @@ lz_encode(lzma_coder *coder, lzma_allocator *allocator,
 
 
 static bool
-lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator,
+lz_encoder_prepare(lzma_mf *mf, const lzma_allocator *allocator,
 		const lzma_lz_options *lz_options)
 {
 	// For now, the dictionary size is limited to 1.5 GiB. This may grow
@@ -325,25 +328,22 @@ lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator,
 		hs += HASH_4_SIZE;
 */
 
-	// If the above code calculating hs is modified, make sure that
-	// this assertion stays valid (UINT32_MAX / 5 is not strictly the
-	// exact limit). If it doesn't, you need to calculate that
-	// hash_size_sum + sons_count cannot overflow.
-	assert(hs < UINT32_MAX / 5);
-
-	const uint32_t old_count = mf->hash_size_sum + mf->sons_count;
-	mf->hash_size_sum = hs;
+	const uint32_t old_hash_count = mf->hash_count;
+	const uint32_t old_sons_count = mf->sons_count;
+	mf->hash_count = hs;
 	mf->sons_count = mf->cyclic_size;
 	if (is_bt)
 		mf->sons_count *= 2;
 
-	const uint32_t new_count = mf->hash_size_sum + mf->sons_count;
-
 	// Deallocate the old hash array if it exists and has different size
 	// than what is needed now.
-	if (old_count != new_count) {
+	if (old_hash_count != mf->hash_count
+			|| old_sons_count != mf->sons_count) {
 		lzma_free(mf->hash, allocator);
 		mf->hash = NULL;
+
+		lzma_free(mf->son, allocator);
+		mf->son = NULL;
 	}
 
 	// Maximum number of match finder cycles
@@ -360,14 +360,23 @@ lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator,
 
 
 static bool
-lz_encoder_init(lzma_mf *mf, lzma_allocator *allocator,
+lz_encoder_init(lzma_mf *mf, const lzma_allocator *allocator,
 		const lzma_lz_options *lz_options)
 {
 	// Allocate the history buffer.
 	if (mf->buffer == NULL) {
-		mf->buffer = lzma_alloc(mf->size, allocator);
+		// lzma_memcmplen() is used for the dictionary buffer
+		// so we need to allocate a few extra bytes to prevent
+		// it from reading past the end of the buffer.
+		mf->buffer = lzma_alloc(mf->size + LZMA_MEMCMPLEN_EXTRA,
+				allocator);
 		if (mf->buffer == NULL)
 			return true;
+
+		// Keep Valgrind happy with lzma_memcmplen() and initialize
+		// the extra bytes whose value may get read but which will
+		// effectively get ignored.
+		memzero(mf->buffer + mf->size, LZMA_MEMCMPLEN_EXTRA);
 	}
 
 	// Use cyclic_size as initial mf->offset. This allows
@@ -381,44 +390,49 @@ lz_encoder_init(lzma_mf *mf, lzma_allocator *allocator,
 	mf->write_pos = 0;
 	mf->pending = 0;
 
-	// Allocate match finder's hash array.
-	const size_t alloc_count = mf->hash_size_sum + mf->sons_count;
-
 #if UINT32_MAX >= SIZE_MAX / 4
 	// Check for integer overflow. (Huge dictionaries are not
 	// possible on 32-bit CPU.)
-	if (alloc_count > SIZE_MAX / sizeof(uint32_t))
+	if (mf->hash_count > SIZE_MAX / sizeof(uint32_t)
+			|| mf->sons_count > SIZE_MAX / sizeof(uint32_t))
 		return true;
 #endif
 
+	// Allocate and initialize the hash table. Since EMPTY_HASH_VALUE
+	// is zero, we can use lzma_alloc_zero() or memzero() for mf->hash.
+	//
+	// We don't need to initialize mf->son, but not doing that may
+	// make Valgrind complain in normalization (see normalize() in
+	// lz_encoder_mf.c). Skipping the initialization is *very* good
+	// when big dictionary is used but only small amount of data gets
+	// actually compressed: most of the mf->son won't get actually
+	// allocated by the kernel, so we avoid wasting RAM and improve
+	// initialization speed a lot.
 	if (mf->hash == NULL) {
-		mf->hash = lzma_alloc(alloc_count * sizeof(uint32_t),
+		mf->hash = lzma_alloc_zero(mf->hash_count * sizeof(uint32_t),
 				allocator);
-		if (mf->hash == NULL)
+		mf->son = lzma_alloc(mf->sons_count * sizeof(uint32_t),
+				allocator);
+
+		if (mf->hash == NULL || mf->son == NULL) {
+			lzma_free(mf->hash, allocator);
+			mf->hash = NULL;
+
+			lzma_free(mf->son, allocator);
+			mf->son = NULL;
+
 			return true;
+		}
+	} else {
+/*
+		for (uint32_t i = 0; i < mf->hash_count; ++i)
+			mf->hash[i] = EMPTY_HASH_VALUE;
+*/
+		memzero(mf->hash, mf->hash_count * sizeof(uint32_t));
 	}
 
-	mf->son = mf->hash + mf->hash_size_sum;
 	mf->cyclic_pos = 0;
 
-	// Initialize the hash table. Since EMPTY_HASH_VALUE is zero, we
-	// can use memset().
-/*
-	for (uint32_t i = 0; i < hash_size_sum; ++i)
-		mf->hash[i] = EMPTY_HASH_VALUE;
-*/
-	memzero(mf->hash, (size_t)(mf->hash_size_sum) * sizeof(uint32_t));
-
-	// We don't need to initialize mf->son, but not doing that will
-	// make Valgrind complain in normalization (see normalize() in
-	// lz_encoder_mf.c).
-	//
-	// Skipping this initialization is *very* good when big dictionary is
-	// used but only small amount of data gets actually compressed: most
-	// of the mf->hash won't get actually allocated by the kernel, so
-	// we avoid wasting RAM and improve initialization speed a lot.
-	//memzero(mf->son, (size_t)(mf->sons_count) * sizeof(uint32_t));
-
 	// Handle preset dictionary.
 	if (lz_options->preset_dict != NULL
 			&& lz_options->preset_dict_size > 0) {
@@ -445,7 +459,8 @@ lzma_lz_encoder_memusage(const lzma_lz_options *lz_options)
 	lzma_mf mf = {
 		.buffer = NULL,
 		.hash = NULL,
-		.hash_size_sum = 0,
+		.son = NULL,
+		.hash_count = 0,
 		.sons_count = 0,
 	};
 
@@ -454,17 +469,17 @@ lzma_lz_encoder_memusage(const lzma_lz_options *lz_options)
 		return UINT64_MAX;
 
 	// Calculate the memory usage.
-	return (uint64_t)(mf.hash_size_sum + mf.sons_count)
-				* sizeof(uint32_t)
-			+ (uint64_t)(mf.size) + sizeof(lzma_coder);
+	return ((uint64_t)(mf.hash_count) + mf.sons_count) * sizeof(uint32_t)
+			+ mf.size + sizeof(lzma_coder);
 }
 
 
 static void
-lz_encoder_end(lzma_coder *coder, lzma_allocator *allocator)
+lz_encoder_end(lzma_coder *coder, const lzma_allocator *allocator)
 {
 	lzma_next_end(&coder->next, allocator);
 
+	lzma_free(coder->mf.son, allocator);
 	lzma_free(coder->mf.hash, allocator);
 	lzma_free(coder->mf.buffer, allocator);
 
@@ -479,7 +494,7 @@ lz_encoder_end(lzma_coder *coder, lzma_allocator *allocator)
 
 
 static lzma_ret
-lz_encoder_update(lzma_coder *coder, lzma_allocator *allocator,
+lz_encoder_update(lzma_coder *coder, const lzma_allocator *allocator,
 		const lzma_filter *filters_null lzma_attribute((__unused__)),
 		const lzma_filter *reversed_filters)
 {
@@ -495,10 +510,10 @@ lz_encoder_update(lzma_coder *coder, lzma_allocator *allocator,
 
 
 extern lzma_ret
-lzma_lz_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_lz_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter_info *filters,
 		lzma_ret (*lz_init)(lzma_lz_encoder *lz,
-			lzma_allocator *allocator, const void *options,
+			const lzma_allocator *allocator, const void *options,
 			lzma_lz_options *lz_options))
 {
 #ifdef HAVE_SMALL
@@ -522,7 +537,8 @@ lzma_lz_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
 
 		next->coder->mf.buffer = NULL;
 		next->coder->mf.hash = NULL;
-		next->coder->mf.hash_size_sum = 0;
+		next->coder->mf.son = NULL;
+		next->coder->mf.hash_count = 0;
 		next->coder->mf.sons_count = 0;
 
 		next->coder->next = LZMA_NEXT_CODER_INIT;

contrib/xz/src/liblzma/lz/lz_encoder.h

@@ -119,7 +119,7 @@ struct lzma_mf_s {
 	lzma_action action;
 
 	/// Number of elements in hash[]
-	uint32_t hash_size_sum;
+	uint32_t hash_count;
 
 	/// Number of elements in son[]
 	uint32_t sons_count;
@@ -199,7 +199,7 @@ typedef struct {
 			size_t *restrict out_pos, size_t out_size);
 
 	/// Free allocated resources
-	void (*end)(lzma_coder *coder, lzma_allocator *allocator);
+	void (*end)(lzma_coder *coder, const lzma_allocator *allocator);
 
 	/// Update the options in the middle of the encoding.
 	lzma_ret (*options_update)(lzma_coder *coder,
@@ -296,10 +296,10 @@ mf_read(lzma_mf *mf, uint8_t *out, size_t *out_pos, size_t out_size,
 
 
 extern lzma_ret lzma_lz_encoder_init(
-		lzma_next_coder *next, lzma_allocator *allocator,
+		lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter_info *filters,
 		lzma_ret (*lz_init)(lzma_lz_encoder *lz,
-			lzma_allocator *allocator, const void *options,
+			const lzma_allocator *allocator, const void *options,
 			lzma_lz_options *lz_options));
 
 

contrib/xz/src/liblzma/lz/lz_encoder_mf.c

@@ -13,6 +13,7 @@
 
 #include "lz_encoder.h"
 #include "lz_encoder_hash.h"
+#include "memcmplen.h"
 
 
 /// \brief      Find matches starting from the current byte
@@ -65,9 +66,7 @@ lzma_mf_find(lzma_mf *mf, uint32_t *count_ptr, lzma_match *matches)
 			// here because the match distances are zero based.
 			const uint8_t *p2 = p1 - matches[count - 1].dist - 1;
 
-			while (len_best < limit
-					&& p1[len_best] == p2[len_best])
-				++len_best;
+			len_best = lzma_memcmplen(p1, p2, len_best, limit);
 		}
 	}
 
@@ -116,24 +115,27 @@ normalize(lzma_mf *mf)
 			= (MUST_NORMALIZE_POS - mf->cyclic_size);
 				// & (~(UINT32_C(1) << 10) - 1);
 
-	const uint32_t count = mf->hash_size_sum + mf->sons_count;
-	uint32_t *hash = mf->hash;
-
-	for (uint32_t i = 0; i < count; ++i) {
+	for (uint32_t i = 0; i < mf->hash_count; ++i) {
 		// If the distance is greater than the dictionary size,
 		// we can simply mark the hash element as empty.
-		//
-		// NOTE: Only the first mf->hash_size_sum elements are
-		// initialized for sure. There may be uninitialized elements
-		// in mf->son. Since we go through both mf->hash and
-		// mf->son here in normalization, Valgrind may complain
-		// that the "if" below depends on uninitialized value. In
-		// this case it is safe to ignore the warning. See also the
-		// comments in lz_encoder_init() in lz_encoder.c.
-		if (hash[i] <= subvalue)
-			hash[i] = EMPTY_HASH_VALUE;
+		if (mf->hash[i] <= subvalue)
+			mf->hash[i] = EMPTY_HASH_VALUE;
 		else
-			hash[i] -= subvalue;
+			mf->hash[i] -= subvalue;
+	}
+
+	for (uint32_t i = 0; i < mf->sons_count; ++i) {
+		// Do the same for mf->son.
+		//
+		// NOTE: There may be uninitialized elements in mf->son.
+		// Valgrind may complain that the "if" below depends on
+		// an uninitialized value. In this case it is safe to ignore
+		// the warning. See also the comments in lz_encoder_init()
+		// in lz_encoder.c.
+		if (mf->son[i] <= subvalue)
+			mf->son[i] = EMPTY_HASH_VALUE;
+		else
+			mf->son[i] -= subvalue;
 	}
 
 	// Update offset to match the new locations.
@@ -269,10 +271,7 @@ hc_find_func(
 				+ (delta > cyclic_pos ? cyclic_size : 0)];
 
 		if (pb[len_best] == cur[len_best] && pb[0] == cur[0]) {
-			uint32_t len = 0;
-			while (++len != len_limit)
-				if (pb[len] != cur[len])
-					break;
+			uint32_t len = lzma_memcmplen(pb, cur, 1, len_limit);
 
 			if (len_best < len) {
 				len_best = len;
@@ -318,9 +317,8 @@ lzma_mf_hc3_find(lzma_mf *mf, lzma_match *matches)
 	uint32_t len_best = 2;
 
 	if (delta2 < mf->cyclic_size && *(cur - delta2) == *cur) {
-		for ( ; len_best != len_limit; ++len_best)
-			if (*(cur + len_best - delta2) != cur[len_best])
-				break;
+		len_best = lzma_memcmplen(cur - delta2, cur,
+				len_best, len_limit);
 
 		matches[0].len = len_best;
 		matches[0].dist = delta2 - 1;
@@ -397,9 +395,8 @@ lzma_mf_hc4_find(lzma_mf *mf, lzma_match *matches)
 	}
 
 	if (matches_count != 0) {
-		for ( ; len_best != len_limit; ++len_best)
-			if (*(cur + len_best - delta2) != cur[len_best])
-				break;
+		len_best = lzma_memcmplen(cur - delta2, cur,
+				len_best, len_limit);
 
 		matches[matches_count - 1].len = len_best;
 
@@ -484,9 +481,7 @@ bt_find_func(
 		uint32_t len = my_min(len0, len1);
 
 		if (pb[len] == cur[len]) {
-			while (++len != len_limit)
-				if (pb[len] != cur[len])
-					break;
+			len = lzma_memcmplen(pb, cur, len + 1, len_limit);
 
 			if (len_best < len) {
 				len_best = len;
@@ -549,9 +544,7 @@ bt_skip_func(
 		uint32_t len = my_min(len0, len1);
 
 		if (pb[len] == cur[len]) {
-			while (++len != len_limit)
-				if (pb[len] != cur[len])
-					break;
+			len = lzma_memcmplen(pb, cur, len + 1, len_limit);
 
 			if (len == len_limit) {
 				*ptr1 = pair[0];
@@ -639,9 +632,8 @@ lzma_mf_bt3_find(lzma_mf *mf, lzma_match *matches)
 	uint32_t len_best = 2;
 
 	if (delta2 < mf->cyclic_size && *(cur - delta2) == *cur) {
-		for ( ; len_best != len_limit; ++len_best)
-			if (*(cur + len_best - delta2) != cur[len_best])
-				break;
+		len_best = lzma_memcmplen(
+				cur, cur - delta2, len_best, len_limit);
 
 		matches[0].len = len_best;
 		matches[0].dist = delta2 - 1;
@@ -712,9 +704,8 @@ lzma_mf_bt4_find(lzma_mf *mf, lzma_match *matches)
 	}
 
 	if (matches_count != 0) {
-		for ( ; len_best != len_limit; ++len_best)
-			if (*(cur + len_best - delta2) != cur[len_best])
-				break;
+		len_best = lzma_memcmplen(
+				cur, cur - delta2, len_best, len_limit);
 
 		matches[matches_count - 1].len = len_best;
 

contrib/xz/src/liblzma/lzma/fastpos.h

@@ -14,15 +14,15 @@
 #ifndef LZMA_FASTPOS_H
 #define LZMA_FASTPOS_H
 
-// LZMA encodes match distances (positions) by storing the highest two
-// bits using a six-bit value [0, 63], and then the missing lower bits.
-// Dictionary size is also stored using this encoding in the new .lzma
+// LZMA encodes match distances by storing the highest two bits using
+// a six-bit value [0, 63], and then the missing lower bits.
+// Dictionary size is also stored using this encoding in the .xz
 // file format header.
 //
 // fastpos.h provides a way to quickly find out the correct six-bit
 // values. The following table gives some examples of this encoding:
 //
-//      pos   return
+//     dist   return
 //       0       0
 //       1       1
 //       2       2
@@ -48,10 +48,10 @@
 // Provided functions or macros
 // ----------------------------
 //
-// get_pos_slot(pos) is the basic version. get_pos_slot_2(pos)
-// assumes that pos >= FULL_DISTANCES, thus the result is at least
-// FULL_DISTANCES_BITS * 2. Using get_pos_slot(pos) instead of
-// get_pos_slot_2(pos) would give the same result, but get_pos_slot_2(pos)
+// get_dist_slot(dist) is the basic version. get_dist_slot_2(dist)
+// assumes that dist >= FULL_DISTANCES, thus the result is at least
+// FULL_DISTANCES_BITS * 2. Using get_dist_slot(dist) instead of
+// get_dist_slot_2(dist) would give the same result, but get_dist_slot_2(dist)
 // should be tiny bit faster due to the assumption being made.
 //
 //
@@ -76,13 +76,14 @@
 // slightly faster, but sometimes it is a lot slower.
 
 #ifdef HAVE_SMALL
-#	define get_pos_slot(pos) ((pos) <= 4 ? (pos) : get_pos_slot_2(pos))
+#	define get_dist_slot(dist) \
+		((dist) <= 4 ? (dist) : get_dist_slot_2(dist))
 
 static inline uint32_t
-get_pos_slot_2(uint32_t pos)
+get_dist_slot_2(uint32_t dist)
 {
-	const uint32_t i = bsr32(pos);
-	return (i + i) + ((pos >> (i - 1)) & 1);
+	const uint32_t i = bsr32(dist);
+	return (i + i) + ((dist >> (i - 1)) & 1);
 }
 
 
@@ -99,39 +100,39 @@ extern const uint8_t lzma_fastpos[1 << FASTPOS_BITS];
 #define fastpos_limit(extra, n) \
 	(UINT32_C(1) << (FASTPOS_BITS + fastpos_shift(extra, n)))
 
-#define fastpos_result(pos, extra, n) \
-	lzma_fastpos[(pos) >> fastpos_shift(extra, n)] \
+#define fastpos_result(dist, extra, n) \
+	lzma_fastpos[(dist) >> fastpos_shift(extra, n)] \
 			+ 2 * fastpos_shift(extra, n)
 
 
 static inline uint32_t
-get_pos_slot(uint32_t pos)
+get_dist_slot(uint32_t dist)
 {
 	// If it is small enough, we can pick the result directly from
 	// the precalculated table.
-	if (pos < fastpos_limit(0, 0))
-		return lzma_fastpos[pos];
+	if (dist < fastpos_limit(0, 0))
+		return lzma_fastpos[dist];
 
-	if (pos < fastpos_limit(0, 1))
-		return fastpos_result(pos, 0, 1);
+	if (dist < fastpos_limit(0, 1))
+		return fastpos_result(dist, 0, 1);
 
-	return fastpos_result(pos, 0, 2);
+	return fastpos_result(dist, 0, 2);
 }
 
 
 #ifdef FULL_DISTANCES_BITS
 static inline uint32_t
-get_pos_slot_2(uint32_t pos)
+get_dist_slot_2(uint32_t dist)
 {
-	assert(pos >= FULL_DISTANCES);
+	assert(dist >= FULL_DISTANCES);
 
-	if (pos < fastpos_limit(FULL_DISTANCES_BITS - 1, 0))
-		return fastpos_result(pos, FULL_DISTANCES_BITS - 1, 0);
+	if (dist < fastpos_limit(FULL_DISTANCES_BITS - 1, 0))
+		return fastpos_result(dist, FULL_DISTANCES_BITS - 1, 0);
 
-	if (pos < fastpos_limit(FULL_DISTANCES_BITS - 1, 1))
-		return fastpos_result(pos, FULL_DISTANCES_BITS - 1, 1);
+	if (dist < fastpos_limit(FULL_DISTANCES_BITS - 1, 1))
+		return fastpos_result(dist, FULL_DISTANCES_BITS - 1, 1);
 
-	return fastpos_result(pos, FULL_DISTANCES_BITS - 1, 2);
+	return fastpos_result(dist, FULL_DISTANCES_BITS - 1, 2);
 }
 #endif
 

contrib/xz/src/liblzma/lzma/lzma2_decoder.c

@@ -209,7 +209,7 @@ lzma2_decode(lzma_coder *restrict coder, lzma_dict *restrict dict,
 
 
 static void
-lzma2_decoder_end(lzma_coder *coder, lzma_allocator *allocator)
+lzma2_decoder_end(lzma_coder *coder, const lzma_allocator *allocator)
 {
 	assert(coder->lzma.end == NULL);
 	lzma_free(coder->lzma.coder, allocator);
@@ -221,7 +221,7 @@ lzma2_decoder_end(lzma_coder *coder, lzma_allocator *allocator)
 
 
 static lzma_ret
-lzma2_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator,
+lzma2_decoder_init(lzma_lz_decoder *lz, const lzma_allocator *allocator,
 		const void *opt, lzma_lz_options *lz_options)
 {
 	if (lz->coder == NULL) {
@@ -248,7 +248,7 @@ lzma2_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator,
 
 
 extern lzma_ret
-lzma_lzma2_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_lzma2_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter_info *filters)
 {
 	// LZMA2 can only be the last filter in the chain. This is enforced
@@ -269,7 +269,7 @@ lzma_lzma2_decoder_memusage(const void *options)
 
 
 extern lzma_ret
-lzma_lzma2_props_decode(void **options, lzma_allocator *allocator,
+lzma_lzma2_props_decode(void **options, const lzma_allocator *allocator,
 		const uint8_t *props, size_t props_size)
 {
 	if (props_size != 1)

contrib/xz/src/liblzma/lzma/lzma2_decoder.h

@@ -17,12 +17,13 @@
 #include "common.h"
 
 extern lzma_ret lzma_lzma2_decoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters);
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters);
 
 extern uint64_t lzma_lzma2_decoder_memusage(const void *options);
 
 extern lzma_ret lzma_lzma2_props_decode(
-		void **options, lzma_allocator *allocator,
+		void **options, const lzma_allocator *allocator,
 		const uint8_t *props, size_t props_size);
 
 #endif

contrib/xz/src/liblzma/lzma/lzma2_encoder.c

@@ -262,7 +262,7 @@ lzma2_encode(lzma_coder *restrict coder, lzma_mf *restrict mf,
 
 
 static void
-lzma2_encoder_end(lzma_coder *coder, lzma_allocator *allocator)
+lzma2_encoder_end(lzma_coder *coder, const lzma_allocator *allocator)
 {
 	lzma_free(coder->lzma, allocator);
 	lzma_free(coder, allocator);
@@ -304,7 +304,7 @@ lzma2_encoder_options_update(lzma_coder *coder, const lzma_filter *filter)
 
 
 static lzma_ret
-lzma2_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator,
+lzma2_encoder_init(lzma_lz_encoder *lz, const lzma_allocator *allocator,
 		const void *options, lzma_lz_options *lz_options)
 {
 	if (options == NULL)
@@ -349,7 +349,7 @@ lzma2_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator,
 
 
 extern lzma_ret
-lzma_lzma2_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_lzma2_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter_info *filters)
 {
 	return lzma_lz_encoder_init(
@@ -387,7 +387,17 @@ lzma_lzma2_props_encode(const void *options, uint8_t *out)
 	if (d == UINT32_MAX)
 		out[0] = 40;
 	else
-		out[0] = get_pos_slot(d + 1) - 24;
+		out[0] = get_dist_slot(d + 1) - 24;
 
 	return LZMA_OK;
 }
+
+
+extern uint64_t
+lzma_lzma2_block_size(const void *options)
+{
+	const lzma_options_lzma *const opt = options;
+
+	// Use at least 1 MiB to keep compression ratio better.
+	return my_max((uint64_t)(opt->dict_size) * 3, UINT64_C(1) << 20);
+}

contrib/xz/src/liblzma/lzma/lzma2_encoder.h

@@ -31,11 +31,13 @@
 
 
 extern lzma_ret lzma_lzma2_encoder_init(
-		lzma_next_coder *next, lzma_allocator *allocator,
+		lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter_info *filters);
 
 extern uint64_t lzma_lzma2_encoder_memusage(const void *options);
 
 extern lzma_ret lzma_lzma2_props_encode(const void *options, uint8_t *out);
 
+extern uint64_t lzma_lzma2_block_size(const void *options);
+
 #endif

contrib/xz/src/liblzma/lzma/lzma_common.h

@@ -171,53 +171,54 @@ literal_init(probability (*probs)[LITERAL_CODER_SIZE],
 // Match distance //
 ////////////////////
 
-// Different set of probabilities is used for match distances that have very
+// Different sets of probabilities are used for match distances that have very
 // short match length: Lengths of 2, 3, and 4 bytes have a separate set of
 // probabilities for each length. The matches with longer length use a shared
 // set of probabilities.
-#define LEN_TO_POS_STATES 4
+#define DIST_STATES 4
 
 // Macro to get the index of the appropriate probability array.
-#define get_len_to_pos_state(len) \
-	((len) < LEN_TO_POS_STATES + MATCH_LEN_MIN \
+#define get_dist_state(len) \
+	((len) < DIST_STATES + MATCH_LEN_MIN \
 		? (len) - MATCH_LEN_MIN \
-		: LEN_TO_POS_STATES - 1)
+		: DIST_STATES - 1)
 
-// The highest two bits of a match distance (pos slot) are encoded using six
-// bits. See fastpos.h for more explanation.
-#define POS_SLOT_BITS 6
-#define POS_SLOTS (1 << POS_SLOT_BITS)
+// The highest two bits of a match distance (distance slot) are encoded
+// using six bits. See fastpos.h for more explanation.
+#define DIST_SLOT_BITS 6
+#define DIST_SLOTS (1 << DIST_SLOT_BITS)
 
 // Match distances up to 127 are fully encoded using probabilities. Since
-// the highest two bits (pos slot) are always encoded using six bits, the
-// distances 0-3 don't need any additional bits to encode, since the pos
-// slot itself is the same as the actual distance. START_POS_MODEL_INDEX
-// indicates the first pos slot where at least one additional bit is needed.
-#define START_POS_MODEL_INDEX 4
+// the highest two bits (distance slot) are always encoded using six bits,
+// the distances 0-3 don't need any additional bits to encode, since the
+// distance slot itself is the same as the actual distance. DIST_MODEL_START
+// indicates the first distance slot where at least one additional bit is
+// needed.
+#define DIST_MODEL_START 4
 
 // Match distances greater than 127 are encoded in three pieces:
-//   - pos slot: the highest two bits
+//   - distance slot: the highest two bits
 //   - direct bits: 2-26 bits below the highest two bits
 //   - alignment bits: four lowest bits
 //
 // Direct bits don't use any probabilities.
 //
-// The pos slot value of 14 is for distances 128-191 (see the table in
+// The distance slot value of 14 is for distances 128-191 (see the table in
 // fastpos.h to understand why).
-#define END_POS_MODEL_INDEX 14
+#define DIST_MODEL_END 14
 
-// Pos slots that indicate a distance <= 127.
-#define FULL_DISTANCES_BITS (END_POS_MODEL_INDEX / 2)
+// Distance slots that indicate a distance <= 127.
+#define FULL_DISTANCES_BITS (DIST_MODEL_END / 2)
 #define FULL_DISTANCES (1 << FULL_DISTANCES_BITS)
 
 // For match distances greater than 127, only the highest two bits and the
 // lowest four bits (alignment) is encoded using probabilities.
 #define ALIGN_BITS 4
-#define ALIGN_TABLE_SIZE (1 << ALIGN_BITS)
-#define ALIGN_MASK (ALIGN_TABLE_SIZE - 1)
+#define ALIGN_SIZE (1 << ALIGN_BITS)
+#define ALIGN_MASK (ALIGN_SIZE - 1)
 
 // LZMA remembers the four most recent match distances. Reusing these distances
 // tends to take less space than re-encoding the actual distance value.
-#define REP_DISTANCES 4
+#define REPS 4
 
 #endif

contrib/xz/src/liblzma/lzma/lzma_decoder.c

@@ -193,15 +193,15 @@ struct lzma_coder_s {
 	/// Probability tree for the highest two bits of the match distance.
 	/// There is a separate probability tree for match lengths of
 	/// 2 (i.e. MATCH_LEN_MIN), 3, 4, and [5, 273].
-	probability pos_slot[LEN_TO_POS_STATES][POS_SLOTS];
+	probability dist_slot[DIST_STATES][DIST_SLOTS];
 
 	/// Probability trees for additional bits for match distance when the
 	/// distance is in the range [4, 127].
-	probability pos_special[FULL_DISTANCES - END_POS_MODEL_INDEX];
+	probability pos_special[FULL_DISTANCES - DIST_MODEL_END];
 
 	/// Probability tree for the lowest four bits of a match distance
 	/// that is equal to or greater than 128.
-	probability pos_align[ALIGN_TABLE_SIZE];
+	probability pos_align[ALIGN_SIZE];
 
 	/// Length of a normal match
 	lzma_length_decoder match_len_decoder;
@@ -245,8 +245,8 @@ struct lzma_coder_s {
 		SEQ_LITERAL_WRITE,
 		SEQ_IS_REP,
 		seq_len(SEQ_MATCH_LEN),
-		seq_6(SEQ_POS_SLOT),
-		SEQ_POS_MODEL,
+		seq_6(SEQ_DIST_SLOT),
+		SEQ_DIST_MODEL,
 		SEQ_DIRECT,
 		seq_4(SEQ_ALIGN),
 		SEQ_EOPM,
@@ -289,8 +289,12 @@ lzma_decode(lzma_coder *restrict coder, lzma_dict *restrict dictptr,
 	// Initialization //
 	////////////////////
 
-	if (!rc_read_init(&coder->rc, in, in_pos, in_size))
-		return LZMA_OK;
+	{
+		const lzma_ret ret = rc_read_init(
+				&coder->rc, in, in_pos, in_size);
+		if (ret != LZMA_STREAM_END)
+			return ret;
+	}
 
 	///////////////
 	// Variables //
@@ -502,28 +506,28 @@ lzma_decode(lzma_coder *restrict coder, lzma_dict *restrict dictptr,
 
 			// Prepare to decode the highest two bits of the
 			// match distance.
-			probs = coder->pos_slot[get_len_to_pos_state(len)];
+			probs = coder->dist_slot[get_dist_state(len)];
 			symbol = 1;
 
 #ifdef HAVE_SMALL
-	case SEQ_POS_SLOT:
+	case SEQ_DIST_SLOT:
 			do {
-				rc_bit(probs[symbol], , , SEQ_POS_SLOT);
-			} while (symbol < POS_SLOTS);
+				rc_bit(probs[symbol], , , SEQ_DIST_SLOT);
+			} while (symbol < DIST_SLOTS);
 #else
-			rc_bit_case(probs[symbol], , , SEQ_POS_SLOT0);
-			rc_bit_case(probs[symbol], , , SEQ_POS_SLOT1);
-			rc_bit_case(probs[symbol], , , SEQ_POS_SLOT2);
-			rc_bit_case(probs[symbol], , , SEQ_POS_SLOT3);
-			rc_bit_case(probs[symbol], , , SEQ_POS_SLOT4);
-			rc_bit_case(probs[symbol], , , SEQ_POS_SLOT5);
+			rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT0);
+			rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT1);
+			rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT2);
+			rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT3);
+			rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT4);
+			rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT5);
 #endif
 			// Get rid of the highest bit that was needed for
 			// indexing of the probability array.
-			symbol -= POS_SLOTS;
+			symbol -= DIST_SLOTS;
 			assert(symbol <= 63);
 
-			if (symbol < START_POS_MODEL_INDEX) {
+			if (symbol < DIST_MODEL_START) {
 				// Match distances [0, 3] have only two bits.
 				rep0 = symbol;
 			} else {
@@ -533,7 +537,7 @@ lzma_decode(lzma_coder *restrict coder, lzma_dict *restrict dictptr,
 				assert(limit >= 1 && limit <= 30);
 				rep0 = 2 + (symbol & 1);
 
-				if (symbol < END_POS_MODEL_INDEX) {
+				if (symbol < DIST_MODEL_END) {
 					// Prepare to decode the low bits for
 					// a distance of [4, 127].
 					assert(limit <= 5);
@@ -553,12 +557,12 @@ lzma_decode(lzma_coder *restrict coder, lzma_dict *restrict dictptr,
 							- symbol - 1;
 					symbol = 1;
 					offset = 0;
-	case SEQ_POS_MODEL:
+	case SEQ_DIST_MODEL:
 #ifdef HAVE_SMALL
 					do {
 						rc_bit(probs[symbol], ,
 							rep0 += 1 << offset,
-							SEQ_POS_MODEL);
+							SEQ_DIST_MODEL);
 					} while (++offset < limit);
 #else
 					switch (limit) {
@@ -566,25 +570,25 @@ lzma_decode(lzma_coder *restrict coder, lzma_dict *restrict dictptr,
 						assert(offset == 0);
 						rc_bit(probs[symbol], ,
 							rep0 += 1,
-							SEQ_POS_MODEL);
+							SEQ_DIST_MODEL);
 						++offset;
 						--limit;
 					case 4:
 						rc_bit(probs[symbol], ,
 							rep0 += 1 << offset,
-							SEQ_POS_MODEL);
+							SEQ_DIST_MODEL);
 						++offset;
 						--limit;
 					case 3:
 						rc_bit(probs[symbol], ,
 							rep0 += 1 << offset,
-							SEQ_POS_MODEL);
+							SEQ_DIST_MODEL);
 						++offset;
 						--limit;
 					case 2:
 						rc_bit(probs[symbol], ,
 							rep0 += 1 << offset,
-							SEQ_POS_MODEL);
+							SEQ_DIST_MODEL);
 						++offset;
 						--limit;
 					case 1:
@@ -596,7 +600,7 @@ lzma_decode(lzma_coder *restrict coder, lzma_dict *restrict dictptr,
 						// "symbol".
 						rc_bit_last(probs[symbol], ,
 							rep0 += 1 << offset,
-							SEQ_POS_MODEL);
+							SEQ_DIST_MODEL);
 					}
 #endif
 				} else {
@@ -637,7 +641,7 @@ lzma_decode(lzma_coder *restrict coder, lzma_dict *restrict dictptr,
 					rc_bit(coder->pos_align[symbol], ,
 							rep0 += 4, SEQ_ALIGN2);
 	case SEQ_ALIGN3:
-					// Like in SEQ_POS_MODEL, we don't
+					// Like in SEQ_DIST_MODEL, we don't
 					// need "symbol" for anything else
 					// than indexing the probability array.
 					rc_bit_last(coder->pos_align[symbol], ,
@@ -891,10 +895,10 @@ lzma_decoder_reset(lzma_coder *coder, const void *opt)
 		bit_reset(coder->is_rep2[i]);
 	}
 
-	for (uint32_t i = 0; i < LEN_TO_POS_STATES; ++i)
-		bittree_reset(coder->pos_slot[i], POS_SLOT_BITS);
+	for (uint32_t i = 0; i < DIST_STATES; ++i)
+		bittree_reset(coder->dist_slot[i], DIST_SLOT_BITS);
 
-	for (uint32_t i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i)
+	for (uint32_t i = 0; i < FULL_DISTANCES - DIST_MODEL_END; ++i)
 		bit_reset(coder->pos_special[i]);
 
 	bittree_reset(coder->pos_align, ALIGN_BITS);
@@ -933,7 +937,7 @@ lzma_decoder_reset(lzma_coder *coder, const void *opt)
 
 
 extern lzma_ret
-lzma_lzma_decoder_create(lzma_lz_decoder *lz, lzma_allocator *allocator,
+lzma_lzma_decoder_create(lzma_lz_decoder *lz, const lzma_allocator *allocator,
 		const void *opt, lzma_lz_options *lz_options)
 {
 	if (lz->coder == NULL) {
@@ -961,7 +965,7 @@ lzma_lzma_decoder_create(lzma_lz_decoder *lz, lzma_allocator *allocator,
 /// initialization (lzma_lzma_decoder_init() passes function pointer to
 /// the LZ initialization).
 static lzma_ret
-lzma_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator,
+lzma_decoder_init(lzma_lz_decoder *lz, const lzma_allocator *allocator,
 		const void *options, lzma_lz_options *lz_options)
 {
 	if (!is_lclppb_valid(options))
@@ -978,7 +982,7 @@ lzma_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator,
 
 
 extern lzma_ret
-lzma_lzma_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_lzma_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter_info *filters)
 {
 	// LZMA can only be the last filter in the chain. This is enforced
@@ -1025,7 +1029,7 @@ lzma_lzma_decoder_memusage(const void *options)
 
 
 extern lzma_ret
-lzma_lzma_props_decode(void **options, lzma_allocator *allocator,
+lzma_lzma_props_decode(void **options, const lzma_allocator *allocator,
 		const uint8_t *props, size_t props_size)
 {
 	if (props_size != 5)

contrib/xz/src/liblzma/lzma/lzma_decoder.h

@@ -19,12 +19,13 @@
 
 /// Allocates and initializes LZMA decoder
 extern lzma_ret lzma_lzma_decoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters);
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters);
 
 extern uint64_t lzma_lzma_decoder_memusage(const void *options);
 
 extern lzma_ret lzma_lzma_props_decode(
-		void **options, lzma_allocator *allocator,
+		void **options, const lzma_allocator *allocator,
 		const uint8_t *props, size_t props_size);
 
 
@@ -40,7 +41,7 @@ extern bool lzma_lzma_lclppb_decode(
 /// Allocate and setup function pointers only. This is used by LZMA1 and
 /// LZMA2 decoders.
 extern lzma_ret lzma_lzma_decoder_create(
-		lzma_lz_decoder *lz, lzma_allocator *allocator,
+		lzma_lz_decoder *lz, const lzma_allocator *allocator,
 		const void *opt, lzma_lz_options *lz_options);
 
 /// Gets memory usage without validating lc/lp/pb. This is used by LZMA2

contrib/xz/src/liblzma/lzma/lzma_encoder.c

@@ -148,28 +148,28 @@ match(lzma_coder *coder, const uint32_t pos_state,
 	length(&coder->rc, &coder->match_len_encoder, pos_state, len,
 			coder->fast_mode);
 
-	const uint32_t pos_slot = get_pos_slot(distance);
-	const uint32_t len_to_pos_state = get_len_to_pos_state(len);
-	rc_bittree(&coder->rc, coder->pos_slot[len_to_pos_state],
-			POS_SLOT_BITS, pos_slot);
+	const uint32_t dist_slot = get_dist_slot(distance);
+	const uint32_t dist_state = get_dist_state(len);
+	rc_bittree(&coder->rc, coder->dist_slot[dist_state],
+			DIST_SLOT_BITS, dist_slot);
 
-	if (pos_slot >= START_POS_MODEL_INDEX) {
-		const uint32_t footer_bits = (pos_slot >> 1) - 1;
-		const uint32_t base = (2 | (pos_slot & 1)) << footer_bits;
-		const uint32_t pos_reduced = distance - base;
+	if (dist_slot >= DIST_MODEL_START) {
+		const uint32_t footer_bits = (dist_slot >> 1) - 1;
+		const uint32_t base = (2 | (dist_slot & 1)) << footer_bits;
+		const uint32_t dist_reduced = distance - base;
 
-		if (pos_slot < END_POS_MODEL_INDEX) {
-			// Careful here: base - pos_slot - 1 can be -1, but
+		if (dist_slot < DIST_MODEL_END) {
+			// Careful here: base - dist_slot - 1 can be -1, but
 			// rc_bittree_reverse starts at probs[1], not probs[0].
 			rc_bittree_reverse(&coder->rc,
-				coder->pos_special + base - pos_slot - 1,
-				footer_bits, pos_reduced);
+				coder->dist_special + base - dist_slot - 1,
+				footer_bits, dist_reduced);
 		} else {
-			rc_direct(&coder->rc, pos_reduced >> ALIGN_BITS,
+			rc_direct(&coder->rc, dist_reduced >> ALIGN_BITS,
 					footer_bits - ALIGN_BITS);
 			rc_bittree_reverse(
-					&coder->rc, coder->pos_align,
-					ALIGN_BITS, pos_reduced & ALIGN_MASK);
+					&coder->rc, coder->dist_align,
+					ALIGN_BITS, dist_reduced & ALIGN_MASK);
 			++coder->align_price_count;
 		}
 	}
@@ -247,7 +247,7 @@ encode_symbol(lzma_coder *coder, lzma_mf *mf,
 		rc_bit(&coder->rc,
 			&coder->is_match[coder->state][pos_state], 1);
 
-		if (back < REP_DISTANCES) {
+		if (back < REPS) {
 			// It's a repeated match i.e. the same distance
 			// has been used earlier.
 			rc_bit(&coder->rc, &coder->is_rep[coder->state], 1);
@@ -255,7 +255,7 @@ encode_symbol(lzma_coder *coder, lzma_mf *mf,
 		} else {
 			// Normal match
 			rc_bit(&coder->rc, &coder->is_rep[coder->state], 0);
-			match(coder, pos_state, back - REP_DISTANCES, len);
+			match(coder, pos_state, back - REPS, len);
 		}
 	}
 
@@ -353,9 +353,9 @@ lzma_lzma_encode(lzma_coder *restrict coder, lzma_mf *restrict mf,
 
 		// Get optimal match (repeat position and length).
 		// Value ranges for pos:
-		//   - [0, REP_DISTANCES): repeated match
-		//   - [REP_DISTANCES, UINT32_MAX):
-		//     match at (pos - REP_DISTANCES)
+		//   - [0, REPS): repeated match
+		//   - [REPS, UINT32_MAX):
+		//     match at (pos - REPS)
 		//   - UINT32_MAX: not a match but a literal
 		// Value ranges for len:
 		//   - [MATCH_LEN_MIN, MATCH_LEN_MAX]
@@ -487,7 +487,7 @@ lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options)
 
 	// State
 	coder->state = STATE_LIT_LIT;
-	for (size_t i = 0; i < REP_DISTANCES; ++i)
+	for (size_t i = 0; i < REPS; ++i)
 		coder->reps[i] = 0;
 
 	literal_init(coder->literal, options->lc, options->lp);
@@ -505,14 +505,14 @@ lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options)
 		bit_reset(coder->is_rep2[i]);
 	}
 
-	for (size_t i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i)
-		bit_reset(coder->pos_special[i]);
+	for (size_t i = 0; i < FULL_DISTANCES - DIST_MODEL_END; ++i)
+		bit_reset(coder->dist_special[i]);
 
 	// Bit tree encoders
-	for (size_t i = 0; i < LEN_TO_POS_STATES; ++i)
-		bittree_reset(coder->pos_slot[i], POS_SLOT_BITS);
+	for (size_t i = 0; i < DIST_STATES; ++i)
+		bittree_reset(coder->dist_slot[i], DIST_SLOT_BITS);
 
-	bittree_reset(coder->pos_align, ALIGN_BITS);
+	bittree_reset(coder->dist_align, ALIGN_BITS);
 
 	// Length encoders
 	length_encoder_reset(&coder->match_len_encoder,
@@ -545,7 +545,8 @@ lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options)
 
 
 extern lzma_ret
-lzma_lzma_encoder_create(lzma_coder **coder_ptr, lzma_allocator *allocator,
+lzma_lzma_encoder_create(lzma_coder **coder_ptr,
+		const lzma_allocator *allocator,
 		const lzma_options_lzma *options, lzma_lz_options *lz_options)
 {
 	// Allocate lzma_coder if it wasn't already allocated.
@@ -604,7 +605,7 @@ lzma_lzma_encoder_create(lzma_coder **coder_ptr, lzma_allocator *allocator,
 
 
 static lzma_ret
-lzma_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator,
+lzma_encoder_init(lzma_lz_encoder *lz, const lzma_allocator *allocator,
 		const void *options, lzma_lz_options *lz_options)
 {
 	lz->code = &lzma_encode;
@@ -614,7 +615,7 @@ lzma_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator,
 
 
 extern lzma_ret
-lzma_lzma_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_lzma_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter_info *filters)
 {
 	return lzma_lz_encoder_init(

contrib/xz/src/liblzma/lzma/lzma_encoder.h

@@ -18,7 +18,8 @@
 
 
 extern lzma_ret lzma_lzma_encoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters);
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters);
 
 
 extern uint64_t lzma_lzma_encoder_memusage(const void *options);
@@ -35,7 +36,7 @@ extern bool lzma_lzma_lclppb_encode(
 
 /// Initializes raw LZMA encoder; this is used by LZMA2.
 extern lzma_ret lzma_lzma_encoder_create(
-		lzma_coder **coder_ptr, lzma_allocator *allocator,
+		lzma_coder **coder_ptr, const lzma_allocator *allocator,
 		const lzma_options_lzma *options, lzma_lz_options *lz_options);
 
 

contrib/xz/src/liblzma/lzma/lzma_encoder_optimum_fast.c

@@ -10,6 +10,7 @@
 ///////////////////////////////////////////////////////////////////////////////
 
 #include "lzma_encoder_private.h"
+#include "memcmplen.h"
 
 
 #define change_pair(small_dist, big_dist) \
@@ -46,7 +47,7 @@ lzma_lzma_optimum_fast(lzma_coder *restrict coder, lzma_mf *restrict mf,
 	uint32_t rep_len = 0;
 	uint32_t rep_index = 0;
 
-	for (uint32_t i = 0; i < REP_DISTANCES; ++i) {
+	for (uint32_t i = 0; i < REPS; ++i) {
 		// Pointer to the beginning of the match candidate
 		const uint8_t *const buf_back = buf - coder->reps[i] - 1;
 
@@ -57,9 +58,8 @@ lzma_lzma_optimum_fast(lzma_coder *restrict coder, lzma_mf *restrict mf,
 
 		// The first two bytes matched.
 		// Calculate the length of the match.
-		uint32_t len;
-		for (len = 2; len < buf_avail
-				&& buf[len] == buf_back[len]; ++len) ;
+		const uint32_t len = lzma_memcmplen(
+				buf, buf_back, 2, buf_avail);
 
 		// If we have found a repeated match that is at least
 		// nice_len long, return it immediately.
@@ -79,8 +79,7 @@ lzma_lzma_optimum_fast(lzma_coder *restrict coder, lzma_mf *restrict mf,
 	// We didn't find a long enough repeated match. Encode it as a normal
 	// match if the match length is at least nice_len.
 	if (len_main >= nice_len) {
-		*back_res = coder->matches[matches_count - 1].dist
-				+ REP_DISTANCES;
+		*back_res = coder->matches[matches_count - 1].dist + REPS;
 		*len_res = len_main;
 		mf_skip(mf, len_main - 1);
 		return;
@@ -155,24 +154,15 @@ lzma_lzma_optimum_fast(lzma_coder *restrict coder, lzma_mf *restrict mf,
 
 	const uint32_t limit = len_main - 1;
 
-	for (uint32_t i = 0; i < REP_DISTANCES; ++i) {
-		const uint8_t *const buf_back = buf - coder->reps[i] - 1;
-
-		if (not_equal_16(buf, buf_back))
-			continue;
-
-		uint32_t len;
-		for (len = 2; len < limit
-				&& buf[len] == buf_back[len]; ++len) ;
-
-		if (len >= limit) {
+	for (uint32_t i = 0; i < REPS; ++i) {
+		if (memcmp(buf, buf - coder->reps[i] - 1, limit) == 0) {
 			*back_res = UINT32_MAX;
 			*len_res = 1;
 			return;
 		}
 	}
 
-	*back_res = back_main + REP_DISTANCES;
+	*back_res = back_main + REPS;
 	*len_res = len_main;
 	mf_skip(mf, len_main - 2);
 	return;

contrib/xz/src/liblzma/lzma/lzma_encoder_optimum_normal.c

@@ -11,6 +11,7 @@
 
 #include "lzma_encoder_private.h"
 #include "fastpos.h"
+#include "memcmplen.h"
 
 
 ////////////
@@ -108,18 +109,18 @@ get_rep_price(const lzma_coder *const coder, const uint32_t rep_index,
 
 
 static inline uint32_t
-get_pos_len_price(const lzma_coder *const coder, const uint32_t pos,
+get_dist_len_price(const lzma_coder *const coder, const uint32_t dist,
 		const uint32_t len, const uint32_t pos_state)
 {
-	const uint32_t len_to_pos_state = get_len_to_pos_state(len);
+	const uint32_t dist_state = get_dist_state(len);
 	uint32_t price;
 
-	if (pos < FULL_DISTANCES) {
-		price = coder->distances_prices[len_to_pos_state][pos];
+	if (dist < FULL_DISTANCES) {
+		price = coder->dist_prices[dist_state][dist];
 	} else {
-		const uint32_t pos_slot = get_pos_slot_2(pos);
-		price = coder->pos_slot_prices[len_to_pos_state][pos_slot]
-				+ coder->align_prices[pos & ALIGN_MASK];
+		const uint32_t dist_slot = get_dist_slot_2(dist);
+		price = coder->dist_slot_prices[dist_state][dist_slot]
+				+ coder->align_prices[dist & ALIGN_MASK];
 	}
 
 	price += get_len_price(&coder->match_len_encoder, len, pos_state);
@@ -129,55 +130,53 @@ get_pos_len_price(const lzma_coder *const coder, const uint32_t pos,
 
 
 static void
-fill_distances_prices(lzma_coder *coder)
+fill_dist_prices(lzma_coder *coder)
 {
-	for (uint32_t len_to_pos_state = 0;
-			len_to_pos_state < LEN_TO_POS_STATES;
-			++len_to_pos_state) {
+	for (uint32_t dist_state = 0; dist_state < DIST_STATES; ++dist_state) {
 
-		uint32_t *const pos_slot_prices
-				= coder->pos_slot_prices[len_to_pos_state];
+		uint32_t *const dist_slot_prices
+				= coder->dist_slot_prices[dist_state];
 
-		// Price to encode the pos_slot.
-		for (uint32_t pos_slot = 0;
-				pos_slot < coder->dist_table_size; ++pos_slot)
-			pos_slot_prices[pos_slot] = rc_bittree_price(
-					coder->pos_slot[len_to_pos_state],
-					POS_SLOT_BITS, pos_slot);
+		// Price to encode the dist_slot.
+		for (uint32_t dist_slot = 0;
+				dist_slot < coder->dist_table_size; ++dist_slot)
+			dist_slot_prices[dist_slot] = rc_bittree_price(
+					coder->dist_slot[dist_state],
+					DIST_SLOT_BITS, dist_slot);
 
 		// For matches with distance >= FULL_DISTANCES, add the price
 		// of the direct bits part of the match distance. (Align bits
 		// are handled by fill_align_prices()).
-		for (uint32_t pos_slot = END_POS_MODEL_INDEX;
-				pos_slot < coder->dist_table_size; ++pos_slot)
-			pos_slot_prices[pos_slot] += rc_direct_price(
-					((pos_slot >> 1) - 1) - ALIGN_BITS);
+		for (uint32_t dist_slot = DIST_MODEL_END;
+				dist_slot < coder->dist_table_size;
+				++dist_slot)
+			dist_slot_prices[dist_slot] += rc_direct_price(
+					((dist_slot >> 1) - 1) - ALIGN_BITS);
 
 		// Distances in the range [0, 3] are fully encoded with
-		// pos_slot, so they are used for coder->distances_prices
+		// dist_slot, so they are used for coder->dist_prices
 		// as is.
-		for (uint32_t i = 0; i < START_POS_MODEL_INDEX; ++i)
-			coder->distances_prices[len_to_pos_state][i]
-					= pos_slot_prices[i];
+		for (uint32_t i = 0; i < DIST_MODEL_START; ++i)
+			coder->dist_prices[dist_state][i]
+					= dist_slot_prices[i];
 	}
 
-	// Distances in the range [4, 127] depend on pos_slot and pos_special.
-	// We do this in a loop separate from the above loop to avoid
-	// redundant calls to get_pos_slot().
-	for (uint32_t i = START_POS_MODEL_INDEX; i < FULL_DISTANCES; ++i) {
-		const uint32_t pos_slot = get_pos_slot(i);
-		const uint32_t footer_bits = ((pos_slot >> 1) - 1);
-		const uint32_t base = (2 | (pos_slot & 1)) << footer_bits;
+	// Distances in the range [4, 127] depend on dist_slot and
+	// dist_special. We do this in a loop separate from the above
+	// loop to avoid redundant calls to get_dist_slot().
+	for (uint32_t i = DIST_MODEL_START; i < FULL_DISTANCES; ++i) {
+		const uint32_t dist_slot = get_dist_slot(i);
+		const uint32_t footer_bits = ((dist_slot >> 1) - 1);
+		const uint32_t base = (2 | (dist_slot & 1)) << footer_bits;
 		const uint32_t price = rc_bittree_reverse_price(
-				coder->pos_special + base - pos_slot - 1,
+				coder->dist_special + base - dist_slot - 1,
 				footer_bits, i - base);
 
-		for (uint32_t len_to_pos_state = 0;
-				len_to_pos_state < LEN_TO_POS_STATES;
-				++len_to_pos_state)
-			coder->distances_prices[len_to_pos_state][i]
-					= price + coder->pos_slot_prices[
-						len_to_pos_state][pos_slot];
+		for (uint32_t dist_state = 0; dist_state < DIST_STATES;
+				++dist_state)
+			coder->dist_prices[dist_state][i]
+					= price + coder->dist_slot_prices[
+						dist_state][dist_slot];
 	}
 
 	coder->match_price_count = 0;
@@ -188,9 +187,9 @@ fill_distances_prices(lzma_coder *coder)
 static void
 fill_align_prices(lzma_coder *coder)
 {
-	for (uint32_t i = 0; i < ALIGN_TABLE_SIZE; ++i)
+	for (uint32_t i = 0; i < ALIGN_SIZE; ++i)
 		coder->align_prices[i] = rc_bittree_reverse_price(
-				coder->pos_align, ALIGN_BITS, i);
+				coder->dist_align, ALIGN_BITS, i);
 
 	coder->align_price_count = 0;
 	return;
@@ -296,10 +295,10 @@ helper1(lzma_coder *restrict coder, lzma_mf *restrict mf,
 
 	const uint8_t *const buf = mf_ptr(mf) - 1;
 
-	uint32_t rep_lens[REP_DISTANCES];
+	uint32_t rep_lens[REPS];
 	uint32_t rep_max_index = 0;
 
-	for (uint32_t i = 0; i < REP_DISTANCES; ++i) {
+	for (uint32_t i = 0; i < REPS; ++i) {
 		const uint8_t *const buf_back = buf - coder->reps[i] - 1;
 
 		if (not_equal_16(buf, buf_back)) {
@@ -307,13 +306,9 @@ helper1(lzma_coder *restrict coder, lzma_mf *restrict mf,
 			continue;
 		}
 
-		uint32_t len_test;
-		for (len_test = 2; len_test < buf_avail
-				&& buf[len_test] == buf_back[len_test];
-				++len_test) ;
+		rep_lens[i] = lzma_memcmplen(buf, buf_back, 2, buf_avail);
 
-		rep_lens[i] = len_test;
-		if (len_test > rep_lens[rep_max_index])
+		if (rep_lens[i] > rep_lens[rep_max_index])
 			rep_max_index = i;
 	}
 
@@ -326,8 +321,7 @@ helper1(lzma_coder *restrict coder, lzma_mf *restrict mf,
 
 
 	if (len_main >= nice_len) {
-		*back_res = coder->matches[matches_count - 1].dist
-				+ REP_DISTANCES;
+		*back_res = coder->matches[matches_count - 1].dist + REPS;
 		*len_res = len_main;
 		mf_skip(mf, len_main - 1);
 		return UINT32_MAX;
@@ -381,7 +375,7 @@ helper1(lzma_coder *restrict coder, lzma_mf *restrict mf,
 
 	coder->opts[1].pos_prev = 0;
 
-	for (uint32_t i = 0; i < REP_DISTANCES; ++i)
+	for (uint32_t i = 0; i < REPS; ++i)
 		coder->opts[0].backs[i] = coder->reps[i];
 
 	uint32_t len = len_end;
@@ -390,7 +384,7 @@ helper1(lzma_coder *restrict coder, lzma_mf *restrict mf,
 	} while (--len >= 2);
 
 
-	for (uint32_t i = 0; i < REP_DISTANCES; ++i) {
+	for (uint32_t i = 0; i < REPS; ++i) {
 		uint32_t rep_len = rep_lens[i];
 		if (rep_len < 2)
 			continue;
@@ -426,14 +420,13 @@ helper1(lzma_coder *restrict coder, lzma_mf *restrict mf,
 		for(; ; ++len) {
 			const uint32_t dist = coder->matches[i].dist;
 			const uint32_t cur_and_len_price = normal_match_price
-					+ get_pos_len_price(coder,
+					+ get_dist_len_price(coder,
 						dist, len, pos_state);
 
 			if (cur_and_len_price < coder->opts[len].price) {
 				coder->opts[len].price = cur_and_len_price;
 				coder->opts[len].pos_prev = 0;
-				coder->opts[len].back_prev
-						= dist + REP_DISTANCES;
+				coder->opts[len].back_prev = dist + REPS;
 				coder->opts[len].prev_1_is_literal = false;
 			}
 
@@ -463,7 +456,7 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf,
 		if (coder->opts[cur].prev_2) {
 			state = coder->opts[coder->opts[cur].pos_prev_2].state;
 
-			if (coder->opts[cur].back_prev_2 < REP_DISTANCES)
+			if (coder->opts[cur].back_prev_2 < REPS)
 				update_long_rep(state);
 			else
 				update_match(state);
@@ -492,33 +485,33 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf,
 			update_long_rep(state);
 		} else {
 			pos = coder->opts[cur].back_prev;
-			if (pos < REP_DISTANCES)
+			if (pos < REPS)
 				update_long_rep(state);
 			else
 				update_match(state);
 		}
 
-		if (pos < REP_DISTANCES) {
+		if (pos < REPS) {
 			reps[0] = coder->opts[pos_prev].backs[pos];
 
 			uint32_t i;
 			for (i = 1; i <= pos; ++i)
 				reps[i] = coder->opts[pos_prev].backs[i - 1];
 
-			for (; i < REP_DISTANCES; ++i)
+			for (; i < REPS; ++i)
 				reps[i] = coder->opts[pos_prev].backs[i];
 
 		} else {
-			reps[0] = pos - REP_DISTANCES;
+			reps[0] = pos - REPS;
 
-			for (uint32_t i = 1; i < REP_DISTANCES; ++i)
+			for (uint32_t i = 1; i < REPS; ++i)
 				reps[i] = coder->opts[pos_prev].backs[i - 1];
 		}
 	}
 
 	coder->opts[cur].state = state;
 
-	for (uint32_t i = 0; i < REP_DISTANCES; ++i)
+	for (uint32_t i = 0; i < REPS; ++i)
 		coder->opts[cur].backs[i] = reps[i];
 
 	const uint32_t cur_price = coder->opts[cur].price;
@@ -572,11 +565,7 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf,
 		const uint8_t *const buf_back = buf - reps[0] - 1;
 		const uint32_t limit = my_min(buf_avail_full, nice_len + 1);
 
-		uint32_t len_test = 1;
-		while (len_test < limit && buf[len_test] == buf_back[len_test])
-			++len_test;
-
-		--len_test;
+		const uint32_t len_test = lzma_memcmplen(buf, buf_back, 1, limit) - 1;
 
 		if (len_test >= 2) {
 			lzma_lzma_state state_2 = state;
@@ -611,15 +600,12 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf,
 
 	uint32_t start_len = 2; // speed optimization
 
-	for (uint32_t rep_index = 0; rep_index < REP_DISTANCES; ++rep_index) {
+	for (uint32_t rep_index = 0; rep_index < REPS; ++rep_index) {
 		const uint8_t *const buf_back = buf - reps[rep_index] - 1;
 		if (not_equal_16(buf, buf_back))
 			continue;
 
-		uint32_t len_test;
-		for (len_test = 2; len_test < buf_avail
-				&& buf[len_test] == buf_back[len_test];
-				++len_test) ;
+		uint32_t len_test = lzma_memcmplen(buf, buf_back, 2, buf_avail);
 
 		while (len_end < cur + len_test)
 			coder->opts[++len_end].price = RC_INFINITY_PRICE;
@@ -728,14 +714,14 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf,
 		for (uint32_t len_test = start_len; ; ++len_test) {
 			const uint32_t cur_back = coder->matches[i].dist;
 			uint32_t cur_and_len_price = normal_match_price
-					+ get_pos_len_price(coder,
+					+ get_dist_len_price(coder,
 						cur_back, len_test, pos_state);
 
 			if (cur_and_len_price < coder->opts[cur + len_test].price) {
 				coder->opts[cur + len_test].price = cur_and_len_price;
 				coder->opts[cur + len_test].pos_prev = cur;
 				coder->opts[cur + len_test].back_prev
-						= cur_back + REP_DISTANCES;
+						= cur_back + REPS;
 				coder->opts[cur + len_test].prev_1_is_literal = false;
 			}
 
@@ -795,7 +781,7 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf,
 						coder->opts[offset].prev_2 = true;
 						coder->opts[offset].pos_prev_2 = cur;
 						coder->opts[offset].back_prev_2
-								= cur_back + REP_DISTANCES;
+								= cur_back + REPS;
 					}
 					//}
 				}
@@ -831,9 +817,9 @@ lzma_lzma_optimum_normal(lzma_coder *restrict coder, lzma_mf *restrict mf,
 	// In liblzma they were moved into this single place.
 	if (mf->read_ahead == 0) {
 		if (coder->match_price_count >= (1 << 7))
-			fill_distances_prices(coder);
+			fill_dist_prices(coder);
 
-		if (coder->align_price_count >= ALIGN_TABLE_SIZE)
+		if (coder->align_price_count >= ALIGN_SIZE)
 			fill_align_prices(coder);
 	}
 
@@ -845,7 +831,7 @@ lzma_lzma_optimum_normal(lzma_coder *restrict coder, lzma_mf *restrict mf,
 	if (len_end == UINT32_MAX)
 		return;
 
-	uint32_t reps[REP_DISTANCES];
+	uint32_t reps[REPS];
 	memcpy(reps, coder->reps, sizeof(reps));
 
 	uint32_t cur;

contrib/xz/src/liblzma/lzma/lzma_encoder_private.h

@@ -64,7 +64,7 @@ typedef struct {
 	uint32_t pos_prev;  // pos_next;
 	uint32_t back_prev;
 
-	uint32_t backs[REP_DISTANCES];
+	uint32_t backs[REPS];
 
 } lzma_optimal;
 
@@ -77,7 +77,7 @@ struct lzma_coder_s {
 	lzma_lzma_state state;
 
 	/// The four most recent match distances
-	uint32_t reps[REP_DISTANCES];
+	uint32_t reps[REPS];
 
 	/// Array of match candidates
 	lzma_match matches[MATCH_LEN_MAX + 1];
@@ -112,9 +112,9 @@ struct lzma_coder_s {
 	probability is_rep1[STATES];
 	probability is_rep2[STATES];
 	probability is_rep0_long[STATES][POS_STATES_MAX];
-	probability pos_slot[LEN_TO_POS_STATES][POS_SLOTS];
-	probability pos_special[FULL_DISTANCES - END_POS_MODEL_INDEX];
-	probability pos_align[ALIGN_TABLE_SIZE];
+	probability dist_slot[DIST_STATES][DIST_SLOTS];
+	probability dist_special[FULL_DISTANCES - DIST_MODEL_END];
+	probability dist_align[ALIGN_SIZE];
 
 	// These are the same as in lzma_decoder.c except that the encoders
 	// include also price tables.
@@ -122,12 +122,12 @@ struct lzma_coder_s {
 	lzma_length_encoder rep_len_encoder;
 
 	// Price tables
-	uint32_t pos_slot_prices[LEN_TO_POS_STATES][POS_SLOTS];
-	uint32_t distances_prices[LEN_TO_POS_STATES][FULL_DISTANCES];
+	uint32_t dist_slot_prices[DIST_STATES][DIST_SLOTS];
+	uint32_t dist_prices[DIST_STATES][FULL_DISTANCES];
 	uint32_t dist_table_size;
 	uint32_t match_price_count;
 
-	uint32_t align_prices[ALIGN_TABLE_SIZE];
+	uint32_t align_prices[ALIGN_SIZE];
 	uint32_t align_price_count;
 
 	// Optimal

contrib/xz/src/liblzma/rangecoder/range_decoder.h

@@ -25,20 +25,26 @@ typedef struct {
 
 
 /// Reads the first five bytes to initialize the range decoder.
-static inline bool
+static inline lzma_ret
 rc_read_init(lzma_range_decoder *rc, const uint8_t *restrict in,
 		size_t *restrict in_pos, size_t in_size)
 {
 	while (rc->init_bytes_left > 0) {
 		if (*in_pos == in_size)
-			return false;
+			return LZMA_OK;
+
+		// The first byte is always 0x00. It could have been omitted
+		// in LZMA2 but it wasn't, so one byte is wasted in every
+		// LZMA2 chunk.
+		if (rc->init_bytes_left == 5 && in[*in_pos] != 0x00)
+			return LZMA_DATA_ERROR;
 
 		rc->code = (rc->code << 8) | in[*in_pos];
 		++*in_pos;
 		--rc->init_bytes_left;
 	}
 
-	return true;
+	return LZMA_STREAM_END;
 }
 
 

contrib/xz/src/liblzma/simple/arm.c

@@ -45,7 +45,7 @@ arm_code(lzma_simple *simple lzma_attribute((__unused__)),
 
 
 static lzma_ret
-arm_coder_init(lzma_next_coder *next, lzma_allocator *allocator,
+arm_coder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter_info *filters, bool is_encoder)
 {
 	return lzma_simple_coder_init(next, allocator, filters,
@@ -54,7 +54,8 @@ arm_coder_init(lzma_next_coder *next, lzma_allocator *allocator,
 
 
 extern lzma_ret
-lzma_simple_arm_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_simple_arm_encoder_init(lzma_next_coder *next,
+		const lzma_allocator *allocator,
 		const lzma_filter_info *filters)
 {
 	return arm_coder_init(next, allocator, filters, true);
@@ -62,7 +63,8 @@ lzma_simple_arm_encoder_init(lzma_next_coder *next, lzma_allocator *allocator,
 
 
 extern lzma_ret
-lzma_simple_arm_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_simple_arm_decoder_init(lzma_next_coder *next,
+		const lzma_allocator *allocator,
 		const lzma_filter_info *filters)
 {
 	return arm_coder_init(next, allocator, filters, false);

contrib/xz/src/liblzma/simple/armthumb.c

@@ -50,7 +50,7 @@ armthumb_code(lzma_simple *simple lzma_attribute((__unused__)),
 
 
 static lzma_ret
-armthumb_coder_init(lzma_next_coder *next, lzma_allocator *allocator,
+armthumb_coder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter_info *filters, bool is_encoder)
 {
 	return lzma_simple_coder_init(next, allocator, filters,
@@ -60,7 +60,8 @@ armthumb_coder_init(lzma_next_coder *next, lzma_allocator *allocator,
 
 extern lzma_ret
 lzma_simple_armthumb_encoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters)
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters)
 {
 	return armthumb_coder_init(next, allocator, filters, true);
 }
@@ -68,7 +69,8 @@ lzma_simple_armthumb_encoder_init(lzma_next_coder *next,
 
 extern lzma_ret
 lzma_simple_armthumb_decoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters)
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters)
 {
 	return armthumb_coder_init(next, allocator, filters, false);
 }

contrib/xz/src/liblzma/simple/ia64.c

@@ -86,7 +86,7 @@ ia64_code(lzma_simple *simple lzma_attribute((__unused__)),
 
 
 static lzma_ret
-ia64_coder_init(lzma_next_coder *next, lzma_allocator *allocator,
+ia64_coder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter_info *filters, bool is_encoder)
 {
 	return lzma_simple_coder_init(next, allocator, filters,
@@ -96,7 +96,8 @@ ia64_coder_init(lzma_next_coder *next, lzma_allocator *allocator,
 
 extern lzma_ret
 lzma_simple_ia64_encoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters)
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters)
 {
 	return ia64_coder_init(next, allocator, filters, true);
 }
@@ -104,7 +105,8 @@ lzma_simple_ia64_encoder_init(lzma_next_coder *next,
 
 extern lzma_ret
 lzma_simple_ia64_decoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters)
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters)
 {
 	return ia64_coder_init(next, allocator, filters, false);
 }

contrib/xz/src/liblzma/simple/powerpc.c

@@ -49,7 +49,7 @@ powerpc_code(lzma_simple *simple lzma_attribute((__unused__)),
 
 
 static lzma_ret
-powerpc_coder_init(lzma_next_coder *next, lzma_allocator *allocator,
+powerpc_coder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter_info *filters, bool is_encoder)
 {
 	return lzma_simple_coder_init(next, allocator, filters,
@@ -59,7 +59,8 @@ powerpc_coder_init(lzma_next_coder *next, lzma_allocator *allocator,
 
 extern lzma_ret
 lzma_simple_powerpc_encoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters)
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters)
 {
 	return powerpc_coder_init(next, allocator, filters, true);
 }
@@ -67,7 +68,8 @@ lzma_simple_powerpc_encoder_init(lzma_next_coder *next,
 
 extern lzma_ret
 lzma_simple_powerpc_decoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters)
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters)
 {
 	return powerpc_coder_init(next, allocator, filters, false);
 }

contrib/xz/src/liblzma/simple/simple_coder.c

@@ -18,7 +18,7 @@
 
 /// Copied or encodes/decodes more data to out[].
 static lzma_ret
-copy_or_code(lzma_coder *coder, lzma_allocator *allocator,
+copy_or_code(lzma_coder *coder, const lzma_allocator *allocator,
 		const uint8_t *restrict in, size_t *restrict in_pos,
 		size_t in_size, uint8_t *restrict out,
 		size_t *restrict out_pos, size_t out_size, lzma_action action)
@@ -66,7 +66,7 @@ call_filter(lzma_coder *coder, uint8_t *buffer, size_t size)
 
 
 static lzma_ret
-simple_code(lzma_coder *coder, lzma_allocator *allocator,
+simple_code(lzma_coder *coder, const lzma_allocator *allocator,
 		const uint8_t *restrict in, size_t *restrict in_pos,
 		size_t in_size, uint8_t *restrict out,
 		size_t *restrict out_pos, size_t out_size, lzma_action action)
@@ -198,7 +198,7 @@ simple_code(lzma_coder *coder, lzma_allocator *allocator,
 
 
 static void
-simple_coder_end(lzma_coder *coder, lzma_allocator *allocator)
+simple_coder_end(lzma_coder *coder, const lzma_allocator *allocator)
 {
 	lzma_next_end(&coder->next, allocator);
 	lzma_free(coder->simple, allocator);
@@ -208,7 +208,7 @@ simple_coder_end(lzma_coder *coder, lzma_allocator *allocator)
 
 
 static lzma_ret
-simple_coder_update(lzma_coder *coder, lzma_allocator *allocator,
+simple_coder_update(lzma_coder *coder, const lzma_allocator *allocator,
 		const lzma_filter *filters_null lzma_attribute((__unused__)),
 		const lzma_filter *reversed_filters)
 {
@@ -219,7 +219,7 @@ simple_coder_update(lzma_coder *coder, lzma_allocator *allocator,
 
 
 extern lzma_ret
-lzma_simple_coder_init(lzma_next_coder *next, lzma_allocator *allocator,
+lzma_simple_coder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter_info *filters,
 		size_t (*filter)(lzma_simple *simple, uint32_t now_pos,
 			bool is_encoder, uint8_t *buffer, size_t size),

contrib/xz/src/liblzma/simple/simple_coder.h

@@ -17,44 +17,56 @@
 
 
 extern lzma_ret lzma_simple_x86_encoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters);
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters);
 
 extern lzma_ret lzma_simple_x86_decoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters);
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters);
 
 
 extern lzma_ret lzma_simple_powerpc_encoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters);
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters);
 
 extern lzma_ret lzma_simple_powerpc_decoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters);
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters);
 
 
 extern lzma_ret lzma_simple_ia64_encoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters);
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters);
 
 extern lzma_ret lzma_simple_ia64_decoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters);
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters);
 
 
 extern lzma_ret lzma_simple_arm_encoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters);
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters);
 
 extern lzma_ret lzma_simple_arm_decoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters);
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters);
 
 
 extern lzma_ret lzma_simple_armthumb_encoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters);
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters);
 
 extern lzma_ret lzma_simple_armthumb_decoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters);
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters);
 
 
 extern lzma_ret lzma_simple_sparc_encoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters);
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters);
 
 extern lzma_ret lzma_simple_sparc_decoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters);
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters);
 
 #endif

contrib/xz/src/liblzma/simple/simple_decoder.c

@@ -14,7 +14,7 @@
 
 
 extern lzma_ret
-lzma_simple_props_decode(void **options, lzma_allocator *allocator,
+lzma_simple_props_decode(void **options, const lzma_allocator *allocator,
 		const uint8_t *props, size_t props_size)
 {
 	if (props_size == 0)

contrib/xz/src/liblzma/simple/simple_decoder.h

@@ -16,7 +16,7 @@
 #include "simple_coder.h"
 
 extern lzma_ret lzma_simple_props_decode(
-		void **options, lzma_allocator *allocator,
+		void **options, const lzma_allocator *allocator,
 		const uint8_t *props, size_t props_size);
 
 #endif

contrib/xz/src/liblzma/simple/simple_private.h

@@ -66,7 +66,8 @@ struct lzma_coder_s {
 
 
 extern lzma_ret lzma_simple_coder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters,
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters,
 		size_t (*filter)(lzma_simple *simple, uint32_t now_pos,
 			bool is_encoder, uint8_t *buffer, size_t size),
 		size_t simple_size, size_t unfiltered_max,

contrib/xz/src/liblzma/simple/sparc.c

@@ -57,7 +57,7 @@ sparc_code(lzma_simple *simple lzma_attribute((__unused__)),
 
 
 static lzma_ret
-sparc_coder_init(lzma_next_coder *next, lzma_allocator *allocator,
+sparc_coder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		const lzma_filter_info *filters, bool is_encoder)
 {
 	return lzma_simple_coder_init(next, allocator, filters,
@@ -67,7 +67,8 @@ sparc_coder_init(lzma_next_coder *next, lzma_allocator *allocator,
 
 extern lzma_ret
 lzma_simple_sparc_encoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters)
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters)
 {
 	return sparc_coder_init(next, allocator, filters, true);
 }
@@ -75,7 +76,8 @@ lzma_simple_sparc_encoder_init(lzma_next_coder *next,
 
 extern lzma_ret
 lzma_simple_sparc_decoder_init(lzma_next_coder *next,
-		lzma_allocator *allocator, const lzma_filter_info *filters)
+		const lzma_allocator *allocator,
+		const lzma_filter_info *filters)
 {
 	return sparc_coder_init(next, allocator, filters, false);
 }