d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update vendor/libarchive to git 92366744a52f3fa83c3899e375e415a5080a05f2

Browse Source 
Relevant vendor changes:
  PR #905: Support for Zstandard read and write filters
  PR #922: Avoid overflow when reading corrupt cpio archive
  Issue #935: heap-based buffer overflow in xml_data (CVE-2017-14166)
  OSS-Fuzz 2936: Place a limit on the mtree line length
  OSS-Fuzz 2394: Ensure that the ZIP AES extension header is large enough
  OSS-Fuzz 573: Read off-by-one error in RAR archives (CVE-2017-14502)

Security: CVE-2017-14166, CVE-2017-14502
This commit is contained in:
Martin Matuska 2017-09-30 23:33:19 +00:00
parent b85c4c3da0
commit 533ab3c53e
61 changed files with 1531 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -25,6 +25,6 @@ matrix:
before_install:
- if [ `uname` = "Darwin" ]; then brew update; fi
install:
- if [ `uname` = "Darwin" ]; then brew install xz lzop lz4; fi
- if [ `uname` = "Darwin" ]; then brew install xz lzop lz4 zstd; fi
script:
- build/ci_build.sh

25
CMakeLists.txt
View File

@ -533,6 +533,31 @@ IF(LZ4_FOUND)
ENDIF(LZ4_FOUND)
MARK_AS_ADVANCED(CLEAR LZ4_INCLUDE_DIR)
MARK_AS_ADVANCED(CLEAR LZ4_LIBRARY)
#
# Find Zstd
#
IF (ZSTD_INCLUDE_DIR)
# Already in cache, be silent
SET(ZSTD_FIND_QUIETLY TRUE)
ENDIF (ZSTD_INCLUDE_DIR)
FIND_PATH(ZSTD_INCLUDE_DIR zstd.h)
FIND_LIBRARY(ZSTD_LIBRARY NAMES zstd libzstd)
INCLUDE(FindPackageHandleStandardArgs)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(ZSTD DEFAULT_MSG ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
IF(ZSTD_FOUND)
SET(HAVE_ZSTD_H 1)
INCLUDE_DIRECTORIES(${ZSTD_INCLUDE_DIR})
LIST(APPEND ADDITIONAL_LIBS ${ZSTD_LIBRARY})
SET(CMAKE_REQUIRED_LIBRARIES ${ZSTD_LIBRARY})
SET(CMAKE_REQUIRED_INCLUDES ${ZSTD_INCLUDE_DIR})
CHECK_FUNCTION_EXISTS(ZSTD_compressStream HAVE_LIBZSTD)
#
# TODO: test for static library.
#
ENDIF(ZSTD_FOUND)
MARK_AS_ADVANCED(CLEAR ZSTD_INCLUDE_DIR)
MARK_AS_ADVANCED(CLEAR ZSTD_LIBRARY)
#
# Check headers

15
Makefile.am
View File

@ -167,6 +167,7 @@ libarchive_la_SOURCES= \
libarchive/archive_read_support_filter_rpm.c \
libarchive/archive_read_support_filter_uu.c \
libarchive/archive_read_support_filter_xz.c \
libarchive/archive_read_support_filter_zstd.c \
libarchive/archive_read_support_format_7zip.c \
libarchive/archive_read_support_format_all.c \
libarchive/archive_read_support_format_ar.c \
@ -213,6 +214,7 @@ libarchive_la_SOURCES= \
libarchive/archive_write_add_filter_program.c \
libarchive/archive_write_add_filter_uuencode.c \
libarchive/archive_write_add_filter_xz.c \
libarchive/archive_write_add_filter_zstd.c \
libarchive/archive_write_set_format.c \
libarchive/archive_write_set_format_7zip.c \
libarchive/archive_write_set_format_ar.c \
@ -402,6 +404,7 @@ libarchive_test_SOURCES= \
libarchive/test/test_compat_uudecode_large.c \
libarchive/test/test_compat_xz.c \
libarchive/test/test_compat_zip.c \
libarchive/test/test_compat_zstd.c \
libarchive/test/test_empty_write.c \
libarchive/test/test_entry.c \
libarchive/test/test_entry_strmode.c \
@ -559,6 +562,7 @@ libarchive_test_SOURCES= \
libarchive/test/test_write_filter_program.c \
libarchive/test/test_write_filter_uuencode.c \
libarchive/test/test_write_filter_xz.c \
libarchive/test/test_write_filter_zstd.c \
libarchive/test/test_write_format_7zip.c \
libarchive/test/test_write_format_7zip_empty.c \
libarchive/test/test_write_format_7zip_large.c \
@ -677,6 +681,7 @@ libarchive_test_EXTRA_DIST=\
libarchive/test/test_compat_zip_5.zip.uu \
libarchive/test/test_compat_zip_6.zip.uu \
libarchive/test/test_compat_zip_7.xps.uu \
libarchive/test/test_compat_zstd_1.tar.zst.uu \
libarchive/test/test_fuzz.cab.uu \
libarchive/test/test_fuzz.lzh.uu \
libarchive/test/test_fuzz_1.iso.Z.uu \
@ -961,6 +966,7 @@ bsdtar_test_SOURCES= \
tar/test/test_extract_tar_lzma.c \
tar/test/test_extract_tar_lzo.c \
tar/test/test_extract_tar_xz.c \
tar/test/test_extract_tar_zstd.c \
tar/test/test_format_newc.c \
tar/test/test_help.c \
tar/test/test_leading_slash.c \
@ -1000,6 +1006,7 @@ bsdtar_test_SOURCES= \
tar/test/test_option_xattrs.c \
tar/test/test_option_xz.c \
tar/test/test_option_z.c \
tar/test/test_option_zstd.c \
tar/test/test_patterns.c \
tar/test/test_print_longpath.c \
tar/test/test_stdio.c \
@ -1036,6 +1043,7 @@ bsdtar_test_EXTRA_DIST= \
tar/test/test_extract.tar.lrz.uu \
tar/test/test_extract.tar.lz.uu \
tar/test/test_extract.tar.lz4.uu \
tar/test/test_extract.tar.zst.uu \
tar/test/test_extract.tar.lzma.uu \
tar/test/test_extract.tar.lzo.uu \
tar/test/test_extract.tar.xz.uu \
@ -1121,6 +1129,7 @@ bsdcpio_test_SOURCES= \
cpio/test/test_extract_cpio_lzma.c \
cpio/test/test_extract_cpio_lzo.c \
cpio/test/test_extract_cpio_xz.c \
cpio/test/test_extract_cpio_zstd.c \
cpio/test/test_format_newc.c \
cpio/test/test_gcpio_compat.c \
cpio/test/test_missing_file.c \
@ -1151,6 +1160,7 @@ bsdcpio_test_SOURCES= \
cpio/test/test_option_xz.c \
cpio/test/test_option_y.c \
cpio/test/test_option_z.c \
cpio/test/test_option_zstd.c \
cpio/test/test_owner_parse.c \
cpio/test/test_passthrough_dotdot.c \
cpio/test/test_passthrough_reverse.c
@ -1184,6 +1194,7 @@ bsdcpio_test_EXTRA_DIST= \
cpio/test/test_extract.cpio.lrz.uu \
cpio/test/test_extract.cpio.lz.uu \
cpio/test/test_extract.cpio.lz4.uu \
cpio/test/test_extract.cpio.zst.uu \
cpio/test/test_extract.cpio.lzma.uu \
cpio/test/test_extract.cpio.lzo.uu \
cpio/test/test_extract.cpio.xz.uu \
@ -1259,6 +1270,7 @@ bsdcat_test_SOURCES= \
cat/test/test_empty_gz.c \
cat/test/test_empty_lz4.c \
cat/test/test_empty_xz.c \
cat/test/test_empty_zstd.c \
cat/test/test_error.c \
cat/test/test_error_mixed.c \
cat/test/test_expand_Z.c \
@ -1268,6 +1280,7 @@ bsdcat_test_SOURCES= \
cat/test/test_expand_mixed.c \
cat/test/test_expand_plain.c \
cat/test/test_expand_xz.c \
cat/test/test_expand_zstd.c \
cat/test/test_help.c \
cat/test/test_version.c
@ -1294,11 +1307,13 @@ bsdcat_test_EXTRA_DIST= \
cat/test/list.h \
cat/test/test_empty.gz.uu \
cat/test/test_empty.lz4.uu \
cat/test/test_empty.zst.uu \
cat/test/test_empty.xz.uu \
cat/test/test_expand.Z.uu \
cat/test/test_expand.bz2.uu \
cat/test/test_expand.gz.uu \
cat/test/test_expand.lz4.uu \
cat/test/test_expand.zst.uu \
cat/test/test_expand.plain.uu \
cat/test/test_expand.xz.uu \
cat/test/CMakeLists.txt

6
build/cmake/config.h.in
View File

@ -725,6 +725,9 @@ typedef uint64_t uintmax_t;
/* Define to 1 if you have the `z' library (-lz). */
#cmakedefine HAVE_LIBZ 1
/* Define to 1 if you have the `zstd' library (-lzstd). */
#cmakedefine HAVE_LIBZSTD 1
/* Define to 1 if you have the <limits.h> header file. */
#cmakedefine HAVE_LIMITS_H 1
@ -1179,6 +1182,9 @@ typedef uint64_t uintmax_t;
/* Define to 1 if you have the <zlib.h> header file. */
#cmakedefine HAVE_ZLIB_H 1
/* Define to 1 if you have the <zstd.h> header file. */
#cmakedefine HAVE_ZSTD_H 1
/* Define to 1 if you have the `_ctime64_s' function. */
#cmakedefine HAVE__CTIME64_S 1

1
build/pkgconfig/libarchive.pc.in
View File

@ -7,5 +7,6 @@ Name: libarchive
Description: library that can create and read several streaming archive formats
Version: @VERSION@
Cflags: -I${includedir}
Cflags.private: -DLIBARCHIVE_STATIC
Libs: -L${libdir} -larchive
Libs.private: @LIBS@

2
cat/test/CMakeLists.txt
View File

@ -12,6 +12,7 @@ IF(ENABLE_CAT AND ENABLE_TEST)
test_empty_gz.c
test_empty_lz4.c
test_empty_xz.c
test_empty_zstd.c
test_error.c
test_error_mixed.c
test_expand_Z.c
@ -21,6 +22,7 @@ IF(ENABLE_CAT AND ENABLE_TEST)
test_expand_mixed.c
test_expand_plain.c
test_expand_xz.c
test_expand_zstd.c
test_help.c
test_version.c
)

4
cat/test/test_empty.zst.uu Normal file
View File

@ -0,0 +1,4 @@
begin 644 test_empty.zst
-*+4O_010`0``F>G840``
`
end

41
cat/test/test_empty_zstd.c Normal file
View File

@ -0,0 +1,41 @@
/*-
* Copyright (c) 2017 Sean Purcell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "test.h"
DEFINE_TEST(test_empty_zstd)
{
const char *reffile = "test_empty.zst";
int f;
extract_reference_file(reffile);
f = systemf("%s %s >test.out 2>test.err", testprog, reffile);
if (f == 0 || canZstd()) {
assertEqualInt(0, f);
assertEmptyFile("test.out");
assertEmptyFile("test.err");
} else {
skipping("It seems zstd is not supported on this platform");
}
}

4
cat/test/test_expand.zst.uu Normal file
View File

@ -0,0 +1,4 @@
begin 644 test_expand.zst
J*+4O_010Z0``8V]N=&5N=',@;V8@=&5S=%]E>'!A;F0N>G-T+@J;23#F
`
end

41
cat/test/test_expand_zstd.c Normal file
View File

@ -0,0 +1,41 @@
/*-
* Copyright (c) 2017 Sean Purcell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "test.h"
DEFINE_TEST(test_expand_zstd)
{
const char *reffile = "test_expand.zst";
int f;
extract_reference_file(reffile);
f = systemf("%s %s >test.out 2>test.err", testprog, reffile);
if (f == 0 || canZstd()) {
assertEqualInt(0, f);
assertTextFileContents("contents of test_expand.zst.\n", "test.out");
assertEmptyFile("test.err");
} else {
skipping("It seems zstd is not supported on this platform");
}
}

10
configure.ac
View File

@ -367,6 +367,14 @@ if test "x$with_lz4" != "xno"; then
AC_CHECK_LIB(lz4,LZ4_decompress_safe)
fi
AC_ARG_WITH([zstd],
AS_HELP_STRING([--without-zstd], [Don't build support for zstd through libzstd]))
if test "x$with_zstd" != "xno"; then
AC_CHECK_HEADERS([zstd.h])
AC_CHECK_LIB(zstd,ZSTD_compressStream)
fi
AC_ARG_WITH([lzma],
AS_HELP_STRING([--without-lzma], [Don't build support for xz through lzma]))
@ -682,7 +690,7 @@ AC_ARG_ENABLE([xattr],
[Disable Extended Attributes support (default: check)]))
if test "x$enable_xattr" != "xno"; then
AC_SEARCH_LIBS([setxattr], [attr])
AC_SEARCH_LIBS([setxattr], [attr gnu])
AC_CHECK_DECLS([EXTATTR_NAMESPACE_USER], [], [], [#include <sys/types.h>
#include <sys/extattr.h>
])

2
contrib/android/Android.mk
View File

@ -74,6 +74,7 @@ libarchive_src_files := libarchive/archive_acl.c \
libarchive/archive_read_support_filter_rpm.c \
libarchive/archive_read_support_filter_uu.c \
libarchive/archive_read_support_filter_xz.c \
libarchive/archive_read_support_filter_zstd.c \
libarchive/archive_read_support_format_7zip.c \
libarchive/archive_read_support_format_all.c \
libarchive/archive_read_support_format_ar.c \
@ -116,6 +117,7 @@ libarchive_src_files := libarchive/archive_acl.c \
libarchive/archive_write_add_filter_program.c \
libarchive/archive_write_add_filter_uuencode.c \
libarchive/archive_write_add_filter_xz.c \
libarchive/archive_write_add_filter_zstd.c \
libarchive/archive_write_set_format.c \
libarchive/archive_write_set_format_7zip.c \
libarchive/archive_write_set_format_ar.c \

5
cpio/bsdcpio.1
View File

@ -187,6 +187,11 @@ In input mode, this option is ignored.
Compress the archive with lz4-compatible compression before writing it.
In input mode, this option is ignored; lz4 compression is recognized
automatically on input.
.It Fl Fl zstd
(o mode only)
Compress the archive with zstd-compatible compression before writing it.
In input mode, this option is ignored; zstd compression is recognized
automatically on input.
.It Fl Fl lzma
(o mode only)
Compress the file with lzma-compatible compression before writing it.

1
cpio/cmdline.c
View File

@ -92,6 +92,7 @@ static const struct option {
{ "verbose", 0, 'v' },
{ "version", 0, OPTION_VERSION },
{ "xz", 0, 'J' },
{ "zstd", 0, OPTION_ZSTD },
{ NULL, 0, 0 }
};

4
cpio/cpio.c
View File

@ -269,6 +269,7 @@ main(int argc, char *argv[])
case OPTION_LZ4:
case OPTION_LZMA: /* GNU tar, others */
case OPTION_LZOP: /* GNU tar, others */
case OPTION_ZSTD:
cpio->compress = opt;
break;
case 'm': /* POSIX 1997 */
@ -546,6 +547,9 @@ mode_out(struct cpio *cpio)
case OPTION_LZOP:
r = archive_write_add_filter_lzop(cpio->archive);
break;
case OPTION_ZSTD:
r = archive_write_add_filter_zstd(cpio->archive);
break;
case 'j': case 'y':
r = archive_write_add_filter_bzip2(cpio->archive);
break;

3
cpio/cpio.h
View File

@ -111,7 +111,8 @@ enum {
OPTION_PRESERVE_OWNER,
OPTION_QUIET,
OPTION_UUENCODE,
OPTION_VERSION
OPTION_VERSION,
OPTION_ZSTD,
};
int cpio_getopt(struct cpio *cpio);

2
cpio/test/CMakeLists.txt
View File

@ -23,6 +23,7 @@ IF(ENABLE_CPIO AND ENABLE_TEST)
test_extract_cpio_lzma
test_extract_cpio_lzo
test_extract_cpio_xz
test_extract_cpio_zstd
test_format_newc.c
test_gcpio_compat.c
test_missing_file.c
@ -53,6 +54,7 @@ IF(ENABLE_CPIO AND ENABLE_TEST)
test_option_xz.c
test_option_y.c
test_option_z.c
test_option_zstd.c
test_owner_parse.c
test_passthrough_dotdot.c
test_passthrough_reverse.c

6
cpio/test/test_extract.cpio.zst.uu Normal file
View File

@ -0,0 +1,6 @@
begin 644 test_extract.cpio.zst
M*+4O_01090,`,@41%X")&@#'G6T\K16_MR)#=DK)5:.1,2J0HY2"!(1!`!7R
M$(UB`2"*D41;J2UF&)<0!Y7X'TU<%W.\W^R]GO-WW^OO^QX0`%P<]30-!#U`
?!KD!`#XP,_`U4`HT3+RF:#!7Y\V@R)5"7P"^;WEUK@``
`
end

48
cpio/test/test_extract_cpio_zstd.c Normal file
View File

@ -0,0 +1,48 @@
/*-
* Copyright (c) 2017 Sean Purcell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "test.h"
__FBSDID("$FreeBSD$");
DEFINE_TEST(test_extract_cpio_zstd)
{
const char *reffile = "test_extract.cpio.zst";
int f;
extract_reference_file(reffile);
f = systemf("%s -it < %s >test.out 2>test.err", testprog, reffile);
if (f == 0 || canZstd()) {
assertEqualInt(0, systemf("%s -i < %s >test.out 2>test.err",
testprog, reffile));
assertFileExists("file1");
assertTextFileContents("contents of file1.\n", "file1");
assertFileExists("file2");
assertTextFileContents("contents of file2.\n", "file2");
assertEmptyFile("test.out");
assertTextFileContents("1 block\n", "test.err");
} else {
skipping("It seems zstd is not supported on this platform");
}
}

85
cpio/test/test_option_zstd.c Normal file
View File

@ -0,0 +1,85 @@
/*-
* Copyright (c) 2017 Sean Purcell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "test.h"
__FBSDID("$FreeBSD$");
DEFINE_TEST(test_option_zstd)
{
char *p;
int r;
size_t s;
/* Create a file. */
assertMakeFile("f", 0644, "a");
/* Archive it with zstd compression. */
r = systemf("echo f | %s -o --zstd >archive.out 2>archive.err",
testprog);
p = slurpfile(&s, "archive.err");
p[s] = '\0';
if (r != 0) {
if (strstr(p, "Unsupported compression") != NULL) {
skipping("This version of bsdcpio was compiled "
"without zstd support");
goto done;
}
/* POSIX permits different handling of the spawnp
* system call used to launch the subsidiary
* program: */
/* Some systems fail immediately to spawn the new process. */
if (strstr(p, "Can't launch") != NULL && !canZstd()) {
skipping("This version of bsdcpio uses an external zstd program "
"but no such program is available on this system.");
goto done;
}
/* Some systems successfully spawn the new process,
* but fail to exec a program within that process.
* This results in failure at the first attempt to
* write. */
if (strstr(p, "Can't write") != NULL && !canZstd()) {
skipping("This version of bsdcpio uses an external zstd program "
"but no such program is available on this system.");
goto done;
}
/* On some systems the error won't be detected until closing
time, by a 127 exit error returned by waitpid. */
if (strstr(p, "Error closing") != NULL && !canZstd()) {
skipping("This version of bsdcpio uses an external zstd program "
"but no such program is available on this system.");
return;
}
failure("--zstd option is broken: %s", p);
assertEqualInt(r, 0);
goto done;
}
free(p);
/* Check that the archive file has an zstd signature. */
p = slurpfile(&s, "archive.out");
assert(s > 2);
assertEqualMem(p, "\x28\xb5\x2f\xfd", 4);
done:
free(p);
}

2
libarchive/CMakeLists.txt
View File

@ -88,6 +88,7 @@ SET(libarchive_SOURCES
archive_read_support_filter_rpm.c
archive_read_support_filter_uu.c
archive_read_support_filter_xz.c
archive_read_support_filter_zstd.c
archive_read_support_format_7zip.c
archive_read_support_format_all.c
archive_read_support_format_ar.c
@ -134,6 +135,7 @@ SET(libarchive_SOURCES
archive_write_add_filter_program.c
archive_write_add_filter_uuencode.c
archive_write_add_filter_xz.c
archive_write_add_filter_zstd.c
archive_write_set_format.c
archive_write_set_format_7zip.c
archive_write_set_format_ar.c

4
libarchive/archive.h
View File

@ -177,6 +177,7 @@ __LA_DECL const char * archive_zlib_version(void);
__LA_DECL const char * archive_liblzma_version(void);
__LA_DECL const char * archive_bzlib_version(void);
__LA_DECL const char * archive_liblz4_version(void);
__LA_DECL const char * archive_libzstd_version(void);
/* Declare our basic types. */
struct archive;
@ -276,6 +277,7 @@ typedef const char *archive_passphrase_callback(struct archive *,
#define ARCHIVE_FILTER_LZOP 11
#define ARCHIVE_FILTER_GRZIP 12
#define ARCHIVE_FILTER_LZ4 13
#define ARCHIVE_FILTER_ZSTD 14
#if ARCHIVE_VERSION_NUMBER < 4000000
#define ARCHIVE_COMPRESSION_NONE ARCHIVE_FILTER_NONE
@ -433,6 +435,7 @@ __LA_DECL int archive_read_support_filter_program_signature
__LA_DECL int archive_read_support_filter_rpm(struct archive *);
__LA_DECL int archive_read_support_filter_uu(struct archive *);
__LA_DECL int archive_read_support_filter_xz(struct archive *);
__LA_DECL int archive_read_support_filter_zstd(struct archive *);
__LA_DECL int archive_read_support_format_7zip(struct archive *);
__LA_DECL int archive_read_support_format_all(struct archive *);
@ -778,6 +781,7 @@ __LA_DECL int archive_write_add_filter_program(struct archive *,
const char *cmd);
__LA_DECL int archive_write_add_filter_uuencode(struct archive *);
__LA_DECL int archive_write_add_filter_xz(struct archive *);
__LA_DECL int archive_write_add_filter_zstd(struct archive *);
/* A convenience function to set the format based on the code or name. */

6
libarchive/archive_cmdline.c
View File

@ -100,10 +100,10 @@ get_argument(struct archive_string *as, const char *p)
/*
* Set up command line arguments.
* Returns ARChIVE_OK if everything okey.
* Returns ARChIVE_FAILED if there is a lack of the `"' terminator or an
* Returns ARCHIVE_OK if everything okey.
* Returns ARCHIVE_FAILED if there is a lack of the `"' terminator or an
* empty command line.
* Returns ARChIVE_FATAL if no memory.
* Returns ARCHIVE_FATAL if no memory.
*/
int
__archive_cmdline_parse(struct archive_cmdline *data, const char *cmd)

11
libarchive/archive_platform.h
View File

@ -52,6 +52,17 @@
#error Oops: No config.h and no pre-built configuration in archive_platform.h.
#endif
/* On macOS check for some symbols based on the deployment target version. */
#if defined(__APPLE__)
# undef HAVE_FUTIMENS
# undef HAVE_UTIMENSAT
# include <AvailabilityMacros.h>
# if MAC_OS_X_VERSION_MIN_REQUIRED >= 101300
# define HAVE_FUTIMENS 1
# define HAVE_UTIMENSAT 1
# endif
#endif
/* It should be possible to get rid of this by extending the feature-test
* macros to cover Windows API functions, probably along with non-trivial
* refactoring of code to find structures that sit more cleanly on top of

4
libarchive/archive_read_append_filter.c
View File

@ -89,6 +89,10 @@ archive_read_append_filter(struct archive *_a, int code)
strcpy(str, "lz4");
r1 = archive_read_support_filter_lz4(_a);
break;
case ARCHIVE_FILTER_ZSTD:
strcpy(str, "zstd");
r1 = archive_read_support_filter_zstd(_a);
break;
case ARCHIVE_FILTER_LZIP:
strcpy(str, "lzip");
r1 = archive_read_support_filter_lzip(_a);

2
libarchive/archive_read_disk_entry_from_file.c
View File

@ -127,7 +127,7 @@ archive_read_disk_entry_setup_acls(struct archive_read_disk *a,
/*
* Enter working directory and return working pathname of archive_entry.
* If a pointer to an integer is provided and its value is below zero
* open a file descriptor on this pahtname.
* open a file descriptor on this pathname.
*/
const char *
archive_read_disk_entry_setup_path(struct archive_read_disk *a,

6
libarchive/archive_read_filter.3
View File

@ -38,6 +38,7 @@
.Nm archive_read_support_filter_rpm ,
.Nm archive_read_support_filter_uu ,
.Nm archive_read_support_filter_xz ,
.Nm archive_read_support_filter_zstd ,
.Nm archive_read_support_filter_program ,
.Nm archive_read_support_filter_program_signature
.Nd functions for reading streaming archives
@ -73,6 +74,8 @@ Streaming Archive Library (libarchive, -larchive)
.Ft int
.Fn archive_read_support_filter_xz "struct archive *"
.Ft int
.Fn archive_read_support_filter_zstd "struct archive *"
.Ft int
.Fo archive_read_support_filter_program
.Fa "struct archive *"
.Fa "const char *cmd"
@ -99,7 +102,8 @@ Streaming Archive Library (libarchive, -larchive)
.Fn archive_read_support_filter_none ,
.Fn archive_read_support_filter_rpm ,
.Fn archive_read_support_filter_uu ,
.Fn archive_read_support_filter_xz
.Fn archive_read_support_filter_xz ,
.Fn archive_read_support_filter_zstd ,
.Xc
Enables auto-detection code and decompression support for the
specified compression.

2
libarchive/archive_read_support_filter_all.c
View File

@ -71,6 +71,8 @@ archive_read_support_filter_all(struct archive *a)
archive_read_support_filter_grzip(a);
/* Lz4 falls back to "lz4 -d" command-line program. */
archive_read_support_filter_lz4(a);
/* Zstd falls back to "zstd -d" command-line program. */
archive_read_support_filter_zstd(a);
/* Note: We always return ARCHIVE_OK here, even if some of the
* above return ARCHIVE_WARN. The intent here is to enable

292
libarchive/archive_read_support_filter_zstd.c Normal file
View File

@ -0,0 +1,292 @@
/*-
* Copyright (c) 2009-2011 Sean Purcell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "archive_platform.h"
__FBSDID("$FreeBSD$");
#ifdef HAVE_ERRNO_H
#include <errno.h>
#endif
#ifdef HAVE_ERRNO_H
#include <errno.h>
#endif
#include <stdio.h>
#ifdef HAVE_STDLIB_H
#include <stdlib.h>
#endif
#ifdef HAVE_STRING_H
#include <string.h>
#endif
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#if HAVE_ZSTD_H
#include <zstd.h>
#endif
#include "archive.h"
#include "archive_endian.h"
#include "archive_private.h"
#include "archive_read_private.h"
#if HAVE_ZSTD_H && HAVE_LIBZSTD
struct private_data {
ZSTD_DStream *dstream;
unsigned char *out_block;
size_t out_block_size;
int64_t total_out;
char in_frame; /* True = in the middle of a zstd frame. */
char eof; /* True = found end of compressed data. */
};
/* Zstd Filter. */
static ssize_t zstd_filter_read(struct archive_read_filter *, const void**);
static int zstd_filter_close(struct archive_read_filter *);
#endif
/*
* Note that we can detect zstd compressed files even if we can't decompress
* them. (In fact, we like detecting them because we can give better error
* messages.) So the bid framework here gets compiled even if no zstd library
* is available.
*/
static int zstd_bidder_bid(struct archive_read_filter_bidder *,
struct archive_read_filter *);
static int zstd_bidder_init(struct archive_read_filter *);
int
archive_read_support_filter_zstd(struct archive *_a)
{
struct archive_read *a = (struct archive_read *)_a;
struct archive_read_filter_bidder *bidder;
archive_check_magic(_a, ARCHIVE_READ_MAGIC,
ARCHIVE_STATE_NEW, "archive_read_support_filter_zstd");
if (__archive_read_get_bidder(a, &bidder) != ARCHIVE_OK)
return (ARCHIVE_FATAL);
bidder->data = NULL;
bidder->name = "zstd";
bidder->bid = zstd_bidder_bid;
bidder->init = zstd_bidder_init;
bidder->options = NULL;
bidder->free = NULL;
#if HAVE_ZSTD_H && HAVE_LIBZSTD
return (ARCHIVE_OK);
#else
archive_set_error(_a, ARCHIVE_ERRNO_MISC,
"Using external zstd program for zstd decompression");
return (ARCHIVE_WARN);
#endif
}
/*
* Test whether we can handle this data.
*/
static int
zstd_bidder_bid(struct archive_read_filter_bidder *self,
struct archive_read_filter *filter)
{
const unsigned char *buffer;
ssize_t avail;
unsigned prefix;
/* Zstd frame magic values */
const unsigned zstd_magic = 0xFD2FB528U;
(void) self; /* UNUSED */
buffer = __archive_read_filter_ahead(filter, 4, &avail);
if (buffer == NULL)
return (0);
prefix = archive_le32dec(buffer);
if (prefix == zstd_magic)
return (32);
return (0);
}
#if !(HAVE_ZSTD_H && HAVE_LIBZSTD)
/*
* If we don't have the library on this system, we can't do the
* decompression directly. We can, however, try to run "zstd -d"
* in case that's available.
*/
static int
zstd_bidder_init(struct archive_read_filter *self)
{
int r;
r = __archive_read_program(self, "zstd -d -qq");
/* Note: We set the format here even if __archive_read_program()
* above fails. We do, after all, know what the format is
* even if we weren't able to read it. */
self->code = ARCHIVE_FILTER_ZSTD;
self->name = "zstd";
return (r);
}
#else
/*
* Initialize the filter object
*/
static int
zstd_bidder_init(struct archive_read_filter *self)
{
struct private_data *state;
const size_t out_block_size = ZSTD_DStreamOutSize();
void *out_block;
ZSTD_DStream *dstream;
self->code = ARCHIVE_FILTER_ZSTD;
self->name = "zstd";
state = (struct private_data *)calloc(sizeof(*state), 1);
out_block = (unsigned char *)malloc(out_block_size);
dstream = ZSTD_createDStream();
if (state == NULL || out_block == NULL || dstream == NULL) {
free(out_block);
free(state);
ZSTD_freeDStream(dstream); /* supports free on NULL */
archive_set_error(&self->archive->archive, ENOMEM,
"Can't allocate data for zstd decompression");
return (ARCHIVE_FATAL);
}
self->data = state;
state->out_block_size = out_block_size;
state->out_block = out_block;
state->dstream = dstream;
self->read = zstd_filter_read;
self->skip = NULL; /* not supported */
self->close = zstd_filter_close;
state->eof = 0;
state->in_frame = 0;
return (ARCHIVE_OK);
}
static ssize_t
zstd_filter_read(struct archive_read_filter *self, const void **p)
{
struct private_data *state;
size_t decompressed;
ssize_t avail_in;
ZSTD_outBuffer out;
ZSTD_inBuffer in;
state = (struct private_data *)self->data;
out = (ZSTD_outBuffer) { state->out_block, state->out_block_size, 0 };
/* Try to fill the output buffer. */
while (out.pos < out.size && !state->eof) {
if (!state->in_frame) {
const size_t ret = ZSTD_initDStream(state->dstream);
if (ZSTD_isError(ret)) {
archive_set_error(&self->archive->archive,
ARCHIVE_ERRNO_MISC,
"Error initializing zstd decompressor: %s",
ZSTD_getErrorName(ret));
return (ARCHIVE_FATAL);
}
}
in.src = __archive_read_filter_ahead(self->upstream, 1,
&avail_in);
if (avail_in < 0) {
return avail_in;
}
if (in.src == NULL && avail_in == 0) {
if (!state->in_frame) {
/* end of stream */
state->eof = 1;
break;
} else {
archive_set_error(&self->archive->archive,
ARCHIVE_ERRNO_MISC,
"Truncated zstd input");
return (ARCHIVE_FATAL);
}
}
in.size = avail_in;
in.pos = 0;
{
const size_t ret =
ZSTD_decompressStream(state->dstream, &out, &in);
if (ZSTD_isError(ret)) {
archive_set_error(&self->archive->archive,
ARCHIVE_ERRNO_MISC,
"Zstd decompression failed: %s",
ZSTD_getErrorName(ret));
return (ARCHIVE_FATAL);
}
/* Decompressor made some progress */
__archive_read_filter_consume(self->upstream, in.pos);
/* ret guaranteed to be > 0 if frame isn't done yet */
state->in_frame = (ret != 0);
}
}
decompressed = out.pos;
state->total_out += decompressed;
if (decompressed == 0)
*p = NULL;
else
*p = state->out_block;
return (decompressed);
}
/*
* Clean up the decompressor.
*/
static int
zstd_filter_close(struct archive_read_filter *self)
{
struct private_data *state;
state = (struct private_data *)self->data;
ZSTD_freeDStream(state->dstream);
free(state->out_block);
free(state);
return (ARCHIVE_OK);
}
#endif /* HAVE_ZLIB_H && HAVE_LIBZSTD */

7
libarchive/archive_read_support_format_cpio.c
View File

@ -633,6 +633,13 @@ header_newc(struct archive_read *a, struct cpio *cpio,
/* Pad name to 2 more than a multiple of 4. */
*name_pad = (2 - *namelength) & 3;
/* Make sure that the padded name length fits into size_t. */
if (*name_pad > SIZE_MAX - *namelength) {
archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT,
"cpio archive has invalid namelength");
return (ARCHIVE_FATAL);
}
/*
* Note: entry_bytes_remaining is at least 64 bits and
* therefore guaranteed to be big enough for a 33-bit file

10
libarchive/archive_read_support_format_mtree.c
View File

@ -77,6 +77,8 @@ __FBSDID("$FreeBSD: head/lib/libarchive/archive_read_support_format_mtree.c 2011
#define MTREE_HASHTABLE_SIZE 1024
#define MAX_LINE_LEN (1024 * 1024)
struct mtree_option {
struct mtree_option *next;
char *value;
@ -334,6 +336,14 @@ next_line(struct archive_read *a,
size_t nbytes_req = (*ravail+1023) & ~1023U;
ssize_t tested;
/*
* Place an arbitrary limit on the line length.
* mtree is almost free-form input and without line length limits,
* it can consume a lot of memory.
*/
if (len >= MAX_LINE_LEN)
return (-1);
/* Increase reading bytes if it is not enough to at least
* new two lines. */
if (nbytes_req < (size_t)*ravail + 160)

6
libarchive/archive_read_support_format_rar.c
View File

@ -1496,7 +1496,11 @@ read_header(struct archive_read *a, struct archive_entry *entry,
return (ARCHIVE_FATAL);
}
filename[filename_size++] = '\0';
filename[filename_size++] = '\0';
/*
* Do not increment filename_size here as the computations below
* add the space for the terminating NUL explicitly.
*/
filename[filename_size] = '\0';
/* Decoded unicode form is UTF-16BE, so we have to update a string
* conversion object for it. */

2
libarchive/archive_read_support_format_tar.c
View File

@ -2241,7 +2241,7 @@ gnu_add_sparse_entry(struct archive_read *a, struct tar *tar,
else
tar->sparse_list = p;
tar->sparse_last = p;
if (remaining < 0 || offset < 0) {
if (remaining < 0 || offset < 0 || offset > INT64_MAX - remaining) {
archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, "Malformed sparse map data");
return (ARCHIVE_FATAL);
}

32
libarchive/archive_read_support_format_xar.c
View File

@ -1040,6 +1040,9 @@ atol10(const char *p, size_t char_cnt)
uint64_t l;
int digit;
if (char_cnt == 0)
return (0);
l = 0;
digit = *p - '0';
while (digit >= 0 && digit < 10 && char_cnt-- > 0) {
@ -1054,7 +1057,10 @@ atol8(const char *p, size_t char_cnt)
{
int64_t l;
int digit;
if (char_cnt == 0)
return (0);
l = 0;
while (char_cnt-- > 0) {
if (*p >= '0' && *p <= '7')
@ -2623,6 +2629,14 @@ strappend_base64(struct xar *xar,
archive_strncat(as, (const char *)buff, len);
}
static int
is_string(const char *known, const char *data, size_t len)
{
if (strlen(known) != len)
return -1;
return memcmp(data, known, len);
}
static void
xml_data(void *userData, const char *s, int len)
{
@ -2674,26 +2688,26 @@ xml_data(void *userData, const char *s, int len)
archive_strncpy(&(xar->file->symlink), s, len);
break;
case FILE_TYPE:
if (strncmp("file", s, len) == 0 ||
strncmp("hardlink", s, len) == 0)
if (is_string("file", s, len) == 0 ||
is_string("hardlink", s, len) == 0)
xar->file->mode =
(xar->file->mode & ~AE_IFMT) | AE_IFREG;
if (strncmp("directory", s, len) == 0)
if (is_string("directory", s, len) == 0)
xar->file->mode =
(xar->file->mode & ~AE_IFMT) | AE_IFDIR;
if (strncmp("symlink", s, len) == 0)
if (is_string("symlink", s, len) == 0)
xar->file->mode =
(xar->file->mode & ~AE_IFMT) | AE_IFLNK;
if (strncmp("character special", s, len) == 0)
if (is_string("character special", s, len) == 0)
xar->file->mode =
(xar->file->mode & ~AE_IFMT) | AE_IFCHR;
if (strncmp("block special", s, len) == 0)
if (is_string("block special", s, len) == 0)
xar->file->mode =
(xar->file->mode & ~AE_IFMT) | AE_IFBLK;
if (strncmp("socket", s, len) == 0)
if (is_string("socket", s, len) == 0)
xar->file->mode =
(xar->file->mode & ~AE_IFMT) | AE_IFSOCK;
if (strncmp("fifo", s, len) == 0)
if (is_string("fifo", s, len) == 0)
xar->file->mode =
(xar->file->mode & ~AE_IFMT) | AE_IFIFO;
xar->file->has |= HAS_TYPE;

5
libarchive/archive_read_support_format_zip.c
View File

@ -723,6 +723,11 @@ process_extra(struct archive_read *a, const char *p, size_t extra_length, struct
}
case 0x9901:
/* WinZip AES extra data field. */
if (datasize < 6) {
archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT,
"Incomplete AES field");
return ARCHIVE_FAILED;
}
if (p[offset + 2] == 'A' && p[offset + 3] == 'E') {
/* Vendor version. */
zip_entry->aes_extra.vendor =

3
libarchive/archive_string.c
View File

@ -214,7 +214,8 @@ archive_wstring_append(struct archive_wstring *as, const wchar_t *p, size_t s)
{
if (archive_wstring_ensure(as, as->length + s + 1) == NULL)
return (NULL);
wmemmove(as->s + as->length, p, s);
if (s)
wmemmove(as->s + as->length, p, s);
as->length += s;
as->s[as->length] = 0;
return (as);

18
libarchive/archive_version_details.c
View File

@ -45,6 +45,9 @@ __FBSDID("$FreeBSD$");
#ifdef HAVE_LZ4_H
#include <lz4.h>
#endif
#ifdef HAVE_ZSTD_H
#include <zstd.h>
#endif
#include "archive.h"
#include "archive_private.h"
@ -59,6 +62,7 @@ archive_version_details(void)
const char *liblzma = archive_liblzma_version();
const char *bzlib = archive_bzlib_version();
const char *liblz4 = archive_liblz4_version();
const char *libzstd = archive_libzstd_version();
if (!init) {
archive_string_init(&str);
@ -84,6 +88,10 @@ archive_version_details(void)
archive_strcat(&str, " liblz4/");
archive_strcat(&str, liblz4);
}
if (libzstd) {
archive_strcat(&str, " libzstd/");
archive_strcat(&str, libzstd);
}
}
return str.s;
}
@ -131,3 +139,13 @@ archive_liblz4_version(void)
return NULL;
#endif
}
const char *
archive_libzstd_version(void)
{
#if HAVE_ZSTD_H && HAVE_LIBZSTD
return ZSTD_VERSION_STRING;
#else
return NULL;
#endif
}

2
libarchive/archive_write.3
View File

@ -71,7 +71,7 @@ support.
.\"
.Ss Set options
See
.Xr archive_read_set_options 3 .
.Xr archive_write_set_options 3 .
.\"
.Ss Open archive
See

1
libarchive/archive_write_add_filter.c
View File

@ -53,6 +53,7 @@ struct { int code; int (*setter)(struct archive *); } codes[] =
{ ARCHIVE_FILTER_LZOP, archive_write_add_filter_lzip },
{ ARCHIVE_FILTER_UU, archive_write_add_filter_uuencode },
{ ARCHIVE_FILTER_XZ, archive_write_add_filter_xz },
{ ARCHIVE_FILTER_ZSTD, archive_write_add_filter_zstd },
{ -1, NULL }
};

1
libarchive/archive_write_add_filter_by_name.c
View File

@ -57,6 +57,7 @@ struct { const char *name; int (*setter)(struct archive *); } names[] =
{ "lzop", archive_write_add_filter_lzop },
{ "uuencode", archive_write_add_filter_uuencode },
{ "xz", archive_write_add_filter_xz },
{ "zstd", archive_write_add_filter_zstd },
{ NULL, NULL }
};

335
libarchive/archive_write_add_filter_zstd.c Normal file
View File

@ -0,0 +1,335 @@
/*-
* Copyright (c) 2017 Sean Purcell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "archive_platform.h"
__FBSDID("$FreeBSD$");
#ifdef HAVE_ERRNO_H
#include <errno.h>
#endif
#ifdef HAVE_STDLIB_H
#include <stdlib.h>
#endif
#ifdef HAVE_STRING_H
#include <string.h>
#endif
#ifdef HAVE_ZSTD_H
#include <zstd.h>
#endif
#include "archive.h"
#include "archive_private.h"
#include "archive_string.h"
#include "archive_write_private.h"
/* Don't compile this if we don't have zstd.h */
struct private_data {
int compression_level;
#if HAVE_ZSTD_H && HAVE_LIBZSTD
ZSTD_CStream *cstream;
int64_t total_in;
ZSTD_outBuffer out;
#else
struct archive_write_program_data *pdata;
#endif
};
static int archive_compressor_zstd_options(struct archive_write_filter *,
const char *, const char *);
static int archive_compressor_zstd_open(struct archive_write_filter *);
static int archive_compressor_zstd_write(struct archive_write_filter *,
const void *, size_t);
static int archive_compressor_zstd_close(struct archive_write_filter *);
static int archive_compressor_zstd_free(struct archive_write_filter *);
#if HAVE_ZSTD_H && HAVE_LIBZSTD
static int drive_compressor(struct archive_write_filter *,
struct private_data *, int, const void *, size_t);
#endif
/*
* Add a zstd compression filter to this write handle.
*/
int
archive_write_add_filter_zstd(struct archive *_a)
{
struct archive_write *a = (struct archive_write *)_a;
struct archive_write_filter *f = __archive_write_allocate_filter(_a);
struct private_data *data;
archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,
ARCHIVE_STATE_NEW, "archive_write_add_filter_zstd");
data = calloc(1, sizeof(*data));
if (data == NULL) {
archive_set_error(&a->archive, ENOMEM, "Out of memory");
return (ARCHIVE_FATAL);
}
f->data = data;
f->open = &archive_compressor_zstd_open;
f->options = &archive_compressor_zstd_options;
f->close = &archive_compressor_zstd_close;
f->free = &archive_compressor_zstd_free;
f->code = ARCHIVE_FILTER_ZSTD;
f->name = "zstd";
data->compression_level = 3; /* Default level used by the zstd CLI */
#if HAVE_ZSTD_H && HAVE_LIBZSTD
data->cstream = ZSTD_createCStream();
if (data->cstream == NULL) {
free(data);
archive_set_error(&a->archive, ENOMEM,
"Failed to allocate zstd compressor object");
return (ARCHIVE_FATAL);
}
return (ARCHIVE_OK);
#else
data->pdata = __archive_write_program_allocate("zstd");
if (data->pdata == NULL) {
free(data);
archive_set_error(&a->archive, ENOMEM, "Out of memory");
return (ARCHIVE_FATAL);
}
archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
"Using external zstd program");
return (ARCHIVE_WARN);
#endif
}
static int
archive_compressor_zstd_free(struct archive_write_filter *f)
{
struct private_data *data = (struct private_data *)f->data;
#if HAVE_ZSTD_H && HAVE_LIBZSTD
ZSTD_freeCStream(data->cstream);
free(data->out.dst);
#else
__archive_write_program_free(data->pdata);
#endif
free(data);
f->data = NULL;
return (ARCHIVE_OK);
}
/*
* Set write options.
*/
static int
archive_compressor_zstd_options(struct archive_write_filter *f, const char *key,
const char *value)
{
struct private_data *data = (struct private_data *)f->data;
if (strcmp(key, "compression-level") == 0) {
int level = atoi(value);
#if HAVE_ZSTD_H && HAVE_LIBZSTD
if (level < 1 || level > ZSTD_maxCLevel()) {
#else
/* If we don't have the library, hard-code the max level */
if (level < 1 || level > 22) {
#endif
return (ARCHIVE_WARN);
}
data->compression_level = level;
return (ARCHIVE_OK);
}
/* Note: The "warn" return is just to inform the options
* supervisor that we didn't handle it. It will generate
* a suitable error if no one used this option. */
return (ARCHIVE_WARN);
}
#if HAVE_ZSTD_H && HAVE_LIBZSTD
/*
* Setup callback.
*/
static int
archive_compressor_zstd_open(struct archive_write_filter *f)
{
struct private_data *data = (struct private_data *)f->data;
int ret;
ret = __archive_write_open_filter(f->next_filter);
if (ret != ARCHIVE_OK)
return (ret);
if (data->out.dst == NULL) {
size_t bs = ZSTD_CStreamOutSize(), bpb;
if (f->archive->magic == ARCHIVE_WRITE_MAGIC) {
/* Buffer size should be a multiple number of
* the of bytes per block for performance. */
bpb = archive_write_get_bytes_per_block(f->archive);
if (bpb > bs)
bs = bpb;
else if (bpb != 0)
bs -= bs % bpb;
}
data->out.size = bs;
data->out.pos = 0;
data->out.dst
= (unsigned char *)malloc(data->out.size);
if (data->out.dst == NULL) {
archive_set_error(f->archive, ENOMEM,
"Can't allocate data for compression buffer");
return (ARCHIVE_FATAL);
}
}
f->write = archive_compressor_zstd_write;
if (ZSTD_isError(ZSTD_initCStream(data->cstream,
data->compression_level))) {
archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
"Internal error initializing zstd compressor object");
return (ARCHIVE_FATAL);
}
return (ARCHIVE_OK);
}
/*
* Write data to the compressed stream.
*/
static int
archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff,
size_t length)
{
struct private_data *data = (struct private_data *)f->data;
int ret;
/* Update statistics */
data->total_in += length;
if ((ret = drive_compressor(f, data, 0, buff, length)) != ARCHIVE_OK)
return (ret);
return (ARCHIVE_OK);
}
/*
* Finish the compression...
*/
static int
archive_compressor_zstd_close(struct archive_write_filter *f)
{
struct private_data *data = (struct private_data *)f->data;
int r1, r2;
/* Finish zstd frame */
r1 = drive_compressor(f, data, 1, NULL, 0);
r2 = __archive_write_close_filter(f->next_filter);
return r1 < r2 ? r1 : r2;
}
/*
* Utility function to push input data through compressor,
* writing full output blocks as necessary.
*
* Note that this handles both the regular write case (finishing ==
* false) and the end-of-archive case (finishing == true).
*/
static int
drive_compressor(struct archive_write_filter *f,
struct private_data *data, int finishing, const void *src, size_t length)
{
ZSTD_inBuffer in = (ZSTD_inBuffer) { src, length, 0 };
for (;;) {
if (data->out.pos == data->out.size) {
const int ret = __archive_write_filter(f->next_filter,
data->out.dst, data->out.size);
if (ret != ARCHIVE_OK)
return (ARCHIVE_FATAL);
data->out.pos = 0;
}
/* If there's nothing to do, we're done. */
if (!finishing && in.pos == in.size)
return (ARCHIVE_OK);
{
const size_t zstdret = !finishing ?
ZSTD_compressStream(data->cstream, &data->out, &in)
: ZSTD_endStream(data->cstream, &data->out);
if (ZSTD_isError(zstdret)) {
archive_set_error(f->archive,
ARCHIVE_ERRNO_MISC,
"Zstd compression failed: %s",
ZSTD_getErrorName(zstdret));
return (ARCHIVE_FATAL);
}
/* If we're finishing, 0 means nothing left to flush */
if (finishing && zstdret == 0) {
const int ret = __archive_write_filter(f->next_filter,
data->out.dst, data->out.pos);
return (ret);
}
}
}
}
#else /* HAVE_ZSTD_H && HAVE_LIBZSTD */
static int
archive_compressor_zstd_open(struct archive_write_filter *f)
{
struct private_data *data = (struct private_data *)f->data;
struct archive_string as;
int r;
archive_string_init(&as);
archive_string_sprintf(&as, "zstd -%d", data->compression_level);
f->write = archive_compressor_zstd_write;
r = __archive_write_program_open(f, data->pdata, as.s);
archive_string_free(&as);
return (r);
}
static int
archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff,
size_t length)
{
struct private_data *data = (struct private_data *)f->data;
return __archive_write_program_write(f, data->pdata, buff, length);
}
static int
archive_compressor_zstd_close(struct archive_write_filter *f)
{
struct private_data *data = (struct private_data *)f->data;
return __archive_write_program_close(f, data->pdata);
}
#endif /* HAVE_ZSTD_H && HAVE_LIBZSTD */

6
libarchive/archive_write_filter.3
View File

@ -42,7 +42,8 @@
.Nm archive_write_add_filter_none ,
.Nm archive_write_add_filter_program ,
.Nm archive_write_add_filter_uuencode ,
.Nm archive_write_add_filter_xz
.Nm archive_write_add_filter_xz ,
.Nm archive_write_add_filter_zstd ,
.Nd functions enabling output filters
.Sh LIBRARY
Streaming Archive Library (libarchive, -larchive)
@ -76,6 +77,8 @@ Streaming Archive Library (libarchive, -larchive)
.Fn archive_write_add_filter_uuencode "struct archive *"
.Ft int
.Fn archive_write_add_filter_xz "struct archive *"
.Ft int
.Fn archive_write_add_filter_zstd "struct archive *"
.Sh DESCRIPTION
.Bl -tag -width indent
.It Xo
@ -89,6 +92,7 @@ Streaming Archive Library (libarchive, -larchive)
.Fn archive_write_add_filter_lzma ,
.Fn archive_write_add_filter_lzop ,
.Fn archive_write_add_filter_xz ,
.Fn archive_write_add_filter_zstd ,
.Xc
The resulting archive will be compressed as specified.
Note that the compressed output is always properly blocked.

2
libarchive/archive_write_set_format_pax.c
View File

@ -1654,7 +1654,7 @@ build_pax_attribute_name(char *dest, const char *src)
* GNU PAX Format 1.0 requires the special name, which pattern is:
* <dir>/GNUSparseFile.<pid>/<original file name>
*
* Since reproducable archives are more important, use 0 as pid.
* Since reproducible archives are more important, use 0 as pid.
*
* This function is used for only Sparse file, a file type of which
* is regular file.

2
libarchive/test/CMakeLists.txt
View File

@ -69,6 +69,7 @@ IF(ENABLE_TEST)
test_compat_uudecode_large.c
test_compat_xz.c
test_compat_zip.c
test_compat_zstd.c
test_empty_write.c
test_entry.c
test_entry_strmode.c
@ -226,6 +227,7 @@ IF(ENABLE_TEST)
test_write_filter_program.c
test_write_filter_uuencode.c
test_write_filter_xz.c
test_write_filter_zstd.c
test_write_format_7zip.c
test_write_format_7zip_empty.c
test_write_format_7zip_large.c

5
libarchive/test/test_archive_write_add_filter_by_name.c
View File

@ -207,3 +207,8 @@ DEFINE_TEST(test_archive_write_add_filter_by_name_xz)
{
test_filter_by_name("xz", ARCHIVE_FILTER_XZ, cannot);
}
DEFINE_TEST(test_archive_write_add_filter_by_name_zstd)
{
test_filter_by_name("zstd", ARCHIVE_FILTER_ZSTD, canZstd);
}

82
libarchive/test/test_compat_zstd.c Normal file
View File

@ -0,0 +1,82 @@
/*-
* Copyright (c) 2017 Sean Purcell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "test.h"
__FBSDID("$FreeBSD$");
/*
* Verify our ability to read sample files compatibly with 'zstd -d'.
*
* In particular:
* * zstd -d will read multiple zstd streams, concatenating the output
* * zstd -d will skip over zstd skippable frames
*/
static void
compat_zstd(const char *name)
{
const char *n[7] = { "f1", "f2", "f3", "d1/f1", "d1/f2", "d1/f3", NULL };
struct archive_entry *ae;
struct archive *a;
int i, r;
assert((a = archive_read_new()) != NULL);
assertEqualIntA(a, ARCHIVE_OK, archive_read_support_filter_all(a));
r = archive_read_support_filter_zstd(a);
if (r == ARCHIVE_WARN) {
skipping("zstd reading not fully supported on this platform");
assertEqualInt(ARCHIVE_OK, archive_read_free(a));
return;
}
assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a));
extract_reference_file(name);
assertEqualIntA(a, ARCHIVE_OK, archive_read_open_filename(a, name, 2));
/* Read entries, match up names with list above. */
for (i = 0; i < 6; ++i) {
failure("Could not read file %d (%s) from %s", i, n[i], name);
assertEqualIntA(a, ARCHIVE_OK,
archive_read_next_header(a, &ae));
assertEqualString(n[i], archive_entry_pathname(ae));
}
/* Verify the end-of-archive. */
assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae));
/* Verify that the format detection worked. */
assertEqualInt(archive_filter_code(a, 0), ARCHIVE_FILTER_ZSTD);
assertEqualString(archive_filter_name(a, 0), "zstd");
assertEqualInt(archive_format(a), ARCHIVE_FORMAT_TAR_USTAR);
assertEqualInt(ARCHIVE_OK, archive_read_close(a));
assertEqualInt(ARCHIVE_OK, archive_read_free(a));
}
DEFINE_TEST(test_compat_zstd)
{
/* This sample was compressed as 3 separate streams with a zstd skippable
* frame placed in the middle */
compat_zstd("test_compat_zstd_1.tar.zst");
}

12
libarchive/test/test_compat_zstd_1.tar.zst.uu Normal file
View File

@ -0,0 +1,12 @@
begin 644 test_compat_zstd_1.tar.zst
M*+4O_010)0,`HL0.%;`Q&>>\/$2[#IQF[<1+Z3T<0CX]!77&0@R.6+/F,0+I
M.$1A$QE2`J!+*_6[_YT9_W_M1KC-EG*V>10.`,M`%3*@#F#\`-FT#J:1#U1"
M`H1!&R#<!.<"@#3@M58XY1,8`DMMD\@HM2_]!%!=`P`B!1`5H#D!0!.SELJ"
M5#509I*T/YQ^]?H/3T1D>A5\*'"JYIJ;C&4=B2CL(L)*E-IJT/RV?.:A_]_N
MB&[7SDG;/=4&#P";0!5D0`=8T0&R&19,)1^HA`0(@S9`N`G.!0!IP&NM<,K!
M-#8!%A]U]K10*DT8!`````$"`P0HM2_]!%!]`P`B11`6H+$)"%]@,Z6OH`"L
MM$R2MAN&*MSG`W?OJ7+4P*B::VXR`NM(1&$7&58"J*U'_&V^S$/_O]U1N%T[
M)VW7J'+4!A``_4$%^T`],J`8P.0!L@D63"4?J(0$"(,V0+@)S@4`:<!KK7!J
)P51V`E@!9CD#
`
end

9
libarchive/test/test_fuzz.c
View File

@ -412,6 +412,12 @@ DEFINE_TEST(test_fuzz_tar)
"test_compat_lzop_1.tar.lzo",
NULL
};
#endif
#if HAVE_ZSTD_H && HAVE_LIBZSTD
static const char *fileset10[] = {
"test_compat_zstd_1.tar.zst",
NULL
};
#endif
static const struct files filesets[] = {
{0, fileset1}, /* Exercise bzip2 decompressor. */
@ -425,6 +431,9 @@ DEFINE_TEST(test_fuzz_tar)
{0, fileset8},
#if HAVE_LIBLZO2 && HAVE_LZO_LZO1X_H && HAVE_LZO_LZOCONF_H
{0, fileset9}, /* Exercise lzo decompressor. */
#endif
#if HAVE_ZSTD_H && HAVE_LIBZSTD
{0, fileset10}, /* Excercise zstd decompressor. */
#endif
{1, NULL}
};

201
libarchive/test/test_write_filter_zstd.c Normal file
View File

@ -0,0 +1,201 @@
/*-
* Copyright (c) 2017 Sean Purcell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer
* in this position and unchanged.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "test.h"
__FBSDID("$FreeBSD$");
DEFINE_TEST(test_write_filter_zstd)
{
struct archive_entry *ae;
struct archive *a;
char *buff, *data;
size_t buffsize, datasize;
char path[16];
size_t used1, used2;
int i, r;
buffsize = 2000000;
assert(NULL != (buff = (char *)malloc(buffsize)));
if (buff == NULL)
return;
datasize = 10000;
assert(NULL != (data = (char *)malloc(datasize)));
if (data == NULL) {
free(buff);
return;
}
memset(data, 0, datasize);
/*
* Write a 100 files and read them all back.
*/
assert((a = archive_write_new()) != NULL);
assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a));
r = archive_write_add_filter_zstd(a);
if (r != ARCHIVE_OK) {
skipping("zstd writing not supported on this platform");
assertEqualInt(ARCHIVE_OK, archive_write_free(a));
free(buff);
free(data);
return;
}
assertEqualIntA(a, ARCHIVE_OK,
archive_write_set_bytes_per_block(a, 10));
assertEqualInt(ARCHIVE_FILTER_ZSTD, archive_filter_code(a, 0));
assertEqualString("zstd", archive_filter_name(a, 0));
assertEqualIntA(a, ARCHIVE_OK, archive_write_open_memory(a, buff, buffsize, &used1));
assertEqualInt(ARCHIVE_FILTER_ZSTD, archive_filter_code(a, 0));
assertEqualString("zstd", archive_filter_name(a, 0));
assert((ae = archive_entry_new()) != NULL);
archive_entry_set_filetype(ae, AE_IFREG);
archive_entry_set_size(ae, datasize);
for (i = 0; i < 100; i++) {
sprintf(path, "file%03d", i);
archive_entry_copy_pathname(ae, path);
assertEqualIntA(a, ARCHIVE_OK, archive_write_header(a, ae));
assertA(datasize
== (size_t)archive_write_data(a, data, datasize));
}
archive_entry_free(ae);
assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a));
assertEqualInt(ARCHIVE_OK, archive_write_free(a));
assert((a = archive_read_new()) != NULL);
assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a));
r = archive_read_support_filter_zstd(a);
if (r == ARCHIVE_WARN) {
skipping("Can't verify zstd writing by reading back;"
" zstd reading not fully supported on this platform");
} else {
assertEqualIntA(a, ARCHIVE_OK,
archive_read_support_filter_all(a));
assertEqualIntA(a, ARCHIVE_OK,
archive_read_open_memory(a, buff, used1));
for (i = 0; i < 100; i++) {
sprintf(path, "file%03d", i);
if (!assertEqualInt(ARCHIVE_OK,
archive_read_next_header(a, &ae)))
break;
assertEqualString(path, archive_entry_pathname(ae));
assertEqualInt((int)datasize, archive_entry_size(ae));
}
assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
}
assertEqualInt(ARCHIVE_OK, archive_read_free(a));
/*
* Repeat the cycle again, this time setting some compression
* options.
*/
assert((a = archive_write_new()) != NULL);
assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a));
assertEqualIntA(a, ARCHIVE_OK,
archive_write_set_bytes_per_block(a, 10));
assertEqualIntA(a, ARCHIVE_OK, archive_write_add_filter_zstd(a));
assertEqualIntA(a, ARCHIVE_FAILED,
archive_write_set_filter_option(a, NULL, "nonexistent-option", "0"));
assertEqualIntA(a, ARCHIVE_FAILED,
archive_write_set_filter_option(a, NULL, "compression-level", "abc"));
assertEqualIntA(a, ARCHIVE_FAILED,
archive_write_set_filter_option(a, NULL, "compression-level", "25")); /* too big */
assertEqualIntA(a, ARCHIVE_OK,
archive_write_set_filter_option(a, NULL, "compression-level", "9"));
assertEqualIntA(a, ARCHIVE_OK,
archive_write_set_filter_option(a, NULL, "compression-level", "15"));
assertEqualIntA(a, ARCHIVE_OK, archive_write_open_memory(a, buff, buffsize, &used2));
for (i = 0; i < 100; i++) {
sprintf(path, "file%03d", i);
assert((ae = archive_entry_new()) != NULL);
archive_entry_copy_pathname(ae, path);
archive_entry_set_size(ae, datasize);
archive_entry_set_filetype(ae, AE_IFREG);
assertEqualIntA(a, ARCHIVE_OK, archive_write_header(a, ae));
assertA(datasize == (size_t)archive_write_data(a, data, datasize));
archive_entry_free(ae);
}
assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a));
assertEqualInt(ARCHIVE_OK, archive_write_free(a));
failure("compression-level=15 wrote %d bytes, default wrote %d bytes",
(int)used2, (int)used1);
assert(used2 < used1);
assert((a = archive_read_new()) != NULL);
assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a));
r = archive_read_support_filter_zstd(a);
if (r == ARCHIVE_WARN) {
skipping("zstd reading not fully supported on this platform");
} else {
assertEqualIntA(a, ARCHIVE_OK,
archive_read_support_filter_all(a));
assertEqualIntA(a, ARCHIVE_OK,
archive_read_open_memory(a, buff, used2));
for (i = 0; i < 100; i++) {
sprintf(path, "file%03d", i);
failure("Trying to read %s", path);
if (!assertEqualIntA(a, ARCHIVE_OK,
archive_read_next_header(a, &ae)))
break;
assertEqualString(path, archive_entry_pathname(ae));
assertEqualInt((int)datasize, archive_entry_size(ae));
}
assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
}
assertEqualInt(ARCHIVE_OK, archive_read_free(a));
/*
* Test various premature shutdown scenarios to make sure we
* don't crash or leak memory.
*/
assert((a = archive_write_new()) != NULL);
assertEqualIntA(a, ARCHIVE_OK, archive_write_add_filter_zstd(a));
assertEqualInt(ARCHIVE_OK, archive_write_free(a));
assert((a = archive_write_new()) != NULL);
assertEqualIntA(a, ARCHIVE_OK, archive_write_add_filter_zstd(a));
assertEqualInt(ARCHIVE_OK, archive_write_close(a));
assertEqualInt(ARCHIVE_OK, archive_write_free(a));
assert((a = archive_write_new()) != NULL);
assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a));
assertEqualIntA(a, ARCHIVE_OK, archive_write_add_filter_zstd(a));
assertEqualInt(ARCHIVE_OK, archive_write_close(a));
assertEqualInt(ARCHIVE_OK, archive_write_free(a));
assert((a = archive_write_new()) != NULL);
assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a));
assertEqualIntA(a, ARCHIVE_OK, archive_write_add_filter_zstd(a));
assertEqualIntA(a, ARCHIVE_OK, archive_write_open_memory(a, buff, buffsize, &used2));
assertEqualInt(ARCHIVE_OK, archive_write_close(a));
assertEqualInt(ARCHIVE_OK, archive_write_free(a));
/*
* Clean up.
*/
free(data);
free(buff);
}

6
tar/bsdtar.1
View File

@ -342,6 +342,10 @@ In extract or list modes, this option is ignored.
Compress the archive with lz4-compatible compression before writing it.
In input mode, this option is ignored; lz4 compression is recognized
automatically on input.
.It Fl Fl zstd
Compress the archive with zstd-compatible compression before writing it.
In input mode, this option is ignored; lz4 compression is recognized
automatically on input.
.It Fl Fl lzma
(c mode only) Compress the resulting archive with the original LZMA algorithm.
Use of this option is discouraged and new archives should be created with
@ -577,6 +581,8 @@ A decimal integer from 4 to 7 specifying the lz4 compression block size
.It Cm lz4:block-dependence
Use the previous block of the block being compressed for
a compression dictionary to improve compression ratio.
.It Cm zstd:compression-level
A decimal integer from 1 to 22 specifying the zstd compression level.
.It Cm lzop:compression-level
A decimal integer from 1 to 9 specifying the lzop compression level.
.It Cm xz:compression-level

8
tar/bsdtar.c
View File

@ -419,6 +419,7 @@ main(int argc, char **argv)
case OPTION_LZIP: /* GNU tar beginning with 1.23 */
case OPTION_LZMA: /* GNU tar beginning with 1.20 */
case OPTION_LZOP: /* GNU tar beginning with 1.21 */
case OPTION_ZSTD:
if (compression != '\0')
lafe_errc(1, 0,
"Can't specify both -%c and -%c", opt,
@ -427,9 +428,10 @@ main(int argc, char **argv)
switch (opt) {
case OPTION_LRZIP: compression_name = "lrzip"; break;
case OPTION_LZ4: compression_name = "lz4"; break;
case OPTION_LZIP: compression_name = "lzip"; break;
case OPTION_LZMA: compression_name = "lzma"; break;
case OPTION_LZOP: compression_name = "lzop"; break;
case OPTION_LZIP: compression_name = "lzip"; break;
case OPTION_LZMA: compression_name = "lzma"; break;
case OPTION_LZOP: compression_name = "lzop"; break;
case OPTION_ZSTD: compression_name = "zstd"; break;
}
break;
case 'm': /* SUSv2 */

3
tar/bsdtar.h
View File

@ -181,7 +181,8 @@ enum {
OPTION_USE_COMPRESS_PROGRAM,
OPTION_UUENCODE,
OPTION_VERSION,
OPTION_XATTRS
OPTION_XATTRS,
OPTION_ZSTD,
};
int bsdtar_getopt(struct bsdtar *);

1
tar/cmdline.c
View File

@ -160,6 +160,7 @@ static const struct bsdtar_option {
{ "version", 0, OPTION_VERSION },
{ "xattrs", 0, OPTION_XATTRS },
{ "xz", 0, 'J' },
{ "zstd", 0, OPTION_ZSTD },
{ NULL, 0, 0 }
};

4
tar/creation_set.c
View File

@ -80,9 +80,10 @@ get_filter_code(const char *suffix)
{ ".lzma", "lzma" },
{ ".uu", "uuencode" },
{ ".xz", "xz" },
{ ".zst", "zstd"},
{ NULL, NULL }
};
return get_suffix_code(filters, suffix);
}
@ -121,6 +122,7 @@ decompose_alias(const char *suffix)
{ ".tzo", ".tar.lzo" },
{ ".taZ", ".tar.Z" },
{ ".tZ", ".tar.Z" },
{ ".tzst", ".tar.zst" },
{ NULL, NULL }
};

2
tar/test/CMakeLists.txt
View File

@ -22,6 +22,7 @@ IF(ENABLE_TAR AND ENABLE_TEST)
test_extract_tar_lzma.c
test_extract_tar_lzo.c
test_extract_tar_xz.c
test_extract_tar_zstd.c
test_format_newc.c
test_help.c
test_leading_slash.c
@ -61,6 +62,7 @@ IF(ENABLE_TAR AND ENABLE_TEST)
test_option_xattrs.c
test_option_xz.c
test_option_z.c
test_option_zstd.c
test_patterns.c
test_print_longpath.c
test_stdio.c

6
tar/test/test_extract.tar.zst.uu Normal file
View File

@ -0,0 +1,6 @@
begin 644 test_extract.tar.zst
M*+4O_010S0,`<L40$Z`5.(2U_RNV_[]L4V;Z_/R@1:7Y$3;9E`8$D$WI:W1)
M'58'D3->Y+>!0*5E/PM"$7^K^1VI3SS-AX&_W0KQWY!-Z1(`_4$%[$"]<T!A
L(*`#I!DXC4[J!6J8$!DJ$D"9$T*L]#G-$$/A`#`I`-(`UUKAU$Z@"`UXII``
`
end

48
tar/test/test_extract_tar_zstd.c Normal file
View File

@ -0,0 +1,48 @@
/*-
* Copyright (c) 2017 Sean Purcell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "test.h"
__FBSDID("$FreeBSD$");
DEFINE_TEST(test_extract_tar_zstd)
{
const char *reffile = "test_extract.tar.zst";
int f;
extract_reference_file(reffile);
f = systemf("%s -tf %s >test.out 2>test.err", testprog, reffile);
if (f == 0 || canZstd()) {
assertEqualInt(0, systemf("%s -xf %s >test.out 2>test.err",
testprog, reffile));
assertFileExists("file1");
assertTextFileContents("contents of file1.\n", "file1");
assertFileExists("file2");
assertTextFileContents("contents of file2.\n", "file2");
assertEmptyFile("test.out");
assertEmptyFile("test.err");
} else {
skipping("It seems zstd is not supported on this platform");
}
}

2
tar/test/test_option_acls.c
View File

@ -483,7 +483,7 @@ DEFINE_TEST(test_option_acls)
r = compare_acls("f", "acls_acls/f");
assertEqualInt(r, 1);
/* Extractl acls without acls */
/* Extract acls without acls */
assertMakeDir("acls_noacls", 0755);
clear_inheritance_flags("acls_noacls", acltype);
r = systemf("%s -x -C acls_noacls -p --no-acls -f acls.tar >acls_noacls.out 2>acls_noacls.err", testprog);

85
tar/test/test_option_zstd.c Normal file
View File

@ -0,0 +1,85 @@
/*-
* Copyright (c) 2017 Sean Purcell
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "test.h"
__FBSDID("$FreeBSD$");
DEFINE_TEST(test_option_zstd)
{
char *p;
int r;
size_t s;
/* Create a file. */
assertMakeFile("f", 0644, "a");
/* Archive it with lz4 compression. */
r = systemf("%s -cf - --zstd f >archive.out 2>archive.err",
testprog);
p = slurpfile(&s, "archive.err");
p[s] = '\0';
if (r != 0) {
if (strstr(p, "Unsupported compression") != NULL) {
skipping("This version of bsdtar was compiled "
"without zstd support");
goto done;
}
/* POSIX permits different handling of the spawnp
* system call used to launch the subsidiary
* program: */
/* Some systems fail immediately to spawn the new process. */
if (strstr(p, "Can't launch") != NULL && !canZstd()) {
skipping("This version of bsdtar uses an external zstd program "
"but no such program is available on this system.");
goto done;
}
/* Some systems successfully spawn the new process,
* but fail to exec a program within that process.
* This results in failure at the first attempt to
* write. */
if (strstr(p, "Can't write") != NULL && !canZstd()) {
skipping("This version of bsdtar uses an external zstd program "
"but no such program is available on this system.");
goto done;
}
/* On some systems the error won't be detected until closing
time, by a 127 exit error returned by waitpid. */
if (strstr(p, "Error closing") != NULL && !canZstd()) {
skipping("This version of bsdcpio uses an external zstd program "
"but no such program is available on this system.");
return;
}
failure("--zstd option is broken: %s", p);
assertEqualInt(r, 0);
goto done;
}
free(p);
/* Check that the archive file has an lz4 signature. */
p = slurpfile(&s, "archive.out");
assert(s > 2);
assertEqualMem(p, "\x28\xb5\x2f\xfd", 4);
done:
free(p);
}

3
test_utils/test_common.h
View File

@ -329,6 +329,9 @@ int canLrzip(void);
/* Return true if this platform can run the "lz4" program. */
int canLz4(void);
/* Return true if this platform can run the "zstd" program. */
int canZstd(void);
/* Return true if this platform can run the "lzip" program. */
int canLzip(void);

15
test_utils/test_main.c
View File

@ -2318,6 +2318,21 @@ canLz4(void)
return (value);
}
/*
* Can this platform run the zstd program?
*/
int
canZstd(void)
{
static int tested = 0, value = 0;
if (!tested) {
tested = 1;
if (systemf("zstd -V %s", redirectArgs) == 0)
value = 1;
}
return (value);
}
/*
* Can this platform run the lzip program?
*/