MFV r324145,324147:

Sync libarchive with vendor.

Relevant vendor changes:
  PR #905: Support for Zstandard read and write filters
  PR #922: Avoid overflow when reading corrupt cpio archive
  Issue #935: heap-based buffer overflow in xml_data (CVE-2017-14166)
  OSS-Fuzz 2936: Place a limit on the mtree line length
  OSS-Fuzz 2394: Ensure that the ZIP AES extension header is large enough
  OSS-Fuzz 573: Read off-by-one error in RAR archives (CVE-2017-14502)

MFC after:	1 week
Security:	CVE-2017-14166, CVE-2017-14502

contrib/libarchive/cat/test/test_empty.zst.uu

@@ -0,0 +1,4 @@
+begin 644 test_empty.zst
+-*+4O_010`0``F>G840``
+`
+end

contrib/libarchive/cat/test/test_empty_zstd.c

@@ -0,0 +1,41 @@
+/*-
+ * Copyright (c) 2017 Sean Purcell
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include "test.h"
+
+DEFINE_TEST(test_empty_zstd)
+{
+	const char *reffile = "test_empty.zst";
+	int f;
+
+	extract_reference_file(reffile);
+	f = systemf("%s %s >test.out 2>test.err", testprog, reffile);
+	if (f == 0 || canZstd()) {
+		assertEqualInt(0, f);
+		assertEmptyFile("test.out");
+		assertEmptyFile("test.err");
+	} else {
+		skipping("It seems zstd is not supported on this platform");
+	}
+}

contrib/libarchive/cat/test/test_expand.zst.uu

@@ -0,0 +1,4 @@
+begin 644 test_expand.zst
+J*+4O_010Z0``8V]N=&5N=',@;V8@=&5S=%]E>'!A;F0N>G-T+@J;23#F
+`
+end

contrib/libarchive/cat/test/test_expand_zstd.c

@@ -0,0 +1,41 @@
+/*-
+ * Copyright (c) 2017 Sean Purcell
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include "test.h"
+
+DEFINE_TEST(test_expand_zstd)
+{
+	const char *reffile = "test_expand.zst";
+	int f;
+
+	extract_reference_file(reffile);
+	f = systemf("%s %s >test.out 2>test.err", testprog, reffile);
+	if (f == 0 || canZstd()) {
+		assertEqualInt(0, f);
+		assertTextFileContents("contents of test_expand.zst.\n", "test.out");
+		assertEmptyFile("test.err");
+	} else {
+		skipping("It seems zstd is not supported on this platform");
+	}
+}

contrib/libarchive/cpio/bsdcpio.1

@@ -187,6 +187,11 @@ In input mode, this option is ignored.
 Compress the archive with lz4-compatible compression before writing it.
 In input mode, this option is ignored; lz4 compression is recognized
 automatically on input.
+.It Fl Fl zstd
+(o mode only)
+Compress the archive with zstd-compatible compression before writing it.
+In input mode, this option is ignored; zstd compression is recognized
+automatically on input.
 .It Fl Fl lzma
 (o mode only)
 Compress the file with lzma-compatible compression before writing it.

contrib/libarchive/cpio/cmdline.c

@@ -92,6 +92,7 @@ static const struct option {
 	{ "verbose",			0, 'v' },
 	{ "version",			0, OPTION_VERSION },
 	{ "xz",				0, 'J' },
+	{ "zstd",			0, OPTION_ZSTD },
 	{ NULL, 0, 0 }
 };
 

contrib/libarchive/cpio/cpio.c

@@ -269,6 +269,7 @@ main(int argc, char *argv[])
 		case OPTION_LZ4:
 		case OPTION_LZMA: /* GNU tar, others */
 		case OPTION_LZOP: /* GNU tar, others */
+		case OPTION_ZSTD:
 			cpio->compress = opt;
 			break;
 		case 'm': /* POSIX 1997 */
@@ -546,6 +547,9 @@ mode_out(struct cpio *cpio)
 	case OPTION_LZOP:
 		r = archive_write_add_filter_lzop(cpio->archive);
 		break;
+	case OPTION_ZSTD:
+		r = archive_write_add_filter_zstd(cpio->archive);
+		break;
 	case 'j': case 'y':
 		r = archive_write_add_filter_bzip2(cpio->archive);
 		break;

contrib/libarchive/cpio/cpio.h

@@ -111,7 +111,8 @@ enum {
 	OPTION_PRESERVE_OWNER,
 	OPTION_QUIET,
 	OPTION_UUENCODE,
-	OPTION_VERSION
+	OPTION_VERSION,
+	OPTION_ZSTD,
 };
 
 int	cpio_getopt(struct cpio *cpio);

contrib/libarchive/cpio/test/test_extract.cpio.zst.uu

@@ -0,0 +1,6 @@
+begin 644 test_extract.cpio.zst
+M*+4O_01090,`,@41%X")&@#'G6T\K16_MR)#=DK)5:.1,2J0HY2"!(1!`!7R
+M$(UB`2"*D41;J2UF&)<0!Y7X'TU<%W.\W^R]GO-WW^OO^QX0`%P<]30-!#U`
+?!KD!`#XP,_`U4`HT3+RF:#!7Y\V@R)5"7P"^;WEUK@``
+`
+end

contrib/libarchive/cpio/test/test_extract_cpio_zstd.c

@@ -0,0 +1,48 @@
+/*-
+ * Copyright (c) 2017 Sean Purcell
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include "test.h"
+__FBSDID("$FreeBSD$");
+
+DEFINE_TEST(test_extract_cpio_zstd)
+{
+	const char *reffile = "test_extract.cpio.zst";
+	int f;
+
+	extract_reference_file(reffile);
+	f = systemf("%s -it < %s >test.out 2>test.err", testprog, reffile);
+	if (f == 0 || canZstd()) {
+		assertEqualInt(0, systemf("%s -i < %s >test.out 2>test.err",
+		    testprog, reffile));
+
+		assertFileExists("file1");
+		assertTextFileContents("contents of file1.\n", "file1");
+		assertFileExists("file2");
+		assertTextFileContents("contents of file2.\n", "file2");
+		assertEmptyFile("test.out");
+		assertTextFileContents("1 block\n", "test.err");
+	} else {
+		skipping("It seems zstd is not supported on this platform");
+	}
+}

contrib/libarchive/cpio/test/test_option_zstd.c

@@ -0,0 +1,85 @@
+/*-
+ * Copyright (c) 2017 Sean Purcell
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include "test.h"
+__FBSDID("$FreeBSD$");
+
+DEFINE_TEST(test_option_zstd)
+{
+	char *p;
+	int r;
+	size_t s;
+
+	/* Create a file. */
+	assertMakeFile("f", 0644, "a");
+
+	/* Archive it with zstd compression. */
+	r = systemf("echo f | %s -o --zstd >archive.out 2>archive.err",
+	    testprog);
+	p = slurpfile(&s, "archive.err");
+	p[s] = '\0';
+	if (r != 0) {
+		if (strstr(p, "Unsupported compression") != NULL) {
+			skipping("This version of bsdcpio was compiled "
+			    "without zstd support");
+			goto done;
+		}
+		/* POSIX permits different handling of the spawnp
+		 * system call used to launch the subsidiary
+		 * program: */
+		/* Some systems fail immediately to spawn the new process. */
+		if (strstr(p, "Can't launch") != NULL && !canZstd()) {
+			skipping("This version of bsdcpio uses an external zstd program "
+			    "but no such program is available on this system.");
+			goto done;
+		}
+		/* Some systems successfully spawn the new process,
+		 * but fail to exec a program within that process.
+		 * This results in failure at the first attempt to
+		 * write. */
+		if (strstr(p, "Can't write") != NULL && !canZstd()) {
+			skipping("This version of bsdcpio uses an external zstd program "
+			    "but no such program is available on this system.");
+			goto done;
+		}
+		/* On some systems the error won't be detected until closing
+		   time, by a 127 exit error returned by waitpid. */
+		if (strstr(p, "Error closing") != NULL && !canZstd()) {
+			skipping("This version of bsdcpio uses an external zstd program "
+			    "but no such program is available on this system.");
+			return;
+		}
+		failure("--zstd option is broken: %s", p);
+		assertEqualInt(r, 0);
+		goto done;
+	}
+	free(p);
+	/* Check that the archive file has an zstd signature. */
+	p = slurpfile(&s, "archive.out");
+	assert(s > 2);
+	assertEqualMem(p, "\x28\xb5\x2f\xfd", 4);
+
+done:
+	free(p);
+}

contrib/libarchive/libarchive/archive.h

@@ -177,6 +177,7 @@ __LA_DECL const char *  archive_zlib_version(void);
 __LA_DECL const char *  archive_liblzma_version(void);
 __LA_DECL const char *  archive_bzlib_version(void);
 __LA_DECL const char *  archive_liblz4_version(void);
+__LA_DECL const char *  archive_libzstd_version(void);
 
 /* Declare our basic types. */
 struct archive;
@@ -276,6 +277,7 @@ typedef const char *archive_passphrase_callback(struct archive *,
 #define	ARCHIVE_FILTER_LZOP	11
 #define	ARCHIVE_FILTER_GRZIP	12
 #define	ARCHIVE_FILTER_LZ4	13
+#define	ARCHIVE_FILTER_ZSTD	14
 
 #if ARCHIVE_VERSION_NUMBER < 4000000
 #define	ARCHIVE_COMPRESSION_NONE	ARCHIVE_FILTER_NONE
@@ -433,6 +435,7 @@ __LA_DECL int archive_read_support_filter_program_signature
 __LA_DECL int archive_read_support_filter_rpm(struct archive *);
 __LA_DECL int archive_read_support_filter_uu(struct archive *);
 __LA_DECL int archive_read_support_filter_xz(struct archive *);
+__LA_DECL int archive_read_support_filter_zstd(struct archive *);
 
 __LA_DECL int archive_read_support_format_7zip(struct archive *);
 __LA_DECL int archive_read_support_format_all(struct archive *);
@@ -778,6 +781,7 @@ __LA_DECL int archive_write_add_filter_program(struct archive *,
 		     const char *cmd);
 __LA_DECL int archive_write_add_filter_uuencode(struct archive *);
 __LA_DECL int archive_write_add_filter_xz(struct archive *);
+__LA_DECL int archive_write_add_filter_zstd(struct archive *);
 
 
 /* A convenience function to set the format based on the code or name. */

contrib/libarchive/libarchive/archive_cmdline.c

@@ -100,10 +100,10 @@ get_argument(struct archive_string *as, const char *p)
 
 /*
  * Set up command line arguments.
- * Returns ARChIVE_OK if everything okey.
+ * Returns ARCHIVE_OK if everything okey.
- * Returns ARChIVE_FAILED if there is a lack of the `"' terminator or an
+ * Returns ARCHIVE_FAILED if there is a lack of the `"' terminator or an
  * empty command line.
- * Returns ARChIVE_FATAL if no memory.
+ * Returns ARCHIVE_FATAL if no memory.
  */
 int
 __archive_cmdline_parse(struct archive_cmdline *data, const char *cmd)

contrib/libarchive/libarchive/archive_platform.h

@@ -52,6 +52,17 @@
 #error Oops: No config.h and no pre-built configuration in archive_platform.h.
 #endif
 
+/* On macOS check for some symbols based on the deployment target version.  */
+#if defined(__APPLE__)
+# undef HAVE_FUTIMENS
+# undef HAVE_UTIMENSAT
+# include <AvailabilityMacros.h>
+# if MAC_OS_X_VERSION_MIN_REQUIRED >= 101300
+#  define HAVE_FUTIMENS 1
+#  define HAVE_UTIMENSAT 1
+# endif
+#endif
+
 /* It should be possible to get rid of this by extending the feature-test
  * macros to cover Windows API functions, probably along with non-trivial
  * refactoring of code to find structures that sit more cleanly on top of

contrib/libarchive/libarchive/archive_read_append_filter.c

@@ -89,6 +89,10 @@ archive_read_append_filter(struct archive *_a, int code)
       strcpy(str, "lz4");
       r1 = archive_read_support_filter_lz4(_a);
       break;
+    case ARCHIVE_FILTER_ZSTD:
+      strcpy(str, "zstd");
+      r1 = archive_read_support_filter_zstd(_a);
+      break;
     case ARCHIVE_FILTER_LZIP:
       strcpy(str, "lzip");
       r1 = archive_read_support_filter_lzip(_a);

contrib/libarchive/libarchive/archive_read_disk_entry_from_file.c

@@ -127,7 +127,7 @@ archive_read_disk_entry_setup_acls(struct archive_read_disk *a,
 /*
  * Enter working directory and return working pathname of archive_entry.
  * If a pointer to an integer is provided and its value is below zero
- * open a file descriptor on this pahtname.
+ * open a file descriptor on this pathname.
  */
 const char *
 archive_read_disk_entry_setup_path(struct archive_read_disk *a,

contrib/libarchive/libarchive/archive_read_filter.3

@@ -38,6 +38,7 @@
 .Nm archive_read_support_filter_rpm ,
 .Nm archive_read_support_filter_uu ,
 .Nm archive_read_support_filter_xz ,
+.Nm archive_read_support_filter_zstd ,
 .Nm archive_read_support_filter_program ,
 .Nm archive_read_support_filter_program_signature
 .Nd functions for reading streaming archives
@@ -73,6 +74,8 @@ Streaming Archive Library (libarchive, -larchive)
 .Ft int
 .Fn archive_read_support_filter_xz "struct archive *"
 .Ft int
+.Fn archive_read_support_filter_zstd "struct archive *"
+.Ft int
 .Fo archive_read_support_filter_program
 .Fa "struct archive *"
 .Fa "const char *cmd"
@@ -99,7 +102,8 @@ Streaming Archive Library (libarchive, -larchive)
 .Fn archive_read_support_filter_none ,
 .Fn archive_read_support_filter_rpm ,
 .Fn archive_read_support_filter_uu ,
-.Fn archive_read_support_filter_xz
+.Fn archive_read_support_filter_xz ,
+.Fn archive_read_support_filter_zstd ,
 .Xc
 Enables auto-detection code and decompression support for the
 specified compression.

contrib/libarchive/libarchive/archive_read_support_filter_all.c

@@ -71,6 +71,8 @@ archive_read_support_filter_all(struct archive *a)
 	archive_read_support_filter_grzip(a);
 	/* Lz4 falls back to "lz4 -d" command-line program. */
 	archive_read_support_filter_lz4(a);
+	/* Zstd falls back to "zstd -d" command-line program. */
+	archive_read_support_filter_zstd(a);
 
 	/* Note: We always return ARCHIVE_OK here, even if some of the
 	 * above return ARCHIVE_WARN.  The intent here is to enable

contrib/libarchive/libarchive/archive_read_support_filter_zstd.c

@@ -0,0 +1,292 @@
+/*-
+ * Copyright (c) 2009-2011 Sean Purcell
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "archive_platform.h"
+
+__FBSDID("$FreeBSD$");
+
+#ifdef HAVE_ERRNO_H
+#include <errno.h>
+#endif
+
+#ifdef HAVE_ERRNO_H
+#include <errno.h>
+#endif
+#include <stdio.h>
+#ifdef HAVE_STDLIB_H
+#include <stdlib.h>
+#endif
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#if HAVE_ZSTD_H
+#include <zstd.h>
+#endif
+
+#include "archive.h"
+#include "archive_endian.h"
+#include "archive_private.h"
+#include "archive_read_private.h"
+
+#if HAVE_ZSTD_H && HAVE_LIBZSTD
+
+struct private_data {
+	ZSTD_DStream	*dstream;
+	unsigned char	*out_block;
+	size_t		 out_block_size;
+	int64_t		 total_out;
+	char		 in_frame; /* True = in the middle of a zstd frame. */
+	char		 eof; /* True = found end of compressed data. */
+};
+
+/* Zstd Filter. */
+static ssize_t	zstd_filter_read(struct archive_read_filter *, const void**);
+static int	zstd_filter_close(struct archive_read_filter *);
+#endif
+
+/*
+ * Note that we can detect zstd compressed files even if we can't decompress
+ * them.  (In fact, we like detecting them because we can give better error
+ * messages.)  So the bid framework here gets compiled even if no zstd library
+ * is available.
+ */
+static int	zstd_bidder_bid(struct archive_read_filter_bidder *,
+		    struct archive_read_filter *);
+static int	zstd_bidder_init(struct archive_read_filter *);
+
+int
+archive_read_support_filter_zstd(struct archive *_a)
+{
+	struct archive_read *a = (struct archive_read *)_a;
+	struct archive_read_filter_bidder *bidder;
+
+	archive_check_magic(_a, ARCHIVE_READ_MAGIC,
+	    ARCHIVE_STATE_NEW, "archive_read_support_filter_zstd");
+
+	if (__archive_read_get_bidder(a, &bidder) != ARCHIVE_OK)
+		return (ARCHIVE_FATAL);
+
+	bidder->data = NULL;
+	bidder->name = "zstd";
+	bidder->bid = zstd_bidder_bid;
+	bidder->init = zstd_bidder_init;
+	bidder->options = NULL;
+	bidder->free = NULL;
+#if HAVE_ZSTD_H && HAVE_LIBZSTD
+	return (ARCHIVE_OK);
+#else
+	archive_set_error(_a, ARCHIVE_ERRNO_MISC,
+	    "Using external zstd program for zstd decompression");
+	return (ARCHIVE_WARN);
+#endif
+}
+
+/*
+ * Test whether we can handle this data.
+ */
+static int
+zstd_bidder_bid(struct archive_read_filter_bidder *self,
+    struct archive_read_filter *filter)
+{
+	const unsigned char *buffer;
+	ssize_t avail;
+	unsigned prefix;
+
+	/* Zstd frame magic values */
+	const unsigned zstd_magic = 0xFD2FB528U;
+
+	(void) self; /* UNUSED */
+
+	buffer = __archive_read_filter_ahead(filter, 4, &avail);
+	if (buffer == NULL)
+		return (0);
+
+	prefix = archive_le32dec(buffer);
+	if (prefix == zstd_magic)
+		return (32);
+
+	return (0);
+}
+
+#if !(HAVE_ZSTD_H && HAVE_LIBZSTD)
+
+/*
+ * If we don't have the library on this system, we can't do the
+ * decompression directly.  We can, however, try to run "zstd -d"
+ * in case that's available.
+ */
+static int
+zstd_bidder_init(struct archive_read_filter *self)
+{
+	int r;
+
+	r = __archive_read_program(self, "zstd -d -qq");
+	/* Note: We set the format here even if __archive_read_program()
+	 * above fails.  We do, after all, know what the format is
+	 * even if we weren't able to read it. */
+	self->code = ARCHIVE_FILTER_ZSTD;
+	self->name = "zstd";
+	return (r);
+}
+
+#else
+
+/*
+ * Initialize the filter object
+ */
+static int
+zstd_bidder_init(struct archive_read_filter *self)
+{
+	struct private_data *state;
+	const size_t out_block_size = ZSTD_DStreamOutSize();
+	void *out_block;
+	ZSTD_DStream *dstream;
+
+	self->code = ARCHIVE_FILTER_ZSTD;
+	self->name = "zstd";
+
+	state = (struct private_data *)calloc(sizeof(*state), 1);
+	out_block = (unsigned char *)malloc(out_block_size);
+	dstream = ZSTD_createDStream();
+
+	if (state == NULL || out_block == NULL || dstream == NULL) {
+		free(out_block);
+		free(state);
+		ZSTD_freeDStream(dstream); /* supports free on NULL */
+		archive_set_error(&self->archive->archive, ENOMEM,
+		    "Can't allocate data for zstd decompression");
+		return (ARCHIVE_FATAL);
+	}
+
+	self->data = state;
+
+	state->out_block_size = out_block_size;
+	state->out_block = out_block;
+	state->dstream = dstream;
+	self->read = zstd_filter_read;
+	self->skip = NULL; /* not supported */
+	self->close = zstd_filter_close;
+
+	state->eof = 0;
+	state->in_frame = 0;
+
+	return (ARCHIVE_OK);
+}
+
+static ssize_t
+zstd_filter_read(struct archive_read_filter *self, const void **p)
+{
+	struct private_data *state;
+	size_t decompressed;
+	ssize_t avail_in;
+	ZSTD_outBuffer out;
+	ZSTD_inBuffer in;
+
+	state = (struct private_data *)self->data;
+
+	out = (ZSTD_outBuffer) { state->out_block, state->out_block_size, 0 };
+
+	/* Try to fill the output buffer. */
+	while (out.pos < out.size && !state->eof) {
+		if (!state->in_frame) {
+			const size_t ret = ZSTD_initDStream(state->dstream);
+			if (ZSTD_isError(ret)) {
+				archive_set_error(&self->archive->archive,
+				    ARCHIVE_ERRNO_MISC,
+				    "Error initializing zstd decompressor: %s",
+				    ZSTD_getErrorName(ret));
+				return (ARCHIVE_FATAL);
+			}
+		}
+		in.src = __archive_read_filter_ahead(self->upstream, 1,
+		    &avail_in);
+		if (avail_in < 0) {
+			return avail_in;
+		}
+		if (in.src == NULL && avail_in == 0) {
+			if (!state->in_frame) {
+				/* end of stream */
+				state->eof = 1;
+				break;
+			} else {
+				archive_set_error(&self->archive->archive,
+				    ARCHIVE_ERRNO_MISC,
+				    "Truncated zstd input");
+				return (ARCHIVE_FATAL);
+			}
+		}
+		in.size = avail_in;
+		in.pos = 0;
+
+		{
+			const size_t ret =
+			    ZSTD_decompressStream(state->dstream, &out, &in);
+
+			if (ZSTD_isError(ret)) {
+				archive_set_error(&self->archive->archive,
+				    ARCHIVE_ERRNO_MISC,
+				    "Zstd decompression failed: %s",
+				    ZSTD_getErrorName(ret));
+				return (ARCHIVE_FATAL);
+			}
+
+			/* Decompressor made some progress */
+			__archive_read_filter_consume(self->upstream, in.pos);
+
+			/* ret guaranteed to be > 0 if frame isn't done yet */
+			state->in_frame = (ret != 0);
+		}
+	}
+
+	decompressed = out.pos;
+	state->total_out += decompressed;
+	if (decompressed == 0)
+		*p = NULL;
+	else
+		*p = state->out_block;
+	return (decompressed);
+}
+
+/*
+ * Clean up the decompressor.
+ */
+static int
+zstd_filter_close(struct archive_read_filter *self)
+{
+	struct private_data *state;
+
+	state = (struct private_data *)self->data;
+
+	ZSTD_freeDStream(state->dstream);
+	free(state->out_block);
+	free(state);
+
+	return (ARCHIVE_OK);
+}
+
+#endif /* HAVE_ZLIB_H && HAVE_LIBZSTD */

contrib/libarchive/libarchive/archive_read_support_format_cpio.c

@@ -633,6 +633,13 @@ header_newc(struct archive_read *a, struct cpio *cpio,
 	/* Pad name to 2 more than a multiple of 4. */
 	*name_pad = (2 - *namelength) & 3;
 
+	/* Make sure that the padded name length fits into size_t. */
+	if (*name_pad > SIZE_MAX - *namelength) {
+		archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT,
+		    "cpio archive has invalid namelength");
+		return (ARCHIVE_FATAL);
+	}
+
 	/*
 	 * Note: entry_bytes_remaining is at least 64 bits and
 	 * therefore guaranteed to be big enough for a 33-bit file

contrib/libarchive/libarchive/archive_read_support_format_mtree.c

@@ -77,6 +77,8 @@ __FBSDID("$FreeBSD$");
 
 #define	MTREE_HASHTABLE_SIZE 1024
 
+#define	MAX_LINE_LEN		(1024 * 1024)
+
 struct mtree_option {
 	struct mtree_option *next;
 	char *value;
@@ -334,6 +336,14 @@ next_line(struct archive_read *a,
 		size_t nbytes_req = (*ravail+1023) & ~1023U;
 		ssize_t tested;
 
+		/*
+		 * Place an arbitrary limit on the line length.
+		 * mtree is almost free-form input and without line length limits,
+		 * it can consume a lot of memory.
+		 */
+		if (len >= MAX_LINE_LEN)
+			return (-1);
+
 		/* Increase reading bytes if it is not enough to at least
 		 * new two lines. */
 		if (nbytes_req < (size_t)*ravail + 160)

contrib/libarchive/libarchive/archive_read_support_format_rar.c

@@ -1496,7 +1496,11 @@ read_header(struct archive_read *a, struct archive_entry *entry,
         return (ARCHIVE_FATAL);
       }
       filename[filename_size++] = '\0';
-      filename[filename_size++] = '\0';
+      /*
+       * Do not increment filename_size here as the computations below
+       * add the space for the terminating NUL explicitly.
+       */
+      filename[filename_size] = '\0';
 
       /* Decoded unicode form is UTF-16BE, so we have to update a string
        * conversion object for it. */

contrib/libarchive/libarchive/archive_read_support_format_tar.c

@@ -2243,7 +2243,7 @@ gnu_add_sparse_entry(struct archive_read *a, struct tar *tar,
 	else
 		tar->sparse_list = p;
 	tar->sparse_last = p;
-	if (remaining < 0 || offset < 0) {
+	if (remaining < 0 || offset < 0 || offset > INT64_MAX - remaining) {
 		archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, "Malformed sparse map data");
 		return (ARCHIVE_FATAL);
 	}

contrib/libarchive/libarchive/archive_read_support_format_xar.c

@@ -1040,6 +1040,9 @@ atol10(const char *p, size_t char_cnt)
 	uint64_t l;
 	int digit;
 
+	if (char_cnt == 0)
+		return (0);
+
 	l = 0;
 	digit = *p - '0';
 	while (digit >= 0 && digit < 10  && char_cnt-- > 0) {
@@ -1055,6 +1058,9 @@ atol8(const char *p, size_t char_cnt)
 	int64_t l;
 	int digit;
 
+	if (char_cnt == 0)
+		return (0);
+
 	l = 0;
 	while (char_cnt-- > 0) {
 		if (*p >= '0' && *p <= '7')
@@ -2623,6 +2629,14 @@ strappend_base64(struct xar *xar,
 		archive_strncat(as, (const char *)buff, len);
 }
 
+static int
+is_string(const char *known, const char *data, size_t len)
+{
+	if (strlen(known) != len)
+		return -1;
+	return memcmp(data, known, len);
+}
+
 static void
 xml_data(void *userData, const char *s, int len)
 {
@@ -2674,26 +2688,26 @@ xml_data(void *userData, const char *s, int len)
 		archive_strncpy(&(xar->file->symlink), s, len);
 		break;
 	case FILE_TYPE:
-		if (strncmp("file", s, len) == 0 ||
+		if (is_string("file", s, len) == 0 ||
-		    strncmp("hardlink", s, len) == 0)
+		    is_string("hardlink", s, len) == 0)
 			xar->file->mode =
 			    (xar->file->mode & ~AE_IFMT) | AE_IFREG;
-		if (strncmp("directory", s, len) == 0)
+		if (is_string("directory", s, len) == 0)
 			xar->file->mode =
 			    (xar->file->mode & ~AE_IFMT) | AE_IFDIR;
-		if (strncmp("symlink", s, len) == 0)
+		if (is_string("symlink", s, len) == 0)
 			xar->file->mode =
 			    (xar->file->mode & ~AE_IFMT) | AE_IFLNK;
-		if (strncmp("character special", s, len) == 0)
+		if (is_string("character special", s, len) == 0)
 			xar->file->mode =
 			    (xar->file->mode & ~AE_IFMT) | AE_IFCHR;
-		if (strncmp("block special", s, len) == 0)
+		if (is_string("block special", s, len) == 0)
 			xar->file->mode =
 			    (xar->file->mode & ~AE_IFMT) | AE_IFBLK;
-		if (strncmp("socket", s, len) == 0)
+		if (is_string("socket", s, len) == 0)
 			xar->file->mode =
 			    (xar->file->mode & ~AE_IFMT) | AE_IFSOCK;
-		if (strncmp("fifo", s, len) == 0)
+		if (is_string("fifo", s, len) == 0)
 			xar->file->mode =
 			    (xar->file->mode & ~AE_IFMT) | AE_IFIFO;
 		xar->file->has |= HAS_TYPE;

contrib/libarchive/libarchive/archive_read_support_format_zip.c

@@ -723,6 +723,11 @@ process_extra(struct archive_read *a, const char *p, size_t extra_length, struct
 		}
 		case 0x9901:
 			/* WinZip AES extra data field. */
+			if (datasize < 6) {
+				archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT,
+				    "Incomplete AES field");
+				return ARCHIVE_FAILED;
+			}
 			if (p[offset + 2] == 'A' && p[offset + 3] == 'E') {
 				/* Vendor version. */
 				zip_entry->aes_extra.vendor =

contrib/libarchive/libarchive/archive_string.c

@@ -214,7 +214,8 @@ archive_wstring_append(struct archive_wstring *as, const wchar_t *p, size_t s)
 {
 	if (archive_wstring_ensure(as, as->length + s + 1) == NULL)
 		return (NULL);
-	wmemmove(as->s + as->length, p, s);
+	if (s)
+		wmemmove(as->s + as->length, p, s);
 	as->length += s;
 	as->s[as->length] = 0;
 	return (as);

contrib/libarchive/libarchive/archive_version_details.c

@@ -45,6 +45,9 @@ __FBSDID("$FreeBSD$");
 #ifdef HAVE_LZ4_H
 #include <lz4.h>
 #endif
+#ifdef HAVE_ZSTD_H
+#include <zstd.h>
+#endif
 
 #include "archive.h"
 #include "archive_private.h"
@@ -59,6 +62,7 @@ archive_version_details(void)
 	const char *liblzma = archive_liblzma_version();
 	const char *bzlib = archive_bzlib_version();
 	const char *liblz4 = archive_liblz4_version();
+	const char *libzstd = archive_libzstd_version();
 
 	if (!init) {
 		archive_string_init(&str);
@@ -84,6 +88,10 @@ archive_version_details(void)
 			archive_strcat(&str, " liblz4/");
 			archive_strcat(&str, liblz4);
 		}
+		if (libzstd) {
+			archive_strcat(&str, " libzstd/");
+			archive_strcat(&str, libzstd);
+		}
 	}
 	return str.s;
 }
@@ -131,3 +139,13 @@ archive_liblz4_version(void)
 	return NULL;
 #endif
 }
+
+const char *
+archive_libzstd_version(void)
+{
+#if HAVE_ZSTD_H && HAVE_LIBZSTD
+	return ZSTD_VERSION_STRING;
+#else
+	return NULL;
+#endif
+}

contrib/libarchive/libarchive/archive_write.3

@@ -71,7 +71,7 @@ support.
 .\"
 .Ss Set options
 See
-.Xr archive_read_set_options 3 .
+.Xr archive_write_set_options 3 .
 .\"
 .Ss Open archive
 See

contrib/libarchive/libarchive/archive_write_add_filter.c

@@ -53,6 +53,7 @@ struct { int code; int (*setter)(struct archive *); } codes[] =
 	{ ARCHIVE_FILTER_LZOP,		archive_write_add_filter_lzip },
 	{ ARCHIVE_FILTER_UU,		archive_write_add_filter_uuencode },
 	{ ARCHIVE_FILTER_XZ,		archive_write_add_filter_xz },
+	{ ARCHIVE_FILTER_ZSTD,		archive_write_add_filter_zstd },
 	{ -1,			NULL }
 };
 

contrib/libarchive/libarchive/archive_write_add_filter_by_name.c

@@ -57,6 +57,7 @@ struct { const char *name; int (*setter)(struct archive *); } names[] =
 	{ "lzop",		archive_write_add_filter_lzop },
 	{ "uuencode",		archive_write_add_filter_uuencode },
 	{ "xz",			archive_write_add_filter_xz },
+	{ "zstd",		archive_write_add_filter_zstd },
 	{ NULL,			NULL }
 };
 

contrib/libarchive/libarchive/archive_write_add_filter_zstd.c

@@ -0,0 +1,335 @@
+/*-
+ * Copyright (c) 2017 Sean Purcell
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "archive_platform.h"
+
+__FBSDID("$FreeBSD$");
+
+
+#ifdef HAVE_ERRNO_H
+#include <errno.h>
+#endif
+#ifdef HAVE_STDLIB_H
+#include <stdlib.h>
+#endif
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+#ifdef HAVE_ZSTD_H
+#include <zstd.h>
+#endif
+
+#include "archive.h"
+#include "archive_private.h"
+#include "archive_string.h"
+#include "archive_write_private.h"
+
+/* Don't compile this if we don't have zstd.h */
+
+struct private_data {
+	int		 compression_level;
+#if HAVE_ZSTD_H && HAVE_LIBZSTD
+	ZSTD_CStream	*cstream;
+	int64_t		 total_in;
+	ZSTD_outBuffer	 out;
+#else
+	struct archive_write_program_data *pdata;
+#endif
+};
+
+static int archive_compressor_zstd_options(struct archive_write_filter *,
+		    const char *, const char *);
+static int archive_compressor_zstd_open(struct archive_write_filter *);
+static int archive_compressor_zstd_write(struct archive_write_filter *,
+		    const void *, size_t);
+static int archive_compressor_zstd_close(struct archive_write_filter *);
+static int archive_compressor_zstd_free(struct archive_write_filter *);
+#if HAVE_ZSTD_H && HAVE_LIBZSTD
+static int drive_compressor(struct archive_write_filter *,
+		    struct private_data *, int, const void *, size_t);
+#endif
+
+
+/*
+ * Add a zstd compression filter to this write handle.
+ */
+int
+archive_write_add_filter_zstd(struct archive *_a)
+{
+	struct archive_write *a = (struct archive_write *)_a;
+	struct archive_write_filter *f = __archive_write_allocate_filter(_a);
+	struct private_data *data;
+	archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,
+	    ARCHIVE_STATE_NEW, "archive_write_add_filter_zstd");
+
+	data = calloc(1, sizeof(*data));
+	if (data == NULL) {
+		archive_set_error(&a->archive, ENOMEM, "Out of memory");
+		return (ARCHIVE_FATAL);
+	}
+	f->data = data;
+	f->open = &archive_compressor_zstd_open;
+	f->options = &archive_compressor_zstd_options;
+	f->close = &archive_compressor_zstd_close;
+	f->free = &archive_compressor_zstd_free;
+	f->code = ARCHIVE_FILTER_ZSTD;
+	f->name = "zstd";
+	data->compression_level = 3; /* Default level used by the zstd CLI */
+#if HAVE_ZSTD_H && HAVE_LIBZSTD
+	data->cstream = ZSTD_createCStream();
+	if (data->cstream == NULL) {
+		free(data);
+		archive_set_error(&a->archive, ENOMEM,
+		    "Failed to allocate zstd compressor object");
+		return (ARCHIVE_FATAL);
+	}
+
+	return (ARCHIVE_OK);
+#else
+	data->pdata = __archive_write_program_allocate("zstd");
+	if (data->pdata == NULL) {
+		free(data);
+		archive_set_error(&a->archive, ENOMEM, "Out of memory");
+		return (ARCHIVE_FATAL);
+	}
+	archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
+	    "Using external zstd program");
+	return (ARCHIVE_WARN);
+#endif
+}
+
+static int
+archive_compressor_zstd_free(struct archive_write_filter *f)
+{
+	struct private_data *data = (struct private_data *)f->data;
+#if HAVE_ZSTD_H && HAVE_LIBZSTD
+	ZSTD_freeCStream(data->cstream);
+	free(data->out.dst);
+#else
+	__archive_write_program_free(data->pdata);
+#endif
+	free(data);
+	f->data = NULL;
+	return (ARCHIVE_OK);
+}
+
+/*
+ * Set write options.
+ */
+static int
+archive_compressor_zstd_options(struct archive_write_filter *f, const char *key,
+    const char *value)
+{
+	struct private_data *data = (struct private_data *)f->data;
+
+	if (strcmp(key, "compression-level") == 0) {
+		int level = atoi(value);
+#if HAVE_ZSTD_H && HAVE_LIBZSTD
+		if (level < 1 || level > ZSTD_maxCLevel()) {
+#else
+		/* If we don't have the library, hard-code the max level */
+		if (level < 1 || level > 22) {
+#endif
+			return (ARCHIVE_WARN);
+		}
+		data->compression_level = level;
+		return (ARCHIVE_OK);
+	}
+
+	/* Note: The "warn" return is just to inform the options
+	 * supervisor that we didn't handle it.  It will generate
+	 * a suitable error if no one used this option. */
+	return (ARCHIVE_WARN);
+}
+
+#if HAVE_ZSTD_H && HAVE_LIBZSTD
+/*
+ * Setup callback.
+ */
+static int
+archive_compressor_zstd_open(struct archive_write_filter *f)
+{
+	struct private_data *data = (struct private_data *)f->data;
+	int ret;
+
+	ret = __archive_write_open_filter(f->next_filter);
+	if (ret != ARCHIVE_OK)
+		return (ret);
+
+	if (data->out.dst == NULL) {
+		size_t bs = ZSTD_CStreamOutSize(), bpb;
+		if (f->archive->magic == ARCHIVE_WRITE_MAGIC) {
+			/* Buffer size should be a multiple number of
+			 * the of bytes per block for performance. */
+			bpb = archive_write_get_bytes_per_block(f->archive);
+			if (bpb > bs)
+				bs = bpb;
+			else if (bpb != 0)
+				bs -= bs % bpb;
+		}
+		data->out.size = bs;
+		data->out.pos = 0;
+		data->out.dst
+		    = (unsigned char *)malloc(data->out.size);
+		if (data->out.dst == NULL) {
+			archive_set_error(f->archive, ENOMEM,
+			    "Can't allocate data for compression buffer");
+			return (ARCHIVE_FATAL);
+		}
+	}
+
+	f->write = archive_compressor_zstd_write;
+
+	if (ZSTD_isError(ZSTD_initCStream(data->cstream,
+	    data->compression_level))) {
+		archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
+		    "Internal error initializing zstd compressor object");
+		return (ARCHIVE_FATAL);
+	}
+
+	return (ARCHIVE_OK);
+}
+
+/*
+ * Write data to the compressed stream.
+ */
+static int
+archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff,
+    size_t length)
+{
+	struct private_data *data = (struct private_data *)f->data;
+	int ret;
+
+	/* Update statistics */
+	data->total_in += length;
+
+	if ((ret = drive_compressor(f, data, 0, buff, length)) != ARCHIVE_OK)
+		return (ret);
+
+	return (ARCHIVE_OK);
+}
+
+/*
+ * Finish the compression...
+ */
+static int
+archive_compressor_zstd_close(struct archive_write_filter *f)
+{
+	struct private_data *data = (struct private_data *)f->data;
+	int r1, r2;
+
+	/* Finish zstd frame */
+	r1 = drive_compressor(f, data, 1, NULL, 0);
+
+	r2 = __archive_write_close_filter(f->next_filter);
+
+	return r1 < r2 ? r1 : r2;
+}
+
+/*
+ * Utility function to push input data through compressor,
+ * writing full output blocks as necessary.
+ *
+ * Note that this handles both the regular write case (finishing ==
+ * false) and the end-of-archive case (finishing == true).
+ */
+static int
+drive_compressor(struct archive_write_filter *f,
+    struct private_data *data, int finishing, const void *src, size_t length)
+{
+	ZSTD_inBuffer in = (ZSTD_inBuffer) { src, length, 0 };
+
+	for (;;) {
+		if (data->out.pos == data->out.size) {
+			const int ret = __archive_write_filter(f->next_filter,
+			    data->out.dst, data->out.size);
+			if (ret != ARCHIVE_OK)
+				return (ARCHIVE_FATAL);
+			data->out.pos = 0;
+		}
+
+		/* If there's nothing to do, we're done. */
+		if (!finishing && in.pos == in.size)
+			return (ARCHIVE_OK);
+
+		{
+			const size_t zstdret = !finishing ?
+			    ZSTD_compressStream(data->cstream, &data->out, &in)
+			    : ZSTD_endStream(data->cstream, &data->out);
+
+			if (ZSTD_isError(zstdret)) {
+				archive_set_error(f->archive,
+				    ARCHIVE_ERRNO_MISC,
+				    "Zstd compression failed: %s",
+				    ZSTD_getErrorName(zstdret));
+				return (ARCHIVE_FATAL);
+			}
+
+			/* If we're finishing, 0 means nothing left to flush */
+			if (finishing && zstdret == 0) {
+				const int ret = __archive_write_filter(f->next_filter,
+				    data->out.dst, data->out.pos);
+				return (ret);
+			}
+		}
+	}
+}
+
+#else /* HAVE_ZSTD_H && HAVE_LIBZSTD */
+
+static int
+archive_compressor_zstd_open(struct archive_write_filter *f)
+{
+	struct private_data *data = (struct private_data *)f->data;
+	struct archive_string as;
+	int r;
+
+	archive_string_init(&as);
+	archive_string_sprintf(&as, "zstd -%d", data->compression_level);
+
+	f->write = archive_compressor_zstd_write;
+	r = __archive_write_program_open(f, data->pdata, as.s);
+	archive_string_free(&as);
+	return (r);
+}
+
+static int
+archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff,
+    size_t length)
+{
+	struct private_data *data = (struct private_data *)f->data;
+
+	return __archive_write_program_write(f, data->pdata, buff, length);
+}
+
+static int
+archive_compressor_zstd_close(struct archive_write_filter *f)
+{
+	struct private_data *data = (struct private_data *)f->data;
+
+	return __archive_write_program_close(f, data->pdata);
+}
+
+#endif /* HAVE_ZSTD_H && HAVE_LIBZSTD */

contrib/libarchive/libarchive/archive_write_filter.3

@@ -42,7 +42,8 @@
 .Nm archive_write_add_filter_none ,
 .Nm archive_write_add_filter_program ,
 .Nm archive_write_add_filter_uuencode ,
-.Nm archive_write_add_filter_xz
+.Nm archive_write_add_filter_xz ,
+.Nm archive_write_add_filter_zstd ,
 .Nd functions enabling output filters
 .Sh LIBRARY
 Streaming Archive Library (libarchive, -larchive)
@@ -76,6 +77,8 @@ Streaming Archive Library (libarchive, -larchive)
 .Fn archive_write_add_filter_uuencode "struct archive *"
 .Ft int
 .Fn archive_write_add_filter_xz "struct archive *"
+.Ft int
+.Fn archive_write_add_filter_zstd "struct archive *"
 .Sh DESCRIPTION
 .Bl -tag -width indent
 .It Xo
@@ -89,6 +92,7 @@ Streaming Archive Library (libarchive, -larchive)
 .Fn archive_write_add_filter_lzma ,
 .Fn archive_write_add_filter_lzop ,
 .Fn archive_write_add_filter_xz ,
+.Fn archive_write_add_filter_zstd ,
 .Xc
 The resulting archive will be compressed as specified.
 Note that the compressed output is always properly blocked.

contrib/libarchive/libarchive/archive_write_set_format_pax.c

@@ -1654,7 +1654,7 @@ build_pax_attribute_name(char *dest, const char *src)
  * GNU PAX Format 1.0 requires the special name, which pattern is:
  * <dir>/GNUSparseFile.<pid>/<original file name>
  *
- * Since reproducable archives are more important, use 0 as pid.
+ * Since reproducible archives are more important, use 0 as pid.
  *
  * This function is used for only Sparse file, a file type of which
  * is regular file.

contrib/libarchive/libarchive/test/test_archive_write_add_filter_by_name.c

@@ -207,3 +207,8 @@ DEFINE_TEST(test_archive_write_add_filter_by_name_xz)
 {
 	test_filter_by_name("xz", ARCHIVE_FILTER_XZ, cannot);
 }
+
+DEFINE_TEST(test_archive_write_add_filter_by_name_zstd)
+{
+	test_filter_by_name("zstd", ARCHIVE_FILTER_ZSTD, canZstd);
+}

contrib/libarchive/libarchive/test/test_compat_zstd.c

@@ -0,0 +1,82 @@
+/*-
+ * Copyright (c) 2017 Sean Purcell
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include "test.h"
+__FBSDID("$FreeBSD$");
+
+/*
+ * Verify our ability to read sample files compatibly with 'zstd -d'.
+ *
+ * In particular:
+ *  * zstd -d will read multiple zstd streams, concatenating the output
+ *  * zstd -d will skip over zstd skippable frames
+ */
+
+static void
+compat_zstd(const char *name)
+{
+	const char *n[7] = { "f1", "f2", "f3", "d1/f1", "d1/f2", "d1/f3", NULL };
+	struct archive_entry *ae;
+	struct archive *a;
+	int i, r;
+
+	assert((a = archive_read_new()) != NULL);
+	assertEqualIntA(a, ARCHIVE_OK, archive_read_support_filter_all(a));
+	r = archive_read_support_filter_zstd(a);
+	if (r == ARCHIVE_WARN) {
+		skipping("zstd reading not fully supported on this platform");
+		assertEqualInt(ARCHIVE_OK, archive_read_free(a));
+		return;
+	}
+	assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a));
+	extract_reference_file(name);
+	assertEqualIntA(a, ARCHIVE_OK, archive_read_open_filename(a, name, 2));
+
+	/* Read entries, match up names with list above. */
+	for (i = 0; i < 6; ++i) {
+		failure("Could not read file %d (%s) from %s", i, n[i], name);
+		assertEqualIntA(a, ARCHIVE_OK,
+		    archive_read_next_header(a, &ae));
+		assertEqualString(n[i], archive_entry_pathname(ae));
+	}
+
+	/* Verify the end-of-archive. */
+	assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae));
+
+	/* Verify that the format detection worked. */
+	assertEqualInt(archive_filter_code(a, 0), ARCHIVE_FILTER_ZSTD);
+	assertEqualString(archive_filter_name(a, 0), "zstd");
+	assertEqualInt(archive_format(a), ARCHIVE_FORMAT_TAR_USTAR);
+
+	assertEqualInt(ARCHIVE_OK, archive_read_close(a));
+	assertEqualInt(ARCHIVE_OK, archive_read_free(a));
+}
+
+
+DEFINE_TEST(test_compat_zstd)
+{
+	/* This sample was compressed as 3 separate streams with a zstd skippable
+	* frame placed in the middle */
+	compat_zstd("test_compat_zstd_1.tar.zst");
+}

contrib/libarchive/libarchive/test/test_compat_zstd_1.tar.zst.uu

@@ -0,0 +1,12 @@
+begin 644 test_compat_zstd_1.tar.zst
+M*+4O_010)0,`HL0.%;`Q&>>\/$2[#IQF[<1+Z3T<0CX]!77&0@R.6+/F,0+I
+M.$1A$QE2`J!+*_6[_YT9_W_M1KC-EG*V>10.`,M`%3*@#F#\`-FT#J:1#U1"
+M`H1!&R#<!.<"@#3@M58XY1,8`DMMD\@HM2_]!%!=`P`B!1`5H#D!0!.SELJ"
+M5#509I*T/YQ^]?H/3T1D>A5\*'"JYIJ;C&4=B2CL(L)*E-IJT/RV?.:A_]_N
+MB&[7SDG;/=4&#P";0!5D0`=8T0&R&19,)1^HA`0(@S9`N`G.!0!IP&NM<,K!
+M-#8!%A]U]K10*DT8!`````$"`P0HM2_]!%!]`P`B11`6H+$)"%]@,Z6OH`"L
+MM$R2MAN&*MSG`W?OJ7+4P*B::VXR`NM(1&$7&58"J*U'_&V^S$/_O]U1N%T[
+M)VW7J'+4!A``_4$%^T`],J`8P.0!L@D63"4?J(0$"(,V0+@)S@4`:<!KK7!J
+)P51V`E@!9CD#
+`
+end

contrib/libarchive/libarchive/test/test_fuzz.c

@@ -412,6 +412,12 @@ DEFINE_TEST(test_fuzz_tar)
 		"test_compat_lzop_1.tar.lzo",
 		NULL
 	};
+#endif
+#if HAVE_ZSTD_H && HAVE_LIBZSTD
+	static const char *fileset10[] = {
+		"test_compat_zstd_1.tar.zst",
+		NULL
+	};
 #endif
 	static const struct files filesets[] = {
 		{0, fileset1}, /* Exercise bzip2 decompressor. */
@@ -425,6 +431,9 @@ DEFINE_TEST(test_fuzz_tar)
 		{0, fileset8},
 #if HAVE_LIBLZO2 && HAVE_LZO_LZO1X_H && HAVE_LZO_LZOCONF_H
 		{0, fileset9}, /* Exercise lzo decompressor. */
+#endif
+#if HAVE_ZSTD_H && HAVE_LIBZSTD
+		{0, fileset10}, /* Excercise zstd decompressor. */
 #endif
 		{1, NULL}
 	};

contrib/libarchive/libarchive/test/test_write_filter_zstd.c

@@ -0,0 +1,201 @@
+/*-
+ * Copyright (c) 2017 Sean Purcell
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer
+ *    in this position and unchanged.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "test.h"
+__FBSDID("$FreeBSD$");
+
+DEFINE_TEST(test_write_filter_zstd)
+{
+	struct archive_entry *ae;
+	struct archive *a;
+	char *buff, *data;
+	size_t buffsize, datasize;
+	char path[16];
+	size_t used1, used2;
+	int i, r;
+
+	buffsize = 2000000;
+	assert(NULL != (buff = (char *)malloc(buffsize)));
+	if (buff == NULL)
+		return;
+
+	datasize = 10000;
+	assert(NULL != (data = (char *)malloc(datasize)));
+	if (data == NULL) {
+		free(buff);
+		return;
+	}
+	memset(data, 0, datasize);
+
+	/*
+	 * Write a 100 files and read them all back.
+	 */
+	assert((a = archive_write_new()) != NULL);
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a));
+	r = archive_write_add_filter_zstd(a);
+	if (r != ARCHIVE_OK) {
+		skipping("zstd writing not supported on this platform");
+		assertEqualInt(ARCHIVE_OK, archive_write_free(a));
+		free(buff);
+		free(data);
+		return;
+	}
+	assertEqualIntA(a, ARCHIVE_OK,
+	    archive_write_set_bytes_per_block(a, 10));
+	assertEqualInt(ARCHIVE_FILTER_ZSTD, archive_filter_code(a, 0));
+	assertEqualString("zstd", archive_filter_name(a, 0));
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_open_memory(a, buff, buffsize, &used1));
+	assertEqualInt(ARCHIVE_FILTER_ZSTD, archive_filter_code(a, 0));
+	assertEqualString("zstd", archive_filter_name(a, 0));
+	assert((ae = archive_entry_new()) != NULL);
+	archive_entry_set_filetype(ae, AE_IFREG);
+	archive_entry_set_size(ae, datasize);
+	for (i = 0; i < 100; i++) {
+		sprintf(path, "file%03d", i);
+		archive_entry_copy_pathname(ae, path);
+		assertEqualIntA(a, ARCHIVE_OK, archive_write_header(a, ae));
+		assertA(datasize
+		    == (size_t)archive_write_data(a, data, datasize));
+	}
+	archive_entry_free(ae);
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a));
+	assertEqualInt(ARCHIVE_OK, archive_write_free(a));
+
+	assert((a = archive_read_new()) != NULL);
+	assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a));
+	r = archive_read_support_filter_zstd(a);
+	if (r == ARCHIVE_WARN) {
+		skipping("Can't verify zstd writing by reading back;"
+		    " zstd reading not fully supported on this platform");
+	} else {
+		assertEqualIntA(a, ARCHIVE_OK,
+		    archive_read_support_filter_all(a));
+		assertEqualIntA(a, ARCHIVE_OK,
+		    archive_read_open_memory(a, buff, used1));
+		for (i = 0; i < 100; i++) {
+			sprintf(path, "file%03d", i);
+			if (!assertEqualInt(ARCHIVE_OK,
+				archive_read_next_header(a, &ae)))
+				break;
+			assertEqualString(path, archive_entry_pathname(ae));
+			assertEqualInt((int)datasize, archive_entry_size(ae));
+		}
+		assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
+	}
+	assertEqualInt(ARCHIVE_OK, archive_read_free(a));
+
+	/*
+	 * Repeat the cycle again, this time setting some compression
+	 * options.
+	 */
+	assert((a = archive_write_new()) != NULL);
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a));
+	assertEqualIntA(a, ARCHIVE_OK,
+	    archive_write_set_bytes_per_block(a, 10));
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_add_filter_zstd(a));
+	assertEqualIntA(a, ARCHIVE_FAILED,
+	    archive_write_set_filter_option(a, NULL, "nonexistent-option", "0"));
+	assertEqualIntA(a, ARCHIVE_FAILED,
+	    archive_write_set_filter_option(a, NULL, "compression-level", "abc"));
+	assertEqualIntA(a, ARCHIVE_FAILED,
+	    archive_write_set_filter_option(a, NULL, "compression-level", "25")); /* too big */
+	assertEqualIntA(a, ARCHIVE_OK,
+	    archive_write_set_filter_option(a, NULL, "compression-level", "9"));
+	assertEqualIntA(a, ARCHIVE_OK,
+	    archive_write_set_filter_option(a, NULL, "compression-level", "15"));
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_open_memory(a, buff, buffsize, &used2));
+	for (i = 0; i < 100; i++) {
+		sprintf(path, "file%03d", i);
+		assert((ae = archive_entry_new()) != NULL);
+		archive_entry_copy_pathname(ae, path);
+		archive_entry_set_size(ae, datasize);
+		archive_entry_set_filetype(ae, AE_IFREG);
+		assertEqualIntA(a, ARCHIVE_OK, archive_write_header(a, ae));
+		assertA(datasize == (size_t)archive_write_data(a, data, datasize));
+		archive_entry_free(ae);
+	}
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a));
+	assertEqualInt(ARCHIVE_OK, archive_write_free(a));
+
+	failure("compression-level=15 wrote %d bytes, default wrote %d bytes",
+	    (int)used2, (int)used1);
+	assert(used2 < used1);
+
+	assert((a = archive_read_new()) != NULL);
+	assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a));
+	r = archive_read_support_filter_zstd(a);
+	if (r == ARCHIVE_WARN) {
+		skipping("zstd reading not fully supported on this platform");
+	} else {
+		assertEqualIntA(a, ARCHIVE_OK,
+		    archive_read_support_filter_all(a));
+		assertEqualIntA(a, ARCHIVE_OK,
+		    archive_read_open_memory(a, buff, used2));
+		for (i = 0; i < 100; i++) {
+			sprintf(path, "file%03d", i);
+			failure("Trying to read %s", path);
+			if (!assertEqualIntA(a, ARCHIVE_OK,
+				archive_read_next_header(a, &ae)))
+				break;
+			assertEqualString(path, archive_entry_pathname(ae));
+			assertEqualInt((int)datasize, archive_entry_size(ae));
+		}
+		assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
+	}
+	assertEqualInt(ARCHIVE_OK, archive_read_free(a));
+
+	/*
+	 * Test various premature shutdown scenarios to make sure we
+	 * don't crash or leak memory.
+	 */
+	assert((a = archive_write_new()) != NULL);
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_add_filter_zstd(a));
+	assertEqualInt(ARCHIVE_OK, archive_write_free(a));
+
+	assert((a = archive_write_new()) != NULL);
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_add_filter_zstd(a));
+	assertEqualInt(ARCHIVE_OK, archive_write_close(a));
+	assertEqualInt(ARCHIVE_OK, archive_write_free(a));
+
+	assert((a = archive_write_new()) != NULL);
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a));
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_add_filter_zstd(a));
+	assertEqualInt(ARCHIVE_OK, archive_write_close(a));
+	assertEqualInt(ARCHIVE_OK, archive_write_free(a));
+
+	assert((a = archive_write_new()) != NULL);
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a));
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_add_filter_zstd(a));
+	assertEqualIntA(a, ARCHIVE_OK, archive_write_open_memory(a, buff, buffsize, &used2));
+	assertEqualInt(ARCHIVE_OK, archive_write_close(a));
+	assertEqualInt(ARCHIVE_OK, archive_write_free(a));
+
+	/*
+	 * Clean up.
+	 */
+	free(data);
+	free(buff);
+}

contrib/libarchive/tar/bsdtar.1

@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd February 25, 2017
+.Dd October 1, 2017
 .Dt TAR 1
 .Os
 .Sh NAME
@@ -302,19 +302,18 @@ containing the string
 Compress the resulting archive with
 .Xr xz 1 .
 In extract or list modes, this option is ignored.
-Note that, unlike other
+Note that this
 .Nm tar
-implementations, this implementation recognizes XZ compression
+implementation recognizes XZ compression automatically when reading archives.
-automatically when reading archives.
 .It Fl j , Fl Fl bzip , Fl Fl bzip2 , Fl Fl bunzip2
 (c mode only)
 Compress the resulting archive with
 .Xr bzip2 1 .
 In extract or list modes, this option is ignored.
-Note that, unlike other
+Note that this
 .Nm tar
-implementations, this implementation recognizes bzip2 compression
+implementation recognizes bzip2 compression automatically when reading
-automatically when reading archives.
+archives.
 .It Fl k , Fl Fl keep-old-files
 (x mode only)
 Do not overwrite existing files.
@@ -337,25 +336,41 @@ Issue a warning message unless all links to each file are archived.
 Compress the resulting archive with
 .Xr lrzip 1 .
 In extract or list modes, this option is ignored.
+Note that this
+.Nm tar
+implementation recognizes lrzip compression automatically when reading
+archives.
 .It Fl Fl lz4
 (c mode only)
 Compress the archive with lz4-compatible compression before writing it.
-In input mode, this option is ignored; lz4 compression is recognized
+In extract or list modes, this option is ignored.
-automatically on input.
+Note that this
+.Nm tar
+implementation recognizes lz4 compression automatically when reading archives.
+.It Fl Fl zstd
+(c mode only)
+Compress the archive with zstd-compatible compression before writing it.
+In extract or list modes, this option is ignored.
+Note that this
+.Nm tar
+implementation recognizes zstd compression automatically when reading archives.
 .It Fl Fl lzma
 (c mode only) Compress the resulting archive with the original LZMA algorithm.
+In extract or list modes, this option is ignored.
 Use of this option is discouraged and new archives should be created with
 .Fl Fl xz
 instead.
-Note that, unlike other
+Note that this
 .Nm tar
-implementations, this implementation recognizes LZMA compression
+implementation recognizes LZMA compression automatically when reading archives.
-automatically when reading archives.
 .It Fl Fl lzop
 (c mode only)
 Compress the resulting archive with
 .Xr lzop 1 .
 In extract or list modes, this option is ignored.
+Note that this
+.Nm tar
+implementation recognizes LZO compression automatically when reading archives.
 .It Fl m , Fl Fl modification-time
 (x mode only)
 Do not extract modification time.
@@ -577,6 +592,8 @@ A decimal integer from 4 to 7 specifying the lz4 compression block size
 .It Cm lz4:block-dependence
 Use the previous block of the block being compressed for
 a compression dictionary to improve compression ratio.
+.It Cm zstd:compression-level
+A decimal integer from 1 to 22 specifying the zstd compression level.
 .It Cm lzop:compression-level
 A decimal integer from 1 to 9 specifying the lzop compression level.
 .It Cm xz:compression-level
@@ -826,28 +843,28 @@ is run in x mode as root.
 Compress the resulting archive with
 .Xr bzip2 1 .
 In extract or list modes, this option is ignored.
-Note that, unlike other
+Note that this
 .Nm tar
-implementations, this implementation recognizes bzip2 compression
+implementation recognizes bzip2 compression automatically when reading
-automatically when reading archives.
+archives.
 .It Fl Z , Fl Fl compress , Fl Fl uncompress
 (c mode only)
 Compress the resulting archive with
 .Xr compress 1 .
 In extract or list modes, this option is ignored.
-Note that, unlike other
+Note that this
 .Nm tar
-implementations, this implementation recognizes compress compression
+implementation recognizes compress compression automatically when reading
-automatically when reading archives.
+archives.
 .It Fl z , Fl Fl gunzip , Fl Fl gzip
 (c mode only)
 Compress the resulting archive with
 .Xr gzip 1 .
 In extract or list modes, this option is ignored.
-Note that, unlike other
+Note that this
 .Nm tar
-implementations, this implementation recognizes gzip compression
+implementation recognizes gzip compression automatically when reading
-automatically when reading archives.
+archives.
 .El
 .Sh ENVIRONMENT
 The following environment variables affect the execution of

contrib/libarchive/tar/bsdtar.c

@@ -419,6 +419,7 @@ main(int argc, char **argv)
 		case OPTION_LZIP: /* GNU tar beginning with 1.23 */
 		case OPTION_LZMA: /* GNU tar beginning with 1.20 */
 		case OPTION_LZOP: /* GNU tar beginning with 1.21 */
+		case OPTION_ZSTD:
 			if (compression != '\0')
 				lafe_errc(1, 0,
 				    "Can't specify both -%c and -%c", opt,
@@ -430,6 +431,7 @@ main(int argc, char **argv)
 			case OPTION_LZIP: compression_name = "lzip"; break;
 			case OPTION_LZMA: compression_name = "lzma"; break;
 			case OPTION_LZOP: compression_name = "lzop"; break;
+			case OPTION_ZSTD: compression_name = "zstd"; break;
 			}
 			break;
 		case 'm': /* SUSv2 */

contrib/libarchive/tar/bsdtar.h

@@ -181,7 +181,8 @@ enum {
 	OPTION_USE_COMPRESS_PROGRAM,
 	OPTION_UUENCODE,
 	OPTION_VERSION,
-	OPTION_XATTRS
+	OPTION_XATTRS,
+	OPTION_ZSTD,
 };
 
 int	bsdtar_getopt(struct bsdtar *);

contrib/libarchive/tar/cmdline.c

@@ -160,6 +160,7 @@ static const struct bsdtar_option {
 	{ "version",              0, OPTION_VERSION },
 	{ "xattrs",               0, OPTION_XATTRS },
 	{ "xz",                   0, 'J' },
+	{ "zstd",                 0, OPTION_ZSTD },
 	{ NULL, 0, 0 }
 };
 

contrib/libarchive/tar/creation_set.c

@@ -80,6 +80,7 @@ get_filter_code(const char *suffix)
 		{ ".lzma",	"lzma" },
 		{ ".uu",	"uuencode" },
 		{ ".xz",	"xz" },
+		{ ".zst",	"zstd"},
 		{ NULL,		NULL }
 	};
 
@@ -121,6 +122,7 @@ decompose_alias(const char *suffix)
 		{ ".tzo",	".tar.lzo" },
 		{ ".taZ",	".tar.Z" },
 		{ ".tZ",	".tar.Z" },
+		{ ".tzst",	".tar.zst" },
 		{ NULL,		NULL }
 	};
 

contrib/libarchive/tar/test/test_extract.tar.zst.uu

@@ -0,0 +1,6 @@
+begin 644 test_extract.tar.zst
+M*+4O_010S0,`<L40$Z`5.(2U_RNV_[]L4V;Z_/R@1:7Y$3;9E`8$D$WI:W1)
+M'58'D3->Y+>!0*5E/PM"$7^K^1VI3SS-AX&_W0KQWY!-Z1(`_4$%[$"]<T!A
+L(*`#I!DXC4[J!6J8$!DJ$D"9$T*L]#G-$$/A`#`I`-(`UUKAU$Z@"`UXII``
+`
+end

contrib/libarchive/tar/test/test_extract_tar_zstd.c

@@ -0,0 +1,48 @@
+/*-
+ * Copyright (c) 2017 Sean Purcell
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include "test.h"
+__FBSDID("$FreeBSD$");
+
+DEFINE_TEST(test_extract_tar_zstd)
+{
+	const char *reffile = "test_extract.tar.zst";
+	int f;
+
+	extract_reference_file(reffile);
+	f = systemf("%s -tf %s >test.out 2>test.err", testprog, reffile);
+	if (f == 0 || canZstd()) {
+		assertEqualInt(0, systemf("%s -xf %s >test.out 2>test.err",
+		    testprog, reffile));
+
+		assertFileExists("file1");
+		assertTextFileContents("contents of file1.\n", "file1");
+		assertFileExists("file2");
+		assertTextFileContents("contents of file2.\n", "file2");
+		assertEmptyFile("test.out");
+		assertEmptyFile("test.err");
+	} else {
+		skipping("It seems zstd is not supported on this platform");
+	}
+}

contrib/libarchive/tar/test/test_option_acls.c

@@ -483,7 +483,7 @@ DEFINE_TEST(test_option_acls)
 	r = compare_acls("f", "acls_acls/f");
 	assertEqualInt(r, 1);
 
-	/* Extractl acls without acls */
+	/* Extract acls without acls */
 	assertMakeDir("acls_noacls", 0755);
 	clear_inheritance_flags("acls_noacls", acltype);
 	r = systemf("%s -x -C acls_noacls -p --no-acls -f acls.tar >acls_noacls.out 2>acls_noacls.err", testprog);

contrib/libarchive/tar/test/test_option_zstd.c

@@ -0,0 +1,85 @@
+/*-
+ * Copyright (c) 2017 Sean Purcell
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include "test.h"
+__FBSDID("$FreeBSD$");
+
+DEFINE_TEST(test_option_zstd)
+{
+	char *p;
+	int r;
+	size_t s;
+
+	/* Create a file. */
+	assertMakeFile("f", 0644, "a");
+
+	/* Archive it with lz4 compression. */
+	r = systemf("%s -cf - --zstd f >archive.out 2>archive.err",
+	    testprog);
+	p = slurpfile(&s, "archive.err");
+	p[s] = '\0';
+	if (r != 0) {
+		if (strstr(p, "Unsupported compression") != NULL) {
+			skipping("This version of bsdtar was compiled "
+			    "without zstd support");
+			goto done;
+		}
+		/* POSIX permits different handling of the spawnp
+		 * system call used to launch the subsidiary
+		 * program: */
+		/* Some systems fail immediately to spawn the new process. */
+		if (strstr(p, "Can't launch") != NULL && !canZstd()) {
+			skipping("This version of bsdtar uses an external zstd program "
+			    "but no such program is available on this system.");
+			goto done;
+		}
+		/* Some systems successfully spawn the new process,
+		 * but fail to exec a program within that process.
+		 * This results in failure at the first attempt to
+		 * write. */
+		if (strstr(p, "Can't write") != NULL && !canZstd()) {
+			skipping("This version of bsdtar uses an external zstd program "
+			    "but no such program is available on this system.");
+			goto done;
+		}
+		/* On some systems the error won't be detected until closing
+		   time, by a 127 exit error returned by waitpid. */
+		if (strstr(p, "Error closing") != NULL && !canZstd()) {
+			skipping("This version of bsdcpio uses an external zstd program "
+			    "but no such program is available on this system.");
+			return;
+		}
+		failure("--zstd option is broken: %s", p);
+		assertEqualInt(r, 0);
+		goto done;
+	}
+	free(p);
+	/* Check that the archive file has an lz4 signature. */
+	p = slurpfile(&s, "archive.out");
+	assert(s > 2);
+	assertEqualMem(p, "\x28\xb5\x2f\xfd", 4);
+
+done:
+	free(p);
+}

contrib/libarchive/test_utils/test_common.h

@@ -329,6 +329,9 @@ int canLrzip(void);
 /* Return true if this platform can run the "lz4" program. */
 int canLz4(void);
 
+/* Return true if this platform can run the "zstd" program. */
+int canZstd(void);
+
 /* Return true if this platform can run the "lzip" program. */
 int canLzip(void);
 

contrib/libarchive/test_utils/test_main.c

@@ -2318,6 +2318,21 @@ canLz4(void)
 	return (value);
 }
 
+/*
+ * Can this platform run the zstd program?
+ */
+int
+canZstd(void)
+{
+	static int tested = 0, value = 0;
+	if (!tested) {
+		tested = 1;
+		if (systemf("zstd -V %s", redirectArgs) == 0)
+			value = 1;
+	}
+	return (value);
+}
+
 /*
  * Can this platform run the lzip program?
  */

lib/libarchive/Makefile

@@ -94,6 +94,7 @@ SRCS=	archive_acl.c					\
 	archive_read_support_filter_rpm.c		\
 	archive_read_support_filter_uu.c		\
 	archive_read_support_filter_xz.c		\
+	archive_read_support_filter_zstd.c		\
 	archive_read_support_format_7zip.c		\
 	archive_read_support_format_all.c		\
 	archive_read_support_format_ar.c		\
@@ -136,6 +137,7 @@ SRCS=	archive_acl.c					\
 	archive_write_add_filter_program.c		\
 	archive_write_add_filter_uuencode.c		\
 	archive_write_add_filter_xz.c			\
+	archive_write_add_filter_zstd.c			\
 	archive_write_set_format.c			\
 	archive_write_set_format_7zip.c			\
 	archive_write_set_format_ar.c			\

lib/libarchive/tests/Makefile

@@ -83,6 +83,7 @@ TESTS_SRCS= \
 	test_compat_uudecode_large.c		\
 	test_compat_xz.c			\
 	test_compat_zip.c			\
+	test_compat_zstd.c			\
 	test_empty_write.c			\
 	test_entry.c				\
 	test_entry_strmode.c			\
@@ -240,6 +241,7 @@ TESTS_SRCS= \
 	test_write_filter_program.c		\
 	test_write_filter_uuencode.c		\
 	test_write_filter_xz.c			\
+	test_write_filter_zstd.c		\
 	test_write_format_7zip.c		\
 	test_write_format_7zip_empty.c		\
 	test_write_format_7zip_large.c		\
@@ -373,6 +375,7 @@ ${PACKAGE}FILES+=	test_compat_zip_4.zip.uu
 ${PACKAGE}FILES+=	test_compat_zip_5.zip.uu
 ${PACKAGE}FILES+=	test_compat_zip_6.zip.uu
 ${PACKAGE}FILES+=	test_compat_zip_7.xps.uu
+${PACKAGE}FILES+=	test_compat_zstd_1.tar.zst.uu
 ${PACKAGE}FILES+=	test_fuzz.cab.uu
 ${PACKAGE}FILES+=	test_fuzz.lzh.uu
 ${PACKAGE}FILES+=	test_fuzz_1.iso.Z.uu

usr.bin/bsdcat/tests/Makefile

@@ -28,6 +28,7 @@ TESTS_SRCS=	\
 	test_empty_gz.c				\
 	test_empty_lz4.c			\
 	test_empty_xz.c				\
+	test_empty_zstd.c			\
 	test_error.c				\
 	test_error_mixed.c			\
 	test_expand_Z.c				\
@@ -37,6 +38,7 @@ TESTS_SRCS=	\
 	test_expand_mixed.c			\
 	test_expand_plain.c			\
 	test_expand_xz.c			\
+	test_expand_zstd.c			\
 	test_help.c				\
 	test_version.c
 
@@ -59,11 +61,13 @@ CLEANFILES+=	list.h list.h.tmp
 ${PACKAGE}FILES+=	test_empty.gz.uu
 ${PACKAGE}FILES+=	test_empty.lz4.uu
 ${PACKAGE}FILES+=	test_empty.xz.uu
+${PACKAGE}FILES+=	test_empty.zst.uu
 ${PACKAGE}FILES+=	test_expand.Z.uu
 ${PACKAGE}FILES+=	test_expand.bz2.uu
 ${PACKAGE}FILES+=	test_expand.gz.uu
 ${PACKAGE}FILES+=	test_expand.lz4.uu
 ${PACKAGE}FILES+=	test_expand.plain.uu
 ${PACKAGE}FILES+=	test_expand.xz.uu
+${PACKAGE}FILES+=	test_expand.zst.uu
 
 .include <bsd.test.mk>

usr.bin/cpio/tests/Makefile

@@ -43,6 +43,7 @@ TESTS_SRCS=	\
 	test_extract_cpio_lzma.c		\
 	test_extract_cpio_lzo.c			\
 	test_extract_cpio_xz.c			\
+	test_extract_cpio_zstd.c		\
 	test_format_newc.c			\
 	test_gcpio_compat.c			\
 	test_missing_file.c			\
@@ -73,6 +74,7 @@ TESTS_SRCS=	\
 	test_option_xz.c			\
 	test_option_y.c				\
 	test_option_z.c				\
+	test_option_zstd.c			\
 	test_owner_parse.c			\
 	test_passthrough_dotdot.c		\
 	test_passthrough_reverse.c
@@ -104,6 +106,7 @@ ${PACKAGE}FILES+= test_extract.cpio.lz4.uu
 ${PACKAGE}FILES+= test_extract.cpio.lzma.uu
 ${PACKAGE}FILES+= test_extract.cpio.lzo.uu
 ${PACKAGE}FILES+= test_extract.cpio.xz.uu
+${PACKAGE}FILES+= test_extract.cpio.zst.uu
 ${PACKAGE}FILES+= test_gcpio_compat_ref.bin.uu
 ${PACKAGE}FILES+= test_gcpio_compat_ref.crc.uu
 ${PACKAGE}FILES+= test_gcpio_compat_ref.newc.uu

usr.bin/tar/tests/Makefile

@@ -35,6 +35,7 @@ TESTS_SRCS=	\
 	test_extract_tar_lzma.c			\
 	test_extract_tar_lzo.c			\
 	test_extract_tar_xz.c			\
+	test_extract_tar_zstd.c			\
 	test_format_newc.c			\
 	test_help.c				\
 	test_leading_slash.c			\
@@ -74,6 +75,7 @@ TESTS_SRCS=	\
 	test_option_xattrs.c			\
 	test_option_xz.c			\
 	test_option_z.c				\
+	test_option_zstd.c			\
 	test_patterns.c				\
 	test_print_longpath.c			\
 	test_stdio.c				\
@@ -108,6 +110,7 @@ ${PACKAGE}FILES+=	test_extract.tar.lz4.uu
 ${PACKAGE}FILES+=	test_extract.tar.lzma.uu
 ${PACKAGE}FILES+=	test_extract.tar.lzo.uu
 ${PACKAGE}FILES+=	test_extract.tar.xz.uu
+${PACKAGE}FILES+=	test_extract.tar.zst.uu
 ${PACKAGE}FILES+=	test_leading_slash.tar.uu
 ${PACKAGE}FILES+=	test_option_keep_newer_files.tar.Z.uu
 ${PACKAGE}FILES+=	test_option_passphrase.zip.uu