gzip: Add support for decompressing zstd files.

Sponsored by:	Klara, Inc.
Differential Revision: https://reviews.freebsd.org/D37236

usr.bin/gzip/Makefile

@@ -6,7 +6,7 @@
 PROG=		gzip
 MAN=		gzip.1 gzexe.1 zdiff.1 zforce.1 zmore.1 znew.1
 
-LIBADD=		z lzma
+LIBADD=		z lzma zstd
 
 .if ${MK_BZIP2_SUPPORT} != "no"
 LIBADD+=	bz2
@@ -14,6 +14,8 @@ LIBADD+=	bz2
 CFLAGS+=	-DNO_BZIP2_SUPPORT
 .endif
 
+CFLAGS+=	-I${SRCTOP}/sys/contrib/zstd/lib
+
 SCRIPTS=	gzexe zdiff zforce zmore znew
 
 MLINKS+=	gzip.1 gunzip.1 \

usr.bin/gzip/Makefile.depend

@@ -10,6 +10,7 @@ DIRDEPS = \
 	lib/liblzma \
 	lib/libthr \
 	lib/libz \
+	lib/libzstd \
 
 
 .include <dirdeps.mk>

usr.bin/gzip/gzip.1

@@ -25,7 +25,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
-.Dd January 7, 2019
+.Dd November 2, 2022
 .Dt GZIP 1
 .Os
 .Sh NAME
@@ -110,6 +110,7 @@ is also capable of decompressing files compressed using
 .Xr compress 1 ,
 .Xr bzip2 1 ,
 .Ar lzip ,
+.Xr zstd 1 ,
 or
 .Xr xz 1 .
 .Sh OPTIONS
@@ -203,6 +204,7 @@ Report progress to standard error.
 .Sh SEE ALSO
 .Xr bzip2 1 ,
 .Xr compress 1 ,
+.Xr zstd 1 ,
 .Xr xz 1 ,
 .Xr fts 3 ,
 .Xr zlib 3

usr.bin/gzip/gzip.c

@@ -86,6 +86,9 @@ enum filetype {
 #endif
 #ifndef NO_LZ_SUPPORT
 	FT_LZ,
+#endif
+#ifndef NO_ZSTD_SUPPORT
+	FT_ZSTD,
 #endif
 	FT_LAST,
 	FT_UNKNOWN
@@ -118,6 +121,12 @@ enum filetype {
 #define LZ_MAGIC	"LZIP"
 #endif
 
+#ifndef NO_ZSTD_SUPPORT
+#include <zstd.h>
+#define ZSTD_SUFFIX	".zst"
+#define ZSTD_MAGIC	"\050\265\057\375"
+#endif
+
 #define GZ_SUFFIX	".gz"
 
 #define BUFLEN		(64 * 1024)
@@ -164,6 +173,9 @@ static suffixes_t suffixes[] = {
 #endif
 #ifndef NO_LZ_SUPPORT
 	SUFFIX(LZ_SUFFIX,	""),
+#endif
+#ifndef NO_ZSTD_SUPPORT
+	SUFFIX(ZSTD_SUFFIX,	""),
 #endif
 	SUFFIX(GZ_SUFFIX,	""),	/* Overwritten by -S "" */
 #undef SUFFIX
@@ -221,7 +233,7 @@ static	const char *infile;		/* name of file coming in */
 
 static	void	maybe_err(const char *fmt, ...) __printflike(1, 2) __dead2;
 #if !defined(NO_BZIP2_SUPPORT) || !defined(NO_PACK_SUPPORT) ||	\
-    !defined(NO_XZ_SUPPORT)
+    !defined(NO_XZ_SUPPORT) || !defined(NO_ZSTD_SUPPORT)
 static	void	maybe_errx(const char *fmt, ...) __printflike(1, 2) __dead2;
 #endif
 static	void	maybe_warn(const char *fmt, ...) __printflike(1, 2);
@@ -284,6 +296,10 @@ static	off_t	unxz_len(int);
 static	off_t	unlz(int, int, char *, size_t, off_t *);
 #endif
 
+#ifndef NO_ZSTD_SUPPORT
+static	off_t	unzstd(int, int, char *, size_t, off_t *);
+#endif
+
 static const struct option longopts[] = {
 	{ "stdout",		no_argument,		0,	'c' },
 	{ "to-stdout",		no_argument,		0,	'c' },
@@ -467,7 +483,7 @@ maybe_err(const char *fmt, ...)
 }
 
 #if !defined(NO_BZIP2_SUPPORT) || !defined(NO_PACK_SUPPORT) ||	\
-    !defined(NO_XZ_SUPPORT)
+    !defined(NO_XZ_SUPPORT) || !defined(NO_ZSTD_SUPPORT)
 /* ... without an errno. */
 void
 maybe_errx(const char *fmt, ...)
@@ -1098,33 +1114,32 @@ file_gettype(u_char *buf)
 	if (buf[0] == GZIP_MAGIC0 &&
 	    (buf[1] == GZIP_MAGIC1 || buf[1] == GZIP_OMAGIC1))
 		return FT_GZIP;
-	else
 #ifndef NO_BZIP2_SUPPORT
-	if (memcmp(buf, BZIP2_MAGIC, 3) == 0 &&
+	else if (memcmp(buf, BZIP2_MAGIC, 3) == 0 &&
 	    buf[3] >= '0' && buf[3] <= '9')
 		return FT_BZIP2;
-	else
 #endif
 #ifndef NO_COMPRESS_SUPPORT
-	if (memcmp(buf, Z_MAGIC, 2) == 0)
+	else if (memcmp(buf, Z_MAGIC, 2) == 0)
 		return FT_Z;
-	else
 #endif
 #ifndef NO_PACK_SUPPORT
-	if (memcmp(buf, PACK_MAGIC, 2) == 0)
+	else if (memcmp(buf, PACK_MAGIC, 2) == 0)
 		return FT_PACK;
-	else
 #endif
 #ifndef NO_XZ_SUPPORT
-	if (memcmp(buf, XZ_MAGIC, 4) == 0)	/* XXX: We only have 4 bytes */
+	else if (memcmp(buf, XZ_MAGIC, 4) == 0)	/* XXX: We only have 4 bytes */
 		return FT_XZ;
-	else
 #endif
 #ifndef NO_LZ_SUPPORT
-	if (memcmp(buf, LZ_MAGIC, 4) == 0)
+	else if (memcmp(buf, LZ_MAGIC, 4) == 0)
 		return FT_LZ;
-	else
 #endif
+#ifndef NO_ZSTD_SUPPORT
+	else if (memcmp(buf, ZSTD_MAGIC, 4) == 0)
+		return FT_ZSTD;
+#endif
+	else
 		return FT_UNKNOWN;
 }
 
@@ -1586,6 +1601,16 @@ file_uncompress(char *file, char *outfile, size_t outsize)
 		size = unlz(fd, zfd, NULL, 0, NULL);
 		break;
 #endif
+
+#ifndef NO_ZSTD_SUPPORT
+	case FT_ZSTD:
+		if (lflag) {
+			maybe_warnx("no -l with zstd files");
+			goto lose;
+		}
+		size = unzstd(fd, zfd, NULL, 0, NULL);
+		break;
+#endif
 	case FT_UNKNOWN:
 		if (lflag) {
 			maybe_warnx("no -l for unknown filetypes");
@@ -1813,6 +1838,12 @@ handle_stdin(void)
 		usize = unlz(STDIN_FILENO, STDOUT_FILENO,
 			     (char *)fourbytes, sizeof fourbytes, &gsize);
 		break;
+#endif
+#ifndef NO_ZSTD_SUPPORT
+	case FT_ZSTD:
+		usize = unzstd(STDIN_FILENO, STDOUT_FILENO,
+			       (char *)fourbytes, sizeof fourbytes, &gsize);
+		break;
 #endif
 	}
 
@@ -2188,6 +2219,9 @@ display_version(void)
 #ifndef NO_LZ_SUPPORT
 #include "unlz.c"
 #endif
+#ifndef NO_ZSTD_SUPPORT
+#include "unzstd.c"
+#endif
 
 static ssize_t
 read_retry(int fd, void *buf, size_t sz)

usr.bin/gzip/unzstd.c

@@ -0,0 +1,89 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2022 Klara, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/* This file is #included by gzip.c */
+
+static off_t
+unzstd(int in, int out, char *pre, size_t prelen, off_t *bytes_in)
+{
+	static char *ibuf, *obuf;
+	ZSTD_inBuffer zib;
+	ZSTD_outBuffer zob;
+	ZSTD_DCtx *zds;
+	ssize_t res;
+	size_t zres;
+	size_t bytes_out = 0;
+	int eof = 0;
+
+	if (ibuf == NULL)
+		ibuf = malloc(BUFLEN);
+	if (obuf == NULL)
+		obuf = malloc(BUFLEN);
+	if (ibuf == NULL || obuf == NULL)
+		maybe_err("malloc");
+
+	zds = ZSTD_createDStream();
+	ZSTD_initDStream(zds);
+
+	zib.src = pre;
+	zib.size = prelen;
+	zib.pos = 0;
+	if (bytes_in != NULL)
+		*bytes_in = prelen;
+	zob.dst = obuf;
+	zob.size = BUFLEN;
+	zob.pos = 0;
+
+	while (!eof) {
+		if (zib.pos >= zib.size) {
+			res = read(in, ibuf, BUFLEN);
+			if (res < 0)
+				maybe_err("read");
+			if (res == 0)
+				eof = 1;
+			infile_newdata(res);
+			zib.src = ibuf;
+			zib.size = res;
+			zib.pos = 0;
+			if (bytes_in != NULL)
+				*bytes_in += res;
+		}
+		zres = ZSTD_decompressStream(zds, &zob, &zib);
+		if (ZSTD_isError(zres)) {
+			maybe_errx("%s", ZSTD_getErrorName(zres));
+		}
+		if (zob.pos > 0) {
+			res = write(out, obuf, zob.pos);
+			if (res < 0)
+				maybe_err("write");
+			zob.pos = 0;
+			bytes_out += res;
+		}
+	}
+	ZSTD_freeDStream(zds);
+	return (bytes_out);
+}