From bd9cab6fb43aa1cebd5b0c4b05201146dae3c1a9 Mon Sep 17 00:00:00 2001
From: Rui Paulo <rpaulo@FreeBSD.org>
Date: Tue, 17 Feb 2015 23:13:45 +0000
Subject: [PATCH] release: use xz via pipe when compressing the tarballs.

libarchive(3) doesn't support the new liblzma API yet, but this change
allows us to enable multi-threaded xz compression.
``make release'' should now finish in half the time on a machine with
several cores and fast disks (our typical build server).

This behaviour only applies when building a release and it doesn't
affect buildworld/installworld.  To disable threaded xz compression,
set XZ_THREADS=1.

Reviewed by:	gjb
Tested by:	gjb
---
 Makefile.inc1       | 32 +++++++++++++++++---------------
 release/Makefile    | 27 +++++++--------------------
 release/Makefile.vm |  2 +-
 share/mk/bsd.own.mk |  9 +++++++++
 4 files changed, 34 insertions(+), 36 deletions(-)

diff --git a/Makefile.inc1 b/Makefile.inc1
index 4dcbc0f97d61..e3b68dd361b7 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -912,25 +912,25 @@ packageworld:
 .for dist in base ${EXTRA_DISTRIBUTIONS}
 .if defined(NO_ROOT)
 	${_+_}cd ${DESTDIR}/${DISTDIR}/${dist}; \
-	    tar cvJf ${DESTDIR}/${DISTDIR}/${dist}.txz \
-	    --exclude usr/lib/debug \
-	    @${DESTDIR}/${DISTDIR}/${dist}.meta
+	    tar cvf - --exclude usr/lib/debug \
+	    @${DESTDIR}/${DISTDIR}/${dist}.meta | \
+	    ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/${dist}.txz
 .else
 	${_+_}cd ${DESTDIR}/${DISTDIR}/${dist}; \
-	    tar cvJf ${DESTDIR}/${DISTDIR}/${dist}.txz \
-	    --exclude usr/lib/debug .
+	    tar cvf - --exclude usr/lib/debug . | \
+	    ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/${dist}.txz
 .endif
 .endfor
 
 .for dist in ${DEBUG_DISTRIBUTIONS}
 . if defined(NO_ROOT)
 	${_+_}cd ${DESTDIR}/${DISTDIR}/${dist}; \
-	    tar cvJf ${DESTDIR}/${DISTDIR}/${dist}-dbg.txz \
-	    @${DESTDIR}/${DISTDIR}/${dist}.debug.meta
+	    tar cvf - @${DESTDIR}/${DISTDIR}/${dist}.debug.meta | \
+	    ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/${dist}-dbg.txz
 . else
 	${_+_}cd ${DESTDIR}/${DISTDIR}/${dist}; \
-	    tar cvJLf ${DESTDIR}/${DISTDIR}/${dist}-dbg.txz \
-	    usr/lib/debug
+	    tar cvLf - usr/lib/debug | \
+	    ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/${dist}-dbg.txz
 . endif
 .endfor
 
@@ -1138,19 +1138,21 @@ distributekernel distributekernel.debug:
 packagekernel:
 .if defined(NO_ROOT)
 	cd ${DESTDIR}/${DISTDIR}/kernel; \
-	    tar cvJf ${DESTDIR}/${DISTDIR}/kernel.txz \
-	    @${DESTDIR}/${DISTDIR}/kernel.meta
+	    tar cvf - @${DESTDIR}/${DISTDIR}/kernel.meta | \
+	    ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel.txz
 .for _kernel in ${BUILDKERNELS:S/${INSTALLKERNEL}//}
 	cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \
-	    tar cvJf ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.txz \
-	    @${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta
+	    tar cvf - @${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta | \
+	    ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.txz
 .endfor
 .else
 	cd ${DESTDIR}/${DISTDIR}/kernel; \
-	    tar cvJf ${DESTDIR}/${DISTDIR}/kernel.txz .
+	    tar cvf - . | \
+	    ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel.txz
 .for _kernel in ${BUILDKERNELS:S/${INSTALLKERNEL}//}
 	cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \
-	    tar cvJf ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.txz .
+	    tar cvf - . | \
+	    ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.txz
 .endfor
 .endif
 
diff --git a/release/Makefile b/release/Makefile
index 0a3f50972d15..e469307068dd 100644
--- a/release/Makefile
+++ b/release/Makefile
@@ -33,18 +33,12 @@
 #  		with xz(1) (extremely time consuming)
 #  WITH_CLOUDWARE: if set, build cloud hosting disk images with the release
 #  TARGET/TARGET_ARCH: architecture of built release
-#  XZ_FLAGS: Additional arguments to pass to xz(1)
-#  XZ_THREADS: Number of xz(1) threads to use
-#  NO_XZTHREADS: Disable multi-threaded xz(1) compression
 #
 
 WORLDDIR?=	${.CURDIR}/..
 PORTSDIR?=	/usr/ports
 DOCDIR?=	/usr/doc
 RELNOTES_LANG?= en_US.ISO8859-1
-XZCMD?=		/usr/bin/xz
-XZ_FLAGS?=
-XZ_THREADS?=
 
 .if !defined(TARGET) || empty(TARGET)
 TARGET=		${MACHINE}
@@ -59,15 +53,6 @@ TARGET_ARCH=	${TARGET}
 IMAKE=		${MAKE} TARGET_ARCH=${TARGET_ARCH} TARGET=${TARGET}
 DISTDIR=	dist
 
-.if !defined(NO_XZTHREADS) && empty(XZ_THREADS)
-XZ_THREADS=	0
-.else
-XZ_THREADS=	${XZ_THREADS}
-.endif
-.if !empty(XZ_THREADS)
-XZ_FLAGS+=	-T ${XZ_THREADS}
-.endif
-
 # Define OSRELEASE by using newvars.sh
 .if !defined(OSRELEASE) || empty(OSRELEASE)
 .for _V in TYPE BRANCH REVISION
@@ -158,16 +143,18 @@ kernel.txz:
 src.txz:
 	mkdir -p ${DISTDIR}/usr
 	ln -fs ${WORLDDIR} ${DISTDIR}/usr/src
-	cd ${DISTDIR} && tar cLvJf ${.OBJDIR}/src.txz --exclude .svn --exclude .zfs \
-	    --exclude .git --exclude @ --exclude usr/src/release/dist usr/src
+	cd ${DISTDIR} && tar cLvf - --exclude .svn --exclude .zfs \
+	    --exclude .git --exclude @ --exclude usr/src/release/dist usr/src | \
+	    ${XZ_CMD} > ${.OBJDIR}/src.txz
 
 ports.txz:
 	mkdir -p ${DISTDIR}/usr
 	ln -fs ${PORTSDIR} ${DISTDIR}/usr/ports
-	cd ${DISTDIR} && tar cLvJf ${.OBJDIR}/ports.txz \
+	cd ${DISTDIR} && tar cLvf - \
 	    --exclude .git --exclude .svn \
 	    --exclude usr/ports/distfiles --exclude usr/ports/packages \
-	    --exclude 'usr/ports/INDEX*' --exclude work usr/ports
+	    --exclude 'usr/ports/INDEX*' --exclude work usr/ports | \
+	    ${XZ_CMD} > ${.OBJDIR}/ports.txz
 
 reldoc:
 	cd ${.CURDIR}/doc && ${MAKE} all install clean 'FORMATS=html txt' \
@@ -309,7 +296,7 @@ release-install:
 .for I in ${IMAGES}
 	cp -p ${I} ${DESTDIR}/${OSRELEASE}-${I}
 . if defined(WITH_COMPRESSED_IMAGES) && !empty(WITH_COMPRESSED_IMAGES)
-	${XZCMD} ${XZ_FLAGS} -k ${DESTDIR}/${OSRELEASE}-${I}
+	${XZ_CMD} -k ${DESTDIR}/${OSRELEASE}-${I}
 . endif
 .endfor
 	cd ${DESTDIR} && sha256 ${OSRELEASE}* > ${DESTDIR}/CHECKSUM.SHA256
diff --git a/release/Makefile.vm b/release/Makefile.vm
index 34f31665fbce..50365f579eb3 100644
--- a/release/Makefile.vm
+++ b/release/Makefile.vm
@@ -101,7 +101,7 @@ vm-install:
 # the DESTDIR.
 .  for FORMAT in ${VMFORMATS}
 	# Don't keep the originals.  There is a copy in ${.OBJDIR} if needed.
-	${XZCMD} ${XZ_FLAGS} ${DESTDIR}/vmimages/${OSRELEASE}.${FORMAT}
+	${XZ_CMD} ${DESTDIR}/vmimages/${OSRELEASE}.${FORMAT}
 .  endfor
 . endif
 	cd ${DESTDIR}/vmimages && sha256 ${OSRELEASE}* > \
diff --git a/share/mk/bsd.own.mk b/share/mk/bsd.own.mk
index ca77f859705f..0e85b9cb52dd 100644
--- a/share/mk/bsd.own.mk
+++ b/share/mk/bsd.own.mk
@@ -234,6 +234,15 @@ STRIP?=		-s
 COMPRESS_CMD?=	gzip -cn
 COMPRESS_EXT?=	.gz
 
+# Set XZ_THREADS to 1 to disable multi-threading.
+XZ_THREADS?=	0
+
+.if !empty(XZ_THREADS)
+XZ_CMD?=	xz -T ${XZ_THREADS}
+.else
+XZ_CMD?=	xz
+.endif
+
 # Pointer to the top directory into which tests are installed.  Should not be
 # overriden by Makefiles, but the user may choose to set this in src.conf(5).
 TESTSBASE?= /usr/tests