Merge ^/projects/release-install-debug:

- Rework MANIFEST generation and parsing via bsdinstall(8).
- Allow selecting debugging distribution sets during install.
- Rework bsdinstall(8) to fetch remote debug distribution sets
  when they are not available on the local install medium.
- Allow selecting additional non-GENERIC kernels during install.
  At present, GENERIC is still required, and installed by default.

Tested with:	head@r293203
Sponsored by:	The FreeBSD Foundation

Makefile.inc1

@@ -1284,27 +1284,43 @@ packagekernel:
 .if defined(NO_ROOT)
 .if !defined(NO_INSTALLKERNEL)
 	cd ${DESTDIR}/${DISTDIR}/kernel; \
-	    tar cvf - @${DESTDIR}/${DISTDIR}/kernel.meta | \
+	    tar cvf - --exclude '*.debug' \
+	    @${DESTDIR}/${DISTDIR}/kernel.meta | \
 	    ${XZ_CMD} > ${PACKAGEDIR}/kernel.txz
 .endif
+	cd ${DESTDIR}/${DISTDIR}/kernel; \
+	    tar cvf - --include '*/*/*.debug' \
+	    @${DESTDIR}/${DISTDIR}/kernel.meta | \
+	    ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel-dbg.txz
 .if ${BUILDKERNELS:[#]} > 1
 .for _kernel in ${BUILDKERNELS:[2..-1]}
 	cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \
-	    tar cvf - @${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta | \
+	    tar cvf - --exclude '*.debug' \
+	    @${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta | \
 	    ${XZ_CMD} > ${PACKAGEDIR}/kernel.${_kernel}.txz
+	cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \
+	    tar cvf - --include '*/*/*.debug' \
+	    @${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta | \
+	    ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel.${_kernel}-dbg.txz
 .endfor
 .endif
 .else
 .if !defined(NO_INSTALLKERNEL)
 	cd ${DESTDIR}/${DISTDIR}/kernel; \
-	    tar cvf - . | \
+	    tar cvf - --exclude '*.debug' . | \
 	    ${XZ_CMD} > ${PACKAGEDIR}/kernel.txz
 .endif
+	cd ${DESTDIR}/${DISTDIR}/kernel; \
+	    tar cvf - --include '*/*/*.debug' $$(eval find .) | \
+	    ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel-dbg.txz
 .if ${BUILDKERNELS:[#]} > 1
 .for _kernel in ${BUILDKERNELS:[2..-1]}
 	cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \
-	    tar cvf - . | \
+	    tar cvf - --exclude '*.debug' . | \
 	    ${XZ_CMD} > ${PACKAGEDIR}/kernel.${_kernel}.txz
+	cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \
+	    tar cvf - --include '*/*/*.debug' $$(eval find .) | \
+	    ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel.${_kernel}-dbg.txz
 .endfor
 .endif
 .endif

release/Makefile

@@ -176,7 +176,7 @@ disc1: packagesystem
 		MK_DEBUG_FILES=no
 # Copy distfiles
 	mkdir -p ${.TARGET}/usr/freebsd-dist
-	for dist in MANIFEST $$(ls *.txz | grep -v -- '-dbg'); \
+	for dist in MANIFEST $$(ls *.txz | grep -vE -- '(base|lib32)-dbg'); \
 	    do cp $${dist} ${.TARGET}/usr/freebsd-dist; \
 	done
 # Copy documentation, if generated
@@ -225,7 +225,7 @@ dvd: packagesystem
 		MK_TESTS=no MK_DEBUG_FILES=no
 # Copy distfiles
 	mkdir -p ${.TARGET}/usr/freebsd-dist
-	for dist in MANIFEST $$(ls *.txz | grep -v -- '-dbg'); \
+	for dist in MANIFEST $$(ls *.txz | grep -v -- '(base|lib32)-dbg'); \
 	    do cp $${dist} ${.TARGET}/usr/freebsd-dist; \
 	done
 # Copy documentation, if generated

release/amd64/mkisoimages.sh

@@ -56,5 +56,5 @@ NAME="$1"; shift
 publisher="The FreeBSD Project.  http://www.FreeBSD.org/"
 echo "/dev/iso9660/$LABEL / cd9660 ro 0 0" > "$1/etc/fstab"
 makefs -t cd9660 $bootable -o rockridge -o label="$LABEL" -o publisher="$publisher" "$NAME" "$@"
-rm "$1/etc/fstab"
+rm -f "$1/etc/fstab"
 rm -f efiboot.img

release/i386/mkisoimages.sh

@@ -42,4 +42,4 @@ NAME="$1"; shift
 publisher="The FreeBSD Project.  http://www.FreeBSD.org/"
 echo "/dev/iso9660/$LABEL / cd9660 ro 0 0" > "$1/etc/fstab"
 makefs -t cd9660 $bootable -o rockridge -o label="$LABEL" -o publisher="$publisher" "$NAME" "$@"
-rm "$1/etc/fstab"
+rm -f "$1/etc/fstab"

release/pc98/mkisoimages.sh

@@ -42,4 +42,4 @@ NAME="$1"; shift
 publisher="The FreeBSD Project.  http://www.FreeBSD.org/"
 echo "/dev/iso9660/$LABEL / cd9660 ro 0 0" > "$1/etc/fstab"
 makefs -t cd9660 $bootable -o rockridge -o label="$LABEL" -o publisher="$publisher" "$NAME" "$@"
-rm "$1/etc/fstab"
+rm -f "$1/etc/fstab"

release/powerpc/mkisoimages.sh

@@ -64,6 +64,6 @@ NAME="$1"; shift
 publisher="The FreeBSD Project.  http://www.FreeBSD.org/"
 echo "/dev/iso9660/$LABEL / cd9660 ro 0 0" > "$1/etc/fstab"
 makefs -t cd9660 $bootable -o rockridge -o label="$LABEL" -o publisher="$publisher" "$NAME" "$@"
-rm "$1/etc/fstab"
+rm -f "$1/etc/fstab"
-rm /tmp/hfs-boot-block
+rm -f /tmp/hfs-boot-block
 rm -rf "$1/ppc"

release/scripts/make-manifest.sh

@@ -9,18 +9,64 @@
 #
 # $FreeBSD$
 
-desc_base="Base system (MANDATORY)"
+base="Base system"
-desc_kernel="Kernel (MANDATORY)"
+doc="Additional Documentation"
-desc_doc="Additional documentation"
+kernel="Kernel"
-doc_default=off
+ports="Ports tree"
-desc_lib32="32-bit compatibility libraries"
+src="System source tree"
-desc_ports="Ports tree"
+lib32="32-bit compatibility libraries"
-desc_src="System source code"
+tests="Test suite"
-desc_tests="Test suite"
-src_default=off
-tests_default=off
 
-for i in $*; do
+desc_base="${base} (MANDATORY)"
-	echo "`basename $i`	`sha256 -q $i`	`tar tvf $i | wc -l | tr -d ' '`	`basename $i .txz`	\"`eval echo \\\$desc_$(basename $i .txz)`\"	`eval echo \\\${$(basename $i .txz)_default:-on}`"
+desc_base_dbg="${base} (Debugging)"
+desc_doc="${doc}"
+desc_kernel="${kernel} (MANDATORY)"
+desc_kernel_dbg="${kernel} (Debugging)"
+desc_kernel_alt="Alternate ${kernel}"
+desc_kernel_alt_dbg="Alternate ${kernel} (Debugging)"
+desc_lib32="${lib32}"
+desc_lib32_dbg="${lib32} (Debugging)"
+desc_ports="${ports}"
+desc_src="${src}"
+desc_tests="${tests}"
+
+default_doc=off
+default_src=off
+default_tests=off
+default_base_dbg=off
+default_lib32_dbg=off
+default_kernel_alt=off
+default_kernel_dbg=on
+default_kernel_alt_dbg=off
+
+for i in ${*}; do
+	dist="${i}"
+	distname="${i%%.txz}"
+	distname="$(echo ${distname} | tr '-' '_')"
+	distname="$(echo ${distname} | tr 'kernel.' 'kernel_')"
+	hash="$(sha256 -q ${i})"
+	nfiles="$(tar tvf ${i} | wc -l | tr -d ' ')"
+	default="$(eval echo \${default_${distname}:-on})"
+	desc="$(eval echo \"\${desc_${distname}}\")"
+
+	case ${i} in
+		kernel-dbg.txz)
+			desc="${desc_kernel_dbg}"
+			;;
+		kernel.*-dbg.txz)
+			desc="$(eval echo \"${desc_kernel_alt_dbg}\")"
+			desc="${desc}: $(eval echo ${i%%-dbg.txz} | cut -f 2 -d '.')"
+			default="$(eval echo \"${default_kernel_alt_dbg}\")"
+			;;
+		kernel.*.txz)
+			desc="$(eval echo \"${desc_kernel_alt}\")"
+			desc="${desc}: $(eval echo ${i%%.txz} | cut -f 2 -d '.')"
+			default="$(eval echo \"${default_kernel_alt}\")"
+			;;
+		*)
+			;;
+	esac
+
+	printf "${dist}\t${hash}\t${nfiles}\t${distname}\t\"${desc}\"\t${default}\n"
 done
 

release/sparc64/mkisoimages.sh

@@ -38,7 +38,7 @@ BASEBITSDIR="$1"
 publisher="The FreeBSD Project.  http://www.FreeBSD.org/"
 echo "/dev/iso9660/$LABEL / cd9660 ro 0 0" > "$BASEBITSDIR/etc/fstab"
 makefs -t cd9660 -o rockridge -o label="$LABEL" -o publisher="$publisher" "$NAME.tmp" "$@"
-rm "$BASEBITSDIR/etc/fstab"
+rm -f "$BASEBITSDIR/etc/fstab"
 
 if [ "x$BOPT" != "x-b" ]; then
 	mv "$NAME.tmp" "$NAME"

usr.sbin/bsdinstall/scripts/auto

@@ -115,7 +115,8 @@ bsdinstall hostname || error "Set hostname failed"
 
 export DISTRIBUTIONS="base.txz kernel.txz"
 if [ -f $BSDINSTALL_DISTDIR/MANIFEST ]; then
-	DISTMENU=`awk -F'\t' '!/^(kernel|base)/{print $4,$5,$6}' $BSDINSTALL_DISTDIR/MANIFEST`
+	DISTMENU=`awk -F'\t' '!/^(kernel\.txz|base\.txz)/{print $1,$5,$6}' $BSDINSTALL_DISTDIR/MANIFEST`
+	DISTMENU="$(echo ${DISTMENU} | sed -E 's/\.txz//g')"
 
 	exec 3>&1
 	EXTRA_DISTS=$( eval dialog \
@@ -129,16 +130,20 @@ if [ -f $BSDINSTALL_DISTDIR/MANIFEST ]; then
 	done
 fi
 
+LOCAL_DISTRIBUTIONS="MANIFEST"
 FETCH_DISTRIBUTIONS=""
 for dist in $DISTRIBUTIONS; do
 	if [ ! -f $BSDINSTALL_DISTDIR/$dist ]; then
 		FETCH_DISTRIBUTIONS="$FETCH_DISTRIBUTIONS $dist"
+	else
+		LOCAL_DISTRIBUTIONS="$LOCAL_DISTRIBUTIONS $dist"
 	fi
 done
+LOCAL_DISTRIBUTIONS=`echo $LOCAL_DISTRIBUTIONS`	# Trim white space
 FETCH_DISTRIBUTIONS=`echo $FETCH_DISTRIBUTIONS`	# Trim white space
 
 if [ -n "$FETCH_DISTRIBUTIONS" -a -n "$BSDINSTALL_CONFIGCURRENT" ]; then
-	dialog --backtitle "FreeBSD Installer" --title "Network Installation" --msgbox "No installation files were found on the boot volume. The next few screens will allow you to configure networking so that they can be downloaded from the Internet." 0 0
+	dialog --backtitle "FreeBSD Installer" --title "Network Installation" --msgbox "Some installation files were not found on the boot volume. The next few screens will allow you to configure networking so that they can be downloaded from the Internet." 0 0
 	bsdinstall netconfig || error
 	NETCONFIG_DONE=yes
 fi
@@ -299,6 +304,7 @@ esac
 
 if [ ! -z "$FETCH_DISTRIBUTIONS" ]; then
 	ALL_DISTRIBUTIONS="$DISTRIBUTIONS"
+	WANT_DEBUG=
 
 	# Download to a directory in the new system as scratch space
 	BSDINSTALL_FETCHDEST="$BSDINSTALL_CHROOT/usr/freebsd-dist"
@@ -310,15 +316,65 @@ if [ ! -z "$FETCH_DISTRIBUTIONS" ]; then
 		DISTDIR_IS_UNIONFS=1
 		mount_nullfs -o union "$BSDINSTALL_FETCHDEST" "$BSDINSTALL_DISTDIR"
 	else
-		export DISTRIBUTIONS="$ALL_DISTRIBUTIONS"
+		export DISTRIBUTIONS="$FETCH_DISTRIBUTIONS"
 		export BSDINSTALL_DISTDIR="$BSDINSTALL_FETCHDEST"
 	fi
 		
 	export FTP_PASSIVE_MODE=YES
-	bsdinstall distfetch || error "Failed to fetch distribution"
+	# Iterate through the distribution list and set a flag if debugging
+	# distributions have been selected.
+	for _DISTRIBUTION in $DISTRIBUTIONS; do
+		case $_DISTRIBUTION in
+			*-dbg.*)
+				[ -e $BSDINSTALL_DISTDIR/$_DISTRIBUTION ] \
+					&& continue
+				WANT_DEBUG=1
+				DEBUG_LIST="\n$DEBUG_LIST\n$_DISTRIBUTION"
+				;;
+			*)
+				;;
+		esac
+	done
+
+	# Fetch the distributions.
+	bsdinstall distfetch
+	rc=$?
+
+	if [ $rc -ne 0 ]; then
+		# If unable to fetch the remote distributions, recommend
+		# deselecting the debugging distributions, and retrying the
+		# installation, since failure to fetch *-dbg.txz should not
+		# be considered a fatal installation error.
+		msg="Failed to fetch remote distribution"
+		if [ ! -z "$WANT_DEBUG" ]; then
+			# Trim leading and trailing newlines.
+			DEBUG_LIST="${DEBUG_LIST%%\n}"
+			DEBUG_LIST="${DEBUG_LIST##\n}"
+			msg="$msg\n\nPlease deselect the following distributions"
+			msg="$msg and retry the installation:"
+			msg="$msg\n$DEBUG_LIST"
+		fi
+		error "$msg"
+	fi
 	export DISTRIBUTIONS="$ALL_DISTRIBUTIONS"
 fi
 
+if [ ! -z "$LOCAL_DISTRIBUTIONS" ]; then
+	# Download to a directory in the new system as scratch space
+	BSDINSTALL_FETCHDEST="$BSDINSTALL_CHROOT/usr/freebsd-dist"
+	mkdir -p "$BSDINSTALL_FETCHDEST" || error "Could not create directory $BSDINSTALL_FETCHDEST"
+	# Try to use any existing distfiles
+	if [ -d $BSDINSTALL_DISTDIR ]; then
+		DISTDIR_IS_UNIONFS=1
+		mount_nullfs -o union "$BSDINSTALL_FETCHDEST" "$BSDINSTALL_DISTDIR"
+		export BSDINSTALL_DISTDIR="$BSDINSTALL_FETCHDEST"
+	fi
+	env DISTRIBUTIONS="$LOCAL_DISTRIBUTIONS" \
+		BSDINSTALL_DISTSITE="file:///usr/freebsd-dist" \
+		bsdinstall distfetch || \
+		error "Failed to fetch distribution from local media"
+fi
+
 bsdinstall checksum || error "Distribution checksum failed"
 bsdinstall distextract || error "Distribution extract failed"
 bsdinstall rootpass || error "Could not set root password"