Various updates to support new pc-sysinstall directive

"installPackages" that will install packages and all package
dependencies.

PR:		148606
Submitted by:	John Hixon

usr.sbin/pc-sysinstall/backend-query/Makefile

@@ -3,8 +3,8 @@
 FILES=	detect-laptop.sh detect-nics.sh detect-emulation.sh disk-info.sh \
 	disk-list.sh disk-part.sh enable-net.sh get-packages.sh list-config.sh \
 	list-components.sh list-mirrors.sh list-packages.sh list-rsync-backups.sh \
-	list-tzones.sh query-langs.sh send-logs.sh setup-ssh-keys.sh sys-mem.sh \
-	test-live.sh test-netup.sh update-part-list.sh xkeyboard-layouts.sh \
+	list-tzones.sh query-langs.sh send-logs.sh set-mirror.sh setup-ssh-keys.sh \
+	sys-mem.sh test-live.sh test-netup.sh update-part-list.sh xkeyboard-layouts.sh \
 	xkeyboard-models.sh xkeyboard-variants.sh
 FILESMODE=	${BINMODE}
 FILESDIR=${SHAREDIR}/pc-sysinstall/backend-query

usr.sbin/pc-sysinstall/backend-query/get-packages.sh

@@ -31,24 +31,16 @@
 . ${PROGDIR}/backend/functions.sh
 . ${PROGDIR}/backend/functions-packages.sh
 
-DEFAULT_FTP_SERVER="ftp.freebsd.org"
-FTP_SERVER="${1}"
 ID=`id -u`
-
 if [ "${ID}" -ne "0" ]
 then
 	echo "Error: must be root!" 
 	exit 1
 fi
 
-if [ -z "${FTP_SERVER}" ]
-then
-	FTP_SERVER="${DEFAULT_FTP_SERVER}"
-fi
-
 if [ ! -f "${PKGDIR}/INDEX" ]
 then
-	get_package_index "${FTP_SERVER}"
+	get_package_index
 fi
 
 if [ -f "${PKGDIR}/INDEX" ]

usr.sbin/pc-sysinstall/backend-query/set-mirror.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+#-
+# Copyright (c) 2010 iXSystems, Inc.  All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+
+. ${PROGDIR}/backend/functions.sh
+. ${PROGDIR}/backend/functions-ftp.sh
+
+MIRROR="${1}"
+
+if [ -z "${MIRROR}" ]
+then
+	echo "Error: No mirror specified!"
+	exit 1
+fi
+
+set_ftp_mirror "${MIRROR}"
+exit 0

usr.sbin/pc-sysinstall/backend/Makefile

@@ -2,7 +2,7 @@
 
 FILES=	functions-bsdlabel.sh functions-cleanup.sh functions-disk.sh \
 	functions-extractimage.sh functions-ftp.sh functions-installcomponents.sh \
-	functions-localize.sh functions-mountdisk.sh \
+	functions-installpackages.sh functions-localize.sh functions-mountdisk.sh \
 	functions-mountoptical.sh functions-networking.sh \
 	functions-newfs.sh functions-packages.sh functions-parse.sh \
 	functions-runcommands.sh functions-unmount.sh \

usr.sbin/pc-sysinstall/backend/functions-ftp.sh

@@ -30,6 +30,8 @@
 . ${BACKEND}/functions.sh
 . ${BACKEND}/functions-parse.sh
 
+DEFAULT_FTP_SERVER="ftp.freebsd.org"
+
 MAIN_FTP_SERVERS="\
 Main Site: ftp.freebsd.org"
 
@@ -274,7 +276,25 @@ show_mirrors()
     done
     IFS="${SAVE_IFS}"
   fi
-}
+};
+
+set_ftp_mirror()
+{
+	MIRROR="${1}"
+	echo "${MIRROR}" > "${CONFDIR}/mirrors.conf"
+};
+
+get_ftp_mirror()
+{
+	MIRROR="${DEFAULT_FTP_SERVER}"
+	if [ -f "${CONFDIR}/mirrors.conf" ]
+	then
+		MIRROR=`cat "${CONFDIR}/mirrors.conf"`
+	fi
+
+	VAL="${MIRROR}"
+	export VAL
+};
 
 get_ftp_mirrors()
 {
@@ -371,4 +391,4 @@ get_ftp_mirrors()
   fi
 
   export VAL
-}
+};

usr.sbin/pc-sysinstall/backend/functions-installpackages.sh

@@ -0,0 +1,118 @@
+#!/bin/sh
+#-
+# Copyright (c) 2010 iXsystems, Inc.  All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+
+# Functions which check and load any optional packages specified in the config
+
+. ${BACKEND}/functions.sh
+. ${BACKEND}/functions-parse.sh
+
+# Recursively determine all dependencies for this package
+determine_package_dependencies()
+{
+	local PKGNAME="${1}"
+	local DEPFILE="${2}"
+
+	grep "${PKGNAME}" "${DEPFILE}" >/dev/null
+	if [ "$?" -ne "0" ]
+	then
+		echo "${PKGNAME}" >> "${DEPFILE}"
+		get_package_dependencies "${PKGNAME}" "1"
+		local DEPS="${VAL}"
+
+		for d in ${DEPS}
+		do
+			get_all_package_dependencies "${d}" "${DEPFILE}"
+		done
+	fi
+};
+
+# Fetch packages dependencies from a file
+fetch_package_dependencies()
+{
+	local DEPFILE
+	local DEPS
+
+	DEPFILE="${1}"
+	DEPS=`cat "${DEPFILE}"`
+
+	for d in ${DEPS}
+	do
+		get_package_short_name "${d}"
+		SNAME="${VAL}"
+
+		get_package_category "${SNAME}"
+		CATEGORY="${VAL}"
+
+		fetch_package "${CATEGORY}" "${d}"
+	done
+};
+
+# Check for any packages specified, and begin loading them
+install_packages()
+{
+   # First, lets check and see if we even have any packages to install
+	get_value_from_cfg installPackages
+	if [ ! -z "${VAL}" ]
+	then
+	  mkdir -p "${PKGTMPDIR}"
+
+	  HERE=`pwd`
+	  cd "${PKGTMPDIR}"
+
+      # Lets start by cleaning up the string and getting it ready to parse
+      strip_white_space ${VAL}
+      PACKAGES=`echo ${VAL} | sed -e "s|,| |g"`
+      for i in $PACKAGES
+	  do
+		if get_package_name "${i}"
+		then
+			PKGNAME="${VAL}"
+			DEPFILE="${PKGTMPDIR}/.${PKGNAME}.deps"
+
+			touch "${DEPFILE}"
+			determine_package_dependencies "${PKGNAME}" "${DEPFILE}"
+			fetch_package_dependencies "${DEPFILE}"
+
+			# If the package is not already installed, install it!
+			if ! run_chroot_cmd "pkg_info -e ${PKGNAME}"
+			then
+				echo_log "Adding package ${PKGNAME}"
+				pkg_add -C "${FSMNT}" "${PKGNAME}.tbz" >/dev/null 2>&1
+				if [ "$?" -eq "0" ]
+				then
+					echo_log "${PKGNAME} successfully installed!"
+				fi
+			fi
+
+			rm "${DEPFILE}"
+		fi
+	  done
+
+	  #rm -rf "${PKGTMPDIR}"
+	  cd "${HERE}"
+	fi
+};

usr.sbin/pc-sysinstall/backend/functions-packages.sh

@@ -34,29 +34,30 @@
 
 get_package_index()
 {
-	FTP_SERVER="${1}"
-	FTP_DIR="ftp://${FTP_SERVER}/pub/FreeBSD/releases/${FBSD_ARCH}/${FBSD_BRANCH}/packages"
 	INDEX_FILE="INDEX"
 	USE_BZIP2=0
 
+	get_ftp_mirror
+	FTP_SERVER="${VAL}"
+
+	FTP_DIR="ftp://${FTP_SERVER}/pub/FreeBSD/releases/${FBSD_ARCH}/${FBSD_BRANCH}/packages"
+
 	if [ -f "/usr/bin/bzip2" ]
 	then
 		INDEX_FILE="${INDEX_FILE}.bz2"
 		USE_BZIP2=1
 	fi
 
-	ftp "${FTP_DIR}/${INDEX_FILE}"
-	if [ -f "${INDEX_FILE}" ]
-	then
-		if [ "${USE_BZIP2}" -eq  "1" ]
-		then
-			bzip2 -d "${INDEX_FILE}"
-			INDEX_FILE="${INDEX_FILE%.bz2}"
-		fi
+	fetch_file "${FTP_DIR}/${INDEX_FILE}" "${PKGDIR}/${INDEX_FILE}" "1"
 
-		mv "${INDEX_FILE}" "${PKGDIR}"
+	HERE=`pwd`
+	cd "${PKGDIR}"
+	if [ -f "${INDEX_FILE}" ] && [ "${USE_BZIP2}" -eq "1" ]
+	then
+		bzip2 -d "${INDEX_FILE}"
 	fi
-}
+	cd "${HERE}"
+};
 
 parse_package_index()
 {
@@ -67,9 +68,11 @@ parse_package_index()
 
 	while read -r line
 	do
+		PKGNAME=""
 		CATEGORY=""
 		PACKAGE=""
 		DESC=""
+		DEPS=""
 		i=0
 
 		SAVE_IFS="${IFS}"
@@ -77,7 +80,11 @@ parse_package_index()
 
 		for part in ${line}
 		do
-			if [ "${i}" -eq "1" ]
+			if [ "${i}" -eq "0" ]
+			then
+				PKGNAME="${part}"
+
+			elif [ "${i}" -eq "1" ]
 			then
 				PACKAGE=`basename "${part}"`
 
@@ -88,17 +95,23 @@ parse_package_index()
 			elif [ "${i}" -eq "6" ]
 			then
 				CATEGORY=`echo "${part}" | cut -f1 -d' '`
+
+			elif [ "${i}" -eq "8" ]
+			then
+				DEPS="${part}"
 			fi
 
 			i=$((i+1))
 		done
 
 		echo "${CATEGORY}|${PACKAGE}|${DESC}" >> "${INDEX_FILE}.parsed"
+		echo "${PACKAGE}|${PKGNAME}|${DEPS}" >> "${INDEX_FILE}.deps"
+
 		IFS="${SAVE_IFS}"
 	done
 
 	exec 0<&3
-}
+};
 
 show_package_file()
 {
@@ -117,7 +130,7 @@ show_package_file()
 	done
 
 	exec 0<&3
-}
+};
 
 show_packages_by_category()
 {
@@ -128,7 +141,7 @@ show_packages_by_category()
 	grep "^${CATEGORY}|" "${INDEX_FILE}" > "${TMPFILE}"
 	show_package_file "${TMPFILE}"
 	rm "${TMPFILE}"
-}
+};
 
 show_package_by_name()
 {
@@ -140,9 +153,118 @@ show_package_by_name()
 	grep "^${CATEGORY}|${PACKAGE}" "${INDEX_FILE}" > "${TMPFILE}"
 	show_package_file "${TMPFILE}"
 	rm "${TMPFILE}"
-}
+};
 
 show_packages()
 {
 	show_package_file "${PKGDIR}/INDEX.parsed"
-}
+};
+
+get_package_dependencies()
+{
+	PACKAGE="${1}"
+	LONG="${2:-0}"
+	RES=0
+
+	INDEX_FILE="${PKGDIR}/INDEX.deps"
+	REGEX="^${PACKAGE}|"
+
+	if [ "${LONG}" -ne "0" ]
+	then
+		REGEX="^.*|${PACKAGE}|"
+	fi
+
+	LINE=`grep "${REGEX}" "${INDEX_FILE}" 2>/dev/null`
+	DEPS=`echo "${LINE}"|cut -f3 -d'|'`
+
+	VAL="${DEPS}"
+	export VAL
+
+	if [ -z "${VAL}" ]
+	then
+		RES=1
+	fi
+
+	return ${RES}
+};
+
+get_package_name()
+{
+	PACKAGE="${1}"
+	RES=0
+
+	INDEX_FILE="${PKGDIR}/INDEX.deps"
+	REGEX="^${PACKAGE}|"
+	
+	LINE=`grep "${REGEX}" "${INDEX_FILE}" 2>/dev/null`
+	NAME=`echo "${LINE}"|cut -f2 -d'|'`
+
+	VAL="${NAME}"
+	export VAL
+
+	if [ -z "${VAL}" ]
+	then
+		RES=1
+	fi
+
+	return ${RES}
+};
+
+get_package_short_name()
+{
+	PACKAGE="${1}"
+	RES=0
+
+	INDEX_FILE="${PKGDIR}/INDEX.deps"
+	REGEX="^.*|${PACKAGE}|"
+	
+	LINE=`grep "${REGEX}" "${INDEX_FILE}" 2>/dev/null`
+	NAME=`echo "${LINE}"|cut -f1 -d'|'`
+
+	VAL="${NAME}"
+	export VAL
+
+	if [ -z "${VAL}" ]
+	then
+		RES=1
+	fi
+
+	return ${RES}
+};
+
+get_package_category()
+{
+	PACKAGE="${1}"
+	INDEX_FILE="${PKGDIR}/INDEX.parsed"
+	RES=0
+
+	LINE=`grep "|${PACKAGE}|" "${INDEX_FILE}" 2>/dev/null`
+	NAME=`echo "${LINE}"|cut -f1 -d'|'`
+
+	VAL="${NAME}"
+	export VAL
+
+	if [ -z "${VAL}" ]
+	then
+		RES=1
+	fi
+
+	return ${RES}
+};
+
+fetch_package()
+{
+	CATEGORY="${1}"
+	PACKAGE="${2}"
+
+	get_ftp_mirror
+	FTP_SERVER="${VAL}"
+
+	PACKAGE="${PACKAGE}.tbz"
+	if [ ! -f "${PKGTMPDIR}/${PACKAGE}" ]
+	then
+		PKGPATH="${CATEGORY}/${PACKAGE}"
+		FTP_PATH="ftp://${FTP_SERVER}/pub/FreeBSD/releases/${FBSD_ARCH}/${FBSD_BRANCH}/packages/${PKGPATH}"
+		fetch_file "${FTP_PATH}" "${PKGTMPDIR}/" "0"
+	fi
+};

usr.sbin/pc-sysinstall/backend/functions-runcommands.sh

@@ -37,7 +37,10 @@ run_chroot_cmd()
   echo "$CMD" >${FSMNT}/.runcmd.sh
   chmod 755 ${FSMNT}/.runcmd.sh
   chroot ${FSMNT} sh /.runcmd.sh
+  RES=$?
+
   rm ${FSMNT}/.runcmd.sh
+  return ${RES}
 };
 
 run_chroot_script()
@@ -50,8 +53,10 @@ run_chroot_script()
 
   echo_log "Running chroot script: ${SCRIPT}"
   chroot ${FSMNT} /.${SBASE}
+  RES=$?
 
   rm ${FSMNT}/.${SBASE}
+  return ${RES}
 };
 
 
@@ -64,7 +69,10 @@ run_ext_cmd()
   echo "${CMD}"> ${TMPDIR}/.runcmd.sh
   chmod 755 ${TMPDIR}/.runcmd.sh
   sh ${TMPDIR}/.runcmd.sh
+  RES=$?
+
   rm ${TMPDIR}/.runcmd.sh
+  return ${RES}
 };
 
 

usr.sbin/pc-sysinstall/backend/parseconfig.sh

@@ -35,12 +35,15 @@
 . ${BACKEND}/functions-disk.sh
 . ${BACKEND}/functions-extractimage.sh
 . ${BACKEND}/functions-installcomponents.sh
+. ${BACKEND}/functions-installpackages.sh
 . ${BACKEND}/functions-localize.sh
 . ${BACKEND}/functions-mountdisk.sh
 . ${BACKEND}/functions-networking.sh
 . ${BACKEND}/functions-newfs.sh
+. ${BACKEND}/functions-packages.sh
 . ${BACKEND}/functions-parse.sh
 . ${BACKEND}/functions-runcommands.sh
+. ${BACKEND}/functions-ftp.sh
 . ${BACKEND}/functions-unmount.sh
 . ${BACKEND}/functions-upgrade.sh
 . ${BACKEND}/functions-users.sh
@@ -117,6 +120,9 @@ then
   # Check if we have any optional modules to load 
   install_components
 
+  # Check if we have any packages to install
+  install_packages
+
   # Do any localization in configuration
   run_localize
   
@@ -158,6 +164,9 @@ else
   # Check if we have any optional modules to load 
   install_components
 
+  # Check if we have any packages to install
+  install_packages
+
   # All finished, unmount the file-systems
   unmount_upgrade
 

usr.sbin/pc-sysinstall/conf/pc-sysinstall.conf

@@ -51,6 +51,10 @@ export COMPFILEDIR
 COMPTMPDIR="/usr/.componenttmp"
 export COMPTMPDIR
 
+# set the package temp directory, which is relative to FSMNT
+PKGTMPDIR="/usr/.pkgtmp"
+export PKGTMPDIR
+
 # Variables to set the location of installation data
 UZIP_FILE="PCBSD.ufs.uzip"
 TAR_FILE="PCBSD.tbz"

usr.sbin/pc-sysinstall/doc/help-index

@@ -34,10 +34,10 @@ System Query Commands
     list-components
         Returns a listing of the available components which can be installed
 
-    list-mirrors
+    list-mirrors [country]
         Returns a listing of the available FTP mirrors
 
-    list-packages
+    list-packages [category] [package]
         Returns a listing of the available packages
 
     list-rsync-backups <user> <host> <port>
@@ -55,6 +55,9 @@ System Query Commands
     sys-mem
         Return the size of installed system RAM in MegaBytes
 
+    set-mirror <mirror>
+        Set FTP mirror
+
     test-netup
         Test if an internet connection is available
     

usr.sbin/pc-sysinstall/pc-sysinstall/pc-sysinstall.sh

@@ -43,8 +43,11 @@ export PROGDIR
 COMPDIR="${PROGDIR}/components"
 export COMPDIR
 
+CONFDIR="${PROGDIR}/conf"
+export CONFDIR
+
 # Set this to the packages location
-PKGDIR="${PROGDIR}/conf"
+PKGDIR="${CONFDIR}"
 export PKGDIR
 
 # End of user-editable configuration
@@ -184,6 +187,10 @@ case $1 in
   get-packages) ${QUERYDIR}/get-packages.sh "${2}"
   ;;
 
+  # Function to set FTP mirror
+  set-mirror) ${QUERYDIR}/set-mirror.sh "${2}"
+  ;;
+
   # Function which allows setting up of SSH keys
   setup-ssh-keys) ${QUERYDIR}/setup-ssh-keys.sh "${2}" "${3}" "${4}"
   ;;