Add a new package splitting mechanism to replace print-cdrom-packages.sh.
The new system tries to be more automated so that there is less work for the re's to do. It also no longer uses a /usr/ports tree as its input, but uses the generated package build including its INDEX file as its input. It parses the INDEX file, determines which packages should go on which ISO images, and then builds full-fledged trees of packages that can be added as an argument to mkisofs along with the tree built by 'make release' to build a full CD image. The INDEX files in the populated trees are generated with volume media number to make use of sysinstall's multiple volume support so that the user is kindly prompted to insert the appropriate disc for a package if it is not on the current disc. There is still some more tweaking to be done here, but this part needs to be committed. This stuff will all be used to prep the 5.4 release as well. Tested by: kensmith, others on re@ Reviewed by: re
This commit is contained in:
John Baldwin
parent
4b0eb6a716
commit
e738d1ea3d
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=143938
@ -299,7 +299,7 @@ BOOTABLE="-b"
|
||||
DOCREL= doc.1 doc.2
|
||||
.endif
|
||||
|
||||
.if !make(release) && !make(rerelease)
|
||||
.if !make(release) && !make(rerelease) && !make(package-split)
|
||||
BINMAKE!= cd ${.CURDIR}/..; ${MAKE} -V BINMAKE
|
||||
WMAKEENV!= cd ${.CURDIR}/..; \
|
||||
${BINMAKE} ${CROSSENV} -f Makefile.inc1 -V WMAKEENV
|
||||
@ -1214,4 +1214,40 @@ buildBootFloppy:
|
||||
${BOOTINODE} ${FLPLABEL}
|
||||
@echo "Created ${RD}/floppies/${FSIMAGE}.flp"
|
||||
|
||||
.if make(package-split)
|
||||
# Targets related to making a package split
|
||||
#
|
||||
# PKG_COPY instructs the script to copy the actual package files rather than
|
||||
# make hard links
|
||||
# PKG_TREE is the path to the package tree to be split
|
||||
# PKG_DEST is the path to the destination tree to create the split in
|
||||
# PKG_VERBOSE asks for verbose output of the layout process
|
||||
|
||||
.if defined(PKG_COPY)
|
||||
PKG_DO_COPY= cp
|
||||
.else
|
||||
PKG_DO_COPY= ln
|
||||
.endif
|
||||
PKG_INDEX= ${PKG_DEST}/INDEX.master
|
||||
PKG_ENV?=
|
||||
.if defined(TARGET_ARCH)
|
||||
PKG_ENV+= PKG_ARCH=${TARGET_ARCH}
|
||||
.endif
|
||||
.if defined(PKG_VERBOSE)
|
||||
PKG_ENV+= PKG_VERBOSE=1
|
||||
.endif
|
||||
|
||||
package-split:
|
||||
.if !defined(PKG_TREE)
|
||||
@echo "PKG_TREE must be defined" && exit 1
|
||||
.endif
|
||||
.if !defined(PKG_DEST)
|
||||
@echo "PKG_DEST must be defined" && exit 1
|
||||
.endif
|
||||
@env ${PKG_ENV} python ${.CURDIR}/scripts/package-split.py \
|
||||
${PKG_TREE}/INDEX ${PKG_INDEX}
|
||||
@env ${PKG_ENV} sh ${.CURDIR}/scripts/package-trees.sh ${PKG_DO_COPY} \
|
||||
${PKG_INDEX} ${PKG_TREE} ${PKG_DEST}
|
||||
.endif
|
||||
|
||||
.include <bsd.obj.mk>
|
||||
|
||||
195
release/scripts/package-split.py
Normal file
195
release/scripts/package-split.py
Normal file
@ -0,0 +1,195 @@
|
||||
#!/usr/local/bin/python
|
||||
#
|
||||
# This script generates a master INDEX file for the CD images built by the
|
||||
# FreeBSD release engineers. Each disc is given a list of desired packages.
|
||||
# Dependencies of these packages are placed on either the same disc or an
|
||||
# earlier disc. The resulting master INDEX file is then written out.
|
||||
#
|
||||
# Usage: package-split.py <INDEX> <master INDEX>
|
||||
#
|
||||
# $FreeBSD$
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
try:
|
||||
arch = os.environ["PKG_ARCH"]
|
||||
except:
|
||||
arch = os.uname()[4]
|
||||
print "Using arch %s" % (arch)
|
||||
|
||||
if 'PKG_VERBOSE' in os.environ:
|
||||
verbose = 1
|
||||
else:
|
||||
verbose = 0
|
||||
|
||||
# List of packages for disc1. This just includes packages sysinstall can
|
||||
# install as a distribution
|
||||
def disc1_packages():
|
||||
# 5.x only
|
||||
pkgs = ['lang/perl5.8']
|
||||
if arch != 'ia64':
|
||||
pkgs.append('x11/xorg')
|
||||
if arch == 'alpha':
|
||||
pkgs.append('emulators/osf1_base')
|
||||
elif arch == 'i386':
|
||||
pkgs.append('emulators/linux_base-8')
|
||||
# 5.x only
|
||||
if arch == 'i386':
|
||||
pkgs.append('misc/compat22')
|
||||
pkgs.append('misc/compat3x')
|
||||
pkgs.append('misc/compat4x')
|
||||
return pkgs
|
||||
|
||||
# List of packages for disc2. This includes packages that the X desktop
|
||||
# menu depends on (if it still exists) and other "nice to have" packages.
|
||||
# For architectures that use a separate livefs, this is actually disc3.
|
||||
def disc2_packages():
|
||||
# X Desktops
|
||||
pkgs = ['x11/kde3',
|
||||
'x11/gnome2',
|
||||
'x11-wm/afterstep',
|
||||
'x11-wm/windowmaker',
|
||||
'x11-wm/fvwm2',
|
||||
# "Nice to have"
|
||||
'archivers/unzip',
|
||||
'astro/xearth',
|
||||
'devel/gmake',
|
||||
'editors/emacs',
|
||||
'editors/vim',
|
||||
'editors/vim-lite',
|
||||
'editors/xemacs',
|
||||
'emulators/mtools',
|
||||
'graphics/png',
|
||||
'graphics/xv',
|
||||
'irc/xchat2',
|
||||
'mail/exim',
|
||||
'mail/fetchmail',
|
||||
'mail/mutt',
|
||||
'mail/pine4',
|
||||
'mail/popd',
|
||||
'mail/xfmail',
|
||||
'mail/postfix',
|
||||
'misc/bsdiff',
|
||||
'net/cvsup',
|
||||
'net/cvsup-without-gui',
|
||||
'net/rsync',
|
||||
'net/samba',
|
||||
'news/slrn',
|
||||
'news/tin',
|
||||
'print/a2ps-letter',
|
||||
'print/apsfilter',
|
||||
'print/ghostscript-gnu-nox11',
|
||||
'print/gv',
|
||||
'print/psutils-letter',
|
||||
'shells/bash2',
|
||||
'shells/pdksh',
|
||||
'shells/zsh',
|
||||
'security/sudo',
|
||||
'sysutils/portsnap',
|
||||
'sysutils/portupgrade',
|
||||
'www/links',
|
||||
'www/lynx',
|
||||
'x11/rxvt',
|
||||
# Formerly on disc3
|
||||
'lang/gnat',
|
||||
'lang/php4',
|
||||
'lang/php5',
|
||||
'print/teTeX',
|
||||
'security/portaudit',
|
||||
'textproc/docproj-jadetex',
|
||||
'www/apache13',
|
||||
'www/apache13-modssl',
|
||||
'www/apache2']
|
||||
if arch == 'i386':
|
||||
pkgs.append('comms/ltmdm')
|
||||
pkgs.append('print/acroread')
|
||||
pkgs.append('www/opera')
|
||||
return pkgs
|
||||
|
||||
# The list of desired packages
|
||||
def desired_packages():
|
||||
disc1 = disc1_packages()
|
||||
disc2 = disc2_packages()
|
||||
return [disc1, disc2]
|
||||
|
||||
# Suck the entire INDEX file into a two different dictionaries. The first
|
||||
# dictionary maps port names (origins) to package names. The second
|
||||
# dictionary maps a package name to a list of its dependent packages.
|
||||
PACKAGE_COL=0
|
||||
ORIGIN_COL=1
|
||||
DEPENDS_COL=8
|
||||
|
||||
def load_index(index):
|
||||
deps = {}
|
||||
pkgs = {}
|
||||
line_num = 1
|
||||
for line in index:
|
||||
fields = line.split('|')
|
||||
name = fields[PACKAGE_COL]
|
||||
if name in deps:
|
||||
sys.stderr.write('%d: Duplicate package %s\n' % (line_num, name))
|
||||
sys.exit(1)
|
||||
origin = fields[ORIGIN_COL].replace('/usr/ports/', '', 1)
|
||||
if origin in pkgs:
|
||||
sys.stderr.write('%d: Duplicate port %s\n' % (line_num, origin))
|
||||
sys.exit(1)
|
||||
deps[name] = fields[DEPENDS_COL].split()
|
||||
pkgs[origin] = name
|
||||
line_num = line_num + 1
|
||||
return (deps, pkgs)
|
||||
|
||||
# Layout the packages on the various CD images. Here's how it works. We walk
|
||||
# each disc in the list of discs. Within each disc we walk the list of ports.
|
||||
# For each port, we add the package name to a dictionary with the value being
|
||||
# the current disc number. We also add all of the dependent packages. If
|
||||
# a package is already in the dictionary when we go to add it, we just leave
|
||||
# the dictionary as it is. This means that each package ends up on the first
|
||||
# disc that either lists it or contains it as a dependency.
|
||||
def layout_discs(discs, pkgs, deps):
|
||||
disc_num = 1
|
||||
layout = {}
|
||||
for disc in discs:
|
||||
for port in disc:
|
||||
if port not in pkgs:
|
||||
sys.stderr.write('Disc %d: Unable to find package for %s\n' %
|
||||
(disc_num, port))
|
||||
continue
|
||||
pkg = pkgs[port]
|
||||
pkg_list = [pkg] + deps[pkg]
|
||||
for pkg in pkg_list:
|
||||
if pkg not in layout:
|
||||
if verbose:
|
||||
print "--> Adding %s to Disc %d" % (pkg, disc_num)
|
||||
layout[pkg] = disc_num
|
||||
disc_num = disc_num + 1
|
||||
return layout
|
||||
|
||||
# Generate a master INDEX file based on the generated layout. The way this
|
||||
# works is that for each INDEX line, we check to see if the package is in the
|
||||
# layout. If it is, we put that INDEX line into the master INDEX and append
|
||||
# a new field with the disc number to the line.
|
||||
def generate_index(index, layout, master_index):
|
||||
for line in index:
|
||||
pkg = line.split('|')[PACKAGE_COL]
|
||||
if pkg in layout:
|
||||
new_line = '%s|%d\n' % (line.splitlines()[0], layout[pkg])
|
||||
master_index.write(new_line)
|
||||
|
||||
# Verify the command line arguments
|
||||
if len(sys.argv) != 3:
|
||||
sys.stderr.write('Invalid number of arguments\n')
|
||||
sys.stderr.write('Usage: package-split.py <source INDEX> <master INDEX>\n')
|
||||
sys.exit(1)
|
||||
|
||||
print "Loading %s..." % (sys.argv[1])
|
||||
index = file(sys.argv[1])
|
||||
(deps, pkgs) = load_index(index)
|
||||
discs = desired_packages()
|
||||
layout = layout_discs(discs, pkgs, deps)
|
||||
index.seek(0)
|
||||
print "Generating %s..." % (sys.argv[2])
|
||||
master_index = file(sys.argv[2], 'w')
|
||||
generate_index(index, layout, master_index)
|
||||
index.close()
|
||||
master_index.close()
|
||||
58
release/scripts/package-trees.sh
Normal file
58
release/scripts/package-trees.sh
Normal file
@ -0,0 +1,58 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# This script generates the disk layout for the CD images built by the FreeBSD
|
||||
# release engineers as dictated by a specified master INDEX file. Each disc
|
||||
# contains the master INDEX, it's assigned list of packages, and the
|
||||
# appropriate tree of category symlinks.
|
||||
#
|
||||
# Usage: package-tress.sh <copy method> <INDEX> <package tree> <destination>
|
||||
#
|
||||
# $FreeBSD$
|
||||
|
||||
# Verify the command line
|
||||
if [ $# -ne 4 ]; then
|
||||
echo "Invalid number of arguments"
|
||||
echo "Usage: package-trees.sh <copy method> <INDEX> <tree> <destination>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
COPY=$1 ; shift
|
||||
INDEX=$1 ; shift
|
||||
TREE=$1 ; shift
|
||||
DESTDIR=$1 ; shift
|
||||
|
||||
# First, determine the highest disc number.
|
||||
high_disc=`cut -d '|' -f 14 ${INDEX} | sort -n | tail -1`
|
||||
echo "Generating trees for ${high_disc} discs"
|
||||
|
||||
# Second, initialize the trees for each disc
|
||||
for disc in `jot $high_disc`; do
|
||||
rm -rf ${DESTDIR}/disc${disc}
|
||||
mkdir -p ${DESTDIR}/disc${disc}/packages/All
|
||||
cp ${INDEX} ${DESTDIR}/disc${disc}/packages/INDEX
|
||||
done
|
||||
|
||||
# Third, run through the INDEX copying each package to its appropriate CD and
|
||||
# making the appropriate category symlinks
|
||||
while read line; do
|
||||
disc=`echo $line | cut -d '|' -f 14`
|
||||
package=`echo $line | cut -d '|' -f 1`
|
||||
categories=`echo $line | cut -d '|' -f 7`
|
||||
discdir=${DESTDIR}/disc${disc}
|
||||
if [ -n "$PKG_VERBOSE" ]; then
|
||||
echo "--> Copying $package to Disc $disc"
|
||||
fi
|
||||
${COPY} ${TREE}/All/${package}.tbz ${discdir}/packages/All
|
||||
for cat in ${categories}; do
|
||||
catdir=${discdir}/packages/${cat}
|
||||
mkdir -p ${catdir}
|
||||
ln -s ../All/${package}.tbz ${catdir}
|
||||
done
|
||||
done < ${INDEX}
|
||||
|
||||
# Fourth, output du info for the relative size of the trees.
|
||||
discs=""
|
||||
for disc in `jot $high_disc`; do
|
||||
discs="${discs} disc${disc}"
|
||||
done
|
||||
(cd ${DESTDIR}; du -sh ${discs})
|
||||
Loading…
Reference in New Issue
Block a user