MFC r284893, r284895-r284897, r284942, r284968, r284996, r285005:

r284893 (brd):
  Add initial support for building Vagrant images for VMWare.
  Next steps will be adding Virtualbox support and uploading
  to Hashicorp Atlas for others to consume.

 r284895:
  Add default VAGRANT_IMG variable.

 r284896:
  Remove _ACCOUNT and add _USERNAME, _NAME, _VERSION for the
  VAGRANT_${VAR} variables extracted from VAGRANT_UPLOAD_CONF.

  Set ATLAS_${VAR} to VAGRANT_${VAR} if VAGRANT_UPLOAD_CONF
  is set.  There is intent to intentionally have separate
  variants of configuration entries, but the defaults do not
  yet have any reason to be different.

 r284897:
  Instead of hard-coding the PROVIDERS for upload, add the
  VAGRANT_PROVIDERS variable.  Right now, it defaults to only
  vmware_desktop, virtualbox support is to follow at some point.

  While here, fix the hashicorp URL: s/vagrant/atlas/, which
  was result of a sed(1) replace (and my fault).

 r284942 (brd):
  Add Support for uploading Vagrant images to Hashicorp Atlas.

 r284968:
  Default the VAGRANT_VERSION to ${REVISION}-${BRANCH} if
  not set, which expands to '11.0-CURRENT', for example.

  If the branch is -CURRENT, -STABLE, or -PRERELEASE, suffix
  the VAGRANT_VERSION with the snapshot date.

 r284996:
  Fix the gcloud port/package name.

 r285005:
  Remove the HH-MM suffix from the build date suffix.

  It was useful when working out several kinks when testing
  automated image uploading when retrying was necessary, but
  now it is making things much too messy.

Sponsored by:	The FreeBSD Foundation
This commit is contained in:
gjb 2015-07-02 16:17:05 +00:00
parent 5daf218e81
commit 6c9b1bee2d
7 changed files with 342 additions and 7 deletions

View File

@ -17,7 +17,7 @@ AZURE${VAR}!= grep -E ^AZURE${VAR} ${AZURE_UPLOAD_CONF} | awk -F' ' '{print $$2}
.endif
.if ${BRANCH} == "STABLE" || ${BRANCH} == "CURRENT" || ${BRANCH} == "PRERELEASE"
SNAPSHOT_DATE!= date +-%Y-%m-%d-%H-%M
SNAPSHOT_DATE!= date +-%Y-%m-%d
.endif
AZURE_TARGET:= ${OSRELEASE}${SNAPSHOT_DATE}.vhd

View File

@ -6,7 +6,7 @@
#
.if ${BRANCH} == "CURRENT" || ${BRANCH} == "STABLE" || ${BRANCH} == "PRERELEASE"
AMINAMESUFFIX!= date +-%Y-%m-%d-%H-%M
AMINAMESUFFIX!= date +-%Y-%m-%d
.endif
.if defined(EC2PUBLIC)
PUBLISH= --public

View File

@ -19,7 +19,7 @@ CLEANFILES+= ${GCE_UPLOAD_TGTS}
GCE_BUCKET?=
.if ${BRANCH} == "STABLE" || ${BRANCH} == "CURRENT" || ${BRANCH} == "PRERELEASE"
SNAPSHOT_DATE!= date +-%Y-%m-%d-%H-%M
SNAPSHOT_DATE!= date +-%Y-%m-%d
.endif
# Really? Uppercase characters are not allowed? Sigh...
@ -36,13 +36,13 @@ gce-check-depends:
. endif
.endfor
.if !exists(/usr/local/bin/gcutil)
. if !exists(${PORTSDIR}/net/google-cloud-api/Makefile)
. if !exists(${PORTSDIR}/net/google-cloud-sdk/Makefile)
. if !exists(/usr/local/sbin/pkg-static)
env ASSUME_ALWAYS_YES=yes pkg bootstrap -yf
. endif
env ASSUME_ALWAYS_YES=yes pkg install -y net/google-cloud-api
env ASSUME_ALWAYS_YES=yes pkg install -y net/google-cloud-sdk
. else
make -C ${PORTSDIR}/net/google-cloud-api BATCH=1 all install clean
make -C ${PORTSDIR}/net/google-cloud-sdk BATCH=1 all install clean
. endif
.endif

94
release/Makefile.vagrant Normal file
View File

@ -0,0 +1,94 @@
#
# $FreeBSD$
#
#
# Makefile for uploading Vagrant boxes to Hashicorp Atlas
#
VAGRANT_IMG?= ${.OBJDIR}/vagrant.vmdk
VAGRANT_UPLOAD_TGTS= vagrant-check-depends \
atlas-do-upload
CLEANFILES+= ${VAGRANT_UPLOAD_TGTS}
.if defined(VAGRANT_UPLOAD_CONF) && !empty(VAGRANT_UPLOAD_CONF)
. for VAR in _KEY _USERNAME
VAGRANT${VAR}!= grep -E ^VAGRANT${VAR} ${VAGRANT_UPLOAD_CONF} | awk -F' ' '{print $$2}'
ATLAS${VAR}:= ${VAGRANT${VAR}}
. endfor
.endif
.if ${BRANCH} == "STABLE" || ${BRANCH} == "CURRENT" || ${BRANCH} == "PRERELEASE"
SNAPSHOT_DATE!= date +-%Y-%m-%d
.endif
VAGRANT_VERSION?= ${REVISION}-${BRANCH}${SNAPSHOT_DATE}
VAGRANT_TARGET:= ${OSRELEASE}${SNAPSHOT_DATE}.box
VAGRANT_PROVIDERS?= vmware_desktop
#VAGRANT_PROVIDERS+= virtualbox
vagrant-upload: ${VAGRANT_UPLOAD_TGTS}
vagrant-check-depends:
.for VAR in _KEY _USERNAME _VERSION
. if !defined(VAGRANT${VAR}) || empty(VAGRANT${VAR})
@echo "Variable VAGRANT${VAR} cannot be empty."
@false
. endif
.endfor
.if !exists(/usr/local/bin/curl)
. if !exists(${PORTSDIR}/ftp/curl/Makefile)
. if !exists(/usr/local/sbin/pkg-static)
env ASSUME_ALWAYS_YES=yes pkg bootstrap -yf
. endif
env ASSUME_ALWAYS_YES=yes pkg install -y curl
. else
make -C ${PORTSDIR}/ftp/curl BATCH=1 all install clean
. endif
.endif
vagrant-do-package: cw-vagrant
vagrant-do-package-vmware: vagrant-create-vmware-vmx vagrant-do-package
@cd ${.OBJDIR} && echo '{"provider":"vmware_desktop"}' > metadata.json
cd ${.OBJDIR} && tar -czf ${VAGRANT_TARGET} metadata.json vagrant.vmx vagrant.vmdk
touch ${.OBJDIR}/${.TARGET}
atlas-do-upload: vagrant-do-package-vmware
.for PROVIDER in ${VAGRANT_PROVIDERS}
${.CURDIR}/scripts/atlas-upload.sh -b FreeBSD-${REVISION}-${BRANCH} -f ${VAGRANT_TARGET} -p ${PROVIDER} -k ${VAGRANT_KEY} -u ${VAGRANT_USERNAME} -v ${VAGRANT_VERSION}
.endfor
touch ${.OBJDIR}/${.TARGET}
vagrant-create-vmware-vmx:
@cd ${.OBJDIR} && echo '.encoding = "UTF-8"' > vagrant.vmx
@cd ${.OBJDIR} && echo 'bios.bootorder = "hdd,CDROM"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'checkpoint.vmstate = ""' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'cleanshutdown = "TRUE"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'config.version = "8"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'displayname = "${VAGRANT_TARGET}"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'ethernet0.addresstype = "generated"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'ethernet0.bsdname = "en0"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'ethernet0.connectiontype = "nat"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'ethernet0.displayname = "Ethernet"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'ethernet0.linkstatepropagation.enable = "FALSE"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'ethernet0.pcislotnumber = "33"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'ethernet0.present = "TRUE"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'ethernet0.virtualdev = "e1000"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'ethernet0.wakeonpcktrcv = "FALSE"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'floppy0.present = "FALSE"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'guestos = "freebsd-64"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'gui.fullscreenatpoweron = "FALSE"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'gui.viewmodeatpoweron = "windowed"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'memsize = "512"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'sound.startconnected = "FALSE"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'softpoweroff = "TRUE"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'scsi0.pcislotnumber = "16"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'scsi0.present = "TRUE"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'scsi0.virtualdev = "lsilogic"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'scsi0:0.filename = "vagrant.vmdk"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'scsi0:0.present = "TRUE"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'tools.synctime = "TRUE"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'usb.present = "FALSE"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'virtualhw.productcompatibility = "hosted"' >> vagrant.vmx
@cd ${.OBJDIR} && echo 'virtualhw.version = "9"' >> vagrant.vmx

View File

@ -18,7 +18,8 @@ RAW_DESC= Unformatted raw disk image
CLOUDWARE?= AZURE \
EC2 \
GCE \
OPENSTACK
OPENSTACK \
VAGRANT
AZURE_FORMAT= vhdf
AZURE_DESC= Microsoft Azure platform image
AZURE_DISK= ${OSRELEASE}.${AZURE_FORMAT}
@ -31,6 +32,9 @@ GCE_DISK= disk.${GCE_FORMAT}
OPENSTACK_FORMAT=qcow2
OPENSTACK_DESC= OpenStack platform image
OPENSTACK_DISK= ${OSRELEASE}.${OPENSTACK_FORMAT}
VAGRANT_FORMAT= vmdk
VAGRANT_DESC= Vagrant Image
VAGRANT_DISK= ${OSRELEASE}.${VAGRANT_FORMAT}
.if defined(WITH_CLOUDWARE) && !empty(WITH_CLOUDWARE) && !empty(CLOUDWARE)
. for _CW in ${CLOUDWARE}
@ -156,3 +160,4 @@ cloudware-install:
.include "${.CURDIR}/Makefile.ec2"
.include "${.CURDIR}/Makefile.azure"
.include "${.CURDIR}/Makefile.gce"
.include "${.CURDIR}/Makefile.vagrant"

155
release/scripts/atlas-upload.sh Executable file
View File

@ -0,0 +1,155 @@
#!/bin/sh
#-
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
# Upload a Vagrant image to Hashicorp's Atlas service
#
# $FreeBSD$
#
ATLAS_API_URL=''
ATLAS_UPLOAD_URL='https://binstore.hashicorp.com'
VERSION_DESCRIPTION="FreeBSD Snapshot Build"
usage() {
echo "${0} usage:"
echo "-b box-name -f box-to-upload -k api-key -p provider -u user -v version"
return 1
}
main () {
while getopts "b:f:k:p:u:v:" arg; do
case "${arg}" in
b)
BOX="${OPTARG}"
;;
f)
FILE="${OPTARG}"
;;
k)
KEY="${OPTARG}"
;;
p)
PROVIDER="${OPTARG}"
;;
u)
USERNAME="${OPTARG}"
;;
v)
VERSION="${OPTARG}"
;;
*)
;;
esac
done
if [ -z "${BOX}" -o \
-z "${FILE}" -o \
-z "${KEY}" -o \
-z "${PROVIDER}" -o \
-z "${USERNAME}" -o \
-z "${VERSION}" ];
then
usage || exit 0
fi
# Check to see if the box exists or create it
BOXRESULT=$(/usr/local/bin/curl -s "https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX}?access_token=${KEY}")
if [ $? != 0 ]; then
echo "Failed to connect to the API"
exit 2;
fi
echo $BOXRESULT | grep "\"name\":\"${BOX}\"" > /dev/null
if [ $? != 0 ]; then
echo "Creating box: ${BOX}"
/usr/local/bin/curl -s https://atlas.hashicorp.com/api/v1/boxes -X POST -d "box[name]=${BOX}" -d "access_token=${KEY}" > /dev/null
/usr/local/bin/curl -s https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX} -X PUT -d "box[is_private]=false" -d "access_token=${KEY}" > /dev/null
else
echo "Box already exists"
fi
# Check to see if the version exists or create it
VERSIONRESULT=$(/usr/local/bin/curl -s "https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX}/version/${VERSION}?access_token=${KEY}")
if [ $? != 0 ]; then
echo "Failed to connect to the API"
exit 2;
fi
echo $VERSIONRESULT | grep "\"version\":\"${VERSION}\"" > /dev/null
if [ $? != 0 ]; then
echo "Creating version: ${VERSION}"
/usr/local/bin/curl -s https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX}/versions -X POST -d "version[version]=${VERSION}" -d "access_token=${KEY}" > /dev/null
/usr/local/bin/curl -s https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX}/version/${VERSION} -X PUT -d "version[description]=${VERSION_DESCRIPTION}" -d "access_token=${KEY}" > /dev/null
VERSIONRESULT=$(/usr/local/bin/curl -s "https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX}/version/${VERSION}?access_token=${KEY}")
echo $VERSIONRESULT | grep "\"version\":\"${VERSION}\"" > /dev/null
if [ $? != 0 ]; then
echo "Failed to create version"
exit 2
fi
else
echo "Version already exists"
fi
# Check to see if the provider exists or create it
PROVIDERRESULT=$(/usr/local/bin/curl -s "https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX}/version/${VERSION}/provider/${PROVIDER}?access_token=${KEY}")
if [ $? != 0 ]; then
echo "Failed to connect to the API"
exit 2;
fi
echo $PROVIDERRESULT | grep "\"name\":\"${PROVIDER}\"" > /dev/null
if [ $? != 0 ]; then
echo "Creating provider: ${PROVIDER}"
/usr/local/bin/curl -s https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX}/version/${VERSION}/providers -X POST -d "provider[name]=${PROVIDER}" -d "access_token=${KEY}" > /dev/null
else
echo "Provider already exists"
fi
# Request an upload token
TOKENRESULT=$(/usr/local/bin/curl -s "https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX}/version/${VERSION}/provider/${PROVIDER}/upload?access_token=${KEY}")
if [ $? != 0 ]; then
echo "Failed to get the token from the API"
exit 2;
fi
echo ${TOKENRESULT} | grep "\"token\":" > /dev/null
if [ $? != 0 ]; then
echo "No token found from the API"
exit 2
else
TOKEN=$(echo $TOKENRESULT | sed -e 's/.*token":"//' -e 's/".*//')
echo "Uploading to Atlas"
UPLOADRESULT=$(/usr/local/bin/curl -s -X PUT --upload-file ${FILE} ${ATLAS_UPLOAD_URL}/${TOKEN})
# Validate the Upload
echo "Validating"
VALIDRESULT=$(/usr/local/bin/curl -s "https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX}/version/${VERSION}/provider/${PROVIDER}?access_token=${KEY}")
HOSTED_TOKEN=$(echo $VALIDRESULT | sed -e 's/.*hosted_token":"//' -e 's/".*//')
if [ ! -z ${HOSTED_TOKEN} -a ! -z ${TOKEN} -a ${HOSTED_TOKEN} != ${TOKEN} ]; then
echo "Upload failed, try again."
exit 2
fi
# Release the version
echo "Releasing ${VERSION} of ${BOX} in Atlas"
/usr/local/bin/curl -s https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX}/version/${VERSION}/release -X PUT -d "access_token=${KEY}" > /dev/null
fi
}
main "$@"

View File

@ -0,0 +1,81 @@
#!/bin/sh
#
# $FreeBSD$
#
# Packages to install into the image we're creating. This is a deliberately
# minimalist set, providing only the packages necessary to bootstrap.
export VM_EXTRA_PACKAGES="firstboot-freebsd-update firstboot-pkgs"
# Set to a list of third-party software to enable in rc.conf(5).
export VM_RC_LIST="firstboot_freebsd_update firstboot_pkgs"
vm_extra_pre_umount() {
# The firstboot_pkgs rc.d script will download the repository
# catalogue and install or update pkg when the instance first
# launches, so these files would just be replaced anyway; removing
# them from the image allows it to boot faster.
env ASSUME_ALWAYS_YES=yes pkg -c ${DESTDIR} delete -f -y pkg
rm ${DESTDIR}/var/db/pkg/repo-*.sqlite
# The size of the EC2 root disk can be configured at instance launch
# time; expand our filesystem to fill the disk.
echo 'growfs_enable="YES"' >> ${DESTDIR}/etc/rc.conf
# Vagrant instances use DHCP to get their network configuration.
echo 'ifconfig_DEFAULT="SYNCDHCP"' >> ${DESTDIR}/etc/rc.conf
# Enable sshd by default
echo 'sshd_enable="YES"' >> ${DESTDIR}/etc/rc.conf
# Disable DNS lookups by default to make SSH connect quickly
echo 'UseDNS no' >> ${DESTDIR}/etc/ssh/sshd_config
# Disable sendmail
echo 'sendmail_enable="NO"' >> ${DESTDIR}/etc/rc.conf
echo 'sendmail_submit_enable="NO"' >> ${DESTDIR}/etc/rc.conf
echo 'sendmail_outbound_enable="NO"' >> ${DESTDIR}/etc/rc.conf
echo 'sendmail_msp_queue_enable="NO"' >> ${DESTDIR}/etc/rc.conf
# sudo is required
echo 'firstboot_pkgs_list="sudo rsync"' >> ${DESTDIR}/etc/rc.conf
# Create the vagrant user with a password of vagrant
/usr/sbin/pw -R ${DESTDIR} \
groupadd vagrant -g 1001
chroot ${DESTDIR} mkdir -p /home/vagrant
/usr/sbin/pw -R ${DESTDIR} \
useradd vagrant \
-m -M 0755 -w yes -n vagrant -u 1001 -g 1001 -G 0 \
-c 'Vagrant User' -d '/home/vagrant' -s '/bin/csh'
# Change root's password to vagrant
echo 'vagrant' | /usr/sbin/pw -R ${DESTDIR} \
usermod root -h 0
# Configure sudo to allow the vagrant user
echo 'vagrant ALL=(ALL) NOPASSWD: ALL' >> ${DESTDIR}/usr/local/etc/sudoers
# Configure the vagrant ssh key
mkdir ${DESTDIR}/home/vagrant/.ssh
chmod 700 ${DESTDIR}/home/vagrant/.ssh
echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key" > ${DESTDIR}/home/vagrant/.ssh/authorized_keys
chown -R 1001 ${DESTDIR}/home/vagrant/.ssh
chmod 600 ${DESTDIR}/home/vagrant/.ssh/authorized_keys
# Reboot quickly, Don't wait at the panic screen
echo 'debug.trace_on_panic=1' >> ${DESTDIR}/etc/sysctl.conf
echo 'debug.debugger_on_panic=0' >> ${DESTDIR}/etc/sysctl.conf
echo 'kern.panic_reboot_wait_time=0' >> ${DESTDIR}/etc/sysctl.conf
# The console is not interactive, so we might as well boot quickly.
echo 'autoboot_delay="-1"' >> ${DESTDIR}/boot/loader.conf
# The first time the VM boots, the installed "first boot" scripts
# should be allowed to run:
# * growfs (expand the filesystem to fill the provided disk)
# * firstboot_freebsd_update (install critical updates)
# * firstboot_pkgs (install packages)
touch ${DESTDIR}/firstboot
return 0
}