pfSense

#!/bin/sh -T

#

# build.sh

#

# part of pfSense (https://www.pfsense.org)

# Copyright (c) 2004-2013 BSD Perimeter

# Copyright (c) 2013-2016 Electric Sheep Fencing

# Copyright (c) 2014-2021 Rubicon Communications, LLC (Netgate)

# All rights reserved.

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

set +e

usage() {

	echo "Usage $0 [options] [ iso | ova | memstick | memstickserial | memstickadi | all | none ]"

	echo "		all = memstick memstickserial memstickadi"

	echo "		none = upgrade only pkg repo"

	echo "	[ options ]: "

	echo "		--no-buildworld|-c - Will set NO_BUILDWORLD NO_BUILDKERNEL to not build kernel and world"

	echo "		--no-cleanobjdir|-d - Will not clean FreeBSD object built dir to allow restarting a build with NO_CLEAN"

	echo "		--resume-image-build|-r - Includes -c -d and also will just move directly to image creation using pre-staged data"

	echo "		--setup - Install required repo and ports builder require to work"

	echo "		--update-sources - Refetch FreeBSD sources"

	echo "		--rsync-repos - rsync pkg repos"

	echo "		--rsync-snapshots - rsync snapshots images and pkg repos"

	echo "		--clean-builder - clean all builder used data/resources"

	echo "		--build-kernels - build all configured kernels"

	echo "		--build-kernel argument - build specified kernel. Example --build-kernel KERNEL_NAME"

	echo "		--install-extra-kernels argument - Put extra kernel(s) under /kernel image directory. Example --install-extra-kernels KERNEL_NAME_WRAP"

	echo "		--snapshots - Build snapshots"

	echo "		--poudriere-snapshots - Update poudriere packages and send them to PKG_RSYNC_HOSTNAME"

	echo "		--setup-poudriere - Install poudriere and create necessary jails and ports tree"

	echo "		--create-unified-patch - Create a big patch with all changes done on FreeBSD"

	echo "		--update-poudriere-jails [-a ARCH_LIST] - Update poudriere jails using current patch versions"

	echo "		--update-poudriere-ports [-a ARCH_LIST] - Update poudriere ports tree"

	echo "		--update-pkg-repo [-a ARCH_LIST]- Rebuild necessary ports on poudriere and update pkg repo"

	echo "		--upload|-U - Upload pkgs and/or snapshots"

	echo "		--skip-final-rsync|-i - Skip rsync to final server"

	echo "		-V VARNAME - print value of variable VARNAME"

	exit 1

}

export BUILDER_ROOT=$(realpath $(dirname ${0}))

export BUILDER_TOOLS="${BUILDER_ROOT}/tools"

unset _SKIP_REBUILD_PRESTAGE

unset _USE_OLD_DATESTRING

unset pfPORTTOBUILD

unset IMAGETYPE

unset UPLOAD

unset SNAPSHOTS

unset POUDRIERE_SNAPSHOTS

unset ARCH_LIST

BUILDACTION="images"

# Maybe use options for nocleans etc?

while test "$1" != ""; do

	case "${1}" in

		--no-buildworld|-c)

			export NO_BUILDWORLD=YES

			export NO_BUILDKERNEL=YES

			;;

		--no-cleanobjdir|-d)

			export NO_CLEAN_FREEBSD_OBJ=YES

			;;

		--resume-image-build|-r)

			export NO_BUILDWORLD=YES

			export NO_BUILDKERNEL=YES

			export NO_CLEAN_FREEBSD_OBJ=YES

			export DO_NOT_SIGN_PKG_REPO=YES

			_SKIP_REBUILD_PRESTAGE=YES

			_USE_OLD_DATESTRING=YES

			;;

		--setup)

			BUILDACTION="builder_setup"

			;;

		--rsync-repos)

			BUILDACTION="rsync_repos"

			export DO_NOT_SIGN_PKG_REPO=YES

			;;

		--rsync-snapshots)

			BUILDACTION="rsync_snapshots"

			export DO_NOT_SIGN_PKG_REPO=YES

			;;

		--build-kernels)

			BUILDACTION="buildkernels"

			;;

		--install-extra-kernels)

			shift

			if [ $# -eq 0 ]; then

				echo "--build-kernel needs extra parameter."

				echo

				usage

			fi

			export INSTALL_EXTRA_KERNELS="${1}"

			;;

		--snapshots)

			export SNAPSHOTS=1

			;;

		--poudriere-snapshots)

			export POUDRIERE_SNAPSHOTS=1

			;;

		--build-kernel)

			BUILDACTION="buildkernel"

			shift

			if [ $# -eq 0 ]; then

				echo "--build-kernel needs extra parameter."

				echo

				usage

			fi

			export BUILD_KERNELS="${1}"

			;;

		--update-sources)

			BUILDACTION="updatesources"

			;;

		--clean-builder)

			BUILDACTION="cleanbuilder"

			;;

		--setup-poudriere)

			BUILDACTION="setup_poudriere"

			;;

		--create-unified-patch)

			BUILDACTION="create_unified_patch"

			;;

		--update-poudriere-jails)

			BUILDACTION="update_poudriere_jails"

			;;

		-a)

			shift

			if [ $# -eq 0 ]; then

				echo "-a needs extra parameter."

				echo

				usage

			fi

			export ARCH_LIST="${1}"

			;;

		--update-poudriere-ports)

			BUILDACTION="update_poudriere_ports"

			;;

		--update-pkg-repo)

			BUILDACTION="update_pkg_repo"

			;;

		--upload|-U)

			export UPLOAD=1

			;;

		--skip-final-rsync|-i)

			export SKIP_FINAL_RSYNC=1

			;;

		all|none|*iso*|*ova*|*memstick*|*memstickserial*|*memstickadi*)

			BUILDACTION="images"

			IMAGETYPE="${1}"

			;;

		-V)

			_USE_OLD_DATESTRING=YES

			shift

			[ -n "${1}" ] \

				&& var_to_print="${1}"

			;;

		--snapshot-update-status)

			shift

			snapshot_status_message="${1}"

			BUILDACTION="snapshot_status_message"

			_USE_OLD_DATESTRING=YES

			;;

		*)

			_USE_OLD_DATESTRING=YES

			usage

	esac

	shift

done

# Suck in local vars

. ${BUILDER_TOOLS}/builder_defaults.sh

# Let user define ARCH_LIST in build.conf

[ -z "${ARCH_LIST}" -a -n "${DEFAULT_ARCH_LIST}" ] \

	&& ARCH_LIST="${DEFAULT_ARCH_LIST}"

# Suck in script helper functions

. ${BUILDER_TOOLS}/builder_common.sh

# Print var required with -V and exit

if [ -n "${var_to_print}"  ]; then

	eval "echo \$${var_to_print}"

	exit 0

fi

# Update snapshot status and exit

if [ "${BUILDACTION}" = "snapshot_status_message" ]; then

	if [ -z "${POUDRIERE_SNAPSHOTS}" ]; then

		export SNAPSHOTS=1

	fi

	snapshots_update_status "${snapshot_status_message}"

	exit 0

fi

# This should be run first

launch

case $BUILDACTION in

	builder_setup)

		builder_setup

	;;

	buildkernels)

		update_freebsd_sources

		build_all_kernels

	;;

	buildkernel)

		update_freebsd_sources

		build_all_kernels

	;;

	cleanbuilder)

		clean_builder

	;;

	images)

		# It will be handled below

	;;

	updatesources)

		update_freebsd_sources

	;;

	setup_poudriere)

		poudriere_init

	;;

	create_unified_patch)

		poudriere_create_patch

	;;

	update_poudriere_jails)

		poudriere_update_jails

	;;

	update_poudriere_ports)

		poudriere_update_ports

	;;

	rsync_repos)

		export UPLOAD=1

		pkg_repo_rsync "${CORE_PKG_PATH}"

	;;

	rsync_snapshots)

		export UPLOAD=1

		snapshots_scp_files

	;;

	update_pkg_repo)

		if [ -n "${UPLOAD}" -a ! -f /usr/local/bin/rsync ]; then

			echo "ERROR: rsync is not installed, aborting..."

			exit 1

		fi

		poudriere_bulk

	;;

	*)

		usage

	;;

esac

if [ "${BUILDACTION}" != "images" ]; then

	finish

	exit 0

fi

if [ -n "${SNAPSHOTS}" -a -n "${UPLOAD}" ]; then

	_required=" \

		RSYNCIP \

		RSYNCUSER \

		RSYNCPATH \

		PKG_RSYNC_HOSTNAME \

		PKG_RSYNC_USERNAME \

		PKG_RSYNC_SSH_PORT \

		PKG_RSYNC_DESTDIR \

		PKG_REPO_SERVER_DEVEL \

		PKG_REPO_SERVER_RELEASE \

		PKG_REPO_SERVER_STAGING \

		PKG_REPO_BRANCH_DEVEL \

		PKG_REPO_BRANCH_RELEASE"

	for _var in ${_required}; do

		eval "_value=\${$_var}"

		if [ -z "${_value}" ]; then

			echo ">>> ERROR: ${_var} is not defined"

			exit 1

		fi

	done

	if [ ! -f /usr/local/bin/rsync ]; then

		echo "ERROR: rsync is not installed, aborting..."

		exit 1

	fi

fi

if [ $# -gt 1 ]; then

	echo "ERROR: Too many arguments given."

	echo

	usage

fi

if [ -n "${SNAPSHOTS}" -a -z "${IMAGETYPE}" ]; then

	IMAGETYPE="all"

fi

if [ -z "${IMAGETYPE}" ]; then

	echo "ERROR: Need to specify image type to build."

	echo

	usage

fi

if [ "$IMAGETYPE" = "none" ]; then

	_IMAGESTOBUILD=""

elif [ "$IMAGETYPE" = "all" ]; then

	_IMAGESTOBUILD="memstick memstickserial"

	if [ "${TARGET}" = "amd64" ]; then

		_IMAGESTOBUILD="${_IMAGESTOBUILD} memstickadi"

		if [ -n "${_IS_RELEASE}"  ]; then

			_IMAGESTOBUILD="${_IMAGESTOBUILD} ova"

		fi

	fi

else

	_IMAGESTOBUILD="${IMAGETYPE}"

fi

echo ">>> Building image type(s): ${_IMAGESTOBUILD}"

if [ -n "${SNAPSHOTS}" ]; then

	snapshots_update_status ">>> Starting snapshot build operations"

	if pkg update -r ${PRODUCT_NAME} >/dev/null 2>&1; then

		snapshots_update_status ">>> Updating builder packages... "

		pkg upgrade -r ${PRODUCT_NAME} -y -q >/dev/null 2>&1

	fi

fi

if [ -z "${_SKIP_REBUILD_PRESTAGE}" ]; then

	[ -n "${CORE_PKG_PATH}" -a -d "${CORE_PKG_PATH}" ] \

		&& rm -rf ${CORE_PKG_PATH}

	# Cleanup environment before start

	clean_builder

	# Make sure source directories are present.

	update_freebsd_sources

	git_last_commit

	# Ensure binaries are present that builder system requires

	depend_check

	# Build world, kernel and install

	make_world

	# Build kernels

	build_all_kernels

	# Install kernel on installer

	installkernel ${INSTALLER_CHROOT_DIR} ${PRODUCT_NAME}

	# Prepare pre-final staging area

	clone_to_staging_area

	# Install packages needed for Product

	install_pkg_install_ports

	# Create core repo

	core_pkg_create_repo

fi

# Send core repo to staging area

pkg_repo_rsync "${CORE_PKG_PATH}" ignore_final_rsync

export DEFAULT_KERNEL=${DEFAULT_KERNEL_ISO:-"${PRODUCT_NAME}"}

# XXX: Figure out why wait is failing and proper fix

# Global variable to keep track of process running in bg

export _bg_pids=""

for _IMGTOBUILD in $_IMAGESTOBUILD; do

	# Clean up items that should be cleaned each run

	staginareas_clean_each_run

	case "${_IMGTOBUILD}" in

		iso)

			if [ -n "${ISO_VARIANTS}" ]; then

				for _variant in ${ISO_VARIANTS}; do

					create_iso_image ${_variant}

				done

			else

				create_iso_image

			fi

			;;

		memstick)

			if [ -n "${MEMSTICK_VARIANTS}" ]; then

				for _variant in ${MEMSTICK_VARIANTS}; do

					create_memstick_image ${_variant}

				done

			else

				create_memstick_image

			fi

			;;

		memstickserial)

			create_memstick_serial_image

			;;

		memstickadi)

			create_memstick_adi_image

			;;

		ova)

			old_custom_package_list="${custom_package_list}"

			export custom_package_list="${custom_package_list} ${PRODUCT_NAME}-pkg-Open-VM-Tools"

			install_pkg_install_ports

			create_ova_image

			export custom_package_list="${old_custom_package_list}"

			install_pkg_install_ports

			;;

	esac

done

if [ -n "${_bg_pids}" ]; then

	if [ -n "${SNAPSHOTS}" ]; then

		snapshots_update_status ">>> NOTE: waiting for jobs: ${_bg_pids} to finish..."

	else

		echo ">>> NOTE: waiting for jobs: ${_bg_pids} to finish..."

	fi

	wait

	# XXX: For some reason wait is failing, workaround it tracking all PIDs

	while [ -n "${_bg_pids}" ]; do

		_tmp_pids="${_bg_pids}"

		unset _bg_pids

		for p in ${_tmp_pids}; do

			[ -z "${p}" ] \

				&& continue

			kill -0 ${p} >/dev/null 2>&1 \

				&& _bg_pids="${_bg_pids}${_bg_pids:+ }${p}"

		done

		[ -n "${_bg_pids}" ] \

			&& sleep 1

	done

fi

if [ -n "${SNAPSHOTS}" ]; then

	if [ "${IMAGETYPE}" = "none" -a -n "${UPLOAD}" ]; then

		pkg_repo_rsync "${CORE_PKG_PATH}"

	elif [ "${IMAGETYPE}" != "none" ]; then

		snapshots_create_sha256

		# SCP files to snapshot web hosting area

		if [ -n "${UPLOAD}" ]; then

			snapshots_scp_files

		fi

	fi

	# Alert the world that we have some snapshots ready.

	snapshots_update_status ">>> Builder run is complete."

fi

echo ">>> ${IMAGES_FINAL_DIR} now contains:"

(cd ${IMAGES_FINAL_DIR} && find ${IMAGES_FINAL_DIR} -type f)

set -e

# Run final finish routines

finish