pfSense

#!/bin/sh

#

# builder_common.sh

#

# Copyright (c) 2013-2016 Electric Sheep Fencing

#

# FreeSBIE portions of the code

# Copyright (c) 2005 Dario Freni

# and copied from FreeSBIE project

# All rights reserved.

#

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

fi

	echo "${1}" | tr '[[:upper:]]' '[[:lower:]]'

}

git_last_commit() {

	export CURRENT_AUTHOR=$(git -C ${BUILDER_ROOT} log -1 --format='%an')

	echo "$CURRENT_COMMIT" > $SCRATCHDIR/build_commit_info.txt

core_pkg_create_repo() {

	fi

	#

	# For some reason pkg-repo fail without / in the end of directory name

	# so removing it will break command

	#

	# https://github.com/freebsd/pkg/issues/1364

	#

	else

		echo "Failed!"

		print_error_pfS

	fi

	# Use the same directory structure as poudriere does to avoid

	# breaking snapshot repositories during rsync

	ln -sf $(basename ${CORE_PKG_REAL_PATH}) ${CORE_PKG_PATH}/.latest

	ln -sf .latest/packagesite.txz ${CORE_PKG_PATH}/packagesite.txz

core_pkg_create() {

	local _template="${1}"

	local _flavor="${2}"

	local _version="${3}"

	local _root="${4}"

	local _filter="${6}"

	local _abi=$(sed -e "s/%%ARCH%%/${TARGET_ARCH}/g" \

	    ${PKG_REPO_DEFAULT%%.conf}.abi)

	local _altabi_arch=$(get_altabi_arch ${TARGET_ARCH})

	local _altabi=$(sed -e "s/%%ARCH%%/${_altabi_arch}/g" \

	    ${PKG_REPO_DEFAULT%%.conf}.altabi)

		-f "${_flavor}" \

		-v "${_version}" \

		-r "${_root}" \

		-d "${CORE_PKG_REAL_PATH}/All" \

		-A "${_altabi}" \

# This routine will output that something went wrong

print_error_pfS() {

	echo

	echo "####################################"

	echo "Something went wrong, check errors!" >&2

	echo "####################################"

	echo

	echo "NOTE: a lot of times you can run './build.sh --clean-builder' to resolve."

	echo

	echo

	kill $$

	exit 1

}

# This routine will verify that the kernel has been

# installed OK to the staging area.

ensure_kernel_exists() {

	if [ ! -f "$1/boot/kernel/kernel.gz" ]; then

		echo ">>> ERROR: Could not locate $1/boot/kernel.gz"

		print_error_pfS

	fi

	KERNEL_SIZE=$(stat -f "%z" $1/boot/kernel/kernel.gz)

	if [ "$KERNEL_SIZE" -lt 3500 ]; then

		echo ">>> ERROR: Kernel $1/boot/kernel.gz appears to be smaller than it should be: $KERNEL_SIZE"

		print_error_pfS

	fi

}

get_pkg_name() {

	echo "${PRODUCT_NAME}-${1}-${CORE_PKG_VERSION}"

}

# This routine builds all related kernels

build_all_kernels() {

	# Set KERNEL_BUILD_PATH if it has not been set

	if [ -z "${KERNEL_BUILD_PATH}" ]; then

		KERNEL_BUILD_PATH=$SCRATCHDIR/kernels

		echo ">>> KERNEL_BUILD_PATH has not been set. Setting to ${KERNEL_BUILD_PATH}!"

	fi

	[ -d "${KERNEL_BUILD_PATH}" ] \

		&& rm -rf ${KERNEL_BUILD_PATH}

	# Build embedded kernel

	for BUILD_KERNEL in $BUILD_KERNELS; do

		unset KERNCONF

		unset KERNEL_DESTDIR

		unset KERNEL_NAME

		export KERNCONF=$BUILD_KERNEL

		export KERNEL_DESTDIR="$KERNEL_BUILD_PATH/$BUILD_KERNEL"

		export KERNEL_NAME=${BUILD_KERNEL}

		LOGFILE="${BUILDER_LOGS}/kernel.${KERNCONF}.${TARGET}.log"

		echo ">>> Building $BUILD_KERNEL kernel."  | tee -a ${LOGFILE}

			continue

		fi

		buildkernel

		echo ">>> Staging $BUILD_KERNEL kernel..." | tee -a ${LOGFILE}

		installkernel

		ensure_kernel_exists $KERNEL_DESTDIR

		rm -rf $KERNEL_DESTDIR 2>&1 1>/dev/null

	done

}

install_default_kernel() {

	if [ -z "${1}" ]; then

		echo ">>> ERROR: install_default_kernel called without a kernel config name"| tee -a ${LOGFILE}

		print_error_pfS

	fi

	export KERNEL_NAME="${1}"

	echo -n ">>> Installing kernel to be used by image ${KERNEL_NAME}..." | tee -a ${LOGFILE}

	# Copy kernel package to chroot, otherwise pkg won't find it to install

	if ! pkg_chroot_add ${FINAL_CHROOT_DIR} kernel-${KERNEL_NAME}; then

		echo ">>> ERROR: Error installing kernel package $(get_pkg_name kernel-${KERNEL_NAME}).txz" | tee -a ${LOGFILE}

		print_error_pfS

	fi

	pkg_chroot ${FINAL_CHROOT_DIR} set -v 1 -y $(get_pkg_name kernel-${KERNEL_NAME})

	if [ ! -f $FINAL_CHROOT_DIR/boot/kernel/kernel.gz ]; then

		echo ">>> ERROR: No kernel installed on $FINAL_CHROOT_DIR and the resulting image will be unusable. STOPPING!" | tee -a ${LOGFILE}

		print_error_pfS

	fi

	mkdir -p $FINAL_CHROOT_DIR/pkgs

	if [ -z "${2}" -o -n "${INSTALL_EXTRA_KERNELS}" ]; then

			for _EXTRA_KERNEL in $INSTALL_EXTRA_KERNELS; do

					echo -n ". adding ${_EXTRA_KERNEL_PATH} on image /pkgs folder"

					cp ${_EXTRA_KERNEL_PATH} $FINAL_CHROOT_DIR/pkgs

				else

					echo ">>> ERROR: Requested kernel $(get_pkg_name kernel-${_EXTRA_KERNEL}).txz was not found to be put on image /pkgs folder!"

					print_error_pfS

				fi

			done

		fi

	fi

	echo "Done." | tee -a ${LOGFILE}

	unset KERNEL_NAME

}

# This builds FreeBSD (make buildworld)

# Imported from FreeSBIE

make_world() {

	LOGFILE=${BUILDER_LOGS}/buildworld.${TARGET}

		return

	fi

	echo ">>> $(LC_ALL=C date) - Finished build world for ${TARGET} architecture..." | tee -a ${LOGFILE}

	LOGFILE=${BUILDER_LOGS}/installworld.${TARGET}

	echo ">>> LOGFILE set to $LOGFILE." | tee -a ${LOGFILE}

	[ -d "${INSTALLER_CHROOT_DIR}" ] \

		|| mkdir -p ${INSTALLER_CHROOT_DIR}

		-d ${INSTALLER_CHROOT_DIR} \

		|| print_error_pfS

	install -o root -g wheel -m 0755 ${BUILDER_TOOLS}/installer/*.sh \

		${INSTALLER_CHROOT_DIR}/root

	pw -R ${INSTALLER_CHROOT_DIR} usermod root -w yes

		-d ${STAGE_CHROOT_DIR} \

		|| print_error_pfS

	[ -f "${BUILD_CC}" ] || print_error_pfS

		|| mkdir -p ${STAGE_CHROOT_DIR}/usr/local/bin

#	(script -aq $LOGFILE make -C ${FREEBSD_SRC_DIR}/tools/tools/ath/athstats ${makeargs} clean all install || print_error_pfS;) | egrep '^>>>' | tee -a ${LOGFILE}

		echo ">>> Building gnid... " | tee -a ${LOGFILE}

			cd ${GNID_SRC_DIR} && \

				CC=${BUILD_CC} \

				INCLUDE_DIR=${GNID_INCLUDE_DIR} \

				LIBCRYPTO_DIR=${GNID_LIBCRYPTO_DIR} \

			clean gnid \

			${STAGE_CHROOT_DIR}/usr/sbin \

			|| print_error_pfS

		install -o root -g wheel -m 0700 ${GNID_SRC_DIR}/gnid \

			${INSTALLER_CHROOT_DIR}/usr/sbin \

			|| print_error_pfS

	fi

}

# This routine creates a ova image that contains

# a ovf and vmdk file. These files can be imported

# right into vmware or virtual box.

# (and many other emulation platforms)

# http://www.vmware.com/pdf/ovf_whitepaper_specification.pdf

create_ova_image() {

	# XXX create a .ovf php creator that you can pass:

	#     1. populatedSize

	#     2. license

	#     3. product name

	#     4. version

	#     5. number of network interface cards

	#     6. allocationUnits

	#     7. capacity

	#     8. capacityAllocationUnits

	LOGFILE=${BUILDER_LOGS}/ova.${TARGET}.log

	if [ -d "${_mntdir}" ]; then

		# XXX Root cause still didn't found but it doesn't umount

		#     properly on looped builds and then require this extra

		#     check

		while true; do

				&& break

		done

		rm -rf ${OVA_TMP}

	fi

		# first partition size (freebsd-ufs)

		local OVA_FIRST_PART_SIZE_IN_GB=${VMDK_DISK_CAPACITY_IN_GB}

		local OVA_FIRST_PART_SIZE=$((${OVA_FIRST_PART_SIZE_IN_GB}*1024*1024*1024-131072))

		unset OVA_SWAP_PART_SIZE

		# Parameter used by mkimg

		unset OVA_SWAP_PART_PARAM

	else

		# first partition size (freebsd-ufs)

		local OVA_FIRST_PART_SIZE_IN_GB=$((VMDK_DISK_CAPACITY_IN_GB-OVA_SWAP_PART_SIZE_IN_GB))

		# Use first partition size in g

		local OVA_FIRST_PART_SIZE="${OVA_FIRST_PART_SIZE_IN_GB}g"

		local OVA_SWAP_PART_SIZE=$((${OVA_SWAP_PART_SIZE_IN_GB}*1024*1024*1024-131072))

		local OVA_SWAP_PART_PARAM="-p freebsd-swap/swap0::${OVA_SWAP_PART_SIZE}"

	fi

	customize_stagearea_for_image "ova"

	install_default_kernel ${DEFAULT_KERNEL} "no"

	echo ">>> Installing platform specific items..." | tee -a ${LOGFILE}

		echo "/dev/gpt/swap0	none	swap	sw	0	0" >> ${FINAL_CHROOT_DIR}/etc/fstab

	fi

	# Create / partition

	newfs -L ${PRODUCT_NAME} -j /dev/${_md} 2>&1 >>${LOGFILE}

	fi

	sync

	trap "-" 1 2 15 EXIT

	echo -n ">>> Creating raw disk... " | tee -a ${LOGFILE}

		-s gpt \

		-p freebsd-boot:=${FINAL_CHROOT_DIR}/boot/gptboot \

			rm -f ${OVA_TMP}/${OVFUFS}

		fi

			rm -f ${OVA_TMP}/${OVFRAW}

		print_error_pfS

	fi

	# We don't need it anymore

	rm -f ${OVA_TMP}/${OVFUFS} >/dev/null 2>&1

			rm -f ${OVA_TMP}/${OVFRAW}

			rm -f ${OVA_TMP}/${OVFVMDK}

		fi

		print_error_pfS

	# Create OVA file for vmware

	echo ">>> OVA created: $(LC_ALL=C date)" | tee -a ${LOGFILE}

# called from create_ova_image

		print_error_pfS

	#  42   FreeBSD 32-Bit

	#  78   FreeBSD 64-Bit

	if [ "${TARGET}" = "amd64" ]; then

		local _os_id="78"

		local _os_descr="FreeBSD 64-Bit"

		echo ">>> ERROR: Platform not supported for OVA (${TARGET})"

		print_error_pfS

	fi

	local POPULATED_SIZE_IN_BYTES=$((${POPULATED_SIZE}*1024))

	local VMDK_FILE_SIZE=$(stat -f "%z" ${OVA_TMP}/${OVFVMDK})

		-e "s,%%VMDK_DISK_CAPACITY_IN_GB%%,${VMDK_DISK_CAPACITY_IN_GB},g" \

		-e "s,%%POPULATED_SIZE_IN_BYTES%%,${POPULATED_SIZE_IN_BYTES},g" \

		-e "s,%%OS_TYPE%%,${_os_type},g" \

		-e "s,%%PRODUCT_URL%%,${PRODUCT_URL},g" \

		-e "s#%%OVF_INFO%%#${OVF_INFO}#g" \

		${OVFTEMPLATE} > ${OVA_TMP}/${PRODUCT_NAME}.ovf

# Cleans up previous builds

	echo ">>> Cleaning up previous build environment...Please wait!"

	staginareas_clean_each_run

	if [ -d "${STAGE_CHROOT_DIR}" ]; then

	fi

		echo -n ">>> Cleaning ${INSTALLER_CHROOT_DIR}... "

		chflags -R noschg ${INSTALLER_CHROOT_DIR} 2>&1 >/dev/null

		rm -rf ${INSTALLER_CHROOT_DIR}/* 2>/dev/null

		echo "Done."

	fi

			echo -n ">>> Cleaning FreeBSD objects dir staging..."

			echo -n "."

			chflags -R noschg ${OBJTREE} 2>&1 >/dev/null

			echo -n "."

			rm -rf ${OBJTREE}/*

			echo "Done!"

		fi

			rm -rf $KERNEL_BUILD_PATH/*

			echo "Done!"

		fi

	fi

	mkdir -p $KERNEL_BUILD_PATH

	echo -n ">>> Cleaning previously built images..."

	echo -n ">>> Cleaning previous builder logs..."

	if [ -d "$BUILDER_LOGS" ]; then

		rm -rf ${BUILDER_LOGS}

	fi

	mkdir -p ${BUILDER_LOGS}

	echo "Done!"

	echo ">>> Cleaning of builder environment has finished."

}

clone_directory_contents() {

		mkdir -p "$2"

	fi

		if [ -z "${LOGFILE}" ]; then

			echo ">>> ERROR: Argument $1 supplied is not a directory!"

		else

			echo ">>> ERROR: Argument $1 supplied is not a directory!" | tee -a ${LOGFILE}

		fi

		print_error_pfS

	fi

	echo -n ">>> Using TAR to clone $1 to $2 ..."

	tar -C ${1} -c -f - . | tar -C ${2} -x -p -f -

	echo "Done!"

}

clone_to_staging_area() {

	# Clone everything to the final staging area

	echo -n ">>> Cloning everything to ${STAGE_CHROOT_DIR} staging area..."

	LOGFILE=${BUILDER_LOGS}/cloning.${TARGET}.log

	mkdir -p ${STAGE_CHROOT_DIR}/etc/mtree

	mtree -Pcp ${STAGE_CHROOT_DIR}/var > ${STAGE_CHROOT_DIR}/etc/mtree/var.dist

	mtree -Pcp ${STAGE_CHROOT_DIR}/etc > ${STAGE_CHROOT_DIR}/etc/mtree/etc.dist

	if [ -d ${STAGE_CHROOT_DIR}/usr/local/etc ]; then

		mtree -Pcp ${STAGE_CHROOT_DIR}/usr/local/etc > ${STAGE_CHROOT_DIR}/etc/mtree/localetc.dist

	fi

	## Add buildtime and lastcommit information

	# This is used for detecting updates.

	echo "$BUILTDATESTRING" > $STAGE_CHROOT_DIR/etc/version.buildtime

	# Record last commit info if it is available.

	if [ -f $SCRATCHDIR/build_commit_info.txt ]; then

		cp $SCRATCHDIR/build_commit_info.txt $STAGE_CHROOT_DIR/etc/version.lastcommit

	fi

		-e "s,%%PRODUCT_NAME%%,${PRODUCT_NAME},g" \

		-e "s,%%VERSION%%,${_version},g" \

	# Include a sample pkg stable conf to base

	setup_pkg_repo \

		-c \

		-k uid,gid,mode,size,flags,sha256digest \

		-p ${STAGE_CHROOT_DIR} \

		-X ${_exclude_files} \

		-C ${STAGE_CHROOT_DIR} \

		.

	core_pkg_create default-config "" ${CORE_PKG_VERSION} ${STAGE_CHROOT_DIR}

	local DEFAULTCONF=${STAGE_CHROOT_DIR}/conf.default/config.xml

	local _old_lan_if=$(xml sel -t -v "${XML_ROOTOBJ}/interfaces/lan/if" ${DEFAULTCONF})

	xml ed -P -L -u "${XML_ROOTOBJ}/interfaces/lan/if" -v "vmx1" ${DEFAULTCONF}

	# Restore default values to be used by serial package

	xml ed -P -L -u "${XML_ROOTOBJ}/interfaces/lan/if" -v "${_old_lan_if}" ${DEFAULTCONF}

	xml ed -P -s "${XML_ROOTOBJ}/system" -t elem -n "enableserial" \

		${DEFAULTCONF} > ${DEFAULTCONF}.tmp

	xml fo -t ${DEFAULTCONF}.tmp > ${DEFAULTCONF}

	rm -f ${DEFAULTCONF}.tmp

	echo force > ${STAGE_CHROOT_DIR}/cf/conf/enableserial_force

	core_pkg_create default-config-serial "" ${CORE_PKG_VERSION} ${STAGE_CHROOT_DIR}

	rm -f ${STAGE_CHROOT_DIR}/cf/conf/enableserial_force

	rm -f ${STAGE_CHROOT_DIR}/cf/conf/config.xml

	# Make sure pkg is present

	pkg_bootstrap ${STAGE_CHROOT_DIR}

		${TARGET_ARCH} \

		${STAGE_CHROOT_DIR}/tmp/pkg/pkg.conf