Project

General

Profile

« Previous | Next » 

Revision 5796b157

Added by Renato Botelho over 3 years ago

AWS: Simplify logic

Create aws_exec() and replace all direct calls to use it

View differences:

tools/builder_common.sh
1616 1616
			script -aq ${LOGFILE} zfs create ${ZFS_TANK}/poudriere/ports/${POUDRIERE_PORTS_NAME}
1617 1617
			# Download local copy of the ports tree stashed in S3
1618 1618
			echo ">>>  Downloading cached copy of the ports tree from S3.." | tee -a ${LOGFILE}
1619
			script -aq ${LOGFILE} env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
1620
				aws s3 cp s3://pfsense-engineering-build-pkg/${FLAVOR}-ports.tz . --no-progress
1619
			aws_exec s3 cp s3://pfsense-engineering-build-pkg/${FLAVOR}-ports.tz . --no-progress
1621 1620
			script -aq ${LOGFILE} tar --strip-components 1 -xf ${FLAVOR}-ports.tz -C /usr/local/poudriere/ports/${POUDRIERE_PORTS_NAME}
1622 1621
			# Update the ports tree
1623 1622
			(
......
1740 1739
	if [ "${AWS}" = 1 ]; then
1741 1740
		# Download a copy of the distfiles from S3
1742 1741
		echo ">>> Downloading distfile cache from S3.." | tee -a ${LOGFILE}
1743
		script -aq ${LOGFILE} env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
1744
			aws s3 cp s3://pfsense-engineering-build-pkg/distfiles.tar . --no-progress
1742
		aws_exec s3 cp s3://pfsense-engineering-build-pkg/distfiles.tar . --no-progress
1745 1743
		script -aq ${LOGFILE} tar -xf distfiles.tar -C /usr/ports/distfiles
1746 1744
		# Save a list of distfiles
1747 1745
		find /usr/ports/distfiles > pre-build-distfile-list
......
1787 1785
			mkdir objs
1788 1786
			echo ">>> Downloading prebuilt release objs from s3://pfsense-engineering-build-freebsd-obj-tarballs/${FLAVOR}/ ..." | tee -a ${LOGFILE}
1789 1787
			# Download prebuilt release tarballs from previous job
1790
			env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
1791
				aws s3 cp s3://pfsense-engineering-build-freebsd-obj-tarballs/${FLAVOR}/LATEST-${jail_arch} objs --no-progress
1788
			aws_exec s3 cp s3://pfsense-engineering-build-freebsd-obj-tarballs/${FLAVOR}/LATEST-${jail_arch} objs --no-progress
1792 1789
			SRC_COMMIT=`cat objs/LATEST-${jail_arch}`
1793
			env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
1794
				aws s3 cp s3://pfsense-engineering-build-freebsd-obj-tarballs/${FLAVOR}/MANIFEST-${jail_arch}-${SRC_COMMIT} objs --no-progress
1790
			aws_exec s3 cp s3://pfsense-engineering-build-freebsd-obj-tarballs/${FLAVOR}/MANIFEST-${jail_arch}-${SRC_COMMIT} objs --no-progress
1795 1791
			ln -s MANIFEST-${jail_arch}-${SRC_COMMIT} objs/MANIFEST
1796 1792
			for i in base doc kernel src tests; do
1797 1793
				if [ ! -f objs/${i}-${jail_arch}-${SRC_COMMIT}.txz ]; then
1798
					env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
1799
						aws s3 cp s3://pfsense-engineering-build-freebsd-obj-tarballs/${FLAVOR}/${i}-${jail_arch}-${SRC_COMMIT}.txz objs --no-progress
1794
					aws_exec s3 cp s3://pfsense-engineering-build-freebsd-obj-tarballs/${FLAVOR}/${i}-${jail_arch}-${SRC_COMMIT}.txz objs --no-progress
1800 1795
					ln -s ${i}-${jail_arch}-${SRC_COMMIT}.txz objs/${i}.txz
1801 1796
				fi
1802 1797
			done
......
1812 1807
			OLDIFS=${IFS}
1813 1808
			IFS=$'\n'
1814 1809
			echo ">>> Downloading cached pkgs for ${jail_arch} from S3.." | tee -a ${LOGFILE}
1815
			for i in `env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
1816
			    aws s3 ls s3://pfsense-engineering-build-pkg/`; do
1810
			for i in $(aws_exec s3 ls s3://pfsense-engineering-build-pkg/); do
1817 1811
				echo ${i} | awk '{print $4}' | grep pkgs-${jail_arch}.tar > /dev/null
1818 1812
				if [ $? -eq 0 ]; then
1819
					script -aq ${LOGFILE} env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
1820
					    aws s3 cp s3://pfsense-engineering-build-pkg/pkgs-${jail_arch}.tar . --no-progress
1813
					aws_exec s3 cp s3://pfsense-engineering-build-pkg/pkgs-${jail_arch}.tar . --no-progress
1821 1814
					[ ! -d /usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME} ] && mkdir -p /usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME}
1822 1815
					echo "Extracting pkgs-${jail_arch}.tar to /usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME}" | tee -a ${LOGFILE}
1823 1816
					[ ! -d /usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME} ] && mkdir /usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME}
......
1902 1895
	# Save a copy of the past few logs into S3
1903 1896
	DATE=`date +%Y%m%d-%H%M%S`
1904 1897
	script -aq ${LOGFILE} tar --zstd -cf pkg-logs-${jail_arch}-${DATE}.tar -C /usr/local/poudriere/data/logs/bulk/${jail_name}-${POUDRIERE_PORTS_NAME}/latest/ .
1905
	script -aq ${LOGFILE} env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
1906
	    aws s3 cp pkg-logs-${jail_arch}-${DATE}.tar s3://pfsense-engineering-build-pkg/logs/ --no-progress
1898
	aws_exec s3 cp pkg-logs-${jail_arch}-${DATE}.tar s3://pfsense-engineering-build-pkg/logs/ --no-progress
1907 1899
	OLDIFS=${IFS}
1908 1900
	IFS=$'\n'
1909 1901
	local _logtemp=$( mktemp /tmp/loglist.XXXXX )
1910
	for i in `env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
1911
	    aws s3 ls s3://pfsense-engineering-build-pkg/logs/`; do
1902
	for i in $(aws_exec s3 ls s3://pfsense-engineering-build-pkg/logs/); do
1912 1903
		echo ${i} | awk '{print $4}' | grep pkg-logs-${jail_arch} >> ${_logtemp}
1913 1904
	done
1914 1905
	local _maxlogs=5
......
1917 1908
	if [ ${_curlogs} -gt ${_maxlogs} ]; then
1918 1909
		local _extralogs=$(( ${_curlogs} - ${_maxlogs} ))
1919 1910
		for _last in $( head -${_extralogs} ${_logtemp} ); do
1920
			env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
1921
			    aws s3 rm s3://pfsense-engineering-build-pkg/logs/${_last}
1911
			aws_exec s3 rm s3://pfsense-engineering-build-pkg/logs/${_last}
1922 1912
		done
1923 1913
	fi
1924 1914
	IFS=${OLDIFS}
......
1933 1923
	if [ $? = 1 ]; then
1934 1924
		[ -f pkgs-${jail_arch}.tar ] && rm pkgs-${jail_arch}.tar
1935 1925
		script -aq ${LOGFILE} tar -cf pkgs-${jail_arch}.tar -C /usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME} .
1936
		script -aq ${LOGFILE} env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
1937
		    aws s3 cp pkgs-${jail_arch}.tar s3://pfsense-engineering-build-pkg/ --no-progress
1926
		aws_exec s3 cp pkgs-${jail_arch}.tar s3://pfsense-engineering-build-pkg/ --no-progress
1938 1927

  
1939 1928
		save_logs_to_s3
1940 1929
	fi
1941 1930
}
1942 1931

  
1932
aws_exec() {
1933
	script -aq ${LOGFILE} \
1934
	    env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
1935
	    AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
1936
	    AWS_DEFAULT_REGION=us-east-2 \
1937
	    aws $@
1938
	return $?
1939
}
1940

  
1943 1941
poudriere_bulk() {
1944 1942
	local _archs=$(poudriere_possible_archs)
1945 1943
	local _makeconf
......
2081 2079
		if [ $? -eq 1 ]; then
2082 2080
			rm distfiles.tar
2083 2081
			script -aq ${LOGFILE} tar -cf distfiles.tar -C /usr/ports/distfiles .
2084
			script -aq ${LOGFILE} env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
2085
			    aws s3 cp distfiles.tar s3://pfsense-engineering-build-pkg/ --no-progress
2082
			aws_exec s3 cp distfiles.tar s3://pfsense-engineering-build-pkg/ --no-progress
2086 2083
		fi
2087 2084
	fi
2088 2085
}

Also available in: Unified diff