Revision 5796b157
Added by Renato Botelho over 3 years ago
tools/builder_common.sh | ||
---|---|---|
1616 | 1616 |
script -aq ${LOGFILE} zfs create ${ZFS_TANK}/poudriere/ports/${POUDRIERE_PORTS_NAME} |
1617 | 1617 |
# Download local copy of the ports tree stashed in S3 |
1618 | 1618 |
echo ">>> Downloading cached copy of the ports tree from S3.." | tee -a ${LOGFILE} |
1619 |
script -aq ${LOGFILE} env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \ |
|
1620 |
aws s3 cp s3://pfsense-engineering-build-pkg/${FLAVOR}-ports.tz . --no-progress |
|
1619 |
aws_exec s3 cp s3://pfsense-engineering-build-pkg/${FLAVOR}-ports.tz . --no-progress |
|
1621 | 1620 |
script -aq ${LOGFILE} tar --strip-components 1 -xf ${FLAVOR}-ports.tz -C /usr/local/poudriere/ports/${POUDRIERE_PORTS_NAME} |
1622 | 1621 |
# Update the ports tree |
1623 | 1622 |
( |
... | ... | |
1740 | 1739 |
if [ "${AWS}" = 1 ]; then |
1741 | 1740 |
# Download a copy of the distfiles from S3 |
1742 | 1741 |
echo ">>> Downloading distfile cache from S3.." | tee -a ${LOGFILE} |
1743 |
script -aq ${LOGFILE} env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \ |
|
1744 |
aws s3 cp s3://pfsense-engineering-build-pkg/distfiles.tar . --no-progress |
|
1742 |
aws_exec s3 cp s3://pfsense-engineering-build-pkg/distfiles.tar . --no-progress |
|
1745 | 1743 |
script -aq ${LOGFILE} tar -xf distfiles.tar -C /usr/ports/distfiles |
1746 | 1744 |
# Save a list of distfiles |
1747 | 1745 |
find /usr/ports/distfiles > pre-build-distfile-list |
... | ... | |
1787 | 1785 |
mkdir objs |
1788 | 1786 |
echo ">>> Downloading prebuilt release objs from s3://pfsense-engineering-build-freebsd-obj-tarballs/${FLAVOR}/ ..." | tee -a ${LOGFILE} |
1789 | 1787 |
# Download prebuilt release tarballs from previous job |
1790 |
env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \ |
|
1791 |
aws s3 cp s3://pfsense-engineering-build-freebsd-obj-tarballs/${FLAVOR}/LATEST-${jail_arch} objs --no-progress |
|
1788 |
aws_exec s3 cp s3://pfsense-engineering-build-freebsd-obj-tarballs/${FLAVOR}/LATEST-${jail_arch} objs --no-progress |
|
1792 | 1789 |
SRC_COMMIT=`cat objs/LATEST-${jail_arch}` |
1793 |
env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \ |
|
1794 |
aws s3 cp s3://pfsense-engineering-build-freebsd-obj-tarballs/${FLAVOR}/MANIFEST-${jail_arch}-${SRC_COMMIT} objs --no-progress |
|
1790 |
aws_exec s3 cp s3://pfsense-engineering-build-freebsd-obj-tarballs/${FLAVOR}/MANIFEST-${jail_arch}-${SRC_COMMIT} objs --no-progress |
|
1795 | 1791 |
ln -s MANIFEST-${jail_arch}-${SRC_COMMIT} objs/MANIFEST |
1796 | 1792 |
for i in base doc kernel src tests; do |
1797 | 1793 |
if [ ! -f objs/${i}-${jail_arch}-${SRC_COMMIT}.txz ]; then |
1798 |
env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \ |
|
1799 |
aws s3 cp s3://pfsense-engineering-build-freebsd-obj-tarballs/${FLAVOR}/${i}-${jail_arch}-${SRC_COMMIT}.txz objs --no-progress |
|
1794 |
aws_exec s3 cp s3://pfsense-engineering-build-freebsd-obj-tarballs/${FLAVOR}/${i}-${jail_arch}-${SRC_COMMIT}.txz objs --no-progress |
|
1800 | 1795 |
ln -s ${i}-${jail_arch}-${SRC_COMMIT}.txz objs/${i}.txz |
1801 | 1796 |
fi |
1802 | 1797 |
done |
... | ... | |
1812 | 1807 |
OLDIFS=${IFS} |
1813 | 1808 |
IFS=$'\n' |
1814 | 1809 |
echo ">>> Downloading cached pkgs for ${jail_arch} from S3.." | tee -a ${LOGFILE} |
1815 |
for i in `env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \ |
|
1816 |
aws s3 ls s3://pfsense-engineering-build-pkg/`; do |
|
1810 |
for i in $(aws_exec s3 ls s3://pfsense-engineering-build-pkg/); do |
|
1817 | 1811 |
echo ${i} | awk '{print $4}' | grep pkgs-${jail_arch}.tar > /dev/null |
1818 | 1812 |
if [ $? -eq 0 ]; then |
1819 |
script -aq ${LOGFILE} env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \ |
|
1820 |
aws s3 cp s3://pfsense-engineering-build-pkg/pkgs-${jail_arch}.tar . --no-progress |
|
1813 |
aws_exec s3 cp s3://pfsense-engineering-build-pkg/pkgs-${jail_arch}.tar . --no-progress |
|
1821 | 1814 |
[ ! -d /usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME} ] && mkdir -p /usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME} |
1822 | 1815 |
echo "Extracting pkgs-${jail_arch}.tar to /usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME}" | tee -a ${LOGFILE} |
1823 | 1816 |
[ ! -d /usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME} ] && mkdir /usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME} |
... | ... | |
1902 | 1895 |
# Save a copy of the past few logs into S3 |
1903 | 1896 |
DATE=`date +%Y%m%d-%H%M%S` |
1904 | 1897 |
script -aq ${LOGFILE} tar --zstd -cf pkg-logs-${jail_arch}-${DATE}.tar -C /usr/local/poudriere/data/logs/bulk/${jail_name}-${POUDRIERE_PORTS_NAME}/latest/ . |
1905 |
script -aq ${LOGFILE} env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \ |
|
1906 |
aws s3 cp pkg-logs-${jail_arch}-${DATE}.tar s3://pfsense-engineering-build-pkg/logs/ --no-progress |
|
1898 |
aws_exec s3 cp pkg-logs-${jail_arch}-${DATE}.tar s3://pfsense-engineering-build-pkg/logs/ --no-progress |
|
1907 | 1899 |
OLDIFS=${IFS} |
1908 | 1900 |
IFS=$'\n' |
1909 | 1901 |
local _logtemp=$( mktemp /tmp/loglist.XXXXX ) |
1910 |
for i in `env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \ |
|
1911 |
aws s3 ls s3://pfsense-engineering-build-pkg/logs/`; do |
|
1902 |
for i in $(aws_exec s3 ls s3://pfsense-engineering-build-pkg/logs/); do |
|
1912 | 1903 |
echo ${i} | awk '{print $4}' | grep pkg-logs-${jail_arch} >> ${_logtemp} |
1913 | 1904 |
done |
1914 | 1905 |
local _maxlogs=5 |
... | ... | |
1917 | 1908 |
if [ ${_curlogs} -gt ${_maxlogs} ]; then |
1918 | 1909 |
local _extralogs=$(( ${_curlogs} - ${_maxlogs} )) |
1919 | 1910 |
for _last in $( head -${_extralogs} ${_logtemp} ); do |
1920 |
env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \ |
|
1921 |
aws s3 rm s3://pfsense-engineering-build-pkg/logs/${_last} |
|
1911 |
aws_exec s3 rm s3://pfsense-engineering-build-pkg/logs/${_last} |
|
1922 | 1912 |
done |
1923 | 1913 |
fi |
1924 | 1914 |
IFS=${OLDIFS} |
... | ... | |
1933 | 1923 |
if [ $? = 1 ]; then |
1934 | 1924 |
[ -f pkgs-${jail_arch}.tar ] && rm pkgs-${jail_arch}.tar |
1935 | 1925 |
script -aq ${LOGFILE} tar -cf pkgs-${jail_arch}.tar -C /usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME} . |
1936 |
script -aq ${LOGFILE} env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \ |
|
1937 |
aws s3 cp pkgs-${jail_arch}.tar s3://pfsense-engineering-build-pkg/ --no-progress |
|
1926 |
aws_exec s3 cp pkgs-${jail_arch}.tar s3://pfsense-engineering-build-pkg/ --no-progress |
|
1938 | 1927 |
|
1939 | 1928 |
save_logs_to_s3 |
1940 | 1929 |
fi |
1941 | 1930 |
} |
1942 | 1931 |
|
1932 |
aws_exec() { |
|
1933 |
script -aq ${LOGFILE} \ |
|
1934 |
env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \ |
|
1935 |
AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \ |
|
1936 |
AWS_DEFAULT_REGION=us-east-2 \ |
|
1937 |
aws $@ |
|
1938 |
return $? |
|
1939 |
} |
|
1940 |
|
|
1943 | 1941 |
poudriere_bulk() { |
1944 | 1942 |
local _archs=$(poudriere_possible_archs) |
1945 | 1943 |
local _makeconf |
... | ... | |
2081 | 2079 |
if [ $? -eq 1 ]; then |
2082 | 2080 |
rm distfiles.tar |
2083 | 2081 |
script -aq ${LOGFILE} tar -cf distfiles.tar -C /usr/ports/distfiles . |
2084 |
script -aq ${LOGFILE} env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \ |
|
2085 |
aws s3 cp distfiles.tar s3://pfsense-engineering-build-pkg/ --no-progress |
|
2082 |
aws_exec s3 cp distfiles.tar s3://pfsense-engineering-build-pkg/ --no-progress |
|
2086 | 2083 |
fi |
2087 | 2084 |
fi |
2088 | 2085 |
} |
Also available in: Unified diff
AWS: Simplify logic
Create aws_exec() and replace all direct calls to use it