Revision 8d49874f
Added by Brad Davis over 3 years ago
| tools/builder_common.sh | ||
|---|---|---|
| 1607 | 1607 |
print_error_pfS |
| 1608 | 1608 |
fi |
| 1609 | 1609 |
if [ -n "${POUDRIERE_PORTS_GIT_BRANCH}" ]; then
|
| 1610 |
_branch="-B ${POUDRIERE_PORTS_GIT_BRANCH}"
|
|
| 1610 |
_branch="${POUDRIERE_PORTS_GIT_BRANCH}"
|
|
| 1611 | 1611 |
fi |
| 1612 | 1612 |
echo -n ">>> Creating poudriere ports tree, it may take some time... " | tee -a ${LOGFILE}
|
| 1613 |
if ! script -aq ${LOGFILE} poudriere ports -c -p "${POUDRIERE_PORTS_NAME}" -m git -U ${POUDRIERE_PORTS_GIT_URL} ${_branch} >/dev/null 2>&1; then
|
|
| 1614 |
echo "" | tee -a ${LOGFILE}
|
|
| 1615 |
echo ">>> ERROR: Error creating poudriere ports tree, aborting..." | tee -a ${LOGFILE}
|
|
| 1616 |
print_error_pfS |
|
| 1613 |
if [ "${AWS}" = 1 ]; then
|
|
| 1614 |
set -e |
|
| 1615 |
script -aq ${LOGFILE} poudriere ports -c -p "${POUDRIERE_PORTS_NAME}" -m none
|
|
| 1616 |
script -aq ${LOGFILE} zfs create ${ZFS_TANK}/poudriere/ports/${POUDRIERE_PORTS_NAME}
|
|
| 1617 |
# Download local copy of the ports tree stashed in S3 |
|
| 1618 |
echo ">>> Downloading cached copy of the ports tree from S3.." | tee -a ${LOGFILE}
|
|
| 1619 |
script -aq ${LOGFILE} env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
|
|
| 1620 |
aws s3 cp s3://pfsense-engineering-build-pkg/factory-ports.tz . --no-progress |
|
| 1621 |
script -aq ${LOGFILE} tar --strip-components 1 -xf factory-ports.tz -C /usr/local/poudriere/ports/${POUDRIERE_PORTS_NAME}
|
|
| 1622 |
# Update the ports tree |
|
| 1623 |
( |
|
| 1624 |
cd /usr/local/poudriere/ports/${POUDRIERE_PORTS_NAME}
|
|
| 1625 |
echo ">>> Updating cached copy of the ports tree from git.." | tee -a ${LOGFILE}
|
|
| 1626 |
script -aq ${LOGFILE} git pull
|
|
| 1627 |
script -aq ${LOGFILE} git checkout ${_branch}
|
|
| 1628 |
) |
|
| 1629 |
set +e |
|
| 1630 |
else |
|
| 1631 |
if ! script -aq ${LOGFILE} poudriere ports -c -p "${POUDRIERE_PORTS_NAME}" -m git -U ${POUDRIERE_PORTS_GIT_URL} -B ${_branch} >/dev/null 2>&1; then
|
|
| 1632 |
echo "" | tee -a ${LOGFILE}
|
|
| 1633 |
echo ">>> ERROR: Error creating poudriere ports tree, aborting..." | tee -a ${LOGFILE}
|
|
| 1634 |
print_error_pfS |
|
| 1635 |
fi |
|
| 1617 | 1636 |
fi |
| 1618 | 1637 |
echo "Done!" | tee -a ${LOGFILE}
|
| 1619 | 1638 |
poudriere_rename_ports |
| ... | ... | |
| 1718 | 1737 |
mkdir -p /usr/ports/distfiles |
| 1719 | 1738 |
fi |
| 1720 | 1739 |
|
| 1740 |
if [ "${AWS}" = 1 ]; then
|
|
| 1741 |
# Download a copy of the distfiles from S3 |
|
| 1742 |
echo ">>> Downloading distfile cache from S3.." | tee -a ${LOGFILE}
|
|
| 1743 |
script -aq ${LOGFILE} env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
|
|
| 1744 |
aws s3 cp s3://pfsense-engineering-build-pkg/distfiles.tar . --no-progress |
|
| 1745 |
script -aq ${LOGFILE} tar -xf distfiles.tar -C /usr/ports/distfiles
|
|
| 1746 |
# Save a list of distfiles |
|
| 1747 |
find /usr/ports/distfiles > pre-build-distfile-list |
|
| 1748 |
|
|
| 1749 |
fi |
|
| 1750 |
|
|
| 1721 | 1751 |
# Remove old jails |
| 1722 | 1752 |
for jail_arch in ${_archs}; do
|
| 1723 | 1753 |
jail_name=$(poudriere_jail_name ${jail_arch})
|
| ... | ... | |
| 1732 | 1762 |
if poudriere ports -l | grep -q -E "^${POUDRIERE_PORTS_NAME}[[:blank:]]"; then
|
| 1733 | 1763 |
echo ">>> Poudriere ports tree ${POUDRIERE_PORTS_NAME} already exists, deleting it..." | tee -a ${LOGFILE}
|
| 1734 | 1764 |
poudriere ports -d -p "${POUDRIERE_PORTS_NAME}"
|
| 1765 |
if [ "${AWS}" = 1 ]; then
|
|
| 1766 |
for d in `zfs list -o name`; do |
|
| 1767 |
if [ "${d}" = "${ZFS_TANK}/poudriere/ports/${POUDRIERE_PORTS_NAME}" ]; then
|
|
| 1768 |
script -aq ${LOGFILE} zfs destroy ${ZFS_TANK}/poudriere/ports/${POUDRIERE_PORTS_NAME}
|
|
| 1769 |
fi |
|
| 1770 |
done |
|
| 1771 |
fi |
|
| 1735 | 1772 |
fi |
| 1736 | 1773 |
|
| 1737 | 1774 |
local native_xtools="" |
| ... | ... | |
| 1745 | 1782 |
native_xtools="" |
| 1746 | 1783 |
fi |
| 1747 | 1784 |
|
| 1748 |
echo -n ">>> Creating jail ${jail_name}, it may take some time... " | tee -a ${LOGFILE}
|
|
| 1749 |
if ! script -aq ${LOGFILE} poudriere jail -c -j "${jail_name}" -v ${FREEBSD_BRANCH} \
|
|
| 1750 |
-a ${jail_arch} -m git -U ${FREEBSD_REPO_BASE_POUDRIERE} ${native_xtools} >/dev/null 2>&1; then
|
|
| 1751 |
echo "" | tee -a ${LOGFILE}
|
|
| 1752 |
echo ">>> ERROR: Error creating jail ${jail_name}, aborting..." | tee -a ${LOGFILE}
|
|
| 1753 |
print_error_pfS |
|
| 1785 |
echo ">>> Creating jail ${jail_name}, it may take some time... " | tee -a ${LOGFILE}
|
|
| 1786 |
if [ "${AWS}" = "1" ]; then
|
|
| 1787 |
mkdir objs |
|
| 1788 |
echo ">>> Downloading prebuilt release objs from s3://pfsense-engineering-build-freebsd-obj-tarballs/${FLAVOR}/ ..." | tee -a ${LOGFILE}
|
|
| 1789 |
# Download prebuilt release tarballs from previous job |
|
| 1790 |
env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
|
|
| 1791 |
aws s3 cp s3://pfsense-engineering-build-freebsd-obj-tarballs/${FLAVOR}/LATEST-${jail_arch} objs --no-progress
|
|
| 1792 |
SRC_COMMIT=`cat objs/LATEST-${jail_arch}`
|
|
| 1793 |
env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
|
|
| 1794 |
aws s3 cp s3://pfsense-engineering-build-freebsd-obj-tarballs/${FLAVOR}/MANIFEST-${jail_arch}-${SRC_COMMIT} objs --no-progress
|
|
| 1795 |
ln -s MANIFEST-${jail_arch}-${SRC_COMMIT} objs/MANIFEST
|
|
| 1796 |
for i in base doc kernel src tests; do |
|
| 1797 |
if [ ! -f objs/${i}-${jail_arch}-${SRC_COMMIT}.txz ]; then
|
|
| 1798 |
env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
|
|
| 1799 |
aws s3 cp s3://pfsense-engineering-build-freebsd-obj-tarballs/${FLAVOR}/${i}-${jail_arch}-${SRC_COMMIT}.txz objs --no-progress
|
|
| 1800 |
ln -s ${i}-${jail_arch}-${SRC_COMMIT}.txz objs/${i}.txz
|
|
| 1801 |
fi |
|
| 1802 |
done |
|
| 1803 |
|
|
| 1804 |
if ! script -aq ${LOGFILE} poudriere jail -c -j "${jail_name}" -v ${FREEBSD_BRANCH} \
|
|
| 1805 |
-a ${jail_arch} -m url=file://${PWD}/objs >/dev/null 2>&1; then
|
|
| 1806 |
echo "" | tee -a ${LOGFILE}
|
|
| 1807 |
echo ">>> ERROR: Error creating jail ${jail_name}, aborting..." | tee -a ${LOGFILE}
|
|
| 1808 |
print_error_pfS |
|
| 1809 |
fi |
|
| 1810 |
|
|
| 1811 |
# Download a cached pkg repo from S3 |
|
| 1812 |
OLDIFS=${IFS}
|
|
| 1813 |
IFS=$'\n' |
|
| 1814 |
echo ">>> Downloading cached pkgs for ${jail_arch} from S3.." | tee -a ${LOGFILE}
|
|
| 1815 |
for i in `env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
|
|
| 1816 |
aws s3 ls s3://pfsense-engineering-build-pkg/`; do |
|
| 1817 |
echo ${i} | awk '{print $4}' | grep pkgs-${jail_arch}.tar > /dev/null
|
|
| 1818 |
if [ $? -eq 0 ]; then |
|
| 1819 |
script -aq ${LOGFILE} env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
|
|
| 1820 |
aws s3 cp s3://pfsense-engineering-build-pkg/pkgs-${jail_arch}.tar . --no-progress
|
|
| 1821 |
[ ! -d /usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME} ] && mkdir -p /usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME}
|
|
| 1822 |
echo "Extracting pkgs-${jail_arch}.tar to /usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME}" | tee -a ${LOGFILE}
|
|
| 1823 |
[ ! -d /usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME} ] && mkdir /usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME}
|
|
| 1824 |
script -aq ${LOGFILE} tar -xf pkgs-${jail_arch}.tar -C /usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME}
|
|
| 1825 |
# Save a list of pkgs |
|
| 1826 |
cd /usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME}/.latest
|
|
| 1827 |
find . > ${WORKSPACE}/pre-build-pkg-list-${jail_arch}
|
|
| 1828 |
cd ${WORKSPACE}
|
|
| 1829 |
else |
|
| 1830 |
touch pre-build-pkg-list-${jail_arch}
|
|
| 1831 |
fi |
|
| 1832 |
done |
|
| 1833 |
IFS=${OLDIFS}
|
|
| 1834 |
else |
|
| 1835 |
if ! script -aq ${LOGFILE} poudriere jail -c -j "${jail_name}" -v ${FREEBSD_BRANCH} \
|
|
| 1836 |
-a ${jail_arch} -m git -U ${FREEBSD_REPO_BASE_POUDRIERE} ${native_xtools} >/dev/null 2>&1; then
|
|
| 1837 |
echo "" | tee -a ${LOGFILE}
|
|
| 1838 |
echo ">>> ERROR: Error creating jail ${jail_name}, aborting..." | tee -a ${LOGFILE}
|
|
| 1839 |
print_error_pfS |
|
| 1840 |
fi |
|
| 1754 | 1841 |
fi |
| 1755 | 1842 |
echo "Done!" | tee -a ${LOGFILE}
|
| 1756 | 1843 |
done |
| ... | ... | |
| 1811 | 1898 |
fi |
| 1812 | 1899 |
} |
| 1813 | 1900 |
|
| 1901 |
save_logs_to_s3() {
|
|
| 1902 |
# Save a copy of the past few logs into S3 |
|
| 1903 |
DATE=`date +%Y%m%d-%H%M%S` |
|
| 1904 |
script -aq ${LOGFILE} tar --zstd -cf pkg-logs-${jail_arch}-${DATE}.tar -C /usr/local/poudriere/data/logs/bulk/${jail_name}-${POUDRIERE_PORTS_NAME}/latest/ .
|
|
| 1905 |
script -aq ${LOGFILE} env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
|
|
| 1906 |
aws s3 cp pkg-logs-${jail_arch}-${DATE}.tar s3://pfsense-engineering-build-pkg/logs/ --no-progress
|
|
| 1907 |
OLDIFS=${IFS}
|
|
| 1908 |
IFS=$'\n' |
|
| 1909 |
local _logtemp=$( mktemp /tmp/loglist.XXXXX ) |
|
| 1910 |
for i in `env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
|
|
| 1911 |
aws s3 ls s3://pfsense-engineering-build-pkg/logs/`; do |
|
| 1912 |
echo ${i} | awk '{print $4}' | grep pkg-logs-${jail_arch} >> ${_logtemp}
|
|
| 1913 |
done |
|
| 1914 |
local _maxlogs=5 |
|
| 1915 |
local _curlogs=0 |
|
| 1916 |
_curlogs=$( wc -l ${_logtemp} | awk '{print $1}' )
|
|
| 1917 |
if [ ${_curlogs} -gt ${_maxlogs} ]; then
|
|
| 1918 |
local _extralogs=$(( ${_curlogs} - ${_maxlogs} ))
|
|
| 1919 |
for _last in $( head -${_extralogs} ${_logtemp} ); do
|
|
| 1920 |
env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
|
|
| 1921 |
aws s3 rm s3://pfsense-engineering-build-pkg/logs/${_last}
|
|
| 1922 |
done |
|
| 1923 |
fi |
|
| 1924 |
IFS=${OLDIFS}
|
|
| 1925 |
} |
|
| 1926 |
|
|
| 1814 | 1927 |
poudriere_bulk() {
|
| 1815 | 1928 |
local _archs=$(poudriere_possible_archs) |
| 1816 | 1929 |
local _makeconf |
| ... | ... | |
| 1921 | 2034 |
rm -f ${_bulk}.tmp ${_bulk}.exclude
|
| 1922 | 2035 |
fi |
| 1923 | 2036 |
|
| 2037 |
echo ">>> Poudriere bulk started at `date "+%Y/%m/%d %H:%M:%S"` for ${jail_arch}"
|
|
| 1924 | 2038 |
if ! poudriere bulk -f ${_bulk} -j ${jail_name} -p ${POUDRIERE_PORTS_NAME}; then
|
| 1925 | 2039 |
echo ">>> ERROR: Something went wrong..." |
| 2040 |
if [ "${AWS}" = 1 ]; then
|
|
| 2041 |
save_logs_to_s3 |
|
| 2042 |
fi |
|
| 1926 | 2043 |
print_error_pfS |
| 1927 | 2044 |
fi |
| 2045 |
echo ">>> Poudriere bulk complated at `date "+%Y/%m/%d %H:%M:%S"` for ${jail_arch}"
|
|
| 1928 | 2046 |
|
| 1929 | 2047 |
echo ">>> Cleaning up old packages from repo..." |
| 1930 | 2048 |
if ! poudriere pkgclean -f ${_bulk} -j ${jail_name} -p ${POUDRIERE_PORTS_NAME} -y; then
|
| ... | ... | |
| 1932 | 2050 |
print_error_pfS |
| 1933 | 2051 |
fi |
| 1934 | 2052 |
|
| 2053 |
if [ "${AWS}" = 1 ]; then
|
|
| 2054 |
echo ">>> Save a copy of the package repo into S3..." | tee -a ${LOGFILE}
|
|
| 2055 |
cd /usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME}/.latest
|
|
| 2056 |
find . > ${WORKSPACE}/post-build-pkg-list-${jail_arch}
|
|
| 2057 |
cd ${WORKSPACE}
|
|
| 2058 |
diff pre-build-pkg-list-${jail_arch} post-build-pkg-list-${jail_arch} > /dev/null
|
|
| 2059 |
if [ $? = 1 ]; then |
|
| 2060 |
[ -f pkgs-${jail_arch}.tar ] && rm pkgs-${jail_arch}.tar
|
|
| 2061 |
script -aq ${LOGFILE} tar -cf pkgs-${jail_arch}.tar -C /usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME} .
|
|
| 2062 |
script -aq ${LOGFILE} env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
|
|
| 2063 |
aws s3 cp pkgs-${jail_arch}.tar s3://pfsense-engineering-build-pkg/ --no-progress
|
|
| 2064 |
|
|
| 2065 |
save_logs_to_s3 |
|
| 2066 |
fi |
|
| 2067 |
fi |
|
| 2068 |
|
|
| 1935 | 2069 |
pkg_repo_rsync "/usr/local/poudriere/data/packages/${jail_name}-${POUDRIERE_PORTS_NAME}"
|
| 1936 | 2070 |
done |
| 2071 |
|
|
| 2072 |
if [ "${AWS}" = 1 ]; then
|
|
| 2073 |
echo ">>> Save a copy of the distfiles into S3..." | tee -a ${LOGFILE}
|
|
| 2074 |
# Save a copy of the distfiles from S3 |
|
| 2075 |
find /usr/ports/distfiles > post-build-distfile-list |
|
| 2076 |
diff pre-build-distfile-list post-build-distfile-list > /dev/null |
|
| 2077 |
if [ $? -eq 1 ]; then |
|
| 2078 |
rm distfiles.tar |
|
| 2079 |
script -aq ${LOGFILE} tar -cf distfiles.tar -C /usr/ports/distfiles .
|
|
| 2080 |
script -aq ${LOGFILE} env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_DEFAULT_REGION=us-east-2 \
|
|
| 2081 |
aws s3 cp distfiles.tar s3://pfsense-engineering-build-pkg/ --no-progress |
|
| 2082 |
fi |
|
| 2083 |
fi |
|
| 1937 | 2084 |
} |
| 1938 | 2085 |
|
| 1939 | 2086 |
# This routine is called to write out to stdout |
Also available in: Unified diff
Build improvements for using AWS: