Project

General

Profile

« Previous | Next » 

Revision cafd9f97

Added by Brad Davis over 3 years ago

Add support for multiple sites for syncing packages to

Each site must be defined in the PKG_RSYNC_HOSTS variable and then a
PKG_RSYNC_HOSTNAME_$site must be defined as the target, for example:

PKG_RSYNC_HOSTS="nyi"
PKG_RSYNC_HOSTNAME_nyi="nfs1.nyi.netgate.com"

The final rsync hostname can be overridden as
PKG_FINAL_RSYNC_HOSTNAME_$site or will inherit from
PKG_RSYNC_HOSTNAME_$site.

View differences:

tools/builder_common.sh
1396 1396 
		return
1397 1397 
	fi
1398 1398 

  		




  1399
  
  
    
	for _pkg_rsync_hostname in ${PKG_RSYNC_HOSTNAME}; do


  		




  
  1399
  
    
	local _pkg_rsync_site


  		




  
  1400
  
    
	for _pkg_rsync_site in ${PKG_RSYNC_HOSTS}; do


  		




  
  1401
  
    
		eval _pkg_rsync_hostname=\$PKG_RSYNC_HOSTNAME$_pkg_rsync_site


  		




  
  1402
  
    
		if [ -z "${_pkg_rsync_hostname}" ]; then


  		




  
  1403
  
    
			echo "PKG_RSYNC_HOSTNAME$_pkg_rsync_site is empty, skipping.."


  		




  
  1404
  
    
			continue


  		




  
  1405
  
    
		fi


  		




  1400
  1406
  
    
		# Make sure destination directory exist


  		




  1401
  1407
  
    
		ssh -o StrictHostKeyChecking=no -p ${PKG_RSYNC_SSH_PORT} \


  		




  1402
  1408
  
    
			${PKG_RSYNC_USERNAME}@${_pkg_rsync_hostname} \


  		




  ......




  1419
  1425
  
    
		fi


  		




  1420
  1426
  
    

  		




  1421
  1427
  
    
		if [ -n "${_IS_RELEASE}" -o "${_repo_path_param}" = "${CORE_PKG_PATH}" ]; then


  		




  1422
  
  
    
			for _pkg_final_rsync_hostname in ${PKG_FINAL_RSYNC_HOSTNAME}; do


  		




  1423
  
  
    
				# Send .real* directories first to prevent having a broken repo while transfer happens


  		




  1424
  
  
    
				local _cmd="rsync -Have \"ssh -o StrictHostKeyChecking=no -p ${PKG_FINAL_RSYNC_SSH_PORT}\" \


  		




  1425
  
  
    
					--timeout=60 ${PKG_RSYNC_DESTDIR}/./${_repo_base%%-core}* \


  		




  1426
  
  
    
					--include=\"/*\" --include=\"*/.real*\" --include=\"*/.real*/***\" \


  		




  1427
  
  
    
					--exclude=\"*\" \


  		




  1428
  
  
    
					${PKG_FINAL_RSYNC_USERNAME}@${_pkg_final_rsync_hostname}:${PKG_FINAL_RSYNC_DESTDIR}"


  		




  1429
  
  
    

  		




  1430
  
  
    
				echo -n ">>> Sending updated packages to ${_pkg_final_rsync_hostname}... " | tee -a ${_logfile}


  		




  1431
  
  
    
				if script -aq ${_logfile} ssh -o StrictHostKeyChecking=no -p ${PKG_RSYNC_SSH_PORT} \


  		




  1432
  
  
    
					${PKG_RSYNC_USERNAME}@${_pkg_rsync_hostname} ${_cmd} >> ${BUILDER_LOGS}/rsync.log 2>&1; then


  		




  
  1428
  
    
			local _pkg_final_rsync_hostname


  		




  
  1429
  
    
			eval _pkg_final_rsync_hostname=\$PKG_FINAL_RSYNC_HOSTNAME$_pkg_rsync_site


  		




  
  1430
  
    
			if [ -z "${_pkg_final_rsync_hostname}" ]; then


  		




  
  1431
  
    
				_pkg_final_rsync_hostname="$_pkg_rsync_hostname"


  		




  
  1432
  
    
			fi


  		




  
  1433
  
    

  		




  
  1434
  
    
			# Send .real* directories first to prevent having a broken repo while transfer happens


  		




  
  1435
  
    
			local _cmd="rsync -Have \"ssh -o StrictHostKeyChecking=no -p ${PKG_FINAL_RSYNC_SSH_PORT}\" \


  		




  
  1436
  
    
				--timeout=60 ${PKG_RSYNC_DESTDIR}/./${_repo_base%%-core}* \


  		




  
  1437
  
    
				--include=\"/*\" --include=\"*/.real*\" --include=\"*/.real*/***\" \


  		




  
  1438
  
    
				--exclude=\"*\" \


  		




  
  1439
  
    
				${PKG_FINAL_RSYNC_USERNAME}@${_pkg_final_rsync_hostname}:${PKG_FINAL_RSYNC_DESTDIR}"


  		




  
  1440
  
    

  		




  
  1441
  
    
			echo -n ">>> Sending updated packages to ${_pkg_final_rsync_hostname}... " | tee -a ${_logfile}


  		




  
  1442
  
    
			if script -aq ${_logfile} ssh -o StrictHostKeyChecking=no -p ${PKG_RSYNC_SSH_PORT} \


  		




  
  1443
  
    
				${PKG_RSYNC_USERNAME}@${_pkg_rsync_hostname} ${_cmd} >> ${BUILDER_LOGS}/rsync.log 2>&1; then


  		




  
  1444
  
    
				echo "Done!" | tee -a ${_logfile}


  		




  
  1445
  
    
			else


  		




  
  1446
  
    
				echo "Failed!" | tee -a ${_logfile}


  		




  
  1447
  
    
				echo ">>> ERROR: An error occurred sending repo to final hostname"


  		




  
  1448
  
    
				print_error_pfS


  		




  
  1449
  
    
			fi


  		




  
  1450
  
    

  		




  
  1451
  
    
			_cmd="rsync -Have \"ssh -o StrictHostKeyChecking=no -p ${PKG_FINAL_RSYNC_SSH_PORT}\" \


  		




  
  1452
  
    
				--timeout=60 --delete-delay ${PKG_RSYNC_DESTDIR}/./${_repo_base%%-core}* \


  		




  
  1453
  
    
				${PKG_FINAL_RSYNC_USERNAME}@${_pkg_final_rsync_hostname}:${PKG_FINAL_RSYNC_DESTDIR}"


  		




  
  1454
  
    

  		




  
  1455
  
    
			echo -n ">>> Sending updated repositories metadata to ${_pkg_final_rsync_hostname}... " | tee -a ${_logfile}


  		




  
  1456
  
    
			if script -aq ${_logfile} ssh -o StrictHostKeyChecking=no -p ${PKG_RSYNC_SSH_PORT} \


  		




  
  1457
  
    
				${PKG_RSYNC_USERNAME}@${_pkg_rsync_hostname} ${_cmd} >> ${BUILDER_LOGS}/rsync.log 2>&1; then


  		




  
  1458
  
    
				echo "Done!" | tee -a ${_logfile}


  		




  
  1459
  
    
			else


  		




  
  1460
  
    
				echo "Failed!" | tee -a ${_logfile}


  		




  
  1461
  
    
				echo ">>> ERROR: An error occurred sending repo to final hostname"


  		




  
  1462
  
    
				print_error_pfS


  		




  
  1463
  
    
			fi


  		




  
  1464
  
    

  		




  
  1465
  
    
			if [ -z "${PKG_FINAL_S3_PATH}" ]; then


  		




  
  1466
  
    
				continue


  		




  
  1467
  
    
			fi


  		




  
  1468
  
    

  		




  
  1469
  
    
			local _repos=$(ssh -o StrictHostKeyChecking=no -p ${PKG_FINAL_RSYNC_SSH_PORT} \


  		




  
  1470
  
    
			    ${PKG_FINAL_RSYNC_USERNAME}@${_pkg_final_rsync_hostname} \


  		




  
  1471
  
    
			    "ls -1d ${PKG_FINAL_RSYNC_DESTDIR}/${_repo_base%%-core}*")


  		




  
  1472
  
    
			for _repo in ${_repos}; do


  		




  
  1473
  
    
				echo -n ">>> Sending updated packages to AWS ${PKG_FINAL_S3_PATH}... " | tee -a ${_logfile}


  		




  
  1474
  
    
				if script -aq ${_logfile} ssh -o StrictHostKeyChecking=no -p ${PKG_FINAL_RSYNC_SSH_PORT} \


  		




  
  1475
  
    
				    ${PKG_FINAL_RSYNC_USERNAME}@${_pkg_final_rsync_hostname} \


  		




  
  1476
  
    
				    "${_aws_sync_cmd} ${_repo} ${PKG_FINAL_S3_PATH}/$(basename ${_repo})"; then


  		




  1433
  1477
  
    
					echo "Done!" | tee -a ${_logfile}


  		




  1434
  1478
  
    
				else


  		




  1435
  1479
  
    
					echo "Failed!" | tee -a ${_logfile}


  		




  1436
  
  
    
					echo ">>> ERROR: An error occurred sending repo to final hostname"


  		




  
  1480
  
    
					echo ">>> ERROR: An error occurred sending files to AWS S3"


  		




  1437
  1481
  
    
					print_error_pfS


  		




  1438
  1482
  
    
				fi


  		




  1439
  
  
    

  		




  1440
  
  
    
				_cmd="rsync -Have \"ssh -o StrictHostKeyChecking=no -p ${PKG_FINAL_RSYNC_SSH_PORT}\" \


  		




  1441
  
  
    
					--timeout=60 --delete-delay ${PKG_RSYNC_DESTDIR}/./${_repo_base%%-core}* \


  		




  1442
  
  
    
					${PKG_FINAL_RSYNC_USERNAME}@${_pkg_final_rsync_hostname}:${PKG_FINAL_RSYNC_DESTDIR}"


  		




  1443
  
  
    

  		




  1444
  
  
    
				echo -n ">>> Sending updated repositories metadata to ${_pkg_final_rsync_hostname}... " | tee -a ${_logfile}


  		




  1445
  
  
    
				if script -aq ${_logfile} ssh -o StrictHostKeyChecking=no -p ${PKG_RSYNC_SSH_PORT} \


  		




  1446
  
  
    
					${PKG_RSYNC_USERNAME}@${_pkg_rsync_hostname} ${_cmd} >> ${BUILDER_LOGS}/rsync.log 2>&1; then


  		




  
  1483
  
    
				echo -n ">>> Cleaning up packages at AWS ${PKG_FINAL_S3_PATH}... " | tee -a ${_logfile}


  		




  
  1484
  
    
				if script -aq ${_logfile} ssh -o StrictHostKeyChecking=no -p ${PKG_FINAL_RSYNC_SSH_PORT} \


  		




  
  1485
  
    
				    ${PKG_FINAL_RSYNC_USERNAME}@${_pkg_final_rsync_hostname} \


  		




  
  1486
  
    
				    "${_aws_sync_cmd} --delete ${_repo} ${PKG_FINAL_S3_PATH}/$(basename ${_repo})"; then


  		




  1447
  1487
  
    
					echo "Done!" | tee -a ${_logfile}


  		




  1448
  1488
  
    
				else


  		




  1449
  1489
  
    
					echo "Failed!" | tee -a ${_logfile}


  		




  1450
  
  
    
					echo ">>> ERROR: An error occurred sending repo to final hostname"


  		




  
  1490
  
    
					echo ">>> ERROR: An error occurred sending files to AWS S3"


  		




  1451
  1491
  
    
					print_error_pfS


  		




  1452
  1492
  
    
				fi


  		




  1453
  
  
    

  		




  1454
  
  
    
				if [ -z "${PKG_FINAL_S3_PATH}" ]; then


  		




  1455
  
  
    
					continue


  		




  1456
  
  
    
				fi


  		




  1457
  
  
    

  		




  1458
  
  
    
				local _repos=$(ssh -o StrictHostKeyChecking=no -p ${PKG_FINAL_RSYNC_SSH_PORT} \


  		




  1459
  
  
    
				    ${PKG_FINAL_RSYNC_USERNAME}@${_pkg_final_rsync_hostname} \


  		




  1460
  
  
    
				    "ls -1d ${PKG_FINAL_RSYNC_DESTDIR}/${_repo_base%%-core}*")


  		




  1461
  
  
    
				for _repo in ${_repos}; do


  		




  1462
  
  
    
					echo -n ">>> Sending updated packages to AWS ${PKG_FINAL_S3_PATH}... " | tee -a ${_logfile}


  		




  1463
  
  
    
					if script -aq ${_logfile} ssh -o StrictHostKeyChecking=no -p ${PKG_FINAL_RSYNC_SSH_PORT} \


  		




  1464
  
  
    
					    ${PKG_FINAL_RSYNC_USERNAME}@${_pkg_final_rsync_hostname} \


  		




  1465
  
  
    
					    "${_aws_sync_cmd} ${_repo} ${PKG_FINAL_S3_PATH}/$(basename ${_repo})"; then


  		




  1466
  
  
    
						echo "Done!" | tee -a ${_logfile}


  		




  1467
  
  
    
					else


  		




  1468
  
  
    
						echo "Failed!" | tee -a ${_logfile}


  		




  1469
  
  
    
						echo ">>> ERROR: An error occurred sending files to AWS S3"


  		




  1470
  
  
    
						print_error_pfS


  		




  1471
  
  
    
					fi


  		




  1472
  
  
    
					echo -n ">>> Cleaning up packages at AWS ${PKG_FINAL_S3_PATH}... " | tee -a ${_logfile}


  		




  1473
  
  
    
					if script -aq ${_logfile} ssh -o StrictHostKeyChecking=no -p ${PKG_FINAL_RSYNC_SSH_PORT} \


  		




  1474
  
  
    
					    ${PKG_FINAL_RSYNC_USERNAME}@${_pkg_final_rsync_hostname} \


  		




  1475
  
  
    
					    "${_aws_sync_cmd} --delete ${_repo} ${PKG_FINAL_S3_PATH}/$(basename ${_repo})"; then


  		




  1476
  
  
    
						echo "Done!" | tee -a ${_logfile}


  		




  1477
  
  
    
					else


  		




  1478
  
  
    
						echo "Failed!" | tee -a ${_logfile}


  		




  1479
  
  
    
						echo ">>> ERROR: An error occurred sending files to AWS S3"


  		




  1480
  
  
    
						print_error_pfS


  		




  1481
  
  
    
					fi


  		




  1482
  
  
    
				done


  		




  1483
  1493
  
    
			done


  		




  1484
  1494
  
    
		fi


  		




  1485
  1495
  
    
	done


  		












  

    
      tools/builder_defaults.sh
    
  





  267
  267
  
    
STAGING_HOSTNAME=${STAGING_HOSTNAME:-"release-staging.nyi.netgate.com"}


  		




  268
  268
  
    

  		




  269
  269
  
    
# Host to rsync pkg repos from poudriere


  		




  270
  
  
    
export PKG_RSYNC_HOSTNAME=${PKG_RSYNC_HOSTNAME:-"nfs1.nyi.netgate.com"}


  		




  
  270
  
    
export PKG_RSYNC_HOSTS=${PKG_RSYNC_HOSTS:-"nyi"}


  		




  
  271
  
    
export PKG_RSYNC_HOSTNAME_nyi=${PKG_RSYNC_HOSTNAME_nyi:-"nfs1.nyi.netgate.com"}


  		




  271
  272
  
    
export PKG_RSYNC_USERNAME=${PKG_RSYNC_USERNAME:-"wwwsync"}


  		




  272
  273
  
    
export PKG_RSYNC_SSH_PORT=${PKG_RSYNC_SSH_PORT:-"22"}


  		




  273
  274
  
    
export PKG_RSYNC_DESTDIR=${PKG_RSYNC_DESTDIR:-"/storage/files/release-staging/ce/packages"}


  		




  274
  275
  
    

  		




  275
  276
  
    
# Final packages server


  		




  276
  277
  
    
if [ -n "${_IS_RELEASE}" -o -n "${_IS_RC}" ]; then


  		




  277
  
  
    
	export PKG_FINAL_RSYNC_HOSTNAME=${PKG_FINAL_RSYNC_HOSTNAME:-"nfs1.nyi.netgate.com"}


  		




  
  278
  
    
	export PKG_FINAL_RSYNC_HOSTNAME_nyi=${PKG_FINAL_RSYNC_HOSTNAME_nyi:-"nfs1.nyi.netgate.com"}


  		




  278
  279
  
    
	export PKG_FINAL_RSYNC_DESTDIR=${PKG_FINAL_RSYNC_DESTDIR:-"/storage/files/pkg"}


  		




  279
  280
  
    
else


  		




  280
  
  
    
	export PKG_FINAL_RSYNC_HOSTNAME=${PKG_FINAL_RSYNC_HOSTNAME:-"nfs1.nyi.netgate.com"}


  		




  
  281
  
    
	export PKG_FINAL_RSYNC_HOSTNAME_nyi=${PKG_FINAL_RSYNC_HOSTNAME_nyi:-"nfs1.nyi.netgate.com"}


  		




  281
  282
  
    
	export PKG_FINAL_RSYNC_DESTDIR=${PKG_FINAL_RSYNC_DESTDIR:-"/storage/files/beta/packages"}


  		




  282
  283
  
    
fi


  		




  283
  284
  
    
export PKG_FINAL_RSYNC_USERNAME=${PKG_FINAL_RSYNC_USERNAME:-"wwwsync"}


  		












Also available in:  Unified diff