Project

General

Profile

Download (11.1 KB) Statistics
| Branch: | Tag: | Revision:
1 9020723c Renato Botelho do Couto
#!/bin/sh -T
2 6f73c362 Renato Botelho
#
3
# build.sh
4
#
5 ac24dc24 Renato Botelho
# part of pfSense (https://www.pfsense.org)
6 38809d47 Renato Botelho do Couto
# Copyright (c) 2004-2013 BSD Perimeter
7
# Copyright (c) 2013-2016 Electric Sheep Fencing
8 8f585441 Luiz Souza
# Copyright (c) 2014-2021 Rubicon Communications, LLC (Netgate)
9 ac24dc24 Renato Botelho
# All rights reserved.
10 6f73c362 Renato Botelho
#
11 b12ea3fb Renato Botelho
# Licensed under the Apache License, Version 2.0 (the "License");
12
# you may not use this file except in compliance with the License.
13
# You may obtain a copy of the License at
14 6f73c362 Renato Botelho
#
15 b12ea3fb Renato Botelho
# http://www.apache.org/licenses/LICENSE-2.0
16 6f73c362 Renato Botelho
#
17 b12ea3fb Renato Botelho
# Unless required by applicable law or agreed to in writing, software
18
# distributed under the License is distributed on an "AS IS" BASIS,
19
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20
# See the License for the specific language governing permissions and
21
# limitations under the License.
22 6f73c362 Renato Botelho
23
set +e
24
usage() {
25 2ac4be3a Renato Botelho
	echo "Usage $0 [options] [ iso | ova | memstick | memstickserial | memstickadi | all | none ]"
26 70350cb1 Renato Botelho
	echo "		all = memstick memstickserial memstickadi"
27 ddf56629 Renato Botelho
	echo "		none = upgrade only pkg repo"
28 6f73c362 Renato Botelho
	echo "	[ options ]: "
29
	echo "		--no-buildworld|-c - Will set NO_BUILDWORLD NO_BUILDKERNEL to not build kernel and world"
30 6ebd280c Renato Botelho
	echo "		--no-cleanobjdir|-d - Will not clean FreeBSD object built dir to allow restarting a build with NO_CLEAN"
31 6f73c362 Renato Botelho
	echo "		--resume-image-build|-r - Includes -c -d and also will just move directly to image creation using pre-staged data"
32
	echo "		--setup - Install required repo and ports builder require to work"
33 2a5914c6 Renato Botelho
	echo "		--update-sources - Refetch FreeBSD sources"
34 0458a1fd Renato Botelho
	echo "		--rsync-repos - rsync pkg repos"
35 e48f0418 Renato Botelho
	echo "		--rsync-snapshots - rsync snapshots images and pkg repos"
36 6f73c362 Renato Botelho
	echo "		--clean-builder - clean all builder used data/resources"
37
	echo "		--build-kernels - build all configured kernels"
38
	echo "		--build-kernel argument - build specified kernel. Example --build-kernel KERNEL_NAME"
39
	echo "		--install-extra-kernels argument - Put extra kernel(s) under /kernel image directory. Example --install-extra-kernels KERNEL_NAME_WRAP"
40 f6c83d05 Renato Botelho
	echo "		--snapshots - Build snapshots"
41 919c8486 Renato Botelho
	echo "		--poudriere-snapshots - Update poudriere packages and send them to PKG_RSYNC_HOSTNAME"
42 6f73c362 Renato Botelho
	echo "		--setup-poudriere - Install poudriere and create necessary jails and ports tree"
43
	echo "		--create-unified-patch - Create a big patch with all changes done on FreeBSD"
44
	echo "		--update-poudriere-jails [-a ARCH_LIST] - Update poudriere jails using current patch versions"
45 51796ced Luiz Souza
	echo "		--update-poudriere-ports [-a ARCH_LIST] - Update poudriere ports tree"
46 fb192227 Renato Botelho
	echo "		--update-pkg-repo [-a ARCH_LIST]- Rebuild necessary ports on poudriere and update pkg repo"
47 f6c83d05 Renato Botelho
	echo "		--upload|-U - Upload pkgs and/or snapshots"
48 158999e9 Renato Botelho
	echo "		--skip-final-rsync|-i - Skip rsync to final server"
49 6f73c362 Renato Botelho
	echo "		-V VARNAME - print value of variable VARNAME"
50
	exit 1
51
}
52
53 0ce2a7f3 Renato Botelho
export BUILDER_ROOT=$(realpath $(dirname ${0}))
54
export BUILDER_TOOLS="${BUILDER_ROOT}/tools"
55 6f73c362 Renato Botelho
56
unset _SKIP_REBUILD_PRESTAGE
57
unset _USE_OLD_DATESTRING
58
unset pfPORTTOBUILD
59
unset IMAGETYPE
60 f6c83d05 Renato Botelho
unset UPLOAD
61 ab943fc9 Renato Botelho
unset SNAPSHOTS
62 919c8486 Renato Botelho
unset POUDRIERE_SNAPSHOTS
63 fb192227 Renato Botelho
unset ARCH_LIST
64 6f73c362 Renato Botelho
BUILDACTION="images"
65
66
# Maybe use options for nocleans etc?
67
while test "$1" != ""; do
68
	case "${1}" in
69
		--no-buildworld|-c)
70
			export NO_BUILDWORLD=YES
71
			export NO_BUILDKERNEL=YES
72
			;;
73 6ebd280c Renato Botelho
		--no-cleanobjdir|-d)
74 2abc3e80 Renato Botelho
			export NO_CLEAN_FREEBSD_OBJ=YES
75 6f73c362 Renato Botelho
			;;
76
		--resume-image-build|-r)
77
			export NO_BUILDWORLD=YES
78
			export NO_BUILDKERNEL=YES
79 2abc3e80 Renato Botelho
			export NO_CLEAN_FREEBSD_OBJ=YES
80 2f666d34 Renato Botelho
			export DO_NOT_SIGN_PKG_REPO=YES
81 6f73c362 Renato Botelho
			_SKIP_REBUILD_PRESTAGE=YES
82
			_USE_OLD_DATESTRING=YES
83
			;;
84
		--setup)
85
			BUILDACTION="builder_setup"
86
			;;
87 0458a1fd Renato Botelho
		--rsync-repos)
88
			BUILDACTION="rsync_repos"
89 2f20d4dc Renato Botelho
			export DO_NOT_SIGN_PKG_REPO=YES
90 0458a1fd Renato Botelho
			;;
91 e48f0418 Renato Botelho
		--rsync-snapshots)
92
			BUILDACTION="rsync_snapshots"
93
			export DO_NOT_SIGN_PKG_REPO=YES
94
			;;
95 6f73c362 Renato Botelho
		--build-kernels)
96
			BUILDACTION="buildkernels"
97
			;;
98
		--install-extra-kernels)
99
			shift
100
			if [ $# -eq 0 ]; then
101
				echo "--build-kernel needs extra parameter."
102
				echo
103
				usage
104
			fi
105
			export INSTALL_EXTRA_KERNELS="${1}"
106
			;;
107 ab943fc9 Renato Botelho
		--snapshots)
108
			export SNAPSHOTS=1
109
			;;
110 919c8486 Renato Botelho
		--poudriere-snapshots)
111
			export POUDRIERE_SNAPSHOTS=1
112
			;;
113 6f73c362 Renato Botelho
		--build-kernel)
114
			BUILDACTION="buildkernel"
115
			shift
116
			if [ $# -eq 0 ]; then
117
				echo "--build-kernel needs extra parameter."
118
				echo
119
				usage
120
			fi
121
			export BUILD_KERNELS="${1}"
122
			;;
123
		--update-sources)
124
			BUILDACTION="updatesources"
125
			;;
126
		--clean-builder)
127
			BUILDACTION="cleanbuilder"
128
			;;
129
		--setup-poudriere)
130
			BUILDACTION="setup_poudriere"
131
			;;
132
		--create-unified-patch)
133
			BUILDACTION="create_unified_patch"
134
			;;
135
		--update-poudriere-jails)
136
			BUILDACTION="update_poudriere_jails"
137 fb192227 Renato Botelho
			;;
138
		-a)
139 6f73c362 Renato Botelho
			shift
140 fb192227 Renato Botelho
			if [ $# -eq 0 ]; then
141
				echo "-a needs extra parameter."
142
				echo
143
				usage
144 6f73c362 Renato Botelho
			fi
145 fb192227 Renato Botelho
			export ARCH_LIST="${1}"
146 6f73c362 Renato Botelho
			;;
147
		--update-poudriere-ports)
148
			BUILDACTION="update_poudriere_ports"
149
			;;
150
		--update-pkg-repo)
151
			BUILDACTION="update_pkg_repo"
152
			;;
153 f6c83d05 Renato Botelho
		--upload|-U)
154
			export UPLOAD=1
155 6f73c362 Renato Botelho
			;;
156 158999e9 Renato Botelho
		--skip-final-rsync|-i)
157
			export SKIP_FINAL_RSYNC=1
158
			;;
159 2ac4be3a Renato Botelho
		all|none|*iso*|*ova*|*memstick*|*memstickserial*|*memstickadi*)
160 6f73c362 Renato Botelho
			BUILDACTION="images"
161
			IMAGETYPE="${1}"
162
			;;
163
		-V)
164 3ad375d4 Renato Botelho
			_USE_OLD_DATESTRING=YES
165 6f73c362 Renato Botelho
			shift
166
			[ -n "${1}" ] \
167
				&& var_to_print="${1}"
168
			;;
169 ab943fc9 Renato Botelho
		--snapshot-update-status)
170
			shift
171 906ca3b0 Renato Botelho
			snapshot_status_message="${1}"
172
			BUILDACTION="snapshot_status_message"
173 b8e4e210 Renato Botelho
			_USE_OLD_DATESTRING=YES
174 ab943fc9 Renato Botelho
			;;
175 6f73c362 Renato Botelho
		*)
176 33a44737 Renato Botelho
			_USE_OLD_DATESTRING=YES
177 6f73c362 Renato Botelho
			usage
178
	esac
179
	shift
180
done
181
182 5eb2c7c5 Renato Botelho
# Suck in local vars
183
. ${BUILDER_TOOLS}/builder_defaults.sh
184
185 21835143 Renato Botelho
# Let user define ARCH_LIST in build.conf
186
[ -z "${ARCH_LIST}" -a -n "${DEFAULT_ARCH_LIST}" ] \
187
	&& ARCH_LIST="${DEFAULT_ARCH_LIST}"
188
189 6f73c362 Renato Botelho
# Suck in script helper functions
190 0ce2a7f3 Renato Botelho
. ${BUILDER_TOOLS}/builder_common.sh
191 6f73c362 Renato Botelho
192
# Print var required with -V and exit
193
if [ -n "${var_to_print}"  ]; then
194
	eval "echo \$${var_to_print}"
195
	exit 0
196
fi
197
198 ab943fc9 Renato Botelho
# Update snapshot status and exit
199 906ca3b0 Renato Botelho
if [ "${BUILDACTION}" = "snapshot_status_message" ]; then
200 919c8486 Renato Botelho
	if [ -z "${POUDRIERE_SNAPSHOTS}" ]; then
201
		export SNAPSHOTS=1
202
	fi
203 ab943fc9 Renato Botelho
	snapshots_update_status "${snapshot_status_message}"
204
	exit 0
205
fi
206
207 6f73c362 Renato Botelho
# This should be run first
208
launch
209
210
case $BUILDACTION in
211
	builder_setup)
212
		builder_setup
213
	;;
214
	buildkernels)
215
		update_freebsd_sources
216
		build_all_kernels
217
	;;
218
	buildkernel)
219
		update_freebsd_sources
220
		build_all_kernels
221
	;;
222
	cleanbuilder)
223 7e6ab3ed Renato Botelho
		clean_builder
224 6f73c362 Renato Botelho
	;;
225 dd423357 Renato Botelho
	images)
226 6f73c362 Renato Botelho
		# It will be handled below
227
	;;
228
	updatesources)
229 dde49bef Renato Botelho
		update_freebsd_sources
230 6f73c362 Renato Botelho
	;;
231
	setup_poudriere)
232
		poudriere_init
233
	;;
234
	create_unified_patch)
235
		poudriere_create_patch
236
	;;
237
	update_poudriere_jails)
238
		poudriere_update_jails
239
	;;
240
	update_poudriere_ports)
241
		poudriere_update_ports
242
	;;
243 0458a1fd Renato Botelho
	rsync_repos)
244 d610e4ba Renato Botelho
		export UPLOAD=1
245 0458a1fd Renato Botelho
		pkg_repo_rsync "${CORE_PKG_PATH}"
246
	;;
247 e48f0418 Renato Botelho
	rsync_snapshots)
248 d610e4ba Renato Botelho
		export UPLOAD=1
249 e48f0418 Renato Botelho
		snapshots_scp_files
250
	;;
251 6f73c362 Renato Botelho
	update_pkg_repo)
252 f6c83d05 Renato Botelho
		if [ -n "${UPLOAD}" -a ! -f /usr/local/bin/rsync ]; then
253 cfa7f409 Renato Botelho
			echo "ERROR: rsync is not installed, aborting..."
254
			exit 1
255
		fi
256 6f73c362 Renato Botelho
		poudriere_bulk
257
	;;
258
	*)
259
		usage
260
	;;
261
esac
262
263
if [ "${BUILDACTION}" != "images" ]; then
264
	finish
265
	exit 0
266
fi
267
268 f6c83d05 Renato Botelho
if [ -n "${SNAPSHOTS}" -a -n "${UPLOAD}" ]; then
269 49784c55 Renato Botelho
	_required=" \
270
		RSYNCIP \
271
		RSYNCUSER \
272
		RSYNCPATH \
273
		PKG_RSYNC_HOSTNAME \
274
		PKG_RSYNC_USERNAME \
275
		PKG_RSYNC_SSH_PORT \
276
		PKG_RSYNC_DESTDIR \
277 74a4eefb Renato Botelho
		PKG_REPO_SERVER_DEVEL \
278
		PKG_REPO_SERVER_RELEASE \
279
		PKG_REPO_SERVER_STAGING \
280
		PKG_REPO_BRANCH_DEVEL \
281
		PKG_REPO_BRANCH_RELEASE"
282 49784c55 Renato Botelho
283 c6d14b0b Renato Botelho
	for _var in ${_required}; do
284 f238a83f Renato Botelho
		eval "_value=\${$_var}"
285
		if [ -z "${_value}" ]; then
286
			echo ">>> ERROR: ${_var} is not defined"
287
			exit 1
288
		fi
289
	done
290 cfa7f409 Renato Botelho
291
	if [ ! -f /usr/local/bin/rsync ]; then
292
		echo "ERROR: rsync is not installed, aborting..."
293
		exit 1
294
	fi
295 ab943fc9 Renato Botelho
fi
296
297 6f73c362 Renato Botelho
if [ $# -gt 1 ]; then
298
	echo "ERROR: Too many arguments given."
299
	echo
300
	usage
301
fi
302 3edb445e Renato Botelho
303
if [ -n "${SNAPSHOTS}" -a -z "${IMAGETYPE}" ]; then
304
	IMAGETYPE="all"
305
fi
306
307 6f73c362 Renato Botelho
if [ -z "${IMAGETYPE}" ]; then
308
	echo "ERROR: Need to specify image type to build."
309
	echo
310
	usage
311
fi
312
313 ddf56629 Renato Botelho
if [ "$IMAGETYPE" = "none" ]; then
314
	_IMAGESTOBUILD=""
315
elif [ "$IMAGETYPE" = "all" ]; then
316 70350cb1 Renato Botelho
	_IMAGESTOBUILD="memstick memstickserial"
317 6f73c362 Renato Botelho
	if [ "${TARGET}" = "amd64" ]; then
318
		_IMAGESTOBUILD="${_IMAGESTOBUILD} memstickadi"
319 7731df41 Renato Botelho
		if [ -n "${_IS_RELEASE}"  ]; then
320
			_IMAGESTOBUILD="${_IMAGESTOBUILD} ova"
321
		fi
322 6f73c362 Renato Botelho
	fi
323
else
324
	_IMAGESTOBUILD="${IMAGETYPE}"
325
fi
326
327
echo ">>> Building image type(s): ${_IMAGESTOBUILD}"
328
329 ab943fc9 Renato Botelho
if [ -n "${SNAPSHOTS}" ]; then
330
	snapshots_update_status ">>> Starting snapshot build operations"
331
332
	if pkg update -r ${PRODUCT_NAME} >/dev/null 2>&1; then
333
		snapshots_update_status ">>> Updating builder packages... "
334
		pkg upgrade -r ${PRODUCT_NAME} -y -q >/dev/null 2>&1
335
	fi
336
fi
337
338 6f73c362 Renato Botelho
if [ -z "${_SKIP_REBUILD_PRESTAGE}" ]; then
339
	[ -n "${CORE_PKG_PATH}" -a -d "${CORE_PKG_PATH}" ] \
340
		&& rm -rf ${CORE_PKG_PATH}
341
342
	# Cleanup environment before start
343 7e6ab3ed Renato Botelho
	clean_builder
344 6f73c362 Renato Botelho
345
	# Make sure source directories are present.
346 dde49bef Renato Botelho
	update_freebsd_sources
347 6f73c362 Renato Botelho
	git_last_commit
348
349
	# Ensure binaries are present that builder system requires
350 1b40e1a6 Renato Botelho
	depend_check
351 6f73c362 Renato Botelho
352
	# Build world, kernel and install
353
	make_world
354
355
	# Build kernels
356
	build_all_kernels
357
358 7d460897 Renato Botelho
	# Install kernel on installer
359 f41f9c0c Renato Botelho
	installkernel ${INSTALLER_CHROOT_DIR} ${PRODUCT_NAME}
360 7d460897 Renato Botelho
361 6f73c362 Renato Botelho
	# Prepare pre-final staging area
362
	clone_to_staging_area
363
364
	# Install packages needed for Product
365
	install_pkg_install_ports
366
367 31fd94b3 Renato Botelho
	# Create core repo
368
	core_pkg_create_repo
369
fi
370 92a1044d Renato Botelho
371
# Send core repo to staging area
372
pkg_repo_rsync "${CORE_PKG_PATH}" ignore_final_rsync
373
374 6f73c362 Renato Botelho
export DEFAULT_KERNEL=${DEFAULT_KERNEL_ISO:-"${PRODUCT_NAME}"}
375
376 39ef49f6 Renato Botelho
# XXX: Figure out why wait is failing and proper fix
377
# Global variable to keep track of process running in bg
378
export _bg_pids=""
379
380 6f73c362 Renato Botelho
for _IMGTOBUILD in $_IMAGESTOBUILD; do
381
	# Clean up items that should be cleaned each run
382
	staginareas_clean_each_run
383
384 ed44fe76 Renato Botelho
	case "${_IMGTOBUILD}" in
385
		iso)
386 b17d47b6 Renato Botelho
			if [ -n "${ISO_VARIANTS}" ]; then
387
				for _variant in ${ISO_VARIANTS}; do
388
					create_iso_image ${_variant}
389
				done
390
			else
391
				create_iso_image
392
			fi
393 ed44fe76 Renato Botelho
			;;
394
		memstick)
395 b0d0498c Renato Botelho
			if [ -n "${MEMSTICK_VARIANTS}" ]; then
396
				for _variant in ${MEMSTICK_VARIANTS}; do
397
					create_memstick_image ${_variant}
398
				done
399
			else
400
				create_memstick_image
401
			fi
402 ed44fe76 Renato Botelho
			;;
403
		memstickserial)
404
			create_memstick_serial_image
405
			;;
406
		memstickadi)
407
			create_memstick_adi_image
408
			;;
409
		ova)
410 b00400ef Renato Botelho
			old_custom_package_list="${custom_package_list}"
411
			export custom_package_list="${custom_package_list} ${PRODUCT_NAME}-pkg-Open-VM-Tools"
412
			install_pkg_install_ports
413 ed44fe76 Renato Botelho
			create_ova_image
414 b00400ef Renato Botelho
			export custom_package_list="${old_custom_package_list}"
415 ed44fe76 Renato Botelho
			install_pkg_install_ports
416
			;;
417
	esac
418 6f73c362 Renato Botelho
done
419
420 39ef49f6 Renato Botelho
if [ -n "${_bg_pids}" ]; then
421
	if [ -n "${SNAPSHOTS}" ]; then
422
		snapshots_update_status ">>> NOTE: waiting for jobs: ${_bg_pids} to finish..."
423
	else
424
		echo ">>> NOTE: waiting for jobs: ${_bg_pids} to finish..."
425
	fi
426
	wait
427
428
	# XXX: For some reason wait is failing, workaround it tracking all PIDs
429
	while [ -n "${_bg_pids}" ]; do
430
		_tmp_pids="${_bg_pids}"
431
		unset _bg_pids
432
		for p in ${_tmp_pids}; do
433
			[ -z "${p}" ] \
434
				&& continue
435
436
			kill -0 ${p} >/dev/null 2>&1 \
437
				&& _bg_pids="${_bg_pids}${_bg_pids:+ }${p}"
438
		done
439
		[ -n "${_bg_pids}" ] \
440
			&& sleep 1
441
	done
442 cc145494 Renato Botelho
fi
443 6f73c362 Renato Botelho
444 ab943fc9 Renato Botelho
if [ -n "${SNAPSHOTS}" ]; then
445 f6c83d05 Renato Botelho
	if [ "${IMAGETYPE}" = "none" -a -n "${UPLOAD}" ]; then
446 ddf56629 Renato Botelho
		pkg_repo_rsync "${CORE_PKG_PATH}"
447
	elif [ "${IMAGETYPE}" != "none" ]; then
448 f26731b0 Renato Botelho
		snapshots_create_sha256
449 ddf56629 Renato Botelho
		# SCP files to snapshot web hosting area
450 f6c83d05 Renato Botelho
		if [ -n "${UPLOAD}" ]; then
451 ddf56629 Renato Botelho
			snapshots_scp_files
452
		fi
453 ab943fc9 Renato Botelho
	fi
454
	# Alert the world that we have some snapshots ready.
455
	snapshots_update_status ">>> Builder run is complete."
456
fi
457 6f73c362 Renato Botelho
458 6b733d47 Renato Botelho
echo ">>> ${IMAGES_FINAL_DIR} now contains:"
459 b34d81ce Renato Botelho
(cd ${IMAGES_FINAL_DIR} && find ${IMAGES_FINAL_DIR} -type f)
460 6b733d47 Renato Botelho
461 6f73c362 Renato Botelho
set -e
462
# Run final finish routines
463
finish