Project

General

Profile

Download (17.1 KB) Statistics
| Branch: | Tag: | Revision:
1
/*
2
 * gitsync
3
 *
4
 * part of pfSense (https://www.pfsense.org)
5
 * Copyright (c) 2010-2012 Erik Fonnesbeck
6
 * Copyright (c) 2004-2016 Rubicon Communications, LLC (Netgate)
7
 * All rights reserved.
8
 *
9
 * Licensed under the Apache License, Version 2.0 (the "License");
10
 * you may not use this file except in compliance with the License.
11
 * You may obtain a copy of the License at
12
 *
13
 * http://www.apache.org/licenses/LICENSE-2.0
14
 *
15
 * Unless required by applicable law or agreed to in writing, software
16
 * distributed under the License is distributed on an "AS IS" BASIS,
17
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
 * See the License for the specific language governing permissions and
19
 * limitations under the License.
20
 */
21

    
22
require_once("globals.inc");
23
require_once("filter.inc");
24
require_once("shaper.inc");
25
require_once("rrd.inc");
26
require_once("pfsense-utils.inc");
27

    
28
$GIT_PKG = "git"; // Either "git" or the full package URL
29
$GIT_BIN= "/usr/local/bin/git";
30
$GIT_REPO = "git://github.com/pfsense/pfsense.git";
31
$DEFAULT_BRANCH = "RELENG_2_4_0";
32
$CODIR =  "/root/pfsense";
33
$GITSYNC_MERGE = "/root/.gitsync_merge";
34

    
35
/* NOTE: Set branches here */
36
$branches = array(
37
	"master" => "2.4.1 development branch",
38
	"RELENG_2_4_0" => "2.4.0 stable branch",
39
	"build_commit" => "The commit originally used to build the image"
40
);
41

    
42
global $g;
43
global $argv;
44
global $command_split;
45

    
46
if (is_array($command_split)) {
47
	$temp_args = array_slice($command_split, 2);
48
} else {
49
	$temp_args = array_slice($argv, 3);
50
}
51

    
52
$valid_args = array(
53
	"--minimal" => "\tPerform a copy of only the updated files.\n" .
54
	    "\tNot recommended if the system has files modified by any method other\n" .
55
	    "\tthan gitsync.\n",
56
	"--diff" => "\tPerform a copy of only the different and missing files.\n" .
57
	    "\tRecommended for SSD if system has files modified by any method other\n" .
58
	    "\tthan gitsync.\n",
59
	"--show-files" => "\tShow the updated, different and missing files.\n" .
60
	    "\t(when used with --minimal and --diff options)\n",
61
	"--show-command" => "\tShow the constructed command.\n",
62
	"--dry-run" => "\tDry-run only.  No files copied.\n",
63
	"--help" => "\tDisplay this help list.\n"
64
	);
65
$args = array();
66
$arg_count = 0;
67
while (!empty($temp_args)) {
68
	$arg = array_shift($temp_args);
69
	if ($arg[0] == '-') {
70
		switch ($arg) {
71
			case "--help":
72
				echo "Usage: playback gitsync [options] [[repository] <branch>]\nOptions:\n";
73
				foreach ($valid_args as $arg_name => $arg_desc) {
74
					echo $arg_name . "\n" . $arg_desc;
75
				}
76
				exit;
77
			case "--upgrading":
78
				// Disables all interactive functions and neither PHP
79
				// nor the web GUI will be killed or restarted.
80
				$upgrading = true;
81
			case (isset($valid_args[$arg])):
82
				$args[$arg] = true;
83
				break;
84
			default:
85
				echo "Invalid option: {$arg}\nUse --help for usage information.\n";
86
				exit;
87
		}
88
	} else {
89
		$args[$arg_count++] = $arg;
90
	}
91
}
92

    
93
if (!function_exists('post_cvssync_commands')) {
94
function post_cvssync_commands() {
95
	echo "===> Removing FAST-CGI temporary files...\n";
96
	exec("find /tmp -name \"php-fastcgi.socket*\" -exec rm -rf {} \;");
97
	exec("find /tmp -name \"*.tmp\" -exec rm -rf {} \;");
98

    
99
	exec("rm -rf /tmp/xcache/* 2>/dev/null");
100

    
101
	echo "===> Upgrading configuration (if needed)...\n";
102
	convert_config();
103

    
104
	echo "===> Configuring filter...";
105
	exec("/etc/rc.filter_configure_sync");
106
	exec("pfctl -f /tmp/rules.debug");
107
	echo "\n";
108

    
109
	if (file_exists("/etc/rc.php_ini_setup")) {
110
		echo "===> Running /etc/rc.php_ini_setup...";
111
		exec("/etc/rc.php_ini_setup >/dev/null 2>&1");
112
		echo "\n";
113
	}
114

    
115
	/* lock down console if necessary */
116
	echo "===> Locking down the console if needed...\n";
117
	reload_ttys();
118

    
119
	echo "===> Signaling PHP and nginx restart...";
120
	$fd = fopen("/tmp/restart_nginx", "w");
121
	fwrite($fd, "#!/bin/sh\n");
122
	fwrite($fd, "sleep 5\n");
123
	fwrite($fd, "/usr/local/sbin/pfSctl -c 'service restart webgui'\n");
124
	fclose($fd);
125
	mwexec_bg("sh /tmp/restart_nginx");
126
	echo "\n";
127
}
128
}
129

    
130
if (!function_exists('isUrl')) {
131
function isUrl($url = "") {
132
	if ($url) {
133
		if (strstr($url, "rcs.pfsense.org") or
134
			strstr($url, "mainline") or
135
			strstr($url, ".git") or
136
			strstr($url, "git://")) {
137
			return true;
138
		}
139
	}
140
	return false;
141
}
142
}
143

    
144
if (!function_exists('run_cmds')) {
145
function run_cmds($cmds) {
146
	global $debug;
147
	foreach ($cmds as $cmd) {
148
		if ($debug) {
149
			echo "Running $cmd";
150
		}
151
		exec($cmd);
152
	}
153
}
154
}
155

    
156
unlink_if_exists("/tmp/config.cache");
157

    
158
if (!file_exists($GIT_BIN)) {
159
	require_once("pkg-utils.inc");
160

    
161
	echo "Cannot find git, installing...\n";
162
	if (!pkg_call('install -y -q git')) {
163
		echo "\nERROR: Unable to install git pkg.\n";
164
		return;
165
	}
166
}
167

    
168
# Remove mainline if exists (older)
169
if (is_dir("/root/pfsense/mainline")) {
170
	exec("rm -rf /root/pfsense/mainline");
171
}
172

    
173
# Remove RELENG_1_2 if exists (older)
174
if (is_dir("/root/pfsense/RELENG_1_2")) {
175
	exec("rm -rf /root/pfsense/RELENG_1_2");
176
}
177

    
178
# Remove HEAD if exists (older)
179
if (is_dir("/root/pfsense/HEAD")) {
180
	exec("rm -rf /root/pfsense/HEAD");
181
}
182

    
183
if (file_exists("/root/cvssync_backup.tgz")) {
184
	$backup_date = `ls -lah /root/cvssync_backup.tgz | awk '{ print $6,$7,$8 }'`;
185
	$tmp = array("RESTORE" => "Restores prior CVSSync backup data performed at {$backup_date}");
186
	$branches = array_merge($branches, $tmp);
187
}
188

    
189
if (is_dir("$CODIR/pfSenseGITREPO/pfSenseGITREPO")) {
190
	exec("cd $CODIR/pfSenseGITREPO/pfSenseGITREPO && {$GIT_BIN} config remote.origin.url", $output_str, $ret);
191
	if (is_array($output_str) && !empty($output_str[0])) {
192
		$GIT_REPO = $output_str[0];
193
	}
194
	unset($output_str);
195
}
196

    
197
if (!$args[0] && !$upgrading) {
198
	echo "\nCurrent repository is $GIT_REPO\n";
199
	echo "\nPlease select which branch you would like to sync against:\n\n";
200
	foreach ($branches as $branchname => $branchdesc) {
201
		echo "{$branchname} \t {$branchdesc}\n";
202
	}
203
	echo "\nOr alternatively you may enter a custom RCS branch URL (Git or HTTP).\n\n";
204
	$branch = readline("> ");
205
	echo "\n";
206
} else {
207
	$branch = $args[0];
208
}
209

    
210
if ($args[1] == "NOBACKUP") {
211
	$nobackup = true;
212
} else {
213
	$nobackup = false;
214
}
215

    
216
// If the repository has been fetched before, build a list of its branches.
217
if (is_dir("$CODIR/pfSenseGITREPO/pfSenseGITREPO")) {
218
	exec("cd $CODIR/pfSenseGITREPO/pfSenseGITREPO && {$GIT_BIN} branch -r", $branch_list, $ret);
219
	if ($ret == 0 && is_array($branch_list)) {
220
		foreach ($branch_list as $branch_item) {
221
			$branch_item = substr(strrchr($branch_item, "/"), 1);
222
			if (!isset($branches[$branch_item])) {
223
				$branches[$branch_item] = " ";
224
			}
225
		}
226
	}
227
}
228

    
229
$found = false;
230
foreach ($branches as $branchname => $branchdesc) {
231
	if ($branchname == $branch) {
232
		$found = true;
233
		break;
234
	}
235
}
236
if (!$found) {
237
	if (isURL($branch) && !$upgrading) {
238
		if ($args[1]) {
239
			$GIT_REPO = $branch;
240
			$branch = $args[1];
241
			$found = true;
242
		} else {
243
			echo "\n";
244
			echo "NOTE: $branch was not found.\n\n";
245
			$command = readline("Is this a custom GIT URL? [y]? ");
246
			if (strtolower($command) == "y" or $command == "") {
247
				$GIT_REPO = $branch;
248
				$command = readline("Checkout which branch [${DEFAULT_BRANCH}]? ");
249
				if ($command == "") {
250
					$branch = $DEFAULT_BRANCH;
251
				}
252
				if ($command) {
253
					$branch = $command;
254
				}
255
				$found = true;
256
			}
257
		}
258
	}
259
	if (!$found) {
260
		echo "\nNo valid branch found.  Exiting.\n\n";
261
		exit;
262
	}
263
}
264

    
265
$merge_repos = array();
266
if (file_exists($GITSYNC_MERGE)) {
267
	$gitsync_merges = file($GITSYNC_MERGE, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
268
	if (!empty($gitsync_merges) && is_array($gitsync_merges)) {
269
		echo "\n===> Automatic merge list read from ${GITSYNC_MERGE}\n";
270
		foreach ($gitsync_merges as $merge_line_num => $merge_line) {
271
			$merge_comments = explode("#", trim($merge_line));
272
			if (empty($merge_comments[0])) {
273
				continue;
274
			}
275

    
276
			$merge_line = explode(" ", trim($merge_comments[0]));
277
			if (count($merge_line) != 2 || empty($merge_line[0]) || empty($merge_line[1])) {
278
				echo "\nLine " . ($merge_line_num + 1) . " does not have the correct parameter count or has improper spacing.\n";
279
				echo "Expected parameters:  repository_url branch\n";
280
				echo "Line read:  " . implode(" ", $merge_line) . "\n\n";
281
				echo "Aborting automatic merge.\n\n";
282
				$merge_repos = array();
283
				break;
284
			}
285
			$merge_repos[] = array('repo' => $merge_line[0], 'branch' => $merge_line[1]);
286
		}
287
	}
288
}
289
if (!$args[0] && !$upgrading) {
290
	do {
291
		echo "\nAdd a custom RCS branch URL (Git or HTTP) to merge in or press enter if done.\n\n";
292
		$merge_repo = readline("> ");
293
		if (!empty($merge_repo)) {
294
			$merge_branch = readline("Merge which branch [${DEFAULT_BRANCH}]? ");
295
			if ($merge_branch == "") {
296
				$merge_repos[] = array('repo' => $merge_repo, 'branch' => $DEFAULT_BRANCH);
297
			} else if ($merge_branch) {
298
				$merge_repos[] = array('repo' => $merge_repo, 'branch' => $merge_branch);
299
			}
300
		}
301
	} while (!empty($merge_repo));
302
}
303

    
304
if ($branch == "RESTORE") {
305
	if (!file_exists("/root/cvssync_backup.tgz")) {
306
		echo "Sorry, we could not find a previous CVSSync backup file.\n";
307
		exit();
308
	}
309
	echo "===> Restoring previous CVSSync backup... Please wait...\n";
310
	exec("tar Uxpf /root/cvssync_backup.tgz -C /");
311
	post_cvssync_commands();
312
	exit();
313
} else {
314
	$nobackup = true; // do not backup embedded
315
}
316

    
317
if ($nobackup == false) {
318
	echo "===> Backing up current pfSense information...\n";
319
	echo "===> Please wait... ";
320
	exec("tar czPf /root/cvssync_backup.tgz --exclude /root --exclude /dev --exclude /tmp --exclude /var/run --exclude /var/empty /");
321
	$size = filesize("/root/cvssync_backup.tgz");
322
	echo "{$size} bytes.\n\n";
323
	sleep(5);
324
}
325

    
326
echo "===> Checking out $branch\n";
327

    
328
// Git commands for resetting to the specified branch
329
if ($branch == "build_commit") {
330
	$git_cmd = array(
331
		"cd $CODIR/pfSenseGITREPO/pfSenseGITREPO && {$GIT_BIN} branch " . escapeshellarg($branch) . " 2>/dev/null",
332
		"cd $CODIR/pfSenseGITREPO/pfSenseGITREPO && {$GIT_BIN} checkout -f " . escapeshellarg($branch) . " 2>/dev/null",
333
		"cd $CODIR/pfSenseGITREPO/pfSenseGITREPO && {$GIT_BIN} reset --hard " . escapeshellarg(trim(file_get_contents("/etc/version.lastcommit")))
334
	);
335
} else {
336
	$git_cmd = array(
337
		"cd $CODIR/pfSenseGITREPO/pfSenseGITREPO && {$GIT_BIN} branch " . escapeshellarg($branch) . " " . escapeshellarg("origin/{$branch}") . " 2>/dev/null",
338
		"cd $CODIR/pfSenseGITREPO/pfSenseGITREPO && {$GIT_BIN} checkout -f " . escapeshellarg($branch) . " 2>/dev/null",
339
		"cd $CODIR/pfSenseGITREPO/pfSenseGITREPO && {$GIT_BIN} reset --hard " . escapeshellarg("origin/{$branch}")
340
	);
341
}
342

    
343
// Git 'er done!
344
if (is_dir("$CODIR/pfSenseGITREPO/pfSenseGITREPO")) {
345
	echo "===> Fetching updates...\n";
346
	exec("cd $CODIR/pfSenseGITREPO/pfSenseGITREPO && {$GIT_BIN} config remote.origin.url " . escapeshellarg($GIT_REPO));
347
	exec("cd $CODIR/pfSenseGITREPO/pfSenseGITREPO && {$GIT_BIN} fetch");
348
	exec("cd $CODIR/pfSenseGITREPO/pfSenseGITREPO && {$GIT_BIN} clean -f -x -d");
349
	run_cmds($git_cmd);
350
} else {
351
	exec("mkdir -p $CODIR/pfSenseGITREPO");
352
	echo "Executing cd $CODIR/pfSenseGITREPO && {$GIT_BIN} clone $GIT_REPO pfSenseGITREPO\n";
353
	exec("cd $CODIR/pfSenseGITREPO && {$GIT_BIN} clone " . escapeshellarg($GIT_REPO) . " pfSenseGITREPO");
354
	if (is_dir("$CODIR/pfSenseGITREPO/pfSense")) {
355
		exec("mv $CODIR/pfSenseGITREPO/pfSense $CODIR/pfSenseGITREPO/pfSenseGITREPO");
356
	}
357
	if (is_dir("$CODIR/pfSenseGITREPO/mainline")) {
358
		exec("mv $CODIR/pfSenseGITREPO/mainline $CODIR/pfSenseGITREPO/pfSenseGITREPO");
359
	}
360
	run_cmds($git_cmd);
361
}
362

    
363
foreach ($merge_repos as $merge_repo) {
364
	echo "===> Merging branch {$merge_repo['branch']} from {$merge_repo['repo']}\n";
365
	exec("cd $CODIR/pfSenseGITREPO/pfSenseGITREPO && {$GIT_BIN} pull " . escapeshellarg($merge_repo['repo']) . " " . escapeshellarg($merge_repo['branch']), $output_str, $ret);
366
	unset($output_str);
367
	if ($ret <> 0) {
368
		echo "\nMerge failed.  Aborting sync.\n\n";
369
		run_cmds($git_cmd);
370
		exit;
371
	}
372
}
373

    
374
$updated_files_array[1] = array();
375
if (isset($args["--minimal"])) {
376
	if (file_exists("/etc/version.gitsync")) {
377
		$old_revision = trim(file_get_contents("/etc/version.gitsync"));
378
	} else if (file_exists("/etc/version.lastcommit")) {
379
		$old_revision = trim(file_get_contents("/etc/version.lastcommit"));
380
	}
381

    
382
	# Find updated files.
383
	$updated_files = shell_exec("cd $CODIR/pfSenseGITREPO/pfSenseGITREPO && {$GIT_BIN} diff --name-only --relative=src " . escapeshellarg($old_revision));
384

    
385
	# Get the path of each updated file.
386
	preg_match_all('@(?:^(.+?)$)@sim', $updated_files, $updated_files_array, PREG_PATTERN_ORDER);
387

    
388
	if(isset($args["--show-files"])) {
389
		echo "===> Updated Files: \n";
390
		print_r($updated_files_array[1]);
391
	}
392
}
393

    
394
// Save new commit ID for later minimal file copies
395
if(!isset($args["--dry-run"])) {
396
	exec("cd $CODIR/pfSenseGITREPO/pfSenseGITREPO && {$GIT_BIN} rev-parse -q --verify HEAD > /etc/version.gitsync");
397
}
398

    
399
// Remove files that we do not want to overwrite the system with
400
@unlink("{$CODIR}/pfSenseGITREPO/pfSenseGITREPO/src/etc/crontab");
401
@unlink("{$CODIR}/pfSenseGITREPO/pfSenseGITREPO/src/etc/master.passwd");
402
@unlink("{$CODIR}/pfSenseGITREPO/pfSenseGITREPO/src/etc/passwd");
403
@unlink("{$CODIR}/pfSenseGITREPO/pfSenseGITREPO/src/etc/fstab");
404
@unlink("{$CODIR}/pfSenseGITREPO/pfSenseGITREPO/src/etc/ttys");
405
@unlink("{$CODIR}/pfSenseGITREPO/pfSenseGITREPO/src/etc/group");
406
@unlink("{$CODIR}/pfSenseGITREPO/pfSenseGITREPO/src/etc/fstab");
407
@unlink("{$CODIR}/pfSenseGITREPO/pfSenseGITREPO/src/etc/platform");
408
@unlink("{$CODIR}/pfSenseGITREPO/pfSenseGITREPO/src/boot/device.hints");
409
@unlink("{$CODIR}/pfSenseGITREPO/pfSenseGITREPO/src/boot/loader.conf");
410
@unlink("{$CODIR}/pfSenseGITREPO/pfSenseGITREPO/src/boot/loader.rc");
411
@unlink("{$CODIR}/pfSenseGITREPO/pfSenseGITREPO/src/etc/syslog.conf");
412
@unlink("{$CODIR}/pfSenseGITREPO/pfSenseGITREPO/src/root/.shrc");
413
@unlink("{$CODIR}/pfSenseGITREPO/pfSenseGITREPO/src/root/.tcshrc");
414
exec("rm -rf {$CODIR}/pfSenseGITREPO/pfSenseGITREPO/src/conf*");
415
exec("rm -rf {$CODIR}/pfSenseGITREPO/pfSenseGITREPO/src/cf 2>/dev/null");
416
@chmod("{$CODIR}/pfSenseGITREPO/pfSenseGITREPO/src/tmp", 01777);
417

    
418
$different_files_array[1] = array();
419
$missing_files_array[1] = array();
420
if(isset($args["--diff"])) {
421
	# Find different and missing files.
422
	$different_missing_files = shell_exec("/usr/bin/diff -qr / $CODIR/pfSenseGITREPO/pfSenseGITREPO/src/ | /usr/bin/grep -E '^(Files .*? and $CODIR/pfSenseGITREPO/pfSenseGITREPO/src/.*? differ)|(Only in $CODIR/pfSenseGITREPO/pfSenseGITREPO/src/)'");
423

    
424
	# Get the path of each different or missing file.
425
	preg_match_all('@(?:^Files .*? and '.$CODIR.'/pfSenseGITREPO/pfSenseGITREPO/src/(.+?) differ.*?$)@sim', $different_missing_files, $different_files_array, PREG_PATTERN_ORDER);
426
	preg_match_all('@(?:^Only in '.$CODIR.'/pfSenseGITREPO/pfSenseGITREPO/src/(.+?)$)@sim', $different_missing_files, $missing_files_array, PREG_PATTERN_ORDER);
427

    
428
	# Deal with diff's output format of missing files (path: missing_file).
429
	foreach ($missing_files_array[1] as $key => $file) {
430

    
431
		# Most of the time there will be only the one ': ' injected by diff output.  So global replace with dir delimiter (/) is fine.
432
		$tmp = str_replace(": ", "/", $file, $count);
433
		if ($count == 1)
434
			$file = ltrim($tmp, "/");
435

    
436
		# For the very rare case a path component (dir or file) contains ': ' as well, then need to find and replace only the ': ' injected by diff output.
437
		else {
438
			$tmp = $file;
439
			do {
440
				$pos = strrpos($tmp, ": ");
441
				if ($pos) {
442
					$tmp = substr($tmp, 0, $pos);
443
					$res = is_dir("$CODIR/pfSenseGITREPO/pfSenseGITREPO/src/$tmp/");
444
				}
445
			} while (!$res && $pos);
446

    
447
			if ($res)
448
				$file = ltrim($tmp . "/" . substr($file, $pos+2), "/");
449
		}
450

    
451
		$missing_files_array[1][$key] = $file;
452
	}
453

    
454
	if(isset($args["--show-files"])) {
455
		echo "===> Different Files: \n";
456
		print_r($different_files_array[1]);
457
		echo "===> Missing Files: \n";
458
		print_r($missing_files_array[1]);
459
	}
460
}
461

    
462
# Files to be copied.
463
if(isset($args["--minimal"]) || isset($args["--diff"])) {
464
	$files_to_copy_array = array_merge($updated_files_array[1], $different_files_array[1], $missing_files_array[1]);
465
	$files_to_copy_array = array_unique($files_to_copy_array);
466

    
467
	unset($updated_files_array, $different_files_array, $missing_files_array);
468

    
469
	# Convert the list from an array to a space separated quoted string.  Quoted for white space file name support.
470
	if (count($files_to_copy_array) > 0) {	# Leave the string empty/unset if there is nothing to copy.
471
		$files_to_copy = '"' . implode('" "', $files_to_copy_array) . '"';
472
	}
473
	$qty_files_to_copy = count($files_to_copy_array);
474
	unset($files_to_copy_array);
475
} else {
476
	$files_to_copy = '.';
477
	$qty_files_to_copy = chr(8);
478
}
479

    
480
$tar_options = '-C ./src';
481

    
482
echo "===> Installing $qty_files_to_copy new files...\n";
483

    
484
$command = "cd $CODIR/pfSenseGITREPO/pfSenseGITREPO ; tar -cpf - {$tar_options} {$files_to_copy} | (cd / ; tar -Uxpf -)";
485

    
486
if (!empty($files_to_copy)) {
487
	if(isset($args["--show-command"])) {
488
		echo "===> Command: \n$command\n";
489
	}
490
	if(!isset($args["--dry-run"])) {
491
		exec($command);
492
	}
493
} else {
494
	echo "Already up-to-date.\n";
495
	$upgrading = true;
496
}
497

    
498
// Reset the repository to restore the deleted files
499
exec("cd $CODIR/pfSenseGITREPO/pfSenseGITREPO && {$GIT_BIN} reset --hard >/dev/null 2>/dev/null");
500

    
501
// Remove obsolete files
502
$files_to_remove = file("/etc/pfSense.obsoletedfiles");
503
foreach ($files_to_remove as $file_to_remove) {
504
	if (file_exists($file_to_remove)) {
505
		exec("/bin/rm -f $file_to_remove");
506
	}
507
}
508

    
509
if (!$upgrading) {
510
	post_cvssync_commands();
511
}
512

    
513
echo "===> Checkout complete.\n";
514
echo "\n";
515
if (!$upgrading) {
516
	echo "The system is now sync'd and PHP and nginx will be restarted in 5 seconds.\n\n";
517
} else {
518
	echo "The system is now sync'd.\n\n";
519
}
520

    
(13-13/24)