5 Copyright (c) 2003-2010, Andrew Dunstan
7 See accompanying License file for license details
13 use vars qw($dbhost $dbname $dbuser $dbpass $dbport
14 $all_stat $fail_stat $change_stat $green_stat
16 $min_script_version $min_web_script_version
17 $default_host $local_git_clone
20 # force this before we do anything - even load modules
21 BEGIN { $server_time = time; }
24 use Digest::SHA1 qw(sha1_hex);
31 use Storable qw(thaw);
33 require "$ENV{BFConfDir}/BuildFarmWeb.pl";
34 my $buildlogs = "$ENV{BFConfDir}/buildlogs";
36 die "no dbname" unless $dbname;
37 die "no dbuser" unless $dbuser;
39 my $dsn="dbi:Pg:dbname=$dbname";
40 $dsn .= ";host=$dbhost" if $dbhost;
41 $dsn .= ";port=$dbport" if $dbport;
45 my $sig = $query->path_info;
48 my $stage = $query->param('stage');
49 my $ts = $query->param('ts');
50 my $animal = $query->param('animal');
51 my $log = $query->param('log');
52 my $res = $query->param('res');
53 my $conf = $query->param('conf');
54 my $branch = $query->param('branch');
55 my $changed_since_success = $query->param('changed_since_success');
56 my $changed_this_run = $query->param('changed_files');
57 my $log_archive = $query->param('logtar');
58 my $frozen_sconf = $query->param('frozen_sconf') || '';
61 if (open($brhandle,"../htdocs/branches_of_interest.txt"))
63 my @branches_of_interest = <$brhandle>;
65 chomp(@branches_of_interest);
66 unless (grep {$_ eq $branch} @branches_of_interest)
69 "Status: 492 bad branch parameter $branch\nContent-Type: text/plain\n\n",
70 "bad branch parameter $branch\n";
77 "branch=$branch&res=$res&stage=$stage&animal=$animal&".
78 "ts=$ts&log=$log&conf=$conf";
81 "changed_files=$changed_this_run&".
82 "changed_since_success=$changed_since_success&";
84 unless ($animal && $ts && $stage && $sig)
87 "Status: 490 bad parameters\nContent-Type: text/plain\n\n",
88 "bad parameters for request\n";
93 # Want to allow all kinds of named branches
94 #unless ($branch =~ /^(HEAD|REL\d+_\d+_STABLE)$/)
97 # "Status: 492 bad branch parameter $branch\nContent-Type: text/plain\n\n",
98 # "bad branch parameter $branch\n";
104 my $db = DBI->connect($dsn,$dbuser,$dbpass);
106 die $DBI::errstr unless $db;
109 "select secret from buildsystems where name = ? and status = 'approved'";
110 my $sth = $db->prepare($gethost);
111 $sth->execute($animal);
112 my ($secret)=$sth->fetchrow_array();
115 my $tsdiff = time - $ts;
117 my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($ts);
118 $year += 1900; $mon +=1;
120 sprintf("%d-%.2d-%.2d_%.2d:%.2d:%.2d",$year,$mon,$mday,$hour,$min,$sec);
122 if ($ENV{BF_DEBUG} || ($ts > time) || ($ts + 86400 < time ) || (! $secret) )
124 open(TX,">$buildlogs/$animal.$date");
125 print TX "sig=$sig\nlogtar-len=" , length($log_archive),
126 "\nstatus=$res\nstage=$stage\nconf:\n$conf\n",
128 "changed_this_run:\n$changed_this_run\n",
129 "changed_since_success:\n$changed_since_success\n",
130 "frozen_sconf:$frozen_sconf\n",
132 # $query->save(\*TX);
136 unless ($ts < time + 120)
138 my $gmt = gmtime($ts);
139 print "Status: 491 bad ts parameter - $ts ($gmt GMT) is in the future.\n",
140 "Content-Type: text/plain\n\n bad ts parameter - $ts ($gmt GMT) is in the future\n";
145 unless ($ts + 86400 > time)
147 my $gmt = gmtime($ts);
148 print "Status: 491 bad ts parameter - $ts ($gmt GMT) is more than 24 hours ago.\n",
149 "Content-Type: text/plain\n\n bad ts parameter - $ts ($gmt GMT) is more than 24 hours ago.\n";
157 "Status: 495 Unknown System\nContent-Type: text/plain\n\n",
158 "System $animal is unknown\n";
167 my $calc_sig = sha1_hex($content,$secret);
168 my $calc_sig2 = sha1_hex($extra_content,$content,$secret);
170 if ($calc_sig ne $sig && $calc_sig2 ne $sig)
173 print "Status: 450 sig mismatch\nContent-Type: text/plain\n\n";
174 print "$sig mismatches $calc_sig($calc_sig2) on content:\n$content";
179 # undo escape-proofing of base64 data and decode it
180 map {tr/$@/+=/; $_ = decode_base64($_); }
181 ($log, $conf,$changed_this_run,$changed_since_success,$log_archive, $frozen_sconf);
183 if ($log =~/Last file mtime in snapshot: (.*)/)
185 my $snaptime = parsedate($1);
186 my $brch = $branch eq 'HEAD' ? 'master' : $branch;
187 my $last_branch_time = time - (30 * 86400);
188 $last_branch_time = `TZ=UTC GIT_DIR=$local_git_clone git log -1 --pretty=format:\%ct $brch`;
189 if ($snaptime < ($last_branch_time - 86400))
191 print "Status: 493 snapshot too old: $1\nContent-Type: text/plain\n\n";
192 print "snapshot to old: $1\n";
198 ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = gmtime($ts);
199 $year += 1900; $mon +=1;
201 sprintf("%d-%.2d-%.2d %.2d:%.2d:%.2d",$year,$mon,$mday,$hour,$min,$sec);
205 my $dirname = "$buildlogs/tmp.$$.unpacklogs";
212 my $archname = "$buildlogs/tmp.$$.tgz";
213 open($log_handle,">$archname");
215 print $log_handle $log_archive;
218 @log_file_names = `tar -z -C $dirname -xvf $archname 2>/dev/null`;
219 map {s/\s+//g; } @log_file_names;
220 my @qnames = grep { $_ ne 'githead.log' } @log_file_names;
221 map { $_ = qq("$_"); } @qnames;
222 $log_file_names = '{' . join(',',@qnames) . '}';
223 if (-e "$dirname/githead.log" )
225 open(my $githead,"$dirname/githead.log");
226 $githeadref = <$githead>;
237 $client_conf = thaw $frozen_sconf;
240 if ($min_script_version)
242 $client_conf->{script_version} ||= '0.0';
243 my $cli_ver = $client_conf->{script_version} ;
244 $cli_ver =~ s/^REL_//;
245 my ($minmajor,$minminor) = split(/\./,$min_script_version);
246 my ($smajor,$sminor) = split(/\./,$cli_ver);
247 if ($minmajor > $smajor || ($minmajor == $smajor && $minminor > $sminor))
249 print "Status: 460 script version too low\nContent-Type: text/plain\n\n";
251 "Script version is below minimum required\n",
252 "Reported version: $client_conf->{script_version},",
253 "Minumum version required: $min_script_version\n";
259 if ($min_web_script_version)
261 $client_conf->{web_script_version} ||= '0.0';
262 my $cli_ver = $client_conf->{web_script_version} ;
263 $cli_ver =~ s/^REL_//;
264 my ($minmajor,$minminor) = split(/\./,$min_web_script_version);
265 my ($smajor,$sminor) = split(/\./,$cli_ver);
266 if ($minmajor > $smajor || ($minmajor == $smajor && $minminor > $sminor))
268 print "Status: 461 web script version too low\nContent-Type: text/plain\n\n";
270 "Web Script version is below minimum required\n",
271 "Reported version: $client_conf->{web_script_version}, ",
272 "Minumum version required: $min_web_script_version\n"
280 if (not exists $client_conf->{config_opts} )
284 elsif (ref $client_conf->{config_opts} eq 'HASH')
286 # leave out keys with false values
287 @config_flags = grep { $client_conf->{config_opts}->{$_} }
288 keys %{$client_conf->{config_opts}};
290 elsif (ref $client_conf->{config_opts} eq 'ARRAY' )
292 @config_flags = @{$client_conf->{config_opts}};
297 @config_flags = grep {! m/=/ } @config_flags;
298 map {s/\s+//g; $_=qq("$_"); } @config_flags;
299 push @config_flags,'git' if $client_conf->{scm} eq 'git';
300 $config_flags = '{' . join(',',@config_flags) . '}' ;
303 my $scm = $client_conf->{scm} || 'cvs';
304 my $scmurl = $client_conf->{scm_url};
307 insert into build_status
308 (sysname, snapshot,status, stage, log,conf_sum, branch,
309 changed_this_run, changed_since_success,
310 log_archive_filenames , log_archive, build_flags, scm, scmurl,
311 git_head_ref,frozen_conf)
312 values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
317 # this transaction lets us set log_error_verbosity to terse
318 # just for the duration of the transaction. That turns off logging the
319 # bind params, so all the logs don't get stuffed on the postgres logs
324 #$db->do("select set_local_error_terse()");
327 $sth=$db->prepare($logst);
329 $sth->bind_param(1,$animal);
330 $sth->bind_param(2,$dbdate);
331 $sth->bind_param(3,$res & 0x8fffffff); # in case we get a 64 bit int status!
332 $sth->bind_param(4,$stage);
333 $sth->bind_param(5,$log);
334 $sth->bind_param(6,$conf);
335 $sth->bind_param(7,$branch);
336 $sth->bind_param(8,$changed_this_run);
337 $sth->bind_param(9,$changed_since_success);
338 $sth->bind_param(10,$log_file_names);
339 #$sth->bind_param(11,$log_archive,{ pg_type => DBD::Pg::PG_BYTEA });
340 $sth->bind_param(11,undef,{ pg_type => DBD::Pg::PG_BYTEA });
341 $sth->bind_param(12,$config_flags);
342 $sth->bind_param(13,$scm);
343 $sth->bind_param(14,$scmurl);
344 $sth->bind_param(15,$githeadref);
345 $sth->bind_param(16,$frozen_sconf,{ pg_type => DBD::Pg::PG_BYTEA });
347 $sqlres = $sth->execute;
356 insert into build_status_log
357 (sysname, snapshot, branch, log_stage, log_text, stage_duration)
358 values (?, ?, ?, ?, ?, ?)
362 $sth = $db->prepare($logst2);
366 my $stage_start = $ts;
368 foreach my $log_file( @log_file_names )
370 next if $log_file =~ /^githead/;
372 open($handle,"$dirname/$log_file");
373 my $mtime = (stat $handle)[9];
374 my $stage_interval = $mtime - $stage_start;
375 $stage_start = $mtime;
376 my $ltext = <$handle>;
378 $ltext =~ s/\x00/\\0/g;
379 $sqlres = $sth->execute($animal,$dbdate,$branch,$log_file,$ltext,
380 "$stage_interval seconds");
384 $sth->finish unless $sqlres;
391 print "Status: 462 database failure\nContent-Type: text/plain\n\n";
392 print "Your report generated a database failure:\n",
403 my $prevst = <<EOSQL;
405 select coalesce((select distinct on (snapshot) stage
407 where sysname = ? and branch = ? and snapshot < ?
408 order by snapshot desc
409 limit 1), 'NEW') as prev_status
413 $sth=$db->prepare($prevst);
414 $sth->execute($animal,$branch,$dbdate);
415 my $row=$sth->fetchrow_arrayref;
416 my $prev_stat=$row->[0];
421 select operating_system|| ' / ' || os_version as os ,
422 compiler || ' / ' || compiler_version as compiler,
425 where status = 'approved'
430 $sth=$db->prepare($det_st);
431 $sth->execute($animal);
432 $row=$sth->fetchrow_arrayref;
433 my ($os, $compiler,$arch) = @$row;
437 # prevent occasional duplication by forcing serialization of this operation
438 $db->do("lock table dashboard_mat in share row exclusive mode");
439 $db->do("delete from dashboard_mat");
440 $db->do("insert into dashboard_mat select * from dashboard_mat_data");
446 # prevent occasional duplication by forcing serialization of this operation
447 $db->do("lock table nrecent_failures in share row exclusive mode");
448 $db->do("delete from nrecent_failures");
449 $db->do("insert into nrecent_failures select bs.sysname, bs.snapshot, bs.branch from build_status bs where bs.stage <> 'OK' and bs.snapshot > now() - interval '90 days'");
455 print "Content-Type: text/plain\n\n";
456 print "request was on:\n";
457 print "res=$res&stage=$stage&animal=$animal&ts=$ts";
459 my $client_events = $client_conf->{mail_events};
463 my $client_time = $client_conf->{current_ts};
464 open(TX,">>$buildlogs/$animal.$date");
465 print TX "\n",Dumper(\$client_conf),"\n";
466 print TX "server time: $server_time, client time: $client_time\n" if $client_time;
472 if (ref $client_events)
474 my $cbcc = $client_events->{all};
477 push @$bcc_stat, @$cbcc;
479 elsif (defined $cbcc)
481 push @$bcc_stat, $cbcc;
485 $cbcc = $client_events->{all};
488 push @$bcc_stat, @$cbcc;
490 elsif (defined $cbcc)
492 push @$bcc_stat, $cbcc;
495 $cbcc = $client_events->{change};
498 push @$bcc_chg, @$cbcc;
500 elsif (defined $cbcc)
502 push @$bcc_chg, $cbcc;
504 if ($stage eq 'OK' || $prev_stat eq 'OK')
506 $cbcc = $client_events->{green};
509 push @$bcc_chg, @$cbcc;
511 elsif (defined $cbcc)
513 push @$bcc_chg, $cbcc;
519 my $url = $query->url(-base => 1);
522 my $stat_type = $stage eq 'OK' ? 'Status' : 'Failed at Stage';
524 my $mailto = [@$all_stat];
525 push(@$mailto,@$fail_stat) if $stage ne 'OK';
527 my $me = `id -un`; chomp($me);
529 my $host = `hostname`; chomp ($host);
530 $host = $default_host unless ($host =~ m/[.]/ || !defined($default_host));
532 my $from_addr = "Exim Build Farm <$me\@$host>";
533 $from_addr =~ tr /\r\n//d;
535 my $msg = new Mail::Send;
539 $msg->bcc(@$bcc_stat) if (@$bcc_stat);
540 $msg->subject("Exim BuildFarm member $animal Branch $branch $stat_type $stage");
541 $msg->set('From',$from_addr);
546 The Exim BuildFarm member $animal had the following event on branch $branch:
550 The snapshot timestamp for the build that triggered this notification is: $dbdate
552 The specs of this machine are:
557 For more information, see $url/cgi-bin/show_history.pl?nm=$animal&br=$branch
563 exit if ($stage eq $prev_stat);
565 $mailto = [@$change_stat];
566 push(@$mailto,@$green_stat) if ($stage eq 'OK' || $prev_stat eq 'OK');
568 $msg = new Mail::Send;
572 $msg->bcc(@$bcc_chg) if (@$bcc_chg);
574 $stat_type = $prev_stat ne 'OK' ? "changed from $prev_stat failure to $stage" :
575 "changed from OK to $stage";
576 $stat_type = "New member: $stage" if $prev_stat eq 'NEW';
577 $stat_type .= " failure" if $stage ne 'OK';
579 $msg->subject("Exim BuildFarm member $animal Branch $branch Status $stat_type");
580 $msg->set('From',$from_addr);
584 The Exim BuildFarm member $animal had the following event on branch $branch:
588 The snapshot timestamp for the build that triggered this notification is: $dbdate
590 The specs of this machine are:
595 For more information, see $url/cgi-bin/show_history.pl?nm=$animal&br=$branch