From: Todd Lyons Date: Mon, 21 Oct 2013 13:49:58 +0000 (-0700) Subject: Merge branch 'master' of github.com:mrballcb/exim-build-farm-client X-Git-Url: https://git.exim.org/buildfarm-client.git/commitdiff_plain/a475ae642a475267249caa9d561380e2baccf5d9?hp=0a764d3725255d0692d44d4820f908a9877c97ba Merge branch 'master' of github.com:mrballcb/exim-build-farm-client --- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6f1a30a --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +build-farm.conf +build-farm_*.conf +tags +run_web_txn.pl diff --git a/EximBuild/Modules/FileTextArrayFDW.pm b/EximBuild/Modules/FileTextArrayFDW.pm new file mode 100644 index 0000000..f2c5508 --- /dev/null +++ b/EximBuild/Modules/FileTextArrayFDW.pm @@ -0,0 +1,201 @@ + +# Package Namespace is hardcoded. Modules must live in +# EximBuild::Modules + +package EximBuild::Modules::FileTextArrayFDW; + +use EximBuild::Options; +use EximBuild::SCM; + +use strict; + +# strip required namespace from package name +(my $MODULE = __PACKAGE__ ) =~ s/EximBuild::Modules:://; + +use vars qw($VERSION); $VERSION = 'REL_0.1'; + +my $hooks = { + 'checkout' => \&checkout, + 'setup-target' => \&setup_target, + + # 'need-run' => \&need_run, + # 'configure' => \&configure, + 'build' => \&build, + + # 'check' => \&check, + 'install' => \&install, + 'installcheck' => \&installcheck, + 'cleanup' => \&cleanup, +}; + +sub setup +{ + my $class = __PACKAGE__; + + my $buildroot = shift; # where we're building + my $branch = shift; # The branch of exim that's being built. + my $conf = shift; # ref to the whole config object + my $exim = shift; # exim build dir + + #return unless $branch ge 'REL9_1_STABLE' || $branch eq 'HEAD'; + + # could even set up several of these (e.g. for different branches) + my $self = { + buildroot => $buildroot, + eximbranch=> $branch, + bfconf => $conf, + exim => $exim + }; + bless($self, $class); + + my $scmconf ={ + scm => 'git', + scmrepo => 'git://github.com/adunstan/file_text_array_fdw.git', + git_reference => undef, + git_keep_mirror => 'true', + git_ignore_mirror_failure => 'true', + build_root => $self->{buildroot}, + }; + + $self->{scm} = new EximBuild::SCM $scmconf, 'file_text_array_fdw'; + my $where = $self->{scm}->get_build_path(); + $self->{where} = $where; + + # for each instance you create, do: + main::register_module_hooks($self,$hooks); + +} + +sub checkout +{ + my $self = shift; + my $savescmlog = shift; # array ref to the log lines + + print main::time_str(), "checking out $MODULE\n" if $verbose; + + my $scmlog = $self->{scm}->checkout($self->{eximbranch}); + + push(@$savescmlog, + "------------- $MODULE checkout ----------------\n",@$scmlog); +} + +sub setup_target +{ + my $self = shift; + + # copy the code or setup a vpath dir if supported as appropriate + + print main::time_str(), "copying source to ...$self->{where}\n" + if $verbose; + + $self->{scm}->copy_source(undef); + +} + +sub need_run +{ + my $self = shift; + my $run_needed = shift; # ref to flag + + # to force a run do: + # $$run_needed = 1; + + print main::time_str(), "checking if run needed by $MODULE\n" + if $verbose; + +} + +sub configure +{ + my $self = shift; + + print main::time_str(), "configuring $MODULE\n" if $verbose; + +} + +sub build +{ + my $self = shift; + + print main::time_str(), "building $MODULE\n" if $verbose; + + my $cmd = "PATH=../inst:$ENV{PATH} make USE_PGXS=1"; + + my @makeout = `cd $self->{where} && $cmd 2>&1`; + + my $status = $? >>8; + main::writelog("$MODULE-build",\@makeout); + print "======== make log ===========\n",@makeout if ($verbose > 1); + main::send_result("$MODULE-build",$status,\@makeout) if $status; + +} + +sub install +{ + my $self = shift; + + print main::time_str(), "installing $MODULE\n" if $verbose; + + my $cmd = "PATH=../inst:$ENV{PATH} make USE_PGXS=1 install"; + + my @log = `cd $self->{where} && $cmd 2>&1`; + + my $status = $? >>8; + main::writelog("$MODULE-install",\@log); + print "======== install log ===========\n",@log if ($verbose > 1); + main::send_result("$MODULE-install",$status,\@log) if $status; + +} + +sub check +{ + my $self = shift; + + print main::time_str(), "checking ",__PACKAGE__,"\n" if $verbose; +} + +sub installcheck +{ + my $self = shift; + my $locale = shift; + + print main::time_str(), "install-checking $MODULE\n" if $verbose; + + my $cmd = "PATH=../inst:$ENV{PATH} make USE_PGXS=1 installcheck"; + + my @log = `cd $self->{where} && $cmd 2>&1`; + + my $status = $? >>8; + my $installdir = "$self->{buildroot}/$self->{eximbranch}/inst"; + my @logfiles =("$self->{where}/regression.diffs","$installdir/logfile"); + foreach my $logfile(@logfiles) + { + last unless $status; + next unless (-e $logfile ); + push(@log,"\n\n================== $logfile ==================\n"); + my $handle; + open($handle,$logfile); + while(<$handle>) + { + push(@log,$_); + } + close($handle); + } + + main::writelog("$MODULE-installcheck-$locale",\@log); + print "======== installcheck ($locale) log ===========\n",@log + if ($verbose > 1); + main::send_result("$MODULE-installcheck-$locale",$status,\@log) if $status; + +} + +sub cleanup +{ + my $self = shift; + + print main::time_str(), "cleaning up $MODULE\n" if $verbose > 1; + + system("rm -rf $self->{where}"); +} + +1; diff --git a/EximBuild/Modules/Skeleton.pm b/EximBuild/Modules/Skeleton.pm new file mode 100644 index 0000000..ff25e77 --- /dev/null +++ b/EximBuild/Modules/Skeleton.pm @@ -0,0 +1,136 @@ + +# Package Namespace is hardcoded. Modules must live in +# EximBuild::Modules + +package EximBuild::Modules::Skeleton; + +use EximBuild::Options; +use EximBuild::SCM; + +use strict; + +use vars qw($VERSION); $VERSION = 'REL_0.1'; + +my $hooks = { + 'checkout' => \&checkout, + 'setup-target' => \&setup_target, + 'need-run' => \&need_run, + 'configure' => \&configure, + 'build' => \&build, + 'check' => \&check, + 'install' => \&install, + 'installcheck' => \&installcheck, + 'locale-end' => \&locale_end, + 'cleanup' => \&cleanup, +}; + +sub setup +{ + my $class = __PACKAGE__; + + my $buildroot = shift; # where we're building + my $branch = shift; # The branch of exim that's being built. + my $conf = shift; # ref to the whole config object + my $exim = shift; # exim build dir + + # could even set up several of these (e.g. for different branches) + my $self = { + buildroot => $buildroot, + eximbranch=> $branch, + bfconf => $conf, + exim => $exim + }; + bless($self, $class); + + # for each instance you create, do: + main::register_module_hooks($self,$hooks); + +} + +sub checkout +{ + my $self = shift; + my $savescmlog = shift; # array ref to the log lines + + print main::time_str(), "checking out ",__PACKAGE__,"\n" if $verbose; + + push(@$savescmlog,"Skeleton processed checkout\n"); +} + +sub setup_target +{ + my $self = shift; + + # copy the code or setup a vpath dir if supported as appropriate + + print main::time_str(), "setting up ",__PACKAGE__,"\n" if $verbose; + +} + +sub need_run +{ + my $self = shift; + my $run_needed = shift; # ref to flag + + # to force a run do: + # $$run_needed = 1; + + print main::time_str(), "checking if run needed by ",__PACKAGE__,"\n" + if $verbose; + +} + +sub configure +{ + my $self = shift; + + print main::time_str(), "configuring ",__PACKAGE__,"\n" if $verbose; +} + +sub build +{ + my $self = shift; + + print main::time_str(), "building ",__PACKAGE__,"\n" if $verbose; +} + +sub install +{ + my $self = shift; + + print main::time_str(), "installing ",__PACKAGE__,"\n" if $verbose; +} + +sub check +{ + my $self = shift; + + print main::time_str(), "checking ",__PACKAGE__,"\n" if $verbose; +} + +sub installcheck +{ + my $self = shift; + my $locale = shift; + + print main::time_str(), "installchecking $locale",__PACKAGE__,"\n" + if $verbose; +} + +sub locale_end +{ + my $self = shift; + my $locale = shift; + + print main::time_str(), "end of locale $locale processing",__PACKAGE__,"\n" + if $verbose; +} + +sub cleanup +{ + my $self = shift; + + print main::time_str(), "cleaning up ",__PACKAGE__,"\n" if $verbose > 1; +} + +1; diff --git a/EximBuild/Modules/TestUpgrade.pm b/EximBuild/Modules/TestUpgrade.pm new file mode 100644 index 0000000..33fcb58 --- /dev/null +++ b/EximBuild/Modules/TestUpgrade.pm @@ -0,0 +1,114 @@ + +# Package Namespace is hardcoded. Modules must live in +# EximBuild::Modules + +package EximBuild::Modules::TestUpgrade; + +use EximBuild::Options; +use EximBuild::SCM; + +use File::Basename; + +use strict; + +use vars qw($VERSION); $VERSION = 'REL_0.1'; + +my $hooks = { + + # 'checkout' => \&checkout, + # 'setup-target' => \&setup_target, + # 'need-run' => \&need_run, + # 'configure' => \&configure, + # 'build' => \&build, + # 'install' => \&install, + 'check' => \&check, + + # 'cleanup' => \&cleanup, +}; + +sub setup +{ + my $class = __PACKAGE__; + + my $buildroot = shift; # where we're building + my $branch = shift; # The branch of exim that's being built. + my $conf = shift; # ref to the whole config object + my $exim = shift; # exim build dir + + return unless ($branch eq 'HEAD' or $branch ge 'REL9_2'); + + die +"overly long build root $buildroot will cause upgrade problems - try something shorter than 46 chars" + if (length($buildroot) > 46); + + # could even set up several of these (e.g. for different branches) + my $self = { + buildroot => $buildroot, + eximbranch=> $branch, + bfconf => $conf, + exim => $exim + }; + bless($self, $class); + + # for each instance you create, do: + main::register_module_hooks($self,$hooks); + +} + +sub check +{ + my $self = shift; + + return unless main::step_wanted('pg_upgrade-check'); + + print main::time_str(), "checking pg_upgrade\n" if $verbose; + + my $make = $self->{bfconf}->{make}; + + local %ENV = %ENV; + delete $ENV{PGUSER}; + + (my $buildport = $ENV{EXTRA_REGRESS_OPTS}) =~ s/--port=//; + $ENV{PGPORT} = $buildport; + + my @checklog; + + if ($self->{bfconf}->{using_msvc}) + { + chdir "$self->{exim}/src/tools/msvc"; + @checklog = `perl vcregress.pl upgradecheck 2>&1`; + chdir "$self->{buildroot}/$self->{eximbranch}"; + } + else + { + my $cmd = "cd $self->{exim}/contrib/pg_upgrade && $make check"; + @checklog = `$cmd 2>&1`; + } + + my @logfiles = glob("$self->{exim}/contrib/pg_upgrade/*.log"); + foreach my $log (@logfiles) + { + my $fname = basename $log; + local $/ = undef; + my $handle; + open($handle,$log); + my $contents = <$handle>; + close($handle); + push(@checklog, + "=========================== $fname ================\n",$contents); + } + + my $status = $? >>8; + + main::writelog("check-pg_upgrade",\@checklog); + print "======== pg_upgrade check log ===========\n",@checklog + if ($verbose > 1); + main::send_result("pg_upgradeCheck",$status,\@checklog) if $status; + { + no warnings 'once'; + $main::steps_completed .= " pg_upgradeCheck"; + } + +} + +1; diff --git a/EximBuild/Options.pm b/EximBuild/Options.pm new file mode 100644 index 0000000..d0a5583 --- /dev/null +++ b/EximBuild/Options.pm @@ -0,0 +1,94 @@ + +package EximBuild::Options; + +=comment + +Copyright (c) 2003-2010, Andrew Dunstan +Copyright (c) 2013, Todd Lyons + +See accompanying License file for license details + +=cut + +# common options code for buildfarm scripts, so it stays in sync + +use strict; +use warnings; +use Getopt::Long; + +use vars qw(@option_list); + +BEGIN +{ + @option_list =qw( + $forcerun $buildconf $keepall $help + $quiet $from_source $from_source_clean $testmode + $skip_steps $only_steps $override + $nosend $nostatus $verbose + ); +} + +use Exporter (); +our (@ISA, @EXPORT, @EXPORT_OK, %EXPORT_TAGS); + +use vars qw($VERSION); $VERSION = 'REL_0.1'; + +@ISA = qw(Exporter); +@EXPORT = @option_list; +%EXPORT_TAGS = (); +@EXPORT_OK = (); + +our ( + $forcerun, $buildconf, $keepall,$help, + $quiet, $from_source,$from_source_clean, $testmode, + $skip_steps,$only_steps, $overrides, + $nosend, $nostatus, $verbose, +); + +my (%standard_options); + +%standard_options =( + 'nosend' => \$nosend, + 'config=s' => \$buildconf, + 'from-source=s' => \$from_source, + 'from-source-clean=s' => \$from_source_clean, + 'force' => \$forcerun, + 'keepall' => \$keepall, + 'verbose:i' => \$verbose, + 'nostatus' => \$nostatus, + 'test' => \$testmode, + 'help' => \$help, + 'quiet' => \$quiet, + 'skip-steps=s' => \$skip_steps, + 'only-steps=s' => \$only_steps, + 'override=s@' => \$overrides, +); + +$buildconf = "build-farm.conf"; # default value + +# extra options can be used by a wrapper program, such as +# the one that will do the global lock and election, and it will +# still have acces to what it needs to do to invoke run_build. + +sub fetch_options +{ + GetOptions(%standard_options, @_) + || die "bad command line"; + +} + +sub standard_option_list +{ + my @result = (); + foreach my $k ( keys %standard_options ) + { + my $vref = $standard_options{$k}; + next unless defined($$vref); + (my $nicekey = $k) =~ s/[=:].*//; + push(@result, "--$nicekey"); + push(@result,$$vref) if $$vref && $k =~ /[:=]/; + } + return @result; +} + +1; diff --git a/EximBuild/SCM.pm b/EximBuild/SCM.pm new file mode 100644 index 0000000..df8862e --- /dev/null +++ b/EximBuild/SCM.pm @@ -0,0 +1,394 @@ +use strict; + +use File::Find; + +=comment + +Copyright (c) 2003-2010, Andrew Dunstan + +See accompanying License file for license details + +=cut + +########################################################################## +# +# SCM Class and subclasses for specific SCMs (currently CVS and git). +# +######################################################################### + +package EximBuild::SCM; + +use vars qw($VERSION); $VERSION = 'REL_0.1'; + +# factory function to return the right subclass +sub new +{ + my $class = shift; + my $conf = shift; + my $target = shift || 'exim'; + if (defined($conf->{scm}) && $conf->{scm} =~ /^git$/i) + { + $conf->{scm} = 'git'; + return new EximBuild::SCM::Git $conf, $target; + } + #elsif ((defined($conf->{scm}) && $conf->{scm} =~ /^cvs$/i ) + # ||$conf->{csvrepo} + # ||$conf->{cvsmethod}) + #{ + # $conf->{scm} = 'cvs'; + # return new EximBuild::SCM::CVS $conf, $target; + #} + die "only Git currently supported"; +} + +# common routine use for copying the source, called by the +# SCM objects (directly, not as class methods) +sub copy_source +{ + my $using_msvc = shift; + my $target = shift; + my $build_path = shift; + + # annoyingly, there isn't a standard perl module to do a recursive copy + # and I don't want to require use of the non-standard File::Copy::Recursive + system("cp -r $target $build_path 2>&1"); + my $status = $? >> 8; + die "copying directories: $status" if $status; + +} + +# required operations in each subclass: +# new() +# copy_source_required() +# copy_source() +# check_access() +# get_build_path() +# checkout() +# cleanup() +# find_changed() +# get_versions() +# log_id() + +################################## +# +# SCM for git +# +################################## + +package EximBuild::SCM::Git; + +use File::Copy; +use Cwd; + +sub new +{ + my $class = shift; + my $conf = shift; + my $target = shift; + my $self = {}; + $self->{gitrepo} = $conf->{scmrepo} || + "git://git.exim.org/exim.git"; + $self->{reference} = $conf->{git_reference} + if defined($conf->{git_reference}); + $self->{mirror} =( + $target eq 'exim' + ? "$conf->{build_root}/exim.git" + :"$conf->{build_root}/$target-exim.git" + )if $conf->{git_keep_mirror}; + $self->{ignore_mirror_failure} = $conf->{git_ignore_mirror_failure}; + $self->{target} = $target; + return bless $self, $class; +} + +sub copy_source_required +{ + my $self = shift; + + # always copy git + return 1; +} + +sub copy_source +{ + my $self = shift; + my $using_msvc = shift; + my $target = $self->{target}; + my $build_path = $self->{build_path}; + die "no build path" unless $build_path; + + # we don't want to copy the (very large) .git directory + # so we just move it out of the way during the copy + # there might be better ways of doing this, but this should do for now + + move "$target/.git", "./git-save"; + EximBuild::SCM::copy_source($using_msvc,$target,$build_path); + move "./git-save","$target/.git"; +} + +sub get_build_path +{ + my $self = shift; + my $use_vpath = shift; # irrelevant for git + my $target = $self->{target}; + $self->{build_path} = "$target.$$"; + return $self->{build_path}; +} + +sub check_access +{ + + # no login required? + return; +} + +sub log_id +{ + my $self = shift; + main::writelog('githead',[$self->{headref}]) + if $self->{headref}; +} + +sub checkout +{ + + my $self = shift; + my $branch = shift; + my $gitserver = $self->{gitrepo}; + my $target = $self->{target}; + my $status; + + # Msysgit does some horrible things, especially when it expects a drive + # spec and doesn't get one. So we extract it if it exists and use it + # where necessary. + my $drive = ""; + my $cwd = getcwd(); + $drive = substr($cwd,0,2) if $cwd =~ /^[A-Z]:/; + + my @gitlog; + if ($self->{mirror}) + { + + my $mirror = $target eq 'exim' ? 'exim.git' : "$target-exim.git"; + + if (-d $self->{mirror}) + { + @gitlog = `git --git-dir="$self->{mirror}" fetch 2>&1`; + $status = $self->{ignore_mirror_failure} ? 0 : $? >> 8; + } + else + { + my $char1 = substr($gitserver,0,1); + $gitserver = "$drive$gitserver" + if ( $char1 eq '/' or $char1 eq '\\'); + + # this will fail on older git versions + # workaround is to do this manually in the buildroot: + # git clone --bare $gitserver exim.git + # (cd exim.git && git remote add --mirror origin $gitserver) + # or equivalent for other targets + @gitlog = `git clone --mirror $gitserver $self->{mirror} 2>&1`; + $status = $? >>8; + } + if ($status) + { + unshift(@gitlog,"Git mirror failure:\n"); + print @gitlog if ($main::verbose); + main::send_result('Git-mirror',$status,\@gitlog); + } + } + + push @gitlog, "Git arguments:\n". + " branch=$branch gitserver=$gitserver target=$target\n\n"; + + if (-d $target) + { + chdir $target; + my @branches = `git branch 2>&1`; + unless (grep {/^\* bf_$branch$/} @branches) + { + chdir '..'; + print "Missing checked out branch bf_$branch:\n",@branches + if ($main::verbose); + unshift @branches,"Missing checked out branch bf_$branch:\n"; + main::send_result("$target-Git",$status,\@branches); + } + my @pulllog = `git pull 2>&1`; + push(@gitlog,@pulllog); + chdir '..'; + } + else + { + my $reference = + defined($self->{reference}) ?"--reference $self->{reference}" : ""; + + my $base = $self->{mirror} || $gitserver; + + my $char1 = substr($base,0,1); + $base = "$drive$base" + if ( $char1 eq '/' or $char1 eq '\\'); + + my @clonelog = `git clone -q $reference $base $target 2>&1`; + push(@gitlog,@clonelog); + $status = $? >>8; + if (!$status) + { + chdir $target; + + # make sure we don't name the new branch HEAD + # also, safer to checkout origin/master than origin/HEAD, I think + my $rbranch = $branch eq 'HEAD' ? 'master' : $branch; + my @colog = + `git checkout -b bf_$branch --track origin/$rbranch 2>&1`; + push(@gitlog,@colog); + chdir ".."; + } + } + $status = $? >>8; + print "================== git log =====================\n",@gitlog + if ($main::verbose > 1); + + # can't call writelog here because we call cleanlogs after the + # checkout stage, since we only clear out the logs if we find we need to + # do a build run. + # consequence - we don't save the git log if we don't do a run + # doesn't matter too much because if git fails we exit anyway. + + # Don't call git clean here. If the user has left stuff lying around it + # might be important to them, so instead of blowing it away just bitch + # loudly. + + chdir "$target"; + my @gitstat = `git status --porcelain 2>&1`; + chdir ".."; + + my ($headref,$refhandle); + if (open($refhandle,"$target/.git/refs/heads/bf_$branch")) + { + $headref = <$refhandle>; + chomp $headref; + close($refhandle); + $self->{headref} = $headref; + } + + main::send_result("$target-Git",$status,\@gitlog) if ($status); + unless ($main::nosend && $main::nostatus) + { + push(@gitlog,"===========",@gitstat); + main::send_result("$target-Git-Dirty",99,\@gitlog) + if (@gitstat); + } + + # if we were successful, however, we return the info so that + # we can put it in the newly cleaned logdir later on. + return \@gitlog; +} + +sub cleanup +{ + my $self = shift; + my $target = $self->{target}; + chdir $target; + system("git clean -dfxq"); + chdir ".."; +} + +# private Class level routine for getting changed file data +sub parse_log +{ + my $cmd = shift; + my @lines = `$cmd`; + chomp(@lines); + my $commit; + my $list = {}; + foreach my $line (@lines) + { + next if $line =~ /^(Author:|Date:|\s)/; + next unless $line; + if ($line =~ /^commit ([0-9a-zA-Z]+)/) + { + $commit = $1; + } + else + { + + # anything else should be a file name + $line =~ s/\s+$//; # make sure all trailing space is trimmed + $list->{$line} ||= $commit; # keep most recent commit + } + } + return $list; +} + +sub find_changed +{ + my $self = shift; + my $target = $self->{target}; + my $current_snap = shift; + my $last_run_snap = shift; + my $last_success_snap = shift || 0; + my $changed_files = shift; + my $changed_since_success = shift; + + my $cmd = qq{git --git-dir=$target/.git log -n 1 "--pretty=format:%ct"}; + $$current_snap = `$cmd` +0; + + # get the list of changed files and stash the commit data + + if ($last_run_snap) + { + if ($last_success_snap > 0 && $last_success_snap < $last_run_snap) + { + $last_success_snap++; + my $lrsscmd ="git --git-dir=$target/.git log --name-only " + ."--since=$last_success_snap --until=$last_run_snap"; + $self->{changed_since_success} = parse_log($lrsscmd); + } + else + { + $self->{changed_since_success} = {}; + } + $last_run_snap++; + my $lrscmd ="git --git-dir=$target/.git log --name-only " + ."--since=$last_run_snap"; + $self->{changed_since_last_run} = parse_log($lrscmd); + foreach my $file (keys %{$self->{changed_since_last_run}}) + { + delete $self->{changed_since_success}->{$file}; + } + } + else + { + $self->{changed_since_last_run} = {}; + } + + @$changed_files = sort keys %{$self->{changed_since_last_run}}; + @$changed_since_success = sort keys %{$self->{changed_since_success}}; +} + +sub get_versions +{ + my $self = shift; + my $flist = shift; + return unless @$flist; + my @repoversions; + + # for git we have already collected and stashed the info, so we just + # extract it from the stash. + + foreach my $file (@$flist) + { + if (exists $self->{changed_since_last_run}->{$file}) + { + my $commit = $self->{changed_since_last_run}->{$file}; + push(@repoversions,"$file $commit"); + } + elsif (exists $self->{changed_since_success}->{$file}) + { + my $commit = $self->{changed_since_success}->{$file}; + push(@repoversions,"$file $commit"); + } + } + @$flist = @repoversions; +} + +1; diff --git a/EximBuild/WebTxn.pm b/EximBuild/WebTxn.pm new file mode 100644 index 0000000..f71f55a --- /dev/null +++ b/EximBuild/WebTxn.pm @@ -0,0 +1,140 @@ +package EximBuild::WebTxn; + +=comment + +Copyright (c) 2003-2013, Andrew Dunstan + +See accompanying License file for license details + + +Most of this code is imported from the older standalone script run_web_txn.pl +which is now just a shell that calls the function below. It is now only +needed on older Msys installations (i.e. things running perl < 5.8). + +=cut + +use strict; + +use vars qw($VERSION); $VERSION = 'REL_0.1'; + +use vars qw($changed_this_run $changed_since_success $branch $status $stage + $animal $ts $log_data $confsum $target $verbose $secret); + +sub run_web_txn +{ + + my $lrname = shift || 'lastrun-logs'; + + # make these runtime imports so they are loaded by the perl that's running + # the procedure. On older Msys it won't be the same as the one that's + # running run_build.pl. + + require LWP; + import LWP; + require HTTP::Request::Common; + import HTTP::Request::Common; + require MIME::Base64; + import MIME::Base64; + require Digest::SHA; + import Digest::SHA qw(sha1_hex); + require Storable; + import Storable qw(nfreeze); + + my $txfname = "$lrname/web-txn.data"; + my $txdhandle; + $/=undef; + open($txdhandle,"$txfname") or die "opening $txfname: $!"; + my $txdata = <$txdhandle>; + close($txdhandle); + + eval $txdata; + if ($@) + { + warn $@; + return undef; + } + + my $tarname = "$lrname/runlogs.tgz"; + my $tardata=""; + if (open($txdhandle,$tarname)) + { + # This creates the tarball to send to the buildfarm server + binmode $txdhandle; + $tardata=<$txdhandle>; + close($txdhandle); + } + + # add our own version string and time + my $current_ts = time; + my $webscriptversion = "'web_script_version' => '$VERSION',\n"; + my $cts = "'current_ts' => $current_ts,\n"; + + # $2 here helps us to preserve the nice spacing from Data::Dumper + my $scriptline = "((.*)'script_version' => '(REL_)?\\d+\\.\\d+',\n)"; + $confsum =~ s/$scriptline/$1$2$webscriptversion$2$cts/; + my $sconf = $confsum; + $sconf =~ s/.*(\$Script_Config)/$1/ms; + my $Script_Config; + eval $sconf; + + # very modern Storable modules choke on regexes + # the server has no need of them anyway, so just chop them out + # they are still there in the text version used for reporting + foreach my $k ( keys %$Script_Config ) + { + delete $Script_Config->{$k} + if ref($Script_Config->{$k}) eq q(Regexp); + } + my $frozen_sconf = nfreeze($Script_Config); + + # make the base64 data escape-proof; = is probably ok but no harm done + # this ensures that what is seen at the other end is EXACTLY what we + # see when we calculate the signature + + map{ $_=encode_base64($_,""); tr/+=/$@/; }( + $log_data,$confsum,$changed_this_run,$changed_since_success,$tardata, + $frozen_sconf + ); + + my $content = + "changed_files=$changed_this_run&" + . "changed_since_success=$changed_since_success&" + ."branch=$branch&res=$status&stage=$stage&animal=$animal&ts=$ts" + ."&log=$log_data&conf=$confsum"; + my $sig= sha1_hex($content,$secret); + + $content .= "&frozen_sconf=$frozen_sconf"; + + if ($tardata) + { + $content .= "&logtar=$tardata"; + } + + my $ua = new LWP::UserAgent; + $ua->agent("Exim Build Farm Reporter"); + if (my $proxy = $ENV{BF_PROXY}) + { + $ua->proxy('http',$proxy); + } + + my $request=HTTP::Request->new(POST => "$target/$sig"); + $request->content_type("application/x-www-form-urlencoded"); + $request->content($content); + + my $response=$ua->request($request); + + unless ($response->is_success) + { + print + "Query for: stage=$stage&animal=$animal&ts=$ts\n", + "Target: $target/$sig\n"; + print "Status Line: ",$response->status_line,"\n"; + print "Content: \n", $response->content,"\n" + if ($verbose && $response->content); + return undef; + } + + return 1; +} + +1; diff --git a/License.PG b/License.PG new file mode 100644 index 0000000..646a69b --- /dev/null +++ b/License.PG @@ -0,0 +1,20 @@ +This software, the PostgreSQL Build Farm Client, is released under the terms +of the PostgreSQL License. + +Copyright (c) 2003-2010, Andrew Dunstan + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose, without fee, and without a written agreement +is hereby granted, provided that the above copyright notice and this paragraph +and the following two paragraphs appear in all copies. + +IN NO EVENT SHALL Andrew Dunstan BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING +OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF Andrew Dunstan +HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Andrew Dunstan SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, +AND Andrew Dunstan HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, +UPDATES, ENHANCEMENTS, OR MODIFICATIONS. diff --git a/README b/README new file mode 100644 index 0000000..191d334 --- /dev/null +++ b/README @@ -0,0 +1,5 @@ +This is based off the code to run a client member of the PostgreSQL Build Farm, +adapted to run a client member of the Exim BuildFarm. + +See License.PG file for original License and Copyright details. + diff --git a/build-farm.conf.template b/build-farm.conf.template new file mode 100644 index 0000000..f169992 --- /dev/null +++ b/build-farm.conf.template @@ -0,0 +1,222 @@ + +# -*-perl-*- hey - emacs - this is a perl file + +=comment + +Copyright (c) 2003-2010, Andrew Dunstan + +See accompanying License file for license details + +=cut + +package EximBuild; + +use strict; + +use vars qw(%conf); + +# use vars qw($VERSION); $VERSION = 'REL_0.1'; + +my $branch; +{ + no warnings qw(once); + $branch = $main::branch; +} + +# This template assumes that the user running the buildfarm process is "farm" +%conf =( + scm => 'git', + scmrepo => 'git://git.exim.org/exim.git', # default is community repo for either type + # Wishlist for future, track and build from multiple repos. + # Doesn't do anything yet. + repos => { + 'exim' => 'git://www.mrball.net/exim/exim.git', + 'exim.jgh' => 'git://www.mrball.net/exim/exim-jgh.git', + 'exim.pdp' => 'git://www.mrball.net/exim/exim-pdp.git', + 'exim.tlyons' => 'git://www.mrball.net/exim/exim-tlyons.git', + }, + # webref for diffs on server - use default for community + scm_url => undef, + # for --reference on git repo + # git_reference => undef, + # or gmake if required. can include path if necessary. + make => 'make', + # >1 for parallel "make" and "make check" steps + make_jobs => undef, + # default is "tar -z -cf runlogs.tgz *.log" + # replacement must have the same effect + # must be absolute, can be either Unix or Windows style for MSVC + tar_log_cmd => undef, + # this directory must exist before anything will work + build_root => '/home/farm/buildfarm', + # set true to do vpath builds + use_vpath => undef, + + keep_error_builds => 0, + # Linux style, use "*.core" for BSD + core_file_glob => "core*", + + # build process will connect to this URL to upload results + target => "http://eximbuild.mrball.net/cgi-bin/eximstatus.pl", + # update_personality uses this when you want to update your + # machine's info (OS, version, compiler, version) + upgrade_target => "http://eximbuild.mrball.net/cgi-bin/upgrade.pl", + + # Your host alias and password in the BuildFarm + animal => "alias_assigned_by_build_team", + secret => "secret_assigned_by_build_team", + + # if force_every is a scalar it will be used on all branches, like this + # for legacy reasons: + # force_every => 336 , # max hours between builds, undef or 0 = unforced + # we now prefer it to be a hash with branch names as the keys, like this + # + # this setting should be kept conservatively high, or not used at all - + # for the most part it's best to let the script decide if something + # has changed that requires a new run for the branch. + # + # an entry with a name of 'default' matches any branch not named + force_every => { + HEAD => 24*7, + # default => 168, + }, + + # alerts are triggered if the server doesn't see a build on a branch after + # this many hours, and then sent out every so often, + alerts => { + #HEAD => { alert_after => 72, alert_every => 24 }, + }, + + print_success => undef, + + # include / exclude pattern for files whose trigger a build + # if both are specified then they are both applied as filters + # undef means don't ignore anything. + # exclude qr[/(doc|po)/] to ignore changes to docs and po files (recommended) + # undef means null filter. + trigger_exclude => qr[/(doc|po)/], + trigger_include => undef, + + # settings for mail notices - default to notifying nobody + # these lists contain addresses to be notified + # must be complete email addresses, as the email is sent from the server + mail_events =>{ + all => [], # unconditional + fail => [], # if this build fails + change => [], # if this build causes a state change + green => [], # if this build causes a state change to/from OK + }, + + # env settings to apply within build/report process + # these settings will be seen by all the processes, including the + # configure process. + build_env =>{ + # use a dedicated cache for the build farm. this should give us + # very high hit rates and slightly faster cache searching. + CCACHE_DIR => "/home/farm/buildfarm/ccache/$branch", + + ### set this if you need a proxy setting for the + # outbound web transaction that reports the results + # BF_PROXY => 'http://my.proxy.server:portnum/', + }, + + # Environment settings on the make commandline. + # These cause full compile output and don't strip the binary. + make_args => q/FULLECHO='' STRIP_COMMAND=''/, + + # Settings to add to Local/Makefile. These will set or override + # previous definitions of variables. Example: + # LDFLAGS => '-Wall' will create LDFLAGS = '-Wall' + makefile_set =>{ + # comment out if not using ccache + CC => 'ccache gcc', + # Other examples. Could use regex in config_features instead. + #SUPPORT_TLS => 'yes', + #TLS_LIBS => '-lssl -lcrypto', + }, + # Settings to add to Local/Makefile. These will add to variables that + # are already defined earlier in the Makefile. Example: + # LDFLAGS => '-Wall' will create LDFLAGS+='-Wall' + makefile_add =>{ + # Show all warnings and errors + CFLAGS => '-Wall -Werror=format-security', + # Or enable debugging flags + #CFLAGS => '-g -Wall', + #LFLAGS => '-g', + }, + + # Another way to enable things in the Local/Makefile. + # Use a simple regex to change a default to what you want. + config_features=>[ + q(s/^# EXPERIMENTAL_PRDR.*/EXPERIMENTAL_PRDR=yes/), + ], + + # The user compiled as the master exim username. + # Requirement: The buildfarm user that runs the build farm script *MUST* have + # the group of the exim user as a secondary group. Example: + # user=>farm, group=>farm, secondary_groups=>exim + # user=>exim, group=>exim + master_exim_user => "exim", + + # Range of tests to run if enable make_test in optional steps. + #range_num_tests => '1 999', + range_num_tests => '1 999', + # Hardcode some valid version for use during make test + exim_test_version => '4.80', + + optional_steps =>{ + # which optional steps to run and when to run them + # build_docs => {min_hours_since => 24*7}, + make_test => {min_hours_since => 24*7}, + }, + + # locales to test + locales => [qw( C )], + + # Unused + config_opts =>[ + qw() + ], + + # per-branch contents of extra config for check stages. + # each branch has an array of setting lines (no \n required) + # a DEFAULT entry is used for all branches, before any + # branch-specific settings. + extra_config =>{ + DEFAULT => [ + #q(log_line_prefix = '[%c:%l] '), + #"log_connections = 'true'", + #"log_disconnections = 'true'", + #"log_statement = 'all'", + #"fsync = off" + ], + }, + + # Unused + # port number actually used will be based on this param and the branch, + # so we ensure they don't collide + base_port => 5678, + + # Unused + modules => [qw(TestUpgrade)], + +); + +if ($branch eq 'global') +{ + + $conf{branches_to_build} = 'ALL'; + + # or 'HEAD_PLUS_LATEST' or 'HEAD_PLUS_LATEST2' + # or [qw( HEAD RELx_y_STABLE etc )] + +} + +################################## +# +# Can use perl below for +# per branch processing. +# +################################## + +1; diff --git a/run_branches.pl b/run_branches.pl new file mode 100755 index 0000000..702d1a4 --- /dev/null +++ b/run_branches.pl @@ -0,0 +1,244 @@ +#!/usr/bin/perl + +=comment + +Copyright (c) 2003-2010, Andrew Dunstan + +See accompanying License file for license details + +=cut + +use vars qw($VERSION); $VERSION = 'REL_0.1'; + +use strict; +use warnings; +use Fcntl qw(:flock :seek); +use EximBuild::Options; +use File::Basename; + +my %branch_last; +sub branch_last_sort; + +my $run_build; +($run_build = $0) =~ s/run_branches/run_build/; + +my($run_all, $run_one); +my %extra_options =( + 'run-all' => \$run_all, + 'run-one' => \$run_one, +); + +# process the command line +EximBuild::Options::fetch_options(%extra_options); + +# no non-option args allowed here +die("$0: non-option arguments not permitted") + if @ARGV; + +die "only one of --run-all and --run-one permitted" + if ($run_all && $run_one); + +die "need one of --run-all and --run-one" + unless ($run_all || $run_one); + +# set up a "branch" variable for processing the config file +use vars qw($branch); +$branch = 'global'; + +# +# process config file +# +require $buildconf; + +unless ( + ( + ref $EximBuild::conf{branches_to_build} eq 'ARRAY' + &&@{$EximBuild::conf{branches_to_build}} + ) + ||$EximBuild::conf{branches_to_build} =~ + /^(ALL|HEAD_PLUS_LATEST|HEAD_PLUS_LATEST2)$/ + ) +{ + die "no branches_to_build specified in $buildconf"; +} + +my @branches; +if (ref $EximBuild::conf{branches_to_build}) +{ + @branches = @{$EximBuild::conf{branches_to_build}}; +} +elsif ($EximBuild::conf{branches_to_build} =~ + /^(ALL|HEAD_PLUS_LATEST|HEAD_PLUS_LATEST2)$/ ) +{ + + # Need to set the path here so we make sure we pick up the right perl. + # It has to be the perl that the build script would choose + # i.e. specially *not* the MinGW SDK perl that is invoked for the + # build script, which means we need to put the path back the way it was + # when we're done + my $save_path = $ENV{PATH}; + $ENV{PATH} = $EximBuild::conf{build_env}->{PATH} + if ($EximBuild::conf{build_env}->{PATH}); + (my $url = $EximBuild::conf{target}) =~s/cgi-bin.*/branches_of_interest.txt/; + my $branches_of_interest = `perl -MLWP::Simple -e "getprint(q{$url})"`; + die "getting branches of interest" unless $branches_of_interest; + $ENV{PATH} = $save_path; + push(@branches,$_)foreach (split(/\s+/,$branches_of_interest)); + #splice(@branches,0,-2) + # if $EximBuild::conf{branches_to_build} eq 'HEAD_PLUS_LATEST'; + #splice(@branches,0,-3) + # if $EximBuild::conf{branches_to_build} eq 'HEAD_PLUS_LATEST2'; +} + +@branches = apply_throttle(@branches); + +my $global_lock_dir = + $EximBuild::conf{global_lock_dir} + ||$EximBuild::conf{build_root} + ||''; + +unless ($global_lock_dir && -d $global_lock_dir) +{ + die "no global lock directory: $global_lock_dir"; +} + +# acquire the lock + +my $lockfile; + +my $lockfilename = "$global_lock_dir/GLOBAL.lck"; + +open($lockfile, ">$lockfilename") || die "opening lockfile: $!"; + +if ( !flock($lockfile,LOCK_EX|LOCK_NB) ) +{ + print "Another process holds the lock on " ."$lockfilename. Exiting.\n" + if ($verbose); + exit(0); +} + +if ($run_all) +{ + foreach my $brnch(@branches) + { + run_branch($brnch); + } +} +elsif ($run_one) +{ + + # sort the branches by the order in which they last did actual work + # then try running them in that order until one does some work + + %branch_last = map {$_ => find_last_status($_)} @branches; + foreach my $brnch(sort branch_last_sort @branches) + { + run_branch($brnch); + my $new_status = find_last_status($brnch); + last if $new_status != $branch_last{$brnch}; + } +} + +exit 0; + +########################################################## + +sub run_branch +{ + my $branch = shift; + my @args = ($run_build,EximBuild::Options::standard_option_list(), $branch); + + # Explicitly use perl from the path (and not this perl, so don't use $^X) + # This script needs to run on Cygwin with non-cygwin perl if it's running + # in tandem with AS/MinGW perl, since Cygwin perl doesn't honor locks + # the samne way, and the global lock fails. But the build script needs + # to run with the native perl, even on Cygwin, which it picks up from + # the path. (Head exploding yet?). + system("perl",@args); +} + +sub branch_last_sort +{ + return $branch_last{$a} <=> $branch_last{$b}; +} + +sub find_last_status +{ + my $brnch = shift; + my $status_file = + "$EximBuild::conf{build_root}/$brnch/$EximBuild::conf{animal}.last.status"; + return 0 unless (-e $status_file); + my $handle; + open($handle,$status_file) || dir $!; + my $ts = <$handle>; + chomp $ts; + close($handle); + return $ts + 0; +} + +sub apply_throttle +{ + my @branches = @_; + return @branches unless exists $EximBuild::conf{throttle}; + my @result; + my %throttle = %{$EximBuild::conf{throttle}}; + + # implement throttle keywords ALL !HEAD and !RECENT + my @candidates; + my $replacement; + if (exists $throttle{ALL}) + { + @candidates = @branches; + $replacement = $throttle{ALL}; + } + elsif (exists $throttle{'!HEAD'}) + { + @candidates = grep { $_ ne 'HEAD' } @branches; + $replacement = $throttle{'!HEAD'}; + } + elsif (exists $throttle{'!RECENT'}) + { + + # sort branches, make sure we get numeric major version sorting right + my @stable = grep { $_ ne 'HEAD' } @branches; + s/^REL(\d)_/0$1/ foreach (@stable); + @stable = sort @stable; + s/^REL0/REL/ foreach (@stable); + pop @stable; # remove latest + @candidates = @stable; + $replacement = $throttle{'!RECENT'}; + } + foreach my $cand (@candidates) + { + + # only supply this for the branch if there isn't already + # a throttle + $throttle{$cand} ||= $replacement; + } + + # apply throttle filters + foreach my $branch(@branches) + { + my $this_throttle = $throttle{$branch}; + unless (defined $this_throttle) + { + push(@result,$branch); + next; + } + my $minh = $this_throttle->{min_hours_since}; + my $ts = find_last_status($branch); + next + if ( $ts + && (defined $minh) + &&($minh && $minh < ((time - $ts) / 3600.0))); + if (exists $this_throttle->{allowed_hours}) + { + my @allowed_hours = split(/,/,$this_throttle->{allowed_hours}); + my $hour = (localtime(time))[2]; + next unless grep {$_ == $hour} @allowed_hours; + } + push(@result,$branch); + } + + return @result; +} diff --git a/run_build.pl b/run_build.pl new file mode 100755 index 0000000..2383c99 --- /dev/null +++ b/run_build.pl @@ -0,0 +1,1425 @@ +#!/usr/bin/perl + +=comment + +Copyright (c) 2003-2010, Andrew Dunstan + +See accompanying License file for license details + +=cut + +#################################################### + +=comment + + NAME: run_build.pl - script to run exim buildfarm + + SYNOPSIS: + + run_build.pl [option ...] [branchname] + + AUTHOR: Andrew Dunstan + + DOCUMENTATION: + + See http://wiki.exim.org/wiki/PostgreSQL_Buildfarm_Howto + + REPOSITORY: + + https://github.com/EximBuildFarm/client-code + +=cut + +################################################### + +use vars qw($VERSION); $VERSION = 'REL_0.1'; + +use strict; +use warnings; +use Config; +use Fcntl qw(:flock :seek); +use File::Path; +use File::Copy; +use File::Basename; +use File::Temp; +use File::Spec; +use IO::Handle; +use POSIX qw(:signal_h strftime); +use Data::Dumper; +use Cwd qw(abs_path getcwd); +use File::Find (); + +# save a copy of the original enviroment for reporting +# save it early to reduce the risk of prior mangling +use vars qw($orig_env); + +BEGIN +{ + $orig_env = {}; + while (my ($k,$v) = each %ENV) + { + + # report all the keys but only values for whitelisted settings + # this is to stop leaking of things like passwords + $orig_env->{$k} =( + ( + $k =~ /^PG(?!PASSWORD)|MAKE|CC|CPP|FLAG|LIBRAR|INCLUDE/ + ||$k =~/^(HOME|LOGNAME|USER|PATH|SHELL)$/ + ) + ? $v + : 'xxxxxx' + ); + } +} + +use EximBuild::SCM; +use EximBuild::Options; +use EximBuild::WebTxn; + +my %module_hooks; +my $orig_dir = getcwd(); +push @INC, $orig_dir; + +# make sure we exit nicely on any normal interrupt +# so the cleanup handler gets called. +# that lets us stop the db if it's running and +# remove the inst and exim directories +# so the next run can start clean. + +foreach my $sig (qw(INT TERM HUP QUIT)) +{ + $SIG{$sig}=\&interrupt_exit; +} + +# copy command line before processing - so we can later report it +# unmunged + +my @invocation_args = (@ARGV); + +# process the command line +EximBuild::Options::fetch_options(); + +die "only one of --from-source and --from-source-clean allowed" + if ($from_source && $from_source_clean); + +die "only one of --skip-steps and --only-steps allowed" + if ($skip_steps && $only_steps); + +$verbose=1 if (defined($verbose) && $verbose==0); +$verbose ||= 0; # stop complaints about undefined var in numeric comparison + +if ($testmode) +{ + $verbose=1 unless $verbose; + $forcerun = 1; + $nostatus = 1; + $nosend = 1; + +} + +use vars qw(%skip_steps %only_steps); +$skip_steps ||= ""; +if ($skip_steps =~ /\S/) +{ + %skip_steps = map {$_ => 1} split(/\s+/,$skip_steps); +} +$only_steps ||= ""; +if ($only_steps =~ /\S/) +{ + %only_steps = map {$_ => 1} split(/\s+/,$only_steps); +} + +# Currently only specifying a branch is actually used. +# Specifying a different repo is just a wishlist item . +use vars qw($branch $repo); +my ($arg1,$arg2) = (shift,shift); +$branch = $arg2 ? $arg2 : + $arg1 ? $arg1 : + 'HEAD'; +$repo = $arg2 ? $arg1 : 'exim'; +my $explicit_branch = $branch; + +print_help() if ($help); + +# +# process config file +# +require $buildconf; + +# get the config data into some local variables +my ( + $buildroot,$target,$animal, $print_success, + $aux_path,$trigger_exclude,$trigger_include,$secret, + $keep_errs,$force_every, $make, $optional_steps, + $use_vpath,$tar_log_cmd, $using_msvc, $extra_config, + $make_jobs, $core_file_glob + ) + =@EximBuild::conf{ + qw(build_root target animal print_success aux_path trigger_exclude + trigger_include secret keep_error_builds force_every make optional_steps + use_vpath tar_log_cmd using_msvc extra_config make_jobs core_file_glob) + }; + +#default is no parallel build +$make_jobs ||= 1; + +# default core file pattern is Linux, which used to be hardcoded +$core_file_glob ||= 'core*'; + +# legacy name +if (defined($EximBuild::conf{trigger_filter})) +{ + $trigger_exclude = $EximBuild::conf{trigger_filter}; +} + +my $scm_timeout_secs = $EximBuild::conf{scm_timeout_secs} + || $EximBuild::conf{cvs_timeout_secs}; + +print scalar(localtime()),": buildfarm run for $animal:$branch starting\n" + if $verbose; + +# Allow commandline overrides of conf variables +foreach my $arg ( @{$EximBuild::Options::overrides} ) +{ + if (my ($key,$val) = split '=', $arg) + { + $EximBuild::conf{$key} = $val; + printf "Commandline override: '$key' = '%s'\n", $EximBuild::conf{$key} + if $verbose; + } +} + +if (ref($force_every) eq 'HASH') +{ + $force_every = $force_every->{$branch} || $force_every->{default}; +} + +my $config_opts = $EximBuild::conf{config_opts}; +my $scm = new EximBuild::SCM \%EximBuild::conf; + +my $buildport; + +if (exists $EximBuild::conf{base_port}) +{ + $buildport = $EximBuild::conf{base_port}; + if ($branch =~ /REL(\d+)_(\d+)/) + { + $buildport += (10 * ($1 - 7)) + $2; + } +} +else +{ + + # support for legacy config style + $buildport = $EximBuild::conf{branch_ports}->{$branch} || 5999; +} + +$ENV{EXTRA_REGRESS_OPTS} = "--port=$buildport"; + +$tar_log_cmd ||= "tar -z -cf runlogs.tgz *.log"; + +my $logdirname = "lastrun-logs"; + +if ($from_source || $from_source_clean) +{ + $from_source ||= $from_source_clean; + die "sourceroot $from_source not absolute" + unless $from_source =~ m!^/!; + + # we need to know where the lock should go, so unless the path + # contains HEAD we require it to be specified. + die "must specify branch explicitly with from_source" + unless ($explicit_branch || $from_source =~ m!/HEAD/!); + $verbose ||= 1; + $nosend=1; + $nostatus=1; + $use_vpath = undef; + $logdirname = "fromsource-logs"; +} + +my @locales; +if ($branch eq 'HEAD' || $branch ge 'REL8_4') +{ + + # non-C locales are not regression-safe before 8.4 + @locales = @{$EximBuild::conf{locales}} if exists $EximBuild::conf{locales}; +} +unshift(@locales,'C') unless grep {$_ eq "C"} @locales; + +# sanity checks +# several people have run into these + +if ( `uname -s 2>&1 ` =~ /CYGWIN/i ) +{ + my @procs = `ps -ef`; + die "cygserver not running" unless(grep {/cygserver/} @procs); +} +my $ccachedir; +if ( $ccachedir = $EximBuild::conf{build_env}->{CCACHE_DIR} ) +{ + + # ccache is smart enough to create what you tell it is the cache dir, but + # not smart enough to build the whole path. mkpath croaks on error, so + # we just let it. + + mkpath $ccachedir; + $ccachedir = abs_path($ccachedir); +} + +if ($^V lt v5.8.0) +{ + die "no aux_path in config file" unless $aux_path; +} + +die "cannot run as root/Administrator" unless ($using_msvc or $> > 0); + +my $devnull = $using_msvc ? "nul" : "/dev/null"; + +if (!$from_source) +{ + $scm->check_access($using_msvc); +} + +my $st_prefix = "$animal."; + +my $exim = $from_source || $scm->get_build_path($use_vpath); + +# set environment from config +while (my ($envkey,$envval) = each %{$EximBuild::conf{build_env}}) +{ + $ENV{$envkey}=$envval; +} + +# change to buildroot for this branch or die +die "no buildroot" unless $buildroot; + +unless ($buildroot =~ m!^/! + or($using_msvc and $buildroot =~ m![a-z]:[/\\]!i )) +{ + die "buildroot $buildroot not absolute"; +} + +die "$buildroot does not exist or is not a directory" unless -d $buildroot; + +chdir $buildroot || die "chdir to $buildroot: $!"; + +mkdir $branch unless -d $branch; +chdir $branch || die "chdir to $buildroot/$branch"; + +# rename legacy status files/directories +foreach my $oldfile (glob("last*")) +{ + move $oldfile, "$st_prefix$oldfile"; +} + +my $branch_root = getcwd(); + +# make sure we are using GNU make +die "$make is not GNU Make - please fix config file" + unless check_make(); + +# set up modules +foreach my $module (@{$EximBuild::conf{modules}}) +{ + + # fill in the name of the module here, so use double quotes + # so everything BUT the module name needs to be escaped + my $str = qq! + require EximBuild::Modules::$module; + EximBuild::Modules::${module}::setup( + \$buildroot, + \$branch, + \\\%EximBuild::conf, + \$exim); + !; + eval $str; + + # make errors fatal + die $@ if $@; +} + +# acquire the lock + +my $lockfile; +my $have_lock; + +open($lockfile, ">builder.LCK") || die "opening lockfile: $!"; + +# only one builder at a time allowed per branch +# having another build running is not a failure, and so we do not output +# a failure message under this condition. +if ($from_source) +{ + die "acquiring lock in $buildroot/$branch/builder.LCK" + unless flock($lockfile,LOCK_EX|LOCK_NB); +} +elsif ( !flock($lockfile,LOCK_EX|LOCK_NB) ) +{ + print "Another process holds the lock on " + ."$buildroot/$branch/builder.LCK. Exiting." + if ($verbose); + exit(0); +} + +die "$buildroot/$branch has $exim or inst directories!" + if ((!$from_source && -d $exim) || -d "inst"); + +# we are OK to run if we get here +$have_lock = 1; + +# check if file present for forced run +my $forcefile = $st_prefix . "force-one-run"; +if (-e $forcefile) +{ + $forcerun = 1; + unlink $forcefile; +} + +# try to allow core files to be produced. +# another way would be for the calling environment +# to call ulimit. We do this in an eval so failure is +# not fatal. +eval{ + require BSD::Resource; + BSD::Resource->import(); + + # explicit sub calls here. using & keeps compiler happy + my $coreok = setrlimit(&RLIMIT_CORE,&RLIM_INFINITY,&RLIM_INFINITY); + die "setrlimit" unless $coreok; +}; +warn "failed to unlimit core size: $@" if $@ && $verbose > 1; + +# the time we take the snapshot +my $now=time; +my $installdir = "$buildroot/$branch/inst"; +my $dbstarted; + +my $extraconf; + +# cleanup handler for all exits +END +{ + + # clean up temp file + unlink $ENV{TEMP_CONFIG} if $extraconf; + + # if we have the lock we must already be in the build root, so + # removing things there should be safe. + # there should only be anything to cleanup if we didn't have + # success. + if ( $have_lock && -d "$exim") + { + if ($dbstarted) + { + chdir $installdir; + system(qq{"bin/pg_ctl" -D data stop >$devnull 2>&1}); + foreach my $loc (@locales) + { + next unless -d "data-$loc"; + system(qq{"bin/pg_ctl" -D "data-$loc" stop >$devnull 2>&1}); + } + chdir $branch_root; + } + if ( !$from_source && $keep_errs) + { + print "moving kept error trees\n" if $verbose; + my $timestr = strftime "%Y-%m-%d_%H-%M-%S", localtime($now); + unless (move("$exim", "eximkeep.$timestr")) + { + print "error renaming '$exim' to 'eximkeep.$timestr': $!"; + } + if (-d "inst") + { + unless(move("inst", "instkeep.$timestr")) + { + print "error renaming 'inst' to 'instkeep.$timestr': $!"; + } + } + } + else + { + rmtree("inst") unless $keepall; + rmtree("$exim") unless ($from_source || $keepall); + } + + # only keep the cache in cases of success + rmtree("$ccachedir") if $ccachedir; + } + + # get the modules to clean up after themselves + process_module_hooks('cleanup'); + + if ($have_lock) + { + if ($use_vpath) + { + + # vpath builds leave some stuff lying around in the + # source dir, unfortunately. This should clean it up. + $scm->cleanup(); + } + close($lockfile); + unlink("builder.LCK"); + } +} + +# Prepend the DEFAULT settings (if any) to any settings for the +# branch. Since we're mangling this, deep clone $extra_config +# so the config object is kept as given. This is done using +# Dumper() because the MSys DTK perl doesn't have Storable. This +# is less efficient but it hardly matters here for this shallow +# structure. + +$extra_config = eval Dumper($extra_config); + +if ($extra_config && $extra_config->{DEFAULT}) +{ + if (!exists $extra_config->{$branch}) + { + $extra_config->{$branch} = $extra_config->{DEFAULT}; + } + else + { + unshift(@{$extra_config->{$branch}}, @{$extra_config->{DEFAULT}}); + } +} + +if ($extra_config && $extra_config->{$branch}) +{ + my $tmpname; + ($extraconf,$tmpname) =File::Temp::tempfile( + 'buildfarm-XXXXXX', + DIR => File::Spec->tmpdir(), + UNLINK => 1 + ); + die 'no $tmpname!' unless $tmpname; + $ENV{TEMP_CONFIG} = $tmpname; + foreach my $line (@{$extra_config->{$branch}}) + { + print $extraconf "$line\n"; + } + autoflush $extraconf 1; +} + +use vars qw($steps_completed); +$steps_completed = ""; + +my @changed_files; +my @changed_since_success; +my $last_status; +my $last_run_snap; +my $last_success_snap; +my $current_snap; +my @filtered_files; +my $savescmlog = ""; + +if ($from_source_clean) +{ + print time_str(),"cleaning source in $exim ...\n"; + clean_from_source(); +} +elsif (!$from_source) +{ + + # see if we need to run the tests (i.e. if either something has changed or + # we have gone over the force_every heartbeat time) + + print time_str(),"checking out source ...\n" if $verbose; + + my $timeout_pid; + + $timeout_pid = spawn(\&scm_timeout,$scm_timeout_secs) + if $scm_timeout_secs; + + $savescmlog = $scm->checkout($branch); + $steps_completed = "SCM-checkout"; + + process_module_hooks('checkout',$savescmlog); + + if ($timeout_pid) + { + + # don't kill me, I finished in time + if (kill(SIGTERM, $timeout_pid)) + { + + # reap the zombie + waitpid($timeout_pid,0); + } + } + + print time_str(),"checking if build run needed ...\n" if $verbose; + + # transition to new time processing + unlink "last.success"; + + # get the timestamp data + $last_status = find_last('status') || 0; + $last_run_snap = find_last('run.snap'); + $last_success_snap = find_last('success.snap'); + $forcerun = 1 unless (defined($last_run_snap)); + + # updated by find_changed to last mtime of any file in the repo + $current_snap=0; + + # see if we need to force a build + $last_status = 0 + if ( $last_status + && $force_every + &&$last_status+($force_every*3600) < $now); + $last_status = 0 if $forcerun; + + # see what's changed since the last time we did work + $scm->find_changed(\$current_snap,$last_run_snap, $last_success_snap, + \@changed_files,\@changed_since_success); + + #ignore changes to files specified by the trigger exclude filter, if any + if (defined($trigger_exclude)) + { + @filtered_files = grep { !m[$trigger_exclude] } @changed_files; + } + else + { + @filtered_files = @changed_files; + } + + #ignore changes to files NOT specified by the trigger include filter, if any + if (defined($trigger_include)) + { + @filtered_files = grep { m[$trigger_include] } @filtered_files; + } + + my $modules_need_run; + + process_module_hooks('need-run',\$modules_need_run); + + # if no build required do nothing + if ($last_status && !@filtered_files && !$modules_need_run) + { + print time_str(), + "No build required: last status = ",scalar(gmtime($last_status)), + " GMT, current snapshot = ",scalar(gmtime($current_snap))," GMT,", + " changed files = ",scalar(@filtered_files),"\n" + if $verbose; + rmtree("$exim"); + exit 0; + } + + # get version info on both changed files sets + # XXX modules support? + + $scm->get_versions(\@changed_files); + $scm->get_versions(\@changed_since_success); + +} # end of unless ($from_source) + +cleanlogs(); + +writelog('SCM-checkout',$savescmlog) unless $from_source; +$scm->log_id() unless $from_source; + +# copy/create according to vpath/scm settings + +if ($use_vpath) +{ + print time_str(),"creating vpath build dir $exim ...\n" if $verbose; + mkdir $exim || die "making $exim: $!"; +} +elsif (!$from_source && $scm->copy_source_required()) +{ + print time_str(),"copying source to $exim ...\n" if $verbose; + + $scm->copy_source($using_msvc); +} + +process_module_hooks('setup-target'); + +# start working + +set_last('status',$now) unless $nostatus; +set_last('run.snap',$current_snap) unless $nostatus; + +my $started_times = 0; +print time_str(),"running configure ...\n" if $verbose; + +# each of these routines will call send_result, which calls exit, +# on any error, so each step depends on success in the previous +# steps. +configure(); + +make(); + +display_features(); + +make_test() if (check_optional_step('make_test')); + +make_doc() if (check_optional_step('build_docs')); + +##check_port_is_ok($buildport,'Post'); + +# if we get here everything went fine ... + +my $saved_config = get_config_summary(); + +rmtree("inst"); # only keep failures +rmtree("$exim") unless ($from_source || $keepall); + +print(time_str(),"OK\n") if $verbose; + +send_result("OK"); + +exit; + +############## end of main program ########################### + +sub print_help +{ + print qq! +usage: $0 [options] [branch] + + where options are one or more of: + + --nosend = don't send results + --nostatus = don't set status files + --force = force a build run (ignore status files) + --from-source=/path = use source in path, not from SCM + or + --from-source-clean=/path = same as --from-source, run make distclean first + --config=/path/to/file = alternative location for config file + --keepall = keep directories if an error occurs + --verbose[=n] = verbosity (default 1) 2 or more = huge output. + --quiet = suppress normal error message + --test = short for --nosend --nostatus --verbose --force + --skip-steps=list = skip certain steps + --only-steps=list = only do certain steps, not allowed with skip-steps + +Default branch is HEAD. Usually only the --config option should be necessary. + +!; + exit(0); +} + +sub time_str +{ + my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); + return sprintf("[%.2d:%.2d:%.2d] ",$hour, $min, $sec); +} + +sub step_wanted +{ + my $step = shift; + return $only_steps{$step} if $only_steps; + return !$skip_steps{$step} if $skip_steps; + return 1; # default is everything is wanted +} + +sub register_module_hooks +{ + my $who = shift; + my $what = shift; + while (my ($hook,$func) = each %$what) + { + $module_hooks{$hook} ||= []; + push(@{$module_hooks{$hook}},[$func,$who]); + } +} + +sub process_module_hooks +{ + my $hook = shift; + + # pass remaining args (if any) to module func + foreach my $module (@{$module_hooks{$hook}}) + { + my ($func,$module_instance) = @$module; + &$func($module_instance, @_); + } +} + +sub check_optional_step +{ + my $step = shift; + my $oconf; + my $shandle; + + return undef unless ref($oconf = $optional_steps->{$step}); + if ($oconf->{branches}) + { + return undef unless grep {$_ eq $branch} @{$oconf->{branches}}; + } + + my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =localtime(time); + return undef if (exists $oconf->{min_hour} && $hour < $oconf->{min_hour}); + return undef if (exists $oconf->{max_hour} && $hour > $oconf->{max_hour}); + return undef if (exists $oconf->{dow} + &&grep {$_ eq $wday} @{$oconf->{dow}}); + + my $last_step = $last_status = find_last("$step") || 0; + + return undef unless ($forcerun || + time >$last_step + (3600 * $oconf->{min_hours_since})); + set_last("$step") unless $nostatus; + + return 1; +} + +sub clean_from_source +{ + if (-e "$exim/GNUmakefile") + { + + # fixme for MSVC + my @makeout = `cd $exim && $make distclean 2>&1`; + my $status = $? >>8; + writelog('distclean',\@makeout); + print "======== distclean log ===========\n",@makeout if ($verbose > 1); + send_result('distclean',$status,\@makeout) if $status; + } +} + +sub interrupt_exit +{ + my $signame = shift; + print "Exiting on signal $signame\n"; + exit(1); +} + +sub cleanlogs +{ + my $lrname = $st_prefix . $logdirname; + rmtree("$lrname"); + mkdir "$lrname" || die "can't make $lrname dir: $!"; +} + +sub writelog +{ + my $stage = shift; + my $loglines = shift; + my $handle; + my $lrname = $st_prefix . $logdirname; + open($handle,">$lrname/$stage.log") || die $!; + print $handle @$loglines; + close($handle); +} + +sub display_features +{ + return unless step_wanted('features'); + my @out = `cd $exim + src/build-*/exim -C test/confs/0000 -bV `; + my $status = $? >>8; + writelog('features',\@out); + print "======== features log ===========\n",@out if ($verbose > 1); + send_result('Features',$status,\@out) if $status; + $steps_completed .= " Features"; +} + +sub check_make +{ + my @out = `$make -v 2>&1`; + return undef unless ($? == 0 && grep {/GNU Make/} @out); + return 'OK'; +} + +sub make +{ + return unless step_wanted('make'); + print time_str(),"running make ...\n" if $verbose; + my $make_args = join(' ',$EximBuild::conf{make_args}); + my (@makeout); + my $make_cmd = $make; + $make_cmd = "$make -j $make_jobs" + if ($make_jobs > 1 && ($branch eq 'HEAD' || $branch ge 'REL9_1')); + @makeout = `cd $exim/src && $make_cmd $make_args 2>&1`; + my $status = $? >>8; + writelog('make',\@makeout); + print "======== make log ===========\n",@makeout if ($verbose > 1); + send_result('Make',$status,\@makeout) if $status; + $steps_completed .= " Make"; +} + +sub make_doc +{ + return unless step_wanted('make-doc'); + print time_str(),"running make doc ...\n" if $verbose; + + my (@makeout); + @makeout = `cd $exim/doc/doc-docbook/ && \ + EXIM_VER="4.82" $make everything 2>&1`; + my $status = $? >>8; + writelog('make-doc',\@makeout); + print "======== make doc log ===========\n",@makeout if ($verbose > 1); + send_result('Doc',$status,\@makeout) if $status; + $steps_completed .= " Doc"; +} + +sub get_stack_trace +{ + my $bindir = shift; + my $pgdata = shift; + + # no core = no result + my @cores = glob("$pgdata/$core_file_glob"); + return () unless @cores; + + # no gdb = no result + system "gdb --version > $devnull 2>&1"; + my $status = $? >>8; + return () if $status; + + my $cmdfile = "./gdbcmd"; + my $handle; + open($handle, ">$cmdfile"); + print $handle "bt\n"; + close($handle); + + my @trace; + + foreach my $core (@cores) + { + my @onetrace = `gdb -x $cmdfile --batch $bindir/exim $core 2>&1`; + push(@trace, + "\n\n================== stack trace: $core ==================\n", + @onetrace); + } + + unlink $cmdfile; + + return @trace; +} + +sub make_install_check +{ + my $locale = shift; + return unless step_wanted('install-check'); + print time_str(),"running make installcheck ($locale)...\n" if $verbose; + + my @checklog; + unless ($using_msvc) + { + @checklog = `cd $exim/src/test/regress && $make installcheck 2>&1`; + } + else + { + chdir "$exim/src/tools/msvc"; + @checklog = `perl vcregress.pl installcheck 2>&1`; + chdir $branch_root; + } + my $status = $? >>8; + my @logfiles = + ("$exim/src/test/regress/regression.diffs","$installdir/logfile"); + foreach my $logfile(@logfiles) + { + next unless (-e $logfile ); + push(@checklog,"\n\n================== $logfile ==================\n"); + my $handle; + open($handle,$logfile); + while(<$handle>) + { + push(@checklog,$_); + } + close($handle); + } + if ($status) + { + my @trace = + get_stack_trace("$installdir/bin","$installdir/data-$locale"); + push(@checklog,@trace); + } + writelog("install-check-$locale",\@checklog); + print "======== make installcheck log ===========\n",@checklog + if ($verbose > 1); + send_result("InstallCheck-$locale",$status,\@checklog) if $status; + $steps_completed .= " InstallCheck-$locale"; +} + +sub make_isolation_check +{ + my $locale = shift; + return unless step_wanted('isolation-check'); + my @makeout; + unless ($using_msvc) + { + my $cmd = + "cd $exim/src/test/isolation && $make NO_LOCALE=1 installcheck"; + @makeout = `$cmd 2>&1`; + } + else + { + chdir "$exim/src/tools/msvc"; + @makeout = `perl vcregress.pl isolationcheck 2>&1`; + chdir $branch_root; + } + + my $status = $? >>8; + + # get the log files and the regression diffs + my @logs = glob("$exim/src/test/isolation/log/*.log"); + push(@logs,"$installdir/logfile"); + unshift(@logs,"$exim/src/test/isolation/regression.diffs") + if (-e "$exim/src/test/isolation/regression.diffs"); + foreach my $logfile (@logs) + { + push(@makeout,"\n\n================== $logfile ===================\n"); + my $handle; + open($handle,$logfile); + while(<$handle>) + { + push(@makeout,$_); + } + close($handle); + } + if ($status) + { + my @trace = + get_stack_trace("$installdir/bin","$installdir/data-$locale"); + push(@makeout,@trace); + } + writelog('isolation-check',\@makeout); + print "======== make isolation check logs ===========\n",@makeout + if ($verbose > 1); + + send_result('IsolationCheck',$status,\@makeout) if $status; + $steps_completed .= " IsolationCheck"; +} + +sub make_test +{ + return unless step_wanted('test'); + print time_str(),"running make test ...\n" if $verbose; + my $tests_range = $EximBuild::conf{range_num_tests} || "1 4"; + my @makeout; + @makeout =`(cd $exim/test + autoconf && ./configure && $make )2>&1 `; + my $status = $? >>8; + unless($status) + { + my @tmp = `(WORKDIR=\$PWD + cd $exim/test + ./runtest \$WORKDIR/$exim/src/build-*/exim -CONTINUE $tests_range )2>&1`; + $status = $? >>8; + push @makeout, @tmp; + } + writelog('test',\@makeout); + print "======== make test logs ===========\n",@makeout + if ($verbose > 1); + + send_result('Test',$status,\@makeout) if $status; + $steps_completed .= " Test"; +} + +sub make_ecpg_check +{ + return unless step_wanted('ecpg-check'); + my @makeout; + my $ecpg_dir = "$exim/src/interfaces/ecpg"; + if ($using_msvc) + { + chdir "$exim/src/tools/msvc"; + @makeout = `perl vcregress.pl ecpgcheck 2>&1`; + chdir $branch_root; + } + else + { + @makeout = `cd $ecpg_dir && $make NO_LOCALE=1 check 2>&1`; + } + my $status = $? >>8; + + # get the log files and the regression diffs + my @logs = glob("$ecpg_dir/test/log/*.log"); + unshift(@logs,"$ecpg_dir/test/regression.diffs") + if (-e "$ecpg_dir/test/regression.diffs"); + foreach my $logfile (@logs) + { + push(@makeout,"\n\n================== $logfile ===================\n"); + my $handle; + open($handle,$logfile); + while(<$handle>) + { + push(@makeout,$_); + } + close($handle); + } + if ($status) + { + my $base = "$ecpg_dir/test/regress/tmp_check"; + my @trace = + get_stack_trace("$base/install$installdir/bin", "$base/data"); + push(@makeout,@trace); + } + writelog('ecpg-check',\@makeout); + print "======== make ecpg check logs ===========\n",@makeout + if ($verbose > 1); + + send_result('ECPG-Check',$status,\@makeout) if $status; + $steps_completed .= " ECPG-Check"; +} + +sub configure +{ + + my @quoted_opts; + foreach my $c_opt (@$config_opts) + { + if ($c_opt =~ /['"]/) + { + push(@quoted_opts,$c_opt); + } + else + { + push(@quoted_opts,"'$c_opt'"); + } + } + + my $env = $EximBuild::conf{makefile_set}; + my $add = $EximBuild::conf{makefile_add}; + my $features = $EximBuild::conf{config_features}; + + my $envstr = ""; + while (my ($key,$val) = each %$env) + { + $envstr .= "$key='$val'\n"; + } + while (my ($key,$val) = each %$add) + { + $envstr .= "$key+='$val'\n"; + } + + my $conf_path = "src/src/EDITME"; + my $local_conf = "src/Local/Makefile"; + my @confout = `cd $exim; mkdir -p src/Local 2>&1`; + my @tmp = `cd $exim && cp $conf_path $local_conf 2>&1`; + my $status = $? >> 8; + push @confout, @tmp; + if ($status == 0) + { + # First, let's display some directory permissions in case + # permissions are causing problems. + my @dir = split('/',`pwd`); + chomp(@dir); + my $count = scalar @dir; + my $loop = 1; + my $dirs = ''; + while ($loop < $count) + { + my $dir = ""; + foreach my $i (0 .. $loop) + { + $dir .= $dir[$i].'/'; + } + $dirs .= " $dir"; + $loop++; + } + @tmp = `echo "Verify Directory Permissions" + ls -ld $dirs`; + push @confout, @tmp; + # Build the config file from the EDITME template + @tmp = `cd $exim && echo '$envstr' >> $local_conf`; + push @confout, @tmp; + my $exim_user = $EximBuild::conf{master_exim_user} || 'exim'; + @tmp = `echo "Hardcoded Exim user info:"; id $exim_user + cd $exim && perl -pi -e 's/^EXIM_USER=.*/EXIM_USER=$exim_user/' $local_conf`; + push @confout, @tmp; + my $me = `whoami`; chomp $me; + @tmp = `echo "Build Farm user info:"; id $me + cd $exim && perl -pi -e 's/^# CONFIGURE_OWNER=\$/CONFIGURE_OWNER=$me/' $local_conf`; + push @confout, @tmp; + @tmp = `cd $exim && perl -pi -e 's/^# TRUSTED_CONFIG_LIST=/TRUSTED_CONFIG_LIST=/' $local_conf`; + push @confout, @tmp; + @tmp = `cd $exim && perl -pi -e 's/^# WHITELIST_D_MACROS=.*/WHITELIST_D_MACROS=DIR:EXIM_PATH:AA:ACL:ACLRCPT:ACL_MAIL:ACL_PREDATA:ACL_RCPT:AFFIX:ALLOW:ARG1:ARG2:AUTHF:AUTHS:AUTH_ID_DOMAIN:BAD:BANNER:BB:BR:BRB:CERT:COM:COMMAND_USER:CONNECTCOND:CONTROL:CREQCIP:CREQMAC:CRL:CSS:D6:DATA:DCF:DDF:DEFAULTDWC:DELAY:DETAILS:DRATELIMIT:DYNAMIC_OPTION:ELI:ERROR_DETAILS:ERT:FAKE:FALLBACK:FILTER:FILTER_PREPEND_HOME:FORBID:FORBID_SMTP_CODE:FUSER:HAI:HAP:HARDLIMIT:HEADER_LINE_MAXSIZE:HEADER_MAXSIZE:HELO_MSG:HL:HOSTS:HOSTS_AVOID_TLS:HOSTS_MAX_TRY:HVH:IFACE:IGNORE_QUOTA:INC:INSERT:IP1:IP2:LAST:LDAPSERVERS:LENCHECK:LIMIT:LIST:LOG_SELECTOR:LS:MAXNM:MESSAGE_LOGS:MSIZE:NOTDAEMON:ONCE:ONLY:OPT:OPTION:ORDER:PAH:PEX:PORT:PTBC:QDG:QOLL:QUOTA:QUOTA_FILECOUNT:QWM:RCPT_MSG:REMEMBER:REQUIRE:RETRY:RETRY1:RETRY2:RETURN:RETURN_ERROR_DETAILS:REWRITE:ROUTE_DATA:RRATELIMIT:RT:S:SELECTOR:SELF:SERVER:SERVERS:SREQCIP:SREQMAC:SRV:STD:STRICT:SUB:SUBMISSION_OPTIONS:TIMEOUTDEFER:TIMES:TRUSTED:TRYCLEAR:UL:USE_SENDER:UTF8:VALUE:WMF:X:Y/' $local_conf`; + push @confout, @tmp; + @tmp = `cd $exim && perl -pi -e 's/^EXIM_MONITOR=(.*)/# EXIM_MONITOR=\$1/' $local_conf`; + push @confout, @tmp; + for my $feature ( @$features ) + { + @tmp = `cd $exim + perl -pi -e '$feature' $local_conf 2>&1 + echo "Changed feature: $feature" `; + push @confout, @tmp; + } + # Add the final build file to the display output + @tmp = `cd $exim + echo + echo "Contents of Local/Makefile:" + egrep '^[^#]' $local_conf `; + push @confout, @tmp; + # Does not matter what the Exim version is, as long as it is valid. + my $exim_ver = $EximBuild::conf{exim_test_version} || '4.82'; + `cd $exim + echo 'EXIM_RELEASE_VERSION="$exim_ver"' > src/src/version.sh + echo 'EXIM_VARIANT_VERSION=""' >> src/src/version.sh + echo 'EXIM_COMPILE_NUMBER="0"' >> src/src/version.sh`; + } + + print "======== configure output ===========\n",@confout + if ($verbose > 1); + + writelog('configure',\@confout); + + if ($status) + { + send_result('Configure',$status,\@confout); + } + + $steps_completed .= " Configure"; +} + +sub find_last +{ + my $which = shift; + my $stname = $st_prefix . "last.$which"; + my $handle; + open($handle,$stname) or return undef; + my $time = <$handle>; + close($handle); + chomp $time; + return $time + 0; +} + +sub set_last +{ + my $which = shift; + my $stname = $st_prefix . "last.$which"; + my $st_now = shift || time; + my $handle; + open($handle,">$stname") or die "opening $stname: $!"; + print $handle "$st_now\n"; + close($handle); +} + +sub send_result +{ + + # clean up temp file + $extraconf = undef; + + my $stage = shift; + + my $ts = $now || time; + my $status=shift || 0; + my $log = shift || []; + print "======== log passed to send_result ===========\n",@$log + if ($verbose > 1); + + unshift(@$log, + "Last file mtime in snapshot: ", + scalar(gmtime($current_snap)), + " GMT\n","===================================================\n") + unless ($from_source || !$current_snap); + + my $log_data = join("",@$log); + my $confsum = ""; + my $changed_this_run = ""; + my $changed_since_success = ""; + $changed_this_run = join("!",@changed_files) + if @changed_files; + $changed_since_success = join("!",@changed_since_success) + if ($stage ne 'OK' && @changed_since_success); + + if ($stage eq 'OK') + { + $confsum= $saved_config; + } + elsif ($stage !~ /CVS|Git|SCM/ ) + { + $confsum = get_config_summary(); + } + else + { + $confsum = get_script_config_dump(); + } + + my $savedata = Data::Dumper->Dump( + [ + $changed_this_run, $changed_since_success, $branch, $status,$stage, + $animal, $ts,$log_data, $confsum, $target, $verbose, $secret + ], + [ + qw(changed_this_run changed_since_success branch status stage + animal ts log_data confsum target verbose secret) + ] + ); + + my $lrname = $st_prefix . $logdirname; + + # might happen if there is a CVS failure and have never got further + mkdir $lrname unless -d $lrname; + + my $txfname = "$lrname/web-txn.data"; + my $txdhandle; + open($txdhandle,">$txfname"); + print $txdhandle $savedata; + close($txdhandle); + + if ($nosend || $stage eq 'CVS' || $stage eq 'CVS-status' ) + { + print "Branch: $branch\n"; + if ($stage eq 'OK') + { + print "All stages succeeded\n"; + set_last('success.snap',$current_snap) unless $nostatus; + exit(0); + } + else + { + print "Stage $stage failed with status $status\n"; + exit(1); + } + } + + if ($stage !~ /CVS|Git|SCM|Pre-run-port-check/ ) + { + + my @logfiles = glob("$lrname/*.log"); + my %mtimes = map { $_ => (stat $_)[9] } @logfiles; + @logfiles = + map { basename $_ }( sort { $mtimes{$a} <=> $mtimes{$b} } @logfiles ); + my $logfiles = join(' ',@logfiles); + $tar_log_cmd =~ s/\*\.log/$logfiles/; + chdir($lrname); + system("$tar_log_cmd 2>&1 "); + chdir($branch_root); + + } + else + { + + # these would be from an earlier run, since we + # do cleanlogs() after the cvs stage + # so don't send them. + unlink "$lrname/runlogs.tgz"; + } + + my $txstatus; + + # this should now only apply to older Msys installs. All others should + # be running with perl >= 5.8 since that's required to build exim + # anyway + if (!$^V or $^V lt v5.8.0) + { + + unless (-x "$aux_path/run_web_txn.pl") + { + print "Could not locate $aux_path/run_web_txn.pl\n"; + exit(1); + } + + system("$aux_path/run_web_txn.pl $lrname"); + $txstatus = $? >> 8; + } + else + { + $txstatus = EximBuild::WebTxn::run_web_txn($lrname) ? 0 : 1; + + } + + if ($txstatus) + { + print "Web txn failed with status: $txstatus\n"; + + # if the web txn fails, restore the timestamps + # so we try again the next time. + set_last('status',$last_status) unless $nostatus; + set_last('run.snap',$last_run_snap) unless $nostatus; + exit($txstatus); + } + + unless ($stage eq 'OK' || $quiet) + { + print "BuildFarm member $animal failed on $branch stage $stage\n"; + } + + # print "Success!\n",$response->content + # if $print_success; + + set_last('success.snap',$current_snap) if ($stage eq 'OK' && !$nostatus); + + exit 0; +} + +sub get_config_summary +{ + my $handle; + my $config = ""; + # unless ($using_msvc) + # { + # open($handle,"$exim/config.log") || return undef; + # my $start = undef; + # while (<$handle>) + # { + # if (!$start && /created by PostgreSQL configure/) + # { + # $start=1; + # s/It was/This file was/; + # } + # next unless $start; + # last if /Core tests/; + # next if /^\#/; + # next if /= ?/; + + # # split up long configure line + # if (m!\$.*configure.*--with! && length > 70) + # { + # my $pos = index($_," ",70); + # substr($_,$pos+1,0,"\\\n ") if ($pos > 0); + # $pos = index($_," ",140); + # substr($_,$pos+1,0,"\\\n ") if ($pos > 0); + # $pos = index($_," ",210); + # substr($_,$pos+1,0,"\\\n ") if ($pos > 0); + # } + # $config .= $_; + # } + # close($handle); + # $config .= + # "\n========================================================\n"; + # } + $config .= get_script_config_dump(); + return $config; +} + +sub get_script_config_dump +{ + my $conf = { + %EximBuild::conf, # shallow copy + script_version => $VERSION, + invocation_args => \@invocation_args, + steps_completed => $steps_completed, + orig_env => $orig_env, + }; + delete $conf->{secret}; + $Data::Dumper::Sortkeys = 1; + return Data::Dumper->Dump([$conf],['Script_Config']); +} + +sub scm_timeout +{ + my $wait_time = shift; + my $who_to_kill = getpgrp(0); + my $sig = SIGTERM; + $sig = -$sig; + print "waiting $wait_time secs to time out process $who_to_kill\n" + if $verbose; + foreach my $sig (qw(INT TERM HUP QUIT)) + { + $SIG{$sig}='DEFAULT'; + } + sleep($wait_time); + $SIG{TERM} = 'IGNORE'; # so we don't kill ourself, we're exiting anyway + # kill the whole process group + unless (kill $sig,$who_to_kill) + { + print "scm timeout kill failed\n"; + } +} + +sub spawn +{ + my $coderef = shift; + my $pid = fork; + if (defined($pid) && $pid == 0) + { + exit &$coderef(@_); + } + return $pid; +} + diff --git a/run_cron.sh b/run_cron.sh new file mode 100755 index 0000000..8a6f604 --- /dev/null +++ b/run_cron.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +if which dirname >/dev/null; then + BFDIR=`dirname $0` +elif [ "x${BFDIR}" = "x" ]; then + echo "Cannot find BuildFarm client directory. Exiting." + exit 1 +fi + +cd $BFDIR +# Update the build client if new version available +if which git >/dev/null; then + git pull +fi + +./run_branches.pl $@ diff --git a/setnotes.pl b/setnotes.pl new file mode 100755 index 0000000..d4fcb57 --- /dev/null +++ b/setnotes.pl @@ -0,0 +1,120 @@ +#!/usr/bin/perl + +=comment + +Copyright (c) 2003-2010, Andrew Dunstan + +See accompanying License file for license details + +=cut + +use vars qw($VERSION); $VERSION = 'REL_0.1'; + +use strict; +use warnings; +no warnings qw(once); # suppress spurious warning about conf structure + +use LWP; +use HTTP::Request::Common; +use MIME::Base64; +use Digest::SHA qw(sha1_hex); +use Getopt::Long; + +# copy command line before processing - so we can later report it +# unmunged + +my @invocation_args = (@ARGV); + +my $buildconf = "build-farm.conf"; # default value +my ($sys_notes,$help,$del); + +GetOptions( + 'config=s' => \$buildconf, + 'help' => \$help, + 'delete' => \$del, +)|| usage("bad command line"); + +$sys_notes = shift; + +usage("No extra args allowed") + if @_; + +usage("must not specify notes if delete flag used") + if $del && defined($sys_notes); + +usage() + if $help; + +usage("must specify notes") + unless ($del || defined($sys_notes)); + +# +# process config file +# +require $buildconf; + +my ($target,$animal,$secret) =@EximBuild::conf{qw(target animal secret)}; + +$target =~ s/eximstatus.pl/addnotes.pl/; + +# make the base64 data escape-proof; = is probably ok but no harm done +# this ensures that what is seen at the other end is EXACTLY what we +# see when we calculate the signature + +map{ $_ ||= ""; $_ = encode_base64($_,""); tr/+=/$@/; }($sys_notes); + +my $content = "animal=$animal\&sysnotes=$sys_notes"; + +my $sig= sha1_hex($content,$secret); + +# set environment from config +while (my ($envkey,$envval) = each %{$EximBuild::conf{build_env}}) +{ + $ENV{$envkey}=$envval; +} + +my $ua = new LWP::UserAgent; +$ua->agent("Exim Build Farm Reporter"); +if (my $proxy = $ENV{BF_PROXY}) +{ + $ua->proxy('http',$proxy); +} + +my $request=HTTP::Request->new(POST => "$target/$sig"); +$request->content_type("application/x-www-form-urlencoded"); +$request->content($content); + +my $response=$ua->request($request); + +unless ($response->is_success) +{ + print + "Query for: animal=$animal\n", + "Target: $target/$sig\n", + "Query Content: $content\n"; + print "Status Line: ",$response->status_line,"\n"; + print "Content: \n", $response->content,"\n"; + exit 1; +} + +exit(0); + +####################################################################### + +sub usage +{ + my $opt_message = shift; + print "$opt_message\n" if $opt_message; + print < \$buildconf, + 'help' => \$help, + 'os-version=s' => \$os_version, + 'compiler-version=s' => \$compiler_version, +)|| usage("bad command line"); + +usage("No extra args allowed") + if @_; + +usage() + if $help; + +usage("must specify at least one item to change") + unless ($os_version or $compiler_version); + +# +# process config file +# +require $buildconf; + +my ($target,$animal,$secret,$upgrade_target) = + @EximBuild::conf{qw(target animal secret upgrade_target)}; + +# default for old config files +unless ($upgrade_target) +{ + $upgrade_target = $target; + $upgrade_target =~ s/eximstatus.pl/upgrade.pl/; +} + +# make the base64 data escape-proof; = is probably ok but no harm done +# this ensures that what is seen at the other end is EXACTLY what we +# see when we calculate the signature + +map{ $_ ||= ""; $_ = encode_base64($_,""); tr/+=/$@/; } + ($os_version,$compiler_version); + +my $ts = time; + +my $content = "animal=$animal\&ts=$ts"; +$content .= "\&new_os=$os_version" if $os_version; +$content .= "\&new_compiler=$compiler_version" if $compiler_version; + +my $sig= sha1_hex($content,$secret); + +# set environment from config +while (my ($envkey,$envval) = each %{$EximBuild::conf{build_env}}) +{ + $ENV{$envkey}=$envval; +} + +my $ua = new LWP::UserAgent; +$ua->agent("Exim Build Farm Reporter"); +if (my $proxy = $ENV{BF_PROXY}) +{ + $ua->proxy('http',$proxy); +} + +my $request=HTTP::Request->new(POST => "$upgrade_target/$sig"); +$request->content_type("application/x-www-form-urlencoded"); +$request->content($content); + +my $response=$ua->request($request); + +unless ($response->is_success) +{ + print + "Query for: animal=$animal&ts=$ts\n", + "Target: $upgrade_target/$sig\n", + "Query Content: $content\n"; + print "Status Line: ",$response->status_line,"\n"; + print "Content: \n", $response->content,"\n"; + exit 1; +} + +exit(0); + +####################################################################### + +sub usage +{ + my $opt_message = shift; + print "$opt_message\n" if $opt_message; + print <