retire use of egrep

[buildfarm-client.git] / run_build

#!/usr/bin/env perl

=comment

Copyright (c) 2003-2010, Andrew Dunstan

See accompanying License file for license details

=cut

####################################################

=comment

 NAME: run_build - script to run exim buildfarm

 SYNOPSIS:

  run_build [option ...] [branchname]

 AUTHOR: Andrew Dunstan

 DOCUMENTATION:

  See http://wiki.exim.org/wiki/PostgreSQL_Buildfarm_Howto

 REPOSITORY:

  https://github.com/EximBuildFarm/client-code

=cut

###################################################

our $VERSION = 'REL_0.1';

use strict;
use warnings;
use 5.010;
use Config;
use Fcntl qw(:flock :seek);
use File::Path;
use File::Copy;
use File::Basename;
use File::Temp;
use File::Spec;
use FindBin qw'$RealBin';
use IO::Handle;
use POSIX qw(:signal_h strftime);
use Data::Dumper;
use Cwd qw(abs_path getcwd);
use File::Find ();

# save a copy of the original enviroment for reporting
# save it early to reduce the risk of prior mangling
use vars qw($orig_env);

BEGIN
{
    $orig_env = {};
    while (my ($k,$v) = each %ENV)
    {

        # report all the keys but only values for whitelisted settings
        # this is to stop leaking of things like passwords
        $orig_env->{$k} =(
            (
                    $k =~ /^PG(?!PASSWORD)|MAKE|CC|CPP|FLAG|LIBRAR|INCLUDE/
                  ||$k =~/^(HOME|LOGNAME|USER|PATH|SHELL)$/
            )
            ? $v
            : 'xxxxxx'
        );
    }
    push @INC, getcwd();
}

use EximBuild::SCM;
use EximBuild::Options;
use EximBuild::WebTxn;

if ($0 =~ /(.*)\.pl$/) {
    die "$0: Please use `@{[join ' ' => $1, @ARGV]}' instead.\n"
        if -t;
    exec $1, @ARGV;
}

if (umask != 022) {
    umask 022;
    say "$0: forced umask to 022" if -t;
}

my %module_hooks;
my $orig_dir = getcwd();

# make sure we exit nicely on any normal interrupt
# so the cleanup handler gets called.
# that lets us stop the db if it's running and
# remove the inst and exim directories
# so the next run can start clean.

foreach my $sig (qw(INT TERM HUP QUIT))
{
    $SIG{$sig}=\&interrupt_exit;
}

# copy command line before processing - so we can later report it
# unmunged

my @invocation_args = (@ARGV);

# process the command line
EximBuild::Options::fetch_options();

die "only one of --from-source and --from-source-clean allowed"
  if ($from_source && $from_source_clean);

die "only one of --skip-steps and --only-steps allowed"
  if ($skip_steps && $only_steps);

$verbose=1 if (defined($verbose) && $verbose==0);
$verbose ||= 0; # stop complaints about undefined var in numeric comparison

if ($testmode)
{
    $verbose=1 unless $verbose;
    $forcerun = 1;
    $nostatus = 1;
    $nosend = 1;

}

use vars qw(%skip_steps %only_steps);
$skip_steps ||= "";
if ($skip_steps =~ /\S/)
{
    %skip_steps = map {$_ => 1} split(/\s+/,$skip_steps);
}
$only_steps ||= "";
if ($only_steps =~ /\S/)
{
    %only_steps = map {$_ => 1} split(/(\s+|[:,])/,$only_steps);
}

# Currently only specifying a branch is actually used.
# Specifying a different repo is just a wishlist item .
use vars qw($branch $repo);
my ($arg1,$arg2) = (shift,shift);
$branch = $arg2 ? $arg2 :
          $arg1 ? $arg1 :
          'HEAD';
$repo = $arg2 ? $arg1 : 'exim';
my $explicit_branch = $branch;

print_help() if ($help);

#
# process config file
#
require $buildconf;
#use Data::Dumper;
#die Dumper \%EximBuild::conf;


# Does not seem to be necessary
#die "$0: permissions on '$EximBuild::conf{build_root}' should be >= 0775\n"
#    if -d $EximBuild::conf{build_root}
#      and ((stat $EximBuild::conf{build_root})[2] & 0775) != 0775;


# get the config data into some local variables
my (
    $buildroot,$target,$animal, $print_success,
    $aux_path,$trigger_exclude,$trigger_include,$secret,
    $keep_errs,$force_every, $make, $optional_steps,
    $use_vpath,$tar_log_cmd, $using_msvc, $extra_config,
    $make_jobs, $core_file_glob, $global_lock_dir
  )
  =@EximBuild::conf{
    qw(build_root target animal print_success aux_path trigger_exclude
      trigger_include secret keep_error_builds force_every make optional_steps
      use_vpath tar_log_cmd using_msvc extra_config make_jobs core_file_glob global_lock_dir)
  };

# This should be done more generally, for all the scripts
# including the $buildconf. For now this is duplicated
# in a similiar war in run_branches.
$global_lock_dir //= $buildroot // die "$0: need global_lock_dir\n";

die "$0: need read/write permissions on '$global_lock_dir': $!\n"
    if not -r -w $global_lock_dir;

#default is no parallel build
$make_jobs ||= 1;

# default core file pattern is Linux, which used to be hardcoded
$core_file_glob ||= 'core*';

# legacy name
if (defined($EximBuild::conf{trigger_filter}))
{
    $trigger_exclude = $EximBuild::conf{trigger_filter};
}

my  $scm_timeout_secs = $EximBuild::conf{scm_timeout_secs}
  || $EximBuild::conf{cvs_timeout_secs};

print scalar(localtime()),": buildfarm run for $animal:$branch starting\n"
  if $verbose;

# Allow commandline overrides of conf variables
foreach my $arg ( @{$EximBuild::Options::overrides} )
{
  if (my ($key,$val) = split '=', $arg)
  {
    $EximBuild::conf{$key} = $val;
    printf "Commandline override: '$key' = '%s'\n", $EximBuild::conf{$key}
      if $verbose;
  }
}

if (ref($force_every) eq 'HASH')
{
    $force_every = $force_every->{$branch} || $force_every->{default};
}

my $scm = new EximBuild::SCM \%EximBuild::conf;

my $buildport;

if (exists $EximBuild::conf{base_port})
{
    $buildport = $EximBuild::conf{base_port};
    if ($branch =~ /REL(\d+)_(\d+)/)
    {
        $buildport += (10 * ($1 - 7)) + $2;
    }
}
else
{

    # support for legacy config style
    $buildport = $EximBuild::conf{branch_ports}->{$branch} || 5999;
}

$ENV{EXTRA_REGRESS_OPTS} = "--port=$buildport";

$tar_log_cmd ||= "tar -z -cf runlogs.tgz *.log";

my $logdirname = "lastrun-logs";

if ($from_source || $from_source_clean)
{
    $from_source ||= $from_source_clean;
    die "sourceroot $from_source not absolute"
      unless $from_source =~ m!^/!;

    # we need to know where the lock should go, so unless the path
    # contains HEAD we require it to be specified.
    die "must specify branch explicitly with from_source"
      unless ($explicit_branch || $from_source =~ m!/HEAD/!);
    $verbose ||= 1;
    $nosend=1;
    $nostatus=1;
    $use_vpath = undef;
    $logdirname = "fromsource-logs";
}

my @locales;
if ($branch eq 'HEAD' || $branch ge 'REL8_4')
{

    # non-C locales are not regression-safe before 8.4
    @locales = @{$EximBuild::conf{locales}} if exists $EximBuild::conf{locales};
}
unshift(@locales,'C') unless grep {$_ eq "C"} @locales;

# sanity checks
# several people have run into these

if ( `uname -s 2>&1 ` =~ /CYGWIN/i )
{
    my @procs = `ps -ef`;
    die "cygserver not running" unless(grep {/cygserver/} @procs);
}
my $ccachedir;
if ( $ccachedir = $EximBuild::conf{build_env}->{CCACHE_DIR} )
{

    # ccache is smart enough to create what you tell it is the cache dir, but
    # not smart enough to build the whole path. mkpath croaks on error, so
    # we just let it.

    mkpath $ccachedir;
    $ccachedir = abs_path($ccachedir);
}

if ($^V lt v5.8.0)
{
    die "no aux_path in config file" unless $aux_path;
}

die "cannot run as root/Administrator" unless ($using_msvc or $> > 0);

my $devnull = $using_msvc ? "nul" : "/dev/null";

if (!$from_source)
{
    $scm->check_access($using_msvc);
}

my $st_prefix = "$animal.";

my $exim = $from_source  || $scm->get_build_path($use_vpath);

# set environment from config
while (my ($envkey,$envval) = each %{$EximBuild::conf{build_env}})
{
    $ENV{$envkey}=$envval;
}

# change to buildroot for this branch or die
die "no buildroot" unless $buildroot;

unless ($buildroot =~ m!^/!
    or($using_msvc and $buildroot =~ m![a-z]:[/\\]!i ))
{
    die "buildroot $buildroot not absolute";
}

die "$buildroot does not exist or is not a directory" unless -d $buildroot;

chdir $buildroot || die "chdir to $buildroot: $!";

mkdir $branch unless -d $branch;
chdir $branch || die "chdir to $buildroot/$branch";

# rename legacy status files/directories
foreach my $oldfile (glob("last*"))
{
    move $oldfile, "$st_prefix$oldfile";
}

my $branch_root = getcwd();

# Normally we would require GNU Make, but allow farm
# configuration to override this
die "$make is not GNU Make - please fix config file"
  unless check_make();

# set up modules
foreach my $module (@{$EximBuild::conf{modules}})
{

    # fill in the name of the module here, so use double quotes
    # so everything BUT the module name needs to be escaped
    my $str = qq!
         require EximBuild::Modules::$module;
         EximBuild::Modules::${module}::setup(
              \$buildroot,
              \$branch,
              \\\%EximBuild::conf,
              \$exim);
    !;
    eval $str;

    # make errors fatal
    die $@ if $@;
}

# acquire the lock

my $lockfile;
my $have_lock;

open($lockfile, ">builder.LCK") || die "opening lockfile: $!";

# only one builder at a time allowed per branch
# having another build running is not a failure, and so we do not output
# a failure message under this condition.
if ($from_source)
{
    die "acquiring lock in $buildroot/$branch/builder.LCK"
      unless flock($lockfile,LOCK_EX|LOCK_NB);
}
elsif ( !flock($lockfile,LOCK_EX|LOCK_NB) )
{
    print "Another process holds the lock on "
      ."$buildroot/$branch/builder.LCK. Exiting."
      if ($verbose);
    exit(0);
}

die "$buildroot/$branch has $exim or inst directories!"
  if ((!$from_source && -d $exim) || -d "inst");

# we are OK to run if we get here
$have_lock = 1;

# check if file present for forced run
my $forcefile = $st_prefix . "force-one-run";
if (-e $forcefile)
{
    $forcerun = 1;
    unlink $forcefile;
}

# try to allow core files to be produced.
# another way would be for the calling environment
# to call ulimit. We do this in an eval so failure is
# not fatal.
eval{
    require BSD::Resource;
    BSD::Resource->import();

    # explicit sub calls here. using & keeps compiler happy
    my $coreok = setrlimit(&RLIMIT_CORE,&RLIM_INFINITY,&RLIM_INFINITY);
    die "setrlimit" unless $coreok;
};
warn "failed to unlimit core size: $@" if $@ && $verbose > 1;

# the time we take the snapshot
my $now=time;
my $installdir = "$buildroot/$branch/inst";
my $dbstarted;

my $extraconf;

# cleanup handler for all exits
END
{

    # clean up temp file
    unlink $ENV{TEMP_CONFIG} if $extraconf;

    # if we have the lock we must already be in the build root, so
    # removing things there should be safe.
    # there should only be anything to cleanup if we didn't have
    # success.
    if ( $have_lock && -d "$exim")
    {
        if ($dbstarted)
        {
            chdir $installdir;
            system(qq{"bin/pg_ctl" -D data stop >$devnull 2>&1});
            foreach my $loc (@locales)
            {
                next unless -d "data-$loc";
                system(qq{"bin/pg_ctl" -D "data-$loc" stop >$devnull 2>&1});
            }
            chdir $branch_root;
        }
        if ( !$from_source && $keep_errs)
        {
            print "moving kept error trees\n" if $verbose;
            my $timestr = strftime "%Y-%m-%d_%H-%M-%S", localtime($now);
            unless (move("$exim", "eximkeep.$timestr"))
            {
                print "error renaming '$exim' to 'eximkeep.$timestr': $!";
            }
            if (-d "inst")
            {
                unless(move("inst", "instkeep.$timestr"))
                {
                    print "error renaming 'inst' to 'instkeep.$timestr': $!";
                }
            }
        }
        else
        {
            rmtree("inst") unless $keepall;
            rmtree("$exim") unless ($from_source || $keepall);
        }

        # only keep the cache in cases of success
        rmtree("$ccachedir") if $ccachedir;
    }

    # get the modules to clean up after themselves
    process_module_hooks('cleanup');

    if ($have_lock)
    {
        if ($use_vpath)
        {

            # vpath builds leave some stuff lying around in the
            # source dir, unfortunately. This should clean it up.
            $scm->cleanup();
        }
        close($lockfile);
        unlink("builder.LCK");
    }
}

# Prepend the DEFAULT settings (if any) to any settings for the
# branch. Since we're mangling this, deep clone $extra_config
# so the config object is kept as given. This is done using
# Dumper() because the MSys DTK perl doesn't have Storable. This
# is less efficient but it hardly matters here for this shallow
# structure.

$extra_config = eval Dumper($extra_config);

if ($extra_config &&  $extra_config->{DEFAULT})
{
    if (!exists  $extra_config->{$branch})
    {
        $extra_config->{$branch} =      $extra_config->{DEFAULT};
    }
    else
    {
        unshift(@{$extra_config->{$branch}}, @{$extra_config->{DEFAULT}});
    }
}

if ($extra_config && $extra_config->{$branch})
{
    my $tmpname;
    ($extraconf,$tmpname) =File::Temp::tempfile(
        'buildfarm-XXXXXX',
        DIR => File::Spec->tmpdir(),
        UNLINK => 1
    );
    die 'no $tmpname!' unless $tmpname;
    $ENV{TEMP_CONFIG} = $tmpname;
    foreach my $line (@{$extra_config->{$branch}})
    {
        print $extraconf "$line\n";
    }
    autoflush $extraconf 1;
}

use vars qw($steps_completed);
$steps_completed = "";

my @changed_files;
my @changed_since_success;
my $last_config;
my $last_status;
my $last_run_snap;
my $last_success_snap;
my $current_config;
my $current_snap;
my @filtered_files;
my $savescmlog = "";

if ($from_source_clean)
{
    print time_str(),"cleaning source in $exim ...\n";
    clean_from_source();
}
elsif (!$from_source)
{

    # see if we need to run the tests (i.e. if either something has changed or
    # we have gone over the force_every heartbeat time)

    print time_str(),"checking out source ...\n" if $verbose;

    my $timeout_pid;

    $timeout_pid = spawn(\&scm_timeout,$scm_timeout_secs)
      if $scm_timeout_secs;

    $savescmlog = $scm->checkout($branch);
    $steps_completed = "SCM-checkout";

    process_module_hooks('checkout',$savescmlog);

    if ($timeout_pid)
    {

        # don't kill me, I finished in time
        if (kill(SIGTERM, $timeout_pid))
        {

            # reap the zombie
            waitpid($timeout_pid,0);
        }
    }

    print time_str(),"checking if build run needed ...\n" if $verbose;

    # transition to new time processing
    unlink "last.success";

    # get the timestamp data
    $last_config = find_last('config') || 0;
    $last_status = find_last('status') || 0;
    $last_run_snap = find_last('run.snap');
    $last_success_snap = find_last('success.snap');
    $forcerun = 1 unless (defined($last_run_snap));

    # If config file changed, force a rebuild
    ($current_config) = (stat $orig_dir.'/'.$buildconf)[9];
    if (defined $current_config && $current_config > $last_config)
    {
      $last_status = 0;
      set_last('config',$current_config) unless $nostatus;
    }

    # updated by find_changed to last mtime of any file in the repo
    $current_snap=0;

    # see if we need to force a build
    $last_status = 0
      if ( $last_status
        && $force_every
        &&$last_status+($force_every*3600) < $now);
    $last_status = 0 if $forcerun;

    # see what's changed since the last time we did work
    $scm->find_changed(\$current_snap,$last_run_snap, $last_success_snap,
        \@changed_files,\@changed_since_success);

    #ignore changes to files specified by the trigger exclude filter, if any
    if (defined($trigger_exclude))
    {
        @filtered_files = grep { !m[$trigger_exclude] } @changed_files;
    }
    else
    {
        @filtered_files = @changed_files;
    }

    #ignore changes to files NOT specified by the trigger include filter, if any
    if (defined($trigger_include))
    {
        @filtered_files = grep { m[$trigger_include] } @filtered_files;
    }

    my $modules_need_run;

    process_module_hooks('need-run',\$modules_need_run);

    # if no build required do nothing
    if ($last_status && !@filtered_files && !$modules_need_run)
    {
        print time_str(),
          "No build required: last status = ",scalar(gmtime($last_status)),
          " GMT, current snapshot = ",scalar(gmtime($current_snap))," GMT,",
          " changed files = ",scalar(@filtered_files),"\n"
          if $verbose;
        rmtree("$exim");
        exit 0;
    }

    # get version info on both changed files sets
    # XXX modules support?

    $scm->get_versions(\@changed_files);
    $scm->get_versions(\@changed_since_success);

} # end of unless ($from_source)

cleanlogs();

writelog('SCM-checkout',$savescmlog) unless $from_source;
$scm->log_id() unless $from_source;

# copy/create according to vpath/scm settings

if ($use_vpath)
{
    print time_str(),"creating vpath build dir $exim ...\n" if $verbose;
    mkdir $exim || die "making $exim: $!";
}
elsif (!$from_source && $scm->copy_source_required())
{
    print time_str(),"copying source to $exim ...\n" if $verbose;

    $scm->copy_source($using_msvc);
}

process_module_hooks('setup-target');

# start working

set_last('status',$now) unless $nostatus;
set_last('run.snap',$current_snap) unless $nostatus;

my $started_times = 0;
print time_str(),"running configure ...\n" if $verbose;

# each of these routines will call send_result, which calls exit,
# on any error, so each step depends on success in the previous
# steps.
configure();

make();

display_features();

make_test() if (check_optional_step('test'));

make_doc() if (check_optional_step('make-doc'));

##check_port_is_ok($buildport,'Post');

# if we get here everything went fine ...

my $saved_config = get_config_summary();

rmtree("inst"); # only keep failures
rmtree("$exim") unless ($from_source || $keepall);

print(time_str(),"OK\n") if $verbose;

send_result("OK");

exit;

############## end of main program ###########################

sub print_help
{
    print qq!
usage: $0 [options] [branch]

 where options are one or more of:

  --nosend                  = don't send results
  --nostatus                = don't set status files
  --force                   = force a build run (ignore status files)
  --from-source=/path       = use source in path, not from SCM
  or
  --from-source-clean=/path = same as --from-source, run make distclean first
  --config=/path/to/file    = alternative location for config file
  --keepall                 = keep directories if an error occurs
  --verbose[=n]             = verbosity (default 1) 2 or more = huge output.
  --quiet                   = suppress normal error message
  --test                    = short for --nosend --nostatus --verbose --force
  --skip-steps=list         = skip certain steps
  --only-steps=list         = only do certain steps, not allowed with skip-steps
                              lists can be comma, colon, or space separated

Default branch is HEAD. Usually only the --config option should be necessary.

!;
    exit(0);
}

sub time_str
{
    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
    return sprintf("[%.2d:%.2d:%.2d] ",$hour, $min, $sec);
}

sub step_wanted
{
    my $step = shift;
    return $only_steps{$step} if $only_steps;
    return !$skip_steps{$step} if $skip_steps;
    return 1; # default is everything is wanted
}

sub register_module_hooks
{
    my $who = shift;
    my $what = shift;
    while (my ($hook,$func) = each %$what)
    {
        $module_hooks{$hook} ||= [];
        push(@{$module_hooks{$hook}},[$func,$who]);
    }
}

sub process_module_hooks
{
    my $hook = shift;

    # pass remaining args (if any) to module func
    foreach my $module (@{$module_hooks{$hook}})
    {
        my ($func,$module_instance) = @$module;
        &$func($module_instance, @_);
    }
}

sub check_optional_step
{
    my $step = shift;
    my $oconf;
    my $shandle;

    return undef unless ref($oconf = $optional_steps->{$step});
    if ($oconf->{branches})
    {
        return undef unless grep {$_ eq $branch} @{$oconf->{branches}};
    }

    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =localtime(time);
    return undef if (exists $oconf->{min_hour} &&  $hour < $oconf->{min_hour});
    return undef if (exists $oconf->{max_hour} &&  $hour > $oconf->{max_hour});
    return undef if (exists $oconf->{dow}
        &&grep {$_ eq $wday} @{$oconf->{dow}});

    my $last_step = $last_status = find_last("$step") || 0;
    ## If made it *to* these optional steps, we just run them and reset last time
    #return undef unless ($forcerun ||
    #                     time >$last_step + (3600 * $oconf->{min_hours_since}));
    set_last("$step") unless $nostatus;

    return 1;
}

sub clean_from_source
{
    if (-e "$exim/GNUmakefile")
    {

        # fixme for MSVC
        my @makeout = `cd $exim && $make distclean 2>&1`;
        my $status = $? >>8;
        writelog('distclean',\@makeout);
        print "======== distclean log ===========\n",@makeout if ($verbose > 1);
        send_result('distclean',$status,\@makeout) if $status;
    }
}

sub interrupt_exit
{
    my $signame = shift;
    print "Exiting on signal $signame\n";
    exit(1);
}

sub cleanlogs
{
    my $lrname = $st_prefix . $logdirname;
    rmtree("$lrname");
    mkdir "$lrname" || die "can't make $lrname dir: $!";
}

sub writelog
{
    my $stage = shift;
    my $loglines = shift;
    my $handle;
    my $lrname = $st_prefix . $logdirname;
    open($handle,">$lrname/$stage.log") || die $!;
    print $handle @$loglines;
    close($handle);
}

sub display_features
{
    return unless step_wanted('features');
    my @out = `cd $exim
               src/build-*/exim -C test/confs/0000 -bV `;
    my $status = $? >>8;
    writelog('features',\@out);
    print "======== features log ===========\n",@out if ($verbose > 1);
    send_result('Features',$status,\@out) if $status;
    $steps_completed .= " Features";
}

sub check_make
{
    # Allow farm member to configure non-GNU make
    my $non_gnu_make = $EximBuild::conf{non_gnu_make};
    if (!defined $non_gnu_make ||
        (defined $non_gnu_make && $non_gnu_make == 1)) {
      return 'OK';
    }
    my @out = `$make -v 2>&1`;
    return undef unless ($? == 0 && grep {/GNU Make/} @out);
    return 'OK';
}

sub make
{
    return unless step_wanted('make');
    print time_str(),"running make ...\n" if $verbose;
    my $make_args = join(' ',$EximBuild::conf{make_args});
    my (@makeout);
    my $make_cmd = $make;
    $make_cmd = "$make -j $make_jobs"
      if ($make_jobs > 1 && ($branch eq 'HEAD' || $branch ge 'REL9_1'));
    @makeout = `cd $exim/src && $make_cmd $make_args 2>&1`;
    my $status = $? >>8;
    writelog('make',\@makeout);
    print "======== make log ===========\n",@makeout if ($verbose > 1);
    send_result('Make',$status,\@makeout) if $status;
    $steps_completed .= " Make";
}

sub make_doc
{
    return unless step_wanted('make-doc');
    print time_str(),"running make doc ...\n" if $verbose;

    my (@makeout);
    @makeout = `cd $exim/doc/doc-docbook/ && \
                EXIM_VER="4.82" $make everything 2>&1`;
    my $status = $? >>8;
    writelog('make-doc',\@makeout);
    print "======== make doc log ===========\n",@makeout if ($verbose > 1);
    send_result('Doc',$status,\@makeout) if $status;
    $steps_completed .= " Doc";
}

sub get_stack_trace
{
    my $bindir = shift;
    my $pgdata = shift;

    # no core = no result
    my @cores = glob("$pgdata/$core_file_glob");
    return () unless @cores;

    # no gdb = no result
    system "gdb --version > $devnull 2>&1";
    my $status = $? >>8;
    return () if $status;

    my $cmdfile = "./gdbcmd";
    my $handle;
    open($handle, ">$cmdfile");
    print $handle "bt\n";
    close($handle);

    my @trace;

    foreach my $core (@cores)
    {
        my @onetrace = `gdb -x $cmdfile --batch $bindir/exim $core 2>&1`;
        push(@trace,
            "\n\n================== stack trace: $core ==================\n",
            @onetrace);
    }

    unlink $cmdfile;

    return @trace;
}

sub make_install_check
{
    my $locale = shift;
    return unless step_wanted('install-check');
    print time_str(),"running make installcheck ($locale)...\n" if $verbose;

    my @checklog;
    unless ($using_msvc)
    {
        @checklog = `cd $exim/src/test/regress && $make installcheck 2>&1`;
    }
    else
    {
        chdir "$exim/src/tools/msvc";
        @checklog = `perl vcregress.pl installcheck 2>&1`;
        chdir $branch_root;
    }
    my $status = $? >>8;
    my @logfiles =
      ("$exim/src/test/regress/regression.diffs","$installdir/logfile");
    foreach my $logfile(@logfiles)
    {
        next unless (-e $logfile );
        push(@checklog,"\n\n================== $logfile ==================\n");
        my $handle;
        open($handle,$logfile);
        while(<$handle>)
        {
            push(@checklog,$_);
        }
        close($handle);
    }
    if ($status)
    {
        my @trace =
          get_stack_trace("$installdir/bin","$installdir/data-$locale");
        push(@checklog,@trace);
    }
    writelog("install-check-$locale",\@checklog);
    print "======== make installcheck log ===========\n",@checklog
      if ($verbose > 1);
    send_result("InstallCheck-$locale",$status,\@checklog) if $status;
    $steps_completed .= " InstallCheck-$locale";
}

sub make_isolation_check
{
    my $locale = shift;
    return unless step_wanted('isolation-check');
    my @makeout;
    unless ($using_msvc)
    {
        my $cmd =
          "cd $exim/src/test/isolation && $make NO_LOCALE=1 installcheck";
        @makeout = `$cmd 2>&1`;
    }
    else
    {
        chdir "$exim/src/tools/msvc";
        @makeout = `perl vcregress.pl isolationcheck 2>&1`;
        chdir $branch_root;
    }

    my $status = $? >>8;

    # get the log files and the regression diffs
    my @logs = glob("$exim/src/test/isolation/log/*.log");
    push(@logs,"$installdir/logfile");
    unshift(@logs,"$exim/src/test/isolation/regression.diffs")
      if (-e "$exim/src/test/isolation/regression.diffs");
    foreach my $logfile (@logs)
    {
        push(@makeout,"\n\n================== $logfile ===================\n");
        my $handle;
        open($handle,$logfile);
        while(<$handle>)
        {
            push(@makeout,$_);
        }
        close($handle);
    }
    if ($status)
    {
        my @trace =
          get_stack_trace("$installdir/bin","$installdir/data-$locale");
        push(@makeout,@trace);
    }
    writelog('isolation-check',\@makeout);
    print "======== make isolation check logs ===========\n",@makeout
      if ($verbose > 1);

    send_result('IsolationCheck',$status,\@makeout) if $status;
    $steps_completed .= " IsolationCheck";
}

sub make_test
{
    return unless step_wanted('test');
    print time_str(),"running make test ...\n" if $verbose;

    my $tests_range = $EximBuild::conf{range_num_tests} || "1 4";
    # backwards compat for build-farm.conf from before this support was added
    $EximBuild::conf{test_configure_env} = {} unless exists $EximBuild::conf{test_configure_env};
    $EximBuild::conf{test_configure_args} = [] unless exists $EximBuild::conf{test_configure_args};
    $EximBuild::conf{test_run_args} = "" unless exists $EximBuild::conf{test_run_args};

    my @makeout;
    my %saved_env = %ENV;
    foreach my $k (keys %{$EximBuild::conf{test_configure_env}}) {
      $ENV{$k} = $EximBuild::conf{test_configure_env}{$k};
    }
    my $conf_args = '';
    foreach (@{$EximBuild::conf{test_configure_args}}) {
      s/'/'"'"'/g;
      $conf_args .= " '${_}'";
    }
    @makeout =`(cd $exim/test && ./configure $conf_args && $make )2>&1 `;
    %ENV = %saved_env;

    my $status = $? >>8;
    unless($status)
    {
      my @tmp = `(cd $exim/test && ./runtest -CONTINUE $EximBuild::conf{test_run_args} $tests_range )2>&1`;
      $status = $? >>8;
      push @makeout, @tmp;
      # Prepend the failed summary log outputs for ease of reading
      my $fail_summary = "$exim/test/failed-summary.log";
      if (-f $fail_summary)
      {
        @tmp = `cat $fail_summary`;
        push @tmp, "\n";
        unshift @makeout, @tmp;
        unshift @makeout, "Summary of failed tests:\n";
      }
    }
    writelog('test',\@makeout);
    send_result('Test',$status,\@makeout) if $status;

    @makeout = `cat $exim/test/run-summary.log`;
    writelog('test-results',\@makeout);

    print "======== make test logs ===========\n",@makeout
      if ($verbose > 1);
    $steps_completed .= " Test";
}

sub make_ecpg_check
{
    return unless step_wanted('ecpg-check');
    my @makeout;
    my $ecpg_dir = "$exim/src/interfaces/ecpg";
    if ($using_msvc)
    {
        chdir "$exim/src/tools/msvc";
        @makeout = `perl vcregress.pl ecpgcheck 2>&1`;
        chdir $branch_root;
    }
    else
    {
        @makeout = `cd  $ecpg_dir && $make NO_LOCALE=1 check 2>&1`;
    }
    my $status = $? >>8;

    # get the log files and the regression diffs
    my @logs = glob("$ecpg_dir/test/log/*.log");
    unshift(@logs,"$ecpg_dir/test/regression.diffs")
      if (-e "$ecpg_dir/test/regression.diffs");
    foreach my $logfile (@logs)
    {
        push(@makeout,"\n\n================== $logfile ===================\n");
        my $handle;
        open($handle,$logfile);
        while(<$handle>)
        {
            push(@makeout,$_);
        }
        close($handle);
    }
    if ($status)
    {
        my $base = "$ecpg_dir/test/regress/tmp_check";
        my @trace =
          get_stack_trace("$base/install$installdir/bin",       "$base/data");
        push(@makeout,@trace);
    }
    writelog('ecpg-check',\@makeout);
    print "======== make ecpg check logs ===========\n",@makeout
      if ($verbose > 1);

    send_result('ECPG-Check',$status,\@makeout) if $status;
    $steps_completed .= " ECPG-Check";
}

sub configure
{
    return unless step_wanted('configure');
    print time_str(),"creating configuration ...\n" if $verbose;

    my $env = $EximBuild::conf{makefile_set};
    my $add = $EximBuild::conf{makefile_add};
    my $features = $EximBuild::conf{makefile_regex};

    my $envstr = "";
    while (my ($key,$val) = each %$env)
    {
        $envstr .= "$key='$val'\n";
    }
    while (my ($key,$val) = each %$add)
    {
        $envstr .= "$key+='$val'\n";
    }

    my $conf_path = "src/src/EDITME";
    my $local_conf = "src/Local/Makefile";
    my @confout = `cd $exim; mkdir -p src/Local 2>&1`;
    my @tmp = `cd $exim && cp $conf_path $local_conf 2>&1`;
    my $status = $? >> 8;
    push @confout, @tmp;
    if ($status == 0)
    {
        # First, let's display some directory permissions in case
        # permissions are causing problems.
        my @dir = split('/',`pwd`);
        chomp(@dir);
        my $count = scalar @dir;
        my $loop = 1;
        my $dirs = '';
        while ($loop < $count)
        {
          my $dir = "";
          foreach my $i (0 .. $loop)
          {
            $dir .= $dir[$i].'/';
          }
          $dirs .= " $dir";
          $loop++;
        }
        @tmp = `echo "Verify Directory Permissions"
                ls -ld $dirs`;
        push @confout, @tmp;
        # Build the config file from the EDITME template
        @tmp = `cd $exim && echo '$envstr' >> $local_conf`;
        push @confout, @tmp;
        my $exim_user = $EximBuild::conf{master_exim_user} || 'exim';
        @tmp = `echo "Hardcoded Exim user info:"; id $exim_user
          cd $exim && perl -pi -e 's/^EXIM_USER=.*/EXIM_USER=$exim_user/' $local_conf`;
        push @confout, @tmp;
        #my $me = `whoami`; chomp $me;
        my $me = getpwuid($>) // die "$0: getpwuid($>): $!\n";
        @tmp = `echo "Build Farm user info:"; id $me
          cd $exim && perl -pi -e 's/^# CONFIGURE_OWNER=\$/CONFIGURE_OWNER=$me/' $local_conf`;
        push @confout, @tmp;
        my $testdir = `cd $exim && /bin/pwd`; chomp $testdir; $testdir .= "/test";
        my $trcf = "$testdir/trusted-configs";
        my $tecf = "$testdir/test-config";
        @tmp = `cd $exim && perl -pi -e "s%^# TRUSTED_CONFIG_LIST=.*%TRUSTED_CONFIG_LIST=$trcf%" $local_conf`;
        push @confout, @tmp;
        @tmp = `cd $exim && perl -pi -e 's/^# WHITELIST_D_MACROS=.*/WHITELIST_D_MACROS=DIR:EXIM_PATH:AA:ACL:ACLRCPT:ACL_MAIL:ACL_PREDATA:ACL_RCPT:AFFIX:ALLOW:ARG1:ARG2:AUTHF:AUTHS:AUTH_ID_DOMAIN:BAD:BANNER:BB:BR:BRB:CERT:COM:COMMAND_USER:CONNECTCOND:CONTROL:CREQCIP:CREQMAC:CRL:CSS:D6:DATA:DCF:DDF:DEFAULTDWC:DELAY:DETAILS:DRATELIMIT:DYNAMIC_OPTION:ELI:ERROR_DETAILS:ERT:FAKE:FALLBACK:FILTER:FILTER_PREPEND_HOME:FORBID:FORBID_SMTP_CODE:FUSER:HAI:HAP:HARDLIMIT:HEADER_LINE_MAXSIZE:HEADER_MAXSIZE:HELO_MSG:HL:HOSTS:HOSTS_AVOID_TLS:HOSTS_MAX_TRY:HVH:IFACE:IGNORE_QUOTA:INC:INSERT:IP1:IP2:LAST:LDAPSERVERS:LENCHECK:LIMIT:LIST:LOG_SELECTOR:LS:MAXNM:MESSAGE_LOGS:MSIZE:NOTDAEMON:ONCE:ONLY:OPT:OPTION:ORDER:PAH:PEX:PORT:PTBC:QDG:QOLL:QUOTA:QUOTA_FILECOUNT:QWM:RCPT_MSG:REMEMBER:REQUIRE:RETRY:RETRY1:RETRY2:RETURN:RETURN_ERROR_DETAILS:REWRITE:ROUTE_DATA:RRATELIMIT:RT:S:SELECTOR:SELF:SERVER:SERVERS:SREQCIP:SREQMAC:SRV:STD:STRICT:SUB:SUBMISSION_OPTIONS:TIMEOUTDEFER:TIMES:TRUSTED:TRYCLEAR:UL:USE_SENDER:UTF8:VALUE:WMF:X:Y/' $local_conf`;
        push @confout, @tmp;
        @tmp = `cd $exim && perl -pi -e 's/^EXIM_MONITOR=(.*)/# EXIM_MONITOR=\$1/' $local_conf`;
        push @confout, @tmp;
        for my $regex ( @$features )
        {
            @tmp = `cd $exim
                    perl -pi -e '$regex' $local_conf 2>&1
                    echo "Used regex: $regex" `;
            push @confout, @tmp;
        }
        # Add the final build file to the display output
        @tmp = `cd $exim
                echo
                echo "Contents of Local/Makefile:"
                grep -E '^[^#]' $local_conf `;
        push @confout, @tmp;
        # Build the config_opts array to send to the server
        chomp @tmp;
        my @config_opts = grep s/(?:LOOKUP_|EXPERIMENTAL_|SUPPORT_|USE_)(\S+)=.*/$1/,
                          @tmp;
        push @config_opts, grep s/^(?:EXIM_)(PERL|PYTHON)=.*/$1/,
                           @tmp;
        # OpenSSL doesn't have a specific USE flag
        push @config_opts, grep s/^(TLS_LIBS.*-l(ssl|crypto)).*/OPENSSL/,
                           @tmp;
        $EximBuild::conf{config_opts} = \@config_opts;

        # Does not matter what the Exim version is, as long as it is valid.
        my $exim_ver = $EximBuild::conf{exim_test_version} || '4.82';
        `cd $exim
         echo 'EXIM_RELEASE_VERSION="$exim_ver"' > src/src/version.sh
         echo 'EXIM_VARIANT_VERSION=""' >> src/src/version.sh
         echo 'EXIM_COMPILE_NUMBER="0"' >> src/src/version.sh`;

        # Create a trusted-configs list file
        @tmp = `cd $exim && echo "$tecf" > "$trcf"`;
        push @confout, @tmp;
    }

    print "======== configure output ===========\n",@confout
      if ($verbose > 1);

    writelog('configure',\@confout);

    if ($status)
    {
        send_result('Configure',$status,\@confout);
    }

    $steps_completed .= " Configure";
}

sub find_last
{
    my $which = shift;
    my $stname = $st_prefix . "last.$which";
    my $handle;
    open($handle,$stname) or return undef;
    my $time = <$handle>;
    close($handle);
    chomp $time;
    return $time + 0;
}

sub set_last
{
    my $which = shift;
    my $stname = $st_prefix . "last.$which";
    my $st_now = shift || time;
    my $handle;
    open($handle,">$stname") or die "opening $stname: $!";
    print $handle "$st_now\n";
    close($handle);
}

sub send_result
{

    # clean up temp file
    $extraconf = undef;

    my $stage = shift;

    my $ts = $now || time;
    my $status=shift || 0;
    my $log = shift || [];
    print "======== log passed to send_result ===========\n",@$log
      if ($verbose > 1);

    unshift(@$log,
        "Last file mtime in snapshot: ",
        scalar(gmtime($current_snap)),
        " GMT\n","===================================================\n")
      unless ($from_source || !$current_snap);

    my $log_data = join("",@$log);
    my $confsum = "";
    my $changed_this_run = "";
    my $changed_since_success = "";
    $changed_this_run = join("!",@changed_files)
      if @changed_files;
    $changed_since_success = join("!",@changed_since_success)
      if ($stage ne 'OK' && @changed_since_success);

    if ($stage eq 'OK')
    {
        $confsum= $saved_config;
    }
    elsif ($stage !~ /CVS|Git|SCM/ )
    {
        $confsum = get_config_summary();
    }
    else
    {
        $confsum = get_script_config_dump();
    }

    my $savedata = Data::Dumper->Dump(
        [
            $changed_this_run, $changed_since_success, $branch, $status,$stage,
            $animal, $ts,$log_data, $confsum, $target, $verbose, $secret
        ],
        [
            qw(changed_this_run changed_since_success branch status stage
              animal ts log_data confsum target verbose secret)
        ]
    );

    my $lrname = $st_prefix . $logdirname;

    # might happen if there is a CVS failure and have never got further
    mkdir $lrname unless -d $lrname;

    my $txfname = "$lrname/web-txn.data";
    my $txdhandle;
    open($txdhandle,">$txfname");
    print $txdhandle $savedata;
    close($txdhandle);

    if ($nosend || $stage eq 'CVS' || $stage eq 'CVS-status' )
    {
        print "Branch: $branch\n";
        if ($stage eq 'OK')
        {
            print "All stages succeeded\n";
            set_last('success.snap',$current_snap) unless $nostatus;
            exit(0);
        }
        else
        {
            print "Stage $stage failed with status $status\n";
            exit(1);
        }
    }

    if ($stage !~ /CVS|Git|SCM|Pre-run-port-check/ )
    {

        my @logfiles = glob("$lrname/*.log");
        my %mtimes = map { $_ => (stat $_)[9] } @logfiles;
        @logfiles =
          map { basename $_ }( sort { $mtimes{$a} <=> $mtimes{$b} } @logfiles );
        my $logfiles = join(' ',@logfiles);
        $tar_log_cmd =~ s/\*\.log/$logfiles/;
        chdir($lrname);
        system("$tar_log_cmd 2>&1 ");
        chdir($branch_root);

    }
    else
    {

        # these would be from an earlier run, since we
        # do cleanlogs() after the cvs stage
        # so don't send them.
        unlink "$lrname/runlogs.tgz";
    }

    my $txstatus;

    # this should now only apply to older Msys installs. All others should
    # be running with perl >= 5.8 since that's required to build exim
    # anyway
    if (!$^V or $^V lt v5.8.0)
    {

        unless (-x "$aux_path/run_web_txn.pl")
        {
            print "Could not locate $aux_path/run_web_txn.pl\n";
            exit(1);
        }

        system("$aux_path/run_web_txn.pl $lrname");
        $txstatus = $? >> 8;
    }
    else
    {
        $txstatus = EximBuild::WebTxn::run_web_txn($lrname) ? 0 : 1;

    }

    if ($txstatus)
    {
        print "Web txn failed with status: $txstatus\n";

        # if the web txn fails, restore the timestamps
        # so we try again the next time.
        set_last('status',$last_status) unless $nostatus;
        set_last('run.snap',$last_run_snap) unless $nostatus;
        exit($txstatus);
    }

    unless ($stage eq 'OK' || $quiet)
    {
        print "BuildFarm member $animal failed on $branch stage $stage\n";
    }

    #   print "Success!\n",$response->content
    #           if $print_success;

    set_last('success.snap',$current_snap) if ($stage eq 'OK' && !$nostatus);

    exit 0;
}

sub get_config_summary
{
    my $handle;
    my $config = "";
    # unless ($using_msvc)
    # {
    #     open($handle,"$exim/config.log") || return undef;
    #     my $start = undef;
    #     while (<$handle>)
    #     {
    #         if (!$start && /created by PostgreSQL configure/)
    #         {
    #             $start=1;
    #             s/It was/This file was/;
    #         }
    #         next unless $start;
    #         last if /Core tests/;
    #         next if /^\#/;
    #         next if /= <?unknown>?/;

    #         # split up long configure line
    #         if (m!\$.*configure.*--with! && length > 70)
    #         {
    #             my $pos = index($_," ",70);
    #             substr($_,$pos+1,0,"\\\n        ") if ($pos > 0);
    #             $pos = index($_," ",140);
    #             substr($_,$pos+1,0,"\\\n        ") if ($pos > 0);
    #             $pos = index($_," ",210);
    #             substr($_,$pos+1,0,"\\\n        ") if ($pos > 0);
    #         }
    #         $config .= $_;
    #     }
    #     close($handle);
    #     $config .=
    #       "\n========================================================\n";
    # }
    $config .= get_script_config_dump();
    return $config;
}

sub get_script_config_dump
{
    my $conf = {
        %EximBuild::conf,  # shallow copy
        script_version => $VERSION,
        invocation_args => \@invocation_args,
        steps_completed => $steps_completed,
        orig_env => $orig_env,
    };
    delete $conf->{secret};

    if ($conf->{scm} eq 'git') {
        chomp($conf->{farm}{revision} = `cd $RealBin && git describe --tags --always --dirty=+`);
        $conf->{farm}{cwd} = getcwd();
        $conf->{farm}{bindir} = $RealBin;
        $conf->{farm}{perl} = $^V;
    }

    $Data::Dumper::Sortkeys = 1;
    return  Data::Dumper->Dump([$conf],['Script_Config']);
}

sub scm_timeout
{
    my $wait_time = shift;
    my $who_to_kill = getpgrp(0);
    my $sig = SIGTERM;
    $sig = -$sig;
    print "waiting $wait_time secs to time out process $who_to_kill\n"
      if $verbose;
    foreach my $sig (qw(INT TERM HUP QUIT))
    {
        $SIG{$sig}='DEFAULT';
    }
    sleep($wait_time);
    $SIG{TERM} = 'IGNORE'; # so we don't kill ourself, we're exiting anyway
    # kill the whole process group
    unless (kill $sig,$who_to_kill)
    {
        print "scm timeout kill failed\n";
    }
}

sub spawn
{
    my $coderef = shift;
    my $pid = fork;
    if (defined($pid) && $pid == 0)
    {
        exit &$coderef(@_);
    }
    return $pid;
}