X-Git-Url: https://git.exim.org/users/jgh/exim.git/blobdiff_plain/aa3de463f5eed2b8552c61624adbdac824f80951..2c9a0e86055f1e86ca5cdde421f5f8c9a48b0194:/test/runtest

diff --git a/test/runtest b/test/runtest
index e71ee9fec..aa242d589 100755
--- a/test/runtest
+++ b/test/runtest
@@ -13,15 +13,25 @@
 # Placed in the Exim CVS: 06 February 2006                                    #
 ###############################################################################
 
+#use strict;
 require Cwd;
 use Errno;
 use FileHandle;
 use Socket;
+use Time::Local;
 
 
 # Start by initializing some global variables
 
-$testversion = "4.72 (02-Jun-10)";
+$testversion = "4.80 (08-May-12)";
+
+# This gets embedded in the D-H params filename, and the value comes
+# from asking GnuTLS for "normal", but there appears to be no way to
+# use certtool/... to ask what that value currently is.  *sigh*
+# We also clamp it because of NSS interop, see addition of tls_dh_max_bits.
+# This value is correct as of GnuTLS 2.12.18 as clamped by tls_dh_max_bits.
+# normal = 2432   tls_dh_max_bits = 2236
+$gnutls_dh_bits_normal = 2236;
 
 $cf = "bin/cf -exact";
 $cr = "\r";
@@ -65,6 +75,9 @@ $parm_port_d2 = 1226;        # Additional for daemon
 $parm_port_d3 = 1227;        # Additional for daemon
 $parm_port_d4 = 1228;        # Additional for daemon
 
+# Manually set locale
+$ENV{'LC_ALL'} = 'C';
+
 
 
 ###############################################################################
@@ -118,6 +131,12 @@ s?\bV6NET:?$parm_ipv6_test_net:?g;
 }
 
 
+##################################################
+#     Any state to be preserved across tests     #
+##################################################
+
+my $TEST_STATE = {};
+
 
 ##################################################
 #        Subroutine to tidy up and exit          #
@@ -140,6 +159,13 @@ my($spool);
 # than SIGTERM to stop it outputting "Terminated" to the terminal when not in
 # the background.
 
+if (exists $TEST_STATE->{exim_pid})
+  {
+  $pid = $TEST_STATE->{exim_pid};
+  print "Tidyup: killing wait-mode daemon pid=$pid\n";
+  system("sudo kill -SIGINT $pid");
+  }
+
 if (opendir(DIR, "spool"))
   {
   my(@spools) = sort readdir(DIR);
@@ -166,6 +192,8 @@ system("sudo /bin/rm -rf ./spool test-* ./dnszones/*")
   if ($rc == 0 && !$save_output);
 
 system("sudo /bin/rm -rf ./eximdir/*");
+
+print "\nYou were in test $test at the end there.\n\n" if defined $test;
 exit $rc if ($rc >= 0);
 die "** runtest error: $_[1]\n";
 }
@@ -196,14 +224,26 @@ return $newid;
 }
 
 
-# This is used while munging the output from exim_dumpdb. We cheat by assuming
-# that the  date always the same, and just return the number of seconds since
-# midnight.
+# This is used while munging the output from exim_dumpdb.
+# May go wrong across DST changes.
 
 sub date_seconds {
 my($day,$month,$year,$hour,$min,$sec) =
   $_[0] =~ /^(\d\d)-(\w\w\w)-(\d{4})\s(\d\d):(\d\d):(\d\d)/;
-return $hour * 60 * 60 + $min * 60 + $sec;
+my($mon);
+if   ($month =~ /Jan/) {$mon = 0;}
+elsif($month =~ /Feb/) {$mon = 1;}
+elsif($month =~ /Mar/) {$mon = 2;}
+elsif($month =~ /Apr/) {$mon = 3;}
+elsif($month =~ /May/) {$mon = 4;}
+elsif($month =~ /Jun/) {$mon = 5;}
+elsif($month =~ /Jul/) {$mon = 6;}
+elsif($month =~ /Aug/) {$mon = 7;}
+elsif($month =~ /Sep/) {$mon = 8;}
+elsif($month =~ /Oct/) {$mon = 9;}
+elsif($month =~ /Nov/) {$mon = 10;}
+elsif($month =~ /Dec/) {$mon = 11;}
+return timelocal($sec,$min,$hour,$day,$mon,$year);
 }
 
 
@@ -307,6 +347,9 @@ RESET_AFTER_EXTRA_LINE_READ:
   # But convert "name=the.local.host address=127.0.0.1" to use "localhost"
   s/name=the\.local\.host address=127\.0\.0\.1/name=localhost address=127.0.0.1/g;
 
+  # The name of the shell may vary
+  s/\s\Q$parm_shell\E\b/ ENV_SHELL/;
+
   # Replace the path to the testsuite directory
   s?\Q$parm_cwd\E?TESTSUITE?g;
 
@@ -354,9 +397,6 @@ RESET_AFTER_EXTRA_LINE_READ:
   # The message for a non-listening FIFO varies
   s/:[^:]+: while opening named pipe/: Error: while opening named pipe/;
 
-  # The name of the shell may vary
-  s/\s\Q$parm_shell\E\b/ SHELL/;
-
   # Debugging output of lists of hosts may have different sort keys
   s/sort=\S+/sort=xx/ if /^\S+ (?:\d+\.){3}\d+ mx=\S+ sort=\S+/;
 
@@ -441,6 +481,42 @@ RESET_AFTER_EXTRA_LINE_READ:
     \d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d/Exim statistics from <time> to <time>/x;
 
 
+  # ======== TLS certificate algorithms ========
+  # Test machines might have various different TLS library versions supporting
+  # different protocols; can't rely upon TLS 1.2's AES256-GCM-SHA384, so we
+  # treat the standard algorithms the same.
+  # So far, have seen:
+  #   TLSv1:AES256-SHA:256
+  #   TLSv1.2:AES256-GCM-SHA384:256
+  #   TLSv1.2:DHE-RSA-AES256-SHA:256
+  #   TLS1.2:DHE_RSA_AES_128_CBC_SHA1:128
+  # We also need to handle the ciphersuite without the TLS part present, for
+  # client-ssl's output.  We also see some older forced ciphersuites, but
+  # negotiating TLS 1.2 instead of 1.0.
+  # Mail headers (...), log-lines X=..., client-ssl output ...
+  # (and \b doesn't match between ' ' and '(' )
+
+  s/( (?: (?:\b|\s) [\(=] ) | \s )TLSv1\.2:/$1TLSv1:/xg;
+  s/\bAES256-GCM-SHA384\b/AES256-SHA/g;
+  s/\bDHE-RSA-AES256-SHA\b/AES256-SHA/g;
+
+  # GnuTLS have seen:
+  #   TLS1.2:RSA_AES_256_CBC_SHA1:256 (canonical)
+  #   TLS1.2:DHE_RSA_AES_128_CBC_SHA1:128
+  #
+  #   X=TLS1.2:DHE_RSA_AES_256_CBC_SHA256:256
+  #   X=TLS1.2:RSA_AES_256_CBC_SHA1:256
+  #   X=TLS1.1:RSA_AES_256_CBC_SHA1:256
+  #   X=TLS1.0:DHE_RSA_AES_256_CBC_SHA1:256
+  # and as stand-alone cipher:
+  #   DHE-RSA-AES256-SHA256
+  #   DHE-RSA-AES256-SHA
+  # picking latter as canonical simply because regex easier that way.
+  s/\bDHE_RSA_AES_128_CBC_SHA1:128/RSA_AES_256_CBC_SHA1:256/g;
+  s/TLS1.[012]:(DHE_)?RSA_AES_256_CBC_SHA(1|256):256/TLS1.x:xxxxRSA_AES_256_CBC_SHAnnn:256/g;
+  s/\bDHE-RSA-AES256-SHA256\b/DHE-RSA-AES256-SHA/g;
+
+
   # ======== Caller's login, uid, gid, home, gecos ========
 
   s/\Q$parm_caller_home\E/CALLER_HOME/g;   # NOTE: these must be done
@@ -598,7 +674,9 @@ RESET_AFTER_EXTRA_LINE_READ:
   # Handle only the Gnu and MacOS space, dot, plus and at-sign.  A full [[:graph:]]
   # unfortunately matches a non-ls linefull of dashes.
   # Allow the case where we've already picked out the file protection bits.
-  s/^([-d](?:[-r][-w][-SsTtx]){3})[.+@]?( +|$)/\1 /;
+  if (s/^([-d](?:[-r][-w][-SsTtx]){3})[.+@]?( +|$)/$1$2/) {
+    s/ +/ /g;
+  }
 
 
   # ======== Message sizes =========
@@ -644,7 +722,6 @@ RESET_AFTER_EXTRA_LINE_READ:
 
   s/(TLS error on connection (?:from|to) .*? \(SSL_\w+\): error:)(.*)/$1 <<detail omitted>>/;
 
-
   # ======== Maildir things ========
   # timestamp output in maildir processing
   s/(timestamp=|\(timestamp_only\): )\d+/$1ddddddd/g;
@@ -687,6 +764,12 @@ RESET_AFTER_EXTRA_LINE_READ:
   s/^\d\d\d(?=[PFS*])/ddd/;
 
 
+  # ========= Exim lookups ==================
+  # Lookups have a char which depends on the number of lookup types compiled in,
+  # in stderr output.  Replace with a "0".  Recognising this while avoiding
+  # other output is fragile; perhaps the debug output should be revised instead.
+  s%(?<!sqlite)(?<!lsearch\*@)(?<!lsearch\*)(?<!lsearch)[0-?]TESTSUITE/aux-fixed/%0TESTSUITE/aux-fixed/%g;
+
   # ==========================================================
   # Some munging is specific to the specific file types
 
@@ -771,6 +854,15 @@ RESET_AFTER_EXTRA_LINE_READ:
     # be the case
     next if /^changing group to \d+ failed: Operation not permitted/;
 
+    # We might not keep this check; rather than change all the tests, just
+    # ignore it as long as it succeeds; then we only need to change the
+    # TLS tests where tls_require_ciphers has been set.
+    if (m{^changed uid/gid: calling tls_validate_require_cipher}) {
+      my $discard = <IN>;
+      next;
+    }
+    next if /^tls_validate_require_cipher child \d+ ended: status=0x0/;
+
     # We invoke Exim with -D, so we hit this new messag as of Exim 4.73:
     next if /^macros_trusted overridden to true by whitelisting/;
 
@@ -932,7 +1024,7 @@ if ($_[1]) { $_ = "u"; print "... update forced\n"; }
 #             [4] TRUE if this is a log file whose deliveries must be sorted
 #
 # Returns:    0 comparison succeeded or differences to be ignored
-#             1 comparison failed; files were updated (=> re-compare)
+#             1 comparison failed; files may have been updated (=> re-compare)
 #
 # Does not return if the user replies "Q" to a prompt.
 
@@ -1090,9 +1182,10 @@ if (-e $sf)
   print "\n";
   for (;;)
     {
-    interact("Continue, Update & retry, Quit? [Q] ", $force_update);
+    interact("Continue, Retry, Update & retry, Quit? [Q] ", $force_update);
     tests_exit(1) if /^q?$/i;
     return 0 if /^c$/i;
+    return 1 if /^r$/i;
     last if (/^u$/i);
     }
   }
@@ -1124,7 +1217,7 @@ return 1;
 #
 # Arguments: none
 # Returns:   0 if the output compared equal
-#            1 if files were updated and the test must be re-run
+#            1 if re-run needed (files may have been updated)
 
 sub check_output{
 my($yield) = 0;
@@ -1351,17 +1444,21 @@ system("$cmd");
 #            reference to the subtest number, holding previous value
 #            reference to the expected return code value
 #            reference to where to put the command name (for messages)
+#            auxilliary information returned from a previous run
 #
 # Returns:   0 the commmand was executed inline, no subprocess was run
 #            1 a non-exim command was run and waited for
 #            2 an exim command was run and waited for
 #            3 a command was run and not waited for (daemon, server, exim_lock)
 #            4 EOF was encountered after an initial return code line
+# Optionally alse a second parameter, a hash-ref, with auxilliary information:
+#            exim_pid: pid of a run process
 
 sub run_command{
 my($testno) = $_[0];
 my($subtestref) = $_[1];
 my($commandnameref) = $_[3];
+my($aux_info) = $_[4];
 my($yield) = 1;
 
 if (/^(\d+)\s*$/)                # Handle unusual return code
@@ -1519,9 +1616,10 @@ if (/^eximstats\s+(.*)/)
 
 if (/^gnutls/)
   {
-  run_system "sudo cp -p aux-fixed/gnutls-params spool/gnutls-params;" .
-         "sudo chown $parm_eximuser:$parm_eximgroup spool/gnutls-params;" .
-         "sudo chmod 0400 spool/gnutls-params";
+  my $gen_fn = "spool/gnutls-params-$gnutls_dh_bits_normal";
+  run_system "sudo cp -p aux-fixed/gnutls-params $gen_fn;" .
+         "sudo chown $parm_eximuser:$parm_eximgroup $gen_fn;" .
+         "sudo chmod 0400 $gen_fn";
   return 1;
   }
 
@@ -1532,11 +1630,27 @@ if (/^gnutls/)
 
 if (/^killdaemon/)
   {
-  $pid = `cat $parm_cwd/spool/exim-daemon.*`;
-  run_system("sudo /bin/kill -SIGINT $pid");
-  close DAEMONCMD;                                   # Waits for process
-  run_system("sudo /bin/rm -f spool/exim-daemon.*");
-  return 1;
+  my $return_extra = {};
+  if (exists $aux_info->{exim_pid})
+    {
+    $pid = $aux_info->{exim_pid};
+    $return_extra->{exim_pid} = undef;
+    print ">> killdaemon: recovered pid $pid\n" if $debug;
+    if ($pid)
+      {
+      run_system("sudo /bin/kill -SIGINT $pid");
+      wait;
+      }
+    } else {
+    $pid = `cat $parm_cwd/spool/exim-daemon.*`;
+    if ($pid)
+      {
+      run_system("sudo /bin/kill -SIGINT $pid");
+      close DAEMONCMD;                                   # Waits for process
+      }
+    }
+    run_system("sudo /bin/rm -f spool/exim-daemon.*");
+  return (1, $return_extra);
   }
 
 
@@ -1764,6 +1878,10 @@ elsif (/^([A-Z_]+=\S+\s+)?(\d+)?\s*(sudo\s+)?exim(_\S+)?\s+(.*)$/)
 
     my($i);
     for ($i = @msglist; $i > 0; $i--) { $args =~ s/\$msg$i/$msglist[$i-1]/g; }
+    if ( $args =~ /\$msg\d/ )
+      {
+      tests_exit(-1, "Not enough messages in spool, for test $testno line $lineno\n");
+      }
     }
 
   # If -d is specified in $optargs, remove it from $args; i.e. let
@@ -1807,6 +1925,38 @@ elsif (/^([A-Z_]+=\S+\s+)?(\d+)?\s*(sudo\s+)?exim(_\S+)?\s+(.*)$/)
     select(undef, undef, undef, 0.3);             # Let the daemon get going
     return 3;                                     # Don't wait
     }
+  elsif ($cmd =~ /\s-DSERVER=wait:(\d+)\s/)
+    {
+    my $listen_port = $1;
+    my $waitmode_sock = new FileHandle;
+    if ($debug) { printf ">> wait-mode daemon: $cmd\n"; }
+    run_system("sudo mkdir spool/log 2>/dev/null");
+    run_system("sudo chown $parm_eximuser:$parm_eximgroup spool/log");
+
+    my ($s_ip,$s_port) = ('127.0.0.1', $listen_port);
+    my $sin = sockaddr_in($s_port, inet_aton($s_ip))
+        or die "** Failed packing $s_ip:$s_port\n";
+    socket($waitmode_sock, PF_INET, SOCK_STREAM, getprotobyname('tcp'))
+        or die "** Unable to open socket $s_ip:$s_port: $!\n";
+    setsockopt($waitmode_sock, SOL_SOCKET, SO_REUSEADDR, 1)
+        or die "** Unable to setsockopt(SO_REUSEADDR): $!\n";
+    bind($waitmode_sock, $sin)
+        or die "** Unable to bind socket ($s_port): $!\n";
+    listen($waitmode_sock, 5);
+    my $pid = fork();
+    if (not defined $pid) { die "** fork failed: $!\n" }
+    if (not $pid) {
+      close(STDIN);
+      open(STDIN, "<&", $waitmode_sock) or die "** dup sock to stdin failed: $!\n";
+      close($waitmode_sock);
+      print "[$$]>> ${cmd}-server\n" if ($debug);
+      exec "exec ${cmd}-server";
+      exit(1);
+    }
+    while (<SCRIPT>) { $lineno++; last if /^\*{4}\s*$/; }   # Ignore any input
+    select(undef, undef, undef, 0.3);             # Let the daemon get going
+    return (3, { exim_pid => $pid });             # Don't wait
+    }
   }
 
 
@@ -1907,7 +2057,7 @@ else
 # If the first character of the first argument is '/', the argument is taken
 # as the path to the binary.
 
-$parm_exim = (@ARGV > 0 && $ARGV[0] =~ ?^/?)? shift @ARGV : "";
+$parm_exim = (@ARGV > 0 && $ARGV[0] =~ m?^/?)? shift @ARGV : "";
 print "Exim binary is $parm_exim\n" if $parm_exim ne "";
 
 
@@ -2170,6 +2320,8 @@ unlink("$parm_cwd/test-config");
 
 if (defined $parm_support{'Content_Scanning'})
   {
+  my $sock = new FileHandle;
+
   if (system("spamc -h 2>/dev/null >/dev/null") == 0)
     {
     print "The spamc command works:\n";
@@ -2186,23 +2338,23 @@ if (defined $parm_support{'Content_Scanning'})
       {
       my $sin = sockaddr_in($sport, inet_aton($sint))
           or die "** Failed packing $sint:$sport\n";
-      socket(SOCK, PF_INET, SOCK_STREAM, getprotobyname('tcp'))
+      socket($sock, PF_INET, SOCK_STREAM, getprotobyname('tcp'))
           or die "** Unable to open socket $sint:$sport\n";
 
       local $SIG{ALRM} =
           sub { die "** Timeout while connecting to socket $sint:$sport\n"; };
       alarm(5);
-      connect(SOCK, $sin)
+      connect($sock, $sin)
           or die "** Unable to connect to socket $sint:$sport\n";
       alarm(0);
 
-      select((select(SOCK), $| = 1)[0]);
-      print SOCK "bad command\r\n";
+      select((select($sock), $| = 1)[0]);
+      print $sock "bad command\r\n";
 
       $SIG{ALRM} =
           sub { die "** Timeout while reading from socket $sint:$sport\n"; };
       alarm(10);
-      my $res = <SOCK>;
+      my $res = <$sock>;
       alarm(0);
 
       $res =~ m|^SPAMD/|
@@ -2313,18 +2465,18 @@ if (defined $parm_support{'Content_Scanning'})
             {
             die "** Unknown socket domain '$socket_domain' (should not happen)\n";
             }
-          socket(SOCK, $socket_domain, SOCK_STREAM, 0) or die "** Unable to open socket '$parm_clamsocket'\n";
+          socket($sock, $socket_domain, SOCK_STREAM, 0) or die "** Unable to open socket '$parm_clamsocket'\n";
           local $SIG{ALRM} = sub { die "** Timeout while connecting to socket '$parm_clamsocket'\n"; };
           alarm(5);
-          connect(SOCK, $socket) or die "** Unable to connect to socket '$parm_clamsocket'\n";
+          connect($sock, $socket) or die "** Unable to connect to socket '$parm_clamsocket'\n";
           alarm(0);
 
-          my $ofh = select SOCK; $| = 1; select $ofh;
-          print SOCK "PING\n";
+          my $ofh = select $sock; $| = 1; select $ofh;
+          print $sock "PING\n";
 
           $SIG{ALRM} = sub { die "** Timeout while reading from socket '$parm_clamsocket'\n"; };
           alarm(10);
-          my $res = <SOCK>;
+          my $res = <$sock>;
           alarm(0);
 
           $res =~ /PONG/ or die "** Did not get PONG from socket '$parm_clamsocket'. It said: $res\n";
@@ -2576,9 +2728,11 @@ if ($parm_hostname !~ /\./)
   print "\n*** Host name is not fully qualified: this may cause problems ***\n\n";
   }
 
-# Find the user's shell
+if ($parm_hostname =~ /[[:upper:]]/)
+  {
+  print "\n*** Host name has upper case characters: this may cause problems ***\n\n";
+  }
 
-$parm_shell = $ENV{'SHELL'};
 
 
 ##################################################
@@ -2640,7 +2794,7 @@ system("sudo cp eximdir/exim eximdir/exim_exim;" .
 # Certain of the tests make use of some of Exim's utilities. We do not need
 # to be root to copy these.
 
-($parm_exim_dir) = $parm_exim =~ ?^(.*)/exim?;
+($parm_exim_dir) = $parm_exim =~ m?^(.*)/exim?;
 
 $dbm_build_deleted = 0;
 if (defined $parm_lookups{'dbm'} &&
@@ -2899,6 +3053,10 @@ foreach $basedir ("aux-var", "dnszones")
     }
   }
 
+# Set a user's shell, distinguishable from /bin/sh
+
+symlink("/bin/sh","aux-var/sh");
+$ENV{'SHELL'} = $parm_shell = $parm_cwd . "/aux-var/sh";
 
 ##################################################
 #     Create fake DNS zones for this host        #
@@ -2938,7 +3096,16 @@ if ($have_ipv4 && $parm_ipv4 ne "127.0.0.1")
 
 if ($have_ipv6 && $parm_ipv6 ne "::1")
   {
-  my(@components) = split /:/, $parm_ipv6;
+  my($exp_v6) = $parm_ipv6;
+  $exp_v6 =~ s/[^:]//g;
+  if ( $parm_ipv6 =~ /^([^:].+)::$/ ) {
+    $exp_v6 = $1 . ':0' x (9-length($exp_v6));
+  } elsif ( $parm_ipv6 =~ /^(.+)::(.+)$/ ) {
+    $exp_v6 = $1 . ':0' x (8-length($exp_v6)) . ':' . $2;
+  } elsif ( $parm_ipv6 =~ /^::(.+[^:])$/ ) {
+    $exp_v6 = '0:' x (9-length($exp_v6)) . $1;
+  }
+  my(@components) = split /:/, $exp_v6;
   my(@nibbles) = reverse (split /\s*/, shift @components);
   my($sep) =  "";
 
@@ -3150,10 +3317,30 @@ foreach $test (@test_list)
 
     my($commandname) = "";
     my($expectrc) = 0;
-    my($rc) = run_command($testno, \$subtestno, \$expectrc, \$commandname);
+    my($rc, $run_extra) = run_command($testno, \$subtestno, \$expectrc, \$commandname, $TEST_STATE);
     my($cmdrc) = $?;
 
-    print ">> rc=$rc cmdrc=$cmdrc\n" if $debug;
+    if ($debug) {
+      print ">> rc=$rc cmdrc=$cmdrc\n";
+      if (defined $run_extra) {
+        foreach my $k (keys %$run_extra) {
+          my $v = defined $run_extra->{$k} ? qq!"$run_extra->{$k}"! : '<undef>';
+          print ">>   $k -> $v\n";
+        }
+      }
+    }
+    $run_extra = {} unless defined $run_extra;
+    foreach my $k (keys %$run_extra) {
+      if (exists $TEST_STATE->{$k}) {
+        my $nv = defined $run_extra->{$k} ? qq!"$run_extra->{$k}"! : 'removed';
+        print ">> override of $k; was $TEST_STATE->{$k}, now $nv\n" if $debug;
+      }
+      if (defined $run_extra->{$k}) {
+        $TEST_STATE->{$k} = $run_extra->{$k};
+      } elsif (exists $TEST_STATE->{$k}) {
+        delete $TEST_STATE->{$k};
+      }
+    }
 
     # Hit EOF after an initial return code number
 
@@ -3182,10 +3369,10 @@ foreach $test (@test_list)
 
       for (;;)
         {
-        print "\nshow stdErr, show stdOut, Continue (without file comparison), or Quit? [Q] ";
+        print "\nshow stdErr, show stdOut, Retry, Continue (without file comparison), or Quit? [Q] ";
         $_ = <T>;
         tests_exit(1) if /^q?$/i;
-        last if /^c$/i;
+        last if /^[rc]$/i;
         if (/^e$/i)
           {
           system("$more test-stderr");
@@ -3196,6 +3383,7 @@ foreach $test (@test_list)
           }
         }
 
+      $retry = 1 if /^r$/i;
       $docheck = 0;
       }
 
@@ -3218,10 +3406,10 @@ foreach $test (@test_list)
 
         for (;;)
           {
-          print "\nShow server stdout, Continue, or Quit? [Q] ";
+          print "\nShow server stdout, Retry, Continue, or Quit? [Q] ";
           $_ = <T>;
           tests_exit(1) if /^q?$/i;
-          last if /^c$/i;
+          last if /^[rc]$/i;
 
           if (/^s$/i)
             {
@@ -3231,6 +3419,7 @@ foreach $test (@test_list)
             close(S);
             }
           }
+        $retry = 1 if /^r$/i;
         }
       }
     }
@@ -3241,6 +3430,13 @@ foreach $test (@test_list)
   # function returns 0 if all is well, 1 if we should rerun the test (the files
   # have been updated). It does not return if the user responds Q to a prompt.
 
+  if ($retry)
+    {
+    $retry = '0';
+    print (("#" x 79) . "\n");
+    redo;
+    }
+
   if ($docheck)
     {
     if (check_output() != 0)
@@ -3264,4 +3460,4 @@ tests_exit(-1, "No runnable tests selected") if @test_list == 0;
 tests_exit(0);
 
 # End of runtest script
-# vim: set sw=2 :
+# vim: set sw=2 et :