Merge branch 'master' into transp_logging_1031

[users/jgh/exim.git] / test / runtest

diff --git a/test/runtest b/test/runtest
index 2756348f9c4a729fc506c1720640a5b066a00d08..6623d3b4b62c80d80a5621d1050db7eb640089a9 100755 (executable)
--- a/test/runtest
+++ b/test/runtest
@@ -36,7 +36,9 @@ $gnutls_dh_bits_normal = 2236;
 $cf = "bin/cf -exact";
 $cr = "\r";
 $debug = 0;
 $cf = "bin/cf -exact";
 $cr = "\r";
 $debug = 0;

+$force_continue = 0;

 $force_update = 0;
 $force_update = 0;

+$log_failed_filename = "failed-summary.log";

 $more = "less -XF";
 $optargs = "";
 $save_output = 0;
 $more = "less -XF";
 $optargs = "";
 $save_output = 0;


@@ -1006,16 +1008,43 @@ return $yield;
 
 # Arguments: [0] the prompt string
 #            [1] if there is a U in the prompt and $force_update is true
 
 # Arguments: [0] the prompt string
 #            [1] if there is a U in the prompt and $force_update is true

+#            [2] if there is a C in the prompt and $force_continue is true

 # Returns:   nothing (it sets $_)
 
 sub interact{
 print $_[0];
 if ($_[1]) { $_ = "u"; print "... update forced\n"; }
 # Returns:   nothing (it sets $_)
 
 sub interact{
 print $_[0];
 if ($_[1]) { $_ = "u"; print "... update forced\n"; }

+  elsif ($_[2]) { $_ = "c"; print "... continue forced\n"; }

   else { $_ = <T>; }
 }
 
 
 
   else { $_ = <T>; }
 }
 
 
 

+##################################################
+#    Subroutine to log in force_continue mode    #
+##################################################
+
+# In force_continue mode, we just want a terse output to a statically
+# named logfile.  If multiple files in same batch (stdout, stderr, etc)
+# all have mismatches, it will log multiple times.
+#
+# Arguments: [0] the logfile to append to
+#            [1] the testno that failed
+# Returns:   nothing
+
+
+
+sub log_failure {
+  my $logfile = shift();
+  my $testno  = shift();
+  my $detail  = shift() || '';
+  if ( open(my $fh, ">>", $logfile) ) {
+    print $fh "Test $testno $detail failed\n";
+    close $fh;
+  }
+}
+
+

 
 ##################################################
 #    Subroutine to compare one output file       #
 
 ##################################################
 #    Subroutine to compare one output file       #


@@ -1056,6 +1085,7 @@ if (! -e $sf)
     print "Continue, Show, or Quit? [Q] ";
     $_ = <T>;
     tests_exit(1) if /^q?$/i;
     print "Continue, Show, or Quit? [Q] ";
     $_ = <T>;
     tests_exit(1) if /^q?$/i;

+    log_failure($log_failed_filename, $testno, $rf) if (/^c$/i && $force_continue);

     return 0 if /^c$/i;
     last if (/^s$/);
     }
     return 0 if /^c$/i;
     last if (/^s$/);
     }


@@ -1074,8 +1104,9 @@ if (! -e $sf)
   print "\n";
   for (;;)
     {
   print "\n";
   for (;;)
     {

-    interact("Continue, Update & retry, Quit? [Q] ", $force_update);
+    interact("Continue, Update & retry, Quit? [Q] ", $force_update, $force_continue);

     tests_exit(1) if /^q?$/i;
     tests_exit(1) if /^q?$/i;

+    log_failure($log_failed_filename, $testno, $rsf) if (/^c$/i && $force_continue);

     return 0 if /^c$/i;
     last if (/^u$/i);
     }
     return 0 if /^c$/i;
     last if (/^u$/i);
     }


@@ -1191,8 +1222,9 @@ if (-e $sf)
   print "\n";
   for (;;)
     {
   print "\n";
   for (;;)
     {

-    interact("Continue, Retry, Update & retry, Quit? [Q] ", $force_update);
+    interact("Continue, Retry, Update & retry, Quit? [Q] ", $force_update, $force_continue);

     tests_exit(1) if /^q?$/i;
     tests_exit(1) if /^q?$/i;

+    log_failure($log_failed_filename, $testno, $sf) if (/^c$/i && $force_continue);

     return 0 if /^c$/i;
     return 1 if /^r$/i;
     last if (/^u$/i);
     return 0 if /^c$/i;
     return 1 if /^r$/i;
     last if (/^u$/i);


@@ -1342,8 +1374,9 @@ if (! $message_skip)
 
     for (;;)
       {
 
     for (;;)
       {

-      interact("Continue, Update & retry, or Quit? [Q] ", $force_update);
+      interact("Continue, Update & retry, or Quit? [Q] ", $force_update, $force_continue);

       tests_exit(1) if /^q?$/i;
       tests_exit(1) if /^q?$/i;

+      log_failure($log_failed_filename, $testno, "missing email") if (/^c$/i && $force_continue);

       last if /^c$/i;
 
       # For update, we not only have to unlink the file, but we must also
       last if /^c$/i;
 
       # For update, we not only have to unlink the file, but we must also


@@ -1425,8 +1458,9 @@ if (! $msglog_skip)
 
     for (;;)
       {
 
     for (;;)
       {

-      interact("Continue, Update, or Quit? [Q] ", $force_update);
+      interact("Continue, Update, or Quit? [Q] ", $force_update, $force_continue);

       tests_exit(1) if /^q?$/i;
       tests_exit(1) if /^q?$/i;

+      log_failure($log_failed_filename, $testno, "missing msglog") if (/^c$/i && $force_continue);

       last if /^c$/i;
       if (/^u$/i)
         {
       last if /^c$/i;
       if (/^u$/i)
         {


@@ -2131,6 +2165,9 @@ while (@ARGV > 0 && $ARGV[0] =~ /^-/)
     {
     if ($arg eq "-DEBUG")  { $debug = 1; $cr = "\n"; next; }
     if ($arg eq "-DIFF")   { $cf = "diff -u"; next; }
     {
     if ($arg eq "-DEBUG")  { $debug = 1; $cr = "\n"; next; }
     if ($arg eq "-DIFF")   { $cf = "diff -u"; next; }

+    if ($arg eq "-CONTINUE"){$force_continue = 1;
+                             $more = "cat";
+                             next; }

     if ($arg eq "-UPDATE") { $force_update = 1; next; }
     if ($arg eq "-NOIPV4") { $have_ipv4 = 0; next; }
     if ($arg eq "-NOIPV6") { $have_ipv6 = 0; next; }
     if ($arg eq "-UPDATE") { $force_update = 1; next; }
     if ($arg eq "-NOIPV4") { $have_ipv4 = 0; next; }
     if ($arg eq "-NOIPV6") { $have_ipv6 = 0; next; }


@@ -3426,8 +3463,10 @@ foreach $test (@test_list)
       for (;;)
         {
         print "\nshow stdErr, show stdOut, Retry, Continue (without file comparison), or Quit? [Q] ";
       for (;;)
         {
         print "\nshow stdErr, show stdOut, Retry, Continue (without file comparison), or Quit? [Q] ";

-        $_ = <T>;
+        $_ = $force_continue ? "c" : <T>;

         tests_exit(1) if /^q?$/i;
         tests_exit(1) if /^q?$/i;

+        log_failure($log_failed_filename, $testno, "exit code unexpected") if (/^c$/i && $force_continue);
+        print "... continue forced\n" if $force_continue;

         last if /^[rc]$/i;
         if (/^e$/i)
           {
         last if /^[rc]$/i;
         if (/^e$/i)
           {


@@ -3463,8 +3502,10 @@ foreach $test (@test_list)
         for (;;)
           {
           print "\nShow server stdout, Retry, Continue, or Quit? [Q] ";
         for (;;)
           {
           print "\nShow server stdout, Retry, Continue, or Quit? [Q] ";

-          $_ = <T>;
+          $_ = $force_continue ? "c" : <T>;

           tests_exit(1) if /^q?$/i;
           tests_exit(1) if /^q?$/i;

+          log_failure($log_failed_filename, $testno, "exit code unexpected") if (/^c$/i && $force_continue);
+          print "... continue forced\n" if $force_continue;

           last if /^[rc]$/i;
 
           if (/^s$/i)
           last if /^[rc]$/i;
 
           if (/^s$/i)