X-Git-Url: https://git.exim.org/users/heiko/exim.git/blobdiff_plain/ffe0a3577e7b6d490354137ca1aa6e6a9d536c14..e5ab0ba9eb0f4424dfa31128f36b27b6769d7795:/test/runtest

diff --git a/test/runtest b/test/runtest
index 6f13a792c..8f5d393bb 100755
--- a/test/runtest
+++ b/test/runtest
@@ -16,9 +16,9 @@
 ###############################################################################
 
 #use strict;
-use 5.010;
-use feature 'state';	# included in 5.010
+use v5.10.1;
 use warnings;
+use if $^V >= v5.19.11, experimental => 'smartmatch';
 
 use Errno;
 use FileHandle;
@@ -32,8 +32,10 @@ use FindBin qw'$RealBin';
 
 use lib "$RealBin/lib";
 use Exim::Runtest;
+use Exim::Utils qw(uniq numerically);
 
-use if $ENV{DEBUG} && $ENV{DEBUG} =~ /\bruntest\b/ => ('Smart::Comments' => '####');
+use if $ENV{DEBUG} && scalar($ENV{DEBUG} =~ /\bruntest\b/) => 'Smart::Comments' => '####';
+use if $ENV{DEBUG} && scalar($ENV{DEBUG} =~ /\bruntest\b/) => 'Data::Dumper';
 
 use constant TEST_TOP => 8999;
 use constant TEST_SPECIAL_TOP => 9999;
@@ -73,9 +75,6 @@ my $have_ipv4 = 1;
 my $have_ipv6 = 1;
 my $have_largefiles = 0;
 
-my $test_start = 1;
-my $test_end = TEST_TOP;
-
 my @test_list = ();
 
 
@@ -488,9 +487,13 @@ RESET_AFTER_EXTRA_LINE_READ:
     /Tue, 2 Mar 1999 09:44:33 +0000/gx;
 
   # Date/time in logs and in one instance of a filter test
-  s/^\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d(\s[+-]\d\d\d\d)?/1999-03-02 09:44:33/gx;
+  s/^\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d(\s[+-]\d\d\d\d)?\s/1999-03-02 09:44:33 /gx;
+  s/^\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d\.\d{3}(\s[+-]\d\d\d\d)?\s/2017-07-30 18:51:05.712 /gx;
   s/^Logwrite\s"\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d/Logwrite "1999-03-02 09:44:33/gx;
 
+  s/((D|[QD]T)=)\d+s/$1qqs/g;
+  s/((D|[QD]T)=)\d\.\d{3}s/$1q.qqqs/g;
+
   # Date/time in message separators
   s/(?:[A-Z][a-z]{2}\s){2}\d\d\s\d\d:\d\d:\d\d\s\d\d\d\d
     /Tue Mar 02 09:44:33 1999/gx;
@@ -518,9 +521,6 @@ RESET_AFTER_EXTRA_LINE_READ:
   # Date/time in exim -bV output
   s/\d\d-[A-Z][a-z]{2}-\d{4}\s\d\d:\d\d:\d\d/07-Mar-2000 12:21:52/g;
 
-  # Time on queue tolerance
-  s/(QT|D)=1s/$1=0s/;
-
   # Eximstats heading
   s/Exim\sstatistics\sfrom\s\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d\sto\s
     \d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d/Exim statistics from <time> to <time>/x;
@@ -903,7 +903,8 @@ RESET_AFTER_EXTRA_LINE_READ:
     next if /^SSL info:/;
     next if /SSL verify error: depth=0 error=certificate not trusted/;
     s/SSL3_READ_BYTES/ssl3_read_bytes/i;
-    s/^\d+:error:\d+(:SSL routines:ssl3_read_bytes:[^:]+:).*(:SSL alert number \d\d)$/pppp:error:dddddddd$1\[...\]$2/;
+    s/CONNECT_CR_FINISHED/ssl3_read_bytes/i;
+    s/^\d+:error:\d+(?:E\d+)?(:SSL routines:ssl3_read_bytes:[^:]+:).*(:SSL alert number \d\d)$/pppp:error:dddddddd$1\[...\]$2/;
 
     # gnutls version variances
     next if /^Error in the pull function./;
@@ -911,6 +912,9 @@ RESET_AFTER_EXTRA_LINE_READ:
     # optional IDN2 variant conversions.  Accept either IDN1 or IDN2
     s/conversion  strasse.de/conversion  xn--strae-oqa.de/;
     s/conversion: german.xn--strae-oqa.de/conversion: german.straße.de/;
+
+    # subsecond timstamp info in reported header-files
+    s/^(-received_time_usec \.)\d{6}$/$1uuuuuu/;
     }
 
   # ======== stderr ========
@@ -1078,7 +1082,7 @@ RESET_AFTER_EXTRA_LINE_READ:
     if (s/(with \$received_protocol)\}\} \$\{if def:tls_cipher \{\(\$tls_cipher\)\n$/$1/)
       {
       $_ .= <IN>;
-      s/\s+\}\}(?=\(Exim )/\}\} /;
+      s/[ ╎]+\}\}(?=\(Exim )/\}\} /;
       }
     if (/^  condition: def:tls_cipher$/)
       {
@@ -1089,6 +1093,9 @@ RESET_AFTER_EXTRA_LINE_READ:
     # Not all platforms build with DKIM enabled
     next if /^PDKIM >> Body data for hash, canonicalized/;
 
+    # Not all platforms have sendfile support
+    next if /^cannot use sendfile for body: no support$/;
+
     #  Parts of DKIM-specific debug output depend on the time/date
     next if /^date:\w+,\{SP\}/;
     next if /^PDKIM \[[^[]+\] (Header hash|b) computed:/;
@@ -1104,6 +1111,10 @@ RESET_AFTER_EXTRA_LINE_READ:
 
     next if /^(ppppp )?setsockopt FASTOPEN: Protocol not available$/;
 
+    # Specific pointer values reported for DB operations change from run to run
+    s/^(returned from EXIM_DBOPEN: 0x)[0-9a-f]+/$1AAAAAAAA/;
+    s/^(EXIM_DBCLOSE.0x)[0-9a-f]+/$1AAAAAAAA/;
+
     # When Exim is checking the size of directories for maildir, it uses
     # the check_dir_size() function to scan directories. Of course, the order
     # of the files that are obtained using readdir() varies from system to
@@ -1155,6 +1166,13 @@ RESET_AFTER_EXTRA_LINE_READ:
     {
     # Berkeley DB version differences
     next if / Berkeley DB error: /;
+
+    # CHUNKING: exact sizes depend on hostnames in headers
+    s/(=>.* K C="250- \d)\d+ (byte chunk, total \d)\d+/$1nn $2nn/;
+
+    # openssl version variances
+    s/(TLS error on connection .*: error:)[0-9A-F]{8}(:system library):func\(4095\):(No such file or
+ directory)$/$1xxxxxxxx$2:fopen:$3/;
     }
 
   # ======== All files other than stderr ========
@@ -1514,13 +1532,14 @@ $munges =
 
     'optional_config' =>
     { 'stdout' => '/^(
-                  dkim_(canon|domain|private_key|selector|sign_headers|strict)
+                  dkim_(canon|domain|private_key|selector|sign_headers|strict|hash|identity)
                   |gnutls_require_(kx|mac|protocols)
                   |hosts_(requ(est|ire)|try)_(dane|ocsp)
-                  |hosts_(avoid|nopass|require|verify_avoid)_tls
+                  |hosts_(avoid|nopass|noproxy|require|verify_avoid)_tls
                   |socks_proxy
                   |tls_[^ ]*
-                  )($|[ ]=)/x' },
+                  )($|[ ]=)/x'
+    },
 
     'sys_bindir' =>
     { 'mainlog' => 's%/(usr/(local/)?)?bin/%SYSBINDIR/%' },
@@ -1536,6 +1555,12 @@ $munges =
 
     'timeout_errno' =>		# actual errno differs Solaris vs. Linux
     { 'mainlog' => 's/(host deferral .* errno) <\d+> /$1 <EEE> /' },
+
+    'peer_terminated_conn' =>	# actual error differs FreedBSD vs. Linux
+    { 'stderr' => 's/^(  SMTP\()Connection reset by peer(\)<<)$/$1closed$2/' },
+
+    'perl_variants' =>		# result of hash-in-scalar-context changed from bucket-fill to keycount
+    { 'stdout' => 's%^> X/X$%> X%' },
   };
 
 
@@ -2521,6 +2546,7 @@ $more = 'more' if system('which less >/dev/null 2>&1') != 0;
 # options are passed on to Exim calls within the tests. Typically, this is used
 # to turn on Exim debugging while setting up a test.
 
+Getopt::Long::Configure qw(no_getopt_compat);
 GetOptions(
     'debug'    => sub { $debug          = 1; $cr   = "\n" },
     'diff'     => sub { $cf             = 'diff -u' },
@@ -2531,7 +2557,9 @@ GetOptions(
     'keep'     => \$save_output,
     'slow'     => \$slow,
     'valgrind' => \$valgrind,
-    'flavor|flavour=s' => \$flavour,
+    'range=s{2}'       => \my @range_wanted,
+    'test=i@'          => \my @tests_wanted,
+    'flavor|flavour=s' => $flavour,
     'help'             => sub { pod2usage(-exit => 0) },
     'man'              => sub {
         pod2usage(
@@ -2545,16 +2573,15 @@ GetOptions(
 ($parm_exim, @ARGV) = Exim::Runtest::exim_binary(@ARGV);
 print "Exim binary is `$parm_exim'\n" if defined $parm_exim;
 
-# Any subsequent arguments are a range of test numbers.
 
-if (@ARGV)
-  {
-  $test_end = $test_start = shift;
-  $test_end = shift if @ARGV;
-  $test_end = ($test_start >= 9000)? TEST_SPECIAL_TOP : TEST_TOP
-    if $test_end eq '+';
-  die "** Test numbers out of order\n" if ($test_end < $test_start);
-  }
+my @wanted = sort numerically uniq
+  @tests_wanted ? @tests_wanted : (),
+  @range_wanted ? $range_wanted[0] .. $range_wanted[1] : (),
+  @ARGV ? @ARGV == 1 ? $ARGV[0] :
+          $ARGV[1] eq '+' ? $ARGV[0]..($ARGV[0] >= 9000 ? TEST_SPECIAL_TOP : TEST_TOP) :
+          0+$ARGV[0]..0+$ARGV[1]    # add 0 to cope with test numbers starting with zero
+        : ();
+@wanted = 1..TEST_TOP if not @wanted;
 
 ##################################################
 #        Check for sudo access to root           #
@@ -2597,7 +2624,7 @@ $parm_cwd = Cwd::getcwd();
 
 # If $parm_exim is still empty, ask the caller
 
-if ($parm_exim eq '')
+if (not $parm_exim)
   {
   print "** Did not find an Exim binary to test\n";
   for ($i = 0; $i < 5; $i++)
@@ -2710,7 +2737,7 @@ if (defined $parm_trusted_config_list)
   open(TCL, $parm_trusted_config_list) or die "Can't open $parm_trusted_config_list: $!\n";
   my $test_config = getcwd() . '/test-config';
   die "Can't find '$test_config' in TRUSTED_CONFIG_LIST $parm_trusted_config_list."
-  if not grep { /^$test_config$/ } <TCL>;
+  if not grep { /^\Q$test_config\E$/ } <TCL>;
   }
 else
   {
@@ -3390,7 +3417,9 @@ tests_exit(-1, "Failed to unlink $log_summary_filename: $!")
 # because the current binary does not support the right facilities, and also
 # those that are outside the numerical range selected.
 
-print "\nTest range is $test_start to $test_end (flavour $flavour)\n";
+printf "\nWill run %d tests between %d and %d for flavour %s\n",
+  scalar(@wanted), $wanted[0], $wanted[-1], $flavour;
+
 print "Omitting \${dlfunc expansion tests (loadable module not present)\n"
   if $dlfunc_deleted;
 print "Omitting dbm tests (unable to copy exim_dbmbuild)\n"
@@ -3401,8 +3430,8 @@ my @test_dirs = grep { not /^CVS$/ } map { basename $_ } glob 'scripts/*'
   or die tests_exit(-1, "Failed to find test scripts in 'scripts/*`: $!");
 
 # Scan for relevant tests
-
-DIR: for ($i = 0; $i < @test_dirs; $i++)
+# HS12: Needs to be reworked.
+DIR: for (my $i = 0; $i < @test_dirs; $i++)
   {
   my($testdir) = $test_dirs[$i];
   my($wantthis) = 1;
@@ -3413,12 +3442,12 @@ DIR: for ($i = 0; $i < @test_dirs; $i++)
   # test in the next directory.
 
   next DIR if ($i < @test_dirs - 1) &&
-          ($test_start >= substr($test_dirs[$i+1], 0, 4));
+          ($wanted[0] >= substr($test_dirs[$i+1], 0, 4));
 
   # No need to carry on if the end test is less than the first test in this
   # subdirectory.
 
-  last DIR if $test_end < substr($testdir, 0, 4);
+  last DIR if $wanted[-1] < substr($testdir, 0, 4);
 
   # Check requirements, if any.
 
@@ -3475,14 +3504,13 @@ DIR: for ($i = 0; $i < @test_dirs; $i++)
   # We want the tests from this subdirectory, provided they are in the
   # range that was selected.
 
-  @testlist = map { basename $_ } glob "scripts/$testdir/*";
+  @testlist = grep { $_ ~~ @wanted } grep { /^\d+(?:\.\d+)?$/ } map { basename $_ } glob "scripts/$testdir/*";
   tests_exit(-1, "Failed to read test scripts from `scripts/$testdir/*': $!")
     if not @testlist;
 
   foreach $test (@testlist)
     {
-    next if ($test !~ /^\d{4}(?:\.\d+)?$/);
-    if (!$wantthis || $test < $test_start || $test > $test_end)
+    if (!$wantthis)
       {
       log_test($log_summary_filename, $test, '.');
       }
@@ -3493,7 +3521,7 @@ DIR: for ($i = 0; $i < @test_dirs; $i++)
     }
   }
 
-print ">>Test List: @test_list\n", if $debug;
+print ">>Test List:\n", join "\n", @test_list, '' if $debug;
 
 
 ##################################################
@@ -3908,8 +3936,12 @@ foreach $test (@test_list)
 	}
         if ($force_continue)
           {
-          print "\nstderr tail:\n";
+          print "\nstdout tail:\n";
+          print "==================>\n";
+          system("tail -20 test-stdout");
           print "===================\n";
+          print "stderr tail:\n";
+          print "==================>\n";
           system("tail -20 test-stderr");
           print "===================\n";
           print "... continue forced\n";
@@ -4020,7 +4052,7 @@ __END__
 
 =head1 SYNOPSIS
 
- runtest [options] [test0 [test1]]
+ runtest [exim-path] [options] [test0 [test1]]
 
 =head1 DESCRIPTION
 
@@ -4032,6 +4064,10 @@ For legacy reasons the options are not case sensitive.
 
 =over
 
+=item B<--continue>
+
+Do not stop for user interaction or on errors. (default: off)
+
 =item B<--debug>
 
 This option enables the output of debug information when running the
@@ -4040,15 +4076,12 @@ various test commands. (default: off)
 =item B<--diff>
 
 Use C<diff -u> for comparing the expected output with the produced
-output. (default: use a built-in comparation routine)
-
-=item B<--continue>
-
-Do not stop for user interaction or on errors. (default: off)
+output. (default: use a built-in routine)
 
-=item B<--update>
+=item B<--flavor>|B<--flavour> I<flavour>
 
-Automatically update the recorded (expected) data on mismatch. (default: off)
+Override the expected results for results for a specific (OS) flavour.
+(default: unused)
 
 =item B<--[no]ipv4>
 
@@ -4062,18 +4095,26 @@ Skip IPv6 related setup and tests (default: use ipv6)
 
 Keep the various output files produced during a test run. (default: don't keep)
 
+=item B<--range> I<n0> I<n1>
+
+Run tests between (including) I<n0> and I<n1>. A "+" may be used to specify the "last
+test available".
+
 =item B<--slow>
 
-Insert some delays to compensate for a slow system. (default: off)
+Insert some delays to compensate for a slow host system. (default: off)
 
-=item B<--valgrind>
+=item B<--test> I<n>
 
-Start Exim wrapped by I<valgrind>. (default: don't use valgrind)
+Run the specified test. This option may used multiple times.
 
-=item B<--flavor>|B<--flavour> I<flavour>
+=item B<--update>
 
-Override the expected results for results for a specific (OS) flavour.
-(default: unused)
+Automatically update the recorded (expected) data on mismatch. (default: off)
+
+=item B<--valgrind>
+
+Start Exim wrapped by I<valgrind>. (default: don't use valgrind)
 
 =back