X-Git-Url: https://git.exim.org/users/heiko/exim.git/blobdiff_plain/e51462229029e40afa0932dd9c6337192a0dc70b..87359fb001c9fb4d450a35a0b5f6642c16c78c34:/test/runtest diff --git a/test/runtest b/test/runtest index 59f61c6a6..4972aedd8 100755 --- a/test/runtest +++ b/test/runtest @@ -16,9 +16,9 @@ ############################################################################### #use strict; -use 5.010; -use feature 'state'; # included in 5.010 +use v5.10.1; use warnings; +use if $^V >= v5.19.11, experimental => 'smartmatch'; use Errno; use FileHandle; @@ -26,12 +26,19 @@ use Socket; use Time::Local; use Cwd; use File::Basename; +use Pod::Usage; +use Getopt::Long; use FindBin qw'$RealBin'; use lib "$RealBin/lib"; use Exim::Runtest; +use Exim::Utils qw(uniq numerically); -use if $ENV{DEBUG} && $ENV{DEBUG} =~ /\bruntest\b/ => ('Smart::Comments' => '####'); +use if $ENV{DEBUG} && scalar($ENV{DEBUG} =~ /\bruntest\b/) => 'Smart::Comments' => '####'; +use if $ENV{DEBUG} && scalar($ENV{DEBUG} =~ /\bruntest\b/) => 'Data::Dumper'; + +use constant TEST_TOP => 8999; +use constant TEST_SPECIAL_TOP => 9999; # Start by initializing some global variables @@ -57,21 +64,18 @@ my $force_continue = 0; my $force_update = 0; my $log_failed_filename = 'failed-summary.log'; my $log_summary_filename = 'run-summary.log'; -my $more = 'less -XF'; +my @more = qw'less -XF'; my $optargs = ''; my $save_output = 0; my $server_opts = ''; +my $slow = 0; my $valgrind = 0; my $have_ipv4 = 1; my $have_ipv6 = 1; my $have_largefiles = 0; -my $test_start = 1; -my $test_end = $test_top = 8999; -my $test_special_top = 9999; my @test_list = (); -my @test_dirs = (); # Networks to use for DNS tests. We need to choose some networks that will @@ -108,7 +112,7 @@ $ENV{LC_ALL} = 'C'; $ENV{USER} = getpwuid($>) if not exists $ENV{USER}; my ($parm_configure_owner, $parm_configure_group); -my ($parm_ipv4, $parm_ipv6); +my ($parm_ipv4, $parm_ipv6, $parm_ipv6_stripped); my $parm_hostname; ############################################################################### @@ -400,7 +404,7 @@ RESET_AFTER_EXTRA_LINE_READ: # Replace the Exim version number (may appear in various places) # patchexim should have fixed this for us - #s/(Exim) \d+\.\d+[\w_-]*/$1 x.yz/i; + #s/Exim \K\d+[._]\d+[\w_-]*/x.yz/i; # Replace Exim message ids by a unique series s/((?:[^\W_]{6}-){2}[^\W_]{2}) @@ -456,7 +460,9 @@ RESET_AFTER_EXTRA_LINE_READ: { my($date1,$date2,$date3,$expired) = ($1,$2,$3,$4); $expired = '' if !defined $expired; - my($increment) = date_seconds($date3) - date_seconds($date2); + + # Round the time-difference up to nearest even value + my($increment) = ((date_seconds($date3) - date_seconds($date2) + 1) >> 1) << 1; # We used to use globally unique replacement values, but timing # differences make this impossible. Just show the increment on the @@ -470,6 +476,13 @@ RESET_AFTER_EXTRA_LINE_READ: # more_errno values in exim_dumpdb output which are times s/T:(\S+)\s-22\s(\S+)\s/T:$1 -22 xxxx /; + # port numbers in dumpdb output + s/T:([a-z.]+(:[0-9.]+)?):$parm_port_n /T:$1:PORT_N /; + + # port numbers in stderr + s/^set_process_info: .*\]:\K$parm_port_d /PORT_D /; + s/^set_process_info: .*\]:\K$parm_port_s /PORT_S /; + # ======== Dates and times ======== @@ -479,12 +492,24 @@ RESET_AFTER_EXTRA_LINE_READ: # time used was fixed when I first started running automatic Exim tests. # Date/time in header lines and SMTP responses - s/[A-Z][a-z]{2},\s\d\d?\s[A-Z][a-z]{2}\s\d\d\d\d\s\d\d\:\d\d:\d\d\s[-+]\d{4} + s/[A-Z][a-z]{2},\s\d\d?\s[A-Z][a-z]{2}\s\d{4}\s\d\d\:\d\d:\d\d\s[-+]\d{4} /Tue, 2 Mar 1999 09:44:33 +0000/gx; + # and in a French locale + s/\S{4},\s\d\d?\s[^,]+\s\d{4}\s\d\d\:\d\d:\d\d\s[-+]\d{4} + /dim., 10 f\xE9vr 2019 20:05:49 +0000/gx; # Date/time in logs and in one instance of a filter test - s/^\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d(\s[+-]\d\d\d\d)?/1999-03-02 09:44:33/gx; + s/^\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d(\s[+-]\d\d\d\d)?\s/1999-03-02 09:44:33 /gx; + s/^\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d\.\d{3}(\s[+-]\d\d\d\d)?\s/2017-07-30 18:51:05.712 /gx; s/^Logwrite\s"\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d/Logwrite "1999-03-02 09:44:33/gx; + # Date/time in syslog test + s/^SYSLOG:\s\'\K\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d\s/2017-07-30 18:51:05 /gx; + s/^SYSLOG:\s\'\K\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d\.\d{3}\s/2017-07-30 18:51:05.712 /gx; + s/^SYSLOG:\s\'\K\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d\s[+-]\d\d\d\d\s/2017-07-30 18:51:05 +9999 /gx; + s/^SYSLOG:\s\'\K\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d\.\d{3}\s[+-]\d\d\d\d\s/2017-07-30 18:51:05.712 +9999 /gx; + + s/((D|[RQD]T)=)\d+s/$1qqs/g; + s/((D|[RQD]T)=)\d\.\d{3}s/$1q.qqqs/g; # Date/time in message separators s/(?:[A-Z][a-z]{2}\s){2}\d\d\s\d\d:\d\d:\d\d\s\d\d\d\d @@ -502,7 +527,7 @@ RESET_AFTER_EXTRA_LINE_READ: my($next) = $3 - $2; $_ = " first failed=dddd last try=dddd next try=+$next $4\n"; } - s/^(\s*)now=\d+ first_failed=\d+ next_try=\d+ expired=(\d)/$1now=tttt first_failed=tttt next_try=tttt expired=$2/; + s/^(\s*)now=\d+ first_failed=\d+ next_try=\d+ expired=(\w)/$1now=tttt first_failed=tttt next_try=tttt expired=$2/; s/^(\s*)received_time=\d+ diff=\d+ timeout=(\d+)/$1received_time=tttt diff=tttt timeout=$2/; # Time to retry may vary @@ -513,9 +538,6 @@ RESET_AFTER_EXTRA_LINE_READ: # Date/time in exim -bV output s/\d\d-[A-Z][a-z]{2}-\d{4}\s\d\d:\d\d:\d\d/07-Mar-2000 12:21:52/g; - # Time on queue tolerance - s/(QT|D)=1s/$1=0s/; - # Eximstats heading s/Exim\sstatistics\sfrom\s\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d\sto\s \d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d/Exim statistics from <time> to <time>/x; @@ -525,55 +547,122 @@ RESET_AFTER_EXTRA_LINE_READ: s/(could not connect to .*: Connection) reset by peer$/$1 refused/; # ======== TLS certificate algorithms ======== + # + # In Received: headers, convert RFC 8314 style ciphersuite to + # the older (comment) style, keeping only the Auth element + # (discarding kex, cipher, mac). For TLS 1.3 there is no kex + # element (and no _WITH); insert a spurious "RSA". + # Also in $tls_X_cipher_std reporting. + + s/^\s+by \S+ with .+ \K \(TLS1(?:\.[0-3])?\) tls TLS_.*?([^_]+)_WITH.+$/(TLS1.x:ke-$1-AES256-SHAnnn:xxx)/; + s/^\s+by \S+ with .+ \K \(TLS1(?:\.[0-3])?\) tls TLS_.+$/(TLS1.x:ke-RSA-AES256-SHAnnn:xxx)/; + + s/ cipher_ TLS_.*?([^_]+)_WITH.+$/ cipher_ TLS1.x:ke_$1_WITH_ci_mac/; + s/ cipher_ TLS_.*$/ cipher_ TLS1.x:ke_RSA_WITH_ci_mac/; + # Test machines might have various different TLS library versions supporting # different protocols; can't rely upon TLS 1.2's AES256-GCM-SHA384, so we # treat the standard algorithms the same. + # + # TLSversion : KeyExchange? - Authentication/Signature - C_iph_er - MAC : bits + # # So far, have seen: # TLSv1:AES128-GCM-SHA256:128 # TLSv1:AES256-SHA:256 # TLSv1.1:AES256-SHA:256 # TLSv1.2:AES256-GCM-SHA384:256 # TLSv1.2:DHE-RSA-AES256-SHA:256 + # TLSv1.3:TLS_AES_256_GCM_SHA384:256 # TLS1.2:DHE_RSA_AES_128_CBC_SHA1:128 # We also need to handle the ciphersuite without the TLS part present, for # client-ssl's output. We also see some older forced ciphersuites, but # negotiating TLS 1.2 instead of 1.0. # Mail headers (...), log-lines X=..., client-ssl output ... # (and \b doesn't match between ' ' and '(' ) + # + # Retain the authentication algorith field as we want to test that. - s/( (?: (?:\b|\s) [\(=] ) | \s )TLSv1\.[12]:/$1TLSv1:/xg; - s/\bAES128-GCM-SHA256:128\b/AES256-SHA:256/g; - s/\bAES128-GCM-SHA256\b/AES256-SHA/g; - s/\bAES256-GCM-SHA384\b/AES256-SHA/g; - s/\bDHE-RSA-AES256-SHA\b/AES256-SHA/g; + s/( (?: (?:\b|\s) [\(=] ) | \s )TLS1(\.[123])?:/$1TLS1.x:/xg; + s/(?<!ke-)((EC)?DHE-)?(RSA|ECDSA)-AES(128|256)-(GCM-SHA(256|384)|SHA)(?!:)/ke-$3-AES256-SHAnnn/g; + s/(?<!ke-)((EC)?DHE-)?(RSA|ECDSA)-AES(128|256)-(GCM-SHA(256|384)|SHA):(128|256)/ke-$3-AES256-SHAnnn:xxx/g; + + # OpenSSL TLSv1.3 - unsure what to do about the authentication-variant testcases now, + # as it seems the protocol no longer supports a user choice. Replace the "TLS" field with "RSA". + # Also insert a key-exchange field for back-compat, even though 1.3 doesn't do that. + # + # TLSversion : "TLS" - C_iph_er - MAC : ??? + # + s/TLS_AES(_256)?_GCM_SHA384(?!:)/ke-RSA-AES256-SHAnnn/g; + s/:TLS_AES(_256)?_GCM_SHA384:256/:ke-RSA-AES256-SHAnnn:xxx/g; # LibreSSL + # TLSv1:AES256-GCM-SHA384:256 # TLSv1:ECDHE-RSA-CHACHA20-POLY1305:256 - s/\bECDHE-RSA-CHACHA20-POLY1305\b/AES256-SHA/g; + # + # ECDHE-RSA-CHACHA20-POLY1305 + # AES256-GCM-SHA384 + + s/(?<!-)(AES256-GCM-SHA384)/RSA-$1/; + s/(?<!ke-)((EC)?DHE-)?(RSA|ECDSA)-(AES256|CHACHA20)-(GCM-SHA384|POLY1305)(?!:)/ke-$3-AES256-SHAnnn/g; + s/(?<!ke-)((EC)?DHE-)?(RSA|ECDSA)-(AES256|CHACHA20)-(GCM-SHA384|POLY1305):256/ke-$3-AES256-SHAnnn:xxx/g; # GnuTLS have seen: + # TLS1.3:ECDHE_RSA_AES_256_GCM_SHA384:256 + # TLS1.3:ECDHE_SECP256R1__RSA_PSS_RSAE_SHA256__AES_256_GCM__AEAD:256 + # TLS1.3:ECDHE_X25519__RSA_PSS_RSAE_SHA256__AES_256_GCM:256 + # TLS1.3:ECDHE_PSK_SECP256R1__AES_256_GCM__AEAD:256 + # # TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256 # TLS1.2:ECDHE_RSA_AES_128_GCM_SHA256:128 # TLS1.2:RSA_AES_256_CBC_SHA1:256 (canonical) # TLS1.2:DHE_RSA_AES_128_CBC_SHA1:128 + # TLS1.2:ECDHE_SECP256R1__RSA_SHA256__AES_256_GCM:256 + # TLS1.2:ECDHE_SECP256R1__RSA_SHA256__AES_128_CBC__SHA256:128 + # TLS1.2:ECDHE_SECP256R1__ECDSA_SHA512__AES_256_GCM:256 + # TLS1.2:ECDHE_SECP256R1__AES_256_GCM:256 (3.6.7 resumption) + # TLS1.2:ECDHE_RSA_SECP256R1__AES_256_GCM:256 (! 3.5.18 !) + # TLS1.2:RSA__CAMELLIA_256_GCM:256 (leave the cipher name) + # TLS1.2-PKIX:RSA__AES_128_GCM__AEAD:128 (the -PKIX seems to be a 3.1.20 thing) + # TLS1.2-PKIX:ECDHE_RSA_SECP521R1__AES_256_GCM__AEAD:256 # # X=TLS1.2:DHE_RSA_AES_256_CBC_SHA256:256 # X=TLS1.2:RSA_AES_256_CBC_SHA1:256 # X=TLS1.1:RSA_AES_256_CBC_SHA1:256 + # X=TLS1.0:RSA_AES_256_CBC_SHA1:256 # X=TLS1.0:DHE_RSA_AES_256_CBC_SHA1:256 + # X=TLS1.0-PKIX:RSA__AES_256_CBC__SHA1:256 # and as stand-alone cipher: # ECDHE-RSA-AES256-SHA # DHE-RSA-AES256-SHA256 # DHE-RSA-AES256-SHA # picking latter as canonical simply because regex easier that way. - s/\bDHE_RSA_AES_128_CBC_SHA1:128/RSA_AES_256_CBC_SHA1:256/g; - s/TLS1.[012]:((EC)?DHE_)?RSA_AES_(256|128)_(CBC|GCM)_SHA(1|256|384):(256|128)/TLS1.x:xxxxRSA_AES_256_CBC_SHAnnn:256/g; - s/\b(ECDHE-RSA-AES256-SHA|DHE-RSA-AES256-SHA256)\b/AES256-SHA/g; + s/\bDHE_RSA_AES_128_CBC_SHA1:128/RSA-AES256-SHA1:256/g; + s/TLS1.[x0123](-PKIX)?: # TLS version + ((EC)?DHE(_((?<psk>PSK)_)?((?<auth>RSA|ECDSA)_)? + (SECP(256|521)R1|X25519))?__?)? # key-exchange + ((?<auth>RSA|ECDSA)((_PSS_RSAE)?_SHA(512|256))?__?)? # authentication + (?<with>WITH_)? # stdname-with + AES_(256|128)_(CBC|GCM) # cipher + (__?AEAD)? # pseudo-MAC + (__?SHA(1|256|384))? # PRF + :(256|128) # cipher strength + /"TLS1.x:ke-" + . (defined($+{psk}) ? $+{psk} : "") + . (defined($+{auth}) ? $+{auth} : "") + . (defined($+{with}) ? $+{with} : "") + . "-AES256-SHAnnn:xxx"/gex; + s/TLS1.2:RSA__CAMELLIA_256_GCM(_SHA384)?:256/TLS1.2:RSA_CAMELLIA_256_GCM-SHAnnn:256/g; + s/\b(ECDHE-(RSA|ECDSA)-AES256-SHA|DHE-RSA-AES256-SHA256)\b/ke-$2-AES256-SHAnnn/g; + + # Separate reporting of TLS version + s/ver: TLS1(\.[0-3])?$/ver: TLS1.x/; + s/ \(TLS1(\.[0-3])?\) / (TLS1.x) /; # GnuTLS library error message changes - s/No certificate was found/The peer did not send any certificate/g; + s/(No certificate was found|Certificate is required)/The peer did not send any certificate/g; #(dodgy test?) s/\(certificate verification failed\): invalid/\(gnutls_handshake\): The peer did not send any certificate./g; s/\(gnutls_priority_set\): No or insufficient priorities were set/\(gnutls_handshake\): Could not negotiate a supported cipher suite/g; + s/\(gnutls_handshake\): \KNo supported cipher suites have been found.$/Could not negotiate a supported cipher suite./; # (this new one is a generic channel-read error, but the testsuite # only hits it in one place) @@ -649,7 +738,7 @@ RESET_AFTER_EXTRA_LINE_READ: s/\bgid=\d+/gid=gggg/; s/\begid=\d+/egid=gggg/; - s/\bpid=\d+/pid=pppp/; + s/\b(pid=|PID: )\d+/$1pppp/; s/\buid=\d+/uid=uuuu/; s/\beuid=\d+/euid=uuuu/; s/set_process_info:\s+\d+/set_process_info: pppp/; @@ -663,8 +752,12 @@ RESET_AFTER_EXTRA_LINE_READ: s"test-mail/temp\.\d+\."test-mail/temp.pppp."; # Optional pid in log lines - s/^(\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d)(\s[+-]\d\d\d\d|)(\s\[\d+\])/ - "$1$2 [" . new_value($3, "%s", \$next_pid) . "]"/gxe; + s/^(\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d)(\.\d{3}|)(\s[+-]\d{4}|)(\s\[\d+\])/ + "$1$2$3 [" . new_value($4, "%s", \$next_pid) . "]"/gxe; + + # Optional pid in syslog test lines + s/^(SYSLOG:\s\'([-0-9]{10}\s[:.0-9]{8,12}\s([-+]\d{4}\s)?|))(\[\d+\] )/ + "$1\[" . new_value($4, "%s", \$next_pid) . "]"/gxe; # Detect a daemon stderr line with a pid and save the pid for subsequent # removal from following lines. @@ -675,8 +768,9 @@ RESET_AFTER_EXTRA_LINE_READ: s/waiting for children of \d+/waiting for children of pppp/; s/waiting for (\S+) \(\d+\)/waiting for $1 (pppp)/; - # The spool header file name varies with PID - s%^(Writing spool header file: .*/hdr).[0-9]{1,5}%$1.pppp%; + # Most builds are without HAVE_LOCAL_SCAN + next if /^calling local_scan\(\); timeout=300$/; + next if /^local_scan\(\) returned 0 NULL$/; # ======== Port numbers ======== # Incoming port numbers may vary, but not in daemon startup line. @@ -692,7 +786,7 @@ RESET_AFTER_EXTRA_LINE_READ: } # Port in host address in spool file output from -Mvh - s/^-host_address (.*)\.\d+/-host_address $1.9999/; + s/^(--?host_address) (.*)\.\d+/$1 $2.9999/; if ($dynamic_socket and $dynamic_socket->opened and my $port = $dynamic_socket->sockport) { s/^Connecting to 127\.0\.0\.1 port \K$port/<dynamic port>/; @@ -706,6 +800,12 @@ RESET_AFTER_EXTRA_LINE_READ: # Also, the length of space at the end of the host line is dependent # on the length of the longest line, so strip it also on otherwise # un-rewritten lines like localhost + # + # host 127.0.0.1 [127.0.0.1] + # host 10.0.0.1 [10.0.0.1]- + # + # host 127.0.0.1 [127.0.0.1]-- + # host 169.16.16.16 [169.16.16.10] s/^\s+host\s(\S+)\s+(\S+)/ host $1 $2/; s/^\s+(host\s\S+\s\S+)\s+(port=.*)/ host $1 $2/; @@ -714,9 +814,10 @@ RESET_AFTER_EXTRA_LINE_READ: s/host\s\Q$parm_ipv6\E\s\[\Q$parm_ipv6\E\]/host ip6:ip6:ip6:ip6:ip6:ip6:ip6:ip6 [ip6:ip6:ip6:ip6:ip6:ip6:ip6:ip6]/; s/\b\Q$parm_ipv4\E\b/ip4.ip4.ip4.ip4/g; s/(^|\W)\K\Q$parm_ipv6\E/ip6:ip6:ip6:ip6:ip6:ip6:ip6:ip6/g; + s/(^|\W)\K\Q$parm_ipv6_stripped\E/ip6:ip6:ip6:ip6:ip6:ip6:ip6:ip6/g; s/\b\Q$parm_ipv4r\E\b/ip4-reverse/g; s/(^|\W)\K\Q$parm_ipv6r\E/ip6-reverse/g; - s/^(\s+host\s\S+\s+\[\S+\]) +$/$1 /; + s/^\s+host\s\S+\s+\[\S+\]\K +$//; # strip, not collapse the trailing whitespace # ======== Test network IP addresses ======== @@ -726,11 +827,10 @@ RESET_AFTER_EXTRA_LINE_READ: # ======== IP error numbers and messages ======== # These vary between operating systems - s/Can't assign requested address/Network Error/; - s/Cannot assign requested address/Network Error/; + s/Can(no|')t assign requested address/Network Error/; s/Operation timed out/Connection timed out/; s/Address family not supported by protocol family/Network Error/; - s/Network is unreachable/Network Error/; + s/Network( is)? unreachable/Network Error/; s/Invalid argument/Network Error/; s/\(\d+\): Network/(dd): Network/; @@ -792,7 +892,7 @@ RESET_AFTER_EXTRA_LINE_READ: # The sizes of filter files may vary because of the substitution of local # filenames, logins, etc. - s/^\d+(?= bytes read from )/ssss/; + s/^\d+(?= (\(tainted\) )?bytes read from )/ssss/; # ======== OpenSSL error messages ======== @@ -800,7 +900,10 @@ RESET_AFTER_EXTRA_LINE_READ: # numbers, or handle specific bad conditions in different ways, leading to # different wording in the error messages, so we cannot compare them. - s/(TLS error on connection (?:from .* )?\(SSL_\w+\): error:)(.*)/$1 <<detail omitted>>/; +#XXX This loses any trailing "deliving unencypted to" which is unfortunate +# but I can't work out how to deal with that. + s/(TLS session: \(SSL_\w+\): error:)(.*)(?!: delivering)/$1 <<detail omitted>>/; + s/(TLS error on connection from .* \(SSL_\w+\): error:)(.*)/$1 <<detail omitted>>/; next if /SSL verify error: depth=0 error=certificate not trusted/; # ======== Maildir things ======== @@ -813,7 +916,7 @@ RESET_AFTER_EXTRA_LINE_READ: s/renamed tmp\/\d+\.[^.]+\.(\S+) as new\/\d+\.[^.]+\.(\S+)/renamed tmp\/MAILDIR.$1 as new\/MAILDIR.$1/; # Maildir file names in general - s/\b\d+\.H\d+P\d+\b/dddddddddd.HddddddPddddd/; + s/\b\d+\.M\d+P\d+\b/dddddddddd.HddddddPddddd/; # Maildirsize data while (/^\d+S,\d+C\s*$/) @@ -829,14 +932,17 @@ RESET_AFTER_EXTRA_LINE_READ: last if !defined $_; + # SRS timestamps and signatures vary by hostname and from run to run + + s/(?i)SRS0=....=.[^=]?=([^=]+)=([^@]+)\@([^ ]+)/SRS0=ZZZZ=YY=$1=$2\@$3/g; + + # ======== Output from the "fd" program about open descriptors ======== # The statuses seem to be different on different operating systems, but # at least we'll still be checking the number of open fd's. s/max fd = \d+/max fd = dddd/; - s/status=0 RDONLY/STATUS/g; - s/status=1 WRONLY/STATUS/g; - s/status=2 RDWR/STATUS/g; + s/status=[0-9a-f]+ (?:RDONLY|WRONLY|RDWR)/STATUS/g; # ======== Contents of spool files ======== @@ -845,12 +951,6 @@ RESET_AFTER_EXTRA_LINE_READ: s/^\d\d\d(?=[PFS*])/ddd/; - # ========= Exim lookups ================== - # Lookups have a char which depends on the number of lookup types compiled in, - # in stderr output. Replace with a "0". Recognising this while avoiding - # other output is fragile; perhaps the debug output should be revised instead. - s%(?<!sqlite)(?<!lsearch\*@)(?<!lsearch\*)(?<!lsearch)[0-?]TESTSUITE/aux-fixed/%0TESTSUITE/aux-fixed/%g; - # ========================================================== # MIME boundaries in RFC3461 DSN messages s/\d{8,10}-eximdsn-\d+/NNNNNNNNNN-eximdsn-MMMMMMMMMM/; @@ -895,7 +995,9 @@ RESET_AFTER_EXTRA_LINE_READ: next if /^SSL info:/; next if /SSL verify error: depth=0 error=certificate not trusted/; s/SSL3_READ_BYTES/ssl3_read_bytes/i; - s/^\d+:error:\d+(:SSL routines:ssl3_read_bytes:[^:]+:).*(:SSL alert number \d\d)$/pppp:error:dddddddd$1\[...\]$2/; + s/CONNECT_CR_FINISHED/ssl3_read_bytes/i; + s/^\d+:error:\d+(?:E\d+)?(:SSL routines:ssl3_read_bytes:[^:]+:).*(:SSL alert number \d\d)$/pppp:error:dddddddd$1\[...\]$2/; + s/^error:[^:]*:(SSL routines:ssl3_read_bytes:(tls|ssl)v\d+ alert)/error:dddddddd:$1/; # gnutls version variances next if /^Error in the pull function./; @@ -903,6 +1005,26 @@ RESET_AFTER_EXTRA_LINE_READ: # optional IDN2 variant conversions. Accept either IDN1 or IDN2 s/conversion strasse.de/conversion xn--strae-oqa.de/; s/conversion: german.xn--strae-oqa.de/conversion: german.straÃe.de/; + + # subsecond timstamp info in reported header-files + s/^(-received_time_usec \.)\d{6}$/$1uuuuuu/; + + # Postgres server takes varible time to shut down; lives in various places + s/^waiting for server to shut down\.+ done$/waiting for server to shut down.... done/; + s/^\/.*postgres /POSTGRES /; + + # DMARC is not always supported by the build + next if /^dmarc_tld_file =/; + + # ARC is not always supported by the build + next if /^arc_sign =/; + + # TLS resumption is not always supported by the build + next if /^tls_resumption_hosts =/; + next if /^-tls_resumption/; + + # gsasl library version may not support some methods + s/250-AUTH ANONYMOUS PLAIN SCRAM-SHA-1\K SCRAM-SHA-256//; } # ======== stderr ======== @@ -913,15 +1035,27 @@ RESET_AFTER_EXTRA_LINE_READ: s/^Exim version .*/Exim version x.yz ..../; - # Debugging lines for Exim terminations + # Debugging lines for Exim terminations and process-generation s/(?<=^>>>>>>>>>>>>>>>> Exim pid=)\d+(?= terminating)/pppp/; + s/^(proxy-proc \w{5}-pid) \d+$/$1 pppp/; + s/^(?:\s*\d+ )(exec .* -oPX)$/pppp $1/; + next if /(?:postfork: | fork(?:ing|ed) for )/; # IP address lookups use gethostbyname() when IPv6 is not supported, # and gethostbyname2() or getipnodebyname() when it is. s/\b(gethostbyname2?|\bgetipnodebyname)(\(af=inet\))?/get[host|ipnode]byname[2]/; + # we don't care what TZ enviroment the testhost was running + next if /^Reset TZ to/; + + # ========= Exim lookups ================== + # Lookups have a char which depends on the number of lookup types compiled in, + # in stderr output. Replace with a "0". Recognising this while avoiding + # other output is fragile; perhaps the debug output should be revised instead. + s%^\s+(:?closing )?\K[0-?]TESTSUITE/aux-fixed/%0TESTSUITE/aux-fixed/%g; + # drop gnutls version strings next if /GnuTLS compile-time version: \d+[\.\d]+$/; next if /GnuTLS runtime version: \d+[\.\d]+$/; @@ -930,6 +1064,9 @@ RESET_AFTER_EXTRA_LINE_READ: next if /OpenSSL compile-time version: OpenSSL \d+[\.\da-z]+/; next if /OpenSSL runtime version: OpenSSL \d+[\.\da-z]+/; + # this is timing-dependent + next if /^OpenSSL: creating STEK$/; + # drop lookups next if /^Lookups \(built-in\):/; next if /^Loading lookup modules from/; @@ -979,7 +1116,7 @@ RESET_AFTER_EXTRA_LINE_READ: next if /name=localhost address=::1/; # drop pdkim debugging header - next if /^PDKIM <<<<<<<<<<<<<<<<<<<<<<<<<<<<<+$/; + next if /^DKIM( <<<<<<<<<<<<<<<<<<<<<<<<<<<<<+|: no signatures)$/; # Various other IPv6 lines must be omitted too @@ -987,6 +1124,8 @@ RESET_AFTER_EXTRA_LINE_READ: next if /get\[host\|ipnode\]byname\[2\]\(af=inet6\)/; next if /DNS lookup of \S+ \(AAAA\) using fakens/; next if / in dns_ipv4_lookup?/; + next if / writing neg-cache entry for .*AAAA/; + next if /^faking res_search\(AAAA\) response length as 65535/; if (/DNS lookup of \S+ \(AAAA\) gave NO_DATA/) { @@ -994,6 +1133,17 @@ RESET_AFTER_EXTRA_LINE_READ: next; } + # Non-TLS bulds have a different Recieved: header expansion + s/^((.*)\t}}}}by \$primary_hostname \$\{if def:received_protocol \{with \$received_protocol }})\(Exim \$version_number\)$/$1\${if def:tls_in_cipher_std { tls \$tls_in_cipher_std\n$2\t}}(Exim \$version_number)/; + s/^((\s*).*considering: with \$received_protocol }})\(Exim \$version_number\)$/$1\${if def:tls_in_cipher_std { tls \$tls_in_cipher_std\n$2\t}}(Exim \$version_number)/; + if (/condition: def:tls_in_cipher_std$/) + { + $_= <IN>; $_= <IN>; $_= <IN>; $_= <IN>; + $_= <IN>; $_= <IN>; $_= <IN>; $_= <IN>; + $_= <IN>; $_= <IN>; $_= <IN>; next; + } + + # Skip tls_advertise_hosts and hosts_require_tls checks when the options # are unset, because tls ain't always there. @@ -1022,12 +1172,14 @@ RESET_AFTER_EXTRA_LINE_READ: # Some DBM libraries seem to make DBM files on opening with O_RDWR without # O_CREAT; other's don't. In the latter case there is some debugging output # which is not present in the former. Skip the relevant lines (there are - # two of them). + # three of them). - if (/TESTSUITE\/spool\/db\/\S+ appears not to exist: trying to create/) + if (/returned from EXIM_DBOPEN: \(nil\)/) { - $_ = <IN>; - next; + $_ .= <IN>; + s?\Q$parm_cwd\E?TESTSUITE?g; + if (/TESTSUITE\/spool\/db\/\S+ appears not to exist: trying to create/) + { $_ = <IN>; next; } } # Some tests turn on +expand debugging to check on expansions. @@ -1046,17 +1198,49 @@ RESET_AFTER_EXTRA_LINE_READ: # remote port numbers vary s/(Connection request from 127.0.0.1 port) \d{1,5}/$1 sssss/; + # Platform-dependent error strings + s/Operation timed out/Connection timed out/; + + # Platform differences on disconnect + s/unexpected disconnection while reading SMTP command from \[127.0.0.1\] \K\(error: Connection reset by peer\) //; + + # Platform-dependent resolver option bits + s/^ (?:writing|update) neg-cache entry for [^,]+-\K[0-9a-f]+, ttl/xxxx, ttl/; + + # timing variance, run-to-run + s/^time on queue = \K1s/0s/; + + # content-scan: file order can vary in directory + s%unspool_mbox\(\): unlinking 'TESTSUITE/spool/scan/[^/]*/\K[^\']*%FFFFFFFFF%; + # Skip hosts_require_dane checks when the options # are unset, because dane ain't always there. - next if /in\shosts_require_dane\?\sno\s\(option\sunset\)/x; + # daemon notifier socket + s/^(\s*\d+|ppppp) (creating notifier socket)$/ppppp $2/; + s/^ \@(.*exim_daemon_notify)$/ $1/; + s/^(\s*\d+|ppppp) \@?(.*exim_daemon_notify)$/ppppp $2/; + next if /unlinking notifier socket/; + + # DISABLE_OCSP + next if /in hosts_requ(est|ire)_ocsp\? (no|yes)/; + # SUPPORT_PROXY next if /host in hosts_proxy\?/; + # PIPE_CONNECT + next if / in (pipelining_connect_advertise_hosts|hosts_pipe_connect)?\? no /; + # Experimental_International next if / in smtputf8_advertise_hosts\? no \(option unset\)/; + # Experimental_REQUIRETLS + next if / in tls_advertise_requiretls?\? no \(end of list\)/; + + # TCP Fast Open + next if /^(ppppp )?setsockopt FASTOPEN: Network Error/; + # Environment cleaning next if /\w+ in keep_environment\? (yes|no)/; @@ -1070,31 +1254,94 @@ RESET_AFTER_EXTRA_LINE_READ: if (s/(with \$received_protocol)\}\} \$\{if def:tls_cipher \{\(\$tls_cipher\)\n$/$1/) { $_ .= <IN>; - s/\s+\}\}(?=\(Exim )/\}\} /; + s/[\sâ]+\}\}(?=\(Exim )/\}\} /; } - if (/^ condition: def:tls_cipher$/) + if (/^ âââcondition: def:tls_cipher$/) { <IN>; <IN>; <IN>; <IN>; <IN>; <IN>; <IN>; <IN>; <IN>; <IN>; <IN>; next; } # Not all platforms build with DKIM enabled - next if /^PDKIM >> Body data for hash, canonicalized/; + next if /^DKIM >> Body data for hash, canonicalized/; + + # Not all platforms build with SPF enabled + next if /^(spf_conn_init|SPF_dns_exim_new|spf_compile\.c)/; + + # Not all platforms have sendfile support + next if /^cannot use sendfile for body: no support$/; # Parts of DKIM-specific debug output depend on the time/date next if /^date:\w+,\{SP\}/; - next if /^PDKIM \[[^[]+\] (Header hash|b) computed:/; + next if /^DKIM \[[^[]+\] (Header hash|b) computed:/; # Not all platforms support TCP Fast Open, and the compile omits the check - if (s/\S+ in hosts_try_fastopen\? no \(option unset\)\n$//) + if (s/\S+ in hosts_try_fastopen\? (no \(option unset\)|no \(end of list\)|yes \(matched "\*"\))\n$//) { + chomp; $_ .= <IN>; s/ \.\.\. >>> / ... /; + if (s/ non-TFO mode connection attempt to 224.0.0.0, 0 data\b$//) { chomp; $_ .= <IN>; } s/Address family not supported by protocol family/Network Error/; s/Network is unreachable/Network Error/; } - next if /^(ppppp )?setsockopt FASTOPEN: Protocol not available$/; + s/^(Connecting to .* \.\.\. sending) \d+ (nonTFO early-data)$/$1 dd $2/; + + if (/^([0-9: ]* # possible timestamp + Connecting\ to\ [^ ]+\ [^ ]+(\ from\ [^ ]+)?)\ \.\.\. + \ .*TFO\ mode\x20 + (sendto,\ no\ data:\ EINPROGRESS # Linux + |connection\ attempt\ to\ [^,]+,\ 0\ data) # MacOS & no-support + $/x) + { + $_ = $1 . " ... " . <IN>; + s/^(.* \.\.\.) [0-9: ]*connected$/$1 connected/; + + if (/^Connecting to .* \.\.\. connected$/) + { + $_ .= <IN>; + if (/^(Connecting to .* \.\.\. )connected\n\s+SMTP(\(close\)>>|\(Connection refused\)<<)$/) + { + $_ = $1 . "failed: Connection refused\n" . <IN>; + s/^(Connecting .*)\n\s+SMTP\(close\)>>$/$1/; + } + elsif (/^(Connecting to .* \.\.\. connected\n)read response data: size=/) + { $_ = $1; } + + # Date/time in SMTP banner + s/[A-Z][a-z]{2},\s\d\d?\s[A-Z][a-z]{2}\s\d{4}\s\d\d\:\d\d:\d\d\s[-+]\d{4} + /Tue, 2 Mar 1999 09:44:33 +0000/gx; + } + } + + # Specific pointer values reported for DB operations change from run to run + s/^(\s*returned from EXIM_DBOPEN: )(0x)?[0-9a-f]+/${1}0xAAAAAAAA/; + s/^(\s*EXIM_DBCLOSE.)(0x)?[0-9a-f]+/${1}0xAAAAAAAA/; + + # Platform-dependent output during MySQL startup + next if /PerconaFT file system space/; + next if /^Waiting for MySQL server to answer/; + next if /mysqladmin: CREATE DATABASE failed; .* database exists/; + + # Postgres version-dependent differences + s/^initdb: warning: (enabling "trust" authentication for local connections)$/\nWARNING: $1/; + + # Not all builds include DMARC + next if /^DMARC: no (dmarc_tld_file|sender_host_address)$/ ; + + # TLS resumption is not always supported by the build + next if /in tls_resumption_hosts\?/; + + # Platform differences in errno strings + s/ SMTP\(Operation timed out\)<</ SMTP(Connection timed out)<</; + + # Platform differences for errno values (eg. Hurd) + s/^errno = \d+$/errno = EEE/; + s/^writing error \d+: /writing error EEE: /; + + # Some platforms have to flip to slow-mode taint-checking + next if /switching to slow-mode taint checking/; # When Exim is checking the size of directories for maildir, it uses # the check_dir_size() function to scan directories. Of course, the order @@ -1128,6 +1375,7 @@ RESET_AFTER_EXTRA_LINE_READ: /^Support for:/ || /^Routers:/ || /^Transports:/ || + /^Malware:/ || /^log selectors =/ || /^cwd=/ || /^Fixed never_users:/ || @@ -1147,6 +1395,70 @@ RESET_AFTER_EXTRA_LINE_READ: { # Berkeley DB version differences next if / Berkeley DB error: /; + + # CHUNKING: exact sizes depend on hostnames in headers + s/(=>.* K C="250- \d)\d+ (byte chunk, total \d)\d+/$1nn $2nn/; + + # openssl version variances + s/(TLS error on connection [^:]*: error:)[0-9A-F]{8}(:system library):(?:fopen|func\(4095\)):(No such file or directory)$/$1xxxxxxxx$2:fopen:$3/; + s/(DANE attempt failed.*error:)[0-9A-F]{8}(:SSL routines:)(?:(?i)ssl3_get_server_certificate|tls_process_server_certificate|CONNECT_CR_CERT)(?=:certificate verify failed$)/$1xxxxxxxx$2ssl3_get_server_certificate/; + s/(DKIM: validation error: )error:[0-9A-F]{8}:rsa routines:(?:(?i)int_rsa_verify|CRYPTO_internal):(?:bad signature|algorithm mismatch)$/$1Public key signature verification has failed./; + s/ARC: AMS signing: privkey PEM-block import: error:\K[0-9A-F]{8}:(PEM routines):get_name:(no start line)/0906D06C:$1:PEM_read_bio:$2/; + + # gnutls version variances + if (/TLS error on connection \(recv\): .* (Decode error|peer did not send any certificate)/) + { + my $prev = $_; + $_ = <IN>; + if (/error on first read/) + { + s/TLS session: \Kerror on first read:/(gnutls_handshake): A TLS fatal alert has been received.:/; + goto RESET_AFTER_EXTRA_LINE_READ; + } + else + { $_ = $prev; } + } + # translate gnutls error into the openssl one + s/ARC: AMS signing: privkey PEM-block import: \KThe requested data were not available.$/error:0906D06C:PEM routines:PEM_read_bio:no start line/; + + # DKIM timestamps + if ( /(DKIM: d=.*) t=([0-9]*) x=([0-9]*) / ) + { + my ($prefix, $t_diff) = ($1, $3 - $2); + s/DKIM: d=.* t=[0-9]* x=[0-9]* /${prefix} t=T x=T+${t_diff} /; + } + # GnuTLS reports a different keysize vs. OpenSSL, for ed25519 keys + s/signer: [^ ]* bits:\K 256/ 253/; + s/public key too short:\K 256 bits/ 253 bits/; + + # port numbers + s/(?:\[[^\]]*\]:|port )\K$parm_port_d/PORT_D/; + s/(?:\[[^\]]*\]:|port )\K$parm_port_d2/PORT_D2/; + s/(?:\[[^\]]*\]:|port )\K$parm_port_d3/PORT_D3/; + s/(?:\[[^\]]*\]:|port )\K$parm_port_d4/PORT_D4/; + s/(?:\[[^\]]*\]:|port )\K$parm_port_s/PORT_S/; + s/(?:\[[^\]]*\]:|port )\K$parm_port_n/PORT_N/; + s/I=\[[^\]]*\]:\K\d+/ppppp/; + + # Platform differences for errno values (eg. Hurd). Leave 0 and negative numbers alone. + s/R=\w+ T=\w+ defer\K \([1-9]\d*\): / (EEE): /; + + # Platform differences in errno strings + s/Arg list too long/Argument list too long/; + } + + # ======== mail ======== + + elsif ($is_mail) + { + # DKIM timestamps, and signatures depending thereon + if ( /^(\s+)t=([0-9]*); x=([0-9]*); b=[A-Za-z0-9+\/]+$/ ) + { + my ($indent, $t_diff) = ($1, $3 - $2); + s/.*/${indent}t=T; x=T+${t_diff}; b=bbbb;/; + <IN>; + <IN>; + } } # ======== All files other than stderr ======== @@ -1274,8 +1586,8 @@ if (! -e $sf_current) log_failure($log_failed_filename, $testno, $rf); log_test($log_summary_filename, $testno, 'F') if ($force_continue); } - return 1 if /^c$/i; - last if (/^s$/); + return 1 if /^c$/i && $rf !~ /paniclog/ && (!defined $rsf || $rsf !~ /paniclog/); + last if (/^[sc]$/); } foreach $f ($rf, $rsf) @@ -1285,7 +1597,7 @@ if (! -e $sf_current) print "\n"; print "------------ $f -----------\n" if (defined $rf && -s $rf && defined $rsf && -s $rsf); - system("$more '$f'"); + system @more => $f; } } @@ -1367,26 +1679,25 @@ if (-e $sf_current) } } - open(MUNGED, '>', $mf) || tests_exit(-1, "Failed to open $mf: $!"); - for ($i = 0; $i < @munged; $i++) - { print MUNGED $munged[$i]; } - close(MUNGED); + open(my $fh, '>', $mf) or tests_exit(-1, "Failed to open $mf: $!"); + print $fh @munged; } # Deal with log sorting if ($sortfile) { - my(@munged, $i, $j); - open(MUNGED, $mf) || tests_exit(-1, "Failed to open $mf: $!"); - @munged = <MUNGED>; - close(MUNGED); + my @munged = do { + open(my $fh, '<', $mf) or tests_exit(-1, "Failed to open $mf: $!"); + <$fh>; + }; - for ($i = 0; $i < @munged; $i++) + for (my $i = 0; $i < @munged; $i++) { if ($munged[$i] =~ /^[-\d]{10}\s[:\d]{8}\s[-A-Za-z\d]{16}\s[-=*]>/) { + my $j; for ($j = $i + 1; $j < @munged; $j++) { last if $munged[$j] !~ @@ -1398,11 +1709,9 @@ if (-e $sf_current) } } - open(MUNGED, ">$mf") || tests_exit(-1, "Failed to open $mf: $!"); - print MUNGED "**NOTE: The delivery lines in this file have been sorted.\n"; - for ($i = 0; $i < @munged; $i++) - { print MUNGED $munged[$i]; } - close(MUNGED); + open(my $fh, '>', $mf) or tests_exit(-1, "Failed to open $mf: $!"); + print $fh "**NOTE: The delivery lines in this file have been sorted.\n"; + print $fh @munged; } # Do the comparison @@ -1412,7 +1721,7 @@ if (-e $sf_current) # Handle comparison failure print "** Comparison of $mf with $sf_current failed"; - system("$more test-cf"); + system @more => 'test-cf'; print "\n"; for (;;) @@ -1443,8 +1752,7 @@ else # if we deal with a flavour file, we can't delete it, because next time the generic # file would be used again if ($sf_current eq $sf_flavour) { - open(FOO, ">$sf_current"); - close(FOO); + open(my $fh, '>', $sf_current); } else { tests_exit(-1, "Failed to unlink $sf_current") if !unlink($sf_current); @@ -1476,6 +1784,11 @@ $munges = 'gnutls_handshake' => { 'mainlog' => 's/\(gnutls_handshake\): Error in the push function/\(gnutls_handshake\): A TLS packet with unexpected length was received/' }, + 'gnutls_bad_clientcert' => + { 'mainlog' => 's/\(certificate verification failed\): certificate invalid/\(gnutls_handshake\): The peer did not send any certificate./', + 'stdout' => 's/Succeeded in starting TLS/A TLS fatal alert has been received.\nFailed to start TLS' + }, + 'optional_events' => { 'stdout' => '/event_action =/' }, @@ -1495,10 +1808,17 @@ $munges = { 'stderr' => 's/(1[5-9]|23\d)\d\d msec/ssss msec/' }, 'tls_anycipher' => - { 'mainlog' => 's/ X=TLS\S+ / X=TLS_proto_and_cipher /' }, + { 'mainlog' => 's! X=TLS\S+ ! X=TLS_proto_and_cipher !; + s! DN="C=! DN="/C=!; + s! DN="[^,"]*\K,!/!; + s! DN="[^,"]*\K,!/!; + s! DN="[^,"]*\K,!/!; + ', + 'rejectlog' => 's/ X=TLS\S+ / X=TLS_proto_and_cipher /', + }, 'debug_pid' => - { 'stderr' => 's/(^\s{0,4}|(?<=Process )|(?<=child ))\d{1,5}/ppppp/g' }, + { 'stderr' => 's/(^\s{0,4}|(?<=Process )|(?<=child ))\d+/ppppp/g' }, 'optional_dsn_info' => { 'mail' => '/^(X-(Remote-MTA-(smtp-greeting|helo-response)|Exim-Diagnostic|(body|message)-linecount):|Remote-MTA: X-ip;)/' @@ -1506,13 +1826,18 @@ $munges = 'optional_config' => { 'stdout' => '/^( - dkim_(canon|domain|private_key|selector|sign_headers|strict) + dkim_(canon|domain|private_key|selector|sign_headers|strict|hash|identity|timestamps) |gnutls_require_(kx|mac|protocols) + |hosts_pipe_connect |hosts_(requ(est|ire)|try)_(dane|ocsp) - |hosts_(avoid|nopass|require|verify_avoid)_tls + |dane_require_tls_ciphers + |hosts_(avoid|nopass|noproxy|require|verify_avoid)_tls + |pipelining_connect_advertise_hosts |socks_proxy |tls_[^ ]* - )($|[ ]=)/x' }, + |utf8_downconvert + )($|[ ]=)/x' + }, 'sys_bindir' => { 'mainlog' => 's%/(usr/(local/)?)?bin/%SYSBINDIR/%' }, @@ -1522,12 +1847,18 @@ $munges = 'rejectlog' => 's/^(.* SMTP protocol synchronization error .* next input=.{8}).*$/$1<suppressed>/'}, 'debuglog_stdout' => - { 'stdout' => 's/^[ .]*\d\d:\d\d:\d\d\s+\d+ //; + { 'stdout' => 's/^\d\d:\d\d:\d\d\s+\d+ //; s/Process \d+ is ready for new message/Process pppp is ready for new message/' }, 'timeout_errno' => # actual errno differs Solaris vs. Linux - { 'mainlog' => 's/(host deferral .* errno) <\d+> /$1 <EEE> /' }, + { 'mainlog' => 's/((?:host|message) deferral .* errno) <\d+> /$1 <EEE> /' }, + + 'peer_terminated_conn' => # actual error differs FreedBSD vs. Linux + { 'stderr' => 's/^( SMTP\()Connection reset by peer(\)<<)$/$1closed$2/' }, + + 'perl_variants' => # result of hash-in-scalar-context changed from bucket-fill to keycount + { 'stdout' => 's%^> X/X$%> X%' }, }; @@ -1883,34 +2214,37 @@ if (/^dbmbuild\s+(\S+)\s+(\S+)/) if (/^dump\s+(\S+)/) { - my($which) = $1; - my(@temp); + my $which = $1; print ">> ./eximdir/exim_dumpdb $parm_cwd/spool $which\n" if $debug; - open(IN, "./eximdir/exim_dumpdb $parm_cwd/spool $which |"); - open(OUT, ">>test-stdout"); - print OUT "+++++++++++++++++++++++++++\n"; + open(my $in, "-|", './eximdir/exim_dumpdb', "$parm_cwd/spool", $which) or die "Can't run exim_dumpdb: $!"; + open(my $out, ">>test-stdout"); + print $out "+++++++++++++++++++++++++++\n"; if ($which eq "retry") { - $/ = "\n "; - @temp = <IN>; - $/ = "\n"; - - @temp = sort { - my($aa) = split(' ', $a); - my($bb) = split(' ', $b); - return $aa cmp $bb; - } @temp; - + # the sort key is the first part of the retry db dump line, but for + # sorting we (temporarly) replace the own hosts ipv4 with a munged + # version, which matches the munging that is done later + # Why? We must ensure sure, that 127.0.0.1 always sorts first + # map-sort-map: Schwartz's transformation + # test 0099 + my @temp = map { $_->[1] } + sort { $a->[0] cmp $b->[0] } + #map { [ (split)[0] =~ s/\Q$parm_ipv4/ip4.ip4.ip4.ip4/gr, $_ ] } # this is too modern for 5.10.1 + map { + (my $k = (split)[0]) =~ s/\Q$parm_ipv4/ip4.ip4.ip4.ip4/g; + [ $k, $_ ] + } + do { local $/ = "\n "; <$in> }; foreach $item (@temp) { $item =~ s/^\s*(.*)\n(.*)\n?\s*$/$1\n$2/m; - print OUT " $item\n"; + print $out " $item\n"; } } else { - @temp = <IN>; + my @temp = <$in>; if ($which eq "callout") { @temp = sort { @@ -1919,11 +2253,9 @@ if (/^dump\s+(\S+)/) return $aa cmp $bb; } @temp; } - print OUT @temp; + print $out @temp; } - - close(IN); - close(OUT); + close($in); # close it explicitly, otherwise $? does not get set return 1; } @@ -2013,8 +2345,7 @@ if (/^gnutls/) # The "killdaemon" command should ultimately follow the starting of any Exim -# daemon with the -bd option. We kill with SIGINT rather than SIGTERM to stop -# it outputting "Terminated" to the terminal when not in the background. +# daemon with the -bd option. if (/^killdaemon/) { @@ -2026,14 +2357,14 @@ if (/^killdaemon/) print ">> killdaemon: recovered pid $pid\n" if $debug; if ($pid) { - run_system("sudo /bin/kill -INT $pid"); + run_system("sudo /bin/kill -TERM $pid"); wait; } } else { $pid = `cat $parm_cwd/spool/exim-daemon.*`; if ($pid) { - run_system("sudo /bin/kill -INT $pid"); + run_system("sudo /bin/kill -TERM $pid"); close DAEMONCMD; # Waits for process } } @@ -2054,7 +2385,7 @@ elsif (/^millisleep\s+(.*)$/) # The "munge" command selects one of a hardwired set of test-result modifications -# to be made before result compares are run agains the golden set. This lets +# to be made before result compares are run against the golden set. This lets # us account for test-system dependent things which only affect a few, but known, # test-cases. # Currently only the last munge takes effect. @@ -2266,9 +2597,24 @@ elsif (/^((?i:[A-Z\d_]+=\S+\s+)+)?(\d+)?\s*(sudo(?:\s+-u\s+(\w+))?\s+)?exim(_\S+ if ($args =~ /\$msg/) { - my @listcmd = ("$parm_cwd/eximdir/exim", '-bp', + my($queuespec); + if ($args =~ /-qG\w+/) { $queuespec = $&; } + + my @listcmd; + + if (defined $queuespec) + { + @listcmd = ("$parm_cwd/eximdir/exim", '-bp', + $queuespec, + "-DEXIM_PATH=$parm_cwd/eximdir/exim", + -C => "$parm_cwd/test-config"); + } + else + { + @listcmd = ("$parm_cwd/eximdir/exim", '-bp', "-DEXIM_PATH=$parm_cwd/eximdir/exim", -C => "$parm_cwd/test-config"); + } print ">> Getting queue list from:\n>> @listcmd\n" if $debug; # We need the message ids sorted in ascending order. # Message id is: <timestamp>-<pid>-<fractional-time>. On some systems (*BSD) the @@ -2316,7 +2662,6 @@ elsif (/^((?i:[A-Z\d_]+=\S+\s+)+)?(\d+)?\s*(sudo(?:\s+-u\s+(\w+))?\s+)?exim(_\S+ if ($cmd =~ /\s-DSERVER=server\s/ && $cmd !~ /\s-DNOTDAEMON\s/) { - $pidfile = "$parm_cwd/spool/exim-daemon.pid"; if ($debug) { printf ">> daemon: $cmd\n"; } run_system("sudo mkdir spool/log 2>/dev/null"); run_system("sudo chown $parm_eximuser:$parm_eximgroup spool/log"); @@ -2343,7 +2688,8 @@ elsif (/^((?i:[A-Z\d_]+=\S+\s+)+)?(\d+)?\s*(sudo(?:\s+-u\s+(\w+))?\s+)?exim(_\S+ while (<SCRIPT>) { $lineno++; last if /^\*{4}\s*$/; } # Ignore any input # Interlock with daemon startup - while (! stat("$pidfile") ) { select(undef, undef, undef, 0.3); } + for (my $count = 0; ! stat("$pidfile") && $count < 30; $count++ ) + { select(undef, undef, undef, 0.3); } return 3; # Don't wait } elsif ($cmd =~ /\s-DSERVER=wait:(\d+)\s/) @@ -2384,6 +2730,7 @@ elsif (/^background$/) $_ = <SCRIPT>; $lineno++; chomp; + do_substitute($testno); $line = $_; if ($debug) { printf ">> daemon: $line >>test-stdout 2>>test-stderr\n"; } @@ -2492,23 +2839,7 @@ umask 022; # Check for the "less" command # ################################################## -$more = 'more' if system('which less >/dev/null 2>&1') != 0; - - - -################################################## -# Check for sudo access to root # -################################################## - -print "You need to have sudo access to root to run these tests. Checking ...\n"; -if (system('sudo true >/dev/null') != 0) - { - die "** Test for sudo failed: testing abandoned.\n"; - } -else - { - print "Test for sudo OK\n"; - } +@more = 'more' if system('which less >/dev/null 2>&1') != 0; @@ -2520,10 +2851,6 @@ else # as the path to the binary. If the first argument does not start with a # '/' but exists in the file system, it's assumed to be the Exim binary. -($parm_exim, @ARGV) = Exim::Runtest::exim_binary(@ARGV); -print "Exim binary is $parm_exim\n" if $parm_exim ne ''; - - ################################################## # Sort out options and which tests are to be run # @@ -2533,38 +2860,60 @@ print "Exim binary is $parm_exim\n" if $parm_exim ne ''; # options are passed on to Exim calls within the tests. Typically, this is used # to turn on Exim debugging while setting up a test. -while (@ARGV > 0 && $ARGV[0] =~ /^-/) - { - my($arg) = shift @ARGV; - if ($optargs eq '') - { - if ($arg eq "-DEBUG") { $debug = 1; $cr = "\n"; next; } - if ($arg eq "-DIFF") { $cf = "diff -u"; next; } - if ($arg eq "-CONTINUE"){$force_continue = 1; - $more = "cat"; - next; } - if ($arg eq "-UPDATE") { $force_update = 1; next; } - if ($arg eq "-NOIPV4") { $have_ipv4 = 0; next; } - if ($arg eq "-NOIPV6") { $have_ipv6 = 0; next; } - if ($arg eq "-KEEP") { $save_output = 1; next; } - if ($arg eq "-VALGRIND") { $valgrind = 1; next; } - if ($arg =~ /^-FLAVOU?R$/) { $flavour = shift; next; } - } - $optargs .= " $arg"; - } +Getopt::Long::Configure qw(no_getopt_compat); +GetOptions( + 'debug' => sub { $debug = 1; $cr = "\n" }, + 'diff' => sub { $cf = 'diff -u' }, + 'continue' => sub { $force_continue = 1; @more = 'cat' }, + 'update' => \$force_update, + 'ipv4!' => \$have_ipv4, + 'ipv6!' => \$have_ipv6, + 'keep' => \$save_output, + 'slow' => \$slow, + 'valgrind' => \$valgrind, + 'range=s{2}' => \my @range_wanted, + 'test=i@' => \my @tests_wanted, + 'flavor|flavour=s' => \$flavour, + 'help' => sub { pod2usage(-exit => 0) }, + 'man' => sub { + pod2usage( + -exit => 0, + -verbose => 2, + -noperldoc => system('perldoc -V 2>/dev/null 1>&2') + ); + }, +) or pod2usage; -# Any subsequent arguments are a range of test numbers. +($parm_exim, @ARGV) = Exim::Runtest::exim_binary(@ARGV); +print "Exim binary is `$parm_exim'\n" if defined $parm_exim; + + +my @wanted = sort numerically uniq + @tests_wanted ? @tests_wanted : (), + @range_wanted ? $range_wanted[0] .. $range_wanted[1] : (), + @ARGV ? @ARGV == 1 ? $ARGV[0] : + $ARGV[1] eq '+' ? $ARGV[0]..($ARGV[0] >= 9000 ? TEST_SPECIAL_TOP : TEST_TOP) : + 0+$ARGV[0]..0+$ARGV[1] # add 0 to cope with test numbers starting with zero + : (); +@wanted = 1..TEST_TOP if not @wanted; + +################################################## +# Check for sudo access to root # +################################################## -if (@ARGV > 0) +print "You need to have sudo access to root to run these tests. Checking ...\n"; +if (system('sudo true >/dev/null') != 0) + { + die "** Test for sudo failed: testing abandoned.\n"; + } +else { - $test_end = $test_start = $ARGV[0]; - $test_end = $ARGV[1] if (@ARGV > 1); - $test_end = ($test_start >= 9000)? $test_special_top : $test_top - if $test_end eq "+"; - die "** Test numbers out of order\n" if ($test_end < $test_start); + print "Test for sudo OK\n"; } + + ################################################## # Make the command's directory current # ################################################## @@ -2589,7 +2938,7 @@ $parm_cwd = Cwd::getcwd(); # If $parm_exim is still empty, ask the caller -if ($parm_exim eq '') +if (not $parm_exim) { print "** Did not find an Exim binary to test\n"; for ($i = 0; $i < 5; $i++) @@ -2639,7 +2988,6 @@ foreach (@eximinfo) my $git = `git describe --dirty=-XX --match 'exim-4*'`; if (defined $git and $? == 0) { chomp $git; - $version =~ s/^\d+\K\./_/; $git =~ s/^exim-//i; $git =~ s/.*-\Kg([[:xdigit:]]+(?:-XX)?)/$1/; print <<___ @@ -2702,7 +3050,7 @@ if (defined $parm_trusted_config_list) open(TCL, $parm_trusted_config_list) or die "Can't open $parm_trusted_config_list: $!\n"; my $test_config = getcwd() . '/test-config'; die "Can't find '$test_config' in TRUSTED_CONFIG_LIST $parm_trusted_config_list." - if not grep { /^$test_config$/ } <TCL>; + if not grep { /^\Q$test_config\E$/ } <TCL>; } else { @@ -2716,6 +3064,9 @@ die "CONFIGURE_GROUP ($parm_configure_group) does not match the group invoking $ if 0020 & (stat "$parm_cwd/test-config")[2] and $parm_configure_group != $); +die "aux-fixed file is group-writeable; best to strip them all, recursively\n" + if 0020 & (stat "aux-fixed/0037.f-1")[2]; + open(EXIMINFO, "$parm_exim -d-all+transport -bV -C $parm_cwd/test-config -DDIR=$parm_cwd |") || die "** Cannot run $parm_exim: $!\n"; @@ -2727,6 +3078,7 @@ while (<EXIMINFO>) my(@temp); if (/^(Exim|Library) version/) { print; } + if (/Runtime: /) {print; } elsif (/^Size of off_t: (\d+)/) { @@ -2790,6 +3142,15 @@ while (<EXIMINFO>) } } } + + elsif (/^Malware: (.*)/) + { + print; + @temp = split /(\s+)/, $1; + push(@temp, ' '); + %parm_malware = @temp; + } + } close(EXIMINFO); print "-" x 78, "\n"; @@ -3100,6 +3461,12 @@ unless (defined $parm_eximgroup) die "** ABANDONING.\n"; } +if ($parm_caller_home eq $parm_cwd) + { + print "will confuse working dir with homedir; change homedir\n"; + die "** ABANDONING.\n"; + } + print "You need to be in the Exim group to run these tests. Checking ..."; if (`groups` =~ /\b\Q$parm_eximgroup\E\b/) @@ -3114,21 +3481,29 @@ else # Find this host's IP addresses - there may be many, of course, but we keep # one of each type (IPv4 and IPv6). +#XXX it would be good to avoid non-UP interfaces open(IFCONFIG, '-|', (grep { -x "$_/ip" } split /:/, $ENV{PATH}) ? 'ip address' : 'ifconfig -a') or die "** Cannot run 'ip address' or 'ifconfig -a'\n"; while (not ($parm_ipv4 and $parm_ipv6) and defined($_ = <IFCONFIG>)) { - if (not $parm_ipv4 and /^\s*inet(?:\saddr)?:?\s?(\d+\.\d+\.\d+\.\d+)(?:\/\d+)?\s/i) + if (/^(?:[0-9]+: )?([a-z0-9]+): /) { $ifname = $1; } + + if (not $parm_ipv4 and /^\s*inet(?:\saddr(?:ess))?:?\s*(\d+\.\d+\.\d+\.\d+)(?:\/\d+)?\s/i) { - next if $1 =~ /^(?:127|10)\./; + # It would be nice to be able to vary the /16 used for manyhome; we could take + # an option to runtest used here - but we'd also have to pass it on to fakens. + # Possibly an environment variable? + next if $1 eq '0.0.0.0' or $1 =~ /^(?:127|10\.250)\./; $parm_ipv4 = $1; } - if (not $parm_ipv6 and /^\s*inet6(?:\saddr)?:?\s?([abcdef\d:]+)(?:\/\d+)/i) + if ( (not $parm_ipv6 or $parm_ipv6 =~ /%/) + and /^\s*inet6(?:\saddr(?:ess))?:?\s*([abcdef\d:]+)(?:%[^ \/]+)?(?:\/\d+)?/i) { - next if $1 eq '::1' or $1 =~ /^fe80/i; + next if $1 eq '::' or $1 eq '::1' or $1 =~ /^ff00/i or $1 =~ /^fe80::1/i; $parm_ipv6 = $1; + if ($1 =~ /^fe80/i) { $parm_ipv6 .= '%' . $ifname; } } } close(IFCONFIG); @@ -3185,6 +3560,9 @@ else print "IPv4 address is $parm_ipv4\n"; print "IPv6 address is $parm_ipv6\n"; +$parm_ipv6 =~ /^[^%\/]*/; +# drop any %scope from the ipv6, for some uses +($parm_ipv6_stripped = $parm_ipv6) =~ s/%.*//g; # For munging test output, we need the reversed IP addresses. @@ -3194,7 +3572,7 @@ $parm_ipv4r = ($parm_ipv4 !~ /^\d/)? '' : $parm_ipv6r = $parm_ipv6; # Appropriate if not in use if ($parm_ipv6 =~ /^[\da-f]/) { - my(@comps) = split /:/, $parm_ipv6; + my(@comps) = split /:/, $parm_ipv6_stripped; my(@nibbles); foreach $comp (@comps) { @@ -3229,6 +3607,12 @@ if ($parm_hostname =~ /[[:upper:]]/) print "\n*** Host name has upper case characters: this may cause problems ***\n\n"; } +if ($parm_hostname =~ /\.example\.com$/) + { + die "\n*** Host name ends in .example.com; this conflicts with the testsuite use of that domain.\n" + . " Please change the host's name (or comment out this check, and fail several testcases)\n"; + } + ################################################## @@ -3282,7 +3666,6 @@ system("sudo cp eximdir/exim eximdir/exim_exim;" . "sudo chgrp $parm_eximgroup eximdir/exim_exim;" . "sudo chmod 06755 eximdir/exim_exim"); - ################################################## # Make copies of utilities we might need # ################################################## @@ -3325,6 +3708,15 @@ if (system("cp $parm_exim_dir/eximstats eximdir") != 0) tests_exit(-1, "Failed to make a copy of eximstats: $!"); } +# Collect some version information +print '-' x 78, "\n"; +print "Perl version for runtest: $]\n"; +foreach (map { "./eximdir/$_" } qw(exigrep exinext eximstats)) { + # fold (or unfold?) multiline output into a one-liner + print join(', ', map { chomp; $_ } `$_ --version`), "\n"; +} +print '-' x 78, "\n"; + ################################################## # Check that the Exim user can access stuff # @@ -3367,6 +3759,8 @@ else print " OK\n"; } +tests_exit(-1, "Failed to unlink $log_summary_filename: $!") + if not unlink($log_summary_filename) and -e $log_summary_filename; ################################################## # Create a list of available tests # @@ -3380,33 +3774,21 @@ else # because the current binary does not support the right facilities, and also # those that are outside the numerical range selected. -print "\nTest range is $test_start to $test_end (flavour $flavour)\n"; +printf "\nWill run %d tests between %d and %d for flavour %s\n", + scalar(@wanted), $wanted[0], $wanted[-1], $flavour; + print "Omitting \${dlfunc expansion tests (loadable module not present)\n" if $dlfunc_deleted; print "Omitting dbm tests (unable to copy exim_dbmbuild)\n" if $dbm_build_deleted; -opendir(DIR, "scripts") || tests_exit(-1, "Failed to opendir(\"scripts\"): $!"); -@test_dirs = sort readdir(DIR); -closedir(DIR); - -# Remove . and .. and CVS from the list. -for ($i = 0; $i < @test_dirs; $i++) - { - my($d) = $test_dirs[$i]; - if ($d eq "." || $d eq ".." || $d eq "CVS") - { - splice @test_dirs, $i, 1; - $i--; - } - } +my @test_dirs = grep { not /^CVS$/ } map { basename $_ } glob 'scripts/*' + or die tests_exit(-1, "Failed to find test scripts in 'scripts/*`: $!"); # Scan for relevant tests - -tests_exit(-1, "Failed to unlink $log_summary_filename") - if (-e $log_summary_filename && !unlink($log_summary_filename)); -for ($i = 0; $i < @test_dirs; $i++) +# HS12: Needs to be reworked. +DIR: for (my $i = 0; $i < @test_dirs; $i++) { my($testdir) = $test_dirs[$i]; my($wantthis) = 1; @@ -3416,19 +3798,19 @@ for ($i = 0; $i < @test_dirs; $i++) # Skip this directory if the first test is equal or greater than the first # test in the next directory. - next if ($i < @test_dirs - 1) && - ($test_start >= substr($test_dirs[$i+1], 0, 4)); + next DIR if ($i < @test_dirs - 1) && + ($wanted[0] >= substr($test_dirs[$i+1], 0, 4)); # No need to carry on if the end test is less than the first test in this # subdirectory. - last if $test_end < substr($testdir, 0, 4); + last DIR if $wanted[-1] < substr($testdir, 0, 4); # Check requirements, if any. - if (open(REQUIRES, "scripts/$testdir/REQUIRES")) + if (open(my $requires, "scripts/$testdir/REQUIRES")) { - while (<REQUIRES>) + while (<$requires>) { next if /^\s*$/; s/\s+$//; @@ -3456,12 +3838,45 @@ for ($i = 0; $i < @test_dirs; $i++) { if (!defined $parm_transports{$1}) { $wantthis = 0; last; } } + elsif (/^malware (.*)$/) + { + if (!defined $parm_malware{$1}) { $wantthis = 0; last; } + } + elsif (/^feature (.*)$/) + { + # move to a subroutine? + my $eximinfo = "$parm_exim -C $parm_cwd/test-config -DDIR=$parm_cwd -bP macro $1"; + + open (IN, "$parm_cwd/confs/0000") || + tests_exit(-1, "Couldn't open $parm_cwd/confs/0000: $!\n"); + open (OUT, ">test-config") || + tests_exit(-1, "Couldn't open test-config: $!\n"); + while (<IN>) + { + do_substitute($testno); + print OUT; + } + close(IN); + close(OUT); + + system($eximinfo . " >/dev/null 2>&1"); + if ($? != 0) { + unlink("$parm_cwd/test-config"); + $wantthis = 0; + $_ = "feature $1"; + last; + } + unlink("$parm_cwd/test-config"); + } + elsif (/^ipv6-non-linklocal/) + { + if ($parm_ipv6 =~ /%/) { $wantthis = 0; last; } + } else { tests_exit(-1, "Unknown line in \"scripts/$testdir/REQUIRES\": \"$_\""); } } - close(REQUIRES); } else { @@ -3480,15 +3895,13 @@ for ($i = 0; $i < @test_dirs; $i++) # We want the tests from this subdirectory, provided they are in the # range that was selected. - opendir(SUBDIR, "scripts/$testdir") || - tests_exit(-1, "Failed to opendir(\"scripts/$testdir\"): $!"); - @testlist = sort readdir(SUBDIR); - close(SUBDIR); + @testlist = grep { $_ ~~ @wanted } grep { /^\d+(?:\.\d+)?$/ } map { basename $_ } glob "scripts/$testdir/*"; + tests_exit(-1, "Failed to read test scripts from `scripts/$testdir/*': $!") + if not @testlist; foreach $test (@testlist) { - next if ($test !~ /^\d{4}(?:\.\d+)?$/); - if (!$wantthis || $test < $test_start || $test > $test_end) + if (!$wantthis) { log_test($log_summary_filename, $test, '.'); } @@ -3499,7 +3912,7 @@ for ($i = 0; $i < @test_dirs; $i++) } } -print ">>Test List: @test_list\n", if $debug; +print ">>Test List:\n", join "\n", @test_list, '' if $debug; ################################################## @@ -3584,7 +3997,7 @@ if ($have_ipv4 || $have_ipv6) "; for queries that it cannot answer\n\n" . "PASS ON NOT FOUND\n\n"; print OUT "$shortname A $parm_ipv4\n" if $have_ipv4; - print OUT "$shortname AAAA $parm_ipv6\n" if $have_ipv6; + print OUT "$shortname AAAA $parm_ipv6_stripped\n" if $have_ipv6; print OUT "\n; End\n"; close(OUT); } @@ -3592,28 +4005,39 @@ if ($have_ipv4 || $have_ipv6) if ($have_ipv4 && $parm_ipv4 ne "127.0.0.1") { my(@components) = $parm_ipv4 =~ /^(\d+)\.(\d+)\.(\d+)\.(\d+)/; - open(OUT, ">$parm_cwd/dnszones/db.ip4.$components[0]") || - tests_exit(-1, - "Failed to open $parm_cwd/dnszones/db.ip4.$components[0]: $!"); - print OUT "; This is a dynamically constructed fake zone file.\n" . - "; The zone is $components[0].in-addr.arpa.\n\n" . - "$components[3].$components[2].$components[1] PTR $parm_hostname.\n\n" . - "; End\n"; - close(OUT); + + if ($components[0]=='10') + { + open(OUT, ">>$parm_cwd/dnszones/db.ip4.$components[0]") || + tests_exit(-1, "Failed to open $parm_cwd/dnszones/db.ip4.$components[0]: $!"); + print OUT "$components[3].$components[2].$components[1] PTR $parm_hostname.\n\n"; + close(OUT); + } + else + { + open(OUT, ">$parm_cwd/dnszones/db.ip4.$components[0]") || + tests_exit(-1, + "Failed to open $parm_cwd/dnszones/db.ip4.$components[0]: $!"); + print OUT "; This is a dynamically constructed fake zone file.\n" . + "; The zone is $components[0].in-addr.arpa.\n\n" . + "$components[3].$components[2].$components[1] PTR $parm_hostname.\n\n" . + "; End\n"; + close(OUT); + } } -if ($have_ipv6 && $parm_ipv6 ne "::1") +if ($have_ipv6 && $parm_ipv6_stripped ne "::1") { - my($exp_v6) = $parm_ipv6; + my($exp_v6) = $parm_ipv6_stripped; $exp_v6 =~ s/[^:]//g; - if ( $parm_ipv6 =~ /^([^:].+)::$/ ) { + if ( $parm_ipv6_stripped =~ /^([^:].+)::$/ ) { $exp_v6 = $1 . ':0' x (9-length($exp_v6)); - } elsif ( $parm_ipv6 =~ /^(.+)::(.+)$/ ) { + } elsif ( $parm_ipv6_stripped =~ /^(.+)::(.+)$/ ) { $exp_v6 = $1 . ':0' x (8-length($exp_v6)) . ':' . $2; - } elsif ( $parm_ipv6 =~ /^::(.+[^:])$/ ) { + } elsif ( $parm_ipv6_stripped =~ /^::(.+[^:])$/ ) { $exp_v6 = '0:' x (9-length($exp_v6)) . $1; } else { - $exp_v6 = $parm_ipv6; + $exp_v6 = $parm_ipv6_stripped; } my(@components) = split /:/, $exp_v6; my(@nibbles) = reverse (split /\s*/, shift @components); @@ -3669,27 +4093,31 @@ closedir(DIR); if (not $force_continue) { # runtest needs to interact if we're not in continue # mode. It does so by communicate to /dev/tty - open(T, "/dev/tty") or tests_exit(-1, "Failed to open /dev/tty: $!"); + open(T, '<', '/dev/tty') or tests_exit(-1, "Failed to open /dev/tty: $!"); + print "\nPress RETURN to run the tests: "; + <T>; } -print "\nPress RETURN to run the tests: "; -$_ = $force_continue ? "c" : <T>; -print "\n"; - -$lasttestdir = ''; - foreach $test (@test_list) { - local($lineno) = 0; - local($commandno) = 0; - local($subtestno) = 0; + state $lasttestdir = ''; + + local $lineno = 0; + local $commandno = 0; + local $subtestno = 0; + local $sortlog = 0; + (local $testno = $test) =~ s|.*/||; - local($sortlog) = 0; - my($gnutls) = 0; - my($docheck) = 1; - my($thistestdir) = substr($test, 0, -5); + # Leaving traces in the process table and in the environment + # gives us a chance to identify hanging processes (exim daemons) + local $0 = "[runtest $testno]"; + local $ENV{EXIM_TEST_NUMBER} = $testno; + + my $gnutls = 0; + my $docheck = 1; + my $thistestdir = substr($test, 0, -5); $dynamic_socket->close() if $dynamic_socket; @@ -3698,20 +4126,19 @@ foreach $test (@test_list) $gnutls = 0; if (-s "scripts/$thistestdir/REQUIRES") { - my($indent) = ''; + my $indent = ''; print "\n>>> The following tests require: "; - open(IN, "scripts/$thistestdir/REQUIRES") || - tests_exit(-1, "Failed to open scripts/$thistestdir/REQUIRES: $1"); - while (<IN>) + open(my $requires, '<', "scripts/$thistestdir/REQUIRES") || + tests_exit(-1, "Failed to open scripts/$thistestdir/REQUIRES: $!"); + while (<$requires>) { $gnutls = 1 if /^support GnuTLS/; print $indent, $_; $indent = ">>> "; } - close(IN); } + $lasttestdir = $thistestdir; } - $lasttestdir = $thistestdir; # Remove any debris in the spool directory and the test-mail directory # and also the files for collecting stdout and stderr. Then put back @@ -3853,8 +4280,6 @@ foreach $test (@test_list) my($rc, $run_extra) = run_command($testno, \$subtestno, \$expectrc, \$commandname, $TEST_STATE); my($cmdrc) = $?; - $0 = "[runtest $testno]"; - if ($debug) { print ">> rc=$rc cmdrc=$cmdrc\n"; if (defined $run_extra) { @@ -3913,21 +4338,37 @@ foreach $test (@test_list) } if ($force_continue) { - print "\nstderr tail:\n"; + print "\nstdout tail:\n"; + print "==================>\n"; + system("tail -20 test-stdout"); + print "===================\n"; + + print "stderr tail:\n"; + print "==================>\n"; + system("tail -30 test-stderr"); + print "===================\n"; + + print "stdout-server tail:\n"; + print "==================>\n"; + system("tail -20 test-stdout-server"); print "===================\n"; - system("tail -20 test-stderr"); + + print "stderr-server tail:\n"; + print "==================>\n"; + system("tail -30 test-stderr-server"); print "===================\n"; + print "... continue forced\n"; } last if /^[rc]$/i; if (/^e$/i) { - system("$more test-stderr"); + system @more => 'test-stderr'; } elsif (/^o$/i) { - system("$more test-stdout"); + system @more => 'test-stdout'; } } @@ -3994,6 +4435,7 @@ foreach $test (@test_list) if ($docheck) { + sleep 1 if $slow; my $rc = check_output($TEST_STATE->{munge}); log_test($log_summary_filename, $testno, 'P') if ($rc == 0); if ($rc < 2) @@ -4013,7 +4455,84 @@ foreach $test (@test_list) # Exit from the test script # ################################################## -tests_exit(-1, "No runnable tests selected") if @test_list == 0; +tests_exit(-1, "No runnable tests selected") if not @test_list; tests_exit(0); +__END__ + +=head1 NAME + + runtest - run the exim testsuite + +=head1 SYNOPSIS + + runtest [exim-path] [options] [test0 [test1]] + +=head1 DESCRIPTION + +B<runtest> runs the Exim testsuite. + +=head1 OPTIONS + +For legacy reasons the options are not case sensitive. + +=over + +=item B<--continue> + +Do not stop for user interaction or on errors. (default: off) + +=item B<--debug> + +This option enables the output of debug information when running the +various test commands. (default: off) + +=item B<--diff> + +Use C<diff -u> for comparing the expected output with the produced +output. (default: use a built-in routine) + +=item B<--flavor>|B<--flavour> I<flavour> + +Override the expected results for results for a specific (OS) flavour. +(default: unused) + +=item B<--[no]ipv4> + +Skip IPv4 related setup and tests (default: use ipv4) + +=item B<--[no]ipv6> + +Skip IPv6 related setup and tests (default: use ipv6) + +=item B<--keep> + +Keep the various output files produced during a test run. (default: don't keep) + +=item B<--range> I<n0> I<n1> + +Run tests between (including) I<n0> and I<n1>. A "+" may be used to specify the "last +test available". + +=item B<--slow> + +Insert some delays to compensate for a slow host system. (default: off) + +=item B<--test> I<n> + +Run the specified test. This option may used multiple times. + +=item B<--update> + +Automatically update the recorded (expected) data on mismatch. (default: off) + +=item B<--valgrind> + +Start Exim wrapped by I<valgrind>. (default: don't use valgrind) + +=back + +=cut + + # End of runtest script