Enforce STARTTLS sync point, client side
[exim.git] / src / src / transports / smtp.c
index 7d60d2e14ff87079a47a8d3cd559063d94e60866..341acde2d6ab4275fbd997bb8b2ca3d14522be37 100644 (file)
@@ -3,6 +3,7 @@
 *************************************************/
 
 /* Copyright (c) University of Cambridge 1995 - 2018 */
+/* Copyright (c) The Exim Maintainers 2020 */
 /* See the file NOTICE for conditions of use and distribution. */
 
 #include "../exim.h"
@@ -18,195 +19,127 @@ over TCP/IP. The options must be in alphabetic order (note that "_" comes
 before the lower case letters). Some live in the transport_instance block so as
 to be publicly visible; these are flagged with opt_public. */
 
+#define LOFF(field) OPT_OFF(smtp_transport_options_block, field)
+
 optionlist smtp_transport_options[] = {
   { "*expand_multi_domain",             opt_stringptr | opt_hidden | opt_public,
-      (void *)offsetof(transport_instance, expand_multi_domain) },
+      OPT_OFF(transport_instance, expand_multi_domain) },
   { "*expand_retry_include_ip_address", opt_stringptr | opt_hidden,
-       (void *)(offsetof(smtp_transport_options_block, expand_retry_include_ip_address)) },
+      LOFF(expand_retry_include_ip_address) },
 
   { "address_retry_include_sender", opt_bool,
-      (void *)offsetof(smtp_transport_options_block, address_retry_include_sender) },
-  { "allow_localhost",      opt_bool,
-      (void *)offsetof(smtp_transport_options_block, allow_localhost) },
+      LOFF(address_retry_include_sender) },
+  { "allow_localhost",      opt_bool,     LOFF(allow_localhost) },
 #ifdef EXPERIMENTAL_ARC
-  { "arc_sign", opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, arc_sign) },
-#endif
-  { "authenticated_sender", opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, authenticated_sender) },
-  { "authenticated_sender_force", opt_bool,
-      (void *)offsetof(smtp_transport_options_block, authenticated_sender_force) },
-  { "command_timeout",      opt_time,
-      (void *)offsetof(smtp_transport_options_block, command_timeout) },
-  { "connect_timeout",      opt_time,
-      (void *)offsetof(smtp_transport_options_block, connect_timeout) },
+  { "arc_sign", opt_stringptr,            LOFF(arc_sign) },
+#endif
+  { "authenticated_sender", opt_stringptr, LOFF(authenticated_sender) },
+  { "authenticated_sender_force", opt_bool, LOFF(authenticated_sender_force) },
+  { "command_timeout",      opt_time,     LOFF(command_timeout) },
+  { "connect_timeout",      opt_time,     LOFF(connect_timeout) },
   { "connection_max_messages", opt_int | opt_public,
-      (void *)offsetof(transport_instance, connection_max_messages) },
+      OPT_OFF(transport_instance, connection_max_messages) },
 # ifdef SUPPORT_DANE
-  { "dane_require_tls_ciphers", opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, dane_require_tls_ciphers) },
+  { "dane_require_tls_ciphers", opt_stringptr, LOFF(dane_require_tls_ciphers) },
 # endif
-  { "data_timeout",         opt_time,
-      (void *)offsetof(smtp_transport_options_block, data_timeout) },
-  { "delay_after_cutoff", opt_bool,
-      (void *)offsetof(smtp_transport_options_block, delay_after_cutoff) },
+  { "data_timeout",         opt_time,     LOFF(data_timeout) },
+  { "delay_after_cutoff", opt_bool,       LOFF(delay_after_cutoff) },
 #ifndef DISABLE_DKIM
-  { "dkim_canon", opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, dkim.dkim_canon) },
-  { "dkim_domain", opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, dkim.dkim_domain) },
-  { "dkim_hash", opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, dkim.dkim_hash) },
-  { "dkim_identity", opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, dkim.dkim_identity) },
-  { "dkim_private_key", opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, dkim.dkim_private_key) },
-  { "dkim_selector", opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, dkim.dkim_selector) },
-  { "dkim_sign_headers", opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, dkim.dkim_sign_headers) },
-  { "dkim_strict", opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, dkim.dkim_strict) },
-  { "dkim_timestamps", opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, dkim.dkim_timestamps) },
-#endif
-  { "dns_qualify_single",   opt_bool,
-      (void *)offsetof(smtp_transport_options_block, dns_qualify_single) },
-  { "dns_search_parents",   opt_bool,
-      (void *)offsetof(smtp_transport_options_block, dns_search_parents) },
-  { "dnssec_request_domains", opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, dnssec.request) },
-  { "dnssec_require_domains", opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, dnssec.require) },
-  { "dscp",                 opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, dscp) },
-  { "fallback_hosts",       opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, fallback_hosts) },
-  { "final_timeout",        opt_time,
-      (void *)offsetof(smtp_transport_options_block, final_timeout) },
-  { "gethostbyname",        opt_bool,
-      (void *)offsetof(smtp_transport_options_block, gethostbyname) },
-  { "helo_data",            opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, helo_data) },
-  { "hosts",                opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, hosts) },
-  { "hosts_avoid_esmtp",    opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, hosts_avoid_esmtp) },
-  { "hosts_avoid_pipelining", opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, hosts_avoid_pipelining) },
+  { "dkim_canon", opt_stringptr,          LOFF(dkim.dkim_canon) },
+  { "dkim_domain", opt_stringptr,         LOFF(dkim.dkim_domain) },
+  { "dkim_hash", opt_stringptr,                   LOFF(dkim.dkim_hash) },
+  { "dkim_identity", opt_stringptr,       LOFF(dkim.dkim_identity) },
+  { "dkim_private_key", opt_stringptr,    LOFF(dkim.dkim_private_key) },
+  { "dkim_selector", opt_stringptr,       LOFF(dkim.dkim_selector) },
+  { "dkim_sign_headers", opt_stringptr,           LOFF(dkim.dkim_sign_headers) },
+  { "dkim_strict", opt_stringptr,         LOFF(dkim.dkim_strict) },
+  { "dkim_timestamps", opt_stringptr,     LOFF(dkim.dkim_timestamps) },
+#endif
+  { "dns_qualify_single",   opt_bool,     LOFF(dns_qualify_single) },
+  { "dns_search_parents",   opt_bool,     LOFF(dns_search_parents) },
+  { "dnssec_request_domains", opt_stringptr, LOFF(dnssec.request) },
+  { "dnssec_require_domains", opt_stringptr, LOFF(dnssec.require) },
+  { "dscp",                 opt_stringptr, LOFF(dscp) },
+  { "fallback_hosts",       opt_stringptr, LOFF(fallback_hosts) },
+  { "final_timeout",        opt_time,     LOFF(final_timeout) },
+  { "gethostbyname",        opt_bool,     LOFF(gethostbyname) },
+  { "helo_data",            opt_stringptr, LOFF(helo_data) },
+  { "hosts",                opt_stringptr, LOFF(hosts) },
+  { "hosts_avoid_esmtp",    opt_stringptr, LOFF(hosts_avoid_esmtp) },
+  { "hosts_avoid_pipelining", opt_stringptr, LOFF(hosts_avoid_pipelining) },
 #ifndef DISABLE_TLS
-  { "hosts_avoid_tls",      opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, hosts_avoid_tls) },
+  { "hosts_avoid_tls",      opt_stringptr, LOFF(hosts_avoid_tls) },
 #endif
-  { "hosts_max_try",        opt_int,
-      (void *)offsetof(smtp_transport_options_block, hosts_max_try) },
-  { "hosts_max_try_hardlimit", opt_int,
-      (void *)offsetof(smtp_transport_options_block, hosts_max_try_hardlimit) },
+  { "hosts_max_try",        opt_int,      LOFF(hosts_max_try) },
+  { "hosts_max_try_hardlimit", opt_int,           LOFF(hosts_max_try_hardlimit) },
 #ifndef DISABLE_TLS
-  { "hosts_nopass_tls",     opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, hosts_nopass_tls) },
-  { "hosts_noproxy_tls",    opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, hosts_noproxy_tls) },
+  { "hosts_nopass_tls",     opt_stringptr, LOFF(hosts_nopass_tls) },
+  { "hosts_noproxy_tls",    opt_stringptr, LOFF(hosts_noproxy_tls) },
 #endif
-  { "hosts_override",       opt_bool,
-      (void *)offsetof(smtp_transport_options_block, hosts_override) },
+  { "hosts_override",       opt_bool,     LOFF(hosts_override) },
 #ifndef DISABLE_PIPE_CONNECT
-  { "hosts_pipe_connect",   opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, hosts_pipe_connect) },
+  { "hosts_pipe_connect",   opt_stringptr, LOFF(hosts_pipe_connect) },
 #endif
-  { "hosts_randomize",      opt_bool,
-      (void *)offsetof(smtp_transport_options_block, hosts_randomize) },
+  { "hosts_randomize",      opt_bool,     LOFF(hosts_randomize) },
 #if !defined(DISABLE_TLS) && !defined(DISABLE_OCSP)
-  { "hosts_request_ocsp",   opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, hosts_request_ocsp) },
+  { "hosts_request_ocsp",   opt_stringptr, LOFF(hosts_request_ocsp) },
 #endif
-  { "hosts_require_auth",   opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, hosts_require_auth) },
+  { "hosts_require_auth",   opt_stringptr, LOFF(hosts_require_auth) },
 #ifndef DISABLE_TLS
 # ifdef SUPPORT_DANE
-  { "hosts_require_dane",   opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, hosts_require_dane) },
+  { "hosts_require_dane",   opt_stringptr, LOFF(hosts_require_dane) },
 # endif
 # ifndef DISABLE_OCSP
-  { "hosts_require_ocsp",   opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, hosts_require_ocsp) },
+  { "hosts_require_ocsp",   opt_stringptr, LOFF(hosts_require_ocsp) },
 # endif
-  { "hosts_require_tls",    opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, hosts_require_tls) },
+  { "hosts_require_tls",    opt_stringptr, LOFF(hosts_require_tls) },
 #endif
-  { "hosts_try_auth",       opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, hosts_try_auth) },
-  { "hosts_try_chunking",   opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, hosts_try_chunking) },
+  { "hosts_try_auth",       opt_stringptr, LOFF(hosts_try_auth) },
+  { "hosts_try_chunking",   opt_stringptr, LOFF(hosts_try_chunking) },
 #ifdef SUPPORT_DANE
-  { "hosts_try_dane",       opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, hosts_try_dane) },
+  { "hosts_try_dane",       opt_stringptr, LOFF(hosts_try_dane) },
 #endif
-  { "hosts_try_fastopen",   opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, hosts_try_fastopen) },
+  { "hosts_try_fastopen",   opt_stringptr, LOFF(hosts_try_fastopen) },
 #ifndef DISABLE_PRDR
-  { "hosts_try_prdr",       opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, hosts_try_prdr) },
+  { "hosts_try_prdr",       opt_stringptr, LOFF(hosts_try_prdr) },
 #endif
 #ifndef DISABLE_TLS
-  { "hosts_verify_avoid_tls", opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, hosts_verify_avoid_tls) },
-#endif
-  { "interface",            opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, interface) },
-  { "keepalive",            opt_bool,
-      (void *)offsetof(smtp_transport_options_block, keepalive) },
-  { "lmtp_ignore_quota",    opt_bool,
-      (void *)offsetof(smtp_transport_options_block, lmtp_ignore_quota) },
+  { "hosts_verify_avoid_tls", opt_stringptr, LOFF(hosts_verify_avoid_tls) },
+#endif
+  { "interface",            opt_stringptr, LOFF(interface) },
+  { "keepalive",            opt_bool,     LOFF(keepalive) },
+  { "lmtp_ignore_quota",    opt_bool,     LOFF(lmtp_ignore_quota) },
   { "max_rcpt",             opt_int | opt_public,
-      (void *)offsetof(transport_instance, max_addresses) },
+      OPT_OFF(transport_instance, max_addresses) },
+  { "message_linelength_limit", opt_int,   LOFF(message_linelength_limit) },
   { "multi_domain",         opt_expand_bool | opt_public,
-      (void *)offsetof(transport_instance, multi_domain) },
-  { "port",                 opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, port) },
-  { "protocol",             opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, protocol) },
-  { "retry_include_ip_address", opt_expand_bool,
-      (void *)offsetof(smtp_transport_options_block, retry_include_ip_address) },
-  { "serialize_hosts",      opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, serialize_hosts) },
-  { "size_addition",        opt_int,
-      (void *)offsetof(smtp_transport_options_block, size_addition) },
+      OPT_OFF(transport_instance, multi_domain) },
+  { "port",                 opt_stringptr, LOFF(port) },
+  { "protocol",             opt_stringptr, LOFF(protocol) },
+  { "retry_include_ip_address", opt_expand_bool, LOFF(retry_include_ip_address) },
+  { "serialize_hosts",      opt_stringptr, LOFF(serialize_hosts) },
+  { "size_addition",        opt_int,      LOFF(size_addition) },
 #ifdef SUPPORT_SOCKS
-  { "socks_proxy",          opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, socks_proxy) },
+  { "socks_proxy",          opt_stringptr, LOFF(socks_proxy) },
 #endif
 #ifndef DISABLE_TLS
-  { "tls_certificate",      opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, tls_certificate) },
-  { "tls_crl",              opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, tls_crl) },
-  { "tls_dh_min_bits",      opt_int,
-      (void *)offsetof(smtp_transport_options_block, tls_dh_min_bits) },
-  { "tls_privatekey",       opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, tls_privatekey) },
-  { "tls_require_ciphers",  opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, tls_require_ciphers) },
-# ifdef EXPERIMENTAL_TLS_RESUME
-  { "tls_resumption_hosts", opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, tls_resumption_hosts) },
+  { "tls_certificate",      opt_stringptr, LOFF(tls_certificate) },
+  { "tls_crl",              opt_stringptr, LOFF(tls_crl) },
+  { "tls_dh_min_bits",      opt_int,      LOFF(tls_dh_min_bits) },
+  { "tls_privatekey",       opt_stringptr, LOFF(tls_privatekey) },
+  { "tls_require_ciphers",  opt_stringptr, LOFF(tls_require_ciphers) },
+# ifndef DISABLE_TLS_RESUME
+  { "tls_resumption_hosts", opt_stringptr, LOFF(tls_resumption_hosts) },
 # endif
-  { "tls_sni",              opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, tls_sni) },
-  { "tls_tempfail_tryclear", opt_bool,
-      (void *)offsetof(smtp_transport_options_block, tls_tempfail_tryclear) },
-  { "tls_try_verify_hosts", opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, tls_try_verify_hosts) },
-  { "tls_verify_cert_hostnames", opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block,tls_verify_cert_hostnames)},
-  { "tls_verify_certificates", opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, tls_verify_certificates) },
-  { "tls_verify_hosts",     opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, tls_verify_hosts) },
+  { "tls_sni",              opt_stringptr, LOFF(tls_sni) },
+  { "tls_tempfail_tryclear", opt_bool, LOFF(tls_tempfail_tryclear) },
+  { "tls_try_verify_hosts", opt_stringptr, LOFF(tls_try_verify_hosts) },
+  { "tls_verify_cert_hostnames", opt_stringptr, LOFF(tls_verify_cert_hostnames)},
+  { "tls_verify_certificates", opt_stringptr, LOFF(tls_verify_certificates) },
+  { "tls_verify_hosts",     opt_stringptr, LOFF(tls_verify_hosts) },
 #endif
 #ifdef SUPPORT_I18N
-  { "utf8_downconvert",            opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, utf8_downconvert) },
+  { "utf8_downconvert",            opt_stringptr, LOFF(utf8_downconvert) },
 #endif
 };
 
@@ -275,6 +208,7 @@ smtp_transport_options_block smtp_transport_option_defaults = {
   .size_addition =             1024,
   .hosts_max_try =             5,
   .hosts_max_try_hardlimit =   50,
+  .message_linelength_limit =  998,
   .address_retry_include_sender = TRUE,
   .allow_localhost =           FALSE,
   .authenticated_sender_force =        FALSE,
@@ -301,7 +235,7 @@ smtp_transport_options_block smtp_transport_option_defaults = {
   .tls_verify_certificates =   US"system",
   .tls_dh_min_bits =           EXIM_CLIENT_DH_DEFAULT_MIN_BITS,
   .tls_tempfail_tryclear =     TRUE,
-# ifdef EXPERIMENTAL_TLS_RESUME
+# ifndef DISABLE_TLS_RESUME
   .tls_resumption_hosts =      NULL,
 # endif
   .tls_verify_hosts =          NULL,
@@ -309,7 +243,7 @@ smtp_transport_options_block smtp_transport_option_defaults = {
   .tls_verify_cert_hostnames = US"*",
 #endif
 #ifdef SUPPORT_I18N
-  .utf8_downconvert =          NULL,
+  .utf8_downconvert =          US"-1",
 #endif
 #ifndef DISABLE_DKIM
  .dkim =
@@ -430,10 +364,6 @@ smtp_transport_setup(transport_instance *tblock, address_item *addrlist,
 {
 smtp_transport_options_block *ob = SOB tblock->options_block;
 
-errmsg = errmsg;    /* Keep picky compilers happy */
-uid = uid;
-gid = gid;
-
 /* Pass back options if required. This interface is getting very messy. */
 
 if (tf)
@@ -537,6 +467,7 @@ Arguments:
   host           if set, mark addrs as having used this host
   smtp_greeting  from peer
   helo_response  from peer
+  start                 points to timestamp of delivery start
 
 If errno_value has the special value ERRNO_CONNECTTIMEOUT, ETIMEDOUT is put in
 the errno field, and RTEF_CTOUT is ORed into the more_errno field, to indicate
@@ -547,23 +478,29 @@ Returns:       nothing
 
 static void
 set_errno(address_item *addrlist, int errno_value, uschar *msg, int rc,
-  BOOL pass_message, host_item * host
+  BOOL pass_message, host_item * host,
 #ifdef EXPERIMENTAL_DSN_INFO
-  , const uschar * smtp_greeting, const uschar * helo_response
+  const uschar * smtp_greeting, const uschar * helo_response,
 #endif
+  struct timeval * start
   )
 {
 int orvalue = 0;
+struct timeval deliver_time;
+
 if (errno_value == ERRNO_CONNECTTIMEOUT)
   {
   errno_value = ETIMEDOUT;
   orvalue = RTEF_CTOUT;
   }
+timesince(&deliver_time, start);
+
 for (address_item * addr = addrlist; addr; addr = addr->next)
   if (addr->transport_return >= PENDING)
     {
     addr->basic_errno = errno_value;
     addr->more_errno |= orvalue;
+    addr->delivery_time = deliver_time;
     if (msg)
       {
       addr->message = msg;
@@ -587,14 +524,14 @@ for (address_item * addr = addrlist; addr; addr = addr->next)
 }
 
 static void
-set_errno_nohost(address_item *addrlist, int errno_value, uschar *msg, int rc,
-  BOOL pass_message)
+set_errno_nohost(address_item * addrlist, int errno_value, uschar * msg, int rc,
+  BOOL pass_message, struct timeval * start)
 {
-set_errno(addrlist, errno_value, msg, rc, pass_message, NULL
+set_errno(addrlist, errno_value, msg, rc, pass_message, NULL,
 #ifdef EXPERIMENTAL_DSN_INFO
-         , NULL, NULL
+         NULL, NULL,
 #endif
-         );
+         start);
 }
 
 
@@ -766,12 +703,13 @@ deliver_msglog("%s H=%s [%s] %s\n", tod_stamp(tod_log),
 Arguments:
   addr                  the address item containing error information
   host                  the current host
+  evstr                        the event
 
 Returns:   nothing
 */
 
 static void
-deferred_event_raise(address_item *addr, host_item *host)
+deferred_event_raise(address_item * addr, host_item * host, uschar * evstr)
 {
 uschar * action = addr->transport->event_action;
 const uschar * save_domain;
@@ -793,7 +731,7 @@ transport_name = addr->transport->name;
 deliver_domain = addr->domain;
 deliver_localpart = addr->local_part;
 
-(void) event_raise(action, US"msg:host:defer",
+(void) event_raise(action, evstr,
     addr->message
       ? addr->basic_errno > 0
        ? string_sprintf("%s: %s", addr->message, strerror(addr->basic_errno))
@@ -946,7 +884,6 @@ else
     return TRUE;
     }
   dbfn_close(dbm_file);
-  memset(&sx->ehlo_resp, 0, sizeof(ehlo_resp_precis));
   }
 return FALSE;
 }
@@ -973,11 +910,9 @@ names = string_copyn(expand_nstring[1], expand_nlength[1]);
 for (au = auths, authnum = 0; au; au = au->next, authnum++) if (au->client)
   {
   const uschar * list = names;
-  int sep = ' ';
-  uschar name[32];
-
-  while (string_nextinlist(&list, &sep, name, sizeof(name)))
-    if (strcmpic(au->public_name, name) == 0)
+  uschar * s;
+  for (int sep = ' '; s = string_nextinlist(&list, &sep, NULL, 0); )
+    if (strcmpic(au->public_name, s) == 0)
       { authbits |= BIT(authnum); break; }
   }
 
@@ -1150,6 +1085,7 @@ if (sx->pending_MAIL)
   {
   DEBUG(D_transport) debug_printf("%s expect mail\n", __FUNCTION__);
   count--;
+  sx->pending_MAIL = FALSE;
   if (!smtp_read_response(sx, sx->buffer, sizeof(sx->buffer),
                          '2', ob->command_timeout))
     {
@@ -1226,7 +1162,7 @@ while (count-- > 0)
     {
     uschar *message = string_sprintf("SMTP timeout after RCPT TO:<%s>",
                transport_rcpt_address(addr, sx->conn_args.tblock->rcpt_include_affixes));
-    set_errno_nohost(sx->first_addr, ETIMEDOUT, message, DEFER, FALSE);
+    set_errno_nohost(sx->first_addr, ETIMEDOUT, message, DEFER, FALSE, &sx->delivery_start);
     retry_add_item(addr, addr->address_retry_key, 0);
     update_waiting = FALSE;
     return -1;
@@ -1284,38 +1220,55 @@ while (count-- > 0)
        event_defer_errno = addr->more_errno;
        msg_event_raise(US"msg:rcpt:host:defer", addr);
 #endif
+       /* If a 452 and we've had at least one 2xx or 5xx, set next_addr to the
+       start point for another MAIL command. */
 
-       /* Log temporary errors if there are more hosts to be tried.
-       If not, log this last one in the == line. */
-
-       if (sx->conn_args.host->next)
-         if (LOGGING(outgoing_port))
-           log_write(0, LOG_MAIN, "H=%s [%s]:%d %s", sx->conn_args.host->name,
-             sx->conn_args.host->address,
-             sx->port == PORT_NONE ? 25 : sx->port, addr->message);
-         else
-           log_write(0, LOG_MAIN, "H=%s [%s]: %s", sx->conn_args.host->name,
-             sx->conn_args.host->address, addr->message);
+       if (addr->more_errno >> 8 == 52  &&  yield & 3)
+         {
+         if (!sx->RCPT_452)
+           {
+           DEBUG(D_transport)
+             debug_printf("%s: seen first 452 too-many-rcpts\n", __FUNCTION__);
+           sx->RCPT_452 = TRUE;
+           sx->next_addr = addr;
+           }
+         addr->transport_return = PENDING_DEFER;
+         addr->basic_errno = 0;
+         }
+       else
+         {
+         /* Log temporary errors if there are more hosts to be tried.
+         If not, log this last one in the == line. */
+
+         if (sx->conn_args.host->next)
+           if (LOGGING(outgoing_port))
+             log_write(0, LOG_MAIN, "H=%s [%s]:%d %s", sx->conn_args.host->name,
+               sx->conn_args.host->address,
+               sx->port == PORT_NONE ? 25 : sx->port, addr->message);
+           else
+             log_write(0, LOG_MAIN, "H=%s [%s]: %s", sx->conn_args.host->name,
+               sx->conn_args.host->address, addr->message);
 
 #ifndef DISABLE_EVENT
-       else
-         msg_event_raise(US"msg:rcpt:defer", addr);
+         else
+           msg_event_raise(US"msg:rcpt:defer", addr);
 #endif
 
-       /* Do not put this message on the list of those waiting for specific
-       hosts, as otherwise it is likely to be tried too often. */
+         /* Do not put this message on the list of those waiting for specific
+         hosts, as otherwise it is likely to be tried too often. */
 
-       update_waiting = FALSE;
+         update_waiting = FALSE;
 
-       /* Add a retry item for the address so that it doesn't get tried again
-       too soon. If address_retry_include_sender is true, add the sender address
-       to the retry key. */
+         /* Add a retry item for the address so that it doesn't get tried again
+         too soon. If address_retry_include_sender is true, add the sender address
+         to the retry key. */
 
-       retry_add_item(addr,
-         ob->address_retry_include_sender
-           ? string_sprintf("%s:<%s>", addr->address_retry_key, sender_address)
-           : addr->address_retry_key,
-         0);
+         retry_add_item(addr,
+           ob->address_retry_include_sender
+             ? string_sprintf("%s:<%s>", addr->address_retry_key, sender_address)
+             : addr->address_retry_key,
+           0);
+         }
        }
       }
     }
@@ -1427,7 +1380,7 @@ switch(rc)
 
   case ERROR:
     set_errno_nohost(sx->addrlist, ERRNO_AUTHPROB, string_copy(sx->buffer),
-             DEFER, FALSE);
+             DEFER, FALSE, &sx->delivery_start);
     return ERROR;
   }
 return OK;
@@ -1594,9 +1547,10 @@ if (  sx->esmtp
 
 if (require_auth == OK && !f.smtp_authenticated)
   {
+  invalidate_ehlo_cache_entry(sx);
   set_errno_nohost(sx->addrlist, ERRNO_AUTHFAIL,
     string_sprintf("authentication required but %s", fail_reason), DEFER,
-    FALSE);
+    FALSE, &sx->delivery_start);
   return DEFER;
   }
 
@@ -1607,9 +1561,9 @@ return OK;
 /* Construct AUTH appendix string for MAIL TO */
 /*
 Arguments
-  buffer       to build string
+  sx           context for smtp connection
+  p            point in sx->buffer to build string
   addrlist      chain of potential addresses to deliver
-  ob           transport options
 
 Globals                f.smtp_authenticated
                client_authenticated_sender
@@ -1617,29 +1571,31 @@ Return  True on error, otherwise buffer has (possibly empty) terminated string
 */
 
 static BOOL
-smtp_mail_auth_str(uschar *buffer, unsigned bufsize, address_item *addrlist,
-                   smtp_transport_options_block *ob)
+smtp_mail_auth_str(smtp_context * sx, uschar * p, address_item * addrlist)
 {
+smtp_transport_options_block * ob = sx->conn_args.ob;
 uschar * local_authenticated_sender = authenticated_sender;
 
 #ifdef notdef
-  debug_printf("smtp_mail_auth_str: as<%s> os<%s> SA<%s>\n", authenticated_sender, ob->authenticated_sender, f.smtp_authenticated?"Y":"N");
+  debug_printf("smtp_mail_auth_str: as<%s> os<%s> SA<%s>\n",
+    authenticated_sender, ob->authenticated_sender, f.smtp_authenticated?"Y":"N");
 #endif
 
 if (ob->authenticated_sender)
   {
-  uschar *new = expand_string(ob->authenticated_sender);
+  uschar * new = expand_string(ob->authenticated_sender);
   if (!new)
     {
     if (!f.expand_string_forcedfail)
       {
       uschar *message = string_sprintf("failed to expand "
         "authenticated_sender: %s", expand_string_message);
-      set_errno_nohost(addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE);
+      set_errno_nohost(addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE, &sx->delivery_start);
       return TRUE;
       }
     }
-  else if (*new) local_authenticated_sender = new;
+  else if (*new)
+    local_authenticated_sender = new;
   }
 
 /* Add the authenticated sender address if present */
@@ -1647,13 +1603,13 @@ if (ob->authenticated_sender)
 if (  (f.smtp_authenticated || ob->authenticated_sender_force)
    && local_authenticated_sender)
   {
-  string_format_nt(buffer, bufsize, " AUTH=%s",
+  string_format_nt(p, sizeof(sx->buffer) - (p-sx->buffer), " AUTH=%s",
     auth_xtextencode(local_authenticated_sender,
       Ustrlen(local_authenticated_sender)));
   client_authenticated_sender = string_copy(local_authenticated_sender);
   }
 else
-  *buffer= 0;
+  *= 0;
 
 return FALSE;
 }
@@ -1727,7 +1683,7 @@ current_local_identity =
   smtp_local_identity(s_compare->current_sender_address, s_compare->tblock);
 
 if (!(new_sender_address = deliver_get_sender_address(message_id)))
-    return 0;
+    return FALSE;
 
 message_local_identity =
   smtp_local_identity(new_sender_address, s_compare->tblock);
@@ -1951,39 +1907,39 @@ sx->conn_args.ob = ob;
 
 sx->lmtp = strcmpic(ob->protocol, US"lmtp") == 0;
 sx->smtps = strcmpic(ob->protocol, US"smtps") == 0;
-sx->ok = FALSE;
+/* sx->ok = FALSE; */
 sx->send_rset = TRUE;
 sx->send_quit = TRUE;
 sx->setting_up = TRUE;
 sx->esmtp = TRUE;
-sx->esmtp_sent = FALSE;
+/* sx->esmtp_sent = FALSE; */
 #ifdef SUPPORT_I18N
-sx->utf8_needed = FALSE;
+/* sx->utf8_needed = FALSE; */
 #endif
 sx->dsn_all_lasthop = TRUE;
 #ifdef SUPPORT_DANE
-sx->conn_args.dane = FALSE;
+/* sx->conn_args.dane = FALSE; */
 sx->dane_required =
   verify_check_given_host(CUSS &ob->hosts_require_dane, sx->conn_args.host) == OK;
 #endif
 #ifndef DISABLE_PIPE_CONNECT
-sx->early_pipe_active = sx->early_pipe_ok = FALSE;
-sx->ehlo_resp.cleartext_features = sx->ehlo_resp.crypted_features = 0;
-sx->pending_BANNER = sx->pending_EHLO = FALSE;
+/* sx->early_pipe_active = sx->early_pipe_ok = FALSE; */
+/* sx->ehlo_resp.cleartext_features = sx->ehlo_resp.crypted_features = 0; */
+/* sx->pending_BANNER = sx->pending_EHLO = sx->pending_MAIL = FALSE; */
 #endif
 
 if ((sx->max_rcpt = sx->conn_args.tblock->max_addresses) == 0) sx->max_rcpt = 999999;
-sx->peer_offered = 0;
-sx->avoid_option = 0;
+/* sx->peer_offered = 0; */
+/* sx->avoid_option = 0; */
 sx->igquotstr = US"";
 if (!sx->helo_data) sx->helo_data = ob->helo_data;
 #ifdef EXPERIMENTAL_DSN_INFO
-sx->smtp_greeting = NULL;
-sx->helo_response = NULL;
+/* sx->smtp_greeting = NULL; */
+/* sx->helo_response = NULL; */
 #endif
 
 smtp_command = US"initial connection";
-sx->buffer[0] = '\0';
+/* sx->buffer[0] = '\0'; */
 
 /* Set up the buffer for reading SMTP response packets. */
 
@@ -1997,9 +1953,9 @@ sx->inblock.ptrend = sx->inbuffer;
 sx->outblock.buffer = sx->outbuffer;
 sx->outblock.buffersize = sizeof(sx->outbuffer);
 sx->outblock.ptr = sx->outbuffer;
-sx->outblock.cmd_count = 0;
-sx->outblock.authenticating = FALSE;
-sx->outblock.conn_args = NULL;
+/* sx->outblock.cmd_count = 0; */
+/* sx->outblock.authenticating = FALSE; */
+/* sx->outblock.conn_args = NULL; */
 
 /* Reset the parameters of a TLS session. */
 
@@ -2012,7 +1968,7 @@ tls_out.peerdn = NULL;
 tls_out.sni = NULL;
 #endif
 tls_out.ocsp = OCSP_NOT_REQ;
-#ifdef EXPERIMENTAL_TLS_RESUME
+#ifndef DISABLE_TLS_RESUME
 tls_out.resumption = 0;
 #endif
 tls_out.ver = NULL;
@@ -2028,7 +1984,7 @@ tls_modify_variables(&tls_out);
 if (sx->smtps)
   {
   set_errno_nohost(sx->addrlist, ERRNO_TLSFAILURE, US"TLS support not available",
-           DEFER, FALSE);
+           DEFER, FALSE, &sx->delivery_start);
   return ERROR;
   }
 #endif
@@ -2067,7 +2023,7 @@ if (!continue_hostname)
          default:              set_errno_nohost(sx->addrlist, ERRNO_DNSDEFER,
                                  string_sprintf("DANE error: tlsa lookup %s",
                                    rc_to_string(rc)),
-                                 rc, FALSE);
+                                 rc, FALSE, &sx->delivery_start);
 # ifndef DISABLE_EVENT
                                (void) event_raise(sx->conn_args.tblock->event_action,
                                  US"dane:fail", sx->dane_required
@@ -2080,7 +2036,7 @@ if (!continue_hostname)
       {
       set_errno_nohost(sx->addrlist, ERRNO_DNSDEFER,
        string_sprintf("DANE error: %s lookup not DNSSEC", sx->conn_args.host->name),
-       FAIL, FALSE);
+       FAIL, FALSE, &sx->delivery_start);
 # ifndef DISABLE_EVENT
       (void) event_raise(sx->conn_args.tblock->event_action,
        US"dane:fail", US"dane-required");
@@ -2121,6 +2077,7 @@ if (!continue_hostname)
     else DEBUG(D_transport)
       debug_printf("helo needs $sending_ip_address\n");
 
+PIPE_CONNECT_RETRY:
   if (sx->early_pipe_active)
     sx->outblock.conn_args = &sx->conn_args;
   else
@@ -2131,7 +2088,7 @@ if (!continue_hostname)
       set_errno_nohost(sx->addrlist,
        errno == ETIMEDOUT ? ERRNO_CONNECTTIMEOUT : errno,
        sx->verify ? US strerror(errno) : NULL,
-       DEFER, FALSE);
+       DEFER, FALSE, &sx->delivery_start);
       sx->send_quit = FALSE;
       return DEFER;
       }
@@ -2198,7 +2155,7 @@ will be?  Somehow I doubt it. */
        {
        set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL,
          string_sprintf("deferred by smtp:connect event expansion: %s", s),
-         DEFER, FALSE);
+         DEFER, FALSE, &sx->delivery_start);
        yield = DEFER;
        goto SEND_QUIT;
        }
@@ -2212,7 +2169,7 @@ will be?  Somehow I doubt it. */
       {
       message = string_sprintf("failed to expand helo_data: %s",
         expand_string_message);
-      set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE);
+      set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE, &sx->delivery_start);
       yield = DEFER;
       goto SEND_QUIT;
       }
@@ -2419,6 +2376,7 @@ else
     }
   sx->inblock.cctx = sx->outblock.cctx = &sx->cctx;
   smtp_command = big_buffer;
+  sx->peer_offered = smtp_peer_options;
   sx->helo_data = NULL;                /* ensure we re-expand ob->helo_data */
 
   /* For a continued connection with TLS being proxied for us, or a
@@ -2429,7 +2387,6 @@ else
          && cutthrough.is_tls)
      )
     {
-    sx->peer_offered = smtp_peer_options;
     sx->pipelining_used = pipelining_active = !!(smtp_peer_options & OPTION_PIPE);
     HDEBUG(D_transport) debug_printf("continued connection, %s TLS\n",
       continue_proxy_cipher ? "proxied" : "verify conn with");
@@ -2467,7 +2424,10 @@ if (  smtp_peer_options & OPTION_TLS
     {
     HDEBUG(D_transport)
       debug_printf("failed reaping pipelined cmd responses\n");
-    goto RESPONSE_FAILED;
+    close(sx->cctx.sock);
+    sx->cctx.sock = -1;
+    sx->early_pipe_active = FALSE;
+    goto PIPE_CONNECT_RETRY;
     }
 #endif
 
@@ -2501,9 +2461,7 @@ if (  smtp_peer_options & OPTION_TLS
       /* TLS negotiation failed; give an error. From outside, this function may
       be called again to try in clear on a new connection, if the options permit
       it for this host. */
-#ifdef USE_GNUTLS
-  GNUTLS_CONN_FAILED:
-#endif
+  TLS_CONN_FAILED:
       DEBUG(D_tls) debug_printf("TLS session fail: %s\n", tls_errstr);
 
 # ifdef SUPPORT_DANE
@@ -2525,7 +2483,23 @@ if (  smtp_peer_options & OPTION_TLS
       goto TLS_FAILED;
       }
 
-    /* TLS session is set up */
+    /* TLS session is set up.  Check the inblock fill level.  If there is
+    content then as we have not yet done a tls read it must have arrived before
+    the TLS handshake, in-clear.  That violates the sync requirement of the
+    STARTTLS RFC, so fail. */
+
+    if (sx->inblock.ptr != sx->inblock.ptrend)
+      {
+      DEBUG(D_tls)
+       {
+       int i = sx->inblock.ptrend - sx->inblock.ptr;
+       debug_printf("unused data in input buffer after ack for STARTTLS:\n"
+         "'%.*s'%s\n",
+         i > 100 ? 100 : i, sx->inblock.ptr, i > 100 ? "..." : "");
+       }
+      tls_errstr = US"synch error before connect";
+      goto TLS_CONN_FAILED;
+      }
 
     smtp_peer_options_wrap = smtp_peer_options;
     for (address_item * addr = sx->addrlist; addr; addr = addr->next)
@@ -2559,7 +2533,7 @@ if (tls_out.active.sock >= 0)
     {
     uschar *message = string_sprintf("failed to expand helo_data: %s",
       expand_string_message);
-    set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE);
+    set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE, &sx->delivery_start);
     yield = DEFER;
     goto SEND_QUIT;
     }
@@ -2630,7 +2604,7 @@ if (tls_out.active.sock >= 0)
       Can it do that, with all the flexibility we need? */
 
       tls_errstr = US"error on first read";
-      goto GNUTLS_CONN_FAILED;
+      goto TLS_CONN_FAILED;
       }
 #else
       goto RESPONSE_FAILED;
@@ -2801,7 +2775,7 @@ if (sx->addrlist->prop.utf8_msg)
       {
       message = string_sprintf("failed to expand utf8_downconvert: %s",
         expand_string_message);
-      set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE);
+      set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE, &sx->delivery_start);
       yield = DEFER;
       goto SEND_QUIT;
       }
@@ -2861,7 +2835,7 @@ return OK;
       set_errno_nohost(sx->addrlist,
        errno == ETIMEDOUT ? ERRNO_CONNECTTIMEOUT : errno,
        sx->verify ? US strerror(errno) : NULL,
-       DEFER, FALSE);
+       DEFER, FALSE, &sx->delivery_start);
       sx->send_quit = FALSE;
       return DEFER;
       }
@@ -2925,11 +2899,11 @@ FAILED:
 #endif
            ? FAIL : DEFER,
            pass_message,
-           errno == ECONNREFUSED ? NULL : sx->conn_args.host
+           errno == ECONNREFUSED ? NULL : sx->conn_args.host,
 #ifdef EXPERIMENTAL_DSN_INFO
-           , sx->smtp_greeting, sx->helo_response
+           sx->smtp_greeting, sx->helo_response,
 #endif
-           );
+           &sx->delivery_start);
   }
 
 
@@ -3071,10 +3045,7 @@ Other expansion failures are serious. An empty result is ignored, but there is
 otherwise no check - this feature is expected to be used with LMTP and other
 cases where non-standard addresses (e.g. without domains) might be required. */
 
-if (smtp_mail_auth_str(p, sizeof(sx->buffer) - (p-sx->buffer), addrlist, sx->conn_args.ob))
-  return ERROR;
-
-return OK;
+return smtp_mail_auth_str(sx, p, addrlist) ? ERROR : OK;
 }
 
 
@@ -3128,7 +3099,7 @@ int
 smtp_write_mail_and_rcpt_cmds(smtp_context * sx, int * yield)
 {
 address_item * addr;
-int address_count;
+int address_count, pipe_limit;
 int rc;
 
 if (build_mailcmd_options(sx, sx->first_addr) != OK)
@@ -3160,7 +3131,7 @@ sx->pending_MAIL = TRUE;     /* The block starts with MAIL */
     {
     if (s = string_address_utf8_to_alabel(s, &errstr), errstr)
       {
-      set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, errstr, DEFER, FALSE);
+      set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, errstr, DEFER, FALSE, &sx->delivery_start);
       *yield = ERROR;
       return -4;
       }
@@ -3211,11 +3182,11 @@ that max_rcpt will be large, so all addresses will be done at once.
 
 For verify we flush the pipeline after any (the only) rcpt address. */
 
-for (addr = sx->first_addr, address_count = 0;
+for (addr = sx->first_addr, address_count = 0, pipe_limit = 100;
      addr  &&  address_count < sx->max_rcpt;
      addr = addr->next) if (addr->transport_return == PENDING_DEFER)
   {
-  int count;
+  int cmds_sent;
   BOOL no_flush;
   uschar * rcpt_addr;
 
@@ -3223,7 +3194,10 @@ for (addr = sx->first_addr, address_count = 0;
     ? dsn_support_yes : dsn_support_no;
 
   address_count++;
-  no_flush = pipelining_active && !sx->verify
+  if (pipe_limit-- <= 0)
+    { no_flush = FALSE; pipe_limit = 100; }
+  else
+    no_flush = pipelining_active && !sx->verify
          && (!mua_wrapper || addr->next && address_count < sx->max_rcpt);
 
   build_rcptcmd_options(sx, addr);
@@ -3246,13 +3220,13 @@ for (addr = sx->first_addr, address_count = 0;
     }
 #endif
 
-  count = smtp_write_command(sx, no_flush ? SCMD_BUFFER : SCMD_FLUSH,
+  cmds_sent = smtp_write_command(sx, no_flush ? SCMD_BUFFER : SCMD_FLUSH,
     "RCPT TO:<%s>%s%s\r\n", rcpt_addr, sx->igquotstr, sx->buffer);
 
-  if (count < 0) return -5;
-  if (count > 0)
+  if (cmds_sent < 0) return -5;
+  if (cmds_sent > 0)
     {
-    switch(sync_responses(sx, count, 0))
+    switch(sync_responses(sx, cmds_sent, 0))
       {
       case 3: sx->ok = TRUE;                   /* 2xx & 5xx => OK & progress made */
       case 2: sx->completed_addr = TRUE;       /* 5xx (only) => progress made */
@@ -3262,6 +3236,17 @@ for (addr = sx->first_addr, address_count = 0;
              if (!sx->lmtp)                    /*  can't tell about progress yet */
                sx->completed_addr = TRUE;
       case 0:                                  /* No 2xx or 5xx, but no probs */
+             /* If any RCPT got a 452 response then next_addr has been updated
+             for restarting with a new MAIL on the same connection.  Send no more
+             RCPTs for this MAIL. */
+
+             if (sx->RCPT_452)
+               {
+               DEBUG(D_transport) debug_printf("seen 452 too-many-rcpts\n");
+               sx->RCPT_452 = FALSE;
+               /* sx->next_addr has been reset for fast_retry */
+               return 0;
+               }
              break;
 
       case -1: return -3;                      /* Timeout on RCPT */
@@ -3273,7 +3258,6 @@ for (addr = sx->first_addr, address_count = 0;
       case -5: return -1;                      /* TLS first-read error */
 #endif
       }
-    sx->pending_MAIL = FALSE;            /* Dealt with MAIL */
     }
   }      /* Loop for next address */
 
@@ -3312,13 +3296,9 @@ int max_fd = MAX(pfd[0], tls_out.active.sock) + 1;
 int rc, i;
 
 close(pfd[1]);
-if ((rc = fork()))
-  {
-  DEBUG(D_transport) debug_printf("proxy-proc final-pid %d\n", rc);
+if ((rc = exim_fork(US"tls-proxy")))
   _exit(rc < 0 ? EXIT_FAILURE : EXIT_SUCCESS);
-  }
 
-testharness_pause_ms(100); /* let parent debug out */
 set_process_info("proxying TLS connection for continued transport");
 FD_ZERO(&rfds);
 FD_SET(tls_out.active.sock, &rfds);
@@ -3393,7 +3373,7 @@ for (int fd_bits = 3; fd_bits; )
 
 done:
   testharness_pause_ms(100);   /* let logging complete */
-  exim_exit(0, US"TLS proxy");
+  exim_exit(EXIT_SUCCESS);
 }
 #endif
 
@@ -3404,8 +3384,9 @@ done:
 
 /* If continue_hostname is not null, we get here only when continuing to
 deliver down an existing channel. The channel was passed as the standard
-input. TLS is never active on a passed channel; the previous process always
-closes it down before passing the connection on.
+input. TLS is never active on a passed channel; the previous process either
+closes it down before passing the connection on, or inserts a TLS-proxy
+process and passes on a cleartext conection.
 
 Otherwise, we have to make a connection to the remote host, and do the
 initial protocol exchange.
@@ -3441,6 +3422,9 @@ Returns:          OK    - the connection was made and the delivery attempted;
                           and there was a problem setting it up; OR helo_data
                           or add_headers or authenticated_sender is specified
                           for this transport, and the string failed to expand
+
+               For all non-OK returns the first addr of the list carries the
+               time taken for the attempt.
 */
 
 static int
@@ -3452,18 +3436,15 @@ smtp_transport_options_block * ob = SOB tblock->options_block;
 int yield = OK;
 int save_errno;
 int rc;
-struct timeval start_delivery_time;
 
 BOOL pass_message = FALSE;
 uschar *message = NULL;
 uschar new_message_id[MESSAGE_ID_LENGTH + 1];
-
 smtp_context * sx = store_get(sizeof(*sx), TRUE);      /* tainted, for the data buffers */
 
-gettimeofday(&start_delivery_time, NULL);
-suppress_tls = suppress_tls;  /* stop compiler warning when no TLS support */
 *message_defer = FALSE;
 
+memset(sx, 0, sizeof(*sx));
 sx->addrlist = addrlist;
 sx->conn_args.host = host;
 sx->conn_args.host_af = host_af,
@@ -3471,17 +3452,21 @@ sx->port = defport;
 sx->conn_args.interface = interface;
 sx->helo_data = NULL;
 sx->conn_args.tblock = tblock;
-sx->verify = FALSE;
+/* sx->verify = FALSE; */
+gettimeofday(&sx->delivery_start, NULL);
 sx->sync_addr = sx->first_addr = addrlist;
 
 /* Get the channel set up ready for a message (MAIL FROM being the next
 SMTP command to send */
 
 if ((rc = smtp_setup_conn(sx, suppress_tls)) != OK)
+  {
+  timesince(&addrlist->delivery_time, &sx->delivery_start);
   return rc;
+  }
 
 /* If there is a filter command specified for this transport, we can now
-set it up. This cannot be done until the identify of the host is known. */
+set it up. This cannot be done until the identity of the host is known. */
 
 if (tblock->filter_command)
   {
@@ -3495,7 +3480,7 @@ if (tblock->filter_command)
        string_sprintf("%.50s transport", tblock->name), NULL))
     {
     set_errno_nohost(addrlist->next, addrlist->basic_errno, addrlist->message, DEFER,
-      FALSE);
+      FALSE, &sx->delivery_start);
     yield = ERROR;
     goto SEND_QUIT;
     }
@@ -3539,10 +3524,9 @@ always has a sequence number greater than one. */
 
 if (continue_hostname && continue_sequence == 1)
   {
-  sx->peer_offered = smtp_peer_options;
-  sx->pending_MAIL = FALSE;
+  /* sx->pending_MAIL = FALSE; */
   sx->ok = TRUE;
-  sx->next_addr = NULL;
+  /* sx->next_addr = NULL; */
 
   for (address_item * addr = addrlist; addr; addr = addr->next)
     addr->transport_return = PENDING_OK;
@@ -3574,7 +3558,7 @@ else
        {
        /*XXX could we find a better errno than 0 here? */
        set_errno_nohost(addrlist, 0, a->message, FAIL,
-         testflag(a, af_pass_message));
+         testflag(a, af_pass_message), &sx->delivery_start);
        sx->ok = FALSE;
        break;
        }
@@ -3827,7 +3811,7 @@ else
     int len;
     uschar * conf = NULL;
 
-    timesince(&delivery_time, &start_delivery_time);
+    timesince(&delivery_time, &sx->delivery_start);
     sx->send_rset = FALSE;
     pipelining_active = FALSE;
 
@@ -3902,9 +3886,8 @@ else
       actual host that was used. */
 
       addr->transport_return = OK;
-      addr->more_errno = delivery_time.tv_sec;
-      addr->delivery_usec = delivery_time.tv_usec;
       addr->host_used = host;
+      addr->delivery_time = delivery_time;
       addr->special_action = flag;
       addr->message = conf;
 
@@ -3947,53 +3930,53 @@ else
       }
 
 #ifndef DISABLE_PRDR
-      if (sx->prdr_active)
-        {
-       const uschar * overall_message;
+    if (sx->prdr_active)
+      {
+      const uschar * overall_message;
 
-       /* PRDR - get the final, overall response.  For any non-success
-       upgrade all the address statuses. */
+      /* PRDR - get the final, overall response.  For any non-success
+      upgrade all the address statuses. */
 
-        sx->ok = smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2',
-          ob->final_timeout);
-        if (!sx->ok)
+      sx->ok = smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2',
+       ob->final_timeout);
+      if (!sx->ok)
+       {
+       if(errno == 0 && sx->buffer[0] == '4')
          {
-         if(errno == 0 && sx->buffer[0] == '4')
-            {
-            errno = ERRNO_DATA4XX;
-            addrlist->more_errno |= ((sx->buffer[1] - '0')*10 + sx->buffer[2] - '0') << 8;
-            }
-         for (address_item * addr = addrlist; addr != sx->first_addr; addr = addr->next)
-            if (sx->buffer[0] == '5' || addr->transport_return == OK)
-              addr->transport_return = PENDING_OK; /* allow set_errno action */
-         goto RESPONSE_FAILED;
+         errno = ERRNO_DATA4XX;
+         addrlist->more_errno |= ((sx->buffer[1] - '0')*10 + sx->buffer[2] - '0') << 8;
          }
+       for (address_item * addr = addrlist; addr != sx->first_addr; addr = addr->next)
+         if (sx->buffer[0] == '5' || addr->transport_return == OK)
+           addr->transport_return = PENDING_OK; /* allow set_errno action */
+       goto RESPONSE_FAILED;
+       }
 
-       /* Append the overall response to the individual PRDR response for logging
-       and update the journal, or setup retry. */
+      /* Append the overall response to the individual PRDR response for logging
+      and update the journal, or setup retry. */
 
-       overall_message = string_printing(sx->buffer);
-        for (address_item * addr = addrlist; addr != sx->first_addr; addr = addr->next)
-         if (addr->transport_return == OK)
-           addr->message = string_sprintf("%s\\n%s", addr->message, overall_message);
+      overall_message = string_printing(sx->buffer);
+      for (address_item * addr = addrlist; addr != sx->first_addr; addr = addr->next)
+       if (addr->transport_return == OK)
+         addr->message = string_sprintf("%s\\n%s", addr->message, overall_message);
 
-        for (address_item * addr = addrlist; addr != sx->first_addr; addr = addr->next)
-         if (addr->transport_return == OK)
-           {
-           if (testflag(addr, af_homonym))
-             sprintf(CS sx->buffer, "%.500s/%s\n", addr->unique + 3, tblock->name);
-           else
-             sprintf(CS sx->buffer, "%.500s\n", addr->unique);
+      for (address_item * addr = addrlist; addr != sx->first_addr; addr = addr->next)
+       if (addr->transport_return == OK)
+         {
+         if (testflag(addr, af_homonym))
+           sprintf(CS sx->buffer, "%.500s/%s\n", addr->unique + 3, tblock->name);
+         else
+           sprintf(CS sx->buffer, "%.500s\n", addr->unique);
 
-           DEBUG(D_deliver) debug_printf("journalling(PRDR) %s\n", sx->buffer);
-           len = Ustrlen(CS sx->buffer);
-           if (write(journal_fd, sx->buffer, len) != len)
-             log_write(0, LOG_MAIN|LOG_PANIC, "failed to write journal for "
-               "%s: %s", sx->buffer, strerror(errno));
-           }
-         else if (addr->transport_return == DEFER)
-           retry_add_item(addr, addr->address_retry_key, -2);
-       }
+         DEBUG(D_deliver) debug_printf("journalling(PRDR) %s\n", sx->buffer);
+         len = Ustrlen(CS sx->buffer);
+         if (write(journal_fd, sx->buffer, len) != len)
+           log_write(0, LOG_MAIN|LOG_PANIC, "failed to write journal for "
+             "%s: %s", sx->buffer, strerror(errno));
+         }
+       else if (addr->transport_return == DEFER)
+         retry_add_item(addr, addr->address_retry_key, -2);
+      }
 #endif
 
     /* Ensure the journal file is pushed out to disk. */
@@ -4134,14 +4117,13 @@ if (!sx->ok)
       }
     }
 
-  set_errno(addrlist, save_errno, set_message, set_rc, pass_message, host
+  set_errno(addrlist, save_errno, set_message, set_rc, pass_message, host,
 #ifdef EXPERIMENTAL_DSN_INFO
-           , sx->smtp_greeting, sx->helo_response
+           sx->smtp_greeting, sx->helo_response,
 #endif
-           );
+           &sx->delivery_start);
   }
 
-
 /* If all has gone well, send_quit will be set TRUE, implying we can end the
 SMTP session tidily. However, if there were too many addresses to send in one
 message (indicated by first_addr being non-NULL) we want to carry on with the
@@ -4230,9 +4212,10 @@ if (sx->completed_addr && sx->ok && sx->send_quit)
       int socket_fd = sx->cctx.sock;
 
 
-      if (sx->first_addr != NULL)         /* More addresses still to be sent */
-        {                                /*   in this run of the transport */
-        continue_sequence++;             /* Causes * in logging */
+      if (sx->first_addr != NULL)      /* More addresses still to be sent */
+        {                              /*   in this run of the transport */
+        continue_sequence++;           /* Causes * in logging */
+       pipelining_active = sx->pipelining_used;    /* was cleared at DATA */
         goto SEND_MESSAGE;
         }
 
@@ -4273,11 +4256,11 @@ if (sx->completed_addr && sx->ok && sx->send_quit)
            socket_fd = pfd[1];
          else
            set_errno(sx->first_addr, errno, US"internal allocation problem",
-                   DEFER, FALSE, host
+                   DEFER, FALSE, host,
 # ifdef EXPERIMENTAL_DSN_INFO
-                   , sx->smtp_greeting, sx->helo_response
+                   sx->smtp_greeting, sx->helo_response,
 # endif
-                   );
+                   &sx->delivery_start);
          }
       else
 #endif
@@ -4303,10 +4286,9 @@ propagate it from the initial
 #ifndef DISABLE_TLS
        if (tls_out.active.sock >= 0)
          {
-         int pid = fork();
+         int pid = exim_fork(US"tls-proxy-interproc");
          if (pid == 0)         /* child; fork again to disconnect totally */
            {
-           testharness_pause_ms(100); /* let parent debug out */
            /* does not return */
            smtp_proxy_tls(sx->cctx.tls_ctx, sx->buffer, sizeof(sx->buffer), pfd,
                            ob->command_timeout);
@@ -4314,7 +4296,6 @@ propagate it from the initial
 
          if (pid > 0)          /* parent */
            {
-           DEBUG(D_transport) debug_printf("proxy-proc inter-pid %d\n", pid);
            close(pfd[0]);
            /* tidy the inter-proc to disconn the proxy proc */
            waitpid(pid, NULL, 0);
@@ -4334,11 +4315,11 @@ propagate it from the initial
 
     /* If RSET failed and there are addresses left, they get deferred. */
     else
-      set_errno(sx->first_addr, errno, msg, DEFER, FALSE, host
+      set_errno(sx->first_addr, errno, msg, DEFER, FALSE, host,
 #ifdef EXPERIMENTAL_DSN_INFO
-                 , sx->smtp_greeting, sx->helo_response
+                 sx->smtp_greeting, sx->helo_response,
 #endif
-                 );
+                 &sx->delivery_start);
     }
   }
 
@@ -4554,6 +4535,22 @@ DEBUG(D_transport)
       cutthrough.cctx.sock >= 0 ? cutthrough.cctx.sock : 0);
   }
 
+/* Check the restrictions on line length */
+
+if (max_received_linelength > ob->message_linelength_limit)
+  {
+  struct timeval now;
+  gettimeofday(&now, NULL);
+
+  for (address_item * addr = addrlist; addr; addr = addr->next)
+    if (addr->transport_return == DEFER)
+      addr->transport_return = PENDING_DEFER;
+
+  set_errno_nohost(addrlist, ERRNO_SMTPFORMAT,
+    US"message has lines too long for transport", FAIL, TRUE, &now);
+  goto END_TRANSPORT;
+  }
+
 /* Set the flag requesting that these hosts be added to the waiting
 database if the delivery fails temporarily or if we are running with
 queue_smtp or a 2-stage queue run. This gets unset for certain
@@ -4894,15 +4891,19 @@ retry_non_continued:
     were not in it. We don't want to hold up all SMTP deliveries! Except when
     doing a two-stage queue run, don't do this if forcing. */
 
-    if ((!f.deliver_force || f.queue_2stage) && (f.queue_smtp ||
-        match_isinlist(addrlist->domain,
-         (const uschar **)&queue_smtp_domains, 0,
-          &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL) == OK))
+    if (  (!f.deliver_force || f.queue_2stage)
+       && (  f.queue_smtp
+         || match_isinlist(addrlist->domain,
+             CUSS &queue_smtp_domains, 0,
+             &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL) == OK)
+       )
       {
+      DEBUG(D_transport) debug_printf("first-pass routing only\n");
       expired = FALSE;
       for (address_item * addr = addrlist; addr; addr = addr->next)
         if (addr->transport_return == DEFER)
-         addr->message = US"domain matches queue_smtp_domains, or -odqs set";
+         addr->message = US"first-pass only routing due to -odqs, "
+                           "queue_smtp_domains or control=queue";
       continue;      /* With next host */
       }
 
@@ -5063,7 +5064,9 @@ retry_non_continued:
 
     if (f.dont_deliver)
       {
-      set_errno_nohost(addrlist, 0, NULL, OK, FALSE);
+      struct timeval now;
+      gettimeofday(&now, NULL);
+      set_errno_nohost(addrlist, 0, NULL, OK, FALSE, &now);
       for (address_item * addr = addrlist; addr; addr = addr->next)
         {
         addr->host_used = host;
@@ -5151,7 +5154,7 @@ retry_non_continued:
 
 #ifndef DISABLE_EVENT
       if (rc == DEFER)
-        deferred_event_raise(first_addr, host);
+       deferred_event_raise(first_addr, host, US"msg:host:defer");
 #endif
 
       /* If STARTTLS was accepted, but there was a failure in setting up the
@@ -5179,11 +5182,23 @@ retry_non_continued:
         if (rc == DEFER && first_addr->basic_errno != ERRNO_AUTHFAIL)
           write_logs(host, first_addr->message, first_addr->basic_errno);
 # ifndef DISABLE_EVENT
-        if (rc == DEFER)
-          deferred_event_raise(first_addr, host);
+       if (rc == DEFER)
+         deferred_event_raise(first_addr, host, US"msg:host:defer");
 # endif
         }
 #endif /*DISABLE_TLS*/
+
+#ifndef DISABLE_EVENT
+      /* If the last host gave a defer raise a per-message event */
+
+      if (  !(  nexthost
+            && unexpired_hosts_tried < ob->hosts_max_try
+            && total_hosts_tried < ob->hosts_max_try_hardlimit
+            )
+         && (message_defer || rc == DEFER)
+        )
+       deferred_event_raise(first_addr, host, US"msg:defer");
+#endif
       }
 
     /* Delivery attempt finished */