tidying

[exim.git] / src / src / transports / smtp.c

diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index 181081b60eff7fff285d2afa57ca2ec5d44d28fb..0e1aa308d809dcd56795efc94b1c21547bc62af8 100644 (file)
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -61,9 +61,9 @@ optionlist smtp_transport_options[] = {
   { "dns_search_parents",   opt_bool,
       (void *)offsetof(smtp_transport_options_block, dns_search_parents) },
   { "dnssec_request_domains", opt_stringptr,
   { "dns_search_parents",   opt_bool,
       (void *)offsetof(smtp_transport_options_block, dns_search_parents) },
   { "dnssec_request_domains", opt_stringptr,

-      (void *)offsetof(smtp_transport_options_block, dnssec_request_domains) },
+      (void *)offsetof(smtp_transport_options_block, dnssec.request) },

   { "dnssec_require_domains", opt_stringptr,
   { "dnssec_require_domains", opt_stringptr,

-      (void *)offsetof(smtp_transport_options_block, dnssec_require_domains) },
+      (void *)offsetof(smtp_transport_options_block, dnssec.require) },

   { "dscp",                 opt_stringptr,
       (void *)offsetof(smtp_transport_options_block, dscp) },
   { "fallback_hosts",       opt_stringptr,
   { "dscp",                 opt_stringptr,
       (void *)offsetof(smtp_transport_options_block, dscp) },
   { "fallback_hosts",       opt_stringptr,


@@ -159,6 +159,10 @@ optionlist smtp_transport_options[] = {
       (void *)offsetof(smtp_transport_options_block, serialize_hosts) },
   { "size_addition",        opt_int,
       (void *)offsetof(smtp_transport_options_block, size_addition) }
       (void *)offsetof(smtp_transport_options_block, serialize_hosts) },
   { "size_addition",        opt_int,
       (void *)offsetof(smtp_transport_options_block, size_addition) }

+#ifdef EXPERIMENTAL_SOCKS
+ ,{ "socks_proxy",          opt_stringptr,
+      (void *)offsetof(smtp_transport_options_block, socks_proxy) }
+#endif

 #ifdef SUPPORT_TLS
  ,{ "tls_certificate",      opt_stringptr,
       (void *)offsetof(smtp_transport_options_block, tls_certificate) },
 #ifdef SUPPORT_TLS
  ,{ "tls_certificate",      opt_stringptr,
       (void *)offsetof(smtp_transport_options_block, tls_certificate) },


@@ -220,7 +224,7 @@ smtp_transport_options_block smtp_transport_option_defaults = {
 #endif
   NULL,                /* hosts_require_tls */
   NULL,                /* hosts_avoid_tls */
 #endif
   NULL,                /* hosts_require_tls */
   NULL,                /* hosts_avoid_tls */

-  US"*",               /* hosts_verify_avoid_tls */
+  NULL,                /* hosts_verify_avoid_tls */

   NULL,                /* hosts_avoid_pipelining */
   NULL,                /* hosts_avoid_esmtp */
   NULL,                /* hosts_nopass_tls */
   NULL,                /* hosts_avoid_pipelining */
   NULL,                /* hosts_avoid_esmtp */
   NULL,                /* hosts_nopass_tls */


@@ -237,8 +241,7 @@ smtp_transport_options_block smtp_transport_option_defaults = {
   FALSE,               /* gethostbyname */
   TRUE,                /* dns_qualify_single */
   FALSE,               /* dns_search_parents */
   FALSE,               /* gethostbyname */
   TRUE,                /* dns_qualify_single */
   FALSE,               /* dns_search_parents */

-  NULL,                /* dnssec_request_domains */
-  NULL,                /* dnssec_require_domains */
+  { NULL, NULL },      /* dnssec_domains {request,require} */

   TRUE,                /* delay_after_cutoff */
   FALSE,               /* hosts_override */
   FALSE,               /* hosts_randomize */
   TRUE,                /* delay_after_cutoff */
   FALSE,               /* hosts_override */
   FALSE,               /* hosts_randomize */


@@ -246,6 +249,9 @@ smtp_transport_options_block smtp_transport_option_defaults = {
   FALSE,               /* lmtp_ignore_quota */
   NULL,                       /* expand_retry_include_ip_address */
   TRUE                 /* retry_include_ip_address */
   FALSE,               /* lmtp_ignore_quota */
   NULL,                       /* expand_retry_include_ip_address */
   TRUE                 /* retry_include_ip_address */

+#ifdef EXPERIMENTAL_SOCKS
+ ,NULL                 /* socks_proxy */
+#endif

 #ifdef SUPPORT_TLS
  ,NULL,                /* tls_certificate */
   NULL,                /* tls_crl */
 #ifdef SUPPORT_TLS
  ,NULL,                /* tls_certificate */
   NULL,                /* tls_crl */


@@ -273,14 +279,12 @@ smtp_transport_options_block smtp_transport_option_defaults = {
 #endif
 };
 
 #endif
 };
 

-#ifdef EXPERIMENTAL_DSN

 /* some DSN flags for use later */
 
 static int     rf_list[] = {rf_notify_never, rf_notify_success,
                             rf_notify_failure, rf_notify_delay };
 
 static uschar *rf_names[] = { US"NEVER", US"SUCCESS", US"FAILURE", US"DELAY" };
 /* some DSN flags for use later */
 
 static int     rf_list[] = {rf_notify_never, rf_notify_success,
                             rf_notify_failure, rf_notify_delay };
 
 static uschar *rf_names[] = { US"NEVER", US"SUCCESS", US"FAILURE", US"DELAY" };

-#endif

 
 
 
 
 
 


@@ -456,19 +460,19 @@ if (errno_value == ERRNO_CONNECTTIMEOUT)
   orvalue = RTEF_CTOUT;
   }
 for (addr = addrlist; addr != NULL; addr = addr->next)
   orvalue = RTEF_CTOUT;
   }
 for (addr = addrlist; addr != NULL; addr = addr->next)

-  {
-  if (addr->transport_return < PENDING) continue;
-  addr->basic_errno = errno_value;
-  addr->more_errno |= orvalue;
-  if (msg != NULL)
+  if (addr->transport_return >= PENDING)

     {
     {

-    addr->message = msg;
-    if (pass_message) setflag(addr, af_pass_message);
+    addr->basic_errno = errno_value;
+    addr->more_errno |= orvalue;
+    if (msg != NULL)
+      {
+      addr->message = msg;
+      if (pass_message) setflag(addr, af_pass_message);
+      }
+    addr->transport_return = rc;
+    if (host)
+      addr->host_used = host;

     }
     }

-  addr->transport_return = rc;
-  if (host)
-    addr->host_used = host;
-  }

 }
 
 
 }
 
 


@@ -525,7 +529,7 @@ if (*errno_value == ETIMEDOUT)
 
 if (*errno_value == ERRNO_SMTPFORMAT)
   {
 
 if (*errno_value == ERRNO_SMTPFORMAT)
   {

-  uschar *malfresp = string_printing(buffer);
+  const uschar *malfresp = string_printing(buffer);

   while (isspace(*malfresp)) malfresp++;
   *message = *malfresp == 0
     ? string_sprintf("Malformed SMTP reply (an empty line) "
   while (isspace(*malfresp)) malfresp++;
   *message = *malfresp == 0
     ? string_sprintf("Malformed SMTP reply (an empty line) "


@@ -565,11 +569,21 @@ if (*errno_value == ERRNO_WRITEINCOMPLETE)
   return FALSE;
   }
 
   return FALSE;
   }
 

+#ifdef EXPERIMENTAL_INTERNATIONAL
+/* Handle lack of advertised SMTPUTF8, for international message */
+if (*errno_value == ERRNO_UTF8_FWD)
+  {
+  *message = US string_sprintf("utf8 support required but not offered for forwarding");
+  DEBUG(D_deliver|D_transport) debug_printf("%s\n", *message);
+  return TRUE;
+  }
+#endif
+

 /* Handle error responses from the remote mailer. */
 
 if (buffer[0] != 0)
   {
 /* Handle error responses from the remote mailer. */
 
 if (buffer[0] != 0)
   {

-  uschar *s = string_printing(buffer);
+  const uschar *s = string_printing(buffer);

   *message = US string_sprintf("SMTP error from remote mail server after %s%s: "
     "%s", pl, smtp_command, s);
   *pass_message = TRUE;
   *message = US string_sprintf("SMTP error from remote mail server after %s%s: "
     "%s", pl, smtp_command, s);
   *pass_message = TRUE;


@@ -661,7 +675,7 @@ static void
 deferred_event_raise(address_item *addr, host_item *host)
 {
 uschar * action = addr->transport->event_action;
 deferred_event_raise(address_item *addr, host_item *host)
 {
 uschar * action = addr->transport->event_action;

-uschar * save_domain;
+const uschar * save_domain;

 uschar * save_local;
 
 if (!action)
 uschar * save_local;
 
 if (!action)


@@ -695,8 +709,6 @@ router_name = transport_name = NULL;
 }
 #endif
 
 }
 #endif
 

-
-

 /*************************************************
 *           Synchronize SMTP responses           *
 *************************************************/
 /*************************************************
 *           Synchronize SMTP responses           *
 *************************************************/


@@ -833,10 +845,9 @@ while (count-- > 0)
 
   else if (errno == ETIMEDOUT)
     {
 
   else if (errno == ETIMEDOUT)
     {

-    int save_errno = errno;

     uschar *message = string_sprintf("SMTP timeout after RCPT TO:<%s>",
                          transport_rcpt_address(addr, include_affixes));
     uschar *message = string_sprintf("SMTP timeout after RCPT TO:<%s>",
                          transport_rcpt_address(addr, include_affixes));

-    set_errno(addrlist, save_errno, message, DEFER, FALSE, NULL);
+    set_errno(addrlist, ETIMEDOUT, message, DEFER, FALSE, NULL);

     retry_add_item(addr, addr->address_retry_key, 0);
     update_waiting = FALSE;
     return -1;
     retry_add_item(addr, addr->address_retry_key, 0);
     update_waiting = FALSE;
     return -1;


@@ -1085,7 +1096,8 @@ if (is_esmtp && regex_match_and_setup(regex_AUTH, buffer, 0, -1))
          /* Internal problem, message in buffer. */
 
          case ERROR:
          /* Internal problem, message in buffer. */
 
          case ERROR:

-         set_errno(addrlist, 0, string_copy(buffer), DEFER, FALSE, NULL);
+         set_errno(addrlist, ERRNO_AUTHPROB, string_copy(buffer),
+                   DEFER, FALSE, NULL);

          return ERROR;
          }
 
          return ERROR;
          }
 


@@ -1140,7 +1152,7 @@ if (ob->authenticated_sender != NULL)
       {
       uschar *message = string_sprintf("failed to expand "
         "authenticated_sender: %s", expand_string_message);
       {
       uschar *message = string_sprintf("failed to expand "
         "authenticated_sender: %s", expand_string_message);

-      set_errno(addrlist, 0, message, DEFER, FALSE, NULL);
+      set_errno(addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE, NULL);

       return TRUE;
       }
     }
       return TRUE;
       }
     }


@@ -1172,7 +1184,7 @@ tlsa_lookup(const host_item * host, dns_answer * dnsa,
 {
 /* move this out to host.c given the similarity to dns_lookup() ? */
 uschar buffer[300];
 {
 /* move this out to host.c given the similarity to dns_lookup() ? */
 uschar buffer[300];

-uschar * fullname = buffer;
+const uschar * fullname = buffer;

 
 /* TLSA lookup string */
 (void)sprintf(CS buffer, "_%d._tcp.%.256s", host->port, host->name);
 
 /* TLSA lookup string */
 (void)sprintf(CS buffer, "_%d._tcp.%.256s", host->port, host->name);


@@ -1185,10 +1197,7 @@ switch (dns_lookup(dnsa, buffer, T_TLSA, &fullname))
   default:
   case DNS_FAIL:
     if (dane_required)
   default:
   case DNS_FAIL:
     if (dane_required)

-      {
-      log_write(0, LOG_MAIN, "DANE error: TLSA lookup failed");

       return FAIL;
       return FAIL;

-      }

     break;
 
   case DNS_SUCCEED:
     break;
 
   case DNS_SUCCEED:


@@ -1205,6 +1214,80 @@ return OK;
 #endif
 
 
 #endif
 
 

+
+typedef struct smtp_compare_s
+{
+    uschar                          *current_sender_address;
+    struct transport_instance       *tblock;
+} smtp_compare_t;
+
+/*
+Create a unique string that identifies this message, it is based on
+sender_address, helo_data and tls_certificate if enabled.  */
+
+static uschar *
+smtp_local_identity(uschar * sender, struct transport_instance * tblock)
+{
+address_item * addr1;
+uschar * if1 = US"";
+uschar * helo1 = US"";
+#ifdef SUPPORT_TLS
+uschar * tlsc1 = US"";
+#endif
+uschar * save_sender_address = sender_address;
+uschar * local_identity = NULL;
+smtp_transport_options_block * ob =
+  (smtp_transport_options_block *)tblock->options_block;
+
+sender_address = sender;
+
+addr1 = deliver_make_addr (sender, TRUE);
+deliver_set_expansions(addr1);
+
+if (ob->interface)
+  if1 = expand_string(ob->interface);
+
+if (ob->helo_data)
+  helo1 = expand_string(ob->helo_data);
+
+#ifdef SUPPORT_TLS
+if (ob->tls_certificate)
+  tlsc1 = expand_string(ob->tls_certificate);
+local_identity = string_sprintf ("%s^%s^%s", if1, helo1, tlsc1);
+#else
+local_identity = string_sprintf ("%s^%s", if1, helo1);
+#endif
+
+deliver_set_expansions(NULL);
+sender_address = save_sender_address;
+
+return local_identity;
+}
+
+
+
+/* This routine is a callback that is called from transport_check_waiting.
+This function will evaluate the incoming message versus the previous
+message.  If the incoming message is using a different local identity then
+we will veto this new message.  */
+
+static BOOL
+smtp_are_same_identities(uschar * message_id, smtp_compare_t * s_compare)
+{
+uschar * save_sender_address = sender_address;
+uschar * current_local_identity =
+  smtp_local_identity(s_compare->current_sender_address, s_compare->tblock);
+uschar * new_sender_address = deliver_get_sender_address(message_id);
+uschar * message_local_identity =
+  smtp_local_identity(new_sender_address, s_compare->tblock);
+
+sender_address = save_sender_address;
+
+return Ustrcmp(current_local_identity, message_local_identity) == 0;
+}
+
+
+

 /*************************************************
 *       Deliver address list to given host       *
 *************************************************/
 /*************************************************
 *       Deliver address list to given host       *
 *************************************************/


@@ -1278,18 +1361,23 @@ BOOL pass_message = FALSE;
 BOOL prdr_offered = FALSE;
 BOOL prdr_active;
 #endif
 BOOL prdr_offered = FALSE;
 BOOL prdr_active;
 #endif

-#ifdef EXPERIMENTAL_DSN
-BOOL dsn_all_lasthop = TRUE;
+#ifdef EXPERIMENTAL_INTERNATIONAL
+BOOL utf8_needed = FALSE;
+BOOL utf8_offered = FALSE;

 #endif
 #endif

+BOOL dsn_all_lasthop = TRUE;

 #if defined(SUPPORT_TLS) && defined(EXPERIMENTAL_DANE)
 BOOL dane = FALSE;
 #if defined(SUPPORT_TLS) && defined(EXPERIMENTAL_DANE)
 BOOL dane = FALSE;

+BOOL dane_required = verify_check_given_host(&ob->hosts_require_dane, host) == OK;

 dns_answer tlsa_dnsa;
 #endif
 smtp_inblock inblock;
 smtp_outblock outblock;
 int max_rcpt = tblock->max_addresses;
 uschar *igquotstr = US"";
 dns_answer tlsa_dnsa;
 #endif
 smtp_inblock inblock;
 smtp_outblock outblock;
 int max_rcpt = tblock->max_addresses;
 uschar *igquotstr = US"";

+

 uschar *helo_data = NULL;
 uschar *helo_data = NULL;

+

 uschar *message = NULL;
 uschar new_message_id[MESSAGE_ID_LENGTH + 1];
 uschar *p;
 uschar *message = NULL;
 uschar new_message_id[MESSAGE_ID_LENGTH + 1];
 uschar *p;


@@ -1339,7 +1427,8 @@ tls_modify_variables(&tls_out);
 #ifndef SUPPORT_TLS
 if (smtps)
   {
 #ifndef SUPPORT_TLS
 if (smtps)
   {

-  set_errno(addrlist, 0, US"TLS support not available", DEFER, FALSE, NULL);
+  set_errno(addrlist, ERRNO_TLSFAILURE, US"TLS support not available",
+           DEFER, FALSE, NULL);

   return ERROR;
   }
 #endif
   return ERROR;
   }
 #endif


@@ -1352,12 +1441,7 @@ if (continue_hostname == NULL)
   {
   /* This puts port into host->port */
   inblock.sock = outblock.sock =
   {
   /* This puts port into host->port */
   inblock.sock = outblock.sock =

-    smtp_connect(host, host_af, port, interface, ob->connect_timeout,
-                 ob->keepalive, ob->dscp
-#ifdef EXPERIMENTAL_EVENT
-                 , tblock->event_action
-#endif
-               );
+    smtp_connect(host, host_af, port, interface, ob->connect_timeout, tblock);

 
   if (inblock.sock < 0)
     {
 
   if (inblock.sock < 0)
     {


@@ -1368,25 +1452,31 @@ if (continue_hostname == NULL)
 
 #if defined(SUPPORT_TLS) && defined(EXPERIMENTAL_DANE)
     {
 
 #if defined(SUPPORT_TLS) && defined(EXPERIMENTAL_DANE)
     {

-    BOOL dane_required;
-

     tls_out.dane_verified = FALSE;
     tls_out.tlsa_usage = 0;
 
     tls_out.dane_verified = FALSE;
     tls_out.tlsa_usage = 0;
 

-    dane_required = verify_check_given_host(&ob->hosts_require_dane, host) == OK;
-

     if (host->dnssec == DS_YES)
       {
     if (host->dnssec == DS_YES)
       {

-      if(  dane_required
-       || verify_check_given_host(&ob->hosts_try_dane, host) == OK
+      if(  (  dane_required
+          || verify_check_given_host(&ob->hosts_try_dane, host) == OK
+          )
+       && (rc = tlsa_lookup(host, &tlsa_dnsa, dane_required, &dane)) != OK
+       && dane_required

        )
        )

-       if ((rc = tlsa_lookup(host, &tlsa_dnsa, dane_required, &dane)) != OK)
-         return rc;
+       {
+       set_errno(addrlist, ERRNO_DNSDEFER,
+         string_sprintf("DANE error: tlsa lookup %s",
+           rc == DEFER ? "DEFER" : "FAIL"),
+         rc, FALSE, NULL);
+       return  rc;
+       }

       }
     else if (dane_required)
       {
       }
     else if (dane_required)
       {

-      log_write(0, LOG_MAIN, "DANE error: %s lookup not DNSSEC", host->name);
-      return FAIL;
+      set_errno(addrlist, ERRNO_DNSDEFER,
+       string_sprintf("DANE error: %s lookup not DNSSEC", host->name),
+       FAIL, FALSE, NULL);
+      return  FAIL;

       }
 
     if (dane)
       }
 
     if (dane)


@@ -1399,6 +1489,19 @@ if (continue_hostname == NULL)
   delayed till here so that $sending_interface and $sending_port are set. */
 
   helo_data = expand_string(ob->helo_data);
   delayed till here so that $sending_interface and $sending_port are set. */
 
   helo_data = expand_string(ob->helo_data);

+#ifdef EXPERIMENTAL_INTERNATIONAL
+  if (helo_data)
+    {
+    uschar * errstr = NULL;
+    if ((helo_data = string_domain_utf8_to_alabel(helo_data, &errstr)), errstr)
+      {
+      errstr = string_sprintf("failed to expand helo_data: %s", errstr);
+      set_errno(addrlist, ERRNO_EXPANDFAIL, errstr, DEFER, FALSE, NULL);
+      yield = DEFER;
+      goto SEND_QUIT;
+      }
+    }
+#endif

 
   /* The first thing is to wait for an initial OK response. The dreaded "goto"
   is nevertheless a reasonably clean way of programming this kind of logic,
 
   /* The first thing is to wait for an initial OK response. The dreaded "goto"
   is nevertheless a reasonably clean way of programming this kind of logic,


@@ -1417,7 +1520,7 @@ if (continue_hostname == NULL)
       s = event_raise(tblock->event_action, US"smtp:connect", buffer);
       if (s)
        {
       s = event_raise(tblock->event_action, US"smtp:connect", buffer);
       if (s)
        {

-       set_errno(addrlist, 0,
+       set_errno(addrlist, ERRNO_EXPANDFAIL,

          string_sprintf("deferred by smtp:connect event expansion: %s", s),
          DEFER, FALSE, NULL);
        yield = DEFER;
          string_sprintf("deferred by smtp:connect event expansion: %s", s),
          DEFER, FALSE, NULL);
        yield = DEFER;


@@ -1433,7 +1536,7 @@ if (continue_hostname == NULL)
       {
       uschar *message = string_sprintf("failed to expand helo_data: %s",
         expand_string_message);
       {
       uschar *message = string_sprintf("failed to expand helo_data: %s",
         expand_string_message);

-      set_errno(addrlist, 0, message, DEFER, FALSE, NULL);
+      set_errno(addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE, NULL);

       yield = DEFER;
       goto SEND_QUIT;
       }
       yield = DEFER;
       goto SEND_QUIT;
       }


@@ -1540,6 +1643,20 @@ goto SEND_QUIT;
   if (prdr_offered)
     {DEBUG(D_transport) debug_printf("PRDR usable\n");}
 #endif
   if (prdr_offered)
     {DEBUG(D_transport) debug_printf("PRDR usable\n");}
 #endif

+
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  if (addrlist->prop.utf8_msg)
+    {
+    utf8_needed =  !addrlist->prop.utf8_downcvt
+               && !addrlist->prop.utf8_downcvt_maybe;
+    DEBUG(D_transport) if (!utf8_needed) debug_printf("utf8: %s downconvert\n",
+      addrlist->prop.utf8_downcvt ? "mandatory" : "optional");
+
+    utf8_offered = esmtp
+      && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0,
+                   PCRE_EOPT, NULL, 0) >= 0;
+    }
+#endif

   }
 
 /* For continuing deliveries down the same channel, the socket is the standard
   }
 
 /* For continuing deliveries down the same channel, the socket is the standard


@@ -1608,6 +1725,17 @@ if (  tls_offered
 
     if (rc != OK)
       {
 
     if (rc != OK)
       {

+# ifdef EXPERIMENTAL_DANE
+      if (rc == DEFER && dane && !dane_required)
+       {
+       log_write(0, LOG_MAIN, "DANE attempt failed;"
+         " trying CA-root TLS to %s [%s] (not in hosts_require_dane)",
+         host->name, host->address);
+       dane = FALSE;
+       goto TLS_NEGOTIATE;
+       }
+# endif
+

       save_errno = ERRNO_TLSFAILURE;
       message = US"failure while setting up TLS session";
       send_quit = FALSE;
       save_errno = ERRNO_TLSFAILURE;
       message = US"failure while setting up TLS session";
       send_quit = FALSE;


@@ -1648,7 +1776,7 @@ if (tls_out.active >= 0)
       {
       uschar *message = string_sprintf("failed to expand helo_data: %s",
         expand_string_message);
       {
       uschar *message = string_sprintf("failed to expand helo_data: %s",
         expand_string_message);

-      set_errno(addrlist, 0, message, DEFER, FALSE, NULL);
+      set_errno(addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE, NULL);

       yield = DEFER;
       goto SEND_QUIT;
       }
       yield = DEFER;
       goto SEND_QUIT;
       }


@@ -1736,19 +1864,25 @@ if (continue_hostname == NULL
 #ifndef DISABLE_PRDR
   prdr_offered = esmtp
     && pcre_exec(regex_PRDR, NULL, CS buffer, Ustrlen(CS buffer), 0,
 #ifndef DISABLE_PRDR
   prdr_offered = esmtp
     && pcre_exec(regex_PRDR, NULL, CS buffer, Ustrlen(CS buffer), 0,

-      PCRE_EOPT, NULL, 0) >= 0
+                 PCRE_EOPT, NULL, 0) >= 0

     && verify_check_given_host(&ob->hosts_try_prdr, host) == OK;
 
   if (prdr_offered)
     {DEBUG(D_transport) debug_printf("PRDR usable\n");}
 #endif
 
     && verify_check_given_host(&ob->hosts_try_prdr, host) == OK;
 
   if (prdr_offered)
     {DEBUG(D_transport) debug_printf("PRDR usable\n");}
 #endif
 

-#ifdef EXPERIMENTAL_DSN
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  if (addrlist->prop.utf8_msg)
+    utf8_offered = esmtp
+      && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0,
+                   PCRE_EOPT, NULL, 0) >= 0;
+#endif
+

   /* Note if the server supports DSN */
   /* Note if the server supports DSN */

-  smtp_use_dsn = esmtp && pcre_exec(regex_DSN, NULL, CS buffer, (int)Ustrlen(CS buffer), 0,
-       PCRE_EOPT, NULL, 0) >= 0;
+  smtp_use_dsn = esmtp
+    && pcre_exec(regex_DSN, NULL, CS buffer, Ustrlen(CS buffer), 0,
+                 PCRE_EOPT, NULL, 0) >= 0;

   DEBUG(D_transport) debug_printf("use_dsn=%d\n", smtp_use_dsn);
   DEBUG(D_transport) debug_printf("use_dsn=%d\n", smtp_use_dsn);

-#endif

 
   /* Note if the response to EHLO specifies support for the AUTH extension.
   If it has, check that this host is one we want to authenticate to, and do
 
   /* Note if the response to EHLO specifies support for the AUTH extension.
   If it has, check that this host is one we want to authenticate to, and do


@@ -1770,6 +1904,15 @@ message-specific. */
 
 setting_up = FALSE;
 
 
 setting_up = FALSE;
 

+#ifdef EXPERIMENTAL_INTERNATIONAL
+/* If this is an international message we need the host to speak SMTPUTF8 */
+if (utf8_needed && !utf8_offered)
+  {
+  errno = ERRNO_UTF8_FWD;
+  goto RESPONSE_FAILED;
+  }
+#endif
+

 /* If there is a filter command specified for this transport, we can now
 set it up. This cannot be done until the identify of the host is known. */
 
 /* If there is a filter command specified for this transport, we can now
 set it up. This cannot be done until the identify of the host is known. */
 


@@ -1846,19 +1989,25 @@ if (prdr_offered)
   }
 #endif
 
   }
 #endif
 

-#ifdef EXPERIMENTAL_DSN
+#ifdef EXPERIMENTAL_INTERNATIONAL
+if (addrlist->prop.utf8_msg && !addrlist->prop.utf8_downcvt && utf8_offered)
+  sprintf(CS p, " SMTPUTF8"), p += 9;
+#endif
+

 /* check if all addresses have lasthop flag */
 /* do not send RET and ENVID if true */
 /* check if all addresses have lasthop flag */
 /* do not send RET and ENVID if true */

-dsn_all_lasthop = TRUE;
-for (addr = first_addr;
+for (dsn_all_lasthop = TRUE, addr = first_addr;

      address_count < max_rcpt && addr != NULL;
      addr = addr->next)
   if ((addr->dsn_flags & rf_dsnlasthop) != 1)
      address_count < max_rcpt && addr != NULL;
      addr = addr->next)
   if ((addr->dsn_flags & rf_dsnlasthop) != 1)

+    {

     dsn_all_lasthop = FALSE;
     dsn_all_lasthop = FALSE;

+    break;
+    }

 
 /* Add any DSN flags to the mail command */
 
 
 /* Add any DSN flags to the mail command */
 

-if ((smtp_use_dsn) && (dsn_all_lasthop == FALSE))
+if (smtp_use_dsn && !dsn_all_lasthop)

   {
   if (dsn_ret == dsn_ret_hdrs)
     {
   {
   if (dsn_ret == dsn_ret_hdrs)
     {


@@ -1876,7 +2025,6 @@ if ((smtp_use_dsn) && (dsn_all_lasthop == FALSE))
     while (*p) p++;
     }
   }
     while (*p) p++;
     }
   }

-#endif

 
 /* If an authenticated_sender override has been specified for this transport
 instance, expand it. If the expansion is forced to fail, and there was already
 
 /* If an authenticated_sender override has been specified for this transport
 instance, expand it. If the expansion is forced to fail, and there was already


@@ -1899,28 +2047,51 @@ buffer. */
 
 pending_MAIL = TRUE;     /* The block starts with MAIL */
 
 
 pending_MAIL = TRUE;     /* The block starts with MAIL */
 

-rc = smtp_write_command(&outblock, smtp_use_pipelining,
-       "MAIL FROM:<%s>%s\r\n", return_path, buffer);
+  {
+  uschar * s = return_path;
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  uschar * errstr = NULL;
+
+  /* If we must downconvert, do the from-address here.  Remember we had to
+  for the to-addresses (done below), and also (ugly) for re-doing when building
+  the delivery log line. */
+
+  if (addrlist->prop.utf8_msg && (addrlist->prop.utf8_downcvt || !utf8_offered))
+    {
+    if (s = string_address_utf8_to_alabel(return_path, &errstr), errstr)
+      {
+      set_errno(addrlist, ERRNO_EXPANDFAIL, errstr, DEFER, FALSE, NULL);
+      yield = ERROR;
+      goto SEND_QUIT;
+      }
+    setflag(addrlist, af_utf8_downcvt);
+    }
+#endif
+
+  rc = smtp_write_command(&outblock, smtp_use_pipelining,
+         "MAIL FROM:<%s>%s\r\n", s, buffer);
+  }
+

 mail_command = string_copy(big_buffer);  /* Save for later error message */
 
 switch(rc)
   {
   case -1:                /* Transmission error */
 mail_command = string_copy(big_buffer);  /* Save for later error message */
 
 switch(rc)
   {
   case -1:                /* Transmission error */

-  goto SEND_FAILED;
+    goto SEND_FAILED;

 
   case +1:                /* Block was sent */
 
   case +1:                /* Block was sent */

-  if (!smtp_read_response(&inblock, buffer, sizeof(buffer), '2',
+    if (!smtp_read_response(&inblock, buffer, sizeof(buffer), '2',

        ob->command_timeout))
        ob->command_timeout))

-    {
-    if (errno == 0 && buffer[0] == '4')

       {
       {

-      errno = ERRNO_MAIL4XX;
-      addrlist->more_errno |= ((buffer[1] - '0')*10 + buffer[2] - '0') << 8;
+      if (errno == 0 && buffer[0] == '4')
+       {
+       errno = ERRNO_MAIL4XX;
+       addrlist->more_errno |= ((buffer[1] - '0')*10 + buffer[2] - '0') << 8;
+       }
+      goto RESPONSE_FAILED;

       }
       }

-    goto RESPONSE_FAILED;
-    }
-  pending_MAIL = FALSE;
-  break;
+    pending_MAIL = FALSE;
+    break;

   }
 
 /* Pass over all the relevant recipient addresses for this host, which are the
   }
 
 /* Pass over all the relevant recipient addresses for this host, which are the


@@ -1942,17 +2113,15 @@ for (addr = first_addr;
   {
   int count;
   BOOL no_flush;
   {
   int count;
   BOOL no_flush;

+  uschar * rcpt_addr;

 
 

-#ifdef EXPERIMENTAL_DSN

   addr->dsn_aware = smtp_use_dsn ? dsn_support_yes : dsn_support_no;
   addr->dsn_aware = smtp_use_dsn ? dsn_support_yes : dsn_support_no;

-#endif

 
   if (addr->transport_return != PENDING_DEFER) continue;
 
   address_count++;
   no_flush = smtp_use_pipelining && (!mua_wrapper || addr->next != NULL);
 
 
   if (addr->transport_return != PENDING_DEFER) continue;
 
   address_count++;
   no_flush = smtp_use_pipelining && (!mua_wrapper || addr->next != NULL);
 

-#ifdef EXPERIMENTAL_DSN

   /* Add any DSN flags to the rcpt command and add to the sent string */
 
   p = buffer;
   /* Add any DSN flags to the rcpt command and add to the sent string */
 
   p = buffer;


@@ -1983,22 +2152,31 @@ for (addr = first_addr;
       while (*p) p++;
       }
     }
       while (*p) p++;
       }
     }

-#endif
-

 
   /* Now send the RCPT command, and process outstanding responses when
   necessary. After a timeout on RCPT, we just end the function, leaving the
   yield as OK, because this error can often mean that there is a problem with
   just one address, so we don't want to delay the host. */
 
 
   /* Now send the RCPT command, and process outstanding responses when
   necessary. After a timeout on RCPT, we just end the function, leaving the
   yield as OK, because this error can often mean that there is a problem with
   just one address, so we don't want to delay the host. */
 

-#ifdef EXPERIMENTAL_DSN
-  count = smtp_write_command(&outblock, no_flush, "RCPT TO:<%s>%s%s\r\n",
-    transport_rcpt_address(addr, tblock->rcpt_include_affixes), igquotstr, buffer);
-#else
-  count = smtp_write_command(&outblock, no_flush, "RCPT TO:<%s>%s\r\n",
-    transport_rcpt_address(addr, tblock->rcpt_include_affixes), igquotstr);
+  rcpt_addr = transport_rcpt_address(addr, tblock->rcpt_include_affixes);
+
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  {
+  uschar * dummy_errstr;
+  if (  testflag(addrlist, af_utf8_downcvt)
+     && (rcpt_addr = string_address_utf8_to_alabel(rcpt_addr, &dummy_errstr),
+        dummy_errstr
+     )  )
+    {
+    errno = ERRNO_EXPANDFAIL;
+    goto SEND_FAILED;
+    }
+  }

 #endif
 
 #endif
 

+  count = smtp_write_command(&outblock, no_flush, "RCPT TO:<%s>%s%s\r\n",
+    rcpt_addr, igquotstr, buffer);
+

   if (count < 0) goto SEND_FAILED;
   if (count > 0)
     {
   if (count < 0) goto SEND_FAILED;
   if (count > 0)
     {


@@ -2030,16 +2208,15 @@ RCPT. */
 if (mua_wrapper)
   {
   address_item *badaddr;
 if (mua_wrapper)
   {
   address_item *badaddr;

-  for (badaddr = first_addr; badaddr != NULL; badaddr = badaddr->next)
-    {
-    if (badaddr->transport_return != PENDING_OK) break;
-    }
-  if (badaddr != NULL)
-    {
-    set_errno(addrlist, 0, badaddr->message, FAIL,
-      testflag(badaddr, af_pass_message), NULL);
-    ok = FALSE;
-    }
+  for (badaddr = first_addr; badaddr; badaddr = badaddr->next)
+    if (badaddr->transport_return != PENDING_OK)
+      {
+      /*XXX could we find a better errno than 0 here? */
+      set_errno(addrlist, 0, badaddr->message, FAIL,
+       testflag(badaddr, af_pass_message), NULL);
+      ok = FALSE;
+      break;
+      }

   }
 
 /* If ok is TRUE, we know we have got at least one good recipient, and must now
   }
 
 /* If ok is TRUE, we know we have got at least one good recipient, and must now


@@ -2208,8 +2385,9 @@ if (!ok) ok = TRUE; else
           !lmtp
        )
       {
           !lmtp
        )
       {

-      uschar *s = string_printing(buffer);
-      conf = (s == buffer)? (uschar *)string_copy(s) : s;
+      const uschar *s = string_printing(buffer);
+      /* deconst cast ok here as string_printing was checked to have alloc'n'copied */
+      conf = (s == buffer)? (uschar *)string_copy(s) : US s;

       }
 
     /* Process all transported addresses - for LMTP or PRDR, read a status for
       }
 
     /* Process all transported addresses - for LMTP or PRDR, read a status for


@@ -2259,8 +2437,9 @@ if (!ok) ok = TRUE; else
         completed_address = TRUE;   /* NOW we can set this flag */
         if ((log_extra_selector & LX_smtp_confirmation) != 0)
           {
         completed_address = TRUE;   /* NOW we can set this flag */
         if ((log_extra_selector & LX_smtp_confirmation) != 0)
           {

-          uschar *s = string_printing(buffer);
-          conf = (s == buffer)? (uschar *)string_copy(s) : s;
+          const uschar *s = string_printing(buffer);
+         /* deconst cast ok here as string_printing was checked to have alloc'n'copied */
+          conf = (s == buffer)? (uschar *)string_copy(s) : US s;

           }
         }
 
           }
         }
 


@@ -2418,24 +2597,29 @@ if (!ok)
 
     switch(save_errno)
       {
 
     switch(save_errno)
       {

+#ifdef EXPERIMENTAL_INTERNATIONAL
+      case ERRNO_UTF8_FWD:
+        code = '5';
+      /*FALLTHROUGH*/
+#endif

       case 0:
       case ERRNO_MAIL4XX:
       case ERRNO_DATA4XX:
       case 0:
       case ERRNO_MAIL4XX:
       case ERRNO_DATA4XX:

-      message_error = TRUE;
-      break;
+       message_error = TRUE;
+       break;

 
       case ETIMEDOUT:
 
       case ETIMEDOUT:

-      message_error = Ustrncmp(smtp_command,"MAIL",4) == 0 ||
-                      Ustrncmp(smtp_command,"end ",4) == 0;
-      break;
+       message_error = Ustrncmp(smtp_command,"MAIL",4) == 0 ||
+                       Ustrncmp(smtp_command,"end ",4) == 0;
+       break;

 
       case ERRNO_SMTPCLOSED:
 
       case ERRNO_SMTPCLOSED:

-      message_error = Ustrncmp(smtp_command,"end ",4) == 0;
-      break;
+       message_error = Ustrncmp(smtp_command,"end ",4) == 0;
+       break;

 
       default:
 
       default:

-      message_error = FALSE;
-      break;
+       message_error = FALSE;
+       break;

       }
 
     /* Handle the cases that are treated as message errors. These are:
       }
 
     /* Handle the cases that are treated as message errors. These are:


@@ -2443,6 +2627,7 @@ if (!ok)
       (a) negative response or timeout after MAIL
       (b) negative response after DATA
       (c) negative response or timeout or dropped connection after "."
       (a) negative response or timeout after MAIL
       (b) negative response after DATA
       (c) negative response or timeout or dropped connection after "."

+      (d) utf8 support required and not offered

 
     It won't be a negative response or timeout after RCPT, as that is dealt
     with separately above. The action in all cases is to set an appropriate
 
     It won't be a negative response or timeout after RCPT, as that is dealt
     with separately above. The action in all cases is to set an appropriate


@@ -2524,6 +2709,11 @@ DEBUG(D_transport)
 if (completed_address && ok && send_quit)
   {
   BOOL more;
 if (completed_address && ok && send_quit)
   {
   BOOL more;

+  smtp_compare_t t_compare;
+
+  t_compare.tblock = tblock;
+  t_compare.current_sender_address = sender_address;
+

   if (  first_addr != NULL
      || continue_more
      || (  (  tls_out.active < 0
   if (  first_addr != NULL
      || continue_more
      || (  (  tls_out.active < 0


@@ -2531,7 +2721,8 @@ if (completed_address && ok && send_quit)
           )
         &&
            transport_check_waiting(tblock->name, host->name,
           )
         &&
            transport_check_waiting(tblock->name, host->name,

-             tblock->connection_max_messages, new_message_id, &more)
+             tblock->connection_max_messages, new_message_id, &more,
+            (oicf)smtp_are_same_identities, (void*)&t_compare)

      )  )
     {
     uschar *msg;
      )  )
     {
     uschar *msg;


@@ -2728,21 +2919,21 @@ prepare_addresses(address_item *addrlist, host_item *host)
 address_item *first_addr = NULL;
 address_item *addr;
 for (addr = addrlist; addr != NULL; addr = addr->next)
 address_item *first_addr = NULL;
 address_item *addr;
 for (addr = addrlist; addr != NULL; addr = addr->next)

-  {
-  if (addr->transport_return != DEFER) continue;
-  if (first_addr == NULL) first_addr = addr;
-  addr->transport_return = PENDING_DEFER;
-  addr->basic_errno = 0;
-  addr->more_errno = (host->mx >= 0)? 'M' : 'A';
-  addr->message = NULL;
+  if (addr->transport_return == DEFER)
+    {
+    if (first_addr == NULL) first_addr = addr;
+    addr->transport_return = PENDING_DEFER;
+    addr->basic_errno = 0;
+    addr->more_errno = (host->mx >= 0)? 'M' : 'A';
+    addr->message = NULL;

 #ifdef SUPPORT_TLS
 #ifdef SUPPORT_TLS

-  addr->cipher = NULL;
-  addr->ourcert = NULL;
-  addr->peercert = NULL;
-  addr->peerdn = NULL;
-  addr->ocsp = OCSP_NOT_REQ;
+    addr->cipher = NULL;
+    addr->ourcert = NULL;
+    addr->peercert = NULL;
+    addr->peerdn = NULL;
+    addr->ocsp = OCSP_NOT_REQ;

 #endif
 #endif

-  }
+    }

 return first_addr;
 }
 
 return first_addr;
 }
 


@@ -2865,7 +3056,8 @@ if (hostlist == NULL || (ob->hosts_override && ob->hosts != NULL))
   /* This is not the first time this transport has been run in this delivery;
   the host list was built previously. */
 
   /* This is not the first time this transport has been run in this delivery;
   the host list was built previously. */
 

-  else hostlist = ob->hostlist;
+  else
+    hostlist = ob->hostlist;

   }
 
 /* The host list was supplied with the address. If hosts_randomize is set, we
   }
 
 /* The host list was supplied with the address. If hosts_randomize is set, we


@@ -2909,12 +3101,10 @@ else if (ob->hosts_randomize && hostlist->mx == MX_NONE && !continuing)
   hostlist = addrlist->host_list = newlist;
   }
 
   hostlist = addrlist->host_list = newlist;
   }
 

-

 /* Sort out the default port.  */
 
 if (!smtp_get_port(ob->port, addrlist, &port, tid)) return FALSE;
 
 /* Sort out the default port.  */
 
 if (!smtp_get_port(ob->port, addrlist, &port, tid)) return FALSE;
 

-

 /* For each host-plus-IP-address on the list:
 
 .  If this is a continued delivery and the host isn't the one with the
 /* For each host-plus-IP-address on the list:
 
 .  If this is a continued delivery and the host isn't the one with the


@@ -3011,7 +3201,6 @@ for (cutoff_retry = 0; expired &&
       {
       int new_port, flags;
       host_item *hh;
       {
       int new_port, flags;
       host_item *hh;

-      uschar *canonical_name;

 
       if (host->status >= hstatus_unusable)
         {
 
       if (host->status >= hstatus_unusable)
         {


@@ -3039,11 +3228,11 @@ for (cutoff_retry = 0; expired &&
       if (ob->dns_search_parents) flags |= HOST_FIND_SEARCH_PARENTS;
 
       if (ob->gethostbyname || string_is_ip_address(host->name, NULL) != 0)
       if (ob->dns_search_parents) flags |= HOST_FIND_SEARCH_PARENTS;
 
       if (ob->gethostbyname || string_is_ip_address(host->name, NULL) != 0)

-        rc = host_find_byname(host, NULL, flags, &canonical_name, TRUE);
+        rc = host_find_byname(host, NULL, flags, NULL, TRUE);

       else
         rc = host_find_bydns(host, NULL, flags, NULL, NULL, NULL,
       else
         rc = host_find_bydns(host, NULL, flags, NULL, NULL, NULL,

-         ob->dnssec_request_domains, ob->dnssec_require_domains,
-          &canonical_name, NULL);
+         &ob->dnssec,          /* domains for request/require */
+          NULL, NULL);

 
       /* Update the host (and any additional blocks, resulting from
       multihoming) with a host-specific port, if any. */
 
       /* Update the host (and any additional blocks, resulting from
       multihoming) with a host-specific port, if any. */


@@ -3119,7 +3308,8 @@ for (cutoff_retry = 0; expired &&
     doing a two-stage queue run, don't do this if forcing. */
 
     if ((!deliver_force || queue_2stage) && (queue_smtp ||
     doing a two-stage queue run, don't do this if forcing. */
 
     if ((!deliver_force || queue_2stage) && (queue_smtp ||

-        match_isinlist(addrlist->domain, &queue_smtp_domains, 0,
+        match_isinlist(addrlist->domain,
+         (const uschar **)&queue_smtp_domains, 0,

           &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL) == OK))
       {
       expired = FALSE;
           &domainlist_anchor, NULL, MCL_DOMAIN, TRUE, NULL) == OK))
       {
       expired = FALSE;


@@ -3503,16 +3693,12 @@ for (cutoff_retry = 0; expired &&
     case, see if any of them are deferred. */
 
     if (rc == OK)
     case, see if any of them are deferred. */
 
     if (rc == OK)

-      {
-      for (addr = addrlist; addr != NULL; addr = addr->next)
-        {
+      for (addr = addrlist; addr; addr = addr->next)

         if (addr->transport_return == DEFER)
           {
           some_deferred = TRUE;
           break;
           }
         if (addr->transport_return == DEFER)
           {
           some_deferred = TRUE;
           break;
           }

-        }
-      }

 
     /* If no addresses deferred or the result was ERROR, return. We do this for
     ERROR because a failing filter set-up or add_headers expansion is likely to
 
     /* If no addresses deferred or the result was ERROR, return. We do this for
     ERROR because a failing filter set-up or add_headers expansion is likely to