UTF8: MSA downconversions

diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff
index e4bc586a5ce33116d64c29e022a9f0032695fc05..ef6a6dea97da954843f9062562034e66752c1ccb 100644 (file)
--- a/doc/doc-txt/NewStuff
+++ b/doc/doc-txt/NewStuff
@@ -23,7 +23,10 @@ Version 4.86
  6. A commandline option to write a comment into the logfile.
 
  7. If built with EXPERIMENTAL_SOCKS feature enabled, the smtp transport can
-    be configured to make connections via socks5 proxies
+    be configured to make connections via socks5 proxies.
+
+ 8. If built with EXPERIMENTAL_INTERNATIONAL, support is included for
+    the transmission of UTF-8 envelope addresses.
 
 
 Version 4.85

diff --git a/doc/doc-txt/experimental-spec.txt b/doc/doc-txt/experimental-spec.txt
index 819b47fd57fa2351de72c2fdcb6b4baf9a5c2c0d..234bac0ab3e16503d2cb3108b3006473f33a51cc 100644 (file)
--- a/doc/doc-txt/experimental-spec.txt
+++ b/doc/doc-txt/experimental-spec.txt
@@ -1308,12 +1308,22 @@ New expansion operators:
        ${utf8_localpart_to_alabel:str}
        ${utf8_localpart_from_alabel:str}
 
+New "control = utf8_downconvert" ACL modifier,
+sets a flag requiring that addresses are converted to
+a-label form before smtp delivery, for use in a
+Message Submission Agent context.  Can also be
+phrased as "control = utf8_downconvert/1" and is
+mandatory.  The flag defaults to zero and can be cleared
+by "control = utf8_downconvert/0".  The value "-1"
+may also be used, to use a-label for only if the
+destination host does not support SMTPUTF8.
+
 Known issues:
  - Currently LMTP is not supported.
  - DSN unitext handling is not present
  - no provision for converting logging from UTF-8
  - VRFY and EXPN not handled
- - MSA mode not handled (!)
+ - mua_wrapper not handled
 
 --------------------------------------------------------------
 End of file

diff --git a/src/src/acl.c b/src/src/acl.c
index 3f513c3286d48b1fa4c06b6e8d08516170ffdf24..c1402a0ffe5c04c436071d77500d89d8678221ad 100644 (file)
--- a/src/src/acl.c
+++ b/src/src/acl.c
@@ -181,17 +181,17 @@ that follows! */
 
 enum {
   CONTROL_AUTH_UNADVERTISED,
-  #ifdef EXPERIMENTAL_BRIGHTMAIL
+#ifdef EXPERIMENTAL_BRIGHTMAIL
   CONTROL_BMI_RUN,
-  #endif
+#endif
   CONTROL_DEBUG,
-  #ifndef DISABLE_DKIM
+#ifndef DISABLE_DKIM
   CONTROL_DKIM_VERIFY,
-  #endif
-  #ifdef EXPERIMENTAL_DMARC
+#endif
+#ifdef EXPERIMENTAL_DMARC
   CONTROL_DMARC_VERIFY,
   CONTROL_DMARC_FORENSIC,
-  #endif
+#endif
   CONTROL_DSCP,
   CONTROL_ERROR,
   CONTROL_CASEFUL_LOCAL_PART,
@@ -203,11 +203,14 @@ enum {
   CONTROL_QUEUE_ONLY,
   CONTROL_SUBMISSION,
   CONTROL_SUPPRESS_LOCAL_FIXUPS,
-  #ifdef WITH_CONTENT_SCAN
+#ifdef WITH_CONTENT_SCAN
   CONTROL_NO_MBOX_UNSPOOL,
-  #endif
+#endif
   CONTROL_FAKEDEFER,
   CONTROL_FAKEREJECT,
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  CONTROL_UTF8_DOWNCONVERT,
+#endif
   CONTROL_NO_MULTILINE,
   CONTROL_NO_PIPELINING,
   CONTROL_NO_DELAY_FLUSH,
@@ -221,17 +224,17 @@ and should be tidied up. */
 
 static uschar *controls[] = {
   US"allow_auth_unadvertised",
-  #ifdef EXPERIMENTAL_BRIGHTMAIL
+#ifdef EXPERIMENTAL_BRIGHTMAIL
   US"bmi_run",
-  #endif
+#endif
   US"debug",
-  #ifndef DISABLE_DKIM
+#ifndef DISABLE_DKIM
   US"dkim_disable_verify",
-  #endif
-  #ifdef EXPERIMENTAL_DMARC
+#endif
+#ifdef EXPERIMENTAL_DMARC
   US"dmarc_disable_verify",
   US"dmarc_enable_forensic",
-  #endif
+#endif
   US"dscp",
   US"error",
   US"caseful_local_part",
@@ -243,11 +246,14 @@ static uschar *controls[] = {
   US"queue_only",
   US"submission",
   US"suppress_local_fixups",
-  #ifdef WITH_CONTENT_SCAN
+#ifdef WITH_CONTENT_SCAN
   US"no_mbox_unspool",
-  #endif
+#endif
   US"fakedefer",
   US"fakereject",
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  US"utf8_downconvert",
+#endif
   US"no_multiline_responses",
   US"no_pipelining",
   US"no_delay_flush",
@@ -600,26 +606,26 @@ static unsigned int control_forbids[] = {
   (unsigned int)
   ~((1<<ACL_WHERE_CONNECT)|(1<<ACL_WHERE_HELO)),   /* allow_auth_unadvertised */
 
-  #ifdef EXPERIMENTAL_BRIGHTMAIL
+#ifdef EXPERIMENTAL_BRIGHTMAIL
   0,                                               /* bmi_run */
-  #endif
+#endif
 
   0,                                               /* debug */
 
-  #ifndef DISABLE_DKIM
+#ifndef DISABLE_DKIM
   (1<<ACL_WHERE_DATA)|(1<<ACL_WHERE_NOTSMTP)|      /* dkim_disable_verify */
-  #ifndef DISABLE_PRDR
+# ifndef DISABLE_PRDR
     (1<<ACL_WHERE_PRDR)|
-  #endif
+# endif
     (1<<ACL_WHERE_NOTSMTP_START),
-  #endif
+#endif
 
-  #ifdef EXPERIMENTAL_DMARC
+#ifdef EXPERIMENTAL_DMARC
   (1<<ACL_WHERE_DATA)|(1<<ACL_WHERE_NOTSMTP)|      /* dmarc_disable_verify */
     (1<<ACL_WHERE_NOTSMTP_START),
   (1<<ACL_WHERE_DATA)|(1<<ACL_WHERE_NOTSMTP)|      /* dmarc_enable_forensic */
     (1<<ACL_WHERE_NOTSMTP_START),
-  #endif
+#endif
 
   (1<<ACL_WHERE_NOTSMTP)|
     (1<<ACL_WHERE_NOTSMTP_START)|
@@ -663,30 +669,34 @@ static unsigned int control_forbids[] = {
     (1<<ACL_WHERE_PREDATA)|
     (1<<ACL_WHERE_NOTSMTP_START)),
 
-  #ifdef WITH_CONTENT_SCAN
+#ifdef WITH_CONTENT_SCAN
   (unsigned int)
   ~((1<<ACL_WHERE_MAIL)|(1<<ACL_WHERE_RCPT)|       /* no_mbox_unspool */
     (1<<ACL_WHERE_PREDATA)|(1<<ACL_WHERE_DATA)|
     // (1<<ACL_WHERE_PRDR)|    /* Not allow one user to freeze for all */
     (1<<ACL_WHERE_MIME)),
-  #endif
+#endif
 
   (unsigned int)
   ~((1<<ACL_WHERE_MAIL)|(1<<ACL_WHERE_RCPT)|       /* fakedefer */
     (1<<ACL_WHERE_PREDATA)|(1<<ACL_WHERE_DATA)|
-  #ifndef DISABLE_PRDR
+#ifndef DISABLE_PRDR
     (1<<ACL_WHERE_PRDR)|
-  #endif
+#endif
     (1<<ACL_WHERE_MIME)),
 
   (unsigned int)
   ~((1<<ACL_WHERE_MAIL)|(1<<ACL_WHERE_RCPT)|       /* fakereject */
     (1<<ACL_WHERE_PREDATA)|(1<<ACL_WHERE_DATA)|
-  #ifndef DISABLE_PRDR
+#ifndef DISABLE_PRDR
     (1<<ACL_WHERE_PRDR)|
-  #endif
+#endif
     (1<<ACL_WHERE_MIME)),
 
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  0,                                               /* utf8_downconvert */
+#endif
+
   (1<<ACL_WHERE_NOTSMTP)|                          /* no_multiline */
     (1<<ACL_WHERE_NOTSMTP_START),
 
@@ -709,37 +719,40 @@ typedef struct control_def {
 } control_def;
 
 static control_def controls_list[] = {
-  { US"allow_auth_unadvertised", CONTROL_AUTH_UNADVERTISED, FALSE },
+  { US"allow_auth_unadvertised", CONTROL_AUTH_UNADVERTISED,     FALSE },
 #ifdef EXPERIMENTAL_BRIGHTMAIL
-  { US"bmi_run",                 CONTROL_BMI_RUN, FALSE },
+  { US"bmi_run",                 CONTROL_BMI_RUN,               FALSE },
 #endif
-  { US"debug",                   CONTROL_DEBUG, TRUE },
+  { US"debug",                   CONTROL_DEBUG,                 TRUE },
 #ifndef DISABLE_DKIM
-  { US"dkim_disable_verify",     CONTROL_DKIM_VERIFY, FALSE },
+  { US"dkim_disable_verify",     CONTROL_DKIM_VERIFY,           FALSE },
 #endif
 #ifdef EXPERIMENTAL_DMARC
-  { US"dmarc_disable_verify",    CONTROL_DMARC_VERIFY, FALSE },
-  { US"dmarc_enable_forensic",   CONTROL_DMARC_FORENSIC, FALSE },
+  { US"dmarc_disable_verify",    CONTROL_DMARC_VERIFY,          FALSE },
+  { US"dmarc_enable_forensic",   CONTROL_DMARC_FORENSIC,        FALSE },
 #endif
-  { US"dscp",                    CONTROL_DSCP, TRUE },
-  { US"caseful_local_part",      CONTROL_CASEFUL_LOCAL_PART, FALSE },
-  { US"caselower_local_part",    CONTROL_CASELOWER_LOCAL_PART, FALSE },
-  { US"enforce_sync",            CONTROL_ENFORCE_SYNC, FALSE },
-  { US"freeze",                  CONTROL_FREEZE, TRUE },
-  { US"no_callout_flush",        CONTROL_NO_CALLOUT_FLUSH, FALSE },
-  { US"no_delay_flush",          CONTROL_NO_DELAY_FLUSH, FALSE },
-  { US"no_enforce_sync",         CONTROL_NO_ENFORCE_SYNC, FALSE },
-  { US"no_multiline_responses",  CONTROL_NO_MULTILINE, FALSE },
-  { US"no_pipelining",           CONTROL_NO_PIPELINING, FALSE },
-  { US"queue_only",              CONTROL_QUEUE_ONLY, FALSE },
+  { US"dscp",                    CONTROL_DSCP,                  TRUE },
+  { US"caseful_local_part",      CONTROL_CASEFUL_LOCAL_PART,    FALSE },
+  { US"caselower_local_part",    CONTROL_CASELOWER_LOCAL_PART,  FALSE },
+  { US"enforce_sync",            CONTROL_ENFORCE_SYNC,          FALSE },
+  { US"freeze",                  CONTROL_FREEZE,                TRUE },
+  { US"no_callout_flush",        CONTROL_NO_CALLOUT_FLUSH,      FALSE },
+  { US"no_delay_flush",          CONTROL_NO_DELAY_FLUSH,        FALSE },
+  { US"no_enforce_sync",         CONTROL_NO_ENFORCE_SYNC,       FALSE },
+  { US"no_multiline_responses",  CONTROL_NO_MULTILINE,          FALSE },
+  { US"no_pipelining",           CONTROL_NO_PIPELINING,         FALSE },
+  { US"queue_only",              CONTROL_QUEUE_ONLY,            FALSE },
 #ifdef WITH_CONTENT_SCAN
-  { US"no_mbox_unspool",         CONTROL_NO_MBOX_UNSPOOL, FALSE },
+  { US"no_mbox_unspool",         CONTROL_NO_MBOX_UNSPOOL,       FALSE },
 #endif
-  { US"fakedefer",               CONTROL_FAKEDEFER, TRUE },
-  { US"fakereject",              CONTROL_FAKEREJECT, TRUE },
-  { US"submission",              CONTROL_SUBMISSION, TRUE },
+  { US"fakedefer",               CONTROL_FAKEDEFER,             TRUE },
+  { US"fakereject",              CONTROL_FAKEREJECT,            TRUE },
+  { US"submission",              CONTROL_SUBMISSION,            TRUE },
   { US"suppress_local_fixups",   CONTROL_SUPPRESS_LOCAL_FIXUPS, FALSE },
-  { US"cutthrough_delivery",     CONTROL_CUTTHROUGH_DELIVERY, FALSE }
+  { US"cutthrough_delivery",     CONTROL_CUTTHROUGH_DELIVERY,   FALSE },
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  { US"utf8_downconvert",        CONTROL_UTF8_DOWNCONVERT,      TRUE }
+#endif
   };
 
 /* Support data structures for Client SMTP Authorization. acl_verify_csa()
@@ -2080,7 +2093,11 @@ else if (verify_sender_address != NULL)
 
     sender_vaddr = deliver_make_addr(verify_sender_address, TRUE);
 #ifdef EXPERIMENTAL_INTERNATIONAL
-    sender_vaddr->prop.utf8 = message_smtputf8;
+    if ((sender_vaddr->prop.utf8_msg = message_smtputf8))
+      {
+      sender_vaddr->prop.utf8_downcvt =       message_utf8_downconvert == 1;
+      sender_vaddr->prop.utf8_downcvt_maybe = message_utf8_downconvert == -1;
+      }
 #endif
     if (no_details) setflag(sender_vaddr, af_sverify_told);
     if (verify_sender_address[0] != 0)
@@ -3377,6 +3394,24 @@ for (; cb != NULL; cb = cb->next)
                                        arg, *log_msgptr);
          }
        return ERROR;
+
+    #ifdef EXPERIMENTAL_INTERNATIONAL
+       case CONTROL_UTF8_DOWNCONVERT:
+       if (*p == '/')
+         {
+         if (p[1] == '1') { message_utf8_downconvert = 1; p += 2; break; }
+         if (p[1] == '0') { message_utf8_downconvert = 0; p += 2; break; }
+         if (p[1] == '-' && p[2] == '1')
+                          { message_utf8_downconvert = -1; p += 3; break; }
+         *log_msgptr = US"bad option value for control=utf8_downconvert";
+         }
+       else
+         {
+         message_utf8_downconvert = 1; break;
+         }
+       return ERROR;
+    #endif
+
        }
       break;
       }
@@ -3390,14 +3425,9 @@ for (; cb != NULL; cb = cb->next)
       /* Run the dcc backend. */
       rc = dcc_process(&ss);
       /* Modify return code based upon the existance of options. */
-      while ((ss = string_nextinlist(&list, &sep, big_buffer, big_buffer_size))
-            != NULL) {
+      while ((ss = string_nextinlist(&list, &sep, big_buffer, big_buffer_size)))
         if (strcmpic(ss, US"defer_ok") == 0 && rc == DEFER)
-          {
-          /* FAIL so that the message is passed to the next ACL */
-          rc = FAIL;
-          }
-        }
+          rc = FAIL;   /* FAIL so that the message is passed to the next ACL */
       }
     break;
     #endif
@@ -4382,7 +4412,11 @@ if (where == ACL_WHERE_RCPT)
     return DEFER;
     }
 #ifdef EXPERIMENTAL_INTERNATIONAL
-  addr->prop.utf8 = message_smtputf8;
+  if ((addr->prop.utf8_msg = message_smtputf8))
+    {
+    addr->prop.utf8_downcvt =       message_utf8_downconvert == 1;
+    addr->prop.utf8_downcvt_maybe = message_utf8_downconvert == -1;
+    }
 #endif
   deliver_domain = addr->domain;
   deliver_localpart = addr->local_part;

diff --git a/src/src/deliver.c b/src/src/deliver.c
index f208f3eddcd984a84b96caca0d5add080625c7d3..58b9d3a01f4ad4405abe03e4260bdd350f80ad8b 100644 (file)
--- a/src/src/deliver.c
+++ b/src/src/deliver.c
@@ -822,7 +822,14 @@ if (log_extra_selector & LX_incoming_interface  &&  sending_ip_address)
   /* for the port:  string_sprintf("%d", sending_port) */
 
 if ((log_extra_selector & LX_sender_on_delivery) != 0  ||  msg)
-  s = string_append(s, &size, &ptr, 3, US" F=<", sender_address, US">");
+  s = string_append(s, &size, &ptr, 3, US" F=<",
+#ifdef EXPERIMENTAL_INTERNATIONAL
+    testflag(addr, af_utf8_downcvt)
+    ? string_address_utf8_to_alabel(sender_address, NULL)
+    :
+#endif
+      sender_address,
+  US">");
 
 #ifdef EXPERIMENTAL_SRS
 if(addr->prop.srs_sender)
@@ -5595,7 +5602,11 @@ if (process_recipients != RECIP_IGNORE)
       address_item *new = deliver_make_addr(r->address, FALSE);
       new->prop.errors_address = r->errors_to;
 #ifdef EXPERIMENTAL_INTERNATIONAL
-      new->prop.utf8 = message_smtputf8;
+      if ((new->prop.utf8_msg = message_smtputf8))
+       {
+       new->prop.utf8_downcvt =       message_utf8_downconvert == 1;
+       new->prop.utf8_downcvt_maybe = message_utf8_downconvert == -1;
+       }
       DEBUG(D_deliver) if (message_smtputf8) debug_printf("utf8\n");
 #endif
 

diff --git a/src/src/functions.h b/src/src/functions.h
index d1ada38443f5900a3c64a8beb87c75cd8e6e22c8..1708b7a076e7ad8a81ac4561fdd703b7db9fba8b 100644 (file)
--- a/src/src/functions.h
+++ b/src/src/functions.h
@@ -422,8 +422,7 @@ extern const uschar *string_printing2(const uschar *, BOOL);
 extern uschar *string_split_message(uschar *);
 extern uschar *string_unprinting(uschar *);
 #ifdef EXPERIMENTAL_INTERNATIONAL
-extern uschar *string_address_alabel_to_utf8(const uschar *, uschar **);
-extern uschar *string_address_utf8_to_alabel(uschar *, uschar **, int *);
+extern uschar *string_address_utf8_to_alabel(const uschar *, uschar **);
 extern uschar *string_domain_alabel_to_utf8(const uschar *, uschar **);
 extern uschar *string_domain_utf8_to_alabel(const uschar *, uschar **);
 extern uschar *string_localpart_alabel_to_utf8(const uschar *, uschar **);

diff --git a/src/src/globals.c b/src/src/globals.c
index 2bf4d0a02fca999d8e8ab7ba08897dbff4fdebc0..a71c80ed90f4913b05cc17f3094fe19879d9514c 100644 (file)
--- a/src/src/globals.c
+++ b/src/src/globals.c
@@ -915,6 +915,7 @@ int     message_size           = 0;
 uschar *message_size_limit     = US"50M";
 #ifdef EXPERIMENTAL_INTERNATIONAL
 BOOL    message_smtputf8       = FALSE;
+int     message_utf8_downconvert = 0;  /* -1 ifneeded; 0 never; 1 always */
 #endif
 uschar  message_subdir[2]      = { 0, 0 };
 uschar *message_reference      = NULL;

diff --git a/src/src/globals.h b/src/src/globals.h
index 7cbf7bfab3dd05bf79de446a1d6a302ec96971e6..8aa69bff69560d430c86e56662653b3fe295b842 100644 (file)
--- a/src/src/globals.h
+++ b/src/src/globals.h
@@ -572,6 +572,7 @@ extern int     message_size;           /* Size of message */
 extern uschar *message_size_limit;     /* As it says */
 #ifdef EXPERIMENTAL_INTERNATIONAL
 extern BOOL    message_smtputf8;       /* Internationalized mail handling */
+extern int     message_utf8_downconvert; /* convert from utf8 */
 const extern pcre *regex_UTF8;         /* For recognizing SMTPUTF8 settings */
 #endif
 extern uschar  message_subdir[];       /* Subdirectory for messages */

diff --git a/src/src/routers/redirect.c b/src/src/routers/redirect.c
index 8f1c2c3d29aeebfc61ab4746bf1df655dc39d668..9e57c8be65572888d3a2c8f1c366c7957f04344c 100644 (file)
--- a/src/src/routers/redirect.c
+++ b/src/src/routers/redirect.c
@@ -452,7 +452,7 @@ while (generated != NULL)
     }
 
 #ifdef EXPERIMENTAL_INTERNATIONAL
-    next->prop.utf8 = string_is_utf8(next->address)
+    next->prop.utf8_msg = string_is_utf8(next->address)
       || (sender_address && string_is_utf8(sender_address));
 #endif
 
@@ -476,7 +476,7 @@ while (generated != NULL)
       debug_printf("gid=unset ");
 
 #ifdef EXPERIMENTAL_INTERNATIONAL
-    if (next->prop.utf8) debug_printf("utf8 ");
+    if (next->prop.utf8_msg) debug_printf("utf8 ");
 #endif
 
     debug_printf("home=%s\n", next->home_dir);

diff --git a/src/src/spool_in.c b/src/src/spool_in.c
index 742f4b579e582c3bb30bce24ff8e41b56efb86a0..558d955c1dfeff212f4b960bad5b90f4e2a21515 100644 (file)
--- a/src/src/spool_in.c
+++ b/src/src/spool_in.c
@@ -301,6 +301,7 @@ spam_score_int = NULL;
 
 #if defined(EXPERIMENTAL_INTERNATIONAL) && !defined(COMPILE_UTILITY)
 message_smtputf8 = FALSE;
+message_utf8_downconvert = 0;
 #endif
 
 dsn_ret = 0;
@@ -600,6 +601,15 @@ for (;;)
     break;
 #endif
 
+#if defined(EXPERIMENTAL_INTERNATIONAL) && !defined(COMPILE_UTILITY)
+    case 'u':
+    if (Ustrncmp(p, "tf8_downcvt", 11) == 0)
+      message_utf8_downconvert = 1;
+    else if (Ustrncmp(p, "tf8_downcvt_opt", 15) == 0)
+      message_utf8_downconvert = -1;
+    break;
+#endif
+
     default:    /* Present because some compilers complain if all */
     break;      /* possibilities are not covered. */
     }

diff --git a/src/src/spool_out.c b/src/src/spool_out.c
index 6d22bff2c8f9df721d40b6712ddbe0d1e1aca2a1..48f27a8c6951fe9e0e9aa57c243e085d9a3557d4 100644 (file)
--- a/src/src/spool_out.c
+++ b/src/src/spool_out.c
@@ -246,7 +246,12 @@ if (tls_in.ocsp)    fprintf(f, "-tls_ocsp %d\n",   tls_in.ocsp);
 #endif
 
 #ifdef EXPERIMENTAL_INTERNATIONAL
-if (message_smtputf8)    fprintf(f, "-smtputf8\n");
+if (message_smtputf8)
+  {
+  fprintf(f, "-smtputf8\n");
+  if (message_utf8_downconvert)
+    fprintf(f, "-utf8_downcvt%s\n", message_utf8_downconvert < 0 ? "_opt" : "");
+  }
 #endif
 
 /* Write the dsn flags to the spool header file */
@@ -508,3 +513,5 @@ return TRUE;
 #endif
 
 /* End of spool_out.c */
+/* vi: aw ai sw=2
+*/

diff --git a/src/src/string.c b/src/src/string.c
index e169a9f057076ff42995949f7b87f76ee2f9eff6..c50a347c2f94f50258378dff7455e07bcef6656e 100644 (file)
--- a/src/src/string.c
+++ b/src/src/string.c
@@ -1547,14 +1547,35 @@ static uschar *
 string_get_localpart(address_item *addr, uschar *yield, int *sizeptr,
   int *ptrptr)
 {
-if (testflag(addr, af_include_affixes) && addr->prefix != NULL)
-  yield = string_cat(yield, sizeptr, ptrptr, addr->prefix,
-    Ustrlen(addr->prefix));
-yield = string_cat(yield, sizeptr, ptrptr, addr->local_part,
-  Ustrlen(addr->local_part));
-if (testflag(addr, af_include_affixes) && addr->suffix != NULL)
-  yield = string_cat(yield, sizeptr, ptrptr, addr->suffix,
-    Ustrlen(addr->suffix));
+uschar * s;
+
+s = addr->prefix;
+if (testflag(addr, af_include_affixes) && s)
+  {
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  if (testflag(addr, af_utf8_downcvt))
+    s = string_localpart_utf8_to_alabel(s, NULL);
+#endif
+  yield = string_cat(yield, sizeptr, ptrptr, s, Ustrlen(s));
+  }
+
+s = addr->local_part;
+#ifdef EXPERIMENTAL_INTERNATIONAL
+if (testflag(addr, af_utf8_downcvt))
+  s = string_localpart_utf8_to_alabel(s, NULL);
+#endif
+yield = string_cat(yield, sizeptr, ptrptr, s, Ustrlen(s));
+
+s = addr->suffix;
+if (testflag(addr, af_include_affixes) && s)
+  {
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  if (testflag(addr, af_utf8_downcvt))
+    s = string_localpart_utf8_to_alabel(s, NULL);
+#endif
+  yield = string_cat(yield, sizeptr, ptrptr, s, Ustrlen(s));
+  }
+
 return yield;
 }
 
@@ -1615,10 +1636,15 @@ else
   {
   if (addr->local_part != NULL)
     {
+    const uschar * s;
     yield = string_get_localpart(addr, yield, &size, &ptr);
     yield = string_cat(yield, &size, &ptr, US"@", 1);
-    yield = string_cat(yield, &size, &ptr, addr->domain,
-      Ustrlen(addr->domain) );
+    s = addr->domain;
+#ifdef EXPERIMENTAL_INTERNATIONAL
+    if (testflag(addr, af_utf8_downcvt))
+      s = string_localpart_utf8_to_alabel(s, NULL);
+#endif
+    yield = string_cat(yield, &size, &ptr, s, Ustrlen(s) );
     }
   else
     {

diff --git a/src/src/structs.h b/src/src/structs.h
index 99d65cfae5bc193ec3dd9285194418ee246e4a0a..c181f3f6ed0a9db0bf68c9f205dc691b6a9e2c48 100644 (file)
--- a/src/src/structs.h
+++ b/src/src/structs.h
@@ -460,7 +460,9 @@ typedef struct address_item_propagated {
   uschar *srs_sender;             /* Change return path when delivering */
   #endif
   #ifdef EXPERIMENTAL_INTERNATIONAL
-  BOOL    utf8;                          /* requires SMTPUTF8 processing */
+  BOOL    utf8_msg:1;            /* requires SMTPUTF8 processing */
+  BOOL   utf8_downcvt:1;         /* mandatory downconvert on delivery */
+  BOOL   utf8_downcvt_maybe:1;   /* optional downconvert on delivery */
   #endif
 } address_item_propagated;
 
@@ -500,6 +502,9 @@ typedef struct address_item_propagated {
 #ifdef EXPERIMENTAL_DANE
 # define af_dane_verified      0x20000000 /* TLS cert verify done with DANE */
 #endif
+#ifdef EXPERIMENTAL_INTERNATIONAL
+# define af_utf8_downcvt       0x40000000 /* downconvert was done for delivery */
+#endif
 
 /* These flags must be propagated when a child is created */
 

diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index 65bb1de223d533ec5e87638567c61ca731b421d9..7537e6e4b7f64b84c1b31090fca491e633285406 100644 (file)
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -1366,6 +1366,7 @@ BOOL prdr_offered = FALSE;
 BOOL prdr_active;
 #endif
 #ifdef EXPERIMENTAL_INTERNATIONAL
+BOOL utf8_needed = FALSE;
 BOOL utf8_offered = FALSE;
 #endif
 BOOL dsn_all_lasthop = TRUE;
@@ -1642,10 +1643,17 @@ goto SEND_QUIT;
 #endif
 
 #ifdef EXPERIMENTAL_INTERNATIONAL
-  utf8_offered = esmtp
-    && addrlist->prop.utf8
-    && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0,
-                 PCRE_EOPT, NULL, 0) >= 0;
+  if (addrlist->prop.utf8_msg)
+    {
+    utf8_needed =  !addrlist->prop.utf8_downcvt
+               && !addrlist->prop.utf8_downcvt_maybe;
+    DEBUG(D_transport) if (!utf8_needed) debug_printf("utf8: %s downconvert\n",
+      addrlist->prop.utf8_downcvt ? "mandatory" : "optional");
+
+    utf8_offered = esmtp
+      && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0,
+                   PCRE_EOPT, NULL, 0) >= 0;
+    }
 #endif
   }
 
@@ -1862,10 +1870,10 @@ if (continue_hostname == NULL
 #endif
 
 #ifdef EXPERIMENTAL_INTERNATIONAL
-  utf8_offered = esmtp
-    && addrlist->prop.utf8
-    && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0,
-                 PCRE_EOPT, NULL, 0) >= 0;
+  if (addrlist->prop.utf8_msg)
+    utf8_offered = esmtp
+      && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0,
+                   PCRE_EOPT, NULL, 0) >= 0;
 #endif
 
   /* Note if the server supports DSN */
@@ -1896,7 +1904,7 @@ setting_up = FALSE;
 
 #ifdef EXPERIMENTAL_INTERNATIONAL
 /* If this is an international message we need the host to speak SMTPUTF8 */
-if (addrlist->prop.utf8 && !utf8_offered)
+if (utf8_needed && !utf8_offered)
   {
   errno = ERRNO_UTF8_FWD;
   goto RESPONSE_FAILED;
@@ -1980,7 +1988,7 @@ if (prdr_offered)
 #endif
 
 #ifdef EXPERIMENTAL_INTERNATIONAL
-if (addrlist->prop.utf8)
+if (addrlist->prop.utf8_msg && !addrlist->prop.utf8_downcvt && utf8_offered)
   sprintf(CS p, " SMTPUTF8"), p += 9;
 #endif
 
@@ -2037,8 +2045,31 @@ buffer. */
 
 pending_MAIL = TRUE;     /* The block starts with MAIL */
 
-rc = smtp_write_command(&outblock, smtp_use_pipelining,
-       "MAIL FROM:<%s>%s\r\n", return_path, buffer);
+  {
+  uschar * s = return_path;
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  uschar * errstr = NULL;
+
+  /* If we must downconvert, do the from-address here.  Remember we had to
+  for the to-addresses (done below), and also (ugly) for re-doing when building
+  the delivery log line. */
+
+  if (addrlist->prop.utf8_msg && (addrlist->prop.utf8_downcvt || !utf8_offered))
+    {
+    if (s = string_address_utf8_to_alabel(return_path, &errstr), errstr)
+      {
+      set_errno(addrlist, ERRNO_EXPANDFAIL, errstr, DEFER, FALSE, NULL);
+      yield = ERROR;
+      goto SEND_QUIT;
+      }
+    setflag(addrlist, af_utf8_downcvt);
+    }
+#endif
+
+  rc = smtp_write_command(&outblock, smtp_use_pipelining,
+         "MAIL FROM:<%s>%s\r\n", s, buffer);
+  }
+
 mail_command = string_copy(big_buffer);  /* Save for later error message */
 
 switch(rc)
@@ -2080,6 +2111,7 @@ for (addr = first_addr;
   {
   int count;
   BOOL no_flush;
+  uschar * rcpt_addr;
 
   addr->dsn_aware = smtp_use_dsn ? dsn_support_yes : dsn_support_no;
 
@@ -2124,8 +2156,24 @@ for (addr = first_addr;
   yield as OK, because this error can often mean that there is a problem with
   just one address, so we don't want to delay the host. */
 
+  rcpt_addr = transport_rcpt_address(addr, tblock->rcpt_include_affixes);
+
+#ifdef EXPERIMENTAL_INTERNATIONAL
+  {
+  uschar * dummy_errstr;
+  if (  testflag(addrlist, af_utf8_downcvt)
+     && (rcpt_addr = string_address_utf8_to_alabel(rcpt_addr, &dummy_errstr),
+        dummy_errstr
+     )  )
+    {
+    errno = ERRNO_EXPANDFAIL;
+    goto SEND_FAILED;
+    }
+  }
+#endif
+
   count = smtp_write_command(&outblock, no_flush, "RCPT TO:<%s>%s%s\r\n",
-    transport_rcpt_address(addr, tblock->rcpt_include_affixes), igquotstr, buffer);
+    rcpt_addr, igquotstr, buffer);
 
   if (count < 0) goto SEND_FAILED;
   if (count > 0)

diff --git a/src/src/utf8.c b/src/src/utf8.c
index 6bc0c2ed5a0813929c248d430fa0de7d45bdcef2..09ebdf128a082f7eac2e0f3450bd2e8cdb127885 100644 (file)
--- a/src/src/utf8.c
+++ b/src/src/utf8.c
@@ -24,6 +24,7 @@ return FALSE;
 
 /**************************************************/
 /* Domain conversions */
+/* the *err string pointer should be null before the call */
 
 uschar *
 string_domain_utf8_to_alabel(const uschar * utf8, uschar ** err)
@@ -68,6 +69,7 @@ return s;
 
 /**************************************************/
 /* localpart conversions */
+/* the *err string pointer should be null before the call */
 
 
 uschar *
@@ -126,6 +128,31 @@ return res;
 }
 
 
+/**************************************************/
+/* whole address conversion */
+/* the *err string pointer should be null before the call */
+
+uschar *
+string_address_utf8_to_alabel(const uschar * utf8, uschar ** err)
+{
+const uschar * s;
+uschar * l;
+uschar * d;
+
+for (s = utf8; *s; s++)
+  if (*s == '@')
+    {
+    l = string_copyn(utf8, s - utf8);
+    return   (l = string_localpart_utf8_to_alabel(l, err), err && *err)
+         || (d = string_domain_utf8_to_alabel(++s, err),  err && *err)
+      ? NULL
+      : string_sprintf("%s@%s", l, d);
+    }
+return string_localpart_utf8_to_alabel(utf8, err);
+}
+
+
+
 /*************************************************
 *         Report the library versions.           *
 *************************************************/

diff --git a/src/src/verify.c b/src/src/verify.c
index 4e9b563fae19b87505ab9963d8cf2d7456a380e2..28013fa35c152da357f17b643392f653b8c8b03d 100644 (file)
--- a/src/src/verify.c
+++ b/src/src/verify.c
@@ -173,6 +173,9 @@ dbdata_callout_cache new_domain_record;
 dbdata_callout_cache_address new_address_record;
 host_item *host;
 time_t callout_start_time;
+#ifdef EXPERIMENTAL_INTERNATIONAL
+BOOL utf8_offered = FALSE;
+#endif
 
 new_domain_record.result = ccache_unknown;
 new_domain_record.postmaster_result = ccache_unknown;
@@ -921,22 +924,35 @@ can do it there for the non-rcpt-verify case.  For this we keep an addresscount.
       }
 
 #ifdef EXPERIMENTAL_INTERNATIONAL
-    else if (  addr->prop.utf8
+    else if (  addr->prop.utf8_msg
+           && !addr->prop.utf8_downcvt
            && !(  esmtp
                && (  regex_UTF8
                   || ( (regex_UTF8 = regex_must_compile(
                          US"\\n250[\\s\\-]SMTPUTF8(\\s|\\n|$)", FALSE, TRUE)),
                      TRUE
                   )  )
-               && pcre_exec(regex_UTF8, NULL, CS responsebuffer,
-                   Ustrlen(responsebuffer), 0, PCRE_EOPT, NULL, 0) >= 0
-           )   )
+               && (  (utf8_offered = pcre_exec(regex_UTF8, NULL,
+                           CS responsebuffer, Ustrlen(responsebuffer),
+                           0, PCRE_EOPT, NULL, 0) >= 0)
+                  || addr->prop.utf8_downcvt_maybe
+           )   )  )
       {
       HDEBUG(D_acl|D_v) debug_printf("utf8 required but not offered\n");
       errno = ERRNO_UTF8_FWD;
       setflag(addr, af_verify_nsfail);
       done = FALSE;
       }
+    else if (  addr->prop.utf8_msg
+           && (addr->prop.utf8_downcvt || !utf8_offered)
+           && (from_address = string_address_utf8_to_alabel(from_address,
+                                     &addr->message), addr->message)
+           )
+      {
+      errno = ERRNO_EXPANDFAIL;
+      setflag(addr, af_verify_nsfail);
+      done = FALSE;
+      }
 #endif
 
     /* If we haven't authenticated, but are required to, give up. */
@@ -958,7 +974,7 @@ can do it there for the non-rcpt-verify case.  For this we keep an addresscount.
     /* Send the MAIL command */
         (smtp_write_command(&outblock, FALSE,
 #ifdef EXPERIMENTAL_INTERNATIONAL
-         addr->prop.utf8
+         addr->prop.utf8_msg
          ? "MAIL FROM:<%s>%s SMTPUTF8\r\n"
          :
 #endif
@@ -1049,7 +1065,7 @@ can do it there for the non-rcpt-verify case.  For this we keep an addresscount.
 
             smtp_write_command(&outblock, FALSE,
 #ifdef EXPERIMENTAL_INTERNATIONAL
-             addr->prop.utf8
+             addr->prop.utf8_msg
              ? "MAIL FROM:<%s> SMTPUTF8\r\n"
              :
 #endif

diff --git a/test/confs/4201 b/test/confs/4201
index b34c7c1f9ff9f5c2aa7e14cf218cb67908709235..b1fb7a6ee473701d7b8a2f925a47f1dfc552ae5e 100644 (file)
--- a/test/confs/4201
+++ b/test/confs/4201
@@ -39,7 +39,7 @@ begin acl
 check_recipient:
   accept hosts = :
   accept domains = +local_domains
-        local_parts = ^user.*\$
+        local_parts = ^(xn--)?user.*\$
   deny   message = relay not permitted
 
 .else

diff --git a/test/confs/4207 b/test/confs/4207
new file mode 120000 (symlink)
index 0000000..73a348f
--- /dev/null
+++ b/test/confs/4207
@@ -0,0 +1 @@
+4201
\ No newline at end of file

diff --git a/test/log/4207 b/test/log/4207
new file mode 100644 (file)
index 0000000..e1dd4f7
--- /dev/null
+++ b/test/log/4207
@@ -0,0 +1,9 @@
+1999-03-02 09:44:33 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTP on port 1225
+1999-03-02 09:44:33 10HmaX-0005vi-00 <= 他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com U=CALLER P=utf8local-esmtp S=sss for user.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex
+1999-03-02 09:44:33 10HmaY-0005vi-00 <= xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com H=localhost (the.local.host.name) [127.0.0.1] P=esmtp S=sss id=E10HmaX-0005vi-00@the.local.host.name for xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@test.ex
+1999-03-02 09:44:33 10HmaX-0005vi-00 => xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@xn--test.ex- <user.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex> F=<xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com> R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] C="250 OK id=10HmaY-0005vi-00"
+1999-03-02 09:44:33 10HmaX-0005vi-00 Completed
+1999-03-02 09:44:33 Start queue run: pid=pppp -qqff
+1999-03-02 09:44:33 10HmaY-0005vi-00 => :blackhole: <xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@test.ex> R=localuser
+1999-03-02 09:44:33 10HmaY-0005vi-00 Completed
+1999-03-02 09:44:33 End queue run: pid=pppp -qqff

diff --git a/test/scripts/4200-International/4207 b/test/scripts/4200-International/4207
new file mode 100644 (file)
index 0000000..66e77f0
--- /dev/null
+++ b/test/scripts/4200-International/4207
@@ -0,0 +1,24 @@
+# Internationalised mail: control = utf8_downconvert
+# Exim test configuration 4207
+#
+exim -DSERVER=server -bd -oX PORT_D
+****
+#
+#
+# utf-8 from, mandatory downconvert
+exim -bs -odi -DCONTROL="control=utf8_downconvert"
+EHLO client.bh
+MAIL FROM: <他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com> SMTPUTF8
+RCPT TO: <user.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex>
+DATA
+Subject: test
+
+body
+.
+QUIT
+****
+#
+killdaemon
+exim -DSERVER=server -qqff
+****
+no_msglog_check

diff --git a/test/stderr/4207 b/test/stderr/4207
new file mode 100644 (file)
index 0000000..045fadc
--- /dev/null
+++ b/test/stderr/4207
@@ -0,0 +1,2 @@
+
+******** SERVER ********

diff --git a/test/stdout/4207 b/test/stdout/4207
new file mode 100644 (file)
index 0000000..94a4e18
--- /dev/null
+++ b/test/stdout/4207
@@ -0,0 +1,12 @@
+220 the.local.host.name ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000\r
+250-the.local.host.name Hello CALLER at client.bh\r
+250-SIZE 52428800\r
+250-8BITMIME\r
+250-PIPELINING\r
+250-SMTPUTF8\r
+250 HELP\r
+250 OK\r
+250 Accepted\r
+354 Enter message, ending with "." on a line by itself\r
+250 OK id=10HmaX-0005vi-00\r
+221 the.local.host.name closing connection\r