I18N: add a utf8_downconvert option to the smtp transport. Bug 2248

diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt
index d0e3358b8d9d7c040a667cdda06741646a245568..fb5944d959f93261b061a3a8e32f70f88380e744 100644 (file)
--- a/doc/doc-docbook/spec.xfpt
+++ b/doc/doc-docbook/spec.xfpt
@@ -24716,6 +24716,16 @@ The &%tls_verify_certificates%& option must also be set.
 If both this option and &%tls_try_verify_hosts%& are unset
 operation is as if this option selected all hosts.
 
+.new
+.option utf8_downconvert smtp integer!! unset
+.cindex utf8 "address downconversion"
+.cindex i18n "utf8 address downconversion"
+If built with internationalization support,
+this option controls conversion of UTF-8 in message addresses
+to a-label form.
+For details see section &<<SECTi18nMTA>>&.
+.wen
+
 
 
 
@@ -39863,6 +39873,12 @@ If a value is appended it may be:
 If mua_wrapper is set, the utf8_downconvert control
 is initially set to -1.
 
+.new
+The smtp transport has an option &%utf8_downconvert%&.
+If set it must expand to one of the three values described above,
+and it overrides any previously set value.
+.wen
+
 
 There is no explicit support for VRFY and EXPN.
 Configurations supporting these should inspect

diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff
index 4039895b658b6b4e257589d108db68b9196a6663..05c416ec740127b2a244f5c507fbed2ed61112dd 100644 (file)
--- a/doc/doc-txt/NewStuff
+++ b/doc/doc-txt/NewStuff
@@ -19,6 +19,8 @@ Version 4.92
 
  3. EXPERIMENTAL_REQUIRETLS.  See the experimental.spec file.
 
+ 4. If built with SUPPORT_I18N a "utf8_downconvert" option on the smtp transport.
+
 Version 4.91
 --------------
 

diff --git a/doc/doc-txt/OptionLists.txt b/doc/doc-txt/OptionLists.txt
index 05e47c0e425f09d5dc71981a6534ed3b38553d6e..c3ce62d4518b5a1fcba4301279a54a93e6e10afc 100644 (file)
--- a/doc/doc-txt/OptionLists.txt
+++ b/doc/doc-txt/OptionLists.txt
@@ -613,6 +613,7 @@ use_mbx_lock                         boolean         +             appendfile
 use_shell                            boolean         false         pipe              1.70
 user                                 string          +             routers           4.00
                                                      unset         transports        4.00 replaces individual options
+utf8_downcvt                        integer         unset         smtp              4.92 if SUPPORT_I18N
 uucp_from_pattern                    string          +             main              1.75
 uucp_from_sender                     string*         "$1"          main              1.75
 verify                               boolean         true          routers           4.00

diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index 08d1810d6990dc8e7bbf71f8c0485b05ae313816..b95913c59e5375c0cd8c13744da9fee1f60c523f 100644 (file)
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -190,15 +190,18 @@ optionlist smtp_transport_options[] = {
   { "tls_verify_certificates", opt_stringptr,
       (void *)offsetof(smtp_transport_options_block, tls_verify_certificates) },
   { "tls_verify_hosts",     opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, tls_verify_hosts) }
+      (void *)offsetof(smtp_transport_options_block, tls_verify_hosts) },
+#endif
+#ifdef SUPPORT_I18N
+  { "utf8_downconvert",            opt_stringptr,
+      (void *)offsetof(smtp_transport_options_block, utf8_downconvert) },
 #endif
 };
 
 /* Size of the options list. An extern variable has to be used so that its
 address can appear in the tables drtables.c. */
 
-int smtp_transport_options_count =
-  sizeof(smtp_transport_options)/sizeof(optionlist);
+int smtp_transport_options_count = nelem(smtp_transport_options);
 
 
 #ifdef MACRO_PREDEF
@@ -287,6 +290,9 @@ smtp_transport_options_block smtp_transport_option_defaults = {
   .tls_try_verify_hosts =      US"*",
   .tls_verify_cert_hostnames = US"*",
 #endif
+#ifdef SUPPORT_I18N
+  .utf8_downconvert =          NULL,
+#endif
 #ifndef DISABLE_DKIM
  .dkim =
    {.dkim_domain =             NULL,
@@ -2218,6 +2224,38 @@ sx->setting_up = FALSE;
 #ifdef SUPPORT_I18N
 if (sx->addrlist->prop.utf8_msg)
   {
+  uschar * s;
+
+  /* If the transport sets a downconversion mode it overrides any set by ACL
+  for the message. */
+
+  if ((s = sx->ob->utf8_downconvert))
+    {
+    if (!(s = expand_string(s)))
+      {
+      message = string_sprintf("failed to expand utf8_downconvert: %s",
+        expand_string_message);
+      set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE);
+      yield = DEFER;
+      goto SEND_QUIT;
+      }
+    switch (*s)
+      {
+      case '1':        sx->addrlist->prop.utf8_downcvt = TRUE;
+               sx->addrlist->prop.utf8_downcvt_maybe = FALSE;
+               break;
+      case '0':        sx->addrlist->prop.utf8_downcvt = FALSE;
+               sx->addrlist->prop.utf8_downcvt_maybe = FALSE;
+               break;
+      case '-':        if (s[1] == '1')
+                 {
+                 sx->addrlist->prop.utf8_downcvt = FALSE;
+                 sx->addrlist->prop.utf8_downcvt_maybe = TRUE;
+                 }
+               break;
+      }
+    }
+
   sx->utf8_needed = !sx->addrlist->prop.utf8_downcvt
                    && !sx->addrlist->prop.utf8_downcvt_maybe;
   DEBUG(D_transport) if (!sx->utf8_needed)
@@ -2231,7 +2269,7 @@ if (sx->utf8_needed && !(sx->peer_offered & OPTION_UTF8))
   errno = ERRNO_UTF8_FWD;
   goto RESPONSE_FAILED;
   }
-#endif
+#endif /*SUPPORT_I18N*/
 
 #if defined(SUPPORT_TLS) && defined(EXPERIMENTAL_REQUIRETLS)
   /*XXX should tls_requiretls actually be per-addr? */

diff --git a/src/src/transports/smtp.h b/src/src/transports/smtp.h
index 34c49d930a782bd68fddf645ce485cfa93610aa4..df9644377035bda18d69097537296900d36be077 100644 (file)
--- a/src/src/transports/smtp.h
+++ b/src/src/transports/smtp.h
@@ -88,6 +88,9 @@ typedef struct {
   uschar *tls_try_verify_hosts;
   uschar *tls_verify_cert_hostnames;
 #endif
+#ifdef SUPPORT_I18N
+  uschar *utf8_downconvert;
+#endif
 #ifndef DISABLE_DKIM
   struct ob_dkim dkim;
 #endif

diff --git a/test/confs/4201 b/test/confs/4201
index 9242eb063b3553372be960d025721ae6f78a77ae..17afe9f782fcf8f48f5e08477c6e5ddc893c5ac2 100644 (file)
--- a/test/confs/4201
+++ b/test/confs/4201
@@ -124,5 +124,8 @@ local_delivery:
 
 rmt_smtp:
   driver = smtp
+.ifdef STRICT
+  utf8_downconvert = STRICT
+.endif
 
 # End

diff --git a/test/log/4207 b/test/log/4207
index e39c0f5fb65c439e3fef686567ec9978d3b3d7c6..3bd01a3dd41ffd379b1a4f3f1ef4eaa16db1ed48 100644 (file)
--- a/test/log/4207
+++ b/test/log/4207
@@ -1,17 +1,23 @@
-1999-03-02 09:44:33 10HmaX-0005vi-00 <= 他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com U=CALLER P=utf8local-esmtp S=sss for user.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex
-1999-03-02 09:44:33 10HmaX-0005vi-00 => xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@test.ex <user.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex> F=<xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com> R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] C="250 OK id=10HmaY-0005vi-00"
+1999-03-02 09:44:33 10HmaX-0005vi-00 <= 他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com U=CALLER P=utf8local-esmtp S=sss for user1.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex
+1999-03-02 09:44:33 10HmaX-0005vi-00 => xn--user1.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex <user1.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex> F=<xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com> R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] C="250 OK id=10HmaY-0005vi-00"
 1999-03-02 09:44:33 10HmaX-0005vi-00 Completed
-1999-03-02 09:44:33 10HmaZ-0005vi-00 <= 他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com U=CALLER P=utf8local-esmtp S=sss for zuser.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex
-1999-03-02 09:44:33 10HmaZ-0005vi-00 => xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@test.ex <zuser.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex> F=<xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com> R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] C="250 OK id=10HmbA-0005vi-00"
+1999-03-02 09:44:33 10HmaZ-0005vi-00 <= 他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com U=CALLER P=utf8local-esmtp S=sss for zuser2.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex
+1999-03-02 09:44:33 10HmaZ-0005vi-00 => xn--user2.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex <zuser2.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex> F=<xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com> R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] C="250 OK id=10HmbA-0005vi-00"
 1999-03-02 09:44:33 10HmaZ-0005vi-00 Completed
+1999-03-02 09:44:33 10HmbB-0005vi-00 <= 他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com U=CALLER P=utf8local-esmtp S=sss for user3.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex
+1999-03-02 09:44:33 10HmbB-0005vi-00 => xn--user3.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex <user3.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex> F=<xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com> R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] C="250 OK id=10HmbC-0005vi-00"
+1999-03-02 09:44:33 10HmbB-0005vi-00 Completed
 
 ******** SERVER ********
 1999-03-02 09:44:33 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTP on port 1225
-1999-03-02 09:44:33 10HmaY-0005vi-00 <= xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com H=localhost (the.local.host.name) [127.0.0.1] P=esmtp S=sss id=E10HmaX-0005vi-00@the.local.host.name for xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@test.ex
-1999-03-02 09:44:33 10HmbA-0005vi-00 <= xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com H=localhost (the.local.host.name) [127.0.0.1] P=esmtp S=sss id=E10HmaZ-0005vi-00@the.local.host.name for xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@test.ex
+1999-03-02 09:44:33 10HmaY-0005vi-00 <= xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com H=localhost (the.local.host.name) [127.0.0.1] P=esmtp S=sss id=E10HmaX-0005vi-00@the.local.host.name for xn--user1.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex
+1999-03-02 09:44:33 10HmbA-0005vi-00 <= xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com H=localhost (the.local.host.name) [127.0.0.1] P=esmtp S=sss id=E10HmaZ-0005vi-00@the.local.host.name for xn--user2.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex
+1999-03-02 09:44:33 10HmbC-0005vi-00 <= xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com H=localhost (the.local.host.name) [127.0.0.1] P=esmtp S=sss id=E10HmbB-0005vi-00@the.local.host.name for xn--user3.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex
 1999-03-02 09:44:33 Start queue run: pid=pppp -qqff
-1999-03-02 09:44:33 10HmaY-0005vi-00 => :blackhole: <xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@test.ex> R=localuser
+1999-03-02 09:44:33 10HmaY-0005vi-00 => :blackhole: <xn--user1.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex> R=localuser
 1999-03-02 09:44:33 10HmaY-0005vi-00 Completed
-1999-03-02 09:44:33 10HmbA-0005vi-00 => :blackhole: <xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@test.ex> R=localuser
+1999-03-02 09:44:33 10HmbA-0005vi-00 => :blackhole: <xn--user2.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex> R=localuser
 1999-03-02 09:44:33 10HmbA-0005vi-00 Completed
+1999-03-02 09:44:33 10HmbC-0005vi-00 => :blackhole: <xn--user3.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex> R=localuser
+1999-03-02 09:44:33 10HmbC-0005vi-00 Completed
 1999-03-02 09:44:33 End queue run: pid=pppp -qqff

diff --git a/test/runtest b/test/runtest
index 9f556660d066a990c7d846d608cfdfdd3d611044..a38f112afe85a440313b1921735d32c51a2a0208 100755 (executable)
--- a/test/runtest
+++ b/test/runtest
@@ -1613,6 +1613,7 @@ $munges =
                   |hosts_(avoid|nopass|noproxy|require|verify_avoid)_tls
                   |socks_proxy
                   |tls_[^ ]*
+                 |utf8_downconvert
                   )($|[ ]=)/x'
     },
 

diff --git a/test/scripts/0000-Basic/0577 b/test/scripts/0000-Basic/0577
index 1bd510ae34c1e09eb4ab8d2cb8090408a25a35b8..bbd2954c7d5efebc1c82281b733014668414e3ef 100644 (file)
--- a/test/scripts/0000-Basic/0577
+++ b/test/scripts/0000-Basic/0577
@@ -1,2 +1,3 @@
 # Test different variants of .includes
+munge optional_config
 exim -bP config

diff --git a/test/scripts/4200-International/4207 b/test/scripts/4200-International/4207
index 061a2c6b36d291f4d89d81057cdae5a63866e3ca..fa32bec928dfd322fb767d6ce8767135169ca592 100644 (file)
--- a/test/scripts/4200-International/4207
+++ b/test/scripts/4200-International/4207
@@ -9,7 +9,7 @@ exim -DSERVER=server -DOPTION="" -bd -oX PORT_D
 exim -bs -odi -DCONTROL="control=utf8_downconvert"
 EHLO client.bh
 MAIL FROM: <他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com> SMTPUTF8
-RCPT TO: <user.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex>
+RCPT TO: <user1.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex>
 DATA
 Subject: test
 
@@ -22,7 +22,20 @@ QUIT
 exim -bs -odi -DCONTROL="control=utf8_downconvert/-1"
 EHLO client.bh
 MAIL FROM: <他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com> SMTPUTF8
-RCPT TO: <zuser.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex>
+RCPT TO: <zuser2.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex>
+DATA
+Subject: test
+
+body
+.
+QUIT
+****
+#
+# utf-8 from, never-downconvert set by ACL, overridden by transport
+exim -bs -odi -DCONTROL="control=utf8_downconvert/0" -DSTRICT=1
+EHLO client.bh
+MAIL FROM: <他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com> SMTPUTF8
+RCPT TO: <user3.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex>
 DATA
 Subject: test
 

diff --git a/test/stdout/0577 b/test/stdout/0577
index 80cf3da884032fcfb88356fda9c8bbf56688ba8f..ef918a86ba2fde04927cd0306b03911ffae45aa1 100644 (file)
--- a/test/stdout/0577
+++ b/test/stdout/0577
@@ -14,7 +14,6 @@ gecos_name = CALLER_NAME
 dns_cname_loops = 9
 chunking_advertise_hosts =
 # 1 "TESTSUITE/aux-var/std_conf_prefix"
-tls_advertise_hosts =
 # 1 "TESTSUITE/test-config"
 # 2 "TESTSUITE/test-config"
 # 1 "TESTSUITE/confs/0577./aaa"

diff --git a/test/stdout/4207 b/test/stdout/4207
index 948434ff3f2822923c02537949cf10f112e4a6c2..34a2cfe360a273fd5e47d5b6a8825ff8b2c9501e 100644 (file)
--- a/test/stdout/4207
+++ b/test/stdout/4207
@@ -22,3 +22,15 @@
 354 Enter message, ending with "." on a line by itself\r
 250 OK id=10HmaZ-0005vi-00\r
 221 the.local.host.name closing connection\r
+220 the.local.host.name ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000\r
+250-the.local.host.name Hello CALLER at client.bh\r
+250-SIZE 52428800\r
+250-8BITMIME\r
+250-PIPELINING\r
+250-SMTPUTF8\r
+250 HELP\r
+250 OK\r
+250 Accepted\r
+354 Enter message, ending with "." on a line by itself\r
+250 OK id=10HmbB-0005vi-00\r
+221 the.local.host.name closing connection\r