Fix utf8clean not replacing incomplete final character

Before, it was just dropped, but we document that it's replaced by ?.

Tests updated, manual test-case for -be prompt is:

    ${utf8clean:${length_1:フィル}}

diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog
index bf5afe0af08b9669d16aba1837cdc16b159928b3..bc521a6a3bf70bf7d142575508963b48bbf3c5f3 100644 (file)
--- a/doc/doc-txt/ChangeLog
+++ b/doc/doc-txt/ChangeLog
@@ -92,6 +92,10 @@ JH/20 Bug 2296: Fix cutthrough for >1 address redirection.  Previously only
       time.  Either a crash (after delivery) or bogus log data could result.
       Discovery and analysis by Tim Stewart.
 
+PP/03 Make ${utf8clean:} expansion operator detect incomplete final character.
+      Previously if the string ended mid-character, we did not insert the
+      promised '?' replacement.
+
 
 Exim version 4.91
 -----------------

diff --git a/src/src/expand.c b/src/src/expand.c
index b6ff96aee11831dba1bae80554636bcc48a688d6..1508dcfd8cfaef914e44ffaaf2cb5fb58ce903a8 100644 (file)
--- a/src/src/expand.c
+++ b/src/src/expand.c
@@ -1683,7 +1683,7 @@ else if (host_lookup_deferred)
   g = string_catn(g, US";\n\tiprev=temperror", 19);
 else if (host_lookup_failed)
   g = string_catn(g, US";\n\tiprev=fail", 13);
-else 
+else
   return g;
 
 if (sender_host_address)
@@ -7143,12 +7143,13 @@ while (*s != 0)
         {
         int seq_len = 0, index = 0;
         int bytes_left = 0;
-       long codepoint = -1;
+        long codepoint = -1;
+        int complete;
         uschar seq_buff[4];                    /* accumulate utf-8 here */
 
         while (*sub != 0)
          {
-         int complete = 0;
+         complete = 0;
          uschar c = *sub++;
 
          if (bytes_left)
@@ -7213,6 +7214,13 @@ while (*s != 0)
                        /* ASCII character follows incomplete sequence */
              yield = string_catn(yield, &c, 1);
          }
+        /* If given a sequence truncated mid-character, we also want to report ?
+        * Eg, ${length_1:フィル} is one byte, not one character, so we expect
+        * ${utf8clean:${length_1:フィル}} to yield '?' */
+        if (bytes_left != 0)
+          {
+          yield = string_catn(yield, UTF8_REPLACEMENT_CHAR, 1);
+          }
         continue;
         }
 
@@ -7954,7 +7962,7 @@ expand_hide_passwords(uschar * s)
 {
 return (  (  Ustrstr(s, "failed to expand") != NULL
          || Ustrstr(s, "expansion of ")    != NULL
-         ) 
+         )
        && (  Ustrstr(s, "mysql")   != NULL
          || Ustrstr(s, "pgsql")   != NULL
          || Ustrstr(s, "redis")   != NULL
@@ -7964,7 +7972,7 @@ return (  (  Ustrstr(s, "failed to expand") != NULL
          || Ustrstr(s, "ldapi:")  != NULL
          || Ustrstr(s, "ldapdn:") != NULL
          || Ustrstr(s, "ldapm:")  != NULL
-       )  ) 
+       )  )
   ? US"Temporary internal error" : s;
 }
 

diff --git a/test/confs/0600 b/test/confs/0600
index 29bc5bf1a7818f623f70d9f30f39bc55bcef0630..f6405460d7db0149532a6fe8756ad77bb0e9f31a 100644 (file)
--- a/test/confs/0600
+++ b/test/confs/0600
@@ -20,15 +20,17 @@ begin acl
 
 check_data:
   accept logwrite = \
-                       x-test-header-good1: ${utf8clean:$h_x-test-header-good1:}
-                logwrite = \
-                       x-test-header-good2: ${utf8clean:$h_x-test-header-good2:}
-                logwrite = \
-                       x-test-header-too-short: ${utf8clean:$h_x-test-header-too-short:}
-                logwrite = \
-                       x-test-header-too-long: ${utf8clean:$h_x-test-header-too-long:}
-                logwrite = \
-                       x-test-header-too-big: ${utf8clean:$h_x-test-header-too-big:}
+                       x-test-header-good1: ${utf8clean:$h_x-test-header-good1:}
+                logwrite = \
+                       x-test-header-good2: ${utf8clean:$h_x-test-header-good2:}
+                logwrite = \
+                       x-test-header-too-short: ${utf8clean:$h_x-test-header-too-short:}
+                logwrite = \
+                       x-test-header-too-long: ${utf8clean:$h_x-test-header-too-long:}
+                logwrite = \
+                       x-test-header-too-big: ${utf8clean:$h_x-test-header-too-big:}
+                log_write = \
+                       x-test-header-truncated: ${utf8clean:$h_x-test-header-truncated:}
 
 
 

diff --git a/test/log/0600 b/test/log/0600
index 8fc8cfc36800776e0181a0bec16b713b2197ec8e..a18b159d2d1de231f73e85f6fa45139692798d3b 100644 (file)
--- a/test/log/0600
+++ b/test/log/0600
@@ -3,14 +3,16 @@
 1999-03-02 09:44:33 10HmaX-0005vi-00 x-test-header-too-short: ?.?.?.\303\244-?.-\303\234.?..?.-?.-?..-?.-?.-?.-?.-?..-?..?.
 1999-03-02 09:44:33 10HmaX-0005vi-00 x-test-header-too-long: ?????-\303\244-?????--\303\226-\303\204-\302\264-\340\244\221-\340\244\225-\340\244\234-\341\220\201-\341\221\214-\341\221\225-?????\360\253\237\206
 1999-03-02 09:44:33 10HmaX-0005vi-00 x-test-header-too-big: ?-----\363\200\200\200
+1999-03-02 09:44:33 10HmaX-0005vi-00 x-test-header-truncated: ?
 1999-03-02 09:44:33 10HmaX-0005vi-00 <= CALLER@the.local.host.name U=CALLER P=local-smtp S=sss
 1999-03-02 09:44:33 10HmaX-0005vi-00 => CALLER <CALLER@the.local.host.name> R=localuser T=local_delivery
 1999-03-02 09:44:33 10HmaX-0005vi-00 Completed
-1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-good1: 
-1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-good2: 
-1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-too-short: 
-1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-too-long: 
-1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-too-big: 
+1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-good1:
+1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-good2:
+1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-too-short:
+1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-too-long:
+1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-too-big:
+1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-too-truncated:
 1999-03-02 09:44:33 10HmaY-0005vi-00 <= CALLER@the.local.host.name U=CALLER P=local-smtp S=sss
 1999-03-02 09:44:33 10HmaY-0005vi-00 => CALLER <CALLER@the.local.host.name> R=localuser T=local_delivery
 1999-03-02 09:44:33 10HmaY-0005vi-00 Completed

diff --git a/test/scripts/0000-Basic/0600 b/test/scripts/0000-Basic/0600
index 9d5e67b5b984a678fb32e3f3f9b8a41d5d72652e..43f7288225849a3c5f6585d522063fb630cd31d9 100644 (file)
--- a/test/scripts/0000-Basic/0600
+++ b/test/scripts/0000-Basic/0600
@@ -10,6 +10,7 @@ x-test-header-good2: ßüöä€ÜÖÄ´ऑकजᐁᑌᑕ𫝆𫟘
 x-test-header-too-short: Ã.Ã.Ã.ä-â\82.-Ã\9c.Ã..Ã.-Â.-à..-à¤.-à¤.-á\90.-á\91.-á..-ð«\9d..ð«\9f.
 x-test-header-too-long: ø\88\88\88\88-ä-ø\88\88\88\88--Ã\96-Ã\84-´-à¤\91-à¤\95-à¤\9c-á\90\81-á\91\8c-á\91\95-ø\80\80\80\80ð«\9f\86
 x-test-header-too-big: ����-----󀀀
+x-test-header-truncated: ã
 Subject: This is a test message.
 
 This is a test message.