JSON: add jsons extract variant, to strip quotes from string results

diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt
index c40cf23f8395a2a9daafee8bcbdaaa0c603184f2..d3de8763f663f00d03c7a7b872f07d4324628c8c 100644 (file)
--- a/doc/doc-docbook/spec.xfpt
+++ b/doc/doc-docbook/spec.xfpt
@@ -9537,6 +9537,8 @@ This forces an expansion failure (see section &<<SECTforexpfai>>&);
 {<&'string2'&>} must be present for &"fail"& to be recognized.
 
 .vitem "&*${extract json{*&<&'key'&>&*}{*&<&'string1'&>&*}{*&<&'string2'&>&*}&&&
 {<&'string2'&>} must be present for &"fail"& to be recognized.
 
 .vitem "&*${extract json{*&<&'key'&>&*}{*&<&'string1'&>&*}{*&<&'string2'&>&*}&&&

+       {*&<&'string3'&>&*}}*&" &&&
+       "&*${extract jsons{*&<&'key'&>&*}{*&<&'string1'&>&*}{*&<&'string2'&>&*}&&&

        {*&<&'string3'&>&*}}*&"
 .cindex "expansion" "extracting from JSON object"
 .cindex JSON expansions
        {*&<&'string3'&>&*}}*&"
 .cindex "expansion" "extracting from JSON object"
 .cindex JSON expansions


@@ -9551,8 +9553,13 @@ The expanded <&'string1'&> must be of the form:
 The braces, commas and colons, and the quoting of the member name are required;
 the spaces are optional.
 Matching of the key against the member names is done case-sensitively.
 The braces, commas and colons, and the quoting of the member name are required;
 the spaces are optional.
 Matching of the key against the member names is done case-sensitively.

-If a returned value is a JSON string, it retains its leading and
+For the &"json"& variant,
+if a returned value is a JSON string, it retains its leading and

 trailing quotes.
 trailing quotes.

+.new
+For the &"jsons"& variant, which is intended for use with JSON strings, the
+leading and trailing quotes are removed.
+.wen

 . XXX should be a UTF-8 compare
 
 The results of matching are handled as above.
 . XXX should be a UTF-8 compare
 
 The results of matching are handled as above.


@@ -9590,6 +9597,8 @@ empty (for example, the fifth field above).
 
 
 .vitem "&*${extract json{*&<&'number'&>&*}}&&&
 
 
 .vitem "&*${extract json{*&<&'number'&>&*}}&&&

+        {*&<&'string1'&>&*}{*&<&'string2'&>&*}{*&<&'string3'&>&*}}*&" &&&
+       "&*${extract jsons{*&<&'number'&>&*}}&&&

         {*&<&'string1'&>&*}{*&<&'string2'&>&*}{*&<&'string3'&>&*}}*&"
 .cindex "expansion" "extracting from JSON array"
 .cindex JSON expansions
         {*&<&'string1'&>&*}{*&<&'string2'&>&*}{*&<&'string3'&>&*}}*&"
 .cindex "expansion" "extracting from JSON array"
 .cindex JSON expansions


@@ -9598,8 +9607,13 @@ apart from leading and trailing white space, which is ignored.
 
 Field selection and result handling is as above;
 there is no choice of field separator.
 
 Field selection and result handling is as above;
 there is no choice of field separator.

-If a returned value is a JSON string, it retains its leading and
+For the &"json"& variant,
+if a returned value is a JSON string, it retains its leading and

 trailing quotes.
 trailing quotes.

+.new
+For the &"jsons"& variant, which is intended for use with JSON strings, the
+leading and trailing quotes are removed.
+.wen

 
 
 .vitem &*${filter{*&<&'string'&>&*}{*&<&'condition'&>&*}}*&
 
 
 .vitem &*${filter{*&<&'string'&>&*}{*&<&'condition'&>&*}}*&

diff --git a/src/src/expand.c b/src/src/expand.c
index d36d376455787938db4e4d4f50c1e2b244fddfca..79304c8b1ab5ae8ec90a2ad856e79b76a7503d30 100644 (file)
--- a/src/src/expand.c
+++ b/src/src/expand.c
@@ -5635,7 +5635,13 @@ while (*s != 0)
       uschar *sub[3];
       int save_expand_nmax =
         save_expand_strings(save_expand_nstring, save_expand_nlength);
       uschar *sub[3];
       int save_expand_nmax =
         save_expand_strings(save_expand_nstring, save_expand_nlength);

-      enum {extract_basic, extract_json} fmt = extract_basic;
+
+      /* On reflection the original behaviour of extract-json for a string
+      result, leaving it quoted, was a mistake.  But it was already published,
+      hence the addition of jsons.  In a future major version, make json
+      work like josons, and withdraw jsons. */
+
+      enum {extract_basic, extract_json, extract_jsons} fmt = extract_basic;

 
       while (isspace(*s)) s++;
 
 
       while (isspace(*s)) s++;
 


@@ -5643,7 +5649,10 @@ while (*s != 0)
 
       if (*s != '{')                                   /*}*/
        if (Ustrncmp(s, "json", 4) == 0)
 
       if (*s != '{')                                   /*}*/
        if (Ustrncmp(s, "json", 4) == 0)

-         {fmt = extract_json; s += 4;}
+         if (*(s += 4) == 's')
+           {fmt = extract_jsons; s++;}
+         else
+           fmt = extract_json;

 
       /* While skipping we cannot rely on the data for expansions being
       available (eg. $item) hence cannot decide on numeric vs. keyed.
 
       /* While skipping we cannot rely on the data for expansions being
       available (eg. $item) hence cannot decide on numeric vs. keyed.


@@ -5724,7 +5733,7 @@ while (*s != 0)
            if (*p == 0)
              {
              field_number *= x;
            if (*p == 0)
              {
              field_number *= x;

-             if (fmt != extract_json) j = 3;               /* Need 3 args */
+             if (fmt == extract_basic) j = 3;               /* Need 3 args */

              field_number_set = TRUE;
              }
             }
              field_number_set = TRUE;
              }
             }


@@ -5751,6 +5760,7 @@ while (*s != 0)
          break;
 
        case extract_json:
          break;
 
        case extract_json:

+       case extract_jsons:

          {
          uschar * s, * item;
          const uschar * list;
          {
          uschar * s, * item;
          const uschar * list;


@@ -5817,6 +5827,17 @@ while (*s != 0)
              }
            }
          }
              }
            }
          }

+
+         if (  fmt == extract_jsons
+            && lookup_value
+            && !(lookup_value = dewrap(lookup_value, US"\"\"")))
+           {
+           expand_string_message =
+             string_sprintf("%s wrapping string result for extract jsons",
+               expand_string_message);
+           goto EXPAND_FAILED_CURLY;
+           }
+         break;        /* json/s */

        }
 
       /* If no string follows, $value gets substituted; otherwise there can
        }
 
       /* If no string follows, $value gets substituted; otherwise there can

diff --git a/test/scripts/0000-Basic/0002 b/test/scripts/0000-Basic/0002
index 7a9b38dba3ab1d2df5e602008876870d57598e16..5229f87e684272313e4b0fd548426a1cfefcaac7 100644 (file)
--- a/test/scripts/0000-Basic/0002
+++ b/test/scripts/0000-Basic/0002
@@ -893,6 +893,11 @@ ${extract json {Width} \
 ${extract json {2} {[116, 943, 234, 38793]} }
 ${extract json {2} {${extract json{IDs} {\{"other":"foo", "IDs": [116, 943, 234]\} }}} }
 
 ${extract json {2} {[116, 943, 234, 38793]} }
 ${extract json {2} {${extract json{IDs} {\{"other":"foo", "IDs": [116, 943, 234]\} }}} }
 

+${extract json {2} {["red", "green", "blue", "black"]} }
+${extract jsons{2} {["red", "green", "blue", "black"]} }
+<${extract jsons{5} {["red", "green", "blue", "black"]} }>
+expect: <>
+

 ${extract json {seconds} { \{"hours":0, "mins":0, "seconds":59\} }}
 ${extract json {seconds} {${extract json {2} { ["irrelevant", \{"hours":0, "mins":0, "seconds":59\}] }}}}
 
 ${extract json {seconds} { \{"hours":0, "mins":0, "seconds":59\} }}
 ${extract json {seconds} {${extract json {2} { ["irrelevant", \{"hours":0, "mins":0, "seconds":59\}] }}}}
 


@@ -904,6 +909,8 @@ expect: {"1":116, "2":943, "3":234}
 
 <${extract json{nonexistent}{ \{"id": \{"a":101, "b":102\}, "IDs": \{"1":116, "2":943, "3":234\}\} }}>
 expect: <>
 
 <${extract json{nonexistent}{ \{"id": \{"a":101, "b":102\}, "IDs": \{"1":116, "2":943, "3":234\}\} }}>
 expect: <>

+<${extract jsons{nonexistent}{ \{"id": \{"a":101, "b":102\}, "IDs": \{"1":116, "2":943, "3":234\}\} }}>
+expect: <>

 
 ****
 # Test "escape" with print_topbitchars
 
 ****
 # Test "escape" with print_topbitchars

diff --git a/test/stdout/0002 b/test/stdout/0002
index c2f5f2f3c99c648dcb8cd99730b237406bcf4330..df3e3ea88b46bd169b038a944d450f67240f544d 100644 (file)
--- a/test/stdout/0002
+++ b/test/stdout/0002
@@ -836,6 +836,11 @@ xyz
 > 943
 > 943
 > 
 > 943
 > 943
 > 

+> "green"
+> green
+> <>
+> expect: <>
+> 

 > 59
 > 59
 > 
 > 59
 > 59
 > 


@@ -847,6 +852,8 @@ xyz
 > 
 > <>
 > expect: <>
 > 
 > <>
 > expect: <>

+> <>
+> expect: <>

 > 
 > 
 > escape: B7·F2ò
 > 
 > 
 > escape: B7·F2ò