From: Jeremy Harris Date: Sun, 10 Feb 2019 23:50:39 +0000 (+0000) Subject: JSON: add iterative conditions for arrays X-Git-Url: https://git.exim.org/users/jgh/exim.git/commitdiff_plain/7f69e814219268610c9d5c9b724f64a17a78b2cb?ds=sidebyside JSON: add iterative conditions for arrays (cherry picked from commit c5c57c4eafde32a0632c2a00bdc634860fc5d06d) --- diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt index d3de8763f..22f06e3c1 100644 --- a/doc/doc-docbook/spec.xfpt +++ b/doc/doc-docbook/spec.xfpt @@ -9558,7 +9558,7 @@ if a returned value is a JSON string, it retains its leading and trailing quotes. .new For the &"jsons"& variant, which is intended for use with JSON strings, the -leading and trailing quotes are removed. +leading and trailing quotes are removed from the returned value. .wen . XXX should be a UTF-8 compare @@ -9596,7 +9596,7 @@ yields &"99"&. Two successive separators mean that the field between them is empty (for example, the fifth field above). -.vitem "&*${extract json{*&<&'number'&>&*}}&&& +.vitem "&*${extract json {*&<&'number'&>&*}}&&& {*&<&'string1'&>&*}{*&<&'string2'&>&*}{*&<&'string3'&>&*}}*&" &&& "&*${extract jsons{*&<&'number'&>&*}}&&& {*&<&'string1'&>&*}{*&<&'string2'&>&*}{*&<&'string3'&>&*}}*&" @@ -9612,7 +9612,7 @@ if a returned value is a JSON string, it retains its leading and trailing quotes. .new For the &"jsons"& variant, which is intended for use with JSON strings, the -leading and trailing quotes are removed. +leading and trailing quotes are removed from the returned value. .wen @@ -11341,6 +11341,25 @@ being processed, to enable these expansion items to be nested. To scan a named list, expand it with the &*listnamed*& operator. +.new +.vitem "&*forall_json{*&<&'a JSON array'&>&*}{*&<&'a condition'&>&*}*&" &&& + "&*forany_json{*&<&'a JSON array'&>&*}{*&<&'a condition'&>&*}*&" &&& + "&*forall_jsons{*&<&'a JSON array'&>&*}{*&<&'a condition'&>&*}*&" &&& + "&*forany_jsons{*&<&'a JSON array'&>&*}{*&<&'a condition'&>&*}*&" +.cindex JSON "iterative conditions" +.cindex JSON expansions +.cindex expansion "&*forall_json*& condition" +.cindex expansion "&*forany_json*& condition" +.cindex expansion "&*forall_jsons*& condition" +.cindex expansion "&*forany_jsons*& condition" +As for the above, except that the first argument must, after expansion, +be a JSON array. +The array separator is not changeable. +For the &"jsons"& variants the elements are expected to be JSON strings +and have their quotes removed before the evaluation of the condition. +.wen + + .vitem &*ge&~{*&<&'string1'&>&*}{*&<&'string2'&>&*}*& &&& &*gei&~{*&<&'string1'&>&*}{*&<&'string2'&>&*}*& diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff index 8dc3648f5..3b5cda15c 100644 --- a/doc/doc-txt/NewStuff +++ b/doc/doc-txt/NewStuff @@ -11,7 +11,7 @@ Version 4.93 1. An "external" authenticator, per RFC 4422 Appendix A. - 2. A JSON lookup type. + 2. A JSON lookup type, and JSON variants of the forall/any expansion conditions. Version 4.92 -------------- diff --git a/src/src/expand.c b/src/src/expand.c index 79304c8b1..dc0912d03 100644 --- a/src/src/expand.c +++ b/src/src/expand.c @@ -312,7 +312,11 @@ static uschar *cond_table[] = { US"exists", US"first_delivery", US"forall", + US"forall_json", + US"forall_jsons", US"forany", + US"forany_json", + US"forany_jsons", US"ge", US"gei", US"gt", @@ -358,7 +362,11 @@ enum { ECOND_EXISTS, ECOND_FIRST_DELIVERY, ECOND_FORALL, + ECOND_FORALL_JSON, + ECOND_FORALL_JSONS, ECOND_FORANY, + ECOND_FORANY_JSON, + ECOND_FORANY_JSONS, ECOND_STR_GE, ECOND_STR_GEI, ECOND_STR_GT, @@ -2144,6 +2152,89 @@ return ret; +/* Return pointer to dewrapped string, with enclosing specified chars removed. +The given string is modified on return. Leading whitespace is skipped while +looking for the opening wrap character, then the rest is scanned for the trailing +(non-escaped) wrap character. A backslash in the string will act as an escape. + +A nul is written over the trailing wrap, and a pointer to the char after the +leading wrap is returned. + +Arguments: + s String for de-wrapping + wrap Two-char string, the first being the opener, second the closer wrapping + character +Return: + Pointer to de-wrapped string, or NULL on error (with expand_string_message set). +*/ + +static uschar * +dewrap(uschar * s, const uschar * wrap) +{ +uschar * p = s; +unsigned depth = 0; +BOOL quotesmode = wrap[0] == wrap[1]; + +while (isspace(*p)) p++; + +if (*p == *wrap) + { + s = ++p; + wrap++; + while (*p) + { + if (*p == '\\') p++; + else if (!quotesmode && *p == wrap[-1]) depth++; + else if (*p == *wrap) + if (depth == 0) + { + *p = '\0'; + return s; + } + else + depth--; + p++; + } + } +expand_string_message = string_sprintf("missing '%c'", *wrap); +return NULL; +} + + +/* Pull off the leading array or object element, returning +a copy in an allocated string. Update the list pointer. + +The element may itself be an abject or array. +Return NULL when the list is empty. +*/ + +static uschar * +json_nextinlist(const uschar ** list) +{ +unsigned array_depth = 0, object_depth = 0; +const uschar * s = *list, * item; + +while (isspace(*s)) s++; + +for (item = s; + *s && (*s != ',' || array_depth != 0 || object_depth != 0); + s++) + switch (*s) + { + case '[': array_depth++; break; + case ']': array_depth--; break; + case '{': object_depth++; break; + case '}': object_depth--; break; + } +*list = *s ? s+1 : s; +if (item == s) return NULL; +item = string_copyn(item, s - item); +DEBUG(D_expand) debug_printf_indent(" json ele: '%s'\n", item); +return US item; +} + + + /************************************************* * Read and evaluate a condition * *************************************************/ @@ -2170,6 +2261,7 @@ BOOL testfor = TRUE; BOOL tempcond, combined_cond; BOOL *subcondptr; BOOL sub2_honour_dollar = TRUE; +BOOL is_forany, is_json, is_jsons; int rc, cond_type, roffset; int_eximarith_t num[2]; struct stat statbuf; @@ -2945,8 +3037,14 @@ switch(cond_type) /* forall/forany: iterates a condition with different values */ - case ECOND_FORALL: - case ECOND_FORANY: + case ECOND_FORALL: is_forany = FALSE; is_json = FALSE; is_jsons = FALSE; goto FORMANY; + case ECOND_FORANY: is_forany = TRUE; is_json = FALSE; is_jsons = FALSE; goto FORMANY; + case ECOND_FORALL_JSON: is_forany = FALSE; is_json = TRUE; is_jsons = FALSE; goto FORMANY; + case ECOND_FORANY_JSON: is_forany = TRUE; is_json = TRUE; is_jsons = FALSE; goto FORMANY; + case ECOND_FORALL_JSONS: is_forany = FALSE; is_json = TRUE; is_jsons = TRUE; goto FORMANY; + case ECOND_FORANY_JSONS: is_forany = TRUE; is_json = TRUE; is_jsons = TRUE; goto FORMANY; + + FORMANY: { const uschar * list; int sep = 0; @@ -2987,10 +3085,22 @@ switch(cond_type) return NULL; } - if (yield != NULL) *yield = !testfor; + if (yield) *yield = !testfor; list = sub[0]; - while ((iterate_item = string_nextinlist(&list, &sep, NULL, 0)) != NULL) + if (is_json) list = dewrap(string_copy(list), US"[]"); + while ((iterate_item = is_json + ? json_nextinlist(&list) : string_nextinlist(&list, &sep, NULL, 0))) { + if (is_jsons) + if (!(iterate_item = dewrap(iterate_item, US"\"\""))) + { + expand_string_message = + string_sprintf("%s wrapping string result for extract jsons", + expand_string_message); + iterate_item = save_iterate_item; + return NULL; + } + DEBUG(D_expand) debug_printf_indent("%s: $item = \"%s\"\n", name, iterate_item); if (!eval_condition(sub[1], resetok, &tempcond)) { @@ -3002,8 +3112,8 @@ switch(cond_type) DEBUG(D_expand) debug_printf_indent("%s: condition evaluated to %s\n", name, tempcond? "true":"false"); - if (yield != NULL) *yield = (tempcond == testfor); - if (tempcond == (cond_type == ECOND_FORANY)) break; + if (yield) *yield = (tempcond == testfor); + if (tempcond == is_forany) break; } iterate_item = save_iterate_item; @@ -3841,89 +3951,6 @@ return x; -/* Return pointer to dewrapped string, with enclosing specified chars removed. -The given string is modified on return. Leading whitespace is skipped while -looking for the opening wrap character, then the rest is scanned for the trailing -(non-escaped) wrap character. A backslash in the string will act as an escape. - -A nul is written over the trailing wrap, and a pointer to the char after the -leading wrap is returned. - -Arguments: - s String for de-wrapping - wrap Two-char string, the first being the opener, second the closer wrapping - character -Return: - Pointer to de-wrapped string, or NULL on error (with expand_string_message set). -*/ - -static uschar * -dewrap(uschar * s, const uschar * wrap) -{ -uschar * p = s; -unsigned depth = 0; -BOOL quotesmode = wrap[0] == wrap[1]; - -while (isspace(*p)) p++; - -if (*p == *wrap) - { - s = ++p; - wrap++; - while (*p) - { - if (*p == '\\') p++; - else if (!quotesmode && *p == wrap[-1]) depth++; - else if (*p == *wrap) - if (depth == 0) - { - *p = '\0'; - return s; - } - else - depth--; - p++; - } - } -expand_string_message = string_sprintf("missing '%c'", *wrap); -return NULL; -} - - -/* Pull off the leading array or object element, returning -a copy in an allocated string. Update the list pointer. - -The element may itself be an object or array. -Return NULL when the list is empty. -*/ - -uschar * -json_nextinlist(const uschar ** list) -{ -unsigned array_depth = 0, object_depth = 0; -const uschar * s = *list, * item; - -while (isspace(*s)) s++; - -for (item = s; - *s && (*s != ',' || array_depth != 0 || object_depth != 0); - s++) - switch (*s) - { - case '[': array_depth++; break; - case ']': array_depth--; break; - case '{': object_depth++; break; - case '}': object_depth--; break; - } -*list = *s ? s+1 : s; -if (item == s) return NULL; -item = string_copyn(item, s - item); -DEBUG(D_expand) debug_printf_indent(" json ele: '%s'\n", item); -return US item; -} - - - /************************************************* * Expand string * *************************************************/ diff --git a/test/scripts/0000-Basic/0002 b/test/scripts/0000-Basic/0002 index 5229f87e6..65ad69047 100644 --- a/test/scripts/0000-Basic/0002 +++ b/test/scripts/0000-Basic/0002 @@ -912,6 +912,9 @@ expect: <> <${extract jsons{nonexistent}{ \{"id": \{"a":101, "b":102\}, "IDs": \{"1":116, "2":943, "3":234\}\} }}> expect: <> +${if forany_json {[1, 2, 3]}{={$item}{1}}{yes}{no}} +${if forany_jsons{["A", "B", "C"]}{eq{$item}{B}}{yes}{no}} + **** # Test "escape" with print_topbitchars exim -be -DPTBC=print_topbitchars diff --git a/test/stdout/0002 b/test/stdout/0002 index df3e3ea88..f5a97289a 100644 --- a/test/stdout/0002 +++ b/test/stdout/0002 @@ -855,6 +855,9 @@ xyz > <> > expect: <> > +> yes +> yes +> > > escape: B7·F2ò >