X-Git-Url: https://git.exim.org/exim.git/blobdiff_plain/87e9d061c94e3fdd721b7b04ccbdba7a061f6ca3..ae8f9024d8d4fad31457d758022e3186d782929c:/src/src/expand.c diff --git a/src/src/expand.c b/src/src/expand.c index a1ac7d198..ca7d36838 100644 --- a/src/src/expand.c +++ b/src/src/expand.c @@ -462,6 +462,8 @@ static var_entry var_table[] = { { "address_file", vtype_stringptr, &address_file }, { "address_pipe", vtype_stringptr, &address_pipe }, #ifdef EXPERIMENTAL_ARC + { "arc_domains", vtype_string_func, &fn_arc_domains }, + { "arc_oldest_pass", vtype_int, &arc_oldest_pass }, { "arc_state", vtype_stringptr, &arc_state }, { "arc_state_reason", vtype_stringptr, &arc_state_reason }, #endif @@ -562,7 +564,9 @@ static var_entry var_table[] = { { "local_part_data", vtype_stringptr, &deliver_localpart_data }, { "local_part_prefix", vtype_stringptr, &deliver_localpart_prefix }, { "local_part_suffix", vtype_stringptr, &deliver_localpart_suffix }, +#ifdef HAVE_LOCAL_SCAN { "local_scan_data", vtype_stringptr, &local_scan_data }, +#endif { "local_user_gid", vtype_gid, &local_user_gid }, { "local_user_uid", vtype_uid, &local_user_uid }, { "localhost_number", vtype_int, &host_number }, @@ -656,6 +660,9 @@ static var_entry var_table[] = { { "regex_match_string", vtype_stringptr, ®ex_match_string }, #endif { "reply_address", vtype_reply, NULL }, +#if defined(SUPPORT_TLS) && defined(EXPERIMENTAL_REQUIRETLS) + { "requiretls", vtype_bool, &tls_requiretls }, +#endif { "return_path", vtype_stringptr, &return_path }, { "return_size_limit", vtype_int, &bounce_return_size_limit }, { "router_name", vtype_stringptr, &router_name }, @@ -813,6 +820,10 @@ static uschar *mtable_setid[] = static uschar *mtable_sticky[] = { US"--T", US"--t", US"-wT", US"-wt", US"r-T", US"r-t", US"rwT", US"rwt" }; +/* flags for find_header() */ +#define FH_EXISTS_ONLY BIT(0) +#define FH_WANT_RAW BIT(1) +#define FH_WANT_LIST BIT(2) /************************************************* @@ -916,7 +927,7 @@ int rc; uschar *ss = expand_string(condition); if (ss == NULL) { - if (!expand_string_forcedfail && !search_find_defer) + if (!f.expand_string_forcedfail && !f.search_find_defer) log_write(0, LOG_MAIN|LOG_PANIC, "failed to expand condition \"%s\" " "for %s %s: %s", condition, m1, m2, expand_string_message); return FALSE; @@ -1122,20 +1133,20 @@ Returns: NULL if the subfield was not found, or */ static uschar * -expand_getkeyed(uschar *key, const uschar *s) +expand_getkeyed(uschar * key, const uschar * s) { int length = Ustrlen(key); while (isspace(*s)) s++; /* Loop to search for the key */ -while (*s != 0) +while (*s) { int dkeylength; - uschar *data; - const uschar *dkey = s; + uschar * data; + const uschar * dkey = s; - while (*s != 0 && *s != '=' && !isspace(*s)) s++; + while (*s && *s != '=' && !isspace(*s)) s++; dkeylength = s - dkey; while (isspace(*s)) s++; if (*s == '=') while (isspace((*(++s)))); @@ -1246,17 +1257,17 @@ return fieldtext; static uschar * expand_getlistele(int field, const uschar * list) { -const uschar * tlist= list; -int sep= 0; +const uschar * tlist = list; +int sep = 0; uschar dummy; -if(field<0) +if (field < 0) { - for(field++; string_nextinlist(&tlist, &sep, &dummy, 1); ) field++; - sep= 0; + for (field++; string_nextinlist(&tlist, &sep, &dummy, 1); ) field++; + sep = 0; } -if(field==0) return NULL; -while(--field>0 && (string_nextinlist(&list, &sep, &dummy, 1))) ; +if (field == 0) return NULL; +while (--field > 0 && (string_nextinlist(&list, &sep, &dummy, 1))) ; return string_nextinlist(&list, &sep, NULL, 0); } @@ -1517,22 +1528,25 @@ can also return a concatenation of all the header lines. When concatenating specific headers that contain lists of addresses, a comma is inserted between them. Otherwise we use a straight concatenation. Because some messages can have pathologically large number of lines, there is a limit on the length that is -returned. Also, to avoid massive store use which would result from using -string_cat() as it copies and extends strings, we do a preliminary pass to find -out exactly how much store will be needed. On "normal" messages this will be -pretty trivial. +returned. Arguments: name the name of the header, without the leading $header_ or $h_, or NULL if a concatenation of all headers is required - exists_only TRUE if called from a def: test; don't need to build a string; - just return a string that is not "" and not "0" if the header - exists newsize return the size of memory block that was obtained; may be NULL if exists_only is TRUE - want_raw TRUE if called for $rh_ or $rheader_ variables; no processing, - other than concatenating, will be done on the header. Also used - for $message_headers_raw. + flags FH_EXISTS_ONLY + set if called from a def: test; don't need to build a string; + just return a string that is not "" and not "0" if the header + exists + FH_WANT_RAW + set if called for $rh_ or $rheader_ items; no processing, + other than concatenating, will be done on the header. Also used + for $message_headers_raw. + FH_WANT_LIST + Double colon chars in the content, and replace newline with + colon between each element when concatenating; returning a + colon-sep list (elements might contain newlines) charset name of charset to translate MIME words to; used only if want_raw is false; if NULL, no translation is done (this is used for $bh_ and $bheader_) @@ -1542,127 +1556,121 @@ Returns: NULL if the header does not exist, else a pointer to a new */ static uschar * -find_header(uschar *name, BOOL exists_only, int *newsize, BOOL want_raw, - uschar *charset) +find_header(uschar *name, int *newsize, unsigned flags, uschar *charset) { -BOOL found = name == NULL; -int comma = 0; -int len = found? 0 : Ustrlen(name); -int i; -uschar *yield = NULL; -uschar *ptr = NULL; - -/* Loop for two passes - saves code repetition */ - -for (i = 0; i < 2; i++) - { - int size = 0; - header_line *h; - - for (h = header_list; size < header_insert_maxlen && h; h = h->next) - if (h->type != htype_old && h->text) /* NULL => Received: placeholder */ - if (!name || (len <= h->slen && strncmpic(name, h->text, len) == 0)) - { - int ilen; - uschar *t; - - if (exists_only) return US"1"; /* don't need actual string */ - found = TRUE; - t = h->text + len; /* text to insert */ - if (!want_raw) /* unless wanted raw, */ - while (isspace(*t)) t++; /* remove leading white space */ - ilen = h->slen - (t - h->text); /* length to insert */ - - /* Unless wanted raw, remove trailing whitespace, including the - newline. */ - - if (!want_raw) - while (ilen > 0 && isspace(t[ilen-1])) ilen--; +BOOL found = !name; +int len = name ? Ustrlen(name) : 0; +BOOL comma = FALSE; +header_line * h; +gstring * g = NULL; - /* Set comma = 1 if handling a single header and it's one of those - that contains an address list, except when asked for raw headers. Only - need to do this once. */ +for (h = header_list; h; h = h->next) + if (h->type != htype_old && h->text) /* NULL => Received: placeholder */ + if (!name || (len <= h->slen && strncmpic(name, h->text, len) == 0)) + { + uschar * s, * t; + size_t inc; - if (!want_raw && name && comma == 0 && - Ustrchr("BCFRST", h->type) != NULL) - comma = 1; + if (flags & FH_EXISTS_ONLY) + return US"1"; /* don't need actual string */ - /* First pass - compute total store needed; second pass - compute - total store used, including this header. */ + found = TRUE; + s = h->text + len; /* text to insert */ + if (!(flags & FH_WANT_RAW)) /* unless wanted raw, */ + while (isspace(*s)) s++; /* remove leading white space */ + t = h->text + h->slen; /* end-point */ - size += ilen + comma + 1; /* +1 for the newline */ + /* Unless wanted raw, remove trailing whitespace, including the + newline. */ - /* Second pass - concatenate the data, up to a maximum. Note that - the loop stops when size hits the limit. */ + if (flags & FH_WANT_LIST) + while (t > s && t[-1] == '\n') t--; + else if (!(flags & FH_WANT_RAW)) + { + while (t > s && isspace(t[-1])) t--; - if (i != 0) - { - if (size > header_insert_maxlen) - { - ilen -= size - header_insert_maxlen - 1; - comma = 0; - } - Ustrncpy(ptr, t, ilen); - ptr += ilen; + /* Set comma if handling a single header and it's one of those + that contains an address list, except when asked for raw headers. Only + need to do this once. */ - /* For a non-raw header, put in the comma if needed, then add - back the newline we removed above, provided there was some text in - the header. */ + if (name && !comma && Ustrchr("BCFRST", h->type)) comma = TRUE; + } - if (!want_raw && ilen > 0) - { - if (comma != 0) *ptr++ = ','; - *ptr++ = '\n'; - } - } - } + /* Trim the header roughly if we're approaching limits */ + inc = t - s; + if ((g ? g->ptr : 0) + inc > header_insert_maxlen) + inc = header_insert_maxlen - (g ? g->ptr : 0); + + /* For raw just copy the data; for a list, add the data as a colon-sep + list-element; for comma-list add as an unchecked comma,newline sep + list-elemment; for other nonraw add as an unchecked newline-sep list (we + stripped trailing WS above including the newline). We ignore the potential + expansion due to colon-doubling, just leaving the loop if the limit is met + or exceeded. */ + + if (flags & FH_WANT_LIST) + g = string_append_listele_n(g, ':', s, (unsigned)inc); + else if (flags & FH_WANT_RAW) + { + g = string_catn(g, s, (unsigned)inc); + (void) string_from_gstring(g); + } + else if (inc > 0) + if (comma) + g = string_append2_listele_n(g, US",\n", s, (unsigned)inc); + else + g = string_append2_listele_n(g, US"\n", s, (unsigned)inc); - /* At end of first pass, return NULL if no header found. Then truncate size - if necessary, and get the buffer to hold the data, returning the buffer size. - */ + if (g && g->ptr >= header_insert_maxlen) break; + } - if (i == 0) - { - if (!found) return NULL; - if (size > header_insert_maxlen) size = header_insert_maxlen; - *newsize = size + 1; - ptr = yield = store_get(*newsize); - } - } +if (!found) return NULL; /* No header found */ +if (!g) return US""; /* That's all we do for raw header expansion. */ -if (want_raw) - *ptr = 0; +*newsize = g->size; +if (flags & FH_WANT_RAW) + return g->s; -/* Otherwise, remove a final newline and a redundant added comma. Then we do -RFC 2047 decoding, translating the charset if requested. The rfc2047_decode2() -function can return an error with decoded data if the charset translation -fails. If decoding fails, it returns NULL. */ +/* Otherwise do RFC 2047 decoding, translating the charset if requested. +The rfc2047_decode2() function can return an error with decoded data if the +charset translation fails. If decoding fails, it returns NULL. */ else { uschar *decoded, *error; - if (ptr > yield && ptr[-1] == '\n') ptr--; - if (ptr > yield && comma != 0 && ptr[-1] == ',') ptr--; - *ptr = 0; - decoded = rfc2047_decode2(yield, check_rfc2047_length, charset, '?', NULL, + + decoded = rfc2047_decode2(g->s, check_rfc2047_length, charset, '?', NULL, newsize, &error); - if (error != NULL) + if (error) { DEBUG(D_any) debug_printf("*** error in RFC 2047 decoding: %s\n" - " input was: %s\n", error, yield); + " input was: %s\n", error, g->s); } - if (decoded != NULL) yield = decoded; + return decoded ? decoded : g->s; } - -return yield; } -/* Append an "iprev" element to an Autherntication-Results: header +/* Append a "local" element to an Authentication-Results: header +if this was a non-smtp message. +*/ + +static gstring * +authres_local(gstring * g, const uschar * sysname) +{ +if (!f.authentication_local) + return g; +g = string_append(g, 3, US";\n\tlocal=pass (non-smtp, ", sysname, US")"); +if (authenticated_id) g = string_append(g, 2, " u=", authenticated_id); +return g; +} + + +/* Append an "iprev" element to an Authentication-Results: header if we have attempted to get the calling host's name. */ @@ -1670,11 +1678,16 @@ static gstring * authres_iprev(gstring * g) { if (sender_host_name) - return string_append(g, 3, US";\n\tiprev=pass (", sender_host_name, US")"); -if (host_lookup_deferred) - return string_catn(g, US";\n\tiprev=temperror", 19); -if (host_lookup_failed) - return string_catn(g, US";\n\tiprev=fail", 13); + g = string_append(g, 3, US";\n\tiprev=pass (", sender_host_name, US")"); +else if (host_lookup_deferred) + g = string_catn(g, US";\n\tiprev=temperror", 19); +else if (host_lookup_failed) + g = string_catn(g, US";\n\tiprev=fail", 13); +else + return g; + +if (sender_host_address) + g = string_append(g, 2, US" smtp.client-ip=", sender_host_address); return g; } @@ -1690,18 +1703,18 @@ generated from a system filter, but not elsewhere. */ static uschar * fn_recipients(void) { +uschar * s; gstring * g = NULL; int i; -if (!enable_dollar_recipients) return NULL; +if (!f.enable_dollar_recipients) return NULL; for (i = 0; i < recipients_count; i++) { - /*XXX variant of list_appendele? */ - if (i != 0) g = string_catn(g, US", ", 2); - g = string_cat(g, recipients_list[i].address); + s = recipients_list[i].address; + g = string_append2_listele_n(g, US", ", s, Ustrlen(s)); } -return string_from_gstring(g); +return g ? g->s : NULL; } @@ -1786,7 +1799,7 @@ val = vp->value; switch (vp->type) { case vtype_filter_int: - if (!filter_running) return NULL; + if (!f.filter_running) return NULL; /* Fall through */ /* VVVVVVVVVVVV */ case vtype_int: @@ -1845,10 +1858,11 @@ switch (vp->type) return (domain == NULL)? US"" : domain + 1; case vtype_msgheaders: - return find_header(NULL, exists_only, newsize, FALSE, NULL); + return find_header(NULL, newsize, exists_only ? FH_EXISTS_ONLY : 0, NULL); case vtype_msgheaders_raw: - return find_header(NULL, exists_only, newsize, TRUE, NULL); + return find_header(NULL, newsize, + exists_only ? FH_EXISTS_ONLY|FH_WANT_RAW : FH_WANT_RAW, NULL); case vtype_msgbody: /* Pointer to msgbody string */ case vtype_msgbody_end: /* Ditto, the end of the msg */ @@ -1913,15 +1927,18 @@ switch (vp->type) return tod_stamp(tod_log_datestamp_daily); case vtype_reply: /* Get reply address */ - s = find_header(US"reply-to:", exists_only, newsize, TRUE, - headers_charset); - if (s != NULL) while (isspace(*s)) s++; - if (s == NULL || *s == 0) + s = find_header(US"reply-to:", newsize, + exists_only ? FH_EXISTS_ONLY|FH_WANT_RAW : FH_WANT_RAW, + headers_charset); + if (s) while (isspace(*s)) s++; + if (!s || !*s) { *newsize = 0; /* For the *s==0 case */ - s = find_header(US"from:", exists_only, newsize, TRUE, headers_charset); + s = find_header(US"from:", newsize, + exists_only ? FH_EXISTS_ONLY|FH_WANT_RAW : FH_WANT_RAW, + headers_charset); } - if (s != NULL) + if (s) { uschar *t; while (isspace(*s)) s++; @@ -1929,7 +1946,7 @@ switch (vp->type) while (t > s && isspace(t[-1])) t--; *t = 0; } - return (s == NULL)? US"" : s; + return s ? s : US""; case vtype_string_func: { @@ -2208,56 +2225,58 @@ switch(cond_type) yield == NULL we are in a skipping state, and don't care about the answer. */ case ECOND_DEF: - if (*s != ':') { - expand_string_message = US"\":\" expected after \"def\""; - return NULL; - } + uschar * t; - s = read_name(name, 256, s+1, US"_"); + if (*s != ':') + { + expand_string_message = US"\":\" expected after \"def\""; + return NULL; + } - /* Test for a header's existence. If the name contains a closing brace - character, this may be a user error where the terminating colon has been - omitted. Set a flag to adjust a subsequent error message in this case. */ + s = read_name(name, 256, s+1, US"_"); - if (Ustrncmp(name, "h_", 2) == 0 || - Ustrncmp(name, "rh_", 3) == 0 || - Ustrncmp(name, "bh_", 3) == 0 || - Ustrncmp(name, "header_", 7) == 0 || - Ustrncmp(name, "rheader_", 8) == 0 || - Ustrncmp(name, "bheader_", 8) == 0) - { - s = read_header_name(name, 256, s); - /* {-for-text-editors */ - if (Ustrchr(name, '}') != NULL) malformed_header = TRUE; - if (yield != NULL) *yield = - (find_header(name, TRUE, NULL, FALSE, NULL) != NULL) == testfor; - } + /* Test for a header's existence. If the name contains a closing brace + character, this may be a user error where the terminating colon has been + omitted. Set a flag to adjust a subsequent error message in this case. */ - /* Test for a variable's having a non-empty value. A non-existent variable - causes an expansion failure. */ + if ( ( *(t = name) == 'h' + || (*t == 'r' || *t == 'l' || *t == 'b') && *++t == 'h' + ) + && (*++t == '_' || Ustrncmp(t, "eader_", 6) == 0) + ) + { + s = read_header_name(name, 256, s); + /* {-for-text-editors */ + if (Ustrchr(name, '}') != NULL) malformed_header = TRUE; + if (yield) *yield = + (find_header(name, NULL, FH_EXISTS_ONLY, NULL) != NULL) == testfor; + } - else - { - uschar *value = find_variable(name, TRUE, yield == NULL, NULL); - if (value == NULL) + /* Test for a variable's having a non-empty value. A non-existent variable + causes an expansion failure. */ + + else { - expand_string_message = (name[0] == 0)? - string_sprintf("variable name omitted after \"def:\"") : - string_sprintf("unknown variable \"%s\" after \"def:\"", name); - check_variable_error_message(name); - return NULL; + if (!(t = find_variable(name, TRUE, yield == NULL, NULL))) + { + expand_string_message = (name[0] == 0)? + string_sprintf("variable name omitted after \"def:\"") : + string_sprintf("unknown variable \"%s\" after \"def:\"", name); + check_variable_error_message(name); + return NULL; + } + if (yield) *yield = (t[0] != 0) == testfor; } - if (yield != NULL) *yield = (value[0] != 0) == testfor; - } - return s; + return s; + } /* first_delivery tests for first delivery attempt */ case ECOND_FIRST_DELIVERY: - if (yield != NULL) *yield = deliver_firsttime == testfor; + if (yield != NULL) *yield = f.deliver_firsttime == testfor; return s; @@ -2412,7 +2431,7 @@ switch(cond_type) break; case DEFER: - expand_string_forcedfail = TRUE; + f.expand_string_forcedfail = TRUE; /*FALLTHROUGH*/ default: expand_string_message = string_sprintf("error from acl \"%s\"", sub[0]); @@ -3239,8 +3258,8 @@ want this string. Set skipping in the call in the fail case (this will always be the case if we were already skipping). */ sub1 = expand_string_internal(s, TRUE, &s, !yes, TRUE, resetok); -if (sub1 == NULL && (yes || !expand_string_forcedfail)) goto FAILED; -expand_string_forcedfail = FALSE; +if (sub1 == NULL && (yes || !f.expand_string_forcedfail)) goto FAILED; +f.expand_string_forcedfail = FALSE; if (*s++ != '}') { errwhere = US"'yes' part did not end with '}'"; @@ -3269,8 +3288,8 @@ while (isspace(*s)) s++; if (*s == '{') { sub2 = expand_string_internal(s+1, TRUE, &s, yes || skipping, TRUE, resetok); - if (sub2 == NULL && (!yes || !expand_string_forcedfail)) goto FAILED; - expand_string_forcedfail = FALSE; + if (sub2 == NULL && (!yes || !f.expand_string_forcedfail)) goto FAILED; + f.expand_string_forcedfail = FALSE; if (*s++ != '}') { errwhere = US"'no' part did not start with '{'"; @@ -3305,7 +3324,7 @@ else if (*s != '}') } expand_string_message = string_sprintf("\"%s\" failed and \"fail\" requested", type); - expand_string_forcedfail = TRUE; + f.expand_string_forcedfail = TRUE; goto FAILED; } } @@ -3534,6 +3553,26 @@ return yield; } +#ifdef SUPPORT_TLS +static gstring * +cat_file_tls(void * tls_ctx, gstring * yield, uschar * eol) +{ +int rc; +uschar * s; +uschar buffer[1024]; + +while ((rc = tls_read(tls_ctx, buffer, sizeof(buffer))) > 0) + for (s = buffer; rc--; s++) + yield = eol && *s == '\n' + ? string_cat(yield, eol) : string_catn(yield, s, 1); + +/* We assume that all errors, and any returns of zero bytes, +are actually EOF. */ + +(void) string_from_gstring(yield); +return yield; +} +#endif /************************************************* @@ -3810,6 +3849,87 @@ return x; +/* Return pointer to dewrapped string, with enclosing specified chars removed. +The given string is modified on return. Leading whitespace is skipped while +looking for the opening wrap character, then the rest is scanned for the trailing +(non-escaped) wrap character. A backslash in the string will act as an escape. + +A nul is written over the trailing wrap, and a pointer to the char after the +leading wrap is returned. + +Arguments: + s String for de-wrapping + wrap Two-char string, the first being the opener, second the closer wrapping + character +Return: + Pointer to de-wrapped string, or NULL on error (with expand_string_message set). +*/ + +static uschar * +dewrap(uschar * s, const uschar * wrap) +{ +uschar * p = s; +unsigned depth = 0; +BOOL quotesmode = wrap[0] == wrap[1]; + +while (isspace(*p)) p++; + +if (*p == *wrap) + { + s = ++p; + wrap++; + while (*p) + { + if (*p == '\\') p++; + else if (!quotesmode && *p == wrap[-1]) depth++; + else if (*p == *wrap) + if (depth == 0) + { + *p = '\0'; + return s; + } + else + depth--; + p++; + } + } +expand_string_message = string_sprintf("missing '%c'", *wrap); +return NULL; +} + + +/* Pull off the leading array or object element, returning +a copy in an allocated string. Update the list pointer. + +The element may itself be an abject or array. +*/ + +uschar * +json_nextinlist(const uschar ** list) +{ +unsigned array_depth = 0, object_depth = 0; +const uschar * s = *list, * item; + +while (isspace(*s)) s++; + +for (item = s; + *s && (*s != ',' || array_depth != 0 || object_depth != 0); + s++) + switch (*s) + { + case '[': array_depth++; break; + case ']': array_depth--; break; + case '{': object_depth++; break; + case '}': object_depth--; break; + } +*list = *s ? s+1 : s; +item = string_copyn(item, s - item); +DEBUG(D_expand) debug_printf_indent(" json ele: '%s'\n", item); +return US item; +} + + + /************************************************* * Expand string * *************************************************/ @@ -3886,13 +4006,17 @@ BOOL resetok = TRUE; expand_level++; DEBUG(D_expand) - debug_printf_indent(UTF8_DOWN_RIGHT "%s: %s\n", - skipping - ? UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ "scanning" - : "considering", - string); + DEBUG(D_noutf8) + debug_printf_indent("/%s: %s\n", + skipping ? "---scanning" : "considering", string); + else + debug_printf_indent(UTF8_DOWN_RIGHT "%s: %s\n", + skipping + ? UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ "scanning" + : "considering", + string); -expand_string_forcedfail = FALSE; +f.expand_string_forcedfail = FALSE; expand_string_message = US""; while (*s != 0) @@ -3958,6 +4082,7 @@ while (*s != 0) int len; int newsize = 0; gstring * g = NULL; + uschar * t; s = read_name(name, sizeof(name), s, US"_"); @@ -3975,17 +4100,19 @@ while (*s != 0) /* Header */ - if (Ustrncmp(name, "h_", 2) == 0 || - Ustrncmp(name, "rh_", 3) == 0 || - Ustrncmp(name, "bh_", 3) == 0 || - Ustrncmp(name, "header_", 7) == 0 || - Ustrncmp(name, "rheader_", 8) == 0 || - Ustrncmp(name, "bheader_", 8) == 0) + if ( ( *(t = name) == 'h' + || (*t == 'r' || *t == 'l' || *t == 'b') && *++t == 'h' + ) + && (*++t == '_' || Ustrncmp(t, "eader_", 6) == 0) + ) { - BOOL want_raw = (name[0] == 'r')? TRUE : FALSE; - uschar *charset = (name[0] == 'b')? NULL : headers_charset; + unsigned flags = *name == 'r' ? FH_WANT_RAW + : *name == 'l' ? FH_WANT_RAW|FH_WANT_LIST + : 0; + uschar * charset = *name == 'b' ? NULL : headers_charset; + s = read_header_name(name, sizeof(name), s); - value = find_header(name, FALSE, &newsize, want_raw, charset); + value = find_header(name, &newsize, flags, charset); /* If we didn't find the header, and the header contains a closing brace character, this may be a user error where the terminating colon @@ -4116,7 +4243,7 @@ while (*s != 0) continue; case DEFER: - expand_string_forcedfail = TRUE; + f.expand_string_forcedfail = TRUE; /*FALLTHROUGH*/ default: expand_string_message = string_sprintf("error from acl \"%s\"", sub[0]); @@ -4141,6 +4268,7 @@ while (*s != 0) US"Authentication-Results: ", sub_arg[0], US"; none"); yield->ptr -= 6; + yield = authres_local(yield, sub_arg[0]); yield = authres_iprev(yield); yield = authres_smtpauth(yield); #ifdef SUPPORT_SPF @@ -4175,15 +4303,21 @@ while (*s != 0) if (next_s == NULL) goto EXPAND_FAILED; /* message already set */ DEBUG(D_expand) - { - debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ - "condition: %.*s\n", - (int)(next_s - s), s); - debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ - UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ - "result: %s\n", - cond ? "true" : "false"); - } + DEBUG(D_noutf8) + { + debug_printf_indent("|--condition: %.*s\n", (int)(next_s - s), s); + debug_printf_indent("|-----result: %s\n", cond ? "true" : "false"); + } + else + { + debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ + "condition: %.*s\n", + (int)(next_s - s), s); + debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ + UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ + "result: %s\n", + cond ? "true" : "false"); + } s = next_s; @@ -4413,7 +4547,7 @@ while (*s != 0) } lookup_value = search_find(handle, filename, key, partial, affix, affixlen, starflags, &expand_setup); - if (search_find_defer) + if (f.search_find_defer) { expand_string_message = string_sprintf("lookup of \"%s\" gave DEFER: %s", @@ -4522,7 +4656,7 @@ while (*s != 0) expand_string_message = string_sprintf("Perl subroutine \"%s\" returned undef to force " "failure", sub_arg[0]); - expand_string_forcedfail = TRUE; + f.expand_string_forcedfail = TRUE; } goto EXPAND_FAILED; } @@ -4530,7 +4664,7 @@ while (*s != 0) /* Yield succeeded. Ensure forcedfail is unset, just in case it got set during a callback from Perl. */ - expand_string_forcedfail = FALSE; + f.expand_string_forcedfail = FALSE; yield = new_yield; continue; } @@ -4780,10 +4914,14 @@ while (*s != 0) int fd; int timeout = 5; int save_ptr = yield->ptr; - FILE *f; - uschar *arg; - uschar *sub_arg[4]; + FILE * fp; + uschar * arg; + uschar * sub_arg[4]; + uschar * server_name = NULL; + host_item host; BOOL do_shutdown = TRUE; + BOOL do_tls = FALSE; /* Only set under SUPPORT_TLS */ + void * tls_ctx = NULL; /* ditto */ blob reqstr; if (expand_forbid & RDO_READSOCK) @@ -4826,10 +4964,14 @@ while (*s != 0) while ((item = string_nextinlist(&list, &sep, NULL, 0))) if (Ustrncmp(item, US"shutdown=", 9) == 0) - if (Ustrcmp(item + 9, US"no") == 0) - do_shutdown = FALSE; + { if (Ustrcmp(item + 9, US"no") == 0) do_shutdown = FALSE; } +#ifdef SUPPORT_TLS + else if (Ustrncmp(item, US"tls=", 4) == 0) + { if (Ustrcmp(item + 9, US"no") != 0) do_tls = TRUE; } +#endif } - else sub_arg[3] = NULL; /* No eol if no timeout */ + else + sub_arg[3] = NULL; /* No eol if no timeout */ /* If skipping, we don't actually do anything. Otherwise, arrange to connect to either an IP or a Unix socket. */ @@ -4841,8 +4983,10 @@ while (*s != 0) if (Ustrncmp(sub_arg[0], "inet:", 5) == 0) { int port; - uschar * server_name = sub_arg[0] + 5; - uschar * port_name = Ustrrchr(server_name, ':'); + uschar * port_name; + + server_name = sub_arg[0] + 5; + port_name = Ustrrchr(server_name, ':'); /* Sort out the port */ @@ -4878,11 +5022,13 @@ while (*s != 0) } fd = ip_connectedsocket(SOCK_STREAM, server_name, port, port, - timeout, NULL, &expand_string_message, &reqstr); + timeout, &host, &expand_string_message, + do_tls ? NULL : &reqstr); callout_address = NULL; if (fd < 0) - goto SOCK_FAIL; - reqstr.len = 0; + goto SOCK_FAIL; + if (!do_tls) + reqstr.len = 0; } /* Handle a Unix domain socket */ @@ -4902,6 +5048,7 @@ while (*s != 0) sockun.sun_family = AF_UNIX; sprintf(sockun.sun_path, "%.*s", (int)(sizeof(sockun.sun_path)-1), sub_arg[0]); + server_name = US sockun.sun_path; sigalrm_seen = FALSE; alarm(timeout); @@ -4918,12 +5065,32 @@ while (*s != 0) "%s: %s", sub_arg[0], strerror(errno)); goto SOCK_FAIL; } + host.name = server_name; + host.address = US""; } DEBUG(D_expand) debug_printf_indent("connected to socket %s\n", sub_arg[0]); +#ifdef SUPPORT_TLS + if (do_tls) + { + tls_support tls_dummy = {.sni=NULL}; + uschar * errstr; + + if (!(tls_ctx = tls_client_start(fd, &host, NULL, NULL, +# ifdef SUPPORT_DANE + NULL, +# endif + &tls_dummy, &errstr))) + { + expand_string_message = string_sprintf("TLS connect failed: %s", errstr); + goto SOCK_FAIL; + } + } +#endif + /* Allow sequencing of test actions */ - if (running_in_test_harness) millisleep(100); + if (f.running_in_test_harness) millisleep(100); /* Write the request string, if not empty or already done */ @@ -4931,7 +5098,11 @@ while (*s != 0) { DEBUG(D_expand) debug_printf_indent("writing \"%s\" to socket\n", reqstr.data); - if (write(fd, reqstr.data, reqstr.len) != reqstr.len) + if ( ( +#ifdef SUPPORT_TLS + tls_ctx ? tls_write(tls_ctx, reqstr.data, reqstr.len, FALSE) : +#endif + write(fd, reqstr.data, reqstr.len)) != reqstr.len) { expand_string_message = string_sprintf("request write to socket " "failed: %s", strerror(errno)); @@ -4944,20 +5115,34 @@ while (*s != 0) system doesn't have this function, make it conditional. */ #ifdef SHUT_WR - if (do_shutdown) shutdown(fd, SHUT_WR); + if (!tls_ctx && do_shutdown) shutdown(fd, SHUT_WR); #endif - if (running_in_test_harness) millisleep(100); + if (f.running_in_test_harness) millisleep(100); /* Now we need to read from the socket, under a timeout. The function that reads a file can be used. */ - f = fdopen(fd, "rb"); + if (!tls_ctx) + fp = fdopen(fd, "rb"); sigalrm_seen = FALSE; alarm(timeout); - yield = cat_file(f, yield, sub_arg[3]); + yield = +#ifdef SUPPORT_TLS + tls_ctx ? cat_file_tls(tls_ctx, yield, sub_arg[3]) : +#endif + cat_file(fp, yield, sub_arg[3]); alarm(0); - (void)fclose(f); + +#ifdef SUPPORT_TLS + if (tls_ctx) + { + tls_close(tls_ctx, TRUE); + close(fd); + } + else +#endif + (void)fclose(fp); /* After a timeout, we restore the pointer in the result, that is, make sure we add nothing from the socket. */ @@ -5460,6 +5645,16 @@ while (*s != 0) uschar *sub[3]; int save_expand_nmax = save_expand_strings(save_expand_nstring, save_expand_nlength); + enum {extract_basic, extract_json} fmt = extract_basic; + + while (isspace(*s)) s++; + + /* Check for a format-variant specifier */ + + if (*s != '{') /*}*/ + { + if (Ustrncmp(s, "json", 4) == 0) {fmt = extract_json; s += 4;} + } /* While skipping we cannot rely on the data for expansions being available (eg. $item) hence cannot decide on numeric vs. keyed. @@ -5467,11 +5662,10 @@ while (*s != 0) if (skipping) { - while (isspace(*s)) s++; - for (j = 5; j > 0 && *s == '{'; j--) + for (j = 5; j > 0 && *s == '{'; j--) /*'}'*/ { if (!expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok)) - goto EXPAND_FAILED; /*{*/ + goto EXPAND_FAILED; /*'{'*/ if (*s++ != '}') { expand_string_message = US"missing '{' for arg of extract"; @@ -5479,13 +5673,13 @@ while (*s != 0) } while (isspace(*s)) s++; } - if ( Ustrncmp(s, "fail", 4) == 0 + if ( Ustrncmp(s, "fail", 4) == 0 /*'{'*/ && (s[4] == '}' || s[4] == ' ' || s[4] == '\t' || !s[4]) ) { s += 4; while (isspace(*s)) s++; - } + } /*'{'*/ if (*s != '}') { expand_string_message = US"missing '}' closing extract"; @@ -5495,11 +5689,11 @@ while (*s != 0) else for (i = 0, j = 2; i < j; i++) /* Read the proper number of arguments */ { - while (isspace(*s)) s++; - if (*s == '{') /*}*/ + while (isspace(*s)) s++; + if (*s == '{') /*'}'*/ { sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok); - if (sub[i] == NULL) goto EXPAND_FAILED; /*{*/ + if (sub[i] == NULL) goto EXPAND_FAILED; /*'{'*/ if (*s++ != '}') { expand_string_message = string_sprintf( @@ -5510,7 +5704,7 @@ while (*s != 0) /* After removal of leading and trailing white space, the first argument must not be empty; if it consists entirely of digits (optionally preceded by a minus sign), this is a numerical - extraction, and we expect 3 arguments. */ + extraction, and we expect 3 arguments (normal) or 2 (json). */ if (i == 0) { @@ -5541,7 +5735,7 @@ while (*s != 0) if (*p == 0) { field_number *= x; - j = 3; /* Need 3 args */ + if (fmt != extract_json) j = 3; /* Need 3 args */ field_number_set = TRUE; } } @@ -5557,9 +5751,83 @@ while (*s != 0) /* Extract either the numbered or the keyed substring into $value. If skipping, just pretend the extraction failed. */ - lookup_value = skipping? NULL : field_number_set? - expand_gettokened(field_number, sub[1], sub[2]) : - expand_getkeyed(sub[0], sub[1]); + if (skipping) + lookup_value = NULL; + else switch (fmt) + { + case extract_basic: + lookup_value = field_number_set + ? expand_gettokened(field_number, sub[1], sub[2]) + : expand_getkeyed(sub[0], sub[1]); + break; + + case extract_json: + { + uschar * s, * item; + const uschar * list; + + /* Array: Bracket-enclosed and comma-separated. + Object: Brace-enclosed, comma-sep list of name:value pairs */ + + if (!(s = dewrap(sub[1], field_number_set ? US"[]" : US"{}"))) + { + expand_string_message = + string_sprintf("%s wrapping %s for extract json", + expand_string_message, + field_number_set ? "array" : "object"); + goto EXPAND_FAILED_CURLY; + } + + list = s; + if (field_number_set) + { + if (field_number <= 0) + { + expand_string_message = US"first argument of \"extract\" must " + "be greater than zero"; + goto EXPAND_FAILED; + } + while (field_number > 0 && (item = json_nextinlist(&list))) + field_number--; + s = item; + lookup_value = s; + while (*s) s++; + while (--s >= lookup_value && isspace(*s)) *s = '\0'; + } + else + { + lookup_value = NULL; + while ((item = json_nextinlist(&list))) + { + /* Item is: string name-sep value. string is quoted. + Dequote the string and compare with the search key. */ + + if (!(item = dewrap(item, US"\"\""))) + { + expand_string_message = + string_sprintf("%s wrapping string key for extract json", + expand_string_message); + goto EXPAND_FAILED_CURLY; + } + if (Ustrcmp(item, sub[0]) == 0) /*XXX should be a UTF8-compare */ + { + s = item + strlen(item) + 1; + while (isspace(*s)) s++; + if (*s != ':') + { + expand_string_message = string_sprintf( + "missing object value-separator for extract json"); + goto EXPAND_FAILED_CURLY; + } + s++; + while (isspace(*s)) s++; + lookup_value = s; + break; + } + } + } + } + } /* If no string follows, $value gets substituted; otherwise there can be yes/no strings, as for lookup or if. */ @@ -5659,7 +5927,7 @@ while (*s != 0) /* Extract the numbered element into $value. If skipping, just pretend the extraction failed. */ - lookup_value = skipping? NULL : expand_getlistele(field_number, sub[1]); + lookup_value = skipping ? NULL : expand_getlistele(field_number, sub[1]); /* If no string follows, $value gets substituted; otherwise there can be yes/no strings, as for lookup or if. */ @@ -6224,7 +6492,7 @@ while (*s != 0) else { expand_string_message = result == NULL ? US"(no message)" : result; - if(status == FAIL_FORCED) expand_string_forcedfail = TRUE; + if(status == FAIL_FORCED) f.expand_string_forcedfail = TRUE; else if(status != FAIL) log_write(0, LOG_MAIN|LOG_PANIC, "dlfunc{%s}{%s} failed (%d): %s", argv[0], argv[1], status, expand_string_message); @@ -6278,7 +6546,9 @@ while (*s != 0) int c; uschar *arg = NULL; uschar *sub; +#ifdef SUPPORT_TLS var_entry *vp = NULL; +#endif /* Owing to an historical mis-design, an underscore may be part of the operator name, or it may introduce arguments. We therefore first scan the @@ -6842,7 +7112,7 @@ while (*s != 0) "missing in expanding ${addresses:%s}", --sub); goto EXPAND_FAILED; } - parse_allow_group = TRUE; + f.parse_allow_group = TRUE; for (;;) { @@ -6891,7 +7161,7 @@ while (*s != 0) separator. */ if (yield->ptr != save_ptr) yield->ptr--; - parse_allow_group = FALSE; + f.parse_allow_group = FALSE; continue; } @@ -7049,12 +7319,13 @@ while (*s != 0) { int seq_len = 0, index = 0; int bytes_left = 0; - long codepoint = -1; + long codepoint = -1; + int complete; uschar seq_buff[4]; /* accumulate utf-8 here */ while (*sub != 0) { - int complete = 0; + complete = 0; uschar c = *sub++; if (bytes_left) @@ -7119,6 +7390,13 @@ while (*s != 0) /* ASCII character follows incomplete sequence */ yield = string_catn(yield, &c, 1); } + /* If given a sequence truncated mid-character, we also want to report ? + * Eg, ${length_1:フィル} is one byte, not one character, so we expect + * ${utf8clean:${length_1:フィル}} to yield '?' */ + if (bytes_left != 0) + { + yield = string_catn(yield, UTF8_REPLACEMENT_CHAR, 1); + } continue; } @@ -7563,19 +7841,28 @@ if (resetok) store_reset(yield->s + (yield->size = yield->ptr + 1)); else if (resetok_p) *resetok_p = FALSE; DEBUG(D_expand) - { - debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ - "expanding: %.*s\n", - (int)(s - string), string); - debug_printf_indent("%s" - UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ - "result: %s\n", - skipping ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT, - yield->s); - if (skipping) - debug_printf_indent(UTF8_UP_RIGHT UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ - "skipping: result is not used\n"); - } + DEBUG(D_noutf8) + { + debug_printf_indent("|--expanding: %.*s\n", (int)(s - string), string); + debug_printf_indent("%sresult: %s\n", + skipping ? "|-----" : "\\_____", yield->s); + if (skipping) + debug_printf_indent("\\___skipping: result is not used\n"); + } + else + { + debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ + "expanding: %.*s\n", + (int)(s - string), string); + debug_printf_indent("%s" + UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ + "result: %s\n", + skipping ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT, + yield->s); + if (skipping) + debug_printf_indent(UTF8_UP_RIGHT UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ + "skipping: result is not used\n"); + } expand_level--; return yield->s; @@ -7597,16 +7884,25 @@ that is a bad idea, because expand_string_message is in dynamic store. */ EXPAND_FAILED: if (left) *left = s; DEBUG(D_expand) - { - debug_printf_indent(UTF8_VERT_RIGHT "failed to expand: %s\n", - string); - debug_printf_indent("%s" UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ - "error message: %s\n", - expand_string_forcedfail ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT, - expand_string_message); - if (expand_string_forcedfail) - debug_printf_indent(UTF8_UP_RIGHT "failure was forced\n"); - } + DEBUG(D_noutf8) + { + debug_printf_indent("|failed to expand: %s\n", string); + debug_printf_indent("%serror message: %s\n", + f.expand_string_forcedfail ? "|---" : "\\___", expand_string_message); + if (f.expand_string_forcedfail) + debug_printf_indent("\\failure was forced\n"); + } + else + { + debug_printf_indent(UTF8_VERT_RIGHT "failed to expand: %s\n", + string); + debug_printf_indent("%s" UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ + "error message: %s\n", + f.expand_string_forcedfail ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT, + expand_string_message); + if (f.expand_string_forcedfail) + debug_printf_indent(UTF8_UP_RIGHT "failure was forced\n"); + } if (resetok_p && !resetok) *resetok_p = FALSE; expand_level--; return NULL; @@ -7629,7 +7925,7 @@ if (Ustrpbrk(string, "$\\") != NULL) int old_pool = store_pool; uschar * s; - search_find_defer = FALSE; + f.search_find_defer = FALSE; malformed_header = FALSE; store_pool = POOL_MAIN; s = expand_string_internal(string, FALSE, NULL, FALSE, TRUE, NULL); @@ -7822,7 +8118,7 @@ if (svalue == NULL) { *rvalue = bvalue; return OK; } expanded = expand_string(svalue); if (expanded == NULL) { - if (expand_string_forcedfail) + if (f.expand_string_forcedfail) { DEBUG(dbg_opt) debug_printf("expansion of \"%s\" forced failure\n", oname); *rvalue = bvalue; @@ -7860,7 +8156,7 @@ expand_hide_passwords(uschar * s) { return ( ( Ustrstr(s, "failed to expand") != NULL || Ustrstr(s, "expansion of ") != NULL - ) + ) && ( Ustrstr(s, "mysql") != NULL || Ustrstr(s, "pgsql") != NULL || Ustrstr(s, "redis") != NULL @@ -7870,7 +8166,7 @@ return ( ( Ustrstr(s, "failed to expand") != NULL || Ustrstr(s, "ldapi:") != NULL || Ustrstr(s, "ldapdn:") != NULL || Ustrstr(s, "ldapm:") != NULL - ) ) + ) ) ? US"Temporary internal error" : s; } @@ -8079,9 +8375,9 @@ while (fgets(buffer, sizeof(buffer), stdin) != NULL) } else { - if (search_find_defer) printf("search_find deferred\n"); + if (f.search_find_defer) printf("search_find deferred\n"); printf("Failed: %s\n", expand_string_message); - if (expand_string_forcedfail) printf("Forced failure\n"); + if (f.expand_string_forcedfail) printf("Forced failure\n"); printf("\n"); } }