X-Git-Url: https://git.exim.org/exim.git/blobdiff_plain/9ee30919f807678b0bc9f675dcfa73225b486574..d945e180cbe2b26392fb21f7ae4dd2ccc603a81b:/src/src/expand.c diff --git a/src/src/expand.c b/src/src/expand.c index 10f009ce2..a916eee64 100644 --- a/src/src/expand.c +++ b/src/src/expand.c @@ -2,7 +2,7 @@ * Exim - an Internet mail transport agent * *************************************************/ -/* Copyright (c) The Exim Maintainers 2020 - 2022 */ +/* Copyright (c) The Exim Maintainers 2020 - 2023 */ /* Copyright (c) University of Cambridge 1995 - 2018 */ /* See the file NOTICE for conditions of use and distribution. */ /* SPDX-License-Identifier: GPL-2.0-or-later */ @@ -27,13 +27,6 @@ typedef unsigned esi_flags; # ifndef SUPPORT_CRYPTEQ # define SUPPORT_CRYPTEQ # endif -#else - -/* Recursively called function */ - -static uschar *expand_string_internal(const uschar *, esi_flags, const uschar **, BOOL *, BOOL *); -static int_eximarith_t expanded_string_integer(const uschar *, BOOL); - #endif /*!STAND_ALONE*/ #ifdef LOOKUP_LDAP @@ -237,6 +230,7 @@ static uschar *op_table_main[] = { US"expand", US"h", US"hash", + US"headerwrap", US"hex2b64", US"hexquote", US"ipv6denorm", @@ -284,6 +278,7 @@ enum { EOP_EXPAND, EOP_H, EOP_HASH, + EOP_HEADERWRAP, EOP_HEX2B64, EOP_HEXQUOTE, EOP_IPV6DENORM, @@ -478,8 +473,9 @@ typedef struct { int *length; } alblock; -static uschar * fn_recipients(void); typedef uschar * stringptr_fn_t(void); +static uschar * fn_recipients(void); +static uschar * fn_recipients_list(void); static uschar * fn_queue_size(void); /* This table must be kept in alphabetical order. */ @@ -685,7 +681,7 @@ static var_entry var_table[] = { { "qualify_domain", vtype_stringptr, &qualify_domain_sender }, { "qualify_recipient", vtype_stringptr, &qualify_domain_recipient }, { "queue_name", vtype_stringptr, &queue_name }, - { "queue_size", vtype_string_func, &fn_queue_size }, + { "queue_size", vtype_string_func, (void *) &fn_queue_size }, { "rcpt_count", vtype_int, &rcpt_count }, { "rcpt_defer_count", vtype_int, &rcpt_defer_count }, { "rcpt_fail_count", vtype_int, &rcpt_fail_count }, @@ -699,6 +695,7 @@ static var_entry var_table[] = { { "recipient_verify_failure",vtype_stringptr,&recipient_verify_failure }, { "recipients", vtype_string_func, (void *) &fn_recipients }, { "recipients_count", vtype_int, &recipients_count }, + { "recipients_list", vtype_string_func, (void *) &fn_recipients_list }, { "regex_cachesize", vtype_int, ®ex_cachesize },/* undocumented; devel observability */ #ifdef WITH_CONTENT_SCAN { "regex_match_string", vtype_stringptr, ®ex_match_string }, @@ -844,6 +841,7 @@ uschar * fn_arc_domains(void) {return NULL;} uschar * fn_hdrs_added(void) {return NULL;} uschar * fn_queue_size(void) {return NULL;} uschar * fn_recipients(void) {return NULL;} +uschar * fn_recipients_list(void) {return NULL;} uschar * sender_helo_verified_boolstr(void) {return NULL;} uschar * smtp_cmd_hist(void) {return NULL;} @@ -940,6 +938,10 @@ static uschar *mtable_sticky[] = #define FH_WANT_RAW BIT(1) #define FH_WANT_LIST BIT(2) +/* Recursively called function */ +static uschar *expand_string_internal(const uschar *, esi_flags, const uschar **, BOOL *, BOOL *); +static int_eximarith_t expanded_string_integer(const uschar *, BOOL); + /************************************************* * Tables for UTF-8 support * @@ -1667,12 +1669,13 @@ Returns: NULL if the header does not exist, else a pointer to a new */ static uschar * -find_header(uschar *name, int *newsize, unsigned flags, const uschar *charset) +find_header(uschar * name, int * newsize, unsigned flags, const uschar * charset) { BOOL found = !name; int len = name ? Ustrlen(name) : 0; BOOL comma = FALSE; gstring * g = NULL; +uschar * rawhdr; for (header_line * h = header_list; h; h = h->next) if (h->type != htype_old && h->text) /* NULL => Received: placeholder */ @@ -1735,8 +1738,9 @@ if (!g) return US""; /* That's all we do for raw header expansion. */ *newsize = g->size; +rawhdr = string_from_gstring(g); if (flags & FH_WANT_RAW) - return string_from_gstring(g); + return rawhdr; /* Otherwise do RFC 2047 decoding, translating the charset if requested. The rfc2047_decode2() function can return an error with decoded data if the @@ -1744,12 +1748,12 @@ charset translation fails. If decoding fails, it returns NULL. */ else { - uschar * error, * decoded = rfc2047_decode2(string_from_gstring(g), + uschar * error, * decoded = rfc2047_decode2(rawhdr, check_rfc2047_length, charset, '?', NULL, newsize, &error); if (error) DEBUG(D_any) debug_printf("*** error in RFC 2047 decoding: %s\n" - " input was: %s\n", error, g->s); - return decoded ? decoded : string_from_gstring(g); + " input was: %s\n", error, rawhdr); + return decoded ? decoded : rawhdr; } } @@ -1799,22 +1803,40 @@ return g; *************************************************/ /* A recipients list is available only during system message filtering, during ACL processing after DATA, and while expanding pipe commands -generated from a system filter, but not elsewhere. */ +generated from a system filter, but not elsewhere. Note that this does +not check for commas in the elements, and uses comma-space as seperator - +so cannot be used as an exim list as-is. */ static uschar * fn_recipients(void) { -uschar * s; gstring * g = NULL; if (!f.enable_dollar_recipients) return NULL; for (int i = 0; i < recipients_count; i++) { - s = recipients_list[i].address; + const uschar * s = recipients_list[i].address; g = string_append2_listele_n(g, US", ", s, Ustrlen(s)); } -return g ? g->s : NULL; +gstring_release_unused(g); +return string_from_gstring(g); +} + +/* Similar, but as a properly-quoted exim list */ + + +static uschar * +fn_recipients_list(void) +{ +gstring * g = NULL; + +if (!f.enable_dollar_recipients) return NULL; + +for (int i = 0; i < recipients_count; i++) + g = string_append_listele(g, ':', recipients_list[i].address); +gstring_release_unused(g); +return string_from_gstring(g); } @@ -2036,7 +2058,8 @@ switch (vp->type) if (!*ss && deliver_datafile >= 0) /* Read body when needed */ { uschar * body; - off_t start_offset = SPOOL_DATA_START_OFFSET; + off_t start_offset_o = spool_data_start_offset(message_id); + off_t start_offset = start_offset_o; int len = message_body_visible; if (len > message_size) len = message_size; @@ -2048,8 +2071,8 @@ switch (vp->type) if (fstat(deliver_datafile, &statbuf) == 0) { start_offset = statbuf.st_size - len; - if (start_offset < SPOOL_DATA_START_OFFSET) - start_offset = SPOOL_DATA_START_OFFSET; + if (start_offset < start_offset_o) + start_offset = start_offset_o; } } if (lseek(deliver_datafile, start_offset, SEEK_SET) < 0) @@ -2117,7 +2140,7 @@ switch (vp->type) case vtype_string_func: { stringptr_fn_t * fn = (stringptr_fn_t *) val; - uschar* s = fn(); + uschar * s = fn(); return s ? s : US""; } @@ -2382,19 +2405,26 @@ static uschar * json_nextinlist(const uschar ** list) { unsigned array_depth = 0, object_depth = 0; +BOOL quoted = FALSE; const uschar * s = *list, * item; skip_whitespace(&s); for (item = s; - *s && (*s != ',' || array_depth != 0 || object_depth != 0); + *s && (*s != ',' || array_depth != 0 || object_depth != 0 || quoted); s++) - switch (*s) + if (!quoted) switch (*s) { case '[': array_depth++; break; case ']': array_depth--; break; case '{': object_depth++; break; case '}': object_depth--; break; + case '"': quoted = TRUE; + } + else switch(*s) + { + case '\\': s++; break; /* backslash protects one char */ + case '"': quoted = FALSE; break; } *list = *s ? s+1 : s; if (item == s) return NULL; @@ -2732,9 +2762,17 @@ switch(cond_type = identify_operator(&s, &opname)) case ECOND_ISIP: case ECOND_ISIP4: case ECOND_ISIP6: - rc = string_is_ip_address(sub[0], NULL); - *yield = ((cond_type == ECOND_ISIP)? (rc != 0) : - (cond_type == ECOND_ISIP4)? (rc == 4) : (rc == 6)) == testfor; + { + const uschar *errp; + const uschar **errpp; + DEBUG(D_expand) errpp = &errp; else errpp = 0; + if (0 == (rc = string_is_ip_addressX(sub[0], NULL, errpp))) + DEBUG(D_expand) debug_printf("failed: %s\n", errp); + + *yield = ( cond_type == ECOND_ISIP ? rc != 0 : + cond_type == ECOND_ISIP4 ? rc == 4 : rc == 6) == testfor; + } + break; /* Various authentication tests - all optionally compiled */ @@ -3522,7 +3560,7 @@ switch(cond_type = identify_operator(&s, &opname)) /* Match the given local_part against the SRS-encoded pattern */ - re = regex_must_compile(US"^(?i)SRS0=([^=]+)=([A-Z2-7]+)=([^=]*)=(.*)$", + re = regex_must_compile(US"^(?i)SRS0=([^=]+)=([A-Z2-7]{2})=([^=]*)=(.*)$", MCS_CASELESS | MCS_CACHEABLE, FALSE); md = pcre2_match_data_create(4+1, pcre_gen_ctx); if (pcre2_match(re, sub[0], PCRE2_ZERO_TERMINATED, 0, PCRE_EOPT, @@ -3552,53 +3590,50 @@ switch(cond_type = identify_operator(&s, &opname)) /* If a zero-length secret was given, we're done. Otherwise carry on and validate the given SRS local_part againt our secret. */ - if (!*sub[1]) + if (*sub[1]) { - boolvalue = TRUE; - goto srs_result; - } + /* check the timestamp */ + { + struct timeval now; + uschar * ss = sub[0] + ovec[4]; /* substring 2, the timestamp */ + long d; + int n; - /* check the timestamp */ - { - struct timeval now; - uschar * ss = sub[0] + ovec[4]; /* substring 2, the timestamp */ - long d; - int n; + gettimeofday(&now, NULL); + now.tv_sec /= 86400; /* days since epoch */ - gettimeofday(&now, NULL); - now.tv_sec /= 86400; /* days since epoch */ + /* Decode substring 2 from base32 to a number */ - /* Decode substring 2 from base32 to a number */ + for (d = 0, n = ovec[5]-ovec[4]; n; n--) + { + uschar * t = Ustrchr(base32_chars, *ss++); + d = d * 32 + (t - base32_chars); + } - for (d = 0, n = ovec[5]-ovec[4]; n; n--) - { - uschar * t = Ustrchr(base32_chars, *ss++); - d = d * 32 + (t - base32_chars); + if (((now.tv_sec - d) & 0x3ff) > 10) /* days since SRS generated */ + { + DEBUG(D_expand) debug_printf("SRS too old\n"); + goto srs_result; + } } - if (((now.tv_sec - d) & 0x3ff) > 10) /* days since SRS generated */ + /* check length of substring 1, the offered checksum */ + + if (ovec[3]-ovec[2] != 4) { - DEBUG(D_expand) debug_printf("SRS too old\n"); + DEBUG(D_expand) debug_printf("SRS checksum wrong size\n"); goto srs_result; } - } - /* check length of substring 1, the offered checksum */ + /* Hash the address with our secret, and compare that computed checksum + with the one extracted from the arg */ - if (ovec[3]-ovec[2] != 4) - { - DEBUG(D_expand) debug_printf("SRS checksum wrong size\n"); - goto srs_result; - } - - /* Hash the address with our secret, and compare that computed checksum - with the one extracted from the arg */ - - hmac_md5(sub[1], srs_recipient, cksum, sizeof(cksum)); - if (Ustrncmp(cksum, sub[0] + ovec[2], 4) != 0) - { - DEBUG(D_expand) debug_printf("SRS checksum mismatch\n"); - goto srs_result; + hmac_md5(sub[1], srs_recipient, cksum, sizeof(cksum)); + if (Ustrncmp(cksum, sub[0] + ovec[2], 4) != 0) + { + DEBUG(D_expand) debug_printf("SRS checksum mismatch\n"); + goto srs_result; + } } boolvalue = TRUE; @@ -3955,10 +3990,9 @@ if (Ustrlen(key) > 64) hash_source = string_catn(NULL, key_num, 1); hash_source = string_catn(hash_source, daystamp, 3); hash_source = string_cat(hash_source, address); -(void) string_from_gstring(hash_source); DEBUG(D_expand) - debug_printf_indent("prvs: hash source is '%s'\n", hash_source->s); + debug_printf_indent("prvs: hash source is '%Y'\n", hash_source); memset(innerkey, 0x36, 64); memset(outerkey, 0x5c, 64); @@ -4462,30 +4496,17 @@ return yield; /************************************************/ static void debug_expansion_interim(const uschar * what, const uschar * value, int nchar, - BOOL skipping) + esi_flags flags) { -DEBUG(D_noutf8) - debug_printf_indent("|"); -else - debug_printf_indent(UTF8_VERT_RIGHT); +debug_printf_indent("%V", "K"); for (int fill = 11 - Ustrlen(what); fill > 0; fill--) - DEBUG(D_noutf8) - debug_printf("-"); - else - debug_printf(UTF8_HORIZ); + debug_printf("%V", "-"); -debug_printf("%s: %.*s\n", what, nchar, value); +debug_printf("%s: %.*W\n", what, nchar, value); if (is_tainted(value)) - { - DEBUG(D_noutf8) - debug_printf_indent("%s \\__", skipping ? "| " : " "); - else - debug_printf_indent("%s", - skipping - ? UTF8_VERT " " : " " UTF8_UP_RIGHT UTF8_HORIZ UTF8_HORIZ); - debug_printf("(tainted)\n"); - } + debug_printf_indent("%V %V(tainted)\n", + flags & ESI_SKIPPING ? "|" : " ", "\\__"); } @@ -4584,17 +4605,10 @@ while (*s) DEBUG(D_expand) { - DEBUG(D_noutf8) - debug_printf_indent("%c%s: %s\n", - first ? '/' : '|', - flags & ESI_SKIPPING ? "---scanning" : "considering", s); - else - debug_printf_indent("%s%s: %s\n", - first ? UTF8_DOWN_RIGHT : UTF8_VERT_RIGHT, - flags & ESI_SKIPPING - ? UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ "scanning" - : "considering", - s); + debug_printf_indent("%V%V%s: %W\n", + first ? "/" : "K", + flags & ESI_SKIPPING ? "---" : "", + flags & ESI_SKIPPING ? "scanning" : "considering", s); first = FALSE; } @@ -4617,21 +4631,20 @@ while (*s) for (s = t; *s ; s++) if (*s == '\\' && s[1] == 'N') break; DEBUG(D_expand) - debug_expansion_interim(US"protected", t, (int)(s - t), !!(flags & ESI_SKIPPING)); - yield = string_catn(yield, t, s - t); + debug_expansion_interim(US"protected", t, (int)(s - t), flags); + if (!(flags & ESI_SKIPPING)) + yield = string_catn(yield, t, s - t); if (*s) s += 2; } else { uschar ch[1]; DEBUG(D_expand) - DEBUG(D_noutf8) - debug_printf_indent("|backslashed: '\\%c'\n", s[1]); - else - debug_printf_indent(UTF8_VERT_RIGHT "backslashed: '\\%c'\n", s[1]); + debug_printf_indent("%Vbackslashed: '\\%c'\n", "K", s[1]); ch[0] = string_interpret_escape(&s); + if (!(flags & ESI_SKIPPING)) + yield = string_catn(yield, ch, 1); s++; - yield = string_catn(yield, ch, 1); } continue; } @@ -4648,9 +4661,10 @@ while (*s) for (const uschar * t = s+1; *t && *t != '$' && *t != '}' && *t != '\\'; t++) i++; - DEBUG(D_expand) debug_expansion_interim(US"text", s, i, !!(flags & ESI_SKIPPING)); + DEBUG(D_expand) debug_expansion_interim(US"text", s, i, flags); - yield = string_catn(yield, s, i); + if (!(flags & ESI_SKIPPING)) + yield = string_catn(yield, s, i); s += i; continue; } @@ -4676,15 +4690,16 @@ while (*s) /* If this is the first thing to be expanded, release the pre-allocated buffer. */ - if (!yield) - g = store_get(sizeof(gstring), GET_UNTAINTED); - else if (yield->ptr == 0) - { - if (resetok) reset_point = store_reset(reset_point); - yield = NULL; - reset_point = store_mark(); - g = store_get(sizeof(gstring), GET_UNTAINTED); /* alloc _before_ calling find_variable() */ - } + if (!(flags & ESI_SKIPPING)) + if (!yield) + g = store_get(sizeof(gstring), GET_UNTAINTED); + else if (yield->ptr == 0) + { + if (resetok) reset_point = store_reset(reset_point); + yield = NULL; + reset_point = store_mark(); + g = store_get(sizeof(gstring), GET_UNTAINTED); /* alloc _before_ calling find_variable() */ + } /* Header */ @@ -4733,28 +4748,30 @@ while (*s) reset in the middle of the buffer will make it inaccessible. */ len = Ustrlen(value); - DEBUG(D_expand) debug_expansion_interim(US"value", value, len, !!(flags & ESI_SKIPPING)); - if (!yield && newsize != 0) - { - yield = g; - yield->size = newsize; - yield->ptr = len; - yield->s = US value; /* known to be in new store i.e. a copy, so deconst safe */ - } - else - yield = string_catn(yield, value, len); + DEBUG(D_expand) debug_expansion_interim(US"value", value, len, flags); + if (!(flags & ESI_SKIPPING)) + if (!yield && newsize != 0) + { + yield = g; + yield->size = newsize; + yield->ptr = len; + yield->s = US value; /* known to be in new store i.e. a copy, so deconst safe */ + } + else + yield = string_catn(yield, value, len); continue; } - if (isdigit(*s)) + if (isdigit(*s)) /* A $ variable */ { int n; s = read_cnumber(&n, s); if (n >= 0 && n <= expand_nmax) { - DEBUG(D_expand) debug_expansion_interim(US"value", expand_nstring[n], expand_nlength[n], !!(flags & ESI_SKIPPING)); - yield = string_catn(yield, expand_nstring[n], expand_nlength[n]); + DEBUG(D_expand) debug_expansion_interim(US"value", expand_nstring[n], expand_nlength[n], flags); + if (!(flags & ESI_SKIPPING)) + yield = string_catn(yield, expand_nstring[n], expand_nlength[n]); } continue; } @@ -4781,8 +4798,9 @@ while (*s) } if (n >= 0 && n <= expand_nmax) { - DEBUG(D_expand) debug_expansion_interim(US"value", expand_nstring[n], expand_nlength[n], !!(flags & ESI_SKIPPING)); - yield = string_catn(yield, expand_nstring[n], expand_nlength[n]); + DEBUG(D_expand) debug_expansion_interim(US"value", expand_nstring[n], expand_nlength[n], flags); + if (!(flags & ESI_SKIPPING)) + yield = string_catn(yield, expand_nstring[n], expand_nlength[n]); } continue; } @@ -4804,7 +4822,7 @@ while (*s) skipping, but "break" otherwise so we get debug output for the item expansion. */ { - int start = gstring_length(yield); + int expansion_start = gstring_length(yield); switch(item_type) { /* Call an ACL from an expansion. We feed data in via $acl_arg1 - $acl_arg9. @@ -4868,7 +4886,7 @@ while (*s) yield = string_append(yield, 3, US"Authentication-Results: ", sub_arg[0], US"; none"); - yield->ptr -= 6; + yield->ptr -= 6; /* ignore tha ": none" for now */ yield = authres_local(yield, sub_arg[0]); yield = authres_iprev(yield); @@ -4907,9 +4925,9 @@ while (*s) DEBUG(D_expand) { - debug_expansion_interim(US"condition", s, (int)(next_s - s), !!(flags & ESI_SKIPPING)); + debug_expansion_interim(US"condition", s, (int)(next_s - s), flags); debug_expansion_interim(US"result", - cond ? US"true" : US"false", cond ? 4 : 5, !!(flags & ESI_SKIPPING)); + cond ? US"true" : US"false", cond ? 4 : 5, flags); } s = next_s; @@ -5381,18 +5399,18 @@ while (*s) if (iexpire >= inow) { prvscheck_result = US"1"; - DEBUG(D_expand) debug_printf_indent("prvscheck: success, $pvrs_result set to 1\n"); + DEBUG(D_expand) debug_printf_indent("prvscheck: success, $prvscheck_result set to 1\n"); } else { prvscheck_result = NULL; - DEBUG(D_expand) debug_printf_indent("prvscheck: signature expired, $pvrs_result unset\n"); + DEBUG(D_expand) debug_printf_indent("prvscheck: signature expired, $prvscheck_result unset\n"); } } else { prvscheck_result = NULL; - DEBUG(D_expand) debug_printf_indent("prvscheck: hash failure, $pvrs_result unset\n"); + DEBUG(D_expand) debug_printf_indent("prvscheck: hash failure, $prvscheck_result unset\n"); } /* Now expand the final argument. We leave this till now so that @@ -5615,7 +5633,7 @@ while (*s) { FILE * f; const uschar * arg, ** argv; - BOOL late_expand = TRUE; + unsigned late_expand = TSUC_EXPAND_ARGS | TSUC_ALLOW_TAINTED_ARGS | TSUC_ALLOW_RECIPIENTS; if (expand_forbid & RDO_RUN) { @@ -5626,9 +5644,8 @@ while (*s) /* Handle options to the "run" */ while (*s == ',') - { if (Ustrncmp(++s, "preexpand", 9) == 0) - { late_expand = FALSE; s += 9; } + { late_expand = 0; s += 9; } else { const uschar * t = s; @@ -5637,7 +5654,6 @@ while (*s) (int)(t-s), s); goto EXPAND_FAILED; } - } Uskip_whitespace(&s); if (*s != '{') /*}*/ @@ -5689,7 +5705,6 @@ while (*s) late_expand, /* expand args if not already done */ 0, /* not relevant when... */ NULL, /* no transporting address */ - late_expand, /* allow tainted args, when expand-after-split */ US"${run} expansion", /* for error messages */ &expand_string_message)) /* where to put error message */ goto EXPAND_FAILED; @@ -5780,16 +5795,15 @@ while (*s) case 3: goto EXPAND_FAILED; } - yield = string_cat(yield, sub[0]); - o2m = Ustrlen(sub[2]) - 1; - - if (o2m >= 0) for (; oldptr < yield->ptr; oldptr++) + if ( (yield = string_cat(yield, sub[0])) + && (o2m = Ustrlen(sub[2]) - 1) >= 0) + for (; oldptr < yield->ptr; oldptr++) { - uschar *m = Ustrrchr(sub[1], yield->s[oldptr]); + uschar * m = Ustrrchr(sub[1], yield->s[oldptr]); if (m) { int o = m - sub[1]; - yield->s[oldptr] = sub[2][(o < o2m)? o : o2m]; + yield->s[oldptr] = sub[2][o < o2m ? o : o2m]; } } @@ -6541,6 +6555,7 @@ while (*s) goto EXPAND_FAILED_CURLY; /*}*/ } + DEBUG(D_expand) debug_printf_indent("%s: evaluate input list list\n", name); if (!(list = expand_string_internal(s, ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | flags, &s, &resetok, NULL))) goto EXPAND_FAILED; /*{{*/ @@ -6560,6 +6575,7 @@ while (*s) expand_string_message = US"missing '{' for second arg of reduce"; goto EXPAND_FAILED_CURLY; /*}*/ } + DEBUG(D_expand) debug_printf_indent("reduce: initial result list\n"); t = expand_string_internal(s, ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | flags, &s, &resetok, NULL); if (!t) goto EXPAND_FAILED; @@ -6587,6 +6603,7 @@ while (*s) condition for real. For EITEM_MAP and EITEM_REDUCE, do the same, using the normal internal expansion function. */ + DEBUG(D_expand) debug_printf_indent("%s: find end of conditionn\n", name); if (item_type != EITEM_FILTER) temp = expand_string_internal(s, ESI_BRACE_ENDS | ESI_HONOR_DOLLAR | ESI_SKIPPING, &s, &resetok, NULL); @@ -7048,6 +7065,7 @@ while (*s) case 2: case 3: goto EXPAND_FAILED; } + if (flags & ESI_SKIPPING) continue; if (sub[1] && *(sub[1])) { @@ -7062,13 +7080,11 @@ while (*s) { struct timeval now; unsigned long i; - gstring * h = NULL; gettimeofday(&now, NULL); - for (unsigned long i = (now.tv_sec / 86400) & 0x3ff; i; i >>= 5) - h = string_catn(h, &base32_chars[i & 0x1f], 1); - if (h) while (h->ptr > 0) - g = string_catn(g, &h->s[--h->ptr], 1); + i = (now.tv_sec / 86400) & 0x3ff; + g = string_catn(g, &base32_chars[i >> 5], 1); + g = string_catn(g, &base32_chars[i & 0x1f], 1); } g = string_catn(g, US"=", 1); @@ -7107,7 +7123,7 @@ while (*s) it was for good reason */ if (quoted) yield = string_catn(yield, US"\"", 1); - yield = string_catn(yield, g->s, g->ptr); + yield = gstring_append(yield, g); if (quoted) yield = string_catn(yield, US"\"", 1); /* @$original_domain */ @@ -7126,10 +7142,11 @@ while (*s) } /* EITEM_* switch */ /*NOTREACHED*/ - DEBUG(D_expand) - if (yield && (start > 0 || *s)) /* only if not the sole expansion of the line */ + DEBUG(D_expand) /* only if not the sole expansion of the line */ + if (yield && (expansion_start > 0 || *s)) debug_expansion_interim(US"item-res", - yield->s + start, yield->ptr - start, !!(flags & ESI_SKIPPING)); + yield->s + expansion_start, yield->ptr - expansion_start, + flags); continue; NOT_ITEM: ; @@ -7165,6 +7182,7 @@ NOT_ITEM: ; /* Deal specially with operators that might take a certificate variable as we do not want to do the usual expansion. For most, expand the string.*/ + switch(c) { #ifndef DISABLE_TLS @@ -7213,16 +7231,16 @@ NOT_ITEM: ; to the main loop top. */ { - int start = yield->ptr; + unsigned expansion_start = gstring_length(yield); switch(c) { case EOP_BASE32: { - uschar *t; + uschar * t; unsigned long int n = Ustrtoul(sub, &t, 10); gstring * g = NULL; - if (*t != 0) + if (*t) { expand_string_message = string_sprintf("argument for base32 " "operator is \"%s\", which is not a decimal number", sub); @@ -7258,13 +7276,13 @@ NOT_ITEM: ; { uschar *t; unsigned long int n = Ustrtoul(sub, &t, 10); - if (*t != 0) + if (*t) { expand_string_message = string_sprintf("argument for base62 " "operator is \"%s\", which is not a decimal number", sub); goto EXPAND_FAILED; } - yield = string_cat(yield, string_base62(n)); + yield = string_cat(yield, string_base62_32(n)); /*XXX only handles 32b input range. Need variants? */ break; } @@ -7274,7 +7292,7 @@ NOT_ITEM: ; { uschar *tt = sub; unsigned long int n = 0; - while (*tt != 0) + while (*tt) { uschar *t = Ustrchr(base62_chars, *tt++); if (!t) @@ -7424,6 +7442,29 @@ NOT_ITEM: ; goto EXPAND_FAILED; #endif + /* Line-wrap a string as if it is a header line */ + + case EOP_HEADERWRAP: + { + unsigned col = 80, lim = 998; + uschar * s; + + if (arg) + { + const uschar * list = arg; + int sep = '_'; + if ((s = string_nextinlist(&list, &sep, NULL, 0))) + { + col = atoi(CS s); + if ((s = string_nextinlist(&list, &sep, NULL, 0))) + lim = atoi(CS s); + } + } + if ((s = wrap_header(sub, col, lim, US"\t", 8))) + yield = string_cat(yield, s); + } + break; + /* Convert hex encoding to base64 encoding */ case EOP_HEX2B64: @@ -7831,16 +7872,19 @@ NOT_ITEM: ; case EOP_UTF8CLEAN: { - int seq_len = 0, index = 0; - int bytes_left = 0; - long codepoint = -1; - int complete; + int seq_len = 0, index = 0, bytes_left = 0, complete; + u_long codepoint = (u_long)-1; uschar seq_buff[4]; /* accumulate utf-8 here */ /* Manually track tainting, as we deal in individual chars below */ - if (!yield->s || !yield->ptr) + if (!yield) + yield = string_get_tainted(Ustrlen(sub), sub); + else if (!yield->s || !yield->ptr) + { yield->s = store_get(yield->size = Ustrlen(sub), sub); + gstring_reset(yield); + } else if (is_incompatible(yield->s, sub)) gstring_rebuffer(yield, sub); @@ -7863,6 +7907,15 @@ NOT_ITEM: ; if (--bytes_left == 0) /* codepoint complete */ if(codepoint > 0x10FFFF) /* is it too large? */ complete = -1; /* error (RFC3629 limit) */ + else if ( (codepoint & 0x1FF800 ) == 0xD800 ) /* surrogate */ + /* A UTF-16 surrogate (which should be one of a pair that + encode a Unicode codepoint that is outside the Basic + Multilingual Plane). Error, not UTF8. + RFC2279.2 is slightly unclear on this, but + https://unicodebook.readthedocs.io/issues.html#strict-utf8-decoder + says "Surrogates characters are also invalid in UTF-8: + characters in U+D800—U+DFFF have to be rejected." */ + complete = -1; else { /* finished; output utf-8 sequence */ yield = string_catn(yield, seq_buff, seq_len); @@ -7872,27 +7925,25 @@ NOT_ITEM: ; } else /* no bytes left: new sequence */ { - if(!(c & 0x80)) /* 1-byte sequence, US-ASCII, keep it */ + if (!(c & 0x80)) /* 1-byte sequence, US-ASCII, keep it */ { yield = string_catn(yield, &c, 1); continue; } - if((c & 0xe0) == 0xc0) /* 2-byte sequence */ - { - if(c == 0xc0 || c == 0xc1) /* 0xc0 and 0xc1 are illegal */ + if ((c & 0xe0) == 0xc0) /* 2-byte sequence */ + if (c == 0xc0 || c == 0xc1) /* 0xc0 and 0xc1 are illegal */ complete = -1; else { - bytes_left = 1; - codepoint = c & 0x1f; + bytes_left = 1; + codepoint = c & 0x1f; } - } - else if((c & 0xf0) == 0xe0) /* 3-byte sequence */ + else if ((c & 0xf0) == 0xe0) /* 3-byte sequence */ { bytes_left = 2; codepoint = c & 0x0f; } - else if((c & 0xf8) == 0xf0) /* 4-byte sequence */ + else if ((c & 0xf8) == 0xf0) /* 4-byte sequence */ { bytes_left = 3; codepoint = c & 0x07; @@ -7966,7 +8017,7 @@ NOT_ITEM: ; goto EXPAND_FAILED; } yield = string_cat(yield, s); - DEBUG(D_expand) debug_printf_indent("yield: '%s'\n", yield->s); + DEBUG(D_expand) debug_printf_indent("yield: '%Y'\n", yield); break; } @@ -8074,7 +8125,7 @@ NOT_ITEM: ; case EOP_BASE64D: { uschar * s; - int len = b64decode(sub, &s); + int len = b64decode(sub, &s, sub); if (len < 0) { expand_string_message = string_sprintf("string \"%s\" is not " @@ -8275,31 +8326,18 @@ NOT_ITEM: ; DEBUG(D_expand) { - const uschar * s = yield->s + start; - int i = yield->ptr - start; + const uschar * res = string_from_gstring(yield); + const uschar * s = res + expansion_start; + int i = gstring_length(yield) - expansion_start; BOOL tainted = is_tainted(s); - DEBUG(D_noutf8) + debug_printf_indent("%Vop-res: %.*s\n", "K-----", i, s); + if (tainted) { - debug_printf_indent("|-----op-res: %.*s\n", i, s); - if (tainted) - { - debug_printf_indent("%s \\__", flags & ESI_SKIPPING ? "| " : " "); - debug_print_taint(yield->s); - } - } - else - { - debug_printf_indent(UTF8_VERT_RIGHT - UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ - "op-res: %.*s\n", i, s); - if (tainted) - { - debug_printf_indent("%s", - flags & ESI_SKIPPING - ? UTF8_VERT " " : " " UTF8_UP_RIGHT UTF8_HORIZ UTF8_HORIZ); - debug_print_taint(yield->s); - } + debug_printf_indent("%V %V", + flags & ESI_SKIPPING ? "|" : " ", + "\\__"); + debug_print_taint(res); } } continue; @@ -8373,58 +8411,48 @@ if (flags & ESI_BRACE_ENDS && !*s) added to the string. If so, set up an empty string. Add a terminating zero. If left != NULL, return a pointer to the terminator. */ -if (!yield) - yield = string_get(1); -(void) string_from_gstring(yield); -if (left) *left = s; + { + uschar * res; -/* Any stacking store that was used above the final string is no longer needed. -In many cases the final string will be the first one that was got and so there -will be optimal store usage. */ + if (!yield) + yield = string_get(1); + res = string_from_gstring(yield); + if (left) *left = s; -if (resetok) gstring_release_unused(yield); -else if (resetok_p) *resetok_p = FALSE; + /* Any stacking store that was used above the final string is no longer needed. + In many cases the final string will be the first one that was got and so there + will be optimal store usage. */ -DEBUG(D_expand) - { - BOOL tainted = is_tainted(yield->s); - DEBUG(D_noutf8) - { - debug_printf_indent("|--expanding: %.*s\n", (int)(s - string), string); - debug_printf_indent("%sresult: %s\n", - flags & ESI_SKIPPING ? "|-----" : "\\_____", yield->s); - if (tainted) - { - debug_printf_indent("%s \\__", flags & ESI_SKIPPING ? "| " : " "); - debug_print_taint(yield->s); - } - if (flags & ESI_SKIPPING) - debug_printf_indent("\\___skipping: result is not used\n"); - } - else + if (resetok) gstring_release_unused(yield); + else if (resetok_p) *resetok_p = FALSE; + + DEBUG(D_expand) { - debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ - "expanding: %.*s\n", + BOOL tainted = is_tainted(res); + debug_printf_indent("%Vexpanded: %.*W\n", + "K---", (int)(s - string), string); - debug_printf_indent("%s" UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ - "result: %s\n", - flags & ESI_SKIPPING ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT, - yield->s); + debug_printf_indent("%Vresult: ", + flags & ESI_SKIPPING ? "K-----" : "\\_____"); + if (*res || !(flags & ESI_SKIPPING)) + debug_printf("%W\n", res); + else + debug_printf(" %Vskipped%V\n", "<", ">"); if (tainted) { - debug_printf_indent("%s", - flags & ESI_SKIPPING - ? UTF8_VERT " " : " " UTF8_UP_RIGHT UTF8_HORIZ UTF8_HORIZ); - debug_print_taint(yield->s); + debug_printf_indent("%V %V", + flags & ESI_SKIPPING ? "|" : " ", + "\\__" + ); + debug_print_taint(res); } if (flags & ESI_SKIPPING) - debug_printf_indent(UTF8_UP_RIGHT UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ - "skipping: result is not used\n"); + debug_printf_indent("%Vskipping: result is not used\n", "\\___"); } - } -if (textonly_p) *textonly_p = textonly; -expand_level--; -return yield->s; + if (textonly_p) *textonly_p = textonly; + expand_level--; + return res; + } /* This is the failure exit: easiest to program with a goto. We still need to update the pointer to the terminator, for cases of nested calls with "fail". @@ -8445,25 +8473,11 @@ EXPAND_FAILED: if (left) *left = s; DEBUG(D_expand) { - DEBUG(D_noutf8) - { - debug_printf_indent("|failed to expand: %s\n", string); - debug_printf_indent("%serror message: %s\n", - f.expand_string_forcedfail ? "|---" : "\\___", expand_string_message); - if (f.expand_string_forcedfail) - debug_printf_indent("\\failure was forced\n"); - } - else - { - debug_printf_indent(UTF8_VERT_RIGHT "failed to expand: %s\n", - string); - debug_printf_indent("%s" UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ - "error message: %s\n", - f.expand_string_forcedfail ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT, - expand_string_message); - if (f.expand_string_forcedfail) - debug_printf_indent(UTF8_UP_RIGHT "failure was forced\n"); - } + debug_printf_indent("%Vfailed to expand: %s\n", "K", string); + debug_printf_indent("%Verror message: %s\n", + f.expand_string_forcedfail ? "K---" : "\\___", expand_string_message); + if (f.expand_string_forcedfail) + debug_printf_indent("%Vfailure was forced\n", "\\"); } if (resetok_p && !resetok) *resetok_p = FALSE; expand_level--; @@ -8486,13 +8500,12 @@ Returns: the expanded string, or NULL if expansion failed; if failure was const uschar * expand_string_2(const uschar * string, BOOL * textonly_p) { +f.expand_string_forcedfail = f.search_find_defer = malformed_header = FALSE; if (Ustrpbrk(string, "$\\") != NULL) { int old_pool = store_pool; uschar * s; - f.search_find_defer = FALSE; - malformed_header = FALSE; store_pool = POOL_MAIN; s = expand_string_internal(string, ESI_HONOR_DOLLAR, NULL, NULL, textonly_p); store_pool = old_pool; @@ -8666,12 +8679,14 @@ Returns: OK value placed in rvalue */ int -exp_bool(address_item *addr, - uschar *mtype, uschar *mname, unsigned dbg_opt, - uschar *oname, BOOL bvalue, - uschar *svalue, BOOL *rvalue) +exp_bool(address_item * addr, + uschar * mtype, uschar * mname, unsigned dbg_opt, + uschar * oname, BOOL bvalue, + uschar * svalue, BOOL * rvalue) { -uschar *expanded; +uschar * expanded; + +DEBUG(D_expand) debug_printf("try option %s\n", oname); if (!svalue) { *rvalue = bvalue; return OK; } if (!(expanded = expand_string(svalue)))