X-Git-Url: https://git.exim.org/users/jgh/exim.git/blobdiff_plain/c5db348c5e29e93e51389fa0079f829967c5da82..HEAD:/src/src/expand.c diff --git a/src/src/expand.c b/src/src/expand.c index 782467ff7..7986bbd14 100644 --- a/src/src/expand.c +++ b/src/src/expand.c @@ -2,7 +2,7 @@ * Exim - an Internet mail transport agent * *************************************************/ -/* Copyright (c) University of Cambridge 1995 - 2017 */ +/* Copyright (c) University of Cambridge 1995 - 2018 */ /* See the file NOTICE for conditions of use and distribution. */ @@ -17,22 +17,22 @@ static uschar *expand_string_internal(const uschar *, BOOL, const uschar **, BOO static int_eximarith_t expanded_string_integer(const uschar *, BOOL); #ifdef STAND_ALONE -#ifndef SUPPORT_CRYPTEQ -#define SUPPORT_CRYPTEQ -#endif +# ifndef SUPPORT_CRYPTEQ +# define SUPPORT_CRYPTEQ +# endif #endif #ifdef LOOKUP_LDAP -#include "lookups/ldap.h" +# include "lookups/ldap.h" #endif #ifdef SUPPORT_CRYPTEQ -#ifdef CRYPT_H -#include -#endif -#ifndef HAVE_CRYPT16 +# ifdef CRYPT_H +# include +# endif +# ifndef HAVE_CRYPT16 extern char* crypt16(char*, char*); -#endif +# endif #endif /* The handling of crypt16() is a mess. I will record below the analysis of the @@ -103,6 +103,7 @@ alphabetical order. */ static uschar *item_table[] = { US"acl", + US"authresults", US"certextract", US"dlfunc", US"env", @@ -128,11 +129,15 @@ static uschar *item_table[] = { US"run", US"sg", US"sort", +#ifdef EXPERIMENTAL_SRS_NATIVE + US"srs_encode", +#endif US"substr", US"tr" }; enum { EITEM_ACL, + EITEM_AUTHRESULTS, EITEM_CERTEXTRACT, EITEM_DLFUNC, EITEM_ENV, @@ -158,6 +163,9 @@ enum { EITEM_RUN, EITEM_SG, EITEM_SORT, +#ifdef EXPERIMENTAL_SRS_NATIVE + EITEM_SRS_ENCODE, +#endif EITEM_SUBSTR, EITEM_TR }; @@ -205,6 +213,7 @@ static uschar *op_table_main[] = { US"base62d", US"base64", US"base64d", + US"bless", US"domain", US"escape", US"escape8bit", @@ -233,6 +242,7 @@ static uschar *op_table_main[] = { US"rxquote", US"s", US"sha1", + US"sha2", US"sha256", US"sha3", US"stat", @@ -251,6 +261,7 @@ enum { EOP_BASE62D, EOP_BASE64, EOP_BASE64D, + EOP_BLESS, EOP_DOMAIN, EOP_ESCAPE, EOP_ESCAPE8BIT, @@ -279,6 +290,7 @@ enum { EOP_RXQUOTE, EOP_S, EOP_SHA1, + EOP_SHA2, EOP_SHA256, EOP_SHA3, EOP_STAT, @@ -310,11 +322,18 @@ static uschar *cond_table[] = { US"exists", US"first_delivery", US"forall", + US"forall_json", + US"forall_jsons", US"forany", + US"forany_json", + US"forany_jsons", US"ge", US"gei", US"gt", US"gti", +#ifdef EXPERIMENTAL_SRS_NATIVE + US"inbound_srs", +#endif US"inlist", US"inlisti", US"isip", @@ -356,11 +375,18 @@ enum { ECOND_EXISTS, ECOND_FIRST_DELIVERY, ECOND_FORALL, + ECOND_FORALL_JSON, + ECOND_FORALL_JSONS, ECOND_FORANY, + ECOND_FORANY_JSON, + ECOND_FORANY_JSONS, ECOND_STR_GE, ECOND_STR_GEI, ECOND_STR_GT, ECOND_STR_GTI, +#ifdef EXPERIMENTAL_SRS_NATIVE + ECOND_INBOUND_SRS, +#endif ECOND_INLIST, ECOND_INLISTI, ECOND_ISIP, @@ -439,6 +465,7 @@ typedef struct { } alblock; static uschar * fn_recipients(void); +typedef uschar * stringptr_fn_t(void); /* This table must be kept in alphabetical order. */ @@ -459,6 +486,12 @@ static var_entry var_table[] = { { "address_data", vtype_stringptr, &deliver_address_data }, { "address_file", vtype_stringptr, &address_file }, { "address_pipe", vtype_stringptr, &address_pipe }, +#ifdef EXPERIMENTAL_ARC + { "arc_domains", vtype_string_func, (void *) &fn_arc_domains }, + { "arc_oldest_pass", vtype_int, &arc_oldest_pass }, + { "arc_state", vtype_stringptr, &arc_state }, + { "arc_state_reason", vtype_stringptr, &arc_state_reason }, +#endif { "authenticated_fail_id",vtype_stringptr, &authenticated_fail_id }, { "authenticated_id", vtype_stringptr, &authenticated_id }, { "authenticated_sender",vtype_stringptr, &authenticated_sender }, @@ -508,11 +541,10 @@ static var_entry var_table[] = { { "dkim_key_testing", vtype_dkim, (void *)DKIM_KEY_TESTING }, { "dkim_selector", vtype_stringptr, &dkim_signing_selector }, { "dkim_signers", vtype_stringptr, &dkim_signers }, - { "dkim_verify_reason", vtype_dkim, (void *)DKIM_VERIFY_REASON }, - { "dkim_verify_status", vtype_dkim, (void *)DKIM_VERIFY_STATUS}, + { "dkim_verify_reason", vtype_stringptr, &dkim_verify_reason }, + { "dkim_verify_status", vtype_stringptr, &dkim_verify_status }, #endif -#ifdef EXPERIMENTAL_DMARC - { "dmarc_ar_header", vtype_stringptr, &dmarc_ar_header }, +#ifdef SUPPORT_DMARC { "dmarc_domain_policy", vtype_stringptr, &dmarc_domain_policy }, { "dmarc_status", vtype_stringptr, &dmarc_status }, { "dmarc_status_text", vtype_stringptr, &dmarc_status_text }, @@ -536,7 +568,7 @@ static var_entry var_table[] = { { "exim_path", vtype_stringptr, &exim_path }, { "exim_uid", vtype_uid, &exim_uid }, { "exim_version", vtype_stringptr, &version_string }, - { "headers_added", vtype_string_func, &fn_hdrs_added }, + { "headers_added", vtype_string_func, (void *) &fn_hdrs_added }, { "home", vtype_stringptr, &deliver_home }, { "host", vtype_stringptr, &deliver_host }, { "host_address", vtype_stringptr, &deliver_host_address }, @@ -557,7 +589,10 @@ static var_entry var_table[] = { { "local_part_data", vtype_stringptr, &deliver_localpart_data }, { "local_part_prefix", vtype_stringptr, &deliver_localpart_prefix }, { "local_part_suffix", vtype_stringptr, &deliver_localpart_suffix }, + { "local_part_verified", vtype_stringptr, &deliver_localpart_verified }, +#ifdef HAVE_LOCAL_SCAN { "local_scan_data", vtype_stringptr, &local_scan_data }, +#endif { "local_user_gid", vtype_gid, &local_user_gid }, { "local_user_uid", vtype_uid, &local_user_uid }, { "localhost_number", vtype_int, &host_number }, @@ -645,7 +680,7 @@ static var_entry var_table[] = { { "received_time", vtype_int, &received_time.tv_sec }, { "recipient_data", vtype_stringptr, &recipient_data }, { "recipient_verify_failure",vtype_stringptr,&recipient_verify_failure }, - { "recipients", vtype_string_func, &fn_recipients }, + { "recipients", vtype_string_func, (void *) &fn_recipients }, { "recipients_count", vtype_int, &recipients_count }, #ifdef WITH_CONTENT_SCAN { "regex_match_string", vtype_stringptr, ®ex_match_string }, @@ -680,7 +715,7 @@ static var_entry var_table[] = { { "smtp_active_hostname", vtype_stringptr, &smtp_active_hostname }, { "smtp_command", vtype_stringptr, &smtp_cmd_buffer }, { "smtp_command_argument", vtype_stringptr, &smtp_cmd_argument }, - { "smtp_command_history", vtype_string_func, &smtp_cmd_hist }, + { "smtp_command_history", vtype_string_func, (void *) &smtp_cmd_hist }, { "smtp_count_at_connection_start", vtype_int, &smtp_accept_count }, { "smtp_notquit_reason", vtype_stringptr, &smtp_notquit_reason }, { "sn0", vtype_filter_int, &filter_sn[0] }, @@ -700,11 +735,12 @@ static var_entry var_table[] = { { "spam_score", vtype_stringptr, &spam_score }, { "spam_score_int", vtype_stringptr, &spam_score_int }, #endif -#ifdef EXPERIMENTAL_SPF +#ifdef SUPPORT_SPF { "spf_guess", vtype_stringptr, &spf_guess }, { "spf_header_comment", vtype_stringptr, &spf_header_comment }, { "spf_received", vtype_stringptr, &spf_received }, { "spf_result", vtype_stringptr, &spf_result }, + { "spf_result_guessed", vtype_bool, &spf_result_guessed }, { "spf_smtp_comment", vtype_stringptr, &spf_smtp_comment }, #endif { "spool_directory", vtype_stringptr, &spool_directory }, @@ -715,7 +751,11 @@ static var_entry var_table[] = { { "srs_db_key", vtype_stringptr, &srs_db_key }, { "srs_orig_recipient", vtype_stringptr, &srs_orig_recipient }, { "srs_orig_sender", vtype_stringptr, &srs_orig_sender }, +#endif +#if defined(EXPERIMENTAL_SRS) || defined(EXPERIMENTAL_SRS_NATIVE) { "srs_recipient", vtype_stringptr, &srs_recipient }, +#endif +#ifdef EXPERIMENTAL_SRS { "srs_status", vtype_stringptr, &srs_status }, #endif { "thisaddress", vtype_stringptr, &filter_thisaddress }, @@ -728,32 +768,42 @@ static var_entry var_table[] = { { "tls_in_bits", vtype_int, &tls_in.bits }, { "tls_in_certificate_verified", vtype_int, &tls_in.certificate_verified }, { "tls_in_cipher", vtype_stringptr, &tls_in.cipher }, + { "tls_in_cipher_std", vtype_stringptr, &tls_in.cipher_stdname }, { "tls_in_ocsp", vtype_int, &tls_in.ocsp }, { "tls_in_ourcert", vtype_cert, &tls_in.ourcert }, { "tls_in_peercert", vtype_cert, &tls_in.peercert }, { "tls_in_peerdn", vtype_stringptr, &tls_in.peerdn }, -#if defined(SUPPORT_TLS) +#ifdef EXPERIMENTAL_TLS_RESUME + { "tls_in_resumption", vtype_int, &tls_in.resumption }, +#endif +#ifndef DISABLE_TLS { "tls_in_sni", vtype_stringptr, &tls_in.sni }, #endif + { "tls_in_ver", vtype_stringptr, &tls_in.ver }, { "tls_out_bits", vtype_int, &tls_out.bits }, { "tls_out_certificate_verified", vtype_int,&tls_out.certificate_verified }, { "tls_out_cipher", vtype_stringptr, &tls_out.cipher }, -#ifdef EXPERIMENTAL_DANE + { "tls_out_cipher_std", vtype_stringptr, &tls_out.cipher_stdname }, +#ifdef SUPPORT_DANE { "tls_out_dane", vtype_bool, &tls_out.dane_verified }, #endif { "tls_out_ocsp", vtype_int, &tls_out.ocsp }, { "tls_out_ourcert", vtype_cert, &tls_out.ourcert }, { "tls_out_peercert", vtype_cert, &tls_out.peercert }, { "tls_out_peerdn", vtype_stringptr, &tls_out.peerdn }, -#if defined(SUPPORT_TLS) +#ifdef EXPERIMENTAL_TLS_RESUME + { "tls_out_resumption", vtype_int, &tls_out.resumption }, +#endif +#ifndef DISABLE_TLS { "tls_out_sni", vtype_stringptr, &tls_out.sni }, #endif -#ifdef EXPERIMENTAL_DANE +#ifdef SUPPORT_DANE { "tls_out_tlsa_usage", vtype_int, &tls_out.tlsa_usage }, #endif + { "tls_out_ver", vtype_stringptr, &tls_out.ver }, { "tls_peerdn", vtype_stringptr, &tls_in.peerdn }, /* mind the alphabetical order! */ -#if defined(SUPPORT_TLS) +#ifndef DISABLE_TLS { "tls_sni", vtype_stringptr, &tls_in.sni }, /* mind the alphabetical order! */ #endif @@ -807,6 +857,10 @@ static uschar *mtable_setid[] = static uschar *mtable_sticky[] = { US"--T", US"--t", US"-wT", US"-wt", US"r-T", US"r-t", US"rwT", US"rwt" }; +/* flags for find_header() */ +#define FH_EXISTS_ONLY BIT(0) +#define FH_WANT_RAW BIT(1) +#define FH_WANT_LIST BIT(2) /************************************************* @@ -906,18 +960,16 @@ Returns: TRUE if condition is met, FALSE if not BOOL expand_check_condition(uschar *condition, uschar *m1, uschar *m2) { -int rc; -uschar *ss = expand_string(condition); -if (ss == NULL) +uschar * ss = expand_string(condition); +if (!ss) { - if (!expand_string_forcedfail && !search_find_defer) + if (!f.expand_string_forcedfail && !f.search_find_defer) log_write(0, LOG_MAIN|LOG_PANIC, "failed to expand condition \"%s\" " "for %s %s: %s", condition, m1, m2, expand_string_message); return FALSE; } -rc = ss[0] != 0 && Ustrcmp(ss, "0") != 0 && strcmpic(ss, US"no") != 0 && +return *ss && Ustrcmp(ss, "0") != 0 && strcmpic(ss, US"no") != 0 && strcmpic(ss, US"false") != 0; -return rc; } @@ -935,7 +987,7 @@ weirdness they'll twist this into. The result should ideally handle fork(). However, if we're stuck unable to provide this, then we'll fall back to appallingly bad randomness. -If SUPPORT_TLS is defined then this will not be used except as an emergency +If DISABLE_TLS is not defined then this will not be used except as an emergency fallback. Arguments: @@ -943,56 +995,55 @@ Arguments: Returns a random number in range [0, max-1] */ -#ifdef SUPPORT_TLS +#ifndef DISABLE_TLS # define vaguely_random_number vaguely_random_number_fallback #endif int vaguely_random_number(int max) { -#ifdef SUPPORT_TLS +#ifndef DISABLE_TLS # undef vaguely_random_number #endif - static pid_t pid = 0; - pid_t p2; -#if defined(HAVE_SRANDOM) && !defined(HAVE_SRANDOMDEV) - struct timeval tv; -#endif +static pid_t pid = 0; +pid_t p2; - p2 = getpid(); - if (p2 != pid) +if ((p2 = getpid()) != pid) + { + if (pid != 0) { - if (pid != 0) - { #ifdef HAVE_ARC4RANDOM - /* cryptographically strong randomness, common on *BSD platforms, not - so much elsewhere. Alas. */ -#ifndef NOT_HAVE_ARC4RANDOM_STIR - arc4random_stir(); -#endif + /* cryptographically strong randomness, common on *BSD platforms, not + so much elsewhere. Alas. */ +# ifndef NOT_HAVE_ARC4RANDOM_STIR + arc4random_stir(); +# endif #elif defined(HAVE_SRANDOM) || defined(HAVE_SRANDOMDEV) -#ifdef HAVE_SRANDOMDEV - /* uses random(4) for seeding */ - srandomdev(); -#else - gettimeofday(&tv, NULL); - srandom(tv.tv_sec | tv.tv_usec | getpid()); -#endif +# ifdef HAVE_SRANDOMDEV + /* uses random(4) for seeding */ + srandomdev(); +# else + { + struct timeval tv; + gettimeofday(&tv, NULL); + srandom(tv.tv_sec | tv.tv_usec | getpid()); + } +# endif #else - /* Poor randomness and no seeding here */ + /* Poor randomness and no seeding here */ #endif - } - pid = p2; } + pid = p2; + } #ifdef HAVE_ARC4RANDOM - return arc4random() % max; +return arc4random() % max; #elif defined(HAVE_SRANDOM) || defined(HAVE_SRANDOMDEV) - return random() % max; +return random() % max; #else - /* This one returns a 16-bit number, definitely not crypto-strong */ - return random_number(max); +/* This one returns a 16-bit number, definitely not crypto-strong */ +return random_number(max); #endif } @@ -1021,7 +1072,7 @@ static const uschar * read_name(uschar *name, int max, const uschar *s, uschar *extras) { int ptr = 0; -while (*s != 0 && (isalnum(*s) || Ustrchr(extras, *s) != NULL)) +while (*s && (isalnum(*s) || Ustrchr(extras, *s) != NULL)) { if (ptr < max-1) name[ptr++] = *s; s++; @@ -1116,20 +1167,20 @@ Returns: NULL if the subfield was not found, or */ static uschar * -expand_getkeyed(uschar *key, const uschar *s) +expand_getkeyed(uschar * key, const uschar * s) { int length = Ustrlen(key); while (isspace(*s)) s++; /* Loop to search for the key */ -while (*s != 0) +while (*s) { int dkeylength; - uschar *data; - const uschar *dkey = s; + uschar * data; + const uschar * dkey = s; - while (*s != 0 && *s != '=' && !isspace(*s)) s++; + while (*s && *s != '=' && !isspace(*s)) s++; dkeylength = s - dkey; while (isspace(*s)) s++; if (*s == '=') while (isspace((*(++s)))); @@ -1240,17 +1291,17 @@ return fieldtext; static uschar * expand_getlistele(int field, const uschar * list) { -const uschar * tlist= list; -int sep= 0; +const uschar * tlist = list; +int sep = 0; uschar dummy; -if(field<0) +if (field < 0) { - for(field++; string_nextinlist(&tlist, &sep, &dummy, 1); ) field++; - sep= 0; + for (field++; string_nextinlist(&tlist, &sep, &dummy, 1); ) field++; + sep = 0; } -if(field==0) return NULL; -while(--field>0 && (string_nextinlist(&list, &sep, &dummy, 1))) ; +if (field == 0) return NULL; +while (--field > 0 && (string_nextinlist(&list, &sep, &dummy, 1))) ; return string_nextinlist(&list, &sep, NULL, 0); } @@ -1258,7 +1309,7 @@ return string_nextinlist(&list, &sep, NULL, 0); /* Certificate fields, by name. Worry about by-OID later */ /* Names are chosen to not have common prefixes */ -#ifdef SUPPORT_TLS +#ifndef DISABLE_TLS typedef struct { uschar * name; @@ -1284,7 +1335,6 @@ static uschar * expand_getcertele(uschar * field, uschar * certvar) { var_entry * vp; -certfield * cp; if (!(vp = find_var_ent(certvar))) { @@ -1306,9 +1356,9 @@ if (!*(void **)vp->value) if (*field >= '0' && *field <= '9') return tls_cert_ext_by_oid(*(void **)vp->value, field, 0); -for(cp = certfields; - cp < certfields + nelem(certfields); - cp++) +for (certfield * cp = certfields; + cp < certfields + nelem(certfields); + cp++) if (Ustrncmp(cp->name, field, cp->namelen) == 0) { uschar * modifier = *(field += cp->namelen) == ',' @@ -1320,7 +1370,7 @@ expand_string_message = string_sprintf("bad field selector \"%s\" for certextract", field); return NULL; } -#endif /*SUPPORT_TLS*/ +#endif /*DISABLE_TLS*/ /************************************************* * Extract a substring from a string * @@ -1511,22 +1561,25 @@ can also return a concatenation of all the header lines. When concatenating specific headers that contain lists of addresses, a comma is inserted between them. Otherwise we use a straight concatenation. Because some messages can have pathologically large number of lines, there is a limit on the length that is -returned. Also, to avoid massive store use which would result from using -string_cat() as it copies and extends strings, we do a preliminary pass to find -out exactly how much store will be needed. On "normal" messages this will be -pretty trivial. +returned. Arguments: name the name of the header, without the leading $header_ or $h_, or NULL if a concatenation of all headers is required - exists_only TRUE if called from a def: test; don't need to build a string; - just return a string that is not "" and not "0" if the header - exists newsize return the size of memory block that was obtained; may be NULL if exists_only is TRUE - want_raw TRUE if called for $rh_ or $rheader_ variables; no processing, - other than concatenating, will be done on the header. Also used - for $message_headers_raw. + flags FH_EXISTS_ONLY + set if called from a def: test; don't need to build a string; + just return a string that is not "" and not "0" if the header + exists + FH_WANT_RAW + set if called for $rh_ or $rheader_ items; no processing, + other than concatenating, will be done on the header. Also used + for $message_headers_raw. + FH_WANT_LIST + Double colon chars in the content, and replace newline with + colon between each element when concatenating; returning a + colon-sep list (elements might contain newlines) charset name of charset to translate MIME words to; used only if want_raw is false; if NULL, no translation is done (this is used for $bh_ and $bheader_) @@ -1536,126 +1589,133 @@ Returns: NULL if the header does not exist, else a pointer to a new */ static uschar * -find_header(uschar *name, BOOL exists_only, int *newsize, BOOL want_raw, - uschar *charset) +find_header(uschar *name, int *newsize, unsigned flags, uschar *charset) { -BOOL found = name == NULL; -int comma = 0; -int len = found? 0 : Ustrlen(name); -int i; -uschar *yield = NULL; -uschar *ptr = NULL; +BOOL found = !name; +int len = name ? Ustrlen(name) : 0; +BOOL comma = FALSE; +gstring * g = NULL; -/* Loop for two passes - saves code repetition */ +for (header_line * h = header_list; h; h = h->next) + if (h->type != htype_old && h->text) /* NULL => Received: placeholder */ + if (!name || (len <= h->slen && strncmpic(name, h->text, len) == 0)) + { + uschar * s, * t; + size_t inc; -for (i = 0; i < 2; i++) - { - int size = 0; - header_line *h; + if (flags & FH_EXISTS_ONLY) + return US"1"; /* don't need actual string */ - for (h = header_list; size < header_insert_maxlen && h; h = h->next) - if (h->type != htype_old && h->text) /* NULL => Received: placeholder */ - if (!name || (len <= h->slen && strncmpic(name, h->text, len) == 0)) - { - int ilen; - uschar *t; + found = TRUE; + s = h->text + len; /* text to insert */ + if (!(flags & FH_WANT_RAW)) /* unless wanted raw, */ + while (isspace(*s)) s++; /* remove leading white space */ + t = h->text + h->slen; /* end-point */ - if (exists_only) return US"1"; /* don't need actual string */ - found = TRUE; - t = h->text + len; /* text to insert */ - if (!want_raw) /* unless wanted raw, */ - while (isspace(*t)) t++; /* remove leading white space */ - ilen = h->slen - (t - h->text); /* length to insert */ + /* Unless wanted raw, remove trailing whitespace, including the + newline. */ - /* Unless wanted raw, remove trailing whitespace, including the - newline. */ + if (flags & FH_WANT_LIST) + while (t > s && t[-1] == '\n') t--; + else if (!(flags & FH_WANT_RAW)) + { + while (t > s && isspace(t[-1])) t--; - if (!want_raw) - while (ilen > 0 && isspace(t[ilen-1])) ilen--; + /* Set comma if handling a single header and it's one of those + that contains an address list, except when asked for raw headers. Only + need to do this once. */ - /* Set comma = 1 if handling a single header and it's one of those - that contains an address list, except when asked for raw headers. Only - need to do this once. */ + if (name && !comma && Ustrchr("BCFRST", h->type)) comma = TRUE; + } - if (!want_raw && name && comma == 0 && - Ustrchr("BCFRST", h->type) != NULL) - comma = 1; + /* Trim the header roughly if we're approaching limits */ + inc = t - s; + if (gstring_length(g) + inc > header_insert_maxlen) + inc = header_insert_maxlen - gstring_length(g); + + /* For raw just copy the data; for a list, add the data as a colon-sep + list-element; for comma-list add as an unchecked comma,newline sep + list-elemment; for other nonraw add as an unchecked newline-sep list (we + stripped trailing WS above including the newline). We ignore the potential + expansion due to colon-doubling, just leaving the loop if the limit is met + or exceeded. */ + + if (flags & FH_WANT_LIST) + g = string_append_listele_n(g, ':', s, (unsigned)inc); + else if (flags & FH_WANT_RAW) + g = string_catn(g, s, (unsigned)inc); + else if (inc > 0) + g = string_append2_listele_n(g, comma ? US",\n" : US"\n", + s, (unsigned)inc); + + if (gstring_length(g) >= header_insert_maxlen) break; + } - /* First pass - compute total store needed; second pass - compute - total store used, including this header. */ +if (!found) return NULL; /* No header found */ +if (!g) return US""; - size += ilen + comma + 1; /* +1 for the newline */ +/* That's all we do for raw header expansion. */ - /* Second pass - concatenate the data, up to a maximum. Note that - the loop stops when size hits the limit. */ +*newsize = g->size; +if (flags & FH_WANT_RAW) + return string_from_gstring(g); - if (i != 0) - { - if (size > header_insert_maxlen) - { - ilen -= size - header_insert_maxlen - 1; - comma = 0; - } - Ustrncpy(ptr, t, ilen); - ptr += ilen; +/* Otherwise do RFC 2047 decoding, translating the charset if requested. +The rfc2047_decode2() function can return an error with decoded data if the +charset translation fails. If decoding fails, it returns NULL. */ - /* For a non-raw header, put in the comma if needed, then add - back the newline we removed above, provided there was some text in - the header. */ +else + { + uschar * error, * decoded = rfc2047_decode2(string_from_gstring(g), + check_rfc2047_length, charset, '?', NULL, newsize, &error); + if (error) + DEBUG(D_any) debug_printf("*** error in RFC 2047 decoding: %s\n" + " input was: %s\n", error, g->s); + return decoded ? decoded : string_from_gstring(g); + } +} - if (!want_raw && ilen > 0) - { - if (comma != 0) *ptr++ = ','; - *ptr++ = '\n'; - } - } - } - /* At end of first pass, return NULL if no header found. Then truncate size - if necessary, and get the buffer to hold the data, returning the buffer size. - */ - if (i == 0) - { - if (!found) return NULL; - if (size > header_insert_maxlen) size = header_insert_maxlen; - *newsize = size + 1; - ptr = yield = store_get(*newsize); - } - } -/* That's all we do for raw header expansion. */ +/* Append a "local" element to an Authentication-Results: header +if this was a non-smtp message. +*/ + +static gstring * +authres_local(gstring * g, const uschar * sysname) +{ +if (!f.authentication_local) + return g; +g = string_append(g, 3, US";\n\tlocal=pass (non-smtp, ", sysname, US")"); +if (authenticated_id) g = string_append(g, 2, " u=", authenticated_id); +return g; +} -if (want_raw) - *ptr = 0; -/* Otherwise, remove a final newline and a redundant added comma. Then we do -RFC 2047 decoding, translating the charset if requested. The rfc2047_decode2() -function can return an error with decoded data if the charset translation -fails. If decoding fails, it returns NULL. */ +/* Append an "iprev" element to an Authentication-Results: header +if we have attempted to get the calling host's name. +*/ +static gstring * +authres_iprev(gstring * g) +{ +if (sender_host_name) + g = string_append(g, 3, US";\n\tiprev=pass (", sender_host_name, US")"); +else if (host_lookup_deferred) + g = string_catn(g, US";\n\tiprev=temperror", 19); +else if (host_lookup_failed) + g = string_catn(g, US";\n\tiprev=fail", 13); else - { - uschar *decoded, *error; - if (ptr > yield && ptr[-1] == '\n') ptr--; - if (ptr > yield && comma != 0 && ptr[-1] == ',') ptr--; - *ptr = 0; - decoded = rfc2047_decode2(yield, check_rfc2047_length, charset, '?', NULL, - newsize, &error); - if (error != NULL) - { - DEBUG(D_any) debug_printf("*** error in RFC 2047 decoding: %s\n" - " input was: %s\n", error, yield); - } - if (decoded != NULL) yield = decoded; - } + return g; -return yield; +if (sender_host_address) + g = string_append(g, 2, US" smtp.remote-ip=", sender_host_address); +return g; } - /************************************************* * Return list of recipients * *************************************************/ @@ -1666,18 +1726,17 @@ generated from a system filter, but not elsewhere. */ static uschar * fn_recipients(void) { +uschar * s; gstring * g = NULL; -int i; -if (!enable_dollar_recipients) return NULL; +if (!f.enable_dollar_recipients) return NULL; -for (i = 0; i < recipients_count; i++) +for (int i = 0; i < recipients_count; i++) { - /*XXX variant of list_appendele? */ - if (i != 0) g = string_catn(g, US", ", 2); - g = string_cat(g, recipients_list[i].address); + s = recipients_list[i].address; + g = string_append2_listele_n(g, US", ", s, Ustrlen(s)); } -return string_from_gstring(g); +return g ? g->s : NULL; } @@ -1725,8 +1784,13 @@ set, in which case give an error. */ if ((Ustrncmp(name, "acl_c", 5) == 0 || Ustrncmp(name, "acl_m", 5) == 0) && !isalpha(name[5])) { - tree_node *node = - tree_search((name[4] == 'c')? acl_var_c : acl_var_m, name + 4); + tree_node * node = + tree_search(name[4] == 'c' ? acl_var_c : acl_var_m, name + 4); + return node ? node->data.ptr : strict_acl_vars ? NULL : US""; + } +else if (Ustrncmp(name, "r_", 2) == 0) + { + tree_node * node = tree_search(router_var, name + 2); return node ? node->data.ptr : strict_acl_vars ? NULL : US""; } @@ -1762,7 +1826,7 @@ val = vp->value; switch (vp->type) { case vtype_filter_int: - if (!filter_running) return NULL; + if (!f.filter_running) return NULL; /* Fall through */ /* VVVVVVVVVVVV */ case vtype_int: @@ -1797,34 +1861,30 @@ switch (vp->type) return var_buffer; case vtype_host_lookup: /* Lookup if not done so */ - if (sender_host_name == NULL && sender_host_address != NULL && - !host_lookup_failed && host_name_lookup() == OK) + if ( !sender_host_name && sender_host_address + && !host_lookup_failed && host_name_lookup() == OK) host_build_sender_fullhost(); - return (sender_host_name == NULL)? US"" : sender_host_name; + return sender_host_name ? sender_host_name : US""; case vtype_localpart: /* Get local part from address */ - s = *((uschar **)(val)); - if (s == NULL) return US""; - domain = Ustrrchr(s, '@'); - if (domain == NULL) return s; + if (!(s = *((uschar **)(val)))) return US""; + if (!(domain = Ustrrchr(s, '@'))) return s; if (domain - s > sizeof(var_buffer) - 1) log_write(0, LOG_MAIN|LOG_PANIC_DIE, "local part longer than " SIZE_T_FMT " in string expansion", sizeof(var_buffer)); - Ustrncpy(var_buffer, s, domain - s); - var_buffer[domain - s] = 0; - return var_buffer; + return string_copyn(s, domain - s); case vtype_domain: /* Get domain from address */ - s = *((uschar **)(val)); - if (s == NULL) return US""; + if (!(s = *((uschar **)(val)))) return US""; domain = Ustrrchr(s, '@'); - return (domain == NULL)? US"" : domain + 1; + return domain ? domain + 1 : US""; case vtype_msgheaders: - return find_header(NULL, exists_only, newsize, FALSE, NULL); + return find_header(NULL, newsize, exists_only ? FH_EXISTS_ONLY : 0, NULL); case vtype_msgheaders_raw: - return find_header(NULL, exists_only, newsize, TRUE, NULL); + return find_header(NULL, newsize, + exists_only ? FH_EXISTS_ONLY|FH_WANT_RAW : FH_WANT_RAW, NULL); case vtype_msgbody: /* Pointer to msgbody string */ case vtype_msgbody_end: /* Ditto, the end of the msg */ @@ -1889,15 +1949,18 @@ switch (vp->type) return tod_stamp(tod_log_datestamp_daily); case vtype_reply: /* Get reply address */ - s = find_header(US"reply-to:", exists_only, newsize, TRUE, - headers_charset); - if (s != NULL) while (isspace(*s)) s++; - if (s == NULL || *s == 0) + s = find_header(US"reply-to:", newsize, + exists_only ? FH_EXISTS_ONLY|FH_WANT_RAW : FH_WANT_RAW, + headers_charset); + if (s) while (isspace(*s)) s++; + if (!s || !*s) { *newsize = 0; /* For the *s==0 case */ - s = find_header(US"from:", exists_only, newsize, TRUE, headers_charset); + s = find_header(US"from:", newsize, + exists_only ? FH_EXISTS_ONLY|FH_WANT_RAW : FH_WANT_RAW, + headers_charset); } - if (s != NULL) + if (s) { uschar *t; while (isspace(*s)) s++; @@ -1905,18 +1968,18 @@ switch (vp->type) while (t > s && isspace(t[-1])) t--; *t = 0; } - return (s == NULL)? US"" : s; + return s ? s : US""; case vtype_string_func: { - uschar * (*fn)() = val; + stringptr_fn_t * fn = (stringptr_fn_t *) val; return fn(); } case vtype_pspace: { int inodes; - sprintf(CS var_buffer, "%d", + sprintf(CS var_buffer, PR_EXIM_ARITH, receive_statvfs(val == (void *)TRUE, &inodes)); } return var_buffer; @@ -1987,11 +2050,10 @@ static int read_subs(uschar **sub, int n, int m, const uschar **sptr, BOOL skipping, BOOL check_end, uschar *name, BOOL *resetok) { -int i; const uschar *s = *sptr; while (isspace(*s)) s++; -for (i = 0; i < n; i++) +for (int i = 0; i < n; i++) { if (*s != '{') { @@ -2107,6 +2169,260 @@ return ret; +/* Return pointer to dewrapped string, with enclosing specified chars removed. +The given string is modified on return. Leading whitespace is skipped while +looking for the opening wrap character, then the rest is scanned for the trailing +(non-escaped) wrap character. A backslash in the string will act as an escape. + +A nul is written over the trailing wrap, and a pointer to the char after the +leading wrap is returned. + +Arguments: + s String for de-wrapping + wrap Two-char string, the first being the opener, second the closer wrapping + character +Return: + Pointer to de-wrapped string, or NULL on error (with expand_string_message set). +*/ + +static uschar * +dewrap(uschar * s, const uschar * wrap) +{ +uschar * p = s; +unsigned depth = 0; +BOOL quotesmode = wrap[0] == wrap[1]; + +while (isspace(*p)) p++; + +if (*p == *wrap) + { + s = ++p; + wrap++; + while (*p) + { + if (*p == '\\') p++; + else if (!quotesmode && *p == wrap[-1]) depth++; + else if (*p == *wrap) + if (depth == 0) + { + *p = '\0'; + return s; + } + else + depth--; + p++; + } + } +expand_string_message = string_sprintf("missing '%c'", *wrap); +return NULL; +} + + +/* Pull off the leading array or object element, returning +a copy in an allocated string. Update the list pointer. + +The element may itself be an abject or array. +Return NULL when the list is empty. +*/ + +static uschar * +json_nextinlist(const uschar ** list) +{ +unsigned array_depth = 0, object_depth = 0; +const uschar * s = *list, * item; + +while (isspace(*s)) s++; + +for (item = s; + *s && (*s != ',' || array_depth != 0 || object_depth != 0); + s++) + switch (*s) + { + case '[': array_depth++; break; + case ']': array_depth--; break; + case '{': object_depth++; break; + case '}': object_depth--; break; + } +*list = *s ? s+1 : s; +if (item == s) return NULL; +item = string_copyn(item, s - item); +DEBUG(D_expand) debug_printf_indent(" json ele: '%s'\n", item); +return US item; +} + + + +/************************************************/ +/* Return offset in ops table, or -1 if not found. +Repoint to just after the operator in the string. + +Argument: + ss string representation of operator + opname split-out operator name +*/ + +static int +identify_operator(const uschar ** ss, uschar ** opname) +{ +const uschar * s = *ss; +uschar name[256]; + +/* Numeric comparisons are symbolic */ + +if (*s == '=' || *s == '>' || *s == '<') + { + int p = 0; + name[p++] = *s++; + if (*s == '=') + { + name[p++] = '='; + s++; + } + name[p] = 0; + } + +/* All other conditions are named */ + +else + s = read_name(name, sizeof(name), s, US"_"); +*ss = s; + +/* If we haven't read a name, it means some non-alpha character is first. */ + +if (!name[0]) + { + expand_string_message = string_sprintf("condition name expected, " + "but found \"%.16s\"", s); + return -1; + } +if (opname) + *opname = string_copy(name); + +return chop_match(name, cond_table, nelem(cond_table)); +} + + +/************************************************* +* Handle MD5 or SHA-1 computation for HMAC * +*************************************************/ + +/* These are some wrapping functions that enable the HMAC code to be a bit +cleaner. A good compiler will spot the tail recursion. + +Arguments: + type HMAC_MD5 or HMAC_SHA1 + remaining are as for the cryptographic hash functions + +Returns: nothing +*/ + +static void +chash_start(int type, void * base) +{ +if (type == HMAC_MD5) + md5_start((md5 *)base); +else + sha1_start((hctx *)base); +} + +static void +chash_mid(int type, void * base, const uschar * string) +{ +if (type == HMAC_MD5) + md5_mid((md5 *)base, string); +else + sha1_mid((hctx *)base, string); +} + +static void +chash_end(int type, void * base, const uschar * string, int length, + uschar * digest) +{ +if (type == HMAC_MD5) + md5_end((md5 *)base, string, length, digest); +else + sha1_end((hctx *)base, string, length, digest); +} + + + + +/* Do an hmac_md5. The result is _not_ nul-terminated, and is sized as +the smaller of a full hmac_md5 result (16 bytes) or the supplied output buffer. + +Arguments: + key encoding key, nul-terminated + src data to be hashed, nul-terminated + buf output buffer + len size of output buffer +*/ + +static void +hmac_md5(const uschar * key, const uschar * src, uschar * buf, unsigned len) +{ +md5 md5_base; +const uschar * keyptr; +uschar * p; +unsigned int keylen; + +#define MD5_HASHLEN 16 +#define MD5_HASHBLOCKLEN 64 + +uschar keyhash[MD5_HASHLEN]; +uschar innerhash[MD5_HASHLEN]; +uschar finalhash[MD5_HASHLEN]; +uschar innerkey[MD5_HASHBLOCKLEN]; +uschar outerkey[MD5_HASHBLOCKLEN]; + +keyptr = key; +keylen = Ustrlen(keyptr); + +/* If the key is longer than the hash block length, then hash the key +first */ + +if (keylen > MD5_HASHBLOCKLEN) + { + chash_start(HMAC_MD5, &md5_base); + chash_end(HMAC_MD5, &md5_base, keyptr, keylen, keyhash); + keyptr = keyhash; + keylen = MD5_HASHLEN; + } + +/* Now make the inner and outer key values */ + +memset(innerkey, 0x36, MD5_HASHBLOCKLEN); +memset(outerkey, 0x5c, MD5_HASHBLOCKLEN); + +for (int i = 0; i < keylen; i++) + { + innerkey[i] ^= keyptr[i]; + outerkey[i] ^= keyptr[i]; + } + +/* Now do the hashes */ + +chash_start(HMAC_MD5, &md5_base); +chash_mid(HMAC_MD5, &md5_base, innerkey); +chash_end(HMAC_MD5, &md5_base, src, Ustrlen(src), innerhash); + +chash_start(HMAC_MD5, &md5_base); +chash_mid(HMAC_MD5, &md5_base, outerkey); +chash_end(HMAC_MD5, &md5_base, innerhash, MD5_HASHLEN, finalhash); + +/* Encode the final hash as a hex string, limited by output buffer size */ + +p = buf; +for (int i = 0, j = len; i < MD5_HASHLEN; i++) + { + if (j-- <= 0) break; + *p++ = hex_digits[(finalhash[i] & 0xf0) >> 4]; + if (j-- <= 0) break; + *p++ = hex_digits[finalhash[i] & 0x0f]; + } +return; +} + + /************************************************* * Read and evaluate a condition * *************************************************/ @@ -2133,9 +2449,11 @@ BOOL testfor = TRUE; BOOL tempcond, combined_cond; BOOL *subcondptr; BOOL sub2_honour_dollar = TRUE; -int i, rc, cond_type, roffset; +BOOL is_forany, is_json, is_jsons; +int rc, cond_type, roffset; int_eximarith_t num[2]; struct stat statbuf; +uschar * opname; uschar name[256]; const uschar *sub[10]; @@ -2148,99 +2466,71 @@ for (;;) if (*s == '!') { testfor = !testfor; s++; } else break; } -/* Numeric comparisons are symbolic */ - -if (*s == '=' || *s == '>' || *s == '<') - { - int p = 0; - name[p++] = *s++; - if (*s == '=') - { - name[p++] = '='; - s++; - } - name[p] = 0; - } - -/* All other conditions are named */ - -else s = read_name(name, 256, s, US"_"); - -/* If we haven't read a name, it means some non-alpha character is first. */ - -if (name[0] == 0) - { - expand_string_message = string_sprintf("condition name expected, " - "but found \"%.16s\"", s); - return NULL; - } - -/* Find which condition we are dealing with, and switch on it */ - -cond_type = chop_match(name, cond_table, nelem(cond_table)); -switch(cond_type) +switch(cond_type = identify_operator(&s, &opname)) { /* def: tests for a non-empty variable, or for the existence of a header. If yield == NULL we are in a skipping state, and don't care about the answer. */ case ECOND_DEF: - if (*s != ':') { - expand_string_message = US"\":\" expected after \"def\""; - return NULL; - } + uschar * t; - s = read_name(name, 256, s+1, US"_"); + if (*s != ':') + { + expand_string_message = US"\":\" expected after \"def\""; + return NULL; + } - /* Test for a header's existence. If the name contains a closing brace - character, this may be a user error where the terminating colon has been - omitted. Set a flag to adjust a subsequent error message in this case. */ + s = read_name(name, sizeof(name), s+1, US"_"); - if (Ustrncmp(name, "h_", 2) == 0 || - Ustrncmp(name, "rh_", 3) == 0 || - Ustrncmp(name, "bh_", 3) == 0 || - Ustrncmp(name, "header_", 7) == 0 || - Ustrncmp(name, "rheader_", 8) == 0 || - Ustrncmp(name, "bheader_", 8) == 0) - { - s = read_header_name(name, 256, s); - /* {-for-text-editors */ - if (Ustrchr(name, '}') != NULL) malformed_header = TRUE; - if (yield != NULL) *yield = - (find_header(name, TRUE, NULL, FALSE, NULL) != NULL) == testfor; - } + /* Test for a header's existence. If the name contains a closing brace + character, this may be a user error where the terminating colon has been + omitted. Set a flag to adjust a subsequent error message in this case. */ + + if ( ( *(t = name) == 'h' + || (*t == 'r' || *t == 'l' || *t == 'b') && *++t == 'h' + ) + && (*++t == '_' || Ustrncmp(t, "eader_", 6) == 0) + ) + { + s = read_header_name(name, sizeof(name), s); + /* {-for-text-editors */ + if (Ustrchr(name, '}') != NULL) malformed_header = TRUE; + if (yield) *yield = + (find_header(name, NULL, FH_EXISTS_ONLY, NULL) != NULL) == testfor; + } - /* Test for a variable's having a non-empty value. A non-existent variable - causes an expansion failure. */ + /* Test for a variable's having a non-empty value. A non-existent variable + causes an expansion failure. */ - else - { - uschar *value = find_variable(name, TRUE, yield == NULL, NULL); - if (value == NULL) + else { - expand_string_message = (name[0] == 0)? - string_sprintf("variable name omitted after \"def:\"") : - string_sprintf("unknown variable \"%s\" after \"def:\"", name); - check_variable_error_message(name); - return NULL; + if (!(t = find_variable(name, TRUE, yield == NULL, NULL))) + { + expand_string_message = name[0] + ? string_sprintf("unknown variable \"%s\" after \"def:\"", name) + : US"variable name omitted after \"def:\""; + check_variable_error_message(name); + return NULL; + } + if (yield) *yield = (t[0] != 0) == testfor; } - if (yield != NULL) *yield = (value[0] != 0) == testfor; - } - return s; + return s; + } /* first_delivery tests for first delivery attempt */ case ECOND_FIRST_DELIVERY: - if (yield != NULL) *yield = deliver_firsttime == testfor; + if (yield) *yield = f.deliver_firsttime == testfor; return s; /* queue_running tests for any process started by a queue runner */ case ECOND_QUEUE_RUNNING: - if (yield != NULL) *yield = (queue_run_pid != (pid_t)0) == testfor; + if (yield) *yield = (queue_run_pid != (pid_t)0) == testfor; return s; @@ -2267,11 +2557,11 @@ switch(cond_type) if (*s != '{') goto COND_FAILED_CURLY_START; /* }-for-text-editors */ sub[0] = expand_string_internal(s+1, TRUE, &s, yield == NULL, TRUE, resetok); - if (sub[0] == NULL) return NULL; + if (!sub[0]) return NULL; /* {-for-text-editors */ if (*s++ != '}') goto COND_FAILED_CURLY_END; - if (yield == NULL) return s; /* No need to run the test if skipping */ + if (!yield) return s; /* No need to run the test if skipping */ switch(cond_type) { @@ -2373,10 +2663,11 @@ switch(cond_type) case 3: return NULL; } - if (yield != NULL) + if (yield) { + int rc; *resetok = FALSE; /* eval_acl() might allocate; do not reclaim */ - switch(eval_acl(sub, nelem(sub), &user_msg)) + switch(rc = eval_acl(sub, nelem(sub), &user_msg)) { case OK: cond = TRUE; @@ -2388,10 +2679,11 @@ switch(cond_type) break; case DEFER: - expand_string_forcedfail = TRUE; + f.expand_string_forcedfail = TRUE; /*FALLTHROUGH*/ default: - expand_string_message = string_sprintf("error from acl \"%s\"", sub[0]); + expand_string_message = string_sprintf("%s from acl \"%s\"", + rc_names[rc], sub[0]); return NULL; } } @@ -2422,8 +2714,8 @@ switch(cond_type) case 2: case 3: return NULL; } - if (sub[2] == NULL) sub[3] = NULL; /* realm if no service */ - if (yield != NULL) + if (!sub[2]) sub[3] = NULL; /* realm if no service */ + if (yield) { int rc = auth_call_saslauthd(sub[0], sub[1], sub[2], sub[3], &expand_string_message); @@ -2481,7 +2773,7 @@ switch(cond_type) case ECOND_STR_GE: case ECOND_STR_GEI: - for (i = 0; i < 2; i++) + for (int i = 0; i < 2; i++) { /* Sometimes, we don't expand substrings; too many insecure configurations created using match_address{}{} and friends, where the second param @@ -2495,19 +2787,22 @@ switch(cond_type) { if (i == 0) goto COND_FAILED_CURLY_START; expand_string_message = string_sprintf("missing 2nd string in {} " - "after \"%s\"", name); + "after \"%s\"", opname); return NULL; } - sub[i] = expand_string_internal(s+1, TRUE, &s, yield == NULL, - honour_dollar, resetok); - if (sub[i] == NULL) return NULL; + if (!(sub[i] = expand_string_internal(s+1, TRUE, &s, yield == NULL, + honour_dollar, resetok))) + return NULL; + DEBUG(D_expand) if (i == 1 && !sub2_honour_dollar && Ustrchr(sub[1], '$')) + debug_printf_indent("WARNING: the second arg is NOT expanded," + " for security reasons\n"); if (*s++ != '}') goto COND_FAILED_CURLY_END; /* Convert to numerical if required; we know that the names of all the conditions that compare numbers do not start with a letter. This just saves checking for them individually. */ - if (!isalpha(name[0]) && yield != NULL) + if (!isalpha(opname[0]) && yield) if (sub[i][0] == 0) { num[i] = 0; @@ -2517,13 +2812,13 @@ switch(cond_type) else { num[i] = expanded_string_integer(sub[i], FALSE); - if (expand_string_message != NULL) return NULL; + if (expand_string_message) return NULL; } } /* Result not required */ - if (yield == NULL) return s; + if (!yield) return s; /* Do an appropriate comparison */ @@ -2591,9 +2886,8 @@ switch(cond_type) break; case ECOND_MATCH: /* Regular expression match */ - re = pcre_compile(CS sub[1], PCRE_COPT, (const char **)&rerror, &roffset, - NULL); - if (re == NULL) + if (!(re = pcre_compile(CS sub[1], PCRE_COPT, CCSS &rerror, + &roffset, NULL))) { expand_string_message = string_sprintf("regular expression error in " "\"%s\": %s at offset %d", sub[1], rerror, roffset); @@ -2695,16 +2989,15 @@ switch(cond_type) if (sublen == 24) { - uschar *coded = b64encode(digest, 16); + uschar *coded = b64encode(CUS digest, 16); DEBUG(D_auth) debug_printf("crypteq: using MD5+B64 hashing\n" " subject=%s\n crypted=%s\n", coded, sub[1]+5); tempcond = (Ustrcmp(coded, sub[1]+5) == 0); } else if (sublen == 32) { - int i; uschar coded[36]; - for (i = 0; i < 16; i++) sprintf(CS (coded+2*i), "%02X", digest[i]); + for (int i = 0; i < 16; i++) sprintf(CS (coded+2*i), "%02X", digest[i]); coded[32] = 0; DEBUG(D_auth) debug_printf("crypteq: using MD5+hex hashing\n" " subject=%s\n crypted=%s\n", coded, sub[1]+5); @@ -2733,16 +3026,15 @@ switch(cond_type) if (sublen == 28) { - uschar *coded = b64encode(digest, 20); + uschar *coded = b64encode(CUS digest, 20); DEBUG(D_auth) debug_printf("crypteq: using SHA1+B64 hashing\n" " subject=%s\n crypted=%s\n", coded, sub[1]+6); tempcond = (Ustrcmp(coded, sub[1]+6) == 0); } else if (sublen == 40) { - int i; uschar coded[44]; - for (i = 0; i < 20; i++) sprintf(CS (coded+2*i), "%02X", digest[i]); + for (int i = 0; i < 20; i++) sprintf(CS (coded+2*i), "%02X", digest[i]); coded[40] = 0; DEBUG(D_auth) debug_printf("crypteq: using SHA1+hex hashing\n" " subject=%s\n crypted=%s\n", coded, sub[1]+6); @@ -2821,7 +3113,7 @@ switch(cond_type) uschar *save_iterate_item = iterate_item; int (*compare)(const uschar *, const uschar *); - DEBUG(D_expand) debug_printf_indent("condition: %s item: %s\n", name, sub[0]); + DEBUG(D_expand) debug_printf_indent("condition: %s item: %s\n", opname, sub[0]); tempcond = FALSE; compare = cond_type == ECOND_INLISTI @@ -2849,7 +3141,7 @@ switch(cond_type) case ECOND_AND: case ECOND_OR: - subcondptr = (yield == NULL)? NULL : &tempcond; + subcondptr = (yield == NULL) ? NULL : &tempcond; combined_cond = (cond_type == ECOND_AND); while (isspace(*s)) s++; @@ -2863,14 +3155,14 @@ switch(cond_type) if (*s != '{') /* }-for-text-editors */ { expand_string_message = string_sprintf("each subcondition " - "inside an \"%s{...}\" condition must be in its own {}", name); + "inside an \"%s{...}\" condition must be in its own {}", opname); return NULL; } if (!(s = eval_condition(s+1, resetok, subcondptr))) { expand_string_message = string_sprintf("%s inside \"%s{...}\" condition", - expand_string_message, name); + expand_string_message, opname); return NULL; } while (isspace(*s)) s++; @@ -2880,12 +3172,11 @@ switch(cond_type) { /* {-for-text-editors */ expand_string_message = string_sprintf("missing } at end of condition " - "inside \"%s\" group", name); + "inside \"%s\" group", opname); return NULL; } - if (yield != NULL) - { + if (yield) if (cond_type == ECOND_AND) { combined_cond &= tempcond; @@ -2896,28 +3187,33 @@ switch(cond_type) combined_cond |= tempcond; if (combined_cond) subcondptr = NULL; /* once true, don't */ } /* evaluate any more */ - } } - if (yield != NULL) *yield = (combined_cond == testfor); + if (yield) *yield = (combined_cond == testfor); return ++s; /* forall/forany: iterates a condition with different values */ - case ECOND_FORALL: - case ECOND_FORANY: + case ECOND_FORALL: is_forany = FALSE; is_json = FALSE; is_jsons = FALSE; goto FORMANY; + case ECOND_FORANY: is_forany = TRUE; is_json = FALSE; is_jsons = FALSE; goto FORMANY; + case ECOND_FORALL_JSON: is_forany = FALSE; is_json = TRUE; is_jsons = FALSE; goto FORMANY; + case ECOND_FORANY_JSON: is_forany = TRUE; is_json = TRUE; is_jsons = FALSE; goto FORMANY; + case ECOND_FORALL_JSONS: is_forany = FALSE; is_json = TRUE; is_jsons = TRUE; goto FORMANY; + case ECOND_FORANY_JSONS: is_forany = TRUE; is_json = TRUE; is_jsons = TRUE; goto FORMANY; + + FORMANY: { const uschar * list; int sep = 0; uschar *save_iterate_item = iterate_item; - DEBUG(D_expand) debug_printf_indent("condition: %s\n", name); + DEBUG(D_expand) debug_printf_indent("condition: %s\n", opname); while (isspace(*s)) s++; if (*s++ != '{') goto COND_FAILED_CURLY_START; /* }-for-text-editors */ - sub[0] = expand_string_internal(s, TRUE, &s, (yield == NULL), TRUE, resetok); - if (sub[0] == NULL) return NULL; + if (!(sub[0] = expand_string_internal(s, TRUE, &s, yield == NULL, TRUE, resetok))) + return NULL; /* {-for-text-editors */ if (*s++ != '}') goto COND_FAILED_CURLY_END; @@ -2933,7 +3229,7 @@ switch(cond_type) if (!(s = eval_condition(sub[1], resetok, NULL))) { expand_string_message = string_sprintf("%s inside \"%s\" condition", - expand_string_message, name); + expand_string_message, opname); return NULL; } while (isspace(*s)) s++; @@ -2943,27 +3239,39 @@ switch(cond_type) { /* {-for-text-editors */ expand_string_message = string_sprintf("missing } at end of condition " - "inside \"%s\"", name); + "inside \"%s\"", opname); return NULL; } - if (yield != NULL) *yield = !testfor; + if (yield) *yield = !testfor; list = sub[0]; - while ((iterate_item = string_nextinlist(&list, &sep, NULL, 0)) != NULL) + if (is_json) list = dewrap(string_copy(list), US"[]"); + while ((iterate_item = is_json + ? json_nextinlist(&list) : string_nextinlist(&list, &sep, NULL, 0))) { - DEBUG(D_expand) debug_printf_indent("%s: $item = \"%s\"\n", name, iterate_item); + if (is_jsons) + if (!(iterate_item = dewrap(iterate_item, US"\"\""))) + { + expand_string_message = + string_sprintf("%s wrapping string result for extract jsons", + expand_string_message); + iterate_item = save_iterate_item; + return NULL; + } + + DEBUG(D_expand) debug_printf_indent("%s: $item = \"%s\"\n", opname, iterate_item); if (!eval_condition(sub[1], resetok, &tempcond)) { expand_string_message = string_sprintf("%s inside \"%s\" condition", - expand_string_message, name); + expand_string_message, opname); iterate_item = save_iterate_item; return NULL; } - DEBUG(D_expand) debug_printf_indent("%s: condition evaluated to %s\n", name, + DEBUG(D_expand) debug_printf_indent("%s: condition evaluated to %s\n", opname, tempcond? "true":"false"); - if (yield != NULL) *yield = (tempcond == testfor); - if (tempcond == (cond_type == ECOND_FORANY)) break; + if (yield) *yield = (tempcond == testfor); + if (tempcond == is_forany) break; } iterate_item = save_iterate_item; @@ -3045,26 +3353,121 @@ switch(cond_type) } DEBUG(D_expand) debug_printf_indent("%s: condition evaluated to %s\n", ourname, boolvalue? "true":"false"); - if (yield != NULL) *yield = (boolvalue == testfor); + if (yield) *yield = (boolvalue == testfor); return s; } +#ifdef EXPERIMENTAL_SRS_NATIVE + case ECOND_INBOUND_SRS: + /* ${if inbound_srs {local_part}{secret} {yes}{no}} */ + { + uschar * sub[2]; + const pcre * re; + int ovec[3*(4+1)]; + int n; + uschar cksum[4]; + BOOL boolvalue = FALSE; + + switch(read_subs(sub, 2, 2, CUSS &s, yield == NULL, FALSE, US"inbound_srs", resetok)) + { + case 1: expand_string_message = US"too few arguments or bracketing " + "error for inbound_srs"; + case 2: + case 3: return NULL; + } + + /* Match the given local_part against the SRS-encoded pattern */ + + re = regex_must_compile(US"^(?i)SRS0=([^=]+)=([A-Z2-7]+)=([^=]*)=(.*)$", + TRUE, FALSE); + if (pcre_exec(re, NULL, CS sub[0], Ustrlen(sub[0]), 0, PCRE_EOPT, + ovec, nelem(ovec)) < 0) + { + DEBUG(D_expand) debug_printf("no match for SRS'd local-part pattern\n"); + goto srs_result; + } + + /* Side-effect: record the decoded recipient */ + + srs_recipient = string_sprintf("%.*S@%.*S", /* lowercased */ + ovec[9]-ovec[8], sub[0] + ovec[8], /* substring 4 */ + ovec[7]-ovec[6], sub[0] + ovec[6]); /* substring 3 */ + + /* If a zero-length secret was given, we're done. Otherwise carry on + and validate the given SRS local_part againt our secret. */ + + if (!*sub[1]) + { + boolvalue = TRUE; + goto srs_result; + } + + /* check the timestamp */ + { + struct timeval now; + uschar * ss = sub[0] + ovec[4]; /* substring 2, the timestamp */ + long d; + + gettimeofday(&now, NULL); + now.tv_sec /= 86400; /* days since epoch */ + + /* Decode substring 2 from base32 to a number */ + + for (d = 0, n = ovec[5]-ovec[4]; n; n--) + { + uschar * t = Ustrchr(base32_chars, *ss++); + d = d * 32 + (t - base32_chars); + } + + if (((now.tv_sec - d) & 0x3ff) > 10) /* days since SRS generated */ + { + DEBUG(D_expand) debug_printf("SRS too old\n"); + goto srs_result; + } + } + + /* check length of substring 1, the offered checksum */ + + if (ovec[3]-ovec[2] != 4) + { + DEBUG(D_expand) debug_printf("SRS checksum wrong size\n"); + goto srs_result; + } + + /* Hash the address with our secret, and compare that computed checksum + with the one extracted from the arg */ + + hmac_md5(sub[1], srs_recipient, cksum, sizeof(cksum)); + if (Ustrncmp(cksum, sub[0] + ovec[2], 4) != 0) + { + DEBUG(D_expand) debug_printf("SRS checksum mismatch\n"); + goto srs_result; + } + boolvalue = TRUE; + +srs_result: + if (yield) *yield = (boolvalue == testfor); + return s; + } +#endif /*EXPERIMENTAL_SRS_NATIVE*/ + /* Unknown condition */ default: - expand_string_message = string_sprintf("unknown condition \"%s\"", name); - return NULL; + if (!expand_string_message || !*expand_string_message) + expand_string_message = string_sprintf("unknown condition \"%s\"", opname); + return NULL; } /* End switch on condition type */ /* Missing braces at start and end of data */ COND_FAILED_CURLY_START: -expand_string_message = string_sprintf("missing { after \"%s\"", name); +expand_string_message = string_sprintf("missing { after \"%s\"", opname); return NULL; COND_FAILED_CURLY_END: expand_string_message = string_sprintf("missing } at end of \"%s\" condition", - name); + opname); return NULL; /* A condition requires code that is not compiled */ @@ -3074,7 +3477,7 @@ return NULL; !defined(SUPPORT_CRYPTEQ) || !defined(CYRUS_SASLAUTHD_SOCKET) COND_FAILED_NOT_COMPILED: expand_string_message = string_sprintf("support for \"%s\" not compiled", - name); + opname); return NULL; #endif } @@ -3099,8 +3502,7 @@ Returns: the value of expand max to save static int save_expand_strings(uschar **save_expand_nstring, int *save_expand_nlength) { -int i; -for (i = 0; i <= expand_nmax; i++) +for (int i = 0; i <= expand_nmax; i++) { save_expand_nstring[i] = expand_nstring[i]; save_expand_nlength[i] = expand_nlength[i]; @@ -3128,9 +3530,8 @@ static void restore_expand_strings(int save_expand_nmax, uschar **save_expand_nstring, int *save_expand_nlength) { -int i; expand_nmax = save_expand_nmax; -for (i = 0; i <= expand_nmax; i++) +for (int i = 0; i <= expand_nmax; i++) { expand_nstring[i] = save_expand_nstring[i]; expand_nlength[i] = save_expand_nlength[i]; @@ -3212,8 +3613,8 @@ want this string. Set skipping in the call in the fail case (this will always be the case if we were already skipping). */ sub1 = expand_string_internal(s, TRUE, &s, !yes, TRUE, resetok); -if (sub1 == NULL && (yes || !expand_string_forcedfail)) goto FAILED; -expand_string_forcedfail = FALSE; +if (sub1 == NULL && (yes || !f.expand_string_forcedfail)) goto FAILED; +f.expand_string_forcedfail = FALSE; if (*s++ != '}') { errwhere = US"'yes' part did not end with '}'"; @@ -3242,8 +3643,8 @@ while (isspace(*s)) s++; if (*s == '{') { sub2 = expand_string_internal(s+1, TRUE, &s, yes || skipping, TRUE, resetok); - if (sub2 == NULL && (!yes || !expand_string_forcedfail)) goto FAILED; - expand_string_forcedfail = FALSE; + if (sub2 == NULL && (!yes || !f.expand_string_forcedfail)) goto FAILED; + f.expand_string_forcedfail = FALSE; if (*s++ != '}') { errwhere = US"'no' part did not start with '{'"; @@ -3278,7 +3679,7 @@ else if (*s != '}') } expand_string_message = string_sprintf("\"%s\" failed and \"fail\" requested", type); - expand_string_forcedfail = TRUE; + f.expand_string_forcedfail = TRUE; goto FAILED; } } @@ -3322,51 +3723,6 @@ FAILED: -/************************************************* -* Handle MD5 or SHA-1 computation for HMAC * -*************************************************/ - -/* These are some wrapping functions that enable the HMAC code to be a bit -cleaner. A good compiler will spot the tail recursion. - -Arguments: - type HMAC_MD5 or HMAC_SHA1 - remaining are as for the cryptographic hash functions - -Returns: nothing -*/ - -static void -chash_start(int type, void *base) -{ -if (type == HMAC_MD5) - md5_start((md5 *)base); -else - sha1_start((hctx *)base); -} - -static void -chash_mid(int type, void *base, uschar *string) -{ -if (type == HMAC_MD5) - md5_mid((md5 *)base, string); -else - sha1_mid((hctx *)base, string); -} - -static void -chash_end(int type, void *base, uschar *string, int length, uschar *digest) -{ -if (type == HMAC_MD5) - md5_end((md5 *)base, string, length, digest); -else - sha1_end((hctx *)base, string, length, digest); -} - - - - - /******************************************************** * prvs: Get last three digits of days since Jan 1, 1970 * ********************************************************/ @@ -3387,7 +3743,7 @@ Returns: pointer to string containing the last three static uschar * prvs_daystamp(int day_offset) { -uschar *days = store_get(32); /* Need at least 24 for cases */ +uschar *days = store_get(32, FALSE); /* Need at least 24 for cases */ (void)string_format(days, 32, TIME_T_FMT, /* where TIME_T_FMT is %lld */ (time(NULL) + day_offset*86400)/86400); return (Ustrlen(days) >= 3) ? &days[Ustrlen(days)-3] : US"100"; @@ -3419,15 +3775,14 @@ prvs_hmac_sha1(uschar *address, uschar *key, uschar *key_num, uschar *daystamp) { gstring * hash_source; uschar * p; -int i; hctx h; uschar innerhash[20]; uschar finalhash[20]; uschar innerkey[64]; uschar outerkey[64]; -uschar *finalhash_hex = store_get(40); +uschar *finalhash_hex; -if (key_num == NULL) +if (!key_num) key_num = US"0"; if (Ustrlen(key) > 64) @@ -3444,7 +3799,7 @@ DEBUG(D_expand) memset(innerkey, 0x36, 64); memset(outerkey, 0x5c, 64); -for (i = 0; i < Ustrlen(key); i++) +for (int i = 0; i < Ustrlen(key); i++) { innerkey[i] ^= key[i]; outerkey[i] ^= key[i]; @@ -3458,8 +3813,10 @@ chash_start(HMAC_SHA1, &h); chash_mid(HMAC_SHA1, &h, outerkey); chash_end(HMAC_SHA1, &h, innerhash, 20, finalhash); -p = finalhash_hex; -for (i = 0; i < 3; i++) +/* Hashing is deemed sufficient to de-taint any input data */ + +p = finalhash_hex = store_get(40, FALSE); +for (int i = 0; i < 3; i++) { *p++ = hex_digits[(finalhash[i] & 0xf0) >> 4]; *p++ = hex_digits[finalhash[i] & 0x0f]; @@ -3507,6 +3864,27 @@ return yield; } +#ifndef DISABLE_TLS +static gstring * +cat_file_tls(void * tls_ctx, gstring * yield, uschar * eol) +{ +int rc; +uschar buffer[1024]; + +/*XXX could we read direct into a pre-grown string? */ + +while ((rc = tls_read(tls_ctx, buffer, sizeof(buffer))) > 0) + for (uschar * s = buffer; rc--; s++) + yield = eol && *s == '\n' + ? string_cat(yield, eol) : string_catn(yield, s, 1); + +/* We assume that all errors, and any returns of zero bytes, +are actually EOF. */ + +(void) string_from_gstring(yield); +return yield; +} +#endif /************************************************* @@ -3535,17 +3913,15 @@ eval_expr(uschar **sptr, BOOL decimal, uschar **error, BOOL endket) { uschar *s = *sptr; int_eximarith_t x = eval_op_or(&s, decimal, error); -if (*error == NULL) - { + +if (!*error) if (endket) - { if (*s != ')') *error = US"expecting closing parenthesis"; else while (isspace(*(++s))); - } - else if (*s != 0) *error = US"expecting operator"; - } + else if (*s) + *error = US"expecting operator"; *sptr = s; return x; } @@ -3554,12 +3930,12 @@ return x; static int_eximarith_t eval_number(uschar **sptr, BOOL decimal, uschar **error) { -register int c; +int c; int_eximarith_t n; uschar *s = *sptr; + while (isspace(*s)) s++; -c = *s; -if (isdigit(c)) +if (isdigit((c = *s))) { int count; (void)sscanf(CS s, (decimal? SC_EXIM_DEC "%n" : SC_EXIM_ARITH "%n"), &n, &count); @@ -3602,9 +3978,8 @@ if (*s == '+' || *s == '-' || *s == '~') else if (op == '~') x = ~x; } else - { x = eval_number(&s, decimal, error); - } + *sptr = s; return x; } @@ -3615,13 +3990,13 @@ eval_op_mult(uschar **sptr, BOOL decimal, uschar **error) { uschar *s = *sptr; int_eximarith_t x = eval_op_unary(&s, decimal, error); -if (*error == NULL) +if (!*error) { while (*s == '*' || *s == '/' || *s == '%') { int op = *s++; int_eximarith_t y = eval_op_unary(&s, decimal, error); - if (*error != NULL) break; + if (*error) break; /* SIGFPE both on div/mod by zero and on INT_MIN / -1, which would give * a value of INT_MAX+1. Note that INT_MIN * -1 gives INT_MIN for me, which * is a bug somewhere in [gcc 4.2.1, FreeBSD, amd64]. In fact, -N*-M where @@ -3702,7 +4077,7 @@ eval_op_shift(uschar **sptr, BOOL decimal, uschar **error) { uschar *s = *sptr; int_eximarith_t x = eval_op_sum(&s, decimal, error); -if (*error == NULL) +if (!*error) { while ((*s == '<' || *s == '>') && s[1] == s[0]) { @@ -3710,7 +4085,7 @@ if (*error == NULL) int op = *s++; s++; y = eval_op_sum(&s, decimal, error); - if (*error != NULL) break; + if (*error) break; if (op == '<') x <<= y; else x >>= y; } } @@ -3724,14 +4099,14 @@ eval_op_and(uschar **sptr, BOOL decimal, uschar **error) { uschar *s = *sptr; int_eximarith_t x = eval_op_shift(&s, decimal, error); -if (*error == NULL) +if (!*error) { while (*s == '&') { int_eximarith_t y; s++; y = eval_op_shift(&s, decimal, error); - if (*error != NULL) break; + if (*error) break; x &= y; } } @@ -3745,14 +4120,14 @@ eval_op_xor(uschar **sptr, BOOL decimal, uschar **error) { uschar *s = *sptr; int_eximarith_t x = eval_op_and(&s, decimal, error); -if (*error == NULL) +if (!*error) { while (*s == '^') { int_eximarith_t y; s++; y = eval_op_and(&s, decimal, error); - if (*error != NULL) break; + if (*error) break; x ^= y; } } @@ -3766,23 +4141,73 @@ eval_op_or(uschar **sptr, BOOL decimal, uschar **error) { uschar *s = *sptr; int_eximarith_t x = eval_op_xor(&s, decimal, error); -if (*error == NULL) +if (!*error) + { + while (*s == '|') + { + int_eximarith_t y; + s++; + y = eval_op_xor(&s, decimal, error); + if (*error) break; + x |= y; + } + } +*sptr = s; +return x; +} + + + +/************************************************/ +/* Comparison operation for sort expansion. We need to avoid +re-expanding the fields being compared, so need a custom routine. + +Arguments: + cond_type Comparison operator code + leftarg, rightarg Arguments for comparison + +Return true iff (leftarg compare rightarg) +*/ + +static BOOL +sortsbefore(int cond_type, BOOL alpha_cond, + const uschar * leftarg, const uschar * rightarg) +{ +int_eximarith_t l_num, r_num; + +if (!alpha_cond) { - while (*s == '|') + l_num = expanded_string_integer(leftarg, FALSE); + if (expand_string_message) return FALSE; + r_num = expanded_string_integer(rightarg, FALSE); + if (expand_string_message) return FALSE; + + switch (cond_type) { - int_eximarith_t y; - s++; - y = eval_op_xor(&s, decimal, error); - if (*error != NULL) break; - x |= y; + case ECOND_NUM_G: return l_num > r_num; + case ECOND_NUM_GE: return l_num >= r_num; + case ECOND_NUM_L: return l_num < r_num; + case ECOND_NUM_LE: return l_num <= r_num; + default: break; } } -*sptr = s; -return x; +else + switch (cond_type) + { + case ECOND_STR_LT: return Ustrcmp (leftarg, rightarg) < 0; + case ECOND_STR_LTI: return strcmpic(leftarg, rightarg) < 0; + case ECOND_STR_LE: return Ustrcmp (leftarg, rightarg) <= 0; + case ECOND_STR_LEI: return strcmpic(leftarg, rightarg) <= 0; + case ECOND_STR_GT: return Ustrcmp (leftarg, rightarg) > 0; + case ECOND_STR_GTI: return strcmpic(leftarg, rightarg) > 0; + case ECOND_STR_GE: return Ustrcmp (leftarg, rightarg) >= 0; + case ECOND_STR_GEI: return strcmpic(leftarg, rightarg) >= 0; + default: break; + } +return FALSE; /* should not happen */ } - /************************************************* * Expand string * *************************************************/ @@ -3850,6 +4275,7 @@ static uschar * expand_string_internal(const uschar *string, BOOL ket_ends, const uschar **left, BOOL skipping, BOOL honour_dollar, BOOL *resetok_p) { +rmark reset_point = store_mark(); gstring * yield = string_get(Ustrlen(string) + 64); int item_type; const uschar *s = string; @@ -3859,15 +4285,27 @@ BOOL resetok = TRUE; expand_level++; DEBUG(D_expand) - debug_printf_indent(UTF8_DOWN_RIGHT "%s: %s\n", - skipping - ? UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ "scanning" - : "considering", - string); + DEBUG(D_noutf8) + debug_printf_indent("/%s: %s\n", + skipping ? "---scanning" : "considering", string); + else + debug_printf_indent(UTF8_DOWN_RIGHT "%s: %s\n", + skipping + ? UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ "scanning" + : "considering", + string); -expand_string_forcedfail = FALSE; +f.expand_string_forcedfail = FALSE; expand_string_message = US""; +if (is_tainted(string)) + { + expand_string_message = + string_sprintf("attempt to expand tainted string '%s'", s); + log_write(0, LOG_MAIN|LOG_PANIC, "%s", expand_string_message); + goto EXPAND_FAILED; + } + while (*s != 0) { uschar *value; @@ -3931,6 +4369,7 @@ while (*s != 0) int len; int newsize = 0; gstring * g = NULL; + uschar * t; s = read_name(name, sizeof(name), s, US"_"); @@ -3938,27 +4377,30 @@ while (*s != 0) buffer. */ if (!yield) - g = store_get(sizeof(gstring)); + g = store_get(sizeof(gstring), FALSE); else if (yield->ptr == 0) { - if (resetok) store_reset(yield); + if (resetok) reset_point = store_reset(reset_point); yield = NULL; - g = store_get(sizeof(gstring)); /* alloc _before_ calling find_variable() */ + reset_point = store_mark(); + g = store_get(sizeof(gstring), FALSE); /* alloc _before_ calling find_variable() */ } /* Header */ - if (Ustrncmp(name, "h_", 2) == 0 || - Ustrncmp(name, "rh_", 3) == 0 || - Ustrncmp(name, "bh_", 3) == 0 || - Ustrncmp(name, "header_", 7) == 0 || - Ustrncmp(name, "rheader_", 8) == 0 || - Ustrncmp(name, "bheader_", 8) == 0) + if ( ( *(t = name) == 'h' + || (*t == 'r' || *t == 'l' || *t == 'b') && *++t == 'h' + ) + && (*++t == '_' || Ustrncmp(t, "eader_", 6) == 0) + ) { - BOOL want_raw = (name[0] == 'r')? TRUE : FALSE; - uschar *charset = (name[0] == 'b')? NULL : headers_charset; + unsigned flags = *name == 'r' ? FH_WANT_RAW + : *name == 'l' ? FH_WANT_RAW|FH_WANT_LIST + : 0; + uschar * charset = *name == 'b' ? NULL : headers_charset; + s = read_header_name(name, sizeof(name), s); - value = find_header(name, FALSE, &newsize, want_raw, charset); + value = find_header(name, &newsize, flags, charset); /* If we didn't find the header, and the header contains a closing brace character, this may be a user error where the terminating colon @@ -3967,7 +4409,7 @@ while (*s != 0) if (!value) { - if (Ustrchr(name, '}') != NULL) malformed_header = TRUE; + if (Ustrchr(name, '}')) malformed_header = TRUE; continue; } } @@ -4067,6 +4509,7 @@ while (*s != 0) { uschar *sub[10]; /* name + arg1-arg9 (which must match number of acl_arg[]) */ uschar *user_msg; + int rc; switch(read_subs(sub, nelem(sub), 1, &s, skipping, TRUE, US"acl", &resetok)) @@ -4078,7 +4521,7 @@ while (*s != 0) if (skipping) continue; resetok = FALSE; - switch(eval_acl(sub, nelem(sub), &user_msg)) + switch(rc = eval_acl(sub, nelem(sub), &user_msg)) { case OK: case FAIL: @@ -4089,14 +4532,50 @@ while (*s != 0) continue; case DEFER: - expand_string_forcedfail = TRUE; + f.expand_string_forcedfail = TRUE; /*FALLTHROUGH*/ default: - expand_string_message = string_sprintf("error from acl \"%s\"", sub[0]); + expand_string_message = string_sprintf("%s from acl \"%s\"", + rc_names[rc], sub[0]); goto EXPAND_FAILED; } } + case EITEM_AUTHRESULTS: + /* ${authresults {mysystemname}} */ + { + uschar *sub_arg[1]; + + switch(read_subs(sub_arg, nelem(sub_arg), 1, &s, skipping, TRUE, name, + &resetok)) + { + case 1: goto EXPAND_FAILED_CURLY; + case 2: + case 3: goto EXPAND_FAILED; + } + + yield = string_append(yield, 3, + US"Authentication-Results: ", sub_arg[0], US"; none"); + yield->ptr -= 6; + + yield = authres_local(yield, sub_arg[0]); + yield = authres_iprev(yield); + yield = authres_smtpauth(yield); +#ifdef SUPPORT_SPF + yield = authres_spf(yield); +#endif +#ifndef DISABLE_DKIM + yield = authres_dkim(yield); +#endif +#ifdef SUPPORT_DMARC + yield = authres_dmarc(yield); +#endif +#ifdef EXPERIMENTAL_ARC + yield = authres_arc(yield); +#endif + continue; + } + /* Handle conditionals - preserve the values of the numerical expansion variables in case they get changed by a regular expression match in the condition. If not, they retain their external settings. At the end @@ -4110,19 +4589,25 @@ while (*s != 0) save_expand_strings(save_expand_nstring, save_expand_nlength); while (isspace(*s)) s++; - next_s = eval_condition(s, &resetok, skipping ? NULL : &cond); - if (next_s == NULL) goto EXPAND_FAILED; /* message already set */ + if (!(next_s = eval_condition(s, &resetok, skipping ? NULL : &cond))) + goto EXPAND_FAILED; /* message already set */ DEBUG(D_expand) - { - debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ - "condition: %.*s\n", - (int)(next_s - s), s); - debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ - UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ - "result: %s\n", - cond ? "true" : "false"); - } + DEBUG(D_noutf8) + { + debug_printf_indent("|--condition: %.*s\n", (int)(next_s - s), s); + debug_printf_indent("|-----result: %s\n", cond ? "true" : "false"); + } + else + { + debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ + "condition: %.*s\n", + (int)(next_s - s), s); + debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ + UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ + "result: %s\n", + cond ? "true" : "false"); + } s = next_s; @@ -4164,7 +4649,7 @@ while (*s != 0) case 3: goto EXPAND_FAILED; } - if (sub_arg[1] == NULL) /* One argument */ + if (!sub_arg[1]) /* One argument */ { sub_arg[1] = US"/"; /* default separator */ sub_arg[2] = NULL; @@ -4266,7 +4751,7 @@ while (*s != 0) if (!mac_islookup(stype, lookup_querystyle|lookup_absfilequery)) { - if (key == NULL) + if (!key) { expand_string_message = string_sprintf("missing {key} for single-" "key \"%s\" lookup", name); @@ -4275,7 +4760,7 @@ while (*s != 0) } else { - if (key != NULL) + if (key) { expand_string_message = string_sprintf("a single key was given for " "lookup type \"%s\", which is not a single-key lookup type", name); @@ -4293,8 +4778,8 @@ while (*s != 0) expand_string_message = US"missing '{' for lookup file-or-query arg"; goto EXPAND_FAILED_CURLY; } - filename = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok); - if (filename == NULL) goto EXPAND_FAILED; + if (!(filename = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok))) + goto EXPAND_FAILED; if (*s++ != '}') { expand_string_message = US"missing '}' closing lookup file-or-query arg"; @@ -4345,14 +4830,14 @@ while (*s != 0) else { void *handle = search_open(filename, stype, 0, NULL, NULL); - if (handle == NULL) + if (!handle) { expand_string_message = search_error_message; goto EXPAND_FAILED; } lookup_value = search_find(handle, filename, key, partial, affix, affixlen, starflags, &expand_setup); - if (search_find_defer) + if (f.search_find_defer) { expand_string_message = string_sprintf("lookup of \"%s\" gave DEFER: %s", @@ -4427,15 +4912,14 @@ while (*s != 0) if (!opt_perl_started) { uschar *initerror; - if (opt_perl_startup == NULL) + if (!opt_perl_startup) { expand_string_message = US"A setting of perl_startup is needed when " "using the Perl interpreter"; goto EXPAND_FAILED; } DEBUG(D_any) debug_printf("Starting Perl interpreter\n"); - initerror = init_perl(opt_perl_startup); - if (initerror != NULL) + if ((initerror = init_perl(opt_perl_startup))) { expand_string_message = string_sprintf("error in perl_startup code: %s\n", initerror); @@ -4454,14 +4938,14 @@ while (*s != 0) NULL, the yield was undef, indicating a forced failure. Otherwise the message will indicate some kind of Perl error. */ - if (new_yield == NULL) + if (!new_yield) { - if (expand_string_message == NULL) + if (!expand_string_message) { expand_string_message = string_sprintf("Perl subroutine \"%s\" returned undef to force " "failure", sub_arg[0]); - expand_string_forcedfail = TRUE; + f.expand_string_forcedfail = TRUE; } goto EXPAND_FAILED; } @@ -4469,7 +4953,7 @@ while (*s != 0) /* Yield succeeded. Ensure forcedfail is unset, just in case it got set during a callback from Perl. */ - expand_string_forcedfail = FALSE; + f.expand_string_forcedfail = FALSE; yield = new_yield; continue; } @@ -4494,25 +4978,25 @@ while (*s != 0) if (skipping) continue; /* sub_arg[0] is the address */ - domain = Ustrrchr(sub_arg[0],'@'); - if ( (domain == NULL) || (domain == sub_arg[0]) || (Ustrlen(domain) == 1) ) + if ( !(domain = Ustrrchr(sub_arg[0],'@')) + || domain == sub_arg[0] || Ustrlen(domain) == 1) { expand_string_message = US"prvs first argument must be a qualified email address"; goto EXPAND_FAILED; } - /* Calculate the hash. The second argument must be a single-digit + /* Calculate the hash. The third argument must be a single-digit key number, or unset. */ - if (sub_arg[2] != NULL && - (!isdigit(sub_arg[2][0]) || sub_arg[2][1] != 0)) + if ( sub_arg[2] + && (!isdigit(sub_arg[2][0]) || sub_arg[2][1] != 0)) { - expand_string_message = US"prvs second argument must be a single digit"; + expand_string_message = US"prvs third argument must be a single digit"; goto EXPAND_FAILED; } - p = prvs_hmac_sha1(sub_arg[0],sub_arg[1],sub_arg[2],prvs_daystamp(7)); - if (p == NULL) + p = prvs_hmac_sha1(sub_arg[0], sub_arg[1], sub_arg[2], prvs_daystamp(7)); + if (!p) { expand_string_message = US"prvs hmac-sha1 conversion failed"; goto EXPAND_FAILED; @@ -4619,7 +5103,7 @@ while (*s != 0) (void)sscanf(CS now,"%u",&inow); (void)sscanf(CS daystamp,"%u",&iexpire); - /* When "iexpire" is < 7, a "flip" has occured. + /* When "iexpire" is < 7, a "flip" has occurred. Adjust "inow" accordingly. */ if ( (iexpire < 7) && (inow >= 993) ) inow = 0; @@ -4628,7 +5112,7 @@ while (*s != 0) prvscheck_result = US"1"; DEBUG(D_expand) debug_printf_indent("prvscheck: success, $pvrs_result set to 1\n"); } - else + else { prvscheck_result = NULL; DEBUG(D_expand) debug_printf_indent("prvscheck: signature expired, $pvrs_result unset\n"); @@ -4716,13 +5200,16 @@ while (*s != 0) case EITEM_READSOCK: { - int fd; + client_conn_ctx cctx; int timeout = 5; int save_ptr = yield->ptr; - FILE *f; - uschar *arg; - uschar *sub_arg[4]; + FILE * fp = NULL; + uschar * arg; + uschar * sub_arg[4]; + uschar * server_name = NULL; + host_item host; BOOL do_shutdown = TRUE; + BOOL do_tls = FALSE; /* Only set under ! DISABLE_TLS */ blob reqstr; if (expand_forbid & RDO_READSOCK) @@ -4765,10 +5252,14 @@ while (*s != 0) while ((item = string_nextinlist(&list, &sep, NULL, 0))) if (Ustrncmp(item, US"shutdown=", 9) == 0) - if (Ustrcmp(item + 9, US"no") == 0) - do_shutdown = FALSE; + { if (Ustrcmp(item + 9, US"no") == 0) do_shutdown = FALSE; } +#ifndef DISABLE_TLS + else if (Ustrncmp(item, US"tls=", 4) == 0) + { if (Ustrcmp(item + 9, US"no") != 0) do_tls = TRUE; } +#endif } - else sub_arg[3] = NULL; /* No eol if no timeout */ + else + sub_arg[3] = NULL; /* No eol if no timeout */ /* If skipping, we don't actually do anything. Otherwise, arrange to connect to either an IP or a Unix socket. */ @@ -4780,8 +5271,10 @@ while (*s != 0) if (Ustrncmp(sub_arg[0], "inet:", 5) == 0) { int port; - uschar * server_name = sub_arg[0] + 5; - uschar * port_name = Ustrrchr(server_name, ':'); + uschar * port_name; + + server_name = sub_arg[0] + 5; + port_name = Ustrrchr(server_name, ':'); /* Sort out the port */ @@ -4816,12 +5309,15 @@ while (*s != 0) port = ntohs(service_info->s_port); } - fd = ip_connectedsocket(SOCK_STREAM, server_name, port, port, - timeout, NULL, &expand_string_message, &reqstr); + /*XXX we trust that the request is idempotent for TFO. Hmm. */ + cctx.sock = ip_connectedsocket(SOCK_STREAM, server_name, port, port, + timeout, &host, &expand_string_message, + do_tls ? NULL : &reqstr); callout_address = NULL; - if (fd < 0) - goto SOCK_FAIL; - reqstr.len = 0; + if (cctx.sock < 0) + goto SOCK_FAIL; + if (!do_tls) + reqstr.len = 0; } /* Handle a Unix domain socket */ @@ -4831,7 +5327,7 @@ while (*s != 0) struct sockaddr_un sockun; /* don't call this "sun" ! */ int rc; - if ((fd = socket(PF_UNIX, SOCK_STREAM, 0)) == -1) + if ((cctx.sock = socket(PF_UNIX, SOCK_STREAM, 0)) == -1) { expand_string_message = string_sprintf("failed to create socket: %s", strerror(errno)); @@ -4841,11 +5337,12 @@ while (*s != 0) sockun.sun_family = AF_UNIX; sprintf(sockun.sun_path, "%.*s", (int)(sizeof(sockun.sun_path)-1), sub_arg[0]); + server_name = US sockun.sun_path; sigalrm_seen = FALSE; - alarm(timeout); - rc = connect(fd, (struct sockaddr *)(&sockun), sizeof(sockun)); - alarm(0); + ALARM(timeout); + rc = connect(cctx.sock, (struct sockaddr *)(&sockun), sizeof(sockun)); + ALARM_CLR(0); if (sigalrm_seen) { expand_string_message = US "socket connect timed out"; @@ -4857,12 +5354,29 @@ while (*s != 0) "%s: %s", sub_arg[0], strerror(errno)); goto SOCK_FAIL; } + host.name = server_name; + host.address = US""; } DEBUG(D_expand) debug_printf_indent("connected to socket %s\n", sub_arg[0]); +#ifndef DISABLE_TLS + if (do_tls) + { + smtp_connect_args conn_args = {.host = &host }; + tls_support tls_dummy = {.sni=NULL}; + uschar * errstr; + + if (!tls_client_start(&cctx, &conn_args, NULL, &tls_dummy, &errstr)) + { + expand_string_message = string_sprintf("TLS connect failed: %s", errstr); + goto SOCK_FAIL; + } + } +#endif + /* Allow sequencing of test actions */ - if (running_in_test_harness) millisleep(100); + testharness_pause_ms(100); /* Write the request string, if not empty or already done */ @@ -4870,7 +5384,11 @@ while (*s != 0) { DEBUG(D_expand) debug_printf_indent("writing \"%s\" to socket\n", reqstr.data); - if (write(fd, reqstr.data, reqstr.len) != reqstr.len) + if ( ( +#ifndef DISABLE_TLS + do_tls ? tls_write(cctx.tls_ctx, reqstr.data, reqstr.len, FALSE) : +#endif + write(cctx.sock, reqstr.data, reqstr.len)) != reqstr.len) { expand_string_message = string_sprintf("request write to socket " "failed: %s", strerror(errno)); @@ -4883,20 +5401,34 @@ while (*s != 0) system doesn't have this function, make it conditional. */ #ifdef SHUT_WR - if (do_shutdown) shutdown(fd, SHUT_WR); + if (!do_tls && do_shutdown) shutdown(cctx.sock, SHUT_WR); #endif - if (running_in_test_harness) millisleep(100); + testharness_pause_ms(100); /* Now we need to read from the socket, under a timeout. The function that reads a file can be used. */ - f = fdopen(fd, "rb"); + if (!do_tls) + fp = fdopen(cctx.sock, "rb"); sigalrm_seen = FALSE; - alarm(timeout); - yield = cat_file(f, yield, sub_arg[3]); - alarm(0); - (void)fclose(f); + ALARM(timeout); + yield = +#ifndef DISABLE_TLS + do_tls ? cat_file_tls(cctx.tls_ctx, yield, sub_arg[3]) : +#endif + cat_file(fp, yield, sub_arg[3]); + ALARM_CLR(0); + +#ifndef DISABLE_TLS + if (do_tls) + { + tls_close(cctx.tls_ctx, TRUE); + close(cctx.sock); + } + else +#endif + (void)fclose(fp); /* After a timeout, we restore the pointer in the result, that is, make sure we add nothing from the socket. */ @@ -4914,7 +5446,7 @@ while (*s != 0) if (*s == '{') { - if (expand_string_internal(s+1, TRUE, &s, TRUE, TRUE, &resetok) == NULL) + if (!expand_string_internal(s+1, TRUE, &s, TRUE, TRUE, &resetok)) goto EXPAND_FAILED; if (*s++ != '}') { @@ -4973,8 +5505,8 @@ while (*s != 0) expand_string_message = US"missing '{' for command arg of run"; goto EXPAND_FAILED_CURLY; } - arg = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok); - if (arg == NULL) goto EXPAND_FAILED; + if (!(arg = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok))) + goto EXPAND_FAILED; while (isspace(*s)) s++; if (*s++ != '}') { @@ -5019,9 +5551,9 @@ while (*s != 0) resetok = FALSE; f = fdopen(fd_out, "rb"); sigalrm_seen = FALSE; - alarm(60); + ALARM(60); lookup_value = string_from_gstring(cat_file(f, NULL, NULL)); - alarm(0); + ALARM_CLR(0); (void)fclose(f); /* Wait for the process to finish, applying the timeout, and inspect its @@ -5032,7 +5564,7 @@ while (*s != 0) { if (sigalrm_seen || runrc == -256) { - expand_string_message = string_sprintf("command timed out"); + expand_string_message = US"command timed out"; killpg(pid, SIGKILL); /* Kill the whole process group */ } @@ -5087,7 +5619,7 @@ while (*s != 0) if (o2m >= 0) for (; oldptr < yield->ptr; oldptr++) { uschar *m = Ustrrchr(sub[1], yield->s[oldptr]); - if (m != NULL) + if (m) { int o = m - sub[1]; yield->s[oldptr] = sub[2][(o < o2m)? o : o2m]; @@ -5105,7 +5637,6 @@ while (*s != 0) case EITEM_NHASH: case EITEM_SUBSTR: { - int i; int len; uschar *ret; int val[2] = { 0, -1 }; @@ -5127,7 +5658,7 @@ while (*s != 0) string to the last position and make ${length{n}{str}} equivalent to ${substr{0}{n}{str}}. See the defaults for val[] above. */ - if (sub[2] == NULL) + if (!sub[2]) { sub[2] = sub[1]; sub[1] = NULL; @@ -5138,9 +5669,8 @@ while (*s != 0) } } - for (i = 0; i < 2; i++) + for (int i = 0; i < 2; i++) if (sub[i]) { - if (sub[i] == NULL) continue; val[i] = (int)Ustrtol(sub[i], &ret, 10); if (*ret != 0 || (i != 0 && val[i] < 0)) { @@ -5151,13 +5681,13 @@ while (*s != 0) } ret = - (item_type == EITEM_HASH)? - compute_hash(sub[2], val[0], val[1], &len) : - (item_type == EITEM_NHASH)? - compute_nhash(sub[2], val[0], val[1], &len) : - extract_substr(sub[2], val[0], val[1], &len); - - if (ret == NULL) goto EXPAND_FAILED; + item_type == EITEM_HASH + ? compute_hash(sub[2], val[0], val[1], &len) + : item_type == EITEM_NHASH + ? compute_nhash(sub[2], val[0], val[1], &len) + : extract_substr(sub[2], val[0], val[1], &len); + if (!ret) + goto EXPAND_FAILED; yield = string_catn(yield, ret, len); continue; } @@ -5178,7 +5708,7 @@ while (*s != 0) md5 md5_base; hctx sha1_ctx; void *use_base; - int type, i; + int type; int hashlen; /* Number of octets for the hash algorithm's output */ int hashblocklen; /* Number of octets the hash algorithm processes */ uschar *keyptr, *p; @@ -5240,7 +5770,7 @@ while (*s != 0) memset(innerkey, 0x36, hashblocklen); memset(outerkey, 0x5c, hashblocklen); - for (i = 0; i < keylen; i++) + for (int i = 0; i < keylen; i++) { innerkey[i] ^= keyptr[i]; outerkey[i] ^= keyptr[i]; @@ -5259,7 +5789,7 @@ while (*s != 0) /* Encode the final hash as a hex string */ p = finalhash_hex; - for (i = 0; i < hashlen; i++) + for (int i = 0; i < hashlen; i++) { *p++ = hex_digits[(finalhash[i] & 0xf0) >> 4]; *p++ = hex_digits[finalhash[i] & 0x0f]; @@ -5297,10 +5827,8 @@ while (*s != 0) /* Compile the regular expression */ - re = pcre_compile(CS sub[1], PCRE_COPT, (const char **)&rerror, &roffset, - NULL); - - if (re == NULL) + if (!(re = pcre_compile(CS sub[1], PCRE_COPT, CCSS &rerror, + &roffset, NULL))) { expand_string_message = string_sprintf("regular expression error in " "\"%s\": %s at offset %d", sub[1], rerror, roffset); @@ -5321,7 +5849,6 @@ while (*s != 0) int ovector[3*(EXPAND_MAXN+1)]; int n = pcre_exec(re, NULL, CS subject, slen, moffset + moffsetextra, PCRE_EOPT | emptyopt, ovector, nelem(ovector)); - int nn; uschar *insert; /* No match - if we previously set PCRE_NOTEMPTY after a null match, this @@ -5347,7 +5874,7 @@ while (*s != 0) if (n == 0) n = EXPAND_MAXN + 1; expand_nmax = 0; - for (nn = 0; nn < n*2; nn += 2) + for (int nn = 0; nn < n*2; nn += 2) { expand_nstring[expand_nmax] = subject + ovector[nn]; expand_nlength[expand_nmax++] = ovector[nn+1] - ovector[nn]; @@ -5357,8 +5884,8 @@ while (*s != 0) /* Copy the characters before the match, plus the expanded insertion. */ yield = string_catn(yield, subject + moffset, ovector[0] - moffset); - insert = expand_string(sub[2]); - if (insert == NULL) goto EXPAND_FAILED; + if (!(insert = expand_string(sub[2]))) + goto EXPAND_FAILED; yield = string_cat(yield, insert); moffset = ovector[1]; @@ -5391,8 +5918,6 @@ while (*s != 0) case EITEM_EXTRACT: { - int i; - int j; int field_number = 1; BOOL field_number_set = FALSE; uschar *save_lookup_value = lookup_value; @@ -5400,17 +5925,34 @@ while (*s != 0) int save_expand_nmax = save_expand_strings(save_expand_nstring, save_expand_nlength); + /* On reflection the original behaviour of extract-json for a string + result, leaving it quoted, was a mistake. But it was already published, + hence the addition of jsons. In a future major version, make json + work like josons, and withdraw jsons. */ + + enum {extract_basic, extract_json, extract_jsons} fmt = extract_basic; + + while (isspace(*s)) s++; + + /* Check for a format-variant specifier */ + + if (*s != '{') /*}*/ + if (Ustrncmp(s, "json", 4) == 0) + if (*(s += 4) == 's') + {fmt = extract_jsons; s++;} + else + fmt = extract_json; + /* While skipping we cannot rely on the data for expansions being available (eg. $item) hence cannot decide on numeric vs. keyed. Read a maximum of 5 arguments (including the yes/no) */ if (skipping) { - while (isspace(*s)) s++; - for (j = 5; j > 0 && *s == '{'; j--) + for (int j = 5; j > 0 && *s == '{'; j--) /*'}'*/ { if (!expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok)) - goto EXPAND_FAILED; /*{*/ + goto EXPAND_FAILED; /*'{'*/ if (*s++ != '}') { expand_string_message = US"missing '{' for arg of extract"; @@ -5418,13 +5960,13 @@ while (*s != 0) } while (isspace(*s)) s++; } - if ( Ustrncmp(s, "fail", 4) == 0 + if ( Ustrncmp(s, "fail", 4) == 0 /*'{'*/ && (s[4] == '}' || s[4] == ' ' || s[4] == '\t' || !s[4]) ) { s += 4; while (isspace(*s)) s++; - } + } /*'{'*/ if (*s != '}') { expand_string_message = US"missing '}' closing extract"; @@ -5432,13 +5974,13 @@ while (*s != 0) } } - else for (i = 0, j = 2; i < j; i++) /* Read the proper number of arguments */ + else for (int i = 0, j = 2; i < j; i++) /* Read the proper number of arguments */ { - while (isspace(*s)) s++; - if (*s == '{') /*}*/ + while (isspace(*s)) s++; + if (*s == '{') /*'}'*/ { - sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok); - if (sub[i] == NULL) goto EXPAND_FAILED; /*{*/ + if (!(sub[i] = expand_string_internal(s+1, TRUE, &s, skipping, TRUE, &resetok))) + goto EXPAND_FAILED; /*'{'*/ if (*s++ != '}') { expand_string_message = string_sprintf( @@ -5449,7 +5991,7 @@ while (*s != 0) /* After removal of leading and trailing white space, the first argument must not be empty; if it consists entirely of digits (optionally preceded by a minus sign), this is a numerical - extraction, and we expect 3 arguments. */ + extraction, and we expect 3 arguments (normal) or 2 (json). */ if (i == 0) { @@ -5480,7 +6022,7 @@ while (*s != 0) if (*p == 0) { field_number *= x; - j = 3; /* Need 3 args */ + if (fmt == extract_basic) j = 3; /* Need 3 args */ field_number_set = TRUE; } } @@ -5496,9 +6038,96 @@ while (*s != 0) /* Extract either the numbered or the keyed substring into $value. If skipping, just pretend the extraction failed. */ - lookup_value = skipping? NULL : field_number_set? - expand_gettokened(field_number, sub[1], sub[2]) : - expand_getkeyed(sub[0], sub[1]); + if (skipping) + lookup_value = NULL; + else switch (fmt) + { + case extract_basic: + lookup_value = field_number_set + ? expand_gettokened(field_number, sub[1], sub[2]) + : expand_getkeyed(sub[0], sub[1]); + break; + + case extract_json: + case extract_jsons: + { + uschar * s, * item; + const uschar * list; + + /* Array: Bracket-enclosed and comma-separated. + Object: Brace-enclosed, comma-sep list of name:value pairs */ + + if (!(s = dewrap(sub[1], field_number_set ? US"[]" : US"{}"))) + { + expand_string_message = + string_sprintf("%s wrapping %s for extract json", + expand_string_message, + field_number_set ? "array" : "object"); + goto EXPAND_FAILED_CURLY; + } + + list = s; + if (field_number_set) + { + if (field_number <= 0) + { + expand_string_message = US"first argument of \"extract\" must " + "be greater than zero"; + goto EXPAND_FAILED; + } + while (field_number > 0 && (item = json_nextinlist(&list))) + field_number--; + if ((lookup_value = s = item)) + { + while (*s) s++; + while (--s >= lookup_value && isspace(*s)) *s = '\0'; + } + } + else + { + lookup_value = NULL; + while ((item = json_nextinlist(&list))) + { + /* Item is: string name-sep value. string is quoted. + Dequote the string and compare with the search key. */ + + if (!(item = dewrap(item, US"\"\""))) + { + expand_string_message = + string_sprintf("%s wrapping string key for extract json", + expand_string_message); + goto EXPAND_FAILED_CURLY; + } + if (Ustrcmp(item, sub[0]) == 0) /*XXX should be a UTF8-compare */ + { + s = item + Ustrlen(item) + 1; + while (isspace(*s)) s++; + if (*s != ':') + { + expand_string_message = + US"missing object value-separator for extract json"; + goto EXPAND_FAILED_CURLY; + } + s++; + while (isspace(*s)) s++; + lookup_value = s; + break; + } + } + } + } + + if ( fmt == extract_jsons + && lookup_value + && !(lookup_value = dewrap(lookup_value, US"\"\""))) + { + expand_string_message = + string_sprintf("%s wrapping string result for extract jsons", + expand_string_message); + goto EXPAND_FAILED_CURLY; + } + break; /* json/s */ + } /* If no string follows, $value gets substituted; otherwise there can be yes/no strings, as for lookup or if. */ @@ -5528,7 +6157,6 @@ while (*s != 0) case EITEM_LISTEXTRACT: { - int i; int field_number = 1; uschar *save_lookup_value = lookup_value; uschar *sub[2]; @@ -5537,10 +6165,10 @@ while (*s != 0) /* Read the field & list arguments */ - for (i = 0; i < 2; i++) + for (int i = 0; i < 2; i++) { while (isspace(*s)) s++; - if (*s != '{') /*}*/ + if (*s != '{') /*'}'*/ { expand_string_message = string_sprintf( "missing '{' for arg %d of listextract", i+1); @@ -5598,7 +6226,7 @@ while (*s != 0) /* Extract the numbered element into $value. If skipping, just pretend the extraction failed. */ - lookup_value = skipping? NULL : expand_getlistele(field_number, sub[1]); + lookup_value = skipping ? NULL : expand_getlistele(field_number, sub[1]); /* If no string follows, $value gets substituted; otherwise there can be yes/no strings, as for lookup or if. */ @@ -5624,7 +6252,7 @@ while (*s != 0) continue; } -#ifdef SUPPORT_TLS +#ifndef DISABLE_TLS case EITEM_CERTEXTRACT: { uschar *save_lookup_value = lookup_value; @@ -5704,7 +6332,7 @@ while (*s != 0) save_expand_nlength); continue; } -#endif /*SUPPORT_TLS*/ +#endif /*DISABLE_TLS*/ /* Handle list operations */ @@ -5727,8 +6355,8 @@ while (*s != 0) goto EXPAND_FAILED_CURLY; } - list = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok); - if (list == NULL) goto EXPAND_FAILED; + if (!(list = expand_string_internal(s, TRUE, &s, skipping, TRUE, &resetok))) + goto EXPAND_FAILED; if (*s++ != '}') { expand_string_message = @@ -5773,13 +6401,13 @@ while (*s != 0) if (item_type == EITEM_FILTER) { - temp = eval_condition(expr, &resetok, NULL); - if (temp != NULL) s = temp; + if ((temp = eval_condition(expr, &resetok, NULL))) + s = temp; } else temp = expand_string_internal(s, TRUE, &s, TRUE, TRUE, &resetok); - if (temp == NULL) + if (!temp) { expand_string_message = string_sprintf("%s inside \"%s\" item", expand_string_message, name); @@ -5790,7 +6418,8 @@ while (*s != 0) if (*s++ != '}') { /*{*/ expand_string_message = string_sprintf("missing } at end of condition " - "or expression inside \"%s\"", name); + "or expression inside \"%s\"; could be an unquoted } in the content", + name); goto EXPAND_FAILED; } @@ -5816,7 +6445,7 @@ while (*s != 0) if (item_type == EITEM_FILTER) { BOOL condresult; - if (eval_condition(expr, &resetok, &condresult) == NULL) + if (!eval_condition(expr, &resetok, &condresult)) { iterate_item = save_iterate_item; lookup_value = save_lookup_value; @@ -5838,7 +6467,7 @@ while (*s != 0) { uschar * t = expand_string_internal(expr, TRUE, NULL, skipping, TRUE, &resetok); temp = t; - if (temp == NULL) + if (!temp) { iterate_item = save_iterate_item; expand_string_message = string_sprintf("%s inside \"%s\" item", @@ -5909,9 +6538,10 @@ while (*s != 0) case EITEM_SORT: { + int cond_type; int sep = 0; const uschar *srclist, *cmp, *xtract; - uschar *srcitem; + uschar * opname, * srcitem; const uschar *dstlist = NULL, *dstkeylist = NULL; uschar * tmp; uschar *save_iterate_item = iterate_item; @@ -5946,6 +6576,25 @@ while (*s != 0) goto EXPAND_FAILED_CURLY; } + if ((cond_type = identify_operator(&cmp, &opname)) == -1) + { + if (!expand_string_message) + expand_string_message = string_sprintf("unknown condition \"%s\"", s); + goto EXPAND_FAILED; + } + switch(cond_type) + { + case ECOND_NUM_L: case ECOND_NUM_LE: + case ECOND_NUM_G: case ECOND_NUM_GE: + case ECOND_STR_GE: case ECOND_STR_GEI: case ECOND_STR_GT: case ECOND_STR_GTI: + case ECOND_STR_LE: case ECOND_STR_LEI: case ECOND_STR_LT: case ECOND_STR_LTI: + break; + + default: + expand_string_message = US"comparator not handled for sort"; + goto EXPAND_FAILED; + } + while (isspace(*s)) s++; if (*s++ != '{') { @@ -5954,8 +6603,8 @@ while (*s != 0) } xtract = s; - tmp = expand_string_internal(s, TRUE, &s, TRUE, TRUE, &resetok); - if (!tmp) goto EXPAND_FAILED; + if (!(tmp = expand_string_internal(s, TRUE, &s, TRUE, TRUE, &resetok))) + goto EXPAND_FAILED; xtract = string_copyn(xtract, s - xtract); if (*s++ != '}') @@ -5973,11 +6622,10 @@ while (*s != 0) if (skipping) continue; while ((srcitem = string_nextinlist(&srclist, &sep, NULL, 0))) - { - uschar * dstitem; + { + uschar * srcfield, * dstitem; gstring * newlist = NULL; gstring * newkeylist = NULL; - uschar * srcfield; DEBUG(D_expand) debug_printf_indent("%s: $item = \"%s\"\n", name, srcitem); @@ -5998,25 +6646,15 @@ while (*s != 0) while ((dstitem = string_nextinlist(&dstlist, &sep, NULL, 0))) { uschar * dstfield; - uschar * expr; - BOOL before; /* field for comparison */ if (!(dstfield = string_nextinlist(&dstkeylist, &sep, NULL, 0))) goto sort_mismatch; - /* build and run condition string */ - expr = string_sprintf("%s{%s}{%s}", cmp, srcfield, dstfield); - - DEBUG(D_expand) debug_printf_indent("%s: cond = \"%s\"\n", name, expr); - if (!eval_condition(expr, &resetok, &before)) - { - expand_string_message = string_sprintf("comparison in sort: %s", - expr); - goto EXPAND_FAILED; - } + /* String-comparator names start with a letter; numeric names do not */ - if (before) + if (sortsbefore(cond_type, isalpha(opname[0]), + srcfield, dstfield)) { /* New-item sorts before this dst-item. Append new-item, then dst-item, then remainder of dst list. */ @@ -6028,6 +6666,7 @@ while (*s != 0) newlist = string_append_listele(newlist, sep, dstitem); newkeylist = string_append_listele(newkeylist, sep, dstfield); +/*XXX why field-at-a-time copy? Why not just dup the rest of the list? */ while ((dstitem = string_nextinlist(&dstlist, &sep, NULL, 0))) { if (!(dstfield = string_nextinlist(&dstkeylist, &sep, NULL, 0))) @@ -6113,18 +6752,17 @@ while (*s != 0) /* Look up the dynamically loaded object handle in the tree. If it isn't found, dlopen() the file and put the handle in the tree for next time. */ - t = tree_search(dlobj_anchor, argv[0]); - if (t == NULL) + if (!(t = tree_search(dlobj_anchor, argv[0]))) { void *handle = dlopen(CS argv[0], RTLD_LAZY); - if (handle == NULL) + if (!handle) { expand_string_message = string_sprintf("dlopen \"%s\" failed: %s", argv[0], dlerror()); log_write(0, LOG_MAIN|LOG_PANIC, "%s", expand_string_message); goto EXPAND_FAILED; } - t = store_get_perm(sizeof(tree_node) + Ustrlen(argv[0])); + t = store_get_perm(sizeof(tree_node) + Ustrlen(argv[0]), is_tainted(argv[0])); Ustrcpy(t->name, argv[0]); t->data.ptr = handle; (void)tree_insertnode(&dlobj_anchor, t); @@ -6133,8 +6771,7 @@ while (*s != 0) /* Having obtained the dynamically loaded object handle, look up the function pointer. */ - func = (exim_dlfunc_t *)dlsym(t->data.ptr, CS argv[1]); - if (func == NULL) + if (!(func = (exim_dlfunc_t *)dlsym(t->data.ptr, CS argv[1]))) { expand_string_message = string_sprintf("dlsym \"%s\" in \"%s\" failed: " "%s", argv[1], argv[0], dlerror()); @@ -6151,20 +6788,21 @@ while (*s != 0) resetok = FALSE; result = NULL; - for (argc = 0; argv[argc] != NULL; argc++); + for (argc = 0; argv[argc]; argc++); status = func(&result, argc - 2, &argv[2]); if(status == OK) { - if (result == NULL) result = US""; + if (!result) result = US""; yield = string_cat(yield, result); continue; } else { - expand_string_message = result == NULL ? US"(no message)" : result; - if(status == FAIL_FORCED) expand_string_forcedfail = TRUE; - else if(status != FAIL) - log_write(0, LOG_MAIN|LOG_PANIC, "dlfunc{%s}{%s} failed (%d): %s", + expand_string_message = result ? result : US"(no message)"; + if (status == FAIL_FORCED) + f.expand_string_forcedfail = TRUE; + else if (status != FAIL) + log_write(0, LOG_MAIN|LOG_PANIC, "dlfunc{%s}{%s} failed (%d): %s", argv[0], argv[1], status, expand_string_message); goto EXPAND_FAILED; } @@ -6204,6 +6842,62 @@ while (*s != 0) } continue; } + +#ifdef EXPERIMENTAL_SRS_NATIVE + case EITEM_SRS_ENCODE: + /* ${srs_encode {secret} {return_path} {orig_domain}} */ + { + uschar * sub[3]; + uschar cksum[4]; + + switch (read_subs(sub, 3, 3, CUSS &s, skipping, TRUE, name, &resetok)) + { + case 1: goto EXPAND_FAILED_CURLY; + case 2: + case 3: goto EXPAND_FAILED; + } + + yield = string_catn(yield, US"SRS0=", 5); + + /* ${l_4:${hmac{md5}{SRS_SECRET}{${lc:$return_path}}}}= */ + hmac_md5(sub[0], string_copylc(sub[1]), cksum, sizeof(cksum)); + yield = string_catn(yield, cksum, sizeof(cksum)); + yield = string_catn(yield, US"=", 1); + + /* ${base32:${eval:$tod_epoch/86400&0x3ff}}= */ + { + struct timeval now; + unsigned long i; + gstring * g = NULL; + + gettimeofday(&now, NULL); + for (unsigned long i = (now.tv_sec / 86400) & 0x3ff; i; i >>= 5) + g = string_catn(g, &base32_chars[i & 0x1f], 1); + if (g) while (g->ptr > 0) + yield = string_catn(yield, &g->s[--g->ptr], 1); + } + yield = string_catn(yield, US"=", 1); + + /* ${domain:$return_path}=${local_part:$return_path} */ + { + int start, end, domain; + uschar * t = parse_extract_address(sub[1], &expand_string_message, + &start, &end, &domain, FALSE); + if (!t) + goto EXPAND_FAILED; + + if (domain > 0) yield = string_cat(yield, t + domain); + yield = string_catn(yield, US"=", 1); + yield = domain > 0 + ? string_catn(yield, t, domain - 1) : string_cat(yield, t); + } + + /* @$original_domain */ + yield = string_catn(yield, US"@", 1); + yield = string_cat(yield, sub[2]); + continue; + } +#endif /*EXPERIMENTAL_SRS_NATIVE*/ } /* EITEM_* switch */ /* Control reaches here if the name is not recognized as one of the more @@ -6216,7 +6910,9 @@ while (*s != 0) int c; uschar *arg = NULL; uschar *sub; +#ifndef DISABLE_TLS var_entry *vp = NULL; +#endif /* Owing to an historical mis-design, an underscore may be part of the operator name, or it may introduce arguments. We therefore first scan the @@ -6226,18 +6922,18 @@ while (*s != 0) if ((c = chop_match(name, op_table_underscore, nelem(op_table_underscore))) < 0) { - arg = Ustrchr(name, '_'); - if (arg != NULL) *arg = 0; - c = chop_match(name, op_table_main, nelem(op_table_main)); - if (c >= 0) c += nelem(op_table_underscore); - if (arg != NULL) *arg++ = '_'; /* Put back for error messages */ + if ((arg = Ustrchr(name, '_'))) + *arg = 0; + if ((c = chop_match(name, op_table_main, nelem(op_table_main))) >= 0) + c += nelem(op_table_underscore); + if (arg) *arg++ = '_'; /* Put back for error messages */ } /* Deal specially with operators that might take a certificate variable as we do not want to do the usual expansion. For most, expand the string.*/ switch(c) { -#ifdef SUPPORT_TLS +#ifndef DISABLE_TLS case EOP_MD5: case EOP_SHA1: case EOP_SHA256: @@ -6305,11 +7001,10 @@ while (*s != 0) { uschar *tt = sub; unsigned long int n = 0; - uschar * s; while (*tt) { uschar * t = Ustrchr(base32_chars, *tt++); - if (t == NULL) + if (!t) { expand_string_message = string_sprintf("argument for base32d " "operator is \"%s\", which is not a base 32 number", sub); @@ -6317,8 +7012,7 @@ while (*s != 0) } n = n * 32 + (t - base32_chars); } - s = string_sprintf("%ld", n); - yield = string_cat(yield, s); + yield = string_fmt_append(yield, "%ld", n); continue; } @@ -6332,8 +7026,7 @@ while (*s != 0) "operator is \"%s\", which is not a decimal number", sub); goto EXPAND_FAILED; } - t = string_base62(n); - yield = string_cat(yield, t); + yield = string_cat(yield, string_base62(n)); continue; } @@ -6341,13 +7034,12 @@ while (*s != 0) case EOP_BASE62D: { - uschar buf[16]; uschar *tt = sub; unsigned long int n = 0; while (*tt != 0) { uschar *t = Ustrchr(base62_chars, *tt++); - if (t == NULL) + if (!t) { expand_string_message = string_sprintf("argument for base62d " "operator is \"%s\", which is not a base %d number", sub, @@ -6356,15 +7048,28 @@ while (*s != 0) } n = n * BASE_62 + (t - base62_chars); } - (void)sprintf(CS buf, "%ld", n); - yield = string_cat(yield, buf); + yield = string_fmt_append(yield, "%ld", n); continue; } + case EOP_BLESS: + /* This is purely for the convenience of the test harness. Do not enable + it otherwise as it defeats the taint-checking security. */ + + if (f.running_in_test_harness) + yield = string_cat(yield, is_tainted(sub) + ? string_copy_taint(sub, FALSE) : sub); + else + { + DEBUG(D_expand) debug_printf_indent("bless operator not supported\n"); + yield = string_cat(yield, sub); + } + continue; + case EOP_EXPAND: { uschar *expanded = expand_string_internal(sub, FALSE, NULL, skipping, TRUE, &resetok); - if (expanded == NULL) + if (!expanded) { expand_string_message = string_sprintf("internal expansion of \"%s\" failed: %s", sub, @@ -6394,7 +7099,7 @@ while (*s != 0) } case EOP_MD5: -#ifdef SUPPORT_TLS +#ifndef DISABLE_TLS if (vp && *(void **)vp->value) { uschar * cp = tls_cert_fprt_md5(*(void **)vp->value); @@ -6405,17 +7110,15 @@ while (*s != 0) { md5 base; uschar digest[16]; - int j; - char st[33]; md5_start(&base); md5_end(&base, sub, Ustrlen(sub), digest); - for(j = 0; j < 16; j++) sprintf(st+2*j, "%02x", digest[j]); - yield = string_cat(yield, US st); + for (int j = 0; j < 16; j++) + yield = string_fmt_append(yield, "%02x", digest[j]); } continue; case EOP_SHA1: -#ifdef SUPPORT_TLS +#ifndef DISABLE_TLS if (vp && *(void **)vp->value) { uschar * cp = tls_cert_fprt_sha1(*(void **)vp->value); @@ -6426,40 +7129,41 @@ while (*s != 0) { hctx h; uschar digest[20]; - int j; - char st[41]; sha1_start(&h); sha1_end(&h, sub, Ustrlen(sub), digest); - for(j = 0; j < 20; j++) sprintf(st+2*j, "%02X", digest[j]); - yield = string_catn(yield, US st, 40); + for (int j = 0; j < 20; j++) + yield = string_fmt_append(yield, "%02X", digest[j]); } continue; + case EOP_SHA2: case EOP_SHA256: #ifdef EXIM_HAVE_SHA2 if (vp && *(void **)vp->value) - { - uschar * cp = tls_cert_fprt_sha256(*(void **)vp->value); - yield = string_cat(yield, cp); - } + if (c == EOP_SHA256) + yield = string_cat(yield, tls_cert_fprt_sha256(*(void **)vp->value)); + else + expand_string_message = US"sha2_N not supported with certificates"; else { hctx h; blob b; - char st[3]; + hashmethod m = !arg ? HASH_SHA2_256 + : Ustrcmp(arg, "256") == 0 ? HASH_SHA2_256 + : Ustrcmp(arg, "384") == 0 ? HASH_SHA2_384 + : Ustrcmp(arg, "512") == 0 ? HASH_SHA2_512 + : HASH_BADTYPE; - if (!exim_sha_init(&h, HASH_SHA2_256)) + if (m == HASH_BADTYPE || !exim_sha_init(&h, m)) { - expand_string_message = US"unrecognised sha256 variant"; + expand_string_message = US"unrecognised sha2 variant"; goto EXPAND_FAILED; } + exim_sha_update(&h, sub, Ustrlen(sub)); exim_sha_finish(&h, &b); while (b.len-- > 0) - { - sprintf(st, "%02X", *b.data++); - yield = string_catn(yield, US st, 2); - } + yield = string_fmt_append(yield, "%02X", *b.data++); } #else expand_string_message = US"sha256 only supported with TLS"; @@ -6471,7 +7175,6 @@ while (*s != 0) { hctx h; blob b; - char st[3]; hashmethod m = !arg ? HASH_SHA3_256 : Ustrcmp(arg, "224") == 0 ? HASH_SHA3_224 : Ustrcmp(arg, "256") == 0 ? HASH_SHA3_256 @@ -6488,14 +7191,11 @@ while (*s != 0) exim_sha_update(&h, sub, Ustrlen(sub)); exim_sha_finish(&h, &b); while (b.len-- > 0) - { - sprintf(st, "%02X", *b.data++); - yield = string_catn(yield, US st, 2); - } + yield = string_fmt_append(yield, "%02X", *b.data++); } continue; #else - expand_string_message = US"sha3 only supported with GnuTLS 3.5.0 +"; + expand_string_message = US"sha3 only supported with GnuTLS 3.5.0 + or OpenSSL 1.1.1 +"; goto EXPAND_FAILED; #endif @@ -6509,7 +7209,7 @@ while (*s != 0) uschar *out = sub; uschar *enc; - for (enc = sub; *enc != 0; enc++) + for (enc = sub; *enc; enc++) { if (!isxdigit(*enc)) { @@ -6532,9 +7232,7 @@ while (*s != 0) if (isdigit(c)) c -= '0'; else c = toupper(c) - 'A' + 10; if (b == -1) - { b = c << 4; - } else { *out++ = b | c; @@ -6542,7 +7240,7 @@ while (*s != 0) } } - enc = b64encode(sub, out - sub); + enc = b64encode(CUS sub, out - sub); yield = string_cat(yield, enc); continue; } @@ -6555,7 +7253,7 @@ while (*s != 0) while (*(++t) != 0) { if (*t < 0x21 || 0x7E < *t) - yield = string_catn(yield, string_sprintf("\\x%02x", *t), 4); + yield = string_fmt_append(yield, "\\x%02x", *t); else yield = string_catn(yield, t, 1); } @@ -6568,12 +7266,10 @@ while (*s != 0) { int cnt = 0; int sep = 0; - uschar * cp; uschar buffer[256]; - while (string_nextinlist(CUSS &sub, &sep, buffer, sizeof(buffer)) != NULL) cnt++; - cp = string_sprintf("%d", cnt); - yield = string_cat(yield, cp); + while (string_nextinlist(CUSS &sub, &sep, buffer, sizeof(buffer))) cnt++; + yield = string_fmt_append(yield, "%d", cnt); continue; } @@ -6591,7 +7287,7 @@ while (*s != 0) uschar buffer[256]; if (*sub == '+') sub++; - if (arg == NULL) /* no-argument version */ + if (!arg) /* no-argument version */ { if (!(t = tree_search(addresslist_anchor, sub)) && !(t = tree_search(domainlist_anchor, sub)) && @@ -6605,7 +7301,7 @@ while (*s != 0) case 'h': t = tree_search(hostlist_anchor, sub); suffix = US"_h"; break; case 'l': t = tree_search(localpartlist_anchor, sub); suffix = US"_l"; break; default: - expand_string_message = string_sprintf("bad suffix on \"list\" operator"); + expand_string_message = US"bad suffix on \"list\" operator"; goto EXPAND_FAILED; } @@ -6751,16 +7447,12 @@ while (*s != 0) uschar * t = parse_extract_address(sub, &error, &start, &end, &domain, FALSE); if (t) - if (c != EOP_DOMAIN) - { - if (c == EOP_LOCAL_PART && domain != 0) end = start + domain - 1; - yield = string_catn(yield, sub+start, end-start); - } - else if (domain != 0) - { - domain += start; - yield = string_catn(yield, sub+domain, end-domain); - } + if (c != EOP_DOMAIN) + yield = c == EOP_LOCAL_PART && domain > 0 + ? string_catn(yield, t, domain - 1) + : string_cat(yield, t); + else if (domain > 0) + yield = string_cat(yield, t + domain); continue; } @@ -6780,11 +7472,11 @@ while (*s != 0) "missing in expanding ${addresses:%s}", --sub); goto EXPAND_FAILED; } - parse_allow_group = TRUE; + f.parse_allow_group = TRUE; for (;;) { - uschar *p = parse_find_address_end(sub, FALSE); + uschar * p = parse_find_address_end(sub, FALSE); uschar saveend = *p; *p = '\0'; address = parse_extract_address(sub, &error, &start, &end, &domain, @@ -6797,7 +7489,7 @@ while (*s != 0) list, add in a space if the new address begins with the separator character, or is an empty string. */ - if (address != NULL) + if (address) { if (yield->ptr != save_ptr && address[0] == *outsep) yield = string_catn(yield, US" ", 1); @@ -6829,7 +7521,7 @@ while (*s != 0) separator. */ if (yield->ptr != save_ptr) yield->ptr--; - parse_allow_group = FALSE; + f.parse_allow_group = FALSE; continue; } @@ -6845,7 +7537,7 @@ while (*s != 0) case EOP_QUOTE: case EOP_QUOTE_LOCAL_PART: - if (arg == NULL) + if (!arg) { BOOL needs_quote = (*sub == 0); /* TRUE for empty string */ uschar *t = sub - 1; @@ -6893,20 +7585,20 @@ while (*s != 0) int n; uschar *opt = Ustrchr(arg, '_'); - if (opt != NULL) *opt++ = 0; + if (opt) *opt++ = 0; - n = search_findtype(arg, Ustrlen(arg)); - if (n < 0) + if ((n = search_findtype(arg, Ustrlen(arg))) < 0) { expand_string_message = search_error_message; goto EXPAND_FAILED; } - if (lookup_list[n]->quote != NULL) + if (lookup_list[n]->quote) sub = (lookup_list[n]->quote)(sub, opt); - else if (opt != NULL) sub = NULL; + else if (opt) + sub = NULL; - if (sub == NULL) + if (!sub) { expand_string_message = string_sprintf( "\"%s\" unrecognized after \"${quote_%s\"", @@ -6939,9 +7631,9 @@ while (*s != 0) case EOP_RFC2047: { uschar buffer[2048]; - const uschar *string = parse_quote_2047(sub, Ustrlen(sub), headers_charset, - buffer, sizeof(buffer), FALSE); - yield = string_cat(yield, string); + yield = string_cat(yield, + parse_quote_2047(sub, Ustrlen(sub), headers_charset, + buffer, sizeof(buffer), FALSE)); continue; } @@ -6953,7 +7645,7 @@ while (*s != 0) uschar *error; uschar *decoded = rfc2047_decode(sub, check_rfc2047_length, headers_charset, '?', &len, &error); - if (error != NULL) + if (error) { expand_string_message = error; goto EXPAND_FAILED; @@ -6987,12 +7679,13 @@ while (*s != 0) { int seq_len = 0, index = 0; int bytes_left = 0; - long codepoint = -1; + long codepoint = -1; + int complete; uschar seq_buff[4]; /* accumulate utf-8 here */ while (*sub != 0) { - int complete = 0; + complete = 0; uschar c = *sub++; if (bytes_left) @@ -7057,6 +7750,13 @@ while (*s != 0) /* ASCII character follows incomplete sequence */ yield = string_catn(yield, &c, 1); } + /* If given a sequence truncated mid-character, we also want to report ? + * Eg, ${length_1:フィル} is one byte, not one character, so we expect + * ${utf8clean:${length_1:フィル}} to yield '?' */ + if (bytes_left != 0) + { + yield = string_catn(yield, UTF8_REPLACEMENT_CHAR, 1); + } continue; } @@ -7134,13 +7834,12 @@ while (*s != 0) case EOP_ESCAPE8BIT: { - const uschar * s = sub; uschar c; - for (s = sub; (c = *s); s++) + for (const uschar * s = sub; (c = *s); s++) yield = c < 127 && c != '\\' ? string_catn(yield, s, 1) - : string_catn(yield, string_sprintf("\\%03o", c), 4); + : string_fmt_append(yield, "\\%03o", c); continue; } @@ -7152,19 +7851,18 @@ while (*s != 0) uschar *save_sub = sub; uschar *error = NULL; int_eximarith_t n = eval_expr(&sub, (c == EOP_EVAL10), &error, FALSE); - if (error != NULL) + if (error) { expand_string_message = string_sprintf("error in expression " "evaluation: %s (after processing \"%.*s\")", error, (int)(sub-save_sub), save_sub); goto EXPAND_FAILED; } - sprintf(CS var_buffer, PR_EXIM_ARITH, n); - yield = string_cat(yield, var_buffer); + yield = string_fmt_append(yield, PR_EXIM_ARITH, n); continue; } - /* Handle time period formating */ + /* Handle time period formatting */ case EOP_TIME_EVAL: { @@ -7175,8 +7873,7 @@ while (*s != 0) "Exim time interval in \"%s\" operator", sub, name); goto EXPAND_FAILED; } - sprintf(CS var_buffer, "%d", n); - yield = string_cat(yield, var_buffer); + yield = string_fmt_append(yield, "%d", n); continue; } @@ -7200,12 +7897,12 @@ while (*s != 0) case EOP_STR2B64: case EOP_BASE64: { -#ifdef SUPPORT_TLS +#ifndef DISABLE_TLS uschar * s = vp && *(void **)vp->value ? tls_cert_der_b64(*(void **)vp->value) - : b64encode(sub, Ustrlen(sub)); + : b64encode(CUS sub, Ustrlen(sub)); #else - uschar * s = b64encode(sub, Ustrlen(sub)); + uschar * s = b64encode(CUS sub, Ustrlen(sub)); #endif yield = string_cat(yield, s); continue; @@ -7228,12 +7925,8 @@ while (*s != 0) /* strlen returns the length of the string */ case EOP_STRLEN: - { - uschar buff[24]; - (void)sprintf(CS buff, "%d", Ustrlen(sub)); - yield = string_cat(yield, buff); + yield = string_fmt_append(yield, "%d", Ustrlen(sub)); continue; - } /* length_n or l_n takes just the first n characters or the whole string, whichever is the shorter; @@ -7266,7 +7959,7 @@ while (*s != 0) int len; uschar *ret; - if (arg == NULL) + if (!arg) { expand_string_message = string_sprintf("missing values after %s", name); @@ -7314,14 +8007,13 @@ while (*s != 0) /* Perform the required operation */ - ret = - (c == EOP_HASH || c == EOP_H)? - compute_hash(sub, value1, value2, &len) : - (c == EOP_NHASH || c == EOP_NH)? - compute_nhash(sub, value1, value2, &len) : - extract_substr(sub, value1, value2, &len); + ret = c == EOP_HASH || c == EOP_H + ? compute_hash(sub, value1, value2, &len) + : c == EOP_NHASH || c == EOP_NH + ? compute_nhash(sub, value1, value2, &len) + : extract_substr(sub, value1, value2, &len); + if (!ret) goto EXPAND_FAILED; - if (ret == NULL) goto EXPAND_FAILED; yield = string_catn(yield, ret, len); continue; } @@ -7330,14 +8022,12 @@ while (*s != 0) case EOP_STAT: { - uschar *s; uschar smode[12]; uschar **modetable[3]; - int i; mode_t mode; struct stat st; - if ((expand_forbid & RDO_EXISTS) != 0) + if (expand_forbid & RDO_EXISTS) { expand_string_message = US"Use of the stat() expansion is not permitted"; goto EXPAND_FAILED; @@ -7364,20 +8054,20 @@ while (*s != 0) modetable[1] = ((mode & 02000) == 0)? mtable_normal : mtable_setid; modetable[2] = ((mode & 04000) == 0)? mtable_normal : mtable_setid; - for (i = 0; i < 3; i++) + for (int i = 0; i < 3; i++) { memcpy(CS(smode + 7 - i*3), CS(modetable[i][mode & 7]), 3); mode >>= 3; } smode[10] = 0; - s = string_sprintf("mode=%04lo smode=%s inode=%ld device=%ld links=%ld " + yield = string_fmt_append(yield, + "mode=%04lo smode=%s inode=%ld device=%ld links=%ld " "uid=%ld gid=%ld size=" OFF_T_FMT " atime=%ld mtime=%ld ctime=%ld", (long)(st.st_mode & 077777), smode, (long)st.st_ino, (long)st.st_dev, (long)st.st_nlink, (long)st.st_uid, (long)st.st_gid, st.st_size, (long)st.st_atime, (long)st.st_mtime, (long)st.st_ctime); - yield = string_cat(yield, s); continue; } @@ -7385,14 +8075,11 @@ while (*s != 0) case EOP_RANDINT: { - int_eximarith_t max; - uschar *s; + int_eximarith_t max = expanded_string_integer(sub, TRUE); - max = expanded_string_integer(sub, TRUE); - if (expand_string_message != NULL) + if (expand_string_message) goto EXPAND_FAILED; - s = string_sprintf("%d", vaguely_random_number((int)max)); - yield = string_cat(yield, s); + yield = string_fmt_append(yield, "%d", vaguely_random_number((int)max)); continue; } @@ -7418,9 +8105,9 @@ while (*s != 0) /* Unknown operator */ default: - expand_string_message = - string_sprintf("unknown expansion operator \"%s\"", name); - goto EXPAND_FAILED; + expand_string_message = + string_sprintf("unknown expansion operator \"%s\"", name); + goto EXPAND_FAILED; } } @@ -7438,12 +8125,13 @@ while (*s != 0) gstring * g = NULL; if (!yield) - g = store_get(sizeof(gstring)); + g = store_get(sizeof(gstring), FALSE); else if (yield->ptr == 0) { - if (resetok) store_reset(yield); + if (resetok) reset_point = store_reset(reset_point); yield = NULL; - g = store_get(sizeof(gstring)); /* alloc _before_ calling find_variable() */ + reset_point = store_mark(); + g = store_get(sizeof(gstring), FALSE); /* alloc _before_ calling find_variable() */ } if (!(value = find_variable(name, FALSE, skipping, &newsize))) { @@ -7478,10 +8166,9 @@ terminating brace. */ if (ket_ends && *s == 0) { - expand_string_message = malformed_header? - US"missing } at end of string - could be header name not terminated by colon" - : - US"missing } at end of string"; + expand_string_message = malformed_header + ? US"missing } at end of string - could be header name not terminated by colon" + : US"missing } at end of string"; goto EXPAND_FAILED; } @@ -7498,22 +8185,40 @@ if (left) *left = s; In many cases the final string will be the first one that was got and so there will be optimal store usage. */ -if (resetok) store_reset(yield->s + (yield->size = yield->ptr + 1)); +if (resetok) gstring_release_unused(yield); else if (resetok_p) *resetok_p = FALSE; DEBUG(D_expand) { - debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ - "expanding: %.*s\n", - (int)(s - string), string); - debug_printf_indent("%s" - UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ - "result: %s\n", - skipping ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT, - yield->s); - if (skipping) - debug_printf_indent(UTF8_UP_RIGHT UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ - "skipping: result is not used\n"); + BOOL tainted = is_tainted(yield->s); + DEBUG(D_noutf8) + { + debug_printf_indent("|--expanding: %.*s\n", (int)(s - string), string); + debug_printf_indent("%sresult: %s\n", + skipping ? "|-----" : "\\_____", yield->s); + if (tainted) + debug_printf_indent("%s \\__(tainted)\n", + skipping ? "| " : " "); + if (skipping) + debug_printf_indent("\\___skipping: result is not used\n"); + } + else + { + debug_printf_indent(UTF8_VERT_RIGHT UTF8_HORIZ UTF8_HORIZ + "expanding: %.*s\n", + (int)(s - string), string); + debug_printf_indent("%s" UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ + "result: %s\n", + skipping ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT, + yield->s); + if (tainted) + debug_printf_indent("%s(tainted)\n", + skipping + ? UTF8_VERT " " : " " UTF8_UP_RIGHT UTF8_HORIZ UTF8_HORIZ); + if (skipping) + debug_printf_indent(UTF8_UP_RIGHT UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ + "skipping: result is not used\n"); + } } expand_level--; return yield->s; @@ -7536,17 +8241,26 @@ that is a bad idea, because expand_string_message is in dynamic store. */ EXPAND_FAILED: if (left) *left = s; DEBUG(D_expand) - { - debug_printf_indent(UTF8_VERT_RIGHT "failed to expand: %s\n", - string); - debug_printf_indent("%s" UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ - "error message: %s\n", - expand_string_forcedfail ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT, - expand_string_message); - if (expand_string_forcedfail) - debug_printf_indent(UTF8_UP_RIGHT "failure was forced\n"); - } -if (resetok_p) *resetok_p = resetok; + DEBUG(D_noutf8) + { + debug_printf_indent("|failed to expand: %s\n", string); + debug_printf_indent("%serror message: %s\n", + f.expand_string_forcedfail ? "|---" : "\\___", expand_string_message); + if (f.expand_string_forcedfail) + debug_printf_indent("\\failure was forced\n"); + } + else + { + debug_printf_indent(UTF8_VERT_RIGHT "failed to expand: %s\n", + string); + debug_printf_indent("%s" UTF8_HORIZ UTF8_HORIZ UTF8_HORIZ + "error message: %s\n", + f.expand_string_forcedfail ? UTF8_VERT_RIGHT : UTF8_UP_RIGHT, + expand_string_message); + if (f.expand_string_forcedfail) + debug_printf_indent(UTF8_UP_RIGHT "failure was forced\n"); + } +if (resetok_p && !resetok) *resetok_p = FALSE; expand_level--; return NULL; } @@ -7560,28 +8274,35 @@ Returns: the expanded string, or NULL if expansion failed; if failure was due to a lookup deferring, search_find_defer will be TRUE */ -uschar * -expand_string(uschar *string) +const uschar * +expand_cstring(const uschar * string) { -search_find_defer = FALSE; -malformed_header = FALSE; -return (Ustrpbrk(string, "$\\") == NULL)? string : - expand_string_internal(string, FALSE, NULL, FALSE, TRUE, NULL); +if (Ustrpbrk(string, "$\\") != NULL) + { + int old_pool = store_pool; + uschar * s; + + f.search_find_defer = FALSE; + malformed_header = FALSE; + store_pool = POOL_MAIN; + s = expand_string_internal(string, FALSE, NULL, FALSE, TRUE, NULL); + store_pool = old_pool; + return s; + } +return string; } - -const uschar * -expand_cstring(const uschar *string) +uschar * +expand_string(uschar * string) { -search_find_defer = FALSE; -malformed_header = FALSE; -return (Ustrpbrk(string, "$\\") == NULL)? string : - expand_string_internal(string, FALSE, NULL, FALSE, TRUE, NULL); +return US expand_cstring(CUS string); } + + /************************************************* * Expand and copy * *************************************************/ @@ -7654,7 +8375,7 @@ uschar *endptr; /* If expansion failed, expand_string_message will be set. */ -if (s == NULL) return -1; +if (!s) return -1; /* On an overflow, strtol() returns LONG_MAX or LONG_MIN, and sets errno to ERANGE. When there isn't an overflow, errno is not changed, at least on some @@ -7749,12 +8470,11 @@ exp_bool(address_item *addr, uschar *svalue, BOOL *rvalue) { uschar *expanded; -if (svalue == NULL) { *rvalue = bvalue; return OK; } +if (!svalue) { *rvalue = bvalue; return OK; } -expanded = expand_string(svalue); -if (expanded == NULL) +if (!(expanded = expand_string(svalue))) { - if (expand_string_forcedfail) + if (f.expand_string_forcedfail) { DEBUG(dbg_opt) debug_printf("expansion of \"%s\" forced failure\n", oname); *rvalue = bvalue; @@ -7792,7 +8512,7 @@ expand_hide_passwords(uschar * s) { return ( ( Ustrstr(s, "failed to expand") != NULL || Ustrstr(s, "expansion of ") != NULL - ) + ) && ( Ustrstr(s, "mysql") != NULL || Ustrstr(s, "pgsql") != NULL || Ustrstr(s, "redis") != NULL @@ -7802,18 +8522,55 @@ return ( ( Ustrstr(s, "failed to expand") != NULL || Ustrstr(s, "ldapi:") != NULL || Ustrstr(s, "ldapdn:") != NULL || Ustrstr(s, "ldapm:") != NULL - ) ) + ) ) ? US"Temporary internal error" : s; } +/* Read given named file into big_buffer. Use for keying material etc. +The content will have an ascii NUL appended. + +Arguments: + filename as it says + +Return: pointer to buffer, or NULL on error. +*/ + +uschar * +expand_file_big_buffer(const uschar * filename) +{ +int fd, off = 0, len; + +if ((fd = exim_open2(CS filename, O_RDONLY)) < 0) + { + log_write(0, LOG_MAIN | LOG_PANIC, "unable to open file for reading: %s", + filename); + return NULL; + } + +do + { + if ((len = read(fd, big_buffer + off, big_buffer_size - 2 - off)) < 0) + { + (void) close(fd); + log_write(0, LOG_MAIN|LOG_PANIC, "unable to read file: %s", filename); + return NULL; + } + off += len; + } +while (len > 0); + +(void) close(fd); +big_buffer[off] = '\0'; +return big_buffer; +} + + /************************************************* * Error-checking for testsuite * *************************************************/ typedef struct { - const char * filename; - int linenumber; uschar * region_start; uschar * region_end; const uschar *var_name; @@ -7834,31 +8591,36 @@ if (var_data >= e->region_start && var_data < e->region_end) void assert_no_variables(void * ptr, int len, const char * filename, int linenumber) { -err_ctx e = {filename, linenumber, ptr, US ptr + len, NULL }; -int i; -var_entry * v; +err_ctx e = { .region_start = ptr, .region_end = US ptr + len, + .var_name = NULL, .var_data = NULL }; /* check acl_ variables */ tree_walk(acl_var_c, assert_variable_notin, &e); tree_walk(acl_var_m, assert_variable_notin, &e); /* check auth variables */ -for (i = 0; i < AUTH_VARS; i++) if (auth_vars[i]) +for (int i = 0; i < AUTH_VARS; i++) if (auth_vars[i]) assert_variable_notin(US"auth", auth_vars[i], &e); /* check regex variables */ -for (i = 0; i < REGEX_VARS; i++) if (regex_vars[i]) +for (int i = 0; i < REGEX_VARS; i++) if (regex_vars[i]) assert_variable_notin(US"regex", regex_vars[i], &e); /* check known-name variables */ -for (v = var_table; v < var_table + var_table_size; v++) +for (var_entry * v = var_table; v < var_table + var_table_size; v++) if (v->type == vtype_stringptr) assert_variable_notin(US v->name, *(USS v->value), &e); +/* check dns and address trees */ +tree_walk(tree_dns_fails, assert_variable_notin, &e); +tree_walk(tree_duplicates, assert_variable_notin, &e); +tree_walk(tree_nonrecipients, assert_variable_notin, &e); +tree_walk(tree_unusable, assert_variable_notin, &e); + if (e.var_name) log_write(0, LOG_MAIN|LOG_PANIC_DIE, "live variable '%s' destroyed by reset_store at %s:%d\n- value '%.64s'", - e.var_name, e.filename, e.linenumber, e.var_data); + e.var_name, filename, linenumber, e.var_data); } @@ -7882,9 +8644,8 @@ BOOL yield = n >= 0; if (n == 0) n = EXPAND_MAXN + 1; if (yield) { - int nn; - expand_nmax = (setup < 0)? 0 : setup + 1; - for (nn = (setup < 0)? 0 : 2; nn < n*2; nn += 2) + expand_nmax = setup < 0 ? 0 : setup + 1; + for (int nn = setup < 0 ? 0 : 2; nn < n*2; nn += 2) { expand_nstring[expand_nmax] = subject + ovector[nn]; expand_nlength[expand_nmax++] = ovector[nn+1] - ovector[nn]; @@ -7897,7 +8658,6 @@ return yield; int main(int argc, uschar **argv) { -int i; uschar buffer[1024]; debug_selector = D_v; @@ -7905,7 +8665,7 @@ debug_file = stderr; debug_fd = fileno(debug_file); big_buffer = malloc(big_buffer_size); -for (i = 1; i < argc; i++) +for (int i = 1; i < argc; i++) { if (argv[i][0] == '+') { @@ -7956,22 +8716,23 @@ if (opt_perl_startup != NULL) } #endif /* EXIM_PERL */ +/* Thie deliberately regards the input as untainted, so that it can be +expanded; only reasonable since this is a test for string-expansions. */ + while (fgets(buffer, sizeof(buffer), stdin) != NULL) { - void *reset_point = store_get(0); + rmark reset_point = store_mark(); uschar *yield = expand_string(buffer); - if (yield != NULL) - { + if (yield) printf("%s\n", yield); - store_reset(reset_point); - } else { - if (search_find_defer) printf("search_find deferred\n"); + if (f.search_find_defer) printf("search_find deferred\n"); printf("Failed: %s\n", expand_string_message); - if (expand_string_forcedfail) printf("Forced failure\n"); + if (f.expand_string_forcedfail) printf("Forced failure\n"); printf("\n"); } + store_reset(reset_point); } search_tidyup();