X-Git-Url: https://git.exim.org/exim.git/blobdiff_plain/9ad27f49dae5a6375d7951679c90c2e26ad82b5c..e2e0f812212b81ac029f90b57c63ee88455f6a12:/src/src/expand.c diff --git a/src/src/expand.c b/src/src/expand.c index cdc914f5e..88d4e756f 100644 --- a/src/src/expand.c +++ b/src/src/expand.c @@ -3,6 +3,7 @@ *************************************************/ /* Copyright (c) University of Cambridge 1995 - 2018 */ +/* Copyright (c) The Exim Maintainers 2020 */ /* See the file NOTICE for conditions of use and distribution. */ @@ -130,7 +131,7 @@ static uschar *item_table[] = { US"run", US"sg", US"sort", -#ifdef EXPERIMENTAL_SRS_NATIVE +#ifdef SUPPORT_SRS US"srs_encode", #endif US"substr", @@ -165,7 +166,7 @@ enum { EITEM_RUN, EITEM_SG, EITEM_SORT, -#ifdef EXPERIMENTAL_SRS_NATIVE +#ifdef SUPPORT_SRS EITEM_SRS_ENCODE, #endif EITEM_SUBSTR, @@ -215,7 +216,6 @@ static uschar *op_table_main[] = { US"base62d", US"base64", US"base64d", - US"bless", US"domain", US"escape", US"escape8bit", @@ -263,7 +263,6 @@ enum { EOP_BASE62D, EOP_BASE64, EOP_BASE64D, - EOP_BLESS, EOP_DOMAIN, EOP_ESCAPE, EOP_ESCAPE8BIT, @@ -333,7 +332,7 @@ static uschar *cond_table[] = { US"gei", US"gt", US"gti", -#ifdef EXPERIMENTAL_SRS_NATIVE +#ifdef SUPPORT_SRS US"inbound_srs", #endif US"inlist", @@ -386,7 +385,7 @@ enum { ECOND_STR_GEI, ECOND_STR_GT, ECOND_STR_GTI, -#ifdef EXPERIMENTAL_SRS_NATIVE +#ifdef SUPPORT_SRS ECOND_INBOUND_SRS, #endif ECOND_INLIST, @@ -594,7 +593,6 @@ static var_entry var_table[] = { { "local_part_prefix_v", vtype_stringptr, &deliver_localpart_prefix_v }, { "local_part_suffix", vtype_stringptr, &deliver_localpart_suffix }, { "local_part_suffix_v", vtype_stringptr, &deliver_localpart_suffix_v }, - { "local_part_verified", vtype_stringptr, &deliver_localpart_verified }, #ifdef HAVE_LOCAL_SCAN { "local_scan_data", vtype_stringptr, &local_scan_data }, #endif @@ -752,16 +750,16 @@ static var_entry var_table[] = { { "spool_directory", vtype_stringptr, &spool_directory }, { "spool_inodes", vtype_pinodes, (void *)TRUE }, { "spool_space", vtype_pspace, (void *)TRUE }, -#ifdef EXPERIMENTAL_SRS +#ifdef EXPERIMENTAL_SRS_ALT { "srs_db_address", vtype_stringptr, &srs_db_address }, { "srs_db_key", vtype_stringptr, &srs_db_key }, { "srs_orig_recipient", vtype_stringptr, &srs_orig_recipient }, { "srs_orig_sender", vtype_stringptr, &srs_orig_sender }, #endif -#if defined(EXPERIMENTAL_SRS) || defined(EXPERIMENTAL_SRS_NATIVE) +#if defined(EXPERIMENTAL_SRS_ALT) || defined(SUPPORT_SRS) { "srs_recipient", vtype_stringptr, &srs_recipient }, #endif -#ifdef EXPERIMENTAL_SRS +#ifdef EXPERIMENTAL_SRS_ALT { "srs_status", vtype_stringptr, &srs_status }, #endif { "thisaddress", vtype_stringptr, &filter_thisaddress }, @@ -779,7 +777,7 @@ static var_entry var_table[] = { { "tls_in_ourcert", vtype_cert, &tls_in.ourcert }, { "tls_in_peercert", vtype_cert, &tls_in.peercert }, { "tls_in_peerdn", vtype_stringptr, &tls_in.peerdn }, -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME { "tls_in_resumption", vtype_int, &tls_in.resumption }, #endif #ifndef DISABLE_TLS @@ -797,7 +795,7 @@ static var_entry var_table[] = { { "tls_out_ourcert", vtype_cert, &tls_out.ourcert }, { "tls_out_peercert", vtype_cert, &tls_out.peercert }, { "tls_out_peerdn", vtype_stringptr, &tls_out.peerdn }, -#ifdef EXPERIMENTAL_TLS_RESUME +#ifndef DISABLE_TLS_RESUME { "tls_out_resumption", vtype_int, &tls_out.resumption }, #endif #ifndef DISABLE_TLS @@ -1172,8 +1170,8 @@ Returns: NULL if the subfield was not found, or a pointer to the subfield's data */ -static uschar * -expand_getkeyed(uschar * key, const uschar * s) +uschar * +expand_getkeyed(const uschar * key, const uschar * s) { int length = Ustrlen(key); Uskip_whitespace(&s); @@ -1298,15 +1296,16 @@ expand_getlistele(int field, const uschar * list) { const uschar * tlist = list; int sep = 0; -uschar dummy; +/* Tainted mem for the throwaway element copies */ +uschar * dummy = store_get(2, TRUE); if (field < 0) { - for (field++; string_nextinlist(&tlist, &sep, &dummy, 1); ) field++; + for (field++; string_nextinlist(&tlist, &sep, dummy, 1); ) field++; sep = 0; } if (field == 0) return NULL; -while (--field > 0 && (string_nextinlist(&list, &sep, &dummy, 1))) ; +while (--field > 0 && (string_nextinlist(&list, &sep, dummy, 1))) ; return string_nextinlist(&list, &sep, NULL, 0); } @@ -1708,9 +1707,9 @@ authres_iprev(gstring * g) if (sender_host_name) g = string_append(g, 3, US";\n\tiprev=pass (", sender_host_name, US")"); else if (host_lookup_deferred) - g = string_catn(g, US";\n\tiprev=temperror", 19); + g = string_cat(g, US";\n\tiprev=temperror"); else if (host_lookup_failed) - g = string_catn(g, US";\n\tiprev=fail", 13); + g = string_cat(g, US";\n\tiprev=fail"); else return g; @@ -1855,7 +1854,7 @@ Returns: NULL if the variable does not exist, or something non-NULL if exists_only is TRUE */ -static uschar * +static const uschar * find_variable(uschar *name, BOOL exists_only, BOOL skipping, int *newsize) { var_entry * vp; @@ -1893,15 +1892,15 @@ if (Ustrncmp(name, "auth", 4) == 0) { uschar *endptr; int n = Ustrtoul(name + 4, &endptr, 10); - if (*endptr == 0 && n != 0 && n <= AUTH_VARS) - return !auth_vars[n-1] ? US"" : auth_vars[n-1]; + if (!*endptr && n != 0 && n <= AUTH_VARS) + return auth_vars[n-1] ? auth_vars[n-1] : US""; } else if (Ustrncmp(name, "regex", 5) == 0) { uschar *endptr; int n = Ustrtoul(name + 5, &endptr, 10); - if (*endptr == 0 && n != 0 && n <= REGEX_VARS) - return !regex_vars[n-1] ? US"" : regex_vars[n-1]; + if (!*endptr && n != 0 && n <= REGEX_VARS) + return regex_vars[n-1] ? regex_vars[n-1] : US""; } /* For all other variables, search the table */ @@ -1984,11 +1983,12 @@ switch (vp->type) ss = (uschar **)(val); if (!*ss && deliver_datafile >= 0) /* Read body when needed */ { - uschar *body; + uschar * body; off_t start_offset = SPOOL_DATA_START_OFFSET; int len = message_body_visible; + if (len > message_size) len = message_size; - *ss = body = store_malloc(len+1); + *ss = body = store_get(len+1, TRUE); body[0] = 0; if (vp->type == vtype_msgbody_end) { @@ -2003,8 +2003,7 @@ switch (vp->type) if (lseek(deliver_datafile, start_offset, SEEK_SET) < 0) log_write(0, LOG_MAIN|LOG_PANIC_DIE, "deliver_datafile lseek: %s", strerror(errno)); - len = read(deliver_datafile, body, len); - if (len > 0) + if ((len = read(deliver_datafile, body, len)) > 0) { body[len] = 0; if (message_body_newlines) /* Separate loops for efficiency */ @@ -2066,7 +2065,8 @@ switch (vp->type) case vtype_string_func: { stringptr_fn_t * fn = (stringptr_fn_t *) val; - return fn(); + uschar* s = fn(); + return s ? s : US""; } case vtype_pspace: @@ -2438,6 +2438,7 @@ else +#ifdef SUPPORT_SRS /* Do an hmac_md5. The result is _not_ nul-terminated, and is sized as the smaller of a full hmac_md5 result (16 bytes) or the supplied output buffer. @@ -2512,6 +2513,7 @@ for (int i = 0, j = len; i < MD5_HASHLEN; i++) } return; } +#endif /*SUPPORT_SRS*/ /************************************************* @@ -2541,16 +2543,13 @@ BOOL tempcond, combined_cond; BOOL *subcondptr; BOOL sub2_honour_dollar = TRUE; BOOL is_forany, is_json, is_jsons; -int rc, cond_type, roffset; +int rc, cond_type; int_eximarith_t num[2]; struct stat statbuf; uschar * opname; uschar name[256]; const uschar *sub[10]; -const pcre *re; -const uschar *rerror; - for (;;) if (Uskip_whitespace(&s) == '!') { testfor = !testfor; s++; } else break; @@ -2561,7 +2560,7 @@ switch(cond_type = identify_operator(&s, &opname)) case ECOND_DEF: { - uschar * t; + const uschar * t; if (*s != ':') { @@ -2972,15 +2971,24 @@ switch(cond_type = identify_operator(&s, &opname)) break; case ECOND_MATCH: /* Regular expression match */ - if (!(re = pcre_compile(CS sub[1], PCRE_COPT, CCSS &rerror, - &roffset, NULL))) { - expand_string_message = string_sprintf("regular expression error in " - "\"%s\": %s at offset %d", sub[1], rerror, roffset); - return NULL; + const pcre2_code * re; + PCRE2_SIZE offset; + int err; + + if (!(re = pcre2_compile((PCRE2_SPTR)sub[1], PCRE2_ZERO_TERMINATED, + PCRE_COPT, &err, &offset, pcre_cmp_ctx))) + { + uschar errbuf[128]; + pcre2_get_error_message(err, errbuf, sizeof(errbuf)); + expand_string_message = string_sprintf("regular expression error in " + "\"%s\": %s at offset %d", sub[1], errbuf, offset); + return NULL; + } + + tempcond = regex_match_and_setup(re, sub[0], 0, -1); + break; } - tempcond = regex_match_and_setup(re, sub[0], 0, -1); - break; case ECOND_MATCH_ADDRESS: /* Match in an address list */ rc = match_address_list(sub[0], TRUE, FALSE, &(sub[1]), NULL, -1, 0, NULL); @@ -3441,14 +3449,15 @@ switch(cond_type = identify_operator(&s, &opname)) return s; } -#ifdef EXPERIMENTAL_SRS_NATIVE +#ifdef SUPPORT_SRS case ECOND_INBOUND_SRS: /* ${if inbound_srs {local_part}{secret} {yes}{no}} */ { uschar * sub[2]; - const pcre * re; - int ovec[3*(4+1)]; - int n; + const pcre2_code * re; + pcre2_match_data * md; + PCRE2_SIZE * ovec; + int quoting = 0; uschar cksum[4]; BOOL boolvalue = FALSE; @@ -3464,17 +3473,29 @@ switch(cond_type = identify_operator(&s, &opname)) re = regex_must_compile(US"^(?i)SRS0=([^=]+)=([A-Z2-7]+)=([^=]*)=(.*)$", TRUE, FALSE); - if (pcre_exec(re, NULL, CS sub[0], Ustrlen(sub[0]), 0, PCRE_EOPT, - ovec, nelem(ovec)) < 0) + md = pcre2_match_data_create(4+1, pcre_gen_ctx); + if (pcre2_match(re, sub[0], PCRE2_ZERO_TERMINATED, 0, PCRE_EOPT, + md, pcre_mtc_ctx) < 0) { DEBUG(D_expand) debug_printf("no match for SRS'd local-part pattern\n"); goto srs_result; } + ovec = pcre2_get_ovector_pointer(md); + + if (sub[0][0] == '"') + quoting = 1; + else for (uschar * s = sub[0]; *s; s++) + if (!isalnum(*s) && Ustrchr(".!#$%&'*+-/=?^_`{|}~", *s) == NULL) + { quoting = 1; break; } + if (quoting) + DEBUG(D_expand) debug_printf_indent("auto-quoting local part\n"); - /* Side-effect: record the decoded recipient */ + /* Record the (quoted, if needed) decoded recipient as $srs_recipient */ - srs_recipient = string_sprintf("%.*S@%.*S", /* lowercased */ + srs_recipient = string_sprintf("%.*s%.*S%.*s@%.*S", /* lowercased */ + quoting, "\"", ovec[9]-ovec[8], sub[0] + ovec[8], /* substring 4 */ + quoting, "\"", ovec[7]-ovec[6], sub[0] + ovec[6]); /* substring 3 */ /* If a zero-length secret was given, we're done. Otherwise carry on @@ -3491,6 +3512,7 @@ switch(cond_type = identify_operator(&s, &opname)) struct timeval now; uschar * ss = sub[0] + ovec[4]; /* substring 2, the timestamp */ long d; + int n; gettimeofday(&now, NULL); now.tv_sec /= 86400; /* days since epoch */ @@ -3533,7 +3555,7 @@ srs_result: if (yield) *yield = (boolvalue == testfor); return s; } -#endif /*EXPERIMENTAL_SRS_NATIVE*/ +#endif /*SUPPORT_SRS*/ /* Unknown condition */ @@ -3584,7 +3606,7 @@ Returns: the value of expand max to save */ static int -save_expand_strings(uschar **save_expand_nstring, int *save_expand_nlength) +save_expand_strings(const uschar **save_expand_nstring, int *save_expand_nlength) { for (int i = 0; i <= expand_nmax; i++) { @@ -3611,7 +3633,7 @@ Returns: nothing */ static void -restore_expand_strings(int save_expand_nmax, uschar **save_expand_nstring, +restore_expand_strings(int save_expand_nmax, const uschar **save_expand_nstring, int *save_expand_nlength) { expand_nmax = save_expand_nmax; @@ -3927,7 +3949,7 @@ Arguments: Returns: new pointer for expandable string, terminated if non-null */ -static gstring * +gstring * cat_file(FILE *f, gstring *yield, uschar *eol) { uschar buffer[1024]; @@ -3947,7 +3969,7 @@ return yield; #ifndef DISABLE_TLS -static gstring * +gstring * cat_file_tls(void * tls_ctx, gstring * yield, uschar * eol) { int rc; @@ -4289,6 +4311,98 @@ return FALSE; /* should not happen */ } +/* Expand a named list. Return false on failure. */ +static gstring * +expand_listnamed(gstring * yield, const uschar * name, const uschar * listtype) +{ +tree_node *t = NULL; +const uschar * list; +int sep = 0; +uschar * item; +uschar * suffix = US""; +BOOL needsep = FALSE; +#define LISTNAMED_BUF_SIZE 256 +uschar b[LISTNAMED_BUF_SIZE]; +uschar * buffer = b; + +if (*name == '+') name++; +if (!listtype) /* no-argument version */ + { + if ( !(t = tree_search(addresslist_anchor, name)) + && !(t = tree_search(domainlist_anchor, name)) + && !(t = tree_search(hostlist_anchor, name))) + t = tree_search(localpartlist_anchor, name); + } +else switch(*listtype) /* specific list-type version */ + { + case 'a': t = tree_search(addresslist_anchor, name); suffix = US"_a"; break; + case 'd': t = tree_search(domainlist_anchor, name); suffix = US"_d"; break; + case 'h': t = tree_search(hostlist_anchor, name); suffix = US"_h"; break; + case 'l': t = tree_search(localpartlist_anchor, name); suffix = US"_l"; break; + default: + expand_string_message = US"bad suffix on \"list\" operator"; + return yield; + } + +if(!t) + { + expand_string_message = string_sprintf("\"%s\" is not a %snamed list", + name, !listtype?"" + : *listtype=='a'?"address " + : *listtype=='d'?"domain " + : *listtype=='h'?"host " + : *listtype=='l'?"localpart " + : 0); + return yield; + } + +list = ((namedlist_block *)(t->data.ptr))->string; + +/* The list could be quite long so we (re)use a buffer for each element +rather than getting each in new memory */ + +if (is_tainted(list)) buffer = store_get(LISTNAMED_BUF_SIZE, TRUE); +while ((item = string_nextinlist(&list, &sep, buffer, LISTNAMED_BUF_SIZE))) + { + uschar * buf = US" : "; + if (needsep) + yield = string_catn(yield, buf, 3); + else + needsep = TRUE; + + if (*item == '+') /* list item is itself a named list */ + { + yield = expand_listnamed(yield, item, listtype); + if (expand_string_message) + return yield; + } + + else if (sep != ':') /* item from non-colon-sep list, re-quote for colon list-separator */ + { + char tok[3]; + tok[0] = sep; tok[1] = ':'; tok[2] = 0; + + for(char * cp; cp = strpbrk(CCS item, tok); item = US cp) + { + yield = string_catn(yield, item, cp - CS item); + if (*cp++ == ':') /* colon in a non-colon-sep list item, needs doubling */ + yield = string_catn(yield, US"::", 2); + else /* sep in item; should already be doubled; emit once */ + { + yield = string_catn(yield, US tok, 1); + if (*cp == sep) cp++; + } + } + yield = string_cat(yield, item); + } + else + yield = string_cat(yield, item); + } +return yield; +} + + + /************************************************* * Expand string * *************************************************/ @@ -4360,7 +4474,7 @@ rmark reset_point = store_mark(); gstring * yield = string_get(Ustrlen(string) + 64); int item_type; const uschar *s = string; -uschar *save_expand_nstring[EXPAND_MAXN+1]; +const uschar *save_expand_nstring[EXPAND_MAXN+1]; int save_expand_nlength[EXPAND_MAXN+1]; BOOL resetok = TRUE; @@ -4379,17 +4493,16 @@ DEBUG(D_expand) f.expand_string_forcedfail = FALSE; expand_string_message = US""; -if (is_tainted(string)) +{ uschar *m; +if ((m = is_tainted2(string, LOG_MAIN|LOG_PANIC, "Tainted string '%s' in expansion", s))) { - expand_string_message = - string_sprintf("attempt to expand tainted string '%s'", s); - log_write(0, LOG_MAIN|LOG_PANIC, "%s", expand_string_message); + expand_string_message = m; goto EXPAND_FAILED; } +} -while (*s != 0) +while (*s) { - uschar *value; uschar name[256]; /* \ escapes the next character, which must exist, or else @@ -4447,6 +4560,7 @@ while (*s != 0) if (isalpha((*(++s)))) { + const uschar * value; int len; int newsize = 0; gstring * g = NULL; @@ -4489,7 +4603,7 @@ while (*s != 0) But there is no error here - nothing gets inserted. */ if (!value) - { + { /*{*/ if (Ustrchr(name, '}')) malformed_header = TRUE; continue; } @@ -4519,7 +4633,7 @@ while (*s != 0) yield = g; yield->size = newsize; yield->ptr = len; - yield->s = value; + yield->s = US value; /* known to be in new store i.e. a copy, so deconst safe */ } else yield = string_catn(yield, value, len); @@ -4773,7 +4887,7 @@ while (*s != 0) int save_expand_nmax = save_expand_strings(save_expand_nstring, save_expand_nlength); - if ((expand_forbid & RDO_LOOKUP) != 0) + if (expand_forbid & RDO_LOOKUP) { expand_string_message = US"lookup expansions are not permitted"; goto EXPAND_FAILED; @@ -4872,21 +4986,7 @@ while (*s != 0) file types, the query (i.e. "key") starts with a file name. */ if (!key) - { - Uskip_whitespace(&filename); - key = filename; - - if (mac_islookup(stype, lookup_querystyle)) - filename = NULL; - else - if (*filename == '/') - { - while (*key && !isspace(*key)) key++; - if (*key) *key++ = '\0'; - } - else - filename = NULL; - } + key = search_args(stype, name, filename, &filename, opts); /* If skipping, don't do the next bit - just lookup_value == NULL, as if the entry was not found. Note that there is no search_close() function. @@ -4917,7 +5017,7 @@ while (*s != 0) { expand_string_message = string_sprintf("lookup of \"%s\" gave DEFER: %s", - string_printing2(key, FALSE), search_error_message); + string_printing2(key, SP_TAB), search_error_message); goto EXPAND_FAILED; } if (expand_setup > 0) expand_nmax = expand_setup; @@ -5099,7 +5199,7 @@ while (*s != 0) { uschar *sub_arg[3]; gstring * g; - const pcre *re; + const pcre2_code *re; uschar *p; /* TF: Ugliness: We want to expand parameter 1 first, then set @@ -5262,7 +5362,7 @@ while (*s != 0) if (!(f = Ufopen(sub_arg[0], "rb"))) { - expand_string_message = string_open_failed(errno, "%s", sub_arg[0]); + expand_string_message = string_open_failed("%s", sub_arg[0]); goto EXPAND_FAILED; } @@ -5276,17 +5376,8 @@ while (*s != 0) case EITEM_READSOCK: { - client_conn_ctx cctx; - int timeout = 5; - int save_ptr = gstring_length(yield); - FILE * fp = NULL; uschar * arg; uschar * sub_arg[4]; - uschar * server_name = NULL; - host_item host; - BOOL do_shutdown = TRUE; - BOOL do_tls = FALSE; /* Only set under ! DISABLE_TLS */ - blob reqstr; if (expand_forbid & RDO_READSOCK) { @@ -5304,218 +5395,80 @@ while (*s != 0) case 3: goto EXPAND_FAILED; } - /* Grab the request string, if any */ - - reqstr.data = sub_arg[1]; - reqstr.len = Ustrlen(sub_arg[1]); - - /* Sort out timeout, if given. The second arg is a list with the first element - being a time value. Any more are options of form "name=value". Currently the - only options recognised are "shutdown" and "tls". */ - - if (sub_arg[2]) - { - const uschar * list = sub_arg[2]; - uschar * item; - int sep = 0; - - if ( !(item = string_nextinlist(&list, &sep, NULL, 0)) - || !*item - || (timeout = readconf_readtime(item, 0, FALSE)) < 0) - { - expand_string_message = string_sprintf("bad time value %s", item); - goto EXPAND_FAILED; - } - - while ((item = string_nextinlist(&list, &sep, NULL, 0))) - if (Ustrncmp(item, US"shutdown=", 9) == 0) - { if (Ustrcmp(item + 9, US"no") == 0) do_shutdown = FALSE; } -#ifndef DISABLE_TLS - else if (Ustrncmp(item, US"tls=", 4) == 0) - { if (Ustrcmp(item + 9, US"no") != 0) do_tls = TRUE; } -#endif - } - else - sub_arg[3] = NULL; /* No eol if no timeout */ - /* If skipping, we don't actually do anything. Otherwise, arrange to connect to either an IP or a Unix socket. */ if (!skipping) { - /* Handle an IP (internet) domain */ - - if (Ustrncmp(sub_arg[0], "inet:", 5) == 0) - { - int port; - uschar * port_name; - - server_name = sub_arg[0] + 5; - port_name = Ustrrchr(server_name, ':'); - - /* Sort out the port */ - - if (!port_name) - { - expand_string_message = - string_sprintf("missing port for readsocket %s", sub_arg[0]); - goto EXPAND_FAILED; - } - *port_name++ = 0; /* Terminate server name */ - - if (isdigit(*port_name)) - { - uschar *end; - port = Ustrtol(port_name, &end, 0); - if (end != port_name + Ustrlen(port_name)) - { - expand_string_message = - string_sprintf("invalid port number %s", port_name); - goto EXPAND_FAILED; - } - } - else - { - struct servent *service_info = getservbyname(CS port_name, "tcp"); - if (!service_info) - { - expand_string_message = string_sprintf("unknown port \"%s\"", - port_name); - goto EXPAND_FAILED; - } - port = ntohs(service_info->s_port); - } - - /*XXX we trust that the request is idempotent for TFO. Hmm. */ - cctx.sock = ip_connectedsocket(SOCK_STREAM, server_name, port, port, - timeout, &host, &expand_string_message, - do_tls ? NULL : &reqstr); - callout_address = NULL; - if (cctx.sock < 0) - goto SOCK_FAIL; - if (!do_tls) - reqstr.len = 0; - } - - /* Handle a Unix domain socket */ - - else - { - struct sockaddr_un sockun; /* don't call this "sun" ! */ - int rc; - - if ((cctx.sock = socket(PF_UNIX, SOCK_STREAM, 0)) == -1) - { - expand_string_message = string_sprintf("failed to create socket: %s", - strerror(errno)); - goto SOCK_FAIL; - } - - sockun.sun_family = AF_UNIX; - sprintf(sockun.sun_path, "%.*s", (int)(sizeof(sockun.sun_path)-1), - sub_arg[0]); - server_name = US sockun.sun_path; - - sigalrm_seen = FALSE; - ALARM(timeout); - rc = connect(cctx.sock, (struct sockaddr *)(&sockun), sizeof(sockun)); - ALARM_CLR(0); - if (sigalrm_seen) - { - expand_string_message = US "socket connect timed out"; - goto SOCK_FAIL; - } - if (rc < 0) - { - expand_string_message = string_sprintf("failed to connect to socket " - "%s: %s", sub_arg[0], strerror(errno)); - goto SOCK_FAIL; - } - host.name = server_name; - host.address = US""; - } + int stype = search_findtype(US"readsock", 8); + gstring * g = NULL; + void * handle; + int expand_setup = -1; + uschar * s; - DEBUG(D_expand) debug_printf_indent("connected to socket %s\n", sub_arg[0]); + /* If the reqstr is empty, flag that and set a dummy */ -#ifndef DISABLE_TLS - if (do_tls) + if (!sub_arg[1][0]) { - smtp_connect_args conn_args = {.host = &host }; - tls_support tls_dummy = {.sni=NULL}; - uschar * errstr; - - if (!tls_client_start(&cctx, &conn_args, NULL, &tls_dummy, &errstr)) - { - expand_string_message = string_sprintf("TLS connect failed: %s", errstr); - goto SOCK_FAIL; - } + g = string_append_listele(g, ',', US"send=no"); + sub_arg[1] = US"DUMMY"; } -#endif - - /* Allow sequencing of test actions */ - testharness_pause_ms(100); - /* Write the request string, if not empty or already done */ + /* Re-marshall the options */ - if (reqstr.len) - { - DEBUG(D_expand) debug_printf_indent("writing \"%s\" to socket\n", - reqstr.data); - if ( ( -#ifndef DISABLE_TLS - do_tls ? tls_write(cctx.tls_ctx, reqstr.data, reqstr.len, FALSE) : -#endif - write(cctx.sock, reqstr.data, reqstr.len)) != reqstr.len) - { - expand_string_message = string_sprintf("request write to socket " - "failed: %s", strerror(errno)); - goto SOCK_FAIL; - } - } - - /* Shut down the sending side of the socket. This helps some servers to - recognise that it is their turn to do some work. Just in case some - system doesn't have this function, make it conditional. */ - -#ifdef SHUT_WR - if (!do_tls && do_shutdown) shutdown(cctx.sock, SHUT_WR); -#endif + if (sub_arg[2]) + { + const uschar * list = sub_arg[2]; + uschar * item; + int sep = 0; + + /* First option has no tag and is timeout */ + if ((item = string_nextinlist(&list, &sep, NULL, 0))) + g = string_append_listele(g, ',', + string_sprintf("timeout=%s", item)); + + /* The rest of the options from the expansion */ + while ((item = string_nextinlist(&list, &sep, NULL, 0))) + g = string_append_listele(g, ',', item); + + /* possibly plus an EOL string. Process with escapes, to protect + from list-processing. The only current user of eol= in search + options is the readsock expansion. */ + + if (sub_arg[3] && *sub_arg[3]) + g = string_append_listele(g, ',', + string_sprintf("eol=%s", + string_printing2(sub_arg[3], SP_TAB|SP_SPACE))); + } - testharness_pause_ms(100); + /* Gat a (possibly cached) handle for the connection */ - /* Now we need to read from the socket, under a timeout. The function - that reads a file can be used. */ + if (!(handle = search_open(sub_arg[0], stype, 0, NULL, NULL))) + { + if (*expand_string_message) goto EXPAND_FAILED; + expand_string_message = search_error_message; + search_error_message = NULL; + goto SOCK_FAIL; + } - if (!do_tls) - fp = fdopen(cctx.sock, "rb"); - sigalrm_seen = FALSE; - ALARM(timeout); - yield = -#ifndef DISABLE_TLS - do_tls ? cat_file_tls(cctx.tls_ctx, yield, sub_arg[3]) : -#endif - cat_file(fp, yield, sub_arg[3]); - ALARM_CLR(0); + /* Get (possibly cached) results for the lookup */ + /* sspec: sub_arg[0] req: sub_arg[1] opts: g */ -#ifndef DISABLE_TLS - if (do_tls) + if ((s = search_find(handle, sub_arg[0], sub_arg[1], -1, NULL, 0, 0, + &expand_setup, string_from_gstring(g)))) + yield = string_cat(yield, s); + else if (f.search_find_defer) { - tls_close(cctx.tls_ctx, TRUE); - close(cctx.sock); + expand_string_message = search_error_message; + search_error_message = NULL; + goto SOCK_FAIL; } else -#endif - (void)fclose(fp); - - /* After a timeout, we restore the pointer in the result, that is, - make sure we add nothing from the socket. */ - - if (sigalrm_seen) - { - if (yield) yield->ptr = save_ptr; - expand_string_message = US "socket read timed out"; - goto SOCK_FAIL; - } + { /* should not happen, at present */ + expand_string_message = search_error_message; + search_error_message = NULL; + goto SOCK_FAIL; + } } /* The whole thing has worked (or we were skipping). If there is a @@ -5886,11 +5839,11 @@ while (*s != 0) case EITEM_SG: { - const pcre *re; + const pcre2_code * re; int moffset, moffsetextra, slen; - int roffset; - int emptyopt; - const uschar *rerror; + PCRE2_SIZE roffset; + pcre2_match_data * md; + int err, emptyopt; uschar *subject; uschar *sub[3]; int save_expand_nmax = @@ -5905,13 +5858,16 @@ while (*s != 0) /* Compile the regular expression */ - if (!(re = pcre_compile(CS sub[1], PCRE_COPT, CCSS &rerror, - &roffset, NULL))) + if (!(re = pcre2_compile((PCRE2_SPTR)sub[1], PCRE2_ZERO_TERMINATED, + PCRE_COPT, &err, &roffset, pcre_cmp_ctx))) { + uschar errbuf[128]; + pcre2_get_error_message(err, errbuf, sizeof(errbuf)); expand_string_message = string_sprintf("regular expression error in " - "\"%s\": %s at offset %d", sub[1], rerror, roffset); + "\"%s\": %s at offset %ld", sub[1], errbuf, (long)roffset); goto EXPAND_FAILED; } + md = pcre2_match_data_create(EXPAND_MAXN + 1, pcre_gen_ctx); /* Now run a loop to do the substitutions as often as necessary. It ends when there are no more matches. Take care over matches of the null string; @@ -5924,9 +5880,9 @@ while (*s != 0) for (;;) { - int ovector[3*(EXPAND_MAXN+1)]; - int n = pcre_exec(re, NULL, CS subject, slen, moffset + moffsetextra, - PCRE_EOPT | emptyopt, ovector, nelem(ovector)); + PCRE2_SIZE * ovec = pcre2_get_ovector_pointer(md); + int n = pcre2_match(re, (PCRE2_SPTR)subject, slen, moffset + moffsetextra, + PCRE_EOPT | emptyopt, md, pcre_mtc_ctx); uschar *insert; /* No match - if we previously set PCRE_NOTEMPTY after a null match, this @@ -5954,19 +5910,19 @@ while (*s != 0) expand_nmax = 0; for (int nn = 0; nn < n*2; nn += 2) { - expand_nstring[expand_nmax] = subject + ovector[nn]; - expand_nlength[expand_nmax++] = ovector[nn+1] - ovector[nn]; + expand_nstring[expand_nmax] = subject + ovec[nn]; + expand_nlength[expand_nmax++] = ovec[nn+1] - ovec[nn]; } expand_nmax--; /* Copy the characters before the match, plus the expanded insertion. */ - yield = string_catn(yield, subject + moffset, ovector[0] - moffset); + yield = string_catn(yield, subject + moffset, ovec[0] - moffset); if (!(insert = expand_string(sub[2]))) goto EXPAND_FAILED; yield = string_cat(yield, insert); - moffset = ovector[1]; + moffset = ovec[1]; moffsetextra = 0; emptyopt = 0; @@ -5977,10 +5933,10 @@ while (*s != 0) string at the same point. If this fails (picked up above) we advance to the next character. */ - if (ovector[0] == ovector[1]) + if (ovec[0] == ovec[1]) { - if (ovector[0] == slen) break; - emptyopt = PCRE_NOTEMPTY | PCRE_ANCHORED; + if (ovec[0] == slen) break; + emptyopt = PCRE2_NOTEMPTY | PCRE2_ANCHORED; } } @@ -6334,11 +6290,12 @@ while (*s != 0) case 2: case 3: goto EXPAND_FAILED; } - for (uschar sep = *sub[0], c; c = *sub[1]; sub[1]++) + if (*sub[1]) for (uschar sep = *sub[0], c; c = *sub[1]; sub[1]++) { if (c == sep) yield = string_catn(yield, sub[1], 1); yield = string_catn(yield, sub[1], 1); } + else yield = string_catn(yield, US" ", 1); continue; } @@ -6487,13 +6444,10 @@ while (*s != 0) condition for real. For EITEM_MAP and EITEM_REDUCE, do the same, using the normal internal expansion function. */ - if (item_type == EITEM_FILTER) - { - if ((temp = eval_condition(expr, &resetok, NULL))) - s = temp; - } - else + if (item_type != EITEM_FILTER) temp = expand_string_internal(s, TRUE, &s, TRUE, TRUE, &resetok); + else + if ((temp = eval_condition(expr, &resetok, NULL))) s = temp; if (!temp) { @@ -6502,7 +6456,7 @@ while (*s != 0) goto EXPAND_FAILED; } - Uskip_whitespace(&s); + Uskip_whitespace(&s); /*{*/ if (*s++ != '}') { /*{*/ expand_string_message = string_sprintf("missing } at end of condition " @@ -6931,12 +6885,14 @@ while (*s != 0) continue; } -#ifdef EXPERIMENTAL_SRS_NATIVE +#ifdef SUPPORT_SRS case EITEM_SRS_ENCODE: /* ${srs_encode {secret} {return_path} {orig_domain}} */ { uschar * sub[3]; uschar cksum[4]; + gstring * g = NULL; + BOOL quoted = FALSE; switch (read_subs(sub, 3, 3, CUSS &s, skipping, TRUE, name, &resetok)) { @@ -6945,47 +6901,71 @@ while (*s != 0) case 3: goto EXPAND_FAILED; } - yield = string_catn(yield, US"SRS0=", 5); + g = string_catn(g, US"SRS0=", 5); /* ${l_4:${hmac{md5}{SRS_SECRET}{${lc:$return_path}}}}= */ hmac_md5(sub[0], string_copylc(sub[1]), cksum, sizeof(cksum)); - yield = string_catn(yield, cksum, sizeof(cksum)); - yield = string_catn(yield, US"=", 1); + g = string_catn(g, cksum, sizeof(cksum)); + g = string_catn(g, US"=", 1); /* ${base32:${eval:$tod_epoch/86400&0x3ff}}= */ { struct timeval now; unsigned long i; - gstring * g = NULL; + gstring * h = NULL; gettimeofday(&now, NULL); for (unsigned long i = (now.tv_sec / 86400) & 0x3ff; i; i >>= 5) - g = string_catn(g, &base32_chars[i & 0x1f], 1); - if (g) while (g->ptr > 0) - yield = string_catn(yield, &g->s[--g->ptr], 1); + h = string_catn(h, &base32_chars[i & 0x1f], 1); + if (h) while (h->ptr > 0) + g = string_catn(g, &h->s[--h->ptr], 1); } - yield = string_catn(yield, US"=", 1); + g = string_catn(g, US"=", 1); /* ${domain:$return_path}=${local_part:$return_path} */ { int start, end, domain; uschar * t = parse_extract_address(sub[1], &expand_string_message, &start, &end, &domain, FALSE); + uschar * s; + if (!t) goto EXPAND_FAILED; - if (domain > 0) yield = string_cat(yield, t + domain); - yield = string_catn(yield, US"=", 1); - yield = domain > 0 - ? string_catn(yield, t, domain - 1) : string_cat(yield, t); + if (domain > 0) g = string_cat(g, t + domain); + g = string_catn(g, US"=", 1); + + s = domain > 0 ? string_copyn(t, domain - 1) : t; + if ((quoted = Ustrchr(s, '"') != NULL)) + { + gstring * h = NULL; + DEBUG(D_expand) debug_printf_indent("auto-quoting local part\n"); + while (*s) /* de-quote */ + { + while (*s && *s != '"') h = string_catn(h, s++, 1); + if (*s) s++; + while (*s && *s != '"') h = string_catn(h, s++, 1); + if (*s) s++; + } + gstring_release_unused(h); + s = string_from_gstring(h); + } + g = string_cat(g, s); } + /* Assume that if the original local_part had quotes + it was for good reason */ + + if (quoted) yield = string_catn(yield, US"\"", 1); + yield = string_catn(yield, g->s, g->ptr); + if (quoted) yield = string_catn(yield, US"\"", 1); + /* @$original_domain */ yield = string_catn(yield, US"@", 1); yield = string_cat(yield, sub[2]); continue; } -#endif /*EXPERIMENTAL_SRS_NATIVE*/ +#endif /*SUPPORT_SRS*/ } /* EITEM_* switch */ /* Control reaches here if the name is not recognized as one of the more @@ -7140,20 +7120,6 @@ while (*s != 0) continue; } - case EOP_BLESS: - /* This is purely for the convenience of the test harness. Do not enable - it otherwise as it defeats the taint-checking security. */ - - if (f.running_in_test_harness) - yield = string_cat(yield, is_tainted(sub) - ? string_copy_taint(sub, FALSE) : sub); - else - { - DEBUG(D_expand) debug_printf_indent("bless operator not supported\n"); - yield = string_cat(yield, sub); - } - continue; - case EOP_EXPAND: { uschar *expanded = expand_string_internal(sub, FALSE, NULL, skipping, TRUE, &resetok); @@ -7352,11 +7318,10 @@ while (*s != 0) case EOP_LISTCOUNT: { - int cnt = 0; - int sep = 0; - uschar buffer[256]; + int cnt = 0, sep = 0; + uschar * buf = store_get(2, is_tainted(sub)); - while (string_nextinlist(CUSS &sub, &sep, buffer, sizeof(buffer))) cnt++; + while (string_nextinlist(CUSS &sub, &sep, buf, 1)) cnt++; yield = string_fmt_append(yield, "%d", cnt); continue; } @@ -7365,86 +7330,11 @@ while (*s != 0) /* handles nested named lists; requotes as colon-sep list */ case EOP_LISTNAMED: - { - tree_node *t = NULL; - const uschar * list; - int sep = 0; - uschar * item; - uschar * suffix = US""; - BOOL needsep = FALSE; - uschar buffer[256]; - - if (*sub == '+') sub++; - if (!arg) /* no-argument version */ - { - if (!(t = tree_search(addresslist_anchor, sub)) && - !(t = tree_search(domainlist_anchor, sub)) && - !(t = tree_search(hostlist_anchor, sub))) - t = tree_search(localpartlist_anchor, sub); - } - else switch(*arg) /* specific list-type version */ - { - case 'a': t = tree_search(addresslist_anchor, sub); suffix = US"_a"; break; - case 'd': t = tree_search(domainlist_anchor, sub); suffix = US"_d"; break; - case 'h': t = tree_search(hostlist_anchor, sub); suffix = US"_h"; break; - case 'l': t = tree_search(localpartlist_anchor, sub); suffix = US"_l"; break; - default: - expand_string_message = US"bad suffix on \"list\" operator"; - goto EXPAND_FAILED; - } - - if(!t) - { - expand_string_message = string_sprintf("\"%s\" is not a %snamed list", - sub, !arg?"" - : *arg=='a'?"address " - : *arg=='d'?"domain " - : *arg=='h'?"host " - : *arg=='l'?"localpart " - : 0); + expand_string_message = NULL; + yield = expand_listnamed(yield, sub, arg); + if (expand_string_message) goto EXPAND_FAILED; - } - - list = ((namedlist_block *)(t->data.ptr))->string; - - while ((item = string_nextinlist(&list, &sep, buffer, sizeof(buffer)))) - { - uschar * buf = US" : "; - if (needsep) - yield = string_catn(yield, buf, 3); - else - needsep = TRUE; - - if (*item == '+') /* list item is itself a named list */ - { - uschar * sub = string_sprintf("${listnamed%s:%s}", suffix, item); - item = expand_string_internal(sub, FALSE, NULL, FALSE, TRUE, &resetok); - } - else if (sep != ':') /* item from non-colon-sep list, re-quote for colon list-separator */ - { - char * cp; - char tok[3]; - tok[0] = sep; tok[1] = ':'; tok[2] = 0; - while ((cp= strpbrk(CCS item, tok))) - { - yield = string_catn(yield, item, cp - CS item); - if (*cp++ == ':') /* colon in a non-colon-sep list item, needs doubling */ - { - yield = string_catn(yield, US"::", 2); - item = US cp; - } - else /* sep in item; should already be doubled; emit once */ - { - yield = string_catn(yield, US tok, 1); - if (*cp == sep) cp++; - item = US cp; - } - } - } - yield = string_cat(yield, item); - } continue; - } /* quote a list-item for the given list-separator */ @@ -7456,11 +7346,11 @@ while (*s != 0) int count; uschar *endptr; int binary[4]; - int mask, maskoffset; - int type = string_is_ip_address(sub, &maskoffset); + int type, mask, maskoffset; + BOOL normalised; uschar buffer[64]; - if (type == 0) + if ((type = string_is_ip_address(sub, &maskoffset)) == 0) { expand_string_message = string_sprintf("\"%s\" is not an IP address", sub); @@ -7476,13 +7366,18 @@ while (*s != 0) mask = Ustrtol(sub + maskoffset + 1, &endptr, 10); - if (*endptr != 0 || mask < 0 || mask > ((type == 4)? 32 : 128)) + if (*endptr || mask < 0 || mask > (type == 4 ? 32 : 128)) { expand_string_message = string_sprintf("mask value too big in \"%s\"", sub); goto EXPAND_FAILED; } + /* If an optional 'n' was given, ipv6 gets normalised output: + colons rather than dots, and zero-compressed. */ + + normalised = arg && *arg == 'n'; + /* Convert the address to binary integer(s) and apply the mask */ sub[maskoffset] = 0; @@ -7491,8 +7386,14 @@ while (*s != 0) /* Convert to masked textual format and add to output. */ - yield = string_catn(yield, buffer, - host_nmtoa(count, binary, mask, buffer, '.')); + if (type == 4 || !normalised) + yield = string_catn(yield, buffer, + host_nmtoa(count, binary, mask, buffer, '.')); + else + { + ipv6_nmtoa(binary, buffer); + yield = string_fmt_append(yield, "%s/%d", buffer, mask); + } continue; } @@ -7632,24 +7533,20 @@ while (*s != 0) uschar *t = sub - 1; if (c == EOP_QUOTE) - { - while (!needs_quote && *(++t) != 0) + while (!needs_quote && *++t) needs_quote = !isalnum(*t) && !strchr("_-.", *t); - } + else /* EOP_QUOTE_LOCAL_PART */ - { - while (!needs_quote && *(++t) != 0) - needs_quote = !isalnum(*t) && - strchr("!#$%&'*+-/=?^_`{|}~", *t) == NULL && - (*t != '.' || t == sub || t[1] == 0); - } + while (!needs_quote && *++t) + needs_quote = !isalnum(*t) + && strchr("!#$%&'*+-/=?^_`{|}~", *t) == NULL + && (*t != '.' || t == sub || !t[1]); if (needs_quote) { yield = string_catn(yield, US"\"", 1); t = sub - 1; - while (*(++t) != 0) - { + while (*++t) if (*t == '\n') yield = string_catn(yield, US"\\n", 2); else if (*t == '\r') @@ -7660,10 +7557,10 @@ while (*s != 0) yield = string_catn(yield, US"\\", 1); yield = string_catn(yield, t, 1); } - } yield = string_catn(yield, US"\"", 1); } - else yield = string_cat(yield, sub); + else + yield = string_cat(yield, sub); continue; } @@ -7718,13 +7615,10 @@ while (*s != 0) prescribed by the RFC, if there are characters that need to be encoded */ case EOP_RFC2047: - { - uschar buffer[2048]; yield = string_cat(yield, parse_quote_2047(sub, Ustrlen(sub), headers_charset, - buffer, sizeof(buffer), FALSE)); + FALSE)); continue; - } /* RFC 2047 decode */ @@ -7775,10 +7669,12 @@ while (*s != 0) /* Manually track tainting, as we deal in individual chars below */ if (is_tainted(sub)) + { if (yield->s && yield->ptr) gstring_rebuffer(yield); else yield->s = store_get(yield->size = Ustrlen(sub), TRUE); + } /* Check the UTF-8, byte-by-byte */ @@ -8218,6 +8114,7 @@ while (*s != 0) /*{*/ if (*s++ == '}') { + const uschar * value; int len; int newsize = 0; gstring * g = NULL; @@ -8244,7 +8141,7 @@ while (*s != 0) yield = g; yield->size = newsize; yield->ptr = len; - yield->s = value; + yield->s = US value; /* known to be in new store i.e. a copy, so deconst safe */ } else yield = string_catn(yield, value, len); @@ -8339,6 +8236,7 @@ that is a bad idea, because expand_string_message is in dynamic store. */ EXPAND_FAILED: if (left) *left = s; DEBUG(D_expand) + { DEBUG(D_noutf8) { debug_printf_indent("|failed to expand: %s\n", string); @@ -8358,6 +8256,7 @@ DEBUG(D_expand) if (f.expand_string_forcedfail) debug_printf_indent(UTF8_UP_RIGHT "failure was forced\n"); } + } if (resetok_p && !resetok) *resetok_p = FALSE; expand_level--; return NULL; @@ -8665,6 +8564,7 @@ typedef struct { const uschar *var_data; } err_ctx; +/* Called via tree_walk, which allows nonconst name/data. Our usage is const. */ static void assert_variable_notin(uschar * var_name, uschar * var_data, void * ctx) { @@ -8686,13 +8586,14 @@ err_ctx e = { .region_start = ptr, .region_end = US ptr + len, tree_walk(acl_var_c, assert_variable_notin, &e); tree_walk(acl_var_m, assert_variable_notin, &e); -/* check auth variables */ +/* check auth variables. +assert_variable_notin() treats as const, so deconst is safe. */ for (int i = 0; i < AUTH_VARS; i++) if (auth_vars[i]) - assert_variable_notin(US"auth", auth_vars[i], &e); + assert_variable_notin(US"auth", US auth_vars[i], &e); -/* check regex variables */ +/* check regex variables. assert_variable_notin() treats as const. */ for (int i = 0; i < REGEX_VARS; i++) if (regex_vars[i]) - assert_variable_notin(US"regex", regex_vars[i], &e); + assert_variable_notin(US"regex", US regex_vars[i], &e); /* check known-name variables */ for (var_entry * v = var_table; v < var_table + var_table_size; v++) @@ -8723,11 +8624,11 @@ if (e.var_name) BOOL -regex_match_and_setup(const pcre *re, uschar *subject, int options, int setup) +regex_match_and_setup(const pcre2_code *re, uschar *subject, int options, int setup) { -int ovector[3*(EXPAND_MAXN+1)]; +int ovec[3*(EXPAND_MAXN+1)]; int n = pcre_exec(re, NULL, subject, Ustrlen(subject), 0, PCRE_EOPT|options, - ovector, nelem(ovector)); + ovec, nelem(ovec)); BOOL yield = n >= 0; if (n == 0) n = EXPAND_MAXN + 1; if (yield) @@ -8735,8 +8636,8 @@ if (yield) expand_nmax = setup < 0 ? 0 : setup + 1; for (int nn = setup < 0 ? 0 : 2; nn < n*2; nn += 2) { - expand_nstring[expand_nmax] = subject + ovector[nn]; - expand_nlength[expand_nmax++] = ovector[nn+1] - ovector[nn]; + expand_nstring[expand_nmax] = subject + ovec[nn]; + expand_nlength[expand_nmax++] = ovec[nn+1] - ovec[nn]; } expand_nmax--; } @@ -8752,6 +8653,7 @@ debug_selector = D_v; debug_file = stderr; debug_fd = fileno(debug_file); big_buffer = malloc(big_buffer_size); +store_init(); for (int i = 1; i < argc; i++) {