From c246a1de8851d7810d53de3cda953d88f7139639 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sat, 28 Oct 2017 21:36:13 +0100 Subject: [PATCH] Add macro support to -be expansion test mode. Bug 1623 --- doc/doc-docbook/spec.xfpt | 7 ++ doc/doc-txt/NewStuff | 3 + src/src/exim.c | 73 +++++++---- src/src/functions.h | 2 + src/src/readconf.c | 231 +++++++++++++++++++++-------------- test/scripts/0000-Basic/0542 | 10 +- test/stdout/0542 | 10 +- 7 files changed, 208 insertions(+), 128 deletions(-) diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt index a9a048ecb..3fe2afb3e 100644 --- a/doc/doc-docbook/spec.xfpt +++ b/doc/doc-docbook/spec.xfpt @@ -2791,6 +2791,13 @@ files or databases you are using, you must exit and restart Exim before trying the same lookup again. Otherwise, because each Exim process caches the results of lookups, you will just get the same result as before. +.new +Macro processing is done on lines before string-expansion: new macros can be +defined and macros will be expanded. +Because macros in thc config file are often used for secrets, those are only +available to admin users. +.wen + .vitem &%-bem%&&~<&'filename'&> .oindex "&%-bem%&" .cindex "testing" "string expansion" diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff index 6d875d5f4..22af13554 100644 --- a/doc/doc-txt/NewStuff +++ b/doc/doc-txt/NewStuff @@ -61,6 +61,9 @@ Version 4.90 15. TCP Fast Open used, with data-on-SYN, for client SMTP via SOCKS5 proxy, for ${readsocket } expansions, and for ClamAV. +16. The "-be" expansion test mode now supports macros. Macros are expanded + in test lines, and new macros can be defined. + Version 4.89 ------------ diff --git a/src/src/exim.c b/src/src/exim.c index 7dd084534..d71af0ac4 100644 --- a/src/src/exim.c +++ b/src/src/exim.c @@ -1457,6 +1457,39 @@ return TRUE; } +/************************************************* +* Expansion testing * +*************************************************/ + +/* Expand and print one item, doing macro-processing. + +Arguments: + item line for expansion +*/ + +static void +expansion_test_line(uschar * line) +{ +int len; +BOOL dummy_macexp; + +Ustrncpy(big_buffer, line, big_buffer_size); +big_buffer[big_buffer_size-1] = '\0'; +len = Ustrlen(big_buffer); + +(void) macros_expand(0, &len, &dummy_macexp); + +if (isupper(big_buffer[0])) + { + if (macro_read_assignment(big_buffer)) + printf("Defined macro '%s'\n", mlast->name); + } +else + if ((line = expand_string(big_buffer))) printf("%s\n", CS line); + else printf("Failed: %s\n", expand_string_message); +} + + /************************************************* * Entry point and high-level code * *************************************************/ @@ -4988,7 +5021,7 @@ if (expansion_test) /* Read a test message from a file. We fudge it up to be on stdin, saving stdin itself for later reading of expansion strings. */ - else if (expansion_test_message != NULL) + else if (expansion_test_message) { int save_stdin = dup(0); int fd = Uopen(expansion_test_message, O_RDONLY, 0); @@ -5008,6 +5041,10 @@ if (expansion_test) clearerr(stdin); /* Required by Darwin */ } + /* Only admin users may see config-file macros this way */ + + if (!admin_user) macros = mlast = NULL; + /* Allow $recipients for this testing */ enable_dollar_recipients = TRUE; @@ -5015,15 +5052,8 @@ if (expansion_test) /* Expand command line items */ if (recipients_arg < argc) - { while (recipients_arg < argc) - { - uschar *s = argv[recipients_arg++]; - uschar *ss = expand_string(s); - if (ss == NULL) printf ("Failed: %s\n", expand_string_message); - else printf("%s\n", CS ss); - } - } + expansion_test_line(argv[recipients_arg++]); /* Read stdin */ @@ -5031,25 +5061,18 @@ if (expansion_test) { char *(*fn_readline)(const char *) = NULL; void (*fn_addhist)(const char *) = NULL; + uschar * s; - #ifdef USE_READLINE +#ifdef USE_READLINE void *dlhandle = set_readline(&fn_readline, &fn_addhist); - #endif +#endif - for (;;) - { - uschar *ss; - uschar *source = get_stdinput(fn_readline, fn_addhist); - if (source == NULL) break; - ss = expand_string(source); - if (ss == NULL) - printf ("Failed: %s\n", expand_string_message); - else printf("%s\n", CS ss); - } + while (s = get_stdinput(fn_readline, fn_addhist)) + expansion_test_line(s); - #ifdef USE_READLINE - if (dlhandle != NULL) dlclose(dlhandle); - #endif +#ifdef USE_READLINE + if (dlhandle) dlclose(dlhandle); +#endif } /* The data file will be open after -Mset */ @@ -5060,7 +5083,7 @@ if (expansion_test) deliver_datafile = -1; } - exim_exit(EXIT_SUCCESS, US"main"); + exim_exit(EXIT_SUCCESS, US"main: expansion test"); } diff --git a/src/src/functions.h b/src/src/functions.h index 1fe561f56..e50fa6f92 100644 --- a/src/src/functions.h +++ b/src/src/functions.h @@ -257,6 +257,8 @@ extern int log_create_as_exim(uschar *); extern void log_close_all(void); extern macro_item * macro_create(const uschar *, const uschar *, BOOL); +extern BOOL macro_read_assignment(uschar *); +extern uschar *macros_expand(int, int *, BOOL *); extern void mainlog_close(void); #ifdef WITH_CONTENT_SCAN extern int malware(const uschar *, int); diff --git a/src/src/readconf.c b/src/src/readconf.c index b34372c44..8d5f38c58 100644 --- a/src/src/readconf.c +++ b/src/src/readconf.c @@ -615,7 +615,10 @@ m->namelen = Ustrlen(name); m->replen = Ustrlen(val); m->name = name; m->replacement = val; -mlast->next = m; +if (mlast) + mlast->next = m; +else + macros = m; mlast = m; return m; } @@ -630,11 +633,11 @@ non-command line, macros is permitted using '==' instead of '='. Arguments: s points to the start of the logical line -Returns: nothing +Returns: FALSE iff fatal error */ -static void -read_macro_assignment(uschar *s) +BOOL +macro_read_assignment(uschar *s) { uschar name[64]; int namelen = 0; @@ -644,15 +647,21 @@ macro_item *m; while (isalnum(*s) || *s == '_') { if (namelen >= sizeof(name) - 1) - log_write(0, LOG_PANIC_DIE|LOG_CONFIG_IN, + { + log_write(0, LOG_PANIC|LOG_CONFIG_IN, "macro name too long (maximum is " SIZE_T_FMT " characters)", sizeof(name) - 1); + return FALSE; + } name[namelen++] = *s++; } name[namelen] = 0; while (isspace(*s)) s++; if (*s++ != '=') - log_write(0, LOG_PANIC_DIE|LOG_CONFIG_IN, "malformed macro definition"); + { + log_write(0, LOG_PANIC|LOG_CONFIG_IN, "malformed macro definition"); + return FALSE; + } if (*s == '=') { @@ -675,15 +684,21 @@ for (m = macros; m; m = m->next) if (Ustrcmp(m->name, name) == 0) { if (!m->command_line && !redef) - log_write(0, LOG_CONFIG|LOG_PANIC_DIE, "macro \"%s\" is already " + { + log_write(0, LOG_CONFIG|LOG_PANIC, "macro \"%s\" is already " "defined (use \"==\" if you want to redefine it", name); + return FALSE; + } break; } if (m->namelen < namelen && Ustrstr(name, m->name) != NULL) - log_write(0, LOG_CONFIG|LOG_PANIC_DIE, "\"%s\" cannot be defined as " + { + log_write(0, LOG_CONFIG|LOG_PANIC, "\"%s\" cannot be defined as " "a macro because previously defined macro \"%s\" is a substring", name, m->name); + return FALSE; + } /* We cannot have this test, because it is documented that a substring macro is permitted (there is even an example). @@ -697,7 +712,7 @@ for (m = macros; m; m = m->next) /* Check for an overriding command-line definition. */ -if (m && m->command_line) return; +if (m && m->command_line) return TRUE; /* Redefinition must refer to an existing macro. */ @@ -708,18 +723,119 @@ if (redef) m->replacement = string_copy(s); } else - log_write(0, LOG_CONFIG|LOG_PANIC_DIE, "can't redefine an undefined macro " + { + log_write(0, LOG_CONFIG|LOG_PANIC, "can't redefine an undefined macro " "\"%s\"", name); + return FALSE; + } /* We have a new definition. */ else (void) macro_create(string_copy(name), string_copy(s), FALSE); +return TRUE; } +/* Process line for macros. The line is in big_buffer starting at offset len. +Expand big_buffer if needed. Handle definitions of new macros, and +imacro expansions, rewriting the line in thw buffer. + +Arguments: + len Offset in buffer of start of line + newlen Pointer to offset of end of line, updated on return + macro_found Pointer to return that a macro was expanded + +Return: pointer to first nonblank char in line +*/ + +uschar * +macros_expand(int len, int * newlen, BOOL * macro_found) +{ +uschar * ss = big_buffer + len; +uschar * s; +macro_item * m; + +/* Find the true start of the physical line - leading spaces are always +ignored. */ + +while (isspace(*ss)) ss++; + +/* Process the physical line for macros. If this is the start of the logical +line, skip over initial text at the start of the line if it starts with an +upper case character followed by a sequence of name characters and an equals +sign, because that is the definition of a new macro, and we don't do +replacement therein. */ + +s = ss; +if (len == 0 && isupper(*s)) + { + while (isalnum(*s) || *s == '_') s++; + while (isspace(*s)) s++; + if (*s != '=') s = ss; /* Not a macro definition */ + } + +/* Skip leading chars which cannot start a macro name, to avoid multiple +pointless rescans in Ustrstr calls. */ + +while (*s && !isupper(*s) && *s != '_') s++; + +/* For each defined macro, scan the line (from after XXX= if present), +replacing all occurrences of the macro. */ + +*macro_found = FALSE; +for (m = macros; m; m = m->next) + { + uschar * p, *pp; + uschar * t = s; + + while ((p = Ustrstr(t, m->name)) != NULL) + { + int moveby; + +/* fprintf(stderr, "%s: matched '%s' in '%s'\n", __FUNCTION__, m->name, ss); */ + /* Expand the buffer if necessary */ + + while (*newlen - m->namelen + m->replen + 1 > big_buffer_size) + { + int newsize = big_buffer_size + BIG_BUFFER_SIZE; + uschar *newbuffer = store_malloc(newsize); + memcpy(newbuffer, big_buffer, *newlen + 1); + p = newbuffer + (p - big_buffer); + s = newbuffer + (s - big_buffer); + ss = newbuffer + (ss - big_buffer); + t = newbuffer + (t - big_buffer); + big_buffer_size = newsize; + store_free(big_buffer); + big_buffer = newbuffer; + } + + /* Shuffle the remaining characters up or down in the buffer before + copying in the replacement text. Don't rescan the replacement for this + same macro. */ + + pp = p + m->namelen; + if ((moveby = m->replen - m->namelen) != 0) + { + memmove(p + m->replen, pp, (big_buffer + *newlen) - pp + 1); + *newlen += moveby; + } + Ustrncpy(p, m->replacement, m->replen); + t = p + m->replen; + while (*t && !isupper(*t) && *t != '_') t++; + *macro_found = TRUE; + } + } + +/* An empty macro replacement at the start of a line could mean that ss no +longer points to the first non-blank character. */ + +while (isspace(*ss)) ss++; +return ss; +} + /************************************************* * Read configuration line * *************************************************/ @@ -749,7 +865,6 @@ int startoffset = 0; /* To first non-blank char in logical line */ int len = 0; /* Of logical line so far */ int newlen; uschar *s, *ss; -macro_item *m; BOOL macro_found; /* Loop for handling continuation lines, skipping comments, and dealing with @@ -810,82 +925,7 @@ for (;;) newlen += Ustrlen(big_buffer + newlen); } - /* Find the true start of the physical line - leading spaces are always - ignored. */ - - ss = big_buffer + len; - while (isspace(*ss)) ss++; - - /* Process the physical line for macros. If this is the start of the logical - line, skip over initial text at the start of the line if it starts with an - upper case character followed by a sequence of name characters and an equals - sign, because that is the definition of a new macro, and we don't do - replacement therein. */ - - s = ss; - if (len == 0 && isupper(*s)) - { - while (isalnum(*s) || *s == '_') s++; - while (isspace(*s)) s++; - if (*s != '=') s = ss; /* Not a macro definition */ - } - - /* Skip leading chars which cannot start a macro name, to avoid multiple - pointless rescans in Ustrstr calls. */ - - while (*s && !isupper(*s) && *s != '_') s++; - - /* For each defined macro, scan the line (from after XXX= if present), - replacing all occurrences of the macro. */ - - macro_found = FALSE; - for (m = macros; m; m = m->next) - { - uschar * p, *pp; - uschar * t = s; - - while ((p = Ustrstr(t, m->name)) != NULL) - { - int moveby; - -/* fprintf(stderr, "%s: matched '%s' in '%s'\n", __FUNCTION__, m->name, ss); */ - /* Expand the buffer if necessary */ - - while (newlen - m->namelen + m->replen + 1 > big_buffer_size) - { - int newsize = big_buffer_size + BIG_BUFFER_SIZE; - uschar *newbuffer = store_malloc(newsize); - memcpy(newbuffer, big_buffer, newlen + 1); - p = newbuffer + (p - big_buffer); - s = newbuffer + (s - big_buffer); - ss = newbuffer + (ss - big_buffer); - t = newbuffer + (t - big_buffer); - big_buffer_size = newsize; - store_free(big_buffer); - big_buffer = newbuffer; - } - - /* Shuffle the remaining characters up or down in the buffer before - copying in the replacement text. Don't rescan the replacement for this - same macro. */ - - pp = p + m->namelen; - if ((moveby = m->replen - m->namelen) != 0) - { - memmove(p + m->replen, pp, (big_buffer + newlen) - pp + 1); - newlen += moveby; - } - Ustrncpy(p, m->replacement, m->replen); - t = p + m->replen; - while (*t && !isupper(*t) && *t != '_') t++; - macro_found = TRUE; - } - } - - /* An empty macro replacement at the start of a line could mean that ss no - longer points to the first non-blank character. */ - - while (isspace(*ss)) ss++; + ss = macros_expand(len, &newlen, ¯o_found); /* Check for comment lines - these are physical lines. */ @@ -3277,7 +3317,8 @@ while ((s = get_config_line())) log_write(0, LOG_PANIC_DIE|LOG_CONFIG_IN, "found unexpected BOM (Byte Order Mark)"); - if (isupper(s[0])) read_macro_assignment(s); + if (isupper(s[0])) + { if (!macro_read_assignment(s)) exim_exit(EXIT_FAILURE, US""); } else if (Ustrncmp(s, "domainlist", 10) == 0) read_named_list(&domainlist_anchor, &domainlist_count, @@ -3724,7 +3765,7 @@ while ((buffer = get_config_line()) != NULL) (d->info->init)(d); d = NULL; } - read_macro_assignment(buffer); + if (!macro_read_assignment(buffer)) exim_exit(EXIT_FAILURE, US""); continue; } @@ -4225,7 +4266,7 @@ between ACLs. */ acl_line = get_config_line(); -while(acl_line != NULL) +while(acl_line) { uschar name[64]; tree_node *node; @@ -4234,7 +4275,7 @@ while(acl_line != NULL) p = readconf_readname(name, sizeof(name), acl_line); if (isupper(*name) && *p == '=') { - read_macro_assignment(acl_line); + if (!macro_read_assignment(acl_line)) exim_exit(EXIT_FAILURE, US""); acl_line = get_config_line(); continue; } @@ -4278,7 +4319,7 @@ log_write(0, LOG_PANIC_DIE|LOG_CONFIG_IN, "local_scan() options not supported: " #else uschar *p; -while ((p = get_config_line()) != NULL) +while ((p = get_config_line())) { (void) readconf_handle_option(p, local_scan_options, local_scan_options_count, NULL, US"local_scan option \"%s\" unknown"); @@ -4368,14 +4409,14 @@ while(next_section[0] != 0) void readconf_save_config(const uschar *s) { - save_config_line(string_sprintf("# Exim Configuration (%s)", - running_in_test_harness ? US"X" : s)); +save_config_line(string_sprintf("# Exim Configuration (%s)", + running_in_test_harness ? US"X" : s)); } static void save_config_position(const uschar *file, int line) { - save_config_line(string_sprintf("# %d \"%s\"", line, file)); +save_config_line(string_sprintf("# %d \"%s\"", line, file)); } /* Append a pre-parsed logical line to the config lines store, diff --git a/test/scripts/0000-Basic/0542 b/test/scripts/0000-Basic/0542 index 0c6362bde..1c8c03b5f 100644 --- a/test/scripts/0000-Basic/0542 +++ b/test/scripts/0000-Basic/0542 @@ -7,13 +7,15 @@ Subject: The subject is not the object This is the body of the message. Make the line longer than any header. **** sudo exim -be -Mset $msg1 -From: $h_from: -Subject: $h_subject: +from: $h_from: +subject: $h_subject: message_body_size=$message_body_size message_id=$message_id message_exim_id=$message_exim_id max_received_linelength=$max_received_linelength recipients=$recipients +TESTING_MACROS=$recipients +(TESTING_MACROS) **** exim -bs mail from: @@ -29,8 +31,8 @@ This is the body of the message. Make the line much longer than any header. quit **** sudo exim -be -Mset $msg2 -From: $h_from: -Subject: $h_subject: +from: $h_from: +subject: $h_subject: message_body_size=$message_body_size message_id=$message_id message_exim_id=$message_exim_id diff --git a/test/stdout/0542 b/test/stdout/0542 index ab29dc9c0..ce255a744 100644 --- a/test/stdout/0542 +++ b/test/stdout/0542 @@ -1,10 +1,12 @@ -> From: Himself -> Subject: The subject is not the object +> from: Himself +> subject: The subject is not the object > message_body_size=71 > message_id=10HmaX-0005vi-00 > message_exim_id=10HmaX-0005vi-00 > max_received_linelength=70 > recipients=userx@test.x, usery@test.ex +> Defined macro 'TESTING_MACROS' +> (userx@test.x, usery@test.ex) > 220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 250 OK @@ -13,8 +15,8 @@ 354 Enter message, ending with "." on a line by itself 250 OK id=10HmaY-0005vi-00 221 myhost.test.ex closing connection -> From: Himself -> Subject: The subject is not the object +> from: Himself +> subject: The subject is not the object > message_body_size=76 > message_id=10HmaY-0005vi-00 > message_exim_id=10HmaY-0005vi-00 -- 2.30.2