From 5f17d2f09733cc77355841fbf095a95186cc9fe5 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sun, 5 Jul 2020 16:32:27 +0100 Subject: [PATCH] Support ret-full on lsearch. Bug 2611 --- doc/doc-docbook/spec.xfpt | 7 +++ doc/doc-txt/NewStuff | 3 + src/src/lookups/lsearch.c | 110 +++++++++++++++++++---------------- test/scripts/0000-Basic/0002 | 54 ++++++++++++----- test/stdout/0002 | 22 +++++++ 5 files changed, 131 insertions(+), 65 deletions(-) diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt index 9a49373f3..6d440e938 100644 --- a/doc/doc-docbook/spec.xfpt +++ b/doc/doc-docbook/spec.xfpt @@ -6859,6 +6859,13 @@ the implicit key is the host's IP address rather than its name (see section &*Warning 3*&: Do not use an IPv4-mapped IPv6 address for a key; use the IPv4, in dotted-quad form. (Exim converts IPv4-mapped IPv6 addresses to this notation before executing the lookup.) + +.new +One option is supported, "ret=full", to request the return of the entire line +rather than omitting the key porttion. +Note however that the key portion will have been de-quoted. +.wen + .next .cindex lookup json .cindex json "lookup type" diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff index ecb9814f4..ac6383ca4 100644 --- a/doc/doc-txt/NewStuff +++ b/doc/doc-txt/NewStuff @@ -33,6 +33,9 @@ Version 4.95 db file, replacing the previous prefix to the SQL string (which had issues when the SQL used tainted values). + 9. Lsearch lookups accept a "ret=full" option, to return both the portion + of the line matching the key, and the remainder. + Version 4.94 ------------ diff --git a/src/src/lookups/lsearch.c b/src/src/lookups/lsearch.c index 9df7a167f..4da422e9e 100644 --- a/src/src/lookups/lsearch.c +++ b/src/src/lookups/lsearch.c @@ -69,29 +69,38 @@ but people do occasionally do weird things. */ static int internal_lsearch_find(void * handle, const uschar * filename, const uschar * keystring, int length, uschar ** result, uschar ** errmsg, - int type) + int type, const uschar * opts) { -FILE *f = (FILE *)handle; -BOOL last_was_eol = TRUE; -BOOL this_is_eol = TRUE; +FILE *f = handle; +BOOL ret_full = FALSE; int old_pool = store_pool; rmark reset_point = NULL; uschar buffer[4096]; +if (opts) + { + int sep = ','; + uschar * ele; + + while ((ele = string_nextinlist(&opts, &sep, NULL, 0))) + if (Ustrcmp(ele, "ret=full") == 0) + { ret_full = TRUE; break; } + } + /* Wildcard searches may use up some store, because of expansions. We don't want them to fill up our search store. What we do is set the pool to the main pool and get a point to reset to later. Wildcard searches could also issue lookups, but internal_search_find will take care of that, and the cache will be safely stored in the search pool again. */ -if(type == LSEARCH_WILD || type == LSEARCH_NWILD) +if (type == LSEARCH_WILD || type == LSEARCH_NWILD) { store_pool = POOL_MAIN; reset_point = store_mark(); } rewind(f); -for (last_was_eol = TRUE; +for (BOOL this_is_eol, last_was_eol = TRUE; Ufgets(buffer, sizeof(buffer), f) != NULL; last_was_eol = this_is_eol) { @@ -137,21 +146,22 @@ for (last_was_eol = TRUE; if (*s == '\"') { uschar *t = s++; - while (*s != 0 && *s != '\"') + while (*s && *s != '\"') { - if (*s == '\\') *t++ = string_interpret_escape(CUSS &s); - else *t++ = *s; + *t++ = *s == '\\' ? string_interpret_escape(CUSS &s) : *s; s++; } - if (*s != 0) s++; /* Past terminating " */ linekeylength = t - buffer; + if (*s) s++; /* Past terminating " */ + if (ret_full) + Ustrcpy(t, s); /* copy the rest of line does also */ } /* Otherwise it is terminated by a colon or white space */ else { - while (*s != 0 && *s != ':' && !isspace(*s)) s++; + while (*s && *s != ':' && !isspace(*s)) s++; linekeylength = s - buffer; } @@ -162,9 +172,9 @@ for (last_was_eol = TRUE; /* A plain lsearch treats each key as a literal */ case LSEARCH_PLAIN: - if (linekeylength != length || strncmpic(buffer, keystring, length) != 0) - continue; - break; /* Key matched */ + if (linekeylength != length || strncmpic(buffer, keystring, length) != 0) + continue; + break; /* Key matched */ /* A wild lsearch treats each key as a possible wildcarded string; no expansion is done for nwildlsearch. */ @@ -181,7 +191,7 @@ for (last_was_eol = TRUE; UCHAR_MAX+1, /* Single-item list */ NULL, /* No anchor */ NULL, /* No caching */ - MCL_STRING + ((type == LSEARCH_WILD)? 0:MCL_NOEXPAND), + MCL_STRING + (type == LSEARCH_WILD ? 0 : MCL_NOEXPAND), TRUE, /* Caseless */ NULL); buffer[linekeylength] = save; @@ -189,47 +199,47 @@ for (last_was_eol = TRUE; if (rc == DEFER) return DEFER; } - /* The key has matched. If the search involved a regular expression, it - might have caused numerical variables to be set. However, their values will - be in the wrong storage pool for external use. Copying them to the standard - pool is not feasible because of the caching of lookup results - a repeated - lookup will not match the regular expression again. Therefore, we flatten - all numeric variables at this point. */ + /* The key has matched. If the search involved a regular expression, it + might have caused numerical variables to be set. However, their values will + be in the wrong storage pool for external use. Copying them to the standard + pool is not feasible because of the caching of lookup results - a repeated + lookup will not match the regular expression again. Therefore, we drop + all numeric variables at this point. */ - expand_nmax = -1; - break; + expand_nmax = -1; + break; /* Compare an ip address against a list of network/ip addresses. We have to allow for the "*" case specially. */ case LSEARCH_IP: - if (linekeylength == 1 && buffer[0] == '*') - { - if (length != 1 || keystring[0] != '*') continue; - } - else if (length == 1 && keystring[0] == '*') continue; - else - { - int maskoffset; - int save = buffer[linekeylength]; - buffer[linekeylength] = 0; - if (string_is_ip_address(buffer, &maskoffset) == 0 || - !host_is_in_net(keystring, buffer, maskoffset)) continue; - buffer[linekeylength] = save; - } - break; /* Key matched */ + if (linekeylength == 1 && buffer[0] == '*') + { + if (length != 1 || keystring[0] != '*') continue; + } + else if (length == 1 && keystring[0] == '*') continue; + else + { + int maskoffset; + int save = buffer[linekeylength]; + buffer[linekeylength] = 0; + if (string_is_ip_address(buffer, &maskoffset) == 0 || + !host_is_in_net(keystring, buffer, maskoffset)) continue; + buffer[linekeylength] = save; + } + break; /* Key matched */ } /* The key has matched. Skip spaces after the key, and allow an optional colon after the spaces. This is an odd specification, but it's for compatibility. */ - while (isspace((uschar)*s)) s++; - if (*s == ':') - { - s++; - while (isspace((uschar)*s)) s++; - } + if (!ret_full) + if (Uskip_whitespace(&s) == ':') + { + s++; + Uskip_whitespace(&s); + } /* Reset dynamic store, if we need to, and revert to the search pool */ @@ -248,7 +258,9 @@ for (last_was_eol = TRUE; this_is_comment = FALSE; yield = string_get(100); - if (*s != 0) + if (ret_full) + yield = string_cat(yield, buffer); + else if (*s) yield = string_cat(yield, s); /* Now handle continuations */ @@ -317,7 +329,7 @@ lsearch_find(void * handle, const uschar * filename, const uschar * keystring, const uschar * opts) { return internal_lsearch_find(handle, filename, keystring, length, result, - errmsg, LSEARCH_PLAIN); + errmsg, LSEARCH_PLAIN, opts); } @@ -334,7 +346,7 @@ wildlsearch_find(void * handle, const uschar * filename, const uschar * keystrin const uschar * opts) { return internal_lsearch_find(handle, filename, keystring, length, result, - errmsg, LSEARCH_WILD); + errmsg, LSEARCH_WILD, opts); } @@ -351,7 +363,7 @@ nwildlsearch_find(void * handle, const uschar * filename, const uschar * keystri const uschar * opts) { return internal_lsearch_find(handle, filename, keystring, length, result, - errmsg, LSEARCH_NWILD); + errmsg, LSEARCH_NWILD, opts); } @@ -371,7 +383,7 @@ iplsearch_find(void * handle, uschar const * filename, const uschar * keystring, if ((length == 1 && keystring[0] == '*') || string_is_ip_address(keystring, NULL) != 0) return internal_lsearch_find(handle, filename, keystring, length, result, - errmsg, LSEARCH_IP); + errmsg, LSEARCH_IP, opts); *errmsg = string_sprintf("\"%s\" is not a valid iplsearch key (an IP " "address, with optional CIDR mask, is wanted): " diff --git a/test/scripts/0000-Basic/0002 b/test/scripts/0000-Basic/0002 index a8fc0bcb2..4d170ec68 100644 --- a/test/scripts/0000-Basic/0002 +++ b/test/scripts/0000-Basic/0002 @@ -526,28 +526,50 @@ acl if: ${if acl {{a_defer}{argN}{arg2}} {Y:$value}{N:$value}} # Lookups: DIR is the testing directory. In this test we can only use the # lookups that are required in all cases. -${lookup{postmaster}lsearch{DIR/aux-fixed/0002.aliases}{$value}fail} +${lookup{postmaster}lsearch {DIR/aux-fixed/0002.aliases}{$value}fail} +${lookup{postmaster}lsearch,ret=full{DIR/aux-fixed/0002.aliases}{$value}fail} ${lookup{x@y}lsearch*@{DIR/aux-fixed/0002.starat}{$value}fail} -${lookup{x@z}lsearch*{DIR/aux-fixed/0002.starat}{$value}fail} +${lookup{x@z}lsearch* {DIR/aux-fixed/0002.starat}{$value}fail} ${lookup{x@z}lsearch*@{DIR/aux-fixed/0002.starat}{$value}fail} ${lookup{x@w}lsearch*@{DIR/aux-fixed/0002.starat}{$value}fail} -${lookup{a.b.c.d}partial-lsearch{DIR/aux-fixed/0002.domains}{$value}fail} -${lookup{x.y.z}partial-lsearch{DIR/aux-fixed/0002.domains}{$value}{failed x.y.z}} -${lookup{p.q}partial-lsearch{DIR/aux-fixed/0002.domains}{$value}fail} -${lookup{o.p.q}partial-lsearch{DIR/aux-fixed/0002.domains}{$value}fail} -${lookup{m.n.o.p.q}partial-lsearch{DIR/aux-fixed/0002.domains}{$value}fail} -${lookup{x.y.z}partial1-lsearch{DIR/aux-fixed/0002.domains}{$value}fail} -${lookup{x.y.z}partial0-lsearch{DIR/aux-fixed/0002.domains}{$value}fail} - -q1: ${lookup{abc}lsearch{DIR/aux-fixed/0002.quoted}} -q2: ${lookup{xyz}lsearch{DIR/aux-fixed/0002.quoted}} -q3: ${lookup{pqr}lsearch{DIR/aux-fixed/0002.quoted}} -q4: ${lookup{a:b}lsearch{DIR/aux-fixed/0002.quoted}} -q5: ${lookup{"quoted"}lsearch{DIR/aux-fixed/0002.quoted}} +${lookup{x@y}lsearch*@,ret=full {DIR/aux-fixed/0002.starat}{$value}fail} +${lookup{x@z}lsearch*,ret=full {DIR/aux-fixed/0002.starat}{$value}fail} +${lookup{x@z}lsearch*@,ret=full {DIR/aux-fixed/0002.starat}{$value}fail} +${lookup{x@w}lsearch*@,ret=full {DIR/aux-fixed/0002.starat}{$value}fail} + +${lookup{a.b.c.d} partial-lsearch {DIR/aux-fixed/0002.domains}{$value}fail} +${lookup{x.y.z} partial-lsearch {DIR/aux-fixed/0002.domains}{$value}{failed x.y.z}} +${lookup{p.q} partial-lsearch {DIR/aux-fixed/0002.domains}{$value}fail} +${lookup{o.p.q} partial-lsearch {DIR/aux-fixed/0002.domains}{$value}fail} +${lookup{m.n.o.p.q}partial-lsearch {DIR/aux-fixed/0002.domains}{$value}fail} +${lookup{x.y.z} partial1-lsearch{DIR/aux-fixed/0002.domains}{$value}fail} +${lookup{x.y.z} partial0-lsearch{DIR/aux-fixed/0002.domains}{$value}fail} + +${lookup{a.b.c.d} partial-lsearch,ret=full {DIR/aux-fixed/0002.domains}{$value}fail} +${lookup{x.y.z} partial-lsearch,ret=full {DIR/aux-fixed/0002.domains}{$value}{failed x.y.z}} +${lookup{p.q} partial-lsearch,ret=full {DIR/aux-fixed/0002.domains}{$value}fail} +${lookup{o.p.q} partial-lsearch,ret=full {DIR/aux-fixed/0002.domains}{$value}fail} +${lookup{m.n.o.p.q}partial-lsearch,ret=full {DIR/aux-fixed/0002.domains}{$value}fail} +${lookup{x.y.z} partial1-lsearch,ret=full{DIR/aux-fixed/0002.domains}{$value}fail} +${lookup{x.y.z} partial0-lsearch,ret=full{DIR/aux-fixed/0002.domains}{$value}fail} + +q1: ${lookup{abc} lsearch{DIR/aux-fixed/0002.quoted}} +q2: ${lookup{xyz} lsearch{DIR/aux-fixed/0002.quoted}} +q3: ${lookup{pqr} lsearch{DIR/aux-fixed/0002.quoted}} +q4: ${lookup{a:b} lsearch{DIR/aux-fixed/0002.quoted}} +q5: ${lookup{"quoted"} lsearch{DIR/aux-fixed/0002.quoted}} q6: ${lookup{white space}lsearch{DIR/aux-fixed/0002.quoted}} -q7: ${lookup{b\\s}lsearch{DIR/aux-fixed/0002.quoted}} +q7: ${lookup{b\\s} lsearch{DIR/aux-fixed/0002.quoted}} + +q1: ${lookup{abc} lsearch,ret=full{DIR/aux-fixed/0002.quoted}} +q2: ${lookup{xyz} lsearch,ret=full{DIR/aux-fixed/0002.quoted}} +q3: ${lookup{pqr} lsearch,ret=full{DIR/aux-fixed/0002.quoted}} +q4: ${lookup{a:b} lsearch,ret=full{DIR/aux-fixed/0002.quoted}} +q5: ${lookup{"quoted"} lsearch,ret=full{DIR/aux-fixed/0002.quoted}} +q6: ${lookup{white space}lsearch,ret=full{DIR/aux-fixed/0002.quoted}} +q7: ${lookup{b\\s} lsearch,ret=full{DIR/aux-fixed/0002.quoted}} abc: ${lookup{abc}wildlsearch{DIR/aux-var/0002.wild}} a.b.c: ${lookup{a.b.c}wildlsearch{DIR/aux-var/0002.wild}} diff --git a/test/stdout/0002 b/test/stdout/0002 index b8ff36122..d0e8b5d7b 100644 --- a/test/stdout/0002 +++ b/test/stdout/0002 @@ -491,12 +491,18 @@ newline tab\134backslash ~tilde\177DEL\200\201. > # lookups that are required in all cases. > > CALLER +> postmaster: CALLER > > ==X@Y > ==* > ==*@Z > ==* > +> x@y ==X@Y +> * ==* +> *@z ==*@Z +> * ==* +> > data for a.b.c.d > failed x.y.z > data for *.p.q @@ -505,6 +511,14 @@ newline tab\134backslash ~tilde\177DEL\200\201. > Failed: "lookup" failed and "fail" requested > data for * > +> a.b.c.d: data for a.b.c.d +> failed x.y.z +> *.p.q: data for *.p.q +> *.p.q: data for *.p.q +> *.p.q: data for *.p.q +> Failed: "lookup" failed and "fail" requested +> *: data for * +> > q1: "abc" > q2: "xyz": > q3: "pqr" : @@ -513,6 +527,14 @@ newline tab\134backslash ~tilde\177DEL\200\201. > q6: "white space" > q7: "b\\s" > +> q1: abc "abc" +> q2: xyz: "xyz": +> q3: pqr : "pqr" : +> q4: a:b "a:b" abc continued data (make sure not key) +> q5: "quoted" "\"quoted\"" +> q6: white space "white space" +> q7: b\s "b\\s" +> > abc: abc > a.b.c: *.b.c > ab.c: *b.c -- 2.30.2