Support ret-full on lsearch. Bug 2611

diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt
index 9a49373f35616b6ccf089999ae174ee0575945c7..6d440e9382b15fa034f3e787d35cc89077f9e998 100644 (file)
--- a/doc/doc-docbook/spec.xfpt
+++ b/doc/doc-docbook/spec.xfpt
@@ -6859,6 +6859,13 @@ the implicit key is the host's IP address rather than its name (see section
 &*Warning 3*&: Do not use an IPv4-mapped IPv6 address for a key; use the
 IPv4, in dotted-quad form. (Exim converts IPv4-mapped IPv6 addresses to this
 notation before executing the lookup.)
 &*Warning 3*&: Do not use an IPv4-mapped IPv6 address for a key; use the
 IPv4, in dotted-quad form. (Exim converts IPv4-mapped IPv6 addresses to this
 notation before executing the lookup.)

+
+.new
+One option is supported, "ret=full", to request the return of the entire line
+rather than omitting the key porttion.
+Note however that the key portion will have been de-quoted.
+.wen
+

 .next
 .cindex lookup json
 .cindex json "lookup type"
 .next
 .cindex lookup json
 .cindex json "lookup type"

diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff
index ecb9814f4e6ae91d6608d81a1169d92985452104..ac6383ca4f8f37b2a6a8011ae3a482d9e642e545 100644 (file)
--- a/doc/doc-txt/NewStuff
+++ b/doc/doc-txt/NewStuff
@@ -33,6 +33,9 @@ Version 4.95
     db file, replacing the previous prefix to the SQL string (which had
     issues when the SQL used tainted values).
 
     db file, replacing the previous prefix to the SQL string (which had
     issues when the SQL used tainted values).
 

+ 9. Lsearch lookups accept a "ret=full" option, to return both the portion
+    of the line matching the key, and the remainder.
+

 
 Version 4.94
 ------------
 
 Version 4.94
 ------------

diff --git a/src/src/lookups/lsearch.c b/src/src/lookups/lsearch.c
index 9df7a167f11bb9c130231caa1ae249259f80888c..4da422e9eb51e1e17f945a45d2638a4108029855 100644 (file)
--- a/src/src/lookups/lsearch.c
+++ b/src/src/lookups/lsearch.c
@@ -69,29 +69,38 @@ but people do occasionally do weird things. */
 static int
 internal_lsearch_find(void * handle, const uschar * filename,
   const uschar * keystring, int length, uschar ** result, uschar ** errmsg,
 static int
 internal_lsearch_find(void * handle, const uschar * filename,
   const uschar * keystring, int length, uschar ** result, uschar ** errmsg,

- int type)
+  int type, const uschar * opts)

 {
 {

-FILE *f = (FILE *)handle;
-BOOL last_was_eol = TRUE;
-BOOL this_is_eol = TRUE;
+FILE *f = handle;
+BOOL ret_full = FALSE;

 int old_pool = store_pool;
 rmark reset_point = NULL;
 uschar buffer[4096];
 
 int old_pool = store_pool;
 rmark reset_point = NULL;
 uschar buffer[4096];
 

+if (opts)
+  {
+  int sep = ',';
+  uschar * ele;
+
+  while ((ele = string_nextinlist(&opts, &sep, NULL, 0)))
+    if (Ustrcmp(ele, "ret=full") == 0)
+      { ret_full = TRUE; break; }
+  }
+

 /* Wildcard searches may use up some store, because of expansions. We don't
 want them to fill up our search store. What we do is set the pool to the main
 pool and get a point to reset to later. Wildcard searches could also issue
 lookups, but internal_search_find will take care of that, and the cache will be
 safely stored in the search pool again. */
 
 /* Wildcard searches may use up some store, because of expansions. We don't
 want them to fill up our search store. What we do is set the pool to the main
 pool and get a point to reset to later. Wildcard searches could also issue
 lookups, but internal_search_find will take care of that, and the cache will be
 safely stored in the search pool again. */
 

-if(type == LSEARCH_WILD || type == LSEARCH_NWILD)
+if (type == LSEARCH_WILD || type == LSEARCH_NWILD)

   {
   store_pool = POOL_MAIN;
   reset_point = store_mark();
   }
 
 rewind(f);
   {
   store_pool = POOL_MAIN;
   reset_point = store_mark();
   }
 
 rewind(f);

-for (last_was_eol = TRUE;
+for (BOOL this_is_eol, last_was_eol = TRUE;

      Ufgets(buffer, sizeof(buffer), f) != NULL;
      last_was_eol = this_is_eol)
   {
      Ufgets(buffer, sizeof(buffer), f) != NULL;
      last_was_eol = this_is_eol)
   {


@@ -137,21 +146,22 @@ for (last_was_eol = TRUE;
   if (*s == '\"')
     {
     uschar *t = s++;
   if (*s == '\"')
     {
     uschar *t = s++;

-    while (*s != 0 && *s != '\"')
+    while (*s && *s != '\"')

       {
       {

-      if (*s == '\\') *t++ = string_interpret_escape(CUSS &s);
-        else *t++ = *s;
+      *t++ = *s == '\\' ? string_interpret_escape(CUSS &s) : *s;

       s++;
       }
       s++;
       }

-    if (*s != 0) s++;               /* Past terminating " */

     linekeylength = t - buffer;
     linekeylength = t - buffer;

+    if (*s) s++;                       /* Past terminating " */
+    if (ret_full)
+      Ustrcpy(t, s);                   /* copy the rest of line does also */

     }
 
   /* Otherwise it is terminated by a colon or white space */
 
   else
     {
     }
 
   /* Otherwise it is terminated by a colon or white space */
 
   else
     {

-    while (*s != 0 && *s != ':' && !isspace(*s)) s++;
+    while (*s && *s != ':' && !isspace(*s)) s++;

     linekeylength = s - buffer;
     }
 
     linekeylength = s - buffer;
     }
 


@@ -162,9 +172,9 @@ for (last_was_eol = TRUE;
     /* A plain lsearch treats each key as a literal */
 
     case LSEARCH_PLAIN:
     /* A plain lsearch treats each key as a literal */
 
     case LSEARCH_PLAIN:

-    if (linekeylength != length || strncmpic(buffer, keystring, length) != 0)
-      continue;
-    break;      /* Key matched */
+      if (linekeylength != length || strncmpic(buffer, keystring, length) != 0)
+       continue;
+      break;      /* Key matched */

 
     /* A wild lsearch treats each key as a possible wildcarded string; no
     expansion is done for nwildlsearch. */
 
     /* A wild lsearch treats each key as a possible wildcarded string; no
     expansion is done for nwildlsearch. */


@@ -181,7 +191,7 @@ for (last_was_eol = TRUE;
         UCHAR_MAX+1,              /* Single-item list */
         NULL,                     /* No anchor */
         NULL,                     /* No caching */
         UCHAR_MAX+1,              /* Single-item list */
         NULL,                     /* No anchor */
         NULL,                     /* No caching */

-        MCL_STRING + ((type == LSEARCH_WILD)? 0:MCL_NOEXPAND),
+        MCL_STRING + (type == LSEARCH_WILD ? 0 : MCL_NOEXPAND),

         TRUE,                     /* Caseless */
         NULL);
       buffer[linekeylength] = save;
         TRUE,                     /* Caseless */
         NULL);
       buffer[linekeylength] = save;


@@ -189,47 +199,47 @@ for (last_was_eol = TRUE;
       if (rc == DEFER) return DEFER;
       }
 
       if (rc == DEFER) return DEFER;
       }
 

-    /* The key has matched. If the search involved a regular expression, it
-    might have caused numerical variables to be set. However, their values will
-    be in the wrong storage pool for external use. Copying them to the standard
-    pool is not feasible because of the caching of lookup results - a repeated
-    lookup will not match the regular expression again. Therefore, we flatten
-    all numeric variables at this point. */
+      /* The key has matched. If the search involved a regular expression, it
+      might have caused numerical variables to be set. However, their values will
+      be in the wrong storage pool for external use. Copying them to the standard
+      pool is not feasible because of the caching of lookup results - a repeated
+      lookup will not match the regular expression again. Therefore, we drop
+      all numeric variables at this point. */

 
 

-    expand_nmax = -1;
-    break;
+      expand_nmax = -1;
+      break;

 
     /* Compare an ip address against a list of network/ip addresses. We have to
     allow for the "*" case specially. */
 
     case LSEARCH_IP:
 
     /* Compare an ip address against a list of network/ip addresses. We have to
     allow for the "*" case specially. */
 
     case LSEARCH_IP:

-    if (linekeylength == 1 && buffer[0] == '*')
-      {
-      if (length != 1 || keystring[0] != '*') continue;
-      }
-    else if (length == 1 && keystring[0] == '*') continue;
-    else
-      {
-      int maskoffset;
-      int save = buffer[linekeylength];
-      buffer[linekeylength] = 0;
-      if (string_is_ip_address(buffer, &maskoffset) == 0 ||
-          !host_is_in_net(keystring, buffer, maskoffset)) continue;
-      buffer[linekeylength] = save;
-      }
-    break;      /* Key matched */
+      if (linekeylength == 1 && buffer[0] == '*')
+       {
+       if (length != 1 || keystring[0] != '*') continue;
+       }
+      else if (length == 1 && keystring[0] == '*') continue;
+      else
+       {
+       int maskoffset;
+       int save = buffer[linekeylength];
+       buffer[linekeylength] = 0;
+       if (string_is_ip_address(buffer, &maskoffset) == 0 ||
+           !host_is_in_net(keystring, buffer, maskoffset)) continue;
+       buffer[linekeylength] = save;
+       }
+      break;      /* Key matched */

     }
 
   /* The key has matched. Skip spaces after the key, and allow an optional
   colon after the spaces. This is an odd specification, but it's for
   compatibility. */
 
     }
 
   /* The key has matched. Skip spaces after the key, and allow an optional
   colon after the spaces. This is an odd specification, but it's for
   compatibility. */
 

-  while (isspace((uschar)*s)) s++;
-  if (*s == ':')
-    {
-    s++;
-    while (isspace((uschar)*s)) s++;
-    }
+  if (!ret_full)
+    if (Uskip_whitespace(&s) == ':')
+      {
+      s++;
+      Uskip_whitespace(&s);
+      }

 
   /* Reset dynamic store, if we need to, and revert to the search pool */
 
 
   /* Reset dynamic store, if we need to, and revert to the search pool */
 


@@ -248,7 +258,9 @@ for (last_was_eol = TRUE;
 
   this_is_comment = FALSE;
   yield = string_get(100);
 
   this_is_comment = FALSE;
   yield = string_get(100);

-  if (*s != 0)
+  if (ret_full)
+    yield = string_cat(yield, buffer);
+  else if (*s)

     yield = string_cat(yield, s);
 
   /* Now handle continuations */
     yield = string_cat(yield, s);
 
   /* Now handle continuations */


@@ -317,7 +329,7 @@ lsearch_find(void * handle, const uschar * filename, const uschar * keystring,
   const uschar * opts)
 {
 return internal_lsearch_find(handle, filename, keystring, length, result,
   const uschar * opts)
 {
 return internal_lsearch_find(handle, filename, keystring, length, result,

-  errmsg, LSEARCH_PLAIN);
+  errmsg, LSEARCH_PLAIN, opts);

 }
 
 
 }
 
 


@@ -334,7 +346,7 @@ wildlsearch_find(void * handle, const uschar * filename, const uschar * keystrin
   const uschar * opts)
 {
 return internal_lsearch_find(handle, filename, keystring, length, result,
   const uschar * opts)
 {
 return internal_lsearch_find(handle, filename, keystring, length, result,

-  errmsg, LSEARCH_WILD);
+  errmsg, LSEARCH_WILD, opts);

 }
 
 
 }
 
 


@@ -351,7 +363,7 @@ nwildlsearch_find(void * handle, const uschar * filename, const uschar * keystri
   const uschar * opts)
 {
 return internal_lsearch_find(handle, filename, keystring, length, result,
   const uschar * opts)
 {
 return internal_lsearch_find(handle, filename, keystring, length, result,

-  errmsg, LSEARCH_NWILD);
+  errmsg, LSEARCH_NWILD, opts);

 }
 
 
 }
 
 


@@ -371,7 +383,7 @@ iplsearch_find(void * handle, uschar const * filename, const uschar * keystring,
 if ((length == 1 && keystring[0] == '*') ||
     string_is_ip_address(keystring, NULL) != 0)
   return internal_lsearch_find(handle, filename, keystring, length, result,
 if ((length == 1 && keystring[0] == '*') ||
     string_is_ip_address(keystring, NULL) != 0)
   return internal_lsearch_find(handle, filename, keystring, length, result,

-    errmsg, LSEARCH_IP);
+    errmsg, LSEARCH_IP, opts);

 
 *errmsg = string_sprintf("\"%s\" is not a valid iplsearch key (an IP "
 "address, with optional CIDR mask, is wanted): "
 
 *errmsg = string_sprintf("\"%s\" is not a valid iplsearch key (an IP "
 "address, with optional CIDR mask, is wanted): "

diff --git a/test/scripts/0000-Basic/0002 b/test/scripts/0000-Basic/0002
index a8fc0bcb274770b166eb3496a73e8a64950b4240..4d170ec687570a6f14fcdfd8b3c2f8a2daf445e5 100644 (file)
--- a/test/scripts/0000-Basic/0002
+++ b/test/scripts/0000-Basic/0002
@@ -526,28 +526,50 @@ acl if: ${if acl {{a_defer}{argN}{arg2}} {Y:$value}{N:$value}}
 # Lookups: DIR is the testing directory. In this test we can only use the
 # lookups that are required in all cases.
 
 # Lookups: DIR is the testing directory. In this test we can only use the
 # lookups that are required in all cases.
 

-${lookup{postmaster}lsearch{DIR/aux-fixed/0002.aliases}{$value}fail}
+${lookup{postmaster}lsearch         {DIR/aux-fixed/0002.aliases}{$value}fail}
+${lookup{postmaster}lsearch,ret=full{DIR/aux-fixed/0002.aliases}{$value}fail}

 
 ${lookup{x@y}lsearch*@{DIR/aux-fixed/0002.starat}{$value}fail}
 
 ${lookup{x@y}lsearch*@{DIR/aux-fixed/0002.starat}{$value}fail}

-${lookup{x@z}lsearch*{DIR/aux-fixed/0002.starat}{$value}fail}
+${lookup{x@z}lsearch* {DIR/aux-fixed/0002.starat}{$value}fail}

 ${lookup{x@z}lsearch*@{DIR/aux-fixed/0002.starat}{$value}fail}
 ${lookup{x@w}lsearch*@{DIR/aux-fixed/0002.starat}{$value}fail}
 
 ${lookup{x@z}lsearch*@{DIR/aux-fixed/0002.starat}{$value}fail}
 ${lookup{x@w}lsearch*@{DIR/aux-fixed/0002.starat}{$value}fail}
 

-${lookup{a.b.c.d}partial-lsearch{DIR/aux-fixed/0002.domains}{$value}fail}
-${lookup{x.y.z}partial-lsearch{DIR/aux-fixed/0002.domains}{$value}{failed x.y.z}}
-${lookup{p.q}partial-lsearch{DIR/aux-fixed/0002.domains}{$value}fail}
-${lookup{o.p.q}partial-lsearch{DIR/aux-fixed/0002.domains}{$value}fail}
-${lookup{m.n.o.p.q}partial-lsearch{DIR/aux-fixed/0002.domains}{$value}fail}
-${lookup{x.y.z}partial1-lsearch{DIR/aux-fixed/0002.domains}{$value}fail}
-${lookup{x.y.z}partial0-lsearch{DIR/aux-fixed/0002.domains}{$value}fail}
-
-q1:  ${lookup{abc}lsearch{DIR/aux-fixed/0002.quoted}}
-q2:  ${lookup{xyz}lsearch{DIR/aux-fixed/0002.quoted}}
-q3:  ${lookup{pqr}lsearch{DIR/aux-fixed/0002.quoted}}
-q4:  ${lookup{a:b}lsearch{DIR/aux-fixed/0002.quoted}}
-q5:  ${lookup{"quoted"}lsearch{DIR/aux-fixed/0002.quoted}}
+${lookup{x@y}lsearch*@,ret=full {DIR/aux-fixed/0002.starat}{$value}fail}
+${lookup{x@z}lsearch*,ret=full  {DIR/aux-fixed/0002.starat}{$value}fail}
+${lookup{x@z}lsearch*@,ret=full {DIR/aux-fixed/0002.starat}{$value}fail}
+${lookup{x@w}lsearch*@,ret=full {DIR/aux-fixed/0002.starat}{$value}fail}
+
+${lookup{a.b.c.d}  partial-lsearch {DIR/aux-fixed/0002.domains}{$value}fail}
+${lookup{x.y.z}    partial-lsearch {DIR/aux-fixed/0002.domains}{$value}{failed x.y.z}}
+${lookup{p.q}      partial-lsearch {DIR/aux-fixed/0002.domains}{$value}fail}
+${lookup{o.p.q}    partial-lsearch {DIR/aux-fixed/0002.domains}{$value}fail}
+${lookup{m.n.o.p.q}partial-lsearch {DIR/aux-fixed/0002.domains}{$value}fail}
+${lookup{x.y.z}    partial1-lsearch{DIR/aux-fixed/0002.domains}{$value}fail}
+${lookup{x.y.z}    partial0-lsearch{DIR/aux-fixed/0002.domains}{$value}fail}
+
+${lookup{a.b.c.d}  partial-lsearch,ret=full {DIR/aux-fixed/0002.domains}{$value}fail}
+${lookup{x.y.z}    partial-lsearch,ret=full {DIR/aux-fixed/0002.domains}{$value}{failed x.y.z}}
+${lookup{p.q}      partial-lsearch,ret=full {DIR/aux-fixed/0002.domains}{$value}fail}
+${lookup{o.p.q}    partial-lsearch,ret=full {DIR/aux-fixed/0002.domains}{$value}fail}
+${lookup{m.n.o.p.q}partial-lsearch,ret=full {DIR/aux-fixed/0002.domains}{$value}fail}
+${lookup{x.y.z}    partial1-lsearch,ret=full{DIR/aux-fixed/0002.domains}{$value}fail}
+${lookup{x.y.z}    partial0-lsearch,ret=full{DIR/aux-fixed/0002.domains}{$value}fail}
+
+q1:  ${lookup{abc}        lsearch{DIR/aux-fixed/0002.quoted}}
+q2:  ${lookup{xyz}        lsearch{DIR/aux-fixed/0002.quoted}}
+q3:  ${lookup{pqr}        lsearch{DIR/aux-fixed/0002.quoted}}
+q4:  ${lookup{a:b}        lsearch{DIR/aux-fixed/0002.quoted}}
+q5:  ${lookup{"quoted"}   lsearch{DIR/aux-fixed/0002.quoted}}

 q6:  ${lookup{white space}lsearch{DIR/aux-fixed/0002.quoted}}
 q6:  ${lookup{white space}lsearch{DIR/aux-fixed/0002.quoted}}

-q7:  ${lookup{b\\s}lsearch{DIR/aux-fixed/0002.quoted}}
+q7:  ${lookup{b\\s}       lsearch{DIR/aux-fixed/0002.quoted}}
+
+q1:  ${lookup{abc}        lsearch,ret=full{DIR/aux-fixed/0002.quoted}}
+q2:  ${lookup{xyz}        lsearch,ret=full{DIR/aux-fixed/0002.quoted}}
+q3:  ${lookup{pqr}        lsearch,ret=full{DIR/aux-fixed/0002.quoted}}
+q4:  ${lookup{a:b}        lsearch,ret=full{DIR/aux-fixed/0002.quoted}}
+q5:  ${lookup{"quoted"}   lsearch,ret=full{DIR/aux-fixed/0002.quoted}}
+q6:  ${lookup{white space}lsearch,ret=full{DIR/aux-fixed/0002.quoted}}
+q7:  ${lookup{b\\s}       lsearch,ret=full{DIR/aux-fixed/0002.quoted}}

 
 abc:   ${lookup{abc}wildlsearch{DIR/aux-var/0002.wild}}
 a.b.c: ${lookup{a.b.c}wildlsearch{DIR/aux-var/0002.wild}}
 
 abc:   ${lookup{abc}wildlsearch{DIR/aux-var/0002.wild}}
 a.b.c: ${lookup{a.b.c}wildlsearch{DIR/aux-var/0002.wild}}

diff --git a/test/stdout/0002 b/test/stdout/0002
index b8ff36122c53752ec4bc3a0499e206f2f90c1b54..d0e8b5d7bf51822c2afff929af871b8dd0f15c31 100644 (file)
--- a/test/stdout/0002
+++ b/test/stdout/0002
@@ -491,12 +491,18 @@ newline   tab\134backslash ~tilde\177DEL\200\201.
 > # lookups that are required in all cases.
 > 
 > CALLER
 > # lookups that are required in all cases.
 > 
 > CALLER

+> postmaster: CALLER

 > 
 > ==X@Y
 > ==*
 > ==*@Z
 > ==*
 > 
 > 
 > ==X@Y
 > ==*
 > ==*@Z
 > ==*
 > 

+> x@y  ==X@Y
+> *    ==*
+> *@z  ==*@Z
+> *    ==*
+> 

 > data for a.b.c.d
 > failed x.y.z
 > data for *.p.q
 > data for a.b.c.d
 > failed x.y.z
 > data for *.p.q


@@ -505,6 +511,14 @@ newline    tab\134backslash ~tilde\177DEL\200\201.
 > Failed: "lookup" failed and "fail" requested
 > data for *
 > 
 > Failed: "lookup" failed and "fail" requested
 > data for *
 > 

+> a.b.c.d: data for a.b.c.d
+> failed x.y.z
+> *.p.q:   data for *.p.q
+> *.p.q:   data for *.p.q
+> *.p.q:   data for *.p.q
+> Failed: "lookup" failed and "fail" requested
+> *:       data for *
+> 

 > q1:  "abc"
 > q2:  "xyz":
 > q3:  "pqr"  :
 > q1:  "abc"
 > q2:  "xyz":
 > q3:  "pqr"  :


@@ -513,6 +527,14 @@ newline    tab\134backslash ~tilde\177DEL\200\201.
 > q6:  "white space"
 > q7:  "b\\s"
 > 
 > q6:  "white space"
 > q7:  "b\\s"
 > 

+> q1:  abc           "abc"
+> q2:  xyz:          "xyz":
+> q3:  pqr  :        "pqr"  :
+> q4:  a:b           "a:b" abc continued data (make sure not key)
+> q5:  "quoted"    "\"quoted\""
+> q6:  white space   "white space"
+> q7:  b\s          "b\\s"
+> 

 > abc:   abc
 > a.b.c: *.b.c
 > ab.c:  *b.c
 > abc:   abc
 > a.b.c: *.b.c
 > ab.c:  *b.c