Lookups: lazy-load modules

diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt
index 4728d9e78d8943a420766c57eaf2f805578d3cf2..600eee4afbef200a9de892f55e82b0b51263c0ac 100644 (file)
--- a/doc/doc-docbook/spec.xfpt
+++ b/doc/doc-docbook/spec.xfpt
@@ -2084,7 +2084,10 @@ library dependencies without requiring all systems to install all of those
 dependencies.
 .new
 Any combination of lookup types can be built this way.
 dependencies.
 .new
 Any combination of lookup types can be built this way.

-All of the lookup modules found as an Exim process starts will be loaded.
+Lookup types that provide several variants will be loaded as
+Exim starts.
+Types that provide only one method are not loaded until used by
+the runtime configuration.

 .wen
 
 For building
 .wen
 
 For building

diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog
index 0cefe5b7cc3759f5d5e3906ee73e10341f3c1afc..22c934b82e7fa88367a03fac8d46fa796027b508 100644 (file)
--- a/doc/doc-txt/ChangeLog
+++ b/doc/doc-txt/ChangeLog
@@ -51,6 +51,11 @@ JH/10 Bug 3108: On platforms not providing strchrnul() [OpenBSD] supply a proper
       different size.  This resulted in crashes while processing DKIM signatures
       of received messages.  Identification and fix from Qualys Security.
 
       different size.  This resulted in crashes while processing DKIM signatures
       of received messages.  Identification and fix from Qualys Security.
 

+JH/11 Lookups built as dynamic-load modules which support a single lookup
+      type are now only loaded if required by the config. Previously all lookup
+      modules present in the modules directory were loaded; this now applies
+      only to those supporting multiple types.
+

 Exim version 4.98
 -----------------
 
 Exim version 4.98
 -----------------
 

diff --git a/src/src/drtables.c b/src/src/drtables.c
index 0c3dd70491099e1a33be7c0960c2ea724fc07ef5..a144085c5a797397c356ecce68ad46b45fab503b 100644 (file)
--- a/src/src/drtables.c
+++ b/src/src/drtables.c
@@ -340,14 +340,90 @@ extern lookup_module_info whoson_lookup_module_info;
 extern lookup_module_info readsock_lookup_module_info;
 
 
 extern lookup_module_info readsock_lookup_module_info;
 
 

+#ifdef LOOKUP_MODULE_DIR
+/* Try to load a lookup module with the given name.
+
+Arguments:
+    name               name of the lookup
+    errstr             if not NULL, place "open fail" error message here
+
+Return: boolean success
+*/
+
+static BOOL
+lookup_mod_load(const uschar * name, uschar ** errstr)
+{
+const uschar * path = string_sprintf(
+  LOOKUP_MODULE_DIR "/%s_lookup." DYNLIB_FN_EXT, name);
+void * dl;
+struct lookup_module_info * info;
+const char * errormsg;
+
+if (!(dl = dlopen(CS path, RTLD_NOW)))
+  {
+  if (errstr)
+    *errstr = string_sprintf("Error loading %s: %s", name, dlerror());
+  else
+    (void) dlerror();          /* clear out error state */
+  return FALSE;
+  }
+
+/* FreeBSD nsdispatch() can trigger dlerror() errors about
+_nss_cache_cycle_prevention_function; we need to clear the dlerror()
+state before calling dlsym(), so that any error afterwards only comes
+from dlsym().  */
+
+errormsg = dlerror();
+
+info = (struct lookup_module_info *) dlsym(dl, "_lookup_module_info");
+if ((errormsg = dlerror()))
+  {
+  fprintf(stderr, "%s does not appear to be a lookup module (%s)\n", name, errormsg);
+  log_write(0, LOG_MAIN|LOG_PANIC, "%s does not appear to be a lookup module (%s)", name, errormsg);
+  dlclose(dl);
+  return FALSE;
+  }
+if (info->magic != LOOKUP_MODULE_INFO_MAGIC)
+  {
+  fprintf(stderr, "Lookup module %s is not compatible with this version of Exim\n", name);
+  log_write(0, LOG_MAIN|LOG_PANIC, "Lookup module %s is not compatible with this version of Exim", name);
+  dlclose(dl);
+  return FALSE;
+  }
+
+addlookupmodule(info);
+DEBUG(D_lookup) debug_printf_indent("Loaded \"%s\" (%d lookup type%s)\n",
+                                   name, info->lookupcount,
+                                   info->lookupcount > 1 ? "s" : "");
+return TRUE;
+}
+
+
+/* Try to load a lookup module, assuming the module name is the same
+as the lookup type name.  This will only work for single-method modules.
+Other have to be always-load (see the RE in init_lookup_list() below).
+*/
+
+BOOL
+lookup_one_mod_load(const uschar * name, uschar ** errstr)
+{
+if (!lookup_mod_load(name, errstr)) return FALSE;
+/*XXX notify daemon? */
+return TRUE;
+}
+
+#endif /*LOOKUP_MODULE_DIR*/
+
+
+
+
+

 void
 init_lookup_list(void)
 {
 #ifdef LOOKUP_MODULE_DIR
 DIR * dd;
 void
 init_lookup_list(void)
 {
 #ifdef LOOKUP_MODULE_DIR
 DIR * dd;

-struct dirent * ent;

 int countmodules = 0;
 int countmodules = 0;

-int moduleerrors = 0;

 #endif
 static BOOL lookup_list_init_done = FALSE;
 
 #endif
 static BOOL lookup_list_init_done = FALSE;
 


@@ -441,6 +517,9 @@ implemented by a lookup module. */
 
 addlookupmodule(&readsock_lookup_module_info);
 
 
 addlookupmodule(&readsock_lookup_module_info);
 

+DEBUG(D_lookup) debug_printf("Total %d built-in lookups\n", lookup_list_count);
+
+

 #ifdef LOOKUP_MODULE_DIR
 if (!(dd = exim_opendir(CUS LOOKUP_MODULE_DIR)))
   {
 #ifdef LOOKUP_MODULE_DIR
 if (!(dd = exim_opendir(CUS LOOKUP_MODULE_DIR)))
   {


@@ -450,81 +529,39 @@ if (!(dd = exim_opendir(CUS LOOKUP_MODULE_DIR)))
   }
 else
   {
   }
 else
   {

+  /* Look specifically for modules we know offer several lookup types and
+  load them now, since we cannot load-on-first-use. */
+
+  struct dirent * ent;

   const pcre2_code * regex_islookupmod = regex_must_compile(
   const pcre2_code * regex_islookupmod = regex_must_compile(

-    US"_lookup\\." DYNLIB_FN_EXT "$", MCS_NOFLAGS, TRUE);
+    US"(lsearch|ldap|nis)_lookup\\." DYNLIB_FN_EXT "$", MCS_NOFLAGS, TRUE);

 
   DEBUG(D_lookup) debug_printf("Loading lookup modules from %s\n", LOOKUP_MODULE_DIR);
   while ((ent = readdir(dd)))
     {
     char * name = ent->d_name;
     int len = (int)strlen(name);
 
   DEBUG(D_lookup) debug_printf("Loading lookup modules from %s\n", LOOKUP_MODULE_DIR);
   while ((ent = readdir(dd)))
     {
     char * name = ent->d_name;
     int len = (int)strlen(name);

-    if (regex_match(regex_islookupmod, US name, len, NULL))
+    if (regex_match_and_setup(regex_islookupmod, US name, 0, 0))

       {
       {

-      int pathnamelen = len + (int)strlen(LOOKUP_MODULE_DIR) + 2;
-      void *dl;
-      struct lookup_module_info *info;
-      const char *errormsg;
-
-      /* SRH: am I being paranoid here or what? */
-      if (pathnamelen > big_buffer_size)
-       {
-       fprintf(stderr, "Loading lookup modules: %s/%s: name too long\n", LOOKUP_MODULE_DIR, name);
-       log_write(0, LOG_MAIN|LOG_PANIC, "%s/%s: name too long\n", LOOKUP_MODULE_DIR, name);
-       continue;
-       }
-
-      /* SRH: snprintf here? */
-      sprintf(CS big_buffer, "%s/%s", LOOKUP_MODULE_DIR, name);
-
-      if (!(dl = dlopen(CS big_buffer, RTLD_NOW)))
+      uschar * errstr;
+      if (lookup_mod_load(expand_nstring[1], &errstr))
+       countmodules++;
+      else

        {
        {

-       errormsg = dlerror();
-       fprintf(stderr, "Error loading %s: %s\n", name, errormsg);
-       log_write(0, LOG_MAIN|LOG_PANIC, "Error loading lookup module %s: %s\n", name, errormsg);
-       moduleerrors++;
-       continue;
+       fprintf(stderr, "%s\n", errstr);
+       log_write(0, LOG_MAIN|LOG_PANIC, "%s", errstr);

        }
        }

-
-      /* FreeBSD nsdispatch() can trigger dlerror() errors about
-      _nss_cache_cycle_prevention_function; we need to clear the dlerror()
-      state before calling dlsym(), so that any error afterwards only comes
-      from dlsym().  */
-
-      errormsg = dlerror();
-
-      info = (struct lookup_module_info*) dlsym(dl, "_lookup_module_info");
-      if ((errormsg = dlerror()))
-       {
-       fprintf(stderr, "%s does not appear to be a lookup module (%s)\n", name, errormsg);
-       log_write(0, LOG_MAIN|LOG_PANIC, "%s does not appear to be a lookup module (%s)\n", name, errormsg);
-       dlclose(dl);
-       moduleerrors++;
-       continue;
-       }
-      if (info->magic != LOOKUP_MODULE_INFO_MAGIC)
-       {
-       fprintf(stderr, "Lookup module %s is not compatible with this version of Exim\n", name);
-       log_write(0, LOG_MAIN|LOG_PANIC, "Lookup module %s is not compatible with this version of Exim\n", name);
-       dlclose(dl);
-       moduleerrors++;
-       continue;
-       }
-
-      addlookupmodule(info);
-      DEBUG(D_lookup) debug_printf("Loaded \"%s\" (%d lookup types)\n", name, info->lookupcount);
-      countmodules++;

       }
     }
       }
     }

-  store_free((void*)regex_islookupmod);

   closedir(dd);
   }
 
 DEBUG(D_lookup) debug_printf("Loaded %d lookup modules\n", countmodules);
 #endif
 
   closedir(dd);
   }
 
 DEBUG(D_lookup) debug_printf("Loaded %d lookup modules\n", countmodules);
 #endif
 

-DEBUG(D_lookup) debug_printf("Total %d lookups\n", lookup_list_count);

 
 }
 
 
 }
 

+

 #endif /*!MACRO_PREDEF*/
 /* End of drtables.c */
 #endif /*!MACRO_PREDEF*/
 /* End of drtables.c */

diff --git a/src/src/exim.c b/src/src/exim.c
index 1a22c9af2528fcd358bba883254f441312e7b40c..de1f484341b9f3b917aff2c25074ca8586208a8f 100644 (file)
--- a/src/src/exim.c
+++ b/src/src/exim.c
@@ -1806,6 +1806,10 @@ len = Ustrlen(big_buffer);
 
 (void) macros_expand(0, &len, &dummy_macexp);
 
 
 (void) macros_expand(0, &len, &dummy_macexp);
 

+#ifdef LOOKUP_MODULE_DIR
+//mod_load_check(big_buffer);
+#endif
+

 if (isupper(big_buffer[0]))
   {
   if (macro_read_assignment(big_buffer))
 if (isupper(big_buffer[0]))
   {
   if (macro_read_assignment(big_buffer))

diff --git a/src/src/functions.h b/src/src/functions.h
index b97895b8a5d2fccb8c93016aec033b4b416c598e..be60d5fc9a49ad7dbb793687f19e4b475d2a1f5f 100644 (file)
--- a/src/src/functions.h
+++ b/src/src/functions.h
@@ -339,6 +339,9 @@ extern const uschar *local_part_quote(const uschar *);
 extern int     log_open_as_exim(const uschar * const);
 extern void    log_close_all(void);
 extern const lookup_info * lookup_with_acq_num(unsigned);
 extern int     log_open_as_exim(const uschar * const);
 extern void    log_close_all(void);
 extern const lookup_info * lookup_with_acq_num(unsigned);

+#ifdef LOOKUP_MODULE_DIR
+extern BOOL    lookup_one_mod_load(const uschar *, uschar **);
+#endif

 
 
 extern macro_item * macro_create(const uschar *, const uschar *, BOOL);
 
 
 extern macro_item * macro_create(const uschar *, const uschar *, BOOL);


@@ -388,6 +391,9 @@ extern void    moan_tell_someone(uschar *, address_item *,
 extern BOOL    moan_to_sender(int, error_block *, header_line *, FILE *, BOOL);
 extern void    moan_write_from(FILE *);
 extern void    moan_write_references(FILE *, uschar *);
 extern BOOL    moan_to_sender(int, error_block *, header_line *, FILE *, BOOL);
 extern void    moan_write_from(FILE *);
 extern void    moan_write_references(FILE *, uschar *);

+#ifdef LOOKUP_MODULE_DIR
+//extern void    mod_load_check(const uschar *);
+#endif

 extern FILE   *modefopen(const uschar *, const char *, mode_t);
 
 extern int     open_cutthrough_connection( address_item * addr );
 extern FILE   *modefopen(const uschar *, const char *, mode_t);
 
 extern int     open_cutthrough_connection( address_item * addr );

diff --git a/src/src/match.c b/src/src/match.c
index 0b3632f7e49a06983cdb26a5eb88ca39c4404167..dc88485a4d8f696e6a07440e2b59eb4f28ddb2cc 100644 (file)
--- a/src/src/match.c
+++ b/src/src/match.c
@@ -272,7 +272,8 @@ the part of the string preceding the semicolon. */
 li = search_findtype_partial(pattern, &partial, &affix, &affixlen,
   &starflags, &opts);
 *semicolon = ';';
 li = search_findtype_partial(pattern, &partial, &affix, &affixlen,
   &starflags, &opts);
 *semicolon = ';';

-if (!li) log_write(0, LOG_MAIN|LOG_PANIC_DIE, "%s", search_error_message);
+if (!li)
+  log_write(0, LOG_MAIN|LOG_PANIC_DIE, "%s", search_error_message);

 
 /* Partial matching is not appropriate for certain lookups (e.g. when looking
 up user@domain for sender rejection). There's a flag to disable it. */
 
 /* Partial matching is not appropriate for certain lookups (e.g. when looking
 up user@domain for sender rejection). There's a flag to disable it. */

diff --git a/src/src/readconf.c b/src/src/readconf.c
index 532e0774c6fdfc934222090eabdf8282ee5b6514..a090cfb30c1109eda3ddf937e7fbf5cf397731f7 100644 (file)
--- a/src/src/readconf.c
+++ b/src/src/readconf.c
@@ -1232,6 +1232,12 @@ if (strncmpic(s, US"begin ", 6) == 0)
   return NULL;
   }
 
   return NULL;
   }
 

+#ifdef LOOKUP_MODULE_DIR
+/* Check for any required module load operations */
+
+//mod_load_check(s);
+#endif
+

 /* Return the first non-blank character. */
 
 return s;
 /* Return the first non-blank character. */
 
 return s;

diff --git a/src/src/search.c b/src/src/search.c
index efa720f257cbf825765332041a7d04cb3a4f4d7e..ad55e878010d0ac2a7bdce3b9846937b391a00b0 100644 (file)
--- a/src/src/search.c
+++ b/src/src/search.c
@@ -51,9 +51,17 @@ static rmark search_reset_point = NULL;
 *      Validate a plain lookup type name         *
 *************************************************/
 
 *      Validate a plain lookup type name         *
 *************************************************/
 

+static const lookup_info *
+internal_search_findtype(const uschar * name)
+{
+tree_node * t = tree_search(lookups_tree, name);
+return t ? t->data.ptr : NULL;
+}
+

 /* Only those names that are recognized and whose code is included in the
 /* Only those names that are recognized and whose code is included in the

-binary give an OK response. Use a binary chop search now that the list has got
-so long.
+binary give an OK response. Types are held in a binary tree for fast location
+and dynamic insertion.  If not initially found, try to load a module if
+any were compiled.

 
 Arguments:
   name       lookup type name - not necessarily zero terminated (e.g. dbm*)
 
 Arguments:
   name       lookup type name - not necessarily zero terminated (e.g. dbm*)


@@ -66,18 +74,32 @@ Returns:     ptr to info struct for the lookup,
 const lookup_info *
 search_findtype(const uschar * name, int len)
 {
 const lookup_info *
 search_findtype(const uschar * name, int len)
 {

-const uschar * s = name[len] ? string_copyn(name, len) : name;
-tree_node * t = tree_search(lookups_tree, s);
+const lookup_info * li;
+
+if (name[len])
+  name = string_copyn(name, len);
+if ((li = internal_search_findtype(name)))
+  return li;

 
 

-if (t) return t->data.ptr;
+#ifdef LOOKUP_MODULE_DIR
+    DEBUG(D_lookup)
+      debug_printf_indent("searchtype %s not initially found\n", name);

 
 

-search_error_message = string_sprintf("unknown lookup type \"%s\"", s);
+    if (lookup_one_mod_load(name, NULL))
+      if ((li = internal_search_findtype(name)))
+       return li;
+      else
+       { DEBUG(D_lookup) debug_printf_indent("find retry failed\n"); }
+    else DEBUG(D_lookup)
+      debug_printf_indent("scan modules dir for %s failed\n", name);
+#endif
+
+search_error_message  = string_sprintf("unknown lookup type \"%s\"", name);

 return NULL;
 }
 
 
 
 return NULL;
 }
 
 
 

-

 /*************************************************
 *       Validate a full lookup type name         *
 *************************************************/
 /*************************************************
 *       Validate a full lookup type name         *
 *************************************************/


@@ -207,8 +229,8 @@ Arguments:
 Return:        keyquery        the search-type (for single-key) or query (for query-type)
  */
 uschar *
 Return:        keyquery        the search-type (for single-key) or query (for query-type)
  */
 uschar *

-search_args(const lookup_info * li, uschar * search, uschar * query, uschar ** fnamep,
-  const uschar * opts)
+search_args(const lookup_info * li, uschar * search, uschar * query,
+  uschar ** fnamep, const uschar * opts)

 {
 Uskip_whitespace(&query);
 if (mac_islookup(li, lookup_absfilequery))
 {
 Uskip_whitespace(&query);
 if (mac_islookup(li, lookup_absfilequery))


@@ -430,7 +452,7 @@ if (li->type == lookup_absfile && open_filecount >= lookup_open_max)
       ((search_cache *)(open_bot->data.ptr))->down = NULL;
     else
       open_top = NULL;
       ((search_cache *)(open_bot->data.ptr))->down = NULL;
     else
       open_top = NULL;

-    ((c->li)->close)(c->handle);
+    (c->li->close)(c->handle);

     c->handle = NULL;
     open_filecount--;
     }
     c->handle = NULL;
     open_filecount--;
     }

diff --git a/src/src/store.c b/src/src/store.c
index c135b43839f081a1d75657e1e75663341f54a4bb..4824b5c54fc89a9dd43f9a2b02ca51a969d8f1eb 100644 (file)
--- a/src/src/store.c
+++ b/src/src/store.c
@@ -661,7 +661,7 @@ find variants but shared quote functions. */
 BOOL
 is_quoted_like(const void * p, unsigned quoter)
 {
 BOOL
 is_quoted_like(const void * p, unsigned quoter)
 {

-const uschar * p_name, * q_name;
+const uschar * p_name, * q_name = NULL;

 const lookup_info * p_li, * q_li;
 void * p_qfn, * q_qfn;
 
 const lookup_info * p_li, * q_li;
 void * p_qfn, * q_qfn;
 

diff --git a/src/src/structs.h b/src/src/structs.h
index 702c7c72c8adf25f44f08d0cc6678699ac825be7..9492dbac23454e9fd202fbcc0df848a887010986 100644 (file)
--- a/src/src/structs.h
+++ b/src/src/structs.h
@@ -742,7 +742,7 @@ to close. */
 
 typedef struct search_cache {
   void   *handle;                 /* lookup handle, or NULL if closed */
 
 typedef struct search_cache {
   void   *handle;                 /* lookup handle, or NULL if closed */

-  const lookup_info *li;         /* info struct for search type */
+  const lookup_info * li;        /* info struct for search type */

   tree_node *up;                  /* LRU up pointer */
   tree_node *down;                /* LRU down pointer */
   tree_node *item_cache;          /* tree of cached results */
   tree_node *up;                  /* LRU up pointer */
   tree_node *down;                /* LRU down pointer */
   tree_node *item_cache;          /* tree of cached results */

diff --git a/src/src/verify.c b/src/src/verify.c
index f936de50a761555fca95ab628f10d7e796371dc8..05b15858a34aeddcf245ef9dcbd942c63426e661 100644 (file)
--- a/src/src/verify.c
+++ b/src/src/verify.c
@@ -3059,11 +3059,10 @@ else
 
 if (iplookup)
   {
 
 if (iplookup)
   {

-  int insize;

   const lookup_info * li;
   const lookup_info * li;

-  int incoming[4];
-  void *handle;
-  uschar *filename, *key, *result;
+  int incoming[4], insize;
+  void * handle;
+  uschar * filename, * key, * result;

   uschar buffer[64];
 
   /* Find the search type */
   uschar buffer[64];
 
   /* Find the search type */


@@ -3094,11 +3093,10 @@ if (iplookup)
     }
   else   /* Single-key style */
     {
     }
   else   /* Single-key style */
     {

-    int sep = (Ustrcmp(li->name, "iplsearch") == 0)?
-      ':' : '.';
+    int sep = Ustrcmp(li->name, "iplsearch") == 0 ? ':' : '.';

     insize = host_aton(cb->host_address, incoming);
     host_mask(insize, incoming, mlen);
     insize = host_aton(cb->host_address, incoming);
     host_mask(insize, incoming, mlen);

-    (void)host_nmtoa(insize, incoming, mlen, buffer, sep);
+    (void) host_nmtoa(insize, incoming, mlen, buffer, sep);

     key = buffer;
     filename = semicolon + 1;
     }
     key = buffer;
     filename = semicolon + 1;
     }

diff --git a/test/runtest b/test/runtest
index e806244848ab6ae7d91994b056f715f931c9d48f..3e10a5ab7e5dcb88764d4a4c75d502cefbc1e2e3 100755 (executable)
--- a/test/runtest
+++ b/test/runtest
@@ -1297,11 +1297,12 @@ RESET_AFTER_EXTRA_LINE_READ:
     next if /^tls_set_watch\(\) fail on '\/usr\/(?:lib\/ssl|local\/openssl3\/etc\/pki\/tls)\/cert.pem': No such file or directory$/;
 
     # drop lookups
     next if /^tls_set_watch\(\) fail on '\/usr\/(?:lib\/ssl|local\/openssl3\/etc\/pki\/tls)\/cert.pem': No such file or directory$/;
 
     # drop lookups

-    next if /^$time_pid?(?: Lookups\ \((?:built-in|dynamic)\):
-                                       | Loaded\ "[^.]+\.so"\ \(\d+\ lookup\ types\)
-                                       | Loading\ lookup\ modules\ from
-                                       | Loaded\ \d+\ lookup\ modules
-                                       | Total\ \d+\ lookups)/x;
+    next if /(?: Lookups\ \((?:built-in|dynamic)\):
+             | searchtype\ \w+\ not\ initially\ found
+             | Loaded\ "\w+"\ \(\d+\ lookup\ types?\)
+             | Loading\ lookup\ modules\ from
+             | Loaded\ \d+\ lookup\ modules
+             | Total\ \d+\ built-in\ lookups)/x;

 
     # drop loads of dyn-module drivers
     next if /^$time_pid?(?:Loading\ \w+\ (?:router|transport|auth)\ driver\ from
 
     # drop loads of dyn-module drivers
     next if /^$time_pid?(?:Loading\ \w+\ (?:router|transport|auth)\ driver\ from