From b2a6e91b334223c00d07dd3a7ca6d71626d21bf3 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sat, 24 Aug 2024 22:57:07 +0100 Subject: [PATCH] Lookups: lazy-load modules --- doc/doc-docbook/spec.xfpt | 5 +- doc/doc-txt/ChangeLog | 5 ++ src/src/drtables.c | 151 ++++++++++++++++++++++++-------------- src/src/exim.c | 4 + src/src/functions.h | 6 ++ src/src/match.c | 3 +- src/src/readconf.c | 6 ++ src/src/search.c | 42 ++++++++--- src/src/store.c | 2 +- src/src/structs.h | 2 +- src/src/verify.c | 12 ++- test/runtest | 11 +-- 12 files changed, 166 insertions(+), 83 deletions(-) diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt index 4728d9e78..600eee4af 100644 --- a/doc/doc-docbook/spec.xfpt +++ b/doc/doc-docbook/spec.xfpt @@ -2084,7 +2084,10 @@ library dependencies without requiring all systems to install all of those dependencies. .new Any combination of lookup types can be built this way. -All of the lookup modules found as an Exim process starts will be loaded. +Lookup types that provide several variants will be loaded as +Exim starts. +Types that provide only one method are not loaded until used by +the runtime configuration. .wen For building diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog index 0cefe5b7c..22c934b82 100644 --- a/doc/doc-txt/ChangeLog +++ b/doc/doc-txt/ChangeLog @@ -51,6 +51,11 @@ JH/10 Bug 3108: On platforms not providing strchrnul() [OpenBSD] supply a proper different size. This resulted in crashes while processing DKIM signatures of received messages. Identification and fix from Qualys Security. +JH/11 Lookups built as dynamic-load modules which support a single lookup + type are now only loaded if required by the config. Previously all lookup + modules present in the modules directory were loaded; this now applies + only to those supporting multiple types. + Exim version 4.98 ----------------- diff --git a/src/src/drtables.c b/src/src/drtables.c index 0c3dd7049..a144085c5 100644 --- a/src/src/drtables.c +++ b/src/src/drtables.c @@ -340,14 +340,90 @@ extern lookup_module_info whoson_lookup_module_info; extern lookup_module_info readsock_lookup_module_info; +#ifdef LOOKUP_MODULE_DIR +/* Try to load a lookup module with the given name. + +Arguments: + name name of the lookup + errstr if not NULL, place "open fail" error message here + +Return: boolean success +*/ + +static BOOL +lookup_mod_load(const uschar * name, uschar ** errstr) +{ +const uschar * path = string_sprintf( + LOOKUP_MODULE_DIR "/%s_lookup." DYNLIB_FN_EXT, name); +void * dl; +struct lookup_module_info * info; +const char * errormsg; + +if (!(dl = dlopen(CS path, RTLD_NOW))) + { + if (errstr) + *errstr = string_sprintf("Error loading %s: %s", name, dlerror()); + else + (void) dlerror(); /* clear out error state */ + return FALSE; + } + +/* FreeBSD nsdispatch() can trigger dlerror() errors about +_nss_cache_cycle_prevention_function; we need to clear the dlerror() +state before calling dlsym(), so that any error afterwards only comes +from dlsym(). */ + +errormsg = dlerror(); + +info = (struct lookup_module_info *) dlsym(dl, "_lookup_module_info"); +if ((errormsg = dlerror())) + { + fprintf(stderr, "%s does not appear to be a lookup module (%s)\n", name, errormsg); + log_write(0, LOG_MAIN|LOG_PANIC, "%s does not appear to be a lookup module (%s)", name, errormsg); + dlclose(dl); + return FALSE; + } +if (info->magic != LOOKUP_MODULE_INFO_MAGIC) + { + fprintf(stderr, "Lookup module %s is not compatible with this version of Exim\n", name); + log_write(0, LOG_MAIN|LOG_PANIC, "Lookup module %s is not compatible with this version of Exim", name); + dlclose(dl); + return FALSE; + } + +addlookupmodule(info); +DEBUG(D_lookup) debug_printf_indent("Loaded \"%s\" (%d lookup type%s)\n", + name, info->lookupcount, + info->lookupcount > 1 ? "s" : ""); +return TRUE; +} + + +/* Try to load a lookup module, assuming the module name is the same +as the lookup type name. This will only work for single-method modules. +Other have to be always-load (see the RE in init_lookup_list() below). +*/ + +BOOL +lookup_one_mod_load(const uschar * name, uschar ** errstr) +{ +if (!lookup_mod_load(name, errstr)) return FALSE; +/*XXX notify daemon? */ +return TRUE; +} + +#endif /*LOOKUP_MODULE_DIR*/ + + + + + void init_lookup_list(void) { #ifdef LOOKUP_MODULE_DIR DIR * dd; -struct dirent * ent; int countmodules = 0; -int moduleerrors = 0; #endif static BOOL lookup_list_init_done = FALSE; @@ -441,6 +517,9 @@ implemented by a lookup module. */ addlookupmodule(&readsock_lookup_module_info); +DEBUG(D_lookup) debug_printf("Total %d built-in lookups\n", lookup_list_count); + + #ifdef LOOKUP_MODULE_DIR if (!(dd = exim_opendir(CUS LOOKUP_MODULE_DIR))) { @@ -450,81 +529,39 @@ if (!(dd = exim_opendir(CUS LOOKUP_MODULE_DIR))) } else { + /* Look specifically for modules we know offer several lookup types and + load them now, since we cannot load-on-first-use. */ + + struct dirent * ent; const pcre2_code * regex_islookupmod = regex_must_compile( - US"_lookup\\." DYNLIB_FN_EXT "$", MCS_NOFLAGS, TRUE); + US"(lsearch|ldap|nis)_lookup\\." DYNLIB_FN_EXT "$", MCS_NOFLAGS, TRUE); DEBUG(D_lookup) debug_printf("Loading lookup modules from %s\n", LOOKUP_MODULE_DIR); while ((ent = readdir(dd))) { char * name = ent->d_name; int len = (int)strlen(name); - if (regex_match(regex_islookupmod, US name, len, NULL)) + if (regex_match_and_setup(regex_islookupmod, US name, 0, 0)) { - int pathnamelen = len + (int)strlen(LOOKUP_MODULE_DIR) + 2; - void *dl; - struct lookup_module_info *info; - const char *errormsg; - - /* SRH: am I being paranoid here or what? */ - if (pathnamelen > big_buffer_size) - { - fprintf(stderr, "Loading lookup modules: %s/%s: name too long\n", LOOKUP_MODULE_DIR, name); - log_write(0, LOG_MAIN|LOG_PANIC, "%s/%s: name too long\n", LOOKUP_MODULE_DIR, name); - continue; - } - - /* SRH: snprintf here? */ - sprintf(CS big_buffer, "%s/%s", LOOKUP_MODULE_DIR, name); - - if (!(dl = dlopen(CS big_buffer, RTLD_NOW))) + uschar * errstr; + if (lookup_mod_load(expand_nstring[1], &errstr)) + countmodules++; + else { - errormsg = dlerror(); - fprintf(stderr, "Error loading %s: %s\n", name, errormsg); - log_write(0, LOG_MAIN|LOG_PANIC, "Error loading lookup module %s: %s\n", name, errormsg); - moduleerrors++; - continue; + fprintf(stderr, "%s\n", errstr); + log_write(0, LOG_MAIN|LOG_PANIC, "%s", errstr); } - - /* FreeBSD nsdispatch() can trigger dlerror() errors about - _nss_cache_cycle_prevention_function; we need to clear the dlerror() - state before calling dlsym(), so that any error afterwards only comes - from dlsym(). */ - - errormsg = dlerror(); - - info = (struct lookup_module_info*) dlsym(dl, "_lookup_module_info"); - if ((errormsg = dlerror())) - { - fprintf(stderr, "%s does not appear to be a lookup module (%s)\n", name, errormsg); - log_write(0, LOG_MAIN|LOG_PANIC, "%s does not appear to be a lookup module (%s)\n", name, errormsg); - dlclose(dl); - moduleerrors++; - continue; - } - if (info->magic != LOOKUP_MODULE_INFO_MAGIC) - { - fprintf(stderr, "Lookup module %s is not compatible with this version of Exim\n", name); - log_write(0, LOG_MAIN|LOG_PANIC, "Lookup module %s is not compatible with this version of Exim\n", name); - dlclose(dl); - moduleerrors++; - continue; - } - - addlookupmodule(info); - DEBUG(D_lookup) debug_printf("Loaded \"%s\" (%d lookup types)\n", name, info->lookupcount); - countmodules++; } } - store_free((void*)regex_islookupmod); closedir(dd); } DEBUG(D_lookup) debug_printf("Loaded %d lookup modules\n", countmodules); #endif -DEBUG(D_lookup) debug_printf("Total %d lookups\n", lookup_list_count); } + #endif /*!MACRO_PREDEF*/ /* End of drtables.c */ diff --git a/src/src/exim.c b/src/src/exim.c index 1a22c9af2..de1f48434 100644 --- a/src/src/exim.c +++ b/src/src/exim.c @@ -1806,6 +1806,10 @@ len = Ustrlen(big_buffer); (void) macros_expand(0, &len, &dummy_macexp); +#ifdef LOOKUP_MODULE_DIR +//mod_load_check(big_buffer); +#endif + if (isupper(big_buffer[0])) { if (macro_read_assignment(big_buffer)) diff --git a/src/src/functions.h b/src/src/functions.h index b97895b8a..be60d5fc9 100644 --- a/src/src/functions.h +++ b/src/src/functions.h @@ -339,6 +339,9 @@ extern const uschar *local_part_quote(const uschar *); extern int log_open_as_exim(const uschar * const); extern void log_close_all(void); extern const lookup_info * lookup_with_acq_num(unsigned); +#ifdef LOOKUP_MODULE_DIR +extern BOOL lookup_one_mod_load(const uschar *, uschar **); +#endif extern macro_item * macro_create(const uschar *, const uschar *, BOOL); @@ -388,6 +391,9 @@ extern void moan_tell_someone(uschar *, address_item *, extern BOOL moan_to_sender(int, error_block *, header_line *, FILE *, BOOL); extern void moan_write_from(FILE *); extern void moan_write_references(FILE *, uschar *); +#ifdef LOOKUP_MODULE_DIR +//extern void mod_load_check(const uschar *); +#endif extern FILE *modefopen(const uschar *, const char *, mode_t); extern int open_cutthrough_connection( address_item * addr ); diff --git a/src/src/match.c b/src/src/match.c index 0b3632f7e..dc88485a4 100644 --- a/src/src/match.c +++ b/src/src/match.c @@ -272,7 +272,8 @@ the part of the string preceding the semicolon. */ li = search_findtype_partial(pattern, &partial, &affix, &affixlen, &starflags, &opts); *semicolon = ';'; -if (!li) log_write(0, LOG_MAIN|LOG_PANIC_DIE, "%s", search_error_message); +if (!li) + log_write(0, LOG_MAIN|LOG_PANIC_DIE, "%s", search_error_message); /* Partial matching is not appropriate for certain lookups (e.g. when looking up user@domain for sender rejection). There's a flag to disable it. */ diff --git a/src/src/readconf.c b/src/src/readconf.c index 532e0774c..a090cfb30 100644 --- a/src/src/readconf.c +++ b/src/src/readconf.c @@ -1232,6 +1232,12 @@ if (strncmpic(s, US"begin ", 6) == 0) return NULL; } +#ifdef LOOKUP_MODULE_DIR +/* Check for any required module load operations */ + +//mod_load_check(s); +#endif + /* Return the first non-blank character. */ return s; diff --git a/src/src/search.c b/src/src/search.c index efa720f25..ad55e8780 100644 --- a/src/src/search.c +++ b/src/src/search.c @@ -51,9 +51,17 @@ static rmark search_reset_point = NULL; * Validate a plain lookup type name * *************************************************/ +static const lookup_info * +internal_search_findtype(const uschar * name) +{ +tree_node * t = tree_search(lookups_tree, name); +return t ? t->data.ptr : NULL; +} + /* Only those names that are recognized and whose code is included in the -binary give an OK response. Use a binary chop search now that the list has got -so long. +binary give an OK response. Types are held in a binary tree for fast location +and dynamic insertion. If not initially found, try to load a module if +any were compiled. Arguments: name lookup type name - not necessarily zero terminated (e.g. dbm*) @@ -66,18 +74,32 @@ Returns: ptr to info struct for the lookup, const lookup_info * search_findtype(const uschar * name, int len) { -const uschar * s = name[len] ? string_copyn(name, len) : name; -tree_node * t = tree_search(lookups_tree, s); +const lookup_info * li; + +if (name[len]) + name = string_copyn(name, len); +if ((li = internal_search_findtype(name))) + return li; -if (t) return t->data.ptr; +#ifdef LOOKUP_MODULE_DIR + DEBUG(D_lookup) + debug_printf_indent("searchtype %s not initially found\n", name); -search_error_message = string_sprintf("unknown lookup type \"%s\"", s); + if (lookup_one_mod_load(name, NULL)) + if ((li = internal_search_findtype(name))) + return li; + else + { DEBUG(D_lookup) debug_printf_indent("find retry failed\n"); } + else DEBUG(D_lookup) + debug_printf_indent("scan modules dir for %s failed\n", name); +#endif + +search_error_message = string_sprintf("unknown lookup type \"%s\"", name); return NULL; } - /************************************************* * Validate a full lookup type name * *************************************************/ @@ -207,8 +229,8 @@ Arguments: Return: keyquery the search-type (for single-key) or query (for query-type) */ uschar * -search_args(const lookup_info * li, uschar * search, uschar * query, uschar ** fnamep, - const uschar * opts) +search_args(const lookup_info * li, uschar * search, uschar * query, + uschar ** fnamep, const uschar * opts) { Uskip_whitespace(&query); if (mac_islookup(li, lookup_absfilequery)) @@ -430,7 +452,7 @@ if (li->type == lookup_absfile && open_filecount >= lookup_open_max) ((search_cache *)(open_bot->data.ptr))->down = NULL; else open_top = NULL; - ((c->li)->close)(c->handle); + (c->li->close)(c->handle); c->handle = NULL; open_filecount--; } diff --git a/src/src/store.c b/src/src/store.c index c135b4383..4824b5c54 100644 --- a/src/src/store.c +++ b/src/src/store.c @@ -661,7 +661,7 @@ find variants but shared quote functions. */ BOOL is_quoted_like(const void * p, unsigned quoter) { -const uschar * p_name, * q_name; +const uschar * p_name, * q_name = NULL; const lookup_info * p_li, * q_li; void * p_qfn, * q_qfn; diff --git a/src/src/structs.h b/src/src/structs.h index 702c7c72c..9492dbac2 100644 --- a/src/src/structs.h +++ b/src/src/structs.h @@ -742,7 +742,7 @@ to close. */ typedef struct search_cache { void *handle; /* lookup handle, or NULL if closed */ - const lookup_info *li; /* info struct for search type */ + const lookup_info * li; /* info struct for search type */ tree_node *up; /* LRU up pointer */ tree_node *down; /* LRU down pointer */ tree_node *item_cache; /* tree of cached results */ diff --git a/src/src/verify.c b/src/src/verify.c index f936de50a..05b15858a 100644 --- a/src/src/verify.c +++ b/src/src/verify.c @@ -3059,11 +3059,10 @@ else if (iplookup) { - int insize; const lookup_info * li; - int incoming[4]; - void *handle; - uschar *filename, *key, *result; + int incoming[4], insize; + void * handle; + uschar * filename, * key, * result; uschar buffer[64]; /* Find the search type */ @@ -3094,11 +3093,10 @@ if (iplookup) } else /* Single-key style */ { - int sep = (Ustrcmp(li->name, "iplsearch") == 0)? - ':' : '.'; + int sep = Ustrcmp(li->name, "iplsearch") == 0 ? ':' : '.'; insize = host_aton(cb->host_address, incoming); host_mask(insize, incoming, mlen); - (void)host_nmtoa(insize, incoming, mlen, buffer, sep); + (void) host_nmtoa(insize, incoming, mlen, buffer, sep); key = buffer; filename = semicolon + 1; } diff --git a/test/runtest b/test/runtest index e80624484..3e10a5ab7 100755 --- a/test/runtest +++ b/test/runtest @@ -1297,11 +1297,12 @@ RESET_AFTER_EXTRA_LINE_READ: next if /^tls_set_watch\(\) fail on '\/usr\/(?:lib\/ssl|local\/openssl3\/etc\/pki\/tls)\/cert.pem': No such file or directory$/; # drop lookups - next if /^$time_pid?(?: Lookups\ \((?:built-in|dynamic)\): - | Loaded\ "[^.]+\.so"\ \(\d+\ lookup\ types\) - | Loading\ lookup\ modules\ from - | Loaded\ \d+\ lookup\ modules - | Total\ \d+\ lookups)/x; + next if /(?: Lookups\ \((?:built-in|dynamic)\): + | searchtype\ \w+\ not\ initially\ found + | Loaded\ "\w+"\ \(\d+\ lookup\ types?\) + | Loading\ lookup\ modules\ from + | Loaded\ \d+\ lookup\ modules + | Total\ \d+\ built-in\ lookups)/x; # drop loads of dyn-module drivers next if /^$time_pid?(?:Loading\ \w+\ (?:router|transport|auth)\ driver\ from -- 2.30.2