1 /*************************************************
2 * Exim - an Internet mail transport agent *
3 *************************************************/
5 /* Copyright (c) University of Cambridge 1995 - 2015 */
6 /* Copyright (c) The Exim Maintainers 2020 */
7 /* See the file NOTICE for conditions of use and distribution. */
9 /* A set of functions to search databases in various formats. An open
10 database is represented by a void * value which is returned from a lookup-
11 specific "open" function. These are now all held in individual modules in the
12 lookups subdirectory and the functions here form a generic interface.
14 Caching is used to improve performance. Open files are cached until a tidyup
15 function is called, and for each file the result of the last lookup is cached.
16 However, if too many files are opened, some of those that are not in use have
17 to be closed. Those open items that use real files are kept on a LRU chain to
20 All the data is held in permanent store so as to be independent of the stacking
21 pool that is reset from time to time. In fact, we use malloc'd store so that it
22 can be freed when the caches are tidied up. It isn't actually clear whether
23 this is a benefit or not, to be honest. */
28 /* Tree in which to cache open files until tidyup called. */
30 static tree_node *search_tree = NULL;
32 /* Two-way chain of open databases that use real files. This is maintained in
33 recently-used order for the purposes of closing the least recently used when
34 too many files are open. */
36 static tree_node *open_top = NULL;
37 static tree_node *open_bot = NULL;
39 /* Count of open databases that use real files */
41 static int open_filecount = 0;
43 /* Allow us to reset store used for lookups and lookup caching */
45 static rmark search_reset_point = NULL;
49 /*************************************************
50 * Validate a plain lookup type name *
51 *************************************************/
53 /* Only those names that are recognized and whose code is included in the
54 binary give an OK response. Use a binary chop search now that the list has got
58 name lookup type name - not necessarily zero terminated (e.g. dbm*)
59 len length of the name
61 Returns: +ve => valid lookup name; value is offset in lookup_list
62 -ve => invalid name; message in search_error_message.
66 search_findtype(const uschar *name, int len)
69 int top = lookup_list_count;
72 int mid = (top + bot)/2;
73 int c = Ustrncmp(name, lookup_list[mid]->name, len);
75 /* If c == 0 we have matched the incoming name with the start of the search
76 type name. However, some search types are substrings of others (e.g. nis and
77 nisplus) so we need to check that the lengths are the same. The length of the
78 type name cannot be shorter (else c would not be 0); if it is not equal it
79 must be longer, and in that case, the incoming name comes before the name we
80 are testing. By leaving c == 0 when the lengths are different, and doing a
81 > 0 test below, this all falls out correctly. */
83 if (c == 0 && Ustrlen(lookup_list[mid]->name) == len)
85 if (lookup_list[mid]->find != NULL) return mid;
86 search_error_message = string_sprintf("lookup type \"%.*s\" is not "
87 "available (not in the binary - check buildtime LOOKUP configuration)",
92 if (c > 0) bot = mid + 1; else top = mid;
95 search_error_message = string_sprintf("unknown lookup type \"%.*s\"",len,name);
101 /*************************************************
102 * Validate a full lookup type name *
103 *************************************************/
105 /* This function recognizes the "partial-" prefix and also terminating * and *@
109 name the full lookup type name
110 ptypeptr where to put the partial type
111 after subtraction of 1024 or 2048:
112 negative => no partial matching
113 non-negative => minimum number of non-wild components
114 ptypeaff where to put a pointer to the affix
115 the affix is within name if supplied therein
116 otherwise it's a literal string
117 afflen the length of the affix
118 starflags where to put the SEARCH_STAR and SEARCH_STARAT flags
119 opts where to put the options
121 Returns: +ve => valid lookup name; value is offset in lookup_list
122 -ve => invalid name; message in search_error_message.
126 search_findtype_partial(const uschar *name, int *ptypeptr, const uschar **ptypeaff,
127 int *afflen, int *starflags, const uschar ** opts)
131 const uschar *ss = name;
137 /* Check for a partial matching type. It must start with "partial", optionally
138 followed by a sequence of digits. If this is followed by "-", the affix is the
139 default "*." string. Otherwise we expect an affix in parentheses. Affixes are a
140 limited number of characters, not including parens. */
142 if (Ustrncmp(name, "partial", 7) == 0)
148 while (isdigit(*ss)) pv = pv*10 + *ss++ - '0';
150 else pv = 2; /* Default number of wild components */
155 while (ispunct(*ss) && *ss != ')') ss++;
156 if (*ss != ')') goto BAD_TYPE;
157 *afflen = ss++ - *ptypeaff;
159 else if (*ss++ == '-')
167 search_error_message = string_sprintf("format error in lookup type \"%s\"",
173 /* Now we are left with a lookup name, possibly followed by * or *@,
174 and then by options starting with a "," */
177 if ((t = Ustrchr(ss, '*')))
180 *starflags |= (t[1] == '@' ? SEARCH_STARAT : SEARCH_STAR);
185 if ((t = Ustrchr(t, ',')))
188 if (l < len) len = l;
189 *opts = string_copy(t+1);
194 /* Check for the individual search type. Only those that are actually in the
195 binary are valid. For query-style types, "partial" and default types are
198 stype = search_findtype(ss, len);
199 if (stype >= 0 && mac_islookup(stype, lookup_querystyle))
203 search_error_message = string_sprintf("\"partial\" is not permitted "
204 "for lookup type \"%s\"", ss);
207 if ((*starflags & (SEARCH_STAR|SEARCH_STARAT)) != 0)
209 search_error_message = string_sprintf("defaults using \"*\" or \"*@\" are "
210 "not permitted for lookup type \"%s\"", ss);
221 /*************************************************
222 * Release cached resources *
223 *************************************************/
225 /* When search_open is called it caches the "file" that it opens in
226 search_tree. The name of the tree node is a concatenation of the search type
227 with the file name. For query-style lookups, the file name is empty. Real files
228 are normally closed only when this tidyup routine is called, typically at the
229 end of sections of code where a number of lookups might occur. However, if too
230 many files are open simultaneously, some get closed beforehand. They can't be
231 removed from the tree. There is also a general tidyup function which is called
232 for the lookup driver, if it exists.
234 First, there is an internal, recursive subroutine.
236 Argument: a pointer to a search_openfile tree node
241 tidyup_subtree(tree_node *t)
243 search_cache * c = (search_cache *)(t->data.ptr);
244 if (t->left) tidyup_subtree(t->left);
245 if (t->right) tidyup_subtree(t->right);
246 if (c && c->handle && lookup_list[c->search_type]->close)
247 lookup_list[c->search_type]->close(c->handle);
251 /* The external entry point
260 int old_pool = store_pool;
262 DEBUG(D_lookup) debug_printf_indent("search_tidyup called\n");
264 /* Close individually each cached open file. */
266 store_pool = POOL_SEARCH;
269 tidyup_subtree(search_tree);
272 open_top = open_bot = NULL;
275 /* Call the general tidyup entry for any drivers that have one. */
277 for (int i = 0; i < lookup_list_count; i++) if (lookup_list[i]->tidy)
278 (lookup_list[i]->tidy)();
280 if (search_reset_point) search_reset_point = store_reset(search_reset_point);
281 store_pool = old_pool;
287 /*************************************************
288 * Open search database *
289 *************************************************/
291 /* A mode, and lists of owners and groups, are passed over for checking in
292 the cases where the database is one or more files. Return NULL, with a message
293 pointed to by message, in cases of error.
295 For search types that use a file or files, check up on the mode after
296 opening. It is tempting to do a stat before opening the file, and use it as
297 an existence check. However, doing that opens a small security loophole in
298 that the status could be changed before the file is opened. Can't quite see
299 what problems this might lead to, but you can't be too careful where security
300 is concerned. Fstat() on an open file can normally be expected to succeed,
301 but there are some NFS states where it does not.
303 There are two styles of query: (1) in the "single-key+file" style, a single
304 key string and a file name are given, for example, for linear searches, DBM
305 files, or for NIS. (2) In the "query" style, no "filename" is given; instead
306 just a single query string is passed. This applies to multiple-key lookup
309 Before opening, scan the tree of cached files to see if this file is already
310 open for the correct search type. If so, return the saved handle. If not, put
311 the handle in the tree for possible subsequent use. See search_tidyup above for
312 closing all the cached files.
314 A count of open databases which use real files is maintained, and if this
315 gets too large, we have to close a cached file. Its entry remains in the tree,
316 but is marked closed.
319 filename the name of the file for single-key+file style lookups,
320 NULL for query-style lookups
321 search_type the type of search required
322 modemask if a real single file is used, this specifies mode bits that
323 must not be set; otherwise it is ignored
324 owners if a real single file is used, this specifies the possible
325 owners of the file; otherwise it is ignored
326 owngroups if a real single file is used, this specifies the possible
327 group owners of the file; otherwise it is ignored
329 Returns: an identifying handle for the open database;
330 this is the pointer to the tree block in the
331 cache of open files; return NULL on open failure, with
332 a message in search_error_message
336 search_open(const uschar * filename, int search_type, int modemask,
337 uid_t * owners, gid_t * owngroups)
342 lookup_info *lk = lookup_list[search_type];
343 uschar keybuffer[256];
344 int old_pool = store_pool;
346 if (filename && is_tainted2(filename, LOG_MAIN|LOG_PANIC, "Tainted filename for search '%s'", filename))
349 /* Change to the search store pool and remember our reset point */
351 store_pool = POOL_SEARCH;
352 if (!search_reset_point) search_reset_point = store_mark();
354 DEBUG(D_lookup) debug_printf_indent("search_open: %s \"%s\"\n", lk->name,
355 filename ? filename : US"NULL");
357 /* See if we already have this open for this type of search, and if so,
358 pass back the tree block as the handle. The key for the tree node is the search
359 type plus '0' concatenated with the file name. There may be entries in the tree
360 with closed files if a lot of files have been opened. */
362 sprintf(CS keybuffer, "%c%.254s", search_type + '0',
363 filename ? filename : US"");
365 if ((t = tree_search(search_tree, keybuffer)))
367 if ((c = (search_cache *)t->data.ptr)->handle)
369 DEBUG(D_lookup) debug_printf_indent(" cached open\n");
370 store_pool = old_pool;
373 DEBUG(D_lookup) debug_printf_indent(" cached closed\n");
376 /* Otherwise, we need to open the file or database - each search type has its
377 own code, which is now split off into separately compiled modules. Before doing
378 this, if the search type is one that uses real files, check on the number that
379 we are holding open in the cache. If the limit is reached, close the least
380 recently used one. */
382 if (lk->type == lookup_absfile && open_filecount >= lookup_open_max)
384 log_write(0, LOG_MAIN|LOG_PANIC, "too many lookups open, but can't find "
388 search_cache *c = (search_cache *)(open_bot->data.ptr);
389 DEBUG(D_lookup) debug_printf_indent("Too many lookup files open\n closing %s\n",
391 if ((open_bot = c->up))
392 ((search_cache *)(open_bot->data.ptr))->down = NULL;
395 ((lookup_list[c->search_type])->close)(c->handle);
400 /* If opening is successful, call the file-checking function if there is one,
401 and if all is still well, enter the open database into the tree. */
403 if (!(handle = (lk->open)(filename, &search_error_message)))
405 store_pool = old_pool;
410 && !lk->check(handle, filename, modemask, owners, owngroups,
411 &search_error_message))
414 store_pool = old_pool;
418 /* If this is a search type that uses real files, keep count. */
420 if (lk->type == lookup_absfile) open_filecount++;
422 /* If we found a previously opened entry in the tree, re-use it; otherwise
423 insert a new entry. On re-use, leave any cached lookup data and the lookup
428 t = store_get(sizeof(tree_node) + Ustrlen(keybuffer), FALSE);
429 t->data.ptr = c = store_get(sizeof(search_cache), FALSE);
430 c->item_cache = NULL;
431 Ustrcpy(t->name, keybuffer);
432 tree_insertnode(&search_tree, t);
434 else c = t->data.ptr;
437 c->search_type = search_type;
438 c->up = c->down = NULL;
440 store_pool = old_pool;
448 /*************************************************
449 * Internal function: Find one item in database *
450 *************************************************/
452 /* The answer is always put into dynamic store. The last lookup for each handle
456 handle the handle from search_open; points to tree node
457 filename the filename that was handed to search_open, or
458 NULL for query-style searches
459 keystring the keystring for single-key+file lookups, or
460 the querystring for query-style lookups
461 opts type-specific options
463 Returns: a pointer to a dynamic string containing the answer,
464 or NULL if the query failed or was deferred; in the
465 latter case, search_find_defer is set TRUE; after an unusual
466 failure, there may be a message in search_error_message.
470 internal_search_find(void * handle, const uschar * filename, uschar * keystring,
473 tree_node * t = (tree_node *)handle;
474 search_cache * c = (search_cache *)(t->data.ptr);
475 expiring_data * e = NULL; /* compiler quietening */
476 uschar * data = NULL;
477 int search_type = t->name[0] - '0';
478 int old_pool = store_pool;
480 /* Lookups that return DEFER may not always set an error message. So that
481 the callers don't have to test for NULL, set an empty string. */
483 search_error_message = US"";
484 f.search_find_defer = FALSE;
486 DEBUG(D_lookup) debug_printf_indent("internal_search_find: file=\"%s\"\n "
487 "type=%s key=\"%s\" opts=%s%s%s\n", filename,
488 lookup_list[search_type]->name, keystring,
489 opts ? "\"" : "", opts, opts ? "\"" : "");
491 /* Insurance. If the keystring is empty, just fail. */
493 if (keystring[0] == 0) return NULL;
495 /* Use the special store pool for search data */
497 store_pool = POOL_SEARCH;
499 /* Look up the data for the key, unless it is already in the cache for this
500 file. No need to check c->item_cache for NULL, tree_search will do so. */
502 if ( (t = tree_search(c->item_cache, keystring))
503 && (!(e = t->data.ptr)->expiry || e->expiry > time(NULL))
504 && (!opts && !e->opts || opts && e->opts && Ustrcmp(opts, e->opts) == 0)
506 { /* Data was in the cache already; set the pointer from the tree node */
508 DEBUG(D_lookup) debug_printf_indent("cached data used for lookup of %s%s%s\n",
510 filename ? US"\n in " : US"", filename ? filename : US"");
514 uint do_cache = UINT_MAX;
515 int keylength = Ustrlen(keystring);
520 debug_printf_indent("cached data found but %s; ",
521 e->expiry && e->expiry <= time(NULL) ? "out-of-date" : "wrong opts");
522 debug_printf_indent("%s lookup required for %s%s%s\n",
523 filename ? US"file" : US"database",
525 filename ? US"\n in " : US"", filename ? filename : US"");
528 /* Call the code for the different kinds of search. DEFER is handled
529 like FAIL, except that search_find_defer is set so the caller can
530 distinguish if necessary. */
532 if (lookup_list[search_type]->find(c->handle, filename, keystring, keylength,
533 &data, &search_error_message, &do_cache, opts) == DEFER)
534 f.search_find_defer = TRUE;
536 /* A record that has been found is now in data, which is either NULL
537 or points to a bit of dynamic store. Cache the result of the lookup if
538 caching is permitted. Lookups can disable caching, when they did something
539 that changes their data. The mysql and pgsql lookups do this when an
540 UPDATE/INSERT query was executed. */
544 if (!t) /* No existing entry. Create new one. */
546 int len = keylength + 1;
547 e = store_get(sizeof(expiring_data) + sizeof(tree_node) + len, is_tainted(keystring));
548 t = (tree_node *)(e+1);
549 memcpy(t->name, keystring, len);
551 tree_insertnode(&c->item_cache, t);
553 /* Else previous, out-of-date cache entry. Update with the */
554 /* new result and forget the old one */
555 e->expiry = do_cache == UINT_MAX ? 0 : time(NULL)+do_cache;
556 e->opts = opts ? string_copy(opts) : NULL;
560 /* If caching was disabled, empty the cache tree. We just set the cache
561 pointer to NULL here, because we cannot release the store at this stage. */
565 DEBUG(D_lookup) debug_printf_indent("lookup forced cache cleanup\n");
566 c->item_cache = NULL; /* forget all lookups on this connection */
573 debug_printf_indent("lookup yielded: %s\n", data);
574 else if (f.search_find_defer)
575 debug_printf_indent("lookup deferred: %s\n", search_error_message);
576 else debug_printf_indent("lookup failed\n");
579 /* Return it in new dynamic store in the regular pool */
581 store_pool = old_pool;
582 return data ? string_copy(data) : NULL;
588 /*************************************************
589 * Find one item in database, possibly wildcarded *
590 *************************************************/
592 /* This function calls the internal function above; once only if there
593 is no partial matching, but repeatedly when partial matching is requested.
596 handle the handle from search_open
597 filename the filename that was handed to search_open, or
598 NULL for query-style searches
599 keystring the keystring for single-key+file lookups, or
600 the querystring for query-style lookups
601 partial -1 means no partial matching;
602 otherwise it's the minimum number of components;
603 affix the affix string for partial matching
604 affixlen the length of the affix string
605 starflags SEARCH_STAR and SEARCH_STARAT flags
606 expand_setup pointer to offset for setting up expansion strings;
608 opts type-specific options
610 Returns: a pointer to a dynamic string containing the answer,
611 or NULL if the query failed or was deferred; in the
612 latter case, search_find_defer is set TRUE
616 search_find(void * handle, const uschar * filename, uschar * keystring,
617 int partial, const uschar * affix, int affixlen, int starflags,
618 int * expand_setup, const uschar * opts)
620 tree_node *t = (tree_node *)handle;
621 BOOL set_null_wild = FALSE;
626 if (partial < 0) affixlen = 99; /* So that "NULL" prints */
627 debug_printf_indent("search_find: file=\"%s\"\n key=\"%s\" "
628 "partial=%d affix=%.*s starflags=%x opts=%s%s%s\n",
629 filename ? filename : US"NULL",
630 keystring, partial, affixlen, affix, starflags,
631 opts ? "\"" : "", opts, opts ? "\"" : "");
635 /* Arrange to put this database at the top of the LRU chain if it is a type
636 that opens real files. */
638 if ( open_top != (tree_node *)handle
639 && lookup_list[t->name[0]-'0']->type == lookup_absfile)
641 search_cache *c = (search_cache *)(t->data.ptr);
642 tree_node *up = c->up;
643 tree_node *down = c->down;
645 /* Cut it out of the list. A newly opened file will a NULL up pointer.
646 Otherwise there will be a non-NULL up pointer, since we checked above that
647 this block isn't already at the top of the list. */
651 ((search_cache *)(up->data.ptr))->down = down;
653 ((search_cache *)(down->data.ptr))->up = up;
658 /* Now put it at the head of the list. */
662 if (!open_top) open_bot = t;
663 else ((search_cache *)(open_top->data.ptr))->up = t;
669 debug_printf_indent("LRU list:\n");
670 for (tree_node *t = open_top; t; )
672 search_cache *c = (search_cache *)(t->data.ptr);
673 debug_printf_indent(" %s\n", t->name);
674 if (t == open_bot) debug_printf_indent(" End\n");
679 /* First of all, try to match the key string verbatim. If matched a complete
680 entry but could have been partial, flag to set up variables. */
682 yield = internal_search_find(handle, filename, keystring, opts);
683 if (f.search_find_defer) return NULL;
685 if (yield) { if (partial >= 0) set_null_wild = TRUE; }
687 /* Not matched a complete entry; handle partial lookups, but only if the full
688 search didn't defer. Don't use string_sprintf() to construct the initial key,
689 just in case the original key is too long for the string_sprintf() buffer (it
690 *has* happened!). The case of a zero-length affix has to be treated specially.
693 else if (partial >= 0)
695 int len = Ustrlen(keystring);
698 /* Try with the affix on the front, except for a zero-length affix */
700 if (affixlen == 0) keystring2 = keystring; else
702 keystring2 = store_get(len + affixlen + 1,
703 is_tainted(keystring) || is_tainted(affix));
704 Ustrncpy(keystring2, affix, affixlen);
705 Ustrcpy(keystring2 + affixlen, keystring);
706 DEBUG(D_lookup) debug_printf_indent("trying partial match %s\n", keystring2);
707 yield = internal_search_find(handle, filename, keystring2, opts);
708 if (f.search_find_defer) return NULL;
711 /* The key in its entirety did not match a wild entry; try chopping off
712 leading components. */
717 uschar *keystring3 = keystring2 + affixlen;
718 uschar *s = keystring3;
719 while (*s != 0) if (*s++ == '.') dotcount++;
721 while (dotcount-- >= partial)
723 while (*keystring3 != 0 && *keystring3 != '.') keystring3++;
725 /* If we get right to the end of the string (which will be the last time
726 through this loop), we've failed if the affix is null. Otherwise do one
727 last lookup for the affix itself, but if it is longer than 1 character,
728 remove the last character if it is ".". */
730 if (*keystring3 == 0)
732 if (affixlen < 1) break;
733 if (affixlen > 1 && affix[affixlen-1] == '.') affixlen--;
734 Ustrncpy(keystring2, affix, affixlen);
735 keystring2[affixlen] = 0;
736 keystring3 = keystring2;
740 keystring3 -= affixlen - 1;
741 if (affixlen > 0) Ustrncpy(keystring3, affix, affixlen);
744 DEBUG(D_lookup) debug_printf_indent("trying partial match %s\n", keystring3);
745 yield = internal_search_find(handle, filename, keystring3, opts);
746 if (f.search_find_defer) return NULL;
749 /* First variable is the wild part; second is the fixed part. Take care
750 to get it right when keystring3 is just "*". */
752 if (expand_setup && *expand_setup >= 0)
754 int fixedlength = Ustrlen(keystring3) - affixlen;
755 int wildlength = Ustrlen(keystring) - fixedlength - 1;
757 expand_nstring[*expand_setup] = keystring;
758 expand_nlength[*expand_setup] = wildlength;
760 expand_nstring[*expand_setup] = keystring + wildlength + 1;
761 expand_nlength[*expand_setup] = (fixedlength < 0)? 0 : fixedlength;
765 keystring3 += affixlen;
769 else set_null_wild = TRUE; /* Matched a wild entry without any wild part */
772 /* If nothing has been matched, but the option to look for "*@" is set, try
773 replacing everything to the left of @ by *. After a match, the wild part
774 is set to the string to the left of the @. */
776 if (!yield && starflags & SEARCH_STARAT)
778 uschar *atat = Ustrrchr(keystring, '@');
779 if (atat != NULL && atat > keystring)
782 savechar = *(--atat);
785 DEBUG(D_lookup) debug_printf_indent("trying default match %s\n", atat);
786 yield = internal_search_find(handle, filename, atat, opts);
788 if (f.search_find_defer) return NULL;
790 if (yield && expand_setup && *expand_setup >= 0)
793 expand_nstring[*expand_setup] = keystring;
794 expand_nlength[*expand_setup] = atat - keystring + 1;
796 expand_nstring[*expand_setup] = keystring;
797 expand_nlength[*expand_setup] = 0;
802 /* If we still haven't matched anything, and the option to look for "*" is set,
803 try that. If we do match, the first variable (the wild part) is the whole key,
804 and the second is empty. */
806 if (!yield && starflags & (SEARCH_STAR|SEARCH_STARAT))
808 DEBUG(D_lookup) debug_printf_indent("trying to match *\n");
809 yield = internal_search_find(handle, filename, US"*", opts);
810 if (yield && expand_setup && *expand_setup >= 0)
813 expand_nstring[*expand_setup] = keystring;
814 expand_nlength[*expand_setup] = Ustrlen(keystring);
816 expand_nstring[*expand_setup] = keystring;
817 expand_nlength[*expand_setup] = 0;
821 /* If this was a potentially partial lookup, and we matched either a
822 complete non-wild domain entry, or we matched a wild-carded entry without
823 chopping off any of the domain components, set up the expansion variables
824 (if required) so that the first one is empty, and the second one is the
825 fixed part of the domain. The set_null_wild flag is set only when yield is not
828 if (set_null_wild && expand_setup && *expand_setup >= 0)
831 expand_nstring[*expand_setup] = keystring;
832 expand_nlength[*expand_setup] = 0;
834 expand_nstring[*expand_setup] = keystring;
835 expand_nlength[*expand_setup] = Ustrlen(keystring);
838 /* If we have a result, check the options to see if the key was wanted rather
839 than the result. Return a de-tainted version of the key on the grounds that
840 it have been validated by the lookup. */
845 for (uschar * ele; ele = string_nextinlist(&opts, &sep, NULL, 0); )
846 if (Ustrcmp(ele, "ret=key") == 0)
848 DEBUG(D_lookup) debug_printf_indent("lookup ret=key: %s\n", keystring);
849 yield = string_copy_taint(keystring, FALSE);
857 /* End of search.c */