1 /*************************************************
2 * Exim - an Internet mail transport agent *
3 *************************************************/
5 /* Copyright (c) University of Cambridge 1995 - 2015 */
6 /* See the file NOTICE for conditions of use and distribution. */
8 /* A set of functions to search databases in various formats. An open
9 database is represented by a void * value which is returned from a lookup-
10 specific "open" function. These are now all held in individual modules in the
11 lookups subdirectory and the functions here form a generic interface.
13 Caching is used to improve performance. Open files are cached until a tidyup
14 function is called, and for each file the result of the last lookup is cached.
15 However, if too many files are opened, some of those that are not in use have
16 to be closed. Those open items that use real files are kept on a LRU chain to
19 All the data is held in permanent store so as to be independent of the stacking
20 pool that is reset from time to time. In fact, we use malloc'd store so that it
21 can be freed when the caches are tidied up. It isn't actually clear whether
22 this is a benefit or not, to be honest. */
27 /* Tree in which to cache open files until tidyup called. */
29 static tree_node *search_tree = NULL;
31 /* Two-way chain of open databases that use real files. This is maintained in
32 recently-used order for the purposes of closing the least recently used when
33 too many files are open. */
35 static tree_node *open_top = NULL;
36 static tree_node *open_bot = NULL;
38 /* Count of open databases that use real files */
40 static int open_filecount = 0;
42 /* Allow us to reset store used for lookups and lookup caching */
44 static rmark search_reset_point = NULL;
48 /*************************************************
49 * Validate a plain lookup type name *
50 *************************************************/
52 /* Only those names that are recognized and whose code is included in the
53 binary give an OK response. Use a binary chop search now that the list has got
57 name lookup type name - not necessarily zero terminated (e.g. dbm*)
58 len length of the name
60 Returns: +ve => valid lookup name; value is offset in lookup_list
61 -ve => invalid name; message in search_error_message.
65 search_findtype(const uschar *name, int len)
68 int top = lookup_list_count;
71 int mid = (top + bot)/2;
72 int c = Ustrncmp(name, lookup_list[mid]->name, len);
74 /* If c == 0 we have matched the incoming name with the start of the search
75 type name. However, some search types are substrings of others (e.g. nis and
76 nisplus) so we need to check that the lengths are the same. The length of the
77 type name cannot be shorter (else c would not be 0); if it is not equal it
78 must be longer, and in that case, the incoming name comes before the name we
79 are testing. By leaving c == 0 when the lengths are different, and doing a
80 > 0 test below, this all falls out correctly. */
82 if (c == 0 && Ustrlen(lookup_list[mid]->name) == len)
84 if (lookup_list[mid]->find != NULL) return mid;
85 search_error_message = string_sprintf("lookup type \"%.*s\" is not "
86 "available (not in the binary - check buildtime LOOKUP configuration)",
91 if (c > 0) bot = mid + 1; else top = mid;
94 search_error_message = string_sprintf("unknown lookup type \"%.*s\"",len,name);
100 /*************************************************
101 * Validate a full lookup type name *
102 *************************************************/
104 /* This function recognizes the "partial-" prefix and also terminating * and *@
108 name the full lookup type name
109 ptypeptr where to put the partial type
110 after subtraction of 1024 or 2048:
111 negative => no partial matching
112 non-negative => minimum number of non-wild components
113 ptypeaff where to put a pointer to the affix
114 the affix is within name if supplied therein
115 otherwise it's a literal string
116 afflen the length of the affix
117 starflags where to put the SEARCH_STAR and SEARCH_STARAT flags
119 Returns: +ve => valid lookup name; value is offset in lookup_list
120 -ve => invalid name; message in search_error_message.
124 search_findtype_partial(const uschar *name, int *ptypeptr, const uschar **ptypeaff,
125 int *afflen, int *starflags)
129 const uschar *ss = name;
134 /* Check for a partial matching type. It must start with "partial", optionally
135 followed by a sequence of digits. If this is followed by "-", the affix is the
136 default "*." string. Otherwise we expect an affix in parentheses. Affixes are a
137 limited number of characters, not including parens. */
139 if (Ustrncmp(name, "partial", 7) == 0)
145 while (isdigit(*ss)) pv = pv*10 + *ss++ - '0';
147 else pv = 2; /* Default number of wild components */
152 while (ispunct(*ss) && *ss != ')') ss++;
153 if (*ss != ')') goto BAD_TYPE;
154 *afflen = ss++ - *ptypeaff;
156 else if (*ss++ == '-')
164 search_error_message = string_sprintf("format error in lookup type \"%s\"",
170 /* Now we are left with a lookup name, possibly followed by * or *@. */
173 if (len >= 2 && Ustrncmp(ss + len - 2, "*@", 2) == 0)
175 *starflags |= SEARCH_STARAT;
178 else if (len >= 1 && ss[len-1] == '*')
180 *starflags |= SEARCH_STAR;
184 /* Check for the individual search type. Only those that are actually in the
185 binary are valid. For query-style types, "partial" and default types are
188 stype = search_findtype(ss, len);
189 if (stype >= 0 && mac_islookup(stype, lookup_querystyle))
193 search_error_message = string_sprintf("\"partial\" is not permitted "
194 "for lookup type \"%s\"", ss);
197 if ((*starflags & (SEARCH_STAR|SEARCH_STARAT)) != 0)
199 search_error_message = string_sprintf("defaults using \"*\" or \"*@\" are "
200 "not permitted for lookup type \"%s\"", ss);
211 /*************************************************
212 * Release cached resources *
213 *************************************************/
215 /* When search_open is called it caches the "file" that it opens in
216 search_tree. The name of the tree node is a concatenation of the search type
217 with the file name. For query-style lookups, the file name is empty. Real files
218 are normally closed only when this tidyup routine is called, typically at the
219 end of sections of code where a number of lookups might occur. However, if too
220 many files are open simultaneously, some get closed beforehand. They can't be
221 removed from the tree. There is also a general tidyup function which is called
222 for the lookup driver, if it exists.
224 First, there is an internal, recursive subroutine.
226 Argument: a pointer to a search_openfile tree node
231 tidyup_subtree(tree_node *t)
233 search_cache *c = (search_cache *)(t->data.ptr);
234 if (t->left != NULL) tidyup_subtree(t->left);
235 if (t->right != NULL) tidyup_subtree(t->right);
238 lookup_list[c->search_type]->close != NULL)
239 lookup_list[c->search_type]->close(c->handle);
243 /* The external entry point
252 int old_pool = store_pool;
254 DEBUG(D_lookup) debug_printf_indent("search_tidyup called\n");
256 /* Close individually each cached open file. */
258 store_pool = POOL_SEARCH;
261 tidyup_subtree(search_tree);
264 open_top = open_bot = NULL;
267 /* Call the general tidyup entry for any drivers that have one. */
269 for (int i = 0; i < lookup_list_count; i++) if (lookup_list[i]->tidy)
270 (lookup_list[i]->tidy)();
272 if (search_reset_point) search_reset_point = store_reset(search_reset_point);
273 store_pool = old_pool;
279 /*************************************************
280 * Open search database *
281 *************************************************/
283 /* A mode, and lists of owners and groups, are passed over for checking in
284 the cases where the database is one or more files. Return NULL, with a message
285 pointed to by message, in cases of error.
287 For search types that use a file or files, check up on the mode after
288 opening. It is tempting to do a stat before opening the file, and use it as
289 an existence check. However, doing that opens a small security loophole in
290 that the status could be changed before the file is opened. Can't quite see
291 what problems this might lead to, but you can't be too careful where security
292 is concerned. Fstat() on an open file can normally be expected to succeed,
293 but there are some NFS states where it does not.
295 There are two styles of query: (1) in the "single-key+file" style, a single
296 key string and a file name are given, for example, for linear searches, DBM
297 files, or for NIS. (2) In the "query" style, no "filename" is given; instead
298 just a single query string is passed. This applies to multiple-key lookup
301 Before opening, scan the tree of cached files to see if this file is already
302 open for the correct search type. If so, return the saved handle. If not, put
303 the handle in the tree for possible subsequent use. See search_tidyup above for
304 closing all the cached files.
306 A count of open databases which use real files is maintained, and if this
307 gets too large, we have to close a cached file. Its entry remains in the tree,
308 but is marked closed.
311 filename the name of the file for single-key+file style lookups,
312 NULL for query-style lookups
313 search_type the type of search required
314 modemask if a real single file is used, this specifies mode bits that
315 must not be set; otherwise it is ignored
316 owners if a real single file is used, this specifies the possible
317 owners of the file; otherwise it is ignored
318 owngroups if a real single file is used, this specifies the possible
319 group owners of the file; otherwise it is ignored
321 Returns: an identifying handle for the open database;
322 this is the pointer to the tree block in the
323 cache of open files; return NULL on open failure, with
324 a message in search_error_message
328 search_open(const uschar * filename, int search_type, int modemask,
329 uid_t * owners, gid_t * owngroups)
334 lookup_info *lk = lookup_list[search_type];
335 uschar keybuffer[256];
336 int old_pool = store_pool;
338 if (filename && is_tainted(filename))
340 log_write(0, LOG_MAIN|LOG_PANIC,
341 "Tainted filename for search: '%s'", filename);
345 /* Change to the search store pool and remember our reset point */
347 store_pool = POOL_SEARCH;
348 if (!search_reset_point) search_reset_point = store_mark();
350 DEBUG(D_lookup) debug_printf_indent("search_open: %s \"%s\"\n", lk->name,
351 filename ? filename : US"NULL");
353 /* See if we already have this open for this type of search, and if so,
354 pass back the tree block as the handle. The key for the tree node is the search
355 type plus '0' concatenated with the file name. There may be entries in the tree
356 with closed files if a lot of files have been opened. */
358 sprintf(CS keybuffer, "%c%.254s", search_type + '0',
359 filename ? filename : US"");
361 if ((t = tree_search(search_tree, keybuffer)))
363 if ((c = (search_cache *)t->data.ptr)->handle)
365 DEBUG(D_lookup) debug_printf_indent(" cached open\n");
366 store_pool = old_pool;
369 DEBUG(D_lookup) debug_printf_indent(" cached closed\n");
372 /* Otherwise, we need to open the file or database - each search type has its
373 own code, which is now split off into separately compiled modules. Before doing
374 this, if the search type is one that uses real files, check on the number that
375 we are holding open in the cache. If the limit is reached, close the least
376 recently used one. */
378 if (lk->type == lookup_absfile && open_filecount >= lookup_open_max)
380 log_write(0, LOG_MAIN|LOG_PANIC, "too many lookups open, but can't find "
384 search_cache *c = (search_cache *)(open_bot->data.ptr);
385 DEBUG(D_lookup) debug_printf_indent("Too many lookup files open\n closing %s\n",
387 if ((open_bot = c->up))
388 ((search_cache *)(open_bot->data.ptr))->down = NULL;
391 ((lookup_list[c->search_type])->close)(c->handle);
396 /* If opening is successful, call the file-checking function if there is one,
397 and if all is still well, enter the open database into the tree. */
399 if (!(handle = (lk->open)(filename, &search_error_message)))
401 store_pool = old_pool;
406 && !lk->check(handle, filename, modemask, owners, owngroups,
407 &search_error_message))
410 store_pool = old_pool;
414 /* If this is a search type that uses real files, keep count. */
416 if (lk->type == lookup_absfile) open_filecount++;
418 /* If we found a previously opened entry in the tree, re-use it; otherwise
419 insert a new entry. On re-use, leave any cached lookup data and the lookup
424 t = store_get(sizeof(tree_node) + Ustrlen(keybuffer), FALSE);
425 t->data.ptr = c = store_get(sizeof(search_cache), FALSE);
426 c->item_cache = NULL;
427 Ustrcpy(t->name, keybuffer);
428 tree_insertnode(&search_tree, t);
430 else c = t->data.ptr;
433 c->search_type = search_type;
434 c->up = c->down = NULL;
436 store_pool = old_pool;
444 /*************************************************
445 * Internal function: Find one item in database *
446 *************************************************/
448 /* The answer is always put into dynamic store. The last lookup for each handle
452 handle the handle from search_open; points to tree node
453 filename the filename that was handed to search_open, or
454 NULL for query-style searches
455 keystring the keystring for single-key+file lookups, or
456 the querystring for query-style lookups
458 Returns: a pointer to a dynamic string containing the answer,
459 or NULL if the query failed or was deferred; in the
460 latter case, search_find_defer is set TRUE; after an unusual
461 failure, there may be a message in search_error_message.
465 internal_search_find(void * handle, const uschar * filename, uschar * keystring)
467 tree_node * t = (tree_node *)handle;
468 search_cache * c = (search_cache *)(t->data.ptr);
469 expiring_data * e = NULL; /* compiler quietening */
470 uschar * data = NULL;
471 int search_type = t->name[0] - '0';
472 int old_pool = store_pool;
474 /* Lookups that return DEFER may not always set an error message. So that
475 the callers don't have to test for NULL, set an empty string. */
477 search_error_message = US"";
478 f.search_find_defer = FALSE;
480 DEBUG(D_lookup) debug_printf_indent("internal_search_find: file=\"%s\"\n "
481 "type=%s key=\"%s\"\n", filename,
482 lookup_list[search_type]->name, keystring);
484 /* Insurance. If the keystring is empty, just fail. */
486 if (keystring[0] == 0) return NULL;
488 /* Use the special store pool for search data */
490 store_pool = POOL_SEARCH;
492 /* Look up the data for the key, unless it is already in the cache for this
493 file. No need to check c->item_cache for NULL, tree_search will do so. */
495 if ( (t = tree_search(c->item_cache, keystring))
496 && (!(e = t->data.ptr)->expiry || e->expiry > time(NULL))
498 { /* Data was in the cache already; set the pointer from the tree node */
500 DEBUG(D_lookup) debug_printf_indent("cached data used for lookup of %s%s%s\n",
502 filename ? US"\n in " : US"", filename ? filename : US"");
506 uint do_cache = UINT_MAX;
507 int keylength = Ustrlen(keystring);
511 if (t) debug_printf_indent("cached data found but past valid time; ");
512 debug_printf_indent("%s lookup required for %s%s%s\n",
513 filename ? US"file" : US"database",
515 filename ? US"\n in " : US"", filename ? filename : US"");
518 /* Call the code for the different kinds of search. DEFER is handled
519 like FAIL, except that search_find_defer is set so the caller can
520 distinguish if necessary. */
522 if (lookup_list[search_type]->find(c->handle, filename, keystring, keylength,
523 &data, &search_error_message, &do_cache) == DEFER)
524 f.search_find_defer = TRUE;
526 /* A record that has been found is now in data, which is either NULL
527 or points to a bit of dynamic store. Cache the result of the lookup if
528 caching is permitted. Lookups can disable caching, when they did something
529 that changes their data. The mysql and pgsql lookups do this when an
530 UPDATE/INSERT query was executed. */
534 int len = keylength + 1;
536 if (t) /* Previous, out-of-date cache entry. Update with the */
537 { /* new result and forget the old one */
538 e->expiry = do_cache == UINT_MAX ? 0 : time(NULL)+do_cache;
543 e = store_get(sizeof(expiring_data) + sizeof(tree_node) + len, is_tainted(keystring));
544 e->expiry = do_cache == UINT_MAX ? 0 : time(NULL)+do_cache;
546 t = (tree_node *)(e+1);
547 memcpy(t->name, keystring, len);
549 tree_insertnode(&c->item_cache, t);
553 /* If caching was disabled, empty the cache tree. We just set the cache
554 pointer to NULL here, because we cannot release the store at this stage. */
558 DEBUG(D_lookup) debug_printf_indent("lookup forced cache cleanup\n");
559 c->item_cache = NULL;
566 debug_printf_indent("lookup yielded: %s\n", data);
567 else if (f.search_find_defer)
568 debug_printf_indent("lookup deferred: %s\n", search_error_message);
569 else debug_printf_indent("lookup failed\n");
572 /* Return it in new dynamic store in the regular pool */
574 store_pool = old_pool;
575 return data ? string_copy(data) : NULL;
581 /*************************************************
582 * Find one item in database, possibly wildcarded *
583 *************************************************/
585 /* This function calls the internal function above; once only if there
586 is no partial matching, but repeatedly when partial matching is requested.
589 handle the handle from search_open
590 filename the filename that was handed to search_open, or
591 NULL for query-style searches
592 keystring the keystring for single-key+file lookups, or
593 the querystring for query-style lookups
594 partial -1 means no partial matching;
595 otherwise it's the minimum number of components;
596 affix the affix string for partial matching
597 affixlen the length of the affix string
598 starflags SEARCH_STAR and SEARCH_STARAT flags
599 expand_setup pointer to offset for setting up expansion strings;
602 Returns: a pointer to a dynamic string containing the answer,
603 or NULL if the query failed or was deferred; in the
604 latter case, search_find_defer is set TRUE
608 search_find(void * handle, const uschar * filename, uschar * keystring,
609 int partial, const uschar * affix, int affixlen, int starflags,
612 tree_node *t = (tree_node *)handle;
613 BOOL set_null_wild = FALSE;
618 if (partial < 0) affixlen = 99; /* So that "NULL" prints */
619 debug_printf_indent("search_find: file=\"%s\"\n key=\"%s\" "
620 "partial=%d affix=%.*s starflags=%x\n",
621 (filename == NULL)? US"NULL" : filename,
622 keystring, partial, affixlen, affix, starflags);
625 /* Arrange to put this database at the top of the LRU chain if it is a type
626 that opens real files. */
628 if ( open_top != (tree_node *)handle
629 && lookup_list[t->name[0]-'0']->type == lookup_absfile)
631 search_cache *c = (search_cache *)(t->data.ptr);
632 tree_node *up = c->up;
633 tree_node *down = c->down;
635 /* Cut it out of the list. A newly opened file will a NULL up pointer.
636 Otherwise there will be a non-NULL up pointer, since we checked above that
637 this block isn't already at the top of the list. */
641 ((search_cache *)(up->data.ptr))->down = down;
643 ((search_cache *)(down->data.ptr))->up = up;
648 /* Now put it at the head of the list. */
652 if (!open_top) open_bot = t;
653 else ((search_cache *)(open_top->data.ptr))->up = t;
659 debug_printf_indent("LRU list:\n");
660 for (tree_node *t = open_top; t; )
662 search_cache *c = (search_cache *)(t->data.ptr);
663 debug_printf_indent(" %s\n", t->name);
664 if (t == open_bot) debug_printf_indent(" End\n");
669 /* First of all, try to match the key string verbatim. If matched a complete
670 entry but could have been partial, flag to set up variables. */
672 yield = internal_search_find(handle, filename, keystring);
673 if (f.search_find_defer) return NULL;
675 if (yield) { if (partial >= 0) set_null_wild = TRUE; }
677 /* Not matched a complete entry; handle partial lookups, but only if the full
678 search didn't defer. Don't use string_sprintf() to construct the initial key,
679 just in case the original key is too long for the string_sprintf() buffer (it
680 *has* happened!). The case of a zero-length affix has to be treated specially.
683 else if (partial >= 0)
685 int len = Ustrlen(keystring);
688 /* Try with the affix on the front, except for a zero-length affix */
690 if (affixlen == 0) keystring2 = keystring; else
692 keystring2 = store_get(len + affixlen + 1,
693 is_tainted(keystring) || is_tainted(affix));
694 Ustrncpy(keystring2, affix, affixlen);
695 Ustrcpy(keystring2 + affixlen, keystring);
696 DEBUG(D_lookup) debug_printf_indent("trying partial match %s\n", keystring2);
697 yield = internal_search_find(handle, filename, keystring2);
698 if (f.search_find_defer) return NULL;
701 /* The key in its entirety did not match a wild entry; try chopping off
702 leading components. */
707 uschar *keystring3 = keystring2 + affixlen;
708 uschar *s = keystring3;
709 while (*s != 0) if (*s++ == '.') dotcount++;
711 while (dotcount-- >= partial)
713 while (*keystring3 != 0 && *keystring3 != '.') keystring3++;
715 /* If we get right to the end of the string (which will be the last time
716 through this loop), we've failed if the affix is null. Otherwise do one
717 last lookup for the affix itself, but if it is longer than 1 character,
718 remove the last character if it is ".". */
720 if (*keystring3 == 0)
722 if (affixlen < 1) break;
723 if (affixlen > 1 && affix[affixlen-1] == '.') affixlen--;
724 Ustrncpy(keystring2, affix, affixlen);
725 keystring2[affixlen] = 0;
726 keystring3 = keystring2;
730 keystring3 -= affixlen - 1;
731 if (affixlen > 0) Ustrncpy(keystring3, affix, affixlen);
734 DEBUG(D_lookup) debug_printf_indent("trying partial match %s\n", keystring3);
735 yield = internal_search_find(handle, filename, keystring3);
736 if (f.search_find_defer) return NULL;
739 /* First variable is the wild part; second is the fixed part. Take care
740 to get it right when keystring3 is just "*". */
742 if (expand_setup && *expand_setup >= 0)
744 int fixedlength = Ustrlen(keystring3) - affixlen;
745 int wildlength = Ustrlen(keystring) - fixedlength - 1;
747 expand_nstring[*expand_setup] = keystring;
748 expand_nlength[*expand_setup] = wildlength;
750 expand_nstring[*expand_setup] = keystring + wildlength + 1;
751 expand_nlength[*expand_setup] = (fixedlength < 0)? 0 : fixedlength;
755 keystring3 += affixlen;
759 else set_null_wild = TRUE; /* Matched a wild entry without any wild part */
762 /* If nothing has been matched, but the option to look for "*@" is set, try
763 replacing everything to the left of @ by *. After a match, the wild part
764 is set to the string to the left of the @. */
766 if (!yield && starflags & SEARCH_STARAT)
768 uschar *atat = Ustrrchr(keystring, '@');
769 if (atat != NULL && atat > keystring)
772 savechar = *(--atat);
775 DEBUG(D_lookup) debug_printf_indent("trying default match %s\n", atat);
776 yield = internal_search_find(handle, filename, atat);
778 if (f.search_find_defer) return NULL;
780 if (yield && expand_setup && *expand_setup >= 0)
783 expand_nstring[*expand_setup] = keystring;
784 expand_nlength[*expand_setup] = atat - keystring + 1;
786 expand_nstring[*expand_setup] = keystring;
787 expand_nlength[*expand_setup] = 0;
792 /* If we still haven't matched anything, and the option to look for "*" is set,
793 try that. If we do match, the first variable (the wild part) is the whole key,
794 and the second is empty. */
796 if (!yield && starflags & (SEARCH_STAR|SEARCH_STARAT))
798 DEBUG(D_lookup) debug_printf_indent("trying to match *\n");
799 yield = internal_search_find(handle, filename, US"*");
800 if (yield && expand_setup && *expand_setup >= 0)
803 expand_nstring[*expand_setup] = keystring;
804 expand_nlength[*expand_setup] = Ustrlen(keystring);
806 expand_nstring[*expand_setup] = keystring;
807 expand_nlength[*expand_setup] = 0;
811 /* If this was a potentially partial lookup, and we matched either a
812 complete non-wild domain entry, or we matched a wild-carded entry without
813 chopping off any of the domain components, set up the expansion variables
814 (if required) so that the first one is empty, and the second one is the
815 fixed part of the domain. The set_null_wild flag is set only when yield is not
818 if (set_null_wild && expand_setup && *expand_setup >= 0)
821 expand_nstring[*expand_setup] = keystring;
822 expand_nlength[*expand_setup] = 0;
824 expand_nstring[*expand_setup] = keystring;
825 expand_nlength[*expand_setup] = Ustrlen(keystring);
831 /* End of search.c */