1 /*************************************************
2 * Exim - an Internet mail transport agent *
3 *************************************************/
5 /* Copyright (c) The Exim Maintainers 2020 - 2024 */
6 /* Copyright (c) University of Cambridge 1995 - 2018 */
7 /* See the file NOTICE for conditions of use and distribution. */
8 /* SPDX-License-Identifier: GPL-2.0-or-later */
13 /* We have buffers holding path names for database files.
14 PATH_MAX could be used here, but would be wasting memory, as we deal
15 with database files like $spooldirectory/db/<name> */
19 /* Functions for accessing Exim's hints database, which consists of a number of
20 different DBM files. This module does not contain code for reading DBM files
21 for (e.g.) alias expansion. That is all contained within the general search
22 functions. As Exim now has support for several DBM interfaces, all the relevant
23 functions are called as inlinable functions from an included file.
25 All the data in Exim's database is in the nature of *hints*. Therefore it
26 doesn't matter if it gets destroyed by accident. These functions are not
27 supposed to implement a "safe" database.
29 Keys are passed in as C strings, and the terminating zero *is* used when
30 building the dbm files. This just makes life easier when scanning the files
33 Synchronization is required on the database files, and this is achieved by
34 means of locking on independent lock files. (Earlier attempts to lock on the
35 DBM files themselves were never completely successful.) Since callers may in
36 general want to do more than one read or write while holding the lock, there
37 are separate open and close functions. However, the calling modules should
38 arrange to hold the locks for the bare minimum of time.
41 exim_lockfile_needed facilities predicate
42 dbfn_open_path full pathname; no lock taken, readonly
43 dbfn_open takes lockfile or opens transaction
44 dbfn_open_multi only if transactions supported;
45 no lock or transaction taken
46 dbfn_close release lockfile or transaction
48 dbfn_transaction_start only if transactions supported
49 dbfn_transaction_commit
50 dbfn_read_klen explicit key length; embedded NUL ok
52 dbfn_read_enforce_length
55 dbfn_scan unused; ifdeffout out
58 ACL ratelimit & seen conditions
59 delivery retry handling
60 delivery serialization
61 TLS session resumption
69 /*************************************************
70 * Open and lock a database file *
71 *************************************************/
73 /* Used by DBM lookups:
74 full pathname for DB file rather than hintsdb name, readonly, no locking. */
77 dbfn_open_path(const uschar * path, open_db * dbblock)
79 uschar * dirname = string_copy(path);
81 dbblock->readonly = TRUE;
83 dbblock->dbptr = !exim_lockfile_needed()
84 ? exim_dbopen_multi(path, dirname, O_RDONLY, 0)
85 : exim_dbopen(path, dirname, O_RDONLY, 0);
86 return dbblock->dbptr ? dbblock : NULL;;
89 /* Ensure the directory for the DB is present */
92 db_dir_make(BOOL panic)
94 (void) directory_make(spool_directory, US"db", EXIMDB_DIRECTORY_MODE, panic);
98 /* Lock a file to protect the DB. Return TRUE for success */
101 lockfile_take(open_db * dbblock, const uschar * filename, BOOL rdonly, BOOL panic)
104 int rc, * fdp = &dbblock->lockfd;
106 priv_drop_temp(exim_uid, exim_gid);
107 if ((*fdp = Uopen(filename, O_RDWR, EXIMDB_LOCKFILE_MODE)) < 0)
110 *fdp = Uopen(filename, O_RDWR|O_CREAT, EXIMDB_LOCKFILE_MODE);
116 log_write(0, LOG_MAIN, "%s",
117 string_open_failed("database lock file %s", filename));
118 errno = 0; /* Indicates locking failure */
122 /* Now we must get a lock on the opened lock file; do this with a blocking
123 lock that times out. */
125 lock_data.l_type = rdonly ? F_RDLCK : F_WRLCK;
126 lock_data.l_whence = lock_data.l_start = lock_data.l_len = 0;
128 DEBUG(D_hints_lookup|D_retry|D_route|D_deliver)
129 debug_printf_indent("locking %s\n", filename);
131 sigalrm_seen = FALSE;
132 ALARM(EXIMDB_LOCK_TIMEOUT);
133 rc = fcntl(*fdp, F_SETLKW, &lock_data);
136 if (sigalrm_seen) errno = ETIMEDOUT;
139 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to get %s lock for %s: %s",
140 rdonly ? "read" : "write", filename,
141 errno == ETIMEDOUT ? "timed out" : strerror(errno));
142 (void)close(*fdp); *fdp = -1;
143 errno = 0; /* Indicates locking failure */
147 DEBUG(D_hints_lookup) debug_printf_indent("locked %s\n", filename);
151 /* Used for accessing Exim's hints databases.
154 name The single-component name of one of Exim's database files.
155 flags Either O_RDONLY or O_RDWR, indicating the type of open required;
157 dbblock Points to an open_db block to be filled in.
158 lof If TRUE, write to the log for actual open failures (locking failures
160 panic If TRUE, panic on failure to create the db directory
162 Returns: NULL if the open failed, or the locking failed. After locking
163 failures, errno is zero.
165 On success, dbblock is returned. This contains the dbm pointer and
166 the fd of the locked lock file.
170 dbfn_open(const uschar * name, int flags, open_db * dbblock,
171 BOOL lof, BOOL panic)
173 int rc, save_errno, dlen, flen;
175 uschar dirname[PATHLEN], filename[PATHLEN];
177 DEBUG(D_hints_lookup) acl_level++;
179 /* The first thing to do is to open a separate file on which to lock. This
180 ensures that Exim has exclusive use of the database before it even tries to
181 open it. Early versions tried to lock on the open database itself, but that
182 gave rise to mysterious problems from time to time - it was suspected that some
183 DB libraries "do things" on their open() calls which break the interlocking.
184 The lock file is never written to, but we open it for writing so we can get a
185 write lock if required. If it does not exist, we create it. This is done
186 separately so we know when we have done it, because when running as root we
187 need to change the ownership - see the bottom of this function. We also try to
188 make the directory as well, just in case. We won't be doing this many times
189 unnecessarily, because usually the lock file will be there. If the directory
190 exists, there is no error. */
192 dlen = snprintf(CS dirname, sizeof(dirname), "%s/db", spool_directory);
194 dbblock->readonly = (flags & O_ACCMODE) == O_RDONLY;
195 dbblock->lockfd = -1;
196 if (!exim_lockfile_needed())
200 flen = Ustrlen(name);
201 snprintf(CS filename, sizeof(filename), "%.*s/%.*s.lockfile",
202 (int)sizeof(filename) - dlen - flen - 11, dirname,
204 if (!lockfile_take(dbblock, filename, flags == O_RDONLY, panic))
206 DEBUG(D_hints_lookup) acl_level--;
211 /* At this point we have an opened and locked separate lock file, that is,
212 exclusive access to the database, so we can go ahead and open it. If we are
213 expected to create it, don't do so at first, again so that we can detect
214 whether we need to change its ownership (see comments about the lock file
215 above.) There have been regular reports of crashes while opening hints
216 databases - often this is caused by non-matching db.h and the library. To make
217 it easy to pin this down, there are now debug statements on either side of the
220 snprintf(CS filename, sizeof(filename), "%.*s/%s", dlen, dirname, name);
222 priv_drop_temp(exim_uid, exim_gid);
223 dbblock->dbptr = dbblock->readonly && !exim_lockfile_needed()
224 ? exim_dbopen_multi(filename, dirname, flags & O_ACCMODE, EXIMDB_MODE)
225 : exim_dbopen(filename, dirname, flags & O_ACCMODE, EXIMDB_MODE);
227 if (!dbblock->dbptr && errno == ENOENT && flags & O_CREAT)
229 DEBUG(D_hints_lookup)
230 debug_printf_indent("%s appears not to exist: trying to create\n", filename);
231 dbblock->dbptr = exim_dbopen(filename, dirname, flags, EXIMDB_MODE);
236 /* If the open has failed, return NULL, leaving errno set. If lof is TRUE,
237 log the event - also for debugging - but debug only if the file just doesn't
243 if (lof && save_errno != ENOENT)
244 log_write(0, LOG_MAIN, "%s", string_open_failed("DB file %s",
247 DEBUG(D_hints_lookup)
248 debug_printf_indent("%s\n", CS string_open_failed("DB file %s",
250 (void)close(dbblock->lockfd);
251 dbblock->lockfd = -1;
255 /* Pass back the block containing the opened database handle and the open fd
258 DEBUG(D_hints_lookup) acl_level--;
264 /* Only for transaction-capable DB types. Open without locking or
265 starting a transaction. "lof" and "panic" always true; read/write mode.
269 dbfn_open_multi(const uschar * name, int flags, open_db * dbblock)
271 int rc, save_errno, dlen;
273 uschar dirname[PATHLEN], filename[PATHLEN];
275 DEBUG(D_hints_lookup) acl_level++;
277 dbblock->lockfd = -1;
278 dbblock->readonly = (flags & O_ACCMODE) == O_RDONLY;
281 dlen = snprintf(CS dirname, sizeof(dirname), "%s/db", spool_directory);
282 snprintf(CS filename, sizeof(filename), "%.*s/%s", dlen, dirname, name);
284 priv_drop_temp(exim_uid, exim_gid);
285 dbblock->dbptr = exim_dbopen_multi(filename, dirname, flags & O_ACCMODE, EXIMDB_MODE);
286 if (!dbblock->dbptr && errno == ENOENT && flags & O_CREAT)
288 DEBUG(D_hints_lookup)
289 debug_printf_indent("%s appears not to exist: trying to create\n", filename);
290 dbblock->dbptr = exim_dbopen_multi(filename, dirname, flags, EXIMDB_MODE);
295 /* If the open has failed, return NULL, leaving errno set. If lof is TRUE,
296 log the event - also for debugging - but debug only if the file just doesn't
302 if (save_errno != ENOENT)
303 log_write(0, LOG_MAIN, "%s", string_open_failed("DB file %s",
306 DEBUG(D_hints_lookup)
307 debug_printf_indent("%s\n", CS string_open_failed("DB file %s",
312 /* Pass back the block containing the opened database handle */
313 DEBUG(D_hints_lookup) acl_level--;
318 /* Return: boolean success */
320 dbfn_transaction_start(open_db * dbp)
322 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_transaction_start\n");
323 if (!dbp->readonly) return exim_dbtransaction_start(dbp->dbptr);
327 dbfn_transaction_commit(open_db * dbp)
329 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_transaction_commit\n");
330 if (!dbp->readonly) exim_dbtransaction_commit(dbp->dbptr);
335 /*************************************************
336 * Unlock and close a database file *
337 *************************************************/
339 /* Closing a file automatically unlocks it, so after closing the database, just
340 close the lock file if there was one.
342 Argument: a pointer to an open database block
347 dbfn_close(open_db * dbp)
349 int * fdp = &dbp->lockfd;
351 if (dbp->readonly && !exim_lockfile_needed())
352 exim_dbclose_multi(dbp->dbptr);
354 exim_dbclose(dbp->dbptr);
356 if (*fdp >= 0) (void)close(*fdp);
357 DEBUG(D_hints_lookup)
358 debug_printf_indent("closed hints database%s\n",
359 *fdp < 0 ? "" : " and lockfile");
365 dbfn_close_multi(open_db * dbp)
367 exim_dbclose_multi(dbp->dbptr);
368 DEBUG(D_hints_lookup)
369 debug_printf_indent("closed hints database\n");
375 /*************************************************
376 * Read from database file *
377 *************************************************/
379 /* Read, using a defined-length key (permitting embedded NULs).
381 Passing back the pointer unchanged is useless, because there is
382 no guarantee of alignment. Since all the records used by Exim need
383 to be properly aligned to pick out the timestamps, etc., we might as
384 well do the copying centrally here.
387 dbblock a pointer to an open database block
388 key the key of the record to be read
389 klen length of key including a terminating NUL (if present)
390 length a pointer to an int into which to return the length, if not NULL
392 Returns: a pointer to the retrieved record, or
393 NULL if the record is not found
397 dbfn_read_klen(open_db * dbblock, const uschar * key, int klen, int * length)
400 EXIM_DATUM key_datum, result_datum;
401 uschar * key_copy = store_get(klen, key);
404 memcpy(key_copy, key, klen);
406 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_read: key=%.*s\n", klen, key);
408 exim_datum_init(&key_datum); /* Some DBM libraries require the datum */
409 exim_datum_init(&result_datum); /* to be cleared before use. */
410 exim_datum_data_set(&key_datum, key_copy);
411 exim_datum_size_set(&key_datum, klen);
413 if (!exim_dbget(dbblock->dbptr, &key_datum, &result_datum))
415 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_read: null return\n");
419 /* Assume the data store could have been tainted. Properly, we should
420 store the taint status with the data. */
422 dlen = exim_datum_size_get(&result_datum);
423 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_read: size %u return\n", dlen);
425 yield = store_get(dlen+1, GET_TAINTED);
426 memcpy(yield, exim_datum_data_get(&result_datum), dlen);
427 ((uschar *)yield)[dlen] = '\0';
428 if (length) *length = dlen;
430 exim_datum_free(&result_datum); /* Some DBM libs require freeing */
435 /* Read, using a NUL-terminated key.
437 Most calls don't need the length, so there is a macro called dbfn_read which
438 has two arguments; it calls this function adding NULL as the third.
441 dbblock a pointer to an open database block
442 key the key of the record to be read (NUL-terminated)
443 lenp a pointer to an int into which to return the data length,
446 Returns: a pointer to the retrieved record, or
447 NULL if the record is not found
451 dbfn_read_with_length(open_db * dbblock, const uschar * key, int * lenp)
453 return dbfn_read_klen(dbblock, key, Ustrlen(key)+1, lenp);
458 /* Read a record. If the length is not as expected then delete it, write
459 an error log line, delete the record and return NULL.
460 Use this for fixed-size records (so not retry or wait records).
463 dbblock a pointer to an open database block
464 key the key of the record to be read
465 length the expected record length
467 Returns: a pointer to the retrieved record, or
468 NULL if the record is not found/bad
472 dbfn_read_enforce_length(open_db * dbblock, const uschar * key, size_t length)
475 void * yield = dbfn_read_with_length(dbblock, key, &rlen);
479 if (rlen == length) return yield;
480 log_write(0, LOG_MAIN|LOG_PANIC, "Bad db record size for '%s'", key);
481 dbfn_delete(dbblock, key);
486 /*************************************************
487 * Write to database file *
488 *************************************************/
492 dbblock a pointer to an open database block
493 key the key of the record to be written
494 ptr a pointer to the record to be written
495 length the length of the record to be written
497 Returns: the yield of the underlying dbm or db "write" function. If this
498 is dbm, the value is zero for OK.
502 dbfn_write(open_db *dbblock, const uschar *key, void *ptr, int length)
504 EXIM_DATUM key_datum, value_datum;
505 dbdata_generic *gptr = (dbdata_generic *)ptr;
506 int klen = Ustrlen(key) + 1;
507 uschar * key_copy = store_get(klen, key);
509 memcpy(key_copy, key, klen);
510 gptr->time_stamp = time(NULL);
512 DEBUG(D_hints_lookup)
513 debug_printf_indent("dbfn_write: key=%s datalen %d\n", key, length);
515 exim_datum_init(&key_datum); /* Some DBM libraries require the datum */
516 exim_datum_init(&value_datum); /* to be cleared before use. */
517 exim_datum_data_set(&key_datum, key_copy);
518 exim_datum_size_set(&key_datum, klen);
519 exim_datum_data_set(&value_datum, ptr);
520 exim_datum_size_set(&value_datum, length);
521 return exim_dbput(dbblock->dbptr, &key_datum, &value_datum);
526 /*************************************************
527 * Delete record from database file *
528 *************************************************/
532 dbblock a pointer to an open database block
533 key the key of the record to be deleted
535 Returns: the yield of the underlying dbm or db "delete" function.
539 dbfn_delete(open_db *dbblock, const uschar *key)
541 int klen = Ustrlen(key) + 1, rc;
542 uschar * key_copy = store_get(klen, key);
543 EXIM_DATUM key_datum;
545 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_delete: key=%s\n", key);
547 memcpy(key_copy, key, klen);
548 exim_datum_init(&key_datum); /* Some DBM libraries require clearing */
549 exim_datum_data_set(&key_datum, key_copy);
550 exim_datum_size_set(&key_datum, klen);
551 rc = exim_dbdel(dbblock->dbptr, &key_datum);
552 DEBUG(D_hints_lookup) if (rc != EXIM_DBPUTB_OK)
553 debug_printf_indent(" exim_dbdel: fail\n");
560 /* XXX This appears to be unused. There's a separate implementation
561 in dbutils.c for dumpdb and fixdb, using the same underlying support.
564 /*************************************************
565 * Scan the keys of a database file *
566 *************************************************/
570 dbblock a pointer to an open database block
571 start TRUE if starting a new scan
572 FALSE if continuing with the current scan
573 cursor a pointer to a pointer to a cursor anchor, for those dbm libraries
574 that use the notion of a cursor
576 Returns: the next record from the file, or
577 NULL if there are no more
581 dbfn_scan(open_db *dbblock, BOOL start, EXIM_CURSOR **cursor)
583 EXIM_DATUM key_datum, value_datum;
586 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_scan\n");
588 /* Some dbm require an initialization */
590 if (start) *cursor = exim_dbcreate_cursor(dbblock->dbptr);
592 exim_datum_init(&key_datum); /* Some DBM libraries require the datum */
593 exim_datum_init(&value_datum); /* to be cleared before use. */
595 yield = exim_dbscan(dbblock->dbptr, &key_datum, &value_datum, start, *cursor)
596 ? US exim_datum_data_get(&key_datum) : NULL;
598 /* Some dbm require a termination */
600 if (!yield) exim_dbdelete_cursor(*cursor);
607 /*************************************************
608 **************************************************
609 * Stand-alone test program *
610 **************************************************
611 *************************************************/
616 main(int argc, char **cargv)
619 int max_db = sizeof(dbblock)/sizeof(open_db);
623 dbdata_wait *dbwait = NULL;
624 uschar **argv = USS cargv;
626 uschar structbuffer[1024];
630 printf("Usage: test_dbfn directory\n");
631 printf("The subdirectory called \"db\" in the given directory is used for\n");
632 printf("the files used in this test program.\n");
638 spool_directory = argv[1];
639 debug_selector = D_all - D_memory;
641 big_buffer = malloc(big_buffer_size);
643 for (i = 0; i < max_db; i++) dbblock[i].dbptr = NULL;
645 printf("\nExim's db functions tester: interface type is %s\n", EXIM_DBTYPE);
646 printf("DBM library: ");
648 #ifdef DB_VERSION_STRING
649 printf("Berkeley DB: %s\n", DB_VERSION_STRING);
650 #elif defined(BTREEVERSION) && defined(HASHVERSION)
652 printf("probably Berkeley DB version 1.8x (native mode)\n");
654 printf("probably Berkeley DB version 1.8x (compatibility mode)\n");
656 #elif defined(_DBM_RDONLY) || defined(dbm_dirfno)
657 printf("probably ndbm\n");
658 #elif defined(USE_TDB)
659 printf("using tdb\n");
662 printf("probably GDBM (native mode)\n");
664 printf("probably GDBM (compatibility mode)\n");
668 /* Test the functions */
670 printf("\nTest the functions\n> ");
672 while (Ufgets(buffer, 256, stdin) != NULL)
674 int len = Ustrlen(buffer);
678 uschar *cmd = buffer;
679 while (len > 0 && isspace((uschar)buffer[len-1])) len--;
682 if (isdigit((uschar)*cmd))
685 while (isdigit((uschar)*cmd)) cmd++;
686 Uskip_whitespace(&cmd);
689 if (Ustrncmp(cmd, "open", 4) == 0)
694 Uskip_whitespace(&s);
696 for (i = 0; i < max_db; i++)
697 if (dbblock[i].dbptr == NULL) break;
701 printf("Too many open databases\n> ");
706 odb = dbfn_open(s, O_RDWR|O_CREAT, dbblock + i, TRUE, TRUE);
712 printf("opened %d\n", current);
714 /* Other error cases will have written messages */
715 else if (errno == ENOENT)
717 printf("open failed: %s%s\n", strerror(errno),
719 " (or other Berkeley DB error)"
727 else if (Ustrncmp(cmd, "write", 5) == 0)
730 uschar * key = cmd + 5, * data;
734 printf("No current database\n");
738 Uskip_whitespace(&key);
740 Uskip_nonwhite(&data);
742 Uskip_whitespace(&data);
744 dbwait = (dbdata_wait *)(&structbuffer);
745 Ustrcpy(dbwait->text, data);
749 rc = dbfn_write(dbblock + current, key, dbwait,
750 Ustrlen(data) + sizeof(dbdata_wait));
752 if (rc != 0) printf("Failed: %s\n", strerror(errno));
755 else if (Ustrncmp(cmd, "read", 4) == 0)
757 uschar * key = cmd + 4;
760 printf("No current database\n");
763 Uskip_whitespace(&key);
766 dbwait = (dbdata_wait *)dbfn_read_with_length(dbblock+ current, key, NULL);
768 printf("%s\n", (dbwait == NULL)? "<not found>" : CS dbwait->text);
771 else if (Ustrncmp(cmd, "delete", 6) == 0)
773 uschar * key = cmd + 6;
776 printf("No current database\n");
779 Uskip_whitespace(&key);
780 dbfn_delete(dbblock + current, key);
783 else if (Ustrncmp(cmd, "scan", 4) == 0)
786 BOOL startflag = TRUE;
788 uschar keybuffer[256];
791 printf("No current database\n");
795 while ((key = dbfn_scan(dbblock + current, startflag, &cursor)) != NULL)
798 Ustrcpy(keybuffer, key);
799 dbwait = (dbdata_wait *)dbfn_read_with_length(dbblock + current,
801 printf("%s: %s\n", keybuffer, dbwait->text);
804 printf("End of scan\n");
807 else if (Ustrncmp(cmd, "close", 5) == 0)
809 uschar * s = cmd + 5;
810 Uskip_whitespace(&s);
812 if (i >= max_db || dbblock[i].dbptr == NULL) printf("Not open\n"); else
815 dbfn_close(dbblock + i);
817 dbblock[i].dbptr = NULL;
818 if (i == current) current = -1;
822 else if (Ustrncmp(cmd, "file", 4) == 0)
824 uschar * s = cmd + 4;
825 Uskip_whitespace(&s);
827 if (i >= max_db || dbblock[i].dbptr == NULL) printf("Not open\n");
831 else if (Ustrncmp(cmd, "time", 4) == 0)
833 showtime = ~showtime;
834 printf("Timing %s\n", showtime? "on" : "off");
837 else if (Ustrcmp(cmd, "q") == 0 || Ustrncmp(cmd, "quit", 4) == 0) break;
839 else if (Ustrncmp(cmd, "help", 4) == 0)
841 printf("close [<number>] close file [<number>]\n");
842 printf("delete <key> remove record from current file\n");
843 printf("file <number> make file <number> current\n");
844 printf("open <name> open db file\n");
845 printf("q[uit] exit program\n");
846 printf("read <key> read record from current file\n");
847 printf("scan scan current file\n");
848 printf("time time display on/off\n");
849 printf("write <key> <rest-of-line> write record to current file\n");
852 else printf("Eh?\n");
854 if (showtime && stop >= start)
855 printf("start=%d stop=%d difference=%d\n", (int)start, (int)stop,
856 (int)(stop - start));
861 for (i = 0; i < max_db; i++)
863 if (dbblock[i].dbptr != NULL)
865 printf("\nClosing %d", i);
866 dbfn_close(dbblock + i);