1 /*************************************************
2 * Exim - an Internet mail transport agent *
3 *************************************************/
5 /* Copyright (c) The Exim Maintainers 2020 - 2024 */
6 /* Copyright (c) University of Cambridge 1995 - 2018 */
7 /* See the file NOTICE for conditions of use and distribution. */
8 /* SPDX-License-Identifier: GPL-2.0-or-later */
13 /* We have buffers holding path names for database files.
14 PATH_MAX could be used here, but would be wasting memory, as we deal
15 with database files like $spooldirectory/db/<name> */
19 /* Functions for accessing Exim's hints database, which consists of a number of
20 different DBM files. This module does not contain code for reading DBM files
21 for (e.g.) alias expansion. That is all contained within the general search
22 functions. As Exim now has support for several DBM interfaces, all the relevant
23 functions are called as inlinable functions from an included file.
25 All the data in Exim's database is in the nature of *hints*. Therefore it
26 doesn't matter if it gets destroyed by accident. These functions are not
27 supposed to implement a "safe" database.
29 Keys are passed in as C strings, and the terminating zero *is* used when
30 building the dbm files. This just makes life easier when scanning the files
33 Synchronization is required on the database files, and this is achieved by
34 means of locking on independent lock files. (Earlier attempts to lock on the
35 DBM files themselves were never completely successful.) Since callers may in
36 general want to do more than one read or write while holding the lock, there
37 are separate open and close functions. However, the calling modules should
38 arrange to hold the locks for the bare minimum of time.
41 exim_lockfile_needed facilities predicate
42 dbfn_open_path full pathname; no lock taken, readonly
43 dbfn_open takes lockfile or opens transaction
44 dbfn_open_multi only if transactions supported;
45 no lock or transaction taken
46 dbfn_close release lockfile or transaction
48 dbfn_transaction_start only if transactions supported
49 dbfn_transaction_commit
50 dbfn_read_klen explicit key length; embedded NUL ok
52 dbfn_read_enforce_length
55 dbfn_scan unused; ifdeffout out
58 ACL ratelimit & seen conditions
59 delivery retry handling
60 delivery serialization
61 TLS session resumption
69 /*************************************************
70 * Open and lock a database file *
71 *************************************************/
73 /* Used by DBM lookups:
74 full pathname for DB file rather than hintsdb name, readonly, no locking. */
77 dbfn_open_path(const uschar * path, open_db * dbblock)
79 uschar * dirname = string_copy(path);
81 dbblock->readonly = TRUE;
83 dbblock->dbptr = !exim_lockfile_needed()
84 ? exim_dbopen_multi(path, dirname, O_RDONLY, 0)
85 : exim_dbopen(path, dirname, O_RDONLY, 0);
86 return dbblock->dbptr ? dbblock : NULL;;
89 /* Ensure the directory for the DB is present */
92 db_dir_make(BOOL panic)
94 (void) directory_make(spool_directory, US"db", EXIMDB_DIRECTORY_MODE, panic);
98 /* Lock a file to protect the DB. Return TRUE for success */
101 lockfile_take(open_db * dbblock, const uschar * filename, BOOL rdonly, BOOL panic)
104 int rc, * fdp = &dbblock->lockfd;
106 priv_drop_temp(exim_uid, exim_gid);
107 if ((*fdp = Uopen(filename, O_RDWR, EXIMDB_LOCKFILE_MODE)) < 0)
110 *fdp = Uopen(filename, O_RDWR|O_CREAT, EXIMDB_LOCKFILE_MODE);
116 log_write(0, LOG_MAIN, "%s",
117 string_open_failed("database lock file %s", filename));
118 errno = 0; /* Indicates locking failure */
122 /* Now we must get a lock on the opened lock file; do this with a blocking
123 lock that times out. */
125 lock_data.l_type = rdonly ? F_RDLCK : F_WRLCK;
126 lock_data.l_whence = lock_data.l_start = lock_data.l_len = 0;
128 DEBUG(D_hints_lookup|D_retry|D_route|D_deliver)
129 debug_printf_indent("locking %s\n", filename);
131 sigalrm_seen = FALSE;
132 ALARM(EXIMDB_LOCK_TIMEOUT);
133 rc = fcntl(*fdp, F_SETLKW, &lock_data);
136 if (sigalrm_seen) errno = ETIMEDOUT;
139 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to get %s lock for %s: %s",
140 rdonly ? "read" : "write", filename,
141 errno == ETIMEDOUT ? "timed out" : strerror(errno));
142 (void)close(*fdp); *fdp = -1;
143 errno = 0; /* Indicates locking failure */
147 DEBUG(D_hints_lookup) debug_printf_indent("locked %s\n", filename);
151 /* Used for accessing Exim's hints databases.
154 name The single-component name of one of Exim's database files.
155 flags Either O_RDONLY or O_RDWR, indicating the type of open required;
157 dbblock Points to an open_db block to be filled in.
158 lof If TRUE, write to the log for actual open failures (locking failures
160 panic If TRUE, panic on failure to create the db directory
162 Returns: NULL if the open failed, or the locking failed. After locking
163 failures, errno is zero.
165 On success, dbblock is returned. This contains the dbm pointer and
166 the fd of the locked lock file.
170 dbfn_open(const uschar * name, int flags, open_db * dbblock,
171 BOOL lof, BOOL panic)
173 int rc, save_errno, dlen, flen;
175 uschar dirname[PATHLEN], filename[PATHLEN];
177 DEBUG(D_hints_lookup) acl_level++;
179 /* The first thing to do is to open a separate file on which to lock. This
180 ensures that Exim has exclusive use of the database before it even tries to
181 open it. Early versions tried to lock on the open database itself, but that
182 gave rise to mysterious problems from time to time - it was suspected that some
183 DB libraries "do things" on their open() calls which break the interlocking.
184 The lock file is never written to, but we open it for writing so we can get a
185 write lock if required. If it does not exist, we create it. This is done
186 separately so we know when we have done it, because when running as root we
187 need to change the ownership - see the bottom of this function. We also try to
188 make the directory as well, just in case. We won't be doing this many times
189 unnecessarily, because usually the lock file will be there. If the directory
190 exists, there is no error. */
192 dlen = snprintf(CS dirname, sizeof(dirname), "%s/db", spool_directory);
194 dbblock->readonly = (flags & O_ACCMODE) == O_RDONLY;
195 dbblock->lockfd = -1;
196 if (!exim_lockfile_needed())
200 flen = Ustrlen(name);
201 snprintf(CS filename, sizeof(filename), "%.*s/%.*s.lockfile",
202 (int)sizeof(filename) - dlen - flen - 11, dirname,
204 if (!lockfile_take(dbblock, filename, flags == O_RDONLY, panic))
206 DEBUG(D_hints_lookup) acl_level--;
211 /* At this point we have an opened and locked separate lock file, that is,
212 exclusive access to the database, so we can go ahead and open it. If we are
213 expected to create it, don't do so at first, again so that we can detect
214 whether we need to change its ownership (see comments about the lock file
215 above.) There have been regular reports of crashes while opening hints
216 databases - often this is caused by non-matching db.h and the library. To make
217 it easy to pin this down, there are now debug statements on either side of the
220 snprintf(CS filename, sizeof(filename), "%.*s/%s", dlen, dirname, name);
222 priv_drop_temp(exim_uid, exim_gid);
223 dbblock->dbptr = dbblock->readonly && !exim_lockfile_needed()
224 ? exim_dbopen_multi(filename, dirname, flags & O_ACCMODE, EXIMDB_MODE)
225 : exim_dbopen(filename, dirname, flags & O_ACCMODE, EXIMDB_MODE);
227 if (!dbblock->dbptr && errno == ENOENT && flags & O_CREAT)
229 DEBUG(D_hints_lookup)
230 debug_printf_indent("%s appears not to exist: trying to create\n", filename);
231 dbblock->dbptr = exim_dbopen(filename, dirname, flags, EXIMDB_MODE);
236 /* If the open has failed, return NULL, leaving errno set. If lof is TRUE,
237 log the event - also for debugging - but debug only if the file just doesn't
243 if (lof && save_errno != ENOENT)
244 log_write(0, LOG_MAIN, "%s", string_open_failed("DB file %s",
247 DEBUG(D_hints_lookup)
248 debug_printf_indent("%s\n", CS string_open_failed("DB file %s",
250 (void)close(dbblock->lockfd);
251 dbblock->lockfd = -1;
255 /* Pass back the block containing the opened database handle and the open fd
258 DEBUG(D_hints_lookup) acl_level--;
264 /* Only for transaction-capable DB types. Open without locking or
265 starting a transaction. "lof" and "panic" always true; read/write mode.
269 dbfn_open_multi(const uschar * name, int flags, open_db * dbblock)
271 int rc, save_errno, dlen;
273 uschar dirname[PATHLEN], filename[PATHLEN];
275 DEBUG(D_hints_lookup) acl_level++;
277 dbblock->lockfd = -1;
278 dbblock->readonly = (flags & O_ACCMODE) == O_RDONLY;
281 dlen = snprintf(CS dirname, sizeof(dirname), "%s/db", spool_directory);
282 snprintf(CS filename, sizeof(filename), "%.*s/%s", dlen, dirname, name);
284 priv_drop_temp(exim_uid, exim_gid);
285 dbblock->dbptr = exim_dbopen_multi(filename, dirname, flags & O_ACCMODE, EXIMDB_MODE);
286 if (!dbblock->dbptr && errno == ENOENT && flags & O_CREAT)
288 DEBUG(D_hints_lookup)
289 debug_printf_indent("%s appears not to exist: trying to create\n", filename);
290 dbblock->dbptr = exim_dbopen_multi(filename, dirname, flags, EXIMDB_MODE);
295 /* If the open has failed, return NULL, leaving errno set. If lof is TRUE,
296 log the event - also for debugging - but debug only if the file just doesn't
302 if (save_errno != ENOENT)
303 log_write(0, LOG_MAIN, "%s", string_open_failed("DB file %s",
306 DEBUG(D_hints_lookup)
307 debug_printf_indent("%s\n", CS string_open_failed("DB file %s",
312 /* Pass back the block containing the opened database handle */
313 DEBUG(D_hints_lookup) acl_level--;
318 /* Return: boolean success */
320 dbfn_transaction_start(open_db * dbp)
322 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_transaction_start\n");
323 if (!dbp->readonly) return exim_dbtransaction_start(dbp->dbptr);
327 dbfn_transaction_commit(open_db * dbp)
329 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_transaction_commit\n");
330 if (!dbp->readonly) exim_dbtransaction_commit(dbp->dbptr);
335 /*************************************************
336 * Unlock and close a database file *
337 *************************************************/
339 /* Closing a file automatically unlocks it, so after closing the database, just
340 close the lock file if there was one.
342 Argument: a pointer to an open database block
347 dbfn_close(open_db * dbp)
349 int * fdp = &dbp->lockfd;
351 if (dbp->readonly && !exim_lockfile_needed())
352 exim_dbclose_multi(dbp->dbptr);
354 exim_dbclose(dbp->dbptr);
356 if (*fdp >= 0) (void)close(*fdp);
357 DEBUG(D_hints_lookup)
358 debug_printf_indent("closed hints database%s\n",
359 *fdp < 0 ? "" : " and lockfile");
365 dbfn_close_multi(open_db * dbp)
367 exim_dbclose_multi(dbp->dbptr);
368 DEBUG(D_hints_lookup)
369 debug_printf_indent("closed hints database\n");
375 /*************************************************
376 * Read from database file *
377 *************************************************/
379 /* Read, using a defined-length key (permitting embedded NULs).
381 Passing back the pointer unchanged is useless, because there is
382 no guarantee of alignment. Since all the records used by Exim need
383 to be properly aligned to pick out the timestamps, etc., we might as
384 well do the copying centrally here.
387 dbblock a pointer to an open database block
388 key the key of the record to be read
389 klen length of key including a terminating NUL (if present)
390 length a pointer to an int into which to return the length, if not NULL
392 Returns: a pointer to the retrieved record, or
393 NULL if the record is not found
397 dbfn_read_klen(open_db * dbblock, const uschar * key, int klen, int * length)
400 EXIM_DATUM key_datum, result_datum;
401 uschar * key_copy = store_get(klen, key);
404 memcpy(key_copy, key, klen);
406 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_read: key=%.*s\n", klen, key);
408 exim_datum_init(&key_datum); /* Some DBM libraries require the datum */
409 exim_datum_init(&result_datum); /* to be cleared before use. */
410 exim_datum_data_set(&key_datum, key_copy);
411 exim_datum_size_set(&key_datum, klen);
413 if (!exim_dbget(dbblock->dbptr, &key_datum, &result_datum))
415 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_read: null return\n");
419 /* Assume the data store could have been tainted. Properly, we should
420 store the taint status with the data. */
422 dlen = exim_datum_size_get(&result_datum);
423 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_read: size %u return\n", dlen);
425 yield = store_get(dlen+1, GET_TAINTED);
426 memcpy(yield, exim_datum_data_get(&result_datum), dlen);
427 ((uschar *)yield)[dlen] = '\0';
428 if (length) *length = dlen;
430 exim_datum_free(&result_datum); /* Some DBM libs require freeing */
435 /* Read, using a NUL-terminated key.
437 Most calls don't need the length, so there is a macro called dbfn_read which
438 has two arguments; it calls this function adding NULL as the third.
441 dbblock a pointer to an open database block
442 key the key of the record to be read (NUL-terminated)
443 lenp a pointer to an int into which to return the data length,
446 Returns: a pointer to the retrieved record, or
447 NULL if the record is not found
451 dbfn_read_with_length(open_db * dbblock, const uschar * key, int * lenp)
453 return dbfn_read_klen(dbblock, key, Ustrlen(key)+1, lenp);
458 /* Read a record. If the length is not as expected then delete it, write
459 an error log line, delete the record and return NULL.
460 Use this for fixed-size records (so not retry or wait records).
463 dbblock a pointer to an open database block
464 key the key of the record to be read
465 length the expected record length
467 Returns: a pointer to the retrieved record, or
468 NULL if the record is not found/bad
472 dbfn_read_enforce_length(open_db * dbblock, const uschar * key, size_t length)
475 void * yield = dbfn_read_with_length(dbblock, key, &rlen);
479 if (rlen == length) return yield;
480 log_write(0, LOG_MAIN|LOG_PANIC, "Bad db record size for '%s'", key);
481 dbfn_delete(dbblock, key);
486 /*************************************************
487 * Write to database file *
488 *************************************************/
492 dbblock a pointer to an open database block
493 key the key of the record to be written
494 ptr a pointer to the record to be written
495 length the length of the record to be written
497 Returns: the yield of the underlying dbm or db "write" function. If this
498 is dbm, the value is zero for OK.
502 dbfn_write(open_db *dbblock, const uschar *key, void *ptr, int length)
504 EXIM_DATUM key_datum, value_datum;
505 dbdata_generic *gptr = (dbdata_generic *)ptr;
506 int klen = Ustrlen(key) + 1;
507 uschar * key_copy = store_get(klen, key);
509 memcpy(key_copy, key, klen);
510 gptr->time_stamp = time(NULL);
512 DEBUG(D_hints_lookup)
513 debug_printf_indent("dbfn_write: key=%s datalen %d\n", key, length);
515 exim_datum_init(&key_datum); /* Some DBM libraries require the datum */
516 exim_datum_init(&value_datum); /* to be cleared before use. */
517 exim_datum_data_set(&key_datum, key_copy);
518 exim_datum_size_set(&key_datum, klen);
519 exim_datum_data_set(&value_datum, ptr);
520 exim_datum_size_set(&value_datum, length);
521 return exim_dbput(dbblock->dbptr, &key_datum, &value_datum);
526 /*************************************************
527 * Delete record from database file *
528 *************************************************/
532 dbblock a pointer to an open database block
533 key the key of the record to be deleted
535 Returns: the yield of the underlying dbm or db "delete" function.
539 dbfn_delete(open_db *dbblock, const uschar *key)
541 int klen = Ustrlen(key) + 1;
542 uschar * key_copy = store_get(klen, key);
543 EXIM_DATUM key_datum;
545 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_delete: key=%s\n", key);
547 memcpy(key_copy, key, klen);
548 exim_datum_init(&key_datum); /* Some DBM libraries require clearing */
549 exim_datum_data_set(&key_datum, key_copy);
550 exim_datum_size_set(&key_datum, klen);
551 return exim_dbdel(dbblock->dbptr, &key_datum);
557 /* XXX This appears to be unused. There's a separate implementation
558 in dbutils.c for dumpdb and fixdb, using the same underlying support.
561 /*************************************************
562 * Scan the keys of a database file *
563 *************************************************/
567 dbblock a pointer to an open database block
568 start TRUE if starting a new scan
569 FALSE if continuing with the current scan
570 cursor a pointer to a pointer to a cursor anchor, for those dbm libraries
571 that use the notion of a cursor
573 Returns: the next record from the file, or
574 NULL if there are no more
578 dbfn_scan(open_db *dbblock, BOOL start, EXIM_CURSOR **cursor)
580 EXIM_DATUM key_datum, value_datum;
583 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_scan\n");
585 /* Some dbm require an initialization */
587 if (start) *cursor = exim_dbcreate_cursor(dbblock->dbptr);
589 exim_datum_init(&key_datum); /* Some DBM libraries require the datum */
590 exim_datum_init(&value_datum); /* to be cleared before use. */
592 yield = exim_dbscan(dbblock->dbptr, &key_datum, &value_datum, start, *cursor)
593 ? US exim_datum_data_get(&key_datum) : NULL;
595 /* Some dbm require a termination */
597 if (!yield) exim_dbdelete_cursor(*cursor);
604 /*************************************************
605 **************************************************
606 * Stand-alone test program *
607 **************************************************
608 *************************************************/
613 main(int argc, char **cargv)
616 int max_db = sizeof(dbblock)/sizeof(open_db);
620 dbdata_wait *dbwait = NULL;
621 uschar **argv = USS cargv;
623 uschar structbuffer[1024];
627 printf("Usage: test_dbfn directory\n");
628 printf("The subdirectory called \"db\" in the given directory is used for\n");
629 printf("the files used in this test program.\n");
635 spool_directory = argv[1];
636 debug_selector = D_all - D_memory;
638 big_buffer = malloc(big_buffer_size);
640 for (i = 0; i < max_db; i++) dbblock[i].dbptr = NULL;
642 printf("\nExim's db functions tester: interface type is %s\n", EXIM_DBTYPE);
643 printf("DBM library: ");
645 #ifdef DB_VERSION_STRING
646 printf("Berkeley DB: %s\n", DB_VERSION_STRING);
647 #elif defined(BTREEVERSION) && defined(HASHVERSION)
649 printf("probably Berkeley DB version 1.8x (native mode)\n");
651 printf("probably Berkeley DB version 1.8x (compatibility mode)\n");
653 #elif defined(_DBM_RDONLY) || defined(dbm_dirfno)
654 printf("probably ndbm\n");
655 #elif defined(USE_TDB)
656 printf("using tdb\n");
659 printf("probably GDBM (native mode)\n");
661 printf("probably GDBM (compatibility mode)\n");
665 /* Test the functions */
667 printf("\nTest the functions\n> ");
669 while (Ufgets(buffer, 256, stdin) != NULL)
671 int len = Ustrlen(buffer);
675 uschar *cmd = buffer;
676 while (len > 0 && isspace((uschar)buffer[len-1])) len--;
679 if (isdigit((uschar)*cmd))
682 while (isdigit((uschar)*cmd)) cmd++;
683 Uskip_whitespace(&cmd);
686 if (Ustrncmp(cmd, "open", 4) == 0)
691 Uskip_whitespace(&s);
693 for (i = 0; i < max_db; i++)
694 if (dbblock[i].dbptr == NULL) break;
698 printf("Too many open databases\n> ");
703 odb = dbfn_open(s, O_RDWR|O_CREAT, dbblock + i, TRUE, TRUE);
709 printf("opened %d\n", current);
711 /* Other error cases will have written messages */
712 else if (errno == ENOENT)
714 printf("open failed: %s%s\n", strerror(errno),
716 " (or other Berkeley DB error)"
724 else if (Ustrncmp(cmd, "write", 5) == 0)
727 uschar * key = cmd + 5, * data;
731 printf("No current database\n");
735 Uskip_whitespace(&key);
737 Uskip_nonwhite(&data);
739 Uskip_whitespace(&data);
741 dbwait = (dbdata_wait *)(&structbuffer);
742 Ustrcpy(dbwait->text, data);
746 rc = dbfn_write(dbblock + current, key, dbwait,
747 Ustrlen(data) + sizeof(dbdata_wait));
749 if (rc != 0) printf("Failed: %s\n", strerror(errno));
752 else if (Ustrncmp(cmd, "read", 4) == 0)
754 uschar * key = cmd + 4;
757 printf("No current database\n");
760 Uskip_whitespace(&key);
763 dbwait = (dbdata_wait *)dbfn_read_with_length(dbblock+ current, key, NULL);
765 printf("%s\n", (dbwait == NULL)? "<not found>" : CS dbwait->text);
768 else if (Ustrncmp(cmd, "delete", 6) == 0)
770 uschar * key = cmd + 6;
773 printf("No current database\n");
776 Uskip_whitespace(&key);
777 dbfn_delete(dbblock + current, key);
780 else if (Ustrncmp(cmd, "scan", 4) == 0)
783 BOOL startflag = TRUE;
785 uschar keybuffer[256];
788 printf("No current database\n");
792 while ((key = dbfn_scan(dbblock + current, startflag, &cursor)) != NULL)
795 Ustrcpy(keybuffer, key);
796 dbwait = (dbdata_wait *)dbfn_read_with_length(dbblock + current,
798 printf("%s: %s\n", keybuffer, dbwait->text);
801 printf("End of scan\n");
804 else if (Ustrncmp(cmd, "close", 5) == 0)
806 uschar * s = cmd + 5;
807 Uskip_whitespace(&s);
809 if (i >= max_db || dbblock[i].dbptr == NULL) printf("Not open\n"); else
812 dbfn_close(dbblock + i);
814 dbblock[i].dbptr = NULL;
815 if (i == current) current = -1;
819 else if (Ustrncmp(cmd, "file", 4) == 0)
821 uschar * s = cmd + 4;
822 Uskip_whitespace(&s);
824 if (i >= max_db || dbblock[i].dbptr == NULL) printf("Not open\n");
828 else if (Ustrncmp(cmd, "time", 4) == 0)
830 showtime = ~showtime;
831 printf("Timing %s\n", showtime? "on" : "off");
834 else if (Ustrcmp(cmd, "q") == 0 || Ustrncmp(cmd, "quit", 4) == 0) break;
836 else if (Ustrncmp(cmd, "help", 4) == 0)
838 printf("close [<number>] close file [<number>]\n");
839 printf("delete <key> remove record from current file\n");
840 printf("file <number> make file <number> current\n");
841 printf("open <name> open db file\n");
842 printf("q[uit] exit program\n");
843 printf("read <key> read record from current file\n");
844 printf("scan scan current file\n");
845 printf("time time display on/off\n");
846 printf("write <key> <rest-of-line> write record to current file\n");
849 else printf("Eh?\n");
851 if (showtime && stop >= start)
852 printf("start=%d stop=%d difference=%d\n", (int)start, (int)stop,
853 (int)(stop - start));
858 for (i = 0; i < max_db; i++)
860 if (dbblock[i].dbptr != NULL)
862 printf("\nClosing %d", i);
863 dbfn_close(dbblock + i);