1 /*************************************************
2 * Exim - an Internet mail transport agent *
3 *************************************************/
5 /* Copyright (c) The Exim Maintainers 2020 - 2024 */
6 /* Copyright (c) University of Cambridge 1995 - 2018 */
7 /* See the file NOTICE for conditions of use and distribution. */
8 /* SPDX-License-Identifier: GPL-2.0-or-later */
13 /* We have buffers holding path names for database files.
14 PATH_MAX could be used here, but would be wasting memory, as we deal
15 with database files like $spooldirectory/db/<name> */
19 /* Functions for accessing Exim's hints database, which consists of a number of
20 different DBM files. This module does not contain code for reading DBM files
21 for (e.g.) alias expansion. That is all contained within the general search
22 functions. As Exim now has support for several DBM interfaces, all the relevant
23 functions are called as inlinable functions from an included file.
25 All the data in Exim's database is in the nature of *hints*. Therefore it
26 doesn't matter if it gets destroyed by accident. These functions are not
27 supposed to implement a "safe" database.
29 Keys are passed in as C strings, and the terminating zero *is* used when
30 building the dbm files. This just makes life easier when scanning the files
33 Synchronization is required on the database files, and this is achieved by
34 means of locking on independent lock files. (Earlier attempts to lock on the
35 DBM files themselves were never completely successful.) Since callers may in
36 general want to do more than one read or write while holding the lock, there
37 are separate open and close functions. However, the calling modules should
38 arrange to hold the locks for the bare minimum of time.
44 dbfn_read_enforce_length
47 dbfn_scan unused; ifdeffout out
50 ACL ratelimit & seen conditions
51 delivery retry handling
52 delivery serialization
53 TLS session resumption
60 /*************************************************
61 * Open and lock a database file *
62 *************************************************/
64 /* Ensure the directory for the DB is present */
67 db_dir_make(BOOL panic)
69 (void) directory_make(spool_directory, US"db", EXIMDB_DIRECTORY_MODE, panic);
73 /* Lock a file to protect the DB. Return TRUE for success */
76 lockfile_take(open_db * dbblock, const uschar * filename, BOOL rdonly, BOOL panic)
79 int rc, * fdp = &dbblock->lockfd;
81 priv_drop_temp(exim_uid, exim_gid);
82 if ((*fdp = Uopen(filename, O_RDWR, EXIMDB_LOCKFILE_MODE)) < 0)
85 *fdp = Uopen(filename, O_RDWR|O_CREAT, EXIMDB_LOCKFILE_MODE);
91 log_write(0, LOG_MAIN, "%s",
92 string_open_failed("database lock file %s", filename));
93 errno = 0; /* Indicates locking failure */
97 /* Now we must get a lock on the opened lock file; do this with a blocking
98 lock that times out. */
100 lock_data.l_type = rdonly ? F_RDLCK : F_WRLCK;
101 lock_data.l_whence = lock_data.l_start = lock_data.l_len = 0;
103 DEBUG(D_hints_lookup|D_retry|D_route|D_deliver)
104 debug_printf_indent("locking %s\n", filename);
106 sigalrm_seen = FALSE;
107 ALARM(EXIMDB_LOCK_TIMEOUT);
108 rc = fcntl(*fdp, F_SETLKW, &lock_data);
111 if (sigalrm_seen) errno = ETIMEDOUT;
114 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to get %s lock for %s: %s",
115 rdonly ? "read" : "write", filename,
116 errno == ETIMEDOUT ? "timed out" : strerror(errno));
117 (void)close(*fdp); *fdp = -1;
118 errno = 0; /* Indicates locking failure */
122 DEBUG(D_hints_lookup) debug_printf_indent("locked %s\n", filename);
126 /* Used for accessing Exim's hints databases.
129 name The single-component name of one of Exim's database files.
130 flags Either O_RDONLY or O_RDWR, indicating the type of open required;
132 dbblock Points to an open_db block to be filled in.
133 lof If TRUE, write to the log for actual open failures (locking failures
135 panic If TRUE, panic on failure to create the db directory
137 Returns: NULL if the open failed, or the locking failed. After locking
138 failures, errno is zero.
140 On success, dbblock is returned. This contains the dbm pointer and
141 the fd of the locked lock file.
145 dbfn_open(const uschar * name, int flags, open_db * dbblock,
146 BOOL lof, BOOL panic)
148 int rc, save_errno, dlen, flen;
150 uschar dirname[PATHLEN], filename[PATHLEN];
152 DEBUG(D_hints_lookup) acl_level++;
154 /* The first thing to do is to open a separate file on which to lock. This
155 ensures that Exim has exclusive use of the database before it even tries to
156 open it. Early versions tried to lock on the open database itself, but that
157 gave rise to mysterious problems from time to time - it was suspected that some
158 DB libraries "do things" on their open() calls which break the interlocking.
159 The lock file is never written to, but we open it for writing so we can get a
160 write lock if required. If it does not exist, we create it. This is done
161 separately so we know when we have done it, because when running as root we
162 need to change the ownership - see the bottom of this function. We also try to
163 make the directory as well, just in case. We won't be doing this many times
164 unnecessarily, because usually the lock file will be there. If the directory
165 exists, there is no error. */
167 dlen = snprintf(CS dirname, sizeof(dirname), "%s/db", spool_directory);
169 dbblock->lockfd = -1;
170 if (!exim_lockfile_needed())
174 flen = Ustrlen(name);
175 snprintf(CS filename, sizeof(filename), "%.*s/%.*s.lockfile",
176 (int)sizeof(filename) - dlen - flen - 11, dirname,
178 if (!lockfile_take(dbblock, filename, flags == O_RDONLY, panic))
180 DEBUG(D_hints_lookup) acl_level--;
185 /* At this point we have an opened and locked separate lock file, that is,
186 exclusive access to the database, so we can go ahead and open it. If we are
187 expected to create it, don't do so at first, again so that we can detect
188 whether we need to change its ownership (see comments about the lock file
189 above.) There have been regular reports of crashes while opening hints
190 databases - often this is caused by non-matching db.h and the library. To make
191 it easy to pin this down, there are now debug statements on either side of the
194 snprintf(CS filename, sizeof(filename), "%.*s/%s", dlen, dirname, name);
196 priv_drop_temp(exim_uid, exim_gid);
197 dbblock->dbptr = exim_dbopen(filename, dirname, flags & O_ACCMODE, EXIMDB_MODE);
198 if (!dbblock->dbptr && errno == ENOENT && flags & O_CREAT)
200 DEBUG(D_hints_lookup)
201 debug_printf_indent("%s appears not to exist: trying to create\n", filename);
202 dbblock->dbptr = exim_dbopen(filename, dirname, flags, EXIMDB_MODE);
207 /* If the open has failed, return NULL, leaving errno set. If lof is TRUE,
208 log the event - also for debugging - but debug only if the file just doesn't
214 if (lof && save_errno != ENOENT)
215 log_write(0, LOG_MAIN, "%s", string_open_failed("DB file %s",
218 DEBUG(D_hints_lookup)
219 debug_printf_indent("%s\n", CS string_open_failed("DB file %s",
221 (void)close(dbblock->lockfd);
222 dbblock->lockfd = -1;
224 DEBUG(D_hints_lookup) acl_level--;
228 DEBUG(D_hints_lookup)
229 debug_printf_indent("opened hints database %s: flags=%s%s\n", filename,
230 (flags & O_ACCMODE) == O_RDONLY ? "O_RDONLY"
231 : (flags & O_ACCMODE) == O_RDWR ? "O_RDWR"
233 flags & O_CREAT ? "|O_CREAT" : "");
235 /* Pass back the block containing the opened database handle and the open fd
243 /* Only for transaction-capable DB types. Open without locking or
244 starting a transaction. "lof" and "panic" always true; read/write mode.
248 dbfn_open_multi(const uschar * name, int flags, open_db * dbblock)
250 int rc, save_errno, dlen;
252 uschar dirname[PATHLEN], filename[PATHLEN];
254 DEBUG(D_hints_lookup) acl_level++;
256 dbblock->lockfd = -1;
259 dlen = snprintf(CS dirname, sizeof(dirname), "%s/db", spool_directory);
260 snprintf(CS filename, sizeof(filename), "%.*s/%s", dlen, dirname, name);
262 priv_drop_temp(exim_uid, exim_gid);
263 dbblock->dbptr = exim_dbopen_multi(filename, dirname, flags & O_ACCMODE, EXIMDB_MODE);
264 if (!dbblock->dbptr && errno == ENOENT && flags & O_CREAT)
266 DEBUG(D_hints_lookup)
267 debug_printf_indent("%s appears not to exist: trying to create\n", filename);
268 dbblock->dbptr = exim_dbopen_multi(filename, dirname, flags, EXIMDB_MODE);
273 /* If the open has failed, return NULL, leaving errno set. If lof is TRUE,
274 log the event - also for debugging - but debug only if the file just doesn't
280 if (save_errno != ENOENT)
281 log_write(0, LOG_MAIN, "%s", string_open_failed("DB file %s",
284 DEBUG(D_hints_lookup)
285 debug_printf_indent("%s\n", CS string_open_failed("DB file %s",
288 DEBUG(D_hints_lookup) acl_level--;
292 DEBUG(D_hints_lookup)
293 debug_printf_indent("opened hints database %s for transactions: NOLOCK flags=%s%s\n",
295 (flags & O_ACCMODE) == O_RDONLY ? "O_RDONLY"
296 : (flags & O_ACCMODE) == O_RDWR ? "O_RDWR"
298 flags & O_CREAT ? "|O_CREAT" : "");
300 /* Pass back the block containing the opened database handle */
307 dbfn_transaction_start(open_db * dbp)
309 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_transaction_start\n");
310 return exim_dbtransaction_start(dbp->dbptr);
313 dbfn_transaction_commit(open_db * dbp)
315 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_transaction_commit\n");
316 exim_dbtransaction_commit(dbp->dbptr);
321 /*************************************************
322 * Unlock and close a database file *
323 *************************************************/
325 /* Closing a file automatically unlocks it, so after closing the database, just
326 close the lock file if there was one.
328 Argument: a pointer to an open database block
333 dbfn_close(open_db * dbp)
335 int * fdp = &dbp->lockfd;
337 exim_dbclose(dbp->dbptr);
338 if (*fdp >= 0) (void)close(*fdp);
339 DEBUG(D_hints_lookup)
341 debug_printf_indent("closed hints database%s\n",
342 *fdp < 0 ? "" : " and lockfile");
350 dbfn_close_multi(open_db * dbp)
352 exim_dbclose_multi(dbp->dbptr);
353 DEBUG(D_hints_lookup)
355 debug_printf_indent("closed hints database\n");
363 /*************************************************
364 * Read from database file *
365 *************************************************/
367 /* Passing back the pointer unchanged is useless, because there is
368 no guarantee of alignment. Since all the records used by Exim need
369 to be properly aligned to pick out the timestamps, etc., we might as
370 well do the copying centrally here.
372 Most calls don't need the length, so there is a macro called dbfn_read which
373 has two arguments; it calls this function adding NULL as the third.
376 dbblock a pointer to an open database block
377 key the key of the record to be read
378 length a pointer to an int into which to return the length, if not NULL
380 Returns: a pointer to the retrieved record, or
381 NULL if the record is not found
385 dbfn_read_with_length(open_db * dbblock, const uschar * key, int * length)
388 EXIM_DATUM key_datum, result_datum;
389 int klen = Ustrlen(key) + 1;
390 uschar * key_copy = store_get(klen, key);
393 memcpy(key_copy, key, klen);
395 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_read: key=%s\n", key);
397 exim_datum_init(&key_datum); /* Some DBM libraries require the datum */
398 exim_datum_init(&result_datum); /* to be cleared before use. */
399 exim_datum_data_set(&key_datum, key_copy);
400 exim_datum_size_set(&key_datum, klen);
402 if (!exim_dbget(dbblock->dbptr, &key_datum, &result_datum))
404 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_read: null return\n");
408 /* Assume the data store could have been tainted. Properly, we should
409 store the taint status with the data. */
411 dlen = exim_datum_size_get(&result_datum);
412 yield = store_get(dlen, GET_TAINTED);
413 memcpy(yield, exim_datum_data_get(&result_datum), dlen);
414 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_read: size %u return\n", dlen);
415 if (length) *length = dlen;
417 exim_datum_free(&result_datum); /* Some DBM libs require freeing */
422 /* Read a record. If the length is not as expected then delete it, write
423 an error log line, delete the record and return NULL.
424 Use this for fixed-size records (so not retry or wait records).
427 dbblock a pointer to an open database block
428 key the key of the record to be read
429 length the expected record length
431 Returns: a pointer to the retrieved record, or
432 NULL if the record is not found/bad
436 dbfn_read_enforce_length(open_db * dbblock, const uschar * key, size_t length)
439 void * yield = dbfn_read_with_length(dbblock, key, &rlen);
443 if (rlen == length) return yield;
444 log_write(0, LOG_MAIN|LOG_PANIC, "Bad db record size for '%s'", key);
445 dbfn_delete(dbblock, key);
451 /*************************************************
452 * Write to database file *
453 *************************************************/
457 dbblock a pointer to an open database block
458 key the key of the record to be written
459 ptr a pointer to the record to be written
460 length the length of the record to be written
462 Returns: the yield of the underlying dbm or db "write" function. If this
463 is dbm, the value is zero for OK.
467 dbfn_write(open_db *dbblock, const uschar *key, void *ptr, int length)
469 EXIM_DATUM key_datum, value_datum;
470 dbdata_generic *gptr = (dbdata_generic *)ptr;
471 int klen = Ustrlen(key) + 1;
472 uschar * key_copy = store_get(klen, key);
474 memcpy(key_copy, key, klen);
475 gptr->time_stamp = time(NULL);
477 DEBUG(D_hints_lookup)
478 debug_printf_indent("dbfn_write: key=%s datalen %d\n", key, length);
480 exim_datum_init(&key_datum); /* Some DBM libraries require the datum */
481 exim_datum_init(&value_datum); /* to be cleared before use. */
482 exim_datum_data_set(&key_datum, key_copy);
483 exim_datum_size_set(&key_datum, klen);
484 exim_datum_data_set(&value_datum, ptr);
485 exim_datum_size_set(&value_datum, length);
486 return exim_dbput(dbblock->dbptr, &key_datum, &value_datum);
491 /*************************************************
492 * Delete record from database file *
493 *************************************************/
497 dbblock a pointer to an open database block
498 key the key of the record to be deleted
500 Returns: the yield of the underlying dbm or db "delete" function.
504 dbfn_delete(open_db *dbblock, const uschar *key)
506 int klen = Ustrlen(key) + 1;
507 uschar * key_copy = store_get(klen, key);
508 EXIM_DATUM key_datum;
510 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_delete: key=%s\n", key);
512 memcpy(key_copy, key, klen);
513 exim_datum_init(&key_datum); /* Some DBM libraries require clearing */
514 exim_datum_data_set(&key_datum, key_copy);
515 exim_datum_size_set(&key_datum, klen);
516 return exim_dbdel(dbblock->dbptr, &key_datum);
522 /* XXX This appears to be unused. There's a separate implementation
523 in dbutils.c for dumpdb and fixdb, using the same underlying support.
526 /*************************************************
527 * Scan the keys of a database file *
528 *************************************************/
532 dbblock a pointer to an open database block
533 start TRUE if starting a new scan
534 FALSE if continuing with the current scan
535 cursor a pointer to a pointer to a cursor anchor, for those dbm libraries
536 that use the notion of a cursor
538 Returns: the next record from the file, or
539 NULL if there are no more
543 dbfn_scan(open_db *dbblock, BOOL start, EXIM_CURSOR **cursor)
545 EXIM_DATUM key_datum, value_datum;
548 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_scan\n");
550 /* Some dbm require an initialization */
552 if (start) *cursor = exim_dbcreate_cursor(dbblock->dbptr);
554 exim_datum_init(&key_datum); /* Some DBM libraries require the datum */
555 exim_datum_init(&value_datum); /* to be cleared before use. */
557 yield = exim_dbscan(dbblock->dbptr, &key_datum, &value_datum, start, *cursor)
558 ? US exim_datum_data_get(&key_datum) : NULL;
560 /* Some dbm require a termination */
562 if (!yield) exim_dbdelete_cursor(*cursor);
569 /*************************************************
570 **************************************************
571 * Stand-alone test program *
572 **************************************************
573 *************************************************/
578 main(int argc, char **cargv)
581 int max_db = sizeof(dbblock)/sizeof(open_db);
585 dbdata_wait *dbwait = NULL;
586 uschar **argv = USS cargv;
588 uschar structbuffer[1024];
592 printf("Usage: test_dbfn directory\n");
593 printf("The subdirectory called \"db\" in the given directory is used for\n");
594 printf("the files used in this test program.\n");
600 spool_directory = argv[1];
601 debug_selector = D_all - D_memory;
603 big_buffer = malloc(big_buffer_size);
605 for (i = 0; i < max_db; i++) dbblock[i].dbptr = NULL;
607 printf("\nExim's db functions tester: interface type is %s\n", EXIM_DBTYPE);
608 printf("DBM library: ");
610 #ifdef DB_VERSION_STRING
611 printf("Berkeley DB: %s\n", DB_VERSION_STRING);
612 #elif defined(BTREEVERSION) && defined(HASHVERSION)
614 printf("probably Berkeley DB version 1.8x (native mode)\n");
616 printf("probably Berkeley DB version 1.8x (compatibility mode)\n");
618 #elif defined(_DBM_RDONLY) || defined(dbm_dirfno)
619 printf("probably ndbm\n");
620 #elif defined(USE_TDB)
621 printf("using tdb\n");
624 printf("probably GDBM (native mode)\n");
626 printf("probably GDBM (compatibility mode)\n");
630 /* Test the functions */
632 printf("\nTest the functions\n> ");
634 while (Ufgets(buffer, 256, stdin) != NULL)
636 int len = Ustrlen(buffer);
640 uschar *cmd = buffer;
641 while (len > 0 && isspace((uschar)buffer[len-1])) len--;
644 if (isdigit((uschar)*cmd))
647 while (isdigit((uschar)*cmd)) cmd++;
648 Uskip_whitespace(&cmd);
651 if (Ustrncmp(cmd, "open", 4) == 0)
656 Uskip_whitespace(&s);
658 for (i = 0; i < max_db; i++)
659 if (dbblock[i].dbptr == NULL) break;
663 printf("Too many open databases\n> ");
668 odb = dbfn_open(s, O_RDWR|O_CREAT, dbblock + i, TRUE, TRUE);
674 printf("opened %d\n", current);
676 /* Other error cases will have written messages */
677 else if (errno == ENOENT)
679 printf("open failed: %s%s\n", strerror(errno),
681 " (or other Berkeley DB error)"
689 else if (Ustrncmp(cmd, "write", 5) == 0)
692 uschar * key = cmd + 5, * data;
696 printf("No current database\n");
700 Uskip_whitespace(&key);
702 Uskip_nonwhite(&data);
704 Uskip_whitespace(&data);
706 dbwait = (dbdata_wait *)(&structbuffer);
707 Ustrcpy(dbwait->text, data);
711 rc = dbfn_write(dbblock + current, key, dbwait,
712 Ustrlen(data) + sizeof(dbdata_wait));
714 if (rc != 0) printf("Failed: %s\n", strerror(errno));
717 else if (Ustrncmp(cmd, "read", 4) == 0)
719 uschar * key = cmd + 4;
722 printf("No current database\n");
725 Uskip_whitespace(&key);
728 dbwait = (dbdata_wait *)dbfn_read_with_length(dbblock+ current, key, NULL);
730 printf("%s\n", (dbwait == NULL)? "<not found>" : CS dbwait->text);
733 else if (Ustrncmp(cmd, "delete", 6) == 0)
735 uschar * key = cmd + 6;
738 printf("No current database\n");
741 Uskip_whitespace(&key);
742 dbfn_delete(dbblock + current, key);
745 else if (Ustrncmp(cmd, "scan", 4) == 0)
748 BOOL startflag = TRUE;
750 uschar keybuffer[256];
753 printf("No current database\n");
757 while ((key = dbfn_scan(dbblock + current, startflag, &cursor)) != NULL)
760 Ustrcpy(keybuffer, key);
761 dbwait = (dbdata_wait *)dbfn_read_with_length(dbblock + current,
763 printf("%s: %s\n", keybuffer, dbwait->text);
766 printf("End of scan\n");
769 else if (Ustrncmp(cmd, "close", 5) == 0)
771 uschar * s = cmd + 5;
772 Uskip_whitespace(&s);
774 if (i >= max_db || dbblock[i].dbptr == NULL) printf("Not open\n"); else
777 dbfn_close(dbblock + i);
779 dbblock[i].dbptr = NULL;
780 if (i == current) current = -1;
784 else if (Ustrncmp(cmd, "file", 4) == 0)
786 uschar * s = cmd + 4;
787 Uskip_whitespace(&s);
789 if (i >= max_db || dbblock[i].dbptr == NULL) printf("Not open\n");
793 else if (Ustrncmp(cmd, "time", 4) == 0)
795 showtime = ~showtime;
796 printf("Timing %s\n", showtime? "on" : "off");
799 else if (Ustrcmp(cmd, "q") == 0 || Ustrncmp(cmd, "quit", 4) == 0) break;
801 else if (Ustrncmp(cmd, "help", 4) == 0)
803 printf("close [<number>] close file [<number>]\n");
804 printf("delete <key> remove record from current file\n");
805 printf("file <number> make file <number> current\n");
806 printf("open <name> open db file\n");
807 printf("q[uit] exit program\n");
808 printf("read <key> read record from current file\n");
809 printf("scan scan current file\n");
810 printf("time time display on/off\n");
811 printf("write <key> <rest-of-line> write record to current file\n");
814 else printf("Eh?\n");
816 if (showtime && stop >= start)
817 printf("start=%d stop=%d difference=%d\n", (int)start, (int)stop,
818 (int)(stop - start));
823 for (i = 0; i < max_db; i++)
825 if (dbblock[i].dbptr != NULL)
827 printf("\nClosing %d", i);
828 dbfn_close(dbblock + i);