1 /*************************************************
2 * Exim - an Internet mail transport agent *
3 *************************************************/
5 /* Copyright (c) The Exim Maintainers 2020 - 2024 */
6 /* Copyright (c) University of Cambridge 1995 - 2018 */
7 /* See the file NOTICE for conditions of use and distribution. */
8 /* SPDX-License-Identifier: GPL-2.0-or-later */
13 /* We have buffers holding path names for database files.
14 PATH_MAX could be used here, but would be wasting memory, as we deal
15 with database files like $spooldirectory/db/<name> */
19 /* Functions for accessing Exim's hints database, which consists of a number of
20 different DBM files. This module does not contain code for reading DBM files
21 for (e.g.) alias expansion. That is all contained within the general search
22 functions. As Exim now has support for several DBM interfaces, all the relevant
23 functions are called as inlinable functions from an included file.
25 All the data in Exim's database is in the nature of *hints*. Therefore it
26 doesn't matter if it gets destroyed by accident. These functions are not
27 supposed to implement a "safe" database.
29 Keys are passed in as C strings, and the terminating zero *is* used when
30 building the dbm files. This just makes life easier when scanning the files
33 Synchronization is required on the database files, and this is achieved by
34 means of locking on independent lock files. (Earlier attempts to lock on the
35 DBM files themselves were never completely successful.) Since callers may in
36 general want to do more than one read or write while holding the lock, there
37 are separate open and close functions. However, the calling modules should
38 arrange to hold the locks for the bare minimum of time.
41 exim_lockfile_needed facilities predicate
42 dbfn_open_path full pathname; no lock taken, readonly
43 dbfn_open takes lockfile or opens transaction
44 dbfn_open_multi only if transactions supported;
45 no lock or transaction taken
46 dbfn_close release lockfile or transaction
48 dbfn_transaction_start only if transactions supported
49 dbfn_transaction_commit
50 dbfn_read_klen explicit key length; embedded NUL ok
52 dbfn_read_enforce_length
55 dbfn_scan unused; ifdeffout out
58 ACL ratelimit & seen conditions
59 delivery retry handling
60 delivery serialization
61 TLS session resumption
69 /*************************************************
70 * Open and lock a database file *
71 *************************************************/
73 /* Used by DBM lookups:
74 full pathname for DB file rather than hintsdb name, readonly, no locking. */
77 dbfn_open_path(const uschar * path, open_db * dbblock)
79 uschar * dirname = string_copy(path);
81 dbblock->readonly = TRUE;
83 dbblock->dbptr = !exim_lockfile_needed()
84 ? exim_dbopen_multi(path, dirname, O_RDONLY, 0)
85 : exim_dbopen(path, dirname, O_RDONLY, 0);
86 return dbblock->dbptr ? dbblock : NULL;;
89 /* Ensure the directory for the DB is present */
92 db_dir_make(BOOL panic)
94 (void) directory_make(spool_directory, US"db", EXIMDB_DIRECTORY_MODE, panic);
98 /* Lock a file to protect the DB. Return TRUE for success */
101 lockfile_take(open_db * dbblock, const uschar * filename, BOOL rdonly, BOOL panic)
104 int rc, * fdp = &dbblock->lockfd;
106 priv_drop_temp(exim_uid, exim_gid);
107 if ((*fdp = Uopen(filename, O_RDWR, EXIMDB_LOCKFILE_MODE)) < 0)
110 *fdp = Uopen(filename, O_RDWR|O_CREAT, EXIMDB_LOCKFILE_MODE);
116 log_write(0, LOG_MAIN, "%s",
117 string_open_failed("database lock file %s", filename));
118 errno = 0; /* Indicates locking failure */
122 /* Now we must get a lock on the opened lock file; do this with a blocking
123 lock that times out. */
125 lock_data.l_type = rdonly ? F_RDLCK : F_WRLCK;
126 lock_data.l_whence = lock_data.l_start = lock_data.l_len = 0;
128 DEBUG(D_hints_lookup|D_retry|D_route|D_deliver)
129 debug_printf_indent("locking %s\n", filename);
131 sigalrm_seen = FALSE;
132 ALARM(EXIMDB_LOCK_TIMEOUT);
133 rc = fcntl(*fdp, F_SETLKW, &lock_data);
136 if (sigalrm_seen) errno = ETIMEDOUT;
139 log_write(0, LOG_MAIN|LOG_PANIC, "Failed to get %s lock for %s: %s",
140 rdonly ? "read" : "write", filename,
141 errno == ETIMEDOUT ? "timed out" : strerror(errno));
142 (void)close(*fdp); *fdp = -1;
143 errno = 0; /* Indicates locking failure */
147 DEBUG(D_hints_lookup) debug_printf_indent("locked %s\n", filename);
151 /* Used for accessing Exim's hints databases.
154 name The single-component name of one of Exim's database files.
155 flags Either O_RDONLY or O_RDWR, indicating the type of open required;
157 dbblock Points to an open_db block to be filled in.
158 lof If TRUE, write to the log for actual open failures (locking failures
160 panic If TRUE, panic on failure to create the db directory
162 Returns: NULL if the open failed, or the locking failed. After locking
163 failures, errno is zero.
165 On success, dbblock is returned. This contains the dbm pointer and
166 the fd of the locked lock file.
170 dbfn_open(const uschar * name, int flags, open_db * dbblock,
171 BOOL lof, BOOL panic)
173 int rc, save_errno, dlen, flen;
175 uschar dirname[PATHLEN], filename[PATHLEN];
177 DEBUG(D_hints_lookup) acl_level++;
179 /* The first thing to do is to open a separate file on which to lock. This
180 ensures that Exim has exclusive use of the database before it even tries to
181 open it. Early versions tried to lock on the open database itself, but that
182 gave rise to mysterious problems from time to time - it was suspected that some
183 DB libraries "do things" on their open() calls which break the interlocking.
184 The lock file is never written to, but we open it for writing so we can get a
185 write lock if required. If it does not exist, we create it. This is done
186 separately so we know when we have done it, because when running as root we
187 need to change the ownership - see the bottom of this function. We also try to
188 make the directory as well, just in case. We won't be doing this many times
189 unnecessarily, because usually the lock file will be there. If the directory
190 exists, there is no error. */
192 dlen = snprintf(CS dirname, sizeof(dirname), "%s/db", spool_directory);
194 dbblock->readonly = (flags & O_ACCMODE) == O_RDONLY;
195 dbblock->lockfd = -1;
196 if (!exim_lockfile_needed())
200 flen = Ustrlen(name);
201 snprintf(CS filename, sizeof(filename), "%.*s/%.*s.lockfile",
202 (int)sizeof(filename) - dlen - flen - 11, dirname,
204 if (!lockfile_take(dbblock, filename, flags == O_RDONLY, panic))
206 DEBUG(D_hints_lookup) acl_level--;
211 /* At this point we have an opened and locked separate lock file, that is,
212 exclusive access to the database, so we can go ahead and open it. If we are
213 expected to create it, don't do so at first, again so that we can detect
214 whether we need to change its ownership (see comments about the lock file
215 above.) There have been regular reports of crashes while opening hints
216 databases - often this is caused by non-matching db.h and the library. To make
217 it easy to pin this down, there are now debug statements on either side of the
220 snprintf(CS filename, sizeof(filename), "%.*s/%s", dlen, dirname, name);
222 priv_drop_temp(exim_uid, exim_gid);
223 dbblock->dbptr = dbblock->readonly && !exim_lockfile_needed()
224 ? exim_dbopen_multi(filename, dirname, flags & O_ACCMODE, EXIMDB_MODE)
225 : exim_dbopen(filename, dirname, flags & O_ACCMODE, EXIMDB_MODE);
227 if (!dbblock->dbptr && errno == ENOENT && flags & O_CREAT)
229 DEBUG(D_hints_lookup)
230 debug_printf_indent("%s appears not to exist: trying to create\n", filename);
231 dbblock->dbptr = exim_dbopen(filename, dirname, flags, EXIMDB_MODE);
236 /* If the open has failed, return NULL, leaving errno set. If lof is TRUE,
237 log the event - also for debugging - but debug only if the file just doesn't
243 if (lof && save_errno != ENOENT)
244 log_write(0, LOG_MAIN, "%s", string_open_failed("DB file %s",
247 DEBUG(D_hints_lookup)
248 debug_printf_indent("%s\n", CS string_open_failed("DB file %s",
250 (void)close(dbblock->lockfd);
251 dbblock->lockfd = -1;
255 /* Pass back the block containing the opened database handle and the open fd
258 DEBUG(D_hints_lookup) acl_level--;
264 /* Only for transaction-capable DB types. Open without locking or
265 starting a transaction. "lof" and "panic" always true; read/write mode.
269 dbfn_open_multi(const uschar * name, int flags, open_db * dbblock)
271 int rc, save_errno, dlen;
273 uschar dirname[PATHLEN], filename[PATHLEN];
275 DEBUG(D_hints_lookup) acl_level++;
277 dbblock->lockfd = -1;
278 dbblock->readonly = (flags & O_ACCMODE) == O_RDONLY;
281 dlen = snprintf(CS dirname, sizeof(dirname), "%s/db", spool_directory);
282 snprintf(CS filename, sizeof(filename), "%.*s/%s", dlen, dirname, name);
284 priv_drop_temp(exim_uid, exim_gid);
285 dbblock->dbptr = exim_dbopen_multi(filename, dirname, flags & O_ACCMODE, EXIMDB_MODE);
286 if (!dbblock->dbptr && errno == ENOENT && flags & O_CREAT)
288 DEBUG(D_hints_lookup)
289 debug_printf_indent("%s appears not to exist: trying to create\n", filename);
290 dbblock->dbptr = exim_dbopen_multi(filename, dirname, flags, EXIMDB_MODE);
295 /* If the open has failed, return NULL, leaving errno set. If lof is TRUE,
296 log the event - also for debugging - but debug only if the file just doesn't
302 if (save_errno != ENOENT)
303 log_write(0, LOG_MAIN, "%s", string_open_failed("DB file %s",
306 DEBUG(D_hints_lookup)
307 debug_printf_indent("%s\n", CS string_open_failed("DB file %s",
312 /* Pass back the block containing the opened database handle */
313 DEBUG(D_hints_lookup) acl_level--;
318 /* Return: boolean success */
320 dbfn_transaction_start(open_db * dbp)
322 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_transaction_start\n");
323 if (!dbp->readonly) return exim_dbtransaction_start(dbp->dbptr);
327 dbfn_transaction_commit(open_db * dbp)
329 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_transaction_commit\n");
330 if (!dbp->readonly) exim_dbtransaction_commit(dbp->dbptr);
335 /*************************************************
336 * Unlock and close a database file *
337 *************************************************/
339 /* Closing a file automatically unlocks it, so after closing the database, just
340 close the lock file if there was one.
342 Argument: a pointer to an open database block
347 dbfn_close(open_db * dbp)
349 int * fdp = &dbp->lockfd;
351 if (dbp->readonly && !exim_lockfile_needed())
352 exim_dbclose_multi(dbp->dbptr);
354 exim_dbclose(dbp->dbptr);
356 if (*fdp >= 0) (void)close(*fdp);
357 DEBUG(D_hints_lookup)
358 debug_printf_indent("closed hints database%s\n",
359 *fdp < 0 ? "" : " and lockfile");
365 dbfn_close_multi(open_db * dbp)
367 exim_dbclose_multi(dbp->dbptr);
368 DEBUG(D_hints_lookup)
369 debug_printf_indent("closed hints database\n");
375 /*************************************************
376 * Read from database file *
377 *************************************************/
379 /* Passing back the pointer unchanged is useless, because there is
380 no guarantee of alignment. Since all the records used by Exim need
381 to be properly aligned to pick out the timestamps, etc., we might as
382 well do the copying centrally here.
384 Most calls don't need the length, so there is a macro called dbfn_read which
385 has two arguments; it calls this function adding NULL as the third.
388 dbblock a pointer to an open database block
389 key the key of the record to be read
390 klen length of key including a terminating NUL (if present)
391 length a pointer to an int into which to return the length, if not NULL
393 Returns: a pointer to the retrieved record, or
394 NULL if the record is not found
398 dbfn_read_klen(open_db * dbblock, const uschar * key, int klen, int * length)
401 EXIM_DATUM key_datum, result_datum;
402 uschar * key_copy = store_get(klen, key);
405 memcpy(key_copy, key, klen);
407 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_read: key=%.*s\n", klen, key);
409 exim_datum_init(&key_datum); /* Some DBM libraries require the datum */
410 exim_datum_init(&result_datum); /* to be cleared before use. */
411 exim_datum_data_set(&key_datum, key_copy);
412 exim_datum_size_set(&key_datum, klen);
414 if (!exim_dbget(dbblock->dbptr, &key_datum, &result_datum))
416 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_read: null return\n");
420 /* Assume the data store could have been tainted. Properly, we should
421 store the taint status with the data. */
423 dlen = exim_datum_size_get(&result_datum);
424 yield = store_get(dlen, GET_TAINTED);
425 memcpy(yield, exim_datum_data_get(&result_datum), dlen);
426 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_read: size %u return\n", dlen);
427 if (length) *length = dlen;
429 exim_datum_free(&result_datum); /* Some DBM libs require freeing */
436 dbblock a pointer to an open database block
437 key the key of the record to be read (NUL-terminated)
438 lenp a pointer to an int into which to return the data length,
441 Returns: a pointer to the retrieved record, or
442 NULL if the record is not found
446 dbfn_read_with_length(open_db * dbblock, const uschar * key, int * lenp)
448 return dbfn_read_klen(dbblock, key, Ustrlen(key)+1, lenp);
453 /* Read a record. If the length is not as expected then delete it, write
454 an error log line, delete the record and return NULL.
455 Use this for fixed-size records (so not retry or wait records).
458 dbblock a pointer to an open database block
459 key the key of the record to be read
460 length the expected record length
462 Returns: a pointer to the retrieved record, or
463 NULL if the record is not found/bad
467 dbfn_read_enforce_length(open_db * dbblock, const uschar * key, size_t length)
470 void * yield = dbfn_read_with_length(dbblock, key, &rlen);
474 if (rlen == length) return yield;
475 log_write(0, LOG_MAIN|LOG_PANIC, "Bad db record size for '%s'", key);
476 dbfn_delete(dbblock, key);
481 /*************************************************
482 * Write to database file *
483 *************************************************/
487 dbblock a pointer to an open database block
488 key the key of the record to be written
489 ptr a pointer to the record to be written
490 length the length of the record to be written
492 Returns: the yield of the underlying dbm or db "write" function. If this
493 is dbm, the value is zero for OK.
497 dbfn_write(open_db *dbblock, const uschar *key, void *ptr, int length)
499 EXIM_DATUM key_datum, value_datum;
500 dbdata_generic *gptr = (dbdata_generic *)ptr;
501 int klen = Ustrlen(key) + 1;
502 uschar * key_copy = store_get(klen, key);
504 memcpy(key_copy, key, klen);
505 gptr->time_stamp = time(NULL);
507 DEBUG(D_hints_lookup)
508 debug_printf_indent("dbfn_write: key=%s datalen %d\n", key, length);
510 exim_datum_init(&key_datum); /* Some DBM libraries require the datum */
511 exim_datum_init(&value_datum); /* to be cleared before use. */
512 exim_datum_data_set(&key_datum, key_copy);
513 exim_datum_size_set(&key_datum, klen);
514 exim_datum_data_set(&value_datum, ptr);
515 exim_datum_size_set(&value_datum, length);
516 return exim_dbput(dbblock->dbptr, &key_datum, &value_datum);
521 /*************************************************
522 * Delete record from database file *
523 *************************************************/
527 dbblock a pointer to an open database block
528 key the key of the record to be deleted
530 Returns: the yield of the underlying dbm or db "delete" function.
534 dbfn_delete(open_db *dbblock, const uschar *key)
536 int klen = Ustrlen(key) + 1;
537 uschar * key_copy = store_get(klen, key);
538 EXIM_DATUM key_datum;
540 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_delete: key=%s\n", key);
542 memcpy(key_copy, key, klen);
543 exim_datum_init(&key_datum); /* Some DBM libraries require clearing */
544 exim_datum_data_set(&key_datum, key_copy);
545 exim_datum_size_set(&key_datum, klen);
546 return exim_dbdel(dbblock->dbptr, &key_datum);
552 /* XXX This appears to be unused. There's a separate implementation
553 in dbutils.c for dumpdb and fixdb, using the same underlying support.
556 /*************************************************
557 * Scan the keys of a database file *
558 *************************************************/
562 dbblock a pointer to an open database block
563 start TRUE if starting a new scan
564 FALSE if continuing with the current scan
565 cursor a pointer to a pointer to a cursor anchor, for those dbm libraries
566 that use the notion of a cursor
568 Returns: the next record from the file, or
569 NULL if there are no more
573 dbfn_scan(open_db *dbblock, BOOL start, EXIM_CURSOR **cursor)
575 EXIM_DATUM key_datum, value_datum;
578 DEBUG(D_hints_lookup) debug_printf_indent("dbfn_scan\n");
580 /* Some dbm require an initialization */
582 if (start) *cursor = exim_dbcreate_cursor(dbblock->dbptr);
584 exim_datum_init(&key_datum); /* Some DBM libraries require the datum */
585 exim_datum_init(&value_datum); /* to be cleared before use. */
587 yield = exim_dbscan(dbblock->dbptr, &key_datum, &value_datum, start, *cursor)
588 ? US exim_datum_data_get(&key_datum) : NULL;
590 /* Some dbm require a termination */
592 if (!yield) exim_dbdelete_cursor(*cursor);
599 /*************************************************
600 **************************************************
601 * Stand-alone test program *
602 **************************************************
603 *************************************************/
608 main(int argc, char **cargv)
611 int max_db = sizeof(dbblock)/sizeof(open_db);
615 dbdata_wait *dbwait = NULL;
616 uschar **argv = USS cargv;
618 uschar structbuffer[1024];
622 printf("Usage: test_dbfn directory\n");
623 printf("The subdirectory called \"db\" in the given directory is used for\n");
624 printf("the files used in this test program.\n");
630 spool_directory = argv[1];
631 debug_selector = D_all - D_memory;
633 big_buffer = malloc(big_buffer_size);
635 for (i = 0; i < max_db; i++) dbblock[i].dbptr = NULL;
637 printf("\nExim's db functions tester: interface type is %s\n", EXIM_DBTYPE);
638 printf("DBM library: ");
640 #ifdef DB_VERSION_STRING
641 printf("Berkeley DB: %s\n", DB_VERSION_STRING);
642 #elif defined(BTREEVERSION) && defined(HASHVERSION)
644 printf("probably Berkeley DB version 1.8x (native mode)\n");
646 printf("probably Berkeley DB version 1.8x (compatibility mode)\n");
648 #elif defined(_DBM_RDONLY) || defined(dbm_dirfno)
649 printf("probably ndbm\n");
650 #elif defined(USE_TDB)
651 printf("using tdb\n");
654 printf("probably GDBM (native mode)\n");
656 printf("probably GDBM (compatibility mode)\n");
660 /* Test the functions */
662 printf("\nTest the functions\n> ");
664 while (Ufgets(buffer, 256, stdin) != NULL)
666 int len = Ustrlen(buffer);
670 uschar *cmd = buffer;
671 while (len > 0 && isspace((uschar)buffer[len-1])) len--;
674 if (isdigit((uschar)*cmd))
677 while (isdigit((uschar)*cmd)) cmd++;
678 Uskip_whitespace(&cmd);
681 if (Ustrncmp(cmd, "open", 4) == 0)
686 Uskip_whitespace(&s);
688 for (i = 0; i < max_db; i++)
689 if (dbblock[i].dbptr == NULL) break;
693 printf("Too many open databases\n> ");
698 odb = dbfn_open(s, O_RDWR|O_CREAT, dbblock + i, TRUE, TRUE);
704 printf("opened %d\n", current);
706 /* Other error cases will have written messages */
707 else if (errno == ENOENT)
709 printf("open failed: %s%s\n", strerror(errno),
711 " (or other Berkeley DB error)"
719 else if (Ustrncmp(cmd, "write", 5) == 0)
722 uschar * key = cmd + 5, * data;
726 printf("No current database\n");
730 Uskip_whitespace(&key);
732 Uskip_nonwhite(&data);
734 Uskip_whitespace(&data);
736 dbwait = (dbdata_wait *)(&structbuffer);
737 Ustrcpy(dbwait->text, data);
741 rc = dbfn_write(dbblock + current, key, dbwait,
742 Ustrlen(data) + sizeof(dbdata_wait));
744 if (rc != 0) printf("Failed: %s\n", strerror(errno));
747 else if (Ustrncmp(cmd, "read", 4) == 0)
749 uschar * key = cmd + 4;
752 printf("No current database\n");
755 Uskip_whitespace(&key);
758 dbwait = (dbdata_wait *)dbfn_read_with_length(dbblock+ current, key, NULL);
760 printf("%s\n", (dbwait == NULL)? "<not found>" : CS dbwait->text);
763 else if (Ustrncmp(cmd, "delete", 6) == 0)
765 uschar * key = cmd + 6;
768 printf("No current database\n");
771 Uskip_whitespace(&key);
772 dbfn_delete(dbblock + current, key);
775 else if (Ustrncmp(cmd, "scan", 4) == 0)
778 BOOL startflag = TRUE;
780 uschar keybuffer[256];
783 printf("No current database\n");
787 while ((key = dbfn_scan(dbblock + current, startflag, &cursor)) != NULL)
790 Ustrcpy(keybuffer, key);
791 dbwait = (dbdata_wait *)dbfn_read_with_length(dbblock + current,
793 printf("%s: %s\n", keybuffer, dbwait->text);
796 printf("End of scan\n");
799 else if (Ustrncmp(cmd, "close", 5) == 0)
801 uschar * s = cmd + 5;
802 Uskip_whitespace(&s);
804 if (i >= max_db || dbblock[i].dbptr == NULL) printf("Not open\n"); else
807 dbfn_close(dbblock + i);
809 dbblock[i].dbptr = NULL;
810 if (i == current) current = -1;
814 else if (Ustrncmp(cmd, "file", 4) == 0)
816 uschar * s = cmd + 4;
817 Uskip_whitespace(&s);
819 if (i >= max_db || dbblock[i].dbptr == NULL) printf("Not open\n");
823 else if (Ustrncmp(cmd, "time", 4) == 0)
825 showtime = ~showtime;
826 printf("Timing %s\n", showtime? "on" : "off");
829 else if (Ustrcmp(cmd, "q") == 0 || Ustrncmp(cmd, "quit", 4) == 0) break;
831 else if (Ustrncmp(cmd, "help", 4) == 0)
833 printf("close [<number>] close file [<number>]\n");
834 printf("delete <key> remove record from current file\n");
835 printf("file <number> make file <number> current\n");
836 printf("open <name> open db file\n");
837 printf("q[uit] exit program\n");
838 printf("read <key> read record from current file\n");
839 printf("scan scan current file\n");
840 printf("time time display on/off\n");
841 printf("write <key> <rest-of-line> write record to current file\n");
844 else printf("Eh?\n");
846 if (showtime && stop >= start)
847 printf("start=%d stop=%d difference=%d\n", (int)start, (int)stop,
848 (int)(stop - start));
853 for (i = 0; i < max_db; i++)
855 if (dbblock[i].dbptr != NULL)
857 printf("\nClosing %d", i);
858 dbfn_close(dbblock + i);