From: Jeremy Harris Date: Sat, 27 Jul 2024 13:17:12 +0000 (+0100) Subject: Fix dmbjz on sqlite X-Git-Url: https://git.exim.org/exim.git/commitdiff_plain/5914065c4651621ed7a5b832094ed9fe83bb2ca4 Fix dmbjz on sqlite --- diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog index 2a7a70c47..7a4f1757e 100644 --- a/doc/doc-txt/ChangeLog +++ b/doc/doc-txt/ChangeLog @@ -22,6 +22,11 @@ JH/03 With dkim_verify_minimal, avoid calling the DKIM ACL after the first JH/04 Remove the docs and support scripts dealing with conversion of Exim version 3 installations. +JH/05 Fix hintsdb support for dbmjz when compiled using sqlite3. Previously + the backend support assumed keys would be simple C strings, but dbmjz + uses keys with embedded NUL bytes. The builtin hintsdb use is unaffected, + but installations using dbmjz will need to rebuild those DBs. + Exim version 4.98 ----------------- diff --git a/src/OS/Makefile-Base b/src/OS/Makefile-Base index 4df9451f8..f494249bf 100644 --- a/src/OS/Makefile-Base +++ b/src/OS/Makefile-Base @@ -544,7 +544,7 @@ exim: buildlookups buildauths pdkim/pdkim.a \ # The utility for dumping the contents of an exim database -OBJ_DUMPDB = exim_dumpdb.o util-os.o util-store.o +OBJ_DUMPDB = exim_dumpdb.o util-os.o util-store.o util-xtextencode.o exim_dumpdb: $(OBJ_DUMPDB) @echo "$(LNCC) -o exim_dumpdb" @@ -559,7 +559,7 @@ exim_dumpdb: $(OBJ_DUMPDB) # The utility for interrogating/fixing the contents of an exim database -OBJ_FIXDB = exim_fixdb.o util-os.o util-store.o util-md5.o +OBJ_FIXDB = exim_fixdb.o util-os.o util-store.o util-md5.o util-xtextencode.o exim_fixdb: $(OBJ_FIXDB) @echo "$(LNCC) -o exim_fixdb" @@ -574,11 +574,12 @@ exim_fixdb: $(OBJ_FIXDB) # The utility for tidying the contents of an exim database -OBJ_TIDYDB = exim_tidydb.o util-os.o util-store.o +OBJ_TIDYDB = exim_tidydb.o util-os.o util-store.o util-xtextencode.o exim_tidydb: $(OBJ_TIDYDB) @echo "$(LNCC) -o exim_tidydb" - $(FE)$(LNCC) $(CFLAGS) $(INCLUDE) -o exim_tidydb $(LFLAGS) $(OBJ_TIDYDB) \ + $(FE)$(LNCC) $(CFLAGS) $(INCLUDE) -o exim_tidydb $(LFLAGS) \ + $(OBJ_TIDYDB) \ $(LIBS) $(EXTRALIBS) $(DBMLIB) @if [ x"$(STRIP_COMMAND)" != x"" ]; then \ echo $(STRIP_COMMAND) exim_tidydb; \ @@ -589,9 +590,12 @@ exim_tidydb: $(OBJ_TIDYDB) # The utility for building dbm files -exim_dbmbuild: exim_dbmbuild.o +OBJ_DBMBUILD = exim_dbmbuild.o util-xtextencode.o + +exim_dbmbuild: $(OBJ_DBMBUILD) @echo "$(LNCC) -o exim_dbmbuild" - $(FE)$(LNCC) $(CFLAGS) $(INCLUDE) -o exim_dbmbuild $(LFLAGS) exim_dbmbuild.o \ + $(FE)$(LNCC) $(CFLAGS) $(INCLUDE) -o exim_dbmbuild $(LFLAGS) \ + $(OBJ_DBMBUILD) \ $(LIBS) $(EXTRALIBS) $(DBMLIB) @if [ x"$(STRIP_COMMAND)" != x"" ]; then \ echo $(STRIP_COMMAND) exim_dbmbuild; \ @@ -636,6 +640,7 @@ OBJ_MONBIN = util-host_address.o \ util-string.o \ util-tod.o \ util-tree.o \ + util-xtextencode.o \ $(MONBIN) eximon.bin: $(EXIMON_EDITME) eximon $(OBJ_MONBIN) ../exim_monitor/em_version.c \ @@ -794,6 +799,10 @@ util-tree.o: $(HDRS) tree.c @echo "$(CC) -DCOMPILE_UTILITY tree.c" $(FE)$(CC) -c $(CFLAGS) $(INCLUDE) -DCOMPILE_UTILITY -o util-tree.o tree.c +util-xtextencode.o: $(HDRS) xtextencode.c + @echo "$(CC) -DCOMPILE_UTILITY xtextencode.c" + $(FE)$(CC) -c $(CFLAGS) $(INCLUDE) -DCOMPILE_UTILITY -o util-xtextencode.o xtextencode.c + util-os.o: $(HDRS) os.c @echo "$(CC) -DCOMPILE_UTILITY os.c" $(FE)$(CC) -c $(CFLAGS) $(INCLUDE) \ diff --git a/src/src/dbfn.c b/src/src/dbfn.c index b9eef503f..1d24769db 100644 --- a/src/src/dbfn.c +++ b/src/src/dbfn.c @@ -600,7 +600,7 @@ yield = exim_dbscan(dbblock->dbptr, &key_datum, &value_datum, start, *cursor) if (!yield) exim_dbdelete_cursor(*cursor); return yield; } -#endif +#endif /*notdef*/ diff --git a/src/src/hintsdb/hints_sqlite.h b/src/src/hintsdb/hints_sqlite.h index d90c2b878..6ef214521 100644 --- a/src/src/hintsdb/hints_sqlite.h +++ b/src/src/hintsdb/hints_sqlite.h @@ -25,7 +25,13 @@ backend provider. */ /* Some text for messages */ # define EXIM_DBTYPE "sqlite3" -# /* Access functions */ +/* Utility functionss */ + +extern uschar *xtextencode(uschar *, int); +extern int xtextdecode(uschar *, uschar**); + + +/* Access functions */ static inline BOOL exim_lockfile_needed(void) @@ -86,15 +92,17 @@ sqlite3_stmt * statement; int ret; res->len = (size_t) -1; -/* fprintf(stderr, "exim_dbget__(%s)\n", s); */ +/* DEBUG(D_hints_lookup) debug_printf_indent("exim_dbget__(%s)\n", s); */ if ((ret = sqlite3_prepare_v2(dbp, CCS s, -1, &statement, NULL)) != SQLITE_OK) { -/* fprintf(stderr, "prepare fail: %s\n", sqlite3_errmsg(dbp)); */ + DEBUG(D_hints_lookup) + debug_printf_indent("prepare fail: %s\n", sqlite3_errmsg(dbp)); return FALSE; } if (sqlite3_step(statement) != SQLITE_ROW) { -/* fprintf(stderr, "step fail: %s\n", sqlite3_errmsg(dbp)); */ + /* DEBUG(D_hints_lookup) + debug_printf_indent("step fail: %s\n", sqlite3_errmsg(dbp)); */ sqlite3_finalize(statement); return FALSE; } @@ -108,7 +116,8 @@ res->data = store_get(res->len +1, GET_TAINTED); # endif memcpy(res->data, sqlite3_column_blob(statement, 0), res->len); res->data[res->len] = '\0'; -/* fprintf(stderr, "res %d bytes: '%.*s'\n", (int)res->len, (int)res->len, res->data); */ +/* DEBUG(D_hints_lookup) debug_printf_indent("res %d bytes: '%.*s'\n", + (int)res->len, (int)res->len, res->data); */ sqlite3_finalize(statement); return TRUE; } @@ -116,22 +125,30 @@ return TRUE; static inline BOOL exim_dbget(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * res) { -# define FMT "SELECT dat FROM tbl WHERE ky = '%.*s';" -uschar * qry; +# define FMT "SELECT dat FROM tbl WHERE ky = '%s';" +uschar * encoded_key, * qry; int i; BOOL ret; # ifdef COMPILE_UTILITY -/* fprintf(stderr, "exim_dbget(k len %d '%.*s')\n", (int)key->len, (int)key->len, key->data); */ -i = snprintf(NULL, 0, FMT, (int) key->len, key->data)+1; +if (!(encoded_key = xtextencode(key->data, key->len))) + return FALSE; +# else +encoded_key = xtextencode(key->data, key->len); +# endif +/* DEBUG(D_hints_lookup) debug_printf_indent("exim_dbget(k len %d '%s')\n", + (int)key->len, encoded_key); */ + +# ifdef COMPILE_UTILITY +i = snprintf(NULL, 0, FMT, encoded_key) + 1; if (!(qry = malloc(i))) return FALSE; -snprintf(CS qry, i, FMT, (int) key->len, key->data); +snprintf(CS qry, i, FMT, encoded_key); ret = exim_dbget__(dbp, qry, res); free(qry); +free(encoded_key); # else -/* fprintf(stderr, "exim_dbget(k len %d '%.*s')\n", (int)key->len, (int)key->len, key->data); */ -qry = string_sprintf(FMT, (int) key->len, key->data); +qry = string_sprintf(FMT, encoded_key); ret = exim_dbget__(dbp, qry, res); # endif @@ -140,7 +157,11 @@ return ret; } /* Note that we return claiming a duplicate record for any error. -It seem not uncommon to get a "database is locked" error. */ +It seem not uncommon to get a "database is locked" error. + +Keys are stored xtext-encoded (which is mostly readable, for plaintext). +Values are stored in a BLOB type in the DB, for which the SQL interface +is hex-encoded. */ # define EXIM_DBPUTB_OK 0 # define EXIM_DBPUTB_DUP (-1) @@ -148,8 +169,8 @@ static inline int exim_s_dbp(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data, const uschar * alt) { int hlen = data->len * 2, off = 0, res; -# define FMT "INSERT OR %s INTO tbl (ky,dat) VALUES ('%.*s', X'%.*s');" -uschar * qry; +# define FMT "INSERT OR %s INTO tbl (ky,dat) VALUES ('%s', X'%.*s');" +uschar * encoded_key, * qry; # ifdef COMPILE_UTILITY uschar * hex = malloc(hlen+1); if (!hex) return EXIM_DBPUTB_DUP; /* best we can do */ @@ -157,27 +178,37 @@ if (!hex) return EXIM_DBPUTB_DUP; /* best we can do */ uschar * hex = store_get(hlen+1, data->data); # endif +/* Encode the value for the SQL API */ + for (const uschar * s = data->data, * t = s + data->len; s < t; s++, off += 2) sprintf(CS hex + off, "%02X", *s); # ifdef COMPILE_UTILITY -res = snprintf(CS hex, 0, FMT, alt, (int) key->len, key->data, hlen, hex) +1; +if (!(encoded_key = xtextencode(key->data, key->len))) + return EXIM_DBPUTB_DUP; +res = snprintf(CS hex, 0, FMT, alt, encoded_key, hlen, hex) +1; if (!(qry = malloc(res))) return EXIM_DBPUTB_DUP; -snprintf(CS qry, res, FMT, alt, (int) key->len, key->data, hlen, hex); -/* fprintf(stderr, "exim_s_dbp(%s)\n", qry); */ +snprintf(CS qry, res, FMT, alt, encoded_key, hlen, hex); +DEBUG(D_hints_lookup) debug_printf_indent("exim_s_dbp(%s)\n", qry); + res = sqlite3_exec(dbp, CS qry, NULL, NULL, NULL); free(qry); +free(encoded_key); free(hex); + # else -qry = string_sprintf(FMT, alt, (int) key->len, key->data, hlen, hex); -/* fprintf(stderr, "exim_s_dbp(%s)\n", qry); */ +encoded_key = xtextencode(key->data, key->len); +qry = string_sprintf(FMT, alt, encoded_key, hlen, hex); +/* DEBUG(D_hints_lookup) debug_printf_indent("exim_s_dbp(%s)\n", qry); */ + res = sqlite3_exec(dbp, CS qry, NULL, NULL, NULL); -/* fprintf(stderr, "exim_s_dbp res %d\n", res); */ +/* DEBUG(D_hints_lookup) debug_printf_indent("exim_s_dbp res %d\n", res); */ # endif # ifdef COMPILE_UTILITY if (res != SQLITE_OK) - fprintf(stderr, "sqlite3_exec: %s\n", sqlite3_errmsg(dbp)); + DEBUG(D_hints_lookup) + debug_printf_indent("sqlite3_exec: %s\n", sqlite3_errmsg(dbp)); # endif return res == SQLITE_OK ? EXIM_DBPUTB_OK : EXIM_DBPUTB_DUP; @@ -189,7 +220,7 @@ return res == SQLITE_OK ? EXIM_DBPUTB_OK : EXIM_DBPUTB_DUP; static inline int exim_dbput(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) { -/* fprintf(stderr, "exim_dbput()\n"); */ +/* DEBUG(D_hints_lookup) debug_printf_indent("exim_dbput()\n"); */ (void) exim_s_dbp(dbp, key, data, US"REPLACE"); return 0; } @@ -208,22 +239,27 @@ return exim_s_dbp(dbp, key, data, US"ABORT"); static inline int exim_dbdel(EXIM_DB * dbp, EXIM_DATUM * key) { -# define FMT "DELETE FROM tbl WHERE ky = '%.*s';" -uschar * qry; +# define FMT "DELETE FROM tbl WHERE ky = '%s';" +uschar * encoded_key, * qry; int res; # ifdef COMPILE_UTILITY -res = snprintf(NULL, 0, FMT, (int) key->len, key->data) +1; /* res includes nul */ +if (!(encoded_key = xtextencode(key->data, key->len))) + return EXIM_DBPUTB_DUP; +res = snprintf(NULL, 0, FMT, encoded_key) +1; /* res includes nul */ if (!(qry = malloc(res))) return SQLITE_NOMEM; -snprintf(CS qry, res, FMT, (int) key->len, key->data); +snprintf(CS qry, res, FMT, encoded_key); res = sqlite3_exec(dbp, CS qry, NULL, NULL, NULL); free(qry); + # else -qry = string_sprintf(FMT, (int) key->len, key->data); +encoded_key = xtextencode(key->data, key->len); +qry = string_sprintf(FMT, encoded_key); res = sqlite3_exec(dbp, CS qry, NULL, NULL, NULL); + # endif -return res; +return res == SQLITE_OK ? EXIM_DBPUTB_OK : EXIM_DBPUTB_DUP; # undef FMT } @@ -245,31 +281,42 @@ return c; } /* EXIM_DBSCAN */ -/* Note that we return the (next) key, not the record value */ +/* Note that we return the (next) key, not the record value. +We allocate memory for the return. */ + static inline BOOL exim_dbscan(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * res, BOOL first, EXIM_CURSOR * cursor) { # define FMT "SELECT ky FROM tbl ORDER BY ky LIMIT 1 OFFSET %d;" uschar * qry; -int i; +EXIM_DATUM encoded_key; BOOL ret; # ifdef COMPILE_UTILITY -i = snprintf(NULL, 0, FMT, *cursor)+1; +int i = snprintf(NULL, 0, FMT, *cursor)+1; + if (!(qry = malloc(i))) return FALSE; snprintf(CS qry, i, FMT, *cursor); -/* fprintf(stderr, "exim_dbscan(%s)\n", qry); */ -ret = exim_dbget__(dbp, qry, key); +DEBUG(D_hints_lookup) debug_printf_indent("exim_dbscan(%s)\n", qry); +ret = exim_dbget__(dbp, qry, &encoded_key); free(qry); -/* fprintf(stderr, "exim_dbscan ret %c\n", ret ? 'T':'F'); */ -# else + +# else /*!COMPILE_UTILITY*/ qry = string_sprintf(FMT, *cursor); -/* fprintf(stderr, "exim_dbscan(%s)\n", qry); */ -ret = exim_dbget__(dbp, qry, key); -/* fprintf(stderr, "exim_dbscan ret %c\n", ret ? 'T':'F'); */ -# endif -if (ret) *cursor = *cursor + 1; +DEBUG(D_hints_lookup) debug_printf_indent("exim_dbscan(%s)\n", qry); +ret = exim_dbget__(dbp, qry, &encoded_key); + +# endif /*COMPILE_UTILITY*/ + +DEBUG(D_hints_lookup) + debug_printf_indent("exim_dbscan ret %c\n", ret ? 'T':'F'); + +if (ret) + { + key->len = xtextdecode(encoded_key.data, &key->data); + *cursor = *cursor + 1; + } return ret; # undef FMT } diff --git a/src/src/string.c b/src/src/string.c index aa768d03c..5583d93de 100644 --- a/src/src/string.c +++ b/src/src/string.c @@ -1626,7 +1626,7 @@ while (*fp) case 'W': /* Maybe mark up ctrls, spaces & newlines */ s = va_arg(ap, char *); - if (Ustrpbrk(s, " \n") && !IS_DEBUG(D_noutf8)) + if (s && !IS_DEBUG(D_noutf8)) { gstring * zg = NULL; int p = precision; @@ -1660,7 +1660,7 @@ while (*fp) } } if (zg) { s = CS zg->s; slen = gstring_length(zg); } - else { s = null; slen = Ustrlen(s); } + else { s = ""; slen = 0; } } else { diff --git a/src/src/xclient.c b/src/src/xclient.c index af2f287b9..af7a94b2d 100644 --- a/src/src/xclient.c +++ b/src/src/xclient.c @@ -10,7 +10,9 @@ #ifdef EXPERIMENTAL_XCLIENT -/* From https://www.postfix.org/XCLIENT_README.html I infer two generations of +/* This is a proxy protocol. + +From https://www.postfix.org/XCLIENT_README.html I infer two generations of protocol. The more recent one obviates the utility of the HELO attribute, since it mandates the proxy always sending a HELO/EHLO smtp command following (a successful) XCLIENT command, and that will carry a NELO name (which we assume, diff --git a/src/src/xtextencode.c b/src/src/xtextencode.c index fa2b26bed..71489bd4f 100644 --- a/src/src/xtextencode.c +++ b/src/src/xtextencode.c @@ -26,8 +26,9 @@ Returns: a pointer to the zero-terminated xtext string, which is in working store */ +#ifndef COMPILE_UTILITY uschar * -xtextencode(uschar *clear, int len) +xtextencode(uschar * clear, int len) { gstring * g = NULL; for(uschar ch; len > 0; len--, clear++) @@ -38,6 +39,31 @@ gstring_release_unused(g); return string_from_gstring(g); } +#else /*COMPILE_UTILITY*/ +uschar * +xtextencode(uschar * clear, int len) +{ +int enc_len = 1, i = len; /* enc_len includes space for terminating NUL */ +uschar * yield, * s; + +for (s = clear; i; i--, s++) + { + uschar ch = *s; + enc_len += ch < 33 || ch > 126 || ch == '+' || ch == '=' + ? 3 : 1; + } +if (!(yield = s = malloc(enc_len))) + return NULL; +for(uschar ch; len > 0; len--, clear++) + if ((ch = *clear) < 33 || ch > 126 || ch == '+' || ch == '=') + s += sprintf(CS s, "+%.02X", ch); + else + *s++ = ch; +*s = '\0'; +return yield; +} + +#endif /*COMPILE_UTILITY*/ /************************************************* * Decode byte-string in xtext * @@ -64,24 +90,29 @@ int xtextdecode(uschar * code, uschar ** ptr) { int x; +#ifdef COMPILE_UTILITY +uschar * result = malloc(Ustrlen(code) + 1); +#else uschar * result = store_get(Ustrlen(code) + 1, code); -*ptr = result; +#endif -while ((x = (*code++)) != 0) +*ptr = result; +while ((x = (*code++))) { if (x < 33 || x > 127 || x == '=') return -1; if (x == '+') { - register int y; + int y; if (!isxdigit((x = (*code++)))) return -1; y = ((isdigit(x))? x - '0' : (tolower(x) - 'a' + 10)) << 4; if (!isxdigit((x = (*code++)))) return -1; *result++ = y | ((isdigit(x))? x - '0' : (tolower(x) - 'a' + 10)); } - else *result++ = x; + else + *result++ = x; } -*result = 0; +*result = '\0'; return result - *ptr; } diff --git a/test/confs/0195 b/test/confs/0195 index 46f1029ef..fb69964d1 100644 --- a/test/confs/0195 +++ b/test/confs/0195 @@ -7,7 +7,7 @@ primary_hostname = myhost.test.ex # ----- Main settings ----- domainlist local_domains = test.ex : *.test.ex - +queue_run_in_order # ----- Routers ----- diff --git a/test/scripts/0000-Basic/0195 b/test/scripts/0000-Basic/0195 index 3c487ded1..5ce0b3e94 100644 --- a/test/scripts/0000-Basic/0195 +++ b/test/scripts/0000-Basic/0195 @@ -1,21 +1,30 @@ # retry times on local addresses +# +# All three should be deferred exim -odi userx usery userz This is a test message. **** +# retry records should have been created dump retry sleep 1 +# retry record should mean no delivery tried exim -R usery **** +# should still be all three records dump retry sleep 1 +# insert another message; should be deferred only on first delivery try exim -odi usery one-defer: set second message **** +# all three records should still be there dump retry sleep 1 +# msg2 has had it's single-defer so should get delivered exim -Mc $msg2 **** +# the record for that localpart should have been removed as a result dump retry sleep 1 exim -q