X-Git-Url: https://git.exim.org/exim.git/blobdiff_plain/fba5586e6d47f55e024e97681c166e857c4f3d1c..HEAD:/src/src/deliver.c diff --git a/src/src/deliver.c b/src/src/deliver.c index 9d459167e..0ba5f81b0 100644 --- a/src/src/deliver.c +++ b/src/src/deliver.c @@ -2,7 +2,7 @@ * Exim - an Internet mail transport agent * *************************************************/ -/* Copyright (c) The Exim Maintainers 2020 - 2023 */ +/* Copyright (c) The Exim Maintainers 2020 - 2024 */ /* Copyright (c) University of Cambridge 1995 - 2018 */ /* See the file NOTICE for conditions of use and distribution. */ /* SPDX-License-Identifier: GPL-2.0-or-later */ @@ -27,7 +27,7 @@ typedef struct pardata { int transport_count; /* returned transport count value */ BOOL done; /* no more data needed */ uschar *msg; /* error message */ - uschar *return_path; /* return_path for these addresses */ + const uschar *return_path; /* return_path for these addresses */ } pardata; /* Values for the process_recipients variable */ @@ -77,7 +77,7 @@ static pardata *parlist = NULL; static struct pollfd *parpoll; static int return_count; static uschar *frozen_info = US""; -static uschar *used_return_path = NULL; +static const uschar * used_return_path = NULL; @@ -144,7 +144,7 @@ Returns: a pointer to an initialized address_item */ address_item * -deliver_make_addr(uschar *address, BOOL copy) +deliver_make_addr(const uschar * address, BOOL copy) { address_item * addr = store_get(sizeof(address_item), GET_UNTAINTED); *addr = address_defaults; @@ -575,7 +575,7 @@ Returns: TRUE or FALSE */ static BOOL -same_strings(uschar *one, uschar *two) +same_strings(const uschar * one, const uschar * two) { if (one == two) return TRUE; /* Includes the case where both NULL */ if (!one || !two) return FALSE; @@ -680,7 +680,7 @@ else if (testflag(addr, af_homonym)) { if (addr->transport) tree_add_nonrecipient( - string_sprintf("%s/%s", addr->unique + 3, addr->transport->name)); + string_sprintf("%s/%s", addr->unique + 3, addr->transport->drinst.name)); } /* Non-homonymous child address */ @@ -791,7 +791,7 @@ g = string_append(g, 3, US" [", h->address, US"]"); if (LOGGING(outgoing_port)) g = string_fmt_append(g, ":%d", h->port); -if (continue_sequence > 1) /*XXX this is wrong for a dropped proxyconn. Would have to pass back from transport */ +if (testflag(addr, af_cont_conn)) g = string_catn(g, US"*", 1); #ifdef SUPPORT_SOCKS @@ -859,9 +859,10 @@ Return: string expansion from listener, or NULL */ uschar * -event_raise(uschar * action, const uschar * event, uschar * ev_data, int * errnop) +event_raise(const uschar * action, const uschar * event, const uschar * ev_data, + int * errnop) { -uschar * s; +const uschar * s; if (action) { DEBUG(D_deliver) @@ -872,7 +873,7 @@ if (action) event_name = event; event_data = ev_data; - if (!(s = expand_string(action)) && *expand_string_message) + if (!(s = expand_cstring(action)) && *expand_string_message) log_write(0, LOG_MAIN|LOG_PANIC, "failed to expand event_action %s in %s: %s\n", event, transport_name ? transport_name : US"main", expand_string_message); @@ -880,7 +881,8 @@ if (action) event_name = event_data = NULL; /* If the expansion returns anything but an empty string, flag for - the caller to modify his normal processing + the caller to modify his normal processing. Copy the string to + de-const it. */ if (s && *s) { @@ -888,7 +890,7 @@ if (action) debug_printf("Event(%s): event_action returned \"%s\"\n", event, s); if (errnop) *errnop = ERRNO_EVENT; - return s; + return string_copy(s); } } return NULL; @@ -898,12 +900,14 @@ void msg_event_raise(const uschar * event, const address_item * addr) { const uschar * save_domain = deliver_domain; -uschar * save_local = deliver_localpart; +const uschar * save_local = deliver_localpart; const uschar * save_host = deliver_host; const uschar * save_address = deliver_host_address; +const uschar * save_rn = router_name; +const uschar * save_tn = transport_name; const int save_port = deliver_host_port; -router_name = addr->router ? addr->router->name : NULL; +router_name = addr->router ? addr->router->drinst.name : NULL; deliver_domain = addr->domain; deliver_localpart = addr->local_part; deliver_host = addr->host_used ? addr->host_used->name : NULL; @@ -921,13 +925,14 @@ if (!addr->transport) } else { - transport_name = addr->transport->name; + const uschar * dr_name = addr->transport->drinst.driver_name; + transport_name = addr->transport->drinst.name; (void) event_raise(addr->transport->event_action, event, addr->host_used - || Ustrcmp(addr->transport->driver_name, "smtp") == 0 - || Ustrcmp(addr->transport->driver_name, "lmtp") == 0 - || Ustrcmp(addr->transport->driver_name, "autoreply") == 0 + || Ustrcmp(dr_name, "smtp") == 0 + || Ustrcmp(dr_name, "lmtp") == 0 + || Ustrcmp(dr_name, "autoreply") == 0 ? addr->message : NULL, NULL); } @@ -937,7 +942,8 @@ deliver_host_address = save_address; deliver_host = save_host; deliver_localpart = save_local; deliver_domain = save_domain; -router_name = transport_name = NULL; +router_name = save_rn; +transport_name = save_tn; } #endif /*DISABLE_EVENT*/ @@ -950,13 +956,13 @@ router_name = transport_name = NULL; * Generate local part for logging * *************************************************/ -static uschar * -string_get_lpart_sub(const address_item * addr, uschar * s) +static const uschar * +string_get_lpart_sub(const address_item * addr, const uschar * s) { #ifdef SUPPORT_I18N if (testflag(addr, af_utf8_downcvt)) { - uschar * t = string_localpart_utf8_to_alabel(s, NULL); + const uschar * t = string_localpart_utf8_to_alabel(s, NULL); return t ? t : s; /* t is NULL on a failed conversion */ } #endif @@ -975,7 +981,7 @@ Returns: the new value of the buffer pointer static gstring * string_get_localpart(address_item * addr, gstring * yield) { -uschar * s; +const uschar * s; if (testflag(addr, af_include_affixes) && (s = addr->prefix)) yield = string_cat(yield, string_get_lpart_sub(addr, s)); @@ -1029,7 +1035,8 @@ so won't necessarily look like a path. Add extra text for this case. */ if ( testflag(addr, af_pfr) || ( success && addr->router && addr->router->log_as_local - && addr->transport && addr->transport->info->local + && addr->transport + && ((transport_info *)addr->transport->drinst.info)->local ) ) { if (testflag(addr, af_file) && addr->local_part[0] != '/') @@ -1173,16 +1180,16 @@ if (msg) /* For a delivery from a system filter, there may not be a router */ if (addr->router) - g = string_append(g, 2, US" R=", addr->router->name); + g = string_append(g, 2, US" R=", addr->router->drinst.name); -g = string_append(g, 2, US" T=", addr->transport->name); +g = string_append(g, 2, US" T=", addr->transport->drinst.name); if (LOGGING(delivery_size)) g = string_fmt_append(g, " S=%d", transport_count); /* Local delivery */ -if (addr->transport->info->local) +if (((transport_info *)addr->transport->drinst.info)->local) { if (addr->host_list) g = string_append(g, 2, US" H=", addr->host_list->name); @@ -1245,11 +1252,20 @@ else g = string_catn(g, US" K", 2); } +#ifndef DISABLE_DKIM + if (addr->dkim_used && LOGGING(dkim_verbose)) + { + g = string_catn(g, US" DKIM=", 6); + g = string_cat(g, addr->dkim_used); + } +#endif + /* confirmation message (SMTP (host_used) and LMTP (driver_name)) */ if ( LOGGING(smtp_confirmation) && addr->message - && (addr->host_used || Ustrcmp(addr->transport->driver_name, "lmtp") == 0) + && ( addr->host_used + || Ustrcmp(addr->transport->drinst.driver_name, "lmtp") == 0) ) { unsigned lim = big_buffer_size < 1024 ? big_buffer_size : 1024; @@ -1317,7 +1333,7 @@ so nothing has been done at all, both variables contain null strings. */ if (driver_name) { if (driver_kind[1] == 't' && addr->router) - g = string_append(g, 2, US" R=", addr->router->name); + g = string_append(g, 2, US" R=", addr->router->drinst.name); g = string_fmt_append(g, " %c=%s", toupper(driver_kind[1]), driver_name); } else if (driver_kind) @@ -1394,9 +1410,9 @@ if (used_return_path && LOGGING(return_path_on_delivery)) g = string_append(g, 3, US" P=<", used_return_path, US">"); if (addr->router) - g = string_append(g, 2, US" R=", addr->router->name); + g = string_append(g, 2, US" R=", addr->router->drinst.name); if (addr->transport) - g = string_append(g, 2, US" T=", addr->transport->name); + g = string_append(g, 2, US" T=", addr->transport->drinst.name); if (addr->host_used) g = d_hostlog(g, addr); @@ -1465,7 +1481,7 @@ if (driver_type == EXIM_DTYPE_TRANSPORT) { if (addr->transport) { - driver_name = addr->transport->name; + driver_name = addr->transport->drinst.name; driver_kind = US" transport"; f.disable_logging = addr->transport->disable_logging; } @@ -1475,7 +1491,7 @@ else if (driver_type == EXIM_DTYPE_ROUTER) { if (addr->router) { - driver_name = addr->router->name; + driver_name = addr->router->drinst.name; driver_kind = US" router"; f.disable_logging = addr->router->disable_logging; } @@ -1519,7 +1535,7 @@ if (addr->return_file >= 0 && addr->return_filename) if (fstat(addr->return_file, &statbuf) == 0 && statbuf.st_size > 0) { - transport_instance *tb = addr->transport; + transport_instance * tb = addr->transport; /* Handle logging options */ @@ -1528,11 +1544,11 @@ if (addr->return_file >= 0 && addr->return_filename) || result == DEFER && tb->log_defer_output ) { - uschar *s; - FILE *f = Ufopen(addr->return_filename, "rb"); + uschar * s; + FILE * f = Ufopen(addr->return_filename, "rb"); if (!f) log_write(0, LOG_MAIN|LOG_PANIC, "failed to open %s to log output " - "from %s transport: %s", addr->return_filename, tb->name, + "from %s transport: %s", addr->return_filename, tb->drinst.name, strerror(errno)); else if ((s = US Ufgets(big_buffer, big_buffer_size, f))) @@ -1543,7 +1559,7 @@ if (addr->return_file >= 0 && addr->return_filename) *p = 0; sp = string_printing(big_buffer); log_write(0, LOG_MAIN, "<%s>: %s transport output: %s", - addr->address, tb->name, sp); + addr->address, tb->drinst.name, sp); } (void)fclose(f); } @@ -1576,12 +1592,6 @@ if (addr->return_file >= 0 && addr->return_filename) (void)close(addr->return_file); } -/* Check if the transport notifed continue-conn status explicitly, and -update our knowlege. */ - -if (testflag(addr, af_new_conn)) continue_sequence = 1; -else if (testflag(addr, af_cont_conn)) continue_sequence++; - /* The success case happens only after delivery by a transport. */ if (result == OK) @@ -1850,8 +1860,9 @@ if (tp->gid_set) } else if (tp->expand_gid) { - if (!route_find_expanded_group(tp->expand_gid, tp->name, US"transport", gidp, - &(addr->message))) + GET_OPTION("group"); + if (!route_find_expanded_group(tp->expand_gid, tp->drinst.name, US"transport", + gidp, &addr->message)) { common_error(FALSE, addr, ERRNO_GIDFAIL, NULL); return FALSE; @@ -1877,8 +1888,9 @@ it does not provide a passwd value from which a gid can be taken. */ else if (tp->expand_uid) { struct passwd *pw; - if (!route_find_expanded_user(tp->expand_uid, tp->name, US"transport", &pw, - uidp, &(addr->message))) + GET_OPTION("user"); + if (!route_find_expanded_user(tp->expand_uid, tp->drinst.name, US"transport", + &pw, uidp, &(addr->message))) { common_error(FALSE, addr, ERRNO_UIDFAIL, NULL); return FALSE; @@ -1931,7 +1943,7 @@ a uid, it must also provide a gid. */ if (!gid_set) { common_error(TRUE, addr, ERRNO_GIDFAIL, US"User set without group for " - "%s transport", tp->name); + "%s transport", tp->drinst.name); return FALSE; } @@ -1946,7 +1958,7 @@ nuname = check_never_users(*uidp, never_users) if (nuname) { common_error(TRUE, addr, ERRNO_UIDFAIL, US"User %ld set for %s transport " - "is on the %s list", (long int)(*uidp), tp->name, nuname); + "is on the %s list", (long int)(*uidp), tp->drinst.name, nuname); return FALSE; } @@ -1980,6 +1992,7 @@ check_message_size(transport_instance *tp, address_item *addr) int rc = OK; int size_limit; +GET_OPTION("message_size_limit"); deliver_set_expansions(addr); size_limit = expand_string_integer(tp->message_size_limit, TRUE); deliver_set_expansions(NULL); @@ -1989,9 +2002,9 @@ if (expand_string_message) rc = DEFER; addr->message = size_limit == -1 ? string_sprintf("failed to expand message_size_limit " - "in %s transport: %s", tp->name, expand_string_message) + "in %s transport: %s", tp->drinst.name, expand_string_message) : string_sprintf("invalid message_size_limit " - "in %s transport: %s", tp->name, expand_string_message); + "in %s transport: %s", tp->drinst.name, expand_string_message); } else if (size_limit > 0 && message_size > size_limit) { @@ -2028,13 +2041,14 @@ static BOOL previously_transported(address_item *addr, BOOL testing) { uschar * s = string_sprintf("%s/%s", - addr->unique + (testflag(addr, af_homonym)? 3:0), addr->transport->name); + addr->unique + (testflag(addr, af_homonym) ? 3:0), + addr->transport->drinst.name); if (tree_search(tree_nonrecipients, s) != 0) { DEBUG(D_deliver|D_route|D_transport) debug_printf("%s was previously delivered (%s transport): discarded\n", - addr->address, addr->transport->name); + addr->address, addr->transport->drinst.name); if (!testing) child_done(addr, tod_stamp(tod_log)); return TRUE; } @@ -2132,7 +2146,8 @@ int pfd[2]; pid_t pid; uschar *working_directory; address_item *addr2; -transport_instance *tp = addr->transport; +transport_instance * tp = addr->transport; +const uschar * trname = tp->drinst.name; /* Set up the return path from the errors or sender address. If the transport has its own return path setting, expand it and replace the existing value. */ @@ -2142,6 +2157,7 @@ if(addr->prop.errors_address) else return_path = sender_address; +GET_OPTION("return_path"); if (tp->return_path) { uschar * new_return_path = expand_string(tp->return_path); @@ -2151,7 +2167,7 @@ if (tp->return_path) { common_error(TRUE, addr, ERRNO_EXPANDFAIL, US"Failed to expand return path \"%s\" in %s transport: %s", - tp->return_path, tp->name, expand_string_message); + tp->return_path, trname, expand_string_message); return; } } @@ -2171,6 +2187,7 @@ if (!findugid(addr, tp, &uid, &gid, &use_initgroups)) return; home directory set in the address may already be expanded; a flag is set to indicate that. In other cases we must expand it. */ +GET_OPTION("home_directory"); if ( (deliver_home = tp->home_dir) /* Set in transport, or */ || ( (deliver_home = addr->home_dir) /* Set in address and */ && !testflag(addr, af_home_expanded) /* not expanded */ @@ -2181,14 +2198,14 @@ if ( (deliver_home = tp->home_dir) /* Set in transport, or */ if (!(deliver_home = expand_string(rawhome))) { common_error(TRUE, addr, ERRNO_EXPANDFAIL, US"home directory \"%s\" failed " - "to expand for %s transport: %s", rawhome, tp->name, + "to expand for %s transport: %s", rawhome, trname, expand_string_message); return; } if (*deliver_home != '/') { common_error(TRUE, addr, ERRNO_NOTABSOLUTE, US"home directory path \"%s\" " - "is not absolute for %s transport", deliver_home, tp->name); + "is not absolute for %s transport", deliver_home, trname); return; } } @@ -2200,6 +2217,7 @@ all users have access. It is necessary to be in a visible directory for some operating systems when running pipes, as some commands (e.g. "rm" under Solaris 2.5) require this. */ +GET_OPTION("current_directory"); working_directory = tp->current_dir ? tp->current_dir : addr->current_dir; if (working_directory) { @@ -2207,14 +2225,14 @@ if (working_directory) if (!(working_directory = expand_string(raw))) { common_error(TRUE, addr, ERRNO_EXPANDFAIL, US"current directory \"%s\" " - "failed to expand for %s transport: %s", raw, tp->name, + "failed to expand for %s transport: %s", raw, trname, expand_string_message); return; } if (*working_directory != '/') { common_error(TRUE, addr, ERRNO_NOTABSOLUTE, US"current directory path " - "\"%s\" is not absolute for %s transport", working_directory, tp->name); + "\"%s\" is not absolute for %s transport", working_directory, trname); return; } } @@ -2234,12 +2252,12 @@ if ( !shadowing addr->return_filename = spool_fname(US"msglog", message_subdir, message_id, - string_sprintf("-%d-%d", getpid(), return_count++)); + string_sprintf("-%ld-%d", (long)getpid(), return_count++)); if ((addr->return_file = open_msglog_file(addr->return_filename, 0400, &error)) < 0) { common_error(TRUE, addr, errno, US"Unable to %s file for %s transport " - "to return message: %s", error, tp->name, strerror(errno)); + "to return message: %s", error, trname, strerror(errno)); return; } } @@ -2336,7 +2354,7 @@ if ((pid = exim_fork(US"delivery-local")) == 0) FD_CLOEXEC); exim_setugid(uid, gid, use_initgroups, string_sprintf("local delivery to %s <%s> transport=%s", addr->local_part, - addr->address, addr->transport->name)); + addr->address, addr->transport->drinst.name)); DEBUG(D_deliver) { @@ -2360,14 +2378,14 @@ if ((pid = exim_fork(US"delivery-local")) == 0) { BOOL ok = TRUE; set_process_info("delivering %s to %s using %s", message_id, - addr->local_part, tp->name); + addr->local_part, trname); /* Setting these globals in the subprocess means we need never clear them */ - transport_name = tp->name; - if (addr->router) router_name = addr->router->name; - driver_srcfile = tp->srcfile; - driver_srcline = tp->srcline; + transport_name = trname; + if (addr->router) router_name = addr->router->drinst.name; + driver_srcfile = tp->drinst.srcfile; + driver_srcline = tp->drinst.srcline; /* If a transport filter has been specified, set up its argument list. Any errors will get put into the address, and FALSE yielded. */ @@ -2383,8 +2401,9 @@ if ((pid = exim_fork(US"delivery-local")) == 0) if (ok) { + transport_info * ti = tp->drinst.info; debug_print_string(tp->debug_string); - replicate = !(tp->info->code)(addr->transport, addr); + replicate = !(ti->code)(addr->transport, addr); } } @@ -2444,8 +2463,7 @@ if ((pid = exim_fork(US"delivery-local")) == 0) and close the pipe we were writing down before exiting. */ (void)close(pfd[pipe_write]); - search_tidyup(); - exit(EXIT_SUCCESS); + exim_exit(EXIT_SUCCESS); } /* Back in the main process: panic if the fork did not succeed. This seems @@ -2540,7 +2558,7 @@ if (!shadowing) if (addr2->transport_return == OK) { if (testflag(addr2, af_homonym)) - sprintf(CS big_buffer, "%.500s/%s\n", addr2->unique + 3, tp->name); + sprintf(CS big_buffer, "%.500s/%s\n", addr2->unique + 3, trname); else sprintf(CS big_buffer, "%.500s\n", addr2->unique); @@ -2575,7 +2593,7 @@ while ((rc = wait(&status)) != pid) if (rc < 0 && errno == ECHILD) /* Process has vanished */ { log_write(0, LOG_MAIN, "%s transport process vanished unexpectedly", - addr->transport->driver_name); + addr->transport->drinst.driver_name); status = 0; break; } @@ -2589,7 +2607,7 @@ if ((status & 0xffff) != 0) addr->special_action = SPECIAL_FREEZE; log_write(0, LOG_MAIN|LOG_PANIC, "%s transport process returned non-zero " "status 0x%04x: %s %d", - addr->transport->driver_name, + addr->transport->drinst.driver_name, status, msb == 0 ? "terminated by signal" : "exit code", code); @@ -2597,36 +2615,40 @@ if ((status & 0xffff) != 0) /* If SPECIAL_WARN is set in the top address, send a warning message. */ -if (addr->special_action == SPECIAL_WARN && addr->transport->warn_message) +if (addr->special_action == SPECIAL_WARN) { - int fd; - uschar *warn_message; - pid_t pid; + uschar * warn_message = addr->transport->warn_message; + GET_OPTION("quota_warn_message"); + if (warn_message) + { + int fd; + pid_t pid; - DEBUG(D_deliver) debug_printf("Warning message requested by transport\n"); + DEBUG(D_deliver) debug_printf("Warning message requested by transport\n"); - if (!(warn_message = expand_string(addr->transport->warn_message))) - log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand \"%s\" (warning " - "message for %s transport): %s", addr->transport->warn_message, - addr->transport->name, expand_string_message); + if (!(warn_message = expand_string(warn_message))) + log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand \"%s\" (warning " + "message for %s transport): %s", addr->transport->warn_message, + addr->transport->drinst.name, expand_string_message); - else if ((pid = child_open_exim(&fd, US"tpt-warning-message")) > 0) - { - FILE *f = fdopen(fd, "wb"); - if (errors_reply_to && !contains_header(US"Reply-To", warn_message)) - fprintf(f, "Reply-To: %s\n", errors_reply_to); - fprintf(f, "Auto-Submitted: auto-replied\n"); - if (!contains_header(US"From", warn_message)) - moan_write_from(f); - fprintf(f, "%s", CS warn_message); + else if ((pid = child_open_exim(&fd, US"tpt-warning-message")) > 0) + { + FILE * f = fdopen(fd, "wb"); + if (errors_reply_to && !contains_header(US"Reply-To", warn_message)) + fprintf(f, "Reply-To: %s\n", errors_reply_to); + fprintf(f, "Auto-Submitted: auto-replied\n"); + if (!contains_header(US"From", warn_message)) + moan_write_from(f); + fprintf(f, "%s", CS warn_message); - /* Close and wait for child process to complete, without a timeout. */ + /* Close and wait for child process to complete, without a timeout. */ - (void)fclose(f); - (void)child_close(pid, 0); - } + (void)fclose(f); + (void)child_close(pid, 0); + } - addr->special_action = SPECIAL_NONE; + addr->special_action = SPECIAL_NONE; + } } } @@ -2640,28 +2662,30 @@ the key for the hints database used for the concurrency count. */ static BOOL tpt_parallel_check(transport_instance * tp, address_item * addr, uschar ** key) { +const uschar * trname = tp->drinst.name; unsigned max_parallel; +GET_OPTION("max_parallel"); if (!tp->max_parallel) return FALSE; max_parallel = (unsigned) expand_string_integer(tp->max_parallel, TRUE); if (expand_string_message) { log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand max_parallel option " - "in %s transport (%s): %s", tp->name, addr->address, + "in %s transport (%s): %s", trname, addr->address, expand_string_message); return TRUE; } if (max_parallel > 0) { - uschar * serialize_key = string_sprintf("tpt-serialize-%s", tp->name); + uschar * serialize_key = string_sprintf("tpt-serialize-%s", trname); if (!enq_start(serialize_key, max_parallel)) { address_item * next; DEBUG(D_transport) debug_printf("skipping tpt %s because concurrency limit %u reached\n", - tp->name, max_parallel); + trname, max_parallel); do { next = addr->next; @@ -2695,8 +2719,7 @@ Returns: Nothing static void do_local_deliveries(void) { -open_db dbblock; -open_db *dbm_file = NULL; +open_db dbblock, * dbm_file = NULL; time_t now = time(NULL); /* Loop until we have exhausted the supply of local deliveries */ @@ -2708,8 +2731,9 @@ while (addr_local) address_item *addr2, *addr3, *nextaddr; int logflags = LOG_MAIN; int logchar = f.dont_deliver? '*' : '='; - transport_instance *tp; + transport_instance * tp; uschar * serialize_key = NULL; + const uschar * trname; /* Pick the first undelivered address off the chain */ @@ -2727,11 +2751,12 @@ while (addr_local) logflags |= LOG_PANIC; f.disable_logging = FALSE; /* Jic */ addr->message = addr->router - ? string_sprintf("No transport set by %s router", addr->router->name) + ? string_sprintf("No transport set by %s router", addr->router->drinst.name) : US"No transport set by system filter"; post_process_one(addr, DEFER, logflags, EXIM_DTYPE_TRANSPORT, 0); continue; } + trname = tp->drinst.name; /* Check that this base address hasn't previously been delivered to this transport. The check is necessary at this point to handle homonymic addresses @@ -2765,6 +2790,7 @@ while (addr_local) /* Expand the batch_id string for comparison with other addresses. Expansion failure suppresses batching. */ + GET_OPTION("batch_id"); if (tp->batch_id) { deliver_set_expansions(addr); @@ -2773,7 +2799,7 @@ while (addr_local) if (!batch_id) { log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand batch_id option " - "in %s transport (%s): %s", tp->name, addr->address, + "in %s transport (%s): %s", trname, addr->address, expand_string_message); batch_count = tp->batch_max; } @@ -2819,17 +2845,18 @@ while (addr_local) if (ok && batch_id) { - uschar *bid; - address_item *save_nextnext = next->next; + uschar * bid; + address_item * save_nextnext = next->next; next->next = NULL; /* Expansion for a single address */ deliver_set_expansions(next); next->next = save_nextnext; + GET_OPTION("batch_id"); bid = expand_string(tp->batch_id); deliver_set_expansions(NULL); if (!bid) { log_write(0, LOG_MAIN|LOG_PANIC, "Failed to expand batch_id option " - "in %s transport (%s): %s", tp->name, next->address, + "in %s transport (%s): %s", trname, next->address, expand_string_message); ok = FALSE; } @@ -2879,7 +2906,12 @@ while (addr_local) of these checks, rather than for all local deliveries, because some local deliveries (e.g. to pipes) can take a substantial time. */ - if (!(dbm_file = dbfn_open(US"retry", O_RDONLY, &dbblock, FALSE, TRUE))) + if (continue_retry_db && continue_retry_db != (open_db *)-1) + { + DEBUG(D_hints_lookup) debug_printf("using cached retry hintsdb handle\n"); + dbm_file = continue_retry_db; + } + else if (!(dbm_file = dbfn_open(US"retry", O_RDONLY, &dbblock, FALSE, TRUE))) DEBUG(D_deliver|D_retry|D_hints_lookup) debug_printf("no retry data available\n"); @@ -2888,61 +2920,66 @@ while (addr_local) while (addr2) { BOOL ok = TRUE; /* to deliver this address */ - uschar *retry_key; - - /* Set up the retry key to include the domain or not, and change its - leading character from "R" to "T". Must make a copy before doing this, - because the old key may be pointed to from a "delete" retry item after - a routing delay. */ - retry_key = string_copy( - tp->retry_use_local_part ? addr2->address_retry_key : - addr2->domain_retry_key); - *retry_key = 'T'; + if (f.queue_2stage) + { + DEBUG(D_deliver) + debug_printf_indent("no router retry check (ph1 qrun)\n"); + } + else + { + /* Set up the retry key to include the domain or not, and change its + leading character from "R" to "T". Must make a copy before doing this, + because the old key may be pointed to from a "delete" retry item after + a routing delay. */ + uschar * retry_key = string_copy(tp->retry_use_local_part + ? addr2->address_retry_key : addr2->domain_retry_key); + *retry_key = 'T'; - /* Inspect the retry data. If there is no hints file, delivery happens. */ + /* Inspect the retry data. If there is no hints file, delivery happens. */ - if (dbm_file) - { - dbdata_retry *retry_record = dbfn_read(dbm_file, retry_key); + if (dbm_file) + { + dbdata_retry * retry_record = dbfn_read(dbm_file, retry_key); - /* If there is no retry record, delivery happens. If there is, - remember it exists so it can be deleted after a successful delivery. */ + /* If there is no retry record, delivery happens. If there is, + remember it exists so it can be deleted after a successful delivery. */ - if (retry_record) - { - setflag(addr2, af_lt_retry_exists); + if (retry_record) + { + setflag(addr2, af_lt_retry_exists); - /* A retry record exists for this address. If queue running and not - forcing, inspect its contents. If the record is too old, or if its - retry time has come, or if it has passed its cutoff time, delivery - will go ahead. */ + /* A retry record exists for this address. If queue running and not + forcing, inspect its contents. If the record is too old, or if its + retry time has come, or if it has passed its cutoff time, delivery + will go ahead. */ - DEBUG(D_retry) - { - debug_printf("retry record exists: age=%s ", - readconf_printtime(now - retry_record->time_stamp)); - debug_printf("(max %s)\n", readconf_printtime(retry_data_expire)); - debug_printf(" time to retry = %s expired = %d\n", - readconf_printtime(retry_record->next_try - now), - retry_record->expired); - } + DEBUG(D_retry) + { + debug_printf("retry record exists: age=%s ", + readconf_printtime(now - retry_record->time_stamp)); + debug_printf("(max %s)\n", readconf_printtime(retry_data_expire)); + debug_printf(" time to retry = %s expired = %d\n", + readconf_printtime(retry_record->next_try - now), + retry_record->expired); + } - if (f.queue_running && !f.deliver_force) - { - ok = (now - retry_record->time_stamp > retry_data_expire) - || (now >= retry_record->next_try) - || retry_record->expired; + if (f.queue_running && !f.deliver_force) + { + ok = (now - retry_record->time_stamp > retry_data_expire) + || (now >= retry_record->next_try) + || retry_record->expired; - /* If we haven't reached the retry time, there is one more check - to do, which is for the ultimate address timeout. */ + /* If we haven't reached the retry time, there is one more check + to do, which is for the ultimate address timeout. */ - if (!ok) - ok = retry_ultimate_address_timeout(retry_key, addr2->domain, - retry_record, now); - } - } - else DEBUG(D_retry) debug_printf("no retry record exists\n"); + if (!ok) + ok = retry_ultimate_address_timeout(retry_key, addr2->domain, + retry_record, now); + } + } + else DEBUG(D_retry) debug_printf("no retry record exists\n"); + } } /* This address is to be delivered. Leave it on the chain. */ @@ -2968,7 +3005,11 @@ while (addr_local) } } - if (dbm_file) dbfn_close(dbm_file); + if (dbm_file) + if (dbm_file != continue_retry_db) + { dbfn_close(dbm_file); dbm_file = NULL; } + else + DEBUG(D_hints_lookup) debug_printf("retaining retry hintsdb handle\n"); /* If there are no addresses left on the chain, they all deferred. Loop for the next set of addresses. */ @@ -3016,15 +3057,15 @@ while (addr_local) if ( tp->shadow && ( !tp->shadow_condition - || expand_check_condition(tp->shadow_condition, tp->name, US"transport") + || expand_check_condition(tp->shadow_condition, trname, US"transport") ) ) { - transport_instance *stp; - address_item *shadow_addr = NULL; - address_item **last = &shadow_addr; + transport_instance * stp; + address_item * shadow_addr = NULL; + address_item ** last = &shadow_addr; - for (stp = transports; stp; stp = stp->next) - if (Ustrcmp(stp->name, tp->shadow) == 0) break; + for (stp = transports; stp; stp = stp->drinst.next) + if (Ustrcmp(stp->drinst.name, tp->shadow) == 0) break; if (!stp) log_write(0, LOG_MAIN|LOG_PANIC, "shadow transport \"%s\" not found ", @@ -3054,6 +3095,7 @@ while (addr_local) if (shadow_addr) { + const uschar * s_trname = stp->drinst.name; int save_count = transport_count; DEBUG(D_deliver|D_transport) @@ -3065,8 +3107,8 @@ while (addr_local) int sresult = shadow_addr->transport_return; *(uschar **)shadow_addr->shadow_message = sresult == OK - ? string_sprintf(" ST=%s", stp->name) - : string_sprintf(" ST=%s (%s%s%s)", stp->name, + ? string_sprintf(" ST=%s", s_trname) + : string_sprintf(" ST=%s (%s%s%s)", s_trname, shadow_addr->basic_errno <= 0 ? US"" : US strerror(shadow_addr->basic_errno), @@ -3081,7 +3123,7 @@ while (addr_local) DEBUG(D_deliver|D_transport) debug_printf("%s shadow transport returned %s for %s\n", - stp->name, rc_to_string(sresult), shadow_addr->address); + s_trname, rc_to_string(sresult), shadow_addr->address); } DEBUG(D_deliver|D_transport) @@ -3110,7 +3152,7 @@ while (addr_local) DEBUG(D_deliver|D_transport) debug_printf("%s transport returned %s for %s\n", - tp->name, rc_to_string(result), addr2->address); + trname, rc_to_string(result), addr2->address); /* If there is a retry_record, or if delivery is deferred, build a retry item for setting a new retry time or deleting the old retry record from @@ -3121,7 +3163,7 @@ while (addr_local) if (result == DEFER || testflag(addr2, af_lt_retry_exists)) { int flags = result == DEFER ? 0 : rf_delete; - uschar *retry_key = string_copy(tp->retry_use_local_part + uschar * retry_key = string_copy(tp->retry_use_local_part ? addr2->address_retry_key : addr2->domain_retry_key); *retry_key = 'T'; retry_add_item(addr2, retry_key, flags); @@ -3296,6 +3338,9 @@ int fd = p->fd; uschar *msg = p->msg; BOOL done = p->done; +continue_hostname = NULL; +continue_transport = NULL; + /* Loop through all items, reading from the pipe when necessary. The pipe used to be non-blocking. But I do not see a reason for using non-blocking I/O here, as the preceding poll() tells us, if data is available for reading. @@ -3313,8 +3358,8 @@ same channel (pipe). */ -DEBUG(D_deliver) debug_printf("reading pipe for subprocess %d (%s)\n", - (int)p->pid, eop? "ended" : "not ended yet"); +DEBUG(D_deliver) debug_printf("reading pipe for subprocess %ld (%s)\n", + (long)p->pid, eop? "ended" : "not ended yet"); while (!done) { @@ -3326,8 +3371,9 @@ while (!done) size_t required = PIPE_HEADER_SIZE; /* first the pipehaeder, later the data */ ssize_t got; - DEBUG(D_deliver) debug_printf( - "expect %lu bytes (pipeheader) from tpt process %d\n", (u_long)required, pid); + DEBUG(D_deliver) + debug_printf("expect %lu bytes (pipeheader) from tpt process %ld\n", + (u_long)required, (long)pid); /* We require(!) all the PIPE_HEADER_SIZE bytes here, as we know, they're written in a timely manner, so waiting for the write shouldn't hurt a lot. @@ -3337,16 +3383,16 @@ while (!done) if ((got = readn(fd, pipeheader, required)) != required) { msg = string_sprintf("got " SSIZE_T_FMT " of %d bytes (pipeheader) " - "from transport process %d for transport %s", - got, PIPE_HEADER_SIZE, pid, addr->transport->driver_name); + "from transport process %ld for transport %s", + got, PIPE_HEADER_SIZE, (long)pid, addr->transport->drinst.driver_name); done = TRUE; break; } pipeheader[PIPE_HEADER_SIZE] = '\0'; DEBUG(D_deliver) - debug_printf("got %ld bytes (pipeheader) '%c' from transport process %d\n", - (long) got, *id, pid); + debug_printf("got %ld bytes (pipeheader) '%c' from transport process %ld\n", + (long) got, *id, (long)pid); { /* If we can't decode the pipeheader, the subprocess seems to have a @@ -3356,16 +3402,16 @@ while (!done) if (*endc) { msg = string_sprintf("failed to read pipe " - "from transport process %d for transport %s: error decoding size from header", - pid, addr->transport->driver_name); + "from transport process %ld for transport %s: error decoding size from header", + (long)pid, addr ? addr->transport->drinst.driver_name : US"?"); done = TRUE; break; } } DEBUG(D_deliver) - debug_printf("expect %lu bytes (pipedata) from transport process %d\n", - (u_long)required, pid); + debug_printf("expect %lu bytes (pipedata) from transport process %ld\n", + (u_long)required, (long)pid); /* Same as above, the transport process will write the bytes announced in a timely manner, so we can just wait for the bytes, getting less than expected @@ -3373,8 +3419,8 @@ while (!done) if ((got = readn(fd, big_buffer, required)) != required) { msg = string_sprintf("got only " SSIZE_T_FMT " of " SIZE_T_FMT - " bytes (pipedata) from transport process %d for transport %s", - got, required, pid, addr->transport->driver_name); + " bytes (pipedata) from transport process %ld for transport %s", + got, required, (long)pid, addr->transport->drinst.driver_name); done = TRUE; break; } @@ -3563,15 +3609,22 @@ while (!done) { ADDR_MISMATCH: msg = string_sprintf("address count mismatch for data read from pipe " - "for transport process %d for transport %s", pid, - addrlist->transport->driver_name); + "for transport process %ld for transport %s", + (long)pid, addrlist->transport->drinst.driver_name); done = TRUE; break; } switch (*subid) { - case 3: /* explicit notification of continued-connection (non)use; +#ifndef DISABLE_DKIM + case '4': /* DKIM information */ + addr->dkim_used = string_copy(ptr); + while(*ptr++); + break; +#endif + + case '3': /* explicit notification of continued-connection (non)use; overrides caller's knowlege. */ if (*ptr & BIT(1)) setflag(addr, af_new_conn); else if (*ptr & BIT(2)) setflag(addr, af_cont_conn); @@ -3633,11 +3686,20 @@ while (!done) h->dnssec = *ptr == '2' ? DS_YES : *ptr == '1' ? DS_NO : DS_UNK; - ptr++; addr->host_used = h; } - else ptr++; + ptr++; + continue_flags = 0; +#ifndef DISABLE_TLS + if (testflag(addr, af_cert_verified)) continue_flags |= CTF_CV; +# ifdef SUPPORT_DANE + if (testflag(addr, af_dane_verified)) continue_flags |= CTF_DV; +# endif +# ifndef DISABLE_TLS_RESUME + if (testflag(addr, af_tls_resume)) continue_flags |= CTF_TR; +# endif +#endif /* Finished with this address */ addr = addr->next; @@ -3653,28 +3715,90 @@ while (!done) while (*ptr++) ; break; - /* Z marks the logical end of the data. It is followed by '0' if + /* Z0 marks the logical end of the data. It is followed by '0' if continue_transport was NULL at the end of transporting, otherwise '1'. - We need to know when it becomes NULL during a delivery down a passed SMTP - channel so that we don't try to pass anything more down it. Of course, for - most normal messages it will remain NULL all the time. */ + Those are now for historical reasons only; we always clear the continued + channel info, and then set it explicitly if the transport indicates it + is still open, because it could differ for each transport we are running in + parallel. + + Z1 is a suggested message_id to handle next, used during a + continued-transport sequence. */ case 'Z': - if (*ptr == '0') + switch (*subid) { - continue_transport = NULL; - continue_hostname = NULL; + case '0': /* End marker */ + done = TRUE; + DEBUG(D_deliver) debug_printf("Z0%c item read\n", *ptr); + break; + case '1': /* Suggested continuation message */ + Ustrncpy(continue_next_id, ptr, MESSAGE_ID_LENGTH); + continue_sequence = atoi(CS ptr + MESSAGE_ID_LENGTH + 1); + DEBUG(D_deliver) debug_printf("continue_next_id: %s seq %d\n", + continue_next_id, continue_sequence); + break; + case '2': /* Continued transport, host & addr */ + { + int recvd_fd; + + DEBUG(D_any) if (Ustrcmp(process_purpose, "continued-delivery") != 0) + debug_printf("%s becomes continued-delivery\n", process_purpose); + process_purpose = US"continued-delivery"; + continue_transport = string_copy(ptr); while (*ptr++) ; + continue_hostname = string_copy(ptr); while (*ptr++) ; + continue_host_address = string_copy(ptr); while (*ptr++) ; + continue_sequence = atoi(CS ptr); + + dup2((recvd_fd = recv_fd_from_sock(fd)), 0); + close(recvd_fd); + + DEBUG(D_deliver) + debug_printf("continue: fd %d tpt %s host '%s' addr '%s' seq %d\n", + recvd_fd, continue_transport, continue_hostname, + continue_host_address, continue_sequence); + break; + } + case '3': /* Continued conn info */ + smtp_peer_options = ptr[0]; + f.smtp_authenticated = ptr[1] & 1; + break; +#ifndef DISABLE_TLS + case '4': /* Continued TLS info */ + continue_proxy_cipher = string_copy(ptr); + break; + case '5': /* Continued DANE info */ + case '6': /* Continued TLS info */ +# ifdef SUPPORT_DANE + continue_proxy_dane = *subid == '5'; +# endif + continue_proxy_sni = *ptr ? string_copy(ptr) : NULL; + break; +#endif +#ifndef DISABLE_ESMTP_LIMITS + case '7': /* Continued peer limits */ + sscanf(CS ptr, "%u %u %u", + &continue_limit_mail, &continue_limit_rcpt, + &continue_limit_rcptdom); + break; +#endif +#ifdef SUPPORT_SOCKS + case '8': /* Continued proxy info */ + proxy_local_address = string_copy(ptr); while (*ptr++) ; + proxy_local_port = atoi(CS ptr); while (*ptr++) ; + proxy_external_address = string_copy(ptr); while (*ptr++) ; + proxy_external_port = atoi(CS ptr); + break; +#endif } - done = TRUE; - DEBUG(D_deliver) debug_printf("Z0%c item read\n", *ptr); break; /* Anything else is a disaster. */ default: msg = string_sprintf("malformed data (%d) read from pipe for transport " - "process %d for transport %s", ptr[-1], pid, - addr->transport->driver_name); + "process %ld for transport %s", ptr[-1], (long)pid, + addr ? addr->transport->drinst.driver_name : US"?"); done = TRUE; break; } @@ -3707,8 +3831,8 @@ something is wrong. */ if (!msg && addr) msg = string_sprintf("insufficient address data read from pipe " - "for transport process %d for transport %s", pid, - addr->transport->driver_name); + "for transport process %ld for transport %s", (long)pid, + addr->transport->drinst.driver_name); /* If an error message is set, something has gone wrong in getting back the delivery data. Put the message into each address and freeze it. */ @@ -3719,7 +3843,8 @@ if (msg) addr->transport_return = DEFER; addr->special_action = SPECIAL_FREEZE; addr->message = msg; - log_write(0, LOG_MAIN|LOG_PANIC, "Delivery status for %s: %s\n", addr->address, addr->message); + log_write(0, LOG_MAIN|LOG_PANIC, "Delivery status for %s: %s\n", + addr->address, addr->message); } /* Return TRUE to indicate we have got all we need from this process, even @@ -3912,8 +4037,8 @@ for (;;) /* Normally we do not repeat this loop */ { if ((pid = parlist[poffset].pid) != 0 && kill(pid, 0) == 0) { - DEBUG(D_deliver) debug_printf("process %d still exists: assume " - "stolen by strace\n", (int)pid); + DEBUG(D_deliver) debug_printf("process %ld still exists: assume " + "stolen by strace\n", (long)pid); break; /* With poffset set */ } } @@ -3974,8 +4099,8 @@ for (;;) /* Normally we do not repeat this loop */ if (endedpid == pid) goto PROCESS_DONE; if (endedpid != (pid_t)(-1) || errno != EINTR) log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Unexpected error return " - "%d (errno = %d) from waitpid() for process %d", - (int)endedpid, errno, (int)pid); + "%d (errno = %d) from waitpid() for process %ld", + (int)endedpid, errno, (long)pid); } } } @@ -3997,8 +4122,8 @@ for (;;) /* Normally we do not repeat this loop */ /* This situation is an error, but it's probably better to carry on looking for another process than to give up (as we used to do). */ - log_write(0, LOG_MAIN|LOG_PANIC, "Process %d finished: not found in remote " - "transport process list", pid); + log_write(0, LOG_MAIN|LOG_PANIC, "Process %ld finished: not found in remote " + "transport process list", (long)pid); } /* End of the "for" loop */ /* Come here when all the data was completely read after a poll(), and @@ -4009,9 +4134,9 @@ PROCESS_DONE: DEBUG(D_deliver) { if (status == 0) - debug_printf("remote delivery process %d ended\n", (int)pid); + debug_printf("remote delivery process %ld ended\n", (long)pid); else - debug_printf("remote delivery process %d ended: status=%04x\n", (int)pid, + debug_printf("remote delivery process %ld ended: status=%04x\n", (long)pid, status); } @@ -4034,7 +4159,7 @@ if ((status & 0xffff) != 0) msg = string_sprintf("%s transport process returned non-zero status 0x%04x: " "%s %d", - addrlist->transport->driver_name, + addrlist->transport->drinst.driver_name, status, msb == 0 ? "terminated by signal" : "exit code", code); @@ -4102,7 +4227,7 @@ while (parcount > max) { transport_instance * tp = doneaddr->transport; if (tp->max_parallel) - enq_end(string_sprintf("tpt-serialize-%s", tp->name)); + enq_end(string_sprintf("tpt-serialize-%s", tp->drinst.name)); remote_post_process(doneaddr, LOG_MAIN, NULL, fallback); } @@ -4277,7 +4402,7 @@ So look out for the place it gets used. if (tp->expand_multi_domain) deliver_set_expansions(addr); - if (exp_bool(addr, US"transport", tp->name, D_transport, + if (exp_bool(addr, US"transport", tp->drinst.name, D_transport, US"multi_domain", tp->multi_domain, tp->expand_multi_domain, &multi_domain) != OK) { @@ -4293,6 +4418,7 @@ So look out for the place it gets used. transport splitting further by max_rcp. So we potentially lose some parallellism. */ + GET_OPTION("max_rcpt"); address_count_max = mua_wrapper || Ustrchr(tp->max_addresses, '$') ? UNLIMITED_ADDRS : expand_max_rcpt(tp->max_addresses); @@ -4340,8 +4466,9 @@ So look out for the place it gets used. && address_count_max < remote_delivery_count/remote_max_parallel ) { - int new_max = remote_delivery_count/remote_max_parallel; - int message_max = tp->connection_max_messages; + int new_max = remote_delivery_count/remote_max_parallel, message_max; + GET_OPTION("connection_max_messages"); + message_max = tp->connection_max_messages; if (connection_max_messages >= 0) message_max = connection_max_messages; message_max -= continue_sequence - 1; if (message_max > 0 && new_max > address_count_max * message_max) @@ -4386,7 +4513,7 @@ nonmatch domains || ( ( (void)(!tp->expand_multi_domain || ((void)deliver_set_expansions(next), 1)), exp_bool(addr, - US"transport", next->transport->name, D_transport, + US"transport", next->transport->drinst.name, D_transport, US"multi_domain", next->transport->multi_domain, next->transport->expand_multi_domain, &md) == OK ) @@ -4434,14 +4561,13 @@ nonmatch domains /* Compute the return path, expanding a new one if required. The old one must be set first, as it might be referred to in the expansion. */ - if(addr->prop.errors_address) - return_path = addr->prop.errors_address; - else - return_path = sender_address; + return_path = addr->prop.errors_address + ? addr->prop.errors_address : sender_address; + GET_OPTION("return_path"); if (tp->return_path) { - uschar *new_return_path = expand_string(tp->return_path); + uschar * new_return_path = expand_string(tp->return_path); if (new_return_path) return_path = new_return_path; else if (!f.expand_string_forcedfail) @@ -4498,7 +4624,7 @@ nonmatch domains f.continue_more = FALSE; /* In case got set for the last lot */ if (continue_transport) { - BOOL ok = Ustrcmp(continue_transport, tp->name) == 0; + BOOL ok = Ustrcmp(continue_transport, tp->drinst.name) == 0; /*XXX do we need to check for a DANEd conn vs. a change of domain? */ /* If the transport is about to override the host list do not check @@ -4509,11 +4635,11 @@ nonmatch domains if (ok) { - smtp_transport_options_block * ob; + transport_info * ti = tp->drinst.info; + smtp_transport_options_block * ob = tp->drinst.options_block; - if ( !( Ustrcmp(tp->info->driver_name, "smtp") == 0 - && (ob = (smtp_transport_options_block *)tp->options_block) - && ob->hosts_override && ob->hosts + if ( !( Ustrcmp(ti->drinfo.driver_name, "smtp") == 0 + && ob && ob->hosts_override && ob->hosts ) && addr->host_list ) @@ -4532,8 +4658,8 @@ nonmatch domains if (!ok) { DEBUG(D_deliver) debug_printf("not suitable for continue_transport (%s)\n", - Ustrcmp(continue_transport, tp->name) != 0 - ? string_sprintf("tpt %s vs %s", continue_transport, tp->name) + Ustrcmp(continue_transport, tp->drinst.name) != 0 + ? string_sprintf("tpt %s vs %s", continue_transport, tp->drinst.name) : string_sprintf("no host matching %s", continue_hostname)); if (serialize_key) enq_end(serialize_key); @@ -4542,7 +4668,8 @@ nonmatch domains for (next = addr; ; next = next->next) { next->host_list = next->fallback_hosts; - DEBUG(D_deliver) debug_printf("%s queued for fallback host(s)\n", next->address); + DEBUG(D_deliver) + debug_printf("%s queued for fallback host(s)\n", next->address); if (!next->next) break; } next->next = addr_fallback; @@ -4562,19 +4689,30 @@ nonmatch domains continue; } + } - /* Set a flag indicating whether there are further addresses that list - the continued host. This tells the transport to leave the channel open, - but not to pass it to another delivery process. We'd like to do that - for non-continue_transport cases too but the knowlege of which host is - connected to is too hard to manage. Perhaps we need a finer-grain - interface to the transport. */ + /* Once we hit the max number of parallel transports set a flag indicating + whether there are further addresses that list the same host. This tells the + transport to leave the channel open for us. */ +/*XXX maybe we should *count* possible further's, and set continue_more if +parmax * tpt-max is exceeded? */ - for (next = addr_remote; next && !f.continue_more; next = next->next) - for (host_item * h = next->host_list; h; h = h->next) - if (Ustrcmp(h->name, continue_hostname) == 0) - { f.continue_more = TRUE; break; } + if (parcount+1 >= remote_max_parallel) + { + host_item * h1 = addr->host_list; + if (h1) + { + const uschar * name = continue_hostname ? continue_hostname : h1->name; + for (next = addr_remote; next && !f.continue_more; next = next->next) + for (host_item * h = next->host_list; h; h = h->next) + if (Ustrcmp(h->name, name) == 0) + { f.continue_more = TRUE; break; } + } } + else DEBUG(D_deliver) + debug_printf( + "not reached parallelism limit (%d/%d) so not setting continue_more\n", + parcount+1, remote_max_parallel); /* The transports set up the process info themselves as they may connect to more than one remote machine. They also have to set up the filter @@ -4587,13 +4725,16 @@ nonmatch domains fails, it is probably because the value of remote_max_parallel is so large that too many file descriptors for pipes have been created. Arrange to wait for a process to finish, and then try again. If we still can't - create a pipe when all processes have finished, break the retry loop. */ + create a pipe when all processes have finished, break the retry loop. + Use socketpair() rather than pipe() so we can pass an fd back from the + transport process. + */ while (!pipe_done) { - if (pipe(pfd) == 0) pipe_done = TRUE; - else if (parcount > 0) parmax = parcount; - else break; + if (socketpair(AF_UNIX, SOCK_STREAM, 0, pfd) == 0) pipe_done = TRUE; + else if (parcount > 0) parmax = parcount; + else break; /* We need to make the reading end of the pipe non-blocking. There are two different options for this. Exim is cunningly (I hope!) coded so @@ -4651,17 +4792,47 @@ all pipes, so I do not see a reason to use non-blocking IO here search_tidyup(); - if ((pid = exim_fork(US"transport")) == 0) +/* +A continued-tpt will, in the tpt parent here, call par_reduce for +the one child. But we are hoping to never do continued-transport... +SO.... we may have called par_reduce for a single child, above when we'd +hit the limit on child-count. Possibly multiple times with different +transports and target hosts. Does it matter if several return a suggested +next-id, and we lose all but the last? Hmm. Less parallel working would +happen. Perhaps still do continued-tpt once one has been set? No, that won't +work for all cases. +BAH. +Could take the initial continued-tpt hit, and then do the next-id thing? + +do_remote_deliveries par_reduce par_wait par_read_pipe +*/ + + /*XXX what about firsttime? */ + /*XXX also, ph1? Note tp->name would possibly change per message, + so a check/close/open would be needed. Might was to change that var name + "continue_wait_db" as we'd be using it for a non-continued-transport + context. */ + if (continue_transport && !exim_lockfile_needed()) + if (!continue_wait_db) + { + continue_wait_db = dbfn_open_multi( + string_sprintf("wait-%.200s", continue_transport), + O_RDWR, + (open_db *) store_get(sizeof(open_db), GET_UNTAINTED)); + continue_next_id[0] = '\0'; + } + + if ((pid = exim_fork(f.queue_2stage ? US"transport ph1":US"transport")) == 0) { int fd = pfd[pipe_write]; host_item *h; /* Setting these globals in the subprocess means we need never clear them */ - transport_name = tp->name; - if (addr->router) router_name = addr->router->name; - driver_srcfile = tp->srcfile; - driver_srcline = tp->srcline; + transport_name = tp->drinst.name; + if (addr->router) router_name = addr->router->drinst.name; + driver_srcfile = tp->drinst.srcfile; + driver_srcline = tp->drinst.srcline; /* There are weird circumstances in which logging is disabled */ f.disable_logging = tp->disable_logging; @@ -4717,19 +4888,24 @@ all pipes, so I do not see a reason to use non-blocking IO here exim_setugid(uid, gid, use_initgroups, string_sprintf("remote delivery to %s with transport=%s", - addr->address, tp->name)); + addr->address, tp->drinst.name)); /* Close the unwanted half of this process' pipe, set the process state, and run the transport. Afterwards, transport_count will contain the number of bytes written. */ (void)close(pfd[pipe_read]); - set_process_info("delivering %s using %s", message_id, tp->name); + set_process_info("delivering %s using %s", message_id, tp->drinst.name); debug_print_string(tp->debug_string); - if (!(tp->info->code)(addr->transport, addr)) replicate_status(addr); + + { + transport_info * ti = tp->drinst.info; + if (!(ti->code)(addr->transport, addr)) /* Call the transport */ + replicate_status(addr); + } set_process_info("delivering %s (just run %s for %s%s in subprocess)", - message_id, tp->name, addr->address, addr->next ? ", ..." : ""); + message_id, tp->drinst.name, addr->address, addr->next ? ", ..." : ""); /* Ensure any cached resources that we used are now released */ @@ -4743,7 +4919,11 @@ all pipes, so I do not see a reason to use non-blocking IO here is flagged by an identifying byte, and is then in a fixed format (with strings terminated by zeros), and there is a final terminator at the end. The host information and retry information is all attached to - the first address, so that gets sent at the start. */ + the first address, so that gets sent at the start. + + Result item tags: + A C D H I K L P R S T X Z + */ /* Host unusability information: for most success cases this will be null. */ @@ -4752,7 +4932,7 @@ all pipes, so I do not see a reason to use non-blocking IO here { if (!h->address || h->status < hstatus_unusable) continue; sprintf(CS big_buffer, "%c%c%s", h->status, h->why, h->address); - rmt_dlv_checked_write(fd, 'H', '0', big_buffer, Ustrlen(big_buffer+2) + 3); + rmt_dlv_checked_write(fd, 'H','0', big_buffer, Ustrlen(big_buffer+2) + 3); } /* The number of bytes written. This is the same for each address. Even @@ -4766,23 +4946,24 @@ all pipes, so I do not see a reason to use non-blocking IO here /* Information about what happened to each address. Four item types are used: an optional 'X' item first, for TLS information, then an optional "C" item for any client-auth info followed by 'R' items for any retry settings, - and finally an 'A' item for the remaining data. */ + and finally an 'A' item for the remaining data. The actual recipient address + is not sent but is implicit in the address-chain being handled. */ for(; addr; addr = addr->next) { - uschar *ptr; + uschar * ptr; - /* The certificate verification status goes into the flags */ +#ifndef DISABLE_TLS + /* The certificate verification status goes into the flags, in A0 */ if (tls_out.certificate_verified) setflag(addr, af_cert_verified); -#ifdef SUPPORT_DANE +# ifdef SUPPORT_DANE if (tls_out.dane_verified) setflag(addr, af_dane_verified); -#endif +# endif # ifndef DISABLE_TLS_RESUME if (tls_out.resumption & RESUME_USED) setflag(addr, af_tls_resume); # endif /* Use an X item only if there's something to send */ -#ifndef DISABLE_TLS if (addr->cipher) { ptr = big_buffer + sprintf(CS big_buffer, "%.128s", addr->cipher) + 1; @@ -4886,6 +5067,15 @@ all pipes, so I do not see a reason to use non-blocking IO here rmt_dlv_checked_write(fd, 'R', '0', big_buffer, ptr - big_buffer); } +#ifndef DISABLE_DKIM + if (addr->dkim_used && LOGGING(dkim_verbose)) + { + DEBUG(D_deliver) debug_printf("dkim used: %s\n", addr->dkim_used); + ptr = big_buffer + sprintf(CS big_buffer, "%.128s", addr->dkim_used) + 1; + rmt_dlv_checked_write(fd, 'A', '4', big_buffer, ptr - big_buffer); + } +#endif + if (testflag(addr, af_new_conn) || testflag(addr, af_cont_conn)) { DEBUG(D_deliver) debug_printf("%scontinued-connection\n", @@ -4930,7 +5120,13 @@ all pipes, so I do not see a reason to use non-blocking IO here #endif /* The rest of the information goes in an 'A0' item. */ - +#ifdef notdef + DEBUG(D_deliver) + debug_printf("%s %s for MAIL\n", + addr->special_action == '=' ? "initial RCPT" + : addr->special_action == '-' ? "additional RCPT" : "?", + addr->address); +#endif sprintf(CS big_buffer, "%c%c", addr->transport_return, addr->special_action); ptr = big_buffer + 2; memcpy(ptr, &addr->basic_errno, sizeof(addr->basic_errno)); @@ -4970,21 +5166,83 @@ all pipes, so I do not see a reason to use non-blocking IO here if (LOGGING(incoming_interface) && sending_ip_address) #endif { - uschar * ptr; - ptr = big_buffer + sprintf(CS big_buffer, "%.128s", sending_ip_address) + 1; + uschar * ptr = big_buffer + + sprintf(CS big_buffer, "%.128s", sending_ip_address) + 1; ptr += sprintf(CS ptr, "%d", sending_port) + 1; rmt_dlv_checked_write(fd, 'I', '0', big_buffer, ptr - big_buffer); } + /* Continuation message-id, if a continuation is for that reason, + and the next sequence number (MAIL FROM count) for the connection. */ + + if (*continue_next_id) + rmt_dlv_checked_write(fd, 'Z', '1', big_buffer, + sprintf(CS big_buffer, "%.*s %u", + MESSAGE_ID_LENGTH, continue_next_id, continue_sequence+1) + 1); + + /* Connection details, only on the first suggested continuation for + wait-db ones, but for all continue-more ones (though any after the + delivery proc has the info are pointless). */ + + if (continue_hostname && continue_fd >= 0) + { + { + uschar * ptr = big_buffer; + ptr += sprintf(CS ptr, "%.128s", continue_transport) + 1; + ptr += sprintf(CS ptr, "%.128s", continue_hostname) + 1; + ptr += sprintf(CS ptr, "%.128s", continue_host_address) + 1; + ptr += sprintf(CS ptr, "%u", continue_sequence+1) + 1; + rmt_dlv_checked_write(fd, 'Z', '2', big_buffer, ptr - big_buffer); + send_fd_over_socket(fd, continue_fd); + } + + big_buffer[0] = smtp_peer_options; + big_buffer[1] = f.smtp_authenticated ? 1 : 0; + rmt_dlv_checked_write(fd, 'Z', '3', big_buffer, 2); + + if (tls_out.active.sock >= 0 || continue_proxy_cipher) + rmt_dlv_checked_write(fd, 'Z', '4', big_buffer, + sprintf(CS big_buffer, "%.128s", continue_proxy_cipher) + 1); + + if (tls_out.sni) + rmt_dlv_checked_write(fd, 'Z', +#ifdef SUPPORT_DANE + tls_out.dane_verified ? '5' : '6', +#else + '6', +#endif + tls_out.sni, Ustrlen(tls_out.sni)+1); + +#ifndef DISABLE_ESMTP_LIMITS + if (continue_limit_mail || continue_limit_rcpt || continue_limit_rcptdom) + rmt_dlv_checked_write(fd, 'Z', '7', big_buffer, + sprintf(CS big_buffer, "%u %u %u", + continue_limit_mail, continue_limit_rcpt, + continue_limit_rcptdom) + 1); +#endif + +#ifdef SUPPORT_SOCKS + if (proxy_session) + { + uschar * ptr = big_buffer; + ptr += sprintf(CS ptr, "%.128s", proxy_local_address) + 1; + ptr += sprintf(CS ptr, "%u", proxy_local_port) + 1; + ptr += sprintf(CS ptr, "%.128s", proxy_external_address) + 1; + ptr += sprintf(CS ptr, "%u", proxy_external_port) + 1; + rmt_dlv_checked_write(fd, 'Z', '8', big_buffer, ptr - big_buffer); + } +#endif + } + /* Add termination flag, close the pipe, and that's it. The character - after 'Z' indicates whether continue_transport is now NULL or not. + after "Z0" indicates whether continue_transport is now NULL or not. A change from non-NULL to NULL indicates a problem with a continuing connection. */ big_buffer[0] = continue_transport ? '1' : '0'; rmt_dlv_checked_write(fd, 'Z', '0', big_buffer, 1); (void)close(fd); - exit(EXIT_SUCCESS); + exim_exit(EXIT_SUCCESS); } /* Back in the mainline: close the unwanted half of the pipe. */ @@ -5037,14 +5295,20 @@ all pipes, so I do not see a reason to use non-blocking IO here (continue_transport gets set to NULL) before we consider any other addresses in this message. */ - if (continue_transport) par_reduce(0, fallback); + if (continue_transport) + { + par_reduce(0, fallback); + if (!*continue_next_id && continue_wait_db) + { dbfn_close_multi(continue_wait_db); continue_wait_db = NULL; } + } /* Otherwise, if we are running in the test harness, wait a bit, to let the newly created process get going before we create another process. This should ensure repeatability in the tests. Wait long enough for most cases to complete the transport. */ - else testharness_pause_ms(600); + else + testharness_pause_ms(600); continue; @@ -5086,7 +5350,7 @@ Returns: OK int deliver_split_address(address_item * addr) { -uschar * address = addr->address; +const uschar * address = addr->address; uschar * domain; uschar * t; int len; @@ -5103,7 +5367,7 @@ where they are locally interpreted. [The new draft "821" is more explicit on this, Jan 1999.] We know the syntax is valid, so this can be done by simply removing quoting backslashes and any unquoted doublequotes. */ -t = addr->cc_local_part = store_get(len+1, address); +addr->cc_local_part = t = store_get(len+1, address); while(len-- > 0) { int c = *address++; @@ -5115,7 +5379,7 @@ while(len-- > 0) } else *t++ = c; } -*t = 0; +*t = '\0'; /* We do the percent hack only for those domains that are listed in percent_hack_domains. A loop is required, to copy with multiple %-hacks. */ @@ -5123,8 +5387,8 @@ percent_hack_domains. A loop is required, to copy with multiple %-hacks. */ if (percent_hack_domains) { int rc; - uschar *new_address = NULL; - uschar *local_part = addr->cc_local_part; + uschar * new_address = NULL; + const uschar * local_part = addr->cc_local_part; deliver_domain = addr->domain; /* set $domain */ @@ -5230,10 +5494,12 @@ static int continue_closedown(void) { if (continue_transport) - for (transport_instance * t = transports; t; t = t->next) - if (Ustrcmp(t->name, continue_transport) == 0) + for (transport_instance * t = transports; t; t = t->drinst.next) + if (Ustrcmp(t->drinst.name, continue_transport) == 0) { - if (t->info->closedown) (t->info->closedown)(t); + transport_info * ti = t->drinst.info; + if (ti->closedown) (ti->closedown)(t); + continue_transport = NULL; break; } return DELIVER_NOT_ATTEMPTED; @@ -5261,12 +5527,12 @@ Returns: TRUE if the address is not hidden */ static BOOL -print_address_information(address_item *addr, FILE *f, uschar *si, uschar *sc, - uschar *se) +print_address_information(address_item * addr, FILE * f, uschar * si, + uschar * sc, uschar * se) { BOOL yield = TRUE; -uschar *printed = US""; -address_item *ancestor = addr; +const uschar * printed = US""; +address_item * ancestor = addr; while (ancestor->parent) ancestor = ancestor->parent; fprintf(f, "%s", CS si); @@ -5281,8 +5547,8 @@ else if (!testflag(addr, af_pfr) || !addr->parent) else { - uschar *s = addr->address; - uschar *ss; + const uschar * s = addr->address; + const uschar * ss; if (addr->address[0] == '>') { ss = US"mail"; s++; } else if (addr->address[0] == '|') ss = US"pipe"; @@ -5296,7 +5562,7 @@ fprintf(f, "%s", CS string_printing(printed)); if (ancestor != addr) { - uschar *original = ancestor->onetime_parent; + const uschar * original = ancestor->onetime_parent; if (!original) original= ancestor->address; if (strcmpic(original, printed) != 0) fprintf(f, "%s(%sgenerated from %s)", sc, @@ -5460,16 +5726,14 @@ Returns: nothing */ static void -do_duplicate_check(address_item **anchor) +do_duplicate_check(address_item ** anchor) { -address_item *addr; +address_item * addr; while ((addr = *anchor)) { - tree_node *tnode; + tree_node * tnode; if (testflag(addr, af_pfr)) - { - anchor = &(addr->next); - } + anchor = &addr->next; else if ((tnode = tree_search(tree_duplicates, addr->unique))) { DEBUG(D_deliver|D_route) @@ -5482,7 +5746,7 @@ while ((addr = *anchor)) else { tree_add_duplicate(addr->unique, addr); - anchor = &(addr->next); + anchor = &addr->next; } } } @@ -5533,18 +5797,18 @@ return actual_time; static FILE * expand_open(const uschar * filename, - const uschar * varname, const uschar * reason) + const uschar * optname, const uschar * reason) { const uschar * s = expand_cstring(filename); FILE * fp = NULL; if (!s || !*s) log_write(0, LOG_MAIN|LOG_PANIC, - "Failed to expand %s: '%s'\n", varname, filename); + "Failed to expand %s: '%s'\n", optname, filename); else if (*s != '/' || is_tainted(s)) log_write(0, LOG_MAIN|LOG_PANIC, "%s is not %s after expansion: '%s'\n", - varname, *s == '/' ? "untainted" : "absolute", s); + optname, *s == '/' ? "untainted" : "absolute", s); else if (!(fp = Ufopen(s, "rb"))) log_write(0, LOG_MAIN|LOG_PANIC, "Failed to open %s for %s " "message texts: %s", s, reason, strerror(errno)); @@ -5593,9 +5857,9 @@ if (!(bounce_recipient = addr_failed->prop.errors_address)) /* Make a subprocess to send a message, using its stdin */ if ((pid = child_open_exim(&fd, US"bounce-message")) < 0) - log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Process %d (parent %d) failed to " - "create child process to send failure message: %s", getpid(), - getppid(), strerror(errno)); + log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Process %ld (parent %ld) failed to " + "create child process to send failure message: %s", + (long)getpid(), (long)getppid(), strerror(errno)); /* Creation of child succeeded */ @@ -5675,6 +5939,7 @@ else /* Open a template file if one is provided. Log failure to open, but carry on - default texts will be used. */ + GET_OPTION("bounce_message_file"); if (bounce_message_file) emf = expand_open(bounce_message_file, US"bounce_message_file", US"error"); @@ -5840,7 +6105,7 @@ wording. */ { /* must be decoded from xtext: see RFC 3461:6.3a */ uschar * xdec_envid; - if (auth_xtextdecode(dsn_envid, &xdec_envid) > 0) + if (xtextdecode(dsn_envid, &xdec_envid) > 0) fprintf(fp, "Original-Envelope-ID: %s\n", dsn_envid); else fprintf(fp, "X-Original-Envelope-ID: error decoding xtext formatted ENVID\n"); @@ -6034,6 +6299,7 @@ transport_ctx tctx = {{0}}; if (pid <= 0) return FALSE; +GET_OPTION("warn_message_file"); if (warn_message_file) wmf = expand_open(warn_message_file, US"warn_message_file", US"warning"); @@ -6144,7 +6410,7 @@ if (dsn_envid) { /* must be decoded from xtext: see RFC 3461:6.3a */ uschar *xdec_envid; - if (auth_xtextdecode(dsn_envid, &xdec_envid) > 0) + if (xtextdecode(dsn_envid, &xdec_envid) > 0) fprintf(f,"Original-Envelope-ID: %s\n", dsn_envid); else fprintf(f,"X-Original-Envelope-ID: error decoding xtext formatted ENVID\n"); @@ -6211,7 +6477,7 @@ for (const address_item * a = addr_succeed; a; a = a->next) "DSN: envid: %s ret: %d\n" "DSN: Final recipient: %s\n" "DSN: Remote SMTP server supports DSN: %d\n", - a->router ? a->router->name : US"(unknown)", + a->router ? a->router->drinst.name : US"(unknown)", a->address, sender_address, a->dsn_orcpt ? a->dsn_orcpt : US"NULL", @@ -6243,13 +6509,13 @@ if (addr_senddsn) int fd; pid_t pid = child_open_exim(&fd, US"DSN"); - DEBUG(D_deliver) debug_printf("DSN: child_open_exim returns: %d\n", pid); + DEBUG(D_deliver) debug_printf("DSN: child_open_exim returns: %ld\n", (long)pid); if (pid < 0) /* Creation of child failed */ { - log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Process %d (parent %d) failed to " - "create child process to send success-dsn message: %s", getpid(), - getppid(), strerror(errno)); + log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Process %ld (parent %ld) failed to " + "create child process to send success-dsn message: %s", + (long)getpid(), (long)getppid(), strerror(errno)); DEBUG(D_deliver) debug_printf("DSN: child_open_exim failed\n"); } @@ -6303,7 +6569,7 @@ if (addr_senddsn) if (dsn_envid) { /* must be decoded from xtext: see RFC 3461:6.3a */ uschar * xdec_envid; - if (auth_xtextdecode(dsn_envid, &xdec_envid) > 0) + if (xtextdecode(dsn_envid, &xdec_envid) > 0) fprintf(f, "Original-Envelope-ID: %s\n", dsn_envid); else fprintf(f, "X-Original-Envelope-ID: error decoding xtext formatted ENVID\n"); @@ -6390,16 +6656,19 @@ Returns: When the global variable mua_wrapper is FALSE: int deliver_message(const uschar * id, BOOL forced, BOOL give_up) { -int i, rc; -int final_yield = DELIVER_ATTEMPTED_NORMAL; -time_t now = time(NULL); -address_item *addr_last = NULL; -uschar *filter_message = NULL; -int process_recipients = RECIP_ACCEPT; -open_db dbblock; -open_db *dbm_file; +int i, rc, final_yield, process_recipients; +time_t now; +address_item * addr_last; +uschar * filter_message, * info; +open_db dbblock, * dbm_file = NULL; extern int acl_where; -uschar *info; + +CONTINUED_ID: +final_yield = DELIVER_ATTEMPTED_NORMAL; +now = time(NULL); +addr_last = NULL; +filter_message = NULL; +process_recipients = RECIP_ACCEPT; #ifdef MEASURE_TIMING report_time_since(×tamp_startup, US"delivery start"); /* testcase 0022, 2100 */ @@ -6407,7 +6676,7 @@ report_time_since(×tamp_startup, US"delivery start"); /* testcase 0022, 210 info = queue_run_pid == (pid_t)0 ? string_sprintf("delivering %s", id) - : string_sprintf("delivering %s (queue run pid %d)", id, queue_run_pid); + : string_sprintf("delivering %s (queue run pid %ld)", id, (long)queue_run_pid); /* If the D_process_info bit is on, set_process_info() will output debugging information. If not, we want to show this initial information if D_deliver or @@ -6474,7 +6743,7 @@ opening the data file, message_subdir gets set. */ if ((deliver_datafile = spool_open_datafile(id)) < 0) return continue_closedown(); /* yields DELIVER_NOT_ATTEMPTED */ -/* tHe value of message_size at this point has been set to the data length, +/* The value of message_size at this point has been set to the data length, plus one for the blank line that notionally precedes the data. */ /* Now read the contents of the header file, which will set up the headers in @@ -6733,6 +7002,7 @@ else if (system_filter && process_recipients != RECIP_FAIL_TIMEOUT) /* Any error in the filter file causes a delivery to be abandoned. */ + GET_OPTION("system_filter"); redirect.string = system_filter; redirect.isfile = TRUE; redirect.check_owner = redirect.check_group = FALSE; @@ -6752,10 +7022,7 @@ else if (system_filter && process_recipients != RECIP_FAIL_TIMEOUT) RDO_REALLOG | RDO_REWRITE, NULL, /* No :include: restriction (not used in filter) */ - NULL, /* No sieve vacation directory (not sieve!) */ - NULL, /* No sieve enotify mailto owner (not sieve!) */ - NULL, /* No sieve user address (not sieve!) */ - NULL, /* No sieve subaddress (not sieve!) */ + NULL, /* No sieve info (not sieve!) */ &ugid, /* uid/gid data */ &addr_new, /* Where to hang generated addresses */ &filter_message, /* Where to put error message */ @@ -6912,12 +7179,14 @@ else if (system_filter && process_recipients != RECIP_FAIL_TIMEOUT) if (p->address[0] == '|') { type = US"pipe"; + GET_OPTION("system_filter_pipe_transport"); tpname = system_filter_pipe_transport; address_pipe = p->address; } else if (p->address[0] == '>') { type = US"reply"; + GET_OPTION("system_filter_reply_transport"); tpname = system_filter_reply_transport; } else @@ -6925,11 +7194,13 @@ else if (system_filter && process_recipients != RECIP_FAIL_TIMEOUT) if (p->address[Ustrlen(p->address)-1] == '/') { type = US"directory"; + GET_OPTION("system_filter_directory_transport"); tpname = system_filter_directory_transport; } else { type = US"file"; + GET_OPTION("system_filter_file_transport"); tpname = system_filter_file_transport; } address_file = p->address; @@ -6957,12 +7228,9 @@ else if (system_filter && process_recipients != RECIP_FAIL_TIMEOUT) if (tpname) { transport_instance *tp; - for (tp = transports; tp; tp = tp->next) - if (Ustrcmp(tp->name, tpname) == 0) - { - p->transport = tp; - break; - } + for (tp = transports; tp; tp = tp->drinst.next) + if (Ustrcmp(tp->drinst.name, tpname) == 0) + { p->transport = tp; break; } if (!tp) p->message = string_sprintf("failed to find \"%s\" transport " "for system filter delivery", tpname); @@ -6973,7 +7241,7 @@ else if (system_filter && process_recipients != RECIP_FAIL_TIMEOUT) if (!p->transport) { - address_item *badp = p; + address_item * badp = p; p = p->next; if (!addr_last) addr_new = p; else addr_last->next = p; badp->local_part = badp->address; /* Needed for log line */ @@ -7114,9 +7382,10 @@ if (process_recipients != RECIP_IGNORE) #ifndef DISABLE_EVENT if (process_recipients != RECIP_ACCEPT && event_action) { - uschar * save_local = deliver_localpart; + const uschar * save_local = deliver_localpart; const uschar * save_domain = deliver_domain; - uschar * addr = new->address, * errmsg = NULL; + const uschar * addr = new->address; + uschar * errmsg = NULL; int start, end, dom; if (!parse_extract_address(addr, &errmsg, &start, &end, &dom, TRUE)) @@ -7196,11 +7465,42 @@ while (addr_new) /* Loop until all addresses dealt with */ address_item * addr, * parent; /* Failure to open the retry database is treated the same as if it does - not exist. In both cases, dbm_file is NULL. */ + not exist. In both cases, dbm_file is NULL. For the first stage of a 2-phase + queue run don't bother checking domain- or address-retry info; they will take + effect on the second stage. */ + + if (!f.queue_2stage) + { + /* If we have transaction-capable hintsdbs, open the retry db without + locking, and leave open for the transport process and for subsequent + deliveries. Use a writeable open as we can keep it open all the way through + to writing retry records if needed due to message fails. + If the open fails, tag that explicitly for the transport but retry the open + next time around, in case it was created in the interim. + If non-transaction, we are only reading records at this stage and + we close the db before running the transport. + Either way we do a non-creating open. */ + + if (continue_retry_db == (open_db *)-1) + continue_retry_db = NULL; + + if (continue_retry_db) + { + DEBUG(D_hints_lookup) debug_printf("using cached retry hintsdb handle\n"); + dbm_file = continue_retry_db; + } + else if (!exim_lockfile_needed()) + { + dbm_file = dbfn_open_multi(US"retry", O_RDWR, &dbblock); + continue_retry_db = dbm_file ? dbm_file : (open_db *)-1; + } + else + dbm_file = dbfn_open(US"retry", O_RDONLY, &dbblock, FALSE, TRUE); - if (!(dbm_file = dbfn_open(US"retry", O_RDONLY, &dbblock, FALSE, TRUE))) - DEBUG(D_deliver|D_retry|D_route|D_hints_lookup) - debug_printf("no retry data available\n"); + if (!dbm_file) + DEBUG(D_deliver|D_retry|D_route|D_hints_lookup) + debug_printf("no retry data available\n"); + } /* Scan the current batch of new addresses, to handle pipes, files and autoreplies, and determine which others are ready for routing. */ @@ -7209,7 +7509,7 @@ while (addr_new) /* Loop until all addresses dealt with */ { int rc; tree_node * tnode; - dbdata_retry * domain_retry_record, * address_retry_record; + dbdata_retry * domain_retry_record = NULL, * address_retry_record = NULL; addr = addr_new; addr_new = addr->next; @@ -7246,7 +7546,7 @@ while (addr_new) /* Loop until all addresses dealt with */ addr->unique = string_sprintf("%s:%s", addr->address, addr->parent->unique + - (testflag(addr->parent, af_homonym)? 3:0)); + (testflag(addr->parent, af_homonym) ? 3:0)); addr->address_retry_key = addr->domain_retry_key = string_sprintf("T:%s", addr->unique); @@ -7344,7 +7644,7 @@ while (addr_new) /* Loop until all addresses dealt with */ transport_instance * save_t = addr->transport; transport_instance * t = store_get(sizeof(*t), save_t); *t = *save_t; - t->name = US"**bypassed**"; + t->drinst.name = US"**bypassed**"; addr->transport = t; (void)post_process_one(addr, OK, LOG_MAIN, EXIM_DTYPE_TRANSPORT, '='); addr->transport= save_t; @@ -7355,7 +7655,7 @@ while (addr_new) /* Loop until all addresses dealt with */ delivery. */ DEBUG(D_deliver|D_route) - debug_printf("queued for %s transport\n", addr->transport->name); + debug_printf("queued for %s transport\n", addr->transport->drinst.name); addr->next = addr_local; addr_local = addr; continue; /* with the next new address */ @@ -7438,70 +7738,82 @@ while (addr_new) /* Loop until all addresses dealt with */ continue; } - /* Get the routing retry status, saving the two retry keys (with and - without the local part) for subsequent use. If there is no retry record for - the standard address routing retry key, we look for the same key with the - sender attached, because this form is used by the smtp transport after a - 4xx response to RCPT when address_retry_include_sender is true. */ - - addr->domain_retry_key = string_sprintf("R:%s", addr->domain); - addr->address_retry_key = string_sprintf("R:%s@%s", addr->local_part, - addr->domain); - - if (dbm_file) + if (f.queue_2stage) { - domain_retry_record = dbfn_read(dbm_file, addr->domain_retry_key); - if ( domain_retry_record - && now - domain_retry_record->time_stamp > retry_data_expire - ) + DEBUG(D_deliver) + debug_printf_indent("no router retry check (ph1 qrun)\n"); + } + else + { + /* Get the routing retry status, saving the two retry keys (with and + without the local part) for subsequent use. If there is no retry record + for the standard address routing retry key, we look for the same key with + the sender attached, because this form is used by the smtp transport after + a 4xx response to RCPT when address_retry_include_sender is true. */ + + DEBUG(D_deliver|D_retry) { - DEBUG(D_deliver|D_retry) - debug_printf("domain retry record present but expired\n"); - domain_retry_record = NULL; /* Ignore if too old */ + debug_printf_indent("checking router retry status\n"); + acl_level++; } + addr->domain_retry_key = string_sprintf("R:%s", addr->domain); + addr->address_retry_key = string_sprintf("R:%s@%s", addr->local_part, + addr->domain); - address_retry_record = dbfn_read(dbm_file, addr->address_retry_key); - if ( address_retry_record - && now - address_retry_record->time_stamp > retry_data_expire - ) + if (dbm_file) { - DEBUG(D_deliver|D_retry) - debug_printf("address retry record present but expired\n"); - address_retry_record = NULL; /* Ignore if too old */ - } + domain_retry_record = dbfn_read(dbm_file, addr->domain_retry_key); + if ( domain_retry_record + && now - domain_retry_record->time_stamp > retry_data_expire + ) + { + DEBUG(D_deliver|D_retry) + debug_printf_indent("domain retry record present but expired\n"); + domain_retry_record = NULL; /* Ignore if too old */ + } - if (!address_retry_record) - { - uschar *altkey = string_sprintf("%s:<%s>", addr->address_retry_key, - sender_address); - address_retry_record = dbfn_read(dbm_file, altkey); - if ( address_retry_record - && now - address_retry_record->time_stamp > retry_data_expire) + address_retry_record = dbfn_read(dbm_file, addr->address_retry_key); + if ( address_retry_record + && now - address_retry_record->time_stamp > retry_data_expire + ) { DEBUG(D_deliver|D_retry) - debug_printf("address retry record present but expired\n"); - address_retry_record = NULL; /* Ignore if too old */ + debug_printf_indent("address retry record present but expired\n"); + address_retry_record = NULL; /* Ignore if too old */ } - } - } - else - domain_retry_record = address_retry_record = NULL; - DEBUG(D_deliver|D_retry) - { - if (!domain_retry_record) - debug_printf("no domain retry record\n"); - else - debug_printf("have domain retry record; next_try = now%+d\n", - f.running_in_test_harness ? 0 : - (int)(domain_retry_record->next_try - now)); + if (!address_retry_record) + { + uschar *altkey = string_sprintf("%s:<%s>", addr->address_retry_key, + sender_address); + address_retry_record = dbfn_read(dbm_file, altkey); + if ( address_retry_record + && now - address_retry_record->time_stamp > retry_data_expire) + { + DEBUG(D_deliver|D_retry) + debug_printf_indent("address retry record present but expired\n"); + address_retry_record = NULL; /* Ignore if too old */ + } + } + } - if (!address_retry_record) - debug_printf("no address retry record\n"); - else - debug_printf("have address retry record; next_try = now%+d\n", - f.running_in_test_harness ? 0 : - (int)(address_retry_record->next_try - now)); + DEBUG(D_deliver|D_retry) + { + if (!domain_retry_record) + debug_printf_indent("no domain retry record\n"); + else + debug_printf_indent("have domain retry record; next_try = now%+d\n", + f.running_in_test_harness ? 0 : + (int)(domain_retry_record->next_try - now)); + + if (!address_retry_record) + debug_printf_indent("no address retry record\n"); + else + debug_printf_indent("have address retry record; next_try = now%+d\n", + f.running_in_test_harness ? 0 : + (int)(address_retry_record->next_try - now)); + acl_level--; + } } /* If we are sending a message down an existing SMTP connection, we must @@ -7605,10 +7917,15 @@ while (addr_new) /* Loop until all addresses dealt with */ } } - /* The database is closed while routing is actually happening. Requests to - update it are put on a chain and all processed together at the end. */ + /* If not transaction-capable, the database is closed while routing is + actually happening. Requests to update it are put on a chain and all processed + together at the end. */ - if (dbm_file) dbfn_close(dbm_file); + if (dbm_file) + if (exim_lockfile_needed()) + { dbfn_close(dbm_file); continue_retry_db = dbm_file = NULL; } + else + DEBUG(D_hints_lookup) debug_printf("retaining retry hintsdb handle\n"); /* If queue_domains is set, we don't even want to try routing addresses in those domains. During queue runs, queue_domains is forced to be unset. @@ -7683,7 +8000,7 @@ while (addr_new) /* Loop until all addresses dealt with */ else if (testflag(addr, af_dr_retry_exists)) { - uschar *altkey = string_sprintf("%s:<%s>", addr->address_retry_key, + uschar * altkey = string_sprintf("%s:<%s>", addr->address_retry_key, sender_address); retry_add_item(addr, altkey, rf_delete); retry_add_item(addr, addr->address_retry_key, rf_delete); @@ -7777,8 +8094,7 @@ while (addr_new) /* Loop until all addresses dealt with */ } } /* Continue with routing the next address. */ } /* Loop to process any child addresses that the routers created, and - any rerouted addresses that got put back on the new chain. */ - + any rerouted addresses that got put back on the new chain. */ /* Debugging: show the results of the routing */ @@ -7878,12 +8194,7 @@ if ( mua_wrapper /* If this is a run to continue deliveries to an external channel that is -already set up, defer any local deliveries. - -jgh 2020/12/20: I don't see why; locals should be quick. -The defer goes back to version 1.62 in 1997. A local being still deliverable -during a continued run might result from something like a defer during the -original delivery, eg. in a DB lookup. Unlikely but possible. +already set up, defer any local deliveries because we are handling remotes. To avoid delaying a local when combined with a callout-hold for a remote delivery, test continue_sequence rather than continue_transport. */ @@ -8131,9 +8442,10 @@ if (mua_wrapper) /* In a normal configuration, we now update the retry database. This is done in one fell swoop at the end in order not to keep opening and closing (and -locking) the database. The code for handling retries is hived off into a -separate module for convenience. We pass it the addresses of the various -chains, because deferred addresses can get moved onto the failed chain if the +locking) the database (at least, for non-transaction-capable DBs. +The code for handling retries is hived off into a separate module for +convenience. We pass it the addresses of the various chains, +because deferred addresses can get moved onto the failed chain if the retry cutoff time has expired for all alternative destinations. Bypass the updating of the database if the -N flag is set, which is a debugging thing that prevents actual delivery. */ @@ -8235,6 +8547,13 @@ f.disable_logging = FALSE; /* In case left set */ DELIVERY_TIDYUP: +if (dbm_file) /* Can only be continue_retry_db */ + { + DEBUG(D_hints_lookup) debug_printf("final close of cached retry db\n"); + dbfn_close_multi(continue_retry_db); + continue_retry_db = dbm_file = NULL; + } + /* If there are now no deferred addresses, we are done. Preserve the message log if so configured, and we are using them. Otherwise, sling it. Then delete the message itself. */ @@ -8365,7 +8684,7 @@ else if (addr_defer != (address_item *)(+1)) for (i = 0; i < recipients_count; i++) { - uschar *r = recipients_list[i].address; + const uschar * r = recipients_list[i].address; if (Ustrcmp(otaddr->onetime_parent, r) == 0) t = i; if (Ustrcmp(otaddr->address, r) == 0) break; } @@ -8393,7 +8712,7 @@ else if (addr_defer != (address_item *)(+1)) if (sender_address[0]) { - uschar * s = addr->prop.errors_address; + const uschar * s = addr->prop.errors_address; if (!s) s = sender_address; if (Ustrstr(recipients, s) == NULL) recipients = string_sprintf("%s%s%s", recipients, @@ -8412,54 +8731,57 @@ else if (addr_defer != (address_item *)(+1)) || addr_defer->dsn_flags & rf_notify_delay ) && delay_warning[1] > 0 - && sender_address[0] != 0 - && ( !delay_warning_condition - || expand_check_condition(delay_warning_condition, - US"delay_warning", US"option") - ) - ) + && sender_address[0] != 0) { - int count; - int show_time; - int queue_time = time(NULL) - received_time.tv_sec; - - queue_time = test_harness_fudged_queue_time(queue_time); - - /* See how many warnings we should have sent by now */ + GET_OPTION("delay_warning_condition"); + if ( ( !delay_warning_condition + || expand_check_condition(delay_warning_condition, + US"delay_warning", US"option") + ) + ) + { + int count; + int show_time; + int queue_time = time(NULL) - received_time.tv_sec; - for (count = 0; count < delay_warning[1]; count++) - if (queue_time < delay_warning[count+2]) break; + queue_time = test_harness_fudged_queue_time(queue_time); - show_time = delay_warning[count+1]; + /* See how many warnings we should have sent by now */ - if (count >= delay_warning[1]) - { - int extra; - int last_gap = show_time; - if (count > 1) last_gap -= delay_warning[count]; - extra = (queue_time - delay_warning[count+1])/last_gap; - show_time += last_gap * extra; - count += extra; - } + for (count = 0; count < delay_warning[1]; count++) + if (queue_time < delay_warning[count+2]) break; - DEBUG(D_deliver) - { - debug_printf("time on queue = %s id %s addr %s\n", - readconf_printtime(queue_time), message_id, addr_defer->address); - debug_printf("warning counts: required %d done %d\n", count, - warning_count); - } + show_time = delay_warning[count+1]; - /* We have computed the number of warnings there should have been by now. - If there haven't been enough, send one, and up the count to what it should - have been. */ + if (count >= delay_warning[1]) + { + int extra; + int last_gap = show_time; + if (count > 1) last_gap -= delay_warning[count]; + extra = (queue_time - delay_warning[count+1])/last_gap; + show_time += last_gap * extra; + count += extra; + } - if (warning_count < count) - if (send_warning_message(recipients, queue_time, show_time)) + DEBUG(D_deliver) { - warning_count = count; - update_spool = TRUE; /* Ensure spool rewritten */ + debug_printf("time on queue = %s id %s addr %s\n", + readconf_printtime(queue_time), message_id, addr_defer->address); + debug_printf("warning counts: required %d done %d\n", count, + warning_count); } + + /* We have computed the number of warnings there should have been by now. + If there haven't been enough, send one, and up the count to what it should + have been. */ + + if (warning_count < count) + if (send_warning_message(recipients, queue_time, show_time)) + { + warning_count = count; + update_spool = TRUE; /* Ensure spool rewritten */ + } + } } /* Clear deliver_domain */ @@ -8574,6 +8896,17 @@ DEBUG(D_deliver) debug_printf("end delivery of %s\n", id); report_time_since(×tamp_startup, US"delivery end"); /* testcase 0005 */ #endif +/* If the transport suggested another message to deliver, go round again. */ + +if (final_yield == DELIVER_ATTEMPTED_NORMAL && *continue_next_id) + { + addr_defer = addr_failed = addr_succeed = NULL; + tree_duplicates = NULL; /* discard dups info from old message */ + id = string_copyn(continue_next_id, MESSAGE_ID_LENGTH); + continue_next_id[0] = '\0'; + goto CONTINUED_ID; + } + /* It is unlikely that there will be any cached resources, since they are released after routing, and in the delivery subprocesses. However, it's possible for an expansion for something afterwards (for example,