X-Git-Url: https://git.exim.org/exim.git/blobdiff_plain/6acb441b40bbcded2e85819c71a068db713e7ca6..9ae8613607b12257e1fe59c603119bc5d4cd3474:/src/src/transports/smtp.c diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c index 301d84c2e..ddd6ad7e1 100644 --- a/src/src/transports/smtp.c +++ b/src/src/transports/smtp.c @@ -2,9 +2,10 @@ * Exim - an Internet mail transport agent * *************************************************/ +/* Copyright (c) The Exim Maintainers 2020 - 2023 */ /* Copyright (c) University of Cambridge 1995 - 2018 */ -/* Copyright (c) The Exim Maintainers 2020 */ /* See the file NOTICE for conditions of use and distribution. */ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #include "../exim.h" #include "smtp.h" @@ -64,6 +65,9 @@ optionlist smtp_transport_options[] = { { "final_timeout", opt_time, LOFF(final_timeout) }, { "gethostbyname", opt_bool, LOFF(gethostbyname) }, { "helo_data", opt_stringptr, LOFF(helo_data) }, +#if !defined(DISABLE_TLS) && !defined(DISABLE_TLS_RESUME) + { "host_name_extract", opt_stringptr, LOFF(host_name_extract) }, +# endif { "hosts", opt_stringptr, LOFF(hosts) }, { "hosts_avoid_esmtp", opt_stringptr, LOFF(hosts_avoid_esmtp) }, { "hosts_avoid_pipelining", opt_stringptr, LOFF(hosts_avoid_pipelining) }, @@ -84,6 +88,7 @@ optionlist smtp_transport_options[] = { #if !defined(DISABLE_TLS) && !defined(DISABLE_OCSP) { "hosts_request_ocsp", opt_stringptr, LOFF(hosts_request_ocsp) }, #endif + { "hosts_require_alpn", opt_stringptr, LOFF(hosts_require_alpn) }, { "hosts_require_auth", opt_stringptr, LOFF(hosts_require_auth) }, #ifndef DISABLE_TLS # ifdef SUPPORT_DANE @@ -109,7 +114,7 @@ optionlist smtp_transport_options[] = { { "interface", opt_stringptr, LOFF(interface) }, { "keepalive", opt_bool, LOFF(keepalive) }, { "lmtp_ignore_quota", opt_bool, LOFF(lmtp_ignore_quota) }, - { "max_rcpt", opt_int | opt_public, + { "max_rcpt", opt_stringptr | opt_public, OPT_OFF(transport_instance, max_addresses) }, { "message_linelength_limit", opt_int, LOFF(message_linelength_limit) }, { "multi_domain", opt_expand_bool | opt_public, @@ -123,6 +128,7 @@ optionlist smtp_transport_options[] = { { "socks_proxy", opt_stringptr, LOFF(socks_proxy) }, #endif #ifndef DISABLE_TLS + { "tls_alpn", opt_stringptr, LOFF(tls_alpn) }, { "tls_certificate", opt_stringptr, LOFF(tls_certificate) }, { "tls_crl", opt_stringptr, LOFF(tls_crl) }, { "tls_dh_min_bits", opt_int, LOFF(tls_dh_min_bits) }, @@ -192,9 +198,7 @@ smtp_transport_options_block smtp_transport_option_defaults = { .keepalive = TRUE, .retry_include_ip_address = TRUE, #ifndef DISABLE_TLS -# if defined(SUPPORT_SYSDEFAULT_CABUNDLE) || !defined(USE_GNUTLS) .tls_verify_certificates = US"system", -# endif .tls_dh_min_bits = EXIM_CLIENT_DH_DEFAULT_MIN_BITS, .tls_tempfail_tryclear = TRUE, .tls_try_verify_hosts = US"*", @@ -231,49 +235,56 @@ static BOOL pipelining_active; /* current transaction is in pipe mode */ static unsigned ehlo_response(uschar * buf, unsigned checks); +/* sync_responses() return codes */ + +#define RESP_BIT_HAD_5XX BIT(1) +#define RESP_BIT_HAD_2XX BIT(0) +#define RESP_HAD_2_AND_5 (RESP_BIT_HAD_2XX | RESP_BIT_HAD_5XX) +#define RESP_NOERROR 0 +#define RESP_RCPT_TIMEO -1 +#define RESP_RCPT_ERROR -2 +#define RESP_MAIL_OR_DATA_ERROR -3 +#define RESP_EPIPE_EHLO_ERR -4 +#define RESP_EHLO_ERR_TLS -5 /******************************************************************************/ void smtp_deliver_init(void) { -if (!regex_PIPELINING) regex_PIPELINING = - regex_must_compile(US"\\n250[\\s\\-]PIPELINING(\\s|\\n|$)", FALSE, TRUE); - -if (!regex_SIZE) regex_SIZE = - regex_must_compile(US"\\n250[\\s\\-]SIZE(\\s|\\n|$)", FALSE, TRUE); - -if (!regex_AUTH) regex_AUTH = - regex_must_compile(AUTHS_REGEX, FALSE, TRUE); +struct list + { + const pcre2_code ** re; + const uschar * string; + } list[] = + { + { ®ex_AUTH, AUTHS_REGEX }, + { ®ex_CHUNKING, US"\\n250[\\s\\-]CHUNKING(\\s|\\n|$)" }, + { ®ex_DSN, US"\\n250[\\s\\-]DSN(\\s|\\n|$)" }, + { ®ex_IGNOREQUOTA, US"\\n250[\\s\\-]IGNOREQUOTA(\\s|\\n|$)" }, + { ®ex_PIPELINING, US"\\n250[\\s\\-]PIPELINING(\\s|\\n|$)" }, + { ®ex_SIZE, US"\\n250[\\s\\-]SIZE(\\s|\\n|$)" }, #ifndef DISABLE_TLS -if (!regex_STARTTLS) regex_STARTTLS = - regex_must_compile(US"\\n250[\\s\\-]STARTTLS(\\s|\\n|$)", FALSE, TRUE); + { ®ex_STARTTLS, US"\\n250[\\s\\-]STARTTLS(\\s|\\n|$)" }, #endif - -if (!regex_CHUNKING) regex_CHUNKING = - regex_must_compile(US"\\n250[\\s\\-]CHUNKING(\\s|\\n|$)", FALSE, TRUE); - #ifndef DISABLE_PRDR -if (!regex_PRDR) regex_PRDR = - regex_must_compile(US"\\n250[\\s\\-]PRDR(\\s|\\n|$)", FALSE, TRUE); + { ®ex_PRDR, US"\\n250[\\s\\-]PRDR(\\s|\\n|$)" }, #endif - #ifdef SUPPORT_I18N -if (!regex_UTF8) regex_UTF8 = - regex_must_compile(US"\\n250[\\s\\-]SMTPUTF8(\\s|\\n|$)", FALSE, TRUE); + { ®ex_UTF8, US"\\n250[\\s\\-]SMTPUTF8(\\s|\\n|$)" }, #endif - -if (!regex_DSN) regex_DSN = - regex_must_compile(US"\\n250[\\s\\-]DSN(\\s|\\n|$)", FALSE, TRUE); - -if (!regex_IGNOREQUOTA) regex_IGNOREQUOTA = - regex_must_compile(US"\\n250[\\s\\-]IGNOREQUOTA(\\s|\\n|$)", FALSE, TRUE); - #ifndef DISABLE_PIPE_CONNECT -if (!regex_EARLY_PIPE) regex_EARLY_PIPE = - regex_must_compile(US"\\n250[\\s\\-]" EARLY_PIPE_FEATURE_NAME "(\\s|\\n|$)", FALSE, TRUE); + { ®ex_EARLY_PIPE, US"\\n250[\\s\\-]" EARLY_PIPE_FEATURE_NAME "(\\s|\\n|$)" }, #endif +#ifndef DISABLE_ESMTP_LIMITS + { ®ex_LIMITS, US"\\n250[\\s\\-]LIMITS\\s" }, +#endif + }; + +for (struct list * l = list; l < list + nelem(list); l++) + if (!*l->re) + *l->re = regex_must_compile(l->string, MCS_NOFLAGS, TRUE); } @@ -349,7 +360,8 @@ Returns: nothing void smtp_transport_init(transport_instance *tblock) { -smtp_transport_options_block *ob = SOB tblock->options_block; +smtp_transport_options_block * ob = SOB tblock->options_block; +int old_pool = store_pool; /* Retry_use_local_part defaults FALSE if unset */ @@ -380,12 +392,14 @@ if (ob->command_timeout <= 0 || ob->data_timeout <= 0 || /* If hosts_override is set and there are local hosts, set the global flag that stops verify from showing router hosts. */ -if (ob->hosts_override && ob->hosts != NULL) tblock->overrides_hosts = TRUE; +if (ob->hosts_override && ob->hosts) tblock->overrides_hosts = TRUE; /* If there are any fallback hosts listed, build a chain of host items for them, but do not do any lookups at this time. */ -host_build_hostlist(&(ob->fallback_hostlist), ob->fallback_hosts, FALSE); +store_pool = POOL_PERM; +host_build_hostlist(&ob->fallback_hostlist, ob->fallback_hosts, FALSE); +store_pool = old_pool; } @@ -519,7 +533,7 @@ switch(*errno_value) case ERRNO_SMTPFORMAT: /* Handle malformed SMTP response */ s = string_printing(buffer); - while (isspace(*s)) s++; + Uskip_whitespace(&s); *message = *s == 0 ? string_sprintf("Malformed SMTP reply (an empty line) " "in response to %s%s", pl, smtp_command) @@ -620,8 +634,8 @@ if (suffix) else message = string_fmt_append(message, " %s", exim_errstr(basic_errno)); -log_write(0, LOG_MAIN, "%s", string_from_gstring(message)); -deliver_msglog("%s %s\n", tod_stamp(tod_log), message->s); +log_write(0, LOG_MAIN, "%Y", message); +deliver_msglog("%s %.*s\n", tod_stamp(tod_log), message->ptr, message->s); } static void @@ -653,8 +667,7 @@ static void deferred_event_raise(address_item * addr, host_item * host, uschar * evstr) { uschar * action = addr->transport->event_action; -const uschar * save_domain; -uschar * save_local; +const uschar * save_domain, * save_local; if (!action) return; @@ -679,7 +692,8 @@ deliver_localpart = addr->local_part; : string_copy(addr->message) : addr->basic_errno > 0 ? string_copy(US strerror(addr->basic_errno)) - : NULL); + : NULL, + NULL); deliver_localpart = save_local; deliver_domain = save_domain; @@ -712,7 +726,19 @@ return count; static BOOL smtp_reap_banner(smtp_context * sx) { -BOOL good_response = smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), +BOOL good_response; +#if defined(__linux__) && defined(TCP_QUICKACK) + { /* Hack to get QUICKACK disabled; has to be right after 3whs, and has to on->off */ + int sock = sx->cctx.sock; + struct pollfd p = {.fd = sock, .events = POLLOUT}; + if (poll(&p, 1, 1000) >= 0) /* retval test solely for compiler quitening */ + { + (void) setsockopt(sock, IPPROTO_TCP, TCP_QUICKACK, US &on, sizeof(on)); + (void) setsockopt(sock, IPPROTO_TCP, TCP_QUICKACK, US &off, sizeof(off)); + } + } +#endif +good_response = smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2', (SOB sx->conn_args.ob)->command_timeout); #ifdef EXPERIMENTAL_DSN_INFO sx->smtp_greeting = string_copy(sx->buffer); @@ -740,12 +766,111 @@ sx->helo_response = string_copy(sx->buffer); #endif #ifndef DISABLE_EVENT (void) event_raise(sx->conn_args.tblock->event_action, - US"smtp:ehlo", sx->buffer); + US"smtp:ehlo", sx->buffer, NULL); #endif return TRUE; } +#if !defined(DISABLE_TLS) && !defined(DISABLE_TLS_RESUME) + +/* Grab a string differentiating server behind a loadbalancer, for TLS +resumption when such servers do not share a session-cache */ + +static void +ehlo_response_lbserver(smtp_context * sx, const uschar * name_extract) +{ +const uschar * s; +uschar * save_item = iterate_item; + +if (sx->conn_args.have_lbserver) + return; +iterate_item = sx->buffer; +s = expand_cstring(name_extract); +iterate_item = save_item; +sx->conn_args.host_lbserver = s && !*s ? NULL : s; +sx->conn_args.have_lbserver = TRUE; +} +#endif + + + +/******************************************************************************/ + +#ifndef DISABLE_ESMTP_LIMITS +/* If TLS, or TLS not offered, called with the EHLO response in the buffer. +Check it for a LIMITS keyword and parse values into the smtp context structure. + +We don't bother with peers that we won't talk TLS to, even though they can, +just ignore their LIMITS advice (if any) and treat them as if they do not. +This saves us dealing with a duplicate set of values. */ + +static void +ehlo_response_limits_read(smtp_context * sx) +{ +uschar * match; + +/* matches up to just after the first space after the keyword */ + +if (regex_match(regex_LIMITS, sx->buffer, -1, &match)) + for (const uschar * s = sx->buffer + Ustrlen(match); *s; ) + { + if (Uskip_whitespace(&s) == '\n') break; + + if (strncmpic(s, US"MAILMAX=", 8) == 0) + { + sx->peer_limit_mail = atoi(CS (s += 8)); + while (isdigit(*s)) s++; + } + else if (strncmpic(s, US"RCPTMAX=", 8) == 0) + { + sx->peer_limit_rcpt = atoi(CS (s += 8)); + while (isdigit(*s)) s++; + } + else if (strncmpic(s, US"RCPTDOMAINMAX=", 14) == 0) + { + sx->peer_limit_rcptdom = atoi(CS (s += 14)); + while (isdigit(*s)) s++; + } + else + while (*s && !isspace(*s)) s++; + } +} + +/* Apply given values to the current connection */ +static void +ehlo_limits_apply(smtp_context * sx, + unsigned limit_mail, unsigned limit_rcpt, unsigned limit_rcptdom) +{ +if (limit_mail && limit_mail < sx->max_mail) sx->max_mail = limit_mail; +if (limit_rcpt && limit_rcpt < sx->max_rcpt) sx->max_rcpt = limit_rcpt; +if (limit_rcptdom) + { + DEBUG(D_transport) debug_printf("will treat as !multi_domain\n"); + sx->single_rcpt_domain = TRUE; + } +} + +/* Apply values from EHLO-resp to the current connection */ +static void +ehlo_response_limits_apply(smtp_context * sx) +{ +ehlo_limits_apply(sx, sx->peer_limit_mail, sx->peer_limit_rcpt, + sx->peer_limit_rcptdom); +} + +/* Apply values read from cache to the current connection */ +static void +ehlo_cache_limits_apply(smtp_context * sx) +{ +# ifndef DISABLE_PIPE_CONNECT +ehlo_limits_apply(sx, sx->ehlo_resp.limit_mail, sx->ehlo_resp.limit_rcpt, + sx->ehlo_resp.limit_rcptdom); +# endif +} +#endif /*EXPERIMENTAL_ESMTP_LIMITS*/ + +/******************************************************************************/ #ifndef DISABLE_PIPE_CONNECT static uschar * @@ -759,19 +884,45 @@ return Ustrchr(host->address, ':') host->port == PORT_NONE ? sx->port : host->port); } +/* Cache EHLO-response info for use by early-pipe. +Called +- During a normal flow on EHLO response (either cleartext or under TLS), + when we are willing to do PIPECONNECT and it is offered +- During an early-pipe flow on receiving the actual EHLO response and noting + disparity versus the cached info used, when PIPECONNECT is still being offered + +We assume that suitable values have been set in the sx.ehlo_resp structure for +features and auths; we handle the copy of limits. */ + static void -write_ehlo_cache_entry(const smtp_context * sx) +write_ehlo_cache_entry(smtp_context * sx) { open_db dbblock, * dbm_file; +# ifndef DISABLE_ESMTP_LIMITS +sx->ehlo_resp.limit_mail = sx->peer_limit_mail; +sx->ehlo_resp.limit_rcpt = sx->peer_limit_rcpt; +sx->ehlo_resp.limit_rcptdom = sx->peer_limit_rcptdom; +# endif + if ((dbm_file = dbfn_open(US"misc", O_RDWR, &dbblock, TRUE, TRUE))) { uschar * ehlo_resp_key = ehlo_cache_key(sx); dbdata_ehlo_resp er = { .data = sx->ehlo_resp }; - HDEBUG(D_transport) debug_printf("writing clr %04x/%04x cry %04x/%04x\n", - sx->ehlo_resp.cleartext_features, sx->ehlo_resp.cleartext_auths, - sx->ehlo_resp.crypted_features, sx->ehlo_resp.crypted_auths); + HDEBUG(D_transport) +# ifndef DISABLE_ESMTP_LIMITS + if (sx->ehlo_resp.limit_mail || sx->ehlo_resp.limit_rcpt || sx->ehlo_resp.limit_rcptdom) + debug_printf("writing clr %04x/%04x cry %04x/%04x lim %05d/%05d/%05d\n", + sx->ehlo_resp.cleartext_features, sx->ehlo_resp.cleartext_auths, + sx->ehlo_resp.crypted_features, sx->ehlo_resp.crypted_auths, + sx->ehlo_resp.limit_mail, sx->ehlo_resp.limit_rcpt, + sx->ehlo_resp.limit_rcptdom); + else +# endif + debug_printf("writing clr %04x/%04x cry %04x/%04x\n", + sx->ehlo_resp.cleartext_features, sx->ehlo_resp.cleartext_auths, + sx->ehlo_resp.crypted_features, sx->ehlo_resp.crypted_auths); dbfn_write(dbm_file, ehlo_resp_key, &er, (int)sizeof(er)); dbfn_close(dbm_file); @@ -787,6 +938,17 @@ if ( sx->early_pipe_active && (dbm_file = dbfn_open(US"misc", O_RDWR, &dbblock, TRUE, TRUE))) { uschar * ehlo_resp_key = ehlo_cache_key(sx); + HDEBUG(D_transport) + { + dbdata_ehlo_resp * er; + + if (!(er = dbfn_read_enforce_length(dbm_file, ehlo_resp_key, sizeof(dbdata_ehlo_resp)))) + debug_printf("no ehlo-resp record!\n"); + else + debug_printf("ehlo-resp record is %.0f seconds old\n", + difftime(time(NULL), er->time_stamp)); + } + dbfn_delete(dbm_file, ehlo_resp_key); dbfn_close(dbm_file); } @@ -816,12 +978,26 @@ else } else { + DEBUG(D_transport) +# ifndef DISABLE_ESMTP_LIMITS + if (er->data.limit_mail || er->data.limit_rcpt || er->data.limit_rcptdom) + debug_printf("EHLO response bits from cache:" + " cleartext 0x%04x/0x%04x crypted 0x%04x/0x%04x lim %05d/%05d/%05d\n", + er->data.cleartext_features, er->data.cleartext_auths, + er->data.crypted_features, er->data.crypted_auths, + er->data.limit_mail, er->data.limit_rcpt, er->data.limit_rcptdom); + else +# endif + debug_printf("EHLO response bits from cache:" + " cleartext 0x%04x/0x%04x crypted 0x%04x/0x%04x\n", + er->data.cleartext_features, er->data.cleartext_auths, + er->data.crypted_features, er->data.crypted_auths); + sx->ehlo_resp = er->data; +# ifndef DISABLE_ESMTP_LIMITS + ehlo_cache_limits_apply(sx); +# endif dbfn_close(dbm_file); - DEBUG(D_transport) debug_printf( - "EHLO response bits from cache: cleartext 0x%04x/0x%04x crypted 0x%04x/0x%04x\n", - er->data.cleartext_features, er->data.cleartext_auths, - er->data.crypted_features, er->data.crypted_auths); return TRUE; } dbfn_close(dbm_file); @@ -832,7 +1008,7 @@ return FALSE; /* Return an auths bitmap for the set of AUTH methods offered by the server -which match our authenticators. */ +which match our client-side authenticators. */ static unsigned short study_ehlo_auths(smtp_context * sx) @@ -843,7 +1019,7 @@ uschar authnum; unsigned short authbits = 0; if (!sx->esmtp) return 0; -if (!regex_AUTH) regex_AUTH = regex_must_compile(AUTHS_REGEX, FALSE, TRUE); +if (!regex_AUTH) regex_AUTH = regex_must_compile(AUTHS_REGEX, MCS_NOFLAGS, TRUE); if (!regex_match_and_setup(regex_AUTH, sx->buffer, 0, -1)) return 0; expand_nmax = -1; /* reset */ names = string_copyn(expand_nstring[1], expand_nlength[1]); @@ -858,7 +1034,7 @@ for (au = auths, authnum = 0; au; au = au->next, authnum++) if (au->client) } DEBUG(D_transport) - debug_printf("server offers %s AUTH, methods '%s', bitmap 0x%04x\n", + debug_printf("server offers %s AUTH, methods '%s', usable-bitmap 0x%04x\n", tls_out.active.sock >= 0 ? "crypted" : "plaintext", names, authbits); if (tls_out.active.sock >= 0) @@ -899,6 +1075,8 @@ sx->pending_EHLO = FALSE; if (pending_BANNER) { + const uschar * s; + DEBUG(D_transport) debug_printf("%s expect banner\n", __FUNCTION__); (*countp)--; if (!smtp_reap_banner(sx)) @@ -907,6 +1085,14 @@ if (pending_BANNER) if (tls_out.active.sock >= 0) rc = DEFER; goto fail; } + /*XXX EXPERIMENTAL_ESMTP_LIMITS ? */ + +# ifndef DISABLE_TLS_RESUME + GET_OPTION("host_name_extract"); + s = ((smtp_transport_options_block *)sx->conn_args.ob)->host_name_extract; + if (!s) s = HNE_DEFAULT; + ehlo_response_lbserver(sx, s); +# endif } if (pending_EHLO) @@ -923,8 +1109,9 @@ if (pending_EHLO) goto fail; } - /* Compare the actual EHLO response to the cached value we assumed; - on difference, dump or rewrite the cache and arrange for a retry. */ + /* Compare the actual EHLO response extensions and AUTH methods to the cached + value we assumed; on difference, dump or rewrite the cache and arrange for a + retry. */ ap = tls_out.active.sock < 0 ? &sx->ehlo_resp.cleartext_auths : &sx->ehlo_resp.crypted_auths; @@ -934,6 +1121,10 @@ if (pending_EHLO) | OPTION_CHUNKING | OPTION_PRDR | OPTION_DSN | OPTION_PIPE | OPTION_SIZE | OPTION_UTF8 | OPTION_EARLY_PIPE ); +# ifndef DISABLE_ESMTP_LIMITS + if (tls_out.active.sock >= 0 || !(peer_offered & OPTION_TLS)) + ehlo_response_limits_read(sx); +# endif if ( peer_offered != sx->peer_offered || (authbits = study_ehlo_auths(sx)) != *ap) { @@ -941,16 +1132,47 @@ if (pending_EHLO) debug_printf("EHLO %s extensions changed, 0x%04x/0x%04x -> 0x%04x/0x%04x\n", tls_out.active.sock < 0 ? "cleartext" : "crypted", sx->peer_offered, *ap, peer_offered, authbits); - *(tls_out.active.sock < 0 - ? &sx->ehlo_resp.cleartext_features : &sx->ehlo_resp.crypted_features) = peer_offered; - *ap = authbits; if (peer_offered & OPTION_EARLY_PIPE) + { + *(tls_out.active.sock < 0 + ? &sx->ehlo_resp.cleartext_features : &sx->ehlo_resp.crypted_features) = + peer_offered; + *ap = authbits; write_ehlo_cache_entry(sx); + } else + { invalidate_ehlo_cache_entry(sx); + sx->early_pipe_active = FALSE; /* cancel further early-pipe on this conn */ + } return OK; /* just carry on */ } +# ifndef DISABLE_ESMTP_LIMITS + /* If we are handling LIMITS, compare the actual EHLO LIMITS values with the + cached values and invalidate cache if different. OK to carry on with + connect since values are advisory. */ + { + if ( (tls_out.active.sock >= 0 || !(peer_offered & OPTION_TLS)) + && ( sx->peer_limit_mail != sx->ehlo_resp.limit_mail + || sx->peer_limit_rcpt != sx->ehlo_resp.limit_rcpt + || sx->peer_limit_rcptdom != sx->ehlo_resp.limit_rcptdom + ) ) + { + HDEBUG(D_transport) + { + debug_printf("EHLO LIMITS changed:"); + if (sx->peer_limit_mail != sx->ehlo_resp.limit_mail) + debug_printf(" MAILMAX %u -> %u\n", sx->ehlo_resp.limit_mail, sx->peer_limit_mail); + else if (sx->peer_limit_rcpt != sx->ehlo_resp.limit_rcpt) + debug_printf(" RCPTMAX %u -> %u\n", sx->ehlo_resp.limit_rcpt, sx->peer_limit_rcpt); + else + debug_printf(" RCPTDOMAINMAX %u -> %u\n", sx->ehlo_resp.limit_rcptdom, sx->peer_limit_rcptdom); + } + invalidate_ehlo_cache_entry(sx); + } + } +# endif } return OK; @@ -959,7 +1181,7 @@ fail: (void) smtp_discard_responses(sx, sx->conn_args.ob, *countp); return rc; } -#endif +#endif /*!DISABLE_PIPE_CONNECT*/ /************************************************* @@ -1015,7 +1237,7 @@ int yield = 0; #ifndef DISABLE_PIPE_CONNECT int rc; if ((rc = smtp_reap_early_pipe(sx, &count)) != OK) - return rc == FAIL ? -4 : -5; + return rc == FAIL ? RESP_EPIPE_EHLO_ERR : RESP_EHLO_ERR_TLS; #endif /* Handle the response for a MAIL command. On error, reinstate the original @@ -1026,14 +1248,14 @@ if (sx->pending_MAIL) { DEBUG(D_transport) debug_printf("%s expect mail\n", __FUNCTION__); count--; - sx->pending_MAIL = FALSE; + sx->pending_MAIL = sx->RCPT_452 = FALSE; if (!smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2', ob->command_timeout)) { DEBUG(D_transport) debug_printf("bad response for MAIL\n"); Ustrcpy(big_buffer, mail_command); /* Fits, because it came from there! */ if (errno == ERRNO_TLSFAILURE) - return -5; + return RESP_EHLO_ERR_TLS; if (errno == 0 && sx->buffer[0] != 0) { int save_errno = 0; @@ -1053,7 +1275,7 @@ if (sx->pending_MAIL) addr->host_used = sx->conn_args.host; addr = addr->next; } - return -3; + return RESP_MAIL_OR_DATA_ERROR; } } @@ -1067,16 +1289,16 @@ while (count-- > 0) { while (addr->transport_return != PENDING_DEFER) if (!(addr = addr->next)) - return -2; + return RESP_RCPT_ERROR; /* The address was accepted */ addr->host_used = sx->conn_args.host; - DEBUG(D_transport) debug_printf("%s expect rcpt\n", __FUNCTION__); + DEBUG(D_transport) debug_printf("%s expect rcpt for %s\n", __FUNCTION__, addr->address); if (smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2', ob->command_timeout)) { - yield |= 1; + yield |= RESP_BIT_HAD_2XX; addr->transport_return = PENDING_OK; /* If af_dr_retry_exists is set, there was a routing delay on this address; @@ -1085,7 +1307,7 @@ while (count-- > 0) if (testflag(addr, af_dr_retry_exists)) { - uschar *altkey = string_sprintf("%s:<%s>", addr->address_retry_key, + uschar * altkey = string_sprintf("%s:<%s>", addr->address_retry_key, sender_address); retry_add_item(addr, altkey, rf_delete); retry_add_item(addr, addr->address_retry_key, rf_delete); @@ -1095,18 +1317,18 @@ while (count-- > 0) /* Error on first TLS read */ else if (errno == ERRNO_TLSFAILURE) - return -5; + return RESP_EHLO_ERR_TLS; /* Timeout while reading the response */ else if (errno == ETIMEDOUT) { - uschar *message = string_sprintf("SMTP timeout after RCPT TO:<%s>", + uschar * message = string_sprintf("SMTP timeout after RCPT TO:<%s>", transport_rcpt_address(addr, sx->conn_args.tblock->rcpt_include_affixes)); set_errno_nohost(sx->first_addr, ETIMEDOUT, message, DEFER, FALSE, &sx->delivery_start); retry_add_item(addr, addr->address_retry_key, 0); update_waiting = FALSE; - return -1; + return RESP_RCPT_TIMEO; } /* Handle other errors in obtaining an SMTP response by returning -1. This @@ -1124,7 +1346,7 @@ while (count-- > 0) g = string_fmt_append_f(g, SVFMT_TAINT_NOCHK, "RCPT TO:<%s>", transport_rcpt_address(addr, sx->conn_args.tblock->rcpt_include_affixes)); string_from_gstring(g); - return -2; + return RESP_RCPT_ERROR; } /* Handle SMTP permanent and temporary response codes. */ @@ -1144,7 +1366,7 @@ while (count-- > 0) if (sx->buffer[0] == '5') { addr->transport_return = FAIL; - yield |= 2; + yield |= RESP_BIT_HAD_5XX; } /* The response was 4xx */ @@ -1164,9 +1386,9 @@ while (count-- > 0) /* If a 452 and we've had at least one 2xx or 5xx, set next_addr to the start point for another MAIL command. */ - if (addr->more_errno >> 8 == 52 && yield & 3) + if (addr->more_errno >> 8 == 52 && yield > 0) { - if (!sx->RCPT_452) + if (!sx->RCPT_452) /* initialised at MAIL-ack above */ { DEBUG(D_transport) debug_printf("%s: seen first 452 too-many-rcpts\n", __FUNCTION__); @@ -1213,6 +1435,8 @@ while (count-- > 0) } } } + if (count && !(addr = addr->next)) + return RESP_RCPT_ERROR; } /* Loop for next RCPT response */ /* Update where to start at for the next block of responses, unless we @@ -1234,16 +1458,16 @@ if (pending_DATA != 0) BOOL pass_message; if (errno == ERRNO_TLSFAILURE) /* Error on first TLS read */ - return -5; + return RESP_EHLO_ERR_TLS; - if (pending_DATA > 0 || (yield & 1) != 0) + if (pending_DATA > 0 || yield & RESP_BIT_HAD_2XX) { if (errno == 0 && sx->buffer[0] == '4') { errno = ERRNO_DATA4XX; sx->first_addr->more_errno |= ((sx->buffer[1] - '0')*10 + sx->buffer[2] - '0') << 8; } - return -3; + return RESP_MAIL_OR_DATA_ERROR; } (void)check_response(sx->conn_args.host, &errno, 0, sx->buffer, &code, &msg, &pass_message); DEBUG(D_transport) debug_printf("%s\nerror for DATA ignored: pipelining " @@ -1271,11 +1495,18 @@ smtp_transport_options_block * ob = sx->conn_args.ob; /* transport options */ host_item * host = sx->conn_args.host; /* host to deliver to */ int rc; +/* Set up globals for error messages */ + +authenticator_name = au->name; +driver_srcfile = au->srcfile; +driver_srcline = au->srcline; + sx->outblock.authenticating = TRUE; rc = (au->info->clientcode)(au, sx, ob->command_timeout, sx->buffer, sizeof(sx->buffer)); sx->outblock.authenticating = FALSE; -DEBUG(D_transport) debug_printf("%s authenticator yielded %d\n", au->name, rc); +driver_srcfile = authenticator_name = NULL; driver_srcline = 0; +DEBUG(D_transport) debug_printf("%s authenticator yielded %s\n", au->name, rc_names[rc]); /* A temporary authentication failure must hold up delivery to this host. After a permanent authentication failure, we carry on @@ -1299,10 +1530,25 @@ switch(rc) /* Failure after reading a response */ case FAIL: + { + uschar * logmsg = NULL; + if (errno != 0 || sx->buffer[0] != '5') return FAIL; - log_write(0, LOG_MAIN, "%s authenticator failed H=%s [%s] %s", - au->name, host->name, host->address, sx->buffer); +#ifndef DISABLE_EVENT + { + uschar * save_name = sender_host_authenticated; + sender_host_authenticated = au->name; + if ((logmsg = event_raise(sx->conn_args.tblock->event_action, US"auth:fail", + sx->buffer, NULL))) + log_write(0, LOG_MAIN, "%s", logmsg); + sender_host_authenticated = save_name; + } +#endif + if (!logmsg) + log_write(0, LOG_MAIN, "%s authenticator failed H=%s [%s] %s", + au->name, host->name, host->address, sx->buffer); break; + } /* Failure by some other means. In effect, the authenticator decided it wasn't prepared to handle this case. Typically this @@ -1362,7 +1608,7 @@ f.smtp_authenticated = FALSE; client_authenticator = client_authenticated_id = client_authenticated_sender = NULL; if (!regex_AUTH) - regex_AUTH = regex_must_compile(AUTHS_REGEX, FALSE, TRUE); + regex_AUTH = regex_must_compile(AUTHS_REGEX, MCS_NOFLAGS, TRUE); /* Is the server offering AUTH? */ @@ -1448,8 +1694,8 @@ if ( sx->esmtp { DEBUG(D_transport) debug_printf("skipping %s authenticator: %s\n", au->name, - (au->client)? "client_condition is false" : - "not configured as a client"); + au->client ? "client_condition is false" + : "not configured as a client"); continue; } @@ -1457,15 +1703,14 @@ if ( sx->esmtp while (*p) { - int len = Ustrlen(au->public_name); - int rc; + int len = Ustrlen(au->public_name), rc; - while (isspace(*p)) p++; + Uskip_whitespace(&p); if (strncmpic(au->public_name, p, len) != 0 || - (p[len] != 0 && !isspace(p[len]))) + (p[len] && !isspace(p[len]))) { - while (*p != 0 && !isspace(*p)) p++; + while (*p && !isspace(*p)) p++; continue; } @@ -1488,7 +1733,9 @@ if ( sx->esmtp if (require_auth == OK && !f.smtp_authenticated) { +#ifndef DISABLE_PIPE_CONNECT invalidate_ehlo_cache_entry(sx); +#endif set_errno_nohost(sx->addrlist, ERRNO_AUTHFAIL, string_sprintf("authentication required but %s", fail_reason), DEFER, FALSE, &sx->delivery_start); @@ -1522,6 +1769,7 @@ uschar * local_authenticated_sender = authenticated_sender; authenticated_sender, ob->authenticated_sender, f.smtp_authenticated?"Y":"N"); #endif +GET_OPTION("authenticated_sender"); if (ob->authenticated_sender) { uschar * new = expand_string(ob->authenticated_sender); @@ -1559,7 +1807,7 @@ return FALSE; typedef struct smtp_compare_s { - uschar * current_sender_address; + const uschar * current_sender_address; struct transport_instance * tblock; } smtp_compare_t; @@ -1569,7 +1817,7 @@ sender_address, helo_data and tls_certificate if enabled. */ static uschar * -smtp_local_identity(uschar * sender, struct transport_instance * tblock) +smtp_local_identity(const uschar * sender, struct transport_instance * tblock) { address_item * addr1; uschar * if1 = US""; @@ -1577,7 +1825,7 @@ uschar * helo1 = US""; #ifndef DISABLE_TLS uschar * tlsc1 = US""; #endif -uschar * save_sender_address = sender_address; +const uschar * save_sender_address = sender_address; uschar * local_identity = NULL; smtp_transport_options_block * ob = SOB tblock->options_block; @@ -1623,8 +1871,9 @@ uschar * message_local_identity, current_local_identity = smtp_local_identity(s_compare->current_sender_address, s_compare->tblock); -if (!(new_sender_address = deliver_get_sender_address(message_id))) - return FALSE; +if (!(new_sender_address = spool_sender_from_msgid(message_id))) + return FALSE; + message_local_identity = smtp_local_identity(new_sender_address, s_compare->tblock); @@ -1637,57 +1886,65 @@ return Ustrcmp(current_local_identity, message_local_identity) == 0; static unsigned ehlo_response(uschar * buf, unsigned checks) { -size_t bsize = Ustrlen(buf); +PCRE2_SIZE bsize = Ustrlen(buf); +pcre2_match_data * md = pcre2_match_data_create(1, pcre_gen_ctx); /* debug_printf("%s: check for 0x%04x\n", __FUNCTION__, checks); */ #ifndef DISABLE_TLS if ( checks & OPTION_TLS - && pcre_exec(regex_STARTTLS, NULL, CS buf, bsize, 0, PCRE_EOPT, NULL, 0) < 0) + && pcre2_match(regex_STARTTLS, + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) #endif checks &= ~OPTION_TLS; if ( checks & OPTION_IGNQ - && pcre_exec(regex_IGNOREQUOTA, NULL, CS buf, bsize, 0, - PCRE_EOPT, NULL, 0) < 0) + && pcre2_match(regex_IGNOREQUOTA, + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) checks &= ~OPTION_IGNQ; if ( checks & OPTION_CHUNKING - && pcre_exec(regex_CHUNKING, NULL, CS buf, bsize, 0, PCRE_EOPT, NULL, 0) < 0) + && pcre2_match(regex_CHUNKING, + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) checks &= ~OPTION_CHUNKING; #ifndef DISABLE_PRDR if ( checks & OPTION_PRDR - && pcre_exec(regex_PRDR, NULL, CS buf, bsize, 0, PCRE_EOPT, NULL, 0) < 0) + && pcre2_match(regex_PRDR, + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) #endif checks &= ~OPTION_PRDR; #ifdef SUPPORT_I18N if ( checks & OPTION_UTF8 - && pcre_exec(regex_UTF8, NULL, CS buf, bsize, 0, PCRE_EOPT, NULL, 0) < 0) + && pcre2_match(regex_UTF8, + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) #endif checks &= ~OPTION_UTF8; if ( checks & OPTION_DSN - && pcre_exec(regex_DSN, NULL, CS buf, bsize, 0, PCRE_EOPT, NULL, 0) < 0) + && pcre2_match(regex_DSN, + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) checks &= ~OPTION_DSN; if ( checks & OPTION_PIPE - && pcre_exec(regex_PIPELINING, NULL, CS buf, bsize, 0, - PCRE_EOPT, NULL, 0) < 0) + && pcre2_match(regex_PIPELINING, + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) checks &= ~OPTION_PIPE; if ( checks & OPTION_SIZE - && pcre_exec(regex_SIZE, NULL, CS buf, bsize, 0, PCRE_EOPT, NULL, 0) < 0) + && pcre2_match(regex_SIZE, + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) checks &= ~OPTION_SIZE; #ifndef DISABLE_PIPE_CONNECT if ( checks & OPTION_EARLY_PIPE - && pcre_exec(regex_EARLY_PIPE, NULL, CS buf, bsize, 0, - PCRE_EOPT, NULL, 0) < 0) + && pcre2_match(regex_EARLY_PIPE, + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) #endif checks &= ~OPTION_EARLY_PIPE; +/* pcre2_match_data_free(md); gen ctx needs no free */ /* debug_printf("%s: found 0x%04x\n", __FUNCTION__, checks); */ return checks; } @@ -1763,18 +2020,18 @@ if (flags & tc_reap_prev && prev_cmd_count > 0) switch(sync_responses(sx, prev_cmd_count, 0)) { - case 1: /* 2xx (only) => OK */ - case 3: sx->good_RCPT = TRUE; /* 2xx & 5xx => OK & progress made */ - case 2: sx->completed_addr = TRUE; /* 5xx (only) => progress made */ - case 0: break; /* No 2xx or 5xx, but no probs */ + case RESP_BIT_HAD_2XX: /* OK */ + case RESP_HAD_2_AND_5: sx->good_RCPT = TRUE; /* OK & progress made */ + case RESP_BIT_HAD_5XX: sx->completed_addr = TRUE; /* progress made */ + case RESP_NOERROR: break; /* No 2xx or 5xx, but no probs */ - case -5: errno = ERRNO_TLSFAILURE; - return DEFER; + case RESP_EHLO_ERR_TLS:errno = ERRNO_TLSFAILURE; + return DEFER; #ifndef DISABLE_PIPE_CONNECT - case -4: /* non-2xx for pipelined banner or EHLO */ + case RESP_EPIPE_EHLO_ERR: /* non-2xx for pipelined banner or EHLO */ #endif - case -1: /* Timeout on RCPT */ - default: return ERROR; /* I/O error, or any MAIL/DATA error */ + case RESP_RCPT_TIMEO: /* Timeout on RCPT */ + default: return ERROR;/* I/O error, or any MAIL/DATA error */ } cmd_count = 1; if (!sx->pending_BDAT) @@ -1812,6 +2069,36 @@ return OK; +#ifdef SUPPORT_DANE +static int +check_force_dane_conn(smtp_context * sx, smtp_transport_options_block * ob) +{ +int rc; +if( sx->dane_required + || verify_check_given_host(CUSS &ob->hosts_try_dane, sx->conn_args.host) == OK + ) + switch (rc = tlsa_lookup(sx->conn_args.host, &sx->conn_args.tlsa_dnsa, sx->dane_required)) + { + case OK: sx->conn_args.dane = TRUE; + ob->tls_tempfail_tryclear = FALSE; /* force TLS */ + ob->tls_sni = sx->conn_args.host->name; /* force SNI */ + break; + case FAIL_FORCED: break; + default: set_errno_nohost(sx->addrlist, ERRNO_DNSDEFER, + string_sprintf("DANE error: tlsa lookup %s", + rc_to_string(rc)), + rc, FALSE, &sx->delivery_start); +# ifndef DISABLE_EVENT + (void) event_raise(sx->conn_args.tblock->event_action, + US"dane:fail", sx->dane_required + ? US"dane-required" : US"dnssec-invalid", + NULL); +# endif + return rc; + } +return OK; +} +#endif /************************************************* @@ -1820,7 +2107,7 @@ return OK; /* Arguments: - ctx connection context + sx connection context suppress_tls if TRUE, don't attempt a TLS connection - this is set for a second attempt after TLS initialization fails @@ -1844,43 +2131,32 @@ int yield = OK; uschar * tls_errstr; #endif +/* Many lines of clearing individual elements of *sx that used to +be here have been replaced by a full memset to zero (de41aff051). +There are two callers, this file and verify.c . Now we only set +up nonzero elements. */ + sx->conn_args.ob = ob; sx->lmtp = strcmpic(ob->protocol, US"lmtp") == 0; sx->smtps = strcmpic(ob->protocol, US"smtps") == 0; -/* sx->ok = FALSE; */ sx->send_rset = TRUE; sx->send_quit = TRUE; sx->setting_up = TRUE; sx->esmtp = TRUE; -/* sx->esmtp_sent = FALSE; */ -#ifdef SUPPORT_I18N -/* sx->utf8_needed = FALSE; */ -#endif sx->dsn_all_lasthop = TRUE; #ifdef SUPPORT_DANE -/* sx->conn_args.dane = FALSE; */ sx->dane_required = verify_check_given_host(CUSS &ob->hosts_require_dane, sx->conn_args.host) == OK; #endif -#ifndef DISABLE_PIPE_CONNECT -/* sx->early_pipe_active = sx->early_pipe_ok = FALSE; */ -/* sx->ehlo_resp.cleartext_features = sx->ehlo_resp.crypted_features = 0; */ -/* sx->pending_BANNER = sx->pending_EHLO = sx->pending_MAIL = FALSE; */ -#endif -if ((sx->max_rcpt = sx->conn_args.tblock->max_addresses) == 0) sx->max_rcpt = 999999; -/* sx->peer_offered = 0; */ -/* sx->avoid_option = 0; */ +if ((sx->max_mail = sx->conn_args.tblock->connection_max_messages) == 0) + sx->max_mail = UNLIMITED_ADDRS; +sx->max_rcpt = expand_max_rcpt(sx->conn_args.tblock->max_addresses); sx->igquotstr = US""; if (!sx->helo_data) sx->helo_data = ob->helo_data; -#ifdef EXPERIMENTAL_DSN_INFO -/* sx->smtp_greeting = NULL; */ -/* sx->helo_response = NULL; */ -#endif smtp_command = US"initial connection"; -/* sx->buffer[0] = '\0'; */ /* Set up the buffer for reading SMTP response packets. */ @@ -1894,9 +2170,6 @@ sx->inblock.ptrend = sx->inbuffer; sx->outblock.buffer = sx->outbuffer; sx->outblock.buffersize = sizeof(sx->outbuffer); sx->outblock.ptr = sx->outbuffer; -/* sx->outblock.cmd_count = 0; */ -/* sx->outblock.authenticating = FALSE; */ -/* sx->outblock.conn_args = NULL; */ /* Reset the parameters of a TLS session. */ @@ -1938,32 +2211,14 @@ if (continue_hostname && continue_proxy_cipher) const uschar * sni = US""; # ifdef SUPPORT_DANE - /* Check if the message will be DANE-verified; if so force its SNI */ + /* Check if the message will be DANE-verified; if so force TLS and its SNI */ tls_out.dane_verified = FALSE; smtp_port_for_connect(sx->conn_args.host, sx->port); if ( sx->conn_args.host->dnssec == DS_YES - && ( sx->dane_required - || verify_check_given_host(CUSS &ob->hosts_try_dane, sx->conn_args.host) == OK - ) ) - switch (rc = tlsa_lookup(sx->conn_args.host, &sx->conn_args.tlsa_dnsa, sx->dane_required)) - { - case OK: sx->conn_args.dane = TRUE; - ob->tls_tempfail_tryclear = FALSE; /* force TLS */ - ob->tls_sni = sx->first_addr->domain; /* force SNI */ - break; - case FAIL_FORCED: break; - default: set_errno_nohost(sx->addrlist, ERRNO_DNSDEFER, - string_sprintf("DANE error: tlsa lookup %s", - rc_to_string(rc)), - rc, FALSE, &sx->delivery_start); -# ifndef DISABLE_EVENT - (void) event_raise(sx->conn_args.tblock->event_action, - US"dane:fail", sx->dane_required - ? US"dane-required" : US"dnssec-invalid"); -# endif - return rc; - } + && (rc = check_force_dane_conn(sx, ob)) != OK + ) + return rc; # endif /* If the SNI or the DANE status required for the new message differs from the @@ -1991,7 +2246,7 @@ if (continue_hostname && continue_proxy_cipher) DEBUG(D_transport) debug_printf("Closing proxied-TLS connection due to SNI mismatch\n"); - HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP>> QUIT\n"); + smtp_debug_cmd(US"QUIT", 0); write(0, "QUIT\r\n", 6); close(0); continue_hostname = continue_proxy_cipher = NULL; @@ -2012,6 +2267,11 @@ if (!continue_hostname) if (sx->verify) HDEBUG(D_verify) debug_printf("interface=%s port=%d\n", sx->conn_args.interface, sx->port); + /* Arrange to report to calling process this is a new connection */ + + clearflag(sx->first_addr, af_cont_conn); + setflag(sx->first_addr, af_new_conn); + /* Get the actual port the connection will use, into sx->conn_args.host */ smtp_port_for_connect(sx->conn_args.host, sx->port); @@ -2025,27 +2285,8 @@ if (!continue_hostname) if (sx->conn_args.host->dnssec == DS_YES) { int rc; - if( sx->dane_required - || verify_check_given_host(CUSS &ob->hosts_try_dane, sx->conn_args.host) == OK - ) - switch (rc = tlsa_lookup(sx->conn_args.host, &sx->conn_args.tlsa_dnsa, sx->dane_required)) - { - case OK: sx->conn_args.dane = TRUE; - ob->tls_tempfail_tryclear = FALSE; /* force TLS */ - ob->tls_sni = sx->first_addr->domain; /* force SNI */ - break; - case FAIL_FORCED: break; - default: set_errno_nohost(sx->addrlist, ERRNO_DNSDEFER, - string_sprintf("DANE error: tlsa lookup %s", - rc_to_string(rc)), - rc, FALSE, &sx->delivery_start); -# ifndef DISABLE_EVENT - (void) event_raise(sx->conn_args.tblock->event_action, - US"dane:fail", sx->dane_required - ? US"dane-required" : US"dnssec-invalid"); -# endif - return rc; - } + if ((rc = check_force_dane_conn(sx, ob)) != OK) + return rc; } else if (sx->dane_required) { @@ -2054,7 +2295,7 @@ if (!continue_hostname) FAIL, FALSE, &sx->delivery_start); # ifndef DISABLE_EVENT (void) event_raise(sx->conn_args.tblock->event_action, - US"dane:fail", US"dane-required"); + US"dane:fail", US"dane-required", NULL); # endif return FAIL; } @@ -2065,7 +2306,13 @@ if (!continue_hostname) sx->cctx.tls_ctx = NULL; sx->inblock.cctx = sx->outblock.cctx = &sx->cctx; +#ifndef DISABLE_ESMTP_LIMITS + sx->peer_limit_mail = sx->peer_limit_rcpt = sx->peer_limit_rcptdom = +#endif sx->avoid_option = sx->peer_offered = smtp_peer_options = 0; +#ifndef DISABLE_CLIENT_CMD_LOG + client_cmd_log = NULL; +#endif #ifndef DISABLE_PIPE_CONNECT if ( verify_check_given_host(CUSS &ob->hosts_pipe_connect, @@ -2075,6 +2322,7 @@ if (!continue_hostname) the helo string might use it avoid doing early-pipelining. */ if ( !sx->helo_data + || sx->conn_args.interface || !Ustrstr(sx->helo_data, "$sending_ip_address") || Ustrstr(sx->helo_data, "def:sending_ip_address") ) @@ -2084,21 +2332,29 @@ if (!continue_hostname) && sx->ehlo_resp.cleartext_features & OPTION_EARLY_PIPE) { DEBUG(D_transport) - debug_printf("Using cached cleartext PIPE_CONNECT\n"); + debug_printf("Using cached cleartext PIPECONNECT\n"); sx->early_pipe_active = TRUE; sx->peer_offered = sx->ehlo_resp.cleartext_features; } } else DEBUG(D_transport) - debug_printf("helo needs $sending_ip_address\n"); + debug_printf("helo needs $sending_ip_address; avoid early-pipelining\n"); PIPE_CONNECT_RETRY: if (sx->early_pipe_active) + { sx->outblock.conn_args = &sx->conn_args; + (void) smtp_boundsock(&sx->conn_args); + } else #endif { - if ((sx->cctx.sock = smtp_connect(&sx->conn_args, NULL)) < 0) + blob lazy_conn = {.data = NULL}; + /* For TLS-connect, a TFO lazy-connect is useful since the Client Hello + can go on the TCP SYN. */ + + if ((sx->cctx.sock = smtp_connect(&sx->conn_args, + sx->smtps ? &lazy_conn : NULL)) < 0) { set_errno_nohost(sx->addrlist, errno == ETIMEDOUT ? ERRNO_CONNECTTIMEOUT : errno, @@ -2107,13 +2363,19 @@ PIPE_CONNECT_RETRY: sx->send_quit = FALSE; return DEFER; } +#ifdef TCP_QUICKACK + (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, TCP_QUICKACK, US &off, + sizeof(off)); +#endif } /* Expand the greeting message while waiting for the initial response. (Makes sense if helo_data contains ${lookup dnsdb ...} stuff). The expansion is - delayed till here so that $sending_interface and $sending_port are set. */ -/*XXX early-pipe: they still will not be. Is there any way to find out what they -will be? Somehow I doubt it. */ + delayed till here so that $sending_ip_address and $sending_port are set. + Those will be known even for a TFO lazy-connect, having been set by the bind(). + For early-pipe, we are ok if binding to a local interface; otherwise (if + $sending_ip_address is seen in helo_data) we disabled early-pipe above. */ + GET_OPTION("helo_data"); if (sx->helo_data) if (!(sx->helo_data = expand_string(sx->helo_data))) if (sx->verify) @@ -2152,10 +2414,6 @@ will be? Somehow I doubt it. */ else #endif { -#ifdef TCP_QUICKACK - (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, TCP_QUICKACK, US &off, - sizeof(off)); -#endif if (!smtp_reap_banner(sx)) goto RESPONSE_FAILED; } @@ -2165,7 +2423,7 @@ will be? Somehow I doubt it. */ uschar * s; lookup_dnssec_authenticated = sx->conn_args.host->dnssec==DS_YES ? US"yes" : sx->conn_args.host->dnssec==DS_NO ? US"no" : NULL; - s = event_raise(sx->conn_args.tblock->event_action, US"smtp:connect", sx->buffer); + s = event_raise(sx->conn_args.tblock->event_action, US"smtp:connect", sx->buffer, NULL); if (s) { set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, @@ -2233,10 +2491,22 @@ goto SEND_QUIT; #ifndef DISABLE_TLS if (sx->smtps) { + const uschar * s; + smtp_peer_options |= OPTION_TLS; suppress_tls = FALSE; ob->tls_tempfail_tryclear = FALSE; smtp_command = US"SSL-on-connect"; + +# ifndef DISABLE_TLS_RESUME + /* Having no EHLO response yet, cannot peek there for a servername to detect + an LB. Call this anyway, so that a dummy host_name_extract option value can + force resumption attempts. */ + + GET_OPTION("host_name_extract"); + if (!(s = ob->host_name_extract)) s = US"never-LB"; + ehlo_response_lbserver(sx, s); +# endif goto TLS_NEGOTIATE; } #endif @@ -2267,7 +2537,7 @@ goto SEND_QUIT; DEBUG(D_transport) debug_printf("may need to auth, so pipeline no further\n"); if (smtp_write_command(sx, SCMD_FLUSH, NULL) < 0) goto SEND_FAILED; - if (sync_responses(sx, 2, 0) != 0) + if (sync_responses(sx, 2, 0) != RESP_NOERROR) { HDEBUG(D_transport) debug_printf("failed reaping pipelined cmd responses\n"); @@ -2294,8 +2564,8 @@ goto SEND_QUIT; int n = sizeof(sx->buffer); uschar * rsp = sx->buffer; - if (sx->esmtp_sent && (n = Ustrlen(sx->buffer)) < sizeof(sx->buffer)/2) - { rsp = sx->buffer + n + 1; n = sizeof(sx->buffer) - n; } + if (sx->esmtp_sent && (n = Ustrlen(sx->buffer) + 1) < sizeof(sx->buffer)/2) + { rsp = sx->buffer + n; n = sizeof(sx->buffer) - n; } if (smtp_write_command(sx, SCMD_FLUSH, "HELO %s\r\n", sx->helo_data) < 0) goto SEND_FAILED; @@ -2324,6 +2594,8 @@ goto SEND_QUIT; if (!sx->early_pipe_active) #endif { + const uschar * s; + sx->peer_offered = ehlo_response(sx->buffer, OPTION_TLS /* others checked later */ #ifndef DISABLE_PIPE_CONNECT @@ -2338,6 +2610,13 @@ goto SEND_QUIT; ) #endif ); +#ifndef DISABLE_ESMTP_LIMITS + if (tls_out.active.sock >= 0 || !(sx->peer_offered & OPTION_TLS)) + { + ehlo_response_limits_read(sx); + ehlo_response_limits_apply(sx); + } +#endif #ifndef DISABLE_PIPE_CONNECT if (sx->early_pipe_ok) { @@ -2346,11 +2625,16 @@ goto SEND_QUIT; if ( (sx->peer_offered & (OPTION_PIPE | OPTION_EARLY_PIPE)) == (OPTION_PIPE | OPTION_EARLY_PIPE)) { - DEBUG(D_transport) debug_printf("PIPE_CONNECT usable in future for this IP\n"); + DEBUG(D_transport) debug_printf("PIPECONNECT usable in future for this IP\n"); sx->ehlo_resp.cleartext_auths = study_ehlo_auths(sx); write_ehlo_cache_entry(sx); } } +#endif +#ifndef DISABLE_TLS_RESUME + GET_OPTION("host_name_extract"); + if (!(s = ob->host_name_extract)) s = HNE_DEFAULT; + ehlo_response_lbserver(sx, s); #endif } @@ -2392,6 +2676,13 @@ else sx->inblock.cctx = sx->outblock.cctx = &sx->cctx; smtp_command = big_buffer; sx->peer_offered = smtp_peer_options; +#ifndef DISABLE_ESMTP_LIMITS + /* Limits passed by cmdline over exec. */ + ehlo_limits_apply(sx, + sx->peer_limit_mail = continue_limit_mail, + sx->peer_limit_rcpt = continue_limit_rcpt, + sx->peer_limit_rcptdom = continue_limit_rcptdom); +#endif sx->helo_data = NULL; /* ensure we re-expand ob->helo_data */ /* For a continued connection with TLS being proxied for us, or a @@ -2433,16 +2724,20 @@ if ( smtp_peer_options & OPTION_TLS #ifndef DISABLE_PIPE_CONNECT /* If doing early-pipelining reap the banner and EHLO-response but leave - the response for the STARTTLS we just sent alone. */ + the response for the STARTTLS we just sent alone. On fail, assume wrong + cached capability and retry with the pipelining disabled. */ - if (sx->early_pipe_active && sync_responses(sx, 2, 0) != 0) + if (sx->early_pipe_active) { - HDEBUG(D_transport) - debug_printf("failed reaping pipelined cmd responses\n"); - close(sx->cctx.sock); - sx->cctx.sock = -1; - sx->early_pipe_active = FALSE; - goto PIPE_CONNECT_RETRY; + if (sync_responses(sx, 2, 0) != RESP_NOERROR) + { + HDEBUG(D_transport) + debug_printf("failed reaping pipelined cmd responses\n"); + close(sx->cctx.sock); + sx->cctx.sock = -1; + sx->early_pipe_active = FALSE; + goto PIPE_CONNECT_RETRY; + } } #endif @@ -2471,6 +2766,7 @@ if ( smtp_peer_options & OPTION_TLS else TLS_NEGOTIATE: { + sx->conn_args.sending_ip_address = sending_ip_address; if (!tls_client_start(&sx->cctx, &sx->conn_args, sx->addrlist, &tls_out, &tls_errstr)) { /* TLS negotiation failed; give an error. From outside, this function may @@ -2487,7 +2783,7 @@ if ( smtp_peer_options & OPTION_TLS sx->conn_args.host->name, sx->conn_args.host->address, tls_errstr); # ifndef DISABLE_EVENT (void) event_raise(sx->conn_args.tblock->event_action, - US"dane:fail", US"validation-failure"); /* could do with better detail */ + US"dane:fail", US"validation-failure", NULL); /* could do with better detail */ # endif } # endif @@ -2497,6 +2793,7 @@ if ( smtp_peer_options & OPTION_TLS sx->send_quit = FALSE; goto TLS_FAILED; } + sx->send_tlsclose = TRUE; /* TLS session is set up. Check the inblock fill level. If there is content then as we have not yet done a tls read it must have arrived before @@ -2544,13 +2841,17 @@ if (tls_out.active.sock >= 0) { uschar * greeting_cmd; - if (!sx->helo_data && !(sx->helo_data = expand_string(ob->helo_data))) + if (!sx->helo_data) { - uschar *message = string_sprintf("failed to expand helo_data: %s", - expand_string_message); - set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE, &sx->delivery_start); - yield = DEFER; - goto SEND_QUIT; + GET_OPTION("helo_data"); + if (!(sx->helo_data = expand_string(ob->helo_data))) + { + uschar *message = string_sprintf("failed to expand helo_data: %s", + expand_string_message); + set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE, &sx->delivery_start); + yield = DEFER; + goto SEND_QUIT; + } } #ifndef DISABLE_PIPE_CONNECT @@ -2565,9 +2866,17 @@ if (tls_out.active.sock >= 0) sx->peer_offered = sx->ehlo_resp.crypted_features; if ((sx->early_pipe_active = !!(sx->ehlo_resp.crypted_features & OPTION_EARLY_PIPE))) - DEBUG(D_transport) debug_printf("Using cached crypted PIPE_CONNECT\n"); + DEBUG(D_transport) debug_printf("Using cached crypted PIPECONNECT\n"); } #endif +#ifdef EXPERIMMENTAL_ESMTP_LIMITS + /* As we are about to send another EHLO, forget any LIMITS received so far. */ + sx->peer_limit_mail = sx->peer_limit_rcpt = sx->peer_limit_rcptdom = 0; + if ((sx->max_mail = sx->conn_args.tblock->connection_max_message) == 0) + sx->max_mail = UNLIMITED_ADDRS; + sx->max_rcpt = expand_max_rcpt(sx->conn_args.tblock->max_addresses); + sx->single_rcpt_domain = FALSE; +#endif /* For SMTPS we need to wait for the initial OK response. */ if (sx->smtps) @@ -2647,7 +2956,8 @@ else if ( sx->smtps (void) event_raise(sx->conn_args.tblock->event_action, US"dane:fail", smtp_peer_options & OPTION_TLS ? US"validation-failure" /* could do with better detail */ - : US"starttls-not-supported"); + : US"starttls-not-supported", + NULL); # endif goto TLS_FAILED; } @@ -2658,7 +2968,7 @@ so its response needs to be analyzed. If TLS is not active and this is a continued session down a previously-used socket, we haven't just done EHLO, so we skip this. */ -if (continue_hostname == NULL +if ( !continue_hostname #ifndef DISABLE_TLS || tls_out.active.sock >= 0 #endif @@ -2697,6 +3007,14 @@ if (continue_hostname == NULL if (tls_out.active.sock >= 0) sx->ehlo_resp.crypted_features = sx->peer_offered; #endif + +#ifndef DISABLE_ESMTP_LIMITS + if (tls_out.active.sock >= 0 || !(sx->peer_offered & OPTION_TLS)) + { + ehlo_response_limits_read(sx); + ehlo_response_limits_apply(sx); + } +#endif } /* Set for IGNOREQUOTA if the response to LHLO specifies support and the @@ -2721,18 +3039,18 @@ if (continue_hostname == NULL smtp_peer_options & OPTION_PIPE ? "" : "not "); if ( sx->peer_offered & OPTION_CHUNKING - && verify_check_given_host(CUSS &ob->hosts_try_chunking, sx->conn_args.host) != OK) - sx->peer_offered &= ~OPTION_CHUNKING; + && verify_check_given_host(CUSS &ob->hosts_try_chunking, sx->conn_args.host) == OK) + smtp_peer_options |= OPTION_CHUNKING; - if (sx->peer_offered & OPTION_CHUNKING) + if (smtp_peer_options & OPTION_CHUNKING) DEBUG(D_transport) debug_printf("CHUNKING usable\n"); #ifndef DISABLE_PRDR if ( sx->peer_offered & OPTION_PRDR - && verify_check_given_host(CUSS &ob->hosts_try_prdr, sx->conn_args.host) != OK) - sx->peer_offered &= ~OPTION_PRDR; + && verify_check_given_host(CUSS &ob->hosts_try_prdr, sx->conn_args.host) == OK) + smtp_peer_options |= OPTION_PRDR; - if (sx->peer_offered & OPTION_PRDR) + if (smtp_peer_options & OPTION_PRDR) DEBUG(D_transport) debug_printf("PRDR usable\n"); #endif @@ -2749,7 +3067,7 @@ if (continue_hostname == NULL && ( sx->ehlo_resp.cleartext_features | sx->ehlo_resp.crypted_features) & OPTION_EARLY_PIPE) { - DEBUG(D_transport) debug_printf("PIPE_CONNECT usable in future for this IP\n"); + DEBUG(D_transport) debug_printf("PIPECONNECT usable in future for this IP\n"); sx->ehlo_resp.crypted_auths = study_ehlo_auths(sx); write_ehlo_cache_entry(sx); } @@ -2784,6 +3102,7 @@ if (sx->addrlist->prop.utf8_msg) /* If the transport sets a downconversion mode it overrides any set by ACL for the message. */ + GET_OPTION("utf8_downconvert"); if ((s = ob->utf8_downconvert)) { if (!(s = expand_string(s))) @@ -2864,7 +3183,7 @@ return OK; SEND_FAILED: code = '4'; - message = US string_sprintf("send() to %s [%s] failed: %s", + message = US string_sprintf("smtp send to %s [%s] failed: %s", sx->conn_args.host->name, sx->conn_args.host->address, strerror(errno)); sx->send_quit = FALSE; yield = DEFER; @@ -2930,7 +3249,11 @@ if (sx->send_quit) #ifndef DISABLE_TLS if (sx->cctx.tls_ctx) { - tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_NOWAIT); + if (sx->send_tlsclose) + { + tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_NOWAIT); + sx->send_tlsclose = FALSE; + } sx->cctx.tls_ctx = NULL; } #endif @@ -2953,9 +3276,10 @@ if (sx->send_quit) sx->cctx.sock = -1; #ifndef DISABLE_EVENT -(void) event_raise(sx->conn_args.tblock->event_action, US"tcp:close", NULL); +(void) event_raise(sx->conn_args.tblock->event_action, US"tcp:close", NULL, NULL); #endif +smtp_debug_cmd_report(); continue_transport = NULL; continue_hostname = NULL; return yield; @@ -2998,7 +3322,7 @@ Or just forget about lines? Or inflate by a fixed proportion? */ request that */ sx->prdr_active = FALSE; -if (sx->peer_offered & OPTION_PRDR) +if (smtp_peer_options & OPTION_PRDR) for (address_item * addr = addrlist; addr; addr = addr->next) if (addr->transport_return == PENDING_DEFER) { @@ -3026,7 +3350,7 @@ if ( sx->peer_offered & OPTION_UTF8 /* check if all addresses have DSN-lasthop flag; do not send RET and ENVID if so */ for (sx->dsn_all_lasthop = TRUE, addr = addrlist, address_count = 0; - addr && address_count < sx->max_rcpt; + addr && address_count < sx->max_rcpt; /*XXX maybe also || sx->single_rcpt_domain ? */ addr = addr->next) if (addr->transport_return == PENDING_DEFER) { address_count++; @@ -3100,27 +3424,24 @@ if (sx->peer_offered & OPTION_DSN && !(addr->dsn_flags & rf_dsnlasthop)) -/* -Return: - 0 good, rcpt results in addr->transport_return (PENDING_OK, DEFER, FAIL) - -1 MAIL response error - -2 any non-MAIL read i/o error - -3 non-MAIL response timeout - -4 internal error; channel still usable - -5 transmit failed +/* Send MAIL FROM and RCPT TO commands. +See sw_mrc_t definition for return codes. */ -int +sw_mrc_t smtp_write_mail_and_rcpt_cmds(smtp_context * sx, int * yield) { address_item * addr; +#ifndef DISABLE_ESMTP_LIMITS +address_item * restart_addr = NULL; +#endif int address_count, pipe_limit; int rc; if (build_mailcmd_options(sx, sx->first_addr) != OK) { *yield = ERROR; - return -4; + return sw_mrc_bad_internal; } /* From here until we send the DATA command, we can make use of PIPELINING @@ -3132,7 +3453,7 @@ buffer. */ sx->pending_MAIL = TRUE; /* The block starts with MAIL */ { - uschar * s = sx->from_addr; + const uschar * s = sx->from_addr; #ifdef SUPPORT_I18N uschar * errstr = NULL; @@ -3148,7 +3469,7 @@ sx->pending_MAIL = TRUE; /* The block starts with MAIL */ { set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, errstr, DEFER, FALSE, &sx->delivery_start); *yield = ERROR; - return -4; + return sw_mrc_bad_internal; } setflag(sx->addrlist, af_utf8_downcvt); } @@ -3163,7 +3484,7 @@ mail_command = string_copy(big_buffer); /* Save for later error message */ switch(rc) { case -1: /* Transmission error */ - return -5; + return sw_mrc_bad_mail; case +1: /* Cmd was sent */ if (!smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2', @@ -3174,7 +3495,7 @@ switch(rc) errno = ERRNO_MAIL4XX; sx->addrlist->more_errno |= ((sx->buffer[1] - '0')*10 + sx->buffer[2] - '0') << 8; } - return -1; + return sw_mrc_bad_mail; } sx->pending_MAIL = FALSE; break; @@ -3198,12 +3519,30 @@ that max_rcpt will be large, so all addresses will be done at once. For verify we flush the pipeline after any (the only) rcpt address. */ for (addr = sx->first_addr, address_count = 0, pipe_limit = 100; - addr && address_count < sx->max_rcpt; + addr && address_count < sx->max_rcpt; addr = addr->next) if (addr->transport_return == PENDING_DEFER) { int cmds_sent; BOOL no_flush; - uschar * rcpt_addr; + const uschar * rcpt_addr; + +#ifndef DISABLE_ESMTP_LIMITS + if ( sx->single_rcpt_domain /* restriction on domains */ + && address_count > 0 /* not first being sent */ + && Ustrcmp(addr->domain, sx->first_addr->domain) != 0 /* dom diff from first */ + ) + { + DEBUG(D_transport) debug_printf("skipping different domain %s\n", addr->domain); + + /* Ensure the smtp-response reaper does not think the address had a RCPT + command sent for it. Reset to PENDING_DEFER in smtp_deliver(), where we + goto SEND_MESSAGE. */ + + addr->transport_return = SKIP; + if (!restart_addr) restart_addr = addr; /* note restart point */ + continue; /* skip this one */ + } +#endif addr->dsn_aware = sx->peer_offered & OPTION_DSN ? dsn_support_yes : dsn_support_no; @@ -3231,26 +3570,26 @@ for (addr = sx->first_addr, address_count = 0, pipe_limit = 100; { /*XXX could we use a per-address errstr here? Not fail the whole send? */ errno = ERRNO_EXPANDFAIL; - return -5; /*XXX too harsh? */ + return sw_mrc_tx_fail; /*XXX too harsh? */ } #endif cmds_sent = smtp_write_command(sx, no_flush ? SCMD_BUFFER : SCMD_FLUSH, "RCPT TO:<%s>%s%s\r\n", rcpt_addr, sx->igquotstr, sx->buffer); - if (cmds_sent < 0) return -5; + if (cmds_sent < 0) return sw_mrc_tx_fail; if (cmds_sent > 0) { switch(sync_responses(sx, cmds_sent, 0)) { - case 3: sx->ok = TRUE; /* 2xx & 5xx => OK & progress made */ - case 2: sx->completed_addr = TRUE; /* 5xx (only) => progress made */ + case RESP_HAD_2_AND_5: sx->ok = TRUE; /* OK & progress made */ + case RESP_BIT_HAD_5XX: sx->completed_addr = TRUE; /* progress made */ break; - case 1: sx->ok = TRUE; /* 2xx (only) => OK, but if LMTP, */ + case RESP_BIT_HAD_2XX: sx->ok = TRUE; /* OK, but if LMTP, */ if (!sx->lmtp) /* can't tell about progress yet */ sx->completed_addr = TRUE; - case 0: /* No 2xx or 5xx, but no probs */ + case RESP_NOERROR: /* No 2xx or 5xx, but no probs */ /* If any RCPT got a 452 response then next_addr has been updated for restarting with a new MAIL on the same connection. Send no more RCPTs for this MAIL. */ @@ -3260,24 +3599,28 @@ for (addr = sx->first_addr, address_count = 0, pipe_limit = 100; DEBUG(D_transport) debug_printf("seen 452 too-many-rcpts\n"); sx->RCPT_452 = FALSE; /* sx->next_addr has been reset for fast_retry */ - return 0; + return sw_mrc_ok; } break; - case -1: return -3; /* Timeout on RCPT */ - case -2: return -2; /* non-MAIL read i/o error */ - default: return -1; /* any MAIL error */ + case RESP_RCPT_TIMEO: return sw_mrc_nonmail_read_timeo; + case RESP_RCPT_ERROR: return sw_mrc_bad_read; + default: return sw_mrc_bad_mail; /* any MAIL error */ #ifndef DISABLE_PIPE_CONNECT - case -4: return -1; /* non-2xx for pipelined banner or EHLO */ - case -5: return -1; /* TLS first-read error */ + case RESP_EPIPE_EHLO_ERR: return sw_mrc_bad_mail; /* non-2xx for pipelined banner or EHLO */ + case RESP_EHLO_ERR_TLS: return sw_mrc_bad_mail; /* TLS first-read error */ #endif } } } /* Loop for next address */ +#ifndef DISABLE_ESMTP_LIMITS +sx->next_addr = restart_addr ? restart_addr : addr; +#else sx->next_addr = addr; -return 0; +#endif +return sw_mrc_ok; } @@ -3298,40 +3641,35 @@ Arguments: bufsiz size of buffer pfd pipe filedescriptor array; [0] is comms to proxied process timeout per-read timeout, seconds + host hostname of remote Does not return. */ void smtp_proxy_tls(void * ct_ctx, uschar * buf, size_t bsize, int * pfd, - int timeout) + int timeout, const uschar * host) { -fd_set rfds, efds; -int max_fd = MAX(pfd[0], tls_out.active.sock) + 1; +struct pollfd p[2] = {{.fd = tls_out.active.sock, .events = POLLIN}, + {.fd = pfd[0], .events = POLLIN}}; int rc, i; +BOOL send_tls_shutdown = TRUE; close(pfd[1]); if ((rc = exim_fork(US"tls-proxy"))) _exit(rc < 0 ? EXIT_FAILURE : EXIT_SUCCESS); -set_process_info("proxying TLS connection for continued transport"); -FD_ZERO(&rfds); -FD_SET(tls_out.active.sock, &rfds); -FD_SET(pfd[0], &rfds); +set_process_info("proxying TLS connection for continued transport to %s\n", host); -for (int fd_bits = 3; fd_bits; ) +do { time_t time_left = timeout; time_t time_start = time(NULL); /* wait for data */ - efds = rfds; do { - struct timeval tv = { time_left, 0 }; - - rc = select(max_fd, - (SELECT_ARG2_TYPE *)&rfds, NULL, (SELECT_ARG2_TYPE *)&efds, &tv); + rc = poll(p, 2, time_left * 1000); if (rc < 0 && errno == EINTR) if ((time_left -= time(NULL) - time_start) > 0) continue; @@ -3342,51 +3680,56 @@ for (int fd_bits = 3; fd_bits; ) goto done; } - if (FD_ISSET(tls_out.active.sock, &efds) || FD_ISSET(pfd[0], &efds)) + /* For errors where not readable, bomb out */ + + if (p[0].revents & POLLERR || p[1].revents & POLLERR) { DEBUG(D_transport) debug_printf("select: exceptional cond on %s fd\n", - FD_ISSET(pfd[0], &efds) ? "proxy" : "tls"); - goto done; + p[0].revents & POLLERR ? "tls" : "proxy"); + if (!(p[0].revents & POLLIN || p[1].events & POLLIN)) + goto done; + DEBUG(D_transport) debug_printf("- but also readable; no exit yet\n"); } } - while (rc < 0 || !(FD_ISSET(tls_out.active.sock, &rfds) || FD_ISSET(pfd[0], &rfds))); + while (rc < 0 || !(p[0].revents & POLLIN || p[1].revents & POLLIN)); /* handle inbound data */ - if (FD_ISSET(tls_out.active.sock, &rfds)) - if ((rc = tls_read(ct_ctx, buf, bsize)) <= 0) - { - fd_bits &= ~1; - FD_CLR(tls_out.active.sock, &rfds); + if (p[0].revents & POLLIN) + if ((rc = tls_read(ct_ctx, buf, bsize)) <= 0) /* Expect -1 for EOF; */ + { /* that reaps the TLS Close Notify record */ + p[0].fd = -1; shutdown(pfd[0], SHUT_WR); timeout = 5; } else - { for (int nbytes = 0; rc - nbytes > 0; nbytes += i) if ((i = write(pfd[0], buf + nbytes, rc - nbytes)) < 0) goto done; - } - else if (fd_bits & 1) - FD_SET(tls_out.active.sock, &rfds); - /* handle outbound data */ - if (FD_ISSET(pfd[0], &rfds)) + /* Handle outbound data. We cannot combine payload and the TLS-close + due to the limitations of the (pipe) channel feeding us. Maybe use a unix-domain + socket? */ + if (p[1].revents & POLLIN) if ((rc = read(pfd[0], buf, bsize)) <= 0) { - fd_bits = 0; - tls_close(ct_ctx, TLS_SHUTDOWN_NOWAIT); - ct_ctx = NULL; + p[1].fd = -1; + +# ifdef EXIM_TCP_CORK /* Use _CORK to get TLS Close Notify in FIN segment */ + (void) setsockopt(tls_out.active.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on)); +# endif + tls_shutdown_wr(ct_ctx); + send_tls_shutdown = FALSE; + shutdown(tls_out.active.sock, SHUT_WR); } else - { for (int nbytes = 0; rc - nbytes > 0; nbytes += i) if ((i = tls_write(ct_ctx, buf + nbytes, rc - nbytes, FALSE)) < 0) goto done; - } - else if (fd_bits & 2) - FD_SET(pfd[0], &rfds); } +while (p[0].fd >= 0 || p[1].fd >= 0); done: + if (send_tls_shutdown) tls_close(ct_ctx, TLS_SHUTDOWN_NOWAIT); + ct_ctx = NULL; testharness_pause_ms(100); /* let logging complete */ exim_exit(EXIT_SUCCESS); } @@ -3452,30 +3795,34 @@ int yield = OK; int save_errno; int rc; -BOOL pass_message = FALSE; uschar *message = NULL; uschar new_message_id[MESSAGE_ID_LENGTH + 1]; -smtp_context * sx = store_get(sizeof(*sx), TRUE); /* tainted, for the data buffers */ +smtp_context * sx = store_get(sizeof(*sx), GET_TAINTED); /* tainted, for the data buffers */ +BOOL pass_message = FALSE; +#ifndef DISABLE_ESMTP_LIMITS +BOOL mail_limit = FALSE; +#endif #ifdef SUPPORT_DANE BOOL dane_held; #endif +BOOL tcw_done = FALSE, tcw = FALSE; *message_defer = FALSE; memset(sx, 0, sizeof(*sx)); sx->addrlist = addrlist; sx->conn_args.host = host; -sx->conn_args.host_af = host_af, +sx->conn_args.host_af = host_af; sx->port = defport; sx->conn_args.interface = interface; sx->helo_data = NULL; sx->conn_args.tblock = tblock; -/* sx->verify = FALSE; */ +sx->conn_args.sock = -1; gettimeofday(&sx->delivery_start, NULL); sx->sync_addr = sx->first_addr = addrlist; +REPEAT_CONN: #ifdef SUPPORT_DANE -DANE_DOMAINS: dane_held = FALSE; #endif @@ -3520,8 +3867,8 @@ if (tblock->filter_command) yield ERROR. */ if (!transport_set_up_command(&transport_filter_argv, - tblock->filter_command, TRUE, DEFER, addrlist, - string_sprintf("%.50s transport", tblock->name), NULL)) + tblock->filter_command, TSUC_EXPAND_ARGS, DEFER, addrlist, + string_sprintf("%.50s transport filter", tblock->name), NULL)) { set_errno_nohost(addrlist->next, addrlist->basic_errno, addrlist->message, DEFER, FALSE, &sx->delivery_start); @@ -3532,7 +3879,7 @@ if (tblock->filter_command) if ( transport_filter_argv && *transport_filter_argv && **transport_filter_argv - && sx->peer_offered & OPTION_CHUNKING + && smtp_peer_options & OPTION_CHUNKING #ifndef DISABLE_DKIM /* When dkim signing, chunking is handled even with a transport-filter */ && !(ob->dkim.dkim_private_key && ob->dkim.dkim_domain && ob->dkim.dkim_selector) @@ -3540,7 +3887,7 @@ if (tblock->filter_command) #endif ) { - sx->peer_offered &= ~OPTION_CHUNKING; + smtp_peer_options &= ~OPTION_CHUNKING; DEBUG(D_transport) debug_printf("CHUNKING not usable due to transport filter\n"); } } @@ -3581,11 +3928,12 @@ else switch(smtp_write_mail_and_rcpt_cmds(sx, &yield)) { - case 0: break; - case -1: case -2: goto RESPONSE_FAILED; - case -3: goto END_OFF; - case -4: goto SEND_QUIT; - default: goto SEND_FAILED; + case sw_mrc_ok: break; + case sw_mrc_bad_mail: goto RESPONSE_FAILED; + case sw_mrc_bad_read: goto RESPONSE_FAILED; + case sw_mrc_nonmail_read_timeo: goto END_OFF; + case sw_mrc_bad_internal: goto SEND_QUIT; + default: goto SEND_FAILED; } /* If we are an MUA wrapper, abort if any RCPTs were rejected, either @@ -3617,23 +3965,25 @@ are pipelining. The responses are all handled by sync_responses(). If using CHUNKING, do not send a BDAT until we know how big a chunk we want to send is. */ -if ( !(sx->peer_offered & OPTION_CHUNKING) +if ( !(smtp_peer_options & OPTION_CHUNKING) && (sx->ok || (pipelining_active && !mua_wrapper))) { int count = smtp_write_command(sx, SCMD_FLUSH, "DATA\r\n"); if (count < 0) goto SEND_FAILED; + switch(sync_responses(sx, count, sx->ok ? +1 : -1)) { - case 3: sx->ok = TRUE; /* 2xx & 5xx => OK & progress made */ - case 2: sx->completed_addr = TRUE; /* 5xx (only) => progress made */ - break; + case RESP_HAD_2_AND_5: sx->ok = TRUE; /* OK & progress made */ + case RESP_BIT_HAD_5XX: sx->completed_addr = TRUE; /* progress made */ + break; - case 1: sx->ok = TRUE; /* 2xx (only) => OK, but if LMTP, */ - if (!sx->lmtp) sx->completed_addr = TRUE; /* can't tell about progress yet */ - case 0: break; /* No 2xx or 5xx, but no probs */ + case RESP_BIT_HAD_2XX: sx->ok = TRUE; /* OK, but if LMTP, */ + if (!sx->lmtp) /* can't tell about progress yet */ + sx->completed_addr = TRUE; + case RESP_NOERROR: break; /* No 2xx or 5xx, but no probs */ - case -1: goto END_OFF; /* Timeout on RCPT */ + case RESP_RCPT_TIMEO: goto END_OFF; #ifndef DISABLE_PIPE_CONNECT case -5: /* TLS first-read error */ @@ -3654,7 +4004,7 @@ well as body. Set the appropriate timeout value to be used for each chunk. (Haven't been able to make it work using select() for writing yet.) */ if ( !sx->ok - && (!(sx->peer_offered & OPTION_CHUNKING) || !pipelining_active)) + && (!(smtp_peer_options & OPTION_CHUNKING) || !pipelining_active)) { /* Save the first address of the next batch. */ sx->first_addr = sx->next_addr; @@ -3683,7 +4033,7 @@ else of responses. The callback needs a whole bunch of state so set up a transport-context structure to be passed around. */ - if (sx->peer_offered & OPTION_CHUNKING) + if (smtp_peer_options & OPTION_CHUNKING) { tctx.check_string = tctx.escape_string = NULL; tctx.options |= topt_use_bdat; @@ -3708,10 +4058,10 @@ else transport_write_timeout = ob->data_timeout; smtp_command = US"sending data block"; /* For error messages */ DEBUG(D_transport|D_v) - if (sx->peer_offered & OPTION_CHUNKING) + if (smtp_peer_options & OPTION_CHUNKING) debug_printf(" will write message using CHUNKING\n"); else - debug_printf(" SMTP>> writing message and terminating \".\"\n"); + debug_printf(" SMTP>> (writing message)\n"); transport_count = 0; #ifndef DISABLE_DKIM @@ -3748,6 +4098,46 @@ else report_time_since(&t0, US"dkim_exim_sign_init (delta)"); # endif } +#endif + + /* See if we can pipeline QUIT. Reasons not to are + - pipelining not active + - not ok to send quit + - errors in amtp transation responses + - more addrs to send for this message or this host + - this message was being retried + - more messages for this host + If we can, we want the message-write to not flush (the tail end of) its data out. */ + + if ( sx->pipelining_used + && (sx->ok && sx->completed_addr || smtp_peer_options & OPTION_CHUNKING) + && sx->send_quit + && !(sx->first_addr || f.continue_more) + && f.deliver_firsttime + ) + { + smtp_compare_t t_compare = + {.tblock = tblock, .current_sender_address = sender_address}; + + tcw_done = TRUE; + tcw = +#ifndef DISABLE_TLS + ( tls_out.active.sock < 0 && !continue_proxy_cipher + || verify_check_given_host(CUSS &ob->hosts_nopass_tls, host) != OK + ) + && +#endif + transport_check_waiting(tblock->name, host->name, + tblock->connection_max_messages, new_message_id, + (oicf)smtp_are_same_identities, (void*)&t_compare); + if (!tcw) + { + HDEBUG(D_transport) debug_printf("will pipeline QUIT\n"); + tctx.options |= topt_no_flush; + } + } + +#ifndef DISABLE_DKIM sx->ok = dkim_transport_write_message(&tctx, &ob->dkim, CUSS &message); #else sx->ok = transport_write_message(&tctx, 0); @@ -3776,27 +4166,70 @@ else smtp_command = US"end of data"; - if (sx->peer_offered & OPTION_CHUNKING && sx->cmd_count > 1) + /* If we can pipeline a QUIT with the data them send it now. If a new message + for this host appeared in the queue while data was being sent, we will not see + it and it will have to wait for a queue run. If there was one but another + thread took it, we might attempt to send it - but locking of spoolfiles will + detect that. Use _MORE to get QUIT in FIN segment. */ + + if (tcw_done && !tcw) + { + /*XXX jgh 2021/03/10 google et. al screwup. G, at least, sends TCP FIN in response to TLS + close-notify. Under TLS 1.3, violating RFC. + However, TLS 1.2 does not have half-close semantics. */ + + if ( sx->cctx.tls_ctx +#if 0 && !defined(DISABLE_TLS) + && Ustrcmp(tls_out.ver, "TLS1.3") != 0 +#endif + || !f.deliver_firsttime + ) + { /* Send QUIT now and not later */ + (void)smtp_write_command(sx, SCMD_FLUSH, "QUIT\r\n"); + sx->send_quit = FALSE; + } + else + { /* add QUIT to the output buffer */ + (void)smtp_write_command(sx, SCMD_MORE, "QUIT\r\n"); + sx->send_quit = FALSE; /* avoid sending it later */ + +#ifndef DISABLE_TLS + if (sx->cctx.tls_ctx && sx->send_tlsclose) /* need to send TLS Close Notify */ + { +# ifdef EXIM_TCP_CORK /* Use _CORK to get Close Notify in FIN segment */ + (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on)); +# endif + tls_shutdown_wr(sx->cctx.tls_ctx); + sx->send_tlsclose = FALSE; /* avoid later repeat */ + } +#endif + HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(shutdown)>>\n"); + shutdown(sx->cctx.sock, SHUT_WR); /* flush output buffer, with TCP FIN */ + } + } + + if (smtp_peer_options & OPTION_CHUNKING && sx->cmd_count > 1) { /* Reap any outstanding MAIL & RCPT commands, but not a DATA-go-ahead */ switch(sync_responses(sx, sx->cmd_count-1, 0)) { - case 3: sx->ok = TRUE; /* 2xx & 5xx => OK & progress made */ - case 2: sx->completed_addr = TRUE; /* 5xx (only) => progress made */ - break; + case RESP_HAD_2_AND_5: sx->ok = TRUE; /* OK & progress made */ + case RESP_BIT_HAD_5XX: sx->completed_addr = TRUE; /* progress made */ + break; - case 1: sx->ok = TRUE; /* 2xx (only) => OK, but if LMTP, */ - if (!sx->lmtp) sx->completed_addr = TRUE; /* can't tell about progress yet */ - case 0: break; /* No 2xx or 5xx, but no probs */ + case RESP_BIT_HAD_2XX: sx->ok = TRUE; /* OK, but if LMTP, */ + if (!sx->lmtp) /* can't tell about progress yet */ + sx->completed_addr = TRUE; + case RESP_NOERROR: break; /* No 2xx or 5xx, but no probs */ - case -1: goto END_OFF; /* Timeout on RCPT */ + case RESP_RCPT_TIMEO: goto END_OFF; /* Timeout on RCPT */ #ifndef DISABLE_PIPE_CONNECT - case -5: /* TLS first-read error */ - case -4: HDEBUG(D_transport) + case RESP_EHLO_ERR_TLS: /* TLS first-read error */ + case RESP_EPIPE_EHLO_ERR: HDEBUG(D_transport) debug_printf("failed reaping pipelined cmd responses\n"); #endif - default: goto RESPONSE_FAILED; /* I/O error, or any MAIL/DATA error */ + default: goto RESPONSE_FAILED; /* I/O error, or any MAIL/DATA error */ } } @@ -3868,15 +4301,16 @@ else !sx->lmtp ) { - const uschar *s = string_printing(sx->buffer); + const uschar * s = string_printing(sx->buffer); /* deconst cast ok here as string_printing was checked to have alloc'n'copied */ - conf = (s == sx->buffer)? US string_copy(s) : US s; + conf = s == sx->buffer ? US string_copy(s) : US s; } /* Process all transported addresses - for LMTP or PRDR, read a status for - each one. */ + each one. We used to drop out at first_addr, until someone returned a 452 + followed by a 250... and we screwed up the accepted addresses. */ - for (address_item * addr = addrlist; addr != sx->first_addr; addr = addr->next) + for (address_item * addr = addrlist; addr; addr = addr->next) { if (addr->transport_return != PENDING_OK) continue; @@ -3948,7 +4382,7 @@ else #ifndef DISABLE_PRDR if (sx->prdr_active) setflag(addr, af_prdr_used); #endif - if (sx->peer_offered & OPTION_CHUNKING) setflag(addr, af_chunking_used); + if (smtp_peer_options & OPTION_CHUNKING) setflag(addr, af_chunking_used); flag = '-'; #ifndef DISABLE_PRDR @@ -3965,7 +4399,7 @@ else else sprintf(CS sx->buffer, "%.500s\n", addr->unique); - DEBUG(D_deliver) debug_printf("S:journalling %s\n", sx->buffer); + DEBUG(D_deliver) debug_printf("S:journalling %s", sx->buffer); len = Ustrlen(CS sx->buffer); if (write(journal_fd, sx->buffer, len) != len) log_write(0, LOG_MAIN|LOG_PANIC, "failed to write journal for " @@ -4019,6 +4453,7 @@ else "%s: %s", sx->buffer, strerror(errno)); } else if (addr->transport_return == DEFER) + /*XXX magic value -2 ? maybe host+message ? */ retry_add_item(addr, addr->address_retry_key, -2); } #endif @@ -4049,7 +4484,8 @@ if (!sx->ok) { save_errno = errno; message = NULL; - sx->send_quit = check_response(host, &save_errno, addrlist->more_errno, + /* Clear send_quit flag if needed. Do not set. */ + sx->send_quit &= check_response(host, &save_errno, addrlist->more_errno, sx->buffer, &code, &message, &pass_message); goto FAILED; } @@ -4058,7 +4494,7 @@ if (!sx->ok) { save_errno = errno; code = '4'; - message = string_sprintf("send() to %s [%s] failed: %s", + message = string_sprintf("smtp send to %s [%s] failed: %s", host->name, host->address, message ? message : US strerror(save_errno)); sx->send_quit = FALSE; goto FAILED; @@ -4091,15 +4527,51 @@ if (!sx->ok) break; case ERRNO_SMTPCLOSED: - message_error = Ustrncmp(smtp_command,"end ",4) == 0; + /* If the peer closed the TCP connection after end-of-data, but before + we could send QUIT, do TLS close, etc - call it a message error. + Otherwise, if all the recipients have been dealt with, call a close no + error at all; each address_item should have a suitable result already + (2xx: PENDING_OK, 4xx: DEFER, 5xx: FAIL) */ + + if (!(message_error = Ustrncmp(smtp_command,"end ",4) == 0)) + { + address_item * addr; + for (addr = sx->addrlist; addr; addr = addr->next) + if (addr->transport_return == PENDING_DEFER) + break; + if (!addr) /* all rcpts fates determined */ + { + log_write(0, LOG_MAIN, "peer close after all rcpt responses;" + " converting i/o-error to no-error"); + sx->ok = TRUE; + goto happy; + } + } break; +#ifndef DISABLE_DKIM + case EACCES: + /* DKIM signing failure: avoid thinking we pipelined quit, + just abandon the message and close the socket. */ + + message_error = FALSE; +# ifndef DISABLE_TLS + if (sx->cctx.tls_ctx) + { + tls_close(sx->cctx.tls_ctx, + sx->send_tlsclose ? TLS_SHUTDOWN_WAIT : TLS_SHUTDOWN_WONLY); + sx->cctx.tls_ctx = NULL; + } +# endif + break; +#endif default: message_error = FALSE; break; } - /* Handle the cases that are treated as message errors. These are: + /* Handle the cases that are treated as message errors (as opposed to + host-errors). These are: (a) negative response or timeout after MAIL (b) negative response after DATA @@ -4203,89 +4675,110 @@ connection to a new process. However, not all servers can handle this (Exim can), so we do not pass such a connection on if the host matches hosts_nopass_tls. */ +happy: + DEBUG(D_transport) debug_printf("ok=%d send_quit=%d send_rset=%d continue_more=%d " "yield=%d first_address is %sNULL\n", sx->ok, sx->send_quit, sx->send_rset, f.continue_more, yield, sx->first_addr ? "not " : ""); if (sx->completed_addr && sx->ok && sx->send_quit) - { - smtp_compare_t t_compare; - - t_compare.tblock = tblock; - t_compare.current_sender_address = sender_address; - - if ( sx->first_addr != NULL /* more addrs for this message */ - || f.continue_more /* more addrs for coninued-host */ - || ( -#ifndef DISABLE_TLS - ( tls_out.active.sock < 0 && !continue_proxy_cipher - || verify_check_given_host(CUSS &ob->hosts_nopass_tls, host) != OK - ) - && +#ifndef DISABLE_ESMTP_LIMITS + if (mail_limit = continue_sequence >= sx->max_mail) + { + DEBUG(D_transport) + debug_printf("reached limit %u for MAILs per conn\n", sx->max_mail); + } + else #endif - transport_check_waiting(tblock->name, host->name, - tblock->connection_max_messages, new_message_id, - (oicf)smtp_are_same_identities, (void*)&t_compare) - ) ) { - uschar *msg; - BOOL pass_message; + smtp_compare_t t_compare = + {.tblock = tblock, .current_sender_address = sender_address}; - if (sx->send_rset) - if (! (sx->ok = smtp_write_command(sx, SCMD_FLUSH, "RSET\r\n") >= 0)) - { - msg = US string_sprintf("send() to %s [%s] failed: %s", host->name, - host->address, strerror(errno)); - sx->send_quit = FALSE; - } - else if (! (sx->ok = smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), - '2', ob->command_timeout))) - { - int code; - sx->send_quit = check_response(host, &errno, 0, sx->buffer, &code, &msg, - &pass_message); - if (!sx->send_quit) - { - DEBUG(D_transport) debug_printf("H=%s [%s] %s\n", - host->name, host->address, msg); - } - } - - /* Either RSET was not needed, or it succeeded */ - - if (sx->ok) - { + if ( sx->first_addr /* more addrs for this message */ + || f.continue_more /* more addrs for continued-host */ + || tcw_done && tcw /* more messages for host */ + || ( #ifndef DISABLE_TLS - int pfd[2]; + ( tls_out.active.sock < 0 && !continue_proxy_cipher + || verify_check_given_host(CUSS &ob->hosts_nopass_tls, host) != OK + ) + && #endif - int socket_fd = sx->cctx.sock; - + transport_check_waiting(tblock->name, host->name, + sx->max_mail, new_message_id, + (oicf)smtp_are_same_identities, (void*)&t_compare) + ) ) + { + uschar *msg; + BOOL pass_message; - if (sx->first_addr != NULL) /* More addresses still to be sent */ - { /* for this message */ - continue_sequence++; /* Causes * in logging */ - pipelining_active = sx->pipelining_used; /* was cleared at DATA */ - goto SEND_MESSAGE; - } + if (sx->send_rset) + if (! (sx->ok = smtp_write_command(sx, SCMD_FLUSH, "RSET\r\n") >= 0)) + { + msg = US string_sprintf("smtp send to %s [%s] failed: %s", host->name, + host->address, strerror(errno)); + sx->send_quit = FALSE; + } + else if (! (sx->ok = smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), + '2', ob->command_timeout))) + { + int code; + sx->send_quit = check_response(host, &errno, 0, sx->buffer, &code, &msg, + &pass_message); + if (!sx->send_quit) + { + DEBUG(D_transport) debug_printf("H=%s [%s] %s\n", + host->name, host->address, msg); + } + } - /* Unless caller said it already has more messages listed for this host, - pass the connection on to a new Exim process (below, the call to - transport_pass_socket). If the caller has more ready, just return with - the connection still open. */ + /* Either RSET was not needed, or it succeeded */ + if (sx->ok) + { #ifndef DISABLE_TLS - if (tls_out.active.sock >= 0) - if ( f.continue_more - || verify_check_given_host(CUSS &ob->hosts_noproxy_tls, host) == OK) - { - /* Before passing the socket on, or returning to caller with it still - open, we must shut down TLS. Not all MTAs allow for the continuation - of the SMTP session when TLS is shut down. We test for this by sending - a new EHLO. If we don't get a good response, we don't attempt to pass - the socket on. */ + int pfd[2]; +#endif + int socket_fd = sx->cctx.sock; + + if (sx->first_addr) /* More addresses still to be sent */ + { /* for this message */ +#ifndef DISABLE_ESMTP_LIMITS + /* Any that we marked as skipped, reset to do now */ + for (address_item * a = sx->first_addr; a; a = a->next) + if (a->transport_return == SKIP) + a->transport_return = PENDING_DEFER; +#endif + continue_sequence++; /* for consistency */ + clearflag(sx->first_addr, af_new_conn); + setflag(sx->first_addr, af_cont_conn); /* Causes * in logging */ + pipelining_active = sx->pipelining_used; /* was cleared at DATA */ + goto SEND_MESSAGE; + } - tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_WAIT); + /* Unless caller said it already has more messages listed for this host, + pass the connection on to a new Exim process (below, the call to + transport_pass_socket). If the caller has more ready, just return with + the connection still open. */ + +#ifndef DISABLE_TLS + if (tls_out.active.sock >= 0) + if ( f.continue_more + || verify_check_given_host(CUSS &ob->hosts_noproxy_tls, host) == OK) + { + /* Before passing the socket on, or returning to caller with it still + open, we must shut down TLS. Not all MTAs allow for the continuation + of the SMTP session when TLS is shut down. We test for this by sending + a new EHLO. If we don't get a good response, we don't attempt to pass + the socket on. + NB: TLS close is *required* per RFC 9266 when tls-exporter info has + been used, which we do under TLSv1.3 for the gsasl SCRAM*PLUS methods. + But we were always doing it anyway. */ + + tls_close(sx->cctx.tls_ctx, + sx->send_tlsclose ? TLS_SHUTDOWN_WAIT : TLS_SHUTDOWN_WONLY); + sx->send_tlsclose = FALSE; sx->cctx.tls_ctx = NULL; tls_out.active.sock = -1; smtp_peer_options = smtp_peer_options_wrap; @@ -4295,119 +4788,110 @@ if (sx->completed_addr && sx->ok && sx->send_quit) && smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2', ob->command_timeout); - if (sx->ok && f.continue_more) - goto TIDYUP; /* More addresses for another run */ - } - else - { - /* Set up a pipe for proxying TLS for the new transport process */ - - smtp_peer_options |= OPTION_TLS; - if ((sx->ok = socketpair(AF_UNIX, SOCK_STREAM, 0, pfd) == 0)) - socket_fd = pfd[1]; + if (sx->ok && f.continue_more) + goto TIDYUP; /* More addresses for another run */ + } else - set_errno(sx->first_addr, errno, US"internal allocation problem", - DEFER, FALSE, host, + { + /* Set up a pipe for proxying TLS for the new transport process */ + + smtp_peer_options |= OPTION_TLS; + if ((sx->ok = socketpair(AF_UNIX, SOCK_STREAM, 0, pfd) == 0)) + socket_fd = pfd[1]; + else + set_errno(sx->first_addr, errno, US"internal allocation problem", + DEFER, FALSE, host, # ifdef EXPERIMENTAL_DSN_INFO - sx->smtp_greeting, sx->helo_response, + sx->smtp_greeting, sx->helo_response, # endif - &sx->delivery_start); - } - else + &sx->delivery_start); + } + else #endif - if (f.continue_more) - goto TIDYUP; /* More addresses for another run */ + if (f.continue_more) + goto TIDYUP; /* More addresses for another run */ - /* If the socket is successfully passed, we mustn't send QUIT (or - indeed anything!) from here. */ + /* If the socket is successfully passed, we mustn't send QUIT (or + indeed anything!) from here. */ -/*XXX DSN_INFO: assume likely to do new HELO; but for greet we'll want to -propagate it from the initial -*/ - if (sx->ok && transport_pass_socket(tblock->name, host->name, - host->address, new_message_id, socket_fd)) - { - sx->send_quit = FALSE; - - /* We have passed the client socket to a fresh transport process. - If TLS is still active, we need to proxy it for the transport we - just passed the baton to. Fork a child to to do it, and return to - get logging done asap. Which way to place the work makes assumptions - about post-fork prioritisation which may not hold on all platforms. */ -#ifndef DISABLE_TLS - if (tls_out.active.sock >= 0) + /*XXX DSN_INFO: assume likely to do new HELO; but for greet we'll want to + propagate it from the initial + */ + if (sx->ok && transport_pass_socket(tblock->name, host->name, + host->address, new_message_id, socket_fd +#ifndef DISABLE_ESMTP_LIMITS + , sx->peer_limit_mail, sx->peer_limit_rcpt, sx->peer_limit_rcptdom +#endif + )) { - int pid = exim_fork(US"tls-proxy-interproc"); - if (pid == 0) /* child; fork again to disconnect totally */ - { - /* does not return */ - smtp_proxy_tls(sx->cctx.tls_ctx, sx->buffer, sizeof(sx->buffer), pfd, - ob->command_timeout); - } + sx->send_quit = FALSE; - if (pid > 0) /* parent */ + /* We have passed the client socket to a fresh transport process. + If TLS is still active, we need to proxy it for the transport we + just passed the baton to. Fork a child to to do it, and return to + get logging done asap. Which way to place the work makes assumptions + about post-fork prioritisation which may not hold on all platforms. */ +#ifndef DISABLE_TLS + if (tls_out.active.sock >= 0) { - close(pfd[0]); - /* tidy the inter-proc to disconn the proxy proc */ - waitpid(pid, NULL, 0); - tls_close(sx->cctx.tls_ctx, TLS_NO_SHUTDOWN); - sx->cctx.tls_ctx = NULL; - (void)close(sx->cctx.sock); - sx->cctx.sock = -1; - continue_transport = NULL; - continue_hostname = NULL; - goto TIDYUP; + int pid = exim_fork(US"tls-proxy-interproc"); + if (pid == 0) /* child; fork again to disconnect totally */ + { + /* does not return */ + smtp_proxy_tls(sx->cctx.tls_ctx, sx->buffer, sizeof(sx->buffer), pfd, + ob->command_timeout, host->name); + } + + if (pid > 0) /* parent */ + { + close(pfd[0]); + /* tidy the inter-proc to disconn the proxy proc */ + waitpid(pid, NULL, 0); + tls_close(sx->cctx.tls_ctx, TLS_NO_SHUTDOWN); + sx->cctx.tls_ctx = NULL; + (void)close(sx->cctx.sock); + sx->cctx.sock = -1; + continue_transport = NULL; + continue_hostname = NULL; + goto TIDYUP; + } + log_write(0, LOG_PANIC_DIE, "fork failed"); } - log_write(0, LOG_PANIC_DIE, "fork failed"); - } #endif + } } - } - /* If RSET failed and there are addresses left, they get deferred. */ - else - set_errno(sx->first_addr, errno, msg, DEFER, FALSE, host, + /* If RSET failed and there are addresses left, they get deferred. */ + else + set_errno(sx->first_addr, errno, msg, DEFER, FALSE, host, #ifdef EXPERIMENTAL_DSN_INFO - sx->smtp_greeting, sx->helo_response, + sx->smtp_greeting, sx->helo_response, #endif - &sx->delivery_start); + &sx->delivery_start); + } } - } /* End off tidily with QUIT unless the connection has died or the socket has -been passed to another process. There has been discussion on the net about what -to do after sending QUIT. The wording of the RFC suggests that it is necessary -to wait for a response, but on the other hand, there isn't anything one can do -with an error response, other than log it. Exim used to do that. However, -further discussion suggested that it is positively advantageous not to wait for -the response, but to close the session immediately. This is supposed to move -the TCP/IP TIME_WAIT state from the server to the client, thereby removing some -load from the server. (Hosts that are both servers and clients may not see much -difference, of course.) Further discussion indicated that this was safe to do -on Unix systems which have decent implementations of TCP/IP that leave the -connection around for a while (TIME_WAIT) after the application has gone away. -This enables the response sent by the server to be properly ACKed rather than -timed out, as can happen on broken TCP/IP implementations on other OS. - -This change is being made on 31-Jul-98. After over a year of trouble-free -operation, the old commented-out code was removed on 17-Sep-99. */ +been passed to another process. */ SEND_QUIT: if (sx->send_quit) - { -#ifdef EXIM_TCP_CORK - (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on)); + { /* Use _MORE to get QUIT in FIN segment */ + (void)smtp_write_command(sx, SCMD_MORE, "QUIT\r\n"); +#ifndef DISABLE_TLS + if (sx->cctx.tls_ctx && sx->send_tlsclose) + { +# ifdef EXIM_TCP_CORK /* Use _CORK to get TLS Close Notify in FIN segment */ + (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on)); +# endif + tls_shutdown_wr(sx->cctx.tls_ctx); + sx->send_tlsclose = FALSE; + } #endif - (void)smtp_write_command(sx, SCMD_FLUSH, "QUIT\r\n"); } END_OFF: -#ifndef DISABLE_TLS -tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_NOWAIT); -sx->cctx.tls_ctx = NULL; -#endif - /* Close the socket, and return the appropriate value, first setting works because the NULL setting is passed back to the calling process, and remote_max_parallel is forced to 1 when delivering over an existing connection, @@ -4418,20 +4902,74 @@ writing RSET might have failed, or there may be other addresses whose hosts are specified in the transports, and therefore not visible at top level, in which case continue_more won't get set. */ -HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(close)>>\n"); if (sx->send_quit) { + /* This flushes data queued in the socket, being the QUIT and any TLS Close, + sending them along with the client FIN flag. Us (we hope) sending FIN first + means we (client) take the TIME_WAIT state, so the server (which likely has a + higher connection rate) does not have to. */ + + HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(shutdown)>>\n"); shutdown(sx->cctx.sock, SHUT_WR); - millisleep(20); - testharness_pause_ms(200); - if (fcntl(sx->cctx.sock, F_SETFL, O_NONBLOCK) == 0) - for (int i = 16; read(sx->cctx.sock, sx->inbuffer, sizeof(sx->inbuffer)) > 0 && i > 0;) - i--; /* drain socket */ } + +if (sx->send_quit || tcw_done && !tcw) + { + /* Wait for (we hope) ack of our QUIT, and a server FIN. Discard any data + received, then discard the socket. Any packet received after then, or receive + data still in the socket, will get a RST - hence the pause/drain. */ + + /* Reap the response to QUIT, timing out after one second */ + (void) smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2', 1); +#ifndef DISABLE_TLS + if (sx->cctx.tls_ctx) + { + int n; + + /* Reap the TLS Close Notify from the server, timing out after one second */ + sigalrm_seen = FALSE; + ALARM(1); + do + n = tls_read(sx->cctx.tls_ctx, sx->inbuffer, sizeof(sx->inbuffer)); + while (!sigalrm_seen && n > 0); + ALARM_CLR(0); + + if (sx->send_tlsclose) + { +# ifdef EXIM_TCP_CORK + (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on)); +# endif + tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_WAIT); + } + else + tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_WONLY); + sx->cctx.tls_ctx = NULL; + } +#endif + + /* Drain any trailing data from the socket before close, to avoid sending a RST */ + + if ( poll_one_fd(sx->cctx.sock, POLLIN, 20) != 0 /* 20ms */ + && fcntl(sx->cctx.sock, F_SETFL, O_NONBLOCK) == 0) + for (int i = 16, n; /* drain socket */ + (n = read(sx->cctx.sock, sx->inbuffer, sizeof(sx->inbuffer))) > 0 && i > 0; + i--) HDEBUG(D_transport|D_acl|D_v) + { + int m = MIN(n, 64); + debug_printf_indent(" SMTP(drain %d bytes)<< %.*s\n", n, m, sx->inbuffer); + for (m = 0; m < n; m++) + debug_printf("0x%02x\n", sx->inbuffer[m]); + } + } +HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(close)>>\n"); (void)close(sx->cctx.sock); +sx->cctx.sock = -1; +continue_transport = NULL; +continue_hostname = NULL; +smtp_debug_cmd_report(); #ifndef DISABLE_EVENT -(void) event_raise(tblock->event_action, US"tcp:close", NULL); +(void) event_raise(tblock->event_action, US"tcp:close", NULL, NULL); #endif #ifdef SUPPORT_DANE @@ -4448,15 +4986,30 @@ if (dane_held) to get the domain string for SNI */ sx->first_addr = a; + clearflag(a, af_cont_conn); + setflag(a, af_new_conn); /* clear * from logging */ DEBUG(D_transport) debug_printf("DANE: go-around for %s\n", a->domain); } } - goto DANE_DOMAINS; + continue_sequence = 1; /* for consistency */ + goto REPEAT_CONN; + } +#endif + +#ifndef DISABLE_ESMTP_LIMITS +if (mail_limit && sx->first_addr) + { + /* Reset the sequence count since we closed the connection. This is flagged + on the pipe back to the delivery process so that a non-continued-conn delivery + is logged. */ + + continue_sequence = 1; /* for consistency */ + clearflag(sx->first_addr, af_cont_conn); + setflag(sx->first_addr, af_new_conn); /* clear * from logging */ + goto REPEAT_CONN; } #endif -continue_transport = NULL; -continue_hostname = NULL; return yield; TIDYUP: @@ -4493,7 +5046,7 @@ smtp_transport_closedown(transport_instance *tblock) { smtp_transport_options_block * ob = SOB tblock->options_block; client_conn_ctx cctx; -smtp_context sx; +smtp_context sx = {0}; uschar buffer[256]; uschar inbuffer[4096]; uschar outbuffer[16]; @@ -4541,16 +5094,16 @@ Returns: the first address for this delivery */ static address_item * -prepare_addresses(address_item *addrlist, host_item *host) +prepare_addresses(address_item * addrlist, host_item * host) { -address_item *first_addr = NULL; +address_item * first_addr = NULL; for (address_item * addr = addrlist; addr; addr = addr->next) if (addr->transport_return == DEFER) { if (!first_addr) first_addr = addr; addr->transport_return = PENDING_DEFER; addr->basic_errno = 0; - addr->more_errno = (host->mx >= 0)? 'M' : 'A'; + addr->more_errno = host->mx >= 0 ? 'M' : 'A'; addr->message = NULL; #ifndef DISABLE_TLS addr->cipher = NULL; @@ -4582,24 +5135,17 @@ FALSE. */ BOOL smtp_transport_entry( - transport_instance *tblock, /* data for this instantiation */ - address_item *addrlist) /* addresses we are working on */ + transport_instance * tblock, /* data for this instantiation */ + address_item * addrlist) /* addresses we are working on */ { int defport; -int hosts_defer = 0; -int hosts_fail = 0; -int hosts_looked_up = 0; -int hosts_retry = 0; -int hosts_serial = 0; -int hosts_total = 0; -int total_hosts_tried = 0; +int hosts_defer = 0, hosts_fail = 0, hosts_looked_up = 0; +int hosts_retry = 0, hosts_serial = 0, hosts_total = 0, total_hosts_tried = 0; BOOL expired = TRUE; -uschar *expanded_hosts = NULL; -uschar *pistring; -uschar *tid = string_sprintf("%s transport", tblock->name); -smtp_transport_options_block *ob = SOB tblock->options_block; -host_item *hostlist = addrlist->host_list; -host_item *host = NULL; +uschar * expanded_hosts = NULL, * pistring; +uschar * tid = string_sprintf("%s transport", tblock->name); +smtp_transport_options_block * ob = SOB tblock->options_block; +host_item * hostlist = addrlist->host_list, * host = NULL; DEBUG(D_transport) { @@ -4639,7 +5185,7 @@ database if the delivery fails temporarily or if we are running with queue_smtp or a 2-stage queue run. This gets unset for certain kinds of error, typically those that are specific to the message. */ -update_waiting = TRUE; +update_waiting = TRUE; /* If a host list is not defined for the addresses - they must all have the same one in order to be passed to a single transport - or if the transport has @@ -4671,7 +5217,7 @@ if (!hostlist || (ob->hosts_override && ob->hosts)) { uschar *s = ob->hosts; - if (Ustrchr(s, '$') != NULL) + if (Ustrchr(s, '$')) { if (!(expanded_hosts = expand_string(s))) { @@ -4827,16 +5373,23 @@ retry_non_continued: && total_hosts_tried < ob->hosts_max_try_hardlimit; host = nexthost) { - int rc; - int host_af; - BOOL host_is_expired = FALSE; - BOOL message_defer = FALSE; - BOOL some_deferred = FALSE; - address_item *first_addr = NULL; - uschar *interface = NULL; - uschar *retry_host_key = NULL; - uschar *retry_message_key = NULL; - uschar *serialize_key = NULL; + int rc, host_af; + BOOL host_is_expired = FALSE, message_defer = FALSE, some_deferred = FALSE; + address_item * first_addr = NULL; + uschar * interface = NULL; + const uschar * retry_host_key = NULL, * retry_message_key = NULL; + uschar * serialize_key = NULL; + + /* Deal slightly better with a possible Linux kernel bug that results + in intermittent TFO-conn fails deep into the TCP flow. Bug 2907 tracks. + Hack: Clear TFO option for any further hosts on this tpt run. */ + + if (total_hosts_tried > 0) + { + DEBUG(D_transport|D_acl|D_v) + debug_printf("Clearing TFO as not first host for message\n"); + ob->hosts_try_fastopen = US""; + } /* Default next host is next host. :-) But this can vary if the hosts_max_try limit is hit (see below). It may also be reset if a host @@ -5019,10 +5572,11 @@ retry_non_continued: because connections to the same host from a different interface should be treated separately. */ - host_af = Ustrchr(host->address, ':') == NULL ? AF_INET : AF_INET6; + host_af = Ustrchr(host->address, ':') ? AF_INET6 : AF_INET; { - uschar * s = ob->interface; - if (s && *s) + uschar * s; + GET_OPTION("interface"); + if ((s = ob->interface) && *s) { if (!smtp_get_interface(s, host_af, addrlist, &interface, tid)) return FALSE; @@ -5132,7 +5686,14 @@ retry_non_continued: out the result of previous attempts, and finding the first address that is still to be delivered. */ - first_addr = prepare_addresses(addrlist, host); + if (!(first_addr = prepare_addresses(addrlist, host))) + { + /* Obscure situation; at least one case (bug 3059, fixed) where + a previous host try returned DEFER, but having moved all + recipients away from DEFER (the waiting-to-be-done state). */ + DEBUG(D_transport) debug_printf("no pending recipients\n"); + goto END_TRANSPORT; + } DEBUG(D_transport) debug_printf("delivering %s to %s [%s] (%s%s)\n", message_id, host->name, host->address, addrlist->address, @@ -5184,11 +5745,11 @@ retry_non_continued: { host_item * thost; /* Make a copy of the host if it is local to this invocation - of the transport. */ + of the transport. */ if (expanded_hosts) { - thost = store_get(sizeof(host_item), FALSE); + thost = store_get(sizeof(host_item), GET_UNTAINTED); *thost = *host; thost->name = string_copy(host->name); thost->address = string_copy(host->address); @@ -5313,9 +5874,7 @@ retry_non_continued: ob->expand_retry_include_ip_address, &incl_ip) != OK) incl_ip = TRUE; /* error; use most-specific retry record */ - retry_host_key = incl_ip - ? string_sprintf("T:%S:%s%s", host->name, host->address, pistring) - : string_sprintf("T:%S%s", host->name, pistring); + retry_host_key = retry_host_key_build(host, incl_ip, pistring); } /* If a delivery of another message over an existing SMTP connection @@ -5361,10 +5920,8 @@ retry_non_continued: ob->expand_retry_include_ip_address, &incl_ip) != OK) incl_ip = TRUE; /* error; use most-specific retry record */ - retry_message_key = incl_ip - ? string_sprintf("T:%S:%s%s:%s", host->name, host->address, pistring, - message_id) - : string_sprintf("T:%S%s:%s", host->name, pistring, message_id); + retry_message_key = string_sprintf("%s:%s", + retry_host_key_build(host, incl_ip, pistring), message_id); } retry_add_item(addrlist, retry_message_key, rf_message | rf_host | delete_flag); @@ -5378,20 +5935,14 @@ retry_non_continued: if (rc == OK) for (address_item * addr = addrlist; addr; addr = addr->next) if (addr->transport_return == DEFER) - { - some_deferred = TRUE; - break; - } + { some_deferred = TRUE; break; } /* If no addresses deferred or the result was ERROR, return. We do this for ERROR because a failing filter set-up or add_headers expansion is likely to fail for any host we try. */ if (rc == ERROR || (rc == OK && !some_deferred)) - { - DEBUG(D_transport) debug_printf("Leaving %s transport\n", tblock->name); - return TRUE; /* Each address has its status */ - } + goto END_TRANSPORT; /* If the result was DEFER or some individual addresses deferred, let the loop run to try other hosts with the deferred addresses, except for the @@ -5411,7 +5962,7 @@ retry_non_continued: if ((rc == DEFER || some_deferred) && nexthost) { BOOL timedout; - retry_config *retry = retry_find_config(host->name, NULL, 0, 0); + retry_config * retry = retry_find_config(host->name, NULL, 0, 0); if (retry && retry->rules) {