X-Git-Url: https://git.exim.org/exim.git/blobdiff_plain/a1108b5118d32e969c5fe91b2110944f7483a7cb..1d28cc061677bd07d9bed48dd84bd5c590247043:/src/src/transports/smtp.c diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c index 52940572b..1183fa478 100644 --- a/src/src/transports/smtp.c +++ b/src/src/transports/smtp.c @@ -2,9 +2,10 @@ * Exim - an Internet mail transport agent * *************************************************/ +/* Copyright (c) The Exim Maintainers 2020 - 2022 */ /* Copyright (c) University of Cambridge 1995 - 2018 */ -/* Copyright (c) The Exim Maintainers 2020 */ /* See the file NOTICE for conditions of use and distribution. */ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #include "../exim.h" #include "smtp.h" @@ -64,6 +65,9 @@ optionlist smtp_transport_options[] = { { "final_timeout", opt_time, LOFF(final_timeout) }, { "gethostbyname", opt_bool, LOFF(gethostbyname) }, { "helo_data", opt_stringptr, LOFF(helo_data) }, +#if !defined(DISABLE_TLS) && !defined(DISABLE_TLS_RESUME) + { "host_name_extract", opt_stringptr, LOFF(host_name_extract) }, +# endif { "hosts", opt_stringptr, LOFF(hosts) }, { "hosts_avoid_esmtp", opt_stringptr, LOFF(hosts_avoid_esmtp) }, { "hosts_avoid_pipelining", opt_stringptr, LOFF(hosts_avoid_pipelining) }, @@ -84,6 +88,7 @@ optionlist smtp_transport_options[] = { #if !defined(DISABLE_TLS) && !defined(DISABLE_OCSP) { "hosts_request_ocsp", opt_stringptr, LOFF(hosts_request_ocsp) }, #endif + { "hosts_require_alpn", opt_stringptr, LOFF(hosts_require_alpn) }, { "hosts_require_auth", opt_stringptr, LOFF(hosts_require_auth) }, #ifndef DISABLE_TLS # ifdef SUPPORT_DANE @@ -123,6 +128,7 @@ optionlist smtp_transport_options[] = { { "socks_proxy", opt_stringptr, LOFF(socks_proxy) }, #endif #ifndef DISABLE_TLS + { "tls_alpn", opt_stringptr, LOFF(tls_alpn) }, { "tls_certificate", opt_stringptr, LOFF(tls_certificate) }, { "tls_crl", opt_stringptr, LOFF(tls_crl) }, { "tls_dh_min_bits", opt_int, LOFF(tls_dh_min_bits) }, @@ -192,13 +198,14 @@ smtp_transport_options_block smtp_transport_option_defaults = { .keepalive = TRUE, .retry_include_ip_address = TRUE, #ifndef DISABLE_TLS -# if defined(SUPPORT_SYSDEFAULT_CABUNDLE) || !defined(USE_GNUTLS) .tls_verify_certificates = US"system", -# endif .tls_dh_min_bits = EXIM_CLIENT_DH_DEFAULT_MIN_BITS, .tls_tempfail_tryclear = TRUE, .tls_try_verify_hosts = US"*", .tls_verify_cert_hostnames = US"*", +# ifndef DISABLE_TLS_RESUME + .host_name_extract = US"${if and {{match{$host}{.outlook.com\\$}} {match{$item}{\\N^250-([\\w.]+)\\s\\N}}} {$1}}", +# endif #endif #ifdef SUPPORT_I18N .utf8_downconvert = US"-1", @@ -237,48 +244,39 @@ static unsigned ehlo_response(uschar * buf, unsigned checks); void smtp_deliver_init(void) { -if (!regex_PIPELINING) regex_PIPELINING = - regex_must_compile(US"\\n250[\\s\\-]PIPELINING(\\s|\\n|$)", FALSE, TRUE); - -if (!regex_SIZE) regex_SIZE = - regex_must_compile(US"\\n250[\\s\\-]SIZE(\\s|\\n|$)", FALSE, TRUE); - -if (!regex_AUTH) regex_AUTH = - regex_must_compile(AUTHS_REGEX, FALSE, TRUE); +struct list + { + const pcre2_code ** re; + const uschar * string; + } list[] = + { + { ®ex_AUTH, AUTHS_REGEX }, + { ®ex_CHUNKING, US"\\n250[\\s\\-]CHUNKING(\\s|\\n|$)" }, + { ®ex_DSN, US"\\n250[\\s\\-]DSN(\\s|\\n|$)" }, + { ®ex_IGNOREQUOTA, US"\\n250[\\s\\-]IGNOREQUOTA(\\s|\\n|$)" }, + { ®ex_PIPELINING, US"\\n250[\\s\\-]PIPELINING(\\s|\\n|$)" }, + { ®ex_SIZE, US"\\n250[\\s\\-]SIZE(\\s|\\n|$)" }, #ifndef DISABLE_TLS -if (!regex_STARTTLS) regex_STARTTLS = - regex_must_compile(US"\\n250[\\s\\-]STARTTLS(\\s|\\n|$)", FALSE, TRUE); + { ®ex_STARTTLS, US"\\n250[\\s\\-]STARTTLS(\\s|\\n|$)" }, #endif - -if (!regex_CHUNKING) regex_CHUNKING = - regex_must_compile(US"\\n250[\\s\\-]CHUNKING(\\s|\\n|$)", FALSE, TRUE); - #ifndef DISABLE_PRDR -if (!regex_PRDR) regex_PRDR = - regex_must_compile(US"\\n250[\\s\\-]PRDR(\\s|\\n|$)", FALSE, TRUE); + { ®ex_PRDR, US"\\n250[\\s\\-]PRDR(\\s|\\n|$)" }, #endif - #ifdef SUPPORT_I18N -if (!regex_UTF8) regex_UTF8 = - regex_must_compile(US"\\n250[\\s\\-]SMTPUTF8(\\s|\\n|$)", FALSE, TRUE); + { ®ex_UTF8, US"\\n250[\\s\\-]SMTPUTF8(\\s|\\n|$)" }, #endif - -if (!regex_DSN) regex_DSN = - regex_must_compile(US"\\n250[\\s\\-]DSN(\\s|\\n|$)", FALSE, TRUE); - -if (!regex_IGNOREQUOTA) regex_IGNOREQUOTA = - regex_must_compile(US"\\n250[\\s\\-]IGNOREQUOTA(\\s|\\n|$)", FALSE, TRUE); - #ifndef DISABLE_PIPE_CONNECT -if (!regex_EARLY_PIPE) regex_EARLY_PIPE = - regex_must_compile(US"\\n250[\\s\\-]" EARLY_PIPE_FEATURE_NAME "(\\s|\\n|$)", FALSE, TRUE); + { ®ex_EARLY_PIPE, US"\\n250[\\s\\-]" EARLY_PIPE_FEATURE_NAME "(\\s|\\n|$)" }, #endif - #ifdef EXPERIMENTAL_ESMTP_LIMITS -if (!regex_LIMITS) regex_LIMITS = - regex_must_compile(US"\\n250[\\s\\-]LIMITS\\s", FALSE, TRUE); + { ®ex_LIMITS, US"\\n250[\\s\\-]LIMITS\\s" }, #endif + }; + +for (struct list * l = list; l < list + nelem(list); l++) + if (!*l->re) + *l->re = regex_must_compile(l->string, MCS_NOFLAGS, TRUE); } @@ -355,6 +353,7 @@ void smtp_transport_init(transport_instance *tblock) { smtp_transport_options_block *ob = SOB tblock->options_block; +int old_pool = store_pool; /* Retry_use_local_part defaults FALSE if unset */ @@ -390,7 +389,9 @@ if (ob->hosts_override && ob->hosts) tblock->overrides_hosts = TRUE; /* If there are any fallback hosts listed, build a chain of host items for them, but do not do any lookups at this time. */ -host_build_hostlist(&(ob->fallback_hostlist), ob->fallback_hosts, FALSE); +store_pool = POOL_PERM; +host_build_hostlist(&ob->fallback_hostlist, ob->fallback_hosts, FALSE); +store_pool = old_pool; } @@ -684,7 +685,8 @@ deliver_localpart = addr->local_part; : string_copy(addr->message) : addr->basic_errno > 0 ? string_copy(US strerror(addr->basic_errno)) - : NULL); + : NULL, + NULL); deliver_localpart = save_local; deliver_domain = save_domain; @@ -722,9 +724,11 @@ BOOL good_response; { /* Hack to get QUICKACK disabled; has to be right after 3whs, and has to on->off */ int sock = sx->cctx.sock; struct pollfd p = {.fd = sock, .events = POLLOUT}; - int rc = poll(&p, 1, 1000); - (void) setsockopt(sock, IPPROTO_TCP, TCP_QUICKACK, US &on, sizeof(on)); - (void) setsockopt(sock, IPPROTO_TCP, TCP_QUICKACK, US &off, sizeof(off)); + if (poll(&p, 1, 1000) >= 0) /* retval test solely for compiler quitening */ + { + (void) setsockopt(sock, IPPROTO_TCP, TCP_QUICKACK, US &on, sizeof(on)); + (void) setsockopt(sock, IPPROTO_TCP, TCP_QUICKACK, US &off, sizeof(off)); + } } #endif good_response = smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), @@ -755,12 +759,34 @@ sx->helo_response = string_copy(sx->buffer); #endif #ifndef DISABLE_EVENT (void) event_raise(sx->conn_args.tblock->event_action, - US"smtp:ehlo", sx->buffer); + US"smtp:ehlo", sx->buffer, NULL); #endif return TRUE; } +/* Grab a string differentiating server behind a loadbalancer, for TLS +resumption when such servers do not share a session-cache */ + +static void +ehlo_response_lbserver(smtp_context * sx, smtp_transport_options_block * ob) +{ +#if !defined(DISABLE_TLS) && !defined(DISABLE_TLS_RESUME) +const uschar * s; +uschar * save_item = iterate_item; + +if (sx->conn_args.have_lbserver) + return; +iterate_item = sx->buffer; +s = expand_cstring(ob->host_name_extract); +iterate_item = save_item; +sx->conn_args.host_lbserver = s && !*s ? NULL : s; +sx->conn_args.have_lbserver = TRUE; +#endif +} + + + /******************************************************************************/ #ifdef EXPERIMENTAL_ESMTP_LIMITS @@ -774,13 +800,12 @@ This saves us dealing with a duplicate set of values. */ static void ehlo_response_limits_read(smtp_context * sx) { -int ovec[3]; /* results vector for a main-match only */ +uschar * match; /* matches up to just after the first space after the keyword */ -if (pcre_exec(regex_LIMITS, NULL, CS sx->buffer, Ustrlen(sx->buffer), - 0, PCRE_EOPT, ovec, nelem(ovec)) >= 0) - for (const uschar * s = sx->buffer + ovec[1]; *s; ) +if (regex_match(regex_LIMITS, sx->buffer, -1, &match)) + for (const uschar * s = sx->buffer + Ustrlen(match); *s; ) { while (isspace(*s)) s++; if (*s == '\n') break; @@ -831,10 +856,12 @@ ehlo_limits_apply(sx, sx->peer_limit_mail, sx->peer_limit_rcpt, static void ehlo_cache_limits_apply(smtp_context * sx) { +# ifndef DISABLE_PIPE_CONNECT ehlo_limits_apply(sx, sx->ehlo_resp.limit_mail, sx->ehlo_resp.limit_rcpt, sx->ehlo_resp.limit_rcptdom); +# endif } -#endif +#endif /*EXPERIMENTAL_ESMTP_LIMITS*/ /******************************************************************************/ @@ -853,9 +880,9 @@ return Ustrchr(host->address, ':') /* Cache EHLO-response info for use by early-pipe. Called - During a normal flow on EHLO response (either cleartext or under TLS), - when we are willing to do PIPE_CONNECT and it is offered + when we are willing to do PIPECONNECT and it is offered - During an early-pipe flow on receiving the actual EHLO response and noting - disparity versus the cached info used, when PIPE_CONNECT is still being offered + disparity versus the cached info used, when PIPECONNECT is still being offered We assume that suitable values have been set in the sx.ehlo_resp structure for features and auths; we handle the copy of limits. */ @@ -865,11 +892,11 @@ write_ehlo_cache_entry(smtp_context * sx) { open_db dbblock, * dbm_file; -#ifdef EXPERIMENTAL_ESMTP_LIMITS +# ifdef EXPERIMENTAL_ESMTP_LIMITS sx->ehlo_resp.limit_mail = sx->peer_limit_mail; sx->ehlo_resp.limit_rcpt = sx->peer_limit_rcpt; sx->ehlo_resp.limit_rcptdom = sx->peer_limit_rcptdom; -#endif +# endif if ((dbm_file = dbfn_open(US"misc", O_RDWR, &dbblock, TRUE, TRUE))) { @@ -877,7 +904,7 @@ if ((dbm_file = dbfn_open(US"misc", O_RDWR, &dbblock, TRUE, TRUE))) dbdata_ehlo_resp er = { .data = sx->ehlo_resp }; HDEBUG(D_transport) -#ifdef EXPERIMENTAL_ESMTP_LIMITS +# ifdef EXPERIMENTAL_ESMTP_LIMITS if (sx->ehlo_resp.limit_mail || sx->ehlo_resp.limit_rcpt || sx->ehlo_resp.limit_rcptdom) debug_printf("writing clr %04x/%04x cry %04x/%04x lim %05d/%05d/%05d\n", sx->ehlo_resp.cleartext_features, sx->ehlo_resp.cleartext_auths, @@ -885,7 +912,7 @@ if ((dbm_file = dbfn_open(US"misc", O_RDWR, &dbblock, TRUE, TRUE))) sx->ehlo_resp.limit_mail, sx->ehlo_resp.limit_rcpt, sx->ehlo_resp.limit_rcptdom); else -#endif +# endif debug_printf("writing clr %04x/%04x cry %04x/%04x\n", sx->ehlo_resp.cleartext_features, sx->ehlo_resp.cleartext_auths, sx->ehlo_resp.crypted_features, sx->ehlo_resp.crypted_auths); @@ -934,7 +961,7 @@ else else { DEBUG(D_transport) -#ifdef EXPERIMENTAL_ESMTP_LIMITS +# ifdef EXPERIMENTAL_ESMTP_LIMITS if (er->data.limit_mail || er->data.limit_rcpt || er->data.limit_rcptdom) debug_printf("EHLO response bits from cache:" " cleartext 0x%04x/0x%04x crypted 0x%04x/0x%04x lim %05d/%05d/%05d\n", @@ -942,16 +969,16 @@ else er->data.crypted_features, er->data.crypted_auths, er->data.limit_mail, er->data.limit_rcpt, er->data.limit_rcptdom); else -#endif +# endif debug_printf("EHLO response bits from cache:" " cleartext 0x%04x/0x%04x crypted 0x%04x/0x%04x\n", er->data.cleartext_features, er->data.cleartext_auths, er->data.crypted_features, er->data.crypted_auths); sx->ehlo_resp = er->data; -#ifdef EXPERIMENTAL_ESMTP_LIMITS +# ifdef EXPERIMENTAL_ESMTP_LIMITS ehlo_cache_limits_apply(sx); -#endif +# endif dbfn_close(dbm_file); return TRUE; } @@ -974,7 +1001,7 @@ uschar authnum; unsigned short authbits = 0; if (!sx->esmtp) return 0; -if (!regex_AUTH) regex_AUTH = regex_must_compile(AUTHS_REGEX, FALSE, TRUE); +if (!regex_AUTH) regex_AUTH = regex_must_compile(AUTHS_REGEX, MCS_NOFLAGS, TRUE); if (!regex_match_and_setup(regex_AUTH, sx->buffer, 0, -1)) return 0; expand_nmax = -1; /* reset */ names = string_copyn(expand_nstring[1], expand_nlength[1]); @@ -1038,6 +1065,8 @@ if (pending_BANNER) if (tls_out.active.sock >= 0) rc = DEFER; goto fail; } + /*XXX EXPERIMENTAL_ESMTP_LIMITS ? */ + ehlo_response_lbserver(sx, sx->conn_args.ob); } if (pending_EHLO) @@ -1066,10 +1095,10 @@ if (pending_EHLO) | OPTION_CHUNKING | OPTION_PRDR | OPTION_DSN | OPTION_PIPE | OPTION_SIZE | OPTION_UTF8 | OPTION_EARLY_PIPE ); -#ifdef EXPERIMENTAL_ESMTP_LIMITS +# ifdef EXPERIMENTAL_ESMTP_LIMITS if (tls_out.active.sock >= 0 || !(peer_offered & OPTION_TLS)) ehlo_response_limits_read(sx); -#endif +# endif if ( peer_offered != sx->peer_offered || (authbits = study_ehlo_auths(sx)) != *ap) { @@ -1090,7 +1119,7 @@ if (pending_EHLO) return OK; /* just carry on */ } -#ifdef EXPERIMENTAL_ESMTP_LIMITS +# ifdef EXPERIMENTAL_ESMTP_LIMITS /* If we are handling LIMITS, compare the actual EHLO LIMITS values with the cached values and invalidate cache if different. OK to carry on with connect since values are advisory. */ @@ -1114,7 +1143,7 @@ if (pending_EHLO) invalidate_ehlo_cache_entry(sx); } } -#endif +# endif } return OK; @@ -1437,11 +1466,18 @@ smtp_transport_options_block * ob = sx->conn_args.ob; /* transport options */ host_item * host = sx->conn_args.host; /* host to deliver to */ int rc; +/* Set up globals for error messages */ + +authenticator_name = au->name; +driver_srcfile = au->srcfile; +driver_srcline = au->srcline; + sx->outblock.authenticating = TRUE; rc = (au->info->clientcode)(au, sx, ob->command_timeout, sx->buffer, sizeof(sx->buffer)); sx->outblock.authenticating = FALSE; -DEBUG(D_transport) debug_printf("%s authenticator yielded %d\n", au->name, rc); +driver_srcfile = authenticator_name = NULL; driver_srcline = 0; +DEBUG(D_transport) debug_printf("%s authenticator yielded %s\n", au->name, rc_names[rc]); /* A temporary authentication failure must hold up delivery to this host. After a permanent authentication failure, we carry on @@ -1465,10 +1501,25 @@ switch(rc) /* Failure after reading a response */ case FAIL: + { + uschar * logmsg = NULL; + if (errno != 0 || sx->buffer[0] != '5') return FAIL; - log_write(0, LOG_MAIN, "%s authenticator failed H=%s [%s] %s", - au->name, host->name, host->address, sx->buffer); +#ifndef DISABLE_EVENT + { + uschar * save_name = sender_host_authenticated; + sender_host_authenticated = au->name; + if ((logmsg = event_raise(sx->conn_args.tblock->event_action, US"auth:fail", + sx->buffer, NULL))) + log_write(0, LOG_MAIN, "%s", logmsg); + sender_host_authenticated = save_name; + } +#endif + if (!logmsg) + log_write(0, LOG_MAIN, "%s authenticator failed H=%s [%s] %s", + au->name, host->name, host->address, sx->buffer); break; + } /* Failure by some other means. In effect, the authenticator decided it wasn't prepared to handle this case. Typically this @@ -1528,7 +1579,7 @@ f.smtp_authenticated = FALSE; client_authenticator = client_authenticated_id = client_authenticated_sender = NULL; if (!regex_AUTH) - regex_AUTH = regex_must_compile(AUTHS_REGEX, FALSE, TRUE); + regex_AUTH = regex_must_compile(AUTHS_REGEX, MCS_NOFLAGS, TRUE); /* Is the server offering AUTH? */ @@ -1806,57 +1857,65 @@ return Ustrcmp(current_local_identity, message_local_identity) == 0; static unsigned ehlo_response(uschar * buf, unsigned checks) { -size_t bsize = Ustrlen(buf); +PCRE2_SIZE bsize = Ustrlen(buf); +pcre2_match_data * md = pcre2_match_data_create(1, pcre_gen_ctx); /* debug_printf("%s: check for 0x%04x\n", __FUNCTION__, checks); */ #ifndef DISABLE_TLS if ( checks & OPTION_TLS - && pcre_exec(regex_STARTTLS, NULL, CS buf, bsize, 0, PCRE_EOPT, NULL, 0) < 0) + && pcre2_match(regex_STARTTLS, + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) #endif checks &= ~OPTION_TLS; if ( checks & OPTION_IGNQ - && pcre_exec(regex_IGNOREQUOTA, NULL, CS buf, bsize, 0, - PCRE_EOPT, NULL, 0) < 0) + && pcre2_match(regex_IGNOREQUOTA, + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) checks &= ~OPTION_IGNQ; if ( checks & OPTION_CHUNKING - && pcre_exec(regex_CHUNKING, NULL, CS buf, bsize, 0, PCRE_EOPT, NULL, 0) < 0) + && pcre2_match(regex_CHUNKING, + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) checks &= ~OPTION_CHUNKING; #ifndef DISABLE_PRDR if ( checks & OPTION_PRDR - && pcre_exec(regex_PRDR, NULL, CS buf, bsize, 0, PCRE_EOPT, NULL, 0) < 0) + && pcre2_match(regex_PRDR, + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) #endif checks &= ~OPTION_PRDR; #ifdef SUPPORT_I18N if ( checks & OPTION_UTF8 - && pcre_exec(regex_UTF8, NULL, CS buf, bsize, 0, PCRE_EOPT, NULL, 0) < 0) + && pcre2_match(regex_UTF8, + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) #endif checks &= ~OPTION_UTF8; if ( checks & OPTION_DSN - && pcre_exec(regex_DSN, NULL, CS buf, bsize, 0, PCRE_EOPT, NULL, 0) < 0) + && pcre2_match(regex_DSN, + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) checks &= ~OPTION_DSN; if ( checks & OPTION_PIPE - && pcre_exec(regex_PIPELINING, NULL, CS buf, bsize, 0, - PCRE_EOPT, NULL, 0) < 0) + && pcre2_match(regex_PIPELINING, + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) checks &= ~OPTION_PIPE; if ( checks & OPTION_SIZE - && pcre_exec(regex_SIZE, NULL, CS buf, bsize, 0, PCRE_EOPT, NULL, 0) < 0) + && pcre2_match(regex_SIZE, + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) checks &= ~OPTION_SIZE; #ifndef DISABLE_PIPE_CONNECT if ( checks & OPTION_EARLY_PIPE - && pcre_exec(regex_EARLY_PIPE, NULL, CS buf, bsize, 0, - PCRE_EOPT, NULL, 0) < 0) + && pcre2_match(regex_EARLY_PIPE, + (PCRE2_SPTR)buf, bsize, 0, PCRE_EOPT, md, pcre_gen_mtc_ctx) < 0) #endif checks &= ~OPTION_EARLY_PIPE; +/* pcre2_match_data_free(md); gen ctx needs no free */ /* debug_printf("%s: found 0x%04x\n", __FUNCTION__, checks); */ return checks; } @@ -1981,6 +2040,38 @@ return OK; +#ifdef SUPPORT_DANE +static int +check_force_dane_conn(smtp_context * sx, smtp_transport_options_block * ob) +{ +int rc; +if( sx->dane_required + || verify_check_given_host(CUSS &ob->hosts_try_dane, sx->conn_args.host) == OK + ) + switch (rc = tlsa_lookup(sx->conn_args.host, &sx->conn_args.tlsa_dnsa, sx->dane_required)) + { + case OK: sx->conn_args.dane = TRUE; + ob->tls_tempfail_tryclear = FALSE; /* force TLS */ + ob->tls_sni = sx->conn_args.host->name; /* force SNI */ + break; + case FAIL_FORCED: break; + default: set_errno_nohost(sx->addrlist, ERRNO_DNSDEFER, + string_sprintf("DANE error: tlsa lookup %s", + rc_to_string(rc)), + rc, FALSE, &sx->delivery_start); +# ifndef DISABLE_EVENT + (void) event_raise(sx->conn_args.tblock->event_action, + US"dane:fail", sx->dane_required + ? US"dane-required" : US"dnssec-invalid", + NULL); +# endif + return rc; + } +return OK; +} +#endif + + /************************************************* * Make connection for given message * *************************************************/ @@ -2011,44 +2102,31 @@ int yield = OK; uschar * tls_errstr; #endif +/* Many lines of clearing individual elements of *sx that used to +be here have been replaced by a full memset to zero (de41aff051). +There are two callers, this file and verify.c . Now we only set +up nonzero elements. */ + sx->conn_args.ob = ob; sx->lmtp = strcmpic(ob->protocol, US"lmtp") == 0; sx->smtps = strcmpic(ob->protocol, US"smtps") == 0; -/* sx->ok = FALSE; */ sx->send_rset = TRUE; sx->send_quit = TRUE; sx->setting_up = TRUE; sx->esmtp = TRUE; -/* sx->esmtp_sent = FALSE; */ -#ifdef SUPPORT_I18N -/* sx->utf8_needed = FALSE; */ -#endif sx->dsn_all_lasthop = TRUE; #ifdef SUPPORT_DANE -/* sx->conn_args.dane = FALSE; */ sx->dane_required = verify_check_given_host(CUSS &ob->hosts_require_dane, sx->conn_args.host) == OK; #endif -#ifndef DISABLE_PIPE_CONNECT -/* sx->early_pipe_active = sx->early_pipe_ok = FALSE; */ -/* sx->ehlo_resp.cleartext_features = sx->ehlo_resp.crypted_features = 0; */ -/* sx->pending_BANNER = sx->pending_EHLO = sx->pending_MAIL = FALSE; */ -#endif if ((sx->max_mail = sx->conn_args.tblock->connection_max_messages) == 0) sx->max_mail = 999999; if ((sx->max_rcpt = sx->conn_args.tblock->max_addresses) == 0) sx->max_rcpt = 999999; -/* sx->peer_offered = 0; */ -/* sx->avoid_option = 0; */ sx->igquotstr = US""; if (!sx->helo_data) sx->helo_data = ob->helo_data; -#ifdef EXPERIMENTAL_DSN_INFO -/* sx->smtp_greeting = NULL; */ -/* sx->helo_response = NULL; */ -#endif smtp_command = US"initial connection"; -/* sx->buffer[0] = '\0'; */ /* Set up the buffer for reading SMTP response packets. */ @@ -2062,9 +2140,6 @@ sx->inblock.ptrend = sx->inbuffer; sx->outblock.buffer = sx->outbuffer; sx->outblock.buffersize = sizeof(sx->outbuffer); sx->outblock.ptr = sx->outbuffer; -/* sx->outblock.cmd_count = 0; */ -/* sx->outblock.authenticating = FALSE; */ -/* sx->outblock.conn_args = NULL; */ /* Reset the parameters of a TLS session. */ @@ -2106,32 +2181,14 @@ if (continue_hostname && continue_proxy_cipher) const uschar * sni = US""; # ifdef SUPPORT_DANE - /* Check if the message will be DANE-verified; if so force its SNI */ + /* Check if the message will be DANE-verified; if so force TLS and its SNI */ tls_out.dane_verified = FALSE; smtp_port_for_connect(sx->conn_args.host, sx->port); if ( sx->conn_args.host->dnssec == DS_YES - && ( sx->dane_required - || verify_check_given_host(CUSS &ob->hosts_try_dane, sx->conn_args.host) == OK - ) ) - switch (rc = tlsa_lookup(sx->conn_args.host, &sx->conn_args.tlsa_dnsa, sx->dane_required)) - { - case OK: sx->conn_args.dane = TRUE; - ob->tls_tempfail_tryclear = FALSE; /* force TLS */ - ob->tls_sni = sx->first_addr->domain; /* force SNI */ - break; - case FAIL_FORCED: break; - default: set_errno_nohost(sx->addrlist, ERRNO_DNSDEFER, - string_sprintf("DANE error: tlsa lookup %s", - rc_to_string(rc)), - rc, FALSE, &sx->delivery_start); -# ifndef DISABLE_EVENT - (void) event_raise(sx->conn_args.tblock->event_action, - US"dane:fail", sx->dane_required - ? US"dane-required" : US"dnssec-invalid"); -# endif - return rc; - } + && (rc = check_force_dane_conn(sx, ob)) != OK + ) + return rc; # endif /* If the SNI or the DANE status required for the new message differs from the @@ -2159,7 +2216,7 @@ if (continue_hostname && continue_proxy_cipher) DEBUG(D_transport) debug_printf("Closing proxied-TLS connection due to SNI mismatch\n"); - HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP>> QUIT\n"); + smtp_debug_cmd(US"QUIT", 0); write(0, "QUIT\r\n", 6); close(0); continue_hostname = continue_proxy_cipher = NULL; @@ -2198,27 +2255,8 @@ if (!continue_hostname) if (sx->conn_args.host->dnssec == DS_YES) { int rc; - if( sx->dane_required - || verify_check_given_host(CUSS &ob->hosts_try_dane, sx->conn_args.host) == OK - ) - switch (rc = tlsa_lookup(sx->conn_args.host, &sx->conn_args.tlsa_dnsa, sx->dane_required)) - { - case OK: sx->conn_args.dane = TRUE; - ob->tls_tempfail_tryclear = FALSE; /* force TLS */ - ob->tls_sni = sx->first_addr->domain; /* force SNI */ - break; - case FAIL_FORCED: break; - default: set_errno_nohost(sx->addrlist, ERRNO_DNSDEFER, - string_sprintf("DANE error: tlsa lookup %s", - rc_to_string(rc)), - rc, FALSE, &sx->delivery_start); -# ifndef DISABLE_EVENT - (void) event_raise(sx->conn_args.tblock->event_action, - US"dane:fail", sx->dane_required - ? US"dane-required" : US"dnssec-invalid"); -# endif - return rc; - } + if ((rc = check_force_dane_conn(sx, ob)) != OK) + return rc; } else if (sx->dane_required) { @@ -2227,7 +2265,7 @@ if (!continue_hostname) FAIL, FALSE, &sx->delivery_start); # ifndef DISABLE_EVENT (void) event_raise(sx->conn_args.tblock->event_action, - US"dane:fail", US"dane-required"); + US"dane:fail", US"dane-required", NULL); # endif return FAIL; } @@ -2242,6 +2280,9 @@ if (!continue_hostname) sx->peer_limit_mail = sx->peer_limit_rcpt = sx->peer_limit_rcptdom = #endif sx->avoid_option = sx->peer_offered = smtp_peer_options = 0; +#ifndef DISABLE_CLIENT_CMD_LOG + client_cmd_log = NULL; +#endif #ifndef DISABLE_PIPE_CONNECT if ( verify_check_given_host(CUSS &ob->hosts_pipe_connect, @@ -2251,6 +2292,7 @@ if (!continue_hostname) the helo string might use it avoid doing early-pipelining. */ if ( !sx->helo_data + || sx->conn_args.interface || !Ustrstr(sx->helo_data, "$sending_ip_address") || Ustrstr(sx->helo_data, "def:sending_ip_address") ) @@ -2260,17 +2302,20 @@ if (!continue_hostname) && sx->ehlo_resp.cleartext_features & OPTION_EARLY_PIPE) { DEBUG(D_transport) - debug_printf("Using cached cleartext PIPE_CONNECT\n"); + debug_printf("Using cached cleartext PIPECONNECT\n"); sx->early_pipe_active = TRUE; sx->peer_offered = sx->ehlo_resp.cleartext_features; } } else DEBUG(D_transport) - debug_printf("helo needs $sending_ip_address\n"); + debug_printf("helo needs $sending_ip_address; avoid early-pipelining\n"); PIPE_CONNECT_RETRY: if (sx->early_pipe_active) + { sx->outblock.conn_args = &sx->conn_args; + (void) smtp_boundsock(&sx->conn_args); + } else #endif { @@ -2295,9 +2340,10 @@ PIPE_CONNECT_RETRY: } /* Expand the greeting message while waiting for the initial response. (Makes sense if helo_data contains ${lookup dnsdb ...} stuff). The expansion is - delayed till here so that $sending_interface and $sending_port are set. */ -/*XXX early-pipe: they still will not be. Is there any way to find out what they -will be? Somehow I doubt it. */ + delayed till here so that $sending_ip_address and $sending_port are set. + Those will be known even for a TFO lazy-connect, having been set by the bind(). + For early-pipe, we are ok if binding to a local interface; otherwise (if + $sending_ip_address is seen in helo_data) we disabled early-pipe above. */ if (sx->helo_data) if (!(sx->helo_data = expand_string(sx->helo_data))) @@ -2346,7 +2392,7 @@ will be? Somehow I doubt it. */ uschar * s; lookup_dnssec_authenticated = sx->conn_args.host->dnssec==DS_YES ? US"yes" : sx->conn_args.host->dnssec==DS_NO ? US"no" : NULL; - s = event_raise(sx->conn_args.tblock->event_action, US"smtp:connect", sx->buffer); + s = event_raise(sx->conn_args.tblock->event_action, US"smtp:connect", sx->buffer, NULL); if (s) { set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, @@ -2475,8 +2521,8 @@ goto SEND_QUIT; int n = sizeof(sx->buffer); uschar * rsp = sx->buffer; - if (sx->esmtp_sent && (n = Ustrlen(sx->buffer)) < sizeof(sx->buffer)/2) - { rsp = sx->buffer + n + 1; n = sizeof(sx->buffer) - n; } + if (sx->esmtp_sent && (n = Ustrlen(sx->buffer) + 1) < sizeof(sx->buffer)/2) + { rsp = sx->buffer + n; n = sizeof(sx->buffer) - n; } if (smtp_write_command(sx, SCMD_FLUSH, "HELO %s\r\n", sx->helo_data) < 0) goto SEND_FAILED; @@ -2534,12 +2580,13 @@ goto SEND_QUIT; if ( (sx->peer_offered & (OPTION_PIPE | OPTION_EARLY_PIPE)) == (OPTION_PIPE | OPTION_EARLY_PIPE)) { - DEBUG(D_transport) debug_printf("PIPE_CONNECT usable in future for this IP\n"); + DEBUG(D_transport) debug_printf("PIPECONNECT usable in future for this IP\n"); sx->ehlo_resp.cleartext_auths = study_ehlo_auths(sx); write_ehlo_cache_entry(sx); } } #endif + ehlo_response_lbserver(sx, ob); } /* Set tls_offered if the response to EHLO specifies support for STARTTLS. */ @@ -2631,14 +2678,17 @@ if ( smtp_peer_options & OPTION_TLS the response for the STARTTLS we just sent alone. On fail, assume wrong cached capability and retry with the pipelining disabled. */ - if (sx->early_pipe_active && sync_responses(sx, 2, 0) != 0) + if (sx->early_pipe_active) { - HDEBUG(D_transport) - debug_printf("failed reaping pipelined cmd responses\n"); - close(sx->cctx.sock); - sx->cctx.sock = -1; - sx->early_pipe_active = FALSE; - goto PIPE_CONNECT_RETRY; + if (sync_responses(sx, 2, 0) != 0) + { + HDEBUG(D_transport) + debug_printf("failed reaping pipelined cmd responses\n"); + close(sx->cctx.sock); + sx->cctx.sock = -1; + sx->early_pipe_active = FALSE; + goto PIPE_CONNECT_RETRY; + } } #endif @@ -2667,6 +2717,7 @@ if ( smtp_peer_options & OPTION_TLS else TLS_NEGOTIATE: { + sx->conn_args.sending_ip_address = sending_ip_address; if (!tls_client_start(&sx->cctx, &sx->conn_args, sx->addrlist, &tls_out, &tls_errstr)) { /* TLS negotiation failed; give an error. From outside, this function may @@ -2683,7 +2734,7 @@ if ( smtp_peer_options & OPTION_TLS sx->conn_args.host->name, sx->conn_args.host->address, tls_errstr); # ifndef DISABLE_EVENT (void) event_raise(sx->conn_args.tblock->event_action, - US"dane:fail", US"validation-failure"); /* could do with better detail */ + US"dane:fail", US"validation-failure", NULL); /* could do with better detail */ # endif } # endif @@ -2762,7 +2813,7 @@ if (tls_out.active.sock >= 0) sx->peer_offered = sx->ehlo_resp.crypted_features; if ((sx->early_pipe_active = !!(sx->ehlo_resp.crypted_features & OPTION_EARLY_PIPE))) - DEBUG(D_transport) debug_printf("Using cached crypted PIPE_CONNECT\n"); + DEBUG(D_transport) debug_printf("Using cached crypted PIPECONNECT\n"); } #endif #ifdef EXPERIMMENTAL_ESMTP_LIMITS @@ -2851,7 +2902,8 @@ else if ( sx->smtps (void) event_raise(sx->conn_args.tblock->event_action, US"dane:fail", smtp_peer_options & OPTION_TLS ? US"validation-failure" /* could do with better detail */ - : US"starttls-not-supported"); + : US"starttls-not-supported", + NULL); # endif goto TLS_FAILED; } @@ -2933,18 +2985,18 @@ if ( !continue_hostname smtp_peer_options & OPTION_PIPE ? "" : "not "); if ( sx->peer_offered & OPTION_CHUNKING - && verify_check_given_host(CUSS &ob->hosts_try_chunking, sx->conn_args.host) != OK) - sx->peer_offered &= ~OPTION_CHUNKING; + && verify_check_given_host(CUSS &ob->hosts_try_chunking, sx->conn_args.host) == OK) + smtp_peer_options |= OPTION_CHUNKING; - if (sx->peer_offered & OPTION_CHUNKING) + if (smtp_peer_options & OPTION_CHUNKING) DEBUG(D_transport) debug_printf("CHUNKING usable\n"); #ifndef DISABLE_PRDR if ( sx->peer_offered & OPTION_PRDR - && verify_check_given_host(CUSS &ob->hosts_try_prdr, sx->conn_args.host) != OK) - sx->peer_offered &= ~OPTION_PRDR; + && verify_check_given_host(CUSS &ob->hosts_try_prdr, sx->conn_args.host) == OK) + smtp_peer_options |= OPTION_PRDR; - if (sx->peer_offered & OPTION_PRDR) + if (smtp_peer_options & OPTION_PRDR) DEBUG(D_transport) debug_printf("PRDR usable\n"); #endif @@ -2961,7 +3013,7 @@ if ( !continue_hostname && ( sx->ehlo_resp.cleartext_features | sx->ehlo_resp.crypted_features) & OPTION_EARLY_PIPE) { - DEBUG(D_transport) debug_printf("PIPE_CONNECT usable in future for this IP\n"); + DEBUG(D_transport) debug_printf("PIPECONNECT usable in future for this IP\n"); sx->ehlo_resp.crypted_auths = study_ehlo_auths(sx); write_ehlo_cache_entry(sx); } @@ -3076,7 +3128,7 @@ return OK; SEND_FAILED: code = '4'; - message = US string_sprintf("send() to %s [%s] failed: %s", + message = US string_sprintf("smtp send to %s [%s] failed: %s", sx->conn_args.host->name, sx->conn_args.host->address, strerror(errno)); sx->send_quit = FALSE; yield = DEFER; @@ -3169,9 +3221,10 @@ if (sx->send_quit) sx->cctx.sock = -1; #ifndef DISABLE_EVENT -(void) event_raise(sx->conn_args.tblock->event_action, US"tcp:close", NULL); +(void) event_raise(sx->conn_args.tblock->event_action, US"tcp:close", NULL, NULL); #endif +smtp_debug_cmd_report(); continue_transport = NULL; continue_hostname = NULL; return yield; @@ -3214,7 +3267,7 @@ Or just forget about lines? Or inflate by a fixed proportion? */ request that */ sx->prdr_active = FALSE; -if (sx->peer_offered & OPTION_PRDR) +if (smtp_peer_options & OPTION_PRDR) for (address_item * addr = addrlist; addr; addr = addr->next) if (addr->transport_return == PENDING_DEFER) { @@ -3539,16 +3592,17 @@ Arguments: bufsiz size of buffer pfd pipe filedescriptor array; [0] is comms to proxied process timeout per-read timeout, seconds + host hostname of remote Does not return. */ void smtp_proxy_tls(void * ct_ctx, uschar * buf, size_t bsize, int * pfd, - int timeout) + int timeout, const uschar * host) { -fd_set rfds, efds; -int max_fd = MAX(pfd[0], tls_out.active.sock) + 1; +struct pollfd p[2] = {{.fd = tls_out.active.sock, .events = POLLIN}, + {.fd = pfd[0], .events = POLLIN}}; int rc, i; BOOL send_tls_shutdown = TRUE; @@ -3556,24 +3610,17 @@ close(pfd[1]); if ((rc = exim_fork(US"tls-proxy"))) _exit(rc < 0 ? EXIT_FAILURE : EXIT_SUCCESS); -set_process_info("proxying TLS connection for continued transport"); -FD_ZERO(&rfds); -FD_SET(tls_out.active.sock, &rfds); -FD_SET(pfd[0], &rfds); +set_process_info("proxying TLS connection for continued transport to %s\n", host); -for (int fd_bits = 3; fd_bits; ) +do { time_t time_left = timeout; time_t time_start = time(NULL); /* wait for data */ - efds = rfds; do { - struct timeval tv = { time_left, 0 }; - - rc = select(max_fd, - (SELECT_ARG2_TYPE *)&rfds, NULL, (SELECT_ARG2_TYPE *)&efds, &tv); + rc = poll(p, 2, time_left * 1000); if (rc < 0 && errno == EINTR) if ((time_left -= time(NULL) - time_start) > 0) continue; @@ -3586,39 +3633,36 @@ for (int fd_bits = 3; fd_bits; ) /* For errors where not readable, bomb out */ - if (FD_ISSET(tls_out.active.sock, &efds) || FD_ISSET(pfd[0], &efds)) + if (p[0].revents & POLLERR || p[1].revents & POLLERR) { DEBUG(D_transport) debug_printf("select: exceptional cond on %s fd\n", - FD_ISSET(pfd[0], &efds) ? "proxy" : "tls"); - if (!(FD_ISSET(tls_out.active.sock, &rfds) || FD_ISSET(pfd[0], &rfds))) + p[0].revents & POLLERR ? "tls" : "proxy"); + if (!(p[0].revents & POLLIN || p[1].events & POLLIN)) goto done; DEBUG(D_transport) debug_printf("- but also readable; no exit yet\n"); } } - while (rc < 0 || !(FD_ISSET(tls_out.active.sock, &rfds) || FD_ISSET(pfd[0], &rfds))); + while (rc < 0 || !(p[0].revents & POLLIN || p[1].revents & POLLIN)); /* handle inbound data */ - if (FD_ISSET(tls_out.active.sock, &rfds)) + if (p[0].revents & POLLIN) if ((rc = tls_read(ct_ctx, buf, bsize)) <= 0) /* Expect -1 for EOF; */ { /* that reaps the TLS Close Notify record */ - fd_bits &= ~1; - FD_CLR(tls_out.active.sock, &rfds); + p[0].fd = -1; shutdown(pfd[0], SHUT_WR); timeout = 5; } else - { for (int nbytes = 0; rc - nbytes > 0; nbytes += i) if ((i = write(pfd[0], buf + nbytes, rc - nbytes)) < 0) goto done; - } /* Handle outbound data. We cannot combine payload and the TLS-close - due to the limitations of the (pipe) channel feeding us. */ - if (FD_ISSET(pfd[0], &rfds)) + due to the limitations of the (pipe) channel feeding us. Maybe use a unix-domain + socket? */ + if (p[1].revents & POLLIN) if ((rc = read(pfd[0], buf, bsize)) <= 0) { - fd_bits &= ~2; - FD_CLR(pfd[0], &rfds); + p[1].fd = -1; # ifdef EXIM_TCP_CORK /* Use _CORK to get TLS Close Notify in FIN segment */ (void) setsockopt(tls_out.active.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on)); @@ -3628,15 +3672,11 @@ for (int fd_bits = 3; fd_bits; ) shutdown(tls_out.active.sock, SHUT_WR); } else - { for (int nbytes = 0; rc - nbytes > 0; nbytes += i) if ((i = tls_write(ct_ctx, buf + nbytes, rc - nbytes, FALSE)) < 0) goto done; - } - - if (fd_bits & 1) FD_SET(tls_out.active.sock, &rfds); - if (fd_bits & 2) FD_SET(pfd[0], &rfds); } +while (p[0].fd >= 0 || p[1].fd >= 0); done: if (send_tls_shutdown) tls_close(ct_ctx, TLS_SHUTDOWN_NOWAIT); @@ -3708,7 +3748,7 @@ int rc; uschar *message = NULL; uschar new_message_id[MESSAGE_ID_LENGTH + 1]; -smtp_context * sx = store_get(sizeof(*sx), TRUE); /* tainted, for the data buffers */ +smtp_context * sx = store_get(sizeof(*sx), GET_TAINTED); /* tainted, for the data buffers */ BOOL pass_message = FALSE; #ifdef EXPERIMENTAL_ESMTP_LIMITS BOOL mail_limit = FALSE; @@ -3723,12 +3763,12 @@ BOOL tcw_done = FALSE, tcw = FALSE; memset(sx, 0, sizeof(*sx)); sx->addrlist = addrlist; sx->conn_args.host = host; -sx->conn_args.host_af = host_af, +sx->conn_args.host_af = host_af; sx->port = defport; sx->conn_args.interface = interface; sx->helo_data = NULL; sx->conn_args.tblock = tblock; -/* sx->verify = FALSE; */ +sx->conn_args.sock = -1; gettimeofday(&sx->delivery_start, NULL); sx->sync_addr = sx->first_addr = addrlist; @@ -3778,8 +3818,8 @@ if (tblock->filter_command) yield ERROR. */ if (!transport_set_up_command(&transport_filter_argv, - tblock->filter_command, TRUE, DEFER, addrlist, - string_sprintf("%.50s transport", tblock->name), NULL)) + tblock->filter_command, TRUE, DEFER, addrlist, FALSE, + string_sprintf("%.50s transport filter", tblock->name), NULL)) { set_errno_nohost(addrlist->next, addrlist->basic_errno, addrlist->message, DEFER, FALSE, &sx->delivery_start); @@ -3790,7 +3830,7 @@ if (tblock->filter_command) if ( transport_filter_argv && *transport_filter_argv && **transport_filter_argv - && sx->peer_offered & OPTION_CHUNKING + && smtp_peer_options & OPTION_CHUNKING #ifndef DISABLE_DKIM /* When dkim signing, chunking is handled even with a transport-filter */ && !(ob->dkim.dkim_private_key && ob->dkim.dkim_domain && ob->dkim.dkim_selector) @@ -3798,7 +3838,7 @@ if (tblock->filter_command) #endif ) { - sx->peer_offered &= ~OPTION_CHUNKING; + smtp_peer_options &= ~OPTION_CHUNKING; DEBUG(D_transport) debug_printf("CHUNKING not usable due to transport filter\n"); } } @@ -3875,7 +3915,7 @@ are pipelining. The responses are all handled by sync_responses(). If using CHUNKING, do not send a BDAT until we know how big a chunk we want to send is. */ -if ( !(sx->peer_offered & OPTION_CHUNKING) +if ( !(smtp_peer_options & OPTION_CHUNKING) && (sx->ok || (pipelining_active && !mua_wrapper))) { int count = smtp_write_command(sx, SCMD_FLUSH, "DATA\r\n"); @@ -3912,7 +3952,7 @@ well as body. Set the appropriate timeout value to be used for each chunk. (Haven't been able to make it work using select() for writing yet.) */ if ( !sx->ok - && (!(sx->peer_offered & OPTION_CHUNKING) || !pipelining_active)) + && (!(smtp_peer_options & OPTION_CHUNKING) || !pipelining_active)) { /* Save the first address of the next batch. */ sx->first_addr = sx->next_addr; @@ -3941,7 +3981,7 @@ else of responses. The callback needs a whole bunch of state so set up a transport-context structure to be passed around. */ - if (sx->peer_offered & OPTION_CHUNKING) + if (smtp_peer_options & OPTION_CHUNKING) { tctx.check_string = tctx.escape_string = NULL; tctx.options |= topt_use_bdat; @@ -3966,10 +4006,10 @@ else transport_write_timeout = ob->data_timeout; smtp_command = US"sending data block"; /* For error messages */ DEBUG(D_transport|D_v) - if (sx->peer_offered & OPTION_CHUNKING) + if (smtp_peer_options & OPTION_CHUNKING) debug_printf(" will write message using CHUNKING\n"); else - debug_printf(" SMTP>> writing message and terminating \".\"\n"); + debug_printf(" SMTP>> (writing message)\n"); transport_count = 0; #ifndef DISABLE_DKIM @@ -4018,7 +4058,7 @@ else If we can, we want the message-write to not flush (the tail end of) its data out. */ if ( sx->pipelining_used - && (sx->ok && sx->completed_addr || sx->peer_offered & OPTION_CHUNKING) + && (sx->ok && sx->completed_addr || smtp_peer_options & OPTION_CHUNKING) && sx->send_quit && !(sx->first_addr || f.continue_more) && f.deliver_firsttime @@ -4102,7 +4142,7 @@ else sx->send_quit = FALSE; /* avoid sending it later */ #ifndef DISABLE_TLS - if (sx->cctx.tls_ctx) /* need to send TLS Cloe Notify */ + if (sx->cctx.tls_ctx && sx->send_tlsclose) /* need to send TLS Close Notify */ { # ifdef EXIM_TCP_CORK /* Use _CORK to get Close Notify in FIN segment */ (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on)); @@ -4116,7 +4156,7 @@ else } } - if (sx->peer_offered & OPTION_CHUNKING && sx->cmd_count > 1) + if (smtp_peer_options & OPTION_CHUNKING && sx->cmd_count > 1) { /* Reap any outstanding MAIL & RCPT commands, but not a DATA-go-ahead */ switch(sync_responses(sx, sx->cmd_count-1, 0)) @@ -4289,7 +4329,7 @@ else #ifndef DISABLE_PRDR if (sx->prdr_active) setflag(addr, af_prdr_used); #endif - if (sx->peer_offered & OPTION_CHUNKING) setflag(addr, af_chunking_used); + if (smtp_peer_options & OPTION_CHUNKING) setflag(addr, af_chunking_used); flag = '-'; #ifndef DISABLE_PRDR @@ -4360,6 +4400,7 @@ else "%s: %s", sx->buffer, strerror(errno)); } else if (addr->transport_return == DEFER) + /*XXX magic value -2 ? maybe host+message ? */ retry_add_item(addr, addr->address_retry_key, -2); } #endif @@ -4400,7 +4441,7 @@ if (!sx->ok) { save_errno = errno; code = '4'; - message = string_sprintf("send() to %s [%s] failed: %s", + message = string_sprintf("smtp send to %s [%s] failed: %s", host->name, host->address, message ? message : US strerror(save_errno)); sx->send_quit = FALSE; goto FAILED; @@ -4436,6 +4477,22 @@ if (!sx->ok) message_error = Ustrncmp(smtp_command,"end ",4) == 0; break; +#ifndef DISABLE_DKIM + case EACCES: + /* DKIM signing failure: avoid thinking we pipelined quit, + just abandon the message and close the socket. */ + + message_error = FALSE; +# ifndef DISABLE_TLS + if (sx->cctx.tls_ctx) + { + tls_close(sx->cctx.tls_ctx, + sx->send_tlsclose ? TLS_SHUTDOWN_WAIT : TLS_SHUTDOWN_WONLY); + sx->cctx.tls_ctx = NULL; + } +# endif + break; +#endif default: message_error = FALSE; break; @@ -4584,7 +4641,7 @@ if (sx->completed_addr && sx->ok && sx->send_quit) if (sx->send_rset) if (! (sx->ok = smtp_write_command(sx, SCMD_FLUSH, "RSET\r\n") >= 0)) { - msg = US string_sprintf("send() to %s [%s] failed: %s", host->name, + msg = US string_sprintf("smtp send to %s [%s] failed: %s", host->name, host->address, strerror(errno)); sx->send_quit = FALSE; } @@ -4639,9 +4696,13 @@ if (sx->completed_addr && sx->ok && sx->send_quit) open, we must shut down TLS. Not all MTAs allow for the continuation of the SMTP session when TLS is shut down. We test for this by sending a new EHLO. If we don't get a good response, we don't attempt to pass - the socket on. */ + the socket on. + NB: TLS close is *required* per RFC 9266 when tls-exporter info has + been used, which we do under TLSv1.3 for the gsasl SCRAM*PLUS methods. + But we were always doing it anyway. */ - tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_WAIT); + tls_close(sx->cctx.tls_ctx, + sx->send_tlsclose ? TLS_SHUTDOWN_WAIT : TLS_SHUTDOWN_WONLY); sx->send_tlsclose = FALSE; sx->cctx.tls_ctx = NULL; tls_out.active.sock = -1; @@ -4703,7 +4764,7 @@ if (sx->completed_addr && sx->ok && sx->send_quit) { /* does not return */ smtp_proxy_tls(sx->cctx.tls_ctx, sx->buffer, sizeof(sx->buffer), pfd, - ob->command_timeout); + ob->command_timeout, host->name); } if (pid > 0) /* parent */ @@ -4743,7 +4804,7 @@ if (sx->send_quit) { /* Use _MORE to get QUIT in FIN segment */ (void)smtp_write_command(sx, SCMD_MORE, "QUIT\r\n"); #ifndef DISABLE_TLS - if (sx->cctx.tls_ctx) + if (sx->cctx.tls_ctx && sx->send_tlsclose) { # ifdef EXIM_TCP_CORK /* Use _CORK to get TLS Close Notify in FIN segment */ (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on)); @@ -4798,15 +4859,23 @@ if (sx->send_quit || tcw_done && !tcw) while (!sigalrm_seen && n > 0); ALARM_CLR(0); + if (sx->send_tlsclose) + { # ifdef EXIM_TCP_CORK - (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on)); + (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on)); # endif - tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_WAIT); + tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_WAIT); + } + else + tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_WONLY); sx->cctx.tls_ctx = NULL; } #endif - millisleep(20); - if (fcntl(sx->cctx.sock, F_SETFL, O_NONBLOCK) == 0) + + /* Drain any trailing data from the socket before close, to avoid sending a RST */ + + if ( poll_one_fd(sx->cctx.sock, POLLIN, 20) != 0 /* 20ms */ + && fcntl(sx->cctx.sock, F_SETFL, O_NONBLOCK) == 0) for (int i = 16, n; /* drain socket */ (n = read(sx->cctx.sock, sx->inbuffer, sizeof(sx->inbuffer))) > 0 && i > 0; i--) HDEBUG(D_transport|D_acl|D_v) @@ -4822,9 +4891,10 @@ HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(close)>>\n"); sx->cctx.sock = -1; continue_transport = NULL; continue_hostname = NULL; +smtp_debug_cmd_report(); #ifndef DISABLE_EVENT -(void) event_raise(tblock->event_action, US"tcp:close", NULL); +(void) event_raise(tblock->event_action, US"tcp:close", NULL, NULL); #endif #ifdef SUPPORT_DANE @@ -5246,6 +5316,17 @@ retry_non_continued: uschar *retry_message_key = NULL; uschar *serialize_key = NULL; + /* Deal slightly better with a possible Linux kernel bug that results + in intermittent TFO-conn fails deep into the TCP flow. Bug 2907 tracks. + Hack: Clear TFO option for any further hosts on this tpt run. */ + + if (total_hosts_tried > 0) + { + DEBUG(D_transport|D_acl|D_v) + debug_printf("Clearing TFO as not first host for message\n"); + ob->hosts_try_fastopen = US""; + } + /* Default next host is next host. :-) But this can vary if the hosts_max_try limit is hit (see below). It may also be reset if a host address is looked up here (in case the host was multihomed). */ @@ -5596,7 +5677,7 @@ retry_non_continued: if (expanded_hosts) { - thost = store_get(sizeof(host_item), FALSE); + thost = store_get(sizeof(host_item), GET_UNTAINTED); *thost = *host; thost->name = string_copy(host->name); thost->address = string_copy(host->address);