#if defined(TCP_FASTOPEN) && !defined(MSG_FASTOPEN)
# define MSG_FASTOPEN 0x20000000
#endif
+#define EXIM_HAVE_TCPI_UNACKED
/* End */
break;
case 'T':
- setflag(addr, af_tcp_fastopen);
+ setflag(addr, af_tcp_fastopen_conn);
+ if (*subid > '0') setflag(addr, af_tcp_fastopen);
break;
case 'D':
if (testflag(addr, af_chunking_used))
rmt_dlv_checked_write(fd, 'K', '0', NULL, 0);
- if (testflag(addr, af_tcp_fastopen))
- rmt_dlv_checked_write(fd, 'T', '0', NULL, 0);
+ if (testflag(addr, af_tcp_fastopen_conn))
+ rmt_dlv_checked_write(fd, 'T',
+ testflag(addr, af_tcp_fastopen) ? '1' : '0', NULL, 0);
memcpy(big_buffer, &addr->dsn_aware, sizeof(addr->dsn_aware));
rmt_dlv_checked_write(fd, 'D', '0', big_buffer, sizeof(addr->dsn_aware));
BOOL tcp_in_fastopen = FALSE;
BOOL tcp_in_fastopen_logged = FALSE;
BOOL tcp_nodelay = TRUE;
-BOOL tcp_out_fastopen = FALSE;
+int tcp_out_fastopen = 0;
BOOL tcp_out_fastopen_logged= FALSE;
#ifdef USE_TCP_WRAPPERS
uschar *tcp_wrappers_daemon_name = US TCP_WRAPPERS_DAEMON_NAME;
extern BOOL tcp_fastopen_ok; /* appears to be supported by kernel */
extern blob tcp_fastopen_nodata; /* for zero-data TFO connect requests */
-extern BOOL tcp_in_fastopen; /* conn used fastopen */
+extern BOOL tcp_in_fastopen; /* conn usefully used fastopen */
extern BOOL tcp_in_fastopen_logged; /* one-time logging */
extern BOOL tcp_nodelay; /* Controls TCP_NODELAY on daemon */
-extern BOOL tcp_out_fastopen; /* conn used fastopen */
+extern int tcp_out_fastopen; /* 0: no 1: conn used 2: useful */
extern BOOL tcp_out_fastopen_logged; /* one-time logging */
#ifdef USE_TCP_WRAPPERS
extern uschar *tcp_wrappers_daemon_name; /* tcpwrappers daemon lookup name */
{
if ((rc = sendto(sock, fastopen->data, fastopen->len,
MSG_FASTOPEN | MSG_DONTWAIT, s_ptr, s_len)) >= 0)
+ /* seen for with-data, experimental TFO option, with-cookie case */
+ /* seen for with-data, proper TFO opt, with-cookie case */
{
- DEBUG(D_transport|D_v)
- debug_printf("TCP_FASTOPEN mode connection, with data\n");
- tcp_out_fastopen = TRUE;
+ DEBUG(D_transport|D_v) debug_printf("TFO mode connection attempt, %s data\n",
+ fastopen->len > 0 ? "with" : "no");
+ tcp_out_fastopen = fastopen->len > 0 ? 2 : 1;
}
- else if (errno == EINPROGRESS) /* expected for nonready peer */
- {
+ else if (errno == EINPROGRESS) /* expected if we had no cookie for peer */
+ /* seen for no-data, proper TFO option, both cookie-request and with-cookie cases */
+ /* apparently no visibility of the diffference at this point */
+ /* seen for with-data, proper TFO opt, cookie-req */
+ /* with netwk delay, post-conn tcp_info sees unacked 1 for R, 2 for C; code in smtp_out.c */
+ /* ? older Experimental TFO option behaviour ? */
+ { /* queue unsent data */
+ DEBUG(D_transport|D_v) debug_printf("TFO mode sendto, %s data: EINPROGRESS\n",
+ fastopen->len > 0 ? "with" : "no");
if (!fastopen->data)
{
- DEBUG(D_transport|D_v)
- debug_printf("TCP_FASTOPEN mode connection, no data\n");
- tcp_out_fastopen = TRUE;
+ tcp_out_fastopen = 1; /* we tried; unknown if useful yet */
rc = 0;
}
- else if ( (rc = send(sock, fastopen->data, fastopen->len, 0)) < 0
- && errno == EINPROGRESS) /* expected for nonready peer */
- rc = 0;
+ else
+ rc = send(sock, fastopen->data, fastopen->len, 0);
}
else if(errno == EOPNOTSUPP)
{
&& tinfo.tcpi_state == TCP_SYN_RECV
)
{
- DEBUG(D_receive) debug_printf("TCP_FASTOPEN mode connection\n");
+ DEBUG(D_receive) debug_printf("TCP_FASTOPEN mode connection (state TCP_SYN_RECV)\n");
tcp_in_fastopen = TRUE;
}
# endif
+#ifdef TCP_FASTOPEN
+static void
+tfo_out_check(int sock)
+{
+# if defined(TCP_INFO) && defined(EXIM_HAVE_TCPI_UNACKED)
+struct tcp_info tinfo;
+socklen_t len = sizeof(tinfo);
+
+if (getsockopt(sock, IPPROTO_TCP, TCP_INFO, &tinfo, &len) == 0)
+ {
+ switch (tcp_out_fastopen)
+ {
+ /* This is a somewhat dubious detection method; totally undocumented so likely
+ to fail in future kernels. There seems to be no documented way. What we really
+ want to know is if the server sent smtp-banner data before our ACK of his SYN,ACK
+ hit him. What this (possibly?) detects is whether we sent a TFO cookie with our
+ SYN, as distinct from a TFO request. This gets a false-positive when the server
+ key is rotated; we send the old one (which this test sees) but the server returns
+ the new one and does not send its SMTP banner before we ACK his SYN,ACK.
+ To force that rotation case:
+ '# echo -n "00000000-00000000-00000000-0000000" >/proc/sys/net/ipv4/tcp_fastopen_key'
+ The kernel seems to be counting unack'd packets. */
+
+ case 1:
+ if (tinfo.tcpi_unacked > 1)
+ {
+ DEBUG(D_transport|D_v)
+ debug_printf("TCP_FASTOPEN tcpi_unacked %d\n", tinfo.tcpi_unacked);
+ tcp_out_fastopen = 2;
+ }
+ break;
+
+#ifdef notdef /* This seems to always fire, meaning that we cannot tell
+ whether the server accepted data we sent. For now assume
+ that it did. */
+
+ /* If there was data-on-SYN but we had to retrasnmit it, declare no TFO */
+
+ case 2:
+ if (!(tinfo.tcpi_options & TCPI_OPT_SYN_DATA))
+ {
+ DEBUG(D_transport|D_v) debug_printf("TFO: had to retransmit\n");
+ tcp_out_fastopen = 0;
+ }
+ break;
+#endif
+ }
+
+ }
+# endif
+}
+#endif
+
+
/* Arguments as for smtp_connect(), plus
early_data if non-NULL, data to be sent - preferably in the TCP SYN segment
int dscp_option;
int sock;
int save_errno = 0;
+const blob * fastopen = NULL;
+
#ifndef DISABLE_EVENT
deliver_host_address = host->address;
else
{
- const blob * fastopen = NULL;
-
#ifdef TCP_FASTOPEN
if (verify_check_given_host(&ob->hosts_try_fastopen, host) == OK)
fastopen = early_data ? early_data : &tcp_fastopen_nodata;
return -1;
}
if (ob->keepalive) ip_keepalive(sock, host->address, TRUE);
+#ifdef TCP_FASTOPEN
+ if (fastopen) tfo_out_check(sock);
+#endif
return sock;
}
}
BOOL af_cert_verified:1; /* delivered with verified TLS cert */
BOOL af_pass_message:1; /* pass message in bounces */
BOOL af_bad_reply:1; /* filter could not generate autoreply */
- BOOL af_tcp_fastopen:1; /* delivery used TCP Fast Open */
+ BOOL af_tcp_fastopen_conn:1; /* delivery connection used TCP Fast Open */
+ BOOL af_tcp_fastopen:1; /* delivery usefuly used TCP Fast Open */
#ifndef DISABLE_PRDR
BOOL af_prdr_used:1; /* delivery used SMTP PRDR */
#endif
uschar * rcpt_addr;
if (tcp_out_fastopen && !tcp_out_fastopen_logged)
- setflag(addr, af_tcp_fastopen);
+ {
+ setflag(addr, af_tcp_fastopen_conn);
+ if (tcp_out_fastopen > 1) setflag(addr, af_tcp_fastopen);
+ }
addr->dsn_aware = sx->peer_offered & OPTION_DSN
? dsn_support_yes : dsn_support_no;
for (i = 0; i<len; i++) debug_printf(" %02x", s[i]);
debug_printf("\n");
}
- if ( send(fd, s, len, 0) < 0
- || !fd_ready(fd, tmo-time(NULL))
- || read(fd, s, 2) != 2
- )
+ if (send(fd, s, len, 0) < 0)
+ return FAIL;
+#ifdef TCP_QUICKACK
+ (void) setsockopt(fd, IPPROTO_TCP, TCP_QUICKACK, US &off, sizeof(off));
+#endif
+ if (!fd_ready(fd, tmo-time(NULL)) || read(fd, s, 2) != 2)
return FAIL;
HDEBUG(D_transport|D_acl|D_v)
debug_printf_indent(" SOCKS<< %02x %02x\n", s[0], s[1]);
/* expect method response */
+#ifdef TCP_QUICKACK
+(void) setsockopt(fd, IPPROTO_TCP, TCP_QUICKACK, US &off, sizeof(off));
+#endif
+
if ( !fd_ready(fd, tmo-time(NULL))
|| read(fd, buf, 2) != 2
)
1999-03-02 09:44:33 10HmaX-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local-esmtp S=sss
-1999-03-02 09:44:33 10HmaX-0005vi-00 => user_tfo@test.ex R=my_main_router T=my_smtp H=127.0.0.1 [127.0.0.1]:1224 PRX=[127.0.0.1]:1225 TFO C="250 accepted OK"
+1999-03-02 09:44:33 10HmaX-0005vi-00 => user_tfo@test.ex R=my_main_router T=my_smtp H=127.0.0.1 [127.0.0.1]:1224 PRX=[127.0.0.1]:1225 C="250 accepted OK"
1999-03-02 09:44:33 10HmaX-0005vi-00 Completed
1999-03-02 09:44:33 10HmaY-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local-esmtp S=sss
1999-03-02 09:44:33 10HmaY-0005vi-00 => user_tfo@test.ex R=my_main_router T=my_smtp H=127.0.0.1 [127.0.0.1]:1224 PRX=[127.0.0.1]:1225 TFO C="250 accepted OK"
1999-03-02 09:44:33 10HmaY-0005vi-00 Completed
+1999-03-02 09:44:33 10HmaZ-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local-esmtp S=sss
+1999-03-02 09:44:33 10HmaZ-0005vi-00 => user_tfo@test.ex R=my_main_router T=my_smtp H=127.0.0.1 [127.0.0.1]:1224 PRX=[127.0.0.1]:1225 TFO C="250 accepted OK"
+1999-03-02 09:44:33 10HmaZ-0005vi-00 Completed
# A packet capture on the loopback interface will show the TFO
# option on the SYN, but the fast-output SMTP banner will not
# be seen unless you also deliberately emulate a long path:
-# 'sudo tc qdisc add dev lo root netem delay 100ms'
+# 'sudo tc qdisc add dev lo root netem delay 50ms'
+# You'll need kernel-modules-extra installed, or you get
+# an unhelpful error from RTNETLINK.
+# To tidy up: 'sudo tc qdisc delete dev lo root'
+#
+sudo perl
+system ("tc qdisc add dev lo root netem delay 50ms");
+****
#
# First time runs will see a TFO request option only; subsequent
# ones should see the TFO cookie and fast-output SMTP banner
# The client log => lint.ex should have a "TFO" element.
# Assuming this is the first run since boot, the a@test recipient will not.
#
+sudo perl
+system ("ip tcp_metrics delete 127.0.0.1");
+****
+#
# The server log <= line for b@test.ex should have a "TFO" element, but
# this will only be obtained when the above delay is inserted into the
# loopback net path.
****
sleep 3
#
+#
+sudo perl
+system ("tc qdisc delete dev lo root");
+****
+#
killdaemon
no_msglog_check
munge loopback
#
#
+# TFO both clients and server, no cookie yet
+sudo perl
+system ("ip tcp_metrics delete 127.0.0.1");
+****
+#
+server -tfo PORT_D
+<<\x05\x01\x00
+>>\x05\x00
+<<\x05\x01\x00\x01\x7f\x00\x00\x01\x04\xc8
+>>\x05\x00\x00\x01\x7f\x00\x00\x01\xbe\xef
+220 Connected OK
+EHLO
+250-server id
+250
+MAIL FROM
+250
+RCPT TO
+250
+DATA
+354 go ahead
+.
+250 accepted OK
+QUIT
+250 bye
+****
+#
+#
+exim -odi -bs -DOPT=
+ehlo test.ex
+mail from:<>
+rcpt to:<user_tfo@test.ex>
+data
+Date: Fri, 17 Dec 2004 14:35:01 +0100
+Subject: message should be sent
+
+connection trying TFO via proxy; no cookie yet
+.
+quit
+****
+#
+#
+#
# TFO client, not server
server PORT_D
<<\x05\x01\x00
quit
****
#
+millisleep 500
#
# Ends
354 Enter message, ending with "." on a line by itself\r
250 OK id=10HmaY-0005vi-00\r
221 myhost.test.ex closing connection\r
+220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000\r
+250-myhost.test.ex Hello CALLER at test.ex\r
+250-SIZE 52428800\r
+250-8BITMIME\r
+250-PIPELINING\r
+250 HELP\r
+250 OK\r
+250 Accepted\r
+354 Enter message, ending with "." on a line by itself\r
+250 OK id=10HmaZ-0005vi-00\r
+221 myhost.test.ex closing connection\r
******** SERVER ********
Listening on port 1225 ...
RCPT TO
250
DATA
+354 go ahead
+R
+250 accepted OK
+QUIT
+250 bye
+End of script
+Listening on port 1225 ...
+Connection request from [ip4.ip4.ip4.ip4]
+<<\x05\x01\x00
+>>\x05\x00
+<<\x05\x01\x00\x01\x7f\x00\x00\x01\x04\xc8
+>>\x05\x00\x00\x01\x7f\x00\x00\x01\xbe\xef
+220 Connected OK
+EHLO
+250-server id
+250
+MAIL FROM
+250
+RCPT TO
+250
+DATA
354 do me
R
250 accepted OK