TFO: better detection of client fast-open connections (again)

diff --git a/src/OS/os.h-Linux b/src/OS/os.h-Linux
index cc1f3cab2df2544983f3fc42aac851c107fe971b..f6d35772b73ddc666b5e694861d4f3c55994f586 100644 (file)
--- a/src/OS/os.h-Linux
+++ b/src/OS/os.h-Linux
@@ -79,6 +79,7 @@ then change the 0 to 1 in the next block. */
 #if defined(TCP_FASTOPEN) && !defined(MSG_FASTOPEN)
 # define MSG_FASTOPEN 0x20000000
 #endif
+#define EXIM_HAVE_TCPI_UNACKED
 
 
 /* End */

diff --git a/src/src/deliver.c b/src/src/deliver.c
index 73921980e263f4bb1ae94cce318398020c814fd9..648c63d691d6e977ceac9705a93dcbf619cb6d99 100644 (file)
--- a/src/src/deliver.c
+++ b/src/src/deliver.c
@@ -3535,7 +3535,8 @@ while (!done)
       break;
 
     case 'T':
-      setflag(addr, af_tcp_fastopen);
+      setflag(addr, af_tcp_fastopen_conn);
+      if (*subid > '0') setflag(addr, af_tcp_fastopen);
       break;
 
     case 'D':
@@ -4837,8 +4838,9 @@ all pipes, so I do not see a reason to use non-blocking IO here
       if (testflag(addr, af_chunking_used))
        rmt_dlv_checked_write(fd, 'K', '0', NULL, 0);
 
-      if (testflag(addr, af_tcp_fastopen))
-       rmt_dlv_checked_write(fd, 'T', '0', NULL, 0);
+      if (testflag(addr, af_tcp_fastopen_conn))
+       rmt_dlv_checked_write(fd, 'T',
+         testflag(addr, af_tcp_fastopen) ? '1' : '0', NULL, 0);
 
       memcpy(big_buffer, &addr->dsn_aware, sizeof(addr->dsn_aware));
       rmt_dlv_checked_write(fd, 'D', '0', big_buffer, sizeof(addr->dsn_aware));

diff --git a/src/src/globals.c b/src/src/globals.c
index 57041fc4e2b05d217832459dfe1475d83650134f..2a53835e9e8d8b3fbe65d6b9b6879b78dc7ce5d9 100644 (file)
--- a/src/src/globals.c
+++ b/src/src/globals.c
@@ -1421,7 +1421,7 @@ blob      tcp_fastopen_nodata    = { .data = NULL, .len = 0 };
 BOOL    tcp_in_fastopen        = FALSE;
 BOOL    tcp_in_fastopen_logged = FALSE;
 BOOL    tcp_nodelay            = TRUE;
-BOOL    tcp_out_fastopen       = FALSE;
+int     tcp_out_fastopen       = 0;
 BOOL    tcp_out_fastopen_logged= FALSE;
 #ifdef USE_TCP_WRAPPERS
 uschar *tcp_wrappers_daemon_name = US TCP_WRAPPERS_DAEMON_NAME;

diff --git a/src/src/globals.h b/src/src/globals.h
index 2957587b0c2b299c5c1878d2395530024c6dc801..62336c27553e042e8cfa05331e736aa1f36b540c 100644 (file)
--- a/src/src/globals.h
+++ b/src/src/globals.h
@@ -923,10 +923,10 @@ extern BOOL    system_filtering;       /* TRUE when running system filter */
 
 extern BOOL    tcp_fastopen_ok;               /* appears to be supported by kernel */
 extern blob    tcp_fastopen_nodata;    /* for zero-data TFO connect requests */
-extern BOOL    tcp_in_fastopen;               /* conn used fastopen */
+extern BOOL    tcp_in_fastopen;               /* conn usefully used fastopen */
 extern BOOL    tcp_in_fastopen_logged; /* one-time logging */
 extern BOOL    tcp_nodelay;            /* Controls TCP_NODELAY on daemon */
-extern BOOL    tcp_out_fastopen;       /* conn used fastopen */
+extern int     tcp_out_fastopen;       /* 0: no  1: conn used  2: useful */
 extern BOOL    tcp_out_fastopen_logged; /* one-time logging */
 #ifdef USE_TCP_WRAPPERS
 extern uschar *tcp_wrappers_daemon_name; /* tcpwrappers daemon lookup name */

diff --git a/src/src/ip.c b/src/src/ip.c
index 266eaf41484c9f9840a155b5d7f322b6126b36d0..eb01229e6b7ec3181e6304018f780b5c799f8fd3 100644 (file)
--- a/src/src/ip.c
+++ b/src/src/ip.c
@@ -236,22 +236,26 @@ if (fastopen)
   {
   if ((rc = sendto(sock, fastopen->data, fastopen->len,
                    MSG_FASTOPEN | MSG_DONTWAIT, s_ptr, s_len)) >= 0)
+       /* seen for with-data, experimental TFO option, with-cookie case */
     {
-    DEBUG(D_transport|D_v)
-      debug_printf("TCP_FASTOPEN mode connection, with data\n");
-    tcp_out_fastopen = TRUE;
+    DEBUG(D_transport|D_v) debug_printf("TFO mode connection attempt, %s data\n",
+      fastopen->len > 0 ? "with"  : "no");
+    tcp_out_fastopen = fastopen->len > 0 ?  2 : 1;
     }
   else if (errno == EINPROGRESS)       /* expected for nonready peer */
-    {
+       /* seen for no-data, proper TFO option, both cookie-request and with-cookie cases */
+       /*  apparently no visibility of the diffference at this point */
+       /*   with netwk delay, post-conn tcp_info sees unacked 1 for R, 2 for C; code in smtp_out.c */
+       /* ? older Experimental TFO option behaviour ? */
+    {                                  /* queue unsent data */
     if (!fastopen->data)
       {
-      DEBUG(D_transport|D_v)
-       debug_printf("TCP_FASTOPEN mode connection, no data\n");
-      tcp_out_fastopen = TRUE;
+      tcp_out_fastopen = 1;    /* we tried; unknown if useful yet */
       rc = 0;
       }
     else if (  (rc = send(sock, fastopen->data, fastopen->len, 0)) < 0
-           && errno == EINPROGRESS)    /* expected for nonready peer */
+           && errno == EINPROGRESS     /* expected for nonready peer */
+           )
       rc = 0;
     }
   else if(errno == EOPNOTSUPP)

diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c
index 36f6856772d0b02f74e2041c6e8b8986b3732fa8..0f8d5599b6a9173411ff094e1a961ec2c665b626 100644 (file)
--- a/src/src/smtp_in.c
+++ b/src/src/smtp_in.c
@@ -2346,7 +2346,7 @@ if (  getsockopt(fileno(smtp_out), IPPROTO_TCP, TCP_INFO, &tinfo, &len) == 0
    && tinfo.tcpi_state == TCP_SYN_RECV
    )
   {
-  DEBUG(D_receive) debug_printf("TCP_FASTOPEN mode connection\n");
+  DEBUG(D_receive) debug_printf("TCP_FASTOPEN mode connection (state TCP_SYN_RECV)\n");
   tcp_in_fastopen = TRUE;
   }
 # endif

diff --git a/src/src/smtp_out.c b/src/src/smtp_out.c
index db33ac66e26990c7ba0be6222ff0dd58106ee604..786f8b592802d1e05df8cf932e6de1fe92a3325f 100644 (file)
--- a/src/src/smtp_out.c
+++ b/src/src/smtp_out.c
@@ -140,6 +140,60 @@ return TRUE;
 
 
 
+#ifdef TCP_FASTOPEN
+static void
+tfo_out_check(int sock)
+{
+# if defined(TCP_INFO) && defined(EXIM_HAVE_TCPI_UNACKED)
+struct tcp_info tinfo;
+socklen_t len = sizeof(tinfo);
+
+if (getsockopt(sock, IPPROTO_TCP, TCP_INFO, &tinfo, &len) == 0)
+  {
+  switch (tcp_out_fastopen)
+    {
+      /* This is a somewhat dubious detection method; totally undocumented so likely
+      to fail in future kernels.  There seems to be no documented way.  What we really
+      want to know is if the server sent smtp-banner data before our ACK of his SYN,ACK
+      hit him.  What this (possibly?) detects is whether we sent a TFO cookie with our
+      SYN, as distinct from a TFO request.  This gets a false-positive when the server
+      key is rotated; we send the old one (which this test sees) but the server returns
+      the new one and does not send its SMTP banner before we ACK his SYN,ACK.
+       To force that rotation case:
+       '# echo -n "00000000-00000000-00000000-0000000" >/proc/sys/net/ipv4/tcp_fastopen_key'
+      The kernel seems to be counting unack'd packets. */
+
+    case 1:
+      if (tinfo.tcpi_unacked > 1)
+       {
+       DEBUG(D_transport|D_v)
+         debug_printf("TCP_FASTOPEN tcpi_unacked %d\n", tinfo.tcpi_unacked);
+       tcp_out_fastopen = 2;
+       }
+      break;
+
+#ifdef notdef          /* This seems to always fire, meaning that we cannot tell
+                       whether the server accepted data we sent.  For now assume
+                       that it did. */
+
+      /* If there was data-on-SYN but we had to retrasnmit it, declare no TFO */
+
+    case 2:
+      if (!(tinfo.tcpi_options & TCPI_OPT_SYN_DATA))
+       {
+       DEBUG(D_transport|D_v) debug_printf("TFO: had to retransmit\n");
+       tcp_out_fastopen = 0;
+       }
+      break;
+#endif
+    }
+
+  }
+# endif
+}
+#endif
+
+
 /* Arguments as for smtp_connect(), plus
   early_data   if non-NULL, data to be sent - preferably in the TCP SYN segment
 
@@ -158,6 +212,8 @@ int dscp_level;
 int dscp_option;
 int sock;
 int save_errno = 0;
+const blob * fastopen = NULL;
+
 
 #ifndef DISABLE_EVENT
 deliver_host_address = host->address;
@@ -207,8 +263,6 @@ requested some early-data then include that in the TFO request. */
 
 else
   {
-  const blob * fastopen = NULL;
-
 #ifdef TCP_FASTOPEN
   if (verify_check_given_host(&ob->hosts_try_fastopen, host) == OK)
     fastopen = early_data ? early_data : &tcp_fastopen_nodata;
@@ -254,6 +308,9 @@ else
     return -1;
     }
   if (ob->keepalive) ip_keepalive(sock, host->address, TRUE);
+#ifdef TCP_FASTOPEN
+  if (fastopen) tfo_out_check(sock);
+#endif
   return sock;
   }
 }

diff --git a/src/src/structs.h b/src/src/structs.h
index d8ac19ab8c8eaf3671a88f082da0ddb6b2f1424f..c16899a0c7a3ace69082250938c8afb5183f363f 100644 (file)
--- a/src/src/structs.h
+++ b/src/src/structs.h
@@ -610,7 +610,8 @@ typedef struct address_item {
     BOOL af_cert_verified:1;           /* delivered with verified TLS cert */
     BOOL af_pass_message:1;            /* pass message in bounces */
     BOOL af_bad_reply:1;               /* filter could not generate autoreply */
-    BOOL af_tcp_fastopen:1;            /* delivery used TCP Fast Open */
+    BOOL af_tcp_fastopen_conn:1;       /* delivery connection used TCP Fast Open */
+    BOOL af_tcp_fastopen:1;            /* delivery usefuly used TCP Fast Open */
 #ifndef DISABLE_PRDR
     BOOL af_prdr_used:1;               /* delivery used SMTP PRDR */
 #endif

diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index 461b26c4a2b3817692bad84a99ec088b9520a731..14cfde72aa814dd5284b46d690b537d50b627f30 100644 (file)
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -2511,7 +2511,10 @@ for (addr = sx->first_addr, address_count = 0;
   uschar * rcpt_addr;
 
   if (tcp_out_fastopen && !tcp_out_fastopen_logged)
-    setflag(addr, af_tcp_fastopen);
+    {
+    setflag(addr, af_tcp_fastopen_conn);
+    if (tcp_out_fastopen > 1) setflag(addr, af_tcp_fastopen);
+    }
 
   addr->dsn_aware = sx->peer_offered & OPTION_DSN
     ? dsn_support_yes : dsn_support_no;

diff --git a/src/src/transports/smtp_socks.c b/src/src/transports/smtp_socks.c
index 1368849d66a1415cdc50fea322a8cb30d64ed189..c9907dd545c2c3805bd5a31b8229889d419f6405 100644 (file)
--- a/src/src/transports/smtp_socks.c
+++ b/src/src/transports/smtp_socks.c
@@ -126,10 +126,12 @@ switch(method)
       for (i = 0; i<len; i++) debug_printf(" %02x", s[i]);
       debug_printf("\n");
       }
-    if (  send(fd, s, len, 0) < 0
-       || !fd_ready(fd, tmo-time(NULL))
-       || read(fd, s, 2) != 2
-       )
+    if (send(fd, s, len, 0) < 0)
+      return FAIL;
+#ifdef TCP_QUICKACK
+    (void) setsockopt(fd, IPPROTO_TCP, TCP_QUICKACK, US &off, sizeof(off));
+#endif
+    if (!fd_ready(fd, tmo-time(NULL)) || read(fd, s, 2) != 2)
       return FAIL;
     HDEBUG(D_transport|D_acl|D_v)
       debug_printf_indent("  SOCKS<< %02x %02x\n", s[0], s[1]);
@@ -315,6 +317,10 @@ HDEBUG(D_transport|D_acl|D_v) debug_printf_indent("  SOCKS>> 05 01 %02x\n", sob-
 
 /* expect method response */
 
+#ifdef TCP_QUICKACK
+(void) setsockopt(fd, IPPROTO_TCP, TCP_QUICKACK, US &off, sizeof(off));
+#endif
+
 if (  !fd_ready(fd, tmo-time(NULL))
    || read(fd, buf, 2) != 2
    )

diff --git a/test/scripts/1990-TCP-Fast-Open/1990 b/test/scripts/1990-TCP-Fast-Open/1990
index cbedd3622eeeed6bfd505b17dad793865f0c6dbf..4f5758f5a9353cae06e989e778f19846d2a0e2e6 100644 (file)
--- a/test/scripts/1990-TCP-Fast-Open/1990
+++ b/test/scripts/1990-TCP-Fast-Open/1990
@@ -8,6 +8,9 @@
 # option on the SYN, but the fast-output SMTP banner will not
 # be seen unless you also deliberately emulate a long path:
 # 'sudo tc qdisc add dev lo root netem delay 100ms'
+# You'll need kernel-modules-extra installed, or you get
+# an unhelpful error from RTNETLINK.
+# To tidy up:  'sudo tc qdisc delete dev lo root'
 #
 # First time runs will see a TFO request option only; subsequent
 # ones should see the TFO cookie and fast-output SMTP banner
@@ -21,6 +24,12 @@
 # this will only be obtained when the above delay is inserted into the
 # loopback net path.
 #
+# this attempt to tidy up does not work
+#sudo perl
+#open (my $fh, "/proc/sys/net/ipv4/tcp_fastopen_key");
+#print $fh "00000000-00000000-00000000-00000000";
+#close $fh;
+#****
 #
 #
 # FreeBSD: it looks like you have to compile a custom kernel, with