TCP Fast Open support

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.60])
-AC_INIT([sniproxy], [0.6.0])
+AC_INIT([sniproxy], [0.6.0+git.17.g6867569])
 AC_CONFIG_SRCDIR([src/sniproxy.c])
 AC_CONFIG_MACRO_DIR([m4])
 AM_INIT_AUTOMAKE([subdir-objects])

redhat/sniproxy.spec

@@ -1,5 +1,5 @@
 Name: sniproxy
-Version: 0.6.0
+Version: 0.6.0+git.17.g6867569
 Release: 1%{?dist}
 Summary: Transparent TLS and HTTP layer 4 proxy with SNI support
 

src/buffer.c

@@ -27,7 +27,6 @@
 #include <stdlib.h> /* malloc */
 #include <string.h> /* memcpy */
 #include <sys/types.h>
-#include <sys/socket.h>
 #include <sys/uio.h>
 #include <time.h>
 #include <errno.h>
@@ -147,6 +146,28 @@ buffer_send(struct Buffer *buffer, int sockfd, int flags, struct ev_loop *loop)
     return bytes;
 }
 
+ssize_t
+buffer_sendto(struct Buffer *buffer, int sockfd, int flags,
+        const struct sockaddr *dest_addr, socklen_t dest_addr_len,
+        struct ev_loop *loop) {
+    struct iovec iov[2];
+    struct msghdr msg = {
+        .msg_name = (struct sockaddr *)dest_addr,
+        .msg_namelen = dest_addr_len,
+        .msg_iov = iov,
+        .msg_iovlen = setup_read_iov(buffer, iov, 0)
+    };
+
+    ssize_t bytes = sendmsg(sockfd, &msg, flags);
+
+    buffer->last_send = ev_now(loop);
+
+    if (bytes > 0)
+        advance_read_position(buffer, (size_t)bytes);
+
+    return bytes;
+}
+
 /*
  * Read data from file into buffer
  */

src/buffer.h

@@ -28,6 +28,7 @@
 
 #include <stdio.h>
 #include <sys/types.h>
+#include <sys/socket.h>
 #include <ev.h>
 
 
@@ -47,6 +48,7 @@ void free_buffer(struct Buffer *);
 
 ssize_t buffer_recv(struct Buffer *, int, int, struct ev_loop *);
 ssize_t buffer_send(struct Buffer *, int, int, struct ev_loop *);
+ssize_t buffer_sendto(struct Buffer *, int, int, const struct sockaddr *, socklen_t, struct ev_loop *);
 ssize_t buffer_read(struct Buffer *, int);
 ssize_t buffer_write(struct Buffer *, int);
 ssize_t buffer_resize(struct Buffer *, size_t);

src/config.c

@@ -105,6 +105,10 @@ static const struct Keyword listener_stanza_grammar[] = {
         .keyword="reuseport",
         .parse_arg=(int(*)(void *, const char *))accept_listener_reuseport,
     },
+    {
+        .keyword="fastopen",
+        .parse_arg=(int(*)(void *, const char *))accept_listener_fastopen,
+    },
     {
         .keyword="ipv6_v6only",
         .parse_arg=(int(*)(void *, const char *))accept_listener_ipv6_v6only,

src/connection.c

@@ -33,6 +33,7 @@
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
+#include <netinet/tcp.h>
 #include <netdb.h> /* getaddrinfo */
 #include <unistd.h> /* close */
 #include <fcntl.h>
@@ -253,7 +254,9 @@ connection_cb(struct ev_loop *loop, struct ev_io *w, int revents) {
     /* Receive first in case the socket was closed */
     if (revents & EV_READ && buffer_room(input_buffer)) {
         ssize_t bytes_received = buffer_recv(input_buffer, w->fd, 0, loop);
-        if (bytes_received < 0 && !IS_TEMPORARY_SOCKERR(errno)) {
+        if (bytes_received < 0 && 
+            !IS_TEMPORARY_SOCKERR(errno) && 
+            !con->server.addr_once) {
             warn("recv(%s): %s, closing connection",
                     socket_name,
                     strerror(errno));
@@ -268,8 +271,19 @@ connection_cb(struct ev_loop *loop, struct ev_io *w, int revents) {
 
     /* Transmit */
     if (revents & EV_WRITE && buffer_len(output_buffer)) {
-        ssize_t bytes_transmitted = buffer_send(output_buffer, w->fd, 0, loop);
-        if (bytes_transmitted < 0 && !IS_TEMPORARY_SOCKERR(errno)) {
+        ssize_t bytes_transmitted = -1;
+        if (!is_client && con->server.addr_once) {
+            bytes_transmitted = 
+                buffer_sendto(output_buffer, w->fd,
+                              MSG_FASTOPEN, con->server.addr_once,
+                              con->server.addr_len, loop);
+            con->server.addr_once = NULL;
+        } else {
+            bytes_transmitted = buffer_send(output_buffer, w->fd, 0, loop);
+        }
+
+        if (bytes_transmitted < 0 && 
+            errno != EINPROGRESS && !IS_TEMPORARY_SOCKERR(errno)) {
             warn("send(%s): %s, closing connection",
                     socket_name,
                     strerror(errno));
@@ -628,22 +642,27 @@ initiate_server_connect(struct Connection *con, struct ev_loop *loop) {
     fcntl(sockfd, F_SETFL, flags | O_NONBLOCK);
 #endif
 
+    int on = 1;
+#ifdef TCP_NODELAY
+    int result = setsockopt(sockfd, SOL_TCP, TCP_NODELAY, &on, sizeof(on));
+#else
+    int result = -EPERM;
+    /* XXX error: not implemented would be better, but this shouldn't be
+     * reached since it is prohibited in the configuration parser. */
+#endif
+    result = setsockopt(sockfd, SOL_SOCKET, SO_KEEPALIVE, &on, sizeof(on));
+
     if (con->listener->transparent_proxy &&
             con->client.addr.ss_family == con->server.addr.ss_family) {
 #ifdef IP_TRANSPARENT
-        int on = 1;
-        int result = setsockopt(sockfd, SOL_IP, IP_TRANSPARENT, &on, sizeof(on));
-#else
-        int result = -EPERM;
-        /* XXX error: not implemented would be better, but this shouldn't be
-         * reached since it is prohibited in the configuration parser. */
-#endif
+        result = setsockopt(sockfd, SOL_IP, IP_TRANSPARENT, &on, sizeof(on));
         if (result < 0) {
             err("setsockopt IP_TRANSPARENT failed: %s", strerror(errno));
             close(sockfd);
             abort_connection(con);
             return;
         }
+#endif
 
         result = bind(sockfd, (struct sockaddr *)&con->client.addr,
                 con->client.addr_len);
@@ -680,18 +699,29 @@ initiate_server_connect(struct Connection *con, struct ev_loop *loop) {
         }
     }
 
-    int result = connect(sockfd,
-            (struct sockaddr *)&con->server.addr,
-            con->server.addr_len);
-    /* TODO retry connect in EADDRNOTAVAIL case */
-    if (result < 0 && errno != EINPROGRESS) {
-        close(sockfd);
-        char server[INET6_ADDRSTRLEN + 8];
-        warn("Failed to open connection to %s: %s",
-                display_sockaddr(&con->server.addr, server, sizeof(server)),
-                strerror(errno));
-        abort_connection(con);
-        return;
+#ifndef MSG_FASTOPEN
+    con->listener->fastopen = 0;
+    con->server.addr_once = NULL;
+    warn("TCP Fast Open for client sockets not supported in this build");
+#endif
+
+    if (con->listener->fastopen == 1 || 
+        con->listener->fastopen == 3) {
+        con->server.addr_once = (struct sockaddr *)&con->server.addr;
+    } else {
+        result = connect(sockfd,
+                (struct sockaddr *)&con->server.addr,
+                con->server.addr_len);
+        /* TODO retry connect in EADDRNOTAVAIL case */
+        if (result < 0 && errno != EINPROGRESS) {
+            close(sockfd);
+            char server[INET6_ADDRSTRLEN + 8];
+            warn("Failed to open connection to %s: %s",
+                    display_sockaddr(&con->server.addr, server, sizeof(server)),
+                    strerror(errno));
+            abort_connection(con);
+            return;
+        }    
     }
 
     if (getsockname(sockfd, (struct sockaddr *)&con->server.local_addr,

src/connection.h

@@ -47,6 +47,7 @@ struct Connection {
 
     struct {
         struct sockaddr_storage addr, local_addr;
+        struct sockaddr *addr_once;
         socklen_t addr_len, local_addr_len;
         struct ev_io watcher;
         struct Buffer *buffer;

src/listener.c

@@ -38,6 +38,7 @@
 #include <sys/socket.h>
 #include <sys/un.h>
 #include <netinet/in.h>
+#include <netinet/tcp.h>
 #include <arpa/inet.h>
 #include <assert.h>
 #include "address.h"
@@ -216,6 +217,7 @@ new_listener() {
     listener->table_name = NULL;
     listener->access_log = NULL;
     listener->log_bad_requests = 0;
+    listener->fastopen = 0;
     listener->reuseport = 0;
     listener->ipv6_v6only = 0;
     listener->transparent_proxy = 0;
@@ -292,6 +294,28 @@ accept_listener_protocol(struct Listener *listener, const char *protocol) {
     return 1;
 }
 
+int
+accept_listener_fastopen(struct Listener *listener, const char *fastopen) {
+    listener->fastopen = parse_boolean(fastopen);
+    if (listener->fastopen)
+        listener->fastopen = 3;
+    else if (strcasecmp(fastopen, "frontend") == 0)
+        listener->fastopen = 2;
+    else if (strcasecmp(fastopen, "backend") == 0)
+        listener->fastopen = 1;
+    else
+        return 0;
+
+#ifndef TCP_FASTOPEN
+    if (listener->fastopen != -1) {
+        err("TCP Fast Open not supported in this build");
+        return 0;
+    }
+#endif
+
+    return 1;
+}
+
 int
 accept_listener_reuseport(struct Listener *listener, const char *reuseport) {
     listener->reuseport = parse_boolean(reuseport);
@@ -536,6 +560,10 @@ init_listener(struct Listener *listener, const struct Table_head *tables,
         return result;
     }
 
+#ifdef TCP_NODELAY
+    result = setsockopt(sockfd, SOL_TCP, TCP_NODELAY, &on, sizeof(on));
+#endif
+
     if (listener->reuseport == 1) {
 #ifdef SO_REUSEPORT
         /* set SO_REUSEPORT on server socket to allow binding of multiple
@@ -567,6 +595,19 @@ init_listener(struct Listener *listener, const struct Table_head *tables,
         }
     }
 
+#ifdef TCP_FASTOPEN
+    if (listener->fastopen == 2 ||
+        listener->fastopen == 3) {
+        int qlen = SOMAXCONN;
+        result = setsockopt(sockfd, SOL_TCP, TCP_FASTOPEN, &qlen, sizeof(qlen));
+        if (result < 0) {
+            err("setsockopt TCP_FASTOPEN failed: %s", strerror(errno));
+            close(sockfd);
+            return result;
+        }        
+    }
+#endif
+
     result = bind(sockfd, address_sa(listener->address),
             address_sa_len(listener->address));
     if (result < 0 && errno == EACCES) {

src/listener.h

@@ -39,7 +39,7 @@ struct Listener {
     const struct Protocol *protocol;
     char *table_name;
     struct Logger *access_log;
-    int log_bad_requests, reuseport, transparent_proxy, ipv6_v6only;
+    int log_bad_requests, fastopen, reuseport, transparent_proxy, ipv6_v6only;
     int fallback_use_proxy_header;
 
     /* Runtime fields */
@@ -58,6 +58,7 @@ int accept_listener_table_name(struct Listener *, const char *);
 int accept_listener_fallback_address(struct Listener *, const char *);
 int accept_listener_source_address(struct Listener *, const char *);
 int accept_listener_protocol(struct Listener *, const char *);
+int accept_listener_fastopen(struct Listener *, const char *);
 int accept_listener_reuseport(struct Listener *, const char *);
 int accept_listener_ipv6_v6only(struct Listener *, const char *);
 int accept_listener_bad_request_action(struct Listener *, const char *);