tcp: Add connection union type

Currently, the tables for spliced and non-spliced connections are entirely
separate, with different types in different arrays.  We want to unify them.
As a first step, create a union type which can represent either a spliced
or non-spliced connection.  For them to be distinguishable, the individual
types need to have a common header added, with a bit indicating which type
this structure is.

This comes at the cost of increasing the size of tcp_tap_conn to over one
(64 byte) cacheline.  This isn't ideal, but it makes things simpler for now
and we'll re-optimize this later.

Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>

tcp.c

@@ -288,6 +288,7 @@
 #include <sys/uio.h>
 #include <unistd.h>
 #include <time.h>
+#include <assert.h>
 
 #include <linux/tcp.h> /* For struct tcp_info */
 
@@ -601,6 +602,7 @@ static inline struct tcp_tap_conn *conn_at_idx(int index)
 {
 	if ((index < 0) || (index >= TCP_MAX_CONNS))
 		return NULL;
+	assert(!(CONN(index)->c.spliced));
 	return CONN(index);
 }
 
@@ -2096,6 +2098,7 @@ static void tcp_conn_from_tap(struct ctx *c, int af, const void *addr,
 	}
 
 	conn = CONN(c->tcp.conn_count++);
+	conn->c.spliced = false;
 	conn->sock = s;
 	conn->timer = -1;
 	conn_event(c, conn, TAP_SYN_RCVD);
@@ -2764,6 +2767,7 @@ static void tcp_conn_from_sock(struct ctx *c, union epoll_ref ref,
 		return;
 
 	conn = CONN(c->tcp.conn_count++);
+	conn->c.spliced = false;
 	conn->sock = s;
 	conn->timer = -1;
 	conn->ws_to_tap = conn->ws_from_tap = 0;

tcp_conn.h

@@ -11,8 +11,19 @@
 
 #define TCP_HASH_BUCKET_BITS		(TCP_CONN_INDEX_BITS + 1)
 
+/**
+ * struct tcp_conn_common - Common fields for spliced and non-spliced
+ * @spliced:		Is this a spliced connection?
+ */
+struct tcp_conn_common {
+	bool spliced	:1;
+};
+
+extern const char *tcp_common_flag_str[];
+
 /**
  * struct tcp_tap_conn - Descriptor for a TCP connection (not spliced)
+ * @c:			Fields common with tcp_splice_conn
  * @next_index:		Connection index of next item in hash chain, -1 for none
  * @tap_mss:		MSS advertised by tap/guest, rounded to 2 ^ TCP_MSS_BITS
  * @sock:		Socket descriptor number
@@ -40,6 +51,9 @@
  * @seq_init_from_tap:	Initial sequence number from tap
  */
 struct tcp_tap_conn {
+	/* Must be first element to match tcp_splice_conn */
+	struct tcp_conn_common c;
+
 	int	 	next_index	:TCP_CONN_INDEX_BITS + 2;
 
 #define TCP_RETRANS_BITS		3
@@ -122,6 +136,7 @@ struct tcp_tap_conn {
 
 /**
  * struct tcp_splice_conn - Descriptor for a spliced TCP connection
+ * @c:			Fields common with tcp_tap_conn
  * @a:			File descriptor number of socket for accepted connection
  * @pipe_a_b:		Pipe ends for splice() from @a to @b
  * @b:			File descriptor number of peer connected socket
@@ -134,6 +149,9 @@ struct tcp_tap_conn {
  * @b_written:		Bytes written to @b (not fully written from one @a read)
 */
 struct tcp_splice_conn {
+	/* Must be first element to match tcp_tap_conn */
+	struct tcp_conn_common c;
+
 	int a;
 	int pipe_a_b[2];
 	int b;
@@ -165,4 +183,16 @@ struct tcp_splice_conn {
 	uint32_t b_written;
 };
 
+/**
+ * union tcp_conn - Descriptor for a TCP connection (spliced or non-spliced)
+ * @c:			Fields common between all variants
+ * @tap:		Fields specific to non-spliced connections
+ * @splice:		Fields specific to spliced connections
+*/
+union tcp_conn {
+	struct tcp_conn_common c;
+	struct tcp_tap_conn tap;
+	struct tcp_splice_conn splice;
+};
+
 #endif /* TCP_CONN_H */

tcp_splice.c

@@ -46,6 +46,7 @@
 #include <sys/epoll.h>
 #include <sys/types.h>
 #include <sys/socket.h>
+#include <assert.h>
 
 #include "util.h"
 #include "passt.h"
@@ -554,6 +555,7 @@ void tcp_sock_handler_splice(struct ctx *c, union epoll_ref ref,
 		}
 
 		conn = CONN(c->tcp.splice_conn_count++);
+		conn->c.spliced = true;
 		conn->a = s;
 		conn->flags = ref.r.p.tcp.tcp.v6 ? SPLICE_V6 : 0;