mirrors/passt
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tcp, udp: Receive batching doesn't pay off when writing single frames to tap

Browse source 
In pasta mode, when we get data from sockets and write it as single
frames to the tap device, we batch receive operations considerably,
and then (conceptually) split the data in many smaller writes.

It looked like an obvious choice, but performance is actually better
if we receive data in many small frame-sized recvmsg()/recvmmsg().

The syscall overhead with the previous behaviour, observed by perf,
comes predominantly from write operations, but receiving data in
shorter chunks probably improves cache locality by a considerable
amount.

Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
This commit is contained in:
Stefano Brivio 2022-02-20 03:52:44 +01:00
parent 9afd87b733
commit 6c93111864
2 changed files with 37 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
tcp.c
View file

@ -343,7 +343,9 @@
#define MAX_TAP_CONNS (128 * 1024) #define MAX_TAP_CONNS (128 * 1024)
#define MAX_SPLICE_CONNS (128 * 1024) #define MAX_SPLICE_CONNS (128 * 1024)
#define TCP_TAP_FRAMES 256 #define TCP_TAP_FRAMES_MEM 256
#define TCP_TAP_FRAMES \
(c->mode == MODE_PASST ? TCP_TAP_FRAMES_MEM : 1)
#define MAX_PIPE_SIZE (2UL * 1024 * 1024) #define MAX_PIPE_SIZE (2UL * 1024 * 1024)
@ -609,7 +611,7 @@ static struct tcp4_l2_buf_t {
#else #else
} __attribute__ ((packed, aligned(__alignof__(unsigned int)))) } __attribute__ ((packed, aligned(__alignof__(unsigned int))))
#endif #endif
tcp4_l2_buf[TCP_TAP_FRAMES]; tcp4_l2_buf[TCP_TAP_FRAMES_MEM];
static unsigned int tcp4_l2_buf_used; static unsigned int tcp4_l2_buf_used;
static size_t tcp4_l2_buf_bytes; static size_t tcp4_l2_buf_bytes;
@ -640,21 +642,21 @@ struct tcp6_l2_buf_t {
#else #else
} __attribute__ ((packed, aligned(__alignof__(unsigned int)))) } __attribute__ ((packed, aligned(__alignof__(unsigned int))))
#endif #endif
tcp6_l2_buf[TCP_TAP_FRAMES]; tcp6_l2_buf[TCP_TAP_FRAMES_MEM];
static unsigned int tcp6_l2_buf_used; static unsigned int tcp6_l2_buf_used;
static size_t tcp6_l2_buf_bytes; static size_t tcp6_l2_buf_bytes;
/* recvmsg()/sendmsg() data for tap */ /* recvmsg()/sendmsg() data for tap */
static char tcp_buf_discard [MAX_WINDOW]; static char tcp_buf_discard [MAX_WINDOW];
static struct iovec iov_sock [TCP_TAP_FRAMES + 1]; static struct iovec iov_sock [TCP_TAP_FRAMES_MEM + 1];
static struct iovec tcp4_l2_iov_tap [TCP_TAP_FRAMES]; static struct iovec tcp4_l2_iov_tap [TCP_TAP_FRAMES_MEM];
static struct iovec tcp6_l2_iov_tap [TCP_TAP_FRAMES]; static struct iovec tcp6_l2_iov_tap [TCP_TAP_FRAMES_MEM];
static struct iovec tcp4_l2_flags_iov_tap [TCP_TAP_FRAMES]; static struct iovec tcp4_l2_flags_iov_tap [TCP_TAP_FRAMES_MEM];
static struct iovec tcp6_l2_flags_iov_tap [TCP_TAP_FRAMES]; static struct iovec tcp6_l2_flags_iov_tap [TCP_TAP_FRAMES_MEM];
static struct mmsghdr tcp_l2_mh_tap [TCP_TAP_FRAMES]; static struct mmsghdr tcp_l2_mh_tap [TCP_TAP_FRAMES_MEM];
/* sendmsg() to socket */ /* sendmsg() to socket */
static struct iovec tcp_tap_iov [UIO_MAXIOV]; static struct iovec tcp_tap_iov [UIO_MAXIOV];
@ -688,7 +690,7 @@ static struct tcp4_l2_flags_buf_t {
#else #else
} __attribute__ ((packed, aligned(__alignof__(unsigned int)))) } __attribute__ ((packed, aligned(__alignof__(unsigned int))))
#endif #endif
tcp4_l2_flags_buf[TCP_TAP_FRAMES]; tcp4_l2_flags_buf[TCP_TAP_FRAMES_MEM];
static int tcp4_l2_flags_buf_used; static int tcp4_l2_flags_buf_used;
@ -717,7 +719,7 @@ static struct tcp6_l2_flags_buf_t {
#else #else
} __attribute__ ((packed, aligned(__alignof__(unsigned int)))) } __attribute__ ((packed, aligned(__alignof__(unsigned int))))
#endif #endif
tcp6_l2_flags_buf[TCP_TAP_FRAMES]; tcp6_l2_flags_buf[TCP_TAP_FRAMES_MEM];
static int tcp6_l2_flags_buf_used; static int tcp6_l2_flags_buf_used;
@ -916,7 +918,7 @@ void tcp_update_l2_buf(unsigned char *eth_d, unsigned char *eth_s,
{ {
int i; int i;
for (i = 0; i < TCP_TAP_FRAMES; i++) { for (i = 0; i < TCP_TAP_FRAMES_MEM; i++) {
struct tcp4_l2_flags_buf_t *b4f = &tcp4_l2_flags_buf[i]; struct tcp4_l2_flags_buf_t *b4f = &tcp4_l2_flags_buf[i];
struct tcp6_l2_flags_buf_t *b6f = &tcp6_l2_flags_buf[i]; struct tcp6_l2_flags_buf_t *b6f = &tcp6_l2_flags_buf[i];
struct tcp4_l2_buf_t *b4 = &tcp4_l2_buf[i]; struct tcp4_l2_buf_t *b4 = &tcp4_l2_buf[i];
@ -982,12 +984,13 @@ static void tcp_sock4_iov_init(void)
}; };
} }
for (i = 0, iov = tcp4_l2_iov_tap; i < TCP_TAP_FRAMES; i++, iov++) { for (i = 0, iov = tcp4_l2_iov_tap; i < TCP_TAP_FRAMES_MEM; i++, iov++) {
iov->iov_base = &tcp4_l2_buf[i].vnet_len; iov->iov_base = &tcp4_l2_buf[i].vnet_len;
iov->iov_len = MSS_DEFAULT; iov->iov_len = MSS_DEFAULT;
} }
for (i = 0, iov = tcp4_l2_flags_iov_tap; i < TCP_TAP_FRAMES; i++, iov++) for (i = 0, iov = tcp4_l2_flags_iov_tap; i < TCP_TAP_FRAMES_MEM;
i++, iov++)
iov->iov_base = &tcp4_l2_flags_buf[i].vnet_len; iov->iov_base = &tcp4_l2_flags_buf[i].vnet_len;
} }
@ -1015,12 +1018,13 @@ static void tcp_sock6_iov_init(void)
}; };
} }
for (i = 0, iov = tcp6_l2_iov_tap; i < TCP_TAP_FRAMES; i++, iov++) { for (i = 0, iov = tcp6_l2_iov_tap; i < TCP_TAP_FRAMES_MEM; i++, iov++) {
iov->iov_base = &tcp6_l2_buf[i].vnet_len; iov->iov_base = &tcp6_l2_buf[i].vnet_len;
iov->iov_len = MSS_DEFAULT; iov->iov_len = MSS_DEFAULT;
} }
for (i = 0, iov = tcp6_l2_flags_iov_tap; i < TCP_TAP_FRAMES; i++, iov++) for (i = 0, iov = tcp6_l2_flags_iov_tap; i < TCP_TAP_FRAMES_MEM;
i++, iov++)
iov->iov_base = &tcp6_l2_flags_buf[i].vnet_len; iov->iov_base = &tcp6_l2_flags_buf[i].vnet_len;
} }

33
udp.c
View file

@ -118,7 +118,8 @@
#define UDP_CONN_TIMEOUT 180 /* s, timeout for ephemeral or local bind */ #define UDP_CONN_TIMEOUT 180 /* s, timeout for ephemeral or local bind */
#define UDP_SPLICE_FRAMES 128 #define UDP_SPLICE_FRAMES 128
#define UDP_TAP_FRAMES 128 #define UDP_TAP_FRAMES_MEM 128
#define UDP_TAP_FRAMES (c->mode == MODE_PASST ? UDP_TAP_FRAMES_MEM : 1)
/** /**
* struct udp_tap_port - Port tracking based on tap-facing source port * struct udp_tap_port - Port tracking based on tap-facing source port
@ -204,7 +205,7 @@ static struct udp4_l2_buf_t {
uint8_t data[USHRT_MAX - uint8_t data[USHRT_MAX -
(sizeof(struct iphdr) + sizeof(struct udphdr))]; (sizeof(struct iphdr) + sizeof(struct udphdr))];
} __attribute__ ((packed, aligned(__alignof__(unsigned int)))) } __attribute__ ((packed, aligned(__alignof__(unsigned int))))
udp4_l2_buf[UDP_TAP_FRAMES]; udp4_l2_buf[UDP_TAP_FRAMES_MEM];
/** /**
* udp6_l2_buf_t - Pre-cooked IPv6 packet buffers for tap connections * udp6_l2_buf_t - Pre-cooked IPv6 packet buffers for tap connections
@ -234,23 +235,23 @@ struct udp6_l2_buf_t {
#else #else
} __attribute__ ((packed, aligned(__alignof__(unsigned int)))) } __attribute__ ((packed, aligned(__alignof__(unsigned int))))
#endif #endif
udp6_l2_buf[UDP_TAP_FRAMES]; udp6_l2_buf[UDP_TAP_FRAMES_MEM];
static struct sockaddr_storage udp_splice_namebuf; static struct sockaddr_storage udp_splice_namebuf;
static uint8_t udp_splice_buf[UDP_SPLICE_FRAMES][USHRT_MAX]; static uint8_t udp_splice_buf[UDP_SPLICE_FRAMES][USHRT_MAX];
/* recvmmsg()/sendmmsg() data for tap */ /* recvmmsg()/sendmmsg() data for tap */
static struct iovec udp4_l2_iov_sock [UDP_TAP_FRAMES]; static struct iovec udp4_l2_iov_sock [UDP_TAP_FRAMES_MEM];
static struct iovec udp6_l2_iov_sock [UDP_TAP_FRAMES]; static struct iovec udp6_l2_iov_sock [UDP_TAP_FRAMES_MEM];
static struct iovec udp4_l2_iov_tap [UDP_TAP_FRAMES]; static struct iovec udp4_l2_iov_tap [UDP_TAP_FRAMES_MEM];
static struct iovec udp6_l2_iov_tap [UDP_TAP_FRAMES]; static struct iovec udp6_l2_iov_tap [UDP_TAP_FRAMES_MEM];
static struct mmsghdr udp4_l2_mh_sock [UDP_TAP_FRAMES]; static struct mmsghdr udp4_l2_mh_sock [UDP_TAP_FRAMES_MEM];
static struct mmsghdr udp6_l2_mh_sock [UDP_TAP_FRAMES]; static struct mmsghdr udp6_l2_mh_sock [UDP_TAP_FRAMES_MEM];
static struct mmsghdr udp4_l2_mh_tap [UDP_TAP_FRAMES]; static struct mmsghdr udp4_l2_mh_tap [UDP_TAP_FRAMES_MEM];
static struct mmsghdr udp6_l2_mh_tap [UDP_TAP_FRAMES]; static struct mmsghdr udp6_l2_mh_tap [UDP_TAP_FRAMES_MEM];
/* recvmmsg()/sendmmsg() data for "spliced" connections */ /* recvmmsg()/sendmmsg() data for "spliced" connections */
static struct iovec udp_splice_iov_recv [UDP_SPLICE_FRAMES]; static struct iovec udp_splice_iov_recv [UDP_SPLICE_FRAMES];
@ -310,7 +311,7 @@ void udp_update_l2_buf(unsigned char *eth_d, unsigned char *eth_s,
{ {
int i; int i;
for (i = 0; i < UDP_TAP_FRAMES; i++) { for (i = 0; i < UDP_TAP_FRAMES_MEM; i++) {
struct udp4_l2_buf_t *b4 = &udp4_l2_buf[i]; struct udp4_l2_buf_t *b4 = &udp4_l2_buf[i];
struct udp6_l2_buf_t *b6 = &udp6_l2_buf[i]; struct udp6_l2_buf_t *b6 = &udp6_l2_buf[i];
@ -354,7 +355,7 @@ static void udp_sock4_iov_init(void)
}; };
} }
for (i = 0, h = udp4_l2_mh_sock; i < UDP_TAP_FRAMES; i++, h++) { for (i = 0, h = udp4_l2_mh_sock; i < UDP_TAP_FRAMES_MEM; i++, h++) {
struct msghdr *mh = &h->msg_hdr; struct msghdr *mh = &h->msg_hdr;
mh->msg_name = &udp4_l2_buf[i].s_in; mh->msg_name = &udp4_l2_buf[i].s_in;
@ -366,7 +367,7 @@ static void udp_sock4_iov_init(void)
mh->msg_iovlen = 1; mh->msg_iovlen = 1;
} }
for (i = 0, h = udp4_l2_mh_tap; i < UDP_TAP_FRAMES; i++, h++) { for (i = 0, h = udp4_l2_mh_tap; i < UDP_TAP_FRAMES_MEM; i++, h++) {
struct msghdr *mh = &h->msg_hdr; struct msghdr *mh = &h->msg_hdr;
udp4_l2_iov_tap[i].iov_base = &udp4_l2_buf[i].vnet_len; udp4_l2_iov_tap[i].iov_base = &udp4_l2_buf[i].vnet_len;
@ -394,7 +395,7 @@ static void udp_sock6_iov_init(void)
}; };
} }
for (i = 0, h = udp6_l2_mh_sock; i < UDP_TAP_FRAMES; i++, h++) { for (i = 0, h = udp6_l2_mh_sock; i < UDP_TAP_FRAMES_MEM; i++, h++) {
struct msghdr *mh = &h->msg_hdr; struct msghdr *mh = &h->msg_hdr;
mh->msg_name = &udp6_l2_buf[i].s_in6; mh->msg_name = &udp6_l2_buf[i].s_in6;
@ -406,7 +407,7 @@ static void udp_sock6_iov_init(void)
mh->msg_iovlen = 1; mh->msg_iovlen = 1;
} }
for (i = 0, h = udp6_l2_mh_tap; i < UDP_TAP_FRAMES; i++, h++) { for (i = 0, h = udp6_l2_mh_tap; i < UDP_TAP_FRAMES_MEM; i++, h++) {
struct msghdr *mh = &h->msg_hdr; struct msghdr *mh = &h->msg_hdr;
udp6_l2_iov_tap[i].iov_base = &udp6_l2_buf[i].vnet_len; udp6_l2_iov_tap[i].iov_base = &udp6_l2_buf[i].vnet_len;