Skip to content

Commit a1a00a8

Browse files
jsitnickijsitnicki
committed
Add tests for configurable TIME-WAIT reuse delay
This set of tests accompanies the kernel patch set which adds a knob for setting the TIME-WAIT reuse delay [1]. Tests cover connection reincarnation through TIME-WAIT reuse with the default reuse delay (1 second) as well as with a reuse delay radically shortened down to 1 millisecond. We exercise both the happy and failure scenarios, where TW reuse is not possible because the required delay period has not elapsed yet. [1] https://lore.kernel.org/all/20241204-jakub-krn-909-poc-msec-tw-tstamp-v1-0-8b54467a0f34@cloudflare.com/
1 parent 8051382 commit a1a00a8

8 files changed

+377
-0
lines changed

‎gtests/net/packetdrill/symbols_linux.c‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,8 @@ struct int_symbol platform_symbols_table[] = {
168168
#endif
169169
{ IPV6_TCLASS, "IPV6_TCLASS" },
170170
{ IPV6_HOPLIMIT, "IPV6_HOPLIMIT" },
171+
{ IP_BIND_ADDRESS_NO_PORT, "IP_BIND_ADDRESS_NO_PORT" },
172+
{ IP_LOCAL_PORT_RANGE, "IP_LOCAL_PORT_RANGE" },
171173

172174
{ TCP_NODELAY, "TCP_NODELAY" },
173175
{ TCP_MAXSEG, "TCP_MAXSEG" },

‎gtests/net/tcp/ts_recent/tw_reuse/fast_tw_reuse_fail_after_500usec.pkt‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
// Check that TIME-WAIT 4-tuple can't be reused after less than a millisecond
2+
3+
--ip_version=ipv4
4+
--tcp_ts_tick_usecs=1000
5+
--tolerance_percent=1
6+
7+
`
8+
../../common/defaults.sh
9+
../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse=1 # global enable
10+
../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse_delay=1 # msec
11+
`
12+
13+
//
14+
// Prime TIME-WAIT - open first connection and close it.
15+
// Use source port 0xdead (57005).
16+
//
17+
18+
0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
19+
20+
+0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
21+
+0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
22+
23+
+0 connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress)
24+
25+
+0 > S 0:0(0) <mss 1460,sackOK,TS val 1001 ecr 0,nop,wscale 8>
26+
+.001 < S. 0:0(0) ack 1 win 65535 <mss 1460,sackOK,TS val 3001 ecr 1001,nop,wscale 8>
27+
+0 > . 1:1(0) ack 1 <nop,nop,TS val 1002 ecr 3001>
28+
29+
+0 close(3) = 0
30+
31+
+0 > F. 1:1(0) ack 1 <nop,nop,TS val 1002 ecr 3001>
32+
+.001 < F. 1:1(0) ack 2 win 8 <nop,nop,TS val 3002 ecr 1002>
33+
+0 > . 2:2(0) ack 2 <nop,nop,TS val 1003 ecr 3002>
34+
35+
//
36+
// Reincarnation after less than 1 msec - open and close second connection.
37+
// Use the same source port.
38+
//
39+
40+
+0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
41+
42+
+0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
43+
+0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
44+
45+
// Delay for 500 usec and assume it takes packetdrill less than 500 usec to
46+
// reincarnate a connection (the interval from last ACK to next connect()), so
47+
// the total delay is less than 1 msec. On a consumer laptop connection
48+
// reincarnation takes ~200 usec.
49+
+.0005 connect(3, ..., ...) = -1 EADDRNOTAVAIL (Cannot assign requested address)
50+
51+
+0 close(3) = 0
52+

‎gtests/net/tcp/ts_recent/tw_reuse/fast_tw_reuse_ok_after_1msec.pkt‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
// Check that TIME-WAIT 4-tuple can be reused after a 1 millisecond period
2+
3+
--ip_version=ipv4
4+
--tcp_ts_tick_usecs=1000
5+
--tolerance_percent=1
6+
7+
`
8+
../../common/defaults.sh
9+
../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse=1 # global enable
10+
../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse_delay=1 # msec
11+
`
12+
13+
//
14+
// Prime TIME-WAIT - open first connection and close it.
15+
// Use source port 0xdead (57005).
16+
//
17+
18+
0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
19+
20+
+0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
21+
+0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
22+
23+
+0 connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress)
24+
25+
+0 > S 0:0(0) <mss 1460,sackOK,TS val 1001 ecr 0,nop,wscale 8>
26+
+.001 < S. 0:0(0) ack 1 win 65535 <mss 1460,sackOK,TS val 3001 ecr 1001,nop,wscale 8>
27+
+0 > . 1:1(0) ack 1 <...>
28+
29+
+0 close(3) = 0
30+
31+
+0 > F. 1:1(0) ack 1 <nop,nop,TS val 1002 ecr 3001>
32+
+.001 < F. 1:1(0) ack 2 win 8 <nop,nop,TS val 3002 ecr 1002>
33+
+0 > . 2:2(0) ack 2 <nop,nop,TS val 1003 ecr 3002>
34+
35+
//
36+
// Reincarnation after 1 millisecond - open and close second connection.
37+
// Use the same source port.
38+
//
39+
40+
+0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
41+
42+
+0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
43+
+0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
44+
45+
46+
// Delay for 1 millisecond, plus an extra millisecond to avoid test flakiness.
47+
// Kernel TIME-WAIT reuse timer has millisecond resolution. Actual reuse delay
48+
// can be up to 1 msec longer than the setting due to timestamp rounding.
49+
+.002 connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress)
50+
51+
// NOTE: Packetdrill doesn't actually track timestamps across connection
52+
// reincarnations despite the fact that the random offset doesn't change.
53+
+0 > S 0:0(0) <mss 1460,sackOK,TS val 1005 ecr 3002,nop,wscale 8>
54+
+.001 < S. 0:0(0) ack 1 win 65535 <mss 1460,sackOK,TS val 3003 ecr 1005,nop,wscale 8>
55+
+0 > . 1:1(0) ack 1 <nop,nop,TS val 1006 ecr 3003>
56+
57+
+0 close(3) = 0
58+
59+
+0 > F. 1:1(0) ack 1 <nop,nop,TS val 1006 ecr 3003>
60+
+.001 < F. 1:1(0) ack 2 win 8 <nop,nop,TS val 3004 ecr 1006>
61+
+0 > . 2:2(0) ack 2 <nop,nop,TS val 1007 ecr 3004>

‎gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_fail_after_0sec.pkt‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
// Check that TIME-WAIT 4-tuple can't be reused immediately
2+
3+
--ip_version=ipv4
4+
--tcp_ts_tick_usecs=1000
5+
--tolerance_percent=1
6+
7+
`
8+
../../common/defaults.sh
9+
../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse=1 # global enable
10+
../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse_delay=1000 # msec
11+
`
12+
13+
//
14+
// Prime TIME-WAIT - open first connection and close it.
15+
// Use source port 0xdead (57005).
16+
//
17+
18+
0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
19+
20+
+0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
21+
+0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
22+
23+
+0 connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress)
24+
25+
+0 > S 0:0(0) <mss 1460,sackOK,TS val 1001 ecr 0,nop,wscale 8>
26+
+.001 < S. 0:0(0) ack 1 win 65535 <mss 1460,sackOK,TS val 3001 ecr 1001,nop,wscale 8>
27+
+0 > . 1:1(0) ack 1 <nop,nop,TS val 1002 ecr 3001>
28+
29+
+0 close(3) = 0
30+
31+
+0 > F. 1:1(0) ack 1 <nop,nop,TS val 1002 ecr 3001>
32+
+.001 < F. 1:1(0) ack 2 win 8 <nop,nop,TS val 3002 ecr 1002>
33+
+0 > . 2:2(0) ack 2 <nop,nop,TS val 1003 ecr 3002>
34+
35+
//
36+
// Reincarnation without delay - open and close second connection.
37+
// Use the same source port.
38+
//
39+
40+
+0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
41+
42+
+0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
43+
+0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
44+
45+
+0.00 connect(3, ..., ...) = -1 EADDRNOTAVAIL (Cannot assign requested address)
46+
47+
+0 close(3) = 0

‎gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_fail_after_1sec_without_tsopt.pkt‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
// Check that TIME-WAIT 4-tuple can't be reused after the TIME-WAIT reuse delay
2+
// when peer doesn't support TSopt
3+
4+
--ip_version=ipv4
5+
--tcp_ts_tick_usecs=1000
6+
--tolerance_percent=1
7+
8+
`
9+
../../common/defaults.sh
10+
../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse=1 # global enable
11+
../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse_delay=1000 # msec
12+
`
13+
14+
//
15+
// Prime TIME-WAIT - open first connection and close it.
16+
// Use source port 0xdead (57005).
17+
//
18+
19+
0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
20+
21+
+0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
22+
+0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
23+
24+
+0 connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress)
25+
26+
+0 > S 0:0(0) <mss 1460,sackOK,TS val 1001 ecr 0,nop,wscale 8>
27+
+.001 < S. 0:0(0) ack 1 win 65535 <mss 1460,sackOK,nop,nop,nop,wscale 8>
28+
+0 > . 1:1(0) ack 1
29+
30+
+0 close(3) = 0
31+
32+
+0 > F. 1:1(0) ack 1
33+
+.001 < F. 1:1(0) ack 2 win 8
34+
+0 > . 2:2(0) ack 2
35+
36+
//
37+
// Reincarnation after 1 second - open and close second connection.
38+
// Use same source port.
39+
//
40+
41+
+0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
42+
43+
+0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
44+
+0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
45+
46+
+1.00 connect(3, ..., ...) = -1 EADDRNOTAVAIL (Cannot assign requested address)
47+
48+
+0 close(3) = 0

‎gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_fail_after_990msec.pkt‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
// Check that TCP 4-tuple can't be reincarnated when we are close to TIME-WAIT
2+
// reuse threshold (990 msec have elapsed).
3+
4+
--ip_version=ipv4
5+
--tcp_ts_tick_usecs=1000
6+
--tolerance_percent=1
7+
8+
`
9+
../../common/defaults.sh
10+
../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse=1 # global enable
11+
../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse_delay=1000 # msec
12+
`
13+
14+
//
15+
// Prime TIME-WAIT - open first connection and close it.
16+
// Use source port 0xdead (57005).
17+
//
18+
19+
0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
20+
21+
+0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
22+
+0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
23+
24+
+0 connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress)
25+
26+
+0 > S 0:0(0) <mss 1460,sackOK,TS val 1001 ecr 0,nop,wscale 8>
27+
+.001 < S. 0:0(0) ack 1 win 65535 <mss 1460,sackOK,TS val 3001 ecr 1001,nop,wscale 8>
28+
+0 > . 1:1(0) ack 1 <nop,nop,TS val 1002 ecr 3001>
29+
30+
+0 close(3) = 0
31+
32+
+0 > F. 1:1(0) ack 1 <nop,nop,TS val 1002 ecr 3001>
33+
+.001 < F. 1:1(0) ack 2 win 8 <nop,nop,TS val 3002 ecr 1002>
34+
+0 > . 2:2(0) ack 2 <nop,nop,TS val 1003 ecr 3002>
35+
36+
//
37+
// Reincarnation after almost 1 second - open and close second connection.
38+
// Use the same source port.
39+
//
40+
41+
+0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
42+
43+
+0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
44+
+0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
45+
46+
+0.99 connect(3, ..., ...) = -1 EADDRNOTAVAIL (Cannot assign requested address)
47+
48+
+0 close(3) = 0

‎gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_ok_after_0sec_with_portlock.pkt‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// Check that TIME-WAIT 4-tuple can be reused immediately if bind() was used to
2+
// reserve the source port.
3+
4+
--ip_version=ipv4
5+
--bind_port=57005
6+
--tcp_ts_tick_usecs=1000
7+
--tolerance_percent=1
8+
9+
`
10+
../../common/defaults.sh
11+
../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse=1 # global enable
12+
../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse_delay=1000 # msec
13+
`
14+
15+
//
16+
// Prime TIME-WAIT - open first connection and close it.
17+
// Use source port 57005. Reserve it with bind().
18+
//
19+
20+
0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
21+
22+
+0 setsockopt(3, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0
23+
+0 bind(3, ..., ...) = 0
24+
25+
+0 connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress)
26+
27+
+0 > S 0:0(0) <mss 1460,sackOK,TS val 1001 ecr 0,nop,wscale 8>
28+
+.001 < S. 0:0(0) ack 1 win 65535 <mss 1460,sackOK,TS val 3001 ecr 1001,nop,wscale 8>
29+
+0 > . 1:1(0) ack 1 <nop,nop,TS val 1002 ecr 3001>
30+
31+
+0 close(3) = 0
32+
33+
+0 > F. 1:1(0) ack 1 <nop,nop,TS val 1002 ecr 3001>
34+
+.001 < F. 1:1(0) ack 2 win 8 <nop,nop,TS val 3002 ecr 1002>
35+
+0 > . 2:2(0) ack 2 <nop,nop,TS val 1003 ecr 3002>
36+
37+
//
38+
// Reincarnation without delay - open and close second connection.
39+
// Use same source port.
40+
//
41+
42+
+0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
43+
44+
+0 setsockopt(3, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0
45+
+0 bind(3, ..., ...) = 0
46+
47+
+0.00 connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress)
48+
49+
// NOTE: Packetdrill doesn't actually track timestamps across connection
50+
// reincarnations despite the fact that the random offset doesn't change.
51+
+0 > S 0:0(0) <mss 1460,sackOK,TS val 1003 ecr 0,nop,wscale 8>
52+
+.001 < S. 0:0(0) ack 1 win 65535 <mss 1460,sackOK,TS val 3002 ecr 1003,nop,wscale 8>
53+
+0 > . 1:1(0) ack 1 <nop,nop,TS val 1004 ecr 3002>
54+
55+
+0 close(3) = 0
56+
57+
+0 > F. 1:1(0) ack 1 <nop,nop,TS val 1004 ecr 3002>
58+
+.001 < F. 1:1(0) ack 2 win 8 <nop,nop,TS val 3003 ecr 1004>
59+
+0 > . 2:2(0) ack 2 <nop,nop,TS val 1005 ecr 3003>

‎gtests/net/tcp/ts_recent/tw_reuse/tw_reuse_ok_after_1sec.pkt‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
// Check that TIME-WAIT 4-tuple can be reused after a 1 second period.
2+
3+
--ip_version=ipv4
4+
--tcp_ts_tick_usecs=1000
5+
--tolerance_percent=1
6+
7+
`
8+
../../common/defaults.sh
9+
../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse=1 # global enable
10+
../../../tcp/common/set_sysctls.py /proc/sys/net/ipv4/tcp_tw_reuse_delay=1000 # msec
11+
`
12+
13+
//
14+
// Prime TIME-WAIT - open first connection and close it.
15+
// Use source port 0xdead (57005).
16+
//
17+
18+
0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
19+
20+
+0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
21+
+0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
22+
23+
+0 connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress)
24+
25+
+0 > S 0:0(0) <mss 1460,sackOK,TS val 1001 ecr 0,nop,wscale 8>
26+
+.001 < S. 0:0(0) ack 1 win 65535 <mss 1460,sackOK,TS val 3001 ecr 1001,nop,wscale 8>
27+
+0 > . 1:1(0) ack 1 <...>
28+
29+
+0 close(3) = 0
30+
31+
+0 > F. 1:1(0) ack 1 <nop,nop,TS val 1002 ecr 3001>
32+
+.001 < F. 1:1(0) ack 2 win 8 <nop,nop,TS val 3002 ecr 1002>
33+
+0 > . 2:2(0) ack 2 <nop,nop,TS val 1003 ecr 3002>
34+
35+
//
36+
// Reincarnation after 1 second - open and close second connection.
37+
// Use the same source port.
38+
//
39+
40+
+0 socket(..., SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP) = 3
41+
42+
+0 setsockopt(3, SOL_IP, IP_BIND_ADDRESS_NO_PORT, [1], 4) = 0
43+
+0 setsockopt(3, SOL_IP, IP_LOCAL_PORT_RANGE, [0xdeaddead], 4) = 0
44+
45+
// Avoid spurious failures with an extra 1 millisecond delay.
46+
// Kernel TIME-WAIT reuse timer has millisecond resolution. Actual reuse delay
47+
// can be up to 1 msec longer than the setting due to timestamp rounding.
48+
+1.01 connect(3, ..., ...) = -1 EINPROGRESS (Operation now in progress)
49+
50+
// NOTE: Packetdrill doesn't actually track timestamps across connection
51+
// reincarnations despite the fact that the random offset doesn't change.
52+
+0 > S 0:0(0) <mss 1460,sackOK,TS val 2004 ecr 3002,nop,wscale 8>
53+
+.001 < S. 0:0(0) ack 1 win 65535 <mss 1460,sackOK,TS val 4003 ecr 2004,nop,wscale 8>
54+
+0 > . 1:1(0) ack 1 <nop,nop,TS val 2005 ecr 4003>
55+
56+
+0 close(3) = 0
57+
58+
+0 > F. 1:1(0) ack 1 <nop,nop,TS val 2005 ecr 4003>
59+
+.001 < F. 1:1(0) ack 2 win 8 <nop,nop,TS val 4004 ecr 2005>
60+
+0 > . 2:2(0) ack 2 <nop,nop,TS val 2006 ecr 4004>

0 commit comments

Comments
 (0)