1#!/bin/bash
2# SPDX-License-Identifier: GPL-2.0
3
4# Double quotes to prevent globbing and word splitting is recommended in new
5# code but we accept it, especially because there were too many before having
6# address all other issues detected by shellcheck.
7#shellcheck disable=SC2086
8
9ret=0
10sin=""
11sinfail=""
12sout=""
13cin=""
14cinfail=""
15cinsent=""
16cout=""
17capout=""
18ns1=""
19ns2=""
20ksft_skip=4
21timeout_poll=30
22timeout_test=$((timeout_poll * 2 + 1))
23capture=0
24checksum=0
25ip_mptcp=0
26check_invert=0
27validate_checksum=0
28init=0
29evts_ns1=""
30evts_ns2=""
31evts_ns1_pid=0
32evts_ns2_pid=0
33
34declare -A all_tests
35declare -a only_tests_ids
36declare -a only_tests_names
37declare -A failed_tests
38TEST_COUNT=0
39TEST_NAME=""
40nr_blank=40
41
42export FAILING_LINKS=""
43
44# generated using "nfbpf_compile '(ip && (ip[54] & 0xf0) == 0x30) ||
45#				  (ip6 && (ip6[74] & 0xf0) == 0x30)'"
46CBPF_MPTCP_SUBOPTION_ADD_ADDR="14,
47			       48 0 0 0,
48			       84 0 0 240,
49			       21 0 3 64,
50			       48 0 0 54,
51			       84 0 0 240,
52			       21 6 7 48,
53			       48 0 0 0,
54			       84 0 0 240,
55			       21 0 4 96,
56			       48 0 0 74,
57			       84 0 0 240,
58			       21 0 1 48,
59			       6 0 0 65535,
60			       6 0 0 0"
61
62init_partial()
63{
64	capout=$(mktemp)
65
66	local sec rndh
67	sec=$(date +%s)
68	rndh=$(printf %x $sec)-$(mktemp -u XXXXXX)
69
70	ns1="ns1-$rndh"
71	ns2="ns2-$rndh"
72
73	local netns
74	for netns in "$ns1" "$ns2"; do
75		ip netns add $netns || exit $ksft_skip
76		ip -net $netns link set lo up
77		ip netns exec $netns sysctl -q net.mptcp.enabled=1
78		ip netns exec $netns sysctl -q net.mptcp.pm_type=0
79		ip netns exec $netns sysctl -q net.ipv4.conf.all.rp_filter=0
80		ip netns exec $netns sysctl -q net.ipv4.conf.default.rp_filter=0
81		if [ $checksum -eq 1 ]; then
82			ip netns exec $netns sysctl -q net.mptcp.checksum_enabled=1
83		fi
84	done
85
86	check_invert=0
87	validate_checksum=$checksum
88	FAILING_LINKS=""
89
90	#  ns1         ns2
91	# ns1eth1    ns2eth1
92	# ns1eth2    ns2eth2
93	# ns1eth3    ns2eth3
94	# ns1eth4    ns2eth4
95
96	local i
97	for i in $(seq 1 4); do
98		ip link add ns1eth$i netns "$ns1" type veth peer name ns2eth$i netns "$ns2"
99		ip -net "$ns1" addr add 10.0.$i.1/24 dev ns1eth$i
100		ip -net "$ns1" addr add dead:beef:$i::1/64 dev ns1eth$i nodad
101		ip -net "$ns1" link set ns1eth$i up
102
103		ip -net "$ns2" addr add 10.0.$i.2/24 dev ns2eth$i
104		ip -net "$ns2" addr add dead:beef:$i::2/64 dev ns2eth$i nodad
105		ip -net "$ns2" link set ns2eth$i up
106
107		# let $ns2 reach any $ns1 address from any interface
108		ip -net "$ns2" route add default via 10.0.$i.1 dev ns2eth$i metric 10$i
109		ip -net "$ns2" route add default via dead:beef:$i::1 dev ns2eth$i metric 10$i
110	done
111}
112
113init_shapers()
114{
115	local i
116	for i in $(seq 1 4); do
117		tc -n $ns1 qdisc add dev ns1eth$i root netem rate 20mbit delay 1
118		tc -n $ns2 qdisc add dev ns2eth$i root netem rate 20mbit delay 1
119	done
120}
121
122cleanup_partial()
123{
124	rm -f "$capout"
125
126	local netns
127	for netns in "$ns1" "$ns2"; do
128		ip netns del $netns
129		rm -f /tmp/$netns.{nstat,out}
130	done
131}
132
133check_tools()
134{
135	if ! ip -Version &> /dev/null; then
136		echo "SKIP: Could not run test without ip tool"
137		exit $ksft_skip
138	fi
139
140	if ! iptables -V &> /dev/null; then
141		echo "SKIP: Could not run all tests without iptables tool"
142		exit $ksft_skip
143	fi
144
145	if ! ip6tables -V &> /dev/null; then
146		echo "SKIP: Could not run all tests without ip6tables tool"
147		exit $ksft_skip
148	fi
149}
150
151init() {
152	init=1
153
154	check_tools
155
156	sin=$(mktemp)
157	sout=$(mktemp)
158	cin=$(mktemp)
159	cinsent=$(mktemp)
160	cout=$(mktemp)
161	evts_ns1=$(mktemp)
162	evts_ns2=$(mktemp)
163
164	trap cleanup EXIT
165
166	make_file "$cin" "client" 1
167	make_file "$sin" "server" 1
168}
169
170cleanup()
171{
172	rm -f "$cin" "$cout" "$sinfail"
173	rm -f "$sin" "$sout" "$cinsent" "$cinfail"
174	rm -rf $evts_ns1 $evts_ns2
175	cleanup_partial
176}
177
178skip_test()
179{
180	if [ "${#only_tests_ids[@]}" -eq 0 ] && [ "${#only_tests_names[@]}" -eq 0 ]; then
181		return 1
182	fi
183
184	local i
185	for i in "${only_tests_ids[@]}"; do
186		if [ "${TEST_COUNT}" -eq "${i}" ]; then
187			return 1
188		fi
189	done
190	for i in "${only_tests_names[@]}"; do
191		if [ "${TEST_NAME}" = "${i}" ]; then
192			return 1
193		fi
194	done
195
196	return 0
197}
198
199# $1: test name
200reset()
201{
202	TEST_NAME="${1}"
203
204	TEST_COUNT=$((TEST_COUNT+1))
205
206	if skip_test; then
207		return 1
208	fi
209
210	if [ "${init}" != "1" ]; then
211		init
212	else
213		cleanup_partial
214	fi
215
216	init_partial
217
218	return 0
219}
220
221# $1: test name
222reset_with_cookies()
223{
224	reset "${1}" || return 1
225
226	local netns
227	for netns in "$ns1" "$ns2"; do
228		ip netns exec $netns sysctl -q net.ipv4.tcp_syncookies=2
229	done
230}
231
232# $1: test name
233reset_with_add_addr_timeout()
234{
235	local ip="${2:-4}"
236	local tables
237
238	reset "${1}" || return 1
239
240	tables="iptables"
241	if [ $ip -eq 6 ]; then
242		tables="ip6tables"
243	fi
244
245	ip netns exec $ns1 sysctl -q net.mptcp.add_addr_timeout=1
246	ip netns exec $ns2 $tables -A OUTPUT -p tcp \
247		-m tcp --tcp-option 30 \
248		-m bpf --bytecode \
249		"$CBPF_MPTCP_SUBOPTION_ADD_ADDR" \
250		-j DROP
251}
252
253# $1: test name
254reset_with_checksum()
255{
256	local ns1_enable=$1
257	local ns2_enable=$2
258
259	reset "checksum test ${1} ${2}" || return 1
260
261	ip netns exec $ns1 sysctl -q net.mptcp.checksum_enabled=$ns1_enable
262	ip netns exec $ns2 sysctl -q net.mptcp.checksum_enabled=$ns2_enable
263
264	validate_checksum=1
265}
266
267reset_with_allow_join_id0()
268{
269	local ns1_enable=$2
270	local ns2_enable=$3
271
272	reset "${1}" || return 1
273
274	ip netns exec $ns1 sysctl -q net.mptcp.allow_join_initial_addr_port=$ns1_enable
275	ip netns exec $ns2 sysctl -q net.mptcp.allow_join_initial_addr_port=$ns2_enable
276}
277
278# Modify TCP payload without corrupting the TCP packet
279#
280# This rule inverts a 8-bit word at byte offset 148 for the 2nd TCP ACK packets
281# carrying enough data.
282# Once it is done, the TCP Checksum field is updated so the packet is still
283# considered as valid at the TCP level.
284# Because the MPTCP checksum, covering the TCP options and data, has not been
285# updated, the modification will be detected and an MP_FAIL will be emitted:
286# what we want to validate here without corrupting "random" MPTCP options.
287#
288# To avoid having tc producing this pr_info() message for each TCP ACK packets
289# not carrying enough data:
290#
291#     tc action pedit offset 162 out of bounds
292#
293# Netfilter is used to mark packets with enough data.
294reset_with_fail()
295{
296	reset "${1}" || return 1
297
298	ip netns exec $ns1 sysctl -q net.mptcp.checksum_enabled=1
299	ip netns exec $ns2 sysctl -q net.mptcp.checksum_enabled=1
300
301	check_invert=1
302	validate_checksum=1
303	local i="$2"
304	local ip="${3:-4}"
305	local tables
306
307	tables="iptables"
308	if [ $ip -eq 6 ]; then
309		tables="ip6tables"
310	fi
311
312	ip netns exec $ns2 $tables \
313		-t mangle \
314		-A OUTPUT \
315		-o ns2eth$i \
316		-p tcp \
317		-m length --length 150:9999 \
318		-m statistic --mode nth --packet 1 --every 99999 \
319		-j MARK --set-mark 42 || exit 1
320
321	tc -n $ns2 qdisc add dev ns2eth$i clsact || exit 1
322	tc -n $ns2 filter add dev ns2eth$i egress \
323		protocol ip prio 1000 \
324		handle 42 fw \
325		action pedit munge offset 148 u8 invert \
326		pipe csum tcp \
327		index 100 || exit 1
328}
329
330reset_with_events()
331{
332	reset "${1}" || return 1
333
334	:> "$evts_ns1"
335	:> "$evts_ns2"
336	ip netns exec $ns1 ./pm_nl_ctl events >> "$evts_ns1" 2>&1 &
337	evts_ns1_pid=$!
338	ip netns exec $ns2 ./pm_nl_ctl events >> "$evts_ns2" 2>&1 &
339	evts_ns2_pid=$!
340}
341
342fail_test()
343{
344	ret=1
345	failed_tests[${TEST_COUNT}]="${TEST_NAME}"
346}
347
348get_failed_tests_ids()
349{
350	# sorted
351	local i
352	for i in "${!failed_tests[@]}"; do
353		echo "${i}"
354	done | sort -n
355}
356
357print_file_err()
358{
359	ls -l "$1" 1>&2
360	echo "Trailing bytes are: "
361	tail -c 27 "$1"
362}
363
364check_transfer()
365{
366	local in=$1
367	local out=$2
368	local what=$3
369	local bytes=$4
370	local i a b
371
372	local line
373	if [ -n "$bytes" ]; then
374		# when truncating we must check the size explicitly
375		local out_size=$(wc -c $out | awk '{print $1}')
376		if [ $out_size -ne $bytes ]; then
377			echo "[ FAIL ] $what output file has wrong size ($out_size, $bytes)"
378			fail_test
379			return 1
380		fi
381		bytes="--bytes=${bytes}"
382	fi
383	cmp -l "$in" "$out" ${bytes} | while read -r i a b; do
384		local sum=$((0${a} + 0${b}))
385		if [ $check_invert -eq 0 ] || [ $sum -ne $((0xff)) ]; then
386			echo "[ FAIL ] $what does not match (in, out):"
387			print_file_err "$in"
388			print_file_err "$out"
389			fail_test
390
391			return 1
392		else
393			echo "$what has inverted byte at ${i}"
394		fi
395	done
396
397	return 0
398}
399
400do_ping()
401{
402	local listener_ns="$1"
403	local connector_ns="$2"
404	local connect_addr="$3"
405
406	if ! ip netns exec ${connector_ns} ping -q -c 1 $connect_addr >/dev/null; then
407		echo "$listener_ns -> $connect_addr connectivity [ FAIL ]" 1>&2
408		fail_test
409	fi
410}
411
412link_failure()
413{
414	local ns="$1"
415
416	if [ -z "$FAILING_LINKS" ]; then
417		l=$((RANDOM%4))
418		FAILING_LINKS=$((l+1))
419	fi
420
421	local l
422	for l in $FAILING_LINKS; do
423		local veth="ns1eth$l"
424		ip -net "$ns" link set "$veth" down
425	done
426}
427
428# $1: IP address
429is_v6()
430{
431	[ -z "${1##*:*}" ]
432}
433
434# $1: ns, $2: port
435wait_local_port_listen()
436{
437	local listener_ns="${1}"
438	local port="${2}"
439
440	local port_hex
441	port_hex="$(printf "%04X" "${port}")"
442
443	local i
444	for i in $(seq 10); do
445		ip netns exec "${listener_ns}" cat /proc/net/tcp* | \
446			awk "BEGIN {rc=1} {if (\$2 ~ /:${port_hex}\$/ && \$4 ~ /0A/) {rc=0; exit}} END {exit rc}" &&
447			break
448		sleep 0.1
449	done
450}
451
452rm_addr_count()
453{
454	local ns=${1}
455
456	ip netns exec ${ns} nstat -as | grep MPTcpExtRmAddr | awk '{print $2}'
457}
458
459# $1: ns, $2: old rm_addr counter in $ns
460wait_rm_addr()
461{
462	local ns="${1}"
463	local old_cnt="${2}"
464	local cnt
465
466	local i
467	for i in $(seq 10); do
468		cnt=$(rm_addr_count ${ns})
469		[ "$cnt" = "${old_cnt}" ] || break
470		sleep 0.1
471	done
472}
473
474wait_mpj()
475{
476	local ns="${1}"
477	local cnt old_cnt
478
479	old_cnt=$(ip netns exec ${ns} nstat -as | grep MPJoinAckRx | awk '{print $2}')
480
481	local i
482	for i in $(seq 10); do
483		cnt=$(ip netns exec ${ns} nstat -as | grep MPJoinAckRx | awk '{print $2}')
484		[ "$cnt" = "${old_cnt}" ] || break
485		sleep 0.1
486	done
487}
488
489kill_wait()
490{
491	kill $1 > /dev/null 2>&1
492	wait $1 2>/dev/null
493}
494
495kill_events_pids()
496{
497	kill_wait $evts_ns1_pid
498	kill_wait $evts_ns2_pid
499}
500
501kill_tests_wait()
502{
503	kill -SIGUSR1 $(ip netns pids $ns2) $(ip netns pids $ns1)
504	wait
505}
506
507pm_nl_set_limits()
508{
509	local ns=$1
510	local addrs=$2
511	local subflows=$3
512
513	if [ $ip_mptcp -eq 1 ]; then
514		ip -n $ns mptcp limits set add_addr_accepted $addrs subflows $subflows
515	else
516		ip netns exec $ns ./pm_nl_ctl limits $addrs $subflows
517	fi
518}
519
520pm_nl_add_endpoint()
521{
522	local ns=$1
523	local addr=$2
524	local flags _flags
525	local port _port
526	local dev _dev
527	local id _id
528	local nr=2
529
530	local p
531	for p in "${@}"
532	do
533		if [ $p = "flags" ]; then
534			eval _flags=\$"$nr"
535			[ -n "$_flags" ]; flags="flags $_flags"
536		fi
537		if [ $p = "dev" ]; then
538			eval _dev=\$"$nr"
539			[ -n "$_dev" ]; dev="dev $_dev"
540		fi
541		if [ $p = "id" ]; then
542			eval _id=\$"$nr"
543			[ -n "$_id" ]; id="id $_id"
544		fi
545		if [ $p = "port" ]; then
546			eval _port=\$"$nr"
547			[ -n "$_port" ]; port="port $_port"
548		fi
549
550		nr=$((nr + 1))
551	done
552
553	if [ $ip_mptcp -eq 1 ]; then
554		ip -n $ns mptcp endpoint add $addr ${_flags//","/" "} $dev $id $port
555	else
556		ip netns exec $ns ./pm_nl_ctl add $addr $flags $dev $id $port
557	fi
558}
559
560pm_nl_del_endpoint()
561{
562	local ns=$1
563	local id=$2
564	local addr=$3
565
566	if [ $ip_mptcp -eq 1 ]; then
567		ip -n $ns mptcp endpoint delete id $id $addr
568	else
569		ip netns exec $ns ./pm_nl_ctl del $id $addr
570	fi
571}
572
573pm_nl_flush_endpoint()
574{
575	local ns=$1
576
577	if [ $ip_mptcp -eq 1 ]; then
578		ip -n $ns mptcp endpoint flush
579	else
580		ip netns exec $ns ./pm_nl_ctl flush
581	fi
582}
583
584pm_nl_show_endpoints()
585{
586	local ns=$1
587
588	if [ $ip_mptcp -eq 1 ]; then
589		ip -n $ns mptcp endpoint show
590	else
591		ip netns exec $ns ./pm_nl_ctl dump
592	fi
593}
594
595pm_nl_change_endpoint()
596{
597	local ns=$1
598	local id=$2
599	local flags=$3
600
601	if [ $ip_mptcp -eq 1 ]; then
602		ip -n $ns mptcp endpoint change id $id ${flags//","/" "}
603	else
604		ip netns exec $ns ./pm_nl_ctl set id $id flags $flags
605	fi
606}
607
608pm_nl_check_endpoint()
609{
610	local line expected_line
611	local need_title=$1
612	local msg="$2"
613	local ns=$3
614	local addr=$4
615	local _flags=""
616	local flags
617	local _port
618	local port
619	local dev
620	local _id
621	local id
622
623	if [ "${need_title}" = 1 ]; then
624		printf "%03u %-36s %s" "${TEST_COUNT}" "${TEST_NAME}" "${msg}"
625	else
626		printf "%-${nr_blank}s %s" " " "${msg}"
627	fi
628
629	shift 4
630	while [ -n "$1" ]; do
631		if [ $1 = "flags" ]; then
632			_flags=$2
633			[ -n "$_flags" ]; flags="flags $_flags"
634			shift
635		elif [ $1 = "dev" ]; then
636			[ -n "$2" ]; dev="dev $1"
637			shift
638		elif [ $1 = "id" ]; then
639			_id=$2
640			[ -n "$_id" ]; id="id $_id"
641			shift
642		elif [ $1 = "port" ]; then
643			_port=$2
644			[ -n "$_port" ]; port=" port $_port"
645			shift
646		fi
647
648		shift
649	done
650
651	if [ -z "$id" ]; then
652		echo "[skip] bad test - missing endpoint id"
653		return
654	fi
655
656	if [ $ip_mptcp -eq 1 ]; then
657		line=$(ip -n $ns mptcp endpoint show $id)
658		# the dump order is: address id flags port dev
659		expected_line="$addr"
660		[ -n "$addr" ] && expected_line="$expected_line $addr"
661		expected_line="$expected_line $id"
662		[ -n "$_flags" ] && expected_line="$expected_line ${_flags//","/" "}"
663		[ -n "$dev" ] && expected_line="$expected_line $dev"
664		[ -n "$port" ] && expected_line="$expected_line $port"
665	else
666		line=$(ip netns exec $ns ./pm_nl_ctl get $_id)
667		# the dump order is: id flags dev address port
668		expected_line="$id"
669		[ -n "$flags" ] && expected_line="$expected_line $flags"
670		[ -n "$dev" ] && expected_line="$expected_line $dev"
671		[ -n "$addr" ] && expected_line="$expected_line $addr"
672		[ -n "$_port" ] && expected_line="$expected_line $_port"
673	fi
674	if [ "$line" = "$expected_line" ]; then
675		echo "[ ok ]"
676	else
677		echo "[fail] expected '$expected_line' found '$line'"
678		fail_test
679	fi
680}
681
682filter_tcp_from()
683{
684	local ns="${1}"
685	local src="${2}"
686	local target="${3}"
687
688	ip netns exec "${ns}" iptables -A INPUT -s "${src}" -p tcp -j "${target}"
689}
690
691do_transfer()
692{
693	local listener_ns="$1"
694	local connector_ns="$2"
695	local cl_proto="$3"
696	local srv_proto="$4"
697	local connect_addr="$5"
698	local test_link_fail="$6"
699	local addr_nr_ns1="$7"
700	local addr_nr_ns2="$8"
701	local speed="$9"
702	local sflags="${10}"
703
704	local port=$((10000 + TEST_COUNT - 1))
705	local cappid
706	local userspace_pm=0
707
708	:> "$cout"
709	:> "$sout"
710	:> "$capout"
711
712	if [ $capture -eq 1 ]; then
713		local capuser
714		if [ -z $SUDO_USER ] ; then
715			capuser=""
716		else
717			capuser="-Z $SUDO_USER"
718		fi
719
720		capfile=$(printf "mp_join-%02u-%s.pcap" "$TEST_COUNT" "${listener_ns}")
721
722		echo "Capturing traffic for test $TEST_COUNT into $capfile"
723		ip netns exec ${listener_ns} tcpdump -i any -s 65535 -B 32768 $capuser -w $capfile > "$capout" 2>&1 &
724		cappid=$!
725
726		sleep 1
727	fi
728
729	NSTAT_HISTORY=/tmp/${listener_ns}.nstat ip netns exec ${listener_ns} \
730		nstat -n
731	NSTAT_HISTORY=/tmp/${connector_ns}.nstat ip netns exec ${connector_ns} \
732		nstat -n
733
734	local extra_args
735	if [ $speed = "fast" ]; then
736		extra_args="-j"
737	elif [ $speed = "slow" ]; then
738		extra_args="-r 50"
739	elif [[ $speed = "speed_"* ]]; then
740		extra_args="-r ${speed:6}"
741	fi
742
743	if [[ "${addr_nr_ns1}" = "userspace_"* ]]; then
744		userspace_pm=1
745		addr_nr_ns1=${addr_nr_ns1:10}
746	fi
747
748	local flags="subflow"
749	local extra_cl_args=""
750	local extra_srv_args=""
751	local trunc_size=""
752	if [[ "${addr_nr_ns2}" = "fastclose_"* ]]; then
753		if [ ${test_link_fail} -le 1 ]; then
754			echo "fastclose tests need test_link_fail argument"
755			fail_test
756			return 1
757		fi
758
759		# disconnect
760		trunc_size=${test_link_fail}
761		local side=${addr_nr_ns2:10}
762
763		if [ ${side} = "client" ]; then
764			extra_cl_args="-f ${test_link_fail}"
765			extra_srv_args="-f -1"
766		elif [ ${side} = "server" ]; then
767			extra_srv_args="-f ${test_link_fail}"
768			extra_cl_args="-f -1"
769		else
770			echo "wrong/unknown fastclose spec ${side}"
771			fail_test
772			return 1
773		fi
774		addr_nr_ns2=0
775	elif [[ "${addr_nr_ns2}" = "userspace_"* ]]; then
776		userspace_pm=1
777		addr_nr_ns2=${addr_nr_ns2:10}
778	elif [[ "${addr_nr_ns2}" = "fullmesh_"* ]]; then
779		flags="${flags},fullmesh"
780		addr_nr_ns2=${addr_nr_ns2:9}
781	fi
782
783	local local_addr
784	if is_v6 "${connect_addr}"; then
785		local_addr="::"
786	else
787		local_addr="0.0.0.0"
788	fi
789
790	extra_srv_args="$extra_args $extra_srv_args"
791	if [ "$test_link_fail" -gt 1 ];then
792		timeout ${timeout_test} \
793			ip netns exec ${listener_ns} \
794				./mptcp_connect -t ${timeout_poll} -l -p $port -s ${srv_proto} \
795					$extra_srv_args ${local_addr} < "$sinfail" > "$sout" &
796	else
797		timeout ${timeout_test} \
798			ip netns exec ${listener_ns} \
799				./mptcp_connect -t ${timeout_poll} -l -p $port -s ${srv_proto} \
800					$extra_srv_args ${local_addr} < "$sin" > "$sout" &
801	fi
802	local spid=$!
803
804	wait_local_port_listen "${listener_ns}" "${port}"
805
806	extra_cl_args="$extra_args $extra_cl_args"
807	if [ "$test_link_fail" -eq 0 ];then
808		timeout ${timeout_test} \
809			ip netns exec ${connector_ns} \
810				./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \
811					$extra_cl_args $connect_addr < "$cin" > "$cout" &
812	elif [ "$test_link_fail" -eq 1 ] || [ "$test_link_fail" -eq 2 ];then
813		( cat "$cinfail" ; sleep 2; link_failure $listener_ns ; cat "$cinfail" ) | \
814			tee "$cinsent" | \
815			timeout ${timeout_test} \
816				ip netns exec ${connector_ns} \
817					./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \
818						$extra_cl_args $connect_addr > "$cout" &
819	else
820		tee "$cinsent" < "$cinfail" | \
821			timeout ${timeout_test} \
822				ip netns exec ${connector_ns} \
823					./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \
824						$extra_cl_args $connect_addr > "$cout" &
825	fi
826	local cpid=$!
827
828	# let the mptcp subflow be established in background before
829	# do endpoint manipulation
830	if [ $addr_nr_ns1 != "0" ] || [ $addr_nr_ns2 != "0" ]; then
831		sleep 1
832	fi
833
834	if [ $addr_nr_ns1 -gt 0 ]; then
835		local counter=2
836		local add_nr_ns1=${addr_nr_ns1}
837		local id=10
838		local tk
839		while [ $add_nr_ns1 -gt 0 ]; do
840			local addr
841			if is_v6 "${connect_addr}"; then
842				addr="dead:beef:$counter::1"
843			else
844				addr="10.0.$counter.1"
845			fi
846			if [ $userspace_pm -eq 0 ]; then
847				pm_nl_add_endpoint $ns1 $addr flags signal
848			else
849				tk=$(grep "type:1," "$evts_ns1" |
850				     sed -n 's/.*\(token:\)\([[:digit:]]*\).*$/\2/p;q')
851				ip netns exec ${listener_ns} ./pm_nl_ctl ann $addr token $tk id $id
852				sleep 1
853				ip netns exec ${listener_ns} ./pm_nl_ctl rem token $tk id $id
854			fi
855
856			counter=$((counter + 1))
857			add_nr_ns1=$((add_nr_ns1 - 1))
858			id=$((id + 1))
859		done
860	elif [ $addr_nr_ns1 -lt 0 ]; then
861		local rm_nr_ns1=$((-addr_nr_ns1))
862		if [ $rm_nr_ns1 -lt 8 ]; then
863			local counter=0
864			local line
865			pm_nl_show_endpoints ${listener_ns} | while read -r line; do
866				# shellcheck disable=SC2206 # we do want to split per word
867				local arr=($line)
868				local nr=0
869
870				local i
871				for i in "${arr[@]}"; do
872					if [ $i = "id" ]; then
873						if [ $counter -eq $rm_nr_ns1 ]; then
874							break
875						fi
876						id=${arr[$nr+1]}
877						rm_addr=$(rm_addr_count ${connector_ns})
878						pm_nl_del_endpoint ${listener_ns} $id
879						wait_rm_addr ${connector_ns} ${rm_addr}
880						counter=$((counter + 1))
881					fi
882					nr=$((nr + 1))
883				done
884			done
885		elif [ $rm_nr_ns1 -eq 8 ]; then
886			pm_nl_flush_endpoint ${listener_ns}
887		elif [ $rm_nr_ns1 -eq 9 ]; then
888			pm_nl_del_endpoint ${listener_ns} 0 ${connect_addr}
889		fi
890	fi
891
892	# if newly added endpoints must be deleted, give the background msk
893	# some time to created them
894	[ $addr_nr_ns1 -gt 0 ] && [ $addr_nr_ns2 -lt 0 ] && sleep 1
895
896	if [ $addr_nr_ns2 -gt 0 ]; then
897		local add_nr_ns2=${addr_nr_ns2}
898		local counter=3
899		local id=20
900		local tk da dp sp
901		while [ $add_nr_ns2 -gt 0 ]; do
902			local addr
903			if is_v6 "${connect_addr}"; then
904				addr="dead:beef:$counter::2"
905			else
906				addr="10.0.$counter.2"
907			fi
908			if [ $userspace_pm -eq 0 ]; then
909				pm_nl_add_endpoint $ns2 $addr flags $flags
910			else
911				tk=$(sed -n 's/.*\(token:\)\([[:digit:]]*\).*$/\2/p;q' "$evts_ns2")
912				da=$(sed -n 's/.*\(daddr4:\)\([0-9.]*\).*$/\2/p;q' "$evts_ns2")
913				dp=$(sed -n 's/.*\(dport:\)\([[:digit:]]*\).*$/\2/p;q' "$evts_ns2")
914				ip netns exec ${connector_ns} ./pm_nl_ctl csf lip $addr lid $id \
915									rip $da rport $dp token $tk
916				sleep 1
917				sp=$(grep "type:10" "$evts_ns2" |
918				     sed -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q')
919				ip netns exec ${connector_ns} ./pm_nl_ctl dsf lip $addr lport $sp \
920									rip $da rport $dp token $tk
921			fi
922			counter=$((counter + 1))
923			add_nr_ns2=$((add_nr_ns2 - 1))
924			id=$((id + 1))
925		done
926	elif [ $addr_nr_ns2 -lt 0 ]; then
927		local rm_nr_ns2=$((-addr_nr_ns2))
928		if [ $rm_nr_ns2 -lt 8 ]; then
929			local counter=0
930			local line
931			pm_nl_show_endpoints ${connector_ns} | while read -r line; do
932				# shellcheck disable=SC2206 # we do want to split per word
933				local arr=($line)
934				local nr=0
935
936				local i
937				for i in "${arr[@]}"; do
938					if [ $i = "id" ]; then
939						if [ $counter -eq $rm_nr_ns2 ]; then
940							break
941						fi
942						local id rm_addr
943						# rm_addr are serialized, allow the previous one to
944						# complete
945						id=${arr[$nr+1]}
946						rm_addr=$(rm_addr_count ${listener_ns})
947						pm_nl_del_endpoint ${connector_ns} $id
948						wait_rm_addr ${listener_ns} ${rm_addr}
949						counter=$((counter + 1))
950					fi
951					nr=$((nr + 1))
952				done
953			done
954		elif [ $rm_nr_ns2 -eq 8 ]; then
955			pm_nl_flush_endpoint ${connector_ns}
956		elif [ $rm_nr_ns2 -eq 9 ]; then
957			local addr
958			if is_v6 "${connect_addr}"; then
959				addr="dead:beef:1::2"
960			else
961				addr="10.0.1.2"
962			fi
963			pm_nl_del_endpoint ${connector_ns} 0 $addr
964		fi
965	fi
966
967	if [ -n "${sflags}" ]; then
968		sleep 1
969
970		local netns
971		for netns in "$ns1" "$ns2"; do
972			local line
973			pm_nl_show_endpoints $netns | while read -r line; do
974				# shellcheck disable=SC2206 # we do want to split per word
975				local arr=($line)
976				local nr=0
977				local id
978
979				local i
980				for i in "${arr[@]}"; do
981					if [ $i = "id" ]; then
982						id=${arr[$nr+1]}
983					fi
984					nr=$((nr + 1))
985				done
986				pm_nl_change_endpoint $netns $id $sflags
987			done
988		done
989	fi
990
991	wait $cpid
992	local retc=$?
993	wait $spid
994	local rets=$?
995
996	if [ $capture -eq 1 ]; then
997	    sleep 1
998	    kill $cappid
999	fi
1000
1001	NSTAT_HISTORY=/tmp/${listener_ns}.nstat ip netns exec ${listener_ns} \
1002		nstat | grep Tcp > /tmp/${listener_ns}.out
1003	NSTAT_HISTORY=/tmp/${connector_ns}.nstat ip netns exec ${connector_ns} \
1004		nstat | grep Tcp > /tmp/${connector_ns}.out
1005
1006	if [ ${rets} -ne 0 ] || [ ${retc} -ne 0 ]; then
1007		echo " client exit code $retc, server $rets" 1>&2
1008		echo -e "\nnetns ${listener_ns} socket stat for ${port}:" 1>&2
1009		ip netns exec ${listener_ns} ss -Menita 1>&2 -o "sport = :$port"
1010		cat /tmp/${listener_ns}.out
1011		echo -e "\nnetns ${connector_ns} socket stat for ${port}:" 1>&2
1012		ip netns exec ${connector_ns} ss -Menita 1>&2 -o "dport = :$port"
1013		cat /tmp/${connector_ns}.out
1014
1015		cat "$capout"
1016		fail_test
1017		return 1
1018	fi
1019
1020	if [ "$test_link_fail" -gt 1 ];then
1021		check_transfer $sinfail $cout "file received by client" $trunc_size
1022	else
1023		check_transfer $sin $cout "file received by client" $trunc_size
1024	fi
1025	retc=$?
1026	if [ "$test_link_fail" -eq 0 ];then
1027		check_transfer $cin $sout "file received by server" $trunc_size
1028	else
1029		check_transfer $cinsent $sout "file received by server" $trunc_size
1030	fi
1031	rets=$?
1032
1033	if [ $retc -eq 0 ] && [ $rets -eq 0 ];then
1034		cat "$capout"
1035		return 0
1036	fi
1037
1038	cat "$capout"
1039	return 1
1040}
1041
1042make_file()
1043{
1044	local name=$1
1045	local who=$2
1046	local size=$3
1047
1048	dd if=/dev/urandom of="$name" bs=1024 count=$size 2> /dev/null
1049	echo -e "\nMPTCP_TEST_FILE_END_MARKER" >> "$name"
1050
1051	echo "Created $name (size $size KB) containing data sent by $who"
1052}
1053
1054run_tests()
1055{
1056	local listener_ns="$1"
1057	local connector_ns="$2"
1058	local connect_addr="$3"
1059	local test_linkfail="${4:-0}"
1060	local addr_nr_ns1="${5:-0}"
1061	local addr_nr_ns2="${6:-0}"
1062	local speed="${7:-fast}"
1063	local sflags="${8:-""}"
1064
1065	local size
1066
1067	# The values above 2 are reused to make test files
1068	# with the given sizes (KB)
1069	if [ "$test_linkfail" -gt 2 ]; then
1070		size=$test_linkfail
1071
1072		if [ -z "$cinfail" ]; then
1073			cinfail=$(mktemp)
1074		fi
1075		make_file "$cinfail" "client" $size
1076	# create the input file for the failure test when
1077	# the first failure test run
1078	elif [ "$test_linkfail" -ne 0 ] && [ -z "$cinfail" ]; then
1079		# the client file must be considerably larger
1080		# of the maximum expected cwin value, or the
1081		# link utilization will be not predicable
1082		size=$((RANDOM%2))
1083		size=$((size+1))
1084		size=$((size*8192))
1085		size=$((size + ( RANDOM % 8192) ))
1086
1087		cinfail=$(mktemp)
1088		make_file "$cinfail" "client" $size
1089	fi
1090
1091	if [ "$test_linkfail" -gt 2 ]; then
1092		size=$test_linkfail
1093
1094		if [ -z "$sinfail" ]; then
1095			sinfail=$(mktemp)
1096		fi
1097		make_file "$sinfail" "server" $size
1098	elif [ "$test_linkfail" -eq 2 ] && [ -z "$sinfail" ]; then
1099		size=$((RANDOM%16))
1100		size=$((size+1))
1101		size=$((size*2048))
1102
1103		sinfail=$(mktemp)
1104		make_file "$sinfail" "server" $size
1105	fi
1106
1107	do_transfer ${listener_ns} ${connector_ns} MPTCP MPTCP ${connect_addr} \
1108		${test_linkfail} ${addr_nr_ns1} ${addr_nr_ns2} ${speed} ${sflags}
1109}
1110
1111dump_stats()
1112{
1113	echo Server ns stats
1114	ip netns exec $ns1 nstat -as | grep Tcp
1115	echo Client ns stats
1116	ip netns exec $ns2 nstat -as | grep Tcp
1117}
1118
1119chk_csum_nr()
1120{
1121	local csum_ns1=${1:-0}
1122	local csum_ns2=${2:-0}
1123	local count
1124	local dump_stats
1125	local extra_msg=""
1126	local allow_multi_errors_ns1=0
1127	local allow_multi_errors_ns2=0
1128
1129	if [[ "${csum_ns1}" = "+"* ]]; then
1130		allow_multi_errors_ns1=1
1131		csum_ns1=${csum_ns1:1}
1132	fi
1133	if [[ "${csum_ns2}" = "+"* ]]; then
1134		allow_multi_errors_ns2=1
1135		csum_ns2=${csum_ns2:1}
1136	fi
1137
1138	printf "%-${nr_blank}s %s" " " "sum"
1139	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtDataCsumErr | awk '{print $2}')
1140	[ -z "$count" ] && count=0
1141	if [ "$count" != "$csum_ns1" ]; then
1142		extra_msg="$extra_msg ns1=$count"
1143	fi
1144	if { [ "$count" != $csum_ns1 ] && [ $allow_multi_errors_ns1 -eq 0 ]; } ||
1145	   { [ "$count" -lt $csum_ns1 ] && [ $allow_multi_errors_ns1 -eq 1 ]; }; then
1146		echo "[fail] got $count data checksum error[s] expected $csum_ns1"
1147		fail_test
1148		dump_stats=1
1149	else
1150		echo -n "[ ok ]"
1151	fi
1152	echo -n " - csum  "
1153	count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtDataCsumErr | awk '{print $2}')
1154	[ -z "$count" ] && count=0
1155	if [ "$count" != "$csum_ns2" ]; then
1156		extra_msg="$extra_msg ns2=$count"
1157	fi
1158	if { [ "$count" != $csum_ns2 ] && [ $allow_multi_errors_ns2 -eq 0 ]; } ||
1159	   { [ "$count" -lt $csum_ns2 ] && [ $allow_multi_errors_ns2 -eq 1 ]; }; then
1160		echo "[fail] got $count data checksum error[s] expected $csum_ns2"
1161		fail_test
1162		dump_stats=1
1163	else
1164		echo -n "[ ok ]"
1165	fi
1166	[ "${dump_stats}" = 1 ] && dump_stats
1167
1168	echo "$extra_msg"
1169}
1170
1171chk_fail_nr()
1172{
1173	local fail_tx=$1
1174	local fail_rx=$2
1175	local ns_invert=${3:-""}
1176	local count
1177	local dump_stats
1178	local ns_tx=$ns1
1179	local ns_rx=$ns2
1180	local extra_msg=""
1181	local allow_tx_lost=0
1182	local allow_rx_lost=0
1183
1184	if [[ $ns_invert = "invert" ]]; then
1185		ns_tx=$ns2
1186		ns_rx=$ns1
1187		extra_msg=" invert"
1188	fi
1189
1190	if [[ "${fail_tx}" = "-"* ]]; then
1191		allow_tx_lost=1
1192		fail_tx=${fail_tx:1}
1193	fi
1194	if [[ "${fail_rx}" = "-"* ]]; then
1195		allow_rx_lost=1
1196		fail_rx=${fail_rx:1}
1197	fi
1198
1199	printf "%-${nr_blank}s %s" " " "ftx"
1200	count=$(ip netns exec $ns_tx nstat -as | grep MPTcpExtMPFailTx | awk '{print $2}')
1201	[ -z "$count" ] && count=0
1202	if [ "$count" != "$fail_tx" ]; then
1203		extra_msg="$extra_msg,tx=$count"
1204	fi
1205	if { [ "$count" != "$fail_tx" ] && [ $allow_tx_lost -eq 0 ]; } ||
1206	   { [ "$count" -gt "$fail_tx" ] && [ $allow_tx_lost -eq 1 ]; }; then
1207		echo "[fail] got $count MP_FAIL[s] TX expected $fail_tx"
1208		fail_test
1209		dump_stats=1
1210	else
1211		echo -n "[ ok ]"
1212	fi
1213
1214	echo -n " - failrx"
1215	count=$(ip netns exec $ns_rx nstat -as | grep MPTcpExtMPFailRx | awk '{print $2}')
1216	[ -z "$count" ] && count=0
1217	if [ "$count" != "$fail_rx" ]; then
1218		extra_msg="$extra_msg,rx=$count"
1219	fi
1220	if { [ "$count" != "$fail_rx" ] && [ $allow_rx_lost -eq 0 ]; } ||
1221	   { [ "$count" -gt "$fail_rx" ] && [ $allow_rx_lost -eq 1 ]; }; then
1222		echo "[fail] got $count MP_FAIL[s] RX expected $fail_rx"
1223		fail_test
1224		dump_stats=1
1225	else
1226		echo -n "[ ok ]"
1227	fi
1228
1229	[ "${dump_stats}" = 1 ] && dump_stats
1230
1231	echo "$extra_msg"
1232}
1233
1234chk_fclose_nr()
1235{
1236	local fclose_tx=$1
1237	local fclose_rx=$2
1238	local ns_invert=$3
1239	local count
1240	local dump_stats
1241	local ns_tx=$ns2
1242	local ns_rx=$ns1
1243	local extra_msg="   "
1244
1245	if [[ $ns_invert = "invert" ]]; then
1246		ns_tx=$ns1
1247		ns_rx=$ns2
1248		extra_msg=${extra_msg}"invert"
1249	fi
1250
1251	printf "%-${nr_blank}s %s" " " "ctx"
1252	count=$(ip netns exec $ns_tx nstat -as | grep MPTcpExtMPFastcloseTx | awk '{print $2}')
1253	[ -z "$count" ] && count=0
1254	[ "$count" != "$fclose_tx" ] && extra_msg="$extra_msg,tx=$count"
1255	if [ "$count" != "$fclose_tx" ]; then
1256		echo "[fail] got $count MP_FASTCLOSE[s] TX expected $fclose_tx"
1257		fail_test
1258		dump_stats=1
1259	else
1260		echo -n "[ ok ]"
1261	fi
1262
1263	echo -n " - fclzrx"
1264	count=$(ip netns exec $ns_rx nstat -as | grep MPTcpExtMPFastcloseRx | awk '{print $2}')
1265	[ -z "$count" ] && count=0
1266	[ "$count" != "$fclose_rx" ] && extra_msg="$extra_msg,rx=$count"
1267	if [ "$count" != "$fclose_rx" ]; then
1268		echo "[fail] got $count MP_FASTCLOSE[s] RX expected $fclose_rx"
1269		fail_test
1270		dump_stats=1
1271	else
1272		echo -n "[ ok ]"
1273	fi
1274
1275	[ "${dump_stats}" = 1 ] && dump_stats
1276
1277	echo "$extra_msg"
1278}
1279
1280chk_rst_nr()
1281{
1282	local rst_tx=$1
1283	local rst_rx=$2
1284	local ns_invert=${3:-""}
1285	local count
1286	local dump_stats
1287	local ns_tx=$ns1
1288	local ns_rx=$ns2
1289	local extra_msg=""
1290
1291	if [[ $ns_invert = "invert" ]]; then
1292		ns_tx=$ns2
1293		ns_rx=$ns1
1294		extra_msg="   invert"
1295	fi
1296
1297	printf "%-${nr_blank}s %s" " " "rtx"
1298	count=$(ip netns exec $ns_tx nstat -as | grep MPTcpExtMPRstTx | awk '{print $2}')
1299	[ -z "$count" ] && count=0
1300	if [ $count -lt $rst_tx ]; then
1301		echo "[fail] got $count MP_RST[s] TX expected $rst_tx"
1302		fail_test
1303		dump_stats=1
1304	else
1305		echo -n "[ ok ]"
1306	fi
1307
1308	echo -n " - rstrx "
1309	count=$(ip netns exec $ns_rx nstat -as | grep MPTcpExtMPRstRx | awk '{print $2}')
1310	[ -z "$count" ] && count=0
1311	if [ "$count" -lt "$rst_rx" ]; then
1312		echo "[fail] got $count MP_RST[s] RX expected $rst_rx"
1313		fail_test
1314		dump_stats=1
1315	else
1316		echo -n "[ ok ]"
1317	fi
1318
1319	[ "${dump_stats}" = 1 ] && dump_stats
1320
1321	echo "$extra_msg"
1322}
1323
1324chk_infi_nr()
1325{
1326	local infi_tx=$1
1327	local infi_rx=$2
1328	local count
1329	local dump_stats
1330
1331	printf "%-${nr_blank}s %s" " " "itx"
1332	count=$(ip netns exec $ns2 nstat -as | grep InfiniteMapTx | awk '{print $2}')
1333	[ -z "$count" ] && count=0
1334	if [ "$count" != "$infi_tx" ]; then
1335		echo "[fail] got $count infinite map[s] TX expected $infi_tx"
1336		fail_test
1337		dump_stats=1
1338	else
1339		echo -n "[ ok ]"
1340	fi
1341
1342	echo -n " - infirx"
1343	count=$(ip netns exec $ns1 nstat -as | grep InfiniteMapRx | awk '{print $2}')
1344	[ -z "$count" ] && count=0
1345	if [ "$count" != "$infi_rx" ]; then
1346		echo "[fail] got $count infinite map[s] RX expected $infi_rx"
1347		fail_test
1348		dump_stats=1
1349	else
1350		echo "[ ok ]"
1351	fi
1352
1353	[ "${dump_stats}" = 1 ] && dump_stats
1354}
1355
1356chk_join_nr()
1357{
1358	local syn_nr=$1
1359	local syn_ack_nr=$2
1360	local ack_nr=$3
1361	local csum_ns1=${4:-0}
1362	local csum_ns2=${5:-0}
1363	local fail_nr=${6:-0}
1364	local rst_nr=${7:-0}
1365	local infi_nr=${8:-0}
1366	local corrupted_pkts=${9:-0}
1367	local count
1368	local dump_stats
1369	local with_cookie
1370	local title="${TEST_NAME}"
1371
1372	if [ "${corrupted_pkts}" -gt 0 ]; then
1373		title+=": ${corrupted_pkts} corrupted pkts"
1374	fi
1375
1376	printf "%03u %-36s %s" "${TEST_COUNT}" "${title}" "syn"
1377	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinSynRx | awk '{print $2}')
1378	[ -z "$count" ] && count=0
1379	if [ "$count" != "$syn_nr" ]; then
1380		echo "[fail] got $count JOIN[s] syn expected $syn_nr"
1381		fail_test
1382		dump_stats=1
1383	else
1384		echo -n "[ ok ]"
1385	fi
1386
1387	echo -n " - synack"
1388	with_cookie=$(ip netns exec $ns2 sysctl -n net.ipv4.tcp_syncookies)
1389	count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtMPJoinSynAckRx | awk '{print $2}')
1390	[ -z "$count" ] && count=0
1391	if [ "$count" != "$syn_ack_nr" ]; then
1392		# simult connections exceeding the limit with cookie enabled could go up to
1393		# synack validation as the conn limit can be enforced reliably only after
1394		# the subflow creation
1395		if [ "$with_cookie" = 2 ] && [ "$count" -gt "$syn_ack_nr" ] && [ "$count" -le "$syn_nr" ]; then
1396			echo -n "[ ok ]"
1397		else
1398			echo "[fail] got $count JOIN[s] synack expected $syn_ack_nr"
1399			fail_test
1400			dump_stats=1
1401		fi
1402	else
1403		echo -n "[ ok ]"
1404	fi
1405
1406	echo -n " - ack"
1407	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinAckRx | awk '{print $2}')
1408	[ -z "$count" ] && count=0
1409	if [ "$count" != "$ack_nr" ]; then
1410		echo "[fail] got $count JOIN[s] ack expected $ack_nr"
1411		fail_test
1412		dump_stats=1
1413	else
1414		echo "[ ok ]"
1415	fi
1416	[ "${dump_stats}" = 1 ] && dump_stats
1417	if [ $validate_checksum -eq 1 ]; then
1418		chk_csum_nr $csum_ns1 $csum_ns2
1419		chk_fail_nr $fail_nr $fail_nr
1420		chk_rst_nr $rst_nr $rst_nr
1421		chk_infi_nr $infi_nr $infi_nr
1422	fi
1423}
1424
1425# a negative value for 'stale_max' means no upper bound:
1426# for bidirectional transfer, if one peer sleep for a while
1427# - as these tests do - we can have a quite high number of
1428# stale/recover conversions, proportional to
1429# sleep duration/ MPTCP-level RTX interval.
1430chk_stale_nr()
1431{
1432	local ns=$1
1433	local stale_min=$2
1434	local stale_max=$3
1435	local stale_delta=$4
1436	local dump_stats
1437	local stale_nr
1438	local recover_nr
1439
1440	printf "%-${nr_blank}s %-18s" " " "stale"
1441	stale_nr=$(ip netns exec $ns nstat -as | grep MPTcpExtSubflowStale | awk '{print $2}')
1442	[ -z "$stale_nr" ] && stale_nr=0
1443	recover_nr=$(ip netns exec $ns nstat -as | grep MPTcpExtSubflowRecover | awk '{print $2}')
1444	[ -z "$recover_nr" ] && recover_nr=0
1445
1446	if [ $stale_nr -lt $stale_min ] ||
1447	   { [ $stale_max -gt 0 ] && [ $stale_nr -gt $stale_max ]; } ||
1448	   [ $((stale_nr - recover_nr)) -ne $stale_delta ]; then
1449		echo "[fail] got $stale_nr stale[s] $recover_nr recover[s], " \
1450		     " expected stale in range [$stale_min..$stale_max]," \
1451		     " stale-recover delta $stale_delta "
1452		fail_test
1453		dump_stats=1
1454	else
1455		echo "[ ok ]"
1456	fi
1457
1458	if [ "${dump_stats}" = 1 ]; then
1459		echo $ns stats
1460		ip netns exec $ns ip -s link show
1461		ip netns exec $ns nstat -as | grep MPTcp
1462	fi
1463}
1464
1465chk_add_nr()
1466{
1467	local add_nr=$1
1468	local echo_nr=$2
1469	local port_nr=${3:-0}
1470	local syn_nr=${4:-$port_nr}
1471	local syn_ack_nr=${5:-$port_nr}
1472	local ack_nr=${6:-$port_nr}
1473	local mis_syn_nr=${7:-0}
1474	local mis_ack_nr=${8:-0}
1475	local count
1476	local dump_stats
1477	local timeout
1478
1479	timeout=$(ip netns exec $ns1 sysctl -n net.mptcp.add_addr_timeout)
1480
1481	printf "%-${nr_blank}s %s" " " "add"
1482	count=$(ip netns exec $ns2 nstat -as MPTcpExtAddAddr | grep MPTcpExtAddAddr | awk '{print $2}')
1483	[ -z "$count" ] && count=0
1484
1485	# if the test configured a short timeout tolerate greater then expected
1486	# add addrs options, due to retransmissions
1487	if [ "$count" != "$add_nr" ] && { [ "$timeout" -gt 1 ] || [ "$count" -lt "$add_nr" ]; }; then
1488		echo "[fail] got $count ADD_ADDR[s] expected $add_nr"
1489		fail_test
1490		dump_stats=1
1491	else
1492		echo -n "[ ok ]"
1493	fi
1494
1495	echo -n " - echo  "
1496	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtEchoAdd | awk '{print $2}')
1497	[ -z "$count" ] && count=0
1498	if [ "$count" != "$echo_nr" ]; then
1499		echo "[fail] got $count ADD_ADDR echo[s] expected $echo_nr"
1500		fail_test
1501		dump_stats=1
1502	else
1503		echo -n "[ ok ]"
1504	fi
1505
1506	if [ $port_nr -gt 0 ]; then
1507		echo -n " - pt "
1508		count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtPortAdd | awk '{print $2}')
1509		[ -z "$count" ] && count=0
1510		if [ "$count" != "$port_nr" ]; then
1511			echo "[fail] got $count ADD_ADDR[s] with a port-number expected $port_nr"
1512			fail_test
1513			dump_stats=1
1514		else
1515			echo "[ ok ]"
1516		fi
1517
1518		printf "%-${nr_blank}s %s" " " "syn"
1519		count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinPortSynRx |
1520			awk '{print $2}')
1521		[ -z "$count" ] && count=0
1522		if [ "$count" != "$syn_nr" ]; then
1523			echo "[fail] got $count JOIN[s] syn with a different \
1524				port-number expected $syn_nr"
1525			fail_test
1526			dump_stats=1
1527		else
1528			echo -n "[ ok ]"
1529		fi
1530
1531		echo -n " - synack"
1532		count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtMPJoinPortSynAckRx |
1533			awk '{print $2}')
1534		[ -z "$count" ] && count=0
1535		if [ "$count" != "$syn_ack_nr" ]; then
1536			echo "[fail] got $count JOIN[s] synack with a different \
1537				port-number expected $syn_ack_nr"
1538			fail_test
1539			dump_stats=1
1540		else
1541			echo -n "[ ok ]"
1542		fi
1543
1544		echo -n " - ack"
1545		count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinPortAckRx |
1546			awk '{print $2}')
1547		[ -z "$count" ] && count=0
1548		if [ "$count" != "$ack_nr" ]; then
1549			echo "[fail] got $count JOIN[s] ack with a different \
1550				port-number expected $ack_nr"
1551			fail_test
1552			dump_stats=1
1553		else
1554			echo "[ ok ]"
1555		fi
1556
1557		printf "%-${nr_blank}s %s" " " "syn"
1558		count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMismatchPortSynRx |
1559			awk '{print $2}')
1560		[ -z "$count" ] && count=0
1561		if [ "$count" != "$mis_syn_nr" ]; then
1562			echo "[fail] got $count JOIN[s] syn with a mismatched \
1563				port-number expected $mis_syn_nr"
1564			fail_test
1565			dump_stats=1
1566		else
1567			echo -n "[ ok ]"
1568		fi
1569
1570		echo -n " - ack   "
1571		count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMismatchPortAckRx |
1572			awk '{print $2}')
1573		[ -z "$count" ] && count=0
1574		if [ "$count" != "$mis_ack_nr" ]; then
1575			echo "[fail] got $count JOIN[s] ack with a mismatched \
1576				port-number expected $mis_ack_nr"
1577			fail_test
1578			dump_stats=1
1579		else
1580			echo "[ ok ]"
1581		fi
1582	else
1583		echo ""
1584	fi
1585
1586	[ "${dump_stats}" = 1 ] && dump_stats
1587}
1588
1589chk_rm_nr()
1590{
1591	local rm_addr_nr=$1
1592	local rm_subflow_nr=$2
1593	local invert
1594	local simult
1595	local count
1596	local dump_stats
1597	local addr_ns=$ns1
1598	local subflow_ns=$ns2
1599	local extra_msg=""
1600
1601	shift 2
1602	while [ -n "$1" ]; do
1603		[ "$1" = "invert" ] && invert=true
1604		[ "$1" = "simult" ] && simult=true
1605		shift
1606	done
1607
1608	if [ -z $invert ]; then
1609		addr_ns=$ns1
1610		subflow_ns=$ns2
1611	elif [ $invert = "true" ]; then
1612		addr_ns=$ns2
1613		subflow_ns=$ns1
1614		extra_msg="   invert"
1615	fi
1616
1617	printf "%-${nr_blank}s %s" " " "rm "
1618	count=$(ip netns exec $addr_ns nstat -as | grep MPTcpExtRmAddr | awk '{print $2}')
1619	[ -z "$count" ] && count=0
1620	if [ "$count" != "$rm_addr_nr" ]; then
1621		echo "[fail] got $count RM_ADDR[s] expected $rm_addr_nr"
1622		fail_test
1623		dump_stats=1
1624	else
1625		echo -n "[ ok ]"
1626	fi
1627
1628	echo -n " - rmsf  "
1629	count=$(ip netns exec $subflow_ns nstat -as | grep MPTcpExtRmSubflow | awk '{print $2}')
1630	[ -z "$count" ] && count=0
1631	if [ -n "$simult" ]; then
1632		local cnt suffix
1633
1634		cnt=$(ip netns exec $addr_ns nstat -as | grep MPTcpExtRmSubflow | awk '{print $2}')
1635
1636		# in case of simult flush, the subflow removal count on each side is
1637		# unreliable
1638		[ -z "$cnt" ] && cnt=0
1639		count=$((count + cnt))
1640		[ "$count" != "$rm_subflow_nr" ] && suffix="$count in [$rm_subflow_nr:$((rm_subflow_nr*2))]"
1641		if [ $count -ge "$rm_subflow_nr" ] && \
1642		   [ "$count" -le "$((rm_subflow_nr *2 ))" ]; then
1643			echo "[ ok ] $suffix"
1644		else
1645			echo "[fail] got $count RM_SUBFLOW[s] expected in range [$rm_subflow_nr:$((rm_subflow_nr*2))]"
1646			fail_test
1647			dump_stats=1
1648		fi
1649		return
1650	fi
1651	if [ "$count" != "$rm_subflow_nr" ]; then
1652		echo "[fail] got $count RM_SUBFLOW[s] expected $rm_subflow_nr"
1653		fail_test
1654		dump_stats=1
1655	else
1656		echo -n "[ ok ]"
1657	fi
1658
1659	[ "${dump_stats}" = 1 ] && dump_stats
1660
1661	echo "$extra_msg"
1662}
1663
1664chk_prio_nr()
1665{
1666	local mp_prio_nr_tx=$1
1667	local mp_prio_nr_rx=$2
1668	local count
1669	local dump_stats
1670
1671	printf "%-${nr_blank}s %s" " " "ptx"
1672	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPPrioTx | awk '{print $2}')
1673	[ -z "$count" ] && count=0
1674	if [ "$count" != "$mp_prio_nr_tx" ]; then
1675		echo "[fail] got $count MP_PRIO[s] TX expected $mp_prio_nr_tx"
1676		fail_test
1677		dump_stats=1
1678	else
1679		echo -n "[ ok ]"
1680	fi
1681
1682	echo -n " - prx   "
1683	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPPrioRx | awk '{print $2}')
1684	[ -z "$count" ] && count=0
1685	if [ "$count" != "$mp_prio_nr_rx" ]; then
1686		echo "[fail] got $count MP_PRIO[s] RX expected $mp_prio_nr_rx"
1687		fail_test
1688		dump_stats=1
1689	else
1690		echo "[ ok ]"
1691	fi
1692
1693	[ "${dump_stats}" = 1 ] && dump_stats
1694}
1695
1696chk_subflow_nr()
1697{
1698	local need_title="$1"
1699	local msg="$2"
1700	local subflow_nr=$3
1701	local cnt1
1702	local cnt2
1703	local dump_stats
1704
1705	if [ -n "${need_title}" ]; then
1706		printf "%03u %-36s %s" "${TEST_COUNT}" "${TEST_NAME}" "${msg}"
1707	else
1708		printf "%-${nr_blank}s %s" " " "${msg}"
1709	fi
1710
1711	cnt1=$(ss -N $ns1 -tOni | grep -c token)
1712	cnt2=$(ss -N $ns2 -tOni | grep -c token)
1713	if [ "$cnt1" != "$subflow_nr" -o "$cnt2" != "$subflow_nr" ]; then
1714		echo "[fail] got $cnt1:$cnt2 subflows expected $subflow_nr"
1715		fail_test
1716		dump_stats=1
1717	else
1718		echo "[ ok ]"
1719	fi
1720
1721	if [ "${dump_stats}" = 1 ]; then
1722		ss -N $ns1 -tOni
1723		ss -N $ns1 -tOni | grep token
1724		ip -n $ns1 mptcp endpoint
1725		dump_stats
1726	fi
1727}
1728
1729chk_link_usage()
1730{
1731	local ns=$1
1732	local link=$2
1733	local out=$3
1734	local expected_rate=$4
1735
1736	local tx_link tx_total
1737	tx_link=$(ip netns exec $ns cat /sys/class/net/$link/statistics/tx_bytes)
1738	tx_total=$(stat --format=%s $out)
1739	local tx_rate=$((tx_link * 100 / tx_total))
1740	local tolerance=5
1741
1742	printf "%-${nr_blank}s %-18s" " " "link usage"
1743	if [ $tx_rate -lt $((expected_rate - tolerance)) ] || \
1744	   [ $tx_rate -gt $((expected_rate + tolerance)) ]; then
1745		echo "[fail] got $tx_rate% usage, expected $expected_rate%"
1746		fail_test
1747	else
1748		echo "[ ok ]"
1749	fi
1750}
1751
1752wait_attempt_fail()
1753{
1754	local timeout_ms=$((timeout_poll * 1000))
1755	local time=0
1756	local ns=$1
1757
1758	while [ $time -lt $timeout_ms ]; do
1759		local cnt
1760
1761		cnt=$(ip netns exec $ns nstat -as TcpAttemptFails | grep TcpAttemptFails | awk '{print $2}')
1762
1763		[ "$cnt" = 1 ] && return 1
1764		time=$((time + 100))
1765		sleep 0.1
1766	done
1767	return 1
1768}
1769
1770set_userspace_pm()
1771{
1772	local ns=$1
1773
1774	ip netns exec $ns sysctl -q net.mptcp.pm_type=1
1775}
1776
1777subflows_tests()
1778{
1779	if reset "no JOIN"; then
1780		run_tests $ns1 $ns2 10.0.1.1
1781		chk_join_nr 0 0 0
1782	fi
1783
1784	# subflow limited by client
1785	if reset "single subflow, limited by client"; then
1786		pm_nl_set_limits $ns1 0 0
1787		pm_nl_set_limits $ns2 0 0
1788		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1789		run_tests $ns1 $ns2 10.0.1.1
1790		chk_join_nr 0 0 0
1791	fi
1792
1793	# subflow limited by server
1794	if reset "single subflow, limited by server"; then
1795		pm_nl_set_limits $ns1 0 0
1796		pm_nl_set_limits $ns2 0 1
1797		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1798		run_tests $ns1 $ns2 10.0.1.1
1799		chk_join_nr 1 1 0
1800	fi
1801
1802	# subflow
1803	if reset "single subflow"; then
1804		pm_nl_set_limits $ns1 0 1
1805		pm_nl_set_limits $ns2 0 1
1806		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1807		run_tests $ns1 $ns2 10.0.1.1
1808		chk_join_nr 1 1 1
1809	fi
1810
1811	# multiple subflows
1812	if reset "multiple subflows"; then
1813		pm_nl_set_limits $ns1 0 2
1814		pm_nl_set_limits $ns2 0 2
1815		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1816		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1817		run_tests $ns1 $ns2 10.0.1.1
1818		chk_join_nr 2 2 2
1819	fi
1820
1821	# multiple subflows limited by server
1822	if reset "multiple subflows, limited by server"; then
1823		pm_nl_set_limits $ns1 0 1
1824		pm_nl_set_limits $ns2 0 2
1825		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1826		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1827		run_tests $ns1 $ns2 10.0.1.1
1828		chk_join_nr 2 2 1
1829	fi
1830
1831	# single subflow, dev
1832	if reset "single subflow, dev"; then
1833		pm_nl_set_limits $ns1 0 1
1834		pm_nl_set_limits $ns2 0 1
1835		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow dev ns2eth3
1836		run_tests $ns1 $ns2 10.0.1.1
1837		chk_join_nr 1 1 1
1838	fi
1839}
1840
1841subflows_error_tests()
1842{
1843	# If a single subflow is configured, and matches the MPC src
1844	# address, no additional subflow should be created
1845	if reset "no MPC reuse with single endpoint"; then
1846		pm_nl_set_limits $ns1 0 1
1847		pm_nl_set_limits $ns2 0 1
1848		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow
1849		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
1850		chk_join_nr 0 0 0
1851	fi
1852
1853	# multiple subflows, with subflow creation error
1854	if reset "multi subflows, with failing subflow"; then
1855		pm_nl_set_limits $ns1 0 2
1856		pm_nl_set_limits $ns2 0 2
1857		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1858		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1859		filter_tcp_from $ns1 10.0.3.2 REJECT
1860		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
1861		chk_join_nr 1 1 1
1862	fi
1863
1864	# multiple subflows, with subflow timeout on MPJ
1865	if reset "multi subflows, with subflow timeout"; then
1866		pm_nl_set_limits $ns1 0 2
1867		pm_nl_set_limits $ns2 0 2
1868		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1869		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1870		filter_tcp_from $ns1 10.0.3.2 DROP
1871		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
1872		chk_join_nr 1 1 1
1873	fi
1874
1875	# multiple subflows, check that the endpoint corresponding to
1876	# closed subflow (due to reset) is not reused if additional
1877	# subflows are added later
1878	if reset "multi subflows, fair usage on close"; then
1879		pm_nl_set_limits $ns1 0 1
1880		pm_nl_set_limits $ns2 0 1
1881		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1882		filter_tcp_from $ns1 10.0.3.2 REJECT
1883		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow &
1884
1885		# mpj subflow will be in TW after the reset
1886		wait_attempt_fail $ns2
1887		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1888		wait
1889
1890		# additional subflow could be created only if the PM select
1891		# the later endpoint, skipping the already used one
1892		chk_join_nr 1 1 1
1893	fi
1894}
1895
1896signal_address_tests()
1897{
1898	# add_address, unused
1899	if reset "unused signal address"; then
1900		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1901		run_tests $ns1 $ns2 10.0.1.1
1902		chk_join_nr 0 0 0
1903		chk_add_nr 1 1
1904	fi
1905
1906	# accept and use add_addr
1907	if reset "signal address"; then
1908		pm_nl_set_limits $ns1 0 1
1909		pm_nl_set_limits $ns2 1 1
1910		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1911		run_tests $ns1 $ns2 10.0.1.1
1912		chk_join_nr 1 1 1
1913		chk_add_nr 1 1
1914	fi
1915
1916	# accept and use add_addr with an additional subflow
1917	# note: signal address in server ns and local addresses in client ns must
1918	# belong to different subnets or one of the listed local address could be
1919	# used for 'add_addr' subflow
1920	if reset "subflow and signal"; then
1921		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1922		pm_nl_set_limits $ns1 0 2
1923		pm_nl_set_limits $ns2 1 2
1924		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1925		run_tests $ns1 $ns2 10.0.1.1
1926		chk_join_nr 2 2 2
1927		chk_add_nr 1 1
1928	fi
1929
1930	# accept and use add_addr with additional subflows
1931	if reset "multiple subflows and signal"; then
1932		pm_nl_set_limits $ns1 0 3
1933		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1934		pm_nl_set_limits $ns2 1 3
1935		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1936		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
1937		run_tests $ns1 $ns2 10.0.1.1
1938		chk_join_nr 3 3 3
1939		chk_add_nr 1 1
1940	fi
1941
1942	# signal addresses
1943	if reset "signal addresses"; then
1944		pm_nl_set_limits $ns1 3 3
1945		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1946		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
1947		pm_nl_add_endpoint $ns1 10.0.4.1 flags signal
1948		pm_nl_set_limits $ns2 3 3
1949		run_tests $ns1 $ns2 10.0.1.1
1950		chk_join_nr 3 3 3
1951		chk_add_nr 3 3
1952	fi
1953
1954	# signal invalid addresses
1955	if reset "signal invalid addresses"; then
1956		pm_nl_set_limits $ns1 3 3
1957		pm_nl_add_endpoint $ns1 10.0.12.1 flags signal
1958		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
1959		pm_nl_add_endpoint $ns1 10.0.14.1 flags signal
1960		pm_nl_set_limits $ns2 3 3
1961		run_tests $ns1 $ns2 10.0.1.1
1962		chk_join_nr 1 1 1
1963		chk_add_nr 3 3
1964	fi
1965
1966	# signal addresses race test
1967	if reset "signal addresses race test"; then
1968		pm_nl_set_limits $ns1 4 4
1969		pm_nl_set_limits $ns2 4 4
1970		pm_nl_add_endpoint $ns1 10.0.1.1 flags signal
1971		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1972		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
1973		pm_nl_add_endpoint $ns1 10.0.4.1 flags signal
1974		pm_nl_add_endpoint $ns2 10.0.1.2 flags signal
1975		pm_nl_add_endpoint $ns2 10.0.2.2 flags signal
1976		pm_nl_add_endpoint $ns2 10.0.3.2 flags signal
1977		pm_nl_add_endpoint $ns2 10.0.4.2 flags signal
1978
1979		# the peer could possibly miss some addr notification, allow retransmission
1980		ip netns exec $ns1 sysctl -q net.mptcp.add_addr_timeout=1
1981		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
1982		chk_join_nr 3 3 3
1983
1984		# the server will not signal the address terminating
1985		# the MPC subflow
1986		chk_add_nr 3 3
1987	fi
1988}
1989
1990link_failure_tests()
1991{
1992	# accept and use add_addr with additional subflows and link loss
1993	if reset "multiple flows, signal, link failure"; then
1994		# without any b/w limit each veth could spool the packets and get
1995		# them acked at xmit time, so that the corresponding subflow will
1996		# have almost always no outstanding pkts, the scheduler will pick
1997		# always the first subflow and we will have hard time testing
1998		# active backup and link switch-over.
1999		# Let's set some arbitrary (low) virtual link limits.
2000		init_shapers
2001		pm_nl_set_limits $ns1 0 3
2002		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
2003		pm_nl_set_limits $ns2 1 3
2004		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow
2005		pm_nl_add_endpoint $ns2 10.0.4.2 dev ns2eth4 flags subflow
2006		run_tests $ns1 $ns2 10.0.1.1 1
2007		chk_join_nr 3 3 3
2008		chk_add_nr 1 1
2009		chk_stale_nr $ns2 1 5 1
2010	fi
2011
2012	# accept and use add_addr with additional subflows and link loss
2013	# for bidirectional transfer
2014	if reset "multi flows, signal, bidi, link fail"; then
2015		init_shapers
2016		pm_nl_set_limits $ns1 0 3
2017		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
2018		pm_nl_set_limits $ns2 1 3
2019		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow
2020		pm_nl_add_endpoint $ns2 10.0.4.2 dev ns2eth4 flags subflow
2021		run_tests $ns1 $ns2 10.0.1.1 2
2022		chk_join_nr 3 3 3
2023		chk_add_nr 1 1
2024		chk_stale_nr $ns2 1 -1 1
2025	fi
2026
2027	# 2 subflows plus 1 backup subflow with a lossy link, backup
2028	# will never be used
2029	if reset "backup subflow unused, link failure"; then
2030		init_shapers
2031		pm_nl_set_limits $ns1 0 2
2032		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
2033		pm_nl_set_limits $ns2 1 2
2034		FAILING_LINKS="1"
2035		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow,backup
2036		run_tests $ns1 $ns2 10.0.1.1 1
2037		chk_join_nr 2 2 2
2038		chk_add_nr 1 1
2039		chk_link_usage $ns2 ns2eth3 $cinsent 0
2040	fi
2041
2042	# 2 lossy links after half transfer, backup will get half of
2043	# the traffic
2044	if reset "backup flow used, multi links fail"; then
2045		init_shapers
2046		pm_nl_set_limits $ns1 0 2
2047		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
2048		pm_nl_set_limits $ns2 1 2
2049		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow,backup
2050		FAILING_LINKS="1 2"
2051		run_tests $ns1 $ns2 10.0.1.1 1
2052		chk_join_nr 2 2 2
2053		chk_add_nr 1 1
2054		chk_stale_nr $ns2 2 4 2
2055		chk_link_usage $ns2 ns2eth3 $cinsent 50
2056	fi
2057
2058	# use a backup subflow with the first subflow on a lossy link
2059	# for bidirectional transfer
2060	if reset "backup flow used, bidi, link failure"; then
2061		init_shapers
2062		pm_nl_set_limits $ns1 0 2
2063		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
2064		pm_nl_set_limits $ns2 1 3
2065		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow,backup
2066		FAILING_LINKS="1 2"
2067		run_tests $ns1 $ns2 10.0.1.1 2
2068		chk_join_nr 2 2 2
2069		chk_add_nr 1 1
2070		chk_stale_nr $ns2 1 -1 2
2071		chk_link_usage $ns2 ns2eth3 $cinsent 50
2072	fi
2073}
2074
2075add_addr_timeout_tests()
2076{
2077	# add_addr timeout
2078	if reset_with_add_addr_timeout "signal address, ADD_ADDR timeout"; then
2079		pm_nl_set_limits $ns1 0 1
2080		pm_nl_set_limits $ns2 1 1
2081		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2082		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
2083		chk_join_nr 1 1 1
2084		chk_add_nr 4 0
2085	fi
2086
2087	# add_addr timeout IPv6
2088	if reset_with_add_addr_timeout "signal address, ADD_ADDR6 timeout" 6; then
2089		pm_nl_set_limits $ns1 0 1
2090		pm_nl_set_limits $ns2 1 1
2091		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2092		run_tests $ns1 $ns2 dead:beef:1::1 0 0 0 slow
2093		chk_join_nr 1 1 1
2094		chk_add_nr 4 0
2095	fi
2096
2097	# signal addresses timeout
2098	if reset_with_add_addr_timeout "signal addresses, ADD_ADDR timeout"; then
2099		pm_nl_set_limits $ns1 2 2
2100		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2101		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2102		pm_nl_set_limits $ns2 2 2
2103		run_tests $ns1 $ns2 10.0.1.1 0 0 0 speed_10
2104		chk_join_nr 2 2 2
2105		chk_add_nr 8 0
2106	fi
2107
2108	# signal invalid addresses timeout
2109	if reset_with_add_addr_timeout "invalid address, ADD_ADDR timeout"; then
2110		pm_nl_set_limits $ns1 2 2
2111		pm_nl_add_endpoint $ns1 10.0.12.1 flags signal
2112		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2113		pm_nl_set_limits $ns2 2 2
2114		run_tests $ns1 $ns2 10.0.1.1 0 0 0 speed_10
2115		chk_join_nr 1 1 1
2116		chk_add_nr 8 0
2117	fi
2118}
2119
2120remove_tests()
2121{
2122	# single subflow, remove
2123	if reset "remove single subflow"; then
2124		pm_nl_set_limits $ns1 0 1
2125		pm_nl_set_limits $ns2 0 1
2126		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2127		run_tests $ns1 $ns2 10.0.1.1 0 0 -1 slow
2128		chk_join_nr 1 1 1
2129		chk_rm_nr 1 1
2130	fi
2131
2132	# multiple subflows, remove
2133	if reset "remove multiple subflows"; then
2134		pm_nl_set_limits $ns1 0 2
2135		pm_nl_set_limits $ns2 0 2
2136		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
2137		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2138		run_tests $ns1 $ns2 10.0.1.1 0 0 -2 slow
2139		chk_join_nr 2 2 2
2140		chk_rm_nr 2 2
2141	fi
2142
2143	# single address, remove
2144	if reset "remove single address"; then
2145		pm_nl_set_limits $ns1 0 1
2146		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2147		pm_nl_set_limits $ns2 1 1
2148		run_tests $ns1 $ns2 10.0.1.1 0 -1 0 slow
2149		chk_join_nr 1 1 1
2150		chk_add_nr 1 1
2151		chk_rm_nr 1 1 invert
2152	fi
2153
2154	# subflow and signal, remove
2155	if reset "remove subflow and signal"; then
2156		pm_nl_set_limits $ns1 0 2
2157		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2158		pm_nl_set_limits $ns2 1 2
2159		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2160		run_tests $ns1 $ns2 10.0.1.1 0 -1 -1 slow
2161		chk_join_nr 2 2 2
2162		chk_add_nr 1 1
2163		chk_rm_nr 1 1
2164	fi
2165
2166	# subflows and signal, remove
2167	if reset "remove subflows and signal"; then
2168		pm_nl_set_limits $ns1 0 3
2169		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2170		pm_nl_set_limits $ns2 1 3
2171		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2172		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2173		run_tests $ns1 $ns2 10.0.1.1 0 -1 -2 speed_10
2174		chk_join_nr 3 3 3
2175		chk_add_nr 1 1
2176		chk_rm_nr 2 2
2177	fi
2178
2179	# addresses remove
2180	if reset "remove addresses"; then
2181		pm_nl_set_limits $ns1 3 3
2182		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal id 250
2183		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2184		pm_nl_add_endpoint $ns1 10.0.4.1 flags signal
2185		pm_nl_set_limits $ns2 3 3
2186		run_tests $ns1 $ns2 10.0.1.1 0 -3 0 speed_10
2187		chk_join_nr 3 3 3
2188		chk_add_nr 3 3
2189		chk_rm_nr 3 3 invert
2190	fi
2191
2192	# invalid addresses remove
2193	if reset "remove invalid addresses"; then
2194		pm_nl_set_limits $ns1 3 3
2195		pm_nl_add_endpoint $ns1 10.0.12.1 flags signal
2196		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2197		pm_nl_add_endpoint $ns1 10.0.14.1 flags signal
2198		pm_nl_set_limits $ns2 3 3
2199		run_tests $ns1 $ns2 10.0.1.1 0 -3 0 speed_10
2200		chk_join_nr 1 1 1
2201		chk_add_nr 3 3
2202		chk_rm_nr 3 1 invert
2203	fi
2204
2205	# subflows and signal, flush
2206	if reset "flush subflows and signal"; then
2207		pm_nl_set_limits $ns1 0 3
2208		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2209		pm_nl_set_limits $ns2 1 3
2210		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2211		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2212		run_tests $ns1 $ns2 10.0.1.1 0 -8 -8 slow
2213		chk_join_nr 3 3 3
2214		chk_add_nr 1 1
2215		chk_rm_nr 1 3 invert simult
2216	fi
2217
2218	# subflows flush
2219	if reset "flush subflows"; then
2220		pm_nl_set_limits $ns1 3 3
2221		pm_nl_set_limits $ns2 3 3
2222		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow id 150
2223		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2224		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2225		run_tests $ns1 $ns2 10.0.1.1 0 -8 -8 slow
2226		chk_join_nr 3 3 3
2227		chk_rm_nr 0 3 simult
2228	fi
2229
2230	# addresses flush
2231	if reset "flush addresses"; then
2232		pm_nl_set_limits $ns1 3 3
2233		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal id 250
2234		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2235		pm_nl_add_endpoint $ns1 10.0.4.1 flags signal
2236		pm_nl_set_limits $ns2 3 3
2237		run_tests $ns1 $ns2 10.0.1.1 0 -8 -8 slow
2238		chk_join_nr 3 3 3
2239		chk_add_nr 3 3
2240		chk_rm_nr 3 3 invert simult
2241	fi
2242
2243	# invalid addresses flush
2244	if reset "flush invalid addresses"; then
2245		pm_nl_set_limits $ns1 3 3
2246		pm_nl_add_endpoint $ns1 10.0.12.1 flags signal
2247		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2248		pm_nl_add_endpoint $ns1 10.0.14.1 flags signal
2249		pm_nl_set_limits $ns2 3 3
2250		run_tests $ns1 $ns2 10.0.1.1 0 -8 0 slow
2251		chk_join_nr 1 1 1
2252		chk_add_nr 3 3
2253		chk_rm_nr 3 1 invert
2254	fi
2255
2256	# remove id 0 subflow
2257	if reset "remove id 0 subflow"; then
2258		pm_nl_set_limits $ns1 0 1
2259		pm_nl_set_limits $ns2 0 1
2260		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2261		run_tests $ns1 $ns2 10.0.1.1 0 0 -9 slow
2262		chk_join_nr 1 1 1
2263		chk_rm_nr 1 1
2264	fi
2265
2266	# remove id 0 address
2267	if reset "remove id 0 address"; then
2268		pm_nl_set_limits $ns1 0 1
2269		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2270		pm_nl_set_limits $ns2 1 1
2271		run_tests $ns1 $ns2 10.0.1.1 0 -9 0 slow
2272		chk_join_nr 1 1 1
2273		chk_add_nr 1 1
2274		chk_rm_nr 1 1 invert
2275	fi
2276}
2277
2278add_tests()
2279{
2280	# add single subflow
2281	if reset "add single subflow"; then
2282		pm_nl_set_limits $ns1 0 1
2283		pm_nl_set_limits $ns2 0 1
2284		run_tests $ns1 $ns2 10.0.1.1 0 0 1 slow
2285		chk_join_nr 1 1 1
2286	fi
2287
2288	# add signal address
2289	if reset "add signal address"; then
2290		pm_nl_set_limits $ns1 0 1
2291		pm_nl_set_limits $ns2 1 1
2292		run_tests $ns1 $ns2 10.0.1.1 0 1 0 slow
2293		chk_join_nr 1 1 1
2294		chk_add_nr 1 1
2295	fi
2296
2297	# add multiple subflows
2298	if reset "add multiple subflows"; then
2299		pm_nl_set_limits $ns1 0 2
2300		pm_nl_set_limits $ns2 0 2
2301		run_tests $ns1 $ns2 10.0.1.1 0 0 2 slow
2302		chk_join_nr 2 2 2
2303	fi
2304
2305	# add multiple subflows IPv6
2306	if reset "add multiple subflows IPv6"; then
2307		pm_nl_set_limits $ns1 0 2
2308		pm_nl_set_limits $ns2 0 2
2309		run_tests $ns1 $ns2 dead:beef:1::1 0 0 2 slow
2310		chk_join_nr 2 2 2
2311	fi
2312
2313	# add multiple addresses IPv6
2314	if reset "add multiple addresses IPv6"; then
2315		pm_nl_set_limits $ns1 0 2
2316		pm_nl_set_limits $ns2 2 2
2317		run_tests $ns1 $ns2 dead:beef:1::1 0 2 0 slow
2318		chk_join_nr 2 2 2
2319		chk_add_nr 2 2
2320	fi
2321}
2322
2323ipv6_tests()
2324{
2325	# subflow IPv6
2326	if reset "single subflow IPv6"; then
2327		pm_nl_set_limits $ns1 0 1
2328		pm_nl_set_limits $ns2 0 1
2329		pm_nl_add_endpoint $ns2 dead:beef:3::2 dev ns2eth3 flags subflow
2330		run_tests $ns1 $ns2 dead:beef:1::1 0 0 0 slow
2331		chk_join_nr 1 1 1
2332	fi
2333
2334	# add_address, unused IPv6
2335	if reset "unused signal address IPv6"; then
2336		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2337		run_tests $ns1 $ns2 dead:beef:1::1 0 0 0 slow
2338		chk_join_nr 0 0 0
2339		chk_add_nr 1 1
2340	fi
2341
2342	# signal address IPv6
2343	if reset "single address IPv6"; then
2344		pm_nl_set_limits $ns1 0 1
2345		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2346		pm_nl_set_limits $ns2 1 1
2347		run_tests $ns1 $ns2 dead:beef:1::1 0 0 0 slow
2348		chk_join_nr 1 1 1
2349		chk_add_nr 1 1
2350	fi
2351
2352	# single address IPv6, remove
2353	if reset "remove single address IPv6"; then
2354		pm_nl_set_limits $ns1 0 1
2355		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2356		pm_nl_set_limits $ns2 1 1
2357		run_tests $ns1 $ns2 dead:beef:1::1 0 -1 0 slow
2358		chk_join_nr 1 1 1
2359		chk_add_nr 1 1
2360		chk_rm_nr 1 1 invert
2361	fi
2362
2363	# subflow and signal IPv6, remove
2364	if reset "remove subflow and signal IPv6"; then
2365		pm_nl_set_limits $ns1 0 2
2366		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2367		pm_nl_set_limits $ns2 1 2
2368		pm_nl_add_endpoint $ns2 dead:beef:3::2 dev ns2eth3 flags subflow
2369		run_tests $ns1 $ns2 dead:beef:1::1 0 -1 -1 slow
2370		chk_join_nr 2 2 2
2371		chk_add_nr 1 1
2372		chk_rm_nr 1 1
2373	fi
2374}
2375
2376v4mapped_tests()
2377{
2378	# subflow IPv4-mapped to IPv4-mapped
2379	if reset "single subflow IPv4-mapped"; then
2380		pm_nl_set_limits $ns1 0 1
2381		pm_nl_set_limits $ns2 0 1
2382		pm_nl_add_endpoint $ns2 "::ffff:10.0.3.2" flags subflow
2383		run_tests $ns1 $ns2 "::ffff:10.0.1.1"
2384		chk_join_nr 1 1 1
2385	fi
2386
2387	# signal address IPv4-mapped with IPv4-mapped sk
2388	if reset "signal address IPv4-mapped"; then
2389		pm_nl_set_limits $ns1 0 1
2390		pm_nl_set_limits $ns2 1 1
2391		pm_nl_add_endpoint $ns1 "::ffff:10.0.2.1" flags signal
2392		run_tests $ns1 $ns2 "::ffff:10.0.1.1"
2393		chk_join_nr 1 1 1
2394		chk_add_nr 1 1
2395	fi
2396
2397	# subflow v4-map-v6
2398	if reset "single subflow v4-map-v6"; then
2399		pm_nl_set_limits $ns1 0 1
2400		pm_nl_set_limits $ns2 0 1
2401		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2402		run_tests $ns1 $ns2 "::ffff:10.0.1.1"
2403		chk_join_nr 1 1 1
2404	fi
2405
2406	# signal address v4-map-v6
2407	if reset "signal address v4-map-v6"; then
2408		pm_nl_set_limits $ns1 0 1
2409		pm_nl_set_limits $ns2 1 1
2410		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2411		run_tests $ns1 $ns2 "::ffff:10.0.1.1"
2412		chk_join_nr 1 1 1
2413		chk_add_nr 1 1
2414	fi
2415
2416	# subflow v6-map-v4
2417	if reset "single subflow v6-map-v4"; then
2418		pm_nl_set_limits $ns1 0 1
2419		pm_nl_set_limits $ns2 0 1
2420		pm_nl_add_endpoint $ns2 "::ffff:10.0.3.2" flags subflow
2421		run_tests $ns1 $ns2 10.0.1.1
2422		chk_join_nr 1 1 1
2423	fi
2424
2425	# signal address v6-map-v4
2426	if reset "signal address v6-map-v4"; then
2427		pm_nl_set_limits $ns1 0 1
2428		pm_nl_set_limits $ns2 1 1
2429		pm_nl_add_endpoint $ns1 "::ffff:10.0.2.1" flags signal
2430		run_tests $ns1 $ns2 10.0.1.1
2431		chk_join_nr 1 1 1
2432		chk_add_nr 1 1
2433	fi
2434
2435	# no subflow IPv6 to v4 address
2436	if reset "no JOIN with diff families v4-v6"; then
2437		pm_nl_set_limits $ns1 0 1
2438		pm_nl_set_limits $ns2 0 1
2439		pm_nl_add_endpoint $ns2 dead:beef:2::2 flags subflow
2440		run_tests $ns1 $ns2 10.0.1.1
2441		chk_join_nr 0 0 0
2442	fi
2443
2444	# no subflow IPv6 to v4 address even if v6 has a valid v4 at the end
2445	if reset "no JOIN with diff families v4-v6-2"; then
2446		pm_nl_set_limits $ns1 0 1
2447		pm_nl_set_limits $ns2 0 1
2448		pm_nl_add_endpoint $ns2 dead:beef:2::10.0.3.2 flags subflow
2449		run_tests $ns1 $ns2 10.0.1.1
2450		chk_join_nr 0 0 0
2451	fi
2452
2453	# no subflow IPv4 to v6 address, no need to slow down too then
2454	if reset "no JOIN with diff families v6-v4"; then
2455		pm_nl_set_limits $ns1 0 1
2456		pm_nl_set_limits $ns2 0 1
2457		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2458		run_tests $ns1 $ns2 dead:beef:1::1
2459		chk_join_nr 0 0 0
2460	fi
2461}
2462
2463backup_tests()
2464{
2465	# single subflow, backup
2466	if reset "single subflow, backup"; then
2467		pm_nl_set_limits $ns1 0 1
2468		pm_nl_set_limits $ns2 0 1
2469		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow,backup
2470		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow nobackup
2471		chk_join_nr 1 1 1
2472		chk_prio_nr 0 1
2473	fi
2474
2475	# single address, backup
2476	if reset "single address, backup"; then
2477		pm_nl_set_limits $ns1 0 1
2478		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2479		pm_nl_set_limits $ns2 1 1
2480		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
2481		chk_join_nr 1 1 1
2482		chk_add_nr 1 1
2483		chk_prio_nr 1 1
2484	fi
2485
2486	# single address with port, backup
2487	if reset "single address with port, backup"; then
2488		pm_nl_set_limits $ns1 0 1
2489		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2490		pm_nl_set_limits $ns2 1 1
2491		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
2492		chk_join_nr 1 1 1
2493		chk_add_nr 1 1
2494		chk_prio_nr 1 1
2495	fi
2496
2497	if reset "mpc backup"; then
2498		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow,backup
2499		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
2500		chk_join_nr 0 0 0
2501		chk_prio_nr 0 1
2502	fi
2503
2504	if reset "mpc backup both sides"; then
2505		pm_nl_add_endpoint $ns1 10.0.1.1 flags subflow,backup
2506		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow,backup
2507		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
2508		chk_join_nr 0 0 0
2509		chk_prio_nr 1 1
2510	fi
2511
2512	if reset "mpc switch to backup"; then
2513		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow
2514		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
2515		chk_join_nr 0 0 0
2516		chk_prio_nr 0 1
2517	fi
2518
2519	if reset "mpc switch to backup both sides"; then
2520		pm_nl_add_endpoint $ns1 10.0.1.1 flags subflow
2521		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow
2522		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
2523		chk_join_nr 0 0 0
2524		chk_prio_nr 1 1
2525	fi
2526}
2527
2528LISTENER_CREATED=15 #MPTCP_EVENT_LISTENER_CREATED
2529LISTENER_CLOSED=16  #MPTCP_EVENT_LISTENER_CLOSED
2530
2531AF_INET=2
2532AF_INET6=10
2533
2534verify_listener_events()
2535{
2536	local evt=$1
2537	local e_type=$2
2538	local e_family=$3
2539	local e_saddr=$4
2540	local e_sport=$5
2541	local type
2542	local family
2543	local saddr
2544	local sport
2545
2546	if [ $e_type = $LISTENER_CREATED ]; then
2547		stdbuf -o0 -e0 printf "\t\t\t\t\t CREATE_LISTENER %s:%s"\
2548			$e_saddr $e_sport
2549	elif [ $e_type = $LISTENER_CLOSED ]; then
2550		stdbuf -o0 -e0 printf "\t\t\t\t\t CLOSE_LISTENER %s:%s "\
2551			$e_saddr $e_sport
2552	fi
2553
2554	type=$(grep "type:$e_type," $evt |
2555	       sed --unbuffered -n 's/.*\(type:\)\([[:digit:]]*\).*$/\2/p;q')
2556	family=$(grep "type:$e_type," $evt |
2557		 sed --unbuffered -n 's/.*\(family:\)\([[:digit:]]*\).*$/\2/p;q')
2558	sport=$(grep "type:$e_type," $evt |
2559		sed --unbuffered -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q')
2560	if [ $family ] && [ $family = $AF_INET6 ]; then
2561		saddr=$(grep "type:$e_type," $evt |
2562			sed --unbuffered -n 's/.*\(saddr6:\)\([0-9a-f:.]*\).*$/\2/p;q')
2563	else
2564		saddr=$(grep "type:$e_type," $evt |
2565			sed --unbuffered -n 's/.*\(saddr4:\)\([0-9.]*\).*$/\2/p;q')
2566	fi
2567
2568	if [ $type ] && [ $type = $e_type ] &&
2569	   [ $family ] && [ $family = $e_family ] &&
2570	   [ $saddr ] && [ $saddr = $e_saddr ] &&
2571	   [ $sport ] && [ $sport = $e_sport ]; then
2572		stdbuf -o0 -e0 printf "[ ok ]\n"
2573		return 0
2574	fi
2575	fail_test
2576	stdbuf -o0 -e0 printf "[fail]\n"
2577}
2578
2579add_addr_ports_tests()
2580{
2581	# signal address with port
2582	if reset "signal address with port"; then
2583		pm_nl_set_limits $ns1 0 1
2584		pm_nl_set_limits $ns2 1 1
2585		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2586		run_tests $ns1 $ns2 10.0.1.1
2587		chk_join_nr 1 1 1
2588		chk_add_nr 1 1 1
2589	fi
2590
2591	# subflow and signal with port
2592	if reset "subflow and signal with port"; then
2593		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2594		pm_nl_set_limits $ns1 0 2
2595		pm_nl_set_limits $ns2 1 2
2596		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2597		run_tests $ns1 $ns2 10.0.1.1
2598		chk_join_nr 2 2 2
2599		chk_add_nr 1 1 1
2600	fi
2601
2602	# single address with port, remove
2603	# pm listener events
2604	if reset_with_events "remove single address with port"; then
2605		pm_nl_set_limits $ns1 0 1
2606		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2607		pm_nl_set_limits $ns2 1 1
2608		run_tests $ns1 $ns2 10.0.1.1 0 -1 0 slow
2609		chk_join_nr 1 1 1
2610		chk_add_nr 1 1 1
2611		chk_rm_nr 1 1 invert
2612
2613		verify_listener_events $evts_ns1 $LISTENER_CREATED $AF_INET 10.0.2.1 10100
2614		verify_listener_events $evts_ns1 $LISTENER_CLOSED $AF_INET 10.0.2.1 10100
2615		kill_events_pids
2616	fi
2617
2618	# subflow and signal with port, remove
2619	if reset "remove subflow and signal with port"; then
2620		pm_nl_set_limits $ns1 0 2
2621		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2622		pm_nl_set_limits $ns2 1 2
2623		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2624		run_tests $ns1 $ns2 10.0.1.1 0 -1 -1 slow
2625		chk_join_nr 2 2 2
2626		chk_add_nr 1 1 1
2627		chk_rm_nr 1 1
2628	fi
2629
2630	# subflows and signal with port, flush
2631	if reset "flush subflows and signal with port"; then
2632		pm_nl_set_limits $ns1 0 3
2633		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2634		pm_nl_set_limits $ns2 1 3
2635		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2636		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2637		run_tests $ns1 $ns2 10.0.1.1 0 -8 -2 slow
2638		chk_join_nr 3 3 3
2639		chk_add_nr 1 1
2640		chk_rm_nr 1 3 invert simult
2641	fi
2642
2643	# multiple addresses with port
2644	if reset "multiple addresses with port"; then
2645		pm_nl_set_limits $ns1 2 2
2646		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2647		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal port 10100
2648		pm_nl_set_limits $ns2 2 2
2649		run_tests $ns1 $ns2 10.0.1.1
2650		chk_join_nr 2 2 2
2651		chk_add_nr 2 2 2
2652	fi
2653
2654	# multiple addresses with ports
2655	if reset "multiple addresses with ports"; then
2656		pm_nl_set_limits $ns1 2 2
2657		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2658		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal port 10101
2659		pm_nl_set_limits $ns2 2 2
2660		run_tests $ns1 $ns2 10.0.1.1
2661		chk_join_nr 2 2 2
2662		chk_add_nr 2 2 2
2663	fi
2664}
2665
2666syncookies_tests()
2667{
2668	# single subflow, syncookies
2669	if reset_with_cookies "single subflow with syn cookies"; then
2670		pm_nl_set_limits $ns1 0 1
2671		pm_nl_set_limits $ns2 0 1
2672		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2673		run_tests $ns1 $ns2 10.0.1.1
2674		chk_join_nr 1 1 1
2675	fi
2676
2677	# multiple subflows with syn cookies
2678	if reset_with_cookies "multiple subflows with syn cookies"; then
2679		pm_nl_set_limits $ns1 0 2
2680		pm_nl_set_limits $ns2 0 2
2681		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2682		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
2683		run_tests $ns1 $ns2 10.0.1.1
2684		chk_join_nr 2 2 2
2685	fi
2686
2687	# multiple subflows limited by server
2688	if reset_with_cookies "subflows limited by server w cookies"; then
2689		pm_nl_set_limits $ns1 0 1
2690		pm_nl_set_limits $ns2 0 2
2691		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2692		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
2693		run_tests $ns1 $ns2 10.0.1.1
2694		chk_join_nr 2 1 1
2695	fi
2696
2697	# test signal address with cookies
2698	if reset_with_cookies "signal address with syn cookies"; then
2699		pm_nl_set_limits $ns1 0 1
2700		pm_nl_set_limits $ns2 1 1
2701		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2702		run_tests $ns1 $ns2 10.0.1.1
2703		chk_join_nr 1 1 1
2704		chk_add_nr 1 1
2705	fi
2706
2707	# test cookie with subflow and signal
2708	if reset_with_cookies "subflow and signal w cookies"; then
2709		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2710		pm_nl_set_limits $ns1 0 2
2711		pm_nl_set_limits $ns2 1 2
2712		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2713		run_tests $ns1 $ns2 10.0.1.1
2714		chk_join_nr 2 2 2
2715		chk_add_nr 1 1
2716	fi
2717
2718	# accept and use add_addr with additional subflows
2719	if reset_with_cookies "subflows and signal w. cookies"; then
2720		pm_nl_set_limits $ns1 0 3
2721		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2722		pm_nl_set_limits $ns2 1 3
2723		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2724		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2725		run_tests $ns1 $ns2 10.0.1.1
2726		chk_join_nr 3 3 3
2727		chk_add_nr 1 1
2728	fi
2729}
2730
2731checksum_tests()
2732{
2733	# checksum test 0 0
2734	if reset_with_checksum 0 0; then
2735		pm_nl_set_limits $ns1 0 1
2736		pm_nl_set_limits $ns2 0 1
2737		run_tests $ns1 $ns2 10.0.1.1
2738		chk_join_nr 0 0 0
2739	fi
2740
2741	# checksum test 1 1
2742	if reset_with_checksum 1 1; then
2743		pm_nl_set_limits $ns1 0 1
2744		pm_nl_set_limits $ns2 0 1
2745		run_tests $ns1 $ns2 10.0.1.1
2746		chk_join_nr 0 0 0
2747	fi
2748
2749	# checksum test 0 1
2750	if reset_with_checksum 0 1; then
2751		pm_nl_set_limits $ns1 0 1
2752		pm_nl_set_limits $ns2 0 1
2753		run_tests $ns1 $ns2 10.0.1.1
2754		chk_join_nr 0 0 0
2755	fi
2756
2757	# checksum test 1 0
2758	if reset_with_checksum 1 0; then
2759		pm_nl_set_limits $ns1 0 1
2760		pm_nl_set_limits $ns2 0 1
2761		run_tests $ns1 $ns2 10.0.1.1
2762		chk_join_nr 0 0 0
2763	fi
2764}
2765
2766deny_join_id0_tests()
2767{
2768	# subflow allow join id0 ns1
2769	if reset_with_allow_join_id0 "single subflow allow join id0 ns1" 1 0; then
2770		pm_nl_set_limits $ns1 1 1
2771		pm_nl_set_limits $ns2 1 1
2772		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2773		run_tests $ns1 $ns2 10.0.1.1
2774		chk_join_nr 1 1 1
2775	fi
2776
2777	# subflow allow join id0 ns2
2778	if reset_with_allow_join_id0 "single subflow allow join id0 ns2" 0 1; then
2779		pm_nl_set_limits $ns1 1 1
2780		pm_nl_set_limits $ns2 1 1
2781		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2782		run_tests $ns1 $ns2 10.0.1.1
2783		chk_join_nr 0 0 0
2784	fi
2785
2786	# signal address allow join id0 ns1
2787	# ADD_ADDRs are not affected by allow_join_id0 value.
2788	if reset_with_allow_join_id0 "signal address allow join id0 ns1" 1 0; then
2789		pm_nl_set_limits $ns1 1 1
2790		pm_nl_set_limits $ns2 1 1
2791		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2792		run_tests $ns1 $ns2 10.0.1.1
2793		chk_join_nr 1 1 1
2794		chk_add_nr 1 1
2795	fi
2796
2797	# signal address allow join id0 ns2
2798	# ADD_ADDRs are not affected by allow_join_id0 value.
2799	if reset_with_allow_join_id0 "signal address allow join id0 ns2" 0 1; then
2800		pm_nl_set_limits $ns1 1 1
2801		pm_nl_set_limits $ns2 1 1
2802		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2803		run_tests $ns1 $ns2 10.0.1.1
2804		chk_join_nr 1 1 1
2805		chk_add_nr 1 1
2806	fi
2807
2808	# subflow and address allow join id0 ns1
2809	if reset_with_allow_join_id0 "subflow and address allow join id0 1" 1 0; then
2810		pm_nl_set_limits $ns1 2 2
2811		pm_nl_set_limits $ns2 2 2
2812		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2813		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2814		run_tests $ns1 $ns2 10.0.1.1
2815		chk_join_nr 2 2 2
2816	fi
2817
2818	# subflow and address allow join id0 ns2
2819	if reset_with_allow_join_id0 "subflow and address allow join id0 2" 0 1; then
2820		pm_nl_set_limits $ns1 2 2
2821		pm_nl_set_limits $ns2 2 2
2822		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2823		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2824		run_tests $ns1 $ns2 10.0.1.1
2825		chk_join_nr 1 1 1
2826	fi
2827}
2828
2829fullmesh_tests()
2830{
2831	# fullmesh 1
2832	# 2 fullmesh addrs in ns2, added before the connection,
2833	# 1 non-fullmesh addr in ns1, added during the connection.
2834	if reset "fullmesh test 2x1"; then
2835		pm_nl_set_limits $ns1 0 4
2836		pm_nl_set_limits $ns2 1 4
2837		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow,fullmesh
2838		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow,fullmesh
2839		run_tests $ns1 $ns2 10.0.1.1 0 1 0 slow
2840		chk_join_nr 4 4 4
2841		chk_add_nr 1 1
2842	fi
2843
2844	# fullmesh 2
2845	# 1 non-fullmesh addr in ns1, added before the connection,
2846	# 1 fullmesh addr in ns2, added during the connection.
2847	if reset "fullmesh test 1x1"; then
2848		pm_nl_set_limits $ns1 1 3
2849		pm_nl_set_limits $ns2 1 3
2850		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2851		run_tests $ns1 $ns2 10.0.1.1 0 0 fullmesh_1 slow
2852		chk_join_nr 3 3 3
2853		chk_add_nr 1 1
2854	fi
2855
2856	# fullmesh 3
2857	# 1 non-fullmesh addr in ns1, added before the connection,
2858	# 2 fullmesh addrs in ns2, added during the connection.
2859	if reset "fullmesh test 1x2"; then
2860		pm_nl_set_limits $ns1 2 5
2861		pm_nl_set_limits $ns2 1 5
2862		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2863		run_tests $ns1 $ns2 10.0.1.1 0 0 fullmesh_2 slow
2864		chk_join_nr 5 5 5
2865		chk_add_nr 1 1
2866	fi
2867
2868	# fullmesh 4
2869	# 1 non-fullmesh addr in ns1, added before the connection,
2870	# 2 fullmesh addrs in ns2, added during the connection,
2871	# limit max_subflows to 4.
2872	if reset "fullmesh test 1x2, limited"; then
2873		pm_nl_set_limits $ns1 2 4
2874		pm_nl_set_limits $ns2 1 4
2875		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2876		run_tests $ns1 $ns2 10.0.1.1 0 0 fullmesh_2 slow
2877		chk_join_nr 4 4 4
2878		chk_add_nr 1 1
2879	fi
2880
2881	# set fullmesh flag
2882	if reset "set fullmesh flag test"; then
2883		pm_nl_set_limits $ns1 4 4
2884		pm_nl_add_endpoint $ns1 10.0.2.1 flags subflow
2885		pm_nl_set_limits $ns2 4 4
2886		run_tests $ns1 $ns2 10.0.1.1 0 0 1 slow fullmesh
2887		chk_join_nr 2 2 2
2888		chk_rm_nr 0 1
2889	fi
2890
2891	# set nofullmesh flag
2892	if reset "set nofullmesh flag test"; then
2893		pm_nl_set_limits $ns1 4 4
2894		pm_nl_add_endpoint $ns1 10.0.2.1 flags subflow,fullmesh
2895		pm_nl_set_limits $ns2 4 4
2896		run_tests $ns1 $ns2 10.0.1.1 0 0 fullmesh_1 slow nofullmesh
2897		chk_join_nr 2 2 2
2898		chk_rm_nr 0 1
2899	fi
2900
2901	# set backup,fullmesh flags
2902	if reset "set backup,fullmesh flags test"; then
2903		pm_nl_set_limits $ns1 4 4
2904		pm_nl_add_endpoint $ns1 10.0.2.1 flags subflow
2905		pm_nl_set_limits $ns2 4 4
2906		run_tests $ns1 $ns2 10.0.1.1 0 0 1 slow backup,fullmesh
2907		chk_join_nr 2 2 2
2908		chk_prio_nr 0 1
2909		chk_rm_nr 0 1
2910	fi
2911
2912	# set nobackup,nofullmesh flags
2913	if reset "set nobackup,nofullmesh flags test"; then
2914		pm_nl_set_limits $ns1 4 4
2915		pm_nl_set_limits $ns2 4 4
2916		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow,backup,fullmesh
2917		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow nobackup,nofullmesh
2918		chk_join_nr 2 2 2
2919		chk_prio_nr 0 1
2920		chk_rm_nr 0 1
2921	fi
2922}
2923
2924fastclose_tests()
2925{
2926	if reset "fastclose test"; then
2927		run_tests $ns1 $ns2 10.0.1.1 1024 0 fastclose_client
2928		chk_join_nr 0 0 0
2929		chk_fclose_nr 1 1
2930		chk_rst_nr 1 1 invert
2931	fi
2932
2933	if reset "fastclose server test"; then
2934		run_tests $ns1 $ns2 10.0.1.1 1024 0 fastclose_server
2935		chk_join_nr 0 0 0
2936		chk_fclose_nr 1 1 invert
2937		chk_rst_nr 1 1
2938	fi
2939}
2940
2941pedit_action_pkts()
2942{
2943	tc -n $ns2 -j -s action show action pedit index 100 | \
2944		grep "packets" | \
2945		sed 's/.*"packets":\([0-9]\+\),.*/\1/'
2946}
2947
2948fail_tests()
2949{
2950	# single subflow
2951	if reset_with_fail "Infinite map" 1; then
2952		run_tests $ns1 $ns2 10.0.1.1 128
2953		chk_join_nr 0 0 0 +1 +0 1 0 1 "$(pedit_action_pkts)"
2954		chk_fail_nr 1 -1 invert
2955	fi
2956
2957	# multiple subflows
2958	if reset_with_fail "MP_FAIL MP_RST" 2; then
2959		tc -n $ns2 qdisc add dev ns2eth1 root netem rate 1mbit delay 5
2960		pm_nl_set_limits $ns1 0 1
2961		pm_nl_set_limits $ns2 0 1
2962		pm_nl_add_endpoint $ns2 10.0.2.2 dev ns2eth2 flags subflow
2963		run_tests $ns1 $ns2 10.0.1.1 1024
2964		chk_join_nr 1 1 1 1 0 1 1 0 "$(pedit_action_pkts)"
2965	fi
2966}
2967
2968userspace_tests()
2969{
2970	# userspace pm type prevents add_addr
2971	if reset "userspace pm type prevents add_addr"; then
2972		set_userspace_pm $ns1
2973		pm_nl_set_limits $ns1 0 2
2974		pm_nl_set_limits $ns2 0 2
2975		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2976		run_tests $ns1 $ns2 10.0.1.1
2977		chk_join_nr 0 0 0
2978		chk_add_nr 0 0
2979	fi
2980
2981	# userspace pm type does not echo add_addr without daemon
2982	if reset "userspace pm no echo w/o daemon"; then
2983		set_userspace_pm $ns2
2984		pm_nl_set_limits $ns1 0 2
2985		pm_nl_set_limits $ns2 0 2
2986		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2987		run_tests $ns1 $ns2 10.0.1.1
2988		chk_join_nr 0 0 0
2989		chk_add_nr 1 0
2990	fi
2991
2992	# userspace pm type rejects join
2993	if reset "userspace pm type rejects join"; then
2994		set_userspace_pm $ns1
2995		pm_nl_set_limits $ns1 1 1
2996		pm_nl_set_limits $ns2 1 1
2997		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2998		run_tests $ns1 $ns2 10.0.1.1
2999		chk_join_nr 1 1 0
3000	fi
3001
3002	# userspace pm type does not send join
3003	if reset "userspace pm type does not send join"; then
3004		set_userspace_pm $ns2
3005		pm_nl_set_limits $ns1 1 1
3006		pm_nl_set_limits $ns2 1 1
3007		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
3008		run_tests $ns1 $ns2 10.0.1.1
3009		chk_join_nr 0 0 0
3010	fi
3011
3012	# userspace pm type prevents mp_prio
3013	if reset "userspace pm type prevents mp_prio"; then
3014		set_userspace_pm $ns1
3015		pm_nl_set_limits $ns1 1 1
3016		pm_nl_set_limits $ns2 1 1
3017		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
3018		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
3019		chk_join_nr 1 1 0
3020		chk_prio_nr 0 0
3021	fi
3022
3023	# userspace pm type prevents rm_addr
3024	if reset "userspace pm type prevents rm_addr"; then
3025		set_userspace_pm $ns1
3026		set_userspace_pm $ns2
3027		pm_nl_set_limits $ns1 0 1
3028		pm_nl_set_limits $ns2 0 1
3029		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
3030		run_tests $ns1 $ns2 10.0.1.1 0 0 -1 slow
3031		chk_join_nr 0 0 0
3032		chk_rm_nr 0 0
3033	fi
3034
3035	# userspace pm add & remove address
3036	if reset_with_events "userspace pm add & remove address"; then
3037		set_userspace_pm $ns1
3038		pm_nl_set_limits $ns2 1 1
3039		run_tests $ns1 $ns2 10.0.1.1 0 userspace_1 0 slow
3040		chk_join_nr 1 1 1
3041		chk_add_nr 1 1
3042		chk_rm_nr 1 1 invert
3043		kill_events_pids
3044	fi
3045
3046	# userspace pm create destroy subflow
3047	if reset_with_events "userspace pm create destroy subflow"; then
3048		set_userspace_pm $ns2
3049		pm_nl_set_limits $ns1 0 1
3050		run_tests $ns1 $ns2 10.0.1.1 0 0 userspace_1 slow
3051		chk_join_nr 1 1 1
3052		chk_rm_nr 0 1
3053		kill_events_pids
3054	fi
3055}
3056
3057endpoint_tests()
3058{
3059	# userspace pm type prevents add_addr
3060	if reset "implicit EP"; then
3061		pm_nl_set_limits $ns1 2 2
3062		pm_nl_set_limits $ns2 2 2
3063		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
3064		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow 2>/dev/null &
3065
3066		wait_mpj $ns1
3067		pm_nl_check_endpoint 1 "creation" \
3068			$ns2 10.0.2.2 id 1 flags implicit
3069
3070		pm_nl_add_endpoint $ns2 10.0.2.2 id 33
3071		pm_nl_check_endpoint 0 "ID change is prevented" \
3072			$ns2 10.0.2.2 id 1 flags implicit
3073
3074		pm_nl_add_endpoint $ns2 10.0.2.2 flags signal
3075		pm_nl_check_endpoint 0 "modif is allowed" \
3076			$ns2 10.0.2.2 id 1 flags signal
3077		kill_tests_wait
3078	fi
3079
3080	if reset "delete and re-add"; then
3081		pm_nl_set_limits $ns1 1 1
3082		pm_nl_set_limits $ns2 1 1
3083		pm_nl_add_endpoint $ns2 10.0.2.2 id 2 dev ns2eth2 flags subflow
3084		run_tests $ns1 $ns2 10.0.1.1 4 0 0 speed_20 2>/dev/null &
3085
3086		wait_mpj $ns2
3087		pm_nl_del_endpoint $ns2 2 10.0.2.2
3088		sleep 0.5
3089		chk_subflow_nr needtitle "after delete" 1
3090
3091		pm_nl_add_endpoint $ns2 10.0.2.2 dev ns2eth2 flags subflow
3092		wait_mpj $ns2
3093		chk_subflow_nr "" "after re-add" 2
3094		kill_tests_wait
3095	fi
3096}
3097
3098# [$1: error message]
3099usage()
3100{
3101	if [ -n "${1}" ]; then
3102		echo "${1}"
3103		ret=1
3104	fi
3105
3106	echo "mptcp_join usage:"
3107
3108	local key
3109	for key in "${!all_tests[@]}"; do
3110		echo "  -${key} ${all_tests[${key}]}"
3111	done
3112
3113	echo "  -c capture pcap files"
3114	echo "  -C enable data checksum"
3115	echo "  -i use ip mptcp"
3116	echo "  -h help"
3117
3118	echo "[test ids|names]"
3119
3120	exit ${ret}
3121}
3122
3123
3124# Use a "simple" array to force an specific order we cannot have with an associative one
3125all_tests_sorted=(
3126	f@subflows_tests
3127	e@subflows_error_tests
3128	s@signal_address_tests
3129	l@link_failure_tests
3130	t@add_addr_timeout_tests
3131	r@remove_tests
3132	a@add_tests
3133	6@ipv6_tests
3134	4@v4mapped_tests
3135	b@backup_tests
3136	p@add_addr_ports_tests
3137	k@syncookies_tests
3138	S@checksum_tests
3139	d@deny_join_id0_tests
3140	m@fullmesh_tests
3141	z@fastclose_tests
3142	F@fail_tests
3143	u@userspace_tests
3144	I@endpoint_tests
3145)
3146
3147all_tests_args=""
3148all_tests_names=()
3149for subtests in "${all_tests_sorted[@]}"; do
3150	key="${subtests%@*}"
3151	value="${subtests#*@}"
3152
3153	all_tests_args+="${key}"
3154	all_tests_names+=("${value}")
3155	all_tests[${key}]="${value}"
3156done
3157
3158tests=()
3159while getopts "${all_tests_args}cCih" opt; do
3160	case $opt in
3161		["${all_tests_args}"])
3162			tests+=("${all_tests[${opt}]}")
3163			;;
3164		c)
3165			capture=1
3166			;;
3167		C)
3168			checksum=1
3169			;;
3170		i)
3171			ip_mptcp=1
3172			;;
3173		h)
3174			usage
3175			;;
3176		*)
3177			usage "Unknown option: -${opt}"
3178			;;
3179	esac
3180done
3181
3182shift $((OPTIND - 1))
3183
3184for arg in "${@}"; do
3185	if [[ "${arg}" =~ ^[0-9]+$ ]]; then
3186		only_tests_ids+=("${arg}")
3187	else
3188		only_tests_names+=("${arg}")
3189	fi
3190done
3191
3192if [ ${#tests[@]} -eq 0 ]; then
3193	tests=("${all_tests_names[@]}")
3194fi
3195
3196for subtests in "${tests[@]}"; do
3197	"${subtests}"
3198done
3199
3200if [ ${ret} -ne 0 ]; then
3201	echo
3202	echo "${#failed_tests[@]} failure(s) has(ve) been detected:"
3203	for i in $(get_failed_tests_ids); do
3204		echo -e "\t- ${i}: ${failed_tests[${i}]}"
3205	done
3206	echo
3207fi
3208
3209exit $ret
3210