1#!/bin/bash
2# SPDX-License-Identifier: GPL-2.0
3
4# Double quotes to prevent globbing and word splitting is recommended in new
5# code but we accept it, especially because there were too many before having
6# address all other issues detected by shellcheck.
7#shellcheck disable=SC2086
8
9ret=0
10sin=""
11sinfail=""
12sout=""
13cin=""
14cinfail=""
15cinsent=""
16cout=""
17capout=""
18ns1=""
19ns2=""
20ksft_skip=4
21timeout_poll=30
22timeout_test=$((timeout_poll * 2 + 1))
23capture=0
24checksum=0
25ip_mptcp=0
26check_invert=0
27validate_checksum=0
28init=0
29evts_ns1=""
30evts_ns2=""
31evts_ns1_pid=0
32evts_ns2_pid=0
33
34declare -A all_tests
35declare -a only_tests_ids
36declare -a only_tests_names
37declare -A failed_tests
38TEST_COUNT=0
39TEST_NAME=""
40nr_blank=40
41
42export FAILING_LINKS=""
43
44# generated using "nfbpf_compile '(ip && (ip[54] & 0xf0) == 0x30) ||
45#				  (ip6 && (ip6[74] & 0xf0) == 0x30)'"
46CBPF_MPTCP_SUBOPTION_ADD_ADDR="14,
47			       48 0 0 0,
48			       84 0 0 240,
49			       21 0 3 64,
50			       48 0 0 54,
51			       84 0 0 240,
52			       21 6 7 48,
53			       48 0 0 0,
54			       84 0 0 240,
55			       21 0 4 96,
56			       48 0 0 74,
57			       84 0 0 240,
58			       21 0 1 48,
59			       6 0 0 65535,
60			       6 0 0 0"
61
62init_partial()
63{
64	capout=$(mktemp)
65
66	local sec rndh
67	sec=$(date +%s)
68	rndh=$(printf %x $sec)-$(mktemp -u XXXXXX)
69
70	ns1="ns1-$rndh"
71	ns2="ns2-$rndh"
72
73	local netns
74	for netns in "$ns1" "$ns2"; do
75		ip netns add $netns || exit $ksft_skip
76		ip -net $netns link set lo up
77		ip netns exec $netns sysctl -q net.mptcp.enabled=1
78		ip netns exec $netns sysctl -q net.mptcp.pm_type=0
79		ip netns exec $netns sysctl -q net.ipv4.conf.all.rp_filter=0
80		ip netns exec $netns sysctl -q net.ipv4.conf.default.rp_filter=0
81		if [ $checksum -eq 1 ]; then
82			ip netns exec $netns sysctl -q net.mptcp.checksum_enabled=1
83		fi
84	done
85
86	check_invert=0
87	validate_checksum=$checksum
88	FAILING_LINKS=""
89
90	#  ns1         ns2
91	# ns1eth1    ns2eth1
92	# ns1eth2    ns2eth2
93	# ns1eth3    ns2eth3
94	# ns1eth4    ns2eth4
95
96	local i
97	for i in $(seq 1 4); do
98		ip link add ns1eth$i netns "$ns1" type veth peer name ns2eth$i netns "$ns2"
99		ip -net "$ns1" addr add 10.0.$i.1/24 dev ns1eth$i
100		ip -net "$ns1" addr add dead:beef:$i::1/64 dev ns1eth$i nodad
101		ip -net "$ns1" link set ns1eth$i up
102
103		ip -net "$ns2" addr add 10.0.$i.2/24 dev ns2eth$i
104		ip -net "$ns2" addr add dead:beef:$i::2/64 dev ns2eth$i nodad
105		ip -net "$ns2" link set ns2eth$i up
106
107		# let $ns2 reach any $ns1 address from any interface
108		ip -net "$ns2" route add default via 10.0.$i.1 dev ns2eth$i metric 10$i
109		ip -net "$ns2" route add default via dead:beef:$i::1 dev ns2eth$i metric 10$i
110	done
111}
112
113init_shapers()
114{
115	local i
116	for i in $(seq 1 4); do
117		tc -n $ns1 qdisc add dev ns1eth$i root netem rate 20mbit delay 1
118		tc -n $ns2 qdisc add dev ns2eth$i root netem rate 20mbit delay 1
119	done
120}
121
122cleanup_partial()
123{
124	rm -f "$capout"
125
126	local netns
127	for netns in "$ns1" "$ns2"; do
128		ip netns del $netns
129		rm -f /tmp/$netns.{nstat,out}
130	done
131}
132
133check_tools()
134{
135	if ! ip -Version &> /dev/null; then
136		echo "SKIP: Could not run test without ip tool"
137		exit $ksft_skip
138	fi
139
140	if ! iptables -V &> /dev/null; then
141		echo "SKIP: Could not run all tests without iptables tool"
142		exit $ksft_skip
143	fi
144
145	if ! ip6tables -V &> /dev/null; then
146		echo "SKIP: Could not run all tests without ip6tables tool"
147		exit $ksft_skip
148	fi
149}
150
151init() {
152	init=1
153
154	check_tools
155
156	sin=$(mktemp)
157	sout=$(mktemp)
158	cin=$(mktemp)
159	cinsent=$(mktemp)
160	cout=$(mktemp)
161	evts_ns1=$(mktemp)
162	evts_ns2=$(mktemp)
163
164	trap cleanup EXIT
165
166	make_file "$cin" "client" 1
167	make_file "$sin" "server" 1
168}
169
170cleanup()
171{
172	rm -f "$cin" "$cout" "$sinfail"
173	rm -f "$sin" "$sout" "$cinsent" "$cinfail"
174	rm -rf $evts_ns1 $evts_ns2
175	cleanup_partial
176}
177
178skip_test()
179{
180	if [ "${#only_tests_ids[@]}" -eq 0 ] && [ "${#only_tests_names[@]}" -eq 0 ]; then
181		return 1
182	fi
183
184	local i
185	for i in "${only_tests_ids[@]}"; do
186		if [ "${TEST_COUNT}" -eq "${i}" ]; then
187			return 1
188		fi
189	done
190	for i in "${only_tests_names[@]}"; do
191		if [ "${TEST_NAME}" = "${i}" ]; then
192			return 1
193		fi
194	done
195
196	return 0
197}
198
199# $1: test name
200reset()
201{
202	TEST_NAME="${1}"
203
204	TEST_COUNT=$((TEST_COUNT+1))
205
206	if skip_test; then
207		return 1
208	fi
209
210	if [ "${init}" != "1" ]; then
211		init
212	else
213		cleanup_partial
214	fi
215
216	init_partial
217
218	return 0
219}
220
221# $1: test name
222reset_with_cookies()
223{
224	reset "${1}" || return 1
225
226	local netns
227	for netns in "$ns1" "$ns2"; do
228		ip netns exec $netns sysctl -q net.ipv4.tcp_syncookies=2
229	done
230}
231
232# $1: test name
233reset_with_add_addr_timeout()
234{
235	local ip="${2:-4}"
236	local tables
237
238	reset "${1}" || return 1
239
240	tables="iptables"
241	if [ $ip -eq 6 ]; then
242		tables="ip6tables"
243	fi
244
245	ip netns exec $ns1 sysctl -q net.mptcp.add_addr_timeout=1
246	ip netns exec $ns2 $tables -A OUTPUT -p tcp \
247		-m tcp --tcp-option 30 \
248		-m bpf --bytecode \
249		"$CBPF_MPTCP_SUBOPTION_ADD_ADDR" \
250		-j DROP
251}
252
253# $1: test name
254reset_with_checksum()
255{
256	local ns1_enable=$1
257	local ns2_enable=$2
258
259	reset "checksum test ${1} ${2}" || return 1
260
261	ip netns exec $ns1 sysctl -q net.mptcp.checksum_enabled=$ns1_enable
262	ip netns exec $ns2 sysctl -q net.mptcp.checksum_enabled=$ns2_enable
263
264	validate_checksum=1
265}
266
267reset_with_allow_join_id0()
268{
269	local ns1_enable=$2
270	local ns2_enable=$3
271
272	reset "${1}" || return 1
273
274	ip netns exec $ns1 sysctl -q net.mptcp.allow_join_initial_addr_port=$ns1_enable
275	ip netns exec $ns2 sysctl -q net.mptcp.allow_join_initial_addr_port=$ns2_enable
276}
277
278# Modify TCP payload without corrupting the TCP packet
279#
280# This rule inverts a 8-bit word at byte offset 148 for the 2nd TCP ACK packets
281# carrying enough data.
282# Once it is done, the TCP Checksum field is updated so the packet is still
283# considered as valid at the TCP level.
284# Because the MPTCP checksum, covering the TCP options and data, has not been
285# updated, the modification will be detected and an MP_FAIL will be emitted:
286# what we want to validate here without corrupting "random" MPTCP options.
287#
288# To avoid having tc producing this pr_info() message for each TCP ACK packets
289# not carrying enough data:
290#
291#     tc action pedit offset 162 out of bounds
292#
293# Netfilter is used to mark packets with enough data.
294reset_with_fail()
295{
296	reset "${1}" || return 1
297
298	ip netns exec $ns1 sysctl -q net.mptcp.checksum_enabled=1
299	ip netns exec $ns2 sysctl -q net.mptcp.checksum_enabled=1
300
301	check_invert=1
302	validate_checksum=1
303	local i="$2"
304	local ip="${3:-4}"
305	local tables
306
307	tables="iptables"
308	if [ $ip -eq 6 ]; then
309		tables="ip6tables"
310	fi
311
312	ip netns exec $ns2 $tables \
313		-t mangle \
314		-A OUTPUT \
315		-o ns2eth$i \
316		-p tcp \
317		-m length --length 150:9999 \
318		-m statistic --mode nth --packet 1 --every 99999 \
319		-j MARK --set-mark 42 || exit 1
320
321	tc -n $ns2 qdisc add dev ns2eth$i clsact || exit 1
322	tc -n $ns2 filter add dev ns2eth$i egress \
323		protocol ip prio 1000 \
324		handle 42 fw \
325		action pedit munge offset 148 u8 invert \
326		pipe csum tcp \
327		index 100 || exit 1
328}
329
330reset_with_events()
331{
332	reset "${1}" || return 1
333
334	:> "$evts_ns1"
335	:> "$evts_ns2"
336	ip netns exec $ns1 ./pm_nl_ctl events >> "$evts_ns1" 2>&1 &
337	evts_ns1_pid=$!
338	ip netns exec $ns2 ./pm_nl_ctl events >> "$evts_ns2" 2>&1 &
339	evts_ns2_pid=$!
340}
341
342fail_test()
343{
344	ret=1
345	failed_tests[${TEST_COUNT}]="${TEST_NAME}"
346}
347
348get_failed_tests_ids()
349{
350	# sorted
351	local i
352	for i in "${!failed_tests[@]}"; do
353		echo "${i}"
354	done | sort -n
355}
356
357print_file_err()
358{
359	ls -l "$1" 1>&2
360	echo "Trailing bytes are: "
361	tail -c 27 "$1"
362}
363
364check_transfer()
365{
366	local in=$1
367	local out=$2
368	local what=$3
369	local bytes=$4
370	local i a b
371
372	local line
373	if [ -n "$bytes" ]; then
374		# when truncating we must check the size explicitly
375		local out_size=$(wc -c $out | awk '{print $1}')
376		if [ $out_size -ne $bytes ]; then
377			echo "[ FAIL ] $what output file has wrong size ($out_size, $bytes)"
378			fail_test
379			return 1
380		fi
381		bytes="--bytes=${bytes}"
382	fi
383	cmp -l "$in" "$out" ${bytes} | while read -r i a b; do
384		local sum=$((0${a} + 0${b}))
385		if [ $check_invert -eq 0 ] || [ $sum -ne $((0xff)) ]; then
386			echo "[ FAIL ] $what does not match (in, out):"
387			print_file_err "$in"
388			print_file_err "$out"
389			fail_test
390
391			return 1
392		else
393			echo "$what has inverted byte at ${i}"
394		fi
395	done
396
397	return 0
398}
399
400do_ping()
401{
402	local listener_ns="$1"
403	local connector_ns="$2"
404	local connect_addr="$3"
405
406	if ! ip netns exec ${connector_ns} ping -q -c 1 $connect_addr >/dev/null; then
407		echo "$listener_ns -> $connect_addr connectivity [ FAIL ]" 1>&2
408		fail_test
409	fi
410}
411
412link_failure()
413{
414	local ns="$1"
415
416	if [ -z "$FAILING_LINKS" ]; then
417		l=$((RANDOM%4))
418		FAILING_LINKS=$((l+1))
419	fi
420
421	local l
422	for l in $FAILING_LINKS; do
423		local veth="ns1eth$l"
424		ip -net "$ns" link set "$veth" down
425	done
426}
427
428# $1: IP address
429is_v6()
430{
431	[ -z "${1##*:*}" ]
432}
433
434# $1: ns, $2: port
435wait_local_port_listen()
436{
437	local listener_ns="${1}"
438	local port="${2}"
439
440	local port_hex
441	port_hex="$(printf "%04X" "${port}")"
442
443	local i
444	for i in $(seq 10); do
445		ip netns exec "${listener_ns}" cat /proc/net/tcp* | \
446			awk "BEGIN {rc=1} {if (\$2 ~ /:${port_hex}\$/ && \$4 ~ /0A/) {rc=0; exit}} END {exit rc}" &&
447			break
448		sleep 0.1
449	done
450}
451
452rm_addr_count()
453{
454	local ns=${1}
455
456	ip netns exec ${ns} nstat -as | grep MPTcpExtRmAddr | awk '{print $2}'
457}
458
459# $1: ns, $2: old rm_addr counter in $ns
460wait_rm_addr()
461{
462	local ns="${1}"
463	local old_cnt="${2}"
464	local cnt
465
466	local i
467	for i in $(seq 10); do
468		cnt=$(rm_addr_count ${ns})
469		[ "$cnt" = "${old_cnt}" ] || break
470		sleep 0.1
471	done
472}
473
474wait_mpj()
475{
476	local ns="${1}"
477	local cnt old_cnt
478
479	old_cnt=$(ip netns exec ${ns} nstat -as | grep MPJoinAckRx | awk '{print $2}')
480
481	local i
482	for i in $(seq 10); do
483		cnt=$(ip netns exec ${ns} nstat -as | grep MPJoinAckRx | awk '{print $2}')
484		[ "$cnt" = "${old_cnt}" ] || break
485		sleep 0.1
486	done
487}
488
489kill_wait()
490{
491	kill $1 > /dev/null 2>&1
492	wait $1 2>/dev/null
493}
494
495kill_events_pids()
496{
497	kill_wait $evts_ns1_pid
498	kill_wait $evts_ns2_pid
499}
500
501pm_nl_set_limits()
502{
503	local ns=$1
504	local addrs=$2
505	local subflows=$3
506
507	if [ $ip_mptcp -eq 1 ]; then
508		ip -n $ns mptcp limits set add_addr_accepted $addrs subflows $subflows
509	else
510		ip netns exec $ns ./pm_nl_ctl limits $addrs $subflows
511	fi
512}
513
514pm_nl_add_endpoint()
515{
516	local ns=$1
517	local addr=$2
518	local flags _flags
519	local port _port
520	local dev _dev
521	local id _id
522	local nr=2
523
524	local p
525	for p in "${@}"
526	do
527		if [ $p = "flags" ]; then
528			eval _flags=\$"$nr"
529			[ -n "$_flags" ]; flags="flags $_flags"
530		fi
531		if [ $p = "dev" ]; then
532			eval _dev=\$"$nr"
533			[ -n "$_dev" ]; dev="dev $_dev"
534		fi
535		if [ $p = "id" ]; then
536			eval _id=\$"$nr"
537			[ -n "$_id" ]; id="id $_id"
538		fi
539		if [ $p = "port" ]; then
540			eval _port=\$"$nr"
541			[ -n "$_port" ]; port="port $_port"
542		fi
543
544		nr=$((nr + 1))
545	done
546
547	if [ $ip_mptcp -eq 1 ]; then
548		ip -n $ns mptcp endpoint add $addr ${_flags//","/" "} $dev $id $port
549	else
550		ip netns exec $ns ./pm_nl_ctl add $addr $flags $dev $id $port
551	fi
552}
553
554pm_nl_del_endpoint()
555{
556	local ns=$1
557	local id=$2
558	local addr=$3
559
560	if [ $ip_mptcp -eq 1 ]; then
561		ip -n $ns mptcp endpoint delete id $id $addr
562	else
563		ip netns exec $ns ./pm_nl_ctl del $id $addr
564	fi
565}
566
567pm_nl_flush_endpoint()
568{
569	local ns=$1
570
571	if [ $ip_mptcp -eq 1 ]; then
572		ip -n $ns mptcp endpoint flush
573	else
574		ip netns exec $ns ./pm_nl_ctl flush
575	fi
576}
577
578pm_nl_show_endpoints()
579{
580	local ns=$1
581
582	if [ $ip_mptcp -eq 1 ]; then
583		ip -n $ns mptcp endpoint show
584	else
585		ip netns exec $ns ./pm_nl_ctl dump
586	fi
587}
588
589pm_nl_change_endpoint()
590{
591	local ns=$1
592	local id=$2
593	local flags=$3
594
595	if [ $ip_mptcp -eq 1 ]; then
596		ip -n $ns mptcp endpoint change id $id ${flags//","/" "}
597	else
598		ip netns exec $ns ./pm_nl_ctl set id $id flags $flags
599	fi
600}
601
602pm_nl_check_endpoint()
603{
604	local line expected_line
605	local need_title=$1
606	local msg="$2"
607	local ns=$3
608	local addr=$4
609	local _flags=""
610	local flags
611	local _port
612	local port
613	local dev
614	local _id
615	local id
616
617	if [ "${need_title}" = 1 ]; then
618		printf "%03u %-36s %s" "${TEST_COUNT}" "${TEST_NAME}" "${msg}"
619	else
620		printf "%-${nr_blank}s %s" " " "${msg}"
621	fi
622
623	shift 4
624	while [ -n "$1" ]; do
625		if [ $1 = "flags" ]; then
626			_flags=$2
627			[ -n "$_flags" ]; flags="flags $_flags"
628			shift
629		elif [ $1 = "dev" ]; then
630			[ -n "$2" ]; dev="dev $1"
631			shift
632		elif [ $1 = "id" ]; then
633			_id=$2
634			[ -n "$_id" ]; id="id $_id"
635			shift
636		elif [ $1 = "port" ]; then
637			_port=$2
638			[ -n "$_port" ]; port=" port $_port"
639			shift
640		fi
641
642		shift
643	done
644
645	if [ -z "$id" ]; then
646		echo "[skip] bad test - missing endpoint id"
647		return
648	fi
649
650	if [ $ip_mptcp -eq 1 ]; then
651		line=$(ip -n $ns mptcp endpoint show $id)
652		# the dump order is: address id flags port dev
653		expected_line="$addr"
654		[ -n "$addr" ] && expected_line="$expected_line $addr"
655		expected_line="$expected_line $id"
656		[ -n "$_flags" ] && expected_line="$expected_line ${_flags//","/" "}"
657		[ -n "$dev" ] && expected_line="$expected_line $dev"
658		[ -n "$port" ] && expected_line="$expected_line $port"
659	else
660		line=$(ip netns exec $ns ./pm_nl_ctl get $_id)
661		# the dump order is: id flags dev address port
662		expected_line="$id"
663		[ -n "$flags" ] && expected_line="$expected_line $flags"
664		[ -n "$dev" ] && expected_line="$expected_line $dev"
665		[ -n "$addr" ] && expected_line="$expected_line $addr"
666		[ -n "$_port" ] && expected_line="$expected_line $_port"
667	fi
668	if [ "$line" = "$expected_line" ]; then
669		echo "[ ok ]"
670	else
671		echo "[fail] expected '$expected_line' found '$line'"
672		fail_test
673	fi
674}
675
676filter_tcp_from()
677{
678	local ns="${1}"
679	local src="${2}"
680	local target="${3}"
681
682	ip netns exec "${ns}" iptables -A INPUT -s "${src}" -p tcp -j "${target}"
683}
684
685do_transfer()
686{
687	local listener_ns="$1"
688	local connector_ns="$2"
689	local cl_proto="$3"
690	local srv_proto="$4"
691	local connect_addr="$5"
692	local test_link_fail="$6"
693	local addr_nr_ns1="$7"
694	local addr_nr_ns2="$8"
695	local speed="$9"
696	local sflags="${10}"
697
698	local port=$((10000 + TEST_COUNT - 1))
699	local cappid
700	local userspace_pm=0
701
702	:> "$cout"
703	:> "$sout"
704	:> "$capout"
705
706	if [ $capture -eq 1 ]; then
707		local capuser
708		if [ -z $SUDO_USER ] ; then
709			capuser=""
710		else
711			capuser="-Z $SUDO_USER"
712		fi
713
714		capfile=$(printf "mp_join-%02u-%s.pcap" "$TEST_COUNT" "${listener_ns}")
715
716		echo "Capturing traffic for test $TEST_COUNT into $capfile"
717		ip netns exec ${listener_ns} tcpdump -i any -s 65535 -B 32768 $capuser -w $capfile > "$capout" 2>&1 &
718		cappid=$!
719
720		sleep 1
721	fi
722
723	NSTAT_HISTORY=/tmp/${listener_ns}.nstat ip netns exec ${listener_ns} \
724		nstat -n
725	NSTAT_HISTORY=/tmp/${connector_ns}.nstat ip netns exec ${connector_ns} \
726		nstat -n
727
728	local extra_args
729	if [ $speed = "fast" ]; then
730		extra_args="-j"
731	elif [ $speed = "slow" ]; then
732		extra_args="-r 50"
733	elif [[ $speed = "speed_"* ]]; then
734		extra_args="-r ${speed:6}"
735	fi
736
737	if [[ "${addr_nr_ns1}" = "userspace_"* ]]; then
738		userspace_pm=1
739		addr_nr_ns1=${addr_nr_ns1:10}
740	fi
741
742	local flags="subflow"
743	local extra_cl_args=""
744	local extra_srv_args=""
745	local trunc_size=""
746	if [[ "${addr_nr_ns2}" = "fastclose_"* ]]; then
747		if [ ${test_link_fail} -le 1 ]; then
748			echo "fastclose tests need test_link_fail argument"
749			fail_test
750			return 1
751		fi
752
753		# disconnect
754		trunc_size=${test_link_fail}
755		local side=${addr_nr_ns2:10}
756
757		if [ ${side} = "client" ]; then
758			extra_cl_args="-f ${test_link_fail}"
759			extra_srv_args="-f -1"
760		elif [ ${side} = "server" ]; then
761			extra_srv_args="-f ${test_link_fail}"
762			extra_cl_args="-f -1"
763		else
764			echo "wrong/unknown fastclose spec ${side}"
765			fail_test
766			return 1
767		fi
768		addr_nr_ns2=0
769	elif [[ "${addr_nr_ns2}" = "userspace_"* ]]; then
770		userspace_pm=1
771		addr_nr_ns2=${addr_nr_ns2:10}
772	elif [[ "${addr_nr_ns2}" = "fullmesh_"* ]]; then
773		flags="${flags},fullmesh"
774		addr_nr_ns2=${addr_nr_ns2:9}
775	fi
776
777	local local_addr
778	if is_v6 "${connect_addr}"; then
779		local_addr="::"
780	else
781		local_addr="0.0.0.0"
782	fi
783
784	extra_srv_args="$extra_args $extra_srv_args"
785	if [ "$test_link_fail" -gt 1 ];then
786		timeout ${timeout_test} \
787			ip netns exec ${listener_ns} \
788				./mptcp_connect -t ${timeout_poll} -l -p $port -s ${srv_proto} \
789					$extra_srv_args ${local_addr} < "$sinfail" > "$sout" &
790	else
791		timeout ${timeout_test} \
792			ip netns exec ${listener_ns} \
793				./mptcp_connect -t ${timeout_poll} -l -p $port -s ${srv_proto} \
794					$extra_srv_args ${local_addr} < "$sin" > "$sout" &
795	fi
796	local spid=$!
797
798	wait_local_port_listen "${listener_ns}" "${port}"
799
800	extra_cl_args="$extra_args $extra_cl_args"
801	if [ "$test_link_fail" -eq 0 ];then
802		timeout ${timeout_test} \
803			ip netns exec ${connector_ns} \
804				./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \
805					$extra_cl_args $connect_addr < "$cin" > "$cout" &
806	elif [ "$test_link_fail" -eq 1 ] || [ "$test_link_fail" -eq 2 ];then
807		( cat "$cinfail" ; sleep 2; link_failure $listener_ns ; cat "$cinfail" ) | \
808			tee "$cinsent" | \
809			timeout ${timeout_test} \
810				ip netns exec ${connector_ns} \
811					./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \
812						$extra_cl_args $connect_addr > "$cout" &
813	else
814		tee "$cinsent" < "$cinfail" | \
815			timeout ${timeout_test} \
816				ip netns exec ${connector_ns} \
817					./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \
818						$extra_cl_args $connect_addr > "$cout" &
819	fi
820	local cpid=$!
821
822	# let the mptcp subflow be established in background before
823	# do endpoint manipulation
824	if [ $addr_nr_ns1 != "0" ] || [ $addr_nr_ns2 != "0" ]; then
825		sleep 1
826	fi
827
828	if [ $addr_nr_ns1 -gt 0 ]; then
829		local counter=2
830		local add_nr_ns1=${addr_nr_ns1}
831		local id=10
832		local tk
833		while [ $add_nr_ns1 -gt 0 ]; do
834			local addr
835			if is_v6 "${connect_addr}"; then
836				addr="dead:beef:$counter::1"
837			else
838				addr="10.0.$counter.1"
839			fi
840			if [ $userspace_pm -eq 0 ]; then
841				pm_nl_add_endpoint $ns1 $addr flags signal
842			else
843				tk=$(grep "type:1," "$evts_ns1" |
844				     sed -n 's/.*\(token:\)\([[:digit:]]*\).*$/\2/p;q')
845				ip netns exec ${listener_ns} ./pm_nl_ctl ann $addr token $tk id $id
846				sleep 1
847				ip netns exec ${listener_ns} ./pm_nl_ctl rem token $tk id $id
848			fi
849
850			counter=$((counter + 1))
851			add_nr_ns1=$((add_nr_ns1 - 1))
852			id=$((id + 1))
853		done
854	elif [ $addr_nr_ns1 -lt 0 ]; then
855		local rm_nr_ns1=$((-addr_nr_ns1))
856		if [ $rm_nr_ns1 -lt 8 ]; then
857			local counter=0
858			local line
859			pm_nl_show_endpoints ${listener_ns} | while read -r line; do
860				# shellcheck disable=SC2206 # we do want to split per word
861				local arr=($line)
862				local nr=0
863
864				local i
865				for i in "${arr[@]}"; do
866					if [ $i = "id" ]; then
867						if [ $counter -eq $rm_nr_ns1 ]; then
868							break
869						fi
870						id=${arr[$nr+1]}
871						rm_addr=$(rm_addr_count ${connector_ns})
872						pm_nl_del_endpoint ${listener_ns} $id
873						wait_rm_addr ${connector_ns} ${rm_addr}
874						counter=$((counter + 1))
875					fi
876					nr=$((nr + 1))
877				done
878			done
879		elif [ $rm_nr_ns1 -eq 8 ]; then
880			pm_nl_flush_endpoint ${listener_ns}
881		elif [ $rm_nr_ns1 -eq 9 ]; then
882			pm_nl_del_endpoint ${listener_ns} 0 ${connect_addr}
883		fi
884	fi
885
886	# if newly added endpoints must be deleted, give the background msk
887	# some time to created them
888	[ $addr_nr_ns1 -gt 0 ] && [ $addr_nr_ns2 -lt 0 ] && sleep 1
889
890	if [ $addr_nr_ns2 -gt 0 ]; then
891		local add_nr_ns2=${addr_nr_ns2}
892		local counter=3
893		local id=20
894		local tk da dp sp
895		while [ $add_nr_ns2 -gt 0 ]; do
896			local addr
897			if is_v6 "${connect_addr}"; then
898				addr="dead:beef:$counter::2"
899			else
900				addr="10.0.$counter.2"
901			fi
902			if [ $userspace_pm -eq 0 ]; then
903				pm_nl_add_endpoint $ns2 $addr flags $flags
904			else
905				tk=$(sed -n 's/.*\(token:\)\([[:digit:]]*\).*$/\2/p;q' "$evts_ns2")
906				da=$(sed -n 's/.*\(daddr4:\)\([0-9.]*\).*$/\2/p;q' "$evts_ns2")
907				dp=$(sed -n 's/.*\(dport:\)\([[:digit:]]*\).*$/\2/p;q' "$evts_ns2")
908				ip netns exec ${connector_ns} ./pm_nl_ctl csf lip $addr lid $id \
909									rip $da rport $dp token $tk
910				sleep 1
911				sp=$(grep "type:10" "$evts_ns2" |
912				     sed -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q')
913				ip netns exec ${connector_ns} ./pm_nl_ctl dsf lip $addr lport $sp \
914									rip $da rport $dp token $tk
915			fi
916			counter=$((counter + 1))
917			add_nr_ns2=$((add_nr_ns2 - 1))
918			id=$((id + 1))
919		done
920	elif [ $addr_nr_ns2 -lt 0 ]; then
921		local rm_nr_ns2=$((-addr_nr_ns2))
922		if [ $rm_nr_ns2 -lt 8 ]; then
923			local counter=0
924			local line
925			pm_nl_show_endpoints ${connector_ns} | while read -r line; do
926				# shellcheck disable=SC2206 # we do want to split per word
927				local arr=($line)
928				local nr=0
929
930				local i
931				for i in "${arr[@]}"; do
932					if [ $i = "id" ]; then
933						if [ $counter -eq $rm_nr_ns2 ]; then
934							break
935						fi
936						local id rm_addr
937						# rm_addr are serialized, allow the previous one to
938						# complete
939						id=${arr[$nr+1]}
940						rm_addr=$(rm_addr_count ${listener_ns})
941						pm_nl_del_endpoint ${connector_ns} $id
942						wait_rm_addr ${listener_ns} ${rm_addr}
943						counter=$((counter + 1))
944					fi
945					nr=$((nr + 1))
946				done
947			done
948		elif [ $rm_nr_ns2 -eq 8 ]; then
949			pm_nl_flush_endpoint ${connector_ns}
950		elif [ $rm_nr_ns2 -eq 9 ]; then
951			local addr
952			if is_v6 "${connect_addr}"; then
953				addr="dead:beef:1::2"
954			else
955				addr="10.0.1.2"
956			fi
957			pm_nl_del_endpoint ${connector_ns} 0 $addr
958		fi
959	fi
960
961	if [ -n "${sflags}" ]; then
962		sleep 1
963
964		local netns
965		for netns in "$ns1" "$ns2"; do
966			local line
967			pm_nl_show_endpoints $netns | while read -r line; do
968				# shellcheck disable=SC2206 # we do want to split per word
969				local arr=($line)
970				local nr=0
971				local id
972
973				local i
974				for i in "${arr[@]}"; do
975					if [ $i = "id" ]; then
976						id=${arr[$nr+1]}
977					fi
978					nr=$((nr + 1))
979				done
980				pm_nl_change_endpoint $netns $id $sflags
981			done
982		done
983	fi
984
985	wait $cpid
986	local retc=$?
987	wait $spid
988	local rets=$?
989
990	if [ $capture -eq 1 ]; then
991	    sleep 1
992	    kill $cappid
993	fi
994
995	NSTAT_HISTORY=/tmp/${listener_ns}.nstat ip netns exec ${listener_ns} \
996		nstat | grep Tcp > /tmp/${listener_ns}.out
997	NSTAT_HISTORY=/tmp/${connector_ns}.nstat ip netns exec ${connector_ns} \
998		nstat | grep Tcp > /tmp/${connector_ns}.out
999
1000	if [ ${rets} -ne 0 ] || [ ${retc} -ne 0 ]; then
1001		echo " client exit code $retc, server $rets" 1>&2
1002		echo -e "\nnetns ${listener_ns} socket stat for ${port}:" 1>&2
1003		ip netns exec ${listener_ns} ss -Menita 1>&2 -o "sport = :$port"
1004		cat /tmp/${listener_ns}.out
1005		echo -e "\nnetns ${connector_ns} socket stat for ${port}:" 1>&2
1006		ip netns exec ${connector_ns} ss -Menita 1>&2 -o "dport = :$port"
1007		cat /tmp/${connector_ns}.out
1008
1009		cat "$capout"
1010		fail_test
1011		return 1
1012	fi
1013
1014	if [ "$test_link_fail" -gt 1 ];then
1015		check_transfer $sinfail $cout "file received by client" $trunc_size
1016	else
1017		check_transfer $sin $cout "file received by client" $trunc_size
1018	fi
1019	retc=$?
1020	if [ "$test_link_fail" -eq 0 ];then
1021		check_transfer $cin $sout "file received by server" $trunc_size
1022	else
1023		check_transfer $cinsent $sout "file received by server" $trunc_size
1024	fi
1025	rets=$?
1026
1027	if [ $retc -eq 0 ] && [ $rets -eq 0 ];then
1028		cat "$capout"
1029		return 0
1030	fi
1031
1032	cat "$capout"
1033	return 1
1034}
1035
1036make_file()
1037{
1038	local name=$1
1039	local who=$2
1040	local size=$3
1041
1042	dd if=/dev/urandom of="$name" bs=1024 count=$size 2> /dev/null
1043	echo -e "\nMPTCP_TEST_FILE_END_MARKER" >> "$name"
1044
1045	echo "Created $name (size $size KB) containing data sent by $who"
1046}
1047
1048run_tests()
1049{
1050	local listener_ns="$1"
1051	local connector_ns="$2"
1052	local connect_addr="$3"
1053	local test_linkfail="${4:-0}"
1054	local addr_nr_ns1="${5:-0}"
1055	local addr_nr_ns2="${6:-0}"
1056	local speed="${7:-fast}"
1057	local sflags="${8:-""}"
1058
1059	local size
1060
1061	# The values above 2 are reused to make test files
1062	# with the given sizes (KB)
1063	if [ "$test_linkfail" -gt 2 ]; then
1064		size=$test_linkfail
1065
1066		if [ -z "$cinfail" ]; then
1067			cinfail=$(mktemp)
1068		fi
1069		make_file "$cinfail" "client" $size
1070	# create the input file for the failure test when
1071	# the first failure test run
1072	elif [ "$test_linkfail" -ne 0 ] && [ -z "$cinfail" ]; then
1073		# the client file must be considerably larger
1074		# of the maximum expected cwin value, or the
1075		# link utilization will be not predicable
1076		size=$((RANDOM%2))
1077		size=$((size+1))
1078		size=$((size*8192))
1079		size=$((size + ( RANDOM % 8192) ))
1080
1081		cinfail=$(mktemp)
1082		make_file "$cinfail" "client" $size
1083	fi
1084
1085	if [ "$test_linkfail" -gt 2 ]; then
1086		size=$test_linkfail
1087
1088		if [ -z "$sinfail" ]; then
1089			sinfail=$(mktemp)
1090		fi
1091		make_file "$sinfail" "server" $size
1092	elif [ "$test_linkfail" -eq 2 ] && [ -z "$sinfail" ]; then
1093		size=$((RANDOM%16))
1094		size=$((size+1))
1095		size=$((size*2048))
1096
1097		sinfail=$(mktemp)
1098		make_file "$sinfail" "server" $size
1099	fi
1100
1101	do_transfer ${listener_ns} ${connector_ns} MPTCP MPTCP ${connect_addr} \
1102		${test_linkfail} ${addr_nr_ns1} ${addr_nr_ns2} ${speed} ${sflags}
1103}
1104
1105dump_stats()
1106{
1107	echo Server ns stats
1108	ip netns exec $ns1 nstat -as | grep Tcp
1109	echo Client ns stats
1110	ip netns exec $ns2 nstat -as | grep Tcp
1111}
1112
1113chk_csum_nr()
1114{
1115	local csum_ns1=${1:-0}
1116	local csum_ns2=${2:-0}
1117	local count
1118	local dump_stats
1119	local extra_msg=""
1120	local allow_multi_errors_ns1=0
1121	local allow_multi_errors_ns2=0
1122
1123	if [[ "${csum_ns1}" = "+"* ]]; then
1124		allow_multi_errors_ns1=1
1125		csum_ns1=${csum_ns1:1}
1126	fi
1127	if [[ "${csum_ns2}" = "+"* ]]; then
1128		allow_multi_errors_ns2=1
1129		csum_ns2=${csum_ns2:1}
1130	fi
1131
1132	printf "%-${nr_blank}s %s" " " "sum"
1133	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtDataCsumErr | awk '{print $2}')
1134	[ -z "$count" ] && count=0
1135	if [ "$count" != "$csum_ns1" ]; then
1136		extra_msg="$extra_msg ns1=$count"
1137	fi
1138	if { [ "$count" != $csum_ns1 ] && [ $allow_multi_errors_ns1 -eq 0 ]; } ||
1139	   { [ "$count" -lt $csum_ns1 ] && [ $allow_multi_errors_ns1 -eq 1 ]; }; then
1140		echo "[fail] got $count data checksum error[s] expected $csum_ns1"
1141		fail_test
1142		dump_stats=1
1143	else
1144		echo -n "[ ok ]"
1145	fi
1146	echo -n " - csum  "
1147	count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtDataCsumErr | awk '{print $2}')
1148	[ -z "$count" ] && count=0
1149	if [ "$count" != "$csum_ns2" ]; then
1150		extra_msg="$extra_msg ns2=$count"
1151	fi
1152	if { [ "$count" != $csum_ns2 ] && [ $allow_multi_errors_ns2 -eq 0 ]; } ||
1153	   { [ "$count" -lt $csum_ns2 ] && [ $allow_multi_errors_ns2 -eq 1 ]; }; then
1154		echo "[fail] got $count data checksum error[s] expected $csum_ns2"
1155		fail_test
1156		dump_stats=1
1157	else
1158		echo -n "[ ok ]"
1159	fi
1160	[ "${dump_stats}" = 1 ] && dump_stats
1161
1162	echo "$extra_msg"
1163}
1164
1165chk_fail_nr()
1166{
1167	local fail_tx=$1
1168	local fail_rx=$2
1169	local ns_invert=${3:-""}
1170	local count
1171	local dump_stats
1172	local ns_tx=$ns1
1173	local ns_rx=$ns2
1174	local extra_msg=""
1175	local allow_tx_lost=0
1176	local allow_rx_lost=0
1177
1178	if [[ $ns_invert = "invert" ]]; then
1179		ns_tx=$ns2
1180		ns_rx=$ns1
1181		extra_msg=" invert"
1182	fi
1183
1184	if [[ "${fail_tx}" = "-"* ]]; then
1185		allow_tx_lost=1
1186		fail_tx=${fail_tx:1}
1187	fi
1188	if [[ "${fail_rx}" = "-"* ]]; then
1189		allow_rx_lost=1
1190		fail_rx=${fail_rx:1}
1191	fi
1192
1193	printf "%-${nr_blank}s %s" " " "ftx"
1194	count=$(ip netns exec $ns_tx nstat -as | grep MPTcpExtMPFailTx | awk '{print $2}')
1195	[ -z "$count" ] && count=0
1196	if [ "$count" != "$fail_tx" ]; then
1197		extra_msg="$extra_msg,tx=$count"
1198	fi
1199	if { [ "$count" != "$fail_tx" ] && [ $allow_tx_lost -eq 0 ]; } ||
1200	   { [ "$count" -gt "$fail_tx" ] && [ $allow_tx_lost -eq 1 ]; }; then
1201		echo "[fail] got $count MP_FAIL[s] TX expected $fail_tx"
1202		fail_test
1203		dump_stats=1
1204	else
1205		echo -n "[ ok ]"
1206	fi
1207
1208	echo -n " - failrx"
1209	count=$(ip netns exec $ns_rx nstat -as | grep MPTcpExtMPFailRx | awk '{print $2}')
1210	[ -z "$count" ] && count=0
1211	if [ "$count" != "$fail_rx" ]; then
1212		extra_msg="$extra_msg,rx=$count"
1213	fi
1214	if { [ "$count" != "$fail_rx" ] && [ $allow_rx_lost -eq 0 ]; } ||
1215	   { [ "$count" -gt "$fail_rx" ] && [ $allow_rx_lost -eq 1 ]; }; then
1216		echo "[fail] got $count MP_FAIL[s] RX expected $fail_rx"
1217		fail_test
1218		dump_stats=1
1219	else
1220		echo -n "[ ok ]"
1221	fi
1222
1223	[ "${dump_stats}" = 1 ] && dump_stats
1224
1225	echo "$extra_msg"
1226}
1227
1228chk_fclose_nr()
1229{
1230	local fclose_tx=$1
1231	local fclose_rx=$2
1232	local ns_invert=$3
1233	local count
1234	local dump_stats
1235	local ns_tx=$ns2
1236	local ns_rx=$ns1
1237	local extra_msg="   "
1238
1239	if [[ $ns_invert = "invert" ]]; then
1240		ns_tx=$ns1
1241		ns_rx=$ns2
1242		extra_msg=${extra_msg}"invert"
1243	fi
1244
1245	printf "%-${nr_blank}s %s" " " "ctx"
1246	count=$(ip netns exec $ns_tx nstat -as | grep MPTcpExtMPFastcloseTx | awk '{print $2}')
1247	[ -z "$count" ] && count=0
1248	[ "$count" != "$fclose_tx" ] && extra_msg="$extra_msg,tx=$count"
1249	if [ "$count" != "$fclose_tx" ]; then
1250		echo "[fail] got $count MP_FASTCLOSE[s] TX expected $fclose_tx"
1251		fail_test
1252		dump_stats=1
1253	else
1254		echo -n "[ ok ]"
1255	fi
1256
1257	echo -n " - fclzrx"
1258	count=$(ip netns exec $ns_rx nstat -as | grep MPTcpExtMPFastcloseRx | awk '{print $2}')
1259	[ -z "$count" ] && count=0
1260	[ "$count" != "$fclose_rx" ] && extra_msg="$extra_msg,rx=$count"
1261	if [ "$count" != "$fclose_rx" ]; then
1262		echo "[fail] got $count MP_FASTCLOSE[s] RX expected $fclose_rx"
1263		fail_test
1264		dump_stats=1
1265	else
1266		echo -n "[ ok ]"
1267	fi
1268
1269	[ "${dump_stats}" = 1 ] && dump_stats
1270
1271	echo "$extra_msg"
1272}
1273
1274chk_rst_nr()
1275{
1276	local rst_tx=$1
1277	local rst_rx=$2
1278	local ns_invert=${3:-""}
1279	local count
1280	local dump_stats
1281	local ns_tx=$ns1
1282	local ns_rx=$ns2
1283	local extra_msg=""
1284
1285	if [[ $ns_invert = "invert" ]]; then
1286		ns_tx=$ns2
1287		ns_rx=$ns1
1288		extra_msg="   invert"
1289	fi
1290
1291	printf "%-${nr_blank}s %s" " " "rtx"
1292	count=$(ip netns exec $ns_tx nstat -as | grep MPTcpExtMPRstTx | awk '{print $2}')
1293	[ -z "$count" ] && count=0
1294	if [ $count -lt $rst_tx ]; then
1295		echo "[fail] got $count MP_RST[s] TX expected $rst_tx"
1296		fail_test
1297		dump_stats=1
1298	else
1299		echo -n "[ ok ]"
1300	fi
1301
1302	echo -n " - rstrx "
1303	count=$(ip netns exec $ns_rx nstat -as | grep MPTcpExtMPRstRx | awk '{print $2}')
1304	[ -z "$count" ] && count=0
1305	if [ "$count" -lt "$rst_rx" ]; then
1306		echo "[fail] got $count MP_RST[s] RX expected $rst_rx"
1307		fail_test
1308		dump_stats=1
1309	else
1310		echo -n "[ ok ]"
1311	fi
1312
1313	[ "${dump_stats}" = 1 ] && dump_stats
1314
1315	echo "$extra_msg"
1316}
1317
1318chk_infi_nr()
1319{
1320	local infi_tx=$1
1321	local infi_rx=$2
1322	local count
1323	local dump_stats
1324
1325	printf "%-${nr_blank}s %s" " " "itx"
1326	count=$(ip netns exec $ns2 nstat -as | grep InfiniteMapTx | awk '{print $2}')
1327	[ -z "$count" ] && count=0
1328	if [ "$count" != "$infi_tx" ]; then
1329		echo "[fail] got $count infinite map[s] TX expected $infi_tx"
1330		fail_test
1331		dump_stats=1
1332	else
1333		echo -n "[ ok ]"
1334	fi
1335
1336	echo -n " - infirx"
1337	count=$(ip netns exec $ns1 nstat -as | grep InfiniteMapRx | awk '{print $2}')
1338	[ -z "$count" ] && count=0
1339	if [ "$count" != "$infi_rx" ]; then
1340		echo "[fail] got $count infinite map[s] RX expected $infi_rx"
1341		fail_test
1342		dump_stats=1
1343	else
1344		echo "[ ok ]"
1345	fi
1346
1347	[ "${dump_stats}" = 1 ] && dump_stats
1348}
1349
1350chk_join_nr()
1351{
1352	local syn_nr=$1
1353	local syn_ack_nr=$2
1354	local ack_nr=$3
1355	local csum_ns1=${4:-0}
1356	local csum_ns2=${5:-0}
1357	local fail_nr=${6:-0}
1358	local rst_nr=${7:-0}
1359	local infi_nr=${8:-0}
1360	local corrupted_pkts=${9:-0}
1361	local count
1362	local dump_stats
1363	local with_cookie
1364	local title="${TEST_NAME}"
1365
1366	if [ "${corrupted_pkts}" -gt 0 ]; then
1367		title+=": ${corrupted_pkts} corrupted pkts"
1368	fi
1369
1370	printf "%03u %-36s %s" "${TEST_COUNT}" "${title}" "syn"
1371	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinSynRx | awk '{print $2}')
1372	[ -z "$count" ] && count=0
1373	if [ "$count" != "$syn_nr" ]; then
1374		echo "[fail] got $count JOIN[s] syn expected $syn_nr"
1375		fail_test
1376		dump_stats=1
1377	else
1378		echo -n "[ ok ]"
1379	fi
1380
1381	echo -n " - synack"
1382	with_cookie=$(ip netns exec $ns2 sysctl -n net.ipv4.tcp_syncookies)
1383	count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtMPJoinSynAckRx | awk '{print $2}')
1384	[ -z "$count" ] && count=0
1385	if [ "$count" != "$syn_ack_nr" ]; then
1386		# simult connections exceeding the limit with cookie enabled could go up to
1387		# synack validation as the conn limit can be enforced reliably only after
1388		# the subflow creation
1389		if [ "$with_cookie" = 2 ] && [ "$count" -gt "$syn_ack_nr" ] && [ "$count" -le "$syn_nr" ]; then
1390			echo -n "[ ok ]"
1391		else
1392			echo "[fail] got $count JOIN[s] synack expected $syn_ack_nr"
1393			fail_test
1394			dump_stats=1
1395		fi
1396	else
1397		echo -n "[ ok ]"
1398	fi
1399
1400	echo -n " - ack"
1401	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinAckRx | awk '{print $2}')
1402	[ -z "$count" ] && count=0
1403	if [ "$count" != "$ack_nr" ]; then
1404		echo "[fail] got $count JOIN[s] ack expected $ack_nr"
1405		fail_test
1406		dump_stats=1
1407	else
1408		echo "[ ok ]"
1409	fi
1410	[ "${dump_stats}" = 1 ] && dump_stats
1411	if [ $validate_checksum -eq 1 ]; then
1412		chk_csum_nr $csum_ns1 $csum_ns2
1413		chk_fail_nr $fail_nr $fail_nr
1414		chk_rst_nr $rst_nr $rst_nr
1415		chk_infi_nr $infi_nr $infi_nr
1416	fi
1417}
1418
1419# a negative value for 'stale_max' means no upper bound:
1420# for bidirectional transfer, if one peer sleep for a while
1421# - as these tests do - we can have a quite high number of
1422# stale/recover conversions, proportional to
1423# sleep duration/ MPTCP-level RTX interval.
1424chk_stale_nr()
1425{
1426	local ns=$1
1427	local stale_min=$2
1428	local stale_max=$3
1429	local stale_delta=$4
1430	local dump_stats
1431	local stale_nr
1432	local recover_nr
1433
1434	printf "%-${nr_blank}s %-18s" " " "stale"
1435	stale_nr=$(ip netns exec $ns nstat -as | grep MPTcpExtSubflowStale | awk '{print $2}')
1436	[ -z "$stale_nr" ] && stale_nr=0
1437	recover_nr=$(ip netns exec $ns nstat -as | grep MPTcpExtSubflowRecover | awk '{print $2}')
1438	[ -z "$recover_nr" ] && recover_nr=0
1439
1440	if [ $stale_nr -lt $stale_min ] ||
1441	   { [ $stale_max -gt 0 ] && [ $stale_nr -gt $stale_max ]; } ||
1442	   [ $((stale_nr - recover_nr)) -ne $stale_delta ]; then
1443		echo "[fail] got $stale_nr stale[s] $recover_nr recover[s], " \
1444		     " expected stale in range [$stale_min..$stale_max]," \
1445		     " stale-recover delta $stale_delta "
1446		fail_test
1447		dump_stats=1
1448	else
1449		echo "[ ok ]"
1450	fi
1451
1452	if [ "${dump_stats}" = 1 ]; then
1453		echo $ns stats
1454		ip netns exec $ns ip -s link show
1455		ip netns exec $ns nstat -as | grep MPTcp
1456	fi
1457}
1458
1459chk_add_nr()
1460{
1461	local add_nr=$1
1462	local echo_nr=$2
1463	local port_nr=${3:-0}
1464	local syn_nr=${4:-$port_nr}
1465	local syn_ack_nr=${5:-$port_nr}
1466	local ack_nr=${6:-$port_nr}
1467	local mis_syn_nr=${7:-0}
1468	local mis_ack_nr=${8:-0}
1469	local count
1470	local dump_stats
1471	local timeout
1472
1473	timeout=$(ip netns exec $ns1 sysctl -n net.mptcp.add_addr_timeout)
1474
1475	printf "%-${nr_blank}s %s" " " "add"
1476	count=$(ip netns exec $ns2 nstat -as MPTcpExtAddAddr | grep MPTcpExtAddAddr | awk '{print $2}')
1477	[ -z "$count" ] && count=0
1478
1479	# if the test configured a short timeout tolerate greater then expected
1480	# add addrs options, due to retransmissions
1481	if [ "$count" != "$add_nr" ] && { [ "$timeout" -gt 1 ] || [ "$count" -lt "$add_nr" ]; }; then
1482		echo "[fail] got $count ADD_ADDR[s] expected $add_nr"
1483		fail_test
1484		dump_stats=1
1485	else
1486		echo -n "[ ok ]"
1487	fi
1488
1489	echo -n " - echo  "
1490	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtEchoAdd | awk '{print $2}')
1491	[ -z "$count" ] && count=0
1492	if [ "$count" != "$echo_nr" ]; then
1493		echo "[fail] got $count ADD_ADDR echo[s] expected $echo_nr"
1494		fail_test
1495		dump_stats=1
1496	else
1497		echo -n "[ ok ]"
1498	fi
1499
1500	if [ $port_nr -gt 0 ]; then
1501		echo -n " - pt "
1502		count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtPortAdd | awk '{print $2}')
1503		[ -z "$count" ] && count=0
1504		if [ "$count" != "$port_nr" ]; then
1505			echo "[fail] got $count ADD_ADDR[s] with a port-number expected $port_nr"
1506			fail_test
1507			dump_stats=1
1508		else
1509			echo "[ ok ]"
1510		fi
1511
1512		printf "%-${nr_blank}s %s" " " "syn"
1513		count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinPortSynRx |
1514			awk '{print $2}')
1515		[ -z "$count" ] && count=0
1516		if [ "$count" != "$syn_nr" ]; then
1517			echo "[fail] got $count JOIN[s] syn with a different \
1518				port-number expected $syn_nr"
1519			fail_test
1520			dump_stats=1
1521		else
1522			echo -n "[ ok ]"
1523		fi
1524
1525		echo -n " - synack"
1526		count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtMPJoinPortSynAckRx |
1527			awk '{print $2}')
1528		[ -z "$count" ] && count=0
1529		if [ "$count" != "$syn_ack_nr" ]; then
1530			echo "[fail] got $count JOIN[s] synack with a different \
1531				port-number expected $syn_ack_nr"
1532			fail_test
1533			dump_stats=1
1534		else
1535			echo -n "[ ok ]"
1536		fi
1537
1538		echo -n " - ack"
1539		count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinPortAckRx |
1540			awk '{print $2}')
1541		[ -z "$count" ] && count=0
1542		if [ "$count" != "$ack_nr" ]; then
1543			echo "[fail] got $count JOIN[s] ack with a different \
1544				port-number expected $ack_nr"
1545			fail_test
1546			dump_stats=1
1547		else
1548			echo "[ ok ]"
1549		fi
1550
1551		printf "%-${nr_blank}s %s" " " "syn"
1552		count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMismatchPortSynRx |
1553			awk '{print $2}')
1554		[ -z "$count" ] && count=0
1555		if [ "$count" != "$mis_syn_nr" ]; then
1556			echo "[fail] got $count JOIN[s] syn with a mismatched \
1557				port-number expected $mis_syn_nr"
1558			fail_test
1559			dump_stats=1
1560		else
1561			echo -n "[ ok ]"
1562		fi
1563
1564		echo -n " - ack   "
1565		count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMismatchPortAckRx |
1566			awk '{print $2}')
1567		[ -z "$count" ] && count=0
1568		if [ "$count" != "$mis_ack_nr" ]; then
1569			echo "[fail] got $count JOIN[s] ack with a mismatched \
1570				port-number expected $mis_ack_nr"
1571			fail_test
1572			dump_stats=1
1573		else
1574			echo "[ ok ]"
1575		fi
1576	else
1577		echo ""
1578	fi
1579
1580	[ "${dump_stats}" = 1 ] && dump_stats
1581}
1582
1583chk_rm_nr()
1584{
1585	local rm_addr_nr=$1
1586	local rm_subflow_nr=$2
1587	local invert
1588	local simult
1589	local count
1590	local dump_stats
1591	local addr_ns=$ns1
1592	local subflow_ns=$ns2
1593	local extra_msg=""
1594
1595	shift 2
1596	while [ -n "$1" ]; do
1597		[ "$1" = "invert" ] && invert=true
1598		[ "$1" = "simult" ] && simult=true
1599		shift
1600	done
1601
1602	if [ -z $invert ]; then
1603		addr_ns=$ns1
1604		subflow_ns=$ns2
1605	elif [ $invert = "true" ]; then
1606		addr_ns=$ns2
1607		subflow_ns=$ns1
1608		extra_msg="   invert"
1609	fi
1610
1611	printf "%-${nr_blank}s %s" " " "rm "
1612	count=$(ip netns exec $addr_ns nstat -as | grep MPTcpExtRmAddr | awk '{print $2}')
1613	[ -z "$count" ] && count=0
1614	if [ "$count" != "$rm_addr_nr" ]; then
1615		echo "[fail] got $count RM_ADDR[s] expected $rm_addr_nr"
1616		fail_test
1617		dump_stats=1
1618	else
1619		echo -n "[ ok ]"
1620	fi
1621
1622	echo -n " - rmsf  "
1623	count=$(ip netns exec $subflow_ns nstat -as | grep MPTcpExtRmSubflow | awk '{print $2}')
1624	[ -z "$count" ] && count=0
1625	if [ -n "$simult" ]; then
1626		local cnt suffix
1627
1628		cnt=$(ip netns exec $addr_ns nstat -as | grep MPTcpExtRmSubflow | awk '{print $2}')
1629
1630		# in case of simult flush, the subflow removal count on each side is
1631		# unreliable
1632		[ -z "$cnt" ] && cnt=0
1633		count=$((count + cnt))
1634		[ "$count" != "$rm_subflow_nr" ] && suffix="$count in [$rm_subflow_nr:$((rm_subflow_nr*2))]"
1635		if [ $count -ge "$rm_subflow_nr" ] && \
1636		   [ "$count" -le "$((rm_subflow_nr *2 ))" ]; then
1637			echo "[ ok ] $suffix"
1638		else
1639			echo "[fail] got $count RM_SUBFLOW[s] expected in range [$rm_subflow_nr:$((rm_subflow_nr*2))]"
1640			fail_test
1641			dump_stats=1
1642		fi
1643		return
1644	fi
1645	if [ "$count" != "$rm_subflow_nr" ]; then
1646		echo "[fail] got $count RM_SUBFLOW[s] expected $rm_subflow_nr"
1647		fail_test
1648		dump_stats=1
1649	else
1650		echo -n "[ ok ]"
1651	fi
1652
1653	[ "${dump_stats}" = 1 ] && dump_stats
1654
1655	echo "$extra_msg"
1656}
1657
1658chk_prio_nr()
1659{
1660	local mp_prio_nr_tx=$1
1661	local mp_prio_nr_rx=$2
1662	local count
1663	local dump_stats
1664
1665	printf "%-${nr_blank}s %s" " " "ptx"
1666	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPPrioTx | awk '{print $2}')
1667	[ -z "$count" ] && count=0
1668	if [ "$count" != "$mp_prio_nr_tx" ]; then
1669		echo "[fail] got $count MP_PRIO[s] TX expected $mp_prio_nr_tx"
1670		fail_test
1671		dump_stats=1
1672	else
1673		echo -n "[ ok ]"
1674	fi
1675
1676	echo -n " - prx   "
1677	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPPrioRx | awk '{print $2}')
1678	[ -z "$count" ] && count=0
1679	if [ "$count" != "$mp_prio_nr_rx" ]; then
1680		echo "[fail] got $count MP_PRIO[s] RX expected $mp_prio_nr_rx"
1681		fail_test
1682		dump_stats=1
1683	else
1684		echo "[ ok ]"
1685	fi
1686
1687	[ "${dump_stats}" = 1 ] && dump_stats
1688}
1689
1690chk_subflow_nr()
1691{
1692	local need_title="$1"
1693	local msg="$2"
1694	local subflow_nr=$3
1695	local cnt1
1696	local cnt2
1697
1698	if [ -n "${need_title}" ]; then
1699		printf "%03u %-36s %s" "${TEST_COUNT}" "${TEST_NAME}" "${msg}"
1700	else
1701		printf "%-${nr_blank}s %s" " " "${msg}"
1702	fi
1703
1704	cnt1=$(ss -N $ns1 -tOni | grep -c token)
1705	cnt2=$(ss -N $ns2 -tOni | grep -c token)
1706	if [ "$cnt1" != "$subflow_nr" -o "$cnt2" != "$subflow_nr" ]; then
1707		echo "[fail] got $cnt1:$cnt2 subflows expected $subflow_nr"
1708		fail_test
1709		dump_stats=1
1710	else
1711		echo "[ ok ]"
1712	fi
1713
1714	[ "${dump_stats}" = 1 ] && ( ss -N $ns1 -tOni ; ss -N $ns1 -tOni | grep token; ip -n $ns1 mptcp endpoint )
1715}
1716
1717chk_link_usage()
1718{
1719	local ns=$1
1720	local link=$2
1721	local out=$3
1722	local expected_rate=$4
1723
1724	local tx_link tx_total
1725	tx_link=$(ip netns exec $ns cat /sys/class/net/$link/statistics/tx_bytes)
1726	tx_total=$(stat --format=%s $out)
1727	local tx_rate=$((tx_link * 100 / tx_total))
1728	local tolerance=5
1729
1730	printf "%-${nr_blank}s %-18s" " " "link usage"
1731	if [ $tx_rate -lt $((expected_rate - tolerance)) ] || \
1732	   [ $tx_rate -gt $((expected_rate + tolerance)) ]; then
1733		echo "[fail] got $tx_rate% usage, expected $expected_rate%"
1734		fail_test
1735	else
1736		echo "[ ok ]"
1737	fi
1738}
1739
1740wait_attempt_fail()
1741{
1742	local timeout_ms=$((timeout_poll * 1000))
1743	local time=0
1744	local ns=$1
1745
1746	while [ $time -lt $timeout_ms ]; do
1747		local cnt
1748
1749		cnt=$(ip netns exec $ns nstat -as TcpAttemptFails | grep TcpAttemptFails | awk '{print $2}')
1750
1751		[ "$cnt" = 1 ] && return 1
1752		time=$((time + 100))
1753		sleep 0.1
1754	done
1755	return 1
1756}
1757
1758set_userspace_pm()
1759{
1760	local ns=$1
1761
1762	ip netns exec $ns sysctl -q net.mptcp.pm_type=1
1763}
1764
1765subflows_tests()
1766{
1767	if reset "no JOIN"; then
1768		run_tests $ns1 $ns2 10.0.1.1
1769		chk_join_nr 0 0 0
1770	fi
1771
1772	# subflow limited by client
1773	if reset "single subflow, limited by client"; then
1774		pm_nl_set_limits $ns1 0 0
1775		pm_nl_set_limits $ns2 0 0
1776		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1777		run_tests $ns1 $ns2 10.0.1.1
1778		chk_join_nr 0 0 0
1779	fi
1780
1781	# subflow limited by server
1782	if reset "single subflow, limited by server"; then
1783		pm_nl_set_limits $ns1 0 0
1784		pm_nl_set_limits $ns2 0 1
1785		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1786		run_tests $ns1 $ns2 10.0.1.1
1787		chk_join_nr 1 1 0
1788	fi
1789
1790	# subflow
1791	if reset "single subflow"; then
1792		pm_nl_set_limits $ns1 0 1
1793		pm_nl_set_limits $ns2 0 1
1794		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1795		run_tests $ns1 $ns2 10.0.1.1
1796		chk_join_nr 1 1 1
1797	fi
1798
1799	# multiple subflows
1800	if reset "multiple subflows"; then
1801		pm_nl_set_limits $ns1 0 2
1802		pm_nl_set_limits $ns2 0 2
1803		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1804		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1805		run_tests $ns1 $ns2 10.0.1.1
1806		chk_join_nr 2 2 2
1807	fi
1808
1809	# multiple subflows limited by server
1810	if reset "multiple subflows, limited by server"; then
1811		pm_nl_set_limits $ns1 0 1
1812		pm_nl_set_limits $ns2 0 2
1813		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1814		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1815		run_tests $ns1 $ns2 10.0.1.1
1816		chk_join_nr 2 2 1
1817	fi
1818
1819	# single subflow, dev
1820	if reset "single subflow, dev"; then
1821		pm_nl_set_limits $ns1 0 1
1822		pm_nl_set_limits $ns2 0 1
1823		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow dev ns2eth3
1824		run_tests $ns1 $ns2 10.0.1.1
1825		chk_join_nr 1 1 1
1826	fi
1827}
1828
1829subflows_error_tests()
1830{
1831	# If a single subflow is configured, and matches the MPC src
1832	# address, no additional subflow should be created
1833	if reset "no MPC reuse with single endpoint"; then
1834		pm_nl_set_limits $ns1 0 1
1835		pm_nl_set_limits $ns2 0 1
1836		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow
1837		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
1838		chk_join_nr 0 0 0
1839	fi
1840
1841	# multiple subflows, with subflow creation error
1842	if reset "multi subflows, with failing subflow"; then
1843		pm_nl_set_limits $ns1 0 2
1844		pm_nl_set_limits $ns2 0 2
1845		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1846		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1847		filter_tcp_from $ns1 10.0.3.2 REJECT
1848		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
1849		chk_join_nr 1 1 1
1850	fi
1851
1852	# multiple subflows, with subflow timeout on MPJ
1853	if reset "multi subflows, with subflow timeout"; then
1854		pm_nl_set_limits $ns1 0 2
1855		pm_nl_set_limits $ns2 0 2
1856		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1857		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1858		filter_tcp_from $ns1 10.0.3.2 DROP
1859		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
1860		chk_join_nr 1 1 1
1861	fi
1862
1863	# multiple subflows, check that the endpoint corresponding to
1864	# closed subflow (due to reset) is not reused if additional
1865	# subflows are added later
1866	if reset "multi subflows, fair usage on close"; then
1867		pm_nl_set_limits $ns1 0 1
1868		pm_nl_set_limits $ns2 0 1
1869		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1870		filter_tcp_from $ns1 10.0.3.2 REJECT
1871		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow &
1872
1873		# mpj subflow will be in TW after the reset
1874		wait_attempt_fail $ns2
1875		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1876		wait
1877
1878		# additional subflow could be created only if the PM select
1879		# the later endpoint, skipping the already used one
1880		chk_join_nr 1 1 1
1881	fi
1882}
1883
1884signal_address_tests()
1885{
1886	# add_address, unused
1887	if reset "unused signal address"; then
1888		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1889		run_tests $ns1 $ns2 10.0.1.1
1890		chk_join_nr 0 0 0
1891		chk_add_nr 1 1
1892	fi
1893
1894	# accept and use add_addr
1895	if reset "signal address"; then
1896		pm_nl_set_limits $ns1 0 1
1897		pm_nl_set_limits $ns2 1 1
1898		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1899		run_tests $ns1 $ns2 10.0.1.1
1900		chk_join_nr 1 1 1
1901		chk_add_nr 1 1
1902	fi
1903
1904	# accept and use add_addr with an additional subflow
1905	# note: signal address in server ns and local addresses in client ns must
1906	# belong to different subnets or one of the listed local address could be
1907	# used for 'add_addr' subflow
1908	if reset "subflow and signal"; then
1909		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1910		pm_nl_set_limits $ns1 0 2
1911		pm_nl_set_limits $ns2 1 2
1912		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1913		run_tests $ns1 $ns2 10.0.1.1
1914		chk_join_nr 2 2 2
1915		chk_add_nr 1 1
1916	fi
1917
1918	# accept and use add_addr with additional subflows
1919	if reset "multiple subflows and signal"; then
1920		pm_nl_set_limits $ns1 0 3
1921		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1922		pm_nl_set_limits $ns2 1 3
1923		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1924		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
1925		run_tests $ns1 $ns2 10.0.1.1
1926		chk_join_nr 3 3 3
1927		chk_add_nr 1 1
1928	fi
1929
1930	# signal addresses
1931	if reset "signal addresses"; then
1932		pm_nl_set_limits $ns1 3 3
1933		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1934		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
1935		pm_nl_add_endpoint $ns1 10.0.4.1 flags signal
1936		pm_nl_set_limits $ns2 3 3
1937		run_tests $ns1 $ns2 10.0.1.1
1938		chk_join_nr 3 3 3
1939		chk_add_nr 3 3
1940	fi
1941
1942	# signal invalid addresses
1943	if reset "signal invalid addresses"; then
1944		pm_nl_set_limits $ns1 3 3
1945		pm_nl_add_endpoint $ns1 10.0.12.1 flags signal
1946		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
1947		pm_nl_add_endpoint $ns1 10.0.14.1 flags signal
1948		pm_nl_set_limits $ns2 3 3
1949		run_tests $ns1 $ns2 10.0.1.1
1950		chk_join_nr 1 1 1
1951		chk_add_nr 3 3
1952	fi
1953
1954	# signal addresses race test
1955	if reset "signal addresses race test"; then
1956		pm_nl_set_limits $ns1 4 4
1957		pm_nl_set_limits $ns2 4 4
1958		pm_nl_add_endpoint $ns1 10.0.1.1 flags signal
1959		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1960		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
1961		pm_nl_add_endpoint $ns1 10.0.4.1 flags signal
1962		pm_nl_add_endpoint $ns2 10.0.1.2 flags signal
1963		pm_nl_add_endpoint $ns2 10.0.2.2 flags signal
1964		pm_nl_add_endpoint $ns2 10.0.3.2 flags signal
1965		pm_nl_add_endpoint $ns2 10.0.4.2 flags signal
1966
1967		# the peer could possibly miss some addr notification, allow retransmission
1968		ip netns exec $ns1 sysctl -q net.mptcp.add_addr_timeout=1
1969		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
1970		chk_join_nr 3 3 3
1971
1972		# the server will not signal the address terminating
1973		# the MPC subflow
1974		chk_add_nr 3 3
1975	fi
1976}
1977
1978link_failure_tests()
1979{
1980	# accept and use add_addr with additional subflows and link loss
1981	if reset "multiple flows, signal, link failure"; then
1982		# without any b/w limit each veth could spool the packets and get
1983		# them acked at xmit time, so that the corresponding subflow will
1984		# have almost always no outstanding pkts, the scheduler will pick
1985		# always the first subflow and we will have hard time testing
1986		# active backup and link switch-over.
1987		# Let's set some arbitrary (low) virtual link limits.
1988		init_shapers
1989		pm_nl_set_limits $ns1 0 3
1990		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
1991		pm_nl_set_limits $ns2 1 3
1992		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow
1993		pm_nl_add_endpoint $ns2 10.0.4.2 dev ns2eth4 flags subflow
1994		run_tests $ns1 $ns2 10.0.1.1 1
1995		chk_join_nr 3 3 3
1996		chk_add_nr 1 1
1997		chk_stale_nr $ns2 1 5 1
1998	fi
1999
2000	# accept and use add_addr with additional subflows and link loss
2001	# for bidirectional transfer
2002	if reset "multi flows, signal, bidi, link fail"; then
2003		init_shapers
2004		pm_nl_set_limits $ns1 0 3
2005		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
2006		pm_nl_set_limits $ns2 1 3
2007		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow
2008		pm_nl_add_endpoint $ns2 10.0.4.2 dev ns2eth4 flags subflow
2009		run_tests $ns1 $ns2 10.0.1.1 2
2010		chk_join_nr 3 3 3
2011		chk_add_nr 1 1
2012		chk_stale_nr $ns2 1 -1 1
2013	fi
2014
2015	# 2 subflows plus 1 backup subflow with a lossy link, backup
2016	# will never be used
2017	if reset "backup subflow unused, link failure"; then
2018		init_shapers
2019		pm_nl_set_limits $ns1 0 2
2020		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
2021		pm_nl_set_limits $ns2 1 2
2022		FAILING_LINKS="1"
2023		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow,backup
2024		run_tests $ns1 $ns2 10.0.1.1 1
2025		chk_join_nr 2 2 2
2026		chk_add_nr 1 1
2027		chk_link_usage $ns2 ns2eth3 $cinsent 0
2028	fi
2029
2030	# 2 lossy links after half transfer, backup will get half of
2031	# the traffic
2032	if reset "backup flow used, multi links fail"; then
2033		init_shapers
2034		pm_nl_set_limits $ns1 0 2
2035		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
2036		pm_nl_set_limits $ns2 1 2
2037		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow,backup
2038		FAILING_LINKS="1 2"
2039		run_tests $ns1 $ns2 10.0.1.1 1
2040		chk_join_nr 2 2 2
2041		chk_add_nr 1 1
2042		chk_stale_nr $ns2 2 4 2
2043		chk_link_usage $ns2 ns2eth3 $cinsent 50
2044	fi
2045
2046	# use a backup subflow with the first subflow on a lossy link
2047	# for bidirectional transfer
2048	if reset "backup flow used, bidi, link failure"; then
2049		init_shapers
2050		pm_nl_set_limits $ns1 0 2
2051		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
2052		pm_nl_set_limits $ns2 1 3
2053		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow,backup
2054		FAILING_LINKS="1 2"
2055		run_tests $ns1 $ns2 10.0.1.1 2
2056		chk_join_nr 2 2 2
2057		chk_add_nr 1 1
2058		chk_stale_nr $ns2 1 -1 2
2059		chk_link_usage $ns2 ns2eth3 $cinsent 50
2060	fi
2061}
2062
2063add_addr_timeout_tests()
2064{
2065	# add_addr timeout
2066	if reset_with_add_addr_timeout "signal address, ADD_ADDR timeout"; then
2067		pm_nl_set_limits $ns1 0 1
2068		pm_nl_set_limits $ns2 1 1
2069		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2070		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
2071		chk_join_nr 1 1 1
2072		chk_add_nr 4 0
2073	fi
2074
2075	# add_addr timeout IPv6
2076	if reset_with_add_addr_timeout "signal address, ADD_ADDR6 timeout" 6; then
2077		pm_nl_set_limits $ns1 0 1
2078		pm_nl_set_limits $ns2 1 1
2079		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2080		run_tests $ns1 $ns2 dead:beef:1::1 0 0 0 slow
2081		chk_join_nr 1 1 1
2082		chk_add_nr 4 0
2083	fi
2084
2085	# signal addresses timeout
2086	if reset_with_add_addr_timeout "signal addresses, ADD_ADDR timeout"; then
2087		pm_nl_set_limits $ns1 2 2
2088		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2089		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2090		pm_nl_set_limits $ns2 2 2
2091		run_tests $ns1 $ns2 10.0.1.1 0 0 0 speed_10
2092		chk_join_nr 2 2 2
2093		chk_add_nr 8 0
2094	fi
2095
2096	# signal invalid addresses timeout
2097	if reset_with_add_addr_timeout "invalid address, ADD_ADDR timeout"; then
2098		pm_nl_set_limits $ns1 2 2
2099		pm_nl_add_endpoint $ns1 10.0.12.1 flags signal
2100		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2101		pm_nl_set_limits $ns2 2 2
2102		run_tests $ns1 $ns2 10.0.1.1 0 0 0 speed_10
2103		chk_join_nr 1 1 1
2104		chk_add_nr 8 0
2105	fi
2106}
2107
2108remove_tests()
2109{
2110	# single subflow, remove
2111	if reset "remove single subflow"; then
2112		pm_nl_set_limits $ns1 0 1
2113		pm_nl_set_limits $ns2 0 1
2114		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2115		run_tests $ns1 $ns2 10.0.1.1 0 0 -1 slow
2116		chk_join_nr 1 1 1
2117		chk_rm_nr 1 1
2118	fi
2119
2120	# multiple subflows, remove
2121	if reset "remove multiple subflows"; then
2122		pm_nl_set_limits $ns1 0 2
2123		pm_nl_set_limits $ns2 0 2
2124		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
2125		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2126		run_tests $ns1 $ns2 10.0.1.1 0 0 -2 slow
2127		chk_join_nr 2 2 2
2128		chk_rm_nr 2 2
2129	fi
2130
2131	# single address, remove
2132	if reset "remove single address"; then
2133		pm_nl_set_limits $ns1 0 1
2134		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2135		pm_nl_set_limits $ns2 1 1
2136		run_tests $ns1 $ns2 10.0.1.1 0 -1 0 slow
2137		chk_join_nr 1 1 1
2138		chk_add_nr 1 1
2139		chk_rm_nr 1 1 invert
2140	fi
2141
2142	# subflow and signal, remove
2143	if reset "remove subflow and signal"; then
2144		pm_nl_set_limits $ns1 0 2
2145		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2146		pm_nl_set_limits $ns2 1 2
2147		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2148		run_tests $ns1 $ns2 10.0.1.1 0 -1 -1 slow
2149		chk_join_nr 2 2 2
2150		chk_add_nr 1 1
2151		chk_rm_nr 1 1
2152	fi
2153
2154	# subflows and signal, remove
2155	if reset "remove subflows and signal"; then
2156		pm_nl_set_limits $ns1 0 3
2157		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2158		pm_nl_set_limits $ns2 1 3
2159		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2160		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2161		run_tests $ns1 $ns2 10.0.1.1 0 -1 -2 speed_10
2162		chk_join_nr 3 3 3
2163		chk_add_nr 1 1
2164		chk_rm_nr 2 2
2165	fi
2166
2167	# addresses remove
2168	if reset "remove addresses"; then
2169		pm_nl_set_limits $ns1 3 3
2170		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal id 250
2171		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2172		pm_nl_add_endpoint $ns1 10.0.4.1 flags signal
2173		pm_nl_set_limits $ns2 3 3
2174		run_tests $ns1 $ns2 10.0.1.1 0 -3 0 speed_10
2175		chk_join_nr 3 3 3
2176		chk_add_nr 3 3
2177		chk_rm_nr 3 3 invert
2178	fi
2179
2180	# invalid addresses remove
2181	if reset "remove invalid addresses"; then
2182		pm_nl_set_limits $ns1 3 3
2183		pm_nl_add_endpoint $ns1 10.0.12.1 flags signal
2184		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2185		pm_nl_add_endpoint $ns1 10.0.14.1 flags signal
2186		pm_nl_set_limits $ns2 3 3
2187		run_tests $ns1 $ns2 10.0.1.1 0 -3 0 speed_10
2188		chk_join_nr 1 1 1
2189		chk_add_nr 3 3
2190		chk_rm_nr 3 1 invert
2191	fi
2192
2193	# subflows and signal, flush
2194	if reset "flush subflows and signal"; then
2195		pm_nl_set_limits $ns1 0 3
2196		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2197		pm_nl_set_limits $ns2 1 3
2198		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2199		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2200		run_tests $ns1 $ns2 10.0.1.1 0 -8 -8 slow
2201		chk_join_nr 3 3 3
2202		chk_add_nr 1 1
2203		chk_rm_nr 1 3 invert simult
2204	fi
2205
2206	# subflows flush
2207	if reset "flush subflows"; then
2208		pm_nl_set_limits $ns1 3 3
2209		pm_nl_set_limits $ns2 3 3
2210		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow id 150
2211		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2212		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2213		run_tests $ns1 $ns2 10.0.1.1 0 -8 -8 slow
2214		chk_join_nr 3 3 3
2215		chk_rm_nr 0 3 simult
2216	fi
2217
2218	# addresses flush
2219	if reset "flush addresses"; then
2220		pm_nl_set_limits $ns1 3 3
2221		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal id 250
2222		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2223		pm_nl_add_endpoint $ns1 10.0.4.1 flags signal
2224		pm_nl_set_limits $ns2 3 3
2225		run_tests $ns1 $ns2 10.0.1.1 0 -8 -8 slow
2226		chk_join_nr 3 3 3
2227		chk_add_nr 3 3
2228		chk_rm_nr 3 3 invert simult
2229	fi
2230
2231	# invalid addresses flush
2232	if reset "flush invalid addresses"; then
2233		pm_nl_set_limits $ns1 3 3
2234		pm_nl_add_endpoint $ns1 10.0.12.1 flags signal
2235		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2236		pm_nl_add_endpoint $ns1 10.0.14.1 flags signal
2237		pm_nl_set_limits $ns2 3 3
2238		run_tests $ns1 $ns2 10.0.1.1 0 -8 0 slow
2239		chk_join_nr 1 1 1
2240		chk_add_nr 3 3
2241		chk_rm_nr 3 1 invert
2242	fi
2243
2244	# remove id 0 subflow
2245	if reset "remove id 0 subflow"; then
2246		pm_nl_set_limits $ns1 0 1
2247		pm_nl_set_limits $ns2 0 1
2248		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2249		run_tests $ns1 $ns2 10.0.1.1 0 0 -9 slow
2250		chk_join_nr 1 1 1
2251		chk_rm_nr 1 1
2252	fi
2253
2254	# remove id 0 address
2255	if reset "remove id 0 address"; then
2256		pm_nl_set_limits $ns1 0 1
2257		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2258		pm_nl_set_limits $ns2 1 1
2259		run_tests $ns1 $ns2 10.0.1.1 0 -9 0 slow
2260		chk_join_nr 1 1 1
2261		chk_add_nr 1 1
2262		chk_rm_nr 1 1 invert
2263	fi
2264}
2265
2266add_tests()
2267{
2268	# add single subflow
2269	if reset "add single subflow"; then
2270		pm_nl_set_limits $ns1 0 1
2271		pm_nl_set_limits $ns2 0 1
2272		run_tests $ns1 $ns2 10.0.1.1 0 0 1 slow
2273		chk_join_nr 1 1 1
2274	fi
2275
2276	# add signal address
2277	if reset "add signal address"; then
2278		pm_nl_set_limits $ns1 0 1
2279		pm_nl_set_limits $ns2 1 1
2280		run_tests $ns1 $ns2 10.0.1.1 0 1 0 slow
2281		chk_join_nr 1 1 1
2282		chk_add_nr 1 1
2283	fi
2284
2285	# add multiple subflows
2286	if reset "add multiple subflows"; then
2287		pm_nl_set_limits $ns1 0 2
2288		pm_nl_set_limits $ns2 0 2
2289		run_tests $ns1 $ns2 10.0.1.1 0 0 2 slow
2290		chk_join_nr 2 2 2
2291	fi
2292
2293	# add multiple subflows IPv6
2294	if reset "add multiple subflows IPv6"; then
2295		pm_nl_set_limits $ns1 0 2
2296		pm_nl_set_limits $ns2 0 2
2297		run_tests $ns1 $ns2 dead:beef:1::1 0 0 2 slow
2298		chk_join_nr 2 2 2
2299	fi
2300
2301	# add multiple addresses IPv6
2302	if reset "add multiple addresses IPv6"; then
2303		pm_nl_set_limits $ns1 0 2
2304		pm_nl_set_limits $ns2 2 2
2305		run_tests $ns1 $ns2 dead:beef:1::1 0 2 0 slow
2306		chk_join_nr 2 2 2
2307		chk_add_nr 2 2
2308	fi
2309}
2310
2311ipv6_tests()
2312{
2313	# subflow IPv6
2314	if reset "single subflow IPv6"; then
2315		pm_nl_set_limits $ns1 0 1
2316		pm_nl_set_limits $ns2 0 1
2317		pm_nl_add_endpoint $ns2 dead:beef:3::2 dev ns2eth3 flags subflow
2318		run_tests $ns1 $ns2 dead:beef:1::1 0 0 0 slow
2319		chk_join_nr 1 1 1
2320	fi
2321
2322	# add_address, unused IPv6
2323	if reset "unused signal address IPv6"; then
2324		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2325		run_tests $ns1 $ns2 dead:beef:1::1 0 0 0 slow
2326		chk_join_nr 0 0 0
2327		chk_add_nr 1 1
2328	fi
2329
2330	# signal address IPv6
2331	if reset "single address IPv6"; then
2332		pm_nl_set_limits $ns1 0 1
2333		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2334		pm_nl_set_limits $ns2 1 1
2335		run_tests $ns1 $ns2 dead:beef:1::1 0 0 0 slow
2336		chk_join_nr 1 1 1
2337		chk_add_nr 1 1
2338	fi
2339
2340	# single address IPv6, remove
2341	if reset "remove single address IPv6"; then
2342		pm_nl_set_limits $ns1 0 1
2343		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2344		pm_nl_set_limits $ns2 1 1
2345		run_tests $ns1 $ns2 dead:beef:1::1 0 -1 0 slow
2346		chk_join_nr 1 1 1
2347		chk_add_nr 1 1
2348		chk_rm_nr 1 1 invert
2349	fi
2350
2351	# subflow and signal IPv6, remove
2352	if reset "remove subflow and signal IPv6"; then
2353		pm_nl_set_limits $ns1 0 2
2354		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2355		pm_nl_set_limits $ns2 1 2
2356		pm_nl_add_endpoint $ns2 dead:beef:3::2 dev ns2eth3 flags subflow
2357		run_tests $ns1 $ns2 dead:beef:1::1 0 -1 -1 slow
2358		chk_join_nr 2 2 2
2359		chk_add_nr 1 1
2360		chk_rm_nr 1 1
2361	fi
2362}
2363
2364v4mapped_tests()
2365{
2366	# subflow IPv4-mapped to IPv4-mapped
2367	if reset "single subflow IPv4-mapped"; then
2368		pm_nl_set_limits $ns1 0 1
2369		pm_nl_set_limits $ns2 0 1
2370		pm_nl_add_endpoint $ns2 "::ffff:10.0.3.2" flags subflow
2371		run_tests $ns1 $ns2 "::ffff:10.0.1.1"
2372		chk_join_nr 1 1 1
2373	fi
2374
2375	# signal address IPv4-mapped with IPv4-mapped sk
2376	if reset "signal address IPv4-mapped"; then
2377		pm_nl_set_limits $ns1 0 1
2378		pm_nl_set_limits $ns2 1 1
2379		pm_nl_add_endpoint $ns1 "::ffff:10.0.2.1" flags signal
2380		run_tests $ns1 $ns2 "::ffff:10.0.1.1"
2381		chk_join_nr 1 1 1
2382		chk_add_nr 1 1
2383	fi
2384
2385	# subflow v4-map-v6
2386	if reset "single subflow v4-map-v6"; then
2387		pm_nl_set_limits $ns1 0 1
2388		pm_nl_set_limits $ns2 0 1
2389		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2390		run_tests $ns1 $ns2 "::ffff:10.0.1.1"
2391		chk_join_nr 1 1 1
2392	fi
2393
2394	# signal address v4-map-v6
2395	if reset "signal address v4-map-v6"; then
2396		pm_nl_set_limits $ns1 0 1
2397		pm_nl_set_limits $ns2 1 1
2398		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2399		run_tests $ns1 $ns2 "::ffff:10.0.1.1"
2400		chk_join_nr 1 1 1
2401		chk_add_nr 1 1
2402	fi
2403
2404	# subflow v6-map-v4
2405	if reset "single subflow v6-map-v4"; then
2406		pm_nl_set_limits $ns1 0 1
2407		pm_nl_set_limits $ns2 0 1
2408		pm_nl_add_endpoint $ns2 "::ffff:10.0.3.2" flags subflow
2409		run_tests $ns1 $ns2 10.0.1.1
2410		chk_join_nr 1 1 1
2411	fi
2412
2413	# signal address v6-map-v4
2414	if reset "signal address v6-map-v4"; then
2415		pm_nl_set_limits $ns1 0 1
2416		pm_nl_set_limits $ns2 1 1
2417		pm_nl_add_endpoint $ns1 "::ffff:10.0.2.1" flags signal
2418		run_tests $ns1 $ns2 10.0.1.1
2419		chk_join_nr 1 1 1
2420		chk_add_nr 1 1
2421	fi
2422
2423	# no subflow IPv6 to v4 address
2424	if reset "no JOIN with diff families v4-v6"; then
2425		pm_nl_set_limits $ns1 0 1
2426		pm_nl_set_limits $ns2 0 1
2427		pm_nl_add_endpoint $ns2 dead:beef:2::2 flags subflow
2428		run_tests $ns1 $ns2 10.0.1.1
2429		chk_join_nr 0 0 0
2430	fi
2431
2432	# no subflow IPv6 to v4 address even if v6 has a valid v4 at the end
2433	if reset "no JOIN with diff families v4-v6-2"; then
2434		pm_nl_set_limits $ns1 0 1
2435		pm_nl_set_limits $ns2 0 1
2436		pm_nl_add_endpoint $ns2 dead:beef:2::10.0.3.2 flags subflow
2437		run_tests $ns1 $ns2 10.0.1.1
2438		chk_join_nr 0 0 0
2439	fi
2440
2441	# no subflow IPv4 to v6 address, no need to slow down too then
2442	if reset "no JOIN with diff families v6-v4"; then
2443		pm_nl_set_limits $ns1 0 1
2444		pm_nl_set_limits $ns2 0 1
2445		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2446		run_tests $ns1 $ns2 dead:beef:1::1
2447		chk_join_nr 0 0 0
2448	fi
2449}
2450
2451backup_tests()
2452{
2453	# single subflow, backup
2454	if reset "single subflow, backup"; then
2455		pm_nl_set_limits $ns1 0 1
2456		pm_nl_set_limits $ns2 0 1
2457		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow,backup
2458		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow nobackup
2459		chk_join_nr 1 1 1
2460		chk_prio_nr 0 1
2461	fi
2462
2463	# single address, backup
2464	if reset "single address, backup"; then
2465		pm_nl_set_limits $ns1 0 1
2466		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2467		pm_nl_set_limits $ns2 1 1
2468		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
2469		chk_join_nr 1 1 1
2470		chk_add_nr 1 1
2471		chk_prio_nr 1 1
2472	fi
2473
2474	# single address with port, backup
2475	if reset "single address with port, backup"; then
2476		pm_nl_set_limits $ns1 0 1
2477		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2478		pm_nl_set_limits $ns2 1 1
2479		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
2480		chk_join_nr 1 1 1
2481		chk_add_nr 1 1
2482		chk_prio_nr 1 1
2483	fi
2484
2485	if reset "mpc backup"; then
2486		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow,backup
2487		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
2488		chk_join_nr 0 0 0
2489		chk_prio_nr 0 1
2490	fi
2491
2492	if reset "mpc backup both sides"; then
2493		pm_nl_add_endpoint $ns1 10.0.1.1 flags subflow,backup
2494		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow,backup
2495		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
2496		chk_join_nr 0 0 0
2497		chk_prio_nr 1 1
2498	fi
2499
2500	if reset "mpc switch to backup"; then
2501		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow
2502		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
2503		chk_join_nr 0 0 0
2504		chk_prio_nr 0 1
2505	fi
2506
2507	if reset "mpc switch to backup both sides"; then
2508		pm_nl_add_endpoint $ns1 10.0.1.1 flags subflow
2509		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow
2510		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
2511		chk_join_nr 0 0 0
2512		chk_prio_nr 1 1
2513	fi
2514}
2515
2516LISTENER_CREATED=15 #MPTCP_EVENT_LISTENER_CREATED
2517LISTENER_CLOSED=16  #MPTCP_EVENT_LISTENER_CLOSED
2518
2519AF_INET=2
2520AF_INET6=10
2521
2522verify_listener_events()
2523{
2524	local evt=$1
2525	local e_type=$2
2526	local e_family=$3
2527	local e_saddr=$4
2528	local e_sport=$5
2529	local type
2530	local family
2531	local saddr
2532	local sport
2533
2534	if [ $e_type = $LISTENER_CREATED ]; then
2535		stdbuf -o0 -e0 printf "\t\t\t\t\t CREATE_LISTENER %s:%s"\
2536			$e_saddr $e_sport
2537	elif [ $e_type = $LISTENER_CLOSED ]; then
2538		stdbuf -o0 -e0 printf "\t\t\t\t\t CLOSE_LISTENER %s:%s "\
2539			$e_saddr $e_sport
2540	fi
2541
2542	type=$(grep "type:$e_type," $evt |
2543	       sed --unbuffered -n 's/.*\(type:\)\([[:digit:]]*\).*$/\2/p;q')
2544	family=$(grep "type:$e_type," $evt |
2545		 sed --unbuffered -n 's/.*\(family:\)\([[:digit:]]*\).*$/\2/p;q')
2546	sport=$(grep "type:$e_type," $evt |
2547		sed --unbuffered -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q')
2548	if [ $family ] && [ $family = $AF_INET6 ]; then
2549		saddr=$(grep "type:$e_type," $evt |
2550			sed --unbuffered -n 's/.*\(saddr6:\)\([0-9a-f:.]*\).*$/\2/p;q')
2551	else
2552		saddr=$(grep "type:$e_type," $evt |
2553			sed --unbuffered -n 's/.*\(saddr4:\)\([0-9.]*\).*$/\2/p;q')
2554	fi
2555
2556	if [ $type ] && [ $type = $e_type ] &&
2557	   [ $family ] && [ $family = $e_family ] &&
2558	   [ $saddr ] && [ $saddr = $e_saddr ] &&
2559	   [ $sport ] && [ $sport = $e_sport ]; then
2560		stdbuf -o0 -e0 printf "[ ok ]\n"
2561		return 0
2562	fi
2563	fail_test
2564	stdbuf -o0 -e0 printf "[fail]\n"
2565}
2566
2567add_addr_ports_tests()
2568{
2569	# signal address with port
2570	if reset "signal address with port"; then
2571		pm_nl_set_limits $ns1 0 1
2572		pm_nl_set_limits $ns2 1 1
2573		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2574		run_tests $ns1 $ns2 10.0.1.1
2575		chk_join_nr 1 1 1
2576		chk_add_nr 1 1 1
2577	fi
2578
2579	# subflow and signal with port
2580	if reset "subflow and signal with port"; then
2581		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2582		pm_nl_set_limits $ns1 0 2
2583		pm_nl_set_limits $ns2 1 2
2584		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2585		run_tests $ns1 $ns2 10.0.1.1
2586		chk_join_nr 2 2 2
2587		chk_add_nr 1 1 1
2588	fi
2589
2590	# single address with port, remove
2591	# pm listener events
2592	if reset_with_events "remove single address with port"; then
2593		pm_nl_set_limits $ns1 0 1
2594		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2595		pm_nl_set_limits $ns2 1 1
2596		run_tests $ns1 $ns2 10.0.1.1 0 -1 0 slow
2597		chk_join_nr 1 1 1
2598		chk_add_nr 1 1 1
2599		chk_rm_nr 1 1 invert
2600
2601		verify_listener_events $evts_ns1 $LISTENER_CREATED $AF_INET 10.0.2.1 10100
2602		verify_listener_events $evts_ns1 $LISTENER_CLOSED $AF_INET 10.0.2.1 10100
2603		kill_events_pids
2604	fi
2605
2606	# subflow and signal with port, remove
2607	if reset "remove subflow and signal with port"; then
2608		pm_nl_set_limits $ns1 0 2
2609		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2610		pm_nl_set_limits $ns2 1 2
2611		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2612		run_tests $ns1 $ns2 10.0.1.1 0 -1 -1 slow
2613		chk_join_nr 2 2 2
2614		chk_add_nr 1 1 1
2615		chk_rm_nr 1 1
2616	fi
2617
2618	# subflows and signal with port, flush
2619	if reset "flush subflows and signal with port"; then
2620		pm_nl_set_limits $ns1 0 3
2621		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2622		pm_nl_set_limits $ns2 1 3
2623		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2624		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2625		run_tests $ns1 $ns2 10.0.1.1 0 -8 -2 slow
2626		chk_join_nr 3 3 3
2627		chk_add_nr 1 1
2628		chk_rm_nr 1 3 invert simult
2629	fi
2630
2631	# multiple addresses with port
2632	if reset "multiple addresses with port"; then
2633		pm_nl_set_limits $ns1 2 2
2634		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2635		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal port 10100
2636		pm_nl_set_limits $ns2 2 2
2637		run_tests $ns1 $ns2 10.0.1.1
2638		chk_join_nr 2 2 2
2639		chk_add_nr 2 2 2
2640	fi
2641
2642	# multiple addresses with ports
2643	if reset "multiple addresses with ports"; then
2644		pm_nl_set_limits $ns1 2 2
2645		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2646		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal port 10101
2647		pm_nl_set_limits $ns2 2 2
2648		run_tests $ns1 $ns2 10.0.1.1
2649		chk_join_nr 2 2 2
2650		chk_add_nr 2 2 2
2651	fi
2652}
2653
2654syncookies_tests()
2655{
2656	# single subflow, syncookies
2657	if reset_with_cookies "single subflow with syn cookies"; then
2658		pm_nl_set_limits $ns1 0 1
2659		pm_nl_set_limits $ns2 0 1
2660		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2661		run_tests $ns1 $ns2 10.0.1.1
2662		chk_join_nr 1 1 1
2663	fi
2664
2665	# multiple subflows with syn cookies
2666	if reset_with_cookies "multiple subflows with syn cookies"; then
2667		pm_nl_set_limits $ns1 0 2
2668		pm_nl_set_limits $ns2 0 2
2669		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2670		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
2671		run_tests $ns1 $ns2 10.0.1.1
2672		chk_join_nr 2 2 2
2673	fi
2674
2675	# multiple subflows limited by server
2676	if reset_with_cookies "subflows limited by server w cookies"; then
2677		pm_nl_set_limits $ns1 0 1
2678		pm_nl_set_limits $ns2 0 2
2679		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2680		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
2681		run_tests $ns1 $ns2 10.0.1.1
2682		chk_join_nr 2 1 1
2683	fi
2684
2685	# test signal address with cookies
2686	if reset_with_cookies "signal address with syn cookies"; then
2687		pm_nl_set_limits $ns1 0 1
2688		pm_nl_set_limits $ns2 1 1
2689		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2690		run_tests $ns1 $ns2 10.0.1.1
2691		chk_join_nr 1 1 1
2692		chk_add_nr 1 1
2693	fi
2694
2695	# test cookie with subflow and signal
2696	if reset_with_cookies "subflow and signal w cookies"; then
2697		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2698		pm_nl_set_limits $ns1 0 2
2699		pm_nl_set_limits $ns2 1 2
2700		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2701		run_tests $ns1 $ns2 10.0.1.1
2702		chk_join_nr 2 2 2
2703		chk_add_nr 1 1
2704	fi
2705
2706	# accept and use add_addr with additional subflows
2707	if reset_with_cookies "subflows and signal w. cookies"; then
2708		pm_nl_set_limits $ns1 0 3
2709		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2710		pm_nl_set_limits $ns2 1 3
2711		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2712		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2713		run_tests $ns1 $ns2 10.0.1.1
2714		chk_join_nr 3 3 3
2715		chk_add_nr 1 1
2716	fi
2717}
2718
2719checksum_tests()
2720{
2721	# checksum test 0 0
2722	if reset_with_checksum 0 0; then
2723		pm_nl_set_limits $ns1 0 1
2724		pm_nl_set_limits $ns2 0 1
2725		run_tests $ns1 $ns2 10.0.1.1
2726		chk_join_nr 0 0 0
2727	fi
2728
2729	# checksum test 1 1
2730	if reset_with_checksum 1 1; then
2731		pm_nl_set_limits $ns1 0 1
2732		pm_nl_set_limits $ns2 0 1
2733		run_tests $ns1 $ns2 10.0.1.1
2734		chk_join_nr 0 0 0
2735	fi
2736
2737	# checksum test 0 1
2738	if reset_with_checksum 0 1; then
2739		pm_nl_set_limits $ns1 0 1
2740		pm_nl_set_limits $ns2 0 1
2741		run_tests $ns1 $ns2 10.0.1.1
2742		chk_join_nr 0 0 0
2743	fi
2744
2745	# checksum test 1 0
2746	if reset_with_checksum 1 0; then
2747		pm_nl_set_limits $ns1 0 1
2748		pm_nl_set_limits $ns2 0 1
2749		run_tests $ns1 $ns2 10.0.1.1
2750		chk_join_nr 0 0 0
2751	fi
2752}
2753
2754deny_join_id0_tests()
2755{
2756	# subflow allow join id0 ns1
2757	if reset_with_allow_join_id0 "single subflow allow join id0 ns1" 1 0; then
2758		pm_nl_set_limits $ns1 1 1
2759		pm_nl_set_limits $ns2 1 1
2760		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2761		run_tests $ns1 $ns2 10.0.1.1
2762		chk_join_nr 1 1 1
2763	fi
2764
2765	# subflow allow join id0 ns2
2766	if reset_with_allow_join_id0 "single subflow allow join id0 ns2" 0 1; then
2767		pm_nl_set_limits $ns1 1 1
2768		pm_nl_set_limits $ns2 1 1
2769		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2770		run_tests $ns1 $ns2 10.0.1.1
2771		chk_join_nr 0 0 0
2772	fi
2773
2774	# signal address allow join id0 ns1
2775	# ADD_ADDRs are not affected by allow_join_id0 value.
2776	if reset_with_allow_join_id0 "signal address allow join id0 ns1" 1 0; then
2777		pm_nl_set_limits $ns1 1 1
2778		pm_nl_set_limits $ns2 1 1
2779		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2780		run_tests $ns1 $ns2 10.0.1.1
2781		chk_join_nr 1 1 1
2782		chk_add_nr 1 1
2783	fi
2784
2785	# signal address allow join id0 ns2
2786	# ADD_ADDRs are not affected by allow_join_id0 value.
2787	if reset_with_allow_join_id0 "signal address allow join id0 ns2" 0 1; then
2788		pm_nl_set_limits $ns1 1 1
2789		pm_nl_set_limits $ns2 1 1
2790		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2791		run_tests $ns1 $ns2 10.0.1.1
2792		chk_join_nr 1 1 1
2793		chk_add_nr 1 1
2794	fi
2795
2796	# subflow and address allow join id0 ns1
2797	if reset_with_allow_join_id0 "subflow and address allow join id0 1" 1 0; then
2798		pm_nl_set_limits $ns1 2 2
2799		pm_nl_set_limits $ns2 2 2
2800		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2801		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2802		run_tests $ns1 $ns2 10.0.1.1
2803		chk_join_nr 2 2 2
2804	fi
2805
2806	# subflow and address allow join id0 ns2
2807	if reset_with_allow_join_id0 "subflow and address allow join id0 2" 0 1; then
2808		pm_nl_set_limits $ns1 2 2
2809		pm_nl_set_limits $ns2 2 2
2810		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2811		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2812		run_tests $ns1 $ns2 10.0.1.1
2813		chk_join_nr 1 1 1
2814	fi
2815}
2816
2817fullmesh_tests()
2818{
2819	# fullmesh 1
2820	# 2 fullmesh addrs in ns2, added before the connection,
2821	# 1 non-fullmesh addr in ns1, added during the connection.
2822	if reset "fullmesh test 2x1"; then
2823		pm_nl_set_limits $ns1 0 4
2824		pm_nl_set_limits $ns2 1 4
2825		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow,fullmesh
2826		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow,fullmesh
2827		run_tests $ns1 $ns2 10.0.1.1 0 1 0 slow
2828		chk_join_nr 4 4 4
2829		chk_add_nr 1 1
2830	fi
2831
2832	# fullmesh 2
2833	# 1 non-fullmesh addr in ns1, added before the connection,
2834	# 1 fullmesh addr in ns2, added during the connection.
2835	if reset "fullmesh test 1x1"; then
2836		pm_nl_set_limits $ns1 1 3
2837		pm_nl_set_limits $ns2 1 3
2838		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2839		run_tests $ns1 $ns2 10.0.1.1 0 0 fullmesh_1 slow
2840		chk_join_nr 3 3 3
2841		chk_add_nr 1 1
2842	fi
2843
2844	# fullmesh 3
2845	# 1 non-fullmesh addr in ns1, added before the connection,
2846	# 2 fullmesh addrs in ns2, added during the connection.
2847	if reset "fullmesh test 1x2"; then
2848		pm_nl_set_limits $ns1 2 5
2849		pm_nl_set_limits $ns2 1 5
2850		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2851		run_tests $ns1 $ns2 10.0.1.1 0 0 fullmesh_2 slow
2852		chk_join_nr 5 5 5
2853		chk_add_nr 1 1
2854	fi
2855
2856	# fullmesh 4
2857	# 1 non-fullmesh addr in ns1, added before the connection,
2858	# 2 fullmesh addrs in ns2, added during the connection,
2859	# limit max_subflows to 4.
2860	if reset "fullmesh test 1x2, limited"; then
2861		pm_nl_set_limits $ns1 2 4
2862		pm_nl_set_limits $ns2 1 4
2863		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2864		run_tests $ns1 $ns2 10.0.1.1 0 0 fullmesh_2 slow
2865		chk_join_nr 4 4 4
2866		chk_add_nr 1 1
2867	fi
2868
2869	# set fullmesh flag
2870	if reset "set fullmesh flag test"; then
2871		pm_nl_set_limits $ns1 4 4
2872		pm_nl_add_endpoint $ns1 10.0.2.1 flags subflow
2873		pm_nl_set_limits $ns2 4 4
2874		run_tests $ns1 $ns2 10.0.1.1 0 0 1 slow fullmesh
2875		chk_join_nr 2 2 2
2876		chk_rm_nr 0 1
2877	fi
2878
2879	# set nofullmesh flag
2880	if reset "set nofullmesh flag test"; then
2881		pm_nl_set_limits $ns1 4 4
2882		pm_nl_add_endpoint $ns1 10.0.2.1 flags subflow,fullmesh
2883		pm_nl_set_limits $ns2 4 4
2884		run_tests $ns1 $ns2 10.0.1.1 0 0 fullmesh_1 slow nofullmesh
2885		chk_join_nr 2 2 2
2886		chk_rm_nr 0 1
2887	fi
2888
2889	# set backup,fullmesh flags
2890	if reset "set backup,fullmesh flags test"; then
2891		pm_nl_set_limits $ns1 4 4
2892		pm_nl_add_endpoint $ns1 10.0.2.1 flags subflow
2893		pm_nl_set_limits $ns2 4 4
2894		run_tests $ns1 $ns2 10.0.1.1 0 0 1 slow backup,fullmesh
2895		chk_join_nr 2 2 2
2896		chk_prio_nr 0 1
2897		chk_rm_nr 0 1
2898	fi
2899
2900	# set nobackup,nofullmesh flags
2901	if reset "set nobackup,nofullmesh flags test"; then
2902		pm_nl_set_limits $ns1 4 4
2903		pm_nl_set_limits $ns2 4 4
2904		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow,backup,fullmesh
2905		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow nobackup,nofullmesh
2906		chk_join_nr 2 2 2
2907		chk_prio_nr 0 1
2908		chk_rm_nr 0 1
2909	fi
2910}
2911
2912fastclose_tests()
2913{
2914	if reset "fastclose test"; then
2915		run_tests $ns1 $ns2 10.0.1.1 1024 0 fastclose_client
2916		chk_join_nr 0 0 0
2917		chk_fclose_nr 1 1
2918		chk_rst_nr 1 1 invert
2919	fi
2920
2921	if reset "fastclose server test"; then
2922		run_tests $ns1 $ns2 10.0.1.1 1024 0 fastclose_server
2923		chk_join_nr 0 0 0
2924		chk_fclose_nr 1 1 invert
2925		chk_rst_nr 1 1
2926	fi
2927}
2928
2929pedit_action_pkts()
2930{
2931	tc -n $ns2 -j -s action show action pedit index 100 | \
2932		grep "packets" | \
2933		sed 's/.*"packets":\([0-9]\+\),.*/\1/'
2934}
2935
2936fail_tests()
2937{
2938	# single subflow
2939	if reset_with_fail "Infinite map" 1; then
2940		run_tests $ns1 $ns2 10.0.1.1 128
2941		chk_join_nr 0 0 0 +1 +0 1 0 1 "$(pedit_action_pkts)"
2942		chk_fail_nr 1 -1 invert
2943	fi
2944
2945	# multiple subflows
2946	if reset_with_fail "MP_FAIL MP_RST" 2; then
2947		tc -n $ns2 qdisc add dev ns2eth1 root netem rate 1mbit delay 5
2948		pm_nl_set_limits $ns1 0 1
2949		pm_nl_set_limits $ns2 0 1
2950		pm_nl_add_endpoint $ns2 10.0.2.2 dev ns2eth2 flags subflow
2951		run_tests $ns1 $ns2 10.0.1.1 1024
2952		chk_join_nr 1 1 1 1 0 1 1 0 "$(pedit_action_pkts)"
2953	fi
2954}
2955
2956userspace_tests()
2957{
2958	# userspace pm type prevents add_addr
2959	if reset "userspace pm type prevents add_addr"; then
2960		set_userspace_pm $ns1
2961		pm_nl_set_limits $ns1 0 2
2962		pm_nl_set_limits $ns2 0 2
2963		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2964		run_tests $ns1 $ns2 10.0.1.1
2965		chk_join_nr 0 0 0
2966		chk_add_nr 0 0
2967	fi
2968
2969	# userspace pm type does not echo add_addr without daemon
2970	if reset "userspace pm no echo w/o daemon"; then
2971		set_userspace_pm $ns2
2972		pm_nl_set_limits $ns1 0 2
2973		pm_nl_set_limits $ns2 0 2
2974		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2975		run_tests $ns1 $ns2 10.0.1.1
2976		chk_join_nr 0 0 0
2977		chk_add_nr 1 0
2978	fi
2979
2980	# userspace pm type rejects join
2981	if reset "userspace pm type rejects join"; then
2982		set_userspace_pm $ns1
2983		pm_nl_set_limits $ns1 1 1
2984		pm_nl_set_limits $ns2 1 1
2985		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2986		run_tests $ns1 $ns2 10.0.1.1
2987		chk_join_nr 1 1 0
2988	fi
2989
2990	# userspace pm type does not send join
2991	if reset "userspace pm type does not send join"; then
2992		set_userspace_pm $ns2
2993		pm_nl_set_limits $ns1 1 1
2994		pm_nl_set_limits $ns2 1 1
2995		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2996		run_tests $ns1 $ns2 10.0.1.1
2997		chk_join_nr 0 0 0
2998	fi
2999
3000	# userspace pm type prevents mp_prio
3001	if reset "userspace pm type prevents mp_prio"; then
3002		set_userspace_pm $ns1
3003		pm_nl_set_limits $ns1 1 1
3004		pm_nl_set_limits $ns2 1 1
3005		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
3006		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
3007		chk_join_nr 1 1 0
3008		chk_prio_nr 0 0
3009	fi
3010
3011	# userspace pm type prevents rm_addr
3012	if reset "userspace pm type prevents rm_addr"; then
3013		set_userspace_pm $ns1
3014		set_userspace_pm $ns2
3015		pm_nl_set_limits $ns1 0 1
3016		pm_nl_set_limits $ns2 0 1
3017		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
3018		run_tests $ns1 $ns2 10.0.1.1 0 0 -1 slow
3019		chk_join_nr 0 0 0
3020		chk_rm_nr 0 0
3021	fi
3022
3023	# userspace pm add & remove address
3024	if reset_with_events "userspace pm add & remove address"; then
3025		set_userspace_pm $ns1
3026		pm_nl_set_limits $ns2 1 1
3027		run_tests $ns1 $ns2 10.0.1.1 0 userspace_1 0 slow
3028		chk_join_nr 1 1 1
3029		chk_add_nr 1 1
3030		chk_rm_nr 1 1 invert
3031		kill_events_pids
3032	fi
3033
3034	# userspace pm create destroy subflow
3035	if reset_with_events "userspace pm create destroy subflow"; then
3036		set_userspace_pm $ns2
3037		pm_nl_set_limits $ns1 0 1
3038		run_tests $ns1 $ns2 10.0.1.1 0 0 userspace_1 slow
3039		chk_join_nr 1 1 1
3040		chk_rm_nr 0 1
3041		kill_events_pids
3042	fi
3043}
3044
3045endpoint_tests()
3046{
3047	# userspace pm type prevents add_addr
3048	if reset "implicit EP"; then
3049		pm_nl_set_limits $ns1 2 2
3050		pm_nl_set_limits $ns2 2 2
3051		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
3052		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow &
3053
3054		wait_mpj $ns1
3055		pm_nl_check_endpoint 1 "creation" \
3056			$ns2 10.0.2.2 id 1 flags implicit
3057
3058		pm_nl_add_endpoint $ns2 10.0.2.2 id 33
3059		pm_nl_check_endpoint 0 "ID change is prevented" \
3060			$ns2 10.0.2.2 id 1 flags implicit
3061
3062		pm_nl_add_endpoint $ns2 10.0.2.2 flags signal
3063		pm_nl_check_endpoint 0 "modif is allowed" \
3064			$ns2 10.0.2.2 id 1 flags signal
3065		wait
3066	fi
3067
3068	if reset "delete and re-add"; then
3069		pm_nl_set_limits $ns1 1 1
3070		pm_nl_set_limits $ns2 1 1
3071		pm_nl_add_endpoint $ns2 10.0.2.2 id 2 dev ns2eth2 flags subflow
3072		run_tests $ns1 $ns2 10.0.1.1 4 0 0 slow &
3073
3074		wait_mpj $ns2
3075		pm_nl_del_endpoint $ns2 2 10.0.2.2
3076		sleep 0.5
3077		chk_subflow_nr needtitle "after delete" 1
3078
3079		pm_nl_add_endpoint $ns2 10.0.2.2 dev ns2eth2 flags subflow
3080		wait_mpj $ns2
3081		chk_subflow_nr "" "after re-add" 2
3082		wait
3083	fi
3084}
3085
3086# [$1: error message]
3087usage()
3088{
3089	if [ -n "${1}" ]; then
3090		echo "${1}"
3091		ret=1
3092	fi
3093
3094	echo "mptcp_join usage:"
3095
3096	local key
3097	for key in "${!all_tests[@]}"; do
3098		echo "  -${key} ${all_tests[${key}]}"
3099	done
3100
3101	echo "  -c capture pcap files"
3102	echo "  -C enable data checksum"
3103	echo "  -i use ip mptcp"
3104	echo "  -h help"
3105
3106	echo "[test ids|names]"
3107
3108	exit ${ret}
3109}
3110
3111
3112# Use a "simple" array to force an specific order we cannot have with an associative one
3113all_tests_sorted=(
3114	f@subflows_tests
3115	e@subflows_error_tests
3116	s@signal_address_tests
3117	l@link_failure_tests
3118	t@add_addr_timeout_tests
3119	r@remove_tests
3120	a@add_tests
3121	6@ipv6_tests
3122	4@v4mapped_tests
3123	b@backup_tests
3124	p@add_addr_ports_tests
3125	k@syncookies_tests
3126	S@checksum_tests
3127	d@deny_join_id0_tests
3128	m@fullmesh_tests
3129	z@fastclose_tests
3130	F@fail_tests
3131	u@userspace_tests
3132	I@endpoint_tests
3133)
3134
3135all_tests_args=""
3136all_tests_names=()
3137for subtests in "${all_tests_sorted[@]}"; do
3138	key="${subtests%@*}"
3139	value="${subtests#*@}"
3140
3141	all_tests_args+="${key}"
3142	all_tests_names+=("${value}")
3143	all_tests[${key}]="${value}"
3144done
3145
3146tests=()
3147while getopts "${all_tests_args}cCih" opt; do
3148	case $opt in
3149		["${all_tests_args}"])
3150			tests+=("${all_tests[${opt}]}")
3151			;;
3152		c)
3153			capture=1
3154			;;
3155		C)
3156			checksum=1
3157			;;
3158		i)
3159			ip_mptcp=1
3160			;;
3161		h)
3162			usage
3163			;;
3164		*)
3165			usage "Unknown option: -${opt}"
3166			;;
3167	esac
3168done
3169
3170shift $((OPTIND - 1))
3171
3172for arg in "${@}"; do
3173	if [[ "${arg}" =~ ^[0-9]+$ ]]; then
3174		only_tests_ids+=("${arg}")
3175	else
3176		only_tests_names+=("${arg}")
3177	fi
3178done
3179
3180if [ ${#tests[@]} -eq 0 ]; then
3181	tests=("${all_tests_names[@]}")
3182fi
3183
3184for subtests in "${tests[@]}"; do
3185	"${subtests}"
3186done
3187
3188if [ ${ret} -ne 0 ]; then
3189	echo
3190	echo "${#failed_tests[@]} failure(s) has(ve) been detected:"
3191	for i in $(get_failed_tests_ids); do
3192		echo -e "\t- ${i}: ${failed_tests[${i}]}"
3193	done
3194	echo
3195fi
3196
3197exit $ret
3198