xref: /openbmc/linux/tools/testing/selftests/net/mptcp/mptcp_join.sh (revision a266ef69b890f099069cf51bb40572611c435a54)
1#!/bin/bash
2# SPDX-License-Identifier: GPL-2.0
3
4# Double quotes to prevent globbing and word splitting is recommended in new
5# code but we accept it, especially because there were too many before having
6# address all other issues detected by shellcheck.
7#shellcheck disable=SC2086
8
9ret=0
10sin=""
11sinfail=""
12sout=""
13cin=""
14cinfail=""
15cinsent=""
16cout=""
17capout=""
18ns1=""
19ns2=""
20ksft_skip=4
21timeout_poll=30
22timeout_test=$((timeout_poll * 2 + 1))
23capture=0
24checksum=0
25ip_mptcp=0
26check_invert=0
27validate_checksum=0
28init=0
29evts_ns1=""
30evts_ns2=""
31evts_ns1_pid=0
32evts_ns2_pid=0
33
34declare -A all_tests
35declare -a only_tests_ids
36declare -a only_tests_names
37declare -A failed_tests
38TEST_COUNT=0
39TEST_NAME=""
40nr_blank=40
41
42export FAILING_LINKS=""
43
44# generated using "nfbpf_compile '(ip && (ip[54] & 0xf0) == 0x30) ||
45#				  (ip6 && (ip6[74] & 0xf0) == 0x30)'"
46CBPF_MPTCP_SUBOPTION_ADD_ADDR="14,
47			       48 0 0 0,
48			       84 0 0 240,
49			       21 0 3 64,
50			       48 0 0 54,
51			       84 0 0 240,
52			       21 6 7 48,
53			       48 0 0 0,
54			       84 0 0 240,
55			       21 0 4 96,
56			       48 0 0 74,
57			       84 0 0 240,
58			       21 0 1 48,
59			       6 0 0 65535,
60			       6 0 0 0"
61
62init_partial()
63{
64	capout=$(mktemp)
65
66	local sec rndh
67	sec=$(date +%s)
68	rndh=$(printf %x $sec)-$(mktemp -u XXXXXX)
69
70	ns1="ns1-$rndh"
71	ns2="ns2-$rndh"
72
73	local netns
74	for netns in "$ns1" "$ns2"; do
75		ip netns add $netns || exit $ksft_skip
76		ip -net $netns link set lo up
77		ip netns exec $netns sysctl -q net.mptcp.enabled=1
78		ip netns exec $netns sysctl -q net.mptcp.pm_type=0
79		ip netns exec $netns sysctl -q net.ipv4.conf.all.rp_filter=0
80		ip netns exec $netns sysctl -q net.ipv4.conf.default.rp_filter=0
81		if [ $checksum -eq 1 ]; then
82			ip netns exec $netns sysctl -q net.mptcp.checksum_enabled=1
83		fi
84	done
85
86	check_invert=0
87	validate_checksum=$checksum
88	FAILING_LINKS=""
89
90	#  ns1         ns2
91	# ns1eth1    ns2eth1
92	# ns1eth2    ns2eth2
93	# ns1eth3    ns2eth3
94	# ns1eth4    ns2eth4
95
96	local i
97	for i in $(seq 1 4); do
98		ip link add ns1eth$i netns "$ns1" type veth peer name ns2eth$i netns "$ns2"
99		ip -net "$ns1" addr add 10.0.$i.1/24 dev ns1eth$i
100		ip -net "$ns1" addr add dead:beef:$i::1/64 dev ns1eth$i nodad
101		ip -net "$ns1" link set ns1eth$i up
102
103		ip -net "$ns2" addr add 10.0.$i.2/24 dev ns2eth$i
104		ip -net "$ns2" addr add dead:beef:$i::2/64 dev ns2eth$i nodad
105		ip -net "$ns2" link set ns2eth$i up
106
107		# let $ns2 reach any $ns1 address from any interface
108		ip -net "$ns2" route add default via 10.0.$i.1 dev ns2eth$i metric 10$i
109		ip -net "$ns2" route add default via dead:beef:$i::1 dev ns2eth$i metric 10$i
110	done
111}
112
113init_shapers()
114{
115	local i
116	for i in $(seq 1 4); do
117		tc -n $ns1 qdisc add dev ns1eth$i root netem rate 20mbit delay 1
118		tc -n $ns2 qdisc add dev ns2eth$i root netem rate 20mbit delay 1
119	done
120}
121
122cleanup_partial()
123{
124	rm -f "$capout"
125
126	local netns
127	for netns in "$ns1" "$ns2"; do
128		ip netns del $netns
129		rm -f /tmp/$netns.{nstat,out}
130	done
131}
132
133check_tools()
134{
135	if ! ip -Version &> /dev/null; then
136		echo "SKIP: Could not run test without ip tool"
137		exit $ksft_skip
138	fi
139
140	if ! iptables -V &> /dev/null; then
141		echo "SKIP: Could not run all tests without iptables tool"
142		exit $ksft_skip
143	fi
144
145	if ! ip6tables -V &> /dev/null; then
146		echo "SKIP: Could not run all tests without ip6tables tool"
147		exit $ksft_skip
148	fi
149}
150
151init() {
152	init=1
153
154	check_tools
155
156	sin=$(mktemp)
157	sout=$(mktemp)
158	cin=$(mktemp)
159	cinsent=$(mktemp)
160	cout=$(mktemp)
161	evts_ns1=$(mktemp)
162	evts_ns2=$(mktemp)
163
164	trap cleanup EXIT
165
166	make_file "$cin" "client" 1
167	make_file "$sin" "server" 1
168}
169
170cleanup()
171{
172	rm -f "$cin" "$cout" "$sinfail"
173	rm -f "$sin" "$sout" "$cinsent" "$cinfail"
174	rm -rf $evts_ns1 $evts_ns2
175	cleanup_partial
176}
177
178skip_test()
179{
180	if [ "${#only_tests_ids[@]}" -eq 0 ] && [ "${#only_tests_names[@]}" -eq 0 ]; then
181		return 1
182	fi
183
184	local i
185	for i in "${only_tests_ids[@]}"; do
186		if [ "${TEST_COUNT}" -eq "${i}" ]; then
187			return 1
188		fi
189	done
190	for i in "${only_tests_names[@]}"; do
191		if [ "${TEST_NAME}" = "${i}" ]; then
192			return 1
193		fi
194	done
195
196	return 0
197}
198
199# $1: test name
200reset()
201{
202	TEST_NAME="${1}"
203
204	TEST_COUNT=$((TEST_COUNT+1))
205
206	if skip_test; then
207		return 1
208	fi
209
210	if [ "${init}" != "1" ]; then
211		init
212	else
213		cleanup_partial
214	fi
215
216	init_partial
217
218	return 0
219}
220
221# $1: test name
222reset_with_cookies()
223{
224	reset "${1}" || return 1
225
226	local netns
227	for netns in "$ns1" "$ns2"; do
228		ip netns exec $netns sysctl -q net.ipv4.tcp_syncookies=2
229	done
230}
231
232# $1: test name
233reset_with_add_addr_timeout()
234{
235	local ip="${2:-4}"
236	local tables
237
238	reset "${1}" || return 1
239
240	tables="iptables"
241	if [ $ip -eq 6 ]; then
242		tables="ip6tables"
243	fi
244
245	ip netns exec $ns1 sysctl -q net.mptcp.add_addr_timeout=1
246	ip netns exec $ns2 $tables -A OUTPUT -p tcp \
247		-m tcp --tcp-option 30 \
248		-m bpf --bytecode \
249		"$CBPF_MPTCP_SUBOPTION_ADD_ADDR" \
250		-j DROP
251}
252
253# $1: test name
254reset_with_checksum()
255{
256	local ns1_enable=$1
257	local ns2_enable=$2
258
259	reset "checksum test ${1} ${2}" || return 1
260
261	ip netns exec $ns1 sysctl -q net.mptcp.checksum_enabled=$ns1_enable
262	ip netns exec $ns2 sysctl -q net.mptcp.checksum_enabled=$ns2_enable
263
264	validate_checksum=1
265}
266
267reset_with_allow_join_id0()
268{
269	local ns1_enable=$2
270	local ns2_enable=$3
271
272	reset "${1}" || return 1
273
274	ip netns exec $ns1 sysctl -q net.mptcp.allow_join_initial_addr_port=$ns1_enable
275	ip netns exec $ns2 sysctl -q net.mptcp.allow_join_initial_addr_port=$ns2_enable
276}
277
278# Modify TCP payload without corrupting the TCP packet
279#
280# This rule inverts a 8-bit word at byte offset 148 for the 2nd TCP ACK packets
281# carrying enough data.
282# Once it is done, the TCP Checksum field is updated so the packet is still
283# considered as valid at the TCP level.
284# Because the MPTCP checksum, covering the TCP options and data, has not been
285# updated, the modification will be detected and an MP_FAIL will be emitted:
286# what we want to validate here without corrupting "random" MPTCP options.
287#
288# To avoid having tc producing this pr_info() message for each TCP ACK packets
289# not carrying enough data:
290#
291#     tc action pedit offset 162 out of bounds
292#
293# Netfilter is used to mark packets with enough data.
294reset_with_fail()
295{
296	reset "${1}" || return 1
297
298	ip netns exec $ns1 sysctl -q net.mptcp.checksum_enabled=1
299	ip netns exec $ns2 sysctl -q net.mptcp.checksum_enabled=1
300
301	check_invert=1
302	validate_checksum=1
303	local i="$2"
304	local ip="${3:-4}"
305	local tables
306
307	tables="iptables"
308	if [ $ip -eq 6 ]; then
309		tables="ip6tables"
310	fi
311
312	ip netns exec $ns2 $tables \
313		-t mangle \
314		-A OUTPUT \
315		-o ns2eth$i \
316		-p tcp \
317		-m length --length 150:9999 \
318		-m statistic --mode nth --packet 1 --every 99999 \
319		-j MARK --set-mark 42 || exit 1
320
321	tc -n $ns2 qdisc add dev ns2eth$i clsact || exit 1
322	tc -n $ns2 filter add dev ns2eth$i egress \
323		protocol ip prio 1000 \
324		handle 42 fw \
325		action pedit munge offset 148 u8 invert \
326		pipe csum tcp \
327		index 100 || exit 1
328}
329
330reset_with_events()
331{
332	reset "${1}" || return 1
333
334	:> "$evts_ns1"
335	:> "$evts_ns2"
336	ip netns exec $ns1 ./pm_nl_ctl events >> "$evts_ns1" 2>&1 &
337	evts_ns1_pid=$!
338	ip netns exec $ns2 ./pm_nl_ctl events >> "$evts_ns2" 2>&1 &
339	evts_ns2_pid=$!
340}
341
342fail_test()
343{
344	ret=1
345	failed_tests[${TEST_COUNT}]="${TEST_NAME}"
346}
347
348get_failed_tests_ids()
349{
350	# sorted
351	local i
352	for i in "${!failed_tests[@]}"; do
353		echo "${i}"
354	done | sort -n
355}
356
357print_file_err()
358{
359	ls -l "$1" 1>&2
360	echo "Trailing bytes are: "
361	tail -c 27 "$1"
362}
363
364check_transfer()
365{
366	local in=$1
367	local out=$2
368	local what=$3
369	local bytes=$4
370	local i a b
371
372	local line
373	if [ -n "$bytes" ]; then
374		# when truncating we must check the size explicitly
375		local out_size=$(wc -c $out | awk '{print $1}')
376		if [ $out_size -ne $bytes ]; then
377			echo "[ FAIL ] $what output file has wrong size ($out_size, $bytes)"
378			fail_test
379			return 1
380		fi
381		bytes="--bytes=${bytes}"
382	fi
383	cmp -l "$in" "$out" ${bytes} | while read -r i a b; do
384		local sum=$((0${a} + 0${b}))
385		if [ $check_invert -eq 0 ] || [ $sum -ne $((0xff)) ]; then
386			echo "[ FAIL ] $what does not match (in, out):"
387			print_file_err "$in"
388			print_file_err "$out"
389			fail_test
390
391			return 1
392		else
393			echo "$what has inverted byte at ${i}"
394		fi
395	done
396
397	return 0
398}
399
400do_ping()
401{
402	local listener_ns="$1"
403	local connector_ns="$2"
404	local connect_addr="$3"
405
406	if ! ip netns exec ${connector_ns} ping -q -c 1 $connect_addr >/dev/null; then
407		echo "$listener_ns -> $connect_addr connectivity [ FAIL ]" 1>&2
408		fail_test
409	fi
410}
411
412link_failure()
413{
414	local ns="$1"
415
416	if [ -z "$FAILING_LINKS" ]; then
417		l=$((RANDOM%4))
418		FAILING_LINKS=$((l+1))
419	fi
420
421	local l
422	for l in $FAILING_LINKS; do
423		local veth="ns1eth$l"
424		ip -net "$ns" link set "$veth" down
425	done
426}
427
428# $1: IP address
429is_v6()
430{
431	[ -z "${1##*:*}" ]
432}
433
434# $1: ns, $2: port
435wait_local_port_listen()
436{
437	local listener_ns="${1}"
438	local port="${2}"
439
440	local port_hex
441	port_hex="$(printf "%04X" "${port}")"
442
443	local i
444	for i in $(seq 10); do
445		ip netns exec "${listener_ns}" cat /proc/net/tcp* | \
446			awk "BEGIN {rc=1} {if (\$2 ~ /:${port_hex}\$/ && \$4 ~ /0A/) {rc=0; exit}} END {exit rc}" &&
447			break
448		sleep 0.1
449	done
450}
451
452rm_addr_count()
453{
454	local ns=${1}
455
456	ip netns exec ${ns} nstat -as | grep MPTcpExtRmAddr | awk '{print $2}'
457}
458
459# $1: ns, $2: old rm_addr counter in $ns
460wait_rm_addr()
461{
462	local ns="${1}"
463	local old_cnt="${2}"
464	local cnt
465
466	local i
467	for i in $(seq 10); do
468		cnt=$(rm_addr_count ${ns})
469		[ "$cnt" = "${old_cnt}" ] || break
470		sleep 0.1
471	done
472}
473
474wait_mpj()
475{
476	local ns="${1}"
477	local cnt old_cnt
478
479	old_cnt=$(ip netns exec ${ns} nstat -as | grep MPJoinAckRx | awk '{print $2}')
480
481	local i
482	for i in $(seq 10); do
483		cnt=$(ip netns exec ${ns} nstat -as | grep MPJoinAckRx | awk '{print $2}')
484		[ "$cnt" = "${old_cnt}" ] || break
485		sleep 0.1
486	done
487}
488
489kill_wait()
490{
491	kill $1 > /dev/null 2>&1
492	wait $1 2>/dev/null
493}
494
495kill_events_pids()
496{
497	kill_wait $evts_ns1_pid
498	kill_wait $evts_ns2_pid
499}
500
501pm_nl_set_limits()
502{
503	local ns=$1
504	local addrs=$2
505	local subflows=$3
506
507	if [ $ip_mptcp -eq 1 ]; then
508		ip -n $ns mptcp limits set add_addr_accepted $addrs subflows $subflows
509	else
510		ip netns exec $ns ./pm_nl_ctl limits $addrs $subflows
511	fi
512}
513
514pm_nl_add_endpoint()
515{
516	local ns=$1
517	local addr=$2
518	local flags _flags
519	local port _port
520	local dev _dev
521	local id _id
522	local nr=2
523
524	local p
525	for p in "${@}"
526	do
527		if [ $p = "flags" ]; then
528			eval _flags=\$"$nr"
529			[ -n "$_flags" ]; flags="flags $_flags"
530		fi
531		if [ $p = "dev" ]; then
532			eval _dev=\$"$nr"
533			[ -n "$_dev" ]; dev="dev $_dev"
534		fi
535		if [ $p = "id" ]; then
536			eval _id=\$"$nr"
537			[ -n "$_id" ]; id="id $_id"
538		fi
539		if [ $p = "port" ]; then
540			eval _port=\$"$nr"
541			[ -n "$_port" ]; port="port $_port"
542		fi
543
544		nr=$((nr + 1))
545	done
546
547	if [ $ip_mptcp -eq 1 ]; then
548		ip -n $ns mptcp endpoint add $addr ${_flags//","/" "} $dev $id $port
549	else
550		ip netns exec $ns ./pm_nl_ctl add $addr $flags $dev $id $port
551	fi
552}
553
554pm_nl_del_endpoint()
555{
556	local ns=$1
557	local id=$2
558	local addr=$3
559
560	if [ $ip_mptcp -eq 1 ]; then
561		ip -n $ns mptcp endpoint delete id $id $addr
562	else
563		ip netns exec $ns ./pm_nl_ctl del $id $addr
564	fi
565}
566
567pm_nl_flush_endpoint()
568{
569	local ns=$1
570
571	if [ $ip_mptcp -eq 1 ]; then
572		ip -n $ns mptcp endpoint flush
573	else
574		ip netns exec $ns ./pm_nl_ctl flush
575	fi
576}
577
578pm_nl_show_endpoints()
579{
580	local ns=$1
581
582	if [ $ip_mptcp -eq 1 ]; then
583		ip -n $ns mptcp endpoint show
584	else
585		ip netns exec $ns ./pm_nl_ctl dump
586	fi
587}
588
589pm_nl_change_endpoint()
590{
591	local ns=$1
592	local id=$2
593	local flags=$3
594
595	if [ $ip_mptcp -eq 1 ]; then
596		ip -n $ns mptcp endpoint change id $id ${flags//","/" "}
597	else
598		ip netns exec $ns ./pm_nl_ctl set id $id flags $flags
599	fi
600}
601
602pm_nl_check_endpoint()
603{
604	local line expected_line
605	local need_title=$1
606	local msg="$2"
607	local ns=$3
608	local addr=$4
609	local _flags=""
610	local flags
611	local _port
612	local port
613	local dev
614	local _id
615	local id
616
617	if [ "${need_title}" = 1 ]; then
618		printf "%03u %-36s %s" "${TEST_COUNT}" "${TEST_NAME}" "${msg}"
619	else
620		printf "%-${nr_blank}s %s" " " "${msg}"
621	fi
622
623	shift 4
624	while [ -n "$1" ]; do
625		if [ $1 = "flags" ]; then
626			_flags=$2
627			[ -n "$_flags" ]; flags="flags $_flags"
628			shift
629		elif [ $1 = "dev" ]; then
630			[ -n "$2" ]; dev="dev $1"
631			shift
632		elif [ $1 = "id" ]; then
633			_id=$2
634			[ -n "$_id" ]; id="id $_id"
635			shift
636		elif [ $1 = "port" ]; then
637			_port=$2
638			[ -n "$_port" ]; port=" port $_port"
639			shift
640		fi
641
642		shift
643	done
644
645	if [ -z "$id" ]; then
646		echo "[skip] bad test - missing endpoint id"
647		return
648	fi
649
650	if [ $ip_mptcp -eq 1 ]; then
651		line=$(ip -n $ns mptcp endpoint show $id)
652		# the dump order is: address id flags port dev
653		expected_line="$addr"
654		[ -n "$addr" ] && expected_line="$expected_line $addr"
655		expected_line="$expected_line $id"
656		[ -n "$_flags" ] && expected_line="$expected_line ${_flags//","/" "}"
657		[ -n "$dev" ] && expected_line="$expected_line $dev"
658		[ -n "$port" ] && expected_line="$expected_line $port"
659	else
660		line=$(ip netns exec $ns ./pm_nl_ctl get $_id)
661		# the dump order is: id flags dev address port
662		expected_line="$id"
663		[ -n "$flags" ] && expected_line="$expected_line $flags"
664		[ -n "$dev" ] && expected_line="$expected_line $dev"
665		[ -n "$addr" ] && expected_line="$expected_line $addr"
666		[ -n "$_port" ] && expected_line="$expected_line $_port"
667	fi
668	if [ "$line" = "$expected_line" ]; then
669		echo "[ ok ]"
670	else
671		echo "[fail] expected '$expected_line' found '$line'"
672		fail_test
673	fi
674}
675
676filter_tcp_from()
677{
678	local ns="${1}"
679	local src="${2}"
680	local target="${3}"
681
682	ip netns exec "${ns}" iptables -A INPUT -s "${src}" -p tcp -j "${target}"
683}
684
685do_transfer()
686{
687	local listener_ns="$1"
688	local connector_ns="$2"
689	local cl_proto="$3"
690	local srv_proto="$4"
691	local connect_addr="$5"
692	local test_link_fail="$6"
693	local addr_nr_ns1="$7"
694	local addr_nr_ns2="$8"
695	local speed="$9"
696	local sflags="${10}"
697
698	local port=$((10000 + TEST_COUNT - 1))
699	local cappid
700	local userspace_pm=0
701
702	:> "$cout"
703	:> "$sout"
704	:> "$capout"
705
706	if [ $capture -eq 1 ]; then
707		local capuser
708		if [ -z $SUDO_USER ] ; then
709			capuser=""
710		else
711			capuser="-Z $SUDO_USER"
712		fi
713
714		capfile=$(printf "mp_join-%02u-%s.pcap" "$TEST_COUNT" "${listener_ns}")
715
716		echo "Capturing traffic for test $TEST_COUNT into $capfile"
717		ip netns exec ${listener_ns} tcpdump -i any -s 65535 -B 32768 $capuser -w $capfile > "$capout" 2>&1 &
718		cappid=$!
719
720		sleep 1
721	fi
722
723	NSTAT_HISTORY=/tmp/${listener_ns}.nstat ip netns exec ${listener_ns} \
724		nstat -n
725	NSTAT_HISTORY=/tmp/${connector_ns}.nstat ip netns exec ${connector_ns} \
726		nstat -n
727
728	local extra_args
729	if [ $speed = "fast" ]; then
730		extra_args="-j"
731	elif [ $speed = "slow" ]; then
732		extra_args="-r 50"
733	elif [[ $speed = "speed_"* ]]; then
734		extra_args="-r ${speed:6}"
735	fi
736
737	if [[ "${addr_nr_ns1}" = "userspace_"* ]]; then
738		userspace_pm=1
739		addr_nr_ns1=${addr_nr_ns1:10}
740	fi
741
742	local flags="subflow"
743	local extra_cl_args=""
744	local extra_srv_args=""
745	local trunc_size=""
746	if [[ "${addr_nr_ns2}" = "fastclose_"* ]]; then
747		if [ ${test_link_fail} -le 1 ]; then
748			echo "fastclose tests need test_link_fail argument"
749			fail_test
750			return 1
751		fi
752
753		# disconnect
754		trunc_size=${test_link_fail}
755		local side=${addr_nr_ns2:10}
756
757		if [ ${side} = "client" ]; then
758			extra_cl_args="-f ${test_link_fail}"
759			extra_srv_args="-f -1"
760		elif [ ${side} = "server" ]; then
761			extra_srv_args="-f ${test_link_fail}"
762			extra_cl_args="-f -1"
763		else
764			echo "wrong/unknown fastclose spec ${side}"
765			fail_test
766			return 1
767		fi
768		addr_nr_ns2=0
769	elif [[ "${addr_nr_ns2}" = "userspace_"* ]]; then
770		userspace_pm=1
771		addr_nr_ns2=${addr_nr_ns2:10}
772	elif [[ "${addr_nr_ns2}" = "fullmesh_"* ]]; then
773		flags="${flags},fullmesh"
774		addr_nr_ns2=${addr_nr_ns2:9}
775	fi
776
777	extra_srv_args="$extra_args $extra_srv_args"
778	if [ "$test_link_fail" -gt 1 ];then
779		timeout ${timeout_test} \
780			ip netns exec ${listener_ns} \
781				./mptcp_connect -t ${timeout_poll} -l -p $port -s ${srv_proto} \
782					$extra_srv_args "::" < "$sinfail" > "$sout" &
783	else
784		timeout ${timeout_test} \
785			ip netns exec ${listener_ns} \
786				./mptcp_connect -t ${timeout_poll} -l -p $port -s ${srv_proto} \
787					$extra_srv_args "::" < "$sin" > "$sout" &
788	fi
789	local spid=$!
790
791	wait_local_port_listen "${listener_ns}" "${port}"
792
793	extra_cl_args="$extra_args $extra_cl_args"
794	if [ "$test_link_fail" -eq 0 ];then
795		timeout ${timeout_test} \
796			ip netns exec ${connector_ns} \
797				./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \
798					$extra_cl_args $connect_addr < "$cin" > "$cout" &
799	elif [ "$test_link_fail" -eq 1 ] || [ "$test_link_fail" -eq 2 ];then
800		( cat "$cinfail" ; sleep 2; link_failure $listener_ns ; cat "$cinfail" ) | \
801			tee "$cinsent" | \
802			timeout ${timeout_test} \
803				ip netns exec ${connector_ns} \
804					./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \
805						$extra_cl_args $connect_addr > "$cout" &
806	else
807		tee "$cinsent" < "$cinfail" | \
808			timeout ${timeout_test} \
809				ip netns exec ${connector_ns} \
810					./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \
811						$extra_cl_args $connect_addr > "$cout" &
812	fi
813	local cpid=$!
814
815	# let the mptcp subflow be established in background before
816	# do endpoint manipulation
817	if [ $addr_nr_ns1 != "0" ] || [ $addr_nr_ns2 != "0" ]; then
818		sleep 1
819	fi
820
821	if [ $addr_nr_ns1 -gt 0 ]; then
822		local counter=2
823		local add_nr_ns1=${addr_nr_ns1}
824		local id=10
825		local tk
826		while [ $add_nr_ns1 -gt 0 ]; do
827			local addr
828			if is_v6 "${connect_addr}"; then
829				addr="dead:beef:$counter::1"
830			else
831				addr="10.0.$counter.1"
832			fi
833			if [ $userspace_pm -eq 0 ]; then
834				pm_nl_add_endpoint $ns1 $addr flags signal
835			else
836				tk=$(grep "type:1," "$evts_ns1" |
837				     sed -n 's/.*\(token:\)\([[:digit:]]*\).*$/\2/p;q')
838				ip netns exec ${listener_ns} ./pm_nl_ctl ann $addr token $tk id $id
839				sleep 1
840				ip netns exec ${listener_ns} ./pm_nl_ctl rem token $tk id $id
841			fi
842
843			counter=$((counter + 1))
844			add_nr_ns1=$((add_nr_ns1 - 1))
845			id=$((id + 1))
846		done
847	elif [ $addr_nr_ns1 -lt 0 ]; then
848		local rm_nr_ns1=$((-addr_nr_ns1))
849		if [ $rm_nr_ns1 -lt 8 ]; then
850			local counter=0
851			local line
852			pm_nl_show_endpoints ${listener_ns} | while read -r line; do
853				# shellcheck disable=SC2206 # we do want to split per word
854				local arr=($line)
855				local nr=0
856
857				local i
858				for i in "${arr[@]}"; do
859					if [ $i = "id" ]; then
860						if [ $counter -eq $rm_nr_ns1 ]; then
861							break
862						fi
863						id=${arr[$nr+1]}
864						rm_addr=$(rm_addr_count ${connector_ns})
865						pm_nl_del_endpoint ${listener_ns} $id
866						wait_rm_addr ${connector_ns} ${rm_addr}
867						counter=$((counter + 1))
868					fi
869					nr=$((nr + 1))
870				done
871			done
872		elif [ $rm_nr_ns1 -eq 8 ]; then
873			pm_nl_flush_endpoint ${listener_ns}
874		elif [ $rm_nr_ns1 -eq 9 ]; then
875			pm_nl_del_endpoint ${listener_ns} 0 ${connect_addr}
876		fi
877	fi
878
879	# if newly added endpoints must be deleted, give the background msk
880	# some time to created them
881	[ $addr_nr_ns1 -gt 0 ] && [ $addr_nr_ns2 -lt 0 ] && sleep 1
882
883	if [ $addr_nr_ns2 -gt 0 ]; then
884		local add_nr_ns2=${addr_nr_ns2}
885		local counter=3
886		local id=20
887		local tk da dp sp
888		while [ $add_nr_ns2 -gt 0 ]; do
889			local addr
890			if is_v6 "${connect_addr}"; then
891				addr="dead:beef:$counter::2"
892			else
893				addr="10.0.$counter.2"
894			fi
895			if [ $userspace_pm -eq 0 ]; then
896				pm_nl_add_endpoint $ns2 $addr flags $flags
897			else
898				tk=$(sed -n 's/.*\(token:\)\([[:digit:]]*\).*$/\2/p;q' "$evts_ns2")
899				da=$(sed -n 's/.*\(daddr4:\)\([0-9.]*\).*$/\2/p;q' "$evts_ns2")
900				dp=$(sed -n 's/.*\(dport:\)\([[:digit:]]*\).*$/\2/p;q' "$evts_ns2")
901				ip netns exec ${connector_ns} ./pm_nl_ctl csf lip $addr lid $id \
902									rip $da rport $dp token $tk
903				sleep 1
904				sp=$(grep "type:10" "$evts_ns2" |
905				     sed -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q')
906				ip netns exec ${connector_ns} ./pm_nl_ctl dsf lip $addr lport $sp \
907									rip $da rport $dp token $tk
908			fi
909			counter=$((counter + 1))
910			add_nr_ns2=$((add_nr_ns2 - 1))
911			id=$((id + 1))
912		done
913	elif [ $addr_nr_ns2 -lt 0 ]; then
914		local rm_nr_ns2=$((-addr_nr_ns2))
915		if [ $rm_nr_ns2 -lt 8 ]; then
916			local counter=0
917			local line
918			pm_nl_show_endpoints ${connector_ns} | while read -r line; do
919				# shellcheck disable=SC2206 # we do want to split per word
920				local arr=($line)
921				local nr=0
922
923				local i
924				for i in "${arr[@]}"; do
925					if [ $i = "id" ]; then
926						if [ $counter -eq $rm_nr_ns2 ]; then
927							break
928						fi
929						local id rm_addr
930						# rm_addr are serialized, allow the previous one to
931						# complete
932						id=${arr[$nr+1]}
933						rm_addr=$(rm_addr_count ${listener_ns})
934						pm_nl_del_endpoint ${connector_ns} $id
935						wait_rm_addr ${listener_ns} ${rm_addr}
936						counter=$((counter + 1))
937					fi
938					nr=$((nr + 1))
939				done
940			done
941		elif [ $rm_nr_ns2 -eq 8 ]; then
942			pm_nl_flush_endpoint ${connector_ns}
943		elif [ $rm_nr_ns2 -eq 9 ]; then
944			local addr
945			if is_v6 "${connect_addr}"; then
946				addr="dead:beef:1::2"
947			else
948				addr="10.0.1.2"
949			fi
950			pm_nl_del_endpoint ${connector_ns} 0 $addr
951		fi
952	fi
953
954	if [ -n "${sflags}" ]; then
955		sleep 1
956
957		local netns
958		for netns in "$ns1" "$ns2"; do
959			local line
960			pm_nl_show_endpoints $netns | while read -r line; do
961				# shellcheck disable=SC2206 # we do want to split per word
962				local arr=($line)
963				local nr=0
964				local id
965
966				local i
967				for i in "${arr[@]}"; do
968					if [ $i = "id" ]; then
969						id=${arr[$nr+1]}
970					fi
971					nr=$((nr + 1))
972				done
973				pm_nl_change_endpoint $netns $id $sflags
974			done
975		done
976	fi
977
978	wait $cpid
979	local retc=$?
980	wait $spid
981	local rets=$?
982
983	if [ $capture -eq 1 ]; then
984	    sleep 1
985	    kill $cappid
986	fi
987
988	NSTAT_HISTORY=/tmp/${listener_ns}.nstat ip netns exec ${listener_ns} \
989		nstat | grep Tcp > /tmp/${listener_ns}.out
990	NSTAT_HISTORY=/tmp/${connector_ns}.nstat ip netns exec ${connector_ns} \
991		nstat | grep Tcp > /tmp/${connector_ns}.out
992
993	if [ ${rets} -ne 0 ] || [ ${retc} -ne 0 ]; then
994		echo " client exit code $retc, server $rets" 1>&2
995		echo -e "\nnetns ${listener_ns} socket stat for ${port}:" 1>&2
996		ip netns exec ${listener_ns} ss -Menita 1>&2 -o "sport = :$port"
997		cat /tmp/${listener_ns}.out
998		echo -e "\nnetns ${connector_ns} socket stat for ${port}:" 1>&2
999		ip netns exec ${connector_ns} ss -Menita 1>&2 -o "dport = :$port"
1000		cat /tmp/${connector_ns}.out
1001
1002		cat "$capout"
1003		fail_test
1004		return 1
1005	fi
1006
1007	if [ "$test_link_fail" -gt 1 ];then
1008		check_transfer $sinfail $cout "file received by client" $trunc_size
1009	else
1010		check_transfer $sin $cout "file received by client" $trunc_size
1011	fi
1012	retc=$?
1013	if [ "$test_link_fail" -eq 0 ];then
1014		check_transfer $cin $sout "file received by server" $trunc_size
1015	else
1016		check_transfer $cinsent $sout "file received by server" $trunc_size
1017	fi
1018	rets=$?
1019
1020	if [ $retc -eq 0 ] && [ $rets -eq 0 ];then
1021		cat "$capout"
1022		return 0
1023	fi
1024
1025	cat "$capout"
1026	return 1
1027}
1028
1029make_file()
1030{
1031	local name=$1
1032	local who=$2
1033	local size=$3
1034
1035	dd if=/dev/urandom of="$name" bs=1024 count=$size 2> /dev/null
1036	echo -e "\nMPTCP_TEST_FILE_END_MARKER" >> "$name"
1037
1038	echo "Created $name (size $size KB) containing data sent by $who"
1039}
1040
1041run_tests()
1042{
1043	local listener_ns="$1"
1044	local connector_ns="$2"
1045	local connect_addr="$3"
1046	local test_linkfail="${4:-0}"
1047	local addr_nr_ns1="${5:-0}"
1048	local addr_nr_ns2="${6:-0}"
1049	local speed="${7:-fast}"
1050	local sflags="${8:-""}"
1051
1052	local size
1053
1054	# The values above 2 are reused to make test files
1055	# with the given sizes (KB)
1056	if [ "$test_linkfail" -gt 2 ]; then
1057		size=$test_linkfail
1058
1059		if [ -z "$cinfail" ]; then
1060			cinfail=$(mktemp)
1061		fi
1062		make_file "$cinfail" "client" $size
1063	# create the input file for the failure test when
1064	# the first failure test run
1065	elif [ "$test_linkfail" -ne 0 ] && [ -z "$cinfail" ]; then
1066		# the client file must be considerably larger
1067		# of the maximum expected cwin value, or the
1068		# link utilization will be not predicable
1069		size=$((RANDOM%2))
1070		size=$((size+1))
1071		size=$((size*8192))
1072		size=$((size + ( RANDOM % 8192) ))
1073
1074		cinfail=$(mktemp)
1075		make_file "$cinfail" "client" $size
1076	fi
1077
1078	if [ "$test_linkfail" -gt 2 ]; then
1079		size=$test_linkfail
1080
1081		if [ -z "$sinfail" ]; then
1082			sinfail=$(mktemp)
1083		fi
1084		make_file "$sinfail" "server" $size
1085	elif [ "$test_linkfail" -eq 2 ] && [ -z "$sinfail" ]; then
1086		size=$((RANDOM%16))
1087		size=$((size+1))
1088		size=$((size*2048))
1089
1090		sinfail=$(mktemp)
1091		make_file "$sinfail" "server" $size
1092	fi
1093
1094	do_transfer ${listener_ns} ${connector_ns} MPTCP MPTCP ${connect_addr} \
1095		${test_linkfail} ${addr_nr_ns1} ${addr_nr_ns2} ${speed} ${sflags}
1096}
1097
1098dump_stats()
1099{
1100	echo Server ns stats
1101	ip netns exec $ns1 nstat -as | grep Tcp
1102	echo Client ns stats
1103	ip netns exec $ns2 nstat -as | grep Tcp
1104}
1105
1106chk_csum_nr()
1107{
1108	local csum_ns1=${1:-0}
1109	local csum_ns2=${2:-0}
1110	local count
1111	local dump_stats
1112	local extra_msg=""
1113	local allow_multi_errors_ns1=0
1114	local allow_multi_errors_ns2=0
1115
1116	if [[ "${csum_ns1}" = "+"* ]]; then
1117		allow_multi_errors_ns1=1
1118		csum_ns1=${csum_ns1:1}
1119	fi
1120	if [[ "${csum_ns2}" = "+"* ]]; then
1121		allow_multi_errors_ns2=1
1122		csum_ns2=${csum_ns2:1}
1123	fi
1124
1125	printf "%-${nr_blank}s %s" " " "sum"
1126	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtDataCsumErr | awk '{print $2}')
1127	[ -z "$count" ] && count=0
1128	if [ "$count" != "$csum_ns1" ]; then
1129		extra_msg="$extra_msg ns1=$count"
1130	fi
1131	if { [ "$count" != $csum_ns1 ] && [ $allow_multi_errors_ns1 -eq 0 ]; } ||
1132	   { [ "$count" -lt $csum_ns1 ] && [ $allow_multi_errors_ns1 -eq 1 ]; }; then
1133		echo "[fail] got $count data checksum error[s] expected $csum_ns1"
1134		fail_test
1135		dump_stats=1
1136	else
1137		echo -n "[ ok ]"
1138	fi
1139	echo -n " - csum  "
1140	count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtDataCsumErr | awk '{print $2}')
1141	[ -z "$count" ] && count=0
1142	if [ "$count" != "$csum_ns2" ]; then
1143		extra_msg="$extra_msg ns2=$count"
1144	fi
1145	if { [ "$count" != $csum_ns2 ] && [ $allow_multi_errors_ns2 -eq 0 ]; } ||
1146	   { [ "$count" -lt $csum_ns2 ] && [ $allow_multi_errors_ns2 -eq 1 ]; }; then
1147		echo "[fail] got $count data checksum error[s] expected $csum_ns2"
1148		fail_test
1149		dump_stats=1
1150	else
1151		echo -n "[ ok ]"
1152	fi
1153	[ "${dump_stats}" = 1 ] && dump_stats
1154
1155	echo "$extra_msg"
1156}
1157
1158chk_fail_nr()
1159{
1160	local fail_tx=$1
1161	local fail_rx=$2
1162	local ns_invert=${3:-""}
1163	local count
1164	local dump_stats
1165	local ns_tx=$ns1
1166	local ns_rx=$ns2
1167	local extra_msg=""
1168	local allow_tx_lost=0
1169	local allow_rx_lost=0
1170
1171	if [[ $ns_invert = "invert" ]]; then
1172		ns_tx=$ns2
1173		ns_rx=$ns1
1174		extra_msg=" invert"
1175	fi
1176
1177	if [[ "${fail_tx}" = "-"* ]]; then
1178		allow_tx_lost=1
1179		fail_tx=${fail_tx:1}
1180	fi
1181	if [[ "${fail_rx}" = "-"* ]]; then
1182		allow_rx_lost=1
1183		fail_rx=${fail_rx:1}
1184	fi
1185
1186	printf "%-${nr_blank}s %s" " " "ftx"
1187	count=$(ip netns exec $ns_tx nstat -as | grep MPTcpExtMPFailTx | awk '{print $2}')
1188	[ -z "$count" ] && count=0
1189	if [ "$count" != "$fail_tx" ]; then
1190		extra_msg="$extra_msg,tx=$count"
1191	fi
1192	if { [ "$count" != "$fail_tx" ] && [ $allow_tx_lost -eq 0 ]; } ||
1193	   { [ "$count" -gt "$fail_tx" ] && [ $allow_tx_lost -eq 1 ]; }; then
1194		echo "[fail] got $count MP_FAIL[s] TX expected $fail_tx"
1195		fail_test
1196		dump_stats=1
1197	else
1198		echo -n "[ ok ]"
1199	fi
1200
1201	echo -n " - failrx"
1202	count=$(ip netns exec $ns_rx nstat -as | grep MPTcpExtMPFailRx | awk '{print $2}')
1203	[ -z "$count" ] && count=0
1204	if [ "$count" != "$fail_rx" ]; then
1205		extra_msg="$extra_msg,rx=$count"
1206	fi
1207	if { [ "$count" != "$fail_rx" ] && [ $allow_rx_lost -eq 0 ]; } ||
1208	   { [ "$count" -gt "$fail_rx" ] && [ $allow_rx_lost -eq 1 ]; }; then
1209		echo "[fail] got $count MP_FAIL[s] RX expected $fail_rx"
1210		fail_test
1211		dump_stats=1
1212	else
1213		echo -n "[ ok ]"
1214	fi
1215
1216	[ "${dump_stats}" = 1 ] && dump_stats
1217
1218	echo "$extra_msg"
1219}
1220
1221chk_fclose_nr()
1222{
1223	local fclose_tx=$1
1224	local fclose_rx=$2
1225	local ns_invert=$3
1226	local count
1227	local dump_stats
1228	local ns_tx=$ns2
1229	local ns_rx=$ns1
1230	local extra_msg="   "
1231
1232	if [[ $ns_invert = "invert" ]]; then
1233		ns_tx=$ns1
1234		ns_rx=$ns2
1235		extra_msg=${extra_msg}"invert"
1236	fi
1237
1238	printf "%-${nr_blank}s %s" " " "ctx"
1239	count=$(ip netns exec $ns_tx nstat -as | grep MPTcpExtMPFastcloseTx | awk '{print $2}')
1240	[ -z "$count" ] && count=0
1241	[ "$count" != "$fclose_tx" ] && extra_msg="$extra_msg,tx=$count"
1242	if [ "$count" != "$fclose_tx" ]; then
1243		echo "[fail] got $count MP_FASTCLOSE[s] TX expected $fclose_tx"
1244		fail_test
1245		dump_stats=1
1246	else
1247		echo -n "[ ok ]"
1248	fi
1249
1250	echo -n " - fclzrx"
1251	count=$(ip netns exec $ns_rx nstat -as | grep MPTcpExtMPFastcloseRx | awk '{print $2}')
1252	[ -z "$count" ] && count=0
1253	[ "$count" != "$fclose_rx" ] && extra_msg="$extra_msg,rx=$count"
1254	if [ "$count" != "$fclose_rx" ]; then
1255		echo "[fail] got $count MP_FASTCLOSE[s] RX expected $fclose_rx"
1256		fail_test
1257		dump_stats=1
1258	else
1259		echo -n "[ ok ]"
1260	fi
1261
1262	[ "${dump_stats}" = 1 ] && dump_stats
1263
1264	echo "$extra_msg"
1265}
1266
1267chk_rst_nr()
1268{
1269	local rst_tx=$1
1270	local rst_rx=$2
1271	local ns_invert=${3:-""}
1272	local count
1273	local dump_stats
1274	local ns_tx=$ns1
1275	local ns_rx=$ns2
1276	local extra_msg=""
1277
1278	if [[ $ns_invert = "invert" ]]; then
1279		ns_tx=$ns2
1280		ns_rx=$ns1
1281		extra_msg="   invert"
1282	fi
1283
1284	printf "%-${nr_blank}s %s" " " "rtx"
1285	count=$(ip netns exec $ns_tx nstat -as | grep MPTcpExtMPRstTx | awk '{print $2}')
1286	[ -z "$count" ] && count=0
1287	if [ $count -lt $rst_tx ]; then
1288		echo "[fail] got $count MP_RST[s] TX expected $rst_tx"
1289		fail_test
1290		dump_stats=1
1291	else
1292		echo -n "[ ok ]"
1293	fi
1294
1295	echo -n " - rstrx "
1296	count=$(ip netns exec $ns_rx nstat -as | grep MPTcpExtMPRstRx | awk '{print $2}')
1297	[ -z "$count" ] && count=0
1298	if [ "$count" -lt "$rst_rx" ]; then
1299		echo "[fail] got $count MP_RST[s] RX expected $rst_rx"
1300		fail_test
1301		dump_stats=1
1302	else
1303		echo -n "[ ok ]"
1304	fi
1305
1306	[ "${dump_stats}" = 1 ] && dump_stats
1307
1308	echo "$extra_msg"
1309}
1310
1311chk_infi_nr()
1312{
1313	local infi_tx=$1
1314	local infi_rx=$2
1315	local count
1316	local dump_stats
1317
1318	printf "%-${nr_blank}s %s" " " "itx"
1319	count=$(ip netns exec $ns2 nstat -as | grep InfiniteMapTx | awk '{print $2}')
1320	[ -z "$count" ] && count=0
1321	if [ "$count" != "$infi_tx" ]; then
1322		echo "[fail] got $count infinite map[s] TX expected $infi_tx"
1323		fail_test
1324		dump_stats=1
1325	else
1326		echo -n "[ ok ]"
1327	fi
1328
1329	echo -n " - infirx"
1330	count=$(ip netns exec $ns1 nstat -as | grep InfiniteMapRx | awk '{print $2}')
1331	[ -z "$count" ] && count=0
1332	if [ "$count" != "$infi_rx" ]; then
1333		echo "[fail] got $count infinite map[s] RX expected $infi_rx"
1334		fail_test
1335		dump_stats=1
1336	else
1337		echo "[ ok ]"
1338	fi
1339
1340	[ "${dump_stats}" = 1 ] && dump_stats
1341}
1342
1343chk_join_nr()
1344{
1345	local syn_nr=$1
1346	local syn_ack_nr=$2
1347	local ack_nr=$3
1348	local csum_ns1=${4:-0}
1349	local csum_ns2=${5:-0}
1350	local fail_nr=${6:-0}
1351	local rst_nr=${7:-0}
1352	local infi_nr=${8:-0}
1353	local corrupted_pkts=${9:-0}
1354	local count
1355	local dump_stats
1356	local with_cookie
1357	local title="${TEST_NAME}"
1358
1359	if [ "${corrupted_pkts}" -gt 0 ]; then
1360		title+=": ${corrupted_pkts} corrupted pkts"
1361	fi
1362
1363	printf "%03u %-36s %s" "${TEST_COUNT}" "${title}" "syn"
1364	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinSynRx | awk '{print $2}')
1365	[ -z "$count" ] && count=0
1366	if [ "$count" != "$syn_nr" ]; then
1367		echo "[fail] got $count JOIN[s] syn expected $syn_nr"
1368		fail_test
1369		dump_stats=1
1370	else
1371		echo -n "[ ok ]"
1372	fi
1373
1374	echo -n " - synack"
1375	with_cookie=$(ip netns exec $ns2 sysctl -n net.ipv4.tcp_syncookies)
1376	count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtMPJoinSynAckRx | awk '{print $2}')
1377	[ -z "$count" ] && count=0
1378	if [ "$count" != "$syn_ack_nr" ]; then
1379		# simult connections exceeding the limit with cookie enabled could go up to
1380		# synack validation as the conn limit can be enforced reliably only after
1381		# the subflow creation
1382		if [ "$with_cookie" = 2 ] && [ "$count" -gt "$syn_ack_nr" ] && [ "$count" -le "$syn_nr" ]; then
1383			echo -n "[ ok ]"
1384		else
1385			echo "[fail] got $count JOIN[s] synack expected $syn_ack_nr"
1386			fail_test
1387			dump_stats=1
1388		fi
1389	else
1390		echo -n "[ ok ]"
1391	fi
1392
1393	echo -n " - ack"
1394	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinAckRx | awk '{print $2}')
1395	[ -z "$count" ] && count=0
1396	if [ "$count" != "$ack_nr" ]; then
1397		echo "[fail] got $count JOIN[s] ack expected $ack_nr"
1398		fail_test
1399		dump_stats=1
1400	else
1401		echo "[ ok ]"
1402	fi
1403	[ "${dump_stats}" = 1 ] && dump_stats
1404	if [ $validate_checksum -eq 1 ]; then
1405		chk_csum_nr $csum_ns1 $csum_ns2
1406		chk_fail_nr $fail_nr $fail_nr
1407		chk_rst_nr $rst_nr $rst_nr
1408		chk_infi_nr $infi_nr $infi_nr
1409	fi
1410}
1411
1412# a negative value for 'stale_max' means no upper bound:
1413# for bidirectional transfer, if one peer sleep for a while
1414# - as these tests do - we can have a quite high number of
1415# stale/recover conversions, proportional to
1416# sleep duration/ MPTCP-level RTX interval.
1417chk_stale_nr()
1418{
1419	local ns=$1
1420	local stale_min=$2
1421	local stale_max=$3
1422	local stale_delta=$4
1423	local dump_stats
1424	local stale_nr
1425	local recover_nr
1426
1427	printf "%-${nr_blank}s %-18s" " " "stale"
1428	stale_nr=$(ip netns exec $ns nstat -as | grep MPTcpExtSubflowStale | awk '{print $2}')
1429	[ -z "$stale_nr" ] && stale_nr=0
1430	recover_nr=$(ip netns exec $ns nstat -as | grep MPTcpExtSubflowRecover | awk '{print $2}')
1431	[ -z "$recover_nr" ] && recover_nr=0
1432
1433	if [ $stale_nr -lt $stale_min ] ||
1434	   { [ $stale_max -gt 0 ] && [ $stale_nr -gt $stale_max ]; } ||
1435	   [ $((stale_nr - recover_nr)) -ne $stale_delta ]; then
1436		echo "[fail] got $stale_nr stale[s] $recover_nr recover[s], " \
1437		     " expected stale in range [$stale_min..$stale_max]," \
1438		     " stale-recover delta $stale_delta "
1439		fail_test
1440		dump_stats=1
1441	else
1442		echo "[ ok ]"
1443	fi
1444
1445	if [ "${dump_stats}" = 1 ]; then
1446		echo $ns stats
1447		ip netns exec $ns ip -s link show
1448		ip netns exec $ns nstat -as | grep MPTcp
1449	fi
1450}
1451
1452chk_add_nr()
1453{
1454	local add_nr=$1
1455	local echo_nr=$2
1456	local port_nr=${3:-0}
1457	local syn_nr=${4:-$port_nr}
1458	local syn_ack_nr=${5:-$port_nr}
1459	local ack_nr=${6:-$port_nr}
1460	local mis_syn_nr=${7:-0}
1461	local mis_ack_nr=${8:-0}
1462	local count
1463	local dump_stats
1464	local timeout
1465
1466	timeout=$(ip netns exec $ns1 sysctl -n net.mptcp.add_addr_timeout)
1467
1468	printf "%-${nr_blank}s %s" " " "add"
1469	count=$(ip netns exec $ns2 nstat -as MPTcpExtAddAddr | grep MPTcpExtAddAddr | awk '{print $2}')
1470	[ -z "$count" ] && count=0
1471
1472	# if the test configured a short timeout tolerate greater then expected
1473	# add addrs options, due to retransmissions
1474	if [ "$count" != "$add_nr" ] && { [ "$timeout" -gt 1 ] || [ "$count" -lt "$add_nr" ]; }; then
1475		echo "[fail] got $count ADD_ADDR[s] expected $add_nr"
1476		fail_test
1477		dump_stats=1
1478	else
1479		echo -n "[ ok ]"
1480	fi
1481
1482	echo -n " - echo  "
1483	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtEchoAdd | awk '{print $2}')
1484	[ -z "$count" ] && count=0
1485	if [ "$count" != "$echo_nr" ]; then
1486		echo "[fail] got $count ADD_ADDR echo[s] expected $echo_nr"
1487		fail_test
1488		dump_stats=1
1489	else
1490		echo -n "[ ok ]"
1491	fi
1492
1493	if [ $port_nr -gt 0 ]; then
1494		echo -n " - pt "
1495		count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtPortAdd | awk '{print $2}')
1496		[ -z "$count" ] && count=0
1497		if [ "$count" != "$port_nr" ]; then
1498			echo "[fail] got $count ADD_ADDR[s] with a port-number expected $port_nr"
1499			fail_test
1500			dump_stats=1
1501		else
1502			echo "[ ok ]"
1503		fi
1504
1505		printf "%-${nr_blank}s %s" " " "syn"
1506		count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinPortSynRx |
1507			awk '{print $2}')
1508		[ -z "$count" ] && count=0
1509		if [ "$count" != "$syn_nr" ]; then
1510			echo "[fail] got $count JOIN[s] syn with a different \
1511				port-number expected $syn_nr"
1512			fail_test
1513			dump_stats=1
1514		else
1515			echo -n "[ ok ]"
1516		fi
1517
1518		echo -n " - synack"
1519		count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtMPJoinPortSynAckRx |
1520			awk '{print $2}')
1521		[ -z "$count" ] && count=0
1522		if [ "$count" != "$syn_ack_nr" ]; then
1523			echo "[fail] got $count JOIN[s] synack with a different \
1524				port-number expected $syn_ack_nr"
1525			fail_test
1526			dump_stats=1
1527		else
1528			echo -n "[ ok ]"
1529		fi
1530
1531		echo -n " - ack"
1532		count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinPortAckRx |
1533			awk '{print $2}')
1534		[ -z "$count" ] && count=0
1535		if [ "$count" != "$ack_nr" ]; then
1536			echo "[fail] got $count JOIN[s] ack with a different \
1537				port-number expected $ack_nr"
1538			fail_test
1539			dump_stats=1
1540		else
1541			echo "[ ok ]"
1542		fi
1543
1544		printf "%-${nr_blank}s %s" " " "syn"
1545		count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMismatchPortSynRx |
1546			awk '{print $2}')
1547		[ -z "$count" ] && count=0
1548		if [ "$count" != "$mis_syn_nr" ]; then
1549			echo "[fail] got $count JOIN[s] syn with a mismatched \
1550				port-number expected $mis_syn_nr"
1551			fail_test
1552			dump_stats=1
1553		else
1554			echo -n "[ ok ]"
1555		fi
1556
1557		echo -n " - ack   "
1558		count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMismatchPortAckRx |
1559			awk '{print $2}')
1560		[ -z "$count" ] && count=0
1561		if [ "$count" != "$mis_ack_nr" ]; then
1562			echo "[fail] got $count JOIN[s] ack with a mismatched \
1563				port-number expected $mis_ack_nr"
1564			fail_test
1565			dump_stats=1
1566		else
1567			echo "[ ok ]"
1568		fi
1569	else
1570		echo ""
1571	fi
1572
1573	[ "${dump_stats}" = 1 ] && dump_stats
1574}
1575
1576chk_rm_nr()
1577{
1578	local rm_addr_nr=$1
1579	local rm_subflow_nr=$2
1580	local invert
1581	local simult
1582	local count
1583	local dump_stats
1584	local addr_ns=$ns1
1585	local subflow_ns=$ns2
1586	local extra_msg=""
1587
1588	shift 2
1589	while [ -n "$1" ]; do
1590		[ "$1" = "invert" ] && invert=true
1591		[ "$1" = "simult" ] && simult=true
1592		shift
1593	done
1594
1595	if [ -z $invert ]; then
1596		addr_ns=$ns1
1597		subflow_ns=$ns2
1598	elif [ $invert = "true" ]; then
1599		addr_ns=$ns2
1600		subflow_ns=$ns1
1601		extra_msg="   invert"
1602	fi
1603
1604	printf "%-${nr_blank}s %s" " " "rm "
1605	count=$(ip netns exec $addr_ns nstat -as | grep MPTcpExtRmAddr | awk '{print $2}')
1606	[ -z "$count" ] && count=0
1607	if [ "$count" != "$rm_addr_nr" ]; then
1608		echo "[fail] got $count RM_ADDR[s] expected $rm_addr_nr"
1609		fail_test
1610		dump_stats=1
1611	else
1612		echo -n "[ ok ]"
1613	fi
1614
1615	echo -n " - rmsf  "
1616	count=$(ip netns exec $subflow_ns nstat -as | grep MPTcpExtRmSubflow | awk '{print $2}')
1617	[ -z "$count" ] && count=0
1618	if [ -n "$simult" ]; then
1619		local cnt suffix
1620
1621		cnt=$(ip netns exec $addr_ns nstat -as | grep MPTcpExtRmSubflow | awk '{print $2}')
1622
1623		# in case of simult flush, the subflow removal count on each side is
1624		# unreliable
1625		[ -z "$cnt" ] && cnt=0
1626		count=$((count + cnt))
1627		[ "$count" != "$rm_subflow_nr" ] && suffix="$count in [$rm_subflow_nr:$((rm_subflow_nr*2))]"
1628		if [ $count -ge "$rm_subflow_nr" ] && \
1629		   [ "$count" -le "$((rm_subflow_nr *2 ))" ]; then
1630			echo "[ ok ] $suffix"
1631		else
1632			echo "[fail] got $count RM_SUBFLOW[s] expected in range [$rm_subflow_nr:$((rm_subflow_nr*2))]"
1633			fail_test
1634			dump_stats=1
1635		fi
1636		return
1637	fi
1638	if [ "$count" != "$rm_subflow_nr" ]; then
1639		echo "[fail] got $count RM_SUBFLOW[s] expected $rm_subflow_nr"
1640		fail_test
1641		dump_stats=1
1642	else
1643		echo -n "[ ok ]"
1644	fi
1645
1646	[ "${dump_stats}" = 1 ] && dump_stats
1647
1648	echo "$extra_msg"
1649}
1650
1651chk_prio_nr()
1652{
1653	local mp_prio_nr_tx=$1
1654	local mp_prio_nr_rx=$2
1655	local count
1656	local dump_stats
1657
1658	printf "%-${nr_blank}s %s" " " "ptx"
1659	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPPrioTx | awk '{print $2}')
1660	[ -z "$count" ] && count=0
1661	if [ "$count" != "$mp_prio_nr_tx" ]; then
1662		echo "[fail] got $count MP_PRIO[s] TX expected $mp_prio_nr_tx"
1663		fail_test
1664		dump_stats=1
1665	else
1666		echo -n "[ ok ]"
1667	fi
1668
1669	echo -n " - prx   "
1670	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPPrioRx | awk '{print $2}')
1671	[ -z "$count" ] && count=0
1672	if [ "$count" != "$mp_prio_nr_rx" ]; then
1673		echo "[fail] got $count MP_PRIO[s] RX expected $mp_prio_nr_rx"
1674		fail_test
1675		dump_stats=1
1676	else
1677		echo "[ ok ]"
1678	fi
1679
1680	[ "${dump_stats}" = 1 ] && dump_stats
1681}
1682
1683chk_subflow_nr()
1684{
1685	local need_title="$1"
1686	local msg="$2"
1687	local subflow_nr=$3
1688	local cnt1
1689	local cnt2
1690
1691	if [ -n "${need_title}" ]; then
1692		printf "%03u %-36s %s" "${TEST_COUNT}" "${TEST_NAME}" "${msg}"
1693	else
1694		printf "%-${nr_blank}s %s" " " "${msg}"
1695	fi
1696
1697	cnt1=$(ss -N $ns1 -tOni | grep -c token)
1698	cnt2=$(ss -N $ns2 -tOni | grep -c token)
1699	if [ "$cnt1" != "$subflow_nr" -o "$cnt2" != "$subflow_nr" ]; then
1700		echo "[fail] got $cnt1:$cnt2 subflows expected $subflow_nr"
1701		fail_test
1702		dump_stats=1
1703	else
1704		echo "[ ok ]"
1705	fi
1706
1707	[ "${dump_stats}" = 1 ] && ( ss -N $ns1 -tOni ; ss -N $ns1 -tOni | grep token; ip -n $ns1 mptcp endpoint )
1708}
1709
1710chk_link_usage()
1711{
1712	local ns=$1
1713	local link=$2
1714	local out=$3
1715	local expected_rate=$4
1716
1717	local tx_link tx_total
1718	tx_link=$(ip netns exec $ns cat /sys/class/net/$link/statistics/tx_bytes)
1719	tx_total=$(stat --format=%s $out)
1720	local tx_rate=$((tx_link * 100 / tx_total))
1721	local tolerance=5
1722
1723	printf "%-${nr_blank}s %-18s" " " "link usage"
1724	if [ $tx_rate -lt $((expected_rate - tolerance)) ] || \
1725	   [ $tx_rate -gt $((expected_rate + tolerance)) ]; then
1726		echo "[fail] got $tx_rate% usage, expected $expected_rate%"
1727		fail_test
1728	else
1729		echo "[ ok ]"
1730	fi
1731}
1732
1733wait_attempt_fail()
1734{
1735	local timeout_ms=$((timeout_poll * 1000))
1736	local time=0
1737	local ns=$1
1738
1739	while [ $time -lt $timeout_ms ]; do
1740		local cnt
1741
1742		cnt=$(ip netns exec $ns nstat -as TcpAttemptFails | grep TcpAttemptFails | awk '{print $2}')
1743
1744		[ "$cnt" = 1 ] && return 1
1745		time=$((time + 100))
1746		sleep 0.1
1747	done
1748	return 1
1749}
1750
1751set_userspace_pm()
1752{
1753	local ns=$1
1754
1755	ip netns exec $ns sysctl -q net.mptcp.pm_type=1
1756}
1757
1758subflows_tests()
1759{
1760	if reset "no JOIN"; then
1761		run_tests $ns1 $ns2 10.0.1.1
1762		chk_join_nr 0 0 0
1763	fi
1764
1765	# subflow limited by client
1766	if reset "single subflow, limited by client"; then
1767		pm_nl_set_limits $ns1 0 0
1768		pm_nl_set_limits $ns2 0 0
1769		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1770		run_tests $ns1 $ns2 10.0.1.1
1771		chk_join_nr 0 0 0
1772	fi
1773
1774	# subflow limited by server
1775	if reset "single subflow, limited by server"; then
1776		pm_nl_set_limits $ns1 0 0
1777		pm_nl_set_limits $ns2 0 1
1778		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1779		run_tests $ns1 $ns2 10.0.1.1
1780		chk_join_nr 1 1 0
1781	fi
1782
1783	# subflow
1784	if reset "single subflow"; then
1785		pm_nl_set_limits $ns1 0 1
1786		pm_nl_set_limits $ns2 0 1
1787		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1788		run_tests $ns1 $ns2 10.0.1.1
1789		chk_join_nr 1 1 1
1790	fi
1791
1792	# multiple subflows
1793	if reset "multiple subflows"; then
1794		pm_nl_set_limits $ns1 0 2
1795		pm_nl_set_limits $ns2 0 2
1796		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1797		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1798		run_tests $ns1 $ns2 10.0.1.1
1799		chk_join_nr 2 2 2
1800	fi
1801
1802	# multiple subflows limited by server
1803	if reset "multiple subflows, limited by server"; then
1804		pm_nl_set_limits $ns1 0 1
1805		pm_nl_set_limits $ns2 0 2
1806		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1807		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1808		run_tests $ns1 $ns2 10.0.1.1
1809		chk_join_nr 2 2 1
1810	fi
1811
1812	# single subflow, dev
1813	if reset "single subflow, dev"; then
1814		pm_nl_set_limits $ns1 0 1
1815		pm_nl_set_limits $ns2 0 1
1816		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow dev ns2eth3
1817		run_tests $ns1 $ns2 10.0.1.1
1818		chk_join_nr 1 1 1
1819	fi
1820}
1821
1822subflows_error_tests()
1823{
1824	# If a single subflow is configured, and matches the MPC src
1825	# address, no additional subflow should be created
1826	if reset "no MPC reuse with single endpoint"; then
1827		pm_nl_set_limits $ns1 0 1
1828		pm_nl_set_limits $ns2 0 1
1829		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow
1830		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
1831		chk_join_nr 0 0 0
1832	fi
1833
1834	# multiple subflows, with subflow creation error
1835	if reset "multi subflows, with failing subflow"; then
1836		pm_nl_set_limits $ns1 0 2
1837		pm_nl_set_limits $ns2 0 2
1838		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1839		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1840		filter_tcp_from $ns1 10.0.3.2 REJECT
1841		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
1842		chk_join_nr 1 1 1
1843	fi
1844
1845	# multiple subflows, with subflow timeout on MPJ
1846	if reset "multi subflows, with subflow timeout"; then
1847		pm_nl_set_limits $ns1 0 2
1848		pm_nl_set_limits $ns2 0 2
1849		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1850		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1851		filter_tcp_from $ns1 10.0.3.2 DROP
1852		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
1853		chk_join_nr 1 1 1
1854	fi
1855
1856	# multiple subflows, check that the endpoint corresponding to
1857	# closed subflow (due to reset) is not reused if additional
1858	# subflows are added later
1859	if reset "multi subflows, fair usage on close"; then
1860		pm_nl_set_limits $ns1 0 1
1861		pm_nl_set_limits $ns2 0 1
1862		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1863		filter_tcp_from $ns1 10.0.3.2 REJECT
1864		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow &
1865
1866		# mpj subflow will be in TW after the reset
1867		wait_attempt_fail $ns2
1868		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1869		wait
1870
1871		# additional subflow could be created only if the PM select
1872		# the later endpoint, skipping the already used one
1873		chk_join_nr 1 1 1
1874	fi
1875}
1876
1877signal_address_tests()
1878{
1879	# add_address, unused
1880	if reset "unused signal address"; then
1881		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1882		run_tests $ns1 $ns2 10.0.1.1
1883		chk_join_nr 0 0 0
1884		chk_add_nr 1 1
1885	fi
1886
1887	# accept and use add_addr
1888	if reset "signal address"; then
1889		pm_nl_set_limits $ns1 0 1
1890		pm_nl_set_limits $ns2 1 1
1891		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1892		run_tests $ns1 $ns2 10.0.1.1
1893		chk_join_nr 1 1 1
1894		chk_add_nr 1 1
1895	fi
1896
1897	# accept and use add_addr with an additional subflow
1898	# note: signal address in server ns and local addresses in client ns must
1899	# belong to different subnets or one of the listed local address could be
1900	# used for 'add_addr' subflow
1901	if reset "subflow and signal"; then
1902		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1903		pm_nl_set_limits $ns1 0 2
1904		pm_nl_set_limits $ns2 1 2
1905		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1906		run_tests $ns1 $ns2 10.0.1.1
1907		chk_join_nr 2 2 2
1908		chk_add_nr 1 1
1909	fi
1910
1911	# accept and use add_addr with additional subflows
1912	if reset "multiple subflows and signal"; then
1913		pm_nl_set_limits $ns1 0 3
1914		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1915		pm_nl_set_limits $ns2 1 3
1916		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1917		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
1918		run_tests $ns1 $ns2 10.0.1.1
1919		chk_join_nr 3 3 3
1920		chk_add_nr 1 1
1921	fi
1922
1923	# signal addresses
1924	if reset "signal addresses"; then
1925		pm_nl_set_limits $ns1 3 3
1926		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1927		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
1928		pm_nl_add_endpoint $ns1 10.0.4.1 flags signal
1929		pm_nl_set_limits $ns2 3 3
1930		run_tests $ns1 $ns2 10.0.1.1
1931		chk_join_nr 3 3 3
1932		chk_add_nr 3 3
1933	fi
1934
1935	# signal invalid addresses
1936	if reset "signal invalid addresses"; then
1937		pm_nl_set_limits $ns1 3 3
1938		pm_nl_add_endpoint $ns1 10.0.12.1 flags signal
1939		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
1940		pm_nl_add_endpoint $ns1 10.0.14.1 flags signal
1941		pm_nl_set_limits $ns2 3 3
1942		run_tests $ns1 $ns2 10.0.1.1
1943		chk_join_nr 1 1 1
1944		chk_add_nr 3 3
1945	fi
1946
1947	# signal addresses race test
1948	if reset "signal addresses race test"; then
1949		pm_nl_set_limits $ns1 4 4
1950		pm_nl_set_limits $ns2 4 4
1951		pm_nl_add_endpoint $ns1 10.0.1.1 flags signal
1952		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1953		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
1954		pm_nl_add_endpoint $ns1 10.0.4.1 flags signal
1955		pm_nl_add_endpoint $ns2 10.0.1.2 flags signal
1956		pm_nl_add_endpoint $ns2 10.0.2.2 flags signal
1957		pm_nl_add_endpoint $ns2 10.0.3.2 flags signal
1958		pm_nl_add_endpoint $ns2 10.0.4.2 flags signal
1959
1960		# the peer could possibly miss some addr notification, allow retransmission
1961		ip netns exec $ns1 sysctl -q net.mptcp.add_addr_timeout=1
1962		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
1963		chk_join_nr 3 3 3
1964
1965		# the server will not signal the address terminating
1966		# the MPC subflow
1967		chk_add_nr 3 3
1968	fi
1969}
1970
1971link_failure_tests()
1972{
1973	# accept and use add_addr with additional subflows and link loss
1974	if reset "multiple flows, signal, link failure"; then
1975		# without any b/w limit each veth could spool the packets and get
1976		# them acked at xmit time, so that the corresponding subflow will
1977		# have almost always no outstanding pkts, the scheduler will pick
1978		# always the first subflow and we will have hard time testing
1979		# active backup and link switch-over.
1980		# Let's set some arbitrary (low) virtual link limits.
1981		init_shapers
1982		pm_nl_set_limits $ns1 0 3
1983		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
1984		pm_nl_set_limits $ns2 1 3
1985		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow
1986		pm_nl_add_endpoint $ns2 10.0.4.2 dev ns2eth4 flags subflow
1987		run_tests $ns1 $ns2 10.0.1.1 1
1988		chk_join_nr 3 3 3
1989		chk_add_nr 1 1
1990		chk_stale_nr $ns2 1 5 1
1991	fi
1992
1993	# accept and use add_addr with additional subflows and link loss
1994	# for bidirectional transfer
1995	if reset "multi flows, signal, bidi, link fail"; then
1996		init_shapers
1997		pm_nl_set_limits $ns1 0 3
1998		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
1999		pm_nl_set_limits $ns2 1 3
2000		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow
2001		pm_nl_add_endpoint $ns2 10.0.4.2 dev ns2eth4 flags subflow
2002		run_tests $ns1 $ns2 10.0.1.1 2
2003		chk_join_nr 3 3 3
2004		chk_add_nr 1 1
2005		chk_stale_nr $ns2 1 -1 1
2006	fi
2007
2008	# 2 subflows plus 1 backup subflow with a lossy link, backup
2009	# will never be used
2010	if reset "backup subflow unused, link failure"; then
2011		init_shapers
2012		pm_nl_set_limits $ns1 0 2
2013		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
2014		pm_nl_set_limits $ns2 1 2
2015		FAILING_LINKS="1"
2016		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow,backup
2017		run_tests $ns1 $ns2 10.0.1.1 1
2018		chk_join_nr 2 2 2
2019		chk_add_nr 1 1
2020		chk_link_usage $ns2 ns2eth3 $cinsent 0
2021	fi
2022
2023	# 2 lossy links after half transfer, backup will get half of
2024	# the traffic
2025	if reset "backup flow used, multi links fail"; then
2026		init_shapers
2027		pm_nl_set_limits $ns1 0 2
2028		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
2029		pm_nl_set_limits $ns2 1 2
2030		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow,backup
2031		FAILING_LINKS="1 2"
2032		run_tests $ns1 $ns2 10.0.1.1 1
2033		chk_join_nr 2 2 2
2034		chk_add_nr 1 1
2035		chk_stale_nr $ns2 2 4 2
2036		chk_link_usage $ns2 ns2eth3 $cinsent 50
2037	fi
2038
2039	# use a backup subflow with the first subflow on a lossy link
2040	# for bidirectional transfer
2041	if reset "backup flow used, bidi, link failure"; then
2042		init_shapers
2043		pm_nl_set_limits $ns1 0 2
2044		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
2045		pm_nl_set_limits $ns2 1 3
2046		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow,backup
2047		FAILING_LINKS="1 2"
2048		run_tests $ns1 $ns2 10.0.1.1 2
2049		chk_join_nr 2 2 2
2050		chk_add_nr 1 1
2051		chk_stale_nr $ns2 1 -1 2
2052		chk_link_usage $ns2 ns2eth3 $cinsent 50
2053	fi
2054}
2055
2056add_addr_timeout_tests()
2057{
2058	# add_addr timeout
2059	if reset_with_add_addr_timeout "signal address, ADD_ADDR timeout"; then
2060		pm_nl_set_limits $ns1 0 1
2061		pm_nl_set_limits $ns2 1 1
2062		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2063		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
2064		chk_join_nr 1 1 1
2065		chk_add_nr 4 0
2066	fi
2067
2068	# add_addr timeout IPv6
2069	if reset_with_add_addr_timeout "signal address, ADD_ADDR6 timeout" 6; then
2070		pm_nl_set_limits $ns1 0 1
2071		pm_nl_set_limits $ns2 1 1
2072		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2073		run_tests $ns1 $ns2 dead:beef:1::1 0 0 0 slow
2074		chk_join_nr 1 1 1
2075		chk_add_nr 4 0
2076	fi
2077
2078	# signal addresses timeout
2079	if reset_with_add_addr_timeout "signal addresses, ADD_ADDR timeout"; then
2080		pm_nl_set_limits $ns1 2 2
2081		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2082		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2083		pm_nl_set_limits $ns2 2 2
2084		run_tests $ns1 $ns2 10.0.1.1 0 0 0 speed_10
2085		chk_join_nr 2 2 2
2086		chk_add_nr 8 0
2087	fi
2088
2089	# signal invalid addresses timeout
2090	if reset_with_add_addr_timeout "invalid address, ADD_ADDR timeout"; then
2091		pm_nl_set_limits $ns1 2 2
2092		pm_nl_add_endpoint $ns1 10.0.12.1 flags signal
2093		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2094		pm_nl_set_limits $ns2 2 2
2095		run_tests $ns1 $ns2 10.0.1.1 0 0 0 speed_10
2096		chk_join_nr 1 1 1
2097		chk_add_nr 8 0
2098	fi
2099}
2100
2101remove_tests()
2102{
2103	# single subflow, remove
2104	if reset "remove single subflow"; then
2105		pm_nl_set_limits $ns1 0 1
2106		pm_nl_set_limits $ns2 0 1
2107		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2108		run_tests $ns1 $ns2 10.0.1.1 0 0 -1 slow
2109		chk_join_nr 1 1 1
2110		chk_rm_nr 1 1
2111	fi
2112
2113	# multiple subflows, remove
2114	if reset "remove multiple subflows"; then
2115		pm_nl_set_limits $ns1 0 2
2116		pm_nl_set_limits $ns2 0 2
2117		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
2118		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2119		run_tests $ns1 $ns2 10.0.1.1 0 0 -2 slow
2120		chk_join_nr 2 2 2
2121		chk_rm_nr 2 2
2122	fi
2123
2124	# single address, remove
2125	if reset "remove single address"; then
2126		pm_nl_set_limits $ns1 0 1
2127		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2128		pm_nl_set_limits $ns2 1 1
2129		run_tests $ns1 $ns2 10.0.1.1 0 -1 0 slow
2130		chk_join_nr 1 1 1
2131		chk_add_nr 1 1
2132		chk_rm_nr 1 1 invert
2133	fi
2134
2135	# subflow and signal, remove
2136	if reset "remove subflow and signal"; then
2137		pm_nl_set_limits $ns1 0 2
2138		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2139		pm_nl_set_limits $ns2 1 2
2140		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2141		run_tests $ns1 $ns2 10.0.1.1 0 -1 -1 slow
2142		chk_join_nr 2 2 2
2143		chk_add_nr 1 1
2144		chk_rm_nr 1 1
2145	fi
2146
2147	# subflows and signal, remove
2148	if reset "remove subflows and signal"; then
2149		pm_nl_set_limits $ns1 0 3
2150		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2151		pm_nl_set_limits $ns2 1 3
2152		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2153		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2154		run_tests $ns1 $ns2 10.0.1.1 0 -1 -2 speed_10
2155		chk_join_nr 3 3 3
2156		chk_add_nr 1 1
2157		chk_rm_nr 2 2
2158	fi
2159
2160	# addresses remove
2161	if reset "remove addresses"; then
2162		pm_nl_set_limits $ns1 3 3
2163		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal id 250
2164		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2165		pm_nl_add_endpoint $ns1 10.0.4.1 flags signal
2166		pm_nl_set_limits $ns2 3 3
2167		run_tests $ns1 $ns2 10.0.1.1 0 -3 0 speed_10
2168		chk_join_nr 3 3 3
2169		chk_add_nr 3 3
2170		chk_rm_nr 3 3 invert
2171	fi
2172
2173	# invalid addresses remove
2174	if reset "remove invalid addresses"; then
2175		pm_nl_set_limits $ns1 3 3
2176		pm_nl_add_endpoint $ns1 10.0.12.1 flags signal
2177		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2178		pm_nl_add_endpoint $ns1 10.0.14.1 flags signal
2179		pm_nl_set_limits $ns2 3 3
2180		run_tests $ns1 $ns2 10.0.1.1 0 -3 0 speed_10
2181		chk_join_nr 1 1 1
2182		chk_add_nr 3 3
2183		chk_rm_nr 3 1 invert
2184	fi
2185
2186	# subflows and signal, flush
2187	if reset "flush subflows and signal"; then
2188		pm_nl_set_limits $ns1 0 3
2189		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2190		pm_nl_set_limits $ns2 1 3
2191		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2192		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2193		run_tests $ns1 $ns2 10.0.1.1 0 -8 -8 slow
2194		chk_join_nr 3 3 3
2195		chk_add_nr 1 1
2196		chk_rm_nr 1 3 invert simult
2197	fi
2198
2199	# subflows flush
2200	if reset "flush subflows"; then
2201		pm_nl_set_limits $ns1 3 3
2202		pm_nl_set_limits $ns2 3 3
2203		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow id 150
2204		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2205		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2206		run_tests $ns1 $ns2 10.0.1.1 0 -8 -8 slow
2207		chk_join_nr 3 3 3
2208		chk_rm_nr 0 3 simult
2209	fi
2210
2211	# addresses flush
2212	if reset "flush addresses"; then
2213		pm_nl_set_limits $ns1 3 3
2214		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal id 250
2215		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2216		pm_nl_add_endpoint $ns1 10.0.4.1 flags signal
2217		pm_nl_set_limits $ns2 3 3
2218		run_tests $ns1 $ns2 10.0.1.1 0 -8 -8 slow
2219		chk_join_nr 3 3 3
2220		chk_add_nr 3 3
2221		chk_rm_nr 3 3 invert simult
2222	fi
2223
2224	# invalid addresses flush
2225	if reset "flush invalid addresses"; then
2226		pm_nl_set_limits $ns1 3 3
2227		pm_nl_add_endpoint $ns1 10.0.12.1 flags signal
2228		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2229		pm_nl_add_endpoint $ns1 10.0.14.1 flags signal
2230		pm_nl_set_limits $ns2 3 3
2231		run_tests $ns1 $ns2 10.0.1.1 0 -8 0 slow
2232		chk_join_nr 1 1 1
2233		chk_add_nr 3 3
2234		chk_rm_nr 3 1 invert
2235	fi
2236
2237	# remove id 0 subflow
2238	if reset "remove id 0 subflow"; then
2239		pm_nl_set_limits $ns1 0 1
2240		pm_nl_set_limits $ns2 0 1
2241		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2242		run_tests $ns1 $ns2 10.0.1.1 0 0 -9 slow
2243		chk_join_nr 1 1 1
2244		chk_rm_nr 1 1
2245	fi
2246
2247	# remove id 0 address
2248	if reset "remove id 0 address"; then
2249		pm_nl_set_limits $ns1 0 1
2250		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2251		pm_nl_set_limits $ns2 1 1
2252		run_tests $ns1 $ns2 10.0.1.1 0 -9 0 slow
2253		chk_join_nr 1 1 1
2254		chk_add_nr 1 1
2255		chk_rm_nr 1 1 invert
2256	fi
2257}
2258
2259add_tests()
2260{
2261	# add single subflow
2262	if reset "add single subflow"; then
2263		pm_nl_set_limits $ns1 0 1
2264		pm_nl_set_limits $ns2 0 1
2265		run_tests $ns1 $ns2 10.0.1.1 0 0 1 slow
2266		chk_join_nr 1 1 1
2267	fi
2268
2269	# add signal address
2270	if reset "add signal address"; then
2271		pm_nl_set_limits $ns1 0 1
2272		pm_nl_set_limits $ns2 1 1
2273		run_tests $ns1 $ns2 10.0.1.1 0 1 0 slow
2274		chk_join_nr 1 1 1
2275		chk_add_nr 1 1
2276	fi
2277
2278	# add multiple subflows
2279	if reset "add multiple subflows"; then
2280		pm_nl_set_limits $ns1 0 2
2281		pm_nl_set_limits $ns2 0 2
2282		run_tests $ns1 $ns2 10.0.1.1 0 0 2 slow
2283		chk_join_nr 2 2 2
2284	fi
2285
2286	# add multiple subflows IPv6
2287	if reset "add multiple subflows IPv6"; then
2288		pm_nl_set_limits $ns1 0 2
2289		pm_nl_set_limits $ns2 0 2
2290		run_tests $ns1 $ns2 dead:beef:1::1 0 0 2 slow
2291		chk_join_nr 2 2 2
2292	fi
2293
2294	# add multiple addresses IPv6
2295	if reset "add multiple addresses IPv6"; then
2296		pm_nl_set_limits $ns1 0 2
2297		pm_nl_set_limits $ns2 2 2
2298		run_tests $ns1 $ns2 dead:beef:1::1 0 2 0 slow
2299		chk_join_nr 2 2 2
2300		chk_add_nr 2 2
2301	fi
2302}
2303
2304ipv6_tests()
2305{
2306	# subflow IPv6
2307	if reset "single subflow IPv6"; then
2308		pm_nl_set_limits $ns1 0 1
2309		pm_nl_set_limits $ns2 0 1
2310		pm_nl_add_endpoint $ns2 dead:beef:3::2 dev ns2eth3 flags subflow
2311		run_tests $ns1 $ns2 dead:beef:1::1 0 0 0 slow
2312		chk_join_nr 1 1 1
2313	fi
2314
2315	# add_address, unused IPv6
2316	if reset "unused signal address IPv6"; then
2317		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2318		run_tests $ns1 $ns2 dead:beef:1::1 0 0 0 slow
2319		chk_join_nr 0 0 0
2320		chk_add_nr 1 1
2321	fi
2322
2323	# signal address IPv6
2324	if reset "single address IPv6"; then
2325		pm_nl_set_limits $ns1 0 1
2326		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2327		pm_nl_set_limits $ns2 1 1
2328		run_tests $ns1 $ns2 dead:beef:1::1 0 0 0 slow
2329		chk_join_nr 1 1 1
2330		chk_add_nr 1 1
2331	fi
2332
2333	# single address IPv6, remove
2334	if reset "remove single address IPv6"; then
2335		pm_nl_set_limits $ns1 0 1
2336		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2337		pm_nl_set_limits $ns2 1 1
2338		run_tests $ns1 $ns2 dead:beef:1::1 0 -1 0 slow
2339		chk_join_nr 1 1 1
2340		chk_add_nr 1 1
2341		chk_rm_nr 1 1 invert
2342	fi
2343
2344	# subflow and signal IPv6, remove
2345	if reset "remove subflow and signal IPv6"; then
2346		pm_nl_set_limits $ns1 0 2
2347		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2348		pm_nl_set_limits $ns2 1 2
2349		pm_nl_add_endpoint $ns2 dead:beef:3::2 dev ns2eth3 flags subflow
2350		run_tests $ns1 $ns2 dead:beef:1::1 0 -1 -1 slow
2351		chk_join_nr 2 2 2
2352		chk_add_nr 1 1
2353		chk_rm_nr 1 1
2354	fi
2355}
2356
2357v4mapped_tests()
2358{
2359	# subflow IPv4-mapped to IPv4-mapped
2360	if reset "single subflow IPv4-mapped"; then
2361		pm_nl_set_limits $ns1 0 1
2362		pm_nl_set_limits $ns2 0 1
2363		pm_nl_add_endpoint $ns2 "::ffff:10.0.3.2" flags subflow
2364		run_tests $ns1 $ns2 "::ffff:10.0.1.1"
2365		chk_join_nr 1 1 1
2366	fi
2367
2368	# signal address IPv4-mapped with IPv4-mapped sk
2369	if reset "signal address IPv4-mapped"; then
2370		pm_nl_set_limits $ns1 0 1
2371		pm_nl_set_limits $ns2 1 1
2372		pm_nl_add_endpoint $ns1 "::ffff:10.0.2.1" flags signal
2373		run_tests $ns1 $ns2 "::ffff:10.0.1.1"
2374		chk_join_nr 1 1 1
2375		chk_add_nr 1 1
2376	fi
2377
2378	# subflow v4-map-v6
2379	if reset "single subflow v4-map-v6"; then
2380		pm_nl_set_limits $ns1 0 1
2381		pm_nl_set_limits $ns2 0 1
2382		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2383		run_tests $ns1 $ns2 "::ffff:10.0.1.1"
2384		chk_join_nr 1 1 1
2385	fi
2386
2387	# signal address v4-map-v6
2388	if reset "signal address v4-map-v6"; then
2389		pm_nl_set_limits $ns1 0 1
2390		pm_nl_set_limits $ns2 1 1
2391		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2392		run_tests $ns1 $ns2 "::ffff:10.0.1.1"
2393		chk_join_nr 1 1 1
2394		chk_add_nr 1 1
2395	fi
2396
2397	# subflow v6-map-v4
2398	if reset "single subflow v6-map-v4"; then
2399		pm_nl_set_limits $ns1 0 1
2400		pm_nl_set_limits $ns2 0 1
2401		pm_nl_add_endpoint $ns2 "::ffff:10.0.3.2" flags subflow
2402		run_tests $ns1 $ns2 10.0.1.1
2403		chk_join_nr 1 1 1
2404	fi
2405
2406	# signal address v6-map-v4
2407	if reset "signal address v6-map-v4"; then
2408		pm_nl_set_limits $ns1 0 1
2409		pm_nl_set_limits $ns2 1 1
2410		pm_nl_add_endpoint $ns1 "::ffff:10.0.2.1" flags signal
2411		run_tests $ns1 $ns2 10.0.1.1
2412		chk_join_nr 1 1 1
2413		chk_add_nr 1 1
2414	fi
2415
2416	# no subflow IPv6 to v4 address
2417	if reset "no JOIN with diff families v4-v6"; then
2418		pm_nl_set_limits $ns1 0 1
2419		pm_nl_set_limits $ns2 0 1
2420		pm_nl_add_endpoint $ns2 dead:beef:2::2 flags subflow
2421		run_tests $ns1 $ns2 10.0.1.1
2422		chk_join_nr 0 0 0
2423	fi
2424
2425	# no subflow IPv6 to v4 address even if v6 has a valid v4 at the end
2426	if reset "no JOIN with diff families v4-v6-2"; then
2427		pm_nl_set_limits $ns1 0 1
2428		pm_nl_set_limits $ns2 0 1
2429		pm_nl_add_endpoint $ns2 dead:beef:2::10.0.3.2 flags subflow
2430		run_tests $ns1 $ns2 10.0.1.1
2431		chk_join_nr 0 0 0
2432	fi
2433
2434	# no subflow IPv4 to v6 address, no need to slow down too then
2435	if reset "no JOIN with diff families v6-v4"; then
2436		pm_nl_set_limits $ns1 0 1
2437		pm_nl_set_limits $ns2 0 1
2438		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2439		run_tests $ns1 $ns2 dead:beef:1::1
2440		chk_join_nr 0 0 0
2441	fi
2442}
2443
2444mixed_tests()
2445{
2446	if reset "IPv4 sockets do not use IPv6 addresses"; then
2447		pm_nl_set_limits $ns1 0 1
2448		pm_nl_set_limits $ns2 1 1
2449		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2450		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
2451		chk_join_nr 0 0 0
2452	fi
2453
2454	# Need an IPv6 mptcp socket to allow subflows of both families
2455	if reset "simult IPv4 and IPv6 subflows"; then
2456		pm_nl_set_limits $ns1 0 1
2457		pm_nl_set_limits $ns2 1 1
2458		pm_nl_add_endpoint $ns1 10.0.1.1 flags signal
2459		run_tests $ns1 $ns2 dead:beef:2::1 0 0 0 slow
2460		chk_join_nr 1 1 1
2461	fi
2462
2463	# cross families subflows will not be created even in fullmesh mode
2464	if reset "simult IPv4 and IPv6 subflows, fullmesh 1x1"; then
2465		pm_nl_set_limits $ns1 0 4
2466		pm_nl_set_limits $ns2 1 4
2467		pm_nl_add_endpoint $ns2 dead:beef:2::2 flags subflow,fullmesh
2468		pm_nl_add_endpoint $ns1 10.0.1.1 flags signal
2469		run_tests $ns1 $ns2 dead:beef:2::1 0 0 0 slow
2470		chk_join_nr 1 1 1
2471	fi
2472
2473	# fullmesh still tries to create all the possibly subflows with
2474	# matching family
2475	if reset "simult IPv4 and IPv6 subflows, fullmesh 2x2"; then
2476		pm_nl_set_limits $ns1 0 4
2477		pm_nl_set_limits $ns2 2 4
2478		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2479		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2480		run_tests $ns1 $ns2 dead:beef:1::1 0 0 fullmesh_1 slow
2481		chk_join_nr 4 4 4
2482	fi
2483}
2484
2485backup_tests()
2486{
2487	# single subflow, backup
2488	if reset "single subflow, backup"; then
2489		pm_nl_set_limits $ns1 0 1
2490		pm_nl_set_limits $ns2 0 1
2491		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow,backup
2492		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow nobackup
2493		chk_join_nr 1 1 1
2494		chk_prio_nr 0 1
2495	fi
2496
2497	# single address, backup
2498	if reset "single address, backup"; then
2499		pm_nl_set_limits $ns1 0 1
2500		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2501		pm_nl_set_limits $ns2 1 1
2502		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
2503		chk_join_nr 1 1 1
2504		chk_add_nr 1 1
2505		chk_prio_nr 1 1
2506	fi
2507
2508	# single address with port, backup
2509	if reset "single address with port, backup"; then
2510		pm_nl_set_limits $ns1 0 1
2511		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2512		pm_nl_set_limits $ns2 1 1
2513		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
2514		chk_join_nr 1 1 1
2515		chk_add_nr 1 1
2516		chk_prio_nr 1 1
2517	fi
2518
2519	if reset "mpc backup"; then
2520		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow,backup
2521		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
2522		chk_join_nr 0 0 0
2523		chk_prio_nr 0 1
2524	fi
2525
2526	if reset "mpc backup both sides"; then
2527		pm_nl_add_endpoint $ns1 10.0.1.1 flags subflow,backup
2528		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow,backup
2529		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
2530		chk_join_nr 0 0 0
2531		chk_prio_nr 1 1
2532	fi
2533
2534	if reset "mpc switch to backup"; then
2535		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow
2536		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
2537		chk_join_nr 0 0 0
2538		chk_prio_nr 0 1
2539	fi
2540
2541	if reset "mpc switch to backup both sides"; then
2542		pm_nl_add_endpoint $ns1 10.0.1.1 flags subflow
2543		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow
2544		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
2545		chk_join_nr 0 0 0
2546		chk_prio_nr 1 1
2547	fi
2548}
2549
2550LISTENER_CREATED=15 #MPTCP_EVENT_LISTENER_CREATED
2551LISTENER_CLOSED=16  #MPTCP_EVENT_LISTENER_CLOSED
2552
2553AF_INET=2
2554AF_INET6=10
2555
2556verify_listener_events()
2557{
2558	local evt=$1
2559	local e_type=$2
2560	local e_family=$3
2561	local e_saddr=$4
2562	local e_sport=$5
2563	local type
2564	local family
2565	local saddr
2566	local sport
2567
2568	if [ $e_type = $LISTENER_CREATED ]; then
2569		stdbuf -o0 -e0 printf "\t\t\t\t\t CREATE_LISTENER %s:%s"\
2570			$e_saddr $e_sport
2571	elif [ $e_type = $LISTENER_CLOSED ]; then
2572		stdbuf -o0 -e0 printf "\t\t\t\t\t CLOSE_LISTENER %s:%s "\
2573			$e_saddr $e_sport
2574	fi
2575
2576	type=$(grep "type:$e_type," $evt |
2577	       sed --unbuffered -n 's/.*\(type:\)\([[:digit:]]*\).*$/\2/p;q')
2578	family=$(grep "type:$e_type," $evt |
2579		 sed --unbuffered -n 's/.*\(family:\)\([[:digit:]]*\).*$/\2/p;q')
2580	sport=$(grep "type:$e_type," $evt |
2581		sed --unbuffered -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q')
2582	if [ $family ] && [ $family = $AF_INET6 ]; then
2583		saddr=$(grep "type:$e_type," $evt |
2584			sed --unbuffered -n 's/.*\(saddr6:\)\([0-9a-f:.]*\).*$/\2/p;q')
2585	else
2586		saddr=$(grep "type:$e_type," $evt |
2587			sed --unbuffered -n 's/.*\(saddr4:\)\([0-9.]*\).*$/\2/p;q')
2588	fi
2589
2590	if [ $type ] && [ $type = $e_type ] &&
2591	   [ $family ] && [ $family = $e_family ] &&
2592	   [ $saddr ] && [ $saddr = $e_saddr ] &&
2593	   [ $sport ] && [ $sport = $e_sport ]; then
2594		stdbuf -o0 -e0 printf "[ ok ]\n"
2595		return 0
2596	fi
2597	fail_test
2598	stdbuf -o0 -e0 printf "[fail]\n"
2599}
2600
2601add_addr_ports_tests()
2602{
2603	# signal address with port
2604	if reset "signal address with port"; then
2605		pm_nl_set_limits $ns1 0 1
2606		pm_nl_set_limits $ns2 1 1
2607		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2608		run_tests $ns1 $ns2 10.0.1.1
2609		chk_join_nr 1 1 1
2610		chk_add_nr 1 1 1
2611	fi
2612
2613	# subflow and signal with port
2614	if reset "subflow and signal with port"; then
2615		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2616		pm_nl_set_limits $ns1 0 2
2617		pm_nl_set_limits $ns2 1 2
2618		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2619		run_tests $ns1 $ns2 10.0.1.1
2620		chk_join_nr 2 2 2
2621		chk_add_nr 1 1 1
2622	fi
2623
2624	# single address with port, remove
2625	# pm listener events
2626	if reset_with_events "remove single address with port"; then
2627		pm_nl_set_limits $ns1 0 1
2628		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2629		pm_nl_set_limits $ns2 1 1
2630		run_tests $ns1 $ns2 10.0.1.1 0 -1 0 slow
2631		chk_join_nr 1 1 1
2632		chk_add_nr 1 1 1
2633		chk_rm_nr 1 1 invert
2634
2635		verify_listener_events $evts_ns1 $LISTENER_CREATED $AF_INET 10.0.2.1 10100
2636		verify_listener_events $evts_ns1 $LISTENER_CLOSED $AF_INET 10.0.2.1 10100
2637		kill_events_pids
2638	fi
2639
2640	# subflow and signal with port, remove
2641	if reset "remove subflow and signal with port"; then
2642		pm_nl_set_limits $ns1 0 2
2643		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2644		pm_nl_set_limits $ns2 1 2
2645		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2646		run_tests $ns1 $ns2 10.0.1.1 0 -1 -1 slow
2647		chk_join_nr 2 2 2
2648		chk_add_nr 1 1 1
2649		chk_rm_nr 1 1
2650	fi
2651
2652	# subflows and signal with port, flush
2653	if reset "flush subflows and signal with port"; then
2654		pm_nl_set_limits $ns1 0 3
2655		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2656		pm_nl_set_limits $ns2 1 3
2657		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2658		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2659		run_tests $ns1 $ns2 10.0.1.1 0 -8 -2 slow
2660		chk_join_nr 3 3 3
2661		chk_add_nr 1 1
2662		chk_rm_nr 1 3 invert simult
2663	fi
2664
2665	# multiple addresses with port
2666	if reset "multiple addresses with port"; then
2667		pm_nl_set_limits $ns1 2 2
2668		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2669		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal port 10100
2670		pm_nl_set_limits $ns2 2 2
2671		run_tests $ns1 $ns2 10.0.1.1
2672		chk_join_nr 2 2 2
2673		chk_add_nr 2 2 2
2674	fi
2675
2676	# multiple addresses with ports
2677	if reset "multiple addresses with ports"; then
2678		pm_nl_set_limits $ns1 2 2
2679		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2680		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal port 10101
2681		pm_nl_set_limits $ns2 2 2
2682		run_tests $ns1 $ns2 10.0.1.1
2683		chk_join_nr 2 2 2
2684		chk_add_nr 2 2 2
2685	fi
2686}
2687
2688syncookies_tests()
2689{
2690	# single subflow, syncookies
2691	if reset_with_cookies "single subflow with syn cookies"; then
2692		pm_nl_set_limits $ns1 0 1
2693		pm_nl_set_limits $ns2 0 1
2694		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2695		run_tests $ns1 $ns2 10.0.1.1
2696		chk_join_nr 1 1 1
2697	fi
2698
2699	# multiple subflows with syn cookies
2700	if reset_with_cookies "multiple subflows with syn cookies"; then
2701		pm_nl_set_limits $ns1 0 2
2702		pm_nl_set_limits $ns2 0 2
2703		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2704		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
2705		run_tests $ns1 $ns2 10.0.1.1
2706		chk_join_nr 2 2 2
2707	fi
2708
2709	# multiple subflows limited by server
2710	if reset_with_cookies "subflows limited by server w cookies"; then
2711		pm_nl_set_limits $ns1 0 1
2712		pm_nl_set_limits $ns2 0 2
2713		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2714		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
2715		run_tests $ns1 $ns2 10.0.1.1
2716		chk_join_nr 2 1 1
2717	fi
2718
2719	# test signal address with cookies
2720	if reset_with_cookies "signal address with syn cookies"; then
2721		pm_nl_set_limits $ns1 0 1
2722		pm_nl_set_limits $ns2 1 1
2723		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2724		run_tests $ns1 $ns2 10.0.1.1
2725		chk_join_nr 1 1 1
2726		chk_add_nr 1 1
2727	fi
2728
2729	# test cookie with subflow and signal
2730	if reset_with_cookies "subflow and signal w cookies"; then
2731		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2732		pm_nl_set_limits $ns1 0 2
2733		pm_nl_set_limits $ns2 1 2
2734		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2735		run_tests $ns1 $ns2 10.0.1.1
2736		chk_join_nr 2 2 2
2737		chk_add_nr 1 1
2738	fi
2739
2740	# accept and use add_addr with additional subflows
2741	if reset_with_cookies "subflows and signal w. cookies"; then
2742		pm_nl_set_limits $ns1 0 3
2743		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2744		pm_nl_set_limits $ns2 1 3
2745		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2746		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2747		run_tests $ns1 $ns2 10.0.1.1
2748		chk_join_nr 3 3 3
2749		chk_add_nr 1 1
2750	fi
2751}
2752
2753checksum_tests()
2754{
2755	# checksum test 0 0
2756	if reset_with_checksum 0 0; then
2757		pm_nl_set_limits $ns1 0 1
2758		pm_nl_set_limits $ns2 0 1
2759		run_tests $ns1 $ns2 10.0.1.1
2760		chk_join_nr 0 0 0
2761	fi
2762
2763	# checksum test 1 1
2764	if reset_with_checksum 1 1; then
2765		pm_nl_set_limits $ns1 0 1
2766		pm_nl_set_limits $ns2 0 1
2767		run_tests $ns1 $ns2 10.0.1.1
2768		chk_join_nr 0 0 0
2769	fi
2770
2771	# checksum test 0 1
2772	if reset_with_checksum 0 1; then
2773		pm_nl_set_limits $ns1 0 1
2774		pm_nl_set_limits $ns2 0 1
2775		run_tests $ns1 $ns2 10.0.1.1
2776		chk_join_nr 0 0 0
2777	fi
2778
2779	# checksum test 1 0
2780	if reset_with_checksum 1 0; then
2781		pm_nl_set_limits $ns1 0 1
2782		pm_nl_set_limits $ns2 0 1
2783		run_tests $ns1 $ns2 10.0.1.1
2784		chk_join_nr 0 0 0
2785	fi
2786}
2787
2788deny_join_id0_tests()
2789{
2790	# subflow allow join id0 ns1
2791	if reset_with_allow_join_id0 "single subflow allow join id0 ns1" 1 0; then
2792		pm_nl_set_limits $ns1 1 1
2793		pm_nl_set_limits $ns2 1 1
2794		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2795		run_tests $ns1 $ns2 10.0.1.1
2796		chk_join_nr 1 1 1
2797	fi
2798
2799	# subflow allow join id0 ns2
2800	if reset_with_allow_join_id0 "single subflow allow join id0 ns2" 0 1; then
2801		pm_nl_set_limits $ns1 1 1
2802		pm_nl_set_limits $ns2 1 1
2803		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2804		run_tests $ns1 $ns2 10.0.1.1
2805		chk_join_nr 0 0 0
2806	fi
2807
2808	# signal address allow join id0 ns1
2809	# ADD_ADDRs are not affected by allow_join_id0 value.
2810	if reset_with_allow_join_id0 "signal address allow join id0 ns1" 1 0; then
2811		pm_nl_set_limits $ns1 1 1
2812		pm_nl_set_limits $ns2 1 1
2813		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2814		run_tests $ns1 $ns2 10.0.1.1
2815		chk_join_nr 1 1 1
2816		chk_add_nr 1 1
2817	fi
2818
2819	# signal address allow join id0 ns2
2820	# ADD_ADDRs are not affected by allow_join_id0 value.
2821	if reset_with_allow_join_id0 "signal address allow join id0 ns2" 0 1; then
2822		pm_nl_set_limits $ns1 1 1
2823		pm_nl_set_limits $ns2 1 1
2824		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2825		run_tests $ns1 $ns2 10.0.1.1
2826		chk_join_nr 1 1 1
2827		chk_add_nr 1 1
2828	fi
2829
2830	# subflow and address allow join id0 ns1
2831	if reset_with_allow_join_id0 "subflow and address allow join id0 1" 1 0; then
2832		pm_nl_set_limits $ns1 2 2
2833		pm_nl_set_limits $ns2 2 2
2834		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2835		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2836		run_tests $ns1 $ns2 10.0.1.1
2837		chk_join_nr 2 2 2
2838	fi
2839
2840	# subflow and address allow join id0 ns2
2841	if reset_with_allow_join_id0 "subflow and address allow join id0 2" 0 1; then
2842		pm_nl_set_limits $ns1 2 2
2843		pm_nl_set_limits $ns2 2 2
2844		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2845		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2846		run_tests $ns1 $ns2 10.0.1.1
2847		chk_join_nr 1 1 1
2848	fi
2849}
2850
2851fullmesh_tests()
2852{
2853	# fullmesh 1
2854	# 2 fullmesh addrs in ns2, added before the connection,
2855	# 1 non-fullmesh addr in ns1, added during the connection.
2856	if reset "fullmesh test 2x1"; then
2857		pm_nl_set_limits $ns1 0 4
2858		pm_nl_set_limits $ns2 1 4
2859		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow,fullmesh
2860		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow,fullmesh
2861		run_tests $ns1 $ns2 10.0.1.1 0 1 0 slow
2862		chk_join_nr 4 4 4
2863		chk_add_nr 1 1
2864	fi
2865
2866	# fullmesh 2
2867	# 1 non-fullmesh addr in ns1, added before the connection,
2868	# 1 fullmesh addr in ns2, added during the connection.
2869	if reset "fullmesh test 1x1"; then
2870		pm_nl_set_limits $ns1 1 3
2871		pm_nl_set_limits $ns2 1 3
2872		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2873		run_tests $ns1 $ns2 10.0.1.1 0 0 fullmesh_1 slow
2874		chk_join_nr 3 3 3
2875		chk_add_nr 1 1
2876	fi
2877
2878	# fullmesh 3
2879	# 1 non-fullmesh addr in ns1, added before the connection,
2880	# 2 fullmesh addrs in ns2, added during the connection.
2881	if reset "fullmesh test 1x2"; then
2882		pm_nl_set_limits $ns1 2 5
2883		pm_nl_set_limits $ns2 1 5
2884		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2885		run_tests $ns1 $ns2 10.0.1.1 0 0 fullmesh_2 slow
2886		chk_join_nr 5 5 5
2887		chk_add_nr 1 1
2888	fi
2889
2890	# fullmesh 4
2891	# 1 non-fullmesh addr in ns1, added before the connection,
2892	# 2 fullmesh addrs in ns2, added during the connection,
2893	# limit max_subflows to 4.
2894	if reset "fullmesh test 1x2, limited"; then
2895		pm_nl_set_limits $ns1 2 4
2896		pm_nl_set_limits $ns2 1 4
2897		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2898		run_tests $ns1 $ns2 10.0.1.1 0 0 fullmesh_2 slow
2899		chk_join_nr 4 4 4
2900		chk_add_nr 1 1
2901	fi
2902
2903	# set fullmesh flag
2904	if reset "set fullmesh flag test"; then
2905		pm_nl_set_limits $ns1 4 4
2906		pm_nl_add_endpoint $ns1 10.0.2.1 flags subflow
2907		pm_nl_set_limits $ns2 4 4
2908		run_tests $ns1 $ns2 10.0.1.1 0 0 1 slow fullmesh
2909		chk_join_nr 2 2 2
2910		chk_rm_nr 0 1
2911	fi
2912
2913	# set nofullmesh flag
2914	if reset "set nofullmesh flag test"; then
2915		pm_nl_set_limits $ns1 4 4
2916		pm_nl_add_endpoint $ns1 10.0.2.1 flags subflow,fullmesh
2917		pm_nl_set_limits $ns2 4 4
2918		run_tests $ns1 $ns2 10.0.1.1 0 0 fullmesh_1 slow nofullmesh
2919		chk_join_nr 2 2 2
2920		chk_rm_nr 0 1
2921	fi
2922
2923	# set backup,fullmesh flags
2924	if reset "set backup,fullmesh flags test"; then
2925		pm_nl_set_limits $ns1 4 4
2926		pm_nl_add_endpoint $ns1 10.0.2.1 flags subflow
2927		pm_nl_set_limits $ns2 4 4
2928		run_tests $ns1 $ns2 10.0.1.1 0 0 1 slow backup,fullmesh
2929		chk_join_nr 2 2 2
2930		chk_prio_nr 0 1
2931		chk_rm_nr 0 1
2932	fi
2933
2934	# set nobackup,nofullmesh flags
2935	if reset "set nobackup,nofullmesh flags test"; then
2936		pm_nl_set_limits $ns1 4 4
2937		pm_nl_set_limits $ns2 4 4
2938		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow,backup,fullmesh
2939		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow nobackup,nofullmesh
2940		chk_join_nr 2 2 2
2941		chk_prio_nr 0 1
2942		chk_rm_nr 0 1
2943	fi
2944}
2945
2946fastclose_tests()
2947{
2948	if reset "fastclose test"; then
2949		run_tests $ns1 $ns2 10.0.1.1 1024 0 fastclose_client
2950		chk_join_nr 0 0 0
2951		chk_fclose_nr 1 1
2952		chk_rst_nr 1 1 invert
2953	fi
2954
2955	if reset "fastclose server test"; then
2956		run_tests $ns1 $ns2 10.0.1.1 1024 0 fastclose_server
2957		chk_join_nr 0 0 0
2958		chk_fclose_nr 1 1 invert
2959		chk_rst_nr 1 1
2960	fi
2961}
2962
2963pedit_action_pkts()
2964{
2965	tc -n $ns2 -j -s action show action pedit index 100 | \
2966		grep "packets" | \
2967		sed 's/.*"packets":\([0-9]\+\),.*/\1/'
2968}
2969
2970fail_tests()
2971{
2972	# single subflow
2973	if reset_with_fail "Infinite map" 1; then
2974		run_tests $ns1 $ns2 10.0.1.1 128
2975		chk_join_nr 0 0 0 +1 +0 1 0 1 "$(pedit_action_pkts)"
2976		chk_fail_nr 1 -1 invert
2977	fi
2978
2979	# multiple subflows
2980	if reset_with_fail "MP_FAIL MP_RST" 2; then
2981		tc -n $ns2 qdisc add dev ns2eth1 root netem rate 1mbit delay 5
2982		pm_nl_set_limits $ns1 0 1
2983		pm_nl_set_limits $ns2 0 1
2984		pm_nl_add_endpoint $ns2 10.0.2.2 dev ns2eth2 flags subflow
2985		run_tests $ns1 $ns2 10.0.1.1 1024
2986		chk_join_nr 1 1 1 1 0 1 1 0 "$(pedit_action_pkts)"
2987	fi
2988}
2989
2990userspace_tests()
2991{
2992	# userspace pm type prevents add_addr
2993	if reset "userspace pm type prevents add_addr"; then
2994		set_userspace_pm $ns1
2995		pm_nl_set_limits $ns1 0 2
2996		pm_nl_set_limits $ns2 0 2
2997		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2998		run_tests $ns1 $ns2 10.0.1.1
2999		chk_join_nr 0 0 0
3000		chk_add_nr 0 0
3001	fi
3002
3003	# userspace pm type does not echo add_addr without daemon
3004	if reset "userspace pm no echo w/o daemon"; then
3005		set_userspace_pm $ns2
3006		pm_nl_set_limits $ns1 0 2
3007		pm_nl_set_limits $ns2 0 2
3008		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
3009		run_tests $ns1 $ns2 10.0.1.1
3010		chk_join_nr 0 0 0
3011		chk_add_nr 1 0
3012	fi
3013
3014	# userspace pm type rejects join
3015	if reset "userspace pm type rejects join"; then
3016		set_userspace_pm $ns1
3017		pm_nl_set_limits $ns1 1 1
3018		pm_nl_set_limits $ns2 1 1
3019		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
3020		run_tests $ns1 $ns2 10.0.1.1
3021		chk_join_nr 1 1 0
3022	fi
3023
3024	# userspace pm type does not send join
3025	if reset "userspace pm type does not send join"; then
3026		set_userspace_pm $ns2
3027		pm_nl_set_limits $ns1 1 1
3028		pm_nl_set_limits $ns2 1 1
3029		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
3030		run_tests $ns1 $ns2 10.0.1.1
3031		chk_join_nr 0 0 0
3032	fi
3033
3034	# userspace pm type prevents mp_prio
3035	if reset "userspace pm type prevents mp_prio"; then
3036		set_userspace_pm $ns1
3037		pm_nl_set_limits $ns1 1 1
3038		pm_nl_set_limits $ns2 1 1
3039		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
3040		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
3041		chk_join_nr 1 1 0
3042		chk_prio_nr 0 0
3043	fi
3044
3045	# userspace pm type prevents rm_addr
3046	if reset "userspace pm type prevents rm_addr"; then
3047		set_userspace_pm $ns1
3048		set_userspace_pm $ns2
3049		pm_nl_set_limits $ns1 0 1
3050		pm_nl_set_limits $ns2 0 1
3051		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
3052		run_tests $ns1 $ns2 10.0.1.1 0 0 -1 slow
3053		chk_join_nr 0 0 0
3054		chk_rm_nr 0 0
3055	fi
3056
3057	# userspace pm add & remove address
3058	if reset_with_events "userspace pm add & remove address"; then
3059		set_userspace_pm $ns1
3060		pm_nl_set_limits $ns2 1 1
3061		run_tests $ns1 $ns2 10.0.1.1 0 userspace_1 0 slow
3062		chk_join_nr 1 1 1
3063		chk_add_nr 1 1
3064		chk_rm_nr 1 1 invert
3065		kill_events_pids
3066	fi
3067
3068	# userspace pm create destroy subflow
3069	if reset_with_events "userspace pm create destroy subflow"; then
3070		set_userspace_pm $ns2
3071		pm_nl_set_limits $ns1 0 1
3072		run_tests $ns1 $ns2 10.0.1.1 0 0 userspace_1 slow
3073		chk_join_nr 1 1 1
3074		chk_rm_nr 0 1
3075		kill_events_pids
3076	fi
3077}
3078
3079endpoint_tests()
3080{
3081	# userspace pm type prevents add_addr
3082	if reset "implicit EP"; then
3083		pm_nl_set_limits $ns1 2 2
3084		pm_nl_set_limits $ns2 2 2
3085		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
3086		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow &
3087
3088		wait_mpj $ns1
3089		pm_nl_check_endpoint 1 "creation" \
3090			$ns2 10.0.2.2 id 1 flags implicit
3091
3092		pm_nl_add_endpoint $ns2 10.0.2.2 id 33
3093		pm_nl_check_endpoint 0 "ID change is prevented" \
3094			$ns2 10.0.2.2 id 1 flags implicit
3095
3096		pm_nl_add_endpoint $ns2 10.0.2.2 flags signal
3097		pm_nl_check_endpoint 0 "modif is allowed" \
3098			$ns2 10.0.2.2 id 1 flags signal
3099		wait
3100	fi
3101
3102	if reset "delete and re-add"; then
3103		pm_nl_set_limits $ns1 1 1
3104		pm_nl_set_limits $ns2 1 1
3105		pm_nl_add_endpoint $ns2 10.0.2.2 id 2 dev ns2eth2 flags subflow
3106		run_tests $ns1 $ns2 10.0.1.1 4 0 0 slow &
3107
3108		wait_mpj $ns2
3109		pm_nl_del_endpoint $ns2 2 10.0.2.2
3110		sleep 0.5
3111		chk_subflow_nr needtitle "after delete" 1
3112
3113		pm_nl_add_endpoint $ns2 10.0.2.2 dev ns2eth2 flags subflow
3114		wait_mpj $ns2
3115		chk_subflow_nr "" "after re-add" 2
3116		wait
3117	fi
3118}
3119
3120# [$1: error message]
3121usage()
3122{
3123	if [ -n "${1}" ]; then
3124		echo "${1}"
3125		ret=1
3126	fi
3127
3128	echo "mptcp_join usage:"
3129
3130	local key
3131	for key in "${!all_tests[@]}"; do
3132		echo "  -${key} ${all_tests[${key}]}"
3133	done
3134
3135	echo "  -c capture pcap files"
3136	echo "  -C enable data checksum"
3137	echo "  -i use ip mptcp"
3138	echo "  -h help"
3139
3140	echo "[test ids|names]"
3141
3142	exit ${ret}
3143}
3144
3145
3146# Use a "simple" array to force an specific order we cannot have with an associative one
3147all_tests_sorted=(
3148	f@subflows_tests
3149	e@subflows_error_tests
3150	s@signal_address_tests
3151	l@link_failure_tests
3152	t@add_addr_timeout_tests
3153	r@remove_tests
3154	a@add_tests
3155	6@ipv6_tests
3156	4@v4mapped_tests
3157	M@mixed_tests
3158	b@backup_tests
3159	p@add_addr_ports_tests
3160	k@syncookies_tests
3161	S@checksum_tests
3162	d@deny_join_id0_tests
3163	m@fullmesh_tests
3164	z@fastclose_tests
3165	F@fail_tests
3166	u@userspace_tests
3167	I@endpoint_tests
3168)
3169
3170all_tests_args=""
3171all_tests_names=()
3172for subtests in "${all_tests_sorted[@]}"; do
3173	key="${subtests%@*}"
3174	value="${subtests#*@}"
3175
3176	all_tests_args+="${key}"
3177	all_tests_names+=("${value}")
3178	all_tests[${key}]="${value}"
3179done
3180
3181tests=()
3182while getopts "${all_tests_args}cCih" opt; do
3183	case $opt in
3184		["${all_tests_args}"])
3185			tests+=("${all_tests[${opt}]}")
3186			;;
3187		c)
3188			capture=1
3189			;;
3190		C)
3191			checksum=1
3192			;;
3193		i)
3194			ip_mptcp=1
3195			;;
3196		h)
3197			usage
3198			;;
3199		*)
3200			usage "Unknown option: -${opt}"
3201			;;
3202	esac
3203done
3204
3205shift $((OPTIND - 1))
3206
3207for arg in "${@}"; do
3208	if [[ "${arg}" =~ ^[0-9]+$ ]]; then
3209		only_tests_ids+=("${arg}")
3210	else
3211		only_tests_names+=("${arg}")
3212	fi
3213done
3214
3215if [ ${#tests[@]} -eq 0 ]; then
3216	tests=("${all_tests_names[@]}")
3217fi
3218
3219for subtests in "${tests[@]}"; do
3220	"${subtests}"
3221done
3222
3223if [ ${ret} -ne 0 ]; then
3224	echo
3225	echo "${#failed_tests[@]} failure(s) has(ve) been detected:"
3226	for i in $(get_failed_tests_ids); do
3227		echo -e "\t- ${i}: ${failed_tests[${i}]}"
3228	done
3229	echo
3230fi
3231
3232exit $ret
3233