xref: /openbmc/linux/tools/testing/selftests/net/mptcp/mptcp_join.sh (revision 9df839a711aee437390b16ee39cf0b5c1620be6a)
1#!/bin/bash
2# SPDX-License-Identifier: GPL-2.0
3
4# Double quotes to prevent globbing and word splitting is recommended in new
5# code but we accept it, especially because there were too many before having
6# address all other issues detected by shellcheck.
7#shellcheck disable=SC2086
8
9ret=0
10sin=""
11sinfail=""
12sout=""
13cin=""
14cinfail=""
15cinsent=""
16cout=""
17capout=""
18ns1=""
19ns2=""
20ksft_skip=4
21timeout_poll=30
22timeout_test=$((timeout_poll * 2 + 1))
23capture=0
24checksum=0
25ip_mptcp=0
26check_invert=0
27validate_checksum=0
28init=0
29evts_ns1=""
30evts_ns2=""
31evts_ns1_pid=0
32evts_ns2_pid=0
33
34declare -A all_tests
35declare -a only_tests_ids
36declare -a only_tests_names
37declare -A failed_tests
38TEST_COUNT=0
39TEST_NAME=""
40nr_blank=40
41
42export FAILING_LINKS=""
43
44# generated using "nfbpf_compile '(ip && (ip[54] & 0xf0) == 0x30) ||
45#				  (ip6 && (ip6[74] & 0xf0) == 0x30)'"
46CBPF_MPTCP_SUBOPTION_ADD_ADDR="14,
47			       48 0 0 0,
48			       84 0 0 240,
49			       21 0 3 64,
50			       48 0 0 54,
51			       84 0 0 240,
52			       21 6 7 48,
53			       48 0 0 0,
54			       84 0 0 240,
55			       21 0 4 96,
56			       48 0 0 74,
57			       84 0 0 240,
58			       21 0 1 48,
59			       6 0 0 65535,
60			       6 0 0 0"
61
62init_partial()
63{
64	capout=$(mktemp)
65
66	local sec rndh
67	sec=$(date +%s)
68	rndh=$(printf %x $sec)-$(mktemp -u XXXXXX)
69
70	ns1="ns1-$rndh"
71	ns2="ns2-$rndh"
72
73	local netns
74	for netns in "$ns1" "$ns2"; do
75		ip netns add $netns || exit $ksft_skip
76		ip -net $netns link set lo up
77		ip netns exec $netns sysctl -q net.mptcp.enabled=1
78		ip netns exec $netns sysctl -q net.mptcp.pm_type=0
79		ip netns exec $netns sysctl -q net.ipv4.conf.all.rp_filter=0
80		ip netns exec $netns sysctl -q net.ipv4.conf.default.rp_filter=0
81		if [ $checksum -eq 1 ]; then
82			ip netns exec $netns sysctl -q net.mptcp.checksum_enabled=1
83		fi
84	done
85
86	check_invert=0
87	validate_checksum=$checksum
88	FAILING_LINKS=""
89
90	#  ns1         ns2
91	# ns1eth1    ns2eth1
92	# ns1eth2    ns2eth2
93	# ns1eth3    ns2eth3
94	# ns1eth4    ns2eth4
95
96	local i
97	for i in $(seq 1 4); do
98		ip link add ns1eth$i netns "$ns1" type veth peer name ns2eth$i netns "$ns2"
99		ip -net "$ns1" addr add 10.0.$i.1/24 dev ns1eth$i
100		ip -net "$ns1" addr add dead:beef:$i::1/64 dev ns1eth$i nodad
101		ip -net "$ns1" link set ns1eth$i up
102
103		ip -net "$ns2" addr add 10.0.$i.2/24 dev ns2eth$i
104		ip -net "$ns2" addr add dead:beef:$i::2/64 dev ns2eth$i nodad
105		ip -net "$ns2" link set ns2eth$i up
106
107		# let $ns2 reach any $ns1 address from any interface
108		ip -net "$ns2" route add default via 10.0.$i.1 dev ns2eth$i metric 10$i
109		ip -net "$ns2" route add default via dead:beef:$i::1 dev ns2eth$i metric 10$i
110	done
111}
112
113init_shapers()
114{
115	local i
116	for i in $(seq 1 4); do
117		tc -n $ns1 qdisc add dev ns1eth$i root netem rate 20mbit delay 1
118		tc -n $ns2 qdisc add dev ns2eth$i root netem rate 20mbit delay 1
119	done
120}
121
122cleanup_partial()
123{
124	rm -f "$capout"
125
126	local netns
127	for netns in "$ns1" "$ns2"; do
128		ip netns del $netns
129		rm -f /tmp/$netns.{nstat,out}
130	done
131}
132
133check_tools()
134{
135	if ! ip -Version &> /dev/null; then
136		echo "SKIP: Could not run test without ip tool"
137		exit $ksft_skip
138	fi
139
140	if ! iptables -V &> /dev/null; then
141		echo "SKIP: Could not run all tests without iptables tool"
142		exit $ksft_skip
143	fi
144
145	if ! ip6tables -V &> /dev/null; then
146		echo "SKIP: Could not run all tests without ip6tables tool"
147		exit $ksft_skip
148	fi
149}
150
151init() {
152	init=1
153
154	check_tools
155
156	sin=$(mktemp)
157	sout=$(mktemp)
158	cin=$(mktemp)
159	cinsent=$(mktemp)
160	cout=$(mktemp)
161	evts_ns1=$(mktemp)
162	evts_ns2=$(mktemp)
163
164	trap cleanup EXIT
165
166	make_file "$cin" "client" 1
167	make_file "$sin" "server" 1
168}
169
170cleanup()
171{
172	rm -f "$cin" "$cout" "$sinfail"
173	rm -f "$sin" "$sout" "$cinsent" "$cinfail"
174	rm -rf $evts_ns1 $evts_ns2
175	cleanup_partial
176}
177
178skip_test()
179{
180	if [ "${#only_tests_ids[@]}" -eq 0 ] && [ "${#only_tests_names[@]}" -eq 0 ]; then
181		return 1
182	fi
183
184	local i
185	for i in "${only_tests_ids[@]}"; do
186		if [ "${TEST_COUNT}" -eq "${i}" ]; then
187			return 1
188		fi
189	done
190	for i in "${only_tests_names[@]}"; do
191		if [ "${TEST_NAME}" = "${i}" ]; then
192			return 1
193		fi
194	done
195
196	return 0
197}
198
199# $1: test name
200reset()
201{
202	TEST_NAME="${1}"
203
204	TEST_COUNT=$((TEST_COUNT+1))
205
206	if skip_test; then
207		return 1
208	fi
209
210	if [ "${init}" != "1" ]; then
211		init
212	else
213		cleanup_partial
214	fi
215
216	init_partial
217
218	return 0
219}
220
221# $1: test name
222reset_with_cookies()
223{
224	reset "${1}" || return 1
225
226	local netns
227	for netns in "$ns1" "$ns2"; do
228		ip netns exec $netns sysctl -q net.ipv4.tcp_syncookies=2
229	done
230}
231
232# $1: test name
233reset_with_add_addr_timeout()
234{
235	local ip="${2:-4}"
236	local tables
237
238	reset "${1}" || return 1
239
240	tables="iptables"
241	if [ $ip -eq 6 ]; then
242		tables="ip6tables"
243	fi
244
245	ip netns exec $ns1 sysctl -q net.mptcp.add_addr_timeout=1
246	ip netns exec $ns2 $tables -A OUTPUT -p tcp \
247		-m tcp --tcp-option 30 \
248		-m bpf --bytecode \
249		"$CBPF_MPTCP_SUBOPTION_ADD_ADDR" \
250		-j DROP
251}
252
253# $1: test name
254reset_with_checksum()
255{
256	local ns1_enable=$1
257	local ns2_enable=$2
258
259	reset "checksum test ${1} ${2}" || return 1
260
261	ip netns exec $ns1 sysctl -q net.mptcp.checksum_enabled=$ns1_enable
262	ip netns exec $ns2 sysctl -q net.mptcp.checksum_enabled=$ns2_enable
263
264	validate_checksum=1
265}
266
267reset_with_allow_join_id0()
268{
269	local ns1_enable=$2
270	local ns2_enable=$3
271
272	reset "${1}" || return 1
273
274	ip netns exec $ns1 sysctl -q net.mptcp.allow_join_initial_addr_port=$ns1_enable
275	ip netns exec $ns2 sysctl -q net.mptcp.allow_join_initial_addr_port=$ns2_enable
276}
277
278# Modify TCP payload without corrupting the TCP packet
279#
280# This rule inverts a 8-bit word at byte offset 148 for the 2nd TCP ACK packets
281# carrying enough data.
282# Once it is done, the TCP Checksum field is updated so the packet is still
283# considered as valid at the TCP level.
284# Because the MPTCP checksum, covering the TCP options and data, has not been
285# updated, the modification will be detected and an MP_FAIL will be emitted:
286# what we want to validate here without corrupting "random" MPTCP options.
287#
288# To avoid having tc producing this pr_info() message for each TCP ACK packets
289# not carrying enough data:
290#
291#     tc action pedit offset 162 out of bounds
292#
293# Netfilter is used to mark packets with enough data.
294reset_with_fail()
295{
296	reset "${1}" || return 1
297
298	ip netns exec $ns1 sysctl -q net.mptcp.checksum_enabled=1
299	ip netns exec $ns2 sysctl -q net.mptcp.checksum_enabled=1
300
301	check_invert=1
302	validate_checksum=1
303	local i="$2"
304	local ip="${3:-4}"
305	local tables
306
307	tables="iptables"
308	if [ $ip -eq 6 ]; then
309		tables="ip6tables"
310	fi
311
312	ip netns exec $ns2 $tables \
313		-t mangle \
314		-A OUTPUT \
315		-o ns2eth$i \
316		-p tcp \
317		-m length --length 150:9999 \
318		-m statistic --mode nth --packet 1 --every 99999 \
319		-j MARK --set-mark 42 || exit 1
320
321	tc -n $ns2 qdisc add dev ns2eth$i clsact || exit 1
322	tc -n $ns2 filter add dev ns2eth$i egress \
323		protocol ip prio 1000 \
324		handle 42 fw \
325		action pedit munge offset 148 u8 invert \
326		pipe csum tcp \
327		index 100 || exit 1
328}
329
330reset_with_events()
331{
332	reset "${1}" || return 1
333
334	:> "$evts_ns1"
335	:> "$evts_ns2"
336	ip netns exec $ns1 ./pm_nl_ctl events >> "$evts_ns1" 2>&1 &
337	evts_ns1_pid=$!
338	ip netns exec $ns2 ./pm_nl_ctl events >> "$evts_ns2" 2>&1 &
339	evts_ns2_pid=$!
340}
341
342fail_test()
343{
344	ret=1
345	failed_tests[${TEST_COUNT}]="${TEST_NAME}"
346}
347
348get_failed_tests_ids()
349{
350	# sorted
351	local i
352	for i in "${!failed_tests[@]}"; do
353		echo "${i}"
354	done | sort -n
355}
356
357print_file_err()
358{
359	ls -l "$1" 1>&2
360	echo "Trailing bytes are: "
361	tail -c 27 "$1"
362}
363
364check_transfer()
365{
366	local in=$1
367	local out=$2
368	local what=$3
369	local bytes=$4
370	local i a b
371
372	local line
373	if [ -n "$bytes" ]; then
374		# when truncating we must check the size explicitly
375		local out_size=$(wc -c $out | awk '{print $1}')
376		if [ $out_size -ne $bytes ]; then
377			echo "[ FAIL ] $what output file has wrong size ($out_size, $bytes)"
378			fail_test
379			return 1
380		fi
381		bytes="--bytes=${bytes}"
382	fi
383	cmp -l "$in" "$out" ${bytes} | while read -r i a b; do
384		local sum=$((0${a} + 0${b}))
385		if [ $check_invert -eq 0 ] || [ $sum -ne $((0xff)) ]; then
386			echo "[ FAIL ] $what does not match (in, out):"
387			print_file_err "$in"
388			print_file_err "$out"
389			fail_test
390
391			return 1
392		else
393			echo "$what has inverted byte at ${i}"
394		fi
395	done
396
397	return 0
398}
399
400do_ping()
401{
402	local listener_ns="$1"
403	local connector_ns="$2"
404	local connect_addr="$3"
405
406	if ! ip netns exec ${connector_ns} ping -q -c 1 $connect_addr >/dev/null; then
407		echo "$listener_ns -> $connect_addr connectivity [ FAIL ]" 1>&2
408		fail_test
409	fi
410}
411
412link_failure()
413{
414	local ns="$1"
415
416	if [ -z "$FAILING_LINKS" ]; then
417		l=$((RANDOM%4))
418		FAILING_LINKS=$((l+1))
419	fi
420
421	local l
422	for l in $FAILING_LINKS; do
423		local veth="ns1eth$l"
424		ip -net "$ns" link set "$veth" down
425	done
426}
427
428# $1: IP address
429is_v6()
430{
431	[ -z "${1##*:*}" ]
432}
433
434# $1: ns, $2: port
435wait_local_port_listen()
436{
437	local listener_ns="${1}"
438	local port="${2}"
439
440	local port_hex
441	port_hex="$(printf "%04X" "${port}")"
442
443	local i
444	for i in $(seq 10); do
445		ip netns exec "${listener_ns}" cat /proc/net/tcp* | \
446			awk "BEGIN {rc=1} {if (\$2 ~ /:${port_hex}\$/ && \$4 ~ /0A/) {rc=0; exit}} END {exit rc}" &&
447			break
448		sleep 0.1
449	done
450}
451
452rm_addr_count()
453{
454	local ns=${1}
455
456	ip netns exec ${ns} nstat -as | grep MPTcpExtRmAddr | awk '{print $2}'
457}
458
459# $1: ns, $2: old rm_addr counter in $ns
460wait_rm_addr()
461{
462	local ns="${1}"
463	local old_cnt="${2}"
464	local cnt
465
466	local i
467	for i in $(seq 10); do
468		cnt=$(rm_addr_count ${ns})
469		[ "$cnt" = "${old_cnt}" ] || break
470		sleep 0.1
471	done
472}
473
474wait_mpj()
475{
476	local ns="${1}"
477	local cnt old_cnt
478
479	old_cnt=$(ip netns exec ${ns} nstat -as | grep MPJoinAckRx | awk '{print $2}')
480
481	local i
482	for i in $(seq 10); do
483		cnt=$(ip netns exec ${ns} nstat -as | grep MPJoinAckRx | awk '{print $2}')
484		[ "$cnt" = "${old_cnt}" ] || break
485		sleep 0.1
486	done
487}
488
489kill_wait()
490{
491	kill $1 > /dev/null 2>&1
492	wait $1 2>/dev/null
493}
494
495kill_events_pids()
496{
497	kill_wait $evts_ns1_pid
498	kill_wait $evts_ns2_pid
499}
500
501kill_tests_wait()
502{
503	kill -SIGUSR1 $(ip netns pids $ns2) $(ip netns pids $ns1)
504	wait
505}
506
507pm_nl_set_limits()
508{
509	local ns=$1
510	local addrs=$2
511	local subflows=$3
512
513	if [ $ip_mptcp -eq 1 ]; then
514		ip -n $ns mptcp limits set add_addr_accepted $addrs subflows $subflows
515	else
516		ip netns exec $ns ./pm_nl_ctl limits $addrs $subflows
517	fi
518}
519
520pm_nl_add_endpoint()
521{
522	local ns=$1
523	local addr=$2
524	local flags _flags
525	local port _port
526	local dev _dev
527	local id _id
528	local nr=2
529
530	local p
531	for p in "${@}"
532	do
533		if [ $p = "flags" ]; then
534			eval _flags=\$"$nr"
535			[ -n "$_flags" ]; flags="flags $_flags"
536		fi
537		if [ $p = "dev" ]; then
538			eval _dev=\$"$nr"
539			[ -n "$_dev" ]; dev="dev $_dev"
540		fi
541		if [ $p = "id" ]; then
542			eval _id=\$"$nr"
543			[ -n "$_id" ]; id="id $_id"
544		fi
545		if [ $p = "port" ]; then
546			eval _port=\$"$nr"
547			[ -n "$_port" ]; port="port $_port"
548		fi
549
550		nr=$((nr + 1))
551	done
552
553	if [ $ip_mptcp -eq 1 ]; then
554		ip -n $ns mptcp endpoint add $addr ${_flags//","/" "} $dev $id $port
555	else
556		ip netns exec $ns ./pm_nl_ctl add $addr $flags $dev $id $port
557	fi
558}
559
560pm_nl_del_endpoint()
561{
562	local ns=$1
563	local id=$2
564	local addr=$3
565
566	if [ $ip_mptcp -eq 1 ]; then
567		ip -n $ns mptcp endpoint delete id $id $addr
568	else
569		ip netns exec $ns ./pm_nl_ctl del $id $addr
570	fi
571}
572
573pm_nl_flush_endpoint()
574{
575	local ns=$1
576
577	if [ $ip_mptcp -eq 1 ]; then
578		ip -n $ns mptcp endpoint flush
579	else
580		ip netns exec $ns ./pm_nl_ctl flush
581	fi
582}
583
584pm_nl_show_endpoints()
585{
586	local ns=$1
587
588	if [ $ip_mptcp -eq 1 ]; then
589		ip -n $ns mptcp endpoint show
590	else
591		ip netns exec $ns ./pm_nl_ctl dump
592	fi
593}
594
595pm_nl_change_endpoint()
596{
597	local ns=$1
598	local id=$2
599	local flags=$3
600
601	if [ $ip_mptcp -eq 1 ]; then
602		ip -n $ns mptcp endpoint change id $id ${flags//","/" "}
603	else
604		ip netns exec $ns ./pm_nl_ctl set id $id flags $flags
605	fi
606}
607
608pm_nl_check_endpoint()
609{
610	local line expected_line
611	local need_title=$1
612	local msg="$2"
613	local ns=$3
614	local addr=$4
615	local _flags=""
616	local flags
617	local _port
618	local port
619	local dev
620	local _id
621	local id
622
623	if [ "${need_title}" = 1 ]; then
624		printf "%03u %-36s %s" "${TEST_COUNT}" "${TEST_NAME}" "${msg}"
625	else
626		printf "%-${nr_blank}s %s" " " "${msg}"
627	fi
628
629	shift 4
630	while [ -n "$1" ]; do
631		if [ $1 = "flags" ]; then
632			_flags=$2
633			[ -n "$_flags" ]; flags="flags $_flags"
634			shift
635		elif [ $1 = "dev" ]; then
636			[ -n "$2" ]; dev="dev $1"
637			shift
638		elif [ $1 = "id" ]; then
639			_id=$2
640			[ -n "$_id" ]; id="id $_id"
641			shift
642		elif [ $1 = "port" ]; then
643			_port=$2
644			[ -n "$_port" ]; port=" port $_port"
645			shift
646		fi
647
648		shift
649	done
650
651	if [ -z "$id" ]; then
652		echo "[skip] bad test - missing endpoint id"
653		return
654	fi
655
656	if [ $ip_mptcp -eq 1 ]; then
657		line=$(ip -n $ns mptcp endpoint show $id)
658		# the dump order is: address id flags port dev
659		expected_line="$addr"
660		[ -n "$addr" ] && expected_line="$expected_line $addr"
661		expected_line="$expected_line $id"
662		[ -n "$_flags" ] && expected_line="$expected_line ${_flags//","/" "}"
663		[ -n "$dev" ] && expected_line="$expected_line $dev"
664		[ -n "$port" ] && expected_line="$expected_line $port"
665	else
666		line=$(ip netns exec $ns ./pm_nl_ctl get $_id)
667		# the dump order is: id flags dev address port
668		expected_line="$id"
669		[ -n "$flags" ] && expected_line="$expected_line $flags"
670		[ -n "$dev" ] && expected_line="$expected_line $dev"
671		[ -n "$addr" ] && expected_line="$expected_line $addr"
672		[ -n "$_port" ] && expected_line="$expected_line $_port"
673	fi
674	if [ "$line" = "$expected_line" ]; then
675		echo "[ ok ]"
676	else
677		echo "[fail] expected '$expected_line' found '$line'"
678		fail_test
679	fi
680}
681
682filter_tcp_from()
683{
684	local ns="${1}"
685	local src="${2}"
686	local target="${3}"
687
688	ip netns exec "${ns}" iptables -A INPUT -s "${src}" -p tcp -j "${target}"
689}
690
691do_transfer()
692{
693	local listener_ns="$1"
694	local connector_ns="$2"
695	local cl_proto="$3"
696	local srv_proto="$4"
697	local connect_addr="$5"
698	local test_link_fail="$6"
699	local addr_nr_ns1="$7"
700	local addr_nr_ns2="$8"
701	local speed="$9"
702	local sflags="${10}"
703
704	local port=$((10000 + TEST_COUNT - 1))
705	local cappid
706	local userspace_pm=0
707
708	:> "$cout"
709	:> "$sout"
710	:> "$capout"
711
712	if [ $capture -eq 1 ]; then
713		local capuser
714		if [ -z $SUDO_USER ] ; then
715			capuser=""
716		else
717			capuser="-Z $SUDO_USER"
718		fi
719
720		capfile=$(printf "mp_join-%02u-%s.pcap" "$TEST_COUNT" "${listener_ns}")
721
722		echo "Capturing traffic for test $TEST_COUNT into $capfile"
723		ip netns exec ${listener_ns} tcpdump -i any -s 65535 -B 32768 $capuser -w $capfile > "$capout" 2>&1 &
724		cappid=$!
725
726		sleep 1
727	fi
728
729	NSTAT_HISTORY=/tmp/${listener_ns}.nstat ip netns exec ${listener_ns} \
730		nstat -n
731	NSTAT_HISTORY=/tmp/${connector_ns}.nstat ip netns exec ${connector_ns} \
732		nstat -n
733
734	local extra_args
735	if [ $speed = "fast" ]; then
736		extra_args="-j"
737	elif [ $speed = "slow" ]; then
738		extra_args="-r 50"
739	elif [[ $speed = "speed_"* ]]; then
740		extra_args="-r ${speed:6}"
741	fi
742
743	if [[ "${addr_nr_ns1}" = "userspace_"* ]]; then
744		userspace_pm=1
745		addr_nr_ns1=${addr_nr_ns1:10}
746	fi
747
748	local flags="subflow"
749	local extra_cl_args=""
750	local extra_srv_args=""
751	local trunc_size=""
752	if [[ "${addr_nr_ns2}" = "fastclose_"* ]]; then
753		if [ ${test_link_fail} -le 1 ]; then
754			echo "fastclose tests need test_link_fail argument"
755			fail_test
756			return 1
757		fi
758
759		# disconnect
760		trunc_size=${test_link_fail}
761		local side=${addr_nr_ns2:10}
762
763		if [ ${side} = "client" ]; then
764			extra_cl_args="-f ${test_link_fail}"
765			extra_srv_args="-f -1"
766		elif [ ${side} = "server" ]; then
767			extra_srv_args="-f ${test_link_fail}"
768			extra_cl_args="-f -1"
769		else
770			echo "wrong/unknown fastclose spec ${side}"
771			fail_test
772			return 1
773		fi
774		addr_nr_ns2=0
775	elif [[ "${addr_nr_ns2}" = "userspace_"* ]]; then
776		userspace_pm=1
777		addr_nr_ns2=${addr_nr_ns2:10}
778	elif [[ "${addr_nr_ns2}" = "fullmesh_"* ]]; then
779		flags="${flags},fullmesh"
780		addr_nr_ns2=${addr_nr_ns2:9}
781	fi
782
783	extra_srv_args="$extra_args $extra_srv_args"
784	if [ "$test_link_fail" -gt 1 ];then
785		timeout ${timeout_test} \
786			ip netns exec ${listener_ns} \
787				./mptcp_connect -t ${timeout_poll} -l -p $port -s ${srv_proto} \
788					$extra_srv_args "::" < "$sinfail" > "$sout" &
789	else
790		timeout ${timeout_test} \
791			ip netns exec ${listener_ns} \
792				./mptcp_connect -t ${timeout_poll} -l -p $port -s ${srv_proto} \
793					$extra_srv_args "::" < "$sin" > "$sout" &
794	fi
795	local spid=$!
796
797	wait_local_port_listen "${listener_ns}" "${port}"
798
799	extra_cl_args="$extra_args $extra_cl_args"
800	if [ "$test_link_fail" -eq 0 ];then
801		timeout ${timeout_test} \
802			ip netns exec ${connector_ns} \
803				./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \
804					$extra_cl_args $connect_addr < "$cin" > "$cout" &
805	elif [ "$test_link_fail" -eq 1 ] || [ "$test_link_fail" -eq 2 ];then
806		( cat "$cinfail" ; sleep 2; link_failure $listener_ns ; cat "$cinfail" ) | \
807			tee "$cinsent" | \
808			timeout ${timeout_test} \
809				ip netns exec ${connector_ns} \
810					./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \
811						$extra_cl_args $connect_addr > "$cout" &
812	else
813		tee "$cinsent" < "$cinfail" | \
814			timeout ${timeout_test} \
815				ip netns exec ${connector_ns} \
816					./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \
817						$extra_cl_args $connect_addr > "$cout" &
818	fi
819	local cpid=$!
820
821	# let the mptcp subflow be established in background before
822	# do endpoint manipulation
823	if [ $addr_nr_ns1 != "0" ] || [ $addr_nr_ns2 != "0" ]; then
824		sleep 1
825	fi
826
827	if [ $addr_nr_ns1 -gt 0 ]; then
828		local counter=2
829		local add_nr_ns1=${addr_nr_ns1}
830		local id=10
831		local tk
832		while [ $add_nr_ns1 -gt 0 ]; do
833			local addr
834			if is_v6 "${connect_addr}"; then
835				addr="dead:beef:$counter::1"
836			else
837				addr="10.0.$counter.1"
838			fi
839			if [ $userspace_pm -eq 0 ]; then
840				pm_nl_add_endpoint $ns1 $addr flags signal
841			else
842				tk=$(grep "type:1," "$evts_ns1" |
843				     sed -n 's/.*\(token:\)\([[:digit:]]*\).*$/\2/p;q')
844				ip netns exec ${listener_ns} ./pm_nl_ctl ann $addr token $tk id $id
845				sleep 1
846				ip netns exec ${listener_ns} ./pm_nl_ctl rem token $tk id $id
847			fi
848
849			counter=$((counter + 1))
850			add_nr_ns1=$((add_nr_ns1 - 1))
851			id=$((id + 1))
852		done
853	elif [ $addr_nr_ns1 -lt 0 ]; then
854		local rm_nr_ns1=$((-addr_nr_ns1))
855		if [ $rm_nr_ns1 -lt 8 ]; then
856			local counter=0
857			local line
858			pm_nl_show_endpoints ${listener_ns} | while read -r line; do
859				# shellcheck disable=SC2206 # we do want to split per word
860				local arr=($line)
861				local nr=0
862
863				local i
864				for i in "${arr[@]}"; do
865					if [ $i = "id" ]; then
866						if [ $counter -eq $rm_nr_ns1 ]; then
867							break
868						fi
869						id=${arr[$nr+1]}
870						rm_addr=$(rm_addr_count ${connector_ns})
871						pm_nl_del_endpoint ${listener_ns} $id
872						wait_rm_addr ${connector_ns} ${rm_addr}
873						counter=$((counter + 1))
874					fi
875					nr=$((nr + 1))
876				done
877			done
878		elif [ $rm_nr_ns1 -eq 8 ]; then
879			pm_nl_flush_endpoint ${listener_ns}
880		elif [ $rm_nr_ns1 -eq 9 ]; then
881			pm_nl_del_endpoint ${listener_ns} 0 ${connect_addr}
882		fi
883	fi
884
885	# if newly added endpoints must be deleted, give the background msk
886	# some time to created them
887	[ $addr_nr_ns1 -gt 0 ] && [ $addr_nr_ns2 -lt 0 ] && sleep 1
888
889	if [ $addr_nr_ns2 -gt 0 ]; then
890		local add_nr_ns2=${addr_nr_ns2}
891		local counter=3
892		local id=20
893		local tk da dp sp
894		while [ $add_nr_ns2 -gt 0 ]; do
895			local addr
896			if is_v6 "${connect_addr}"; then
897				addr="dead:beef:$counter::2"
898			else
899				addr="10.0.$counter.2"
900			fi
901			if [ $userspace_pm -eq 0 ]; then
902				pm_nl_add_endpoint $ns2 $addr flags $flags
903			else
904				tk=$(sed -n 's/.*\(token:\)\([[:digit:]]*\).*$/\2/p;q' "$evts_ns2")
905				da=$(sed -n 's/.*\(daddr4:\)\([0-9.]*\).*$/\2/p;q' "$evts_ns2")
906				dp=$(sed -n 's/.*\(dport:\)\([[:digit:]]*\).*$/\2/p;q' "$evts_ns2")
907				ip netns exec ${connector_ns} ./pm_nl_ctl csf lip $addr lid $id \
908									rip $da rport $dp token $tk
909				sleep 1
910				sp=$(grep "type:10" "$evts_ns2" |
911				     sed -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q')
912				ip netns exec ${connector_ns} ./pm_nl_ctl dsf lip $addr lport $sp \
913									rip $da rport $dp token $tk
914			fi
915			counter=$((counter + 1))
916			add_nr_ns2=$((add_nr_ns2 - 1))
917			id=$((id + 1))
918		done
919	elif [ $addr_nr_ns2 -lt 0 ]; then
920		local rm_nr_ns2=$((-addr_nr_ns2))
921		if [ $rm_nr_ns2 -lt 8 ]; then
922			local counter=0
923			local line
924			pm_nl_show_endpoints ${connector_ns} | while read -r line; do
925				# shellcheck disable=SC2206 # we do want to split per word
926				local arr=($line)
927				local nr=0
928
929				local i
930				for i in "${arr[@]}"; do
931					if [ $i = "id" ]; then
932						if [ $counter -eq $rm_nr_ns2 ]; then
933							break
934						fi
935						local id rm_addr
936						# rm_addr are serialized, allow the previous one to
937						# complete
938						id=${arr[$nr+1]}
939						rm_addr=$(rm_addr_count ${listener_ns})
940						pm_nl_del_endpoint ${connector_ns} $id
941						wait_rm_addr ${listener_ns} ${rm_addr}
942						counter=$((counter + 1))
943					fi
944					nr=$((nr + 1))
945				done
946			done
947		elif [ $rm_nr_ns2 -eq 8 ]; then
948			pm_nl_flush_endpoint ${connector_ns}
949		elif [ $rm_nr_ns2 -eq 9 ]; then
950			local addr
951			if is_v6 "${connect_addr}"; then
952				addr="dead:beef:1::2"
953			else
954				addr="10.0.1.2"
955			fi
956			pm_nl_del_endpoint ${connector_ns} 0 $addr
957		fi
958	fi
959
960	if [ -n "${sflags}" ]; then
961		sleep 1
962
963		local netns
964		for netns in "$ns1" "$ns2"; do
965			local line
966			pm_nl_show_endpoints $netns | while read -r line; do
967				# shellcheck disable=SC2206 # we do want to split per word
968				local arr=($line)
969				local nr=0
970				local id
971
972				local i
973				for i in "${arr[@]}"; do
974					if [ $i = "id" ]; then
975						id=${arr[$nr+1]}
976					fi
977					nr=$((nr + 1))
978				done
979				pm_nl_change_endpoint $netns $id $sflags
980			done
981		done
982	fi
983
984	wait $cpid
985	local retc=$?
986	wait $spid
987	local rets=$?
988
989	if [ $capture -eq 1 ]; then
990	    sleep 1
991	    kill $cappid
992	fi
993
994	NSTAT_HISTORY=/tmp/${listener_ns}.nstat ip netns exec ${listener_ns} \
995		nstat | grep Tcp > /tmp/${listener_ns}.out
996	NSTAT_HISTORY=/tmp/${connector_ns}.nstat ip netns exec ${connector_ns} \
997		nstat | grep Tcp > /tmp/${connector_ns}.out
998
999	if [ ${rets} -ne 0 ] || [ ${retc} -ne 0 ]; then
1000		echo " client exit code $retc, server $rets" 1>&2
1001		echo -e "\nnetns ${listener_ns} socket stat for ${port}:" 1>&2
1002		ip netns exec ${listener_ns} ss -Menita 1>&2 -o "sport = :$port"
1003		cat /tmp/${listener_ns}.out
1004		echo -e "\nnetns ${connector_ns} socket stat for ${port}:" 1>&2
1005		ip netns exec ${connector_ns} ss -Menita 1>&2 -o "dport = :$port"
1006		cat /tmp/${connector_ns}.out
1007
1008		cat "$capout"
1009		fail_test
1010		return 1
1011	fi
1012
1013	if [ "$test_link_fail" -gt 1 ];then
1014		check_transfer $sinfail $cout "file received by client" $trunc_size
1015	else
1016		check_transfer $sin $cout "file received by client" $trunc_size
1017	fi
1018	retc=$?
1019	if [ "$test_link_fail" -eq 0 ];then
1020		check_transfer $cin $sout "file received by server" $trunc_size
1021	else
1022		check_transfer $cinsent $sout "file received by server" $trunc_size
1023	fi
1024	rets=$?
1025
1026	if [ $retc -eq 0 ] && [ $rets -eq 0 ];then
1027		cat "$capout"
1028		return 0
1029	fi
1030
1031	cat "$capout"
1032	return 1
1033}
1034
1035make_file()
1036{
1037	local name=$1
1038	local who=$2
1039	local size=$3
1040
1041	dd if=/dev/urandom of="$name" bs=1024 count=$size 2> /dev/null
1042	echo -e "\nMPTCP_TEST_FILE_END_MARKER" >> "$name"
1043
1044	echo "Created $name (size $size KB) containing data sent by $who"
1045}
1046
1047run_tests()
1048{
1049	local listener_ns="$1"
1050	local connector_ns="$2"
1051	local connect_addr="$3"
1052	local test_linkfail="${4:-0}"
1053	local addr_nr_ns1="${5:-0}"
1054	local addr_nr_ns2="${6:-0}"
1055	local speed="${7:-fast}"
1056	local sflags="${8:-""}"
1057
1058	local size
1059
1060	# The values above 2 are reused to make test files
1061	# with the given sizes (KB)
1062	if [ "$test_linkfail" -gt 2 ]; then
1063		size=$test_linkfail
1064
1065		if [ -z "$cinfail" ]; then
1066			cinfail=$(mktemp)
1067		fi
1068		make_file "$cinfail" "client" $size
1069	# create the input file for the failure test when
1070	# the first failure test run
1071	elif [ "$test_linkfail" -ne 0 ] && [ -z "$cinfail" ]; then
1072		# the client file must be considerably larger
1073		# of the maximum expected cwin value, or the
1074		# link utilization will be not predicable
1075		size=$((RANDOM%2))
1076		size=$((size+1))
1077		size=$((size*8192))
1078		size=$((size + ( RANDOM % 8192) ))
1079
1080		cinfail=$(mktemp)
1081		make_file "$cinfail" "client" $size
1082	fi
1083
1084	if [ "$test_linkfail" -gt 2 ]; then
1085		size=$test_linkfail
1086
1087		if [ -z "$sinfail" ]; then
1088			sinfail=$(mktemp)
1089		fi
1090		make_file "$sinfail" "server" $size
1091	elif [ "$test_linkfail" -eq 2 ] && [ -z "$sinfail" ]; then
1092		size=$((RANDOM%16))
1093		size=$((size+1))
1094		size=$((size*2048))
1095
1096		sinfail=$(mktemp)
1097		make_file "$sinfail" "server" $size
1098	fi
1099
1100	do_transfer ${listener_ns} ${connector_ns} MPTCP MPTCP ${connect_addr} \
1101		${test_linkfail} ${addr_nr_ns1} ${addr_nr_ns2} ${speed} ${sflags}
1102}
1103
1104dump_stats()
1105{
1106	echo Server ns stats
1107	ip netns exec $ns1 nstat -as | grep Tcp
1108	echo Client ns stats
1109	ip netns exec $ns2 nstat -as | grep Tcp
1110}
1111
1112chk_csum_nr()
1113{
1114	local csum_ns1=${1:-0}
1115	local csum_ns2=${2:-0}
1116	local count
1117	local dump_stats
1118	local extra_msg=""
1119	local allow_multi_errors_ns1=0
1120	local allow_multi_errors_ns2=0
1121
1122	if [[ "${csum_ns1}" = "+"* ]]; then
1123		allow_multi_errors_ns1=1
1124		csum_ns1=${csum_ns1:1}
1125	fi
1126	if [[ "${csum_ns2}" = "+"* ]]; then
1127		allow_multi_errors_ns2=1
1128		csum_ns2=${csum_ns2:1}
1129	fi
1130
1131	printf "%-${nr_blank}s %s" " " "sum"
1132	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtDataCsumErr | awk '{print $2}')
1133	[ -z "$count" ] && count=0
1134	if [ "$count" != "$csum_ns1" ]; then
1135		extra_msg="$extra_msg ns1=$count"
1136	fi
1137	if { [ "$count" != $csum_ns1 ] && [ $allow_multi_errors_ns1 -eq 0 ]; } ||
1138	   { [ "$count" -lt $csum_ns1 ] && [ $allow_multi_errors_ns1 -eq 1 ]; }; then
1139		echo "[fail] got $count data checksum error[s] expected $csum_ns1"
1140		fail_test
1141		dump_stats=1
1142	else
1143		echo -n "[ ok ]"
1144	fi
1145	echo -n " - csum  "
1146	count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtDataCsumErr | awk '{print $2}')
1147	[ -z "$count" ] && count=0
1148	if [ "$count" != "$csum_ns2" ]; then
1149		extra_msg="$extra_msg ns2=$count"
1150	fi
1151	if { [ "$count" != $csum_ns2 ] && [ $allow_multi_errors_ns2 -eq 0 ]; } ||
1152	   { [ "$count" -lt $csum_ns2 ] && [ $allow_multi_errors_ns2 -eq 1 ]; }; then
1153		echo "[fail] got $count data checksum error[s] expected $csum_ns2"
1154		fail_test
1155		dump_stats=1
1156	else
1157		echo -n "[ ok ]"
1158	fi
1159	[ "${dump_stats}" = 1 ] && dump_stats
1160
1161	echo "$extra_msg"
1162}
1163
1164chk_fail_nr()
1165{
1166	local fail_tx=$1
1167	local fail_rx=$2
1168	local ns_invert=${3:-""}
1169	local count
1170	local dump_stats
1171	local ns_tx=$ns1
1172	local ns_rx=$ns2
1173	local extra_msg=""
1174	local allow_tx_lost=0
1175	local allow_rx_lost=0
1176
1177	if [[ $ns_invert = "invert" ]]; then
1178		ns_tx=$ns2
1179		ns_rx=$ns1
1180		extra_msg=" invert"
1181	fi
1182
1183	if [[ "${fail_tx}" = "-"* ]]; then
1184		allow_tx_lost=1
1185		fail_tx=${fail_tx:1}
1186	fi
1187	if [[ "${fail_rx}" = "-"* ]]; then
1188		allow_rx_lost=1
1189		fail_rx=${fail_rx:1}
1190	fi
1191
1192	printf "%-${nr_blank}s %s" " " "ftx"
1193	count=$(ip netns exec $ns_tx nstat -as | grep MPTcpExtMPFailTx | awk '{print $2}')
1194	[ -z "$count" ] && count=0
1195	if [ "$count" != "$fail_tx" ]; then
1196		extra_msg="$extra_msg,tx=$count"
1197	fi
1198	if { [ "$count" != "$fail_tx" ] && [ $allow_tx_lost -eq 0 ]; } ||
1199	   { [ "$count" -gt "$fail_tx" ] && [ $allow_tx_lost -eq 1 ]; }; then
1200		echo "[fail] got $count MP_FAIL[s] TX expected $fail_tx"
1201		fail_test
1202		dump_stats=1
1203	else
1204		echo -n "[ ok ]"
1205	fi
1206
1207	echo -n " - failrx"
1208	count=$(ip netns exec $ns_rx nstat -as | grep MPTcpExtMPFailRx | awk '{print $2}')
1209	[ -z "$count" ] && count=0
1210	if [ "$count" != "$fail_rx" ]; then
1211		extra_msg="$extra_msg,rx=$count"
1212	fi
1213	if { [ "$count" != "$fail_rx" ] && [ $allow_rx_lost -eq 0 ]; } ||
1214	   { [ "$count" -gt "$fail_rx" ] && [ $allow_rx_lost -eq 1 ]; }; then
1215		echo "[fail] got $count MP_FAIL[s] RX expected $fail_rx"
1216		fail_test
1217		dump_stats=1
1218	else
1219		echo -n "[ ok ]"
1220	fi
1221
1222	[ "${dump_stats}" = 1 ] && dump_stats
1223
1224	echo "$extra_msg"
1225}
1226
1227chk_fclose_nr()
1228{
1229	local fclose_tx=$1
1230	local fclose_rx=$2
1231	local ns_invert=$3
1232	local count
1233	local dump_stats
1234	local ns_tx=$ns2
1235	local ns_rx=$ns1
1236	local extra_msg="   "
1237
1238	if [[ $ns_invert = "invert" ]]; then
1239		ns_tx=$ns1
1240		ns_rx=$ns2
1241		extra_msg=${extra_msg}"invert"
1242	fi
1243
1244	printf "%-${nr_blank}s %s" " " "ctx"
1245	count=$(ip netns exec $ns_tx nstat -as | grep MPTcpExtMPFastcloseTx | awk '{print $2}')
1246	[ -z "$count" ] && count=0
1247	[ "$count" != "$fclose_tx" ] && extra_msg="$extra_msg,tx=$count"
1248	if [ "$count" != "$fclose_tx" ]; then
1249		echo "[fail] got $count MP_FASTCLOSE[s] TX expected $fclose_tx"
1250		fail_test
1251		dump_stats=1
1252	else
1253		echo -n "[ ok ]"
1254	fi
1255
1256	echo -n " - fclzrx"
1257	count=$(ip netns exec $ns_rx nstat -as | grep MPTcpExtMPFastcloseRx | awk '{print $2}')
1258	[ -z "$count" ] && count=0
1259	[ "$count" != "$fclose_rx" ] && extra_msg="$extra_msg,rx=$count"
1260	if [ "$count" != "$fclose_rx" ]; then
1261		echo "[fail] got $count MP_FASTCLOSE[s] RX expected $fclose_rx"
1262		fail_test
1263		dump_stats=1
1264	else
1265		echo -n "[ ok ]"
1266	fi
1267
1268	[ "${dump_stats}" = 1 ] && dump_stats
1269
1270	echo "$extra_msg"
1271}
1272
1273chk_rst_nr()
1274{
1275	local rst_tx=$1
1276	local rst_rx=$2
1277	local ns_invert=${3:-""}
1278	local count
1279	local dump_stats
1280	local ns_tx=$ns1
1281	local ns_rx=$ns2
1282	local extra_msg=""
1283
1284	if [[ $ns_invert = "invert" ]]; then
1285		ns_tx=$ns2
1286		ns_rx=$ns1
1287		extra_msg="   invert"
1288	fi
1289
1290	printf "%-${nr_blank}s %s" " " "rtx"
1291	count=$(ip netns exec $ns_tx nstat -as | grep MPTcpExtMPRstTx | awk '{print $2}')
1292	[ -z "$count" ] && count=0
1293	if [ $count -lt $rst_tx ]; then
1294		echo "[fail] got $count MP_RST[s] TX expected $rst_tx"
1295		fail_test
1296		dump_stats=1
1297	else
1298		echo -n "[ ok ]"
1299	fi
1300
1301	echo -n " - rstrx "
1302	count=$(ip netns exec $ns_rx nstat -as | grep MPTcpExtMPRstRx | awk '{print $2}')
1303	[ -z "$count" ] && count=0
1304	if [ "$count" -lt "$rst_rx" ]; then
1305		echo "[fail] got $count MP_RST[s] RX expected $rst_rx"
1306		fail_test
1307		dump_stats=1
1308	else
1309		echo -n "[ ok ]"
1310	fi
1311
1312	[ "${dump_stats}" = 1 ] && dump_stats
1313
1314	echo "$extra_msg"
1315}
1316
1317chk_infi_nr()
1318{
1319	local infi_tx=$1
1320	local infi_rx=$2
1321	local count
1322	local dump_stats
1323
1324	printf "%-${nr_blank}s %s" " " "itx"
1325	count=$(ip netns exec $ns2 nstat -as | grep InfiniteMapTx | awk '{print $2}')
1326	[ -z "$count" ] && count=0
1327	if [ "$count" != "$infi_tx" ]; then
1328		echo "[fail] got $count infinite map[s] TX expected $infi_tx"
1329		fail_test
1330		dump_stats=1
1331	else
1332		echo -n "[ ok ]"
1333	fi
1334
1335	echo -n " - infirx"
1336	count=$(ip netns exec $ns1 nstat -as | grep InfiniteMapRx | awk '{print $2}')
1337	[ -z "$count" ] && count=0
1338	if [ "$count" != "$infi_rx" ]; then
1339		echo "[fail] got $count infinite map[s] RX expected $infi_rx"
1340		fail_test
1341		dump_stats=1
1342	else
1343		echo "[ ok ]"
1344	fi
1345
1346	[ "${dump_stats}" = 1 ] && dump_stats
1347}
1348
1349chk_join_nr()
1350{
1351	local syn_nr=$1
1352	local syn_ack_nr=$2
1353	local ack_nr=$3
1354	local csum_ns1=${4:-0}
1355	local csum_ns2=${5:-0}
1356	local fail_nr=${6:-0}
1357	local rst_nr=${7:-0}
1358	local infi_nr=${8:-0}
1359	local corrupted_pkts=${9:-0}
1360	local count
1361	local dump_stats
1362	local with_cookie
1363	local title="${TEST_NAME}"
1364
1365	if [ "${corrupted_pkts}" -gt 0 ]; then
1366		title+=": ${corrupted_pkts} corrupted pkts"
1367	fi
1368
1369	printf "%03u %-36s %s" "${TEST_COUNT}" "${title}" "syn"
1370	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinSynRx | awk '{print $2}')
1371	[ -z "$count" ] && count=0
1372	if [ "$count" != "$syn_nr" ]; then
1373		echo "[fail] got $count JOIN[s] syn expected $syn_nr"
1374		fail_test
1375		dump_stats=1
1376	else
1377		echo -n "[ ok ]"
1378	fi
1379
1380	echo -n " - synack"
1381	with_cookie=$(ip netns exec $ns2 sysctl -n net.ipv4.tcp_syncookies)
1382	count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtMPJoinSynAckRx | awk '{print $2}')
1383	[ -z "$count" ] && count=0
1384	if [ "$count" != "$syn_ack_nr" ]; then
1385		# simult connections exceeding the limit with cookie enabled could go up to
1386		# synack validation as the conn limit can be enforced reliably only after
1387		# the subflow creation
1388		if [ "$with_cookie" = 2 ] && [ "$count" -gt "$syn_ack_nr" ] && [ "$count" -le "$syn_nr" ]; then
1389			echo -n "[ ok ]"
1390		else
1391			echo "[fail] got $count JOIN[s] synack expected $syn_ack_nr"
1392			fail_test
1393			dump_stats=1
1394		fi
1395	else
1396		echo -n "[ ok ]"
1397	fi
1398
1399	echo -n " - ack"
1400	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinAckRx | awk '{print $2}')
1401	[ -z "$count" ] && count=0
1402	if [ "$count" != "$ack_nr" ]; then
1403		echo "[fail] got $count JOIN[s] ack expected $ack_nr"
1404		fail_test
1405		dump_stats=1
1406	else
1407		echo "[ ok ]"
1408	fi
1409	[ "${dump_stats}" = 1 ] && dump_stats
1410	if [ $validate_checksum -eq 1 ]; then
1411		chk_csum_nr $csum_ns1 $csum_ns2
1412		chk_fail_nr $fail_nr $fail_nr
1413		chk_rst_nr $rst_nr $rst_nr
1414		chk_infi_nr $infi_nr $infi_nr
1415	fi
1416}
1417
1418# a negative value for 'stale_max' means no upper bound:
1419# for bidirectional transfer, if one peer sleep for a while
1420# - as these tests do - we can have a quite high number of
1421# stale/recover conversions, proportional to
1422# sleep duration/ MPTCP-level RTX interval.
1423chk_stale_nr()
1424{
1425	local ns=$1
1426	local stale_min=$2
1427	local stale_max=$3
1428	local stale_delta=$4
1429	local dump_stats
1430	local stale_nr
1431	local recover_nr
1432
1433	printf "%-${nr_blank}s %-18s" " " "stale"
1434	stale_nr=$(ip netns exec $ns nstat -as | grep MPTcpExtSubflowStale | awk '{print $2}')
1435	[ -z "$stale_nr" ] && stale_nr=0
1436	recover_nr=$(ip netns exec $ns nstat -as | grep MPTcpExtSubflowRecover | awk '{print $2}')
1437	[ -z "$recover_nr" ] && recover_nr=0
1438
1439	if [ $stale_nr -lt $stale_min ] ||
1440	   { [ $stale_max -gt 0 ] && [ $stale_nr -gt $stale_max ]; } ||
1441	   [ $((stale_nr - recover_nr)) -ne $stale_delta ]; then
1442		echo "[fail] got $stale_nr stale[s] $recover_nr recover[s], " \
1443		     " expected stale in range [$stale_min..$stale_max]," \
1444		     " stale-recover delta $stale_delta "
1445		fail_test
1446		dump_stats=1
1447	else
1448		echo "[ ok ]"
1449	fi
1450
1451	if [ "${dump_stats}" = 1 ]; then
1452		echo $ns stats
1453		ip netns exec $ns ip -s link show
1454		ip netns exec $ns nstat -as | grep MPTcp
1455	fi
1456}
1457
1458chk_add_nr()
1459{
1460	local add_nr=$1
1461	local echo_nr=$2
1462	local port_nr=${3:-0}
1463	local syn_nr=${4:-$port_nr}
1464	local syn_ack_nr=${5:-$port_nr}
1465	local ack_nr=${6:-$port_nr}
1466	local mis_syn_nr=${7:-0}
1467	local mis_ack_nr=${8:-0}
1468	local count
1469	local dump_stats
1470	local timeout
1471
1472	timeout=$(ip netns exec $ns1 sysctl -n net.mptcp.add_addr_timeout)
1473
1474	printf "%-${nr_blank}s %s" " " "add"
1475	count=$(ip netns exec $ns2 nstat -as MPTcpExtAddAddr | grep MPTcpExtAddAddr | awk '{print $2}')
1476	[ -z "$count" ] && count=0
1477
1478	# if the test configured a short timeout tolerate greater then expected
1479	# add addrs options, due to retransmissions
1480	if [ "$count" != "$add_nr" ] && { [ "$timeout" -gt 1 ] || [ "$count" -lt "$add_nr" ]; }; then
1481		echo "[fail] got $count ADD_ADDR[s] expected $add_nr"
1482		fail_test
1483		dump_stats=1
1484	else
1485		echo -n "[ ok ]"
1486	fi
1487
1488	echo -n " - echo  "
1489	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtEchoAdd | awk '{print $2}')
1490	[ -z "$count" ] && count=0
1491	if [ "$count" != "$echo_nr" ]; then
1492		echo "[fail] got $count ADD_ADDR echo[s] expected $echo_nr"
1493		fail_test
1494		dump_stats=1
1495	else
1496		echo -n "[ ok ]"
1497	fi
1498
1499	if [ $port_nr -gt 0 ]; then
1500		echo -n " - pt "
1501		count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtPortAdd | awk '{print $2}')
1502		[ -z "$count" ] && count=0
1503		if [ "$count" != "$port_nr" ]; then
1504			echo "[fail] got $count ADD_ADDR[s] with a port-number expected $port_nr"
1505			fail_test
1506			dump_stats=1
1507		else
1508			echo "[ ok ]"
1509		fi
1510
1511		printf "%-${nr_blank}s %s" " " "syn"
1512		count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinPortSynRx |
1513			awk '{print $2}')
1514		[ -z "$count" ] && count=0
1515		if [ "$count" != "$syn_nr" ]; then
1516			echo "[fail] got $count JOIN[s] syn with a different \
1517				port-number expected $syn_nr"
1518			fail_test
1519			dump_stats=1
1520		else
1521			echo -n "[ ok ]"
1522		fi
1523
1524		echo -n " - synack"
1525		count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtMPJoinPortSynAckRx |
1526			awk '{print $2}')
1527		[ -z "$count" ] && count=0
1528		if [ "$count" != "$syn_ack_nr" ]; then
1529			echo "[fail] got $count JOIN[s] synack with a different \
1530				port-number expected $syn_ack_nr"
1531			fail_test
1532			dump_stats=1
1533		else
1534			echo -n "[ ok ]"
1535		fi
1536
1537		echo -n " - ack"
1538		count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinPortAckRx |
1539			awk '{print $2}')
1540		[ -z "$count" ] && count=0
1541		if [ "$count" != "$ack_nr" ]; then
1542			echo "[fail] got $count JOIN[s] ack with a different \
1543				port-number expected $ack_nr"
1544			fail_test
1545			dump_stats=1
1546		else
1547			echo "[ ok ]"
1548		fi
1549
1550		printf "%-${nr_blank}s %s" " " "syn"
1551		count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMismatchPortSynRx |
1552			awk '{print $2}')
1553		[ -z "$count" ] && count=0
1554		if [ "$count" != "$mis_syn_nr" ]; then
1555			echo "[fail] got $count JOIN[s] syn with a mismatched \
1556				port-number expected $mis_syn_nr"
1557			fail_test
1558			dump_stats=1
1559		else
1560			echo -n "[ ok ]"
1561		fi
1562
1563		echo -n " - ack   "
1564		count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMismatchPortAckRx |
1565			awk '{print $2}')
1566		[ -z "$count" ] && count=0
1567		if [ "$count" != "$mis_ack_nr" ]; then
1568			echo "[fail] got $count JOIN[s] ack with a mismatched \
1569				port-number expected $mis_ack_nr"
1570			fail_test
1571			dump_stats=1
1572		else
1573			echo "[ ok ]"
1574		fi
1575	else
1576		echo ""
1577	fi
1578
1579	[ "${dump_stats}" = 1 ] && dump_stats
1580}
1581
1582chk_rm_nr()
1583{
1584	local rm_addr_nr=$1
1585	local rm_subflow_nr=$2
1586	local invert
1587	local simult
1588	local count
1589	local dump_stats
1590	local addr_ns=$ns1
1591	local subflow_ns=$ns2
1592	local extra_msg=""
1593
1594	shift 2
1595	while [ -n "$1" ]; do
1596		[ "$1" = "invert" ] && invert=true
1597		[ "$1" = "simult" ] && simult=true
1598		shift
1599	done
1600
1601	if [ -z $invert ]; then
1602		addr_ns=$ns1
1603		subflow_ns=$ns2
1604	elif [ $invert = "true" ]; then
1605		addr_ns=$ns2
1606		subflow_ns=$ns1
1607		extra_msg="   invert"
1608	fi
1609
1610	printf "%-${nr_blank}s %s" " " "rm "
1611	count=$(ip netns exec $addr_ns nstat -as | grep MPTcpExtRmAddr | awk '{print $2}')
1612	[ -z "$count" ] && count=0
1613	if [ "$count" != "$rm_addr_nr" ]; then
1614		echo "[fail] got $count RM_ADDR[s] expected $rm_addr_nr"
1615		fail_test
1616		dump_stats=1
1617	else
1618		echo -n "[ ok ]"
1619	fi
1620
1621	echo -n " - rmsf  "
1622	count=$(ip netns exec $subflow_ns nstat -as | grep MPTcpExtRmSubflow | awk '{print $2}')
1623	[ -z "$count" ] && count=0
1624	if [ -n "$simult" ]; then
1625		local cnt suffix
1626
1627		cnt=$(ip netns exec $addr_ns nstat -as | grep MPTcpExtRmSubflow | awk '{print $2}')
1628
1629		# in case of simult flush, the subflow removal count on each side is
1630		# unreliable
1631		[ -z "$cnt" ] && cnt=0
1632		count=$((count + cnt))
1633		[ "$count" != "$rm_subflow_nr" ] && suffix="$count in [$rm_subflow_nr:$((rm_subflow_nr*2))]"
1634		if [ $count -ge "$rm_subflow_nr" ] && \
1635		   [ "$count" -le "$((rm_subflow_nr *2 ))" ]; then
1636			echo "[ ok ] $suffix"
1637		else
1638			echo "[fail] got $count RM_SUBFLOW[s] expected in range [$rm_subflow_nr:$((rm_subflow_nr*2))]"
1639			fail_test
1640			dump_stats=1
1641		fi
1642		return
1643	fi
1644	if [ "$count" != "$rm_subflow_nr" ]; then
1645		echo "[fail] got $count RM_SUBFLOW[s] expected $rm_subflow_nr"
1646		fail_test
1647		dump_stats=1
1648	else
1649		echo -n "[ ok ]"
1650	fi
1651
1652	[ "${dump_stats}" = 1 ] && dump_stats
1653
1654	echo "$extra_msg"
1655}
1656
1657chk_prio_nr()
1658{
1659	local mp_prio_nr_tx=$1
1660	local mp_prio_nr_rx=$2
1661	local count
1662	local dump_stats
1663
1664	printf "%-${nr_blank}s %s" " " "ptx"
1665	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPPrioTx | awk '{print $2}')
1666	[ -z "$count" ] && count=0
1667	if [ "$count" != "$mp_prio_nr_tx" ]; then
1668		echo "[fail] got $count MP_PRIO[s] TX expected $mp_prio_nr_tx"
1669		fail_test
1670		dump_stats=1
1671	else
1672		echo -n "[ ok ]"
1673	fi
1674
1675	echo -n " - prx   "
1676	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPPrioRx | awk '{print $2}')
1677	[ -z "$count" ] && count=0
1678	if [ "$count" != "$mp_prio_nr_rx" ]; then
1679		echo "[fail] got $count MP_PRIO[s] RX expected $mp_prio_nr_rx"
1680		fail_test
1681		dump_stats=1
1682	else
1683		echo "[ ok ]"
1684	fi
1685
1686	[ "${dump_stats}" = 1 ] && dump_stats
1687}
1688
1689chk_subflow_nr()
1690{
1691	local need_title="$1"
1692	local msg="$2"
1693	local subflow_nr=$3
1694	local cnt1
1695	local cnt2
1696	local dump_stats
1697
1698	if [ -n "${need_title}" ]; then
1699		printf "%03u %-36s %s" "${TEST_COUNT}" "${TEST_NAME}" "${msg}"
1700	else
1701		printf "%-${nr_blank}s %s" " " "${msg}"
1702	fi
1703
1704	cnt1=$(ss -N $ns1 -tOni | grep -c token)
1705	cnt2=$(ss -N $ns2 -tOni | grep -c token)
1706	if [ "$cnt1" != "$subflow_nr" -o "$cnt2" != "$subflow_nr" ]; then
1707		echo "[fail] got $cnt1:$cnt2 subflows expected $subflow_nr"
1708		fail_test
1709		dump_stats=1
1710	else
1711		echo "[ ok ]"
1712	fi
1713
1714	if [ "${dump_stats}" = 1 ]; then
1715		ss -N $ns1 -tOni
1716		ss -N $ns1 -tOni | grep token
1717		ip -n $ns1 mptcp endpoint
1718		dump_stats
1719	fi
1720}
1721
1722chk_mptcp_info()
1723{
1724	local nr_info=$1
1725	local info
1726	local cnt1
1727	local cnt2
1728	local dump_stats
1729
1730	if [[ $nr_info = "subflows_"* ]]; then
1731		info="subflows"
1732		nr_info=${nr_info:9}
1733	else
1734		echo "[fail] unsupported argument: $nr_info"
1735		fail_test
1736		return 1
1737	fi
1738
1739	printf "%-${nr_blank}s %-30s" " " "mptcp_info $info=$nr_info"
1740
1741	cnt1=$(ss -N $ns1 -inmHM | grep "$info:" |
1742		sed -n 's/.*\('"$info"':\)\([[:digit:]]*\).*$/\2/p;q')
1743	[ -z "$cnt1" ] && cnt1=0
1744	cnt2=$(ss -N $ns2 -inmHM | grep "$info:" |
1745		sed -n 's/.*\('"$info"':\)\([[:digit:]]*\).*$/\2/p;q')
1746	[ -z "$cnt2" ] && cnt2=0
1747	if [ "$cnt1" != "$nr_info" ] || [ "$cnt2" != "$nr_info" ]; then
1748		echo "[fail] got $cnt1:$cnt2 $info expected $nr_info"
1749		fail_test
1750		dump_stats=1
1751	else
1752		echo "[ ok ]"
1753	fi
1754
1755	if [ "$dump_stats" = 1 ]; then
1756		ss -N $ns1 -inmHM
1757		ss -N $ns2 -inmHM
1758		dump_stats
1759	fi
1760}
1761
1762chk_link_usage()
1763{
1764	local ns=$1
1765	local link=$2
1766	local out=$3
1767	local expected_rate=$4
1768
1769	local tx_link tx_total
1770	tx_link=$(ip netns exec $ns cat /sys/class/net/$link/statistics/tx_bytes)
1771	tx_total=$(stat --format=%s $out)
1772	local tx_rate=$((tx_link * 100 / tx_total))
1773	local tolerance=5
1774
1775	printf "%-${nr_blank}s %-18s" " " "link usage"
1776	if [ $tx_rate -lt $((expected_rate - tolerance)) ] || \
1777	   [ $tx_rate -gt $((expected_rate + tolerance)) ]; then
1778		echo "[fail] got $tx_rate% usage, expected $expected_rate%"
1779		fail_test
1780	else
1781		echo "[ ok ]"
1782	fi
1783}
1784
1785wait_attempt_fail()
1786{
1787	local timeout_ms=$((timeout_poll * 1000))
1788	local time=0
1789	local ns=$1
1790
1791	while [ $time -lt $timeout_ms ]; do
1792		local cnt
1793
1794		cnt=$(ip netns exec $ns nstat -as TcpAttemptFails | grep TcpAttemptFails | awk '{print $2}')
1795
1796		[ "$cnt" = 1 ] && return 1
1797		time=$((time + 100))
1798		sleep 0.1
1799	done
1800	return 1
1801}
1802
1803set_userspace_pm()
1804{
1805	local ns=$1
1806
1807	ip netns exec $ns sysctl -q net.mptcp.pm_type=1
1808}
1809
1810subflows_tests()
1811{
1812	if reset "no JOIN"; then
1813		run_tests $ns1 $ns2 10.0.1.1
1814		chk_join_nr 0 0 0
1815	fi
1816
1817	# subflow limited by client
1818	if reset "single subflow, limited by client"; then
1819		pm_nl_set_limits $ns1 0 0
1820		pm_nl_set_limits $ns2 0 0
1821		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1822		run_tests $ns1 $ns2 10.0.1.1
1823		chk_join_nr 0 0 0
1824	fi
1825
1826	# subflow limited by server
1827	if reset "single subflow, limited by server"; then
1828		pm_nl_set_limits $ns1 0 0
1829		pm_nl_set_limits $ns2 0 1
1830		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1831		run_tests $ns1 $ns2 10.0.1.1
1832		chk_join_nr 1 1 0
1833	fi
1834
1835	# subflow
1836	if reset "single subflow"; then
1837		pm_nl_set_limits $ns1 0 1
1838		pm_nl_set_limits $ns2 0 1
1839		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1840		run_tests $ns1 $ns2 10.0.1.1
1841		chk_join_nr 1 1 1
1842	fi
1843
1844	# multiple subflows
1845	if reset "multiple subflows"; then
1846		pm_nl_set_limits $ns1 0 2
1847		pm_nl_set_limits $ns2 0 2
1848		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1849		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1850		run_tests $ns1 $ns2 10.0.1.1
1851		chk_join_nr 2 2 2
1852	fi
1853
1854	# multiple subflows limited by server
1855	if reset "multiple subflows, limited by server"; then
1856		pm_nl_set_limits $ns1 0 1
1857		pm_nl_set_limits $ns2 0 2
1858		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1859		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1860		run_tests $ns1 $ns2 10.0.1.1
1861		chk_join_nr 2 2 1
1862	fi
1863
1864	# single subflow, dev
1865	if reset "single subflow, dev"; then
1866		pm_nl_set_limits $ns1 0 1
1867		pm_nl_set_limits $ns2 0 1
1868		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow dev ns2eth3
1869		run_tests $ns1 $ns2 10.0.1.1
1870		chk_join_nr 1 1 1
1871	fi
1872}
1873
1874subflows_error_tests()
1875{
1876	# If a single subflow is configured, and matches the MPC src
1877	# address, no additional subflow should be created
1878	if reset "no MPC reuse with single endpoint"; then
1879		pm_nl_set_limits $ns1 0 1
1880		pm_nl_set_limits $ns2 0 1
1881		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow
1882		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
1883		chk_join_nr 0 0 0
1884	fi
1885
1886	# multiple subflows, with subflow creation error
1887	if reset "multi subflows, with failing subflow"; then
1888		pm_nl_set_limits $ns1 0 2
1889		pm_nl_set_limits $ns2 0 2
1890		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1891		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1892		filter_tcp_from $ns1 10.0.3.2 REJECT
1893		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
1894		chk_join_nr 1 1 1
1895	fi
1896
1897	# multiple subflows, with subflow timeout on MPJ
1898	if reset "multi subflows, with subflow timeout"; then
1899		pm_nl_set_limits $ns1 0 2
1900		pm_nl_set_limits $ns2 0 2
1901		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1902		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1903		filter_tcp_from $ns1 10.0.3.2 DROP
1904		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
1905		chk_join_nr 1 1 1
1906	fi
1907
1908	# multiple subflows, check that the endpoint corresponding to
1909	# closed subflow (due to reset) is not reused if additional
1910	# subflows are added later
1911	if reset "multi subflows, fair usage on close"; then
1912		pm_nl_set_limits $ns1 0 1
1913		pm_nl_set_limits $ns2 0 1
1914		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1915		filter_tcp_from $ns1 10.0.3.2 REJECT
1916		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow &
1917
1918		# mpj subflow will be in TW after the reset
1919		wait_attempt_fail $ns2
1920		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1921		wait
1922
1923		# additional subflow could be created only if the PM select
1924		# the later endpoint, skipping the already used one
1925		chk_join_nr 1 1 1
1926	fi
1927}
1928
1929signal_address_tests()
1930{
1931	# add_address, unused
1932	if reset "unused signal address"; then
1933		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1934		run_tests $ns1 $ns2 10.0.1.1
1935		chk_join_nr 0 0 0
1936		chk_add_nr 1 1
1937	fi
1938
1939	# accept and use add_addr
1940	if reset "signal address"; then
1941		pm_nl_set_limits $ns1 0 1
1942		pm_nl_set_limits $ns2 1 1
1943		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1944		run_tests $ns1 $ns2 10.0.1.1
1945		chk_join_nr 1 1 1
1946		chk_add_nr 1 1
1947	fi
1948
1949	# accept and use add_addr with an additional subflow
1950	# note: signal address in server ns and local addresses in client ns must
1951	# belong to different subnets or one of the listed local address could be
1952	# used for 'add_addr' subflow
1953	if reset "subflow and signal"; then
1954		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1955		pm_nl_set_limits $ns1 0 2
1956		pm_nl_set_limits $ns2 1 2
1957		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1958		run_tests $ns1 $ns2 10.0.1.1
1959		chk_join_nr 2 2 2
1960		chk_add_nr 1 1
1961	fi
1962
1963	# accept and use add_addr with additional subflows
1964	if reset "multiple subflows and signal"; then
1965		pm_nl_set_limits $ns1 0 3
1966		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1967		pm_nl_set_limits $ns2 1 3
1968		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1969		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
1970		run_tests $ns1 $ns2 10.0.1.1
1971		chk_join_nr 3 3 3
1972		chk_add_nr 1 1
1973	fi
1974
1975	# signal addresses
1976	if reset "signal addresses"; then
1977		pm_nl_set_limits $ns1 3 3
1978		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1979		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
1980		pm_nl_add_endpoint $ns1 10.0.4.1 flags signal
1981		pm_nl_set_limits $ns2 3 3
1982		run_tests $ns1 $ns2 10.0.1.1
1983		chk_join_nr 3 3 3
1984		chk_add_nr 3 3
1985	fi
1986
1987	# signal invalid addresses
1988	if reset "signal invalid addresses"; then
1989		pm_nl_set_limits $ns1 3 3
1990		pm_nl_add_endpoint $ns1 10.0.12.1 flags signal
1991		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
1992		pm_nl_add_endpoint $ns1 10.0.14.1 flags signal
1993		pm_nl_set_limits $ns2 3 3
1994		run_tests $ns1 $ns2 10.0.1.1
1995		chk_join_nr 1 1 1
1996		chk_add_nr 3 3
1997	fi
1998
1999	# signal addresses race test
2000	if reset "signal addresses race test"; then
2001		pm_nl_set_limits $ns1 4 4
2002		pm_nl_set_limits $ns2 4 4
2003		pm_nl_add_endpoint $ns1 10.0.1.1 flags signal
2004		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2005		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2006		pm_nl_add_endpoint $ns1 10.0.4.1 flags signal
2007		pm_nl_add_endpoint $ns2 10.0.1.2 flags signal
2008		pm_nl_add_endpoint $ns2 10.0.2.2 flags signal
2009		pm_nl_add_endpoint $ns2 10.0.3.2 flags signal
2010		pm_nl_add_endpoint $ns2 10.0.4.2 flags signal
2011
2012		# the peer could possibly miss some addr notification, allow retransmission
2013		ip netns exec $ns1 sysctl -q net.mptcp.add_addr_timeout=1
2014		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
2015		chk_join_nr 3 3 3
2016
2017		# the server will not signal the address terminating
2018		# the MPC subflow
2019		chk_add_nr 3 3
2020	fi
2021}
2022
2023link_failure_tests()
2024{
2025	# accept and use add_addr with additional subflows and link loss
2026	if reset "multiple flows, signal, link failure"; then
2027		# without any b/w limit each veth could spool the packets and get
2028		# them acked at xmit time, so that the corresponding subflow will
2029		# have almost always no outstanding pkts, the scheduler will pick
2030		# always the first subflow and we will have hard time testing
2031		# active backup and link switch-over.
2032		# Let's set some arbitrary (low) virtual link limits.
2033		init_shapers
2034		pm_nl_set_limits $ns1 0 3
2035		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
2036		pm_nl_set_limits $ns2 1 3
2037		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow
2038		pm_nl_add_endpoint $ns2 10.0.4.2 dev ns2eth4 flags subflow
2039		run_tests $ns1 $ns2 10.0.1.1 1
2040		chk_join_nr 3 3 3
2041		chk_add_nr 1 1
2042		chk_stale_nr $ns2 1 5 1
2043	fi
2044
2045	# accept and use add_addr with additional subflows and link loss
2046	# for bidirectional transfer
2047	if reset "multi flows, signal, bidi, link fail"; then
2048		init_shapers
2049		pm_nl_set_limits $ns1 0 3
2050		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
2051		pm_nl_set_limits $ns2 1 3
2052		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow
2053		pm_nl_add_endpoint $ns2 10.0.4.2 dev ns2eth4 flags subflow
2054		run_tests $ns1 $ns2 10.0.1.1 2
2055		chk_join_nr 3 3 3
2056		chk_add_nr 1 1
2057		chk_stale_nr $ns2 1 -1 1
2058	fi
2059
2060	# 2 subflows plus 1 backup subflow with a lossy link, backup
2061	# will never be used
2062	if reset "backup subflow unused, link failure"; then
2063		init_shapers
2064		pm_nl_set_limits $ns1 0 2
2065		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
2066		pm_nl_set_limits $ns2 1 2
2067		FAILING_LINKS="1"
2068		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow,backup
2069		run_tests $ns1 $ns2 10.0.1.1 1
2070		chk_join_nr 2 2 2
2071		chk_add_nr 1 1
2072		chk_link_usage $ns2 ns2eth3 $cinsent 0
2073	fi
2074
2075	# 2 lossy links after half transfer, backup will get half of
2076	# the traffic
2077	if reset "backup flow used, multi links fail"; then
2078		init_shapers
2079		pm_nl_set_limits $ns1 0 2
2080		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
2081		pm_nl_set_limits $ns2 1 2
2082		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow,backup
2083		FAILING_LINKS="1 2"
2084		run_tests $ns1 $ns2 10.0.1.1 1
2085		chk_join_nr 2 2 2
2086		chk_add_nr 1 1
2087		chk_stale_nr $ns2 2 4 2
2088		chk_link_usage $ns2 ns2eth3 $cinsent 50
2089	fi
2090
2091	# use a backup subflow with the first subflow on a lossy link
2092	# for bidirectional transfer
2093	if reset "backup flow used, bidi, link failure"; then
2094		init_shapers
2095		pm_nl_set_limits $ns1 0 2
2096		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
2097		pm_nl_set_limits $ns2 1 3
2098		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow,backup
2099		FAILING_LINKS="1 2"
2100		run_tests $ns1 $ns2 10.0.1.1 2
2101		chk_join_nr 2 2 2
2102		chk_add_nr 1 1
2103		chk_stale_nr $ns2 1 -1 2
2104		chk_link_usage $ns2 ns2eth3 $cinsent 50
2105	fi
2106}
2107
2108add_addr_timeout_tests()
2109{
2110	# add_addr timeout
2111	if reset_with_add_addr_timeout "signal address, ADD_ADDR timeout"; then
2112		pm_nl_set_limits $ns1 0 1
2113		pm_nl_set_limits $ns2 1 1
2114		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2115		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
2116		chk_join_nr 1 1 1
2117		chk_add_nr 4 0
2118	fi
2119
2120	# add_addr timeout IPv6
2121	if reset_with_add_addr_timeout "signal address, ADD_ADDR6 timeout" 6; then
2122		pm_nl_set_limits $ns1 0 1
2123		pm_nl_set_limits $ns2 1 1
2124		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2125		run_tests $ns1 $ns2 dead:beef:1::1 0 0 0 slow
2126		chk_join_nr 1 1 1
2127		chk_add_nr 4 0
2128	fi
2129
2130	# signal addresses timeout
2131	if reset_with_add_addr_timeout "signal addresses, ADD_ADDR timeout"; then
2132		pm_nl_set_limits $ns1 2 2
2133		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2134		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2135		pm_nl_set_limits $ns2 2 2
2136		run_tests $ns1 $ns2 10.0.1.1 0 0 0 speed_10
2137		chk_join_nr 2 2 2
2138		chk_add_nr 8 0
2139	fi
2140
2141	# signal invalid addresses timeout
2142	if reset_with_add_addr_timeout "invalid address, ADD_ADDR timeout"; then
2143		pm_nl_set_limits $ns1 2 2
2144		pm_nl_add_endpoint $ns1 10.0.12.1 flags signal
2145		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2146		pm_nl_set_limits $ns2 2 2
2147		run_tests $ns1 $ns2 10.0.1.1 0 0 0 speed_10
2148		chk_join_nr 1 1 1
2149		chk_add_nr 8 0
2150	fi
2151}
2152
2153remove_tests()
2154{
2155	# single subflow, remove
2156	if reset "remove single subflow"; then
2157		pm_nl_set_limits $ns1 0 1
2158		pm_nl_set_limits $ns2 0 1
2159		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2160		run_tests $ns1 $ns2 10.0.1.1 0 0 -1 slow
2161		chk_join_nr 1 1 1
2162		chk_rm_nr 1 1
2163	fi
2164
2165	# multiple subflows, remove
2166	if reset "remove multiple subflows"; then
2167		pm_nl_set_limits $ns1 0 2
2168		pm_nl_set_limits $ns2 0 2
2169		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
2170		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2171		run_tests $ns1 $ns2 10.0.1.1 0 0 -2 slow
2172		chk_join_nr 2 2 2
2173		chk_rm_nr 2 2
2174	fi
2175
2176	# single address, remove
2177	if reset "remove single address"; then
2178		pm_nl_set_limits $ns1 0 1
2179		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2180		pm_nl_set_limits $ns2 1 1
2181		run_tests $ns1 $ns2 10.0.1.1 0 -1 0 slow
2182		chk_join_nr 1 1 1
2183		chk_add_nr 1 1
2184		chk_rm_nr 1 1 invert
2185	fi
2186
2187	# subflow and signal, remove
2188	if reset "remove subflow and signal"; then
2189		pm_nl_set_limits $ns1 0 2
2190		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2191		pm_nl_set_limits $ns2 1 2
2192		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2193		run_tests $ns1 $ns2 10.0.1.1 0 -1 -1 slow
2194		chk_join_nr 2 2 2
2195		chk_add_nr 1 1
2196		chk_rm_nr 1 1
2197	fi
2198
2199	# subflows and signal, remove
2200	if reset "remove subflows and signal"; then
2201		pm_nl_set_limits $ns1 0 3
2202		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2203		pm_nl_set_limits $ns2 1 3
2204		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2205		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2206		run_tests $ns1 $ns2 10.0.1.1 0 -1 -2 speed_10
2207		chk_join_nr 3 3 3
2208		chk_add_nr 1 1
2209		chk_rm_nr 2 2
2210	fi
2211
2212	# addresses remove
2213	if reset "remove addresses"; then
2214		pm_nl_set_limits $ns1 3 3
2215		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal id 250
2216		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2217		pm_nl_add_endpoint $ns1 10.0.4.1 flags signal
2218		pm_nl_set_limits $ns2 3 3
2219		run_tests $ns1 $ns2 10.0.1.1 0 -3 0 speed_10
2220		chk_join_nr 3 3 3
2221		chk_add_nr 3 3
2222		chk_rm_nr 3 3 invert
2223	fi
2224
2225	# invalid addresses remove
2226	if reset "remove invalid addresses"; then
2227		pm_nl_set_limits $ns1 3 3
2228		pm_nl_add_endpoint $ns1 10.0.12.1 flags signal
2229		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2230		pm_nl_add_endpoint $ns1 10.0.14.1 flags signal
2231		pm_nl_set_limits $ns2 3 3
2232		run_tests $ns1 $ns2 10.0.1.1 0 -3 0 speed_10
2233		chk_join_nr 1 1 1
2234		chk_add_nr 3 3
2235		chk_rm_nr 3 1 invert
2236	fi
2237
2238	# subflows and signal, flush
2239	if reset "flush subflows and signal"; then
2240		pm_nl_set_limits $ns1 0 3
2241		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2242		pm_nl_set_limits $ns2 1 3
2243		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2244		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2245		run_tests $ns1 $ns2 10.0.1.1 0 -8 -8 slow
2246		chk_join_nr 3 3 3
2247		chk_add_nr 1 1
2248		chk_rm_nr 1 3 invert simult
2249	fi
2250
2251	# subflows flush
2252	if reset "flush subflows"; then
2253		pm_nl_set_limits $ns1 3 3
2254		pm_nl_set_limits $ns2 3 3
2255		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow id 150
2256		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2257		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2258		run_tests $ns1 $ns2 10.0.1.1 0 -8 -8 slow
2259		chk_join_nr 3 3 3
2260		chk_rm_nr 0 3 simult
2261	fi
2262
2263	# addresses flush
2264	if reset "flush addresses"; then
2265		pm_nl_set_limits $ns1 3 3
2266		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal id 250
2267		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2268		pm_nl_add_endpoint $ns1 10.0.4.1 flags signal
2269		pm_nl_set_limits $ns2 3 3
2270		run_tests $ns1 $ns2 10.0.1.1 0 -8 -8 slow
2271		chk_join_nr 3 3 3
2272		chk_add_nr 3 3
2273		chk_rm_nr 3 3 invert simult
2274	fi
2275
2276	# invalid addresses flush
2277	if reset "flush invalid addresses"; then
2278		pm_nl_set_limits $ns1 3 3
2279		pm_nl_add_endpoint $ns1 10.0.12.1 flags signal
2280		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2281		pm_nl_add_endpoint $ns1 10.0.14.1 flags signal
2282		pm_nl_set_limits $ns2 3 3
2283		run_tests $ns1 $ns2 10.0.1.1 0 -8 0 slow
2284		chk_join_nr 1 1 1
2285		chk_add_nr 3 3
2286		chk_rm_nr 3 1 invert
2287	fi
2288
2289	# remove id 0 subflow
2290	if reset "remove id 0 subflow"; then
2291		pm_nl_set_limits $ns1 0 1
2292		pm_nl_set_limits $ns2 0 1
2293		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2294		run_tests $ns1 $ns2 10.0.1.1 0 0 -9 slow
2295		chk_join_nr 1 1 1
2296		chk_rm_nr 1 1
2297	fi
2298
2299	# remove id 0 address
2300	if reset "remove id 0 address"; then
2301		pm_nl_set_limits $ns1 0 1
2302		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2303		pm_nl_set_limits $ns2 1 1
2304		run_tests $ns1 $ns2 10.0.1.1 0 -9 0 slow
2305		chk_join_nr 1 1 1
2306		chk_add_nr 1 1
2307		chk_rm_nr 1 1 invert
2308	fi
2309}
2310
2311add_tests()
2312{
2313	# add single subflow
2314	if reset "add single subflow"; then
2315		pm_nl_set_limits $ns1 0 1
2316		pm_nl_set_limits $ns2 0 1
2317		run_tests $ns1 $ns2 10.0.1.1 0 0 1 slow
2318		chk_join_nr 1 1 1
2319	fi
2320
2321	# add signal address
2322	if reset "add signal address"; then
2323		pm_nl_set_limits $ns1 0 1
2324		pm_nl_set_limits $ns2 1 1
2325		run_tests $ns1 $ns2 10.0.1.1 0 1 0 slow
2326		chk_join_nr 1 1 1
2327		chk_add_nr 1 1
2328	fi
2329
2330	# add multiple subflows
2331	if reset "add multiple subflows"; then
2332		pm_nl_set_limits $ns1 0 2
2333		pm_nl_set_limits $ns2 0 2
2334		run_tests $ns1 $ns2 10.0.1.1 0 0 2 slow
2335		chk_join_nr 2 2 2
2336	fi
2337
2338	# add multiple subflows IPv6
2339	if reset "add multiple subflows IPv6"; then
2340		pm_nl_set_limits $ns1 0 2
2341		pm_nl_set_limits $ns2 0 2
2342		run_tests $ns1 $ns2 dead:beef:1::1 0 0 2 slow
2343		chk_join_nr 2 2 2
2344	fi
2345
2346	# add multiple addresses IPv6
2347	if reset "add multiple addresses IPv6"; then
2348		pm_nl_set_limits $ns1 0 2
2349		pm_nl_set_limits $ns2 2 2
2350		run_tests $ns1 $ns2 dead:beef:1::1 0 2 0 slow
2351		chk_join_nr 2 2 2
2352		chk_add_nr 2 2
2353	fi
2354}
2355
2356ipv6_tests()
2357{
2358	# subflow IPv6
2359	if reset "single subflow IPv6"; then
2360		pm_nl_set_limits $ns1 0 1
2361		pm_nl_set_limits $ns2 0 1
2362		pm_nl_add_endpoint $ns2 dead:beef:3::2 dev ns2eth3 flags subflow
2363		run_tests $ns1 $ns2 dead:beef:1::1 0 0 0 slow
2364		chk_join_nr 1 1 1
2365	fi
2366
2367	# add_address, unused IPv6
2368	if reset "unused signal address IPv6"; then
2369		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2370		run_tests $ns1 $ns2 dead:beef:1::1 0 0 0 slow
2371		chk_join_nr 0 0 0
2372		chk_add_nr 1 1
2373	fi
2374
2375	# signal address IPv6
2376	if reset "single address IPv6"; then
2377		pm_nl_set_limits $ns1 0 1
2378		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2379		pm_nl_set_limits $ns2 1 1
2380		run_tests $ns1 $ns2 dead:beef:1::1 0 0 0 slow
2381		chk_join_nr 1 1 1
2382		chk_add_nr 1 1
2383	fi
2384
2385	# single address IPv6, remove
2386	if reset "remove single address IPv6"; then
2387		pm_nl_set_limits $ns1 0 1
2388		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2389		pm_nl_set_limits $ns2 1 1
2390		run_tests $ns1 $ns2 dead:beef:1::1 0 -1 0 slow
2391		chk_join_nr 1 1 1
2392		chk_add_nr 1 1
2393		chk_rm_nr 1 1 invert
2394	fi
2395
2396	# subflow and signal IPv6, remove
2397	if reset "remove subflow and signal IPv6"; then
2398		pm_nl_set_limits $ns1 0 2
2399		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2400		pm_nl_set_limits $ns2 1 2
2401		pm_nl_add_endpoint $ns2 dead:beef:3::2 dev ns2eth3 flags subflow
2402		run_tests $ns1 $ns2 dead:beef:1::1 0 -1 -1 slow
2403		chk_join_nr 2 2 2
2404		chk_add_nr 1 1
2405		chk_rm_nr 1 1
2406	fi
2407}
2408
2409v4mapped_tests()
2410{
2411	# subflow IPv4-mapped to IPv4-mapped
2412	if reset "single subflow IPv4-mapped"; then
2413		pm_nl_set_limits $ns1 0 1
2414		pm_nl_set_limits $ns2 0 1
2415		pm_nl_add_endpoint $ns2 "::ffff:10.0.3.2" flags subflow
2416		run_tests $ns1 $ns2 "::ffff:10.0.1.1"
2417		chk_join_nr 1 1 1
2418	fi
2419
2420	# signal address IPv4-mapped with IPv4-mapped sk
2421	if reset "signal address IPv4-mapped"; then
2422		pm_nl_set_limits $ns1 0 1
2423		pm_nl_set_limits $ns2 1 1
2424		pm_nl_add_endpoint $ns1 "::ffff:10.0.2.1" flags signal
2425		run_tests $ns1 $ns2 "::ffff:10.0.1.1"
2426		chk_join_nr 1 1 1
2427		chk_add_nr 1 1
2428	fi
2429
2430	# subflow v4-map-v6
2431	if reset "single subflow v4-map-v6"; then
2432		pm_nl_set_limits $ns1 0 1
2433		pm_nl_set_limits $ns2 0 1
2434		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2435		run_tests $ns1 $ns2 "::ffff:10.0.1.1"
2436		chk_join_nr 1 1 1
2437	fi
2438
2439	# signal address v4-map-v6
2440	if reset "signal address v4-map-v6"; then
2441		pm_nl_set_limits $ns1 0 1
2442		pm_nl_set_limits $ns2 1 1
2443		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2444		run_tests $ns1 $ns2 "::ffff:10.0.1.1"
2445		chk_join_nr 1 1 1
2446		chk_add_nr 1 1
2447	fi
2448
2449	# subflow v6-map-v4
2450	if reset "single subflow v6-map-v4"; then
2451		pm_nl_set_limits $ns1 0 1
2452		pm_nl_set_limits $ns2 0 1
2453		pm_nl_add_endpoint $ns2 "::ffff:10.0.3.2" flags subflow
2454		run_tests $ns1 $ns2 10.0.1.1
2455		chk_join_nr 1 1 1
2456	fi
2457
2458	# signal address v6-map-v4
2459	if reset "signal address v6-map-v4"; then
2460		pm_nl_set_limits $ns1 0 1
2461		pm_nl_set_limits $ns2 1 1
2462		pm_nl_add_endpoint $ns1 "::ffff:10.0.2.1" flags signal
2463		run_tests $ns1 $ns2 10.0.1.1
2464		chk_join_nr 1 1 1
2465		chk_add_nr 1 1
2466	fi
2467
2468	# no subflow IPv6 to v4 address
2469	if reset "no JOIN with diff families v4-v6"; then
2470		pm_nl_set_limits $ns1 0 1
2471		pm_nl_set_limits $ns2 0 1
2472		pm_nl_add_endpoint $ns2 dead:beef:2::2 flags subflow
2473		run_tests $ns1 $ns2 10.0.1.1
2474		chk_join_nr 0 0 0
2475	fi
2476
2477	# no subflow IPv6 to v4 address even if v6 has a valid v4 at the end
2478	if reset "no JOIN with diff families v4-v6-2"; then
2479		pm_nl_set_limits $ns1 0 1
2480		pm_nl_set_limits $ns2 0 1
2481		pm_nl_add_endpoint $ns2 dead:beef:2::10.0.3.2 flags subflow
2482		run_tests $ns1 $ns2 10.0.1.1
2483		chk_join_nr 0 0 0
2484	fi
2485
2486	# no subflow IPv4 to v6 address, no need to slow down too then
2487	if reset "no JOIN with diff families v6-v4"; then
2488		pm_nl_set_limits $ns1 0 1
2489		pm_nl_set_limits $ns2 0 1
2490		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2491		run_tests $ns1 $ns2 dead:beef:1::1
2492		chk_join_nr 0 0 0
2493	fi
2494}
2495
2496mixed_tests()
2497{
2498	if reset "IPv4 sockets do not use IPv6 addresses"; then
2499		pm_nl_set_limits $ns1 0 1
2500		pm_nl_set_limits $ns2 1 1
2501		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2502		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
2503		chk_join_nr 0 0 0
2504	fi
2505
2506	# Need an IPv6 mptcp socket to allow subflows of both families
2507	if reset "simult IPv4 and IPv6 subflows"; then
2508		pm_nl_set_limits $ns1 0 1
2509		pm_nl_set_limits $ns2 1 1
2510		pm_nl_add_endpoint $ns1 10.0.1.1 flags signal
2511		run_tests $ns1 $ns2 dead:beef:2::1 0 0 0 slow
2512		chk_join_nr 1 1 1
2513	fi
2514
2515	# cross families subflows will not be created even in fullmesh mode
2516	if reset "simult IPv4 and IPv6 subflows, fullmesh 1x1"; then
2517		pm_nl_set_limits $ns1 0 4
2518		pm_nl_set_limits $ns2 1 4
2519		pm_nl_add_endpoint $ns2 dead:beef:2::2 flags subflow,fullmesh
2520		pm_nl_add_endpoint $ns1 10.0.1.1 flags signal
2521		run_tests $ns1 $ns2 dead:beef:2::1 0 0 0 slow
2522		chk_join_nr 1 1 1
2523	fi
2524
2525	# fullmesh still tries to create all the possibly subflows with
2526	# matching family
2527	if reset "simult IPv4 and IPv6 subflows, fullmesh 2x2"; then
2528		pm_nl_set_limits $ns1 0 4
2529		pm_nl_set_limits $ns2 2 4
2530		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2531		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2532		run_tests $ns1 $ns2 dead:beef:1::1 0 0 fullmesh_1 slow
2533		chk_join_nr 4 4 4
2534	fi
2535}
2536
2537backup_tests()
2538{
2539	# single subflow, backup
2540	if reset "single subflow, backup"; then
2541		pm_nl_set_limits $ns1 0 1
2542		pm_nl_set_limits $ns2 0 1
2543		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow,backup
2544		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow nobackup
2545		chk_join_nr 1 1 1
2546		chk_prio_nr 0 1
2547	fi
2548
2549	# single address, backup
2550	if reset "single address, backup"; then
2551		pm_nl_set_limits $ns1 0 1
2552		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2553		pm_nl_set_limits $ns2 1 1
2554		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
2555		chk_join_nr 1 1 1
2556		chk_add_nr 1 1
2557		chk_prio_nr 1 1
2558	fi
2559
2560	# single address with port, backup
2561	if reset "single address with port, backup"; then
2562		pm_nl_set_limits $ns1 0 1
2563		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2564		pm_nl_set_limits $ns2 1 1
2565		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
2566		chk_join_nr 1 1 1
2567		chk_add_nr 1 1
2568		chk_prio_nr 1 1
2569	fi
2570
2571	if reset "mpc backup"; then
2572		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow,backup
2573		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
2574		chk_join_nr 0 0 0
2575		chk_prio_nr 0 1
2576	fi
2577
2578	if reset "mpc backup both sides"; then
2579		pm_nl_add_endpoint $ns1 10.0.1.1 flags subflow,backup
2580		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow,backup
2581		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
2582		chk_join_nr 0 0 0
2583		chk_prio_nr 1 1
2584	fi
2585
2586	if reset "mpc switch to backup"; then
2587		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow
2588		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
2589		chk_join_nr 0 0 0
2590		chk_prio_nr 0 1
2591	fi
2592
2593	if reset "mpc switch to backup both sides"; then
2594		pm_nl_add_endpoint $ns1 10.0.1.1 flags subflow
2595		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow
2596		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
2597		chk_join_nr 0 0 0
2598		chk_prio_nr 1 1
2599	fi
2600}
2601
2602LISTENER_CREATED=15 #MPTCP_EVENT_LISTENER_CREATED
2603LISTENER_CLOSED=16  #MPTCP_EVENT_LISTENER_CLOSED
2604
2605AF_INET=2
2606AF_INET6=10
2607
2608verify_listener_events()
2609{
2610	local evt=$1
2611	local e_type=$2
2612	local e_family=$3
2613	local e_saddr=$4
2614	local e_sport=$5
2615	local type
2616	local family
2617	local saddr
2618	local sport
2619
2620	if [ $e_type = $LISTENER_CREATED ]; then
2621		stdbuf -o0 -e0 printf "\t\t\t\t\t CREATE_LISTENER %s:%s"\
2622			$e_saddr $e_sport
2623	elif [ $e_type = $LISTENER_CLOSED ]; then
2624		stdbuf -o0 -e0 printf "\t\t\t\t\t CLOSE_LISTENER %s:%s "\
2625			$e_saddr $e_sport
2626	fi
2627
2628	type=$(grep "type:$e_type," $evt |
2629	       sed --unbuffered -n 's/.*\(type:\)\([[:digit:]]*\).*$/\2/p;q')
2630	family=$(grep "type:$e_type," $evt |
2631		 sed --unbuffered -n 's/.*\(family:\)\([[:digit:]]*\).*$/\2/p;q')
2632	sport=$(grep "type:$e_type," $evt |
2633		sed --unbuffered -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q')
2634	if [ $family ] && [ $family = $AF_INET6 ]; then
2635		saddr=$(grep "type:$e_type," $evt |
2636			sed --unbuffered -n 's/.*\(saddr6:\)\([0-9a-f:.]*\).*$/\2/p;q')
2637	else
2638		saddr=$(grep "type:$e_type," $evt |
2639			sed --unbuffered -n 's/.*\(saddr4:\)\([0-9.]*\).*$/\2/p;q')
2640	fi
2641
2642	if [ $type ] && [ $type = $e_type ] &&
2643	   [ $family ] && [ $family = $e_family ] &&
2644	   [ $saddr ] && [ $saddr = $e_saddr ] &&
2645	   [ $sport ] && [ $sport = $e_sport ]; then
2646		stdbuf -o0 -e0 printf "[ ok ]\n"
2647		return 0
2648	fi
2649	fail_test
2650	stdbuf -o0 -e0 printf "[fail]\n"
2651}
2652
2653add_addr_ports_tests()
2654{
2655	# signal address with port
2656	if reset "signal address with port"; then
2657		pm_nl_set_limits $ns1 0 1
2658		pm_nl_set_limits $ns2 1 1
2659		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2660		run_tests $ns1 $ns2 10.0.1.1
2661		chk_join_nr 1 1 1
2662		chk_add_nr 1 1 1
2663	fi
2664
2665	# subflow and signal with port
2666	if reset "subflow and signal with port"; then
2667		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2668		pm_nl_set_limits $ns1 0 2
2669		pm_nl_set_limits $ns2 1 2
2670		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2671		run_tests $ns1 $ns2 10.0.1.1
2672		chk_join_nr 2 2 2
2673		chk_add_nr 1 1 1
2674	fi
2675
2676	# single address with port, remove
2677	# pm listener events
2678	if reset_with_events "remove single address with port"; then
2679		pm_nl_set_limits $ns1 0 1
2680		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2681		pm_nl_set_limits $ns2 1 1
2682		run_tests $ns1 $ns2 10.0.1.1 0 -1 0 slow
2683		chk_join_nr 1 1 1
2684		chk_add_nr 1 1 1
2685		chk_rm_nr 1 1 invert
2686
2687		verify_listener_events $evts_ns1 $LISTENER_CREATED $AF_INET 10.0.2.1 10100
2688		verify_listener_events $evts_ns1 $LISTENER_CLOSED $AF_INET 10.0.2.1 10100
2689		kill_events_pids
2690	fi
2691
2692	# subflow and signal with port, remove
2693	if reset "remove subflow and signal with port"; then
2694		pm_nl_set_limits $ns1 0 2
2695		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2696		pm_nl_set_limits $ns2 1 2
2697		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2698		run_tests $ns1 $ns2 10.0.1.1 0 -1 -1 slow
2699		chk_join_nr 2 2 2
2700		chk_add_nr 1 1 1
2701		chk_rm_nr 1 1
2702	fi
2703
2704	# subflows and signal with port, flush
2705	if reset "flush subflows and signal with port"; then
2706		pm_nl_set_limits $ns1 0 3
2707		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2708		pm_nl_set_limits $ns2 1 3
2709		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2710		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2711		run_tests $ns1 $ns2 10.0.1.1 0 -8 -2 slow
2712		chk_join_nr 3 3 3
2713		chk_add_nr 1 1
2714		chk_rm_nr 1 3 invert simult
2715	fi
2716
2717	# multiple addresses with port
2718	if reset "multiple addresses with port"; then
2719		pm_nl_set_limits $ns1 2 2
2720		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2721		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal port 10100
2722		pm_nl_set_limits $ns2 2 2
2723		run_tests $ns1 $ns2 10.0.1.1
2724		chk_join_nr 2 2 2
2725		chk_add_nr 2 2 2
2726	fi
2727
2728	# multiple addresses with ports
2729	if reset "multiple addresses with ports"; then
2730		pm_nl_set_limits $ns1 2 2
2731		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2732		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal port 10101
2733		pm_nl_set_limits $ns2 2 2
2734		run_tests $ns1 $ns2 10.0.1.1
2735		chk_join_nr 2 2 2
2736		chk_add_nr 2 2 2
2737	fi
2738}
2739
2740syncookies_tests()
2741{
2742	# single subflow, syncookies
2743	if reset_with_cookies "single subflow with syn cookies"; then
2744		pm_nl_set_limits $ns1 0 1
2745		pm_nl_set_limits $ns2 0 1
2746		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2747		run_tests $ns1 $ns2 10.0.1.1
2748		chk_join_nr 1 1 1
2749	fi
2750
2751	# multiple subflows with syn cookies
2752	if reset_with_cookies "multiple subflows with syn cookies"; then
2753		pm_nl_set_limits $ns1 0 2
2754		pm_nl_set_limits $ns2 0 2
2755		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2756		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
2757		run_tests $ns1 $ns2 10.0.1.1
2758		chk_join_nr 2 2 2
2759	fi
2760
2761	# multiple subflows limited by server
2762	if reset_with_cookies "subflows limited by server w cookies"; then
2763		pm_nl_set_limits $ns1 0 1
2764		pm_nl_set_limits $ns2 0 2
2765		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2766		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
2767		run_tests $ns1 $ns2 10.0.1.1
2768		chk_join_nr 2 1 1
2769	fi
2770
2771	# test signal address with cookies
2772	if reset_with_cookies "signal address with syn cookies"; then
2773		pm_nl_set_limits $ns1 0 1
2774		pm_nl_set_limits $ns2 1 1
2775		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2776		run_tests $ns1 $ns2 10.0.1.1
2777		chk_join_nr 1 1 1
2778		chk_add_nr 1 1
2779	fi
2780
2781	# test cookie with subflow and signal
2782	if reset_with_cookies "subflow and signal w cookies"; then
2783		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2784		pm_nl_set_limits $ns1 0 2
2785		pm_nl_set_limits $ns2 1 2
2786		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2787		run_tests $ns1 $ns2 10.0.1.1
2788		chk_join_nr 2 2 2
2789		chk_add_nr 1 1
2790	fi
2791
2792	# accept and use add_addr with additional subflows
2793	if reset_with_cookies "subflows and signal w. cookies"; then
2794		pm_nl_set_limits $ns1 0 3
2795		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2796		pm_nl_set_limits $ns2 1 3
2797		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2798		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2799		run_tests $ns1 $ns2 10.0.1.1
2800		chk_join_nr 3 3 3
2801		chk_add_nr 1 1
2802	fi
2803}
2804
2805checksum_tests()
2806{
2807	# checksum test 0 0
2808	if reset_with_checksum 0 0; then
2809		pm_nl_set_limits $ns1 0 1
2810		pm_nl_set_limits $ns2 0 1
2811		run_tests $ns1 $ns2 10.0.1.1
2812		chk_join_nr 0 0 0
2813	fi
2814
2815	# checksum test 1 1
2816	if reset_with_checksum 1 1; then
2817		pm_nl_set_limits $ns1 0 1
2818		pm_nl_set_limits $ns2 0 1
2819		run_tests $ns1 $ns2 10.0.1.1
2820		chk_join_nr 0 0 0
2821	fi
2822
2823	# checksum test 0 1
2824	if reset_with_checksum 0 1; then
2825		pm_nl_set_limits $ns1 0 1
2826		pm_nl_set_limits $ns2 0 1
2827		run_tests $ns1 $ns2 10.0.1.1
2828		chk_join_nr 0 0 0
2829	fi
2830
2831	# checksum test 1 0
2832	if reset_with_checksum 1 0; then
2833		pm_nl_set_limits $ns1 0 1
2834		pm_nl_set_limits $ns2 0 1
2835		run_tests $ns1 $ns2 10.0.1.1
2836		chk_join_nr 0 0 0
2837	fi
2838}
2839
2840deny_join_id0_tests()
2841{
2842	# subflow allow join id0 ns1
2843	if reset_with_allow_join_id0 "single subflow allow join id0 ns1" 1 0; then
2844		pm_nl_set_limits $ns1 1 1
2845		pm_nl_set_limits $ns2 1 1
2846		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2847		run_tests $ns1 $ns2 10.0.1.1
2848		chk_join_nr 1 1 1
2849	fi
2850
2851	# subflow allow join id0 ns2
2852	if reset_with_allow_join_id0 "single subflow allow join id0 ns2" 0 1; then
2853		pm_nl_set_limits $ns1 1 1
2854		pm_nl_set_limits $ns2 1 1
2855		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2856		run_tests $ns1 $ns2 10.0.1.1
2857		chk_join_nr 0 0 0
2858	fi
2859
2860	# signal address allow join id0 ns1
2861	# ADD_ADDRs are not affected by allow_join_id0 value.
2862	if reset_with_allow_join_id0 "signal address allow join id0 ns1" 1 0; then
2863		pm_nl_set_limits $ns1 1 1
2864		pm_nl_set_limits $ns2 1 1
2865		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2866		run_tests $ns1 $ns2 10.0.1.1
2867		chk_join_nr 1 1 1
2868		chk_add_nr 1 1
2869	fi
2870
2871	# signal address allow join id0 ns2
2872	# ADD_ADDRs are not affected by allow_join_id0 value.
2873	if reset_with_allow_join_id0 "signal address allow join id0 ns2" 0 1; then
2874		pm_nl_set_limits $ns1 1 1
2875		pm_nl_set_limits $ns2 1 1
2876		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2877		run_tests $ns1 $ns2 10.0.1.1
2878		chk_join_nr 1 1 1
2879		chk_add_nr 1 1
2880	fi
2881
2882	# subflow and address allow join id0 ns1
2883	if reset_with_allow_join_id0 "subflow and address allow join id0 1" 1 0; then
2884		pm_nl_set_limits $ns1 2 2
2885		pm_nl_set_limits $ns2 2 2
2886		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2887		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2888		run_tests $ns1 $ns2 10.0.1.1
2889		chk_join_nr 2 2 2
2890	fi
2891
2892	# subflow and address allow join id0 ns2
2893	if reset_with_allow_join_id0 "subflow and address allow join id0 2" 0 1; then
2894		pm_nl_set_limits $ns1 2 2
2895		pm_nl_set_limits $ns2 2 2
2896		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2897		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2898		run_tests $ns1 $ns2 10.0.1.1
2899		chk_join_nr 1 1 1
2900	fi
2901}
2902
2903fullmesh_tests()
2904{
2905	# fullmesh 1
2906	# 2 fullmesh addrs in ns2, added before the connection,
2907	# 1 non-fullmesh addr in ns1, added during the connection.
2908	if reset "fullmesh test 2x1"; then
2909		pm_nl_set_limits $ns1 0 4
2910		pm_nl_set_limits $ns2 1 4
2911		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow,fullmesh
2912		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow,fullmesh
2913		run_tests $ns1 $ns2 10.0.1.1 0 1 0 slow
2914		chk_join_nr 4 4 4
2915		chk_add_nr 1 1
2916	fi
2917
2918	# fullmesh 2
2919	# 1 non-fullmesh addr in ns1, added before the connection,
2920	# 1 fullmesh addr in ns2, added during the connection.
2921	if reset "fullmesh test 1x1"; then
2922		pm_nl_set_limits $ns1 1 3
2923		pm_nl_set_limits $ns2 1 3
2924		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2925		run_tests $ns1 $ns2 10.0.1.1 0 0 fullmesh_1 slow
2926		chk_join_nr 3 3 3
2927		chk_add_nr 1 1
2928	fi
2929
2930	# fullmesh 3
2931	# 1 non-fullmesh addr in ns1, added before the connection,
2932	# 2 fullmesh addrs in ns2, added during the connection.
2933	if reset "fullmesh test 1x2"; then
2934		pm_nl_set_limits $ns1 2 5
2935		pm_nl_set_limits $ns2 1 5
2936		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2937		run_tests $ns1 $ns2 10.0.1.1 0 0 fullmesh_2 slow
2938		chk_join_nr 5 5 5
2939		chk_add_nr 1 1
2940	fi
2941
2942	# fullmesh 4
2943	# 1 non-fullmesh addr in ns1, added before the connection,
2944	# 2 fullmesh addrs in ns2, added during the connection,
2945	# limit max_subflows to 4.
2946	if reset "fullmesh test 1x2, limited"; then
2947		pm_nl_set_limits $ns1 2 4
2948		pm_nl_set_limits $ns2 1 4
2949		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2950		run_tests $ns1 $ns2 10.0.1.1 0 0 fullmesh_2 slow
2951		chk_join_nr 4 4 4
2952		chk_add_nr 1 1
2953	fi
2954
2955	# set fullmesh flag
2956	if reset "set fullmesh flag test"; then
2957		pm_nl_set_limits $ns1 4 4
2958		pm_nl_add_endpoint $ns1 10.0.2.1 flags subflow
2959		pm_nl_set_limits $ns2 4 4
2960		run_tests $ns1 $ns2 10.0.1.1 0 0 1 slow fullmesh
2961		chk_join_nr 2 2 2
2962		chk_rm_nr 0 1
2963	fi
2964
2965	# set nofullmesh flag
2966	if reset "set nofullmesh flag test"; then
2967		pm_nl_set_limits $ns1 4 4
2968		pm_nl_add_endpoint $ns1 10.0.2.1 flags subflow,fullmesh
2969		pm_nl_set_limits $ns2 4 4
2970		run_tests $ns1 $ns2 10.0.1.1 0 0 fullmesh_1 slow nofullmesh
2971		chk_join_nr 2 2 2
2972		chk_rm_nr 0 1
2973	fi
2974
2975	# set backup,fullmesh flags
2976	if reset "set backup,fullmesh flags test"; then
2977		pm_nl_set_limits $ns1 4 4
2978		pm_nl_add_endpoint $ns1 10.0.2.1 flags subflow
2979		pm_nl_set_limits $ns2 4 4
2980		run_tests $ns1 $ns2 10.0.1.1 0 0 1 slow backup,fullmesh
2981		chk_join_nr 2 2 2
2982		chk_prio_nr 0 1
2983		chk_rm_nr 0 1
2984	fi
2985
2986	# set nobackup,nofullmesh flags
2987	if reset "set nobackup,nofullmesh flags test"; then
2988		pm_nl_set_limits $ns1 4 4
2989		pm_nl_set_limits $ns2 4 4
2990		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow,backup,fullmesh
2991		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow nobackup,nofullmesh
2992		chk_join_nr 2 2 2
2993		chk_prio_nr 0 1
2994		chk_rm_nr 0 1
2995	fi
2996}
2997
2998fastclose_tests()
2999{
3000	if reset "fastclose test"; then
3001		run_tests $ns1 $ns2 10.0.1.1 1024 0 fastclose_client
3002		chk_join_nr 0 0 0
3003		chk_fclose_nr 1 1
3004		chk_rst_nr 1 1 invert
3005	fi
3006
3007	if reset "fastclose server test"; then
3008		run_tests $ns1 $ns2 10.0.1.1 1024 0 fastclose_server
3009		chk_join_nr 0 0 0
3010		chk_fclose_nr 1 1 invert
3011		chk_rst_nr 1 1
3012	fi
3013}
3014
3015pedit_action_pkts()
3016{
3017	tc -n $ns2 -j -s action show action pedit index 100 | \
3018		grep "packets" | \
3019		sed 's/.*"packets":\([0-9]\+\),.*/\1/'
3020}
3021
3022fail_tests()
3023{
3024	# single subflow
3025	if reset_with_fail "Infinite map" 1; then
3026		run_tests $ns1 $ns2 10.0.1.1 128
3027		chk_join_nr 0 0 0 +1 +0 1 0 1 "$(pedit_action_pkts)"
3028		chk_fail_nr 1 -1 invert
3029	fi
3030
3031	# multiple subflows
3032	if reset_with_fail "MP_FAIL MP_RST" 2; then
3033		tc -n $ns2 qdisc add dev ns2eth1 root netem rate 1mbit delay 5
3034		pm_nl_set_limits $ns1 0 1
3035		pm_nl_set_limits $ns2 0 1
3036		pm_nl_add_endpoint $ns2 10.0.2.2 dev ns2eth2 flags subflow
3037		run_tests $ns1 $ns2 10.0.1.1 1024
3038		chk_join_nr 1 1 1 1 0 1 1 0 "$(pedit_action_pkts)"
3039	fi
3040}
3041
3042userspace_tests()
3043{
3044	# userspace pm type prevents add_addr
3045	if reset "userspace pm type prevents add_addr"; then
3046		set_userspace_pm $ns1
3047		pm_nl_set_limits $ns1 0 2
3048		pm_nl_set_limits $ns2 0 2
3049		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
3050		run_tests $ns1 $ns2 10.0.1.1
3051		chk_join_nr 0 0 0
3052		chk_add_nr 0 0
3053	fi
3054
3055	# userspace pm type does not echo add_addr without daemon
3056	if reset "userspace pm no echo w/o daemon"; then
3057		set_userspace_pm $ns2
3058		pm_nl_set_limits $ns1 0 2
3059		pm_nl_set_limits $ns2 0 2
3060		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
3061		run_tests $ns1 $ns2 10.0.1.1
3062		chk_join_nr 0 0 0
3063		chk_add_nr 1 0
3064	fi
3065
3066	# userspace pm type rejects join
3067	if reset "userspace pm type rejects join"; then
3068		set_userspace_pm $ns1
3069		pm_nl_set_limits $ns1 1 1
3070		pm_nl_set_limits $ns2 1 1
3071		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
3072		run_tests $ns1 $ns2 10.0.1.1
3073		chk_join_nr 1 1 0
3074	fi
3075
3076	# userspace pm type does not send join
3077	if reset "userspace pm type does not send join"; then
3078		set_userspace_pm $ns2
3079		pm_nl_set_limits $ns1 1 1
3080		pm_nl_set_limits $ns2 1 1
3081		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
3082		run_tests $ns1 $ns2 10.0.1.1
3083		chk_join_nr 0 0 0
3084	fi
3085
3086	# userspace pm type prevents mp_prio
3087	if reset "userspace pm type prevents mp_prio"; then
3088		set_userspace_pm $ns1
3089		pm_nl_set_limits $ns1 1 1
3090		pm_nl_set_limits $ns2 1 1
3091		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
3092		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
3093		chk_join_nr 1 1 0
3094		chk_prio_nr 0 0
3095	fi
3096
3097	# userspace pm type prevents rm_addr
3098	if reset "userspace pm type prevents rm_addr"; then
3099		set_userspace_pm $ns1
3100		set_userspace_pm $ns2
3101		pm_nl_set_limits $ns1 0 1
3102		pm_nl_set_limits $ns2 0 1
3103		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
3104		run_tests $ns1 $ns2 10.0.1.1 0 0 -1 slow
3105		chk_join_nr 0 0 0
3106		chk_rm_nr 0 0
3107	fi
3108
3109	# userspace pm add & remove address
3110	if reset_with_events "userspace pm add & remove address"; then
3111		set_userspace_pm $ns1
3112		pm_nl_set_limits $ns2 1 1
3113		run_tests $ns1 $ns2 10.0.1.1 0 userspace_1 0 slow
3114		chk_join_nr 1 1 1
3115		chk_add_nr 1 1
3116		chk_rm_nr 1 1 invert
3117		kill_events_pids
3118	fi
3119
3120	# userspace pm create destroy subflow
3121	if reset_with_events "userspace pm create destroy subflow"; then
3122		set_userspace_pm $ns2
3123		pm_nl_set_limits $ns1 0 1
3124		run_tests $ns1 $ns2 10.0.1.1 0 0 userspace_1 slow
3125		chk_join_nr 1 1 1
3126		chk_rm_nr 0 1
3127		kill_events_pids
3128	fi
3129}
3130
3131endpoint_tests()
3132{
3133	# userspace pm type prevents add_addr
3134	if reset "implicit EP"; then
3135		pm_nl_set_limits $ns1 2 2
3136		pm_nl_set_limits $ns2 2 2
3137		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
3138		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow 2>/dev/null &
3139
3140		wait_mpj $ns1
3141		pm_nl_check_endpoint 1 "creation" \
3142			$ns2 10.0.2.2 id 1 flags implicit
3143
3144		pm_nl_add_endpoint $ns2 10.0.2.2 id 33
3145		pm_nl_check_endpoint 0 "ID change is prevented" \
3146			$ns2 10.0.2.2 id 1 flags implicit
3147
3148		pm_nl_add_endpoint $ns2 10.0.2.2 flags signal
3149		pm_nl_check_endpoint 0 "modif is allowed" \
3150			$ns2 10.0.2.2 id 1 flags signal
3151		kill_tests_wait
3152	fi
3153
3154	if reset "delete and re-add"; then
3155		pm_nl_set_limits $ns1 1 1
3156		pm_nl_set_limits $ns2 1 1
3157		pm_nl_add_endpoint $ns2 10.0.2.2 id 2 dev ns2eth2 flags subflow
3158		run_tests $ns1 $ns2 10.0.1.1 4 0 0 speed_20 2>/dev/null &
3159
3160		wait_mpj $ns2
3161		chk_subflow_nr needtitle "before delete" 2
3162		chk_mptcp_info subflows_1
3163
3164		pm_nl_del_endpoint $ns2 2 10.0.2.2
3165		sleep 0.5
3166		chk_subflow_nr "" "after delete" 1
3167		chk_mptcp_info subflows_0
3168
3169		pm_nl_add_endpoint $ns2 10.0.2.2 dev ns2eth2 flags subflow
3170		wait_mpj $ns2
3171		chk_subflow_nr "" "after re-add" 2
3172		chk_mptcp_info subflows_1
3173		kill_tests_wait
3174	fi
3175}
3176
3177# [$1: error message]
3178usage()
3179{
3180	if [ -n "${1}" ]; then
3181		echo "${1}"
3182		ret=1
3183	fi
3184
3185	echo "mptcp_join usage:"
3186
3187	local key
3188	for key in "${!all_tests[@]}"; do
3189		echo "  -${key} ${all_tests[${key}]}"
3190	done
3191
3192	echo "  -c capture pcap files"
3193	echo "  -C enable data checksum"
3194	echo "  -i use ip mptcp"
3195	echo "  -h help"
3196
3197	echo "[test ids|names]"
3198
3199	exit ${ret}
3200}
3201
3202
3203# Use a "simple" array to force an specific order we cannot have with an associative one
3204all_tests_sorted=(
3205	f@subflows_tests
3206	e@subflows_error_tests
3207	s@signal_address_tests
3208	l@link_failure_tests
3209	t@add_addr_timeout_tests
3210	r@remove_tests
3211	a@add_tests
3212	6@ipv6_tests
3213	4@v4mapped_tests
3214	M@mixed_tests
3215	b@backup_tests
3216	p@add_addr_ports_tests
3217	k@syncookies_tests
3218	S@checksum_tests
3219	d@deny_join_id0_tests
3220	m@fullmesh_tests
3221	z@fastclose_tests
3222	F@fail_tests
3223	u@userspace_tests
3224	I@endpoint_tests
3225)
3226
3227all_tests_args=""
3228all_tests_names=()
3229for subtests in "${all_tests_sorted[@]}"; do
3230	key="${subtests%@*}"
3231	value="${subtests#*@}"
3232
3233	all_tests_args+="${key}"
3234	all_tests_names+=("${value}")
3235	all_tests[${key}]="${value}"
3236done
3237
3238tests=()
3239while getopts "${all_tests_args}cCih" opt; do
3240	case $opt in
3241		["${all_tests_args}"])
3242			tests+=("${all_tests[${opt}]}")
3243			;;
3244		c)
3245			capture=1
3246			;;
3247		C)
3248			checksum=1
3249			;;
3250		i)
3251			ip_mptcp=1
3252			;;
3253		h)
3254			usage
3255			;;
3256		*)
3257			usage "Unknown option: -${opt}"
3258			;;
3259	esac
3260done
3261
3262shift $((OPTIND - 1))
3263
3264for arg in "${@}"; do
3265	if [[ "${arg}" =~ ^[0-9]+$ ]]; then
3266		only_tests_ids+=("${arg}")
3267	else
3268		only_tests_names+=("${arg}")
3269	fi
3270done
3271
3272if [ ${#tests[@]} -eq 0 ]; then
3273	tests=("${all_tests_names[@]}")
3274fi
3275
3276for subtests in "${tests[@]}"; do
3277	"${subtests}"
3278done
3279
3280if [ ${ret} -ne 0 ]; then
3281	echo
3282	echo "${#failed_tests[@]} failure(s) has(ve) been detected:"
3283	for i in $(get_failed_tests_ids); do
3284		echo -e "\t- ${i}: ${failed_tests[${i}]}"
3285	done
3286	echo
3287fi
3288
3289exit $ret
3290