1#!/bin/bash
2# SPDX-License-Identifier: GPL-2.0
3
4# Double quotes to prevent globbing and word splitting is recommended in new
5# code but we accept it, especially because there were too many before having
6# address all other issues detected by shellcheck.
7#shellcheck disable=SC2086
8
9ret=0
10sin=""
11sinfail=""
12sout=""
13cin=""
14cinfail=""
15cinsent=""
16cout=""
17capout=""
18ns1=""
19ns2=""
20ksft_skip=4
21timeout_poll=30
22timeout_test=$((timeout_poll * 2 + 1))
23capture=0
24checksum=0
25ip_mptcp=0
26check_invert=0
27validate_checksum=0
28init=0
29
30declare -A all_tests
31declare -a only_tests_ids
32declare -a only_tests_names
33declare -A failed_tests
34TEST_COUNT=0
35TEST_NAME=""
36nr_blank=40
37
38export FAILING_LINKS=""
39
40# generated using "nfbpf_compile '(ip && (ip[54] & 0xf0) == 0x30) ||
41#				  (ip6 && (ip6[74] & 0xf0) == 0x30)'"
42CBPF_MPTCP_SUBOPTION_ADD_ADDR="14,
43			       48 0 0 0,
44			       84 0 0 240,
45			       21 0 3 64,
46			       48 0 0 54,
47			       84 0 0 240,
48			       21 6 7 48,
49			       48 0 0 0,
50			       84 0 0 240,
51			       21 0 4 96,
52			       48 0 0 74,
53			       84 0 0 240,
54			       21 0 1 48,
55			       6 0 0 65535,
56			       6 0 0 0"
57
58init_partial()
59{
60	capout=$(mktemp)
61
62	local rndh
63	rndh=$(mktemp -u XXXXXX)
64
65	ns1="ns1-$rndh"
66	ns2="ns2-$rndh"
67
68	local netns
69	for netns in "$ns1" "$ns2"; do
70		ip netns add $netns || exit $ksft_skip
71		ip -net $netns link set lo up
72		ip netns exec $netns sysctl -q net.mptcp.enabled=1
73		ip netns exec $netns sysctl -q net.mptcp.pm_type=0
74		ip netns exec $netns sysctl -q net.ipv4.conf.all.rp_filter=0
75		ip netns exec $netns sysctl -q net.ipv4.conf.default.rp_filter=0
76		if [ $checksum -eq 1 ]; then
77			ip netns exec $netns sysctl -q net.mptcp.checksum_enabled=1
78		fi
79	done
80
81	check_invert=0
82	validate_checksum=$checksum
83	FAILING_LINKS=""
84
85	#  ns1         ns2
86	# ns1eth1    ns2eth1
87	# ns1eth2    ns2eth2
88	# ns1eth3    ns2eth3
89	# ns1eth4    ns2eth4
90
91	local i
92	for i in $(seq 1 4); do
93		ip link add ns1eth$i netns "$ns1" type veth peer name ns2eth$i netns "$ns2"
94		ip -net "$ns1" addr add 10.0.$i.1/24 dev ns1eth$i
95		ip -net "$ns1" addr add dead:beef:$i::1/64 dev ns1eth$i nodad
96		ip -net "$ns1" link set ns1eth$i up
97
98		ip -net "$ns2" addr add 10.0.$i.2/24 dev ns2eth$i
99		ip -net "$ns2" addr add dead:beef:$i::2/64 dev ns2eth$i nodad
100		ip -net "$ns2" link set ns2eth$i up
101
102		# let $ns2 reach any $ns1 address from any interface
103		ip -net "$ns2" route add default via 10.0.$i.1 dev ns2eth$i metric 10$i
104		ip -net "$ns2" route add default via dead:beef:$i::1 dev ns2eth$i metric 10$i
105	done
106}
107
108init_shapers()
109{
110	local i
111	for i in $(seq 1 4); do
112		tc -n $ns1 qdisc add dev ns1eth$i root netem rate 20mbit delay 1
113		tc -n $ns2 qdisc add dev ns2eth$i root netem rate 20mbit delay 1
114	done
115}
116
117cleanup_partial()
118{
119	rm -f "$capout"
120
121	local netns
122	for netns in "$ns1" "$ns2"; do
123		ip netns del $netns
124		rm -f /tmp/$netns.{nstat,out}
125	done
126}
127
128check_tools()
129{
130	if ! ip -Version &> /dev/null; then
131		echo "SKIP: Could not run test without ip tool"
132		exit $ksft_skip
133	fi
134
135	if ! iptables -V &> /dev/null; then
136		echo "SKIP: Could not run all tests without iptables tool"
137		exit $ksft_skip
138	fi
139
140	if ! ip6tables -V &> /dev/null; then
141		echo "SKIP: Could not run all tests without ip6tables tool"
142		exit $ksft_skip
143	fi
144}
145
146init() {
147	init=1
148
149	check_tools
150
151	sin=$(mktemp)
152	sout=$(mktemp)
153	cin=$(mktemp)
154	cinsent=$(mktemp)
155	cout=$(mktemp)
156
157	trap cleanup EXIT
158
159	make_file "$cin" "client" 1
160	make_file "$sin" "server" 1
161}
162
163cleanup()
164{
165	rm -f "$cin" "$cout" "$sinfail"
166	rm -f "$sin" "$sout" "$cinsent" "$cinfail"
167	cleanup_partial
168}
169
170skip_test()
171{
172	if [ "${#only_tests_ids[@]}" -eq 0 ] && [ "${#only_tests_names[@]}" -eq 0 ]; then
173		return 1
174	fi
175
176	local i
177	for i in "${only_tests_ids[@]}"; do
178		if [ "${TEST_COUNT}" -eq "${i}" ]; then
179			return 1
180		fi
181	done
182	for i in "${only_tests_names[@]}"; do
183		if [ "${TEST_NAME}" = "${i}" ]; then
184			return 1
185		fi
186	done
187
188	return 0
189}
190
191# $1: test name
192reset()
193{
194	TEST_NAME="${1}"
195
196	TEST_COUNT=$((TEST_COUNT+1))
197
198	if skip_test; then
199		return 1
200	fi
201
202	if [ "${init}" != "1" ]; then
203		init
204	else
205		cleanup_partial
206	fi
207
208	init_partial
209
210	return 0
211}
212
213# $1: test name
214reset_with_cookies()
215{
216	reset "${1}" || return 1
217
218	local netns
219	for netns in "$ns1" "$ns2"; do
220		ip netns exec $netns sysctl -q net.ipv4.tcp_syncookies=2
221	done
222}
223
224# $1: test name
225reset_with_add_addr_timeout()
226{
227	local ip="${2:-4}"
228	local tables
229
230	reset "${1}" || return 1
231
232	tables="iptables"
233	if [ $ip -eq 6 ]; then
234		tables="ip6tables"
235	fi
236
237	ip netns exec $ns1 sysctl -q net.mptcp.add_addr_timeout=1
238	ip netns exec $ns2 $tables -A OUTPUT -p tcp \
239		-m tcp --tcp-option 30 \
240		-m bpf --bytecode \
241		"$CBPF_MPTCP_SUBOPTION_ADD_ADDR" \
242		-j DROP
243}
244
245# $1: test name
246reset_with_checksum()
247{
248	local ns1_enable=$1
249	local ns2_enable=$2
250
251	reset "checksum test ${1} ${2}" || return 1
252
253	ip netns exec $ns1 sysctl -q net.mptcp.checksum_enabled=$ns1_enable
254	ip netns exec $ns2 sysctl -q net.mptcp.checksum_enabled=$ns2_enable
255
256	validate_checksum=1
257}
258
259reset_with_allow_join_id0()
260{
261	local ns1_enable=$2
262	local ns2_enable=$3
263
264	reset "${1}" || return 1
265
266	ip netns exec $ns1 sysctl -q net.mptcp.allow_join_initial_addr_port=$ns1_enable
267	ip netns exec $ns2 sysctl -q net.mptcp.allow_join_initial_addr_port=$ns2_enable
268}
269
270# Modify TCP payload without corrupting the TCP packet
271#
272# This rule inverts a 8-bit word at byte offset 148 for the 2nd TCP ACK packets
273# carrying enough data.
274# Once it is done, the TCP Checksum field is updated so the packet is still
275# considered as valid at the TCP level.
276# Because the MPTCP checksum, covering the TCP options and data, has not been
277# updated, the modification will be detected and an MP_FAIL will be emitted:
278# what we want to validate here without corrupting "random" MPTCP options.
279#
280# To avoid having tc producing this pr_info() message for each TCP ACK packets
281# not carrying enough data:
282#
283#     tc action pedit offset 162 out of bounds
284#
285# Netfilter is used to mark packets with enough data.
286reset_with_fail()
287{
288	reset "${1}" || return 1
289
290	ip netns exec $ns1 sysctl -q net.mptcp.checksum_enabled=1
291	ip netns exec $ns2 sysctl -q net.mptcp.checksum_enabled=1
292
293	check_invert=1
294	validate_checksum=1
295	local i="$2"
296	local ip="${3:-4}"
297	local tables
298
299	tables="iptables"
300	if [ $ip -eq 6 ]; then
301		tables="ip6tables"
302	fi
303
304	ip netns exec $ns2 $tables \
305		-t mangle \
306		-A OUTPUT \
307		-o ns2eth$i \
308		-p tcp \
309		-m length --length 150:9999 \
310		-m statistic --mode nth --packet 1 --every 99999 \
311		-j MARK --set-mark 42 || exit 1
312
313	tc -n $ns2 qdisc add dev ns2eth$i clsact || exit 1
314	tc -n $ns2 filter add dev ns2eth$i egress \
315		protocol ip prio 1000 \
316		handle 42 fw \
317		action pedit munge offset 148 u8 invert \
318		pipe csum tcp \
319		index 100 || exit 1
320}
321
322fail_test()
323{
324	ret=1
325	failed_tests[${TEST_COUNT}]="${TEST_NAME}"
326}
327
328get_failed_tests_ids()
329{
330	# sorted
331	local i
332	for i in "${!failed_tests[@]}"; do
333		echo "${i}"
334	done | sort -n
335}
336
337print_file_err()
338{
339	ls -l "$1" 1>&2
340	echo "Trailing bytes are: "
341	tail -c 27 "$1"
342}
343
344check_transfer()
345{
346	local in=$1
347	local out=$2
348	local what=$3
349	local i a b
350
351	local line
352	cmp -l "$in" "$out" | while read -r i a b; do
353		local sum=$((0${a} + 0${b}))
354		if [ $check_invert -eq 0 ] || [ $sum -ne $((0xff)) ]; then
355			echo "[ FAIL ] $what does not match (in, out):"
356			print_file_err "$in"
357			print_file_err "$out"
358			fail_test
359
360			return 1
361		else
362			echo "$what has inverted byte at ${i}"
363		fi
364	done
365
366	return 0
367}
368
369do_ping()
370{
371	local listener_ns="$1"
372	local connector_ns="$2"
373	local connect_addr="$3"
374
375	if ! ip netns exec ${connector_ns} ping -q -c 1 $connect_addr >/dev/null; then
376		echo "$listener_ns -> $connect_addr connectivity [ FAIL ]" 1>&2
377		fail_test
378	fi
379}
380
381link_failure()
382{
383	local ns="$1"
384
385	if [ -z "$FAILING_LINKS" ]; then
386		l=$((RANDOM%4))
387		FAILING_LINKS=$((l+1))
388	fi
389
390	local l
391	for l in $FAILING_LINKS; do
392		local veth="ns1eth$l"
393		ip -net "$ns" link set "$veth" down
394	done
395}
396
397# $1: IP address
398is_v6()
399{
400	[ -z "${1##*:*}" ]
401}
402
403# $1: ns, $2: port
404wait_local_port_listen()
405{
406	local listener_ns="${1}"
407	local port="${2}"
408
409	local port_hex
410	port_hex="$(printf "%04X" "${port}")"
411
412	local i
413	for i in $(seq 10); do
414		ip netns exec "${listener_ns}" cat /proc/net/tcp* | \
415			awk "BEGIN {rc=1} {if (\$2 ~ /:${port_hex}\$/ && \$4 ~ /0A/) {rc=0; exit}} END {exit rc}" &&
416			break
417		sleep 0.1
418	done
419}
420
421rm_addr_count()
422{
423	local ns=${1}
424
425	ip netns exec ${ns} nstat -as | grep MPTcpExtRmAddr | awk '{print $2}'
426}
427
428# $1: ns, $2: old rm_addr counter in $ns
429wait_rm_addr()
430{
431	local ns="${1}"
432	local old_cnt="${2}"
433	local cnt
434
435	local i
436	for i in $(seq 10); do
437		cnt=$(rm_addr_count ${ns})
438		[ "$cnt" = "${old_cnt}" ] || break
439		sleep 0.1
440	done
441}
442
443wait_mpj()
444{
445	local ns="${1}"
446	local cnt old_cnt
447
448	old_cnt=$(ip netns exec ${ns} nstat -as | grep MPJoinAckRx | awk '{print $2}')
449
450	local i
451	for i in $(seq 10); do
452		cnt=$(ip netns exec ${ns} nstat -as | grep MPJoinAckRx | awk '{print $2}')
453		[ "$cnt" = "${old_cnt}" ] || break
454		sleep 0.1
455	done
456}
457
458kill_wait()
459{
460	kill $1 > /dev/null 2>&1
461	wait $1 2>/dev/null
462}
463
464pm_nl_set_limits()
465{
466	local ns=$1
467	local addrs=$2
468	local subflows=$3
469
470	if [ $ip_mptcp -eq 1 ]; then
471		ip -n $ns mptcp limits set add_addr_accepted $addrs subflows $subflows
472	else
473		ip netns exec $ns ./pm_nl_ctl limits $addrs $subflows
474	fi
475}
476
477pm_nl_add_endpoint()
478{
479	local ns=$1
480	local addr=$2
481	local flags _flags
482	local port _port
483	local dev _dev
484	local id _id
485	local nr=2
486
487	local p
488	for p in "${@}"
489	do
490		if [ $p = "flags" ]; then
491			eval _flags=\$"$nr"
492			[ -n "$_flags" ]; flags="flags $_flags"
493		fi
494		if [ $p = "dev" ]; then
495			eval _dev=\$"$nr"
496			[ -n "$_dev" ]; dev="dev $_dev"
497		fi
498		if [ $p = "id" ]; then
499			eval _id=\$"$nr"
500			[ -n "$_id" ]; id="id $_id"
501		fi
502		if [ $p = "port" ]; then
503			eval _port=\$"$nr"
504			[ -n "$_port" ]; port="port $_port"
505		fi
506
507		nr=$((nr + 1))
508	done
509
510	if [ $ip_mptcp -eq 1 ]; then
511		ip -n $ns mptcp endpoint add $addr ${_flags//","/" "} $dev $id $port
512	else
513		ip netns exec $ns ./pm_nl_ctl add $addr $flags $dev $id $port
514	fi
515}
516
517pm_nl_del_endpoint()
518{
519	local ns=$1
520	local id=$2
521	local addr=$3
522
523	if [ $ip_mptcp -eq 1 ]; then
524		ip -n $ns mptcp endpoint delete id $id $addr
525	else
526		ip netns exec $ns ./pm_nl_ctl del $id $addr
527	fi
528}
529
530pm_nl_flush_endpoint()
531{
532	local ns=$1
533
534	if [ $ip_mptcp -eq 1 ]; then
535		ip -n $ns mptcp endpoint flush
536	else
537		ip netns exec $ns ./pm_nl_ctl flush
538	fi
539}
540
541pm_nl_show_endpoints()
542{
543	local ns=$1
544
545	if [ $ip_mptcp -eq 1 ]; then
546		ip -n $ns mptcp endpoint show
547	else
548		ip netns exec $ns ./pm_nl_ctl dump
549	fi
550}
551
552pm_nl_change_endpoint()
553{
554	local ns=$1
555	local id=$2
556	local flags=$3
557
558	if [ $ip_mptcp -eq 1 ]; then
559		ip -n $ns mptcp endpoint change id $id ${flags//","/" "}
560	else
561		ip netns exec $ns ./pm_nl_ctl set id $id flags $flags
562	fi
563}
564
565pm_nl_check_endpoint()
566{
567	local line expected_line
568	local need_title=$1
569	local msg="$2"
570	local ns=$3
571	local addr=$4
572	local _flags=""
573	local flags
574	local _port
575	local port
576	local dev
577	local _id
578	local id
579
580	if [ "${need_title}" = 1 ]; then
581		printf "%03u %-36s %s" "${TEST_COUNT}" "${TEST_NAME}" "${msg}"
582	else
583		printf "%-${nr_blank}s %s" " " "${msg}"
584	fi
585
586	shift 4
587	while [ -n "$1" ]; do
588		if [ $1 = "flags" ]; then
589			_flags=$2
590			[ -n "$_flags" ]; flags="flags $_flags"
591			shift
592		elif [ $1 = "dev" ]; then
593			[ -n "$2" ]; dev="dev $1"
594			shift
595		elif [ $1 = "id" ]; then
596			_id=$2
597			[ -n "$_id" ]; id="id $_id"
598			shift
599		elif [ $1 = "port" ]; then
600			_port=$2
601			[ -n "$_port" ]; port=" port $_port"
602			shift
603		fi
604
605		shift
606	done
607
608	if [ -z "$id" ]; then
609		echo "[skip] bad test - missing endpoint id"
610		return
611	fi
612
613	if [ $ip_mptcp -eq 1 ]; then
614		line=$(ip -n $ns mptcp endpoint show $id)
615		# the dump order is: address id flags port dev
616		expected_line="$addr"
617		[ -n "$addr" ] && expected_line="$expected_line $addr"
618		expected_line="$expected_line $id"
619		[ -n "$_flags" ] && expected_line="$expected_line ${_flags//","/" "}"
620		[ -n "$dev" ] && expected_line="$expected_line $dev"
621		[ -n "$port" ] && expected_line="$expected_line $port"
622	else
623		line=$(ip netns exec $ns ./pm_nl_ctl get $_id)
624		# the dump order is: id flags dev address port
625		expected_line="$id"
626		[ -n "$flags" ] && expected_line="$expected_line $flags"
627		[ -n "$dev" ] && expected_line="$expected_line $dev"
628		[ -n "$addr" ] && expected_line="$expected_line $addr"
629		[ -n "$_port" ] && expected_line="$expected_line $_port"
630	fi
631	if [ "$line" = "$expected_line" ]; then
632		echo "[ ok ]"
633	else
634		echo "[fail] expected '$expected_line' found '$line'"
635		fail_test
636	fi
637}
638
639filter_tcp_from()
640{
641	local ns="${1}"
642	local src="${2}"
643	local target="${3}"
644
645	ip netns exec "${ns}" iptables -A INPUT -s "${src}" -p tcp -j "${target}"
646}
647
648do_transfer()
649{
650	local listener_ns="$1"
651	local connector_ns="$2"
652	local cl_proto="$3"
653	local srv_proto="$4"
654	local connect_addr="$5"
655	local test_link_fail="$6"
656	local addr_nr_ns1="$7"
657	local addr_nr_ns2="$8"
658	local speed="$9"
659	local sflags="${10}"
660
661	local port=$((10000 + TEST_COUNT - 1))
662	local cappid
663	local userspace_pm=0
664	local evts_ns1
665	local evts_ns1_pid
666	local evts_ns2
667	local evts_ns2_pid
668
669	:> "$cout"
670	:> "$sout"
671	:> "$capout"
672
673	if [ $capture -eq 1 ]; then
674		local capuser
675		if [ -z $SUDO_USER ] ; then
676			capuser=""
677		else
678			capuser="-Z $SUDO_USER"
679		fi
680
681		capfile=$(printf "mp_join-%02u-%s.pcap" "$TEST_COUNT" "${listener_ns}")
682
683		echo "Capturing traffic for test $TEST_COUNT into $capfile"
684		ip netns exec ${listener_ns} tcpdump -i any -s 65535 -B 32768 $capuser -w $capfile > "$capout" 2>&1 &
685		cappid=$!
686
687		sleep 1
688	fi
689
690	NSTAT_HISTORY=/tmp/${listener_ns}.nstat ip netns exec ${listener_ns} \
691		nstat -n
692	NSTAT_HISTORY=/tmp/${connector_ns}.nstat ip netns exec ${connector_ns} \
693		nstat -n
694
695	local extra_args
696	if [ $speed = "fast" ]; then
697		extra_args="-j"
698	elif [ $speed = "slow" ]; then
699		extra_args="-r 50"
700	elif [[ $speed = "speed_"* ]]; then
701		extra_args="-r ${speed:6}"
702	fi
703
704	if [[ "${addr_nr_ns1}" = "userspace_"* ]]; then
705		userspace_pm=1
706		addr_nr_ns1=${addr_nr_ns1:10}
707	fi
708
709	if [[ "${addr_nr_ns2}" = "fastclose_"* ]]; then
710		# disconnect
711		extra_args="$extra_args -I ${addr_nr_ns2:10}"
712		addr_nr_ns2=0
713	elif [[ "${addr_nr_ns2}" = "userspace_"* ]]; then
714		userspace_pm=1
715		addr_nr_ns2=${addr_nr_ns2:10}
716	fi
717
718	if [ $userspace_pm -eq 1 ]; then
719		evts_ns1=$(mktemp)
720		evts_ns2=$(mktemp)
721		:> "$evts_ns1"
722		:> "$evts_ns2"
723		ip netns exec ${listener_ns} ./pm_nl_ctl events >> "$evts_ns1" 2>&1 &
724		evts_ns1_pid=$!
725		ip netns exec ${connector_ns} ./pm_nl_ctl events >> "$evts_ns2" 2>&1 &
726		evts_ns2_pid=$!
727	fi
728
729	local local_addr
730	if is_v6 "${connect_addr}"; then
731		local_addr="::"
732	else
733		local_addr="0.0.0.0"
734	fi
735
736	if [ "$test_link_fail" -gt 1 ];then
737		timeout ${timeout_test} \
738			ip netns exec ${listener_ns} \
739				./mptcp_connect -t ${timeout_poll} -l -p $port -s ${srv_proto} \
740					$extra_args ${local_addr} < "$sinfail" > "$sout" &
741	else
742		timeout ${timeout_test} \
743			ip netns exec ${listener_ns} \
744				./mptcp_connect -t ${timeout_poll} -l -p $port -s ${srv_proto} \
745					$extra_args ${local_addr} < "$sin" > "$sout" &
746	fi
747	local spid=$!
748
749	wait_local_port_listen "${listener_ns}" "${port}"
750
751	if [ "$test_link_fail" -eq 0 ];then
752		timeout ${timeout_test} \
753			ip netns exec ${connector_ns} \
754				./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \
755					$extra_args $connect_addr < "$cin" > "$cout" &
756	elif [ "$test_link_fail" -eq 1 ] || [ "$test_link_fail" -eq 2 ];then
757		( cat "$cinfail" ; sleep 2; link_failure $listener_ns ; cat "$cinfail" ) | \
758			tee "$cinsent" | \
759			timeout ${timeout_test} \
760				ip netns exec ${connector_ns} \
761					./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \
762						$extra_args $connect_addr > "$cout" &
763	else
764		tee "$cinsent" < "$cinfail" | \
765			timeout ${timeout_test} \
766				ip netns exec ${connector_ns} \
767					./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \
768						$extra_args $connect_addr > "$cout" &
769	fi
770	local cpid=$!
771
772	# let the mptcp subflow be established in background before
773	# do endpoint manipulation
774	if [ $addr_nr_ns1 != "0" ] || [ $addr_nr_ns2 != "0" ]; then
775		sleep 1
776	fi
777
778	if [ $addr_nr_ns1 -gt 0 ]; then
779		local counter=2
780		local add_nr_ns1=${addr_nr_ns1}
781		local id=10
782		local tk
783		while [ $add_nr_ns1 -gt 0 ]; do
784			local addr
785			if is_v6 "${connect_addr}"; then
786				addr="dead:beef:$counter::1"
787			else
788				addr="10.0.$counter.1"
789			fi
790			if [ $userspace_pm -eq 0 ]; then
791				pm_nl_add_endpoint $ns1 $addr flags signal
792			else
793				tk=$(sed -n 's/.*\(token:\)\([[:digit:]]*\).*$/\2/p;q' "$evts_ns1")
794				ip netns exec ${listener_ns} ./pm_nl_ctl ann $addr token $tk id $id
795				sleep 1
796				ip netns exec ${listener_ns} ./pm_nl_ctl rem token $tk id $id
797			fi
798
799			counter=$((counter + 1))
800			add_nr_ns1=$((add_nr_ns1 - 1))
801			id=$((id + 1))
802		done
803	elif [ $addr_nr_ns1 -lt 0 ]; then
804		local rm_nr_ns1=$((-addr_nr_ns1))
805		if [ $rm_nr_ns1 -lt 8 ]; then
806			local counter=0
807			local line
808			pm_nl_show_endpoints ${listener_ns} | while read -r line; do
809				# shellcheck disable=SC2206 # we do want to split per word
810				local arr=($line)
811				local nr=0
812
813				local i
814				for i in "${arr[@]}"; do
815					if [ $i = "id" ]; then
816						if [ $counter -eq $rm_nr_ns1 ]; then
817							break
818						fi
819						id=${arr[$nr+1]}
820						rm_addr=$(rm_addr_count ${connector_ns})
821						pm_nl_del_endpoint ${listener_ns} $id
822						wait_rm_addr ${connector_ns} ${rm_addr}
823						counter=$((counter + 1))
824					fi
825					nr=$((nr + 1))
826				done
827			done
828		elif [ $rm_nr_ns1 -eq 8 ]; then
829			pm_nl_flush_endpoint ${listener_ns}
830		elif [ $rm_nr_ns1 -eq 9 ]; then
831			pm_nl_del_endpoint ${listener_ns} 0 ${connect_addr}
832		fi
833	fi
834
835	local flags="subflow"
836	if [[ "${addr_nr_ns2}" = "fullmesh_"* ]]; then
837		flags="${flags},fullmesh"
838		addr_nr_ns2=${addr_nr_ns2:9}
839	fi
840
841	# if newly added endpoints must be deleted, give the background msk
842	# some time to created them
843	[ $addr_nr_ns1 -gt 0 ] && [ $addr_nr_ns2 -lt 0 ] && sleep 1
844
845	if [ $addr_nr_ns2 -gt 0 ]; then
846		local add_nr_ns2=${addr_nr_ns2}
847		local counter=3
848		local id=20
849		local tk da dp sp
850		while [ $add_nr_ns2 -gt 0 ]; do
851			local addr
852			if is_v6 "${connect_addr}"; then
853				addr="dead:beef:$counter::2"
854			else
855				addr="10.0.$counter.2"
856			fi
857			if [ $userspace_pm -eq 0 ]; then
858				pm_nl_add_endpoint $ns2 $addr flags $flags
859			else
860				tk=$(sed -n 's/.*\(token:\)\([[:digit:]]*\).*$/\2/p;q' "$evts_ns2")
861				da=$(sed -n 's/.*\(daddr4:\)\([0-9.]*\).*$/\2/p;q' "$evts_ns2")
862				dp=$(sed -n 's/.*\(dport:\)\([[:digit:]]*\).*$/\2/p;q' "$evts_ns2")
863				ip netns exec ${connector_ns} ./pm_nl_ctl csf lip $addr lid $id \
864									rip $da rport $dp token $tk
865				sleep 1
866				sp=$(grep "type:10" "$evts_ns2" |
867				     sed -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q')
868				ip netns exec ${connector_ns} ./pm_nl_ctl dsf lip $addr lport $sp \
869									rip $da rport $dp token $tk
870			fi
871			counter=$((counter + 1))
872			add_nr_ns2=$((add_nr_ns2 - 1))
873			id=$((id + 1))
874		done
875	elif [ $addr_nr_ns2 -lt 0 ]; then
876		local rm_nr_ns2=$((-addr_nr_ns2))
877		if [ $rm_nr_ns2 -lt 8 ]; then
878			local counter=0
879			local line
880			pm_nl_show_endpoints ${connector_ns} | while read -r line; do
881				# shellcheck disable=SC2206 # we do want to split per word
882				local arr=($line)
883				local nr=0
884
885				local i
886				for i in "${arr[@]}"; do
887					if [ $i = "id" ]; then
888						if [ $counter -eq $rm_nr_ns2 ]; then
889							break
890						fi
891						local id rm_addr
892						# rm_addr are serialized, allow the previous one to
893						# complete
894						id=${arr[$nr+1]}
895						rm_addr=$(rm_addr_count ${listener_ns})
896						pm_nl_del_endpoint ${connector_ns} $id
897						wait_rm_addr ${listener_ns} ${rm_addr}
898						counter=$((counter + 1))
899					fi
900					nr=$((nr + 1))
901				done
902			done
903		elif [ $rm_nr_ns2 -eq 8 ]; then
904			pm_nl_flush_endpoint ${connector_ns}
905		elif [ $rm_nr_ns2 -eq 9 ]; then
906			local addr
907			if is_v6 "${connect_addr}"; then
908				addr="dead:beef:1::2"
909			else
910				addr="10.0.1.2"
911			fi
912			pm_nl_del_endpoint ${connector_ns} 0 $addr
913		fi
914	fi
915
916	if [ -n "${sflags}" ]; then
917		sleep 1
918
919		local netns
920		for netns in "$ns1" "$ns2"; do
921			local line
922			pm_nl_show_endpoints $netns | while read -r line; do
923				# shellcheck disable=SC2206 # we do want to split per word
924				local arr=($line)
925				local nr=0
926				local id
927
928				local i
929				for i in "${arr[@]}"; do
930					if [ $i = "id" ]; then
931						id=${arr[$nr+1]}
932					fi
933					nr=$((nr + 1))
934				done
935				pm_nl_change_endpoint $netns $id $sflags
936			done
937		done
938	fi
939
940	wait $cpid
941	local retc=$?
942	wait $spid
943	local rets=$?
944
945	if [ $capture -eq 1 ]; then
946	    sleep 1
947	    kill $cappid
948	fi
949
950	if [ $userspace_pm -eq 1 ]; then
951		kill_wait $evts_ns1_pid
952		kill_wait $evts_ns2_pid
953		rm -rf $evts_ns1 $evts_ns2
954	fi
955
956	NSTAT_HISTORY=/tmp/${listener_ns}.nstat ip netns exec ${listener_ns} \
957		nstat | grep Tcp > /tmp/${listener_ns}.out
958	NSTAT_HISTORY=/tmp/${connector_ns}.nstat ip netns exec ${connector_ns} \
959		nstat | grep Tcp > /tmp/${connector_ns}.out
960
961	if [ ${rets} -ne 0 ] || [ ${retc} -ne 0 ]; then
962		echo " client exit code $retc, server $rets" 1>&2
963		echo -e "\nnetns ${listener_ns} socket stat for ${port}:" 1>&2
964		ip netns exec ${listener_ns} ss -Menita 1>&2 -o "sport = :$port"
965		cat /tmp/${listener_ns}.out
966		echo -e "\nnetns ${connector_ns} socket stat for ${port}:" 1>&2
967		ip netns exec ${connector_ns} ss -Menita 1>&2 -o "dport = :$port"
968		cat /tmp/${connector_ns}.out
969
970		cat "$capout"
971		fail_test
972		return 1
973	fi
974
975	if [ "$test_link_fail" -gt 1 ];then
976		check_transfer $sinfail $cout "file received by client"
977	else
978		check_transfer $sin $cout "file received by client"
979	fi
980	retc=$?
981	if [ "$test_link_fail" -eq 0 ];then
982		check_transfer $cin $sout "file received by server"
983	else
984		check_transfer $cinsent $sout "file received by server"
985	fi
986	rets=$?
987
988	if [ $retc -eq 0 ] && [ $rets -eq 0 ];then
989		cat "$capout"
990		return 0
991	fi
992
993	cat "$capout"
994	return 1
995}
996
997make_file()
998{
999	local name=$1
1000	local who=$2
1001	local size=$3
1002
1003	dd if=/dev/urandom of="$name" bs=1024 count=$size 2> /dev/null
1004	echo -e "\nMPTCP_TEST_FILE_END_MARKER" >> "$name"
1005
1006	echo "Created $name (size $size KB) containing data sent by $who"
1007}
1008
1009run_tests()
1010{
1011	local listener_ns="$1"
1012	local connector_ns="$2"
1013	local connect_addr="$3"
1014	local test_linkfail="${4:-0}"
1015	local addr_nr_ns1="${5:-0}"
1016	local addr_nr_ns2="${6:-0}"
1017	local speed="${7:-fast}"
1018	local sflags="${8:-""}"
1019
1020	local size
1021
1022	# The values above 2 are reused to make test files
1023	# with the given sizes (KB)
1024	if [ "$test_linkfail" -gt 2 ]; then
1025		size=$test_linkfail
1026
1027		if [ -z "$cinfail" ]; then
1028			cinfail=$(mktemp)
1029		fi
1030		make_file "$cinfail" "client" $size
1031	# create the input file for the failure test when
1032	# the first failure test run
1033	elif [ "$test_linkfail" -ne 0 ] && [ -z "$cinfail" ]; then
1034		# the client file must be considerably larger
1035		# of the maximum expected cwin value, or the
1036		# link utilization will be not predicable
1037		size=$((RANDOM%2))
1038		size=$((size+1))
1039		size=$((size*8192))
1040		size=$((size + ( RANDOM % 8192) ))
1041
1042		cinfail=$(mktemp)
1043		make_file "$cinfail" "client" $size
1044	fi
1045
1046	if [ "$test_linkfail" -gt 2 ]; then
1047		size=$test_linkfail
1048
1049		if [ -z "$sinfail" ]; then
1050			sinfail=$(mktemp)
1051		fi
1052		make_file "$sinfail" "server" $size
1053	elif [ "$test_linkfail" -eq 2 ] && [ -z "$sinfail" ]; then
1054		size=$((RANDOM%16))
1055		size=$((size+1))
1056		size=$((size*2048))
1057
1058		sinfail=$(mktemp)
1059		make_file "$sinfail" "server" $size
1060	fi
1061
1062	do_transfer ${listener_ns} ${connector_ns} MPTCP MPTCP ${connect_addr} \
1063		${test_linkfail} ${addr_nr_ns1} ${addr_nr_ns2} ${speed} ${sflags}
1064}
1065
1066dump_stats()
1067{
1068	echo Server ns stats
1069	ip netns exec $ns1 nstat -as | grep Tcp
1070	echo Client ns stats
1071	ip netns exec $ns2 nstat -as | grep Tcp
1072}
1073
1074chk_csum_nr()
1075{
1076	local csum_ns1=${1:-0}
1077	local csum_ns2=${2:-0}
1078	local count
1079	local dump_stats
1080	local extra_msg=""
1081	local allow_multi_errors_ns1=0
1082	local allow_multi_errors_ns2=0
1083
1084	if [[ "${csum_ns1}" = "+"* ]]; then
1085		allow_multi_errors_ns1=1
1086		csum_ns1=${csum_ns1:1}
1087	fi
1088	if [[ "${csum_ns2}" = "+"* ]]; then
1089		allow_multi_errors_ns2=1
1090		csum_ns2=${csum_ns2:1}
1091	fi
1092
1093	printf "%-${nr_blank}s %s" " " "sum"
1094	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtDataCsumErr | awk '{print $2}')
1095	[ -z "$count" ] && count=0
1096	if [ "$count" != "$csum_ns1" ]; then
1097		extra_msg="$extra_msg ns1=$count"
1098	fi
1099	if { [ "$count" != $csum_ns1 ] && [ $allow_multi_errors_ns1 -eq 0 ]; } ||
1100	   { [ "$count" -lt $csum_ns1 ] && [ $allow_multi_errors_ns1 -eq 1 ]; }; then
1101		echo "[fail] got $count data checksum error[s] expected $csum_ns1"
1102		fail_test
1103		dump_stats=1
1104	else
1105		echo -n "[ ok ]"
1106	fi
1107	echo -n " - csum  "
1108	count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtDataCsumErr | awk '{print $2}')
1109	[ -z "$count" ] && count=0
1110	if [ "$count" != "$csum_ns2" ]; then
1111		extra_msg="$extra_msg ns2=$count"
1112	fi
1113	if { [ "$count" != $csum_ns2 ] && [ $allow_multi_errors_ns2 -eq 0 ]; } ||
1114	   { [ "$count" -lt $csum_ns2 ] && [ $allow_multi_errors_ns2 -eq 1 ]; }; then
1115		echo "[fail] got $count data checksum error[s] expected $csum_ns2"
1116		fail_test
1117		dump_stats=1
1118	else
1119		echo -n "[ ok ]"
1120	fi
1121	[ "${dump_stats}" = 1 ] && dump_stats
1122
1123	echo "$extra_msg"
1124}
1125
1126chk_fail_nr()
1127{
1128	local fail_tx=$1
1129	local fail_rx=$2
1130	local ns_invert=${3:-""}
1131	local count
1132	local dump_stats
1133	local ns_tx=$ns1
1134	local ns_rx=$ns2
1135	local extra_msg=""
1136	local allow_tx_lost=0
1137	local allow_rx_lost=0
1138
1139	if [[ $ns_invert = "invert" ]]; then
1140		ns_tx=$ns2
1141		ns_rx=$ns1
1142		extra_msg=" invert"
1143	fi
1144
1145	if [[ "${fail_tx}" = "-"* ]]; then
1146		allow_tx_lost=1
1147		fail_tx=${fail_tx:1}
1148	fi
1149	if [[ "${fail_rx}" = "-"* ]]; then
1150		allow_rx_lost=1
1151		fail_rx=${fail_rx:1}
1152	fi
1153
1154	printf "%-${nr_blank}s %s" " " "ftx"
1155	count=$(ip netns exec $ns_tx nstat -as | grep MPTcpExtMPFailTx | awk '{print $2}')
1156	[ -z "$count" ] && count=0
1157	if [ "$count" != "$fail_tx" ]; then
1158		extra_msg="$extra_msg,tx=$count"
1159	fi
1160	if { [ "$count" != "$fail_tx" ] && [ $allow_tx_lost -eq 0 ]; } ||
1161	   { [ "$count" -gt "$fail_tx" ] && [ $allow_tx_lost -eq 1 ]; }; then
1162		echo "[fail] got $count MP_FAIL[s] TX expected $fail_tx"
1163		fail_test
1164		dump_stats=1
1165	else
1166		echo -n "[ ok ]"
1167	fi
1168
1169	echo -n " - failrx"
1170	count=$(ip netns exec $ns_rx nstat -as | grep MPTcpExtMPFailRx | awk '{print $2}')
1171	[ -z "$count" ] && count=0
1172	if [ "$count" != "$fail_rx" ]; then
1173		extra_msg="$extra_msg,rx=$count"
1174	fi
1175	if { [ "$count" != "$fail_rx" ] && [ $allow_rx_lost -eq 0 ]; } ||
1176	   { [ "$count" -gt "$fail_rx" ] && [ $allow_rx_lost -eq 1 ]; }; then
1177		echo "[fail] got $count MP_FAIL[s] RX expected $fail_rx"
1178		fail_test
1179		dump_stats=1
1180	else
1181		echo -n "[ ok ]"
1182	fi
1183
1184	[ "${dump_stats}" = 1 ] && dump_stats
1185
1186	echo "$extra_msg"
1187}
1188
1189chk_fclose_nr()
1190{
1191	local fclose_tx=$1
1192	local fclose_rx=$2
1193	local count
1194	local dump_stats
1195
1196	printf "%-${nr_blank}s %s" " " "ctx"
1197	count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtMPFastcloseTx | awk '{print $2}')
1198	[ -z "$count" ] && count=0
1199	if [ "$count" != "$fclose_tx" ]; then
1200		echo "[fail] got $count MP_FASTCLOSE[s] TX expected $fclose_tx"
1201		fail_test
1202		dump_stats=1
1203	else
1204		echo -n "[ ok ]"
1205	fi
1206
1207	echo -n " - fclzrx"
1208	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPFastcloseRx | awk '{print $2}')
1209	[ -z "$count" ] && count=0
1210	if [ "$count" != "$fclose_rx" ]; then
1211		echo "[fail] got $count MP_FASTCLOSE[s] RX expected $fclose_rx"
1212		fail_test
1213		dump_stats=1
1214	else
1215		echo "[ ok ]"
1216	fi
1217
1218	[ "${dump_stats}" = 1 ] && dump_stats
1219}
1220
1221chk_rst_nr()
1222{
1223	local rst_tx=$1
1224	local rst_rx=$2
1225	local ns_invert=${3:-""}
1226	local count
1227	local dump_stats
1228	local ns_tx=$ns1
1229	local ns_rx=$ns2
1230	local extra_msg=""
1231
1232	if [[ $ns_invert = "invert" ]]; then
1233		ns_tx=$ns2
1234		ns_rx=$ns1
1235		extra_msg="   invert"
1236	fi
1237
1238	printf "%-${nr_blank}s %s" " " "rtx"
1239	count=$(ip netns exec $ns_tx nstat -as | grep MPTcpExtMPRstTx | awk '{print $2}')
1240	[ -z "$count" ] && count=0
1241	if [ "$count" != "$rst_tx" ]; then
1242		echo "[fail] got $count MP_RST[s] TX expected $rst_tx"
1243		fail_test
1244		dump_stats=1
1245	else
1246		echo -n "[ ok ]"
1247	fi
1248
1249	echo -n " - rstrx "
1250	count=$(ip netns exec $ns_rx nstat -as | grep MPTcpExtMPRstRx | awk '{print $2}')
1251	[ -z "$count" ] && count=0
1252	if [ "$count" != "$rst_rx" ]; then
1253		echo "[fail] got $count MP_RST[s] RX expected $rst_rx"
1254		fail_test
1255		dump_stats=1
1256	else
1257		echo -n "[ ok ]"
1258	fi
1259
1260	[ "${dump_stats}" = 1 ] && dump_stats
1261
1262	echo "$extra_msg"
1263}
1264
1265chk_infi_nr()
1266{
1267	local infi_tx=$1
1268	local infi_rx=$2
1269	local count
1270	local dump_stats
1271
1272	printf "%-${nr_blank}s %s" " " "itx"
1273	count=$(ip netns exec $ns2 nstat -as | grep InfiniteMapTx | awk '{print $2}')
1274	[ -z "$count" ] && count=0
1275	if [ "$count" != "$infi_tx" ]; then
1276		echo "[fail] got $count infinite map[s] TX expected $infi_tx"
1277		fail_test
1278		dump_stats=1
1279	else
1280		echo -n "[ ok ]"
1281	fi
1282
1283	echo -n " - infirx"
1284	count=$(ip netns exec $ns1 nstat -as | grep InfiniteMapRx | awk '{print $2}')
1285	[ -z "$count" ] && count=0
1286	if [ "$count" != "$infi_rx" ]; then
1287		echo "[fail] got $count infinite map[s] RX expected $infi_rx"
1288		fail_test
1289		dump_stats=1
1290	else
1291		echo "[ ok ]"
1292	fi
1293
1294	[ "${dump_stats}" = 1 ] && dump_stats
1295}
1296
1297chk_join_nr()
1298{
1299	local syn_nr=$1
1300	local syn_ack_nr=$2
1301	local ack_nr=$3
1302	local csum_ns1=${4:-0}
1303	local csum_ns2=${5:-0}
1304	local fail_nr=${6:-0}
1305	local rst_nr=${7:-0}
1306	local infi_nr=${8:-0}
1307	local corrupted_pkts=${9:-0}
1308	local count
1309	local dump_stats
1310	local with_cookie
1311	local title="${TEST_NAME}"
1312
1313	if [ "${corrupted_pkts}" -gt 0 ]; then
1314		title+=": ${corrupted_pkts} corrupted pkts"
1315	fi
1316
1317	printf "%03u %-36s %s" "${TEST_COUNT}" "${title}" "syn"
1318	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinSynRx | awk '{print $2}')
1319	[ -z "$count" ] && count=0
1320	if [ "$count" != "$syn_nr" ]; then
1321		echo "[fail] got $count JOIN[s] syn expected $syn_nr"
1322		fail_test
1323		dump_stats=1
1324	else
1325		echo -n "[ ok ]"
1326	fi
1327
1328	echo -n " - synack"
1329	with_cookie=$(ip netns exec $ns2 sysctl -n net.ipv4.tcp_syncookies)
1330	count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtMPJoinSynAckRx | awk '{print $2}')
1331	[ -z "$count" ] && count=0
1332	if [ "$count" != "$syn_ack_nr" ]; then
1333		# simult connections exceeding the limit with cookie enabled could go up to
1334		# synack validation as the conn limit can be enforced reliably only after
1335		# the subflow creation
1336		if [ "$with_cookie" = 2 ] && [ "$count" -gt "$syn_ack_nr" ] && [ "$count" -le "$syn_nr" ]; then
1337			echo -n "[ ok ]"
1338		else
1339			echo "[fail] got $count JOIN[s] synack expected $syn_ack_nr"
1340			fail_test
1341			dump_stats=1
1342		fi
1343	else
1344		echo -n "[ ok ]"
1345	fi
1346
1347	echo -n " - ack"
1348	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinAckRx | awk '{print $2}')
1349	[ -z "$count" ] && count=0
1350	if [ "$count" != "$ack_nr" ]; then
1351		echo "[fail] got $count JOIN[s] ack expected $ack_nr"
1352		fail_test
1353		dump_stats=1
1354	else
1355		echo "[ ok ]"
1356	fi
1357	[ "${dump_stats}" = 1 ] && dump_stats
1358	if [ $validate_checksum -eq 1 ]; then
1359		chk_csum_nr $csum_ns1 $csum_ns2
1360		chk_fail_nr $fail_nr $fail_nr
1361		chk_rst_nr $rst_nr $rst_nr
1362		chk_infi_nr $infi_nr $infi_nr
1363	fi
1364}
1365
1366# a negative value for 'stale_max' means no upper bound:
1367# for bidirectional transfer, if one peer sleep for a while
1368# - as these tests do - we can have a quite high number of
1369# stale/recover conversions, proportional to
1370# sleep duration/ MPTCP-level RTX interval.
1371chk_stale_nr()
1372{
1373	local ns=$1
1374	local stale_min=$2
1375	local stale_max=$3
1376	local stale_delta=$4
1377	local dump_stats
1378	local stale_nr
1379	local recover_nr
1380
1381	printf "%-${nr_blank}s %-18s" " " "stale"
1382	stale_nr=$(ip netns exec $ns nstat -as | grep MPTcpExtSubflowStale | awk '{print $2}')
1383	[ -z "$stale_nr" ] && stale_nr=0
1384	recover_nr=$(ip netns exec $ns nstat -as | grep MPTcpExtSubflowRecover | awk '{print $2}')
1385	[ -z "$recover_nr" ] && recover_nr=0
1386
1387	if [ $stale_nr -lt $stale_min ] ||
1388	   { [ $stale_max -gt 0 ] && [ $stale_nr -gt $stale_max ]; } ||
1389	   [ $((stale_nr - recover_nr)) -ne $stale_delta ]; then
1390		echo "[fail] got $stale_nr stale[s] $recover_nr recover[s], " \
1391		     " expected stale in range [$stale_min..$stale_max]," \
1392		     " stale-recover delta $stale_delta "
1393		fail_test
1394		dump_stats=1
1395	else
1396		echo "[ ok ]"
1397	fi
1398
1399	if [ "${dump_stats}" = 1 ]; then
1400		echo $ns stats
1401		ip netns exec $ns ip -s link show
1402		ip netns exec $ns nstat -as | grep MPTcp
1403	fi
1404}
1405
1406chk_add_nr()
1407{
1408	local add_nr=$1
1409	local echo_nr=$2
1410	local port_nr=${3:-0}
1411	local syn_nr=${4:-$port_nr}
1412	local syn_ack_nr=${5:-$port_nr}
1413	local ack_nr=${6:-$port_nr}
1414	local mis_syn_nr=${7:-0}
1415	local mis_ack_nr=${8:-0}
1416	local count
1417	local dump_stats
1418	local timeout
1419
1420	timeout=$(ip netns exec $ns1 sysctl -n net.mptcp.add_addr_timeout)
1421
1422	printf "%-${nr_blank}s %s" " " "add"
1423	count=$(ip netns exec $ns2 nstat -as MPTcpExtAddAddr | grep MPTcpExtAddAddr | awk '{print $2}')
1424	[ -z "$count" ] && count=0
1425
1426	# if the test configured a short timeout tolerate greater then expected
1427	# add addrs options, due to retransmissions
1428	if [ "$count" != "$add_nr" ] && { [ "$timeout" -gt 1 ] || [ "$count" -lt "$add_nr" ]; }; then
1429		echo "[fail] got $count ADD_ADDR[s] expected $add_nr"
1430		fail_test
1431		dump_stats=1
1432	else
1433		echo -n "[ ok ]"
1434	fi
1435
1436	echo -n " - echo  "
1437	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtEchoAdd | awk '{print $2}')
1438	[ -z "$count" ] && count=0
1439	if [ "$count" != "$echo_nr" ]; then
1440		echo "[fail] got $count ADD_ADDR echo[s] expected $echo_nr"
1441		fail_test
1442		dump_stats=1
1443	else
1444		echo -n "[ ok ]"
1445	fi
1446
1447	if [ $port_nr -gt 0 ]; then
1448		echo -n " - pt "
1449		count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtPortAdd | awk '{print $2}')
1450		[ -z "$count" ] && count=0
1451		if [ "$count" != "$port_nr" ]; then
1452			echo "[fail] got $count ADD_ADDR[s] with a port-number expected $port_nr"
1453			fail_test
1454			dump_stats=1
1455		else
1456			echo "[ ok ]"
1457		fi
1458
1459		printf "%-${nr_blank}s %s" " " "syn"
1460		count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinPortSynRx |
1461			awk '{print $2}')
1462		[ -z "$count" ] && count=0
1463		if [ "$count" != "$syn_nr" ]; then
1464			echo "[fail] got $count JOIN[s] syn with a different \
1465				port-number expected $syn_nr"
1466			fail_test
1467			dump_stats=1
1468		else
1469			echo -n "[ ok ]"
1470		fi
1471
1472		echo -n " - synack"
1473		count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtMPJoinPortSynAckRx |
1474			awk '{print $2}')
1475		[ -z "$count" ] && count=0
1476		if [ "$count" != "$syn_ack_nr" ]; then
1477			echo "[fail] got $count JOIN[s] synack with a different \
1478				port-number expected $syn_ack_nr"
1479			fail_test
1480			dump_stats=1
1481		else
1482			echo -n "[ ok ]"
1483		fi
1484
1485		echo -n " - ack"
1486		count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPJoinPortAckRx |
1487			awk '{print $2}')
1488		[ -z "$count" ] && count=0
1489		if [ "$count" != "$ack_nr" ]; then
1490			echo "[fail] got $count JOIN[s] ack with a different \
1491				port-number expected $ack_nr"
1492			fail_test
1493			dump_stats=1
1494		else
1495			echo "[ ok ]"
1496		fi
1497
1498		printf "%-${nr_blank}s %s" " " "syn"
1499		count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMismatchPortSynRx |
1500			awk '{print $2}')
1501		[ -z "$count" ] && count=0
1502		if [ "$count" != "$mis_syn_nr" ]; then
1503			echo "[fail] got $count JOIN[s] syn with a mismatched \
1504				port-number expected $mis_syn_nr"
1505			fail_test
1506			dump_stats=1
1507		else
1508			echo -n "[ ok ]"
1509		fi
1510
1511		echo -n " - ack   "
1512		count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMismatchPortAckRx |
1513			awk '{print $2}')
1514		[ -z "$count" ] && count=0
1515		if [ "$count" != "$mis_ack_nr" ]; then
1516			echo "[fail] got $count JOIN[s] ack with a mismatched \
1517				port-number expected $mis_ack_nr"
1518			fail_test
1519			dump_stats=1
1520		else
1521			echo "[ ok ]"
1522		fi
1523	else
1524		echo ""
1525	fi
1526
1527	[ "${dump_stats}" = 1 ] && dump_stats
1528}
1529
1530chk_rm_nr()
1531{
1532	local rm_addr_nr=$1
1533	local rm_subflow_nr=$2
1534	local invert
1535	local simult
1536	local count
1537	local dump_stats
1538	local addr_ns=$ns1
1539	local subflow_ns=$ns2
1540	local extra_msg=""
1541
1542	shift 2
1543	while [ -n "$1" ]; do
1544		[ "$1" = "invert" ] && invert=true
1545		[ "$1" = "simult" ] && simult=true
1546		shift
1547	done
1548
1549	if [ -z $invert ]; then
1550		addr_ns=$ns1
1551		subflow_ns=$ns2
1552	elif [ $invert = "true" ]; then
1553		addr_ns=$ns2
1554		subflow_ns=$ns1
1555		extra_msg="   invert"
1556	fi
1557
1558	printf "%-${nr_blank}s %s" " " "rm "
1559	count=$(ip netns exec $addr_ns nstat -as | grep MPTcpExtRmAddr | awk '{print $2}')
1560	[ -z "$count" ] && count=0
1561	if [ "$count" != "$rm_addr_nr" ]; then
1562		echo "[fail] got $count RM_ADDR[s] expected $rm_addr_nr"
1563		fail_test
1564		dump_stats=1
1565	else
1566		echo -n "[ ok ]"
1567	fi
1568
1569	echo -n " - rmsf  "
1570	count=$(ip netns exec $subflow_ns nstat -as | grep MPTcpExtRmSubflow | awk '{print $2}')
1571	[ -z "$count" ] && count=0
1572	if [ -n "$simult" ]; then
1573		local cnt suffix
1574
1575		cnt=$(ip netns exec $addr_ns nstat -as | grep MPTcpExtRmSubflow | awk '{print $2}')
1576
1577		# in case of simult flush, the subflow removal count on each side is
1578		# unreliable
1579		[ -z "$cnt" ] && cnt=0
1580		count=$((count + cnt))
1581		[ "$count" != "$rm_subflow_nr" ] && suffix="$count in [$rm_subflow_nr:$((rm_subflow_nr*2))]"
1582		if [ $count -ge "$rm_subflow_nr" ] && \
1583		   [ "$count" -le "$((rm_subflow_nr *2 ))" ]; then
1584			echo "[ ok ] $suffix"
1585		else
1586			echo "[fail] got $count RM_SUBFLOW[s] expected in range [$rm_subflow_nr:$((rm_subflow_nr*2))]"
1587			fail_test
1588			dump_stats=1
1589		fi
1590		return
1591	fi
1592	if [ "$count" != "$rm_subflow_nr" ]; then
1593		echo "[fail] got $count RM_SUBFLOW[s] expected $rm_subflow_nr"
1594		fail_test
1595		dump_stats=1
1596	else
1597		echo -n "[ ok ]"
1598	fi
1599
1600	[ "${dump_stats}" = 1 ] && dump_stats
1601
1602	echo "$extra_msg"
1603}
1604
1605chk_prio_nr()
1606{
1607	local mp_prio_nr_tx=$1
1608	local mp_prio_nr_rx=$2
1609	local count
1610	local dump_stats
1611
1612	printf "%-${nr_blank}s %s" " " "ptx"
1613	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPPrioTx | awk '{print $2}')
1614	[ -z "$count" ] && count=0
1615	if [ "$count" != "$mp_prio_nr_tx" ]; then
1616		echo "[fail] got $count MP_PRIO[s] TX expected $mp_prio_nr_tx"
1617		fail_test
1618		dump_stats=1
1619	else
1620		echo -n "[ ok ]"
1621	fi
1622
1623	echo -n " - prx   "
1624	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPPrioRx | awk '{print $2}')
1625	[ -z "$count" ] && count=0
1626	if [ "$count" != "$mp_prio_nr_rx" ]; then
1627		echo "[fail] got $count MP_PRIO[s] RX expected $mp_prio_nr_rx"
1628		fail_test
1629		dump_stats=1
1630	else
1631		echo "[ ok ]"
1632	fi
1633
1634	[ "${dump_stats}" = 1 ] && dump_stats
1635}
1636
1637chk_subflow_nr()
1638{
1639	local need_title="$1"
1640	local msg="$2"
1641	local subflow_nr=$3
1642	local cnt1
1643	local cnt2
1644
1645	if [ -n "${need_title}" ]; then
1646		printf "%03u %-36s %s" "${TEST_COUNT}" "${TEST_NAME}" "${msg}"
1647	else
1648		printf "%-${nr_blank}s %s" " " "${msg}"
1649	fi
1650
1651	cnt1=$(ss -N $ns1 -tOni | grep -c token)
1652	cnt2=$(ss -N $ns2 -tOni | grep -c token)
1653	if [ "$cnt1" != "$subflow_nr" -o "$cnt2" != "$subflow_nr" ]; then
1654		echo "[fail] got $cnt1:$cnt2 subflows expected $subflow_nr"
1655		fail_test
1656		dump_stats=1
1657	else
1658		echo "[ ok ]"
1659	fi
1660
1661	[ "${dump_stats}" = 1 ] && ( ss -N $ns1 -tOni ; ss -N $ns1 -tOni | grep token; ip -n $ns1 mptcp endpoint )
1662}
1663
1664chk_link_usage()
1665{
1666	local ns=$1
1667	local link=$2
1668	local out=$3
1669	local expected_rate=$4
1670
1671	local tx_link tx_total
1672	tx_link=$(ip netns exec $ns cat /sys/class/net/$link/statistics/tx_bytes)
1673	tx_total=$(stat --format=%s $out)
1674	local tx_rate=$((tx_link * 100 / tx_total))
1675	local tolerance=5
1676
1677	printf "%-${nr_blank}s %-18s" " " "link usage"
1678	if [ $tx_rate -lt $((expected_rate - tolerance)) ] || \
1679	   [ $tx_rate -gt $((expected_rate + tolerance)) ]; then
1680		echo "[fail] got $tx_rate% usage, expected $expected_rate%"
1681		fail_test
1682	else
1683		echo "[ ok ]"
1684	fi
1685}
1686
1687wait_attempt_fail()
1688{
1689	local timeout_ms=$((timeout_poll * 1000))
1690	local time=0
1691	local ns=$1
1692
1693	while [ $time -lt $timeout_ms ]; do
1694		local cnt
1695
1696		cnt=$(ip netns exec $ns nstat -as TcpAttemptFails | grep TcpAttemptFails | awk '{print $2}')
1697
1698		[ "$cnt" = 1 ] && return 1
1699		time=$((time + 100))
1700		sleep 0.1
1701	done
1702	return 1
1703}
1704
1705set_userspace_pm()
1706{
1707	local ns=$1
1708
1709	ip netns exec $ns sysctl -q net.mptcp.pm_type=1
1710}
1711
1712subflows_tests()
1713{
1714	if reset "no JOIN"; then
1715		run_tests $ns1 $ns2 10.0.1.1
1716		chk_join_nr 0 0 0
1717	fi
1718
1719	# subflow limited by client
1720	if reset "single subflow, limited by client"; then
1721		pm_nl_set_limits $ns1 0 0
1722		pm_nl_set_limits $ns2 0 0
1723		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1724		run_tests $ns1 $ns2 10.0.1.1
1725		chk_join_nr 0 0 0
1726	fi
1727
1728	# subflow limited by server
1729	if reset "single subflow, limited by server"; then
1730		pm_nl_set_limits $ns1 0 0
1731		pm_nl_set_limits $ns2 0 1
1732		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1733		run_tests $ns1 $ns2 10.0.1.1
1734		chk_join_nr 1 1 0
1735	fi
1736
1737	# subflow
1738	if reset "single subflow"; then
1739		pm_nl_set_limits $ns1 0 1
1740		pm_nl_set_limits $ns2 0 1
1741		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1742		run_tests $ns1 $ns2 10.0.1.1
1743		chk_join_nr 1 1 1
1744	fi
1745
1746	# multiple subflows
1747	if reset "multiple subflows"; then
1748		pm_nl_set_limits $ns1 0 2
1749		pm_nl_set_limits $ns2 0 2
1750		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1751		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1752		run_tests $ns1 $ns2 10.0.1.1
1753		chk_join_nr 2 2 2
1754	fi
1755
1756	# multiple subflows limited by server
1757	if reset "multiple subflows, limited by server"; then
1758		pm_nl_set_limits $ns1 0 1
1759		pm_nl_set_limits $ns2 0 2
1760		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1761		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1762		run_tests $ns1 $ns2 10.0.1.1
1763		chk_join_nr 2 2 1
1764	fi
1765
1766	# single subflow, dev
1767	if reset "single subflow, dev"; then
1768		pm_nl_set_limits $ns1 0 1
1769		pm_nl_set_limits $ns2 0 1
1770		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow dev ns2eth3
1771		run_tests $ns1 $ns2 10.0.1.1
1772		chk_join_nr 1 1 1
1773	fi
1774}
1775
1776subflows_error_tests()
1777{
1778	# If a single subflow is configured, and matches the MPC src
1779	# address, no additional subflow should be created
1780	if reset "no MPC reuse with single endpoint"; then
1781		pm_nl_set_limits $ns1 0 1
1782		pm_nl_set_limits $ns2 0 1
1783		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow
1784		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
1785		chk_join_nr 0 0 0
1786	fi
1787
1788	# multiple subflows, with subflow creation error
1789	if reset "multi subflows, with failing subflow"; then
1790		pm_nl_set_limits $ns1 0 2
1791		pm_nl_set_limits $ns2 0 2
1792		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1793		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1794		filter_tcp_from $ns1 10.0.3.2 REJECT
1795		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
1796		chk_join_nr 1 1 1
1797	fi
1798
1799	# multiple subflows, with subflow timeout on MPJ
1800	if reset "multi subflows, with subflow timeout"; then
1801		pm_nl_set_limits $ns1 0 2
1802		pm_nl_set_limits $ns2 0 2
1803		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1804		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1805		filter_tcp_from $ns1 10.0.3.2 DROP
1806		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
1807		chk_join_nr 1 1 1
1808	fi
1809
1810	# multiple subflows, check that the endpoint corresponding to
1811	# closed subflow (due to reset) is not reused if additional
1812	# subflows are added later
1813	if reset "multi subflows, fair usage on close"; then
1814		pm_nl_set_limits $ns1 0 1
1815		pm_nl_set_limits $ns2 0 1
1816		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1817		filter_tcp_from $ns1 10.0.3.2 REJECT
1818		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow &
1819
1820		# mpj subflow will be in TW after the reset
1821		wait_attempt_fail $ns2
1822		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
1823		wait
1824
1825		# additional subflow could be created only if the PM select
1826		# the later endpoint, skipping the already used one
1827		chk_join_nr 1 1 1
1828	fi
1829}
1830
1831signal_address_tests()
1832{
1833	# add_address, unused
1834	if reset "unused signal address"; then
1835		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1836		run_tests $ns1 $ns2 10.0.1.1
1837		chk_join_nr 0 0 0
1838		chk_add_nr 1 1
1839	fi
1840
1841	# accept and use add_addr
1842	if reset "signal address"; then
1843		pm_nl_set_limits $ns1 0 1
1844		pm_nl_set_limits $ns2 1 1
1845		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1846		run_tests $ns1 $ns2 10.0.1.1
1847		chk_join_nr 1 1 1
1848		chk_add_nr 1 1
1849	fi
1850
1851	# accept and use add_addr with an additional subflow
1852	# note: signal address in server ns and local addresses in client ns must
1853	# belong to different subnets or one of the listed local address could be
1854	# used for 'add_addr' subflow
1855	if reset "subflow and signal"; then
1856		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1857		pm_nl_set_limits $ns1 0 2
1858		pm_nl_set_limits $ns2 1 2
1859		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1860		run_tests $ns1 $ns2 10.0.1.1
1861		chk_join_nr 2 2 2
1862		chk_add_nr 1 1
1863	fi
1864
1865	# accept and use add_addr with additional subflows
1866	if reset "multiple subflows and signal"; then
1867		pm_nl_set_limits $ns1 0 3
1868		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1869		pm_nl_set_limits $ns2 1 3
1870		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
1871		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
1872		run_tests $ns1 $ns2 10.0.1.1
1873		chk_join_nr 3 3 3
1874		chk_add_nr 1 1
1875	fi
1876
1877	# signal addresses
1878	if reset "signal addresses"; then
1879		pm_nl_set_limits $ns1 3 3
1880		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1881		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
1882		pm_nl_add_endpoint $ns1 10.0.4.1 flags signal
1883		pm_nl_set_limits $ns2 3 3
1884		run_tests $ns1 $ns2 10.0.1.1
1885		chk_join_nr 3 3 3
1886		chk_add_nr 3 3
1887	fi
1888
1889	# signal invalid addresses
1890	if reset "signal invalid addresses"; then
1891		pm_nl_set_limits $ns1 3 3
1892		pm_nl_add_endpoint $ns1 10.0.12.1 flags signal
1893		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
1894		pm_nl_add_endpoint $ns1 10.0.14.1 flags signal
1895		pm_nl_set_limits $ns2 3 3
1896		run_tests $ns1 $ns2 10.0.1.1
1897		chk_join_nr 1 1 1
1898		chk_add_nr 3 3
1899	fi
1900
1901	# signal addresses race test
1902	if reset "signal addresses race test"; then
1903		pm_nl_set_limits $ns1 4 4
1904		pm_nl_set_limits $ns2 4 4
1905		pm_nl_add_endpoint $ns1 10.0.1.1 flags signal
1906		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
1907		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
1908		pm_nl_add_endpoint $ns1 10.0.4.1 flags signal
1909		pm_nl_add_endpoint $ns2 10.0.1.2 flags signal
1910		pm_nl_add_endpoint $ns2 10.0.2.2 flags signal
1911		pm_nl_add_endpoint $ns2 10.0.3.2 flags signal
1912		pm_nl_add_endpoint $ns2 10.0.4.2 flags signal
1913
1914		# the peer could possibly miss some addr notification, allow retransmission
1915		ip netns exec $ns1 sysctl -q net.mptcp.add_addr_timeout=1
1916		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
1917		chk_join_nr 3 3 3
1918
1919		# the server will not signal the address terminating
1920		# the MPC subflow
1921		chk_add_nr 3 3
1922	fi
1923}
1924
1925link_failure_tests()
1926{
1927	# accept and use add_addr with additional subflows and link loss
1928	if reset "multiple flows, signal, link failure"; then
1929		# without any b/w limit each veth could spool the packets and get
1930		# them acked at xmit time, so that the corresponding subflow will
1931		# have almost always no outstanding pkts, the scheduler will pick
1932		# always the first subflow and we will have hard time testing
1933		# active backup and link switch-over.
1934		# Let's set some arbitrary (low) virtual link limits.
1935		init_shapers
1936		pm_nl_set_limits $ns1 0 3
1937		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
1938		pm_nl_set_limits $ns2 1 3
1939		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow
1940		pm_nl_add_endpoint $ns2 10.0.4.2 dev ns2eth4 flags subflow
1941		run_tests $ns1 $ns2 10.0.1.1 1
1942		chk_join_nr 3 3 3
1943		chk_add_nr 1 1
1944		chk_stale_nr $ns2 1 5 1
1945	fi
1946
1947	# accept and use add_addr with additional subflows and link loss
1948	# for bidirectional transfer
1949	if reset "multi flows, signal, bidi, link fail"; then
1950		init_shapers
1951		pm_nl_set_limits $ns1 0 3
1952		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
1953		pm_nl_set_limits $ns2 1 3
1954		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow
1955		pm_nl_add_endpoint $ns2 10.0.4.2 dev ns2eth4 flags subflow
1956		run_tests $ns1 $ns2 10.0.1.1 2
1957		chk_join_nr 3 3 3
1958		chk_add_nr 1 1
1959		chk_stale_nr $ns2 1 -1 1
1960	fi
1961
1962	# 2 subflows plus 1 backup subflow with a lossy link, backup
1963	# will never be used
1964	if reset "backup subflow unused, link failure"; then
1965		init_shapers
1966		pm_nl_set_limits $ns1 0 2
1967		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
1968		pm_nl_set_limits $ns2 1 2
1969		FAILING_LINKS="1"
1970		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow,backup
1971		run_tests $ns1 $ns2 10.0.1.1 1
1972		chk_join_nr 2 2 2
1973		chk_add_nr 1 1
1974		chk_link_usage $ns2 ns2eth3 $cinsent 0
1975	fi
1976
1977	# 2 lossy links after half transfer, backup will get half of
1978	# the traffic
1979	if reset "backup flow used, multi links fail"; then
1980		init_shapers
1981		pm_nl_set_limits $ns1 0 2
1982		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
1983		pm_nl_set_limits $ns2 1 2
1984		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow,backup
1985		FAILING_LINKS="1 2"
1986		run_tests $ns1 $ns2 10.0.1.1 1
1987		chk_join_nr 2 2 2
1988		chk_add_nr 1 1
1989		chk_stale_nr $ns2 2 4 2
1990		chk_link_usage $ns2 ns2eth3 $cinsent 50
1991	fi
1992
1993	# use a backup subflow with the first subflow on a lossy link
1994	# for bidirectional transfer
1995	if reset "backup flow used, bidi, link failure"; then
1996		init_shapers
1997		pm_nl_set_limits $ns1 0 2
1998		pm_nl_add_endpoint $ns1 10.0.2.1 dev ns1eth2 flags signal
1999		pm_nl_set_limits $ns2 1 3
2000		pm_nl_add_endpoint $ns2 10.0.3.2 dev ns2eth3 flags subflow,backup
2001		FAILING_LINKS="1 2"
2002		run_tests $ns1 $ns2 10.0.1.1 2
2003		chk_join_nr 2 2 2
2004		chk_add_nr 1 1
2005		chk_stale_nr $ns2 1 -1 2
2006		chk_link_usage $ns2 ns2eth3 $cinsent 50
2007	fi
2008}
2009
2010add_addr_timeout_tests()
2011{
2012	# add_addr timeout
2013	if reset_with_add_addr_timeout "signal address, ADD_ADDR timeout"; then
2014		pm_nl_set_limits $ns1 0 1
2015		pm_nl_set_limits $ns2 1 1
2016		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2017		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
2018		chk_join_nr 1 1 1
2019		chk_add_nr 4 0
2020	fi
2021
2022	# add_addr timeout IPv6
2023	if reset_with_add_addr_timeout "signal address, ADD_ADDR6 timeout" 6; then
2024		pm_nl_set_limits $ns1 0 1
2025		pm_nl_set_limits $ns2 1 1
2026		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2027		run_tests $ns1 $ns2 dead:beef:1::1 0 0 0 slow
2028		chk_join_nr 1 1 1
2029		chk_add_nr 4 0
2030	fi
2031
2032	# signal addresses timeout
2033	if reset_with_add_addr_timeout "signal addresses, ADD_ADDR timeout"; then
2034		pm_nl_set_limits $ns1 2 2
2035		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2036		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2037		pm_nl_set_limits $ns2 2 2
2038		run_tests $ns1 $ns2 10.0.1.1 0 0 0 speed_10
2039		chk_join_nr 2 2 2
2040		chk_add_nr 8 0
2041	fi
2042
2043	# signal invalid addresses timeout
2044	if reset_with_add_addr_timeout "invalid address, ADD_ADDR timeout"; then
2045		pm_nl_set_limits $ns1 2 2
2046		pm_nl_add_endpoint $ns1 10.0.12.1 flags signal
2047		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2048		pm_nl_set_limits $ns2 2 2
2049		run_tests $ns1 $ns2 10.0.1.1 0 0 0 speed_10
2050		chk_join_nr 1 1 1
2051		chk_add_nr 8 0
2052	fi
2053}
2054
2055remove_tests()
2056{
2057	# single subflow, remove
2058	if reset "remove single subflow"; then
2059		pm_nl_set_limits $ns1 0 1
2060		pm_nl_set_limits $ns2 0 1
2061		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2062		run_tests $ns1 $ns2 10.0.1.1 0 0 -1 slow
2063		chk_join_nr 1 1 1
2064		chk_rm_nr 1 1
2065	fi
2066
2067	# multiple subflows, remove
2068	if reset "remove multiple subflows"; then
2069		pm_nl_set_limits $ns1 0 2
2070		pm_nl_set_limits $ns2 0 2
2071		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
2072		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2073		run_tests $ns1 $ns2 10.0.1.1 0 0 -2 slow
2074		chk_join_nr 2 2 2
2075		chk_rm_nr 2 2
2076	fi
2077
2078	# single address, remove
2079	if reset "remove single address"; then
2080		pm_nl_set_limits $ns1 0 1
2081		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2082		pm_nl_set_limits $ns2 1 1
2083		run_tests $ns1 $ns2 10.0.1.1 0 -1 0 slow
2084		chk_join_nr 1 1 1
2085		chk_add_nr 1 1
2086		chk_rm_nr 1 1 invert
2087	fi
2088
2089	# subflow and signal, remove
2090	if reset "remove subflow and signal"; then
2091		pm_nl_set_limits $ns1 0 2
2092		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2093		pm_nl_set_limits $ns2 1 2
2094		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2095		run_tests $ns1 $ns2 10.0.1.1 0 -1 -1 slow
2096		chk_join_nr 2 2 2
2097		chk_add_nr 1 1
2098		chk_rm_nr 1 1
2099	fi
2100
2101	# subflows and signal, remove
2102	if reset "remove subflows and signal"; then
2103		pm_nl_set_limits $ns1 0 3
2104		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2105		pm_nl_set_limits $ns2 1 3
2106		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2107		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2108		run_tests $ns1 $ns2 10.0.1.1 0 -1 -2 slow
2109		chk_join_nr 3 3 3
2110		chk_add_nr 1 1
2111		chk_rm_nr 2 2
2112	fi
2113
2114	# addresses remove
2115	if reset "remove addresses"; then
2116		pm_nl_set_limits $ns1 3 3
2117		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal id 250
2118		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2119		pm_nl_add_endpoint $ns1 10.0.4.1 flags signal
2120		pm_nl_set_limits $ns2 3 3
2121		run_tests $ns1 $ns2 10.0.1.1 0 -3 0 slow
2122		chk_join_nr 3 3 3
2123		chk_add_nr 3 3
2124		chk_rm_nr 3 3 invert
2125	fi
2126
2127	# invalid addresses remove
2128	if reset "remove invalid addresses"; then
2129		pm_nl_set_limits $ns1 3 3
2130		pm_nl_add_endpoint $ns1 10.0.12.1 flags signal
2131		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2132		pm_nl_add_endpoint $ns1 10.0.14.1 flags signal
2133		pm_nl_set_limits $ns2 3 3
2134		run_tests $ns1 $ns2 10.0.1.1 0 -3 0 slow
2135		chk_join_nr 1 1 1
2136		chk_add_nr 3 3
2137		chk_rm_nr 3 1 invert
2138	fi
2139
2140	# subflows and signal, flush
2141	if reset "flush subflows and signal"; then
2142		pm_nl_set_limits $ns1 0 3
2143		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2144		pm_nl_set_limits $ns2 1 3
2145		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2146		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2147		run_tests $ns1 $ns2 10.0.1.1 0 -8 -8 slow
2148		chk_join_nr 3 3 3
2149		chk_add_nr 1 1
2150		chk_rm_nr 1 3 invert simult
2151	fi
2152
2153	# subflows flush
2154	if reset "flush subflows"; then
2155		pm_nl_set_limits $ns1 3 3
2156		pm_nl_set_limits $ns2 3 3
2157		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow id 150
2158		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2159		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2160		run_tests $ns1 $ns2 10.0.1.1 0 -8 -8 slow
2161		chk_join_nr 3 3 3
2162		chk_rm_nr 0 3 simult
2163	fi
2164
2165	# addresses flush
2166	if reset "flush addresses"; then
2167		pm_nl_set_limits $ns1 3 3
2168		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal id 250
2169		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2170		pm_nl_add_endpoint $ns1 10.0.4.1 flags signal
2171		pm_nl_set_limits $ns2 3 3
2172		run_tests $ns1 $ns2 10.0.1.1 0 -8 -8 slow
2173		chk_join_nr 3 3 3
2174		chk_add_nr 3 3
2175		chk_rm_nr 3 3 invert simult
2176	fi
2177
2178	# invalid addresses flush
2179	if reset "flush invalid addresses"; then
2180		pm_nl_set_limits $ns1 3 3
2181		pm_nl_add_endpoint $ns1 10.0.12.1 flags signal
2182		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal
2183		pm_nl_add_endpoint $ns1 10.0.14.1 flags signal
2184		pm_nl_set_limits $ns2 3 3
2185		run_tests $ns1 $ns2 10.0.1.1 0 -8 0 slow
2186		chk_join_nr 1 1 1
2187		chk_add_nr 3 3
2188		chk_rm_nr 3 1 invert
2189	fi
2190
2191	# remove id 0 subflow
2192	if reset "remove id 0 subflow"; then
2193		pm_nl_set_limits $ns1 0 1
2194		pm_nl_set_limits $ns2 0 1
2195		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2196		run_tests $ns1 $ns2 10.0.1.1 0 0 -9 slow
2197		chk_join_nr 1 1 1
2198		chk_rm_nr 1 1
2199	fi
2200
2201	# remove id 0 address
2202	if reset "remove id 0 address"; then
2203		pm_nl_set_limits $ns1 0 1
2204		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2205		pm_nl_set_limits $ns2 1 1
2206		run_tests $ns1 $ns2 10.0.1.1 0 -9 0 slow
2207		chk_join_nr 1 1 1
2208		chk_add_nr 1 1
2209		chk_rm_nr 1 1 invert
2210	fi
2211}
2212
2213add_tests()
2214{
2215	# add single subflow
2216	if reset "add single subflow"; then
2217		pm_nl_set_limits $ns1 0 1
2218		pm_nl_set_limits $ns2 0 1
2219		run_tests $ns1 $ns2 10.0.1.1 0 0 1 slow
2220		chk_join_nr 1 1 1
2221	fi
2222
2223	# add signal address
2224	if reset "add signal address"; then
2225		pm_nl_set_limits $ns1 0 1
2226		pm_nl_set_limits $ns2 1 1
2227		run_tests $ns1 $ns2 10.0.1.1 0 1 0 slow
2228		chk_join_nr 1 1 1
2229		chk_add_nr 1 1
2230	fi
2231
2232	# add multiple subflows
2233	if reset "add multiple subflows"; then
2234		pm_nl_set_limits $ns1 0 2
2235		pm_nl_set_limits $ns2 0 2
2236		run_tests $ns1 $ns2 10.0.1.1 0 0 2 slow
2237		chk_join_nr 2 2 2
2238	fi
2239
2240	# add multiple subflows IPv6
2241	if reset "add multiple subflows IPv6"; then
2242		pm_nl_set_limits $ns1 0 2
2243		pm_nl_set_limits $ns2 0 2
2244		run_tests $ns1 $ns2 dead:beef:1::1 0 0 2 slow
2245		chk_join_nr 2 2 2
2246	fi
2247
2248	# add multiple addresses IPv6
2249	if reset "add multiple addresses IPv6"; then
2250		pm_nl_set_limits $ns1 0 2
2251		pm_nl_set_limits $ns2 2 2
2252		run_tests $ns1 $ns2 dead:beef:1::1 0 2 0 slow
2253		chk_join_nr 2 2 2
2254		chk_add_nr 2 2
2255	fi
2256}
2257
2258ipv6_tests()
2259{
2260	# subflow IPv6
2261	if reset "single subflow IPv6"; then
2262		pm_nl_set_limits $ns1 0 1
2263		pm_nl_set_limits $ns2 0 1
2264		pm_nl_add_endpoint $ns2 dead:beef:3::2 dev ns2eth3 flags subflow
2265		run_tests $ns1 $ns2 dead:beef:1::1 0 0 0 slow
2266		chk_join_nr 1 1 1
2267	fi
2268
2269	# add_address, unused IPv6
2270	if reset "unused signal address IPv6"; then
2271		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2272		run_tests $ns1 $ns2 dead:beef:1::1 0 0 0 slow
2273		chk_join_nr 0 0 0
2274		chk_add_nr 1 1
2275	fi
2276
2277	# signal address IPv6
2278	if reset "single address IPv6"; then
2279		pm_nl_set_limits $ns1 0 1
2280		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2281		pm_nl_set_limits $ns2 1 1
2282		run_tests $ns1 $ns2 dead:beef:1::1 0 0 0 slow
2283		chk_join_nr 1 1 1
2284		chk_add_nr 1 1
2285	fi
2286
2287	# single address IPv6, remove
2288	if reset "remove single address IPv6"; then
2289		pm_nl_set_limits $ns1 0 1
2290		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2291		pm_nl_set_limits $ns2 1 1
2292		run_tests $ns1 $ns2 dead:beef:1::1 0 -1 0 slow
2293		chk_join_nr 1 1 1
2294		chk_add_nr 1 1
2295		chk_rm_nr 1 1 invert
2296	fi
2297
2298	# subflow and signal IPv6, remove
2299	if reset "remove subflow and signal IPv6"; then
2300		pm_nl_set_limits $ns1 0 2
2301		pm_nl_add_endpoint $ns1 dead:beef:2::1 flags signal
2302		pm_nl_set_limits $ns2 1 2
2303		pm_nl_add_endpoint $ns2 dead:beef:3::2 dev ns2eth3 flags subflow
2304		run_tests $ns1 $ns2 dead:beef:1::1 0 -1 -1 slow
2305		chk_join_nr 2 2 2
2306		chk_add_nr 1 1
2307		chk_rm_nr 1 1
2308	fi
2309}
2310
2311v4mapped_tests()
2312{
2313	# subflow IPv4-mapped to IPv4-mapped
2314	if reset "single subflow IPv4-mapped"; then
2315		pm_nl_set_limits $ns1 0 1
2316		pm_nl_set_limits $ns2 0 1
2317		pm_nl_add_endpoint $ns2 "::ffff:10.0.3.2" flags subflow
2318		run_tests $ns1 $ns2 "::ffff:10.0.1.1"
2319		chk_join_nr 1 1 1
2320	fi
2321
2322	# signal address IPv4-mapped with IPv4-mapped sk
2323	if reset "signal address IPv4-mapped"; then
2324		pm_nl_set_limits $ns1 0 1
2325		pm_nl_set_limits $ns2 1 1
2326		pm_nl_add_endpoint $ns1 "::ffff:10.0.2.1" flags signal
2327		run_tests $ns1 $ns2 "::ffff:10.0.1.1"
2328		chk_join_nr 1 1 1
2329		chk_add_nr 1 1
2330	fi
2331
2332	# subflow v4-map-v6
2333	if reset "single subflow v4-map-v6"; then
2334		pm_nl_set_limits $ns1 0 1
2335		pm_nl_set_limits $ns2 0 1
2336		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2337		run_tests $ns1 $ns2 "::ffff:10.0.1.1"
2338		chk_join_nr 1 1 1
2339	fi
2340
2341	# signal address v4-map-v6
2342	if reset "signal address v4-map-v6"; then
2343		pm_nl_set_limits $ns1 0 1
2344		pm_nl_set_limits $ns2 1 1
2345		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2346		run_tests $ns1 $ns2 "::ffff:10.0.1.1"
2347		chk_join_nr 1 1 1
2348		chk_add_nr 1 1
2349	fi
2350
2351	# subflow v6-map-v4
2352	if reset "single subflow v6-map-v4"; then
2353		pm_nl_set_limits $ns1 0 1
2354		pm_nl_set_limits $ns2 0 1
2355		pm_nl_add_endpoint $ns2 "::ffff:10.0.3.2" flags subflow
2356		run_tests $ns1 $ns2 10.0.1.1
2357		chk_join_nr 1 1 1
2358	fi
2359
2360	# signal address v6-map-v4
2361	if reset "signal address v6-map-v4"; then
2362		pm_nl_set_limits $ns1 0 1
2363		pm_nl_set_limits $ns2 1 1
2364		pm_nl_add_endpoint $ns1 "::ffff:10.0.2.1" flags signal
2365		run_tests $ns1 $ns2 10.0.1.1
2366		chk_join_nr 1 1 1
2367		chk_add_nr 1 1
2368	fi
2369
2370	# no subflow IPv6 to v4 address
2371	if reset "no JOIN with diff families v4-v6"; then
2372		pm_nl_set_limits $ns1 0 1
2373		pm_nl_set_limits $ns2 0 1
2374		pm_nl_add_endpoint $ns2 dead:beef:2::2 flags subflow
2375		run_tests $ns1 $ns2 10.0.1.1
2376		chk_join_nr 0 0 0
2377	fi
2378
2379	# no subflow IPv6 to v4 address even if v6 has a valid v4 at the end
2380	if reset "no JOIN with diff families v4-v6-2"; then
2381		pm_nl_set_limits $ns1 0 1
2382		pm_nl_set_limits $ns2 0 1
2383		pm_nl_add_endpoint $ns2 dead:beef:2::10.0.3.2 flags subflow
2384		run_tests $ns1 $ns2 10.0.1.1
2385		chk_join_nr 0 0 0
2386	fi
2387
2388	# no subflow IPv4 to v6 address, no need to slow down too then
2389	if reset "no JOIN with diff families v6-v4"; then
2390		pm_nl_set_limits $ns1 0 1
2391		pm_nl_set_limits $ns2 0 1
2392		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2393		run_tests $ns1 $ns2 dead:beef:1::1
2394		chk_join_nr 0 0 0
2395	fi
2396}
2397
2398backup_tests()
2399{
2400	# single subflow, backup
2401	if reset "single subflow, backup"; then
2402		pm_nl_set_limits $ns1 0 1
2403		pm_nl_set_limits $ns2 0 1
2404		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow,backup
2405		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow nobackup
2406		chk_join_nr 1 1 1
2407		chk_prio_nr 0 1
2408	fi
2409
2410	# single address, backup
2411	if reset "single address, backup"; then
2412		pm_nl_set_limits $ns1 0 1
2413		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2414		pm_nl_set_limits $ns2 1 1
2415		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
2416		chk_join_nr 1 1 1
2417		chk_add_nr 1 1
2418		chk_prio_nr 1 1
2419	fi
2420
2421	# single address with port, backup
2422	if reset "single address with port, backup"; then
2423		pm_nl_set_limits $ns1 0 1
2424		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2425		pm_nl_set_limits $ns2 1 1
2426		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
2427		chk_join_nr 1 1 1
2428		chk_add_nr 1 1
2429		chk_prio_nr 1 1
2430	fi
2431
2432	if reset "mpc backup"; then
2433		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow,backup
2434		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
2435		chk_join_nr 0 0 0
2436		chk_prio_nr 0 1
2437	fi
2438
2439	if reset "mpc backup both sides"; then
2440		pm_nl_add_endpoint $ns1 10.0.1.1 flags subflow,backup
2441		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow,backup
2442		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow
2443		chk_join_nr 0 0 0
2444		chk_prio_nr 1 1
2445	fi
2446
2447	if reset "mpc switch to backup"; then
2448		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow
2449		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
2450		chk_join_nr 0 0 0
2451		chk_prio_nr 0 1
2452	fi
2453
2454	if reset "mpc switch to backup both sides"; then
2455		pm_nl_add_endpoint $ns1 10.0.1.1 flags subflow
2456		pm_nl_add_endpoint $ns2 10.0.1.2 flags subflow
2457		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
2458		chk_join_nr 0 0 0
2459		chk_prio_nr 1 1
2460	fi
2461}
2462
2463add_addr_ports_tests()
2464{
2465	# signal address with port
2466	if reset "signal address with port"; then
2467		pm_nl_set_limits $ns1 0 1
2468		pm_nl_set_limits $ns2 1 1
2469		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2470		run_tests $ns1 $ns2 10.0.1.1
2471		chk_join_nr 1 1 1
2472		chk_add_nr 1 1 1
2473	fi
2474
2475	# subflow and signal with port
2476	if reset "subflow and signal with port"; then
2477		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2478		pm_nl_set_limits $ns1 0 2
2479		pm_nl_set_limits $ns2 1 2
2480		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2481		run_tests $ns1 $ns2 10.0.1.1
2482		chk_join_nr 2 2 2
2483		chk_add_nr 1 1 1
2484	fi
2485
2486	# single address with port, remove
2487	if reset "remove single address with port"; then
2488		pm_nl_set_limits $ns1 0 1
2489		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2490		pm_nl_set_limits $ns2 1 1
2491		run_tests $ns1 $ns2 10.0.1.1 0 -1 0 slow
2492		chk_join_nr 1 1 1
2493		chk_add_nr 1 1 1
2494		chk_rm_nr 1 1 invert
2495	fi
2496
2497	# subflow and signal with port, remove
2498	if reset "remove subflow and signal with port"; then
2499		pm_nl_set_limits $ns1 0 2
2500		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2501		pm_nl_set_limits $ns2 1 2
2502		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2503		run_tests $ns1 $ns2 10.0.1.1 0 -1 -1 slow
2504		chk_join_nr 2 2 2
2505		chk_add_nr 1 1 1
2506		chk_rm_nr 1 1
2507	fi
2508
2509	# subflows and signal with port, flush
2510	if reset "flush subflows and signal with port"; then
2511		pm_nl_set_limits $ns1 0 3
2512		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2513		pm_nl_set_limits $ns2 1 3
2514		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2515		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2516		run_tests $ns1 $ns2 10.0.1.1 0 -8 -2 slow
2517		chk_join_nr 3 3 3
2518		chk_add_nr 1 1
2519		chk_rm_nr 1 3 invert simult
2520	fi
2521
2522	# multiple addresses with port
2523	if reset "multiple addresses with port"; then
2524		pm_nl_set_limits $ns1 2 2
2525		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2526		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal port 10100
2527		pm_nl_set_limits $ns2 2 2
2528		run_tests $ns1 $ns2 10.0.1.1
2529		chk_join_nr 2 2 2
2530		chk_add_nr 2 2 2
2531	fi
2532
2533	# multiple addresses with ports
2534	if reset "multiple addresses with ports"; then
2535		pm_nl_set_limits $ns1 2 2
2536		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal port 10100
2537		pm_nl_add_endpoint $ns1 10.0.3.1 flags signal port 10101
2538		pm_nl_set_limits $ns2 2 2
2539		run_tests $ns1 $ns2 10.0.1.1
2540		chk_join_nr 2 2 2
2541		chk_add_nr 2 2 2
2542	fi
2543}
2544
2545syncookies_tests()
2546{
2547	# single subflow, syncookies
2548	if reset_with_cookies "single subflow with syn cookies"; then
2549		pm_nl_set_limits $ns1 0 1
2550		pm_nl_set_limits $ns2 0 1
2551		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2552		run_tests $ns1 $ns2 10.0.1.1
2553		chk_join_nr 1 1 1
2554	fi
2555
2556	# multiple subflows with syn cookies
2557	if reset_with_cookies "multiple subflows with syn cookies"; then
2558		pm_nl_set_limits $ns1 0 2
2559		pm_nl_set_limits $ns2 0 2
2560		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2561		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
2562		run_tests $ns1 $ns2 10.0.1.1
2563		chk_join_nr 2 2 2
2564	fi
2565
2566	# multiple subflows limited by server
2567	if reset_with_cookies "subflows limited by server w cookies"; then
2568		pm_nl_set_limits $ns1 0 1
2569		pm_nl_set_limits $ns2 0 2
2570		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2571		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow
2572		run_tests $ns1 $ns2 10.0.1.1
2573		chk_join_nr 2 1 1
2574	fi
2575
2576	# test signal address with cookies
2577	if reset_with_cookies "signal address with syn cookies"; then
2578		pm_nl_set_limits $ns1 0 1
2579		pm_nl_set_limits $ns2 1 1
2580		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2581		run_tests $ns1 $ns2 10.0.1.1
2582		chk_join_nr 1 1 1
2583		chk_add_nr 1 1
2584	fi
2585
2586	# test cookie with subflow and signal
2587	if reset_with_cookies "subflow and signal w cookies"; then
2588		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2589		pm_nl_set_limits $ns1 0 2
2590		pm_nl_set_limits $ns2 1 2
2591		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2592		run_tests $ns1 $ns2 10.0.1.1
2593		chk_join_nr 2 2 2
2594		chk_add_nr 1 1
2595	fi
2596
2597	# accept and use add_addr with additional subflows
2598	if reset_with_cookies "subflows and signal w. cookies"; then
2599		pm_nl_set_limits $ns1 0 3
2600		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2601		pm_nl_set_limits $ns2 1 3
2602		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2603		pm_nl_add_endpoint $ns2 10.0.4.2 flags subflow
2604		run_tests $ns1 $ns2 10.0.1.1
2605		chk_join_nr 3 3 3
2606		chk_add_nr 1 1
2607	fi
2608}
2609
2610checksum_tests()
2611{
2612	# checksum test 0 0
2613	if reset_with_checksum 0 0; then
2614		pm_nl_set_limits $ns1 0 1
2615		pm_nl_set_limits $ns2 0 1
2616		run_tests $ns1 $ns2 10.0.1.1
2617		chk_join_nr 0 0 0
2618	fi
2619
2620	# checksum test 1 1
2621	if reset_with_checksum 1 1; then
2622		pm_nl_set_limits $ns1 0 1
2623		pm_nl_set_limits $ns2 0 1
2624		run_tests $ns1 $ns2 10.0.1.1
2625		chk_join_nr 0 0 0
2626	fi
2627
2628	# checksum test 0 1
2629	if reset_with_checksum 0 1; then
2630		pm_nl_set_limits $ns1 0 1
2631		pm_nl_set_limits $ns2 0 1
2632		run_tests $ns1 $ns2 10.0.1.1
2633		chk_join_nr 0 0 0
2634	fi
2635
2636	# checksum test 1 0
2637	if reset_with_checksum 1 0; then
2638		pm_nl_set_limits $ns1 0 1
2639		pm_nl_set_limits $ns2 0 1
2640		run_tests $ns1 $ns2 10.0.1.1
2641		chk_join_nr 0 0 0
2642	fi
2643}
2644
2645deny_join_id0_tests()
2646{
2647	# subflow allow join id0 ns1
2648	if reset_with_allow_join_id0 "single subflow allow join id0 ns1" 1 0; then
2649		pm_nl_set_limits $ns1 1 1
2650		pm_nl_set_limits $ns2 1 1
2651		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2652		run_tests $ns1 $ns2 10.0.1.1
2653		chk_join_nr 1 1 1
2654	fi
2655
2656	# subflow allow join id0 ns2
2657	if reset_with_allow_join_id0 "single subflow allow join id0 ns2" 0 1; then
2658		pm_nl_set_limits $ns1 1 1
2659		pm_nl_set_limits $ns2 1 1
2660		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2661		run_tests $ns1 $ns2 10.0.1.1
2662		chk_join_nr 0 0 0
2663	fi
2664
2665	# signal address allow join id0 ns1
2666	# ADD_ADDRs are not affected by allow_join_id0 value.
2667	if reset_with_allow_join_id0 "signal address allow join id0 ns1" 1 0; then
2668		pm_nl_set_limits $ns1 1 1
2669		pm_nl_set_limits $ns2 1 1
2670		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2671		run_tests $ns1 $ns2 10.0.1.1
2672		chk_join_nr 1 1 1
2673		chk_add_nr 1 1
2674	fi
2675
2676	# signal address allow join id0 ns2
2677	# ADD_ADDRs are not affected by allow_join_id0 value.
2678	if reset_with_allow_join_id0 "signal address allow join id0 ns2" 0 1; then
2679		pm_nl_set_limits $ns1 1 1
2680		pm_nl_set_limits $ns2 1 1
2681		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2682		run_tests $ns1 $ns2 10.0.1.1
2683		chk_join_nr 1 1 1
2684		chk_add_nr 1 1
2685	fi
2686
2687	# subflow and address allow join id0 ns1
2688	if reset_with_allow_join_id0 "subflow and address allow join id0 1" 1 0; then
2689		pm_nl_set_limits $ns1 2 2
2690		pm_nl_set_limits $ns2 2 2
2691		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2692		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2693		run_tests $ns1 $ns2 10.0.1.1
2694		chk_join_nr 2 2 2
2695	fi
2696
2697	# subflow and address allow join id0 ns2
2698	if reset_with_allow_join_id0 "subflow and address allow join id0 2" 0 1; then
2699		pm_nl_set_limits $ns1 2 2
2700		pm_nl_set_limits $ns2 2 2
2701		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2702		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2703		run_tests $ns1 $ns2 10.0.1.1
2704		chk_join_nr 1 1 1
2705	fi
2706}
2707
2708fullmesh_tests()
2709{
2710	# fullmesh 1
2711	# 2 fullmesh addrs in ns2, added before the connection,
2712	# 1 non-fullmesh addr in ns1, added during the connection.
2713	if reset "fullmesh test 2x1"; then
2714		pm_nl_set_limits $ns1 0 4
2715		pm_nl_set_limits $ns2 1 4
2716		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow,fullmesh
2717		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow,fullmesh
2718		run_tests $ns1 $ns2 10.0.1.1 0 1 0 slow
2719		chk_join_nr 4 4 4
2720		chk_add_nr 1 1
2721	fi
2722
2723	# fullmesh 2
2724	# 1 non-fullmesh addr in ns1, added before the connection,
2725	# 1 fullmesh addr in ns2, added during the connection.
2726	if reset "fullmesh test 1x1"; then
2727		pm_nl_set_limits $ns1 1 3
2728		pm_nl_set_limits $ns2 1 3
2729		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2730		run_tests $ns1 $ns2 10.0.1.1 0 0 fullmesh_1 slow
2731		chk_join_nr 3 3 3
2732		chk_add_nr 1 1
2733	fi
2734
2735	# fullmesh 3
2736	# 1 non-fullmesh addr in ns1, added before the connection,
2737	# 2 fullmesh addrs in ns2, added during the connection.
2738	if reset "fullmesh test 1x2"; then
2739		pm_nl_set_limits $ns1 2 5
2740		pm_nl_set_limits $ns2 1 5
2741		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2742		run_tests $ns1 $ns2 10.0.1.1 0 0 fullmesh_2 slow
2743		chk_join_nr 5 5 5
2744		chk_add_nr 1 1
2745	fi
2746
2747	# fullmesh 4
2748	# 1 non-fullmesh addr in ns1, added before the connection,
2749	# 2 fullmesh addrs in ns2, added during the connection,
2750	# limit max_subflows to 4.
2751	if reset "fullmesh test 1x2, limited"; then
2752		pm_nl_set_limits $ns1 2 4
2753		pm_nl_set_limits $ns2 1 4
2754		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2755		run_tests $ns1 $ns2 10.0.1.1 0 0 fullmesh_2 slow
2756		chk_join_nr 4 4 4
2757		chk_add_nr 1 1
2758	fi
2759
2760	# set fullmesh flag
2761	if reset "set fullmesh flag test"; then
2762		pm_nl_set_limits $ns1 4 4
2763		pm_nl_add_endpoint $ns1 10.0.2.1 flags subflow
2764		pm_nl_set_limits $ns2 4 4
2765		run_tests $ns1 $ns2 10.0.1.1 0 0 1 slow fullmesh
2766		chk_join_nr 2 2 2
2767		chk_rm_nr 0 1
2768	fi
2769
2770	# set nofullmesh flag
2771	if reset "set nofullmesh flag test"; then
2772		pm_nl_set_limits $ns1 4 4
2773		pm_nl_add_endpoint $ns1 10.0.2.1 flags subflow,fullmesh
2774		pm_nl_set_limits $ns2 4 4
2775		run_tests $ns1 $ns2 10.0.1.1 0 0 fullmesh_1 slow nofullmesh
2776		chk_join_nr 2 2 2
2777		chk_rm_nr 0 1
2778	fi
2779
2780	# set backup,fullmesh flags
2781	if reset "set backup,fullmesh flags test"; then
2782		pm_nl_set_limits $ns1 4 4
2783		pm_nl_add_endpoint $ns1 10.0.2.1 flags subflow
2784		pm_nl_set_limits $ns2 4 4
2785		run_tests $ns1 $ns2 10.0.1.1 0 0 1 slow backup,fullmesh
2786		chk_join_nr 2 2 2
2787		chk_prio_nr 0 1
2788		chk_rm_nr 0 1
2789	fi
2790
2791	# set nobackup,nofullmesh flags
2792	if reset "set nobackup,nofullmesh flags test"; then
2793		pm_nl_set_limits $ns1 4 4
2794		pm_nl_set_limits $ns2 4 4
2795		pm_nl_add_endpoint $ns2 10.0.2.2 flags subflow,backup,fullmesh
2796		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow nobackup,nofullmesh
2797		chk_join_nr 2 2 2
2798		chk_prio_nr 0 1
2799		chk_rm_nr 0 1
2800	fi
2801}
2802
2803fastclose_tests()
2804{
2805	if reset "fastclose test"; then
2806		run_tests $ns1 $ns2 10.0.1.1 1024 0 fastclose_2
2807		chk_join_nr 0 0 0
2808		chk_fclose_nr 1 1
2809		chk_rst_nr 1 1 invert
2810	fi
2811}
2812
2813pedit_action_pkts()
2814{
2815	tc -n $ns2 -j -s action show action pedit index 100 | \
2816		grep "packets" | \
2817		sed 's/.*"packets":\([0-9]\+\),.*/\1/'
2818}
2819
2820fail_tests()
2821{
2822	# single subflow
2823	if reset_with_fail "Infinite map" 1; then
2824		run_tests $ns1 $ns2 10.0.1.1 128
2825		chk_join_nr 0 0 0 +1 +0 1 0 1 "$(pedit_action_pkts)"
2826		chk_fail_nr 1 -1 invert
2827	fi
2828
2829	# multiple subflows
2830	if reset_with_fail "MP_FAIL MP_RST" 2; then
2831		tc -n $ns2 qdisc add dev ns2eth1 root netem rate 1mbit delay 5
2832		pm_nl_set_limits $ns1 0 1
2833		pm_nl_set_limits $ns2 0 1
2834		pm_nl_add_endpoint $ns2 10.0.2.2 dev ns2eth2 flags subflow
2835		run_tests $ns1 $ns2 10.0.1.1 1024
2836		chk_join_nr 1 1 1 1 0 1 1 0 "$(pedit_action_pkts)"
2837	fi
2838}
2839
2840userspace_tests()
2841{
2842	# userspace pm type prevents add_addr
2843	if reset "userspace pm type prevents add_addr"; then
2844		set_userspace_pm $ns1
2845		pm_nl_set_limits $ns1 0 2
2846		pm_nl_set_limits $ns2 0 2
2847		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2848		run_tests $ns1 $ns2 10.0.1.1
2849		chk_join_nr 0 0 0
2850		chk_add_nr 0 0
2851	fi
2852
2853	# userspace pm type does not echo add_addr without daemon
2854	if reset "userspace pm no echo w/o daemon"; then
2855		set_userspace_pm $ns2
2856		pm_nl_set_limits $ns1 0 2
2857		pm_nl_set_limits $ns2 0 2
2858		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2859		run_tests $ns1 $ns2 10.0.1.1
2860		chk_join_nr 0 0 0
2861		chk_add_nr 1 0
2862	fi
2863
2864	# userspace pm type rejects join
2865	if reset "userspace pm type rejects join"; then
2866		set_userspace_pm $ns1
2867		pm_nl_set_limits $ns1 1 1
2868		pm_nl_set_limits $ns2 1 1
2869		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2870		run_tests $ns1 $ns2 10.0.1.1
2871		chk_join_nr 1 1 0
2872	fi
2873
2874	# userspace pm type does not send join
2875	if reset "userspace pm type does not send join"; then
2876		set_userspace_pm $ns2
2877		pm_nl_set_limits $ns1 1 1
2878		pm_nl_set_limits $ns2 1 1
2879		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2880		run_tests $ns1 $ns2 10.0.1.1
2881		chk_join_nr 0 0 0
2882	fi
2883
2884	# userspace pm type prevents mp_prio
2885	if reset "userspace pm type prevents mp_prio"; then
2886		set_userspace_pm $ns1
2887		pm_nl_set_limits $ns1 1 1
2888		pm_nl_set_limits $ns2 1 1
2889		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2890		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow backup
2891		chk_join_nr 1 1 0
2892		chk_prio_nr 0 0
2893	fi
2894
2895	# userspace pm type prevents rm_addr
2896	if reset "userspace pm type prevents rm_addr"; then
2897		set_userspace_pm $ns1
2898		set_userspace_pm $ns2
2899		pm_nl_set_limits $ns1 0 1
2900		pm_nl_set_limits $ns2 0 1
2901		pm_nl_add_endpoint $ns2 10.0.3.2 flags subflow
2902		run_tests $ns1 $ns2 10.0.1.1 0 0 -1 slow
2903		chk_join_nr 0 0 0
2904		chk_rm_nr 0 0
2905	fi
2906
2907	# userspace pm add & remove address
2908	if reset "userspace pm add & remove address"; then
2909		set_userspace_pm $ns1
2910		pm_nl_set_limits $ns2 1 1
2911		run_tests $ns1 $ns2 10.0.1.1 0 userspace_1 0 slow
2912		chk_join_nr 1 1 1
2913		chk_add_nr 1 1
2914		chk_rm_nr 1 1 invert
2915	fi
2916
2917	# userspace pm create destroy subflow
2918	if reset "userspace pm create destroy subflow"; then
2919		set_userspace_pm $ns2
2920		pm_nl_set_limits $ns1 0 1
2921		run_tests $ns1 $ns2 10.0.1.1 0 0 userspace_1 slow
2922		chk_join_nr 1 1 1
2923		chk_rm_nr 0 1
2924	fi
2925}
2926
2927endpoint_tests()
2928{
2929	# userspace pm type prevents add_addr
2930	if reset "implicit EP"; then
2931		pm_nl_set_limits $ns1 2 2
2932		pm_nl_set_limits $ns2 2 2
2933		pm_nl_add_endpoint $ns1 10.0.2.1 flags signal
2934		run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow &
2935
2936		wait_mpj $ns1
2937		pm_nl_check_endpoint 1 "creation" \
2938			$ns2 10.0.2.2 id 1 flags implicit
2939
2940		pm_nl_add_endpoint $ns2 10.0.2.2 id 33
2941		pm_nl_check_endpoint 0 "ID change is prevented" \
2942			$ns2 10.0.2.2 id 1 flags implicit
2943
2944		pm_nl_add_endpoint $ns2 10.0.2.2 flags signal
2945		pm_nl_check_endpoint 0 "modif is allowed" \
2946			$ns2 10.0.2.2 id 1 flags signal
2947		wait
2948	fi
2949
2950	if reset "delete and re-add"; then
2951		pm_nl_set_limits $ns1 1 1
2952		pm_nl_set_limits $ns2 1 1
2953		pm_nl_add_endpoint $ns2 10.0.2.2 id 2 dev ns2eth2 flags subflow
2954		run_tests $ns1 $ns2 10.0.1.1 4 0 0 slow &
2955
2956		wait_mpj $ns2
2957		pm_nl_del_endpoint $ns2 2 10.0.2.2
2958		sleep 0.5
2959		chk_subflow_nr needtitle "after delete" 1
2960
2961		pm_nl_add_endpoint $ns2 10.0.2.2 dev ns2eth2 flags subflow
2962		wait_mpj $ns2
2963		chk_subflow_nr "" "after re-add" 2
2964		wait
2965	fi
2966}
2967
2968# [$1: error message]
2969usage()
2970{
2971	if [ -n "${1}" ]; then
2972		echo "${1}"
2973		ret=1
2974	fi
2975
2976	echo "mptcp_join usage:"
2977
2978	local key
2979	for key in "${!all_tests[@]}"; do
2980		echo "  -${key} ${all_tests[${key}]}"
2981	done
2982
2983	echo "  -c capture pcap files"
2984	echo "  -C enable data checksum"
2985	echo "  -i use ip mptcp"
2986	echo "  -h help"
2987
2988	echo "[test ids|names]"
2989
2990	exit ${ret}
2991}
2992
2993
2994# Use a "simple" array to force an specific order we cannot have with an associative one
2995all_tests_sorted=(
2996	f@subflows_tests
2997	e@subflows_error_tests
2998	s@signal_address_tests
2999	l@link_failure_tests
3000	t@add_addr_timeout_tests
3001	r@remove_tests
3002	a@add_tests
3003	6@ipv6_tests
3004	4@v4mapped_tests
3005	b@backup_tests
3006	p@add_addr_ports_tests
3007	k@syncookies_tests
3008	S@checksum_tests
3009	d@deny_join_id0_tests
3010	m@fullmesh_tests
3011	z@fastclose_tests
3012	F@fail_tests
3013	u@userspace_tests
3014	I@endpoint_tests
3015)
3016
3017all_tests_args=""
3018all_tests_names=()
3019for subtests in "${all_tests_sorted[@]}"; do
3020	key="${subtests%@*}"
3021	value="${subtests#*@}"
3022
3023	all_tests_args+="${key}"
3024	all_tests_names+=("${value}")
3025	all_tests[${key}]="${value}"
3026done
3027
3028tests=()
3029while getopts "${all_tests_args}cCih" opt; do
3030	case $opt in
3031		["${all_tests_args}"])
3032			tests+=("${all_tests[${opt}]}")
3033			;;
3034		c)
3035			capture=1
3036			;;
3037		C)
3038			checksum=1
3039			;;
3040		i)
3041			ip_mptcp=1
3042			;;
3043		h)
3044			usage
3045			;;
3046		*)
3047			usage "Unknown option: -${opt}"
3048			;;
3049	esac
3050done
3051
3052shift $((OPTIND - 1))
3053
3054for arg in "${@}"; do
3055	if [[ "${arg}" =~ ^[0-9]+$ ]]; then
3056		only_tests_ids+=("${arg}")
3057	else
3058		only_tests_names+=("${arg}")
3059	fi
3060done
3061
3062if [ ${#tests[@]} -eq 0 ]; then
3063	tests=("${all_tests_names[@]}")
3064fi
3065
3066for subtests in "${tests[@]}"; do
3067	"${subtests}"
3068done
3069
3070if [ ${ret} -ne 0 ]; then
3071	echo
3072	echo "${#failed_tests[@]} failure(s) has(ve) been detected:"
3073	for i in $(get_failed_tests_ids); do
3074		echo -e "\t- ${i}: ${failed_tests[${i}]}"
3075	done
3076	echo
3077fi
3078
3079exit $ret
3080