1#!/bin/sh 2# SPDX-License-Identifier: GPL-2.0 3# 4# nft_concat_range.sh - Tests for sets with concatenation of ranged fields 5# 6# Copyright (c) 2019 Red Hat GmbH 7# 8# Author: Stefano Brivio <sbrivio@redhat.com> 9# 10# shellcheck disable=SC2154,SC2034,SC2016,SC2030,SC2031 11# ^ Configuration and templates sourced with eval, counters reused in subshells 12 13KSELFTEST_SKIP=4 14 15# Available test groups: 16# - reported_issues: check for issues that were reported in the past 17# - correctness: check that packets match given entries, and only those 18# - concurrency: attempt races between insertion, deletion and lookup 19# - timeout: check that packets match entries until they expire 20# - performance: estimate matching rate, compare with rbtree and hash baselines 21TESTS="reported_issues correctness concurrency timeout" 22[ "${quicktest}" != "1" ] && TESTS="${TESTS} performance" 23 24# Set types, defined by TYPE_ variables below 25TYPES="net_port port_net net6_port port_proto net6_port_mac net6_port_mac_proto 26 net_port_net net_mac net_mac_icmp net6_mac_icmp net6_port_net6_port 27 net_port_mac_proto_net" 28 29# Reported bugs, also described by TYPE_ variables below 30BUGS="flush_remove_add" 31 32# List of possible paths to pktgen script from kernel tree for performance tests 33PKTGEN_SCRIPT_PATHS=" 34 ../../../samples/pktgen/pktgen_bench_xmit_mode_netif_receive.sh 35 pktgen/pktgen_bench_xmit_mode_netif_receive.sh" 36 37# Definition of set types: 38# display display text for test report 39# type_spec nftables set type specifier 40# chain_spec nftables type specifier for rules mapping to set 41# dst call sequence of format_*() functions for destination fields 42# src call sequence of format_*() functions for source fields 43# start initial integer used to generate addresses and ports 44# count count of entries to generate and match 45# src_delta number summed to destination generator for source fields 46# tools list of tools for correctness and timeout tests, any can be used 47# proto L4 protocol of test packets 48# 49# race_repeat race attempts per thread, 0 disables concurrency test for type 50# flood_tools list of tools for concurrency tests, any can be used 51# flood_proto L4 protocol of test packets for concurrency tests 52# flood_spec nftables type specifier for concurrency tests 53# 54# perf_duration duration of single pktgen injection test 55# perf_spec nftables type specifier for performance tests 56# perf_dst format_*() functions for destination fields in performance test 57# perf_src format_*() functions for source fields in performance test 58# perf_entries number of set entries for performance test 59# perf_proto L3 protocol of test packets 60TYPE_net_port=" 61display net,port 62type_spec ipv4_addr . inet_service 63chain_spec ip daddr . udp dport 64dst addr4 port 65src 66start 1 67count 5 68src_delta 2000 69tools sendip nc bash 70proto udp 71 72race_repeat 3 73flood_tools iperf3 iperf netperf 74flood_proto udp 75flood_spec ip daddr . udp dport 76 77perf_duration 5 78perf_spec ip daddr . udp dport 79perf_dst addr4 port 80perf_src 81perf_entries 1000 82perf_proto ipv4 83" 84 85TYPE_port_net=" 86display port,net 87type_spec inet_service . ipv4_addr 88chain_spec udp dport . ip daddr 89dst port addr4 90src 91start 1 92count 5 93src_delta 2000 94tools sendip nc bash 95proto udp 96 97race_repeat 3 98flood_tools iperf3 iperf netperf 99flood_proto udp 100flood_spec udp dport . ip daddr 101 102perf_duration 5 103perf_spec udp dport . ip daddr 104perf_dst port addr4 105perf_src 106perf_entries 100 107perf_proto ipv4 108" 109 110TYPE_net6_port=" 111display net6,port 112type_spec ipv6_addr . inet_service 113chain_spec ip6 daddr . udp dport 114dst addr6 port 115src 116start 10 117count 5 118src_delta 2000 119tools sendip nc bash 120proto udp6 121 122race_repeat 3 123flood_tools iperf3 iperf netperf 124flood_proto tcp6 125flood_spec ip6 daddr . udp dport 126 127perf_duration 5 128perf_spec ip6 daddr . udp dport 129perf_dst addr6 port 130perf_src 131perf_entries 1000 132perf_proto ipv6 133" 134 135TYPE_port_proto=" 136display port,proto 137type_spec inet_service . inet_proto 138chain_spec udp dport . meta l4proto 139dst port proto 140src 141start 1 142count 5 143src_delta 2000 144tools sendip nc bash 145proto udp 146 147race_repeat 0 148 149perf_duration 5 150perf_spec udp dport . meta l4proto 151perf_dst port proto 152perf_src 153perf_entries 30000 154perf_proto ipv4 155" 156 157TYPE_net6_port_mac=" 158display net6,port,mac 159type_spec ipv6_addr . inet_service . ether_addr 160chain_spec ip6 daddr . udp dport . ether saddr 161dst addr6 port 162src mac 163start 10 164count 5 165src_delta 2000 166tools sendip nc bash 167proto udp6 168 169race_repeat 0 170 171perf_duration 5 172perf_spec ip6 daddr . udp dport . ether daddr 173perf_dst addr6 port mac 174perf_src 175perf_entries 10 176perf_proto ipv6 177" 178 179TYPE_net6_port_mac_proto=" 180display net6,port,mac,proto 181type_spec ipv6_addr . inet_service . ether_addr . inet_proto 182chain_spec ip6 daddr . udp dport . ether saddr . meta l4proto 183dst addr6 port 184src mac proto 185start 10 186count 5 187src_delta 2000 188tools sendip nc bash 189proto udp6 190 191race_repeat 0 192 193perf_duration 5 194perf_spec ip6 daddr . udp dport . ether daddr . meta l4proto 195perf_dst addr6 port mac proto 196perf_src 197perf_entries 1000 198perf_proto ipv6 199" 200 201TYPE_net_port_net=" 202display net,port,net 203type_spec ipv4_addr . inet_service . ipv4_addr 204chain_spec ip daddr . udp dport . ip saddr 205dst addr4 port 206src addr4 207start 1 208count 5 209src_delta 2000 210tools sendip nc bash 211proto udp 212 213race_repeat 3 214flood_tools iperf3 iperf netperf 215flood_proto tcp 216flood_spec ip daddr . udp dport . ip saddr 217 218perf_duration 0 219" 220 221TYPE_net6_port_net6_port=" 222display net6,port,net6,port 223type_spec ipv6_addr . inet_service . ipv6_addr . inet_service 224chain_spec ip6 daddr . udp dport . ip6 saddr . udp sport 225dst addr6 port 226src addr6 port 227start 10 228count 5 229src_delta 2000 230tools sendip nc 231proto udp6 232 233race_repeat 3 234flood_tools iperf3 iperf netperf 235flood_proto tcp6 236flood_spec ip6 daddr . tcp dport . ip6 saddr . tcp sport 237 238perf_duration 0 239" 240 241TYPE_net_port_mac_proto_net=" 242display net,port,mac,proto,net 243type_spec ipv4_addr . inet_service . ether_addr . inet_proto . ipv4_addr 244chain_spec ip daddr . udp dport . ether saddr . meta l4proto . ip saddr 245dst addr4 port 246src mac proto addr4 247start 1 248count 5 249src_delta 2000 250tools sendip nc bash 251proto udp 252 253race_repeat 0 254 255perf_duration 0 256" 257 258TYPE_net_mac=" 259display net,mac 260type_spec ipv4_addr . ether_addr 261chain_spec ip daddr . ether saddr 262dst addr4 263src mac 264start 1 265count 5 266src_delta 2000 267tools sendip nc bash 268proto udp 269 270race_repeat 0 271 272perf_duration 5 273perf_spec ip daddr . ether daddr 274perf_dst addr4 mac 275perf_src 276perf_entries 1000 277perf_proto ipv4 278" 279 280TYPE_net_mac_icmp=" 281display net,mac - ICMP 282type_spec ipv4_addr . ether_addr 283chain_spec ip daddr . ether saddr 284dst addr4 285src mac 286start 1 287count 5 288src_delta 2000 289tools ping 290proto icmp 291 292race_repeat 0 293 294perf_duration 0 295" 296 297TYPE_net6_mac_icmp=" 298display net6,mac - ICMPv6 299type_spec ipv6_addr . ether_addr 300chain_spec ip6 daddr . ether saddr 301dst addr6 302src mac 303start 10 304count 50 305src_delta 2000 306tools ping 307proto icmp6 308 309race_repeat 0 310 311perf_duration 0 312" 313 314TYPE_net_port_proto_net=" 315display net,port,proto,net 316type_spec ipv4_addr . inet_service . inet_proto . ipv4_addr 317chain_spec ip daddr . udp dport . meta l4proto . ip saddr 318dst addr4 port proto 319src addr4 320start 1 321count 5 322src_delta 2000 323tools sendip nc 324proto udp 325 326race_repeat 3 327flood_tools iperf3 iperf netperf 328flood_proto tcp 329flood_spec ip daddr . tcp dport . meta l4proto . ip saddr 330 331perf_duration 0 332" 333 334# Definition of tests for bugs reported in the past: 335# display display text for test report 336TYPE_flush_remove_add=" 337display Add two elements, flush, re-add 338" 339 340# Set template for all tests, types and rules are filled in depending on test 341set_template=' 342flush ruleset 343 344table inet filter { 345 counter test { 346 packets 0 bytes 0 347 } 348 349 set test { 350 type ${type_spec} 351 flags interval,timeout 352 } 353 354 chain input { 355 type filter hook prerouting priority 0; policy accept; 356 ${chain_spec} @test counter name \"test\" 357 } 358} 359 360table netdev perf { 361 counter test { 362 packets 0 bytes 0 363 } 364 365 counter match { 366 packets 0 bytes 0 367 } 368 369 set test { 370 type ${type_spec} 371 flags interval 372 } 373 374 set norange { 375 type ${type_spec} 376 } 377 378 set noconcat { 379 type ${type_spec%% *} 380 flags interval 381 } 382 383 chain test { 384 type filter hook ingress device veth_a priority 0; 385 } 386} 387' 388 389err_buf= 390info_buf= 391 392# Append string to error buffer 393err() { 394 err_buf="${err_buf}${1} 395" 396} 397 398# Append string to information buffer 399info() { 400 info_buf="${info_buf}${1} 401" 402} 403 404# Flush error buffer to stdout 405err_flush() { 406 printf "%s" "${err_buf}" 407 err_buf= 408} 409 410# Flush information buffer to stdout 411info_flush() { 412 printf "%s" "${info_buf}" 413 info_buf= 414} 415 416# Setup veth pair: this namespace receives traffic, B generates it 417setup_veth() { 418 ip netns add B 419 ip link add veth_a type veth peer name veth_b || return 1 420 421 ip link set veth_a up 422 ip link set veth_b netns B 423 424 ip -n B link set veth_b up 425 426 ip addr add dev veth_a 10.0.0.1 427 ip route add default dev veth_a 428 429 ip -6 addr add fe80::1/64 dev veth_a nodad 430 ip -6 addr add 2001:db8::1/64 dev veth_a nodad 431 ip -6 route add default dev veth_a 432 433 ip -n B route add default dev veth_b 434 435 ip -6 -n B addr add fe80::2/64 dev veth_b nodad 436 ip -6 -n B addr add 2001:db8::2/64 dev veth_b nodad 437 ip -6 -n B route add default dev veth_b 438 439 B() { 440 ip netns exec B "$@" >/dev/null 2>&1 441 } 442 443 sleep 2 444} 445 446# Fill in set template and initialise set 447setup_set() { 448 eval "echo \"${set_template}\"" | nft -f - 449} 450 451# Check that at least one of the needed tools is available 452check_tools() { 453 [ -z "${tools}" ] && return 0 454 455 __tools= 456 for tool in ${tools}; do 457 if [ "${tool}" = "nc" ] && [ "${proto}" = "udp6" ] && \ 458 ! nc -u -w0 1.1.1.1 1 2>/dev/null; then 459 # Some GNU netcat builds might not support IPv6 460 __tools="${__tools} netcat-openbsd" 461 continue 462 fi 463 __tools="${__tools} ${tool}" 464 465 command -v "${tool}" >/dev/null && return 0 466 done 467 err "need one of:${__tools}, skipping" && return 1 468} 469 470# Set up function to send ICMP packets 471setup_send_icmp() { 472 send_icmp() { 473 B ping -c1 -W1 "${dst_addr4}" >/dev/null 2>&1 474 } 475} 476 477# Set up function to send ICMPv6 packets 478setup_send_icmp6() { 479 if command -v ping6 >/dev/null; then 480 send_icmp6() { 481 ip -6 addr add "${dst_addr6}" dev veth_a nodad \ 482 2>/dev/null 483 B ping6 -q -c1 -W1 "${dst_addr6}" 484 } 485 else 486 send_icmp6() { 487 ip -6 addr add "${dst_addr6}" dev veth_a nodad \ 488 2>/dev/null 489 B ping -q -6 -c1 -W1 "${dst_addr6}" 490 } 491 fi 492} 493 494# Set up function to send single UDP packets on IPv4 495setup_send_udp() { 496 if command -v sendip >/dev/null; then 497 send_udp() { 498 [ -n "${src_port}" ] && src_port="-us ${src_port}" 499 [ -n "${dst_port}" ] && dst_port="-ud ${dst_port}" 500 [ -n "${src_addr4}" ] && src_addr4="-is ${src_addr4}" 501 502 # shellcheck disable=SC2086 # sendip needs split options 503 B sendip -p ipv4 -p udp ${src_addr4} ${src_port} \ 504 ${dst_port} "${dst_addr4}" 505 506 src_port= 507 dst_port= 508 src_addr4= 509 } 510 elif command -v nc >/dev/null; then 511 if nc -u -w0 1.1.1.1 1 2>/dev/null; then 512 # OpenBSD netcat 513 nc_opt="-w0" 514 else 515 # GNU netcat 516 nc_opt="-q0" 517 fi 518 519 send_udp() { 520 if [ -n "${src_addr4}" ]; then 521 B ip addr add "${src_addr4}" dev veth_b 522 __src_addr4="-s ${src_addr4}" 523 fi 524 ip addr add "${dst_addr4}" dev veth_a 2>/dev/null 525 [ -n "${src_port}" ] && src_port="-p ${src_port}" 526 527 echo "" | B nc -u "${nc_opt}" "${__src_addr4}" \ 528 "${src_port}" "${dst_addr4}" "${dst_port}" 529 530 src_addr4= 531 src_port= 532 } 533 elif [ -z "$(bash -c 'type -p')" ]; then 534 send_udp() { 535 ip addr add "${dst_addr4}" dev veth_a 2>/dev/null 536 if [ -n "${src_addr4}" ]; then 537 B ip addr add "${src_addr4}/16" dev veth_b 538 B ip route add default dev veth_b 539 fi 540 541 B bash -c "echo > /dev/udp/${dst_addr4}/${dst_port}" 542 543 if [ -n "${src_addr4}" ]; then 544 B ip addr del "${src_addr4}/16" dev veth_b 545 fi 546 src_addr4= 547 } 548 else 549 return 1 550 fi 551} 552 553# Set up function to send single UDP packets on IPv6 554setup_send_udp6() { 555 if command -v sendip >/dev/null; then 556 send_udp6() { 557 [ -n "${src_port}" ] && src_port="-us ${src_port}" 558 [ -n "${dst_port}" ] && dst_port="-ud ${dst_port}" 559 if [ -n "${src_addr6}" ]; then 560 src_addr6="-6s ${src_addr6}" 561 else 562 src_addr6="-6s 2001:db8::2" 563 fi 564 ip -6 addr add "${dst_addr6}" dev veth_a nodad \ 565 2>/dev/null 566 567 # shellcheck disable=SC2086 # this needs split options 568 B sendip -p ipv6 -p udp ${src_addr6} ${src_port} \ 569 ${dst_port} "${dst_addr6}" 570 571 src_port= 572 dst_port= 573 src_addr6= 574 } 575 elif command -v nc >/dev/null && nc -u -w0 1.1.1.1 1 2>/dev/null; then 576 # GNU netcat might not work with IPv6, try next tool 577 send_udp6() { 578 ip -6 addr add "${dst_addr6}" dev veth_a nodad \ 579 2>/dev/null 580 if [ -n "${src_addr6}" ]; then 581 B ip addr add "${src_addr6}" dev veth_b nodad 582 else 583 src_addr6="2001:db8::2" 584 fi 585 [ -n "${src_port}" ] && src_port="-p ${src_port}" 586 587 # shellcheck disable=SC2086 # this needs split options 588 echo "" | B nc -u w0 "-s${src_addr6}" ${src_port} \ 589 ${dst_addr6} ${dst_port} 590 591 src_addr6= 592 src_port= 593 } 594 elif [ -z "$(bash -c 'type -p')" ]; then 595 send_udp6() { 596 ip -6 addr add "${dst_addr6}" dev veth_a nodad \ 597 2>/dev/null 598 B ip addr add "${src_addr6}" dev veth_b nodad 599 B bash -c "echo > /dev/udp/${dst_addr6}/${dst_port}" 600 ip -6 addr del "${dst_addr6}" dev veth_a 2>/dev/null 601 } 602 else 603 return 1 604 fi 605} 606 607# Set up function to send TCP traffic on IPv4 608setup_flood_tcp() { 609 if command -v iperf3 >/dev/null; then 610 flood_tcp() { 611 [ -n "${dst_port}" ] && dst_port="-p ${dst_port}" 612 if [ -n "${src_addr4}" ]; then 613 B ip addr add "${src_addr4}/16" dev veth_b 614 src_addr4="-B ${src_addr4}" 615 else 616 B ip addr add dev veth_b 10.0.0.2 617 src_addr4="-B 10.0.0.2" 618 fi 619 if [ -n "${src_port}" ]; then 620 src_port="--cport ${src_port}" 621 fi 622 B ip route add default dev veth_b 2>/dev/null 623 ip addr add "${dst_addr4}" dev veth_a 2>/dev/null 624 625 # shellcheck disable=SC2086 # this needs split options 626 iperf3 -s -DB "${dst_addr4}" ${dst_port} >/dev/null 2>&1 627 sleep 2 628 629 # shellcheck disable=SC2086 # this needs split options 630 B iperf3 -c "${dst_addr4}" ${dst_port} ${src_port} \ 631 ${src_addr4} -l16 -t 1000 632 633 src_addr4= 634 src_port= 635 dst_port= 636 } 637 elif command -v iperf >/dev/null; then 638 flood_tcp() { 639 [ -n "${dst_port}" ] && dst_port="-p ${dst_port}" 640 if [ -n "${src_addr4}" ]; then 641 B ip addr add "${src_addr4}/16" dev veth_b 642 src_addr4="-B ${src_addr4}" 643 else 644 B ip addr add dev veth_b 10.0.0.2 2>/dev/null 645 src_addr4="-B 10.0.0.2" 646 fi 647 if [ -n "${src_port}" ]; then 648 src_addr4="${src_addr4}:${src_port}" 649 fi 650 B ip route add default dev veth_b 651 ip addr add "${dst_addr4}" dev veth_a 2>/dev/null 652 653 # shellcheck disable=SC2086 # this needs split options 654 iperf -s -DB "${dst_addr4}" ${dst_port} >/dev/null 2>&1 655 sleep 2 656 657 # shellcheck disable=SC2086 # this needs split options 658 B iperf -c "${dst_addr4}" ${dst_port} ${src_addr4} \ 659 -l20 -t 1000 660 661 src_addr4= 662 src_port= 663 dst_port= 664 } 665 elif command -v netperf >/dev/null; then 666 flood_tcp() { 667 [ -n "${dst_port}" ] && dst_port="-p ${dst_port}" 668 if [ -n "${src_addr4}" ]; then 669 B ip addr add "${src_addr4}/16" dev veth_b 670 else 671 B ip addr add dev veth_b 10.0.0.2 672 src_addr4="10.0.0.2" 673 fi 674 if [ -n "${src_port}" ]; then 675 dst_port="${dst_port},${src_port}" 676 fi 677 B ip route add default dev veth_b 678 ip addr add "${dst_addr4}" dev veth_a 2>/dev/null 679 680 # shellcheck disable=SC2086 # this needs split options 681 netserver -4 ${dst_port} -L "${dst_addr4}" \ 682 >/dev/null 2>&1 683 sleep 2 684 685 # shellcheck disable=SC2086 # this needs split options 686 B netperf -4 -H "${dst_addr4}" ${dst_port} \ 687 -L "${src_addr4}" -l 1000 -t TCP_STREAM 688 689 src_addr4= 690 src_port= 691 dst_port= 692 } 693 else 694 return 1 695 fi 696} 697 698# Set up function to send TCP traffic on IPv6 699setup_flood_tcp6() { 700 if command -v iperf3 >/dev/null; then 701 flood_tcp6() { 702 [ -n "${dst_port}" ] && dst_port="-p ${dst_port}" 703 if [ -n "${src_addr6}" ]; then 704 B ip addr add "${src_addr6}" dev veth_b nodad 705 src_addr6="-B ${src_addr6}" 706 else 707 src_addr6="-B 2001:db8::2" 708 fi 709 if [ -n "${src_port}" ]; then 710 src_port="--cport ${src_port}" 711 fi 712 B ip route add default dev veth_b 713 ip -6 addr add "${dst_addr6}" dev veth_a nodad \ 714 2>/dev/null 715 716 # shellcheck disable=SC2086 # this needs split options 717 iperf3 -s -DB "${dst_addr6}" ${dst_port} >/dev/null 2>&1 718 sleep 2 719 720 # shellcheck disable=SC2086 # this needs split options 721 B iperf3 -c "${dst_addr6}" ${dst_port} \ 722 ${src_port} ${src_addr6} -l16 -t 1000 723 724 src_addr6= 725 src_port= 726 dst_port= 727 } 728 elif command -v iperf >/dev/null; then 729 flood_tcp6() { 730 [ -n "${dst_port}" ] && dst_port="-p ${dst_port}" 731 if [ -n "${src_addr6}" ]; then 732 B ip addr add "${src_addr6}" dev veth_b nodad 733 src_addr6="-B ${src_addr6}" 734 else 735 src_addr6="-B 2001:db8::2" 736 fi 737 if [ -n "${src_port}" ]; then 738 src_addr6="${src_addr6}:${src_port}" 739 fi 740 B ip route add default dev veth_b 741 ip -6 addr add "${dst_addr6}" dev veth_a nodad \ 742 2>/dev/null 743 744 # shellcheck disable=SC2086 # this needs split options 745 iperf -s -VDB "${dst_addr6}" ${dst_port} >/dev/null 2>&1 746 sleep 2 747 748 # shellcheck disable=SC2086 # this needs split options 749 B iperf -c "${dst_addr6}" -V ${dst_port} \ 750 ${src_addr6} -l1 -t 1000 751 752 src_addr6= 753 src_port= 754 dst_port= 755 } 756 elif command -v netperf >/dev/null; then 757 flood_tcp6() { 758 [ -n "${dst_port}" ] && dst_port="-p ${dst_port}" 759 if [ -n "${src_addr6}" ]; then 760 B ip addr add "${src_addr6}" dev veth_b nodad 761 else 762 src_addr6="2001:db8::2" 763 fi 764 if [ -n "${src_port}" ]; then 765 dst_port="${dst_port},${src_port}" 766 fi 767 B ip route add default dev veth_b 768 ip -6 addr add "${dst_addr6}" dev veth_a nodad \ 769 2>/dev/null 770 771 # shellcheck disable=SC2086 # this needs split options 772 netserver -6 ${dst_port} -L "${dst_addr6}" \ 773 >/dev/null 2>&1 774 sleep 2 775 776 # shellcheck disable=SC2086 # this needs split options 777 B netperf -6 -H "${dst_addr6}" ${dst_port} \ 778 -L "${src_addr6}" -l 1000 -t TCP_STREAM 779 780 src_addr6= 781 src_port= 782 dst_port= 783 } 784 else 785 return 1 786 fi 787} 788 789# Set up function to send UDP traffic on IPv4 790setup_flood_udp() { 791 if command -v iperf3 >/dev/null; then 792 flood_udp() { 793 [ -n "${dst_port}" ] && dst_port="-p ${dst_port}" 794 if [ -n "${src_addr4}" ]; then 795 B ip addr add "${src_addr4}/16" dev veth_b 796 src_addr4="-B ${src_addr4}" 797 else 798 B ip addr add dev veth_b 10.0.0.2 2>/dev/null 799 src_addr4="-B 10.0.0.2" 800 fi 801 if [ -n "${src_port}" ]; then 802 src_port="--cport ${src_port}" 803 fi 804 B ip route add default dev veth_b 805 ip addr add "${dst_addr4}" dev veth_a 2>/dev/null 806 807 # shellcheck disable=SC2086 # this needs split options 808 iperf3 -s -DB "${dst_addr4}" ${dst_port} 809 sleep 2 810 811 # shellcheck disable=SC2086 # this needs split options 812 B iperf3 -u -c "${dst_addr4}" -Z -b 100M -l16 -t1000 \ 813 ${dst_port} ${src_port} ${src_addr4} 814 815 src_addr4= 816 src_port= 817 dst_port= 818 } 819 elif command -v iperf >/dev/null; then 820 flood_udp() { 821 [ -n "${dst_port}" ] && dst_port="-p ${dst_port}" 822 if [ -n "${src_addr4}" ]; then 823 B ip addr add "${src_addr4}/16" dev veth_b 824 src_addr4="-B ${src_addr4}" 825 else 826 B ip addr add dev veth_b 10.0.0.2 827 src_addr4="-B 10.0.0.2" 828 fi 829 if [ -n "${src_port}" ]; then 830 src_addr4="${src_addr4}:${src_port}" 831 fi 832 B ip route add default dev veth_b 833 ip addr add "${dst_addr4}" dev veth_a 2>/dev/null 834 835 # shellcheck disable=SC2086 # this needs split options 836 iperf -u -sDB "${dst_addr4}" ${dst_port} >/dev/null 2>&1 837 sleep 2 838 839 # shellcheck disable=SC2086 # this needs split options 840 B iperf -u -c "${dst_addr4}" -b 100M -l1 -t1000 \ 841 ${dst_port} ${src_addr4} 842 843 src_addr4= 844 src_port= 845 dst_port= 846 } 847 elif command -v netperf >/dev/null; then 848 flood_udp() { 849 [ -n "${dst_port}" ] && dst_port="-p ${dst_port}" 850 if [ -n "${src_addr4}" ]; then 851 B ip addr add "${src_addr4}/16" dev veth_b 852 else 853 B ip addr add dev veth_b 10.0.0.2 854 src_addr4="10.0.0.2" 855 fi 856 if [ -n "${src_port}" ]; then 857 dst_port="${dst_port},${src_port}" 858 fi 859 B ip route add default dev veth_b 860 ip addr add "${dst_addr4}" dev veth_a 2>/dev/null 861 862 # shellcheck disable=SC2086 # this needs split options 863 netserver -4 ${dst_port} -L "${dst_addr4}" \ 864 >/dev/null 2>&1 865 sleep 2 866 867 # shellcheck disable=SC2086 # this needs split options 868 B netperf -4 -H "${dst_addr4}" ${dst_port} \ 869 -L "${src_addr4}" -l 1000 -t UDP_STREAM 870 871 src_addr4= 872 src_port= 873 dst_port= 874 } 875 else 876 return 1 877 fi 878} 879 880# Find pktgen script and set up function to start pktgen injection 881setup_perf() { 882 for pktgen_script_path in ${PKTGEN_SCRIPT_PATHS} __notfound; do 883 command -v "${pktgen_script_path}" >/dev/null && break 884 done 885 [ "${pktgen_script_path}" = "__notfound" ] && return 1 886 887 perf_ipv4() { 888 ${pktgen_script_path} -s80 \ 889 -i veth_a -d "${dst_addr4}" -p "${dst_port}" \ 890 -m "${dst_mac}" \ 891 -t $(($(nproc) / 5 + 1)) -b10000 -n0 2>/dev/null & 892 perf_pid=$! 893 } 894 perf_ipv6() { 895 IP6=6 ${pktgen_script_path} -s100 \ 896 -i veth_a -d "${dst_addr6}" -p "${dst_port}" \ 897 -m "${dst_mac}" \ 898 -t $(($(nproc) / 5 + 1)) -b10000 -n0 2>/dev/null & 899 perf_pid=$! 900 } 901} 902 903# Clean up before each test 904cleanup() { 905 nft reset counter inet filter test >/dev/null 2>&1 906 nft flush ruleset >/dev/null 2>&1 907 ip link del dummy0 2>/dev/null 908 ip route del default 2>/dev/null 909 ip -6 route del default 2>/dev/null 910 ip netns del B 2>/dev/null 911 ip link del veth_a 2>/dev/null 912 timeout= 913 killall iperf3 2>/dev/null 914 killall iperf 2>/dev/null 915 killall netperf 2>/dev/null 916 killall netserver 2>/dev/null 917 rm -f ${tmp} 918 sleep 2 919} 920 921# Entry point for setup functions 922setup() { 923 if [ "$(id -u)" -ne 0 ]; then 924 echo " need to run as root" 925 exit ${KSELFTEST_SKIP} 926 fi 927 928 cleanup 929 check_tools || return 1 930 for arg do 931 if ! eval setup_"${arg}"; then 932 err " ${arg} not supported" 933 return 1 934 fi 935 done 936} 937 938# Format integer into IPv4 address, summing 10.0.0.5 (arbitrary) to it 939format_addr4() { 940 a=$((${1} + 16777216 * 10 + 5)) 941 printf "%i.%i.%i.%i" \ 942 "$((a / 16777216))" "$((a % 16777216 / 65536))" \ 943 "$((a % 65536 / 256))" "$((a % 256))" 944} 945 946# Format integer into IPv6 address, summing 2001:db8:: to it 947format_addr6() { 948 printf "2001:db8::%04x:%04x" "$((${1} / 65536))" "$((${1} % 65536))" 949} 950 951# Format integer into EUI-48 address, summing 00:01:00:00:00:00 to it 952format_mac() { 953 printf "00:01:%02x:%02x:%02x:%02x" \ 954 "$((${1} / 16777216))" "$((${1} % 16777216 / 65536))" \ 955 "$((${1} % 65536 / 256))" "$((${1} % 256))" 956} 957 958# Format integer into port, avoid 0 port 959format_port() { 960 printf "%i" "$((${1} % 65534 + 1))" 961} 962 963# Drop suffixed '6' from L4 protocol, if any 964format_proto() { 965 printf "%s" "${proto}" | tr -d 6 966} 967 968# Format destination and source fields into nft concatenated type 969format() { 970 __start= 971 __end= 972 __expr="{ " 973 974 for f in ${dst}; do 975 [ "${__expr}" != "{ " ] && __expr="${__expr} . " 976 977 __start="$(eval format_"${f}" "${start}")" 978 __end="$(eval format_"${f}" "${end}")" 979 980 if [ "${f}" = "proto" ]; then 981 __expr="${__expr}${__start}" 982 else 983 __expr="${__expr}${__start}-${__end}" 984 fi 985 done 986 for f in ${src}; do 987 __expr="${__expr} . " 988 __start="$(eval format_"${f}" "${srcstart}")" 989 __end="$(eval format_"${f}" "${srcend}")" 990 991 if [ "${f}" = "proto" ]; then 992 __expr="${__expr}${__start}" 993 else 994 __expr="${__expr}${__start}-${__end}" 995 fi 996 done 997 998 if [ -n "${timeout}" ]; then 999 echo "${__expr} timeout ${timeout}s }" 1000 else 1001 echo "${__expr} }" 1002 fi 1003} 1004 1005# Format destination and source fields into nft type, start element only 1006format_norange() { 1007 __expr="{ " 1008 1009 for f in ${dst}; do 1010 [ "${__expr}" != "{ " ] && __expr="${__expr} . " 1011 1012 __expr="${__expr}$(eval format_"${f}" "${start}")" 1013 done 1014 for f in ${src}; do 1015 __expr="${__expr} . $(eval format_"${f}" "${start}")" 1016 done 1017 1018 echo "${__expr} }" 1019} 1020 1021# Format first destination field into nft type 1022format_noconcat() { 1023 for f in ${dst}; do 1024 __start="$(eval format_"${f}" "${start}")" 1025 __end="$(eval format_"${f}" "${end}")" 1026 1027 if [ "${f}" = "proto" ]; then 1028 echo "{ ${__start} }" 1029 else 1030 echo "{ ${__start}-${__end} }" 1031 fi 1032 return 1033 done 1034} 1035 1036# Add single entry to 'test' set in 'inet filter' table 1037add() { 1038 if ! nft add element inet filter test "${1}"; then 1039 err "Failed to add ${1} given ruleset:" 1040 err "$(nft -a list ruleset)" 1041 return 1 1042 fi 1043} 1044 1045# Format and output entries for sets in 'netdev perf' table 1046add_perf() { 1047 if [ "${1}" = "test" ]; then 1048 echo "add element netdev perf test $(format)" 1049 elif [ "${1}" = "norange" ]; then 1050 echo "add element netdev perf norange $(format_norange)" 1051 elif [ "${1}" = "noconcat" ]; then 1052 echo "add element netdev perf noconcat $(format_noconcat)" 1053 fi 1054} 1055 1056# Add single entry to 'norange' set in 'netdev perf' table 1057add_perf_norange() { 1058 if ! nft add element netdev perf norange "${1}"; then 1059 err "Failed to add ${1} given ruleset:" 1060 err "$(nft -a list ruleset)" 1061 return 1 1062 fi 1063} 1064 1065# Add single entry to 'noconcat' set in 'netdev perf' table 1066add_perf_noconcat() { 1067 if ! nft add element netdev perf noconcat "${1}"; then 1068 err "Failed to add ${1} given ruleset:" 1069 err "$(nft -a list ruleset)" 1070 return 1 1071 fi 1072} 1073 1074# Delete single entry from set 1075del() { 1076 if ! nft delete element inet filter test "${1}"; then 1077 err "Failed to delete ${1} given ruleset:" 1078 err "$(nft -a list ruleset)" 1079 return 1 1080 fi 1081} 1082 1083# Return packet count from 'test' counter in 'inet filter' table 1084count_packets() { 1085 found=0 1086 for token in $(nft list counter inet filter test); do 1087 [ ${found} -eq 1 ] && echo "${token}" && return 1088 [ "${token}" = "packets" ] && found=1 1089 done 1090} 1091 1092# Return packet count from 'test' counter in 'netdev perf' table 1093count_perf_packets() { 1094 found=0 1095 for token in $(nft list counter netdev perf test); do 1096 [ ${found} -eq 1 ] && echo "${token}" && return 1097 [ "${token}" = "packets" ] && found=1 1098 done 1099} 1100 1101# Set MAC addresses, send traffic according to specifier 1102flood() { 1103 ip link set veth_a address "$(format_mac "${1}")" 1104 ip -n B link set veth_b address "$(format_mac "${2}")" 1105 1106 for f in ${dst}; do 1107 eval dst_"$f"=\$\(format_\$f "${1}"\) 1108 done 1109 for f in ${src}; do 1110 eval src_"$f"=\$\(format_\$f "${2}"\) 1111 done 1112 eval flood_\$proto 1113} 1114 1115# Set MAC addresses, start pktgen injection 1116perf() { 1117 dst_mac="$(format_mac "${1}")" 1118 ip link set veth_a address "${dst_mac}" 1119 1120 for f in ${dst}; do 1121 eval dst_"$f"=\$\(format_\$f "${1}"\) 1122 done 1123 for f in ${src}; do 1124 eval src_"$f"=\$\(format_\$f "${2}"\) 1125 done 1126 eval perf_\$perf_proto 1127} 1128 1129# Set MAC addresses, send single packet, check that it matches, reset counter 1130send_match() { 1131 ip link set veth_a address "$(format_mac "${1}")" 1132 ip -n B link set veth_b address "$(format_mac "${2}")" 1133 1134 for f in ${dst}; do 1135 eval dst_"$f"=\$\(format_\$f "${1}"\) 1136 done 1137 for f in ${src}; do 1138 eval src_"$f"=\$\(format_\$f "${2}"\) 1139 done 1140 eval send_\$proto 1141 if [ "$(count_packets)" != "1" ]; then 1142 err "${proto} packet to:" 1143 err " $(for f in ${dst}; do 1144 eval format_\$f "${1}"; printf ' '; done)" 1145 err "from:" 1146 err " $(for f in ${src}; do 1147 eval format_\$f "${2}"; printf ' '; done)" 1148 err "should have matched ruleset:" 1149 err "$(nft -a list ruleset)" 1150 return 1 1151 fi 1152 nft reset counter inet filter test >/dev/null 1153} 1154 1155# Set MAC addresses, send single packet, check that it doesn't match 1156send_nomatch() { 1157 ip link set veth_a address "$(format_mac "${1}")" 1158 ip -n B link set veth_b address "$(format_mac "${2}")" 1159 1160 for f in ${dst}; do 1161 eval dst_"$f"=\$\(format_\$f "${1}"\) 1162 done 1163 for f in ${src}; do 1164 eval src_"$f"=\$\(format_\$f "${2}"\) 1165 done 1166 eval send_\$proto 1167 if [ "$(count_packets)" != "0" ]; then 1168 err "${proto} packet to:" 1169 err " $(for f in ${dst}; do 1170 eval format_\$f "${1}"; printf ' '; done)" 1171 err "from:" 1172 err " $(for f in ${src}; do 1173 eval format_\$f "${2}"; printf ' '; done)" 1174 err "should not have matched ruleset:" 1175 err "$(nft -a list ruleset)" 1176 return 1 1177 fi 1178} 1179 1180# Correctness test template: 1181# - add ranged element, check that packets match it 1182# - check that packets outside range don't match it 1183# - remove some elements, check that packets don't match anymore 1184test_correctness() { 1185 setup veth send_"${proto}" set || return ${KSELFTEST_SKIP} 1186 1187 range_size=1 1188 for i in $(seq "${start}" $((start + count))); do 1189 end=$((start + range_size)) 1190 1191 # Avoid negative or zero-sized port ranges 1192 if [ $((end / 65534)) -gt $((start / 65534)) ]; then 1193 start=${end} 1194 end=$((end + 1)) 1195 fi 1196 srcstart=$((start + src_delta)) 1197 srcend=$((end + src_delta)) 1198 1199 add "$(format)" || return 1 1200 for j in $(seq ${start} $((range_size / 2 + 1)) ${end}); do 1201 send_match "${j}" $((j + src_delta)) || return 1 1202 done 1203 send_nomatch $((end + 1)) $((end + 1 + src_delta)) || return 1 1204 1205 # Delete elements now and then 1206 if [ $((i % 3)) -eq 0 ]; then 1207 del "$(format)" || return 1 1208 for j in $(seq ${start} \ 1209 $((range_size / 2 + 1)) ${end}); do 1210 send_nomatch "${j}" $((j + src_delta)) \ 1211 || return 1 1212 done 1213 fi 1214 1215 range_size=$((range_size + 1)) 1216 start=$((end + range_size)) 1217 done 1218} 1219 1220# Concurrency test template: 1221# - add all the elements 1222# - start a thread for each physical thread that: 1223# - adds all the elements 1224# - flushes the set 1225# - adds all the elements 1226# - flushes the entire ruleset 1227# - adds the set back 1228# - adds all the elements 1229# - delete all the elements 1230test_concurrency() { 1231 proto=${flood_proto} 1232 tools=${flood_tools} 1233 chain_spec=${flood_spec} 1234 setup veth flood_"${proto}" set || return ${KSELFTEST_SKIP} 1235 1236 range_size=1 1237 cstart=${start} 1238 flood_pids= 1239 for i in $(seq ${start} $((start + count))); do 1240 end=$((start + range_size)) 1241 srcstart=$((start + src_delta)) 1242 srcend=$((end + src_delta)) 1243 1244 add "$(format)" || return 1 1245 1246 flood "${i}" $((i + src_delta)) & flood_pids="${flood_pids} $!" 1247 1248 range_size=$((range_size + 1)) 1249 start=$((end + range_size)) 1250 done 1251 1252 sleep 10 1253 1254 pids= 1255 for c in $(seq 1 "$(nproc)"); do ( 1256 for r in $(seq 1 "${race_repeat}"); do 1257 range_size=1 1258 1259 # $start needs to be local to this subshell 1260 # shellcheck disable=SC2030 1261 start=${cstart} 1262 for i in $(seq ${start} $((start + count))); do 1263 end=$((start + range_size)) 1264 srcstart=$((start + src_delta)) 1265 srcend=$((end + src_delta)) 1266 1267 add "$(format)" 2>/dev/null 1268 1269 range_size=$((range_size + 1)) 1270 start=$((end + range_size)) 1271 done 1272 1273 nft flush inet filter test 2>/dev/null 1274 1275 range_size=1 1276 start=${cstart} 1277 for i in $(seq ${start} $((start + count))); do 1278 end=$((start + range_size)) 1279 srcstart=$((start + src_delta)) 1280 srcend=$((end + src_delta)) 1281 1282 add "$(format)" 2>/dev/null 1283 1284 range_size=$((range_size + 1)) 1285 start=$((end + range_size)) 1286 done 1287 1288 nft flush ruleset 1289 setup set 2>/dev/null 1290 1291 range_size=1 1292 start=${cstart} 1293 for i in $(seq ${start} $((start + count))); do 1294 end=$((start + range_size)) 1295 srcstart=$((start + src_delta)) 1296 srcend=$((end + src_delta)) 1297 1298 add "$(format)" 2>/dev/null 1299 1300 range_size=$((range_size + 1)) 1301 start=$((end + range_size)) 1302 done 1303 1304 range_size=1 1305 start=${cstart} 1306 for i in $(seq ${start} $((start + count))); do 1307 end=$((start + range_size)) 1308 srcstart=$((start + src_delta)) 1309 srcend=$((end + src_delta)) 1310 1311 del "$(format)" 2>/dev/null 1312 1313 range_size=$((range_size + 1)) 1314 start=$((end + range_size)) 1315 done 1316 done 1317 ) & pids="${pids} $!" 1318 done 1319 1320 # shellcheck disable=SC2046,SC2086 # word splitting wanted here 1321 wait $(for pid in ${pids}; do echo ${pid}; done) 1322 # shellcheck disable=SC2046,SC2086 1323 kill $(for pid in ${flood_pids}; do echo ${pid}; done) 2>/dev/null 1324 # shellcheck disable=SC2046,SC2086 1325 wait $(for pid in ${flood_pids}; do echo ${pid}; done) 2>/dev/null 1326 1327 return 0 1328} 1329 1330# Timeout test template: 1331# - add all the elements with 3s timeout while checking that packets match 1332# - wait 3s after the last insertion, check that packets don't match any entry 1333test_timeout() { 1334 setup veth send_"${proto}" set || return ${KSELFTEST_SKIP} 1335 1336 timeout=3 1337 range_size=1 1338 for i in $(seq "${start}" $((start + count))); do 1339 end=$((start + range_size)) 1340 srcstart=$((start + src_delta)) 1341 srcend=$((end + src_delta)) 1342 1343 add "$(format)" || return 1 1344 1345 for j in $(seq ${start} $((range_size / 2 + 1)) ${end}); do 1346 send_match "${j}" $((j + src_delta)) || return 1 1347 done 1348 1349 range_size=$((range_size + 1)) 1350 start=$((end + range_size)) 1351 done 1352 sleep 3 1353 for i in $(seq ${start} $((start + count))); do 1354 end=$((start + range_size)) 1355 srcstart=$((start + src_delta)) 1356 srcend=$((end + src_delta)) 1357 1358 for j in $(seq ${start} $((range_size / 2 + 1)) ${end}); do 1359 send_nomatch "${j}" $((j + src_delta)) || return 1 1360 done 1361 1362 range_size=$((range_size + 1)) 1363 start=$((end + range_size)) 1364 done 1365} 1366 1367# Performance test template: 1368# - add concatenated ranged entries 1369# - add non-ranged concatenated entries (for hash set matching rate baseline) 1370# - add ranged entries with first field only (for rbhash baseline) 1371# - start pktgen injection directly on device rx path of this namespace 1372# - measure drop only rate, hash and rbtree baselines, then matching rate 1373test_performance() { 1374 chain_spec=${perf_spec} 1375 dst="${perf_dst}" 1376 src="${perf_src}" 1377 setup veth perf set || return ${KSELFTEST_SKIP} 1378 1379 first=${start} 1380 range_size=1 1381 for set in test norange noconcat; do 1382 start=${first} 1383 for i in $(seq ${start} $((start + perf_entries))); do 1384 end=$((start + range_size)) 1385 srcstart=$((start + src_delta)) 1386 srcend=$((end + src_delta)) 1387 1388 if [ $((end / 65534)) -gt $((start / 65534)) ]; then 1389 start=${end} 1390 end=$((end + 1)) 1391 elif [ ${start} -eq ${end} ]; then 1392 end=$((start + 1)) 1393 fi 1394 1395 add_perf ${set} 1396 1397 start=$((end + range_size)) 1398 done > "${tmp}" 1399 nft -f "${tmp}" 1400 done 1401 1402 perf $((end - 1)) ${srcstart} 1403 1404 sleep 2 1405 1406 nft add rule netdev perf test counter name \"test\" drop 1407 nft reset counter netdev perf test >/dev/null 2>&1 1408 sleep "${perf_duration}" 1409 pps="$(printf %10s $(($(count_perf_packets) / perf_duration)))" 1410 info " baseline (drop from netdev hook): ${pps}pps" 1411 handle="$(nft -a list chain netdev perf test | grep counter)" 1412 handle="${handle##* }" 1413 nft delete rule netdev perf test handle "${handle}" 1414 1415 nft add rule "netdev perf test ${chain_spec} @norange \ 1416 counter name \"test\" drop" 1417 nft reset counter netdev perf test >/dev/null 2>&1 1418 sleep "${perf_duration}" 1419 pps="$(printf %10s $(($(count_perf_packets) / perf_duration)))" 1420 info " baseline hash (non-ranged entries): ${pps}pps" 1421 handle="$(nft -a list chain netdev perf test | grep counter)" 1422 handle="${handle##* }" 1423 nft delete rule netdev perf test handle "${handle}" 1424 1425 nft add rule "netdev perf test ${chain_spec%%. *} @noconcat \ 1426 counter name \"test\" drop" 1427 nft reset counter netdev perf test >/dev/null 2>&1 1428 sleep "${perf_duration}" 1429 pps="$(printf %10s $(($(count_perf_packets) / perf_duration)))" 1430 info " baseline rbtree (match on first field only): ${pps}pps" 1431 handle="$(nft -a list chain netdev perf test | grep counter)" 1432 handle="${handle##* }" 1433 nft delete rule netdev perf test handle "${handle}" 1434 1435 nft add rule "netdev perf test ${chain_spec} @test \ 1436 counter name \"test\" drop" 1437 nft reset counter netdev perf test >/dev/null 2>&1 1438 sleep "${perf_duration}" 1439 pps="$(printf %10s $(($(count_perf_packets) / perf_duration)))" 1440 p5="$(printf %5s "${perf_entries}")" 1441 info " set with ${p5} full, ranged entries: ${pps}pps" 1442 kill "${perf_pid}" 1443} 1444 1445test_bug_flush_remove_add() { 1446 set_cmd='{ set s { type ipv4_addr . inet_service; flags interval; }; }' 1447 elem1='{ 10.0.0.1 . 22-25, 10.0.0.1 . 10-20 }' 1448 elem2='{ 10.0.0.1 . 10-20, 10.0.0.1 . 22-25 }' 1449 for i in `seq 1 100`; do 1450 nft add table t ${set_cmd} || return ${KSELFTEST_SKIP} 1451 nft add element t s ${elem1} 2>/dev/null || return 1 1452 nft flush set t s 2>/dev/null || return 1 1453 nft add element t s ${elem2} 2>/dev/null || return 1 1454 done 1455 nft flush ruleset 1456} 1457 1458test_reported_issues() { 1459 eval test_bug_"${subtest}" 1460} 1461 1462# Run everything in a separate network namespace 1463[ "${1}" != "run" ] && { unshare -n "${0}" run; exit $?; } 1464tmp="$(mktemp)" 1465trap cleanup EXIT 1466 1467# Entry point for test runs 1468passed=0 1469for name in ${TESTS}; do 1470 printf "TEST: %s\n" "$(echo ${name} | tr '_' ' ')" 1471 if [ "${name}" = "reported_issues" ]; then 1472 SUBTESTS="${BUGS}" 1473 else 1474 SUBTESTS="${TYPES}" 1475 fi 1476 1477 for subtest in ${SUBTESTS}; do 1478 eval desc=\$TYPE_"${subtest}" 1479 IFS=' 1480' 1481 for __line in ${desc}; do 1482 # shellcheck disable=SC2086 1483 eval ${__line%% *}=\"${__line##* }\"; 1484 done 1485 IFS=' 1486' 1487 1488 if [ "${name}" = "concurrency" ] && \ 1489 [ "${race_repeat}" = "0" ]; then 1490 continue 1491 fi 1492 if [ "${name}" = "performance" ] && \ 1493 [ "${perf_duration}" = "0" ]; then 1494 continue 1495 fi 1496 1497 printf " %-60s " "${display}" 1498 eval test_"${name}" 1499 ret=$? 1500 1501 if [ $ret -eq 0 ]; then 1502 printf "[ OK ]\n" 1503 info_flush 1504 passed=$((passed + 1)) 1505 elif [ $ret -eq 1 ]; then 1506 printf "[FAIL]\n" 1507 err_flush 1508 exit 1 1509 elif [ $ret -eq ${KSELFTEST_SKIP} ]; then 1510 printf "[SKIP]\n" 1511 err_flush 1512 fi 1513 done 1514done 1515 1516[ ${passed} -eq 0 ] && exit ${KSELFTEST_SKIP} 1517