1 // SPDX-License-Identifier: GPL-2.0 2 3 /** 4 * Test XDP bonding support 5 * 6 * Sets up two bonded veth pairs between two fresh namespaces 7 * and verifies that XDP_TX program loaded on a bond device 8 * are correctly loaded onto the slave devices and XDP_TX'd 9 * packets are balanced using bonding. 10 */ 11 12 #define _GNU_SOURCE 13 #include <sched.h> 14 #include <net/if.h> 15 #include <linux/if_link.h> 16 #include "test_progs.h" 17 #include "network_helpers.h" 18 #include <linux/if_bonding.h> 19 #include <linux/limits.h> 20 #include <linux/udp.h> 21 22 #include "xdp_dummy.skel.h" 23 #include "xdp_redirect_multi_kern.skel.h" 24 #include "xdp_tx.skel.h" 25 26 #define BOND1_MAC {0x00, 0x11, 0x22, 0x33, 0x44, 0x55} 27 #define BOND1_MAC_STR "00:11:22:33:44:55" 28 #define BOND2_MAC {0x00, 0x22, 0x33, 0x44, 0x55, 0x66} 29 #define BOND2_MAC_STR "00:22:33:44:55:66" 30 #define NPACKETS 100 31 32 static int root_netns_fd = -1; 33 34 static void restore_root_netns(void) 35 { 36 ASSERT_OK(setns(root_netns_fd, CLONE_NEWNET), "restore_root_netns"); 37 } 38 39 static int setns_by_name(char *name) 40 { 41 int nsfd, err; 42 char nspath[PATH_MAX]; 43 44 snprintf(nspath, sizeof(nspath), "%s/%s", "/var/run/netns", name); 45 nsfd = open(nspath, O_RDONLY | O_CLOEXEC); 46 if (nsfd < 0) 47 return -1; 48 49 err = setns(nsfd, CLONE_NEWNET); 50 close(nsfd); 51 return err; 52 } 53 54 static int get_rx_packets(const char *iface) 55 { 56 FILE *f; 57 char line[512]; 58 int iface_len = strlen(iface); 59 60 f = fopen("/proc/net/dev", "r"); 61 if (!f) 62 return -1; 63 64 while (fgets(line, sizeof(line), f)) { 65 char *p = line; 66 67 while (*p == ' ') 68 p++; /* skip whitespace */ 69 if (!strncmp(p, iface, iface_len)) { 70 p += iface_len; 71 if (*p++ != ':') 72 continue; 73 while (*p == ' ') 74 p++; /* skip whitespace */ 75 while (*p && *p != ' ') 76 p++; /* skip rx bytes */ 77 while (*p == ' ') 78 p++; /* skip whitespace */ 79 fclose(f); 80 return atoi(p); 81 } 82 } 83 fclose(f); 84 return -1; 85 } 86 87 #define MAX_BPF_LINKS 8 88 89 struct skeletons { 90 struct xdp_dummy *xdp_dummy; 91 struct xdp_tx *xdp_tx; 92 struct xdp_redirect_multi_kern *xdp_redirect_multi_kern; 93 94 int nlinks; 95 struct bpf_link *links[MAX_BPF_LINKS]; 96 }; 97 98 static int xdp_attach(struct skeletons *skeletons, struct bpf_program *prog, char *iface) 99 { 100 struct bpf_link *link; 101 int ifindex; 102 103 ifindex = if_nametoindex(iface); 104 if (!ASSERT_GT(ifindex, 0, "get ifindex")) 105 return -1; 106 107 if (!ASSERT_LE(skeletons->nlinks+1, MAX_BPF_LINKS, "too many XDP programs attached")) 108 return -1; 109 110 link = bpf_program__attach_xdp(prog, ifindex); 111 if (!ASSERT_OK_PTR(link, "attach xdp program")) 112 return -1; 113 114 skeletons->links[skeletons->nlinks++] = link; 115 return 0; 116 } 117 118 enum { 119 BOND_ONE_NO_ATTACH = 0, 120 BOND_BOTH_AND_ATTACH, 121 }; 122 123 static const char * const mode_names[] = { 124 [BOND_MODE_ROUNDROBIN] = "balance-rr", 125 [BOND_MODE_ACTIVEBACKUP] = "active-backup", 126 [BOND_MODE_XOR] = "balance-xor", 127 [BOND_MODE_BROADCAST] = "broadcast", 128 [BOND_MODE_8023AD] = "802.3ad", 129 [BOND_MODE_TLB] = "balance-tlb", 130 [BOND_MODE_ALB] = "balance-alb", 131 }; 132 133 static const char * const xmit_policy_names[] = { 134 [BOND_XMIT_POLICY_LAYER2] = "layer2", 135 [BOND_XMIT_POLICY_LAYER34] = "layer3+4", 136 [BOND_XMIT_POLICY_LAYER23] = "layer2+3", 137 [BOND_XMIT_POLICY_ENCAP23] = "encap2+3", 138 [BOND_XMIT_POLICY_ENCAP34] = "encap3+4", 139 }; 140 141 static int bonding_setup(struct skeletons *skeletons, int mode, int xmit_policy, 142 int bond_both_attach) 143 { 144 SYS(fail, "ip netns add ns_dst"); 145 SYS(fail, "ip link add veth1_1 type veth peer name veth2_1 netns ns_dst"); 146 SYS(fail, "ip link add veth1_2 type veth peer name veth2_2 netns ns_dst"); 147 148 SYS(fail, "ip link add bond1 type bond mode %s xmit_hash_policy %s", 149 mode_names[mode], xmit_policy_names[xmit_policy]); 150 SYS(fail, "ip link set bond1 up address " BOND1_MAC_STR " addrgenmode none"); 151 SYS(fail, "ip -netns ns_dst link add bond2 type bond mode %s xmit_hash_policy %s", 152 mode_names[mode], xmit_policy_names[xmit_policy]); 153 SYS(fail, "ip -netns ns_dst link set bond2 up address " BOND2_MAC_STR " addrgenmode none"); 154 155 SYS(fail, "ip link set veth1_1 master bond1"); 156 if (bond_both_attach == BOND_BOTH_AND_ATTACH) { 157 SYS(fail, "ip link set veth1_2 master bond1"); 158 } else { 159 SYS(fail, "ip link set veth1_2 up addrgenmode none"); 160 161 if (xdp_attach(skeletons, skeletons->xdp_dummy->progs.xdp_dummy_prog, "veth1_2")) 162 return -1; 163 } 164 165 SYS(fail, "ip -netns ns_dst link set veth2_1 master bond2"); 166 167 if (bond_both_attach == BOND_BOTH_AND_ATTACH) 168 SYS(fail, "ip -netns ns_dst link set veth2_2 master bond2"); 169 else 170 SYS(fail, "ip -netns ns_dst link set veth2_2 up addrgenmode none"); 171 172 /* Load a dummy program on sending side as with veth peer needs to have a 173 * XDP program loaded as well. 174 */ 175 if (xdp_attach(skeletons, skeletons->xdp_dummy->progs.xdp_dummy_prog, "bond1")) 176 return -1; 177 178 if (bond_both_attach == BOND_BOTH_AND_ATTACH) { 179 if (!ASSERT_OK(setns_by_name("ns_dst"), "set netns to ns_dst")) 180 return -1; 181 182 if (xdp_attach(skeletons, skeletons->xdp_tx->progs.xdp_tx, "bond2")) 183 return -1; 184 185 restore_root_netns(); 186 } 187 188 return 0; 189 fail: 190 return -1; 191 } 192 193 static void bonding_cleanup(struct skeletons *skeletons) 194 { 195 restore_root_netns(); 196 while (skeletons->nlinks) { 197 skeletons->nlinks--; 198 bpf_link__destroy(skeletons->links[skeletons->nlinks]); 199 } 200 ASSERT_OK(system("ip link delete bond1"), "delete bond1"); 201 ASSERT_OK(system("ip link delete veth1_1"), "delete veth1_1"); 202 ASSERT_OK(system("ip link delete veth1_2"), "delete veth1_2"); 203 ASSERT_OK(system("ip netns delete ns_dst"), "delete ns_dst"); 204 } 205 206 static int send_udp_packets(int vary_dst_ip) 207 { 208 struct ethhdr eh = { 209 .h_source = BOND1_MAC, 210 .h_dest = BOND2_MAC, 211 .h_proto = htons(ETH_P_IP), 212 }; 213 struct iphdr iph = {}; 214 struct udphdr uh = {}; 215 uint8_t buf[128]; 216 int i, s = -1; 217 int ifindex; 218 219 s = socket(AF_PACKET, SOCK_RAW, IPPROTO_RAW); 220 if (!ASSERT_GE(s, 0, "socket")) 221 goto err; 222 223 ifindex = if_nametoindex("bond1"); 224 if (!ASSERT_GT(ifindex, 0, "get bond1 ifindex")) 225 goto err; 226 227 iph.ihl = 5; 228 iph.version = 4; 229 iph.tos = 16; 230 iph.id = 1; 231 iph.ttl = 64; 232 iph.protocol = IPPROTO_UDP; 233 iph.saddr = 1; 234 iph.daddr = 2; 235 iph.tot_len = htons(sizeof(buf) - ETH_HLEN); 236 iph.check = 0; 237 238 for (i = 1; i <= NPACKETS; i++) { 239 int n; 240 struct sockaddr_ll saddr_ll = { 241 .sll_ifindex = ifindex, 242 .sll_halen = ETH_ALEN, 243 .sll_addr = BOND2_MAC, 244 }; 245 246 /* vary the UDP destination port for even distribution with roundrobin/xor modes */ 247 uh.dest++; 248 249 if (vary_dst_ip) 250 iph.daddr++; 251 252 /* construct a packet */ 253 memcpy(buf, &eh, sizeof(eh)); 254 memcpy(buf + sizeof(eh), &iph, sizeof(iph)); 255 memcpy(buf + sizeof(eh) + sizeof(iph), &uh, sizeof(uh)); 256 257 n = sendto(s, buf, sizeof(buf), 0, (struct sockaddr *)&saddr_ll, sizeof(saddr_ll)); 258 if (!ASSERT_EQ(n, sizeof(buf), "sendto")) 259 goto err; 260 } 261 262 return 0; 263 264 err: 265 if (s >= 0) 266 close(s); 267 return -1; 268 } 269 270 static void test_xdp_bonding_with_mode(struct skeletons *skeletons, int mode, int xmit_policy) 271 { 272 int bond1_rx; 273 274 if (bonding_setup(skeletons, mode, xmit_policy, BOND_BOTH_AND_ATTACH)) 275 goto out; 276 277 if (send_udp_packets(xmit_policy != BOND_XMIT_POLICY_LAYER34)) 278 goto out; 279 280 bond1_rx = get_rx_packets("bond1"); 281 ASSERT_EQ(bond1_rx, NPACKETS, "expected more received packets"); 282 283 switch (mode) { 284 case BOND_MODE_ROUNDROBIN: 285 case BOND_MODE_XOR: { 286 int veth1_rx = get_rx_packets("veth1_1"); 287 int veth2_rx = get_rx_packets("veth1_2"); 288 int diff = abs(veth1_rx - veth2_rx); 289 290 ASSERT_GE(veth1_rx + veth2_rx, NPACKETS, "expected more packets"); 291 292 switch (xmit_policy) { 293 case BOND_XMIT_POLICY_LAYER2: 294 ASSERT_GE(diff, NPACKETS, 295 "expected packets on only one of the interfaces"); 296 break; 297 case BOND_XMIT_POLICY_LAYER23: 298 case BOND_XMIT_POLICY_LAYER34: 299 ASSERT_LT(diff, NPACKETS/2, 300 "expected even distribution of packets"); 301 break; 302 default: 303 PRINT_FAIL("Unimplemented xmit_policy=%d\n", xmit_policy); 304 break; 305 } 306 break; 307 } 308 case BOND_MODE_ACTIVEBACKUP: { 309 int veth1_rx = get_rx_packets("veth1_1"); 310 int veth2_rx = get_rx_packets("veth1_2"); 311 int diff = abs(veth1_rx - veth2_rx); 312 313 ASSERT_GE(diff, NPACKETS, 314 "expected packets on only one of the interfaces"); 315 break; 316 } 317 default: 318 PRINT_FAIL("Unimplemented xmit_policy=%d\n", xmit_policy); 319 break; 320 } 321 322 out: 323 bonding_cleanup(skeletons); 324 } 325 326 /* Test the broadcast redirection using xdp_redirect_map_multi_prog and adding 327 * all the interfaces to it and checking that broadcasting won't send the packet 328 * to neither the ingress bond device (bond2) or its slave (veth2_1). 329 */ 330 static void test_xdp_bonding_redirect_multi(struct skeletons *skeletons) 331 { 332 static const char * const ifaces[] = {"bond2", "veth2_1", "veth2_2"}; 333 int veth1_1_rx, veth1_2_rx; 334 int err; 335 336 if (bonding_setup(skeletons, BOND_MODE_ROUNDROBIN, BOND_XMIT_POLICY_LAYER23, 337 BOND_ONE_NO_ATTACH)) 338 goto out; 339 340 341 if (!ASSERT_OK(setns_by_name("ns_dst"), "could not set netns to ns_dst")) 342 goto out; 343 344 /* populate the devmap with the relevant interfaces */ 345 for (int i = 0; i < ARRAY_SIZE(ifaces); i++) { 346 int ifindex = if_nametoindex(ifaces[i]); 347 int map_fd = bpf_map__fd(skeletons->xdp_redirect_multi_kern->maps.map_all); 348 349 if (!ASSERT_GT(ifindex, 0, "could not get interface index")) 350 goto out; 351 352 err = bpf_map_update_elem(map_fd, &ifindex, &ifindex, 0); 353 if (!ASSERT_OK(err, "add interface to map_all")) 354 goto out; 355 } 356 357 if (xdp_attach(skeletons, 358 skeletons->xdp_redirect_multi_kern->progs.xdp_redirect_map_multi_prog, 359 "bond2")) 360 goto out; 361 362 restore_root_netns(); 363 364 if (send_udp_packets(BOND_MODE_ROUNDROBIN)) 365 goto out; 366 367 veth1_1_rx = get_rx_packets("veth1_1"); 368 veth1_2_rx = get_rx_packets("veth1_2"); 369 370 ASSERT_EQ(veth1_1_rx, 0, "expected no packets on veth1_1"); 371 ASSERT_GE(veth1_2_rx, NPACKETS, "expected packets on veth1_2"); 372 373 out: 374 restore_root_netns(); 375 bonding_cleanup(skeletons); 376 } 377 378 /* Test that XDP programs cannot be attached to both the bond master and slaves simultaneously */ 379 static void test_xdp_bonding_attach(struct skeletons *skeletons) 380 { 381 struct bpf_link *link = NULL; 382 struct bpf_link *link2 = NULL; 383 int veth, bond, err; 384 385 if (!ASSERT_OK(system("ip link add veth type veth"), "add veth")) 386 goto out; 387 if (!ASSERT_OK(system("ip link add bond type bond"), "add bond")) 388 goto out; 389 390 veth = if_nametoindex("veth"); 391 if (!ASSERT_GE(veth, 0, "if_nametoindex veth")) 392 goto out; 393 bond = if_nametoindex("bond"); 394 if (!ASSERT_GE(bond, 0, "if_nametoindex bond")) 395 goto out; 396 397 /* enslaving with a XDP program loaded is allowed */ 398 link = bpf_program__attach_xdp(skeletons->xdp_dummy->progs.xdp_dummy_prog, veth); 399 if (!ASSERT_OK_PTR(link, "attach program to veth")) 400 goto out; 401 402 err = system("ip link set veth master bond"); 403 if (!ASSERT_OK(err, "set veth master")) 404 goto out; 405 406 bpf_link__destroy(link); 407 link = NULL; 408 409 /* attaching to slave when master has no program is allowed */ 410 link = bpf_program__attach_xdp(skeletons->xdp_dummy->progs.xdp_dummy_prog, veth); 411 if (!ASSERT_OK_PTR(link, "attach program to slave when enslaved")) 412 goto out; 413 414 /* attaching to master not allowed when slave has program loaded */ 415 link2 = bpf_program__attach_xdp(skeletons->xdp_dummy->progs.xdp_dummy_prog, bond); 416 if (!ASSERT_ERR_PTR(link2, "attach program to master when slave has program")) 417 goto out; 418 419 bpf_link__destroy(link); 420 link = NULL; 421 422 /* attaching XDP program to master allowed when slave has no program */ 423 link = bpf_program__attach_xdp(skeletons->xdp_dummy->progs.xdp_dummy_prog, bond); 424 if (!ASSERT_OK_PTR(link, "attach program to master")) 425 goto out; 426 427 /* attaching to slave not allowed when master has program loaded */ 428 link2 = bpf_program__attach_xdp(skeletons->xdp_dummy->progs.xdp_dummy_prog, veth); 429 if (!ASSERT_ERR_PTR(link2, "attach program to slave when master has program")) 430 goto out; 431 432 bpf_link__destroy(link); 433 link = NULL; 434 435 /* test program unwinding with a non-XDP slave */ 436 if (!ASSERT_OK(system("ip link add vxlan type vxlan id 1 remote 1.2.3.4 dstport 0 dev lo"), 437 "add vxlan")) 438 goto out; 439 440 err = system("ip link set vxlan master bond"); 441 if (!ASSERT_OK(err, "set vxlan master")) 442 goto out; 443 444 /* attaching not allowed when one slave does not support XDP */ 445 link = bpf_program__attach_xdp(skeletons->xdp_dummy->progs.xdp_dummy_prog, bond); 446 if (!ASSERT_ERR_PTR(link, "attach program to master when slave does not support XDP")) 447 goto out; 448 449 out: 450 bpf_link__destroy(link); 451 bpf_link__destroy(link2); 452 453 system("ip link del veth"); 454 system("ip link del bond"); 455 system("ip link del vxlan"); 456 } 457 458 /* Test with nested bonding devices to catch issue with negative jump label count */ 459 static void test_xdp_bonding_nested(struct skeletons *skeletons) 460 { 461 struct bpf_link *link = NULL; 462 int bond, err; 463 464 if (!ASSERT_OK(system("ip link add bond type bond"), "add bond")) 465 goto out; 466 467 bond = if_nametoindex("bond"); 468 if (!ASSERT_GE(bond, 0, "if_nametoindex bond")) 469 goto out; 470 471 if (!ASSERT_OK(system("ip link add bond_nest1 type bond"), "add bond_nest1")) 472 goto out; 473 474 err = system("ip link set bond_nest1 master bond"); 475 if (!ASSERT_OK(err, "set bond_nest1 master")) 476 goto out; 477 478 if (!ASSERT_OK(system("ip link add bond_nest2 type bond"), "add bond_nest1")) 479 goto out; 480 481 err = system("ip link set bond_nest2 master bond_nest1"); 482 if (!ASSERT_OK(err, "set bond_nest2 master")) 483 goto out; 484 485 link = bpf_program__attach_xdp(skeletons->xdp_dummy->progs.xdp_dummy_prog, bond); 486 ASSERT_OK_PTR(link, "attach program to master"); 487 488 out: 489 bpf_link__destroy(link); 490 system("ip link del bond"); 491 system("ip link del bond_nest1"); 492 system("ip link del bond_nest2"); 493 } 494 495 static int libbpf_debug_print(enum libbpf_print_level level, 496 const char *format, va_list args) 497 { 498 if (level != LIBBPF_WARN) 499 vprintf(format, args); 500 return 0; 501 } 502 503 struct bond_test_case { 504 char *name; 505 int mode; 506 int xmit_policy; 507 }; 508 509 static struct bond_test_case bond_test_cases[] = { 510 { "xdp_bonding_roundrobin", BOND_MODE_ROUNDROBIN, BOND_XMIT_POLICY_LAYER23, }, 511 { "xdp_bonding_activebackup", BOND_MODE_ACTIVEBACKUP, BOND_XMIT_POLICY_LAYER23 }, 512 513 { "xdp_bonding_xor_layer2", BOND_MODE_XOR, BOND_XMIT_POLICY_LAYER2, }, 514 { "xdp_bonding_xor_layer23", BOND_MODE_XOR, BOND_XMIT_POLICY_LAYER23, }, 515 { "xdp_bonding_xor_layer34", BOND_MODE_XOR, BOND_XMIT_POLICY_LAYER34, }, 516 }; 517 518 void serial_test_xdp_bonding(void) 519 { 520 libbpf_print_fn_t old_print_fn; 521 struct skeletons skeletons = {}; 522 int i; 523 524 old_print_fn = libbpf_set_print(libbpf_debug_print); 525 526 root_netns_fd = open("/proc/self/ns/net", O_RDONLY); 527 if (!ASSERT_GE(root_netns_fd, 0, "open /proc/self/ns/net")) 528 goto out; 529 530 skeletons.xdp_dummy = xdp_dummy__open_and_load(); 531 if (!ASSERT_OK_PTR(skeletons.xdp_dummy, "xdp_dummy__open_and_load")) 532 goto out; 533 534 skeletons.xdp_tx = xdp_tx__open_and_load(); 535 if (!ASSERT_OK_PTR(skeletons.xdp_tx, "xdp_tx__open_and_load")) 536 goto out; 537 538 skeletons.xdp_redirect_multi_kern = xdp_redirect_multi_kern__open_and_load(); 539 if (!ASSERT_OK_PTR(skeletons.xdp_redirect_multi_kern, 540 "xdp_redirect_multi_kern__open_and_load")) 541 goto out; 542 543 if (test__start_subtest("xdp_bonding_attach")) 544 test_xdp_bonding_attach(&skeletons); 545 546 if (test__start_subtest("xdp_bonding_nested")) 547 test_xdp_bonding_nested(&skeletons); 548 549 for (i = 0; i < ARRAY_SIZE(bond_test_cases); i++) { 550 struct bond_test_case *test_case = &bond_test_cases[i]; 551 552 if (test__start_subtest(test_case->name)) 553 test_xdp_bonding_with_mode( 554 &skeletons, 555 test_case->mode, 556 test_case->xmit_policy); 557 } 558 559 if (test__start_subtest("xdp_bonding_redirect_multi")) 560 test_xdp_bonding_redirect_multi(&skeletons); 561 562 out: 563 xdp_dummy__destroy(skeletons.xdp_dummy); 564 xdp_tx__destroy(skeletons.xdp_tx); 565 xdp_redirect_multi_kern__destroy(skeletons.xdp_redirect_multi_kern); 566 567 libbpf_set_print(old_print_fn); 568 if (root_netns_fd >= 0) 569 close(root_netns_fd); 570 } 571