1 // SPDX-License-Identifier: GPL-2.0 2 3 /** 4 * Test XDP bonding support 5 * 6 * Sets up two bonded veth pairs between two fresh namespaces 7 * and verifies that XDP_TX program loaded on a bond device 8 * are correctly loaded onto the slave devices and XDP_TX'd 9 * packets are balanced using bonding. 10 */ 11 12 #define _GNU_SOURCE 13 #include <sched.h> 14 #include <net/if.h> 15 #include <linux/if_link.h> 16 #include "test_progs.h" 17 #include "network_helpers.h" 18 #include <linux/if_bonding.h> 19 #include <linux/limits.h> 20 #include <linux/udp.h> 21 22 #include "xdp_dummy.skel.h" 23 #include "xdp_redirect_multi_kern.skel.h" 24 #include "xdp_tx.skel.h" 25 26 #define BOND1_MAC {0x00, 0x11, 0x22, 0x33, 0x44, 0x55} 27 #define BOND1_MAC_STR "00:11:22:33:44:55" 28 #define BOND2_MAC {0x00, 0x22, 0x33, 0x44, 0x55, 0x66} 29 #define BOND2_MAC_STR "00:22:33:44:55:66" 30 #define NPACKETS 100 31 32 static int root_netns_fd = -1; 33 34 static void restore_root_netns(void) 35 { 36 ASSERT_OK(setns(root_netns_fd, CLONE_NEWNET), "restore_root_netns"); 37 } 38 39 static int setns_by_name(char *name) 40 { 41 int nsfd, err; 42 char nspath[PATH_MAX]; 43 44 snprintf(nspath, sizeof(nspath), "%s/%s", "/var/run/netns", name); 45 nsfd = open(nspath, O_RDONLY | O_CLOEXEC); 46 if (nsfd < 0) 47 return -1; 48 49 err = setns(nsfd, CLONE_NEWNET); 50 close(nsfd); 51 return err; 52 } 53 54 static int get_rx_packets(const char *iface) 55 { 56 FILE *f; 57 char line[512]; 58 int iface_len = strlen(iface); 59 60 f = fopen("/proc/net/dev", "r"); 61 if (!f) 62 return -1; 63 64 while (fgets(line, sizeof(line), f)) { 65 char *p = line; 66 67 while (*p == ' ') 68 p++; /* skip whitespace */ 69 if (!strncmp(p, iface, iface_len)) { 70 p += iface_len; 71 if (*p++ != ':') 72 continue; 73 while (*p == ' ') 74 p++; /* skip whitespace */ 75 while (*p && *p != ' ') 76 p++; /* skip rx bytes */ 77 while (*p == ' ') 78 p++; /* skip whitespace */ 79 fclose(f); 80 return atoi(p); 81 } 82 } 83 fclose(f); 84 return -1; 85 } 86 87 #define MAX_BPF_LINKS 8 88 89 struct skeletons { 90 struct xdp_dummy *xdp_dummy; 91 struct xdp_tx *xdp_tx; 92 struct xdp_redirect_multi_kern *xdp_redirect_multi_kern; 93 94 int nlinks; 95 struct bpf_link *links[MAX_BPF_LINKS]; 96 }; 97 98 static int xdp_attach(struct skeletons *skeletons, struct bpf_program *prog, char *iface) 99 { 100 struct bpf_link *link; 101 int ifindex; 102 103 ifindex = if_nametoindex(iface); 104 if (!ASSERT_GT(ifindex, 0, "get ifindex")) 105 return -1; 106 107 if (!ASSERT_LE(skeletons->nlinks+1, MAX_BPF_LINKS, "too many XDP programs attached")) 108 return -1; 109 110 link = bpf_program__attach_xdp(prog, ifindex); 111 if (!ASSERT_OK_PTR(link, "attach xdp program")) 112 return -1; 113 114 skeletons->links[skeletons->nlinks++] = link; 115 return 0; 116 } 117 118 enum { 119 BOND_ONE_NO_ATTACH = 0, 120 BOND_BOTH_AND_ATTACH, 121 }; 122 123 static const char * const mode_names[] = { 124 [BOND_MODE_ROUNDROBIN] = "balance-rr", 125 [BOND_MODE_ACTIVEBACKUP] = "active-backup", 126 [BOND_MODE_XOR] = "balance-xor", 127 [BOND_MODE_BROADCAST] = "broadcast", 128 [BOND_MODE_8023AD] = "802.3ad", 129 [BOND_MODE_TLB] = "balance-tlb", 130 [BOND_MODE_ALB] = "balance-alb", 131 }; 132 133 static const char * const xmit_policy_names[] = { 134 [BOND_XMIT_POLICY_LAYER2] = "layer2", 135 [BOND_XMIT_POLICY_LAYER34] = "layer3+4", 136 [BOND_XMIT_POLICY_LAYER23] = "layer2+3", 137 [BOND_XMIT_POLICY_ENCAP23] = "encap2+3", 138 [BOND_XMIT_POLICY_ENCAP34] = "encap3+4", 139 }; 140 141 static int bonding_setup(struct skeletons *skeletons, int mode, int xmit_policy, 142 int bond_both_attach) 143 { 144 #define SYS(fmt, ...) \ 145 ({ \ 146 char cmd[1024]; \ 147 snprintf(cmd, sizeof(cmd), fmt, ##__VA_ARGS__); \ 148 if (!ASSERT_OK(system(cmd), cmd)) \ 149 return -1; \ 150 }) 151 152 SYS("ip netns add ns_dst"); 153 SYS("ip link add veth1_1 type veth peer name veth2_1 netns ns_dst"); 154 SYS("ip link add veth1_2 type veth peer name veth2_2 netns ns_dst"); 155 156 SYS("ip link add bond1 type bond mode %s xmit_hash_policy %s", 157 mode_names[mode], xmit_policy_names[xmit_policy]); 158 SYS("ip link set bond1 up address " BOND1_MAC_STR " addrgenmode none"); 159 SYS("ip -netns ns_dst link add bond2 type bond mode %s xmit_hash_policy %s", 160 mode_names[mode], xmit_policy_names[xmit_policy]); 161 SYS("ip -netns ns_dst link set bond2 up address " BOND2_MAC_STR " addrgenmode none"); 162 163 SYS("ip link set veth1_1 master bond1"); 164 if (bond_both_attach == BOND_BOTH_AND_ATTACH) { 165 SYS("ip link set veth1_2 master bond1"); 166 } else { 167 SYS("ip link set veth1_2 up addrgenmode none"); 168 169 if (xdp_attach(skeletons, skeletons->xdp_dummy->progs.xdp_dummy_prog, "veth1_2")) 170 return -1; 171 } 172 173 SYS("ip -netns ns_dst link set veth2_1 master bond2"); 174 175 if (bond_both_attach == BOND_BOTH_AND_ATTACH) 176 SYS("ip -netns ns_dst link set veth2_2 master bond2"); 177 else 178 SYS("ip -netns ns_dst link set veth2_2 up addrgenmode none"); 179 180 /* Load a dummy program on sending side as with veth peer needs to have a 181 * XDP program loaded as well. 182 */ 183 if (xdp_attach(skeletons, skeletons->xdp_dummy->progs.xdp_dummy_prog, "bond1")) 184 return -1; 185 186 if (bond_both_attach == BOND_BOTH_AND_ATTACH) { 187 if (!ASSERT_OK(setns_by_name("ns_dst"), "set netns to ns_dst")) 188 return -1; 189 190 if (xdp_attach(skeletons, skeletons->xdp_tx->progs.xdp_tx, "bond2")) 191 return -1; 192 193 restore_root_netns(); 194 } 195 196 return 0; 197 198 #undef SYS 199 } 200 201 static void bonding_cleanup(struct skeletons *skeletons) 202 { 203 restore_root_netns(); 204 while (skeletons->nlinks) { 205 skeletons->nlinks--; 206 bpf_link__destroy(skeletons->links[skeletons->nlinks]); 207 } 208 ASSERT_OK(system("ip link delete bond1"), "delete bond1"); 209 ASSERT_OK(system("ip link delete veth1_1"), "delete veth1_1"); 210 ASSERT_OK(system("ip link delete veth1_2"), "delete veth1_2"); 211 ASSERT_OK(system("ip netns delete ns_dst"), "delete ns_dst"); 212 } 213 214 static int send_udp_packets(int vary_dst_ip) 215 { 216 struct ethhdr eh = { 217 .h_source = BOND1_MAC, 218 .h_dest = BOND2_MAC, 219 .h_proto = htons(ETH_P_IP), 220 }; 221 uint8_t buf[128] = {}; 222 struct iphdr *iph = (struct iphdr *)(buf + sizeof(eh)); 223 struct udphdr *uh = (struct udphdr *)(buf + sizeof(eh) + sizeof(*iph)); 224 int i, s = -1; 225 int ifindex; 226 227 s = socket(AF_PACKET, SOCK_RAW, IPPROTO_RAW); 228 if (!ASSERT_GE(s, 0, "socket")) 229 goto err; 230 231 ifindex = if_nametoindex("bond1"); 232 if (!ASSERT_GT(ifindex, 0, "get bond1 ifindex")) 233 goto err; 234 235 memcpy(buf, &eh, sizeof(eh)); 236 iph->ihl = 5; 237 iph->version = 4; 238 iph->tos = 16; 239 iph->id = 1; 240 iph->ttl = 64; 241 iph->protocol = IPPROTO_UDP; 242 iph->saddr = 1; 243 iph->daddr = 2; 244 iph->tot_len = htons(sizeof(buf) - ETH_HLEN); 245 iph->check = 0; 246 247 for (i = 1; i <= NPACKETS; i++) { 248 int n; 249 struct sockaddr_ll saddr_ll = { 250 .sll_ifindex = ifindex, 251 .sll_halen = ETH_ALEN, 252 .sll_addr = BOND2_MAC, 253 }; 254 255 /* vary the UDP destination port for even distribution with roundrobin/xor modes */ 256 uh->dest++; 257 258 if (vary_dst_ip) 259 iph->daddr++; 260 261 n = sendto(s, buf, sizeof(buf), 0, (struct sockaddr *)&saddr_ll, sizeof(saddr_ll)); 262 if (!ASSERT_EQ(n, sizeof(buf), "sendto")) 263 goto err; 264 } 265 266 return 0; 267 268 err: 269 if (s >= 0) 270 close(s); 271 return -1; 272 } 273 274 static void test_xdp_bonding_with_mode(struct skeletons *skeletons, int mode, int xmit_policy) 275 { 276 int bond1_rx; 277 278 if (bonding_setup(skeletons, mode, xmit_policy, BOND_BOTH_AND_ATTACH)) 279 goto out; 280 281 if (send_udp_packets(xmit_policy != BOND_XMIT_POLICY_LAYER34)) 282 goto out; 283 284 bond1_rx = get_rx_packets("bond1"); 285 ASSERT_EQ(bond1_rx, NPACKETS, "expected more received packets"); 286 287 switch (mode) { 288 case BOND_MODE_ROUNDROBIN: 289 case BOND_MODE_XOR: { 290 int veth1_rx = get_rx_packets("veth1_1"); 291 int veth2_rx = get_rx_packets("veth1_2"); 292 int diff = abs(veth1_rx - veth2_rx); 293 294 ASSERT_GE(veth1_rx + veth2_rx, NPACKETS, "expected more packets"); 295 296 switch (xmit_policy) { 297 case BOND_XMIT_POLICY_LAYER2: 298 ASSERT_GE(diff, NPACKETS, 299 "expected packets on only one of the interfaces"); 300 break; 301 case BOND_XMIT_POLICY_LAYER23: 302 case BOND_XMIT_POLICY_LAYER34: 303 ASSERT_LT(diff, NPACKETS/2, 304 "expected even distribution of packets"); 305 break; 306 default: 307 PRINT_FAIL("Unimplemented xmit_policy=%d\n", xmit_policy); 308 break; 309 } 310 break; 311 } 312 case BOND_MODE_ACTIVEBACKUP: { 313 int veth1_rx = get_rx_packets("veth1_1"); 314 int veth2_rx = get_rx_packets("veth1_2"); 315 int diff = abs(veth1_rx - veth2_rx); 316 317 ASSERT_GE(diff, NPACKETS, 318 "expected packets on only one of the interfaces"); 319 break; 320 } 321 default: 322 PRINT_FAIL("Unimplemented xmit_policy=%d\n", xmit_policy); 323 break; 324 } 325 326 out: 327 bonding_cleanup(skeletons); 328 } 329 330 /* Test the broadcast redirection using xdp_redirect_map_multi_prog and adding 331 * all the interfaces to it and checking that broadcasting won't send the packet 332 * to neither the ingress bond device (bond2) or its slave (veth2_1). 333 */ 334 static void test_xdp_bonding_redirect_multi(struct skeletons *skeletons) 335 { 336 static const char * const ifaces[] = {"bond2", "veth2_1", "veth2_2"}; 337 int veth1_1_rx, veth1_2_rx; 338 int err; 339 340 if (bonding_setup(skeletons, BOND_MODE_ROUNDROBIN, BOND_XMIT_POLICY_LAYER23, 341 BOND_ONE_NO_ATTACH)) 342 goto out; 343 344 345 if (!ASSERT_OK(setns_by_name("ns_dst"), "could not set netns to ns_dst")) 346 goto out; 347 348 /* populate the devmap with the relevant interfaces */ 349 for (int i = 0; i < ARRAY_SIZE(ifaces); i++) { 350 int ifindex = if_nametoindex(ifaces[i]); 351 int map_fd = bpf_map__fd(skeletons->xdp_redirect_multi_kern->maps.map_all); 352 353 if (!ASSERT_GT(ifindex, 0, "could not get interface index")) 354 goto out; 355 356 err = bpf_map_update_elem(map_fd, &ifindex, &ifindex, 0); 357 if (!ASSERT_OK(err, "add interface to map_all")) 358 goto out; 359 } 360 361 if (xdp_attach(skeletons, 362 skeletons->xdp_redirect_multi_kern->progs.xdp_redirect_map_multi_prog, 363 "bond2")) 364 goto out; 365 366 restore_root_netns(); 367 368 if (send_udp_packets(BOND_MODE_ROUNDROBIN)) 369 goto out; 370 371 veth1_1_rx = get_rx_packets("veth1_1"); 372 veth1_2_rx = get_rx_packets("veth1_2"); 373 374 ASSERT_EQ(veth1_1_rx, 0, "expected no packets on veth1_1"); 375 ASSERT_GE(veth1_2_rx, NPACKETS, "expected packets on veth1_2"); 376 377 out: 378 restore_root_netns(); 379 bonding_cleanup(skeletons); 380 } 381 382 /* Test that XDP programs cannot be attached to both the bond master and slaves simultaneously */ 383 static void test_xdp_bonding_attach(struct skeletons *skeletons) 384 { 385 struct bpf_link *link = NULL; 386 struct bpf_link *link2 = NULL; 387 int veth, bond; 388 int err; 389 390 if (!ASSERT_OK(system("ip link add veth type veth"), "add veth")) 391 goto out; 392 if (!ASSERT_OK(system("ip link add bond type bond"), "add bond")) 393 goto out; 394 395 veth = if_nametoindex("veth"); 396 if (!ASSERT_GE(veth, 0, "if_nametoindex veth")) 397 goto out; 398 bond = if_nametoindex("bond"); 399 if (!ASSERT_GE(bond, 0, "if_nametoindex bond")) 400 goto out; 401 402 /* enslaving with a XDP program loaded fails */ 403 link = bpf_program__attach_xdp(skeletons->xdp_dummy->progs.xdp_dummy_prog, veth); 404 if (!ASSERT_OK_PTR(link, "attach program to veth")) 405 goto out; 406 407 err = system("ip link set veth master bond"); 408 if (!ASSERT_NEQ(err, 0, "attaching slave with xdp program expected to fail")) 409 goto out; 410 411 bpf_link__destroy(link); 412 link = NULL; 413 414 err = system("ip link set veth master bond"); 415 if (!ASSERT_OK(err, "set veth master")) 416 goto out; 417 418 /* attaching to slave when master has no program is allowed */ 419 link = bpf_program__attach_xdp(skeletons->xdp_dummy->progs.xdp_dummy_prog, veth); 420 if (!ASSERT_OK_PTR(link, "attach program to slave when enslaved")) 421 goto out; 422 423 /* attaching to master not allowed when slave has program loaded */ 424 link2 = bpf_program__attach_xdp(skeletons->xdp_dummy->progs.xdp_dummy_prog, bond); 425 if (!ASSERT_ERR_PTR(link2, "attach program to master when slave has program")) 426 goto out; 427 428 bpf_link__destroy(link); 429 link = NULL; 430 431 /* attaching XDP program to master allowed when slave has no program */ 432 link = bpf_program__attach_xdp(skeletons->xdp_dummy->progs.xdp_dummy_prog, bond); 433 if (!ASSERT_OK_PTR(link, "attach program to master")) 434 goto out; 435 436 /* attaching to slave not allowed when master has program loaded */ 437 link2 = bpf_program__attach_xdp(skeletons->xdp_dummy->progs.xdp_dummy_prog, bond); 438 ASSERT_ERR_PTR(link2, "attach program to slave when master has program"); 439 440 out: 441 bpf_link__destroy(link); 442 bpf_link__destroy(link2); 443 444 system("ip link del veth"); 445 system("ip link del bond"); 446 } 447 448 static int libbpf_debug_print(enum libbpf_print_level level, 449 const char *format, va_list args) 450 { 451 if (level != LIBBPF_WARN) 452 vprintf(format, args); 453 return 0; 454 } 455 456 struct bond_test_case { 457 char *name; 458 int mode; 459 int xmit_policy; 460 }; 461 462 static struct bond_test_case bond_test_cases[] = { 463 { "xdp_bonding_roundrobin", BOND_MODE_ROUNDROBIN, BOND_XMIT_POLICY_LAYER23, }, 464 { "xdp_bonding_activebackup", BOND_MODE_ACTIVEBACKUP, BOND_XMIT_POLICY_LAYER23 }, 465 466 { "xdp_bonding_xor_layer2", BOND_MODE_XOR, BOND_XMIT_POLICY_LAYER2, }, 467 { "xdp_bonding_xor_layer23", BOND_MODE_XOR, BOND_XMIT_POLICY_LAYER23, }, 468 { "xdp_bonding_xor_layer34", BOND_MODE_XOR, BOND_XMIT_POLICY_LAYER34, }, 469 }; 470 471 void test_xdp_bonding(void) 472 { 473 libbpf_print_fn_t old_print_fn; 474 struct skeletons skeletons = {}; 475 int i; 476 477 old_print_fn = libbpf_set_print(libbpf_debug_print); 478 479 root_netns_fd = open("/proc/self/ns/net", O_RDONLY); 480 if (!ASSERT_GE(root_netns_fd, 0, "open /proc/self/ns/net")) 481 goto out; 482 483 skeletons.xdp_dummy = xdp_dummy__open_and_load(); 484 if (!ASSERT_OK_PTR(skeletons.xdp_dummy, "xdp_dummy__open_and_load")) 485 goto out; 486 487 skeletons.xdp_tx = xdp_tx__open_and_load(); 488 if (!ASSERT_OK_PTR(skeletons.xdp_tx, "xdp_tx__open_and_load")) 489 goto out; 490 491 skeletons.xdp_redirect_multi_kern = xdp_redirect_multi_kern__open_and_load(); 492 if (!ASSERT_OK_PTR(skeletons.xdp_redirect_multi_kern, 493 "xdp_redirect_multi_kern__open_and_load")) 494 goto out; 495 496 if (test__start_subtest("xdp_bonding_attach")) 497 test_xdp_bonding_attach(&skeletons); 498 499 for (i = 0; i < ARRAY_SIZE(bond_test_cases); i++) { 500 struct bond_test_case *test_case = &bond_test_cases[i]; 501 502 if (test__start_subtest(test_case->name)) 503 test_xdp_bonding_with_mode( 504 &skeletons, 505 test_case->mode, 506 test_case->xmit_policy); 507 } 508 509 if (test__start_subtest("xdp_bonding_redirect_multi")) 510 test_xdp_bonding_redirect_multi(&skeletons); 511 512 out: 513 xdp_dummy__destroy(skeletons.xdp_dummy); 514 xdp_tx__destroy(skeletons.xdp_tx); 515 xdp_redirect_multi_kern__destroy(skeletons.xdp_redirect_multi_kern); 516 517 libbpf_set_print(old_print_fn); 518 if (root_netns_fd >= 0) 519 close(root_netns_fd); 520 } 521