1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018 Facebook */ 3 4 #include <stdlib.h> 5 #include <unistd.h> 6 #include <stdbool.h> 7 #include <string.h> 8 #include <errno.h> 9 #include <assert.h> 10 #include <fcntl.h> 11 #include <linux/bpf.h> 12 #include <linux/err.h> 13 #include <linux/types.h> 14 #include <linux/if_ether.h> 15 #include <sys/types.h> 16 #include <sys/epoll.h> 17 #include <sys/socket.h> 18 #include <netinet/in.h> 19 #include <bpf/bpf.h> 20 #include <bpf/libbpf.h> 21 #include "bpf_rlimit.h" 22 #include "bpf_util.h" 23 24 #include "test_progs.h" 25 #include "test_select_reuseport_common.h" 26 27 #define MAX_TEST_NAME 80 28 #define MIN_TCPHDR_LEN 20 29 #define UDPHDR_LEN 8 30 31 #define TCP_SYNCOOKIE_SYSCTL "/proc/sys/net/ipv4/tcp_syncookies" 32 #define TCP_FO_SYSCTL "/proc/sys/net/ipv4/tcp_fastopen" 33 #define REUSEPORT_ARRAY_SIZE 32 34 35 static int result_map, tmp_index_ovr_map, linum_map, data_check_map; 36 static __u32 expected_results[NR_RESULTS]; 37 static int sk_fds[REUSEPORT_ARRAY_SIZE]; 38 static int reuseport_array = -1, outer_map = -1; 39 static int select_by_skb_data_prog; 40 static int saved_tcp_syncookie = -1; 41 static struct bpf_object *obj; 42 static int saved_tcp_fo = -1; 43 static __u32 index_zero; 44 static int epfd; 45 46 static union sa46 { 47 struct sockaddr_in6 v6; 48 struct sockaddr_in v4; 49 sa_family_t family; 50 } srv_sa; 51 52 #define RET_IF(condition, tag, format...) ({ \ 53 if (CHECK_FAIL(condition)) { \ 54 printf(tag " " format); \ 55 return; \ 56 } \ 57 }) 58 59 #define RET_ERR(condition, tag, format...) ({ \ 60 if (CHECK_FAIL(condition)) { \ 61 printf(tag " " format); \ 62 return -1; \ 63 } \ 64 }) 65 66 static int create_maps(void) 67 { 68 struct bpf_create_map_attr attr = {}; 69 70 /* Creating reuseport_array */ 71 attr.name = "reuseport_array"; 72 attr.map_type = BPF_MAP_TYPE_REUSEPORT_SOCKARRAY; 73 attr.key_size = sizeof(__u32); 74 attr.value_size = sizeof(__u32); 75 attr.max_entries = REUSEPORT_ARRAY_SIZE; 76 77 reuseport_array = bpf_create_map_xattr(&attr); 78 RET_ERR(reuseport_array == -1, "creating reuseport_array", 79 "reuseport_array:%d errno:%d\n", reuseport_array, errno); 80 81 /* Creating outer_map */ 82 attr.name = "outer_map"; 83 attr.map_type = BPF_MAP_TYPE_ARRAY_OF_MAPS; 84 attr.key_size = sizeof(__u32); 85 attr.value_size = sizeof(__u32); 86 attr.max_entries = 1; 87 attr.inner_map_fd = reuseport_array; 88 outer_map = bpf_create_map_xattr(&attr); 89 RET_ERR(outer_map == -1, "creating outer_map", 90 "outer_map:%d errno:%d\n", outer_map, errno); 91 92 return 0; 93 } 94 95 static int prepare_bpf_obj(void) 96 { 97 struct bpf_program *prog; 98 struct bpf_map *map; 99 int err; 100 101 obj = bpf_object__open("test_select_reuseport_kern.o"); 102 RET_ERR(IS_ERR_OR_NULL(obj), "open test_select_reuseport_kern.o", 103 "obj:%p PTR_ERR(obj):%ld\n", obj, PTR_ERR(obj)); 104 105 map = bpf_object__find_map_by_name(obj, "outer_map"); 106 RET_ERR(!map, "find outer_map", "!map\n"); 107 err = bpf_map__reuse_fd(map, outer_map); 108 RET_ERR(err, "reuse outer_map", "err:%d\n", err); 109 110 err = bpf_object__load(obj); 111 RET_ERR(err, "load bpf_object", "err:%d\n", err); 112 113 prog = bpf_program__next(NULL, obj); 114 RET_ERR(!prog, "get first bpf_program", "!prog\n"); 115 select_by_skb_data_prog = bpf_program__fd(prog); 116 RET_ERR(select_by_skb_data_prog == -1, "get prog fd", 117 "select_by_skb_data_prog:%d\n", select_by_skb_data_prog); 118 119 map = bpf_object__find_map_by_name(obj, "result_map"); 120 RET_ERR(!map, "find result_map", "!map\n"); 121 result_map = bpf_map__fd(map); 122 RET_ERR(result_map == -1, "get result_map fd", 123 "result_map:%d\n", result_map); 124 125 map = bpf_object__find_map_by_name(obj, "tmp_index_ovr_map"); 126 RET_ERR(!map, "find tmp_index_ovr_map\n", "!map"); 127 tmp_index_ovr_map = bpf_map__fd(map); 128 RET_ERR(tmp_index_ovr_map == -1, "get tmp_index_ovr_map fd", 129 "tmp_index_ovr_map:%d\n", tmp_index_ovr_map); 130 131 map = bpf_object__find_map_by_name(obj, "linum_map"); 132 RET_ERR(!map, "find linum_map", "!map\n"); 133 linum_map = bpf_map__fd(map); 134 RET_ERR(linum_map == -1, "get linum_map fd", 135 "linum_map:%d\n", linum_map); 136 137 map = bpf_object__find_map_by_name(obj, "data_check_map"); 138 RET_ERR(!map, "find data_check_map", "!map\n"); 139 data_check_map = bpf_map__fd(map); 140 RET_ERR(data_check_map == -1, "get data_check_map fd", 141 "data_check_map:%d\n", data_check_map); 142 143 return 0; 144 } 145 146 static void sa46_init_loopback(union sa46 *sa, sa_family_t family) 147 { 148 memset(sa, 0, sizeof(*sa)); 149 sa->family = family; 150 if (sa->family == AF_INET6) 151 sa->v6.sin6_addr = in6addr_loopback; 152 else 153 sa->v4.sin_addr.s_addr = htonl(INADDR_LOOPBACK); 154 } 155 156 static void sa46_init_inany(union sa46 *sa, sa_family_t family) 157 { 158 memset(sa, 0, sizeof(*sa)); 159 sa->family = family; 160 if (sa->family == AF_INET6) 161 sa->v6.sin6_addr = in6addr_any; 162 else 163 sa->v4.sin_addr.s_addr = INADDR_ANY; 164 } 165 166 static int read_int_sysctl(const char *sysctl) 167 { 168 char buf[16]; 169 int fd, ret; 170 171 fd = open(sysctl, 0); 172 RET_ERR(fd == -1, "open(sysctl)", 173 "sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno); 174 175 ret = read(fd, buf, sizeof(buf)); 176 RET_ERR(ret <= 0, "read(sysctl)", 177 "sysctl:%s ret:%d errno:%d\n", sysctl, ret, errno); 178 179 close(fd); 180 return atoi(buf); 181 } 182 183 static int write_int_sysctl(const char *sysctl, int v) 184 { 185 int fd, ret, size; 186 char buf[16]; 187 188 fd = open(sysctl, O_RDWR); 189 RET_ERR(fd == -1, "open(sysctl)", 190 "sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno); 191 192 size = snprintf(buf, sizeof(buf), "%d", v); 193 ret = write(fd, buf, size); 194 RET_ERR(ret != size, "write(sysctl)", 195 "sysctl:%s ret:%d size:%d errno:%d\n", 196 sysctl, ret, size, errno); 197 198 close(fd); 199 return 0; 200 } 201 202 static void restore_sysctls(void) 203 { 204 if (saved_tcp_fo != -1) 205 write_int_sysctl(TCP_FO_SYSCTL, saved_tcp_fo); 206 if (saved_tcp_syncookie != -1) 207 write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, saved_tcp_syncookie); 208 } 209 210 static int enable_fastopen(void) 211 { 212 int fo; 213 214 fo = read_int_sysctl(TCP_FO_SYSCTL); 215 if (fo < 0) 216 return -1; 217 218 return write_int_sysctl(TCP_FO_SYSCTL, fo | 7); 219 } 220 221 static int enable_syncookie(void) 222 { 223 return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 2); 224 } 225 226 static int disable_syncookie(void) 227 { 228 return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 0); 229 } 230 231 static long get_linum(void) 232 { 233 __u32 linum; 234 int err; 235 236 err = bpf_map_lookup_elem(linum_map, &index_zero, &linum); 237 RET_ERR(err == -1, "lookup_elem(linum_map)", "err:%d errno:%d\n", 238 err, errno); 239 240 return linum; 241 } 242 243 static void check_data(int type, sa_family_t family, const struct cmd *cmd, 244 int cli_fd) 245 { 246 struct data_check expected = {}, result; 247 union sa46 cli_sa; 248 socklen_t addrlen; 249 int err; 250 251 addrlen = sizeof(cli_sa); 252 err = getsockname(cli_fd, (struct sockaddr *)&cli_sa, 253 &addrlen); 254 RET_IF(err == -1, "getsockname(cli_fd)", "err:%d errno:%d\n", 255 err, errno); 256 257 err = bpf_map_lookup_elem(data_check_map, &index_zero, &result); 258 RET_IF(err == -1, "lookup_elem(data_check_map)", "err:%d errno:%d\n", 259 err, errno); 260 261 if (type == SOCK_STREAM) { 262 expected.len = MIN_TCPHDR_LEN; 263 expected.ip_protocol = IPPROTO_TCP; 264 } else { 265 expected.len = UDPHDR_LEN; 266 expected.ip_protocol = IPPROTO_UDP; 267 } 268 269 if (family == AF_INET6) { 270 expected.eth_protocol = htons(ETH_P_IPV6); 271 expected.bind_inany = !srv_sa.v6.sin6_addr.s6_addr32[3] && 272 !srv_sa.v6.sin6_addr.s6_addr32[2] && 273 !srv_sa.v6.sin6_addr.s6_addr32[1] && 274 !srv_sa.v6.sin6_addr.s6_addr32[0]; 275 276 memcpy(&expected.skb_addrs[0], cli_sa.v6.sin6_addr.s6_addr32, 277 sizeof(cli_sa.v6.sin6_addr)); 278 memcpy(&expected.skb_addrs[4], &in6addr_loopback, 279 sizeof(in6addr_loopback)); 280 expected.skb_ports[0] = cli_sa.v6.sin6_port; 281 expected.skb_ports[1] = srv_sa.v6.sin6_port; 282 } else { 283 expected.eth_protocol = htons(ETH_P_IP); 284 expected.bind_inany = !srv_sa.v4.sin_addr.s_addr; 285 286 expected.skb_addrs[0] = cli_sa.v4.sin_addr.s_addr; 287 expected.skb_addrs[1] = htonl(INADDR_LOOPBACK); 288 expected.skb_ports[0] = cli_sa.v4.sin_port; 289 expected.skb_ports[1] = srv_sa.v4.sin_port; 290 } 291 292 if (memcmp(&result, &expected, offsetof(struct data_check, 293 equal_check_end))) { 294 printf("unexpected data_check\n"); 295 printf(" result: (0x%x, %u, %u)\n", 296 result.eth_protocol, result.ip_protocol, 297 result.bind_inany); 298 printf("expected: (0x%x, %u, %u)\n", 299 expected.eth_protocol, expected.ip_protocol, 300 expected.bind_inany); 301 RET_IF(1, "data_check result != expected", 302 "bpf_prog_linum:%ld\n", get_linum()); 303 } 304 305 RET_IF(!result.hash, "data_check result.hash empty", 306 "result.hash:%u", result.hash); 307 308 expected.len += cmd ? sizeof(*cmd) : 0; 309 if (type == SOCK_STREAM) 310 RET_IF(expected.len > result.len, "expected.len > result.len", 311 "expected.len:%u result.len:%u bpf_prog_linum:%ld\n", 312 expected.len, result.len, get_linum()); 313 else 314 RET_IF(expected.len != result.len, "expected.len != result.len", 315 "expected.len:%u result.len:%u bpf_prog_linum:%ld\n", 316 expected.len, result.len, get_linum()); 317 } 318 319 static const char *result_to_str(enum result res) 320 { 321 switch (res) { 322 case DROP_ERR_INNER_MAP: 323 return "DROP_ERR_INNER_MAP"; 324 case DROP_ERR_SKB_DATA: 325 return "DROP_ERR_SKB_DATA"; 326 case DROP_ERR_SK_SELECT_REUSEPORT: 327 return "DROP_ERR_SK_SELECT_REUSEPORT"; 328 case DROP_MISC: 329 return "DROP_MISC"; 330 case PASS: 331 return "PASS"; 332 case PASS_ERR_SK_SELECT_REUSEPORT: 333 return "PASS_ERR_SK_SELECT_REUSEPORT"; 334 default: 335 return "UNKNOWN"; 336 } 337 } 338 339 static void check_results(void) 340 { 341 __u32 results[NR_RESULTS]; 342 __u32 i, broken = 0; 343 int err; 344 345 for (i = 0; i < NR_RESULTS; i++) { 346 err = bpf_map_lookup_elem(result_map, &i, &results[i]); 347 RET_IF(err == -1, "lookup_elem(result_map)", 348 "i:%u err:%d errno:%d\n", i, err, errno); 349 } 350 351 for (i = 0; i < NR_RESULTS; i++) { 352 if (results[i] != expected_results[i]) { 353 broken = i; 354 break; 355 } 356 } 357 358 if (i == NR_RESULTS) 359 return; 360 361 printf("unexpected result\n"); 362 printf(" result: ["); 363 printf("%u", results[0]); 364 for (i = 1; i < NR_RESULTS; i++) 365 printf(", %u", results[i]); 366 printf("]\n"); 367 368 printf("expected: ["); 369 printf("%u", expected_results[0]); 370 for (i = 1; i < NR_RESULTS; i++) 371 printf(", %u", expected_results[i]); 372 printf("]\n"); 373 374 printf("mismatch on %s (bpf_prog_linum:%ld)\n", result_to_str(broken), 375 get_linum()); 376 377 CHECK_FAIL(true); 378 } 379 380 static int send_data(int type, sa_family_t family, void *data, size_t len, 381 enum result expected) 382 { 383 union sa46 cli_sa; 384 int fd, err; 385 386 fd = socket(family, type, 0); 387 RET_ERR(fd == -1, "socket()", "fd:%d errno:%d\n", fd, errno); 388 389 sa46_init_loopback(&cli_sa, family); 390 err = bind(fd, (struct sockaddr *)&cli_sa, sizeof(cli_sa)); 391 RET_ERR(fd == -1, "bind(cli_sa)", "err:%d errno:%d\n", err, errno); 392 393 err = sendto(fd, data, len, MSG_FASTOPEN, (struct sockaddr *)&srv_sa, 394 sizeof(srv_sa)); 395 RET_ERR(err != len && expected >= PASS, 396 "sendto()", "family:%u err:%d errno:%d expected:%d\n", 397 family, err, errno, expected); 398 399 return fd; 400 } 401 402 static void do_test(int type, sa_family_t family, struct cmd *cmd, 403 enum result expected) 404 { 405 int nev, srv_fd, cli_fd; 406 struct epoll_event ev; 407 struct cmd rcv_cmd; 408 ssize_t nread; 409 410 cli_fd = send_data(type, family, cmd, cmd ? sizeof(*cmd) : 0, 411 expected); 412 if (cli_fd < 0) 413 return; 414 nev = epoll_wait(epfd, &ev, 1, expected >= PASS ? 5 : 0); 415 RET_IF((nev <= 0 && expected >= PASS) || 416 (nev > 0 && expected < PASS), 417 "nev <> expected", 418 "nev:%d expected:%d type:%d family:%d data:(%d, %d)\n", 419 nev, expected, type, family, 420 cmd ? cmd->reuseport_index : -1, 421 cmd ? cmd->pass_on_failure : -1); 422 check_results(); 423 check_data(type, family, cmd, cli_fd); 424 425 if (expected < PASS) 426 return; 427 428 RET_IF(expected != PASS_ERR_SK_SELECT_REUSEPORT && 429 cmd->reuseport_index != ev.data.u32, 430 "check cmd->reuseport_index", 431 "cmd:(%u, %u) ev.data.u32:%u\n", 432 cmd->pass_on_failure, cmd->reuseport_index, ev.data.u32); 433 434 srv_fd = sk_fds[ev.data.u32]; 435 if (type == SOCK_STREAM) { 436 int new_fd = accept(srv_fd, NULL, 0); 437 438 RET_IF(new_fd == -1, "accept(srv_fd)", 439 "ev.data.u32:%u new_fd:%d errno:%d\n", 440 ev.data.u32, new_fd, errno); 441 442 nread = recv(new_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT); 443 RET_IF(nread != sizeof(rcv_cmd), 444 "recv(new_fd)", 445 "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n", 446 ev.data.u32, nread, sizeof(rcv_cmd), errno); 447 448 close(new_fd); 449 } else { 450 nread = recv(srv_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT); 451 RET_IF(nread != sizeof(rcv_cmd), 452 "recv(sk_fds)", 453 "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n", 454 ev.data.u32, nread, sizeof(rcv_cmd), errno); 455 } 456 457 close(cli_fd); 458 } 459 460 static void test_err_inner_map(int type, sa_family_t family) 461 { 462 struct cmd cmd = { 463 .reuseport_index = 0, 464 .pass_on_failure = 0, 465 }; 466 467 expected_results[DROP_ERR_INNER_MAP]++; 468 do_test(type, family, &cmd, DROP_ERR_INNER_MAP); 469 } 470 471 static void test_err_skb_data(int type, sa_family_t family) 472 { 473 expected_results[DROP_ERR_SKB_DATA]++; 474 do_test(type, family, NULL, DROP_ERR_SKB_DATA); 475 } 476 477 static void test_err_sk_select_port(int type, sa_family_t family) 478 { 479 struct cmd cmd = { 480 .reuseport_index = REUSEPORT_ARRAY_SIZE, 481 .pass_on_failure = 0, 482 }; 483 484 expected_results[DROP_ERR_SK_SELECT_REUSEPORT]++; 485 do_test(type, family, &cmd, DROP_ERR_SK_SELECT_REUSEPORT); 486 } 487 488 static void test_pass(int type, sa_family_t family) 489 { 490 struct cmd cmd; 491 int i; 492 493 cmd.pass_on_failure = 0; 494 for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) { 495 expected_results[PASS]++; 496 cmd.reuseport_index = i; 497 do_test(type, family, &cmd, PASS); 498 } 499 } 500 501 static void test_syncookie(int type, sa_family_t family) 502 { 503 int err, tmp_index = 1; 504 struct cmd cmd = { 505 .reuseport_index = 0, 506 .pass_on_failure = 0, 507 }; 508 509 if (type != SOCK_STREAM) { 510 test__skip(); 511 return; 512 } 513 514 /* 515 * +1 for TCP-SYN and 516 * +1 for the TCP-ACK (ack the syncookie) 517 */ 518 expected_results[PASS] += 2; 519 enable_syncookie(); 520 /* 521 * Simulate TCP-SYN and TCP-ACK are handled by two different sk: 522 * TCP-SYN: select sk_fds[tmp_index = 1] tmp_index is from the 523 * tmp_index_ovr_map 524 * TCP-ACK: select sk_fds[reuseport_index = 0] reuseport_index 525 * is from the cmd.reuseport_index 526 */ 527 err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, 528 &tmp_index, BPF_ANY); 529 RET_IF(err == -1, "update_elem(tmp_index_ovr_map, 0, 1)", 530 "err:%d errno:%d\n", err, errno); 531 do_test(type, family, &cmd, PASS); 532 err = bpf_map_lookup_elem(tmp_index_ovr_map, &index_zero, 533 &tmp_index); 534 RET_IF(err == -1 || tmp_index != -1, 535 "lookup_elem(tmp_index_ovr_map)", 536 "err:%d errno:%d tmp_index:%d\n", 537 err, errno, tmp_index); 538 disable_syncookie(); 539 } 540 541 static void test_pass_on_err(int type, sa_family_t family) 542 { 543 struct cmd cmd = { 544 .reuseport_index = REUSEPORT_ARRAY_SIZE, 545 .pass_on_failure = 1, 546 }; 547 548 expected_results[PASS_ERR_SK_SELECT_REUSEPORT] += 1; 549 do_test(type, family, &cmd, PASS_ERR_SK_SELECT_REUSEPORT); 550 } 551 552 static void test_detach_bpf(int type, sa_family_t family) 553 { 554 #ifdef SO_DETACH_REUSEPORT_BPF 555 __u32 nr_run_before = 0, nr_run_after = 0, tmp, i; 556 struct epoll_event ev; 557 int cli_fd, err, nev; 558 struct cmd cmd = {}; 559 int optvalue = 0; 560 561 err = setsockopt(sk_fds[0], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF, 562 &optvalue, sizeof(optvalue)); 563 RET_IF(err == -1, "setsockopt(SO_DETACH_REUSEPORT_BPF)", 564 "err:%d errno:%d\n", err, errno); 565 566 err = setsockopt(sk_fds[1], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF, 567 &optvalue, sizeof(optvalue)); 568 RET_IF(err == 0 || errno != ENOENT, 569 "setsockopt(SO_DETACH_REUSEPORT_BPF)", 570 "err:%d errno:%d\n", err, errno); 571 572 for (i = 0; i < NR_RESULTS; i++) { 573 err = bpf_map_lookup_elem(result_map, &i, &tmp); 574 RET_IF(err == -1, "lookup_elem(result_map)", 575 "i:%u err:%d errno:%d\n", i, err, errno); 576 nr_run_before += tmp; 577 } 578 579 cli_fd = send_data(type, family, &cmd, sizeof(cmd), PASS); 580 if (cli_fd < 0) 581 return; 582 nev = epoll_wait(epfd, &ev, 1, 5); 583 RET_IF(nev <= 0, "nev <= 0", 584 "nev:%d expected:1 type:%d family:%d data:(0, 0)\n", 585 nev, type, family); 586 587 for (i = 0; i < NR_RESULTS; i++) { 588 err = bpf_map_lookup_elem(result_map, &i, &tmp); 589 RET_IF(err == -1, "lookup_elem(result_map)", 590 "i:%u err:%d errno:%d\n", i, err, errno); 591 nr_run_after += tmp; 592 } 593 594 RET_IF(nr_run_before != nr_run_after, 595 "nr_run_before != nr_run_after", 596 "nr_run_before:%u nr_run_after:%u\n", 597 nr_run_before, nr_run_after); 598 599 close(cli_fd); 600 #else 601 test__skip(); 602 #endif 603 } 604 605 static void prepare_sk_fds(int type, sa_family_t family, bool inany) 606 { 607 const int first = REUSEPORT_ARRAY_SIZE - 1; 608 int i, err, optval = 1; 609 struct epoll_event ev; 610 socklen_t addrlen; 611 612 if (inany) 613 sa46_init_inany(&srv_sa, family); 614 else 615 sa46_init_loopback(&srv_sa, family); 616 addrlen = sizeof(srv_sa); 617 618 /* 619 * The sk_fds[] is filled from the back such that the order 620 * is exactly opposite to the (struct sock_reuseport *)reuse->socks[]. 621 */ 622 for (i = first; i >= 0; i--) { 623 sk_fds[i] = socket(family, type, 0); 624 RET_IF(sk_fds[i] == -1, "socket()", "sk_fds[%d]:%d errno:%d\n", 625 i, sk_fds[i], errno); 626 err = setsockopt(sk_fds[i], SOL_SOCKET, SO_REUSEPORT, 627 &optval, sizeof(optval)); 628 RET_IF(err == -1, "setsockopt(SO_REUSEPORT)", 629 "sk_fds[%d] err:%d errno:%d\n", 630 i, err, errno); 631 632 if (i == first) { 633 err = setsockopt(sk_fds[i], SOL_SOCKET, 634 SO_ATTACH_REUSEPORT_EBPF, 635 &select_by_skb_data_prog, 636 sizeof(select_by_skb_data_prog)); 637 RET_IF(err == -1, "setsockopt(SO_ATTACH_REUEPORT_EBPF)", 638 "err:%d errno:%d\n", err, errno); 639 } 640 641 err = bind(sk_fds[i], (struct sockaddr *)&srv_sa, addrlen); 642 RET_IF(err == -1, "bind()", "sk_fds[%d] err:%d errno:%d\n", 643 i, err, errno); 644 645 if (type == SOCK_STREAM) { 646 err = listen(sk_fds[i], 10); 647 RET_IF(err == -1, "listen()", 648 "sk_fds[%d] err:%d errno:%d\n", 649 i, err, errno); 650 } 651 652 err = bpf_map_update_elem(reuseport_array, &i, &sk_fds[i], 653 BPF_NOEXIST); 654 RET_IF(err == -1, "update_elem(reuseport_array)", 655 "sk_fds[%d] err:%d errno:%d\n", i, err, errno); 656 657 if (i == first) { 658 socklen_t addrlen = sizeof(srv_sa); 659 660 err = getsockname(sk_fds[i], (struct sockaddr *)&srv_sa, 661 &addrlen); 662 RET_IF(err == -1, "getsockname()", 663 "sk_fds[%d] err:%d errno:%d\n", i, err, errno); 664 } 665 } 666 667 epfd = epoll_create(1); 668 RET_IF(epfd == -1, "epoll_create(1)", 669 "epfd:%d errno:%d\n", epfd, errno); 670 671 ev.events = EPOLLIN; 672 for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) { 673 ev.data.u32 = i; 674 err = epoll_ctl(epfd, EPOLL_CTL_ADD, sk_fds[i], &ev); 675 RET_IF(err, "epoll_ctl(EPOLL_CTL_ADD)", "sk_fds[%d]\n", i); 676 } 677 } 678 679 static void setup_per_test(int type, sa_family_t family, bool inany, 680 bool no_inner_map) 681 { 682 int ovr = -1, err; 683 684 prepare_sk_fds(type, family, inany); 685 err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, &ovr, 686 BPF_ANY); 687 RET_IF(err == -1, "update_elem(tmp_index_ovr_map, 0, -1)", 688 "err:%d errno:%d\n", err, errno); 689 690 /* Install reuseport_array to outer_map? */ 691 if (no_inner_map) 692 return; 693 694 err = bpf_map_update_elem(outer_map, &index_zero, &reuseport_array, 695 BPF_ANY); 696 RET_IF(err == -1, "update_elem(outer_map, 0, reuseport_array)", 697 "err:%d errno:%d\n", err, errno); 698 } 699 700 static void cleanup_per_test(bool no_inner_map) 701 { 702 int i, err, zero = 0; 703 704 memset(expected_results, 0, sizeof(expected_results)); 705 706 for (i = 0; i < NR_RESULTS; i++) { 707 err = bpf_map_update_elem(result_map, &i, &zero, BPF_ANY); 708 RET_IF(err, "reset elem in result_map", 709 "i:%u err:%d errno:%d\n", i, err, errno); 710 } 711 712 err = bpf_map_update_elem(linum_map, &zero, &zero, BPF_ANY); 713 RET_IF(err, "reset line number in linum_map", "err:%d errno:%d\n", 714 err, errno); 715 716 for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) 717 close(sk_fds[i]); 718 close(epfd); 719 720 /* Delete reuseport_array from outer_map? */ 721 if (no_inner_map) 722 return; 723 724 err = bpf_map_delete_elem(outer_map, &index_zero); 725 RET_IF(err == -1, "delete_elem(outer_map)", 726 "err:%d errno:%d\n", err, errno); 727 } 728 729 static void cleanup(void) 730 { 731 if (outer_map != -1) 732 close(outer_map); 733 if (reuseport_array != -1) 734 close(reuseport_array); 735 if (obj) 736 bpf_object__close(obj); 737 } 738 739 static const char *family_str(sa_family_t family) 740 { 741 switch (family) { 742 case AF_INET: 743 return "IPv4"; 744 case AF_INET6: 745 return "IPv6"; 746 default: 747 return "unknown"; 748 } 749 } 750 751 static const char *sotype_str(int sotype) 752 { 753 switch (sotype) { 754 case SOCK_STREAM: 755 return "TCP"; 756 case SOCK_DGRAM: 757 return "UDP"; 758 default: 759 return "unknown"; 760 } 761 } 762 763 #define TEST_INIT(fn, ...) { fn, #fn, __VA_ARGS__ } 764 765 static void test_config(int sotype, sa_family_t family, bool inany) 766 { 767 const struct test { 768 void (*fn)(int sotype, sa_family_t family); 769 const char *name; 770 bool no_inner_map; 771 } tests[] = { 772 TEST_INIT(test_err_inner_map, true /* no_inner_map */), 773 TEST_INIT(test_err_skb_data), 774 TEST_INIT(test_err_sk_select_port), 775 TEST_INIT(test_pass), 776 TEST_INIT(test_syncookie), 777 TEST_INIT(test_pass_on_err), 778 TEST_INIT(test_detach_bpf), 779 }; 780 char s[MAX_TEST_NAME]; 781 const struct test *t; 782 783 for (t = tests; t < tests + ARRAY_SIZE(tests); t++) { 784 snprintf(s, sizeof(s), "%s/%s %s %s", 785 family_str(family), sotype_str(sotype), 786 inany ? "INANY" : "LOOPBACK", t->name); 787 788 if (!test__start_subtest(s)) 789 continue; 790 791 setup_per_test(sotype, family, inany, t->no_inner_map); 792 t->fn(sotype, family); 793 cleanup_per_test(t->no_inner_map); 794 } 795 } 796 797 #define BIND_INANY true 798 799 static void test_all(void) 800 { 801 const struct config { 802 int sotype; 803 sa_family_t family; 804 bool inany; 805 } configs[] = { 806 { SOCK_STREAM, AF_INET }, 807 { SOCK_STREAM, AF_INET, BIND_INANY }, 808 { SOCK_STREAM, AF_INET6 }, 809 { SOCK_STREAM, AF_INET6, BIND_INANY }, 810 { SOCK_DGRAM, AF_INET }, 811 { SOCK_DGRAM, AF_INET6 }, 812 }; 813 const struct config *c; 814 815 for (c = configs; c < configs + ARRAY_SIZE(configs); c++) 816 test_config(c->sotype, c->family, c->inany); 817 } 818 819 void test_select_reuseport(void) 820 { 821 if (create_maps()) 822 goto out; 823 if (prepare_bpf_obj()) 824 goto out; 825 826 saved_tcp_fo = read_int_sysctl(TCP_FO_SYSCTL); 827 if (saved_tcp_fo < 0) 828 goto out; 829 saved_tcp_syncookie = read_int_sysctl(TCP_SYNCOOKIE_SYSCTL); 830 if (saved_tcp_syncookie < 0) 831 goto out; 832 833 if (enable_fastopen()) 834 goto out; 835 if (disable_syncookie()) 836 goto out; 837 838 test_all(); 839 out: 840 cleanup(); 841 restore_sysctls(); 842 } 843