1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018 Facebook */ 3 4 #include <stdlib.h> 5 #include <unistd.h> 6 #include <stdbool.h> 7 #include <string.h> 8 #include <errno.h> 9 #include <assert.h> 10 #include <fcntl.h> 11 #include <linux/bpf.h> 12 #include <linux/err.h> 13 #include <linux/types.h> 14 #include <linux/if_ether.h> 15 #include <sys/types.h> 16 #include <sys/epoll.h> 17 #include <sys/socket.h> 18 #include <netinet/in.h> 19 #include <bpf/bpf.h> 20 #include <bpf/libbpf.h> 21 #include "bpf_rlimit.h" 22 #include "bpf_util.h" 23 24 #include "test_progs.h" 25 #include "test_select_reuseport_common.h" 26 27 #define MAX_TEST_NAME 80 28 #define MIN_TCPHDR_LEN 20 29 #define UDPHDR_LEN 8 30 31 #define TCP_SYNCOOKIE_SYSCTL "/proc/sys/net/ipv4/tcp_syncookies" 32 #define TCP_FO_SYSCTL "/proc/sys/net/ipv4/tcp_fastopen" 33 #define REUSEPORT_ARRAY_SIZE 32 34 35 static int result_map, tmp_index_ovr_map, linum_map, data_check_map; 36 static __u32 expected_results[NR_RESULTS]; 37 static int sk_fds[REUSEPORT_ARRAY_SIZE]; 38 static int reuseport_array = -1, outer_map = -1; 39 static enum bpf_map_type inner_map_type; 40 static int select_by_skb_data_prog; 41 static int saved_tcp_syncookie = -1; 42 static struct bpf_object *obj; 43 static int saved_tcp_fo = -1; 44 static __u32 index_zero; 45 static int epfd; 46 47 static union sa46 { 48 struct sockaddr_in6 v6; 49 struct sockaddr_in v4; 50 sa_family_t family; 51 } srv_sa; 52 53 #define RET_IF(condition, tag, format...) ({ \ 54 if (CHECK_FAIL(condition)) { \ 55 printf(tag " " format); \ 56 return; \ 57 } \ 58 }) 59 60 #define RET_ERR(condition, tag, format...) ({ \ 61 if (CHECK_FAIL(condition)) { \ 62 printf(tag " " format); \ 63 return -1; \ 64 } \ 65 }) 66 67 static int create_maps(enum bpf_map_type inner_type) 68 { 69 LIBBPF_OPTS(bpf_map_create_opts, opts); 70 71 inner_map_type = inner_type; 72 73 /* Creating reuseport_array */ 74 reuseport_array = bpf_map_create(inner_type, "reuseport_array", 75 sizeof(__u32), sizeof(__u32), REUSEPORT_ARRAY_SIZE, NULL); 76 RET_ERR(reuseport_array < 0, "creating reuseport_array", 77 "reuseport_array:%d errno:%d\n", reuseport_array, errno); 78 79 /* Creating outer_map */ 80 opts.inner_map_fd = reuseport_array; 81 outer_map = bpf_map_create(BPF_MAP_TYPE_ARRAY_OF_MAPS, "outer_map", 82 sizeof(__u32), sizeof(__u32), 1, &opts); 83 RET_ERR(outer_map < 0, "creating outer_map", 84 "outer_map:%d errno:%d\n", outer_map, errno); 85 86 return 0; 87 } 88 89 static int prepare_bpf_obj(void) 90 { 91 struct bpf_program *prog; 92 struct bpf_map *map; 93 int err; 94 95 obj = bpf_object__open("test_select_reuseport_kern.o"); 96 err = libbpf_get_error(obj); 97 RET_ERR(err, "open test_select_reuseport_kern.o", 98 "obj:%p PTR_ERR(obj):%d\n", obj, err); 99 100 map = bpf_object__find_map_by_name(obj, "outer_map"); 101 RET_ERR(!map, "find outer_map", "!map\n"); 102 err = bpf_map__reuse_fd(map, outer_map); 103 RET_ERR(err, "reuse outer_map", "err:%d\n", err); 104 105 err = bpf_object__load(obj); 106 RET_ERR(err, "load bpf_object", "err:%d\n", err); 107 108 prog = bpf_object__next_program(obj, NULL); 109 RET_ERR(!prog, "get first bpf_program", "!prog\n"); 110 select_by_skb_data_prog = bpf_program__fd(prog); 111 RET_ERR(select_by_skb_data_prog < 0, "get prog fd", 112 "select_by_skb_data_prog:%d\n", select_by_skb_data_prog); 113 114 map = bpf_object__find_map_by_name(obj, "result_map"); 115 RET_ERR(!map, "find result_map", "!map\n"); 116 result_map = bpf_map__fd(map); 117 RET_ERR(result_map < 0, "get result_map fd", 118 "result_map:%d\n", result_map); 119 120 map = bpf_object__find_map_by_name(obj, "tmp_index_ovr_map"); 121 RET_ERR(!map, "find tmp_index_ovr_map\n", "!map"); 122 tmp_index_ovr_map = bpf_map__fd(map); 123 RET_ERR(tmp_index_ovr_map < 0, "get tmp_index_ovr_map fd", 124 "tmp_index_ovr_map:%d\n", tmp_index_ovr_map); 125 126 map = bpf_object__find_map_by_name(obj, "linum_map"); 127 RET_ERR(!map, "find linum_map", "!map\n"); 128 linum_map = bpf_map__fd(map); 129 RET_ERR(linum_map < 0, "get linum_map fd", 130 "linum_map:%d\n", linum_map); 131 132 map = bpf_object__find_map_by_name(obj, "data_check_map"); 133 RET_ERR(!map, "find data_check_map", "!map\n"); 134 data_check_map = bpf_map__fd(map); 135 RET_ERR(data_check_map < 0, "get data_check_map fd", 136 "data_check_map:%d\n", data_check_map); 137 138 return 0; 139 } 140 141 static void sa46_init_loopback(union sa46 *sa, sa_family_t family) 142 { 143 memset(sa, 0, sizeof(*sa)); 144 sa->family = family; 145 if (sa->family == AF_INET6) 146 sa->v6.sin6_addr = in6addr_loopback; 147 else 148 sa->v4.sin_addr.s_addr = htonl(INADDR_LOOPBACK); 149 } 150 151 static void sa46_init_inany(union sa46 *sa, sa_family_t family) 152 { 153 memset(sa, 0, sizeof(*sa)); 154 sa->family = family; 155 if (sa->family == AF_INET6) 156 sa->v6.sin6_addr = in6addr_any; 157 else 158 sa->v4.sin_addr.s_addr = INADDR_ANY; 159 } 160 161 static int read_int_sysctl(const char *sysctl) 162 { 163 char buf[16]; 164 int fd, ret; 165 166 fd = open(sysctl, 0); 167 RET_ERR(fd == -1, "open(sysctl)", 168 "sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno); 169 170 ret = read(fd, buf, sizeof(buf)); 171 RET_ERR(ret <= 0, "read(sysctl)", 172 "sysctl:%s ret:%d errno:%d\n", sysctl, ret, errno); 173 174 close(fd); 175 return atoi(buf); 176 } 177 178 static int write_int_sysctl(const char *sysctl, int v) 179 { 180 int fd, ret, size; 181 char buf[16]; 182 183 fd = open(sysctl, O_RDWR); 184 RET_ERR(fd == -1, "open(sysctl)", 185 "sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno); 186 187 size = snprintf(buf, sizeof(buf), "%d", v); 188 ret = write(fd, buf, size); 189 RET_ERR(ret != size, "write(sysctl)", 190 "sysctl:%s ret:%d size:%d errno:%d\n", 191 sysctl, ret, size, errno); 192 193 close(fd); 194 return 0; 195 } 196 197 static void restore_sysctls(void) 198 { 199 if (saved_tcp_fo != -1) 200 write_int_sysctl(TCP_FO_SYSCTL, saved_tcp_fo); 201 if (saved_tcp_syncookie != -1) 202 write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, saved_tcp_syncookie); 203 } 204 205 static int enable_fastopen(void) 206 { 207 int fo; 208 209 fo = read_int_sysctl(TCP_FO_SYSCTL); 210 if (fo < 0) 211 return -1; 212 213 return write_int_sysctl(TCP_FO_SYSCTL, fo | 7); 214 } 215 216 static int enable_syncookie(void) 217 { 218 return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 2); 219 } 220 221 static int disable_syncookie(void) 222 { 223 return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 0); 224 } 225 226 static long get_linum(void) 227 { 228 __u32 linum; 229 int err; 230 231 err = bpf_map_lookup_elem(linum_map, &index_zero, &linum); 232 RET_ERR(err < 0, "lookup_elem(linum_map)", "err:%d errno:%d\n", 233 err, errno); 234 235 return linum; 236 } 237 238 static void check_data(int type, sa_family_t family, const struct cmd *cmd, 239 int cli_fd) 240 { 241 struct data_check expected = {}, result; 242 union sa46 cli_sa; 243 socklen_t addrlen; 244 int err; 245 246 addrlen = sizeof(cli_sa); 247 err = getsockname(cli_fd, (struct sockaddr *)&cli_sa, 248 &addrlen); 249 RET_IF(err < 0, "getsockname(cli_fd)", "err:%d errno:%d\n", 250 err, errno); 251 252 err = bpf_map_lookup_elem(data_check_map, &index_zero, &result); 253 RET_IF(err < 0, "lookup_elem(data_check_map)", "err:%d errno:%d\n", 254 err, errno); 255 256 if (type == SOCK_STREAM) { 257 expected.len = MIN_TCPHDR_LEN; 258 expected.ip_protocol = IPPROTO_TCP; 259 } else { 260 expected.len = UDPHDR_LEN; 261 expected.ip_protocol = IPPROTO_UDP; 262 } 263 264 if (family == AF_INET6) { 265 expected.eth_protocol = htons(ETH_P_IPV6); 266 expected.bind_inany = !srv_sa.v6.sin6_addr.s6_addr32[3] && 267 !srv_sa.v6.sin6_addr.s6_addr32[2] && 268 !srv_sa.v6.sin6_addr.s6_addr32[1] && 269 !srv_sa.v6.sin6_addr.s6_addr32[0]; 270 271 memcpy(&expected.skb_addrs[0], cli_sa.v6.sin6_addr.s6_addr32, 272 sizeof(cli_sa.v6.sin6_addr)); 273 memcpy(&expected.skb_addrs[4], &in6addr_loopback, 274 sizeof(in6addr_loopback)); 275 expected.skb_ports[0] = cli_sa.v6.sin6_port; 276 expected.skb_ports[1] = srv_sa.v6.sin6_port; 277 } else { 278 expected.eth_protocol = htons(ETH_P_IP); 279 expected.bind_inany = !srv_sa.v4.sin_addr.s_addr; 280 281 expected.skb_addrs[0] = cli_sa.v4.sin_addr.s_addr; 282 expected.skb_addrs[1] = htonl(INADDR_LOOPBACK); 283 expected.skb_ports[0] = cli_sa.v4.sin_port; 284 expected.skb_ports[1] = srv_sa.v4.sin_port; 285 } 286 287 if (memcmp(&result, &expected, offsetof(struct data_check, 288 equal_check_end))) { 289 printf("unexpected data_check\n"); 290 printf(" result: (0x%x, %u, %u)\n", 291 result.eth_protocol, result.ip_protocol, 292 result.bind_inany); 293 printf("expected: (0x%x, %u, %u)\n", 294 expected.eth_protocol, expected.ip_protocol, 295 expected.bind_inany); 296 RET_IF(1, "data_check result != expected", 297 "bpf_prog_linum:%ld\n", get_linum()); 298 } 299 300 RET_IF(!result.hash, "data_check result.hash empty", 301 "result.hash:%u", result.hash); 302 303 expected.len += cmd ? sizeof(*cmd) : 0; 304 if (type == SOCK_STREAM) 305 RET_IF(expected.len > result.len, "expected.len > result.len", 306 "expected.len:%u result.len:%u bpf_prog_linum:%ld\n", 307 expected.len, result.len, get_linum()); 308 else 309 RET_IF(expected.len != result.len, "expected.len != result.len", 310 "expected.len:%u result.len:%u bpf_prog_linum:%ld\n", 311 expected.len, result.len, get_linum()); 312 } 313 314 static const char *result_to_str(enum result res) 315 { 316 switch (res) { 317 case DROP_ERR_INNER_MAP: 318 return "DROP_ERR_INNER_MAP"; 319 case DROP_ERR_SKB_DATA: 320 return "DROP_ERR_SKB_DATA"; 321 case DROP_ERR_SK_SELECT_REUSEPORT: 322 return "DROP_ERR_SK_SELECT_REUSEPORT"; 323 case DROP_MISC: 324 return "DROP_MISC"; 325 case PASS: 326 return "PASS"; 327 case PASS_ERR_SK_SELECT_REUSEPORT: 328 return "PASS_ERR_SK_SELECT_REUSEPORT"; 329 default: 330 return "UNKNOWN"; 331 } 332 } 333 334 static void check_results(void) 335 { 336 __u32 results[NR_RESULTS]; 337 __u32 i, broken = 0; 338 int err; 339 340 for (i = 0; i < NR_RESULTS; i++) { 341 err = bpf_map_lookup_elem(result_map, &i, &results[i]); 342 RET_IF(err < 0, "lookup_elem(result_map)", 343 "i:%u err:%d errno:%d\n", i, err, errno); 344 } 345 346 for (i = 0; i < NR_RESULTS; i++) { 347 if (results[i] != expected_results[i]) { 348 broken = i; 349 break; 350 } 351 } 352 353 if (i == NR_RESULTS) 354 return; 355 356 printf("unexpected result\n"); 357 printf(" result: ["); 358 printf("%u", results[0]); 359 for (i = 1; i < NR_RESULTS; i++) 360 printf(", %u", results[i]); 361 printf("]\n"); 362 363 printf("expected: ["); 364 printf("%u", expected_results[0]); 365 for (i = 1; i < NR_RESULTS; i++) 366 printf(", %u", expected_results[i]); 367 printf("]\n"); 368 369 printf("mismatch on %s (bpf_prog_linum:%ld)\n", result_to_str(broken), 370 get_linum()); 371 372 CHECK_FAIL(true); 373 } 374 375 static int send_data(int type, sa_family_t family, void *data, size_t len, 376 enum result expected) 377 { 378 union sa46 cli_sa; 379 int fd, err; 380 381 fd = socket(family, type, 0); 382 RET_ERR(fd == -1, "socket()", "fd:%d errno:%d\n", fd, errno); 383 384 sa46_init_loopback(&cli_sa, family); 385 err = bind(fd, (struct sockaddr *)&cli_sa, sizeof(cli_sa)); 386 RET_ERR(fd == -1, "bind(cli_sa)", "err:%d errno:%d\n", err, errno); 387 388 err = sendto(fd, data, len, MSG_FASTOPEN, (struct sockaddr *)&srv_sa, 389 sizeof(srv_sa)); 390 RET_ERR(err != len && expected >= PASS, 391 "sendto()", "family:%u err:%d errno:%d expected:%d\n", 392 family, err, errno, expected); 393 394 return fd; 395 } 396 397 static void do_test(int type, sa_family_t family, struct cmd *cmd, 398 enum result expected) 399 { 400 int nev, srv_fd, cli_fd; 401 struct epoll_event ev; 402 struct cmd rcv_cmd; 403 ssize_t nread; 404 405 cli_fd = send_data(type, family, cmd, cmd ? sizeof(*cmd) : 0, 406 expected); 407 if (cli_fd < 0) 408 return; 409 nev = epoll_wait(epfd, &ev, 1, expected >= PASS ? 5 : 0); 410 RET_IF((nev <= 0 && expected >= PASS) || 411 (nev > 0 && expected < PASS), 412 "nev <> expected", 413 "nev:%d expected:%d type:%d family:%d data:(%d, %d)\n", 414 nev, expected, type, family, 415 cmd ? cmd->reuseport_index : -1, 416 cmd ? cmd->pass_on_failure : -1); 417 check_results(); 418 check_data(type, family, cmd, cli_fd); 419 420 if (expected < PASS) 421 return; 422 423 RET_IF(expected != PASS_ERR_SK_SELECT_REUSEPORT && 424 cmd->reuseport_index != ev.data.u32, 425 "check cmd->reuseport_index", 426 "cmd:(%u, %u) ev.data.u32:%u\n", 427 cmd->pass_on_failure, cmd->reuseport_index, ev.data.u32); 428 429 srv_fd = sk_fds[ev.data.u32]; 430 if (type == SOCK_STREAM) { 431 int new_fd = accept(srv_fd, NULL, 0); 432 433 RET_IF(new_fd == -1, "accept(srv_fd)", 434 "ev.data.u32:%u new_fd:%d errno:%d\n", 435 ev.data.u32, new_fd, errno); 436 437 nread = recv(new_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT); 438 RET_IF(nread != sizeof(rcv_cmd), 439 "recv(new_fd)", 440 "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n", 441 ev.data.u32, nread, sizeof(rcv_cmd), errno); 442 443 close(new_fd); 444 } else { 445 nread = recv(srv_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT); 446 RET_IF(nread != sizeof(rcv_cmd), 447 "recv(sk_fds)", 448 "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n", 449 ev.data.u32, nread, sizeof(rcv_cmd), errno); 450 } 451 452 close(cli_fd); 453 } 454 455 static void test_err_inner_map(int type, sa_family_t family) 456 { 457 struct cmd cmd = { 458 .reuseport_index = 0, 459 .pass_on_failure = 0, 460 }; 461 462 expected_results[DROP_ERR_INNER_MAP]++; 463 do_test(type, family, &cmd, DROP_ERR_INNER_MAP); 464 } 465 466 static void test_err_skb_data(int type, sa_family_t family) 467 { 468 expected_results[DROP_ERR_SKB_DATA]++; 469 do_test(type, family, NULL, DROP_ERR_SKB_DATA); 470 } 471 472 static void test_err_sk_select_port(int type, sa_family_t family) 473 { 474 struct cmd cmd = { 475 .reuseport_index = REUSEPORT_ARRAY_SIZE, 476 .pass_on_failure = 0, 477 }; 478 479 expected_results[DROP_ERR_SK_SELECT_REUSEPORT]++; 480 do_test(type, family, &cmd, DROP_ERR_SK_SELECT_REUSEPORT); 481 } 482 483 static void test_pass(int type, sa_family_t family) 484 { 485 struct cmd cmd; 486 int i; 487 488 cmd.pass_on_failure = 0; 489 for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) { 490 expected_results[PASS]++; 491 cmd.reuseport_index = i; 492 do_test(type, family, &cmd, PASS); 493 } 494 } 495 496 static void test_syncookie(int type, sa_family_t family) 497 { 498 int err, tmp_index = 1; 499 struct cmd cmd = { 500 .reuseport_index = 0, 501 .pass_on_failure = 0, 502 }; 503 504 /* 505 * +1 for TCP-SYN and 506 * +1 for the TCP-ACK (ack the syncookie) 507 */ 508 expected_results[PASS] += 2; 509 enable_syncookie(); 510 /* 511 * Simulate TCP-SYN and TCP-ACK are handled by two different sk: 512 * TCP-SYN: select sk_fds[tmp_index = 1] tmp_index is from the 513 * tmp_index_ovr_map 514 * TCP-ACK: select sk_fds[reuseport_index = 0] reuseport_index 515 * is from the cmd.reuseport_index 516 */ 517 err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, 518 &tmp_index, BPF_ANY); 519 RET_IF(err < 0, "update_elem(tmp_index_ovr_map, 0, 1)", 520 "err:%d errno:%d\n", err, errno); 521 do_test(type, family, &cmd, PASS); 522 err = bpf_map_lookup_elem(tmp_index_ovr_map, &index_zero, 523 &tmp_index); 524 RET_IF(err < 0 || tmp_index >= 0, 525 "lookup_elem(tmp_index_ovr_map)", 526 "err:%d errno:%d tmp_index:%d\n", 527 err, errno, tmp_index); 528 disable_syncookie(); 529 } 530 531 static void test_pass_on_err(int type, sa_family_t family) 532 { 533 struct cmd cmd = { 534 .reuseport_index = REUSEPORT_ARRAY_SIZE, 535 .pass_on_failure = 1, 536 }; 537 538 expected_results[PASS_ERR_SK_SELECT_REUSEPORT] += 1; 539 do_test(type, family, &cmd, PASS_ERR_SK_SELECT_REUSEPORT); 540 } 541 542 static void test_detach_bpf(int type, sa_family_t family) 543 { 544 #ifdef SO_DETACH_REUSEPORT_BPF 545 __u32 nr_run_before = 0, nr_run_after = 0, tmp, i; 546 struct epoll_event ev; 547 int cli_fd, err, nev; 548 struct cmd cmd = {}; 549 int optvalue = 0; 550 551 err = setsockopt(sk_fds[0], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF, 552 &optvalue, sizeof(optvalue)); 553 RET_IF(err == -1, "setsockopt(SO_DETACH_REUSEPORT_BPF)", 554 "err:%d errno:%d\n", err, errno); 555 556 err = setsockopt(sk_fds[1], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF, 557 &optvalue, sizeof(optvalue)); 558 RET_IF(err == 0 || errno != ENOENT, 559 "setsockopt(SO_DETACH_REUSEPORT_BPF)", 560 "err:%d errno:%d\n", err, errno); 561 562 for (i = 0; i < NR_RESULTS; i++) { 563 err = bpf_map_lookup_elem(result_map, &i, &tmp); 564 RET_IF(err < 0, "lookup_elem(result_map)", 565 "i:%u err:%d errno:%d\n", i, err, errno); 566 nr_run_before += tmp; 567 } 568 569 cli_fd = send_data(type, family, &cmd, sizeof(cmd), PASS); 570 if (cli_fd < 0) 571 return; 572 nev = epoll_wait(epfd, &ev, 1, 5); 573 RET_IF(nev <= 0, "nev <= 0", 574 "nev:%d expected:1 type:%d family:%d data:(0, 0)\n", 575 nev, type, family); 576 577 for (i = 0; i < NR_RESULTS; i++) { 578 err = bpf_map_lookup_elem(result_map, &i, &tmp); 579 RET_IF(err < 0, "lookup_elem(result_map)", 580 "i:%u err:%d errno:%d\n", i, err, errno); 581 nr_run_after += tmp; 582 } 583 584 RET_IF(nr_run_before != nr_run_after, 585 "nr_run_before != nr_run_after", 586 "nr_run_before:%u nr_run_after:%u\n", 587 nr_run_before, nr_run_after); 588 589 close(cli_fd); 590 #else 591 test__skip(); 592 #endif 593 } 594 595 static void prepare_sk_fds(int type, sa_family_t family, bool inany) 596 { 597 const int first = REUSEPORT_ARRAY_SIZE - 1; 598 int i, err, optval = 1; 599 struct epoll_event ev; 600 socklen_t addrlen; 601 602 if (inany) 603 sa46_init_inany(&srv_sa, family); 604 else 605 sa46_init_loopback(&srv_sa, family); 606 addrlen = sizeof(srv_sa); 607 608 /* 609 * The sk_fds[] is filled from the back such that the order 610 * is exactly opposite to the (struct sock_reuseport *)reuse->socks[]. 611 */ 612 for (i = first; i >= 0; i--) { 613 sk_fds[i] = socket(family, type, 0); 614 RET_IF(sk_fds[i] == -1, "socket()", "sk_fds[%d]:%d errno:%d\n", 615 i, sk_fds[i], errno); 616 err = setsockopt(sk_fds[i], SOL_SOCKET, SO_REUSEPORT, 617 &optval, sizeof(optval)); 618 RET_IF(err == -1, "setsockopt(SO_REUSEPORT)", 619 "sk_fds[%d] err:%d errno:%d\n", 620 i, err, errno); 621 622 if (i == first) { 623 err = setsockopt(sk_fds[i], SOL_SOCKET, 624 SO_ATTACH_REUSEPORT_EBPF, 625 &select_by_skb_data_prog, 626 sizeof(select_by_skb_data_prog)); 627 RET_IF(err < 0, "setsockopt(SO_ATTACH_REUEPORT_EBPF)", 628 "err:%d errno:%d\n", err, errno); 629 } 630 631 err = bind(sk_fds[i], (struct sockaddr *)&srv_sa, addrlen); 632 RET_IF(err < 0, "bind()", "sk_fds[%d] err:%d errno:%d\n", 633 i, err, errno); 634 635 if (type == SOCK_STREAM) { 636 err = listen(sk_fds[i], 10); 637 RET_IF(err < 0, "listen()", 638 "sk_fds[%d] err:%d errno:%d\n", 639 i, err, errno); 640 } 641 642 err = bpf_map_update_elem(reuseport_array, &i, &sk_fds[i], 643 BPF_NOEXIST); 644 RET_IF(err < 0, "update_elem(reuseport_array)", 645 "sk_fds[%d] err:%d errno:%d\n", i, err, errno); 646 647 if (i == first) { 648 socklen_t addrlen = sizeof(srv_sa); 649 650 err = getsockname(sk_fds[i], (struct sockaddr *)&srv_sa, 651 &addrlen); 652 RET_IF(err == -1, "getsockname()", 653 "sk_fds[%d] err:%d errno:%d\n", i, err, errno); 654 } 655 } 656 657 epfd = epoll_create(1); 658 RET_IF(epfd == -1, "epoll_create(1)", 659 "epfd:%d errno:%d\n", epfd, errno); 660 661 ev.events = EPOLLIN; 662 for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) { 663 ev.data.u32 = i; 664 err = epoll_ctl(epfd, EPOLL_CTL_ADD, sk_fds[i], &ev); 665 RET_IF(err, "epoll_ctl(EPOLL_CTL_ADD)", "sk_fds[%d]\n", i); 666 } 667 } 668 669 static void setup_per_test(int type, sa_family_t family, bool inany, 670 bool no_inner_map) 671 { 672 int ovr = -1, err; 673 674 prepare_sk_fds(type, family, inany); 675 err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, &ovr, 676 BPF_ANY); 677 RET_IF(err < 0, "update_elem(tmp_index_ovr_map, 0, -1)", 678 "err:%d errno:%d\n", err, errno); 679 680 /* Install reuseport_array to outer_map? */ 681 if (no_inner_map) 682 return; 683 684 err = bpf_map_update_elem(outer_map, &index_zero, &reuseport_array, 685 BPF_ANY); 686 RET_IF(err < 0, "update_elem(outer_map, 0, reuseport_array)", 687 "err:%d errno:%d\n", err, errno); 688 } 689 690 static void cleanup_per_test(bool no_inner_map) 691 { 692 int i, err, zero = 0; 693 694 memset(expected_results, 0, sizeof(expected_results)); 695 696 for (i = 0; i < NR_RESULTS; i++) { 697 err = bpf_map_update_elem(result_map, &i, &zero, BPF_ANY); 698 RET_IF(err, "reset elem in result_map", 699 "i:%u err:%d errno:%d\n", i, err, errno); 700 } 701 702 err = bpf_map_update_elem(linum_map, &zero, &zero, BPF_ANY); 703 RET_IF(err, "reset line number in linum_map", "err:%d errno:%d\n", 704 err, errno); 705 706 for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) 707 close(sk_fds[i]); 708 close(epfd); 709 710 /* Delete reuseport_array from outer_map? */ 711 if (no_inner_map) 712 return; 713 714 err = bpf_map_delete_elem(outer_map, &index_zero); 715 RET_IF(err < 0, "delete_elem(outer_map)", 716 "err:%d errno:%d\n", err, errno); 717 } 718 719 static void cleanup(void) 720 { 721 if (outer_map >= 0) { 722 close(outer_map); 723 outer_map = -1; 724 } 725 726 if (reuseport_array >= 0) { 727 close(reuseport_array); 728 reuseport_array = -1; 729 } 730 731 if (obj) { 732 bpf_object__close(obj); 733 obj = NULL; 734 } 735 736 memset(expected_results, 0, sizeof(expected_results)); 737 } 738 739 static const char *maptype_str(enum bpf_map_type type) 740 { 741 switch (type) { 742 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: 743 return "reuseport_sockarray"; 744 case BPF_MAP_TYPE_SOCKMAP: 745 return "sockmap"; 746 case BPF_MAP_TYPE_SOCKHASH: 747 return "sockhash"; 748 default: 749 return "unknown"; 750 } 751 } 752 753 static const char *family_str(sa_family_t family) 754 { 755 switch (family) { 756 case AF_INET: 757 return "IPv4"; 758 case AF_INET6: 759 return "IPv6"; 760 default: 761 return "unknown"; 762 } 763 } 764 765 static const char *sotype_str(int sotype) 766 { 767 switch (sotype) { 768 case SOCK_STREAM: 769 return "TCP"; 770 case SOCK_DGRAM: 771 return "UDP"; 772 default: 773 return "unknown"; 774 } 775 } 776 777 #define TEST_INIT(fn_, ...) { .fn = fn_, .name = #fn_, __VA_ARGS__ } 778 779 static void test_config(int sotype, sa_family_t family, bool inany) 780 { 781 const struct test { 782 void (*fn)(int sotype, sa_family_t family); 783 const char *name; 784 bool no_inner_map; 785 int need_sotype; 786 } tests[] = { 787 TEST_INIT(test_err_inner_map, 788 .no_inner_map = true), 789 TEST_INIT(test_err_skb_data), 790 TEST_INIT(test_err_sk_select_port), 791 TEST_INIT(test_pass), 792 TEST_INIT(test_syncookie, 793 .need_sotype = SOCK_STREAM), 794 TEST_INIT(test_pass_on_err), 795 TEST_INIT(test_detach_bpf), 796 }; 797 char s[MAX_TEST_NAME]; 798 const struct test *t; 799 800 for (t = tests; t < tests + ARRAY_SIZE(tests); t++) { 801 if (t->need_sotype && t->need_sotype != sotype) 802 continue; /* test not compatible with socket type */ 803 804 snprintf(s, sizeof(s), "%s %s/%s %s %s", 805 maptype_str(inner_map_type), 806 family_str(family), sotype_str(sotype), 807 inany ? "INANY" : "LOOPBACK", t->name); 808 809 if (!test__start_subtest(s)) 810 continue; 811 812 setup_per_test(sotype, family, inany, t->no_inner_map); 813 t->fn(sotype, family); 814 cleanup_per_test(t->no_inner_map); 815 } 816 } 817 818 #define BIND_INANY true 819 820 static void test_all(void) 821 { 822 const struct config { 823 int sotype; 824 sa_family_t family; 825 bool inany; 826 } configs[] = { 827 { SOCK_STREAM, AF_INET }, 828 { SOCK_STREAM, AF_INET, BIND_INANY }, 829 { SOCK_STREAM, AF_INET6 }, 830 { SOCK_STREAM, AF_INET6, BIND_INANY }, 831 { SOCK_DGRAM, AF_INET }, 832 { SOCK_DGRAM, AF_INET6 }, 833 }; 834 const struct config *c; 835 836 for (c = configs; c < configs + ARRAY_SIZE(configs); c++) 837 test_config(c->sotype, c->family, c->inany); 838 } 839 840 void test_map_type(enum bpf_map_type mt) 841 { 842 if (create_maps(mt)) 843 goto out; 844 if (prepare_bpf_obj()) 845 goto out; 846 847 test_all(); 848 out: 849 cleanup(); 850 } 851 852 void serial_test_select_reuseport(void) 853 { 854 saved_tcp_fo = read_int_sysctl(TCP_FO_SYSCTL); 855 if (saved_tcp_fo < 0) 856 goto out; 857 saved_tcp_syncookie = read_int_sysctl(TCP_SYNCOOKIE_SYSCTL); 858 if (saved_tcp_syncookie < 0) 859 goto out; 860 861 if (enable_fastopen()) 862 goto out; 863 if (disable_syncookie()) 864 goto out; 865 866 test_map_type(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY); 867 test_map_type(BPF_MAP_TYPE_SOCKMAP); 868 test_map_type(BPF_MAP_TYPE_SOCKHASH); 869 out: 870 restore_sysctls(); 871 } 872