1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018 Facebook */ 3 4 #include <stdlib.h> 5 #include <unistd.h> 6 #include <stdbool.h> 7 #include <string.h> 8 #include <errno.h> 9 #include <assert.h> 10 #include <fcntl.h> 11 #include <linux/bpf.h> 12 #include <linux/err.h> 13 #include <linux/types.h> 14 #include <linux/if_ether.h> 15 #include <sys/types.h> 16 #include <sys/epoll.h> 17 #include <sys/socket.h> 18 #include <netinet/in.h> 19 #include <bpf/bpf.h> 20 #include <bpf/libbpf.h> 21 #include "bpf_util.h" 22 23 #include "test_progs.h" 24 #include "test_select_reuseport_common.h" 25 26 #define MAX_TEST_NAME 80 27 #define MIN_TCPHDR_LEN 20 28 #define UDPHDR_LEN 8 29 30 #define TCP_SYNCOOKIE_SYSCTL "/proc/sys/net/ipv4/tcp_syncookies" 31 #define TCP_FO_SYSCTL "/proc/sys/net/ipv4/tcp_fastopen" 32 #define REUSEPORT_ARRAY_SIZE 32 33 34 static int result_map, tmp_index_ovr_map, linum_map, data_check_map; 35 static __u32 expected_results[NR_RESULTS]; 36 static int sk_fds[REUSEPORT_ARRAY_SIZE]; 37 static int reuseport_array = -1, outer_map = -1; 38 static enum bpf_map_type inner_map_type; 39 static int select_by_skb_data_prog; 40 static int saved_tcp_syncookie = -1; 41 static struct bpf_object *obj; 42 static int saved_tcp_fo = -1; 43 static __u32 index_zero; 44 static int epfd; 45 46 static union sa46 { 47 struct sockaddr_in6 v6; 48 struct sockaddr_in v4; 49 sa_family_t family; 50 } srv_sa; 51 52 #define RET_IF(condition, tag, format...) ({ \ 53 if (CHECK_FAIL(condition)) { \ 54 printf(tag " " format); \ 55 return; \ 56 } \ 57 }) 58 59 #define RET_ERR(condition, tag, format...) ({ \ 60 if (CHECK_FAIL(condition)) { \ 61 printf(tag " " format); \ 62 return -1; \ 63 } \ 64 }) 65 66 static int create_maps(enum bpf_map_type inner_type) 67 { 68 LIBBPF_OPTS(bpf_map_create_opts, opts); 69 70 inner_map_type = inner_type; 71 72 /* Creating reuseport_array */ 73 reuseport_array = bpf_map_create(inner_type, "reuseport_array", 74 sizeof(__u32), sizeof(__u32), REUSEPORT_ARRAY_SIZE, NULL); 75 RET_ERR(reuseport_array < 0, "creating reuseport_array", 76 "reuseport_array:%d errno:%d\n", reuseport_array, errno); 77 78 /* Creating outer_map */ 79 opts.inner_map_fd = reuseport_array; 80 outer_map = bpf_map_create(BPF_MAP_TYPE_ARRAY_OF_MAPS, "outer_map", 81 sizeof(__u32), sizeof(__u32), 1, &opts); 82 RET_ERR(outer_map < 0, "creating outer_map", 83 "outer_map:%d errno:%d\n", outer_map, errno); 84 85 return 0; 86 } 87 88 static int prepare_bpf_obj(void) 89 { 90 struct bpf_program *prog; 91 struct bpf_map *map; 92 int err; 93 94 obj = bpf_object__open("test_select_reuseport_kern.o"); 95 err = libbpf_get_error(obj); 96 RET_ERR(err, "open test_select_reuseport_kern.o", 97 "obj:%p PTR_ERR(obj):%d\n", obj, err); 98 99 map = bpf_object__find_map_by_name(obj, "outer_map"); 100 RET_ERR(!map, "find outer_map", "!map\n"); 101 err = bpf_map__reuse_fd(map, outer_map); 102 RET_ERR(err, "reuse outer_map", "err:%d\n", err); 103 104 err = bpf_object__load(obj); 105 RET_ERR(err, "load bpf_object", "err:%d\n", err); 106 107 prog = bpf_object__next_program(obj, NULL); 108 RET_ERR(!prog, "get first bpf_program", "!prog\n"); 109 select_by_skb_data_prog = bpf_program__fd(prog); 110 RET_ERR(select_by_skb_data_prog < 0, "get prog fd", 111 "select_by_skb_data_prog:%d\n", select_by_skb_data_prog); 112 113 map = bpf_object__find_map_by_name(obj, "result_map"); 114 RET_ERR(!map, "find result_map", "!map\n"); 115 result_map = bpf_map__fd(map); 116 RET_ERR(result_map < 0, "get result_map fd", 117 "result_map:%d\n", result_map); 118 119 map = bpf_object__find_map_by_name(obj, "tmp_index_ovr_map"); 120 RET_ERR(!map, "find tmp_index_ovr_map\n", "!map"); 121 tmp_index_ovr_map = bpf_map__fd(map); 122 RET_ERR(tmp_index_ovr_map < 0, "get tmp_index_ovr_map fd", 123 "tmp_index_ovr_map:%d\n", tmp_index_ovr_map); 124 125 map = bpf_object__find_map_by_name(obj, "linum_map"); 126 RET_ERR(!map, "find linum_map", "!map\n"); 127 linum_map = bpf_map__fd(map); 128 RET_ERR(linum_map < 0, "get linum_map fd", 129 "linum_map:%d\n", linum_map); 130 131 map = bpf_object__find_map_by_name(obj, "data_check_map"); 132 RET_ERR(!map, "find data_check_map", "!map\n"); 133 data_check_map = bpf_map__fd(map); 134 RET_ERR(data_check_map < 0, "get data_check_map fd", 135 "data_check_map:%d\n", data_check_map); 136 137 return 0; 138 } 139 140 static void sa46_init_loopback(union sa46 *sa, sa_family_t family) 141 { 142 memset(sa, 0, sizeof(*sa)); 143 sa->family = family; 144 if (sa->family == AF_INET6) 145 sa->v6.sin6_addr = in6addr_loopback; 146 else 147 sa->v4.sin_addr.s_addr = htonl(INADDR_LOOPBACK); 148 } 149 150 static void sa46_init_inany(union sa46 *sa, sa_family_t family) 151 { 152 memset(sa, 0, sizeof(*sa)); 153 sa->family = family; 154 if (sa->family == AF_INET6) 155 sa->v6.sin6_addr = in6addr_any; 156 else 157 sa->v4.sin_addr.s_addr = INADDR_ANY; 158 } 159 160 static int read_int_sysctl(const char *sysctl) 161 { 162 char buf[16]; 163 int fd, ret; 164 165 fd = open(sysctl, 0); 166 RET_ERR(fd == -1, "open(sysctl)", 167 "sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno); 168 169 ret = read(fd, buf, sizeof(buf)); 170 RET_ERR(ret <= 0, "read(sysctl)", 171 "sysctl:%s ret:%d errno:%d\n", sysctl, ret, errno); 172 173 close(fd); 174 return atoi(buf); 175 } 176 177 static int write_int_sysctl(const char *sysctl, int v) 178 { 179 int fd, ret, size; 180 char buf[16]; 181 182 fd = open(sysctl, O_RDWR); 183 RET_ERR(fd == -1, "open(sysctl)", 184 "sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno); 185 186 size = snprintf(buf, sizeof(buf), "%d", v); 187 ret = write(fd, buf, size); 188 RET_ERR(ret != size, "write(sysctl)", 189 "sysctl:%s ret:%d size:%d errno:%d\n", 190 sysctl, ret, size, errno); 191 192 close(fd); 193 return 0; 194 } 195 196 static void restore_sysctls(void) 197 { 198 if (saved_tcp_fo != -1) 199 write_int_sysctl(TCP_FO_SYSCTL, saved_tcp_fo); 200 if (saved_tcp_syncookie != -1) 201 write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, saved_tcp_syncookie); 202 } 203 204 static int enable_fastopen(void) 205 { 206 int fo; 207 208 fo = read_int_sysctl(TCP_FO_SYSCTL); 209 if (fo < 0) 210 return -1; 211 212 return write_int_sysctl(TCP_FO_SYSCTL, fo | 7); 213 } 214 215 static int enable_syncookie(void) 216 { 217 return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 2); 218 } 219 220 static int disable_syncookie(void) 221 { 222 return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 0); 223 } 224 225 static long get_linum(void) 226 { 227 __u32 linum; 228 int err; 229 230 err = bpf_map_lookup_elem(linum_map, &index_zero, &linum); 231 RET_ERR(err < 0, "lookup_elem(linum_map)", "err:%d errno:%d\n", 232 err, errno); 233 234 return linum; 235 } 236 237 static void check_data(int type, sa_family_t family, const struct cmd *cmd, 238 int cli_fd) 239 { 240 struct data_check expected = {}, result; 241 union sa46 cli_sa; 242 socklen_t addrlen; 243 int err; 244 245 addrlen = sizeof(cli_sa); 246 err = getsockname(cli_fd, (struct sockaddr *)&cli_sa, 247 &addrlen); 248 RET_IF(err < 0, "getsockname(cli_fd)", "err:%d errno:%d\n", 249 err, errno); 250 251 err = bpf_map_lookup_elem(data_check_map, &index_zero, &result); 252 RET_IF(err < 0, "lookup_elem(data_check_map)", "err:%d errno:%d\n", 253 err, errno); 254 255 if (type == SOCK_STREAM) { 256 expected.len = MIN_TCPHDR_LEN; 257 expected.ip_protocol = IPPROTO_TCP; 258 } else { 259 expected.len = UDPHDR_LEN; 260 expected.ip_protocol = IPPROTO_UDP; 261 } 262 263 if (family == AF_INET6) { 264 expected.eth_protocol = htons(ETH_P_IPV6); 265 expected.bind_inany = !srv_sa.v6.sin6_addr.s6_addr32[3] && 266 !srv_sa.v6.sin6_addr.s6_addr32[2] && 267 !srv_sa.v6.sin6_addr.s6_addr32[1] && 268 !srv_sa.v6.sin6_addr.s6_addr32[0]; 269 270 memcpy(&expected.skb_addrs[0], cli_sa.v6.sin6_addr.s6_addr32, 271 sizeof(cli_sa.v6.sin6_addr)); 272 memcpy(&expected.skb_addrs[4], &in6addr_loopback, 273 sizeof(in6addr_loopback)); 274 expected.skb_ports[0] = cli_sa.v6.sin6_port; 275 expected.skb_ports[1] = srv_sa.v6.sin6_port; 276 } else { 277 expected.eth_protocol = htons(ETH_P_IP); 278 expected.bind_inany = !srv_sa.v4.sin_addr.s_addr; 279 280 expected.skb_addrs[0] = cli_sa.v4.sin_addr.s_addr; 281 expected.skb_addrs[1] = htonl(INADDR_LOOPBACK); 282 expected.skb_ports[0] = cli_sa.v4.sin_port; 283 expected.skb_ports[1] = srv_sa.v4.sin_port; 284 } 285 286 if (memcmp(&result, &expected, offsetof(struct data_check, 287 equal_check_end))) { 288 printf("unexpected data_check\n"); 289 printf(" result: (0x%x, %u, %u)\n", 290 result.eth_protocol, result.ip_protocol, 291 result.bind_inany); 292 printf("expected: (0x%x, %u, %u)\n", 293 expected.eth_protocol, expected.ip_protocol, 294 expected.bind_inany); 295 RET_IF(1, "data_check result != expected", 296 "bpf_prog_linum:%ld\n", get_linum()); 297 } 298 299 RET_IF(!result.hash, "data_check result.hash empty", 300 "result.hash:%u", result.hash); 301 302 expected.len += cmd ? sizeof(*cmd) : 0; 303 if (type == SOCK_STREAM) 304 RET_IF(expected.len > result.len, "expected.len > result.len", 305 "expected.len:%u result.len:%u bpf_prog_linum:%ld\n", 306 expected.len, result.len, get_linum()); 307 else 308 RET_IF(expected.len != result.len, "expected.len != result.len", 309 "expected.len:%u result.len:%u bpf_prog_linum:%ld\n", 310 expected.len, result.len, get_linum()); 311 } 312 313 static const char *result_to_str(enum result res) 314 { 315 switch (res) { 316 case DROP_ERR_INNER_MAP: 317 return "DROP_ERR_INNER_MAP"; 318 case DROP_ERR_SKB_DATA: 319 return "DROP_ERR_SKB_DATA"; 320 case DROP_ERR_SK_SELECT_REUSEPORT: 321 return "DROP_ERR_SK_SELECT_REUSEPORT"; 322 case DROP_MISC: 323 return "DROP_MISC"; 324 case PASS: 325 return "PASS"; 326 case PASS_ERR_SK_SELECT_REUSEPORT: 327 return "PASS_ERR_SK_SELECT_REUSEPORT"; 328 default: 329 return "UNKNOWN"; 330 } 331 } 332 333 static void check_results(void) 334 { 335 __u32 results[NR_RESULTS]; 336 __u32 i, broken = 0; 337 int err; 338 339 for (i = 0; i < NR_RESULTS; i++) { 340 err = bpf_map_lookup_elem(result_map, &i, &results[i]); 341 RET_IF(err < 0, "lookup_elem(result_map)", 342 "i:%u err:%d errno:%d\n", i, err, errno); 343 } 344 345 for (i = 0; i < NR_RESULTS; i++) { 346 if (results[i] != expected_results[i]) { 347 broken = i; 348 break; 349 } 350 } 351 352 if (i == NR_RESULTS) 353 return; 354 355 printf("unexpected result\n"); 356 printf(" result: ["); 357 printf("%u", results[0]); 358 for (i = 1; i < NR_RESULTS; i++) 359 printf(", %u", results[i]); 360 printf("]\n"); 361 362 printf("expected: ["); 363 printf("%u", expected_results[0]); 364 for (i = 1; i < NR_RESULTS; i++) 365 printf(", %u", expected_results[i]); 366 printf("]\n"); 367 368 printf("mismatch on %s (bpf_prog_linum:%ld)\n", result_to_str(broken), 369 get_linum()); 370 371 CHECK_FAIL(true); 372 } 373 374 static int send_data(int type, sa_family_t family, void *data, size_t len, 375 enum result expected) 376 { 377 union sa46 cli_sa; 378 int fd, err; 379 380 fd = socket(family, type, 0); 381 RET_ERR(fd == -1, "socket()", "fd:%d errno:%d\n", fd, errno); 382 383 sa46_init_loopback(&cli_sa, family); 384 err = bind(fd, (struct sockaddr *)&cli_sa, sizeof(cli_sa)); 385 RET_ERR(fd == -1, "bind(cli_sa)", "err:%d errno:%d\n", err, errno); 386 387 err = sendto(fd, data, len, MSG_FASTOPEN, (struct sockaddr *)&srv_sa, 388 sizeof(srv_sa)); 389 RET_ERR(err != len && expected >= PASS, 390 "sendto()", "family:%u err:%d errno:%d expected:%d\n", 391 family, err, errno, expected); 392 393 return fd; 394 } 395 396 static void do_test(int type, sa_family_t family, struct cmd *cmd, 397 enum result expected) 398 { 399 int nev, srv_fd, cli_fd; 400 struct epoll_event ev; 401 struct cmd rcv_cmd; 402 ssize_t nread; 403 404 cli_fd = send_data(type, family, cmd, cmd ? sizeof(*cmd) : 0, 405 expected); 406 if (cli_fd < 0) 407 return; 408 nev = epoll_wait(epfd, &ev, 1, expected >= PASS ? 5 : 0); 409 RET_IF((nev <= 0 && expected >= PASS) || 410 (nev > 0 && expected < PASS), 411 "nev <> expected", 412 "nev:%d expected:%d type:%d family:%d data:(%d, %d)\n", 413 nev, expected, type, family, 414 cmd ? cmd->reuseport_index : -1, 415 cmd ? cmd->pass_on_failure : -1); 416 check_results(); 417 check_data(type, family, cmd, cli_fd); 418 419 if (expected < PASS) 420 return; 421 422 RET_IF(expected != PASS_ERR_SK_SELECT_REUSEPORT && 423 cmd->reuseport_index != ev.data.u32, 424 "check cmd->reuseport_index", 425 "cmd:(%u, %u) ev.data.u32:%u\n", 426 cmd->pass_on_failure, cmd->reuseport_index, ev.data.u32); 427 428 srv_fd = sk_fds[ev.data.u32]; 429 if (type == SOCK_STREAM) { 430 int new_fd = accept(srv_fd, NULL, 0); 431 432 RET_IF(new_fd == -1, "accept(srv_fd)", 433 "ev.data.u32:%u new_fd:%d errno:%d\n", 434 ev.data.u32, new_fd, errno); 435 436 nread = recv(new_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT); 437 RET_IF(nread != sizeof(rcv_cmd), 438 "recv(new_fd)", 439 "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n", 440 ev.data.u32, nread, sizeof(rcv_cmd), errno); 441 442 close(new_fd); 443 } else { 444 nread = recv(srv_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT); 445 RET_IF(nread != sizeof(rcv_cmd), 446 "recv(sk_fds)", 447 "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n", 448 ev.data.u32, nread, sizeof(rcv_cmd), errno); 449 } 450 451 close(cli_fd); 452 } 453 454 static void test_err_inner_map(int type, sa_family_t family) 455 { 456 struct cmd cmd = { 457 .reuseport_index = 0, 458 .pass_on_failure = 0, 459 }; 460 461 expected_results[DROP_ERR_INNER_MAP]++; 462 do_test(type, family, &cmd, DROP_ERR_INNER_MAP); 463 } 464 465 static void test_err_skb_data(int type, sa_family_t family) 466 { 467 expected_results[DROP_ERR_SKB_DATA]++; 468 do_test(type, family, NULL, DROP_ERR_SKB_DATA); 469 } 470 471 static void test_err_sk_select_port(int type, sa_family_t family) 472 { 473 struct cmd cmd = { 474 .reuseport_index = REUSEPORT_ARRAY_SIZE, 475 .pass_on_failure = 0, 476 }; 477 478 expected_results[DROP_ERR_SK_SELECT_REUSEPORT]++; 479 do_test(type, family, &cmd, DROP_ERR_SK_SELECT_REUSEPORT); 480 } 481 482 static void test_pass(int type, sa_family_t family) 483 { 484 struct cmd cmd; 485 int i; 486 487 cmd.pass_on_failure = 0; 488 for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) { 489 expected_results[PASS]++; 490 cmd.reuseport_index = i; 491 do_test(type, family, &cmd, PASS); 492 } 493 } 494 495 static void test_syncookie(int type, sa_family_t family) 496 { 497 int err, tmp_index = 1; 498 struct cmd cmd = { 499 .reuseport_index = 0, 500 .pass_on_failure = 0, 501 }; 502 503 /* 504 * +1 for TCP-SYN and 505 * +1 for the TCP-ACK (ack the syncookie) 506 */ 507 expected_results[PASS] += 2; 508 enable_syncookie(); 509 /* 510 * Simulate TCP-SYN and TCP-ACK are handled by two different sk: 511 * TCP-SYN: select sk_fds[tmp_index = 1] tmp_index is from the 512 * tmp_index_ovr_map 513 * TCP-ACK: select sk_fds[reuseport_index = 0] reuseport_index 514 * is from the cmd.reuseport_index 515 */ 516 err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, 517 &tmp_index, BPF_ANY); 518 RET_IF(err < 0, "update_elem(tmp_index_ovr_map, 0, 1)", 519 "err:%d errno:%d\n", err, errno); 520 do_test(type, family, &cmd, PASS); 521 err = bpf_map_lookup_elem(tmp_index_ovr_map, &index_zero, 522 &tmp_index); 523 RET_IF(err < 0 || tmp_index >= 0, 524 "lookup_elem(tmp_index_ovr_map)", 525 "err:%d errno:%d tmp_index:%d\n", 526 err, errno, tmp_index); 527 disable_syncookie(); 528 } 529 530 static void test_pass_on_err(int type, sa_family_t family) 531 { 532 struct cmd cmd = { 533 .reuseport_index = REUSEPORT_ARRAY_SIZE, 534 .pass_on_failure = 1, 535 }; 536 537 expected_results[PASS_ERR_SK_SELECT_REUSEPORT] += 1; 538 do_test(type, family, &cmd, PASS_ERR_SK_SELECT_REUSEPORT); 539 } 540 541 static void test_detach_bpf(int type, sa_family_t family) 542 { 543 #ifdef SO_DETACH_REUSEPORT_BPF 544 __u32 nr_run_before = 0, nr_run_after = 0, tmp, i; 545 struct epoll_event ev; 546 int cli_fd, err, nev; 547 struct cmd cmd = {}; 548 int optvalue = 0; 549 550 err = setsockopt(sk_fds[0], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF, 551 &optvalue, sizeof(optvalue)); 552 RET_IF(err == -1, "setsockopt(SO_DETACH_REUSEPORT_BPF)", 553 "err:%d errno:%d\n", err, errno); 554 555 err = setsockopt(sk_fds[1], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF, 556 &optvalue, sizeof(optvalue)); 557 RET_IF(err == 0 || errno != ENOENT, 558 "setsockopt(SO_DETACH_REUSEPORT_BPF)", 559 "err:%d errno:%d\n", err, errno); 560 561 for (i = 0; i < NR_RESULTS; i++) { 562 err = bpf_map_lookup_elem(result_map, &i, &tmp); 563 RET_IF(err < 0, "lookup_elem(result_map)", 564 "i:%u err:%d errno:%d\n", i, err, errno); 565 nr_run_before += tmp; 566 } 567 568 cli_fd = send_data(type, family, &cmd, sizeof(cmd), PASS); 569 if (cli_fd < 0) 570 return; 571 nev = epoll_wait(epfd, &ev, 1, 5); 572 RET_IF(nev <= 0, "nev <= 0", 573 "nev:%d expected:1 type:%d family:%d data:(0, 0)\n", 574 nev, type, family); 575 576 for (i = 0; i < NR_RESULTS; i++) { 577 err = bpf_map_lookup_elem(result_map, &i, &tmp); 578 RET_IF(err < 0, "lookup_elem(result_map)", 579 "i:%u err:%d errno:%d\n", i, err, errno); 580 nr_run_after += tmp; 581 } 582 583 RET_IF(nr_run_before != nr_run_after, 584 "nr_run_before != nr_run_after", 585 "nr_run_before:%u nr_run_after:%u\n", 586 nr_run_before, nr_run_after); 587 588 close(cli_fd); 589 #else 590 test__skip(); 591 #endif 592 } 593 594 static void prepare_sk_fds(int type, sa_family_t family, bool inany) 595 { 596 const int first = REUSEPORT_ARRAY_SIZE - 1; 597 int i, err, optval = 1; 598 struct epoll_event ev; 599 socklen_t addrlen; 600 601 if (inany) 602 sa46_init_inany(&srv_sa, family); 603 else 604 sa46_init_loopback(&srv_sa, family); 605 addrlen = sizeof(srv_sa); 606 607 /* 608 * The sk_fds[] is filled from the back such that the order 609 * is exactly opposite to the (struct sock_reuseport *)reuse->socks[]. 610 */ 611 for (i = first; i >= 0; i--) { 612 sk_fds[i] = socket(family, type, 0); 613 RET_IF(sk_fds[i] == -1, "socket()", "sk_fds[%d]:%d errno:%d\n", 614 i, sk_fds[i], errno); 615 err = setsockopt(sk_fds[i], SOL_SOCKET, SO_REUSEPORT, 616 &optval, sizeof(optval)); 617 RET_IF(err == -1, "setsockopt(SO_REUSEPORT)", 618 "sk_fds[%d] err:%d errno:%d\n", 619 i, err, errno); 620 621 if (i == first) { 622 err = setsockopt(sk_fds[i], SOL_SOCKET, 623 SO_ATTACH_REUSEPORT_EBPF, 624 &select_by_skb_data_prog, 625 sizeof(select_by_skb_data_prog)); 626 RET_IF(err < 0, "setsockopt(SO_ATTACH_REUEPORT_EBPF)", 627 "err:%d errno:%d\n", err, errno); 628 } 629 630 err = bind(sk_fds[i], (struct sockaddr *)&srv_sa, addrlen); 631 RET_IF(err < 0, "bind()", "sk_fds[%d] err:%d errno:%d\n", 632 i, err, errno); 633 634 if (type == SOCK_STREAM) { 635 err = listen(sk_fds[i], 10); 636 RET_IF(err < 0, "listen()", 637 "sk_fds[%d] err:%d errno:%d\n", 638 i, err, errno); 639 } 640 641 err = bpf_map_update_elem(reuseport_array, &i, &sk_fds[i], 642 BPF_NOEXIST); 643 RET_IF(err < 0, "update_elem(reuseport_array)", 644 "sk_fds[%d] err:%d errno:%d\n", i, err, errno); 645 646 if (i == first) { 647 socklen_t addrlen = sizeof(srv_sa); 648 649 err = getsockname(sk_fds[i], (struct sockaddr *)&srv_sa, 650 &addrlen); 651 RET_IF(err == -1, "getsockname()", 652 "sk_fds[%d] err:%d errno:%d\n", i, err, errno); 653 } 654 } 655 656 epfd = epoll_create(1); 657 RET_IF(epfd == -1, "epoll_create(1)", 658 "epfd:%d errno:%d\n", epfd, errno); 659 660 ev.events = EPOLLIN; 661 for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) { 662 ev.data.u32 = i; 663 err = epoll_ctl(epfd, EPOLL_CTL_ADD, sk_fds[i], &ev); 664 RET_IF(err, "epoll_ctl(EPOLL_CTL_ADD)", "sk_fds[%d]\n", i); 665 } 666 } 667 668 static void setup_per_test(int type, sa_family_t family, bool inany, 669 bool no_inner_map) 670 { 671 int ovr = -1, err; 672 673 prepare_sk_fds(type, family, inany); 674 err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, &ovr, 675 BPF_ANY); 676 RET_IF(err < 0, "update_elem(tmp_index_ovr_map, 0, -1)", 677 "err:%d errno:%d\n", err, errno); 678 679 /* Install reuseport_array to outer_map? */ 680 if (no_inner_map) 681 return; 682 683 err = bpf_map_update_elem(outer_map, &index_zero, &reuseport_array, 684 BPF_ANY); 685 RET_IF(err < 0, "update_elem(outer_map, 0, reuseport_array)", 686 "err:%d errno:%d\n", err, errno); 687 } 688 689 static void cleanup_per_test(bool no_inner_map) 690 { 691 int i, err, zero = 0; 692 693 memset(expected_results, 0, sizeof(expected_results)); 694 695 for (i = 0; i < NR_RESULTS; i++) { 696 err = bpf_map_update_elem(result_map, &i, &zero, BPF_ANY); 697 RET_IF(err, "reset elem in result_map", 698 "i:%u err:%d errno:%d\n", i, err, errno); 699 } 700 701 err = bpf_map_update_elem(linum_map, &zero, &zero, BPF_ANY); 702 RET_IF(err, "reset line number in linum_map", "err:%d errno:%d\n", 703 err, errno); 704 705 for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) 706 close(sk_fds[i]); 707 close(epfd); 708 709 /* Delete reuseport_array from outer_map? */ 710 if (no_inner_map) 711 return; 712 713 err = bpf_map_delete_elem(outer_map, &index_zero); 714 RET_IF(err < 0, "delete_elem(outer_map)", 715 "err:%d errno:%d\n", err, errno); 716 } 717 718 static void cleanup(void) 719 { 720 if (outer_map >= 0) { 721 close(outer_map); 722 outer_map = -1; 723 } 724 725 if (reuseport_array >= 0) { 726 close(reuseport_array); 727 reuseport_array = -1; 728 } 729 730 if (obj) { 731 bpf_object__close(obj); 732 obj = NULL; 733 } 734 735 memset(expected_results, 0, sizeof(expected_results)); 736 } 737 738 static const char *maptype_str(enum bpf_map_type type) 739 { 740 switch (type) { 741 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: 742 return "reuseport_sockarray"; 743 case BPF_MAP_TYPE_SOCKMAP: 744 return "sockmap"; 745 case BPF_MAP_TYPE_SOCKHASH: 746 return "sockhash"; 747 default: 748 return "unknown"; 749 } 750 } 751 752 static const char *family_str(sa_family_t family) 753 { 754 switch (family) { 755 case AF_INET: 756 return "IPv4"; 757 case AF_INET6: 758 return "IPv6"; 759 default: 760 return "unknown"; 761 } 762 } 763 764 static const char *sotype_str(int sotype) 765 { 766 switch (sotype) { 767 case SOCK_STREAM: 768 return "TCP"; 769 case SOCK_DGRAM: 770 return "UDP"; 771 default: 772 return "unknown"; 773 } 774 } 775 776 #define TEST_INIT(fn_, ...) { .fn = fn_, .name = #fn_, __VA_ARGS__ } 777 778 static void test_config(int sotype, sa_family_t family, bool inany) 779 { 780 const struct test { 781 void (*fn)(int sotype, sa_family_t family); 782 const char *name; 783 bool no_inner_map; 784 int need_sotype; 785 } tests[] = { 786 TEST_INIT(test_err_inner_map, 787 .no_inner_map = true), 788 TEST_INIT(test_err_skb_data), 789 TEST_INIT(test_err_sk_select_port), 790 TEST_INIT(test_pass), 791 TEST_INIT(test_syncookie, 792 .need_sotype = SOCK_STREAM), 793 TEST_INIT(test_pass_on_err), 794 TEST_INIT(test_detach_bpf), 795 }; 796 char s[MAX_TEST_NAME]; 797 const struct test *t; 798 799 for (t = tests; t < tests + ARRAY_SIZE(tests); t++) { 800 if (t->need_sotype && t->need_sotype != sotype) 801 continue; /* test not compatible with socket type */ 802 803 snprintf(s, sizeof(s), "%s %s/%s %s %s", 804 maptype_str(inner_map_type), 805 family_str(family), sotype_str(sotype), 806 inany ? "INANY" : "LOOPBACK", t->name); 807 808 if (!test__start_subtest(s)) 809 continue; 810 811 setup_per_test(sotype, family, inany, t->no_inner_map); 812 t->fn(sotype, family); 813 cleanup_per_test(t->no_inner_map); 814 } 815 } 816 817 #define BIND_INANY true 818 819 static void test_all(void) 820 { 821 const struct config { 822 int sotype; 823 sa_family_t family; 824 bool inany; 825 } configs[] = { 826 { SOCK_STREAM, AF_INET }, 827 { SOCK_STREAM, AF_INET, BIND_INANY }, 828 { SOCK_STREAM, AF_INET6 }, 829 { SOCK_STREAM, AF_INET6, BIND_INANY }, 830 { SOCK_DGRAM, AF_INET }, 831 { SOCK_DGRAM, AF_INET6 }, 832 }; 833 const struct config *c; 834 835 for (c = configs; c < configs + ARRAY_SIZE(configs); c++) 836 test_config(c->sotype, c->family, c->inany); 837 } 838 839 void test_map_type(enum bpf_map_type mt) 840 { 841 if (create_maps(mt)) 842 goto out; 843 if (prepare_bpf_obj()) 844 goto out; 845 846 test_all(); 847 out: 848 cleanup(); 849 } 850 851 void serial_test_select_reuseport(void) 852 { 853 saved_tcp_fo = read_int_sysctl(TCP_FO_SYSCTL); 854 if (saved_tcp_fo < 0) 855 goto out; 856 saved_tcp_syncookie = read_int_sysctl(TCP_SYNCOOKIE_SYSCTL); 857 if (saved_tcp_syncookie < 0) 858 goto out; 859 860 if (enable_fastopen()) 861 goto out; 862 if (disable_syncookie()) 863 goto out; 864 865 test_map_type(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY); 866 test_map_type(BPF_MAP_TYPE_SOCKMAP); 867 test_map_type(BPF_MAP_TYPE_SOCKHASH); 868 out: 869 restore_sysctls(); 870 } 871