1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2020 Facebook */ 3 #include <test_progs.h> 4 #include "bpf_iter_ipv6_route.skel.h" 5 #include "bpf_iter_netlink.skel.h" 6 #include "bpf_iter_bpf_map.skel.h" 7 #include "bpf_iter_task.skel.h" 8 #include "bpf_iter_task_stack.skel.h" 9 #include "bpf_iter_task_file.skel.h" 10 #include "bpf_iter_task_vma.skel.h" 11 #include "bpf_iter_task_btf.skel.h" 12 #include "bpf_iter_tcp4.skel.h" 13 #include "bpf_iter_tcp6.skel.h" 14 #include "bpf_iter_udp4.skel.h" 15 #include "bpf_iter_udp6.skel.h" 16 #include "bpf_iter_test_kern1.skel.h" 17 #include "bpf_iter_test_kern2.skel.h" 18 #include "bpf_iter_test_kern3.skel.h" 19 #include "bpf_iter_test_kern4.skel.h" 20 #include "bpf_iter_bpf_hash_map.skel.h" 21 #include "bpf_iter_bpf_percpu_hash_map.skel.h" 22 #include "bpf_iter_bpf_array_map.skel.h" 23 #include "bpf_iter_bpf_percpu_array_map.skel.h" 24 #include "bpf_iter_bpf_sk_storage_helpers.skel.h" 25 #include "bpf_iter_bpf_sk_storage_map.skel.h" 26 #include "bpf_iter_test_kern5.skel.h" 27 #include "bpf_iter_test_kern6.skel.h" 28 29 static int duration; 30 31 static void test_btf_id_or_null(void) 32 { 33 struct bpf_iter_test_kern3 *skel; 34 35 skel = bpf_iter_test_kern3__open_and_load(); 36 if (CHECK(skel, "bpf_iter_test_kern3__open_and_load", 37 "skeleton open_and_load unexpectedly succeeded\n")) { 38 bpf_iter_test_kern3__destroy(skel); 39 return; 40 } 41 } 42 43 static void do_dummy_read(struct bpf_program *prog) 44 { 45 struct bpf_link *link; 46 char buf[16] = {}; 47 int iter_fd, len; 48 49 link = bpf_program__attach_iter(prog, NULL); 50 if (!ASSERT_OK_PTR(link, "attach_iter")) 51 return; 52 53 iter_fd = bpf_iter_create(bpf_link__fd(link)); 54 if (CHECK(iter_fd < 0, "create_iter", "create_iter failed\n")) 55 goto free_link; 56 57 /* not check contents, but ensure read() ends without error */ 58 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 59 ; 60 CHECK(len < 0, "read", "read failed: %s\n", strerror(errno)); 61 62 close(iter_fd); 63 64 free_link: 65 bpf_link__destroy(link); 66 } 67 68 static int read_fd_into_buffer(int fd, char *buf, int size) 69 { 70 int bufleft = size; 71 int len; 72 73 do { 74 len = read(fd, buf, bufleft); 75 if (len > 0) { 76 buf += len; 77 bufleft -= len; 78 } 79 } while (len > 0); 80 81 return len < 0 ? len : size - bufleft; 82 } 83 84 static void test_ipv6_route(void) 85 { 86 struct bpf_iter_ipv6_route *skel; 87 88 skel = bpf_iter_ipv6_route__open_and_load(); 89 if (CHECK(!skel, "bpf_iter_ipv6_route__open_and_load", 90 "skeleton open_and_load failed\n")) 91 return; 92 93 do_dummy_read(skel->progs.dump_ipv6_route); 94 95 bpf_iter_ipv6_route__destroy(skel); 96 } 97 98 static void test_netlink(void) 99 { 100 struct bpf_iter_netlink *skel; 101 102 skel = bpf_iter_netlink__open_and_load(); 103 if (CHECK(!skel, "bpf_iter_netlink__open_and_load", 104 "skeleton open_and_load failed\n")) 105 return; 106 107 do_dummy_read(skel->progs.dump_netlink); 108 109 bpf_iter_netlink__destroy(skel); 110 } 111 112 static void test_bpf_map(void) 113 { 114 struct bpf_iter_bpf_map *skel; 115 116 skel = bpf_iter_bpf_map__open_and_load(); 117 if (CHECK(!skel, "bpf_iter_bpf_map__open_and_load", 118 "skeleton open_and_load failed\n")) 119 return; 120 121 do_dummy_read(skel->progs.dump_bpf_map); 122 123 bpf_iter_bpf_map__destroy(skel); 124 } 125 126 static void test_task(void) 127 { 128 struct bpf_iter_task *skel; 129 130 skel = bpf_iter_task__open_and_load(); 131 if (CHECK(!skel, "bpf_iter_task__open_and_load", 132 "skeleton open_and_load failed\n")) 133 return; 134 135 do_dummy_read(skel->progs.dump_task); 136 137 bpf_iter_task__destroy(skel); 138 } 139 140 static void test_task_stack(void) 141 { 142 struct bpf_iter_task_stack *skel; 143 144 skel = bpf_iter_task_stack__open_and_load(); 145 if (CHECK(!skel, "bpf_iter_task_stack__open_and_load", 146 "skeleton open_and_load failed\n")) 147 return; 148 149 do_dummy_read(skel->progs.dump_task_stack); 150 do_dummy_read(skel->progs.get_task_user_stacks); 151 152 bpf_iter_task_stack__destroy(skel); 153 } 154 155 static void *do_nothing(void *arg) 156 { 157 pthread_exit(arg); 158 } 159 160 static void test_task_file(void) 161 { 162 struct bpf_iter_task_file *skel; 163 pthread_t thread_id; 164 void *ret; 165 166 skel = bpf_iter_task_file__open_and_load(); 167 if (CHECK(!skel, "bpf_iter_task_file__open_and_load", 168 "skeleton open_and_load failed\n")) 169 return; 170 171 skel->bss->tgid = getpid(); 172 173 if (CHECK(pthread_create(&thread_id, NULL, &do_nothing, NULL), 174 "pthread_create", "pthread_create failed\n")) 175 goto done; 176 177 do_dummy_read(skel->progs.dump_task_file); 178 179 if (CHECK(pthread_join(thread_id, &ret) || ret != NULL, 180 "pthread_join", "pthread_join failed\n")) 181 goto done; 182 183 CHECK(skel->bss->count != 0, "check_count", 184 "invalid non pthread file visit count %d\n", skel->bss->count); 185 186 done: 187 bpf_iter_task_file__destroy(skel); 188 } 189 190 #define TASKBUFSZ 32768 191 192 static char taskbuf[TASKBUFSZ]; 193 194 static int do_btf_read(struct bpf_iter_task_btf *skel) 195 { 196 struct bpf_program *prog = skel->progs.dump_task_struct; 197 struct bpf_iter_task_btf__bss *bss = skel->bss; 198 int iter_fd = -1, err; 199 struct bpf_link *link; 200 char *buf = taskbuf; 201 int ret = 0; 202 203 link = bpf_program__attach_iter(prog, NULL); 204 if (!ASSERT_OK_PTR(link, "attach_iter")) 205 return ret; 206 207 iter_fd = bpf_iter_create(bpf_link__fd(link)); 208 if (CHECK(iter_fd < 0, "create_iter", "create_iter failed\n")) 209 goto free_link; 210 211 err = read_fd_into_buffer(iter_fd, buf, TASKBUFSZ); 212 if (bss->skip) { 213 printf("%s:SKIP:no __builtin_btf_type_id\n", __func__); 214 ret = 1; 215 test__skip(); 216 goto free_link; 217 } 218 219 if (CHECK(err < 0, "read", "read failed: %s\n", strerror(errno))) 220 goto free_link; 221 222 CHECK(strstr(taskbuf, "(struct task_struct)") == NULL, 223 "check for btf representation of task_struct in iter data", 224 "struct task_struct not found"); 225 free_link: 226 if (iter_fd > 0) 227 close(iter_fd); 228 bpf_link__destroy(link); 229 return ret; 230 } 231 232 static void test_task_btf(void) 233 { 234 struct bpf_iter_task_btf__bss *bss; 235 struct bpf_iter_task_btf *skel; 236 int ret; 237 238 skel = bpf_iter_task_btf__open_and_load(); 239 if (CHECK(!skel, "bpf_iter_task_btf__open_and_load", 240 "skeleton open_and_load failed\n")) 241 return; 242 243 bss = skel->bss; 244 245 ret = do_btf_read(skel); 246 if (ret) 247 goto cleanup; 248 249 if (CHECK(bss->tasks == 0, "check if iterated over tasks", 250 "no task iteration, did BPF program run?\n")) 251 goto cleanup; 252 253 CHECK(bss->seq_err != 0, "check for unexpected err", 254 "bpf_seq_printf_btf returned %ld", bss->seq_err); 255 256 cleanup: 257 bpf_iter_task_btf__destroy(skel); 258 } 259 260 static void test_tcp4(void) 261 { 262 struct bpf_iter_tcp4 *skel; 263 264 skel = bpf_iter_tcp4__open_and_load(); 265 if (CHECK(!skel, "bpf_iter_tcp4__open_and_load", 266 "skeleton open_and_load failed\n")) 267 return; 268 269 do_dummy_read(skel->progs.dump_tcp4); 270 271 bpf_iter_tcp4__destroy(skel); 272 } 273 274 static void test_tcp6(void) 275 { 276 struct bpf_iter_tcp6 *skel; 277 278 skel = bpf_iter_tcp6__open_and_load(); 279 if (CHECK(!skel, "bpf_iter_tcp6__open_and_load", 280 "skeleton open_and_load failed\n")) 281 return; 282 283 do_dummy_read(skel->progs.dump_tcp6); 284 285 bpf_iter_tcp6__destroy(skel); 286 } 287 288 static void test_udp4(void) 289 { 290 struct bpf_iter_udp4 *skel; 291 292 skel = bpf_iter_udp4__open_and_load(); 293 if (CHECK(!skel, "bpf_iter_udp4__open_and_load", 294 "skeleton open_and_load failed\n")) 295 return; 296 297 do_dummy_read(skel->progs.dump_udp4); 298 299 bpf_iter_udp4__destroy(skel); 300 } 301 302 static void test_udp6(void) 303 { 304 struct bpf_iter_udp6 *skel; 305 306 skel = bpf_iter_udp6__open_and_load(); 307 if (CHECK(!skel, "bpf_iter_udp6__open_and_load", 308 "skeleton open_and_load failed\n")) 309 return; 310 311 do_dummy_read(skel->progs.dump_udp6); 312 313 bpf_iter_udp6__destroy(skel); 314 } 315 316 /* The expected string is less than 16 bytes */ 317 static int do_read_with_fd(int iter_fd, const char *expected, 318 bool read_one_char) 319 { 320 int err = -1, len, read_buf_len, start; 321 char buf[16] = {}; 322 323 read_buf_len = read_one_char ? 1 : 16; 324 start = 0; 325 while ((len = read(iter_fd, buf + start, read_buf_len)) > 0) { 326 start += len; 327 if (CHECK(start >= 16, "read", "read len %d\n", len)) 328 return -1; 329 read_buf_len = read_one_char ? 1 : 16 - start; 330 } 331 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno))) 332 return -1; 333 334 err = strcmp(buf, expected); 335 if (CHECK(err, "read", "incorrect read result: buf %s, expected %s\n", 336 buf, expected)) 337 return -1; 338 339 return 0; 340 } 341 342 static void test_anon_iter(bool read_one_char) 343 { 344 struct bpf_iter_test_kern1 *skel; 345 struct bpf_link *link; 346 int iter_fd, err; 347 348 skel = bpf_iter_test_kern1__open_and_load(); 349 if (CHECK(!skel, "bpf_iter_test_kern1__open_and_load", 350 "skeleton open_and_load failed\n")) 351 return; 352 353 err = bpf_iter_test_kern1__attach(skel); 354 if (CHECK(err, "bpf_iter_test_kern1__attach", 355 "skeleton attach failed\n")) { 356 goto out; 357 } 358 359 link = skel->links.dump_task; 360 iter_fd = bpf_iter_create(bpf_link__fd(link)); 361 if (CHECK(iter_fd < 0, "create_iter", "create_iter failed\n")) 362 goto out; 363 364 do_read_with_fd(iter_fd, "abcd", read_one_char); 365 close(iter_fd); 366 367 out: 368 bpf_iter_test_kern1__destroy(skel); 369 } 370 371 static int do_read(const char *path, const char *expected) 372 { 373 int err, iter_fd; 374 375 iter_fd = open(path, O_RDONLY); 376 if (CHECK(iter_fd < 0, "open", "open %s failed: %s\n", 377 path, strerror(errno))) 378 return -1; 379 380 err = do_read_with_fd(iter_fd, expected, false); 381 close(iter_fd); 382 return err; 383 } 384 385 static void test_file_iter(void) 386 { 387 const char *path = "/sys/fs/bpf/bpf_iter_test1"; 388 struct bpf_iter_test_kern1 *skel1; 389 struct bpf_iter_test_kern2 *skel2; 390 struct bpf_link *link; 391 int err; 392 393 skel1 = bpf_iter_test_kern1__open_and_load(); 394 if (CHECK(!skel1, "bpf_iter_test_kern1__open_and_load", 395 "skeleton open_and_load failed\n")) 396 return; 397 398 link = bpf_program__attach_iter(skel1->progs.dump_task, NULL); 399 if (!ASSERT_OK_PTR(link, "attach_iter")) 400 goto out; 401 402 /* unlink this path if it exists. */ 403 unlink(path); 404 405 err = bpf_link__pin(link, path); 406 if (CHECK(err, "pin_iter", "pin_iter to %s failed: %d\n", path, err)) 407 goto free_link; 408 409 err = do_read(path, "abcd"); 410 if (err) 411 goto unlink_path; 412 413 /* file based iterator seems working fine. Let us a link update 414 * of the underlying link and `cat` the iterator again, its content 415 * should change. 416 */ 417 skel2 = bpf_iter_test_kern2__open_and_load(); 418 if (CHECK(!skel2, "bpf_iter_test_kern2__open_and_load", 419 "skeleton open_and_load failed\n")) 420 goto unlink_path; 421 422 err = bpf_link__update_program(link, skel2->progs.dump_task); 423 if (CHECK(err, "update_prog", "update_prog failed\n")) 424 goto destroy_skel2; 425 426 do_read(path, "ABCD"); 427 428 destroy_skel2: 429 bpf_iter_test_kern2__destroy(skel2); 430 unlink_path: 431 unlink(path); 432 free_link: 433 bpf_link__destroy(link); 434 out: 435 bpf_iter_test_kern1__destroy(skel1); 436 } 437 438 static void test_overflow(bool test_e2big_overflow, bool ret1) 439 { 440 __u32 map_info_len, total_read_len, expected_read_len; 441 int err, iter_fd, map1_fd, map2_fd, len; 442 struct bpf_map_info map_info = {}; 443 struct bpf_iter_test_kern4 *skel; 444 struct bpf_link *link; 445 __u32 iter_size; 446 char *buf; 447 448 skel = bpf_iter_test_kern4__open(); 449 if (CHECK(!skel, "bpf_iter_test_kern4__open", 450 "skeleton open failed\n")) 451 return; 452 453 /* create two maps: bpf program will only do bpf_seq_write 454 * for these two maps. The goal is one map output almost 455 * fills seq_file buffer and then the other will trigger 456 * overflow and needs restart. 457 */ 458 map1_fd = bpf_create_map(BPF_MAP_TYPE_ARRAY, 4, 8, 1, 0); 459 if (CHECK(map1_fd < 0, "bpf_create_map", 460 "map_creation failed: %s\n", strerror(errno))) 461 goto out; 462 map2_fd = bpf_create_map(BPF_MAP_TYPE_ARRAY, 4, 8, 1, 0); 463 if (CHECK(map2_fd < 0, "bpf_create_map", 464 "map_creation failed: %s\n", strerror(errno))) 465 goto free_map1; 466 467 /* bpf_seq_printf kernel buffer is 8 pages, so one map 468 * bpf_seq_write will mostly fill it, and the other map 469 * will partially fill and then trigger overflow and need 470 * bpf_seq_read restart. 471 */ 472 iter_size = sysconf(_SC_PAGE_SIZE) << 3; 473 474 if (test_e2big_overflow) { 475 skel->rodata->print_len = (iter_size + 8) / 8; 476 expected_read_len = 2 * (iter_size + 8); 477 } else if (!ret1) { 478 skel->rodata->print_len = (iter_size - 8) / 8; 479 expected_read_len = 2 * (iter_size - 8); 480 } else { 481 skel->rodata->print_len = 1; 482 expected_read_len = 2 * 8; 483 } 484 skel->rodata->ret1 = ret1; 485 486 if (CHECK(bpf_iter_test_kern4__load(skel), 487 "bpf_iter_test_kern4__load", "skeleton load failed\n")) 488 goto free_map2; 489 490 /* setup filtering map_id in bpf program */ 491 map_info_len = sizeof(map_info); 492 err = bpf_obj_get_info_by_fd(map1_fd, &map_info, &map_info_len); 493 if (CHECK(err, "get_map_info", "get map info failed: %s\n", 494 strerror(errno))) 495 goto free_map2; 496 skel->bss->map1_id = map_info.id; 497 498 err = bpf_obj_get_info_by_fd(map2_fd, &map_info, &map_info_len); 499 if (CHECK(err, "get_map_info", "get map info failed: %s\n", 500 strerror(errno))) 501 goto free_map2; 502 skel->bss->map2_id = map_info.id; 503 504 link = bpf_program__attach_iter(skel->progs.dump_bpf_map, NULL); 505 if (!ASSERT_OK_PTR(link, "attach_iter")) 506 goto free_map2; 507 508 iter_fd = bpf_iter_create(bpf_link__fd(link)); 509 if (CHECK(iter_fd < 0, "create_iter", "create_iter failed\n")) 510 goto free_link; 511 512 buf = malloc(expected_read_len); 513 if (!buf) 514 goto close_iter; 515 516 /* do read */ 517 total_read_len = 0; 518 if (test_e2big_overflow) { 519 while ((len = read(iter_fd, buf, expected_read_len)) > 0) 520 total_read_len += len; 521 522 CHECK(len != -1 || errno != E2BIG, "read", 523 "expected ret -1, errno E2BIG, but get ret %d, error %s\n", 524 len, strerror(errno)); 525 goto free_buf; 526 } else if (!ret1) { 527 while ((len = read(iter_fd, buf, expected_read_len)) > 0) 528 total_read_len += len; 529 530 if (CHECK(len < 0, "read", "read failed: %s\n", 531 strerror(errno))) 532 goto free_buf; 533 } else { 534 do { 535 len = read(iter_fd, buf, expected_read_len); 536 if (len > 0) 537 total_read_len += len; 538 } while (len > 0 || len == -EAGAIN); 539 540 if (CHECK(len < 0, "read", "read failed: %s\n", 541 strerror(errno))) 542 goto free_buf; 543 } 544 545 if (CHECK(total_read_len != expected_read_len, "read", 546 "total len %u, expected len %u\n", total_read_len, 547 expected_read_len)) 548 goto free_buf; 549 550 if (CHECK(skel->bss->map1_accessed != 1, "map1_accessed", 551 "expected 1 actual %d\n", skel->bss->map1_accessed)) 552 goto free_buf; 553 554 if (CHECK(skel->bss->map2_accessed != 2, "map2_accessed", 555 "expected 2 actual %d\n", skel->bss->map2_accessed)) 556 goto free_buf; 557 558 CHECK(skel->bss->map2_seqnum1 != skel->bss->map2_seqnum2, 559 "map2_seqnum", "two different seqnum %lld %lld\n", 560 skel->bss->map2_seqnum1, skel->bss->map2_seqnum2); 561 562 free_buf: 563 free(buf); 564 close_iter: 565 close(iter_fd); 566 free_link: 567 bpf_link__destroy(link); 568 free_map2: 569 close(map2_fd); 570 free_map1: 571 close(map1_fd); 572 out: 573 bpf_iter_test_kern4__destroy(skel); 574 } 575 576 static void test_bpf_hash_map(void) 577 { 578 __u32 expected_key_a = 0, expected_key_b = 0, expected_key_c = 0; 579 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 580 struct bpf_iter_bpf_hash_map *skel; 581 int err, i, len, map_fd, iter_fd; 582 union bpf_iter_link_info linfo; 583 __u64 val, expected_val = 0; 584 struct bpf_link *link; 585 struct key_t { 586 int a; 587 int b; 588 int c; 589 } key; 590 char buf[64]; 591 592 skel = bpf_iter_bpf_hash_map__open(); 593 if (CHECK(!skel, "bpf_iter_bpf_hash_map__open", 594 "skeleton open failed\n")) 595 return; 596 597 skel->bss->in_test_mode = true; 598 599 err = bpf_iter_bpf_hash_map__load(skel); 600 if (CHECK(!skel, "bpf_iter_bpf_hash_map__load", 601 "skeleton load failed\n")) 602 goto out; 603 604 /* iterator with hashmap2 and hashmap3 should fail */ 605 memset(&linfo, 0, sizeof(linfo)); 606 linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap2); 607 opts.link_info = &linfo; 608 opts.link_info_len = sizeof(linfo); 609 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts); 610 if (!ASSERT_ERR_PTR(link, "attach_iter")) 611 goto out; 612 613 linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap3); 614 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts); 615 if (!ASSERT_ERR_PTR(link, "attach_iter")) 616 goto out; 617 618 /* hashmap1 should be good, update map values here */ 619 map_fd = bpf_map__fd(skel->maps.hashmap1); 620 for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) { 621 key.a = i + 1; 622 key.b = i + 2; 623 key.c = i + 3; 624 val = i + 4; 625 expected_key_a += key.a; 626 expected_key_b += key.b; 627 expected_key_c += key.c; 628 expected_val += val; 629 630 err = bpf_map_update_elem(map_fd, &key, &val, BPF_ANY); 631 if (CHECK(err, "map_update", "map_update failed\n")) 632 goto out; 633 } 634 635 linfo.map.map_fd = map_fd; 636 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts); 637 if (!ASSERT_OK_PTR(link, "attach_iter")) 638 goto out; 639 640 iter_fd = bpf_iter_create(bpf_link__fd(link)); 641 if (CHECK(iter_fd < 0, "create_iter", "create_iter failed\n")) 642 goto free_link; 643 644 /* do some tests */ 645 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 646 ; 647 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno))) 648 goto close_iter; 649 650 /* test results */ 651 if (CHECK(skel->bss->key_sum_a != expected_key_a, 652 "key_sum_a", "got %u expected %u\n", 653 skel->bss->key_sum_a, expected_key_a)) 654 goto close_iter; 655 if (CHECK(skel->bss->key_sum_b != expected_key_b, 656 "key_sum_b", "got %u expected %u\n", 657 skel->bss->key_sum_b, expected_key_b)) 658 goto close_iter; 659 if (CHECK(skel->bss->val_sum != expected_val, 660 "val_sum", "got %llu expected %llu\n", 661 skel->bss->val_sum, expected_val)) 662 goto close_iter; 663 664 close_iter: 665 close(iter_fd); 666 free_link: 667 bpf_link__destroy(link); 668 out: 669 bpf_iter_bpf_hash_map__destroy(skel); 670 } 671 672 static void test_bpf_percpu_hash_map(void) 673 { 674 __u32 expected_key_a = 0, expected_key_b = 0, expected_key_c = 0; 675 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 676 struct bpf_iter_bpf_percpu_hash_map *skel; 677 int err, i, j, len, map_fd, iter_fd; 678 union bpf_iter_link_info linfo; 679 __u32 expected_val = 0; 680 struct bpf_link *link; 681 struct key_t { 682 int a; 683 int b; 684 int c; 685 } key; 686 char buf[64]; 687 void *val; 688 689 val = malloc(8 * bpf_num_possible_cpus()); 690 691 skel = bpf_iter_bpf_percpu_hash_map__open(); 692 if (CHECK(!skel, "bpf_iter_bpf_percpu_hash_map__open", 693 "skeleton open failed\n")) 694 return; 695 696 skel->rodata->num_cpus = bpf_num_possible_cpus(); 697 698 err = bpf_iter_bpf_percpu_hash_map__load(skel); 699 if (CHECK(!skel, "bpf_iter_bpf_percpu_hash_map__load", 700 "skeleton load failed\n")) 701 goto out; 702 703 /* update map values here */ 704 map_fd = bpf_map__fd(skel->maps.hashmap1); 705 for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) { 706 key.a = i + 1; 707 key.b = i + 2; 708 key.c = i + 3; 709 expected_key_a += key.a; 710 expected_key_b += key.b; 711 expected_key_c += key.c; 712 713 for (j = 0; j < bpf_num_possible_cpus(); j++) { 714 *(__u32 *)(val + j * 8) = i + j; 715 expected_val += i + j; 716 } 717 718 err = bpf_map_update_elem(map_fd, &key, val, BPF_ANY); 719 if (CHECK(err, "map_update", "map_update failed\n")) 720 goto out; 721 } 722 723 memset(&linfo, 0, sizeof(linfo)); 724 linfo.map.map_fd = map_fd; 725 opts.link_info = &linfo; 726 opts.link_info_len = sizeof(linfo); 727 link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_hash_map, &opts); 728 if (!ASSERT_OK_PTR(link, "attach_iter")) 729 goto out; 730 731 iter_fd = bpf_iter_create(bpf_link__fd(link)); 732 if (CHECK(iter_fd < 0, "create_iter", "create_iter failed\n")) 733 goto free_link; 734 735 /* do some tests */ 736 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 737 ; 738 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno))) 739 goto close_iter; 740 741 /* test results */ 742 if (CHECK(skel->bss->key_sum_a != expected_key_a, 743 "key_sum_a", "got %u expected %u\n", 744 skel->bss->key_sum_a, expected_key_a)) 745 goto close_iter; 746 if (CHECK(skel->bss->key_sum_b != expected_key_b, 747 "key_sum_b", "got %u expected %u\n", 748 skel->bss->key_sum_b, expected_key_b)) 749 goto close_iter; 750 if (CHECK(skel->bss->val_sum != expected_val, 751 "val_sum", "got %u expected %u\n", 752 skel->bss->val_sum, expected_val)) 753 goto close_iter; 754 755 close_iter: 756 close(iter_fd); 757 free_link: 758 bpf_link__destroy(link); 759 out: 760 bpf_iter_bpf_percpu_hash_map__destroy(skel); 761 } 762 763 static void test_bpf_array_map(void) 764 { 765 __u64 val, expected_val = 0, res_first_val, first_val = 0; 766 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 767 __u32 expected_key = 0, res_first_key; 768 struct bpf_iter_bpf_array_map *skel; 769 union bpf_iter_link_info linfo; 770 int err, i, map_fd, iter_fd; 771 struct bpf_link *link; 772 char buf[64] = {}; 773 int len, start; 774 775 skel = bpf_iter_bpf_array_map__open_and_load(); 776 if (CHECK(!skel, "bpf_iter_bpf_array_map__open_and_load", 777 "skeleton open_and_load failed\n")) 778 return; 779 780 map_fd = bpf_map__fd(skel->maps.arraymap1); 781 for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) { 782 val = i + 4; 783 expected_key += i; 784 expected_val += val; 785 786 if (i == 0) 787 first_val = val; 788 789 err = bpf_map_update_elem(map_fd, &i, &val, BPF_ANY); 790 if (CHECK(err, "map_update", "map_update failed\n")) 791 goto out; 792 } 793 794 memset(&linfo, 0, sizeof(linfo)); 795 linfo.map.map_fd = map_fd; 796 opts.link_info = &linfo; 797 opts.link_info_len = sizeof(linfo); 798 link = bpf_program__attach_iter(skel->progs.dump_bpf_array_map, &opts); 799 if (!ASSERT_OK_PTR(link, "attach_iter")) 800 goto out; 801 802 iter_fd = bpf_iter_create(bpf_link__fd(link)); 803 if (CHECK(iter_fd < 0, "create_iter", "create_iter failed\n")) 804 goto free_link; 805 806 /* do some tests */ 807 start = 0; 808 while ((len = read(iter_fd, buf + start, sizeof(buf) - start)) > 0) 809 start += len; 810 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno))) 811 goto close_iter; 812 813 /* test results */ 814 res_first_key = *(__u32 *)buf; 815 res_first_val = *(__u64 *)(buf + sizeof(__u32)); 816 if (CHECK(res_first_key != 0 || res_first_val != first_val, 817 "bpf_seq_write", 818 "seq_write failure: first key %u vs expected 0, " 819 " first value %llu vs expected %llu\n", 820 res_first_key, res_first_val, first_val)) 821 goto close_iter; 822 823 if (CHECK(skel->bss->key_sum != expected_key, 824 "key_sum", "got %u expected %u\n", 825 skel->bss->key_sum, expected_key)) 826 goto close_iter; 827 if (CHECK(skel->bss->val_sum != expected_val, 828 "val_sum", "got %llu expected %llu\n", 829 skel->bss->val_sum, expected_val)) 830 goto close_iter; 831 832 for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) { 833 err = bpf_map_lookup_elem(map_fd, &i, &val); 834 if (CHECK(err, "map_lookup", "map_lookup failed\n")) 835 goto out; 836 if (CHECK(i != val, "invalid_val", 837 "got value %llu expected %u\n", val, i)) 838 goto out; 839 } 840 841 close_iter: 842 close(iter_fd); 843 free_link: 844 bpf_link__destroy(link); 845 out: 846 bpf_iter_bpf_array_map__destroy(skel); 847 } 848 849 static void test_bpf_percpu_array_map(void) 850 { 851 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 852 struct bpf_iter_bpf_percpu_array_map *skel; 853 __u32 expected_key = 0, expected_val = 0; 854 union bpf_iter_link_info linfo; 855 int err, i, j, map_fd, iter_fd; 856 struct bpf_link *link; 857 char buf[64]; 858 void *val; 859 int len; 860 861 val = malloc(8 * bpf_num_possible_cpus()); 862 863 skel = bpf_iter_bpf_percpu_array_map__open(); 864 if (CHECK(!skel, "bpf_iter_bpf_percpu_array_map__open", 865 "skeleton open failed\n")) 866 return; 867 868 skel->rodata->num_cpus = bpf_num_possible_cpus(); 869 870 err = bpf_iter_bpf_percpu_array_map__load(skel); 871 if (CHECK(!skel, "bpf_iter_bpf_percpu_array_map__load", 872 "skeleton load failed\n")) 873 goto out; 874 875 /* update map values here */ 876 map_fd = bpf_map__fd(skel->maps.arraymap1); 877 for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) { 878 expected_key += i; 879 880 for (j = 0; j < bpf_num_possible_cpus(); j++) { 881 *(__u32 *)(val + j * 8) = i + j; 882 expected_val += i + j; 883 } 884 885 err = bpf_map_update_elem(map_fd, &i, val, BPF_ANY); 886 if (CHECK(err, "map_update", "map_update failed\n")) 887 goto out; 888 } 889 890 memset(&linfo, 0, sizeof(linfo)); 891 linfo.map.map_fd = map_fd; 892 opts.link_info = &linfo; 893 opts.link_info_len = sizeof(linfo); 894 link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_array_map, &opts); 895 if (!ASSERT_OK_PTR(link, "attach_iter")) 896 goto out; 897 898 iter_fd = bpf_iter_create(bpf_link__fd(link)); 899 if (CHECK(iter_fd < 0, "create_iter", "create_iter failed\n")) 900 goto free_link; 901 902 /* do some tests */ 903 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 904 ; 905 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno))) 906 goto close_iter; 907 908 /* test results */ 909 if (CHECK(skel->bss->key_sum != expected_key, 910 "key_sum", "got %u expected %u\n", 911 skel->bss->key_sum, expected_key)) 912 goto close_iter; 913 if (CHECK(skel->bss->val_sum != expected_val, 914 "val_sum", "got %u expected %u\n", 915 skel->bss->val_sum, expected_val)) 916 goto close_iter; 917 918 close_iter: 919 close(iter_fd); 920 free_link: 921 bpf_link__destroy(link); 922 out: 923 bpf_iter_bpf_percpu_array_map__destroy(skel); 924 } 925 926 /* An iterator program deletes all local storage in a map. */ 927 static void test_bpf_sk_storage_delete(void) 928 { 929 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 930 struct bpf_iter_bpf_sk_storage_helpers *skel; 931 union bpf_iter_link_info linfo; 932 int err, len, map_fd, iter_fd; 933 struct bpf_link *link; 934 int sock_fd = -1; 935 __u32 val = 42; 936 char buf[64]; 937 938 skel = bpf_iter_bpf_sk_storage_helpers__open_and_load(); 939 if (CHECK(!skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load", 940 "skeleton open_and_load failed\n")) 941 return; 942 943 map_fd = bpf_map__fd(skel->maps.sk_stg_map); 944 945 sock_fd = socket(AF_INET6, SOCK_STREAM, 0); 946 if (CHECK(sock_fd < 0, "socket", "errno: %d\n", errno)) 947 goto out; 948 err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST); 949 if (CHECK(err, "map_update", "map_update failed\n")) 950 goto out; 951 952 memset(&linfo, 0, sizeof(linfo)); 953 linfo.map.map_fd = map_fd; 954 opts.link_info = &linfo; 955 opts.link_info_len = sizeof(linfo); 956 link = bpf_program__attach_iter(skel->progs.delete_bpf_sk_storage_map, 957 &opts); 958 if (!ASSERT_OK_PTR(link, "attach_iter")) 959 goto out; 960 961 iter_fd = bpf_iter_create(bpf_link__fd(link)); 962 if (CHECK(iter_fd < 0, "create_iter", "create_iter failed\n")) 963 goto free_link; 964 965 /* do some tests */ 966 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 967 ; 968 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno))) 969 goto close_iter; 970 971 /* test results */ 972 err = bpf_map_lookup_elem(map_fd, &sock_fd, &val); 973 if (CHECK(!err || errno != ENOENT, "bpf_map_lookup_elem", 974 "map value wasn't deleted (err=%d, errno=%d)\n", err, errno)) 975 goto close_iter; 976 977 close_iter: 978 close(iter_fd); 979 free_link: 980 bpf_link__destroy(link); 981 out: 982 if (sock_fd >= 0) 983 close(sock_fd); 984 bpf_iter_bpf_sk_storage_helpers__destroy(skel); 985 } 986 987 /* This creates a socket and its local storage. It then runs a task_iter BPF 988 * program that replaces the existing socket local storage with the tgid of the 989 * only task owning a file descriptor to this socket, this process, prog_tests. 990 * It then runs a tcp socket iterator that negates the value in the existing 991 * socket local storage, the test verifies that the resulting value is -pid. 992 */ 993 static void test_bpf_sk_storage_get(void) 994 { 995 struct bpf_iter_bpf_sk_storage_helpers *skel; 996 int err, map_fd, val = -1; 997 int sock_fd = -1; 998 999 skel = bpf_iter_bpf_sk_storage_helpers__open_and_load(); 1000 if (CHECK(!skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load", 1001 "skeleton open_and_load failed\n")) 1002 return; 1003 1004 sock_fd = socket(AF_INET6, SOCK_STREAM, 0); 1005 if (CHECK(sock_fd < 0, "socket", "errno: %d\n", errno)) 1006 goto out; 1007 1008 err = listen(sock_fd, 1); 1009 if (CHECK(err != 0, "listen", "errno: %d\n", errno)) 1010 goto close_socket; 1011 1012 map_fd = bpf_map__fd(skel->maps.sk_stg_map); 1013 1014 err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST); 1015 if (CHECK(err, "bpf_map_update_elem", "map_update_failed\n")) 1016 goto close_socket; 1017 1018 do_dummy_read(skel->progs.fill_socket_owner); 1019 1020 err = bpf_map_lookup_elem(map_fd, &sock_fd, &val); 1021 if (CHECK(err || val != getpid(), "bpf_map_lookup_elem", 1022 "map value wasn't set correctly (expected %d, got %d, err=%d)\n", 1023 getpid(), val, err)) 1024 goto close_socket; 1025 1026 do_dummy_read(skel->progs.negate_socket_local_storage); 1027 1028 err = bpf_map_lookup_elem(map_fd, &sock_fd, &val); 1029 CHECK(err || val != -getpid(), "bpf_map_lookup_elem", 1030 "map value wasn't set correctly (expected %d, got %d, err=%d)\n", 1031 -getpid(), val, err); 1032 1033 close_socket: 1034 close(sock_fd); 1035 out: 1036 bpf_iter_bpf_sk_storage_helpers__destroy(skel); 1037 } 1038 1039 static void test_bpf_sk_storage_map(void) 1040 { 1041 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 1042 int err, i, len, map_fd, iter_fd, num_sockets; 1043 struct bpf_iter_bpf_sk_storage_map *skel; 1044 union bpf_iter_link_info linfo; 1045 int sock_fd[3] = {-1, -1, -1}; 1046 __u32 val, expected_val = 0; 1047 struct bpf_link *link; 1048 char buf[64]; 1049 1050 skel = bpf_iter_bpf_sk_storage_map__open_and_load(); 1051 if (CHECK(!skel, "bpf_iter_bpf_sk_storage_map__open_and_load", 1052 "skeleton open_and_load failed\n")) 1053 return; 1054 1055 map_fd = bpf_map__fd(skel->maps.sk_stg_map); 1056 num_sockets = ARRAY_SIZE(sock_fd); 1057 for (i = 0; i < num_sockets; i++) { 1058 sock_fd[i] = socket(AF_INET6, SOCK_STREAM, 0); 1059 if (CHECK(sock_fd[i] < 0, "socket", "errno: %d\n", errno)) 1060 goto out; 1061 1062 val = i + 1; 1063 expected_val += val; 1064 1065 err = bpf_map_update_elem(map_fd, &sock_fd[i], &val, 1066 BPF_NOEXIST); 1067 if (CHECK(err, "map_update", "map_update failed\n")) 1068 goto out; 1069 } 1070 1071 memset(&linfo, 0, sizeof(linfo)); 1072 linfo.map.map_fd = map_fd; 1073 opts.link_info = &linfo; 1074 opts.link_info_len = sizeof(linfo); 1075 link = bpf_program__attach_iter(skel->progs.dump_bpf_sk_storage_map, &opts); 1076 if (!ASSERT_OK_PTR(link, "attach_iter")) 1077 goto out; 1078 1079 iter_fd = bpf_iter_create(bpf_link__fd(link)); 1080 if (CHECK(iter_fd < 0, "create_iter", "create_iter failed\n")) 1081 goto free_link; 1082 1083 /* do some tests */ 1084 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 1085 ; 1086 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno))) 1087 goto close_iter; 1088 1089 /* test results */ 1090 if (CHECK(skel->bss->ipv6_sk_count != num_sockets, 1091 "ipv6_sk_count", "got %u expected %u\n", 1092 skel->bss->ipv6_sk_count, num_sockets)) 1093 goto close_iter; 1094 1095 if (CHECK(skel->bss->val_sum != expected_val, 1096 "val_sum", "got %u expected %u\n", 1097 skel->bss->val_sum, expected_val)) 1098 goto close_iter; 1099 1100 close_iter: 1101 close(iter_fd); 1102 free_link: 1103 bpf_link__destroy(link); 1104 out: 1105 for (i = 0; i < num_sockets; i++) { 1106 if (sock_fd[i] >= 0) 1107 close(sock_fd[i]); 1108 } 1109 bpf_iter_bpf_sk_storage_map__destroy(skel); 1110 } 1111 1112 static void test_rdonly_buf_out_of_bound(void) 1113 { 1114 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 1115 struct bpf_iter_test_kern5 *skel; 1116 union bpf_iter_link_info linfo; 1117 struct bpf_link *link; 1118 1119 skel = bpf_iter_test_kern5__open_and_load(); 1120 if (CHECK(!skel, "bpf_iter_test_kern5__open_and_load", 1121 "skeleton open_and_load failed\n")) 1122 return; 1123 1124 memset(&linfo, 0, sizeof(linfo)); 1125 linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap1); 1126 opts.link_info = &linfo; 1127 opts.link_info_len = sizeof(linfo); 1128 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts); 1129 if (!ASSERT_ERR_PTR(link, "attach_iter")) 1130 bpf_link__destroy(link); 1131 1132 bpf_iter_test_kern5__destroy(skel); 1133 } 1134 1135 static void test_buf_neg_offset(void) 1136 { 1137 struct bpf_iter_test_kern6 *skel; 1138 1139 skel = bpf_iter_test_kern6__open_and_load(); 1140 if (CHECK(skel, "bpf_iter_test_kern6__open_and_load", 1141 "skeleton open_and_load unexpected success\n")) 1142 bpf_iter_test_kern6__destroy(skel); 1143 } 1144 1145 #define CMP_BUFFER_SIZE 1024 1146 static char task_vma_output[CMP_BUFFER_SIZE]; 1147 static char proc_maps_output[CMP_BUFFER_SIZE]; 1148 1149 /* remove \0 and \t from str, and only keep the first line */ 1150 static void str_strip_first_line(char *str) 1151 { 1152 char *dst = str, *src = str; 1153 1154 do { 1155 if (*src == ' ' || *src == '\t') 1156 src++; 1157 else 1158 *(dst++) = *(src++); 1159 1160 } while (*src != '\0' && *src != '\n'); 1161 1162 *dst = '\0'; 1163 } 1164 1165 #define min(a, b) ((a) < (b) ? (a) : (b)) 1166 1167 static void test_task_vma(void) 1168 { 1169 int err, iter_fd = -1, proc_maps_fd = -1; 1170 struct bpf_iter_task_vma *skel; 1171 int len, read_size = 4; 1172 char maps_path[64]; 1173 1174 skel = bpf_iter_task_vma__open(); 1175 if (CHECK(!skel, "bpf_iter_task_vma__open", "skeleton open failed\n")) 1176 return; 1177 1178 skel->bss->pid = getpid(); 1179 1180 err = bpf_iter_task_vma__load(skel); 1181 if (CHECK(err, "bpf_iter_task_vma__load", "skeleton load failed\n")) 1182 goto out; 1183 1184 skel->links.proc_maps = bpf_program__attach_iter( 1185 skel->progs.proc_maps, NULL); 1186 1187 if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) { 1188 skel->links.proc_maps = NULL; 1189 goto out; 1190 } 1191 1192 iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps)); 1193 if (CHECK(iter_fd < 0, "create_iter", "create_iter failed\n")) 1194 goto out; 1195 1196 /* Read CMP_BUFFER_SIZE (1kB) from bpf_iter. Read in small chunks 1197 * to trigger seq_file corner cases. The expected output is much 1198 * longer than 1kB, so the while loop will terminate. 1199 */ 1200 len = 0; 1201 while (len < CMP_BUFFER_SIZE) { 1202 err = read_fd_into_buffer(iter_fd, task_vma_output + len, 1203 min(read_size, CMP_BUFFER_SIZE - len)); 1204 if (CHECK(err < 0, "read_iter_fd", "read_iter_fd failed\n")) 1205 goto out; 1206 len += err; 1207 } 1208 1209 /* read CMP_BUFFER_SIZE (1kB) from /proc/pid/maps */ 1210 snprintf(maps_path, 64, "/proc/%u/maps", skel->bss->pid); 1211 proc_maps_fd = open(maps_path, O_RDONLY); 1212 if (CHECK(proc_maps_fd < 0, "open_proc_maps", "open_proc_maps failed\n")) 1213 goto out; 1214 err = read_fd_into_buffer(proc_maps_fd, proc_maps_output, CMP_BUFFER_SIZE); 1215 if (CHECK(err < 0, "read_prog_maps_fd", "read_prog_maps_fd failed\n")) 1216 goto out; 1217 1218 /* strip and compare the first line of the two files */ 1219 str_strip_first_line(task_vma_output); 1220 str_strip_first_line(proc_maps_output); 1221 1222 CHECK(strcmp(task_vma_output, proc_maps_output), "compare_output", 1223 "found mismatch\n"); 1224 out: 1225 close(proc_maps_fd); 1226 close(iter_fd); 1227 bpf_iter_task_vma__destroy(skel); 1228 } 1229 1230 void test_bpf_iter(void) 1231 { 1232 if (test__start_subtest("btf_id_or_null")) 1233 test_btf_id_or_null(); 1234 if (test__start_subtest("ipv6_route")) 1235 test_ipv6_route(); 1236 if (test__start_subtest("netlink")) 1237 test_netlink(); 1238 if (test__start_subtest("bpf_map")) 1239 test_bpf_map(); 1240 if (test__start_subtest("task")) 1241 test_task(); 1242 if (test__start_subtest("task_stack")) 1243 test_task_stack(); 1244 if (test__start_subtest("task_file")) 1245 test_task_file(); 1246 if (test__start_subtest("task_vma")) 1247 test_task_vma(); 1248 if (test__start_subtest("task_btf")) 1249 test_task_btf(); 1250 if (test__start_subtest("tcp4")) 1251 test_tcp4(); 1252 if (test__start_subtest("tcp6")) 1253 test_tcp6(); 1254 if (test__start_subtest("udp4")) 1255 test_udp4(); 1256 if (test__start_subtest("udp6")) 1257 test_udp6(); 1258 if (test__start_subtest("anon")) 1259 test_anon_iter(false); 1260 if (test__start_subtest("anon-read-one-char")) 1261 test_anon_iter(true); 1262 if (test__start_subtest("file")) 1263 test_file_iter(); 1264 if (test__start_subtest("overflow")) 1265 test_overflow(false, false); 1266 if (test__start_subtest("overflow-e2big")) 1267 test_overflow(true, false); 1268 if (test__start_subtest("prog-ret-1")) 1269 test_overflow(false, true); 1270 if (test__start_subtest("bpf_hash_map")) 1271 test_bpf_hash_map(); 1272 if (test__start_subtest("bpf_percpu_hash_map")) 1273 test_bpf_percpu_hash_map(); 1274 if (test__start_subtest("bpf_array_map")) 1275 test_bpf_array_map(); 1276 if (test__start_subtest("bpf_percpu_array_map")) 1277 test_bpf_percpu_array_map(); 1278 if (test__start_subtest("bpf_sk_storage_map")) 1279 test_bpf_sk_storage_map(); 1280 if (test__start_subtest("bpf_sk_storage_delete")) 1281 test_bpf_sk_storage_delete(); 1282 if (test__start_subtest("bpf_sk_storage_get")) 1283 test_bpf_sk_storage_get(); 1284 if (test__start_subtest("rdonly-buf-out-of-bound")) 1285 test_rdonly_buf_out_of_bound(); 1286 if (test__start_subtest("buf-neg-offset")) 1287 test_buf_neg_offset(); 1288 } 1289