1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2020 Facebook */ 3 #include <test_progs.h> 4 #include "bpf_iter_ipv6_route.skel.h" 5 #include "bpf_iter_netlink.skel.h" 6 #include "bpf_iter_bpf_map.skel.h" 7 #include "bpf_iter_task.skel.h" 8 #include "bpf_iter_task_stack.skel.h" 9 #include "bpf_iter_task_file.skel.h" 10 #include "bpf_iter_task_vma.skel.h" 11 #include "bpf_iter_task_btf.skel.h" 12 #include "bpf_iter_tcp4.skel.h" 13 #include "bpf_iter_tcp6.skel.h" 14 #include "bpf_iter_udp4.skel.h" 15 #include "bpf_iter_udp6.skel.h" 16 #include "bpf_iter_unix.skel.h" 17 #include "bpf_iter_test_kern1.skel.h" 18 #include "bpf_iter_test_kern2.skel.h" 19 #include "bpf_iter_test_kern3.skel.h" 20 #include "bpf_iter_test_kern4.skel.h" 21 #include "bpf_iter_bpf_hash_map.skel.h" 22 #include "bpf_iter_bpf_percpu_hash_map.skel.h" 23 #include "bpf_iter_bpf_array_map.skel.h" 24 #include "bpf_iter_bpf_percpu_array_map.skel.h" 25 #include "bpf_iter_bpf_sk_storage_helpers.skel.h" 26 #include "bpf_iter_bpf_sk_storage_map.skel.h" 27 #include "bpf_iter_test_kern5.skel.h" 28 #include "bpf_iter_test_kern6.skel.h" 29 #include "bpf_iter_bpf_link.skel.h" 30 #include "bpf_iter_ksym.skel.h" 31 32 static int duration; 33 34 static void test_btf_id_or_null(void) 35 { 36 struct bpf_iter_test_kern3 *skel; 37 38 skel = bpf_iter_test_kern3__open_and_load(); 39 if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern3__open_and_load")) { 40 bpf_iter_test_kern3__destroy(skel); 41 return; 42 } 43 } 44 45 static void do_dummy_read(struct bpf_program *prog) 46 { 47 struct bpf_link *link; 48 char buf[16] = {}; 49 int iter_fd, len; 50 51 link = bpf_program__attach_iter(prog, NULL); 52 if (!ASSERT_OK_PTR(link, "attach_iter")) 53 return; 54 55 iter_fd = bpf_iter_create(bpf_link__fd(link)); 56 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 57 goto free_link; 58 59 /* not check contents, but ensure read() ends without error */ 60 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 61 ; 62 CHECK(len < 0, "read", "read failed: %s\n", strerror(errno)); 63 64 close(iter_fd); 65 66 free_link: 67 bpf_link__destroy(link); 68 } 69 70 static int read_fd_into_buffer(int fd, char *buf, int size) 71 { 72 int bufleft = size; 73 int len; 74 75 do { 76 len = read(fd, buf, bufleft); 77 if (len > 0) { 78 buf += len; 79 bufleft -= len; 80 } 81 } while (len > 0); 82 83 return len < 0 ? len : size - bufleft; 84 } 85 86 static void test_ipv6_route(void) 87 { 88 struct bpf_iter_ipv6_route *skel; 89 90 skel = bpf_iter_ipv6_route__open_and_load(); 91 if (!ASSERT_OK_PTR(skel, "bpf_iter_ipv6_route__open_and_load")) 92 return; 93 94 do_dummy_read(skel->progs.dump_ipv6_route); 95 96 bpf_iter_ipv6_route__destroy(skel); 97 } 98 99 static void test_netlink(void) 100 { 101 struct bpf_iter_netlink *skel; 102 103 skel = bpf_iter_netlink__open_and_load(); 104 if (!ASSERT_OK_PTR(skel, "bpf_iter_netlink__open_and_load")) 105 return; 106 107 do_dummy_read(skel->progs.dump_netlink); 108 109 bpf_iter_netlink__destroy(skel); 110 } 111 112 static void test_bpf_map(void) 113 { 114 struct bpf_iter_bpf_map *skel; 115 116 skel = bpf_iter_bpf_map__open_and_load(); 117 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_map__open_and_load")) 118 return; 119 120 do_dummy_read(skel->progs.dump_bpf_map); 121 122 bpf_iter_bpf_map__destroy(skel); 123 } 124 125 static void test_task(void) 126 { 127 struct bpf_iter_task *skel; 128 129 skel = bpf_iter_task__open_and_load(); 130 if (!ASSERT_OK_PTR(skel, "bpf_iter_task__open_and_load")) 131 return; 132 133 do_dummy_read(skel->progs.dump_task); 134 135 bpf_iter_task__destroy(skel); 136 } 137 138 static void test_task_sleepable(void) 139 { 140 struct bpf_iter_task *skel; 141 142 skel = bpf_iter_task__open_and_load(); 143 if (!ASSERT_OK_PTR(skel, "bpf_iter_task__open_and_load")) 144 return; 145 146 do_dummy_read(skel->progs.dump_task_sleepable); 147 148 ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task, 0, 149 "num_expected_failure_copy_from_user_task"); 150 ASSERT_GT(skel->bss->num_success_copy_from_user_task, 0, 151 "num_success_copy_from_user_task"); 152 153 bpf_iter_task__destroy(skel); 154 } 155 156 static void test_task_stack(void) 157 { 158 struct bpf_iter_task_stack *skel; 159 160 skel = bpf_iter_task_stack__open_and_load(); 161 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_stack__open_and_load")) 162 return; 163 164 do_dummy_read(skel->progs.dump_task_stack); 165 do_dummy_read(skel->progs.get_task_user_stacks); 166 167 bpf_iter_task_stack__destroy(skel); 168 } 169 170 static void *do_nothing(void *arg) 171 { 172 pthread_exit(arg); 173 } 174 175 static void test_task_file(void) 176 { 177 struct bpf_iter_task_file *skel; 178 pthread_t thread_id; 179 void *ret; 180 181 skel = bpf_iter_task_file__open_and_load(); 182 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_file__open_and_load")) 183 return; 184 185 skel->bss->tgid = getpid(); 186 187 if (!ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing, NULL), 188 "pthread_create")) 189 goto done; 190 191 do_dummy_read(skel->progs.dump_task_file); 192 193 if (!ASSERT_FALSE(pthread_join(thread_id, &ret) || ret != NULL, 194 "pthread_join")) 195 goto done; 196 197 ASSERT_EQ(skel->bss->count, 0, "check_count"); 198 199 done: 200 bpf_iter_task_file__destroy(skel); 201 } 202 203 #define TASKBUFSZ 32768 204 205 static char taskbuf[TASKBUFSZ]; 206 207 static int do_btf_read(struct bpf_iter_task_btf *skel) 208 { 209 struct bpf_program *prog = skel->progs.dump_task_struct; 210 struct bpf_iter_task_btf__bss *bss = skel->bss; 211 int iter_fd = -1, err; 212 struct bpf_link *link; 213 char *buf = taskbuf; 214 int ret = 0; 215 216 link = bpf_program__attach_iter(prog, NULL); 217 if (!ASSERT_OK_PTR(link, "attach_iter")) 218 return ret; 219 220 iter_fd = bpf_iter_create(bpf_link__fd(link)); 221 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 222 goto free_link; 223 224 err = read_fd_into_buffer(iter_fd, buf, TASKBUFSZ); 225 if (bss->skip) { 226 printf("%s:SKIP:no __builtin_btf_type_id\n", __func__); 227 ret = 1; 228 test__skip(); 229 goto free_link; 230 } 231 232 if (CHECK(err < 0, "read", "read failed: %s\n", strerror(errno))) 233 goto free_link; 234 235 ASSERT_HAS_SUBSTR(taskbuf, "(struct task_struct)", 236 "check for btf representation of task_struct in iter data"); 237 free_link: 238 if (iter_fd > 0) 239 close(iter_fd); 240 bpf_link__destroy(link); 241 return ret; 242 } 243 244 static void test_task_btf(void) 245 { 246 struct bpf_iter_task_btf__bss *bss; 247 struct bpf_iter_task_btf *skel; 248 int ret; 249 250 skel = bpf_iter_task_btf__open_and_load(); 251 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_btf__open_and_load")) 252 return; 253 254 bss = skel->bss; 255 256 ret = do_btf_read(skel); 257 if (ret) 258 goto cleanup; 259 260 if (!ASSERT_NEQ(bss->tasks, 0, "no task iteration, did BPF program run?")) 261 goto cleanup; 262 263 ASSERT_EQ(bss->seq_err, 0, "check for unexpected err"); 264 265 cleanup: 266 bpf_iter_task_btf__destroy(skel); 267 } 268 269 static void test_tcp4(void) 270 { 271 struct bpf_iter_tcp4 *skel; 272 273 skel = bpf_iter_tcp4__open_and_load(); 274 if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp4__open_and_load")) 275 return; 276 277 do_dummy_read(skel->progs.dump_tcp4); 278 279 bpf_iter_tcp4__destroy(skel); 280 } 281 282 static void test_tcp6(void) 283 { 284 struct bpf_iter_tcp6 *skel; 285 286 skel = bpf_iter_tcp6__open_and_load(); 287 if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp6__open_and_load")) 288 return; 289 290 do_dummy_read(skel->progs.dump_tcp6); 291 292 bpf_iter_tcp6__destroy(skel); 293 } 294 295 static void test_udp4(void) 296 { 297 struct bpf_iter_udp4 *skel; 298 299 skel = bpf_iter_udp4__open_and_load(); 300 if (!ASSERT_OK_PTR(skel, "bpf_iter_udp4__open_and_load")) 301 return; 302 303 do_dummy_read(skel->progs.dump_udp4); 304 305 bpf_iter_udp4__destroy(skel); 306 } 307 308 static void test_udp6(void) 309 { 310 struct bpf_iter_udp6 *skel; 311 312 skel = bpf_iter_udp6__open_and_load(); 313 if (!ASSERT_OK_PTR(skel, "bpf_iter_udp6__open_and_load")) 314 return; 315 316 do_dummy_read(skel->progs.dump_udp6); 317 318 bpf_iter_udp6__destroy(skel); 319 } 320 321 static void test_unix(void) 322 { 323 struct bpf_iter_unix *skel; 324 325 skel = bpf_iter_unix__open_and_load(); 326 if (!ASSERT_OK_PTR(skel, "bpf_iter_unix__open_and_load")) 327 return; 328 329 do_dummy_read(skel->progs.dump_unix); 330 331 bpf_iter_unix__destroy(skel); 332 } 333 334 /* The expected string is less than 16 bytes */ 335 static int do_read_with_fd(int iter_fd, const char *expected, 336 bool read_one_char) 337 { 338 int len, read_buf_len, start; 339 char buf[16] = {}; 340 341 read_buf_len = read_one_char ? 1 : 16; 342 start = 0; 343 while ((len = read(iter_fd, buf + start, read_buf_len)) > 0) { 344 start += len; 345 if (CHECK(start >= 16, "read", "read len %d\n", len)) 346 return -1; 347 read_buf_len = read_one_char ? 1 : 16 - start; 348 } 349 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno))) 350 return -1; 351 352 if (!ASSERT_STREQ(buf, expected, "read")) 353 return -1; 354 355 return 0; 356 } 357 358 static void test_anon_iter(bool read_one_char) 359 { 360 struct bpf_iter_test_kern1 *skel; 361 struct bpf_link *link; 362 int iter_fd, err; 363 364 skel = bpf_iter_test_kern1__open_and_load(); 365 if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern1__open_and_load")) 366 return; 367 368 err = bpf_iter_test_kern1__attach(skel); 369 if (!ASSERT_OK(err, "bpf_iter_test_kern1__attach")) { 370 goto out; 371 } 372 373 link = skel->links.dump_task; 374 iter_fd = bpf_iter_create(bpf_link__fd(link)); 375 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 376 goto out; 377 378 do_read_with_fd(iter_fd, "abcd", read_one_char); 379 close(iter_fd); 380 381 out: 382 bpf_iter_test_kern1__destroy(skel); 383 } 384 385 static int do_read(const char *path, const char *expected) 386 { 387 int err, iter_fd; 388 389 iter_fd = open(path, O_RDONLY); 390 if (CHECK(iter_fd < 0, "open", "open %s failed: %s\n", 391 path, strerror(errno))) 392 return -1; 393 394 err = do_read_with_fd(iter_fd, expected, false); 395 close(iter_fd); 396 return err; 397 } 398 399 static void test_file_iter(void) 400 { 401 const char *path = "/sys/fs/bpf/bpf_iter_test1"; 402 struct bpf_iter_test_kern1 *skel1; 403 struct bpf_iter_test_kern2 *skel2; 404 struct bpf_link *link; 405 int err; 406 407 skel1 = bpf_iter_test_kern1__open_and_load(); 408 if (!ASSERT_OK_PTR(skel1, "bpf_iter_test_kern1__open_and_load")) 409 return; 410 411 link = bpf_program__attach_iter(skel1->progs.dump_task, NULL); 412 if (!ASSERT_OK_PTR(link, "attach_iter")) 413 goto out; 414 415 /* unlink this path if it exists. */ 416 unlink(path); 417 418 err = bpf_link__pin(link, path); 419 if (CHECK(err, "pin_iter", "pin_iter to %s failed: %d\n", path, err)) 420 goto free_link; 421 422 err = do_read(path, "abcd"); 423 if (err) 424 goto unlink_path; 425 426 /* file based iterator seems working fine. Let us a link update 427 * of the underlying link and `cat` the iterator again, its content 428 * should change. 429 */ 430 skel2 = bpf_iter_test_kern2__open_and_load(); 431 if (!ASSERT_OK_PTR(skel2, "bpf_iter_test_kern2__open_and_load")) 432 goto unlink_path; 433 434 err = bpf_link__update_program(link, skel2->progs.dump_task); 435 if (!ASSERT_OK(err, "update_prog")) 436 goto destroy_skel2; 437 438 do_read(path, "ABCD"); 439 440 destroy_skel2: 441 bpf_iter_test_kern2__destroy(skel2); 442 unlink_path: 443 unlink(path); 444 free_link: 445 bpf_link__destroy(link); 446 out: 447 bpf_iter_test_kern1__destroy(skel1); 448 } 449 450 static void test_overflow(bool test_e2big_overflow, bool ret1) 451 { 452 __u32 map_info_len, total_read_len, expected_read_len; 453 int err, iter_fd, map1_fd, map2_fd, len; 454 struct bpf_map_info map_info = {}; 455 struct bpf_iter_test_kern4 *skel; 456 struct bpf_link *link; 457 __u32 iter_size; 458 char *buf; 459 460 skel = bpf_iter_test_kern4__open(); 461 if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern4__open")) 462 return; 463 464 /* create two maps: bpf program will only do bpf_seq_write 465 * for these two maps. The goal is one map output almost 466 * fills seq_file buffer and then the other will trigger 467 * overflow and needs restart. 468 */ 469 map1_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL); 470 if (CHECK(map1_fd < 0, "bpf_map_create", 471 "map_creation failed: %s\n", strerror(errno))) 472 goto out; 473 map2_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL); 474 if (CHECK(map2_fd < 0, "bpf_map_create", 475 "map_creation failed: %s\n", strerror(errno))) 476 goto free_map1; 477 478 /* bpf_seq_printf kernel buffer is 8 pages, so one map 479 * bpf_seq_write will mostly fill it, and the other map 480 * will partially fill and then trigger overflow and need 481 * bpf_seq_read restart. 482 */ 483 iter_size = sysconf(_SC_PAGE_SIZE) << 3; 484 485 if (test_e2big_overflow) { 486 skel->rodata->print_len = (iter_size + 8) / 8; 487 expected_read_len = 2 * (iter_size + 8); 488 } else if (!ret1) { 489 skel->rodata->print_len = (iter_size - 8) / 8; 490 expected_read_len = 2 * (iter_size - 8); 491 } else { 492 skel->rodata->print_len = 1; 493 expected_read_len = 2 * 8; 494 } 495 skel->rodata->ret1 = ret1; 496 497 if (!ASSERT_OK(bpf_iter_test_kern4__load(skel), 498 "bpf_iter_test_kern4__load")) 499 goto free_map2; 500 501 /* setup filtering map_id in bpf program */ 502 map_info_len = sizeof(map_info); 503 err = bpf_obj_get_info_by_fd(map1_fd, &map_info, &map_info_len); 504 if (CHECK(err, "get_map_info", "get map info failed: %s\n", 505 strerror(errno))) 506 goto free_map2; 507 skel->bss->map1_id = map_info.id; 508 509 err = bpf_obj_get_info_by_fd(map2_fd, &map_info, &map_info_len); 510 if (CHECK(err, "get_map_info", "get map info failed: %s\n", 511 strerror(errno))) 512 goto free_map2; 513 skel->bss->map2_id = map_info.id; 514 515 link = bpf_program__attach_iter(skel->progs.dump_bpf_map, NULL); 516 if (!ASSERT_OK_PTR(link, "attach_iter")) 517 goto free_map2; 518 519 iter_fd = bpf_iter_create(bpf_link__fd(link)); 520 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 521 goto free_link; 522 523 buf = malloc(expected_read_len); 524 if (!buf) 525 goto close_iter; 526 527 /* do read */ 528 total_read_len = 0; 529 if (test_e2big_overflow) { 530 while ((len = read(iter_fd, buf, expected_read_len)) > 0) 531 total_read_len += len; 532 533 CHECK(len != -1 || errno != E2BIG, "read", 534 "expected ret -1, errno E2BIG, but get ret %d, error %s\n", 535 len, strerror(errno)); 536 goto free_buf; 537 } else if (!ret1) { 538 while ((len = read(iter_fd, buf, expected_read_len)) > 0) 539 total_read_len += len; 540 541 if (CHECK(len < 0, "read", "read failed: %s\n", 542 strerror(errno))) 543 goto free_buf; 544 } else { 545 do { 546 len = read(iter_fd, buf, expected_read_len); 547 if (len > 0) 548 total_read_len += len; 549 } while (len > 0 || len == -EAGAIN); 550 551 if (CHECK(len < 0, "read", "read failed: %s\n", 552 strerror(errno))) 553 goto free_buf; 554 } 555 556 if (!ASSERT_EQ(total_read_len, expected_read_len, "read")) 557 goto free_buf; 558 559 if (!ASSERT_EQ(skel->bss->map1_accessed, 1, "map1_accessed")) 560 goto free_buf; 561 562 if (!ASSERT_EQ(skel->bss->map2_accessed, 2, "map2_accessed")) 563 goto free_buf; 564 565 ASSERT_EQ(skel->bss->map2_seqnum1, skel->bss->map2_seqnum2, "map2_seqnum"); 566 567 free_buf: 568 free(buf); 569 close_iter: 570 close(iter_fd); 571 free_link: 572 bpf_link__destroy(link); 573 free_map2: 574 close(map2_fd); 575 free_map1: 576 close(map1_fd); 577 out: 578 bpf_iter_test_kern4__destroy(skel); 579 } 580 581 static void test_bpf_hash_map(void) 582 { 583 __u32 expected_key_a = 0, expected_key_b = 0; 584 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 585 struct bpf_iter_bpf_hash_map *skel; 586 int err, i, len, map_fd, iter_fd; 587 union bpf_iter_link_info linfo; 588 __u64 val, expected_val = 0; 589 struct bpf_link *link; 590 struct key_t { 591 int a; 592 int b; 593 int c; 594 } key; 595 char buf[64]; 596 597 skel = bpf_iter_bpf_hash_map__open(); 598 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_hash_map__open")) 599 return; 600 601 skel->bss->in_test_mode = true; 602 603 err = bpf_iter_bpf_hash_map__load(skel); 604 if (!ASSERT_OK(err, "bpf_iter_bpf_hash_map__load")) 605 goto out; 606 607 /* iterator with hashmap2 and hashmap3 should fail */ 608 memset(&linfo, 0, sizeof(linfo)); 609 linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap2); 610 opts.link_info = &linfo; 611 opts.link_info_len = sizeof(linfo); 612 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts); 613 if (!ASSERT_ERR_PTR(link, "attach_iter")) 614 goto out; 615 616 linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap3); 617 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts); 618 if (!ASSERT_ERR_PTR(link, "attach_iter")) 619 goto out; 620 621 /* hashmap1 should be good, update map values here */ 622 map_fd = bpf_map__fd(skel->maps.hashmap1); 623 for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) { 624 key.a = i + 1; 625 key.b = i + 2; 626 key.c = i + 3; 627 val = i + 4; 628 expected_key_a += key.a; 629 expected_key_b += key.b; 630 expected_val += val; 631 632 err = bpf_map_update_elem(map_fd, &key, &val, BPF_ANY); 633 if (!ASSERT_OK(err, "map_update")) 634 goto out; 635 } 636 637 linfo.map.map_fd = map_fd; 638 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts); 639 if (!ASSERT_OK_PTR(link, "attach_iter")) 640 goto out; 641 642 iter_fd = bpf_iter_create(bpf_link__fd(link)); 643 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 644 goto free_link; 645 646 /* do some tests */ 647 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 648 ; 649 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno))) 650 goto close_iter; 651 652 /* test results */ 653 if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a")) 654 goto close_iter; 655 if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b")) 656 goto close_iter; 657 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum")) 658 goto close_iter; 659 660 close_iter: 661 close(iter_fd); 662 free_link: 663 bpf_link__destroy(link); 664 out: 665 bpf_iter_bpf_hash_map__destroy(skel); 666 } 667 668 static void test_bpf_percpu_hash_map(void) 669 { 670 __u32 expected_key_a = 0, expected_key_b = 0; 671 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 672 struct bpf_iter_bpf_percpu_hash_map *skel; 673 int err, i, j, len, map_fd, iter_fd; 674 union bpf_iter_link_info linfo; 675 __u32 expected_val = 0; 676 struct bpf_link *link; 677 struct key_t { 678 int a; 679 int b; 680 int c; 681 } key; 682 char buf[64]; 683 void *val; 684 685 skel = bpf_iter_bpf_percpu_hash_map__open(); 686 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__open")) 687 return; 688 689 skel->rodata->num_cpus = bpf_num_possible_cpus(); 690 val = malloc(8 * bpf_num_possible_cpus()); 691 692 err = bpf_iter_bpf_percpu_hash_map__load(skel); 693 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__load")) 694 goto out; 695 696 /* update map values here */ 697 map_fd = bpf_map__fd(skel->maps.hashmap1); 698 for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) { 699 key.a = i + 1; 700 key.b = i + 2; 701 key.c = i + 3; 702 expected_key_a += key.a; 703 expected_key_b += key.b; 704 705 for (j = 0; j < bpf_num_possible_cpus(); j++) { 706 *(__u32 *)(val + j * 8) = i + j; 707 expected_val += i + j; 708 } 709 710 err = bpf_map_update_elem(map_fd, &key, val, BPF_ANY); 711 if (!ASSERT_OK(err, "map_update")) 712 goto out; 713 } 714 715 memset(&linfo, 0, sizeof(linfo)); 716 linfo.map.map_fd = map_fd; 717 opts.link_info = &linfo; 718 opts.link_info_len = sizeof(linfo); 719 link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_hash_map, &opts); 720 if (!ASSERT_OK_PTR(link, "attach_iter")) 721 goto out; 722 723 iter_fd = bpf_iter_create(bpf_link__fd(link)); 724 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 725 goto free_link; 726 727 /* do some tests */ 728 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 729 ; 730 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno))) 731 goto close_iter; 732 733 /* test results */ 734 if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a")) 735 goto close_iter; 736 if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b")) 737 goto close_iter; 738 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum")) 739 goto close_iter; 740 741 close_iter: 742 close(iter_fd); 743 free_link: 744 bpf_link__destroy(link); 745 out: 746 bpf_iter_bpf_percpu_hash_map__destroy(skel); 747 free(val); 748 } 749 750 static void test_bpf_array_map(void) 751 { 752 __u64 val, expected_val = 0, res_first_val, first_val = 0; 753 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 754 __u32 expected_key = 0, res_first_key; 755 struct bpf_iter_bpf_array_map *skel; 756 union bpf_iter_link_info linfo; 757 int err, i, map_fd, iter_fd; 758 struct bpf_link *link; 759 char buf[64] = {}; 760 int len, start; 761 762 skel = bpf_iter_bpf_array_map__open_and_load(); 763 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load")) 764 return; 765 766 map_fd = bpf_map__fd(skel->maps.arraymap1); 767 for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) { 768 val = i + 4; 769 expected_key += i; 770 expected_val += val; 771 772 if (i == 0) 773 first_val = val; 774 775 err = bpf_map_update_elem(map_fd, &i, &val, BPF_ANY); 776 if (!ASSERT_OK(err, "map_update")) 777 goto out; 778 } 779 780 memset(&linfo, 0, sizeof(linfo)); 781 linfo.map.map_fd = map_fd; 782 opts.link_info = &linfo; 783 opts.link_info_len = sizeof(linfo); 784 link = bpf_program__attach_iter(skel->progs.dump_bpf_array_map, &opts); 785 if (!ASSERT_OK_PTR(link, "attach_iter")) 786 goto out; 787 788 iter_fd = bpf_iter_create(bpf_link__fd(link)); 789 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 790 goto free_link; 791 792 /* do some tests */ 793 start = 0; 794 while ((len = read(iter_fd, buf + start, sizeof(buf) - start)) > 0) 795 start += len; 796 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno))) 797 goto close_iter; 798 799 /* test results */ 800 res_first_key = *(__u32 *)buf; 801 res_first_val = *(__u64 *)(buf + sizeof(__u32)); 802 if (CHECK(res_first_key != 0 || res_first_val != first_val, 803 "bpf_seq_write", 804 "seq_write failure: first key %u vs expected 0, " 805 " first value %llu vs expected %llu\n", 806 res_first_key, res_first_val, first_val)) 807 goto close_iter; 808 809 if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum")) 810 goto close_iter; 811 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum")) 812 goto close_iter; 813 814 for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) { 815 err = bpf_map_lookup_elem(map_fd, &i, &val); 816 if (!ASSERT_OK(err, "map_lookup")) 817 goto out; 818 if (!ASSERT_EQ(i, val, "invalid_val")) 819 goto out; 820 } 821 822 close_iter: 823 close(iter_fd); 824 free_link: 825 bpf_link__destroy(link); 826 out: 827 bpf_iter_bpf_array_map__destroy(skel); 828 } 829 830 static void test_bpf_percpu_array_map(void) 831 { 832 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 833 struct bpf_iter_bpf_percpu_array_map *skel; 834 __u32 expected_key = 0, expected_val = 0; 835 union bpf_iter_link_info linfo; 836 int err, i, j, map_fd, iter_fd; 837 struct bpf_link *link; 838 char buf[64]; 839 void *val; 840 int len; 841 842 skel = bpf_iter_bpf_percpu_array_map__open(); 843 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__open")) 844 return; 845 846 skel->rodata->num_cpus = bpf_num_possible_cpus(); 847 val = malloc(8 * bpf_num_possible_cpus()); 848 849 err = bpf_iter_bpf_percpu_array_map__load(skel); 850 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__load")) 851 goto out; 852 853 /* update map values here */ 854 map_fd = bpf_map__fd(skel->maps.arraymap1); 855 for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) { 856 expected_key += i; 857 858 for (j = 0; j < bpf_num_possible_cpus(); j++) { 859 *(__u32 *)(val + j * 8) = i + j; 860 expected_val += i + j; 861 } 862 863 err = bpf_map_update_elem(map_fd, &i, val, BPF_ANY); 864 if (!ASSERT_OK(err, "map_update")) 865 goto out; 866 } 867 868 memset(&linfo, 0, sizeof(linfo)); 869 linfo.map.map_fd = map_fd; 870 opts.link_info = &linfo; 871 opts.link_info_len = sizeof(linfo); 872 link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_array_map, &opts); 873 if (!ASSERT_OK_PTR(link, "attach_iter")) 874 goto out; 875 876 iter_fd = bpf_iter_create(bpf_link__fd(link)); 877 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 878 goto free_link; 879 880 /* do some tests */ 881 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 882 ; 883 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno))) 884 goto close_iter; 885 886 /* test results */ 887 if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum")) 888 goto close_iter; 889 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum")) 890 goto close_iter; 891 892 close_iter: 893 close(iter_fd); 894 free_link: 895 bpf_link__destroy(link); 896 out: 897 bpf_iter_bpf_percpu_array_map__destroy(skel); 898 free(val); 899 } 900 901 /* An iterator program deletes all local storage in a map. */ 902 static void test_bpf_sk_storage_delete(void) 903 { 904 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 905 struct bpf_iter_bpf_sk_storage_helpers *skel; 906 union bpf_iter_link_info linfo; 907 int err, len, map_fd, iter_fd; 908 struct bpf_link *link; 909 int sock_fd = -1; 910 __u32 val = 42; 911 char buf[64]; 912 913 skel = bpf_iter_bpf_sk_storage_helpers__open_and_load(); 914 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load")) 915 return; 916 917 map_fd = bpf_map__fd(skel->maps.sk_stg_map); 918 919 sock_fd = socket(AF_INET6, SOCK_STREAM, 0); 920 if (!ASSERT_GE(sock_fd, 0, "socket")) 921 goto out; 922 err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST); 923 if (!ASSERT_OK(err, "map_update")) 924 goto out; 925 926 memset(&linfo, 0, sizeof(linfo)); 927 linfo.map.map_fd = map_fd; 928 opts.link_info = &linfo; 929 opts.link_info_len = sizeof(linfo); 930 link = bpf_program__attach_iter(skel->progs.delete_bpf_sk_storage_map, 931 &opts); 932 if (!ASSERT_OK_PTR(link, "attach_iter")) 933 goto out; 934 935 iter_fd = bpf_iter_create(bpf_link__fd(link)); 936 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 937 goto free_link; 938 939 /* do some tests */ 940 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 941 ; 942 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno))) 943 goto close_iter; 944 945 /* test results */ 946 err = bpf_map_lookup_elem(map_fd, &sock_fd, &val); 947 if (CHECK(!err || errno != ENOENT, "bpf_map_lookup_elem", 948 "map value wasn't deleted (err=%d, errno=%d)\n", err, errno)) 949 goto close_iter; 950 951 close_iter: 952 close(iter_fd); 953 free_link: 954 bpf_link__destroy(link); 955 out: 956 if (sock_fd >= 0) 957 close(sock_fd); 958 bpf_iter_bpf_sk_storage_helpers__destroy(skel); 959 } 960 961 /* This creates a socket and its local storage. It then runs a task_iter BPF 962 * program that replaces the existing socket local storage with the tgid of the 963 * only task owning a file descriptor to this socket, this process, prog_tests. 964 * It then runs a tcp socket iterator that negates the value in the existing 965 * socket local storage, the test verifies that the resulting value is -pid. 966 */ 967 static void test_bpf_sk_storage_get(void) 968 { 969 struct bpf_iter_bpf_sk_storage_helpers *skel; 970 int err, map_fd, val = -1; 971 int sock_fd = -1; 972 973 skel = bpf_iter_bpf_sk_storage_helpers__open_and_load(); 974 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load")) 975 return; 976 977 sock_fd = socket(AF_INET6, SOCK_STREAM, 0); 978 if (!ASSERT_GE(sock_fd, 0, "socket")) 979 goto out; 980 981 err = listen(sock_fd, 1); 982 if (!ASSERT_OK(err, "listen")) 983 goto close_socket; 984 985 map_fd = bpf_map__fd(skel->maps.sk_stg_map); 986 987 err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST); 988 if (!ASSERT_OK(err, "bpf_map_update_elem")) 989 goto close_socket; 990 991 do_dummy_read(skel->progs.fill_socket_owner); 992 993 err = bpf_map_lookup_elem(map_fd, &sock_fd, &val); 994 if (CHECK(err || val != getpid(), "bpf_map_lookup_elem", 995 "map value wasn't set correctly (expected %d, got %d, err=%d)\n", 996 getpid(), val, err)) 997 goto close_socket; 998 999 do_dummy_read(skel->progs.negate_socket_local_storage); 1000 1001 err = bpf_map_lookup_elem(map_fd, &sock_fd, &val); 1002 CHECK(err || val != -getpid(), "bpf_map_lookup_elem", 1003 "map value wasn't set correctly (expected %d, got %d, err=%d)\n", 1004 -getpid(), val, err); 1005 1006 close_socket: 1007 close(sock_fd); 1008 out: 1009 bpf_iter_bpf_sk_storage_helpers__destroy(skel); 1010 } 1011 1012 static void test_bpf_sk_storage_map(void) 1013 { 1014 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 1015 int err, i, len, map_fd, iter_fd, num_sockets; 1016 struct bpf_iter_bpf_sk_storage_map *skel; 1017 union bpf_iter_link_info linfo; 1018 int sock_fd[3] = {-1, -1, -1}; 1019 __u32 val, expected_val = 0; 1020 struct bpf_link *link; 1021 char buf[64]; 1022 1023 skel = bpf_iter_bpf_sk_storage_map__open_and_load(); 1024 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_map__open_and_load")) 1025 return; 1026 1027 map_fd = bpf_map__fd(skel->maps.sk_stg_map); 1028 num_sockets = ARRAY_SIZE(sock_fd); 1029 for (i = 0; i < num_sockets; i++) { 1030 sock_fd[i] = socket(AF_INET6, SOCK_STREAM, 0); 1031 if (!ASSERT_GE(sock_fd[i], 0, "socket")) 1032 goto out; 1033 1034 val = i + 1; 1035 expected_val += val; 1036 1037 err = bpf_map_update_elem(map_fd, &sock_fd[i], &val, 1038 BPF_NOEXIST); 1039 if (!ASSERT_OK(err, "map_update")) 1040 goto out; 1041 } 1042 1043 memset(&linfo, 0, sizeof(linfo)); 1044 linfo.map.map_fd = map_fd; 1045 opts.link_info = &linfo; 1046 opts.link_info_len = sizeof(linfo); 1047 link = bpf_program__attach_iter(skel->progs.dump_bpf_sk_storage_map, &opts); 1048 if (!ASSERT_OK_PTR(link, "attach_iter")) 1049 goto out; 1050 1051 iter_fd = bpf_iter_create(bpf_link__fd(link)); 1052 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 1053 goto free_link; 1054 1055 /* do some tests */ 1056 while ((len = read(iter_fd, buf, sizeof(buf))) > 0) 1057 ; 1058 if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno))) 1059 goto close_iter; 1060 1061 /* test results */ 1062 if (!ASSERT_EQ(skel->bss->ipv6_sk_count, num_sockets, "ipv6_sk_count")) 1063 goto close_iter; 1064 1065 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum")) 1066 goto close_iter; 1067 1068 close_iter: 1069 close(iter_fd); 1070 free_link: 1071 bpf_link__destroy(link); 1072 out: 1073 for (i = 0; i < num_sockets; i++) { 1074 if (sock_fd[i] >= 0) 1075 close(sock_fd[i]); 1076 } 1077 bpf_iter_bpf_sk_storage_map__destroy(skel); 1078 } 1079 1080 static void test_rdonly_buf_out_of_bound(void) 1081 { 1082 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); 1083 struct bpf_iter_test_kern5 *skel; 1084 union bpf_iter_link_info linfo; 1085 struct bpf_link *link; 1086 1087 skel = bpf_iter_test_kern5__open_and_load(); 1088 if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern5__open_and_load")) 1089 return; 1090 1091 memset(&linfo, 0, sizeof(linfo)); 1092 linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap1); 1093 opts.link_info = &linfo; 1094 opts.link_info_len = sizeof(linfo); 1095 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts); 1096 if (!ASSERT_ERR_PTR(link, "attach_iter")) 1097 bpf_link__destroy(link); 1098 1099 bpf_iter_test_kern5__destroy(skel); 1100 } 1101 1102 static void test_buf_neg_offset(void) 1103 { 1104 struct bpf_iter_test_kern6 *skel; 1105 1106 skel = bpf_iter_test_kern6__open_and_load(); 1107 if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern6__open_and_load")) 1108 bpf_iter_test_kern6__destroy(skel); 1109 } 1110 1111 static void test_link_iter(void) 1112 { 1113 struct bpf_iter_bpf_link *skel; 1114 1115 skel = bpf_iter_bpf_link__open_and_load(); 1116 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_link__open_and_load")) 1117 return; 1118 1119 do_dummy_read(skel->progs.dump_bpf_link); 1120 1121 bpf_iter_bpf_link__destroy(skel); 1122 } 1123 1124 static void test_ksym_iter(void) 1125 { 1126 struct bpf_iter_ksym *skel; 1127 1128 skel = bpf_iter_ksym__open_and_load(); 1129 if (!ASSERT_OK_PTR(skel, "bpf_iter_ksym__open_and_load")) 1130 return; 1131 1132 do_dummy_read(skel->progs.dump_ksym); 1133 1134 bpf_iter_ksym__destroy(skel); 1135 } 1136 1137 #define CMP_BUFFER_SIZE 1024 1138 static char task_vma_output[CMP_BUFFER_SIZE]; 1139 static char proc_maps_output[CMP_BUFFER_SIZE]; 1140 1141 /* remove \0 and \t from str, and only keep the first line */ 1142 static void str_strip_first_line(char *str) 1143 { 1144 char *dst = str, *src = str; 1145 1146 do { 1147 if (*src == ' ' || *src == '\t') 1148 src++; 1149 else 1150 *(dst++) = *(src++); 1151 1152 } while (*src != '\0' && *src != '\n'); 1153 1154 *dst = '\0'; 1155 } 1156 1157 static void test_task_vma(void) 1158 { 1159 int err, iter_fd = -1, proc_maps_fd = -1; 1160 struct bpf_iter_task_vma *skel; 1161 int len, read_size = 4; 1162 char maps_path[64]; 1163 1164 skel = bpf_iter_task_vma__open(); 1165 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vma__open")) 1166 return; 1167 1168 skel->bss->pid = getpid(); 1169 1170 err = bpf_iter_task_vma__load(skel); 1171 if (!ASSERT_OK(err, "bpf_iter_task_vma__load")) 1172 goto out; 1173 1174 skel->links.proc_maps = bpf_program__attach_iter( 1175 skel->progs.proc_maps, NULL); 1176 1177 if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) { 1178 skel->links.proc_maps = NULL; 1179 goto out; 1180 } 1181 1182 iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps)); 1183 if (!ASSERT_GE(iter_fd, 0, "create_iter")) 1184 goto out; 1185 1186 /* Read CMP_BUFFER_SIZE (1kB) from bpf_iter. Read in small chunks 1187 * to trigger seq_file corner cases. 1188 */ 1189 len = 0; 1190 while (len < CMP_BUFFER_SIZE) { 1191 err = read_fd_into_buffer(iter_fd, task_vma_output + len, 1192 MIN(read_size, CMP_BUFFER_SIZE - len)); 1193 if (!err) 1194 break; 1195 if (!ASSERT_GE(err, 0, "read_iter_fd")) 1196 goto out; 1197 len += err; 1198 } 1199 1200 /* read CMP_BUFFER_SIZE (1kB) from /proc/pid/maps */ 1201 snprintf(maps_path, 64, "/proc/%u/maps", skel->bss->pid); 1202 proc_maps_fd = open(maps_path, O_RDONLY); 1203 if (!ASSERT_GE(proc_maps_fd, 0, "open_proc_maps")) 1204 goto out; 1205 err = read_fd_into_buffer(proc_maps_fd, proc_maps_output, CMP_BUFFER_SIZE); 1206 if (!ASSERT_GE(err, 0, "read_prog_maps_fd")) 1207 goto out; 1208 1209 /* strip and compare the first line of the two files */ 1210 str_strip_first_line(task_vma_output); 1211 str_strip_first_line(proc_maps_output); 1212 1213 ASSERT_STREQ(task_vma_output, proc_maps_output, "compare_output"); 1214 out: 1215 close(proc_maps_fd); 1216 close(iter_fd); 1217 bpf_iter_task_vma__destroy(skel); 1218 } 1219 1220 void test_bpf_iter(void) 1221 { 1222 if (test__start_subtest("btf_id_or_null")) 1223 test_btf_id_or_null(); 1224 if (test__start_subtest("ipv6_route")) 1225 test_ipv6_route(); 1226 if (test__start_subtest("netlink")) 1227 test_netlink(); 1228 if (test__start_subtest("bpf_map")) 1229 test_bpf_map(); 1230 if (test__start_subtest("task")) 1231 test_task(); 1232 if (test__start_subtest("task_sleepable")) 1233 test_task_sleepable(); 1234 if (test__start_subtest("task_stack")) 1235 test_task_stack(); 1236 if (test__start_subtest("task_file")) 1237 test_task_file(); 1238 if (test__start_subtest("task_vma")) 1239 test_task_vma(); 1240 if (test__start_subtest("task_btf")) 1241 test_task_btf(); 1242 if (test__start_subtest("tcp4")) 1243 test_tcp4(); 1244 if (test__start_subtest("tcp6")) 1245 test_tcp6(); 1246 if (test__start_subtest("udp4")) 1247 test_udp4(); 1248 if (test__start_subtest("udp6")) 1249 test_udp6(); 1250 if (test__start_subtest("unix")) 1251 test_unix(); 1252 if (test__start_subtest("anon")) 1253 test_anon_iter(false); 1254 if (test__start_subtest("anon-read-one-char")) 1255 test_anon_iter(true); 1256 if (test__start_subtest("file")) 1257 test_file_iter(); 1258 if (test__start_subtest("overflow")) 1259 test_overflow(false, false); 1260 if (test__start_subtest("overflow-e2big")) 1261 test_overflow(true, false); 1262 if (test__start_subtest("prog-ret-1")) 1263 test_overflow(false, true); 1264 if (test__start_subtest("bpf_hash_map")) 1265 test_bpf_hash_map(); 1266 if (test__start_subtest("bpf_percpu_hash_map")) 1267 test_bpf_percpu_hash_map(); 1268 if (test__start_subtest("bpf_array_map")) 1269 test_bpf_array_map(); 1270 if (test__start_subtest("bpf_percpu_array_map")) 1271 test_bpf_percpu_array_map(); 1272 if (test__start_subtest("bpf_sk_storage_map")) 1273 test_bpf_sk_storage_map(); 1274 if (test__start_subtest("bpf_sk_storage_delete")) 1275 test_bpf_sk_storage_delete(); 1276 if (test__start_subtest("bpf_sk_storage_get")) 1277 test_bpf_sk_storage_get(); 1278 if (test__start_subtest("rdonly-buf-out-of-bound")) 1279 test_rdonly_buf_out_of_bound(); 1280 if (test__start_subtest("buf-neg-offset")) 1281 test_buf_neg_offset(); 1282 if (test__start_subtest("link-iter")) 1283 test_link_iter(); 1284 if (test__start_subtest("ksym")) 1285 test_ksym_iter(); 1286 } 1287