1 /* Copyright (c) 2017 Facebook 2 * 3 * This program is free software; you can redistribute it and/or 4 * modify it under the terms of version 2 of the GNU General Public 5 * License as published by the Free Software Foundation. 6 */ 7 #include <stdio.h> 8 #include <unistd.h> 9 #include <errno.h> 10 #include <string.h> 11 #include <assert.h> 12 #include <stdlib.h> 13 #include <time.h> 14 15 #include <linux/types.h> 16 typedef __u16 __sum16; 17 #include <arpa/inet.h> 18 #include <linux/if_ether.h> 19 #include <linux/if_packet.h> 20 #include <linux/ip.h> 21 #include <linux/ipv6.h> 22 #include <linux/tcp.h> 23 #include <linux/filter.h> 24 #include <linux/perf_event.h> 25 #include <linux/unistd.h> 26 27 #include <sys/ioctl.h> 28 #include <sys/wait.h> 29 #include <sys/types.h> 30 #include <fcntl.h> 31 32 #include <linux/bpf.h> 33 #include <linux/err.h> 34 #include <bpf/bpf.h> 35 #include <bpf/libbpf.h> 36 37 #include "test_iptunnel_common.h" 38 #include "bpf_util.h" 39 #include "bpf_endian.h" 40 #include "bpf_rlimit.h" 41 #include "trace_helpers.h" 42 #include "flow_dissector_load.h" 43 44 static int error_cnt, pass_cnt; 45 static bool jit_enabled; 46 47 #define MAGIC_BYTES 123 48 49 /* ipv4 test vector */ 50 static struct { 51 struct ethhdr eth; 52 struct iphdr iph; 53 struct tcphdr tcp; 54 } __packed pkt_v4 = { 55 .eth.h_proto = __bpf_constant_htons(ETH_P_IP), 56 .iph.ihl = 5, 57 .iph.protocol = IPPROTO_TCP, 58 .iph.tot_len = __bpf_constant_htons(MAGIC_BYTES), 59 .tcp.urg_ptr = 123, 60 .tcp.doff = 5, 61 }; 62 63 /* ipv6 test vector */ 64 static struct { 65 struct ethhdr eth; 66 struct ipv6hdr iph; 67 struct tcphdr tcp; 68 } __packed pkt_v6 = { 69 .eth.h_proto = __bpf_constant_htons(ETH_P_IPV6), 70 .iph.nexthdr = IPPROTO_TCP, 71 .iph.payload_len = __bpf_constant_htons(MAGIC_BYTES), 72 .tcp.urg_ptr = 123, 73 .tcp.doff = 5, 74 }; 75 76 #define _CHECK(condition, tag, duration, format...) ({ \ 77 int __ret = !!(condition); \ 78 if (__ret) { \ 79 error_cnt++; \ 80 printf("%s:FAIL:%s ", __func__, tag); \ 81 printf(format); \ 82 } else { \ 83 pass_cnt++; \ 84 printf("%s:PASS:%s %d nsec\n", __func__, tag, duration);\ 85 } \ 86 __ret; \ 87 }) 88 89 #define CHECK(condition, tag, format...) \ 90 _CHECK(condition, tag, duration, format) 91 #define CHECK_ATTR(condition, tag, format...) \ 92 _CHECK(condition, tag, tattr.duration, format) 93 94 static int bpf_find_map(const char *test, struct bpf_object *obj, 95 const char *name) 96 { 97 struct bpf_map *map; 98 99 map = bpf_object__find_map_by_name(obj, name); 100 if (!map) { 101 printf("%s:FAIL:map '%s' not found\n", test, name); 102 error_cnt++; 103 return -1; 104 } 105 return bpf_map__fd(map); 106 } 107 108 static void test_pkt_access(void) 109 { 110 const char *file = "./test_pkt_access.o"; 111 struct bpf_object *obj; 112 __u32 duration, retval; 113 int err, prog_fd; 114 115 err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); 116 if (err) { 117 error_cnt++; 118 return; 119 } 120 121 err = bpf_prog_test_run(prog_fd, 100000, &pkt_v4, sizeof(pkt_v4), 122 NULL, NULL, &retval, &duration); 123 CHECK(err || retval, "ipv4", 124 "err %d errno %d retval %d duration %d\n", 125 err, errno, retval, duration); 126 127 err = bpf_prog_test_run(prog_fd, 100000, &pkt_v6, sizeof(pkt_v6), 128 NULL, NULL, &retval, &duration); 129 CHECK(err || retval, "ipv6", 130 "err %d errno %d retval %d duration %d\n", 131 err, errno, retval, duration); 132 bpf_object__close(obj); 133 } 134 135 static void test_prog_run_xattr(void) 136 { 137 const char *file = "./test_pkt_access.o"; 138 struct bpf_object *obj; 139 char buf[10]; 140 int err; 141 struct bpf_prog_test_run_attr tattr = { 142 .repeat = 1, 143 .data_in = &pkt_v4, 144 .data_size_in = sizeof(pkt_v4), 145 .data_out = buf, 146 .data_size_out = 5, 147 }; 148 149 err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, 150 &tattr.prog_fd); 151 if (CHECK_ATTR(err, "load", "err %d errno %d\n", err, errno)) 152 return; 153 154 memset(buf, 0, sizeof(buf)); 155 156 err = bpf_prog_test_run_xattr(&tattr); 157 CHECK_ATTR(err != -1 || errno != ENOSPC || tattr.retval, "run", 158 "err %d errno %d retval %d\n", err, errno, tattr.retval); 159 160 CHECK_ATTR(tattr.data_size_out != sizeof(pkt_v4), "data_size_out", 161 "incorrect output size, want %lu have %u\n", 162 sizeof(pkt_v4), tattr.data_size_out); 163 164 CHECK_ATTR(buf[5] != 0, "overflow", 165 "BPF_PROG_TEST_RUN ignored size hint\n"); 166 167 tattr.data_out = NULL; 168 tattr.data_size_out = 0; 169 errno = 0; 170 171 err = bpf_prog_test_run_xattr(&tattr); 172 CHECK_ATTR(err || errno || tattr.retval, "run_no_output", 173 "err %d errno %d retval %d\n", err, errno, tattr.retval); 174 175 tattr.data_size_out = 1; 176 err = bpf_prog_test_run_xattr(&tattr); 177 CHECK_ATTR(err != -EINVAL, "run_wrong_size_out", "err %d\n", err); 178 179 bpf_object__close(obj); 180 } 181 182 static void test_xdp(void) 183 { 184 struct vip key4 = {.protocol = 6, .family = AF_INET}; 185 struct vip key6 = {.protocol = 6, .family = AF_INET6}; 186 struct iptnl_info value4 = {.family = AF_INET}; 187 struct iptnl_info value6 = {.family = AF_INET6}; 188 const char *file = "./test_xdp.o"; 189 struct bpf_object *obj; 190 char buf[128]; 191 struct ipv6hdr *iph6 = (void *)buf + sizeof(struct ethhdr); 192 struct iphdr *iph = (void *)buf + sizeof(struct ethhdr); 193 __u32 duration, retval, size; 194 int err, prog_fd, map_fd; 195 196 err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd); 197 if (err) { 198 error_cnt++; 199 return; 200 } 201 202 map_fd = bpf_find_map(__func__, obj, "vip2tnl"); 203 if (map_fd < 0) 204 goto out; 205 bpf_map_update_elem(map_fd, &key4, &value4, 0); 206 bpf_map_update_elem(map_fd, &key6, &value6, 0); 207 208 err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4), 209 buf, &size, &retval, &duration); 210 211 CHECK(err || retval != XDP_TX || size != 74 || 212 iph->protocol != IPPROTO_IPIP, "ipv4", 213 "err %d errno %d retval %d size %d\n", 214 err, errno, retval, size); 215 216 err = bpf_prog_test_run(prog_fd, 1, &pkt_v6, sizeof(pkt_v6), 217 buf, &size, &retval, &duration); 218 CHECK(err || retval != XDP_TX || size != 114 || 219 iph6->nexthdr != IPPROTO_IPV6, "ipv6", 220 "err %d errno %d retval %d size %d\n", 221 err, errno, retval, size); 222 out: 223 bpf_object__close(obj); 224 } 225 226 static void test_xdp_adjust_tail(void) 227 { 228 const char *file = "./test_adjust_tail.o"; 229 struct bpf_object *obj; 230 char buf[128]; 231 __u32 duration, retval, size; 232 int err, prog_fd; 233 234 err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd); 235 if (err) { 236 error_cnt++; 237 return; 238 } 239 240 err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4), 241 buf, &size, &retval, &duration); 242 243 CHECK(err || retval != XDP_DROP, 244 "ipv4", "err %d errno %d retval %d size %d\n", 245 err, errno, retval, size); 246 247 err = bpf_prog_test_run(prog_fd, 1, &pkt_v6, sizeof(pkt_v6), 248 buf, &size, &retval, &duration); 249 CHECK(err || retval != XDP_TX || size != 54, 250 "ipv6", "err %d errno %d retval %d size %d\n", 251 err, errno, retval, size); 252 bpf_object__close(obj); 253 } 254 255 256 257 #define MAGIC_VAL 0x1234 258 #define NUM_ITER 100000 259 #define VIP_NUM 5 260 261 static void test_l4lb(const char *file) 262 { 263 unsigned int nr_cpus = bpf_num_possible_cpus(); 264 struct vip key = {.protocol = 6}; 265 struct vip_meta { 266 __u32 flags; 267 __u32 vip_num; 268 } value = {.vip_num = VIP_NUM}; 269 __u32 stats_key = VIP_NUM; 270 struct vip_stats { 271 __u64 bytes; 272 __u64 pkts; 273 } stats[nr_cpus]; 274 struct real_definition { 275 union { 276 __be32 dst; 277 __be32 dstv6[4]; 278 }; 279 __u8 flags; 280 } real_def = {.dst = MAGIC_VAL}; 281 __u32 ch_key = 11, real_num = 3; 282 __u32 duration, retval, size; 283 int err, i, prog_fd, map_fd; 284 __u64 bytes = 0, pkts = 0; 285 struct bpf_object *obj; 286 char buf[128]; 287 u32 *magic = (u32 *)buf; 288 289 err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); 290 if (err) { 291 error_cnt++; 292 return; 293 } 294 295 map_fd = bpf_find_map(__func__, obj, "vip_map"); 296 if (map_fd < 0) 297 goto out; 298 bpf_map_update_elem(map_fd, &key, &value, 0); 299 300 map_fd = bpf_find_map(__func__, obj, "ch_rings"); 301 if (map_fd < 0) 302 goto out; 303 bpf_map_update_elem(map_fd, &ch_key, &real_num, 0); 304 305 map_fd = bpf_find_map(__func__, obj, "reals"); 306 if (map_fd < 0) 307 goto out; 308 bpf_map_update_elem(map_fd, &real_num, &real_def, 0); 309 310 err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v4, sizeof(pkt_v4), 311 buf, &size, &retval, &duration); 312 CHECK(err || retval != 7/*TC_ACT_REDIRECT*/ || size != 54 || 313 *magic != MAGIC_VAL, "ipv4", 314 "err %d errno %d retval %d size %d magic %x\n", 315 err, errno, retval, size, *magic); 316 317 err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v6, sizeof(pkt_v6), 318 buf, &size, &retval, &duration); 319 CHECK(err || retval != 7/*TC_ACT_REDIRECT*/ || size != 74 || 320 *magic != MAGIC_VAL, "ipv6", 321 "err %d errno %d retval %d size %d magic %x\n", 322 err, errno, retval, size, *magic); 323 324 map_fd = bpf_find_map(__func__, obj, "stats"); 325 if (map_fd < 0) 326 goto out; 327 bpf_map_lookup_elem(map_fd, &stats_key, stats); 328 for (i = 0; i < nr_cpus; i++) { 329 bytes += stats[i].bytes; 330 pkts += stats[i].pkts; 331 } 332 if (bytes != MAGIC_BYTES * NUM_ITER * 2 || pkts != NUM_ITER * 2) { 333 error_cnt++; 334 printf("test_l4lb:FAIL:stats %lld %lld\n", bytes, pkts); 335 } 336 out: 337 bpf_object__close(obj); 338 } 339 340 static void test_l4lb_all(void) 341 { 342 const char *file1 = "./test_l4lb.o"; 343 const char *file2 = "./test_l4lb_noinline.o"; 344 345 test_l4lb(file1); 346 test_l4lb(file2); 347 } 348 349 static void test_xdp_noinline(void) 350 { 351 const char *file = "./test_xdp_noinline.o"; 352 unsigned int nr_cpus = bpf_num_possible_cpus(); 353 struct vip key = {.protocol = 6}; 354 struct vip_meta { 355 __u32 flags; 356 __u32 vip_num; 357 } value = {.vip_num = VIP_NUM}; 358 __u32 stats_key = VIP_NUM; 359 struct vip_stats { 360 __u64 bytes; 361 __u64 pkts; 362 } stats[nr_cpus]; 363 struct real_definition { 364 union { 365 __be32 dst; 366 __be32 dstv6[4]; 367 }; 368 __u8 flags; 369 } real_def = {.dst = MAGIC_VAL}; 370 __u32 ch_key = 11, real_num = 3; 371 __u32 duration, retval, size; 372 int err, i, prog_fd, map_fd; 373 __u64 bytes = 0, pkts = 0; 374 struct bpf_object *obj; 375 char buf[128]; 376 u32 *magic = (u32 *)buf; 377 378 err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd); 379 if (err) { 380 error_cnt++; 381 return; 382 } 383 384 map_fd = bpf_find_map(__func__, obj, "vip_map"); 385 if (map_fd < 0) 386 goto out; 387 bpf_map_update_elem(map_fd, &key, &value, 0); 388 389 map_fd = bpf_find_map(__func__, obj, "ch_rings"); 390 if (map_fd < 0) 391 goto out; 392 bpf_map_update_elem(map_fd, &ch_key, &real_num, 0); 393 394 map_fd = bpf_find_map(__func__, obj, "reals"); 395 if (map_fd < 0) 396 goto out; 397 bpf_map_update_elem(map_fd, &real_num, &real_def, 0); 398 399 err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v4, sizeof(pkt_v4), 400 buf, &size, &retval, &duration); 401 CHECK(err || retval != 1 || size != 54 || 402 *magic != MAGIC_VAL, "ipv4", 403 "err %d errno %d retval %d size %d magic %x\n", 404 err, errno, retval, size, *magic); 405 406 err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v6, sizeof(pkt_v6), 407 buf, &size, &retval, &duration); 408 CHECK(err || retval != 1 || size != 74 || 409 *magic != MAGIC_VAL, "ipv6", 410 "err %d errno %d retval %d size %d magic %x\n", 411 err, errno, retval, size, *magic); 412 413 map_fd = bpf_find_map(__func__, obj, "stats"); 414 if (map_fd < 0) 415 goto out; 416 bpf_map_lookup_elem(map_fd, &stats_key, stats); 417 for (i = 0; i < nr_cpus; i++) { 418 bytes += stats[i].bytes; 419 pkts += stats[i].pkts; 420 } 421 if (bytes != MAGIC_BYTES * NUM_ITER * 2 || pkts != NUM_ITER * 2) { 422 error_cnt++; 423 printf("test_xdp_noinline:FAIL:stats %lld %lld\n", bytes, pkts); 424 } 425 out: 426 bpf_object__close(obj); 427 } 428 429 static void test_tcp_estats(void) 430 { 431 const char *file = "./test_tcp_estats.o"; 432 int err, prog_fd; 433 struct bpf_object *obj; 434 __u32 duration = 0; 435 436 err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd); 437 CHECK(err, "", "err %d errno %d\n", err, errno); 438 if (err) { 439 error_cnt++; 440 return; 441 } 442 443 bpf_object__close(obj); 444 } 445 446 static inline __u64 ptr_to_u64(const void *ptr) 447 { 448 return (__u64) (unsigned long) ptr; 449 } 450 451 static bool is_jit_enabled(void) 452 { 453 const char *jit_sysctl = "/proc/sys/net/core/bpf_jit_enable"; 454 bool enabled = false; 455 int sysctl_fd; 456 457 sysctl_fd = open(jit_sysctl, 0, O_RDONLY); 458 if (sysctl_fd != -1) { 459 char tmpc; 460 461 if (read(sysctl_fd, &tmpc, sizeof(tmpc)) == 1) 462 enabled = (tmpc != '0'); 463 close(sysctl_fd); 464 } 465 466 return enabled; 467 } 468 469 static void test_bpf_obj_id(void) 470 { 471 const __u64 array_magic_value = 0xfaceb00c; 472 const __u32 array_key = 0; 473 const int nr_iters = 2; 474 const char *file = "./test_obj_id.o"; 475 const char *expected_prog_name = "test_obj_id"; 476 const char *expected_map_name = "test_map_id"; 477 const __u64 nsec_per_sec = 1000000000; 478 479 struct bpf_object *objs[nr_iters]; 480 int prog_fds[nr_iters], map_fds[nr_iters]; 481 /* +1 to test for the info_len returned by kernel */ 482 struct bpf_prog_info prog_infos[nr_iters + 1]; 483 struct bpf_map_info map_infos[nr_iters + 1]; 484 /* Each prog only uses one map. +1 to test nr_map_ids 485 * returned by kernel. 486 */ 487 __u32 map_ids[nr_iters + 1]; 488 char jited_insns[128], xlated_insns[128], zeros[128]; 489 __u32 i, next_id, info_len, nr_id_found, duration = 0; 490 struct timespec real_time_ts, boot_time_ts; 491 int err = 0; 492 __u64 array_value; 493 uid_t my_uid = getuid(); 494 time_t now, load_time; 495 496 err = bpf_prog_get_fd_by_id(0); 497 CHECK(err >= 0 || errno != ENOENT, 498 "get-fd-by-notexist-prog-id", "err %d errno %d\n", err, errno); 499 500 err = bpf_map_get_fd_by_id(0); 501 CHECK(err >= 0 || errno != ENOENT, 502 "get-fd-by-notexist-map-id", "err %d errno %d\n", err, errno); 503 504 for (i = 0; i < nr_iters; i++) 505 objs[i] = NULL; 506 507 /* Check bpf_obj_get_info_by_fd() */ 508 bzero(zeros, sizeof(zeros)); 509 for (i = 0; i < nr_iters; i++) { 510 now = time(NULL); 511 err = bpf_prog_load(file, BPF_PROG_TYPE_SOCKET_FILTER, 512 &objs[i], &prog_fds[i]); 513 /* test_obj_id.o is a dumb prog. It should never fail 514 * to load. 515 */ 516 if (err) 517 error_cnt++; 518 assert(!err); 519 520 /* Insert a magic value to the map */ 521 map_fds[i] = bpf_find_map(__func__, objs[i], "test_map_id"); 522 assert(map_fds[i] >= 0); 523 err = bpf_map_update_elem(map_fds[i], &array_key, 524 &array_magic_value, 0); 525 assert(!err); 526 527 /* Check getting map info */ 528 info_len = sizeof(struct bpf_map_info) * 2; 529 bzero(&map_infos[i], info_len); 530 err = bpf_obj_get_info_by_fd(map_fds[i], &map_infos[i], 531 &info_len); 532 if (CHECK(err || 533 map_infos[i].type != BPF_MAP_TYPE_ARRAY || 534 map_infos[i].key_size != sizeof(__u32) || 535 map_infos[i].value_size != sizeof(__u64) || 536 map_infos[i].max_entries != 1 || 537 map_infos[i].map_flags != 0 || 538 info_len != sizeof(struct bpf_map_info) || 539 strcmp((char *)map_infos[i].name, expected_map_name), 540 "get-map-info(fd)", 541 "err %d errno %d type %d(%d) info_len %u(%Zu) key_size %u value_size %u max_entries %u map_flags %X name %s(%s)\n", 542 err, errno, 543 map_infos[i].type, BPF_MAP_TYPE_ARRAY, 544 info_len, sizeof(struct bpf_map_info), 545 map_infos[i].key_size, 546 map_infos[i].value_size, 547 map_infos[i].max_entries, 548 map_infos[i].map_flags, 549 map_infos[i].name, expected_map_name)) 550 goto done; 551 552 /* Check getting prog info */ 553 info_len = sizeof(struct bpf_prog_info) * 2; 554 bzero(&prog_infos[i], info_len); 555 bzero(jited_insns, sizeof(jited_insns)); 556 bzero(xlated_insns, sizeof(xlated_insns)); 557 prog_infos[i].jited_prog_insns = ptr_to_u64(jited_insns); 558 prog_infos[i].jited_prog_len = sizeof(jited_insns); 559 prog_infos[i].xlated_prog_insns = ptr_to_u64(xlated_insns); 560 prog_infos[i].xlated_prog_len = sizeof(xlated_insns); 561 prog_infos[i].map_ids = ptr_to_u64(map_ids + i); 562 prog_infos[i].nr_map_ids = 2; 563 err = clock_gettime(CLOCK_REALTIME, &real_time_ts); 564 assert(!err); 565 err = clock_gettime(CLOCK_BOOTTIME, &boot_time_ts); 566 assert(!err); 567 err = bpf_obj_get_info_by_fd(prog_fds[i], &prog_infos[i], 568 &info_len); 569 load_time = (real_time_ts.tv_sec - boot_time_ts.tv_sec) 570 + (prog_infos[i].load_time / nsec_per_sec); 571 if (CHECK(err || 572 prog_infos[i].type != BPF_PROG_TYPE_SOCKET_FILTER || 573 info_len != sizeof(struct bpf_prog_info) || 574 (jit_enabled && !prog_infos[i].jited_prog_len) || 575 (jit_enabled && 576 !memcmp(jited_insns, zeros, sizeof(zeros))) || 577 !prog_infos[i].xlated_prog_len || 578 !memcmp(xlated_insns, zeros, sizeof(zeros)) || 579 load_time < now - 60 || load_time > now + 60 || 580 prog_infos[i].created_by_uid != my_uid || 581 prog_infos[i].nr_map_ids != 1 || 582 *(int *)(long)prog_infos[i].map_ids != map_infos[i].id || 583 strcmp((char *)prog_infos[i].name, expected_prog_name), 584 "get-prog-info(fd)", 585 "err %d errno %d i %d type %d(%d) info_len %u(%Zu) jit_enabled %d jited_prog_len %u xlated_prog_len %u jited_prog %d xlated_prog %d load_time %lu(%lu) uid %u(%u) nr_map_ids %u(%u) map_id %u(%u) name %s(%s)\n", 586 err, errno, i, 587 prog_infos[i].type, BPF_PROG_TYPE_SOCKET_FILTER, 588 info_len, sizeof(struct bpf_prog_info), 589 jit_enabled, 590 prog_infos[i].jited_prog_len, 591 prog_infos[i].xlated_prog_len, 592 !!memcmp(jited_insns, zeros, sizeof(zeros)), 593 !!memcmp(xlated_insns, zeros, sizeof(zeros)), 594 load_time, now, 595 prog_infos[i].created_by_uid, my_uid, 596 prog_infos[i].nr_map_ids, 1, 597 *(int *)(long)prog_infos[i].map_ids, map_infos[i].id, 598 prog_infos[i].name, expected_prog_name)) 599 goto done; 600 } 601 602 /* Check bpf_prog_get_next_id() */ 603 nr_id_found = 0; 604 next_id = 0; 605 while (!bpf_prog_get_next_id(next_id, &next_id)) { 606 struct bpf_prog_info prog_info = {}; 607 __u32 saved_map_id; 608 int prog_fd; 609 610 info_len = sizeof(prog_info); 611 612 prog_fd = bpf_prog_get_fd_by_id(next_id); 613 if (prog_fd < 0 && errno == ENOENT) 614 /* The bpf_prog is in the dead row */ 615 continue; 616 if (CHECK(prog_fd < 0, "get-prog-fd(next_id)", 617 "prog_fd %d next_id %d errno %d\n", 618 prog_fd, next_id, errno)) 619 break; 620 621 for (i = 0; i < nr_iters; i++) 622 if (prog_infos[i].id == next_id) 623 break; 624 625 if (i == nr_iters) 626 continue; 627 628 nr_id_found++; 629 630 /* Negative test: 631 * prog_info.nr_map_ids = 1 632 * prog_info.map_ids = NULL 633 */ 634 prog_info.nr_map_ids = 1; 635 err = bpf_obj_get_info_by_fd(prog_fd, &prog_info, &info_len); 636 if (CHECK(!err || errno != EFAULT, 637 "get-prog-fd-bad-nr-map-ids", "err %d errno %d(%d)", 638 err, errno, EFAULT)) 639 break; 640 bzero(&prog_info, sizeof(prog_info)); 641 info_len = sizeof(prog_info); 642 643 saved_map_id = *(int *)((long)prog_infos[i].map_ids); 644 prog_info.map_ids = prog_infos[i].map_ids; 645 prog_info.nr_map_ids = 2; 646 err = bpf_obj_get_info_by_fd(prog_fd, &prog_info, &info_len); 647 prog_infos[i].jited_prog_insns = 0; 648 prog_infos[i].xlated_prog_insns = 0; 649 CHECK(err || info_len != sizeof(struct bpf_prog_info) || 650 memcmp(&prog_info, &prog_infos[i], info_len) || 651 *(int *)(long)prog_info.map_ids != saved_map_id, 652 "get-prog-info(next_id->fd)", 653 "err %d errno %d info_len %u(%Zu) memcmp %d map_id %u(%u)\n", 654 err, errno, info_len, sizeof(struct bpf_prog_info), 655 memcmp(&prog_info, &prog_infos[i], info_len), 656 *(int *)(long)prog_info.map_ids, saved_map_id); 657 close(prog_fd); 658 } 659 CHECK(nr_id_found != nr_iters, 660 "check total prog id found by get_next_id", 661 "nr_id_found %u(%u)\n", 662 nr_id_found, nr_iters); 663 664 /* Check bpf_map_get_next_id() */ 665 nr_id_found = 0; 666 next_id = 0; 667 while (!bpf_map_get_next_id(next_id, &next_id)) { 668 struct bpf_map_info map_info = {}; 669 int map_fd; 670 671 info_len = sizeof(map_info); 672 673 map_fd = bpf_map_get_fd_by_id(next_id); 674 if (map_fd < 0 && errno == ENOENT) 675 /* The bpf_map is in the dead row */ 676 continue; 677 if (CHECK(map_fd < 0, "get-map-fd(next_id)", 678 "map_fd %d next_id %u errno %d\n", 679 map_fd, next_id, errno)) 680 break; 681 682 for (i = 0; i < nr_iters; i++) 683 if (map_infos[i].id == next_id) 684 break; 685 686 if (i == nr_iters) 687 continue; 688 689 nr_id_found++; 690 691 err = bpf_map_lookup_elem(map_fd, &array_key, &array_value); 692 assert(!err); 693 694 err = bpf_obj_get_info_by_fd(map_fd, &map_info, &info_len); 695 CHECK(err || info_len != sizeof(struct bpf_map_info) || 696 memcmp(&map_info, &map_infos[i], info_len) || 697 array_value != array_magic_value, 698 "check get-map-info(next_id->fd)", 699 "err %d errno %d info_len %u(%Zu) memcmp %d array_value %llu(%llu)\n", 700 err, errno, info_len, sizeof(struct bpf_map_info), 701 memcmp(&map_info, &map_infos[i], info_len), 702 array_value, array_magic_value); 703 704 close(map_fd); 705 } 706 CHECK(nr_id_found != nr_iters, 707 "check total map id found by get_next_id", 708 "nr_id_found %u(%u)\n", 709 nr_id_found, nr_iters); 710 711 done: 712 for (i = 0; i < nr_iters; i++) 713 bpf_object__close(objs[i]); 714 } 715 716 static void test_pkt_md_access(void) 717 { 718 const char *file = "./test_pkt_md_access.o"; 719 struct bpf_object *obj; 720 __u32 duration, retval; 721 int err, prog_fd; 722 723 err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); 724 if (err) { 725 error_cnt++; 726 return; 727 } 728 729 err = bpf_prog_test_run(prog_fd, 10, &pkt_v4, sizeof(pkt_v4), 730 NULL, NULL, &retval, &duration); 731 CHECK(err || retval, "", 732 "err %d errno %d retval %d duration %d\n", 733 err, errno, retval, duration); 734 735 bpf_object__close(obj); 736 } 737 738 static void test_obj_name(void) 739 { 740 struct { 741 const char *name; 742 int success; 743 int expected_errno; 744 } tests[] = { 745 { "", 1, 0 }, 746 { "_123456789ABCDE", 1, 0 }, 747 { "_123456789ABCDEF", 0, EINVAL }, 748 { "_123456789ABCD\n", 0, EINVAL }, 749 }; 750 struct bpf_insn prog[] = { 751 BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0), 752 BPF_EXIT_INSN(), 753 }; 754 __u32 duration = 0; 755 int i; 756 757 for (i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) { 758 size_t name_len = strlen(tests[i].name) + 1; 759 union bpf_attr attr; 760 size_t ncopy; 761 int fd; 762 763 /* test different attr.prog_name during BPF_PROG_LOAD */ 764 ncopy = name_len < sizeof(attr.prog_name) ? 765 name_len : sizeof(attr.prog_name); 766 bzero(&attr, sizeof(attr)); 767 attr.prog_type = BPF_PROG_TYPE_SCHED_CLS; 768 attr.insn_cnt = 2; 769 attr.insns = ptr_to_u64(prog); 770 attr.license = ptr_to_u64(""); 771 memcpy(attr.prog_name, tests[i].name, ncopy); 772 773 fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, sizeof(attr)); 774 CHECK((tests[i].success && fd < 0) || 775 (!tests[i].success && fd != -1) || 776 (!tests[i].success && errno != tests[i].expected_errno), 777 "check-bpf-prog-name", 778 "fd %d(%d) errno %d(%d)\n", 779 fd, tests[i].success, errno, tests[i].expected_errno); 780 781 if (fd != -1) 782 close(fd); 783 784 /* test different attr.map_name during BPF_MAP_CREATE */ 785 ncopy = name_len < sizeof(attr.map_name) ? 786 name_len : sizeof(attr.map_name); 787 bzero(&attr, sizeof(attr)); 788 attr.map_type = BPF_MAP_TYPE_ARRAY; 789 attr.key_size = 4; 790 attr.value_size = 4; 791 attr.max_entries = 1; 792 attr.map_flags = 0; 793 memcpy(attr.map_name, tests[i].name, ncopy); 794 fd = syscall(__NR_bpf, BPF_MAP_CREATE, &attr, sizeof(attr)); 795 CHECK((tests[i].success && fd < 0) || 796 (!tests[i].success && fd != -1) || 797 (!tests[i].success && errno != tests[i].expected_errno), 798 "check-bpf-map-name", 799 "fd %d(%d) errno %d(%d)\n", 800 fd, tests[i].success, errno, tests[i].expected_errno); 801 802 if (fd != -1) 803 close(fd); 804 } 805 } 806 807 static void test_tp_attach_query(void) 808 { 809 const int num_progs = 3; 810 int i, j, bytes, efd, err, prog_fd[num_progs], pmu_fd[num_progs]; 811 __u32 duration = 0, info_len, saved_prog_ids[num_progs]; 812 const char *file = "./test_tracepoint.o"; 813 struct perf_event_query_bpf *query; 814 struct perf_event_attr attr = {}; 815 struct bpf_object *obj[num_progs]; 816 struct bpf_prog_info prog_info; 817 char buf[256]; 818 819 snprintf(buf, sizeof(buf), 820 "/sys/kernel/debug/tracing/events/sched/sched_switch/id"); 821 efd = open(buf, O_RDONLY, 0); 822 if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno)) 823 return; 824 bytes = read(efd, buf, sizeof(buf)); 825 close(efd); 826 if (CHECK(bytes <= 0 || bytes >= sizeof(buf), 827 "read", "bytes %d errno %d\n", bytes, errno)) 828 return; 829 830 attr.config = strtol(buf, NULL, 0); 831 attr.type = PERF_TYPE_TRACEPOINT; 832 attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN; 833 attr.sample_period = 1; 834 attr.wakeup_events = 1; 835 836 query = malloc(sizeof(*query) + sizeof(__u32) * num_progs); 837 for (i = 0; i < num_progs; i++) { 838 err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj[i], 839 &prog_fd[i]); 840 if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno)) 841 goto cleanup1; 842 843 bzero(&prog_info, sizeof(prog_info)); 844 prog_info.jited_prog_len = 0; 845 prog_info.xlated_prog_len = 0; 846 prog_info.nr_map_ids = 0; 847 info_len = sizeof(prog_info); 848 err = bpf_obj_get_info_by_fd(prog_fd[i], &prog_info, &info_len); 849 if (CHECK(err, "bpf_obj_get_info_by_fd", "err %d errno %d\n", 850 err, errno)) 851 goto cleanup1; 852 saved_prog_ids[i] = prog_info.id; 853 854 pmu_fd[i] = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 855 0 /* cpu 0 */, -1 /* group id */, 856 0 /* flags */); 857 if (CHECK(pmu_fd[i] < 0, "perf_event_open", "err %d errno %d\n", 858 pmu_fd[i], errno)) 859 goto cleanup2; 860 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_ENABLE, 0); 861 if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n", 862 err, errno)) 863 goto cleanup3; 864 865 if (i == 0) { 866 /* check NULL prog array query */ 867 query->ids_len = num_progs; 868 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query); 869 if (CHECK(err || query->prog_cnt != 0, 870 "perf_event_ioc_query_bpf", 871 "err %d errno %d query->prog_cnt %u\n", 872 err, errno, query->prog_cnt)) 873 goto cleanup3; 874 } 875 876 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_SET_BPF, prog_fd[i]); 877 if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n", 878 err, errno)) 879 goto cleanup3; 880 881 if (i == 1) { 882 /* try to get # of programs only */ 883 query->ids_len = 0; 884 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query); 885 if (CHECK(err || query->prog_cnt != 2, 886 "perf_event_ioc_query_bpf", 887 "err %d errno %d query->prog_cnt %u\n", 888 err, errno, query->prog_cnt)) 889 goto cleanup3; 890 891 /* try a few negative tests */ 892 /* invalid query pointer */ 893 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, 894 (struct perf_event_query_bpf *)0x1); 895 if (CHECK(!err || errno != EFAULT, 896 "perf_event_ioc_query_bpf", 897 "err %d errno %d\n", err, errno)) 898 goto cleanup3; 899 900 /* no enough space */ 901 query->ids_len = 1; 902 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query); 903 if (CHECK(!err || errno != ENOSPC || query->prog_cnt != 2, 904 "perf_event_ioc_query_bpf", 905 "err %d errno %d query->prog_cnt %u\n", 906 err, errno, query->prog_cnt)) 907 goto cleanup3; 908 } 909 910 query->ids_len = num_progs; 911 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query); 912 if (CHECK(err || query->prog_cnt != (i + 1), 913 "perf_event_ioc_query_bpf", 914 "err %d errno %d query->prog_cnt %u\n", 915 err, errno, query->prog_cnt)) 916 goto cleanup3; 917 for (j = 0; j < i + 1; j++) 918 if (CHECK(saved_prog_ids[j] != query->ids[j], 919 "perf_event_ioc_query_bpf", 920 "#%d saved_prog_id %x query prog_id %x\n", 921 j, saved_prog_ids[j], query->ids[j])) 922 goto cleanup3; 923 } 924 925 i = num_progs - 1; 926 for (; i >= 0; i--) { 927 cleanup3: 928 ioctl(pmu_fd[i], PERF_EVENT_IOC_DISABLE); 929 cleanup2: 930 close(pmu_fd[i]); 931 cleanup1: 932 bpf_object__close(obj[i]); 933 } 934 free(query); 935 } 936 937 static int compare_map_keys(int map1_fd, int map2_fd) 938 { 939 __u32 key, next_key; 940 char val_buf[PERF_MAX_STACK_DEPTH * 941 sizeof(struct bpf_stack_build_id)]; 942 int err; 943 944 err = bpf_map_get_next_key(map1_fd, NULL, &key); 945 if (err) 946 return err; 947 err = bpf_map_lookup_elem(map2_fd, &key, val_buf); 948 if (err) 949 return err; 950 951 while (bpf_map_get_next_key(map1_fd, &key, &next_key) == 0) { 952 err = bpf_map_lookup_elem(map2_fd, &next_key, val_buf); 953 if (err) 954 return err; 955 956 key = next_key; 957 } 958 if (errno != ENOENT) 959 return -1; 960 961 return 0; 962 } 963 964 static int compare_stack_ips(int smap_fd, int amap_fd, int stack_trace_len) 965 { 966 __u32 key, next_key, *cur_key_p, *next_key_p; 967 char *val_buf1, *val_buf2; 968 int i, err = 0; 969 970 val_buf1 = malloc(stack_trace_len); 971 val_buf2 = malloc(stack_trace_len); 972 cur_key_p = NULL; 973 next_key_p = &key; 974 while (bpf_map_get_next_key(smap_fd, cur_key_p, next_key_p) == 0) { 975 err = bpf_map_lookup_elem(smap_fd, next_key_p, val_buf1); 976 if (err) 977 goto out; 978 err = bpf_map_lookup_elem(amap_fd, next_key_p, val_buf2); 979 if (err) 980 goto out; 981 for (i = 0; i < stack_trace_len; i++) { 982 if (val_buf1[i] != val_buf2[i]) { 983 err = -1; 984 goto out; 985 } 986 } 987 key = *next_key_p; 988 cur_key_p = &key; 989 next_key_p = &next_key; 990 } 991 if (errno != ENOENT) 992 err = -1; 993 994 out: 995 free(val_buf1); 996 free(val_buf2); 997 return err; 998 } 999 1000 static void test_stacktrace_map() 1001 { 1002 int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd; 1003 const char *file = "./test_stacktrace_map.o"; 1004 int bytes, efd, err, pmu_fd, prog_fd, stack_trace_len; 1005 struct perf_event_attr attr = {}; 1006 __u32 key, val, duration = 0; 1007 struct bpf_object *obj; 1008 char buf[256]; 1009 1010 err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd); 1011 if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno)) 1012 return; 1013 1014 /* Get the ID for the sched/sched_switch tracepoint */ 1015 snprintf(buf, sizeof(buf), 1016 "/sys/kernel/debug/tracing/events/sched/sched_switch/id"); 1017 efd = open(buf, O_RDONLY, 0); 1018 if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno)) 1019 goto close_prog; 1020 1021 bytes = read(efd, buf, sizeof(buf)); 1022 close(efd); 1023 if (bytes <= 0 || bytes >= sizeof(buf)) 1024 goto close_prog; 1025 1026 /* Open the perf event and attach bpf progrram */ 1027 attr.config = strtol(buf, NULL, 0); 1028 attr.type = PERF_TYPE_TRACEPOINT; 1029 attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN; 1030 attr.sample_period = 1; 1031 attr.wakeup_events = 1; 1032 pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 1033 0 /* cpu 0 */, -1 /* group id */, 1034 0 /* flags */); 1035 if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n", 1036 pmu_fd, errno)) 1037 goto close_prog; 1038 1039 err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0); 1040 if (err) 1041 goto disable_pmu; 1042 1043 err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd); 1044 if (err) 1045 goto disable_pmu; 1046 1047 /* find map fds */ 1048 control_map_fd = bpf_find_map(__func__, obj, "control_map"); 1049 if (control_map_fd < 0) 1050 goto disable_pmu; 1051 1052 stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap"); 1053 if (stackid_hmap_fd < 0) 1054 goto disable_pmu; 1055 1056 stackmap_fd = bpf_find_map(__func__, obj, "stackmap"); 1057 if (stackmap_fd < 0) 1058 goto disable_pmu; 1059 1060 stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap"); 1061 if (stack_amap_fd < 0) 1062 goto disable_pmu; 1063 1064 /* give some time for bpf program run */ 1065 sleep(1); 1066 1067 /* disable stack trace collection */ 1068 key = 0; 1069 val = 1; 1070 bpf_map_update_elem(control_map_fd, &key, &val, 0); 1071 1072 /* for every element in stackid_hmap, we can find a corresponding one 1073 * in stackmap, and vise versa. 1074 */ 1075 err = compare_map_keys(stackid_hmap_fd, stackmap_fd); 1076 if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap", 1077 "err %d errno %d\n", err, errno)) 1078 goto disable_pmu_noerr; 1079 1080 err = compare_map_keys(stackmap_fd, stackid_hmap_fd); 1081 if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap", 1082 "err %d errno %d\n", err, errno)) 1083 goto disable_pmu_noerr; 1084 1085 stack_trace_len = PERF_MAX_STACK_DEPTH * sizeof(__u64); 1086 err = compare_stack_ips(stackmap_fd, stack_amap_fd, stack_trace_len); 1087 if (CHECK(err, "compare_stack_ips stackmap vs. stack_amap", 1088 "err %d errno %d\n", err, errno)) 1089 goto disable_pmu_noerr; 1090 1091 goto disable_pmu_noerr; 1092 disable_pmu: 1093 error_cnt++; 1094 disable_pmu_noerr: 1095 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE); 1096 close(pmu_fd); 1097 close_prog: 1098 bpf_object__close(obj); 1099 } 1100 1101 static void test_stacktrace_map_raw_tp() 1102 { 1103 int control_map_fd, stackid_hmap_fd, stackmap_fd; 1104 const char *file = "./test_stacktrace_map.o"; 1105 int efd, err, prog_fd; 1106 __u32 key, val, duration = 0; 1107 struct bpf_object *obj; 1108 1109 err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd); 1110 if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno)) 1111 return; 1112 1113 efd = bpf_raw_tracepoint_open("sched_switch", prog_fd); 1114 if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno)) 1115 goto close_prog; 1116 1117 /* find map fds */ 1118 control_map_fd = bpf_find_map(__func__, obj, "control_map"); 1119 if (control_map_fd < 0) 1120 goto close_prog; 1121 1122 stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap"); 1123 if (stackid_hmap_fd < 0) 1124 goto close_prog; 1125 1126 stackmap_fd = bpf_find_map(__func__, obj, "stackmap"); 1127 if (stackmap_fd < 0) 1128 goto close_prog; 1129 1130 /* give some time for bpf program run */ 1131 sleep(1); 1132 1133 /* disable stack trace collection */ 1134 key = 0; 1135 val = 1; 1136 bpf_map_update_elem(control_map_fd, &key, &val, 0); 1137 1138 /* for every element in stackid_hmap, we can find a corresponding one 1139 * in stackmap, and vise versa. 1140 */ 1141 err = compare_map_keys(stackid_hmap_fd, stackmap_fd); 1142 if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap", 1143 "err %d errno %d\n", err, errno)) 1144 goto close_prog; 1145 1146 err = compare_map_keys(stackmap_fd, stackid_hmap_fd); 1147 if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap", 1148 "err %d errno %d\n", err, errno)) 1149 goto close_prog; 1150 1151 goto close_prog_noerr; 1152 close_prog: 1153 error_cnt++; 1154 close_prog_noerr: 1155 bpf_object__close(obj); 1156 } 1157 1158 static int extract_build_id(char *build_id, size_t size) 1159 { 1160 FILE *fp; 1161 char *line = NULL; 1162 size_t len = 0; 1163 1164 fp = popen("readelf -n ./urandom_read | grep 'Build ID'", "r"); 1165 if (fp == NULL) 1166 return -1; 1167 1168 if (getline(&line, &len, fp) == -1) 1169 goto err; 1170 fclose(fp); 1171 1172 if (len > size) 1173 len = size; 1174 memcpy(build_id, line, len); 1175 build_id[len] = '\0'; 1176 return 0; 1177 err: 1178 fclose(fp); 1179 return -1; 1180 } 1181 1182 static void test_stacktrace_build_id(void) 1183 { 1184 int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd; 1185 const char *file = "./test_stacktrace_build_id.o"; 1186 int bytes, efd, err, pmu_fd, prog_fd, stack_trace_len; 1187 struct perf_event_attr attr = {}; 1188 __u32 key, previous_key, val, duration = 0; 1189 struct bpf_object *obj; 1190 char buf[256]; 1191 int i, j; 1192 struct bpf_stack_build_id id_offs[PERF_MAX_STACK_DEPTH]; 1193 int build_id_matches = 0; 1194 int retry = 1; 1195 1196 retry: 1197 err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd); 1198 if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno)) 1199 goto out; 1200 1201 /* Get the ID for the sched/sched_switch tracepoint */ 1202 snprintf(buf, sizeof(buf), 1203 "/sys/kernel/debug/tracing/events/random/urandom_read/id"); 1204 efd = open(buf, O_RDONLY, 0); 1205 if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno)) 1206 goto close_prog; 1207 1208 bytes = read(efd, buf, sizeof(buf)); 1209 close(efd); 1210 if (CHECK(bytes <= 0 || bytes >= sizeof(buf), 1211 "read", "bytes %d errno %d\n", bytes, errno)) 1212 goto close_prog; 1213 1214 /* Open the perf event and attach bpf progrram */ 1215 attr.config = strtol(buf, NULL, 0); 1216 attr.type = PERF_TYPE_TRACEPOINT; 1217 attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN; 1218 attr.sample_period = 1; 1219 attr.wakeup_events = 1; 1220 pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 1221 0 /* cpu 0 */, -1 /* group id */, 1222 0 /* flags */); 1223 if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n", 1224 pmu_fd, errno)) 1225 goto close_prog; 1226 1227 err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0); 1228 if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n", 1229 err, errno)) 1230 goto close_pmu; 1231 1232 err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd); 1233 if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n", 1234 err, errno)) 1235 goto disable_pmu; 1236 1237 /* find map fds */ 1238 control_map_fd = bpf_find_map(__func__, obj, "control_map"); 1239 if (CHECK(control_map_fd < 0, "bpf_find_map control_map", 1240 "err %d errno %d\n", err, errno)) 1241 goto disable_pmu; 1242 1243 stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap"); 1244 if (CHECK(stackid_hmap_fd < 0, "bpf_find_map stackid_hmap", 1245 "err %d errno %d\n", err, errno)) 1246 goto disable_pmu; 1247 1248 stackmap_fd = bpf_find_map(__func__, obj, "stackmap"); 1249 if (CHECK(stackmap_fd < 0, "bpf_find_map stackmap", "err %d errno %d\n", 1250 err, errno)) 1251 goto disable_pmu; 1252 1253 stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap"); 1254 if (CHECK(stack_amap_fd < 0, "bpf_find_map stack_amap", 1255 "err %d errno %d\n", err, errno)) 1256 goto disable_pmu; 1257 1258 assert(system("dd if=/dev/urandom of=/dev/zero count=4 2> /dev/null") 1259 == 0); 1260 assert(system("./urandom_read") == 0); 1261 /* disable stack trace collection */ 1262 key = 0; 1263 val = 1; 1264 bpf_map_update_elem(control_map_fd, &key, &val, 0); 1265 1266 /* for every element in stackid_hmap, we can find a corresponding one 1267 * in stackmap, and vise versa. 1268 */ 1269 err = compare_map_keys(stackid_hmap_fd, stackmap_fd); 1270 if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap", 1271 "err %d errno %d\n", err, errno)) 1272 goto disable_pmu; 1273 1274 err = compare_map_keys(stackmap_fd, stackid_hmap_fd); 1275 if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap", 1276 "err %d errno %d\n", err, errno)) 1277 goto disable_pmu; 1278 1279 err = extract_build_id(buf, 256); 1280 1281 if (CHECK(err, "get build_id with readelf", 1282 "err %d errno %d\n", err, errno)) 1283 goto disable_pmu; 1284 1285 err = bpf_map_get_next_key(stackmap_fd, NULL, &key); 1286 if (CHECK(err, "get_next_key from stackmap", 1287 "err %d, errno %d\n", err, errno)) 1288 goto disable_pmu; 1289 1290 do { 1291 char build_id[64]; 1292 1293 err = bpf_map_lookup_elem(stackmap_fd, &key, id_offs); 1294 if (CHECK(err, "lookup_elem from stackmap", 1295 "err %d, errno %d\n", err, errno)) 1296 goto disable_pmu; 1297 for (i = 0; i < PERF_MAX_STACK_DEPTH; ++i) 1298 if (id_offs[i].status == BPF_STACK_BUILD_ID_VALID && 1299 id_offs[i].offset != 0) { 1300 for (j = 0; j < 20; ++j) 1301 sprintf(build_id + 2 * j, "%02x", 1302 id_offs[i].build_id[j] & 0xff); 1303 if (strstr(buf, build_id) != NULL) 1304 build_id_matches = 1; 1305 } 1306 previous_key = key; 1307 } while (bpf_map_get_next_key(stackmap_fd, &previous_key, &key) == 0); 1308 1309 /* stack_map_get_build_id_offset() is racy and sometimes can return 1310 * BPF_STACK_BUILD_ID_IP instead of BPF_STACK_BUILD_ID_VALID; 1311 * try it one more time. 1312 */ 1313 if (build_id_matches < 1 && retry--) { 1314 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE); 1315 close(pmu_fd); 1316 bpf_object__close(obj); 1317 printf("%s:WARN:Didn't find expected build ID from the map, retrying\n", 1318 __func__); 1319 goto retry; 1320 } 1321 1322 if (CHECK(build_id_matches < 1, "build id match", 1323 "Didn't find expected build ID from the map\n")) 1324 goto disable_pmu; 1325 1326 stack_trace_len = PERF_MAX_STACK_DEPTH 1327 * sizeof(struct bpf_stack_build_id); 1328 err = compare_stack_ips(stackmap_fd, stack_amap_fd, stack_trace_len); 1329 CHECK(err, "compare_stack_ips stackmap vs. stack_amap", 1330 "err %d errno %d\n", err, errno); 1331 1332 disable_pmu: 1333 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE); 1334 1335 close_pmu: 1336 close(pmu_fd); 1337 1338 close_prog: 1339 bpf_object__close(obj); 1340 1341 out: 1342 return; 1343 } 1344 1345 static void test_stacktrace_build_id_nmi(void) 1346 { 1347 int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd; 1348 const char *file = "./test_stacktrace_build_id.o"; 1349 int err, pmu_fd, prog_fd; 1350 struct perf_event_attr attr = { 1351 .sample_freq = 5000, 1352 .freq = 1, 1353 .type = PERF_TYPE_HARDWARE, 1354 .config = PERF_COUNT_HW_CPU_CYCLES, 1355 }; 1356 __u32 key, previous_key, val, duration = 0; 1357 struct bpf_object *obj; 1358 char buf[256]; 1359 int i, j; 1360 struct bpf_stack_build_id id_offs[PERF_MAX_STACK_DEPTH]; 1361 int build_id_matches = 0; 1362 int retry = 1; 1363 1364 retry: 1365 err = bpf_prog_load(file, BPF_PROG_TYPE_PERF_EVENT, &obj, &prog_fd); 1366 if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno)) 1367 return; 1368 1369 pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 1370 0 /* cpu 0 */, -1 /* group id */, 1371 0 /* flags */); 1372 if (CHECK(pmu_fd < 0, "perf_event_open", 1373 "err %d errno %d. Does the test host support PERF_COUNT_HW_CPU_CYCLES?\n", 1374 pmu_fd, errno)) 1375 goto close_prog; 1376 1377 err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0); 1378 if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n", 1379 err, errno)) 1380 goto close_pmu; 1381 1382 err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd); 1383 if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n", 1384 err, errno)) 1385 goto disable_pmu; 1386 1387 /* find map fds */ 1388 control_map_fd = bpf_find_map(__func__, obj, "control_map"); 1389 if (CHECK(control_map_fd < 0, "bpf_find_map control_map", 1390 "err %d errno %d\n", err, errno)) 1391 goto disable_pmu; 1392 1393 stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap"); 1394 if (CHECK(stackid_hmap_fd < 0, "bpf_find_map stackid_hmap", 1395 "err %d errno %d\n", err, errno)) 1396 goto disable_pmu; 1397 1398 stackmap_fd = bpf_find_map(__func__, obj, "stackmap"); 1399 if (CHECK(stackmap_fd < 0, "bpf_find_map stackmap", "err %d errno %d\n", 1400 err, errno)) 1401 goto disable_pmu; 1402 1403 stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap"); 1404 if (CHECK(stack_amap_fd < 0, "bpf_find_map stack_amap", 1405 "err %d errno %d\n", err, errno)) 1406 goto disable_pmu; 1407 1408 assert(system("dd if=/dev/urandom of=/dev/zero count=4 2> /dev/null") 1409 == 0); 1410 assert(system("taskset 0x1 ./urandom_read 100000") == 0); 1411 /* disable stack trace collection */ 1412 key = 0; 1413 val = 1; 1414 bpf_map_update_elem(control_map_fd, &key, &val, 0); 1415 1416 /* for every element in stackid_hmap, we can find a corresponding one 1417 * in stackmap, and vise versa. 1418 */ 1419 err = compare_map_keys(stackid_hmap_fd, stackmap_fd); 1420 if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap", 1421 "err %d errno %d\n", err, errno)) 1422 goto disable_pmu; 1423 1424 err = compare_map_keys(stackmap_fd, stackid_hmap_fd); 1425 if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap", 1426 "err %d errno %d\n", err, errno)) 1427 goto disable_pmu; 1428 1429 err = extract_build_id(buf, 256); 1430 1431 if (CHECK(err, "get build_id with readelf", 1432 "err %d errno %d\n", err, errno)) 1433 goto disable_pmu; 1434 1435 err = bpf_map_get_next_key(stackmap_fd, NULL, &key); 1436 if (CHECK(err, "get_next_key from stackmap", 1437 "err %d, errno %d\n", err, errno)) 1438 goto disable_pmu; 1439 1440 do { 1441 char build_id[64]; 1442 1443 err = bpf_map_lookup_elem(stackmap_fd, &key, id_offs); 1444 if (CHECK(err, "lookup_elem from stackmap", 1445 "err %d, errno %d\n", err, errno)) 1446 goto disable_pmu; 1447 for (i = 0; i < PERF_MAX_STACK_DEPTH; ++i) 1448 if (id_offs[i].status == BPF_STACK_BUILD_ID_VALID && 1449 id_offs[i].offset != 0) { 1450 for (j = 0; j < 20; ++j) 1451 sprintf(build_id + 2 * j, "%02x", 1452 id_offs[i].build_id[j] & 0xff); 1453 if (strstr(buf, build_id) != NULL) 1454 build_id_matches = 1; 1455 } 1456 previous_key = key; 1457 } while (bpf_map_get_next_key(stackmap_fd, &previous_key, &key) == 0); 1458 1459 /* stack_map_get_build_id_offset() is racy and sometimes can return 1460 * BPF_STACK_BUILD_ID_IP instead of BPF_STACK_BUILD_ID_VALID; 1461 * try it one more time. 1462 */ 1463 if (build_id_matches < 1 && retry--) { 1464 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE); 1465 close(pmu_fd); 1466 bpf_object__close(obj); 1467 printf("%s:WARN:Didn't find expected build ID from the map, retrying\n", 1468 __func__); 1469 goto retry; 1470 } 1471 1472 if (CHECK(build_id_matches < 1, "build id match", 1473 "Didn't find expected build ID from the map\n")) 1474 goto disable_pmu; 1475 1476 /* 1477 * We intentionally skip compare_stack_ips(). This is because we 1478 * only support one in_nmi() ips-to-build_id translation per cpu 1479 * at any time, thus stack_amap here will always fallback to 1480 * BPF_STACK_BUILD_ID_IP; 1481 */ 1482 1483 disable_pmu: 1484 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE); 1485 1486 close_pmu: 1487 close(pmu_fd); 1488 1489 close_prog: 1490 bpf_object__close(obj); 1491 } 1492 1493 #define MAX_CNT_RAWTP 10ull 1494 #define MAX_STACK_RAWTP 100 1495 struct get_stack_trace_t { 1496 int pid; 1497 int kern_stack_size; 1498 int user_stack_size; 1499 int user_stack_buildid_size; 1500 __u64 kern_stack[MAX_STACK_RAWTP]; 1501 __u64 user_stack[MAX_STACK_RAWTP]; 1502 struct bpf_stack_build_id user_stack_buildid[MAX_STACK_RAWTP]; 1503 }; 1504 1505 static int get_stack_print_output(void *data, int size) 1506 { 1507 bool good_kern_stack = false, good_user_stack = false; 1508 const char *nonjit_func = "___bpf_prog_run"; 1509 struct get_stack_trace_t *e = data; 1510 int i, num_stack; 1511 static __u64 cnt; 1512 struct ksym *ks; 1513 1514 cnt++; 1515 1516 if (size < sizeof(struct get_stack_trace_t)) { 1517 __u64 *raw_data = data; 1518 bool found = false; 1519 1520 num_stack = size / sizeof(__u64); 1521 /* If jit is enabled, we do not have a good way to 1522 * verify the sanity of the kernel stack. So we 1523 * just assume it is good if the stack is not empty. 1524 * This could be improved in the future. 1525 */ 1526 if (jit_enabled) { 1527 found = num_stack > 0; 1528 } else { 1529 for (i = 0; i < num_stack; i++) { 1530 ks = ksym_search(raw_data[i]); 1531 if (strcmp(ks->name, nonjit_func) == 0) { 1532 found = true; 1533 break; 1534 } 1535 } 1536 } 1537 if (found) { 1538 good_kern_stack = true; 1539 good_user_stack = true; 1540 } 1541 } else { 1542 num_stack = e->kern_stack_size / sizeof(__u64); 1543 if (jit_enabled) { 1544 good_kern_stack = num_stack > 0; 1545 } else { 1546 for (i = 0; i < num_stack; i++) { 1547 ks = ksym_search(e->kern_stack[i]); 1548 if (strcmp(ks->name, nonjit_func) == 0) { 1549 good_kern_stack = true; 1550 break; 1551 } 1552 } 1553 } 1554 if (e->user_stack_size > 0 && e->user_stack_buildid_size > 0) 1555 good_user_stack = true; 1556 } 1557 if (!good_kern_stack || !good_user_stack) 1558 return LIBBPF_PERF_EVENT_ERROR; 1559 1560 if (cnt == MAX_CNT_RAWTP) 1561 return LIBBPF_PERF_EVENT_DONE; 1562 1563 return LIBBPF_PERF_EVENT_CONT; 1564 } 1565 1566 static void test_get_stack_raw_tp(void) 1567 { 1568 const char *file = "./test_get_stack_rawtp.o"; 1569 int i, efd, err, prog_fd, pmu_fd, perfmap_fd; 1570 struct perf_event_attr attr = {}; 1571 struct timespec tv = {0, 10}; 1572 __u32 key = 0, duration = 0; 1573 struct bpf_object *obj; 1574 1575 err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd); 1576 if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno)) 1577 return; 1578 1579 efd = bpf_raw_tracepoint_open("sys_enter", prog_fd); 1580 if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno)) 1581 goto close_prog; 1582 1583 perfmap_fd = bpf_find_map(__func__, obj, "perfmap"); 1584 if (CHECK(perfmap_fd < 0, "bpf_find_map", "err %d errno %d\n", 1585 perfmap_fd, errno)) 1586 goto close_prog; 1587 1588 err = load_kallsyms(); 1589 if (CHECK(err < 0, "load_kallsyms", "err %d errno %d\n", err, errno)) 1590 goto close_prog; 1591 1592 attr.sample_type = PERF_SAMPLE_RAW; 1593 attr.type = PERF_TYPE_SOFTWARE; 1594 attr.config = PERF_COUNT_SW_BPF_OUTPUT; 1595 pmu_fd = syscall(__NR_perf_event_open, &attr, getpid()/*pid*/, -1/*cpu*/, 1596 -1/*group_fd*/, 0); 1597 if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n", pmu_fd, 1598 errno)) 1599 goto close_prog; 1600 1601 err = bpf_map_update_elem(perfmap_fd, &key, &pmu_fd, BPF_ANY); 1602 if (CHECK(err < 0, "bpf_map_update_elem", "err %d errno %d\n", err, 1603 errno)) 1604 goto close_prog; 1605 1606 err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0); 1607 if (CHECK(err < 0, "ioctl PERF_EVENT_IOC_ENABLE", "err %d errno %d\n", 1608 err, errno)) 1609 goto close_prog; 1610 1611 err = perf_event_mmap(pmu_fd); 1612 if (CHECK(err < 0, "perf_event_mmap", "err %d errno %d\n", err, errno)) 1613 goto close_prog; 1614 1615 /* trigger some syscall action */ 1616 for (i = 0; i < MAX_CNT_RAWTP; i++) 1617 nanosleep(&tv, NULL); 1618 1619 err = perf_event_poller(pmu_fd, get_stack_print_output); 1620 if (CHECK(err < 0, "perf_event_poller", "err %d errno %d\n", err, errno)) 1621 goto close_prog; 1622 1623 goto close_prog_noerr; 1624 close_prog: 1625 error_cnt++; 1626 close_prog_noerr: 1627 bpf_object__close(obj); 1628 } 1629 1630 static void test_task_fd_query_rawtp(void) 1631 { 1632 const char *file = "./test_get_stack_rawtp.o"; 1633 __u64 probe_offset, probe_addr; 1634 __u32 len, prog_id, fd_type; 1635 struct bpf_object *obj; 1636 int efd, err, prog_fd; 1637 __u32 duration = 0; 1638 char buf[256]; 1639 1640 err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd); 1641 if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno)) 1642 return; 1643 1644 efd = bpf_raw_tracepoint_open("sys_enter", prog_fd); 1645 if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno)) 1646 goto close_prog; 1647 1648 /* query (getpid(), efd) */ 1649 len = sizeof(buf); 1650 err = bpf_task_fd_query(getpid(), efd, 0, buf, &len, &prog_id, 1651 &fd_type, &probe_offset, &probe_addr); 1652 if (CHECK(err < 0, "bpf_task_fd_query", "err %d errno %d\n", err, 1653 errno)) 1654 goto close_prog; 1655 1656 err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT && 1657 strcmp(buf, "sys_enter") == 0; 1658 if (CHECK(!err, "check_results", "fd_type %d tp_name %s\n", 1659 fd_type, buf)) 1660 goto close_prog; 1661 1662 /* test zero len */ 1663 len = 0; 1664 err = bpf_task_fd_query(getpid(), efd, 0, buf, &len, &prog_id, 1665 &fd_type, &probe_offset, &probe_addr); 1666 if (CHECK(err < 0, "bpf_task_fd_query (len = 0)", "err %d errno %d\n", 1667 err, errno)) 1668 goto close_prog; 1669 err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT && 1670 len == strlen("sys_enter"); 1671 if (CHECK(!err, "check_results", "fd_type %d len %u\n", fd_type, len)) 1672 goto close_prog; 1673 1674 /* test empty buffer */ 1675 len = sizeof(buf); 1676 err = bpf_task_fd_query(getpid(), efd, 0, 0, &len, &prog_id, 1677 &fd_type, &probe_offset, &probe_addr); 1678 if (CHECK(err < 0, "bpf_task_fd_query (buf = 0)", "err %d errno %d\n", 1679 err, errno)) 1680 goto close_prog; 1681 err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT && 1682 len == strlen("sys_enter"); 1683 if (CHECK(!err, "check_results", "fd_type %d len %u\n", fd_type, len)) 1684 goto close_prog; 1685 1686 /* test smaller buffer */ 1687 len = 3; 1688 err = bpf_task_fd_query(getpid(), efd, 0, buf, &len, &prog_id, 1689 &fd_type, &probe_offset, &probe_addr); 1690 if (CHECK(err >= 0 || errno != ENOSPC, "bpf_task_fd_query (len = 3)", 1691 "err %d errno %d\n", err, errno)) 1692 goto close_prog; 1693 err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT && 1694 len == strlen("sys_enter") && 1695 strcmp(buf, "sy") == 0; 1696 if (CHECK(!err, "check_results", "fd_type %d len %u\n", fd_type, len)) 1697 goto close_prog; 1698 1699 goto close_prog_noerr; 1700 close_prog: 1701 error_cnt++; 1702 close_prog_noerr: 1703 bpf_object__close(obj); 1704 } 1705 1706 static void test_task_fd_query_tp_core(const char *probe_name, 1707 const char *tp_name) 1708 { 1709 const char *file = "./test_tracepoint.o"; 1710 int err, bytes, efd, prog_fd, pmu_fd; 1711 struct perf_event_attr attr = {}; 1712 __u64 probe_offset, probe_addr; 1713 __u32 len, prog_id, fd_type; 1714 struct bpf_object *obj; 1715 __u32 duration = 0; 1716 char buf[256]; 1717 1718 err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd); 1719 if (CHECK(err, "bpf_prog_load", "err %d errno %d\n", err, errno)) 1720 goto close_prog; 1721 1722 snprintf(buf, sizeof(buf), 1723 "/sys/kernel/debug/tracing/events/%s/id", probe_name); 1724 efd = open(buf, O_RDONLY, 0); 1725 if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno)) 1726 goto close_prog; 1727 bytes = read(efd, buf, sizeof(buf)); 1728 close(efd); 1729 if (CHECK(bytes <= 0 || bytes >= sizeof(buf), "read", 1730 "bytes %d errno %d\n", bytes, errno)) 1731 goto close_prog; 1732 1733 attr.config = strtol(buf, NULL, 0); 1734 attr.type = PERF_TYPE_TRACEPOINT; 1735 attr.sample_type = PERF_SAMPLE_RAW; 1736 attr.sample_period = 1; 1737 attr.wakeup_events = 1; 1738 pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 1739 0 /* cpu 0 */, -1 /* group id */, 1740 0 /* flags */); 1741 if (CHECK(err, "perf_event_open", "err %d errno %d\n", err, errno)) 1742 goto close_pmu; 1743 1744 err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0); 1745 if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n", err, 1746 errno)) 1747 goto close_pmu; 1748 1749 err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd); 1750 if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n", err, 1751 errno)) 1752 goto close_pmu; 1753 1754 /* query (getpid(), pmu_fd) */ 1755 len = sizeof(buf); 1756 err = bpf_task_fd_query(getpid(), pmu_fd, 0, buf, &len, &prog_id, 1757 &fd_type, &probe_offset, &probe_addr); 1758 if (CHECK(err < 0, "bpf_task_fd_query", "err %d errno %d\n", err, 1759 errno)) 1760 goto close_pmu; 1761 1762 err = (fd_type == BPF_FD_TYPE_TRACEPOINT) && !strcmp(buf, tp_name); 1763 if (CHECK(!err, "check_results", "fd_type %d tp_name %s\n", 1764 fd_type, buf)) 1765 goto close_pmu; 1766 1767 close(pmu_fd); 1768 goto close_prog_noerr; 1769 1770 close_pmu: 1771 close(pmu_fd); 1772 close_prog: 1773 error_cnt++; 1774 close_prog_noerr: 1775 bpf_object__close(obj); 1776 } 1777 1778 static void test_task_fd_query_tp(void) 1779 { 1780 test_task_fd_query_tp_core("sched/sched_switch", 1781 "sched_switch"); 1782 test_task_fd_query_tp_core("syscalls/sys_enter_read", 1783 "sys_enter_read"); 1784 } 1785 1786 static void test_reference_tracking() 1787 { 1788 const char *file = "./test_sk_lookup_kern.o"; 1789 struct bpf_object *obj; 1790 struct bpf_program *prog; 1791 __u32 duration = 0; 1792 int err = 0; 1793 1794 obj = bpf_object__open(file); 1795 if (IS_ERR(obj)) { 1796 error_cnt++; 1797 return; 1798 } 1799 1800 bpf_object__for_each_program(prog, obj) { 1801 const char *title; 1802 1803 /* Ignore .text sections */ 1804 title = bpf_program__title(prog, false); 1805 if (strstr(title, ".text") != NULL) 1806 continue; 1807 1808 bpf_program__set_type(prog, BPF_PROG_TYPE_SCHED_CLS); 1809 1810 /* Expect verifier failure if test name has 'fail' */ 1811 if (strstr(title, "fail") != NULL) { 1812 libbpf_set_print(NULL, NULL, NULL); 1813 err = !bpf_program__load(prog, "GPL", 0); 1814 libbpf_set_print(printf, printf, NULL); 1815 } else { 1816 err = bpf_program__load(prog, "GPL", 0); 1817 } 1818 CHECK(err, title, "\n"); 1819 } 1820 bpf_object__close(obj); 1821 } 1822 1823 enum { 1824 QUEUE, 1825 STACK, 1826 }; 1827 1828 static void test_queue_stack_map(int type) 1829 { 1830 const int MAP_SIZE = 32; 1831 __u32 vals[MAP_SIZE], duration, retval, size, val; 1832 int i, err, prog_fd, map_in_fd, map_out_fd; 1833 char file[32], buf[128]; 1834 struct bpf_object *obj; 1835 struct iphdr *iph = (void *)buf + sizeof(struct ethhdr); 1836 1837 /* Fill test values to be used */ 1838 for (i = 0; i < MAP_SIZE; i++) 1839 vals[i] = rand(); 1840 1841 if (type == QUEUE) 1842 strncpy(file, "./test_queue_map.o", sizeof(file)); 1843 else if (type == STACK) 1844 strncpy(file, "./test_stack_map.o", sizeof(file)); 1845 else 1846 return; 1847 1848 err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); 1849 if (err) { 1850 error_cnt++; 1851 return; 1852 } 1853 1854 map_in_fd = bpf_find_map(__func__, obj, "map_in"); 1855 if (map_in_fd < 0) 1856 goto out; 1857 1858 map_out_fd = bpf_find_map(__func__, obj, "map_out"); 1859 if (map_out_fd < 0) 1860 goto out; 1861 1862 /* Push 32 elements to the input map */ 1863 for (i = 0; i < MAP_SIZE; i++) { 1864 err = bpf_map_update_elem(map_in_fd, NULL, &vals[i], 0); 1865 if (err) { 1866 error_cnt++; 1867 goto out; 1868 } 1869 } 1870 1871 /* The eBPF program pushes iph.saddr in the output map, 1872 * pops the input map and saves this value in iph.daddr 1873 */ 1874 for (i = 0; i < MAP_SIZE; i++) { 1875 if (type == QUEUE) { 1876 val = vals[i]; 1877 pkt_v4.iph.saddr = vals[i] * 5; 1878 } else if (type == STACK) { 1879 val = vals[MAP_SIZE - 1 - i]; 1880 pkt_v4.iph.saddr = vals[MAP_SIZE - 1 - i] * 5; 1881 } 1882 1883 err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4), 1884 buf, &size, &retval, &duration); 1885 if (err || retval || size != sizeof(pkt_v4) || 1886 iph->daddr != val) 1887 break; 1888 } 1889 1890 CHECK(err || retval || size != sizeof(pkt_v4) || iph->daddr != val, 1891 "bpf_map_pop_elem", 1892 "err %d errno %d retval %d size %d iph->daddr %u\n", 1893 err, errno, retval, size, iph->daddr); 1894 1895 /* Queue is empty, program should return TC_ACT_SHOT */ 1896 err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4), 1897 buf, &size, &retval, &duration); 1898 CHECK(err || retval != 2 /* TC_ACT_SHOT */|| size != sizeof(pkt_v4), 1899 "check-queue-stack-map-empty", 1900 "err %d errno %d retval %d size %d\n", 1901 err, errno, retval, size); 1902 1903 /* Check that the program pushed elements correctly */ 1904 for (i = 0; i < MAP_SIZE; i++) { 1905 err = bpf_map_lookup_and_delete_elem(map_out_fd, NULL, &val); 1906 if (err || val != vals[i] * 5) 1907 break; 1908 } 1909 1910 CHECK(i != MAP_SIZE && (err || val != vals[i] * 5), 1911 "bpf_map_push_elem", "err %d value %u\n", err, val); 1912 1913 out: 1914 pkt_v4.iph.saddr = 0; 1915 bpf_object__close(obj); 1916 } 1917 1918 #define CHECK_FLOW_KEYS(desc, got, expected) \ 1919 CHECK(memcmp(&got, &expected, sizeof(got)) != 0, \ 1920 desc, \ 1921 "nhoff=%u/%u " \ 1922 "thoff=%u/%u " \ 1923 "addr_proto=0x%x/0x%x " \ 1924 "is_frag=%u/%u " \ 1925 "is_first_frag=%u/%u " \ 1926 "is_encap=%u/%u " \ 1927 "n_proto=0x%x/0x%x " \ 1928 "sport=%u/%u " \ 1929 "dport=%u/%u\n", \ 1930 got.nhoff, expected.nhoff, \ 1931 got.thoff, expected.thoff, \ 1932 got.addr_proto, expected.addr_proto, \ 1933 got.is_frag, expected.is_frag, \ 1934 got.is_first_frag, expected.is_first_frag, \ 1935 got.is_encap, expected.is_encap, \ 1936 got.n_proto, expected.n_proto, \ 1937 got.sport, expected.sport, \ 1938 got.dport, expected.dport) 1939 1940 static struct bpf_flow_keys pkt_v4_flow_keys = { 1941 .nhoff = 0, 1942 .thoff = sizeof(struct iphdr), 1943 .addr_proto = ETH_P_IP, 1944 .ip_proto = IPPROTO_TCP, 1945 .n_proto = bpf_htons(ETH_P_IP), 1946 }; 1947 1948 static struct bpf_flow_keys pkt_v6_flow_keys = { 1949 .nhoff = 0, 1950 .thoff = sizeof(struct ipv6hdr), 1951 .addr_proto = ETH_P_IPV6, 1952 .ip_proto = IPPROTO_TCP, 1953 .n_proto = bpf_htons(ETH_P_IPV6), 1954 }; 1955 1956 static void test_flow_dissector(void) 1957 { 1958 struct bpf_flow_keys flow_keys; 1959 struct bpf_object *obj; 1960 __u32 duration, retval; 1961 int err, prog_fd; 1962 __u32 size; 1963 1964 err = bpf_flow_load(&obj, "./bpf_flow.o", "flow_dissector", 1965 "jmp_table", &prog_fd); 1966 if (err) { 1967 error_cnt++; 1968 return; 1969 } 1970 1971 err = bpf_prog_test_run(prog_fd, 10, &pkt_v4, sizeof(pkt_v4), 1972 &flow_keys, &size, &retval, &duration); 1973 CHECK(size != sizeof(flow_keys) || err || retval != 1, "ipv4", 1974 "err %d errno %d retval %d duration %d size %u/%lu\n", 1975 err, errno, retval, duration, size, sizeof(flow_keys)); 1976 CHECK_FLOW_KEYS("ipv4_flow_keys", flow_keys, pkt_v4_flow_keys); 1977 1978 err = bpf_prog_test_run(prog_fd, 10, &pkt_v6, sizeof(pkt_v6), 1979 &flow_keys, &size, &retval, &duration); 1980 CHECK(size != sizeof(flow_keys) || err || retval != 1, "ipv6", 1981 "err %d errno %d retval %d duration %d size %u/%lu\n", 1982 err, errno, retval, duration, size, sizeof(flow_keys)); 1983 CHECK_FLOW_KEYS("ipv6_flow_keys", flow_keys, pkt_v6_flow_keys); 1984 1985 bpf_object__close(obj); 1986 } 1987 1988 int main(void) 1989 { 1990 srand(time(NULL)); 1991 1992 jit_enabled = is_jit_enabled(); 1993 1994 test_pkt_access(); 1995 test_prog_run_xattr(); 1996 test_xdp(); 1997 test_xdp_adjust_tail(); 1998 test_l4lb_all(); 1999 test_xdp_noinline(); 2000 test_tcp_estats(); 2001 test_bpf_obj_id(); 2002 test_pkt_md_access(); 2003 test_obj_name(); 2004 test_tp_attach_query(); 2005 test_stacktrace_map(); 2006 test_stacktrace_build_id(); 2007 test_stacktrace_build_id_nmi(); 2008 test_stacktrace_map_raw_tp(); 2009 test_get_stack_raw_tp(); 2010 test_task_fd_query_rawtp(); 2011 test_task_fd_query_tp(); 2012 test_reference_tracking(); 2013 test_queue_stack_map(QUEUE); 2014 test_queue_stack_map(STACK); 2015 test_flow_dissector(); 2016 2017 printf("Summary: %d PASSED, %d FAILED\n", pass_cnt, error_cnt); 2018 return error_cnt ? EXIT_FAILURE : EXIT_SUCCESS; 2019 } 2020