1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2017-2018 Netronome Systems, Inc. */ 3 4 #define _GNU_SOURCE 5 #include <errno.h> 6 #include <fcntl.h> 7 #include <signal.h> 8 #include <stdarg.h> 9 #include <stdio.h> 10 #include <stdlib.h> 11 #include <string.h> 12 #include <time.h> 13 #include <unistd.h> 14 #include <net/if.h> 15 #include <sys/ioctl.h> 16 #include <sys/types.h> 17 #include <sys/stat.h> 18 #include <sys/syscall.h> 19 20 #include <linux/err.h> 21 #include <linux/perf_event.h> 22 #include <linux/sizes.h> 23 24 #include <bpf/bpf.h> 25 #include <bpf/btf.h> 26 #include <bpf/libbpf.h> 27 28 #include "cfg.h" 29 #include "main.h" 30 #include "xlated_dumper.h" 31 32 const char * const prog_type_name[] = { 33 [BPF_PROG_TYPE_UNSPEC] = "unspec", 34 [BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter", 35 [BPF_PROG_TYPE_KPROBE] = "kprobe", 36 [BPF_PROG_TYPE_SCHED_CLS] = "sched_cls", 37 [BPF_PROG_TYPE_SCHED_ACT] = "sched_act", 38 [BPF_PROG_TYPE_TRACEPOINT] = "tracepoint", 39 [BPF_PROG_TYPE_XDP] = "xdp", 40 [BPF_PROG_TYPE_PERF_EVENT] = "perf_event", 41 [BPF_PROG_TYPE_CGROUP_SKB] = "cgroup_skb", 42 [BPF_PROG_TYPE_CGROUP_SOCK] = "cgroup_sock", 43 [BPF_PROG_TYPE_LWT_IN] = "lwt_in", 44 [BPF_PROG_TYPE_LWT_OUT] = "lwt_out", 45 [BPF_PROG_TYPE_LWT_XMIT] = "lwt_xmit", 46 [BPF_PROG_TYPE_SOCK_OPS] = "sock_ops", 47 [BPF_PROG_TYPE_SK_SKB] = "sk_skb", 48 [BPF_PROG_TYPE_CGROUP_DEVICE] = "cgroup_device", 49 [BPF_PROG_TYPE_SK_MSG] = "sk_msg", 50 [BPF_PROG_TYPE_RAW_TRACEPOINT] = "raw_tracepoint", 51 [BPF_PROG_TYPE_CGROUP_SOCK_ADDR] = "cgroup_sock_addr", 52 [BPF_PROG_TYPE_LWT_SEG6LOCAL] = "lwt_seg6local", 53 [BPF_PROG_TYPE_LIRC_MODE2] = "lirc_mode2", 54 [BPF_PROG_TYPE_SK_REUSEPORT] = "sk_reuseport", 55 [BPF_PROG_TYPE_FLOW_DISSECTOR] = "flow_dissector", 56 [BPF_PROG_TYPE_CGROUP_SYSCTL] = "cgroup_sysctl", 57 [BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE] = "raw_tracepoint_writable", 58 [BPF_PROG_TYPE_CGROUP_SOCKOPT] = "cgroup_sockopt", 59 [BPF_PROG_TYPE_TRACING] = "tracing", 60 [BPF_PROG_TYPE_STRUCT_OPS] = "struct_ops", 61 [BPF_PROG_TYPE_EXT] = "ext", 62 [BPF_PROG_TYPE_SK_LOOKUP] = "sk_lookup", 63 }; 64 65 const size_t prog_type_name_size = ARRAY_SIZE(prog_type_name); 66 67 enum dump_mode { 68 DUMP_JITED, 69 DUMP_XLATED, 70 }; 71 72 static const char * const attach_type_strings[] = { 73 [BPF_SK_SKB_STREAM_PARSER] = "stream_parser", 74 [BPF_SK_SKB_STREAM_VERDICT] = "stream_verdict", 75 [BPF_SK_MSG_VERDICT] = "msg_verdict", 76 [BPF_FLOW_DISSECTOR] = "flow_dissector", 77 [__MAX_BPF_ATTACH_TYPE] = NULL, 78 }; 79 80 static enum bpf_attach_type parse_attach_type(const char *str) 81 { 82 enum bpf_attach_type type; 83 84 for (type = 0; type < __MAX_BPF_ATTACH_TYPE; type++) { 85 if (attach_type_strings[type] && 86 is_prefix(str, attach_type_strings[type])) 87 return type; 88 } 89 90 return __MAX_BPF_ATTACH_TYPE; 91 } 92 93 static void print_boot_time(__u64 nsecs, char *buf, unsigned int size) 94 { 95 struct timespec real_time_ts, boot_time_ts; 96 time_t wallclock_secs; 97 struct tm load_tm; 98 99 buf[--size] = '\0'; 100 101 if (clock_gettime(CLOCK_REALTIME, &real_time_ts) || 102 clock_gettime(CLOCK_BOOTTIME, &boot_time_ts)) { 103 perror("Can't read clocks"); 104 snprintf(buf, size, "%llu", nsecs / 1000000000); 105 return; 106 } 107 108 wallclock_secs = (real_time_ts.tv_sec - boot_time_ts.tv_sec) + 109 (real_time_ts.tv_nsec - boot_time_ts.tv_nsec + nsecs) / 110 1000000000; 111 112 113 if (!localtime_r(&wallclock_secs, &load_tm)) { 114 snprintf(buf, size, "%llu", nsecs / 1000000000); 115 return; 116 } 117 118 if (json_output) 119 strftime(buf, size, "%s", &load_tm); 120 else 121 strftime(buf, size, "%FT%T%z", &load_tm); 122 } 123 124 static void show_prog_maps(int fd, __u32 num_maps) 125 { 126 struct bpf_prog_info info = {}; 127 __u32 len = sizeof(info); 128 __u32 map_ids[num_maps]; 129 unsigned int i; 130 int err; 131 132 info.nr_map_ids = num_maps; 133 info.map_ids = ptr_to_u64(map_ids); 134 135 err = bpf_obj_get_info_by_fd(fd, &info, &len); 136 if (err || !info.nr_map_ids) 137 return; 138 139 if (json_output) { 140 jsonw_name(json_wtr, "map_ids"); 141 jsonw_start_array(json_wtr); 142 for (i = 0; i < info.nr_map_ids; i++) 143 jsonw_uint(json_wtr, map_ids[i]); 144 jsonw_end_array(json_wtr); 145 } else { 146 printf(" map_ids "); 147 for (i = 0; i < info.nr_map_ids; i++) 148 printf("%u%s", map_ids[i], 149 i == info.nr_map_ids - 1 ? "" : ","); 150 } 151 } 152 153 static void print_prog_header_json(struct bpf_prog_info *info) 154 { 155 jsonw_uint_field(json_wtr, "id", info->id); 156 if (info->type < ARRAY_SIZE(prog_type_name)) 157 jsonw_string_field(json_wtr, "type", 158 prog_type_name[info->type]); 159 else 160 jsonw_uint_field(json_wtr, "type", info->type); 161 162 if (*info->name) 163 jsonw_string_field(json_wtr, "name", info->name); 164 165 jsonw_name(json_wtr, "tag"); 166 jsonw_printf(json_wtr, "\"" BPF_TAG_FMT "\"", 167 info->tag[0], info->tag[1], info->tag[2], info->tag[3], 168 info->tag[4], info->tag[5], info->tag[6], info->tag[7]); 169 170 jsonw_bool_field(json_wtr, "gpl_compatible", info->gpl_compatible); 171 if (info->run_time_ns) { 172 jsonw_uint_field(json_wtr, "run_time_ns", info->run_time_ns); 173 jsonw_uint_field(json_wtr, "run_cnt", info->run_cnt); 174 } 175 } 176 177 static void print_prog_json(struct bpf_prog_info *info, int fd) 178 { 179 char *memlock; 180 181 jsonw_start_object(json_wtr); 182 print_prog_header_json(info); 183 print_dev_json(info->ifindex, info->netns_dev, info->netns_ino); 184 185 if (info->load_time) { 186 char buf[32]; 187 188 print_boot_time(info->load_time, buf, sizeof(buf)); 189 190 /* Piggy back on load_time, since 0 uid is a valid one */ 191 jsonw_name(json_wtr, "loaded_at"); 192 jsonw_printf(json_wtr, "%s", buf); 193 jsonw_uint_field(json_wtr, "uid", info->created_by_uid); 194 } 195 196 jsonw_uint_field(json_wtr, "bytes_xlated", info->xlated_prog_len); 197 198 if (info->jited_prog_len) { 199 jsonw_bool_field(json_wtr, "jited", true); 200 jsonw_uint_field(json_wtr, "bytes_jited", info->jited_prog_len); 201 } else { 202 jsonw_bool_field(json_wtr, "jited", false); 203 } 204 205 memlock = get_fdinfo(fd, "memlock"); 206 if (memlock) 207 jsonw_int_field(json_wtr, "bytes_memlock", atoi(memlock)); 208 free(memlock); 209 210 if (info->nr_map_ids) 211 show_prog_maps(fd, info->nr_map_ids); 212 213 if (info->btf_id) 214 jsonw_int_field(json_wtr, "btf_id", info->btf_id); 215 216 if (!hash_empty(prog_table.table)) { 217 struct pinned_obj *obj; 218 219 jsonw_name(json_wtr, "pinned"); 220 jsonw_start_array(json_wtr); 221 hash_for_each_possible(prog_table.table, obj, hash, info->id) { 222 if (obj->id == info->id) 223 jsonw_string(json_wtr, obj->path); 224 } 225 jsonw_end_array(json_wtr); 226 } 227 228 emit_obj_refs_json(&refs_table, info->id, json_wtr); 229 230 jsonw_end_object(json_wtr); 231 } 232 233 static void print_prog_header_plain(struct bpf_prog_info *info) 234 { 235 printf("%u: ", info->id); 236 if (info->type < ARRAY_SIZE(prog_type_name)) 237 printf("%s ", prog_type_name[info->type]); 238 else 239 printf("type %u ", info->type); 240 241 if (*info->name) 242 printf("name %s ", info->name); 243 244 printf("tag "); 245 fprint_hex(stdout, info->tag, BPF_TAG_SIZE, ""); 246 print_dev_plain(info->ifindex, info->netns_dev, info->netns_ino); 247 printf("%s", info->gpl_compatible ? " gpl" : ""); 248 if (info->run_time_ns) 249 printf(" run_time_ns %lld run_cnt %lld", 250 info->run_time_ns, info->run_cnt); 251 printf("\n"); 252 } 253 254 static void print_prog_plain(struct bpf_prog_info *info, int fd) 255 { 256 char *memlock; 257 258 print_prog_header_plain(info); 259 260 if (info->load_time) { 261 char buf[32]; 262 263 print_boot_time(info->load_time, buf, sizeof(buf)); 264 265 /* Piggy back on load_time, since 0 uid is a valid one */ 266 printf("\tloaded_at %s uid %u\n", buf, info->created_by_uid); 267 } 268 269 printf("\txlated %uB", info->xlated_prog_len); 270 271 if (info->jited_prog_len) 272 printf(" jited %uB", info->jited_prog_len); 273 else 274 printf(" not jited"); 275 276 memlock = get_fdinfo(fd, "memlock"); 277 if (memlock) 278 printf(" memlock %sB", memlock); 279 free(memlock); 280 281 if (info->nr_map_ids) 282 show_prog_maps(fd, info->nr_map_ids); 283 284 if (!hash_empty(prog_table.table)) { 285 struct pinned_obj *obj; 286 287 hash_for_each_possible(prog_table.table, obj, hash, info->id) { 288 if (obj->id == info->id) 289 printf("\n\tpinned %s", obj->path); 290 } 291 } 292 293 if (info->btf_id) 294 printf("\n\tbtf_id %d", info->btf_id); 295 296 emit_obj_refs_plain(&refs_table, info->id, "\n\tpids "); 297 298 printf("\n"); 299 } 300 301 static int show_prog(int fd) 302 { 303 struct bpf_prog_info info = {}; 304 __u32 len = sizeof(info); 305 int err; 306 307 err = bpf_obj_get_info_by_fd(fd, &info, &len); 308 if (err) { 309 p_err("can't get prog info: %s", strerror(errno)); 310 return -1; 311 } 312 313 if (json_output) 314 print_prog_json(&info, fd); 315 else 316 print_prog_plain(&info, fd); 317 318 return 0; 319 } 320 321 static int do_show_subset(int argc, char **argv) 322 { 323 int *fds = NULL; 324 int nb_fds, i; 325 int err = -1; 326 327 fds = malloc(sizeof(int)); 328 if (!fds) { 329 p_err("mem alloc failed"); 330 return -1; 331 } 332 nb_fds = prog_parse_fds(&argc, &argv, &fds); 333 if (nb_fds < 1) 334 goto exit_free; 335 336 if (json_output && nb_fds > 1) 337 jsonw_start_array(json_wtr); /* root array */ 338 for (i = 0; i < nb_fds; i++) { 339 err = show_prog(fds[i]); 340 if (err) { 341 for (; i < nb_fds; i++) 342 close(fds[i]); 343 break; 344 } 345 close(fds[i]); 346 } 347 if (json_output && nb_fds > 1) 348 jsonw_end_array(json_wtr); /* root array */ 349 350 exit_free: 351 free(fds); 352 return err; 353 } 354 355 static int do_show(int argc, char **argv) 356 { 357 __u32 id = 0; 358 int err; 359 int fd; 360 361 if (show_pinned) 362 build_pinned_obj_table(&prog_table, BPF_OBJ_PROG); 363 build_obj_refs_table(&refs_table, BPF_OBJ_PROG); 364 365 if (argc == 2) 366 return do_show_subset(argc, argv); 367 368 if (argc) 369 return BAD_ARG(); 370 371 if (json_output) 372 jsonw_start_array(json_wtr); 373 while (true) { 374 err = bpf_prog_get_next_id(id, &id); 375 if (err) { 376 if (errno == ENOENT) { 377 err = 0; 378 break; 379 } 380 p_err("can't get next program: %s%s", strerror(errno), 381 errno == EINVAL ? " -- kernel too old?" : ""); 382 err = -1; 383 break; 384 } 385 386 fd = bpf_prog_get_fd_by_id(id); 387 if (fd < 0) { 388 if (errno == ENOENT) 389 continue; 390 p_err("can't get prog by id (%u): %s", 391 id, strerror(errno)); 392 err = -1; 393 break; 394 } 395 396 err = show_prog(fd); 397 close(fd); 398 if (err) 399 break; 400 } 401 402 if (json_output) 403 jsonw_end_array(json_wtr); 404 405 delete_obj_refs_table(&refs_table); 406 407 return err; 408 } 409 410 static int 411 prog_dump(struct bpf_prog_info *info, enum dump_mode mode, 412 char *filepath, bool opcodes, bool visual, bool linum) 413 { 414 struct bpf_prog_linfo *prog_linfo = NULL; 415 const char *disasm_opt = NULL; 416 struct dump_data dd = {}; 417 void *func_info = NULL; 418 struct btf *btf = NULL; 419 char func_sig[1024]; 420 unsigned char *buf; 421 __u32 member_len; 422 ssize_t n; 423 int fd; 424 425 if (mode == DUMP_JITED) { 426 if (info->jited_prog_len == 0 || !info->jited_prog_insns) { 427 p_info("no instructions returned"); 428 return -1; 429 } 430 buf = (unsigned char *)(info->jited_prog_insns); 431 member_len = info->jited_prog_len; 432 } else { /* DUMP_XLATED */ 433 if (info->xlated_prog_len == 0 || !info->xlated_prog_insns) { 434 p_err("error retrieving insn dump: kernel.kptr_restrict set?"); 435 return -1; 436 } 437 buf = (unsigned char *)info->xlated_prog_insns; 438 member_len = info->xlated_prog_len; 439 } 440 441 if (info->btf_id && btf__get_from_id(info->btf_id, &btf)) { 442 p_err("failed to get btf"); 443 return -1; 444 } 445 446 func_info = (void *)info->func_info; 447 448 if (info->nr_line_info) { 449 prog_linfo = bpf_prog_linfo__new(info); 450 if (!prog_linfo) 451 p_info("error in processing bpf_line_info. continue without it."); 452 } 453 454 if (filepath) { 455 fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, 0600); 456 if (fd < 0) { 457 p_err("can't open file %s: %s", filepath, 458 strerror(errno)); 459 return -1; 460 } 461 462 n = write(fd, buf, member_len); 463 close(fd); 464 if (n != member_len) { 465 p_err("error writing output file: %s", 466 n < 0 ? strerror(errno) : "short write"); 467 return -1; 468 } 469 470 if (json_output) 471 jsonw_null(json_wtr); 472 } else if (mode == DUMP_JITED) { 473 const char *name = NULL; 474 475 if (info->ifindex) { 476 name = ifindex_to_bfd_params(info->ifindex, 477 info->netns_dev, 478 info->netns_ino, 479 &disasm_opt); 480 if (!name) 481 return -1; 482 } 483 484 if (info->nr_jited_func_lens && info->jited_func_lens) { 485 struct kernel_sym *sym = NULL; 486 struct bpf_func_info *record; 487 char sym_name[SYM_MAX_NAME]; 488 unsigned char *img = buf; 489 __u64 *ksyms = NULL; 490 __u32 *lens; 491 __u32 i; 492 if (info->nr_jited_ksyms) { 493 kernel_syms_load(&dd); 494 ksyms = (__u64 *) info->jited_ksyms; 495 } 496 497 if (json_output) 498 jsonw_start_array(json_wtr); 499 500 lens = (__u32 *) info->jited_func_lens; 501 for (i = 0; i < info->nr_jited_func_lens; i++) { 502 if (ksyms) { 503 sym = kernel_syms_search(&dd, ksyms[i]); 504 if (sym) 505 sprintf(sym_name, "%s", sym->name); 506 else 507 sprintf(sym_name, "0x%016llx", ksyms[i]); 508 } else { 509 strcpy(sym_name, "unknown"); 510 } 511 512 if (func_info) { 513 record = func_info + i * info->func_info_rec_size; 514 btf_dumper_type_only(btf, record->type_id, 515 func_sig, 516 sizeof(func_sig)); 517 } 518 519 if (json_output) { 520 jsonw_start_object(json_wtr); 521 if (func_info && func_sig[0] != '\0') { 522 jsonw_name(json_wtr, "proto"); 523 jsonw_string(json_wtr, func_sig); 524 } 525 jsonw_name(json_wtr, "name"); 526 jsonw_string(json_wtr, sym_name); 527 jsonw_name(json_wtr, "insns"); 528 } else { 529 if (func_info && func_sig[0] != '\0') 530 printf("%s:\n", func_sig); 531 printf("%s:\n", sym_name); 532 } 533 534 disasm_print_insn(img, lens[i], opcodes, 535 name, disasm_opt, btf, 536 prog_linfo, ksyms[i], i, 537 linum); 538 539 img += lens[i]; 540 541 if (json_output) 542 jsonw_end_object(json_wtr); 543 else 544 printf("\n"); 545 } 546 547 if (json_output) 548 jsonw_end_array(json_wtr); 549 } else { 550 disasm_print_insn(buf, member_len, opcodes, name, 551 disasm_opt, btf, NULL, 0, 0, false); 552 } 553 } else if (visual) { 554 if (json_output) 555 jsonw_null(json_wtr); 556 else 557 dump_xlated_cfg(buf, member_len); 558 } else { 559 kernel_syms_load(&dd); 560 dd.nr_jited_ksyms = info->nr_jited_ksyms; 561 dd.jited_ksyms = (__u64 *) info->jited_ksyms; 562 dd.btf = btf; 563 dd.func_info = func_info; 564 dd.finfo_rec_size = info->func_info_rec_size; 565 dd.prog_linfo = prog_linfo; 566 567 if (json_output) 568 dump_xlated_json(&dd, buf, member_len, opcodes, 569 linum); 570 else 571 dump_xlated_plain(&dd, buf, member_len, opcodes, 572 linum); 573 kernel_syms_destroy(&dd); 574 } 575 576 return 0; 577 } 578 579 static int do_dump(int argc, char **argv) 580 { 581 struct bpf_prog_info_linear *info_linear; 582 char *filepath = NULL; 583 bool opcodes = false; 584 bool visual = false; 585 enum dump_mode mode; 586 bool linum = false; 587 int *fds = NULL; 588 int nb_fds, i = 0; 589 int err = -1; 590 __u64 arrays; 591 592 if (is_prefix(*argv, "jited")) { 593 if (disasm_init()) 594 return -1; 595 mode = DUMP_JITED; 596 } else if (is_prefix(*argv, "xlated")) { 597 mode = DUMP_XLATED; 598 } else { 599 p_err("expected 'xlated' or 'jited', got: %s", *argv); 600 return -1; 601 } 602 NEXT_ARG(); 603 604 if (argc < 2) 605 usage(); 606 607 fds = malloc(sizeof(int)); 608 if (!fds) { 609 p_err("mem alloc failed"); 610 return -1; 611 } 612 nb_fds = prog_parse_fds(&argc, &argv, &fds); 613 if (nb_fds < 1) 614 goto exit_free; 615 616 if (is_prefix(*argv, "file")) { 617 NEXT_ARG(); 618 if (!argc) { 619 p_err("expected file path"); 620 goto exit_close; 621 } 622 if (nb_fds > 1) { 623 p_err("several programs matched"); 624 goto exit_close; 625 } 626 627 filepath = *argv; 628 NEXT_ARG(); 629 } else if (is_prefix(*argv, "opcodes")) { 630 opcodes = true; 631 NEXT_ARG(); 632 } else if (is_prefix(*argv, "visual")) { 633 if (nb_fds > 1) { 634 p_err("several programs matched"); 635 goto exit_close; 636 } 637 638 visual = true; 639 NEXT_ARG(); 640 } else if (is_prefix(*argv, "linum")) { 641 linum = true; 642 NEXT_ARG(); 643 } 644 645 if (argc) { 646 usage(); 647 goto exit_close; 648 } 649 650 if (mode == DUMP_JITED) 651 arrays = 1UL << BPF_PROG_INFO_JITED_INSNS; 652 else 653 arrays = 1UL << BPF_PROG_INFO_XLATED_INSNS; 654 655 arrays |= 1UL << BPF_PROG_INFO_JITED_KSYMS; 656 arrays |= 1UL << BPF_PROG_INFO_JITED_FUNC_LENS; 657 arrays |= 1UL << BPF_PROG_INFO_FUNC_INFO; 658 arrays |= 1UL << BPF_PROG_INFO_LINE_INFO; 659 arrays |= 1UL << BPF_PROG_INFO_JITED_LINE_INFO; 660 661 if (json_output && nb_fds > 1) 662 jsonw_start_array(json_wtr); /* root array */ 663 for (i = 0; i < nb_fds; i++) { 664 info_linear = bpf_program__get_prog_info_linear(fds[i], arrays); 665 if (IS_ERR_OR_NULL(info_linear)) { 666 p_err("can't get prog info: %s", strerror(errno)); 667 break; 668 } 669 670 if (json_output && nb_fds > 1) { 671 jsonw_start_object(json_wtr); /* prog object */ 672 print_prog_header_json(&info_linear->info); 673 jsonw_name(json_wtr, "insns"); 674 } else if (nb_fds > 1) { 675 print_prog_header_plain(&info_linear->info); 676 } 677 678 err = prog_dump(&info_linear->info, mode, filepath, opcodes, 679 visual, linum); 680 681 if (json_output && nb_fds > 1) 682 jsonw_end_object(json_wtr); /* prog object */ 683 else if (i != nb_fds - 1 && nb_fds > 1) 684 printf("\n"); 685 686 free(info_linear); 687 if (err) 688 break; 689 close(fds[i]); 690 } 691 if (json_output && nb_fds > 1) 692 jsonw_end_array(json_wtr); /* root array */ 693 694 exit_close: 695 for (; i < nb_fds; i++) 696 close(fds[i]); 697 exit_free: 698 free(fds); 699 return err; 700 } 701 702 static int do_pin(int argc, char **argv) 703 { 704 int err; 705 706 err = do_pin_any(argc, argv, prog_parse_fd); 707 if (!err && json_output) 708 jsonw_null(json_wtr); 709 return err; 710 } 711 712 struct map_replace { 713 int idx; 714 int fd; 715 char *name; 716 }; 717 718 static int map_replace_compar(const void *p1, const void *p2) 719 { 720 const struct map_replace *a = p1, *b = p2; 721 722 return a->idx - b->idx; 723 } 724 725 static int parse_attach_detach_args(int argc, char **argv, int *progfd, 726 enum bpf_attach_type *attach_type, 727 int *mapfd) 728 { 729 if (!REQ_ARGS(3)) 730 return -EINVAL; 731 732 *progfd = prog_parse_fd(&argc, &argv); 733 if (*progfd < 0) 734 return *progfd; 735 736 *attach_type = parse_attach_type(*argv); 737 if (*attach_type == __MAX_BPF_ATTACH_TYPE) { 738 p_err("invalid attach/detach type"); 739 return -EINVAL; 740 } 741 742 if (*attach_type == BPF_FLOW_DISSECTOR) { 743 *mapfd = -1; 744 return 0; 745 } 746 747 NEXT_ARG(); 748 if (!REQ_ARGS(2)) 749 return -EINVAL; 750 751 *mapfd = map_parse_fd(&argc, &argv); 752 if (*mapfd < 0) 753 return *mapfd; 754 755 return 0; 756 } 757 758 static int do_attach(int argc, char **argv) 759 { 760 enum bpf_attach_type attach_type; 761 int err, progfd; 762 int mapfd; 763 764 err = parse_attach_detach_args(argc, argv, 765 &progfd, &attach_type, &mapfd); 766 if (err) 767 return err; 768 769 err = bpf_prog_attach(progfd, mapfd, attach_type, 0); 770 if (err) { 771 p_err("failed prog attach to map"); 772 return -EINVAL; 773 } 774 775 if (json_output) 776 jsonw_null(json_wtr); 777 return 0; 778 } 779 780 static int do_detach(int argc, char **argv) 781 { 782 enum bpf_attach_type attach_type; 783 int err, progfd; 784 int mapfd; 785 786 err = parse_attach_detach_args(argc, argv, 787 &progfd, &attach_type, &mapfd); 788 if (err) 789 return err; 790 791 err = bpf_prog_detach2(progfd, mapfd, attach_type); 792 if (err) { 793 p_err("failed prog detach from map"); 794 return -EINVAL; 795 } 796 797 if (json_output) 798 jsonw_null(json_wtr); 799 return 0; 800 } 801 802 static int check_single_stdin(char *file_data_in, char *file_ctx_in) 803 { 804 if (file_data_in && file_ctx_in && 805 !strcmp(file_data_in, "-") && !strcmp(file_ctx_in, "-")) { 806 p_err("cannot use standard input for both data_in and ctx_in"); 807 return -1; 808 } 809 810 return 0; 811 } 812 813 static int get_run_data(const char *fname, void **data_ptr, unsigned int *size) 814 { 815 size_t block_size = 256; 816 size_t buf_size = block_size; 817 size_t nb_read = 0; 818 void *tmp; 819 FILE *f; 820 821 if (!fname) { 822 *data_ptr = NULL; 823 *size = 0; 824 return 0; 825 } 826 827 if (!strcmp(fname, "-")) 828 f = stdin; 829 else 830 f = fopen(fname, "r"); 831 if (!f) { 832 p_err("failed to open %s: %s", fname, strerror(errno)); 833 return -1; 834 } 835 836 *data_ptr = malloc(block_size); 837 if (!*data_ptr) { 838 p_err("failed to allocate memory for data_in/ctx_in: %s", 839 strerror(errno)); 840 goto err_fclose; 841 } 842 843 while ((nb_read += fread(*data_ptr + nb_read, 1, block_size, f))) { 844 if (feof(f)) 845 break; 846 if (ferror(f)) { 847 p_err("failed to read data_in/ctx_in from %s: %s", 848 fname, strerror(errno)); 849 goto err_free; 850 } 851 if (nb_read > buf_size - block_size) { 852 if (buf_size == UINT32_MAX) { 853 p_err("data_in/ctx_in is too long (max: %d)", 854 UINT32_MAX); 855 goto err_free; 856 } 857 /* No space for fread()-ing next chunk; realloc() */ 858 buf_size *= 2; 859 tmp = realloc(*data_ptr, buf_size); 860 if (!tmp) { 861 p_err("failed to reallocate data_in/ctx_in: %s", 862 strerror(errno)); 863 goto err_free; 864 } 865 *data_ptr = tmp; 866 } 867 } 868 if (f != stdin) 869 fclose(f); 870 871 *size = nb_read; 872 return 0; 873 874 err_free: 875 free(*data_ptr); 876 *data_ptr = NULL; 877 err_fclose: 878 if (f != stdin) 879 fclose(f); 880 return -1; 881 } 882 883 static void hex_print(void *data, unsigned int size, FILE *f) 884 { 885 size_t i, j; 886 char c; 887 888 for (i = 0; i < size; i += 16) { 889 /* Row offset */ 890 fprintf(f, "%07zx\t", i); 891 892 /* Hexadecimal values */ 893 for (j = i; j < i + 16 && j < size; j++) 894 fprintf(f, "%02x%s", *(uint8_t *)(data + j), 895 j % 2 ? " " : ""); 896 for (; j < i + 16; j++) 897 fprintf(f, " %s", j % 2 ? " " : ""); 898 899 /* ASCII values (if relevant), '.' otherwise */ 900 fprintf(f, "| "); 901 for (j = i; j < i + 16 && j < size; j++) { 902 c = *(char *)(data + j); 903 if (c < ' ' || c > '~') 904 c = '.'; 905 fprintf(f, "%c%s", c, j == i + 7 ? " " : ""); 906 } 907 908 fprintf(f, "\n"); 909 } 910 } 911 912 static int 913 print_run_output(void *data, unsigned int size, const char *fname, 914 const char *json_key) 915 { 916 size_t nb_written; 917 FILE *f; 918 919 if (!fname) 920 return 0; 921 922 if (!strcmp(fname, "-")) { 923 f = stdout; 924 if (json_output) { 925 jsonw_name(json_wtr, json_key); 926 print_data_json(data, size); 927 } else { 928 hex_print(data, size, f); 929 } 930 return 0; 931 } 932 933 f = fopen(fname, "w"); 934 if (!f) { 935 p_err("failed to open %s: %s", fname, strerror(errno)); 936 return -1; 937 } 938 939 nb_written = fwrite(data, 1, size, f); 940 fclose(f); 941 if (nb_written != size) { 942 p_err("failed to write output data/ctx: %s", strerror(errno)); 943 return -1; 944 } 945 946 return 0; 947 } 948 949 static int alloc_run_data(void **data_ptr, unsigned int size_out) 950 { 951 *data_ptr = calloc(size_out, 1); 952 if (!*data_ptr) { 953 p_err("failed to allocate memory for output data/ctx: %s", 954 strerror(errno)); 955 return -1; 956 } 957 958 return 0; 959 } 960 961 static int do_run(int argc, char **argv) 962 { 963 char *data_fname_in = NULL, *data_fname_out = NULL; 964 char *ctx_fname_in = NULL, *ctx_fname_out = NULL; 965 struct bpf_prog_test_run_attr test_attr = {0}; 966 const unsigned int default_size = SZ_32K; 967 void *data_in = NULL, *data_out = NULL; 968 void *ctx_in = NULL, *ctx_out = NULL; 969 unsigned int repeat = 1; 970 int fd, err; 971 972 if (!REQ_ARGS(4)) 973 return -1; 974 975 fd = prog_parse_fd(&argc, &argv); 976 if (fd < 0) 977 return -1; 978 979 while (argc) { 980 if (detect_common_prefix(*argv, "data_in", "data_out", 981 "data_size_out", NULL)) 982 return -1; 983 if (detect_common_prefix(*argv, "ctx_in", "ctx_out", 984 "ctx_size_out", NULL)) 985 return -1; 986 987 if (is_prefix(*argv, "data_in")) { 988 NEXT_ARG(); 989 if (!REQ_ARGS(1)) 990 return -1; 991 992 data_fname_in = GET_ARG(); 993 if (check_single_stdin(data_fname_in, ctx_fname_in)) 994 return -1; 995 } else if (is_prefix(*argv, "data_out")) { 996 NEXT_ARG(); 997 if (!REQ_ARGS(1)) 998 return -1; 999 1000 data_fname_out = GET_ARG(); 1001 } else if (is_prefix(*argv, "data_size_out")) { 1002 char *endptr; 1003 1004 NEXT_ARG(); 1005 if (!REQ_ARGS(1)) 1006 return -1; 1007 1008 test_attr.data_size_out = strtoul(*argv, &endptr, 0); 1009 if (*endptr) { 1010 p_err("can't parse %s as output data size", 1011 *argv); 1012 return -1; 1013 } 1014 NEXT_ARG(); 1015 } else if (is_prefix(*argv, "ctx_in")) { 1016 NEXT_ARG(); 1017 if (!REQ_ARGS(1)) 1018 return -1; 1019 1020 ctx_fname_in = GET_ARG(); 1021 if (check_single_stdin(data_fname_in, ctx_fname_in)) 1022 return -1; 1023 } else if (is_prefix(*argv, "ctx_out")) { 1024 NEXT_ARG(); 1025 if (!REQ_ARGS(1)) 1026 return -1; 1027 1028 ctx_fname_out = GET_ARG(); 1029 } else if (is_prefix(*argv, "ctx_size_out")) { 1030 char *endptr; 1031 1032 NEXT_ARG(); 1033 if (!REQ_ARGS(1)) 1034 return -1; 1035 1036 test_attr.ctx_size_out = strtoul(*argv, &endptr, 0); 1037 if (*endptr) { 1038 p_err("can't parse %s as output context size", 1039 *argv); 1040 return -1; 1041 } 1042 NEXT_ARG(); 1043 } else if (is_prefix(*argv, "repeat")) { 1044 char *endptr; 1045 1046 NEXT_ARG(); 1047 if (!REQ_ARGS(1)) 1048 return -1; 1049 1050 repeat = strtoul(*argv, &endptr, 0); 1051 if (*endptr) { 1052 p_err("can't parse %s as repeat number", 1053 *argv); 1054 return -1; 1055 } 1056 NEXT_ARG(); 1057 } else { 1058 p_err("expected no more arguments, 'data_in', 'data_out', 'data_size_out', 'ctx_in', 'ctx_out', 'ctx_size_out' or 'repeat', got: '%s'?", 1059 *argv); 1060 return -1; 1061 } 1062 } 1063 1064 err = get_run_data(data_fname_in, &data_in, &test_attr.data_size_in); 1065 if (err) 1066 return -1; 1067 1068 if (data_in) { 1069 if (!test_attr.data_size_out) 1070 test_attr.data_size_out = default_size; 1071 err = alloc_run_data(&data_out, test_attr.data_size_out); 1072 if (err) 1073 goto free_data_in; 1074 } 1075 1076 err = get_run_data(ctx_fname_in, &ctx_in, &test_attr.ctx_size_in); 1077 if (err) 1078 goto free_data_out; 1079 1080 if (ctx_in) { 1081 if (!test_attr.ctx_size_out) 1082 test_attr.ctx_size_out = default_size; 1083 err = alloc_run_data(&ctx_out, test_attr.ctx_size_out); 1084 if (err) 1085 goto free_ctx_in; 1086 } 1087 1088 test_attr.prog_fd = fd; 1089 test_attr.repeat = repeat; 1090 test_attr.data_in = data_in; 1091 test_attr.data_out = data_out; 1092 test_attr.ctx_in = ctx_in; 1093 test_attr.ctx_out = ctx_out; 1094 1095 err = bpf_prog_test_run_xattr(&test_attr); 1096 if (err) { 1097 p_err("failed to run program: %s", strerror(errno)); 1098 goto free_ctx_out; 1099 } 1100 1101 err = 0; 1102 1103 if (json_output) 1104 jsonw_start_object(json_wtr); /* root */ 1105 1106 /* Do not exit on errors occurring when printing output data/context, 1107 * we still want to print return value and duration for program run. 1108 */ 1109 if (test_attr.data_size_out) 1110 err += print_run_output(test_attr.data_out, 1111 test_attr.data_size_out, 1112 data_fname_out, "data_out"); 1113 if (test_attr.ctx_size_out) 1114 err += print_run_output(test_attr.ctx_out, 1115 test_attr.ctx_size_out, 1116 ctx_fname_out, "ctx_out"); 1117 1118 if (json_output) { 1119 jsonw_uint_field(json_wtr, "retval", test_attr.retval); 1120 jsonw_uint_field(json_wtr, "duration", test_attr.duration); 1121 jsonw_end_object(json_wtr); /* root */ 1122 } else { 1123 fprintf(stdout, "Return value: %u, duration%s: %uns\n", 1124 test_attr.retval, 1125 repeat > 1 ? " (average)" : "", test_attr.duration); 1126 } 1127 1128 free_ctx_out: 1129 free(ctx_out); 1130 free_ctx_in: 1131 free(ctx_in); 1132 free_data_out: 1133 free(data_out); 1134 free_data_in: 1135 free(data_in); 1136 1137 return err; 1138 } 1139 1140 static int 1141 get_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 1142 enum bpf_attach_type *expected_attach_type) 1143 { 1144 libbpf_print_fn_t print_backup; 1145 int ret; 1146 1147 ret = libbpf_prog_type_by_name(name, prog_type, expected_attach_type); 1148 if (!ret) 1149 return ret; 1150 1151 /* libbpf_prog_type_by_name() failed, let's re-run with debug level */ 1152 print_backup = libbpf_set_print(print_all_levels); 1153 ret = libbpf_prog_type_by_name(name, prog_type, expected_attach_type); 1154 libbpf_set_print(print_backup); 1155 1156 return ret; 1157 } 1158 1159 static int load_with_options(int argc, char **argv, bool first_prog_only) 1160 { 1161 enum bpf_prog_type common_prog_type = BPF_PROG_TYPE_UNSPEC; 1162 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, open_opts, 1163 .relaxed_maps = relaxed_maps, 1164 ); 1165 struct bpf_object_load_attr load_attr = { 0 }; 1166 enum bpf_attach_type expected_attach_type; 1167 struct map_replace *map_replace = NULL; 1168 struct bpf_program *prog = NULL, *pos; 1169 unsigned int old_map_fds = 0; 1170 const char *pinmaps = NULL; 1171 struct bpf_object *obj; 1172 struct bpf_map *map; 1173 const char *pinfile; 1174 unsigned int i, j; 1175 __u32 ifindex = 0; 1176 const char *file; 1177 int idx, err; 1178 1179 1180 if (!REQ_ARGS(2)) 1181 return -1; 1182 file = GET_ARG(); 1183 pinfile = GET_ARG(); 1184 1185 while (argc) { 1186 if (is_prefix(*argv, "type")) { 1187 char *type; 1188 1189 NEXT_ARG(); 1190 1191 if (common_prog_type != BPF_PROG_TYPE_UNSPEC) { 1192 p_err("program type already specified"); 1193 goto err_free_reuse_maps; 1194 } 1195 if (!REQ_ARGS(1)) 1196 goto err_free_reuse_maps; 1197 1198 /* Put a '/' at the end of type to appease libbpf */ 1199 type = malloc(strlen(*argv) + 2); 1200 if (!type) { 1201 p_err("mem alloc failed"); 1202 goto err_free_reuse_maps; 1203 } 1204 *type = 0; 1205 strcat(type, *argv); 1206 strcat(type, "/"); 1207 1208 err = get_prog_type_by_name(type, &common_prog_type, 1209 &expected_attach_type); 1210 free(type); 1211 if (err < 0) 1212 goto err_free_reuse_maps; 1213 1214 NEXT_ARG(); 1215 } else if (is_prefix(*argv, "map")) { 1216 void *new_map_replace; 1217 char *endptr, *name; 1218 int fd; 1219 1220 NEXT_ARG(); 1221 1222 if (!REQ_ARGS(4)) 1223 goto err_free_reuse_maps; 1224 1225 if (is_prefix(*argv, "idx")) { 1226 NEXT_ARG(); 1227 1228 idx = strtoul(*argv, &endptr, 0); 1229 if (*endptr) { 1230 p_err("can't parse %s as IDX", *argv); 1231 goto err_free_reuse_maps; 1232 } 1233 name = NULL; 1234 } else if (is_prefix(*argv, "name")) { 1235 NEXT_ARG(); 1236 1237 name = *argv; 1238 idx = -1; 1239 } else { 1240 p_err("expected 'idx' or 'name', got: '%s'?", 1241 *argv); 1242 goto err_free_reuse_maps; 1243 } 1244 NEXT_ARG(); 1245 1246 fd = map_parse_fd(&argc, &argv); 1247 if (fd < 0) 1248 goto err_free_reuse_maps; 1249 1250 new_map_replace = reallocarray(map_replace, 1251 old_map_fds + 1, 1252 sizeof(*map_replace)); 1253 if (!new_map_replace) { 1254 p_err("mem alloc failed"); 1255 goto err_free_reuse_maps; 1256 } 1257 map_replace = new_map_replace; 1258 1259 map_replace[old_map_fds].idx = idx; 1260 map_replace[old_map_fds].name = name; 1261 map_replace[old_map_fds].fd = fd; 1262 old_map_fds++; 1263 } else if (is_prefix(*argv, "dev")) { 1264 NEXT_ARG(); 1265 1266 if (ifindex) { 1267 p_err("offload device already specified"); 1268 goto err_free_reuse_maps; 1269 } 1270 if (!REQ_ARGS(1)) 1271 goto err_free_reuse_maps; 1272 1273 ifindex = if_nametoindex(*argv); 1274 if (!ifindex) { 1275 p_err("unrecognized netdevice '%s': %s", 1276 *argv, strerror(errno)); 1277 goto err_free_reuse_maps; 1278 } 1279 NEXT_ARG(); 1280 } else if (is_prefix(*argv, "pinmaps")) { 1281 NEXT_ARG(); 1282 1283 if (!REQ_ARGS(1)) 1284 goto err_free_reuse_maps; 1285 1286 pinmaps = GET_ARG(); 1287 } else { 1288 p_err("expected no more arguments, 'type', 'map' or 'dev', got: '%s'?", 1289 *argv); 1290 goto err_free_reuse_maps; 1291 } 1292 } 1293 1294 set_max_rlimit(); 1295 1296 obj = bpf_object__open_file(file, &open_opts); 1297 if (IS_ERR_OR_NULL(obj)) { 1298 p_err("failed to open object file"); 1299 goto err_free_reuse_maps; 1300 } 1301 1302 bpf_object__for_each_program(pos, obj) { 1303 enum bpf_prog_type prog_type = common_prog_type; 1304 1305 if (prog_type == BPF_PROG_TYPE_UNSPEC) { 1306 const char *sec_name = bpf_program__title(pos, false); 1307 1308 err = get_prog_type_by_name(sec_name, &prog_type, 1309 &expected_attach_type); 1310 if (err < 0) 1311 goto err_close_obj; 1312 } 1313 1314 bpf_program__set_ifindex(pos, ifindex); 1315 bpf_program__set_type(pos, prog_type); 1316 bpf_program__set_expected_attach_type(pos, expected_attach_type); 1317 } 1318 1319 qsort(map_replace, old_map_fds, sizeof(*map_replace), 1320 map_replace_compar); 1321 1322 /* After the sort maps by name will be first on the list, because they 1323 * have idx == -1. Resolve them. 1324 */ 1325 j = 0; 1326 while (j < old_map_fds && map_replace[j].name) { 1327 i = 0; 1328 bpf_object__for_each_map(map, obj) { 1329 if (!strcmp(bpf_map__name(map), map_replace[j].name)) { 1330 map_replace[j].idx = i; 1331 break; 1332 } 1333 i++; 1334 } 1335 if (map_replace[j].idx == -1) { 1336 p_err("unable to find map '%s'", map_replace[j].name); 1337 goto err_close_obj; 1338 } 1339 j++; 1340 } 1341 /* Resort if any names were resolved */ 1342 if (j) 1343 qsort(map_replace, old_map_fds, sizeof(*map_replace), 1344 map_replace_compar); 1345 1346 /* Set ifindex and name reuse */ 1347 j = 0; 1348 idx = 0; 1349 bpf_object__for_each_map(map, obj) { 1350 if (!bpf_map__is_offload_neutral(map)) 1351 bpf_map__set_ifindex(map, ifindex); 1352 1353 if (j < old_map_fds && idx == map_replace[j].idx) { 1354 err = bpf_map__reuse_fd(map, map_replace[j++].fd); 1355 if (err) { 1356 p_err("unable to set up map reuse: %d", err); 1357 goto err_close_obj; 1358 } 1359 1360 /* Next reuse wants to apply to the same map */ 1361 if (j < old_map_fds && map_replace[j].idx == idx) { 1362 p_err("replacement for map idx %d specified more than once", 1363 idx); 1364 goto err_close_obj; 1365 } 1366 } 1367 1368 idx++; 1369 } 1370 if (j < old_map_fds) { 1371 p_err("map idx '%d' not used", map_replace[j].idx); 1372 goto err_close_obj; 1373 } 1374 1375 load_attr.obj = obj; 1376 if (verifier_logs) 1377 /* log_level1 + log_level2 + stats, but not stable UAPI */ 1378 load_attr.log_level = 1 + 2 + 4; 1379 1380 err = bpf_object__load_xattr(&load_attr); 1381 if (err) { 1382 p_err("failed to load object file"); 1383 goto err_close_obj; 1384 } 1385 1386 err = mount_bpffs_for_pin(pinfile); 1387 if (err) 1388 goto err_close_obj; 1389 1390 if (first_prog_only) { 1391 prog = bpf_program__next(NULL, obj); 1392 if (!prog) { 1393 p_err("object file doesn't contain any bpf program"); 1394 goto err_close_obj; 1395 } 1396 1397 err = bpf_obj_pin(bpf_program__fd(prog), pinfile); 1398 if (err) { 1399 p_err("failed to pin program %s", 1400 bpf_program__title(prog, false)); 1401 goto err_close_obj; 1402 } 1403 } else { 1404 err = bpf_object__pin_programs(obj, pinfile); 1405 if (err) { 1406 p_err("failed to pin all programs"); 1407 goto err_close_obj; 1408 } 1409 } 1410 1411 if (pinmaps) { 1412 err = bpf_object__pin_maps(obj, pinmaps); 1413 if (err) { 1414 p_err("failed to pin all maps"); 1415 goto err_unpin; 1416 } 1417 } 1418 1419 if (json_output) 1420 jsonw_null(json_wtr); 1421 1422 bpf_object__close(obj); 1423 for (i = 0; i < old_map_fds; i++) 1424 close(map_replace[i].fd); 1425 free(map_replace); 1426 1427 return 0; 1428 1429 err_unpin: 1430 if (first_prog_only) 1431 unlink(pinfile); 1432 else 1433 bpf_object__unpin_programs(obj, pinfile); 1434 err_close_obj: 1435 bpf_object__close(obj); 1436 err_free_reuse_maps: 1437 for (i = 0; i < old_map_fds; i++) 1438 close(map_replace[i].fd); 1439 free(map_replace); 1440 return -1; 1441 } 1442 1443 static int do_load(int argc, char **argv) 1444 { 1445 return load_with_options(argc, argv, true); 1446 } 1447 1448 static int do_loadall(int argc, char **argv) 1449 { 1450 return load_with_options(argc, argv, false); 1451 } 1452 1453 #ifdef BPFTOOL_WITHOUT_SKELETONS 1454 1455 static int do_profile(int argc, char **argv) 1456 { 1457 p_err("bpftool prog profile command is not supported. Please build bpftool with clang >= 10.0.0"); 1458 return 0; 1459 } 1460 1461 #else /* BPFTOOL_WITHOUT_SKELETONS */ 1462 1463 #include "profiler.skel.h" 1464 1465 struct profile_metric { 1466 const char *name; 1467 struct bpf_perf_event_value val; 1468 struct perf_event_attr attr; 1469 bool selected; 1470 1471 /* calculate ratios like instructions per cycle */ 1472 const int ratio_metric; /* 0 for N/A, 1 for index 0 (cycles) */ 1473 const char *ratio_desc; 1474 const float ratio_mul; 1475 } metrics[] = { 1476 { 1477 .name = "cycles", 1478 .attr = { 1479 .type = PERF_TYPE_HARDWARE, 1480 .config = PERF_COUNT_HW_CPU_CYCLES, 1481 .exclude_user = 1, 1482 }, 1483 }, 1484 { 1485 .name = "instructions", 1486 .attr = { 1487 .type = PERF_TYPE_HARDWARE, 1488 .config = PERF_COUNT_HW_INSTRUCTIONS, 1489 .exclude_user = 1, 1490 }, 1491 .ratio_metric = 1, 1492 .ratio_desc = "insns per cycle", 1493 .ratio_mul = 1.0, 1494 }, 1495 { 1496 .name = "l1d_loads", 1497 .attr = { 1498 .type = PERF_TYPE_HW_CACHE, 1499 .config = 1500 PERF_COUNT_HW_CACHE_L1D | 1501 (PERF_COUNT_HW_CACHE_OP_READ << 8) | 1502 (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16), 1503 .exclude_user = 1, 1504 }, 1505 }, 1506 { 1507 .name = "llc_misses", 1508 .attr = { 1509 .type = PERF_TYPE_HW_CACHE, 1510 .config = 1511 PERF_COUNT_HW_CACHE_LL | 1512 (PERF_COUNT_HW_CACHE_OP_READ << 8) | 1513 (PERF_COUNT_HW_CACHE_RESULT_MISS << 16), 1514 .exclude_user = 1 1515 }, 1516 .ratio_metric = 2, 1517 .ratio_desc = "LLC misses per million insns", 1518 .ratio_mul = 1e6, 1519 }, 1520 }; 1521 1522 static __u64 profile_total_count; 1523 1524 #define MAX_NUM_PROFILE_METRICS 4 1525 1526 static int profile_parse_metrics(int argc, char **argv) 1527 { 1528 unsigned int metric_cnt; 1529 int selected_cnt = 0; 1530 unsigned int i; 1531 1532 metric_cnt = sizeof(metrics) / sizeof(struct profile_metric); 1533 1534 while (argc > 0) { 1535 for (i = 0; i < metric_cnt; i++) { 1536 if (is_prefix(argv[0], metrics[i].name)) { 1537 if (!metrics[i].selected) 1538 selected_cnt++; 1539 metrics[i].selected = true; 1540 break; 1541 } 1542 } 1543 if (i == metric_cnt) { 1544 p_err("unknown metric %s", argv[0]); 1545 return -1; 1546 } 1547 NEXT_ARG(); 1548 } 1549 if (selected_cnt > MAX_NUM_PROFILE_METRICS) { 1550 p_err("too many (%d) metrics, please specify no more than %d metrics at at time", 1551 selected_cnt, MAX_NUM_PROFILE_METRICS); 1552 return -1; 1553 } 1554 return selected_cnt; 1555 } 1556 1557 static void profile_read_values(struct profiler_bpf *obj) 1558 { 1559 __u32 m, cpu, num_cpu = obj->rodata->num_cpu; 1560 int reading_map_fd, count_map_fd; 1561 __u64 counts[num_cpu]; 1562 __u32 key = 0; 1563 int err; 1564 1565 reading_map_fd = bpf_map__fd(obj->maps.accum_readings); 1566 count_map_fd = bpf_map__fd(obj->maps.counts); 1567 if (reading_map_fd < 0 || count_map_fd < 0) { 1568 p_err("failed to get fd for map"); 1569 return; 1570 } 1571 1572 err = bpf_map_lookup_elem(count_map_fd, &key, counts); 1573 if (err) { 1574 p_err("failed to read count_map: %s", strerror(errno)); 1575 return; 1576 } 1577 1578 profile_total_count = 0; 1579 for (cpu = 0; cpu < num_cpu; cpu++) 1580 profile_total_count += counts[cpu]; 1581 1582 for (m = 0; m < ARRAY_SIZE(metrics); m++) { 1583 struct bpf_perf_event_value values[num_cpu]; 1584 1585 if (!metrics[m].selected) 1586 continue; 1587 1588 err = bpf_map_lookup_elem(reading_map_fd, &key, values); 1589 if (err) { 1590 p_err("failed to read reading_map: %s", 1591 strerror(errno)); 1592 return; 1593 } 1594 for (cpu = 0; cpu < num_cpu; cpu++) { 1595 metrics[m].val.counter += values[cpu].counter; 1596 metrics[m].val.enabled += values[cpu].enabled; 1597 metrics[m].val.running += values[cpu].running; 1598 } 1599 key++; 1600 } 1601 } 1602 1603 static void profile_print_readings_json(void) 1604 { 1605 __u32 m; 1606 1607 jsonw_start_array(json_wtr); 1608 for (m = 0; m < ARRAY_SIZE(metrics); m++) { 1609 if (!metrics[m].selected) 1610 continue; 1611 jsonw_start_object(json_wtr); 1612 jsonw_string_field(json_wtr, "metric", metrics[m].name); 1613 jsonw_lluint_field(json_wtr, "run_cnt", profile_total_count); 1614 jsonw_lluint_field(json_wtr, "value", metrics[m].val.counter); 1615 jsonw_lluint_field(json_wtr, "enabled", metrics[m].val.enabled); 1616 jsonw_lluint_field(json_wtr, "running", metrics[m].val.running); 1617 1618 jsonw_end_object(json_wtr); 1619 } 1620 jsonw_end_array(json_wtr); 1621 } 1622 1623 static void profile_print_readings_plain(void) 1624 { 1625 __u32 m; 1626 1627 printf("\n%18llu %-20s\n", profile_total_count, "run_cnt"); 1628 for (m = 0; m < ARRAY_SIZE(metrics); m++) { 1629 struct bpf_perf_event_value *val = &metrics[m].val; 1630 int r; 1631 1632 if (!metrics[m].selected) 1633 continue; 1634 printf("%18llu %-20s", val->counter, metrics[m].name); 1635 1636 r = metrics[m].ratio_metric - 1; 1637 if (r >= 0 && metrics[r].selected && 1638 metrics[r].val.counter > 0) { 1639 printf("# %8.2f %-30s", 1640 val->counter * metrics[m].ratio_mul / 1641 metrics[r].val.counter, 1642 metrics[m].ratio_desc); 1643 } else { 1644 printf("%-41s", ""); 1645 } 1646 1647 if (val->enabled > val->running) 1648 printf("(%4.2f%%)", 1649 val->running * 100.0 / val->enabled); 1650 printf("\n"); 1651 } 1652 } 1653 1654 static void profile_print_readings(void) 1655 { 1656 if (json_output) 1657 profile_print_readings_json(); 1658 else 1659 profile_print_readings_plain(); 1660 } 1661 1662 static char *profile_target_name(int tgt_fd) 1663 { 1664 struct bpf_prog_info_linear *info_linear; 1665 struct bpf_func_info *func_info; 1666 const struct btf_type *t; 1667 char *name = NULL; 1668 struct btf *btf; 1669 1670 info_linear = bpf_program__get_prog_info_linear( 1671 tgt_fd, 1UL << BPF_PROG_INFO_FUNC_INFO); 1672 if (IS_ERR_OR_NULL(info_linear)) { 1673 p_err("failed to get info_linear for prog FD %d", tgt_fd); 1674 return NULL; 1675 } 1676 1677 if (info_linear->info.btf_id == 0 || 1678 btf__get_from_id(info_linear->info.btf_id, &btf)) { 1679 p_err("prog FD %d doesn't have valid btf", tgt_fd); 1680 goto out; 1681 } 1682 1683 func_info = (struct bpf_func_info *)(info_linear->info.func_info); 1684 t = btf__type_by_id(btf, func_info[0].type_id); 1685 if (!t) { 1686 p_err("btf %d doesn't have type %d", 1687 info_linear->info.btf_id, func_info[0].type_id); 1688 goto out; 1689 } 1690 name = strdup(btf__name_by_offset(btf, t->name_off)); 1691 out: 1692 free(info_linear); 1693 return name; 1694 } 1695 1696 static struct profiler_bpf *profile_obj; 1697 static int profile_tgt_fd = -1; 1698 static char *profile_tgt_name; 1699 static int *profile_perf_events; 1700 static int profile_perf_event_cnt; 1701 1702 static void profile_close_perf_events(struct profiler_bpf *obj) 1703 { 1704 int i; 1705 1706 for (i = profile_perf_event_cnt - 1; i >= 0; i--) 1707 close(profile_perf_events[i]); 1708 1709 free(profile_perf_events); 1710 profile_perf_event_cnt = 0; 1711 } 1712 1713 static int profile_open_perf_events(struct profiler_bpf *obj) 1714 { 1715 unsigned int cpu, m; 1716 int map_fd, pmu_fd; 1717 1718 profile_perf_events = calloc( 1719 sizeof(int), obj->rodata->num_cpu * obj->rodata->num_metric); 1720 if (!profile_perf_events) { 1721 p_err("failed to allocate memory for perf_event array: %s", 1722 strerror(errno)); 1723 return -1; 1724 } 1725 map_fd = bpf_map__fd(obj->maps.events); 1726 if (map_fd < 0) { 1727 p_err("failed to get fd for events map"); 1728 return -1; 1729 } 1730 1731 for (m = 0; m < ARRAY_SIZE(metrics); m++) { 1732 if (!metrics[m].selected) 1733 continue; 1734 for (cpu = 0; cpu < obj->rodata->num_cpu; cpu++) { 1735 pmu_fd = syscall(__NR_perf_event_open, &metrics[m].attr, 1736 -1/*pid*/, cpu, -1/*group_fd*/, 0); 1737 if (pmu_fd < 0 || 1738 bpf_map_update_elem(map_fd, &profile_perf_event_cnt, 1739 &pmu_fd, BPF_ANY) || 1740 ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0)) { 1741 p_err("failed to create event %s on cpu %d", 1742 metrics[m].name, cpu); 1743 return -1; 1744 } 1745 profile_perf_events[profile_perf_event_cnt++] = pmu_fd; 1746 } 1747 } 1748 return 0; 1749 } 1750 1751 static void profile_print_and_cleanup(void) 1752 { 1753 profile_close_perf_events(profile_obj); 1754 profile_read_values(profile_obj); 1755 profile_print_readings(); 1756 profiler_bpf__destroy(profile_obj); 1757 1758 close(profile_tgt_fd); 1759 free(profile_tgt_name); 1760 } 1761 1762 static void int_exit(int signo) 1763 { 1764 profile_print_and_cleanup(); 1765 exit(0); 1766 } 1767 1768 static int do_profile(int argc, char **argv) 1769 { 1770 int num_metric, num_cpu, err = -1; 1771 struct bpf_program *prog; 1772 unsigned long duration; 1773 char *endptr; 1774 1775 /* we at least need two args for the prog and one metric */ 1776 if (!REQ_ARGS(3)) 1777 return -EINVAL; 1778 1779 /* parse target fd */ 1780 profile_tgt_fd = prog_parse_fd(&argc, &argv); 1781 if (profile_tgt_fd < 0) { 1782 p_err("failed to parse fd"); 1783 return -1; 1784 } 1785 1786 /* parse profiling optional duration */ 1787 if (argc > 2 && is_prefix(argv[0], "duration")) { 1788 NEXT_ARG(); 1789 duration = strtoul(*argv, &endptr, 0); 1790 if (*endptr) 1791 usage(); 1792 NEXT_ARG(); 1793 } else { 1794 duration = UINT_MAX; 1795 } 1796 1797 num_metric = profile_parse_metrics(argc, argv); 1798 if (num_metric <= 0) 1799 goto out; 1800 1801 num_cpu = libbpf_num_possible_cpus(); 1802 if (num_cpu <= 0) { 1803 p_err("failed to identify number of CPUs"); 1804 goto out; 1805 } 1806 1807 profile_obj = profiler_bpf__open(); 1808 if (!profile_obj) { 1809 p_err("failed to open and/or load BPF object"); 1810 goto out; 1811 } 1812 1813 profile_obj->rodata->num_cpu = num_cpu; 1814 profile_obj->rodata->num_metric = num_metric; 1815 1816 /* adjust map sizes */ 1817 bpf_map__resize(profile_obj->maps.events, num_metric * num_cpu); 1818 bpf_map__resize(profile_obj->maps.fentry_readings, num_metric); 1819 bpf_map__resize(profile_obj->maps.accum_readings, num_metric); 1820 bpf_map__resize(profile_obj->maps.counts, 1); 1821 1822 /* change target name */ 1823 profile_tgt_name = profile_target_name(profile_tgt_fd); 1824 if (!profile_tgt_name) 1825 goto out; 1826 1827 bpf_object__for_each_program(prog, profile_obj->obj) { 1828 err = bpf_program__set_attach_target(prog, profile_tgt_fd, 1829 profile_tgt_name); 1830 if (err) { 1831 p_err("failed to set attach target\n"); 1832 goto out; 1833 } 1834 } 1835 1836 set_max_rlimit(); 1837 err = profiler_bpf__load(profile_obj); 1838 if (err) { 1839 p_err("failed to load profile_obj"); 1840 goto out; 1841 } 1842 1843 err = profile_open_perf_events(profile_obj); 1844 if (err) 1845 goto out; 1846 1847 err = profiler_bpf__attach(profile_obj); 1848 if (err) { 1849 p_err("failed to attach profile_obj"); 1850 goto out; 1851 } 1852 signal(SIGINT, int_exit); 1853 1854 sleep(duration); 1855 profile_print_and_cleanup(); 1856 return 0; 1857 1858 out: 1859 profile_close_perf_events(profile_obj); 1860 if (profile_obj) 1861 profiler_bpf__destroy(profile_obj); 1862 close(profile_tgt_fd); 1863 free(profile_tgt_name); 1864 return err; 1865 } 1866 1867 #endif /* BPFTOOL_WITHOUT_SKELETONS */ 1868 1869 static int do_help(int argc, char **argv) 1870 { 1871 if (json_output) { 1872 jsonw_null(json_wtr); 1873 return 0; 1874 } 1875 1876 fprintf(stderr, 1877 "Usage: %1$s %2$s { show | list } [PROG]\n" 1878 " %1$s %2$s dump xlated PROG [{ file FILE | opcodes | visual | linum }]\n" 1879 " %1$s %2$s dump jited PROG [{ file FILE | opcodes | linum }]\n" 1880 " %1$s %2$s pin PROG FILE\n" 1881 " %1$s %2$s { load | loadall } OBJ PATH \\\n" 1882 " [type TYPE] [dev NAME] \\\n" 1883 " [map { idx IDX | name NAME } MAP]\\\n" 1884 " [pinmaps MAP_DIR]\n" 1885 " %1$s %2$s attach PROG ATTACH_TYPE [MAP]\n" 1886 " %1$s %2$s detach PROG ATTACH_TYPE [MAP]\n" 1887 " %1$s %2$s run PROG \\\n" 1888 " data_in FILE \\\n" 1889 " [data_out FILE [data_size_out L]] \\\n" 1890 " [ctx_in FILE [ctx_out FILE [ctx_size_out M]]] \\\n" 1891 " [repeat N]\n" 1892 " %1$s %2$s profile PROG [duration DURATION] METRICs\n" 1893 " %1$s %2$s tracelog\n" 1894 " %1$s %2$s help\n" 1895 "\n" 1896 " " HELP_SPEC_MAP "\n" 1897 " " HELP_SPEC_PROGRAM "\n" 1898 " TYPE := { socket | kprobe | kretprobe | classifier | action |\n" 1899 " tracepoint | raw_tracepoint | xdp | perf_event | cgroup/skb |\n" 1900 " cgroup/sock | cgroup/dev | lwt_in | lwt_out | lwt_xmit |\n" 1901 " lwt_seg6local | sockops | sk_skb | sk_msg | lirc_mode2 |\n" 1902 " sk_reuseport | flow_dissector | cgroup/sysctl |\n" 1903 " cgroup/bind4 | cgroup/bind6 | cgroup/post_bind4 |\n" 1904 " cgroup/post_bind6 | cgroup/connect4 | cgroup/connect6 |\n" 1905 " cgroup/getpeername4 | cgroup/getpeername6 |\n" 1906 " cgroup/getsockname4 | cgroup/getsockname6 | cgroup/sendmsg4 |\n" 1907 " cgroup/sendmsg6 | cgroup/recvmsg4 | cgroup/recvmsg6 |\n" 1908 " cgroup/getsockopt | cgroup/setsockopt |\n" 1909 " struct_ops | fentry | fexit | freplace | sk_lookup }\n" 1910 " ATTACH_TYPE := { msg_verdict | stream_verdict | stream_parser |\n" 1911 " flow_dissector }\n" 1912 " METRIC := { cycles | instructions | l1d_loads | llc_misses }\n" 1913 " " HELP_SPEC_OPTIONS "\n" 1914 "", 1915 bin_name, argv[-2]); 1916 1917 return 0; 1918 } 1919 1920 static const struct cmd cmds[] = { 1921 { "show", do_show }, 1922 { "list", do_show }, 1923 { "help", do_help }, 1924 { "dump", do_dump }, 1925 { "pin", do_pin }, 1926 { "load", do_load }, 1927 { "loadall", do_loadall }, 1928 { "attach", do_attach }, 1929 { "detach", do_detach }, 1930 { "tracelog", do_tracelog }, 1931 { "run", do_run }, 1932 { "profile", do_profile }, 1933 { 0 } 1934 }; 1935 1936 int do_prog(int argc, char **argv) 1937 { 1938 return cmd_select(cmds, argc, argv, do_help); 1939 } 1940