1 // SPDX-License-Identifier: GPL-2.0 2 #include <errno.h> 3 #include <stdlib.h> 4 #include <bpf/bpf.h> 5 #include <bpf/btf.h> 6 #include <bpf/libbpf.h> 7 #include <linux/btf.h> 8 #include <linux/err.h> 9 #include <linux/string.h> 10 #include <internal/lib.h> 11 #include <symbol/kallsyms.h> 12 #include "bpf-event.h" 13 #include "bpf-utils.h" 14 #include "debug.h" 15 #include "dso.h" 16 #include "symbol.h" 17 #include "machine.h" 18 #include "env.h" 19 #include "session.h" 20 #include "map.h" 21 #include "evlist.h" 22 #include "record.h" 23 #include "util/synthetic-events.h" 24 25 #ifndef HAVE_LIBBPF_BTF__LOAD_FROM_KERNEL_BY_ID 26 struct btf *btf__load_from_kernel_by_id(__u32 id) 27 { 28 struct btf *btf; 29 #pragma GCC diagnostic push 30 #pragma GCC diagnostic ignored "-Wdeprecated-declarations" 31 int err = btf__get_from_id(id, &btf); 32 #pragma GCC diagnostic pop 33 34 return err ? ERR_PTR(err) : btf; 35 } 36 #endif 37 38 int __weak bpf_prog_load(enum bpf_prog_type prog_type, 39 const char *prog_name __maybe_unused, 40 const char *license, 41 const struct bpf_insn *insns, size_t insn_cnt, 42 const struct bpf_prog_load_opts *opts) 43 { 44 #pragma GCC diagnostic push 45 #pragma GCC diagnostic ignored "-Wdeprecated-declarations" 46 return bpf_load_program(prog_type, insns, insn_cnt, license, 47 opts->kern_version, opts->log_buf, opts->log_size); 48 #pragma GCC diagnostic pop 49 } 50 51 struct bpf_program * __weak 52 bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev) 53 { 54 #pragma GCC diagnostic push 55 #pragma GCC diagnostic ignored "-Wdeprecated-declarations" 56 return bpf_program__next(prev, obj); 57 #pragma GCC diagnostic pop 58 } 59 60 struct bpf_map * __weak 61 bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev) 62 { 63 #pragma GCC diagnostic push 64 #pragma GCC diagnostic ignored "-Wdeprecated-declarations" 65 return bpf_map__next(prev, obj); 66 #pragma GCC diagnostic pop 67 } 68 69 const void * __weak 70 btf__raw_data(const struct btf *btf_ro, __u32 *size) 71 { 72 #pragma GCC diagnostic push 73 #pragma GCC diagnostic ignored "-Wdeprecated-declarations" 74 return btf__get_raw_data(btf_ro, size); 75 #pragma GCC diagnostic pop 76 } 77 78 static int snprintf_hex(char *buf, size_t size, unsigned char *data, size_t len) 79 { 80 int ret = 0; 81 size_t i; 82 83 for (i = 0; i < len; i++) 84 ret += snprintf(buf + ret, size - ret, "%02x", data[i]); 85 return ret; 86 } 87 88 static int machine__process_bpf_event_load(struct machine *machine, 89 union perf_event *event, 90 struct perf_sample *sample __maybe_unused) 91 { 92 struct bpf_prog_info_node *info_node; 93 struct perf_env *env = machine->env; 94 struct perf_bpil *info_linear; 95 int id = event->bpf.id; 96 unsigned int i; 97 98 /* perf-record, no need to handle bpf-event */ 99 if (env == NULL) 100 return 0; 101 102 info_node = perf_env__find_bpf_prog_info(env, id); 103 if (!info_node) 104 return 0; 105 info_linear = info_node->info_linear; 106 107 for (i = 0; i < info_linear->info.nr_jited_ksyms; i++) { 108 u64 *addrs = (u64 *)(uintptr_t)(info_linear->info.jited_ksyms); 109 u64 addr = addrs[i]; 110 struct map *map = maps__find(machine__kernel_maps(machine), addr); 111 112 if (map) { 113 map->dso->binary_type = DSO_BINARY_TYPE__BPF_PROG_INFO; 114 map->dso->bpf_prog.id = id; 115 map->dso->bpf_prog.sub_id = i; 116 map->dso->bpf_prog.env = env; 117 } 118 } 119 return 0; 120 } 121 122 int machine__process_bpf(struct machine *machine, union perf_event *event, 123 struct perf_sample *sample) 124 { 125 if (dump_trace) 126 perf_event__fprintf_bpf(event, stdout); 127 128 switch (event->bpf.type) { 129 case PERF_BPF_EVENT_PROG_LOAD: 130 return machine__process_bpf_event_load(machine, event, sample); 131 132 case PERF_BPF_EVENT_PROG_UNLOAD: 133 /* 134 * Do not free bpf_prog_info and btf of the program here, 135 * as annotation still need them. They will be freed at 136 * the end of the session. 137 */ 138 break; 139 default: 140 pr_debug("unexpected bpf event type of %d\n", event->bpf.type); 141 break; 142 } 143 return 0; 144 } 145 146 static int perf_env__fetch_btf(struct perf_env *env, 147 u32 btf_id, 148 struct btf *btf) 149 { 150 struct btf_node *node; 151 u32 data_size; 152 const void *data; 153 154 data = btf__raw_data(btf, &data_size); 155 156 node = malloc(data_size + sizeof(struct btf_node)); 157 if (!node) 158 return -1; 159 160 node->id = btf_id; 161 node->data_size = data_size; 162 memcpy(node->data, data, data_size); 163 164 if (!perf_env__insert_btf(env, node)) { 165 /* Insertion failed because of a duplicate. */ 166 free(node); 167 return -1; 168 } 169 return 0; 170 } 171 172 static int synthesize_bpf_prog_name(char *buf, int size, 173 struct bpf_prog_info *info, 174 struct btf *btf, 175 u32 sub_id) 176 { 177 u8 (*prog_tags)[BPF_TAG_SIZE] = (void *)(uintptr_t)(info->prog_tags); 178 void *func_infos = (void *)(uintptr_t)(info->func_info); 179 u32 sub_prog_cnt = info->nr_jited_ksyms; 180 const struct bpf_func_info *finfo; 181 const char *short_name = NULL; 182 const struct btf_type *t; 183 int name_len; 184 185 name_len = snprintf(buf, size, "bpf_prog_"); 186 name_len += snprintf_hex(buf + name_len, size - name_len, 187 prog_tags[sub_id], BPF_TAG_SIZE); 188 if (btf) { 189 finfo = func_infos + sub_id * info->func_info_rec_size; 190 t = btf__type_by_id(btf, finfo->type_id); 191 short_name = btf__name_by_offset(btf, t->name_off); 192 } else if (sub_id == 0 && sub_prog_cnt == 1) { 193 /* no subprog */ 194 if (info->name[0]) 195 short_name = info->name; 196 } else 197 short_name = "F"; 198 if (short_name) 199 name_len += snprintf(buf + name_len, size - name_len, 200 "_%s", short_name); 201 return name_len; 202 } 203 204 /* 205 * Synthesize PERF_RECORD_KSYMBOL and PERF_RECORD_BPF_EVENT for one bpf 206 * program. One PERF_RECORD_BPF_EVENT is generated for the program. And 207 * one PERF_RECORD_KSYMBOL is generated for each sub program. 208 * 209 * Returns: 210 * 0 for success; 211 * -1 for failures; 212 * -2 for lack of kernel support. 213 */ 214 static int perf_event__synthesize_one_bpf_prog(struct perf_session *session, 215 perf_event__handler_t process, 216 struct machine *machine, 217 int fd, 218 union perf_event *event, 219 struct record_opts *opts) 220 { 221 struct perf_record_ksymbol *ksymbol_event = &event->ksymbol; 222 struct perf_record_bpf_event *bpf_event = &event->bpf; 223 struct perf_tool *tool = session->tool; 224 struct bpf_prog_info_node *info_node; 225 struct perf_bpil *info_linear; 226 struct bpf_prog_info *info; 227 struct btf *btf = NULL; 228 struct perf_env *env; 229 u32 sub_prog_cnt, i; 230 int err = 0; 231 u64 arrays; 232 233 /* 234 * for perf-record and perf-report use header.env; 235 * otherwise, use global perf_env. 236 */ 237 env = session->data ? &session->header.env : &perf_env; 238 239 arrays = 1UL << PERF_BPIL_JITED_KSYMS; 240 arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS; 241 arrays |= 1UL << PERF_BPIL_FUNC_INFO; 242 arrays |= 1UL << PERF_BPIL_PROG_TAGS; 243 arrays |= 1UL << PERF_BPIL_JITED_INSNS; 244 arrays |= 1UL << PERF_BPIL_LINE_INFO; 245 arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO; 246 247 info_linear = get_bpf_prog_info_linear(fd, arrays); 248 if (IS_ERR_OR_NULL(info_linear)) { 249 info_linear = NULL; 250 pr_debug("%s: failed to get BPF program info. aborting\n", __func__); 251 return -1; 252 } 253 254 if (info_linear->info_len < offsetof(struct bpf_prog_info, prog_tags)) { 255 free(info_linear); 256 pr_debug("%s: the kernel is too old, aborting\n", __func__); 257 return -2; 258 } 259 260 info = &info_linear->info; 261 if (!info->jited_ksyms) { 262 free(info_linear); 263 return -1; 264 } 265 266 /* number of ksyms, func_lengths, and tags should match */ 267 sub_prog_cnt = info->nr_jited_ksyms; 268 if (sub_prog_cnt != info->nr_prog_tags || 269 sub_prog_cnt != info->nr_jited_func_lens) { 270 free(info_linear); 271 return -1; 272 } 273 274 /* check BTF func info support */ 275 if (info->btf_id && info->nr_func_info && info->func_info_rec_size) { 276 /* btf func info number should be same as sub_prog_cnt */ 277 if (sub_prog_cnt != info->nr_func_info) { 278 pr_debug("%s: mismatch in BPF sub program count and BTF function info count, aborting\n", __func__); 279 free(info_linear); 280 return -1; 281 } 282 btf = btf__load_from_kernel_by_id(info->btf_id); 283 if (libbpf_get_error(btf)) { 284 pr_debug("%s: failed to get BTF of id %u, aborting\n", __func__, info->btf_id); 285 err = -1; 286 goto out; 287 } 288 perf_env__fetch_btf(env, info->btf_id, btf); 289 } 290 291 /* Synthesize PERF_RECORD_KSYMBOL */ 292 for (i = 0; i < sub_prog_cnt; i++) { 293 __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); 294 __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); 295 int name_len; 296 297 *ksymbol_event = (struct perf_record_ksymbol) { 298 .header = { 299 .type = PERF_RECORD_KSYMBOL, 300 .size = offsetof(struct perf_record_ksymbol, name), 301 }, 302 .addr = prog_addrs[i], 303 .len = prog_lens[i], 304 .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF, 305 .flags = 0, 306 }; 307 308 name_len = synthesize_bpf_prog_name(ksymbol_event->name, 309 KSYM_NAME_LEN, info, btf, i); 310 ksymbol_event->header.size += PERF_ALIGN(name_len + 1, 311 sizeof(u64)); 312 313 memset((void *)event + event->header.size, 0, machine->id_hdr_size); 314 event->header.size += machine->id_hdr_size; 315 err = perf_tool__process_synth_event(tool, event, 316 machine, process); 317 } 318 319 if (!opts->no_bpf_event) { 320 /* Synthesize PERF_RECORD_BPF_EVENT */ 321 *bpf_event = (struct perf_record_bpf_event) { 322 .header = { 323 .type = PERF_RECORD_BPF_EVENT, 324 .size = sizeof(struct perf_record_bpf_event), 325 }, 326 .type = PERF_BPF_EVENT_PROG_LOAD, 327 .flags = 0, 328 .id = info->id, 329 }; 330 memcpy(bpf_event->tag, info->tag, BPF_TAG_SIZE); 331 memset((void *)event + event->header.size, 0, machine->id_hdr_size); 332 event->header.size += machine->id_hdr_size; 333 334 /* save bpf_prog_info to env */ 335 info_node = malloc(sizeof(struct bpf_prog_info_node)); 336 if (!info_node) { 337 err = -1; 338 goto out; 339 } 340 341 info_node->info_linear = info_linear; 342 perf_env__insert_bpf_prog_info(env, info_node); 343 info_linear = NULL; 344 345 /* 346 * process after saving bpf_prog_info to env, so that 347 * required information is ready for look up 348 */ 349 err = perf_tool__process_synth_event(tool, event, 350 machine, process); 351 } 352 353 out: 354 free(info_linear); 355 btf__free(btf); 356 return err ? -1 : 0; 357 } 358 359 struct kallsyms_parse { 360 union perf_event *event; 361 perf_event__handler_t process; 362 struct machine *machine; 363 struct perf_tool *tool; 364 }; 365 366 static int 367 process_bpf_image(char *name, u64 addr, struct kallsyms_parse *data) 368 { 369 struct machine *machine = data->machine; 370 union perf_event *event = data->event; 371 struct perf_record_ksymbol *ksymbol; 372 int len; 373 374 ksymbol = &event->ksymbol; 375 376 *ksymbol = (struct perf_record_ksymbol) { 377 .header = { 378 .type = PERF_RECORD_KSYMBOL, 379 .size = offsetof(struct perf_record_ksymbol, name), 380 }, 381 .addr = addr, 382 .len = page_size, 383 .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF, 384 .flags = 0, 385 }; 386 387 len = scnprintf(ksymbol->name, KSYM_NAME_LEN, "%s", name); 388 ksymbol->header.size += PERF_ALIGN(len + 1, sizeof(u64)); 389 memset((void *) event + event->header.size, 0, machine->id_hdr_size); 390 event->header.size += machine->id_hdr_size; 391 392 return perf_tool__process_synth_event(data->tool, event, machine, 393 data->process); 394 } 395 396 static int 397 kallsyms_process_symbol(void *data, const char *_name, 398 char type __maybe_unused, u64 start) 399 { 400 char disp[KSYM_NAME_LEN]; 401 char *module, *name; 402 unsigned long id; 403 int err = 0; 404 405 module = strchr(_name, '\t'); 406 if (!module) 407 return 0; 408 409 /* We are going after [bpf] module ... */ 410 if (strcmp(module + 1, "[bpf]")) 411 return 0; 412 413 name = memdup(_name, (module - _name) + 1); 414 if (!name) 415 return -ENOMEM; 416 417 name[module - _name] = 0; 418 419 /* .. and only for trampolines and dispatchers */ 420 if ((sscanf(name, "bpf_trampoline_%lu", &id) == 1) || 421 (sscanf(name, "bpf_dispatcher_%s", disp) == 1)) 422 err = process_bpf_image(name, start, data); 423 424 free(name); 425 return err; 426 } 427 428 int perf_event__synthesize_bpf_events(struct perf_session *session, 429 perf_event__handler_t process, 430 struct machine *machine, 431 struct record_opts *opts) 432 { 433 const char *kallsyms_filename = "/proc/kallsyms"; 434 struct kallsyms_parse arg; 435 union perf_event *event; 436 __u32 id = 0; 437 int err; 438 int fd; 439 440 event = malloc(sizeof(event->bpf) + KSYM_NAME_LEN + machine->id_hdr_size); 441 if (!event) 442 return -1; 443 444 /* Synthesize all the bpf programs in system. */ 445 while (true) { 446 err = bpf_prog_get_next_id(id, &id); 447 if (err) { 448 if (errno == ENOENT) { 449 err = 0; 450 break; 451 } 452 pr_debug("%s: can't get next program: %s%s\n", 453 __func__, strerror(errno), 454 errno == EINVAL ? " -- kernel too old?" : ""); 455 /* don't report error on old kernel or EPERM */ 456 err = (errno == EINVAL || errno == EPERM) ? 0 : -1; 457 break; 458 } 459 fd = bpf_prog_get_fd_by_id(id); 460 if (fd < 0) { 461 pr_debug("%s: failed to get fd for prog_id %u\n", 462 __func__, id); 463 continue; 464 } 465 466 err = perf_event__synthesize_one_bpf_prog(session, process, 467 machine, fd, 468 event, opts); 469 close(fd); 470 if (err) { 471 /* do not return error for old kernel */ 472 if (err == -2) 473 err = 0; 474 break; 475 } 476 } 477 478 /* Synthesize all the bpf images - trampolines/dispatchers. */ 479 if (symbol_conf.kallsyms_name != NULL) 480 kallsyms_filename = symbol_conf.kallsyms_name; 481 482 arg = (struct kallsyms_parse) { 483 .event = event, 484 .process = process, 485 .machine = machine, 486 .tool = session->tool, 487 }; 488 489 if (kallsyms__parse(kallsyms_filename, &arg, kallsyms_process_symbol)) { 490 pr_err("%s: failed to synthesize bpf images: %s\n", 491 __func__, strerror(errno)); 492 } 493 494 free(event); 495 return err; 496 } 497 498 static void perf_env__add_bpf_info(struct perf_env *env, u32 id) 499 { 500 struct bpf_prog_info_node *info_node; 501 struct perf_bpil *info_linear; 502 struct btf *btf = NULL; 503 u64 arrays; 504 u32 btf_id; 505 int fd; 506 507 fd = bpf_prog_get_fd_by_id(id); 508 if (fd < 0) 509 return; 510 511 arrays = 1UL << PERF_BPIL_JITED_KSYMS; 512 arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS; 513 arrays |= 1UL << PERF_BPIL_FUNC_INFO; 514 arrays |= 1UL << PERF_BPIL_PROG_TAGS; 515 arrays |= 1UL << PERF_BPIL_JITED_INSNS; 516 arrays |= 1UL << PERF_BPIL_LINE_INFO; 517 arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO; 518 519 info_linear = get_bpf_prog_info_linear(fd, arrays); 520 if (IS_ERR_OR_NULL(info_linear)) { 521 pr_debug("%s: failed to get BPF program info. aborting\n", __func__); 522 goto out; 523 } 524 525 btf_id = info_linear->info.btf_id; 526 527 info_node = malloc(sizeof(struct bpf_prog_info_node)); 528 if (info_node) { 529 info_node->info_linear = info_linear; 530 perf_env__insert_bpf_prog_info(env, info_node); 531 } else 532 free(info_linear); 533 534 if (btf_id == 0) 535 goto out; 536 537 btf = btf__load_from_kernel_by_id(btf_id); 538 if (libbpf_get_error(btf)) { 539 pr_debug("%s: failed to get BTF of id %u, aborting\n", 540 __func__, btf_id); 541 goto out; 542 } 543 perf_env__fetch_btf(env, btf_id, btf); 544 545 out: 546 btf__free(btf); 547 close(fd); 548 } 549 550 static int bpf_event__sb_cb(union perf_event *event, void *data) 551 { 552 struct perf_env *env = data; 553 554 if (event->header.type != PERF_RECORD_BPF_EVENT) 555 return -1; 556 557 switch (event->bpf.type) { 558 case PERF_BPF_EVENT_PROG_LOAD: 559 perf_env__add_bpf_info(env, event->bpf.id); 560 561 case PERF_BPF_EVENT_PROG_UNLOAD: 562 /* 563 * Do not free bpf_prog_info and btf of the program here, 564 * as annotation still need them. They will be freed at 565 * the end of the session. 566 */ 567 break; 568 default: 569 pr_debug("unexpected bpf event type of %d\n", event->bpf.type); 570 break; 571 } 572 573 return 0; 574 } 575 576 int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env) 577 { 578 struct perf_event_attr attr = { 579 .type = PERF_TYPE_SOFTWARE, 580 .config = PERF_COUNT_SW_DUMMY, 581 .sample_id_all = 1, 582 .watermark = 1, 583 .bpf_event = 1, 584 .size = sizeof(attr), /* to capture ABI version */ 585 }; 586 587 /* 588 * Older gcc versions don't support designated initializers, like above, 589 * for unnamed union members, such as the following: 590 */ 591 attr.wakeup_watermark = 1; 592 593 return evlist__add_sb_event(evlist, &attr, bpf_event__sb_cb, env); 594 } 595 596 void bpf_event__print_bpf_prog_info(struct bpf_prog_info *info, 597 struct perf_env *env, 598 FILE *fp) 599 { 600 __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); 601 __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); 602 char name[KSYM_NAME_LEN]; 603 struct btf *btf = NULL; 604 u32 sub_prog_cnt, i; 605 606 sub_prog_cnt = info->nr_jited_ksyms; 607 if (sub_prog_cnt != info->nr_prog_tags || 608 sub_prog_cnt != info->nr_jited_func_lens) 609 return; 610 611 if (info->btf_id) { 612 struct btf_node *node; 613 614 node = perf_env__find_btf(env, info->btf_id); 615 if (node) 616 btf = btf__new((__u8 *)(node->data), 617 node->data_size); 618 } 619 620 if (sub_prog_cnt == 1) { 621 synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, 0); 622 fprintf(fp, "# bpf_prog_info %u: %s addr 0x%llx size %u\n", 623 info->id, name, prog_addrs[0], prog_lens[0]); 624 goto out; 625 } 626 627 fprintf(fp, "# bpf_prog_info %u:\n", info->id); 628 for (i = 0; i < sub_prog_cnt; i++) { 629 synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, i); 630 631 fprintf(fp, "# \tsub_prog %u: %s addr 0x%llx size %u\n", 632 i, name, prog_addrs[i], prog_lens[i]); 633 } 634 out: 635 btf__free(btf); 636 } 637