1 // SPDX-License-Identifier: GPL-2.0 2 #include <errno.h> 3 #include <stdlib.h> 4 #include <bpf/bpf.h> 5 #include <bpf/btf.h> 6 #include <bpf/libbpf.h> 7 #include <linux/btf.h> 8 #include <linux/err.h> 9 #include <linux/string.h> 10 #include <internal/lib.h> 11 #include <symbol/kallsyms.h> 12 #include "bpf-event.h" 13 #include "bpf-utils.h" 14 #include "debug.h" 15 #include "dso.h" 16 #include "symbol.h" 17 #include "machine.h" 18 #include "env.h" 19 #include "session.h" 20 #include "map.h" 21 #include "evlist.h" 22 #include "record.h" 23 #include "util/synthetic-events.h" 24 25 struct btf * __weak btf__load_from_kernel_by_id(__u32 id) 26 { 27 struct btf *btf; 28 #pragma GCC diagnostic push 29 #pragma GCC diagnostic ignored "-Wdeprecated-declarations" 30 int err = btf__get_from_id(id, &btf); 31 #pragma GCC diagnostic pop 32 33 return err ? ERR_PTR(err) : btf; 34 } 35 36 int __weak bpf_prog_load(enum bpf_prog_type prog_type, 37 const char *prog_name __maybe_unused, 38 const char *license, 39 const struct bpf_insn *insns, size_t insn_cnt, 40 const struct bpf_prog_load_opts *opts) 41 { 42 #pragma GCC diagnostic push 43 #pragma GCC diagnostic ignored "-Wdeprecated-declarations" 44 return bpf_load_program(prog_type, insns, insn_cnt, license, 45 opts->kern_version, opts->log_buf, opts->log_size); 46 #pragma GCC diagnostic pop 47 } 48 49 struct bpf_program * __weak 50 bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev) 51 { 52 #pragma GCC diagnostic push 53 #pragma GCC diagnostic ignored "-Wdeprecated-declarations" 54 return bpf_program__next(prev, obj); 55 #pragma GCC diagnostic pop 56 } 57 58 struct bpf_map * __weak 59 bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev) 60 { 61 #pragma GCC diagnostic push 62 #pragma GCC diagnostic ignored "-Wdeprecated-declarations" 63 return bpf_map__next(prev, obj); 64 #pragma GCC diagnostic pop 65 } 66 67 const void * __weak 68 btf__raw_data(const struct btf *btf_ro, __u32 *size) 69 { 70 #pragma GCC diagnostic push 71 #pragma GCC diagnostic ignored "-Wdeprecated-declarations" 72 return btf__get_raw_data(btf_ro, size); 73 #pragma GCC diagnostic pop 74 } 75 76 static int snprintf_hex(char *buf, size_t size, unsigned char *data, size_t len) 77 { 78 int ret = 0; 79 size_t i; 80 81 for (i = 0; i < len; i++) 82 ret += snprintf(buf + ret, size - ret, "%02x", data[i]); 83 return ret; 84 } 85 86 static int machine__process_bpf_event_load(struct machine *machine, 87 union perf_event *event, 88 struct perf_sample *sample __maybe_unused) 89 { 90 struct bpf_prog_info_node *info_node; 91 struct perf_env *env = machine->env; 92 struct perf_bpil *info_linear; 93 int id = event->bpf.id; 94 unsigned int i; 95 96 /* perf-record, no need to handle bpf-event */ 97 if (env == NULL) 98 return 0; 99 100 info_node = perf_env__find_bpf_prog_info(env, id); 101 if (!info_node) 102 return 0; 103 info_linear = info_node->info_linear; 104 105 for (i = 0; i < info_linear->info.nr_jited_ksyms; i++) { 106 u64 *addrs = (u64 *)(uintptr_t)(info_linear->info.jited_ksyms); 107 u64 addr = addrs[i]; 108 struct map *map = maps__find(machine__kernel_maps(machine), addr); 109 110 if (map) { 111 map->dso->binary_type = DSO_BINARY_TYPE__BPF_PROG_INFO; 112 map->dso->bpf_prog.id = id; 113 map->dso->bpf_prog.sub_id = i; 114 map->dso->bpf_prog.env = env; 115 } 116 } 117 return 0; 118 } 119 120 int machine__process_bpf(struct machine *machine, union perf_event *event, 121 struct perf_sample *sample) 122 { 123 if (dump_trace) 124 perf_event__fprintf_bpf(event, stdout); 125 126 switch (event->bpf.type) { 127 case PERF_BPF_EVENT_PROG_LOAD: 128 return machine__process_bpf_event_load(machine, event, sample); 129 130 case PERF_BPF_EVENT_PROG_UNLOAD: 131 /* 132 * Do not free bpf_prog_info and btf of the program here, 133 * as annotation still need them. They will be freed at 134 * the end of the session. 135 */ 136 break; 137 default: 138 pr_debug("unexpected bpf event type of %d\n", event->bpf.type); 139 break; 140 } 141 return 0; 142 } 143 144 static int perf_env__fetch_btf(struct perf_env *env, 145 u32 btf_id, 146 struct btf *btf) 147 { 148 struct btf_node *node; 149 u32 data_size; 150 const void *data; 151 152 data = btf__raw_data(btf, &data_size); 153 154 node = malloc(data_size + sizeof(struct btf_node)); 155 if (!node) 156 return -1; 157 158 node->id = btf_id; 159 node->data_size = data_size; 160 memcpy(node->data, data, data_size); 161 162 if (!perf_env__insert_btf(env, node)) { 163 /* Insertion failed because of a duplicate. */ 164 free(node); 165 return -1; 166 } 167 return 0; 168 } 169 170 static int synthesize_bpf_prog_name(char *buf, int size, 171 struct bpf_prog_info *info, 172 struct btf *btf, 173 u32 sub_id) 174 { 175 u8 (*prog_tags)[BPF_TAG_SIZE] = (void *)(uintptr_t)(info->prog_tags); 176 void *func_infos = (void *)(uintptr_t)(info->func_info); 177 u32 sub_prog_cnt = info->nr_jited_ksyms; 178 const struct bpf_func_info *finfo; 179 const char *short_name = NULL; 180 const struct btf_type *t; 181 int name_len; 182 183 name_len = snprintf(buf, size, "bpf_prog_"); 184 name_len += snprintf_hex(buf + name_len, size - name_len, 185 prog_tags[sub_id], BPF_TAG_SIZE); 186 if (btf) { 187 finfo = func_infos + sub_id * info->func_info_rec_size; 188 t = btf__type_by_id(btf, finfo->type_id); 189 short_name = btf__name_by_offset(btf, t->name_off); 190 } else if (sub_id == 0 && sub_prog_cnt == 1) { 191 /* no subprog */ 192 if (info->name[0]) 193 short_name = info->name; 194 } else 195 short_name = "F"; 196 if (short_name) 197 name_len += snprintf(buf + name_len, size - name_len, 198 "_%s", short_name); 199 return name_len; 200 } 201 202 /* 203 * Synthesize PERF_RECORD_KSYMBOL and PERF_RECORD_BPF_EVENT for one bpf 204 * program. One PERF_RECORD_BPF_EVENT is generated for the program. And 205 * one PERF_RECORD_KSYMBOL is generated for each sub program. 206 * 207 * Returns: 208 * 0 for success; 209 * -1 for failures; 210 * -2 for lack of kernel support. 211 */ 212 static int perf_event__synthesize_one_bpf_prog(struct perf_session *session, 213 perf_event__handler_t process, 214 struct machine *machine, 215 int fd, 216 union perf_event *event, 217 struct record_opts *opts) 218 { 219 struct perf_record_ksymbol *ksymbol_event = &event->ksymbol; 220 struct perf_record_bpf_event *bpf_event = &event->bpf; 221 struct perf_tool *tool = session->tool; 222 struct bpf_prog_info_node *info_node; 223 struct perf_bpil *info_linear; 224 struct bpf_prog_info *info; 225 struct btf *btf = NULL; 226 struct perf_env *env; 227 u32 sub_prog_cnt, i; 228 int err = 0; 229 u64 arrays; 230 231 /* 232 * for perf-record and perf-report use header.env; 233 * otherwise, use global perf_env. 234 */ 235 env = session->data ? &session->header.env : &perf_env; 236 237 arrays = 1UL << PERF_BPIL_JITED_KSYMS; 238 arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS; 239 arrays |= 1UL << PERF_BPIL_FUNC_INFO; 240 arrays |= 1UL << PERF_BPIL_PROG_TAGS; 241 arrays |= 1UL << PERF_BPIL_JITED_INSNS; 242 arrays |= 1UL << PERF_BPIL_LINE_INFO; 243 arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO; 244 245 info_linear = get_bpf_prog_info_linear(fd, arrays); 246 if (IS_ERR_OR_NULL(info_linear)) { 247 info_linear = NULL; 248 pr_debug("%s: failed to get BPF program info. aborting\n", __func__); 249 return -1; 250 } 251 252 if (info_linear->info_len < offsetof(struct bpf_prog_info, prog_tags)) { 253 free(info_linear); 254 pr_debug("%s: the kernel is too old, aborting\n", __func__); 255 return -2; 256 } 257 258 info = &info_linear->info; 259 if (!info->jited_ksyms) { 260 free(info_linear); 261 return -1; 262 } 263 264 /* number of ksyms, func_lengths, and tags should match */ 265 sub_prog_cnt = info->nr_jited_ksyms; 266 if (sub_prog_cnt != info->nr_prog_tags || 267 sub_prog_cnt != info->nr_jited_func_lens) { 268 free(info_linear); 269 return -1; 270 } 271 272 /* check BTF func info support */ 273 if (info->btf_id && info->nr_func_info && info->func_info_rec_size) { 274 /* btf func info number should be same as sub_prog_cnt */ 275 if (sub_prog_cnt != info->nr_func_info) { 276 pr_debug("%s: mismatch in BPF sub program count and BTF function info count, aborting\n", __func__); 277 free(info_linear); 278 return -1; 279 } 280 btf = btf__load_from_kernel_by_id(info->btf_id); 281 if (libbpf_get_error(btf)) { 282 pr_debug("%s: failed to get BTF of id %u, aborting\n", __func__, info->btf_id); 283 err = -1; 284 goto out; 285 } 286 perf_env__fetch_btf(env, info->btf_id, btf); 287 } 288 289 /* Synthesize PERF_RECORD_KSYMBOL */ 290 for (i = 0; i < sub_prog_cnt; i++) { 291 __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); 292 __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); 293 int name_len; 294 295 *ksymbol_event = (struct perf_record_ksymbol) { 296 .header = { 297 .type = PERF_RECORD_KSYMBOL, 298 .size = offsetof(struct perf_record_ksymbol, name), 299 }, 300 .addr = prog_addrs[i], 301 .len = prog_lens[i], 302 .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF, 303 .flags = 0, 304 }; 305 306 name_len = synthesize_bpf_prog_name(ksymbol_event->name, 307 KSYM_NAME_LEN, info, btf, i); 308 ksymbol_event->header.size += PERF_ALIGN(name_len + 1, 309 sizeof(u64)); 310 311 memset((void *)event + event->header.size, 0, machine->id_hdr_size); 312 event->header.size += machine->id_hdr_size; 313 err = perf_tool__process_synth_event(tool, event, 314 machine, process); 315 } 316 317 if (!opts->no_bpf_event) { 318 /* Synthesize PERF_RECORD_BPF_EVENT */ 319 *bpf_event = (struct perf_record_bpf_event) { 320 .header = { 321 .type = PERF_RECORD_BPF_EVENT, 322 .size = sizeof(struct perf_record_bpf_event), 323 }, 324 .type = PERF_BPF_EVENT_PROG_LOAD, 325 .flags = 0, 326 .id = info->id, 327 }; 328 memcpy(bpf_event->tag, info->tag, BPF_TAG_SIZE); 329 memset((void *)event + event->header.size, 0, machine->id_hdr_size); 330 event->header.size += machine->id_hdr_size; 331 332 /* save bpf_prog_info to env */ 333 info_node = malloc(sizeof(struct bpf_prog_info_node)); 334 if (!info_node) { 335 err = -1; 336 goto out; 337 } 338 339 info_node->info_linear = info_linear; 340 perf_env__insert_bpf_prog_info(env, info_node); 341 info_linear = NULL; 342 343 /* 344 * process after saving bpf_prog_info to env, so that 345 * required information is ready for look up 346 */ 347 err = perf_tool__process_synth_event(tool, event, 348 machine, process); 349 } 350 351 out: 352 free(info_linear); 353 btf__free(btf); 354 return err ? -1 : 0; 355 } 356 357 struct kallsyms_parse { 358 union perf_event *event; 359 perf_event__handler_t process; 360 struct machine *machine; 361 struct perf_tool *tool; 362 }; 363 364 static int 365 process_bpf_image(char *name, u64 addr, struct kallsyms_parse *data) 366 { 367 struct machine *machine = data->machine; 368 union perf_event *event = data->event; 369 struct perf_record_ksymbol *ksymbol; 370 int len; 371 372 ksymbol = &event->ksymbol; 373 374 *ksymbol = (struct perf_record_ksymbol) { 375 .header = { 376 .type = PERF_RECORD_KSYMBOL, 377 .size = offsetof(struct perf_record_ksymbol, name), 378 }, 379 .addr = addr, 380 .len = page_size, 381 .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF, 382 .flags = 0, 383 }; 384 385 len = scnprintf(ksymbol->name, KSYM_NAME_LEN, "%s", name); 386 ksymbol->header.size += PERF_ALIGN(len + 1, sizeof(u64)); 387 memset((void *) event + event->header.size, 0, machine->id_hdr_size); 388 event->header.size += machine->id_hdr_size; 389 390 return perf_tool__process_synth_event(data->tool, event, machine, 391 data->process); 392 } 393 394 static int 395 kallsyms_process_symbol(void *data, const char *_name, 396 char type __maybe_unused, u64 start) 397 { 398 char disp[KSYM_NAME_LEN]; 399 char *module, *name; 400 unsigned long id; 401 int err = 0; 402 403 module = strchr(_name, '\t'); 404 if (!module) 405 return 0; 406 407 /* We are going after [bpf] module ... */ 408 if (strcmp(module + 1, "[bpf]")) 409 return 0; 410 411 name = memdup(_name, (module - _name) + 1); 412 if (!name) 413 return -ENOMEM; 414 415 name[module - _name] = 0; 416 417 /* .. and only for trampolines and dispatchers */ 418 if ((sscanf(name, "bpf_trampoline_%lu", &id) == 1) || 419 (sscanf(name, "bpf_dispatcher_%s", disp) == 1)) 420 err = process_bpf_image(name, start, data); 421 422 free(name); 423 return err; 424 } 425 426 int perf_event__synthesize_bpf_events(struct perf_session *session, 427 perf_event__handler_t process, 428 struct machine *machine, 429 struct record_opts *opts) 430 { 431 const char *kallsyms_filename = "/proc/kallsyms"; 432 struct kallsyms_parse arg; 433 union perf_event *event; 434 __u32 id = 0; 435 int err; 436 int fd; 437 438 event = malloc(sizeof(event->bpf) + KSYM_NAME_LEN + machine->id_hdr_size); 439 if (!event) 440 return -1; 441 442 /* Synthesize all the bpf programs in system. */ 443 while (true) { 444 err = bpf_prog_get_next_id(id, &id); 445 if (err) { 446 if (errno == ENOENT) { 447 err = 0; 448 break; 449 } 450 pr_debug("%s: can't get next program: %s%s\n", 451 __func__, strerror(errno), 452 errno == EINVAL ? " -- kernel too old?" : ""); 453 /* don't report error on old kernel or EPERM */ 454 err = (errno == EINVAL || errno == EPERM) ? 0 : -1; 455 break; 456 } 457 fd = bpf_prog_get_fd_by_id(id); 458 if (fd < 0) { 459 pr_debug("%s: failed to get fd for prog_id %u\n", 460 __func__, id); 461 continue; 462 } 463 464 err = perf_event__synthesize_one_bpf_prog(session, process, 465 machine, fd, 466 event, opts); 467 close(fd); 468 if (err) { 469 /* do not return error for old kernel */ 470 if (err == -2) 471 err = 0; 472 break; 473 } 474 } 475 476 /* Synthesize all the bpf images - trampolines/dispatchers. */ 477 if (symbol_conf.kallsyms_name != NULL) 478 kallsyms_filename = symbol_conf.kallsyms_name; 479 480 arg = (struct kallsyms_parse) { 481 .event = event, 482 .process = process, 483 .machine = machine, 484 .tool = session->tool, 485 }; 486 487 if (kallsyms__parse(kallsyms_filename, &arg, kallsyms_process_symbol)) { 488 pr_err("%s: failed to synthesize bpf images: %s\n", 489 __func__, strerror(errno)); 490 } 491 492 free(event); 493 return err; 494 } 495 496 static void perf_env__add_bpf_info(struct perf_env *env, u32 id) 497 { 498 struct bpf_prog_info_node *info_node; 499 struct perf_bpil *info_linear; 500 struct btf *btf = NULL; 501 u64 arrays; 502 u32 btf_id; 503 int fd; 504 505 fd = bpf_prog_get_fd_by_id(id); 506 if (fd < 0) 507 return; 508 509 arrays = 1UL << PERF_BPIL_JITED_KSYMS; 510 arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS; 511 arrays |= 1UL << PERF_BPIL_FUNC_INFO; 512 arrays |= 1UL << PERF_BPIL_PROG_TAGS; 513 arrays |= 1UL << PERF_BPIL_JITED_INSNS; 514 arrays |= 1UL << PERF_BPIL_LINE_INFO; 515 arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO; 516 517 info_linear = get_bpf_prog_info_linear(fd, arrays); 518 if (IS_ERR_OR_NULL(info_linear)) { 519 pr_debug("%s: failed to get BPF program info. aborting\n", __func__); 520 goto out; 521 } 522 523 btf_id = info_linear->info.btf_id; 524 525 info_node = malloc(sizeof(struct bpf_prog_info_node)); 526 if (info_node) { 527 info_node->info_linear = info_linear; 528 perf_env__insert_bpf_prog_info(env, info_node); 529 } else 530 free(info_linear); 531 532 if (btf_id == 0) 533 goto out; 534 535 btf = btf__load_from_kernel_by_id(btf_id); 536 if (libbpf_get_error(btf)) { 537 pr_debug("%s: failed to get BTF of id %u, aborting\n", 538 __func__, btf_id); 539 goto out; 540 } 541 perf_env__fetch_btf(env, btf_id, btf); 542 543 out: 544 btf__free(btf); 545 close(fd); 546 } 547 548 static int bpf_event__sb_cb(union perf_event *event, void *data) 549 { 550 struct perf_env *env = data; 551 552 if (event->header.type != PERF_RECORD_BPF_EVENT) 553 return -1; 554 555 switch (event->bpf.type) { 556 case PERF_BPF_EVENT_PROG_LOAD: 557 perf_env__add_bpf_info(env, event->bpf.id); 558 559 case PERF_BPF_EVENT_PROG_UNLOAD: 560 /* 561 * Do not free bpf_prog_info and btf of the program here, 562 * as annotation still need them. They will be freed at 563 * the end of the session. 564 */ 565 break; 566 default: 567 pr_debug("unexpected bpf event type of %d\n", event->bpf.type); 568 break; 569 } 570 571 return 0; 572 } 573 574 int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env) 575 { 576 struct perf_event_attr attr = { 577 .type = PERF_TYPE_SOFTWARE, 578 .config = PERF_COUNT_SW_DUMMY, 579 .sample_id_all = 1, 580 .watermark = 1, 581 .bpf_event = 1, 582 .size = sizeof(attr), /* to capture ABI version */ 583 }; 584 585 /* 586 * Older gcc versions don't support designated initializers, like above, 587 * for unnamed union members, such as the following: 588 */ 589 attr.wakeup_watermark = 1; 590 591 return evlist__add_sb_event(evlist, &attr, bpf_event__sb_cb, env); 592 } 593 594 void bpf_event__print_bpf_prog_info(struct bpf_prog_info *info, 595 struct perf_env *env, 596 FILE *fp) 597 { 598 __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); 599 __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); 600 char name[KSYM_NAME_LEN]; 601 struct btf *btf = NULL; 602 u32 sub_prog_cnt, i; 603 604 sub_prog_cnt = info->nr_jited_ksyms; 605 if (sub_prog_cnt != info->nr_prog_tags || 606 sub_prog_cnt != info->nr_jited_func_lens) 607 return; 608 609 if (info->btf_id) { 610 struct btf_node *node; 611 612 node = perf_env__find_btf(env, info->btf_id); 613 if (node) 614 btf = btf__new((__u8 *)(node->data), 615 node->data_size); 616 } 617 618 if (sub_prog_cnt == 1) { 619 synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, 0); 620 fprintf(fp, "# bpf_prog_info %u: %s addr 0x%llx size %u\n", 621 info->id, name, prog_addrs[0], prog_lens[0]); 622 goto out; 623 } 624 625 fprintf(fp, "# bpf_prog_info %u:\n", info->id); 626 for (i = 0; i < sub_prog_cnt; i++) { 627 synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, i); 628 629 fprintf(fp, "# \tsub_prog %u: %s addr 0x%llx size %u\n", 630 i, name, prog_addrs[i], prog_lens[i]); 631 } 632 out: 633 btf__free(btf); 634 } 635