1 // SPDX-License-Identifier: GPL-2.0 2 #include <errno.h> 3 #include <stdlib.h> 4 #include <bpf/bpf.h> 5 #include <bpf/btf.h> 6 #include <bpf/libbpf.h> 7 #include <linux/btf.h> 8 #include <linux/err.h> 9 #include <linux/string.h> 10 #include <internal/lib.h> 11 #include <symbol/kallsyms.h> 12 #include "bpf-event.h" 13 #include "debug.h" 14 #include "dso.h" 15 #include "symbol.h" 16 #include "machine.h" 17 #include "env.h" 18 #include "session.h" 19 #include "map.h" 20 #include "evlist.h" 21 #include "record.h" 22 #include "util/synthetic-events.h" 23 24 struct btf * __weak btf__load_from_kernel_by_id(__u32 id) 25 { 26 struct btf *btf; 27 int err = btf__get_from_id(id, &btf); 28 29 return err ? ERR_PTR(err) : btf; 30 } 31 32 #define ptr_to_u64(ptr) ((__u64)(unsigned long)(ptr)) 33 34 static int snprintf_hex(char *buf, size_t size, unsigned char *data, size_t len) 35 { 36 int ret = 0; 37 size_t i; 38 39 for (i = 0; i < len; i++) 40 ret += snprintf(buf + ret, size - ret, "%02x", data[i]); 41 return ret; 42 } 43 44 static int machine__process_bpf_event_load(struct machine *machine, 45 union perf_event *event, 46 struct perf_sample *sample __maybe_unused) 47 { 48 struct bpf_prog_info_linear *info_linear; 49 struct bpf_prog_info_node *info_node; 50 struct perf_env *env = machine->env; 51 int id = event->bpf.id; 52 unsigned int i; 53 54 /* perf-record, no need to handle bpf-event */ 55 if (env == NULL) 56 return 0; 57 58 info_node = perf_env__find_bpf_prog_info(env, id); 59 if (!info_node) 60 return 0; 61 info_linear = info_node->info_linear; 62 63 for (i = 0; i < info_linear->info.nr_jited_ksyms; i++) { 64 u64 *addrs = (u64 *)(uintptr_t)(info_linear->info.jited_ksyms); 65 u64 addr = addrs[i]; 66 struct map *map = maps__find(&machine->kmaps, addr); 67 68 if (map) { 69 map->dso->binary_type = DSO_BINARY_TYPE__BPF_PROG_INFO; 70 map->dso->bpf_prog.id = id; 71 map->dso->bpf_prog.sub_id = i; 72 map->dso->bpf_prog.env = env; 73 } 74 } 75 return 0; 76 } 77 78 int machine__process_bpf(struct machine *machine, union perf_event *event, 79 struct perf_sample *sample) 80 { 81 if (dump_trace) 82 perf_event__fprintf_bpf(event, stdout); 83 84 switch (event->bpf.type) { 85 case PERF_BPF_EVENT_PROG_LOAD: 86 return machine__process_bpf_event_load(machine, event, sample); 87 88 case PERF_BPF_EVENT_PROG_UNLOAD: 89 /* 90 * Do not free bpf_prog_info and btf of the program here, 91 * as annotation still need them. They will be freed at 92 * the end of the session. 93 */ 94 break; 95 default: 96 pr_debug("unexpected bpf event type of %d\n", event->bpf.type); 97 break; 98 } 99 return 0; 100 } 101 102 static int perf_env__fetch_btf(struct perf_env *env, 103 u32 btf_id, 104 struct btf *btf) 105 { 106 struct btf_node *node; 107 u32 data_size; 108 const void *data; 109 110 data = btf__get_raw_data(btf, &data_size); 111 112 node = malloc(data_size + sizeof(struct btf_node)); 113 if (!node) 114 return -1; 115 116 node->id = btf_id; 117 node->data_size = data_size; 118 memcpy(node->data, data, data_size); 119 120 perf_env__insert_btf(env, node); 121 return 0; 122 } 123 124 static int synthesize_bpf_prog_name(char *buf, int size, 125 struct bpf_prog_info *info, 126 struct btf *btf, 127 u32 sub_id) 128 { 129 u8 (*prog_tags)[BPF_TAG_SIZE] = (void *)(uintptr_t)(info->prog_tags); 130 void *func_infos = (void *)(uintptr_t)(info->func_info); 131 u32 sub_prog_cnt = info->nr_jited_ksyms; 132 const struct bpf_func_info *finfo; 133 const char *short_name = NULL; 134 const struct btf_type *t; 135 int name_len; 136 137 name_len = snprintf(buf, size, "bpf_prog_"); 138 name_len += snprintf_hex(buf + name_len, size - name_len, 139 prog_tags[sub_id], BPF_TAG_SIZE); 140 if (btf) { 141 finfo = func_infos + sub_id * info->func_info_rec_size; 142 t = btf__type_by_id(btf, finfo->type_id); 143 short_name = btf__name_by_offset(btf, t->name_off); 144 } else if (sub_id == 0 && sub_prog_cnt == 1) { 145 /* no subprog */ 146 if (info->name[0]) 147 short_name = info->name; 148 } else 149 short_name = "F"; 150 if (short_name) 151 name_len += snprintf(buf + name_len, size - name_len, 152 "_%s", short_name); 153 return name_len; 154 } 155 156 /* 157 * Synthesize PERF_RECORD_KSYMBOL and PERF_RECORD_BPF_EVENT for one bpf 158 * program. One PERF_RECORD_BPF_EVENT is generated for the program. And 159 * one PERF_RECORD_KSYMBOL is generated for each sub program. 160 * 161 * Returns: 162 * 0 for success; 163 * -1 for failures; 164 * -2 for lack of kernel support. 165 */ 166 static int perf_event__synthesize_one_bpf_prog(struct perf_session *session, 167 perf_event__handler_t process, 168 struct machine *machine, 169 int fd, 170 union perf_event *event, 171 struct record_opts *opts) 172 { 173 struct perf_record_ksymbol *ksymbol_event = &event->ksymbol; 174 struct perf_record_bpf_event *bpf_event = &event->bpf; 175 struct bpf_prog_info_linear *info_linear; 176 struct perf_tool *tool = session->tool; 177 struct bpf_prog_info_node *info_node; 178 struct bpf_prog_info *info; 179 struct btf *btf = NULL; 180 struct perf_env *env; 181 u32 sub_prog_cnt, i; 182 int err = 0; 183 u64 arrays; 184 185 /* 186 * for perf-record and perf-report use header.env; 187 * otherwise, use global perf_env. 188 */ 189 env = session->data ? &session->header.env : &perf_env; 190 191 arrays = 1UL << BPF_PROG_INFO_JITED_KSYMS; 192 arrays |= 1UL << BPF_PROG_INFO_JITED_FUNC_LENS; 193 arrays |= 1UL << BPF_PROG_INFO_FUNC_INFO; 194 arrays |= 1UL << BPF_PROG_INFO_PROG_TAGS; 195 arrays |= 1UL << BPF_PROG_INFO_JITED_INSNS; 196 arrays |= 1UL << BPF_PROG_INFO_LINE_INFO; 197 arrays |= 1UL << BPF_PROG_INFO_JITED_LINE_INFO; 198 199 info_linear = bpf_program__get_prog_info_linear(fd, arrays); 200 if (IS_ERR_OR_NULL(info_linear)) { 201 info_linear = NULL; 202 pr_debug("%s: failed to get BPF program info. aborting\n", __func__); 203 return -1; 204 } 205 206 if (info_linear->info_len < offsetof(struct bpf_prog_info, prog_tags)) { 207 free(info_linear); 208 pr_debug("%s: the kernel is too old, aborting\n", __func__); 209 return -2; 210 } 211 212 info = &info_linear->info; 213 if (!info->jited_ksyms) { 214 free(info_linear); 215 return -1; 216 } 217 218 /* number of ksyms, func_lengths, and tags should match */ 219 sub_prog_cnt = info->nr_jited_ksyms; 220 if (sub_prog_cnt != info->nr_prog_tags || 221 sub_prog_cnt != info->nr_jited_func_lens) { 222 free(info_linear); 223 return -1; 224 } 225 226 /* check BTF func info support */ 227 if (info->btf_id && info->nr_func_info && info->func_info_rec_size) { 228 /* btf func info number should be same as sub_prog_cnt */ 229 if (sub_prog_cnt != info->nr_func_info) { 230 pr_debug("%s: mismatch in BPF sub program count and BTF function info count, aborting\n", __func__); 231 free(info_linear); 232 return -1; 233 } 234 btf = btf__load_from_kernel_by_id(info->btf_id); 235 if (libbpf_get_error(btf)) { 236 pr_debug("%s: failed to get BTF of id %u, aborting\n", __func__, info->btf_id); 237 err = -1; 238 goto out; 239 } 240 perf_env__fetch_btf(env, info->btf_id, btf); 241 } 242 243 /* Synthesize PERF_RECORD_KSYMBOL */ 244 for (i = 0; i < sub_prog_cnt; i++) { 245 __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); 246 __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); 247 int name_len; 248 249 *ksymbol_event = (struct perf_record_ksymbol) { 250 .header = { 251 .type = PERF_RECORD_KSYMBOL, 252 .size = offsetof(struct perf_record_ksymbol, name), 253 }, 254 .addr = prog_addrs[i], 255 .len = prog_lens[i], 256 .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF, 257 .flags = 0, 258 }; 259 260 name_len = synthesize_bpf_prog_name(ksymbol_event->name, 261 KSYM_NAME_LEN, info, btf, i); 262 ksymbol_event->header.size += PERF_ALIGN(name_len + 1, 263 sizeof(u64)); 264 265 memset((void *)event + event->header.size, 0, machine->id_hdr_size); 266 event->header.size += machine->id_hdr_size; 267 err = perf_tool__process_synth_event(tool, event, 268 machine, process); 269 } 270 271 if (!opts->no_bpf_event) { 272 /* Synthesize PERF_RECORD_BPF_EVENT */ 273 *bpf_event = (struct perf_record_bpf_event) { 274 .header = { 275 .type = PERF_RECORD_BPF_EVENT, 276 .size = sizeof(struct perf_record_bpf_event), 277 }, 278 .type = PERF_BPF_EVENT_PROG_LOAD, 279 .flags = 0, 280 .id = info->id, 281 }; 282 memcpy(bpf_event->tag, info->tag, BPF_TAG_SIZE); 283 memset((void *)event + event->header.size, 0, machine->id_hdr_size); 284 event->header.size += machine->id_hdr_size; 285 286 /* save bpf_prog_info to env */ 287 info_node = malloc(sizeof(struct bpf_prog_info_node)); 288 if (!info_node) { 289 err = -1; 290 goto out; 291 } 292 293 info_node->info_linear = info_linear; 294 perf_env__insert_bpf_prog_info(env, info_node); 295 info_linear = NULL; 296 297 /* 298 * process after saving bpf_prog_info to env, so that 299 * required information is ready for look up 300 */ 301 err = perf_tool__process_synth_event(tool, event, 302 machine, process); 303 } 304 305 out: 306 free(info_linear); 307 btf__free(btf); 308 return err ? -1 : 0; 309 } 310 311 struct kallsyms_parse { 312 union perf_event *event; 313 perf_event__handler_t process; 314 struct machine *machine; 315 struct perf_tool *tool; 316 }; 317 318 static int 319 process_bpf_image(char *name, u64 addr, struct kallsyms_parse *data) 320 { 321 struct machine *machine = data->machine; 322 union perf_event *event = data->event; 323 struct perf_record_ksymbol *ksymbol; 324 int len; 325 326 ksymbol = &event->ksymbol; 327 328 *ksymbol = (struct perf_record_ksymbol) { 329 .header = { 330 .type = PERF_RECORD_KSYMBOL, 331 .size = offsetof(struct perf_record_ksymbol, name), 332 }, 333 .addr = addr, 334 .len = page_size, 335 .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF, 336 .flags = 0, 337 }; 338 339 len = scnprintf(ksymbol->name, KSYM_NAME_LEN, "%s", name); 340 ksymbol->header.size += PERF_ALIGN(len + 1, sizeof(u64)); 341 memset((void *) event + event->header.size, 0, machine->id_hdr_size); 342 event->header.size += machine->id_hdr_size; 343 344 return perf_tool__process_synth_event(data->tool, event, machine, 345 data->process); 346 } 347 348 static int 349 kallsyms_process_symbol(void *data, const char *_name, 350 char type __maybe_unused, u64 start) 351 { 352 char disp[KSYM_NAME_LEN]; 353 char *module, *name; 354 unsigned long id; 355 int err = 0; 356 357 module = strchr(_name, '\t'); 358 if (!module) 359 return 0; 360 361 /* We are going after [bpf] module ... */ 362 if (strcmp(module + 1, "[bpf]")) 363 return 0; 364 365 name = memdup(_name, (module - _name) + 1); 366 if (!name) 367 return -ENOMEM; 368 369 name[module - _name] = 0; 370 371 /* .. and only for trampolines and dispatchers */ 372 if ((sscanf(name, "bpf_trampoline_%lu", &id) == 1) || 373 (sscanf(name, "bpf_dispatcher_%s", disp) == 1)) 374 err = process_bpf_image(name, start, data); 375 376 free(name); 377 return err; 378 } 379 380 int perf_event__synthesize_bpf_events(struct perf_session *session, 381 perf_event__handler_t process, 382 struct machine *machine, 383 struct record_opts *opts) 384 { 385 const char *kallsyms_filename = "/proc/kallsyms"; 386 struct kallsyms_parse arg; 387 union perf_event *event; 388 __u32 id = 0; 389 int err; 390 int fd; 391 392 event = malloc(sizeof(event->bpf) + KSYM_NAME_LEN + machine->id_hdr_size); 393 if (!event) 394 return -1; 395 396 /* Synthesize all the bpf programs in system. */ 397 while (true) { 398 err = bpf_prog_get_next_id(id, &id); 399 if (err) { 400 if (errno == ENOENT) { 401 err = 0; 402 break; 403 } 404 pr_debug("%s: can't get next program: %s%s\n", 405 __func__, strerror(errno), 406 errno == EINVAL ? " -- kernel too old?" : ""); 407 /* don't report error on old kernel or EPERM */ 408 err = (errno == EINVAL || errno == EPERM) ? 0 : -1; 409 break; 410 } 411 fd = bpf_prog_get_fd_by_id(id); 412 if (fd < 0) { 413 pr_debug("%s: failed to get fd for prog_id %u\n", 414 __func__, id); 415 continue; 416 } 417 418 err = perf_event__synthesize_one_bpf_prog(session, process, 419 machine, fd, 420 event, opts); 421 close(fd); 422 if (err) { 423 /* do not return error for old kernel */ 424 if (err == -2) 425 err = 0; 426 break; 427 } 428 } 429 430 /* Synthesize all the bpf images - trampolines/dispatchers. */ 431 if (symbol_conf.kallsyms_name != NULL) 432 kallsyms_filename = symbol_conf.kallsyms_name; 433 434 arg = (struct kallsyms_parse) { 435 .event = event, 436 .process = process, 437 .machine = machine, 438 .tool = session->tool, 439 }; 440 441 if (kallsyms__parse(kallsyms_filename, &arg, kallsyms_process_symbol)) { 442 pr_err("%s: failed to synthesize bpf images: %s\n", 443 __func__, strerror(errno)); 444 } 445 446 free(event); 447 return err; 448 } 449 450 static void perf_env__add_bpf_info(struct perf_env *env, u32 id) 451 { 452 struct bpf_prog_info_linear *info_linear; 453 struct bpf_prog_info_node *info_node; 454 struct btf *btf = NULL; 455 u64 arrays; 456 u32 btf_id; 457 int fd; 458 459 fd = bpf_prog_get_fd_by_id(id); 460 if (fd < 0) 461 return; 462 463 arrays = 1UL << BPF_PROG_INFO_JITED_KSYMS; 464 arrays |= 1UL << BPF_PROG_INFO_JITED_FUNC_LENS; 465 arrays |= 1UL << BPF_PROG_INFO_FUNC_INFO; 466 arrays |= 1UL << BPF_PROG_INFO_PROG_TAGS; 467 arrays |= 1UL << BPF_PROG_INFO_JITED_INSNS; 468 arrays |= 1UL << BPF_PROG_INFO_LINE_INFO; 469 arrays |= 1UL << BPF_PROG_INFO_JITED_LINE_INFO; 470 471 info_linear = bpf_program__get_prog_info_linear(fd, arrays); 472 if (IS_ERR_OR_NULL(info_linear)) { 473 pr_debug("%s: failed to get BPF program info. aborting\n", __func__); 474 goto out; 475 } 476 477 btf_id = info_linear->info.btf_id; 478 479 info_node = malloc(sizeof(struct bpf_prog_info_node)); 480 if (info_node) { 481 info_node->info_linear = info_linear; 482 perf_env__insert_bpf_prog_info(env, info_node); 483 } else 484 free(info_linear); 485 486 if (btf_id == 0) 487 goto out; 488 489 btf = btf__load_from_kernel_by_id(btf_id); 490 if (libbpf_get_error(btf)) { 491 pr_debug("%s: failed to get BTF of id %u, aborting\n", 492 __func__, btf_id); 493 goto out; 494 } 495 perf_env__fetch_btf(env, btf_id, btf); 496 497 out: 498 btf__free(btf); 499 close(fd); 500 } 501 502 static int bpf_event__sb_cb(union perf_event *event, void *data) 503 { 504 struct perf_env *env = data; 505 506 if (event->header.type != PERF_RECORD_BPF_EVENT) 507 return -1; 508 509 switch (event->bpf.type) { 510 case PERF_BPF_EVENT_PROG_LOAD: 511 perf_env__add_bpf_info(env, event->bpf.id); 512 513 case PERF_BPF_EVENT_PROG_UNLOAD: 514 /* 515 * Do not free bpf_prog_info and btf of the program here, 516 * as annotation still need them. They will be freed at 517 * the end of the session. 518 */ 519 break; 520 default: 521 pr_debug("unexpected bpf event type of %d\n", event->bpf.type); 522 break; 523 } 524 525 return 0; 526 } 527 528 int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env) 529 { 530 struct perf_event_attr attr = { 531 .type = PERF_TYPE_SOFTWARE, 532 .config = PERF_COUNT_SW_DUMMY, 533 .sample_id_all = 1, 534 .watermark = 1, 535 .bpf_event = 1, 536 .size = sizeof(attr), /* to capture ABI version */ 537 }; 538 539 /* 540 * Older gcc versions don't support designated initializers, like above, 541 * for unnamed union members, such as the following: 542 */ 543 attr.wakeup_watermark = 1; 544 545 return evlist__add_sb_event(evlist, &attr, bpf_event__sb_cb, env); 546 } 547 548 void bpf_event__print_bpf_prog_info(struct bpf_prog_info *info, 549 struct perf_env *env, 550 FILE *fp) 551 { 552 __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); 553 __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); 554 char name[KSYM_NAME_LEN]; 555 struct btf *btf = NULL; 556 u32 sub_prog_cnt, i; 557 558 sub_prog_cnt = info->nr_jited_ksyms; 559 if (sub_prog_cnt != info->nr_prog_tags || 560 sub_prog_cnt != info->nr_jited_func_lens) 561 return; 562 563 if (info->btf_id) { 564 struct btf_node *node; 565 566 node = perf_env__find_btf(env, info->btf_id); 567 if (node) 568 btf = btf__new((__u8 *)(node->data), 569 node->data_size); 570 } 571 572 if (sub_prog_cnt == 1) { 573 synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, 0); 574 fprintf(fp, "# bpf_prog_info %u: %s addr 0x%llx size %u\n", 575 info->id, name, prog_addrs[0], prog_lens[0]); 576 return; 577 } 578 579 fprintf(fp, "# bpf_prog_info %u:\n", info->id); 580 for (i = 0; i < sub_prog_cnt; i++) { 581 synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, i); 582 583 fprintf(fp, "# \tsub_prog %u: %s addr 0x%llx size %u\n", 584 i, name, prog_addrs[i], prog_lens[i]); 585 } 586 } 587