1 // SPDX-License-Identifier: GPL-2.0 2 #include <errno.h> 3 #include <stdlib.h> 4 #include <bpf/bpf.h> 5 #include <bpf/btf.h> 6 #include <bpf/libbpf.h> 7 #include <linux/btf.h> 8 #include <linux/err.h> 9 #include <linux/string.h> 10 #include <internal/lib.h> 11 #include <symbol/kallsyms.h> 12 #include "bpf-event.h" 13 #include "bpf-utils.h" 14 #include "debug.h" 15 #include "dso.h" 16 #include "symbol.h" 17 #include "machine.h" 18 #include "env.h" 19 #include "session.h" 20 #include "map.h" 21 #include "evlist.h" 22 #include "record.h" 23 #include "util/synthetic-events.h" 24 25 struct btf * __weak btf__load_from_kernel_by_id(__u32 id) 26 { 27 struct btf *btf; 28 #pragma GCC diagnostic push 29 #pragma GCC diagnostic ignored "-Wdeprecated-declarations" 30 int err = btf__get_from_id(id, &btf); 31 #pragma GCC diagnostic pop 32 33 return err ? ERR_PTR(err) : btf; 34 } 35 36 static int snprintf_hex(char *buf, size_t size, unsigned char *data, size_t len) 37 { 38 int ret = 0; 39 size_t i; 40 41 for (i = 0; i < len; i++) 42 ret += snprintf(buf + ret, size - ret, "%02x", data[i]); 43 return ret; 44 } 45 46 static int machine__process_bpf_event_load(struct machine *machine, 47 union perf_event *event, 48 struct perf_sample *sample __maybe_unused) 49 { 50 struct bpf_prog_info_node *info_node; 51 struct perf_env *env = machine->env; 52 struct perf_bpil *info_linear; 53 int id = event->bpf.id; 54 unsigned int i; 55 56 /* perf-record, no need to handle bpf-event */ 57 if (env == NULL) 58 return 0; 59 60 info_node = perf_env__find_bpf_prog_info(env, id); 61 if (!info_node) 62 return 0; 63 info_linear = info_node->info_linear; 64 65 for (i = 0; i < info_linear->info.nr_jited_ksyms; i++) { 66 u64 *addrs = (u64 *)(uintptr_t)(info_linear->info.jited_ksyms); 67 u64 addr = addrs[i]; 68 struct map *map = maps__find(&machine->kmaps, addr); 69 70 if (map) { 71 map->dso->binary_type = DSO_BINARY_TYPE__BPF_PROG_INFO; 72 map->dso->bpf_prog.id = id; 73 map->dso->bpf_prog.sub_id = i; 74 map->dso->bpf_prog.env = env; 75 } 76 } 77 return 0; 78 } 79 80 int machine__process_bpf(struct machine *machine, union perf_event *event, 81 struct perf_sample *sample) 82 { 83 if (dump_trace) 84 perf_event__fprintf_bpf(event, stdout); 85 86 switch (event->bpf.type) { 87 case PERF_BPF_EVENT_PROG_LOAD: 88 return machine__process_bpf_event_load(machine, event, sample); 89 90 case PERF_BPF_EVENT_PROG_UNLOAD: 91 /* 92 * Do not free bpf_prog_info and btf of the program here, 93 * as annotation still need them. They will be freed at 94 * the end of the session. 95 */ 96 break; 97 default: 98 pr_debug("unexpected bpf event type of %d\n", event->bpf.type); 99 break; 100 } 101 return 0; 102 } 103 104 static int perf_env__fetch_btf(struct perf_env *env, 105 u32 btf_id, 106 struct btf *btf) 107 { 108 struct btf_node *node; 109 u32 data_size; 110 const void *data; 111 112 data = btf__raw_data(btf, &data_size); 113 114 node = malloc(data_size + sizeof(struct btf_node)); 115 if (!node) 116 return -1; 117 118 node->id = btf_id; 119 node->data_size = data_size; 120 memcpy(node->data, data, data_size); 121 122 perf_env__insert_btf(env, node); 123 return 0; 124 } 125 126 static int synthesize_bpf_prog_name(char *buf, int size, 127 struct bpf_prog_info *info, 128 struct btf *btf, 129 u32 sub_id) 130 { 131 u8 (*prog_tags)[BPF_TAG_SIZE] = (void *)(uintptr_t)(info->prog_tags); 132 void *func_infos = (void *)(uintptr_t)(info->func_info); 133 u32 sub_prog_cnt = info->nr_jited_ksyms; 134 const struct bpf_func_info *finfo; 135 const char *short_name = NULL; 136 const struct btf_type *t; 137 int name_len; 138 139 name_len = snprintf(buf, size, "bpf_prog_"); 140 name_len += snprintf_hex(buf + name_len, size - name_len, 141 prog_tags[sub_id], BPF_TAG_SIZE); 142 if (btf) { 143 finfo = func_infos + sub_id * info->func_info_rec_size; 144 t = btf__type_by_id(btf, finfo->type_id); 145 short_name = btf__name_by_offset(btf, t->name_off); 146 } else if (sub_id == 0 && sub_prog_cnt == 1) { 147 /* no subprog */ 148 if (info->name[0]) 149 short_name = info->name; 150 } else 151 short_name = "F"; 152 if (short_name) 153 name_len += snprintf(buf + name_len, size - name_len, 154 "_%s", short_name); 155 return name_len; 156 } 157 158 /* 159 * Synthesize PERF_RECORD_KSYMBOL and PERF_RECORD_BPF_EVENT for one bpf 160 * program. One PERF_RECORD_BPF_EVENT is generated for the program. And 161 * one PERF_RECORD_KSYMBOL is generated for each sub program. 162 * 163 * Returns: 164 * 0 for success; 165 * -1 for failures; 166 * -2 for lack of kernel support. 167 */ 168 static int perf_event__synthesize_one_bpf_prog(struct perf_session *session, 169 perf_event__handler_t process, 170 struct machine *machine, 171 int fd, 172 union perf_event *event, 173 struct record_opts *opts) 174 { 175 struct perf_record_ksymbol *ksymbol_event = &event->ksymbol; 176 struct perf_record_bpf_event *bpf_event = &event->bpf; 177 struct perf_tool *tool = session->tool; 178 struct bpf_prog_info_node *info_node; 179 struct perf_bpil *info_linear; 180 struct bpf_prog_info *info; 181 struct btf *btf = NULL; 182 struct perf_env *env; 183 u32 sub_prog_cnt, i; 184 int err = 0; 185 u64 arrays; 186 187 /* 188 * for perf-record and perf-report use header.env; 189 * otherwise, use global perf_env. 190 */ 191 env = session->data ? &session->header.env : &perf_env; 192 193 arrays = 1UL << PERF_BPIL_JITED_KSYMS; 194 arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS; 195 arrays |= 1UL << PERF_BPIL_FUNC_INFO; 196 arrays |= 1UL << PERF_BPIL_PROG_TAGS; 197 arrays |= 1UL << PERF_BPIL_JITED_INSNS; 198 arrays |= 1UL << PERF_BPIL_LINE_INFO; 199 arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO; 200 201 info_linear = get_bpf_prog_info_linear(fd, arrays); 202 if (IS_ERR_OR_NULL(info_linear)) { 203 info_linear = NULL; 204 pr_debug("%s: failed to get BPF program info. aborting\n", __func__); 205 return -1; 206 } 207 208 if (info_linear->info_len < offsetof(struct bpf_prog_info, prog_tags)) { 209 free(info_linear); 210 pr_debug("%s: the kernel is too old, aborting\n", __func__); 211 return -2; 212 } 213 214 info = &info_linear->info; 215 if (!info->jited_ksyms) { 216 free(info_linear); 217 return -1; 218 } 219 220 /* number of ksyms, func_lengths, and tags should match */ 221 sub_prog_cnt = info->nr_jited_ksyms; 222 if (sub_prog_cnt != info->nr_prog_tags || 223 sub_prog_cnt != info->nr_jited_func_lens) { 224 free(info_linear); 225 return -1; 226 } 227 228 /* check BTF func info support */ 229 if (info->btf_id && info->nr_func_info && info->func_info_rec_size) { 230 /* btf func info number should be same as sub_prog_cnt */ 231 if (sub_prog_cnt != info->nr_func_info) { 232 pr_debug("%s: mismatch in BPF sub program count and BTF function info count, aborting\n", __func__); 233 free(info_linear); 234 return -1; 235 } 236 btf = btf__load_from_kernel_by_id(info->btf_id); 237 if (libbpf_get_error(btf)) { 238 pr_debug("%s: failed to get BTF of id %u, aborting\n", __func__, info->btf_id); 239 err = -1; 240 goto out; 241 } 242 perf_env__fetch_btf(env, info->btf_id, btf); 243 } 244 245 /* Synthesize PERF_RECORD_KSYMBOL */ 246 for (i = 0; i < sub_prog_cnt; i++) { 247 __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); 248 __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); 249 int name_len; 250 251 *ksymbol_event = (struct perf_record_ksymbol) { 252 .header = { 253 .type = PERF_RECORD_KSYMBOL, 254 .size = offsetof(struct perf_record_ksymbol, name), 255 }, 256 .addr = prog_addrs[i], 257 .len = prog_lens[i], 258 .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF, 259 .flags = 0, 260 }; 261 262 name_len = synthesize_bpf_prog_name(ksymbol_event->name, 263 KSYM_NAME_LEN, info, btf, i); 264 ksymbol_event->header.size += PERF_ALIGN(name_len + 1, 265 sizeof(u64)); 266 267 memset((void *)event + event->header.size, 0, machine->id_hdr_size); 268 event->header.size += machine->id_hdr_size; 269 err = perf_tool__process_synth_event(tool, event, 270 machine, process); 271 } 272 273 if (!opts->no_bpf_event) { 274 /* Synthesize PERF_RECORD_BPF_EVENT */ 275 *bpf_event = (struct perf_record_bpf_event) { 276 .header = { 277 .type = PERF_RECORD_BPF_EVENT, 278 .size = sizeof(struct perf_record_bpf_event), 279 }, 280 .type = PERF_BPF_EVENT_PROG_LOAD, 281 .flags = 0, 282 .id = info->id, 283 }; 284 memcpy(bpf_event->tag, info->tag, BPF_TAG_SIZE); 285 memset((void *)event + event->header.size, 0, machine->id_hdr_size); 286 event->header.size += machine->id_hdr_size; 287 288 /* save bpf_prog_info to env */ 289 info_node = malloc(sizeof(struct bpf_prog_info_node)); 290 if (!info_node) { 291 err = -1; 292 goto out; 293 } 294 295 info_node->info_linear = info_linear; 296 perf_env__insert_bpf_prog_info(env, info_node); 297 info_linear = NULL; 298 299 /* 300 * process after saving bpf_prog_info to env, so that 301 * required information is ready for look up 302 */ 303 err = perf_tool__process_synth_event(tool, event, 304 machine, process); 305 } 306 307 out: 308 free(info_linear); 309 btf__free(btf); 310 return err ? -1 : 0; 311 } 312 313 struct kallsyms_parse { 314 union perf_event *event; 315 perf_event__handler_t process; 316 struct machine *machine; 317 struct perf_tool *tool; 318 }; 319 320 static int 321 process_bpf_image(char *name, u64 addr, struct kallsyms_parse *data) 322 { 323 struct machine *machine = data->machine; 324 union perf_event *event = data->event; 325 struct perf_record_ksymbol *ksymbol; 326 int len; 327 328 ksymbol = &event->ksymbol; 329 330 *ksymbol = (struct perf_record_ksymbol) { 331 .header = { 332 .type = PERF_RECORD_KSYMBOL, 333 .size = offsetof(struct perf_record_ksymbol, name), 334 }, 335 .addr = addr, 336 .len = page_size, 337 .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF, 338 .flags = 0, 339 }; 340 341 len = scnprintf(ksymbol->name, KSYM_NAME_LEN, "%s", name); 342 ksymbol->header.size += PERF_ALIGN(len + 1, sizeof(u64)); 343 memset((void *) event + event->header.size, 0, machine->id_hdr_size); 344 event->header.size += machine->id_hdr_size; 345 346 return perf_tool__process_synth_event(data->tool, event, machine, 347 data->process); 348 } 349 350 static int 351 kallsyms_process_symbol(void *data, const char *_name, 352 char type __maybe_unused, u64 start) 353 { 354 char disp[KSYM_NAME_LEN]; 355 char *module, *name; 356 unsigned long id; 357 int err = 0; 358 359 module = strchr(_name, '\t'); 360 if (!module) 361 return 0; 362 363 /* We are going after [bpf] module ... */ 364 if (strcmp(module + 1, "[bpf]")) 365 return 0; 366 367 name = memdup(_name, (module - _name) + 1); 368 if (!name) 369 return -ENOMEM; 370 371 name[module - _name] = 0; 372 373 /* .. and only for trampolines and dispatchers */ 374 if ((sscanf(name, "bpf_trampoline_%lu", &id) == 1) || 375 (sscanf(name, "bpf_dispatcher_%s", disp) == 1)) 376 err = process_bpf_image(name, start, data); 377 378 free(name); 379 return err; 380 } 381 382 int perf_event__synthesize_bpf_events(struct perf_session *session, 383 perf_event__handler_t process, 384 struct machine *machine, 385 struct record_opts *opts) 386 { 387 const char *kallsyms_filename = "/proc/kallsyms"; 388 struct kallsyms_parse arg; 389 union perf_event *event; 390 __u32 id = 0; 391 int err; 392 int fd; 393 394 event = malloc(sizeof(event->bpf) + KSYM_NAME_LEN + machine->id_hdr_size); 395 if (!event) 396 return -1; 397 398 /* Synthesize all the bpf programs in system. */ 399 while (true) { 400 err = bpf_prog_get_next_id(id, &id); 401 if (err) { 402 if (errno == ENOENT) { 403 err = 0; 404 break; 405 } 406 pr_debug("%s: can't get next program: %s%s\n", 407 __func__, strerror(errno), 408 errno == EINVAL ? " -- kernel too old?" : ""); 409 /* don't report error on old kernel or EPERM */ 410 err = (errno == EINVAL || errno == EPERM) ? 0 : -1; 411 break; 412 } 413 fd = bpf_prog_get_fd_by_id(id); 414 if (fd < 0) { 415 pr_debug("%s: failed to get fd for prog_id %u\n", 416 __func__, id); 417 continue; 418 } 419 420 err = perf_event__synthesize_one_bpf_prog(session, process, 421 machine, fd, 422 event, opts); 423 close(fd); 424 if (err) { 425 /* do not return error for old kernel */ 426 if (err == -2) 427 err = 0; 428 break; 429 } 430 } 431 432 /* Synthesize all the bpf images - trampolines/dispatchers. */ 433 if (symbol_conf.kallsyms_name != NULL) 434 kallsyms_filename = symbol_conf.kallsyms_name; 435 436 arg = (struct kallsyms_parse) { 437 .event = event, 438 .process = process, 439 .machine = machine, 440 .tool = session->tool, 441 }; 442 443 if (kallsyms__parse(kallsyms_filename, &arg, kallsyms_process_symbol)) { 444 pr_err("%s: failed to synthesize bpf images: %s\n", 445 __func__, strerror(errno)); 446 } 447 448 free(event); 449 return err; 450 } 451 452 static void perf_env__add_bpf_info(struct perf_env *env, u32 id) 453 { 454 struct bpf_prog_info_node *info_node; 455 struct perf_bpil *info_linear; 456 struct btf *btf = NULL; 457 u64 arrays; 458 u32 btf_id; 459 int fd; 460 461 fd = bpf_prog_get_fd_by_id(id); 462 if (fd < 0) 463 return; 464 465 arrays = 1UL << PERF_BPIL_JITED_KSYMS; 466 arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS; 467 arrays |= 1UL << PERF_BPIL_FUNC_INFO; 468 arrays |= 1UL << PERF_BPIL_PROG_TAGS; 469 arrays |= 1UL << PERF_BPIL_JITED_INSNS; 470 arrays |= 1UL << PERF_BPIL_LINE_INFO; 471 arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO; 472 473 info_linear = get_bpf_prog_info_linear(fd, arrays); 474 if (IS_ERR_OR_NULL(info_linear)) { 475 pr_debug("%s: failed to get BPF program info. aborting\n", __func__); 476 goto out; 477 } 478 479 btf_id = info_linear->info.btf_id; 480 481 info_node = malloc(sizeof(struct bpf_prog_info_node)); 482 if (info_node) { 483 info_node->info_linear = info_linear; 484 perf_env__insert_bpf_prog_info(env, info_node); 485 } else 486 free(info_linear); 487 488 if (btf_id == 0) 489 goto out; 490 491 btf = btf__load_from_kernel_by_id(btf_id); 492 if (libbpf_get_error(btf)) { 493 pr_debug("%s: failed to get BTF of id %u, aborting\n", 494 __func__, btf_id); 495 goto out; 496 } 497 perf_env__fetch_btf(env, btf_id, btf); 498 499 out: 500 btf__free(btf); 501 close(fd); 502 } 503 504 static int bpf_event__sb_cb(union perf_event *event, void *data) 505 { 506 struct perf_env *env = data; 507 508 if (event->header.type != PERF_RECORD_BPF_EVENT) 509 return -1; 510 511 switch (event->bpf.type) { 512 case PERF_BPF_EVENT_PROG_LOAD: 513 perf_env__add_bpf_info(env, event->bpf.id); 514 515 case PERF_BPF_EVENT_PROG_UNLOAD: 516 /* 517 * Do not free bpf_prog_info and btf of the program here, 518 * as annotation still need them. They will be freed at 519 * the end of the session. 520 */ 521 break; 522 default: 523 pr_debug("unexpected bpf event type of %d\n", event->bpf.type); 524 break; 525 } 526 527 return 0; 528 } 529 530 int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env) 531 { 532 struct perf_event_attr attr = { 533 .type = PERF_TYPE_SOFTWARE, 534 .config = PERF_COUNT_SW_DUMMY, 535 .sample_id_all = 1, 536 .watermark = 1, 537 .bpf_event = 1, 538 .size = sizeof(attr), /* to capture ABI version */ 539 }; 540 541 /* 542 * Older gcc versions don't support designated initializers, like above, 543 * for unnamed union members, such as the following: 544 */ 545 attr.wakeup_watermark = 1; 546 547 return evlist__add_sb_event(evlist, &attr, bpf_event__sb_cb, env); 548 } 549 550 void bpf_event__print_bpf_prog_info(struct bpf_prog_info *info, 551 struct perf_env *env, 552 FILE *fp) 553 { 554 __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); 555 __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); 556 char name[KSYM_NAME_LEN]; 557 struct btf *btf = NULL; 558 u32 sub_prog_cnt, i; 559 560 sub_prog_cnt = info->nr_jited_ksyms; 561 if (sub_prog_cnt != info->nr_prog_tags || 562 sub_prog_cnt != info->nr_jited_func_lens) 563 return; 564 565 if (info->btf_id) { 566 struct btf_node *node; 567 568 node = perf_env__find_btf(env, info->btf_id); 569 if (node) 570 btf = btf__new((__u8 *)(node->data), 571 node->data_size); 572 } 573 574 if (sub_prog_cnt == 1) { 575 synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, 0); 576 fprintf(fp, "# bpf_prog_info %u: %s addr 0x%llx size %u\n", 577 info->id, name, prog_addrs[0], prog_lens[0]); 578 goto out; 579 } 580 581 fprintf(fp, "# bpf_prog_info %u:\n", info->id); 582 for (i = 0; i < sub_prog_cnt; i++) { 583 synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, i); 584 585 fprintf(fp, "# \tsub_prog %u: %s addr 0x%llx size %u\n", 586 i, name, prog_addrs[i], prog_lens[i]); 587 } 588 out: 589 btf__free(btf); 590 } 591