1 // SPDX-License-Identifier: GPL-2.0 2 #include <errno.h> 3 #include <stdlib.h> 4 #include <bpf/bpf.h> 5 #include <bpf/btf.h> 6 #include <bpf/libbpf.h> 7 #include <linux/btf.h> 8 #include <linux/err.h> 9 #include <linux/string.h> 10 #include <internal/lib.h> 11 #include <symbol/kallsyms.h> 12 #include "bpf-event.h" 13 #include "bpf-utils.h" 14 #include "debug.h" 15 #include "dso.h" 16 #include "symbol.h" 17 #include "machine.h" 18 #include "env.h" 19 #include "session.h" 20 #include "map.h" 21 #include "evlist.h" 22 #include "record.h" 23 #include "util/synthetic-events.h" 24 25 static int snprintf_hex(char *buf, size_t size, unsigned char *data, size_t len) 26 { 27 int ret = 0; 28 size_t i; 29 30 for (i = 0; i < len; i++) 31 ret += snprintf(buf + ret, size - ret, "%02x", data[i]); 32 return ret; 33 } 34 35 static int machine__process_bpf_event_load(struct machine *machine, 36 union perf_event *event, 37 struct perf_sample *sample __maybe_unused) 38 { 39 struct bpf_prog_info_node *info_node; 40 struct perf_env *env = machine->env; 41 struct perf_bpil *info_linear; 42 int id = event->bpf.id; 43 unsigned int i; 44 45 /* perf-record, no need to handle bpf-event */ 46 if (env == NULL) 47 return 0; 48 49 info_node = perf_env__find_bpf_prog_info(env, id); 50 if (!info_node) 51 return 0; 52 info_linear = info_node->info_linear; 53 54 for (i = 0; i < info_linear->info.nr_jited_ksyms; i++) { 55 u64 *addrs = (u64 *)(uintptr_t)(info_linear->info.jited_ksyms); 56 u64 addr = addrs[i]; 57 struct map *map = maps__find(machine__kernel_maps(machine), addr); 58 59 if (map) { 60 struct dso *dso = map__dso(map); 61 62 dso->binary_type = DSO_BINARY_TYPE__BPF_PROG_INFO; 63 dso->bpf_prog.id = id; 64 dso->bpf_prog.sub_id = i; 65 dso->bpf_prog.env = env; 66 } 67 } 68 return 0; 69 } 70 71 int machine__process_bpf(struct machine *machine, union perf_event *event, 72 struct perf_sample *sample) 73 { 74 if (dump_trace) 75 perf_event__fprintf_bpf(event, stdout); 76 77 switch (event->bpf.type) { 78 case PERF_BPF_EVENT_PROG_LOAD: 79 return machine__process_bpf_event_load(machine, event, sample); 80 81 case PERF_BPF_EVENT_PROG_UNLOAD: 82 /* 83 * Do not free bpf_prog_info and btf of the program here, 84 * as annotation still need them. They will be freed at 85 * the end of the session. 86 */ 87 break; 88 default: 89 pr_debug("unexpected bpf event type of %d\n", event->bpf.type); 90 break; 91 } 92 return 0; 93 } 94 95 static int perf_env__fetch_btf(struct perf_env *env, 96 u32 btf_id, 97 struct btf *btf) 98 { 99 struct btf_node *node; 100 u32 data_size; 101 const void *data; 102 103 data = btf__raw_data(btf, &data_size); 104 105 node = malloc(data_size + sizeof(struct btf_node)); 106 if (!node) 107 return -1; 108 109 node->id = btf_id; 110 node->data_size = data_size; 111 memcpy(node->data, data, data_size); 112 113 if (!perf_env__insert_btf(env, node)) { 114 /* Insertion failed because of a duplicate. */ 115 free(node); 116 return -1; 117 } 118 return 0; 119 } 120 121 static int synthesize_bpf_prog_name(char *buf, int size, 122 struct bpf_prog_info *info, 123 struct btf *btf, 124 u32 sub_id) 125 { 126 u8 (*prog_tags)[BPF_TAG_SIZE] = (void *)(uintptr_t)(info->prog_tags); 127 void *func_infos = (void *)(uintptr_t)(info->func_info); 128 u32 sub_prog_cnt = info->nr_jited_ksyms; 129 const struct bpf_func_info *finfo; 130 const char *short_name = NULL; 131 const struct btf_type *t; 132 int name_len; 133 134 name_len = snprintf(buf, size, "bpf_prog_"); 135 name_len += snprintf_hex(buf + name_len, size - name_len, 136 prog_tags[sub_id], BPF_TAG_SIZE); 137 if (btf) { 138 finfo = func_infos + sub_id * info->func_info_rec_size; 139 t = btf__type_by_id(btf, finfo->type_id); 140 short_name = btf__name_by_offset(btf, t->name_off); 141 } else if (sub_id == 0 && sub_prog_cnt == 1) { 142 /* no subprog */ 143 if (info->name[0]) 144 short_name = info->name; 145 } else 146 short_name = "F"; 147 if (short_name) 148 name_len += snprintf(buf + name_len, size - name_len, 149 "_%s", short_name); 150 return name_len; 151 } 152 153 /* 154 * Synthesize PERF_RECORD_KSYMBOL and PERF_RECORD_BPF_EVENT for one bpf 155 * program. One PERF_RECORD_BPF_EVENT is generated for the program. And 156 * one PERF_RECORD_KSYMBOL is generated for each sub program. 157 * 158 * Returns: 159 * 0 for success; 160 * -1 for failures; 161 * -2 for lack of kernel support. 162 */ 163 static int perf_event__synthesize_one_bpf_prog(struct perf_session *session, 164 perf_event__handler_t process, 165 struct machine *machine, 166 int fd, 167 union perf_event *event, 168 struct record_opts *opts) 169 { 170 struct perf_record_ksymbol *ksymbol_event = &event->ksymbol; 171 struct perf_record_bpf_event *bpf_event = &event->bpf; 172 struct perf_tool *tool = session->tool; 173 struct bpf_prog_info_node *info_node; 174 struct perf_bpil *info_linear; 175 struct bpf_prog_info *info; 176 struct btf *btf = NULL; 177 struct perf_env *env; 178 u32 sub_prog_cnt, i; 179 int err = 0; 180 u64 arrays; 181 182 /* 183 * for perf-record and perf-report use header.env; 184 * otherwise, use global perf_env. 185 */ 186 env = session->data ? &session->header.env : &perf_env; 187 188 arrays = 1UL << PERF_BPIL_JITED_KSYMS; 189 arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS; 190 arrays |= 1UL << PERF_BPIL_FUNC_INFO; 191 arrays |= 1UL << PERF_BPIL_PROG_TAGS; 192 arrays |= 1UL << PERF_BPIL_JITED_INSNS; 193 arrays |= 1UL << PERF_BPIL_LINE_INFO; 194 arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO; 195 196 info_linear = get_bpf_prog_info_linear(fd, arrays); 197 if (IS_ERR_OR_NULL(info_linear)) { 198 info_linear = NULL; 199 pr_debug("%s: failed to get BPF program info. aborting\n", __func__); 200 return -1; 201 } 202 203 if (info_linear->info_len < offsetof(struct bpf_prog_info, prog_tags)) { 204 free(info_linear); 205 pr_debug("%s: the kernel is too old, aborting\n", __func__); 206 return -2; 207 } 208 209 info = &info_linear->info; 210 if (!info->jited_ksyms) { 211 free(info_linear); 212 return -1; 213 } 214 215 /* number of ksyms, func_lengths, and tags should match */ 216 sub_prog_cnt = info->nr_jited_ksyms; 217 if (sub_prog_cnt != info->nr_prog_tags || 218 sub_prog_cnt != info->nr_jited_func_lens) { 219 free(info_linear); 220 return -1; 221 } 222 223 /* check BTF func info support */ 224 if (info->btf_id && info->nr_func_info && info->func_info_rec_size) { 225 /* btf func info number should be same as sub_prog_cnt */ 226 if (sub_prog_cnt != info->nr_func_info) { 227 pr_debug("%s: mismatch in BPF sub program count and BTF function info count, aborting\n", __func__); 228 free(info_linear); 229 return -1; 230 } 231 btf = btf__load_from_kernel_by_id(info->btf_id); 232 if (libbpf_get_error(btf)) { 233 pr_debug("%s: failed to get BTF of id %u, aborting\n", __func__, info->btf_id); 234 err = -1; 235 goto out; 236 } 237 perf_env__fetch_btf(env, info->btf_id, btf); 238 } 239 240 /* Synthesize PERF_RECORD_KSYMBOL */ 241 for (i = 0; i < sub_prog_cnt; i++) { 242 __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); 243 __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); 244 int name_len; 245 246 *ksymbol_event = (struct perf_record_ksymbol) { 247 .header = { 248 .type = PERF_RECORD_KSYMBOL, 249 .size = offsetof(struct perf_record_ksymbol, name), 250 }, 251 .addr = prog_addrs[i], 252 .len = prog_lens[i], 253 .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF, 254 .flags = 0, 255 }; 256 257 name_len = synthesize_bpf_prog_name(ksymbol_event->name, 258 KSYM_NAME_LEN, info, btf, i); 259 ksymbol_event->header.size += PERF_ALIGN(name_len + 1, 260 sizeof(u64)); 261 262 memset((void *)event + event->header.size, 0, machine->id_hdr_size); 263 event->header.size += machine->id_hdr_size; 264 err = perf_tool__process_synth_event(tool, event, 265 machine, process); 266 } 267 268 if (!opts->no_bpf_event) { 269 /* Synthesize PERF_RECORD_BPF_EVENT */ 270 *bpf_event = (struct perf_record_bpf_event) { 271 .header = { 272 .type = PERF_RECORD_BPF_EVENT, 273 .size = sizeof(struct perf_record_bpf_event), 274 }, 275 .type = PERF_BPF_EVENT_PROG_LOAD, 276 .flags = 0, 277 .id = info->id, 278 }; 279 memcpy(bpf_event->tag, info->tag, BPF_TAG_SIZE); 280 memset((void *)event + event->header.size, 0, machine->id_hdr_size); 281 event->header.size += machine->id_hdr_size; 282 283 /* save bpf_prog_info to env */ 284 info_node = malloc(sizeof(struct bpf_prog_info_node)); 285 if (!info_node) { 286 err = -1; 287 goto out; 288 } 289 290 info_node->info_linear = info_linear; 291 perf_env__insert_bpf_prog_info(env, info_node); 292 info_linear = NULL; 293 294 /* 295 * process after saving bpf_prog_info to env, so that 296 * required information is ready for look up 297 */ 298 err = perf_tool__process_synth_event(tool, event, 299 machine, process); 300 } 301 302 out: 303 free(info_linear); 304 btf__free(btf); 305 return err ? -1 : 0; 306 } 307 308 struct kallsyms_parse { 309 union perf_event *event; 310 perf_event__handler_t process; 311 struct machine *machine; 312 struct perf_tool *tool; 313 }; 314 315 static int 316 process_bpf_image(char *name, u64 addr, struct kallsyms_parse *data) 317 { 318 struct machine *machine = data->machine; 319 union perf_event *event = data->event; 320 struct perf_record_ksymbol *ksymbol; 321 int len; 322 323 ksymbol = &event->ksymbol; 324 325 *ksymbol = (struct perf_record_ksymbol) { 326 .header = { 327 .type = PERF_RECORD_KSYMBOL, 328 .size = offsetof(struct perf_record_ksymbol, name), 329 }, 330 .addr = addr, 331 .len = page_size, 332 .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF, 333 .flags = 0, 334 }; 335 336 len = scnprintf(ksymbol->name, KSYM_NAME_LEN, "%s", name); 337 ksymbol->header.size += PERF_ALIGN(len + 1, sizeof(u64)); 338 memset((void *) event + event->header.size, 0, machine->id_hdr_size); 339 event->header.size += machine->id_hdr_size; 340 341 return perf_tool__process_synth_event(data->tool, event, machine, 342 data->process); 343 } 344 345 static int 346 kallsyms_process_symbol(void *data, const char *_name, 347 char type __maybe_unused, u64 start) 348 { 349 char disp[KSYM_NAME_LEN]; 350 char *module, *name; 351 unsigned long id; 352 int err = 0; 353 354 module = strchr(_name, '\t'); 355 if (!module) 356 return 0; 357 358 /* We are going after [bpf] module ... */ 359 if (strcmp(module + 1, "[bpf]")) 360 return 0; 361 362 name = memdup(_name, (module - _name) + 1); 363 if (!name) 364 return -ENOMEM; 365 366 name[module - _name] = 0; 367 368 /* .. and only for trampolines and dispatchers */ 369 if ((sscanf(name, "bpf_trampoline_%lu", &id) == 1) || 370 (sscanf(name, "bpf_dispatcher_%s", disp) == 1)) 371 err = process_bpf_image(name, start, data); 372 373 free(name); 374 return err; 375 } 376 377 int perf_event__synthesize_bpf_events(struct perf_session *session, 378 perf_event__handler_t process, 379 struct machine *machine, 380 struct record_opts *opts) 381 { 382 const char *kallsyms_filename = "/proc/kallsyms"; 383 struct kallsyms_parse arg; 384 union perf_event *event; 385 __u32 id = 0; 386 int err; 387 int fd; 388 389 event = malloc(sizeof(event->bpf) + KSYM_NAME_LEN + machine->id_hdr_size); 390 if (!event) 391 return -1; 392 393 /* Synthesize all the bpf programs in system. */ 394 while (true) { 395 err = bpf_prog_get_next_id(id, &id); 396 if (err) { 397 if (errno == ENOENT) { 398 err = 0; 399 break; 400 } 401 pr_debug("%s: can't get next program: %s%s\n", 402 __func__, strerror(errno), 403 errno == EINVAL ? " -- kernel too old?" : ""); 404 /* don't report error on old kernel or EPERM */ 405 err = (errno == EINVAL || errno == EPERM) ? 0 : -1; 406 break; 407 } 408 fd = bpf_prog_get_fd_by_id(id); 409 if (fd < 0) { 410 pr_debug("%s: failed to get fd for prog_id %u\n", 411 __func__, id); 412 continue; 413 } 414 415 err = perf_event__synthesize_one_bpf_prog(session, process, 416 machine, fd, 417 event, opts); 418 close(fd); 419 if (err) { 420 /* do not return error for old kernel */ 421 if (err == -2) 422 err = 0; 423 break; 424 } 425 } 426 427 /* Synthesize all the bpf images - trampolines/dispatchers. */ 428 if (symbol_conf.kallsyms_name != NULL) 429 kallsyms_filename = symbol_conf.kallsyms_name; 430 431 arg = (struct kallsyms_parse) { 432 .event = event, 433 .process = process, 434 .machine = machine, 435 .tool = session->tool, 436 }; 437 438 if (kallsyms__parse(kallsyms_filename, &arg, kallsyms_process_symbol)) { 439 pr_err("%s: failed to synthesize bpf images: %s\n", 440 __func__, strerror(errno)); 441 } 442 443 free(event); 444 return err; 445 } 446 447 static void perf_env__add_bpf_info(struct perf_env *env, u32 id) 448 { 449 struct bpf_prog_info_node *info_node; 450 struct perf_bpil *info_linear; 451 struct btf *btf = NULL; 452 u64 arrays; 453 u32 btf_id; 454 int fd; 455 456 fd = bpf_prog_get_fd_by_id(id); 457 if (fd < 0) 458 return; 459 460 arrays = 1UL << PERF_BPIL_JITED_KSYMS; 461 arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS; 462 arrays |= 1UL << PERF_BPIL_FUNC_INFO; 463 arrays |= 1UL << PERF_BPIL_PROG_TAGS; 464 arrays |= 1UL << PERF_BPIL_JITED_INSNS; 465 arrays |= 1UL << PERF_BPIL_LINE_INFO; 466 arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO; 467 468 info_linear = get_bpf_prog_info_linear(fd, arrays); 469 if (IS_ERR_OR_NULL(info_linear)) { 470 pr_debug("%s: failed to get BPF program info. aborting\n", __func__); 471 goto out; 472 } 473 474 btf_id = info_linear->info.btf_id; 475 476 info_node = malloc(sizeof(struct bpf_prog_info_node)); 477 if (info_node) { 478 info_node->info_linear = info_linear; 479 perf_env__insert_bpf_prog_info(env, info_node); 480 } else 481 free(info_linear); 482 483 if (btf_id == 0) 484 goto out; 485 486 btf = btf__load_from_kernel_by_id(btf_id); 487 if (libbpf_get_error(btf)) { 488 pr_debug("%s: failed to get BTF of id %u, aborting\n", 489 __func__, btf_id); 490 goto out; 491 } 492 perf_env__fetch_btf(env, btf_id, btf); 493 494 out: 495 btf__free(btf); 496 close(fd); 497 } 498 499 static int bpf_event__sb_cb(union perf_event *event, void *data) 500 { 501 struct perf_env *env = data; 502 503 if (event->header.type != PERF_RECORD_BPF_EVENT) 504 return -1; 505 506 switch (event->bpf.type) { 507 case PERF_BPF_EVENT_PROG_LOAD: 508 perf_env__add_bpf_info(env, event->bpf.id); 509 510 case PERF_BPF_EVENT_PROG_UNLOAD: 511 /* 512 * Do not free bpf_prog_info and btf of the program here, 513 * as annotation still need them. They will be freed at 514 * the end of the session. 515 */ 516 break; 517 default: 518 pr_debug("unexpected bpf event type of %d\n", event->bpf.type); 519 break; 520 } 521 522 return 0; 523 } 524 525 int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env) 526 { 527 struct perf_event_attr attr = { 528 .type = PERF_TYPE_SOFTWARE, 529 .config = PERF_COUNT_SW_DUMMY, 530 .sample_id_all = 1, 531 .watermark = 1, 532 .bpf_event = 1, 533 .size = sizeof(attr), /* to capture ABI version */ 534 }; 535 536 /* 537 * Older gcc versions don't support designated initializers, like above, 538 * for unnamed union members, such as the following: 539 */ 540 attr.wakeup_watermark = 1; 541 542 return evlist__add_sb_event(evlist, &attr, bpf_event__sb_cb, env); 543 } 544 545 void __bpf_event__print_bpf_prog_info(struct bpf_prog_info *info, 546 struct perf_env *env, 547 FILE *fp) 548 { 549 __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); 550 __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); 551 char name[KSYM_NAME_LEN]; 552 struct btf *btf = NULL; 553 u32 sub_prog_cnt, i; 554 555 sub_prog_cnt = info->nr_jited_ksyms; 556 if (sub_prog_cnt != info->nr_prog_tags || 557 sub_prog_cnt != info->nr_jited_func_lens) 558 return; 559 560 if (info->btf_id) { 561 struct btf_node *node; 562 563 node = __perf_env__find_btf(env, info->btf_id); 564 if (node) 565 btf = btf__new((__u8 *)(node->data), 566 node->data_size); 567 } 568 569 if (sub_prog_cnt == 1) { 570 synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, 0); 571 fprintf(fp, "# bpf_prog_info %u: %s addr 0x%llx size %u\n", 572 info->id, name, prog_addrs[0], prog_lens[0]); 573 goto out; 574 } 575 576 fprintf(fp, "# bpf_prog_info %u:\n", info->id); 577 for (i = 0; i < sub_prog_cnt; i++) { 578 synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, i); 579 580 fprintf(fp, "# \tsub_prog %u: %s addr 0x%llx size %u\n", 581 i, name, prog_addrs[i], prog_lens[i]); 582 } 583 out: 584 btf__free(btf); 585 } 586