1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 /* Copyright (c) 2018 Facebook */ 3 4 #include <stdlib.h> 5 #include <string.h> 6 #include <unistd.h> 7 #include <errno.h> 8 #include <linux/err.h> 9 #include <linux/btf.h> 10 #include "btf.h" 11 #include "bpf.h" 12 13 #define elog(fmt, ...) { if (err_log) err_log(fmt, ##__VA_ARGS__); } 14 #define max(a, b) ((a) > (b) ? (a) : (b)) 15 #define min(a, b) ((a) < (b) ? (a) : (b)) 16 17 #define BTF_MAX_NR_TYPES 65535 18 19 #define IS_MODIFIER(k) (((k) == BTF_KIND_TYPEDEF) || \ 20 ((k) == BTF_KIND_VOLATILE) || \ 21 ((k) == BTF_KIND_CONST) || \ 22 ((k) == BTF_KIND_RESTRICT)) 23 24 static struct btf_type btf_void; 25 26 struct btf { 27 union { 28 struct btf_header *hdr; 29 void *data; 30 }; 31 struct btf_type **types; 32 const char *strings; 33 void *nohdr_data; 34 __u32 nr_types; 35 __u32 types_size; 36 __u32 data_size; 37 int fd; 38 }; 39 40 struct btf_ext_info { 41 /* 42 * info points to a deep copy of the individual info section 43 * (e.g. func_info and line_info) from the .BTF.ext. 44 * It does not include the __u32 rec_size. 45 */ 46 void *info; 47 __u32 rec_size; 48 __u32 len; 49 }; 50 51 struct btf_ext { 52 struct btf_ext_info func_info; 53 struct btf_ext_info line_info; 54 }; 55 56 struct btf_ext_info_sec { 57 __u32 sec_name_off; 58 __u32 num_info; 59 /* Followed by num_info * record_size number of bytes */ 60 __u8 data[0]; 61 }; 62 63 /* The minimum bpf_func_info checked by the loader */ 64 struct bpf_func_info_min { 65 __u32 insn_off; 66 __u32 type_id; 67 }; 68 69 /* The minimum bpf_line_info checked by the loader */ 70 struct bpf_line_info_min { 71 __u32 insn_off; 72 __u32 file_name_off; 73 __u32 line_off; 74 __u32 line_col; 75 }; 76 77 static inline __u64 ptr_to_u64(const void *ptr) 78 { 79 return (__u64) (unsigned long) ptr; 80 } 81 82 static int btf_add_type(struct btf *btf, struct btf_type *t) 83 { 84 if (btf->types_size - btf->nr_types < 2) { 85 struct btf_type **new_types; 86 __u32 expand_by, new_size; 87 88 if (btf->types_size == BTF_MAX_NR_TYPES) 89 return -E2BIG; 90 91 expand_by = max(btf->types_size >> 2, 16); 92 new_size = min(BTF_MAX_NR_TYPES, btf->types_size + expand_by); 93 94 new_types = realloc(btf->types, sizeof(*new_types) * new_size); 95 if (!new_types) 96 return -ENOMEM; 97 98 if (btf->nr_types == 0) 99 new_types[0] = &btf_void; 100 101 btf->types = new_types; 102 btf->types_size = new_size; 103 } 104 105 btf->types[++(btf->nr_types)] = t; 106 107 return 0; 108 } 109 110 static int btf_parse_hdr(struct btf *btf, btf_print_fn_t err_log) 111 { 112 const struct btf_header *hdr = btf->hdr; 113 __u32 meta_left; 114 115 if (btf->data_size < sizeof(struct btf_header)) { 116 elog("BTF header not found\n"); 117 return -EINVAL; 118 } 119 120 if (hdr->magic != BTF_MAGIC) { 121 elog("Invalid BTF magic:%x\n", hdr->magic); 122 return -EINVAL; 123 } 124 125 if (hdr->version != BTF_VERSION) { 126 elog("Unsupported BTF version:%u\n", hdr->version); 127 return -ENOTSUP; 128 } 129 130 if (hdr->flags) { 131 elog("Unsupported BTF flags:%x\n", hdr->flags); 132 return -ENOTSUP; 133 } 134 135 meta_left = btf->data_size - sizeof(*hdr); 136 if (!meta_left) { 137 elog("BTF has no data\n"); 138 return -EINVAL; 139 } 140 141 if (meta_left < hdr->type_off) { 142 elog("Invalid BTF type section offset:%u\n", hdr->type_off); 143 return -EINVAL; 144 } 145 146 if (meta_left < hdr->str_off) { 147 elog("Invalid BTF string section offset:%u\n", hdr->str_off); 148 return -EINVAL; 149 } 150 151 if (hdr->type_off >= hdr->str_off) { 152 elog("BTF type section offset >= string section offset. No type?\n"); 153 return -EINVAL; 154 } 155 156 if (hdr->type_off & 0x02) { 157 elog("BTF type section is not aligned to 4 bytes\n"); 158 return -EINVAL; 159 } 160 161 btf->nohdr_data = btf->hdr + 1; 162 163 return 0; 164 } 165 166 static int btf_parse_str_sec(struct btf *btf, btf_print_fn_t err_log) 167 { 168 const struct btf_header *hdr = btf->hdr; 169 const char *start = btf->nohdr_data + hdr->str_off; 170 const char *end = start + btf->hdr->str_len; 171 172 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_NAME_OFFSET || 173 start[0] || end[-1]) { 174 elog("Invalid BTF string section\n"); 175 return -EINVAL; 176 } 177 178 btf->strings = start; 179 180 return 0; 181 } 182 183 static int btf_parse_type_sec(struct btf *btf, btf_print_fn_t err_log) 184 { 185 struct btf_header *hdr = btf->hdr; 186 void *nohdr_data = btf->nohdr_data; 187 void *next_type = nohdr_data + hdr->type_off; 188 void *end_type = nohdr_data + hdr->str_off; 189 190 while (next_type < end_type) { 191 struct btf_type *t = next_type; 192 __u16 vlen = BTF_INFO_VLEN(t->info); 193 int err; 194 195 next_type += sizeof(*t); 196 switch (BTF_INFO_KIND(t->info)) { 197 case BTF_KIND_INT: 198 next_type += sizeof(int); 199 break; 200 case BTF_KIND_ARRAY: 201 next_type += sizeof(struct btf_array); 202 break; 203 case BTF_KIND_STRUCT: 204 case BTF_KIND_UNION: 205 next_type += vlen * sizeof(struct btf_member); 206 break; 207 case BTF_KIND_ENUM: 208 next_type += vlen * sizeof(struct btf_enum); 209 break; 210 case BTF_KIND_FUNC_PROTO: 211 next_type += vlen * sizeof(struct btf_param); 212 break; 213 case BTF_KIND_FUNC: 214 case BTF_KIND_TYPEDEF: 215 case BTF_KIND_PTR: 216 case BTF_KIND_FWD: 217 case BTF_KIND_VOLATILE: 218 case BTF_KIND_CONST: 219 case BTF_KIND_RESTRICT: 220 break; 221 default: 222 elog("Unsupported BTF_KIND:%u\n", 223 BTF_INFO_KIND(t->info)); 224 return -EINVAL; 225 } 226 227 err = btf_add_type(btf, t); 228 if (err) 229 return err; 230 } 231 232 return 0; 233 } 234 235 const struct btf_type *btf__type_by_id(const struct btf *btf, __u32 type_id) 236 { 237 if (type_id > btf->nr_types) 238 return NULL; 239 240 return btf->types[type_id]; 241 } 242 243 static bool btf_type_is_void(const struct btf_type *t) 244 { 245 return t == &btf_void || BTF_INFO_KIND(t->info) == BTF_KIND_FWD; 246 } 247 248 static bool btf_type_is_void_or_null(const struct btf_type *t) 249 { 250 return !t || btf_type_is_void(t); 251 } 252 253 static __s64 btf_type_size(const struct btf_type *t) 254 { 255 switch (BTF_INFO_KIND(t->info)) { 256 case BTF_KIND_INT: 257 case BTF_KIND_STRUCT: 258 case BTF_KIND_UNION: 259 case BTF_KIND_ENUM: 260 return t->size; 261 case BTF_KIND_PTR: 262 return sizeof(void *); 263 default: 264 return -EINVAL; 265 } 266 } 267 268 #define MAX_RESOLVE_DEPTH 32 269 270 __s64 btf__resolve_size(const struct btf *btf, __u32 type_id) 271 { 272 const struct btf_array *array; 273 const struct btf_type *t; 274 __u32 nelems = 1; 275 __s64 size = -1; 276 int i; 277 278 t = btf__type_by_id(btf, type_id); 279 for (i = 0; i < MAX_RESOLVE_DEPTH && !btf_type_is_void_or_null(t); 280 i++) { 281 size = btf_type_size(t); 282 if (size >= 0) 283 break; 284 285 switch (BTF_INFO_KIND(t->info)) { 286 case BTF_KIND_TYPEDEF: 287 case BTF_KIND_VOLATILE: 288 case BTF_KIND_CONST: 289 case BTF_KIND_RESTRICT: 290 type_id = t->type; 291 break; 292 case BTF_KIND_ARRAY: 293 array = (const struct btf_array *)(t + 1); 294 if (nelems && array->nelems > UINT32_MAX / nelems) 295 return -E2BIG; 296 nelems *= array->nelems; 297 type_id = array->type; 298 break; 299 default: 300 return -EINVAL; 301 } 302 303 t = btf__type_by_id(btf, type_id); 304 } 305 306 if (size < 0) 307 return -EINVAL; 308 309 if (nelems && size > UINT32_MAX / nelems) 310 return -E2BIG; 311 312 return nelems * size; 313 } 314 315 int btf__resolve_type(const struct btf *btf, __u32 type_id) 316 { 317 const struct btf_type *t; 318 int depth = 0; 319 320 t = btf__type_by_id(btf, type_id); 321 while (depth < MAX_RESOLVE_DEPTH && 322 !btf_type_is_void_or_null(t) && 323 IS_MODIFIER(BTF_INFO_KIND(t->info))) { 324 type_id = t->type; 325 t = btf__type_by_id(btf, type_id); 326 depth++; 327 } 328 329 if (depth == MAX_RESOLVE_DEPTH || btf_type_is_void_or_null(t)) 330 return -EINVAL; 331 332 return type_id; 333 } 334 335 __s32 btf__find_by_name(const struct btf *btf, const char *type_name) 336 { 337 __u32 i; 338 339 if (!strcmp(type_name, "void")) 340 return 0; 341 342 for (i = 1; i <= btf->nr_types; i++) { 343 const struct btf_type *t = btf->types[i]; 344 const char *name = btf__name_by_offset(btf, t->name_off); 345 346 if (name && !strcmp(type_name, name)) 347 return i; 348 } 349 350 return -ENOENT; 351 } 352 353 void btf__free(struct btf *btf) 354 { 355 if (!btf) 356 return; 357 358 if (btf->fd != -1) 359 close(btf->fd); 360 361 free(btf->data); 362 free(btf->types); 363 free(btf); 364 } 365 366 struct btf *btf__new(__u8 *data, __u32 size, btf_print_fn_t err_log) 367 { 368 __u32 log_buf_size = 0; 369 char *log_buf = NULL; 370 struct btf *btf; 371 int err; 372 373 btf = calloc(1, sizeof(struct btf)); 374 if (!btf) 375 return ERR_PTR(-ENOMEM); 376 377 btf->fd = -1; 378 379 if (err_log) { 380 log_buf = malloc(BPF_LOG_BUF_SIZE); 381 if (!log_buf) { 382 err = -ENOMEM; 383 goto done; 384 } 385 *log_buf = 0; 386 log_buf_size = BPF_LOG_BUF_SIZE; 387 } 388 389 btf->data = malloc(size); 390 if (!btf->data) { 391 err = -ENOMEM; 392 goto done; 393 } 394 395 memcpy(btf->data, data, size); 396 btf->data_size = size; 397 398 btf->fd = bpf_load_btf(btf->data, btf->data_size, 399 log_buf, log_buf_size, false); 400 401 if (btf->fd == -1) { 402 err = -errno; 403 elog("Error loading BTF: %s(%d)\n", strerror(errno), errno); 404 if (log_buf && *log_buf) 405 elog("%s\n", log_buf); 406 goto done; 407 } 408 409 err = btf_parse_hdr(btf, err_log); 410 if (err) 411 goto done; 412 413 err = btf_parse_str_sec(btf, err_log); 414 if (err) 415 goto done; 416 417 err = btf_parse_type_sec(btf, err_log); 418 419 done: 420 free(log_buf); 421 422 if (err) { 423 btf__free(btf); 424 return ERR_PTR(err); 425 } 426 427 return btf; 428 } 429 430 int btf__fd(const struct btf *btf) 431 { 432 return btf->fd; 433 } 434 435 const char *btf__name_by_offset(const struct btf *btf, __u32 offset) 436 { 437 if (offset < btf->hdr->str_len) 438 return &btf->strings[offset]; 439 else 440 return NULL; 441 } 442 443 int btf__get_from_id(__u32 id, struct btf **btf) 444 { 445 struct bpf_btf_info btf_info = { 0 }; 446 __u32 len = sizeof(btf_info); 447 __u32 last_size; 448 int btf_fd; 449 void *ptr; 450 int err; 451 452 err = 0; 453 *btf = NULL; 454 btf_fd = bpf_btf_get_fd_by_id(id); 455 if (btf_fd < 0) 456 return 0; 457 458 /* we won't know btf_size until we call bpf_obj_get_info_by_fd(). so 459 * let's start with a sane default - 4KiB here - and resize it only if 460 * bpf_obj_get_info_by_fd() needs a bigger buffer. 461 */ 462 btf_info.btf_size = 4096; 463 last_size = btf_info.btf_size; 464 ptr = malloc(last_size); 465 if (!ptr) { 466 err = -ENOMEM; 467 goto exit_free; 468 } 469 470 bzero(ptr, last_size); 471 btf_info.btf = ptr_to_u64(ptr); 472 err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len); 473 474 if (!err && btf_info.btf_size > last_size) { 475 void *temp_ptr; 476 477 last_size = btf_info.btf_size; 478 temp_ptr = realloc(ptr, last_size); 479 if (!temp_ptr) { 480 err = -ENOMEM; 481 goto exit_free; 482 } 483 ptr = temp_ptr; 484 bzero(ptr, last_size); 485 btf_info.btf = ptr_to_u64(ptr); 486 err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len); 487 } 488 489 if (err || btf_info.btf_size > last_size) { 490 err = errno; 491 goto exit_free; 492 } 493 494 *btf = btf__new((__u8 *)(long)btf_info.btf, btf_info.btf_size, NULL); 495 if (IS_ERR(*btf)) { 496 err = PTR_ERR(*btf); 497 *btf = NULL; 498 } 499 500 exit_free: 501 close(btf_fd); 502 free(ptr); 503 504 return err; 505 } 506 507 struct btf_ext_sec_copy_param { 508 __u32 off; 509 __u32 len; 510 __u32 min_rec_size; 511 struct btf_ext_info *ext_info; 512 const char *desc; 513 }; 514 515 static int btf_ext_copy_info(struct btf_ext *btf_ext, 516 __u8 *data, __u32 data_size, 517 struct btf_ext_sec_copy_param *ext_sec, 518 btf_print_fn_t err_log) 519 { 520 const struct btf_ext_header *hdr = (struct btf_ext_header *)data; 521 const struct btf_ext_info_sec *sinfo; 522 struct btf_ext_info *ext_info; 523 __u32 info_left, record_size; 524 /* The start of the info sec (including the __u32 record_size). */ 525 const void *info; 526 527 /* data and data_size do not include btf_ext_header from now on */ 528 data = data + hdr->hdr_len; 529 data_size -= hdr->hdr_len; 530 531 if (ext_sec->off & 0x03) { 532 elog(".BTF.ext %s section is not aligned to 4 bytes\n", 533 ext_sec->desc); 534 return -EINVAL; 535 } 536 537 if (data_size < ext_sec->off || 538 ext_sec->len > data_size - ext_sec->off) { 539 elog("%s section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n", 540 ext_sec->desc, ext_sec->off, ext_sec->len); 541 return -EINVAL; 542 } 543 544 info = data + ext_sec->off; 545 info_left = ext_sec->len; 546 547 /* At least a record size */ 548 if (info_left < sizeof(__u32)) { 549 elog(".BTF.ext %s record size not found\n", ext_sec->desc); 550 return -EINVAL; 551 } 552 553 /* The record size needs to meet the minimum standard */ 554 record_size = *(__u32 *)info; 555 if (record_size < ext_sec->min_rec_size || 556 record_size & 0x03) { 557 elog("%s section in .BTF.ext has invalid record size %u\n", 558 ext_sec->desc, record_size); 559 return -EINVAL; 560 } 561 562 sinfo = info + sizeof(__u32); 563 info_left -= sizeof(__u32); 564 565 /* If no records, return failure now so .BTF.ext won't be used. */ 566 if (!info_left) { 567 elog("%s section in .BTF.ext has no records", ext_sec->desc); 568 return -EINVAL; 569 } 570 571 while (info_left) { 572 unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec); 573 __u64 total_record_size; 574 __u32 num_records; 575 576 if (info_left < sec_hdrlen) { 577 elog("%s section header is not found in .BTF.ext\n", 578 ext_sec->desc); 579 return -EINVAL; 580 } 581 582 num_records = sinfo->num_info; 583 if (num_records == 0) { 584 elog("%s section has incorrect num_records in .BTF.ext\n", 585 ext_sec->desc); 586 return -EINVAL; 587 } 588 589 total_record_size = sec_hdrlen + 590 (__u64)num_records * record_size; 591 if (info_left < total_record_size) { 592 elog("%s section has incorrect num_records in .BTF.ext\n", 593 ext_sec->desc); 594 return -EINVAL; 595 } 596 597 info_left -= total_record_size; 598 sinfo = (void *)sinfo + total_record_size; 599 } 600 601 ext_info = ext_sec->ext_info; 602 ext_info->len = ext_sec->len - sizeof(__u32); 603 ext_info->rec_size = record_size; 604 ext_info->info = malloc(ext_info->len); 605 if (!ext_info->info) 606 return -ENOMEM; 607 memcpy(ext_info->info, info + sizeof(__u32), ext_info->len); 608 609 return 0; 610 } 611 612 static int btf_ext_copy_func_info(struct btf_ext *btf_ext, 613 __u8 *data, __u32 data_size, 614 btf_print_fn_t err_log) 615 { 616 const struct btf_ext_header *hdr = (struct btf_ext_header *)data; 617 struct btf_ext_sec_copy_param param = { 618 .off = hdr->func_info_off, 619 .len = hdr->func_info_len, 620 .min_rec_size = sizeof(struct bpf_func_info_min), 621 .ext_info = &btf_ext->func_info, 622 .desc = "func_info" 623 }; 624 625 return btf_ext_copy_info(btf_ext, data, data_size, ¶m, err_log); 626 } 627 628 static int btf_ext_copy_line_info(struct btf_ext *btf_ext, 629 __u8 *data, __u32 data_size, 630 btf_print_fn_t err_log) 631 { 632 const struct btf_ext_header *hdr = (struct btf_ext_header *)data; 633 struct btf_ext_sec_copy_param param = { 634 .off = hdr->line_info_off, 635 .len = hdr->line_info_len, 636 .min_rec_size = sizeof(struct bpf_line_info_min), 637 .ext_info = &btf_ext->line_info, 638 .desc = "line_info", 639 }; 640 641 return btf_ext_copy_info(btf_ext, data, data_size, ¶m, err_log); 642 } 643 644 static int btf_ext_parse_hdr(__u8 *data, __u32 data_size, 645 btf_print_fn_t err_log) 646 { 647 const struct btf_ext_header *hdr = (struct btf_ext_header *)data; 648 649 if (data_size < offsetof(struct btf_ext_header, func_info_off) || 650 data_size < hdr->hdr_len) { 651 elog("BTF.ext header not found"); 652 return -EINVAL; 653 } 654 655 if (hdr->magic != BTF_MAGIC) { 656 elog("Invalid BTF.ext magic:%x\n", hdr->magic); 657 return -EINVAL; 658 } 659 660 if (hdr->version != BTF_VERSION) { 661 elog("Unsupported BTF.ext version:%u\n", hdr->version); 662 return -ENOTSUP; 663 } 664 665 if (hdr->flags) { 666 elog("Unsupported BTF.ext flags:%x\n", hdr->flags); 667 return -ENOTSUP; 668 } 669 670 if (data_size == hdr->hdr_len) { 671 elog("BTF.ext has no data\n"); 672 return -EINVAL; 673 } 674 675 return 0; 676 } 677 678 void btf_ext__free(struct btf_ext *btf_ext) 679 { 680 if (!btf_ext) 681 return; 682 683 free(btf_ext->func_info.info); 684 free(btf_ext->line_info.info); 685 free(btf_ext); 686 } 687 688 struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log) 689 { 690 struct btf_ext *btf_ext; 691 int err; 692 693 err = btf_ext_parse_hdr(data, size, err_log); 694 if (err) 695 return ERR_PTR(err); 696 697 btf_ext = calloc(1, sizeof(struct btf_ext)); 698 if (!btf_ext) 699 return ERR_PTR(-ENOMEM); 700 701 err = btf_ext_copy_func_info(btf_ext, data, size, err_log); 702 if (err) { 703 btf_ext__free(btf_ext); 704 return ERR_PTR(err); 705 } 706 707 err = btf_ext_copy_line_info(btf_ext, data, size, err_log); 708 if (err) { 709 btf_ext__free(btf_ext); 710 return ERR_PTR(err); 711 } 712 713 return btf_ext; 714 } 715 716 static int btf_ext_reloc_info(const struct btf *btf, 717 const struct btf_ext_info *ext_info, 718 const char *sec_name, __u32 insns_cnt, 719 void **info, __u32 *cnt) 720 { 721 __u32 sec_hdrlen = sizeof(struct btf_ext_info_sec); 722 __u32 i, record_size, existing_len, records_len; 723 struct btf_ext_info_sec *sinfo; 724 const char *info_sec_name; 725 __u64 remain_len; 726 void *data; 727 728 record_size = ext_info->rec_size; 729 sinfo = ext_info->info; 730 remain_len = ext_info->len; 731 while (remain_len > 0) { 732 records_len = sinfo->num_info * record_size; 733 info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off); 734 if (strcmp(info_sec_name, sec_name)) { 735 remain_len -= sec_hdrlen + records_len; 736 sinfo = (void *)sinfo + sec_hdrlen + records_len; 737 continue; 738 } 739 740 existing_len = (*cnt) * record_size; 741 data = realloc(*info, existing_len + records_len); 742 if (!data) 743 return -ENOMEM; 744 745 memcpy(data + existing_len, sinfo->data, records_len); 746 /* adjust insn_off only, the rest data will be passed 747 * to the kernel. 748 */ 749 for (i = 0; i < sinfo->num_info; i++) { 750 __u32 *insn_off; 751 752 insn_off = data + existing_len + (i * record_size); 753 *insn_off = *insn_off / sizeof(struct bpf_insn) + 754 insns_cnt; 755 } 756 *info = data; 757 *cnt += sinfo->num_info; 758 return 0; 759 } 760 761 return -ENOENT; 762 } 763 764 int btf_ext__reloc_func_info(const struct btf *btf, const struct btf_ext *btf_ext, 765 const char *sec_name, __u32 insns_cnt, 766 void **func_info, __u32 *cnt) 767 { 768 return btf_ext_reloc_info(btf, &btf_ext->func_info, sec_name, 769 insns_cnt, func_info, cnt); 770 } 771 772 int btf_ext__reloc_line_info(const struct btf *btf, const struct btf_ext *btf_ext, 773 const char *sec_name, __u32 insns_cnt, 774 void **line_info, __u32 *cnt) 775 { 776 return btf_ext_reloc_info(btf, &btf_ext->line_info, sec_name, 777 insns_cnt, line_info, cnt); 778 } 779 780 __u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext) 781 { 782 return btf_ext->func_info.rec_size; 783 } 784 785 __u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext) 786 { 787 return btf_ext->line_info.rec_size; 788 } 789