1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 /* Copyright (c) 2018 Facebook */ 3 4 #include <stdlib.h> 5 #include <string.h> 6 #include <unistd.h> 7 #include <errno.h> 8 #include <linux/err.h> 9 #include <linux/btf.h> 10 #include "btf.h" 11 #include "bpf.h" 12 13 #define elog(fmt, ...) { if (err_log) err_log(fmt, ##__VA_ARGS__); } 14 #define max(a, b) ((a) > (b) ? (a) : (b)) 15 #define min(a, b) ((a) < (b) ? (a) : (b)) 16 17 #define BTF_MAX_NR_TYPES 65535 18 19 #define IS_MODIFIER(k) (((k) == BTF_KIND_TYPEDEF) || \ 20 ((k) == BTF_KIND_VOLATILE) || \ 21 ((k) == BTF_KIND_CONST) || \ 22 ((k) == BTF_KIND_RESTRICT)) 23 24 static struct btf_type btf_void; 25 26 struct btf { 27 union { 28 struct btf_header *hdr; 29 void *data; 30 }; 31 struct btf_type **types; 32 const char *strings; 33 void *nohdr_data; 34 __u32 nr_types; 35 __u32 types_size; 36 __u32 data_size; 37 int fd; 38 }; 39 40 struct btf_ext { 41 void *func_info; 42 __u32 func_info_rec_size; 43 __u32 func_info_len; 44 }; 45 46 /* The minimum bpf_func_info checked by the loader */ 47 struct bpf_func_info_min { 48 __u32 insn_offset; 49 __u32 type_id; 50 }; 51 52 static inline __u64 ptr_to_u64(const void *ptr) 53 { 54 return (__u64) (unsigned long) ptr; 55 } 56 57 static int btf_add_type(struct btf *btf, struct btf_type *t) 58 { 59 if (btf->types_size - btf->nr_types < 2) { 60 struct btf_type **new_types; 61 __u32 expand_by, new_size; 62 63 if (btf->types_size == BTF_MAX_NR_TYPES) 64 return -E2BIG; 65 66 expand_by = max(btf->types_size >> 2, 16); 67 new_size = min(BTF_MAX_NR_TYPES, btf->types_size + expand_by); 68 69 new_types = realloc(btf->types, sizeof(*new_types) * new_size); 70 if (!new_types) 71 return -ENOMEM; 72 73 if (btf->nr_types == 0) 74 new_types[0] = &btf_void; 75 76 btf->types = new_types; 77 btf->types_size = new_size; 78 } 79 80 btf->types[++(btf->nr_types)] = t; 81 82 return 0; 83 } 84 85 static int btf_parse_hdr(struct btf *btf, btf_print_fn_t err_log) 86 { 87 const struct btf_header *hdr = btf->hdr; 88 __u32 meta_left; 89 90 if (btf->data_size < sizeof(struct btf_header)) { 91 elog("BTF header not found\n"); 92 return -EINVAL; 93 } 94 95 if (hdr->magic != BTF_MAGIC) { 96 elog("Invalid BTF magic:%x\n", hdr->magic); 97 return -EINVAL; 98 } 99 100 if (hdr->version != BTF_VERSION) { 101 elog("Unsupported BTF version:%u\n", hdr->version); 102 return -ENOTSUP; 103 } 104 105 if (hdr->flags) { 106 elog("Unsupported BTF flags:%x\n", hdr->flags); 107 return -ENOTSUP; 108 } 109 110 meta_left = btf->data_size - sizeof(*hdr); 111 if (!meta_left) { 112 elog("BTF has no data\n"); 113 return -EINVAL; 114 } 115 116 if (meta_left < hdr->type_off) { 117 elog("Invalid BTF type section offset:%u\n", hdr->type_off); 118 return -EINVAL; 119 } 120 121 if (meta_left < hdr->str_off) { 122 elog("Invalid BTF string section offset:%u\n", hdr->str_off); 123 return -EINVAL; 124 } 125 126 if (hdr->type_off >= hdr->str_off) { 127 elog("BTF type section offset >= string section offset. No type?\n"); 128 return -EINVAL; 129 } 130 131 if (hdr->type_off & 0x02) { 132 elog("BTF type section is not aligned to 4 bytes\n"); 133 return -EINVAL; 134 } 135 136 btf->nohdr_data = btf->hdr + 1; 137 138 return 0; 139 } 140 141 static int btf_parse_str_sec(struct btf *btf, btf_print_fn_t err_log) 142 { 143 const struct btf_header *hdr = btf->hdr; 144 const char *start = btf->nohdr_data + hdr->str_off; 145 const char *end = start + btf->hdr->str_len; 146 147 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_NAME_OFFSET || 148 start[0] || end[-1]) { 149 elog("Invalid BTF string section\n"); 150 return -EINVAL; 151 } 152 153 btf->strings = start; 154 155 return 0; 156 } 157 158 static int btf_parse_type_sec(struct btf *btf, btf_print_fn_t err_log) 159 { 160 struct btf_header *hdr = btf->hdr; 161 void *nohdr_data = btf->nohdr_data; 162 void *next_type = nohdr_data + hdr->type_off; 163 void *end_type = nohdr_data + hdr->str_off; 164 165 while (next_type < end_type) { 166 struct btf_type *t = next_type; 167 __u16 vlen = BTF_INFO_VLEN(t->info); 168 int err; 169 170 next_type += sizeof(*t); 171 switch (BTF_INFO_KIND(t->info)) { 172 case BTF_KIND_INT: 173 next_type += sizeof(int); 174 break; 175 case BTF_KIND_ARRAY: 176 next_type += sizeof(struct btf_array); 177 break; 178 case BTF_KIND_STRUCT: 179 case BTF_KIND_UNION: 180 next_type += vlen * sizeof(struct btf_member); 181 break; 182 case BTF_KIND_ENUM: 183 next_type += vlen * sizeof(struct btf_enum); 184 break; 185 case BTF_KIND_FUNC_PROTO: 186 next_type += vlen * sizeof(struct btf_param); 187 break; 188 case BTF_KIND_FUNC: 189 case BTF_KIND_TYPEDEF: 190 case BTF_KIND_PTR: 191 case BTF_KIND_FWD: 192 case BTF_KIND_VOLATILE: 193 case BTF_KIND_CONST: 194 case BTF_KIND_RESTRICT: 195 break; 196 default: 197 elog("Unsupported BTF_KIND:%u\n", 198 BTF_INFO_KIND(t->info)); 199 return -EINVAL; 200 } 201 202 err = btf_add_type(btf, t); 203 if (err) 204 return err; 205 } 206 207 return 0; 208 } 209 210 const struct btf_type *btf__type_by_id(const struct btf *btf, __u32 type_id) 211 { 212 if (type_id > btf->nr_types) 213 return NULL; 214 215 return btf->types[type_id]; 216 } 217 218 static bool btf_type_is_void(const struct btf_type *t) 219 { 220 return t == &btf_void || BTF_INFO_KIND(t->info) == BTF_KIND_FWD; 221 } 222 223 static bool btf_type_is_void_or_null(const struct btf_type *t) 224 { 225 return !t || btf_type_is_void(t); 226 } 227 228 static __s64 btf_type_size(const struct btf_type *t) 229 { 230 switch (BTF_INFO_KIND(t->info)) { 231 case BTF_KIND_INT: 232 case BTF_KIND_STRUCT: 233 case BTF_KIND_UNION: 234 case BTF_KIND_ENUM: 235 return t->size; 236 case BTF_KIND_PTR: 237 return sizeof(void *); 238 default: 239 return -EINVAL; 240 } 241 } 242 243 #define MAX_RESOLVE_DEPTH 32 244 245 __s64 btf__resolve_size(const struct btf *btf, __u32 type_id) 246 { 247 const struct btf_array *array; 248 const struct btf_type *t; 249 __u32 nelems = 1; 250 __s64 size = -1; 251 int i; 252 253 t = btf__type_by_id(btf, type_id); 254 for (i = 0; i < MAX_RESOLVE_DEPTH && !btf_type_is_void_or_null(t); 255 i++) { 256 size = btf_type_size(t); 257 if (size >= 0) 258 break; 259 260 switch (BTF_INFO_KIND(t->info)) { 261 case BTF_KIND_TYPEDEF: 262 case BTF_KIND_VOLATILE: 263 case BTF_KIND_CONST: 264 case BTF_KIND_RESTRICT: 265 type_id = t->type; 266 break; 267 case BTF_KIND_ARRAY: 268 array = (const struct btf_array *)(t + 1); 269 if (nelems && array->nelems > UINT32_MAX / nelems) 270 return -E2BIG; 271 nelems *= array->nelems; 272 type_id = array->type; 273 break; 274 default: 275 return -EINVAL; 276 } 277 278 t = btf__type_by_id(btf, type_id); 279 } 280 281 if (size < 0) 282 return -EINVAL; 283 284 if (nelems && size > UINT32_MAX / nelems) 285 return -E2BIG; 286 287 return nelems * size; 288 } 289 290 int btf__resolve_type(const struct btf *btf, __u32 type_id) 291 { 292 const struct btf_type *t; 293 int depth = 0; 294 295 t = btf__type_by_id(btf, type_id); 296 while (depth < MAX_RESOLVE_DEPTH && 297 !btf_type_is_void_or_null(t) && 298 IS_MODIFIER(BTF_INFO_KIND(t->info))) { 299 type_id = t->type; 300 t = btf__type_by_id(btf, type_id); 301 depth++; 302 } 303 304 if (depth == MAX_RESOLVE_DEPTH || btf_type_is_void_or_null(t)) 305 return -EINVAL; 306 307 return type_id; 308 } 309 310 __s32 btf__find_by_name(const struct btf *btf, const char *type_name) 311 { 312 __u32 i; 313 314 if (!strcmp(type_name, "void")) 315 return 0; 316 317 for (i = 1; i <= btf->nr_types; i++) { 318 const struct btf_type *t = btf->types[i]; 319 const char *name = btf__name_by_offset(btf, t->name_off); 320 321 if (name && !strcmp(type_name, name)) 322 return i; 323 } 324 325 return -ENOENT; 326 } 327 328 void btf__free(struct btf *btf) 329 { 330 if (!btf) 331 return; 332 333 if (btf->fd != -1) 334 close(btf->fd); 335 336 free(btf->data); 337 free(btf->types); 338 free(btf); 339 } 340 341 struct btf *btf__new(__u8 *data, __u32 size, btf_print_fn_t err_log) 342 { 343 __u32 log_buf_size = 0; 344 char *log_buf = NULL; 345 struct btf *btf; 346 int err; 347 348 btf = calloc(1, sizeof(struct btf)); 349 if (!btf) 350 return ERR_PTR(-ENOMEM); 351 352 btf->fd = -1; 353 354 if (err_log) { 355 log_buf = malloc(BPF_LOG_BUF_SIZE); 356 if (!log_buf) { 357 err = -ENOMEM; 358 goto done; 359 } 360 *log_buf = 0; 361 log_buf_size = BPF_LOG_BUF_SIZE; 362 } 363 364 btf->data = malloc(size); 365 if (!btf->data) { 366 err = -ENOMEM; 367 goto done; 368 } 369 370 memcpy(btf->data, data, size); 371 btf->data_size = size; 372 373 btf->fd = bpf_load_btf(btf->data, btf->data_size, 374 log_buf, log_buf_size, false); 375 376 if (btf->fd == -1) { 377 err = -errno; 378 elog("Error loading BTF: %s(%d)\n", strerror(errno), errno); 379 if (log_buf && *log_buf) 380 elog("%s\n", log_buf); 381 goto done; 382 } 383 384 err = btf_parse_hdr(btf, err_log); 385 if (err) 386 goto done; 387 388 err = btf_parse_str_sec(btf, err_log); 389 if (err) 390 goto done; 391 392 err = btf_parse_type_sec(btf, err_log); 393 394 done: 395 free(log_buf); 396 397 if (err) { 398 btf__free(btf); 399 return ERR_PTR(err); 400 } 401 402 return btf; 403 } 404 405 int btf__fd(const struct btf *btf) 406 { 407 return btf->fd; 408 } 409 410 const char *btf__name_by_offset(const struct btf *btf, __u32 offset) 411 { 412 if (offset < btf->hdr->str_len) 413 return &btf->strings[offset]; 414 else 415 return NULL; 416 } 417 418 int btf__get_from_id(__u32 id, struct btf **btf) 419 { 420 struct bpf_btf_info btf_info = { 0 }; 421 __u32 len = sizeof(btf_info); 422 __u32 last_size; 423 int btf_fd; 424 void *ptr; 425 int err; 426 427 err = 0; 428 *btf = NULL; 429 btf_fd = bpf_btf_get_fd_by_id(id); 430 if (btf_fd < 0) 431 return 0; 432 433 /* we won't know btf_size until we call bpf_obj_get_info_by_fd(). so 434 * let's start with a sane default - 4KiB here - and resize it only if 435 * bpf_obj_get_info_by_fd() needs a bigger buffer. 436 */ 437 btf_info.btf_size = 4096; 438 last_size = btf_info.btf_size; 439 ptr = malloc(last_size); 440 if (!ptr) { 441 err = -ENOMEM; 442 goto exit_free; 443 } 444 445 bzero(ptr, last_size); 446 btf_info.btf = ptr_to_u64(ptr); 447 err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len); 448 449 if (!err && btf_info.btf_size > last_size) { 450 void *temp_ptr; 451 452 last_size = btf_info.btf_size; 453 temp_ptr = realloc(ptr, last_size); 454 if (!temp_ptr) { 455 err = -ENOMEM; 456 goto exit_free; 457 } 458 ptr = temp_ptr; 459 bzero(ptr, last_size); 460 btf_info.btf = ptr_to_u64(ptr); 461 err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len); 462 } 463 464 if (err || btf_info.btf_size > last_size) { 465 err = errno; 466 goto exit_free; 467 } 468 469 *btf = btf__new((__u8 *)(long)btf_info.btf, btf_info.btf_size, NULL); 470 if (IS_ERR(*btf)) { 471 err = PTR_ERR(*btf); 472 *btf = NULL; 473 } 474 475 exit_free: 476 close(btf_fd); 477 free(ptr); 478 479 return err; 480 } 481 482 static int btf_ext_validate_func_info(const void *finfo, __u32 size, 483 btf_print_fn_t err_log) 484 { 485 int sec_hdrlen = sizeof(struct btf_sec_func_info); 486 __u32 size_left, num_records, record_size; 487 const struct btf_sec_func_info *sinfo; 488 __u64 total_record_size; 489 490 /* At least a func_info record size */ 491 if (size < sizeof(__u32)) { 492 elog("BTF.ext func_info record size not found"); 493 return -EINVAL; 494 } 495 496 /* The record size needs to meet below minimum standard */ 497 record_size = *(__u32 *)finfo; 498 if (record_size < sizeof(struct bpf_func_info_min) || 499 record_size % sizeof(__u32)) { 500 elog("BTF.ext func_info invalid record size"); 501 return -EINVAL; 502 } 503 504 sinfo = finfo + sizeof(__u32); 505 size_left = size - sizeof(__u32); 506 507 /* If no func_info records, return failure now so .BTF.ext 508 * won't be used. 509 */ 510 if (!size_left) { 511 elog("BTF.ext no func info records"); 512 return -EINVAL; 513 } 514 515 while (size_left) { 516 if (size_left < sec_hdrlen) { 517 elog("BTF.ext func_info header not found"); 518 return -EINVAL; 519 } 520 521 num_records = sinfo->num_func_info; 522 if (num_records == 0) { 523 elog("incorrect BTF.ext num_func_info"); 524 return -EINVAL; 525 } 526 527 total_record_size = sec_hdrlen + 528 (__u64)num_records * record_size; 529 if (size_left < total_record_size) { 530 elog("incorrect BTF.ext num_func_info"); 531 return -EINVAL; 532 } 533 534 size_left -= total_record_size; 535 sinfo = (void *)sinfo + total_record_size; 536 } 537 538 return 0; 539 } 540 541 static int btf_ext_parse_hdr(__u8 *data, __u32 data_size, 542 btf_print_fn_t err_log) 543 { 544 const struct btf_ext_header *hdr = (struct btf_ext_header *)data; 545 __u32 meta_left, last_func_info_pos; 546 void *finfo; 547 548 if (data_size < offsetof(struct btf_ext_header, func_info_off) || 549 data_size < hdr->hdr_len) { 550 elog("BTF.ext header not found"); 551 return -EINVAL; 552 } 553 554 if (hdr->magic != BTF_MAGIC) { 555 elog("Invalid BTF.ext magic:%x\n", hdr->magic); 556 return -EINVAL; 557 } 558 559 if (hdr->version != BTF_VERSION) { 560 elog("Unsupported BTF.ext version:%u\n", hdr->version); 561 return -ENOTSUP; 562 } 563 564 if (hdr->flags) { 565 elog("Unsupported BTF.ext flags:%x\n", hdr->flags); 566 return -ENOTSUP; 567 } 568 569 meta_left = data_size - hdr->hdr_len; 570 if (!meta_left) { 571 elog("BTF.ext has no data\n"); 572 return -EINVAL; 573 } 574 575 if (meta_left < hdr->func_info_off) { 576 elog("Invalid BTF.ext func_info section offset:%u\n", 577 hdr->func_info_off); 578 return -EINVAL; 579 } 580 581 if (hdr->func_info_off & 0x03) { 582 elog("BTF.ext func_info section is not aligned to 4 bytes\n"); 583 return -EINVAL; 584 } 585 586 last_func_info_pos = hdr->hdr_len + hdr->func_info_off + 587 hdr->func_info_len; 588 if (last_func_info_pos > data_size) { 589 elog("Invalid BTF.ext func_info section size:%u\n", 590 hdr->func_info_len); 591 return -EINVAL; 592 } 593 594 finfo = data + hdr->hdr_len + hdr->func_info_off; 595 return btf_ext_validate_func_info(finfo, hdr->func_info_len, 596 err_log); 597 } 598 599 void btf_ext__free(struct btf_ext *btf_ext) 600 { 601 if (!btf_ext) 602 return; 603 604 free(btf_ext->func_info); 605 free(btf_ext); 606 } 607 608 struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log) 609 { 610 const struct btf_ext_header *hdr; 611 struct btf_ext *btf_ext; 612 void *org_fdata, *fdata; 613 __u32 hdrlen, size_u32; 614 int err; 615 616 err = btf_ext_parse_hdr(data, size, err_log); 617 if (err) 618 return ERR_PTR(err); 619 620 btf_ext = calloc(1, sizeof(struct btf_ext)); 621 if (!btf_ext) 622 return ERR_PTR(-ENOMEM); 623 624 hdr = (const struct btf_ext_header *)data; 625 hdrlen = hdr->hdr_len; 626 size_u32 = sizeof(__u32); 627 fdata = malloc(hdr->func_info_len - size_u32); 628 if (!fdata) { 629 free(btf_ext); 630 return ERR_PTR(-ENOMEM); 631 } 632 633 /* remember record size and copy rest of func_info data */ 634 org_fdata = data + hdrlen + hdr->func_info_off; 635 btf_ext->func_info_rec_size = *(__u32 *)org_fdata; 636 memcpy(fdata, org_fdata + size_u32, hdr->func_info_len - size_u32); 637 btf_ext->func_info = fdata; 638 btf_ext->func_info_len = hdr->func_info_len - size_u32; 639 640 return btf_ext; 641 } 642 643 int btf_ext__reloc_init(struct btf *btf, struct btf_ext *btf_ext, 644 const char *sec_name, void **func_info, 645 __u32 *func_info_rec_size, __u32 *func_info_len) 646 { 647 __u32 sec_hdrlen = sizeof(struct btf_sec_func_info); 648 __u32 i, record_size, records_len; 649 struct btf_sec_func_info *sinfo; 650 const char *info_sec_name; 651 __s64 remain_len; 652 void *data; 653 654 record_size = btf_ext->func_info_rec_size; 655 sinfo = btf_ext->func_info; 656 remain_len = btf_ext->func_info_len; 657 658 while (remain_len > 0) { 659 records_len = sinfo->num_func_info * record_size; 660 info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off); 661 if (strcmp(info_sec_name, sec_name)) { 662 remain_len -= sec_hdrlen + records_len; 663 sinfo = (void *)sinfo + sec_hdrlen + records_len; 664 continue; 665 } 666 667 data = malloc(records_len); 668 if (!data) 669 return -ENOMEM; 670 671 memcpy(data, sinfo->data, records_len); 672 673 /* adjust the insn_offset, the data in .BTF.ext is 674 * the actual byte offset, and the kernel expects 675 * the offset in term of bpf_insn. 676 * 677 * adjust the insn offset only, the rest data will 678 * be passed to kernel. 679 */ 680 for (i = 0; i < sinfo->num_func_info; i++) { 681 struct bpf_func_info_min *record; 682 683 record = data + i * record_size; 684 record->insn_offset /= sizeof(struct bpf_insn); 685 } 686 687 *func_info = data; 688 *func_info_len = records_len; 689 *func_info_rec_size = record_size; 690 return 0; 691 } 692 693 return -EINVAL; 694 } 695 696 int btf_ext__reloc(struct btf *btf, struct btf_ext *btf_ext, 697 const char *sec_name, __u32 insns_cnt, 698 void **func_info, __u32 *func_info_len) 699 { 700 __u32 sec_hdrlen = sizeof(struct btf_sec_func_info); 701 __u32 i, record_size, existing_flen, records_len; 702 struct btf_sec_func_info *sinfo; 703 const char *info_sec_name; 704 __u64 remain_len; 705 void *data; 706 707 record_size = btf_ext->func_info_rec_size; 708 sinfo = btf_ext->func_info; 709 remain_len = btf_ext->func_info_len; 710 while (remain_len > 0) { 711 records_len = sinfo->num_func_info * record_size; 712 info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off); 713 if (strcmp(info_sec_name, sec_name)) { 714 remain_len -= sec_hdrlen + records_len; 715 sinfo = (void *)sinfo + sec_hdrlen + records_len; 716 continue; 717 } 718 719 existing_flen = *func_info_len; 720 data = realloc(*func_info, existing_flen + records_len); 721 if (!data) 722 return -ENOMEM; 723 724 memcpy(data + existing_flen, sinfo->data, records_len); 725 /* adjust insn_offset only, the rest data will be passed 726 * to the kernel. 727 */ 728 for (i = 0; i < sinfo->num_func_info; i++) { 729 struct bpf_func_info_min *record; 730 731 record = data + existing_flen + i * record_size; 732 record->insn_offset = 733 record->insn_offset / sizeof(struct bpf_insn) + 734 insns_cnt; 735 } 736 *func_info = data; 737 *func_info_len = existing_flen + records_len; 738 return 0; 739 } 740 741 return -EINVAL; 742 } 743