1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 /* 3 * BPF static linker 4 * 5 * Copyright (c) 2021 Facebook 6 */ 7 #include <stdbool.h> 8 #include <stddef.h> 9 #include <stdio.h> 10 #include <stdlib.h> 11 #include <string.h> 12 #include <unistd.h> 13 #include <errno.h> 14 #include <linux/err.h> 15 #include <linux/btf.h> 16 #include <elf.h> 17 #include <libelf.h> 18 #include <gelf.h> 19 #include <fcntl.h> 20 #include "libbpf.h" 21 #include "btf.h" 22 #include "libbpf_internal.h" 23 #include "strset.h" 24 25 struct src_sec { 26 const char *sec_name; 27 /* positional (not necessarily ELF) index in an array of sections */ 28 int id; 29 /* positional (not necessarily ELF) index of a matching section in a final object file */ 30 int dst_id; 31 /* section data offset in a matching output section */ 32 int dst_off; 33 /* whether section is omitted from the final ELF file */ 34 bool skipped; 35 /* whether section is an ephemeral section, not mapped to an ELF section */ 36 bool ephemeral; 37 38 /* ELF info */ 39 size_t sec_idx; 40 Elf_Scn *scn; 41 Elf64_Shdr *shdr; 42 Elf_Data *data; 43 44 /* corresponding BTF DATASEC type ID */ 45 int sec_type_id; 46 }; 47 48 struct src_obj { 49 const char *filename; 50 int fd; 51 Elf *elf; 52 /* Section header strings section index */ 53 size_t shstrs_sec_idx; 54 /* SYMTAB section index */ 55 size_t symtab_sec_idx; 56 57 struct btf *btf; 58 struct btf_ext *btf_ext; 59 60 /* List of sections (including ephemeral). Slot zero is unused. */ 61 struct src_sec *secs; 62 int sec_cnt; 63 64 /* mapping of symbol indices from src to dst ELF */ 65 int *sym_map; 66 /* mapping from the src BTF type IDs to dst ones */ 67 int *btf_type_map; 68 }; 69 70 /* single .BTF.ext data section */ 71 struct btf_ext_sec_data { 72 size_t rec_cnt; 73 __u32 rec_sz; 74 void *recs; 75 }; 76 77 struct dst_sec { 78 char *sec_name; 79 /* positional (not necessarily ELF) index in an array of sections */ 80 int id; 81 82 /* ELF info */ 83 size_t sec_idx; 84 Elf_Scn *scn; 85 Elf64_Shdr *shdr; 86 Elf_Data *data; 87 88 /* final output section size */ 89 int sec_sz; 90 /* final output contents of the section */ 91 void *raw_data; 92 93 /* corresponding STT_SECTION symbol index in SYMTAB */ 94 int sec_sym_idx; 95 96 /* section's DATASEC variable info, emitted on BTF finalization */ 97 bool has_btf; 98 int sec_var_cnt; 99 struct btf_var_secinfo *sec_vars; 100 101 /* section's .BTF.ext data */ 102 struct btf_ext_sec_data func_info; 103 struct btf_ext_sec_data line_info; 104 struct btf_ext_sec_data core_relo_info; 105 }; 106 107 struct bpf_linker { 108 char *filename; 109 int fd; 110 Elf *elf; 111 Elf64_Ehdr *elf_hdr; 112 113 /* Output sections metadata */ 114 struct dst_sec *secs; 115 int sec_cnt; 116 117 struct strset *strtab_strs; /* STRTAB unique strings */ 118 size_t strtab_sec_idx; /* STRTAB section index */ 119 size_t symtab_sec_idx; /* SYMTAB section index */ 120 121 struct btf *btf; 122 struct btf_ext *btf_ext; 123 }; 124 125 #define pr_warn_elf(fmt, ...) \ 126 do { \ 127 libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1)); \ 128 } while (0) 129 130 static int init_output_elf(struct bpf_linker *linker, const char *file); 131 132 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, struct src_obj *obj); 133 static int linker_sanity_check_elf(struct src_obj *obj); 134 static int linker_sanity_check_btf(struct src_obj *obj); 135 static int linker_sanity_check_btf_ext(struct src_obj *obj); 136 static int linker_fixup_btf(struct src_obj *obj); 137 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj); 138 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj); 139 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj); 140 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj); 141 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj); 142 143 static int finalize_btf(struct bpf_linker *linker); 144 static int finalize_btf_ext(struct bpf_linker *linker); 145 146 void bpf_linker__free(struct bpf_linker *linker) 147 { 148 int i; 149 150 if (!linker) 151 return; 152 153 free(linker->filename); 154 155 if (linker->elf) 156 elf_end(linker->elf); 157 158 if (linker->fd >= 0) 159 close(linker->fd); 160 161 strset__free(linker->strtab_strs); 162 163 btf__free(linker->btf); 164 btf_ext__free(linker->btf_ext); 165 166 for (i = 1; i < linker->sec_cnt; i++) { 167 struct dst_sec *sec = &linker->secs[i]; 168 169 free(sec->sec_name); 170 free(sec->raw_data); 171 free(sec->sec_vars); 172 173 free(sec->func_info.recs); 174 free(sec->line_info.recs); 175 free(sec->core_relo_info.recs); 176 } 177 free(linker->secs); 178 179 free(linker); 180 } 181 182 struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts) 183 { 184 struct bpf_linker *linker; 185 int err; 186 187 if (!OPTS_VALID(opts, bpf_linker_opts)) 188 return NULL; 189 190 if (elf_version(EV_CURRENT) == EV_NONE) { 191 pr_warn_elf("libelf initialization failed"); 192 return NULL; 193 } 194 195 linker = calloc(1, sizeof(*linker)); 196 if (!linker) 197 return NULL; 198 199 linker->fd = -1; 200 201 err = init_output_elf(linker, filename); 202 if (err) 203 goto err_out; 204 205 return linker; 206 207 err_out: 208 bpf_linker__free(linker); 209 return NULL; 210 } 211 212 static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name) 213 { 214 struct dst_sec *secs = linker->secs, *sec; 215 size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2; 216 217 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs)); 218 if (!secs) 219 return NULL; 220 221 /* zero out newly allocated memory */ 222 memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs)); 223 224 linker->secs = secs; 225 linker->sec_cnt = new_cnt; 226 227 sec = &linker->secs[new_cnt - 1]; 228 sec->id = new_cnt - 1; 229 sec->sec_name = strdup(sec_name); 230 if (!sec->sec_name) 231 return NULL; 232 233 return sec; 234 } 235 236 static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx) 237 { 238 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx]; 239 Elf64_Sym *syms, *sym; 240 size_t sym_cnt = symtab->sec_sz / sizeof(*sym); 241 242 syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym)); 243 if (!syms) 244 return NULL; 245 246 sym = &syms[sym_cnt]; 247 memset(sym, 0, sizeof(*sym)); 248 249 symtab->raw_data = syms; 250 symtab->sec_sz += sizeof(*sym); 251 symtab->shdr->sh_size += sizeof(*sym); 252 symtab->data->d_size += sizeof(*sym); 253 254 if (sym_idx) 255 *sym_idx = sym_cnt; 256 257 return sym; 258 } 259 260 static int init_output_elf(struct bpf_linker *linker, const char *file) 261 { 262 int err, str_off; 263 Elf64_Sym *init_sym; 264 struct dst_sec *sec; 265 266 linker->filename = strdup(file); 267 if (!linker->filename) 268 return -ENOMEM; 269 270 linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0644); 271 if (linker->fd < 0) { 272 err = -errno; 273 pr_warn("failed to create '%s': %d\n", file, err); 274 return err; 275 } 276 277 linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL); 278 if (!linker->elf) { 279 pr_warn_elf("failed to create ELF object"); 280 return -EINVAL; 281 } 282 283 /* ELF header */ 284 linker->elf_hdr = elf64_newehdr(linker->elf); 285 if (!linker->elf_hdr){ 286 pr_warn_elf("failed to create ELF header"); 287 return -EINVAL; 288 } 289 290 linker->elf_hdr->e_machine = EM_BPF; 291 linker->elf_hdr->e_type = ET_REL; 292 #if __BYTE_ORDER == __LITTLE_ENDIAN 293 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB; 294 #elif __BYTE_ORDER == __BIG_ENDIAN 295 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB; 296 #else 297 #error "Unknown __BYTE_ORDER" 298 #endif 299 300 /* STRTAB */ 301 /* initialize strset with an empty string to conform to ELF */ 302 linker->strtab_strs = strset__new(INT_MAX, "", sizeof("")); 303 if (libbpf_get_error(linker->strtab_strs)) 304 return libbpf_get_error(linker->strtab_strs); 305 306 sec = add_dst_sec(linker, ".strtab"); 307 if (!sec) 308 return -ENOMEM; 309 310 sec->scn = elf_newscn(linker->elf); 311 if (!sec->scn) { 312 pr_warn_elf("failed to create STRTAB section"); 313 return -EINVAL; 314 } 315 316 sec->shdr = elf64_getshdr(sec->scn); 317 if (!sec->shdr) 318 return -EINVAL; 319 320 sec->data = elf_newdata(sec->scn); 321 if (!sec->data) { 322 pr_warn_elf("failed to create STRTAB data"); 323 return -EINVAL; 324 } 325 326 str_off = strset__add_str(linker->strtab_strs, sec->sec_name); 327 if (str_off < 0) 328 return str_off; 329 330 sec->sec_idx = elf_ndxscn(sec->scn); 331 linker->elf_hdr->e_shstrndx = sec->sec_idx; 332 linker->strtab_sec_idx = sec->sec_idx; 333 334 sec->shdr->sh_name = str_off; 335 sec->shdr->sh_type = SHT_STRTAB; 336 sec->shdr->sh_flags = SHF_STRINGS; 337 sec->shdr->sh_offset = 0; 338 sec->shdr->sh_link = 0; 339 sec->shdr->sh_info = 0; 340 sec->shdr->sh_addralign = 1; 341 sec->shdr->sh_size = sec->sec_sz = 0; 342 sec->shdr->sh_entsize = 0; 343 344 /* SYMTAB */ 345 sec = add_dst_sec(linker, ".symtab"); 346 if (!sec) 347 return -ENOMEM; 348 349 sec->scn = elf_newscn(linker->elf); 350 if (!sec->scn) { 351 pr_warn_elf("failed to create SYMTAB section"); 352 return -EINVAL; 353 } 354 355 sec->shdr = elf64_getshdr(sec->scn); 356 if (!sec->shdr) 357 return -EINVAL; 358 359 sec->data = elf_newdata(sec->scn); 360 if (!sec->data) { 361 pr_warn_elf("failed to create SYMTAB data"); 362 return -EINVAL; 363 } 364 365 str_off = strset__add_str(linker->strtab_strs, sec->sec_name); 366 if (str_off < 0) 367 return str_off; 368 369 sec->sec_idx = elf_ndxscn(sec->scn); 370 linker->symtab_sec_idx = sec->sec_idx; 371 372 sec->shdr->sh_name = str_off; 373 sec->shdr->sh_type = SHT_SYMTAB; 374 sec->shdr->sh_flags = 0; 375 sec->shdr->sh_offset = 0; 376 sec->shdr->sh_link = linker->strtab_sec_idx; 377 /* sh_info should be one greater than the index of the last local 378 * symbol (i.e., binding is STB_LOCAL). But why and who cares? 379 */ 380 sec->shdr->sh_info = 0; 381 sec->shdr->sh_addralign = 8; 382 sec->shdr->sh_entsize = sizeof(Elf64_Sym); 383 384 /* .BTF */ 385 linker->btf = btf__new_empty(); 386 err = libbpf_get_error(linker->btf); 387 if (err) 388 return err; 389 390 /* add the special all-zero symbol */ 391 init_sym = add_new_sym(linker, NULL); 392 if (!init_sym) 393 return -EINVAL; 394 395 init_sym->st_name = 0; 396 init_sym->st_info = 0; 397 init_sym->st_other = 0; 398 init_sym->st_shndx = SHN_UNDEF; 399 init_sym->st_value = 0; 400 init_sym->st_size = 0; 401 402 return 0; 403 } 404 405 int bpf_linker__add_file(struct bpf_linker *linker, const char *filename) 406 { 407 struct src_obj obj = {}; 408 int err = 0; 409 410 if (!linker->elf) 411 return -EINVAL; 412 413 err = err ?: linker_load_obj_file(linker, filename, &obj); 414 err = err ?: linker_append_sec_data(linker, &obj); 415 err = err ?: linker_append_elf_syms(linker, &obj); 416 err = err ?: linker_append_elf_relos(linker, &obj); 417 err = err ?: linker_append_btf(linker, &obj); 418 err = err ?: linker_append_btf_ext(linker, &obj); 419 420 /* free up src_obj resources */ 421 free(obj.btf_type_map); 422 btf__free(obj.btf); 423 btf_ext__free(obj.btf_ext); 424 free(obj.secs); 425 free(obj.sym_map); 426 if (obj.elf) 427 elf_end(obj.elf); 428 if (obj.fd >= 0) 429 close(obj.fd); 430 431 return err; 432 } 433 434 static bool is_dwarf_sec_name(const char *name) 435 { 436 /* approximation, but the actual list is too long */ 437 return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0; 438 } 439 440 static bool is_ignored_sec(struct src_sec *sec) 441 { 442 Elf64_Shdr *shdr = sec->shdr; 443 const char *name = sec->sec_name; 444 445 /* no special handling of .strtab */ 446 if (shdr->sh_type == SHT_STRTAB) 447 return true; 448 449 /* ignore .llvm_addrsig section as well */ 450 if (shdr->sh_type == SHT_LLVM_ADDRSIG) 451 return true; 452 453 /* no subprograms will lead to an empty .text section, ignore it */ 454 if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 && 455 strcmp(sec->sec_name, ".text") == 0) 456 return true; 457 458 /* DWARF sections */ 459 if (is_dwarf_sec_name(sec->sec_name)) 460 return true; 461 462 if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) { 463 name += sizeof(".rel") - 1; 464 /* DWARF section relocations */ 465 if (is_dwarf_sec_name(name)) 466 return true; 467 468 /* .BTF and .BTF.ext don't need relocations */ 469 if (strcmp(name, BTF_ELF_SEC) == 0 || 470 strcmp(name, BTF_EXT_ELF_SEC) == 0) 471 return true; 472 } 473 474 return false; 475 } 476 477 static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name) 478 { 479 struct src_sec *secs = obj->secs, *sec; 480 size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2; 481 482 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs)); 483 if (!secs) 484 return NULL; 485 486 /* zero out newly allocated memory */ 487 memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs)); 488 489 obj->secs = secs; 490 obj->sec_cnt = new_cnt; 491 492 sec = &obj->secs[new_cnt - 1]; 493 sec->id = new_cnt - 1; 494 sec->sec_name = sec_name; 495 496 return sec; 497 } 498 499 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, struct src_obj *obj) 500 { 501 #if __BYTE_ORDER == __LITTLE_ENDIAN 502 const int host_endianness = ELFDATA2LSB; 503 #elif __BYTE_ORDER == __BIG_ENDIAN 504 const int host_endianness = ELFDATA2MSB; 505 #else 506 #error "Unknown __BYTE_ORDER" 507 #endif 508 int err = 0; 509 Elf_Scn *scn; 510 Elf_Data *data; 511 Elf64_Ehdr *ehdr; 512 Elf64_Shdr *shdr; 513 struct src_sec *sec; 514 515 pr_debug("linker: adding object file '%s'...\n", filename); 516 517 obj->filename = filename; 518 519 obj->fd = open(filename, O_RDONLY); 520 if (obj->fd < 0) { 521 err = -errno; 522 pr_warn("failed to open file '%s': %d\n", filename, err); 523 return err; 524 } 525 obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL); 526 if (!obj->elf) { 527 err = -errno; 528 pr_warn_elf("failed to parse ELF file '%s'", filename); 529 return err; 530 } 531 532 /* Sanity check ELF file high-level properties */ 533 ehdr = elf64_getehdr(obj->elf); 534 if (!ehdr) { 535 err = -errno; 536 pr_warn_elf("failed to get ELF header for %s", filename); 537 return err; 538 } 539 if (ehdr->e_ident[EI_DATA] != host_endianness) { 540 err = -EOPNOTSUPP; 541 pr_warn_elf("unsupported byte order of ELF file %s", filename); 542 return err; 543 } 544 if (ehdr->e_type != ET_REL 545 || ehdr->e_machine != EM_BPF 546 || ehdr->e_ident[EI_CLASS] != ELFCLASS64) { 547 err = -EOPNOTSUPP; 548 pr_warn_elf("unsupported kind of ELF file %s", filename); 549 return err; 550 } 551 552 if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) { 553 err = -errno; 554 pr_warn_elf("failed to get SHSTRTAB section index for %s", filename); 555 return err; 556 } 557 558 scn = NULL; 559 while ((scn = elf_nextscn(obj->elf, scn)) != NULL) { 560 size_t sec_idx = elf_ndxscn(scn); 561 const char *sec_name; 562 563 shdr = elf64_getshdr(scn); 564 if (!shdr) { 565 err = -errno; 566 pr_warn_elf("failed to get section #%zu header for %s", 567 sec_idx, filename); 568 return err; 569 } 570 571 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name); 572 if (!sec_name) { 573 err = -errno; 574 pr_warn_elf("failed to get section #%zu name for %s", 575 sec_idx, filename); 576 return err; 577 } 578 579 data = elf_getdata(scn, 0); 580 if (!data) { 581 err = -errno; 582 pr_warn_elf("failed to get section #%zu (%s) data from %s", 583 sec_idx, sec_name, filename); 584 return err; 585 } 586 587 sec = add_src_sec(obj, sec_name); 588 if (!sec) 589 return -ENOMEM; 590 591 sec->scn = scn; 592 sec->shdr = shdr; 593 sec->data = data; 594 sec->sec_idx = elf_ndxscn(scn); 595 596 if (is_ignored_sec(sec)) { 597 sec->skipped = true; 598 continue; 599 } 600 601 switch (shdr->sh_type) { 602 case SHT_SYMTAB: 603 if (obj->symtab_sec_idx) { 604 err = -EOPNOTSUPP; 605 pr_warn("multiple SYMTAB sections found, not supported\n"); 606 return err; 607 } 608 obj->symtab_sec_idx = sec_idx; 609 break; 610 case SHT_STRTAB: 611 /* we'll construct our own string table */ 612 break; 613 case SHT_PROGBITS: 614 if (strcmp(sec_name, BTF_ELF_SEC) == 0) { 615 obj->btf = btf__new(data->d_buf, shdr->sh_size); 616 err = libbpf_get_error(obj->btf); 617 if (err) { 618 pr_warn("failed to parse .BTF from %s: %d\n", filename, err); 619 return err; 620 } 621 sec->skipped = true; 622 continue; 623 } 624 if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) { 625 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size); 626 err = libbpf_get_error(obj->btf_ext); 627 if (err) { 628 pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err); 629 return err; 630 } 631 sec->skipped = true; 632 continue; 633 } 634 635 /* data & code */ 636 break; 637 case SHT_NOBITS: 638 /* BSS */ 639 break; 640 case SHT_REL: 641 /* relocations */ 642 break; 643 default: 644 pr_warn("unrecognized section #%zu (%s) in %s\n", 645 sec_idx, sec_name, filename); 646 err = -EINVAL; 647 return err; 648 } 649 } 650 651 err = err ?: linker_sanity_check_elf(obj); 652 err = err ?: linker_sanity_check_btf(obj); 653 err = err ?: linker_sanity_check_btf_ext(obj); 654 err = err ?: linker_fixup_btf(obj); 655 656 return err; 657 } 658 659 static bool is_pow_of_2(size_t x) 660 { 661 return x && (x & (x - 1)) == 0; 662 } 663 664 static int linker_sanity_check_elf(struct src_obj *obj) 665 { 666 struct src_sec *sec, *link_sec; 667 int i, j, n; 668 669 if (!obj->symtab_sec_idx) { 670 pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename); 671 return -EINVAL; 672 } 673 if (!obj->shstrs_sec_idx) { 674 pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename); 675 return -EINVAL; 676 } 677 678 for (i = 1; i < obj->sec_cnt; i++) { 679 sec = &obj->secs[i]; 680 681 if (sec->sec_name[0] == '\0') { 682 pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename); 683 return -EINVAL; 684 } 685 686 if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) 687 return -EINVAL; 688 if (sec->shdr->sh_addralign != sec->data->d_align) 689 return -EINVAL; 690 691 if (sec->shdr->sh_size != sec->data->d_size) 692 return -EINVAL; 693 694 switch (sec->shdr->sh_type) { 695 case SHT_SYMTAB: { 696 Elf64_Sym *sym; 697 698 if (sec->shdr->sh_entsize != sizeof(Elf64_Sym)) 699 return -EINVAL; 700 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0) 701 return -EINVAL; 702 703 if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) { 704 pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n", 705 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); 706 return -EINVAL; 707 } 708 link_sec = &obj->secs[sec->shdr->sh_link]; 709 if (link_sec->shdr->sh_type != SHT_STRTAB) { 710 pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n", 711 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); 712 return -EINVAL; 713 } 714 715 n = sec->shdr->sh_size / sec->shdr->sh_entsize; 716 sym = sec->data->d_buf; 717 for (j = 0; j < n; j++, sym++) { 718 if (sym->st_shndx 719 && sym->st_shndx < SHN_LORESERVE 720 && sym->st_shndx >= obj->sec_cnt) { 721 pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n", 722 j, sec->sec_idx, (size_t)sym->st_shndx, obj->filename); 723 return -EINVAL; 724 } 725 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION) { 726 if (sym->st_value != 0) 727 return -EINVAL; 728 } 729 } 730 break; 731 } 732 case SHT_STRTAB: 733 break; 734 case SHT_PROGBITS: 735 if (sec->shdr->sh_flags & SHF_EXECINSTR) { 736 if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) 737 return -EINVAL; 738 } 739 break; 740 case SHT_NOBITS: 741 break; 742 case SHT_REL: { 743 Elf64_Rel *relo; 744 struct src_sec *sym_sec; 745 746 if (sec->shdr->sh_entsize != sizeof(Elf64_Rel)) 747 return -EINVAL; 748 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0) 749 return -EINVAL; 750 751 /* SHT_REL's sh_link should point to SYMTAB */ 752 if (sec->shdr->sh_link != obj->symtab_sec_idx) { 753 pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n", 754 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); 755 return -EINVAL; 756 } 757 758 /* SHT_REL's sh_info points to relocated section */ 759 if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) { 760 pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n", 761 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename); 762 return -EINVAL; 763 } 764 link_sec = &obj->secs[sec->shdr->sh_info]; 765 766 /* .rel<secname> -> <secname> pattern is followed */ 767 if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0 768 || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) { 769 pr_warn("ELF relo section #%zu name has invalid name in %s\n", 770 sec->sec_idx, obj->filename); 771 return -EINVAL; 772 } 773 774 /* don't further validate relocations for ignored sections */ 775 if (link_sec->skipped) 776 break; 777 778 /* relocatable section is data or instructions */ 779 if (link_sec->shdr->sh_type != SHT_PROGBITS 780 && link_sec->shdr->sh_type != SHT_NOBITS) { 781 pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n", 782 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename); 783 return -EINVAL; 784 } 785 786 /* check sanity of each relocation */ 787 n = sec->shdr->sh_size / sec->shdr->sh_entsize; 788 relo = sec->data->d_buf; 789 sym_sec = &obj->secs[obj->symtab_sec_idx]; 790 for (j = 0; j < n; j++, relo++) { 791 size_t sym_idx = ELF64_R_SYM(relo->r_info); 792 size_t sym_type = ELF64_R_TYPE(relo->r_info); 793 794 if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32) { 795 pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n", 796 j, sec->sec_idx, sym_type, obj->filename); 797 return -EINVAL; 798 } 799 800 if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) { 801 pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n", 802 j, sec->sec_idx, sym_idx, obj->filename); 803 return -EINVAL; 804 } 805 806 if (link_sec->shdr->sh_flags & SHF_EXECINSTR) { 807 if (relo->r_offset % sizeof(struct bpf_insn) != 0) { 808 pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n", 809 j, sec->sec_idx, sym_idx, obj->filename); 810 return -EINVAL; 811 } 812 } 813 } 814 break; 815 } 816 case SHT_LLVM_ADDRSIG: 817 break; 818 default: 819 pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n", 820 sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename); 821 return -EINVAL; 822 } 823 } 824 825 return 0; 826 } 827 828 static int check_btf_type_id(__u32 *type_id, void *ctx) 829 { 830 struct btf *btf = ctx; 831 832 if (*type_id > btf__get_nr_types(btf)) 833 return -EINVAL; 834 835 return 0; 836 } 837 838 static int check_btf_str_off(__u32 *str_off, void *ctx) 839 { 840 struct btf *btf = ctx; 841 const char *s; 842 843 s = btf__str_by_offset(btf, *str_off); 844 845 if (!s) 846 return -EINVAL; 847 848 return 0; 849 } 850 851 static int linker_sanity_check_btf(struct src_obj *obj) 852 { 853 struct btf_type *t; 854 int i, n, err = 0; 855 856 if (!obj->btf) 857 return 0; 858 859 n = btf__get_nr_types(obj->btf); 860 for (i = 1; i <= n; i++) { 861 t = btf_type_by_id(obj->btf, i); 862 863 err = err ?: btf_type_visit_type_ids(t, check_btf_type_id, obj->btf); 864 err = err ?: btf_type_visit_str_offs(t, check_btf_str_off, obj->btf); 865 if (err) 866 return err; 867 } 868 869 return 0; 870 } 871 872 static int linker_sanity_check_btf_ext(struct src_obj *obj) 873 { 874 int err = 0; 875 876 if (!obj->btf_ext) 877 return 0; 878 879 /* can't use .BTF.ext without .BTF */ 880 if (!obj->btf) 881 return -EINVAL; 882 883 err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf); 884 err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf); 885 if (err) 886 return err; 887 888 return 0; 889 } 890 891 static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec) 892 { 893 Elf_Scn *scn; 894 Elf_Data *data; 895 Elf64_Shdr *shdr; 896 int name_off; 897 898 dst_sec->sec_sz = 0; 899 dst_sec->sec_idx = 0; 900 901 /* ephemeral sections are just thin section shells lacking most parts */ 902 if (src_sec->ephemeral) 903 return 0; 904 905 scn = elf_newscn(linker->elf); 906 if (!scn) 907 return -1; 908 data = elf_newdata(scn); 909 if (!data) 910 return -1; 911 shdr = elf64_getshdr(scn); 912 if (!shdr) 913 return -1; 914 915 dst_sec->scn = scn; 916 dst_sec->shdr = shdr; 917 dst_sec->data = data; 918 dst_sec->sec_idx = elf_ndxscn(scn); 919 920 name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name); 921 if (name_off < 0) 922 return name_off; 923 924 shdr->sh_name = name_off; 925 shdr->sh_type = src_sec->shdr->sh_type; 926 shdr->sh_flags = src_sec->shdr->sh_flags; 927 shdr->sh_size = 0; 928 /* sh_link and sh_info have different meaning for different types of 929 * sections, so we leave it up to the caller code to fill them in, if 930 * necessary 931 */ 932 shdr->sh_link = 0; 933 shdr->sh_info = 0; 934 shdr->sh_addralign = src_sec->shdr->sh_addralign; 935 shdr->sh_entsize = src_sec->shdr->sh_entsize; 936 937 data->d_type = src_sec->data->d_type; 938 data->d_size = 0; 939 data->d_buf = NULL; 940 data->d_align = src_sec->data->d_align; 941 data->d_off = 0; 942 943 return 0; 944 } 945 946 static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name) 947 { 948 struct dst_sec *sec; 949 int i; 950 951 for (i = 1; i < linker->sec_cnt; i++) { 952 sec = &linker->secs[i]; 953 954 if (strcmp(sec->sec_name, sec_name) == 0) 955 return sec; 956 } 957 958 return NULL; 959 } 960 961 static bool secs_match(struct dst_sec *dst, struct src_sec *src) 962 { 963 if (dst->shdr->sh_type != src->shdr->sh_type) { 964 pr_warn("sec %s types mismatch\n", dst->sec_name); 965 return false; 966 } 967 if (dst->shdr->sh_flags != src->shdr->sh_flags) { 968 pr_warn("sec %s flags mismatch\n", dst->sec_name); 969 return false; 970 } 971 if (dst->shdr->sh_entsize != src->shdr->sh_entsize) { 972 pr_warn("sec %s entsize mismatch\n", dst->sec_name); 973 return false; 974 } 975 976 return true; 977 } 978 979 static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec) 980 { 981 if (dst_sec->sec_sz != src_sec->shdr->sh_size) 982 return false; 983 if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0) 984 return false; 985 return true; 986 } 987 988 static int extend_sec(struct dst_sec *dst, struct src_sec *src) 989 { 990 void *tmp; 991 size_t dst_align = dst->shdr->sh_addralign; 992 size_t src_align = src->shdr->sh_addralign; 993 size_t dst_align_sz, dst_final_sz; 994 995 if (dst_align == 0) 996 dst_align = 1; 997 if (dst_align < src_align) 998 dst_align = src_align; 999 1000 dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align; 1001 1002 /* no need to re-align final size */ 1003 dst_final_sz = dst_align_sz + src->shdr->sh_size; 1004 1005 if (src->shdr->sh_type != SHT_NOBITS) { 1006 tmp = realloc(dst->raw_data, dst_final_sz); 1007 if (!tmp) 1008 return -ENOMEM; 1009 dst->raw_data = tmp; 1010 1011 /* pad dst section, if it's alignment forced size increase */ 1012 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz); 1013 /* now copy src data at a properly aligned offset */ 1014 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size); 1015 } 1016 1017 dst->sec_sz = dst_final_sz; 1018 dst->shdr->sh_size = dst_final_sz; 1019 dst->data->d_size = dst_final_sz; 1020 1021 dst->shdr->sh_addralign = dst_align; 1022 dst->data->d_align = dst_align; 1023 1024 src->dst_off = dst_align_sz; 1025 1026 return 0; 1027 } 1028 1029 static bool is_data_sec(struct src_sec *sec) 1030 { 1031 if (!sec || sec->skipped) 1032 return false; 1033 /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */ 1034 if (sec->ephemeral) 1035 return true; 1036 return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS; 1037 } 1038 1039 static bool is_relo_sec(struct src_sec *sec) 1040 { 1041 if (!sec || sec->skipped || sec->ephemeral) 1042 return false; 1043 return sec->shdr->sh_type == SHT_REL; 1044 } 1045 1046 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj) 1047 { 1048 int i, err; 1049 1050 for (i = 1; i < obj->sec_cnt; i++) { 1051 struct src_sec *src_sec; 1052 struct dst_sec *dst_sec; 1053 1054 src_sec = &obj->secs[i]; 1055 if (!is_data_sec(src_sec)) 1056 continue; 1057 1058 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name); 1059 if (!dst_sec) { 1060 dst_sec = add_dst_sec(linker, src_sec->sec_name); 1061 if (!dst_sec) 1062 return -ENOMEM; 1063 err = init_sec(linker, dst_sec, src_sec); 1064 if (err) { 1065 pr_warn("failed to init section '%s'\n", src_sec->sec_name); 1066 return err; 1067 } 1068 } else { 1069 if (!secs_match(dst_sec, src_sec)) { 1070 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name); 1071 return -1; 1072 } 1073 1074 /* "license" and "version" sections are deduped */ 1075 if (strcmp(src_sec->sec_name, "license") == 0 1076 || strcmp(src_sec->sec_name, "version") == 0) { 1077 if (!sec_content_is_same(dst_sec, src_sec)) { 1078 pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name); 1079 return -EINVAL; 1080 } 1081 src_sec->skipped = true; 1082 src_sec->dst_id = dst_sec->id; 1083 continue; 1084 } 1085 } 1086 1087 /* record mapped section index */ 1088 src_sec->dst_id = dst_sec->id; 1089 1090 if (src_sec->ephemeral) 1091 continue; 1092 1093 err = extend_sec(dst_sec, src_sec); 1094 if (err) 1095 return err; 1096 } 1097 1098 return 0; 1099 } 1100 1101 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj) 1102 { 1103 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx]; 1104 Elf64_Sym *sym = symtab->data->d_buf, *dst_sym; 1105 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize; 1106 int str_sec_idx = symtab->shdr->sh_link; 1107 1108 obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map)); 1109 if (!obj->sym_map) 1110 return -ENOMEM; 1111 1112 for (i = 0; i < n; i++, sym++) { 1113 struct src_sec *src_sec = NULL; 1114 struct dst_sec *dst_sec = NULL; 1115 const char *sym_name; 1116 size_t dst_sym_idx; 1117 int name_off; 1118 1119 /* we already have all-zero initial symbol */ 1120 if (sym->st_name == 0 && sym->st_info == 0 && 1121 sym->st_other == 0 && sym->st_shndx == SHN_UNDEF && 1122 sym->st_value == 0 && sym->st_size ==0) 1123 continue; 1124 1125 sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name); 1126 if (!sym_name) { 1127 pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename); 1128 return -1; 1129 } 1130 1131 if (sym->st_shndx && sym->st_shndx < SHN_LORESERVE) { 1132 src_sec = &obj->secs[sym->st_shndx]; 1133 if (src_sec->skipped) 1134 continue; 1135 dst_sec = &linker->secs[src_sec->dst_id]; 1136 1137 /* allow only one STT_SECTION symbol per section */ 1138 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && dst_sec->sec_sym_idx) { 1139 obj->sym_map[i] = dst_sec->sec_sym_idx; 1140 continue; 1141 } 1142 } 1143 1144 name_off = strset__add_str(linker->strtab_strs, sym_name); 1145 if (name_off < 0) 1146 return name_off; 1147 1148 dst_sym = add_new_sym(linker, &dst_sym_idx); 1149 if (!dst_sym) 1150 return -ENOMEM; 1151 1152 dst_sym->st_name = name_off; 1153 dst_sym->st_info = sym->st_info; 1154 dst_sym->st_other = sym->st_other; 1155 dst_sym->st_shndx = src_sec ? dst_sec->sec_idx : sym->st_shndx; 1156 dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value; 1157 dst_sym->st_size = sym->st_size; 1158 1159 obj->sym_map[i] = dst_sym_idx; 1160 1161 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && dst_sym) { 1162 dst_sec->sec_sym_idx = dst_sym_idx; 1163 dst_sym->st_value = 0; 1164 } 1165 1166 } 1167 1168 return 0; 1169 } 1170 1171 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj) 1172 { 1173 struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx]; 1174 struct dst_sec *dst_symtab = &linker->secs[linker->symtab_sec_idx]; 1175 int i, err; 1176 1177 for (i = 1; i < obj->sec_cnt; i++) { 1178 struct src_sec *src_sec, *src_linked_sec; 1179 struct dst_sec *dst_sec, *dst_linked_sec; 1180 Elf64_Rel *src_rel, *dst_rel; 1181 int j, n; 1182 1183 src_sec = &obj->secs[i]; 1184 if (!is_relo_sec(src_sec)) 1185 continue; 1186 1187 /* shdr->sh_info points to relocatable section */ 1188 src_linked_sec = &obj->secs[src_sec->shdr->sh_info]; 1189 if (src_linked_sec->skipped) 1190 continue; 1191 1192 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name); 1193 if (!dst_sec) { 1194 dst_sec = add_dst_sec(linker, src_sec->sec_name); 1195 if (!dst_sec) 1196 return -ENOMEM; 1197 err = init_sec(linker, dst_sec, src_sec); 1198 if (err) { 1199 pr_warn("failed to init section '%s'\n", src_sec->sec_name); 1200 return err; 1201 } 1202 } else if (!secs_match(dst_sec, src_sec)) { 1203 pr_warn("Secs %s are not compatible\n", src_sec->sec_name); 1204 return -1; 1205 } 1206 1207 /* shdr->sh_link points to SYMTAB */ 1208 dst_sec->shdr->sh_link = linker->symtab_sec_idx; 1209 1210 /* shdr->sh_info points to relocated section */ 1211 dst_linked_sec = &linker->secs[src_linked_sec->dst_id]; 1212 dst_sec->shdr->sh_info = dst_linked_sec->sec_idx; 1213 1214 src_sec->dst_id = dst_sec->id; 1215 err = extend_sec(dst_sec, src_sec); 1216 if (err) 1217 return err; 1218 1219 src_rel = src_sec->data->d_buf; 1220 dst_rel = dst_sec->raw_data + src_sec->dst_off; 1221 n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize; 1222 for (j = 0; j < n; j++, src_rel++, dst_rel++) { 1223 size_t src_sym_idx = ELF64_R_SYM(src_rel->r_info); 1224 size_t sym_type = ELF64_R_TYPE(src_rel->r_info); 1225 Elf64_Sym *src_sym, *dst_sym; 1226 size_t dst_sym_idx; 1227 1228 src_sym_idx = ELF64_R_SYM(src_rel->r_info); 1229 src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx; 1230 1231 dst_sym_idx = obj->sym_map[src_sym_idx]; 1232 dst_sym = dst_symtab->raw_data + sizeof(*dst_sym) * dst_sym_idx; 1233 dst_rel->r_offset += src_linked_sec->dst_off; 1234 sym_type = ELF64_R_TYPE(src_rel->r_info); 1235 dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type); 1236 1237 if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) { 1238 struct src_sec *sec = &obj->secs[src_sym->st_shndx]; 1239 struct bpf_insn *insn; 1240 1241 if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) { 1242 /* calls to the very first static function inside 1243 * .text section at offset 0 will 1244 * reference section symbol, not the 1245 * function symbol. Fix that up, 1246 * otherwise it won't be possible to 1247 * relocate calls to two different 1248 * static functions with the same name 1249 * (rom two different object files) 1250 */ 1251 insn = dst_linked_sec->raw_data + dst_rel->r_offset; 1252 if (insn->code == (BPF_JMP | BPF_CALL)) 1253 insn->imm += sec->dst_off / sizeof(struct bpf_insn); 1254 else 1255 insn->imm += sec->dst_off; 1256 } else { 1257 pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n"); 1258 return -EINVAL; 1259 } 1260 } 1261 1262 } 1263 } 1264 1265 return 0; 1266 } 1267 1268 static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name) 1269 { 1270 struct src_sec *sec; 1271 int i; 1272 1273 for (i = 1; i < obj->sec_cnt; i++) { 1274 sec = &obj->secs[i]; 1275 1276 if (strcmp(sec->sec_name, sec_name) == 0) 1277 return sec; 1278 } 1279 1280 return NULL; 1281 } 1282 1283 static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx, 1284 int sym_type, const char *sym_name) 1285 { 1286 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx]; 1287 Elf64_Sym *sym = symtab->data->d_buf; 1288 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize; 1289 int str_sec_idx = symtab->shdr->sh_link; 1290 const char *name; 1291 1292 for (i = 0; i < n; i++, sym++) { 1293 if (sym->st_shndx != sec_idx) 1294 continue; 1295 if (ELF64_ST_TYPE(sym->st_info) != sym_type) 1296 continue; 1297 1298 name = elf_strptr(obj->elf, str_sec_idx, sym->st_name); 1299 if (!name) 1300 return NULL; 1301 1302 if (strcmp(sym_name, name) != 0) 1303 continue; 1304 1305 return sym; 1306 } 1307 1308 return NULL; 1309 } 1310 1311 static int linker_fixup_btf(struct src_obj *obj) 1312 { 1313 const char *sec_name; 1314 struct src_sec *sec; 1315 int i, j, n, m; 1316 1317 if (!obj->btf) 1318 return 0; 1319 1320 n = btf__get_nr_types(obj->btf); 1321 for (i = 1; i <= n; i++) { 1322 struct btf_var_secinfo *vi; 1323 struct btf_type *t; 1324 1325 t = btf_type_by_id(obj->btf, i); 1326 if (btf_kind(t) != BTF_KIND_DATASEC) 1327 continue; 1328 1329 sec_name = btf__str_by_offset(obj->btf, t->name_off); 1330 sec = find_src_sec_by_name(obj, sec_name); 1331 if (sec) { 1332 /* record actual section size, unless ephemeral */ 1333 if (sec->shdr) 1334 t->size = sec->shdr->sh_size; 1335 } else { 1336 /* BTF can have some sections that are not represented 1337 * in ELF, e.g., .kconfig and .ksyms, which are used 1338 * for special extern variables. Here we'll 1339 * pre-create "section shells" for them to be able to 1340 * keep track of extra per-section metadata later 1341 * (e.g., BTF variables). 1342 */ 1343 sec = add_src_sec(obj, sec_name); 1344 if (!sec) 1345 return -ENOMEM; 1346 1347 sec->ephemeral = true; 1348 sec->sec_idx = 0; /* will match UNDEF shndx in ELF */ 1349 } 1350 1351 /* remember ELF section and its BTF type ID match */ 1352 sec->sec_type_id = i; 1353 1354 /* fix up variable offsets */ 1355 vi = btf_var_secinfos(t); 1356 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) { 1357 const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type); 1358 const char *var_name = btf__str_by_offset(obj->btf, vt->name_off); 1359 int var_linkage = btf_var(vt)->linkage; 1360 Elf64_Sym *sym; 1361 1362 /* no need to patch up static or extern vars */ 1363 if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED) 1364 continue; 1365 1366 sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name); 1367 if (!sym) { 1368 pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name); 1369 return -ENOENT; 1370 } 1371 1372 vi->offset = sym->st_value; 1373 } 1374 } 1375 1376 return 0; 1377 } 1378 1379 static int remap_type_id(__u32 *type_id, void *ctx) 1380 { 1381 int *id_map = ctx; 1382 1383 *type_id = id_map[*type_id]; 1384 1385 return 0; 1386 } 1387 1388 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj) 1389 { 1390 const struct btf_type *t; 1391 int i, j, n, start_id, id; 1392 1393 if (!obj->btf) 1394 return 0; 1395 1396 start_id = btf__get_nr_types(linker->btf) + 1; 1397 n = btf__get_nr_types(obj->btf); 1398 1399 obj->btf_type_map = calloc(n + 1, sizeof(int)); 1400 if (!obj->btf_type_map) 1401 return -ENOMEM; 1402 1403 for (i = 1; i <= n; i++) { 1404 t = btf__type_by_id(obj->btf, i); 1405 1406 /* DATASECs are handled specially below */ 1407 if (btf_kind(t) == BTF_KIND_DATASEC) 1408 continue; 1409 1410 id = btf__add_type(linker->btf, obj->btf, t); 1411 if (id < 0) { 1412 pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename); 1413 return id; 1414 } 1415 1416 obj->btf_type_map[i] = id; 1417 } 1418 1419 /* remap all the types except DATASECs */ 1420 n = btf__get_nr_types(linker->btf); 1421 for (i = start_id; i <= n; i++) { 1422 struct btf_type *dst_t = btf_type_by_id(linker->btf, i); 1423 1424 if (btf_type_visit_type_ids(dst_t, remap_type_id, obj->btf_type_map)) 1425 return -EINVAL; 1426 } 1427 1428 /* append DATASEC info */ 1429 for (i = 1; i < obj->sec_cnt; i++) { 1430 struct src_sec *src_sec; 1431 struct dst_sec *dst_sec; 1432 const struct btf_var_secinfo *src_var; 1433 struct btf_var_secinfo *dst_var; 1434 1435 src_sec = &obj->secs[i]; 1436 if (!src_sec->sec_type_id || src_sec->skipped) 1437 continue; 1438 dst_sec = &linker->secs[src_sec->dst_id]; 1439 1440 /* Mark section as having BTF regardless of the presence of 1441 * variables. In some cases compiler might generate empty BTF 1442 * with no variables information. E.g., when promoting local 1443 * array/structure variable initial values and BPF object 1444 * file otherwise has no read-only static variables in 1445 * .rodata. We need to preserve such empty BTF and just set 1446 * correct section size. 1447 */ 1448 dst_sec->has_btf = true; 1449 1450 t = btf__type_by_id(obj->btf, src_sec->sec_type_id); 1451 src_var = btf_var_secinfos(t); 1452 n = btf_vlen(t); 1453 for (j = 0; j < n; j++, src_var++) { 1454 void *sec_vars = dst_sec->sec_vars; 1455 1456 sec_vars = libbpf_reallocarray(sec_vars, 1457 dst_sec->sec_var_cnt + 1, 1458 sizeof(*dst_sec->sec_vars)); 1459 if (!sec_vars) 1460 return -ENOMEM; 1461 1462 dst_sec->sec_vars = sec_vars; 1463 dst_sec->sec_var_cnt++; 1464 1465 dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1]; 1466 dst_var->type = obj->btf_type_map[src_var->type]; 1467 dst_var->size = src_var->size; 1468 dst_var->offset = src_sec->dst_off + src_var->offset; 1469 } 1470 } 1471 1472 return 0; 1473 } 1474 1475 static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec) 1476 { 1477 void *tmp; 1478 1479 tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz); 1480 if (!tmp) 1481 return NULL; 1482 ext_data->recs = tmp; 1483 1484 tmp += ext_data->rec_cnt * ext_data->rec_sz; 1485 memcpy(tmp, src_rec, ext_data->rec_sz); 1486 1487 ext_data->rec_cnt++; 1488 1489 return tmp; 1490 } 1491 1492 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj) 1493 { 1494 const struct btf_ext_info_sec *ext_sec; 1495 const char *sec_name, *s; 1496 struct src_sec *src_sec; 1497 struct dst_sec *dst_sec; 1498 int rec_sz, str_off, i; 1499 1500 if (!obj->btf_ext) 1501 return 0; 1502 1503 rec_sz = obj->btf_ext->func_info.rec_size; 1504 for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) { 1505 struct bpf_func_info_min *src_rec, *dst_rec; 1506 1507 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); 1508 src_sec = find_src_sec_by_name(obj, sec_name); 1509 if (!src_sec) { 1510 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name); 1511 return -EINVAL; 1512 } 1513 dst_sec = &linker->secs[src_sec->dst_id]; 1514 1515 if (dst_sec->func_info.rec_sz == 0) 1516 dst_sec->func_info.rec_sz = rec_sz; 1517 if (dst_sec->func_info.rec_sz != rec_sz) { 1518 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name); 1519 return -EINVAL; 1520 } 1521 1522 for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) { 1523 dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec); 1524 if (!dst_rec) 1525 return -ENOMEM; 1526 1527 dst_rec->insn_off += src_sec->dst_off; 1528 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id]; 1529 } 1530 } 1531 1532 rec_sz = obj->btf_ext->line_info.rec_size; 1533 for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) { 1534 struct bpf_line_info_min *src_rec, *dst_rec; 1535 1536 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); 1537 src_sec = find_src_sec_by_name(obj, sec_name); 1538 if (!src_sec) { 1539 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name); 1540 return -EINVAL; 1541 } 1542 dst_sec = &linker->secs[src_sec->dst_id]; 1543 1544 if (dst_sec->line_info.rec_sz == 0) 1545 dst_sec->line_info.rec_sz = rec_sz; 1546 if (dst_sec->line_info.rec_sz != rec_sz) { 1547 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name); 1548 return -EINVAL; 1549 } 1550 1551 for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) { 1552 dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec); 1553 if (!dst_rec) 1554 return -ENOMEM; 1555 1556 dst_rec->insn_off += src_sec->dst_off; 1557 1558 s = btf__str_by_offset(obj->btf, src_rec->file_name_off); 1559 str_off = btf__add_str(linker->btf, s); 1560 if (str_off < 0) 1561 return -ENOMEM; 1562 dst_rec->file_name_off = str_off; 1563 1564 s = btf__str_by_offset(obj->btf, src_rec->line_off); 1565 str_off = btf__add_str(linker->btf, s); 1566 if (str_off < 0) 1567 return -ENOMEM; 1568 dst_rec->line_off = str_off; 1569 1570 /* dst_rec->line_col is fine */ 1571 } 1572 } 1573 1574 rec_sz = obj->btf_ext->core_relo_info.rec_size; 1575 for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) { 1576 struct bpf_core_relo *src_rec, *dst_rec; 1577 1578 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); 1579 src_sec = find_src_sec_by_name(obj, sec_name); 1580 if (!src_sec) { 1581 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name); 1582 return -EINVAL; 1583 } 1584 dst_sec = &linker->secs[src_sec->dst_id]; 1585 1586 if (dst_sec->core_relo_info.rec_sz == 0) 1587 dst_sec->core_relo_info.rec_sz = rec_sz; 1588 if (dst_sec->core_relo_info.rec_sz != rec_sz) { 1589 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name); 1590 return -EINVAL; 1591 } 1592 1593 for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) { 1594 dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec); 1595 if (!dst_rec) 1596 return -ENOMEM; 1597 1598 dst_rec->insn_off += src_sec->dst_off; 1599 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id]; 1600 1601 s = btf__str_by_offset(obj->btf, src_rec->access_str_off); 1602 str_off = btf__add_str(linker->btf, s); 1603 if (str_off < 0) 1604 return -ENOMEM; 1605 dst_rec->access_str_off = str_off; 1606 1607 /* dst_rec->kind is fine */ 1608 } 1609 } 1610 1611 return 0; 1612 } 1613 1614 int bpf_linker__finalize(struct bpf_linker *linker) 1615 { 1616 struct dst_sec *sec; 1617 size_t strs_sz; 1618 const void *strs; 1619 int err, i; 1620 1621 if (!linker->elf) 1622 return -EINVAL; 1623 1624 err = finalize_btf(linker); 1625 if (err) 1626 return err; 1627 1628 /* Finalize strings */ 1629 strs_sz = strset__data_size(linker->strtab_strs); 1630 strs = strset__data(linker->strtab_strs); 1631 1632 sec = &linker->secs[linker->strtab_sec_idx]; 1633 sec->data->d_align = 1; 1634 sec->data->d_off = 0LL; 1635 sec->data->d_buf = (void *)strs; 1636 sec->data->d_type = ELF_T_BYTE; 1637 sec->data->d_size = strs_sz; 1638 sec->shdr->sh_size = strs_sz; 1639 1640 for (i = 1; i < linker->sec_cnt; i++) { 1641 sec = &linker->secs[i]; 1642 1643 /* STRTAB is handled specially above */ 1644 if (sec->sec_idx == linker->strtab_sec_idx) 1645 continue; 1646 1647 /* special ephemeral sections (.ksyms, .kconfig, etc) */ 1648 if (!sec->scn) 1649 continue; 1650 1651 sec->data->d_buf = sec->raw_data; 1652 } 1653 1654 /* Finalize ELF layout */ 1655 if (elf_update(linker->elf, ELF_C_NULL) < 0) { 1656 err = -errno; 1657 pr_warn_elf("failed to finalize ELF layout"); 1658 return err; 1659 } 1660 1661 /* Write out final ELF contents */ 1662 if (elf_update(linker->elf, ELF_C_WRITE) < 0) { 1663 err = -errno; 1664 pr_warn_elf("failed to write ELF contents"); 1665 return err; 1666 } 1667 1668 elf_end(linker->elf); 1669 close(linker->fd); 1670 1671 linker->elf = NULL; 1672 linker->fd = -1; 1673 1674 return 0; 1675 } 1676 1677 static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name, 1678 size_t align, const void *raw_data, size_t raw_sz) 1679 { 1680 Elf_Scn *scn; 1681 Elf_Data *data; 1682 Elf64_Shdr *shdr; 1683 int name_off; 1684 1685 name_off = strset__add_str(linker->strtab_strs, sec_name); 1686 if (name_off < 0) 1687 return name_off; 1688 1689 scn = elf_newscn(linker->elf); 1690 if (!scn) 1691 return -ENOMEM; 1692 data = elf_newdata(scn); 1693 if (!data) 1694 return -ENOMEM; 1695 shdr = elf64_getshdr(scn); 1696 if (!shdr) 1697 return -EINVAL; 1698 1699 shdr->sh_name = name_off; 1700 shdr->sh_type = SHT_PROGBITS; 1701 shdr->sh_flags = 0; 1702 shdr->sh_size = raw_sz; 1703 shdr->sh_link = 0; 1704 shdr->sh_info = 0; 1705 shdr->sh_addralign = align; 1706 shdr->sh_entsize = 0; 1707 1708 data->d_type = ELF_T_BYTE; 1709 data->d_size = raw_sz; 1710 data->d_buf = (void *)raw_data; 1711 data->d_align = align; 1712 data->d_off = 0; 1713 1714 return 0; 1715 } 1716 1717 static int finalize_btf(struct bpf_linker *linker) 1718 { 1719 struct btf *btf = linker->btf; 1720 const void *raw_data; 1721 int i, j, id, err; 1722 __u32 raw_sz; 1723 1724 /* bail out if no BTF data was produced */ 1725 if (btf__get_nr_types(linker->btf) == 0) 1726 return 0; 1727 1728 for (i = 1; i < linker->sec_cnt; i++) { 1729 struct dst_sec *sec = &linker->secs[i]; 1730 1731 if (!sec->has_btf) 1732 continue; 1733 1734 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz); 1735 if (id < 0) { 1736 pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n", 1737 sec->sec_name, id); 1738 return id; 1739 } 1740 1741 for (j = 0; j < sec->sec_var_cnt; j++) { 1742 struct btf_var_secinfo *vi = &sec->sec_vars[j]; 1743 1744 if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size)) 1745 return -EINVAL; 1746 } 1747 } 1748 1749 err = finalize_btf_ext(linker); 1750 if (err) { 1751 pr_warn(".BTF.ext generation failed: %d\n", err); 1752 return err; 1753 } 1754 1755 err = btf__dedup(linker->btf, linker->btf_ext, NULL); 1756 if (err) { 1757 pr_warn("BTF dedup failed: %d\n", err); 1758 return err; 1759 } 1760 1761 /* Emit .BTF section */ 1762 raw_data = btf__get_raw_data(linker->btf, &raw_sz); 1763 if (!raw_data) 1764 return -ENOMEM; 1765 1766 err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz); 1767 if (err) { 1768 pr_warn("failed to write out .BTF ELF section: %d\n", err); 1769 return err; 1770 } 1771 1772 /* Emit .BTF.ext section */ 1773 if (linker->btf_ext) { 1774 raw_data = btf_ext__get_raw_data(linker->btf_ext, &raw_sz); 1775 if (!raw_data) 1776 return -ENOMEM; 1777 1778 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz); 1779 if (err) { 1780 pr_warn("failed to write out .BTF.ext ELF section: %d\n", err); 1781 return err; 1782 } 1783 } 1784 1785 return 0; 1786 } 1787 1788 static int emit_btf_ext_data(struct bpf_linker *linker, void *output, 1789 const char *sec_name, struct btf_ext_sec_data *sec_data) 1790 { 1791 struct btf_ext_info_sec *sec_info; 1792 void *cur = output; 1793 int str_off; 1794 size_t sz; 1795 1796 if (!sec_data->rec_cnt) 1797 return 0; 1798 1799 str_off = btf__add_str(linker->btf, sec_name); 1800 if (str_off < 0) 1801 return -ENOMEM; 1802 1803 sec_info = cur; 1804 sec_info->sec_name_off = str_off; 1805 sec_info->num_info = sec_data->rec_cnt; 1806 cur += sizeof(struct btf_ext_info_sec); 1807 1808 sz = sec_data->rec_cnt * sec_data->rec_sz; 1809 memcpy(cur, sec_data->recs, sz); 1810 cur += sz; 1811 1812 return cur - output; 1813 } 1814 1815 static int finalize_btf_ext(struct bpf_linker *linker) 1816 { 1817 size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0; 1818 size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0; 1819 struct btf_ext_header *hdr; 1820 void *data, *cur; 1821 int i, err, sz; 1822 1823 /* validate that all sections have the same .BTF.ext record sizes 1824 * and calculate total data size for each type of data (func info, 1825 * line info, core relos) 1826 */ 1827 for (i = 1; i < linker->sec_cnt; i++) { 1828 struct dst_sec *sec = &linker->secs[i]; 1829 1830 if (sec->func_info.rec_cnt) { 1831 if (func_rec_sz == 0) 1832 func_rec_sz = sec->func_info.rec_sz; 1833 if (func_rec_sz != sec->func_info.rec_sz) { 1834 pr_warn("mismatch in func_info record size %zu != %u\n", 1835 func_rec_sz, sec->func_info.rec_sz); 1836 return -EINVAL; 1837 } 1838 1839 funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt; 1840 } 1841 if (sec->line_info.rec_cnt) { 1842 if (line_rec_sz == 0) 1843 line_rec_sz = sec->line_info.rec_sz; 1844 if (line_rec_sz != sec->line_info.rec_sz) { 1845 pr_warn("mismatch in line_info record size %zu != %u\n", 1846 line_rec_sz, sec->line_info.rec_sz); 1847 return -EINVAL; 1848 } 1849 1850 lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt; 1851 } 1852 if (sec->core_relo_info.rec_cnt) { 1853 if (core_relo_rec_sz == 0) 1854 core_relo_rec_sz = sec->core_relo_info.rec_sz; 1855 if (core_relo_rec_sz != sec->core_relo_info.rec_sz) { 1856 pr_warn("mismatch in core_relo_info record size %zu != %u\n", 1857 core_relo_rec_sz, sec->core_relo_info.rec_sz); 1858 return -EINVAL; 1859 } 1860 1861 core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt; 1862 } 1863 } 1864 1865 if (!funcs_sz && !lines_sz && !core_relos_sz) 1866 return 0; 1867 1868 total_sz += sizeof(struct btf_ext_header); 1869 if (funcs_sz) { 1870 funcs_sz += sizeof(__u32); /* record size prefix */ 1871 total_sz += funcs_sz; 1872 } 1873 if (lines_sz) { 1874 lines_sz += sizeof(__u32); /* record size prefix */ 1875 total_sz += lines_sz; 1876 } 1877 if (core_relos_sz) { 1878 core_relos_sz += sizeof(__u32); /* record size prefix */ 1879 total_sz += core_relos_sz; 1880 } 1881 1882 cur = data = calloc(1, total_sz); 1883 if (!data) 1884 return -ENOMEM; 1885 1886 hdr = cur; 1887 hdr->magic = BTF_MAGIC; 1888 hdr->version = BTF_VERSION; 1889 hdr->flags = 0; 1890 hdr->hdr_len = sizeof(struct btf_ext_header); 1891 cur += sizeof(struct btf_ext_header); 1892 1893 /* All offsets are in bytes relative to the end of this header */ 1894 hdr->func_info_off = 0; 1895 hdr->func_info_len = funcs_sz; 1896 hdr->line_info_off = funcs_sz; 1897 hdr->line_info_len = lines_sz; 1898 hdr->core_relo_off = funcs_sz + lines_sz;; 1899 hdr->core_relo_len = core_relos_sz; 1900 1901 if (funcs_sz) { 1902 *(__u32 *)cur = func_rec_sz; 1903 cur += sizeof(__u32); 1904 1905 for (i = 1; i < linker->sec_cnt; i++) { 1906 struct dst_sec *sec = &linker->secs[i]; 1907 1908 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info); 1909 if (sz < 0) { 1910 err = sz; 1911 goto out; 1912 } 1913 1914 cur += sz; 1915 } 1916 } 1917 1918 if (lines_sz) { 1919 *(__u32 *)cur = line_rec_sz; 1920 cur += sizeof(__u32); 1921 1922 for (i = 1; i < linker->sec_cnt; i++) { 1923 struct dst_sec *sec = &linker->secs[i]; 1924 1925 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info); 1926 if (sz < 0) { 1927 err = sz; 1928 goto out; 1929 } 1930 1931 cur += sz; 1932 } 1933 } 1934 1935 if (core_relos_sz) { 1936 *(__u32 *)cur = core_relo_rec_sz; 1937 cur += sizeof(__u32); 1938 1939 for (i = 1; i < linker->sec_cnt; i++) { 1940 struct dst_sec *sec = &linker->secs[i]; 1941 1942 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info); 1943 if (sz < 0) { 1944 err = sz; 1945 goto out; 1946 } 1947 1948 cur += sz; 1949 } 1950 } 1951 1952 linker->btf_ext = btf_ext__new(data, total_sz); 1953 err = libbpf_get_error(linker->btf_ext); 1954 if (err) { 1955 linker->btf_ext = NULL; 1956 pr_warn("failed to parse final .BTF.ext data: %d\n", err); 1957 goto out; 1958 } 1959 1960 out: 1961 free(data); 1962 return err; 1963 } 1964