1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 /* 3 * BPF static linker 4 * 5 * Copyright (c) 2021 Facebook 6 */ 7 #include <stdbool.h> 8 #include <stddef.h> 9 #include <stdio.h> 10 #include <stdlib.h> 11 #include <string.h> 12 #include <unistd.h> 13 #include <errno.h> 14 #include <linux/err.h> 15 #include <linux/btf.h> 16 #include <elf.h> 17 #include <libelf.h> 18 #include <fcntl.h> 19 #include "libbpf.h" 20 #include "btf.h" 21 #include "libbpf_internal.h" 22 #include "strset.h" 23 24 #define BTF_EXTERN_SEC ".extern" 25 26 struct src_sec { 27 const char *sec_name; 28 /* positional (not necessarily ELF) index in an array of sections */ 29 int id; 30 /* positional (not necessarily ELF) index of a matching section in a final object file */ 31 int dst_id; 32 /* section data offset in a matching output section */ 33 int dst_off; 34 /* whether section is omitted from the final ELF file */ 35 bool skipped; 36 /* whether section is an ephemeral section, not mapped to an ELF section */ 37 bool ephemeral; 38 39 /* ELF info */ 40 size_t sec_idx; 41 Elf_Scn *scn; 42 Elf64_Shdr *shdr; 43 Elf_Data *data; 44 45 /* corresponding BTF DATASEC type ID */ 46 int sec_type_id; 47 }; 48 49 struct src_obj { 50 const char *filename; 51 int fd; 52 Elf *elf; 53 /* Section header strings section index */ 54 size_t shstrs_sec_idx; 55 /* SYMTAB section index */ 56 size_t symtab_sec_idx; 57 58 struct btf *btf; 59 struct btf_ext *btf_ext; 60 61 /* List of sections (including ephemeral). Slot zero is unused. */ 62 struct src_sec *secs; 63 int sec_cnt; 64 65 /* mapping of symbol indices from src to dst ELF */ 66 int *sym_map; 67 /* mapping from the src BTF type IDs to dst ones */ 68 int *btf_type_map; 69 }; 70 71 /* single .BTF.ext data section */ 72 struct btf_ext_sec_data { 73 size_t rec_cnt; 74 __u32 rec_sz; 75 void *recs; 76 }; 77 78 struct glob_sym { 79 /* ELF symbol index */ 80 int sym_idx; 81 /* associated section id for .ksyms, .kconfig, etc, but not .extern */ 82 int sec_id; 83 /* extern name offset in STRTAB */ 84 int name_off; 85 /* optional associated BTF type ID */ 86 int btf_id; 87 /* BTF type ID to which VAR/FUNC type is pointing to; used for 88 * rewriting types when extern VAR/FUNC is resolved to a concrete 89 * definition 90 */ 91 int underlying_btf_id; 92 /* sec_var index in the corresponding dst_sec, if exists */ 93 int var_idx; 94 95 /* extern or resolved/global symbol */ 96 bool is_extern; 97 /* weak or strong symbol, never goes back from strong to weak */ 98 bool is_weak; 99 }; 100 101 struct dst_sec { 102 char *sec_name; 103 /* positional (not necessarily ELF) index in an array of sections */ 104 int id; 105 106 bool ephemeral; 107 108 /* ELF info */ 109 size_t sec_idx; 110 Elf_Scn *scn; 111 Elf64_Shdr *shdr; 112 Elf_Data *data; 113 114 /* final output section size */ 115 int sec_sz; 116 /* final output contents of the section */ 117 void *raw_data; 118 119 /* corresponding STT_SECTION symbol index in SYMTAB */ 120 int sec_sym_idx; 121 122 /* section's DATASEC variable info, emitted on BTF finalization */ 123 bool has_btf; 124 int sec_var_cnt; 125 struct btf_var_secinfo *sec_vars; 126 127 /* section's .BTF.ext data */ 128 struct btf_ext_sec_data func_info; 129 struct btf_ext_sec_data line_info; 130 struct btf_ext_sec_data core_relo_info; 131 }; 132 133 struct bpf_linker { 134 char *filename; 135 int fd; 136 Elf *elf; 137 Elf64_Ehdr *elf_hdr; 138 139 /* Output sections metadata */ 140 struct dst_sec *secs; 141 int sec_cnt; 142 143 struct strset *strtab_strs; /* STRTAB unique strings */ 144 size_t strtab_sec_idx; /* STRTAB section index */ 145 size_t symtab_sec_idx; /* SYMTAB section index */ 146 147 struct btf *btf; 148 struct btf_ext *btf_ext; 149 150 /* global (including extern) ELF symbols */ 151 int glob_sym_cnt; 152 struct glob_sym *glob_syms; 153 }; 154 155 #define pr_warn_elf(fmt, ...) \ 156 libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1)) 157 158 static int init_output_elf(struct bpf_linker *linker, const char *file); 159 160 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, 161 const struct bpf_linker_file_opts *opts, 162 struct src_obj *obj); 163 static int linker_sanity_check_elf(struct src_obj *obj); 164 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec); 165 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec); 166 static int linker_sanity_check_btf(struct src_obj *obj); 167 static int linker_sanity_check_btf_ext(struct src_obj *obj); 168 static int linker_fixup_btf(struct src_obj *obj); 169 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj); 170 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj); 171 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj, 172 Elf64_Sym *sym, const char *sym_name, int src_sym_idx); 173 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj); 174 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj); 175 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj); 176 177 static int finalize_btf(struct bpf_linker *linker); 178 static int finalize_btf_ext(struct bpf_linker *linker); 179 bpf_linker__free(struct bpf_linker * linker)180 void bpf_linker__free(struct bpf_linker *linker) 181 { 182 int i; 183 184 if (!linker) 185 return; 186 187 free(linker->filename); 188 189 if (linker->elf) 190 elf_end(linker->elf); 191 192 if (linker->fd >= 0) 193 close(linker->fd); 194 195 strset__free(linker->strtab_strs); 196 197 btf__free(linker->btf); 198 btf_ext__free(linker->btf_ext); 199 200 for (i = 1; i < linker->sec_cnt; i++) { 201 struct dst_sec *sec = &linker->secs[i]; 202 203 free(sec->sec_name); 204 free(sec->raw_data); 205 free(sec->sec_vars); 206 207 free(sec->func_info.recs); 208 free(sec->line_info.recs); 209 free(sec->core_relo_info.recs); 210 } 211 free(linker->secs); 212 213 free(linker->glob_syms); 214 free(linker); 215 } 216 bpf_linker__new(const char * filename,struct bpf_linker_opts * opts)217 struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts) 218 { 219 struct bpf_linker *linker; 220 int err; 221 222 if (!OPTS_VALID(opts, bpf_linker_opts)) 223 return errno = EINVAL, NULL; 224 225 if (elf_version(EV_CURRENT) == EV_NONE) { 226 pr_warn_elf("libelf initialization failed"); 227 return errno = EINVAL, NULL; 228 } 229 230 linker = calloc(1, sizeof(*linker)); 231 if (!linker) 232 return errno = ENOMEM, NULL; 233 234 linker->fd = -1; 235 236 err = init_output_elf(linker, filename); 237 if (err) 238 goto err_out; 239 240 return linker; 241 242 err_out: 243 bpf_linker__free(linker); 244 return errno = -err, NULL; 245 } 246 add_dst_sec(struct bpf_linker * linker,const char * sec_name)247 static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name) 248 { 249 struct dst_sec *secs = linker->secs, *sec; 250 size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2; 251 252 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs)); 253 if (!secs) 254 return NULL; 255 256 /* zero out newly allocated memory */ 257 memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs)); 258 259 linker->secs = secs; 260 linker->sec_cnt = new_cnt; 261 262 sec = &linker->secs[new_cnt - 1]; 263 sec->id = new_cnt - 1; 264 sec->sec_name = strdup(sec_name); 265 if (!sec->sec_name) 266 return NULL; 267 268 return sec; 269 } 270 add_new_sym(struct bpf_linker * linker,size_t * sym_idx)271 static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx) 272 { 273 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx]; 274 Elf64_Sym *syms, *sym; 275 size_t sym_cnt = symtab->sec_sz / sizeof(*sym); 276 277 syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym)); 278 if (!syms) 279 return NULL; 280 281 sym = &syms[sym_cnt]; 282 memset(sym, 0, sizeof(*sym)); 283 284 symtab->raw_data = syms; 285 symtab->sec_sz += sizeof(*sym); 286 symtab->shdr->sh_size += sizeof(*sym); 287 symtab->data->d_size += sizeof(*sym); 288 289 if (sym_idx) 290 *sym_idx = sym_cnt; 291 292 return sym; 293 } 294 init_output_elf(struct bpf_linker * linker,const char * file)295 static int init_output_elf(struct bpf_linker *linker, const char *file) 296 { 297 int err, str_off; 298 Elf64_Sym *init_sym; 299 struct dst_sec *sec; 300 301 linker->filename = strdup(file); 302 if (!linker->filename) 303 return -ENOMEM; 304 305 linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644); 306 if (linker->fd < 0) { 307 err = -errno; 308 pr_warn("failed to create '%s': %d\n", file, err); 309 return err; 310 } 311 312 linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL); 313 if (!linker->elf) { 314 pr_warn_elf("failed to create ELF object"); 315 return -EINVAL; 316 } 317 318 /* ELF header */ 319 linker->elf_hdr = elf64_newehdr(linker->elf); 320 if (!linker->elf_hdr) { 321 pr_warn_elf("failed to create ELF header"); 322 return -EINVAL; 323 } 324 325 linker->elf_hdr->e_machine = EM_BPF; 326 linker->elf_hdr->e_type = ET_REL; 327 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 328 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB; 329 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 330 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB; 331 #else 332 #error "Unknown __BYTE_ORDER__" 333 #endif 334 335 /* STRTAB */ 336 /* initialize strset with an empty string to conform to ELF */ 337 linker->strtab_strs = strset__new(INT_MAX, "", sizeof("")); 338 if (libbpf_get_error(linker->strtab_strs)) 339 return libbpf_get_error(linker->strtab_strs); 340 341 sec = add_dst_sec(linker, ".strtab"); 342 if (!sec) 343 return -ENOMEM; 344 345 sec->scn = elf_newscn(linker->elf); 346 if (!sec->scn) { 347 pr_warn_elf("failed to create STRTAB section"); 348 return -EINVAL; 349 } 350 351 sec->shdr = elf64_getshdr(sec->scn); 352 if (!sec->shdr) 353 return -EINVAL; 354 355 sec->data = elf_newdata(sec->scn); 356 if (!sec->data) { 357 pr_warn_elf("failed to create STRTAB data"); 358 return -EINVAL; 359 } 360 361 str_off = strset__add_str(linker->strtab_strs, sec->sec_name); 362 if (str_off < 0) 363 return str_off; 364 365 sec->sec_idx = elf_ndxscn(sec->scn); 366 linker->elf_hdr->e_shstrndx = sec->sec_idx; 367 linker->strtab_sec_idx = sec->sec_idx; 368 369 sec->shdr->sh_name = str_off; 370 sec->shdr->sh_type = SHT_STRTAB; 371 sec->shdr->sh_flags = SHF_STRINGS; 372 sec->shdr->sh_offset = 0; 373 sec->shdr->sh_link = 0; 374 sec->shdr->sh_info = 0; 375 sec->shdr->sh_addralign = 1; 376 sec->shdr->sh_size = sec->sec_sz = 0; 377 sec->shdr->sh_entsize = 0; 378 379 /* SYMTAB */ 380 sec = add_dst_sec(linker, ".symtab"); 381 if (!sec) 382 return -ENOMEM; 383 384 sec->scn = elf_newscn(linker->elf); 385 if (!sec->scn) { 386 pr_warn_elf("failed to create SYMTAB section"); 387 return -EINVAL; 388 } 389 390 sec->shdr = elf64_getshdr(sec->scn); 391 if (!sec->shdr) 392 return -EINVAL; 393 394 sec->data = elf_newdata(sec->scn); 395 if (!sec->data) { 396 pr_warn_elf("failed to create SYMTAB data"); 397 return -EINVAL; 398 } 399 /* Ensure libelf translates byte-order of symbol records */ 400 sec->data->d_type = ELF_T_SYM; 401 402 str_off = strset__add_str(linker->strtab_strs, sec->sec_name); 403 if (str_off < 0) 404 return str_off; 405 406 sec->sec_idx = elf_ndxscn(sec->scn); 407 linker->symtab_sec_idx = sec->sec_idx; 408 409 sec->shdr->sh_name = str_off; 410 sec->shdr->sh_type = SHT_SYMTAB; 411 sec->shdr->sh_flags = 0; 412 sec->shdr->sh_offset = 0; 413 sec->shdr->sh_link = linker->strtab_sec_idx; 414 /* sh_info should be one greater than the index of the last local 415 * symbol (i.e., binding is STB_LOCAL). But why and who cares? 416 */ 417 sec->shdr->sh_info = 0; 418 sec->shdr->sh_addralign = 8; 419 sec->shdr->sh_entsize = sizeof(Elf64_Sym); 420 421 /* .BTF */ 422 linker->btf = btf__new_empty(); 423 err = libbpf_get_error(linker->btf); 424 if (err) 425 return err; 426 427 /* add the special all-zero symbol */ 428 init_sym = add_new_sym(linker, NULL); 429 if (!init_sym) 430 return -EINVAL; 431 432 init_sym->st_name = 0; 433 init_sym->st_info = 0; 434 init_sym->st_other = 0; 435 init_sym->st_shndx = SHN_UNDEF; 436 init_sym->st_value = 0; 437 init_sym->st_size = 0; 438 439 return 0; 440 } 441 bpf_linker__add_file(struct bpf_linker * linker,const char * filename,const struct bpf_linker_file_opts * opts)442 int bpf_linker__add_file(struct bpf_linker *linker, const char *filename, 443 const struct bpf_linker_file_opts *opts) 444 { 445 struct src_obj obj = {}; 446 int err = 0; 447 448 if (!OPTS_VALID(opts, bpf_linker_file_opts)) 449 return libbpf_err(-EINVAL); 450 451 if (!linker->elf) 452 return libbpf_err(-EINVAL); 453 454 err = err ?: linker_load_obj_file(linker, filename, opts, &obj); 455 err = err ?: linker_append_sec_data(linker, &obj); 456 err = err ?: linker_append_elf_syms(linker, &obj); 457 err = err ?: linker_append_elf_relos(linker, &obj); 458 err = err ?: linker_append_btf(linker, &obj); 459 err = err ?: linker_append_btf_ext(linker, &obj); 460 461 /* free up src_obj resources */ 462 free(obj.btf_type_map); 463 btf__free(obj.btf); 464 btf_ext__free(obj.btf_ext); 465 free(obj.secs); 466 free(obj.sym_map); 467 if (obj.elf) 468 elf_end(obj.elf); 469 if (obj.fd >= 0) 470 close(obj.fd); 471 472 return libbpf_err(err); 473 } 474 is_dwarf_sec_name(const char * name)475 static bool is_dwarf_sec_name(const char *name) 476 { 477 /* approximation, but the actual list is too long */ 478 return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0; 479 } 480 is_ignored_sec(struct src_sec * sec)481 static bool is_ignored_sec(struct src_sec *sec) 482 { 483 Elf64_Shdr *shdr = sec->shdr; 484 const char *name = sec->sec_name; 485 486 /* no special handling of .strtab */ 487 if (shdr->sh_type == SHT_STRTAB) 488 return true; 489 490 /* ignore .llvm_addrsig section as well */ 491 if (shdr->sh_type == SHT_LLVM_ADDRSIG) 492 return true; 493 494 /* no subprograms will lead to an empty .text section, ignore it */ 495 if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 && 496 strcmp(sec->sec_name, ".text") == 0) 497 return true; 498 499 /* DWARF sections */ 500 if (is_dwarf_sec_name(sec->sec_name)) 501 return true; 502 503 if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) { 504 name += sizeof(".rel") - 1; 505 /* DWARF section relocations */ 506 if (is_dwarf_sec_name(name)) 507 return true; 508 509 /* .BTF and .BTF.ext don't need relocations */ 510 if (strcmp(name, BTF_ELF_SEC) == 0 || 511 strcmp(name, BTF_EXT_ELF_SEC) == 0) 512 return true; 513 } 514 515 return false; 516 } 517 add_src_sec(struct src_obj * obj,const char * sec_name)518 static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name) 519 { 520 struct src_sec *secs = obj->secs, *sec; 521 size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2; 522 523 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs)); 524 if (!secs) 525 return NULL; 526 527 /* zero out newly allocated memory */ 528 memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs)); 529 530 obj->secs = secs; 531 obj->sec_cnt = new_cnt; 532 533 sec = &obj->secs[new_cnt - 1]; 534 sec->id = new_cnt - 1; 535 sec->sec_name = sec_name; 536 537 return sec; 538 } 539 linker_load_obj_file(struct bpf_linker * linker,const char * filename,const struct bpf_linker_file_opts * opts,struct src_obj * obj)540 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, 541 const struct bpf_linker_file_opts *opts, 542 struct src_obj *obj) 543 { 544 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 545 const int host_endianness = ELFDATA2LSB; 546 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 547 const int host_endianness = ELFDATA2MSB; 548 #else 549 #error "Unknown __BYTE_ORDER__" 550 #endif 551 int err = 0; 552 Elf_Scn *scn; 553 Elf_Data *data; 554 Elf64_Ehdr *ehdr; 555 Elf64_Shdr *shdr; 556 struct src_sec *sec; 557 558 pr_debug("linker: adding object file '%s'...\n", filename); 559 560 obj->filename = filename; 561 562 obj->fd = open(filename, O_RDONLY | O_CLOEXEC); 563 if (obj->fd < 0) { 564 err = -errno; 565 pr_warn("failed to open file '%s': %d\n", filename, err); 566 return err; 567 } 568 obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL); 569 if (!obj->elf) { 570 pr_warn_elf("failed to parse ELF file '%s'", filename); 571 return -EINVAL; 572 } 573 574 /* Sanity check ELF file high-level properties */ 575 ehdr = elf64_getehdr(obj->elf); 576 if (!ehdr) { 577 pr_warn_elf("failed to get ELF header for %s", filename); 578 return -EINVAL; 579 } 580 if (ehdr->e_ident[EI_DATA] != host_endianness) { 581 err = -EOPNOTSUPP; 582 pr_warn_elf("unsupported byte order of ELF file %s", filename); 583 return err; 584 } 585 if (ehdr->e_type != ET_REL 586 || ehdr->e_machine != EM_BPF 587 || ehdr->e_ident[EI_CLASS] != ELFCLASS64) { 588 err = -EOPNOTSUPP; 589 pr_warn_elf("unsupported kind of ELF file %s", filename); 590 return err; 591 } 592 593 if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) { 594 pr_warn_elf("failed to get SHSTRTAB section index for %s", filename); 595 return -EINVAL; 596 } 597 598 scn = NULL; 599 while ((scn = elf_nextscn(obj->elf, scn)) != NULL) { 600 size_t sec_idx = elf_ndxscn(scn); 601 const char *sec_name; 602 603 shdr = elf64_getshdr(scn); 604 if (!shdr) { 605 pr_warn_elf("failed to get section #%zu header for %s", 606 sec_idx, filename); 607 return -EINVAL; 608 } 609 610 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name); 611 if (!sec_name) { 612 pr_warn_elf("failed to get section #%zu name for %s", 613 sec_idx, filename); 614 return -EINVAL; 615 } 616 617 data = elf_getdata(scn, 0); 618 if (!data) { 619 pr_warn_elf("failed to get section #%zu (%s) data from %s", 620 sec_idx, sec_name, filename); 621 return -EINVAL; 622 } 623 624 sec = add_src_sec(obj, sec_name); 625 if (!sec) 626 return -ENOMEM; 627 628 sec->scn = scn; 629 sec->shdr = shdr; 630 sec->data = data; 631 sec->sec_idx = elf_ndxscn(scn); 632 633 if (is_ignored_sec(sec)) { 634 sec->skipped = true; 635 continue; 636 } 637 638 switch (shdr->sh_type) { 639 case SHT_SYMTAB: 640 if (obj->symtab_sec_idx) { 641 err = -EOPNOTSUPP; 642 pr_warn("multiple SYMTAB sections found, not supported\n"); 643 return err; 644 } 645 obj->symtab_sec_idx = sec_idx; 646 break; 647 case SHT_STRTAB: 648 /* we'll construct our own string table */ 649 break; 650 case SHT_PROGBITS: 651 if (strcmp(sec_name, BTF_ELF_SEC) == 0) { 652 obj->btf = btf__new(data->d_buf, shdr->sh_size); 653 err = libbpf_get_error(obj->btf); 654 if (err) { 655 pr_warn("failed to parse .BTF from %s: %d\n", filename, err); 656 return err; 657 } 658 sec->skipped = true; 659 continue; 660 } 661 if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) { 662 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size); 663 err = libbpf_get_error(obj->btf_ext); 664 if (err) { 665 pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err); 666 return err; 667 } 668 sec->skipped = true; 669 continue; 670 } 671 672 /* data & code */ 673 break; 674 case SHT_NOBITS: 675 /* BSS */ 676 break; 677 case SHT_REL: 678 /* relocations */ 679 break; 680 default: 681 pr_warn("unrecognized section #%zu (%s) in %s\n", 682 sec_idx, sec_name, filename); 683 err = -EINVAL; 684 return err; 685 } 686 } 687 688 err = err ?: linker_sanity_check_elf(obj); 689 err = err ?: linker_sanity_check_btf(obj); 690 err = err ?: linker_sanity_check_btf_ext(obj); 691 err = err ?: linker_fixup_btf(obj); 692 693 return err; 694 } 695 linker_sanity_check_elf(struct src_obj * obj)696 static int linker_sanity_check_elf(struct src_obj *obj) 697 { 698 struct src_sec *sec; 699 int i, err; 700 701 if (!obj->symtab_sec_idx) { 702 pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename); 703 return -EINVAL; 704 } 705 if (!obj->shstrs_sec_idx) { 706 pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename); 707 return -EINVAL; 708 } 709 710 for (i = 1; i < obj->sec_cnt; i++) { 711 sec = &obj->secs[i]; 712 713 if (sec->sec_name[0] == '\0') { 714 pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename); 715 return -EINVAL; 716 } 717 718 if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) 719 return -EINVAL; 720 if (sec->shdr->sh_addralign != sec->data->d_align) 721 return -EINVAL; 722 723 if (sec->shdr->sh_size != sec->data->d_size) 724 return -EINVAL; 725 726 switch (sec->shdr->sh_type) { 727 case SHT_SYMTAB: 728 err = linker_sanity_check_elf_symtab(obj, sec); 729 if (err) 730 return err; 731 break; 732 case SHT_STRTAB: 733 break; 734 case SHT_PROGBITS: 735 if (sec->shdr->sh_flags & SHF_EXECINSTR) { 736 if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) 737 return -EINVAL; 738 } 739 break; 740 case SHT_NOBITS: 741 break; 742 case SHT_REL: 743 err = linker_sanity_check_elf_relos(obj, sec); 744 if (err) 745 return err; 746 break; 747 case SHT_LLVM_ADDRSIG: 748 break; 749 default: 750 pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n", 751 sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename); 752 return -EINVAL; 753 } 754 } 755 756 return 0; 757 } 758 linker_sanity_check_elf_symtab(struct src_obj * obj,struct src_sec * sec)759 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec) 760 { 761 struct src_sec *link_sec; 762 Elf64_Sym *sym; 763 int i, n; 764 765 if (sec->shdr->sh_entsize != sizeof(Elf64_Sym)) 766 return -EINVAL; 767 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0) 768 return -EINVAL; 769 770 if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) { 771 pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n", 772 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); 773 return -EINVAL; 774 } 775 link_sec = &obj->secs[sec->shdr->sh_link]; 776 if (link_sec->shdr->sh_type != SHT_STRTAB) { 777 pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n", 778 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); 779 return -EINVAL; 780 } 781 782 n = sec->shdr->sh_size / sec->shdr->sh_entsize; 783 sym = sec->data->d_buf; 784 for (i = 0; i < n; i++, sym++) { 785 int sym_type = ELF64_ST_TYPE(sym->st_info); 786 int sym_bind = ELF64_ST_BIND(sym->st_info); 787 int sym_vis = ELF64_ST_VISIBILITY(sym->st_other); 788 789 if (i == 0) { 790 if (sym->st_name != 0 || sym->st_info != 0 791 || sym->st_other != 0 || sym->st_shndx != 0 792 || sym->st_value != 0 || sym->st_size != 0) { 793 pr_warn("ELF sym #0 is invalid in %s\n", obj->filename); 794 return -EINVAL; 795 } 796 continue; 797 } 798 if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) { 799 pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n", 800 i, sec->sec_idx, sym_bind); 801 return -EINVAL; 802 } 803 if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) { 804 pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n", 805 i, sec->sec_idx, sym_vis); 806 return -EINVAL; 807 } 808 if (sym->st_shndx == 0) { 809 if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL 810 || sym->st_value != 0 || sym->st_size != 0) { 811 pr_warn("ELF sym #%d is invalid extern symbol in %s\n", 812 i, obj->filename); 813 814 return -EINVAL; 815 } 816 continue; 817 } 818 if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) { 819 pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n", 820 i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename); 821 return -EINVAL; 822 } 823 if (sym_type == STT_SECTION) { 824 if (sym->st_value != 0) 825 return -EINVAL; 826 continue; 827 } 828 } 829 830 return 0; 831 } 832 linker_sanity_check_elf_relos(struct src_obj * obj,struct src_sec * sec)833 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec) 834 { 835 struct src_sec *link_sec, *sym_sec; 836 Elf64_Rel *relo; 837 int i, n; 838 839 if (sec->shdr->sh_entsize != sizeof(Elf64_Rel)) 840 return -EINVAL; 841 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0) 842 return -EINVAL; 843 844 /* SHT_REL's sh_link should point to SYMTAB */ 845 if (sec->shdr->sh_link != obj->symtab_sec_idx) { 846 pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n", 847 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); 848 return -EINVAL; 849 } 850 851 /* SHT_REL's sh_info points to relocated section */ 852 if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) { 853 pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n", 854 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename); 855 return -EINVAL; 856 } 857 link_sec = &obj->secs[sec->shdr->sh_info]; 858 859 /* .rel<secname> -> <secname> pattern is followed */ 860 if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0 861 || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) { 862 pr_warn("ELF relo section #%zu name has invalid name in %s\n", 863 sec->sec_idx, obj->filename); 864 return -EINVAL; 865 } 866 867 /* don't further validate relocations for ignored sections */ 868 if (link_sec->skipped) 869 return 0; 870 871 /* relocatable section is data or instructions */ 872 if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) { 873 pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n", 874 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename); 875 return -EINVAL; 876 } 877 878 /* check sanity of each relocation */ 879 n = sec->shdr->sh_size / sec->shdr->sh_entsize; 880 relo = sec->data->d_buf; 881 sym_sec = &obj->secs[obj->symtab_sec_idx]; 882 for (i = 0; i < n; i++, relo++) { 883 size_t sym_idx = ELF64_R_SYM(relo->r_info); 884 size_t sym_type = ELF64_R_TYPE(relo->r_info); 885 886 if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 && 887 sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) { 888 pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n", 889 i, sec->sec_idx, sym_type, obj->filename); 890 return -EINVAL; 891 } 892 893 if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) { 894 pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n", 895 i, sec->sec_idx, sym_idx, obj->filename); 896 return -EINVAL; 897 } 898 899 if (link_sec->shdr->sh_flags & SHF_EXECINSTR) { 900 if (relo->r_offset % sizeof(struct bpf_insn) != 0) { 901 pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n", 902 i, sec->sec_idx, sym_idx, obj->filename); 903 return -EINVAL; 904 } 905 } 906 } 907 908 return 0; 909 } 910 check_btf_type_id(__u32 * type_id,void * ctx)911 static int check_btf_type_id(__u32 *type_id, void *ctx) 912 { 913 struct btf *btf = ctx; 914 915 if (*type_id >= btf__type_cnt(btf)) 916 return -EINVAL; 917 918 return 0; 919 } 920 check_btf_str_off(__u32 * str_off,void * ctx)921 static int check_btf_str_off(__u32 *str_off, void *ctx) 922 { 923 struct btf *btf = ctx; 924 const char *s; 925 926 s = btf__str_by_offset(btf, *str_off); 927 928 if (!s) 929 return -EINVAL; 930 931 return 0; 932 } 933 linker_sanity_check_btf(struct src_obj * obj)934 static int linker_sanity_check_btf(struct src_obj *obj) 935 { 936 struct btf_type *t; 937 int i, n, err = 0; 938 939 if (!obj->btf) 940 return 0; 941 942 n = btf__type_cnt(obj->btf); 943 for (i = 1; i < n; i++) { 944 t = btf_type_by_id(obj->btf, i); 945 946 err = err ?: btf_type_visit_type_ids(t, check_btf_type_id, obj->btf); 947 err = err ?: btf_type_visit_str_offs(t, check_btf_str_off, obj->btf); 948 if (err) 949 return err; 950 } 951 952 return 0; 953 } 954 linker_sanity_check_btf_ext(struct src_obj * obj)955 static int linker_sanity_check_btf_ext(struct src_obj *obj) 956 { 957 int err = 0; 958 959 if (!obj->btf_ext) 960 return 0; 961 962 /* can't use .BTF.ext without .BTF */ 963 if (!obj->btf) 964 return -EINVAL; 965 966 err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf); 967 err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf); 968 if (err) 969 return err; 970 971 return 0; 972 } 973 init_sec(struct bpf_linker * linker,struct dst_sec * dst_sec,struct src_sec * src_sec)974 static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec) 975 { 976 Elf_Scn *scn; 977 Elf_Data *data; 978 Elf64_Shdr *shdr; 979 int name_off; 980 981 dst_sec->sec_sz = 0; 982 dst_sec->sec_idx = 0; 983 dst_sec->ephemeral = src_sec->ephemeral; 984 985 /* ephemeral sections are just thin section shells lacking most parts */ 986 if (src_sec->ephemeral) 987 return 0; 988 989 scn = elf_newscn(linker->elf); 990 if (!scn) 991 return -ENOMEM; 992 data = elf_newdata(scn); 993 if (!data) 994 return -ENOMEM; 995 shdr = elf64_getshdr(scn); 996 if (!shdr) 997 return -ENOMEM; 998 999 dst_sec->scn = scn; 1000 dst_sec->shdr = shdr; 1001 dst_sec->data = data; 1002 dst_sec->sec_idx = elf_ndxscn(scn); 1003 1004 name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name); 1005 if (name_off < 0) 1006 return name_off; 1007 1008 shdr->sh_name = name_off; 1009 shdr->sh_type = src_sec->shdr->sh_type; 1010 shdr->sh_flags = src_sec->shdr->sh_flags; 1011 shdr->sh_size = 0; 1012 /* sh_link and sh_info have different meaning for different types of 1013 * sections, so we leave it up to the caller code to fill them in, if 1014 * necessary 1015 */ 1016 shdr->sh_link = 0; 1017 shdr->sh_info = 0; 1018 shdr->sh_addralign = src_sec->shdr->sh_addralign; 1019 shdr->sh_entsize = src_sec->shdr->sh_entsize; 1020 1021 data->d_type = src_sec->data->d_type; 1022 data->d_size = 0; 1023 data->d_buf = NULL; 1024 data->d_align = src_sec->data->d_align; 1025 data->d_off = 0; 1026 1027 return 0; 1028 } 1029 find_dst_sec_by_name(struct bpf_linker * linker,const char * sec_name)1030 static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name) 1031 { 1032 struct dst_sec *sec; 1033 int i; 1034 1035 for (i = 1; i < linker->sec_cnt; i++) { 1036 sec = &linker->secs[i]; 1037 1038 if (strcmp(sec->sec_name, sec_name) == 0) 1039 return sec; 1040 } 1041 1042 return NULL; 1043 } 1044 secs_match(struct dst_sec * dst,struct src_sec * src)1045 static bool secs_match(struct dst_sec *dst, struct src_sec *src) 1046 { 1047 if (dst->ephemeral || src->ephemeral) 1048 return true; 1049 1050 if (dst->shdr->sh_type != src->shdr->sh_type) { 1051 pr_warn("sec %s types mismatch\n", dst->sec_name); 1052 return false; 1053 } 1054 if (dst->shdr->sh_flags != src->shdr->sh_flags) { 1055 pr_warn("sec %s flags mismatch\n", dst->sec_name); 1056 return false; 1057 } 1058 if (dst->shdr->sh_entsize != src->shdr->sh_entsize) { 1059 pr_warn("sec %s entsize mismatch\n", dst->sec_name); 1060 return false; 1061 } 1062 1063 return true; 1064 } 1065 sec_content_is_same(struct dst_sec * dst_sec,struct src_sec * src_sec)1066 static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec) 1067 { 1068 if (dst_sec->sec_sz != src_sec->shdr->sh_size) 1069 return false; 1070 if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0) 1071 return false; 1072 return true; 1073 } 1074 extend_sec(struct bpf_linker * linker,struct dst_sec * dst,struct src_sec * src)1075 static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src) 1076 { 1077 void *tmp; 1078 size_t dst_align, src_align; 1079 size_t dst_align_sz, dst_final_sz; 1080 int err; 1081 1082 /* Ephemeral source section doesn't contribute anything to ELF 1083 * section data. 1084 */ 1085 if (src->ephemeral) 1086 return 0; 1087 1088 /* Some sections (like .maps) can contain both externs (and thus be 1089 * ephemeral) and non-externs (map definitions). So it's possible that 1090 * it has to be "upgraded" from ephemeral to non-ephemeral when the 1091 * first non-ephemeral entity appears. In such case, we add ELF 1092 * section, data, etc. 1093 */ 1094 if (dst->ephemeral) { 1095 err = init_sec(linker, dst, src); 1096 if (err) 1097 return err; 1098 } 1099 1100 dst_align = dst->shdr->sh_addralign; 1101 src_align = src->shdr->sh_addralign; 1102 if (dst_align == 0) 1103 dst_align = 1; 1104 if (dst_align < src_align) 1105 dst_align = src_align; 1106 1107 dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align; 1108 1109 /* no need to re-align final size */ 1110 dst_final_sz = dst_align_sz + src->shdr->sh_size; 1111 1112 if (src->shdr->sh_type != SHT_NOBITS) { 1113 tmp = realloc(dst->raw_data, dst_final_sz); 1114 /* If dst_align_sz == 0, realloc() behaves in a special way: 1115 * 1. When dst->raw_data is NULL it returns: 1116 * "either NULL or a pointer suitable to be passed to free()" [1]. 1117 * 2. When dst->raw_data is not-NULL it frees dst->raw_data and returns NULL, 1118 * thus invalidating any "pointer suitable to be passed to free()" obtained 1119 * at step (1). 1120 * 1121 * The dst_align_sz > 0 check avoids error exit after (2), otherwise 1122 * dst->raw_data would be freed again in bpf_linker__free(). 1123 * 1124 * [1] man 3 realloc 1125 */ 1126 if (!tmp && dst_align_sz > 0) 1127 return -ENOMEM; 1128 dst->raw_data = tmp; 1129 1130 /* pad dst section, if it's alignment forced size increase */ 1131 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz); 1132 /* now copy src data at a properly aligned offset */ 1133 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size); 1134 } 1135 1136 dst->sec_sz = dst_final_sz; 1137 dst->shdr->sh_size = dst_final_sz; 1138 dst->data->d_size = dst_final_sz; 1139 1140 dst->shdr->sh_addralign = dst_align; 1141 dst->data->d_align = dst_align; 1142 1143 src->dst_off = dst_align_sz; 1144 1145 return 0; 1146 } 1147 is_data_sec(struct src_sec * sec)1148 static bool is_data_sec(struct src_sec *sec) 1149 { 1150 if (!sec || sec->skipped) 1151 return false; 1152 /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */ 1153 if (sec->ephemeral) 1154 return true; 1155 return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS; 1156 } 1157 is_relo_sec(struct src_sec * sec)1158 static bool is_relo_sec(struct src_sec *sec) 1159 { 1160 if (!sec || sec->skipped || sec->ephemeral) 1161 return false; 1162 return sec->shdr->sh_type == SHT_REL; 1163 } 1164 linker_append_sec_data(struct bpf_linker * linker,struct src_obj * obj)1165 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj) 1166 { 1167 int i, err; 1168 1169 for (i = 1; i < obj->sec_cnt; i++) { 1170 struct src_sec *src_sec; 1171 struct dst_sec *dst_sec; 1172 1173 src_sec = &obj->secs[i]; 1174 if (!is_data_sec(src_sec)) 1175 continue; 1176 1177 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name); 1178 if (!dst_sec) { 1179 dst_sec = add_dst_sec(linker, src_sec->sec_name); 1180 if (!dst_sec) 1181 return -ENOMEM; 1182 err = init_sec(linker, dst_sec, src_sec); 1183 if (err) { 1184 pr_warn("failed to init section '%s'\n", src_sec->sec_name); 1185 return err; 1186 } 1187 } else { 1188 if (!secs_match(dst_sec, src_sec)) { 1189 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name); 1190 return -1; 1191 } 1192 1193 /* "license" and "version" sections are deduped */ 1194 if (strcmp(src_sec->sec_name, "license") == 0 1195 || strcmp(src_sec->sec_name, "version") == 0) { 1196 if (!sec_content_is_same(dst_sec, src_sec)) { 1197 pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name); 1198 return -EINVAL; 1199 } 1200 src_sec->skipped = true; 1201 src_sec->dst_id = dst_sec->id; 1202 continue; 1203 } 1204 } 1205 1206 /* record mapped section index */ 1207 src_sec->dst_id = dst_sec->id; 1208 1209 err = extend_sec(linker, dst_sec, src_sec); 1210 if (err) 1211 return err; 1212 } 1213 1214 return 0; 1215 } 1216 linker_append_elf_syms(struct bpf_linker * linker,struct src_obj * obj)1217 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj) 1218 { 1219 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx]; 1220 Elf64_Sym *sym = symtab->data->d_buf; 1221 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err; 1222 int str_sec_idx = symtab->shdr->sh_link; 1223 const char *sym_name; 1224 1225 obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map)); 1226 if (!obj->sym_map) 1227 return -ENOMEM; 1228 1229 for (i = 0; i < n; i++, sym++) { 1230 /* We already validated all-zero symbol #0 and we already 1231 * appended it preventively to the final SYMTAB, so skip it. 1232 */ 1233 if (i == 0) 1234 continue; 1235 1236 sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name); 1237 if (!sym_name) { 1238 pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename); 1239 return -EINVAL; 1240 } 1241 1242 err = linker_append_elf_sym(linker, obj, sym, sym_name, i); 1243 if (err) 1244 return err; 1245 } 1246 1247 return 0; 1248 } 1249 get_sym_by_idx(struct bpf_linker * linker,size_t sym_idx)1250 static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx) 1251 { 1252 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx]; 1253 Elf64_Sym *syms = symtab->raw_data; 1254 1255 return &syms[sym_idx]; 1256 } 1257 find_glob_sym(struct bpf_linker * linker,const char * sym_name)1258 static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name) 1259 { 1260 struct glob_sym *glob_sym; 1261 const char *name; 1262 int i; 1263 1264 for (i = 0; i < linker->glob_sym_cnt; i++) { 1265 glob_sym = &linker->glob_syms[i]; 1266 name = strset__data(linker->strtab_strs) + glob_sym->name_off; 1267 1268 if (strcmp(name, sym_name) == 0) 1269 return glob_sym; 1270 } 1271 1272 return NULL; 1273 } 1274 add_glob_sym(struct bpf_linker * linker)1275 static struct glob_sym *add_glob_sym(struct bpf_linker *linker) 1276 { 1277 struct glob_sym *syms, *sym; 1278 1279 syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1, 1280 sizeof(*linker->glob_syms)); 1281 if (!syms) 1282 return NULL; 1283 1284 sym = &syms[linker->glob_sym_cnt]; 1285 memset(sym, 0, sizeof(*sym)); 1286 sym->var_idx = -1; 1287 1288 linker->glob_syms = syms; 1289 linker->glob_sym_cnt++; 1290 1291 return sym; 1292 } 1293 glob_sym_btf_matches(const char * sym_name,bool exact,const struct btf * btf1,__u32 id1,const struct btf * btf2,__u32 id2)1294 static bool glob_sym_btf_matches(const char *sym_name, bool exact, 1295 const struct btf *btf1, __u32 id1, 1296 const struct btf *btf2, __u32 id2) 1297 { 1298 const struct btf_type *t1, *t2; 1299 bool is_static1, is_static2; 1300 const char *n1, *n2; 1301 int i, n; 1302 1303 recur: 1304 n1 = n2 = NULL; 1305 t1 = skip_mods_and_typedefs(btf1, id1, &id1); 1306 t2 = skip_mods_and_typedefs(btf2, id2, &id2); 1307 1308 /* check if only one side is FWD, otherwise handle with common logic */ 1309 if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) { 1310 n1 = btf__str_by_offset(btf1, t1->name_off); 1311 n2 = btf__str_by_offset(btf2, t2->name_off); 1312 if (strcmp(n1, n2) != 0) { 1313 pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n", 1314 sym_name, n1, n2); 1315 return false; 1316 } 1317 /* validate if FWD kind matches concrete kind */ 1318 if (btf_is_fwd(t1)) { 1319 if (btf_kflag(t1) && btf_is_union(t2)) 1320 return true; 1321 if (!btf_kflag(t1) && btf_is_struct(t2)) 1322 return true; 1323 pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n", 1324 sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2)); 1325 } else { 1326 if (btf_kflag(t2) && btf_is_union(t1)) 1327 return true; 1328 if (!btf_kflag(t2) && btf_is_struct(t1)) 1329 return true; 1330 pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n", 1331 sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1)); 1332 } 1333 return false; 1334 } 1335 1336 if (btf_kind(t1) != btf_kind(t2)) { 1337 pr_warn("global '%s': incompatible BTF kinds %s and %s\n", 1338 sym_name, btf_kind_str(t1), btf_kind_str(t2)); 1339 return false; 1340 } 1341 1342 switch (btf_kind(t1)) { 1343 case BTF_KIND_STRUCT: 1344 case BTF_KIND_UNION: 1345 case BTF_KIND_ENUM: 1346 case BTF_KIND_ENUM64: 1347 case BTF_KIND_FWD: 1348 case BTF_KIND_FUNC: 1349 case BTF_KIND_VAR: 1350 n1 = btf__str_by_offset(btf1, t1->name_off); 1351 n2 = btf__str_by_offset(btf2, t2->name_off); 1352 if (strcmp(n1, n2) != 0) { 1353 pr_warn("global '%s': incompatible %s names '%s' and '%s'\n", 1354 sym_name, btf_kind_str(t1), n1, n2); 1355 return false; 1356 } 1357 break; 1358 default: 1359 break; 1360 } 1361 1362 switch (btf_kind(t1)) { 1363 case BTF_KIND_UNKN: /* void */ 1364 case BTF_KIND_FWD: 1365 return true; 1366 case BTF_KIND_INT: 1367 case BTF_KIND_FLOAT: 1368 case BTF_KIND_ENUM: 1369 case BTF_KIND_ENUM64: 1370 /* ignore encoding for int and enum values for enum */ 1371 if (t1->size != t2->size) { 1372 pr_warn("global '%s': incompatible %s '%s' size %u and %u\n", 1373 sym_name, btf_kind_str(t1), n1, t1->size, t2->size); 1374 return false; 1375 } 1376 return true; 1377 case BTF_KIND_PTR: 1378 /* just validate overall shape of the referenced type, so no 1379 * contents comparison for struct/union, and allowd fwd vs 1380 * struct/union 1381 */ 1382 exact = false; 1383 id1 = t1->type; 1384 id2 = t2->type; 1385 goto recur; 1386 case BTF_KIND_ARRAY: 1387 /* ignore index type and array size */ 1388 id1 = btf_array(t1)->type; 1389 id2 = btf_array(t2)->type; 1390 goto recur; 1391 case BTF_KIND_FUNC: 1392 /* extern and global linkages are compatible */ 1393 is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC; 1394 is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC; 1395 if (is_static1 != is_static2) { 1396 pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1); 1397 return false; 1398 } 1399 1400 id1 = t1->type; 1401 id2 = t2->type; 1402 goto recur; 1403 case BTF_KIND_VAR: 1404 /* extern and global linkages are compatible */ 1405 is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC; 1406 is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC; 1407 if (is_static1 != is_static2) { 1408 pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1); 1409 return false; 1410 } 1411 1412 id1 = t1->type; 1413 id2 = t2->type; 1414 goto recur; 1415 case BTF_KIND_STRUCT: 1416 case BTF_KIND_UNION: { 1417 const struct btf_member *m1, *m2; 1418 1419 if (!exact) 1420 return true; 1421 1422 if (btf_vlen(t1) != btf_vlen(t2)) { 1423 pr_warn("global '%s': incompatible number of %s fields %u and %u\n", 1424 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2)); 1425 return false; 1426 } 1427 1428 n = btf_vlen(t1); 1429 m1 = btf_members(t1); 1430 m2 = btf_members(t2); 1431 for (i = 0; i < n; i++, m1++, m2++) { 1432 n1 = btf__str_by_offset(btf1, m1->name_off); 1433 n2 = btf__str_by_offset(btf2, m2->name_off); 1434 if (strcmp(n1, n2) != 0) { 1435 pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n", 1436 sym_name, i, n1, n2); 1437 return false; 1438 } 1439 if (m1->offset != m2->offset) { 1440 pr_warn("global '%s': incompatible field #%d ('%s') offsets\n", 1441 sym_name, i, n1); 1442 return false; 1443 } 1444 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type)) 1445 return false; 1446 } 1447 1448 return true; 1449 } 1450 case BTF_KIND_FUNC_PROTO: { 1451 const struct btf_param *m1, *m2; 1452 1453 if (btf_vlen(t1) != btf_vlen(t2)) { 1454 pr_warn("global '%s': incompatible number of %s params %u and %u\n", 1455 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2)); 1456 return false; 1457 } 1458 1459 n = btf_vlen(t1); 1460 m1 = btf_params(t1); 1461 m2 = btf_params(t2); 1462 for (i = 0; i < n; i++, m1++, m2++) { 1463 /* ignore func arg names */ 1464 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type)) 1465 return false; 1466 } 1467 1468 /* now check return type as well */ 1469 id1 = t1->type; 1470 id2 = t2->type; 1471 goto recur; 1472 } 1473 1474 /* skip_mods_and_typedefs() make this impossible */ 1475 case BTF_KIND_TYPEDEF: 1476 case BTF_KIND_VOLATILE: 1477 case BTF_KIND_CONST: 1478 case BTF_KIND_RESTRICT: 1479 /* DATASECs are never compared with each other */ 1480 case BTF_KIND_DATASEC: 1481 default: 1482 pr_warn("global '%s': unsupported BTF kind %s\n", 1483 sym_name, btf_kind_str(t1)); 1484 return false; 1485 } 1486 } 1487 map_defs_match(const char * sym_name,const struct btf * main_btf,const struct btf_map_def * main_def,const struct btf_map_def * main_inner_def,const struct btf * extra_btf,const struct btf_map_def * extra_def,const struct btf_map_def * extra_inner_def)1488 static bool map_defs_match(const char *sym_name, 1489 const struct btf *main_btf, 1490 const struct btf_map_def *main_def, 1491 const struct btf_map_def *main_inner_def, 1492 const struct btf *extra_btf, 1493 const struct btf_map_def *extra_def, 1494 const struct btf_map_def *extra_inner_def) 1495 { 1496 const char *reason; 1497 1498 if (main_def->map_type != extra_def->map_type) { 1499 reason = "type"; 1500 goto mismatch; 1501 } 1502 1503 /* check key type/size match */ 1504 if (main_def->key_size != extra_def->key_size) { 1505 reason = "key_size"; 1506 goto mismatch; 1507 } 1508 if (!!main_def->key_type_id != !!extra_def->key_type_id) { 1509 reason = "key type"; 1510 goto mismatch; 1511 } 1512 if ((main_def->parts & MAP_DEF_KEY_TYPE) 1513 && !glob_sym_btf_matches(sym_name, true /*exact*/, 1514 main_btf, main_def->key_type_id, 1515 extra_btf, extra_def->key_type_id)) { 1516 reason = "key type"; 1517 goto mismatch; 1518 } 1519 1520 /* validate value type/size match */ 1521 if (main_def->value_size != extra_def->value_size) { 1522 reason = "value_size"; 1523 goto mismatch; 1524 } 1525 if (!!main_def->value_type_id != !!extra_def->value_type_id) { 1526 reason = "value type"; 1527 goto mismatch; 1528 } 1529 if ((main_def->parts & MAP_DEF_VALUE_TYPE) 1530 && !glob_sym_btf_matches(sym_name, true /*exact*/, 1531 main_btf, main_def->value_type_id, 1532 extra_btf, extra_def->value_type_id)) { 1533 reason = "key type"; 1534 goto mismatch; 1535 } 1536 1537 if (main_def->max_entries != extra_def->max_entries) { 1538 reason = "max_entries"; 1539 goto mismatch; 1540 } 1541 if (main_def->map_flags != extra_def->map_flags) { 1542 reason = "map_flags"; 1543 goto mismatch; 1544 } 1545 if (main_def->numa_node != extra_def->numa_node) { 1546 reason = "numa_node"; 1547 goto mismatch; 1548 } 1549 if (main_def->pinning != extra_def->pinning) { 1550 reason = "pinning"; 1551 goto mismatch; 1552 } 1553 1554 if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) { 1555 reason = "inner map"; 1556 goto mismatch; 1557 } 1558 1559 if (main_def->parts & MAP_DEF_INNER_MAP) { 1560 char inner_map_name[128]; 1561 1562 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name); 1563 1564 return map_defs_match(inner_map_name, 1565 main_btf, main_inner_def, NULL, 1566 extra_btf, extra_inner_def, NULL); 1567 } 1568 1569 return true; 1570 1571 mismatch: 1572 pr_warn("global '%s': map %s mismatch\n", sym_name, reason); 1573 return false; 1574 } 1575 glob_map_defs_match(const char * sym_name,struct bpf_linker * linker,struct glob_sym * glob_sym,struct src_obj * obj,Elf64_Sym * sym,int btf_id)1576 static bool glob_map_defs_match(const char *sym_name, 1577 struct bpf_linker *linker, struct glob_sym *glob_sym, 1578 struct src_obj *obj, Elf64_Sym *sym, int btf_id) 1579 { 1580 struct btf_map_def dst_def = {}, dst_inner_def = {}; 1581 struct btf_map_def src_def = {}, src_inner_def = {}; 1582 const struct btf_type *t; 1583 int err; 1584 1585 t = btf__type_by_id(obj->btf, btf_id); 1586 if (!btf_is_var(t)) { 1587 pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id); 1588 return false; 1589 } 1590 t = skip_mods_and_typedefs(obj->btf, t->type, NULL); 1591 1592 err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def); 1593 if (err) { 1594 pr_warn("global '%s': invalid map definition\n", sym_name); 1595 return false; 1596 } 1597 1598 /* re-parse existing map definition */ 1599 t = btf__type_by_id(linker->btf, glob_sym->btf_id); 1600 t = skip_mods_and_typedefs(linker->btf, t->type, NULL); 1601 err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def); 1602 if (err) { 1603 /* this should not happen, because we already validated it */ 1604 pr_warn("global '%s': invalid dst map definition\n", sym_name); 1605 return false; 1606 } 1607 1608 /* Currently extern map definition has to be complete and match 1609 * concrete map definition exactly. This restriction might be lifted 1610 * in the future. 1611 */ 1612 return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def, 1613 obj->btf, &src_def, &src_inner_def); 1614 } 1615 glob_syms_match(const char * sym_name,struct bpf_linker * linker,struct glob_sym * glob_sym,struct src_obj * obj,Elf64_Sym * sym,size_t sym_idx,int btf_id)1616 static bool glob_syms_match(const char *sym_name, 1617 struct bpf_linker *linker, struct glob_sym *glob_sym, 1618 struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id) 1619 { 1620 const struct btf_type *src_t; 1621 1622 /* if we are dealing with externs, BTF types describing both global 1623 * and extern VARs/FUNCs should be completely present in all files 1624 */ 1625 if (!glob_sym->btf_id || !btf_id) { 1626 pr_warn("BTF info is missing for global symbol '%s'\n", sym_name); 1627 return false; 1628 } 1629 1630 src_t = btf__type_by_id(obj->btf, btf_id); 1631 if (!btf_is_var(src_t) && !btf_is_func(src_t)) { 1632 pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n", 1633 btf_kind_str(src_t), sym_name); 1634 return false; 1635 } 1636 1637 /* deal with .maps definitions specially */ 1638 if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0) 1639 return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id); 1640 1641 if (!glob_sym_btf_matches(sym_name, true /*exact*/, 1642 linker->btf, glob_sym->btf_id, obj->btf, btf_id)) 1643 return false; 1644 1645 return true; 1646 } 1647 btf_is_non_static(const struct btf_type * t)1648 static bool btf_is_non_static(const struct btf_type *t) 1649 { 1650 return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC) 1651 || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC); 1652 } 1653 find_glob_sym_btf(struct src_obj * obj,Elf64_Sym * sym,const char * sym_name,int * out_btf_sec_id,int * out_btf_id)1654 static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name, 1655 int *out_btf_sec_id, int *out_btf_id) 1656 { 1657 int i, j, n, m, btf_id = 0; 1658 const struct btf_type *t; 1659 const struct btf_var_secinfo *vi; 1660 const char *name; 1661 1662 if (!obj->btf) { 1663 pr_warn("failed to find BTF info for object '%s'\n", obj->filename); 1664 return -EINVAL; 1665 } 1666 1667 n = btf__type_cnt(obj->btf); 1668 for (i = 1; i < n; i++) { 1669 t = btf__type_by_id(obj->btf, i); 1670 1671 /* some global and extern FUNCs and VARs might not be associated with any 1672 * DATASEC, so try to detect them in the same pass 1673 */ 1674 if (btf_is_non_static(t)) { 1675 name = btf__str_by_offset(obj->btf, t->name_off); 1676 if (strcmp(name, sym_name) != 0) 1677 continue; 1678 1679 /* remember and still try to find DATASEC */ 1680 btf_id = i; 1681 continue; 1682 } 1683 1684 if (!btf_is_datasec(t)) 1685 continue; 1686 1687 vi = btf_var_secinfos(t); 1688 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) { 1689 t = btf__type_by_id(obj->btf, vi->type); 1690 name = btf__str_by_offset(obj->btf, t->name_off); 1691 1692 if (strcmp(name, sym_name) != 0) 1693 continue; 1694 if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC) 1695 continue; 1696 if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC) 1697 continue; 1698 1699 if (btf_id && btf_id != vi->type) { 1700 pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n", 1701 sym_name, btf_id, vi->type); 1702 return -EINVAL; 1703 } 1704 1705 *out_btf_sec_id = i; 1706 *out_btf_id = vi->type; 1707 1708 return 0; 1709 } 1710 } 1711 1712 /* free-floating extern or global FUNC */ 1713 if (btf_id) { 1714 *out_btf_sec_id = 0; 1715 *out_btf_id = btf_id; 1716 return 0; 1717 } 1718 1719 pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name); 1720 return -ENOENT; 1721 } 1722 find_src_sec_by_name(struct src_obj * obj,const char * sec_name)1723 static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name) 1724 { 1725 struct src_sec *sec; 1726 int i; 1727 1728 for (i = 1; i < obj->sec_cnt; i++) { 1729 sec = &obj->secs[i]; 1730 1731 if (strcmp(sec->sec_name, sec_name) == 0) 1732 return sec; 1733 } 1734 1735 return NULL; 1736 } 1737 complete_extern_btf_info(struct btf * dst_btf,int dst_id,struct btf * src_btf,int src_id)1738 static int complete_extern_btf_info(struct btf *dst_btf, int dst_id, 1739 struct btf *src_btf, int src_id) 1740 { 1741 struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id); 1742 struct btf_type *src_t = btf_type_by_id(src_btf, src_id); 1743 struct btf_param *src_p, *dst_p; 1744 const char *s; 1745 int i, n, off; 1746 1747 /* We already made sure that source and destination types (FUNC or 1748 * VAR) match in terms of types and argument names. 1749 */ 1750 if (btf_is_var(dst_t)) { 1751 btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED; 1752 return 0; 1753 } 1754 1755 dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0); 1756 1757 /* now onto FUNC_PROTO types */ 1758 src_t = btf_type_by_id(src_btf, src_t->type); 1759 dst_t = btf_type_by_id(dst_btf, dst_t->type); 1760 1761 /* Fill in all the argument names, which for extern FUNCs are missing. 1762 * We'll end up with two copies of FUNCs/VARs for externs, but that 1763 * will be taken care of by BTF dedup at the very end. 1764 * It might be that BTF types for extern in one file has less/more BTF 1765 * information (e.g., FWD instead of full STRUCT/UNION information), 1766 * but that should be (in most cases, subject to BTF dedup rules) 1767 * handled and resolved by BTF dedup algorithm as well, so we won't 1768 * worry about it. Our only job is to make sure that argument names 1769 * are populated on both sides, otherwise BTF dedup will pedantically 1770 * consider them different. 1771 */ 1772 src_p = btf_params(src_t); 1773 dst_p = btf_params(dst_t); 1774 for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) { 1775 if (!src_p->name_off) 1776 continue; 1777 1778 /* src_btf has more complete info, so add name to dst_btf */ 1779 s = btf__str_by_offset(src_btf, src_p->name_off); 1780 off = btf__add_str(dst_btf, s); 1781 if (off < 0) 1782 return off; 1783 dst_p->name_off = off; 1784 } 1785 return 0; 1786 } 1787 sym_update_bind(Elf64_Sym * sym,int sym_bind)1788 static void sym_update_bind(Elf64_Sym *sym, int sym_bind) 1789 { 1790 sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info)); 1791 } 1792 sym_update_type(Elf64_Sym * sym,int sym_type)1793 static void sym_update_type(Elf64_Sym *sym, int sym_type) 1794 { 1795 sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type); 1796 } 1797 sym_update_visibility(Elf64_Sym * sym,int sym_vis)1798 static void sym_update_visibility(Elf64_Sym *sym, int sym_vis) 1799 { 1800 /* libelf doesn't provide setters for ST_VISIBILITY, 1801 * but it is stored in the lower 2 bits of st_other 1802 */ 1803 sym->st_other &= ~0x03; 1804 sym->st_other |= sym_vis; 1805 } 1806 linker_append_elf_sym(struct bpf_linker * linker,struct src_obj * obj,Elf64_Sym * sym,const char * sym_name,int src_sym_idx)1807 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj, 1808 Elf64_Sym *sym, const char *sym_name, int src_sym_idx) 1809 { 1810 struct src_sec *src_sec = NULL; 1811 struct dst_sec *dst_sec = NULL; 1812 struct glob_sym *glob_sym = NULL; 1813 int name_off, sym_type, sym_bind, sym_vis, err; 1814 int btf_sec_id = 0, btf_id = 0; 1815 size_t dst_sym_idx; 1816 Elf64_Sym *dst_sym; 1817 bool sym_is_extern; 1818 1819 sym_type = ELF64_ST_TYPE(sym->st_info); 1820 sym_bind = ELF64_ST_BIND(sym->st_info); 1821 sym_vis = ELF64_ST_VISIBILITY(sym->st_other); 1822 sym_is_extern = sym->st_shndx == SHN_UNDEF; 1823 1824 if (sym_is_extern) { 1825 if (!obj->btf) { 1826 pr_warn("externs without BTF info are not supported\n"); 1827 return -ENOTSUP; 1828 } 1829 } else if (sym->st_shndx < SHN_LORESERVE) { 1830 src_sec = &obj->secs[sym->st_shndx]; 1831 if (src_sec->skipped) 1832 return 0; 1833 dst_sec = &linker->secs[src_sec->dst_id]; 1834 1835 /* allow only one STT_SECTION symbol per section */ 1836 if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) { 1837 obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx; 1838 return 0; 1839 } 1840 } 1841 1842 if (sym_bind == STB_LOCAL) 1843 goto add_sym; 1844 1845 /* find matching BTF info */ 1846 err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id); 1847 if (err) 1848 return err; 1849 1850 if (sym_is_extern && btf_sec_id) { 1851 const char *sec_name = NULL; 1852 const struct btf_type *t; 1853 1854 t = btf__type_by_id(obj->btf, btf_sec_id); 1855 sec_name = btf__str_by_offset(obj->btf, t->name_off); 1856 1857 /* Clang puts unannotated extern vars into 1858 * '.extern' BTF DATASEC. Treat them the same 1859 * as unannotated extern funcs (which are 1860 * currently not put into any DATASECs). 1861 * Those don't have associated src_sec/dst_sec. 1862 */ 1863 if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) { 1864 src_sec = find_src_sec_by_name(obj, sec_name); 1865 if (!src_sec) { 1866 pr_warn("failed to find matching ELF sec '%s'\n", sec_name); 1867 return -ENOENT; 1868 } 1869 dst_sec = &linker->secs[src_sec->dst_id]; 1870 } 1871 } 1872 1873 glob_sym = find_glob_sym(linker, sym_name); 1874 if (glob_sym) { 1875 /* Preventively resolve to existing symbol. This is 1876 * needed for further relocation symbol remapping in 1877 * the next step of linking. 1878 */ 1879 obj->sym_map[src_sym_idx] = glob_sym->sym_idx; 1880 1881 /* If both symbols are non-externs, at least one of 1882 * them has to be STB_WEAK, otherwise they are in 1883 * a conflict with each other. 1884 */ 1885 if (!sym_is_extern && !glob_sym->is_extern 1886 && !glob_sym->is_weak && sym_bind != STB_WEAK) { 1887 pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n", 1888 src_sym_idx, sym_name, obj->filename); 1889 return -EINVAL; 1890 } 1891 1892 if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id)) 1893 return -EINVAL; 1894 1895 dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx); 1896 1897 /* If new symbol is strong, then force dst_sym to be strong as 1898 * well; this way a mix of weak and non-weak extern 1899 * definitions will end up being strong. 1900 */ 1901 if (sym_bind == STB_GLOBAL) { 1902 /* We still need to preserve type (NOTYPE or 1903 * OBJECT/FUNC, depending on whether the symbol is 1904 * extern or not) 1905 */ 1906 sym_update_bind(dst_sym, STB_GLOBAL); 1907 glob_sym->is_weak = false; 1908 } 1909 1910 /* Non-default visibility is "contaminating", with stricter 1911 * visibility overwriting more permissive ones, even if more 1912 * permissive visibility comes from just an extern definition. 1913 * Currently only STV_DEFAULT and STV_HIDDEN are allowed and 1914 * ensured by ELF symbol sanity checks above. 1915 */ 1916 if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other)) 1917 sym_update_visibility(dst_sym, sym_vis); 1918 1919 /* If the new symbol is extern, then regardless if 1920 * existing symbol is extern or resolved global, just 1921 * keep the existing one untouched. 1922 */ 1923 if (sym_is_extern) 1924 return 0; 1925 1926 /* If existing symbol is a strong resolved symbol, bail out, 1927 * because we lost resolution battle have nothing to 1928 * contribute. We already checked abover that there is no 1929 * strong-strong conflict. We also already tightened binding 1930 * and visibility, so nothing else to contribute at that point. 1931 */ 1932 if (!glob_sym->is_extern && sym_bind == STB_WEAK) 1933 return 0; 1934 1935 /* At this point, new symbol is strong non-extern, 1936 * so overwrite glob_sym with new symbol information. 1937 * Preserve binding and visibility. 1938 */ 1939 sym_update_type(dst_sym, sym_type); 1940 dst_sym->st_shndx = dst_sec->sec_idx; 1941 dst_sym->st_value = src_sec->dst_off + sym->st_value; 1942 dst_sym->st_size = sym->st_size; 1943 1944 /* see comment below about dst_sec->id vs dst_sec->sec_idx */ 1945 glob_sym->sec_id = dst_sec->id; 1946 glob_sym->is_extern = false; 1947 1948 if (complete_extern_btf_info(linker->btf, glob_sym->btf_id, 1949 obj->btf, btf_id)) 1950 return -EINVAL; 1951 1952 /* request updating VAR's/FUNC's underlying BTF type when appending BTF type */ 1953 glob_sym->underlying_btf_id = 0; 1954 1955 obj->sym_map[src_sym_idx] = glob_sym->sym_idx; 1956 return 0; 1957 } 1958 1959 add_sym: 1960 name_off = strset__add_str(linker->strtab_strs, sym_name); 1961 if (name_off < 0) 1962 return name_off; 1963 1964 dst_sym = add_new_sym(linker, &dst_sym_idx); 1965 if (!dst_sym) 1966 return -ENOMEM; 1967 1968 dst_sym->st_name = name_off; 1969 dst_sym->st_info = sym->st_info; 1970 dst_sym->st_other = sym->st_other; 1971 dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx; 1972 dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value; 1973 dst_sym->st_size = sym->st_size; 1974 1975 obj->sym_map[src_sym_idx] = dst_sym_idx; 1976 1977 if (sym_type == STT_SECTION && dst_sec) { 1978 dst_sec->sec_sym_idx = dst_sym_idx; 1979 dst_sym->st_value = 0; 1980 } 1981 1982 if (sym_bind != STB_LOCAL) { 1983 glob_sym = add_glob_sym(linker); 1984 if (!glob_sym) 1985 return -ENOMEM; 1986 1987 glob_sym->sym_idx = dst_sym_idx; 1988 /* we use dst_sec->id (and not dst_sec->sec_idx), because 1989 * ephemeral sections (.kconfig, .ksyms, etc) don't have 1990 * sec_idx (as they don't have corresponding ELF section), but 1991 * still have id. .extern doesn't have even ephemeral section 1992 * associated with it, so dst_sec->id == dst_sec->sec_idx == 0. 1993 */ 1994 glob_sym->sec_id = dst_sec ? dst_sec->id : 0; 1995 glob_sym->name_off = name_off; 1996 /* we will fill btf_id in during BTF merging step */ 1997 glob_sym->btf_id = 0; 1998 glob_sym->is_extern = sym_is_extern; 1999 glob_sym->is_weak = sym_bind == STB_WEAK; 2000 } 2001 2002 return 0; 2003 } 2004 linker_append_elf_relos(struct bpf_linker * linker,struct src_obj * obj)2005 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj) 2006 { 2007 struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx]; 2008 int i, err; 2009 2010 for (i = 1; i < obj->sec_cnt; i++) { 2011 struct src_sec *src_sec, *src_linked_sec; 2012 struct dst_sec *dst_sec, *dst_linked_sec; 2013 Elf64_Rel *src_rel, *dst_rel; 2014 int j, n; 2015 2016 src_sec = &obj->secs[i]; 2017 if (!is_relo_sec(src_sec)) 2018 continue; 2019 2020 /* shdr->sh_info points to relocatable section */ 2021 src_linked_sec = &obj->secs[src_sec->shdr->sh_info]; 2022 if (src_linked_sec->skipped) 2023 continue; 2024 2025 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name); 2026 if (!dst_sec) { 2027 dst_sec = add_dst_sec(linker, src_sec->sec_name); 2028 if (!dst_sec) 2029 return -ENOMEM; 2030 err = init_sec(linker, dst_sec, src_sec); 2031 if (err) { 2032 pr_warn("failed to init section '%s'\n", src_sec->sec_name); 2033 return err; 2034 } 2035 } else if (!secs_match(dst_sec, src_sec)) { 2036 pr_warn("sections %s are not compatible\n", src_sec->sec_name); 2037 return -1; 2038 } 2039 2040 /* shdr->sh_link points to SYMTAB */ 2041 dst_sec->shdr->sh_link = linker->symtab_sec_idx; 2042 2043 /* shdr->sh_info points to relocated section */ 2044 dst_linked_sec = &linker->secs[src_linked_sec->dst_id]; 2045 dst_sec->shdr->sh_info = dst_linked_sec->sec_idx; 2046 2047 src_sec->dst_id = dst_sec->id; 2048 err = extend_sec(linker, dst_sec, src_sec); 2049 if (err) 2050 return err; 2051 2052 src_rel = src_sec->data->d_buf; 2053 dst_rel = dst_sec->raw_data + src_sec->dst_off; 2054 n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize; 2055 for (j = 0; j < n; j++, src_rel++, dst_rel++) { 2056 size_t src_sym_idx, dst_sym_idx, sym_type; 2057 Elf64_Sym *src_sym; 2058 2059 src_sym_idx = ELF64_R_SYM(src_rel->r_info); 2060 src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx; 2061 2062 dst_sym_idx = obj->sym_map[src_sym_idx]; 2063 dst_rel->r_offset += src_linked_sec->dst_off; 2064 sym_type = ELF64_R_TYPE(src_rel->r_info); 2065 dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type); 2066 2067 if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) { 2068 struct src_sec *sec = &obj->secs[src_sym->st_shndx]; 2069 struct bpf_insn *insn; 2070 2071 if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) { 2072 /* calls to the very first static function inside 2073 * .text section at offset 0 will 2074 * reference section symbol, not the 2075 * function symbol. Fix that up, 2076 * otherwise it won't be possible to 2077 * relocate calls to two different 2078 * static functions with the same name 2079 * (rom two different object files) 2080 */ 2081 insn = dst_linked_sec->raw_data + dst_rel->r_offset; 2082 if (insn->code == (BPF_JMP | BPF_CALL)) 2083 insn->imm += sec->dst_off / sizeof(struct bpf_insn); 2084 else 2085 insn->imm += sec->dst_off; 2086 } else { 2087 pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n"); 2088 return -EINVAL; 2089 } 2090 } 2091 2092 } 2093 } 2094 2095 return 0; 2096 } 2097 find_sym_by_name(struct src_obj * obj,size_t sec_idx,int sym_type,const char * sym_name)2098 static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx, 2099 int sym_type, const char *sym_name) 2100 { 2101 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx]; 2102 Elf64_Sym *sym = symtab->data->d_buf; 2103 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize; 2104 int str_sec_idx = symtab->shdr->sh_link; 2105 const char *name; 2106 2107 for (i = 0; i < n; i++, sym++) { 2108 if (sym->st_shndx != sec_idx) 2109 continue; 2110 if (ELF64_ST_TYPE(sym->st_info) != sym_type) 2111 continue; 2112 2113 name = elf_strptr(obj->elf, str_sec_idx, sym->st_name); 2114 if (!name) 2115 return NULL; 2116 2117 if (strcmp(sym_name, name) != 0) 2118 continue; 2119 2120 return sym; 2121 } 2122 2123 return NULL; 2124 } 2125 linker_fixup_btf(struct src_obj * obj)2126 static int linker_fixup_btf(struct src_obj *obj) 2127 { 2128 const char *sec_name; 2129 struct src_sec *sec; 2130 int i, j, n, m; 2131 2132 if (!obj->btf) 2133 return 0; 2134 2135 n = btf__type_cnt(obj->btf); 2136 for (i = 1; i < n; i++) { 2137 struct btf_var_secinfo *vi; 2138 struct btf_type *t; 2139 2140 t = btf_type_by_id(obj->btf, i); 2141 if (btf_kind(t) != BTF_KIND_DATASEC) 2142 continue; 2143 2144 sec_name = btf__str_by_offset(obj->btf, t->name_off); 2145 sec = find_src_sec_by_name(obj, sec_name); 2146 if (sec) { 2147 /* record actual section size, unless ephemeral */ 2148 if (sec->shdr) 2149 t->size = sec->shdr->sh_size; 2150 } else { 2151 /* BTF can have some sections that are not represented 2152 * in ELF, e.g., .kconfig, .ksyms, .extern, which are used 2153 * for special extern variables. 2154 * 2155 * For all but one such special (ephemeral) 2156 * sections, we pre-create "section shells" to be able 2157 * to keep track of extra per-section metadata later 2158 * (e.g., those BTF extern variables). 2159 * 2160 * .extern is even more special, though, because it 2161 * contains extern variables that need to be resolved 2162 * by static linker, not libbpf and kernel. When such 2163 * externs are resolved, we are going to remove them 2164 * from .extern BTF section and might end up not 2165 * needing it at all. Each resolved extern should have 2166 * matching non-extern VAR/FUNC in other sections. 2167 * 2168 * We do support leaving some of the externs 2169 * unresolved, though, to support cases of building 2170 * libraries, which will later be linked against final 2171 * BPF applications. So if at finalization we still 2172 * see unresolved externs, we'll create .extern 2173 * section on our own. 2174 */ 2175 if (strcmp(sec_name, BTF_EXTERN_SEC) == 0) 2176 continue; 2177 2178 sec = add_src_sec(obj, sec_name); 2179 if (!sec) 2180 return -ENOMEM; 2181 2182 sec->ephemeral = true; 2183 sec->sec_idx = 0; /* will match UNDEF shndx in ELF */ 2184 } 2185 2186 /* remember ELF section and its BTF type ID match */ 2187 sec->sec_type_id = i; 2188 2189 /* fix up variable offsets */ 2190 vi = btf_var_secinfos(t); 2191 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) { 2192 const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type); 2193 const char *var_name; 2194 int var_linkage; 2195 Elf64_Sym *sym; 2196 2197 /* could be a variable or function */ 2198 if (!btf_is_var(vt)) 2199 continue; 2200 2201 var_name = btf__str_by_offset(obj->btf, vt->name_off); 2202 var_linkage = btf_var(vt)->linkage; 2203 2204 /* no need to patch up static or extern vars */ 2205 if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED) 2206 continue; 2207 2208 sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name); 2209 if (!sym) { 2210 pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name); 2211 return -ENOENT; 2212 } 2213 2214 vi->offset = sym->st_value; 2215 } 2216 } 2217 2218 return 0; 2219 } 2220 remap_type_id(__u32 * type_id,void * ctx)2221 static int remap_type_id(__u32 *type_id, void *ctx) 2222 { 2223 int *id_map = ctx; 2224 int new_id = id_map[*type_id]; 2225 2226 /* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */ 2227 if (new_id == 0 && *type_id != 0) { 2228 pr_warn("failed to find new ID mapping for original BTF type ID %u\n", *type_id); 2229 return -EINVAL; 2230 } 2231 2232 *type_id = id_map[*type_id]; 2233 2234 return 0; 2235 } 2236 linker_append_btf(struct bpf_linker * linker,struct src_obj * obj)2237 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj) 2238 { 2239 const struct btf_type *t; 2240 int i, j, n, start_id, id; 2241 const char *name; 2242 2243 if (!obj->btf) 2244 return 0; 2245 2246 start_id = btf__type_cnt(linker->btf); 2247 n = btf__type_cnt(obj->btf); 2248 2249 obj->btf_type_map = calloc(n + 1, sizeof(int)); 2250 if (!obj->btf_type_map) 2251 return -ENOMEM; 2252 2253 for (i = 1; i < n; i++) { 2254 struct glob_sym *glob_sym = NULL; 2255 2256 t = btf__type_by_id(obj->btf, i); 2257 2258 /* DATASECs are handled specially below */ 2259 if (btf_kind(t) == BTF_KIND_DATASEC) 2260 continue; 2261 2262 if (btf_is_non_static(t)) { 2263 /* there should be glob_sym already */ 2264 name = btf__str_by_offset(obj->btf, t->name_off); 2265 glob_sym = find_glob_sym(linker, name); 2266 2267 /* VARs without corresponding glob_sym are those that 2268 * belong to skipped/deduplicated sections (i.e., 2269 * license and version), so just skip them 2270 */ 2271 if (!glob_sym) 2272 continue; 2273 2274 /* linker_append_elf_sym() might have requested 2275 * updating underlying type ID, if extern was resolved 2276 * to strong symbol or weak got upgraded to non-weak 2277 */ 2278 if (glob_sym->underlying_btf_id == 0) 2279 glob_sym->underlying_btf_id = -t->type; 2280 2281 /* globals from previous object files that match our 2282 * VAR/FUNC already have a corresponding associated 2283 * BTF type, so just make sure to use it 2284 */ 2285 if (glob_sym->btf_id) { 2286 /* reuse existing BTF type for global var/func */ 2287 obj->btf_type_map[i] = glob_sym->btf_id; 2288 continue; 2289 } 2290 } 2291 2292 id = btf__add_type(linker->btf, obj->btf, t); 2293 if (id < 0) { 2294 pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename); 2295 return id; 2296 } 2297 2298 obj->btf_type_map[i] = id; 2299 2300 /* record just appended BTF type for var/func */ 2301 if (glob_sym) { 2302 glob_sym->btf_id = id; 2303 glob_sym->underlying_btf_id = -t->type; 2304 } 2305 } 2306 2307 /* remap all the types except DATASECs */ 2308 n = btf__type_cnt(linker->btf); 2309 for (i = start_id; i < n; i++) { 2310 struct btf_type *dst_t = btf_type_by_id(linker->btf, i); 2311 2312 if (btf_type_visit_type_ids(dst_t, remap_type_id, obj->btf_type_map)) 2313 return -EINVAL; 2314 } 2315 2316 /* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's 2317 * actual type), if necessary 2318 */ 2319 for (i = 0; i < linker->glob_sym_cnt; i++) { 2320 struct glob_sym *glob_sym = &linker->glob_syms[i]; 2321 struct btf_type *glob_t; 2322 2323 if (glob_sym->underlying_btf_id >= 0) 2324 continue; 2325 2326 glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id]; 2327 2328 glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id); 2329 glob_t->type = glob_sym->underlying_btf_id; 2330 } 2331 2332 /* append DATASEC info */ 2333 for (i = 1; i < obj->sec_cnt; i++) { 2334 struct src_sec *src_sec; 2335 struct dst_sec *dst_sec; 2336 const struct btf_var_secinfo *src_var; 2337 struct btf_var_secinfo *dst_var; 2338 2339 src_sec = &obj->secs[i]; 2340 if (!src_sec->sec_type_id || src_sec->skipped) 2341 continue; 2342 dst_sec = &linker->secs[src_sec->dst_id]; 2343 2344 /* Mark section as having BTF regardless of the presence of 2345 * variables. In some cases compiler might generate empty BTF 2346 * with no variables information. E.g., when promoting local 2347 * array/structure variable initial values and BPF object 2348 * file otherwise has no read-only static variables in 2349 * .rodata. We need to preserve such empty BTF and just set 2350 * correct section size. 2351 */ 2352 dst_sec->has_btf = true; 2353 2354 t = btf__type_by_id(obj->btf, src_sec->sec_type_id); 2355 src_var = btf_var_secinfos(t); 2356 n = btf_vlen(t); 2357 for (j = 0; j < n; j++, src_var++) { 2358 void *sec_vars = dst_sec->sec_vars; 2359 int new_id = obj->btf_type_map[src_var->type]; 2360 struct glob_sym *glob_sym = NULL; 2361 2362 t = btf_type_by_id(linker->btf, new_id); 2363 if (btf_is_non_static(t)) { 2364 name = btf__str_by_offset(linker->btf, t->name_off); 2365 glob_sym = find_glob_sym(linker, name); 2366 if (glob_sym->sec_id != dst_sec->id) { 2367 pr_warn("global '%s': section mismatch %d vs %d\n", 2368 name, glob_sym->sec_id, dst_sec->id); 2369 return -EINVAL; 2370 } 2371 } 2372 2373 /* If there is already a member (VAR or FUNC) mapped 2374 * to the same type, don't add a duplicate entry. 2375 * This will happen when multiple object files define 2376 * the same extern VARs/FUNCs. 2377 */ 2378 if (glob_sym && glob_sym->var_idx >= 0) { 2379 __s64 sz; 2380 2381 dst_var = &dst_sec->sec_vars[glob_sym->var_idx]; 2382 /* Because underlying BTF type might have 2383 * changed, so might its size have changed, so 2384 * re-calculate and update it in sec_var. 2385 */ 2386 sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id); 2387 if (sz < 0) { 2388 pr_warn("global '%s': failed to resolve size of underlying type: %d\n", 2389 name, (int)sz); 2390 return -EINVAL; 2391 } 2392 dst_var->size = sz; 2393 continue; 2394 } 2395 2396 sec_vars = libbpf_reallocarray(sec_vars, 2397 dst_sec->sec_var_cnt + 1, 2398 sizeof(*dst_sec->sec_vars)); 2399 if (!sec_vars) 2400 return -ENOMEM; 2401 2402 dst_sec->sec_vars = sec_vars; 2403 dst_sec->sec_var_cnt++; 2404 2405 dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1]; 2406 dst_var->type = obj->btf_type_map[src_var->type]; 2407 dst_var->size = src_var->size; 2408 dst_var->offset = src_sec->dst_off + src_var->offset; 2409 2410 if (glob_sym) 2411 glob_sym->var_idx = dst_sec->sec_var_cnt - 1; 2412 } 2413 } 2414 2415 return 0; 2416 } 2417 add_btf_ext_rec(struct btf_ext_sec_data * ext_data,const void * src_rec)2418 static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec) 2419 { 2420 void *tmp; 2421 2422 tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz); 2423 if (!tmp) 2424 return NULL; 2425 ext_data->recs = tmp; 2426 2427 tmp += ext_data->rec_cnt * ext_data->rec_sz; 2428 memcpy(tmp, src_rec, ext_data->rec_sz); 2429 2430 ext_data->rec_cnt++; 2431 2432 return tmp; 2433 } 2434 linker_append_btf_ext(struct bpf_linker * linker,struct src_obj * obj)2435 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj) 2436 { 2437 const struct btf_ext_info_sec *ext_sec; 2438 const char *sec_name, *s; 2439 struct src_sec *src_sec; 2440 struct dst_sec *dst_sec; 2441 int rec_sz, str_off, i; 2442 2443 if (!obj->btf_ext) 2444 return 0; 2445 2446 rec_sz = obj->btf_ext->func_info.rec_size; 2447 for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) { 2448 struct bpf_func_info_min *src_rec, *dst_rec; 2449 2450 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); 2451 src_sec = find_src_sec_by_name(obj, sec_name); 2452 if (!src_sec) { 2453 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name); 2454 return -EINVAL; 2455 } 2456 dst_sec = &linker->secs[src_sec->dst_id]; 2457 2458 if (dst_sec->func_info.rec_sz == 0) 2459 dst_sec->func_info.rec_sz = rec_sz; 2460 if (dst_sec->func_info.rec_sz != rec_sz) { 2461 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name); 2462 return -EINVAL; 2463 } 2464 2465 for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) { 2466 dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec); 2467 if (!dst_rec) 2468 return -ENOMEM; 2469 2470 dst_rec->insn_off += src_sec->dst_off; 2471 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id]; 2472 } 2473 } 2474 2475 rec_sz = obj->btf_ext->line_info.rec_size; 2476 for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) { 2477 struct bpf_line_info_min *src_rec, *dst_rec; 2478 2479 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); 2480 src_sec = find_src_sec_by_name(obj, sec_name); 2481 if (!src_sec) { 2482 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name); 2483 return -EINVAL; 2484 } 2485 dst_sec = &linker->secs[src_sec->dst_id]; 2486 2487 if (dst_sec->line_info.rec_sz == 0) 2488 dst_sec->line_info.rec_sz = rec_sz; 2489 if (dst_sec->line_info.rec_sz != rec_sz) { 2490 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name); 2491 return -EINVAL; 2492 } 2493 2494 for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) { 2495 dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec); 2496 if (!dst_rec) 2497 return -ENOMEM; 2498 2499 dst_rec->insn_off += src_sec->dst_off; 2500 2501 s = btf__str_by_offset(obj->btf, src_rec->file_name_off); 2502 str_off = btf__add_str(linker->btf, s); 2503 if (str_off < 0) 2504 return -ENOMEM; 2505 dst_rec->file_name_off = str_off; 2506 2507 s = btf__str_by_offset(obj->btf, src_rec->line_off); 2508 str_off = btf__add_str(linker->btf, s); 2509 if (str_off < 0) 2510 return -ENOMEM; 2511 dst_rec->line_off = str_off; 2512 2513 /* dst_rec->line_col is fine */ 2514 } 2515 } 2516 2517 rec_sz = obj->btf_ext->core_relo_info.rec_size; 2518 for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) { 2519 struct bpf_core_relo *src_rec, *dst_rec; 2520 2521 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); 2522 src_sec = find_src_sec_by_name(obj, sec_name); 2523 if (!src_sec) { 2524 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name); 2525 return -EINVAL; 2526 } 2527 dst_sec = &linker->secs[src_sec->dst_id]; 2528 2529 if (dst_sec->core_relo_info.rec_sz == 0) 2530 dst_sec->core_relo_info.rec_sz = rec_sz; 2531 if (dst_sec->core_relo_info.rec_sz != rec_sz) { 2532 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name); 2533 return -EINVAL; 2534 } 2535 2536 for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) { 2537 dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec); 2538 if (!dst_rec) 2539 return -ENOMEM; 2540 2541 dst_rec->insn_off += src_sec->dst_off; 2542 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id]; 2543 2544 s = btf__str_by_offset(obj->btf, src_rec->access_str_off); 2545 str_off = btf__add_str(linker->btf, s); 2546 if (str_off < 0) 2547 return -ENOMEM; 2548 dst_rec->access_str_off = str_off; 2549 2550 /* dst_rec->kind is fine */ 2551 } 2552 } 2553 2554 return 0; 2555 } 2556 bpf_linker__finalize(struct bpf_linker * linker)2557 int bpf_linker__finalize(struct bpf_linker *linker) 2558 { 2559 struct dst_sec *sec; 2560 size_t strs_sz; 2561 const void *strs; 2562 int err, i; 2563 2564 if (!linker->elf) 2565 return libbpf_err(-EINVAL); 2566 2567 err = finalize_btf(linker); 2568 if (err) 2569 return libbpf_err(err); 2570 2571 /* Finalize strings */ 2572 strs_sz = strset__data_size(linker->strtab_strs); 2573 strs = strset__data(linker->strtab_strs); 2574 2575 sec = &linker->secs[linker->strtab_sec_idx]; 2576 sec->data->d_align = 1; 2577 sec->data->d_off = 0LL; 2578 sec->data->d_buf = (void *)strs; 2579 sec->data->d_type = ELF_T_BYTE; 2580 sec->data->d_size = strs_sz; 2581 sec->shdr->sh_size = strs_sz; 2582 2583 for (i = 1; i < linker->sec_cnt; i++) { 2584 sec = &linker->secs[i]; 2585 2586 /* STRTAB is handled specially above */ 2587 if (sec->sec_idx == linker->strtab_sec_idx) 2588 continue; 2589 2590 /* special ephemeral sections (.ksyms, .kconfig, etc) */ 2591 if (!sec->scn) 2592 continue; 2593 2594 sec->data->d_buf = sec->raw_data; 2595 } 2596 2597 /* Finalize ELF layout */ 2598 if (elf_update(linker->elf, ELF_C_NULL) < 0) { 2599 err = -EINVAL; 2600 pr_warn_elf("failed to finalize ELF layout"); 2601 return libbpf_err(err); 2602 } 2603 2604 /* Write out final ELF contents */ 2605 if (elf_update(linker->elf, ELF_C_WRITE) < 0) { 2606 err = -EINVAL; 2607 pr_warn_elf("failed to write ELF contents"); 2608 return libbpf_err(err); 2609 } 2610 2611 elf_end(linker->elf); 2612 close(linker->fd); 2613 2614 linker->elf = NULL; 2615 linker->fd = -1; 2616 2617 return 0; 2618 } 2619 emit_elf_data_sec(struct bpf_linker * linker,const char * sec_name,size_t align,const void * raw_data,size_t raw_sz)2620 static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name, 2621 size_t align, const void *raw_data, size_t raw_sz) 2622 { 2623 Elf_Scn *scn; 2624 Elf_Data *data; 2625 Elf64_Shdr *shdr; 2626 int name_off; 2627 2628 name_off = strset__add_str(linker->strtab_strs, sec_name); 2629 if (name_off < 0) 2630 return name_off; 2631 2632 scn = elf_newscn(linker->elf); 2633 if (!scn) 2634 return -ENOMEM; 2635 data = elf_newdata(scn); 2636 if (!data) 2637 return -ENOMEM; 2638 shdr = elf64_getshdr(scn); 2639 if (!shdr) 2640 return -EINVAL; 2641 2642 shdr->sh_name = name_off; 2643 shdr->sh_type = SHT_PROGBITS; 2644 shdr->sh_flags = 0; 2645 shdr->sh_size = raw_sz; 2646 shdr->sh_link = 0; 2647 shdr->sh_info = 0; 2648 shdr->sh_addralign = align; 2649 shdr->sh_entsize = 0; 2650 2651 data->d_type = ELF_T_BYTE; 2652 data->d_size = raw_sz; 2653 data->d_buf = (void *)raw_data; 2654 data->d_align = align; 2655 data->d_off = 0; 2656 2657 return 0; 2658 } 2659 finalize_btf(struct bpf_linker * linker)2660 static int finalize_btf(struct bpf_linker *linker) 2661 { 2662 LIBBPF_OPTS(btf_dedup_opts, opts); 2663 struct btf *btf = linker->btf; 2664 const void *raw_data; 2665 int i, j, id, err; 2666 __u32 raw_sz; 2667 2668 /* bail out if no BTF data was produced */ 2669 if (btf__type_cnt(linker->btf) == 1) 2670 return 0; 2671 2672 for (i = 1; i < linker->sec_cnt; i++) { 2673 struct dst_sec *sec = &linker->secs[i]; 2674 2675 if (!sec->has_btf) 2676 continue; 2677 2678 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz); 2679 if (id < 0) { 2680 pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n", 2681 sec->sec_name, id); 2682 return id; 2683 } 2684 2685 for (j = 0; j < sec->sec_var_cnt; j++) { 2686 struct btf_var_secinfo *vi = &sec->sec_vars[j]; 2687 2688 if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size)) 2689 return -EINVAL; 2690 } 2691 } 2692 2693 err = finalize_btf_ext(linker); 2694 if (err) { 2695 pr_warn(".BTF.ext generation failed: %d\n", err); 2696 return err; 2697 } 2698 2699 opts.btf_ext = linker->btf_ext; 2700 err = btf__dedup(linker->btf, &opts); 2701 if (err) { 2702 pr_warn("BTF dedup failed: %d\n", err); 2703 return err; 2704 } 2705 2706 /* Emit .BTF section */ 2707 raw_data = btf__raw_data(linker->btf, &raw_sz); 2708 if (!raw_data) 2709 return -ENOMEM; 2710 2711 err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz); 2712 if (err) { 2713 pr_warn("failed to write out .BTF ELF section: %d\n", err); 2714 return err; 2715 } 2716 2717 /* Emit .BTF.ext section */ 2718 if (linker->btf_ext) { 2719 raw_data = btf_ext__get_raw_data(linker->btf_ext, &raw_sz); 2720 if (!raw_data) 2721 return -ENOMEM; 2722 2723 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz); 2724 if (err) { 2725 pr_warn("failed to write out .BTF.ext ELF section: %d\n", err); 2726 return err; 2727 } 2728 } 2729 2730 return 0; 2731 } 2732 emit_btf_ext_data(struct bpf_linker * linker,void * output,const char * sec_name,struct btf_ext_sec_data * sec_data)2733 static int emit_btf_ext_data(struct bpf_linker *linker, void *output, 2734 const char *sec_name, struct btf_ext_sec_data *sec_data) 2735 { 2736 struct btf_ext_info_sec *sec_info; 2737 void *cur = output; 2738 int str_off; 2739 size_t sz; 2740 2741 if (!sec_data->rec_cnt) 2742 return 0; 2743 2744 str_off = btf__add_str(linker->btf, sec_name); 2745 if (str_off < 0) 2746 return -ENOMEM; 2747 2748 sec_info = cur; 2749 sec_info->sec_name_off = str_off; 2750 sec_info->num_info = sec_data->rec_cnt; 2751 cur += sizeof(struct btf_ext_info_sec); 2752 2753 sz = sec_data->rec_cnt * sec_data->rec_sz; 2754 memcpy(cur, sec_data->recs, sz); 2755 cur += sz; 2756 2757 return cur - output; 2758 } 2759 finalize_btf_ext(struct bpf_linker * linker)2760 static int finalize_btf_ext(struct bpf_linker *linker) 2761 { 2762 size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0; 2763 size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0; 2764 struct btf_ext_header *hdr; 2765 void *data, *cur; 2766 int i, err, sz; 2767 2768 /* validate that all sections have the same .BTF.ext record sizes 2769 * and calculate total data size for each type of data (func info, 2770 * line info, core relos) 2771 */ 2772 for (i = 1; i < linker->sec_cnt; i++) { 2773 struct dst_sec *sec = &linker->secs[i]; 2774 2775 if (sec->func_info.rec_cnt) { 2776 if (func_rec_sz == 0) 2777 func_rec_sz = sec->func_info.rec_sz; 2778 if (func_rec_sz != sec->func_info.rec_sz) { 2779 pr_warn("mismatch in func_info record size %zu != %u\n", 2780 func_rec_sz, sec->func_info.rec_sz); 2781 return -EINVAL; 2782 } 2783 2784 funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt; 2785 } 2786 if (sec->line_info.rec_cnt) { 2787 if (line_rec_sz == 0) 2788 line_rec_sz = sec->line_info.rec_sz; 2789 if (line_rec_sz != sec->line_info.rec_sz) { 2790 pr_warn("mismatch in line_info record size %zu != %u\n", 2791 line_rec_sz, sec->line_info.rec_sz); 2792 return -EINVAL; 2793 } 2794 2795 lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt; 2796 } 2797 if (sec->core_relo_info.rec_cnt) { 2798 if (core_relo_rec_sz == 0) 2799 core_relo_rec_sz = sec->core_relo_info.rec_sz; 2800 if (core_relo_rec_sz != sec->core_relo_info.rec_sz) { 2801 pr_warn("mismatch in core_relo_info record size %zu != %u\n", 2802 core_relo_rec_sz, sec->core_relo_info.rec_sz); 2803 return -EINVAL; 2804 } 2805 2806 core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt; 2807 } 2808 } 2809 2810 if (!funcs_sz && !lines_sz && !core_relos_sz) 2811 return 0; 2812 2813 total_sz += sizeof(struct btf_ext_header); 2814 if (funcs_sz) { 2815 funcs_sz += sizeof(__u32); /* record size prefix */ 2816 total_sz += funcs_sz; 2817 } 2818 if (lines_sz) { 2819 lines_sz += sizeof(__u32); /* record size prefix */ 2820 total_sz += lines_sz; 2821 } 2822 if (core_relos_sz) { 2823 core_relos_sz += sizeof(__u32); /* record size prefix */ 2824 total_sz += core_relos_sz; 2825 } 2826 2827 cur = data = calloc(1, total_sz); 2828 if (!data) 2829 return -ENOMEM; 2830 2831 hdr = cur; 2832 hdr->magic = BTF_MAGIC; 2833 hdr->version = BTF_VERSION; 2834 hdr->flags = 0; 2835 hdr->hdr_len = sizeof(struct btf_ext_header); 2836 cur += sizeof(struct btf_ext_header); 2837 2838 /* All offsets are in bytes relative to the end of this header */ 2839 hdr->func_info_off = 0; 2840 hdr->func_info_len = funcs_sz; 2841 hdr->line_info_off = funcs_sz; 2842 hdr->line_info_len = lines_sz; 2843 hdr->core_relo_off = funcs_sz + lines_sz; 2844 hdr->core_relo_len = core_relos_sz; 2845 2846 if (funcs_sz) { 2847 *(__u32 *)cur = func_rec_sz; 2848 cur += sizeof(__u32); 2849 2850 for (i = 1; i < linker->sec_cnt; i++) { 2851 struct dst_sec *sec = &linker->secs[i]; 2852 2853 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info); 2854 if (sz < 0) { 2855 err = sz; 2856 goto out; 2857 } 2858 2859 cur += sz; 2860 } 2861 } 2862 2863 if (lines_sz) { 2864 *(__u32 *)cur = line_rec_sz; 2865 cur += sizeof(__u32); 2866 2867 for (i = 1; i < linker->sec_cnt; i++) { 2868 struct dst_sec *sec = &linker->secs[i]; 2869 2870 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info); 2871 if (sz < 0) { 2872 err = sz; 2873 goto out; 2874 } 2875 2876 cur += sz; 2877 } 2878 } 2879 2880 if (core_relos_sz) { 2881 *(__u32 *)cur = core_relo_rec_sz; 2882 cur += sizeof(__u32); 2883 2884 for (i = 1; i < linker->sec_cnt; i++) { 2885 struct dst_sec *sec = &linker->secs[i]; 2886 2887 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info); 2888 if (sz < 0) { 2889 err = sz; 2890 goto out; 2891 } 2892 2893 cur += sz; 2894 } 2895 } 2896 2897 linker->btf_ext = btf_ext__new(data, total_sz); 2898 err = libbpf_get_error(linker->btf_ext); 2899 if (err) { 2900 linker->btf_ext = NULL; 2901 pr_warn("failed to parse final .BTF.ext data: %d\n", err); 2902 goto out; 2903 } 2904 2905 out: 2906 free(data); 2907 return err; 2908 } 2909