1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com> 4 */ 5 6 #include <string.h> 7 #include <stdlib.h> 8 #include <inttypes.h> 9 #include <sys/mman.h> 10 11 #include <arch/elf.h> 12 #include <objtool/builtin.h> 13 #include <objtool/cfi.h> 14 #include <objtool/arch.h> 15 #include <objtool/check.h> 16 #include <objtool/special.h> 17 #include <objtool/warn.h> 18 #include <objtool/endianness.h> 19 20 #include <linux/objtool.h> 21 #include <linux/hashtable.h> 22 #include <linux/kernel.h> 23 #include <linux/static_call_types.h> 24 25 struct alternative { 26 struct list_head list; 27 struct instruction *insn; 28 bool skip_orig; 29 }; 30 31 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache; 32 33 static struct cfi_init_state initial_func_cfi; 34 static struct cfi_state init_cfi; 35 static struct cfi_state func_cfi; 36 37 struct instruction *find_insn(struct objtool_file *file, 38 struct section *sec, unsigned long offset) 39 { 40 struct instruction *insn; 41 42 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) { 43 if (insn->sec == sec && insn->offset == offset) 44 return insn; 45 } 46 47 return NULL; 48 } 49 50 static struct instruction *next_insn_same_sec(struct objtool_file *file, 51 struct instruction *insn) 52 { 53 struct instruction *next = list_next_entry(insn, list); 54 55 if (!next || &next->list == &file->insn_list || next->sec != insn->sec) 56 return NULL; 57 58 return next; 59 } 60 61 static struct instruction *next_insn_same_func(struct objtool_file *file, 62 struct instruction *insn) 63 { 64 struct instruction *next = list_next_entry(insn, list); 65 struct symbol *func = insn_func(insn); 66 67 if (!func) 68 return NULL; 69 70 if (&next->list != &file->insn_list && insn_func(next) == func) 71 return next; 72 73 /* Check if we're already in the subfunction: */ 74 if (func == func->cfunc) 75 return NULL; 76 77 /* Move to the subfunction: */ 78 return find_insn(file, func->cfunc->sec, func->cfunc->offset); 79 } 80 81 static struct instruction *prev_insn_same_sym(struct objtool_file *file, 82 struct instruction *insn) 83 { 84 struct instruction *prev = list_prev_entry(insn, list); 85 86 if (&prev->list != &file->insn_list && insn_func(prev) == insn_func(insn)) 87 return prev; 88 89 return NULL; 90 } 91 92 #define func_for_each_insn(file, func, insn) \ 93 for (insn = find_insn(file, func->sec, func->offset); \ 94 insn; \ 95 insn = next_insn_same_func(file, insn)) 96 97 #define sym_for_each_insn(file, sym, insn) \ 98 for (insn = find_insn(file, sym->sec, sym->offset); \ 99 insn && &insn->list != &file->insn_list && \ 100 insn->sec == sym->sec && \ 101 insn->offset < sym->offset + sym->len; \ 102 insn = list_next_entry(insn, list)) 103 104 #define sym_for_each_insn_continue_reverse(file, sym, insn) \ 105 for (insn = list_prev_entry(insn, list); \ 106 &insn->list != &file->insn_list && \ 107 insn->sec == sym->sec && insn->offset >= sym->offset; \ 108 insn = list_prev_entry(insn, list)) 109 110 #define sec_for_each_insn_from(file, insn) \ 111 for (; insn; insn = next_insn_same_sec(file, insn)) 112 113 #define sec_for_each_insn_continue(file, insn) \ 114 for (insn = next_insn_same_sec(file, insn); insn; \ 115 insn = next_insn_same_sec(file, insn)) 116 117 static bool is_jump_table_jump(struct instruction *insn) 118 { 119 struct alt_group *alt_group = insn->alt_group; 120 121 if (insn->jump_table) 122 return true; 123 124 /* Retpoline alternative for a jump table? */ 125 return alt_group && alt_group->orig_group && 126 alt_group->orig_group->first_insn->jump_table; 127 } 128 129 static bool is_sibling_call(struct instruction *insn) 130 { 131 /* 132 * Assume only STT_FUNC calls have jump-tables. 133 */ 134 if (insn_func(insn)) { 135 /* An indirect jump is either a sibling call or a jump to a table. */ 136 if (insn->type == INSN_JUMP_DYNAMIC) 137 return !is_jump_table_jump(insn); 138 } 139 140 /* add_jump_destinations() sets insn->call_dest for sibling calls. */ 141 return (is_static_jump(insn) && insn->call_dest); 142 } 143 144 /* 145 * This checks to see if the given function is a "noreturn" function. 146 * 147 * For global functions which are outside the scope of this object file, we 148 * have to keep a manual list of them. 149 * 150 * For local functions, we have to detect them manually by simply looking for 151 * the lack of a return instruction. 152 */ 153 static bool __dead_end_function(struct objtool_file *file, struct symbol *func, 154 int recursion) 155 { 156 int i; 157 struct instruction *insn; 158 bool empty = true; 159 160 /* 161 * Unfortunately these have to be hard coded because the noreturn 162 * attribute isn't provided in ELF data. Keep 'em sorted. 163 */ 164 static const char * const global_noreturns[] = { 165 "__invalid_creds", 166 "__module_put_and_kthread_exit", 167 "__reiserfs_panic", 168 "__stack_chk_fail", 169 "__ubsan_handle_builtin_unreachable", 170 "cpu_bringup_and_idle", 171 "cpu_startup_entry", 172 "do_exit", 173 "do_group_exit", 174 "do_task_dead", 175 "ex_handler_msr_mce", 176 "fortify_panic", 177 "kthread_complete_and_exit", 178 "kthread_exit", 179 "kunit_try_catch_throw", 180 "lbug_with_loc", 181 "machine_real_restart", 182 "make_task_dead", 183 "panic", 184 "rewind_stack_and_make_dead", 185 "sev_es_terminate", 186 "snp_abort", 187 "stop_this_cpu", 188 "usercopy_abort", 189 "xen_start_kernel", 190 }; 191 192 if (!func) 193 return false; 194 195 if (func->bind == STB_WEAK) 196 return false; 197 198 if (func->bind == STB_GLOBAL) 199 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++) 200 if (!strcmp(func->name, global_noreturns[i])) 201 return true; 202 203 if (!func->len) 204 return false; 205 206 insn = find_insn(file, func->sec, func->offset); 207 if (!insn || !insn_func(insn)) 208 return false; 209 210 func_for_each_insn(file, func, insn) { 211 empty = false; 212 213 if (insn->type == INSN_RETURN) 214 return false; 215 } 216 217 if (empty) 218 return false; 219 220 /* 221 * A function can have a sibling call instead of a return. In that 222 * case, the function's dead-end status depends on whether the target 223 * of the sibling call returns. 224 */ 225 func_for_each_insn(file, func, insn) { 226 if (is_sibling_call(insn)) { 227 struct instruction *dest = insn->jump_dest; 228 229 if (!dest) 230 /* sibling call to another file */ 231 return false; 232 233 /* local sibling call */ 234 if (recursion == 5) { 235 /* 236 * Infinite recursion: two functions have 237 * sibling calls to each other. This is a very 238 * rare case. It means they aren't dead ends. 239 */ 240 return false; 241 } 242 243 return __dead_end_function(file, insn_func(dest), recursion+1); 244 } 245 } 246 247 return true; 248 } 249 250 static bool dead_end_function(struct objtool_file *file, struct symbol *func) 251 { 252 return __dead_end_function(file, func, 0); 253 } 254 255 static void init_cfi_state(struct cfi_state *cfi) 256 { 257 int i; 258 259 for (i = 0; i < CFI_NUM_REGS; i++) { 260 cfi->regs[i].base = CFI_UNDEFINED; 261 cfi->vals[i].base = CFI_UNDEFINED; 262 } 263 cfi->cfa.base = CFI_UNDEFINED; 264 cfi->drap_reg = CFI_UNDEFINED; 265 cfi->drap_offset = -1; 266 } 267 268 static void init_insn_state(struct objtool_file *file, struct insn_state *state, 269 struct section *sec) 270 { 271 memset(state, 0, sizeof(*state)); 272 init_cfi_state(&state->cfi); 273 274 /* 275 * We need the full vmlinux for noinstr validation, otherwise we can 276 * not correctly determine insn->call_dest->sec (external symbols do 277 * not have a section). 278 */ 279 if (opts.link && opts.noinstr && sec) 280 state->noinstr = sec->noinstr; 281 } 282 283 static struct cfi_state *cfi_alloc(void) 284 { 285 struct cfi_state *cfi = calloc(sizeof(struct cfi_state), 1); 286 if (!cfi) { 287 WARN("calloc failed"); 288 exit(1); 289 } 290 nr_cfi++; 291 return cfi; 292 } 293 294 static int cfi_bits; 295 static struct hlist_head *cfi_hash; 296 297 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2) 298 { 299 return memcmp((void *)cfi1 + sizeof(cfi1->hash), 300 (void *)cfi2 + sizeof(cfi2->hash), 301 sizeof(struct cfi_state) - sizeof(struct hlist_node)); 302 } 303 304 static inline u32 cfi_key(struct cfi_state *cfi) 305 { 306 return jhash((void *)cfi + sizeof(cfi->hash), 307 sizeof(*cfi) - sizeof(cfi->hash), 0); 308 } 309 310 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi) 311 { 312 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 313 struct cfi_state *obj; 314 315 hlist_for_each_entry(obj, head, hash) { 316 if (!cficmp(cfi, obj)) { 317 nr_cfi_cache++; 318 return obj; 319 } 320 } 321 322 obj = cfi_alloc(); 323 *obj = *cfi; 324 hlist_add_head(&obj->hash, head); 325 326 return obj; 327 } 328 329 static void cfi_hash_add(struct cfi_state *cfi) 330 { 331 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 332 333 hlist_add_head(&cfi->hash, head); 334 } 335 336 static void *cfi_hash_alloc(unsigned long size) 337 { 338 cfi_bits = max(10, ilog2(size)); 339 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits, 340 PROT_READ|PROT_WRITE, 341 MAP_PRIVATE|MAP_ANON, -1, 0); 342 if (cfi_hash == (void *)-1L) { 343 WARN("mmap fail cfi_hash"); 344 cfi_hash = NULL; 345 } else if (opts.stats) { 346 printf("cfi_bits: %d\n", cfi_bits); 347 } 348 349 return cfi_hash; 350 } 351 352 static unsigned long nr_insns; 353 static unsigned long nr_insns_visited; 354 355 /* 356 * Call the arch-specific instruction decoder for all the instructions and add 357 * them to the global instruction list. 358 */ 359 static int decode_instructions(struct objtool_file *file) 360 { 361 struct section *sec; 362 struct symbol *func; 363 unsigned long offset; 364 struct instruction *insn; 365 int ret; 366 367 for_each_sec(file, sec) { 368 369 if (!(sec->sh.sh_flags & SHF_EXECINSTR)) 370 continue; 371 372 if (strcmp(sec->name, ".altinstr_replacement") && 373 strcmp(sec->name, ".altinstr_aux") && 374 strncmp(sec->name, ".discard.", 9)) 375 sec->text = true; 376 377 if (!strcmp(sec->name, ".noinstr.text") || 378 !strcmp(sec->name, ".entry.text") || 379 !strcmp(sec->name, ".cpuidle.text") || 380 !strncmp(sec->name, ".text.__x86.", 12)) 381 sec->noinstr = true; 382 383 /* 384 * .init.text code is ran before userspace and thus doesn't 385 * strictly need retpolines, except for modules which are 386 * loaded late, they very much do need retpoline in their 387 * .init.text 388 */ 389 if (!strcmp(sec->name, ".init.text") && !opts.module) 390 sec->init = true; 391 392 for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) { 393 insn = malloc(sizeof(*insn)); 394 if (!insn) { 395 WARN("malloc failed"); 396 return -1; 397 } 398 memset(insn, 0, sizeof(*insn)); 399 INIT_LIST_HEAD(&insn->alts); 400 INIT_LIST_HEAD(&insn->stack_ops); 401 INIT_LIST_HEAD(&insn->call_node); 402 403 insn->sec = sec; 404 insn->offset = offset; 405 406 ret = arch_decode_instruction(file, sec, offset, 407 sec->sh.sh_size - offset, 408 &insn->len, &insn->type, 409 &insn->immediate, 410 &insn->stack_ops); 411 if (ret) 412 goto err; 413 414 /* 415 * By default, "ud2" is a dead end unless otherwise 416 * annotated, because GCC 7 inserts it for certain 417 * divide-by-zero cases. 418 */ 419 if (insn->type == INSN_BUG) 420 insn->dead_end = true; 421 422 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset)); 423 list_add_tail(&insn->list, &file->insn_list); 424 nr_insns++; 425 } 426 427 list_for_each_entry(func, &sec->symbol_list, list) { 428 if (func->type != STT_NOTYPE && func->type != STT_FUNC) 429 continue; 430 431 if (func->offset == sec->sh.sh_size) { 432 /* Heuristic: likely an "end" symbol */ 433 if (func->type == STT_NOTYPE) 434 continue; 435 WARN("%s(): STT_FUNC at end of section", 436 func->name); 437 return -1; 438 } 439 440 if (func->return_thunk || func->alias != func) 441 continue; 442 443 if (!find_insn(file, sec, func->offset)) { 444 WARN("%s(): can't find starting instruction", 445 func->name); 446 return -1; 447 } 448 449 sym_for_each_insn(file, func, insn) { 450 insn->sym = func; 451 if (func->type == STT_FUNC && 452 insn->type == INSN_ENDBR && 453 list_empty(&insn->call_node)) { 454 if (insn->offset == func->offset) { 455 list_add_tail(&insn->call_node, &file->endbr_list); 456 file->nr_endbr++; 457 } else { 458 file->nr_endbr_int++; 459 } 460 } 461 } 462 } 463 } 464 465 if (opts.stats) 466 printf("nr_insns: %lu\n", nr_insns); 467 468 return 0; 469 470 err: 471 free(insn); 472 return ret; 473 } 474 475 /* 476 * Read the pv_ops[] .data table to find the static initialized values. 477 */ 478 static int add_pv_ops(struct objtool_file *file, const char *symname) 479 { 480 struct symbol *sym, *func; 481 unsigned long off, end; 482 struct reloc *rel; 483 int idx; 484 485 sym = find_symbol_by_name(file->elf, symname); 486 if (!sym) 487 return 0; 488 489 off = sym->offset; 490 end = off + sym->len; 491 for (;;) { 492 rel = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off); 493 if (!rel) 494 break; 495 496 func = rel->sym; 497 if (func->type == STT_SECTION) 498 func = find_symbol_by_offset(rel->sym->sec, rel->addend); 499 500 idx = (rel->offset - sym->offset) / sizeof(unsigned long); 501 502 objtool_pv_add(file, idx, func); 503 504 off = rel->offset + 1; 505 if (off > end) 506 break; 507 } 508 509 return 0; 510 } 511 512 /* 513 * Allocate and initialize file->pv_ops[]. 514 */ 515 static int init_pv_ops(struct objtool_file *file) 516 { 517 static const char *pv_ops_tables[] = { 518 "pv_ops", 519 "xen_cpu_ops", 520 "xen_irq_ops", 521 "xen_mmu_ops", 522 NULL, 523 }; 524 const char *pv_ops; 525 struct symbol *sym; 526 int idx, nr; 527 528 if (!opts.noinstr) 529 return 0; 530 531 file->pv_ops = NULL; 532 533 sym = find_symbol_by_name(file->elf, "pv_ops"); 534 if (!sym) 535 return 0; 536 537 nr = sym->len / sizeof(unsigned long); 538 file->pv_ops = calloc(sizeof(struct pv_state), nr); 539 if (!file->pv_ops) 540 return -1; 541 542 for (idx = 0; idx < nr; idx++) 543 INIT_LIST_HEAD(&file->pv_ops[idx].targets); 544 545 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++) 546 add_pv_ops(file, pv_ops); 547 548 return 0; 549 } 550 551 static struct instruction *find_last_insn(struct objtool_file *file, 552 struct section *sec) 553 { 554 struct instruction *insn = NULL; 555 unsigned int offset; 556 unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0; 557 558 for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--) 559 insn = find_insn(file, sec, offset); 560 561 return insn; 562 } 563 564 /* 565 * Mark "ud2" instructions and manually annotated dead ends. 566 */ 567 static int add_dead_ends(struct objtool_file *file) 568 { 569 struct section *sec; 570 struct reloc *reloc; 571 struct instruction *insn; 572 573 /* 574 * Check for manually annotated dead ends. 575 */ 576 sec = find_section_by_name(file->elf, ".rela.discard.unreachable"); 577 if (!sec) 578 goto reachable; 579 580 list_for_each_entry(reloc, &sec->reloc_list, list) { 581 if (reloc->sym->type != STT_SECTION) { 582 WARN("unexpected relocation symbol type in %s", sec->name); 583 return -1; 584 } 585 insn = find_insn(file, reloc->sym->sec, reloc->addend); 586 if (insn) 587 insn = list_prev_entry(insn, list); 588 else if (reloc->addend == reloc->sym->sec->sh.sh_size) { 589 insn = find_last_insn(file, reloc->sym->sec); 590 if (!insn) { 591 WARN("can't find unreachable insn at %s+0x%" PRIx64, 592 reloc->sym->sec->name, reloc->addend); 593 return -1; 594 } 595 } else { 596 WARN("can't find unreachable insn at %s+0x%" PRIx64, 597 reloc->sym->sec->name, reloc->addend); 598 return -1; 599 } 600 601 insn->dead_end = true; 602 } 603 604 reachable: 605 /* 606 * These manually annotated reachable checks are needed for GCC 4.4, 607 * where the Linux unreachable() macro isn't supported. In that case 608 * GCC doesn't know the "ud2" is fatal, so it generates code as if it's 609 * not a dead end. 610 */ 611 sec = find_section_by_name(file->elf, ".rela.discard.reachable"); 612 if (!sec) 613 return 0; 614 615 list_for_each_entry(reloc, &sec->reloc_list, list) { 616 if (reloc->sym->type != STT_SECTION) { 617 WARN("unexpected relocation symbol type in %s", sec->name); 618 return -1; 619 } 620 insn = find_insn(file, reloc->sym->sec, reloc->addend); 621 if (insn) 622 insn = list_prev_entry(insn, list); 623 else if (reloc->addend == reloc->sym->sec->sh.sh_size) { 624 insn = find_last_insn(file, reloc->sym->sec); 625 if (!insn) { 626 WARN("can't find reachable insn at %s+0x%" PRIx64, 627 reloc->sym->sec->name, reloc->addend); 628 return -1; 629 } 630 } else { 631 WARN("can't find reachable insn at %s+0x%" PRIx64, 632 reloc->sym->sec->name, reloc->addend); 633 return -1; 634 } 635 636 insn->dead_end = false; 637 } 638 639 return 0; 640 } 641 642 static int create_static_call_sections(struct objtool_file *file) 643 { 644 struct section *sec; 645 struct static_call_site *site; 646 struct instruction *insn; 647 struct symbol *key_sym; 648 char *key_name, *tmp; 649 int idx; 650 651 sec = find_section_by_name(file->elf, ".static_call_sites"); 652 if (sec) { 653 INIT_LIST_HEAD(&file->static_call_list); 654 WARN("file already has .static_call_sites section, skipping"); 655 return 0; 656 } 657 658 if (list_empty(&file->static_call_list)) 659 return 0; 660 661 idx = 0; 662 list_for_each_entry(insn, &file->static_call_list, call_node) 663 idx++; 664 665 sec = elf_create_section(file->elf, ".static_call_sites", SHF_WRITE, 666 sizeof(struct static_call_site), idx); 667 if (!sec) 668 return -1; 669 670 idx = 0; 671 list_for_each_entry(insn, &file->static_call_list, call_node) { 672 673 site = (struct static_call_site *)sec->data->d_buf + idx; 674 memset(site, 0, sizeof(struct static_call_site)); 675 676 /* populate reloc for 'addr' */ 677 if (elf_add_reloc_to_insn(file->elf, sec, 678 idx * sizeof(struct static_call_site), 679 R_X86_64_PC32, 680 insn->sec, insn->offset)) 681 return -1; 682 683 /* find key symbol */ 684 key_name = strdup(insn->call_dest->name); 685 if (!key_name) { 686 perror("strdup"); 687 return -1; 688 } 689 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR, 690 STATIC_CALL_TRAMP_PREFIX_LEN)) { 691 WARN("static_call: trampoline name malformed: %s", key_name); 692 return -1; 693 } 694 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN; 695 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN); 696 697 key_sym = find_symbol_by_name(file->elf, tmp); 698 if (!key_sym) { 699 if (!opts.module) { 700 WARN("static_call: can't find static_call_key symbol: %s", tmp); 701 return -1; 702 } 703 704 /* 705 * For modules(), the key might not be exported, which 706 * means the module can make static calls but isn't 707 * allowed to change them. 708 * 709 * In that case we temporarily set the key to be the 710 * trampoline address. This is fixed up in 711 * static_call_add_module(). 712 */ 713 key_sym = insn->call_dest; 714 } 715 free(key_name); 716 717 /* populate reloc for 'key' */ 718 if (elf_add_reloc(file->elf, sec, 719 idx * sizeof(struct static_call_site) + 4, 720 R_X86_64_PC32, key_sym, 721 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL)) 722 return -1; 723 724 idx++; 725 } 726 727 return 0; 728 } 729 730 static int create_retpoline_sites_sections(struct objtool_file *file) 731 { 732 struct instruction *insn; 733 struct section *sec; 734 int idx; 735 736 sec = find_section_by_name(file->elf, ".retpoline_sites"); 737 if (sec) { 738 WARN("file already has .retpoline_sites, skipping"); 739 return 0; 740 } 741 742 idx = 0; 743 list_for_each_entry(insn, &file->retpoline_call_list, call_node) 744 idx++; 745 746 if (!idx) 747 return 0; 748 749 sec = elf_create_section(file->elf, ".retpoline_sites", 0, 750 sizeof(int), idx); 751 if (!sec) { 752 WARN("elf_create_section: .retpoline_sites"); 753 return -1; 754 } 755 756 idx = 0; 757 list_for_each_entry(insn, &file->retpoline_call_list, call_node) { 758 759 int *site = (int *)sec->data->d_buf + idx; 760 *site = 0; 761 762 if (elf_add_reloc_to_insn(file->elf, sec, 763 idx * sizeof(int), 764 R_X86_64_PC32, 765 insn->sec, insn->offset)) { 766 WARN("elf_add_reloc_to_insn: .retpoline_sites"); 767 return -1; 768 } 769 770 idx++; 771 } 772 773 return 0; 774 } 775 776 static int create_return_sites_sections(struct objtool_file *file) 777 { 778 struct instruction *insn; 779 struct section *sec; 780 int idx; 781 782 sec = find_section_by_name(file->elf, ".return_sites"); 783 if (sec) { 784 WARN("file already has .return_sites, skipping"); 785 return 0; 786 } 787 788 idx = 0; 789 list_for_each_entry(insn, &file->return_thunk_list, call_node) 790 idx++; 791 792 if (!idx) 793 return 0; 794 795 sec = elf_create_section(file->elf, ".return_sites", 0, 796 sizeof(int), idx); 797 if (!sec) { 798 WARN("elf_create_section: .return_sites"); 799 return -1; 800 } 801 802 idx = 0; 803 list_for_each_entry(insn, &file->return_thunk_list, call_node) { 804 805 int *site = (int *)sec->data->d_buf + idx; 806 *site = 0; 807 808 if (elf_add_reloc_to_insn(file->elf, sec, 809 idx * sizeof(int), 810 R_X86_64_PC32, 811 insn->sec, insn->offset)) { 812 WARN("elf_add_reloc_to_insn: .return_sites"); 813 return -1; 814 } 815 816 idx++; 817 } 818 819 return 0; 820 } 821 822 static int create_ibt_endbr_seal_sections(struct objtool_file *file) 823 { 824 struct instruction *insn; 825 struct section *sec; 826 int idx; 827 828 sec = find_section_by_name(file->elf, ".ibt_endbr_seal"); 829 if (sec) { 830 WARN("file already has .ibt_endbr_seal, skipping"); 831 return 0; 832 } 833 834 idx = 0; 835 list_for_each_entry(insn, &file->endbr_list, call_node) 836 idx++; 837 838 if (opts.stats) { 839 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr); 840 printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int); 841 printf("ibt: superfluous ENDBR: %d\n", idx); 842 } 843 844 if (!idx) 845 return 0; 846 847 sec = elf_create_section(file->elf, ".ibt_endbr_seal", 0, 848 sizeof(int), idx); 849 if (!sec) { 850 WARN("elf_create_section: .ibt_endbr_seal"); 851 return -1; 852 } 853 854 idx = 0; 855 list_for_each_entry(insn, &file->endbr_list, call_node) { 856 857 int *site = (int *)sec->data->d_buf + idx; 858 *site = 0; 859 860 if (elf_add_reloc_to_insn(file->elf, sec, 861 idx * sizeof(int), 862 R_X86_64_PC32, 863 insn->sec, insn->offset)) { 864 WARN("elf_add_reloc_to_insn: .ibt_endbr_seal"); 865 return -1; 866 } 867 868 idx++; 869 } 870 871 return 0; 872 } 873 874 static int create_cfi_sections(struct objtool_file *file) 875 { 876 struct section *sec, *s; 877 struct symbol *sym; 878 unsigned int *loc; 879 int idx; 880 881 sec = find_section_by_name(file->elf, ".cfi_sites"); 882 if (sec) { 883 INIT_LIST_HEAD(&file->call_list); 884 WARN("file already has .cfi_sites section, skipping"); 885 return 0; 886 } 887 888 idx = 0; 889 for_each_sec(file, s) { 890 if (!s->text) 891 continue; 892 893 list_for_each_entry(sym, &s->symbol_list, list) { 894 if (sym->type != STT_FUNC) 895 continue; 896 897 if (strncmp(sym->name, "__cfi_", 6)) 898 continue; 899 900 idx++; 901 } 902 } 903 904 sec = elf_create_section(file->elf, ".cfi_sites", 0, sizeof(unsigned int), idx); 905 if (!sec) 906 return -1; 907 908 idx = 0; 909 for_each_sec(file, s) { 910 if (!s->text) 911 continue; 912 913 list_for_each_entry(sym, &s->symbol_list, list) { 914 if (sym->type != STT_FUNC) 915 continue; 916 917 if (strncmp(sym->name, "__cfi_", 6)) 918 continue; 919 920 loc = (unsigned int *)sec->data->d_buf + idx; 921 memset(loc, 0, sizeof(unsigned int)); 922 923 if (elf_add_reloc_to_insn(file->elf, sec, 924 idx * sizeof(unsigned int), 925 R_X86_64_PC32, 926 s, sym->offset)) 927 return -1; 928 929 idx++; 930 } 931 } 932 933 return 0; 934 } 935 936 static int create_mcount_loc_sections(struct objtool_file *file) 937 { 938 int addrsize = elf_class_addrsize(file->elf); 939 struct instruction *insn; 940 struct section *sec; 941 int idx; 942 943 sec = find_section_by_name(file->elf, "__mcount_loc"); 944 if (sec) { 945 INIT_LIST_HEAD(&file->mcount_loc_list); 946 WARN("file already has __mcount_loc section, skipping"); 947 return 0; 948 } 949 950 if (list_empty(&file->mcount_loc_list)) 951 return 0; 952 953 idx = 0; 954 list_for_each_entry(insn, &file->mcount_loc_list, call_node) 955 idx++; 956 957 sec = elf_create_section(file->elf, "__mcount_loc", 0, addrsize, idx); 958 if (!sec) 959 return -1; 960 961 sec->sh.sh_addralign = addrsize; 962 963 idx = 0; 964 list_for_each_entry(insn, &file->mcount_loc_list, call_node) { 965 void *loc; 966 967 loc = sec->data->d_buf + idx; 968 memset(loc, 0, addrsize); 969 970 if (elf_add_reloc_to_insn(file->elf, sec, idx, 971 addrsize == sizeof(u64) ? R_ABS64 : R_ABS32, 972 insn->sec, insn->offset)) 973 return -1; 974 975 idx += addrsize; 976 } 977 978 return 0; 979 } 980 981 static int create_direct_call_sections(struct objtool_file *file) 982 { 983 struct instruction *insn; 984 struct section *sec; 985 unsigned int *loc; 986 int idx; 987 988 sec = find_section_by_name(file->elf, ".call_sites"); 989 if (sec) { 990 INIT_LIST_HEAD(&file->call_list); 991 WARN("file already has .call_sites section, skipping"); 992 return 0; 993 } 994 995 if (list_empty(&file->call_list)) 996 return 0; 997 998 idx = 0; 999 list_for_each_entry(insn, &file->call_list, call_node) 1000 idx++; 1001 1002 sec = elf_create_section(file->elf, ".call_sites", 0, sizeof(unsigned int), idx); 1003 if (!sec) 1004 return -1; 1005 1006 idx = 0; 1007 list_for_each_entry(insn, &file->call_list, call_node) { 1008 1009 loc = (unsigned int *)sec->data->d_buf + idx; 1010 memset(loc, 0, sizeof(unsigned int)); 1011 1012 if (elf_add_reloc_to_insn(file->elf, sec, 1013 idx * sizeof(unsigned int), 1014 R_X86_64_PC32, 1015 insn->sec, insn->offset)) 1016 return -1; 1017 1018 idx++; 1019 } 1020 1021 return 0; 1022 } 1023 1024 /* 1025 * Warnings shouldn't be reported for ignored functions. 1026 */ 1027 static void add_ignores(struct objtool_file *file) 1028 { 1029 struct instruction *insn; 1030 struct section *sec; 1031 struct symbol *func; 1032 struct reloc *reloc; 1033 1034 sec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard"); 1035 if (!sec) 1036 return; 1037 1038 list_for_each_entry(reloc, &sec->reloc_list, list) { 1039 switch (reloc->sym->type) { 1040 case STT_FUNC: 1041 func = reloc->sym; 1042 break; 1043 1044 case STT_SECTION: 1045 func = find_func_by_offset(reloc->sym->sec, reloc->addend); 1046 if (!func) 1047 continue; 1048 break; 1049 1050 default: 1051 WARN("unexpected relocation symbol type in %s: %d", sec->name, reloc->sym->type); 1052 continue; 1053 } 1054 1055 func_for_each_insn(file, func, insn) 1056 insn->ignore = true; 1057 } 1058 } 1059 1060 /* 1061 * This is a whitelist of functions that is allowed to be called with AC set. 1062 * The list is meant to be minimal and only contains compiler instrumentation 1063 * ABI and a few functions used to implement *_{to,from}_user() functions. 1064 * 1065 * These functions must not directly change AC, but may PUSHF/POPF. 1066 */ 1067 static const char *uaccess_safe_builtin[] = { 1068 /* KASAN */ 1069 "kasan_report", 1070 "kasan_check_range", 1071 /* KASAN out-of-line */ 1072 "__asan_loadN_noabort", 1073 "__asan_load1_noabort", 1074 "__asan_load2_noabort", 1075 "__asan_load4_noabort", 1076 "__asan_load8_noabort", 1077 "__asan_load16_noabort", 1078 "__asan_storeN_noabort", 1079 "__asan_store1_noabort", 1080 "__asan_store2_noabort", 1081 "__asan_store4_noabort", 1082 "__asan_store8_noabort", 1083 "__asan_store16_noabort", 1084 "__kasan_check_read", 1085 "__kasan_check_write", 1086 /* KASAN in-line */ 1087 "__asan_report_load_n_noabort", 1088 "__asan_report_load1_noabort", 1089 "__asan_report_load2_noabort", 1090 "__asan_report_load4_noabort", 1091 "__asan_report_load8_noabort", 1092 "__asan_report_load16_noabort", 1093 "__asan_report_store_n_noabort", 1094 "__asan_report_store1_noabort", 1095 "__asan_report_store2_noabort", 1096 "__asan_report_store4_noabort", 1097 "__asan_report_store8_noabort", 1098 "__asan_report_store16_noabort", 1099 /* KCSAN */ 1100 "__kcsan_check_access", 1101 "__kcsan_mb", 1102 "__kcsan_wmb", 1103 "__kcsan_rmb", 1104 "__kcsan_release", 1105 "kcsan_found_watchpoint", 1106 "kcsan_setup_watchpoint", 1107 "kcsan_check_scoped_accesses", 1108 "kcsan_disable_current", 1109 "kcsan_enable_current_nowarn", 1110 /* KCSAN/TSAN */ 1111 "__tsan_func_entry", 1112 "__tsan_func_exit", 1113 "__tsan_read_range", 1114 "__tsan_write_range", 1115 "__tsan_read1", 1116 "__tsan_read2", 1117 "__tsan_read4", 1118 "__tsan_read8", 1119 "__tsan_read16", 1120 "__tsan_write1", 1121 "__tsan_write2", 1122 "__tsan_write4", 1123 "__tsan_write8", 1124 "__tsan_write16", 1125 "__tsan_read_write1", 1126 "__tsan_read_write2", 1127 "__tsan_read_write4", 1128 "__tsan_read_write8", 1129 "__tsan_read_write16", 1130 "__tsan_volatile_read1", 1131 "__tsan_volatile_read2", 1132 "__tsan_volatile_read4", 1133 "__tsan_volatile_read8", 1134 "__tsan_volatile_read16", 1135 "__tsan_volatile_write1", 1136 "__tsan_volatile_write2", 1137 "__tsan_volatile_write4", 1138 "__tsan_volatile_write8", 1139 "__tsan_volatile_write16", 1140 "__tsan_atomic8_load", 1141 "__tsan_atomic16_load", 1142 "__tsan_atomic32_load", 1143 "__tsan_atomic64_load", 1144 "__tsan_atomic8_store", 1145 "__tsan_atomic16_store", 1146 "__tsan_atomic32_store", 1147 "__tsan_atomic64_store", 1148 "__tsan_atomic8_exchange", 1149 "__tsan_atomic16_exchange", 1150 "__tsan_atomic32_exchange", 1151 "__tsan_atomic64_exchange", 1152 "__tsan_atomic8_fetch_add", 1153 "__tsan_atomic16_fetch_add", 1154 "__tsan_atomic32_fetch_add", 1155 "__tsan_atomic64_fetch_add", 1156 "__tsan_atomic8_fetch_sub", 1157 "__tsan_atomic16_fetch_sub", 1158 "__tsan_atomic32_fetch_sub", 1159 "__tsan_atomic64_fetch_sub", 1160 "__tsan_atomic8_fetch_and", 1161 "__tsan_atomic16_fetch_and", 1162 "__tsan_atomic32_fetch_and", 1163 "__tsan_atomic64_fetch_and", 1164 "__tsan_atomic8_fetch_or", 1165 "__tsan_atomic16_fetch_or", 1166 "__tsan_atomic32_fetch_or", 1167 "__tsan_atomic64_fetch_or", 1168 "__tsan_atomic8_fetch_xor", 1169 "__tsan_atomic16_fetch_xor", 1170 "__tsan_atomic32_fetch_xor", 1171 "__tsan_atomic64_fetch_xor", 1172 "__tsan_atomic8_fetch_nand", 1173 "__tsan_atomic16_fetch_nand", 1174 "__tsan_atomic32_fetch_nand", 1175 "__tsan_atomic64_fetch_nand", 1176 "__tsan_atomic8_compare_exchange_strong", 1177 "__tsan_atomic16_compare_exchange_strong", 1178 "__tsan_atomic32_compare_exchange_strong", 1179 "__tsan_atomic64_compare_exchange_strong", 1180 "__tsan_atomic8_compare_exchange_weak", 1181 "__tsan_atomic16_compare_exchange_weak", 1182 "__tsan_atomic32_compare_exchange_weak", 1183 "__tsan_atomic64_compare_exchange_weak", 1184 "__tsan_atomic8_compare_exchange_val", 1185 "__tsan_atomic16_compare_exchange_val", 1186 "__tsan_atomic32_compare_exchange_val", 1187 "__tsan_atomic64_compare_exchange_val", 1188 "__tsan_atomic_thread_fence", 1189 "__tsan_atomic_signal_fence", 1190 /* KCOV */ 1191 "write_comp_data", 1192 "check_kcov_mode", 1193 "__sanitizer_cov_trace_pc", 1194 "__sanitizer_cov_trace_const_cmp1", 1195 "__sanitizer_cov_trace_const_cmp2", 1196 "__sanitizer_cov_trace_const_cmp4", 1197 "__sanitizer_cov_trace_const_cmp8", 1198 "__sanitizer_cov_trace_cmp1", 1199 "__sanitizer_cov_trace_cmp2", 1200 "__sanitizer_cov_trace_cmp4", 1201 "__sanitizer_cov_trace_cmp8", 1202 "__sanitizer_cov_trace_switch", 1203 /* KMSAN */ 1204 "kmsan_copy_to_user", 1205 "kmsan_report", 1206 "kmsan_unpoison_entry_regs", 1207 "kmsan_unpoison_memory", 1208 "__msan_chain_origin", 1209 "__msan_get_context_state", 1210 "__msan_instrument_asm_store", 1211 "__msan_metadata_ptr_for_load_1", 1212 "__msan_metadata_ptr_for_load_2", 1213 "__msan_metadata_ptr_for_load_4", 1214 "__msan_metadata_ptr_for_load_8", 1215 "__msan_metadata_ptr_for_load_n", 1216 "__msan_metadata_ptr_for_store_1", 1217 "__msan_metadata_ptr_for_store_2", 1218 "__msan_metadata_ptr_for_store_4", 1219 "__msan_metadata_ptr_for_store_8", 1220 "__msan_metadata_ptr_for_store_n", 1221 "__msan_poison_alloca", 1222 "__msan_warning", 1223 /* UBSAN */ 1224 "ubsan_type_mismatch_common", 1225 "__ubsan_handle_type_mismatch", 1226 "__ubsan_handle_type_mismatch_v1", 1227 "__ubsan_handle_shift_out_of_bounds", 1228 "__ubsan_handle_load_invalid_value", 1229 /* misc */ 1230 "csum_partial_copy_generic", 1231 "copy_mc_fragile", 1232 "copy_mc_fragile_handle_tail", 1233 "copy_mc_enhanced_fast_string", 1234 "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */ 1235 "clear_user_erms", 1236 "clear_user_rep_good", 1237 "clear_user_original", 1238 NULL 1239 }; 1240 1241 static void add_uaccess_safe(struct objtool_file *file) 1242 { 1243 struct symbol *func; 1244 const char **name; 1245 1246 if (!opts.uaccess) 1247 return; 1248 1249 for (name = uaccess_safe_builtin; *name; name++) { 1250 func = find_symbol_by_name(file->elf, *name); 1251 if (!func) 1252 continue; 1253 1254 func->uaccess_safe = true; 1255 } 1256 } 1257 1258 /* 1259 * FIXME: For now, just ignore any alternatives which add retpolines. This is 1260 * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline. 1261 * But it at least allows objtool to understand the control flow *around* the 1262 * retpoline. 1263 */ 1264 static int add_ignore_alternatives(struct objtool_file *file) 1265 { 1266 struct section *sec; 1267 struct reloc *reloc; 1268 struct instruction *insn; 1269 1270 sec = find_section_by_name(file->elf, ".rela.discard.ignore_alts"); 1271 if (!sec) 1272 return 0; 1273 1274 list_for_each_entry(reloc, &sec->reloc_list, list) { 1275 if (reloc->sym->type != STT_SECTION) { 1276 WARN("unexpected relocation symbol type in %s", sec->name); 1277 return -1; 1278 } 1279 1280 insn = find_insn(file, reloc->sym->sec, reloc->addend); 1281 if (!insn) { 1282 WARN("bad .discard.ignore_alts entry"); 1283 return -1; 1284 } 1285 1286 insn->ignore_alts = true; 1287 } 1288 1289 return 0; 1290 } 1291 1292 __weak bool arch_is_retpoline(struct symbol *sym) 1293 { 1294 return false; 1295 } 1296 1297 __weak bool arch_is_rethunk(struct symbol *sym) 1298 { 1299 return false; 1300 } 1301 1302 #define NEGATIVE_RELOC ((void *)-1L) 1303 1304 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn) 1305 { 1306 if (insn->reloc == NEGATIVE_RELOC) 1307 return NULL; 1308 1309 if (!insn->reloc) { 1310 if (!file) 1311 return NULL; 1312 1313 insn->reloc = find_reloc_by_dest_range(file->elf, insn->sec, 1314 insn->offset, insn->len); 1315 if (!insn->reloc) { 1316 insn->reloc = NEGATIVE_RELOC; 1317 return NULL; 1318 } 1319 } 1320 1321 return insn->reloc; 1322 } 1323 1324 static void remove_insn_ops(struct instruction *insn) 1325 { 1326 struct stack_op *op, *tmp; 1327 1328 list_for_each_entry_safe(op, tmp, &insn->stack_ops, list) { 1329 list_del(&op->list); 1330 free(op); 1331 } 1332 } 1333 1334 static void annotate_call_site(struct objtool_file *file, 1335 struct instruction *insn, bool sibling) 1336 { 1337 struct reloc *reloc = insn_reloc(file, insn); 1338 struct symbol *sym = insn->call_dest; 1339 1340 if (!sym) 1341 sym = reloc->sym; 1342 1343 /* 1344 * Alternative replacement code is just template code which is 1345 * sometimes copied to the original instruction. For now, don't 1346 * annotate it. (In the future we might consider annotating the 1347 * original instruction if/when it ever makes sense to do so.) 1348 */ 1349 if (!strcmp(insn->sec->name, ".altinstr_replacement")) 1350 return; 1351 1352 if (sym->static_call_tramp) { 1353 list_add_tail(&insn->call_node, &file->static_call_list); 1354 return; 1355 } 1356 1357 if (sym->retpoline_thunk) { 1358 list_add_tail(&insn->call_node, &file->retpoline_call_list); 1359 return; 1360 } 1361 1362 /* 1363 * Many compilers cannot disable KCOV or sanitizer calls with a function 1364 * attribute so they need a little help, NOP out any such calls from 1365 * noinstr text. 1366 */ 1367 if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) { 1368 if (reloc) { 1369 reloc->type = R_NONE; 1370 elf_write_reloc(file->elf, reloc); 1371 } 1372 1373 elf_write_insn(file->elf, insn->sec, 1374 insn->offset, insn->len, 1375 sibling ? arch_ret_insn(insn->len) 1376 : arch_nop_insn(insn->len)); 1377 1378 insn->type = sibling ? INSN_RETURN : INSN_NOP; 1379 1380 if (sibling) { 1381 /* 1382 * We've replaced the tail-call JMP insn by two new 1383 * insn: RET; INT3, except we only have a single struct 1384 * insn here. Mark it retpoline_safe to avoid the SLS 1385 * warning, instead of adding another insn. 1386 */ 1387 insn->retpoline_safe = true; 1388 } 1389 1390 return; 1391 } 1392 1393 if (opts.mcount && sym->fentry) { 1394 if (sibling) 1395 WARN_FUNC("Tail call to __fentry__ !?!?", insn->sec, insn->offset); 1396 if (opts.mnop) { 1397 if (reloc) { 1398 reloc->type = R_NONE; 1399 elf_write_reloc(file->elf, reloc); 1400 } 1401 1402 elf_write_insn(file->elf, insn->sec, 1403 insn->offset, insn->len, 1404 arch_nop_insn(insn->len)); 1405 1406 insn->type = INSN_NOP; 1407 } 1408 1409 list_add_tail(&insn->call_node, &file->mcount_loc_list); 1410 return; 1411 } 1412 1413 if (insn->type == INSN_CALL && !insn->sec->init) 1414 list_add_tail(&insn->call_node, &file->call_list); 1415 1416 if (!sibling && dead_end_function(file, sym)) 1417 insn->dead_end = true; 1418 } 1419 1420 static void add_call_dest(struct objtool_file *file, struct instruction *insn, 1421 struct symbol *dest, bool sibling) 1422 { 1423 insn->call_dest = dest; 1424 if (!dest) 1425 return; 1426 1427 /* 1428 * Whatever stack impact regular CALLs have, should be undone 1429 * by the RETURN of the called function. 1430 * 1431 * Annotated intra-function calls retain the stack_ops but 1432 * are converted to JUMP, see read_intra_function_calls(). 1433 */ 1434 remove_insn_ops(insn); 1435 1436 annotate_call_site(file, insn, sibling); 1437 } 1438 1439 static void add_retpoline_call(struct objtool_file *file, struct instruction *insn) 1440 { 1441 /* 1442 * Retpoline calls/jumps are really dynamic calls/jumps in disguise, 1443 * so convert them accordingly. 1444 */ 1445 switch (insn->type) { 1446 case INSN_CALL: 1447 insn->type = INSN_CALL_DYNAMIC; 1448 break; 1449 case INSN_JUMP_UNCONDITIONAL: 1450 insn->type = INSN_JUMP_DYNAMIC; 1451 break; 1452 case INSN_JUMP_CONDITIONAL: 1453 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL; 1454 break; 1455 default: 1456 return; 1457 } 1458 1459 insn->retpoline_safe = true; 1460 1461 /* 1462 * Whatever stack impact regular CALLs have, should be undone 1463 * by the RETURN of the called function. 1464 * 1465 * Annotated intra-function calls retain the stack_ops but 1466 * are converted to JUMP, see read_intra_function_calls(). 1467 */ 1468 remove_insn_ops(insn); 1469 1470 annotate_call_site(file, insn, false); 1471 } 1472 1473 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add) 1474 { 1475 /* 1476 * Return thunk tail calls are really just returns in disguise, 1477 * so convert them accordingly. 1478 */ 1479 insn->type = INSN_RETURN; 1480 insn->retpoline_safe = true; 1481 1482 if (add) 1483 list_add_tail(&insn->call_node, &file->return_thunk_list); 1484 } 1485 1486 static bool is_first_func_insn(struct objtool_file *file, 1487 struct instruction *insn, struct symbol *sym) 1488 { 1489 if (insn->offset == sym->offset) 1490 return true; 1491 1492 /* Allow direct CALL/JMP past ENDBR */ 1493 if (opts.ibt) { 1494 struct instruction *prev = prev_insn_same_sym(file, insn); 1495 1496 if (prev && prev->type == INSN_ENDBR && 1497 insn->offset == sym->offset + prev->len) 1498 return true; 1499 } 1500 1501 return false; 1502 } 1503 1504 /* 1505 * A sibling call is a tail-call to another symbol -- to differentiate from a 1506 * recursive tail-call which is to the same symbol. 1507 */ 1508 static bool jump_is_sibling_call(struct objtool_file *file, 1509 struct instruction *from, struct instruction *to) 1510 { 1511 struct symbol *fs = from->sym; 1512 struct symbol *ts = to->sym; 1513 1514 /* Not a sibling call if from/to a symbol hole */ 1515 if (!fs || !ts) 1516 return false; 1517 1518 /* Not a sibling call if not targeting the start of a symbol. */ 1519 if (!is_first_func_insn(file, to, ts)) 1520 return false; 1521 1522 /* Disallow sibling calls into STT_NOTYPE */ 1523 if (ts->type == STT_NOTYPE) 1524 return false; 1525 1526 /* Must not be self to be a sibling */ 1527 return fs->pfunc != ts->pfunc; 1528 } 1529 1530 /* 1531 * Find the destination instructions for all jumps. 1532 */ 1533 static int add_jump_destinations(struct objtool_file *file) 1534 { 1535 struct instruction *insn, *jump_dest; 1536 struct reloc *reloc; 1537 struct section *dest_sec; 1538 unsigned long dest_off; 1539 1540 for_each_insn(file, insn) { 1541 if (insn->jump_dest) { 1542 /* 1543 * handle_group_alt() may have previously set 1544 * 'jump_dest' for some alternatives. 1545 */ 1546 continue; 1547 } 1548 if (!is_static_jump(insn)) 1549 continue; 1550 1551 reloc = insn_reloc(file, insn); 1552 if (!reloc) { 1553 dest_sec = insn->sec; 1554 dest_off = arch_jump_destination(insn); 1555 } else if (reloc->sym->type == STT_SECTION) { 1556 dest_sec = reloc->sym->sec; 1557 dest_off = arch_dest_reloc_offset(reloc->addend); 1558 } else if (reloc->sym->retpoline_thunk) { 1559 add_retpoline_call(file, insn); 1560 continue; 1561 } else if (reloc->sym->return_thunk) { 1562 add_return_call(file, insn, true); 1563 continue; 1564 } else if (insn_func(insn)) { 1565 /* 1566 * External sibling call or internal sibling call with 1567 * STT_FUNC reloc. 1568 */ 1569 add_call_dest(file, insn, reloc->sym, true); 1570 continue; 1571 } else if (reloc->sym->sec->idx) { 1572 dest_sec = reloc->sym->sec; 1573 dest_off = reloc->sym->sym.st_value + 1574 arch_dest_reloc_offset(reloc->addend); 1575 } else { 1576 /* non-func asm code jumping to another file */ 1577 continue; 1578 } 1579 1580 jump_dest = find_insn(file, dest_sec, dest_off); 1581 if (!jump_dest) { 1582 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off); 1583 1584 /* 1585 * This is a special case for zen_untrain_ret(). 1586 * It jumps to __x86_return_thunk(), but objtool 1587 * can't find the thunk's starting RET 1588 * instruction, because the RET is also in the 1589 * middle of another instruction. Objtool only 1590 * knows about the outer instruction. 1591 */ 1592 if (sym && sym->return_thunk) { 1593 add_return_call(file, insn, false); 1594 continue; 1595 } 1596 1597 WARN_FUNC("can't find jump dest instruction at %s+0x%lx", 1598 insn->sec, insn->offset, dest_sec->name, 1599 dest_off); 1600 return -1; 1601 } 1602 1603 /* 1604 * Cross-function jump. 1605 */ 1606 if (insn_func(insn) && insn_func(jump_dest) && 1607 insn_func(insn) != insn_func(jump_dest)) { 1608 1609 /* 1610 * For GCC 8+, create parent/child links for any cold 1611 * subfunctions. This is _mostly_ redundant with a 1612 * similar initialization in read_symbols(). 1613 * 1614 * If a function has aliases, we want the *first* such 1615 * function in the symbol table to be the subfunction's 1616 * parent. In that case we overwrite the 1617 * initialization done in read_symbols(). 1618 * 1619 * However this code can't completely replace the 1620 * read_symbols() code because this doesn't detect the 1621 * case where the parent function's only reference to a 1622 * subfunction is through a jump table. 1623 */ 1624 if (!strstr(insn_func(insn)->name, ".cold") && 1625 strstr(insn_func(jump_dest)->name, ".cold")) { 1626 insn_func(insn)->cfunc = insn_func(jump_dest); 1627 insn_func(jump_dest)->pfunc = insn_func(insn); 1628 } 1629 } 1630 1631 if (jump_is_sibling_call(file, insn, jump_dest)) { 1632 /* 1633 * Internal sibling call without reloc or with 1634 * STT_SECTION reloc. 1635 */ 1636 add_call_dest(file, insn, insn_func(jump_dest), true); 1637 continue; 1638 } 1639 1640 insn->jump_dest = jump_dest; 1641 } 1642 1643 return 0; 1644 } 1645 1646 static struct symbol *find_call_destination(struct section *sec, unsigned long offset) 1647 { 1648 struct symbol *call_dest; 1649 1650 call_dest = find_func_by_offset(sec, offset); 1651 if (!call_dest) 1652 call_dest = find_symbol_by_offset(sec, offset); 1653 1654 return call_dest; 1655 } 1656 1657 /* 1658 * Find the destination instructions for all calls. 1659 */ 1660 static int add_call_destinations(struct objtool_file *file) 1661 { 1662 struct instruction *insn; 1663 unsigned long dest_off; 1664 struct symbol *dest; 1665 struct reloc *reloc; 1666 1667 for_each_insn(file, insn) { 1668 if (insn->type != INSN_CALL) 1669 continue; 1670 1671 reloc = insn_reloc(file, insn); 1672 if (!reloc) { 1673 dest_off = arch_jump_destination(insn); 1674 dest = find_call_destination(insn->sec, dest_off); 1675 1676 add_call_dest(file, insn, dest, false); 1677 1678 if (insn->ignore) 1679 continue; 1680 1681 if (!insn->call_dest) { 1682 WARN_FUNC("unannotated intra-function call", insn->sec, insn->offset); 1683 return -1; 1684 } 1685 1686 if (insn_func(insn) && insn->call_dest->type != STT_FUNC) { 1687 WARN_FUNC("unsupported call to non-function", 1688 insn->sec, insn->offset); 1689 return -1; 1690 } 1691 1692 } else if (reloc->sym->type == STT_SECTION) { 1693 dest_off = arch_dest_reloc_offset(reloc->addend); 1694 dest = find_call_destination(reloc->sym->sec, dest_off); 1695 if (!dest) { 1696 WARN_FUNC("can't find call dest symbol at %s+0x%lx", 1697 insn->sec, insn->offset, 1698 reloc->sym->sec->name, 1699 dest_off); 1700 return -1; 1701 } 1702 1703 add_call_dest(file, insn, dest, false); 1704 1705 } else if (reloc->sym->retpoline_thunk) { 1706 add_retpoline_call(file, insn); 1707 1708 } else 1709 add_call_dest(file, insn, reloc->sym, false); 1710 } 1711 1712 return 0; 1713 } 1714 1715 /* 1716 * The .alternatives section requires some extra special care over and above 1717 * other special sections because alternatives are patched in place. 1718 */ 1719 static int handle_group_alt(struct objtool_file *file, 1720 struct special_alt *special_alt, 1721 struct instruction *orig_insn, 1722 struct instruction **new_insn) 1723 { 1724 struct instruction *last_orig_insn, *last_new_insn = NULL, *insn, *nop = NULL; 1725 struct alt_group *orig_alt_group, *new_alt_group; 1726 unsigned long dest_off; 1727 1728 1729 orig_alt_group = malloc(sizeof(*orig_alt_group)); 1730 if (!orig_alt_group) { 1731 WARN("malloc failed"); 1732 return -1; 1733 } 1734 orig_alt_group->cfi = calloc(special_alt->orig_len, 1735 sizeof(struct cfi_state *)); 1736 if (!orig_alt_group->cfi) { 1737 WARN("calloc failed"); 1738 return -1; 1739 } 1740 1741 last_orig_insn = NULL; 1742 insn = orig_insn; 1743 sec_for_each_insn_from(file, insn) { 1744 if (insn->offset >= special_alt->orig_off + special_alt->orig_len) 1745 break; 1746 1747 insn->alt_group = orig_alt_group; 1748 last_orig_insn = insn; 1749 } 1750 orig_alt_group->orig_group = NULL; 1751 orig_alt_group->first_insn = orig_insn; 1752 orig_alt_group->last_insn = last_orig_insn; 1753 1754 1755 new_alt_group = malloc(sizeof(*new_alt_group)); 1756 if (!new_alt_group) { 1757 WARN("malloc failed"); 1758 return -1; 1759 } 1760 1761 if (special_alt->new_len < special_alt->orig_len) { 1762 /* 1763 * Insert a fake nop at the end to make the replacement 1764 * alt_group the same size as the original. This is needed to 1765 * allow propagate_alt_cfi() to do its magic. When the last 1766 * instruction affects the stack, the instruction after it (the 1767 * nop) will propagate the new state to the shared CFI array. 1768 */ 1769 nop = malloc(sizeof(*nop)); 1770 if (!nop) { 1771 WARN("malloc failed"); 1772 return -1; 1773 } 1774 memset(nop, 0, sizeof(*nop)); 1775 INIT_LIST_HEAD(&nop->alts); 1776 INIT_LIST_HEAD(&nop->stack_ops); 1777 1778 nop->sec = special_alt->new_sec; 1779 nop->offset = special_alt->new_off + special_alt->new_len; 1780 nop->len = special_alt->orig_len - special_alt->new_len; 1781 nop->type = INSN_NOP; 1782 nop->sym = orig_insn->sym; 1783 nop->alt_group = new_alt_group; 1784 nop->ignore = orig_insn->ignore_alts; 1785 } 1786 1787 if (!special_alt->new_len) { 1788 *new_insn = nop; 1789 goto end; 1790 } 1791 1792 insn = *new_insn; 1793 sec_for_each_insn_from(file, insn) { 1794 struct reloc *alt_reloc; 1795 1796 if (insn->offset >= special_alt->new_off + special_alt->new_len) 1797 break; 1798 1799 last_new_insn = insn; 1800 1801 insn->ignore = orig_insn->ignore_alts; 1802 insn->sym = orig_insn->sym; 1803 insn->alt_group = new_alt_group; 1804 1805 /* 1806 * Since alternative replacement code is copy/pasted by the 1807 * kernel after applying relocations, generally such code can't 1808 * have relative-address relocation references to outside the 1809 * .altinstr_replacement section, unless the arch's 1810 * alternatives code can adjust the relative offsets 1811 * accordingly. 1812 */ 1813 alt_reloc = insn_reloc(file, insn); 1814 if (alt_reloc && arch_pc_relative_reloc(alt_reloc) && 1815 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) { 1816 1817 WARN_FUNC("unsupported relocation in alternatives section", 1818 insn->sec, insn->offset); 1819 return -1; 1820 } 1821 1822 if (!is_static_jump(insn)) 1823 continue; 1824 1825 if (!insn->immediate) 1826 continue; 1827 1828 dest_off = arch_jump_destination(insn); 1829 if (dest_off == special_alt->new_off + special_alt->new_len) { 1830 insn->jump_dest = next_insn_same_sec(file, last_orig_insn); 1831 if (!insn->jump_dest) { 1832 WARN_FUNC("can't find alternative jump destination", 1833 insn->sec, insn->offset); 1834 return -1; 1835 } 1836 } 1837 } 1838 1839 if (!last_new_insn) { 1840 WARN_FUNC("can't find last new alternative instruction", 1841 special_alt->new_sec, special_alt->new_off); 1842 return -1; 1843 } 1844 1845 if (nop) 1846 list_add(&nop->list, &last_new_insn->list); 1847 end: 1848 new_alt_group->orig_group = orig_alt_group; 1849 new_alt_group->first_insn = *new_insn; 1850 new_alt_group->last_insn = nop ? : last_new_insn; 1851 new_alt_group->cfi = orig_alt_group->cfi; 1852 return 0; 1853 } 1854 1855 /* 1856 * A jump table entry can either convert a nop to a jump or a jump to a nop. 1857 * If the original instruction is a jump, make the alt entry an effective nop 1858 * by just skipping the original instruction. 1859 */ 1860 static int handle_jump_alt(struct objtool_file *file, 1861 struct special_alt *special_alt, 1862 struct instruction *orig_insn, 1863 struct instruction **new_insn) 1864 { 1865 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL && 1866 orig_insn->type != INSN_NOP) { 1867 1868 WARN_FUNC("unsupported instruction at jump label", 1869 orig_insn->sec, orig_insn->offset); 1870 return -1; 1871 } 1872 1873 if (opts.hack_jump_label && special_alt->key_addend & 2) { 1874 struct reloc *reloc = insn_reloc(file, orig_insn); 1875 1876 if (reloc) { 1877 reloc->type = R_NONE; 1878 elf_write_reloc(file->elf, reloc); 1879 } 1880 elf_write_insn(file->elf, orig_insn->sec, 1881 orig_insn->offset, orig_insn->len, 1882 arch_nop_insn(orig_insn->len)); 1883 orig_insn->type = INSN_NOP; 1884 } 1885 1886 if (orig_insn->type == INSN_NOP) { 1887 if (orig_insn->len == 2) 1888 file->jl_nop_short++; 1889 else 1890 file->jl_nop_long++; 1891 1892 return 0; 1893 } 1894 1895 if (orig_insn->len == 2) 1896 file->jl_short++; 1897 else 1898 file->jl_long++; 1899 1900 *new_insn = list_next_entry(orig_insn, list); 1901 return 0; 1902 } 1903 1904 /* 1905 * Read all the special sections which have alternate instructions which can be 1906 * patched in or redirected to at runtime. Each instruction having alternate 1907 * instruction(s) has them added to its insn->alts list, which will be 1908 * traversed in validate_branch(). 1909 */ 1910 static int add_special_section_alts(struct objtool_file *file) 1911 { 1912 struct list_head special_alts; 1913 struct instruction *orig_insn, *new_insn; 1914 struct special_alt *special_alt, *tmp; 1915 struct alternative *alt; 1916 int ret; 1917 1918 ret = special_get_alts(file->elf, &special_alts); 1919 if (ret) 1920 return ret; 1921 1922 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) { 1923 1924 orig_insn = find_insn(file, special_alt->orig_sec, 1925 special_alt->orig_off); 1926 if (!orig_insn) { 1927 WARN_FUNC("special: can't find orig instruction", 1928 special_alt->orig_sec, special_alt->orig_off); 1929 ret = -1; 1930 goto out; 1931 } 1932 1933 new_insn = NULL; 1934 if (!special_alt->group || special_alt->new_len) { 1935 new_insn = find_insn(file, special_alt->new_sec, 1936 special_alt->new_off); 1937 if (!new_insn) { 1938 WARN_FUNC("special: can't find new instruction", 1939 special_alt->new_sec, 1940 special_alt->new_off); 1941 ret = -1; 1942 goto out; 1943 } 1944 } 1945 1946 if (special_alt->group) { 1947 if (!special_alt->orig_len) { 1948 WARN_FUNC("empty alternative entry", 1949 orig_insn->sec, orig_insn->offset); 1950 continue; 1951 } 1952 1953 ret = handle_group_alt(file, special_alt, orig_insn, 1954 &new_insn); 1955 if (ret) 1956 goto out; 1957 } else if (special_alt->jump_or_nop) { 1958 ret = handle_jump_alt(file, special_alt, orig_insn, 1959 &new_insn); 1960 if (ret) 1961 goto out; 1962 } 1963 1964 alt = malloc(sizeof(*alt)); 1965 if (!alt) { 1966 WARN("malloc failed"); 1967 ret = -1; 1968 goto out; 1969 } 1970 1971 alt->insn = new_insn; 1972 alt->skip_orig = special_alt->skip_orig; 1973 orig_insn->ignore_alts |= special_alt->skip_alt; 1974 list_add_tail(&alt->list, &orig_insn->alts); 1975 1976 list_del(&special_alt->list); 1977 free(special_alt); 1978 } 1979 1980 if (opts.stats) { 1981 printf("jl\\\tNOP\tJMP\n"); 1982 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short); 1983 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long); 1984 } 1985 1986 out: 1987 return ret; 1988 } 1989 1990 static int add_jump_table(struct objtool_file *file, struct instruction *insn, 1991 struct reloc *table) 1992 { 1993 struct reloc *reloc = table; 1994 struct instruction *dest_insn; 1995 struct alternative *alt; 1996 struct symbol *pfunc = insn_func(insn)->pfunc; 1997 unsigned int prev_offset = 0; 1998 1999 /* 2000 * Each @reloc is a switch table relocation which points to the target 2001 * instruction. 2002 */ 2003 list_for_each_entry_from(reloc, &table->sec->reloc_list, list) { 2004 2005 /* Check for the end of the table: */ 2006 if (reloc != table && reloc->jump_table_start) 2007 break; 2008 2009 /* Make sure the table entries are consecutive: */ 2010 if (prev_offset && reloc->offset != prev_offset + 8) 2011 break; 2012 2013 /* Detect function pointers from contiguous objects: */ 2014 if (reloc->sym->sec == pfunc->sec && 2015 reloc->addend == pfunc->offset) 2016 break; 2017 2018 dest_insn = find_insn(file, reloc->sym->sec, reloc->addend); 2019 if (!dest_insn) 2020 break; 2021 2022 /* Make sure the destination is in the same function: */ 2023 if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc) 2024 break; 2025 2026 alt = malloc(sizeof(*alt)); 2027 if (!alt) { 2028 WARN("malloc failed"); 2029 return -1; 2030 } 2031 2032 alt->insn = dest_insn; 2033 list_add_tail(&alt->list, &insn->alts); 2034 prev_offset = reloc->offset; 2035 } 2036 2037 if (!prev_offset) { 2038 WARN_FUNC("can't find switch jump table", 2039 insn->sec, insn->offset); 2040 return -1; 2041 } 2042 2043 return 0; 2044 } 2045 2046 /* 2047 * find_jump_table() - Given a dynamic jump, find the switch jump table 2048 * associated with it. 2049 */ 2050 static struct reloc *find_jump_table(struct objtool_file *file, 2051 struct symbol *func, 2052 struct instruction *insn) 2053 { 2054 struct reloc *table_reloc; 2055 struct instruction *dest_insn, *orig_insn = insn; 2056 2057 /* 2058 * Backward search using the @first_jump_src links, these help avoid 2059 * much of the 'in between' code. Which avoids us getting confused by 2060 * it. 2061 */ 2062 for (; 2063 insn && insn_func(insn) && insn_func(insn)->pfunc == func; 2064 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) { 2065 2066 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC) 2067 break; 2068 2069 /* allow small jumps within the range */ 2070 if (insn->type == INSN_JUMP_UNCONDITIONAL && 2071 insn->jump_dest && 2072 (insn->jump_dest->offset <= insn->offset || 2073 insn->jump_dest->offset > orig_insn->offset)) 2074 break; 2075 2076 table_reloc = arch_find_switch_table(file, insn); 2077 if (!table_reloc) 2078 continue; 2079 dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend); 2080 if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func) 2081 continue; 2082 2083 return table_reloc; 2084 } 2085 2086 return NULL; 2087 } 2088 2089 /* 2090 * First pass: Mark the head of each jump table so that in the next pass, 2091 * we know when a given jump table ends and the next one starts. 2092 */ 2093 static void mark_func_jump_tables(struct objtool_file *file, 2094 struct symbol *func) 2095 { 2096 struct instruction *insn, *last = NULL; 2097 struct reloc *reloc; 2098 2099 func_for_each_insn(file, func, insn) { 2100 if (!last) 2101 last = insn; 2102 2103 /* 2104 * Store back-pointers for unconditional forward jumps such 2105 * that find_jump_table() can back-track using those and 2106 * avoid some potentially confusing code. 2107 */ 2108 if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest && 2109 insn->offset > last->offset && 2110 insn->jump_dest->offset > insn->offset && 2111 !insn->jump_dest->first_jump_src) { 2112 2113 insn->jump_dest->first_jump_src = insn; 2114 last = insn->jump_dest; 2115 } 2116 2117 if (insn->type != INSN_JUMP_DYNAMIC) 2118 continue; 2119 2120 reloc = find_jump_table(file, func, insn); 2121 if (reloc) { 2122 reloc->jump_table_start = true; 2123 insn->jump_table = reloc; 2124 } 2125 } 2126 } 2127 2128 static int add_func_jump_tables(struct objtool_file *file, 2129 struct symbol *func) 2130 { 2131 struct instruction *insn; 2132 int ret; 2133 2134 func_for_each_insn(file, func, insn) { 2135 if (!insn->jump_table) 2136 continue; 2137 2138 ret = add_jump_table(file, insn, insn->jump_table); 2139 if (ret) 2140 return ret; 2141 } 2142 2143 return 0; 2144 } 2145 2146 /* 2147 * For some switch statements, gcc generates a jump table in the .rodata 2148 * section which contains a list of addresses within the function to jump to. 2149 * This finds these jump tables and adds them to the insn->alts lists. 2150 */ 2151 static int add_jump_table_alts(struct objtool_file *file) 2152 { 2153 struct section *sec; 2154 struct symbol *func; 2155 int ret; 2156 2157 if (!file->rodata) 2158 return 0; 2159 2160 for_each_sec(file, sec) { 2161 list_for_each_entry(func, &sec->symbol_list, list) { 2162 if (func->type != STT_FUNC) 2163 continue; 2164 2165 mark_func_jump_tables(file, func); 2166 ret = add_func_jump_tables(file, func); 2167 if (ret) 2168 return ret; 2169 } 2170 } 2171 2172 return 0; 2173 } 2174 2175 static void set_func_state(struct cfi_state *state) 2176 { 2177 state->cfa = initial_func_cfi.cfa; 2178 memcpy(&state->regs, &initial_func_cfi.regs, 2179 CFI_NUM_REGS * sizeof(struct cfi_reg)); 2180 state->stack_size = initial_func_cfi.cfa.offset; 2181 } 2182 2183 static int read_unwind_hints(struct objtool_file *file) 2184 { 2185 struct cfi_state cfi = init_cfi; 2186 struct section *sec, *relocsec; 2187 struct unwind_hint *hint; 2188 struct instruction *insn; 2189 struct reloc *reloc; 2190 int i; 2191 2192 sec = find_section_by_name(file->elf, ".discard.unwind_hints"); 2193 if (!sec) 2194 return 0; 2195 2196 relocsec = sec->reloc; 2197 if (!relocsec) { 2198 WARN("missing .rela.discard.unwind_hints section"); 2199 return -1; 2200 } 2201 2202 if (sec->sh.sh_size % sizeof(struct unwind_hint)) { 2203 WARN("struct unwind_hint size mismatch"); 2204 return -1; 2205 } 2206 2207 file->hints = true; 2208 2209 for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) { 2210 hint = (struct unwind_hint *)sec->data->d_buf + i; 2211 2212 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint)); 2213 if (!reloc) { 2214 WARN("can't find reloc for unwind_hints[%d]", i); 2215 return -1; 2216 } 2217 2218 insn = find_insn(file, reloc->sym->sec, reloc->addend); 2219 if (!insn) { 2220 WARN("can't find insn for unwind_hints[%d]", i); 2221 return -1; 2222 } 2223 2224 insn->hint = true; 2225 2226 if (hint->type == UNWIND_HINT_TYPE_SAVE) { 2227 insn->hint = false; 2228 insn->save = true; 2229 continue; 2230 } 2231 2232 if (hint->type == UNWIND_HINT_TYPE_RESTORE) { 2233 insn->restore = true; 2234 continue; 2235 } 2236 2237 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) { 2238 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset); 2239 2240 if (sym && sym->bind == STB_GLOBAL) { 2241 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) { 2242 WARN_FUNC("UNWIND_HINT_IRET_REGS without ENDBR", 2243 insn->sec, insn->offset); 2244 } 2245 2246 insn->entry = 1; 2247 } 2248 } 2249 2250 if (hint->type == UNWIND_HINT_TYPE_ENTRY) { 2251 hint->type = UNWIND_HINT_TYPE_CALL; 2252 insn->entry = 1; 2253 } 2254 2255 if (hint->type == UNWIND_HINT_TYPE_FUNC) { 2256 insn->cfi = &func_cfi; 2257 continue; 2258 } 2259 2260 if (insn->cfi) 2261 cfi = *(insn->cfi); 2262 2263 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) { 2264 WARN_FUNC("unsupported unwind_hint sp base reg %d", 2265 insn->sec, insn->offset, hint->sp_reg); 2266 return -1; 2267 } 2268 2269 cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset); 2270 cfi.type = hint->type; 2271 cfi.end = hint->end; 2272 2273 insn->cfi = cfi_hash_find_or_add(&cfi); 2274 } 2275 2276 return 0; 2277 } 2278 2279 static int read_noendbr_hints(struct objtool_file *file) 2280 { 2281 struct section *sec; 2282 struct instruction *insn; 2283 struct reloc *reloc; 2284 2285 sec = find_section_by_name(file->elf, ".rela.discard.noendbr"); 2286 if (!sec) 2287 return 0; 2288 2289 list_for_each_entry(reloc, &sec->reloc_list, list) { 2290 insn = find_insn(file, reloc->sym->sec, reloc->sym->offset + reloc->addend); 2291 if (!insn) { 2292 WARN("bad .discard.noendbr entry"); 2293 return -1; 2294 } 2295 2296 insn->noendbr = 1; 2297 } 2298 2299 return 0; 2300 } 2301 2302 static int read_retpoline_hints(struct objtool_file *file) 2303 { 2304 struct section *sec; 2305 struct instruction *insn; 2306 struct reloc *reloc; 2307 2308 sec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe"); 2309 if (!sec) 2310 return 0; 2311 2312 list_for_each_entry(reloc, &sec->reloc_list, list) { 2313 if (reloc->sym->type != STT_SECTION) { 2314 WARN("unexpected relocation symbol type in %s", sec->name); 2315 return -1; 2316 } 2317 2318 insn = find_insn(file, reloc->sym->sec, reloc->addend); 2319 if (!insn) { 2320 WARN("bad .discard.retpoline_safe entry"); 2321 return -1; 2322 } 2323 2324 if (insn->type != INSN_JUMP_DYNAMIC && 2325 insn->type != INSN_CALL_DYNAMIC && 2326 insn->type != INSN_RETURN && 2327 insn->type != INSN_NOP) { 2328 WARN_FUNC("retpoline_safe hint not an indirect jump/call/ret/nop", 2329 insn->sec, insn->offset); 2330 return -1; 2331 } 2332 2333 insn->retpoline_safe = true; 2334 } 2335 2336 return 0; 2337 } 2338 2339 static int read_instr_hints(struct objtool_file *file) 2340 { 2341 struct section *sec; 2342 struct instruction *insn; 2343 struct reloc *reloc; 2344 2345 sec = find_section_by_name(file->elf, ".rela.discard.instr_end"); 2346 if (!sec) 2347 return 0; 2348 2349 list_for_each_entry(reloc, &sec->reloc_list, list) { 2350 if (reloc->sym->type != STT_SECTION) { 2351 WARN("unexpected relocation symbol type in %s", sec->name); 2352 return -1; 2353 } 2354 2355 insn = find_insn(file, reloc->sym->sec, reloc->addend); 2356 if (!insn) { 2357 WARN("bad .discard.instr_end entry"); 2358 return -1; 2359 } 2360 2361 insn->instr--; 2362 } 2363 2364 sec = find_section_by_name(file->elf, ".rela.discard.instr_begin"); 2365 if (!sec) 2366 return 0; 2367 2368 list_for_each_entry(reloc, &sec->reloc_list, list) { 2369 if (reloc->sym->type != STT_SECTION) { 2370 WARN("unexpected relocation symbol type in %s", sec->name); 2371 return -1; 2372 } 2373 2374 insn = find_insn(file, reloc->sym->sec, reloc->addend); 2375 if (!insn) { 2376 WARN("bad .discard.instr_begin entry"); 2377 return -1; 2378 } 2379 2380 insn->instr++; 2381 } 2382 2383 return 0; 2384 } 2385 2386 static int read_intra_function_calls(struct objtool_file *file) 2387 { 2388 struct instruction *insn; 2389 struct section *sec; 2390 struct reloc *reloc; 2391 2392 sec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls"); 2393 if (!sec) 2394 return 0; 2395 2396 list_for_each_entry(reloc, &sec->reloc_list, list) { 2397 unsigned long dest_off; 2398 2399 if (reloc->sym->type != STT_SECTION) { 2400 WARN("unexpected relocation symbol type in %s", 2401 sec->name); 2402 return -1; 2403 } 2404 2405 insn = find_insn(file, reloc->sym->sec, reloc->addend); 2406 if (!insn) { 2407 WARN("bad .discard.intra_function_call entry"); 2408 return -1; 2409 } 2410 2411 if (insn->type != INSN_CALL) { 2412 WARN_FUNC("intra_function_call not a direct call", 2413 insn->sec, insn->offset); 2414 return -1; 2415 } 2416 2417 /* 2418 * Treat intra-function CALLs as JMPs, but with a stack_op. 2419 * See add_call_destinations(), which strips stack_ops from 2420 * normal CALLs. 2421 */ 2422 insn->type = INSN_JUMP_UNCONDITIONAL; 2423 2424 dest_off = arch_jump_destination(insn); 2425 insn->jump_dest = find_insn(file, insn->sec, dest_off); 2426 if (!insn->jump_dest) { 2427 WARN_FUNC("can't find call dest at %s+0x%lx", 2428 insn->sec, insn->offset, 2429 insn->sec->name, dest_off); 2430 return -1; 2431 } 2432 } 2433 2434 return 0; 2435 } 2436 2437 /* 2438 * Return true if name matches an instrumentation function, where calls to that 2439 * function from noinstr code can safely be removed, but compilers won't do so. 2440 */ 2441 static bool is_profiling_func(const char *name) 2442 { 2443 /* 2444 * Many compilers cannot disable KCOV with a function attribute. 2445 */ 2446 if (!strncmp(name, "__sanitizer_cov_", 16)) 2447 return true; 2448 2449 /* 2450 * Some compilers currently do not remove __tsan_func_entry/exit nor 2451 * __tsan_atomic_signal_fence (used for barrier instrumentation) with 2452 * the __no_sanitize_thread attribute, remove them. Once the kernel's 2453 * minimum Clang version is 14.0, this can be removed. 2454 */ 2455 if (!strncmp(name, "__tsan_func_", 12) || 2456 !strcmp(name, "__tsan_atomic_signal_fence")) 2457 return true; 2458 2459 return false; 2460 } 2461 2462 static int classify_symbols(struct objtool_file *file) 2463 { 2464 struct section *sec; 2465 struct symbol *func; 2466 2467 for_each_sec(file, sec) { 2468 list_for_each_entry(func, &sec->symbol_list, list) { 2469 if (func->bind != STB_GLOBAL) 2470 continue; 2471 2472 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR, 2473 strlen(STATIC_CALL_TRAMP_PREFIX_STR))) 2474 func->static_call_tramp = true; 2475 2476 if (arch_is_retpoline(func)) 2477 func->retpoline_thunk = true; 2478 2479 if (arch_is_rethunk(func)) 2480 func->return_thunk = true; 2481 2482 if (arch_ftrace_match(func->name)) 2483 func->fentry = true; 2484 2485 if (is_profiling_func(func->name)) 2486 func->profiling_func = true; 2487 } 2488 } 2489 2490 return 0; 2491 } 2492 2493 static void mark_rodata(struct objtool_file *file) 2494 { 2495 struct section *sec; 2496 bool found = false; 2497 2498 /* 2499 * Search for the following rodata sections, each of which can 2500 * potentially contain jump tables: 2501 * 2502 * - .rodata: can contain GCC switch tables 2503 * - .rodata.<func>: same, if -fdata-sections is being used 2504 * - .rodata..c_jump_table: contains C annotated jump tables 2505 * 2506 * .rodata.str1.* sections are ignored; they don't contain jump tables. 2507 */ 2508 for_each_sec(file, sec) { 2509 if (!strncmp(sec->name, ".rodata", 7) && 2510 !strstr(sec->name, ".str1.")) { 2511 sec->rodata = true; 2512 found = true; 2513 } 2514 } 2515 2516 file->rodata = found; 2517 } 2518 2519 static int decode_sections(struct objtool_file *file) 2520 { 2521 int ret; 2522 2523 mark_rodata(file); 2524 2525 ret = init_pv_ops(file); 2526 if (ret) 2527 return ret; 2528 2529 /* 2530 * Must be before add_{jump_call}_destination. 2531 */ 2532 ret = classify_symbols(file); 2533 if (ret) 2534 return ret; 2535 2536 ret = decode_instructions(file); 2537 if (ret) 2538 return ret; 2539 2540 add_ignores(file); 2541 add_uaccess_safe(file); 2542 2543 ret = add_ignore_alternatives(file); 2544 if (ret) 2545 return ret; 2546 2547 /* 2548 * Must be before read_unwind_hints() since that needs insn->noendbr. 2549 */ 2550 ret = read_noendbr_hints(file); 2551 if (ret) 2552 return ret; 2553 2554 /* 2555 * Must be before add_jump_destinations(), which depends on 'func' 2556 * being set for alternatives, to enable proper sibling call detection. 2557 */ 2558 if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) { 2559 ret = add_special_section_alts(file); 2560 if (ret) 2561 return ret; 2562 } 2563 2564 ret = add_jump_destinations(file); 2565 if (ret) 2566 return ret; 2567 2568 /* 2569 * Must be before add_call_destination(); it changes INSN_CALL to 2570 * INSN_JUMP. 2571 */ 2572 ret = read_intra_function_calls(file); 2573 if (ret) 2574 return ret; 2575 2576 ret = add_call_destinations(file); 2577 if (ret) 2578 return ret; 2579 2580 /* 2581 * Must be after add_call_destinations() such that it can override 2582 * dead_end_function() marks. 2583 */ 2584 ret = add_dead_ends(file); 2585 if (ret) 2586 return ret; 2587 2588 ret = add_jump_table_alts(file); 2589 if (ret) 2590 return ret; 2591 2592 ret = read_unwind_hints(file); 2593 if (ret) 2594 return ret; 2595 2596 ret = read_retpoline_hints(file); 2597 if (ret) 2598 return ret; 2599 2600 ret = read_instr_hints(file); 2601 if (ret) 2602 return ret; 2603 2604 return 0; 2605 } 2606 2607 static bool is_fentry_call(struct instruction *insn) 2608 { 2609 if (insn->type == INSN_CALL && 2610 insn->call_dest && 2611 insn->call_dest->fentry) 2612 return true; 2613 2614 return false; 2615 } 2616 2617 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state) 2618 { 2619 struct cfi_state *cfi = &state->cfi; 2620 int i; 2621 2622 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap) 2623 return true; 2624 2625 if (cfi->cfa.offset != initial_func_cfi.cfa.offset) 2626 return true; 2627 2628 if (cfi->stack_size != initial_func_cfi.cfa.offset) 2629 return true; 2630 2631 for (i = 0; i < CFI_NUM_REGS; i++) { 2632 if (cfi->regs[i].base != initial_func_cfi.regs[i].base || 2633 cfi->regs[i].offset != initial_func_cfi.regs[i].offset) 2634 return true; 2635 } 2636 2637 return false; 2638 } 2639 2640 static bool check_reg_frame_pos(const struct cfi_reg *reg, 2641 int expected_offset) 2642 { 2643 return reg->base == CFI_CFA && 2644 reg->offset == expected_offset; 2645 } 2646 2647 static bool has_valid_stack_frame(struct insn_state *state) 2648 { 2649 struct cfi_state *cfi = &state->cfi; 2650 2651 if (cfi->cfa.base == CFI_BP && 2652 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) && 2653 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8)) 2654 return true; 2655 2656 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP) 2657 return true; 2658 2659 return false; 2660 } 2661 2662 static int update_cfi_state_regs(struct instruction *insn, 2663 struct cfi_state *cfi, 2664 struct stack_op *op) 2665 { 2666 struct cfi_reg *cfa = &cfi->cfa; 2667 2668 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT) 2669 return 0; 2670 2671 /* push */ 2672 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF) 2673 cfa->offset += 8; 2674 2675 /* pop */ 2676 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF) 2677 cfa->offset -= 8; 2678 2679 /* add immediate to sp */ 2680 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD && 2681 op->dest.reg == CFI_SP && op->src.reg == CFI_SP) 2682 cfa->offset -= op->src.offset; 2683 2684 return 0; 2685 } 2686 2687 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset) 2688 { 2689 if (arch_callee_saved_reg(reg) && 2690 cfi->regs[reg].base == CFI_UNDEFINED) { 2691 cfi->regs[reg].base = base; 2692 cfi->regs[reg].offset = offset; 2693 } 2694 } 2695 2696 static void restore_reg(struct cfi_state *cfi, unsigned char reg) 2697 { 2698 cfi->regs[reg].base = initial_func_cfi.regs[reg].base; 2699 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset; 2700 } 2701 2702 /* 2703 * A note about DRAP stack alignment: 2704 * 2705 * GCC has the concept of a DRAP register, which is used to help keep track of 2706 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP 2707 * register. The typical DRAP pattern is: 2708 * 2709 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10 2710 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp 2711 * 41 ff 72 f8 pushq -0x8(%r10) 2712 * 55 push %rbp 2713 * 48 89 e5 mov %rsp,%rbp 2714 * (more pushes) 2715 * 41 52 push %r10 2716 * ... 2717 * 41 5a pop %r10 2718 * (more pops) 2719 * 5d pop %rbp 2720 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2721 * c3 retq 2722 * 2723 * There are some variations in the epilogues, like: 2724 * 2725 * 5b pop %rbx 2726 * 41 5a pop %r10 2727 * 41 5c pop %r12 2728 * 41 5d pop %r13 2729 * 41 5e pop %r14 2730 * c9 leaveq 2731 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2732 * c3 retq 2733 * 2734 * and: 2735 * 2736 * 4c 8b 55 e8 mov -0x18(%rbp),%r10 2737 * 48 8b 5d e0 mov -0x20(%rbp),%rbx 2738 * 4c 8b 65 f0 mov -0x10(%rbp),%r12 2739 * 4c 8b 6d f8 mov -0x8(%rbp),%r13 2740 * c9 leaveq 2741 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2742 * c3 retq 2743 * 2744 * Sometimes r13 is used as the DRAP register, in which case it's saved and 2745 * restored beforehand: 2746 * 2747 * 41 55 push %r13 2748 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13 2749 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp 2750 * ... 2751 * 49 8d 65 f0 lea -0x10(%r13),%rsp 2752 * 41 5d pop %r13 2753 * c3 retq 2754 */ 2755 static int update_cfi_state(struct instruction *insn, 2756 struct instruction *next_insn, 2757 struct cfi_state *cfi, struct stack_op *op) 2758 { 2759 struct cfi_reg *cfa = &cfi->cfa; 2760 struct cfi_reg *regs = cfi->regs; 2761 2762 /* stack operations don't make sense with an undefined CFA */ 2763 if (cfa->base == CFI_UNDEFINED) { 2764 if (insn_func(insn)) { 2765 WARN_FUNC("undefined stack state", insn->sec, insn->offset); 2766 return -1; 2767 } 2768 return 0; 2769 } 2770 2771 if (cfi->type == UNWIND_HINT_TYPE_REGS || 2772 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL) 2773 return update_cfi_state_regs(insn, cfi, op); 2774 2775 switch (op->dest.type) { 2776 2777 case OP_DEST_REG: 2778 switch (op->src.type) { 2779 2780 case OP_SRC_REG: 2781 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP && 2782 cfa->base == CFI_SP && 2783 check_reg_frame_pos(®s[CFI_BP], -cfa->offset)) { 2784 2785 /* mov %rsp, %rbp */ 2786 cfa->base = op->dest.reg; 2787 cfi->bp_scratch = false; 2788 } 2789 2790 else if (op->src.reg == CFI_SP && 2791 op->dest.reg == CFI_BP && cfi->drap) { 2792 2793 /* drap: mov %rsp, %rbp */ 2794 regs[CFI_BP].base = CFI_BP; 2795 regs[CFI_BP].offset = -cfi->stack_size; 2796 cfi->bp_scratch = false; 2797 } 2798 2799 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 2800 2801 /* 2802 * mov %rsp, %reg 2803 * 2804 * This is needed for the rare case where GCC 2805 * does: 2806 * 2807 * mov %rsp, %rax 2808 * ... 2809 * mov %rax, %rsp 2810 */ 2811 cfi->vals[op->dest.reg].base = CFI_CFA; 2812 cfi->vals[op->dest.reg].offset = -cfi->stack_size; 2813 } 2814 2815 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP && 2816 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) { 2817 2818 /* 2819 * mov %rbp, %rsp 2820 * 2821 * Restore the original stack pointer (Clang). 2822 */ 2823 cfi->stack_size = -cfi->regs[CFI_BP].offset; 2824 } 2825 2826 else if (op->dest.reg == cfa->base) { 2827 2828 /* mov %reg, %rsp */ 2829 if (cfa->base == CFI_SP && 2830 cfi->vals[op->src.reg].base == CFI_CFA) { 2831 2832 /* 2833 * This is needed for the rare case 2834 * where GCC does something dumb like: 2835 * 2836 * lea 0x8(%rsp), %rcx 2837 * ... 2838 * mov %rcx, %rsp 2839 */ 2840 cfa->offset = -cfi->vals[op->src.reg].offset; 2841 cfi->stack_size = cfa->offset; 2842 2843 } else if (cfa->base == CFI_SP && 2844 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 2845 cfi->vals[op->src.reg].offset == cfa->offset) { 2846 2847 /* 2848 * Stack swizzle: 2849 * 2850 * 1: mov %rsp, (%[tos]) 2851 * 2: mov %[tos], %rsp 2852 * ... 2853 * 3: pop %rsp 2854 * 2855 * Where: 2856 * 2857 * 1 - places a pointer to the previous 2858 * stack at the Top-of-Stack of the 2859 * new stack. 2860 * 2861 * 2 - switches to the new stack. 2862 * 2863 * 3 - pops the Top-of-Stack to restore 2864 * the original stack. 2865 * 2866 * Note: we set base to SP_INDIRECT 2867 * here and preserve offset. Therefore 2868 * when the unwinder reaches ToS it 2869 * will dereference SP and then add the 2870 * offset to find the next frame, IOW: 2871 * (%rsp) + offset. 2872 */ 2873 cfa->base = CFI_SP_INDIRECT; 2874 2875 } else { 2876 cfa->base = CFI_UNDEFINED; 2877 cfa->offset = 0; 2878 } 2879 } 2880 2881 else if (op->dest.reg == CFI_SP && 2882 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 2883 cfi->vals[op->src.reg].offset == cfa->offset) { 2884 2885 /* 2886 * The same stack swizzle case 2) as above. But 2887 * because we can't change cfa->base, case 3) 2888 * will become a regular POP. Pretend we're a 2889 * PUSH so things don't go unbalanced. 2890 */ 2891 cfi->stack_size += 8; 2892 } 2893 2894 2895 break; 2896 2897 case OP_SRC_ADD: 2898 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) { 2899 2900 /* add imm, %rsp */ 2901 cfi->stack_size -= op->src.offset; 2902 if (cfa->base == CFI_SP) 2903 cfa->offset -= op->src.offset; 2904 break; 2905 } 2906 2907 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) { 2908 2909 /* lea disp(%rbp), %rsp */ 2910 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset); 2911 break; 2912 } 2913 2914 if (!cfi->drap && op->src.reg == CFI_SP && 2915 op->dest.reg == CFI_BP && cfa->base == CFI_SP && 2916 check_reg_frame_pos(®s[CFI_BP], -cfa->offset + op->src.offset)) { 2917 2918 /* lea disp(%rsp), %rbp */ 2919 cfa->base = CFI_BP; 2920 cfa->offset -= op->src.offset; 2921 cfi->bp_scratch = false; 2922 break; 2923 } 2924 2925 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 2926 2927 /* drap: lea disp(%rsp), %drap */ 2928 cfi->drap_reg = op->dest.reg; 2929 2930 /* 2931 * lea disp(%rsp), %reg 2932 * 2933 * This is needed for the rare case where GCC 2934 * does something dumb like: 2935 * 2936 * lea 0x8(%rsp), %rcx 2937 * ... 2938 * mov %rcx, %rsp 2939 */ 2940 cfi->vals[op->dest.reg].base = CFI_CFA; 2941 cfi->vals[op->dest.reg].offset = \ 2942 -cfi->stack_size + op->src.offset; 2943 2944 break; 2945 } 2946 2947 if (cfi->drap && op->dest.reg == CFI_SP && 2948 op->src.reg == cfi->drap_reg) { 2949 2950 /* drap: lea disp(%drap), %rsp */ 2951 cfa->base = CFI_SP; 2952 cfa->offset = cfi->stack_size = -op->src.offset; 2953 cfi->drap_reg = CFI_UNDEFINED; 2954 cfi->drap = false; 2955 break; 2956 } 2957 2958 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) { 2959 WARN_FUNC("unsupported stack register modification", 2960 insn->sec, insn->offset); 2961 return -1; 2962 } 2963 2964 break; 2965 2966 case OP_SRC_AND: 2967 if (op->dest.reg != CFI_SP || 2968 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) || 2969 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) { 2970 WARN_FUNC("unsupported stack pointer realignment", 2971 insn->sec, insn->offset); 2972 return -1; 2973 } 2974 2975 if (cfi->drap_reg != CFI_UNDEFINED) { 2976 /* drap: and imm, %rsp */ 2977 cfa->base = cfi->drap_reg; 2978 cfa->offset = cfi->stack_size = 0; 2979 cfi->drap = true; 2980 } 2981 2982 /* 2983 * Older versions of GCC (4.8ish) realign the stack 2984 * without DRAP, with a frame pointer. 2985 */ 2986 2987 break; 2988 2989 case OP_SRC_POP: 2990 case OP_SRC_POPF: 2991 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) { 2992 2993 /* pop %rsp; # restore from a stack swizzle */ 2994 cfa->base = CFI_SP; 2995 break; 2996 } 2997 2998 if (!cfi->drap && op->dest.reg == cfa->base) { 2999 3000 /* pop %rbp */ 3001 cfa->base = CFI_SP; 3002 } 3003 3004 if (cfi->drap && cfa->base == CFI_BP_INDIRECT && 3005 op->dest.reg == cfi->drap_reg && 3006 cfi->drap_offset == -cfi->stack_size) { 3007 3008 /* drap: pop %drap */ 3009 cfa->base = cfi->drap_reg; 3010 cfa->offset = 0; 3011 cfi->drap_offset = -1; 3012 3013 } else if (cfi->stack_size == -regs[op->dest.reg].offset) { 3014 3015 /* pop %reg */ 3016 restore_reg(cfi, op->dest.reg); 3017 } 3018 3019 cfi->stack_size -= 8; 3020 if (cfa->base == CFI_SP) 3021 cfa->offset -= 8; 3022 3023 break; 3024 3025 case OP_SRC_REG_INDIRECT: 3026 if (!cfi->drap && op->dest.reg == cfa->base && 3027 op->dest.reg == CFI_BP) { 3028 3029 /* mov disp(%rsp), %rbp */ 3030 cfa->base = CFI_SP; 3031 cfa->offset = cfi->stack_size; 3032 } 3033 3034 if (cfi->drap && op->src.reg == CFI_BP && 3035 op->src.offset == cfi->drap_offset) { 3036 3037 /* drap: mov disp(%rbp), %drap */ 3038 cfa->base = cfi->drap_reg; 3039 cfa->offset = 0; 3040 cfi->drap_offset = -1; 3041 } 3042 3043 if (cfi->drap && op->src.reg == CFI_BP && 3044 op->src.offset == regs[op->dest.reg].offset) { 3045 3046 /* drap: mov disp(%rbp), %reg */ 3047 restore_reg(cfi, op->dest.reg); 3048 3049 } else if (op->src.reg == cfa->base && 3050 op->src.offset == regs[op->dest.reg].offset + cfa->offset) { 3051 3052 /* mov disp(%rbp), %reg */ 3053 /* mov disp(%rsp), %reg */ 3054 restore_reg(cfi, op->dest.reg); 3055 3056 } else if (op->src.reg == CFI_SP && 3057 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) { 3058 3059 /* mov disp(%rsp), %reg */ 3060 restore_reg(cfi, op->dest.reg); 3061 } 3062 3063 break; 3064 3065 default: 3066 WARN_FUNC("unknown stack-related instruction", 3067 insn->sec, insn->offset); 3068 return -1; 3069 } 3070 3071 break; 3072 3073 case OP_DEST_PUSH: 3074 case OP_DEST_PUSHF: 3075 cfi->stack_size += 8; 3076 if (cfa->base == CFI_SP) 3077 cfa->offset += 8; 3078 3079 if (op->src.type != OP_SRC_REG) 3080 break; 3081 3082 if (cfi->drap) { 3083 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 3084 3085 /* drap: push %drap */ 3086 cfa->base = CFI_BP_INDIRECT; 3087 cfa->offset = -cfi->stack_size; 3088 3089 /* save drap so we know when to restore it */ 3090 cfi->drap_offset = -cfi->stack_size; 3091 3092 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) { 3093 3094 /* drap: push %rbp */ 3095 cfi->stack_size = 0; 3096 3097 } else { 3098 3099 /* drap: push %reg */ 3100 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size); 3101 } 3102 3103 } else { 3104 3105 /* push %reg */ 3106 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size); 3107 } 3108 3109 /* detect when asm code uses rbp as a scratch register */ 3110 if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP && 3111 cfa->base != CFI_BP) 3112 cfi->bp_scratch = true; 3113 break; 3114 3115 case OP_DEST_REG_INDIRECT: 3116 3117 if (cfi->drap) { 3118 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 3119 3120 /* drap: mov %drap, disp(%rbp) */ 3121 cfa->base = CFI_BP_INDIRECT; 3122 cfa->offset = op->dest.offset; 3123 3124 /* save drap offset so we know when to restore it */ 3125 cfi->drap_offset = op->dest.offset; 3126 } else { 3127 3128 /* drap: mov reg, disp(%rbp) */ 3129 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset); 3130 } 3131 3132 } else if (op->dest.reg == cfa->base) { 3133 3134 /* mov reg, disp(%rbp) */ 3135 /* mov reg, disp(%rsp) */ 3136 save_reg(cfi, op->src.reg, CFI_CFA, 3137 op->dest.offset - cfi->cfa.offset); 3138 3139 } else if (op->dest.reg == CFI_SP) { 3140 3141 /* mov reg, disp(%rsp) */ 3142 save_reg(cfi, op->src.reg, CFI_CFA, 3143 op->dest.offset - cfi->stack_size); 3144 3145 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) { 3146 3147 /* mov %rsp, (%reg); # setup a stack swizzle. */ 3148 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT; 3149 cfi->vals[op->dest.reg].offset = cfa->offset; 3150 } 3151 3152 break; 3153 3154 case OP_DEST_MEM: 3155 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) { 3156 WARN_FUNC("unknown stack-related memory operation", 3157 insn->sec, insn->offset); 3158 return -1; 3159 } 3160 3161 /* pop mem */ 3162 cfi->stack_size -= 8; 3163 if (cfa->base == CFI_SP) 3164 cfa->offset -= 8; 3165 3166 break; 3167 3168 default: 3169 WARN_FUNC("unknown stack-related instruction", 3170 insn->sec, insn->offset); 3171 return -1; 3172 } 3173 3174 return 0; 3175 } 3176 3177 /* 3178 * The stack layouts of alternatives instructions can sometimes diverge when 3179 * they have stack modifications. That's fine as long as the potential stack 3180 * layouts don't conflict at any given potential instruction boundary. 3181 * 3182 * Flatten the CFIs of the different alternative code streams (both original 3183 * and replacement) into a single shared CFI array which can be used to detect 3184 * conflicts and nicely feed a linear array of ORC entries to the unwinder. 3185 */ 3186 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn) 3187 { 3188 struct cfi_state **alt_cfi; 3189 int group_off; 3190 3191 if (!insn->alt_group) 3192 return 0; 3193 3194 if (!insn->cfi) { 3195 WARN("CFI missing"); 3196 return -1; 3197 } 3198 3199 alt_cfi = insn->alt_group->cfi; 3200 group_off = insn->offset - insn->alt_group->first_insn->offset; 3201 3202 if (!alt_cfi[group_off]) { 3203 alt_cfi[group_off] = insn->cfi; 3204 } else { 3205 if (cficmp(alt_cfi[group_off], insn->cfi)) { 3206 WARN_FUNC("stack layout conflict in alternatives", 3207 insn->sec, insn->offset); 3208 return -1; 3209 } 3210 } 3211 3212 return 0; 3213 } 3214 3215 static int handle_insn_ops(struct instruction *insn, 3216 struct instruction *next_insn, 3217 struct insn_state *state) 3218 { 3219 struct stack_op *op; 3220 3221 list_for_each_entry(op, &insn->stack_ops, list) { 3222 3223 if (update_cfi_state(insn, next_insn, &state->cfi, op)) 3224 return 1; 3225 3226 if (!insn->alt_group) 3227 continue; 3228 3229 if (op->dest.type == OP_DEST_PUSHF) { 3230 if (!state->uaccess_stack) { 3231 state->uaccess_stack = 1; 3232 } else if (state->uaccess_stack >> 31) { 3233 WARN_FUNC("PUSHF stack exhausted", 3234 insn->sec, insn->offset); 3235 return 1; 3236 } 3237 state->uaccess_stack <<= 1; 3238 state->uaccess_stack |= state->uaccess; 3239 } 3240 3241 if (op->src.type == OP_SRC_POPF) { 3242 if (state->uaccess_stack) { 3243 state->uaccess = state->uaccess_stack & 1; 3244 state->uaccess_stack >>= 1; 3245 if (state->uaccess_stack == 1) 3246 state->uaccess_stack = 0; 3247 } 3248 } 3249 } 3250 3251 return 0; 3252 } 3253 3254 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2) 3255 { 3256 struct cfi_state *cfi1 = insn->cfi; 3257 int i; 3258 3259 if (!cfi1) { 3260 WARN("CFI missing"); 3261 return false; 3262 } 3263 3264 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) { 3265 3266 WARN_FUNC("stack state mismatch: cfa1=%d%+d cfa2=%d%+d", 3267 insn->sec, insn->offset, 3268 cfi1->cfa.base, cfi1->cfa.offset, 3269 cfi2->cfa.base, cfi2->cfa.offset); 3270 3271 } else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) { 3272 for (i = 0; i < CFI_NUM_REGS; i++) { 3273 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i], 3274 sizeof(struct cfi_reg))) 3275 continue; 3276 3277 WARN_FUNC("stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d", 3278 insn->sec, insn->offset, 3279 i, cfi1->regs[i].base, cfi1->regs[i].offset, 3280 i, cfi2->regs[i].base, cfi2->regs[i].offset); 3281 break; 3282 } 3283 3284 } else if (cfi1->type != cfi2->type) { 3285 3286 WARN_FUNC("stack state mismatch: type1=%d type2=%d", 3287 insn->sec, insn->offset, cfi1->type, cfi2->type); 3288 3289 } else if (cfi1->drap != cfi2->drap || 3290 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) || 3291 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) { 3292 3293 WARN_FUNC("stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)", 3294 insn->sec, insn->offset, 3295 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset, 3296 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset); 3297 3298 } else 3299 return true; 3300 3301 return false; 3302 } 3303 3304 static inline bool func_uaccess_safe(struct symbol *func) 3305 { 3306 if (func) 3307 return func->uaccess_safe; 3308 3309 return false; 3310 } 3311 3312 static inline const char *call_dest_name(struct instruction *insn) 3313 { 3314 static char pvname[19]; 3315 struct reloc *rel; 3316 int idx; 3317 3318 if (insn->call_dest) 3319 return insn->call_dest->name; 3320 3321 rel = insn_reloc(NULL, insn); 3322 if (rel && !strcmp(rel->sym->name, "pv_ops")) { 3323 idx = (rel->addend / sizeof(void *)); 3324 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx); 3325 return pvname; 3326 } 3327 3328 return "{dynamic}"; 3329 } 3330 3331 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn) 3332 { 3333 struct symbol *target; 3334 struct reloc *rel; 3335 int idx; 3336 3337 rel = insn_reloc(file, insn); 3338 if (!rel || strcmp(rel->sym->name, "pv_ops")) 3339 return false; 3340 3341 idx = (arch_dest_reloc_offset(rel->addend) / sizeof(void *)); 3342 3343 if (file->pv_ops[idx].clean) 3344 return true; 3345 3346 file->pv_ops[idx].clean = true; 3347 3348 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) { 3349 if (!target->sec->noinstr) { 3350 WARN("pv_ops[%d]: %s", idx, target->name); 3351 file->pv_ops[idx].clean = false; 3352 } 3353 } 3354 3355 return file->pv_ops[idx].clean; 3356 } 3357 3358 static inline bool noinstr_call_dest(struct objtool_file *file, 3359 struct instruction *insn, 3360 struct symbol *func) 3361 { 3362 /* 3363 * We can't deal with indirect function calls at present; 3364 * assume they're instrumented. 3365 */ 3366 if (!func) { 3367 if (file->pv_ops) 3368 return pv_call_dest(file, insn); 3369 3370 return false; 3371 } 3372 3373 /* 3374 * If the symbol is from a noinstr section; we good. 3375 */ 3376 if (func->sec->noinstr) 3377 return true; 3378 3379 /* 3380 * If the symbol is a static_call trampoline, we can't tell. 3381 */ 3382 if (func->static_call_tramp) 3383 return true; 3384 3385 /* 3386 * The __ubsan_handle_*() calls are like WARN(), they only happen when 3387 * something 'BAD' happened. At the risk of taking the machine down, 3388 * let them proceed to get the message out. 3389 */ 3390 if (!strncmp(func->name, "__ubsan_handle_", 15)) 3391 return true; 3392 3393 return false; 3394 } 3395 3396 static int validate_call(struct objtool_file *file, 3397 struct instruction *insn, 3398 struct insn_state *state) 3399 { 3400 if (state->noinstr && state->instr <= 0 && 3401 !noinstr_call_dest(file, insn, insn->call_dest)) { 3402 WARN_FUNC("call to %s() leaves .noinstr.text section", 3403 insn->sec, insn->offset, call_dest_name(insn)); 3404 return 1; 3405 } 3406 3407 if (state->uaccess && !func_uaccess_safe(insn->call_dest)) { 3408 WARN_FUNC("call to %s() with UACCESS enabled", 3409 insn->sec, insn->offset, call_dest_name(insn)); 3410 return 1; 3411 } 3412 3413 if (state->df) { 3414 WARN_FUNC("call to %s() with DF set", 3415 insn->sec, insn->offset, call_dest_name(insn)); 3416 return 1; 3417 } 3418 3419 return 0; 3420 } 3421 3422 static int validate_sibling_call(struct objtool_file *file, 3423 struct instruction *insn, 3424 struct insn_state *state) 3425 { 3426 if (insn_func(insn) && has_modified_stack_frame(insn, state)) { 3427 WARN_FUNC("sibling call from callable instruction with modified stack frame", 3428 insn->sec, insn->offset); 3429 return 1; 3430 } 3431 3432 return validate_call(file, insn, state); 3433 } 3434 3435 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state) 3436 { 3437 if (state->noinstr && state->instr > 0) { 3438 WARN_FUNC("return with instrumentation enabled", 3439 insn->sec, insn->offset); 3440 return 1; 3441 } 3442 3443 if (state->uaccess && !func_uaccess_safe(func)) { 3444 WARN_FUNC("return with UACCESS enabled", 3445 insn->sec, insn->offset); 3446 return 1; 3447 } 3448 3449 if (!state->uaccess && func_uaccess_safe(func)) { 3450 WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function", 3451 insn->sec, insn->offset); 3452 return 1; 3453 } 3454 3455 if (state->df) { 3456 WARN_FUNC("return with DF set", 3457 insn->sec, insn->offset); 3458 return 1; 3459 } 3460 3461 if (func && has_modified_stack_frame(insn, state)) { 3462 WARN_FUNC("return with modified stack frame", 3463 insn->sec, insn->offset); 3464 return 1; 3465 } 3466 3467 if (state->cfi.bp_scratch) { 3468 WARN_FUNC("BP used as a scratch register", 3469 insn->sec, insn->offset); 3470 return 1; 3471 } 3472 3473 return 0; 3474 } 3475 3476 static struct instruction *next_insn_to_validate(struct objtool_file *file, 3477 struct instruction *insn) 3478 { 3479 struct alt_group *alt_group = insn->alt_group; 3480 3481 /* 3482 * Simulate the fact that alternatives are patched in-place. When the 3483 * end of a replacement alt_group is reached, redirect objtool flow to 3484 * the end of the original alt_group. 3485 */ 3486 if (alt_group && insn == alt_group->last_insn && alt_group->orig_group) 3487 return next_insn_same_sec(file, alt_group->orig_group->last_insn); 3488 3489 return next_insn_same_sec(file, insn); 3490 } 3491 3492 /* 3493 * Follow the branch starting at the given instruction, and recursively follow 3494 * any other branches (jumps). Meanwhile, track the frame pointer state at 3495 * each instruction and validate all the rules described in 3496 * tools/objtool/Documentation/objtool.txt. 3497 */ 3498 static int validate_branch(struct objtool_file *file, struct symbol *func, 3499 struct instruction *insn, struct insn_state state) 3500 { 3501 struct alternative *alt; 3502 struct instruction *next_insn, *prev_insn = NULL; 3503 struct section *sec; 3504 u8 visited; 3505 int ret; 3506 3507 sec = insn->sec; 3508 3509 while (1) { 3510 next_insn = next_insn_to_validate(file, insn); 3511 3512 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) { 3513 /* Ignore KCFI type preambles, which always fall through */ 3514 if (!strncmp(func->name, "__cfi_", 6) || 3515 !strncmp(func->name, "__pfx_", 6)) 3516 return 0; 3517 3518 WARN("%s() falls through to next function %s()", 3519 func->name, insn_func(insn)->name); 3520 return 1; 3521 } 3522 3523 if (func && insn->ignore) { 3524 WARN_FUNC("BUG: why am I validating an ignored function?", 3525 sec, insn->offset); 3526 return 1; 3527 } 3528 3529 visited = VISITED_BRANCH << state.uaccess; 3530 if (insn->visited & VISITED_BRANCH_MASK) { 3531 if (!insn->hint && !insn_cfi_match(insn, &state.cfi)) 3532 return 1; 3533 3534 if (insn->visited & visited) 3535 return 0; 3536 } else { 3537 nr_insns_visited++; 3538 } 3539 3540 if (state.noinstr) 3541 state.instr += insn->instr; 3542 3543 if (insn->hint) { 3544 if (insn->restore) { 3545 struct instruction *save_insn, *i; 3546 3547 i = insn; 3548 save_insn = NULL; 3549 3550 sym_for_each_insn_continue_reverse(file, func, i) { 3551 if (i->save) { 3552 save_insn = i; 3553 break; 3554 } 3555 } 3556 3557 if (!save_insn) { 3558 WARN_FUNC("no corresponding CFI save for CFI restore", 3559 sec, insn->offset); 3560 return 1; 3561 } 3562 3563 if (!save_insn->visited) { 3564 WARN_FUNC("objtool isn't smart enough to handle this CFI save/restore combo", 3565 sec, insn->offset); 3566 return 1; 3567 } 3568 3569 insn->cfi = save_insn->cfi; 3570 nr_cfi_reused++; 3571 } 3572 3573 state.cfi = *insn->cfi; 3574 } else { 3575 /* XXX track if we actually changed state.cfi */ 3576 3577 if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) { 3578 insn->cfi = prev_insn->cfi; 3579 nr_cfi_reused++; 3580 } else { 3581 insn->cfi = cfi_hash_find_or_add(&state.cfi); 3582 } 3583 } 3584 3585 insn->visited |= visited; 3586 3587 if (propagate_alt_cfi(file, insn)) 3588 return 1; 3589 3590 if (!insn->ignore_alts && !list_empty(&insn->alts)) { 3591 bool skip_orig = false; 3592 3593 list_for_each_entry(alt, &insn->alts, list) { 3594 if (alt->skip_orig) 3595 skip_orig = true; 3596 3597 ret = validate_branch(file, func, alt->insn, state); 3598 if (ret) { 3599 if (opts.backtrace) 3600 BT_FUNC("(alt)", insn); 3601 return ret; 3602 } 3603 } 3604 3605 if (skip_orig) 3606 return 0; 3607 } 3608 3609 if (handle_insn_ops(insn, next_insn, &state)) 3610 return 1; 3611 3612 switch (insn->type) { 3613 3614 case INSN_RETURN: 3615 return validate_return(func, insn, &state); 3616 3617 case INSN_CALL: 3618 case INSN_CALL_DYNAMIC: 3619 ret = validate_call(file, insn, &state); 3620 if (ret) 3621 return ret; 3622 3623 if (opts.stackval && func && !is_fentry_call(insn) && 3624 !has_valid_stack_frame(&state)) { 3625 WARN_FUNC("call without frame pointer save/setup", 3626 sec, insn->offset); 3627 return 1; 3628 } 3629 3630 if (insn->dead_end) 3631 return 0; 3632 3633 break; 3634 3635 case INSN_JUMP_CONDITIONAL: 3636 case INSN_JUMP_UNCONDITIONAL: 3637 if (is_sibling_call(insn)) { 3638 ret = validate_sibling_call(file, insn, &state); 3639 if (ret) 3640 return ret; 3641 3642 } else if (insn->jump_dest) { 3643 ret = validate_branch(file, func, 3644 insn->jump_dest, state); 3645 if (ret) { 3646 if (opts.backtrace) 3647 BT_FUNC("(branch)", insn); 3648 return ret; 3649 } 3650 } 3651 3652 if (insn->type == INSN_JUMP_UNCONDITIONAL) 3653 return 0; 3654 3655 break; 3656 3657 case INSN_JUMP_DYNAMIC: 3658 case INSN_JUMP_DYNAMIC_CONDITIONAL: 3659 if (is_sibling_call(insn)) { 3660 ret = validate_sibling_call(file, insn, &state); 3661 if (ret) 3662 return ret; 3663 } 3664 3665 if (insn->type == INSN_JUMP_DYNAMIC) 3666 return 0; 3667 3668 break; 3669 3670 case INSN_CONTEXT_SWITCH: 3671 if (func && (!next_insn || !next_insn->hint)) { 3672 WARN_FUNC("unsupported instruction in callable function", 3673 sec, insn->offset); 3674 return 1; 3675 } 3676 return 0; 3677 3678 case INSN_STAC: 3679 if (state.uaccess) { 3680 WARN_FUNC("recursive UACCESS enable", sec, insn->offset); 3681 return 1; 3682 } 3683 3684 state.uaccess = true; 3685 break; 3686 3687 case INSN_CLAC: 3688 if (!state.uaccess && func) { 3689 WARN_FUNC("redundant UACCESS disable", sec, insn->offset); 3690 return 1; 3691 } 3692 3693 if (func_uaccess_safe(func) && !state.uaccess_stack) { 3694 WARN_FUNC("UACCESS-safe disables UACCESS", sec, insn->offset); 3695 return 1; 3696 } 3697 3698 state.uaccess = false; 3699 break; 3700 3701 case INSN_STD: 3702 if (state.df) { 3703 WARN_FUNC("recursive STD", sec, insn->offset); 3704 return 1; 3705 } 3706 3707 state.df = true; 3708 break; 3709 3710 case INSN_CLD: 3711 if (!state.df && func) { 3712 WARN_FUNC("redundant CLD", sec, insn->offset); 3713 return 1; 3714 } 3715 3716 state.df = false; 3717 break; 3718 3719 default: 3720 break; 3721 } 3722 3723 if (insn->dead_end) 3724 return 0; 3725 3726 if (!next_insn) { 3727 if (state.cfi.cfa.base == CFI_UNDEFINED) 3728 return 0; 3729 WARN("%s: unexpected end of section", sec->name); 3730 return 1; 3731 } 3732 3733 prev_insn = insn; 3734 insn = next_insn; 3735 } 3736 3737 return 0; 3738 } 3739 3740 static int validate_unwind_hints(struct objtool_file *file, struct section *sec) 3741 { 3742 struct instruction *insn; 3743 struct insn_state state; 3744 int ret, warnings = 0; 3745 3746 if (!file->hints) 3747 return 0; 3748 3749 init_insn_state(file, &state, sec); 3750 3751 if (sec) { 3752 insn = find_insn(file, sec, 0); 3753 if (!insn) 3754 return 0; 3755 } else { 3756 insn = list_first_entry(&file->insn_list, typeof(*insn), list); 3757 } 3758 3759 while (&insn->list != &file->insn_list && (!sec || insn->sec == sec)) { 3760 if (insn->hint && !insn->visited && !insn->ignore) { 3761 ret = validate_branch(file, insn_func(insn), insn, state); 3762 if (ret && opts.backtrace) 3763 BT_FUNC("<=== (hint)", insn); 3764 warnings += ret; 3765 } 3766 3767 insn = list_next_entry(insn, list); 3768 } 3769 3770 return warnings; 3771 } 3772 3773 /* 3774 * Validate rethunk entry constraint: must untrain RET before the first RET. 3775 * 3776 * Follow every branch (intra-function) and ensure ANNOTATE_UNRET_END comes 3777 * before an actual RET instruction. 3778 */ 3779 static int validate_entry(struct objtool_file *file, struct instruction *insn) 3780 { 3781 struct instruction *next, *dest; 3782 int ret, warnings = 0; 3783 3784 for (;;) { 3785 next = next_insn_to_validate(file, insn); 3786 3787 if (insn->visited & VISITED_ENTRY) 3788 return 0; 3789 3790 insn->visited |= VISITED_ENTRY; 3791 3792 if (!insn->ignore_alts && !list_empty(&insn->alts)) { 3793 struct alternative *alt; 3794 bool skip_orig = false; 3795 3796 list_for_each_entry(alt, &insn->alts, list) { 3797 if (alt->skip_orig) 3798 skip_orig = true; 3799 3800 ret = validate_entry(file, alt->insn); 3801 if (ret) { 3802 if (opts.backtrace) 3803 BT_FUNC("(alt)", insn); 3804 return ret; 3805 } 3806 } 3807 3808 if (skip_orig) 3809 return 0; 3810 } 3811 3812 switch (insn->type) { 3813 3814 case INSN_CALL_DYNAMIC: 3815 case INSN_JUMP_DYNAMIC: 3816 case INSN_JUMP_DYNAMIC_CONDITIONAL: 3817 WARN_FUNC("early indirect call", insn->sec, insn->offset); 3818 return 1; 3819 3820 case INSN_JUMP_UNCONDITIONAL: 3821 case INSN_JUMP_CONDITIONAL: 3822 if (!is_sibling_call(insn)) { 3823 if (!insn->jump_dest) { 3824 WARN_FUNC("unresolved jump target after linking?!?", 3825 insn->sec, insn->offset); 3826 return -1; 3827 } 3828 ret = validate_entry(file, insn->jump_dest); 3829 if (ret) { 3830 if (opts.backtrace) { 3831 BT_FUNC("(branch%s)", insn, 3832 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : ""); 3833 } 3834 return ret; 3835 } 3836 3837 if (insn->type == INSN_JUMP_UNCONDITIONAL) 3838 return 0; 3839 3840 break; 3841 } 3842 3843 /* fallthrough */ 3844 case INSN_CALL: 3845 dest = find_insn(file, insn->call_dest->sec, 3846 insn->call_dest->offset); 3847 if (!dest) { 3848 WARN("Unresolved function after linking!?: %s", 3849 insn->call_dest->name); 3850 return -1; 3851 } 3852 3853 ret = validate_entry(file, dest); 3854 if (ret) { 3855 if (opts.backtrace) 3856 BT_FUNC("(call)", insn); 3857 return ret; 3858 } 3859 /* 3860 * If a call returns without error, it must have seen UNTRAIN_RET. 3861 * Therefore any non-error return is a success. 3862 */ 3863 return 0; 3864 3865 case INSN_RETURN: 3866 WARN_FUNC("RET before UNTRAIN", insn->sec, insn->offset); 3867 return 1; 3868 3869 case INSN_NOP: 3870 if (insn->retpoline_safe) 3871 return 0; 3872 break; 3873 3874 default: 3875 break; 3876 } 3877 3878 if (!next) { 3879 WARN_FUNC("teh end!", insn->sec, insn->offset); 3880 return -1; 3881 } 3882 insn = next; 3883 } 3884 3885 return warnings; 3886 } 3887 3888 /* 3889 * Validate that all branches starting at 'insn->entry' encounter UNRET_END 3890 * before RET. 3891 */ 3892 static int validate_unret(struct objtool_file *file) 3893 { 3894 struct instruction *insn; 3895 int ret, warnings = 0; 3896 3897 for_each_insn(file, insn) { 3898 if (!insn->entry) 3899 continue; 3900 3901 ret = validate_entry(file, insn); 3902 if (ret < 0) { 3903 WARN_FUNC("Failed UNRET validation", insn->sec, insn->offset); 3904 return ret; 3905 } 3906 warnings += ret; 3907 } 3908 3909 return warnings; 3910 } 3911 3912 static int validate_retpoline(struct objtool_file *file) 3913 { 3914 struct instruction *insn; 3915 int warnings = 0; 3916 3917 for_each_insn(file, insn) { 3918 if (insn->type != INSN_JUMP_DYNAMIC && 3919 insn->type != INSN_CALL_DYNAMIC && 3920 insn->type != INSN_RETURN) 3921 continue; 3922 3923 if (insn->retpoline_safe) 3924 continue; 3925 3926 if (insn->sec->init) 3927 continue; 3928 3929 if (insn->type == INSN_RETURN) { 3930 if (opts.rethunk) { 3931 WARN_FUNC("'naked' return found in RETHUNK build", 3932 insn->sec, insn->offset); 3933 } else 3934 continue; 3935 } else { 3936 WARN_FUNC("indirect %s found in RETPOLINE build", 3937 insn->sec, insn->offset, 3938 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call"); 3939 } 3940 3941 warnings++; 3942 } 3943 3944 return warnings; 3945 } 3946 3947 static bool is_kasan_insn(struct instruction *insn) 3948 { 3949 return (insn->type == INSN_CALL && 3950 !strcmp(insn->call_dest->name, "__asan_handle_no_return")); 3951 } 3952 3953 static bool is_ubsan_insn(struct instruction *insn) 3954 { 3955 return (insn->type == INSN_CALL && 3956 !strcmp(insn->call_dest->name, 3957 "__ubsan_handle_builtin_unreachable")); 3958 } 3959 3960 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn) 3961 { 3962 int i; 3963 struct instruction *prev_insn; 3964 3965 if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP) 3966 return true; 3967 3968 /* 3969 * Ignore alternative replacement instructions. This can happen 3970 * when a whitelisted function uses one of the ALTERNATIVE macros. 3971 */ 3972 if (!strcmp(insn->sec->name, ".altinstr_replacement") || 3973 !strcmp(insn->sec->name, ".altinstr_aux")) 3974 return true; 3975 3976 /* 3977 * Whole archive runs might encounter dead code from weak symbols. 3978 * This is where the linker will have dropped the weak symbol in 3979 * favour of a regular symbol, but leaves the code in place. 3980 * 3981 * In this case we'll find a piece of code (whole function) that is not 3982 * covered by a !section symbol. Ignore them. 3983 */ 3984 if (opts.link && !insn_func(insn)) { 3985 int size = find_symbol_hole_containing(insn->sec, insn->offset); 3986 unsigned long end = insn->offset + size; 3987 3988 if (!size) /* not a hole */ 3989 return false; 3990 3991 if (size < 0) /* hole until the end */ 3992 return true; 3993 3994 sec_for_each_insn_continue(file, insn) { 3995 /* 3996 * If we reach a visited instruction at or before the 3997 * end of the hole, ignore the unreachable. 3998 */ 3999 if (insn->visited) 4000 return true; 4001 4002 if (insn->offset >= end) 4003 break; 4004 4005 /* 4006 * If this hole jumps to a .cold function, mark it ignore too. 4007 */ 4008 if (insn->jump_dest && insn_func(insn->jump_dest) && 4009 strstr(insn_func(insn->jump_dest)->name, ".cold")) { 4010 struct instruction *dest = insn->jump_dest; 4011 func_for_each_insn(file, insn_func(dest), dest) 4012 dest->ignore = true; 4013 } 4014 } 4015 4016 return false; 4017 } 4018 4019 if (!insn_func(insn)) 4020 return false; 4021 4022 if (insn_func(insn)->static_call_tramp) 4023 return true; 4024 4025 /* 4026 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees 4027 * __builtin_unreachable(). The BUG() macro has an unreachable() after 4028 * the UD2, which causes GCC's undefined trap logic to emit another UD2 4029 * (or occasionally a JMP to UD2). 4030 * 4031 * It may also insert a UD2 after calling a __noreturn function. 4032 */ 4033 prev_insn = list_prev_entry(insn, list); 4034 if ((prev_insn->dead_end || dead_end_function(file, prev_insn->call_dest)) && 4035 (insn->type == INSN_BUG || 4036 (insn->type == INSN_JUMP_UNCONDITIONAL && 4037 insn->jump_dest && insn->jump_dest->type == INSN_BUG))) 4038 return true; 4039 4040 /* 4041 * Check if this (or a subsequent) instruction is related to 4042 * CONFIG_UBSAN or CONFIG_KASAN. 4043 * 4044 * End the search at 5 instructions to avoid going into the weeds. 4045 */ 4046 for (i = 0; i < 5; i++) { 4047 4048 if (is_kasan_insn(insn) || is_ubsan_insn(insn)) 4049 return true; 4050 4051 if (insn->type == INSN_JUMP_UNCONDITIONAL) { 4052 if (insn->jump_dest && 4053 insn_func(insn->jump_dest) == insn_func(insn)) { 4054 insn = insn->jump_dest; 4055 continue; 4056 } 4057 4058 break; 4059 } 4060 4061 if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len) 4062 break; 4063 4064 insn = list_next_entry(insn, list); 4065 } 4066 4067 return false; 4068 } 4069 4070 static int add_prefix_symbol(struct objtool_file *file, struct symbol *func, 4071 struct instruction *insn) 4072 { 4073 if (!opts.prefix) 4074 return 0; 4075 4076 for (;;) { 4077 struct instruction *prev = list_prev_entry(insn, list); 4078 u64 offset; 4079 4080 if (&prev->list == &file->insn_list) 4081 break; 4082 4083 if (prev->type != INSN_NOP) 4084 break; 4085 4086 offset = func->offset - prev->offset; 4087 if (offset >= opts.prefix) { 4088 if (offset == opts.prefix) { 4089 /* 4090 * Since the sec->symbol_list is ordered by 4091 * offset (see elf_add_symbol()) the added 4092 * symbol will not be seen by the iteration in 4093 * validate_section(). 4094 * 4095 * Hence the lack of list_for_each_entry_safe() 4096 * there. 4097 * 4098 * The direct concequence is that prefix symbols 4099 * don't get visited (because pointless), except 4100 * for the logic in ignore_unreachable_insn() 4101 * that needs the terminating insn to be visited 4102 * otherwise it will report the hole. 4103 * 4104 * Hence mark the first instruction of the 4105 * prefix symbol as visisted. 4106 */ 4107 prev->visited |= VISITED_BRANCH; 4108 elf_create_prefix_symbol(file->elf, func, opts.prefix); 4109 } 4110 break; 4111 } 4112 insn = prev; 4113 } 4114 4115 return 0; 4116 } 4117 4118 static int validate_symbol(struct objtool_file *file, struct section *sec, 4119 struct symbol *sym, struct insn_state *state) 4120 { 4121 struct instruction *insn; 4122 int ret; 4123 4124 if (!sym->len) { 4125 WARN("%s() is missing an ELF size annotation", sym->name); 4126 return 1; 4127 } 4128 4129 if (sym->pfunc != sym || sym->alias != sym) 4130 return 0; 4131 4132 insn = find_insn(file, sec, sym->offset); 4133 if (!insn || insn->ignore || insn->visited) 4134 return 0; 4135 4136 add_prefix_symbol(file, sym, insn); 4137 4138 state->uaccess = sym->uaccess_safe; 4139 4140 ret = validate_branch(file, insn_func(insn), insn, *state); 4141 if (ret && opts.backtrace) 4142 BT_FUNC("<=== (sym)", insn); 4143 return ret; 4144 } 4145 4146 static int validate_section(struct objtool_file *file, struct section *sec) 4147 { 4148 struct insn_state state; 4149 struct symbol *func; 4150 int warnings = 0; 4151 4152 list_for_each_entry(func, &sec->symbol_list, list) { 4153 if (func->type != STT_FUNC) 4154 continue; 4155 4156 init_insn_state(file, &state, sec); 4157 set_func_state(&state.cfi); 4158 4159 warnings += validate_symbol(file, sec, func, &state); 4160 } 4161 4162 return warnings; 4163 } 4164 4165 static int validate_noinstr_sections(struct objtool_file *file) 4166 { 4167 struct section *sec; 4168 int warnings = 0; 4169 4170 sec = find_section_by_name(file->elf, ".noinstr.text"); 4171 if (sec) { 4172 warnings += validate_section(file, sec); 4173 warnings += validate_unwind_hints(file, sec); 4174 } 4175 4176 sec = find_section_by_name(file->elf, ".entry.text"); 4177 if (sec) { 4178 warnings += validate_section(file, sec); 4179 warnings += validate_unwind_hints(file, sec); 4180 } 4181 4182 sec = find_section_by_name(file->elf, ".cpuidle.text"); 4183 if (sec) { 4184 warnings += validate_section(file, sec); 4185 warnings += validate_unwind_hints(file, sec); 4186 } 4187 4188 return warnings; 4189 } 4190 4191 static int validate_functions(struct objtool_file *file) 4192 { 4193 struct section *sec; 4194 int warnings = 0; 4195 4196 for_each_sec(file, sec) { 4197 if (!(sec->sh.sh_flags & SHF_EXECINSTR)) 4198 continue; 4199 4200 warnings += validate_section(file, sec); 4201 } 4202 4203 return warnings; 4204 } 4205 4206 static void mark_endbr_used(struct instruction *insn) 4207 { 4208 if (!list_empty(&insn->call_node)) 4209 list_del_init(&insn->call_node); 4210 } 4211 4212 static bool noendbr_range(struct objtool_file *file, struct instruction *insn) 4213 { 4214 struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1); 4215 struct instruction *first; 4216 4217 if (!sym) 4218 return false; 4219 4220 first = find_insn(file, sym->sec, sym->offset); 4221 if (!first) 4222 return false; 4223 4224 if (first->type != INSN_ENDBR && !first->noendbr) 4225 return false; 4226 4227 return insn->offset == sym->offset + sym->len; 4228 } 4229 4230 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn) 4231 { 4232 struct instruction *dest; 4233 struct reloc *reloc; 4234 unsigned long off; 4235 int warnings = 0; 4236 4237 /* 4238 * Looking for function pointer load relocations. Ignore 4239 * direct/indirect branches: 4240 */ 4241 switch (insn->type) { 4242 case INSN_CALL: 4243 case INSN_CALL_DYNAMIC: 4244 case INSN_JUMP_CONDITIONAL: 4245 case INSN_JUMP_UNCONDITIONAL: 4246 case INSN_JUMP_DYNAMIC: 4247 case INSN_JUMP_DYNAMIC_CONDITIONAL: 4248 case INSN_RETURN: 4249 case INSN_NOP: 4250 return 0; 4251 default: 4252 break; 4253 } 4254 4255 for (reloc = insn_reloc(file, insn); 4256 reloc; 4257 reloc = find_reloc_by_dest_range(file->elf, insn->sec, 4258 reloc->offset + 1, 4259 (insn->offset + insn->len) - (reloc->offset + 1))) { 4260 4261 /* 4262 * static_call_update() references the trampoline, which 4263 * doesn't have (or need) ENDBR. Skip warning in that case. 4264 */ 4265 if (reloc->sym->static_call_tramp) 4266 continue; 4267 4268 off = reloc->sym->offset; 4269 if (reloc->type == R_X86_64_PC32 || reloc->type == R_X86_64_PLT32) 4270 off += arch_dest_reloc_offset(reloc->addend); 4271 else 4272 off += reloc->addend; 4273 4274 dest = find_insn(file, reloc->sym->sec, off); 4275 if (!dest) 4276 continue; 4277 4278 if (dest->type == INSN_ENDBR) { 4279 mark_endbr_used(dest); 4280 continue; 4281 } 4282 4283 if (insn_func(dest) && insn_func(dest) == insn_func(insn)) { 4284 /* 4285 * Anything from->to self is either _THIS_IP_ or 4286 * IRET-to-self. 4287 * 4288 * There is no sane way to annotate _THIS_IP_ since the 4289 * compiler treats the relocation as a constant and is 4290 * happy to fold in offsets, skewing any annotation we 4291 * do, leading to vast amounts of false-positives. 4292 * 4293 * There's also compiler generated _THIS_IP_ through 4294 * KCOV and such which we have no hope of annotating. 4295 * 4296 * As such, blanket accept self-references without 4297 * issue. 4298 */ 4299 continue; 4300 } 4301 4302 /* 4303 * Accept anything ANNOTATE_NOENDBR. 4304 */ 4305 if (dest->noendbr) 4306 continue; 4307 4308 /* 4309 * Accept if this is the instruction after a symbol 4310 * that is (no)endbr -- typical code-range usage. 4311 */ 4312 if (noendbr_range(file, dest)) 4313 continue; 4314 4315 WARN_FUNC("relocation to !ENDBR: %s", 4316 insn->sec, insn->offset, 4317 offstr(dest->sec, dest->offset)); 4318 4319 warnings++; 4320 } 4321 4322 return warnings; 4323 } 4324 4325 static int validate_ibt_data_reloc(struct objtool_file *file, 4326 struct reloc *reloc) 4327 { 4328 struct instruction *dest; 4329 4330 dest = find_insn(file, reloc->sym->sec, 4331 reloc->sym->offset + reloc->addend); 4332 if (!dest) 4333 return 0; 4334 4335 if (dest->type == INSN_ENDBR) { 4336 mark_endbr_used(dest); 4337 return 0; 4338 } 4339 4340 if (dest->noendbr) 4341 return 0; 4342 4343 WARN_FUNC("data relocation to !ENDBR: %s", 4344 reloc->sec->base, reloc->offset, 4345 offstr(dest->sec, dest->offset)); 4346 4347 return 1; 4348 } 4349 4350 /* 4351 * Validate IBT rules and remove used ENDBR instructions from the seal list. 4352 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with 4353 * NOPs) later, in create_ibt_endbr_seal_sections(). 4354 */ 4355 static int validate_ibt(struct objtool_file *file) 4356 { 4357 struct section *sec; 4358 struct reloc *reloc; 4359 struct instruction *insn; 4360 int warnings = 0; 4361 4362 for_each_insn(file, insn) 4363 warnings += validate_ibt_insn(file, insn); 4364 4365 for_each_sec(file, sec) { 4366 4367 /* Already done by validate_ibt_insn() */ 4368 if (sec->sh.sh_flags & SHF_EXECINSTR) 4369 continue; 4370 4371 if (!sec->reloc) 4372 continue; 4373 4374 /* 4375 * These sections can reference text addresses, but not with 4376 * the intent to indirect branch to them. 4377 */ 4378 if ((!strncmp(sec->name, ".discard", 8) && 4379 strcmp(sec->name, ".discard.ibt_endbr_noseal")) || 4380 !strncmp(sec->name, ".debug", 6) || 4381 !strcmp(sec->name, ".altinstructions") || 4382 !strcmp(sec->name, ".ibt_endbr_seal") || 4383 !strcmp(sec->name, ".orc_unwind_ip") || 4384 !strcmp(sec->name, ".parainstructions") || 4385 !strcmp(sec->name, ".retpoline_sites") || 4386 !strcmp(sec->name, ".smp_locks") || 4387 !strcmp(sec->name, ".static_call_sites") || 4388 !strcmp(sec->name, "_error_injection_whitelist") || 4389 !strcmp(sec->name, "_kprobe_blacklist") || 4390 !strcmp(sec->name, "__bug_table") || 4391 !strcmp(sec->name, "__ex_table") || 4392 !strcmp(sec->name, "__jump_table") || 4393 !strcmp(sec->name, "__mcount_loc") || 4394 !strcmp(sec->name, ".kcfi_traps") || 4395 strstr(sec->name, "__patchable_function_entries")) 4396 continue; 4397 4398 list_for_each_entry(reloc, &sec->reloc->reloc_list, list) 4399 warnings += validate_ibt_data_reloc(file, reloc); 4400 } 4401 4402 return warnings; 4403 } 4404 4405 static int validate_sls(struct objtool_file *file) 4406 { 4407 struct instruction *insn, *next_insn; 4408 int warnings = 0; 4409 4410 for_each_insn(file, insn) { 4411 next_insn = next_insn_same_sec(file, insn); 4412 4413 if (insn->retpoline_safe) 4414 continue; 4415 4416 switch (insn->type) { 4417 case INSN_RETURN: 4418 if (!next_insn || next_insn->type != INSN_TRAP) { 4419 WARN_FUNC("missing int3 after ret", 4420 insn->sec, insn->offset); 4421 warnings++; 4422 } 4423 4424 break; 4425 case INSN_JUMP_DYNAMIC: 4426 if (!next_insn || next_insn->type != INSN_TRAP) { 4427 WARN_FUNC("missing int3 after indirect jump", 4428 insn->sec, insn->offset); 4429 warnings++; 4430 } 4431 break; 4432 default: 4433 break; 4434 } 4435 } 4436 4437 return warnings; 4438 } 4439 4440 static int validate_reachable_instructions(struct objtool_file *file) 4441 { 4442 struct instruction *insn; 4443 4444 if (file->ignore_unreachables) 4445 return 0; 4446 4447 for_each_insn(file, insn) { 4448 if (insn->visited || ignore_unreachable_insn(file, insn)) 4449 continue; 4450 4451 WARN_FUNC("unreachable instruction", insn->sec, insn->offset); 4452 return 1; 4453 } 4454 4455 return 0; 4456 } 4457 4458 int check(struct objtool_file *file) 4459 { 4460 int ret, warnings = 0; 4461 4462 arch_initial_func_cfi_state(&initial_func_cfi); 4463 init_cfi_state(&init_cfi); 4464 init_cfi_state(&func_cfi); 4465 set_func_state(&func_cfi); 4466 4467 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3))) 4468 goto out; 4469 4470 cfi_hash_add(&init_cfi); 4471 cfi_hash_add(&func_cfi); 4472 4473 ret = decode_sections(file); 4474 if (ret < 0) 4475 goto out; 4476 4477 warnings += ret; 4478 4479 if (list_empty(&file->insn_list)) 4480 goto out; 4481 4482 if (opts.retpoline) { 4483 ret = validate_retpoline(file); 4484 if (ret < 0) 4485 return ret; 4486 warnings += ret; 4487 } 4488 4489 if (opts.stackval || opts.orc || opts.uaccess) { 4490 ret = validate_functions(file); 4491 if (ret < 0) 4492 goto out; 4493 warnings += ret; 4494 4495 ret = validate_unwind_hints(file, NULL); 4496 if (ret < 0) 4497 goto out; 4498 warnings += ret; 4499 4500 if (!warnings) { 4501 ret = validate_reachable_instructions(file); 4502 if (ret < 0) 4503 goto out; 4504 warnings += ret; 4505 } 4506 4507 } else if (opts.noinstr) { 4508 ret = validate_noinstr_sections(file); 4509 if (ret < 0) 4510 goto out; 4511 warnings += ret; 4512 } 4513 4514 if (opts.unret) { 4515 /* 4516 * Must be after validate_branch() and friends, it plays 4517 * further games with insn->visited. 4518 */ 4519 ret = validate_unret(file); 4520 if (ret < 0) 4521 return ret; 4522 warnings += ret; 4523 } 4524 4525 if (opts.ibt) { 4526 ret = validate_ibt(file); 4527 if (ret < 0) 4528 goto out; 4529 warnings += ret; 4530 } 4531 4532 if (opts.sls) { 4533 ret = validate_sls(file); 4534 if (ret < 0) 4535 goto out; 4536 warnings += ret; 4537 } 4538 4539 if (opts.static_call) { 4540 ret = create_static_call_sections(file); 4541 if (ret < 0) 4542 goto out; 4543 warnings += ret; 4544 } 4545 4546 if (opts.retpoline) { 4547 ret = create_retpoline_sites_sections(file); 4548 if (ret < 0) 4549 goto out; 4550 warnings += ret; 4551 } 4552 4553 if (opts.cfi) { 4554 ret = create_cfi_sections(file); 4555 if (ret < 0) 4556 goto out; 4557 warnings += ret; 4558 } 4559 4560 if (opts.rethunk) { 4561 ret = create_return_sites_sections(file); 4562 if (ret < 0) 4563 goto out; 4564 warnings += ret; 4565 4566 if (opts.hack_skylake) { 4567 ret = create_direct_call_sections(file); 4568 if (ret < 0) 4569 goto out; 4570 warnings += ret; 4571 } 4572 } 4573 4574 if (opts.mcount) { 4575 ret = create_mcount_loc_sections(file); 4576 if (ret < 0) 4577 goto out; 4578 warnings += ret; 4579 } 4580 4581 if (opts.ibt) { 4582 ret = create_ibt_endbr_seal_sections(file); 4583 if (ret < 0) 4584 goto out; 4585 warnings += ret; 4586 } 4587 4588 if (opts.orc && !list_empty(&file->insn_list)) { 4589 ret = orc_create(file); 4590 if (ret < 0) 4591 goto out; 4592 warnings += ret; 4593 } 4594 4595 4596 if (opts.stats) { 4597 printf("nr_insns_visited: %ld\n", nr_insns_visited); 4598 printf("nr_cfi: %ld\n", nr_cfi); 4599 printf("nr_cfi_reused: %ld\n", nr_cfi_reused); 4600 printf("nr_cfi_cache: %ld\n", nr_cfi_cache); 4601 } 4602 4603 out: 4604 /* 4605 * For now, don't fail the kernel build on fatal warnings. These 4606 * errors are still fairly common due to the growing matrix of 4607 * supported toolchains and their recent pace of change. 4608 */ 4609 return 0; 4610 } 4611