1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2015 Josh Poimboeuf <jpoimboe@redhat.com> 4 */ 5 6 #include <stdio.h> 7 #include <stdlib.h> 8 9 #define unlikely(cond) (cond) 10 #include <asm/insn.h> 11 #include "../../../arch/x86/lib/inat.c" 12 #include "../../../arch/x86/lib/insn.c" 13 14 #define CONFIG_64BIT 1 15 #include <asm/nops.h> 16 17 #include <asm/orc_types.h> 18 #include <objtool/check.h> 19 #include <objtool/elf.h> 20 #include <objtool/arch.h> 21 #include <objtool/warn.h> 22 #include <arch/elf.h> 23 24 static unsigned char op_to_cfi_reg[][2] = { 25 {CFI_AX, CFI_R8}, 26 {CFI_CX, CFI_R9}, 27 {CFI_DX, CFI_R10}, 28 {CFI_BX, CFI_R11}, 29 {CFI_SP, CFI_R12}, 30 {CFI_BP, CFI_R13}, 31 {CFI_SI, CFI_R14}, 32 {CFI_DI, CFI_R15}, 33 }; 34 35 static int is_x86_64(const struct elf *elf) 36 { 37 switch (elf->ehdr.e_machine) { 38 case EM_X86_64: 39 return 1; 40 case EM_386: 41 return 0; 42 default: 43 WARN("unexpected ELF machine type %d", elf->ehdr.e_machine); 44 return -1; 45 } 46 } 47 48 bool arch_callee_saved_reg(unsigned char reg) 49 { 50 switch (reg) { 51 case CFI_BP: 52 case CFI_BX: 53 case CFI_R12: 54 case CFI_R13: 55 case CFI_R14: 56 case CFI_R15: 57 return true; 58 59 case CFI_AX: 60 case CFI_CX: 61 case CFI_DX: 62 case CFI_SI: 63 case CFI_DI: 64 case CFI_SP: 65 case CFI_R8: 66 case CFI_R9: 67 case CFI_R10: 68 case CFI_R11: 69 case CFI_RA: 70 default: 71 return false; 72 } 73 } 74 75 unsigned long arch_dest_reloc_offset(int addend) 76 { 77 return addend + 4; 78 } 79 80 unsigned long arch_jump_destination(struct instruction *insn) 81 { 82 return insn->offset + insn->len + insn->immediate; 83 } 84 85 #define ADD_OP(op) \ 86 if (!(op = calloc(1, sizeof(*op)))) \ 87 return -1; \ 88 else for (list_add_tail(&op->list, ops_list); op; op = NULL) 89 90 int arch_decode_instruction(const struct elf *elf, const struct section *sec, 91 unsigned long offset, unsigned int maxlen, 92 unsigned int *len, enum insn_type *type, 93 unsigned long *immediate, 94 struct list_head *ops_list) 95 { 96 struct insn insn; 97 int x86_64, sign, ret; 98 unsigned char op1, op2, rex = 0, rex_b = 0, rex_r = 0, rex_w = 0, 99 rex_x = 0, modrm = 0, modrm_mod = 0, modrm_rm = 0, 100 modrm_reg = 0, sib = 0; 101 struct stack_op *op = NULL; 102 struct symbol *sym; 103 104 x86_64 = is_x86_64(elf); 105 if (x86_64 == -1) 106 return -1; 107 108 ret = insn_decode(&insn, sec->data->d_buf + offset, maxlen, 109 x86_64 ? INSN_MODE_64 : INSN_MODE_32); 110 if (ret < 0) { 111 WARN("can't decode instruction at %s:0x%lx", sec->name, offset); 112 return -1; 113 } 114 115 *len = insn.length; 116 *type = INSN_OTHER; 117 118 if (insn.vex_prefix.nbytes) 119 return 0; 120 121 op1 = insn.opcode.bytes[0]; 122 op2 = insn.opcode.bytes[1]; 123 124 if (insn.rex_prefix.nbytes) { 125 rex = insn.rex_prefix.bytes[0]; 126 rex_w = X86_REX_W(rex) >> 3; 127 rex_r = X86_REX_R(rex) >> 2; 128 rex_x = X86_REX_X(rex) >> 1; 129 rex_b = X86_REX_B(rex); 130 } 131 132 if (insn.modrm.nbytes) { 133 modrm = insn.modrm.bytes[0]; 134 modrm_mod = X86_MODRM_MOD(modrm); 135 modrm_reg = X86_MODRM_REG(modrm); 136 modrm_rm = X86_MODRM_RM(modrm); 137 } 138 139 if (insn.sib.nbytes) 140 sib = insn.sib.bytes[0]; 141 142 switch (op1) { 143 144 case 0x1: 145 case 0x29: 146 if (rex_w && !rex_b && modrm_mod == 3 && modrm_rm == 4) { 147 148 /* add/sub reg, %rsp */ 149 ADD_OP(op) { 150 op->src.type = OP_SRC_ADD; 151 op->src.reg = op_to_cfi_reg[modrm_reg][rex_r]; 152 op->dest.type = OP_DEST_REG; 153 op->dest.reg = CFI_SP; 154 } 155 } 156 break; 157 158 case 0x50 ... 0x57: 159 160 /* push reg */ 161 ADD_OP(op) { 162 op->src.type = OP_SRC_REG; 163 op->src.reg = op_to_cfi_reg[op1 & 0x7][rex_b]; 164 op->dest.type = OP_DEST_PUSH; 165 } 166 167 break; 168 169 case 0x58 ... 0x5f: 170 171 /* pop reg */ 172 ADD_OP(op) { 173 op->src.type = OP_SRC_POP; 174 op->dest.type = OP_DEST_REG; 175 op->dest.reg = op_to_cfi_reg[op1 & 0x7][rex_b]; 176 } 177 178 break; 179 180 case 0x68: 181 case 0x6a: 182 /* push immediate */ 183 ADD_OP(op) { 184 op->src.type = OP_SRC_CONST; 185 op->dest.type = OP_DEST_PUSH; 186 } 187 break; 188 189 case 0x70 ... 0x7f: 190 *type = INSN_JUMP_CONDITIONAL; 191 break; 192 193 case 0x81: 194 case 0x83: 195 if (rex != 0x48) 196 break; 197 198 if (modrm == 0xe4) { 199 /* and imm, %rsp */ 200 ADD_OP(op) { 201 op->src.type = OP_SRC_AND; 202 op->src.reg = CFI_SP; 203 op->src.offset = insn.immediate.value; 204 op->dest.type = OP_DEST_REG; 205 op->dest.reg = CFI_SP; 206 } 207 break; 208 } 209 210 if (modrm == 0xc4) 211 sign = 1; 212 else if (modrm == 0xec) 213 sign = -1; 214 else 215 break; 216 217 /* add/sub imm, %rsp */ 218 ADD_OP(op) { 219 op->src.type = OP_SRC_ADD; 220 op->src.reg = CFI_SP; 221 op->src.offset = insn.immediate.value * sign; 222 op->dest.type = OP_DEST_REG; 223 op->dest.reg = CFI_SP; 224 } 225 break; 226 227 case 0x89: 228 if (rex_w && !rex_r && modrm_reg == 4) { 229 230 if (modrm_mod == 3) { 231 /* mov %rsp, reg */ 232 ADD_OP(op) { 233 op->src.type = OP_SRC_REG; 234 op->src.reg = CFI_SP; 235 op->dest.type = OP_DEST_REG; 236 op->dest.reg = op_to_cfi_reg[modrm_rm][rex_b]; 237 } 238 break; 239 240 } else { 241 /* skip nontrivial SIB */ 242 if (modrm_rm == 4 && !(sib == 0x24 && rex_b == rex_x)) 243 break; 244 245 /* skip RIP relative displacement */ 246 if (modrm_rm == 5 && modrm_mod == 0) 247 break; 248 249 /* mov %rsp, disp(%reg) */ 250 ADD_OP(op) { 251 op->src.type = OP_SRC_REG; 252 op->src.reg = CFI_SP; 253 op->dest.type = OP_DEST_REG_INDIRECT; 254 op->dest.reg = op_to_cfi_reg[modrm_rm][rex_b]; 255 op->dest.offset = insn.displacement.value; 256 } 257 break; 258 } 259 260 break; 261 } 262 263 if (rex_w && !rex_b && modrm_mod == 3 && modrm_rm == 4) { 264 265 /* mov reg, %rsp */ 266 ADD_OP(op) { 267 op->src.type = OP_SRC_REG; 268 op->src.reg = op_to_cfi_reg[modrm_reg][rex_r]; 269 op->dest.type = OP_DEST_REG; 270 op->dest.reg = CFI_SP; 271 } 272 break; 273 } 274 275 /* fallthrough */ 276 case 0x88: 277 if (!rex_b && 278 (modrm_mod == 1 || modrm_mod == 2) && modrm_rm == 5) { 279 280 /* mov reg, disp(%rbp) */ 281 ADD_OP(op) { 282 op->src.type = OP_SRC_REG; 283 op->src.reg = op_to_cfi_reg[modrm_reg][rex_r]; 284 op->dest.type = OP_DEST_REG_INDIRECT; 285 op->dest.reg = CFI_BP; 286 op->dest.offset = insn.displacement.value; 287 } 288 break; 289 } 290 291 if (rex_w && !rex_b && modrm_rm == 4 && sib == 0x24) { 292 293 /* mov reg, disp(%rsp) */ 294 ADD_OP(op) { 295 op->src.type = OP_SRC_REG; 296 op->src.reg = op_to_cfi_reg[modrm_reg][rex_r]; 297 op->dest.type = OP_DEST_REG_INDIRECT; 298 op->dest.reg = CFI_SP; 299 op->dest.offset = insn.displacement.value; 300 } 301 break; 302 } 303 304 break; 305 306 case 0x8b: 307 if (rex_w && !rex_b && modrm_mod == 1 && modrm_rm == 5) { 308 309 /* mov disp(%rbp), reg */ 310 ADD_OP(op) { 311 op->src.type = OP_SRC_REG_INDIRECT; 312 op->src.reg = CFI_BP; 313 op->src.offset = insn.displacement.value; 314 op->dest.type = OP_DEST_REG; 315 op->dest.reg = op_to_cfi_reg[modrm_reg][rex_r]; 316 } 317 318 } else if (rex_w && !rex_b && sib == 0x24 && 319 modrm_mod != 3 && modrm_rm == 4) { 320 321 /* mov disp(%rsp), reg */ 322 ADD_OP(op) { 323 op->src.type = OP_SRC_REG_INDIRECT; 324 op->src.reg = CFI_SP; 325 op->src.offset = insn.displacement.value; 326 op->dest.type = OP_DEST_REG; 327 op->dest.reg = op_to_cfi_reg[modrm_reg][rex_r]; 328 } 329 } 330 331 break; 332 333 case 0x8d: 334 if (sib == 0x24 && rex_w && !rex_b && !rex_x) { 335 336 ADD_OP(op) { 337 if (!insn.displacement.value) { 338 /* lea (%rsp), reg */ 339 op->src.type = OP_SRC_REG; 340 } else { 341 /* lea disp(%rsp), reg */ 342 op->src.type = OP_SRC_ADD; 343 op->src.offset = insn.displacement.value; 344 } 345 op->src.reg = CFI_SP; 346 op->dest.type = OP_DEST_REG; 347 op->dest.reg = op_to_cfi_reg[modrm_reg][rex_r]; 348 } 349 350 } else if (rex == 0x48 && modrm == 0x65) { 351 352 /* lea disp(%rbp), %rsp */ 353 ADD_OP(op) { 354 op->src.type = OP_SRC_ADD; 355 op->src.reg = CFI_BP; 356 op->src.offset = insn.displacement.value; 357 op->dest.type = OP_DEST_REG; 358 op->dest.reg = CFI_SP; 359 } 360 361 } else if (rex == 0x49 && modrm == 0x62 && 362 insn.displacement.value == -8) { 363 364 /* 365 * lea -0x8(%r10), %rsp 366 * 367 * Restoring rsp back to its original value after a 368 * stack realignment. 369 */ 370 ADD_OP(op) { 371 op->src.type = OP_SRC_ADD; 372 op->src.reg = CFI_R10; 373 op->src.offset = -8; 374 op->dest.type = OP_DEST_REG; 375 op->dest.reg = CFI_SP; 376 } 377 378 } else if (rex == 0x49 && modrm == 0x65 && 379 insn.displacement.value == -16) { 380 381 /* 382 * lea -0x10(%r13), %rsp 383 * 384 * Restoring rsp back to its original value after a 385 * stack realignment. 386 */ 387 ADD_OP(op) { 388 op->src.type = OP_SRC_ADD; 389 op->src.reg = CFI_R13; 390 op->src.offset = -16; 391 op->dest.type = OP_DEST_REG; 392 op->dest.reg = CFI_SP; 393 } 394 } 395 396 break; 397 398 case 0x8f: 399 /* pop to mem */ 400 ADD_OP(op) { 401 op->src.type = OP_SRC_POP; 402 op->dest.type = OP_DEST_MEM; 403 } 404 break; 405 406 case 0x90: 407 *type = INSN_NOP; 408 break; 409 410 case 0x9c: 411 /* pushf */ 412 ADD_OP(op) { 413 op->src.type = OP_SRC_CONST; 414 op->dest.type = OP_DEST_PUSHF; 415 } 416 break; 417 418 case 0x9d: 419 /* popf */ 420 ADD_OP(op) { 421 op->src.type = OP_SRC_POPF; 422 op->dest.type = OP_DEST_MEM; 423 } 424 break; 425 426 case 0x0f: 427 428 if (op2 == 0x01) { 429 430 if (modrm == 0xca) 431 *type = INSN_CLAC; 432 else if (modrm == 0xcb) 433 *type = INSN_STAC; 434 435 } else if (op2 >= 0x80 && op2 <= 0x8f) { 436 437 *type = INSN_JUMP_CONDITIONAL; 438 439 } else if (op2 == 0x05 || op2 == 0x07 || op2 == 0x34 || 440 op2 == 0x35) { 441 442 /* sysenter, sysret */ 443 *type = INSN_CONTEXT_SWITCH; 444 445 } else if (op2 == 0x0b || op2 == 0xb9) { 446 447 /* ud2 */ 448 *type = INSN_BUG; 449 450 } else if (op2 == 0x0d || op2 == 0x1f) { 451 452 /* nopl/nopw */ 453 *type = INSN_NOP; 454 455 } else if (op2 == 0xa0 || op2 == 0xa8) { 456 457 /* push fs/gs */ 458 ADD_OP(op) { 459 op->src.type = OP_SRC_CONST; 460 op->dest.type = OP_DEST_PUSH; 461 } 462 463 } else if (op2 == 0xa1 || op2 == 0xa9) { 464 465 /* pop fs/gs */ 466 ADD_OP(op) { 467 op->src.type = OP_SRC_POP; 468 op->dest.type = OP_DEST_MEM; 469 } 470 } 471 472 break; 473 474 case 0xc9: 475 /* 476 * leave 477 * 478 * equivalent to: 479 * mov bp, sp 480 * pop bp 481 */ 482 ADD_OP(op) 483 op->dest.type = OP_DEST_LEAVE; 484 485 break; 486 487 case 0xe3: 488 /* jecxz/jrcxz */ 489 *type = INSN_JUMP_CONDITIONAL; 490 break; 491 492 case 0xe9: 493 case 0xeb: 494 *type = INSN_JUMP_UNCONDITIONAL; 495 break; 496 497 case 0xc2: 498 case 0xc3: 499 *type = INSN_RETURN; 500 break; 501 502 case 0xcf: /* iret */ 503 /* 504 * Handle sync_core(), which has an IRET to self. 505 * All other IRET are in STT_NONE entry code. 506 */ 507 sym = find_symbol_containing(sec, offset); 508 if (sym && sym->type == STT_FUNC) { 509 ADD_OP(op) { 510 /* add $40, %rsp */ 511 op->src.type = OP_SRC_ADD; 512 op->src.reg = CFI_SP; 513 op->src.offset = 5*8; 514 op->dest.type = OP_DEST_REG; 515 op->dest.reg = CFI_SP; 516 } 517 break; 518 } 519 520 /* fallthrough */ 521 522 case 0xca: /* retf */ 523 case 0xcb: /* retf */ 524 *type = INSN_CONTEXT_SWITCH; 525 break; 526 527 case 0xe8: 528 *type = INSN_CALL; 529 /* 530 * For the impact on the stack, a CALL behaves like 531 * a PUSH of an immediate value (the return address). 532 */ 533 ADD_OP(op) { 534 op->src.type = OP_SRC_CONST; 535 op->dest.type = OP_DEST_PUSH; 536 } 537 break; 538 539 case 0xfc: 540 *type = INSN_CLD; 541 break; 542 543 case 0xfd: 544 *type = INSN_STD; 545 break; 546 547 case 0xff: 548 if (modrm_reg == 2 || modrm_reg == 3) 549 550 *type = INSN_CALL_DYNAMIC; 551 552 else if (modrm_reg == 4) 553 554 *type = INSN_JUMP_DYNAMIC; 555 556 else if (modrm_reg == 5) 557 558 /* jmpf */ 559 *type = INSN_CONTEXT_SWITCH; 560 561 else if (modrm_reg == 6) { 562 563 /* push from mem */ 564 ADD_OP(op) { 565 op->src.type = OP_SRC_CONST; 566 op->dest.type = OP_DEST_PUSH; 567 } 568 } 569 570 break; 571 572 default: 573 break; 574 } 575 576 *immediate = insn.immediate.nbytes ? insn.immediate.value : 0; 577 578 return 0; 579 } 580 581 void arch_initial_func_cfi_state(struct cfi_init_state *state) 582 { 583 int i; 584 585 for (i = 0; i < CFI_NUM_REGS; i++) { 586 state->regs[i].base = CFI_UNDEFINED; 587 state->regs[i].offset = 0; 588 } 589 590 /* initial CFA (call frame address) */ 591 state->cfa.base = CFI_SP; 592 state->cfa.offset = 8; 593 594 /* initial RA (return address) */ 595 state->regs[CFI_RA].base = CFI_CFA; 596 state->regs[CFI_RA].offset = -8; 597 } 598 599 const char *arch_nop_insn(int len) 600 { 601 static const char nops[5][5] = { 602 { BYTES_NOP1 }, 603 { BYTES_NOP2 }, 604 { BYTES_NOP3 }, 605 { BYTES_NOP4 }, 606 { BYTES_NOP5 }, 607 }; 608 609 if (len < 1 || len > 5) { 610 WARN("invalid NOP size: %d\n", len); 611 return NULL; 612 } 613 614 return nops[len-1]; 615 } 616 617 /* asm/alternative.h ? */ 618 619 #define ALTINSTR_FLAG_INV (1 << 15) 620 #define ALT_NOT(feat) ((feat) | ALTINSTR_FLAG_INV) 621 622 struct alt_instr { 623 s32 instr_offset; /* original instruction */ 624 s32 repl_offset; /* offset to replacement instruction */ 625 u16 cpuid; /* cpuid bit set for replacement */ 626 u8 instrlen; /* length of original instruction */ 627 u8 replacementlen; /* length of new instruction */ 628 } __packed; 629 630 static int elf_add_alternative(struct elf *elf, 631 struct instruction *orig, struct symbol *sym, 632 int cpuid, u8 orig_len, u8 repl_len) 633 { 634 const int size = sizeof(struct alt_instr); 635 struct alt_instr *alt; 636 struct section *sec; 637 Elf_Scn *s; 638 639 sec = find_section_by_name(elf, ".altinstructions"); 640 if (!sec) { 641 sec = elf_create_section(elf, ".altinstructions", 642 SHF_WRITE, size, 0); 643 644 if (!sec) { 645 WARN_ELF("elf_create_section"); 646 return -1; 647 } 648 } 649 650 s = elf_getscn(elf->elf, sec->idx); 651 if (!s) { 652 WARN_ELF("elf_getscn"); 653 return -1; 654 } 655 656 sec->data = elf_newdata(s); 657 if (!sec->data) { 658 WARN_ELF("elf_newdata"); 659 return -1; 660 } 661 662 sec->data->d_size = size; 663 sec->data->d_align = 1; 664 665 alt = sec->data->d_buf = malloc(size); 666 if (!sec->data->d_buf) { 667 perror("malloc"); 668 return -1; 669 } 670 memset(sec->data->d_buf, 0, size); 671 672 if (elf_add_reloc_to_insn(elf, sec, sec->sh.sh_size, 673 R_X86_64_PC32, orig->sec, orig->offset)) { 674 WARN("elf_create_reloc: alt_instr::instr_offset"); 675 return -1; 676 } 677 678 if (elf_add_reloc(elf, sec, sec->sh.sh_size + 4, 679 R_X86_64_PC32, sym, 0)) { 680 WARN("elf_create_reloc: alt_instr::repl_offset"); 681 return -1; 682 } 683 684 alt->cpuid = cpuid; 685 alt->instrlen = orig_len; 686 alt->replacementlen = repl_len; 687 688 sec->sh.sh_size += size; 689 sec->changed = true; 690 691 return 0; 692 } 693 694 #define X86_FEATURE_RETPOLINE ( 7*32+12) 695 696 int arch_rewrite_retpolines(struct objtool_file *file) 697 { 698 struct instruction *insn; 699 struct reloc *reloc; 700 struct symbol *sym; 701 char name[32] = ""; 702 703 list_for_each_entry(insn, &file->retpoline_call_list, call_node) { 704 705 if (!strcmp(insn->sec->name, ".text.__x86.indirect_thunk")) 706 continue; 707 708 reloc = insn->reloc; 709 710 sprintf(name, "__x86_indirect_alt_%s_%s", 711 insn->type == INSN_JUMP_DYNAMIC ? "jmp" : "call", 712 reloc->sym->name + 21); 713 714 sym = find_symbol_by_name(file->elf, name); 715 if (!sym) { 716 sym = elf_create_undef_symbol(file->elf, name); 717 if (!sym) { 718 WARN("elf_create_undef_symbol"); 719 return -1; 720 } 721 } 722 723 if (elf_add_alternative(file->elf, insn, sym, 724 ALT_NOT(X86_FEATURE_RETPOLINE), 5, 5)) { 725 WARN("elf_add_alternative"); 726 return -1; 727 } 728 } 729 730 return 0; 731 } 732 733 int arch_decode_hint_reg(struct instruction *insn, u8 sp_reg) 734 { 735 struct cfi_reg *cfa = &insn->cfi.cfa; 736 737 switch (sp_reg) { 738 case ORC_REG_UNDEFINED: 739 cfa->base = CFI_UNDEFINED; 740 break; 741 case ORC_REG_SP: 742 cfa->base = CFI_SP; 743 break; 744 case ORC_REG_BP: 745 cfa->base = CFI_BP; 746 break; 747 case ORC_REG_SP_INDIRECT: 748 cfa->base = CFI_SP_INDIRECT; 749 break; 750 case ORC_REG_R10: 751 cfa->base = CFI_R10; 752 break; 753 case ORC_REG_R13: 754 cfa->base = CFI_R13; 755 break; 756 case ORC_REG_DI: 757 cfa->base = CFI_DI; 758 break; 759 case ORC_REG_DX: 760 cfa->base = CFI_DX; 761 break; 762 default: 763 return -1; 764 } 765 766 return 0; 767 } 768 769 bool arch_is_retpoline(struct symbol *sym) 770 { 771 return !strncmp(sym->name, "__x86_indirect_", 15); 772 } 773