Lines Matching +full:check +full:- +full:cfi +full:- +full:x86_64
1 // SPDX-License-Identifier: GPL-2.0-only
21 #include <asm/text-patching.h>
32 #include <asm/asm-prototypes.h>
38 #define MAX_PATCH_LEN (255-1)
59 __setup("debug-alternative", debug_alt);
68 __setup("noreplace-smp", setup_noreplace_smp);
85 for (j = 0; j < (len) - 1; j++) \
129 * In order not to issue an ORC stack depth tracking CFI entry (Call Frame Info)
130 * for every single-byte NOP, try to generate the maximally available NOP of
131 * size <= ASM_NOP_MAX such that only a single CFI entry is generated (vs one for
132 * each single-byte NOPs). If @len to fill out is > ASM_NOP_MAX, pad with INT3 and
173 if (insn->opcode.bytes[0] == 0x90 && in insn_is_nop()
174 (!insn->prefixes.nbytes || insn->prefixes.bytes[0] != 0xF3)) in insn_is_nop()
178 if (insn->opcode.bytes[0] == 0x0F && insn->opcode.bytes[1] == 0x1F) in insn_is_nop()
187 * Find the offset of the first non-NOP instruction starting at @offset
212 int i = *next - insn->length; in __optimize_nops()
214 switch (insn->opcode.bytes[0]) { in __optimize_nops()
218 *target = *next + insn->immediate.value; in __optimize_nops()
229 add_nop(instr + nop, *next - nop); in __optimize_nops()
277 * src_imm = target - src_next_ip (1)
281 * dst_imm = target - dst_next_ip (2)
289 * dst_imm = (src_imm + src_next_ip) - dst_next_ip (3)
300 * dst_imm = src_imm + (src + ip_offset) - (dst + ip_offset)
301 * = src_imm + src - dst + ip_offset - ip_offset
302 * = src_imm + src - dst (5)
311 BUG_ON((v >> 31) != (v >> (n_-1))); \
368 src - dest); in apply_relocation()
376 imm += src - dest; in apply_relocation()
377 imm += JMP32_INSN_SIZE - JMP8_INSN_SIZE; in apply_relocation()
382 memset(&buf[i+2], INT3_INSN_OPCODE, insn.length - 2); in apply_relocation()
392 src - dest); in apply_relocation()
415 DPRINTK(ALT, "alt table %px, -> %px", start, end); in apply_alternatives()
421 * conversion and triggers a false-positive out-of-bound report. in apply_alternatives()
439 instr = (u8 *)&a->instr_offset + a->instr_offset; in apply_alternatives()
440 replacement = (u8 *)&a->repl_offset + a->repl_offset; in apply_alternatives()
441 BUG_ON(a->instrlen > sizeof(insn_buff)); in apply_alternatives()
442 BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32); in apply_alternatives()
446 * - feature is present in apply_alternatives()
447 * - feature not present but ALT_FLAG_NOT is set to mean, in apply_alternatives()
450 if (!boot_cpu_has(a->cpuid) == !(a->flags & ALT_FLAG_NOT)) { in apply_alternatives()
451 optimize_nops_inplace(instr, a->instrlen); in apply_alternatives()
456 (a->flags & ALT_FLAG_NOT) ? "!" : "", in apply_alternatives()
457 a->cpuid >> 5, in apply_alternatives()
458 a->cpuid & 0x1f, in apply_alternatives()
459 instr, instr, a->instrlen, in apply_alternatives()
460 replacement, a->replacementlen); in apply_alternatives()
462 memcpy(insn_buff, replacement, a->replacementlen); in apply_alternatives()
463 insn_buff_sz = a->replacementlen; in apply_alternatives()
465 for (; insn_buff_sz < a->instrlen; insn_buff_sz++) in apply_alternatives()
468 apply_relocation(insn_buff, a->instrlen, instr, replacement, a->replacementlen); in apply_alternatives()
470 DUMP_BYTES(ALT, instr, a->instrlen, "%px: old_insn: ", instr); in apply_alternatives()
471 DUMP_BYTES(ALT, replacement, a->replacementlen, "%px: rpl_insn: ", replacement); in apply_alternatives()
482 /* Jcc.d32 second opcode byte is in the range: 0x80-0x8f */ in is_jcc32()
483 return insn->opcode.bytes[0] == 0x0f && (insn->opcode.bytes[1] & 0xf0) == 0x80; in is_jcc32()
507 return -1; in emit_indirect()
512 reg -= 8; in emit_indirect()
526 u8 op = insn->opcode.bytes[0]; in emit_call_track_retpoline()
531 * tail-calls. Deal with them. in emit_call_track_retpoline()
535 op = insn->opcode.bytes[1]; in emit_call_track_retpoline()
539 if (insn->length == 6) in emit_call_track_retpoline()
540 bytes[i++] = 0x2e; /* CS-prefix */ in emit_call_track_retpoline()
560 return -1; in emit_call_track_retpoline()
563 WARN_ON_ONCE(i != insn->length); in emit_call_track_retpoline()
590 target = addr + insn->length + insn->immediate.value; in patch_retpoline()
591 reg = target - __x86_indirect_thunk_array; in patch_retpoline()
594 return -1; in patch_retpoline()
604 return -1; in patch_retpoline()
607 op = insn->opcode.bytes[0]; in patch_retpoline()
623 cc = insn->opcode.bytes[1] & 0xf; in patch_retpoline()
627 bytes[i++] = insn->length - 2; /* sizeof(Jcc.d8) == 2 */ in patch_retpoline()
653 if (op == JMP32_INSN_OPCODE && i < insn->length) in patch_retpoline()
656 for (; i < insn->length;) in patch_retpoline()
663 * Generated by 'objtool --retpoline'.
714 * Rewrite the compiler generated return thunk tail-calls.
737 for (; i < insn->length;) in patch_return()
766 "missing return thunk: %pS-%pS: %*ph", in apply_returns()
820 * Generated by: objtool --ibt
834 poison_cfi(addr - 16); in apply_seal_endbr()
858 * Re-hash the CFI hash with a boot-time seed while making sure the result is
864 while (unlikely(is_endbr(hash) || is_endbr(-hash))) { in cfi_rehash()
876 return -EINVAL; in cfi_parse_cmdline()
897 pr_err("Ignoring unknown cfi option (%s).", str); in cfi_parse_cmdline()
905 early_param("cfi", cfi_parse_cmdline);
926 * movl $(-0x12345678),%r10d // 6 movl $0x12345678,%r10d // 6
927 * addl $-15(%r11),%r10d // 4 sub $16,%r11 // 4
948 #define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start)
963 #define fineibt_caller_size (fineibt_caller_end - fineibt_caller_start)
966 #define fineibt_caller_jmp (fineibt_caller_size - 2)
985 return -*(u32 *)(addr + 2); in decode_caller_hash()
989 return -*(u32 *)(addr + 2); in decode_caller_hash()
1009 addr -= fineibt_caller_size; in cfi_disable_callers()
1023 * Re-enable kCFI, undo what cfi_disable_callers() did. in cfi_enable_callers()
1032 addr -= fineibt_caller_size; in cfi_enable_callers()
1053 if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n", in cfi_rand_preamble()
1055 return -EINVAL; in cfi_rand_preamble()
1073 if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n", in cfi_rewrite_preamble()
1075 return -EINVAL; in cfi_rewrite_preamble()
1105 addr -= fineibt_caller_size; in cfi_rand_callers()
1108 hash = -cfi_rehash(hash); in cfi_rand_callers()
1124 addr -= fineibt_caller_size; in cfi_rewrite_callers()
1154 * rewrite them. This disables all CFI. If this succeeds but any of the in __apply_fineibt()
1155 * later stages fails, we're without CFI. in __apply_fineibt()
1177 pr_info("Disabling CFI\n"); in __apply_fineibt()
1190 /* place the FineIBT preamble at func()-16 */ in __apply_fineibt()
1195 /* rewrite the callers to target func()-16 */ in __apply_fineibt()
1204 pr_info("Using FineIBT CFI\n"); in __apply_fineibt()
1212 pr_err("Something went horribly wrong trying to rewrite the CFI implementation.\n"); in __apply_fineibt()
1342 smp->mod = mod; in alternatives_smp_module_add()
1343 smp->name = name; in alternatives_smp_module_add()
1344 smp->locks = locks; in alternatives_smp_module_add()
1345 smp->locks_end = locks_end; in alternatives_smp_module_add()
1346 smp->text = text; in alternatives_smp_module_add()
1347 smp->text_end = text_end; in alternatives_smp_module_add()
1348 DPRINTK(SMP, "locks %p -> %p, text %p -> %p, name %s\n", in alternatives_smp_module_add()
1349 smp->locks, smp->locks_end, in alternatives_smp_module_add()
1350 smp->text, smp->text_end, smp->name); in alternatives_smp_module_add()
1352 list_add_tail(&smp->next, &smp_alt_modules); in alternatives_smp_module_add()
1365 if (mod != item->mod) in alternatives_smp_module_del()
1367 list_del(&item->next); in alternatives_smp_module_del()
1389 alternatives_smp_lock(mod->locks, mod->locks_end, in alternatives_enable_smp()
1390 mod->text, mod->text_end); in alternatives_enable_smp()
1397 * Return 1 if the address range is reserved for SMP-alternatives.
1410 if (mod->text > text_end || mod->text_end < text_start) in alternatives_text_reserved()
1412 for (poff = mod->locks; poff < mod->locks_end; poff++) { in alternatives_text_reserved()
1435 len -= noplen; in add_nops()
1448 BUG_ON(p->len > MAX_PATCH_LEN); in apply_paravirt()
1450 memcpy(insn_buff, p->instr, p->len); in apply_paravirt()
1451 used = paravirt_patch(p->type, insn_buff, (unsigned long)p->instr, p->len); in apply_paravirt()
1453 BUG_ON(used > p->len); in apply_paravirt()
1456 add_nops(insn_buff + used, p->len - used); in apply_paravirt()
1457 text_poke_early(p->instr, insn_buff, p->len); in apply_paravirt()
1465 * Self-test for the INT3 based CALL emulation code.
1489 " .size int3_magic, .-int3_magic\n"
1500 struct pt_regs *regs = args->regs; in int3_exception_notify()
1510 if (regs->ip - INT3_INSN_SIZE != selftest) in int3_exception_notify()
1522 .priority = INT_MAX-1, /* last */ in int3_selftest()
1529 * Basically: int3_magic(&val); but really complicated :-) in int3_selftest()
1560 * instruction and additionally on x86_64 we get a RIP-relative LEA: in alt_reloc_selftest()
1563 * call +0 # 5d5: R_X86_64_PLT32 __alt_reloc_selftest-0x4 in alt_reloc_selftest()
1588 * Don't stop machine check exceptions while patching. in alternative_instructions()
1593 * goes into machine check the others follow quickly, so we don't in alternative_instructions()
1594 * expect a machine check to cause undue problems during to code in alternative_instructions()
1671 * text_poke_early - Update instructions on a live kernel at boot time
1690 * Modules text is marked initially as non-executable, so the in text_poke_early()
1691 * code cannot be running and speculative code-fetches are in text_poke_early()
1716 * temporary page-table mappings that are required for these write operations to
1823 * flush_tlb_mm_range(), which is intended for non-global PTEs. in __text_poke()
1828 * The lock is not really needed, but this allows to avoid open-coding. in __text_poke()
1868 * Loading the previous page-table hierarchy requires a serializing in __text_poke()
1870 * Xen-PV is assumed to serialize execution in a similar manner. in __text_poke()
1896 * text_poke - Update instructions on a live kernel
1919 * text_poke_kgdb - Update instructions on a live kernel by kgdb
1950 s = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(ptr), len - patched); in text_poke_copy_locked()
1959 * text_poke_copy - Copy instructions into (an unused part of) RX memory
1979 * text_poke_set - memset into (an unused part of) RX memory
2000 s = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(ptr), len - patched); in text_poke_set()
2048 if (!raw_atomic_inc_not_zero(&desc->refs)) in try_get_desc()
2059 raw_atomic_dec(&desc->refs); in put_desc()
2064 return _stext + tp->rel_addr; in text_poke_addr()
2072 return -1; in patch_cmp()
2090 * bp_desc with non-zero refcount: in poke_int3_handler()
2105 ip = (void *) regs->ip - INT3_INSN_SIZE; in poke_int3_handler()
2110 if (unlikely(desc->nr_entries > 1)) { in poke_int3_handler()
2111 tp = __inline_bsearch(ip, desc->vec, desc->nr_entries, in poke_int3_handler()
2117 tp = desc->vec; in poke_int3_handler()
2122 ip += tp->len; in poke_int3_handler()
2124 switch (tp->opcode) { in poke_int3_handler()
2137 int3_emulate_call(regs, (long)ip + tp->disp); in poke_int3_handler()
2142 int3_emulate_jmp(regs, (long)ip + tp->disp); in poke_int3_handler()
2146 int3_emulate_jcc(regs, tp->opcode & 0xf, (long)ip, tp->disp); in poke_int3_handler()
2165 * text_poke_bp_batch() -- update instructions on live kernel on SMP
2169 * Modify multi-byte instruction by using int3 breakpoint on SMP.
2174 * - For each entry in the vector:
2175 * - add a int3 trap to the address that will be patched
2176 * - sync cores
2177 * - For each entry in the vector:
2178 * - update all but the first byte of the patched range
2179 * - sync cores
2180 * - For each entry in the vector:
2181 * - replace the first byte (int3) by the first byte of
2183 * - sync cores
2198 * ensure reading a non-zero refcount provides up to date bp_desc data. in text_poke_bp_batch()
2237 if (len - INT3_INSN_SIZE > 0) { in text_poke_bp_batch()
2240 len - INT3_INSN_SIZE); in text_poke_bp_batch()
2250 len - INT3_INSN_SIZE); in text_poke_bp_batch()
2259 * - write INT3 byte in text_poke_bp_batch()
2260 * - IPI-SYNC in text_poke_bp_batch()
2261 * - write instruction tail in text_poke_bp_batch()
2266 * - emit RECORD_TEXT_POKE with the new instruction in text_poke_bp_batch()
2267 * - IPI-SYNC in text_poke_bp_batch()
2268 * - write first byte in text_poke_bp_batch()
2269 * - IPI-SYNC in text_poke_bp_batch()
2326 memcpy((void *)tp->text, opcode+i, len-i); in text_poke_loc_init()
2333 tp->rel_addr = addr - (void *)_stext; in text_poke_loc_init()
2334 tp->len = len; in text_poke_loc_init()
2335 tp->opcode = insn.opcode.bytes[0]; in text_poke_loc_init()
2341 tp->opcode = insn.opcode.bytes[1] - 0x10; in text_poke_loc_init()
2344 switch (tp->opcode) { in text_poke_loc_init()
2353 BUG_ON(tp->text[i] != INT3_INSN_OPCODE); in text_poke_loc_init()
2360 switch (tp->opcode) { in text_poke_loc_init()
2369 tp->disp = insn.immediate.value; in text_poke_loc_init()
2374 case 2: /* NOP2 -- emulate as JMP8+0 */ in text_poke_loc_init()
2376 tp->opcode = JMP8_INSN_OPCODE; in text_poke_loc_init()
2377 tp->disp = 0; in text_poke_loc_init()
2380 case 5: /* NOP5 -- emulate as JMP32+0 */ in text_poke_loc_init()
2382 tp->opcode = JMP32_INSN_OPCODE; in text_poke_loc_init()
2383 tp->disp = 0; in text_poke_loc_init()
2407 tp = &tp_vec[tp_vec_nr - 1]; in tp_order_fail()
2438 * text_poke_bp() -- update instructions on live kernel on SMP