1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */ 235de5b06SAndy Lutomirski #ifndef _ASM_X86_TEXT_PATCHING_H 335de5b06SAndy Lutomirski #define _ASM_X86_TEXT_PATCHING_H 435de5b06SAndy Lutomirski 535de5b06SAndy Lutomirski #include <linux/types.h> 635de5b06SAndy Lutomirski #include <linux/stddef.h> 735de5b06SAndy Lutomirski #include <asm/ptrace.h> 835de5b06SAndy Lutomirski 935de5b06SAndy Lutomirski struct paravirt_patch_site; 1035de5b06SAndy Lutomirski #ifdef CONFIG_PARAVIRT 1135de5b06SAndy Lutomirski void apply_paravirt(struct paravirt_patch_site *start, 1235de5b06SAndy Lutomirski struct paravirt_patch_site *end); 1335de5b06SAndy Lutomirski #else 1435de5b06SAndy Lutomirski static inline void apply_paravirt(struct paravirt_patch_site *start, 1535de5b06SAndy Lutomirski struct paravirt_patch_site *end) 1635de5b06SAndy Lutomirski {} 1735de5b06SAndy Lutomirski #define __parainstructions NULL 1835de5b06SAndy Lutomirski #define __parainstructions_end NULL 1935de5b06SAndy Lutomirski #endif 2035de5b06SAndy Lutomirski 21c0213b0aSDaniel Bristot de Oliveira /* 22c0213b0aSDaniel Bristot de Oliveira * Currently, the max observed size in the kernel code is 23c0213b0aSDaniel Bristot de Oliveira * JUMP_LABEL_NOP_SIZE/RELATIVEJUMP_SIZE, which are 5. 24c0213b0aSDaniel Bristot de Oliveira * Raise it if needed. 25c0213b0aSDaniel Bristot de Oliveira */ 26c0213b0aSDaniel Bristot de Oliveira #define POKE_MAX_OPCODE_SIZE 5 27c0213b0aSDaniel Bristot de Oliveira 280a203df5SNadav Amit extern void text_poke_early(void *addr, const void *opcode, size_t len); 2935de5b06SAndy Lutomirski 3035de5b06SAndy Lutomirski /* 3135de5b06SAndy Lutomirski * Clear and restore the kernel write-protection flag on the local CPU. 3235de5b06SAndy Lutomirski * Allows the kernel to edit read-only pages. 3335de5b06SAndy Lutomirski * Side-effect: any interrupt handler running between save and restore will have 3435de5b06SAndy Lutomirski * the ability to write to read-only pages. 3535de5b06SAndy Lutomirski * 3635de5b06SAndy Lutomirski * Warning: 3735de5b06SAndy Lutomirski * Code patching in the UP case is safe if NMIs and MCE handlers are stopped and 3835de5b06SAndy Lutomirski * no thread can be preempted in the instructions being modified (no iret to an 3935de5b06SAndy Lutomirski * invalid instruction possible) or if the instructions are changed from a 4035de5b06SAndy Lutomirski * consistent state to another consistent state atomically. 4132b1cbe3SMarco Ammon * On the local CPU you need to be protected against NMI or MCE handlers seeing 4232b1cbe3SMarco Ammon * an inconsistent instruction while you patch. 4335de5b06SAndy Lutomirski */ 4435de5b06SAndy Lutomirski extern void *text_poke(void *addr, const void *opcode, size_t len); 455c02ece8SPeter Zijlstra extern void text_poke_sync(void); 46e836673cSNadav Amit extern void *text_poke_kgdb(void *addr, const void *opcode, size_t len); 470e06b403SSong Liu extern void *text_poke_copy(void *addr, const void *opcode, size_t len); 48*aadd1b67SSong Liu extern void *text_poke_set(void *addr, int c, size_t len); 4935de5b06SAndy Lutomirski extern int poke_int3_handler(struct pt_regs *regs); 50c3d6324fSPeter Zijlstra extern void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate); 5118cbc8beSPeter Zijlstra 5218cbc8beSPeter Zijlstra extern void text_poke_queue(void *addr, const void *opcode, size_t len, const void *emulate); 5318cbc8beSPeter Zijlstra extern void text_poke_finish(void); 5418cbc8beSPeter Zijlstra 554b33dadfSPeter Zijlstra #define INT3_INSN_SIZE 1 56c3d6324fSPeter Zijlstra #define INT3_INSN_OPCODE 0xCC 57c3d6324fSPeter Zijlstra 58c43a43e4SPeter Zijlstra #define RET_INSN_SIZE 1 59c43a43e4SPeter Zijlstra #define RET_INSN_OPCODE 0xC3 60c43a43e4SPeter Zijlstra 614b33dadfSPeter Zijlstra #define CALL_INSN_SIZE 5 62c3d6324fSPeter Zijlstra #define CALL_INSN_OPCODE 0xE8 63c3d6324fSPeter Zijlstra 64c3d6324fSPeter Zijlstra #define JMP32_INSN_SIZE 5 65c3d6324fSPeter Zijlstra #define JMP32_INSN_OPCODE 0xE9 66c3d6324fSPeter Zijlstra 67c3d6324fSPeter Zijlstra #define JMP8_INSN_SIZE 2 68c3d6324fSPeter Zijlstra #define JMP8_INSN_OPCODE 0xEB 694b33dadfSPeter Zijlstra 70ab09e95cSPeter Zijlstra #define DISP32_SIZE 4 71ab09e95cSPeter Zijlstra 724979fb53SThomas Gleixner static __always_inline int text_opcode_size(u8 opcode) 73254d2c04SPeter Zijlstra { 74254d2c04SPeter Zijlstra int size = 0; 75254d2c04SPeter Zijlstra 76254d2c04SPeter Zijlstra #define __CASE(insn) \ 77254d2c04SPeter Zijlstra case insn##_INSN_OPCODE: size = insn##_INSN_SIZE; break 78254d2c04SPeter Zijlstra 79254d2c04SPeter Zijlstra switch(opcode) { 80254d2c04SPeter Zijlstra __CASE(INT3); 81c43a43e4SPeter Zijlstra __CASE(RET); 82254d2c04SPeter Zijlstra __CASE(CALL); 83254d2c04SPeter Zijlstra __CASE(JMP32); 84254d2c04SPeter Zijlstra __CASE(JMP8); 85254d2c04SPeter Zijlstra } 86254d2c04SPeter Zijlstra 87254d2c04SPeter Zijlstra #undef __CASE 88254d2c04SPeter Zijlstra 89254d2c04SPeter Zijlstra return size; 90254d2c04SPeter Zijlstra } 91254d2c04SPeter Zijlstra 9267c1d4a2SPeter Zijlstra union text_poke_insn { 9367c1d4a2SPeter Zijlstra u8 text[POKE_MAX_OPCODE_SIZE]; 9467c1d4a2SPeter Zijlstra struct { 9567c1d4a2SPeter Zijlstra u8 opcode; 9667c1d4a2SPeter Zijlstra s32 disp; 9767c1d4a2SPeter Zijlstra } __attribute__((packed)); 9867c1d4a2SPeter Zijlstra }; 9967c1d4a2SPeter Zijlstra 10067c1d4a2SPeter Zijlstra static __always_inline 101ba27d1a8SPeter Zijlstra void __text_gen_insn(void *buf, u8 opcode, const void *addr, const void *dest, int size) 10267c1d4a2SPeter Zijlstra { 103ba27d1a8SPeter Zijlstra union text_poke_insn *insn = buf; 104ba27d1a8SPeter Zijlstra 105ba27d1a8SPeter Zijlstra BUG_ON(size < text_opcode_size(opcode)); 10667c1d4a2SPeter Zijlstra 107bbf92368SPeter Zijlstra /* 108bbf92368SPeter Zijlstra * Hide the addresses to avoid the compiler folding in constants when 109bbf92368SPeter Zijlstra * referencing code, these can mess up annotations like 110bbf92368SPeter Zijlstra * ANNOTATE_NOENDBR. 111bbf92368SPeter Zijlstra */ 112ba27d1a8SPeter Zijlstra OPTIMIZER_HIDE_VAR(insn); 113bbf92368SPeter Zijlstra OPTIMIZER_HIDE_VAR(addr); 114bbf92368SPeter Zijlstra OPTIMIZER_HIDE_VAR(dest); 115bbf92368SPeter Zijlstra 116ba27d1a8SPeter Zijlstra insn->opcode = opcode; 11767c1d4a2SPeter Zijlstra 11867c1d4a2SPeter Zijlstra if (size > 1) { 119ba27d1a8SPeter Zijlstra insn->disp = (long)dest - (long)(addr + size); 12067c1d4a2SPeter Zijlstra if (size == 2) { 12167c1d4a2SPeter Zijlstra /* 122bbf92368SPeter Zijlstra * Ensure that for JMP8 the displacement 12367c1d4a2SPeter Zijlstra * actually fits the signed byte. 12467c1d4a2SPeter Zijlstra */ 125ba27d1a8SPeter Zijlstra BUG_ON((insn->disp >> 31) != (insn->disp >> 7)); 126ba27d1a8SPeter Zijlstra } 12767c1d4a2SPeter Zijlstra } 12867c1d4a2SPeter Zijlstra } 12967c1d4a2SPeter Zijlstra 130ba27d1a8SPeter Zijlstra static __always_inline 13167c1d4a2SPeter Zijlstra void *text_gen_insn(u8 opcode, const void *addr, const void *dest) 13267c1d4a2SPeter Zijlstra { 13367c1d4a2SPeter Zijlstra static union text_poke_insn insn; /* per instance */ 134ba27d1a8SPeter Zijlstra __text_gen_insn(&insn, opcode, addr, dest, text_opcode_size(opcode)); 13567c1d4a2SPeter Zijlstra return &insn.text; 13667c1d4a2SPeter Zijlstra } 137254d2c04SPeter Zijlstra 138254d2c04SPeter Zijlstra extern int after_bootmem; 139254d2c04SPeter Zijlstra extern __ro_after_init struct mm_struct *poking_mm; 140254d2c04SPeter Zijlstra extern __ro_after_init unsigned long poking_addr; 141254d2c04SPeter Zijlstra 142254d2c04SPeter Zijlstra #ifndef CONFIG_UML_X86 1434979fb53SThomas Gleixner static __always_inline 1444979fb53SThomas Gleixner void int3_emulate_jmp(struct pt_regs *regs, unsigned long ip) 145254d2c04SPeter Zijlstra { 146254d2c04SPeter Zijlstra regs->ip = ip; 147254d2c04SPeter Zijlstra } 148254d2c04SPeter Zijlstra 1494979fb53SThomas Gleixner static __always_inline 1504979fb53SThomas Gleixner void int3_emulate_push(struct pt_regs *regs, unsigned long val) 1514b33dadfSPeter Zijlstra { 1524b33dadfSPeter Zijlstra /* 1534b33dadfSPeter Zijlstra * The int3 handler in entry_64.S adds a gap between the 1544b33dadfSPeter Zijlstra * stack where the break point happened, and the saving of 1554b33dadfSPeter Zijlstra * pt_regs. We can extend the original stack because of 1564b33dadfSPeter Zijlstra * this gap. See the idtentry macro's create_gap option. 1578f4a4160SPeter Zijlstra * 1588f4a4160SPeter Zijlstra * Similarly entry_32.S will have a gap on the stack for (any) hardware 1598f4a4160SPeter Zijlstra * exception and pt_regs; see FIXUP_FRAME. 1604b33dadfSPeter Zijlstra */ 1614b33dadfSPeter Zijlstra regs->sp -= sizeof(unsigned long); 1624b33dadfSPeter Zijlstra *(unsigned long *)regs->sp = val; 1634b33dadfSPeter Zijlstra } 1644b33dadfSPeter Zijlstra 1654979fb53SThomas Gleixner static __always_inline 166c43a43e4SPeter Zijlstra unsigned long int3_emulate_pop(struct pt_regs *regs) 167c43a43e4SPeter Zijlstra { 168c43a43e4SPeter Zijlstra unsigned long val = *(unsigned long *)regs->sp; 169c43a43e4SPeter Zijlstra regs->sp += sizeof(unsigned long); 170c43a43e4SPeter Zijlstra return val; 171c43a43e4SPeter Zijlstra } 172c43a43e4SPeter Zijlstra 173c43a43e4SPeter Zijlstra static __always_inline 1744979fb53SThomas Gleixner void int3_emulate_call(struct pt_regs *regs, unsigned long func) 1754b33dadfSPeter Zijlstra { 1764b33dadfSPeter Zijlstra int3_emulate_push(regs, regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE); 1774b33dadfSPeter Zijlstra int3_emulate_jmp(regs, func); 1784b33dadfSPeter Zijlstra } 179c43a43e4SPeter Zijlstra 180c43a43e4SPeter Zijlstra static __always_inline 181c43a43e4SPeter Zijlstra void int3_emulate_ret(struct pt_regs *regs) 182c43a43e4SPeter Zijlstra { 183c43a43e4SPeter Zijlstra unsigned long ip = int3_emulate_pop(regs); 184c43a43e4SPeter Zijlstra int3_emulate_jmp(regs, ip); 185c43a43e4SPeter Zijlstra } 186693713cbSSteven Rostedt (VMware) #endif /* !CONFIG_UML_X86 */ 1874b33dadfSPeter Zijlstra 18835de5b06SAndy Lutomirski #endif /* _ASM_X86_TEXT_PATCHING_H */ 189