1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * x86 implementation of rethook. Mostly copied from arch/x86/kernel/kprobes/core.c. 4 */ 5 #include <linux/bug.h> 6 #include <linux/rethook.h> 7 #include <linux/kprobes.h> 8 #include <linux/objtool.h> 9 10 #include "kprobes/common.h" 11 12 __visible void arch_rethook_trampoline_callback(struct pt_regs *regs); 13 14 #ifndef ANNOTATE_NOENDBR 15 #define ANNOTATE_NOENDBR 16 #endif 17 18 /* 19 * When a target function returns, this code saves registers and calls 20 * arch_rethook_trampoline_callback(), which calls the rethook handler. 21 */ 22 asm( 23 ".text\n" 24 ".global arch_rethook_trampoline\n" 25 ".type arch_rethook_trampoline, @function\n" 26 "arch_rethook_trampoline:\n" 27 #ifdef CONFIG_X86_64 28 ANNOTATE_NOENDBR /* This is only jumped from ret instruction */ 29 /* Push a fake return address to tell the unwinder it's a rethook. */ 30 " pushq $arch_rethook_trampoline\n" 31 UNWIND_HINT_FUNC 32 " pushq $" __stringify(__KERNEL_DS) "\n" 33 /* Save the 'sp - 16', this will be fixed later. */ 34 " pushq %rsp\n" 35 " pushfq\n" 36 SAVE_REGS_STRING 37 " movq %rsp, %rdi\n" 38 " call arch_rethook_trampoline_callback\n" 39 RESTORE_REGS_STRING 40 /* In the callback function, 'regs->flags' is copied to 'regs->ss'. */ 41 " addq $16, %rsp\n" 42 " popfq\n" 43 #else 44 /* Push a fake return address to tell the unwinder it's a rethook. */ 45 " pushl $arch_rethook_trampoline\n" 46 UNWIND_HINT_FUNC 47 " pushl %ss\n" 48 /* Save the 'sp - 8', this will be fixed later. */ 49 " pushl %esp\n" 50 " pushfl\n" 51 SAVE_REGS_STRING 52 " movl %esp, %eax\n" 53 " call arch_rethook_trampoline_callback\n" 54 RESTORE_REGS_STRING 55 /* In the callback function, 'regs->flags' is copied to 'regs->ss'. */ 56 " addl $8, %esp\n" 57 " popfl\n" 58 #endif 59 ASM_RET 60 ".size arch_rethook_trampoline, .-arch_rethook_trampoline\n" 61 ); 62 NOKPROBE_SYMBOL(arch_rethook_trampoline); 63 64 /* 65 * Called from arch_rethook_trampoline 66 */ 67 __used __visible void arch_rethook_trampoline_callback(struct pt_regs *regs) 68 { 69 unsigned long *frame_pointer; 70 71 /* fixup registers */ 72 regs->cs = __KERNEL_CS; 73 #ifdef CONFIG_X86_32 74 regs->gs = 0; 75 #endif 76 regs->ip = (unsigned long)&arch_rethook_trampoline; 77 regs->orig_ax = ~0UL; 78 regs->sp += 2*sizeof(long); 79 frame_pointer = (long *)(regs + 1); 80 81 /* 82 * The return address at 'frame_pointer' is recovered by the 83 * arch_rethook_fixup_return() which called from this 84 * rethook_trampoline_handler(). 85 */ 86 rethook_trampoline_handler(regs, (unsigned long)frame_pointer); 87 88 /* 89 * Copy FLAGS to 'pt_regs::ss' so that arch_rethook_trapmoline() 90 * can do RET right after POPF. 91 */ 92 *(unsigned long *)®s->ss = regs->flags; 93 } 94 NOKPROBE_SYMBOL(arch_rethook_trampoline_callback); 95 96 /* 97 * arch_rethook_trampoline() skips updating frame pointer. The frame pointer 98 * saved in arch_rethook_trampoline_callback() points to the real caller 99 * function's frame pointer. Thus the arch_rethook_trampoline() doesn't have 100 * a standard stack frame with CONFIG_FRAME_POINTER=y. 101 * Let's mark it non-standard function. Anyway, FP unwinder can correctly 102 * unwind without the hint. 103 */ 104 STACK_FRAME_NON_STANDARD_FP(arch_rethook_trampoline); 105 106 /* This is called from rethook_trampoline_handler(). */ 107 void arch_rethook_fixup_return(struct pt_regs *regs, 108 unsigned long correct_ret_addr) 109 { 110 unsigned long *frame_pointer = (void *)(regs + 1); 111 112 /* Replace fake return address with real one. */ 113 *frame_pointer = correct_ret_addr; 114 } 115 NOKPROBE_SYMBOL(arch_rethook_fixup_return); 116 117 void arch_rethook_prepare(struct rethook_node *rh, struct pt_regs *regs, bool mcount) 118 { 119 unsigned long *stack = (unsigned long *)regs->sp; 120 121 rh->ret_addr = stack[0]; 122 rh->frame = regs->sp; 123 124 /* Replace the return addr with trampoline addr */ 125 stack[0] = (unsigned long) arch_rethook_trampoline; 126 } 127 NOKPROBE_SYMBOL(arch_rethook_prepare); 128