1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Copyright (C) 2014 Steven Rostedt, Red Hat Inc 4 */ 5 6#include <linux/linkage.h> 7#include <linux/cfi_types.h> 8#include <asm/ptrace.h> 9#include <asm/ftrace.h> 10#include <asm/export.h> 11#include <asm/nospec-branch.h> 12#include <asm/unwind_hints.h> 13#include <asm/frame.h> 14 15 .code64 16 .section .text, "ax" 17 18#ifdef CONFIG_FRAME_POINTER 19/* Save parent and function stack frames (rip and rbp) */ 20# define MCOUNT_FRAME_SIZE (8+16*2) 21#else 22/* No need to save a stack frame */ 23# define MCOUNT_FRAME_SIZE 0 24#endif /* CONFIG_FRAME_POINTER */ 25 26/* Size of stack used to save mcount regs in save_mcount_regs */ 27#define MCOUNT_REG_SIZE (FRAME_SIZE + MCOUNT_FRAME_SIZE) 28 29/* 30 * gcc -pg option adds a call to 'mcount' in most functions. 31 * When -mfentry is used, the call is to 'fentry' and not 'mcount' 32 * and is done before the function's stack frame is set up. 33 * They both require a set of regs to be saved before calling 34 * any C code and restored before returning back to the function. 35 * 36 * On boot up, all these calls are converted into nops. When tracing 37 * is enabled, the call can jump to either ftrace_caller or 38 * ftrace_regs_caller. Callbacks (tracing functions) that require 39 * ftrace_regs_caller (like kprobes) need to have pt_regs passed to 40 * it. For this reason, the size of the pt_regs structure will be 41 * allocated on the stack and the required mcount registers will 42 * be saved in the locations that pt_regs has them in. 43 */ 44 45/* 46 * @added: the amount of stack added before calling this 47 * 48 * After this is called, the following registers contain: 49 * 50 * %rdi - holds the address that called the trampoline 51 * %rsi - holds the parent function (traced function's return address) 52 * %rdx - holds the original %rbp 53 */ 54.macro save_mcount_regs added=0 55 56#ifdef CONFIG_FRAME_POINTER 57 /* Save the original rbp */ 58 pushq %rbp 59 60 /* 61 * Stack traces will stop at the ftrace trampoline if the frame pointer 62 * is not set up properly. If fentry is used, we need to save a frame 63 * pointer for the parent as well as the function traced, because the 64 * fentry is called before the stack frame is set up, where as mcount 65 * is called afterward. 66 */ 67 68 /* Save the parent pointer (skip orig rbp and our return address) */ 69 pushq \added+8*2(%rsp) 70 pushq %rbp 71 movq %rsp, %rbp 72 /* Save the return address (now skip orig rbp, rbp and parent) */ 73 pushq \added+8*3(%rsp) 74 pushq %rbp 75 movq %rsp, %rbp 76#endif /* CONFIG_FRAME_POINTER */ 77 78 /* 79 * We add enough stack to save all regs. 80 */ 81 subq $(FRAME_SIZE), %rsp 82 movq %rax, RAX(%rsp) 83 movq %rcx, RCX(%rsp) 84 movq %rdx, RDX(%rsp) 85 movq %rsi, RSI(%rsp) 86 movq %rdi, RDI(%rsp) 87 movq %r8, R8(%rsp) 88 movq %r9, R9(%rsp) 89 movq $0, ORIG_RAX(%rsp) 90 /* 91 * Save the original RBP. Even though the mcount ABI does not 92 * require this, it helps out callers. 93 */ 94#ifdef CONFIG_FRAME_POINTER 95 movq MCOUNT_REG_SIZE-8(%rsp), %rdx 96#else 97 movq %rbp, %rdx 98#endif 99 movq %rdx, RBP(%rsp) 100 101 /* Copy the parent address into %rsi (second parameter) */ 102 movq MCOUNT_REG_SIZE+8+\added(%rsp), %rsi 103 104 /* Move RIP to its proper location */ 105 movq MCOUNT_REG_SIZE+\added(%rsp), %rdi 106 movq %rdi, RIP(%rsp) 107 108 /* 109 * Now %rdi (the first parameter) has the return address of 110 * where ftrace_call returns. But the callbacks expect the 111 * address of the call itself. 112 */ 113 subq $MCOUNT_INSN_SIZE, %rdi 114 .endm 115 116.macro restore_mcount_regs save=0 117 118 /* ftrace_regs_caller or frame pointers require this */ 119 movq RBP(%rsp), %rbp 120 121 movq R9(%rsp), %r9 122 movq R8(%rsp), %r8 123 movq RDI(%rsp), %rdi 124 movq RSI(%rsp), %rsi 125 movq RDX(%rsp), %rdx 126 movq RCX(%rsp), %rcx 127 movq RAX(%rsp), %rax 128 129 addq $MCOUNT_REG_SIZE-\save, %rsp 130 131 .endm 132 133SYM_TYPED_FUNC_START(ftrace_stub) 134 RET 135SYM_FUNC_END(ftrace_stub) 136 137SYM_TYPED_FUNC_START(ftrace_stub_graph) 138 RET 139SYM_FUNC_END(ftrace_stub_graph) 140 141#ifdef CONFIG_DYNAMIC_FTRACE 142 143SYM_FUNC_START(__fentry__) 144 RET 145SYM_FUNC_END(__fentry__) 146EXPORT_SYMBOL(__fentry__) 147 148SYM_FUNC_START(ftrace_caller) 149 /* save_mcount_regs fills in first two parameters */ 150 save_mcount_regs 151 152 /* Stack - skipping return address of ftrace_caller */ 153 leaq MCOUNT_REG_SIZE+8(%rsp), %rcx 154 movq %rcx, RSP(%rsp) 155 156SYM_INNER_LABEL(ftrace_caller_op_ptr, SYM_L_GLOBAL) 157 ANNOTATE_NOENDBR 158 /* Load the ftrace_ops into the 3rd parameter */ 159 movq function_trace_op(%rip), %rdx 160 161 /* regs go into 4th parameter */ 162 leaq (%rsp), %rcx 163 164 /* Only ops with REGS flag set should have CS register set */ 165 movq $0, CS(%rsp) 166 167SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) 168 ANNOTATE_NOENDBR 169 call ftrace_stub 170 171 /* Handlers can change the RIP */ 172 movq RIP(%rsp), %rax 173 movq %rax, MCOUNT_REG_SIZE(%rsp) 174 175 restore_mcount_regs 176 177 /* 178 * The code up to this label is copied into trampolines so 179 * think twice before adding any new code or changing the 180 * layout here. 181 */ 182SYM_INNER_LABEL(ftrace_caller_end, SYM_L_GLOBAL) 183 ANNOTATE_NOENDBR 184 RET 185SYM_FUNC_END(ftrace_caller); 186STACK_FRAME_NON_STANDARD_FP(ftrace_caller) 187 188SYM_FUNC_START(ftrace_regs_caller) 189 /* Save the current flags before any operations that can change them */ 190 pushfq 191 192 /* added 8 bytes to save flags */ 193 save_mcount_regs 8 194 /* save_mcount_regs fills in first two parameters */ 195 196SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL) 197 ANNOTATE_NOENDBR 198 /* Load the ftrace_ops into the 3rd parameter */ 199 movq function_trace_op(%rip), %rdx 200 201 /* Save the rest of pt_regs */ 202 movq %r15, R15(%rsp) 203 movq %r14, R14(%rsp) 204 movq %r13, R13(%rsp) 205 movq %r12, R12(%rsp) 206 movq %r11, R11(%rsp) 207 movq %r10, R10(%rsp) 208 movq %rbx, RBX(%rsp) 209 /* Copy saved flags */ 210 movq MCOUNT_REG_SIZE(%rsp), %rcx 211 movq %rcx, EFLAGS(%rsp) 212 /* Kernel segments */ 213 movq $__KERNEL_DS, %rcx 214 movq %rcx, SS(%rsp) 215 movq $__KERNEL_CS, %rcx 216 movq %rcx, CS(%rsp) 217 /* Stack - skipping return address and flags */ 218 leaq MCOUNT_REG_SIZE+8*2(%rsp), %rcx 219 movq %rcx, RSP(%rsp) 220 221 ENCODE_FRAME_POINTER 222 223 /* regs go into 4th parameter */ 224 leaq (%rsp), %rcx 225 226SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) 227 ANNOTATE_NOENDBR 228 call ftrace_stub 229 230 /* Copy flags back to SS, to restore them */ 231 movq EFLAGS(%rsp), %rax 232 movq %rax, MCOUNT_REG_SIZE(%rsp) 233 234 /* Handlers can change the RIP */ 235 movq RIP(%rsp), %rax 236 movq %rax, MCOUNT_REG_SIZE+8(%rsp) 237 238 /* restore the rest of pt_regs */ 239 movq R15(%rsp), %r15 240 movq R14(%rsp), %r14 241 movq R13(%rsp), %r13 242 movq R12(%rsp), %r12 243 movq R10(%rsp), %r10 244 movq RBX(%rsp), %rbx 245 246 movq ORIG_RAX(%rsp), %rax 247 movq %rax, MCOUNT_REG_SIZE-8(%rsp) 248 249 /* 250 * If ORIG_RAX is anything but zero, make this a call to that. 251 * See arch_ftrace_set_direct_caller(). 252 */ 253 testq %rax, %rax 254SYM_INNER_LABEL(ftrace_regs_caller_jmp, SYM_L_GLOBAL) 255 ANNOTATE_NOENDBR 256 jnz 1f 257 258 restore_mcount_regs 259 /* Restore flags */ 260 popfq 261 262 /* 263 * The trampoline will add the return. 264 */ 265SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL) 266 ANNOTATE_NOENDBR 267 RET 268 269 /* Swap the flags with orig_rax */ 2701: movq MCOUNT_REG_SIZE(%rsp), %rdi 271 movq %rdi, MCOUNT_REG_SIZE-8(%rsp) 272 movq %rax, MCOUNT_REG_SIZE(%rsp) 273 274 restore_mcount_regs 8 275 /* Restore flags */ 276 popfq 277 UNWIND_HINT_FUNC 278 RET 279 280SYM_FUNC_END(ftrace_regs_caller) 281STACK_FRAME_NON_STANDARD_FP(ftrace_regs_caller) 282 283 284#else /* ! CONFIG_DYNAMIC_FTRACE */ 285 286SYM_FUNC_START(__fentry__) 287 cmpq $ftrace_stub, ftrace_trace_function 288 jnz trace 289 RET 290 291trace: 292 /* save_mcount_regs fills in first two parameters */ 293 save_mcount_regs 294 295 /* 296 * When DYNAMIC_FTRACE is not defined, ARCH_SUPPORTS_FTRACE_OPS is not 297 * set (see include/asm/ftrace.h and include/linux/ftrace.h). Only the 298 * ip and parent ip are used and the list function is called when 299 * function tracing is enabled. 300 */ 301 movq ftrace_trace_function, %r8 302 CALL_NOSPEC r8 303 restore_mcount_regs 304 305 jmp ftrace_stub 306SYM_FUNC_END(__fentry__) 307EXPORT_SYMBOL(__fentry__) 308STACK_FRAME_NON_STANDARD_FP(__fentry__) 309 310#endif /* CONFIG_DYNAMIC_FTRACE */ 311 312#ifdef CONFIG_FUNCTION_GRAPH_TRACER 313SYM_CODE_START(return_to_handler) 314 UNWIND_HINT_EMPTY 315 ANNOTATE_NOENDBR 316 subq $16, %rsp 317 318 /* Save the return values */ 319 movq %rax, (%rsp) 320 movq %rdx, 8(%rsp) 321 movq %rbp, %rdi 322 323 call ftrace_return_to_handler 324 325 movq %rax, %rdi 326 movq 8(%rsp), %rdx 327 movq (%rsp), %rax 328 329 addq $16, %rsp 330 /* 331 * Jump back to the old return address. This cannot be JMP_NOSPEC rdi 332 * since IBT would demand that contain ENDBR, which simply isn't so for 333 * return addresses. Use a retpoline here to keep the RSB balanced. 334 */ 335 ANNOTATE_INTRA_FUNCTION_CALL 336 call .Ldo_rop 337 int3 338.Ldo_rop: 339 mov %rdi, (%rsp) 340 RET 341SYM_CODE_END(return_to_handler) 342#endif 343