1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Copyright (C) 2014 Steven Rostedt, Red Hat Inc 4 */ 5 6#include <linux/linkage.h> 7#include <asm/ptrace.h> 8#include <asm/ftrace.h> 9#include <asm/export.h> 10#include <asm/nospec-branch.h> 11#include <asm/unwind_hints.h> 12 13 .code64 14 .section .entry.text, "ax" 15 16#ifdef CC_USING_FENTRY 17# define function_hook __fentry__ 18EXPORT_SYMBOL(__fentry__) 19#else 20# define function_hook mcount 21EXPORT_SYMBOL(mcount) 22#endif 23 24#ifdef CONFIG_FRAME_POINTER 25# ifdef CC_USING_FENTRY 26/* Save parent and function stack frames (rip and rbp) */ 27# define MCOUNT_FRAME_SIZE (8+16*2) 28# else 29/* Save just function stack frame (rip and rbp) */ 30# define MCOUNT_FRAME_SIZE (8+16) 31# endif 32#else 33/* No need to save a stack frame */ 34# define MCOUNT_FRAME_SIZE 0 35#endif /* CONFIG_FRAME_POINTER */ 36 37/* Size of stack used to save mcount regs in save_mcount_regs */ 38#define MCOUNT_REG_SIZE (SS+8 + MCOUNT_FRAME_SIZE) 39 40/* 41 * gcc -pg option adds a call to 'mcount' in most functions. 42 * When -mfentry is used, the call is to 'fentry' and not 'mcount' 43 * and is done before the function's stack frame is set up. 44 * They both require a set of regs to be saved before calling 45 * any C code and restored before returning back to the function. 46 * 47 * On boot up, all these calls are converted into nops. When tracing 48 * is enabled, the call can jump to either ftrace_caller or 49 * ftrace_regs_caller. Callbacks (tracing functions) that require 50 * ftrace_regs_caller (like kprobes) need to have pt_regs passed to 51 * it. For this reason, the size of the pt_regs structure will be 52 * allocated on the stack and the required mcount registers will 53 * be saved in the locations that pt_regs has them in. 54 */ 55 56/* 57 * @added: the amount of stack added before calling this 58 * 59 * After this is called, the following registers contain: 60 * 61 * %rdi - holds the address that called the trampoline 62 * %rsi - holds the parent function (traced function's return address) 63 * %rdx - holds the original %rbp 64 */ 65.macro save_mcount_regs added=0 66 67#ifdef CONFIG_FRAME_POINTER 68 /* Save the original rbp */ 69 pushq %rbp 70 71 /* 72 * Stack traces will stop at the ftrace trampoline if the frame pointer 73 * is not set up properly. If fentry is used, we need to save a frame 74 * pointer for the parent as well as the function traced, because the 75 * fentry is called before the stack frame is set up, where as mcount 76 * is called afterward. 77 */ 78#ifdef CC_USING_FENTRY 79 /* Save the parent pointer (skip orig rbp and our return address) */ 80 pushq \added+8*2(%rsp) 81 pushq %rbp 82 movq %rsp, %rbp 83 /* Save the return address (now skip orig rbp, rbp and parent) */ 84 pushq \added+8*3(%rsp) 85#else 86 /* Can't assume that rip is before this (unless added was zero) */ 87 pushq \added+8(%rsp) 88#endif 89 pushq %rbp 90 movq %rsp, %rbp 91#endif /* CONFIG_FRAME_POINTER */ 92 93 /* 94 * We add enough stack to save all regs. 95 */ 96 subq $(MCOUNT_REG_SIZE - MCOUNT_FRAME_SIZE), %rsp 97 movq %rax, RAX(%rsp) 98 movq %rcx, RCX(%rsp) 99 movq %rdx, RDX(%rsp) 100 movq %rsi, RSI(%rsp) 101 movq %rdi, RDI(%rsp) 102 movq %r8, R8(%rsp) 103 movq %r9, R9(%rsp) 104 /* 105 * Save the original RBP. Even though the mcount ABI does not 106 * require this, it helps out callers. 107 */ 108#ifdef CONFIG_FRAME_POINTER 109 movq MCOUNT_REG_SIZE-8(%rsp), %rdx 110#else 111 movq %rbp, %rdx 112#endif 113 movq %rdx, RBP(%rsp) 114 115 /* Copy the parent address into %rsi (second parameter) */ 116#ifdef CC_USING_FENTRY 117 movq MCOUNT_REG_SIZE+8+\added(%rsp), %rsi 118#else 119 /* %rdx contains original %rbp */ 120 movq 8(%rdx), %rsi 121#endif 122 123 /* Move RIP to its proper location */ 124 movq MCOUNT_REG_SIZE+\added(%rsp), %rdi 125 movq %rdi, RIP(%rsp) 126 127 /* 128 * Now %rdi (the first parameter) has the return address of 129 * where ftrace_call returns. But the callbacks expect the 130 * address of the call itself. 131 */ 132 subq $MCOUNT_INSN_SIZE, %rdi 133 .endm 134 135.macro restore_mcount_regs 136 movq R9(%rsp), %r9 137 movq R8(%rsp), %r8 138 movq RDI(%rsp), %rdi 139 movq RSI(%rsp), %rsi 140 movq RDX(%rsp), %rdx 141 movq RCX(%rsp), %rcx 142 movq RAX(%rsp), %rax 143 144 /* ftrace_regs_caller can modify %rbp */ 145 movq RBP(%rsp), %rbp 146 147 addq $MCOUNT_REG_SIZE, %rsp 148 149 .endm 150 151#ifdef CONFIG_DYNAMIC_FTRACE 152 153ENTRY(function_hook) 154 retq 155ENDPROC(function_hook) 156 157ENTRY(ftrace_caller) 158 /* save_mcount_regs fills in first two parameters */ 159 save_mcount_regs 160 161GLOBAL(ftrace_caller_op_ptr) 162 /* Load the ftrace_ops into the 3rd parameter */ 163 movq function_trace_op(%rip), %rdx 164 165 /* regs go into 4th parameter (but make it NULL) */ 166 movq $0, %rcx 167 168GLOBAL(ftrace_call) 169 call ftrace_stub 170 171 restore_mcount_regs 172 173 /* 174 * The copied trampoline must call ftrace_epilogue as it 175 * still may need to call the function graph tracer. 176 * 177 * The code up to this label is copied into trampolines so 178 * think twice before adding any new code or changing the 179 * layout here. 180 */ 181GLOBAL(ftrace_epilogue) 182 183#ifdef CONFIG_FUNCTION_GRAPH_TRACER 184GLOBAL(ftrace_graph_call) 185 jmp ftrace_stub 186#endif 187 188/* This is weak to keep gas from relaxing the jumps */ 189WEAK(ftrace_stub) 190 retq 191ENDPROC(ftrace_caller) 192 193ENTRY(ftrace_regs_caller) 194 /* Save the current flags before any operations that can change them */ 195 pushfq 196 197 /* added 8 bytes to save flags */ 198 save_mcount_regs 8 199 /* save_mcount_regs fills in first two parameters */ 200 201GLOBAL(ftrace_regs_caller_op_ptr) 202 /* Load the ftrace_ops into the 3rd parameter */ 203 movq function_trace_op(%rip), %rdx 204 205 /* Save the rest of pt_regs */ 206 movq %r15, R15(%rsp) 207 movq %r14, R14(%rsp) 208 movq %r13, R13(%rsp) 209 movq %r12, R12(%rsp) 210 movq %r11, R11(%rsp) 211 movq %r10, R10(%rsp) 212 movq %rbx, RBX(%rsp) 213 /* Copy saved flags */ 214 movq MCOUNT_REG_SIZE(%rsp), %rcx 215 movq %rcx, EFLAGS(%rsp) 216 /* Kernel segments */ 217 movq $__KERNEL_DS, %rcx 218 movq %rcx, SS(%rsp) 219 movq $__KERNEL_CS, %rcx 220 movq %rcx, CS(%rsp) 221 /* Stack - skipping return address and flags */ 222 leaq MCOUNT_REG_SIZE+8*2(%rsp), %rcx 223 movq %rcx, RSP(%rsp) 224 225 /* regs go into 4th parameter */ 226 leaq (%rsp), %rcx 227 228GLOBAL(ftrace_regs_call) 229 call ftrace_stub 230 231 /* Copy flags back to SS, to restore them */ 232 movq EFLAGS(%rsp), %rax 233 movq %rax, MCOUNT_REG_SIZE(%rsp) 234 235 /* Handlers can change the RIP */ 236 movq RIP(%rsp), %rax 237 movq %rax, MCOUNT_REG_SIZE+8(%rsp) 238 239 /* restore the rest of pt_regs */ 240 movq R15(%rsp), %r15 241 movq R14(%rsp), %r14 242 movq R13(%rsp), %r13 243 movq R12(%rsp), %r12 244 movq R10(%rsp), %r10 245 movq RBX(%rsp), %rbx 246 247 restore_mcount_regs 248 249 /* Restore flags */ 250 popfq 251 252 /* 253 * As this jmp to ftrace_epilogue can be a short jump 254 * it must not be copied into the trampoline. 255 * The trampoline will add the code to jump 256 * to the return. 257 */ 258GLOBAL(ftrace_regs_caller_end) 259 260 jmp ftrace_epilogue 261 262ENDPROC(ftrace_regs_caller) 263 264 265#else /* ! CONFIG_DYNAMIC_FTRACE */ 266 267ENTRY(function_hook) 268 cmpq $ftrace_stub, ftrace_trace_function 269 jnz trace 270 271fgraph_trace: 272#ifdef CONFIG_FUNCTION_GRAPH_TRACER 273 cmpq $ftrace_stub, ftrace_graph_return 274 jnz ftrace_graph_caller 275 276 cmpq $ftrace_graph_entry_stub, ftrace_graph_entry 277 jnz ftrace_graph_caller 278#endif 279 280GLOBAL(ftrace_stub) 281 retq 282 283trace: 284 /* save_mcount_regs fills in first two parameters */ 285 save_mcount_regs 286 287 /* 288 * When DYNAMIC_FTRACE is not defined, ARCH_SUPPORTS_FTRACE_OPS is not 289 * set (see include/asm/ftrace.h and include/linux/ftrace.h). Only the 290 * ip and parent ip are used and the list function is called when 291 * function tracing is enabled. 292 */ 293 movq ftrace_trace_function, %r8 294 CALL_NOSPEC %r8 295 restore_mcount_regs 296 297 jmp fgraph_trace 298ENDPROC(function_hook) 299#endif /* CONFIG_DYNAMIC_FTRACE */ 300 301#ifdef CONFIG_FUNCTION_GRAPH_TRACER 302ENTRY(ftrace_graph_caller) 303 /* Saves rbp into %rdx and fills first parameter */ 304 save_mcount_regs 305 306#ifdef CC_USING_FENTRY 307 leaq MCOUNT_REG_SIZE+8(%rsp), %rsi 308 movq $0, %rdx /* No framepointers needed */ 309#else 310 /* Save address of the return address of traced function */ 311 leaq 8(%rdx), %rsi 312 /* ftrace does sanity checks against frame pointers */ 313 movq (%rdx), %rdx 314#endif 315 call prepare_ftrace_return 316 317 restore_mcount_regs 318 319 retq 320ENDPROC(ftrace_graph_caller) 321 322ENTRY(return_to_handler) 323 UNWIND_HINT_EMPTY 324 subq $24, %rsp 325 326 /* Save the return values */ 327 movq %rax, (%rsp) 328 movq %rdx, 8(%rsp) 329 movq %rbp, %rdi 330 331 call ftrace_return_to_handler 332 333 movq %rax, %rdi 334 movq 8(%rsp), %rdx 335 movq (%rsp), %rax 336 addq $24, %rsp 337 JMP_NOSPEC %rdi 338END(return_to_handler) 339#endif 340