1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * arch/arm64/kernel/entry-ftrace.S 4 * 5 * Copyright (C) 2013 Linaro Limited 6 * Author: AKASHI Takahiro <takahiro.akashi@linaro.org> 7 */ 8 9#include <linux/linkage.h> 10#include <linux/cfi_types.h> 11#include <asm/asm-offsets.h> 12#include <asm/assembler.h> 13#include <asm/ftrace.h> 14#include <asm/insn.h> 15 16#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS 17/* 18 * Due to -fpatchable-function-entry=2, the compiler has placed two NOPs before 19 * the regular function prologue. For an enabled callsite, ftrace_init_nop() and 20 * ftrace_make_call() have patched those NOPs to: 21 * 22 * MOV X9, LR 23 * BL ftrace_caller 24 * 25 * Each instrumented function follows the AAPCS, so here x0-x8 and x18-x30 are 26 * live (x18 holds the Shadow Call Stack pointer), and x9-x17 are safe to 27 * clobber. 28 * 29 * We save the callsite's context into a struct ftrace_regs before invoking any 30 * ftrace callbacks. So that we can get a sensible backtrace, we create frame 31 * records for the callsite and the ftrace entry assembly. This is not 32 * sufficient for reliable stacktrace: until we create the callsite stack 33 * record, its caller is missing from the LR and existing chain of frame 34 * records. 35 */ 36SYM_CODE_START(ftrace_caller) 37 bti c 38 39 /* Save original SP */ 40 mov x10, sp 41 42 /* Make room for ftrace regs, plus two frame records */ 43 sub sp, sp, #(FREGS_SIZE + 32) 44 45 /* Save function arguments */ 46 stp x0, x1, [sp, #FREGS_X0] 47 stp x2, x3, [sp, #FREGS_X2] 48 stp x4, x5, [sp, #FREGS_X4] 49 stp x6, x7, [sp, #FREGS_X6] 50 str x8, [sp, #FREGS_X8] 51 52 /* Save the callsite's FP, LR, SP */ 53 str x29, [sp, #FREGS_FP] 54 str x9, [sp, #FREGS_LR] 55 str x10, [sp, #FREGS_SP] 56 57 /* Save the PC after the ftrace callsite */ 58 str x30, [sp, #FREGS_PC] 59 60 /* Create a frame record for the callsite above the ftrace regs */ 61 stp x29, x9, [sp, #FREGS_SIZE + 16] 62 add x29, sp, #FREGS_SIZE + 16 63 64 /* Create our frame record above the ftrace regs */ 65 stp x29, x30, [sp, #FREGS_SIZE] 66 add x29, sp, #FREGS_SIZE 67 68 /* Prepare arguments for the the tracer func */ 69 sub x0, x30, #AARCH64_INSN_SIZE // ip (callsite's BL insn) 70 mov x1, x9 // parent_ip (callsite's LR) 71 mov x3, sp // regs 72 73#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS 74 /* 75 * The literal pointer to the ops is at an 8-byte aligned boundary 76 * which is either 12 or 16 bytes before the BL instruction in the call 77 * site. See ftrace_call_adjust() for details. 78 * 79 * Therefore here the LR points at `literal + 16` or `literal + 20`, 80 * and we can find the address of the literal in either case by 81 * aligning to an 8-byte boundary and subtracting 16. We do the 82 * alignment first as this allows us to fold the subtraction into the 83 * LDR. 84 */ 85 bic x2, x30, 0x7 86 ldr x2, [x2, #-16] // op 87 88 ldr x4, [x2, #FTRACE_OPS_FUNC] // op->func 89 blr x4 // op->func(ip, parent_ip, op, regs) 90 91#else 92 ldr_l x2, function_trace_op // op 93 94SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) 95 bl ftrace_stub // func(ip, parent_ip, op, regs) 96#endif 97 98/* 99 * At the callsite x0-x8 and x19-x30 were live. Any C code will have preserved 100 * x19-x29 per the AAPCS, and we created frame records upon entry, so we need 101 * to restore x0-x8, x29, and x30. 102 */ 103 /* Restore function arguments */ 104 ldp x0, x1, [sp, #FREGS_X0] 105 ldp x2, x3, [sp, #FREGS_X2] 106 ldp x4, x5, [sp, #FREGS_X4] 107 ldp x6, x7, [sp, #FREGS_X6] 108 ldr x8, [sp, #FREGS_X8] 109 110 /* Restore the callsite's FP, LR, PC */ 111 ldr x29, [sp, #FREGS_FP] 112 ldr x30, [sp, #FREGS_LR] 113 ldr x9, [sp, #FREGS_PC] 114 115 /* Restore the callsite's SP */ 116 add sp, sp, #FREGS_SIZE + 32 117 118 ret x9 119SYM_CODE_END(ftrace_caller) 120 121#else /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */ 122 123/* 124 * Gcc with -pg will put the following code in the beginning of each function: 125 * mov x0, x30 126 * bl _mcount 127 * [function's body ...] 128 * "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic 129 * ftrace is enabled. 130 * 131 * Please note that x0 as an argument will not be used here because we can 132 * get lr(x30) of instrumented function at any time by winding up call stack 133 * as long as the kernel is compiled without -fomit-frame-pointer. 134 * (or CONFIG_FRAME_POINTER, this is forced on arm64) 135 * 136 * stack layout after mcount_enter in _mcount(): 137 * 138 * current sp/fp => 0:+-----+ 139 * in _mcount() | x29 | -> instrumented function's fp 140 * +-----+ 141 * | x30 | -> _mcount()'s lr (= instrumented function's pc) 142 * old sp => +16:+-----+ 143 * when instrumented | | 144 * function calls | ... | 145 * _mcount() | | 146 * | | 147 * instrumented => +xx:+-----+ 148 * function's fp | x29 | -> parent's fp 149 * +-----+ 150 * | x30 | -> instrumented function's lr (= parent's pc) 151 * +-----+ 152 * | ... | 153 */ 154 155 .macro mcount_enter 156 stp x29, x30, [sp, #-16]! 157 mov x29, sp 158 .endm 159 160 .macro mcount_exit 161 ldp x29, x30, [sp], #16 162 ret 163 .endm 164 165 .macro mcount_adjust_addr rd, rn 166 sub \rd, \rn, #AARCH64_INSN_SIZE 167 .endm 168 169 /* for instrumented function's parent */ 170 .macro mcount_get_parent_fp reg 171 ldr \reg, [x29] 172 ldr \reg, [\reg] 173 .endm 174 175 /* for instrumented function */ 176 .macro mcount_get_pc0 reg 177 mcount_adjust_addr \reg, x30 178 .endm 179 180 .macro mcount_get_pc reg 181 ldr \reg, [x29, #8] 182 mcount_adjust_addr \reg, \reg 183 .endm 184 185 .macro mcount_get_lr reg 186 ldr \reg, [x29] 187 ldr \reg, [\reg, #8] 188 .endm 189 190 .macro mcount_get_lr_addr reg 191 ldr \reg, [x29] 192 add \reg, \reg, #8 193 .endm 194 195/* 196 * _mcount() is used to build the kernel with -pg option, but all the branch 197 * instructions to _mcount() are replaced to NOP initially at kernel start up, 198 * and later on, NOP to branch to ftrace_caller() when enabled or branch to 199 * NOP when disabled per-function base. 200 */ 201SYM_FUNC_START(_mcount) 202 ret 203SYM_FUNC_END(_mcount) 204EXPORT_SYMBOL(_mcount) 205NOKPROBE(_mcount) 206 207/* 208 * void ftrace_caller(unsigned long return_address) 209 * @return_address: return address to instrumented function 210 * 211 * This function is a counterpart of _mcount() in 'static' ftrace, and 212 * makes calls to: 213 * - tracer function to probe instrumented function's entry, 214 * - ftrace_graph_caller to set up an exit hook 215 */ 216SYM_FUNC_START(ftrace_caller) 217 mcount_enter 218 219 mcount_get_pc0 x0 // function's pc 220 mcount_get_lr x1 // function's lr 221 222SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) // tracer(pc, lr); 223 nop // This will be replaced with "bl xxx" 224 // where xxx can be any kind of tracer. 225 226#ifdef CONFIG_FUNCTION_GRAPH_TRACER 227SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) // ftrace_graph_caller(); 228 nop // If enabled, this will be replaced 229 // "b ftrace_graph_caller" 230#endif 231 232 mcount_exit 233SYM_FUNC_END(ftrace_caller) 234 235#ifdef CONFIG_FUNCTION_GRAPH_TRACER 236/* 237 * void ftrace_graph_caller(void) 238 * 239 * Called from _mcount() or ftrace_caller() when function_graph tracer is 240 * selected. 241 * This function w/ prepare_ftrace_return() fakes link register's value on 242 * the call stack in order to intercept instrumented function's return path 243 * and run return_to_handler() later on its exit. 244 */ 245SYM_FUNC_START(ftrace_graph_caller) 246 mcount_get_pc x0 // function's pc 247 mcount_get_lr_addr x1 // pointer to function's saved lr 248 mcount_get_parent_fp x2 // parent's fp 249 bl prepare_ftrace_return // prepare_ftrace_return(pc, &lr, fp) 250 251 mcount_exit 252SYM_FUNC_END(ftrace_graph_caller) 253#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 254#endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */ 255 256SYM_TYPED_FUNC_START(ftrace_stub) 257 ret 258SYM_FUNC_END(ftrace_stub) 259 260#ifdef CONFIG_FUNCTION_GRAPH_TRACER 261SYM_TYPED_FUNC_START(ftrace_stub_graph) 262 ret 263SYM_FUNC_END(ftrace_stub_graph) 264 265/* 266 * void return_to_handler(void) 267 * 268 * Run ftrace_return_to_handler() before going back to parent. 269 * @fp is checked against the value passed by ftrace_graph_caller(). 270 */ 271SYM_CODE_START(return_to_handler) 272 /* save return value regs */ 273 sub sp, sp, #64 274 stp x0, x1, [sp] 275 stp x2, x3, [sp, #16] 276 stp x4, x5, [sp, #32] 277 stp x6, x7, [sp, #48] 278 279 mov x0, x29 // parent's fp 280 bl ftrace_return_to_handler// addr = ftrace_return_to_hander(fp); 281 mov x30, x0 // restore the original return address 282 283 /* restore return value regs */ 284 ldp x0, x1, [sp] 285 ldp x2, x3, [sp, #16] 286 ldp x4, x5, [sp, #32] 287 ldp x6, x7, [sp, #48] 288 add sp, sp, #64 289 290 ret 291SYM_CODE_END(return_to_handler) 292#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 293