1d2912cb1SThomas Gleixner/* SPDX-License-Identifier: GPL-2.0-only */ 2819e50e2SAKASHI Takahiro/* 3819e50e2SAKASHI Takahiro * arch/arm64/kernel/entry-ftrace.S 4819e50e2SAKASHI Takahiro * 5819e50e2SAKASHI Takahiro * Copyright (C) 2013 Linaro Limited 6819e50e2SAKASHI Takahiro * Author: AKASHI Takahiro <takahiro.akashi@linaro.org> 7819e50e2SAKASHI Takahiro */ 8819e50e2SAKASHI Takahiro 9819e50e2SAKASHI Takahiro#include <linux/linkage.h> 10883bbbffSPeter Zijlstra#include <linux/cfi_types.h> 113b23e499STorsten Duwe#include <asm/asm-offsets.h> 12f705d954SArnd Bergmann#include <asm/assembler.h> 13819e50e2SAKASHI Takahiro#include <asm/ftrace.h> 14819e50e2SAKASHI Takahiro#include <asm/insn.h> 15819e50e2SAKASHI Takahiro 1626299b3fSMark Rutland#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS 173b23e499STorsten Duwe/* 183b23e499STorsten Duwe * Due to -fpatchable-function-entry=2, the compiler has placed two NOPs before 193b23e499STorsten Duwe * the regular function prologue. For an enabled callsite, ftrace_init_nop() and 203b23e499STorsten Duwe * ftrace_make_call() have patched those NOPs to: 213b23e499STorsten Duwe * 223b23e499STorsten Duwe * MOV X9, LR 2326299b3fSMark Rutland * BL ftrace_caller 243b23e499STorsten Duwe * 25258c3d62SWill Deacon * Each instrumented function follows the AAPCS, so here x0-x8 and x18-x30 are 26258c3d62SWill Deacon * live (x18 holds the Shadow Call Stack pointer), and x9-x17 are safe to 27258c3d62SWill Deacon * clobber. 283b23e499STorsten Duwe * 2926299b3fSMark Rutland * We save the callsite's context into a struct ftrace_regs before invoking any 3026299b3fSMark Rutland * ftrace callbacks. So that we can get a sensible backtrace, we create frame 3126299b3fSMark Rutland * records for the callsite and the ftrace entry assembly. This is not 3226299b3fSMark Rutland * sufficient for reliable stacktrace: until we create the callsite stack 3326299b3fSMark Rutland * record, its caller is missing from the LR and existing chain of frame 3426299b3fSMark Rutland * records. 353b23e499STorsten Duwe */ 36e434b08bSMark BrownSYM_CODE_START(ftrace_caller) 37742a15b1SMark Brown bti c 383b23e499STorsten Duwe 3926299b3fSMark Rutland /* Save original SP */ 4026299b3fSMark Rutland mov x10, sp 4126299b3fSMark Rutland 4226299b3fSMark Rutland /* Make room for ftrace regs, plus two frame records */ 4326299b3fSMark Rutland sub sp, sp, #(FREGS_SIZE + 32) 4426299b3fSMark Rutland 4526299b3fSMark Rutland /* Save function arguments */ 4626299b3fSMark Rutland stp x0, x1, [sp, #FREGS_X0] 4726299b3fSMark Rutland stp x2, x3, [sp, #FREGS_X2] 4826299b3fSMark Rutland stp x4, x5, [sp, #FREGS_X4] 4926299b3fSMark Rutland stp x6, x7, [sp, #FREGS_X6] 5026299b3fSMark Rutland str x8, [sp, #FREGS_X8] 5126299b3fSMark Rutland 5226299b3fSMark Rutland /* Save the callsite's FP, LR, SP */ 5326299b3fSMark Rutland str x29, [sp, #FREGS_FP] 5426299b3fSMark Rutland str x9, [sp, #FREGS_LR] 5526299b3fSMark Rutland str x10, [sp, #FREGS_SP] 5626299b3fSMark Rutland 5726299b3fSMark Rutland /* Save the PC after the ftrace callsite */ 5826299b3fSMark Rutland str x30, [sp, #FREGS_PC] 5926299b3fSMark Rutland 6026299b3fSMark Rutland /* Create a frame record for the callsite above the ftrace regs */ 6126299b3fSMark Rutland stp x29, x9, [sp, #FREGS_SIZE + 16] 6226299b3fSMark Rutland add x29, sp, #FREGS_SIZE + 16 6326299b3fSMark Rutland 6426299b3fSMark Rutland /* Create our frame record above the ftrace regs */ 6526299b3fSMark Rutland stp x29, x30, [sp, #FREGS_SIZE] 6626299b3fSMark Rutland add x29, sp, #FREGS_SIZE 6726299b3fSMark Rutland 68*baaf553dSMark Rutland /* Prepare arguments for the the tracer func */ 693b23e499STorsten Duwe sub x0, x30, #AARCH64_INSN_SIZE // ip (callsite's BL insn) 703b23e499STorsten Duwe mov x1, x9 // parent_ip (callsite's LR) 713b23e499STorsten Duwe mov x3, sp // regs 723b23e499STorsten Duwe 73*baaf553dSMark Rutland#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS 74*baaf553dSMark Rutland /* 75*baaf553dSMark Rutland * The literal pointer to the ops is at an 8-byte aligned boundary 76*baaf553dSMark Rutland * which is either 12 or 16 bytes before the BL instruction in the call 77*baaf553dSMark Rutland * site. See ftrace_call_adjust() for details. 78*baaf553dSMark Rutland * 79*baaf553dSMark Rutland * Therefore here the LR points at `literal + 16` or `literal + 20`, 80*baaf553dSMark Rutland * and we can find the address of the literal in either case by 81*baaf553dSMark Rutland * aligning to an 8-byte boundary and subtracting 16. We do the 82*baaf553dSMark Rutland * alignment first as this allows us to fold the subtraction into the 83*baaf553dSMark Rutland * LDR. 84*baaf553dSMark Rutland */ 85*baaf553dSMark Rutland bic x2, x30, 0x7 86*baaf553dSMark Rutland ldr x2, [x2, #-16] // op 87*baaf553dSMark Rutland 88*baaf553dSMark Rutland ldr x4, [x2, #FTRACE_OPS_FUNC] // op->func 89*baaf553dSMark Rutland blr x4 // op->func(ip, parent_ip, op, regs) 90*baaf553dSMark Rutland 91*baaf553dSMark Rutland#else 92*baaf553dSMark Rutland ldr_l x2, function_trace_op // op 93*baaf553dSMark Rutland 94e2d591d2SMark BrownSYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) 95*baaf553dSMark Rutland bl ftrace_stub // func(ip, parent_ip, op, regs) 96*baaf553dSMark Rutland#endif 973b23e499STorsten Duwe 983b23e499STorsten Duwe/* 993b23e499STorsten Duwe * At the callsite x0-x8 and x19-x30 were live. Any C code will have preserved 1003b23e499STorsten Duwe * x19-x29 per the AAPCS, and we created frame records upon entry, so we need 1013b23e499STorsten Duwe * to restore x0-x8, x29, and x30. 1023b23e499STorsten Duwe */ 1033b23e499STorsten Duwe /* Restore function arguments */ 10426299b3fSMark Rutland ldp x0, x1, [sp, #FREGS_X0] 10526299b3fSMark Rutland ldp x2, x3, [sp, #FREGS_X2] 10626299b3fSMark Rutland ldp x4, x5, [sp, #FREGS_X4] 10726299b3fSMark Rutland ldp x6, x7, [sp, #FREGS_X6] 10826299b3fSMark Rutland ldr x8, [sp, #FREGS_X8] 1093b23e499STorsten Duwe 1103b23e499STorsten Duwe /* Restore the callsite's FP, LR, PC */ 11126299b3fSMark Rutland ldr x29, [sp, #FREGS_FP] 11226299b3fSMark Rutland ldr x30, [sp, #FREGS_LR] 11326299b3fSMark Rutland ldr x9, [sp, #FREGS_PC] 1143b23e499STorsten Duwe 1153b23e499STorsten Duwe /* Restore the callsite's SP */ 11626299b3fSMark Rutland add sp, sp, #FREGS_SIZE + 32 1173b23e499STorsten Duwe 1183b23e499STorsten Duwe ret x9 11926299b3fSMark RutlandSYM_CODE_END(ftrace_caller) 1203b23e499STorsten Duwe 12126299b3fSMark Rutland#else /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */ 1223b23e499STorsten Duwe 123819e50e2SAKASHI Takahiro/* 124819e50e2SAKASHI Takahiro * Gcc with -pg will put the following code in the beginning of each function: 125819e50e2SAKASHI Takahiro * mov x0, x30 126819e50e2SAKASHI Takahiro * bl _mcount 127819e50e2SAKASHI Takahiro * [function's body ...] 128819e50e2SAKASHI Takahiro * "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic 129819e50e2SAKASHI Takahiro * ftrace is enabled. 130819e50e2SAKASHI Takahiro * 131819e50e2SAKASHI Takahiro * Please note that x0 as an argument will not be used here because we can 132819e50e2SAKASHI Takahiro * get lr(x30) of instrumented function at any time by winding up call stack 133819e50e2SAKASHI Takahiro * as long as the kernel is compiled without -fomit-frame-pointer. 134819e50e2SAKASHI Takahiro * (or CONFIG_FRAME_POINTER, this is forced on arm64) 135819e50e2SAKASHI Takahiro * 136819e50e2SAKASHI Takahiro * stack layout after mcount_enter in _mcount(): 137819e50e2SAKASHI Takahiro * 138819e50e2SAKASHI Takahiro * current sp/fp => 0:+-----+ 139819e50e2SAKASHI Takahiro * in _mcount() | x29 | -> instrumented function's fp 140819e50e2SAKASHI Takahiro * +-----+ 141819e50e2SAKASHI Takahiro * | x30 | -> _mcount()'s lr (= instrumented function's pc) 142819e50e2SAKASHI Takahiro * old sp => +16:+-----+ 143819e50e2SAKASHI Takahiro * when instrumented | | 144819e50e2SAKASHI Takahiro * function calls | ... | 145819e50e2SAKASHI Takahiro * _mcount() | | 146819e50e2SAKASHI Takahiro * | | 147819e50e2SAKASHI Takahiro * instrumented => +xx:+-----+ 148819e50e2SAKASHI Takahiro * function's fp | x29 | -> parent's fp 149819e50e2SAKASHI Takahiro * +-----+ 150819e50e2SAKASHI Takahiro * | x30 | -> instrumented function's lr (= parent's pc) 151819e50e2SAKASHI Takahiro * +-----+ 152819e50e2SAKASHI Takahiro * | ... | 153819e50e2SAKASHI Takahiro */ 154819e50e2SAKASHI Takahiro 155819e50e2SAKASHI Takahiro .macro mcount_enter 156819e50e2SAKASHI Takahiro stp x29, x30, [sp, #-16]! 157819e50e2SAKASHI Takahiro mov x29, sp 158819e50e2SAKASHI Takahiro .endm 159819e50e2SAKASHI Takahiro 160819e50e2SAKASHI Takahiro .macro mcount_exit 161819e50e2SAKASHI Takahiro ldp x29, x30, [sp], #16 162819e50e2SAKASHI Takahiro ret 163819e50e2SAKASHI Takahiro .endm 164819e50e2SAKASHI Takahiro 165819e50e2SAKASHI Takahiro .macro mcount_adjust_addr rd, rn 166819e50e2SAKASHI Takahiro sub \rd, \rn, #AARCH64_INSN_SIZE 167819e50e2SAKASHI Takahiro .endm 168819e50e2SAKASHI Takahiro 169819e50e2SAKASHI Takahiro /* for instrumented function's parent */ 170819e50e2SAKASHI Takahiro .macro mcount_get_parent_fp reg 171819e50e2SAKASHI Takahiro ldr \reg, [x29] 172819e50e2SAKASHI Takahiro ldr \reg, [\reg] 173819e50e2SAKASHI Takahiro .endm 174819e50e2SAKASHI Takahiro 175819e50e2SAKASHI Takahiro /* for instrumented function */ 176819e50e2SAKASHI Takahiro .macro mcount_get_pc0 reg 177819e50e2SAKASHI Takahiro mcount_adjust_addr \reg, x30 178819e50e2SAKASHI Takahiro .endm 179819e50e2SAKASHI Takahiro 180819e50e2SAKASHI Takahiro .macro mcount_get_pc reg 181819e50e2SAKASHI Takahiro ldr \reg, [x29, #8] 182819e50e2SAKASHI Takahiro mcount_adjust_addr \reg, \reg 183819e50e2SAKASHI Takahiro .endm 184819e50e2SAKASHI Takahiro 185819e50e2SAKASHI Takahiro .macro mcount_get_lr reg 186819e50e2SAKASHI Takahiro ldr \reg, [x29] 187819e50e2SAKASHI Takahiro ldr \reg, [\reg, #8] 188819e50e2SAKASHI Takahiro .endm 189819e50e2SAKASHI Takahiro 190819e50e2SAKASHI Takahiro .macro mcount_get_lr_addr reg 191819e50e2SAKASHI Takahiro ldr \reg, [x29] 192819e50e2SAKASHI Takahiro add \reg, \reg, #8 193819e50e2SAKASHI Takahiro .endm 194819e50e2SAKASHI Takahiro 195bd7d38dbSAKASHI Takahiro/* 196bd7d38dbSAKASHI Takahiro * _mcount() is used to build the kernel with -pg option, but all the branch 197bd7d38dbSAKASHI Takahiro * instructions to _mcount() are replaced to NOP initially at kernel start up, 198bd7d38dbSAKASHI Takahiro * and later on, NOP to branch to ftrace_caller() when enabled or branch to 199bd7d38dbSAKASHI Takahiro * NOP when disabled per-function base. 200bd7d38dbSAKASHI Takahiro */ 201e2d591d2SMark BrownSYM_FUNC_START(_mcount) 202bd7d38dbSAKASHI Takahiro ret 203e2d591d2SMark BrownSYM_FUNC_END(_mcount) 204dbd31962SMark RutlandEXPORT_SYMBOL(_mcount) 205dbd31962SMark RutlandNOKPROBE(_mcount) 206bd7d38dbSAKASHI Takahiro 207bd7d38dbSAKASHI Takahiro/* 208bd7d38dbSAKASHI Takahiro * void ftrace_caller(unsigned long return_address) 209bd7d38dbSAKASHI Takahiro * @return_address: return address to instrumented function 210bd7d38dbSAKASHI Takahiro * 211bd7d38dbSAKASHI Takahiro * This function is a counterpart of _mcount() in 'static' ftrace, and 212bd7d38dbSAKASHI Takahiro * makes calls to: 213bd7d38dbSAKASHI Takahiro * - tracer function to probe instrumented function's entry, 214bd7d38dbSAKASHI Takahiro * - ftrace_graph_caller to set up an exit hook 215bd7d38dbSAKASHI Takahiro */ 216e2d591d2SMark BrownSYM_FUNC_START(ftrace_caller) 217bd7d38dbSAKASHI Takahiro mcount_enter 218bd7d38dbSAKASHI Takahiro 219bd7d38dbSAKASHI Takahiro mcount_get_pc0 x0 // function's pc 220bd7d38dbSAKASHI Takahiro mcount_get_lr x1 // function's lr 221bd7d38dbSAKASHI Takahiro 222e2d591d2SMark BrownSYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) // tracer(pc, lr); 223bd7d38dbSAKASHI Takahiro nop // This will be replaced with "bl xxx" 224bd7d38dbSAKASHI Takahiro // where xxx can be any kind of tracer. 225bd7d38dbSAKASHI Takahiro 226bd7d38dbSAKASHI Takahiro#ifdef CONFIG_FUNCTION_GRAPH_TRACER 22769d113b5SKunihiko HayashiSYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) // ftrace_graph_caller(); 228bd7d38dbSAKASHI Takahiro nop // If enabled, this will be replaced 229bd7d38dbSAKASHI Takahiro // "b ftrace_graph_caller" 230bd7d38dbSAKASHI Takahiro#endif 231bd7d38dbSAKASHI Takahiro 232bd7d38dbSAKASHI Takahiro mcount_exit 233e2d591d2SMark BrownSYM_FUNC_END(ftrace_caller) 234819e50e2SAKASHI Takahiro 235819e50e2SAKASHI Takahiro#ifdef CONFIG_FUNCTION_GRAPH_TRACER 236819e50e2SAKASHI Takahiro/* 237819e50e2SAKASHI Takahiro * void ftrace_graph_caller(void) 238819e50e2SAKASHI Takahiro * 239819e50e2SAKASHI Takahiro * Called from _mcount() or ftrace_caller() when function_graph tracer is 240819e50e2SAKASHI Takahiro * selected. 241819e50e2SAKASHI Takahiro * This function w/ prepare_ftrace_return() fakes link register's value on 242819e50e2SAKASHI Takahiro * the call stack in order to intercept instrumented function's return path 243819e50e2SAKASHI Takahiro * and run return_to_handler() later on its exit. 244819e50e2SAKASHI Takahiro */ 245e2d591d2SMark BrownSYM_FUNC_START(ftrace_graph_caller) 2467dc48bf9SMark Rutland mcount_get_pc x0 // function's pc 2477dc48bf9SMark Rutland mcount_get_lr_addr x1 // pointer to function's saved lr 248819e50e2SAKASHI Takahiro mcount_get_parent_fp x2 // parent's fp 2497dc48bf9SMark Rutland bl prepare_ftrace_return // prepare_ftrace_return(pc, &lr, fp) 250819e50e2SAKASHI Takahiro 251819e50e2SAKASHI Takahiro mcount_exit 252e2d591d2SMark BrownSYM_FUNC_END(ftrace_graph_caller) 2533b23e499STorsten Duwe#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 25426299b3fSMark Rutland#endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */ 255819e50e2SAKASHI Takahiro 256883bbbffSPeter ZijlstraSYM_TYPED_FUNC_START(ftrace_stub) 2573b23e499STorsten Duwe ret 258e2d591d2SMark BrownSYM_FUNC_END(ftrace_stub) 2593b23e499STorsten Duwe 2602598ac6eSSami Tolvanen#ifdef CONFIG_FUNCTION_GRAPH_TRACER 261883bbbffSPeter ZijlstraSYM_TYPED_FUNC_START(ftrace_stub_graph) 262883bbbffSPeter Zijlstra ret 263883bbbffSPeter ZijlstraSYM_FUNC_END(ftrace_stub_graph) 264883bbbffSPeter Zijlstra 265819e50e2SAKASHI Takahiro/* 266819e50e2SAKASHI Takahiro * void return_to_handler(void) 267819e50e2SAKASHI Takahiro * 268819e50e2SAKASHI Takahiro * Run ftrace_return_to_handler() before going back to parent. 2695c176affSMark Rutland * @fp is checked against the value passed by ftrace_graph_caller(). 270819e50e2SAKASHI Takahiro */ 2711e4729edSMark BrownSYM_CODE_START(return_to_handler) 27249e258e0SMark Rutland /* save return value regs */ 27349e258e0SMark Rutland sub sp, sp, #64 27449e258e0SMark Rutland stp x0, x1, [sp] 27549e258e0SMark Rutland stp x2, x3, [sp, #16] 27649e258e0SMark Rutland stp x4, x5, [sp, #32] 27749e258e0SMark Rutland stp x6, x7, [sp, #48] 27849e258e0SMark Rutland 279819e50e2SAKASHI Takahiro mov x0, x29 // parent's fp 280819e50e2SAKASHI Takahiro bl ftrace_return_to_handler// addr = ftrace_return_to_hander(fp); 281819e50e2SAKASHI Takahiro mov x30, x0 // restore the original return address 28249e258e0SMark Rutland 28349e258e0SMark Rutland /* restore return value regs */ 28449e258e0SMark Rutland ldp x0, x1, [sp] 28549e258e0SMark Rutland ldp x2, x3, [sp, #16] 28649e258e0SMark Rutland ldp x4, x5, [sp, #32] 28749e258e0SMark Rutland ldp x6, x7, [sp, #48] 28849e258e0SMark Rutland add sp, sp, #64 28949e258e0SMark Rutland 290819e50e2SAKASHI Takahiro ret 2911e4729edSMark BrownSYM_CODE_END(return_to_handler) 292819e50e2SAKASHI Takahiro#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 293