1d2912cb1SThomas Gleixner/* SPDX-License-Identifier: GPL-2.0-only */ 2819e50e2SAKASHI Takahiro/* 3819e50e2SAKASHI Takahiro * arch/arm64/kernel/entry-ftrace.S 4819e50e2SAKASHI Takahiro * 5819e50e2SAKASHI Takahiro * Copyright (C) 2013 Linaro Limited 6819e50e2SAKASHI Takahiro * Author: AKASHI Takahiro <takahiro.akashi@linaro.org> 7819e50e2SAKASHI Takahiro */ 8819e50e2SAKASHI Takahiro 9819e50e2SAKASHI Takahiro#include <linux/linkage.h> 10f705d954SArnd Bergmann#include <asm/assembler.h> 11819e50e2SAKASHI Takahiro#include <asm/ftrace.h> 12819e50e2SAKASHI Takahiro#include <asm/insn.h> 13819e50e2SAKASHI Takahiro 14819e50e2SAKASHI Takahiro/* 15819e50e2SAKASHI Takahiro * Gcc with -pg will put the following code in the beginning of each function: 16819e50e2SAKASHI Takahiro * mov x0, x30 17819e50e2SAKASHI Takahiro * bl _mcount 18819e50e2SAKASHI Takahiro * [function's body ...] 19819e50e2SAKASHI Takahiro * "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic 20819e50e2SAKASHI Takahiro * ftrace is enabled. 21819e50e2SAKASHI Takahiro * 22819e50e2SAKASHI Takahiro * Please note that x0 as an argument will not be used here because we can 23819e50e2SAKASHI Takahiro * get lr(x30) of instrumented function at any time by winding up call stack 24819e50e2SAKASHI Takahiro * as long as the kernel is compiled without -fomit-frame-pointer. 25819e50e2SAKASHI Takahiro * (or CONFIG_FRAME_POINTER, this is forced on arm64) 26819e50e2SAKASHI Takahiro * 27819e50e2SAKASHI Takahiro * stack layout after mcount_enter in _mcount(): 28819e50e2SAKASHI Takahiro * 29819e50e2SAKASHI Takahiro * current sp/fp => 0:+-----+ 30819e50e2SAKASHI Takahiro * in _mcount() | x29 | -> instrumented function's fp 31819e50e2SAKASHI Takahiro * +-----+ 32819e50e2SAKASHI Takahiro * | x30 | -> _mcount()'s lr (= instrumented function's pc) 33819e50e2SAKASHI Takahiro * old sp => +16:+-----+ 34819e50e2SAKASHI Takahiro * when instrumented | | 35819e50e2SAKASHI Takahiro * function calls | ... | 36819e50e2SAKASHI Takahiro * _mcount() | | 37819e50e2SAKASHI Takahiro * | | 38819e50e2SAKASHI Takahiro * instrumented => +xx:+-----+ 39819e50e2SAKASHI Takahiro * function's fp | x29 | -> parent's fp 40819e50e2SAKASHI Takahiro * +-----+ 41819e50e2SAKASHI Takahiro * | x30 | -> instrumented function's lr (= parent's pc) 42819e50e2SAKASHI Takahiro * +-----+ 43819e50e2SAKASHI Takahiro * | ... | 44819e50e2SAKASHI Takahiro */ 45819e50e2SAKASHI Takahiro 46819e50e2SAKASHI Takahiro .macro mcount_enter 47819e50e2SAKASHI Takahiro stp x29, x30, [sp, #-16]! 48819e50e2SAKASHI Takahiro mov x29, sp 49819e50e2SAKASHI Takahiro .endm 50819e50e2SAKASHI Takahiro 51819e50e2SAKASHI Takahiro .macro mcount_exit 52819e50e2SAKASHI Takahiro ldp x29, x30, [sp], #16 53819e50e2SAKASHI Takahiro ret 54819e50e2SAKASHI Takahiro .endm 55819e50e2SAKASHI Takahiro 56819e50e2SAKASHI Takahiro .macro mcount_adjust_addr rd, rn 57819e50e2SAKASHI Takahiro sub \rd, \rn, #AARCH64_INSN_SIZE 58819e50e2SAKASHI Takahiro .endm 59819e50e2SAKASHI Takahiro 60819e50e2SAKASHI Takahiro /* for instrumented function's parent */ 61819e50e2SAKASHI Takahiro .macro mcount_get_parent_fp reg 62819e50e2SAKASHI Takahiro ldr \reg, [x29] 63819e50e2SAKASHI Takahiro ldr \reg, [\reg] 64819e50e2SAKASHI Takahiro .endm 65819e50e2SAKASHI Takahiro 66819e50e2SAKASHI Takahiro /* for instrumented function */ 67819e50e2SAKASHI Takahiro .macro mcount_get_pc0 reg 68819e50e2SAKASHI Takahiro mcount_adjust_addr \reg, x30 69819e50e2SAKASHI Takahiro .endm 70819e50e2SAKASHI Takahiro 71819e50e2SAKASHI Takahiro .macro mcount_get_pc reg 72819e50e2SAKASHI Takahiro ldr \reg, [x29, #8] 73819e50e2SAKASHI Takahiro mcount_adjust_addr \reg, \reg 74819e50e2SAKASHI Takahiro .endm 75819e50e2SAKASHI Takahiro 76819e50e2SAKASHI Takahiro .macro mcount_get_lr reg 77819e50e2SAKASHI Takahiro ldr \reg, [x29] 78819e50e2SAKASHI Takahiro ldr \reg, [\reg, #8] 79819e50e2SAKASHI Takahiro .endm 80819e50e2SAKASHI Takahiro 81819e50e2SAKASHI Takahiro .macro mcount_get_lr_addr reg 82819e50e2SAKASHI Takahiro ldr \reg, [x29] 83819e50e2SAKASHI Takahiro add \reg, \reg, #8 84819e50e2SAKASHI Takahiro .endm 85819e50e2SAKASHI Takahiro 86bd7d38dbSAKASHI Takahiro#ifndef CONFIG_DYNAMIC_FTRACE 87819e50e2SAKASHI Takahiro/* 88819e50e2SAKASHI Takahiro * void _mcount(unsigned long return_address) 89819e50e2SAKASHI Takahiro * @return_address: return address to instrumented function 90819e50e2SAKASHI Takahiro * 91819e50e2SAKASHI Takahiro * This function makes calls, if enabled, to: 92819e50e2SAKASHI Takahiro * - tracer function to probe instrumented function's entry, 93819e50e2SAKASHI Takahiro * - ftrace_graph_caller to set up an exit hook 94819e50e2SAKASHI Takahiro */ 95819e50e2SAKASHI TakahiroENTRY(_mcount) 96819e50e2SAKASHI Takahiro mcount_enter 97819e50e2SAKASHI Takahiro 98829d2bd1SMark Rutland ldr_l x2, ftrace_trace_function 99819e50e2SAKASHI Takahiro adr x0, ftrace_stub 100819e50e2SAKASHI Takahiro cmp x0, x2 // if (ftrace_trace_function 101819e50e2SAKASHI Takahiro b.eq skip_ftrace_call // != ftrace_stub) { 102819e50e2SAKASHI Takahiro 103819e50e2SAKASHI Takahiro mcount_get_pc x0 // function's pc 104819e50e2SAKASHI Takahiro mcount_get_lr x1 // function's lr (= parent's pc) 105819e50e2SAKASHI Takahiro blr x2 // (*ftrace_trace_function)(pc, lr); 106819e50e2SAKASHI Takahiro 107d125bffcSJulien Thierryskip_ftrace_call: // } 108d125bffcSJulien Thierry#ifdef CONFIG_FUNCTION_GRAPH_TRACER 109829d2bd1SMark Rutland ldr_l x2, ftrace_graph_return 110f1ba46eeSArd Biesheuvel cmp x0, x2 // if ((ftrace_graph_return 111f1ba46eeSArd Biesheuvel b.ne ftrace_graph_caller // != ftrace_stub) 112819e50e2SAKASHI Takahiro 113829d2bd1SMark Rutland ldr_l x2, ftrace_graph_entry // || (ftrace_graph_entry 114829d2bd1SMark Rutland adr_l x0, ftrace_graph_entry_stub // != ftrace_graph_entry_stub)) 115819e50e2SAKASHI Takahiro cmp x0, x2 116819e50e2SAKASHI Takahiro b.ne ftrace_graph_caller // ftrace_graph_caller(); 117819e50e2SAKASHI Takahiro#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 118d125bffcSJulien Thierry mcount_exit 119819e50e2SAKASHI TakahiroENDPROC(_mcount) 120dbd31962SMark RutlandEXPORT_SYMBOL(_mcount) 121dbd31962SMark RutlandNOKPROBE(_mcount) 122819e50e2SAKASHI Takahiro 123bd7d38dbSAKASHI Takahiro#else /* CONFIG_DYNAMIC_FTRACE */ 124bd7d38dbSAKASHI Takahiro/* 125bd7d38dbSAKASHI Takahiro * _mcount() is used to build the kernel with -pg option, but all the branch 126bd7d38dbSAKASHI Takahiro * instructions to _mcount() are replaced to NOP initially at kernel start up, 127bd7d38dbSAKASHI Takahiro * and later on, NOP to branch to ftrace_caller() when enabled or branch to 128bd7d38dbSAKASHI Takahiro * NOP when disabled per-function base. 129bd7d38dbSAKASHI Takahiro */ 130bd7d38dbSAKASHI TakahiroENTRY(_mcount) 131bd7d38dbSAKASHI Takahiro ret 132bd7d38dbSAKASHI TakahiroENDPROC(_mcount) 133dbd31962SMark RutlandEXPORT_SYMBOL(_mcount) 134dbd31962SMark RutlandNOKPROBE(_mcount) 135bd7d38dbSAKASHI Takahiro 136bd7d38dbSAKASHI Takahiro/* 137bd7d38dbSAKASHI Takahiro * void ftrace_caller(unsigned long return_address) 138bd7d38dbSAKASHI Takahiro * @return_address: return address to instrumented function 139bd7d38dbSAKASHI Takahiro * 140bd7d38dbSAKASHI Takahiro * This function is a counterpart of _mcount() in 'static' ftrace, and 141bd7d38dbSAKASHI Takahiro * makes calls to: 142bd7d38dbSAKASHI Takahiro * - tracer function to probe instrumented function's entry, 143bd7d38dbSAKASHI Takahiro * - ftrace_graph_caller to set up an exit hook 144bd7d38dbSAKASHI Takahiro */ 145bd7d38dbSAKASHI TakahiroENTRY(ftrace_caller) 146bd7d38dbSAKASHI Takahiro mcount_enter 147bd7d38dbSAKASHI Takahiro 148bd7d38dbSAKASHI Takahiro mcount_get_pc0 x0 // function's pc 149bd7d38dbSAKASHI Takahiro mcount_get_lr x1 // function's lr 150bd7d38dbSAKASHI Takahiro 151e4fe1966SMark RutlandGLOBAL(ftrace_call) // tracer(pc, lr); 152bd7d38dbSAKASHI Takahiro nop // This will be replaced with "bl xxx" 153bd7d38dbSAKASHI Takahiro // where xxx can be any kind of tracer. 154bd7d38dbSAKASHI Takahiro 155bd7d38dbSAKASHI Takahiro#ifdef CONFIG_FUNCTION_GRAPH_TRACER 156e4fe1966SMark RutlandGLOBAL(ftrace_graph_call) // ftrace_graph_caller(); 157bd7d38dbSAKASHI Takahiro nop // If enabled, this will be replaced 158bd7d38dbSAKASHI Takahiro // "b ftrace_graph_caller" 159bd7d38dbSAKASHI Takahiro#endif 160bd7d38dbSAKASHI Takahiro 161bd7d38dbSAKASHI Takahiro mcount_exit 162bd7d38dbSAKASHI TakahiroENDPROC(ftrace_caller) 163bd7d38dbSAKASHI Takahiro#endif /* CONFIG_DYNAMIC_FTRACE */ 164bd7d38dbSAKASHI Takahiro 165819e50e2SAKASHI TakahiroENTRY(ftrace_stub) 166819e50e2SAKASHI Takahiro ret 167819e50e2SAKASHI TakahiroENDPROC(ftrace_stub) 168819e50e2SAKASHI Takahiro 169819e50e2SAKASHI Takahiro#ifdef CONFIG_FUNCTION_GRAPH_TRACER 170819e50e2SAKASHI Takahiro/* 171819e50e2SAKASHI Takahiro * void ftrace_graph_caller(void) 172819e50e2SAKASHI Takahiro * 173819e50e2SAKASHI Takahiro * Called from _mcount() or ftrace_caller() when function_graph tracer is 174819e50e2SAKASHI Takahiro * selected. 175819e50e2SAKASHI Takahiro * This function w/ prepare_ftrace_return() fakes link register's value on 176819e50e2SAKASHI Takahiro * the call stack in order to intercept instrumented function's return path 177819e50e2SAKASHI Takahiro * and run return_to_handler() later on its exit. 178819e50e2SAKASHI Takahiro */ 179819e50e2SAKASHI TakahiroENTRY(ftrace_graph_caller) 1807dc48bf9SMark Rutland mcount_get_pc x0 // function's pc 1817dc48bf9SMark Rutland mcount_get_lr_addr x1 // pointer to function's saved lr 182819e50e2SAKASHI Takahiro mcount_get_parent_fp x2 // parent's fp 1837dc48bf9SMark Rutland bl prepare_ftrace_return // prepare_ftrace_return(pc, &lr, fp) 184819e50e2SAKASHI Takahiro 185819e50e2SAKASHI Takahiro mcount_exit 186819e50e2SAKASHI TakahiroENDPROC(ftrace_graph_caller) 187819e50e2SAKASHI Takahiro 188819e50e2SAKASHI Takahiro/* 189819e50e2SAKASHI Takahiro * void return_to_handler(void) 190819e50e2SAKASHI Takahiro * 191819e50e2SAKASHI Takahiro * Run ftrace_return_to_handler() before going back to parent. 1925c176affSMark Rutland * @fp is checked against the value passed by ftrace_graph_caller(). 193819e50e2SAKASHI Takahiro */ 194819e50e2SAKASHI TakahiroENTRY(return_to_handler) 19549e258e0SMark Rutland /* save return value regs */ 19649e258e0SMark Rutland sub sp, sp, #64 19749e258e0SMark Rutland stp x0, x1, [sp] 19849e258e0SMark Rutland stp x2, x3, [sp, #16] 19949e258e0SMark Rutland stp x4, x5, [sp, #32] 20049e258e0SMark Rutland stp x6, x7, [sp, #48] 20149e258e0SMark Rutland 202819e50e2SAKASHI Takahiro mov x0, x29 // parent's fp 203819e50e2SAKASHI Takahiro bl ftrace_return_to_handler// addr = ftrace_return_to_hander(fp); 204819e50e2SAKASHI Takahiro mov x30, x0 // restore the original return address 20549e258e0SMark Rutland 20649e258e0SMark Rutland /* restore return value regs */ 20749e258e0SMark Rutland ldp x0, x1, [sp] 20849e258e0SMark Rutland ldp x2, x3, [sp, #16] 20949e258e0SMark Rutland ldp x4, x5, [sp, #32] 21049e258e0SMark Rutland ldp x6, x7, [sp, #48] 21149e258e0SMark Rutland add sp, sp, #64 21249e258e0SMark Rutland 213819e50e2SAKASHI Takahiro ret 214819e50e2SAKASHI TakahiroEND(return_to_handler) 215819e50e2SAKASHI Takahiro#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 216