1819e50e2SAKASHI Takahiro/* 2819e50e2SAKASHI Takahiro * arch/arm64/kernel/entry-ftrace.S 3819e50e2SAKASHI Takahiro * 4819e50e2SAKASHI Takahiro * Copyright (C) 2013 Linaro Limited 5819e50e2SAKASHI Takahiro * Author: AKASHI Takahiro <takahiro.akashi@linaro.org> 6819e50e2SAKASHI Takahiro * 7819e50e2SAKASHI Takahiro * This program is free software; you can redistribute it and/or modify 8819e50e2SAKASHI Takahiro * it under the terms of the GNU General Public License version 2 as 9819e50e2SAKASHI Takahiro * published by the Free Software Foundation. 10819e50e2SAKASHI Takahiro */ 11819e50e2SAKASHI Takahiro 12819e50e2SAKASHI Takahiro#include <linux/linkage.h> 13f705d954SArnd Bergmann#include <asm/assembler.h> 14819e50e2SAKASHI Takahiro#include <asm/ftrace.h> 15819e50e2SAKASHI Takahiro#include <asm/insn.h> 16819e50e2SAKASHI Takahiro 17819e50e2SAKASHI Takahiro/* 18819e50e2SAKASHI Takahiro * Gcc with -pg will put the following code in the beginning of each function: 19819e50e2SAKASHI Takahiro * mov x0, x30 20819e50e2SAKASHI Takahiro * bl _mcount 21819e50e2SAKASHI Takahiro * [function's body ...] 22819e50e2SAKASHI Takahiro * "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic 23819e50e2SAKASHI Takahiro * ftrace is enabled. 24819e50e2SAKASHI Takahiro * 25819e50e2SAKASHI Takahiro * Please note that x0 as an argument will not be used here because we can 26819e50e2SAKASHI Takahiro * get lr(x30) of instrumented function at any time by winding up call stack 27819e50e2SAKASHI Takahiro * as long as the kernel is compiled without -fomit-frame-pointer. 28819e50e2SAKASHI Takahiro * (or CONFIG_FRAME_POINTER, this is forced on arm64) 29819e50e2SAKASHI Takahiro * 30819e50e2SAKASHI Takahiro * stack layout after mcount_enter in _mcount(): 31819e50e2SAKASHI Takahiro * 32819e50e2SAKASHI Takahiro * current sp/fp => 0:+-----+ 33819e50e2SAKASHI Takahiro * in _mcount() | x29 | -> instrumented function's fp 34819e50e2SAKASHI Takahiro * +-----+ 35819e50e2SAKASHI Takahiro * | x30 | -> _mcount()'s lr (= instrumented function's pc) 36819e50e2SAKASHI Takahiro * old sp => +16:+-----+ 37819e50e2SAKASHI Takahiro * when instrumented | | 38819e50e2SAKASHI Takahiro * function calls | ... | 39819e50e2SAKASHI Takahiro * _mcount() | | 40819e50e2SAKASHI Takahiro * | | 41819e50e2SAKASHI Takahiro * instrumented => +xx:+-----+ 42819e50e2SAKASHI Takahiro * function's fp | x29 | -> parent's fp 43819e50e2SAKASHI Takahiro * +-----+ 44819e50e2SAKASHI Takahiro * | x30 | -> instrumented function's lr (= parent's pc) 45819e50e2SAKASHI Takahiro * +-----+ 46819e50e2SAKASHI Takahiro * | ... | 47819e50e2SAKASHI Takahiro */ 48819e50e2SAKASHI Takahiro 49819e50e2SAKASHI Takahiro .macro mcount_enter 50819e50e2SAKASHI Takahiro stp x29, x30, [sp, #-16]! 51819e50e2SAKASHI Takahiro mov x29, sp 52819e50e2SAKASHI Takahiro .endm 53819e50e2SAKASHI Takahiro 54819e50e2SAKASHI Takahiro .macro mcount_exit 55819e50e2SAKASHI Takahiro ldp x29, x30, [sp], #16 56819e50e2SAKASHI Takahiro ret 57819e50e2SAKASHI Takahiro .endm 58819e50e2SAKASHI Takahiro 59819e50e2SAKASHI Takahiro .macro mcount_adjust_addr rd, rn 60819e50e2SAKASHI Takahiro sub \rd, \rn, #AARCH64_INSN_SIZE 61819e50e2SAKASHI Takahiro .endm 62819e50e2SAKASHI Takahiro 63819e50e2SAKASHI Takahiro /* for instrumented function's parent */ 64819e50e2SAKASHI Takahiro .macro mcount_get_parent_fp reg 65819e50e2SAKASHI Takahiro ldr \reg, [x29] 66819e50e2SAKASHI Takahiro ldr \reg, [\reg] 67819e50e2SAKASHI Takahiro .endm 68819e50e2SAKASHI Takahiro 69819e50e2SAKASHI Takahiro /* for instrumented function */ 70819e50e2SAKASHI Takahiro .macro mcount_get_pc0 reg 71819e50e2SAKASHI Takahiro mcount_adjust_addr \reg, x30 72819e50e2SAKASHI Takahiro .endm 73819e50e2SAKASHI Takahiro 74819e50e2SAKASHI Takahiro .macro mcount_get_pc reg 75819e50e2SAKASHI Takahiro ldr \reg, [x29, #8] 76819e50e2SAKASHI Takahiro mcount_adjust_addr \reg, \reg 77819e50e2SAKASHI Takahiro .endm 78819e50e2SAKASHI Takahiro 79819e50e2SAKASHI Takahiro .macro mcount_get_lr reg 80819e50e2SAKASHI Takahiro ldr \reg, [x29] 81819e50e2SAKASHI Takahiro ldr \reg, [\reg, #8] 82819e50e2SAKASHI Takahiro .endm 83819e50e2SAKASHI Takahiro 84819e50e2SAKASHI Takahiro .macro mcount_get_lr_addr reg 85819e50e2SAKASHI Takahiro ldr \reg, [x29] 86819e50e2SAKASHI Takahiro add \reg, \reg, #8 87819e50e2SAKASHI Takahiro .endm 88819e50e2SAKASHI Takahiro 89bd7d38dbSAKASHI Takahiro#ifndef CONFIG_DYNAMIC_FTRACE 90819e50e2SAKASHI Takahiro/* 91819e50e2SAKASHI Takahiro * void _mcount(unsigned long return_address) 92819e50e2SAKASHI Takahiro * @return_address: return address to instrumented function 93819e50e2SAKASHI Takahiro * 94819e50e2SAKASHI Takahiro * This function makes calls, if enabled, to: 95819e50e2SAKASHI Takahiro * - tracer function to probe instrumented function's entry, 96819e50e2SAKASHI Takahiro * - ftrace_graph_caller to set up an exit hook 97819e50e2SAKASHI Takahiro */ 98819e50e2SAKASHI TakahiroENTRY(_mcount) 99819e50e2SAKASHI Takahiro mcount_enter 100819e50e2SAKASHI Takahiro 101829d2bd1SMark Rutland ldr_l x2, ftrace_trace_function 102819e50e2SAKASHI Takahiro adr x0, ftrace_stub 103819e50e2SAKASHI Takahiro cmp x0, x2 // if (ftrace_trace_function 104819e50e2SAKASHI Takahiro b.eq skip_ftrace_call // != ftrace_stub) { 105819e50e2SAKASHI Takahiro 106819e50e2SAKASHI Takahiro mcount_get_pc x0 // function's pc 107819e50e2SAKASHI Takahiro mcount_get_lr x1 // function's lr (= parent's pc) 108819e50e2SAKASHI Takahiro blr x2 // (*ftrace_trace_function)(pc, lr); 109819e50e2SAKASHI Takahiro 110d125bffcSJulien Thierryskip_ftrace_call: // } 111d125bffcSJulien Thierry#ifdef CONFIG_FUNCTION_GRAPH_TRACER 112829d2bd1SMark Rutland ldr_l x2, ftrace_graph_return 113f1ba46eeSArd Biesheuvel cmp x0, x2 // if ((ftrace_graph_return 114f1ba46eeSArd Biesheuvel b.ne ftrace_graph_caller // != ftrace_stub) 115819e50e2SAKASHI Takahiro 116829d2bd1SMark Rutland ldr_l x2, ftrace_graph_entry // || (ftrace_graph_entry 117829d2bd1SMark Rutland adr_l x0, ftrace_graph_entry_stub // != ftrace_graph_entry_stub)) 118819e50e2SAKASHI Takahiro cmp x0, x2 119819e50e2SAKASHI Takahiro b.ne ftrace_graph_caller // ftrace_graph_caller(); 120819e50e2SAKASHI Takahiro#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 121d125bffcSJulien Thierry mcount_exit 122819e50e2SAKASHI TakahiroENDPROC(_mcount) 123819e50e2SAKASHI Takahiro 124bd7d38dbSAKASHI Takahiro#else /* CONFIG_DYNAMIC_FTRACE */ 125bd7d38dbSAKASHI Takahiro/* 126bd7d38dbSAKASHI Takahiro * _mcount() is used to build the kernel with -pg option, but all the branch 127bd7d38dbSAKASHI Takahiro * instructions to _mcount() are replaced to NOP initially at kernel start up, 128bd7d38dbSAKASHI Takahiro * and later on, NOP to branch to ftrace_caller() when enabled or branch to 129bd7d38dbSAKASHI Takahiro * NOP when disabled per-function base. 130bd7d38dbSAKASHI Takahiro */ 131bd7d38dbSAKASHI TakahiroENTRY(_mcount) 132bd7d38dbSAKASHI Takahiro ret 133bd7d38dbSAKASHI TakahiroENDPROC(_mcount) 134bd7d38dbSAKASHI Takahiro 135bd7d38dbSAKASHI Takahiro/* 136bd7d38dbSAKASHI Takahiro * void ftrace_caller(unsigned long return_address) 137bd7d38dbSAKASHI Takahiro * @return_address: return address to instrumented function 138bd7d38dbSAKASHI Takahiro * 139bd7d38dbSAKASHI Takahiro * This function is a counterpart of _mcount() in 'static' ftrace, and 140bd7d38dbSAKASHI Takahiro * makes calls to: 141bd7d38dbSAKASHI Takahiro * - tracer function to probe instrumented function's entry, 142bd7d38dbSAKASHI Takahiro * - ftrace_graph_caller to set up an exit hook 143bd7d38dbSAKASHI Takahiro */ 144bd7d38dbSAKASHI TakahiroENTRY(ftrace_caller) 145bd7d38dbSAKASHI Takahiro mcount_enter 146bd7d38dbSAKASHI Takahiro 147bd7d38dbSAKASHI Takahiro mcount_get_pc0 x0 // function's pc 148bd7d38dbSAKASHI Takahiro mcount_get_lr x1 // function's lr 149bd7d38dbSAKASHI Takahiro 150e4fe1966SMark RutlandGLOBAL(ftrace_call) // tracer(pc, lr); 151bd7d38dbSAKASHI Takahiro nop // This will be replaced with "bl xxx" 152bd7d38dbSAKASHI Takahiro // where xxx can be any kind of tracer. 153bd7d38dbSAKASHI Takahiro 154bd7d38dbSAKASHI Takahiro#ifdef CONFIG_FUNCTION_GRAPH_TRACER 155e4fe1966SMark RutlandGLOBAL(ftrace_graph_call) // ftrace_graph_caller(); 156bd7d38dbSAKASHI Takahiro nop // If enabled, this will be replaced 157bd7d38dbSAKASHI Takahiro // "b ftrace_graph_caller" 158bd7d38dbSAKASHI Takahiro#endif 159bd7d38dbSAKASHI Takahiro 160bd7d38dbSAKASHI Takahiro mcount_exit 161bd7d38dbSAKASHI TakahiroENDPROC(ftrace_caller) 162bd7d38dbSAKASHI Takahiro#endif /* CONFIG_DYNAMIC_FTRACE */ 163bd7d38dbSAKASHI Takahiro 164819e50e2SAKASHI TakahiroENTRY(ftrace_stub) 165819e50e2SAKASHI Takahiro ret 166819e50e2SAKASHI TakahiroENDPROC(ftrace_stub) 167819e50e2SAKASHI Takahiro 168819e50e2SAKASHI Takahiro#ifdef CONFIG_FUNCTION_GRAPH_TRACER 169819e50e2SAKASHI Takahiro/* 170819e50e2SAKASHI Takahiro * void ftrace_graph_caller(void) 171819e50e2SAKASHI Takahiro * 172819e50e2SAKASHI Takahiro * Called from _mcount() or ftrace_caller() when function_graph tracer is 173819e50e2SAKASHI Takahiro * selected. 174819e50e2SAKASHI Takahiro * This function w/ prepare_ftrace_return() fakes link register's value on 175819e50e2SAKASHI Takahiro * the call stack in order to intercept instrumented function's return path 176819e50e2SAKASHI Takahiro * and run return_to_handler() later on its exit. 177819e50e2SAKASHI Takahiro */ 178819e50e2SAKASHI TakahiroENTRY(ftrace_graph_caller) 1797dc48bf9SMark Rutland mcount_get_pc x0 // function's pc 1807dc48bf9SMark Rutland mcount_get_lr_addr x1 // pointer to function's saved lr 181819e50e2SAKASHI Takahiro mcount_get_parent_fp x2 // parent's fp 1827dc48bf9SMark Rutland bl prepare_ftrace_return // prepare_ftrace_return(pc, &lr, fp) 183819e50e2SAKASHI Takahiro 184819e50e2SAKASHI Takahiro mcount_exit 185819e50e2SAKASHI TakahiroENDPROC(ftrace_graph_caller) 186819e50e2SAKASHI Takahiro 187819e50e2SAKASHI Takahiro/* 188819e50e2SAKASHI Takahiro * void return_to_handler(void) 189819e50e2SAKASHI Takahiro * 190819e50e2SAKASHI Takahiro * Run ftrace_return_to_handler() before going back to parent. 1915c176affSMark Rutland * @fp is checked against the value passed by ftrace_graph_caller(). 192819e50e2SAKASHI Takahiro */ 193819e50e2SAKASHI TakahiroENTRY(return_to_handler) 19449e258e0SMark Rutland /* save return value regs */ 19549e258e0SMark Rutland sub sp, sp, #64 19649e258e0SMark Rutland stp x0, x1, [sp] 19749e258e0SMark Rutland stp x2, x3, [sp, #16] 19849e258e0SMark Rutland stp x4, x5, [sp, #32] 19949e258e0SMark Rutland stp x6, x7, [sp, #48] 20049e258e0SMark Rutland 201819e50e2SAKASHI Takahiro mov x0, x29 // parent's fp 202819e50e2SAKASHI Takahiro bl ftrace_return_to_handler// addr = ftrace_return_to_hander(fp); 203819e50e2SAKASHI Takahiro mov x30, x0 // restore the original return address 20449e258e0SMark Rutland 20549e258e0SMark Rutland /* restore return value regs */ 20649e258e0SMark Rutland ldp x0, x1, [sp] 20749e258e0SMark Rutland ldp x2, x3, [sp, #16] 20849e258e0SMark Rutland ldp x4, x5, [sp, #32] 20949e258e0SMark Rutland ldp x6, x7, [sp, #48] 21049e258e0SMark Rutland add sp, sp, #64 21149e258e0SMark Rutland 212819e50e2SAKASHI Takahiro ret 213819e50e2SAKASHI TakahiroEND(return_to_handler) 214819e50e2SAKASHI Takahiro#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 215