1819e50e2SAKASHI Takahiro/* 2819e50e2SAKASHI Takahiro * arch/arm64/kernel/entry-ftrace.S 3819e50e2SAKASHI Takahiro * 4819e50e2SAKASHI Takahiro * Copyright (C) 2013 Linaro Limited 5819e50e2SAKASHI Takahiro * Author: AKASHI Takahiro <takahiro.akashi@linaro.org> 6819e50e2SAKASHI Takahiro * 7819e50e2SAKASHI Takahiro * This program is free software; you can redistribute it and/or modify 8819e50e2SAKASHI Takahiro * it under the terms of the GNU General Public License version 2 as 9819e50e2SAKASHI Takahiro * published by the Free Software Foundation. 10819e50e2SAKASHI Takahiro */ 11819e50e2SAKASHI Takahiro 12819e50e2SAKASHI Takahiro#include <linux/linkage.h> 13f705d954SArnd Bergmann#include <asm/assembler.h> 14819e50e2SAKASHI Takahiro#include <asm/ftrace.h> 15819e50e2SAKASHI Takahiro#include <asm/insn.h> 16819e50e2SAKASHI Takahiro 17819e50e2SAKASHI Takahiro/* 18819e50e2SAKASHI Takahiro * Gcc with -pg will put the following code in the beginning of each function: 19819e50e2SAKASHI Takahiro * mov x0, x30 20819e50e2SAKASHI Takahiro * bl _mcount 21819e50e2SAKASHI Takahiro * [function's body ...] 22819e50e2SAKASHI Takahiro * "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic 23819e50e2SAKASHI Takahiro * ftrace is enabled. 24819e50e2SAKASHI Takahiro * 25819e50e2SAKASHI Takahiro * Please note that x0 as an argument will not be used here because we can 26819e50e2SAKASHI Takahiro * get lr(x30) of instrumented function at any time by winding up call stack 27819e50e2SAKASHI Takahiro * as long as the kernel is compiled without -fomit-frame-pointer. 28819e50e2SAKASHI Takahiro * (or CONFIG_FRAME_POINTER, this is forced on arm64) 29819e50e2SAKASHI Takahiro * 30819e50e2SAKASHI Takahiro * stack layout after mcount_enter in _mcount(): 31819e50e2SAKASHI Takahiro * 32819e50e2SAKASHI Takahiro * current sp/fp => 0:+-----+ 33819e50e2SAKASHI Takahiro * in _mcount() | x29 | -> instrumented function's fp 34819e50e2SAKASHI Takahiro * +-----+ 35819e50e2SAKASHI Takahiro * | x30 | -> _mcount()'s lr (= instrumented function's pc) 36819e50e2SAKASHI Takahiro * old sp => +16:+-----+ 37819e50e2SAKASHI Takahiro * when instrumented | | 38819e50e2SAKASHI Takahiro * function calls | ... | 39819e50e2SAKASHI Takahiro * _mcount() | | 40819e50e2SAKASHI Takahiro * | | 41819e50e2SAKASHI Takahiro * instrumented => +xx:+-----+ 42819e50e2SAKASHI Takahiro * function's fp | x29 | -> parent's fp 43819e50e2SAKASHI Takahiro * +-----+ 44819e50e2SAKASHI Takahiro * | x30 | -> instrumented function's lr (= parent's pc) 45819e50e2SAKASHI Takahiro * +-----+ 46819e50e2SAKASHI Takahiro * | ... | 47819e50e2SAKASHI Takahiro */ 48819e50e2SAKASHI Takahiro 49819e50e2SAKASHI Takahiro .macro mcount_enter 50819e50e2SAKASHI Takahiro stp x29, x30, [sp, #-16]! 51819e50e2SAKASHI Takahiro mov x29, sp 52819e50e2SAKASHI Takahiro .endm 53819e50e2SAKASHI Takahiro 54819e50e2SAKASHI Takahiro .macro mcount_exit 55819e50e2SAKASHI Takahiro ldp x29, x30, [sp], #16 56819e50e2SAKASHI Takahiro ret 57819e50e2SAKASHI Takahiro .endm 58819e50e2SAKASHI Takahiro 59819e50e2SAKASHI Takahiro .macro mcount_adjust_addr rd, rn 60819e50e2SAKASHI Takahiro sub \rd, \rn, #AARCH64_INSN_SIZE 61819e50e2SAKASHI Takahiro .endm 62819e50e2SAKASHI Takahiro 63819e50e2SAKASHI Takahiro /* for instrumented function's parent */ 64819e50e2SAKASHI Takahiro .macro mcount_get_parent_fp reg 65819e50e2SAKASHI Takahiro ldr \reg, [x29] 66819e50e2SAKASHI Takahiro ldr \reg, [\reg] 67819e50e2SAKASHI Takahiro .endm 68819e50e2SAKASHI Takahiro 69819e50e2SAKASHI Takahiro /* for instrumented function */ 70819e50e2SAKASHI Takahiro .macro mcount_get_pc0 reg 71819e50e2SAKASHI Takahiro mcount_adjust_addr \reg, x30 72819e50e2SAKASHI Takahiro .endm 73819e50e2SAKASHI Takahiro 74819e50e2SAKASHI Takahiro .macro mcount_get_pc reg 75819e50e2SAKASHI Takahiro ldr \reg, [x29, #8] 76819e50e2SAKASHI Takahiro mcount_adjust_addr \reg, \reg 77819e50e2SAKASHI Takahiro .endm 78819e50e2SAKASHI Takahiro 79819e50e2SAKASHI Takahiro .macro mcount_get_lr reg 80819e50e2SAKASHI Takahiro ldr \reg, [x29] 81819e50e2SAKASHI Takahiro ldr \reg, [\reg, #8] 82819e50e2SAKASHI Takahiro mcount_adjust_addr \reg, \reg 83819e50e2SAKASHI Takahiro .endm 84819e50e2SAKASHI Takahiro 85819e50e2SAKASHI Takahiro .macro mcount_get_lr_addr reg 86819e50e2SAKASHI Takahiro ldr \reg, [x29] 87819e50e2SAKASHI Takahiro add \reg, \reg, #8 88819e50e2SAKASHI Takahiro .endm 89819e50e2SAKASHI Takahiro 90bd7d38dbSAKASHI Takahiro#ifndef CONFIG_DYNAMIC_FTRACE 91819e50e2SAKASHI Takahiro/* 92819e50e2SAKASHI Takahiro * void _mcount(unsigned long return_address) 93819e50e2SAKASHI Takahiro * @return_address: return address to instrumented function 94819e50e2SAKASHI Takahiro * 95819e50e2SAKASHI Takahiro * This function makes calls, if enabled, to: 96819e50e2SAKASHI Takahiro * - tracer function to probe instrumented function's entry, 97819e50e2SAKASHI Takahiro * - ftrace_graph_caller to set up an exit hook 98819e50e2SAKASHI Takahiro */ 99819e50e2SAKASHI TakahiroENTRY(_mcount) 100819e50e2SAKASHI Takahiro mcount_enter 101819e50e2SAKASHI Takahiro 102829d2bd1SMark Rutland ldr_l x2, ftrace_trace_function 103819e50e2SAKASHI Takahiro adr x0, ftrace_stub 104819e50e2SAKASHI Takahiro cmp x0, x2 // if (ftrace_trace_function 105819e50e2SAKASHI Takahiro b.eq skip_ftrace_call // != ftrace_stub) { 106819e50e2SAKASHI Takahiro 107819e50e2SAKASHI Takahiro mcount_get_pc x0 // function's pc 108819e50e2SAKASHI Takahiro mcount_get_lr x1 // function's lr (= parent's pc) 109819e50e2SAKASHI Takahiro blr x2 // (*ftrace_trace_function)(pc, lr); 110819e50e2SAKASHI Takahiro 111d125bffcSJulien Thierryskip_ftrace_call: // } 112d125bffcSJulien Thierry#ifdef CONFIG_FUNCTION_GRAPH_TRACER 113829d2bd1SMark Rutland ldr_l x2, ftrace_graph_return 114f1ba46eeSArd Biesheuvel cmp x0, x2 // if ((ftrace_graph_return 115f1ba46eeSArd Biesheuvel b.ne ftrace_graph_caller // != ftrace_stub) 116819e50e2SAKASHI Takahiro 117829d2bd1SMark Rutland ldr_l x2, ftrace_graph_entry // || (ftrace_graph_entry 118829d2bd1SMark Rutland adr_l x0, ftrace_graph_entry_stub // != ftrace_graph_entry_stub)) 119819e50e2SAKASHI Takahiro cmp x0, x2 120819e50e2SAKASHI Takahiro b.ne ftrace_graph_caller // ftrace_graph_caller(); 121819e50e2SAKASHI Takahiro#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 122d125bffcSJulien Thierry mcount_exit 123819e50e2SAKASHI TakahiroENDPROC(_mcount) 124819e50e2SAKASHI Takahiro 125bd7d38dbSAKASHI Takahiro#else /* CONFIG_DYNAMIC_FTRACE */ 126bd7d38dbSAKASHI Takahiro/* 127bd7d38dbSAKASHI Takahiro * _mcount() is used to build the kernel with -pg option, but all the branch 128bd7d38dbSAKASHI Takahiro * instructions to _mcount() are replaced to NOP initially at kernel start up, 129bd7d38dbSAKASHI Takahiro * and later on, NOP to branch to ftrace_caller() when enabled or branch to 130bd7d38dbSAKASHI Takahiro * NOP when disabled per-function base. 131bd7d38dbSAKASHI Takahiro */ 132bd7d38dbSAKASHI TakahiroENTRY(_mcount) 133bd7d38dbSAKASHI Takahiro ret 134bd7d38dbSAKASHI TakahiroENDPROC(_mcount) 135bd7d38dbSAKASHI Takahiro 136bd7d38dbSAKASHI Takahiro/* 137bd7d38dbSAKASHI Takahiro * void ftrace_caller(unsigned long return_address) 138bd7d38dbSAKASHI Takahiro * @return_address: return address to instrumented function 139bd7d38dbSAKASHI Takahiro * 140bd7d38dbSAKASHI Takahiro * This function is a counterpart of _mcount() in 'static' ftrace, and 141bd7d38dbSAKASHI Takahiro * makes calls to: 142bd7d38dbSAKASHI Takahiro * - tracer function to probe instrumented function's entry, 143bd7d38dbSAKASHI Takahiro * - ftrace_graph_caller to set up an exit hook 144bd7d38dbSAKASHI Takahiro */ 145bd7d38dbSAKASHI TakahiroENTRY(ftrace_caller) 146bd7d38dbSAKASHI Takahiro mcount_enter 147bd7d38dbSAKASHI Takahiro 148bd7d38dbSAKASHI Takahiro mcount_get_pc0 x0 // function's pc 149bd7d38dbSAKASHI Takahiro mcount_get_lr x1 // function's lr 150bd7d38dbSAKASHI Takahiro 151e4fe1966SMark RutlandGLOBAL(ftrace_call) // tracer(pc, lr); 152bd7d38dbSAKASHI Takahiro nop // This will be replaced with "bl xxx" 153bd7d38dbSAKASHI Takahiro // where xxx can be any kind of tracer. 154bd7d38dbSAKASHI Takahiro 155bd7d38dbSAKASHI Takahiro#ifdef CONFIG_FUNCTION_GRAPH_TRACER 156e4fe1966SMark RutlandGLOBAL(ftrace_graph_call) // ftrace_graph_caller(); 157bd7d38dbSAKASHI Takahiro nop // If enabled, this will be replaced 158bd7d38dbSAKASHI Takahiro // "b ftrace_graph_caller" 159bd7d38dbSAKASHI Takahiro#endif 160bd7d38dbSAKASHI Takahiro 161bd7d38dbSAKASHI Takahiro mcount_exit 162bd7d38dbSAKASHI TakahiroENDPROC(ftrace_caller) 163bd7d38dbSAKASHI Takahiro#endif /* CONFIG_DYNAMIC_FTRACE */ 164bd7d38dbSAKASHI Takahiro 165819e50e2SAKASHI TakahiroENTRY(ftrace_stub) 166819e50e2SAKASHI Takahiro ret 167819e50e2SAKASHI TakahiroENDPROC(ftrace_stub) 168819e50e2SAKASHI Takahiro 169819e50e2SAKASHI Takahiro#ifdef CONFIG_FUNCTION_GRAPH_TRACER 170ee556d00SLi Bin /* save return value regs*/ 171ee556d00SLi Bin .macro save_return_regs 172ee556d00SLi Bin sub sp, sp, #64 173ee556d00SLi Bin stp x0, x1, [sp] 174ee556d00SLi Bin stp x2, x3, [sp, #16] 175ee556d00SLi Bin stp x4, x5, [sp, #32] 176ee556d00SLi Bin stp x6, x7, [sp, #48] 177ee556d00SLi Bin .endm 178ee556d00SLi Bin 179ee556d00SLi Bin /* restore return value regs*/ 180ee556d00SLi Bin .macro restore_return_regs 181ee556d00SLi Bin ldp x0, x1, [sp] 182ee556d00SLi Bin ldp x2, x3, [sp, #16] 183ee556d00SLi Bin ldp x4, x5, [sp, #32] 184ee556d00SLi Bin ldp x6, x7, [sp, #48] 185ee556d00SLi Bin add sp, sp, #64 186ee556d00SLi Bin .endm 187ee556d00SLi Bin 188819e50e2SAKASHI Takahiro/* 189819e50e2SAKASHI Takahiro * void ftrace_graph_caller(void) 190819e50e2SAKASHI Takahiro * 191819e50e2SAKASHI Takahiro * Called from _mcount() or ftrace_caller() when function_graph tracer is 192819e50e2SAKASHI Takahiro * selected. 193819e50e2SAKASHI Takahiro * This function w/ prepare_ftrace_return() fakes link register's value on 194819e50e2SAKASHI Takahiro * the call stack in order to intercept instrumented function's return path 195819e50e2SAKASHI Takahiro * and run return_to_handler() later on its exit. 196819e50e2SAKASHI Takahiro */ 197819e50e2SAKASHI TakahiroENTRY(ftrace_graph_caller) 198819e50e2SAKASHI Takahiro mcount_get_lr_addr x0 // pointer to function's saved lr 199819e50e2SAKASHI Takahiro mcount_get_pc x1 // function's pc 200819e50e2SAKASHI Takahiro mcount_get_parent_fp x2 // parent's fp 201819e50e2SAKASHI Takahiro bl prepare_ftrace_return // prepare_ftrace_return(&lr, pc, fp) 202819e50e2SAKASHI Takahiro 203819e50e2SAKASHI Takahiro mcount_exit 204819e50e2SAKASHI TakahiroENDPROC(ftrace_graph_caller) 205819e50e2SAKASHI Takahiro 206819e50e2SAKASHI Takahiro/* 207819e50e2SAKASHI Takahiro * void return_to_handler(void) 208819e50e2SAKASHI Takahiro * 209819e50e2SAKASHI Takahiro * Run ftrace_return_to_handler() before going back to parent. 2105c176affSMark Rutland * @fp is checked against the value passed by ftrace_graph_caller(). 211819e50e2SAKASHI Takahiro */ 212819e50e2SAKASHI TakahiroENTRY(return_to_handler) 213ee556d00SLi Bin save_return_regs 214819e50e2SAKASHI Takahiro mov x0, x29 // parent's fp 215819e50e2SAKASHI Takahiro bl ftrace_return_to_handler// addr = ftrace_return_to_hander(fp); 216819e50e2SAKASHI Takahiro mov x30, x0 // restore the original return address 217ee556d00SLi Bin restore_return_regs 218819e50e2SAKASHI Takahiro ret 219819e50e2SAKASHI TakahiroEND(return_to_handler) 220819e50e2SAKASHI Takahiro#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 221