1/* 2 * arch/arm64/kernel/entry-ftrace.S 3 * 4 * Copyright (C) 2013 Linaro Limited 5 * Author: AKASHI Takahiro <takahiro.akashi@linaro.org> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 12#include <linux/linkage.h> 13#include <asm/ftrace.h> 14#include <asm/insn.h> 15 16/* 17 * Gcc with -pg will put the following code in the beginning of each function: 18 * mov x0, x30 19 * bl _mcount 20 * [function's body ...] 21 * "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic 22 * ftrace is enabled. 23 * 24 * Please note that x0 as an argument will not be used here because we can 25 * get lr(x30) of instrumented function at any time by winding up call stack 26 * as long as the kernel is compiled without -fomit-frame-pointer. 27 * (or CONFIG_FRAME_POINTER, this is forced on arm64) 28 * 29 * stack layout after mcount_enter in _mcount(): 30 * 31 * current sp/fp => 0:+-----+ 32 * in _mcount() | x29 | -> instrumented function's fp 33 * +-----+ 34 * | x30 | -> _mcount()'s lr (= instrumented function's pc) 35 * old sp => +16:+-----+ 36 * when instrumented | | 37 * function calls | ... | 38 * _mcount() | | 39 * | | 40 * instrumented => +xx:+-----+ 41 * function's fp | x29 | -> parent's fp 42 * +-----+ 43 * | x30 | -> instrumented function's lr (= parent's pc) 44 * +-----+ 45 * | ... | 46 */ 47 48 .macro mcount_enter 49 stp x29, x30, [sp, #-16]! 50 mov x29, sp 51 .endm 52 53 .macro mcount_exit 54 ldp x29, x30, [sp], #16 55 ret 56 .endm 57 58 .macro mcount_adjust_addr rd, rn 59 sub \rd, \rn, #AARCH64_INSN_SIZE 60 .endm 61 62 /* for instrumented function's parent */ 63 .macro mcount_get_parent_fp reg 64 ldr \reg, [x29] 65 ldr \reg, [\reg] 66 .endm 67 68 /* for instrumented function */ 69 .macro mcount_get_pc0 reg 70 mcount_adjust_addr \reg, x30 71 .endm 72 73 .macro mcount_get_pc reg 74 ldr \reg, [x29, #8] 75 mcount_adjust_addr \reg, \reg 76 .endm 77 78 .macro mcount_get_lr reg 79 ldr \reg, [x29] 80 ldr \reg, [\reg, #8] 81 mcount_adjust_addr \reg, \reg 82 .endm 83 84 .macro mcount_get_lr_addr reg 85 ldr \reg, [x29] 86 add \reg, \reg, #8 87 .endm 88 89#ifndef CONFIG_DYNAMIC_FTRACE 90/* 91 * void _mcount(unsigned long return_address) 92 * @return_address: return address to instrumented function 93 * 94 * This function makes calls, if enabled, to: 95 * - tracer function to probe instrumented function's entry, 96 * - ftrace_graph_caller to set up an exit hook 97 */ 98ENTRY(_mcount) 99#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST 100 ldr x0, =ftrace_trace_stop 101 ldr x0, [x0] // if ftrace_trace_stop 102 ret // return; 103#endif 104 mcount_enter 105 106 ldr x0, =ftrace_trace_function 107 ldr x2, [x0] 108 adr x0, ftrace_stub 109 cmp x0, x2 // if (ftrace_trace_function 110 b.eq skip_ftrace_call // != ftrace_stub) { 111 112 mcount_get_pc x0 // function's pc 113 mcount_get_lr x1 // function's lr (= parent's pc) 114 blr x2 // (*ftrace_trace_function)(pc, lr); 115 116#ifndef CONFIG_FUNCTION_GRAPH_TRACER 117skip_ftrace_call: // return; 118 mcount_exit // } 119#else 120 mcount_exit // return; 121 // } 122skip_ftrace_call: 123 ldr x1, =ftrace_graph_return 124 ldr x2, [x1] // if ((ftrace_graph_return 125 cmp x0, x2 // != ftrace_stub) 126 b.ne ftrace_graph_caller 127 128 ldr x1, =ftrace_graph_entry // || (ftrace_graph_entry 129 ldr x2, [x1] // != ftrace_graph_entry_stub)) 130 ldr x0, =ftrace_graph_entry_stub 131 cmp x0, x2 132 b.ne ftrace_graph_caller // ftrace_graph_caller(); 133 134 mcount_exit 135#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 136ENDPROC(_mcount) 137 138#else /* CONFIG_DYNAMIC_FTRACE */ 139/* 140 * _mcount() is used to build the kernel with -pg option, but all the branch 141 * instructions to _mcount() are replaced to NOP initially at kernel start up, 142 * and later on, NOP to branch to ftrace_caller() when enabled or branch to 143 * NOP when disabled per-function base. 144 */ 145ENTRY(_mcount) 146 ret 147ENDPROC(_mcount) 148 149/* 150 * void ftrace_caller(unsigned long return_address) 151 * @return_address: return address to instrumented function 152 * 153 * This function is a counterpart of _mcount() in 'static' ftrace, and 154 * makes calls to: 155 * - tracer function to probe instrumented function's entry, 156 * - ftrace_graph_caller to set up an exit hook 157 */ 158ENTRY(ftrace_caller) 159 mcount_enter 160 161 mcount_get_pc0 x0 // function's pc 162 mcount_get_lr x1 // function's lr 163 164 .global ftrace_call 165ftrace_call: // tracer(pc, lr); 166 nop // This will be replaced with "bl xxx" 167 // where xxx can be any kind of tracer. 168 169#ifdef CONFIG_FUNCTION_GRAPH_TRACER 170 .global ftrace_graph_call 171ftrace_graph_call: // ftrace_graph_caller(); 172 nop // If enabled, this will be replaced 173 // "b ftrace_graph_caller" 174#endif 175 176 mcount_exit 177ENDPROC(ftrace_caller) 178#endif /* CONFIG_DYNAMIC_FTRACE */ 179 180ENTRY(ftrace_stub) 181 ret 182ENDPROC(ftrace_stub) 183 184#ifdef CONFIG_FUNCTION_GRAPH_TRACER 185/* 186 * void ftrace_graph_caller(void) 187 * 188 * Called from _mcount() or ftrace_caller() when function_graph tracer is 189 * selected. 190 * This function w/ prepare_ftrace_return() fakes link register's value on 191 * the call stack in order to intercept instrumented function's return path 192 * and run return_to_handler() later on its exit. 193 */ 194ENTRY(ftrace_graph_caller) 195 mcount_get_lr_addr x0 // pointer to function's saved lr 196 mcount_get_pc x1 // function's pc 197 mcount_get_parent_fp x2 // parent's fp 198 bl prepare_ftrace_return // prepare_ftrace_return(&lr, pc, fp) 199 200 mcount_exit 201ENDPROC(ftrace_graph_caller) 202 203/* 204 * void return_to_handler(void) 205 * 206 * Run ftrace_return_to_handler() before going back to parent. 207 * @fp is checked against the value passed by ftrace_graph_caller() 208 * only when CONFIG_HAVE_FUNCTION_GRAPH_FP_TEST is enabled. 209 */ 210ENTRY(return_to_handler) 211 str x0, [sp, #-16]! 212 mov x0, x29 // parent's fp 213 bl ftrace_return_to_handler// addr = ftrace_return_to_hander(fp); 214 mov x30, x0 // restore the original return address 215 ldr x0, [sp], #16 216 ret 217END(return_to_handler) 218#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 219