xref: /openbmc/linux/arch/arm64/kernel/entry-ftrace.S (revision aeb64ff3) 
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * arch/arm64/kernel/entry-ftrace.S
4 *
5 * Copyright (C) 2013 Linaro Limited
6 * Author: AKASHI Takahiro <takahiro.akashi@linaro.org>
7 */
8
9#include <linux/linkage.h>
10#include <asm/asm-offsets.h>
11#include <asm/assembler.h>
12#include <asm/ftrace.h>
13#include <asm/insn.h>
14
15#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
16/*
17 * Due to -fpatchable-function-entry=2, the compiler has placed two NOPs before
18 * the regular function prologue. For an enabled callsite, ftrace_init_nop() and
19 * ftrace_make_call() have patched those NOPs to:
20 *
21 * 	MOV	X9, LR
22 * 	BL	<entry>
23 *
24 * ... where <entry> is either ftrace_caller or ftrace_regs_caller.
25 *
26 * Each instrumented function follows the AAPCS, so here x0-x8 and x19-x30 are
27 * live, and x9-x18 are safe to clobber.
28 *
29 * We save the callsite's context into a pt_regs before invoking any ftrace
30 * callbacks. So that we can get a sensible backtrace, we create a stack record
31 * for the callsite and the ftrace entry assembly. This is not sufficient for
32 * reliable stacktrace: until we create the callsite stack record, its caller
33 * is missing from the LR and existing chain of frame records.
34 */
35	.macro  ftrace_regs_entry, allregs=0
36	/* Make room for pt_regs, plus a callee frame */
37	sub	sp, sp, #(S_FRAME_SIZE + 16)
38
39	/* Save function arguments (and x9 for simplicity) */
40	stp	x0, x1, [sp, #S_X0]
41	stp	x2, x3, [sp, #S_X2]
42	stp	x4, x5, [sp, #S_X4]
43	stp	x6, x7, [sp, #S_X6]
44	stp	x8, x9, [sp, #S_X8]
45
46	/* Optionally save the callee-saved registers, always save the FP */
47	.if \allregs == 1
48	stp	x10, x11, [sp, #S_X10]
49	stp	x12, x13, [sp, #S_X12]
50	stp	x14, x15, [sp, #S_X14]
51	stp	x16, x17, [sp, #S_X16]
52	stp	x18, x19, [sp, #S_X18]
53	stp	x20, x21, [sp, #S_X20]
54	stp	x22, x23, [sp, #S_X22]
55	stp	x24, x25, [sp, #S_X24]
56	stp	x26, x27, [sp, #S_X26]
57	stp	x28, x29, [sp, #S_X28]
58	.else
59	str	x29, [sp, #S_FP]
60	.endif
61
62	/* Save the callsite's SP and LR */
63	add	x10, sp, #(S_FRAME_SIZE + 16)
64	stp	x9, x10, [sp, #S_LR]
65
66	/* Save the PC after the ftrace callsite */
67	str	x30, [sp, #S_PC]
68
69	/* Create a frame record for the callsite above pt_regs */
70	stp	x29, x9, [sp, #S_FRAME_SIZE]
71	add	x29, sp, #S_FRAME_SIZE
72
73	/* Create our frame record within pt_regs. */
74	stp	x29, x30, [sp, #S_STACKFRAME]
75	add	x29, sp, #S_STACKFRAME
76	.endm
77
78ENTRY(ftrace_regs_caller)
79	ftrace_regs_entry	1
80	b	ftrace_common
81ENDPROC(ftrace_regs_caller)
82
83ENTRY(ftrace_caller)
84	ftrace_regs_entry	0
85	b	ftrace_common
86ENDPROC(ftrace_caller)
87
88ENTRY(ftrace_common)
89	sub	x0, x30, #AARCH64_INSN_SIZE	// ip (callsite's BL insn)
90	mov	x1, x9				// parent_ip (callsite's LR)
91	ldr_l	x2, function_trace_op		// op
92	mov	x3, sp				// regs
93
94GLOBAL(ftrace_call)
95	bl	ftrace_stub
96
97#ifdef CONFIG_FUNCTION_GRAPH_TRACER
98GLOBAL(ftrace_graph_call)		// ftrace_graph_caller();
99	nop				// If enabled, this will be replaced
100					// "b ftrace_graph_caller"
101#endif
102
103/*
104 * At the callsite x0-x8 and x19-x30 were live. Any C code will have preserved
105 * x19-x29 per the AAPCS, and we created frame records upon entry, so we need
106 * to restore x0-x8, x29, and x30.
107 */
108ftrace_common_return:
109	/* Restore function arguments */
110	ldp	x0, x1, [sp]
111	ldp	x2, x3, [sp, #S_X2]
112	ldp	x4, x5, [sp, #S_X4]
113	ldp	x6, x7, [sp, #S_X6]
114	ldr	x8, [sp, #S_X8]
115
116	/* Restore the callsite's FP, LR, PC */
117	ldr	x29, [sp, #S_FP]
118	ldr	x30, [sp, #S_LR]
119	ldr	x9, [sp, #S_PC]
120
121	/* Restore the callsite's SP */
122	add	sp, sp, #S_FRAME_SIZE + 16
123
124	ret	x9
125ENDPROC(ftrace_common)
126
127#ifdef CONFIG_FUNCTION_GRAPH_TRACER
128ENTRY(ftrace_graph_caller)
129	ldr	x0, [sp, #S_PC]
130	sub	x0, x0, #AARCH64_INSN_SIZE	// ip (callsite's BL insn)
131	add	x1, sp, #S_LR			// parent_ip (callsite's LR)
132	ldr	x2, [sp, #S_FRAME_SIZE]	   	// parent fp (callsite's FP)
133	bl	prepare_ftrace_return
134	b	ftrace_common_return
135ENDPROC(ftrace_graph_caller)
136#endif
137
138#else /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
139
140/*
141 * Gcc with -pg will put the following code in the beginning of each function:
142 *      mov x0, x30
143 *      bl _mcount
144 *	[function's body ...]
145 * "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic
146 * ftrace is enabled.
147 *
148 * Please note that x0 as an argument will not be used here because we can
149 * get lr(x30) of instrumented function at any time by winding up call stack
150 * as long as the kernel is compiled without -fomit-frame-pointer.
151 * (or CONFIG_FRAME_POINTER, this is forced on arm64)
152 *
153 * stack layout after mcount_enter in _mcount():
154 *
155 * current sp/fp =>  0:+-----+
156 * in _mcount()        | x29 | -> instrumented function's fp
157 *                     +-----+
158 *                     | x30 | -> _mcount()'s lr (= instrumented function's pc)
159 * old sp       => +16:+-----+
160 * when instrumented   |     |
161 * function calls      | ... |
162 * _mcount()           |     |
163 *                     |     |
164 * instrumented => +xx:+-----+
165 * function's fp       | x29 | -> parent's fp
166 *                     +-----+
167 *                     | x30 | -> instrumented function's lr (= parent's pc)
168 *                     +-----+
169 *                     | ... |
170 */
171
172	.macro mcount_enter
173	stp	x29, x30, [sp, #-16]!
174	mov	x29, sp
175	.endm
176
177	.macro mcount_exit
178	ldp	x29, x30, [sp], #16
179	ret
180	.endm
181
182	.macro mcount_adjust_addr rd, rn
183	sub	\rd, \rn, #AARCH64_INSN_SIZE
184	.endm
185
186	/* for instrumented function's parent */
187	.macro mcount_get_parent_fp reg
188	ldr	\reg, [x29]
189	ldr	\reg, [\reg]
190	.endm
191
192	/* for instrumented function */
193	.macro mcount_get_pc0 reg
194	mcount_adjust_addr	\reg, x30
195	.endm
196
197	.macro mcount_get_pc reg
198	ldr	\reg, [x29, #8]
199	mcount_adjust_addr	\reg, \reg
200	.endm
201
202	.macro mcount_get_lr reg
203	ldr	\reg, [x29]
204	ldr	\reg, [\reg, #8]
205	.endm
206
207	.macro mcount_get_lr_addr reg
208	ldr	\reg, [x29]
209	add	\reg, \reg, #8
210	.endm
211
212#ifndef CONFIG_DYNAMIC_FTRACE
213/*
214 * void _mcount(unsigned long return_address)
215 * @return_address: return address to instrumented function
216 *
217 * This function makes calls, if enabled, to:
218 *     - tracer function to probe instrumented function's entry,
219 *     - ftrace_graph_caller to set up an exit hook
220 */
221ENTRY(_mcount)
222	mcount_enter
223
224	ldr_l	x2, ftrace_trace_function
225	adr	x0, ftrace_stub
226	cmp	x0, x2			// if (ftrace_trace_function
227	b.eq	skip_ftrace_call	//     != ftrace_stub) {
228
229	mcount_get_pc	x0		//       function's pc
230	mcount_get_lr	x1		//       function's lr (= parent's pc)
231	blr	x2			//   (*ftrace_trace_function)(pc, lr);
232
233skip_ftrace_call:			// }
234#ifdef CONFIG_FUNCTION_GRAPH_TRACER
235	ldr_l	x2, ftrace_graph_return
236	cmp	x0, x2			//   if ((ftrace_graph_return
237	b.ne	ftrace_graph_caller	//        != ftrace_stub)
238
239	ldr_l	x2, ftrace_graph_entry	//     || (ftrace_graph_entry
240	adr_l	x0, ftrace_graph_entry_stub //     != ftrace_graph_entry_stub))
241	cmp	x0, x2
242	b.ne	ftrace_graph_caller	//     ftrace_graph_caller();
243#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
244	mcount_exit
245ENDPROC(_mcount)
246EXPORT_SYMBOL(_mcount)
247NOKPROBE(_mcount)
248
249#else /* CONFIG_DYNAMIC_FTRACE */
250/*
251 * _mcount() is used to build the kernel with -pg option, but all the branch
252 * instructions to _mcount() are replaced to NOP initially at kernel start up,
253 * and later on, NOP to branch to ftrace_caller() when enabled or branch to
254 * NOP when disabled per-function base.
255 */
256ENTRY(_mcount)
257	ret
258ENDPROC(_mcount)
259EXPORT_SYMBOL(_mcount)
260NOKPROBE(_mcount)
261
262/*
263 * void ftrace_caller(unsigned long return_address)
264 * @return_address: return address to instrumented function
265 *
266 * This function is a counterpart of _mcount() in 'static' ftrace, and
267 * makes calls to:
268 *     - tracer function to probe instrumented function's entry,
269 *     - ftrace_graph_caller to set up an exit hook
270 */
271ENTRY(ftrace_caller)
272	mcount_enter
273
274	mcount_get_pc0	x0		//     function's pc
275	mcount_get_lr	x1		//     function's lr
276
277GLOBAL(ftrace_call)			// tracer(pc, lr);
278	nop				// This will be replaced with "bl xxx"
279					// where xxx can be any kind of tracer.
280
281#ifdef CONFIG_FUNCTION_GRAPH_TRACER
282GLOBAL(ftrace_graph_call)		// ftrace_graph_caller();
283	nop				// If enabled, this will be replaced
284					// "b ftrace_graph_caller"
285#endif
286
287	mcount_exit
288ENDPROC(ftrace_caller)
289#endif /* CONFIG_DYNAMIC_FTRACE */
290
291#ifdef CONFIG_FUNCTION_GRAPH_TRACER
292/*
293 * void ftrace_graph_caller(void)
294 *
295 * Called from _mcount() or ftrace_caller() when function_graph tracer is
296 * selected.
297 * This function w/ prepare_ftrace_return() fakes link register's value on
298 * the call stack in order to intercept instrumented function's return path
299 * and run return_to_handler() later on its exit.
300 */
301ENTRY(ftrace_graph_caller)
302	mcount_get_pc		  x0	//     function's pc
303	mcount_get_lr_addr	  x1	//     pointer to function's saved lr
304	mcount_get_parent_fp	  x2	//     parent's fp
305	bl	prepare_ftrace_return	// prepare_ftrace_return(pc, &lr, fp)
306
307	mcount_exit
308ENDPROC(ftrace_graph_caller)
309#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
310#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
311
312ENTRY(ftrace_stub)
313	ret
314ENDPROC(ftrace_stub)
315
316#ifdef CONFIG_FUNCTION_GRAPH_TRACER
317/*
318 * void return_to_handler(void)
319 *
320 * Run ftrace_return_to_handler() before going back to parent.
321 * @fp is checked against the value passed by ftrace_graph_caller().
322 */
323ENTRY(return_to_handler)
324	/* save return value regs */
325	sub sp, sp, #64
326	stp x0, x1, [sp]
327	stp x2, x3, [sp, #16]
328	stp x4, x5, [sp, #32]
329	stp x6, x7, [sp, #48]
330
331	mov	x0, x29			//     parent's fp
332	bl	ftrace_return_to_handler// addr = ftrace_return_to_hander(fp);
333	mov	x30, x0			// restore the original return address
334
335	/* restore return value regs */
336	ldp x0, x1, [sp]
337	ldp x2, x3, [sp, #16]
338	ldp x4, x5, [sp, #32]
339	ldp x6, x7, [sp, #48]
340	add sp, sp, #64
341
342	ret
343END(return_to_handler)
344#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
345