xref: /openbmc/linux/arch/arm64/kernel/entry-ftrace.S (revision 01ab991f) 
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * arch/arm64/kernel/entry-ftrace.S
4 *
5 * Copyright (C) 2013 Linaro Limited
6 * Author: AKASHI Takahiro <takahiro.akashi@linaro.org>
7 */
8
9#include <linux/linkage.h>
10#include <linux/cfi_types.h>
11#include <asm/asm-offsets.h>
12#include <asm/assembler.h>
13#include <asm/ftrace.h>
14#include <asm/insn.h>
15
16#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
17/*
18 * Due to -fpatchable-function-entry=2, the compiler has placed two NOPs before
19 * the regular function prologue. For an enabled callsite, ftrace_init_nop() and
20 * ftrace_make_call() have patched those NOPs to:
21 *
22 * 	MOV	X9, LR
23 * 	BL	<entry>
24 *
25 * ... where <entry> is either ftrace_caller or ftrace_regs_caller.
26 *
27 * Each instrumented function follows the AAPCS, so here x0-x8 and x18-x30 are
28 * live (x18 holds the Shadow Call Stack pointer), and x9-x17 are safe to
29 * clobber.
30 *
31 * We save the callsite's context into a pt_regs before invoking any ftrace
32 * callbacks. So that we can get a sensible backtrace, we create a stack record
33 * for the callsite and the ftrace entry assembly. This is not sufficient for
34 * reliable stacktrace: until we create the callsite stack record, its caller
35 * is missing from the LR and existing chain of frame records.
36 */
37	.macro  ftrace_regs_entry, allregs=0
38	/* Make room for pt_regs, plus a callee frame */
39	sub	sp, sp, #(PT_REGS_SIZE + 16)
40
41	/* Save function arguments (and x9 for simplicity) */
42	stp	x0, x1, [sp, #S_X0]
43	stp	x2, x3, [sp, #S_X2]
44	stp	x4, x5, [sp, #S_X4]
45	stp	x6, x7, [sp, #S_X6]
46	stp	x8, x9, [sp, #S_X8]
47
48	/* Optionally save the callee-saved registers, always save the FP */
49	.if \allregs == 1
50	stp	x10, x11, [sp, #S_X10]
51	stp	x12, x13, [sp, #S_X12]
52	stp	x14, x15, [sp, #S_X14]
53	stp	x16, x17, [sp, #S_X16]
54	stp	x18, x19, [sp, #S_X18]
55	stp	x20, x21, [sp, #S_X20]
56	stp	x22, x23, [sp, #S_X22]
57	stp	x24, x25, [sp, #S_X24]
58	stp	x26, x27, [sp, #S_X26]
59	stp	x28, x29, [sp, #S_X28]
60	.else
61	str	x29, [sp, #S_FP]
62	.endif
63
64	/* Save the callsite's SP and LR */
65	add	x10, sp, #(PT_REGS_SIZE + 16)
66	stp	x9, x10, [sp, #S_LR]
67
68	/* Save the PC after the ftrace callsite */
69	str	x30, [sp, #S_PC]
70
71	/* Create a frame record for the callsite above pt_regs */
72	stp	x29, x9, [sp, #PT_REGS_SIZE]
73	add	x29, sp, #PT_REGS_SIZE
74
75	/* Create our frame record within pt_regs. */
76	stp	x29, x30, [sp, #S_STACKFRAME]
77	add	x29, sp, #S_STACKFRAME
78	.endm
79
80SYM_CODE_START(ftrace_regs_caller)
81	bti	c
82	ftrace_regs_entry	1
83	b	ftrace_common
84SYM_CODE_END(ftrace_regs_caller)
85
86SYM_CODE_START(ftrace_caller)
87	bti	c
88	ftrace_regs_entry	0
89	b	ftrace_common
90SYM_CODE_END(ftrace_caller)
91
92SYM_CODE_START(ftrace_common)
93	sub	x0, x30, #AARCH64_INSN_SIZE	// ip (callsite's BL insn)
94	mov	x1, x9				// parent_ip (callsite's LR)
95	ldr_l	x2, function_trace_op		// op
96	mov	x3, sp				// regs
97
98SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
99	bl	ftrace_stub
100
101/*
102 * At the callsite x0-x8 and x19-x30 were live. Any C code will have preserved
103 * x19-x29 per the AAPCS, and we created frame records upon entry, so we need
104 * to restore x0-x8, x29, and x30.
105 */
106	/* Restore function arguments */
107	ldp	x0, x1, [sp]
108	ldp	x2, x3, [sp, #S_X2]
109	ldp	x4, x5, [sp, #S_X4]
110	ldp	x6, x7, [sp, #S_X6]
111	ldr	x8, [sp, #S_X8]
112
113	/* Restore the callsite's FP, LR, PC */
114	ldr	x29, [sp, #S_FP]
115	ldr	x30, [sp, #S_LR]
116	ldr	x9, [sp, #S_PC]
117
118	/* Restore the callsite's SP */
119	add	sp, sp, #PT_REGS_SIZE + 16
120
121	ret	x9
122SYM_CODE_END(ftrace_common)
123
124#else /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
125
126/*
127 * Gcc with -pg will put the following code in the beginning of each function:
128 *      mov x0, x30
129 *      bl _mcount
130 *	[function's body ...]
131 * "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic
132 * ftrace is enabled.
133 *
134 * Please note that x0 as an argument will not be used here because we can
135 * get lr(x30) of instrumented function at any time by winding up call stack
136 * as long as the kernel is compiled without -fomit-frame-pointer.
137 * (or CONFIG_FRAME_POINTER, this is forced on arm64)
138 *
139 * stack layout after mcount_enter in _mcount():
140 *
141 * current sp/fp =>  0:+-----+
142 * in _mcount()        | x29 | -> instrumented function's fp
143 *                     +-----+
144 *                     | x30 | -> _mcount()'s lr (= instrumented function's pc)
145 * old sp       => +16:+-----+
146 * when instrumented   |     |
147 * function calls      | ... |
148 * _mcount()           |     |
149 *                     |     |
150 * instrumented => +xx:+-----+
151 * function's fp       | x29 | -> parent's fp
152 *                     +-----+
153 *                     | x30 | -> instrumented function's lr (= parent's pc)
154 *                     +-----+
155 *                     | ... |
156 */
157
158	.macro mcount_enter
159	stp	x29, x30, [sp, #-16]!
160	mov	x29, sp
161	.endm
162
163	.macro mcount_exit
164	ldp	x29, x30, [sp], #16
165	ret
166	.endm
167
168	.macro mcount_adjust_addr rd, rn
169	sub	\rd, \rn, #AARCH64_INSN_SIZE
170	.endm
171
172	/* for instrumented function's parent */
173	.macro mcount_get_parent_fp reg
174	ldr	\reg, [x29]
175	ldr	\reg, [\reg]
176	.endm
177
178	/* for instrumented function */
179	.macro mcount_get_pc0 reg
180	mcount_adjust_addr	\reg, x30
181	.endm
182
183	.macro mcount_get_pc reg
184	ldr	\reg, [x29, #8]
185	mcount_adjust_addr	\reg, \reg
186	.endm
187
188	.macro mcount_get_lr reg
189	ldr	\reg, [x29]
190	ldr	\reg, [\reg, #8]
191	.endm
192
193	.macro mcount_get_lr_addr reg
194	ldr	\reg, [x29]
195	add	\reg, \reg, #8
196	.endm
197
198#ifndef CONFIG_DYNAMIC_FTRACE
199/*
200 * void _mcount(unsigned long return_address)
201 * @return_address: return address to instrumented function
202 *
203 * This function makes calls, if enabled, to:
204 *     - tracer function to probe instrumented function's entry,
205 *     - ftrace_graph_caller to set up an exit hook
206 */
207SYM_FUNC_START(_mcount)
208	mcount_enter
209
210	ldr_l	x2, ftrace_trace_function
211	adr	x0, ftrace_stub
212	cmp	x0, x2			// if (ftrace_trace_function
213	b.eq	skip_ftrace_call	//     != ftrace_stub) {
214
215	mcount_get_pc	x0		//       function's pc
216	mcount_get_lr	x1		//       function's lr (= parent's pc)
217	blr	x2			//   (*ftrace_trace_function)(pc, lr);
218
219skip_ftrace_call:			// }
220#ifdef CONFIG_FUNCTION_GRAPH_TRACER
221	ldr_l	x2, ftrace_graph_return
222	cmp	x0, x2			//   if ((ftrace_graph_return
223	b.ne	ftrace_graph_caller	//        != ftrace_stub)
224
225	ldr_l	x2, ftrace_graph_entry	//     || (ftrace_graph_entry
226	adr_l	x0, ftrace_graph_entry_stub //     != ftrace_graph_entry_stub))
227	cmp	x0, x2
228	b.ne	ftrace_graph_caller	//     ftrace_graph_caller();
229#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
230	mcount_exit
231SYM_FUNC_END(_mcount)
232EXPORT_SYMBOL(_mcount)
233NOKPROBE(_mcount)
234
235#else /* CONFIG_DYNAMIC_FTRACE */
236/*
237 * _mcount() is used to build the kernel with -pg option, but all the branch
238 * instructions to _mcount() are replaced to NOP initially at kernel start up,
239 * and later on, NOP to branch to ftrace_caller() when enabled or branch to
240 * NOP when disabled per-function base.
241 */
242SYM_FUNC_START(_mcount)
243	ret
244SYM_FUNC_END(_mcount)
245EXPORT_SYMBOL(_mcount)
246NOKPROBE(_mcount)
247
248/*
249 * void ftrace_caller(unsigned long return_address)
250 * @return_address: return address to instrumented function
251 *
252 * This function is a counterpart of _mcount() in 'static' ftrace, and
253 * makes calls to:
254 *     - tracer function to probe instrumented function's entry,
255 *     - ftrace_graph_caller to set up an exit hook
256 */
257SYM_FUNC_START(ftrace_caller)
258	mcount_enter
259
260	mcount_get_pc0	x0		//     function's pc
261	mcount_get_lr	x1		//     function's lr
262
263SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)	// tracer(pc, lr);
264	nop				// This will be replaced with "bl xxx"
265					// where xxx can be any kind of tracer.
266
267#ifdef CONFIG_FUNCTION_GRAPH_TRACER
268SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) // ftrace_graph_caller();
269	nop				// If enabled, this will be replaced
270					// "b ftrace_graph_caller"
271#endif
272
273	mcount_exit
274SYM_FUNC_END(ftrace_caller)
275#endif /* CONFIG_DYNAMIC_FTRACE */
276
277#ifdef CONFIG_FUNCTION_GRAPH_TRACER
278/*
279 * void ftrace_graph_caller(void)
280 *
281 * Called from _mcount() or ftrace_caller() when function_graph tracer is
282 * selected.
283 * This function w/ prepare_ftrace_return() fakes link register's value on
284 * the call stack in order to intercept instrumented function's return path
285 * and run return_to_handler() later on its exit.
286 */
287SYM_FUNC_START(ftrace_graph_caller)
288	mcount_get_pc		  x0	//     function's pc
289	mcount_get_lr_addr	  x1	//     pointer to function's saved lr
290	mcount_get_parent_fp	  x2	//     parent's fp
291	bl	prepare_ftrace_return	// prepare_ftrace_return(pc, &lr, fp)
292
293	mcount_exit
294SYM_FUNC_END(ftrace_graph_caller)
295#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
296#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
297
298SYM_TYPED_FUNC_START(ftrace_stub)
299	ret
300SYM_FUNC_END(ftrace_stub)
301
302SYM_TYPED_FUNC_START(ftrace_stub_graph)
303	ret
304SYM_FUNC_END(ftrace_stub_graph)
305
306#ifdef CONFIG_FUNCTION_GRAPH_TRACER
307/*
308 * void return_to_handler(void)
309 *
310 * Run ftrace_return_to_handler() before going back to parent.
311 * @fp is checked against the value passed by ftrace_graph_caller().
312 */
313SYM_CODE_START(return_to_handler)
314	/* save return value regs */
315	sub sp, sp, #64
316	stp x0, x1, [sp]
317	stp x2, x3, [sp, #16]
318	stp x4, x5, [sp, #32]
319	stp x6, x7, [sp, #48]
320
321	mov	x0, x29			//     parent's fp
322	bl	ftrace_return_to_handler// addr = ftrace_return_to_hander(fp);
323	mov	x30, x0			// restore the original return address
324
325	/* restore return value regs */
326	ldp x0, x1, [sp]
327	ldp x2, x3, [sp, #16]
328	ldp x4, x5, [sp, #32]
329	ldp x6, x7, [sp, #48]
330	add sp, sp, #64
331
332	ret
333SYM_CODE_END(return_to_handler)
334#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
335