1014c257cSAbhishek Sagar /*
2014c257cSAbhishek Sagar * Dynamic function tracing support.
3014c257cSAbhishek Sagar *
4014c257cSAbhishek Sagar * Copyright (C) 2008 Abhishek Sagar <sagar.abhishek@gmail.com>
53b6c223bSRabin Vincent * Copyright (C) 2010 Rabin Vincent <rabin@rab.in>
6014c257cSAbhishek Sagar *
7014c257cSAbhishek Sagar * For licencing details, see COPYING.
8014c257cSAbhishek Sagar *
9014c257cSAbhishek Sagar * Defines low-level handling of mcount calls when the kernel
10014c257cSAbhishek Sagar * is compiled with the -pg flag. When using dynamic ftrace, the
113b6c223bSRabin Vincent * mcount call-sites get patched with NOP till they are enabled.
123b6c223bSRabin Vincent * All code mutation routines here are called under stop_machine().
13014c257cSAbhishek Sagar */
14014c257cSAbhishek Sagar
15014c257cSAbhishek Sagar #include <linux/ftrace.h>
163b6c223bSRabin Vincent #include <linux/uaccess.h>
17a672917aSRabin Vincent #include <linux/module.h>
1880d6b0c2SKees Cook #include <linux/stop_machine.h>
19014c257cSAbhishek Sagar
20395a59d0SAbhishek Sagar #include <asm/cacheflush.h>
214394e282SRabin Vincent #include <asm/opcodes.h>
22395a59d0SAbhishek Sagar #include <asm/ftrace.h>
230dc016dbSWang Nan #include <asm/insn.h>
2474d86a70SLaura Abbott #include <asm/set_memory.h>
2541918ec8SArd Biesheuvel #include <asm/stacktrace.h>
265a735583SPeter Zijlstra #include <asm/patch.h>
27d82227cfSRabin Vincent
28ad1c2f39SArd Biesheuvel /*
29ad1c2f39SArd Biesheuvel * The compiler emitted profiling hook consists of
30ad1c2f39SArd Biesheuvel *
31ad1c2f39SArd Biesheuvel * PUSH {LR}
32ad1c2f39SArd Biesheuvel * BL __gnu_mcount_nc
33ad1c2f39SArd Biesheuvel *
34ad1c2f39SArd Biesheuvel * To turn this combined sequence into a NOP, we need to restore the value of
35ad1c2f39SArd Biesheuvel * SP before the PUSH. Let's use an ADD rather than a POP into LR, as LR is not
36ad1c2f39SArd Biesheuvel * modified anyway, and reloading LR from memory is highly likely to be less
37ad1c2f39SArd Biesheuvel * efficient.
38ad1c2f39SArd Biesheuvel */
3972dc43a9SRabin Vincent #ifdef CONFIG_THUMB2_KERNEL
40ad1c2f39SArd Biesheuvel #define NOP 0xf10d0d04 /* add.w sp, sp, #4 */
4172dc43a9SRabin Vincent #else
42ad1c2f39SArd Biesheuvel #define NOP 0xe28dd004 /* add sp, sp, #4 */
4372dc43a9SRabin Vincent #endif
44014c257cSAbhishek Sagar
45376cfa87STim Bird #ifdef CONFIG_DYNAMIC_FTRACE
463b6c223bSRabin Vincent
__ftrace_modify_code(void * data)4780d6b0c2SKees Cook static int __ftrace_modify_code(void *data)
4880d6b0c2SKees Cook {
4980d6b0c2SKees Cook int *command = data;
5080d6b0c2SKees Cook
5180d6b0c2SKees Cook ftrace_modify_all_code(*command);
5280d6b0c2SKees Cook
5380d6b0c2SKees Cook return 0;
5480d6b0c2SKees Cook }
5580d6b0c2SKees Cook
arch_ftrace_update_code(int command)5680d6b0c2SKees Cook void arch_ftrace_update_code(int command)
5780d6b0c2SKees Cook {
5880d6b0c2SKees Cook stop_machine(__ftrace_modify_code, &command, NULL);
5980d6b0c2SKees Cook }
6080d6b0c2SKees Cook
ftrace_nop_replace(struct dyn_ftrace * rec)613b6c223bSRabin Vincent static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec)
623b6c223bSRabin Vincent {
633b6c223bSRabin Vincent return NOP;
643b6c223bSRabin Vincent }
653b6c223bSRabin Vincent
66dc438db5SArd Biesheuvel void ftrace_caller_from_init(void);
67dc438db5SArd Biesheuvel void ftrace_regs_caller_from_init(void);
68dc438db5SArd Biesheuvel
adjust_address(struct dyn_ftrace * rec,unsigned long addr)69dc438db5SArd Biesheuvel static unsigned long __ref adjust_address(struct dyn_ftrace *rec,
70dc438db5SArd Biesheuvel unsigned long addr)
713b6c223bSRabin Vincent {
72dc438db5SArd Biesheuvel if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE) ||
73dc438db5SArd Biesheuvel system_state >= SYSTEM_FREEING_INITMEM ||
74dc438db5SArd Biesheuvel likely(!is_kernel_inittext(rec->ip)))
753b6c223bSRabin Vincent return addr;
76dc438db5SArd Biesheuvel if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) ||
77dc438db5SArd Biesheuvel addr == (unsigned long)&ftrace_caller)
78dc438db5SArd Biesheuvel return (unsigned long)&ftrace_caller_from_init;
79dc438db5SArd Biesheuvel return (unsigned long)&ftrace_regs_caller_from_init;
803b6c223bSRabin Vincent }
813b6c223bSRabin Vincent
ftrace_arch_code_modify_prepare(void)82*3a2bfec0SLi kunyu void ftrace_arch_code_modify_prepare(void)
83a672917aSRabin Vincent {
84a672917aSRabin Vincent }
85a672917aSRabin Vincent
ftrace_arch_code_modify_post_process(void)86*3a2bfec0SLi kunyu void ftrace_arch_code_modify_post_process(void)
87a672917aSRabin Vincent {
8880d6b0c2SKees Cook /* Make sure any TLB misses during machine stop are cleared. */
8980d6b0c2SKees Cook flush_tlb_all();
90a672917aSRabin Vincent }
91a672917aSRabin Vincent
ftrace_call_replace(unsigned long pc,unsigned long addr,bool warn)9279f32b22SAlex Sverdlin static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr,
9379f32b22SAlex Sverdlin bool warn)
94dd686eb1SRabin Vincent {
9579f32b22SAlex Sverdlin return arm_gen_branch_link(pc, addr, warn);
96dd686eb1SRabin Vincent }
97dd686eb1SRabin Vincent
ftrace_modify_code(unsigned long pc,unsigned long old,unsigned long new,bool validate)983b6c223bSRabin Vincent static int ftrace_modify_code(unsigned long pc, unsigned long old,
99dc283d70SRabin Vincent unsigned long new, bool validate)
100014c257cSAbhishek Sagar {
1013b6c223bSRabin Vincent unsigned long replaced;
102014c257cSAbhishek Sagar
103be993e44SPeter Zijlstra if (IS_ENABLED(CONFIG_THUMB2_KERNEL))
1044394e282SRabin Vincent old = __opcode_to_mem_thumb32(old);
105be993e44SPeter Zijlstra else
1064394e282SRabin Vincent old = __opcode_to_mem_arm(old);
1074394e282SRabin Vincent
108dc283d70SRabin Vincent if (validate) {
109fe557319SChristoph Hellwig if (copy_from_kernel_nofault(&replaced, (void *)pc,
110fe557319SChristoph Hellwig MCOUNT_INSN_SIZE))
1113b6c223bSRabin Vincent return -EFAULT;
112014c257cSAbhishek Sagar
1133b6c223bSRabin Vincent if (replaced != old)
1143b6c223bSRabin Vincent return -EINVAL;
115dc283d70SRabin Vincent }
116014c257cSAbhishek Sagar
1175a735583SPeter Zijlstra __patch_text((void *)pc, new);
118014c257cSAbhishek Sagar
1193b6c223bSRabin Vincent return 0;
120014c257cSAbhishek Sagar }
121014c257cSAbhishek Sagar
ftrace_update_ftrace_func(ftrace_func_t func)122014c257cSAbhishek Sagar int ftrace_update_ftrace_func(ftrace_func_t func)
123014c257cSAbhishek Sagar {
124dc283d70SRabin Vincent unsigned long pc;
1253b6c223bSRabin Vincent unsigned long new;
1263b6c223bSRabin Vincent int ret;
127014c257cSAbhishek Sagar
128014c257cSAbhishek Sagar pc = (unsigned long)&ftrace_call;
12979f32b22SAlex Sverdlin new = ftrace_call_replace(pc, (unsigned long)func, true);
1303b6c223bSRabin Vincent
131dc283d70SRabin Vincent ret = ftrace_modify_code(pc, 0, new, false);
1323b6c223bSRabin Vincent
133620176f3SAbel Vesa #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
134620176f3SAbel Vesa if (!ret) {
135620176f3SAbel Vesa pc = (unsigned long)&ftrace_regs_call;
13679f32b22SAlex Sverdlin new = ftrace_call_replace(pc, (unsigned long)func, true);
137620176f3SAbel Vesa
138620176f3SAbel Vesa ret = ftrace_modify_code(pc, 0, new, false);
139620176f3SAbel Vesa }
140620176f3SAbel Vesa #endif
141620176f3SAbel Vesa
142014c257cSAbhishek Sagar return ret;
143014c257cSAbhishek Sagar }
144014c257cSAbhishek Sagar
ftrace_make_call(struct dyn_ftrace * rec,unsigned long addr)1453b6c223bSRabin Vincent int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
1463b6c223bSRabin Vincent {
1473b6c223bSRabin Vincent unsigned long new, old;
1483b6c223bSRabin Vincent unsigned long ip = rec->ip;
14979f32b22SAlex Sverdlin unsigned long aaddr = adjust_address(rec, addr);
15079f32b22SAlex Sverdlin struct module *mod = NULL;
15179f32b22SAlex Sverdlin
15279f32b22SAlex Sverdlin #ifdef CONFIG_ARM_MODULE_PLTS
15379f32b22SAlex Sverdlin mod = rec->arch.mod;
15479f32b22SAlex Sverdlin #endif
1553b6c223bSRabin Vincent
1563b6c223bSRabin Vincent old = ftrace_nop_replace(rec);
157620176f3SAbel Vesa
15879f32b22SAlex Sverdlin new = ftrace_call_replace(ip, aaddr, !mod);
15979f32b22SAlex Sverdlin #ifdef CONFIG_ARM_MODULE_PLTS
16079f32b22SAlex Sverdlin if (!new && mod) {
16179f32b22SAlex Sverdlin aaddr = get_module_plt(mod, ip, aaddr);
16279f32b22SAlex Sverdlin new = ftrace_call_replace(ip, aaddr, true);
16379f32b22SAlex Sverdlin }
16479f32b22SAlex Sverdlin #endif
1653b6c223bSRabin Vincent
166dc283d70SRabin Vincent return ftrace_modify_code(rec->ip, old, new, true);
1673b6c223bSRabin Vincent }
1683b6c223bSRabin Vincent
169620176f3SAbel Vesa #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
170620176f3SAbel Vesa
ftrace_modify_call(struct dyn_ftrace * rec,unsigned long old_addr,unsigned long addr)171620176f3SAbel Vesa int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
172620176f3SAbel Vesa unsigned long addr)
173620176f3SAbel Vesa {
174620176f3SAbel Vesa unsigned long new, old;
175620176f3SAbel Vesa unsigned long ip = rec->ip;
176620176f3SAbel Vesa
17779f32b22SAlex Sverdlin old = ftrace_call_replace(ip, adjust_address(rec, old_addr), true);
178620176f3SAbel Vesa
17979f32b22SAlex Sverdlin new = ftrace_call_replace(ip, adjust_address(rec, addr), true);
180620176f3SAbel Vesa
181620176f3SAbel Vesa return ftrace_modify_code(rec->ip, old, new, true);
182620176f3SAbel Vesa }
183620176f3SAbel Vesa
184620176f3SAbel Vesa #endif
185620176f3SAbel Vesa
ftrace_make_nop(struct module * mod,struct dyn_ftrace * rec,unsigned long addr)1863b6c223bSRabin Vincent int ftrace_make_nop(struct module *mod,
1873b6c223bSRabin Vincent struct dyn_ftrace *rec, unsigned long addr)
1883b6c223bSRabin Vincent {
18979f32b22SAlex Sverdlin unsigned long aaddr = adjust_address(rec, addr);
1903b6c223bSRabin Vincent unsigned long ip = rec->ip;
1913b6c223bSRabin Vincent unsigned long old;
1923b6c223bSRabin Vincent unsigned long new;
1933b6c223bSRabin Vincent int ret;
1943b6c223bSRabin Vincent
19579f32b22SAlex Sverdlin #ifdef CONFIG_ARM_MODULE_PLTS
19679f32b22SAlex Sverdlin /* mod is only supplied during module loading */
19779f32b22SAlex Sverdlin if (!mod)
19879f32b22SAlex Sverdlin mod = rec->arch.mod;
19979f32b22SAlex Sverdlin else
20079f32b22SAlex Sverdlin rec->arch.mod = mod;
20179f32b22SAlex Sverdlin #endif
20279f32b22SAlex Sverdlin
20379f32b22SAlex Sverdlin old = ftrace_call_replace(ip, aaddr,
20479f32b22SAlex Sverdlin !IS_ENABLED(CONFIG_ARM_MODULE_PLTS) || !mod);
20579f32b22SAlex Sverdlin #ifdef CONFIG_ARM_MODULE_PLTS
20679f32b22SAlex Sverdlin if (!old && mod) {
20779f32b22SAlex Sverdlin aaddr = get_module_plt(mod, ip, aaddr);
20879f32b22SAlex Sverdlin old = ftrace_call_replace(ip, aaddr, true);
20979f32b22SAlex Sverdlin }
21079f32b22SAlex Sverdlin #endif
21179f32b22SAlex Sverdlin
2123b6c223bSRabin Vincent new = ftrace_nop_replace(rec);
213dc438db5SArd Biesheuvel /*
214dc438db5SArd Biesheuvel * Locations in .init.text may call __gnu_mcount_mc via a linker
215dc438db5SArd Biesheuvel * emitted veneer if they are too far away from its implementation, and
216dc438db5SArd Biesheuvel * so validation may fail spuriously in such cases. Let's work around
217dc438db5SArd Biesheuvel * this by omitting those from validation.
218dc438db5SArd Biesheuvel */
219dc438db5SArd Biesheuvel ret = ftrace_modify_code(ip, old, new, !is_kernel_inittext(ip));
2203b6c223bSRabin Vincent
2213b6c223bSRabin Vincent return ret;
2223b6c223bSRabin Vincent }
223376cfa87STim Bird #endif /* CONFIG_DYNAMIC_FTRACE */
224376cfa87STim Bird
225376cfa87STim Bird #ifdef CONFIG_FUNCTION_GRAPH_TRACER
22641918ec8SArd Biesheuvel asmlinkage
prepare_ftrace_return(unsigned long * parent,unsigned long self_addr,unsigned long frame_pointer,unsigned long stack_pointer)227376cfa87STim Bird void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
22841918ec8SArd Biesheuvel unsigned long frame_pointer,
22941918ec8SArd Biesheuvel unsigned long stack_pointer)
230376cfa87STim Bird {
231376cfa87STim Bird unsigned long return_hooker = (unsigned long) &return_to_handler;
232376cfa87STim Bird unsigned long old;
233376cfa87STim Bird
234376cfa87STim Bird if (unlikely(atomic_read(¤t->tracing_graph_pause)))
235376cfa87STim Bird return;
236376cfa87STim Bird
237953f534aSArd Biesheuvel if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER)) {
238953f534aSArd Biesheuvel /* FP points one word below parent's top of stack */
239953f534aSArd Biesheuvel frame_pointer += 4;
24041918ec8SArd Biesheuvel } else {
24141918ec8SArd Biesheuvel struct stackframe frame = {
24241918ec8SArd Biesheuvel .fp = frame_pointer,
24341918ec8SArd Biesheuvel .sp = stack_pointer,
24441918ec8SArd Biesheuvel .lr = self_addr,
24541918ec8SArd Biesheuvel .pc = self_addr,
24641918ec8SArd Biesheuvel };
24741918ec8SArd Biesheuvel if (unwind_frame(&frame) < 0)
24841918ec8SArd Biesheuvel return;
24941918ec8SArd Biesheuvel if (frame.lr != self_addr)
25041918ec8SArd Biesheuvel parent = frame.lr_addr;
25141918ec8SArd Biesheuvel frame_pointer = frame.sp;
252953f534aSArd Biesheuvel }
253953f534aSArd Biesheuvel
254376cfa87STim Bird old = *parent;
255376cfa87STim Bird *parent = return_hooker;
256376cfa87STim Bird
257f1f5b14aSSteven Rostedt (VMware) if (function_graph_enter(old, self_addr, frame_pointer, NULL))
2584c36595eSColin Cross *parent = old;
259376cfa87STim Bird }
260dd686eb1SRabin Vincent
261dd686eb1SRabin Vincent #ifdef CONFIG_DYNAMIC_FTRACE
262dd686eb1SRabin Vincent extern unsigned long ftrace_graph_call;
263dd686eb1SRabin Vincent extern unsigned long ftrace_graph_call_old;
264dd686eb1SRabin Vincent extern void ftrace_graph_caller_old(void);
265620176f3SAbel Vesa extern unsigned long ftrace_graph_regs_call;
266620176f3SAbel Vesa extern void ftrace_graph_regs_caller(void);
267dd686eb1SRabin Vincent
__ftrace_modify_caller(unsigned long * callsite,void (* func)(void),bool enable)268dd686eb1SRabin Vincent static int __ftrace_modify_caller(unsigned long *callsite,
269dd686eb1SRabin Vincent void (*func) (void), bool enable)
270dd686eb1SRabin Vincent {
271dd686eb1SRabin Vincent unsigned long caller_fn = (unsigned long) func;
272dd686eb1SRabin Vincent unsigned long pc = (unsigned long) callsite;
273d82227cfSRabin Vincent unsigned long branch = arm_gen_branch(pc, caller_fn);
27441918ec8SArd Biesheuvel unsigned long nop = arm_gen_nop();
275dd686eb1SRabin Vincent unsigned long old = enable ? nop : branch;
276dd686eb1SRabin Vincent unsigned long new = enable ? branch : nop;
277dd686eb1SRabin Vincent
278dc283d70SRabin Vincent return ftrace_modify_code(pc, old, new, true);
279dd686eb1SRabin Vincent }
280dd686eb1SRabin Vincent
ftrace_modify_graph_caller(bool enable)281dd686eb1SRabin Vincent static int ftrace_modify_graph_caller(bool enable)
282dd686eb1SRabin Vincent {
283dd686eb1SRabin Vincent int ret;
284dd686eb1SRabin Vincent
285dd686eb1SRabin Vincent ret = __ftrace_modify_caller(&ftrace_graph_call,
286dd686eb1SRabin Vincent ftrace_graph_caller,
287dd686eb1SRabin Vincent enable);
288dd686eb1SRabin Vincent
289620176f3SAbel Vesa #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
290620176f3SAbel Vesa if (!ret)
291620176f3SAbel Vesa ret = __ftrace_modify_caller(&ftrace_graph_regs_call,
292620176f3SAbel Vesa ftrace_graph_regs_caller,
293620176f3SAbel Vesa enable);
294620176f3SAbel Vesa #endif
295620176f3SAbel Vesa
296620176f3SAbel Vesa
297dd686eb1SRabin Vincent return ret;
298dd686eb1SRabin Vincent }
299dd686eb1SRabin Vincent
ftrace_enable_ftrace_graph_caller(void)300dd686eb1SRabin Vincent int ftrace_enable_ftrace_graph_caller(void)
301dd686eb1SRabin Vincent {
302dd686eb1SRabin Vincent return ftrace_modify_graph_caller(true);
303dd686eb1SRabin Vincent }
304dd686eb1SRabin Vincent
ftrace_disable_ftrace_graph_caller(void)305dd686eb1SRabin Vincent int ftrace_disable_ftrace_graph_caller(void)
306dd686eb1SRabin Vincent {
307dd686eb1SRabin Vincent return ftrace_modify_graph_caller(false);
308dd686eb1SRabin Vincent }
309dd686eb1SRabin Vincent #endif /* CONFIG_DYNAMIC_FTRACE */
310376cfa87STim Bird #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
311