1d2912cb1SThomas Gleixner/* SPDX-License-Identifier: GPL-2.0-only */ 282112379SRussell King 382112379SRussell King#include <asm/assembler.h> 482112379SRussell King#include <asm/ftrace.h> 582112379SRussell King#include <asm/unwind.h> 682112379SRussell King 782112379SRussell King#include "entry-header.S" 882112379SRussell King 982112379SRussell King/* 1082112379SRussell King * When compiling with -pg, gcc inserts a call to the mcount routine at the 1182112379SRussell King * start of every function. In mcount, apart from the function's address (in 1282112379SRussell King * lr), we need to get hold of the function's caller's address. 1382112379SRussell King * 14d3c61619SStefan Agner * Newer GCCs (4.4+) solve this problem by using a version of mcount with call 15d3c61619SStefan Agner * sites like: 1682112379SRussell King * 1782112379SRussell King * push {lr} 1882112379SRussell King * bl __gnu_mcount_nc 1982112379SRussell King * 2082112379SRussell King * With these compilers, frame pointers are not necessary. 2182112379SRussell King * 2282112379SRussell King * mcount can be thought of as a function called in the middle of a subroutine 2382112379SRussell King * call. As such, it needs to be transparent for both the caller and the 2482112379SRussell King * callee: the original lr needs to be restored when leaving mcount, and no 2582112379SRussell King * registers should be clobbered. (In the __gnu_mcount_nc implementation, we 2682112379SRussell King * clobber the ip register. This is OK because the ARM calling convention 2782112379SRussell King * allows it to be clobbered in subroutines and doesn't use it to hold 2882112379SRussell King * parameters.) 2982112379SRussell King * 30ad1c2f39SArd Biesheuvel * When using dynamic ftrace, we patch out the mcount call by a "add sp, #4" 31d3c61619SStefan Agner * instead of the __gnu_mcount_nc call (see arch/arm/kernel/ftrace.c). 3282112379SRussell King */ 3382112379SRussell King 3482112379SRussell King.macro mcount_adjust_addr rd, rn 3582112379SRussell King bic \rd, \rn, #1 @ clear the Thumb bit if present 3682112379SRussell King sub \rd, \rd, #MCOUNT_INSN_SIZE 3782112379SRussell King.endm 3882112379SRussell King 3982112379SRussell King.macro __mcount suffix 4082112379SRussell King mcount_enter 4182112379SRussell King ldr r0, =ftrace_trace_function 4282112379SRussell King ldr r2, [r0] 43dd88b03fSArd Biesheuvel badr r0, .Lftrace_stub 4482112379SRussell King cmp r0, r2 4582112379SRussell King bne 1f 4682112379SRussell King 4782112379SRussell King#ifdef CONFIG_FUNCTION_GRAPH_TRACER 4882112379SRussell King ldr r1, =ftrace_graph_return 4982112379SRussell King ldr r2, [r1] 5082112379SRussell King cmp r0, r2 5182112379SRussell King bne ftrace_graph_caller\suffix 5282112379SRussell King 5382112379SRussell King ldr r1, =ftrace_graph_entry 5482112379SRussell King ldr r2, [r1] 5582112379SRussell King ldr r0, =ftrace_graph_entry_stub 5682112379SRussell King cmp r0, r2 5782112379SRussell King bne ftrace_graph_caller\suffix 5882112379SRussell King#endif 5982112379SRussell King 6082112379SRussell King mcount_exit 6182112379SRussell King 6282112379SRussell King1: mcount_get_lr r1 @ lr of instrumented func 6382112379SRussell King mcount_adjust_addr r0, lr @ instrumented function 6414327c66SRussell King badr lr, 2f 6582112379SRussell King mov pc, r2 6682112379SRussell King2: mcount_exit 6782112379SRussell King.endm 6882112379SRussell King 69620176f3SAbel Vesa#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 70620176f3SAbel Vesa 71620176f3SAbel Vesa.macro __ftrace_regs_caller 72620176f3SAbel Vesa 73620176f3SAbel Vesa sub sp, sp, #8 @ space for PC and CPSR OLD_R0, 74620176f3SAbel Vesa @ OLD_R0 will overwrite previous LR 75620176f3SAbel Vesa 76620176f3SAbel Vesa add ip, sp, #12 @ move in IP the value of SP as it was 77620176f3SAbel Vesa @ before the push {lr} of the mcount mechanism 78620176f3SAbel Vesa 79620176f3SAbel Vesa str lr, [sp, #0] @ store LR instead of PC 80620176f3SAbel Vesa 81620176f3SAbel Vesa ldr lr, [sp, #8] @ get previous LR 82620176f3SAbel Vesa 83620176f3SAbel Vesa str r0, [sp, #8] @ write r0 as OLD_R0 over previous LR 84620176f3SAbel Vesa 85620176f3SAbel Vesa stmdb sp!, {ip, lr} 86620176f3SAbel Vesa stmdb sp!, {r0-r11, lr} 87620176f3SAbel Vesa 88620176f3SAbel Vesa @ stack content at this point: 89620176f3SAbel Vesa @ 0 4 48 52 56 60 64 68 72 90620176f3SAbel Vesa @ R0 | R1 | ... | LR | SP + 4 | previous LR | LR | PSR | OLD_R0 | 91620176f3SAbel Vesa 92620176f3SAbel Vesa mov r3, sp @ struct pt_regs* 93620176f3SAbel Vesa 94620176f3SAbel Vesa ldr r2, =function_trace_op 95620176f3SAbel Vesa ldr r2, [r2] @ pointer to the current 96620176f3SAbel Vesa @ function tracing op 97620176f3SAbel Vesa 98620176f3SAbel Vesa ldr r1, [sp, #S_LR] @ lr of instrumented func 99620176f3SAbel Vesa 100620176f3SAbel Vesa ldr lr, [sp, #S_PC] @ get LR 101620176f3SAbel Vesa 102620176f3SAbel Vesa mcount_adjust_addr r0, lr @ instrumented function 103620176f3SAbel Vesa 104620176f3SAbel Vesa .globl ftrace_regs_call 105620176f3SAbel Vesaftrace_regs_call: 106620176f3SAbel Vesa bl ftrace_stub 107620176f3SAbel Vesa 108620176f3SAbel Vesa#ifdef CONFIG_FUNCTION_GRAPH_TRACER 109620176f3SAbel Vesa .globl ftrace_graph_regs_call 110620176f3SAbel Vesaftrace_graph_regs_call: 111620176f3SAbel Vesa mov r0, r0 112620176f3SAbel Vesa#endif 113620176f3SAbel Vesa 114620176f3SAbel Vesa @ pop saved regs 115620176f3SAbel Vesa ldmia sp!, {r0-r12} @ restore r0 through r12 116620176f3SAbel Vesa ldr ip, [sp, #8] @ restore PC 117620176f3SAbel Vesa ldr lr, [sp, #4] @ restore LR 118620176f3SAbel Vesa ldr sp, [sp, #0] @ restore SP 119620176f3SAbel Vesa mov pc, ip @ return 120620176f3SAbel Vesa.endm 121620176f3SAbel Vesa 122620176f3SAbel Vesa#ifdef CONFIG_FUNCTION_GRAPH_TRACER 123620176f3SAbel Vesa.macro __ftrace_graph_regs_caller 124620176f3SAbel Vesa 125620176f3SAbel Vesa sub r0, fp, #4 @ lr of instrumented routine (parent) 126620176f3SAbel Vesa 127620176f3SAbel Vesa @ called from __ftrace_regs_caller 128620176f3SAbel Vesa ldr r1, [sp, #S_PC] @ instrumented routine (func) 129620176f3SAbel Vesa mcount_adjust_addr r1, r1 130620176f3SAbel Vesa 131620176f3SAbel Vesa mov r2, fp @ frame pointer 132620176f3SAbel Vesa bl prepare_ftrace_return 133620176f3SAbel Vesa 134620176f3SAbel Vesa @ pop registers saved in ftrace_regs_caller 135620176f3SAbel Vesa ldmia sp!, {r0-r12} @ restore r0 through r12 136620176f3SAbel Vesa ldr ip, [sp, #8] @ restore PC 137620176f3SAbel Vesa ldr lr, [sp, #4] @ restore LR 138620176f3SAbel Vesa ldr sp, [sp, #0] @ restore SP 139620176f3SAbel Vesa mov pc, ip @ return 140620176f3SAbel Vesa 141620176f3SAbel Vesa.endm 142620176f3SAbel Vesa#endif 143620176f3SAbel Vesa#endif 144620176f3SAbel Vesa 14582112379SRussell King.macro __ftrace_caller suffix 14682112379SRussell King mcount_enter 14782112379SRussell King 14882112379SRussell King mcount_get_lr r1 @ lr of instrumented func 14982112379SRussell King mcount_adjust_addr r0, lr @ instrumented function 15082112379SRussell King 151620176f3SAbel Vesa#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 152620176f3SAbel Vesa ldr r2, =function_trace_op 153620176f3SAbel Vesa ldr r2, [r2] @ pointer to the current 154620176f3SAbel Vesa @ function tracing op 155620176f3SAbel Vesa mov r3, #0 @ regs is NULL 156620176f3SAbel Vesa#endif 157620176f3SAbel Vesa 15882112379SRussell King .globl ftrace_call\suffix 15982112379SRussell Kingftrace_call\suffix: 16082112379SRussell King bl ftrace_stub 16182112379SRussell King 16282112379SRussell King#ifdef CONFIG_FUNCTION_GRAPH_TRACER 16382112379SRussell King .globl ftrace_graph_call\suffix 16482112379SRussell Kingftrace_graph_call\suffix: 16582112379SRussell King mov r0, r0 16682112379SRussell King#endif 16782112379SRussell King 16882112379SRussell King mcount_exit 16982112379SRussell King.endm 17082112379SRussell King 17182112379SRussell King.macro __ftrace_graph_caller 17282112379SRussell King sub r0, fp, #4 @ &lr of instrumented routine (&parent) 17382112379SRussell King#ifdef CONFIG_DYNAMIC_FTRACE 17482112379SRussell King @ called from __ftrace_caller, saved in mcount_enter 17582112379SRussell King ldr r1, [sp, #16] @ instrumented routine (func) 17682112379SRussell King mcount_adjust_addr r1, r1 17782112379SRussell King#else 17882112379SRussell King @ called from __mcount, untouched in lr 17982112379SRussell King mcount_adjust_addr r1, lr @ instrumented routine (func) 18082112379SRussell King#endif 18182112379SRussell King mov r2, fp @ frame pointer 18282112379SRussell King bl prepare_ftrace_return 18382112379SRussell King mcount_exit 18482112379SRussell King.endm 18582112379SRussell King 18682112379SRussell King/* 18782112379SRussell King * __gnu_mcount_nc 18882112379SRussell King */ 18982112379SRussell King 19082112379SRussell King.macro mcount_enter 19182112379SRussell King/* 19282112379SRussell King * This pad compensates for the push {lr} at the call site. Note that we are 19382112379SRussell King * unable to unwind through a function which does not otherwise save its lr. 19482112379SRussell King */ 19582112379SRussell King UNWIND(.pad #4) 19682112379SRussell King stmdb sp!, {r0-r3, lr} 19782112379SRussell King UNWIND(.save {r0-r3, lr}) 19882112379SRussell King.endm 19982112379SRussell King 20082112379SRussell King.macro mcount_get_lr reg 20182112379SRussell King ldr \reg, [sp, #20] 20282112379SRussell King.endm 20382112379SRussell King 20482112379SRussell King.macro mcount_exit 20582112379SRussell King ldmia sp!, {r0-r3, ip, lr} 20682112379SRussell King ret ip 20782112379SRussell King.endm 20882112379SRussell King 20982112379SRussell KingENTRY(__gnu_mcount_nc) 21082112379SRussell KingUNWIND(.fnstart) 21182112379SRussell King#ifdef CONFIG_DYNAMIC_FTRACE 21282112379SRussell King mov ip, lr 21382112379SRussell King ldmia sp!, {lr} 21482112379SRussell King ret ip 21582112379SRussell King#else 21682112379SRussell King __mcount 21782112379SRussell King#endif 21882112379SRussell KingUNWIND(.fnend) 21982112379SRussell KingENDPROC(__gnu_mcount_nc) 22082112379SRussell King 22182112379SRussell King#ifdef CONFIG_DYNAMIC_FTRACE 22282112379SRussell KingENTRY(ftrace_caller) 22382112379SRussell KingUNWIND(.fnstart) 22482112379SRussell King __ftrace_caller 22582112379SRussell KingUNWIND(.fnend) 22682112379SRussell KingENDPROC(ftrace_caller) 227620176f3SAbel Vesa 228620176f3SAbel Vesa#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 229620176f3SAbel VesaENTRY(ftrace_regs_caller) 230620176f3SAbel VesaUNWIND(.fnstart) 231620176f3SAbel Vesa __ftrace_regs_caller 232620176f3SAbel VesaUNWIND(.fnend) 233620176f3SAbel VesaENDPROC(ftrace_regs_caller) 234620176f3SAbel Vesa#endif 235620176f3SAbel Vesa 23682112379SRussell King#endif 23782112379SRussell King 23882112379SRussell King#ifdef CONFIG_FUNCTION_GRAPH_TRACER 23982112379SRussell KingENTRY(ftrace_graph_caller) 24082112379SRussell KingUNWIND(.fnstart) 24182112379SRussell King __ftrace_graph_caller 24282112379SRussell KingUNWIND(.fnend) 24382112379SRussell KingENDPROC(ftrace_graph_caller) 244620176f3SAbel Vesa 245620176f3SAbel Vesa#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 246620176f3SAbel VesaENTRY(ftrace_graph_regs_caller) 247620176f3SAbel VesaUNWIND(.fnstart) 248620176f3SAbel Vesa __ftrace_graph_regs_caller 249620176f3SAbel VesaUNWIND(.fnend) 250620176f3SAbel VesaENDPROC(ftrace_graph_regs_caller) 251620176f3SAbel Vesa#endif 25282112379SRussell King#endif 25382112379SRussell King 25482112379SRussell King.purgem mcount_enter 25582112379SRussell King.purgem mcount_get_lr 25682112379SRussell King.purgem mcount_exit 25782112379SRussell King 25882112379SRussell King#ifdef CONFIG_FUNCTION_GRAPH_TRACER 25982112379SRussell King .globl return_to_handler 26082112379SRussell Kingreturn_to_handler: 26182112379SRussell King stmdb sp!, {r0-r3} 26282112379SRussell King mov r0, fp @ frame pointer 26382112379SRussell King bl ftrace_return_to_handler 26482112379SRussell King mov lr, r0 @ r0 has real ret addr 26582112379SRussell King ldmia sp!, {r0-r3} 26682112379SRussell King ret lr 26782112379SRussell King#endif 26882112379SRussell King 26982112379SRussell KingENTRY(ftrace_stub) 27082112379SRussell King.Lftrace_stub: 27182112379SRussell King ret lr 27282112379SRussell KingENDPROC(ftrace_stub) 273*dc438db5SArd Biesheuvel 274*dc438db5SArd Biesheuvel#ifdef CONFIG_DYNAMIC_FTRACE 275*dc438db5SArd Biesheuvel 276*dc438db5SArd Biesheuvel __INIT 277*dc438db5SArd Biesheuvel 278*dc438db5SArd Biesheuvel .macro init_tramp, dst:req 279*dc438db5SArd BiesheuvelENTRY(\dst\()_from_init) 280*dc438db5SArd Biesheuvel ldr pc, =\dst 281*dc438db5SArd BiesheuvelENDPROC(\dst\()_from_init) 282*dc438db5SArd Biesheuvel .endm 283*dc438db5SArd Biesheuvel 284*dc438db5SArd Biesheuvel init_tramp ftrace_caller 285*dc438db5SArd Biesheuvel#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 286*dc438db5SArd Biesheuvel init_tramp ftrace_regs_caller 287*dc438db5SArd Biesheuvel#endif 288*dc438db5SArd Biesheuvel#endif 289