1d2912cb1SThomas Gleixner/* SPDX-License-Identifier: GPL-2.0-only */ 282112379SRussell King 382112379SRussell King#include <asm/assembler.h> 482112379SRussell King#include <asm/ftrace.h> 582112379SRussell King#include <asm/unwind.h> 682112379SRussell King 782112379SRussell King#include "entry-header.S" 882112379SRussell King 982112379SRussell King/* 1082112379SRussell King * When compiling with -pg, gcc inserts a call to the mcount routine at the 1182112379SRussell King * start of every function. In mcount, apart from the function's address (in 1282112379SRussell King * lr), we need to get hold of the function's caller's address. 1382112379SRussell King * 14d3c61619SStefan Agner * Newer GCCs (4.4+) solve this problem by using a version of mcount with call 15d3c61619SStefan Agner * sites like: 1682112379SRussell King * 1782112379SRussell King * push {lr} 1882112379SRussell King * bl __gnu_mcount_nc 1982112379SRussell King * 2082112379SRussell King * With these compilers, frame pointers are not necessary. 2182112379SRussell King * 2282112379SRussell King * mcount can be thought of as a function called in the middle of a subroutine 2382112379SRussell King * call. As such, it needs to be transparent for both the caller and the 2482112379SRussell King * callee: the original lr needs to be restored when leaving mcount, and no 25d1196787SArd Biesheuvel * registers should be clobbered. 2682112379SRussell King * 27ad1c2f39SArd Biesheuvel * When using dynamic ftrace, we patch out the mcount call by a "add sp, #4" 28d3c61619SStefan Agner * instead of the __gnu_mcount_nc call (see arch/arm/kernel/ftrace.c). 2982112379SRussell King */ 3082112379SRussell King 3182112379SRussell King.macro mcount_adjust_addr rd, rn 3282112379SRussell King bic \rd, \rn, #1 @ clear the Thumb bit if present 3382112379SRussell King sub \rd, \rd, #MCOUNT_INSN_SIZE 3482112379SRussell King.endm 3582112379SRussell King 3682112379SRussell King.macro __mcount suffix 3782112379SRussell King mcount_enter 3865aa7e34SArd Biesheuvel ldr_va r2, ftrace_trace_function 39dd88b03fSArd Biesheuvel badr r0, .Lftrace_stub 4082112379SRussell King cmp r0, r2 4182112379SRussell King bne 1f 4282112379SRussell King 4382112379SRussell King#ifdef CONFIG_FUNCTION_GRAPH_TRACER 4465aa7e34SArd Biesheuvel ldr_va r2, ftrace_graph_return 4582112379SRussell King cmp r0, r2 4682112379SRussell King bne ftrace_graph_caller\suffix 4782112379SRussell King 4865aa7e34SArd Biesheuvel ldr_va r2, ftrace_graph_entry 4965aa7e34SArd Biesheuvel mov_l r0, ftrace_graph_entry_stub 5082112379SRussell King cmp r0, r2 5182112379SRussell King bne ftrace_graph_caller\suffix 5282112379SRussell King#endif 5382112379SRussell King 5482112379SRussell King mcount_exit 5582112379SRussell King 5682112379SRussell King1: mcount_get_lr r1 @ lr of instrumented func 5782112379SRussell King mcount_adjust_addr r0, lr @ instrumented function 5814327c66SRussell King badr lr, 2f 5982112379SRussell King mov pc, r2 6082112379SRussell King2: mcount_exit 6182112379SRussell King.endm 6282112379SRussell King 63620176f3SAbel Vesa#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 64620176f3SAbel Vesa 65620176f3SAbel Vesa.macro __ftrace_regs_caller 66620176f3SAbel Vesa 67d1196787SArd Biesheuvel str lr, [sp, #-8]! @ store LR as PC and make space for CPSR/OLD_R0, 68620176f3SAbel Vesa @ OLD_R0 will overwrite previous LR 69620176f3SAbel Vesa 70620176f3SAbel Vesa ldr lr, [sp, #8] @ get previous LR 71620176f3SAbel Vesa 72620176f3SAbel Vesa str r0, [sp, #8] @ write r0 as OLD_R0 over previous LR 73620176f3SAbel Vesa 74d1196787SArd Biesheuvel str lr, [sp, #-4]! @ store previous LR as LR 75d1196787SArd Biesheuvel 76d1196787SArd Biesheuvel add lr, sp, #16 @ move in LR the value of SP as it was 77d1196787SArd Biesheuvel @ before the push {lr} of the mcount mechanism 78d1196787SArd Biesheuvel 79d1196787SArd Biesheuvel push {r0-r11, ip, lr} 80620176f3SAbel Vesa 81620176f3SAbel Vesa @ stack content at this point: 82620176f3SAbel Vesa @ 0 4 48 52 56 60 64 68 72 83d1196787SArd Biesheuvel @ R0 | R1 | ... | IP | SP + 4 | previous LR | LR | PSR | OLD_R0 | 84620176f3SAbel Vesa 85620176f3SAbel Vesa mov r3, sp @ struct pt_regs* 86620176f3SAbel Vesa 8765aa7e34SArd Biesheuvel ldr_va r2, function_trace_op @ pointer to the current 88620176f3SAbel Vesa @ function tracing op 89620176f3SAbel Vesa 90620176f3SAbel Vesa ldr r1, [sp, #S_LR] @ lr of instrumented func 91620176f3SAbel Vesa 92620176f3SAbel Vesa ldr lr, [sp, #S_PC] @ get LR 93620176f3SAbel Vesa 94620176f3SAbel Vesa mcount_adjust_addr r0, lr @ instrumented function 95620176f3SAbel Vesa 96620176f3SAbel Vesa .globl ftrace_regs_call 97620176f3SAbel Vesaftrace_regs_call: 98620176f3SAbel Vesa bl ftrace_stub 99620176f3SAbel Vesa 100620176f3SAbel Vesa#ifdef CONFIG_FUNCTION_GRAPH_TRACER 101620176f3SAbel Vesa .globl ftrace_graph_regs_call 102620176f3SAbel Vesaftrace_graph_regs_call: 103*41918ec8SArd BiesheuvelARM( mov r0, r0 ) 104*41918ec8SArd BiesheuvelTHUMB( nop.w ) 105620176f3SAbel Vesa#endif 106620176f3SAbel Vesa 107620176f3SAbel Vesa @ pop saved regs 108d1196787SArd Biesheuvel pop {r0-r11, ip, lr} @ restore r0 through r12 109d1196787SArd Biesheuvel ldr lr, [sp], #4 @ restore LR 110d1196787SArd Biesheuvel ldr pc, [sp], #12 111620176f3SAbel Vesa.endm 112620176f3SAbel Vesa 113620176f3SAbel Vesa#ifdef CONFIG_FUNCTION_GRAPH_TRACER 114620176f3SAbel Vesa.macro __ftrace_graph_regs_caller 115620176f3SAbel Vesa 116*41918ec8SArd Biesheuvel#ifdef CONFIG_UNWINDER_FRAME_POINTER 117620176f3SAbel Vesa sub r0, fp, #4 @ lr of instrumented routine (parent) 118*41918ec8SArd Biesheuvel#else 119*41918ec8SArd Biesheuvel add r0, sp, #S_LR 120*41918ec8SArd Biesheuvel#endif 121620176f3SAbel Vesa 122620176f3SAbel Vesa @ called from __ftrace_regs_caller 123620176f3SAbel Vesa ldr r1, [sp, #S_PC] @ instrumented routine (func) 124620176f3SAbel Vesa mcount_adjust_addr r1, r1 125620176f3SAbel Vesa 126*41918ec8SArd Biesheuvel mov r2, fpreg @ frame pointer 127*41918ec8SArd Biesheuvel add r3, sp, #PT_REGS_SIZE 128620176f3SAbel Vesa bl prepare_ftrace_return 129620176f3SAbel Vesa 130620176f3SAbel Vesa @ pop registers saved in ftrace_regs_caller 131d1196787SArd Biesheuvel pop {r0-r11, ip, lr} @ restore r0 through r12 132d1196787SArd Biesheuvel ldr lr, [sp], #4 @ restore LR 133d1196787SArd Biesheuvel ldr pc, [sp], #12 134620176f3SAbel Vesa 135620176f3SAbel Vesa.endm 136620176f3SAbel Vesa#endif 137620176f3SAbel Vesa#endif 138620176f3SAbel Vesa 13982112379SRussell King.macro __ftrace_caller suffix 14082112379SRussell King mcount_enter 14182112379SRussell King 14282112379SRussell King mcount_get_lr r1 @ lr of instrumented func 14382112379SRussell King mcount_adjust_addr r0, lr @ instrumented function 14482112379SRussell King 145620176f3SAbel Vesa#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 14665aa7e34SArd Biesheuvel ldr_va r2, function_trace_op @ pointer to the current 147620176f3SAbel Vesa @ function tracing op 148620176f3SAbel Vesa mov r3, #0 @ regs is NULL 149620176f3SAbel Vesa#endif 150620176f3SAbel Vesa 15182112379SRussell King .globl ftrace_call\suffix 15282112379SRussell Kingftrace_call\suffix: 15382112379SRussell King bl ftrace_stub 15482112379SRussell King 15582112379SRussell King#ifdef CONFIG_FUNCTION_GRAPH_TRACER 15682112379SRussell King .globl ftrace_graph_call\suffix 15782112379SRussell Kingftrace_graph_call\suffix: 158*41918ec8SArd BiesheuvelARM( mov r0, r0 ) 159*41918ec8SArd BiesheuvelTHUMB( nop.w ) 16082112379SRussell King#endif 16182112379SRussell King 16282112379SRussell King mcount_exit 16382112379SRussell King.endm 16482112379SRussell King 16582112379SRussell King.macro __ftrace_graph_caller 166*41918ec8SArd Biesheuvel#ifdef CONFIG_UNWINDER_FRAME_POINTER 16782112379SRussell King sub r0, fp, #4 @ &lr of instrumented routine (&parent) 168*41918ec8SArd Biesheuvel#else 169*41918ec8SArd Biesheuvel add r0, sp, #20 170*41918ec8SArd Biesheuvel#endif 17182112379SRussell King#ifdef CONFIG_DYNAMIC_FTRACE 17282112379SRussell King @ called from __ftrace_caller, saved in mcount_enter 17382112379SRussell King ldr r1, [sp, #16] @ instrumented routine (func) 17482112379SRussell King mcount_adjust_addr r1, r1 17582112379SRussell King#else 17682112379SRussell King @ called from __mcount, untouched in lr 17782112379SRussell King mcount_adjust_addr r1, lr @ instrumented routine (func) 17882112379SRussell King#endif 179*41918ec8SArd Biesheuvel mov r2, fpreg @ frame pointer 180*41918ec8SArd Biesheuvel add r3, sp, #24 18182112379SRussell King bl prepare_ftrace_return 18282112379SRussell King mcount_exit 18382112379SRussell King.endm 18482112379SRussell King 18582112379SRussell King/* 18682112379SRussell King * __gnu_mcount_nc 18782112379SRussell King */ 18882112379SRussell King 18982112379SRussell King.macro mcount_enter 19082112379SRussell King/* 19182112379SRussell King * This pad compensates for the push {lr} at the call site. Note that we are 19282112379SRussell King * unable to unwind through a function which does not otherwise save its lr. 19382112379SRussell King */ 19482112379SRussell King UNWIND(.pad #4) 19582112379SRussell King stmdb sp!, {r0-r3, lr} 19682112379SRussell King UNWIND(.save {r0-r3, lr}) 19782112379SRussell King.endm 19882112379SRussell King 19982112379SRussell King.macro mcount_get_lr reg 20082112379SRussell King ldr \reg, [sp, #20] 20182112379SRussell King.endm 20282112379SRussell King 20382112379SRussell King.macro mcount_exit 204d1196787SArd Biesheuvel ldmia sp!, {r0-r3} 205d1196787SArd Biesheuvel ldr lr, [sp, #4] 206d1196787SArd Biesheuvel ldr pc, [sp], #8 20782112379SRussell King.endm 20882112379SRussell King 20982112379SRussell KingENTRY(__gnu_mcount_nc) 21082112379SRussell KingUNWIND(.fnstart) 21182112379SRussell King#ifdef CONFIG_DYNAMIC_FTRACE 212d1196787SArd Biesheuvel push {lr} 213d1196787SArd Biesheuvel ldr lr, [sp, #4] 214d1196787SArd Biesheuvel ldr pc, [sp], #8 21582112379SRussell King#else 21682112379SRussell King __mcount 21782112379SRussell King#endif 21882112379SRussell KingUNWIND(.fnend) 21982112379SRussell KingENDPROC(__gnu_mcount_nc) 22082112379SRussell King 22182112379SRussell King#ifdef CONFIG_DYNAMIC_FTRACE 22282112379SRussell KingENTRY(ftrace_caller) 22382112379SRussell KingUNWIND(.fnstart) 22482112379SRussell King __ftrace_caller 22582112379SRussell KingUNWIND(.fnend) 22682112379SRussell KingENDPROC(ftrace_caller) 227620176f3SAbel Vesa 228620176f3SAbel Vesa#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 229620176f3SAbel VesaENTRY(ftrace_regs_caller) 230620176f3SAbel VesaUNWIND(.fnstart) 231620176f3SAbel Vesa __ftrace_regs_caller 232620176f3SAbel VesaUNWIND(.fnend) 233620176f3SAbel VesaENDPROC(ftrace_regs_caller) 234620176f3SAbel Vesa#endif 235620176f3SAbel Vesa 23682112379SRussell King#endif 23782112379SRussell King 23882112379SRussell King#ifdef CONFIG_FUNCTION_GRAPH_TRACER 23982112379SRussell KingENTRY(ftrace_graph_caller) 24082112379SRussell KingUNWIND(.fnstart) 24182112379SRussell King __ftrace_graph_caller 24282112379SRussell KingUNWIND(.fnend) 24382112379SRussell KingENDPROC(ftrace_graph_caller) 244620176f3SAbel Vesa 245620176f3SAbel Vesa#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 246620176f3SAbel VesaENTRY(ftrace_graph_regs_caller) 247620176f3SAbel VesaUNWIND(.fnstart) 248620176f3SAbel Vesa __ftrace_graph_regs_caller 249620176f3SAbel VesaUNWIND(.fnend) 250620176f3SAbel VesaENDPROC(ftrace_graph_regs_caller) 251620176f3SAbel Vesa#endif 25282112379SRussell King#endif 25382112379SRussell King 25482112379SRussell King.purgem mcount_enter 25582112379SRussell King.purgem mcount_get_lr 25682112379SRussell King.purgem mcount_exit 25782112379SRussell King 25882112379SRussell King#ifdef CONFIG_FUNCTION_GRAPH_TRACER 259*41918ec8SArd BiesheuvelENTRY(return_to_handler) 26082112379SRussell King stmdb sp!, {r0-r3} 261953f534aSArd Biesheuvel add r0, sp, #16 @ sp at exit of instrumented routine 26282112379SRussell King bl ftrace_return_to_handler 26382112379SRussell King mov lr, r0 @ r0 has real ret addr 26482112379SRussell King ldmia sp!, {r0-r3} 26582112379SRussell King ret lr 266*41918ec8SArd BiesheuvelENDPROC(return_to_handler) 26782112379SRussell King#endif 26882112379SRussell King 26982112379SRussell KingENTRY(ftrace_stub) 27082112379SRussell King.Lftrace_stub: 27182112379SRussell King ret lr 27282112379SRussell KingENDPROC(ftrace_stub) 273dc438db5SArd Biesheuvel 274dc438db5SArd Biesheuvel#ifdef CONFIG_DYNAMIC_FTRACE 275dc438db5SArd Biesheuvel 276dc438db5SArd Biesheuvel __INIT 277dc438db5SArd Biesheuvel 278dc438db5SArd Biesheuvel .macro init_tramp, dst:req 279dc438db5SArd BiesheuvelENTRY(\dst\()_from_init) 280dc438db5SArd Biesheuvel ldr pc, =\dst 281dc438db5SArd BiesheuvelENDPROC(\dst\()_from_init) 282dc438db5SArd Biesheuvel .endm 283dc438db5SArd Biesheuvel 284dc438db5SArd Biesheuvel init_tramp ftrace_caller 285dc438db5SArd Biesheuvel#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 286dc438db5SArd Biesheuvel init_tramp ftrace_regs_caller 287dc438db5SArd Biesheuvel#endif 288dc438db5SArd Biesheuvel#endif 289