1bad90aa5SNaveen N Rao/* SPDX-License-Identifier: GPL-2.0-or-later */ 2bad90aa5SNaveen N Rao/* 3bad90aa5SNaveen N Rao * Split from ftrace_64.S 4bad90aa5SNaveen N Rao */ 5bad90aa5SNaveen N Rao 6bad90aa5SNaveen N Rao#include <linux/export.h> 7bad90aa5SNaveen N Rao#include <linux/magic.h> 8bad90aa5SNaveen N Rao#include <asm/ppc_asm.h> 9bad90aa5SNaveen N Rao#include <asm/asm-offsets.h> 10bad90aa5SNaveen N Rao#include <asm/ftrace.h> 11bad90aa5SNaveen N Rao#include <asm/ppc-opcode.h> 12bad90aa5SNaveen N Rao#include <asm/thread_info.h> 13bad90aa5SNaveen N Rao#include <asm/bug.h> 14bad90aa5SNaveen N Rao#include <asm/ptrace.h> 15bad90aa5SNaveen N Rao 16bad90aa5SNaveen N Rao/* 17bad90aa5SNaveen N Rao * 18bad90aa5SNaveen N Rao * ftrace_caller()/ftrace_regs_caller() is the function that replaces _mcount() 19bad90aa5SNaveen N Rao * when ftrace is active. 20bad90aa5SNaveen N Rao * 21bad90aa5SNaveen N Rao * We arrive here after a function A calls function B, and we are the trace 22bad90aa5SNaveen N Rao * function for B. When we enter r1 points to A's stack frame, B has not yet 23bad90aa5SNaveen N Rao * had a chance to allocate one yet. 24bad90aa5SNaveen N Rao * 25bad90aa5SNaveen N Rao * Additionally r2 may point either to the TOC for A, or B, depending on 26bad90aa5SNaveen N Rao * whether B did a TOC setup sequence before calling us. 27bad90aa5SNaveen N Rao * 28bad90aa5SNaveen N Rao * On entry the LR points back to the _mcount() call site, and r0 holds the 29bad90aa5SNaveen N Rao * saved LR as it was on entry to B, ie. the original return address at the 30bad90aa5SNaveen N Rao * call site in A. 31bad90aa5SNaveen N Rao * 32bad90aa5SNaveen N Rao * Our job is to save the register state into a struct pt_regs (on the stack) 33bad90aa5SNaveen N Rao * and then arrange for the ftrace function to be called. 34bad90aa5SNaveen N Rao */ 35bad90aa5SNaveen N Rao.macro ftrace_regs_entry allregs 364ad0a4c2SLinus Torvalds /* Create a minimal stack frame for representing B */ 374ad0a4c2SLinus Torvalds PPC_STLU r1, -STACK_FRAME_MIN_SIZE(r1) 384ad0a4c2SLinus Torvalds 39bad90aa5SNaveen N Rao /* Create our stack frame + pt_regs */ 40bad90aa5SNaveen N Rao PPC_STLU r1,-SWITCH_FRAME_SIZE(r1) 41bad90aa5SNaveen N Rao 42bad90aa5SNaveen N Rao /* Save all gprs to pt_regs */ 43bad90aa5SNaveen N Rao SAVE_GPR(0, r1) 44bad90aa5SNaveen N Rao SAVE_GPRS(3, 10, r1) 45bad90aa5SNaveen N Rao 46bad90aa5SNaveen N Rao#ifdef CONFIG_PPC64 47bad90aa5SNaveen N Rao /* Save the original return address in A's stack frame */ 484ad0a4c2SLinus Torvalds std r0, LRSAVE+SWITCH_FRAME_SIZE+STACK_FRAME_MIN_SIZE(r1) 49bad90aa5SNaveen N Rao /* Ok to continue? */ 50bad90aa5SNaveen N Rao lbz r3, PACA_FTRACE_ENABLED(r13) 51bad90aa5SNaveen N Rao cmpdi r3, 0 52bad90aa5SNaveen N Rao beq ftrace_no_trace 53bad90aa5SNaveen N Rao#endif 54bad90aa5SNaveen N Rao 55bad90aa5SNaveen N Rao .if \allregs == 1 56bad90aa5SNaveen N Rao SAVE_GPR(2, r1) 57bad90aa5SNaveen N Rao SAVE_GPRS(11, 31, r1) 58bad90aa5SNaveen N Rao .else 59bad90aa5SNaveen N Rao#ifdef CONFIG_LIVEPATCH_64 60bad90aa5SNaveen N Rao SAVE_GPR(14, r1) 61bad90aa5SNaveen N Rao#endif 62bad90aa5SNaveen N Rao .endif 63bad90aa5SNaveen N Rao 64bad90aa5SNaveen N Rao /* Save previous stack pointer (r1) */ 65*fd728449SNaveen N Rao addi r8, r1, SWITCH_FRAME_SIZE+STACK_FRAME_MIN_SIZE 66bad90aa5SNaveen N Rao PPC_STL r8, GPR1(r1) 67bad90aa5SNaveen N Rao 68bad90aa5SNaveen N Rao .if \allregs == 1 69bad90aa5SNaveen N Rao /* Load special regs for save below */ 70bad90aa5SNaveen N Rao mfmsr r8 71bad90aa5SNaveen N Rao mfctr r9 72bad90aa5SNaveen N Rao mfxer r10 73bad90aa5SNaveen N Rao mfcr r11 74bad90aa5SNaveen N Rao .else 75bad90aa5SNaveen N Rao /* Clear MSR to flag as ftrace_caller versus frace_regs_caller */ 76bad90aa5SNaveen N Rao li r8, 0 77bad90aa5SNaveen N Rao .endif 78bad90aa5SNaveen N Rao 79bad90aa5SNaveen N Rao /* Get the _mcount() call site out of LR */ 80bad90aa5SNaveen N Rao mflr r7 81bad90aa5SNaveen N Rao /* Save it as pt_regs->nip */ 82bad90aa5SNaveen N Rao PPC_STL r7, _NIP(r1) 834ad0a4c2SLinus Torvalds /* Also save it in B's stackframe header for proper unwind */ 844ad0a4c2SLinus Torvalds PPC_STL r7, LRSAVE+SWITCH_FRAME_SIZE(r1) 85bad90aa5SNaveen N Rao /* Save the read LR in pt_regs->link */ 86bad90aa5SNaveen N Rao PPC_STL r0, _LINK(r1) 87bad90aa5SNaveen N Rao 88bad90aa5SNaveen N Rao#ifdef CONFIG_PPC64 89bad90aa5SNaveen N Rao /* Save callee's TOC in the ABI compliant location */ 90bad90aa5SNaveen N Rao std r2, STK_GOT(r1) 91bad90aa5SNaveen N Rao LOAD_PACA_TOC() /* get kernel TOC in r2 */ 92bad90aa5SNaveen N Rao LOAD_REG_ADDR(r3, function_trace_op) 93bad90aa5SNaveen N Rao ld r5,0(r3) 94bad90aa5SNaveen N Rao#else 95bad90aa5SNaveen N Rao lis r3,function_trace_op@ha 96bad90aa5SNaveen N Rao lwz r5,function_trace_op@l(r3) 97bad90aa5SNaveen N Rao#endif 98bad90aa5SNaveen N Rao 99bad90aa5SNaveen N Rao#ifdef CONFIG_LIVEPATCH_64 100bad90aa5SNaveen N Rao mr r14, r7 /* remember old NIP */ 101bad90aa5SNaveen N Rao#endif 102bad90aa5SNaveen N Rao 103bad90aa5SNaveen N Rao /* Calculate ip from nip-4 into r3 for call below */ 104bad90aa5SNaveen N Rao subi r3, r7, MCOUNT_INSN_SIZE 105bad90aa5SNaveen N Rao 106bad90aa5SNaveen N Rao /* Put the original return address in r4 as parent_ip */ 107bad90aa5SNaveen N Rao mr r4, r0 108bad90aa5SNaveen N Rao 109bad90aa5SNaveen N Rao /* Save special regs */ 110bad90aa5SNaveen N Rao PPC_STL r8, _MSR(r1) 111bad90aa5SNaveen N Rao .if \allregs == 1 112bad90aa5SNaveen N Rao PPC_STL r9, _CTR(r1) 113bad90aa5SNaveen N Rao PPC_STL r10, _XER(r1) 114bad90aa5SNaveen N Rao PPC_STL r11, _CCR(r1) 115bad90aa5SNaveen N Rao .endif 116bad90aa5SNaveen N Rao 117bad90aa5SNaveen N Rao /* Load &pt_regs in r6 for call below */ 118bad90aa5SNaveen N Rao addi r6, r1, STACK_INT_FRAME_REGS 119bad90aa5SNaveen N Rao.endm 120bad90aa5SNaveen N Rao 121bad90aa5SNaveen N Rao.macro ftrace_regs_exit allregs 122bad90aa5SNaveen N Rao /* Load ctr with the possibly modified NIP */ 123bad90aa5SNaveen N Rao PPC_LL r3, _NIP(r1) 124bad90aa5SNaveen N Rao mtctr r3 125bad90aa5SNaveen N Rao 126bad90aa5SNaveen N Rao#ifdef CONFIG_LIVEPATCH_64 127bad90aa5SNaveen N Rao cmpd r14, r3 /* has NIP been altered? */ 128bad90aa5SNaveen N Rao#endif 129bad90aa5SNaveen N Rao 130bad90aa5SNaveen N Rao /* Restore gprs */ 131bad90aa5SNaveen N Rao .if \allregs == 1 132bad90aa5SNaveen N Rao REST_GPRS(2, 31, r1) 133bad90aa5SNaveen N Rao .else 134bad90aa5SNaveen N Rao REST_GPRS(3, 10, r1) 135bad90aa5SNaveen N Rao#ifdef CONFIG_LIVEPATCH_64 136bad90aa5SNaveen N Rao REST_GPR(14, r1) 137bad90aa5SNaveen N Rao#endif 138bad90aa5SNaveen N Rao .endif 139bad90aa5SNaveen N Rao 140bad90aa5SNaveen N Rao /* Restore possibly modified LR */ 141bad90aa5SNaveen N Rao PPC_LL r0, _LINK(r1) 142bad90aa5SNaveen N Rao mtlr r0 143bad90aa5SNaveen N Rao 144bad90aa5SNaveen N Rao#ifdef CONFIG_PPC64 145bad90aa5SNaveen N Rao /* Restore callee's TOC */ 146bad90aa5SNaveen N Rao ld r2, STK_GOT(r1) 147bad90aa5SNaveen N Rao#endif 148bad90aa5SNaveen N Rao 149bad90aa5SNaveen N Rao /* Pop our stack frame */ 1504ad0a4c2SLinus Torvalds addi r1, r1, SWITCH_FRAME_SIZE+STACK_FRAME_MIN_SIZE 151bad90aa5SNaveen N Rao 152bad90aa5SNaveen N Rao#ifdef CONFIG_LIVEPATCH_64 153bad90aa5SNaveen N Rao /* Based on the cmpd above, if the NIP was altered handle livepatch */ 154bad90aa5SNaveen N Rao bne- livepatch_handler 155bad90aa5SNaveen N Rao#endif 156bad90aa5SNaveen N Rao bctr /* jump after _mcount site */ 157bad90aa5SNaveen N Rao.endm 158bad90aa5SNaveen N Rao 159bad90aa5SNaveen N Rao_GLOBAL(ftrace_regs_caller) 160bad90aa5SNaveen N Rao ftrace_regs_entry 1 161bad90aa5SNaveen N Rao /* ftrace_call(r3, r4, r5, r6) */ 162bad90aa5SNaveen N Rao.globl ftrace_regs_call 163bad90aa5SNaveen N Raoftrace_regs_call: 164bad90aa5SNaveen N Rao bl ftrace_stub 165bad90aa5SNaveen N Rao nop 166bad90aa5SNaveen N Rao ftrace_regs_exit 1 167bad90aa5SNaveen N Rao 168bad90aa5SNaveen N Rao_GLOBAL(ftrace_caller) 169bad90aa5SNaveen N Rao ftrace_regs_entry 0 170bad90aa5SNaveen N Rao /* ftrace_call(r3, r4, r5, r6) */ 171bad90aa5SNaveen N Rao.globl ftrace_call 172bad90aa5SNaveen N Raoftrace_call: 173bad90aa5SNaveen N Rao bl ftrace_stub 174bad90aa5SNaveen N Rao nop 175bad90aa5SNaveen N Rao ftrace_regs_exit 0 176bad90aa5SNaveen N Rao 177bad90aa5SNaveen N Rao_GLOBAL(ftrace_stub) 178bad90aa5SNaveen N Rao blr 179bad90aa5SNaveen N Rao 180bad90aa5SNaveen N Rao#ifdef CONFIG_PPC64 181bad90aa5SNaveen N Raoftrace_no_trace: 182bad90aa5SNaveen N Rao mflr r3 183bad90aa5SNaveen N Rao mtctr r3 184bad90aa5SNaveen N Rao REST_GPR(3, r1) 185*fd728449SNaveen N Rao addi r1, r1, SWITCH_FRAME_SIZE+STACK_FRAME_MIN_SIZE 186bad90aa5SNaveen N Rao mtlr r0 187bad90aa5SNaveen N Rao bctr 188bad90aa5SNaveen N Rao#endif 189bad90aa5SNaveen N Rao 190bad90aa5SNaveen N Rao#ifdef CONFIG_LIVEPATCH_64 191bad90aa5SNaveen N Rao /* 192bad90aa5SNaveen N Rao * This function runs in the mcount context, between two functions. As 193bad90aa5SNaveen N Rao * such it can only clobber registers which are volatile and used in 194bad90aa5SNaveen N Rao * function linkage. 195bad90aa5SNaveen N Rao * 196bad90aa5SNaveen N Rao * We get here when a function A, calls another function B, but B has 197bad90aa5SNaveen N Rao * been live patched with a new function C. 198bad90aa5SNaveen N Rao * 199bad90aa5SNaveen N Rao * On entry: 200bad90aa5SNaveen N Rao * - we have no stack frame and can not allocate one 201bad90aa5SNaveen N Rao * - LR points back to the original caller (in A) 202bad90aa5SNaveen N Rao * - CTR holds the new NIP in C 203bad90aa5SNaveen N Rao * - r0, r11 & r12 are free 204bad90aa5SNaveen N Rao */ 205bad90aa5SNaveen N Raolivepatch_handler: 206bad90aa5SNaveen N Rao ld r12, PACA_THREAD_INFO(r13) 207bad90aa5SNaveen N Rao 208bad90aa5SNaveen N Rao /* Allocate 3 x 8 bytes */ 209bad90aa5SNaveen N Rao ld r11, TI_livepatch_sp(r12) 210bad90aa5SNaveen N Rao addi r11, r11, 24 211bad90aa5SNaveen N Rao std r11, TI_livepatch_sp(r12) 212bad90aa5SNaveen N Rao 213bad90aa5SNaveen N Rao /* Save toc & real LR on livepatch stack */ 214bad90aa5SNaveen N Rao std r2, -24(r11) 215bad90aa5SNaveen N Rao mflr r12 216bad90aa5SNaveen N Rao std r12, -16(r11) 217bad90aa5SNaveen N Rao 218bad90aa5SNaveen N Rao /* Store stack end marker */ 219bad90aa5SNaveen N Rao lis r12, STACK_END_MAGIC@h 220bad90aa5SNaveen N Rao ori r12, r12, STACK_END_MAGIC@l 221bad90aa5SNaveen N Rao std r12, -8(r11) 222bad90aa5SNaveen N Rao 223bad90aa5SNaveen N Rao /* Put ctr in r12 for global entry and branch there */ 224bad90aa5SNaveen N Rao mfctr r12 225bad90aa5SNaveen N Rao bctrl 226bad90aa5SNaveen N Rao 227bad90aa5SNaveen N Rao /* 228bad90aa5SNaveen N Rao * Now we are returning from the patched function to the original 229bad90aa5SNaveen N Rao * caller A. We are free to use r11, r12 and we can use r2 until we 230bad90aa5SNaveen N Rao * restore it. 231bad90aa5SNaveen N Rao */ 232bad90aa5SNaveen N Rao 233bad90aa5SNaveen N Rao ld r12, PACA_THREAD_INFO(r13) 234bad90aa5SNaveen N Rao 235bad90aa5SNaveen N Rao ld r11, TI_livepatch_sp(r12) 236bad90aa5SNaveen N Rao 237bad90aa5SNaveen N Rao /* Check stack marker hasn't been trashed */ 238bad90aa5SNaveen N Rao lis r2, STACK_END_MAGIC@h 239bad90aa5SNaveen N Rao ori r2, r2, STACK_END_MAGIC@l 240bad90aa5SNaveen N Rao ld r12, -8(r11) 241bad90aa5SNaveen N Rao1: tdne r12, r2 242bad90aa5SNaveen N Rao EMIT_BUG_ENTRY 1b, __FILE__, __LINE__ - 1, 0 243bad90aa5SNaveen N Rao 244bad90aa5SNaveen N Rao /* Restore LR & toc from livepatch stack */ 245bad90aa5SNaveen N Rao ld r12, -16(r11) 246bad90aa5SNaveen N Rao mtlr r12 247bad90aa5SNaveen N Rao ld r2, -24(r11) 248bad90aa5SNaveen N Rao 249bad90aa5SNaveen N Rao /* Pop livepatch stack frame */ 250bad90aa5SNaveen N Rao ld r12, PACA_THREAD_INFO(r13) 251bad90aa5SNaveen N Rao subi r11, r11, 24 252bad90aa5SNaveen N Rao std r11, TI_livepatch_sp(r12) 253bad90aa5SNaveen N Rao 254bad90aa5SNaveen N Rao /* Return to original caller of live patched function */ 255bad90aa5SNaveen N Rao blr 256bad90aa5SNaveen N Rao#endif /* CONFIG_LIVEPATCH */ 257bad90aa5SNaveen N Rao 2580f71dcfbSNaveen N Rao#ifndef CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY 259bad90aa5SNaveen N Rao_GLOBAL(mcount) 260bad90aa5SNaveen N Rao_GLOBAL(_mcount) 261bad90aa5SNaveen N RaoEXPORT_SYMBOL(_mcount) 262bad90aa5SNaveen N Rao mflr r12 263bad90aa5SNaveen N Rao mtctr r12 264bad90aa5SNaveen N Rao mtlr r0 265bad90aa5SNaveen N Rao bctr 2660f71dcfbSNaveen N Rao#endif 267bad90aa5SNaveen N Rao 268bad90aa5SNaveen N Rao#ifdef CONFIG_FUNCTION_GRAPH_TRACER 269bad90aa5SNaveen N Rao_GLOBAL(return_to_handler) 270bad90aa5SNaveen N Rao /* need to save return values */ 271bad90aa5SNaveen N Rao#ifdef CONFIG_PPC64 272bad90aa5SNaveen N Rao std r4, -32(r1) 273bad90aa5SNaveen N Rao std r3, -24(r1) 274bad90aa5SNaveen N Rao /* save TOC */ 275bad90aa5SNaveen N Rao std r2, -16(r1) 276bad90aa5SNaveen N Rao std r31, -8(r1) 277bad90aa5SNaveen N Rao mr r31, r1 278bad90aa5SNaveen N Rao stdu r1, -112(r1) 279bad90aa5SNaveen N Rao 280bad90aa5SNaveen N Rao /* 281bad90aa5SNaveen N Rao * We might be called from a module. 282bad90aa5SNaveen N Rao * Switch to our TOC to run inside the core kernel. 283bad90aa5SNaveen N Rao */ 284bad90aa5SNaveen N Rao LOAD_PACA_TOC() 285bad90aa5SNaveen N Rao#else 286bad90aa5SNaveen N Rao stwu r1, -16(r1) 287bad90aa5SNaveen N Rao stw r3, 8(r1) 288bad90aa5SNaveen N Rao stw r4, 12(r1) 289bad90aa5SNaveen N Rao#endif 290bad90aa5SNaveen N Rao 291bad90aa5SNaveen N Rao bl ftrace_return_to_handler 292bad90aa5SNaveen N Rao nop 293bad90aa5SNaveen N Rao 294bad90aa5SNaveen N Rao /* return value has real return address */ 295bad90aa5SNaveen N Rao mtlr r3 296bad90aa5SNaveen N Rao 297bad90aa5SNaveen N Rao#ifdef CONFIG_PPC64 298bad90aa5SNaveen N Rao ld r1, 0(r1) 299bad90aa5SNaveen N Rao ld r4, -32(r1) 300bad90aa5SNaveen N Rao ld r3, -24(r1) 301bad90aa5SNaveen N Rao ld r2, -16(r1) 302bad90aa5SNaveen N Rao ld r31, -8(r1) 303bad90aa5SNaveen N Rao#else 304bad90aa5SNaveen N Rao lwz r3, 8(r1) 305bad90aa5SNaveen N Rao lwz r4, 12(r1) 306bad90aa5SNaveen N Rao addi r1, r1, 16 307bad90aa5SNaveen N Rao#endif 308bad90aa5SNaveen N Rao 309bad90aa5SNaveen N Rao /* Jump back to real return address */ 310bad90aa5SNaveen N Rao blr 311bad90aa5SNaveen N Rao#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 312bad90aa5SNaveen N Rao 313bad90aa5SNaveen N Rao.pushsection ".tramp.ftrace.text","aw",@progbits; 314bad90aa5SNaveen N Rao.globl ftrace_tramp_text 315bad90aa5SNaveen N Raoftrace_tramp_text: 316bad90aa5SNaveen N Rao .space 32 317bad90aa5SNaveen N Rao.popsection 318bad90aa5SNaveen N Rao 319bad90aa5SNaveen N Rao.pushsection ".tramp.ftrace.init","aw",@progbits; 320bad90aa5SNaveen N Rao.globl ftrace_tramp_init 321bad90aa5SNaveen N Raoftrace_tramp_init: 322bad90aa5SNaveen N Rao .space 32 323bad90aa5SNaveen N Rao.popsection 324