1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Copyright IBM Corp. 2008, 2009 4 * 5 */ 6 7#include <linux/linkage.h> 8#include <asm/asm-offsets.h> 9#include <asm/ftrace.h> 10#include <asm/nospec-insn.h> 11#include <asm/ptrace.h> 12#include <asm/export.h> 13 14 15#define STACK_FRAME_SIZE_PTREGS (STACK_FRAME_OVERHEAD + __PT_SIZE) 16#define STACK_PTREGS (STACK_FRAME_OVERHEAD) 17#define STACK_PTREGS_GPRS (STACK_PTREGS + __PT_GPRS) 18#define STACK_PTREGS_PSW (STACK_PTREGS + __PT_PSW) 19 20#define STACK_FRAME_SIZE_FREGS (STACK_FRAME_OVERHEAD + __FTRACE_REGS_SIZE) 21#define STACK_FREGS (STACK_FRAME_OVERHEAD) 22#define STACK_FREGS_PTREGS (STACK_FRAME_OVERHEAD + __FTRACE_REGS_PT_REGS) 23#define STACK_FREGS_PTREGS_GPRS (STACK_FREGS_PTREGS + __PT_GPRS) 24#define STACK_FREGS_PTREGS_PSW (STACK_FREGS_PTREGS + __PT_PSW) 25#define STACK_FREGS_PTREGS_ORIG_GPR2 (STACK_FREGS_PTREGS + __PT_ORIG_GPR2) 26#define STACK_FREGS_PTREGS_FLAGS (STACK_FREGS_PTREGS + __PT_FLAGS) 27 28/* packed stack: allocate just enough for r14, r15 and backchain */ 29#define TRACED_FUNC_FRAME_SIZE 24 30 31#ifdef CONFIG_FUNCTION_TRACER 32 33 GEN_BR_THUNK %r1 34 GEN_BR_THUNK %r14 35 36 .section .kprobes.text, "ax" 37 38SYM_FUNC_START(ftrace_stub) 39 BR_EX %r14 40SYM_FUNC_END(ftrace_stub) 41 42SYM_CODE_START(ftrace_stub_direct_tramp) 43 lgr %r1, %r0 44 BR_EX %r1 45SYM_CODE_END(ftrace_stub_direct_tramp) 46 47 .macro ftrace_regs_entry, allregs=0 48 stg %r14,(__SF_GPRS+8*8)(%r15) # save traced function caller 49 50 .if \allregs == 1 51 # save psw mask 52 # don't put any instructions clobbering CC before this point 53 epsw %r1,%r14 54 risbg %r14,%r1,0,31,32 55 .endif 56 57 lgr %r1,%r15 58 # allocate stack frame for ftrace_caller to contain traced function 59 aghi %r15,-TRACED_FUNC_FRAME_SIZE 60 stg %r1,__SF_BACKCHAIN(%r15) 61 stg %r0,(__SF_GPRS+8*8)(%r15) 62 stg %r15,(__SF_GPRS+9*8)(%r15) 63 # allocate ftrace_regs and stack frame for ftrace_trace_function 64 aghi %r15,-STACK_FRAME_SIZE_FREGS 65 stg %r1,(STACK_FREGS_PTREGS_GPRS+15*8)(%r15) 66 xc STACK_FREGS_PTREGS_ORIG_GPR2(8,%r15),STACK_FREGS_PTREGS_ORIG_GPR2(%r15) 67 68 .if \allregs == 1 69 stg %r14,(STACK_FREGS_PTREGS_PSW)(%r15) 70 mvghi STACK_FREGS_PTREGS_FLAGS(%r15),_PIF_FTRACE_FULL_REGS 71 .else 72 xc STACK_FREGS_PTREGS_FLAGS(8,%r15),STACK_FREGS_PTREGS_FLAGS(%r15) 73 .endif 74 75 lg %r14,(__SF_GPRS+8*8)(%r1) # restore original return address 76 aghi %r1,-TRACED_FUNC_FRAME_SIZE 77 stg %r1,__SF_BACKCHAIN(%r15) 78 stg %r0,(STACK_FREGS_PTREGS_PSW+8)(%r15) 79 stmg %r2,%r14,(STACK_FREGS_PTREGS_GPRS+2*8)(%r15) 80 .endm 81 82SYM_CODE_START(ftrace_regs_caller) 83 ftrace_regs_entry 1 84 j ftrace_common 85SYM_CODE_END(ftrace_regs_caller) 86 87SYM_CODE_START(ftrace_caller) 88 ftrace_regs_entry 0 89 j ftrace_common 90SYM_CODE_END(ftrace_caller) 91 92SYM_CODE_START(ftrace_common) 93#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES 94 aghik %r2,%r0,-MCOUNT_INSN_SIZE 95 lgrl %r4,function_trace_op 96 lgrl %r1,ftrace_func 97#else 98 lgr %r2,%r0 99 aghi %r2,-MCOUNT_INSN_SIZE 100 larl %r4,function_trace_op 101 lg %r4,0(%r4) 102 larl %r1,ftrace_func 103 lg %r1,0(%r1) 104#endif 105 lgr %r3,%r14 106 la %r5,STACK_FREGS(%r15) 107 BASR_EX %r14,%r1 108#ifdef CONFIG_FUNCTION_GRAPH_TRACER 109# The j instruction gets runtime patched to a nop instruction. 110# See ftrace_enable_ftrace_graph_caller. 111SYM_INNER_LABEL(ftrace_graph_caller, SYM_L_GLOBAL) 112 j .Lftrace_graph_caller_end 113 lmg %r2,%r3,(STACK_FREGS_PTREGS_GPRS+14*8)(%r15) 114 lg %r4,(STACK_FREGS_PTREGS_PSW+8)(%r15) 115 brasl %r14,prepare_ftrace_return 116 stg %r2,(STACK_FREGS_PTREGS_GPRS+14*8)(%r15) 117.Lftrace_graph_caller_end: 118#endif 119 lg %r0,(STACK_FREGS_PTREGS_PSW+8)(%r15) 120#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES 121 ltg %r1,STACK_FREGS_PTREGS_ORIG_GPR2(%r15) 122 locgrz %r1,%r0 123#else 124 lg %r1,STACK_FREGS_PTREGS_ORIG_GPR2(%r15) 125 ltgr %r1,%r1 126 jnz 0f 127 lgr %r1,%r0 128#endif 1290: lmg %r2,%r15,(STACK_FREGS_PTREGS_GPRS+2*8)(%r15) 130 BR_EX %r1 131SYM_CODE_END(ftrace_common) 132 133#ifdef CONFIG_FUNCTION_GRAPH_TRACER 134 135SYM_FUNC_START(return_to_handler) 136 stmg %r2,%r5,32(%r15) 137 lgr %r1,%r15 138 aghi %r15,-(STACK_FRAME_OVERHEAD+__FGRAPH_RET_SIZE) 139 stg %r1,__SF_BACKCHAIN(%r15) 140 aghik %r3,%r15,STACK_FRAME_OVERHEAD 141 stg %r1,__FGRAPH_RET_FP(%r3) 142 stg %r2,__FGRAPH_RET_GPR2(%r3) 143 lgr %r2,%r3 144 brasl %r14,ftrace_return_to_handler 145 aghi %r15,STACK_FRAME_OVERHEAD+__FGRAPH_RET_SIZE 146 lgr %r14,%r2 147 lmg %r2,%r5,32(%r15) 148 BR_EX %r14 149SYM_FUNC_END(return_to_handler) 150 151#endif 152#endif /* CONFIG_FUNCTION_TRACER */ 153 154SYM_CODE_START(ftrace_shared_hotpatch_trampoline_br) 155 lmg %r0,%r1,2(%r1) 156 br %r1 157SYM_INNER_LABEL(ftrace_shared_hotpatch_trampoline_br_end, SYM_L_GLOBAL) 158SYM_CODE_END(ftrace_shared_hotpatch_trampoline_br) 159 160#ifdef CONFIG_EXPOLINE 161SYM_CODE_START(ftrace_shared_hotpatch_trampoline_exrl) 162 lmg %r0,%r1,2(%r1) 163 exrl %r0,0f 164 j . 1650: br %r1 166SYM_INNER_LABEL(ftrace_shared_hotpatch_trampoline_exrl_end, SYM_L_GLOBAL) 167SYM_CODE_END(ftrace_shared_hotpatch_trampoline_exrl) 168#endif /* CONFIG_EXPOLINE */ 169 170#ifdef CONFIG_RETHOOK 171 172SYM_CODE_START(arch_rethook_trampoline) 173 stg %r14,(__SF_GPRS+8*8)(%r15) 174 lay %r15,-STACK_FRAME_SIZE_PTREGS(%r15) 175 stmg %r0,%r14,STACK_PTREGS_GPRS(%r15) 176 177 # store original stack pointer in backchain and pt_regs 178 lay %r7,STACK_FRAME_SIZE_PTREGS(%r15) 179 stg %r7,__SF_BACKCHAIN(%r15) 180 stg %r7,STACK_PTREGS_GPRS+(15*8)(%r15) 181 182 # store full psw 183 epsw %r2,%r3 184 risbg %r3,%r2,0,31,32 185 stg %r3,STACK_PTREGS_PSW(%r15) 186 larl %r1,arch_rethook_trampoline 187 stg %r1,STACK_PTREGS_PSW+8(%r15) 188 189 lay %r2,STACK_PTREGS(%r15) 190 brasl %r14,arch_rethook_trampoline_callback 191 192 mvc __SF_EMPTY(16,%r7),STACK_PTREGS_PSW(%r15) 193 lmg %r0,%r15,STACK_PTREGS_GPRS(%r15) 194 lpswe __SF_EMPTY(%r15) 195SYM_CODE_END(arch_rethook_trampoline) 196 197#endif /* CONFIG_RETHOOK */ 198