1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Copyright (C) 2017 Steven Rostedt, VMware Inc. 4 */ 5 6#include <linux/linkage.h> 7#include <asm/page_types.h> 8#include <asm/segment.h> 9#include <asm/export.h> 10#include <asm/ftrace.h> 11#include <asm/nospec-branch.h> 12 13#ifdef CC_USING_FENTRY 14# define function_hook __fentry__ 15EXPORT_SYMBOL(__fentry__) 16#else 17# define function_hook mcount 18EXPORT_SYMBOL(mcount) 19#endif 20 21#ifdef CONFIG_DYNAMIC_FTRACE 22 23/* mcount uses a frame pointer even if CONFIG_FRAME_POINTER is not set */ 24#if !defined(CC_USING_FENTRY) || defined(CONFIG_FRAME_POINTER) 25# define USING_FRAME_POINTER 26#endif 27 28#ifdef USING_FRAME_POINTER 29# define MCOUNT_FRAME 1 /* using frame = true */ 30#else 31# define MCOUNT_FRAME 0 /* using frame = false */ 32#endif 33 34ENTRY(function_hook) 35 ret 36END(function_hook) 37 38ENTRY(ftrace_caller) 39 40#ifdef USING_FRAME_POINTER 41# ifdef CC_USING_FENTRY 42 /* 43 * Frame pointers are of ip followed by bp. 44 * Since fentry is an immediate jump, we are left with 45 * parent-ip, function-ip. We need to add a frame with 46 * parent-ip followed by ebp. 47 */ 48 pushl 4(%esp) /* parent ip */ 49 pushl %ebp 50 movl %esp, %ebp 51 pushl 2*4(%esp) /* function ip */ 52# endif 53 /* For mcount, the function ip is directly above */ 54 pushl %ebp 55 movl %esp, %ebp 56#endif 57 pushl %eax 58 pushl %ecx 59 pushl %edx 60 pushl $0 /* Pass NULL as regs pointer */ 61 62#ifdef USING_FRAME_POINTER 63 /* Load parent ebp into edx */ 64 movl 4*4(%esp), %edx 65#else 66 /* There's no frame pointer, load the appropriate stack addr instead */ 67 lea 4*4(%esp), %edx 68#endif 69 70 movl (MCOUNT_FRAME+4)*4(%esp), %eax /* load the rip */ 71 /* Get the parent ip */ 72 movl 4(%edx), %edx /* edx has ebp */ 73 74 movl function_trace_op, %ecx 75 subl $MCOUNT_INSN_SIZE, %eax 76 77.globl ftrace_call 78ftrace_call: 79 call ftrace_stub 80 81 addl $4, %esp /* skip NULL pointer */ 82 popl %edx 83 popl %ecx 84 popl %eax 85#ifdef USING_FRAME_POINTER 86 popl %ebp 87# ifdef CC_USING_FENTRY 88 addl $4,%esp /* skip function ip */ 89 popl %ebp /* this is the orig bp */ 90 addl $4, %esp /* skip parent ip */ 91# endif 92#endif 93.Lftrace_ret: 94#ifdef CONFIG_FUNCTION_GRAPH_TRACER 95.globl ftrace_graph_call 96ftrace_graph_call: 97 jmp ftrace_stub 98#endif 99 100/* This is weak to keep gas from relaxing the jumps */ 101WEAK(ftrace_stub) 102 ret 103END(ftrace_caller) 104 105ENTRY(ftrace_regs_caller) 106 /* 107 * i386 does not save SS and ESP when coming from kernel. 108 * Instead, to get sp, ®s->sp is used (see ptrace.h). 109 * Unfortunately, that means eflags must be at the same location 110 * as the current return ip is. We move the return ip into the 111 * regs->ip location, and move flags into the return ip location. 112 */ 113 pushl $__KERNEL_CS 114 pushl 4(%esp) /* Save the return ip */ 115 pushl $0 /* Load 0 into orig_ax */ 116 pushl %gs 117 pushl %fs 118 pushl %es 119 pushl %ds 120 pushl %eax 121 122 /* Get flags and place them into the return ip slot */ 123 pushf 124 popl %eax 125 movl %eax, 8*4(%esp) 126 127 pushl %ebp 128 pushl %edi 129 pushl %esi 130 pushl %edx 131 pushl %ecx 132 pushl %ebx 133 134 movl 12*4(%esp), %eax /* Load ip (1st parameter) */ 135 subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */ 136#ifdef CC_USING_FENTRY 137 movl 15*4(%esp), %edx /* Load parent ip (2nd parameter) */ 138#else 139 movl 0x4(%ebp), %edx /* Load parent ip (2nd parameter) */ 140#endif 141 movl function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */ 142 pushl %esp /* Save pt_regs as 4th parameter */ 143 144GLOBAL(ftrace_regs_call) 145 call ftrace_stub 146 147 addl $4, %esp /* Skip pt_regs */ 148 149 /* restore flags */ 150 push 14*4(%esp) 151 popf 152 153 /* Move return ip back to its original location */ 154 movl 12*4(%esp), %eax 155 movl %eax, 14*4(%esp) 156 157 popl %ebx 158 popl %ecx 159 popl %edx 160 popl %esi 161 popl %edi 162 popl %ebp 163 popl %eax 164 popl %ds 165 popl %es 166 popl %fs 167 popl %gs 168 169 /* use lea to not affect flags */ 170 lea 3*4(%esp), %esp /* Skip orig_ax, ip and cs */ 171 172 jmp .Lftrace_ret 173#else /* ! CONFIG_DYNAMIC_FTRACE */ 174 175ENTRY(function_hook) 176 cmpl $__PAGE_OFFSET, %esp 177 jb ftrace_stub /* Paging not enabled yet? */ 178 179 cmpl $ftrace_stub, ftrace_trace_function 180 jnz .Ltrace 181#ifdef CONFIG_FUNCTION_GRAPH_TRACER 182 cmpl $ftrace_stub, ftrace_graph_return 183 jnz ftrace_graph_caller 184 185 cmpl $ftrace_graph_entry_stub, ftrace_graph_entry 186 jnz ftrace_graph_caller 187#endif 188.globl ftrace_stub 189ftrace_stub: 190 ret 191 192 /* taken from glibc */ 193.Ltrace: 194 pushl %eax 195 pushl %ecx 196 pushl %edx 197 movl 0xc(%esp), %eax 198 movl 0x4(%ebp), %edx 199 subl $MCOUNT_INSN_SIZE, %eax 200 201 movl ftrace_trace_function, %ecx 202 CALL_NOSPEC %ecx 203 204 popl %edx 205 popl %ecx 206 popl %eax 207 jmp ftrace_stub 208END(function_hook) 209#endif /* CONFIG_DYNAMIC_FTRACE */ 210 211#ifdef CONFIG_FUNCTION_GRAPH_TRACER 212ENTRY(ftrace_graph_caller) 213 pushl %eax 214 pushl %ecx 215 pushl %edx 216 movl 3*4(%esp), %eax 217 /* Even with frame pointers, fentry doesn't have one here */ 218#ifdef CC_USING_FENTRY 219 lea 4*4(%esp), %edx 220 movl $0, %ecx 221#else 222 lea 0x4(%ebp), %edx 223 movl (%ebp), %ecx 224#endif 225 subl $MCOUNT_INSN_SIZE, %eax 226 call prepare_ftrace_return 227 popl %edx 228 popl %ecx 229 popl %eax 230 ret 231END(ftrace_graph_caller) 232 233.globl return_to_handler 234return_to_handler: 235 pushl %eax 236 pushl %edx 237#ifdef CC_USING_FENTRY 238 movl $0, %eax 239#else 240 movl %ebp, %eax 241#endif 242 call ftrace_return_to_handler 243 movl %eax, %ecx 244 popl %edx 245 popl %eax 246 JMP_NOSPEC %ecx 247#endif 248