1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Copyright (C) 2017 Steven Rostedt, VMware Inc. 4 */ 5 6#include <linux/linkage.h> 7#include <asm/page_types.h> 8#include <asm/segment.h> 9#include <asm/export.h> 10#include <asm/ftrace.h> 11#include <asm/nospec-branch.h> 12#include <asm/frame.h> 13#include <asm/asm-offsets.h> 14 15# define function_hook __fentry__ 16EXPORT_SYMBOL(__fentry__) 17 18#ifdef CONFIG_FRAME_POINTER 19# define MCOUNT_FRAME 1 /* using frame = true */ 20#else 21# define MCOUNT_FRAME 0 /* using frame = false */ 22#endif 23 24ENTRY(function_hook) 25 ret 26END(function_hook) 27 28ENTRY(ftrace_caller) 29 30#ifdef CONFIG_FRAME_POINTER 31 /* 32 * Frame pointers are of ip followed by bp. 33 * Since fentry is an immediate jump, we are left with 34 * parent-ip, function-ip. We need to add a frame with 35 * parent-ip followed by ebp. 36 */ 37 pushl 4(%esp) /* parent ip */ 38 pushl %ebp 39 movl %esp, %ebp 40 pushl 2*4(%esp) /* function ip */ 41 42 /* For mcount, the function ip is directly above */ 43 pushl %ebp 44 movl %esp, %ebp 45#endif 46 pushl %eax 47 pushl %ecx 48 pushl %edx 49 pushl $0 /* Pass NULL as regs pointer */ 50 51#ifdef CONFIG_FRAME_POINTER 52 /* Load parent ebp into edx */ 53 movl 4*4(%esp), %edx 54#else 55 /* There's no frame pointer, load the appropriate stack addr instead */ 56 lea 4*4(%esp), %edx 57#endif 58 59 movl (MCOUNT_FRAME+4)*4(%esp), %eax /* load the rip */ 60 /* Get the parent ip */ 61 movl 4(%edx), %edx /* edx has ebp */ 62 63 movl function_trace_op, %ecx 64 subl $MCOUNT_INSN_SIZE, %eax 65 66.globl ftrace_call 67ftrace_call: 68 call ftrace_stub 69 70 addl $4, %esp /* skip NULL pointer */ 71 popl %edx 72 popl %ecx 73 popl %eax 74#ifdef CONFIG_FRAME_POINTER 75 popl %ebp 76 addl $4,%esp /* skip function ip */ 77 popl %ebp /* this is the orig bp */ 78 addl $4, %esp /* skip parent ip */ 79#endif 80.Lftrace_ret: 81#ifdef CONFIG_FUNCTION_GRAPH_TRACER 82.globl ftrace_graph_call 83ftrace_graph_call: 84 jmp ftrace_stub 85#endif 86 87/* This is weak to keep gas from relaxing the jumps */ 88WEAK(ftrace_stub) 89 ret 90END(ftrace_caller) 91 92ENTRY(ftrace_regs_caller) 93 /* 94 * We're here from an mcount/fentry CALL, and the stack frame looks like: 95 * 96 * <previous context> 97 * RET-IP 98 * 99 * The purpose of this function is to call out in an emulated INT3 100 * environment with a stack frame like: 101 * 102 * <previous context> 103 * gap / RET-IP 104 * gap 105 * gap 106 * gap 107 * pt_regs 108 * 109 * We do _NOT_ restore: ss, flags, cs, gs, fs, es, ds 110 */ 111 subl $3*4, %esp # RET-IP + 3 gaps 112 pushl %ss # ss 113 pushl %esp # points at ss 114 addl $5*4, (%esp) # make it point at <previous context> 115 pushfl # flags 116 pushl $__KERNEL_CS # cs 117 pushl 7*4(%esp) # ip <- RET-IP 118 pushl $0 # orig_eax 119 120 pushl %gs 121 pushl %fs 122 pushl %es 123 pushl %ds 124 125 pushl %eax 126 pushl %ebp 127 pushl %edi 128 pushl %esi 129 pushl %edx 130 pushl %ecx 131 pushl %ebx 132 133 ENCODE_FRAME_POINTER 134 135 movl PT_EIP(%esp), %eax # 1st argument: IP 136 subl $MCOUNT_INSN_SIZE, %eax 137 movl 21*4(%esp), %edx # 2nd argument: parent ip 138 movl function_trace_op, %ecx # 3rd argument: ftrace_pos 139 pushl %esp # 4th argument: pt_regs 140 141GLOBAL(ftrace_regs_call) 142 call ftrace_stub 143 144 addl $4, %esp # skip 4th argument 145 146 /* place IP below the new SP */ 147 movl PT_OLDESP(%esp), %eax 148 movl PT_EIP(%esp), %ecx 149 movl %ecx, -4(%eax) 150 151 /* place EAX below that */ 152 movl PT_EAX(%esp), %ecx 153 movl %ecx, -8(%eax) 154 155 popl %ebx 156 popl %ecx 157 popl %edx 158 popl %esi 159 popl %edi 160 popl %ebp 161 162 lea -8(%eax), %esp 163 popl %eax 164 165 jmp .Lftrace_ret 166 167#ifdef CONFIG_FUNCTION_GRAPH_TRACER 168ENTRY(ftrace_graph_caller) 169 pushl %eax 170 pushl %ecx 171 pushl %edx 172 movl 3*4(%esp), %eax 173 /* Even with frame pointers, fentry doesn't have one here */ 174 lea 4*4(%esp), %edx 175 movl $0, %ecx 176 subl $MCOUNT_INSN_SIZE, %eax 177 call prepare_ftrace_return 178 popl %edx 179 popl %ecx 180 popl %eax 181 ret 182END(ftrace_graph_caller) 183 184.globl return_to_handler 185return_to_handler: 186 pushl %eax 187 pushl %edx 188 movl $0, %eax 189 call ftrace_return_to_handler 190 movl %eax, %ecx 191 popl %edx 192 popl %eax 193 JMP_NOSPEC %ecx 194#endif 195