1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Copyright (C) 2017 Steven Rostedt, VMware Inc. 4 */ 5 6#include <linux/linkage.h> 7#include <asm/page_types.h> 8#include <asm/segment.h> 9#include <asm/export.h> 10#include <asm/ftrace.h> 11#include <asm/nospec-branch.h> 12 13# define function_hook __fentry__ 14EXPORT_SYMBOL(__fentry__) 15 16#ifdef CONFIG_FRAME_POINTER 17# define MCOUNT_FRAME 1 /* using frame = true */ 18#else 19# define MCOUNT_FRAME 0 /* using frame = false */ 20#endif 21 22ENTRY(function_hook) 23 ret 24END(function_hook) 25 26ENTRY(ftrace_caller) 27 28#ifdef CONFIG_FRAME_POINTER 29 /* 30 * Frame pointers are of ip followed by bp. 31 * Since fentry is an immediate jump, we are left with 32 * parent-ip, function-ip. We need to add a frame with 33 * parent-ip followed by ebp. 34 */ 35 pushl 4(%esp) /* parent ip */ 36 pushl %ebp 37 movl %esp, %ebp 38 pushl 2*4(%esp) /* function ip */ 39 40 /* For mcount, the function ip is directly above */ 41 pushl %ebp 42 movl %esp, %ebp 43#endif 44 pushl %eax 45 pushl %ecx 46 pushl %edx 47 pushl $0 /* Pass NULL as regs pointer */ 48 49#ifdef CONFIG_FRAME_POINTER 50 /* Load parent ebp into edx */ 51 movl 4*4(%esp), %edx 52#else 53 /* There's no frame pointer, load the appropriate stack addr instead */ 54 lea 4*4(%esp), %edx 55#endif 56 57 movl (MCOUNT_FRAME+4)*4(%esp), %eax /* load the rip */ 58 /* Get the parent ip */ 59 movl 4(%edx), %edx /* edx has ebp */ 60 61 movl function_trace_op, %ecx 62 subl $MCOUNT_INSN_SIZE, %eax 63 64.globl ftrace_call 65ftrace_call: 66 call ftrace_stub 67 68 addl $4, %esp /* skip NULL pointer */ 69 popl %edx 70 popl %ecx 71 popl %eax 72#ifdef CONFIG_FRAME_POINTER 73 popl %ebp 74 addl $4,%esp /* skip function ip */ 75 popl %ebp /* this is the orig bp */ 76 addl $4, %esp /* skip parent ip */ 77#endif 78.Lftrace_ret: 79#ifdef CONFIG_FUNCTION_GRAPH_TRACER 80.globl ftrace_graph_call 81ftrace_graph_call: 82 jmp ftrace_stub 83#endif 84 85/* This is weak to keep gas from relaxing the jumps */ 86WEAK(ftrace_stub) 87 ret 88END(ftrace_caller) 89 90ENTRY(ftrace_regs_caller) 91 /* 92 * i386 does not save SS and ESP when coming from kernel. 93 * Instead, to get sp, ®s->sp is used (see ptrace.h). 94 * Unfortunately, that means eflags must be at the same location 95 * as the current return ip is. We move the return ip into the 96 * regs->ip location, and move flags into the return ip location. 97 */ 98 pushl $__KERNEL_CS 99 pushl 4(%esp) /* Save the return ip */ 100 pushl $0 /* Load 0 into orig_ax */ 101 pushl %gs 102 pushl %fs 103 pushl %es 104 pushl %ds 105 pushl %eax 106 107 /* Get flags and place them into the return ip slot */ 108 pushf 109 popl %eax 110 movl %eax, 8*4(%esp) 111 112 pushl %ebp 113 pushl %edi 114 pushl %esi 115 pushl %edx 116 pushl %ecx 117 pushl %ebx 118 119 movl 12*4(%esp), %eax /* Load ip (1st parameter) */ 120 subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */ 121 movl 15*4(%esp), %edx /* Load parent ip (2nd parameter) */ 122 movl function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */ 123 pushl %esp /* Save pt_regs as 4th parameter */ 124 125GLOBAL(ftrace_regs_call) 126 call ftrace_stub 127 128 addl $4, %esp /* Skip pt_regs */ 129 130 /* restore flags */ 131 push 14*4(%esp) 132 popf 133 134 /* Move return ip back to its original location */ 135 movl 12*4(%esp), %eax 136 movl %eax, 14*4(%esp) 137 138 popl %ebx 139 popl %ecx 140 popl %edx 141 popl %esi 142 popl %edi 143 popl %ebp 144 popl %eax 145 popl %ds 146 popl %es 147 popl %fs 148 popl %gs 149 150 /* use lea to not affect flags */ 151 lea 3*4(%esp), %esp /* Skip orig_ax, ip and cs */ 152 153 jmp .Lftrace_ret 154 155#ifdef CONFIG_FUNCTION_GRAPH_TRACER 156ENTRY(ftrace_graph_caller) 157 pushl %eax 158 pushl %ecx 159 pushl %edx 160 movl 3*4(%esp), %eax 161 /* Even with frame pointers, fentry doesn't have one here */ 162 lea 4*4(%esp), %edx 163 movl $0, %ecx 164 subl $MCOUNT_INSN_SIZE, %eax 165 call prepare_ftrace_return 166 popl %edx 167 popl %ecx 168 popl %eax 169 ret 170END(ftrace_graph_caller) 171 172.globl return_to_handler 173return_to_handler: 174 pushl %eax 175 pushl %edx 176 movl $0, %eax 177 call ftrace_return_to_handler 178 movl %eax, %ecx 179 popl %edx 180 popl %eax 181 JMP_NOSPEC %ecx 182#endif 183