1/* 2 * Copyright (C) 2017 Steven Rostedt, VMware Inc. 3 */ 4 5#include <linux/linkage.h> 6#include <asm/page_types.h> 7#include <asm/segment.h> 8#include <asm/export.h> 9#include <asm/ftrace.h> 10 11#ifdef CC_USING_FENTRY 12# define function_hook __fentry__ 13EXPORT_SYMBOL(__fentry__) 14#else 15# define function_hook mcount 16EXPORT_SYMBOL(mcount) 17#endif 18 19#ifdef CONFIG_DYNAMIC_FTRACE 20 21/* mcount uses a frame pointer even if CONFIG_FRAME_POINTER is not set */ 22#if !defined(CC_USING_FENTRY) || defined(CONFIG_FRAME_POINTER) 23# define USING_FRAME_POINTER 24#endif 25 26#ifdef USING_FRAME_POINTER 27# define MCOUNT_FRAME 1 /* using frame = true */ 28#else 29# define MCOUNT_FRAME 0 /* using frame = false */ 30#endif 31 32ENTRY(function_hook) 33 ret 34END(function_hook) 35 36ENTRY(ftrace_caller) 37 38#ifdef USING_FRAME_POINTER 39# ifdef CC_USING_FENTRY 40 /* 41 * Frame pointers are of ip followed by bp. 42 * Since fentry is an immediate jump, we are left with 43 * parent-ip, function-ip. We need to add a frame with 44 * parent-ip followed by ebp. 45 */ 46 pushl 4(%esp) /* parent ip */ 47 pushl %ebp 48 movl %esp, %ebp 49 pushl 2*4(%esp) /* function ip */ 50# endif 51 /* For mcount, the function ip is directly above */ 52 pushl %ebp 53 movl %esp, %ebp 54#endif 55 pushl %eax 56 pushl %ecx 57 pushl %edx 58 pushl $0 /* Pass NULL as regs pointer */ 59 60#ifdef USING_FRAME_POINTER 61 /* Load parent ebp into edx */ 62 movl 4*4(%esp), %edx 63#else 64 /* There's no frame pointer, load the appropriate stack addr instead */ 65 lea 4*4(%esp), %edx 66#endif 67 68 movl (MCOUNT_FRAME+4)*4(%esp), %eax /* load the rip */ 69 /* Get the parent ip */ 70 movl 4(%edx), %edx /* edx has ebp */ 71 72 movl function_trace_op, %ecx 73 subl $MCOUNT_INSN_SIZE, %eax 74 75.globl ftrace_call 76ftrace_call: 77 call ftrace_stub 78 79 addl $4, %esp /* skip NULL pointer */ 80 popl %edx 81 popl %ecx 82 popl %eax 83#ifdef USING_FRAME_POINTER 84 popl %ebp 85# ifdef CC_USING_FENTRY 86 addl $4,%esp /* skip function ip */ 87 popl %ebp /* this is the orig bp */ 88 addl $4, %esp /* skip parent ip */ 89# endif 90#endif 91.Lftrace_ret: 92#ifdef CONFIG_FUNCTION_GRAPH_TRACER 93.globl ftrace_graph_call 94ftrace_graph_call: 95 jmp ftrace_stub 96#endif 97 98/* This is weak to keep gas from relaxing the jumps */ 99WEAK(ftrace_stub) 100 ret 101END(ftrace_caller) 102 103ENTRY(ftrace_regs_caller) 104 /* 105 * i386 does not save SS and ESP when coming from kernel. 106 * Instead, to get sp, ®s->sp is used (see ptrace.h). 107 * Unfortunately, that means eflags must be at the same location 108 * as the current return ip is. We move the return ip into the 109 * regs->ip location, and move flags into the return ip location. 110 */ 111 pushl $__KERNEL_CS 112 pushl 4(%esp) /* Save the return ip */ 113 pushl $0 /* Load 0 into orig_ax */ 114 pushl %gs 115 pushl %fs 116 pushl %es 117 pushl %ds 118 pushl %eax 119 120 /* Get flags and place them into the return ip slot */ 121 pushf 122 popl %eax 123 movl %eax, 8*4(%esp) 124 125 pushl %ebp 126 pushl %edi 127 pushl %esi 128 pushl %edx 129 pushl %ecx 130 pushl %ebx 131 132 movl 12*4(%esp), %eax /* Load ip (1st parameter) */ 133 subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */ 134#ifdef CC_USING_FENTRY 135 movl 15*4(%esp), %edx /* Load parent ip (2nd parameter) */ 136#else 137 movl 0x4(%ebp), %edx /* Load parent ip (2nd parameter) */ 138#endif 139 movl function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */ 140 pushl %esp /* Save pt_regs as 4th parameter */ 141 142GLOBAL(ftrace_regs_call) 143 call ftrace_stub 144 145 addl $4, %esp /* Skip pt_regs */ 146 147 /* restore flags */ 148 push 14*4(%esp) 149 popf 150 151 /* Move return ip back to its original location */ 152 movl 12*4(%esp), %eax 153 movl %eax, 14*4(%esp) 154 155 popl %ebx 156 popl %ecx 157 popl %edx 158 popl %esi 159 popl %edi 160 popl %ebp 161 popl %eax 162 popl %ds 163 popl %es 164 popl %fs 165 popl %gs 166 167 /* use lea to not affect flags */ 168 lea 3*4(%esp), %esp /* Skip orig_ax, ip and cs */ 169 170 jmp .Lftrace_ret 171#else /* ! CONFIG_DYNAMIC_FTRACE */ 172 173ENTRY(function_hook) 174 cmpl $__PAGE_OFFSET, %esp 175 jb ftrace_stub /* Paging not enabled yet? */ 176 177 cmpl $ftrace_stub, ftrace_trace_function 178 jnz .Ltrace 179#ifdef CONFIG_FUNCTION_GRAPH_TRACER 180 cmpl $ftrace_stub, ftrace_graph_return 181 jnz ftrace_graph_caller 182 183 cmpl $ftrace_graph_entry_stub, ftrace_graph_entry 184 jnz ftrace_graph_caller 185#endif 186.globl ftrace_stub 187ftrace_stub: 188 ret 189 190 /* taken from glibc */ 191.Ltrace: 192 pushl %eax 193 pushl %ecx 194 pushl %edx 195 movl 0xc(%esp), %eax 196 movl 0x4(%ebp), %edx 197 subl $MCOUNT_INSN_SIZE, %eax 198 199 call *ftrace_trace_function 200 201 popl %edx 202 popl %ecx 203 popl %eax 204 jmp ftrace_stub 205END(function_hook) 206#endif /* CONFIG_DYNAMIC_FTRACE */ 207 208#ifdef CONFIG_FUNCTION_GRAPH_TRACER 209ENTRY(ftrace_graph_caller) 210 pushl %eax 211 pushl %ecx 212 pushl %edx 213 movl 3*4(%esp), %eax 214 /* Even with frame pointers, fentry doesn't have one here */ 215#ifdef CC_USING_FENTRY 216 lea 4*4(%esp), %edx 217 movl $0, %ecx 218#else 219 lea 0x4(%ebp), %edx 220 movl (%ebp), %ecx 221#endif 222 subl $MCOUNT_INSN_SIZE, %eax 223 call prepare_ftrace_return 224 popl %edx 225 popl %ecx 226 popl %eax 227 ret 228END(ftrace_graph_caller) 229 230.globl return_to_handler 231return_to_handler: 232 pushl %eax 233 pushl %edx 234#ifdef CC_USING_FENTRY 235 movl $0, %eax 236#else 237 movl %ebp, %eax 238#endif 239 call ftrace_return_to_handler 240 movl %eax, %ecx 241 popl %edx 242 popl %eax 243 jmp *%ecx 244#endif 245