1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Dynamic function tracer architecture backend. 4 * 5 * Copyright IBM Corp. 2009,2014 6 * 7 * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>, 8 * Martin Schwidefsky <schwidefsky@de.ibm.com> 9 */ 10 11 #include <linux/moduleloader.h> 12 #include <linux/hardirq.h> 13 #include <linux/uaccess.h> 14 #include <linux/ftrace.h> 15 #include <linux/kernel.h> 16 #include <linux/types.h> 17 #include <linux/kprobes.h> 18 #include <trace/syscall.h> 19 #include <asm/asm-offsets.h> 20 #include <asm/cacheflush.h> 21 #include <asm/set_memory.h> 22 #include "entry.h" 23 24 /* 25 * The mcount code looks like this: 26 * stg %r14,8(%r15) # offset 0 27 * larl %r1,<&counter> # offset 6 28 * brasl %r14,_mcount # offset 12 29 * lg %r14,8(%r15) # offset 18 30 * Total length is 24 bytes. Only the first instruction will be patched 31 * by ftrace_make_call / ftrace_make_nop. 32 * The enabled ftrace code block looks like this: 33 * > brasl %r0,ftrace_caller # offset 0 34 * larl %r1,<&counter> # offset 6 35 * brasl %r14,_mcount # offset 12 36 * lg %r14,8(%r15) # offset 18 37 * The ftrace function gets called with a non-standard C function call ABI 38 * where r0 contains the return address. It is also expected that the called 39 * function only clobbers r0 and r1, but restores r2-r15. 40 * For module code we can't directly jump to ftrace caller, but need a 41 * trampoline (ftrace_plt), which clobbers also r1. 42 * The return point of the ftrace function has offset 24, so execution 43 * continues behind the mcount block. 44 * The disabled ftrace code block looks like this: 45 * > jg .+24 # offset 0 46 * larl %r1,<&counter> # offset 6 47 * brasl %r14,_mcount # offset 12 48 * lg %r14,8(%r15) # offset 18 49 * The jg instruction branches to offset 24 to skip as many instructions 50 * as possible. 51 * In case we use gcc's hotpatch feature the original and also the disabled 52 * function prologue contains only a single six byte instruction and looks 53 * like this: 54 * > brcl 0,0 # offset 0 55 * To enable ftrace the code gets patched like above and afterwards looks 56 * like this: 57 * > brasl %r0,ftrace_caller # offset 0 58 */ 59 60 unsigned long ftrace_plt; 61 62 static inline void ftrace_generate_orig_insn(struct ftrace_insn *insn) 63 { 64 #if defined(CC_USING_HOTPATCH) || defined(CC_USING_NOP_MCOUNT) 65 /* brcl 0,0 */ 66 insn->opc = 0xc004; 67 insn->disp = 0; 68 #else 69 /* stg r14,8(r15) */ 70 insn->opc = 0xe3e0; 71 insn->disp = 0xf0080024; 72 #endif 73 } 74 75 static inline void ftrace_generate_kprobe_nop_insn(struct ftrace_insn *insn) 76 { 77 #ifdef CONFIG_KPROBES 78 insn->opc = BREAKPOINT_INSTRUCTION; 79 insn->disp = KPROBE_ON_FTRACE_NOP; 80 #endif 81 } 82 83 static inline void ftrace_generate_kprobe_call_insn(struct ftrace_insn *insn) 84 { 85 #ifdef CONFIG_KPROBES 86 insn->opc = BREAKPOINT_INSTRUCTION; 87 insn->disp = KPROBE_ON_FTRACE_CALL; 88 #endif 89 } 90 91 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, 92 unsigned long addr) 93 { 94 return 0; 95 } 96 97 int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, 98 unsigned long addr) 99 { 100 struct ftrace_insn orig, new, old; 101 102 if (probe_kernel_read(&old, (void *) rec->ip, sizeof(old))) 103 return -EFAULT; 104 if (addr == MCOUNT_ADDR) { 105 /* Initial code replacement */ 106 ftrace_generate_orig_insn(&orig); 107 ftrace_generate_nop_insn(&new); 108 } else { 109 /* Replace ftrace call with a nop. */ 110 ftrace_generate_call_insn(&orig, rec->ip); 111 ftrace_generate_nop_insn(&new); 112 } 113 /* Verify that the to be replaced code matches what we expect. */ 114 if (memcmp(&orig, &old, sizeof(old))) 115 return -EINVAL; 116 s390_kernel_write((void *) rec->ip, &new, sizeof(new)); 117 return 0; 118 } 119 120 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 121 { 122 struct ftrace_insn orig, new, old; 123 124 if (probe_kernel_read(&old, (void *) rec->ip, sizeof(old))) 125 return -EFAULT; 126 /* Replace nop with an ftrace call. */ 127 ftrace_generate_nop_insn(&orig); 128 ftrace_generate_call_insn(&new, rec->ip); 129 130 /* Verify that the to be replaced code matches what we expect. */ 131 if (memcmp(&orig, &old, sizeof(old))) 132 return -EINVAL; 133 s390_kernel_write((void *) rec->ip, &new, sizeof(new)); 134 return 0; 135 } 136 137 int ftrace_update_ftrace_func(ftrace_func_t func) 138 { 139 return 0; 140 } 141 142 int __init ftrace_dyn_arch_init(void) 143 { 144 return 0; 145 } 146 147 #ifdef CONFIG_MODULES 148 149 static int __init ftrace_plt_init(void) 150 { 151 unsigned int *ip; 152 153 ftrace_plt = (unsigned long) module_alloc(PAGE_SIZE); 154 if (!ftrace_plt) 155 panic("cannot allocate ftrace plt\n"); 156 ip = (unsigned int *) ftrace_plt; 157 ip[0] = 0x0d10e310; /* basr 1,0; lg 1,10(1); br 1 */ 158 ip[1] = 0x100a0004; 159 ip[2] = 0x07f10000; 160 ip[3] = FTRACE_ADDR >> 32; 161 ip[4] = FTRACE_ADDR & 0xffffffff; 162 set_memory_ro(ftrace_plt, 1); 163 return 0; 164 } 165 device_initcall(ftrace_plt_init); 166 167 #endif /* CONFIG_MODULES */ 168 169 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 170 /* 171 * Hook the return address and push it in the stack of return addresses 172 * in current thread info. 173 */ 174 unsigned long prepare_ftrace_return(unsigned long ra, unsigned long sp, 175 unsigned long ip) 176 { 177 if (unlikely(ftrace_graph_is_dead())) 178 goto out; 179 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 180 goto out; 181 ip -= MCOUNT_INSN_SIZE; 182 if (!function_graph_enter(ra, ip, 0, (void *) sp)) 183 ra = (unsigned long) return_to_handler; 184 out: 185 return ra; 186 } 187 NOKPROBE_SYMBOL(prepare_ftrace_return); 188 189 /* 190 * Patch the kernel code at ftrace_graph_caller location. The instruction 191 * there is branch relative on condition. To enable the ftrace graph code 192 * block, we simply patch the mask field of the instruction to zero and 193 * turn the instruction into a nop. 194 * To disable the ftrace graph code the mask field will be patched to 195 * all ones, which turns the instruction into an unconditional branch. 196 */ 197 int ftrace_enable_ftrace_graph_caller(void) 198 { 199 u8 op = 0x04; /* set mask field to zero */ 200 201 s390_kernel_write(__va(ftrace_graph_caller)+1, &op, sizeof(op)); 202 return 0; 203 } 204 205 int ftrace_disable_ftrace_graph_caller(void) 206 { 207 u8 op = 0xf4; /* set mask field to all ones */ 208 209 s390_kernel_write(__va(ftrace_graph_caller)+1, &op, sizeof(op)); 210 return 0; 211 } 212 213 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 214 215 #ifdef CONFIG_KPROBES_ON_FTRACE 216 void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, 217 struct ftrace_ops *ops, struct pt_regs *regs) 218 { 219 struct kprobe_ctlblk *kcb; 220 struct kprobe *p = get_kprobe((kprobe_opcode_t *)ip); 221 222 if (unlikely(!p) || kprobe_disabled(p)) 223 return; 224 225 if (kprobe_running()) { 226 kprobes_inc_nmissed_count(p); 227 return; 228 } 229 230 __this_cpu_write(current_kprobe, p); 231 232 kcb = get_kprobe_ctlblk(); 233 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 234 235 instruction_pointer_set(regs, ip); 236 237 if (!p->pre_handler || !p->pre_handler(p, regs)) { 238 239 instruction_pointer_set(regs, ip + MCOUNT_INSN_SIZE); 240 241 if (unlikely(p->post_handler)) { 242 kcb->kprobe_status = KPROBE_HIT_SSDONE; 243 p->post_handler(p, regs, 0); 244 } 245 } 246 __this_cpu_write(current_kprobe, NULL); 247 } 248 NOKPROBE_SYMBOL(kprobe_ftrace_handler); 249 250 int arch_prepare_kprobe_ftrace(struct kprobe *p) 251 { 252 p->ainsn.insn = NULL; 253 return 0; 254 } 255 #endif 256