1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Dynamic function tracer architecture backend. 4 * 5 * Copyright IBM Corp. 2009,2014 6 * 7 * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>, 8 * Martin Schwidefsky <schwidefsky@de.ibm.com> 9 */ 10 11 #include <linux/moduleloader.h> 12 #include <linux/hardirq.h> 13 #include <linux/uaccess.h> 14 #include <linux/ftrace.h> 15 #include <linux/kernel.h> 16 #include <linux/types.h> 17 #include <linux/kprobes.h> 18 #include <trace/syscall.h> 19 #include <asm/asm-offsets.h> 20 #include <asm/cacheflush.h> 21 #include <asm/set_memory.h> 22 #include "entry.h" 23 24 /* 25 * The mcount code looks like this: 26 * stg %r14,8(%r15) # offset 0 27 * larl %r1,<&counter> # offset 6 28 * brasl %r14,_mcount # offset 12 29 * lg %r14,8(%r15) # offset 18 30 * Total length is 24 bytes. Only the first instruction will be patched 31 * by ftrace_make_call / ftrace_make_nop. 32 * The enabled ftrace code block looks like this: 33 * > brasl %r0,ftrace_caller # offset 0 34 * larl %r1,<&counter> # offset 6 35 * brasl %r14,_mcount # offset 12 36 * lg %r14,8(%r15) # offset 18 37 * The ftrace function gets called with a non-standard C function call ABI 38 * where r0 contains the return address. It is also expected that the called 39 * function only clobbers r0 and r1, but restores r2-r15. 40 * For module code we can't directly jump to ftrace caller, but need a 41 * trampoline (ftrace_plt), which clobbers also r1. 42 * The return point of the ftrace function has offset 24, so execution 43 * continues behind the mcount block. 44 * The disabled ftrace code block looks like this: 45 * > jg .+24 # offset 0 46 * larl %r1,<&counter> # offset 6 47 * brasl %r14,_mcount # offset 12 48 * lg %r14,8(%r15) # offset 18 49 * The jg instruction branches to offset 24 to skip as many instructions 50 * as possible. 51 * In case we use gcc's hotpatch feature the original and also the disabled 52 * function prologue contains only a single six byte instruction and looks 53 * like this: 54 * > brcl 0,0 # offset 0 55 * To enable ftrace the code gets patched like above and afterwards looks 56 * like this: 57 * > brasl %r0,ftrace_caller # offset 0 58 */ 59 60 unsigned long ftrace_plt; 61 62 static inline void ftrace_generate_orig_insn(struct ftrace_insn *insn) 63 { 64 #if defined(CC_USING_HOTPATCH) || defined(CC_USING_NOP_MCOUNT) 65 /* brcl 0,0 */ 66 insn->opc = 0xc004; 67 insn->disp = 0; 68 #else 69 /* stg r14,8(r15) */ 70 insn->opc = 0xe3e0; 71 insn->disp = 0xf0080024; 72 #endif 73 } 74 75 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, 76 unsigned long addr) 77 { 78 return 0; 79 } 80 81 int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, 82 unsigned long addr) 83 { 84 struct ftrace_insn orig, new, old; 85 86 if (copy_from_kernel_nofault(&old, (void *) rec->ip, sizeof(old))) 87 return -EFAULT; 88 if (addr == MCOUNT_ADDR) { 89 /* Initial code replacement */ 90 ftrace_generate_orig_insn(&orig); 91 ftrace_generate_nop_insn(&new); 92 } else { 93 /* Replace ftrace call with a nop. */ 94 ftrace_generate_call_insn(&orig, rec->ip); 95 ftrace_generate_nop_insn(&new); 96 } 97 /* Verify that the to be replaced code matches what we expect. */ 98 if (memcmp(&orig, &old, sizeof(old))) 99 return -EINVAL; 100 s390_kernel_write((void *) rec->ip, &new, sizeof(new)); 101 return 0; 102 } 103 104 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 105 { 106 struct ftrace_insn orig, new, old; 107 108 if (copy_from_kernel_nofault(&old, (void *) rec->ip, sizeof(old))) 109 return -EFAULT; 110 /* Replace nop with an ftrace call. */ 111 ftrace_generate_nop_insn(&orig); 112 ftrace_generate_call_insn(&new, rec->ip); 113 114 /* Verify that the to be replaced code matches what we expect. */ 115 if (memcmp(&orig, &old, sizeof(old))) 116 return -EINVAL; 117 s390_kernel_write((void *) rec->ip, &new, sizeof(new)); 118 return 0; 119 } 120 121 int ftrace_update_ftrace_func(ftrace_func_t func) 122 { 123 return 0; 124 } 125 126 int __init ftrace_dyn_arch_init(void) 127 { 128 return 0; 129 } 130 131 #ifdef CONFIG_MODULES 132 133 static int __init ftrace_plt_init(void) 134 { 135 unsigned int *ip; 136 137 ftrace_plt = (unsigned long) module_alloc(PAGE_SIZE); 138 if (!ftrace_plt) 139 panic("cannot allocate ftrace plt\n"); 140 ip = (unsigned int *) ftrace_plt; 141 ip[0] = 0x0d10e310; /* basr 1,0; lg 1,10(1); br 1 */ 142 ip[1] = 0x100a0004; 143 ip[2] = 0x07f10000; 144 ip[3] = FTRACE_ADDR >> 32; 145 ip[4] = FTRACE_ADDR & 0xffffffff; 146 set_memory_ro(ftrace_plt, 1); 147 return 0; 148 } 149 device_initcall(ftrace_plt_init); 150 151 #endif /* CONFIG_MODULES */ 152 153 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 154 /* 155 * Hook the return address and push it in the stack of return addresses 156 * in current thread info. 157 */ 158 unsigned long prepare_ftrace_return(unsigned long ra, unsigned long sp, 159 unsigned long ip) 160 { 161 if (unlikely(ftrace_graph_is_dead())) 162 goto out; 163 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 164 goto out; 165 ip -= MCOUNT_INSN_SIZE; 166 if (!function_graph_enter(ra, ip, 0, (void *) sp)) 167 ra = (unsigned long) return_to_handler; 168 out: 169 return ra; 170 } 171 NOKPROBE_SYMBOL(prepare_ftrace_return); 172 173 /* 174 * Patch the kernel code at ftrace_graph_caller location. The instruction 175 * there is branch relative on condition. To enable the ftrace graph code 176 * block, we simply patch the mask field of the instruction to zero and 177 * turn the instruction into a nop. 178 * To disable the ftrace graph code the mask field will be patched to 179 * all ones, which turns the instruction into an unconditional branch. 180 */ 181 int ftrace_enable_ftrace_graph_caller(void) 182 { 183 u8 op = 0x04; /* set mask field to zero */ 184 185 s390_kernel_write(__va(ftrace_graph_caller)+1, &op, sizeof(op)); 186 return 0; 187 } 188 189 int ftrace_disable_ftrace_graph_caller(void) 190 { 191 u8 op = 0xf4; /* set mask field to all ones */ 192 193 s390_kernel_write(__va(ftrace_graph_caller)+1, &op, sizeof(op)); 194 return 0; 195 } 196 197 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 198 199 #ifdef CONFIG_KPROBES_ON_FTRACE 200 void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, 201 struct ftrace_ops *ops, struct pt_regs *regs) 202 { 203 struct kprobe_ctlblk *kcb; 204 struct kprobe *p = get_kprobe((kprobe_opcode_t *)ip); 205 206 if (unlikely(!p) || kprobe_disabled(p)) 207 return; 208 209 if (kprobe_running()) { 210 kprobes_inc_nmissed_count(p); 211 return; 212 } 213 214 __this_cpu_write(current_kprobe, p); 215 216 kcb = get_kprobe_ctlblk(); 217 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 218 219 instruction_pointer_set(regs, ip); 220 221 if (!p->pre_handler || !p->pre_handler(p, regs)) { 222 223 instruction_pointer_set(regs, ip + MCOUNT_INSN_SIZE); 224 225 if (unlikely(p->post_handler)) { 226 kcb->kprobe_status = KPROBE_HIT_SSDONE; 227 p->post_handler(p, regs, 0); 228 } 229 } 230 __this_cpu_write(current_kprobe, NULL); 231 } 232 NOKPROBE_SYMBOL(kprobe_ftrace_handler); 233 234 int arch_prepare_kprobe_ftrace(struct kprobe *p) 235 { 236 p->ainsn.insn = NULL; 237 return 0; 238 } 239 #endif 240