1f684199fSMasami Hiramatsu /* 2f684199fSMasami Hiramatsu * Kernel Probes Jump Optimization (Optprobes) 3f684199fSMasami Hiramatsu * 4f684199fSMasami Hiramatsu * This program is free software; you can redistribute it and/or modify 5f684199fSMasami Hiramatsu * it under the terms of the GNU General Public License as published by 6f684199fSMasami Hiramatsu * the Free Software Foundation; either version 2 of the License, or 7f684199fSMasami Hiramatsu * (at your option) any later version. 8f684199fSMasami Hiramatsu * 9f684199fSMasami Hiramatsu * This program is distributed in the hope that it will be useful, 10f684199fSMasami Hiramatsu * but WITHOUT ANY WARRANTY; without even the implied warranty of 11f684199fSMasami Hiramatsu * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12f684199fSMasami Hiramatsu * GNU General Public License for more details. 13f684199fSMasami Hiramatsu * 14f684199fSMasami Hiramatsu * You should have received a copy of the GNU General Public License 15f684199fSMasami Hiramatsu * along with this program; if not, write to the Free Software 16f684199fSMasami Hiramatsu * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17f684199fSMasami Hiramatsu * 18f684199fSMasami Hiramatsu * Copyright (C) IBM Corporation, 2002, 2004 19f684199fSMasami Hiramatsu * Copyright (C) Hitachi Ltd., 2012 20f684199fSMasami Hiramatsu */ 21f684199fSMasami Hiramatsu #include <linux/kprobes.h> 22f684199fSMasami Hiramatsu #include <linux/ptrace.h> 23f684199fSMasami Hiramatsu #include <linux/string.h> 24f684199fSMasami Hiramatsu #include <linux/slab.h> 25f684199fSMasami Hiramatsu #include <linux/hardirq.h> 26f684199fSMasami Hiramatsu #include <linux/preempt.h> 27744c193eSPaul Gortmaker #include <linux/extable.h> 28f684199fSMasami Hiramatsu #include <linux/kdebug.h> 29f684199fSMasami Hiramatsu #include <linux/kallsyms.h> 30f684199fSMasami Hiramatsu #include <linux/ftrace.h> 31f684199fSMasami Hiramatsu 3235de5b06SAndy Lutomirski #include <asm/text-patching.h> 33f684199fSMasami Hiramatsu #include <asm/cacheflush.h> 34f684199fSMasami Hiramatsu #include <asm/desc.h> 35f684199fSMasami Hiramatsu #include <asm/pgtable.h> 367c0f6ba6SLinus Torvalds #include <linux/uaccess.h> 37f684199fSMasami Hiramatsu #include <asm/alternative.h> 38f684199fSMasami Hiramatsu #include <asm/insn.h> 39f684199fSMasami Hiramatsu #include <asm/debugreg.h> 40f684199fSMasami Hiramatsu 41f684199fSMasami Hiramatsu #include "common.h" 42f684199fSMasami Hiramatsu 43f684199fSMasami Hiramatsu unsigned long __recover_optprobed_insn(kprobe_opcode_t *buf, unsigned long addr) 44f684199fSMasami Hiramatsu { 45f684199fSMasami Hiramatsu struct optimized_kprobe *op; 46f684199fSMasami Hiramatsu struct kprobe *kp; 47f684199fSMasami Hiramatsu long offs; 48f684199fSMasami Hiramatsu int i; 49f684199fSMasami Hiramatsu 50f684199fSMasami Hiramatsu for (i = 0; i < RELATIVEJUMP_SIZE; i++) { 51f684199fSMasami Hiramatsu kp = get_kprobe((void *)addr - i); 52f684199fSMasami Hiramatsu /* This function only handles jump-optimized kprobe */ 53f684199fSMasami Hiramatsu if (kp && kprobe_optimized(kp)) { 54f684199fSMasami Hiramatsu op = container_of(kp, struct optimized_kprobe, kp); 55f684199fSMasami Hiramatsu /* If op->list is not empty, op is under optimizing */ 56f684199fSMasami Hiramatsu if (list_empty(&op->list)) 57f684199fSMasami Hiramatsu goto found; 58f684199fSMasami Hiramatsu } 59f684199fSMasami Hiramatsu } 60f684199fSMasami Hiramatsu 61f684199fSMasami Hiramatsu return addr; 62f684199fSMasami Hiramatsu found: 63f684199fSMasami Hiramatsu /* 64f684199fSMasami Hiramatsu * If the kprobe can be optimized, original bytes which can be 65f684199fSMasami Hiramatsu * overwritten by jump destination address. In this case, original 66f684199fSMasami Hiramatsu * bytes must be recovered from op->optinsn.copied_insn buffer. 67f684199fSMasami Hiramatsu */ 68ea1e34fcSMasami Hiramatsu if (probe_kernel_read(buf, (void *)addr, 69ea1e34fcSMasami Hiramatsu MAX_INSN_SIZE * sizeof(kprobe_opcode_t))) 70ea1e34fcSMasami Hiramatsu return 0UL; 71ea1e34fcSMasami Hiramatsu 72f684199fSMasami Hiramatsu if (addr == (unsigned long)kp->addr) { 73f684199fSMasami Hiramatsu buf[0] = kp->opcode; 74f684199fSMasami Hiramatsu memcpy(buf + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE); 75f684199fSMasami Hiramatsu } else { 76f684199fSMasami Hiramatsu offs = addr - (unsigned long)kp->addr - 1; 77f684199fSMasami Hiramatsu memcpy(buf, op->optinsn.copied_insn + offs, RELATIVE_ADDR_SIZE - offs); 78f684199fSMasami Hiramatsu } 79f684199fSMasami Hiramatsu 80f684199fSMasami Hiramatsu return (unsigned long)buf; 81f684199fSMasami Hiramatsu } 82f684199fSMasami Hiramatsu 83f684199fSMasami Hiramatsu /* Insert a move instruction which sets a pointer to eax/rdi (1st arg). */ 847ec8a97aSMasami Hiramatsu static void synthesize_set_arg1(kprobe_opcode_t *addr, unsigned long val) 85f684199fSMasami Hiramatsu { 86f684199fSMasami Hiramatsu #ifdef CONFIG_X86_64 87f684199fSMasami Hiramatsu *addr++ = 0x48; 88f684199fSMasami Hiramatsu *addr++ = 0xbf; 89f684199fSMasami Hiramatsu #else 90f684199fSMasami Hiramatsu *addr++ = 0xb8; 91f684199fSMasami Hiramatsu #endif 92f684199fSMasami Hiramatsu *(unsigned long *)addr = val; 93f684199fSMasami Hiramatsu } 94f684199fSMasami Hiramatsu 9504bb591cSAndi Kleen asm ( 96f684199fSMasami Hiramatsu ".global optprobe_template_entry\n" 97f684199fSMasami Hiramatsu "optprobe_template_entry:\n" 98f684199fSMasami Hiramatsu #ifdef CONFIG_X86_64 99f684199fSMasami Hiramatsu /* We don't bother saving the ss register */ 100f684199fSMasami Hiramatsu " pushq %rsp\n" 101f684199fSMasami Hiramatsu " pushfq\n" 102f684199fSMasami Hiramatsu SAVE_REGS_STRING 103f684199fSMasami Hiramatsu " movq %rsp, %rsi\n" 104f684199fSMasami Hiramatsu ".global optprobe_template_val\n" 105f684199fSMasami Hiramatsu "optprobe_template_val:\n" 106f684199fSMasami Hiramatsu ASM_NOP5 107f684199fSMasami Hiramatsu ASM_NOP5 108f684199fSMasami Hiramatsu ".global optprobe_template_call\n" 109f684199fSMasami Hiramatsu "optprobe_template_call:\n" 110f684199fSMasami Hiramatsu ASM_NOP5 111f684199fSMasami Hiramatsu /* Move flags to rsp */ 112f684199fSMasami Hiramatsu " movq 144(%rsp), %rdx\n" 113f684199fSMasami Hiramatsu " movq %rdx, 152(%rsp)\n" 114f684199fSMasami Hiramatsu RESTORE_REGS_STRING 115f684199fSMasami Hiramatsu /* Skip flags entry */ 116f684199fSMasami Hiramatsu " addq $8, %rsp\n" 117f684199fSMasami Hiramatsu " popfq\n" 118f684199fSMasami Hiramatsu #else /* CONFIG_X86_32 */ 119f684199fSMasami Hiramatsu " pushf\n" 120f684199fSMasami Hiramatsu SAVE_REGS_STRING 121f684199fSMasami Hiramatsu " movl %esp, %edx\n" 122f684199fSMasami Hiramatsu ".global optprobe_template_val\n" 123f684199fSMasami Hiramatsu "optprobe_template_val:\n" 124f684199fSMasami Hiramatsu ASM_NOP5 125f684199fSMasami Hiramatsu ".global optprobe_template_call\n" 126f684199fSMasami Hiramatsu "optprobe_template_call:\n" 127f684199fSMasami Hiramatsu ASM_NOP5 128f684199fSMasami Hiramatsu RESTORE_REGS_STRING 129f684199fSMasami Hiramatsu " addl $4, %esp\n" /* skip cs */ 130f684199fSMasami Hiramatsu " popf\n" 131f684199fSMasami Hiramatsu #endif 132f684199fSMasami Hiramatsu ".global optprobe_template_end\n" 133f684199fSMasami Hiramatsu "optprobe_template_end:\n"); 134f684199fSMasami Hiramatsu 135f684199fSMasami Hiramatsu #define TMPL_MOVE_IDX \ 136f684199fSMasami Hiramatsu ((long)&optprobe_template_val - (long)&optprobe_template_entry) 137f684199fSMasami Hiramatsu #define TMPL_CALL_IDX \ 138f684199fSMasami Hiramatsu ((long)&optprobe_template_call - (long)&optprobe_template_entry) 139f684199fSMasami Hiramatsu #define TMPL_END_IDX \ 140f684199fSMasami Hiramatsu ((long)&optprobe_template_end - (long)&optprobe_template_entry) 141f684199fSMasami Hiramatsu 142f684199fSMasami Hiramatsu #define INT3_SIZE sizeof(kprobe_opcode_t) 143f684199fSMasami Hiramatsu 144f684199fSMasami Hiramatsu /* Optimized kprobe call back function: called from optinsn */ 1459326638cSMasami Hiramatsu static void 1469326638cSMasami Hiramatsu optimized_callback(struct optimized_kprobe *op, struct pt_regs *regs) 147f684199fSMasami Hiramatsu { 148f684199fSMasami Hiramatsu struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 149f684199fSMasami Hiramatsu unsigned long flags; 150f684199fSMasami Hiramatsu 151f684199fSMasami Hiramatsu /* This is possible if op is under delayed unoptimizing */ 152f684199fSMasami Hiramatsu if (kprobe_disabled(&op->kp)) 153f684199fSMasami Hiramatsu return; 154f684199fSMasami Hiramatsu 155f684199fSMasami Hiramatsu local_irq_save(flags); 156f684199fSMasami Hiramatsu if (kprobe_running()) { 157f684199fSMasami Hiramatsu kprobes_inc_nmissed_count(&op->kp); 158f684199fSMasami Hiramatsu } else { 159f684199fSMasami Hiramatsu /* Save skipped registers */ 160f684199fSMasami Hiramatsu #ifdef CONFIG_X86_64 161f684199fSMasami Hiramatsu regs->cs = __KERNEL_CS; 162f684199fSMasami Hiramatsu #else 163f684199fSMasami Hiramatsu regs->cs = __KERNEL_CS | get_kernel_rpl(); 164f684199fSMasami Hiramatsu regs->gs = 0; 165f684199fSMasami Hiramatsu #endif 166f684199fSMasami Hiramatsu regs->ip = (unsigned long)op->kp.addr + INT3_SIZE; 167f684199fSMasami Hiramatsu regs->orig_ax = ~0UL; 168f684199fSMasami Hiramatsu 169f684199fSMasami Hiramatsu __this_cpu_write(current_kprobe, &op->kp); 170f684199fSMasami Hiramatsu kcb->kprobe_status = KPROBE_HIT_ACTIVE; 171f684199fSMasami Hiramatsu opt_pre_handler(&op->kp, regs); 172f684199fSMasami Hiramatsu __this_cpu_write(current_kprobe, NULL); 173f684199fSMasami Hiramatsu } 174f684199fSMasami Hiramatsu local_irq_restore(flags); 175f684199fSMasami Hiramatsu } 1769326638cSMasami Hiramatsu NOKPROBE_SYMBOL(optimized_callback); 177f684199fSMasami Hiramatsu 1787ec8a97aSMasami Hiramatsu static int copy_optimized_instructions(u8 *dest, u8 *src) 179f684199fSMasami Hiramatsu { 180f684199fSMasami Hiramatsu int len = 0, ret; 181f684199fSMasami Hiramatsu 182f684199fSMasami Hiramatsu while (len < RELATIVEJUMP_SIZE) { 183f684199fSMasami Hiramatsu ret = __copy_instruction(dest + len, src + len); 18475013fb1SMasami Hiramatsu if (!ret || !can_boost(dest + len, src + len)) 185f684199fSMasami Hiramatsu return -EINVAL; 186f684199fSMasami Hiramatsu len += ret; 187f684199fSMasami Hiramatsu } 188f684199fSMasami Hiramatsu /* Check whether the address range is reserved */ 189f684199fSMasami Hiramatsu if (ftrace_text_reserved(src, src + len - 1) || 190f684199fSMasami Hiramatsu alternatives_text_reserved(src, src + len - 1) || 191f684199fSMasami Hiramatsu jump_label_text_reserved(src, src + len - 1)) 192f684199fSMasami Hiramatsu return -EBUSY; 193f684199fSMasami Hiramatsu 194f684199fSMasami Hiramatsu return len; 195f684199fSMasami Hiramatsu } 196f684199fSMasami Hiramatsu 197f684199fSMasami Hiramatsu /* Check whether insn is indirect jump */ 1987ec8a97aSMasami Hiramatsu static int insn_is_indirect_jump(struct insn *insn) 199f684199fSMasami Hiramatsu { 200f684199fSMasami Hiramatsu return ((insn->opcode.bytes[0] == 0xff && 201f684199fSMasami Hiramatsu (X86_MODRM_REG(insn->modrm.value) & 6) == 4) || /* Jump */ 202f684199fSMasami Hiramatsu insn->opcode.bytes[0] == 0xea); /* Segment based jump */ 203f684199fSMasami Hiramatsu } 204f684199fSMasami Hiramatsu 205f684199fSMasami Hiramatsu /* Check whether insn jumps into specified address range */ 206f684199fSMasami Hiramatsu static int insn_jump_into_range(struct insn *insn, unsigned long start, int len) 207f684199fSMasami Hiramatsu { 208f684199fSMasami Hiramatsu unsigned long target = 0; 209f684199fSMasami Hiramatsu 210f684199fSMasami Hiramatsu switch (insn->opcode.bytes[0]) { 211f684199fSMasami Hiramatsu case 0xe0: /* loopne */ 212f684199fSMasami Hiramatsu case 0xe1: /* loope */ 213f684199fSMasami Hiramatsu case 0xe2: /* loop */ 214f684199fSMasami Hiramatsu case 0xe3: /* jcxz */ 215f684199fSMasami Hiramatsu case 0xe9: /* near relative jump */ 216f684199fSMasami Hiramatsu case 0xeb: /* short relative jump */ 217f684199fSMasami Hiramatsu break; 218f684199fSMasami Hiramatsu case 0x0f: 219f684199fSMasami Hiramatsu if ((insn->opcode.bytes[1] & 0xf0) == 0x80) /* jcc near */ 220f684199fSMasami Hiramatsu break; 221f684199fSMasami Hiramatsu return 0; 222f684199fSMasami Hiramatsu default: 223f684199fSMasami Hiramatsu if ((insn->opcode.bytes[0] & 0xf0) == 0x70) /* jcc short */ 224f684199fSMasami Hiramatsu break; 225f684199fSMasami Hiramatsu return 0; 226f684199fSMasami Hiramatsu } 227f684199fSMasami Hiramatsu target = (unsigned long)insn->next_byte + insn->immediate.value; 228f684199fSMasami Hiramatsu 229f684199fSMasami Hiramatsu return (start <= target && target <= start + len); 230f684199fSMasami Hiramatsu } 231f684199fSMasami Hiramatsu 232f684199fSMasami Hiramatsu /* Decode whole function to ensure any instructions don't jump into target */ 2337ec8a97aSMasami Hiramatsu static int can_optimize(unsigned long paddr) 234f684199fSMasami Hiramatsu { 235f684199fSMasami Hiramatsu unsigned long addr, size = 0, offset = 0; 236f684199fSMasami Hiramatsu struct insn insn; 237f684199fSMasami Hiramatsu kprobe_opcode_t buf[MAX_INSN_SIZE]; 238f684199fSMasami Hiramatsu 239f684199fSMasami Hiramatsu /* Lookup symbol including addr */ 240f684199fSMasami Hiramatsu if (!kallsyms_lookup_size_offset(paddr, &size, &offset)) 241f684199fSMasami Hiramatsu return 0; 242f684199fSMasami Hiramatsu 243f684199fSMasami Hiramatsu /* 244f684199fSMasami Hiramatsu * Do not optimize in the entry code due to the unstable 245f684199fSMasami Hiramatsu * stack handling. 246f684199fSMasami Hiramatsu */ 247f684199fSMasami Hiramatsu if ((paddr >= (unsigned long)__entry_text_start) && 248f684199fSMasami Hiramatsu (paddr < (unsigned long)__entry_text_end)) 249f684199fSMasami Hiramatsu return 0; 250f684199fSMasami Hiramatsu 251f684199fSMasami Hiramatsu /* Check there is enough space for a relative jump. */ 252f684199fSMasami Hiramatsu if (size - offset < RELATIVEJUMP_SIZE) 253f684199fSMasami Hiramatsu return 0; 254f684199fSMasami Hiramatsu 255f684199fSMasami Hiramatsu /* Decode instructions */ 256f684199fSMasami Hiramatsu addr = paddr - offset; 257f684199fSMasami Hiramatsu while (addr < paddr - offset + size) { /* Decode until function end */ 2586ba48ff4SDave Hansen unsigned long recovered_insn; 259f684199fSMasami Hiramatsu if (search_exception_tables(addr)) 260f684199fSMasami Hiramatsu /* 261f684199fSMasami Hiramatsu * Since some fixup code will jumps into this function, 262f684199fSMasami Hiramatsu * we can't optimize kprobe in this function. 263f684199fSMasami Hiramatsu */ 264f684199fSMasami Hiramatsu return 0; 2656ba48ff4SDave Hansen recovered_insn = recover_probed_instruction(buf, addr); 2662a6730c8SPetr Mladek if (!recovered_insn) 2672a6730c8SPetr Mladek return 0; 2686ba48ff4SDave Hansen kernel_insn_init(&insn, (void *)recovered_insn, MAX_INSN_SIZE); 269f684199fSMasami Hiramatsu insn_get_length(&insn); 270f684199fSMasami Hiramatsu /* Another subsystem puts a breakpoint */ 271f684199fSMasami Hiramatsu if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION) 272f684199fSMasami Hiramatsu return 0; 273f684199fSMasami Hiramatsu /* Recover address */ 274f684199fSMasami Hiramatsu insn.kaddr = (void *)addr; 275f684199fSMasami Hiramatsu insn.next_byte = (void *)(addr + insn.length); 276f684199fSMasami Hiramatsu /* Check any instructions don't jump into target */ 277f684199fSMasami Hiramatsu if (insn_is_indirect_jump(&insn) || 278f684199fSMasami Hiramatsu insn_jump_into_range(&insn, paddr + INT3_SIZE, 279f684199fSMasami Hiramatsu RELATIVE_ADDR_SIZE)) 280f684199fSMasami Hiramatsu return 0; 281f684199fSMasami Hiramatsu addr += insn.length; 282f684199fSMasami Hiramatsu } 283f684199fSMasami Hiramatsu 284f684199fSMasami Hiramatsu return 1; 285f684199fSMasami Hiramatsu } 286f684199fSMasami Hiramatsu 287f684199fSMasami Hiramatsu /* Check optimized_kprobe can actually be optimized. */ 2887ec8a97aSMasami Hiramatsu int arch_check_optimized_kprobe(struct optimized_kprobe *op) 289f684199fSMasami Hiramatsu { 290f684199fSMasami Hiramatsu int i; 291f684199fSMasami Hiramatsu struct kprobe *p; 292f684199fSMasami Hiramatsu 293f684199fSMasami Hiramatsu for (i = 1; i < op->optinsn.size; i++) { 294f684199fSMasami Hiramatsu p = get_kprobe(op->kp.addr + i); 295f684199fSMasami Hiramatsu if (p && !kprobe_disabled(p)) 296f684199fSMasami Hiramatsu return -EEXIST; 297f684199fSMasami Hiramatsu } 298f684199fSMasami Hiramatsu 299f684199fSMasami Hiramatsu return 0; 300f684199fSMasami Hiramatsu } 301f684199fSMasami Hiramatsu 302f684199fSMasami Hiramatsu /* Check the addr is within the optimized instructions. */ 3037ec8a97aSMasami Hiramatsu int arch_within_optimized_kprobe(struct optimized_kprobe *op, 3047ec8a97aSMasami Hiramatsu unsigned long addr) 305f684199fSMasami Hiramatsu { 306f684199fSMasami Hiramatsu return ((unsigned long)op->kp.addr <= addr && 307f684199fSMasami Hiramatsu (unsigned long)op->kp.addr + op->optinsn.size > addr); 308f684199fSMasami Hiramatsu } 309f684199fSMasami Hiramatsu 310f684199fSMasami Hiramatsu /* Free optimized instruction slot */ 3117ec8a97aSMasami Hiramatsu static 312f684199fSMasami Hiramatsu void __arch_remove_optimized_kprobe(struct optimized_kprobe *op, int dirty) 313f684199fSMasami Hiramatsu { 314f684199fSMasami Hiramatsu if (op->optinsn.insn) { 315f684199fSMasami Hiramatsu free_optinsn_slot(op->optinsn.insn, dirty); 316f684199fSMasami Hiramatsu op->optinsn.insn = NULL; 317f684199fSMasami Hiramatsu op->optinsn.size = 0; 318f684199fSMasami Hiramatsu } 319f684199fSMasami Hiramatsu } 320f684199fSMasami Hiramatsu 3217ec8a97aSMasami Hiramatsu void arch_remove_optimized_kprobe(struct optimized_kprobe *op) 322f684199fSMasami Hiramatsu { 323f684199fSMasami Hiramatsu __arch_remove_optimized_kprobe(op, 1); 324f684199fSMasami Hiramatsu } 325f684199fSMasami Hiramatsu 326f684199fSMasami Hiramatsu /* 327f684199fSMasami Hiramatsu * Copy replacing target instructions 328f684199fSMasami Hiramatsu * Target instructions MUST be relocatable (checked inside) 329f684199fSMasami Hiramatsu * This is called when new aggr(opt)probe is allocated or reused. 330f684199fSMasami Hiramatsu */ 331cbf6ab52SMasami Hiramatsu int arch_prepare_optimized_kprobe(struct optimized_kprobe *op, 332cbf6ab52SMasami Hiramatsu struct kprobe *__unused) 333f684199fSMasami Hiramatsu { 334f684199fSMasami Hiramatsu u8 *buf; 335f684199fSMasami Hiramatsu int ret; 336f684199fSMasami Hiramatsu long rel; 337f684199fSMasami Hiramatsu 338f684199fSMasami Hiramatsu if (!can_optimize((unsigned long)op->kp.addr)) 339f684199fSMasami Hiramatsu return -EILSEQ; 340f684199fSMasami Hiramatsu 341f684199fSMasami Hiramatsu op->optinsn.insn = get_optinsn_slot(); 342f684199fSMasami Hiramatsu if (!op->optinsn.insn) 343f684199fSMasami Hiramatsu return -ENOMEM; 344f684199fSMasami Hiramatsu 345f684199fSMasami Hiramatsu /* 346f684199fSMasami Hiramatsu * Verify if the address gap is in 2GB range, because this uses 347f684199fSMasami Hiramatsu * a relative jump. 348f684199fSMasami Hiramatsu */ 349f684199fSMasami Hiramatsu rel = (long)op->optinsn.insn - (long)op->kp.addr + RELATIVEJUMP_SIZE; 350256aae5eSWang Nan if (abs(rel) > 0x7fffffff) { 351256aae5eSWang Nan __arch_remove_optimized_kprobe(op, 0); 352f684199fSMasami Hiramatsu return -ERANGE; 353256aae5eSWang Nan } 354f684199fSMasami Hiramatsu 355f684199fSMasami Hiramatsu buf = (u8 *)op->optinsn.insn; 356d0381c81SMasami Hiramatsu set_memory_rw((unsigned long)buf & PAGE_MASK, 1); 357f684199fSMasami Hiramatsu 358f684199fSMasami Hiramatsu /* Copy instructions into the out-of-line buffer */ 359f684199fSMasami Hiramatsu ret = copy_optimized_instructions(buf + TMPL_END_IDX, op->kp.addr); 360f684199fSMasami Hiramatsu if (ret < 0) { 361f684199fSMasami Hiramatsu __arch_remove_optimized_kprobe(op, 0); 362f684199fSMasami Hiramatsu return ret; 363f684199fSMasami Hiramatsu } 364f684199fSMasami Hiramatsu op->optinsn.size = ret; 365f684199fSMasami Hiramatsu 366f684199fSMasami Hiramatsu /* Copy arch-dep-instance from template */ 367f684199fSMasami Hiramatsu memcpy(buf, &optprobe_template_entry, TMPL_END_IDX); 368f684199fSMasami Hiramatsu 369f684199fSMasami Hiramatsu /* Set probe information */ 370f684199fSMasami Hiramatsu synthesize_set_arg1(buf + TMPL_MOVE_IDX, (unsigned long)op); 371f684199fSMasami Hiramatsu 372f684199fSMasami Hiramatsu /* Set probe function call */ 373f684199fSMasami Hiramatsu synthesize_relcall(buf + TMPL_CALL_IDX, optimized_callback); 374f684199fSMasami Hiramatsu 375f684199fSMasami Hiramatsu /* Set returning jmp instruction at the tail of out-of-line buffer */ 376f684199fSMasami Hiramatsu synthesize_reljump(buf + TMPL_END_IDX + op->optinsn.size, 377f684199fSMasami Hiramatsu (u8 *)op->kp.addr + op->optinsn.size); 378f684199fSMasami Hiramatsu 379d0381c81SMasami Hiramatsu set_memory_ro((unsigned long)buf & PAGE_MASK, 1); 380d0381c81SMasami Hiramatsu 381f684199fSMasami Hiramatsu flush_icache_range((unsigned long) buf, 382f684199fSMasami Hiramatsu (unsigned long) buf + TMPL_END_IDX + 383f684199fSMasami Hiramatsu op->optinsn.size + RELATIVEJUMP_SIZE); 384f684199fSMasami Hiramatsu return 0; 385f684199fSMasami Hiramatsu } 386f684199fSMasami Hiramatsu 387a7b0133eSMasami Hiramatsu /* 388a7b0133eSMasami Hiramatsu * Replace breakpoints (int3) with relative jumps. 389a7b0133eSMasami Hiramatsu * Caller must call with locking kprobe_mutex and text_mutex. 390a7b0133eSMasami Hiramatsu */ 3917ec8a97aSMasami Hiramatsu void arch_optimize_kprobes(struct list_head *oplist) 392f684199fSMasami Hiramatsu { 393a7b0133eSMasami Hiramatsu struct optimized_kprobe *op, *tmp; 394a7b0133eSMasami Hiramatsu u8 insn_buf[RELATIVEJUMP_SIZE]; 395a7b0133eSMasami Hiramatsu 396a7b0133eSMasami Hiramatsu list_for_each_entry_safe(op, tmp, oplist, list) { 397f684199fSMasami Hiramatsu s32 rel = (s32)((long)op->optinsn.insn - 398f684199fSMasami Hiramatsu ((long)op->kp.addr + RELATIVEJUMP_SIZE)); 399f684199fSMasami Hiramatsu 400a7b0133eSMasami Hiramatsu WARN_ON(kprobe_disabled(&op->kp)); 401a7b0133eSMasami Hiramatsu 402f684199fSMasami Hiramatsu /* Backup instructions which will be replaced by jump address */ 403f684199fSMasami Hiramatsu memcpy(op->optinsn.copied_insn, op->kp.addr + INT3_SIZE, 404f684199fSMasami Hiramatsu RELATIVE_ADDR_SIZE); 405f684199fSMasami Hiramatsu 406f684199fSMasami Hiramatsu insn_buf[0] = RELATIVEJUMP_OPCODE; 407f684199fSMasami Hiramatsu *(s32 *)(&insn_buf[1]) = rel; 408f684199fSMasami Hiramatsu 409a7b0133eSMasami Hiramatsu text_poke_bp(op->kp.addr, insn_buf, RELATIVEJUMP_SIZE, 410a7b0133eSMasami Hiramatsu op->optinsn.insn); 411f684199fSMasami Hiramatsu 412f684199fSMasami Hiramatsu list_del_init(&op->list); 413a7b0133eSMasami Hiramatsu } 414f684199fSMasami Hiramatsu } 415f684199fSMasami Hiramatsu 416a7b0133eSMasami Hiramatsu /* Replace a relative jump with a breakpoint (int3). */ 4177ec8a97aSMasami Hiramatsu void arch_unoptimize_kprobe(struct optimized_kprobe *op) 418f684199fSMasami Hiramatsu { 419a7b0133eSMasami Hiramatsu u8 insn_buf[RELATIVEJUMP_SIZE]; 420a7b0133eSMasami Hiramatsu 421f684199fSMasami Hiramatsu /* Set int3 to first byte for kprobes */ 422f684199fSMasami Hiramatsu insn_buf[0] = BREAKPOINT_INSTRUCTION; 423f684199fSMasami Hiramatsu memcpy(insn_buf + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE); 424a7b0133eSMasami Hiramatsu text_poke_bp(op->kp.addr, insn_buf, RELATIVEJUMP_SIZE, 425a7b0133eSMasami Hiramatsu op->optinsn.insn); 426f684199fSMasami Hiramatsu } 427f684199fSMasami Hiramatsu 428f684199fSMasami Hiramatsu /* 429f684199fSMasami Hiramatsu * Recover original instructions and breakpoints from relative jumps. 430f684199fSMasami Hiramatsu * Caller must call with locking kprobe_mutex. 431f684199fSMasami Hiramatsu */ 432f684199fSMasami Hiramatsu extern void arch_unoptimize_kprobes(struct list_head *oplist, 433f684199fSMasami Hiramatsu struct list_head *done_list) 434f684199fSMasami Hiramatsu { 435f684199fSMasami Hiramatsu struct optimized_kprobe *op, *tmp; 436f684199fSMasami Hiramatsu 437f684199fSMasami Hiramatsu list_for_each_entry_safe(op, tmp, oplist, list) { 438a7b0133eSMasami Hiramatsu arch_unoptimize_kprobe(op); 439f684199fSMasami Hiramatsu list_move(&op->list, done_list); 440f684199fSMasami Hiramatsu } 441f684199fSMasami Hiramatsu } 442f684199fSMasami Hiramatsu 4439326638cSMasami Hiramatsu int setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter) 444f684199fSMasami Hiramatsu { 445f684199fSMasami Hiramatsu struct optimized_kprobe *op; 446f684199fSMasami Hiramatsu 447f684199fSMasami Hiramatsu if (p->flags & KPROBE_FLAG_OPTIMIZED) { 448f684199fSMasami Hiramatsu /* This kprobe is really able to run optimized path. */ 449f684199fSMasami Hiramatsu op = container_of(p, struct optimized_kprobe, kp); 450f684199fSMasami Hiramatsu /* Detour through copied instructions */ 451f684199fSMasami Hiramatsu regs->ip = (unsigned long)op->optinsn.insn + TMPL_END_IDX; 452f684199fSMasami Hiramatsu if (!reenter) 453f684199fSMasami Hiramatsu reset_current_kprobe(); 454f684199fSMasami Hiramatsu preempt_enable_no_resched(); 455f684199fSMasami Hiramatsu return 1; 456f684199fSMasami Hiramatsu } 457f684199fSMasami Hiramatsu return 0; 458f684199fSMasami Hiramatsu } 4599326638cSMasami Hiramatsu NOKPROBE_SYMBOL(setup_detour_execution); 460