1f684199fSMasami Hiramatsu /* 2f684199fSMasami Hiramatsu * Kernel Probes Jump Optimization (Optprobes) 3f684199fSMasami Hiramatsu * 4f684199fSMasami Hiramatsu * This program is free software; you can redistribute it and/or modify 5f684199fSMasami Hiramatsu * it under the terms of the GNU General Public License as published by 6f684199fSMasami Hiramatsu * the Free Software Foundation; either version 2 of the License, or 7f684199fSMasami Hiramatsu * (at your option) any later version. 8f684199fSMasami Hiramatsu * 9f684199fSMasami Hiramatsu * This program is distributed in the hope that it will be useful, 10f684199fSMasami Hiramatsu * but WITHOUT ANY WARRANTY; without even the implied warranty of 11f684199fSMasami Hiramatsu * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12f684199fSMasami Hiramatsu * GNU General Public License for more details. 13f684199fSMasami Hiramatsu * 14f684199fSMasami Hiramatsu * You should have received a copy of the GNU General Public License 15f684199fSMasami Hiramatsu * along with this program; if not, write to the Free Software 16f684199fSMasami Hiramatsu * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17f684199fSMasami Hiramatsu * 18f684199fSMasami Hiramatsu * Copyright (C) IBM Corporation, 2002, 2004 19f684199fSMasami Hiramatsu * Copyright (C) Hitachi Ltd., 2012 20f684199fSMasami Hiramatsu */ 21f684199fSMasami Hiramatsu #include <linux/kprobes.h> 22f684199fSMasami Hiramatsu #include <linux/ptrace.h> 23f684199fSMasami Hiramatsu #include <linux/string.h> 24f684199fSMasami Hiramatsu #include <linux/slab.h> 25f684199fSMasami Hiramatsu #include <linux/hardirq.h> 26f684199fSMasami Hiramatsu #include <linux/preempt.h> 27f684199fSMasami Hiramatsu #include <linux/module.h> 28f684199fSMasami Hiramatsu #include <linux/kdebug.h> 29f684199fSMasami Hiramatsu #include <linux/kallsyms.h> 30f684199fSMasami Hiramatsu #include <linux/ftrace.h> 31f684199fSMasami Hiramatsu 32f684199fSMasami Hiramatsu #include <asm/cacheflush.h> 33f684199fSMasami Hiramatsu #include <asm/desc.h> 34f684199fSMasami Hiramatsu #include <asm/pgtable.h> 35f684199fSMasami Hiramatsu #include <asm/uaccess.h> 36f684199fSMasami Hiramatsu #include <asm/alternative.h> 37f684199fSMasami Hiramatsu #include <asm/insn.h> 38f684199fSMasami Hiramatsu #include <asm/debugreg.h> 39f684199fSMasami Hiramatsu 40f684199fSMasami Hiramatsu #include "common.h" 41f684199fSMasami Hiramatsu 42f684199fSMasami Hiramatsu unsigned long __recover_optprobed_insn(kprobe_opcode_t *buf, unsigned long addr) 43f684199fSMasami Hiramatsu { 44f684199fSMasami Hiramatsu struct optimized_kprobe *op; 45f684199fSMasami Hiramatsu struct kprobe *kp; 46f684199fSMasami Hiramatsu long offs; 47f684199fSMasami Hiramatsu int i; 48f684199fSMasami Hiramatsu 49f684199fSMasami Hiramatsu for (i = 0; i < RELATIVEJUMP_SIZE; i++) { 50f684199fSMasami Hiramatsu kp = get_kprobe((void *)addr - i); 51f684199fSMasami Hiramatsu /* This function only handles jump-optimized kprobe */ 52f684199fSMasami Hiramatsu if (kp && kprobe_optimized(kp)) { 53f684199fSMasami Hiramatsu op = container_of(kp, struct optimized_kprobe, kp); 54f684199fSMasami Hiramatsu /* If op->list is not empty, op is under optimizing */ 55f684199fSMasami Hiramatsu if (list_empty(&op->list)) 56f684199fSMasami Hiramatsu goto found; 57f684199fSMasami Hiramatsu } 58f684199fSMasami Hiramatsu } 59f684199fSMasami Hiramatsu 60f684199fSMasami Hiramatsu return addr; 61f684199fSMasami Hiramatsu found: 62f684199fSMasami Hiramatsu /* 63f684199fSMasami Hiramatsu * If the kprobe can be optimized, original bytes which can be 64f684199fSMasami Hiramatsu * overwritten by jump destination address. In this case, original 65f684199fSMasami Hiramatsu * bytes must be recovered from op->optinsn.copied_insn buffer. 66f684199fSMasami Hiramatsu */ 67f684199fSMasami Hiramatsu memcpy(buf, (void *)addr, MAX_INSN_SIZE * sizeof(kprobe_opcode_t)); 68f684199fSMasami Hiramatsu if (addr == (unsigned long)kp->addr) { 69f684199fSMasami Hiramatsu buf[0] = kp->opcode; 70f684199fSMasami Hiramatsu memcpy(buf + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE); 71f684199fSMasami Hiramatsu } else { 72f684199fSMasami Hiramatsu offs = addr - (unsigned long)kp->addr - 1; 73f684199fSMasami Hiramatsu memcpy(buf, op->optinsn.copied_insn + offs, RELATIVE_ADDR_SIZE - offs); 74f684199fSMasami Hiramatsu } 75f684199fSMasami Hiramatsu 76f684199fSMasami Hiramatsu return (unsigned long)buf; 77f684199fSMasami Hiramatsu } 78f684199fSMasami Hiramatsu 79f684199fSMasami Hiramatsu /* Insert a move instruction which sets a pointer to eax/rdi (1st arg). */ 807ec8a97aSMasami Hiramatsu static void synthesize_set_arg1(kprobe_opcode_t *addr, unsigned long val) 81f684199fSMasami Hiramatsu { 82f684199fSMasami Hiramatsu #ifdef CONFIG_X86_64 83f684199fSMasami Hiramatsu *addr++ = 0x48; 84f684199fSMasami Hiramatsu *addr++ = 0xbf; 85f684199fSMasami Hiramatsu #else 86f684199fSMasami Hiramatsu *addr++ = 0xb8; 87f684199fSMasami Hiramatsu #endif 88f684199fSMasami Hiramatsu *(unsigned long *)addr = val; 89f684199fSMasami Hiramatsu } 90f684199fSMasami Hiramatsu 9104bb591cSAndi Kleen asm ( 92f684199fSMasami Hiramatsu ".global optprobe_template_entry\n" 93f684199fSMasami Hiramatsu "optprobe_template_entry:\n" 94f684199fSMasami Hiramatsu #ifdef CONFIG_X86_64 95f684199fSMasami Hiramatsu /* We don't bother saving the ss register */ 96f684199fSMasami Hiramatsu " pushq %rsp\n" 97f684199fSMasami Hiramatsu " pushfq\n" 98f684199fSMasami Hiramatsu SAVE_REGS_STRING 99f684199fSMasami Hiramatsu " movq %rsp, %rsi\n" 100f684199fSMasami Hiramatsu ".global optprobe_template_val\n" 101f684199fSMasami Hiramatsu "optprobe_template_val:\n" 102f684199fSMasami Hiramatsu ASM_NOP5 103f684199fSMasami Hiramatsu ASM_NOP5 104f684199fSMasami Hiramatsu ".global optprobe_template_call\n" 105f684199fSMasami Hiramatsu "optprobe_template_call:\n" 106f684199fSMasami Hiramatsu ASM_NOP5 107f684199fSMasami Hiramatsu /* Move flags to rsp */ 108f684199fSMasami Hiramatsu " movq 144(%rsp), %rdx\n" 109f684199fSMasami Hiramatsu " movq %rdx, 152(%rsp)\n" 110f684199fSMasami Hiramatsu RESTORE_REGS_STRING 111f684199fSMasami Hiramatsu /* Skip flags entry */ 112f684199fSMasami Hiramatsu " addq $8, %rsp\n" 113f684199fSMasami Hiramatsu " popfq\n" 114f684199fSMasami Hiramatsu #else /* CONFIG_X86_32 */ 115f684199fSMasami Hiramatsu " pushf\n" 116f684199fSMasami Hiramatsu SAVE_REGS_STRING 117f684199fSMasami Hiramatsu " movl %esp, %edx\n" 118f684199fSMasami Hiramatsu ".global optprobe_template_val\n" 119f684199fSMasami Hiramatsu "optprobe_template_val:\n" 120f684199fSMasami Hiramatsu ASM_NOP5 121f684199fSMasami Hiramatsu ".global optprobe_template_call\n" 122f684199fSMasami Hiramatsu "optprobe_template_call:\n" 123f684199fSMasami Hiramatsu ASM_NOP5 124f684199fSMasami Hiramatsu RESTORE_REGS_STRING 125f684199fSMasami Hiramatsu " addl $4, %esp\n" /* skip cs */ 126f684199fSMasami Hiramatsu " popf\n" 127f684199fSMasami Hiramatsu #endif 128f684199fSMasami Hiramatsu ".global optprobe_template_end\n" 129f684199fSMasami Hiramatsu "optprobe_template_end:\n"); 130f684199fSMasami Hiramatsu 131f684199fSMasami Hiramatsu #define TMPL_MOVE_IDX \ 132f684199fSMasami Hiramatsu ((long)&optprobe_template_val - (long)&optprobe_template_entry) 133f684199fSMasami Hiramatsu #define TMPL_CALL_IDX \ 134f684199fSMasami Hiramatsu ((long)&optprobe_template_call - (long)&optprobe_template_entry) 135f684199fSMasami Hiramatsu #define TMPL_END_IDX \ 136f684199fSMasami Hiramatsu ((long)&optprobe_template_end - (long)&optprobe_template_entry) 137f684199fSMasami Hiramatsu 138f684199fSMasami Hiramatsu #define INT3_SIZE sizeof(kprobe_opcode_t) 139f684199fSMasami Hiramatsu 140f684199fSMasami Hiramatsu /* Optimized kprobe call back function: called from optinsn */ 1419326638cSMasami Hiramatsu static void 1429326638cSMasami Hiramatsu optimized_callback(struct optimized_kprobe *op, struct pt_regs *regs) 143f684199fSMasami Hiramatsu { 144f684199fSMasami Hiramatsu struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 145f684199fSMasami Hiramatsu unsigned long flags; 146f684199fSMasami Hiramatsu 147f684199fSMasami Hiramatsu /* This is possible if op is under delayed unoptimizing */ 148f684199fSMasami Hiramatsu if (kprobe_disabled(&op->kp)) 149f684199fSMasami Hiramatsu return; 150f684199fSMasami Hiramatsu 151f684199fSMasami Hiramatsu local_irq_save(flags); 152f684199fSMasami Hiramatsu if (kprobe_running()) { 153f684199fSMasami Hiramatsu kprobes_inc_nmissed_count(&op->kp); 154f684199fSMasami Hiramatsu } else { 155f684199fSMasami Hiramatsu /* Save skipped registers */ 156f684199fSMasami Hiramatsu #ifdef CONFIG_X86_64 157f684199fSMasami Hiramatsu regs->cs = __KERNEL_CS; 158f684199fSMasami Hiramatsu #else 159f684199fSMasami Hiramatsu regs->cs = __KERNEL_CS | get_kernel_rpl(); 160f684199fSMasami Hiramatsu regs->gs = 0; 161f684199fSMasami Hiramatsu #endif 162f684199fSMasami Hiramatsu regs->ip = (unsigned long)op->kp.addr + INT3_SIZE; 163f684199fSMasami Hiramatsu regs->orig_ax = ~0UL; 164f684199fSMasami Hiramatsu 165f684199fSMasami Hiramatsu __this_cpu_write(current_kprobe, &op->kp); 166f684199fSMasami Hiramatsu kcb->kprobe_status = KPROBE_HIT_ACTIVE; 167f684199fSMasami Hiramatsu opt_pre_handler(&op->kp, regs); 168f684199fSMasami Hiramatsu __this_cpu_write(current_kprobe, NULL); 169f684199fSMasami Hiramatsu } 170f684199fSMasami Hiramatsu local_irq_restore(flags); 171f684199fSMasami Hiramatsu } 1729326638cSMasami Hiramatsu NOKPROBE_SYMBOL(optimized_callback); 173f684199fSMasami Hiramatsu 1747ec8a97aSMasami Hiramatsu static int copy_optimized_instructions(u8 *dest, u8 *src) 175f684199fSMasami Hiramatsu { 176f684199fSMasami Hiramatsu int len = 0, ret; 177f684199fSMasami Hiramatsu 178f684199fSMasami Hiramatsu while (len < RELATIVEJUMP_SIZE) { 179f684199fSMasami Hiramatsu ret = __copy_instruction(dest + len, src + len); 180f684199fSMasami Hiramatsu if (!ret || !can_boost(dest + len)) 181f684199fSMasami Hiramatsu return -EINVAL; 182f684199fSMasami Hiramatsu len += ret; 183f684199fSMasami Hiramatsu } 184f684199fSMasami Hiramatsu /* Check whether the address range is reserved */ 185f684199fSMasami Hiramatsu if (ftrace_text_reserved(src, src + len - 1) || 186f684199fSMasami Hiramatsu alternatives_text_reserved(src, src + len - 1) || 187f684199fSMasami Hiramatsu jump_label_text_reserved(src, src + len - 1)) 188f684199fSMasami Hiramatsu return -EBUSY; 189f684199fSMasami Hiramatsu 190f684199fSMasami Hiramatsu return len; 191f684199fSMasami Hiramatsu } 192f684199fSMasami Hiramatsu 193f684199fSMasami Hiramatsu /* Check whether insn is indirect jump */ 1947ec8a97aSMasami Hiramatsu static int insn_is_indirect_jump(struct insn *insn) 195f684199fSMasami Hiramatsu { 196f684199fSMasami Hiramatsu return ((insn->opcode.bytes[0] == 0xff && 197f684199fSMasami Hiramatsu (X86_MODRM_REG(insn->modrm.value) & 6) == 4) || /* Jump */ 198f684199fSMasami Hiramatsu insn->opcode.bytes[0] == 0xea); /* Segment based jump */ 199f684199fSMasami Hiramatsu } 200f684199fSMasami Hiramatsu 201f684199fSMasami Hiramatsu /* Check whether insn jumps into specified address range */ 202f684199fSMasami Hiramatsu static int insn_jump_into_range(struct insn *insn, unsigned long start, int len) 203f684199fSMasami Hiramatsu { 204f684199fSMasami Hiramatsu unsigned long target = 0; 205f684199fSMasami Hiramatsu 206f684199fSMasami Hiramatsu switch (insn->opcode.bytes[0]) { 207f684199fSMasami Hiramatsu case 0xe0: /* loopne */ 208f684199fSMasami Hiramatsu case 0xe1: /* loope */ 209f684199fSMasami Hiramatsu case 0xe2: /* loop */ 210f684199fSMasami Hiramatsu case 0xe3: /* jcxz */ 211f684199fSMasami Hiramatsu case 0xe9: /* near relative jump */ 212f684199fSMasami Hiramatsu case 0xeb: /* short relative jump */ 213f684199fSMasami Hiramatsu break; 214f684199fSMasami Hiramatsu case 0x0f: 215f684199fSMasami Hiramatsu if ((insn->opcode.bytes[1] & 0xf0) == 0x80) /* jcc near */ 216f684199fSMasami Hiramatsu break; 217f684199fSMasami Hiramatsu return 0; 218f684199fSMasami Hiramatsu default: 219f684199fSMasami Hiramatsu if ((insn->opcode.bytes[0] & 0xf0) == 0x70) /* jcc short */ 220f684199fSMasami Hiramatsu break; 221f684199fSMasami Hiramatsu return 0; 222f684199fSMasami Hiramatsu } 223f684199fSMasami Hiramatsu target = (unsigned long)insn->next_byte + insn->immediate.value; 224f684199fSMasami Hiramatsu 225f684199fSMasami Hiramatsu return (start <= target && target <= start + len); 226f684199fSMasami Hiramatsu } 227f684199fSMasami Hiramatsu 228f684199fSMasami Hiramatsu /* Decode whole function to ensure any instructions don't jump into target */ 2297ec8a97aSMasami Hiramatsu static int can_optimize(unsigned long paddr) 230f684199fSMasami Hiramatsu { 231f684199fSMasami Hiramatsu unsigned long addr, size = 0, offset = 0; 232f684199fSMasami Hiramatsu struct insn insn; 233f684199fSMasami Hiramatsu kprobe_opcode_t buf[MAX_INSN_SIZE]; 234f684199fSMasami Hiramatsu 235f684199fSMasami Hiramatsu /* Lookup symbol including addr */ 236f684199fSMasami Hiramatsu if (!kallsyms_lookup_size_offset(paddr, &size, &offset)) 237f684199fSMasami Hiramatsu return 0; 238f684199fSMasami Hiramatsu 239f684199fSMasami Hiramatsu /* 240f684199fSMasami Hiramatsu * Do not optimize in the entry code due to the unstable 241f684199fSMasami Hiramatsu * stack handling. 242f684199fSMasami Hiramatsu */ 243f684199fSMasami Hiramatsu if ((paddr >= (unsigned long)__entry_text_start) && 244f684199fSMasami Hiramatsu (paddr < (unsigned long)__entry_text_end)) 245f684199fSMasami Hiramatsu return 0; 246f684199fSMasami Hiramatsu 247f684199fSMasami Hiramatsu /* Check there is enough space for a relative jump. */ 248f684199fSMasami Hiramatsu if (size - offset < RELATIVEJUMP_SIZE) 249f684199fSMasami Hiramatsu return 0; 250f684199fSMasami Hiramatsu 251f684199fSMasami Hiramatsu /* Decode instructions */ 252f684199fSMasami Hiramatsu addr = paddr - offset; 253f684199fSMasami Hiramatsu while (addr < paddr - offset + size) { /* Decode until function end */ 254f684199fSMasami Hiramatsu if (search_exception_tables(addr)) 255f684199fSMasami Hiramatsu /* 256f684199fSMasami Hiramatsu * Since some fixup code will jumps into this function, 257f684199fSMasami Hiramatsu * we can't optimize kprobe in this function. 258f684199fSMasami Hiramatsu */ 259f684199fSMasami Hiramatsu return 0; 260f684199fSMasami Hiramatsu kernel_insn_init(&insn, (void *)recover_probed_instruction(buf, addr)); 261f684199fSMasami Hiramatsu insn_get_length(&insn); 262f684199fSMasami Hiramatsu /* Another subsystem puts a breakpoint */ 263f684199fSMasami Hiramatsu if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION) 264f684199fSMasami Hiramatsu return 0; 265f684199fSMasami Hiramatsu /* Recover address */ 266f684199fSMasami Hiramatsu insn.kaddr = (void *)addr; 267f684199fSMasami Hiramatsu insn.next_byte = (void *)(addr + insn.length); 268f684199fSMasami Hiramatsu /* Check any instructions don't jump into target */ 269f684199fSMasami Hiramatsu if (insn_is_indirect_jump(&insn) || 270f684199fSMasami Hiramatsu insn_jump_into_range(&insn, paddr + INT3_SIZE, 271f684199fSMasami Hiramatsu RELATIVE_ADDR_SIZE)) 272f684199fSMasami Hiramatsu return 0; 273f684199fSMasami Hiramatsu addr += insn.length; 274f684199fSMasami Hiramatsu } 275f684199fSMasami Hiramatsu 276f684199fSMasami Hiramatsu return 1; 277f684199fSMasami Hiramatsu } 278f684199fSMasami Hiramatsu 279f684199fSMasami Hiramatsu /* Check optimized_kprobe can actually be optimized. */ 2807ec8a97aSMasami Hiramatsu int arch_check_optimized_kprobe(struct optimized_kprobe *op) 281f684199fSMasami Hiramatsu { 282f684199fSMasami Hiramatsu int i; 283f684199fSMasami Hiramatsu struct kprobe *p; 284f684199fSMasami Hiramatsu 285f684199fSMasami Hiramatsu for (i = 1; i < op->optinsn.size; i++) { 286f684199fSMasami Hiramatsu p = get_kprobe(op->kp.addr + i); 287f684199fSMasami Hiramatsu if (p && !kprobe_disabled(p)) 288f684199fSMasami Hiramatsu return -EEXIST; 289f684199fSMasami Hiramatsu } 290f684199fSMasami Hiramatsu 291f684199fSMasami Hiramatsu return 0; 292f684199fSMasami Hiramatsu } 293f684199fSMasami Hiramatsu 294f684199fSMasami Hiramatsu /* Check the addr is within the optimized instructions. */ 2957ec8a97aSMasami Hiramatsu int arch_within_optimized_kprobe(struct optimized_kprobe *op, 2967ec8a97aSMasami Hiramatsu unsigned long addr) 297f684199fSMasami Hiramatsu { 298f684199fSMasami Hiramatsu return ((unsigned long)op->kp.addr <= addr && 299f684199fSMasami Hiramatsu (unsigned long)op->kp.addr + op->optinsn.size > addr); 300f684199fSMasami Hiramatsu } 301f684199fSMasami Hiramatsu 302f684199fSMasami Hiramatsu /* Free optimized instruction slot */ 3037ec8a97aSMasami Hiramatsu static 304f684199fSMasami Hiramatsu void __arch_remove_optimized_kprobe(struct optimized_kprobe *op, int dirty) 305f684199fSMasami Hiramatsu { 306f684199fSMasami Hiramatsu if (op->optinsn.insn) { 307f684199fSMasami Hiramatsu free_optinsn_slot(op->optinsn.insn, dirty); 308f684199fSMasami Hiramatsu op->optinsn.insn = NULL; 309f684199fSMasami Hiramatsu op->optinsn.size = 0; 310f684199fSMasami Hiramatsu } 311f684199fSMasami Hiramatsu } 312f684199fSMasami Hiramatsu 3137ec8a97aSMasami Hiramatsu void arch_remove_optimized_kprobe(struct optimized_kprobe *op) 314f684199fSMasami Hiramatsu { 315f684199fSMasami Hiramatsu __arch_remove_optimized_kprobe(op, 1); 316f684199fSMasami Hiramatsu } 317f684199fSMasami Hiramatsu 318f684199fSMasami Hiramatsu /* 319f684199fSMasami Hiramatsu * Copy replacing target instructions 320f684199fSMasami Hiramatsu * Target instructions MUST be relocatable (checked inside) 321f684199fSMasami Hiramatsu * This is called when new aggr(opt)probe is allocated or reused. 322f684199fSMasami Hiramatsu */ 3237ec8a97aSMasami Hiramatsu int arch_prepare_optimized_kprobe(struct optimized_kprobe *op) 324f684199fSMasami Hiramatsu { 325f684199fSMasami Hiramatsu u8 *buf; 326f684199fSMasami Hiramatsu int ret; 327f684199fSMasami Hiramatsu long rel; 328f684199fSMasami Hiramatsu 329f684199fSMasami Hiramatsu if (!can_optimize((unsigned long)op->kp.addr)) 330f684199fSMasami Hiramatsu return -EILSEQ; 331f684199fSMasami Hiramatsu 332f684199fSMasami Hiramatsu op->optinsn.insn = get_optinsn_slot(); 333f684199fSMasami Hiramatsu if (!op->optinsn.insn) 334f684199fSMasami Hiramatsu return -ENOMEM; 335f684199fSMasami Hiramatsu 336f684199fSMasami Hiramatsu /* 337f684199fSMasami Hiramatsu * Verify if the address gap is in 2GB range, because this uses 338f684199fSMasami Hiramatsu * a relative jump. 339f684199fSMasami Hiramatsu */ 340f684199fSMasami Hiramatsu rel = (long)op->optinsn.insn - (long)op->kp.addr + RELATIVEJUMP_SIZE; 341256aae5eSWang Nan if (abs(rel) > 0x7fffffff) { 342256aae5eSWang Nan __arch_remove_optimized_kprobe(op, 0); 343f684199fSMasami Hiramatsu return -ERANGE; 344256aae5eSWang Nan } 345f684199fSMasami Hiramatsu 346f684199fSMasami Hiramatsu buf = (u8 *)op->optinsn.insn; 347f684199fSMasami Hiramatsu 348f684199fSMasami Hiramatsu /* Copy instructions into the out-of-line buffer */ 349f684199fSMasami Hiramatsu ret = copy_optimized_instructions(buf + TMPL_END_IDX, op->kp.addr); 350f684199fSMasami Hiramatsu if (ret < 0) { 351f684199fSMasami Hiramatsu __arch_remove_optimized_kprobe(op, 0); 352f684199fSMasami Hiramatsu return ret; 353f684199fSMasami Hiramatsu } 354f684199fSMasami Hiramatsu op->optinsn.size = ret; 355f684199fSMasami Hiramatsu 356f684199fSMasami Hiramatsu /* Copy arch-dep-instance from template */ 357f684199fSMasami Hiramatsu memcpy(buf, &optprobe_template_entry, TMPL_END_IDX); 358f684199fSMasami Hiramatsu 359f684199fSMasami Hiramatsu /* Set probe information */ 360f684199fSMasami Hiramatsu synthesize_set_arg1(buf + TMPL_MOVE_IDX, (unsigned long)op); 361f684199fSMasami Hiramatsu 362f684199fSMasami Hiramatsu /* Set probe function call */ 363f684199fSMasami Hiramatsu synthesize_relcall(buf + TMPL_CALL_IDX, optimized_callback); 364f684199fSMasami Hiramatsu 365f684199fSMasami Hiramatsu /* Set returning jmp instruction at the tail of out-of-line buffer */ 366f684199fSMasami Hiramatsu synthesize_reljump(buf + TMPL_END_IDX + op->optinsn.size, 367f684199fSMasami Hiramatsu (u8 *)op->kp.addr + op->optinsn.size); 368f684199fSMasami Hiramatsu 369f684199fSMasami Hiramatsu flush_icache_range((unsigned long) buf, 370f684199fSMasami Hiramatsu (unsigned long) buf + TMPL_END_IDX + 371f684199fSMasami Hiramatsu op->optinsn.size + RELATIVEJUMP_SIZE); 372f684199fSMasami Hiramatsu return 0; 373f684199fSMasami Hiramatsu } 374f684199fSMasami Hiramatsu 375a7b0133eSMasami Hiramatsu /* 376a7b0133eSMasami Hiramatsu * Replace breakpoints (int3) with relative jumps. 377a7b0133eSMasami Hiramatsu * Caller must call with locking kprobe_mutex and text_mutex. 378a7b0133eSMasami Hiramatsu */ 3797ec8a97aSMasami Hiramatsu void arch_optimize_kprobes(struct list_head *oplist) 380f684199fSMasami Hiramatsu { 381a7b0133eSMasami Hiramatsu struct optimized_kprobe *op, *tmp; 382a7b0133eSMasami Hiramatsu u8 insn_buf[RELATIVEJUMP_SIZE]; 383a7b0133eSMasami Hiramatsu 384a7b0133eSMasami Hiramatsu list_for_each_entry_safe(op, tmp, oplist, list) { 385f684199fSMasami Hiramatsu s32 rel = (s32)((long)op->optinsn.insn - 386f684199fSMasami Hiramatsu ((long)op->kp.addr + RELATIVEJUMP_SIZE)); 387f684199fSMasami Hiramatsu 388a7b0133eSMasami Hiramatsu WARN_ON(kprobe_disabled(&op->kp)); 389a7b0133eSMasami Hiramatsu 390f684199fSMasami Hiramatsu /* Backup instructions which will be replaced by jump address */ 391f684199fSMasami Hiramatsu memcpy(op->optinsn.copied_insn, op->kp.addr + INT3_SIZE, 392f684199fSMasami Hiramatsu RELATIVE_ADDR_SIZE); 393f684199fSMasami Hiramatsu 394f684199fSMasami Hiramatsu insn_buf[0] = RELATIVEJUMP_OPCODE; 395f684199fSMasami Hiramatsu *(s32 *)(&insn_buf[1]) = rel; 396f684199fSMasami Hiramatsu 397a7b0133eSMasami Hiramatsu text_poke_bp(op->kp.addr, insn_buf, RELATIVEJUMP_SIZE, 398a7b0133eSMasami Hiramatsu op->optinsn.insn); 399f684199fSMasami Hiramatsu 400f684199fSMasami Hiramatsu list_del_init(&op->list); 401a7b0133eSMasami Hiramatsu } 402f684199fSMasami Hiramatsu } 403f684199fSMasami Hiramatsu 404a7b0133eSMasami Hiramatsu /* Replace a relative jump with a breakpoint (int3). */ 4057ec8a97aSMasami Hiramatsu void arch_unoptimize_kprobe(struct optimized_kprobe *op) 406f684199fSMasami Hiramatsu { 407a7b0133eSMasami Hiramatsu u8 insn_buf[RELATIVEJUMP_SIZE]; 408a7b0133eSMasami Hiramatsu 409f684199fSMasami Hiramatsu /* Set int3 to first byte for kprobes */ 410f684199fSMasami Hiramatsu insn_buf[0] = BREAKPOINT_INSTRUCTION; 411f684199fSMasami Hiramatsu memcpy(insn_buf + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE); 412a7b0133eSMasami Hiramatsu text_poke_bp(op->kp.addr, insn_buf, RELATIVEJUMP_SIZE, 413a7b0133eSMasami Hiramatsu op->optinsn.insn); 414f684199fSMasami Hiramatsu } 415f684199fSMasami Hiramatsu 416f684199fSMasami Hiramatsu /* 417f684199fSMasami Hiramatsu * Recover original instructions and breakpoints from relative jumps. 418f684199fSMasami Hiramatsu * Caller must call with locking kprobe_mutex. 419f684199fSMasami Hiramatsu */ 420f684199fSMasami Hiramatsu extern void arch_unoptimize_kprobes(struct list_head *oplist, 421f684199fSMasami Hiramatsu struct list_head *done_list) 422f684199fSMasami Hiramatsu { 423f684199fSMasami Hiramatsu struct optimized_kprobe *op, *tmp; 424f684199fSMasami Hiramatsu 425f684199fSMasami Hiramatsu list_for_each_entry_safe(op, tmp, oplist, list) { 426a7b0133eSMasami Hiramatsu arch_unoptimize_kprobe(op); 427f684199fSMasami Hiramatsu list_move(&op->list, done_list); 428f684199fSMasami Hiramatsu } 429f684199fSMasami Hiramatsu } 430f684199fSMasami Hiramatsu 4319326638cSMasami Hiramatsu int setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter) 432f684199fSMasami Hiramatsu { 433f684199fSMasami Hiramatsu struct optimized_kprobe *op; 434f684199fSMasami Hiramatsu 435f684199fSMasami Hiramatsu if (p->flags & KPROBE_FLAG_OPTIMIZED) { 436f684199fSMasami Hiramatsu /* This kprobe is really able to run optimized path. */ 437f684199fSMasami Hiramatsu op = container_of(p, struct optimized_kprobe, kp); 438f684199fSMasami Hiramatsu /* Detour through copied instructions */ 439f684199fSMasami Hiramatsu regs->ip = (unsigned long)op->optinsn.insn + TMPL_END_IDX; 440f684199fSMasami Hiramatsu if (!reenter) 441f684199fSMasami Hiramatsu reset_current_kprobe(); 442f684199fSMasami Hiramatsu preempt_enable_no_resched(); 443f684199fSMasami Hiramatsu return 1; 444f684199fSMasami Hiramatsu } 445f684199fSMasami Hiramatsu return 0; 446f684199fSMasami Hiramatsu } 4479326638cSMasami Hiramatsu NOKPROBE_SYMBOL(setup_detour_execution); 448