1a5f4374aSIngo Molnar /* 2a5f4374aSIngo Molnar * User-space Probes (UProbes) 3a5f4374aSIngo Molnar * 4a5f4374aSIngo Molnar * This program is free software; you can redistribute it and/or modify 5a5f4374aSIngo Molnar * it under the terms of the GNU General Public License as published by 6a5f4374aSIngo Molnar * the Free Software Foundation; either version 2 of the License, or 7a5f4374aSIngo Molnar * (at your option) any later version. 8a5f4374aSIngo Molnar * 9a5f4374aSIngo Molnar * This program is distributed in the hope that it will be useful, 10a5f4374aSIngo Molnar * but WITHOUT ANY WARRANTY; without even the implied warranty of 11a5f4374aSIngo Molnar * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12a5f4374aSIngo Molnar * GNU General Public License for more details. 13a5f4374aSIngo Molnar * 14a5f4374aSIngo Molnar * You should have received a copy of the GNU General Public License 15a5f4374aSIngo Molnar * along with this program; if not, write to the Free Software 16a5f4374aSIngo Molnar * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17a5f4374aSIngo Molnar * 1835aa621bSIngo Molnar * Copyright (C) IBM Corporation, 2008-2012 19a5f4374aSIngo Molnar * Authors: 20a5f4374aSIngo Molnar * Srikar Dronamraju 21a5f4374aSIngo Molnar * Jim Keniston 2290eec103SPeter Zijlstra * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra 23a5f4374aSIngo Molnar */ 24a5f4374aSIngo Molnar 25a5f4374aSIngo Molnar #include <linux/kernel.h> 26a5f4374aSIngo Molnar #include <linux/highmem.h> 27a5f4374aSIngo Molnar #include <linux/pagemap.h> /* read_mapping_page */ 28a5f4374aSIngo Molnar #include <linux/slab.h> 29a5f4374aSIngo Molnar #include <linux/sched.h> 306e84f315SIngo Molnar #include <linux/sched/mm.h> 31f7ccbae4SIngo Molnar #include <linux/sched/coredump.h> 32e8440c14SJosh Stone #include <linux/export.h> 33a5f4374aSIngo Molnar #include <linux/rmap.h> /* anon_vma_prepare */ 34a5f4374aSIngo Molnar #include <linux/mmu_notifier.h> /* set_pte_at_notify */ 35a5f4374aSIngo Molnar #include <linux/swap.h> /* try_to_free_swap */ 360326f5a9SSrikar Dronamraju #include <linux/ptrace.h> /* user_enable_single_step */ 370326f5a9SSrikar Dronamraju #include <linux/kdebug.h> /* notifier mechanism */ 38194f8dcbSOleg Nesterov #include "../../mm/internal.h" /* munlock_vma_page */ 3932cdba1eSOleg Nesterov #include <linux/percpu-rwsem.h> 40aa59c53fSOleg Nesterov #include <linux/task_work.h> 4140814f68SOleg Nesterov #include <linux/shmem_fs.h> 42a5f4374aSIngo Molnar 43a5f4374aSIngo Molnar #include <linux/uprobes.h> 44a5f4374aSIngo Molnar 45d4b3b638SSrikar Dronamraju #define UINSNS_PER_PAGE (PAGE_SIZE/UPROBE_XOL_SLOT_BYTES) 46d4b3b638SSrikar Dronamraju #define MAX_UPROBE_XOL_SLOTS UINSNS_PER_PAGE 47d4b3b638SSrikar Dronamraju 48a5f4374aSIngo Molnar static struct rb_root uprobes_tree = RB_ROOT; 49441f1eb7SOleg Nesterov /* 50441f1eb7SOleg Nesterov * allows us to skip the uprobe_mmap if there are no uprobe events active 51441f1eb7SOleg Nesterov * at this time. Probably a fine grained per inode count is better? 52441f1eb7SOleg Nesterov */ 53441f1eb7SOleg Nesterov #define no_uprobe_events() RB_EMPTY_ROOT(&uprobes_tree) 54a5f4374aSIngo Molnar 55a5f4374aSIngo Molnar static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */ 56a5f4374aSIngo Molnar 57a5f4374aSIngo Molnar #define UPROBES_HASH_SZ 13 58a5f4374aSIngo Molnar /* serialize uprobe->pending_list */ 59a5f4374aSIngo Molnar static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ]; 60a5f4374aSIngo Molnar #define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ]) 61a5f4374aSIngo Molnar 6232cdba1eSOleg Nesterov static struct percpu_rw_semaphore dup_mmap_sem; 6332cdba1eSOleg Nesterov 64cb9a19feSOleg Nesterov /* Have a copy of original instruction */ 6571434f2fSOleg Nesterov #define UPROBE_COPY_INSN 0 66cb9a19feSOleg Nesterov 673ff54efdSSrikar Dronamraju struct uprobe { 683ff54efdSSrikar Dronamraju struct rb_node rb_node; /* node in the rb tree */ 693ff54efdSSrikar Dronamraju atomic_t ref; 70e591c8d7SOleg Nesterov struct rw_semaphore register_rwsem; 713ff54efdSSrikar Dronamraju struct rw_semaphore consumer_rwsem; 723ff54efdSSrikar Dronamraju struct list_head pending_list; 733ff54efdSSrikar Dronamraju struct uprobe_consumer *consumers; 743ff54efdSSrikar Dronamraju struct inode *inode; /* Also hold a ref to inode */ 753ff54efdSSrikar Dronamraju loff_t offset; 761cc33161SRavi Bangoria loff_t ref_ctr_offset; 7771434f2fSOleg Nesterov unsigned long flags; 78ad439356SOleg Nesterov 79ad439356SOleg Nesterov /* 80ad439356SOleg Nesterov * The generic code assumes that it has two members of unknown type 81ad439356SOleg Nesterov * owned by the arch-specific code: 82ad439356SOleg Nesterov * 83ad439356SOleg Nesterov * insn - copy_insn() saves the original instruction here for 84ad439356SOleg Nesterov * arch_uprobe_analyze_insn(). 85ad439356SOleg Nesterov * 86ad439356SOleg Nesterov * ixol - potentially modified instruction to execute out of 87ad439356SOleg Nesterov * line, copied to xol_area by xol_get_insn_slot(). 88ad439356SOleg Nesterov */ 893ff54efdSSrikar Dronamraju struct arch_uprobe arch; 903ff54efdSSrikar Dronamraju }; 913ff54efdSSrikar Dronamraju 921cc33161SRavi Bangoria struct delayed_uprobe { 931cc33161SRavi Bangoria struct list_head list; 941cc33161SRavi Bangoria struct uprobe *uprobe; 951cc33161SRavi Bangoria struct mm_struct *mm; 961cc33161SRavi Bangoria }; 971cc33161SRavi Bangoria 981cc33161SRavi Bangoria static DEFINE_MUTEX(delayed_uprobe_lock); 991cc33161SRavi Bangoria static LIST_HEAD(delayed_uprobe_list); 1001cc33161SRavi Bangoria 101a5f4374aSIngo Molnar /* 102ad439356SOleg Nesterov * Execute out of line area: anonymous executable mapping installed 103ad439356SOleg Nesterov * by the probed task to execute the copy of the original instruction 104ad439356SOleg Nesterov * mangled by set_swbp(). 105ad439356SOleg Nesterov * 106c912dae6SOleg Nesterov * On a breakpoint hit, thread contests for a slot. It frees the 107c912dae6SOleg Nesterov * slot after singlestep. Currently a fixed number of slots are 108c912dae6SOleg Nesterov * allocated. 109c912dae6SOleg Nesterov */ 110c912dae6SOleg Nesterov struct xol_area { 111c912dae6SOleg Nesterov wait_queue_head_t wq; /* if all slots are busy */ 112c912dae6SOleg Nesterov atomic_t slot_count; /* number of in-use slots */ 113c912dae6SOleg Nesterov unsigned long *bitmap; /* 0 = free slot */ 114c912dae6SOleg Nesterov 115704bde3cSOleg Nesterov struct vm_special_mapping xol_mapping; 116704bde3cSOleg Nesterov struct page *pages[2]; 117c912dae6SOleg Nesterov /* 118c912dae6SOleg Nesterov * We keep the vma's vm_start rather than a pointer to the vma 119c912dae6SOleg Nesterov * itself. The probed process or a naughty kernel module could make 120c912dae6SOleg Nesterov * the vma go away, and we must handle that reasonably gracefully. 121c912dae6SOleg Nesterov */ 122c912dae6SOleg Nesterov unsigned long vaddr; /* Page(s) of instruction slots */ 123c912dae6SOleg Nesterov }; 124c912dae6SOleg Nesterov 125c912dae6SOleg Nesterov /* 126a5f4374aSIngo Molnar * valid_vma: Verify if the specified vma is an executable vma 127a5f4374aSIngo Molnar * Relax restrictions while unregistering: vm_flags might have 128a5f4374aSIngo Molnar * changed after breakpoint was inserted. 129a5f4374aSIngo Molnar * - is_register: indicates if we are in register context. 130a5f4374aSIngo Molnar * - Return 1 if the specified virtual address is in an 131a5f4374aSIngo Molnar * executable vma. 132a5f4374aSIngo Molnar */ 133a5f4374aSIngo Molnar static bool valid_vma(struct vm_area_struct *vma, bool is_register) 134a5f4374aSIngo Molnar { 13513f59c5eSOleg Nesterov vm_flags_t flags = VM_HUGETLB | VM_MAYEXEC | VM_MAYSHARE; 136a5f4374aSIngo Molnar 137e40cfce6SOleg Nesterov if (is_register) 138e40cfce6SOleg Nesterov flags |= VM_WRITE; 139a5f4374aSIngo Molnar 140e40cfce6SOleg Nesterov return vma->vm_file && (vma->vm_flags & flags) == VM_MAYEXEC; 141a5f4374aSIngo Molnar } 142a5f4374aSIngo Molnar 14357683f72SOleg Nesterov static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset) 144a5f4374aSIngo Molnar { 14557683f72SOleg Nesterov return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); 146a5f4374aSIngo Molnar } 147a5f4374aSIngo Molnar 148cb113b47SOleg Nesterov static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr) 149cb113b47SOleg Nesterov { 150cb113b47SOleg Nesterov return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start); 151cb113b47SOleg Nesterov } 152cb113b47SOleg Nesterov 153a5f4374aSIngo Molnar /** 154a5f4374aSIngo Molnar * __replace_page - replace page in vma by new page. 155a5f4374aSIngo Molnar * based on replace_page in mm/ksm.c 156a5f4374aSIngo Molnar * 157a5f4374aSIngo Molnar * @vma: vma that holds the pte pointing to page 158c517ee74SOleg Nesterov * @addr: address the old @page is mapped at 159a5f4374aSIngo Molnar * @page: the cowed page we are replacing by kpage 160a5f4374aSIngo Molnar * @kpage: the modified page we replace page by 161a5f4374aSIngo Molnar * 162a5f4374aSIngo Molnar * Returns 0 on success, -EFAULT on failure. 163a5f4374aSIngo Molnar */ 164c517ee74SOleg Nesterov static int __replace_page(struct vm_area_struct *vma, unsigned long addr, 165bdfaa2eeSOleg Nesterov struct page *old_page, struct page *new_page) 166a5f4374aSIngo Molnar { 167a5f4374aSIngo Molnar struct mm_struct *mm = vma->vm_mm; 16814fa2daaSKirill A. Shutemov struct page_vma_mapped_walk pvmw = { 16914fa2daaSKirill A. Shutemov .page = old_page, 17014fa2daaSKirill A. Shutemov .vma = vma, 17114fa2daaSKirill A. Shutemov .address = addr, 17214fa2daaSKirill A. Shutemov }; 1739f92448cSOleg Nesterov int err; 1746bdb913fSHaggai Eran /* For mmu_notifiers */ 1756bdb913fSHaggai Eran const unsigned long mmun_start = addr; 1766bdb913fSHaggai Eran const unsigned long mmun_end = addr + PAGE_SIZE; 17700501b53SJohannes Weiner struct mem_cgroup *memcg; 17800501b53SJohannes Weiner 17914fa2daaSKirill A. Shutemov VM_BUG_ON_PAGE(PageTransHuge(old_page), old_page); 18014fa2daaSKirill A. Shutemov 181bdfaa2eeSOleg Nesterov err = mem_cgroup_try_charge(new_page, vma->vm_mm, GFP_KERNEL, &memcg, 182f627c2f5SKirill A. Shutemov false); 18300501b53SJohannes Weiner if (err) 18400501b53SJohannes Weiner return err; 185a5f4374aSIngo Molnar 186194f8dcbSOleg Nesterov /* For try_to_free_swap() and munlock_vma_page() below */ 187bdfaa2eeSOleg Nesterov lock_page(old_page); 1889f92448cSOleg Nesterov 1896bdb913fSHaggai Eran mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 1909f92448cSOleg Nesterov err = -EAGAIN; 19114fa2daaSKirill A. Shutemov if (!page_vma_mapped_walk(&pvmw)) { 192bdfaa2eeSOleg Nesterov mem_cgroup_cancel_charge(new_page, memcg, false); 1939f92448cSOleg Nesterov goto unlock; 1946c4687ccSOleg Nesterov } 19514fa2daaSKirill A. Shutemov VM_BUG_ON_PAGE(addr != pvmw.address, old_page); 196a5f4374aSIngo Molnar 197bdfaa2eeSOleg Nesterov get_page(new_page); 198bdfaa2eeSOleg Nesterov page_add_new_anon_rmap(new_page, vma, addr, false); 199bdfaa2eeSOleg Nesterov mem_cgroup_commit_charge(new_page, memcg, false, false); 200bdfaa2eeSOleg Nesterov lru_cache_add_active_or_unevictable(new_page, vma); 201a5f4374aSIngo Molnar 202bdfaa2eeSOleg Nesterov if (!PageAnon(old_page)) { 203bdfaa2eeSOleg Nesterov dec_mm_counter(mm, mm_counter_file(old_page)); 2047396fa81SSrikar Dronamraju inc_mm_counter(mm, MM_ANONPAGES); 2057396fa81SSrikar Dronamraju } 2067396fa81SSrikar Dronamraju 20714fa2daaSKirill A. Shutemov flush_cache_page(vma, addr, pte_pfn(*pvmw.pte)); 20814fa2daaSKirill A. Shutemov ptep_clear_flush_notify(vma, addr, pvmw.pte); 20914fa2daaSKirill A. Shutemov set_pte_at_notify(mm, addr, pvmw.pte, 21014fa2daaSKirill A. Shutemov mk_pte(new_page, vma->vm_page_prot)); 211a5f4374aSIngo Molnar 212bdfaa2eeSOleg Nesterov page_remove_rmap(old_page, false); 213bdfaa2eeSOleg Nesterov if (!page_mapped(old_page)) 214bdfaa2eeSOleg Nesterov try_to_free_swap(old_page); 21514fa2daaSKirill A. Shutemov page_vma_mapped_walk_done(&pvmw); 216a5f4374aSIngo Molnar 217194f8dcbSOleg Nesterov if (vma->vm_flags & VM_LOCKED) 218bdfaa2eeSOleg Nesterov munlock_vma_page(old_page); 219bdfaa2eeSOleg Nesterov put_page(old_page); 220194f8dcbSOleg Nesterov 2219f92448cSOleg Nesterov err = 0; 2229f92448cSOleg Nesterov unlock: 2236bdb913fSHaggai Eran mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 224bdfaa2eeSOleg Nesterov unlock_page(old_page); 2259f92448cSOleg Nesterov return err; 226a5f4374aSIngo Molnar } 227a5f4374aSIngo Molnar 228a5f4374aSIngo Molnar /** 2295cb4ac3aSSrikar Dronamraju * is_swbp_insn - check if instruction is breakpoint instruction. 230a5f4374aSIngo Molnar * @insn: instruction to be checked. 2315cb4ac3aSSrikar Dronamraju * Default implementation of is_swbp_insn 232a5f4374aSIngo Molnar * Returns true if @insn is a breakpoint instruction. 233a5f4374aSIngo Molnar */ 2345cb4ac3aSSrikar Dronamraju bool __weak is_swbp_insn(uprobe_opcode_t *insn) 235a5f4374aSIngo Molnar { 2365cb4ac3aSSrikar Dronamraju return *insn == UPROBE_SWBP_INSN; 237a5f4374aSIngo Molnar } 238a5f4374aSIngo Molnar 2390908ad6eSAnanth N Mavinakayanahalli /** 2400908ad6eSAnanth N Mavinakayanahalli * is_trap_insn - check if instruction is breakpoint instruction. 2410908ad6eSAnanth N Mavinakayanahalli * @insn: instruction to be checked. 2420908ad6eSAnanth N Mavinakayanahalli * Default implementation of is_trap_insn 2430908ad6eSAnanth N Mavinakayanahalli * Returns true if @insn is a breakpoint instruction. 2440908ad6eSAnanth N Mavinakayanahalli * 2450908ad6eSAnanth N Mavinakayanahalli * This function is needed for the case where an architecture has multiple 2460908ad6eSAnanth N Mavinakayanahalli * trap instructions (like powerpc). 2470908ad6eSAnanth N Mavinakayanahalli */ 2480908ad6eSAnanth N Mavinakayanahalli bool __weak is_trap_insn(uprobe_opcode_t *insn) 2490908ad6eSAnanth N Mavinakayanahalli { 2500908ad6eSAnanth N Mavinakayanahalli return is_swbp_insn(insn); 2510908ad6eSAnanth N Mavinakayanahalli } 2520908ad6eSAnanth N Mavinakayanahalli 253ab0d805cSOleg Nesterov static void copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len) 254cceb55aaSOleg Nesterov { 255cceb55aaSOleg Nesterov void *kaddr = kmap_atomic(page); 256ab0d805cSOleg Nesterov memcpy(dst, kaddr + (vaddr & ~PAGE_MASK), len); 257cceb55aaSOleg Nesterov kunmap_atomic(kaddr); 258cceb55aaSOleg Nesterov } 259cceb55aaSOleg Nesterov 2605669cceeSOleg Nesterov static void copy_to_page(struct page *page, unsigned long vaddr, const void *src, int len) 2615669cceeSOleg Nesterov { 2625669cceeSOleg Nesterov void *kaddr = kmap_atomic(page); 2635669cceeSOleg Nesterov memcpy(kaddr + (vaddr & ~PAGE_MASK), src, len); 2645669cceeSOleg Nesterov kunmap_atomic(kaddr); 2655669cceeSOleg Nesterov } 2665669cceeSOleg Nesterov 267ed6f6a50SOleg Nesterov static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode) 268ed6f6a50SOleg Nesterov { 269ed6f6a50SOleg Nesterov uprobe_opcode_t old_opcode; 270ed6f6a50SOleg Nesterov bool is_swbp; 271ed6f6a50SOleg Nesterov 2720908ad6eSAnanth N Mavinakayanahalli /* 2730908ad6eSAnanth N Mavinakayanahalli * Note: We only check if the old_opcode is UPROBE_SWBP_INSN here. 2740908ad6eSAnanth N Mavinakayanahalli * We do not check if it is any other 'trap variant' which could 2750908ad6eSAnanth N Mavinakayanahalli * be conditional trap instruction such as the one powerpc supports. 2760908ad6eSAnanth N Mavinakayanahalli * 2770908ad6eSAnanth N Mavinakayanahalli * The logic is that we do not care if the underlying instruction 2780908ad6eSAnanth N Mavinakayanahalli * is a trap variant; uprobes always wins over any other (gdb) 2790908ad6eSAnanth N Mavinakayanahalli * breakpoint. 2800908ad6eSAnanth N Mavinakayanahalli */ 281ab0d805cSOleg Nesterov copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE); 282ed6f6a50SOleg Nesterov is_swbp = is_swbp_insn(&old_opcode); 283ed6f6a50SOleg Nesterov 284ed6f6a50SOleg Nesterov if (is_swbp_insn(new_opcode)) { 285ed6f6a50SOleg Nesterov if (is_swbp) /* register: already installed? */ 286ed6f6a50SOleg Nesterov return 0; 287ed6f6a50SOleg Nesterov } else { 288ed6f6a50SOleg Nesterov if (!is_swbp) /* unregister: was it changed by us? */ 289076a365bSOleg Nesterov return 0; 290ed6f6a50SOleg Nesterov } 291ed6f6a50SOleg Nesterov 292ed6f6a50SOleg Nesterov return 1; 293ed6f6a50SOleg Nesterov } 294ed6f6a50SOleg Nesterov 2951cc33161SRavi Bangoria static struct delayed_uprobe * 2961cc33161SRavi Bangoria delayed_uprobe_check(struct uprobe *uprobe, struct mm_struct *mm) 2971cc33161SRavi Bangoria { 2981cc33161SRavi Bangoria struct delayed_uprobe *du; 2991cc33161SRavi Bangoria 3001cc33161SRavi Bangoria list_for_each_entry(du, &delayed_uprobe_list, list) 3011cc33161SRavi Bangoria if (du->uprobe == uprobe && du->mm == mm) 3021cc33161SRavi Bangoria return du; 3031cc33161SRavi Bangoria return NULL; 3041cc33161SRavi Bangoria } 3051cc33161SRavi Bangoria 3061cc33161SRavi Bangoria static int delayed_uprobe_add(struct uprobe *uprobe, struct mm_struct *mm) 3071cc33161SRavi Bangoria { 3081cc33161SRavi Bangoria struct delayed_uprobe *du; 3091cc33161SRavi Bangoria 3101cc33161SRavi Bangoria if (delayed_uprobe_check(uprobe, mm)) 3111cc33161SRavi Bangoria return 0; 3121cc33161SRavi Bangoria 3131cc33161SRavi Bangoria du = kzalloc(sizeof(*du), GFP_KERNEL); 3141cc33161SRavi Bangoria if (!du) 3151cc33161SRavi Bangoria return -ENOMEM; 3161cc33161SRavi Bangoria 3171cc33161SRavi Bangoria du->uprobe = uprobe; 3181cc33161SRavi Bangoria du->mm = mm; 3191cc33161SRavi Bangoria list_add(&du->list, &delayed_uprobe_list); 3201cc33161SRavi Bangoria return 0; 3211cc33161SRavi Bangoria } 3221cc33161SRavi Bangoria 3231cc33161SRavi Bangoria static void delayed_uprobe_delete(struct delayed_uprobe *du) 3241cc33161SRavi Bangoria { 3251cc33161SRavi Bangoria if (WARN_ON(!du)) 3261cc33161SRavi Bangoria return; 3271cc33161SRavi Bangoria list_del(&du->list); 3281cc33161SRavi Bangoria kfree(du); 3291cc33161SRavi Bangoria } 3301cc33161SRavi Bangoria 3311cc33161SRavi Bangoria static void delayed_uprobe_remove(struct uprobe *uprobe, struct mm_struct *mm) 3321cc33161SRavi Bangoria { 3331cc33161SRavi Bangoria struct list_head *pos, *q; 3341cc33161SRavi Bangoria struct delayed_uprobe *du; 3351cc33161SRavi Bangoria 3361cc33161SRavi Bangoria if (!uprobe && !mm) 3371cc33161SRavi Bangoria return; 3381cc33161SRavi Bangoria 3391cc33161SRavi Bangoria list_for_each_safe(pos, q, &delayed_uprobe_list) { 3401cc33161SRavi Bangoria du = list_entry(pos, struct delayed_uprobe, list); 3411cc33161SRavi Bangoria 3421cc33161SRavi Bangoria if (uprobe && du->uprobe != uprobe) 3431cc33161SRavi Bangoria continue; 3441cc33161SRavi Bangoria if (mm && du->mm != mm) 3451cc33161SRavi Bangoria continue; 3461cc33161SRavi Bangoria 3471cc33161SRavi Bangoria delayed_uprobe_delete(du); 3481cc33161SRavi Bangoria } 3491cc33161SRavi Bangoria } 3501cc33161SRavi Bangoria 3511cc33161SRavi Bangoria static bool valid_ref_ctr_vma(struct uprobe *uprobe, 3521cc33161SRavi Bangoria struct vm_area_struct *vma) 3531cc33161SRavi Bangoria { 3541cc33161SRavi Bangoria unsigned long vaddr = offset_to_vaddr(vma, uprobe->ref_ctr_offset); 3551cc33161SRavi Bangoria 3561cc33161SRavi Bangoria return uprobe->ref_ctr_offset && 3571cc33161SRavi Bangoria vma->vm_file && 3581cc33161SRavi Bangoria file_inode(vma->vm_file) == uprobe->inode && 3591cc33161SRavi Bangoria (vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE && 3601cc33161SRavi Bangoria vma->vm_start <= vaddr && 3611cc33161SRavi Bangoria vma->vm_end > vaddr; 3621cc33161SRavi Bangoria } 3631cc33161SRavi Bangoria 3641cc33161SRavi Bangoria static struct vm_area_struct * 3651cc33161SRavi Bangoria find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm) 3661cc33161SRavi Bangoria { 3671cc33161SRavi Bangoria struct vm_area_struct *tmp; 3681cc33161SRavi Bangoria 3691cc33161SRavi Bangoria for (tmp = mm->mmap; tmp; tmp = tmp->vm_next) 3701cc33161SRavi Bangoria if (valid_ref_ctr_vma(uprobe, tmp)) 3711cc33161SRavi Bangoria return tmp; 3721cc33161SRavi Bangoria 3731cc33161SRavi Bangoria return NULL; 3741cc33161SRavi Bangoria } 3751cc33161SRavi Bangoria 3761cc33161SRavi Bangoria static int 3771cc33161SRavi Bangoria __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d) 3781cc33161SRavi Bangoria { 3791cc33161SRavi Bangoria void *kaddr; 3801cc33161SRavi Bangoria struct page *page; 3811cc33161SRavi Bangoria struct vm_area_struct *vma; 3821cc33161SRavi Bangoria int ret; 3831cc33161SRavi Bangoria short *ptr; 3841cc33161SRavi Bangoria 3851cc33161SRavi Bangoria if (!vaddr || !d) 3861cc33161SRavi Bangoria return -EINVAL; 3871cc33161SRavi Bangoria 3881cc33161SRavi Bangoria ret = get_user_pages_remote(NULL, mm, vaddr, 1, 3891cc33161SRavi Bangoria FOLL_WRITE, &page, &vma, NULL); 3901cc33161SRavi Bangoria if (unlikely(ret <= 0)) { 3911cc33161SRavi Bangoria /* 3921cc33161SRavi Bangoria * We are asking for 1 page. If get_user_pages_remote() fails, 3931cc33161SRavi Bangoria * it may return 0, in that case we have to return error. 3941cc33161SRavi Bangoria */ 3951cc33161SRavi Bangoria return ret == 0 ? -EBUSY : ret; 3961cc33161SRavi Bangoria } 3971cc33161SRavi Bangoria 3981cc33161SRavi Bangoria kaddr = kmap_atomic(page); 3991cc33161SRavi Bangoria ptr = kaddr + (vaddr & ~PAGE_MASK); 4001cc33161SRavi Bangoria 4011cc33161SRavi Bangoria if (unlikely(*ptr + d < 0)) { 4021cc33161SRavi Bangoria pr_warn("ref_ctr going negative. vaddr: 0x%lx, " 4031cc33161SRavi Bangoria "curr val: %d, delta: %d\n", vaddr, *ptr, d); 4041cc33161SRavi Bangoria ret = -EINVAL; 4051cc33161SRavi Bangoria goto out; 4061cc33161SRavi Bangoria } 4071cc33161SRavi Bangoria 4081cc33161SRavi Bangoria *ptr += d; 4091cc33161SRavi Bangoria ret = 0; 4101cc33161SRavi Bangoria out: 4111cc33161SRavi Bangoria kunmap_atomic(kaddr); 4121cc33161SRavi Bangoria put_page(page); 4131cc33161SRavi Bangoria return ret; 4141cc33161SRavi Bangoria } 4151cc33161SRavi Bangoria 4161cc33161SRavi Bangoria static void update_ref_ctr_warn(struct uprobe *uprobe, 4171cc33161SRavi Bangoria struct mm_struct *mm, short d) 4181cc33161SRavi Bangoria { 4191cc33161SRavi Bangoria pr_warn("ref_ctr %s failed for inode: 0x%lx offset: " 4201cc33161SRavi Bangoria "0x%llx ref_ctr_offset: 0x%llx of mm: 0x%pK\n", 4211cc33161SRavi Bangoria d > 0 ? "increment" : "decrement", uprobe->inode->i_ino, 4221cc33161SRavi Bangoria (unsigned long long) uprobe->offset, 4231cc33161SRavi Bangoria (unsigned long long) uprobe->ref_ctr_offset, mm); 4241cc33161SRavi Bangoria } 4251cc33161SRavi Bangoria 4261cc33161SRavi Bangoria static int update_ref_ctr(struct uprobe *uprobe, struct mm_struct *mm, 4271cc33161SRavi Bangoria short d) 4281cc33161SRavi Bangoria { 4291cc33161SRavi Bangoria struct vm_area_struct *rc_vma; 4301cc33161SRavi Bangoria unsigned long rc_vaddr; 4311cc33161SRavi Bangoria int ret = 0; 4321cc33161SRavi Bangoria 4331cc33161SRavi Bangoria rc_vma = find_ref_ctr_vma(uprobe, mm); 4341cc33161SRavi Bangoria 4351cc33161SRavi Bangoria if (rc_vma) { 4361cc33161SRavi Bangoria rc_vaddr = offset_to_vaddr(rc_vma, uprobe->ref_ctr_offset); 4371cc33161SRavi Bangoria ret = __update_ref_ctr(mm, rc_vaddr, d); 4381cc33161SRavi Bangoria if (ret) 4391cc33161SRavi Bangoria update_ref_ctr_warn(uprobe, mm, d); 4401cc33161SRavi Bangoria 4411cc33161SRavi Bangoria if (d > 0) 4421cc33161SRavi Bangoria return ret; 4431cc33161SRavi Bangoria } 4441cc33161SRavi Bangoria 4451cc33161SRavi Bangoria mutex_lock(&delayed_uprobe_lock); 4461cc33161SRavi Bangoria if (d > 0) 4471cc33161SRavi Bangoria ret = delayed_uprobe_add(uprobe, mm); 4481cc33161SRavi Bangoria else 4491cc33161SRavi Bangoria delayed_uprobe_remove(uprobe, mm); 4501cc33161SRavi Bangoria mutex_unlock(&delayed_uprobe_lock); 4511cc33161SRavi Bangoria 4521cc33161SRavi Bangoria return ret; 4531cc33161SRavi Bangoria } 4541cc33161SRavi Bangoria 455a5f4374aSIngo Molnar /* 456a5f4374aSIngo Molnar * NOTE: 457a5f4374aSIngo Molnar * Expect the breakpoint instruction to be the smallest size instruction for 458a5f4374aSIngo Molnar * the architecture. If an arch has variable length instruction and the 459a5f4374aSIngo Molnar * breakpoint instruction is not of the smallest length instruction 4600908ad6eSAnanth N Mavinakayanahalli * supported by that architecture then we need to modify is_trap_at_addr and 461f72d41faSOleg Nesterov * uprobe_write_opcode accordingly. This would never be a problem for archs 462f72d41faSOleg Nesterov * that have fixed length instructions. 46329dedee0SOleg Nesterov * 464f72d41faSOleg Nesterov * uprobe_write_opcode - write the opcode at a given virtual address. 465a5f4374aSIngo Molnar * @mm: the probed process address space. 466a5f4374aSIngo Molnar * @vaddr: the virtual address to store the opcode. 467a5f4374aSIngo Molnar * @opcode: opcode to be written at @vaddr. 468a5f4374aSIngo Molnar * 46929dedee0SOleg Nesterov * Called with mm->mmap_sem held for write. 470a5f4374aSIngo Molnar * Return 0 (success) or a negative errno. 471a5f4374aSIngo Molnar */ 4726d43743eSRavi Bangoria int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm, 4736d43743eSRavi Bangoria unsigned long vaddr, uprobe_opcode_t opcode) 474a5f4374aSIngo Molnar { 4751cc33161SRavi Bangoria struct uprobe *uprobe; 476a5f4374aSIngo Molnar struct page *old_page, *new_page; 477a5f4374aSIngo Molnar struct vm_area_struct *vma; 4781cc33161SRavi Bangoria int ret, is_register, ref_ctr_updated = 0; 4791cc33161SRavi Bangoria 4801cc33161SRavi Bangoria is_register = is_swbp_insn(&opcode); 4811cc33161SRavi Bangoria uprobe = container_of(auprobe, struct uprobe, arch); 482f403072cSOleg Nesterov 4835323ce71SOleg Nesterov retry: 484a5f4374aSIngo Molnar /* Read the page with vaddr into memory */ 485c8394812SKirill A. Shutemov ret = get_user_pages_remote(NULL, mm, vaddr, 1, 486c8394812SKirill A. Shutemov FOLL_FORCE | FOLL_SPLIT, &old_page, &vma, NULL); 487a5f4374aSIngo Molnar if (ret <= 0) 488a5f4374aSIngo Molnar return ret; 489a5f4374aSIngo Molnar 490ed6f6a50SOleg Nesterov ret = verify_opcode(old_page, vaddr, &opcode); 491ed6f6a50SOleg Nesterov if (ret <= 0) 492ed6f6a50SOleg Nesterov goto put_old; 493ed6f6a50SOleg Nesterov 4941cc33161SRavi Bangoria /* We are going to replace instruction, update ref_ctr. */ 4951cc33161SRavi Bangoria if (!ref_ctr_updated && uprobe->ref_ctr_offset) { 4961cc33161SRavi Bangoria ret = update_ref_ctr(uprobe, mm, is_register ? 1 : -1); 4971cc33161SRavi Bangoria if (ret) 4981cc33161SRavi Bangoria goto put_old; 4991cc33161SRavi Bangoria 5001cc33161SRavi Bangoria ref_ctr_updated = 1; 5011cc33161SRavi Bangoria } 5021cc33161SRavi Bangoria 50329dedee0SOleg Nesterov ret = anon_vma_prepare(vma); 50429dedee0SOleg Nesterov if (ret) 50529dedee0SOleg Nesterov goto put_old; 50629dedee0SOleg Nesterov 507a5f4374aSIngo Molnar ret = -ENOMEM; 508a5f4374aSIngo Molnar new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr); 509a5f4374aSIngo Molnar if (!new_page) 5109f92448cSOleg Nesterov goto put_old; 511a5f4374aSIngo Molnar 51229dedee0SOleg Nesterov __SetPageUptodate(new_page); 5133f47107cSOleg Nesterov copy_highpage(new_page, old_page); 5143f47107cSOleg Nesterov copy_to_page(new_page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE); 515a5f4374aSIngo Molnar 516c517ee74SOleg Nesterov ret = __replace_page(vma, vaddr, old_page, new_page); 51709cbfeafSKirill A. Shutemov put_page(new_page); 5189f92448cSOleg Nesterov put_old: 519a5f4374aSIngo Molnar put_page(old_page); 520a5f4374aSIngo Molnar 5215323ce71SOleg Nesterov if (unlikely(ret == -EAGAIN)) 5225323ce71SOleg Nesterov goto retry; 5231cc33161SRavi Bangoria 5241cc33161SRavi Bangoria /* Revert back reference counter if instruction update failed. */ 5251cc33161SRavi Bangoria if (ret && is_register && ref_ctr_updated) 5261cc33161SRavi Bangoria update_ref_ctr(uprobe, mm, -1); 5271cc33161SRavi Bangoria 528a5f4374aSIngo Molnar return ret; 529a5f4374aSIngo Molnar } 530a5f4374aSIngo Molnar 531a5f4374aSIngo Molnar /** 5325cb4ac3aSSrikar Dronamraju * set_swbp - store breakpoint at a given address. 533e3343e6aSSrikar Dronamraju * @auprobe: arch specific probepoint information. 534a5f4374aSIngo Molnar * @mm: the probed process address space. 535a5f4374aSIngo Molnar * @vaddr: the virtual address to insert the opcode. 536a5f4374aSIngo Molnar * 537a5f4374aSIngo Molnar * For mm @mm, store the breakpoint instruction at @vaddr. 538a5f4374aSIngo Molnar * Return 0 (success) or a negative errno. 539a5f4374aSIngo Molnar */ 5405cb4ac3aSSrikar Dronamraju int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr) 541a5f4374aSIngo Molnar { 5426d43743eSRavi Bangoria return uprobe_write_opcode(auprobe, mm, vaddr, UPROBE_SWBP_INSN); 543a5f4374aSIngo Molnar } 544a5f4374aSIngo Molnar 545a5f4374aSIngo Molnar /** 546a5f4374aSIngo Molnar * set_orig_insn - Restore the original instruction. 547a5f4374aSIngo Molnar * @mm: the probed process address space. 548e3343e6aSSrikar Dronamraju * @auprobe: arch specific probepoint information. 549a5f4374aSIngo Molnar * @vaddr: the virtual address to insert the opcode. 550a5f4374aSIngo Molnar * 551a5f4374aSIngo Molnar * For mm @mm, restore the original opcode (opcode) at @vaddr. 552a5f4374aSIngo Molnar * Return 0 (success) or a negative errno. 553a5f4374aSIngo Molnar */ 554a5f4374aSIngo Molnar int __weak 555ded86e7cSOleg Nesterov set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr) 556a5f4374aSIngo Molnar { 5576d43743eSRavi Bangoria return uprobe_write_opcode(auprobe, mm, vaddr, 5586d43743eSRavi Bangoria *(uprobe_opcode_t *)&auprobe->insn); 559a5f4374aSIngo Molnar } 560a5f4374aSIngo Molnar 561f231722aSOleg Nesterov static struct uprobe *get_uprobe(struct uprobe *uprobe) 562f231722aSOleg Nesterov { 563f231722aSOleg Nesterov atomic_inc(&uprobe->ref); 564f231722aSOleg Nesterov return uprobe; 565f231722aSOleg Nesterov } 566f231722aSOleg Nesterov 567f231722aSOleg Nesterov static void put_uprobe(struct uprobe *uprobe) 568f231722aSOleg Nesterov { 5691cc33161SRavi Bangoria if (atomic_dec_and_test(&uprobe->ref)) { 5701cc33161SRavi Bangoria /* 5711cc33161SRavi Bangoria * If application munmap(exec_vma) before uprobe_unregister() 5721cc33161SRavi Bangoria * gets called, we don't get a chance to remove uprobe from 5731cc33161SRavi Bangoria * delayed_uprobe_list from remove_breakpoint(). Do it here. 5741cc33161SRavi Bangoria */ 5751aed58e6SRavi Bangoria mutex_lock(&delayed_uprobe_lock); 5761cc33161SRavi Bangoria delayed_uprobe_remove(uprobe, NULL); 5771aed58e6SRavi Bangoria mutex_unlock(&delayed_uprobe_lock); 578f231722aSOleg Nesterov kfree(uprobe); 579f231722aSOleg Nesterov } 5801cc33161SRavi Bangoria } 581f231722aSOleg Nesterov 582a5f4374aSIngo Molnar static int match_uprobe(struct uprobe *l, struct uprobe *r) 583a5f4374aSIngo Molnar { 584a5f4374aSIngo Molnar if (l->inode < r->inode) 585a5f4374aSIngo Molnar return -1; 586a5f4374aSIngo Molnar 587a5f4374aSIngo Molnar if (l->inode > r->inode) 588a5f4374aSIngo Molnar return 1; 589a5f4374aSIngo Molnar 590a5f4374aSIngo Molnar if (l->offset < r->offset) 591a5f4374aSIngo Molnar return -1; 592a5f4374aSIngo Molnar 593a5f4374aSIngo Molnar if (l->offset > r->offset) 594a5f4374aSIngo Molnar return 1; 595a5f4374aSIngo Molnar 596a5f4374aSIngo Molnar return 0; 597a5f4374aSIngo Molnar } 598a5f4374aSIngo Molnar 599a5f4374aSIngo Molnar static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset) 600a5f4374aSIngo Molnar { 601a5f4374aSIngo Molnar struct uprobe u = { .inode = inode, .offset = offset }; 602a5f4374aSIngo Molnar struct rb_node *n = uprobes_tree.rb_node; 603a5f4374aSIngo Molnar struct uprobe *uprobe; 604a5f4374aSIngo Molnar int match; 605a5f4374aSIngo Molnar 606a5f4374aSIngo Molnar while (n) { 607a5f4374aSIngo Molnar uprobe = rb_entry(n, struct uprobe, rb_node); 608a5f4374aSIngo Molnar match = match_uprobe(&u, uprobe); 609f231722aSOleg Nesterov if (!match) 610f231722aSOleg Nesterov return get_uprobe(uprobe); 611a5f4374aSIngo Molnar 612a5f4374aSIngo Molnar if (match < 0) 613a5f4374aSIngo Molnar n = n->rb_left; 614a5f4374aSIngo Molnar else 615a5f4374aSIngo Molnar n = n->rb_right; 616a5f4374aSIngo Molnar } 617a5f4374aSIngo Molnar return NULL; 618a5f4374aSIngo Molnar } 619a5f4374aSIngo Molnar 620a5f4374aSIngo Molnar /* 621a5f4374aSIngo Molnar * Find a uprobe corresponding to a given inode:offset 622a5f4374aSIngo Molnar * Acquires uprobes_treelock 623a5f4374aSIngo Molnar */ 624a5f4374aSIngo Molnar static struct uprobe *find_uprobe(struct inode *inode, loff_t offset) 625a5f4374aSIngo Molnar { 626a5f4374aSIngo Molnar struct uprobe *uprobe; 627a5f4374aSIngo Molnar 6286f47caa0SOleg Nesterov spin_lock(&uprobes_treelock); 629a5f4374aSIngo Molnar uprobe = __find_uprobe(inode, offset); 6306f47caa0SOleg Nesterov spin_unlock(&uprobes_treelock); 631a5f4374aSIngo Molnar 632a5f4374aSIngo Molnar return uprobe; 633a5f4374aSIngo Molnar } 634a5f4374aSIngo Molnar 635a5f4374aSIngo Molnar static struct uprobe *__insert_uprobe(struct uprobe *uprobe) 636a5f4374aSIngo Molnar { 637a5f4374aSIngo Molnar struct rb_node **p = &uprobes_tree.rb_node; 638a5f4374aSIngo Molnar struct rb_node *parent = NULL; 639a5f4374aSIngo Molnar struct uprobe *u; 640a5f4374aSIngo Molnar int match; 641a5f4374aSIngo Molnar 642a5f4374aSIngo Molnar while (*p) { 643a5f4374aSIngo Molnar parent = *p; 644a5f4374aSIngo Molnar u = rb_entry(parent, struct uprobe, rb_node); 645a5f4374aSIngo Molnar match = match_uprobe(uprobe, u); 646f231722aSOleg Nesterov if (!match) 647f231722aSOleg Nesterov return get_uprobe(u); 648a5f4374aSIngo Molnar 649a5f4374aSIngo Molnar if (match < 0) 650a5f4374aSIngo Molnar p = &parent->rb_left; 651a5f4374aSIngo Molnar else 652a5f4374aSIngo Molnar p = &parent->rb_right; 653a5f4374aSIngo Molnar 654a5f4374aSIngo Molnar } 655a5f4374aSIngo Molnar 656a5f4374aSIngo Molnar u = NULL; 657a5f4374aSIngo Molnar rb_link_node(&uprobe->rb_node, parent, p); 658a5f4374aSIngo Molnar rb_insert_color(&uprobe->rb_node, &uprobes_tree); 659a5f4374aSIngo Molnar /* get access + creation ref */ 660a5f4374aSIngo Molnar atomic_set(&uprobe->ref, 2); 661a5f4374aSIngo Molnar 662a5f4374aSIngo Molnar return u; 663a5f4374aSIngo Molnar } 664a5f4374aSIngo Molnar 665a5f4374aSIngo Molnar /* 666a5f4374aSIngo Molnar * Acquire uprobes_treelock. 667a5f4374aSIngo Molnar * Matching uprobe already exists in rbtree; 668a5f4374aSIngo Molnar * increment (access refcount) and return the matching uprobe. 669a5f4374aSIngo Molnar * 670a5f4374aSIngo Molnar * No matching uprobe; insert the uprobe in rb_tree; 671a5f4374aSIngo Molnar * get a double refcount (access + creation) and return NULL. 672a5f4374aSIngo Molnar */ 673a5f4374aSIngo Molnar static struct uprobe *insert_uprobe(struct uprobe *uprobe) 674a5f4374aSIngo Molnar { 675a5f4374aSIngo Molnar struct uprobe *u; 676a5f4374aSIngo Molnar 6776f47caa0SOleg Nesterov spin_lock(&uprobes_treelock); 678a5f4374aSIngo Molnar u = __insert_uprobe(uprobe); 6796f47caa0SOleg Nesterov spin_unlock(&uprobes_treelock); 680a5f4374aSIngo Molnar 681a5f4374aSIngo Molnar return u; 682a5f4374aSIngo Molnar } 683a5f4374aSIngo Molnar 68422bad382SRavi Bangoria static void 68522bad382SRavi Bangoria ref_ctr_mismatch_warn(struct uprobe *cur_uprobe, struct uprobe *uprobe) 68622bad382SRavi Bangoria { 68722bad382SRavi Bangoria pr_warn("ref_ctr_offset mismatch. inode: 0x%lx offset: 0x%llx " 68822bad382SRavi Bangoria "ref_ctr_offset(old): 0x%llx ref_ctr_offset(new): 0x%llx\n", 68922bad382SRavi Bangoria uprobe->inode->i_ino, (unsigned long long) uprobe->offset, 69022bad382SRavi Bangoria (unsigned long long) cur_uprobe->ref_ctr_offset, 69122bad382SRavi Bangoria (unsigned long long) uprobe->ref_ctr_offset); 69222bad382SRavi Bangoria } 69322bad382SRavi Bangoria 6941cc33161SRavi Bangoria static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset, 6951cc33161SRavi Bangoria loff_t ref_ctr_offset) 696a5f4374aSIngo Molnar { 697a5f4374aSIngo Molnar struct uprobe *uprobe, *cur_uprobe; 698a5f4374aSIngo Molnar 699a5f4374aSIngo Molnar uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL); 700a5f4374aSIngo Molnar if (!uprobe) 701a5f4374aSIngo Molnar return NULL; 702a5f4374aSIngo Molnar 70361f94203SSong Liu uprobe->inode = inode; 704a5f4374aSIngo Molnar uprobe->offset = offset; 7051cc33161SRavi Bangoria uprobe->ref_ctr_offset = ref_ctr_offset; 706e591c8d7SOleg Nesterov init_rwsem(&uprobe->register_rwsem); 707a5f4374aSIngo Molnar init_rwsem(&uprobe->consumer_rwsem); 708a5f4374aSIngo Molnar 709a5f4374aSIngo Molnar /* add to uprobes_tree, sorted on inode:offset */ 710a5f4374aSIngo Molnar cur_uprobe = insert_uprobe(uprobe); 711a5f4374aSIngo Molnar /* a uprobe exists for this inode:offset combination */ 712a5f4374aSIngo Molnar if (cur_uprobe) { 71322bad382SRavi Bangoria if (cur_uprobe->ref_ctr_offset != uprobe->ref_ctr_offset) { 71422bad382SRavi Bangoria ref_ctr_mismatch_warn(cur_uprobe, uprobe); 71522bad382SRavi Bangoria put_uprobe(cur_uprobe); 71622bad382SRavi Bangoria kfree(uprobe); 71722bad382SRavi Bangoria return ERR_PTR(-EINVAL); 71822bad382SRavi Bangoria } 719a5f4374aSIngo Molnar kfree(uprobe); 720a5f4374aSIngo Molnar uprobe = cur_uprobe; 721a5f4374aSIngo Molnar } 722a5f4374aSIngo Molnar 723a5f4374aSIngo Molnar return uprobe; 724a5f4374aSIngo Molnar } 725a5f4374aSIngo Molnar 7269a98e03cSOleg Nesterov static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc) 727a5f4374aSIngo Molnar { 728a5f4374aSIngo Molnar down_write(&uprobe->consumer_rwsem); 729e3343e6aSSrikar Dronamraju uc->next = uprobe->consumers; 730e3343e6aSSrikar Dronamraju uprobe->consumers = uc; 731a5f4374aSIngo Molnar up_write(&uprobe->consumer_rwsem); 732a5f4374aSIngo Molnar } 733a5f4374aSIngo Molnar 734a5f4374aSIngo Molnar /* 735e3343e6aSSrikar Dronamraju * For uprobe @uprobe, delete the consumer @uc. 736e3343e6aSSrikar Dronamraju * Return true if the @uc is deleted successfully 737a5f4374aSIngo Molnar * or return false. 738a5f4374aSIngo Molnar */ 739e3343e6aSSrikar Dronamraju static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc) 740a5f4374aSIngo Molnar { 741a5f4374aSIngo Molnar struct uprobe_consumer **con; 742a5f4374aSIngo Molnar bool ret = false; 743a5f4374aSIngo Molnar 744a5f4374aSIngo Molnar down_write(&uprobe->consumer_rwsem); 745a5f4374aSIngo Molnar for (con = &uprobe->consumers; *con; con = &(*con)->next) { 746e3343e6aSSrikar Dronamraju if (*con == uc) { 747e3343e6aSSrikar Dronamraju *con = uc->next; 748a5f4374aSIngo Molnar ret = true; 749a5f4374aSIngo Molnar break; 750a5f4374aSIngo Molnar } 751a5f4374aSIngo Molnar } 752a5f4374aSIngo Molnar up_write(&uprobe->consumer_rwsem); 753a5f4374aSIngo Molnar 754a5f4374aSIngo Molnar return ret; 755a5f4374aSIngo Molnar } 756a5f4374aSIngo Molnar 7572ded0980SOleg Nesterov static int __copy_insn(struct address_space *mapping, struct file *filp, 7582ded0980SOleg Nesterov void *insn, int nbytes, loff_t offset) 759a5f4374aSIngo Molnar { 760a5f4374aSIngo Molnar struct page *page; 761a5f4374aSIngo Molnar /* 76240814f68SOleg Nesterov * Ensure that the page that has the original instruction is populated 76340814f68SOleg Nesterov * and in page-cache. If ->readpage == NULL it must be shmem_mapping(), 76440814f68SOleg Nesterov * see uprobe_register(). 765a5f4374aSIngo Molnar */ 76640814f68SOleg Nesterov if (mapping->a_ops->readpage) 76709cbfeafSKirill A. Shutemov page = read_mapping_page(mapping, offset >> PAGE_SHIFT, filp); 76840814f68SOleg Nesterov else 76909cbfeafSKirill A. Shutemov page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT); 770a5f4374aSIngo Molnar if (IS_ERR(page)) 771a5f4374aSIngo Molnar return PTR_ERR(page); 772a5f4374aSIngo Molnar 7732edb7b55SOleg Nesterov copy_from_page(page, offset, insn, nbytes); 77409cbfeafSKirill A. Shutemov put_page(page); 775a5f4374aSIngo Molnar 776a5f4374aSIngo Molnar return 0; 777a5f4374aSIngo Molnar } 778a5f4374aSIngo Molnar 779d436615eSOleg Nesterov static int copy_insn(struct uprobe *uprobe, struct file *filp) 780a5f4374aSIngo Molnar { 7812ded0980SOleg Nesterov struct address_space *mapping = uprobe->inode->i_mapping; 7822ded0980SOleg Nesterov loff_t offs = uprobe->offset; 783803200e2SOleg Nesterov void *insn = &uprobe->arch.insn; 784803200e2SOleg Nesterov int size = sizeof(uprobe->arch.insn); 7852ded0980SOleg Nesterov int len, err = -EIO; 786a5f4374aSIngo Molnar 7872ded0980SOleg Nesterov /* Copy only available bytes, -EIO if nothing was read */ 7882ded0980SOleg Nesterov do { 7892ded0980SOleg Nesterov if (offs >= i_size_read(uprobe->inode)) 7902ded0980SOleg Nesterov break; 791a5f4374aSIngo Molnar 7922ded0980SOleg Nesterov len = min_t(int, size, PAGE_SIZE - (offs & ~PAGE_MASK)); 7932ded0980SOleg Nesterov err = __copy_insn(mapping, filp, insn, len, offs); 794fc36f595SOleg Nesterov if (err) 7952ded0980SOleg Nesterov break; 7962ded0980SOleg Nesterov 7972ded0980SOleg Nesterov insn += len; 7982ded0980SOleg Nesterov offs += len; 7992ded0980SOleg Nesterov size -= len; 8002ded0980SOleg Nesterov } while (size); 8012ded0980SOleg Nesterov 802fc36f595SOleg Nesterov return err; 803a5f4374aSIngo Molnar } 804a5f4374aSIngo Molnar 805cb9a19feSOleg Nesterov static int prepare_uprobe(struct uprobe *uprobe, struct file *file, 806cb9a19feSOleg Nesterov struct mm_struct *mm, unsigned long vaddr) 807cb9a19feSOleg Nesterov { 808cb9a19feSOleg Nesterov int ret = 0; 809cb9a19feSOleg Nesterov 81071434f2fSOleg Nesterov if (test_bit(UPROBE_COPY_INSN, &uprobe->flags)) 811cb9a19feSOleg Nesterov return ret; 812cb9a19feSOleg Nesterov 813d4d3ccc6SOleg Nesterov /* TODO: move this into _register, until then we abuse this sem. */ 814d4d3ccc6SOleg Nesterov down_write(&uprobe->consumer_rwsem); 81571434f2fSOleg Nesterov if (test_bit(UPROBE_COPY_INSN, &uprobe->flags)) 8164710f05fSOleg Nesterov goto out; 8174710f05fSOleg Nesterov 818cb9a19feSOleg Nesterov ret = copy_insn(uprobe, file); 819cb9a19feSOleg Nesterov if (ret) 820cb9a19feSOleg Nesterov goto out; 821cb9a19feSOleg Nesterov 822cb9a19feSOleg Nesterov ret = -ENOTSUPP; 823803200e2SOleg Nesterov if (is_trap_insn((uprobe_opcode_t *)&uprobe->arch.insn)) 824cb9a19feSOleg Nesterov goto out; 825cb9a19feSOleg Nesterov 826cb9a19feSOleg Nesterov ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr); 827cb9a19feSOleg Nesterov if (ret) 828cb9a19feSOleg Nesterov goto out; 829cb9a19feSOleg Nesterov 830f72d41faSOleg Nesterov /* uprobe_write_opcode() assumes we don't cross page boundary */ 831cb9a19feSOleg Nesterov BUG_ON((uprobe->offset & ~PAGE_MASK) + 832cb9a19feSOleg Nesterov UPROBE_SWBP_INSN_SIZE > PAGE_SIZE); 833cb9a19feSOleg Nesterov 834cb9a19feSOleg Nesterov smp_wmb(); /* pairs with rmb() in find_active_uprobe() */ 83571434f2fSOleg Nesterov set_bit(UPROBE_COPY_INSN, &uprobe->flags); 836cb9a19feSOleg Nesterov 837cb9a19feSOleg Nesterov out: 838d4d3ccc6SOleg Nesterov up_write(&uprobe->consumer_rwsem); 8394710f05fSOleg Nesterov 840cb9a19feSOleg Nesterov return ret; 841cb9a19feSOleg Nesterov } 842cb9a19feSOleg Nesterov 8438a7f2fa0SOleg Nesterov static inline bool consumer_filter(struct uprobe_consumer *uc, 8448a7f2fa0SOleg Nesterov enum uprobe_filter_ctx ctx, struct mm_struct *mm) 845806a98bdSOleg Nesterov { 8468a7f2fa0SOleg Nesterov return !uc->filter || uc->filter(uc, ctx, mm); 847806a98bdSOleg Nesterov } 848806a98bdSOleg Nesterov 8498a7f2fa0SOleg Nesterov static bool filter_chain(struct uprobe *uprobe, 8508a7f2fa0SOleg Nesterov enum uprobe_filter_ctx ctx, struct mm_struct *mm) 85163633cbfSOleg Nesterov { 8521ff6fee5SOleg Nesterov struct uprobe_consumer *uc; 8531ff6fee5SOleg Nesterov bool ret = false; 8541ff6fee5SOleg Nesterov 8551ff6fee5SOleg Nesterov down_read(&uprobe->consumer_rwsem); 8561ff6fee5SOleg Nesterov for (uc = uprobe->consumers; uc; uc = uc->next) { 8578a7f2fa0SOleg Nesterov ret = consumer_filter(uc, ctx, mm); 8581ff6fee5SOleg Nesterov if (ret) 8591ff6fee5SOleg Nesterov break; 8601ff6fee5SOleg Nesterov } 8611ff6fee5SOleg Nesterov up_read(&uprobe->consumer_rwsem); 8621ff6fee5SOleg Nesterov 8631ff6fee5SOleg Nesterov return ret; 86463633cbfSOleg Nesterov } 86563633cbfSOleg Nesterov 866e3343e6aSSrikar Dronamraju static int 867e3343e6aSSrikar Dronamraju install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, 868816c03fbSOleg Nesterov struct vm_area_struct *vma, unsigned long vaddr) 869a5f4374aSIngo Molnar { 870f8ac4ec9SOleg Nesterov bool first_uprobe; 871a5f4374aSIngo Molnar int ret; 872a5f4374aSIngo Molnar 873cb9a19feSOleg Nesterov ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr); 874a5f4374aSIngo Molnar if (ret) 875a5f4374aSIngo Molnar return ret; 876a5f4374aSIngo Molnar 877f8ac4ec9SOleg Nesterov /* 878f8ac4ec9SOleg Nesterov * set MMF_HAS_UPROBES in advance for uprobe_pre_sstep_notifier(), 879f8ac4ec9SOleg Nesterov * the task can hit this breakpoint right after __replace_page(). 880f8ac4ec9SOleg Nesterov */ 881f8ac4ec9SOleg Nesterov first_uprobe = !test_bit(MMF_HAS_UPROBES, &mm->flags); 882f8ac4ec9SOleg Nesterov if (first_uprobe) 883f8ac4ec9SOleg Nesterov set_bit(MMF_HAS_UPROBES, &mm->flags); 884f8ac4ec9SOleg Nesterov 885816c03fbSOleg Nesterov ret = set_swbp(&uprobe->arch, mm, vaddr); 8869f68f672SOleg Nesterov if (!ret) 8879f68f672SOleg Nesterov clear_bit(MMF_RECALC_UPROBES, &mm->flags); 8889f68f672SOleg Nesterov else if (first_uprobe) 889f8ac4ec9SOleg Nesterov clear_bit(MMF_HAS_UPROBES, &mm->flags); 890a5f4374aSIngo Molnar 891a5f4374aSIngo Molnar return ret; 892a5f4374aSIngo Molnar } 893a5f4374aSIngo Molnar 894076a365bSOleg Nesterov static int 895816c03fbSOleg Nesterov remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr) 896a5f4374aSIngo Molnar { 8979f68f672SOleg Nesterov set_bit(MMF_RECALC_UPROBES, &mm->flags); 898076a365bSOleg Nesterov return set_orig_insn(&uprobe->arch, mm, vaddr); 899a5f4374aSIngo Molnar } 900a5f4374aSIngo Molnar 90106b7bcd8SOleg Nesterov static inline bool uprobe_is_active(struct uprobe *uprobe) 90206b7bcd8SOleg Nesterov { 90306b7bcd8SOleg Nesterov return !RB_EMPTY_NODE(&uprobe->rb_node); 90406b7bcd8SOleg Nesterov } 9050326f5a9SSrikar Dronamraju /* 906778b032dSOleg Nesterov * There could be threads that have already hit the breakpoint. They 907778b032dSOleg Nesterov * will recheck the current insn and restart if find_uprobe() fails. 908778b032dSOleg Nesterov * See find_active_uprobe(). 9090326f5a9SSrikar Dronamraju */ 910a5f4374aSIngo Molnar static void delete_uprobe(struct uprobe *uprobe) 911a5f4374aSIngo Molnar { 91206b7bcd8SOleg Nesterov if (WARN_ON(!uprobe_is_active(uprobe))) 91306b7bcd8SOleg Nesterov return; 91406b7bcd8SOleg Nesterov 9156f47caa0SOleg Nesterov spin_lock(&uprobes_treelock); 916a5f4374aSIngo Molnar rb_erase(&uprobe->rb_node, &uprobes_tree); 9176f47caa0SOleg Nesterov spin_unlock(&uprobes_treelock); 91806b7bcd8SOleg Nesterov RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */ 919a5f4374aSIngo Molnar put_uprobe(uprobe); 920a5f4374aSIngo Molnar } 921a5f4374aSIngo Molnar 92226872090SOleg Nesterov struct map_info { 92326872090SOleg Nesterov struct map_info *next; 92426872090SOleg Nesterov struct mm_struct *mm; 925816c03fbSOleg Nesterov unsigned long vaddr; 92626872090SOleg Nesterov }; 92726872090SOleg Nesterov 92826872090SOleg Nesterov static inline struct map_info *free_map_info(struct map_info *info) 929a5f4374aSIngo Molnar { 93026872090SOleg Nesterov struct map_info *next = info->next; 93126872090SOleg Nesterov kfree(info); 93226872090SOleg Nesterov return next; 93326872090SOleg Nesterov } 93426872090SOleg Nesterov 93526872090SOleg Nesterov static struct map_info * 93626872090SOleg Nesterov build_map_info(struct address_space *mapping, loff_t offset, bool is_register) 93726872090SOleg Nesterov { 93826872090SOleg Nesterov unsigned long pgoff = offset >> PAGE_SHIFT; 939a5f4374aSIngo Molnar struct vm_area_struct *vma; 94026872090SOleg Nesterov struct map_info *curr = NULL; 94126872090SOleg Nesterov struct map_info *prev = NULL; 94226872090SOleg Nesterov struct map_info *info; 94326872090SOleg Nesterov int more = 0; 944a5f4374aSIngo Molnar 94526872090SOleg Nesterov again: 9464a23717aSDavidlohr Bueso i_mmap_lock_read(mapping); 9476b2dbba8SMichel Lespinasse vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { 948a5f4374aSIngo Molnar if (!valid_vma(vma, is_register)) 949a5f4374aSIngo Molnar continue; 950a5f4374aSIngo Molnar 9517a5bfb66SOleg Nesterov if (!prev && !more) { 9527a5bfb66SOleg Nesterov /* 953c8c06efaSDavidlohr Bueso * Needs GFP_NOWAIT to avoid i_mmap_rwsem recursion through 9547a5bfb66SOleg Nesterov * reclaim. This is optimistic, no harm done if it fails. 9557a5bfb66SOleg Nesterov */ 9567a5bfb66SOleg Nesterov prev = kmalloc(sizeof(struct map_info), 9577a5bfb66SOleg Nesterov GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_NOWARN); 9587a5bfb66SOleg Nesterov if (prev) 9597a5bfb66SOleg Nesterov prev->next = NULL; 9607a5bfb66SOleg Nesterov } 96126872090SOleg Nesterov if (!prev) { 96226872090SOleg Nesterov more++; 96326872090SOleg Nesterov continue; 964a5f4374aSIngo Molnar } 965a5f4374aSIngo Molnar 966388f7934SVegard Nossum if (!mmget_not_zero(vma->vm_mm)) 96726872090SOleg Nesterov continue; 968a5f4374aSIngo Molnar 96926872090SOleg Nesterov info = prev; 97026872090SOleg Nesterov prev = prev->next; 97126872090SOleg Nesterov info->next = curr; 97226872090SOleg Nesterov curr = info; 97326872090SOleg Nesterov 97426872090SOleg Nesterov info->mm = vma->vm_mm; 97557683f72SOleg Nesterov info->vaddr = offset_to_vaddr(vma, offset); 976a5f4374aSIngo Molnar } 9774a23717aSDavidlohr Bueso i_mmap_unlock_read(mapping); 978a5f4374aSIngo Molnar 97926872090SOleg Nesterov if (!more) 98026872090SOleg Nesterov goto out; 981a5f4374aSIngo Molnar 98226872090SOleg Nesterov prev = curr; 98326872090SOleg Nesterov while (curr) { 98426872090SOleg Nesterov mmput(curr->mm); 98526872090SOleg Nesterov curr = curr->next; 98626872090SOleg Nesterov } 98726872090SOleg Nesterov 98826872090SOleg Nesterov do { 98926872090SOleg Nesterov info = kmalloc(sizeof(struct map_info), GFP_KERNEL); 99026872090SOleg Nesterov if (!info) { 99126872090SOleg Nesterov curr = ERR_PTR(-ENOMEM); 99226872090SOleg Nesterov goto out; 99326872090SOleg Nesterov } 99426872090SOleg Nesterov info->next = prev; 99526872090SOleg Nesterov prev = info; 99626872090SOleg Nesterov } while (--more); 99726872090SOleg Nesterov 99826872090SOleg Nesterov goto again; 99926872090SOleg Nesterov out: 100026872090SOleg Nesterov while (prev) 100126872090SOleg Nesterov prev = free_map_info(prev); 100226872090SOleg Nesterov return curr; 1003a5f4374aSIngo Molnar } 1004a5f4374aSIngo Molnar 1005bdf8647cSOleg Nesterov static int 1006bdf8647cSOleg Nesterov register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new) 1007a5f4374aSIngo Molnar { 1008bdf8647cSOleg Nesterov bool is_register = !!new; 100926872090SOleg Nesterov struct map_info *info; 101026872090SOleg Nesterov int err = 0; 101126872090SOleg Nesterov 101232cdba1eSOleg Nesterov percpu_down_write(&dup_mmap_sem); 101326872090SOleg Nesterov info = build_map_info(uprobe->inode->i_mapping, 101426872090SOleg Nesterov uprobe->offset, is_register); 101532cdba1eSOleg Nesterov if (IS_ERR(info)) { 101632cdba1eSOleg Nesterov err = PTR_ERR(info); 101732cdba1eSOleg Nesterov goto out; 101832cdba1eSOleg Nesterov } 101926872090SOleg Nesterov 102026872090SOleg Nesterov while (info) { 102126872090SOleg Nesterov struct mm_struct *mm = info->mm; 1022a5f4374aSIngo Molnar struct vm_area_struct *vma; 1023a5f4374aSIngo Molnar 1024076a365bSOleg Nesterov if (err && is_register) 102526872090SOleg Nesterov goto free; 1026a5f4374aSIngo Molnar 102777fc4af1SOleg Nesterov down_write(&mm->mmap_sem); 1028f4d6dfe5SOleg Nesterov vma = find_vma(mm, info->vaddr); 1029f4d6dfe5SOleg Nesterov if (!vma || !valid_vma(vma, is_register) || 1030f281769eSOleg Nesterov file_inode(vma->vm_file) != uprobe->inode) 103126872090SOleg Nesterov goto unlock; 103226872090SOleg Nesterov 1033f4d6dfe5SOleg Nesterov if (vma->vm_start > info->vaddr || 1034f4d6dfe5SOleg Nesterov vaddr_to_offset(vma, info->vaddr) != uprobe->offset) 103526872090SOleg Nesterov goto unlock; 1036a5f4374aSIngo Molnar 1037806a98bdSOleg Nesterov if (is_register) { 1038806a98bdSOleg Nesterov /* consult only the "caller", new consumer. */ 1039bdf8647cSOleg Nesterov if (consumer_filter(new, 10408a7f2fa0SOleg Nesterov UPROBE_FILTER_REGISTER, mm)) 104126872090SOleg Nesterov err = install_breakpoint(uprobe, mm, vma, info->vaddr); 1042806a98bdSOleg Nesterov } else if (test_bit(MMF_HAS_UPROBES, &mm->flags)) { 10438a7f2fa0SOleg Nesterov if (!filter_chain(uprobe, 10448a7f2fa0SOleg Nesterov UPROBE_FILTER_UNREGISTER, mm)) 1045076a365bSOleg Nesterov err |= remove_breakpoint(uprobe, mm, info->vaddr); 1046806a98bdSOleg Nesterov } 104778f74116SOleg Nesterov 104826872090SOleg Nesterov unlock: 104926872090SOleg Nesterov up_write(&mm->mmap_sem); 105026872090SOleg Nesterov free: 105126872090SOleg Nesterov mmput(mm); 105226872090SOleg Nesterov info = free_map_info(info); 1053a5f4374aSIngo Molnar } 105432cdba1eSOleg Nesterov out: 105532cdba1eSOleg Nesterov percpu_up_write(&dup_mmap_sem); 105626872090SOleg Nesterov return err; 1057a5f4374aSIngo Molnar } 1058a5f4374aSIngo Molnar 105938e967aeSRavi Bangoria static void 106038e967aeSRavi Bangoria __uprobe_unregister(struct uprobe *uprobe, struct uprobe_consumer *uc) 1061a5f4374aSIngo Molnar { 106204aab9b2SOleg Nesterov int err; 1063a5f4374aSIngo Molnar 106406d07139SOleg Nesterov if (WARN_ON(!consumer_del(uprobe, uc))) 106504aab9b2SOleg Nesterov return; 106604aab9b2SOleg Nesterov 1067bdf8647cSOleg Nesterov err = register_for_each_vma(uprobe, NULL); 1068a5f4374aSIngo Molnar /* TODO : cant unregister? schedule a worker thread */ 1069bb929284SOleg Nesterov if (!uprobe->consumers && !err) 107004aab9b2SOleg Nesterov delete_uprobe(uprobe); 107104aab9b2SOleg Nesterov } 1072a5f4374aSIngo Molnar 1073a5f4374aSIngo Molnar /* 10747140ad38SLinus Torvalds * uprobe_unregister - unregister an already registered probe. 107538e967aeSRavi Bangoria * @inode: the file in which the probe has to be removed. 107638e967aeSRavi Bangoria * @offset: offset from the start of the file. 107738e967aeSRavi Bangoria * @uc: identify which probe if multiple probes are colocated. 107838e967aeSRavi Bangoria */ 107938e967aeSRavi Bangoria void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc) 108038e967aeSRavi Bangoria { 108138e967aeSRavi Bangoria struct uprobe *uprobe; 108238e967aeSRavi Bangoria 108338e967aeSRavi Bangoria uprobe = find_uprobe(inode, offset); 108438e967aeSRavi Bangoria if (WARN_ON(!uprobe)) 108538e967aeSRavi Bangoria return; 108638e967aeSRavi Bangoria 108738e967aeSRavi Bangoria down_write(&uprobe->register_rwsem); 108838e967aeSRavi Bangoria __uprobe_unregister(uprobe, uc); 108938e967aeSRavi Bangoria up_write(&uprobe->register_rwsem); 109038e967aeSRavi Bangoria put_uprobe(uprobe); 109138e967aeSRavi Bangoria } 109238e967aeSRavi Bangoria EXPORT_SYMBOL_GPL(uprobe_unregister); 109338e967aeSRavi Bangoria 109438e967aeSRavi Bangoria /* 109538e967aeSRavi Bangoria * __uprobe_register - register a probe 1096a5f4374aSIngo Molnar * @inode: the file in which the probe has to be placed. 1097a5f4374aSIngo Molnar * @offset: offset from the start of the file. 1098e3343e6aSSrikar Dronamraju * @uc: information on howto handle the probe.. 1099a5f4374aSIngo Molnar * 110038e967aeSRavi Bangoria * Apart from the access refcount, __uprobe_register() takes a creation 1101a5f4374aSIngo Molnar * refcount (thro alloc_uprobe) if and only if this @uprobe is getting 1102a5f4374aSIngo Molnar * inserted into the rbtree (i.e first consumer for a @inode:@offset 1103a5f4374aSIngo Molnar * tuple). Creation refcount stops uprobe_unregister from freeing the 1104a5f4374aSIngo Molnar * @uprobe even before the register operation is complete. Creation 1105e3343e6aSSrikar Dronamraju * refcount is released when the last @uc for the @uprobe 110638e967aeSRavi Bangoria * unregisters. Caller of __uprobe_register() is required to keep @inode 110761f94203SSong Liu * (and the containing mount) referenced. 1108a5f4374aSIngo Molnar * 1109a5f4374aSIngo Molnar * Return errno if it cannot successully install probes 1110a5f4374aSIngo Molnar * else return 0 (success) 1111a5f4374aSIngo Molnar */ 111238e967aeSRavi Bangoria static int __uprobe_register(struct inode *inode, loff_t offset, 11131cc33161SRavi Bangoria loff_t ref_ctr_offset, struct uprobe_consumer *uc) 1114a5f4374aSIngo Molnar { 1115a5f4374aSIngo Molnar struct uprobe *uprobe; 1116a5f4374aSIngo Molnar int ret; 1117a5f4374aSIngo Molnar 1118ea024870SAnton Arapov /* Uprobe must have at least one set consumer */ 1119ea024870SAnton Arapov if (!uc->handler && !uc->ret_handler) 1120ea024870SAnton Arapov return -EINVAL; 1121ea024870SAnton Arapov 112240814f68SOleg Nesterov /* copy_insn() uses read_mapping_page() or shmem_read_mapping_page() */ 112340814f68SOleg Nesterov if (!inode->i_mapping->a_ops->readpage && !shmem_mapping(inode->i_mapping)) 112441ccba02SOleg Nesterov return -EIO; 1125f0744af7SOleg Nesterov /* Racy, just to catch the obvious mistakes */ 1126a5f4374aSIngo Molnar if (offset > i_size_read(inode)) 1127a5f4374aSIngo Molnar return -EINVAL; 1128a5f4374aSIngo Molnar 112966d06dffSOleg Nesterov retry: 11301cc33161SRavi Bangoria uprobe = alloc_uprobe(inode, offset, ref_ctr_offset); 113166d06dffSOleg Nesterov if (!uprobe) 113266d06dffSOleg Nesterov return -ENOMEM; 113322bad382SRavi Bangoria if (IS_ERR(uprobe)) 113422bad382SRavi Bangoria return PTR_ERR(uprobe); 113522bad382SRavi Bangoria 113666d06dffSOleg Nesterov /* 113766d06dffSOleg Nesterov * We can race with uprobe_unregister()->delete_uprobe(). 113866d06dffSOleg Nesterov * Check uprobe_is_active() and retry if it is false. 113966d06dffSOleg Nesterov */ 1140e591c8d7SOleg Nesterov down_write(&uprobe->register_rwsem); 114166d06dffSOleg Nesterov ret = -EAGAIN; 114266d06dffSOleg Nesterov if (likely(uprobe_is_active(uprobe))) { 114338e967aeSRavi Bangoria consumer_add(uprobe, uc); 114438e967aeSRavi Bangoria ret = register_for_each_vma(uprobe, uc); 11459a98e03cSOleg Nesterov if (ret) 114604aab9b2SOleg Nesterov __uprobe_unregister(uprobe, uc); 1147a5f4374aSIngo Molnar } 114866d06dffSOleg Nesterov up_write(&uprobe->register_rwsem); 1149a5f4374aSIngo Molnar put_uprobe(uprobe); 1150a5f4374aSIngo Molnar 115166d06dffSOleg Nesterov if (unlikely(ret == -EAGAIN)) 115266d06dffSOleg Nesterov goto retry; 1153a5f4374aSIngo Molnar return ret; 1154a5f4374aSIngo Molnar } 115538e967aeSRavi Bangoria 115638e967aeSRavi Bangoria int uprobe_register(struct inode *inode, loff_t offset, 115738e967aeSRavi Bangoria struct uprobe_consumer *uc) 115838e967aeSRavi Bangoria { 11591cc33161SRavi Bangoria return __uprobe_register(inode, offset, 0, uc); 116038e967aeSRavi Bangoria } 1161e8440c14SJosh Stone EXPORT_SYMBOL_GPL(uprobe_register); 1162a5f4374aSIngo Molnar 11631cc33161SRavi Bangoria int uprobe_register_refctr(struct inode *inode, loff_t offset, 11641cc33161SRavi Bangoria loff_t ref_ctr_offset, struct uprobe_consumer *uc) 11651cc33161SRavi Bangoria { 11661cc33161SRavi Bangoria return __uprobe_register(inode, offset, ref_ctr_offset, uc); 11671cc33161SRavi Bangoria } 11681cc33161SRavi Bangoria EXPORT_SYMBOL_GPL(uprobe_register_refctr); 11691cc33161SRavi Bangoria 1170a5f4374aSIngo Molnar /* 1171788faab7STobias Tefke * uprobe_apply - unregister an already registered probe. 1172bdf8647cSOleg Nesterov * @inode: the file in which the probe has to be removed. 1173bdf8647cSOleg Nesterov * @offset: offset from the start of the file. 1174bdf8647cSOleg Nesterov * @uc: consumer which wants to add more or remove some breakpoints 1175bdf8647cSOleg Nesterov * @add: add or remove the breakpoints 1176bdf8647cSOleg Nesterov */ 1177bdf8647cSOleg Nesterov int uprobe_apply(struct inode *inode, loff_t offset, 1178bdf8647cSOleg Nesterov struct uprobe_consumer *uc, bool add) 1179bdf8647cSOleg Nesterov { 1180bdf8647cSOleg Nesterov struct uprobe *uprobe; 1181bdf8647cSOleg Nesterov struct uprobe_consumer *con; 1182bdf8647cSOleg Nesterov int ret = -ENOENT; 1183bdf8647cSOleg Nesterov 1184bdf8647cSOleg Nesterov uprobe = find_uprobe(inode, offset); 118506d07139SOleg Nesterov if (WARN_ON(!uprobe)) 1186bdf8647cSOleg Nesterov return ret; 1187bdf8647cSOleg Nesterov 1188bdf8647cSOleg Nesterov down_write(&uprobe->register_rwsem); 1189bdf8647cSOleg Nesterov for (con = uprobe->consumers; con && con != uc ; con = con->next) 1190bdf8647cSOleg Nesterov ; 1191bdf8647cSOleg Nesterov if (con) 1192bdf8647cSOleg Nesterov ret = register_for_each_vma(uprobe, add ? uc : NULL); 1193bdf8647cSOleg Nesterov up_write(&uprobe->register_rwsem); 1194bdf8647cSOleg Nesterov put_uprobe(uprobe); 1195bdf8647cSOleg Nesterov 1196bdf8647cSOleg Nesterov return ret; 1197bdf8647cSOleg Nesterov } 1198bdf8647cSOleg Nesterov 1199da1816b1SOleg Nesterov static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) 1200da1816b1SOleg Nesterov { 1201da1816b1SOleg Nesterov struct vm_area_struct *vma; 1202da1816b1SOleg Nesterov int err = 0; 1203da1816b1SOleg Nesterov 1204da1816b1SOleg Nesterov down_read(&mm->mmap_sem); 1205da1816b1SOleg Nesterov for (vma = mm->mmap; vma; vma = vma->vm_next) { 1206da1816b1SOleg Nesterov unsigned long vaddr; 1207da1816b1SOleg Nesterov loff_t offset; 1208da1816b1SOleg Nesterov 1209da1816b1SOleg Nesterov if (!valid_vma(vma, false) || 1210f281769eSOleg Nesterov file_inode(vma->vm_file) != uprobe->inode) 1211da1816b1SOleg Nesterov continue; 1212da1816b1SOleg Nesterov 1213da1816b1SOleg Nesterov offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT; 1214da1816b1SOleg Nesterov if (uprobe->offset < offset || 1215da1816b1SOleg Nesterov uprobe->offset >= offset + vma->vm_end - vma->vm_start) 1216da1816b1SOleg Nesterov continue; 1217da1816b1SOleg Nesterov 1218da1816b1SOleg Nesterov vaddr = offset_to_vaddr(vma, uprobe->offset); 1219da1816b1SOleg Nesterov err |= remove_breakpoint(uprobe, mm, vaddr); 1220da1816b1SOleg Nesterov } 1221da1816b1SOleg Nesterov up_read(&mm->mmap_sem); 1222da1816b1SOleg Nesterov 1223da1816b1SOleg Nesterov return err; 1224da1816b1SOleg Nesterov } 1225da1816b1SOleg Nesterov 1226891c3970SOleg Nesterov static struct rb_node * 1227891c3970SOleg Nesterov find_node_in_range(struct inode *inode, loff_t min, loff_t max) 1228a5f4374aSIngo Molnar { 1229a5f4374aSIngo Molnar struct rb_node *n = uprobes_tree.rb_node; 1230a5f4374aSIngo Molnar 1231a5f4374aSIngo Molnar while (n) { 1232891c3970SOleg Nesterov struct uprobe *u = rb_entry(n, struct uprobe, rb_node); 1233a5f4374aSIngo Molnar 1234891c3970SOleg Nesterov if (inode < u->inode) { 1235a5f4374aSIngo Molnar n = n->rb_left; 1236891c3970SOleg Nesterov } else if (inode > u->inode) { 1237a5f4374aSIngo Molnar n = n->rb_right; 1238891c3970SOleg Nesterov } else { 1239891c3970SOleg Nesterov if (max < u->offset) 1240891c3970SOleg Nesterov n = n->rb_left; 1241891c3970SOleg Nesterov else if (min > u->offset) 1242891c3970SOleg Nesterov n = n->rb_right; 1243891c3970SOleg Nesterov else 1244891c3970SOleg Nesterov break; 1245891c3970SOleg Nesterov } 1246a5f4374aSIngo Molnar } 1247a5f4374aSIngo Molnar 1248891c3970SOleg Nesterov return n; 1249a5f4374aSIngo Molnar } 1250a5f4374aSIngo Molnar 1251a5f4374aSIngo Molnar /* 1252891c3970SOleg Nesterov * For a given range in vma, build a list of probes that need to be inserted. 1253a5f4374aSIngo Molnar */ 1254891c3970SOleg Nesterov static void build_probe_list(struct inode *inode, 1255891c3970SOleg Nesterov struct vm_area_struct *vma, 1256891c3970SOleg Nesterov unsigned long start, unsigned long end, 1257891c3970SOleg Nesterov struct list_head *head) 1258a5f4374aSIngo Molnar { 1259891c3970SOleg Nesterov loff_t min, max; 1260891c3970SOleg Nesterov struct rb_node *n, *t; 1261891c3970SOleg Nesterov struct uprobe *u; 1262891c3970SOleg Nesterov 1263891c3970SOleg Nesterov INIT_LIST_HEAD(head); 1264cb113b47SOleg Nesterov min = vaddr_to_offset(vma, start); 1265891c3970SOleg Nesterov max = min + (end - start) - 1; 1266a5f4374aSIngo Molnar 12676f47caa0SOleg Nesterov spin_lock(&uprobes_treelock); 1268891c3970SOleg Nesterov n = find_node_in_range(inode, min, max); 1269891c3970SOleg Nesterov if (n) { 1270891c3970SOleg Nesterov for (t = n; t; t = rb_prev(t)) { 1271891c3970SOleg Nesterov u = rb_entry(t, struct uprobe, rb_node); 1272891c3970SOleg Nesterov if (u->inode != inode || u->offset < min) 1273a5f4374aSIngo Molnar break; 1274891c3970SOleg Nesterov list_add(&u->pending_list, head); 1275f231722aSOleg Nesterov get_uprobe(u); 1276a5f4374aSIngo Molnar } 1277891c3970SOleg Nesterov for (t = n; (t = rb_next(t)); ) { 1278891c3970SOleg Nesterov u = rb_entry(t, struct uprobe, rb_node); 1279891c3970SOleg Nesterov if (u->inode != inode || u->offset > max) 1280891c3970SOleg Nesterov break; 1281891c3970SOleg Nesterov list_add(&u->pending_list, head); 1282f231722aSOleg Nesterov get_uprobe(u); 1283891c3970SOleg Nesterov } 1284891c3970SOleg Nesterov } 12856f47caa0SOleg Nesterov spin_unlock(&uprobes_treelock); 1286a5f4374aSIngo Molnar } 1287a5f4374aSIngo Molnar 12881cc33161SRavi Bangoria /* @vma contains reference counter, not the probed instruction. */ 12891cc33161SRavi Bangoria static int delayed_ref_ctr_inc(struct vm_area_struct *vma) 12901cc33161SRavi Bangoria { 12911cc33161SRavi Bangoria struct list_head *pos, *q; 12921cc33161SRavi Bangoria struct delayed_uprobe *du; 12931cc33161SRavi Bangoria unsigned long vaddr; 12941cc33161SRavi Bangoria int ret = 0, err = 0; 12951cc33161SRavi Bangoria 12961cc33161SRavi Bangoria mutex_lock(&delayed_uprobe_lock); 12971cc33161SRavi Bangoria list_for_each_safe(pos, q, &delayed_uprobe_list) { 12981cc33161SRavi Bangoria du = list_entry(pos, struct delayed_uprobe, list); 12991cc33161SRavi Bangoria 13001cc33161SRavi Bangoria if (du->mm != vma->vm_mm || 13011cc33161SRavi Bangoria !valid_ref_ctr_vma(du->uprobe, vma)) 13021cc33161SRavi Bangoria continue; 13031cc33161SRavi Bangoria 13041cc33161SRavi Bangoria vaddr = offset_to_vaddr(vma, du->uprobe->ref_ctr_offset); 13051cc33161SRavi Bangoria ret = __update_ref_ctr(vma->vm_mm, vaddr, 1); 13061cc33161SRavi Bangoria if (ret) { 13071cc33161SRavi Bangoria update_ref_ctr_warn(du->uprobe, vma->vm_mm, 1); 13081cc33161SRavi Bangoria if (!err) 13091cc33161SRavi Bangoria err = ret; 13101cc33161SRavi Bangoria } 13111cc33161SRavi Bangoria delayed_uprobe_delete(du); 13121cc33161SRavi Bangoria } 13131cc33161SRavi Bangoria mutex_unlock(&delayed_uprobe_lock); 13141cc33161SRavi Bangoria return err; 13151cc33161SRavi Bangoria } 13161cc33161SRavi Bangoria 1317a5f4374aSIngo Molnar /* 13185e5be71aSOleg Nesterov * Called from mmap_region/vma_adjust with mm->mmap_sem acquired. 1319a5f4374aSIngo Molnar * 13205e5be71aSOleg Nesterov * Currently we ignore all errors and always return 0, the callers 13215e5be71aSOleg Nesterov * can't handle the failure anyway. 1322a5f4374aSIngo Molnar */ 1323a5f4374aSIngo Molnar int uprobe_mmap(struct vm_area_struct *vma) 1324a5f4374aSIngo Molnar { 1325a5f4374aSIngo Molnar struct list_head tmp_list; 1326665605a2SOleg Nesterov struct uprobe *uprobe, *u; 1327a5f4374aSIngo Molnar struct inode *inode; 1328a5f4374aSIngo Molnar 13291cc33161SRavi Bangoria if (no_uprobe_events()) 13301cc33161SRavi Bangoria return 0; 13311cc33161SRavi Bangoria 13321cc33161SRavi Bangoria if (vma->vm_file && 13331cc33161SRavi Bangoria (vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE && 13341cc33161SRavi Bangoria test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags)) 13351cc33161SRavi Bangoria delayed_ref_ctr_inc(vma); 13361cc33161SRavi Bangoria 13371cc33161SRavi Bangoria if (!valid_vma(vma, true)) 1338a5f4374aSIngo Molnar return 0; 1339a5f4374aSIngo Molnar 1340f281769eSOleg Nesterov inode = file_inode(vma->vm_file); 1341a5f4374aSIngo Molnar if (!inode) 1342a5f4374aSIngo Molnar return 0; 1343a5f4374aSIngo Molnar 1344a5f4374aSIngo Molnar mutex_lock(uprobes_mmap_hash(inode)); 1345891c3970SOleg Nesterov build_probe_list(inode, vma, vma->vm_start, vma->vm_end, &tmp_list); 1346806a98bdSOleg Nesterov /* 1347806a98bdSOleg Nesterov * We can race with uprobe_unregister(), this uprobe can be already 1348806a98bdSOleg Nesterov * removed. But in this case filter_chain() must return false, all 1349806a98bdSOleg Nesterov * consumers have gone away. 1350806a98bdSOleg Nesterov */ 1351665605a2SOleg Nesterov list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) { 1352806a98bdSOleg Nesterov if (!fatal_signal_pending(current) && 13538a7f2fa0SOleg Nesterov filter_chain(uprobe, UPROBE_FILTER_MMAP, vma->vm_mm)) { 135457683f72SOleg Nesterov unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset); 13555e5be71aSOleg Nesterov install_breakpoint(uprobe, vma->vm_mm, vma, vaddr); 1356a5f4374aSIngo Molnar } 1357a5f4374aSIngo Molnar put_uprobe(uprobe); 1358a5f4374aSIngo Molnar } 1359a5f4374aSIngo Molnar mutex_unlock(uprobes_mmap_hash(inode)); 1360a5f4374aSIngo Molnar 13615e5be71aSOleg Nesterov return 0; 1362a5f4374aSIngo Molnar } 1363a5f4374aSIngo Molnar 13649f68f672SOleg Nesterov static bool 13659f68f672SOleg Nesterov vma_has_uprobes(struct vm_area_struct *vma, unsigned long start, unsigned long end) 13669f68f672SOleg Nesterov { 13679f68f672SOleg Nesterov loff_t min, max; 13689f68f672SOleg Nesterov struct inode *inode; 13699f68f672SOleg Nesterov struct rb_node *n; 13709f68f672SOleg Nesterov 1371f281769eSOleg Nesterov inode = file_inode(vma->vm_file); 13729f68f672SOleg Nesterov 13739f68f672SOleg Nesterov min = vaddr_to_offset(vma, start); 13749f68f672SOleg Nesterov max = min + (end - start) - 1; 13759f68f672SOleg Nesterov 13769f68f672SOleg Nesterov spin_lock(&uprobes_treelock); 13779f68f672SOleg Nesterov n = find_node_in_range(inode, min, max); 13789f68f672SOleg Nesterov spin_unlock(&uprobes_treelock); 13799f68f672SOleg Nesterov 13809f68f672SOleg Nesterov return !!n; 13819f68f672SOleg Nesterov } 13829f68f672SOleg Nesterov 1383682968e0SSrikar Dronamraju /* 1384682968e0SSrikar Dronamraju * Called in context of a munmap of a vma. 1385682968e0SSrikar Dronamraju */ 1386cbc91f71SSrikar Dronamraju void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end) 1387682968e0SSrikar Dronamraju { 1388441f1eb7SOleg Nesterov if (no_uprobe_events() || !valid_vma(vma, false)) 1389682968e0SSrikar Dronamraju return; 1390682968e0SSrikar Dronamraju 13912fd611a9SOleg Nesterov if (!atomic_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */ 13922fd611a9SOleg Nesterov return; 13932fd611a9SOleg Nesterov 13949f68f672SOleg Nesterov if (!test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags) || 13959f68f672SOleg Nesterov test_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags)) 1396f8ac4ec9SOleg Nesterov return; 1397f8ac4ec9SOleg Nesterov 13989f68f672SOleg Nesterov if (vma_has_uprobes(vma, start, end)) 13999f68f672SOleg Nesterov set_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags); 1400682968e0SSrikar Dronamraju } 1401682968e0SSrikar Dronamraju 1402d4b3b638SSrikar Dronamraju /* Slot allocation for XOL */ 14036441ec8bSOleg Nesterov static int xol_add_vma(struct mm_struct *mm, struct xol_area *area) 1404d4b3b638SSrikar Dronamraju { 1405704bde3cSOleg Nesterov struct vm_area_struct *vma; 1406704bde3cSOleg Nesterov int ret; 1407d4b3b638SSrikar Dronamraju 1408598fdc1dSMichal Hocko if (down_write_killable(&mm->mmap_sem)) 1409598fdc1dSMichal Hocko return -EINTR; 1410598fdc1dSMichal Hocko 1411704bde3cSOleg Nesterov if (mm->uprobes_state.xol_area) { 1412704bde3cSOleg Nesterov ret = -EALREADY; 1413d4b3b638SSrikar Dronamraju goto fail; 1414704bde3cSOleg Nesterov } 1415d4b3b638SSrikar Dronamraju 1416af0d95afSOleg Nesterov if (!area->vaddr) { 1417d4b3b638SSrikar Dronamraju /* Try to map as high as possible, this is only a hint. */ 1418af0d95afSOleg Nesterov area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, 1419af0d95afSOleg Nesterov PAGE_SIZE, 0, 0); 1420d4b3b638SSrikar Dronamraju if (area->vaddr & ~PAGE_MASK) { 1421d4b3b638SSrikar Dronamraju ret = area->vaddr; 1422d4b3b638SSrikar Dronamraju goto fail; 1423d4b3b638SSrikar Dronamraju } 1424af0d95afSOleg Nesterov } 1425d4b3b638SSrikar Dronamraju 1426704bde3cSOleg Nesterov vma = _install_special_mapping(mm, area->vaddr, PAGE_SIZE, 1427704bde3cSOleg Nesterov VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|VM_IO, 1428704bde3cSOleg Nesterov &area->xol_mapping); 1429704bde3cSOleg Nesterov if (IS_ERR(vma)) { 1430704bde3cSOleg Nesterov ret = PTR_ERR(vma); 1431d4b3b638SSrikar Dronamraju goto fail; 1432704bde3cSOleg Nesterov } 1433d4b3b638SSrikar Dronamraju 1434704bde3cSOleg Nesterov ret = 0; 14355c6338b4SPaul E. McKenney /* pairs with get_xol_area() */ 14365c6338b4SPaul E. McKenney smp_store_release(&mm->uprobes_state.xol_area, area); /* ^^^ */ 1437d4b3b638SSrikar Dronamraju fail: 1438d4b3b638SSrikar Dronamraju up_write(&mm->mmap_sem); 1439d4b3b638SSrikar Dronamraju 1440d4b3b638SSrikar Dronamraju return ret; 1441d4b3b638SSrikar Dronamraju } 1442d4b3b638SSrikar Dronamraju 1443af0d95afSOleg Nesterov static struct xol_area *__create_xol_area(unsigned long vaddr) 1444d4b3b638SSrikar Dronamraju { 14459b545df8SOleg Nesterov struct mm_struct *mm = current->mm; 1446e78aebfdSAnton Arapov uprobe_opcode_t insn = UPROBE_SWBP_INSN; 14476441ec8bSOleg Nesterov struct xol_area *area; 14489b545df8SOleg Nesterov 1449af0d95afSOleg Nesterov area = kmalloc(sizeof(*area), GFP_KERNEL); 1450d4b3b638SSrikar Dronamraju if (unlikely(!area)) 1451c8a82538SOleg Nesterov goto out; 1452d4b3b638SSrikar Dronamraju 14536396bb22SKees Cook area->bitmap = kcalloc(BITS_TO_LONGS(UINSNS_PER_PAGE), sizeof(long), 14546396bb22SKees Cook GFP_KERNEL); 1455d4b3b638SSrikar Dronamraju if (!area->bitmap) 1456c8a82538SOleg Nesterov goto free_area; 1457c8a82538SOleg Nesterov 1458704bde3cSOleg Nesterov area->xol_mapping.name = "[uprobes]"; 1459869ae761SOleg Nesterov area->xol_mapping.fault = NULL; 1460704bde3cSOleg Nesterov area->xol_mapping.pages = area->pages; 1461f58bea2fSOleg Nesterov area->pages[0] = alloc_page(GFP_HIGHUSER); 1462f58bea2fSOleg Nesterov if (!area->pages[0]) 1463c8a82538SOleg Nesterov goto free_bitmap; 1464f58bea2fSOleg Nesterov area->pages[1] = NULL; 1465d4b3b638SSrikar Dronamraju 1466af0d95afSOleg Nesterov area->vaddr = vaddr; 1467d4b3b638SSrikar Dronamraju init_waitqueue_head(&area->wq); 14686441ec8bSOleg Nesterov /* Reserve the 1st slot for get_trampoline_vaddr() */ 14696441ec8bSOleg Nesterov set_bit(0, area->bitmap); 14706441ec8bSOleg Nesterov atomic_set(&area->slot_count, 1); 1471297e765eSMarcin Nowakowski arch_uprobe_copy_ixol(area->pages[0], 0, &insn, UPROBE_SWBP_INSN_SIZE); 1472e78aebfdSAnton Arapov 14736441ec8bSOleg Nesterov if (!xol_add_vma(mm, area)) 1474d4b3b638SSrikar Dronamraju return area; 1475d4b3b638SSrikar Dronamraju 1476f58bea2fSOleg Nesterov __free_page(area->pages[0]); 1477c8a82538SOleg Nesterov free_bitmap: 1478d4b3b638SSrikar Dronamraju kfree(area->bitmap); 1479c8a82538SOleg Nesterov free_area: 1480d4b3b638SSrikar Dronamraju kfree(area); 1481c8a82538SOleg Nesterov out: 14826441ec8bSOleg Nesterov return NULL; 14836441ec8bSOleg Nesterov } 14846441ec8bSOleg Nesterov 14856441ec8bSOleg Nesterov /* 14866441ec8bSOleg Nesterov * get_xol_area - Allocate process's xol_area if necessary. 14876441ec8bSOleg Nesterov * This area will be used for storing instructions for execution out of line. 14886441ec8bSOleg Nesterov * 14896441ec8bSOleg Nesterov * Returns the allocated area or NULL. 14906441ec8bSOleg Nesterov */ 14916441ec8bSOleg Nesterov static struct xol_area *get_xol_area(void) 14926441ec8bSOleg Nesterov { 14936441ec8bSOleg Nesterov struct mm_struct *mm = current->mm; 14946441ec8bSOleg Nesterov struct xol_area *area; 14956441ec8bSOleg Nesterov 14966441ec8bSOleg Nesterov if (!mm->uprobes_state.xol_area) 1497af0d95afSOleg Nesterov __create_xol_area(0); 14986441ec8bSOleg Nesterov 14995c6338b4SPaul E. McKenney /* Pairs with xol_add_vma() smp_store_release() */ 15005c6338b4SPaul E. McKenney area = READ_ONCE(mm->uprobes_state.xol_area); /* ^^^ */ 15019b545df8SOleg Nesterov return area; 1502d4b3b638SSrikar Dronamraju } 1503d4b3b638SSrikar Dronamraju 1504d4b3b638SSrikar Dronamraju /* 1505d4b3b638SSrikar Dronamraju * uprobe_clear_state - Free the area allocated for slots. 1506d4b3b638SSrikar Dronamraju */ 1507d4b3b638SSrikar Dronamraju void uprobe_clear_state(struct mm_struct *mm) 1508d4b3b638SSrikar Dronamraju { 1509d4b3b638SSrikar Dronamraju struct xol_area *area = mm->uprobes_state.xol_area; 1510d4b3b638SSrikar Dronamraju 15111cc33161SRavi Bangoria mutex_lock(&delayed_uprobe_lock); 15121cc33161SRavi Bangoria delayed_uprobe_remove(NULL, mm); 15131cc33161SRavi Bangoria mutex_unlock(&delayed_uprobe_lock); 15141cc33161SRavi Bangoria 1515d4b3b638SSrikar Dronamraju if (!area) 1516d4b3b638SSrikar Dronamraju return; 1517d4b3b638SSrikar Dronamraju 1518f58bea2fSOleg Nesterov put_page(area->pages[0]); 1519d4b3b638SSrikar Dronamraju kfree(area->bitmap); 1520d4b3b638SSrikar Dronamraju kfree(area); 1521d4b3b638SSrikar Dronamraju } 1522d4b3b638SSrikar Dronamraju 152332cdba1eSOleg Nesterov void uprobe_start_dup_mmap(void) 152432cdba1eSOleg Nesterov { 152532cdba1eSOleg Nesterov percpu_down_read(&dup_mmap_sem); 152632cdba1eSOleg Nesterov } 152732cdba1eSOleg Nesterov 152832cdba1eSOleg Nesterov void uprobe_end_dup_mmap(void) 152932cdba1eSOleg Nesterov { 153032cdba1eSOleg Nesterov percpu_up_read(&dup_mmap_sem); 153132cdba1eSOleg Nesterov } 153232cdba1eSOleg Nesterov 1533f8ac4ec9SOleg Nesterov void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm) 1534f8ac4ec9SOleg Nesterov { 15359f68f672SOleg Nesterov if (test_bit(MMF_HAS_UPROBES, &oldmm->flags)) { 1536f8ac4ec9SOleg Nesterov set_bit(MMF_HAS_UPROBES, &newmm->flags); 15379f68f672SOleg Nesterov /* unconditionally, dup_mmap() skips VM_DONTCOPY vmas */ 15389f68f672SOleg Nesterov set_bit(MMF_RECALC_UPROBES, &newmm->flags); 15399f68f672SOleg Nesterov } 1540f8ac4ec9SOleg Nesterov } 1541f8ac4ec9SOleg Nesterov 1542d4b3b638SSrikar Dronamraju /* 1543d4b3b638SSrikar Dronamraju * - search for a free slot. 1544d4b3b638SSrikar Dronamraju */ 1545d4b3b638SSrikar Dronamraju static unsigned long xol_take_insn_slot(struct xol_area *area) 1546d4b3b638SSrikar Dronamraju { 1547d4b3b638SSrikar Dronamraju unsigned long slot_addr; 1548d4b3b638SSrikar Dronamraju int slot_nr; 1549d4b3b638SSrikar Dronamraju 1550d4b3b638SSrikar Dronamraju do { 1551d4b3b638SSrikar Dronamraju slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE); 1552d4b3b638SSrikar Dronamraju if (slot_nr < UINSNS_PER_PAGE) { 1553d4b3b638SSrikar Dronamraju if (!test_and_set_bit(slot_nr, area->bitmap)) 1554d4b3b638SSrikar Dronamraju break; 1555d4b3b638SSrikar Dronamraju 1556d4b3b638SSrikar Dronamraju slot_nr = UINSNS_PER_PAGE; 1557d4b3b638SSrikar Dronamraju continue; 1558d4b3b638SSrikar Dronamraju } 1559d4b3b638SSrikar Dronamraju wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE)); 1560d4b3b638SSrikar Dronamraju } while (slot_nr >= UINSNS_PER_PAGE); 1561d4b3b638SSrikar Dronamraju 1562d4b3b638SSrikar Dronamraju slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES); 1563d4b3b638SSrikar Dronamraju atomic_inc(&area->slot_count); 1564d4b3b638SSrikar Dronamraju 1565d4b3b638SSrikar Dronamraju return slot_addr; 1566d4b3b638SSrikar Dronamraju } 1567d4b3b638SSrikar Dronamraju 1568d4b3b638SSrikar Dronamraju /* 1569a6cb3f6dSOleg Nesterov * xol_get_insn_slot - allocate a slot for xol. 1570d4b3b638SSrikar Dronamraju * Returns the allocated slot address or 0. 1571d4b3b638SSrikar Dronamraju */ 1572a6cb3f6dSOleg Nesterov static unsigned long xol_get_insn_slot(struct uprobe *uprobe) 1573d4b3b638SSrikar Dronamraju { 1574d4b3b638SSrikar Dronamraju struct xol_area *area; 1575a6cb3f6dSOleg Nesterov unsigned long xol_vaddr; 1576d4b3b638SSrikar Dronamraju 15779b545df8SOleg Nesterov area = get_xol_area(); 1578d4b3b638SSrikar Dronamraju if (!area) 1579d4b3b638SSrikar Dronamraju return 0; 1580d4b3b638SSrikar Dronamraju 1581a6cb3f6dSOleg Nesterov xol_vaddr = xol_take_insn_slot(area); 1582a6cb3f6dSOleg Nesterov if (unlikely(!xol_vaddr)) 1583d4b3b638SSrikar Dronamraju return 0; 1584d4b3b638SSrikar Dronamraju 1585f58bea2fSOleg Nesterov arch_uprobe_copy_ixol(area->pages[0], xol_vaddr, 1586803200e2SOleg Nesterov &uprobe->arch.ixol, sizeof(uprobe->arch.ixol)); 1587d4b3b638SSrikar Dronamraju 1588a6cb3f6dSOleg Nesterov return xol_vaddr; 1589d4b3b638SSrikar Dronamraju } 1590d4b3b638SSrikar Dronamraju 1591d4b3b638SSrikar Dronamraju /* 1592d4b3b638SSrikar Dronamraju * xol_free_insn_slot - If slot was earlier allocated by 1593d4b3b638SSrikar Dronamraju * @xol_get_insn_slot(), make the slot available for 1594d4b3b638SSrikar Dronamraju * subsequent requests. 1595d4b3b638SSrikar Dronamraju */ 1596d4b3b638SSrikar Dronamraju static void xol_free_insn_slot(struct task_struct *tsk) 1597d4b3b638SSrikar Dronamraju { 1598d4b3b638SSrikar Dronamraju struct xol_area *area; 1599d4b3b638SSrikar Dronamraju unsigned long vma_end; 1600d4b3b638SSrikar Dronamraju unsigned long slot_addr; 1601d4b3b638SSrikar Dronamraju 1602d4b3b638SSrikar Dronamraju if (!tsk->mm || !tsk->mm->uprobes_state.xol_area || !tsk->utask) 1603d4b3b638SSrikar Dronamraju return; 1604d4b3b638SSrikar Dronamraju 1605d4b3b638SSrikar Dronamraju slot_addr = tsk->utask->xol_vaddr; 1606af4355e9SOleg Nesterov if (unlikely(!slot_addr)) 1607d4b3b638SSrikar Dronamraju return; 1608d4b3b638SSrikar Dronamraju 1609d4b3b638SSrikar Dronamraju area = tsk->mm->uprobes_state.xol_area; 1610d4b3b638SSrikar Dronamraju vma_end = area->vaddr + PAGE_SIZE; 1611d4b3b638SSrikar Dronamraju if (area->vaddr <= slot_addr && slot_addr < vma_end) { 1612d4b3b638SSrikar Dronamraju unsigned long offset; 1613d4b3b638SSrikar Dronamraju int slot_nr; 1614d4b3b638SSrikar Dronamraju 1615d4b3b638SSrikar Dronamraju offset = slot_addr - area->vaddr; 1616d4b3b638SSrikar Dronamraju slot_nr = offset / UPROBE_XOL_SLOT_BYTES; 1617d4b3b638SSrikar Dronamraju if (slot_nr >= UINSNS_PER_PAGE) 1618d4b3b638SSrikar Dronamraju return; 1619d4b3b638SSrikar Dronamraju 1620d4b3b638SSrikar Dronamraju clear_bit(slot_nr, area->bitmap); 1621d4b3b638SSrikar Dronamraju atomic_dec(&area->slot_count); 16222a742cedSOleg Nesterov smp_mb__after_atomic(); /* pairs with prepare_to_wait() */ 1623d4b3b638SSrikar Dronamraju if (waitqueue_active(&area->wq)) 1624d4b3b638SSrikar Dronamraju wake_up(&area->wq); 1625d4b3b638SSrikar Dronamraju 1626d4b3b638SSrikar Dronamraju tsk->utask->xol_vaddr = 0; 1627d4b3b638SSrikar Dronamraju } 1628d4b3b638SSrikar Dronamraju } 1629d4b3b638SSrikar Dronamraju 163072e6ae28SVictor Kamensky void __weak arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, 163172e6ae28SVictor Kamensky void *src, unsigned long len) 163272e6ae28SVictor Kamensky { 163372e6ae28SVictor Kamensky /* Initialize the slot */ 163472e6ae28SVictor Kamensky copy_to_page(page, vaddr, src, len); 163572e6ae28SVictor Kamensky 163672e6ae28SVictor Kamensky /* 163772e6ae28SVictor Kamensky * We probably need flush_icache_user_range() but it needs vma. 163872e6ae28SVictor Kamensky * This should work on most of architectures by default. If 163972e6ae28SVictor Kamensky * architecture needs to do something different it can define 164072e6ae28SVictor Kamensky * its own version of the function. 164172e6ae28SVictor Kamensky */ 164272e6ae28SVictor Kamensky flush_dcache_page(page); 164372e6ae28SVictor Kamensky } 164472e6ae28SVictor Kamensky 16450326f5a9SSrikar Dronamraju /** 16460326f5a9SSrikar Dronamraju * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs 16470326f5a9SSrikar Dronamraju * @regs: Reflects the saved state of the task after it has hit a breakpoint 16480326f5a9SSrikar Dronamraju * instruction. 16490326f5a9SSrikar Dronamraju * Return the address of the breakpoint instruction. 16500326f5a9SSrikar Dronamraju */ 16510326f5a9SSrikar Dronamraju unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs) 16520326f5a9SSrikar Dronamraju { 16530326f5a9SSrikar Dronamraju return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE; 16540326f5a9SSrikar Dronamraju } 16550326f5a9SSrikar Dronamraju 1656b02ef20aSOleg Nesterov unsigned long uprobe_get_trap_addr(struct pt_regs *regs) 1657b02ef20aSOleg Nesterov { 1658b02ef20aSOleg Nesterov struct uprobe_task *utask = current->utask; 1659b02ef20aSOleg Nesterov 1660b02ef20aSOleg Nesterov if (unlikely(utask && utask->active_uprobe)) 1661b02ef20aSOleg Nesterov return utask->vaddr; 1662b02ef20aSOleg Nesterov 1663b02ef20aSOleg Nesterov return instruction_pointer(regs); 1664b02ef20aSOleg Nesterov } 1665b02ef20aSOleg Nesterov 16662bb5e840SOleg Nesterov static struct return_instance *free_ret_instance(struct return_instance *ri) 16672bb5e840SOleg Nesterov { 16682bb5e840SOleg Nesterov struct return_instance *next = ri->next; 16692bb5e840SOleg Nesterov put_uprobe(ri->uprobe); 16702bb5e840SOleg Nesterov kfree(ri); 16712bb5e840SOleg Nesterov return next; 16722bb5e840SOleg Nesterov } 16732bb5e840SOleg Nesterov 16740326f5a9SSrikar Dronamraju /* 16750326f5a9SSrikar Dronamraju * Called with no locks held. 1676788faab7STobias Tefke * Called in context of an exiting or an exec-ing thread. 16770326f5a9SSrikar Dronamraju */ 16780326f5a9SSrikar Dronamraju void uprobe_free_utask(struct task_struct *t) 16790326f5a9SSrikar Dronamraju { 16800326f5a9SSrikar Dronamraju struct uprobe_task *utask = t->utask; 16812bb5e840SOleg Nesterov struct return_instance *ri; 16820326f5a9SSrikar Dronamraju 16830326f5a9SSrikar Dronamraju if (!utask) 16840326f5a9SSrikar Dronamraju return; 16850326f5a9SSrikar Dronamraju 16860326f5a9SSrikar Dronamraju if (utask->active_uprobe) 16870326f5a9SSrikar Dronamraju put_uprobe(utask->active_uprobe); 16880326f5a9SSrikar Dronamraju 16890dfd0eb8SAnton Arapov ri = utask->return_instances; 16902bb5e840SOleg Nesterov while (ri) 16912bb5e840SOleg Nesterov ri = free_ret_instance(ri); 16920dfd0eb8SAnton Arapov 1693d4b3b638SSrikar Dronamraju xol_free_insn_slot(t); 16940326f5a9SSrikar Dronamraju kfree(utask); 16950326f5a9SSrikar Dronamraju t->utask = NULL; 16960326f5a9SSrikar Dronamraju } 16970326f5a9SSrikar Dronamraju 16980326f5a9SSrikar Dronamraju /* 16995a2df662SOleg Nesterov * Allocate a uprobe_task object for the task if if necessary. 17005a2df662SOleg Nesterov * Called when the thread hits a breakpoint. 17010326f5a9SSrikar Dronamraju * 17020326f5a9SSrikar Dronamraju * Returns: 17030326f5a9SSrikar Dronamraju * - pointer to new uprobe_task on success 17040326f5a9SSrikar Dronamraju * - NULL otherwise 17050326f5a9SSrikar Dronamraju */ 17065a2df662SOleg Nesterov static struct uprobe_task *get_utask(void) 17070326f5a9SSrikar Dronamraju { 17085a2df662SOleg Nesterov if (!current->utask) 17095a2df662SOleg Nesterov current->utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL); 17105a2df662SOleg Nesterov return current->utask; 17110326f5a9SSrikar Dronamraju } 17120326f5a9SSrikar Dronamraju 1713248d3a7bSOleg Nesterov static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask) 1714248d3a7bSOleg Nesterov { 1715248d3a7bSOleg Nesterov struct uprobe_task *n_utask; 1716248d3a7bSOleg Nesterov struct return_instance **p, *o, *n; 1717248d3a7bSOleg Nesterov 1718248d3a7bSOleg Nesterov n_utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL); 1719248d3a7bSOleg Nesterov if (!n_utask) 1720248d3a7bSOleg Nesterov return -ENOMEM; 1721248d3a7bSOleg Nesterov t->utask = n_utask; 1722248d3a7bSOleg Nesterov 1723248d3a7bSOleg Nesterov p = &n_utask->return_instances; 1724248d3a7bSOleg Nesterov for (o = o_utask->return_instances; o; o = o->next) { 1725248d3a7bSOleg Nesterov n = kmalloc(sizeof(struct return_instance), GFP_KERNEL); 1726248d3a7bSOleg Nesterov if (!n) 1727248d3a7bSOleg Nesterov return -ENOMEM; 1728248d3a7bSOleg Nesterov 1729248d3a7bSOleg Nesterov *n = *o; 1730f231722aSOleg Nesterov get_uprobe(n->uprobe); 1731248d3a7bSOleg Nesterov n->next = NULL; 1732248d3a7bSOleg Nesterov 1733248d3a7bSOleg Nesterov *p = n; 1734248d3a7bSOleg Nesterov p = &n->next; 1735248d3a7bSOleg Nesterov n_utask->depth++; 1736248d3a7bSOleg Nesterov } 1737248d3a7bSOleg Nesterov 1738248d3a7bSOleg Nesterov return 0; 1739248d3a7bSOleg Nesterov } 1740248d3a7bSOleg Nesterov 1741248d3a7bSOleg Nesterov static void uprobe_warn(struct task_struct *t, const char *msg) 1742248d3a7bSOleg Nesterov { 1743248d3a7bSOleg Nesterov pr_warn("uprobe: %s:%d failed to %s\n", 1744248d3a7bSOleg Nesterov current->comm, current->pid, msg); 1745248d3a7bSOleg Nesterov } 1746248d3a7bSOleg Nesterov 1747aa59c53fSOleg Nesterov static void dup_xol_work(struct callback_head *work) 1748aa59c53fSOleg Nesterov { 1749aa59c53fSOleg Nesterov if (current->flags & PF_EXITING) 1750aa59c53fSOleg Nesterov return; 1751aa59c53fSOleg Nesterov 1752598fdc1dSMichal Hocko if (!__create_xol_area(current->utask->dup_xol_addr) && 1753598fdc1dSMichal Hocko !fatal_signal_pending(current)) 1754aa59c53fSOleg Nesterov uprobe_warn(current, "dup xol area"); 1755aa59c53fSOleg Nesterov } 1756aa59c53fSOleg Nesterov 1757e78aebfdSAnton Arapov /* 1758b68e0749SOleg Nesterov * Called in context of a new clone/fork from copy_process. 1759b68e0749SOleg Nesterov */ 17603ab67966SOleg Nesterov void uprobe_copy_process(struct task_struct *t, unsigned long flags) 1761b68e0749SOleg Nesterov { 1762248d3a7bSOleg Nesterov struct uprobe_task *utask = current->utask; 1763248d3a7bSOleg Nesterov struct mm_struct *mm = current->mm; 1764aa59c53fSOleg Nesterov struct xol_area *area; 1765248d3a7bSOleg Nesterov 1766b68e0749SOleg Nesterov t->utask = NULL; 1767248d3a7bSOleg Nesterov 17683ab67966SOleg Nesterov if (!utask || !utask->return_instances) 17693ab67966SOleg Nesterov return; 17703ab67966SOleg Nesterov 17713ab67966SOleg Nesterov if (mm == t->mm && !(flags & CLONE_VFORK)) 1772248d3a7bSOleg Nesterov return; 1773248d3a7bSOleg Nesterov 1774248d3a7bSOleg Nesterov if (dup_utask(t, utask)) 1775248d3a7bSOleg Nesterov return uprobe_warn(t, "dup ret instances"); 1776aa59c53fSOleg Nesterov 1777aa59c53fSOleg Nesterov /* The task can fork() after dup_xol_work() fails */ 1778aa59c53fSOleg Nesterov area = mm->uprobes_state.xol_area; 1779aa59c53fSOleg Nesterov if (!area) 1780aa59c53fSOleg Nesterov return uprobe_warn(t, "dup xol area"); 1781aa59c53fSOleg Nesterov 17823ab67966SOleg Nesterov if (mm == t->mm) 17833ab67966SOleg Nesterov return; 17843ab67966SOleg Nesterov 178532473431SOleg Nesterov t->utask->dup_xol_addr = area->vaddr; 178632473431SOleg Nesterov init_task_work(&t->utask->dup_xol_work, dup_xol_work); 178732473431SOleg Nesterov task_work_add(t, &t->utask->dup_xol_work, true); 1788b68e0749SOleg Nesterov } 1789b68e0749SOleg Nesterov 1790b68e0749SOleg Nesterov /* 1791e78aebfdSAnton Arapov * Current area->vaddr notion assume the trampoline address is always 1792e78aebfdSAnton Arapov * equal area->vaddr. 1793e78aebfdSAnton Arapov * 1794e78aebfdSAnton Arapov * Returns -1 in case the xol_area is not allocated. 1795e78aebfdSAnton Arapov */ 1796e78aebfdSAnton Arapov static unsigned long get_trampoline_vaddr(void) 1797e78aebfdSAnton Arapov { 1798e78aebfdSAnton Arapov struct xol_area *area; 1799e78aebfdSAnton Arapov unsigned long trampoline_vaddr = -1; 1800e78aebfdSAnton Arapov 18015c6338b4SPaul E. McKenney /* Pairs with xol_add_vma() smp_store_release() */ 18025c6338b4SPaul E. McKenney area = READ_ONCE(current->mm->uprobes_state.xol_area); /* ^^^ */ 1803e78aebfdSAnton Arapov if (area) 1804e78aebfdSAnton Arapov trampoline_vaddr = area->vaddr; 1805e78aebfdSAnton Arapov 1806e78aebfdSAnton Arapov return trampoline_vaddr; 1807e78aebfdSAnton Arapov } 1808e78aebfdSAnton Arapov 1809db087ef6SOleg Nesterov static void cleanup_return_instances(struct uprobe_task *utask, bool chained, 1810db087ef6SOleg Nesterov struct pt_regs *regs) 1811a5b7e1a8SOleg Nesterov { 1812a5b7e1a8SOleg Nesterov struct return_instance *ri = utask->return_instances; 1813db087ef6SOleg Nesterov enum rp_check ctx = chained ? RP_CHECK_CHAIN_CALL : RP_CHECK_CALL; 181486dcb702SOleg Nesterov 181586dcb702SOleg Nesterov while (ri && !arch_uretprobe_is_alive(ri, ctx, regs)) { 1816a5b7e1a8SOleg Nesterov ri = free_ret_instance(ri); 1817a5b7e1a8SOleg Nesterov utask->depth--; 1818a5b7e1a8SOleg Nesterov } 1819a5b7e1a8SOleg Nesterov utask->return_instances = ri; 1820a5b7e1a8SOleg Nesterov } 1821a5b7e1a8SOleg Nesterov 18220dfd0eb8SAnton Arapov static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs) 18230dfd0eb8SAnton Arapov { 18240dfd0eb8SAnton Arapov struct return_instance *ri; 18250dfd0eb8SAnton Arapov struct uprobe_task *utask; 18260dfd0eb8SAnton Arapov unsigned long orig_ret_vaddr, trampoline_vaddr; 1827db087ef6SOleg Nesterov bool chained; 18280dfd0eb8SAnton Arapov 18290dfd0eb8SAnton Arapov if (!get_xol_area()) 18300dfd0eb8SAnton Arapov return; 18310dfd0eb8SAnton Arapov 18320dfd0eb8SAnton Arapov utask = get_utask(); 18330dfd0eb8SAnton Arapov if (!utask) 18340dfd0eb8SAnton Arapov return; 18350dfd0eb8SAnton Arapov 1836ded49c55SAnton Arapov if (utask->depth >= MAX_URETPROBE_DEPTH) { 1837ded49c55SAnton Arapov printk_ratelimited(KERN_INFO "uprobe: omit uretprobe due to" 1838ded49c55SAnton Arapov " nestedness limit pid/tgid=%d/%d\n", 1839ded49c55SAnton Arapov current->pid, current->tgid); 1840ded49c55SAnton Arapov return; 1841ded49c55SAnton Arapov } 1842ded49c55SAnton Arapov 18436c58d0e4SOleg Nesterov ri = kmalloc(sizeof(struct return_instance), GFP_KERNEL); 18440dfd0eb8SAnton Arapov if (!ri) 18456c58d0e4SOleg Nesterov return; 18460dfd0eb8SAnton Arapov 18470dfd0eb8SAnton Arapov trampoline_vaddr = get_trampoline_vaddr(); 18480dfd0eb8SAnton Arapov orig_ret_vaddr = arch_uretprobe_hijack_return_addr(trampoline_vaddr, regs); 18490dfd0eb8SAnton Arapov if (orig_ret_vaddr == -1) 18500dfd0eb8SAnton Arapov goto fail; 18510dfd0eb8SAnton Arapov 1852a5b7e1a8SOleg Nesterov /* drop the entries invalidated by longjmp() */ 1853db087ef6SOleg Nesterov chained = (orig_ret_vaddr == trampoline_vaddr); 1854db087ef6SOleg Nesterov cleanup_return_instances(utask, chained, regs); 1855a5b7e1a8SOleg Nesterov 18560dfd0eb8SAnton Arapov /* 18570dfd0eb8SAnton Arapov * We don't want to keep trampoline address in stack, rather keep the 18580dfd0eb8SAnton Arapov * original return address of first caller thru all the consequent 18590dfd0eb8SAnton Arapov * instances. This also makes breakpoint unwrapping easier. 18600dfd0eb8SAnton Arapov */ 1861db087ef6SOleg Nesterov if (chained) { 18620dfd0eb8SAnton Arapov if (!utask->return_instances) { 18630dfd0eb8SAnton Arapov /* 18640dfd0eb8SAnton Arapov * This situation is not possible. Likely we have an 18650dfd0eb8SAnton Arapov * attack from user-space. 18660dfd0eb8SAnton Arapov */ 18676c58d0e4SOleg Nesterov uprobe_warn(current, "handle tail call"); 18680dfd0eb8SAnton Arapov goto fail; 18690dfd0eb8SAnton Arapov } 18700dfd0eb8SAnton Arapov orig_ret_vaddr = utask->return_instances->orig_ret_vaddr; 18710dfd0eb8SAnton Arapov } 18720dfd0eb8SAnton Arapov 1873f231722aSOleg Nesterov ri->uprobe = get_uprobe(uprobe); 18740dfd0eb8SAnton Arapov ri->func = instruction_pointer(regs); 18757b868e48SOleg Nesterov ri->stack = user_stack_pointer(regs); 18760dfd0eb8SAnton Arapov ri->orig_ret_vaddr = orig_ret_vaddr; 18770dfd0eb8SAnton Arapov ri->chained = chained; 18780dfd0eb8SAnton Arapov 1879ded49c55SAnton Arapov utask->depth++; 18800dfd0eb8SAnton Arapov ri->next = utask->return_instances; 18810dfd0eb8SAnton Arapov utask->return_instances = ri; 18820dfd0eb8SAnton Arapov 18830dfd0eb8SAnton Arapov return; 18840dfd0eb8SAnton Arapov fail: 18850dfd0eb8SAnton Arapov kfree(ri); 18860dfd0eb8SAnton Arapov } 18870dfd0eb8SAnton Arapov 18880326f5a9SSrikar Dronamraju /* Prepare to single-step probed instruction out of line. */ 18890326f5a9SSrikar Dronamraju static int 1890a6cb3f6dSOleg Nesterov pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr) 18910326f5a9SSrikar Dronamraju { 1892a6cb3f6dSOleg Nesterov struct uprobe_task *utask; 1893a6cb3f6dSOleg Nesterov unsigned long xol_vaddr; 1894aba51024SOleg Nesterov int err; 1895d4b3b638SSrikar Dronamraju 1896608e7427SOleg Nesterov utask = get_utask(); 1897608e7427SOleg Nesterov if (!utask) 1898608e7427SOleg Nesterov return -ENOMEM; 1899a6cb3f6dSOleg Nesterov 1900a6cb3f6dSOleg Nesterov xol_vaddr = xol_get_insn_slot(uprobe); 1901a6cb3f6dSOleg Nesterov if (!xol_vaddr) 1902a6cb3f6dSOleg Nesterov return -ENOMEM; 1903a6cb3f6dSOleg Nesterov 1904a6cb3f6dSOleg Nesterov utask->xol_vaddr = xol_vaddr; 1905a6cb3f6dSOleg Nesterov utask->vaddr = bp_vaddr; 1906a6cb3f6dSOleg Nesterov 1907aba51024SOleg Nesterov err = arch_uprobe_pre_xol(&uprobe->arch, regs); 1908aba51024SOleg Nesterov if (unlikely(err)) { 1909aba51024SOleg Nesterov xol_free_insn_slot(current); 1910aba51024SOleg Nesterov return err; 1911aba51024SOleg Nesterov } 1912aba51024SOleg Nesterov 1913608e7427SOleg Nesterov utask->active_uprobe = uprobe; 1914608e7427SOleg Nesterov utask->state = UTASK_SSTEP; 1915aba51024SOleg Nesterov return 0; 19160326f5a9SSrikar Dronamraju } 19170326f5a9SSrikar Dronamraju 19180326f5a9SSrikar Dronamraju /* 19190326f5a9SSrikar Dronamraju * If we are singlestepping, then ensure this thread is not connected to 19200326f5a9SSrikar Dronamraju * non-fatal signals until completion of singlestep. When xol insn itself 19210326f5a9SSrikar Dronamraju * triggers the signal, restart the original insn even if the task is 19220326f5a9SSrikar Dronamraju * already SIGKILL'ed (since coredump should report the correct ip). This 19230326f5a9SSrikar Dronamraju * is even more important if the task has a handler for SIGSEGV/etc, The 19240326f5a9SSrikar Dronamraju * _same_ instruction should be repeated again after return from the signal 19250326f5a9SSrikar Dronamraju * handler, and SSTEP can never finish in this case. 19260326f5a9SSrikar Dronamraju */ 19270326f5a9SSrikar Dronamraju bool uprobe_deny_signal(void) 19280326f5a9SSrikar Dronamraju { 19290326f5a9SSrikar Dronamraju struct task_struct *t = current; 19300326f5a9SSrikar Dronamraju struct uprobe_task *utask = t->utask; 19310326f5a9SSrikar Dronamraju 19320326f5a9SSrikar Dronamraju if (likely(!utask || !utask->active_uprobe)) 19330326f5a9SSrikar Dronamraju return false; 19340326f5a9SSrikar Dronamraju 19350326f5a9SSrikar Dronamraju WARN_ON_ONCE(utask->state != UTASK_SSTEP); 19360326f5a9SSrikar Dronamraju 19370326f5a9SSrikar Dronamraju if (signal_pending(t)) { 19380326f5a9SSrikar Dronamraju spin_lock_irq(&t->sighand->siglock); 19390326f5a9SSrikar Dronamraju clear_tsk_thread_flag(t, TIF_SIGPENDING); 19400326f5a9SSrikar Dronamraju spin_unlock_irq(&t->sighand->siglock); 19410326f5a9SSrikar Dronamraju 19420326f5a9SSrikar Dronamraju if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) { 19430326f5a9SSrikar Dronamraju utask->state = UTASK_SSTEP_TRAPPED; 19440326f5a9SSrikar Dronamraju set_tsk_thread_flag(t, TIF_UPROBE); 19450326f5a9SSrikar Dronamraju } 19460326f5a9SSrikar Dronamraju } 19470326f5a9SSrikar Dronamraju 19480326f5a9SSrikar Dronamraju return true; 19490326f5a9SSrikar Dronamraju } 19500326f5a9SSrikar Dronamraju 1951499a4f3eSOleg Nesterov static void mmf_recalc_uprobes(struct mm_struct *mm) 1952499a4f3eSOleg Nesterov { 1953499a4f3eSOleg Nesterov struct vm_area_struct *vma; 1954499a4f3eSOleg Nesterov 1955499a4f3eSOleg Nesterov for (vma = mm->mmap; vma; vma = vma->vm_next) { 1956499a4f3eSOleg Nesterov if (!valid_vma(vma, false)) 1957499a4f3eSOleg Nesterov continue; 1958499a4f3eSOleg Nesterov /* 1959499a4f3eSOleg Nesterov * This is not strictly accurate, we can race with 1960499a4f3eSOleg Nesterov * uprobe_unregister() and see the already removed 1961499a4f3eSOleg Nesterov * uprobe if delete_uprobe() was not yet called. 196263633cbfSOleg Nesterov * Or this uprobe can be filtered out. 1963499a4f3eSOleg Nesterov */ 1964499a4f3eSOleg Nesterov if (vma_has_uprobes(vma, vma->vm_start, vma->vm_end)) 1965499a4f3eSOleg Nesterov return; 1966499a4f3eSOleg Nesterov } 1967499a4f3eSOleg Nesterov 1968499a4f3eSOleg Nesterov clear_bit(MMF_HAS_UPROBES, &mm->flags); 1969499a4f3eSOleg Nesterov } 1970499a4f3eSOleg Nesterov 19710908ad6eSAnanth N Mavinakayanahalli static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr) 1972ec75fba9SOleg Nesterov { 1973ec75fba9SOleg Nesterov struct page *page; 1974ec75fba9SOleg Nesterov uprobe_opcode_t opcode; 1975ec75fba9SOleg Nesterov int result; 1976ec75fba9SOleg Nesterov 1977ec75fba9SOleg Nesterov pagefault_disable(); 1978bd28b145SLinus Torvalds result = __get_user(opcode, (uprobe_opcode_t __user *)vaddr); 1979ec75fba9SOleg Nesterov pagefault_enable(); 1980ec75fba9SOleg Nesterov 1981ec75fba9SOleg Nesterov if (likely(result == 0)) 1982ec75fba9SOleg Nesterov goto out; 1983ec75fba9SOleg Nesterov 19841e987790SDave Hansen /* 19851e987790SDave Hansen * The NULL 'tsk' here ensures that any faults that occur here 19861e987790SDave Hansen * will not be accounted to the task. 'mm' *is* current->mm, 19871e987790SDave Hansen * but we treat this as a 'remote' access since it is 19881e987790SDave Hansen * essentially a kernel access to the memory. 19891e987790SDave Hansen */ 19909beae1eaSLorenzo Stoakes result = get_user_pages_remote(NULL, mm, vaddr, 1, FOLL_FORCE, &page, 19915b56d49fSLorenzo Stoakes NULL, NULL); 1992ec75fba9SOleg Nesterov if (result < 0) 1993ec75fba9SOleg Nesterov return result; 1994ec75fba9SOleg Nesterov 1995ab0d805cSOleg Nesterov copy_from_page(page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE); 1996ec75fba9SOleg Nesterov put_page(page); 1997ec75fba9SOleg Nesterov out: 19980908ad6eSAnanth N Mavinakayanahalli /* This needs to return true for any variant of the trap insn */ 19990908ad6eSAnanth N Mavinakayanahalli return is_trap_insn(&opcode); 2000ec75fba9SOleg Nesterov } 2001ec75fba9SOleg Nesterov 2002d790d346SOleg Nesterov static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp) 20030326f5a9SSrikar Dronamraju { 20043a9ea052SOleg Nesterov struct mm_struct *mm = current->mm; 20053a9ea052SOleg Nesterov struct uprobe *uprobe = NULL; 20060326f5a9SSrikar Dronamraju struct vm_area_struct *vma; 20070326f5a9SSrikar Dronamraju 20080326f5a9SSrikar Dronamraju down_read(&mm->mmap_sem); 20090326f5a9SSrikar Dronamraju vma = find_vma(mm, bp_vaddr); 20103a9ea052SOleg Nesterov if (vma && vma->vm_start <= bp_vaddr) { 20113a9ea052SOleg Nesterov if (valid_vma(vma, false)) { 2012f281769eSOleg Nesterov struct inode *inode = file_inode(vma->vm_file); 2013cb113b47SOleg Nesterov loff_t offset = vaddr_to_offset(vma, bp_vaddr); 20140326f5a9SSrikar Dronamraju 20150326f5a9SSrikar Dronamraju uprobe = find_uprobe(inode, offset); 20160326f5a9SSrikar Dronamraju } 2017d790d346SOleg Nesterov 2018d790d346SOleg Nesterov if (!uprobe) 20190908ad6eSAnanth N Mavinakayanahalli *is_swbp = is_trap_at_addr(mm, bp_vaddr); 2020d790d346SOleg Nesterov } else { 2021d790d346SOleg Nesterov *is_swbp = -EFAULT; 20223a9ea052SOleg Nesterov } 2023499a4f3eSOleg Nesterov 2024499a4f3eSOleg Nesterov if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags)) 2025499a4f3eSOleg Nesterov mmf_recalc_uprobes(mm); 20260326f5a9SSrikar Dronamraju up_read(&mm->mmap_sem); 20270326f5a9SSrikar Dronamraju 20283a9ea052SOleg Nesterov return uprobe; 20293a9ea052SOleg Nesterov } 20303a9ea052SOleg Nesterov 2031da1816b1SOleg Nesterov static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs) 2032da1816b1SOleg Nesterov { 2033da1816b1SOleg Nesterov struct uprobe_consumer *uc; 2034da1816b1SOleg Nesterov int remove = UPROBE_HANDLER_REMOVE; 20350dfd0eb8SAnton Arapov bool need_prep = false; /* prepare return uprobe, when needed */ 2036da1816b1SOleg Nesterov 2037da1816b1SOleg Nesterov down_read(&uprobe->register_rwsem); 2038da1816b1SOleg Nesterov for (uc = uprobe->consumers; uc; uc = uc->next) { 2039ea024870SAnton Arapov int rc = 0; 2040da1816b1SOleg Nesterov 2041ea024870SAnton Arapov if (uc->handler) { 2042ea024870SAnton Arapov rc = uc->handler(uc, regs); 2043da1816b1SOleg Nesterov WARN(rc & ~UPROBE_HANDLER_MASK, 2044da1816b1SOleg Nesterov "bad rc=0x%x from %pf()\n", rc, uc->handler); 2045ea024870SAnton Arapov } 20460dfd0eb8SAnton Arapov 20470dfd0eb8SAnton Arapov if (uc->ret_handler) 20480dfd0eb8SAnton Arapov need_prep = true; 20490dfd0eb8SAnton Arapov 2050da1816b1SOleg Nesterov remove &= rc; 2051da1816b1SOleg Nesterov } 2052da1816b1SOleg Nesterov 20530dfd0eb8SAnton Arapov if (need_prep && !remove) 20540dfd0eb8SAnton Arapov prepare_uretprobe(uprobe, regs); /* put bp at return */ 20550dfd0eb8SAnton Arapov 2056da1816b1SOleg Nesterov if (remove && uprobe->consumers) { 2057da1816b1SOleg Nesterov WARN_ON(!uprobe_is_active(uprobe)); 2058da1816b1SOleg Nesterov unapply_uprobe(uprobe, current->mm); 2059da1816b1SOleg Nesterov } 2060da1816b1SOleg Nesterov up_read(&uprobe->register_rwsem); 2061da1816b1SOleg Nesterov } 2062da1816b1SOleg Nesterov 2063fec8898dSAnton Arapov static void 2064fec8898dSAnton Arapov handle_uretprobe_chain(struct return_instance *ri, struct pt_regs *regs) 2065fec8898dSAnton Arapov { 2066fec8898dSAnton Arapov struct uprobe *uprobe = ri->uprobe; 2067fec8898dSAnton Arapov struct uprobe_consumer *uc; 2068fec8898dSAnton Arapov 2069fec8898dSAnton Arapov down_read(&uprobe->register_rwsem); 2070fec8898dSAnton Arapov for (uc = uprobe->consumers; uc; uc = uc->next) { 2071fec8898dSAnton Arapov if (uc->ret_handler) 2072fec8898dSAnton Arapov uc->ret_handler(uc, ri->func, regs); 2073fec8898dSAnton Arapov } 2074fec8898dSAnton Arapov up_read(&uprobe->register_rwsem); 2075fec8898dSAnton Arapov } 2076fec8898dSAnton Arapov 2077a83cfeb9SOleg Nesterov static struct return_instance *find_next_ret_chain(struct return_instance *ri) 2078a83cfeb9SOleg Nesterov { 2079a83cfeb9SOleg Nesterov bool chained; 2080a83cfeb9SOleg Nesterov 2081a83cfeb9SOleg Nesterov do { 2082a83cfeb9SOleg Nesterov chained = ri->chained; 2083a83cfeb9SOleg Nesterov ri = ri->next; /* can't be NULL if chained */ 2084a83cfeb9SOleg Nesterov } while (chained); 2085a83cfeb9SOleg Nesterov 2086a83cfeb9SOleg Nesterov return ri; 2087a83cfeb9SOleg Nesterov } 2088a83cfeb9SOleg Nesterov 20890b5256c7SOleg Nesterov static void handle_trampoline(struct pt_regs *regs) 2090fec8898dSAnton Arapov { 2091fec8898dSAnton Arapov struct uprobe_task *utask; 2092a83cfeb9SOleg Nesterov struct return_instance *ri, *next; 20935eeb50deSOleg Nesterov bool valid; 2094fec8898dSAnton Arapov 2095fec8898dSAnton Arapov utask = current->utask; 2096fec8898dSAnton Arapov if (!utask) 20970b5256c7SOleg Nesterov goto sigill; 2098fec8898dSAnton Arapov 2099fec8898dSAnton Arapov ri = utask->return_instances; 2100fec8898dSAnton Arapov if (!ri) 21010b5256c7SOleg Nesterov goto sigill; 2102fec8898dSAnton Arapov 21035eeb50deSOleg Nesterov do { 2104fec8898dSAnton Arapov /* 21055eeb50deSOleg Nesterov * We should throw out the frames invalidated by longjmp(). 21065eeb50deSOleg Nesterov * If this chain is valid, then the next one should be alive 21075eeb50deSOleg Nesterov * or NULL; the latter case means that nobody but ri->func 21085eeb50deSOleg Nesterov * could hit this trampoline on return. TODO: sigaltstack(). 2109fec8898dSAnton Arapov */ 21105eeb50deSOleg Nesterov next = find_next_ret_chain(ri); 211186dcb702SOleg Nesterov valid = !next || arch_uretprobe_is_alive(next, RP_CHECK_RET, regs); 21125eeb50deSOleg Nesterov 2113fec8898dSAnton Arapov instruction_pointer_set(regs, ri->orig_ret_vaddr); 2114a83cfeb9SOleg Nesterov do { 21155eeb50deSOleg Nesterov if (valid) 2116fec8898dSAnton Arapov handle_uretprobe_chain(ri, regs); 21172bb5e840SOleg Nesterov ri = free_ret_instance(ri); 2118878b5a6eSOleg Nesterov utask->depth--; 2119a83cfeb9SOleg Nesterov } while (ri != next); 21205eeb50deSOleg Nesterov } while (!valid); 2121fec8898dSAnton Arapov 2122fec8898dSAnton Arapov utask->return_instances = ri; 21230b5256c7SOleg Nesterov return; 2124fec8898dSAnton Arapov 21250b5256c7SOleg Nesterov sigill: 21260b5256c7SOleg Nesterov uprobe_warn(current, "handle uretprobe, sending SIGILL."); 212755a3235fSEric W. Biederman force_sig(SIGILL, current); 21280b5256c7SOleg Nesterov 2129fec8898dSAnton Arapov } 2130fec8898dSAnton Arapov 21316fe50a28SDavid A. Long bool __weak arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs) 21326fe50a28SDavid A. Long { 21336fe50a28SDavid A. Long return false; 21346fe50a28SDavid A. Long } 21356fe50a28SDavid A. Long 213686dcb702SOleg Nesterov bool __weak arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx, 213786dcb702SOleg Nesterov struct pt_regs *regs) 213897da8976SOleg Nesterov { 213997da8976SOleg Nesterov return true; 214097da8976SOleg Nesterov } 214197da8976SOleg Nesterov 21423a9ea052SOleg Nesterov /* 21433a9ea052SOleg Nesterov * Run handler and ask thread to singlestep. 21443a9ea052SOleg Nesterov * Ensure all non-fatal signals cannot interrupt thread while it singlesteps. 21453a9ea052SOleg Nesterov */ 21463a9ea052SOleg Nesterov static void handle_swbp(struct pt_regs *regs) 21473a9ea052SOleg Nesterov { 21483a9ea052SOleg Nesterov struct uprobe *uprobe; 21493a9ea052SOleg Nesterov unsigned long bp_vaddr; 215056bb4cf6SOleg Nesterov int uninitialized_var(is_swbp); 21513a9ea052SOleg Nesterov 21523a9ea052SOleg Nesterov bp_vaddr = uprobe_get_swbp_addr(regs); 21530b5256c7SOleg Nesterov if (bp_vaddr == get_trampoline_vaddr()) 21540b5256c7SOleg Nesterov return handle_trampoline(regs); 2155fec8898dSAnton Arapov 2156fec8898dSAnton Arapov uprobe = find_active_uprobe(bp_vaddr, &is_swbp); 21570326f5a9SSrikar Dronamraju if (!uprobe) { 215856bb4cf6SOleg Nesterov if (is_swbp > 0) { 21590326f5a9SSrikar Dronamraju /* No matching uprobe; signal SIGTRAP. */ 21600326f5a9SSrikar Dronamraju send_sig(SIGTRAP, current, 0); 216156bb4cf6SOleg Nesterov } else { 216256bb4cf6SOleg Nesterov /* 216356bb4cf6SOleg Nesterov * Either we raced with uprobe_unregister() or we can't 216456bb4cf6SOleg Nesterov * access this memory. The latter is only possible if 216556bb4cf6SOleg Nesterov * another thread plays with our ->mm. In both cases 216656bb4cf6SOleg Nesterov * we can simply restart. If this vma was unmapped we 216756bb4cf6SOleg Nesterov * can pretend this insn was not executed yet and get 216856bb4cf6SOleg Nesterov * the (correct) SIGSEGV after restart. 216956bb4cf6SOleg Nesterov */ 217056bb4cf6SOleg Nesterov instruction_pointer_set(regs, bp_vaddr); 217156bb4cf6SOleg Nesterov } 21720326f5a9SSrikar Dronamraju return; 21730326f5a9SSrikar Dronamraju } 217474e59dfcSOleg Nesterov 217574e59dfcSOleg Nesterov /* change it in advance for ->handler() and restart */ 217674e59dfcSOleg Nesterov instruction_pointer_set(regs, bp_vaddr); 217774e59dfcSOleg Nesterov 2178142b18ddSOleg Nesterov /* 2179142b18ddSOleg Nesterov * TODO: move copy_insn/etc into _register and remove this hack. 2180142b18ddSOleg Nesterov * After we hit the bp, _unregister + _register can install the 2181142b18ddSOleg Nesterov * new and not-yet-analyzed uprobe at the same address, restart. 2182142b18ddSOleg Nesterov */ 2183142b18ddSOleg Nesterov smp_rmb(); /* pairs with wmb() in install_breakpoint() */ 218471434f2fSOleg Nesterov if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags))) 218574e59dfcSOleg Nesterov goto out; 21860326f5a9SSrikar Dronamraju 218772fd293aSOleg Nesterov /* Tracing handlers use ->utask to communicate with fetch methods */ 218872fd293aSOleg Nesterov if (!get_utask()) 218972fd293aSOleg Nesterov goto out; 219072fd293aSOleg Nesterov 21916fe50a28SDavid A. Long if (arch_uprobe_ignore(&uprobe->arch, regs)) 21926fe50a28SDavid A. Long goto out; 21936fe50a28SDavid A. Long 21940326f5a9SSrikar Dronamraju handler_chain(uprobe, regs); 21956fe50a28SDavid A. Long 21968a6b1732SOleg Nesterov if (arch_uprobe_skip_sstep(&uprobe->arch, regs)) 21970578a970SOleg Nesterov goto out; 21980326f5a9SSrikar Dronamraju 2199608e7427SOleg Nesterov if (!pre_ssout(uprobe, regs, bp_vaddr)) 22000326f5a9SSrikar Dronamraju return; 22010326f5a9SSrikar Dronamraju 22028a6b1732SOleg Nesterov /* arch_uprobe_skip_sstep() succeeded, or restart if can't singlestep */ 22030578a970SOleg Nesterov out: 22040326f5a9SSrikar Dronamraju put_uprobe(uprobe); 22050326f5a9SSrikar Dronamraju } 22060326f5a9SSrikar Dronamraju 22070326f5a9SSrikar Dronamraju /* 22080326f5a9SSrikar Dronamraju * Perform required fix-ups and disable singlestep. 22090326f5a9SSrikar Dronamraju * Allow pending signals to take effect. 22100326f5a9SSrikar Dronamraju */ 22110326f5a9SSrikar Dronamraju static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs) 22120326f5a9SSrikar Dronamraju { 22130326f5a9SSrikar Dronamraju struct uprobe *uprobe; 2214014940baSOleg Nesterov int err = 0; 22150326f5a9SSrikar Dronamraju 22160326f5a9SSrikar Dronamraju uprobe = utask->active_uprobe; 22170326f5a9SSrikar Dronamraju if (utask->state == UTASK_SSTEP_ACK) 2218014940baSOleg Nesterov err = arch_uprobe_post_xol(&uprobe->arch, regs); 22190326f5a9SSrikar Dronamraju else if (utask->state == UTASK_SSTEP_TRAPPED) 22200326f5a9SSrikar Dronamraju arch_uprobe_abort_xol(&uprobe->arch, regs); 22210326f5a9SSrikar Dronamraju else 22220326f5a9SSrikar Dronamraju WARN_ON_ONCE(1); 22230326f5a9SSrikar Dronamraju 22240326f5a9SSrikar Dronamraju put_uprobe(uprobe); 22250326f5a9SSrikar Dronamraju utask->active_uprobe = NULL; 22260326f5a9SSrikar Dronamraju utask->state = UTASK_RUNNING; 2227d4b3b638SSrikar Dronamraju xol_free_insn_slot(current); 22280326f5a9SSrikar Dronamraju 22290326f5a9SSrikar Dronamraju spin_lock_irq(¤t->sighand->siglock); 22300326f5a9SSrikar Dronamraju recalc_sigpending(); /* see uprobe_deny_signal() */ 22310326f5a9SSrikar Dronamraju spin_unlock_irq(¤t->sighand->siglock); 2232014940baSOleg Nesterov 2233014940baSOleg Nesterov if (unlikely(err)) { 2234014940baSOleg Nesterov uprobe_warn(current, "execute the probed insn, sending SIGILL."); 223555a3235fSEric W. Biederman force_sig(SIGILL, current); 2236014940baSOleg Nesterov } 22370326f5a9SSrikar Dronamraju } 22380326f5a9SSrikar Dronamraju 22390326f5a9SSrikar Dronamraju /* 22401b08e907SOleg Nesterov * On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag and 22411b08e907SOleg Nesterov * allows the thread to return from interrupt. After that handle_swbp() 22421b08e907SOleg Nesterov * sets utask->active_uprobe. 22430326f5a9SSrikar Dronamraju * 22441b08e907SOleg Nesterov * On singlestep exception, singlestep notifier sets the TIF_UPROBE flag 22451b08e907SOleg Nesterov * and allows the thread to return from interrupt. 22460326f5a9SSrikar Dronamraju * 22470326f5a9SSrikar Dronamraju * While returning to userspace, thread notices the TIF_UPROBE flag and calls 22480326f5a9SSrikar Dronamraju * uprobe_notify_resume(). 22490326f5a9SSrikar Dronamraju */ 22500326f5a9SSrikar Dronamraju void uprobe_notify_resume(struct pt_regs *regs) 22510326f5a9SSrikar Dronamraju { 22520326f5a9SSrikar Dronamraju struct uprobe_task *utask; 22530326f5a9SSrikar Dronamraju 2254db023ea5SOleg Nesterov clear_thread_flag(TIF_UPROBE); 2255db023ea5SOleg Nesterov 22560326f5a9SSrikar Dronamraju utask = current->utask; 22571b08e907SOleg Nesterov if (utask && utask->active_uprobe) 22580326f5a9SSrikar Dronamraju handle_singlestep(utask, regs); 22591b08e907SOleg Nesterov else 22601b08e907SOleg Nesterov handle_swbp(regs); 22610326f5a9SSrikar Dronamraju } 22620326f5a9SSrikar Dronamraju 22630326f5a9SSrikar Dronamraju /* 22640326f5a9SSrikar Dronamraju * uprobe_pre_sstep_notifier gets called from interrupt context as part of 22650326f5a9SSrikar Dronamraju * notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit. 22660326f5a9SSrikar Dronamraju */ 22670326f5a9SSrikar Dronamraju int uprobe_pre_sstep_notifier(struct pt_regs *regs) 22680326f5a9SSrikar Dronamraju { 22690dfd0eb8SAnton Arapov if (!current->mm) 22700dfd0eb8SAnton Arapov return 0; 22710dfd0eb8SAnton Arapov 22720dfd0eb8SAnton Arapov if (!test_bit(MMF_HAS_UPROBES, ¤t->mm->flags) && 22730dfd0eb8SAnton Arapov (!current->utask || !current->utask->return_instances)) 22740326f5a9SSrikar Dronamraju return 0; 22750326f5a9SSrikar Dronamraju 22760326f5a9SSrikar Dronamraju set_thread_flag(TIF_UPROBE); 22770326f5a9SSrikar Dronamraju return 1; 22780326f5a9SSrikar Dronamraju } 22790326f5a9SSrikar Dronamraju 22800326f5a9SSrikar Dronamraju /* 22810326f5a9SSrikar Dronamraju * uprobe_post_sstep_notifier gets called in interrupt context as part of notifier 22820326f5a9SSrikar Dronamraju * mechanism. Set TIF_UPROBE flag and indicate completion of singlestep. 22830326f5a9SSrikar Dronamraju */ 22840326f5a9SSrikar Dronamraju int uprobe_post_sstep_notifier(struct pt_regs *regs) 22850326f5a9SSrikar Dronamraju { 22860326f5a9SSrikar Dronamraju struct uprobe_task *utask = current->utask; 22870326f5a9SSrikar Dronamraju 22880326f5a9SSrikar Dronamraju if (!current->mm || !utask || !utask->active_uprobe) 22890326f5a9SSrikar Dronamraju /* task is currently not uprobed */ 22900326f5a9SSrikar Dronamraju return 0; 22910326f5a9SSrikar Dronamraju 22920326f5a9SSrikar Dronamraju utask->state = UTASK_SSTEP_ACK; 22930326f5a9SSrikar Dronamraju set_thread_flag(TIF_UPROBE); 22940326f5a9SSrikar Dronamraju return 1; 22950326f5a9SSrikar Dronamraju } 22960326f5a9SSrikar Dronamraju 22970326f5a9SSrikar Dronamraju static struct notifier_block uprobe_exception_nb = { 22980326f5a9SSrikar Dronamraju .notifier_call = arch_uprobe_exception_notify, 22990326f5a9SSrikar Dronamraju .priority = INT_MAX-1, /* notified after kprobes, kgdb */ 23000326f5a9SSrikar Dronamraju }; 23010326f5a9SSrikar Dronamraju 2302a5f4374aSIngo Molnar static int __init init_uprobes(void) 2303a5f4374aSIngo Molnar { 2304a5f4374aSIngo Molnar int i; 2305a5f4374aSIngo Molnar 230666d06dffSOleg Nesterov for (i = 0; i < UPROBES_HASH_SZ; i++) 2307a5f4374aSIngo Molnar mutex_init(&uprobes_mmap_mutex[i]); 23080326f5a9SSrikar Dronamraju 230932cdba1eSOleg Nesterov if (percpu_init_rwsem(&dup_mmap_sem)) 231032cdba1eSOleg Nesterov return -ENOMEM; 231132cdba1eSOleg Nesterov 23120326f5a9SSrikar Dronamraju return register_die_notifier(&uprobe_exception_nb); 2313a5f4374aSIngo Molnar } 2314736e89d9SOleg Nesterov __initcall(init_uprobes); 2315